Skip to content

J
jio
Commit f864e6e4 authored by Romain Courteaud's avatar Romain Courteaud
Browse files
Options

ReplicateStorage: use a property as attachment hash 

Speed up attachment replication by using a document property to check if there was a change.
The attachment content is NEVER checked, as this configuration expect the document to be modified when an attachment is created, modified or deleted.
In case of conflict, follow the document conflict resolution result.
parent 8abcfa83
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 5501 additions and 71 deletions
src/jio.storage/replicatestorage.js
View file @ f864e6e4
This diff is collapsed.
test/jio.storage/replicatestorage_fastrepair.tests.js
View file @ f864e6e4
......@@ -141,6 +141,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: true,
hash: "foo dynetag"
});
})
......@@ -260,6 +261,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: true,
hash: "foo dynetag"
});
})
......@@ -386,6 +388,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: true,
hash: "foo dynetag"
});
})
......@@ -424,6 +427,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: false,
hash: "bar dynetag"
});
})
......@@ -493,6 +497,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: true,
hash: "foo dynetag"
});
})
......@@ -567,6 +572,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: true,
hash: "foo dynetag"
});
})
......@@ -642,6 +648,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: true,
hash: "foo dynetag"
});
})
......@@ -719,6 +726,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: true,
hash: "foo dynetag"
});
})
......@@ -795,6 +803,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: true,
hash: "foo dynetag"
});
})
......@@ -870,6 +879,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: false,
hash: "bar dynetag"
});
})
......@@ -945,6 +955,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: false,
hash: "bar dynetag"
});
})
......@@ -1021,6 +1032,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: false,
hash: "bar dynetag"
});
})
......@@ -1099,6 +1111,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: false,
hash: "bar dynetag"
});
})
......@@ -1296,6 +1309,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: true,
hash: "foo dynetag"
});
})
......@@ -1336,6 +1350,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: true,
hash: "foo dynetag"
});
})
......@@ -1379,6 +1394,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: true,
hash: "foo2 dynetag"
});
})
......@@ -1454,6 +1470,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: true,
hash: "foo2 dynetag"
});
})
......@@ -1536,6 +1553,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: true,
hash: "foo dynetag"
});
})
......@@ -1582,6 +1600,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: false,
hash: "foo3 dynetag"
});
})
......@@ -1667,6 +1686,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: true,
hash: "foo dynetag"
});
})
......@@ -1719,6 +1739,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: true,
hash: "foo dynetag"
});
})
......@@ -1789,6 +1810,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: true,
hash: "foo4 dynetag"
});
})
......@@ -1880,6 +1902,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: false,
hash: "foo5 dynetag"
});
})
......@@ -1971,6 +1994,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: true,
hash: "foo dynetag"
});
})
......@@ -2032,6 +2056,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: true,
hash: "foo99 dynetag"
});
})
......@@ -2161,6 +2186,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: true,
hash: "foo dynetag"
});
})
......@@ -2171,7 +2197,7 @@
test("local document deletion with attachment", function () {
stop();
expect(7);
expect(10);
var id,
context = this,
......@@ -2206,26 +2232,26 @@
.then(function () {
return context.jio.__storage._remote_sub_storage.get(id);
})
.then(function (result) {
deepEqual(result, {
title: "foo",
foo_etag: "foo etag"
});
})
.then(function () {
return context.jio.__storage._remote_sub_storage
.getAttachment(id, 'foo', {format: 'text'});
})
.then(function (result) {
equal(result, big_string);
.fail(function (error) {
ok(error instanceof jIO.util.jIOError);
equal(error.message, "Cannot find document: " + id);
equal(error.status_code, 404);
})
.then(function () {
return context.jio.__storage._signature_sub_storage.get(id);
})
.then(function (result) {
deepEqual(result, {
hash: "foo dynetag"
});
.fail(function (error) {
ok(error instanceof jIO.util.jIOError);
equal(
error.message.indexOf(
"Cannot find attachment: " +
"_replicate_ae15d2189153f083c0e4a845fd580b1d86f7a512 , " +
"jio_document/"
),
0,
error.message
);
equal(error.status_code, 404);
})
.always(function () {
start();
......@@ -2350,6 +2376,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: true,
hash: "foo dynetag"
});
})
......@@ -2360,7 +2387,7 @@
test("remote document deletion with attachment", function () {
stop();
expect(7);
expect(10);
var id,
context = this,
......@@ -2394,25 +2421,26 @@
.then(function () {
return context.jio.get(id);
})
.then(function (result) {
deepEqual(result, {
title: "foo",
foo_etag: "foo etag"
});
})
.then(function () {
return context.jio.getAttachment(id, 'foo', {format: 'text'});
})
.then(function (result) {
equal(result, big_string);
.fail(function (error) {
ok(error instanceof jIO.util.jIOError);
equal(error.message, "Cannot find document: " + id);
equal(error.status_code, 404);
})
.then(function () {
return context.jio.__storage._signature_sub_storage.get(id);
})
.then(function (result) {
deepEqual(result, {
hash: "foo dynetag"
});
.fail(function (error) {
ok(error instanceof jIO.util.jIOError);
equal(
error.message.indexOf(
"Cannot find attachment: " +
"_replicate_ae15d2189153f083c0e4a845fd580b1d86f7a512 , " +
"jio_document/"
),
0,
error.message
);
equal(error.status_code, 404);
})
.always(function () {
start();
......@@ -2522,6 +2550,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: false,
hash: "foo99 dynetag"
});
})
......@@ -2710,6 +2739,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: false,
hash: "foo99 dynetag"
});
})
......@@ -2800,6 +2830,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: true,
hash: "foo dynetag"
});
})
......@@ -2846,6 +2877,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: true,
hash: "foo99 dynetag"
});
})
......@@ -2972,6 +3004,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: true,
hash: "foo99 dynetag"
});
})
......@@ -3099,6 +3132,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: true,
hash: "foo99 dynetag"
});
})
......@@ -3185,6 +3219,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: true,
hash: "foo99 dynetag"
});
})
......@@ -3312,6 +3347,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: true,
hash: "foo99 dynetag"
});
})
......@@ -3582,6 +3618,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: true,
hash: "foo dynetag"
});
})
......
test/jio.storage/replicatestorage_repair.tests.js
View file @ f864e6e4
......@@ -71,6 +71,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: true,
hash: "5ea9013447539ad65de308cbd75b5826a2ae30e5"
});
})
......@@ -232,6 +233,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: true,
hash: "5ea9013447539ad65de308cbd75b5826a2ae30e5"
});
})
......@@ -343,6 +345,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: true,
hash: "5ea9013447539ad65de308cbd75b5826a2ae30e5"
});
})
......@@ -379,6 +382,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: false,
hash: "6799f3ea80e325b89f19589282a343c376c1f1af"
});
})
......@@ -522,6 +526,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: true,
hash: "5ea9013447539ad65de308cbd75b5826a2ae30e5"
});
})
......@@ -587,6 +592,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: true,
hash: "5ea9013447539ad65de308cbd75b5826a2ae30e5"
});
})
......@@ -653,6 +659,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: true,
hash: "5ea9013447539ad65de308cbd75b5826a2ae30e5"
});
})
......@@ -718,6 +725,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: true,
hash: "a0a1b37cee3709101b752c56e59b9d66cce09961"
});
})
......@@ -783,6 +791,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: false,
hash: "6799f3ea80e325b89f19589282a343c376c1f1af"
});
})
......@@ -848,6 +857,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: false,
hash: "6799f3ea80e325b89f19589282a343c376c1f1af"
});
})
......@@ -914,6 +924,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: false,
hash: "45efa2292d54cc4ce1f726ea197bc0b9721fc1dc"
});
})
......@@ -981,6 +992,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: false,
hash: "6799f3ea80e325b89f19589282a343c376c1f1af"
});
})
......@@ -1155,6 +1167,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: true,
hash: "5ea9013447539ad65de308cbd75b5826a2ae30e5"
});
})
......@@ -1194,6 +1207,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: true,
hash: "5ea9013447539ad65de308cbd75b5826a2ae30e5"
});
})
......@@ -1236,6 +1250,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: true,
hash: "9819187e39531fdc9bcfd40dbc6a7d3c78fe8dab"
});
})
......@@ -1296,6 +1311,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: true,
hash: "9819187e39531fdc9bcfd40dbc6a7d3c78fe8dab"
});
})
......@@ -1363,6 +1379,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: true,
hash: "5ea9013447539ad65de308cbd75b5826a2ae30e5"
});
})
......@@ -1408,6 +1425,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: false,
hash: "4b1dde0f80ac38514771a9d25b5278e38f560e0f"
});
})
......@@ -1478,6 +1496,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: true,
hash: "5ea9013447539ad65de308cbd75b5826a2ae30e5"
});
})
......@@ -1524,6 +1543,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: true,
hash: "5ea9013447539ad65de308cbd75b5826a2ae30e5"
});
})
......@@ -1575,6 +1595,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: true,
hash: "6f700e813022233a785692585484c21cb5a412fd"
});
})
......@@ -1645,6 +1666,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: false,
hash: "7bea6f87fd1dda14e340e5b14836cc8578fd615f"
});
})
......@@ -1715,6 +1737,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: true,
hash: "5ea9013447539ad65de308cbd75b5826a2ae30e5"
});
})
......@@ -1768,6 +1791,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: true,
hash: "8ed3a474128b6e0c0c7d3dd51b1a06ebfbf6722f"
});
})
......@@ -1883,6 +1907,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: true,
hash: "5ea9013447539ad65de308cbd75b5826a2ae30e5"
});
})
......@@ -1945,6 +1970,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: true,
hash: "5ea9013447539ad65de308cbd75b5826a2ae30e5"
});
})
......@@ -2057,6 +2083,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: true,
hash: "5ea9013447539ad65de308cbd75b5826a2ae30e5"
});
})
......@@ -2117,6 +2144,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: true,
hash: "5ea9013447539ad65de308cbd75b5826a2ae30e5"
});
})
......@@ -2221,6 +2249,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: false,
hash: "8ed3a474128b6e0c0c7d3dd51b1a06ebfbf6722f"
});
})
......@@ -2368,6 +2397,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: false,
hash: "8ed3a474128b6e0c0c7d3dd51b1a06ebfbf6722f"
});
})
......@@ -2438,6 +2468,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: true,
hash: "5ea9013447539ad65de308cbd75b5826a2ae30e5"
});
})
......@@ -2483,6 +2514,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: true,
hash: "8ed3a474128b6e0c0c7d3dd51b1a06ebfbf6722f"
});
})
......@@ -2592,6 +2624,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: true,
hash: "8ed3a474128b6e0c0c7d3dd51b1a06ebfbf6722f"
});
})
......@@ -2703,6 +2736,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: true,
hash: "8ed3a474128b6e0c0c7d3dd51b1a06ebfbf6722f"
});
})
......@@ -2774,6 +2808,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: true,
hash: "8ed3a474128b6e0c0c7d3dd51b1a06ebfbf6722f"
});
})
......@@ -2884,6 +2919,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: true,
hash: "8ed3a474128b6e0c0c7d3dd51b1a06ebfbf6722f"
});
})
......@@ -3112,6 +3148,7 @@
})
.then(function (result) {
deepEqual(result, {
from_local: true,
hash: "5ea9013447539ad65de308cbd75b5826a2ae30e5"
});
})
......
test/jio.storage/replicatestorage_repairattachment.tests.js
View file @ f864e6e4
......@@ -3742,7 +3742,10 @@
.get(id);
})
.then(function (result) {
deepEqual(result, {hash: "5ea9013447539ad65de308cbd75b5826a2ae30e5"});
deepEqual(result, {
from_local: true,
hash: "5ea9013447539ad65de308cbd75b5826a2ae30e5"
});
// remote document untouched
return context.jio.__storage._remote_sub_storage.get(id);
......@@ -3910,7 +3913,10 @@
.get(id);
})
.then(function (result) {
deepEqual(result, {hash: "5ea9013447539ad65de308cbd75b5826a2ae30e5"});
deepEqual(result, {
from_local: true,
hash: "5ea9013447539ad65de308cbd75b5826a2ae30e5"
});
// local document untouched
return context.jio.get(id);
......@@ -4080,7 +4086,10 @@
.get(id);
})
.then(function (result) {
deepEqual(result, {hash: "9819187e39531fdc9bcfd40dbc6a7d3c78fe8dab"});
deepEqual(result, {
from_local: false,
hash: "9819187e39531fdc9bcfd40dbc6a7d3c78fe8dab"
});
// remote document untouched
return context.jio.__storage._remote_sub_storage.get(id);
......@@ -4246,7 +4255,10 @@
.get(id);
})
.then(function (result) {
deepEqual(result, {hash: "9819187e39531fdc9bcfd40dbc6a7d3c78fe8dab"});
deepEqual(result, {
from_local: true,
hash: "9819187e39531fdc9bcfd40dbc6a7d3c78fe8dab"
});
// local document untouched
return context.jio.get(id);
......@@ -4417,7 +4429,10 @@
.get(id);
})
.then(function (result) {
deepEqual(result, {hash: "9819187e39531fdc9bcfd40dbc6a7d3c78fe8dab"});
deepEqual(result, {
from_local: false,
hash: "9819187e39531fdc9bcfd40dbc6a7d3c78fe8dab"
});
// remote document untouched
return context.jio.__storage._remote_sub_storage.get(id);
......@@ -4584,7 +4599,10 @@
.get(id);
})
.then(function (result) {
deepEqual(result, {hash: "9819187e39531fdc9bcfd40dbc6a7d3c78fe8dab"});
deepEqual(result, {
from_local: true,
hash: "9819187e39531fdc9bcfd40dbc6a7d3c78fe8dab"
});
// local document untouched
return context.jio.get(id);
......
test/tests.html
View file @ f864e6e4
......@@ -43,6 +43,7 @@
<script src="jio.storage/replicatestorage_repair.tests.js"></script>
<script src="jio.storage/replicatestorage_repairattachment.tests.js"></script>
<script src="jio.storage/replicatestorage_fastrepair.tests.js"></script>
<script src="jio.storage/replicatestorage_fastrepairattachment.tests.js"></script>
<script src="jio.storage/shastorage.tests.js"></script>
<!--script src="jio.storage/qiniustorage.tests.js"></script-->
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7