From 3914e4e4087ebeb6b40fd134da95c274b880b1c1 Mon Sep 17 00:00:00 2001
From: Romain Courteaud <romain@nexedi.com>
Date: Fri, 29 May 2015 16:46:54 +0200
Subject: [PATCH] ReplicateStorage: allow to reduce number of checks during
synchronization
---
src/jio.storage/replicatestorage.js | 69 +++-
test/jio.storage/replicatestorage.tests.js | 408 ++++++++++++++++++++-
2 files changed, 465 insertions(+), 12 deletions(-)
diff --git a/src/jio.storage/replicatestorage.js b/src/jio.storage/replicatestorage.js
index 5f828c4..e04b71e 100644
--- a/src/jio.storage/replicatestorage.js
+++ b/src/jio.storage/replicatestorage.js
@@ -53,6 +53,30 @@
});
this._use_remote_post = spec.use_remote_post || false;
+ this._check_local_modification = spec.check_local_modification;
+ if (this._check_local_modification === undefined) {
+ this._check_local_modification = true;
+ }
+ this._check_local_creation = spec.check_local_creation;
+ if (this._check_local_creation === undefined) {
+ this._check_local_creation = true;
+ }
+ this._check_local_deletion = spec.check_local_deletion;
+ if (this._check_local_deletion === undefined) {
+ this._check_local_deletion = true;
+ }
+ this._check_remote_modification = spec.check_remote_modification;
+ if (this._check_remote_modification === undefined) {
+ this._check_remote_modification = true;
+ }
+ this._check_remote_creation = spec.check_remote_creation;
+ if (this._check_remote_creation === undefined) {
+ this._check_remote_creation = true;
+ }
+ this._check_remote_deletion = spec.check_remote_deletion;
+ if (this._check_remote_deletion === undefined) {
+ this._check_remote_deletion = true;
+ }
}
ReplicateStorage.prototype.remove = function (id) {
@@ -304,19 +328,25 @@
signature_dict[result_list[1].data.rows[i].id] = i;
}
}
- for (key in local_dict) {
- if (local_dict.hasOwnProperty(key)) {
- if (!signature_dict.hasOwnProperty(key)) {
- checkLocalCreation(queue, source, destination, key, options);
+ if (options.check_creation === true) {
+ for (key in local_dict) {
+ if (local_dict.hasOwnProperty(key)) {
+ if (!signature_dict.hasOwnProperty(key)) {
+ checkLocalCreation(queue, source, destination, key, options);
+ }
}
}
}
for (key in signature_dict) {
if (signature_dict.hasOwnProperty(key)) {
if (local_dict.hasOwnProperty(key)) {
- checkSignatureDifference(queue, source, destination, key);
+ if (options.check_modification === true) {
+ checkSignatureDifference(queue, source, destination, key);
+ }
} else {
- checkLocalDeletion(queue, destination, key, source);
+ if (options.check_deletion === true) {
+ checkLocalDeletion(queue, destination, key, source);
+ }
}
}
}
@@ -360,13 +390,30 @@
})
.push(function () {
- return pushStorage(context._local_sub_storage,
- context._remote_sub_storage,
- {use_post: context._use_remote_post});
+ if (context._check_local_modification ||
+ context._check_local_creation ||
+ context._check_local_deletion) {
+ return pushStorage(context._local_sub_storage,
+ context._remote_sub_storage,
+ {
+ use_post: context._use_remote_post,
+ check_modification: context._check_local_modification,
+ check_creation: context._check_local_creation,
+ check_deletion: context._check_local_deletion
+ });
+ }
})
.push(function () {
- return pushStorage(context._remote_sub_storage,
- context._local_sub_storage, {});
+ if (context._check_remote_modification ||
+ context._check_remote_creation ||
+ context._check_remote_deletion) {
+ return pushStorage(context._remote_sub_storage,
+ context._local_sub_storage, {
+ check_modification: context._check_remote_modification,
+ check_creation: context._check_remote_creation,
+ check_deletion: context._check_remote_deletion
+ });
+ }
});
};
diff --git a/test/jio.storage/replicatestorage.tests.js b/test/jio.storage/replicatestorage.tests.js
index bc7bf20..07342aa 100644
--- a/test/jio.storage/replicatestorage.tests.js
+++ b/test/jio.storage/replicatestorage.tests.js
@@ -46,6 +46,12 @@
deepEqual(jio.__storage._query_options, {});
equal(jio.__storage._use_remote_post, false);
+ equal(jio.__storage._check_local_creation, true);
+ equal(jio.__storage._check_local_deletion, true);
+ equal(jio.__storage._check_local_modification, true);
+ equal(jio.__storage._check_remote_creation, true);
+ equal(jio.__storage._check_remote_deletion, true);
+ equal(jio.__storage._check_remote_modification, true);
equal(jio.__storage._signature_hash,
"_replicate_7209dfbcaff00f6637f939fdd71fa896793ed385");
@@ -73,7 +79,13 @@
type: "replicatestorage500"
},
query: {query: 'portal_type: "Foo"', limit: [0, 1234567890]},
- use_remote_post: true
+ use_remote_post: true,
+ check_local_creation: false,
+ check_local_deletion: false,
+ check_local_modification: false,
+ check_remote_creation: false,
+ check_remote_deletion: false,
+ check_remote_modification: false
});
deepEqual(
@@ -81,6 +93,12 @@
{query: 'portal_type: "Foo"', limit: [0, 1234567890]}
);
equal(jio.__storage._use_remote_post, true);
+ equal(jio.__storage._check_local_creation, false);
+ equal(jio.__storage._check_local_deletion, false);
+ equal(jio.__storage._check_local_modification, false);
+ equal(jio.__storage._check_remote_creation, false);
+ equal(jio.__storage._check_remote_deletion, false);
+ equal(jio.__storage._check_remote_modification, false);
equal(jio.__storage._signature_hash,
"_replicate_623653d45a4e770a2c9f6b71e3144d18ee1b5bec");
@@ -434,6 +452,63 @@
});
});
+ test("local document creation not checked", function () {
+ stop();
+ expect(6);
+
+ this.jio = jIO.createJIO({
+ type: "replicate",
+ check_local_creation: false,
+ local_sub_storage: {
+ type: "uuid",
+ sub_storage: {
+ type: "memory"
+ }
+ },
+ remote_sub_storage: {
+ type: "uuid",
+ sub_storage: {
+ type: "memory"
+ }
+ }
+ });
+
+ var id,
+ context = this;
+
+ context.jio.post({"title": "foo"})
+ .then(function (result) {
+ id = result;
+ return context.jio.repair();
+ })
+ .then(function () {
+ return context.jio.get(id);
+ })
+ .then(function (result) {
+ deepEqual(result, {
+ title: "foo"
+ });
+ })
+ .then(function () {
+ return context.jio.__storage._remote_sub_storage.get(id);
+ })
+ .fail(function (error) {
+ ok(error instanceof jIO.util.jIOError);
+ equal(error.message, "Cannot find document: " + id);
+ equal(error.status_code, 404);
+ })
+ .then(function () {
+ return context.jio.__storage._signature_sub_storage.get(id);
+ })
+ .fail(function (error) {
+ ok(error instanceof jIO.util.jIOError);
+ equal(error.status_code, 404);
+ })
+ .always(function () {
+ start();
+ });
+ });
+
test("local document creation and use remote post", function () {
stop();
expect(12);
@@ -677,6 +752,66 @@
});
});
+ test("remote document creation not checked", function () {
+ stop();
+ expect(6);
+
+ var id,
+ context = this;
+
+ this.jio = jIO.createJIO({
+ type: "replicate",
+ check_remote_creation: false,
+ local_sub_storage: {
+ type: "uuid",
+ sub_storage: {
+ type: "memory"
+ }
+ },
+ remote_sub_storage: {
+ type: "uuid",
+ sub_storage: {
+ type: "memory"
+ }
+ }
+ });
+
+ context.jio.__storage._remote_sub_storage.post({"title": "bar"})
+ .then(function (result) {
+ id = result;
+ return context.jio.repair();
+ })
+ .then(function () {
+ return context.jio.__storage._remote_sub_storage.get(id);
+ })
+ .then(function (result) {
+ deepEqual(result, {
+ title: "bar"
+ });
+ })
+ .then(function () {
+ return context.jio.get(id);
+ })
+ .fail(function (error) {
+ ok(error instanceof jIO.util.jIOError);
+ equal(error.message, "Cannot find document: " + id);
+ equal(error.status_code, 404);
+ })
+ .then(function () {
+ return context.jio.__storage._signature_sub_storage.get(id);
+ })
+ .fail(function (error) {
+ ok(error instanceof jIO.util.jIOError);
+ equal(error.status_code, 404);
+ })
+ .fail(function (error) {
+ ok(false, error);
+ })
+ .always(function () {
+ start();
+ });
+ });
+
test("local and remote document creations", function () {
stop();
expect(5);
@@ -823,6 +958,73 @@
});
});
+ test("local document modification not checked", function () {
+ stop();
+ expect(3);
+
+ var id,
+ context = this;
+
+ this.jio = jIO.createJIO({
+ type: "replicate",
+ check_local_modification: false,
+ local_sub_storage: {
+ type: "uuid",
+ sub_storage: {
+ type: "memory"
+ }
+ },
+ remote_sub_storage: {
+ type: "uuid",
+ sub_storage: {
+ type: "memory"
+ }
+ }
+ });
+
+ context.jio.post({"title": "foo"})
+ .then(function (result) {
+ id = result;
+ return context.jio.repair();
+ })
+ .then(function () {
+ return context.jio.put(id, {"title": "foo2"});
+ })
+ .then(function () {
+ return context.jio.repair();
+ })
+ .then(function () {
+ return context.jio.get(id);
+ })
+ .then(function (result) {
+ deepEqual(result, {
+ title: "foo2"
+ });
+ })
+ .then(function () {
+ return context.jio.__storage._remote_sub_storage.get(id);
+ })
+ .then(function (result) {
+ deepEqual(result, {
+ title: "foo"
+ });
+ })
+ .then(function () {
+ return context.jio.__storage._signature_sub_storage.get(id);
+ })
+ .then(function (result) {
+ deepEqual(result, {
+ hash: "5ea9013447539ad65de308cbd75b5826a2ae30e5"
+ });
+ })
+ .fail(function (error) {
+ ok(false, error);
+ })
+ .always(function () {
+ start();
+ });
+ });
+
test("remote document modification", function () {
stop();
expect(2);
@@ -868,6 +1070,76 @@
});
});
+ test("remote document modification not checked", function () {
+ stop();
+ expect(3);
+
+ var id,
+ context = this;
+
+ this.jio = jIO.createJIO({
+ type: "replicate",
+ check_remote_modification: false,
+ local_sub_storage: {
+ type: "uuid",
+ sub_storage: {
+ type: "memory"
+ }
+ },
+ remote_sub_storage: {
+ type: "uuid",
+ sub_storage: {
+ type: "memory"
+ }
+ }
+ });
+
+ context.jio.post({"title": "foo"})
+ .then(function (result) {
+ id = result;
+ return context.jio.repair();
+ })
+ .then(function () {
+ return context.jio.__storage._remote_sub_storage.put(
+ id,
+ {"title": "foo3"}
+ );
+ })
+ .then(function () {
+ return context.jio.repair();
+ })
+ .then(function () {
+ return context.jio.__storage._remote_sub_storage.get(id);
+ })
+ .then(function (result) {
+ deepEqual(result, {
+ title: "foo3"
+ });
+ })
+ .then(function () {
+ return context.jio.get(id);
+ })
+ .then(function (result) {
+ deepEqual(result, {
+ title: "foo"
+ });
+ })
+ .then(function () {
+ return context.jio.__storage._signature_sub_storage.get(id);
+ })
+ .then(function (result) {
+ deepEqual(result, {
+ hash: "5ea9013447539ad65de308cbd75b5826a2ae30e5"
+ });
+ })
+ .fail(function (error) {
+ ok(false, error);
+ })
+ .always(function () {
+ start();
+ });
+ });
+
test("local and remote document modifications", function () {
stop();
expect(4);
@@ -992,6 +1264,73 @@
});
});
+ test("local document deletion not checked", function () {
+ stop();
+ expect(6);
+
+ var id,
+ context = this;
+
+ this.jio = jIO.createJIO({
+ type: "replicate",
+ check_local_deletion: false,
+ local_sub_storage: {
+ type: "uuid",
+ sub_storage: {
+ type: "memory"
+ }
+ },
+ remote_sub_storage: {
+ type: "uuid",
+ sub_storage: {
+ type: "memory"
+ }
+ }
+ });
+
+ context.jio.post({"title": "foo"})
+ .then(function (result) {
+ id = result;
+ return context.jio.repair();
+ })
+ .then(function () {
+ return context.jio.remove(id);
+ })
+ .then(function () {
+ return context.jio.repair();
+ })
+ .then(function () {
+ ok(true, "Removal correctly synced");
+ })
+ .then(function () {
+ return context.jio.get(id);
+ })
+ .fail(function (error) {
+ ok(error instanceof jIO.util.jIOError);
+ equal(error.message, "Cannot find document: " + id);
+ equal(error.status_code, 404);
+ })
+ .then(function () {
+ return context.jio.__storage._remote_sub_storage.get(id);
+ })
+ .then(function (result) {
+ deepEqual(result, {
+ title: "foo"
+ });
+ })
+ .then(function () {
+ return context.jio.__storage._signature_sub_storage.get(id);
+ })
+ .then(function (result) {
+ deepEqual(result, {
+ hash: "5ea9013447539ad65de308cbd75b5826a2ae30e5"
+ });
+ })
+ .always(function () {
+ start();
+ });
+ });
+
test("remote document deletion", function () {
stop();
expect(6);
@@ -1037,6 +1376,73 @@
});
});
+ test("remote document deletion not checked", function () {
+ stop();
+ expect(6);
+
+ var id,
+ context = this;
+
+ this.jio = jIO.createJIO({
+ type: "replicate",
+ check_remote_deletion: false,
+ local_sub_storage: {
+ type: "uuid",
+ sub_storage: {
+ type: "memory"
+ }
+ },
+ remote_sub_storage: {
+ type: "uuid",
+ sub_storage: {
+ type: "memory"
+ }
+ }
+ });
+
+ context.jio.post({"title": "foo"})
+ .then(function (result) {
+ id = result;
+ return context.jio.repair();
+ })
+ .then(function () {
+ return context.jio.__storage._remote_sub_storage.remove(id);
+ })
+ .then(function () {
+ return context.jio.repair();
+ })
+ .then(function () {
+ ok(true, "Removal correctly synced");
+ })
+ .then(function () {
+ return context.jio.__storage._remote_sub_storage.get(id);
+ })
+ .fail(function (error) {
+ ok(error instanceof jIO.util.jIOError);
+ equal(error.message, "Cannot find document: " + id);
+ equal(error.status_code, 404);
+ })
+ .then(function () {
+ return context.jio.get(id);
+ })
+ .then(function (result) {
+ deepEqual(result, {
+ title: "foo"
+ });
+ })
+ .then(function () {
+ return context.jio.__storage._signature_sub_storage.get(id);
+ })
+ .then(function (result) {
+ deepEqual(result, {
+ hash: "5ea9013447539ad65de308cbd75b5826a2ae30e5"
+ });
+ })
+ .always(function () {
+ start();
+ });
+ });
+
test("local and remote document deletions", function () {
stop();
expect(8);
--
2.30.9