Skip to content

jio
jio
Commit b6b14699 authored by Romain Courteaud's avatar Romain Courteaud
Browse files
Options

[replicate] Handle document removed because of not query matching 

If a document stop matching the replicate query, it will be considered as deleted.
But its attachments will still be reachable and previously prevented the remote document deletion.
parent 2e7d0590
Hide whitespace changes
Inline Side-by-side
Showing with 564 additions and 35 deletions
src/jio.storage/replicatestorage.js
View file @ b6b14699
...@@ -1262,7 +1262,8 @@ ...@@ -1262,7 +1262,8 @@
// ie, replication should prevent losing user data // ie, replication should prevent losing user data
// Synchronize attachments before, to ensure // Synchronize attachments before, to ensure
// all of them will be deleted too // all of them will be deleted too
var result; var result,
previous_report_length;
if (context._signature_hash_key !== undefined) { if (context._signature_hash_key !== undefined) {
if (options.conflict) { if (options.conflict) {
report.log(id, options.from_local ? LOG_FORCE_DELETE_REMOTE : report.log(id, options.from_local ? LOG_FORCE_DELETE_REMOTE :
...@@ -1276,12 +1277,21 @@ ...@@ -1276,12 +1277,21 @@
return context._signature_sub_storage.remove(id); return context._signature_sub_storage.remove(id);
}); });
} else { } else {
previous_report_length = report._list.length;
result = repairDocumentAttachment(context, id, report) result = repairDocumentAttachment(context, id, report)
.push(function () { .push(function () {
return destination.allAttachments(id); var next_report_length = report._list.length,
}) has_error = false,
.push(function (attachment_dict) { i;
if (JSON.stringify(attachment_dict) === "{}") { // Check if there was an error during attachment replication
for (i = previous_report_length; i < next_report_length; i += 1) {
if ((report._list[i][1] === id) &&
(report._list[i][0] < 100)) {
has_error = true;
}
}
if (!has_error) {
// Attachment repication has been correctly resolved
if (options.conflict) { if (options.conflict) {
report.log(id, options.from_local ? LOG_FORCE_DELETE_REMOTE : report.log(id, options.from_local ? LOG_FORCE_DELETE_REMOTE :
LOG_FORCE_DELETE_LOCAL); LOG_FORCE_DELETE_LOCAL);
...@@ -1295,8 +1305,7 @@ ...@@ -1295,8 +1305,7 @@
}); });
} }
report.log(id, options.from_local ? LOG_UNEXPECTED_REMOTE_ATTACHMENT : report.log(id, options.from_local ? LOG_UNEXPECTED_REMOTE_ATTACHMENT :
LOG_UNEXPECTED_LOCAL_ATTACHMENT, LOG_UNEXPECTED_LOCAL_ATTACHMENT);
JSON.stringify(attachment_dict));
}, function (error) { }, function (error) {
if ((error instanceof jIO.util.jIOError) && if ((error instanceof jIO.util.jIOError) &&
(error.status_code === 404)) { (error.status_code === 404)) {
......
test/jio.storage/replicatestorage_repair.tests.js
View file @ b6b14699
...@@ -1992,7 +1992,7 @@ ...@@ -1992,7 +1992,7 @@
test("local document deletion with attachment", function () { test("local document deletion with attachment", function () {
stop(); stop();
expect(7); expect(12);
var id, var id,
context = this, context = this,
...@@ -2013,9 +2013,9 @@ ...@@ -2013,9 +2013,9 @@
.then(function () { .then(function () {
return context.jio.repair(); return context.jio.repair();
}) })
.fail(function (report) { .then(function (report) {
deepEqual(report._list, [ deepEqual(report._list, [
[report.LOG_UNEXPECTED_REMOTE_ATTACHMENT, id, '{"foo":{}}'] [report.LOG_DELETE_REMOTE, id]
]); ]);
}) })
.then(function () { .then(function () {
...@@ -2029,26 +2029,27 @@ ...@@ -2029,26 +2029,27 @@
.then(function () { .then(function () {
return context.jio.__storage._remote_sub_storage.get(id); return context.jio.__storage._remote_sub_storage.get(id);
}) })
.then(function (result) { .fail(function (error) {
deepEqual(result, { ok(error instanceof jIO.util.jIOError);
title: "foo" equal(error.message, "Cannot find document: " + id);
}); equal(error.status_code, 404);
}) })
.then(function () { .then(function () {
return context.jio.__storage._remote_sub_storage return context.jio.__storage._remote_sub_storage
.getAttachment(id, 'foo', {format: 'text'}); .getAttachment(id, 'foo', {format: 'text'});
}) })
.then(function (result) { .fail(function (error) {
equal(result, big_string); ok(error instanceof jIO.util.jIOError);
equal(error.message, "Cannot find attachment: " + id + " , foo");
equal(error.status_code, 404);
}) })
.then(function () { .then(function () {
return context.jio.__storage._signature_sub_storage.get(id); return context.jio.__storage._signature_sub_storage.get(id);
}) })
.then(function (result) { .fail(function (error) {
deepEqual(result, { ok(error instanceof jIO.util.jIOError);
from_local: true, // equal(error.message, "Cannot find document: " + id);
hash: "5ea9013447539ad65de308cbd75b5826a2ae30e5" equal(error.status_code, 404);
});
}) })
.always(function () { .always(function () {
start(); start();
...@@ -2177,7 +2178,7 @@ ...@@ -2177,7 +2178,7 @@
test("remote document deletion with attachment", function () { test("remote document deletion with attachment", function () {
stop(); stop();
expect(7); expect(12);
var id, var id,
context = this, context = this,
...@@ -2197,9 +2198,9 @@ ...@@ -2197,9 +2198,9 @@
.then(function () { .then(function () {
return context.jio.repair(); return context.jio.repair();
}) })
.fail(function (report) { .then(function (report) {
deepEqual(report._list, [ deepEqual(report._list, [
[report.LOG_UNEXPECTED_LOCAL_ATTACHMENT, id, '{"foo":{}}'] [report.LOG_DELETE_LOCAL, id]
]); ]);
}) })
.then(function () { .then(function () {
...@@ -2213,25 +2214,25 @@ ...@@ -2213,25 +2214,25 @@
.then(function () { .then(function () {
return context.jio.get(id); return context.jio.get(id);
}) })
.then(function (result) { .fail(function (error) {
deepEqual(result, { ok(error instanceof jIO.util.jIOError);
title: "foo" equal(error.message, "Cannot find document: " + id);
}); equal(error.status_code, 404);
}) })
.then(function () { .then(function () {
return context.jio.getAttachment(id, 'foo', {format: 'text'}); return context.jio.getAttachment(id, 'foo', {format: 'text'});
}) })
.then(function (result) { .fail(function (error) {
equal(result, big_string); ok(error instanceof jIO.util.jIOError);
equal(error.message, "Cannot find attachment: " + id + " , foo");
equal(error.status_code, 404);
}) })
.then(function () { .then(function () {
return context.jio.__storage._signature_sub_storage.get(id); return context.jio.__storage._signature_sub_storage.get(id);
}) })
.then(function (result) { .fail(function (error) {
deepEqual(result, { ok(error instanceof jIO.util.jIOError);
from_local: true, equal(error.status_code, 404);
hash: "5ea9013447539ad65de308cbd75b5826a2ae30e5"
});
}) })
.always(function () { .always(function () {
start(); start();
......
test/jio.storage/replicatestorage_repairattachment.tests.js
View file @ b6b14699
...@@ -4928,4 +4928,523 @@ ...@@ -4928,4 +4928,523 @@
}); });
}); });
///////////////////////////////////////
// Query parameter handling
///////////////////////////////////////
test("local document stop matching query with attachment", function () {
stop();
expect(4);
var id,
second_id,
context = this,
blob = new Blob(['a']);
this.jio = jIO.createJIO({
type: "replicate",
report_level: 1000,
query: {sort_on: [['title', 'descending']], limit: [0, 1]},
check_local_attachment_modification: true,
check_local_attachment_creation: true,
check_local_attachment_deletion: true,
check_remote_attachment_modification: true,
check_remote_attachment_creation: true,
check_remote_attachment_deletion: true,
local_sub_storage: {
type: "uuid",
sub_storage: {
type: "query",
sub_storage: {
type: "memory"
}
}
},
remote_sub_storage: {
type: "uuid",
sub_storage: {
type: "query",
sub_storage: {
type: "memory"
}
}
}
});
context.jio.post({"title": "a"})
.then(function (result) {
id = result;
return context.jio.putAttachment(id, 'foo', blob);
})
.then(function () {
return context.jio.repair();
})
.then(function (report) {
deepEqual(report._list, [
[report.LOG_PUT_REMOTE, id],
[report.LOG_PUT_REMOTE_ATTACHMENT, id, 'foo']
]);
ok(true, 'first repair success');
// Create another document, which will make the first one
// not matching the query anymore
// and so, will be considered as deleted
return context.jio.post({"title": "b"});
})
.then(function (result) {
second_id = result;
return context.jio.repair();
})
.then(function (report) {
deepEqual(report._list, [
[report.LOG_PUT_REMOTE, second_id],
[report.LOG_NO_CHANGE_ATTACHMENT, id, 'foo'],
[report.LOG_DELETE_REMOTE, id]
]);
ok(true, 'second repair success');
})
.fail(function (error) {
ok(false, error);
})
.always(function () {
start();
});
});
test("local document stop matching query with att. conflict", function () {
stop();
expect(4);
var id,
second_id,
context = this,
blob = new Blob(['a']),
blob2 = new Blob(['b']),
blob3 = new Blob(['c']);
this.jio = jIO.createJIO({
type: "replicate",
report_level: 1000,
query: {sort_on: [['title', 'descending']], limit: [0, 1]},
check_local_attachment_modification: true,
check_local_attachment_creation: true,
check_local_attachment_deletion: true,
check_remote_attachment_modification: true,
check_remote_attachment_creation: true,
check_remote_attachment_deletion: true,
local_sub_storage: {
type: "uuid",
sub_storage: {
type: "query",
sub_storage: {
type: "memory"
}
}
},
remote_sub_storage: {
type: "uuid",
sub_storage: {
type: "query",
sub_storage: {
type: "memory"
}
}
}
});
context.jio.post({"title": "a"})
.then(function (result) {
id = result;
return context.jio.putAttachment(id, 'foo', blob);
})
.then(function () {
return context.jio.repair();
})
.then(function (report) {
deepEqual(report._list, [
[report.LOG_PUT_REMOTE, id],
[report.LOG_PUT_REMOTE_ATTACHMENT, id, 'foo']
]);
ok(true, 'first repair success');
// Create another document, which will make the first one
// not matching the query anymore
// and so, will be considered as deleted
return RSVP.all([
context.jio.post({"title": "b"}),
context.jio.putAttachment(id, 'foo', blob2),
context.jio.__storage._remote_sub_storage.putAttachment(
id,
"foo",
blob3
)
]);
})
.then(function (result) {
second_id = result[0];
return context.jio.repair();
})
.fail(function (report) {
deepEqual(report._list, [
[report.LOG_PUT_REMOTE, second_id],
[report.LOG_UNRESOLVED_ATTACHMENT_CONFLICT, id, 'foo'],
[report.LOG_UNEXPECTED_REMOTE_ATTACHMENT, id]
]);
ok(true, 'second repair success');
})
.fail(function (error) {
ok(false, error);
})
.always(function () {
start();
});
});
test("local document stop matching query with att. conf. res.", function () {
stop();
expect(4);
var id,
second_id,
context = this,
blob = new Blob(['a']),
blob2 = new Blob(['b']),
blob3 = new Blob(['c']);
this.jio = jIO.createJIO({
type: "replicate",
report_level: 1000,
query: {sort_on: [['title', 'descending']], limit: [0, 1]},
check_local_attachment_modification: true,
check_local_attachment_creation: true,
check_local_attachment_deletion: true,
check_remote_attachment_modification: true,
check_remote_attachment_creation: true,
check_remote_attachment_deletion: true,
conflict_handling: 2,
local_sub_storage: {
type: "uuid",
sub_storage: {
type: "query",
sub_storage: {
type: "memory"
}
}
},
remote_sub_storage: {
type: "uuid",
sub_storage: {
type: "query",
sub_storage: {
type: "memory"
}
}
}
});
context.jio.post({"title": "a"})
.then(function (result) {
id = result;
return context.jio.putAttachment(id, 'foo', blob);
})
.then(function () {
return context.jio.repair();
})
.then(function (report) {
deepEqual(report._list, [
[report.LOG_PUT_REMOTE, id],
[report.LOG_PUT_REMOTE_ATTACHMENT, id, 'foo']
]);
ok(true, 'first repair success');
// Create another document, which will make the first one
// not matching the query anymore
// and so, will be considered as deleted
return RSVP.all([
context.jio.post({"title": "b"}),
context.jio.putAttachment(id, 'foo', blob2),
context.jio.__storage._remote_sub_storage.putAttachment(
id,
"foo",
blob3
)
]);
})
.then(function (result) {
second_id = result[0];
return context.jio.repair();
})
.then(function (report) {
deepEqual(report._list, [
[report.LOG_PUT_REMOTE, second_id],
[report.LOG_FORCE_PUT_LOCAL_ATTACHMENT, id, 'foo'],
[report.LOG_DELETE_REMOTE, id]
]);
ok(true, 'second repair success');
})
.fail(function (error) {
ok(false, error);
})
.always(function () {
start();
});
});
test("remote document stop matching query with attachment", function () {
stop();
expect(4);
var id,
second_id,
context = this,
blob = new Blob(['a']);
this.jio = jIO.createJIO({
type: "replicate",
report_level: 1000,
query: {sort_on: [['title', 'descending']], limit: [0, 1]},
check_local_attachment_modification: true,
check_local_attachment_creation: true,
check_local_attachment_deletion: true,
check_remote_attachment_modification: true,
check_remote_attachment_creation: true,
check_remote_attachment_deletion: true,
local_sub_storage: {
type: "uuid",
sub_storage: {
type: "query",
sub_storage: {
type: "memory"
}
}
},
remote_sub_storage: {
type: "uuid",
sub_storage: {
type: "query",
sub_storage: {
type: "memory"
}
}
}
});
context.jio.post({"title": "a"})
.then(function (result) {
id = result;
return context.jio.putAttachment(id, 'foo', blob);
})
.then(function () {
return context.jio.repair();
})
.then(function (report) {
deepEqual(report._list, [
[report.LOG_PUT_REMOTE, id],
[report.LOG_PUT_REMOTE_ATTACHMENT, id, 'foo']
]);
ok(true, 'first repair success');
// Create another document, which will make the first one
// not matching the query anymore
// and so, will be considered as deleted
return context.jio.__storage._remote_sub_storage.post({"title": "b"});
})
.then(function (result) {
second_id = result;
return context.jio.repair();
})
.then(function (report) {
deepEqual(report._list, [
[report.LOG_PUT_LOCAL, second_id],
[report.LOG_NO_CHANGE_ATTACHMENT, id, 'foo'],
[report.LOG_DELETE_LOCAL, id]
]);
ok(true, 'second repair success');
})
.fail(function (error) {
ok(false, error);
})
.always(function () {
start();
});
});
test("remote document stop matching query with att. conflict", function () {
stop();
expect(4);
var id,
second_id,
context = this,
blob = new Blob(['a']),
blob2 = new Blob(['b']),
blob3 = new Blob(['c']);
this.jio = jIO.createJIO({
type: "replicate",
report_level: 1000,
query: {sort_on: [['title', 'descending']], limit: [0, 1]},
check_local_attachment_modification: true,
check_local_attachment_creation: true,
check_local_attachment_deletion: true,
check_remote_attachment_modification: true,
check_remote_attachment_creation: true,
check_remote_attachment_deletion: true,
local_sub_storage: {
type: "uuid",
sub_storage: {
type: "query",
sub_storage: {
type: "memory"
}
}
},
remote_sub_storage: {
type: "uuid",
sub_storage: {
type: "query",
sub_storage: {
type: "memory"
}
}
}
});
context.jio.post({"title": "a"})
.then(function (result) {
id = result;
return context.jio.putAttachment(id, 'foo', blob);
})
.then(function () {
return context.jio.repair();
})
.then(function (report) {
deepEqual(report._list, [
[report.LOG_PUT_REMOTE, id],
[report.LOG_PUT_REMOTE_ATTACHMENT, id, 'foo']
]);
ok(true, 'first repair success');
// Create another document, which will make the first one
// not matching the query anymore
// and so, will be considered as deleted
return RSVP.all([
context.jio.__storage._remote_sub_storage.post({"title": "b"}),
context.jio.putAttachment(id, 'foo', blob2),
context.jio.__storage._remote_sub_storage.putAttachment(
id,
"foo",
blob3
)
]);
})
.then(function (result) {
second_id = result[0];
return context.jio.repair();
})
.fail(function (report) {
deepEqual(report._list, [
[report.LOG_PUT_LOCAL, second_id],
[report.LOG_UNRESOLVED_ATTACHMENT_CONFLICT, id, 'foo'],
[report.LOG_UNEXPECTED_LOCAL_ATTACHMENT, id]
]);
ok(true, 'second repair success');
})
.fail(function (error) {
ok(false, error);
})
.always(function () {
start();
});
});
test("remote document stop matching query with att. conf. res.", function () {
stop();
expect(4);
var id,
second_id,
context = this,
blob = new Blob(['a']),
blob2 = new Blob(['b']),
blob3 = new Blob(['c']);
this.jio = jIO.createJIO({
type: "replicate",
report_level: 1000,
query: {sort_on: [['title', 'descending']], limit: [0, 1]},
check_local_attachment_modification: true,
check_local_attachment_creation: true,
check_local_attachment_deletion: true,
check_remote_attachment_modification: true,
check_remote_attachment_creation: true,
check_remote_attachment_deletion: true,
conflict_handling: 1,
local_sub_storage: {
type: "uuid",
sub_storage: {
type: "query",
sub_storage: {
type: "memory"
}
}
},
remote_sub_storage: {
type: "uuid",
sub_storage: {
type: "query",
sub_storage: {
type: "memory"
}
}
}
});
context.jio.post({"title": "a"})
.then(function (result) {
id = result;
return context.jio.putAttachment(id, 'foo', blob);
})
.then(function () {
return context.jio.repair();
})
.then(function (report) {
deepEqual(report._list, [
[report.LOG_PUT_REMOTE, id],
[report.LOG_PUT_REMOTE_ATTACHMENT, id, 'foo']
]);
ok(true, 'first repair success');
// Create another document, which will make the first one
// not matching the query anymore
// and so, will be considered as deleted
return RSVP.all([
context.jio.__storage._remote_sub_storage.post({"title": "b"}),
context.jio.putAttachment(id, 'foo', blob2),
context.jio.__storage._remote_sub_storage.putAttachment(
id,
"foo",
blob3
)
]);
})
.then(function (result) {
second_id = result[0];
return context.jio.repair();
})
.then(function (report) {
deepEqual(report._list, [
[report.LOG_PUT_LOCAL, second_id],
[report.LOG_FORCE_PUT_REMOTE_ATTACHMENT, id, 'foo'],
[report.LOG_DELETE_LOCAL, id]
]);
ok(true, 'second repair success');
})
.fail(function (error) {
ok(false, error);
})
.always(function () {
start();
});
});
}(jIO, QUnit, Blob, RSVP)); }(jIO, QUnit, Blob, RSVP));
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7