Skip to content

J
jio
Commit 17dec26b authored by preetwinder's avatar preetwinder Committed by Romain Courteaud
Browse files
Options

Add support for manual repair

parent 9ebbd6e0
Hide whitespace changes
Inline Side-by-side
Showing with 421 additions and 114 deletions
src/jio.storage/indexstorage2.js
View file @ 17dec26b
......@@ -19,10 +19,10 @@
*/
/*jslint nomen: true */
/*global indexedDB, jIO, RSVP, IDBOpenDBRequest, DOMError, Event,
parseStringToObject, Set*/
parseStringToObject, Set, DOMException*/
(function (indexedDB, jIO, RSVP, IDBOpenDBRequest, DOMError,
parseStringToObject) {
parseStringToObject, DOMException) {
"use strict";
function IndexStorage2(description) {
......@@ -35,11 +35,16 @@
throw new TypeError("IndexStorage2 'index_keys' description property " +
"must be an Array");
}
if (description.version && (typeof description.version !== "number")) {
throw new TypeError("IndexStorage2 'version' description property " +
"must be a number");
}
this._sub_storage_description = description.sub_storage;
this._sub_storage = jIO.createJIO(description.sub_storage);
this._database_name = "jio:" + description.database;
this._index_keys = description.index_keys || [];
this._version = description.version || undefined;
this._version = description.version;
this._signature_storage_name = description.database + "_signatures";
}
IndexStorage2.prototype.hasCapacity = function (name) {
......@@ -76,55 +81,194 @@
});
}
function iterateCursor(on, query, limit) {
return new RSVP.Promise(function (resolve, reject) {
var result = [], count = 0, cursor;
cursor = on.openKeyCursor(query);
cursor.onsuccess = function (cursor) {
if (cursor.target.result && count !== limit) {
count += 1;
result.push({id: cursor.target.result.primaryKey, value: {}});
cursor.target.result.continue();
} else {
resolve(result);
}
};
cursor.onerror = function (error) {
reject(error.message);
};
});
}
function VirtualIDB(description) {
this._write_operations = description.write_operations;
this._operations = description.operations;
}
VirtualIDB.prototype.put = function () {
this._write_operations.put.push(arguments);
VirtualIDB.prototype.hasCapacity = function (name) {
return (name === "list");
};
VirtualIDB.prototype.hasCapacity = function (name) {
return (name === 'list') || (name === 'select');
VirtualIDB.prototype.put = function (id, value) {
var context = this;
return new RSVP.Promise(function (resolve, reject) {
context._operations.push({type: "put", arguments: [id, value],
onsuccess: resolve, onerror: reject});
});
};
VirtualIDB.prototype.remove = function (id) {
var context = this;
return new RSVP.Promise(function (resolve, reject) {
context._operations.push({type: "remove", arguments: [id],
onsuccess: resolve, onerror: reject});
});
};
VirtualIDB.prototype.get = function (id) {
throw new jIO.util.jIOError("Cannot find document: " + id, 404);
var context = this;
return new RSVP.Promise(function (resolve, reject) {
context._operations.push({type: "get", arguments: [id],
onsuccess: resolve, onerror: reject});
});
};
VirtualIDB.prototype.buildQuery = function (options) {
var context = this;
return new RSVP.Promise(function (resolve, reject) {
context._operations.push({type: "buildQuery", arguments: [options],
onsuccess: resolve, onerror: reject});
});
};
VirtualIDB.prototype.buildQuery = function () {
return [];
VirtualIDB.prototype.allAttachments = function () {
return {};
};
jIO.addStorage("virtualidb", VirtualIDB);
function getRepairStorage(write_operations, sub_storage_description) {
function getRepairStorage(operations, sub_storage_description,
signature_storage_name) {
return jIO.createJIO({
type: "replicate",
local_sub_storage: sub_storage_description,
check_local_modification: false,
check_local_deletion: false,
check_local_creation: true,
check_remote_modification: false,
check_remote_creation: false,
check_remote_deletion: false,
remote_sub_storage: {
type: "virtualidb",
write_operations: write_operations,
operations: operations
},
signature_sub_storage: {
type: "query",
sub_storage: {
type: "memory"
type: "indexeddb",
database: signature_storage_name
}
}
},
check_remote_modification: false,
check_remote_creation: false,
check_remote_deletion: false,
conflict_handling: 1,
parallel_operation_amount: 16
});
}
function handleUpgradeNeeded(evt, index_keys, sub_storage_description) {
var db = evt.target.result, store, i, current_indices, required_indices,
put_promise_list = [], repair_promise, repeatUntilPromiseFulfilled,
write_operations;
function handleVirtualGetSuccess(id, onsuccess, onerror) {
return function (result) {
if (result.target.result === undefined) {
return onerror(new jIO.util.jIOError("Cannot find document: " +
id, 404));
}
return onsuccess(result.target.result.doc);
};
}
function processVirtualOperation(operation, store, index_keys, disable_get) {
var request, get_success_handler;
if (operation.type === "put") {
request = store.put({
id: operation.arguments[0],
doc: filterDocValues(operation.arguments[1], index_keys),
});
request.onerror = operation.onerror;
return {request: request, onsuccess: operation.onsuccess};
}
if (operation.type === "get") {
// if storage was cleared, get can return without checking the database
if (disable_get) {
operation.onerror(new jIO.util.jIOError("Cannot find document: " +
operation.arguments[0], 404));
} else {
get_success_handler = handleVirtualGetSuccess(operation.arguments[0],
operation.onsuccess, operation.onerror);
request = store.get(operation.arguments[0]);
request.onerror = operation.onerror;
return {request: request, onsuccess: get_success_handler};
}
}
if (operation.type === "buildQuery") {
request = iterateCursor(store);
request.then(operation.onsuccess).fail(operation.onerror);
return;
}
if (operation.type === "remove") {
request = store.delete(operation.arguments[0]);
request.onerror = operation.onerror;
return {request: request, onsuccess: operation.onsuccess};
}
}
var transaction_failure_reason;
function repairInTransaction(sub_storage_description, transaction,
index_keys, signature_storage_name, clear_storage) {
var repair_promise, repeatUntilPromiseFulfilled, store,
operations = [];
if (clear_storage) {
indexedDB.deleteDatabase("jio:" + signature_storage_name);
}
store = transaction.objectStore("index-store");
repair_promise = getRepairStorage(operations,
sub_storage_description, signature_storage_name).repair();
repeatUntilPromiseFulfilled = function repeatUntilPromiseFulfilled(
continuation_request,
continuation_resolve
) {
var operation_result, next_continuation_request,
next_continuation_resolve;
continuation_request.onsuccess = function () {
if (continuation_resolve) {
continuation_resolve.apply(null, arguments);
}
while (true) {
if (operations.length === 0) {
break;
}
operation_result = processVirtualOperation(operations.shift(), store,
index_keys, clear_storage);
// use the current request to continue the repeat loop if possible
if (next_continuation_request && operation_result) {
operation_result.request.onsuccess = operation_result.onsuccess;
} else if (operation_result) {
next_continuation_request = operation_result.request;
next_continuation_resolve = operation_result.onsuccess;
}
}
if (repair_promise.isRejected) {
transaction.abort();
transaction_failure_reason = repair_promise.rejectedReason;
return;
}
if (repair_promise.isFulfilled) {
return;
}
return repeatUntilPromiseFulfilled(next_continuation_request ||
store.get("inexistent"), next_continuation_resolve);
};
};
repeatUntilPromiseFulfilled(store.get("inexistent"));
}
function handleUpgradeNeeded(evt, index_keys, sub_storage_description,
signature_storage_name) {
var db = evt.target.result, store, i, current_indices, required_indices;
required_indices = new Set(index_keys.map(function (name) {
return 'Index-' + name;
}));
......@@ -134,12 +278,11 @@
current_indices = new Set(store ? store.indexNames : []);
if (isSubset(current_indices, required_indices)) {
if (!store) {
return;
}
for (i = 0; i < store.indexNames.length; i += 1) {
if (!required_indices.has(store.indexNames[i])) {
store.deleteIndex(store.indexNames[i]);
if (store) {
for (i = 0; i < store.indexNames.length; i += 1) {
if (!required_indices.has(store.indexNames[i])) {
store.deleteIndex(store.indexNames[i]);
}
}
}
} else {
......@@ -155,49 +298,33 @@
store.createIndex('Index-' + index_keys[i],
'doc.' + index_keys[i], { unique: false });
}
write_operations = {put: []};
repair_promise = getRepairStorage(write_operations,
sub_storage_description).repair();
repeatUntilPromiseFulfilled = function repeatUntilPromiseFulfilled(req) {
req.onsuccess = function () {
if (repair_promise.isRejected) {
evt.target.transaction.abort();
return;
}
if (repair_promise.isFulfilled) {
for (i = 0; i < write_operations.put.length; i += 1) {
put_promise_list.push(waitForIDBRequest(store.put({
id: write_operations.put[i][0],
doc: filterDocValues(write_operations.put[i][1], index_keys)
})));
}
write_operations.put = [];
return RSVP.all(put_promise_list);
}
return repeatUntilPromiseFulfilled(store.getAll());
};
};
repeatUntilPromiseFulfilled(store.getAll());
return repairInTransaction(sub_storage_description,
evt.target.transaction, index_keys, signature_storage_name, true);
}
}
function waitForOpenIndexedDB(db_name, version, index_keys,
sub_storage_description, callback) {
sub_storage_description, signature_storage_name, callback) {
function resolver(resolve, reject) {
// Open DB //
var request = indexedDB.open(db_name, version);
request.onerror = function (error) {
var error_sub_message;
if (request.result) {
request.result.close();
}
if ((error !== undefined) &&
(error.target instanceof IDBOpenDBRequest) &&
(error.target.error instanceof DOMError)) {
reject("Connection to: " + db_name + " failed: " +
error.target.error.message);
((error.target.error instanceof DOMError) ||
(error.target.error instanceof DOMException))) {
error_sub_message = error.target.error.message;
if (transaction_failure_reason) {
error_sub_message += " " + transaction_failure_reason;
transaction_failure_reason = undefined;
}
reject("Connection to: " + db_name + " failed: " + error_sub_message);
} else {
reject(error.target.error);
reject(error);
}
};
......@@ -218,7 +345,8 @@
// Create DB if necessary //
request.onupgradeneeded = function (evt) {
handleUpgradeNeeded(evt, index_keys, sub_storage_description);
handleUpgradeNeeded(evt, index_keys, sub_storage_description,
signature_storage_name);
};
request.onversionchange = function () {
......@@ -283,32 +411,14 @@
return new RSVP.Promise(resolver, canceller);
}
IndexStorage2.prototype._iterateCursor = function (on, query, limit) {
return new RSVP.Promise(function (resolve, reject) {
var result_list = [], count = 0, cursor;
cursor = on.openKeyCursor(query);
cursor.onsuccess = function (cursor) {
if (cursor.target.result && count !== limit) {
count += 1;
result_list.push({id: cursor.target.result.primaryKey, value: {}});
cursor.target.result.continue();
} else {
resolve(result_list);
}
};
cursor.onerror = function (error) {
reject(error.message);
};
});
};
IndexStorage2.prototype._runQuery = function (key, value, limit) {
var context = this;
return waitForOpenIndexedDB(context._database_name, context._version,
context._index_keys, context._sub_storage_description, function (db) {
context._index_keys, context._sub_storage_description,
context._signature_storage_name, function (db) {
return waitForTransaction(db, ["index-store"], "readonly",
function (tx) {
return context._iterateCursor(tx.objectStore("index-store")
return iterateCursor(tx.objectStore("index-store")
.index("Index-" + key), value, limit);
});
});
......@@ -349,7 +459,8 @@
return;
}
return waitForOpenIndexedDB(context._database_name, context._version,
context._index_keys, context._sub_storage_description, function (db) {
context._index_keys, context._sub_storage_description,
context._signature_storage_name, function (db) {
return waitForTransaction(db, ["index-store"], "readwrite",
function (tx) {
return waitForIDBRequest(tx.objectStore("index-store").put({
......@@ -381,7 +492,8 @@
return context._sub_storage.remove(id)
.push(function () {
return waitForOpenIndexedDB(context._database_name, context._version,
context._index_keys, context._sub_storage_description, function (db) {
context._index_keys, context._sub_storage_description,
context._signature_storage_name, function (db) {
return waitForTransaction(db, ["index-store"], "readwrite",
function (tx) {
return waitForIDBRequest(tx.objectStore("index-store")
......@@ -391,6 +503,19 @@
});
};
IndexStorage2.prototype.repair = function () {
var context = this;
return waitForOpenIndexedDB(context._database_name, context._version,
context._index_keys, context._sub_storage_description,
context._signature_storage_name, function (db) {
return waitForTransaction(db, ["index-store"], "readwrite",
function (tx) {
return repairInTransaction(context._sub_storage_description, tx,
context._index_keys, context._signature_storage_name);
});
});
};
IndexStorage2.prototype.getAttachment = function () {
return this._sub_storage.getAttachment.apply(this._sub_storage, arguments);
};
......@@ -405,4 +530,5 @@
};
jIO.addStorage("index2", IndexStorage2);
}(indexedDB, jIO, RSVP, IDBOpenDBRequest, DOMError, parseStringToObject));
\ No newline at end of file
}(indexedDB, jIO, RSVP, IDBOpenDBRequest, DOMError, parseStringToObject,
DOMException));
\ No newline at end of file
test/jio.storage/indexstorage2.tests.js
View file @ 17dec26b
......@@ -31,14 +31,19 @@
module = QUnit.module,
throws = QUnit.throws;
function deleteIndexedDB(storage) {
function deleteIndexStorage2(storage) {
return new RSVP.Promise(function resolver(resolve, reject) {
var request = indexedDB.deleteDatabase(
var storage_deletion_request = indexedDB.deleteDatabase(
storage.__storage._database_name
), signature_deletion_request = indexedDB.deleteDatabase(
"jio:" + storage.__storage._signature_storage_name
);
request.onerror = reject;
request.onblocked = reject;
request.onsuccess = resolve;
storage_deletion_request.onerror = reject;
storage_deletion_request.onblocked = reject;
storage_deletion_request.onsuccess = resolve;
signature_deletion_request.onerror = reject;
signature_deletion_request.onblocked = reject;
signature_deletion_request.onsuccess = resolve;
});
}
......@@ -65,7 +70,7 @@
/////////////////////////////////////////////////////////////////
module("indexStorage2.constructor", {
teardown: function () {
deleteIndexedDB(this.jio);
deleteIndexStorage2(this.jio);
}
});
test("Constructor without index_keys", function () {
......@@ -80,6 +85,7 @@
equal(this.jio.__type, "index2");
equal(this.jio.__storage._sub_storage.__type, "dummystorage3");
equal(this.jio.__storage._database_name, "jio:index2_test");
equal(this.jio.__storage._signature_storage_name, "index2_test_signatures");
deepEqual(this.jio.__storage._index_keys, []);
});
......@@ -141,12 +147,32 @@
return true;
}
);
throws(
function () {
this.jio = jIO.createJIO({
type: "index2",
database: "index2_test",
index_keys: ["a", "b"],
version: "1",
sub_storage: {
type: "dummystorage3"
}
});
},
function (error) {
ok(error instanceof TypeError);
equal(error.message, "IndexStorage2 'version' description property" +
" must be a number");
return true;
}
);
});
test("Constructor with index_keys", function () {
test("Constructor with index_keys and version", function () {
this.jio = jIO.createJIO({
type: "index2",
database: "index2_test",
version: 4,
index_keys: ["a", "b"],
sub_storage: {
type: "dummystorage3"
......@@ -156,6 +182,10 @@
equal(this.jio.__type, "index2");
equal(this.jio.__storage._sub_storage.__type, "dummystorage3");
equal(this.jio.__storage._database_name, "jio:index2_test");
equal(this.jio.__storage._version, 4);
equal(this.jio.__storage._signature_storage_name, "index2_test_signatures");
deepEqual(this.jio.__storage._sub_storage_description,
{type: "dummystorage3"});
deepEqual(this.jio.__storage._index_keys, ["a", "b"]);
});
......@@ -164,10 +194,10 @@
/////////////////////////////////////////////////////////////////
module("indexStorage2.hasCapacity", {
teardown: function () {
deleteIndexedDB(this.jio);
deleteIndexStorage2(this.jio);
}
});
test("can list documents", function () {
test("Test various capacities", function () {
this.jio = jIO.createJIO({
type: "index2",
database: "index2_test",
......@@ -213,7 +243,7 @@
});
},
teardown: function () {
deleteIndexedDB(this.jio);
deleteIndexStorage2(this.jio);
}
});
test("Get calls substorage", function () {
......@@ -251,7 +281,7 @@
};
},
teardown: function () {
deleteIndexedDB(this.jio);
deleteIndexStorage2(this.jio);
}
});
......@@ -650,26 +680,21 @@
stop();
expect(8);
dummy_data = {
"32": {id: "32", doc: {"a": "3", "b": "2", "c": "inverse"},
value: {"a": "3", "b": "2", "c": "inverse"}},
"5": {id: "5", doc: {"a": "6", "b": "2", "c": "strong"},
value: {"a": "6", "b": "2", "c": "strong"}},
"14": {id: "14", doc: {"a": "67", "b": "3", "c": "disolve"},
value: {"a": "67", "b": "3", "c": "disolve"}}
};
dummy_data = {};
DummyStorage3.prototype.put = function (id, value) {
dummy_data[id] = {id: id, doc: value, value: value};
return id;
};
DummyStorage3.prototype.get = function (id) {
return dummy_data[id].doc;
if (dummy_data[id]) {
return dummy_data[id].doc;
}
throw new jIO.util.jIOError("Cannot find document: " + id, 404);
};
DummyStorage3.prototype.hasCapacity = function (name) {
return (name === 'list') || (name === 'include') || (name === 'select');
};
DummyStorage3.prototype.buildQuery = function () {
return Object.values(dummy_data);
};
......@@ -781,8 +806,164 @@
context.jio.allDocs({query: 'c: "control"'})
.fail(function (error) {
equal(error.message, "Version change transaction was aborted in" +
" upgradeneeded event handler.");
equal(error, "Connection to: jio:index2_test failed: Version change " +
"transaction was aborted in upgradeneeded event handler. " +
"Error: Capacity 'buildQuery' is not implemented on 'dummystorage3'");
})
.always(function () {
start();
});
});
test("Manual repair", function () {
var context = this, fake_data;
context.jio = jIO.createJIO({
type: "index2",
database: "index2_test",
index_keys: ["a", "c"],
sub_storage: {
type: "dummystorage3"
}
});
stop();
expect(15);
fake_data = {
"1": {a: "id54", b: "9", c: "night"},
"4": {a: "vn92", b: "7", c: "matter"},
"9": {a: "ru23", b: "3", c: "control"},
"42": {a: "k422", b: "100", c: "grape"}
};
DummyStorage3.prototype.hasCapacity = function (name) {
return (name === 'list') || (name === 'select');
};
DummyStorage3.prototype.put = function (id, value) {
fake_data[id] = value;
return id;
};
DummyStorage3.prototype.get = function (id) {
if (fake_data[id]) {
return fake_data[id];
}
throw new jIO.util.jIOError("Cannot find document: " + id, 404);
};
DummyStorage3.prototype.remove = function (id) {
delete fake_data[id];
return id;
};
DummyStorage3.prototype.buildQuery = function () {
var keys = Object.keys(fake_data);
return keys.map(function (v) { return {id: v, value: {}}; });
};
context.jio.allDocs({query: 'c: "control"'})
.then(function (result) {
equal(result.data.total_rows, 1);
deepEqual(result.data.rows, [{id: "9", value: {}}]);
})
.then(function () {
fake_data["2"] = {a: "zu64", b: "1", c: "matter"};
fake_data["13"] = {a: "tk32", b: "9", c: "matter"};
return context.jio.repair();
})
.then(function () {
return context.jio.allDocs({query: 'c: "matter"'});
})
.then(function (result) {
equal(result.data.total_rows, 3);
deepEqual(result.data.rows.sort(idCompare), [{id: "13", value: {}},
{id: "2", value: {}}, {id: "4", value: {}}]);
})
.then(function () {
fake_data["2"] = {a: "zu64", b: "1", c: "observe"};
return context.jio.repair();
})
.then(function () {
return context.jio.allDocs({query: 'c: "observe"'});
})
.then(function (result) {
equal(result.data.total_rows, 1);
deepEqual(result.data.rows, [{id: "2", value: {}}]);
})
.then(function () {
delete fake_data["2"];
return context.jio.repair();
})
.then(function () {
return context.jio.allDocs({query: 'c: "observe"'});
})
.then(function (result) {
equal(result.data.total_rows, 0);
deepEqual(result.data.rows, []);
})
.then(function () {
return RSVP.all([
context.jio.put("43", {a: "t345", b: "101", c: "pear"}),
context.jio.put("44", {a: "j939", b: "121", c: "grape"}),
context.jio.put("45", {a: "q423", b: "131", c: "grape"}),
context.jio.remove("42")
]);
})
.then(function () {
return context.jio.repair();
})
.then(function () {
return context.jio.allDocs({query: "c:grape"});
})
.then(function (result) {
equal(result.data.total_rows, 2);
deepEqual(result.data.rows.sort(idCompare), [{id: "44", value: {}},
{id: "45", value: {}}]);
})
.then(function () {
context.jio = jIO.createJIO({
type: "index2",
database: "index2_test",
index_keys: ["a", "c"],
sub_storage: {
type: "dummystorage3"
}
});
delete fake_data["13"];
fake_data["3"] = {a: "gg38", b: "4", c: "matter"};
fake_data["9"] = {a: "tk32", b: "9", c: "matter"};
return context.jio.repair();
})
.then(function () {
return context.jio.allDocs({query: "c:matter"});
})
.then(function (result) {
equal(result.data.total_rows, 3);
deepEqual(result.data.rows.sort(idCompare), [{id: "3", value: {}},
{id: "4", value: {}}, {id: "9", value: {}}]);
})
.then(function () {
delete fake_data["9"];
fake_data["3"] = {a: "xu76", b: "9", c: "night"};
fake_data["7"] = {a: "bn02", b: "9", c: "matter"};
context.jio = jIO.createJIO({
type: "index2",
database: "index2_test",
index_keys: ["b"],
version: 2,
sub_storage: {
type: "dummystorage3"
}
});
return context.jio.allDocs({query: "b:9"});
})
.then(function (result) {
equal(result.data.total_rows, 3);
deepEqual(result.data.rows.sort(idCompare), [{id: "1", value: {}},
{id: "3", value: {}}, {id: "7", value: {}}]);
})
.then(function () {
return context.jio.allDocs({query: "c:matter"});
})
.fail(function (error) {
equal(error.message,
"Capacity 'query' is not implemented on 'dummystorage3'");
})
.always(function () {
start();
......@@ -794,7 +975,7 @@
/////////////////////////////////////////////////////////////////
module("IndexStorage2.getAttachment", {
teardown: function () {
deleteIndexedDB(this.jio);
deleteIndexStorage2(this.jio);
}
});
test("getAttachment called substorage getAttachment", function () {
......@@ -834,7 +1015,7 @@
/////////////////////////////////////////////////////////////////
module("IndexStorage2.putAttachment", {
teardown: function () {
deleteIndexedDB(this.jio);
deleteIndexStorage2(this.jio);
}
});
test("putAttachment called substorage putAttachment", function () {
......@@ -875,7 +1056,7 @@
/////////////////////////////////////////////////////////////////
module("IndexStorage2.removeAttachment", {
teardown: function () {
deleteIndexedDB(this.jio);
deleteIndexStorage2(this.jio);
}
});
test("removeAttachment called substorage removeAttachment", function () {
......@@ -914,7 +1095,7 @@
/////////////////////////////////////////////////////////////////
module("indexStorage2.put", {
teardown: function () {
deleteIndexedDB(this.jio);
deleteIndexStorage2(this.jio);
}
});
test("Put creates index", function () {
......@@ -998,7 +1179,7 @@
/////////////////////////////////////////////////////////////////
module("indexStorage2.post", {
teardown: function () {
deleteIndexedDB(this.jio);
deleteIndexStorage2(this.jio);
}
});
test("Post creates index", function () {
......@@ -1090,7 +1271,7 @@
/////////////////////////////////////////////////////////////////
module("indexStorage2.remove", {
teardown: function () {
deleteIndexedDB(this.jio);
deleteIndexStorage2(this.jio);
}
});
test("Remove values", function () {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7