Skip to content

J
jio
Commit f71fb6cb authored by Bryan Kaperick's avatar Bryan Kaperick
Browse files
Options

Implemented attachment methods so that scenario_officejs.js runs with... 

Implemented attachment methods so that scenario_officejs.js runs with historystorage as the local sub storage.
parent fede0168
Hide whitespace changes
Inline Side-by-side
Showing with 440 additions and 46 deletions
examples/scenario_officejs.js
View file @ f71fb6cb
...@@ -135,6 +135,7 @@ ...@@ -135,6 +135,7 @@
}, },
count: {} count: {}
}; };
this.jio = jIO.createJIO({ this.jio = jIO.createJIO({
type: "replicate", type: "replicate",
query: { query: {
...@@ -153,7 +154,7 @@ ...@@ -153,7 +154,7 @@
parallel_operation_amount: 10, parallel_operation_amount: 10,
parallel_operation_attachment_amount: 10, parallel_operation_attachment_amount: 10,
local_sub_storage: { local_sub_storage: {
type: "query", type: "history",
sub_storage: { sub_storage: {
type: "uuid", type: "uuid",
sub_storage: { sub_storage: {
...@@ -213,7 +214,6 @@ ...@@ -213,7 +214,6 @@
doc_id = 'foo_module/1', doc_id = 'foo_module/1',
doc = {title: doc_id, portal_type: "Foo", modification_date: 'a'}, doc = {title: doc_id, portal_type: "Foo", modification_date: 'a'},
blob = new Blob(['a']); blob = new Blob(['a']);
putFullDoc(this.jio.__storage._remote_sub_storage, doc_id, doc, blob) putFullDoc(this.jio.__storage._remote_sub_storage, doc_id, doc, blob)
.then(function () { .then(function () {
return test.jio.repair(); return test.jio.repair();
...@@ -279,7 +279,6 @@ ...@@ -279,7 +279,6 @@
test("remote document modification: copy", function () { test("remote document modification: copy", function () {
expect(2); expect(2);
stop(); stop();
var test = this, var test = this,
doc_id = 'foo_module/1', doc_id = 'foo_module/1',
doc = {title: doc_id, portal_type: "Foo", modification_date: 'a'}, doc = {title: doc_id, portal_type: "Foo", modification_date: 'a'},
...@@ -367,7 +366,6 @@ ...@@ -367,7 +366,6 @@
doc_id = 'abc', doc_id = 'abc',
doc = {title: doc_id, portal_type: "Foo", modification_date: 'a'}, doc = {title: doc_id, portal_type: "Foo", modification_date: 'a'},
blob = new Blob(['a']); blob = new Blob(['a']);
putFullDoc(this.jio, doc_id, doc, blob) putFullDoc(this.jio, doc_id, doc, blob)
.then(function () { .then(function () {
resetCount(test.remote_mock_options.count); resetCount(test.remote_mock_options.count);
......
src/jio.storage/historystorage.js
View file @ f71fb6cb
...@@ -3,6 +3,9 @@ ...@@ -3,6 +3,9 @@
(function (jIO) { (function (jIO) {
"use strict"; "use strict";
//local mod/frozen remote
//local del/remote mod
// Used to distinguish between operations done within the same millisecond // Used to distinguish between operations done within the same millisecond
var unique_timestamp = function () { var unique_timestamp = function () {
...@@ -30,11 +33,13 @@ ...@@ -30,11 +33,13 @@
HistoryStorage.prototype.get = function (id_in) { HistoryStorage.prototype.get = function (id_in) {
// Query to get the last edit made to this document // Query to get the last edit made to this document
var substorage = this._sub_storage, var substorage = this._sub_storage,
metadata_query = function (id) {
return "doc_id: " + id + " AND ((op: put) OR (op: remove))";
},
options = { options = {
query: "doc_id: " + id_in, query: metadata_query(id_in),
sort_on: [["timestamp", "descending"]], sort_on: [["timestamp", "descending"]],
limit: [0, 1] limit: [0, 1]
}; };
...@@ -45,7 +50,7 @@ ...@@ -45,7 +50,7 @@
return substorage.get(results.data.rows[0].id); return substorage.get(results.data.rows[0].id);
} }
throw new jIO.util.jIOError( throw new jIO.util.jIOError(
"HistoryStorage: cannot find object '" + id_in + "'", "HistoryStorage: cannot find object '" + id_in + "' (0)",
404 404
); );
}) })
...@@ -67,7 +72,7 @@ ...@@ -67,7 +72,7 @@
// is not found // is not found
if (steps_loc === -1) { if (steps_loc === -1) {
throw new jIO.util.jIOError( throw new jIO.util.jIOError(
"HistoryStorage: cannot find object '" + id_in + "'", "HistoryStorage: cannot find object '" + id_in + "' (1)",
404 404
); );
} }
...@@ -76,7 +81,7 @@ ...@@ -76,7 +81,7 @@
steps = Number(id_in.slice(steps_loc + 2)); steps = Number(id_in.slice(steps_loc + 2));
id_in = id_in.slice(0, steps_loc); id_in = id_in.slice(0, steps_loc);
options = { options = {
query: "doc_id: " + id_in, query: metadata_query(id_in),
sort_on: [["timestamp", "descending"]], sort_on: [["timestamp", "descending"]],
limit: [steps, 1] limit: [steps, 1]
}; };
...@@ -86,7 +91,7 @@ ...@@ -86,7 +91,7 @@
return substorage.get(results.data.rows[0].id); return substorage.get(results.data.rows[0].id);
} }
throw new jIO.util.jIOError( throw new jIO.util.jIOError(
"HistoryStorage: cannot find object '" + id_in + "'", "HistoryStorage: cannot find object '" + id_in + "' (2)",
404 404
); );
}) })
...@@ -95,7 +100,8 @@ ...@@ -95,7 +100,8 @@
return result.doc; return result.doc;
} }
throw new jIO.util.jIOError( throw new jIO.util.jIOError(
"HistoryStorage: cannot find object '" + id_in + "' (removed)", "HistoryStorage: cannot find object '" + id_in +
"' (removed) (1)",
404 404
); );
}); });
...@@ -129,18 +135,143 @@ ...@@ -129,18 +135,143 @@
return this._sub_storage.put(timestamp, metadata); return this._sub_storage.put(timestamp, metadata);
}; };
HistoryStorage.prototype.allAttachments = function () { HistoryStorage.prototype.allAttachments = function (id) {
return this._sub_storage.allAttachments.apply(this._sub_storage, arguments); var substorage = this._sub_storage,
options = {
query: "(doc_id: " + id + ") AND " +
"((op: putAttachment) OR (op: removeAttachment))",
sort_on: [["timestamp", "descending"]]
};
return this._sub_storage.allDocs(options)
.push(function (results) {
var promises = results.data.rows.map(function (data) {
return substorage.get(data.id);
});
return RSVP.all(promises);
})
.push(function (results) {
var seen = {},
attachments = {},
ind,
doc;
for (ind = 0; ind < results.length; ind += 1) {
doc = results[ind];
if (!seen.hasOwnProperty(doc.name)) {
if (doc.op === "putAttachment") {
attachments[doc.name] = {};
}
seen[doc.name] = {};
}
}
return attachments;
});
}; };
HistoryStorage.prototype.getAttachment = function () {
return this._sub_storage.getAttachment.apply(this._sub_storage, arguments); HistoryStorage.prototype.putAttachment = function (id, name, blob) {
var timestamp = unique_timestamp(),
metadata = {
// XXX: remove this attribute once query can sort_on id
timestamp: timestamp,
doc_id: id,
name: name,
op: "putAttachment"
},
substorage = this._sub_storage;
return this._sub_storage.put(timestamp, metadata)
.push(function () {
return substorage.putAttachment(timestamp, name, blob);
});
}; };
HistoryStorage.prototype.putAttachment = function () {
return this._sub_storage.putAttachment.apply(this._sub_storage, arguments); HistoryStorage.prototype.getAttachment = function (id, name) {
// Query to get the last edit made to this document
var substorage = this._sub_storage,
metadata_query = function (id) {
return "(doc_id: " + id +
") AND (name: " + name +
") AND ((op: putAttachment) OR (op: removeAttachment))";
},
options = {
query: metadata_query(id),
sort_on: [["timestamp", "descending"]],
limit: [0, 1]
};
return substorage.allDocs(options)
.push(function (results) {
if (results.data.rows.length > 0) {
return substorage.get(results.data.rows[0].id);
}
throw new jIO.util.jIOError(
"HistoryStorage: cannot find object '" + id + "' (0)",
404
);
})
.push(function (result) {
if (result.op === "putAttachment") {
return substorage.getAttachment(result.timestamp, result.name);
//return result.blob;
}
throw new jIO.util.jIOError(
"HistoryStorage: cannot find object '" + id + "' (removed)",
404
);
// If no documents returned in first query, check if the id is encoding
// revision information
}, function () {
var steps,
steps_loc = id.lastIndexOf("_-");
// If revision signature is not in id_in, than return 404, since id
// is not found
if (steps_loc === -1) {
throw new jIO.util.jIOError(
"HistoryStorage: cannot find object '" + id + "' (1)",
404
);
}
// If revision signature is found, query storage based on this
steps = Number(id.slice(steps_loc + 2));
id = id.slice(0, steps_loc);
options = {
query: metadata_query(id),
sort_on: [["timestamp", "descending"]],
limit: [steps, 1]
};
return substorage.allDocs(options)
.push(function (results) {
if (results.data.rows.length > 0) {
return substorage.get(results.data.rows[0].id);
}
throw new jIO.util.jIOError(
"HistoryStorage: cannot find object '" + id + "' (2)",
404
);
})
.push(function (result) {
if (result.op === "putAttachment") {
return substorage.getAttachment(result.timestamp, result.name);
//return result.blob;
}
throw new jIO.util.jIOError(
"HistoryStorage: cannot find object '" + id + "' (removed) (1)",
404
);
});
});
}; };
HistoryStorage.prototype.removeAttachment = function () {
return this._sub_storage.removeAttachment HistoryStorage.prototype.removeAttachment = function (id, name) {
.apply(this._sub_storage, arguments); var timestamp = unique_timestamp(),
metadata = {
// XXX: remove this attribute once query can sort_on id
timestamp: timestamp,
doc_id: id,
name: name,
op: "removeAttachment"
};
return this._sub_storage.put(timestamp, metadata);
}; };
HistoryStorage.prototype.repair = function () { HistoryStorage.prototype.repair = function () {
return this._sub_storage.repair.apply(this._sub_storage, arguments); return this._sub_storage.repair.apply(this._sub_storage, arguments);
...@@ -150,7 +281,9 @@ ...@@ -150,7 +281,9 @@
}; };
HistoryStorage.prototype.buildQuery = function (options) { HistoryStorage.prototype.buildQuery = function (options) {
if (options.message === "give all docs") {
return this._sub_storage.allDocs(options.opts);
}
if (options === undefined) { if (options === undefined) {
options = {}; options = {};
} }
...@@ -160,16 +293,20 @@ ...@@ -160,16 +293,20 @@
if (options.sort_on === undefined) { if (options.sort_on === undefined) {
options.sort_on = []; options.sort_on = [];
} }
if (options.select_list === undefined) {
options.select_list = [];
}
options.sort_on.push(["timestamp", "descending"]); options.sort_on.push(["timestamp", "descending"]);
options.query = jIO.QueryFactory.create(options.query); options.query = jIO.QueryFactory.create(options.query);
var meta_options = { var meta_options = {
// XXX: I don't believe it's currently possible to query on // XXX: I don't believe it's currently possible to query on
// sub-attributes so for now, we just use the inputted query, which // sub-attributes so for now, we just use the inputted query, which
// obviously will fail // obviously will fail
query: "", //query: "",//(op: put) OR (op: remove)",
// XXX: same here, we cannot sort correctly because we cannot access // XXX: same here, we cannot sort correctly because we cannot access
// attributes of doc // attributes of doc
query: "(op: remove) OR (op: put)",
sort_on: options.sort_on sort_on: options.sort_on
}, },
substorage = this._sub_storage, substorage = this._sub_storage,
...@@ -201,49 +338,61 @@ ...@@ -201,49 +338,61 @@
// Get all documents found in query // Get all documents found in query
.push(function (results) { .push(function (results) {
var promises = results.data.rows.map(function (data) { var promises = results.data.rows.map(function (data) {
return substorage.get(data.id); return substorage.get(data.id);
}); });
return RSVP.all(promises); return RSVP.all(promises);
}) })
.push(function (results) { .push(function (results) {
// Label all documents with their current revision status // Label all documents with their current revision status
var doc, var docum,
revision_tracker = {}, revision_tracker = {},
latest_rev_query; latest_rev_query,
results_reduced;
for (ind = 0; ind < results.length; ind += 1) { for (ind = 0; ind < results.length; ind += 1) {
doc = results[ind]; docum = results[ind];
if (revision_tracker.hasOwnProperty(doc.doc_id)) { if (revision_tracker.hasOwnProperty(docum.doc_id)) {
revision_tracker[doc.doc_id] += 1; revision_tracker[docum.doc_id] += 1;
} else { } else {
revision_tracker[doc.doc_id] = 0; revision_tracker[docum.doc_id] = 0;
} }
doc._REVISION = revision_tracker[doc.doc_id]; if (docum.op === "remove") {
docum.doc = {};
}
results[ind].doc._REVISION = revision_tracker[docum.doc_id];
results[ind].doc.op = docum.op;
} }
// Create a new query to only get non-removed revisions and abide by // Create a new query to only get non-removed revisions and abide by
// whatever the inputted query says // whatever the inputted query says
latest_rev_query = jIO.QueryFactory.create( latest_rev_query = jIO.QueryFactory.create(
"(_REVISION: >= 0) AND (NOT op: remove)" "(_REVISION: >= 0) AND (op: put)"
); );
if (rev_query) { if (rev_query) {
latest_rev_query.query_list[0] = options.query; latest_rev_query.query_list[0] = options.query;
} else { } else {
latest_rev_query.query_list[0] = jIO.QueryFactory.create( latest_rev_query.query_list[0] = jIO.QueryFactory.create(
"(_REVISION: =0)" "(_REVISION: = 0)"
); );
if (options.query.type === "simple" || if (options.query.type === "simple" ||
options.query.type === "complex") { options.query.type === "complex") {
latest_rev_query.query_list.push(options.query); latest_rev_query.query_list.push(options.query);
} }
} }
//return results
return results results_reduced = results
// Only return results which match latest_rev_query // Only return results which match latest_rev_query
.filter(function (doc) { .filter(function (docum) {
return latest_rev_query.match(doc); var filtered_res = latest_rev_query.match(docum.doc);
})
// Remove extra metadata used in revision query
delete docum.doc.op;
delete docum.doc._REVISION;
return filtered_res;
});
return results_reduced
// Only return the correct range of valid results specified by // Only return the correct range of valid results specified by
// options.limit // options.limit
...@@ -257,9 +406,18 @@ ...@@ -257,9 +406,18 @@
// Format results to be expected output of allDocs // Format results to be expected output of allDocs
.map(function (current_doc) { .map(function (current_doc) {
var val = {},
ind,
key;
for (ind = 0; ind < options.select_list.length; ind += 1) {
key = options.select_list[ind];
if (current_doc.doc.hasOwnProperty(key)) {
val[key] = current_doc.doc[key];
}
}
return { return {
doc: current_doc.doc, doc: current_doc.doc,
value: {}, value: val,
id: current_doc.doc_id id: current_doc.doc_id
}; };
}); });
......
test/jio.storage/historystorage.tests.js
View file @ f71fb6cb
...@@ -11,6 +11,16 @@ ...@@ -11,6 +11,16 @@
equal = QUnit.equal, equal = QUnit.equal,
module = QUnit.module; module = QUnit.module;
function putFullDoc(storage, id, doc, attachment_name, attachment) {
return storage.put(id, doc)
.push(function () {
return storage.putAttachment(
id,
attachment_name,
attachment
);
});
}
///////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////
// _revision parameter updating with RSVP all // _revision parameter updating with RSVP all
...@@ -279,6 +289,148 @@ ...@@ -279,6 +289,148 @@
.always(function () {start(); }); .always(function () {start(); });
}); });
**/ **/
/////////////////////////////////////////////////////////////////
// Attachments
/////////////////////////////////////////////////////////////////
module("HistoryStorage.attachments");
test("Testing proper adding/removing attachments",
function () {
stop();
expect(9);
// create storage of type "history" with memory as substorage
var jio = jIO.createJIO({
type: "history",
sub_storage: {
type: "uuid",
sub_storage: {
type: "memory"
//type: "indexeddb",
//database: dbname
}
}
}),
not_history = jIO.createJIO({
type: "query",
sub_storage: {
type: "uuid",
sub_storage: {
type: "memory"
//type: "indexeddb",
//database: dbname
}
}
}),
blob1 = new Blob(['a']),
blob2 = new Blob(['b']),
other_blob = new Blob(['1']);
jio.put("doc", {title: "foo0"})
.push(function () {
return jio.putAttachment("doc", "attached", blob1);
})
.push(function () {
return jio.putAttachment("doc", "attached", blob2);
})
.push(function () {
return jio.putAttachment("doc", "other_attached", other_blob);
})
.push(function () {
return jio.get("doc");
})
.push(function (result) {
deepEqual(result, {
title: "foo0"
}, "Get does not return any attachment information");
return jio.getAttachment("doc", "attached");
})
.push(function (result) {
deepEqual(result,
blob2,
"Return the attachment information with getAttachment"
);
return jio.getAttachment("doc", "attached_-0");
})
.push(function (result) {
deepEqual(result,
blob2,
"Return the attachment information with getAttachment"
);
return jio.getAttachment("doc", "attached_-1");
})
.push(function (result) {
deepEqual(result,
blob1,
"Return the attachment information with getAttachment"
);
return jio.getAttachment("doc", "attached_-2");
})
.push(function () {
ok(false, "This query should have thrown a 404 error");
},
function (error) {
deepEqual(error.status_code,
404,
"Error if you try to go back more revisions than what exists");
return jio.allAttachments("doc");
})
.push(function (results) {
deepEqual(results, {
"attached": {},
"other_attached": {}
}, "allAttachments works as expected.");
return jio.removeAttachment("doc", "attached");
})
.push(function () {
return jio.get("doc");
})
.push(function (result) {
deepEqual(result, {
title: "foo0"
}, "Get does not return any attachment information");
return jio.getAttachment("doc", "attached");
})
.push(function () {
ok(false, "This query should have thrown a 404 error");
},
function (error) {
deepEqual(error.status_code,
404,
"Removed attachments cannot be queried");
return jio.allAttachments("doc");
})
.push(function (results) {
deepEqual(results, {
"other_attached": {}
}, "allAttachments works as expected with a removed attachment");
})
.push(function () {
return jio.allDocs();
})
.push(function (results) {
equal(results.data.rows.length, 1, "Only one document in storage");
return jio.get(results.data.rows[0].id);
})
.push(function (result) {
deepEqual(result, {
"title": "foo0"
});
return not_history.allDocs();
})
.push(function (results) {
return RSVP.all(results.data.rows.map(function (d) {
return not_history.get(d.id);
}));
})
.fail(function (error) {
//console.log(error);
ok(false, error);
})
.always(function () {start(); });
});
///////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////
// Accessing older revisions // Accessing older revisions
...@@ -483,7 +635,7 @@ ...@@ -483,7 +635,7 @@
test("Testing retrieval of older revisions via allDocs calls", test("Testing retrieval of older revisions via allDocs calls",
function () { function () {
stop(); stop();
expect(37); expect(42);
// create storage of type "history" with memory as substorage // create storage of type "history" with memory as substorage
var jio = jIO.createJIO({ var jio = jIO.createJIO({
...@@ -527,7 +679,7 @@ ...@@ -527,7 +679,7 @@
.push(function (result) { .push(function (result) {
deepEqual(result, { deepEqual(result, {
"k": "v3" "k": "v3"
}); }, "One correct result.");
}) })
.push(function () { .push(function () {
return jio.allDocs({ return jio.allDocs({
...@@ -540,7 +692,7 @@ ...@@ -540,7 +692,7 @@
"Only one query returned with options.revision_limit == [1,1]"); "Only one query returned with options.revision_limit == [1,1]");
deepEqual(results.data.rows[0].doc, { deepEqual(results.data.rows[0].doc, {
"k": "v2" "k": "v2"
}); }, "One correct result.");
return jio.allDocs({ return jio.allDocs({
query: "_REVISION : =2" query: "_REVISION : =2"
}); });
...@@ -576,13 +728,14 @@ ...@@ -576,13 +728,14 @@
}); });
}) })
.push(function (results) { .push(function (results) {
equal(results.data.rows.length, 2); equal(results.data.rows.length, 2,
"Only retrieve two most recent revions");
deepEqual(results.data.rows[0].doc, { deepEqual(results.data.rows[0].doc, {
"k": "v3" "k": "v3"
}, "Only retrieve two most recent revions"); }, "First retrieved revision is correct");
deepEqual(results.data.rows[1].doc, { deepEqual(results.data.rows[1].doc, {
"k": "v2" "k": "v2"
}); }, "Second retrieved revision is correct");
}) })
.push(function () { .push(function () {
return jio.remove("doc"); return jio.remove("doc");
...@@ -660,10 +813,10 @@ ...@@ -660,10 +813,10 @@
}) })
.push(function (results) { .push(function (results) {
equal(results.data.rows.length, 1, equal(results.data.rows.length, 1,
"There is only one non-removed doc"); "There is only one non-removed doc.");
deepEqual(results.data.rows[0].doc, { deepEqual(results.data.rows[0].doc, {
"k2": "w1" "k2": "w1"
}); }, "Returned the one correct document.");
}) })
.push(function () { .push(function () {
return jio.remove("doc2"); return jio.remove("doc2");
...@@ -714,6 +867,91 @@ ...@@ -714,6 +867,91 @@
deepEqual(results.data.rows[3].doc, { deepEqual(results.data.rows[3].doc, {
"k": "v1" "k": "v1"
}, "Correct results with options.limit set"); }, "Correct results with options.limit set");
return jio.allDocs({
query: "_REVISION: = 1",
select_list: ["k"]
});
})
.push(function (results) {
equal(results.data.rows.length, 2);
deepEqual(results.data.rows[0].doc, {
"k2": "w1"
});
deepEqual(results.data.rows[0].value, {});
deepEqual(results.data.rows[1].doc, {
"k": "v3"
});
deepEqual(results.data.rows[1].value, {
"k": "v3"
});
})
.fail(function (error) {
//console.log(error);
ok(false, error);
})
.always(function () {start(); });
});
/////////////////////////////////////////////////////////////////
// Complex Queries
/////////////////////////////////////////////////////////////////
module("HistoryStorage.complex_queries");
test("Testing retrieval of older revisions via allDocs calls",
function () {
stop();
expect(3);
// create storage of type "history" with memory as substorage
var jio = jIO.createJIO({
type: "history",
sub_storage: {
type: "uuid",
sub_storage: {
type: "memory"
}
}
}),
doc = {
"modification_date": "a",
"portal_type": "Foo",
"title": "foo_module/1"
},
blob = new Blob(['a']);
putFullDoc(jio, "foo_module/1", doc, "data", blob)
.push(function () {
return jio.get("foo_module/1");
})
.push(function (result) {
deepEqual(result, {
"modification_date": "a",
"portal_type": "Foo",
"title": "foo_module/1"
}, "Can retrieve a document after attachment placed."
);
})
.push(function () {
return jio.allDocs({
query: "portal_type: Foo",
select_list: ["modification_date", "__id", "__id"],
sort_on: [["modification_date", "descending"],
["timestamp", "descending"],
["timestamp", "descending"]
]
});
})
.push(function (results) {
equal(results.data.rows.length, 1);
deepEqual(results.data.rows[0], {
doc: {
"modification_date": "a",
"portal_type": "Foo",
"title": "foo_module/1"
},
id: "foo_module/1",
value: {modification_date: "a"}
});
}) })
.fail(function (error) { .fail(function (error) {
//console.log(error); //console.log(error);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7