Skip to content

jio
jio
Commit c403f6bf authored by Bryan Kaperick's avatar Bryan Kaperick
Browse files
Options

Get and allDocs can now be used to access older revisions without changes to... 

Get and allDocs can now be used to access older revisions without changes to the API.  The approach to doing this with allDocs is very costly right now, but is being fixed for the next commit.
parent 7d25b5ce
Hide whitespace changes
Inline Side-by-side
Showing with 232 additions and 127 deletions
src/jio.storage/bryanstorage.js
View file @ c403f6bf
......@@ -28,18 +28,15 @@
});
}
BryanStorage.prototype.get = function (id_in, steps) {
BryanStorage.prototype.get = function (id_in) {
if (steps === undefined) {
steps = 0;
}
// Query to get the last edit made to this document
var substorage = this._sub_storage,
options = {
query: "doc_id: " + id_in,
sort_on: [["timestamp", "descending"]],
limit: [steps, 1]
limit: [0, 1]
};
return substorage.allDocs(options)
......@@ -52,7 +49,6 @@
404
);
})
.push(function (result) {
if (result.op === "put") {
return result.doc;
......@@ -61,6 +57,48 @@
"bryanstorage: cannot find object '" + id_in + "' (removed)",
404
);
// If no documents returned in first query, check if the id is encoding
// revision information
}, function () {
var steps,
steps_loc = id_in.lastIndexOf("_-");
// If revision signature is not in id_in, than return 404, since id
// is not found
if (steps_loc === -1) {
throw new jIO.util.jIOError(
"bryanstorage: cannot find object '" + id_in + "'",
404
);
}
// If revision signature is found, query storage based on this
steps = Number(id_in.slice(steps_loc + 2));
id_in = id_in.slice(0, steps_loc);
options = {
query: "doc_id: " + id_in,
sort_on: [["timestamp", "descending"]],
limit: [steps, 1]
};
return substorage.allDocs(options)
.push(function (results) {
if (results.data.rows.length > 0) {
return substorage.get(results.data.rows[0].id);
}
throw new jIO.util.jIOError(
"bryanstorage: cannot find object '" + id_in + "'",
404
);
})
.push(function (result) {
if (result.op === "put") {
return result.doc;
}
throw new jIO.util.jIOError(
"bryanstorage: cannot find object '" + id_in + "' (removed)",
404
);
});
});
};
......@@ -77,7 +115,6 @@
doc: data,
op: "put"
};
this._lastseen = timestamp;
return this._sub_storage.put(timestamp, metadata);
};
......@@ -152,34 +189,49 @@
};
BryanStorage.prototype.buildQuery = function (options) {
if (options === undefined) {
options = {};
}
if (options.sort_on === undefined) {
options.sort_on = [];
}
options.sort_on.push(["timestamp", "descending"]);
if (options.limit === undefined) {
options.limit = [0, -1];
if (options.query === undefined) {
options.query = "";
}
// Default behavior is to return only the latest revision of each document
if (options.revision_limit === undefined) {
options.revision_limit = [0, 1];
}
options.query = jIO.QueryFactory.create(options.query);
var meta_options = {
// XXX: I don't believe it's currently possible to query on sub-attributes
// so for now, we just use the inputted query, which obviously will fail
query: options.query,
// XXX: same here, we cannot sort correctly because we cannot access
// attributes of doc
sort_on: options.sort_on
},
// XXX: I don't believe it's currently possible to query on
// sub-attributes so for now, we just use the inputted query, which
// obviously will fail
query: "",
// XXX: same here, we cannot sort correctly because we cannot access
// attributes of doc
sort_on: [["timestamp", "descending"]]
},
substorage = this._sub_storage,
max_num_docs = options.limit[1],
first_doc = options.limit[0];
// Check if query involved _REVISION. If not, we will later place a
// (*) AND (_REVISION: =0) as the default handling of revisions
rev_query = false,
query_obj = options.query,
query_stack = [],
ind;
if (query_obj.hasOwnProperty("query_list")) {
query_stack.push(query_obj);
} else {
rev_query = (query_obj.key === "_REVISION");
}
while (query_stack.length > 0 && (!rev_query)) {
query_obj = query_stack.pop();
for (ind = 0; ind < query_obj.query_list.length; ind += 1) {
if (query_obj.query_list[ind].hasOwnProperty("query_list")) {
query_stack.push(query_obj.query_list[ind]);
} else if (query_obj.query_list[ind].key === "_REVISION") {
rev_query = true;
break;
}
}
}
return this._sub_storage.allDocs(meta_options)
......@@ -191,56 +243,66 @@
return RSVP.all(promises);
})
.push(function (results_array) {
var clean_data = [],
ind,
seen_docs = {},
current_doc,
counter = 0;
// Default behavior is to not limit the number of documents returned
if (max_num_docs === -1) {
max_num_docs = results_array.length;
}
for (ind = 0; ind < results_array.length; ind += 1) {
current_doc = results_array[ind];
// Initialize count of revisions
if (!seen_docs.hasOwnProperty(current_doc.doc_id)) {
seen_docs[current_doc.doc_id] = 0;
.push(function (results) {
// Label all documents with their current revision status
var doc,
revision_tracker = {},
promises;
for (ind = 0; ind < results.length; ind += 1) {
doc = results[ind];
if (revision_tracker.hasOwnProperty(doc.doc_id)) {
revision_tracker[doc.doc_id] += 1;
} else {
revision_tracker[doc.doc_id] = 0;
}
doc._REVISION = revision_tracker[doc.doc_id];
}
// If the latest version of this document has not yet been
// included in query result
if (options.revision_limit[0] <= seen_docs[current_doc.doc_id] &&
seen_docs[current_doc.doc_id] < options.revision_limit[0] +
options.revision_limit[1]) {
// If the latest edit was a put operation, add it to query
// results
if (current_doc.op === "put") {
if (counter >= first_doc) {
// Note the rev attribute added to the output data.
// This guarantees that `this.get(id, rev) === doc`
clean_data.push({
doc: current_doc.doc,
value: {},
id: current_doc.doc_id,
rev: seen_docs[current_doc.doc_id]
});
if (clean_data.length === max_num_docs) {
return clean_data;
}
}
counter += 1;
}
// There must be a faster way
promises = results.map(function (data) {
return substorage.put(data.timestamp, data);
});
return RSVP.all(promises);
})
.push(function () {
var latest_rev_query;
latest_rev_query = jIO.QueryFactory.create(
"(_REVISION: >= 0) AND (NOT op: remove)"
);
if (rev_query) {
latest_rev_query.query_list[0] = options.query;
} else {
latest_rev_query.query_list[0] = jIO.QueryFactory.create(
"(_REVISION: =0)"
);
if (options.query.type === "simple" ||
options.query.type === "complex") {
latest_rev_query.query_list.push(options.query);
}
// Keep track of how many times this doc_id has been seen
seen_docs[current_doc.doc_id] += 1;
}
// In passing results back to allDocs, formatting of query is handled
return clean_data;
// Build a query for final push
options.query = latest_rev_query;
if (options.sort_on === undefined) {
options.sort_on = [];
}
options.sort_on.push(["timestamp", "descending"]);
return substorage.allDocs(options);
})
.push(function (results) {
var promises = results.data.rows.map(function (data) {
return substorage.get(data.id);
});
return RSVP.all(promises);
})
.push(function (results) {
return results
.map(function (current_doc) {
return {
doc: current_doc.doc,
value: {},
id: current_doc.doc_id
};
});
});
};
......
test/jio.storage/bryanstorage.tests.js
View file @ c403f6bf
......@@ -287,7 +287,7 @@
test("Testing proper retrieval of older revisions of documents",
function () {
stop();
expect(12);
expect(18);
// create storage of type "bryan" with memory as substorage
var jio = jIO.createJIO({
......@@ -314,7 +314,7 @@
});
})
.push(function () {
return jio.get("doc", 0);
return jio.get("doc_-0");
})
.push(function (result) {
deepEqual(result, {
......@@ -325,7 +325,7 @@
return jio.put("doc", {"k1": "v1"});
})
.push(function () {
return jio.get("doc", 0);
return jio.get("doc_-0");
})
.push(function (result) {
deepEqual(result, {
......@@ -333,7 +333,7 @@
});
})
.push(function () {
return jio.get("doc", 1);
return jio.get("doc_-1");
})
.push(function (result) {
deepEqual(result, {
......@@ -359,19 +359,19 @@
deepEqual(result,
{"k4": "v4"},
"By default, .get returns latest revision");
return jio.get("doc", 0);
return jio.get("doc");
})
.push(function (result) {
deepEqual(result,
{"k4": "v4"},
".get returns latest revision with second input = 0");
return jio.get("doc", 1);
return jio.get("doc_-1");
})
.push(function (result) {
deepEqual(result,
{"k3": "v3"},
"Walk back one revision with second input = 1");
return jio.get("doc", 2);
return jio.get("doc_-2");
})
.push(function () {
ok(false, "This query should have thrown a 404 error");
......@@ -380,25 +380,25 @@
deepEqual(error.status_code,
404,
"Current state of document is 'removed'.");
return jio.get("doc", 3);
return jio.get("doc_-3");
})
.push(function (result) {
deepEqual(result,
{"k2": "v2"},
"Walk back three revisions with second input = 3");
return jio.get("doc", 4);
return jio.get("doc_-4");
})
.push(function (result) {
deepEqual(result,
{"k1": "v1"},
"Walk back four revisions with second input = 4");
return jio.get("doc", 5);
return jio.get("doc_-5");
})
.push(function (result) {
deepEqual(result,
{"k0": "v0"},
"Walk back five revisions with second input = 5");
return jio.get("doc", 6);
return jio.get("doc_-6");
})
.push(function () {
ok(false, "This query should have thrown a 404 error");
......@@ -408,6 +408,65 @@
404,
"There are only 5 previous states of this document");
})
// Adding documents with problematic doc_id's
.push(function () {
return jio.put("doc_-name", {
"key": "val0"
});
})
.push(function () {
return jio.put("document_-0", {
"key": "and val0"
});
})
.push(function () {
return jio.put("doc_-name", {
"key": "val1"
});
})
.push(function () {
return jio.get("doc_-name");
})
.push(function (result) {
deepEqual(result, {
"key": "val1"
});
return jio.get("doc_-name_-0");
})
.push(function (result) {
deepEqual(result, {
"key": "val1"
});
return jio.get("doc_-name_-1");
})
.push(function (result) {
deepEqual(result, {
"key": "val0"
});
return jio.get("document_-0");
})
.push(function (result) {
deepEqual(result, {
"key": "and val0"
});
return jio.get("document_-0_-0");
})
.push(function (result) {
deepEqual(result, {
"key": "and val0"
});
return jio.get("document_-0_-1");
})
.push(function () {
ok(false, "This query should have thrown a 404 error");
},
function (error) {
deepEqual(error.status_code,
404,
"Document does not have this many revisions.");
})
.fail(function (error) {
//console.log(error);
ok(false, error);
......@@ -416,27 +475,25 @@
});
/////////////////////////////////////////////////////////////////
// Accessing older revisions with two users
// Querying older revisions
/////////////////////////////////////////////////////////////////
module("bryanStorage.querying_old_revisions");
test("Testing retrieval of older revisions via allDocs calls",
function () {
stop();
expect(47);
expect(37);
// create storage of type "bryan" with memory as substorage
var dbname = "db-" + Date.now(),
jio = jIO.createJIO({
type: "bryan",
var jio = jIO.createJIO({
type: "bryan",
sub_storage: {
type: "uuid",
sub_storage: {
type: "uuid",
sub_storage: {
type: "indexeddb",
database: dbname
}
type: "memory"
}
});
}
});
jio.put("doc", {
"k": "v0"
})
......@@ -457,8 +514,7 @@
})
.push(function () {
return jio.allDocs({
query: "",
revision_limit: [0, 1]
query: "_REVISION : 0"
});
})
.push(function (results) {
......@@ -474,8 +530,7 @@
})
.push(function () {
return jio.allDocs({
query: "",
revision_limit: [1, 1]
query: "_REVISION : =1"
});
})
.push(function (results) {
......@@ -486,8 +541,7 @@
"k": "v2"
});
return jio.allDocs({
query: "",
revision_limit: [2, 1]
query: "_REVISION : =2"
});
})
.push(function (results) {
......@@ -498,8 +552,7 @@
"k": "v1"
});
return jio.allDocs({
query: "",
revision_limit: [3, 1]
query: "_REVISION : =3"
});
})
.push(function (results) {
......@@ -510,8 +563,7 @@
"k": "v0"
});
return jio.allDocs({
query: "",
revision_limit: [4, 1]
query: "_REVISION : =4"
});
})
.push(function (results) {
......@@ -519,7 +571,7 @@
})
.push(function () {
return jio.allDocs({
revision_limit: [0, 2]
query: "_REVISION: <= 1"
});
})
.push(function (results) {
......@@ -536,7 +588,7 @@
})
.push(function () {
return jio.allDocs({
query: "",
query: "NOT (_REVISION: >= 1)",
revision_limit: [0, 1]
});
})
......@@ -546,8 +598,7 @@
})
.push(function () {
return jio.allDocs({
query: "",
revision_limit: [1, 3]
query: "(_REVISION: >= 1) AND (_REVISION: <= 3)"
});
})
.push(function (results) {
......@@ -569,7 +620,7 @@
})
.push(function () {
return jio.allDocs({
revision_limit: [1, 3]
query: "(_REVISION: >0) AND (_REVISION: <= 3)"
});
})
.push(function (results) {
......@@ -586,7 +637,7 @@
})
.push(function () {
return jio.allDocs({
revision_limit: [0, 2]
query: "(_REVISION: = 0) OR (_REVISION: = 1)"
});
})
.push(function (results) {
......@@ -609,7 +660,6 @@
.push(function (results) {
equal(results.data.rows.length, 1,
"There is only one non-removed doc");
equal(results.data.rows[0].rev, 0);
deepEqual(results.data.rows[0].doc, {
"k2": "w1"
});
......@@ -619,16 +669,13 @@
})
.push(function () {
return jio.allDocs({
revision_limit: [0, 4]
query:
"_REVISION: 0 OR _REVISION: 1 OR " +
"(_REVISION: >= 2 AND _REVISION: <= 3)"
});
})
.push(function (results) {
equal(results.data.rows.length, 5);
equal(results.data.rows[0].rev, 1, "Rev parameter is correct");
equal(results.data.rows[1].rev, 2, "Rev parameter is correct");
equal(results.data.rows[2].rev, 1, "Rev parameter is correct");
equal(results.data.rows[3].rev, 2, "Rev parameter is correct");
equal(results.data.rows[4].rev, 3, "Rev parameter is correct");
deepEqual(results.data.rows[0].doc, {
"k2": "w1"
});
......@@ -647,17 +694,13 @@
})
.push(function () {
return jio.allDocs({
limit: [1, 4],
revision_limit: [0, 4]
query: "_REVISION: <= 3",
limit: [1, 4]
});
})
.push(function (results) {
equal(results.data.rows.length, 4,
"Correct number of results with options.limit set");
equal(results.data.rows[0].rev, 2, "Rev parameter is correct");
equal(results.data.rows[1].rev, 1, "Rev parameter is correct");
equal(results.data.rows[2].rev, 2, "Rev parameter is correct");
equal(results.data.rows[3].rev, 3, "Rev parameter is correct");
"Correct number of results with optins.limit set");
deepEqual(results.data.rows[0].doc, {
"k2": "w0"
}, "Correct results with options.limit set");
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7