Skip to content

J
jio
Commit a1fa77a9 authored by Bryan Kaperick's avatar Bryan Kaperick
Browse files
Options

Added packing function for removing old data without corruption.

parent 86942633
Show whitespace changes
Inline Side-by-side
Showing with 569 additions and 1 deletion
src/jio.storage/historystorage.js
View file @ a1fa77a9
...@@ -15,6 +15,71 @@ ...@@ -15,6 +15,71 @@
return timestamp + "-" + uuid; return timestamp + "-" + uuid;
} }
function removeOldRevs(
substorage,
results,
keepDoc
) {
var ind,
promises = [],
seen = {},
docum,
log,
start_ind,
new_promises,
doc_id,
checkIsId,
removeDoc;
for (ind = 0; ind < results.data.rows.length; ind += 1) {
docum = results.data.rows[ind];
// Count the number of revisions of each document, and delete older
// ones.
if (!seen.hasOwnProperty(docum.value.doc_id)) {
seen[docum.value.doc_id] = {count: 0};
}
log = seen[docum.value.doc_id];
log.count += 1;
//log.id = docum.id;
// Record the index of the most recent edit that is before the cutoff
if (!log.hasOwnProperty("s") && !keepDoc({doc: docum, log: log})) {
log.s = ind;
}
// Record the index of the most recent put or remove
if ((!log.hasOwnProperty("pr")) &&
(docum.value.op === "put" || docum.value.op === "remove")) {
log.pr = ind;
log.final = ind;
}
if ((docum.op === "putAttachment" || docum.op === "removeAttachment") &&
log.hasOwnProperty(docum.name) &&
!log[docum.name].hasOwnProperty("prA")) {
log[docum.name].prA = ind;
log.final = ind;
}
}
checkIsId = function (d) {
return d.value.doc_id === doc_id;
};
removeDoc = function (d) {
return substorage.remove(d.id);
};
for (doc_id in seen) {
if (seen.hasOwnProperty(doc_id)) {
log = seen[doc_id];
start_ind = Math.max(log.s, log.final + 1);
new_promises = results.data.rows
.slice(start_ind)
.filter(checkIsId)
.map(removeDoc);
promises = promises.concat(new_promises);
}
}
return RSVP.all(promises);
}
function throwCantFindError(id) { function throwCantFindError(id) {
throw new jIO.util.jIOError( throw new jIO.util.jIOError(
"HistoryStorage: cannot find object '" + id + "'", "HistoryStorage: cannot find object '" + id + "'",
...@@ -42,6 +107,34 @@ ...@@ -42,6 +107,34 @@
} else { } else {
this._include_revisions = false; this._include_revisions = false;
} }
var substorage = this._sub_storage;
this.packOldRevisions = function (save_info) {
/**
save_info has this form:
{
keep_latest_num: 10,
keep_active_revs: timestamp
}
keep_latest_num = x: keep at most the x latest copies of each unique doc
keep_active_revs = x: throw away all outdated revisions from before x
**/
var options = {
sort_on: [["timestamp", "descending"]],
select_list: ["doc", "doc_id", "op"]
},
keep_fixed_num = save_info.hasOwnProperty("keep_latest_num");
return substorage.allDocs(options)
.push(function (results) {
if (keep_fixed_num) {
return removeOldRevs(substorage, results, function (data) {
return data.log.count <= save_info.keep_latest_num;
});
}
return removeOldRevs(substorage, results, function (data) {
return data.doc.id > save_info.keep_active_revs;
});
});
};
} }
HistoryStorage.prototype.get = function (id_in) { HistoryStorage.prototype.get = function (id_in) {
...@@ -330,14 +423,15 @@ ...@@ -330,14 +423,15 @@
}; };
return this._sub_storage.put(timestamp, metadata); return this._sub_storage.put(timestamp, metadata);
}; };
HistoryStorage.prototype.repair = function () { HistoryStorage.prototype.repair = function () {
return this._sub_storage.repair.apply(this._sub_storage, arguments); return this._sub_storage.repair.apply(this._sub_storage, arguments);
}; };
HistoryStorage.prototype.hasCapacity = function (name) { HistoryStorage.prototype.hasCapacity = function (name) {
return name === 'list' || name === 'include'; return name === 'list' || name === 'include';
}; };
HistoryStorage.prototype.buildQuery = function (options) { HistoryStorage.prototype.buildQuery = function (options) {
// Set default values // Set default values
if (options === undefined) {options = {}; } if (options === undefined) {options = {}; }
...@@ -442,6 +536,7 @@ ...@@ -442,6 +536,7 @@
// this attachment, then don't include attachment in query // this attachment, then don't include attachment in query
return false; return false;
} }
docum.value.doc = {};
} }
} }
} }
......
test/jio.storage/historystorage.tests.js
View file @ a1fa77a9
...@@ -2586,4 +2586,477 @@ ...@@ -2586,4 +2586,477 @@
}) })
.always(function () {start(); }); .always(function () {start(); });
}); });
module("HistoryStorage.pack", {
setup: function () {
// create storage of type "history" with memory as substorage
var dbname = "db_" + Date.now();
this.jio = jIO.createJIO({
type: "uuid",
sub_storage: {
type: "query",
sub_storage: {
type: "history",
sub_storage: {
type: "query",
sub_storage: {
type: "indexeddb",
database: dbname
}
}
}
}
});
this.history = jIO.createJIO({
type: "uuid",
sub_storage: {
type: "query",
sub_storage: {
type: "history",
include_revisions: true,
sub_storage: {
type: "query",
sub_storage: {
type: "indexeddb",
database: dbname
}
}
}
}
});
this.not_history = jIO.createJIO({
type: "query",
sub_storage: {
type: "uuid",
sub_storage: {
type: "indexeddb",
database: dbname
}
}
});
this.blob = new Blob(['a']);
}
});
test("Verifying pack works with keep_latest_num",
function () {
stop();
expect(2);
var jio = this.jio,
not_history = this.not_history;
return jio.put("doc_a", {title: "rev"})
.push(function () {
return jio.put("doc_a", {title: "rev0"});
})
.push(function () {
return jio.put("doc_a", {title: "rev1"});
})
.push(function () {
return jio.put("doc_b", {title: "data"});
})
.push(function () {
return jio.put("doc_b", {title: "data0"});
})
.push(function () {
return jio.put("doc_a", {title: "rev2"});
})
.push(function () {
return jio.put("doc_b", {title: "data1"});
})
.push(function () {
return jio.put("doc_b", {title: "data2"});
})
.push(function () {
return jio.__storage._sub_storage.__storage._sub_storage
.__storage.packOldRevisions({
keep_latest_num: 2
});
})
.push(function () {
return not_history.allDocs({
sort_on: [["timestamp", "descending"]],
select_list: ["doc", "doc_id", "timestamp", "op"]
});
})
.push(function (results) {
equal(results.data.total_rows, 4, "Correct amount of results");
deepEqual(results.data.rows, [
{
doc: {},
id: results.data.rows[0].id,
value: {
doc: {title: "data2"},
doc_id: "doc_b",
timestamp: results.data.rows[0].id,
op: "put"
}
},
{
doc: {},
id: results.data.rows[1].id,
value: {
doc: {title: "data1"},
doc_id: "doc_b",
timestamp: results.data.rows[1].id,
op: "put"
}
},
{
doc: {},
id: results.data.rows[2].id,
value: {
doc: {title: "rev2"},
doc_id: "doc_a",
timestamp: results.data.rows[2].id,
op: "put"
}
},
{
doc: {},
id: results.data.rows[3].id,
value: {
doc: {title: "rev1"},
doc_id: "doc_a",
timestamp: results.data.rows[3].id,
op: "put"
}
}
],
"Keep the correct documents after pack");
})
.fail(function (error) {
//console.log(error);
ok(false, error);
})
.always(function () {start(); });
});
test("Verifying pack works with fixed timestamp",
function () {
stop();
expect(2);
var jio = this.jio,
not_history = this.not_history,
timestamp;
return jio.allDocs()
.push(function () {
return RSVP.all([
jio.put("doc_a", {title: "old_rev0"}),
jio.put("doc_a", {title: "old_rev1"}),
jio.put("doc_a", {title: "old_rev2"}),
jio.put("doc_b", {title: "old_data0"}),
jio.put("doc_b", {title: "old_data1"}),
jio.put("doc_b", {title: "old_data2"}),
jio.put("doc_c", {title: "latest_bar"})
]);
})
.push(function () {
return not_history.allDocs({sort_on: [["timestamp", "descending"]]});
})
.push(function (results) {
timestamp = results.data.rows[0].id;
return jio.put("doc_a", {title: "latest_rev"});
})
.push(function () {
return jio.put("doc_b", {title: "latest_data"});
})
.push(function () {
return jio.__storage._sub_storage.__storage._sub_storage
.__storage.packOldRevisions({
keep_active_revs: timestamp
});
})
.push(function () {
return not_history.allDocs({
sort_on: [["timestamp", "descending"]],
select_list: ["doc", "doc_id", "timestamp"]
});
})
.push(function (results) {
equal(results.data.total_rows, 3, "Correct amount of results");
deepEqual(results.data.rows, [
{
doc: {},
id: results.data.rows[0].id,
value: {
doc: {title: "latest_data"},
doc_id: "doc_b",
timestamp: results.data.rows[0].id
}
},
{
doc: {},
id: results.data.rows[1].id,
value: {
doc: {title: "latest_rev"},
doc_id: "doc_a",
timestamp: results.data.rows[1].id
}
},
{
doc: {},
id: results.data.rows[2].id,
value: {
doc: {title: "latest_bar"},
doc_id: "doc_c",
timestamp: results.data.rows[2].id
}
}
],
"Keep the correct documents after pack");
})
.fail(function (error) {
//console.log(error);
ok(false, error);
})
.always(function () {start(); });
});
test("Verifying pack works with fixed timestamp and more complex operations",
function () {
stop();
expect(2);
var jio = this.jio,
not_history = this.not_history,
timestamp;
return jio.allDocs()
.push(function () {
return RSVP.all([
jio.put("doc_a", {title: "old_rev0"}),
jio.put("doc_a", {title: "old_rev1"}),
jio.put("doc_a", {title: "old_rev2"}),
jio.put("doc_b", {title: "latest_data"})
]);
})
.push(function () {
return jio.allDocs({sort_on: [["timestamp", "descending"]]});
})
.push(function (results) {
timestamp = results.data.rows[0].id;
return jio.remove("doc_a");
})
.push(function () {
return jio.__storage._sub_storage.__storage._sub_storage
.__storage.packOldRevisions({
keep_active_revs: timestamp
});
})
.push(function () {
return not_history.allDocs({
sort_on: [["timestamp", "descending"]],
select_list: ["doc", "doc_id", "timestamp", "op"]
});
})
.push(function (results) {
deepEqual(results.data.rows, [
{
doc: {},
id: results.data.rows[0].id,
value: {
op: "remove",
doc_id: "doc_a",
timestamp: results.data.rows[0].id
}
},
{
doc: {},
id: results.data.rows[1].id,
value: {
doc: {title: "latest_data"},
doc_id: "doc_b",
op: "put",
timestamp: results.data.rows[1].id
}
}
],
"Keep the correct documents after pack");
})
.push(function () {
return jio.allDocs({
sort_on: [["timestamp", "descending"]],
select_list: ["title"]
});
})
.push(function (results) {
deepEqual(results.data.rows, [
{
doc: {},
id: "doc_b",
value: {title: "latest_data"}
}
],
"Memory not corrupted by pack without include_revisions");
})
.fail(function (error) {
//console.log(error);
ok(false, error);
})
.always(function () {start(); });
});
test("Verifying pack works with fixed timestamp and more complex operations",
function () {
stop();
expect(2);
var jio = this.jio,
not_history = this.not_history,
timestamp;
return jio.allDocs()
.push(function () {
return RSVP.all([
jio.put("doc_a", {title: "old_rev0"}),
jio.put("doc_a", {title: "old_rev1"}),
jio.put("doc_a", {title: "old_rev2"}),
jio.put("doc_b", {title: "latest_data"})
]);
})
.push(function () {
return jio.allDocs({sort_on: [["timestamp", "descending"]]});
})
.push(function (results) {
timestamp = results.data.rows[0].id;
return jio.remove("doc_a");
})
.push(function () {
return jio.__storage._sub_storage.__storage._sub_storage
.__storage.packOldRevisions({
keep_active_revs: timestamp
});
})
.push(function () {
return not_history.allDocs({
sort_on: [["timestamp", "descending"]],
select_list: ["doc", "doc_id", "timestamp", "op"]
});
})
.push(function (results) {
deepEqual(results.data.rows, [
{
doc: {},
id: results.data.rows[0].id,
value: {
op: "remove",
doc_id: "doc_a",
timestamp: results.data.rows[0].id
}
},
{
doc: {},
id: results.data.rows[1].id,
value: {
doc: {title: "latest_data"},
doc_id: "doc_b",
op: "put",
timestamp: results.data.rows[1].id
}
}
],
"Keep the correct documents after pack");
})
.push(function () {
return jio.allDocs({
sort_on: [["timestamp", "descending"]],
select_list: ["title"]
});
})
.push(function (results) {
deepEqual(results.data.rows, [
{
doc: {},
id: "doc_b",
value: {title: "latest_data"}
}
],
"Memory not corrupted by pack without include_revisions");
})
.fail(function (error) {
//console.log(error);
ok(false, error);
})
.always(function () {start(); });
});
test("Verifying pack works with fixed timestamp and more complex operations",
function () {
stop();
expect(2);
var jio = this.jio,
not_history = this.not_history,
timestamp,
blob = this.blob;
return jio.allDocs()
.push(function () {
return RSVP.all([
jio.put("doc_a", {title: "old_rev0"}),
jio.putAttachment("doc_a", "attach_aa", blob),
jio.put("doc_b", {title: "latest_data"})
]);
})
.push(function () {
return jio.allDocs({sort_on: [["timestamp", "descending"]]});
})
.push(function (results) {
timestamp = results.data.rows[0].id;
return jio.remove("doc_a");
})
.push(function () {
return jio.__storage._sub_storage.__storage._sub_storage
.__storage.packOldRevisions({
keep_active_revs: timestamp
});
})
.push(function () {
return not_history.allDocs({
sort_on: [["timestamp", "descending"]],
select_list: ["doc", "doc_id", "timestamp", "op"]
});
})
.push(function (results) {
deepEqual(results.data.rows, [
{
doc: {},
id: results.data.rows[0].id,
value: {
op: "remove",
doc_id: "doc_a",
timestamp: results.data.rows[0].id
}
},
{
doc: {},
id: results.data.rows[1].id,
value: {
doc: {title: "latest_data"},
doc_id: "doc_b",
op: "put",
timestamp: results.data.rows[1].id
}
}
],
"Keep the correct documents after pack");
})
.push(function () {
return jio.allDocs({
sort_on: [["timestamp", "descending"]],
select_list: ["title"]
});
})
.push(function (results) {
deepEqual(results.data.rows, [
{
doc: {},
id: "doc_b",
value: {title: "latest_data"}
}
],
"Memory not corrupted by pack without include_revisions");
})
.fail(function (error) {
//console.log(error);
ok(false, error);
})
.always(function () {start(); });
});
}(jIO, RSVP, Blob, QUnit)); }(jIO, RSVP, Blob, QUnit));
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7