Skip to content

J
jio
Commit 54e878be authored by root's avatar root
Browse files
Options

wip

parent f641ef58
Show whitespace changes
Inline Side-by-side
Showing with 1241 additions and 199 deletions
bryan_tests/ex1.html 0 → 100644
View file @ 54e878be
<html>
<head>
<script type="text/javascript" src="../node_modules/rsvp/dist/rsvp-2.0.4.js">
</script>
<script type="text/javascript" src="../dist/jio-latest.js"></script>
</head>
<body> <p>jio example, see console</p>
<script type="text/javascript">
(
function (jio) {
var storage = jIO.createJIO({"type": "indexeddb", "database": "foo"});
console.log(storage);
function foo(message) {
return storage.put("doc1", message)
.push(function () { return storage.put("doc2", {"hello again":"hi again"})})
.push(function (result) {
console.log(result);
return storage.get("doc1");
})
.push(function (result) {console.log(result) })
.push(undefined, function (error) {
console.log(error);});
}
return foo({"hello":"hi"});
}(jIO)
);
</script>
<p> more text </p>
</body>
</html>
bryan_tests/ex2.html 0 → 100644
View file @ 54e878be
<html>
<head>
<script type="text/javascript" src="../node_modules/rsvp/dist/rsvp-2.0.4.js">
</script>
<script type="text/javascript" src="../dist/jio-latest.js"></script>
</head>
<body> <p>jio example, see console</p>
<script type="text/javascript">
(function (jIO) { // create a new jIO storage
var jio_instance = jIO.createJIO({type:"memory"});
<!-- var jio_instance = jIO.createJIO({type: "indexeddb", "database": "foo"}); -->
// post the metadata for "myVideo"
return jio_instance.put("document", {
title: "My Video",
type: "MovingImage",
format: "video/ogg",
description: "Images Compilation" })
// post a thumbnail attachment
.push(function () {
return jio_instance.putAttachment(
"document",
"thumbnail",
new Blob(["img.png"],{type: "image/jpeg"})
)
})
// post video attachment
.push(function () {
return jio_instance.putAttachment(
"document",
"video",
new Blob(["vid.txt"], {type: "video/ogg"})
)
})
// catch any errors and throw
.push(undefined, function(error) {
console.log(error); throw error;
});
}(jIO));
</script>
<p> more text </p>
</body>
</html>
bryan_tests/ex2.js 0 → 100644
View file @ 54e878be
(function (jIO) { // create a new jIO storage
var jio_instance = jIO.createJIO({type: "local"});
// post the metadata for "myVideo"
return jio_instance.put("document", {
title: "My Video",
type: "MovingImage",
format: "video/ogg",
description: "Images Compilation" })
// post a thumbnail attachment
.push(function () {
return jio_instance.putAttachment(
"document",
"thumbnail",
new Blob([my_image],
{type: "image/jpeg"})
})
// post video attachment
.push(function () {
return jio_instance.putAttachment(
"document",
"video",
new Blob([my_video],
{type: "video/ogg"})
})
// catch any errors and throw
.push(undefined, function(error) {
console.log(error); throw error;
});
}(jIO));
bryan_tests/ex3.html 0 → 100644
View file @ 54e878be
<html>
<head>
<script type="text/javascript" src="../node_modules/rsvp/dist/rsvp-2.0.4.js">
</script>
<script type="text/javascript" src="../dist/jio-latest.js"></script>
</head>
<body> <p>Retrieving local data:</p>
<script type="text/javascript">
(function (jIO) {
// create a new jIO storage
var jio_instance = jIO.createJIO({type: "indexeddb", "database":"foo"});
// post the metadata for "myVideo"
return jio_instance.put("document", {
title: "My Video"
})
// post a thumbnail attachment
.push(function () {
return jio_instance.putAttachment(
"document",
"thumbnail",
new Blob(["img.jpg"], {type: "image/jpeg"}))
})
.push(function () {
return jio_instance.getAttachment(
"document",
"thumbnail",
{format:"text"})
})
.push(function (res) {console.log(res)})
// catch any errors and throw
.push(undefined, function(error) {
console.log(error);
throw error;
});
}(jIO));
</script>
<p> more text </p>
</body>
</html>
bryan_tests/img.jpg 0 → 100644
View file @ 54e878be
test test image
bryan_tests/revision_test.html 0 → 100644
View file @ 54e878be
<html>
<head>
<script type="text/javascript" src="../node_modules/rsvp/dist/rsvp-2.0.4.js">
</script>
<script type="text/javascript" src="../dist/jio-latest.js"></script>
</head>
<body> <p>jio example, see console</p>
<script type="text/javascript">
(function (jIO) {
var jio = jIO.createJIO({
type: "bryan",
sub_storage: {
type: "uuid",
sub_storage: {
//type: "memory"
type: "indexeddb",
database: "testdb_2"
}
}
}),
not_bryan = jIO.createJIO({
type: "uuid",
sub_storage: {
//type: "memory"
type: "indexeddb",
database: "testdb_2"
}
});
jio.put("A", {
"k": "v0"
})
.push(function () {
jio.put("A", {
"k": "v1"
})
})
.push(function () {
jio.put("A", {
"k": "v2"
})
})
.push(function () {
jio.put("A", {
"k": "v3"
})
})
.push(function () {
return not_bryan.buildQuery({
query: ""
});
})
.push(function (results) {
console.log(results);
},
function (err) {
console.log(err);
});
}(jIO));
</script>
<p> more text </p>
</body>
</html>
\ No newline at end of file
bryan_tests/vid.txt 0 → 100644
View file @ 54e878be
test test video test
examples/bu_scenario_officejs.js 0 → 100644
View file @ 54e878be
/*global Blob*/
/*jslint nomen: true, maxlen: 80*/
(function (QUnit, jIO, Blob) {
"use strict";
var test = QUnit.test,
// equal = QUnit.equal,
expect = QUnit.expect,
ok = QUnit.ok,
stop = QUnit.stop,
start = QUnit.start,
deepEqual = QUnit.deepEqual,
module = QUnit.module,
ATTACHMENT = 'data',
i,
name_list = ['get', 'post', 'put', 'buildQuery',
'putAttachment', 'getAttachment', 'allAttachments'];
///////////////////////////////////////////////////////
// Fake Storage
///////////////////////////////////////////////////////
function resetCount(count) {
for (i = 0; i < name_list.length; i += 1) {
count[name_list[i]] = 0;
}
}
function MockStorage(spec) {
this._erp5_storage = jIO.createJIO({
type: "erp5",
url: "http://example.org"
});
this._sub_storage = jIO.createJIO({
type: "query",
sub_storage: {
type: "uuid",
sub_storage: {
type: "memory"
}
}
});
this._options = spec.options;
resetCount(spec.options.count);
}
function mockFunction(name) {
MockStorage.prototype[name] = function () {
this._options.count[name] += 1;
if (this._options.mock.hasOwnProperty(name)) {
return this._options.mock[name].apply(this, arguments);
}
return this._sub_storage[name].apply(this._sub_storage, arguments);
};
}
for (i = 0; i < name_list.length; i += 1) {
mockFunction(name_list[i]);
}
MockStorage.prototype.hasCapacity = function (name) {
return this._erp5_storage.hasCapacity(name);
};
jIO.addStorage('mock', MockStorage);
///////////////////////////////////////////////////////
// Helpers
///////////////////////////////////////////////////////
function putFullDoc(storage, id, doc, attachment) {
return storage.put(id, doc)
.push(function () {
return storage.putAttachment(
id,
ATTACHMENT,
attachment
);
});
}
function equalStorage(storage, doc_tuple_list) {
return storage.allDocs()
.push(function (result) {
var i,
promise_list = [];
for (i = 0; i < result.data.rows.length; i += 1) {
promise_list.push(RSVP.all([
result.data.rows[i].id,
storage.get(result.data.rows[i].id),
storage.getAttachment(result.data.rows[i].id, ATTACHMENT)
]));
}
return RSVP.all(promise_list);
})
.push(function (result) {
deepEqual(result, doc_tuple_list, 'Storage content');
});
}
function isEmptyStorage(storage) {
return equalStorage(storage, []);
}
function equalRemoteStorageCallCount(mock_count, expected_count) {
for (i = 0; i < name_list.length; i += 1) {
if (!expected_count.hasOwnProperty(name_list[i])) {
expected_count[name_list[i]] = 0;
}
}
deepEqual(mock_count, expected_count, 'Expected method call count');
}
///////////////////////////////////////////////////////
// Module
///////////////////////////////////////////////////////
module("scenario_officejs", {
setup: function () {
this.remote_mock_options = {
mock: {
remove: function () {
throw new Error('remove not supported');
},
removeAttachment: function () {
throw new Error('removeAttachment not supported');
},
allAttachments: function () {
return {data: null};
},
post: function (doc) {
var context = this;
return this._sub_storage.post(doc)
.push(function (post_id) {
context._options.last_post_id = post_id;
return post_id;
});
}
},
count: {}
};
this.jio = jIO.createJIO({
type: "replicate",
query: {
query: 'portal_type:"Foo"',
sort_on: [["modification_date", "descending"]]
},
signature_hash_key: 'modification_date',
use_remote_post: true,
conflict_handling: 1,
check_local_attachment_modification: true,
check_local_attachment_creation: true,
check_remote_attachment_modification: true,
check_remote_attachment_creation: true,
check_remote_attachment_deletion: true,
check_local_deletion: false,
parallel_operation_amount: 10,
parallel_operation_attachment_amount: 10,
local_sub_storage: {
type: "query",
sub_storage: {
type: "uuid",
sub_storage: {
type: "memory"
}
}
},
signature_sub_storage: {
type: "query",
sub_storage: {
type: "memory"
}
},
remote_sub_storage: {
type: "saferepair",
sub_storage: {
type: "mock",
options: this.remote_mock_options
}
}
});
}
});
///////////////////////////////////////////////////////
// Do nothing cases
///////////////////////////////////////////////////////
test("empty: nothing to do", function () {
expect(2);
stop();
var test = this;
this.jio.repair()
.then(function () {
return RSVP.all([
isEmptyStorage(test.jio),
equalRemoteStorageCallCount(
test.remote_mock_options.count,
{buildQuery: 1}
)
]);
})
.fail(function (error) {
ok(false, error);
})
.always(function () {
start();
});
});
test("already synced: nothing to do", function () {
expect(2);
stop();
var test = this,
doc_id = 'foo_module/1',
doc = {title: doc_id, portal_type: "Foo", modification_date: 'a'},
blob = new Blob(['a']);
putFullDoc(this.jio.__storage._remote_sub_storage, doc_id, doc, blob)
.then(function () {
return test.jio.repair();
})
.then(function () {
resetCount(test.remote_mock_options.count);
return test.jio.repair();
})
.then(function () {
return RSVP.all([
equalStorage(test.jio, [[doc_id, doc, blob]]),
equalRemoteStorageCallCount(
test.remote_mock_options.count,
{buildQuery: 1}
)
]);
})
.fail(function (error) {
ok(false, error);
})
.always(function () {
start();
});
});
///////////////////////////////////////////////////////
// Remote creation
///////////////////////////////////////////////////////
test("remote document creation: copy", function () {
expect(2);
stop();
var test = this,
doc_id = 'foo_module/1',
doc = {title: doc_id, portal_type: "Foo", modification_date: 'a'},
blob = new Blob(['a']);
putFullDoc(this.jio.__storage._remote_sub_storage, doc_id, doc, blob)
.then(function () {
resetCount(test.remote_mock_options.count);
return test.jio.repair();
})
.then(function () {
return RSVP.all([
equalStorage(test.jio, [[doc_id, doc, blob]]),
equalRemoteStorageCallCount(
test.remote_mock_options.count,
{buildQuery: 1, get: 1, getAttachment: 1, allAttachments: 1}
)
]);
})
.fail(function (error) {
ok(false, error);
})
.always(function () {
start();
});
});
///////////////////////////////////////////////////////
// Remote modification
///////////////////////////////////////////////////////
test("remote document modification: copy", function () {
expect(2);
stop();
var test = this,
doc_id = 'foo_module/1',
doc = {title: doc_id, portal_type: "Foo", modification_date: 'a'},
doc2 = {title: doc_id + 'a', portal_type: "Foo", modification_date: 'b'},
blob = new Blob(['a']),
blob2 = new Blob(['b']);
putFullDoc(this.jio.__storage._remote_sub_storage, doc_id, doc, blob)
.then(function () {
return test.jio.repair();
})
.then(function () {
return putFullDoc(test.jio.__storage._remote_sub_storage, doc_id, doc2,
blob2);
})
.then(function () {
resetCount(test.remote_mock_options.count);
return test.jio.repair();
})
.then(function () {
return RSVP.all([
equalStorage(test.jio, [[doc_id, doc2, blob2]]),
equalRemoteStorageCallCount(
test.remote_mock_options.count,
{buildQuery: 1, get: 1, getAttachment: 1, allAttachments: 1}
)
]);
})
.fail(function (error) {
ok(false, error);
})
.always(function () {
start();
});
});
///////////////////////////////////////////////////////
// Remote hide
///////////////////////////////////////////////////////
test("remote document deletion: delete", function () {
expect(2);
stop();
var test = this,
doc_id = 'foo_module/1',
doc = {title: doc_id, portal_type: "Foo", modification_date: 'a'},
blob = new Blob(['a']);
putFullDoc(this.jio.__storage._remote_sub_storage, doc_id, doc, blob)
.then(function () {
return test.jio.repair();
})
.then(function () {
test.remote_mock_options.mock.buildQuery = function () {
return [];
};
resetCount(test.remote_mock_options.count);
return test.jio.repair();
})
.then(function () {
return RSVP.all([
isEmptyStorage(test.jio),
equalRemoteStorageCallCount(
test.remote_mock_options.count,
{buildQuery: 1}
)
]);
})
.fail(function (error) {
ok(false, error);
})
.always(function () {
start();
});
});
///////////////////////////////////////////////////////
// Local creation
///////////////////////////////////////////////////////
test("local document creation: copy", function () {
expect(3);
stop();
var test = this,
doc_id = 'abc',
doc = {title: doc_id, portal_type: "Foo", modification_date: 'a'},
blob = new Blob(['a']);
putFullDoc(this.jio, doc_id, doc, blob)
.then(function () {
resetCount(test.remote_mock_options.count);
return test.jio.repair();
})
.then(function () {
return RSVP.all([
equalStorage(
test.jio,
[[test.remote_mock_options.last_post_id, doc, blob]]
),
equalRemoteStorageCallCount(
test.remote_mock_options.count,
{buildQuery: 1, post: 1, putAttachment: 1, allAttachments: 1}
)
]);
})
.then(function () {
return equalStorage(
test.jio.__storage._remote_sub_storage,
[[test.remote_mock_options.last_post_id, doc, blob]]
);
})
.fail(function (error) {
ok(false, error);
})
.always(function () {
start();
});
});
///////////////////////////////////////////////////////
// Local modification
///////////////////////////////////////////////////////
test("local document modification: copy", function () {
expect(2);
stop();
var test = this,
doc_id = 'foo_module/1',
doc = {title: doc_id, portal_type: "Foo", modification_date: 'a'},
doc2 = {title: doc_id + 'a', portal_type: "Foo", modification_date: 'b'},
blob = new Blob(['a']),
blob2 = new Blob(['b']),
last_id;
putFullDoc(this.jio, doc_id, doc, blob)
.then(function () {
return test.jio.repair();
})
.then(function () {
last_id = test.remote_mock_options.last_post_id;
return putFullDoc(test.jio, last_id, doc2, blob2);
})
.then(function () {
resetCount(test.remote_mock_options.count);
return test.jio.repair();
})
.then(function () {
return RSVP.all([
equalStorage(
test.jio.__storage._remote_sub_storage,
[[last_id, doc2, blob2]]
),
equalRemoteStorageCallCount(
test.remote_mock_options.count,
{buildQuery: 1, put: 1,
allAttachments: 1, putAttachment: 1}
)
]);
})
.fail(function (error) {
ok(false, error);
})
.always(function () {
start();
});
});
///////////////////////////////////////////////////////
// Conflict
///////////////////////////////////////////////////////
test("both modification: keep local", function () {
expect(2);
stop();
var test = this,
doc_id = 'foo_module/1',
doc = {title: doc_id, portal_type: "Foo", modification_date: 'a'},
doc2 = {title: doc_id + 'a', portal_type: "Foo", modification_date: 'b'},
doc3 = {title: doc_id + 'c', portal_type: "Foo", modification_date: 'c'},
blob = new Blob(['a']),
blob2 = new Blob(['b']),
blob3 = new Blob(['c']);
putFullDoc(this.jio.__storage._remote_sub_storage, doc_id, doc, blob)
.then(function () {
return test.jio.repair();
})
.then(function () {
return RSVP.all([
putFullDoc(test.jio.__storage._remote_sub_storage, doc_id,
doc2, blob2),
putFullDoc(test.jio, doc_id, doc3, blob3)
]);
})
.then(function () {
resetCount(test.remote_mock_options.count);
return test.jio.repair();
})
.then(function () {
return RSVP.all([
equalStorage(
test.jio.__storage._remote_sub_storage,
[[doc_id, doc3, blob3]]
),
equalRemoteStorageCallCount(
test.remote_mock_options.count,
{buildQuery: 1, put: 1,
allAttachments: 1, putAttachment: 1}
)
]);
})
.fail(function (error) {
ok(false, error);
})
.always(function () {
start();
});
});
test("local modification / frozen remote", function () {
expect(2);
stop();
var test = this,
doc_id = 'foo_module/1',
doc = {title: doc_id, portal_type: "Foo", modification_date: 'a'},
doc2 = {title: doc_id + 'a', portal_type: "Foo", modification_date: 'b'},
blob = new Blob(['a']),
blob2 = new Blob(['b']);
putFullDoc(this.jio.__storage._remote_sub_storage, doc_id, doc, blob)
.then(function () {
return test.jio.repair();
})
.then(function () {
return putFullDoc(test.jio, doc_id, doc2, blob2);
})
.then(function () {
test.remote_mock_options.mock.put = function (id) {
if (id === doc_id) {
throw new jIO.util.jIOError('put not allowed', 403);
}
return id;
};
test.remote_mock_options.mock.putAttachment = function (id) {
if (id === doc_id) {
throw new jIO.util.jIOError('putattachment not allowed', 403);
}
return id;
};
resetCount(test.remote_mock_options.count);
return test.jio.repair();
})
.then(function () {
ok(false, 'notimplemented');
})
.fail(function (error) {
ok(false, error);
})
.always(function () {
start();
});
});
///////////////////////////////////////////////////////
// Local deletion (aka, people playing manually with the browser storage)
///////////////////////////////////////////////////////
test("local document deletion: do nothing", function () {
expect(3);
stop();
var test = this,
doc_id = 'foo_module/1',
doc = {title: doc_id, portal_type: "Foo", modification_date: 'a'},
blob = new Blob(['a']);
putFullDoc(this.jio.__storage._remote_sub_storage, doc_id, doc, blob)
.then(function () {
return test.jio.repair();
})
.then(function () {
return test.jio.remove(doc_id);
})
.then(function () {
resetCount(test.remote_mock_options.count);
return test.jio.repair();
})
.then(function () {
return RSVP.all([
isEmptyStorage(test.jio),
equalStorage(
test.jio.__storage._remote_sub_storage,
[[doc_id, doc, blob]]
),
equalRemoteStorageCallCount(
test.remote_mock_options.count,
{buildQuery: 1}
)
]);
})
.fail(function (error) {
ok(false, error);
})
.always(function () {
start();
});
});
test("local attachment deletion: do nothing", function () {
expect(2);
stop();
var test = this,
doc_id = 'foo_module/1',
doc = {title: doc_id, portal_type: "Foo", modification_date: 'a'},
blob = new Blob(['a']);
putFullDoc(this.jio.__storage._remote_sub_storage, doc_id, doc, blob)
.then(function () {
return test.jio.repair();
})
.then(function () {
return test.jio.removeAttachment(doc_id, ATTACHMENT);
})
.then(function () {
resetCount(test.remote_mock_options.count);
return test.jio.repair();
})
.then(function () {
return RSVP.all([
equalStorage(
test.jio.__storage._remote_sub_storage,
[[doc_id, doc, blob]]
),
equalRemoteStorageCallCount(
test.remote_mock_options.count,
{buildQuery: 1}
)
]);
})
.fail(function (error) {
ok(false, error);
})
.always(function () {
start();
});
});
test("local deletion / remote modification", function () {
expect(2);
stop();
var test = this,
doc_id = 'foo_module/1',
doc = {title: doc_id, portal_type: "Foo", modification_date: 'a'},
doc2 = {title: doc_id + 'a', portal_type: "Foo", modification_date: 'b'},
blob = new Blob(['a']),
blob2 = new Blob(['b']);
putFullDoc(this.jio.__storage._remote_sub_storage, doc_id, doc, blob)
.then(function () {
return test.jio.repair();
})
.then(function () {
return RSVP.all([
putFullDoc(test.jio.__storage._remote_sub_storage, doc_id,
doc2, blob2),
test.jio.remove(doc_id)
]);
})
.then(function () {
resetCount(test.remote_mock_options.count);
return test.jio.repair();
})
.then(function () {
return RSVP.all([
equalStorage(
test.jio,
[[doc_id, doc2, blob2]]
),
equalStorage(
test.jio.__storage._remote_sub_storage,
[[doc_id, doc2, blob2]]
),
equalRemoteStorageCallCount(
test.remote_mock_options.count,
{buildQuery: 1, get: 1,
allAttachments: 1, getAttachment: 1}
)
]);
})
.fail(function (error) {
ok(false, error);
})
.always(function () {
start();
});
});
}(QUnit, jIO, Blob));
examples/scenario_officejs.html 0 → 100644
View file @ 54e878be
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>OfficeJS Coverage Scenario</title>
<script src="../node_modules/rsvp/dist/rsvp-2.0.4.js"></script>
<script src="../dist/jio-latest.js"></script>
<link rel="stylesheet" href="../node_modules/grunt-contrib-qunit/test/libs/qunit.css" type="text/css" media="screen"/>
<script src="../node_modules/grunt-contrib-qunit/test/libs/qunit.js" type="text/javascript"></script>
<script src="scenario_officejs.js"></script>
</head>
<body>
<h1 id="qunit-header">OfficeJS Coverage Scenario</h1>
<h2 id="qunit-banner"></h2>
<div id="qunit-testrunner-toolbar"></div>
<h2 id="qunit-userAgent"></h2>
<ol id="qunit-tests"></ol>
<div id="qunit-fixture">test markup, will be hidden</div>
</body>
</html>
examples/scenario_officejs.js
View file @ 54e878be
......@@ -155,12 +155,15 @@
parallel_operation_attachment_amount: 10,
local_sub_storage: {
type: "history",
sub_storage: {
type: "query",
sub_storage: {
type: "uuid",
sub_storage: {
type: "memory"
}
}
}
},
signature_sub_storage: {
type: "query",
......
src/jio.storage/querystorage.js
View file @ 54e878be
......@@ -133,7 +133,6 @@
throw error;
}
}
return substorage.buildQuery(sub_options)
// Include docs if needed
......@@ -198,6 +197,7 @@
if (options.select_list) {
options.select_list.push("__id");
}
console.log("query:",options.query);
result = jIO.QueryFactory.create(options.query || "",
context._key_schema).
exec(data_rows, options);
......
src/jio.storage/replicatestorage.js
View file @ 54e878be
......@@ -228,7 +228,6 @@
number_queue) {
var result_promise_list = [],
i;
function pushAndExecute(queue) {
queue
.push(function () {
......@@ -1297,12 +1296,12 @@
}
ReplicateStorage.prototype.repair = function () {
var context = this,
argument_list = arguments,
skip_document_dict = {},
skip_deleted_document_dict = {},
cache = {};
return new RSVP.Queue()
.push(function () {
// Ensure that the document storage is usable
......@@ -1425,6 +1424,7 @@
context._check_remote_attachment_deletion) {
// Attachments are synchronized if and only if their parent document
// has been also marked as synchronized.
return context._signature_sub_storage.allDocs({
select_list: ['hash', 'attachment_hash', 'from_local']
})
......@@ -1433,6 +1433,7 @@
local_argument_list = [],
row,
len = result.data.total_rows;
//skip_deleted_document_dict = {};
for (i = 0; i < len; i += 1) {
row = result.data.rows[i];
......
src/jio.storage/revisionstorage.js
View file @ 54e878be
......@@ -22,69 +22,74 @@
return re.test(id);
}
function removeOldRevs(
function removeOldRevisions(
substorage,
results,
keepDoc
keepDocument
) {
var ind,
promises = [],
seen = {},
docum,
var index,
promise_list = [],
seen_dict = {},
document,
log,
start_ind,
new_promises,
doc_id,
checkIsId,
removeDoc;
for (ind = 0; ind < results.data.rows.length; ind += 1) {
docum = results.data.rows[ind];
start_index,
new_promise_list,
document_id;
for (index = 0; index < results.data.rows.length; index += 1) {
document = results.data.rows[index];
// Count the number of revisions of each document, and delete older
// ones.
if (!seen.hasOwnProperty(docum.value.doc_id)) {
seen[docum.value.doc_id] = {count: 0};
if (!seen_dict.hasOwnProperty(document.value.document_id)) {
seen_dict[document.value.document_id] = {count: 0};
}
log = seen[docum.value.doc_id];
log = seen_dict[document.value.document_id];
log.count += 1;
//log.id = docum.id;
// Record the index of the most recent edit that is before the cutoff
if (!log.hasOwnProperty("s") && !keepDoc({doc: docum, log: log})) {
log.s = ind;
if (!log.hasOwnProperty("last_edit_before_cutoff") &&
!keepDocument({document: document, log: log})) {
log.last_edit_before_cutoff = index;
}
// Record the index of the most recent put or remove
if ((!log.hasOwnProperty("pr")) &&
(docum.value.op === "put" || docum.value.op === "remove")) {
log.pr = ind;
log.final = ind;
if ((!log.hasOwnProperty("put_or_remove")) &&
(document.value.op === "put" || document.value.op === "remove")) {
log.put_or_remove = index;
log.final = index;
}
if ((docum.op === "putAttachment" || docum.op === "removeAttachment") &&
log.hasOwnProperty(docum.name) &&
!log[docum.name].hasOwnProperty("prA")) {
log[docum.name].prA = ind;
log.final = ind;
if ((document.op === "putAttachment" ||
document.op === "removeAttachment") &&
log.hasOwnProperty(document.name) &&
!log[document.name].hasOwnProperty("put_or_remove_attachment")) {
log[document.name].put_or_remove_attachment = index;
log.final = index;
}
}
checkIsId = function (d) {
return d.value.doc_id === doc_id;
};
removeDoc = function (d) {
return substorage.remove(d.id);
};
for (doc_id in seen) {
if (seen.hasOwnProperty(doc_id)) {
log = seen[doc_id];
start_ind = Math.max(log.s, log.final + 1);
new_promises = results.data.rows
.slice(start_ind)
function checkIsId(document, document_id) {
return document.value.document_id === document_id;
}
function removeDocument(document) {
return substorage.remove(document.id);
}
// Looping over unique documents in storage
for (document_id in seen_dict) {
if (seen_dict.hasOwnProperty(document_id)) {
// Identify where to cut to avoid loss of data
log = seen_dict[document_id];
start_index = Math.max(log.last_edit_before_cutoff, log.final + 1);
// Record all remove promises for this document
new_promise_list = results.data.rows
.slice(start_index)
.filter(checkIsId)
.map(removeDoc);
promises = promises.concat(new_promises);
.map(removeDocument);
promise_list = promise_list.concat(new_promise_list);
}
}
return RSVP.all(promises);
return RSVP.all(promise_list);
}
function throwCantFindError(id) {
......@@ -114,7 +119,6 @@
} else {
this._include_revisions = false;
}
var substorage = this._sub_storage;
}
RevisionStorage.prototype.packOldRevisions = function (save_info) {
......@@ -128,20 +132,20 @@
keep_active_revs = x: throw away all outdated revisions from before x
**/
var substorage = this._sub_storage,
options = {
option_dict = {
sort_on: [["timestamp", "descending"]],
select_list: ["doc", "doc_id", "op"]
select_list: ["document", "document_id", "operation"]
},
keep_fixed_num = save_info.hasOwnProperty("keep_latest_num");
return substorage.allDocs(options)
keep_fixed_number = save_info.hasOwnProperty("keep_latest_number");
return substorage.allDocs(option_dict)
.push(function (results) {
if (keep_fixed_num) {
return removeOldRevs(substorage, results, function (data) {
return data.log.count <= save_info.keep_latest_num;
if (keep_fixed_number) {
return removeOldRevisions(substorage, results, function (data) {
return data.log.count <= save_info.keep_latest_number;
});
}
return removeOldRevs(substorage, results, function (data) {
return data.doc.id > save_info.keep_active_revs;
return removeOldRevisions(substorage, results, function (data) {
return data.doc.id > save_info.keep_active_revisions;
});
});
};
......@@ -150,25 +154,25 @@
// Query to get the last edit made to this document
var substorage = this._sub_storage,
doc_id_query,
document_id_query,
metadata_query,
options;
option_dict;
if (this._include_revisions) {
doc_id_query = new SimpleQuery({
document_id_query = new SimpleQuery({
operator: "<=",
key: "timestamp",
value: id_in
});
} else {
doc_id_query = new SimpleQuery({key: "doc_id", value: id_in});
document_id_query = new SimpleQuery({key: "document_id", value: id_in});
}
// Include id_in as value in query object for safety
metadata_query = new ComplexQuery({
operator: "AND",
query_list: [
doc_id_query,
document_id_query,
new ComplexQuery({
operator: "OR",
query_list: [
......@@ -178,7 +182,7 @@
})
]
});
options = {
option_dict = {
query: metadata_query,
select_list: ["op"],
limit: [0, 1],
......@@ -186,7 +190,7 @@
};
return substorage.allDocs(options)
return substorage.allDocs(option_dict)
.push(function (results) {
if (results.data.total_rows > 0) {
if (results.data.rows[0].value.op === "put") {
......@@ -243,31 +247,31 @@
RevisionStorage.prototype.allAttachments = function (id) {
var substorage = this._sub_storage,
query_obj,
query_object,
query_removed_check,
options,
query_doc_id,
options_remcheck,
include_revs = this._include_revisions,
option_dict,
query_document_id,
check_if_removed_option_list,
include_revisions = this._include_revisions,
have_seen_id = false;
// id is a timestamp, and allAttachments will return attachment versions
// up-to-and-including those made at time id
if (include_revs) {
query_doc_id = new SimpleQuery({
if (include_revisions) {
query_document_id = new SimpleQuery({
operator: "<=",
key: "timestamp",
value: id
});
} else {
query_doc_id = new SimpleQuery({key: "doc_id", value: id});
query_document_id = new SimpleQuery({key: "doc_id", value: id});
have_seen_id = true;
}
query_removed_check = new ComplexQuery({
operator: "AND",
query_list: [
query_doc_id,
query_document_id,
new ComplexQuery({
operator: "OR",
query_list: [
......@@ -278,10 +282,10 @@
]
});
query_obj = new ComplexQuery({
query_object = new ComplexQuery({
operator: "AND",
query_list: [
query_doc_id,
query_document_id,
new ComplexQuery({
operator: "OR",
query_list: [
......@@ -292,19 +296,19 @@
]
});
options_remcheck = {
check_if_removed_option_list = {
query: query_removed_check,
select_list: ["op", "timestamp"],
limit: [0, 1],
sort_on: [["timestamp", "descending"]]
};
options = {
query: query_obj,
option_dict = {
query: query_object,
sort_on: [["timestamp", "descending"]],
select_list: ["op", "name"]
};
return this._sub_storage.allDocs(options_remcheck)
return this._sub_storage.allDocs(check_if_removed_option_list)
// Check the document exists and is not removed
.push(function (results) {
if (results.data.total_rows > 0) {
......@@ -319,19 +323,19 @@
}
})
.push(function () {
return substorage.allDocs(options);
return substorage.allDocs(option_dict);
})
.push(function (results) {
var seen = {},
attachments = [],
attachment_promises = [],
ind,
var seen_dict = {},
attachment_list = [],
attachment_promise_list = [],
i,
entry;
// If input mapped to a real timestamp, then the first query result must
// have the inputted id. Otherwise, unexpected results could arise
// by inputting nonsensical strings as id when include_revisions = true
if (include_revs &&
if (include_revisions &&
results.data.total_rows > 0 &&
results.data.rows[0].id !== id &&
!have_seen_id) {
......@@ -341,20 +345,20 @@
// Only return attachments if:
// (it is the most recent revision) AND (it is a putAttachment)
attachments = results.data.rows.filter(function (docum) {
if (!seen.hasOwnProperty(docum.value.name)) {
var output = (docum.value.op === "putAttachment");
seen[docum.value.name] = {};
attachment_list = results.data.rows.filter(function (document) {
if (!seen_dict.hasOwnProperty(document.value.name)) {
var output = (document.value.op === "putAttachment");
seen_dict[document.value.name] = {};
return output;
}
});
// Assembles object of attachment_name: attachment_object
for (ind = 0; ind < attachments.length; ind += 1) {
entry = attachments[ind];
attachment_promises[entry.value.name] =
for (i = 0; i < attachment_list.length; i += 1) {
entry = attachment_list[i];
attachment_promise_list[entry.value.name] =
substorage.getAttachment(entry.id, entry.value.name);
}
return RSVP.hash(attachment_promises);
return RSVP.hash(attachment_promise_list);
});
};
......@@ -434,13 +438,13 @@
})
]
}),
options = {
option_dict = {
query: metadata_query,
sort_on: [["timestamp", "descending"]],
limit: [0, 1],
select_list: ["op", "name"]
};
return substorage.allDocs(options)
return substorage.allDocs(option_dict)
.push(function (results) {
if (results.data.total_rows > 0) {
// XXX: issue if attachments are put on a removed document
......@@ -474,67 +478,71 @@
return name === 'list' ||
name === 'include' ||
name === 'query' ||
name === 'select';
name === 'select' ||
name === 'limit' ||
name === 'sort';
};
RevisionStorage.prototype.buildQuery = function (options) {
RevisionStorage.prototype.buildQuery = function (option_dict) {
// Set default values
if (options === undefined) {options = {}; }
if (options.query === undefined) {options.query = ""; }
if (options.sort_on === undefined) {options.sort_on = []; }
if (options.select_list === undefined) {options.select_list = []; }
options.query = jIO.QueryFactory.create(options.query);
var meta_options = {
if (option_dict === undefined) {option_dict = {}; }
if (option_dict.query === undefined) {option_dict.query = ""; }
if (option_dict.sort_on === undefined) {option_dict.sort_on = []; }
if (option_dict.select_list === undefined) {option_dict.select_list = []; }
//option_dict.query = jIO.QueryFactory.create(option_dict.query);
var meta_option_dict = {
query: "",
sort_on: [["timestamp", "descending"]],
select_list: ["doc", "op", "doc_id", "timestamp"]
},
include_revs = this._include_revisions;
include_revisions = this._include_revisions;
if (include_revs) {// && options.query.key === "doc_id") {
meta_options.query = options.query;
// When include_revisions is set, the document metadata will be queried
// instead of the documents themselves.
if (include_revisions) {
meta_option_dict.query = option_dict.query;
}
return this._sub_storage.allDocs(meta_options)
.push(function (results) {
results = results.data.rows;
var seen = {},
docs_to_query,
return this._sub_storage.allDocs(meta_option_dict)
.push(function (result_list) {
result_list = result_list.data.rows;
var seen_dict = {},
documents_to_query_list,
i;
if (include_revs) {
if (include_revisions) {
// We only query on versions mapping to puts or putAttachments
results = results.map(function (docum, ind) {
result_list = result_list.map(function (document, index) {
var data_key;
if (docum.value.op === "put") {
return docum;
if (document.value.op === "put") {
return document;
}
if (docum.value.op === "remove") {
docum.value.doc = {};
return docum;
if (document.value.op === "remove") {
document.value.doc = {};
return document;
}
if (docum.value.op === "putAttachment" ||
docum.value.op === "removeAttachment") {
if (document.value.op === "putAttachment" ||
document.value.op === "removeAttachment") {
// putAttachment document does not contain doc metadata, so we
// add it from the most recent non-removed put on same id
docum.value.doc = {};
for (i = ind + 1; i < results.length; i += 1) {
if (results[i].value.doc_id === docum.value.doc_id) {
if (results[i].value.op === "put") {
for (data_key in results[i].value.doc) {
if (results[i].value.doc.hasOwnProperty(data_key)) {
docum.value.doc[data_key] =
results[i].value.doc[data_key];
document.value.doc = {};
for (i = index + 1; i < result_list.length; i += 1) {
if (result_list[i].value.doc_id === document.value.doc_id) {
if (result_list[i].value.op === "put") {
for (data_key in result_list[i].value.doc) {
if (result_list[i].value.doc.hasOwnProperty(data_key)) {
document.value.doc[data_key] =
result_list[i].value.doc[data_key];
}
}
return docum;
return document;
}
// If most recent metadata edit before the attachment edit
// was a remove, then leave doc empty
if (results[i].value.op === "remove") {
return docum;
if (result_list[i].value.op === "remove") {
return document;
}
}
}
......@@ -545,43 +553,43 @@
// Only query on latest revisions of non-removed documents/attachment
// edits
results = results.map(function (docum, ind) {
result_list = result_list.map(function (document, index) {
var data_key;
if (docum.value.op === "put") {
if (document.value.op === "put") {
// Mark as read and include in query
if (!seen.hasOwnProperty(docum.value.doc_id)) {
seen[docum.value.doc_id] = {};
return docum;
if (!seen_dict.hasOwnProperty(document.value.doc_id)) {
seen_dict[document.value.doc_id] = {};
return document;
}
} else if (docum.value.op === "remove" ||
docum.value.op === "removeAttachment") {
} else if (document.value.op === "remove" ||
document.value.op === "removeAttachment") {
// Mark as read but do not include in query
seen[docum.value.doc_id] = {};
} else if (docum.value.op === "putAttachment") {
// If latest edit, mark as read, add document metadata from most
// recent put, and add to query
if (!seen.hasOwnProperty(docum.value.doc_id)) {
seen[docum.value.doc_id] = {};
docum.value.doc = {};
for (i = ind + 1; i < results.length; i += 1) {
if (results[i].value.doc_id === docum.value.doc_id) {
if (results[i].value.op === "put") {
for (data_key in results[i].value.doc) {
if (results[i].value.doc.hasOwnProperty(data_key)) {
docum.value.doc[data_key] =
results[i].value.doc[data_key];
}
}
return docum;
}
if (results[i].value.op === "remove") {
seen_dict[document.value.doc_id] = {};
} else if (document.value.op === "putAttachment") {
// If latest edit, mark as read, add documentent metadata from
// most recent put, and add to query
if (!seen_dict.hasOwnProperty(document.value.doc_id)) {
seen_dict[document.value.doc_id] = {};
document.value.doc = {};
for (i = index + 1; i < result_list.length; i += 1) {
if (result_list[i].value.doc_id === document.value.doc_id) {
if (result_list[i].value.op === "put") {
for (data_key in result_list[i].value.doc) {
if (result_list[i].value.doc.hasOwnProperty(data_key)) {
document.value.doc[data_key] =
result_list[i].value.doc[data_key];
}
}
return document;
}
if (result_list[i].value.op === "remove") {
// If most recent edit on document was a remove before
// this attachment, then don't include attachment in query
return false;
}
docum.value.doc = {};
document.value.doc = {};
}
}
}
......@@ -589,31 +597,36 @@
return false;
});
}
docs_to_query = results
documents_to_query_list = result_list
// Filter out all docs flagged as false in previous map call
.filter(function (docum) {
return docum;
.filter(function (document) {
return document;
})
// Put into correct format to be passed back to query storage
.map(function (docum) {
if (include_revs) {
docum.id = docum.value.timestamp;
//docum.doc = docum.value.doc;
.map(function (document) {
if (include_revisions) {
document.id = document.value.timestamp;
//document.doc = document.value.doc;
} else {
docum.id = docum.value.doc_id;
//docum.doc = docum.value.doc;
}
delete docum.value.doc_id;
delete docum.value.timestamp;
delete docum.value.op;
docum.value = docum.value.doc;
//docum.value = {};
return docum;
document.id = document.value.doc_id;
//document.doc = document.value.doc;
}
delete document.value.doc_id;
delete document.value.timestamp;
delete document.value.op;
var temp = document.value.doc;
document.value = {};
for (var selection in temp) {
if (option_dict.select_list.indexOf(selection) > -1) {
document.value[selection] = temp[selection];
}
}
//document.value = {};
return document;
});
return docs_to_query;
return documents_to_query_list;
});
};
......
src/jio.storage/saferepairstorage.js 0 → 100644
View file @ 54e878be
/*jslint nomen: true*/
/*global jIO, RSVP*/
(function (jIO, RSVP) {
"use strict";
/**
* The jIO SafeRepairStorage extension
*
* @class SafeRepairStorage
* @constructor
*/
function SafeRepairStorage(spec) {
this._sub_storage = jIO.createJIO(spec.sub_storage);
this._id_dict = {};
}
SafeRepairStorage.prototype.get = function () {
return this._sub_storage.get.apply(this._sub_storage, arguments);
};
SafeRepairStorage.prototype.allAttachments = function () {
return this._sub_storage.allAttachments.apply(this._sub_storage, arguments);
};
SafeRepairStorage.prototype.post = function () {
return this._sub_storage.post.apply(this._sub_storage, arguments);
};
SafeRepairStorage.prototype.put = function (id, doc) {
var storage = this;
return this._sub_storage.put.apply(this._sub_storage, arguments)
.push(undefined, function (error) {
if (error instanceof jIO.util.jIOError &&
error.status_code === 403) {
if (storage._id_dict[id]) {
return storage._sub_storage.put(storage._id_dict[id], doc);
}
return storage._sub_storage.post(doc)
.push(function (sub_id) {
storage._id_dict[id] = sub_id;
return sub_id;
});
}
});
};
SafeRepairStorage.prototype.remove = function () {
return;
};
SafeRepairStorage.prototype.getAttachment = function () {
return this._sub_storage.getAttachment.apply(this._sub_storage, arguments);
};
SafeRepairStorage.prototype.putAttachment = function (id, attachment_id,
attachment) {
var storage = this;
return this._sub_storage.putAttachment.apply(this._sub_storage, arguments)
.push(undefined, function (error) {
if (error instanceof jIO.util.jIOError &&
error.status_code === 403) {
return new RSVP.Queue()
.push(function () {
if (storage._id_dict[id]) {
return storage._id_dict[id];
}
return storage._sub_storage.get(id)
.push(function (doc) {
return storage._sub_storage.post(doc);
});
})
.push(function (sub_id) {
storage._id_dict[id] = sub_id;
return storage._sub_storage.putAttachment(sub_id, attachment_id,
attachment);
});
}
});
};
SafeRepairStorage.prototype.removeAttachment = function () {
return;
};
SafeRepairStorage.prototype.repair = function () {
return this._sub_storage.repair.apply(this._sub_storage, arguments);
};
SafeRepairStorage.prototype.hasCapacity = function (name) {
return this._sub_storage.hasCapacity(name);
};
SafeRepairStorage.prototype.buildQuery = function () {
return this._sub_storage.buildQuery.apply(this._sub_storage,
arguments);
};
jIO.addStorage('saferepair', SafeRepairStorage);
}(jIO, RSVP));
\ No newline at end of file
test/jio.storage/revisionstorage.tests.js
View file @ 54e878be
......@@ -1414,13 +1414,20 @@
})
.push(function () {
return RSVP.all([
jio.allDocs(),
jio.allDocs({query: "title: version0"}),
jio.allDocs({limit: [0, 1]}),
jio.allDocs({})
jio.allDocs({select_list: ['title']}),
jio.allDocs({
query: "title: version0",
select_list: ['title']
}),
jio.allDocs({
limit: [0, 1],
select_list: ['title']
}),
jio.allDocs({select_list: ['title']})
]);
})
.push(function (results) {
console.log(results);
var ind = 0;
for (ind = 0; ind < results.length - 1; ind += 1) {
deepEqual(results[ind],
......@@ -1436,12 +1443,14 @@
"Exactly one result returned");
deepEqual(results.data.rows[0], {
doc: {},
value: {},
value: {
title: "version0"
},
//timestamp: timestamp,
id: "doc"
},
"Correct document format is returned."
);
);///
return not_revision.allDocs();
})
.push(function (results) {
......@@ -1609,6 +1618,7 @@
);
})
.push(function () {
console.log("about to fail");
return revision.allDocs({
query: "",
select_list: ["title", "subtitle"]
......@@ -1690,7 +1700,7 @@
"A different storage type can retrieve all versions as expected.");
})
.fail(function (error) {
//console.log(error);
console.log(error);
ok(false, error);
})
.always(function () {start(); });
......@@ -1826,14 +1836,13 @@
.push(function () {
return jio.allDocs({
query: "NOT (date: > 2)",
select_list: ["date", "non-existent-key"],
sort_on: [["date", "ascending"],
["non-existent-key", "ascending"]
]
select_list: ["date"],//, "non-existent-key"],
sort_on: [["date", "ascending"]]//,
//["non-existent-key", "ascending"]]
});
})
.push(function (results) {
equal(results.data.total_rows, 3);
equal(results.data.total_rows, 3, "Correct number of results");
deepEqual(results.data.rows, [
{
doc: {},
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7