Skip to content

J
jio
Commit bd61250f authored by Sven Franck's avatar Sven Franck
Browse files
Options

indexStorage: putAttachment new API and qunit tests

parent 06d0b3f4
Hide whitespace changes
Inline Side-by-side
Showing with 373 additions and 272 deletions
src/jio.storage/indexstorage.js
View file @ bd61250f
......@@ -117,33 +117,49 @@ jIO.addStorageType('indexed', function (spec, my) {
return new_index_object;
};
/**
* Determine if a key/value pair exists in an object by VALUE
* @method searchObjectByValue
* @param {object} indexToSearch The index to search
* @param {string} docid The document id to find
* @param {string} passback The value that should be returned
* @return {boolean} true/false
*/
priv.searchIndexByValue = function (indexToSearch, docid, passback) {
var key, obj, prop;
for (key in indexToSearch) {
obj = indexToSearch[key];
for (prop in obj) {
if (obj[prop] === docid) {
return passback === "bool" ? true : key;
}
}
}
return false;
}
/**
* Find id in indices
* @method docidInIndex
* @method isDocidInIndex
* @param {object} indices The file containing the indeces
* @param {object} doc The document which should be added to the index
* @return {boolean} true/false
*/
priv.docidInIndex = function (indices, doc) {
var i, j, l = priv.indices.length, elements_to_check,
index, index_name, index_length;
priv.isDocidInIndex = function (indices, doc) {
var index, i, l = priv.indices.length;
// loop indices
for (i = 0; i < l; i += 1) {
index = priv.indices[i];
index_name = index["name"];
index_length = index.fields.length;
elements_to_check = priv.getObjectSize(indices[index_name]);
index = {};
index.reference = priv.indices[i];
index.name = index.reference["name"];
index.size = priv.getObjectSize(indices[index.name]);
if (elements_to_check > 0) {
for (var key in indices[index_name]) {
var obj = indices[index_name][key];
for (var prop in obj) {
if (obj[prop] === doc._id) {
return true;
}
}
}
if (index.size > 0) {
if (priv.searchIndexByValue(indices[index.name], doc._id, "bool")) {
return true
};
}
}
return false;
......@@ -154,35 +170,48 @@ jIO.addStorageType('indexed', function (spec, my) {
* @param {object} indices The file containing the indeces
* @param {object} doc The document which should be added to the index
*/
priv.updateIndeces = function (indices, doc) {
var i, j, k, m, value,
index, index_name, index_length, index_field_array,
l = priv.indices.length,
docid = doc._id;
priv.updateIndices = function (indices, doc) {
var i, j, k, m, index,value, label, key, l = priv.indices.length;
// loop indices
for (i = 0; i < l; i += 1) {
index = priv.indices[i];
index_name = index["name"];
index_length = index.fields.length;
index_field_array = [];
// index object (reference and current iteration)
index = {};
index.reference = priv.indices[i];
index.reference_size = index.reference.fields.length;
index.name = index.reference["name"];
index.field_array = [];
index.current = indices[index.name];
index.current_size = priv.getObjectSize(index.current);
// loop index fields [keywords]
for (j = 0; j < index_length; j += 1) {
value = doc[index.fields[j]];
// build array of values to create entries in index
for (j = 0; j < index.reference_size; j += 1) {
label = index.reference.fields[j]
value = doc[label];
if (value !== undefined) {
index_field_array.push(value);
// add a new entry
index.field_array.push(value);
// remove existing entries with same docid
// because items are stored as "keyword:id" pairs this is tricky
if (index.current_size > 0) {
key = priv.searchIndexByValue(indices[index.name], doc._id, "key");
if (!!key) {
delete index.current[key];
}
}
}
}
m = index_field_array.length;
if (m) {
for (k = 0; k < m; k += 1) {
if (indices[index_name] !== undefined) {
if (indices[index_name][index_field_array[k]] === undefined) {
indices[index_name][index_field_array[k]] = [];
// create keyword entries
if (index.current !== undefined) {
m = index.field_array.length;
if (m) {
for (k = 0; k < m; k += 1) {
index.current_keyword = [index.field_array[k]];
if (index.current[index.current_keyword] === undefined) {
index.current[index.current_keyword] = [];
}
indices[index_name][index_field_array[k]].push(docid);
index.current[index.current_keyword].push(doc._id);
}
}
}
......@@ -194,8 +223,9 @@ jIO.addStorageType('indexed', function (spec, my) {
* Post document to substorage and create/update index file(s)
* @method post
* @param {object} command The JIO command
* @param {string} source The source of the function call
*/
that.post = function (command) {
priv.postOrput = function (command, source) {
var f = {}, indices, doc, docid;
doc = command.cloneDoc();
docid = command.getDocId();
......@@ -214,16 +244,25 @@ jIO.addStorageType('indexed', function (spec, my) {
priv.index_suffix,
option,
function (response) {
console.log("index file found, we post(put)");
console.log( indices );
indices = response;
f.postDocument("put");
},
function (err) {
switch (err.status) {
case 404:
indices = priv.createEmptyIndexArray();
f.postDocument("post");
if (source !== 'PUTATTACHMENT') {
indices = priv.createEmptyIndexArray();
f.postDocument("post");
} else {
that.error({
"status": 404,
"statusText": "Not Found",
"error": "not found",
"message": "Document not found",
"reason": "Document not found"
});
return;
}
break;
default:
err.message = "Cannot retrieve index array";
......@@ -234,9 +273,7 @@ jIO.addStorageType('indexed', function (spec, my) {
);
};
f.postDocument = function (index_update_method) {
// if the index file already has an entry with this id,
// the document already exists
if (priv.docidInIndex(indices, doc) && index_update_method === 'POST') {
if (priv.isDocidInIndex(indices, doc) && source === 'POST') {
// POST the document already exists
that.error({
"status": 409,
......@@ -247,20 +284,37 @@ jIO.addStorageType('indexed', function (spec, my) {
});
return;
} else {
indices = priv.updateIndeces(indices, doc);
if (source !== 'PUTATTACHMENT') {
indices = priv.updateIndices(indices, doc);
}
that.addJob(
"post",
source === 'PUTATTACHMENT' ? "putAttachment" : "post",
priv.substorage,
doc,
command.cloneOption(),
function () {
f.sendIndices(index_update_method);
if (source !== 'PUTATTACHMENT') {
f.sendIndices(index_update_method);
} else {
docid = docid + '/' + command.getAttachmentId();
that.success({
"ok": true,
"id": docid
});
}
},
function (err) {
switch (err.status) {
case 409:
// file already exists
f.sendIndices(index_update_method);
if (source !== 'PUTATTACHMENT') {
f.sendIndices(index_update_method);
} else {
that.success({
"ok": true,
"id": docid
});
}
break;
default:
err.message = "Cannot upload document";
......@@ -299,168 +353,68 @@ jIO.addStorageType('indexed', function (spec, my) {
* @method put
* @param {object} command The JIO command
*/
that.put = function (command) {
that.post(command);
that.post = function (command) {
priv.postOrput(command, 'POST');
};
/*
/**
* @method formatToFileObject
* @param {} row
* @return {} obj
*//*
priv.formatToFileObject = function (row) {
var k, obj = {
_id: row.id
};
for (k in row.value) {
if (row.value.hasOwnProperty(k)) {
obj[k] = row.value[k];
}
}
return obj;
* Update the document metadata and update the index
* @method put
* @param {object} command The JIO command
*/
that.put = function (command) {
priv.postOrput(command, 'PUT');
};
*/
/**
* @method allDocs
* @param {} files_object
* @return {} obj
*//*
priv.allDocs = function (files_object) {
var k, obj = {
rows: []
}, i = 0;
for (k in files_object) {
if (files_object.hasOwnProperty(k)) {
obj.rows[i] = {};
obj.rows[i].value = files_object[k];
obj.rows[i].id = obj.rows[i].key = obj.rows[i].value._id;
delete obj.rows[i].value._id;
i += 1;
}
}
obj.total_rows = obj.rows.length;
return obj;
* Add an attachment to a document (no index modification)
* @method putAttachment
* @param {object} command The JIO command
*/
that.putAttachment = function (command) {
priv.postOrput(command, 'PUTATTACHMENT');
};
*/
/**
* @method setFileArray
* @param {} file_array
*//*
priv.setFileArray = function (file_array) {
var i, obj = {};
for (i = 0; i < file_array.length; i += 1) {
obj[file_array[i].id] = priv.formatToFileObject(file_array[i]);
* Get the document metadata or attachment.
* Options:
* - {boolean} revs Add simple revision history (false by default).
* - {boolean} revs_info Add revs info (false by default).
* - {boolean} conflicts Add conflict object (false by default).
* @method get
* @param {object} command The JIO command
*/
that.get = function (command) {
var option, docid;
option = command.cloneOption();
if (option.max_retry === 0) {
option.max_retry = 3;
}
if (command.getAttachmentId() !== undefined) {
docid = command.getDocId() + '/' + command.getAttachmentId();
} else {
docid = command.getDocId();
}
localStorage.setItem(storage_file_object_name, obj);
};
*/
/**
* @method getFileObject
* @param {} docid
* @return {} obj
*//*
priv.getFileObject = function (docid) {
var obj = localStorage.getItem(storage_file_object_name) || {};
return obj[docid];
};
*/
/**
* @method addFile
* @param {} file_obj
*//*
priv.addFile = function (file_obj) {
var obj = localStorage.getItem(storage_file_object_name) || {};
obj[file_obj._id] = file_obj;
localStorage.setItem(storage_file_object_name, obj);
};
*/
/**
* @method removeFile
* @param {} docid
*//*
priv.removeFile = function (docid) {
var obj = localStorage.getItem(storage_file_object_name) || {};
delete obj[docid];
localStorage.setItem(storage_file_object_name, obj);
};
*/
/**
* updates the storage.
* It will retreive all files from a storage. It is an asynchronous task
* so the update can be on going even if IndexedStorage has already
* returned the result.
* @method update
*//*
priv.update = function () {
that.addJob(
'allDocs',
priv.sub_storage_spec,
null,
{max_retry: 3},
"get",
priv.substorage,
docid,
option,
function (response) {
priv.setFileArray(response.rows);
},
function () {}
);
};
*/
/**
* Add put job to substorage and create/update index file(s)
* @method put
* @param {object} command The JIO command
*//*
that.put = function (command) {
var cloned_doc = command.cloneDoc(),
cloned_option = command.cloneOption();
// create/update indexStorage
// fwd job
that.addJob('put', priv.sub_storage_spec, cloned_doc,
cloned_option,
function (response) {
priv.update();
that.success(response);
},
function (error) {
that.error(error);
function (err) {
that.error({
"status": 404,
"statusText": "Not Found",
"error": "not_found",
"message": "Cannot find the attachment",
"reason": "Document/Attachment not found"
});
}
);
};
*//**
* Loads a document.
* @method get
*//*
that.get = function (command) {
// jslint unused var file_array
var success = function (val) {
that.success(val);
},
error = function (err) {
that.error(err);
},
get = function () {
var cloned_option = command.cloneOption();
that.addJob('get', priv.sub_storage_spec, command.cloneDoc(),
cloned_option, success, error);
that.end();
};
priv.indexStorage();
priv.update();
if (command.getOption('metadata_only')) {
setTimeout(function () {
var file_obj = priv.getFileObject(command.getDocId());
if (file_obj && (file_obj._last_modified || file_obj._creation_date)) {
that.success(file_obj);
} else {
get();
}
});
} else {
get();
}
}; // end get
*//**
/**
* Removes a document.
* @method remove
*//*
......
test/jiotests.js
View file @ bd61250f
......@@ -2653,13 +2653,16 @@ test ("Post", function () {
o.tick(o);
// check document
o.indexPost = {
o.fakeIndex = {
"_id": "ipost_indices.json",
"indexAB": {"keyword_abc":["some_id"], "keyword_def":["some_id"]},
"indexA": {"keyword_abc":["some_id"]}
};
deepEqual(
o.jio.get("ipost_indices.json"), o.indexPost, "Check index file"
);
o.jio.get("ipost_indices.json",function(err, response){
o.actualIndex = response;
deepEqual(o.actualIndex, o.fakeIndex, "Check index file");
});
o.tick(o);
// post with escapable characters
o.doc = {"_id": "other_id", "title": "myPost2",
......@@ -2689,7 +2692,7 @@ test ("Put", function(){
"type": "indexed",
"indices": [
{"name":"indexA", "fields":["author"]},
{"name":"indexAB", "fields":["author","findMeC"]}
{"name":"indexAB", "fields":["author","year"]}
],
"sub_storage": {
"type": "local",
......@@ -2711,33 +2714,35 @@ test ("Put", function(){
o.tick(o);
// check index file
o.indexPut = {
"indexA": {"John Doe": ["put1"]},
"indexAB": {"John Doe": ["put1"]},
"_id": "iput_indices.json"
o.fakeIndex = {
"indexA": {"John Doe": ["put1"]},
"indexAB": {"John Doe": ["put1"]},
"_id": "iput_indices.json"
};
deepEqual(
o.jio.get("iput_indices.json"), o.indexPut, "Check index file",
o.index, "Check index file"
);
o.jio.get("iput_indices.json",function(err, response){
o.actualIndex = response;
deepEqual(o.actualIndex, o.fakeIndex, "Check index file");
});
o.tick(o);
// modify a document - modify keyword on index!
// modify document - modify keyword on index!
o.doc = {"_id": "put1", "title": "myPuttter1", "author":"Jane Doe"};
o.spy (o, "value", {"ok": true, "id": "put1"},
"Modify document, update index file");
"Modify document, update keyword on index");
o.jio.put(o.doc, o.f);
o.tick(o);
// check index file
o.index = {
"indexA": {"Jane Doe": ["put1"]},
"indexAB": {"Jane Doe": ["put1"]},
"_id": "iput_indices.json"
o.fakeIndex = {
"indexA": {"Jane Doe": ["put1"]},
"indexAB": {"Jane Doe": ["put1"]},
"_id": "iput_indices.json"
};
deepEqual(
o.jio.get("iput_indices.json"), o.indexPut, "Check index file",
o.index, "Check index file"
);
o.jio.get("iput_indices.json",function(err, response){
o.actualIndex = response;
deepEqual(o.actualIndex, o.fakeIndex, "Check index file");
});
o.tick(o);
// add new document with same keyword!
o.doc = {"_id": "new_doc", "title": "myPut2", "author":"Jane Doe"};
......@@ -2747,15 +2752,36 @@ test ("Put", function(){
o.tick(o);
// check index file
o.index = {
"indexA": {"Jane Doe": ["put1", "new_doc"] },
"indexAB": {"Jane Doe": ["put1", "new_doc"]},
"_id": "iput_indices.json"
o.fakeIndex = {
"indexA": {"Jane Doe": ["put1", "new_doc"] },
"indexAB": {"Jane Doe": ["put1", "new_doc"]},
"_id": "iput_indices.json"
};
deepEqual(
o.jio.get("iput_indices.json"), o.indexPut, "Check index file",
o.index, "Check index file"
);
o.jio.get("iput_indices.json",function(err, response){
o.actualIndex = response;
deepEqual(o.actualIndex, o.fakeIndex, "Check index file");
});
o.tick(o);
// add second keyword to index file
o.doc = {"_id": "put1", "title": "myPut2", "author":"Jane Doe",
"year":"1912"};
o.spy (o, "value", {"ok": true, "id": "put1"},
"add second keyword to index file");
o.jio.put(o.doc, o.f);
o.tick(o);
// check index file
o.fakeIndex = {
"indexA": {"Jane Doe": ["put1"] },
"indexAB": {"Jane Doe": ["put1"],"1912": ["put1"]},
"_id": "iput_indices.json"
};
o.jio.get("iput_indices.json",function(err, response){
o.actualIndex = response;
deepEqual(o.actualIndex, o.fakeIndex, "Check index file");
});
o.tick(o);
// remove a keyword from an existing document
o.doc = {"_id": "new_doc", "title": "myPut2"};
......@@ -2765,64 +2791,185 @@ test ("Put", function(){
o.tick(o);
// check index file
o.index = {
"indexA": {"Jane Doe": ["put1"] },
"indexAB": {"Jane Doe": ["put1"]},
"_id": "iput_indices.json"
o.fakeIndex = {
"indexA": {"Jane Doe": ["put1"] },
"indexAB": {"Jane Doe": ["put1"], "1912": ["put1"]},
"_id": "iput_indices.json"
};
o.jio.get("iput_indices.json",function(err, response){
o.actualIndex = response;
deepEqual(o.actualIndex, o.fakeIndex, "Check index file");
});
o.tick(o);
o.jio.stop();
});
test ("PutAttachment", function(){
// not sure these need to be run, because the index does not change
// and only small modifications have been made to handle putAttachment
// tests are from localStorage putAttachment
var o = generateTools(this);
o.jio = JIO.newJio({
"type": "indexed",
"indices": [
{"name":"indexA", "fields":["author"]},
{"name":"indexAB", "fields":["author","year"]}
],
"sub_storage": {
"type": "local",
"username": "iputatt",
"application_name": "iputatt"
}
});
// putAttachment without doc id
// error 20 -> document id required
o.spy(o, "status", 20, "PutAttachment without doc id");
o.jio.putAttachment({}, o.f);
o.tick(o);
// putAttachment without attachment id
// error 22 -> attachment id required
o.spy(o, "status", 22, "PutAttachment without attachment id");
o.jio.putAttachment({"id": "putattmt1"}, o.f);
o.tick(o);
// putAttachment without document
// error 404 -> not found
o.spy(o, "status", 404, "PutAttachment without document");
o.jio.putAttachment({"id": "putattmt1/putattmt2"}, o.f);
o.tick(o);
// putAttachment with document
o.doc = {"_id": "putattmt1","title": "myPutAttmt1"};
o.spy (o, "value", {"ok": true, "id": "putattmt1"},
"Put underlying document");
o.jio.put(o.doc, o.f);
o.tick(o);
o.spy(o, "value", {"ok": true, "id": "putattmt1/putattmt2"},
"PutAttachment with document, without data");
o.jio.putAttachment({"id": "putattmt1/putattmt2"}, o.f);
o.tick(o);
// check document
deepEqual(
o.jio.get("iput_indices.json"), o.indexPut, "Check index file",
o.index, "Check index file"
localstorage.getItem("jio/localstorage/iputatt/iputatt/putattmt1"),
{
"_id": "putattmt1",
"title": "myPutAttmt1",
"_attachments": {
"putattmt2": {
"length": 0,
// md5("")
"digest": "md5-d41d8cd98f00b204e9800998ecf8427e"
}
}
},
"Check document"
);
// check attachment
deepEqual(
localstorage.getItem(
"jio/localstorage/iputatt/iputatt/putattmt1/putattmt2"),
"", "Check attachment"
);
// update attachment
o.spy(o, "value", {"ok": true, "id": "putattmt1/putattmt2"},
"Update Attachment, with data");
o.jio.putAttachment({"id": "putattmt1/putattmt2", "data": "abc"}, o.f);
o.tick(o);
// check document
deepEqual(
localstorage.getItem("jio/localstorage/iputatt/iputatt/putattmt1"),
{
"_id": "putattmt1",
"title": "myPutAttmt1",
"_attachments": {
"putattmt2": {
"length": 3,
// md5("abc")
"digest": "md5-900150983cd24fb0d6963f7d28e17f72"
}
}
},
"Check document"
);
// check attachment
deepEqual(
localstorage.getItem(
"jio/localstorage/iputatt/iputatt/putattmt1/putattmt2"),
"abc", "Check attachment"
);
o.jio.stop();
});
/*
test ("Get", function(){
var o = generateTools(this);
/*
test ('Document load', function () {
var o = {}; o.clock = this.sandbox.useFakeTimers();
o.clock.tick(base_tick);
o.jio = JIO.newJio({type:'indexed',storage:{type:'dummyall3tries'}});
// loading must take long time with dummyall3tries
o.f = this.spy();
o.jio.get('memo',{max_retry:3,metadata_only:true},o.f);
o.clock.tick(1000);
ok(!o.f.called,'Callback must not be called');
// wait long time too retreive list
o.clock.tick(1000);
o.jio = JIO.newJio({
"type": "local",
"username": "uget",
"application_name": "aget"
});
// now we can test if the document metadata are loaded faster.
o.doc = {_id:'memo',_last_modified:25000,_creation_date:20000};
o.f2 = function (err,val) {
deepEqual (err||val,o.doc,'Document metadata retrieved');
// get inexistent document
o.spy(o, "status", 404, "Get inexistent document");
o.jio.get("get1", o.f);
o.tick(o);
// get inexistent attachment
o.spy(o, "status", 404, "Get inexistent attachment");
o.jio.get("get1/get2", o.f);
o.tick(o);
// adding a document
o.doc_get1 = {
"_id": "get1",
"title": "myGet1"
};
this.spy(o,'f2');
o.jio.get('memo',{max_retry:3,metadata_only:true},o.f2);
o.clock.tick(1000);
if (!o.f2.calledOnce) {
if (o.f2.called) {
ok (false, 'too much results');
} else {
ok (false, 'no response');
}
}
localstorage.setItem("jio/localstorage/uget/aget/get1", o.doc_get1);
// get document
o.spy(o, "value", o.doc_get1, "Get document");
o.jio.get("get1", o.f);
o.tick(o);
// get inexistent attachment (document exists)
o.spy(o, "status", 404, "Get inexistent attachment (document exists)");
o.jio.get("get1/get2", o.f);
o.tick(o);
// test a simple document loading
o.doc2 = {_id:'file',_last_modified:17000,
_creation_date:11000,content:'content file'};
o.f3 = function (err,val) {
deepEqual (err||val,o.doc2,'Simple document loading');
// adding an attachment
o.doc_get1["_attachments"] = {
"get2": {
"length": 2,
// md5("de")
"digest": "md5-5f02f0889301fd7be1ac972c11bf3e7d"
}
};
this.spy(o,'f3');
o.jio.get('file',{max_retry:3},o.f3);
o.clock.tick(2000);
if (!o.f3.calledOnce) {
ok (false, 'no response / too much results');
}
localstorage.setItem("jio/localstorage/uget/aget/get1", o.doc_get1);
localstorage.setItem("jio/localstorage/uget/aget/get1/get2", "de");
// get attachment
o.spy(o, "value", "de", "Get attachment");
o.jio.get("get1/get2", o.f);
o.tick(o);
o.jio.stop();
});
*/
/*
test ('Get document list', function () {
var o = {}; o.clock = this.sandbox.useFakeTimers();
o.clock.tick(base_tick);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7