Skip to content

J
jio
Commit 4c418987 authored by Tristan Cavelier's avatar Tristan Cavelier
Browse files
Options

indexstorage.js: reworked to manage queries, increase speed and to fix some bug

parent c65d2469
Hide whitespace changes
Inline Side-by-side
Showing with 867 additions and 1265 deletions
src/jio.storage/indexstorage.js
View file @ 4c418987
/* /*
* Copyright 2013, Nexedi SA * Copyright 2013, Nexedi SA
* Released under the LGPL license. * Released under the LGPL license.
* http://www.gnu.org/licenses/lgpl.html * http://www.gnu.org/licenses/lgpl.html
*/ */
/*jslint indent: 2, maxlen: 80, sloppy: true, nomen: true */
/*global jIO: true, localStorage: true, setTimeout: true */ /*jslint indent: 2, maxlen: 80, sloppy: true, nomen: true, regexp: true */
/*global jIO: true, localStorage: true, define: true, complex_queries: true */
/** /**
* JIO Index Storage. * JIO Index Storage.
* Manages indexes for specified storages. * Manages indexes for specified storages.
* Description: * Description:
* { * {
* "type": "index", * "type": "index",
* "indices": [ * "indices": [{
* {"indexA",["field_A"]}, * "id": "index_title_subject.json", // doc id where to store indices
* {"indexAB",["field_A","field_B"]} * "index": ["title", "subject"] // metadata to index
* ], * "sub_storage": <sub storage where to store index>
* "field_types": { * (default equal to parent sub_storage field)
* "field_A": "dateTime", * }, {
* "field_B": "string" * "id": "index_year.json",
* }, * "index": "year"
* "storage": [ * ...
* <sub storage description>, * }],
* ... * "sub_storage": <sub storage description>
* ] * }
*
* Sent document metadata will be:
* index_titre_subject.json
* {
* "_id": "index_title_subject.json",
* "indexing": ["title", "subject"],
* "free": [0],
* "location": {
* "foo": 1,
* "bar": 2,
* ...
* },
* "database": [
* {},
* {"_id": "foo", "title": "...", "subject": ...},
* {"_id": "bar", "title": "...", "subject": ...},
* ...
* ]
* } * }
* Index file will contain *
* index_year.json
* { * {
* "_id": "app-name_indices.json", * "_id": "index_year.json",
* "indexA": * "indexing": ["year"],
* "fieldA": { * "free": [1],
* "keyword_abc": ["some_id","some_other_id",...] * "location": {
* } * "foo": 0,
* "bar": 2,
* ...
* }, * },
* "indexAB": { * "database": [
* "fieldA": { * {"_id": "foo", "year": "..."},
* "keyword_abc": ["some_id"] * {},
* }, * {"_id": "bar", "year": "..."},
* "fieldB": { * ...
* "keyword_def": ["some_id"] * ]
* }
* }
* } * }
* NOTES: *
* It may be difficult to "un-sort" multi-field indices, like * A put document will be indexed to the free location if exist, else it will be
* indexAB, because all keywords will be listed regrardless * indexed at the end of the database. The document id will be indexed, also, in
* of underlying field, so an index on author and year would produce * 'location' to quickly replace metadata.
* two entries per record like: *
* * Only one or two loops are executed:
* "William Shakespeare":["id_Romeo_and_Juliet", "id_Othello"], * - one to filter retrieved document list (no query -> no loop)
* "1591":["id_Romeo_and_Juliet"], * - one to format the result to a JIO response
* "1603":["id_Othello"]
*
* So for direct lookups, this should be convient, but for other types
* of queries, it depends
*/ */
jIO.addStorageType('indexed', function (spec, my) { (function () {
"use strict"; "use strict";
var that, priv = {};
spec = spec || {};
that = my.basicStorage(spec, my);
priv.indices = spec.indices; var error_dict = {
priv.field_types = spec.field_types; "Corrupted Index": {
priv.substorage_key = "sub_storage"; "status": 24,
priv.substorage = spec[priv.substorage_key]; "statusText": "Corrupt",
priv.index_indicator = spec.sub_storage.application_name || "index"; "error": "corrupt",
priv.index_suffix = priv.index_indicator + "_indices.json"; "reason": "corrupted index database"
},
"Corrupted Metadata": {
"status": 24,
"statusText": "Corrupt",
"error": "corrupt",
"reason": "corrupted document"
},
"Not Found": {
"status": 404,
"statusText": "Not Found",
"error": "not_found",
"reason": "missing document"
},
"Conflict": {
"status": 409,
"statusText": "Conflicts",
"error": "conflicts",
"reason": "already exist"
}
};
my.env = my.env || spec.env || {}; /**
* Generate a JIO Error Object
*
* @method generateErrorObject
* @param {String} name The error name
* @param {String} message The error message
* @param {String} [reason] The error reason
* @return {Object} A jIO error object
*/
function generateErrorObject(name, message, reason) {
if (!error_dict[name]) {
return {
"status": 0,
"statusText": "Unknown",
"error": "unknown",
"message": message,
"reason": reason || "unknown"
};
}
return {
"status": error_dict[name].status,
"statusText": error_dict[name].statusText,
"error": error_dict[name].error,
"message": message,
"reason": reason || error_dict[name].reason
};
}
that.specToStore = function () { /**
var o = {}; * Get the real type of an object
o[priv.substorage_key] = priv.substorage; * @method type
o.env = my.env; * @param {Any} value The value to check
return o; * @return {String} The value type
}; */
function type(value) {
// returns "String", "Object", "Array", "RegExp", ...
return (/^\[object ([a-zA-Z]+)\]$/).exec(
Object.prototype.toString.call(value)
)[1];
}
/** /**
* Generate a new uuid * Generate a new uuid
* @method generateUuid * @method generateUuid
* @return {string} The new uuid * @return {string} The new uuid
*/ */
priv.generateUuid = function () { function generateUuid() {
var S4 = function () { var S4 = function () {
var i, string = Math.floor( var i, string = Math.floor(
Math.random() * 0x10000 /* 65536 */ Math.random() * 0x10000 /* 65536 */
...@@ -98,863 +162,541 @@ jIO.addStorageType('indexed', function (spec, my) { ...@@ -98,863 +162,541 @@ jIO.addStorageType('indexed', function (spec, my) {
S4() + "-" + S4() + "-" +
S4() + "-" + S4() + "-" +
S4() + S4() + S4(); S4() + S4() + S4();
}; }
/**
* Get number of elements in object
* @method getObjectSize
* @param {object} obj The object to check
* @return {number} size The amount of elements in the object
*/
priv.getObjectSize = function (obj) {
var size = 0, key;
for (key in obj) {
if (obj.hasOwnProperty(key)) {
size += 1;
}
}
return size;
};
/** /**
* Creates an empty indices array * A JSON Index manipulator
* @method createEmptyIndexArray *
* @param {array} indices An array of indices (optional) * @class JSONIndex
* @return {object} The new index array * @constructor
*/ */
priv.createEmptyIndexArray = function (indices) { function JSONIndex(spec) {
var i, k, j = priv.indices.length, new_index, var that = this;
new_index_object = {}, new_index_name, new_index_fields; spec = spec || {};
if (indices === undefined) { /**
for (i = 0; i < j; i += 1) { * The document id
new_index = priv.indices[i]; *
new_index_name = new_index.name; * @property _id
new_index_fields = new_index.fields; * @type String
new_index_object[new_index_name] = {}; */
that._id = spec._id;
// loop index fields and add objects to hold value/id pairs
for (k = 0; k < new_index_fields.length; k += 1) { /**
new_index_object[new_index_name][new_index_fields[k]] = {}; * The array with metadata key to index
} *
* @property _indexing
* @type Array
*/
that._indexing = spec.indexing || [];
/**
* The array of free location index
*
* @property _free
* @type Array
* @default []
*/
that._free = spec.free || [];
/**
* The dictionnary document id -> database index
*
* @property _location
* @type Object
* @default {}
*/
that._location = spec.location || {};
/**
* The database array containing document metadata
*
* @property _database
* @type Array
* @default []
*/
that._database = spec.database || [];
/**
* Adds a metadata object in the database, replace if already exist
*
* @method put
* @param {Object} meta The metadata to add
* @return {Boolean} true if added, false otherwise
*/
that.put = function (meta) {
var underscored_meta_re = /^_.*$/, k, needed_meta = {}, ok = false;
if (typeof meta._id !== "string" && meta._id !== "") {
throw new TypeError("Corrupted Metadata");
} }
} for (k in meta) {
return new_index_object; if (meta.hasOwnProperty(k)) {
}; if (underscored_meta_re.test(k)) {
needed_meta[k] = meta[k];
/** } else if (that._indexing_object[k]) {
* Determine if a key/value pair exists in an object by VALUE needed_meta[k] = meta[k];
* @method searchObjectByValue ok = true;
* @param {object} indexToSearch The index to search
* @param {string} docid The document id to find
* @param {string} passback The value that should be returned
* @return {boolean} true/false
*/
priv.searchIndexByValue = function (indexToSearch, docid, passback) {
var key, obj, prop;
for (key in indexToSearch) {
if (indexToSearch.hasOwnProperty(key)) {
obj = indexToSearch[key];
for (prop in obj) {
if (obj[prop] === docid) {
return passback === "bool" ? true : key;
} }
} }
} }
} if (ok) {
return false; if (typeof that._location[meta._id] === "number") {
}; that._database[that._location[meta._id]] = needed_meta;
} else if (that._free.length > 0) {
/** k = that._free.shift();
* Get element position in array that._database[k] = needed_meta;
* @method getPositionInArray that._location[meta._id] = k;
* @param {object} indices The index file } else {
* @param {object} indices The index file that._database.push(needed_meta);
* @returns {number} i Position of element in array that._location[meta._id] = that._database.length - 1;
*/
priv.getPositionInArray = function (element, array) {
var i, l = array.length;
for (i = 0; i < l; i += 1) {
if (array[i] === element) {
return i;
}
}
return null;
};
/**
* Find id in indices
* @method isDocidInIndex
* @param {object} indices The file containing the indeces
* @param {object} doc The document which should be added to the index
* @return {boolean} true/false
*/
priv.isDocidInIndex = function (indices, doc) {
var index, i, j, label, l = priv.indices.length;
// loop indices
for (i = 0; i < l; i += 1) {
index = {};
index.reference = priv.indices[i];
index.reference_size = index.reference.fields.length;
index.current = indices[index.reference.name];
for (j = 0; j < index.reference_size; j += 1) {
label = index.reference.fields[j];
index.current_size = priv.getObjectSize(index.current[label]);
// check for existing entries to remove (put-update)
if (index.current_size > 0) {
if (priv.searchIndexByValue(index.current[label], doc._id, "bool")) {
return true;
}
} }
return true;
} }
} if (typeof that._location[meta._id] === "number") {
return false; that.remove(meta);
}; }
return false;
};
/** /**
* Clean up indexes when removing a file * Removes a metadata object from the database if exist
* @method cleanIndices *
* @param {object} indices The file containing the indeces * @method remove
* @param {object} doc The document which should be added to the index * @param {Object} meta The metadata to remove
* @return {object} indices The cleaned up file */
*/ that.remove = function (meta) {
priv.cleanIndices = function (indices, doc) { if (typeof meta._id !== "string") {
var i, j, k, index, key, label, l = priv.indices.length; throw new TypeError("Corrupted Metadata");
// loop indices (indexA, indexAB...)
for (i = 0; i < l; i += 1) {
index = {};
index.reference = priv.indices[i];
index.reference_size = index.reference.fields.length;
index.current = indices[index.reference.name];
// loop index fields
for (j = 0; j < index.reference_size; j += 1) {
label = index.reference.fields[j];
index.current_size = priv.getObjectSize(index.current[label]);
// loop field entries
for (k = 0; k < index.current_size; k += 1) {
key = priv.searchIndexByValue(index.current[label], doc._id, "key");
index.result_array = index.current[label][key];
if (!!key) {
// if there is more than one docid in the result array,
// just remove this one and not the whole array
if (index.result_array.length > 1) {
index.result_array.splice(k, 1);
} else {
delete index.current[label][key];
}
}
}
} }
} if (typeof that._location[meta._id] !== "number") {
return indices; throw new ReferenceError("Not Found");
};
/**
* Adds entries to indices
* @method createEmptyIndexArray
* @param {object} indices The file containing the indeces
* @param {object} doc The document which should be added to the index
*/
priv.updateIndices = function (indices, doc) {
var i, j, index, value, label, key, l = priv.indices.length;
// loop indices
for (i = 0; i < l; i += 1) {
index = {};
index.reference = priv.indices[i];
index.reference_size = index.reference.fields.length;
index.current = indices[index.reference.name];
// build array of values to create entries in index
for (j = 0; j < index.reference_size; j += 1) {
label = index.reference.fields[j];
value = doc[label];
if (value !== undefined) {
index.current_size = priv.getObjectSize(index.current[label]);
// check for existing entries to remove (put-update)
if (index.current_size > 0) {
key = priv.searchIndexByValue(
index.current[label],
doc._id,
"key"
);
if (!!key) {
delete index.current[label][key];
}
}
if (index.current[label][value] === undefined) {
index.current[label][value] = [];
}
// add a new entry
index.current[label][value].push(doc._id);
}
} }
} that._database[that._location[meta._id]] = null;
return indices; that._free.push(that._location[meta._id]);
}; delete that._location[meta._id];
};
/**
* Check available indices to find the best one.
* TODOS: NOT NICE, redo
* @method findBestIndexForQuery
* @param {object} syntax of query
* @returns {object} response The query object constructed from Index file
*/
priv.findBestIndexForQuery = function (syntax) {
var i, j, k, l, n, p, o, element, key, block,
search_ids, use_index = [], select_ids = {}, index, query_param,
current_query, current_query_size;
// try to parse into object
if (syntax.query !== undefined) {
current_query = jIO.ComplexQueries.parse(syntax.query);
} else {
current_query = {};
current_query_size = 0;
}
// loop indices
for (i = 0; i < priv.indices.length; i += 1) {
search_ids = [];
block = false;
index = {};
index.reference = priv.indices[i];
index.reference_size = index.reference.fields.length;
if (current_query_size !== 0) {
// rebuild search_ids for iteration
if (current_query.query_list === undefined) {
search_ids.push(current_query.id);
} else {
for (j = 0; j < current_query.query_list.length; j += 1) {
if (priv.getPositionInArray(current_query.query_list[j].id,
search_ids) === null) {
search_ids.push(current_query.query_list[j].id);
}
}
}
// loop search ids and find matches in index /**
for (k = 0; k < search_ids.length; k += 1) { * Checks if the index document is correct
query_param = search_ids[0]; *
for (l = 0; l < index.reference_size; l += 1) { * @method check
if (query_param === index.reference.fields[l]) { */
search_ids.splice( that.check = function () {
priv.getPositionInArray(query_param, search_ids), var id, database_meta;
1 if (typeof that._id !== "string" ||
); that._id === "" ||
} type(that._free) !== "Array" ||
type(that._indexing) !== "Array" ||
type(that._location) !== "Object" ||
type(that._database) !== "Array" ||
that._indexing.length === 0) {
throw new TypeError("Corrupted Index");
}
for (id in that._location) {
if (that._location.hasOwnProperty(id)) {
database_meta = that._database[that._location[id]];
if (type(database_meta) !== "Object" ||
database_meta._id !== id) {
throw new TypeError("Corrupted Index");
} }
} }
} }
};
// rebuild select_ids /**
for (o = 0; o < syntax.filter.select_list.length; o += 1) { * Recreates database indices and remove free space
element = syntax.filter.select_list[o]; *
select_ids[element] = true; * @method repair
*/
that.repair = function () {
var i = 0, meta;
that._free = [];
that._location = {};
if (type(that._database) !== "Array") {
that._database = [];
} }
while (i < that._database.length) {
// search_ids empty = all needed search fields found on index meta = that._database[i];
if (search_ids.length === 0) { if (type(meta) === "Object" &&
p = priv.getObjectSize(select_ids); typeof meta._id === "string" && meta._id !== "" &&
if (p === 0) { !that._location[meta._id]) {
use_index.push({ that._location[meta._id] = i;
"name": index.reference.name, i += 1;
"search": true,
"results": false
});
} else { } else {
for (n = 0; n < index.reference_size; n += 1) { that._database.splice(i, 1);
delete select_ids[index.reference.fields[n]];
}
for (key in select_ids) {
if (select_ids.hasOwnProperty(key)) {
use_index.push({
"name": index.reference.name,
"search": true,
"results": false
});
block = true;
}
}
if (block === false) {
use_index.push({
"name": index.reference.name,
"search": true,
"results": true
});
}
} }
} }
} };
return use_index;
}; /**
* Returns the serialized version of this object (not cloned)
*
* @method serialized
* @return {Object} The serialized version
*/
that.serialized = function () {
return {
"_id": that._id,
"indexing": that._indexing,
"free": that._free,
"location": that._location,
"database": that._database
};
};
that.check();
that._indexing_object = {};
that._indexing.forEach(function (meta_key) {
that._indexing_object[meta_key] = true;
});
}
/** /**
* Converts the indices file into an object usable by complex queries * The JIO index storage constructor
* @method constructQueryObject
* @param {object} indices The index file
* @returns {object} response The query object constructed from Index file
*/
priv.constructQueryObject = function (indices, query_syntax) {
var j, k, l, m, n, use_index, index,
index_name, field_names, field, key, element,
query_index, query_object = [], field_name,
entry;
// returns index-to-use|can-do-query|can-do-query-and-results
use_index = priv.findBestIndexForQuery(query_syntax);
if (use_index.length > 0) {
for (j = 0; j < use_index.length; j += 1) {
index = use_index[j];
// NOTED: the index could be used to:
// (a) get all document ids matching query
// (b) get all document ids and results (= run complex query on index)
// right now, only (b) is supported, because the complex query is
// a single step process. If it was possible to first get the
// relevant document ids, then get the results, the index could be
// used to do the first step plus use GET on the returned documents
if (index.search && index.results) {
index_name = use_index[j].name;
query_index = indices[index_name];
// get fieldnames from this index
for (k = 0; k < priv.indices.length; k += 1) {
if (priv.indices[k].name === use_index[j].name) {
field_names = priv.indices[k].fields;
}
}
for (l = 0; l < field_names.length; l += 1) {
field_name = field_names[l];
// loop entries for this field name
field = query_index[field_name];
for (key in field) {
if (field.hasOwnProperty(key)) {
element = field[key];
// key can be "string" or "number" right now
if (priv.field_types[field_name] === "number") {
key = +key;
}
for (m = 0; m < element.length; m += 1) {
if (priv.searchIndexByValue(
query_object,
element[m],
"bool"
)) {
// loop object
for (n = 0; n < query_object.length; n += 1) {
entry = query_object[n];
if (entry.id === element[m]) {
entry[field_name] = key;
}
}
} else {
entry = {};
entry.id = element[m];
entry[field_name] = key;
query_object.push(entry);
}
}
}
}
}
}
}
}
return query_object;
};
/**
* Build the alldocs response from the index file (overriding substorage)
* @method allDocsResponseFromIndex
* @param {object} command The JIO command
* @param {boolean} include_docs Whether to also supply the document
* @param {object} option The options set for this method
* @returns {object} response The allDocs response
*/ */
priv.allDocsResponseFromIndex = function (indices, include_docs, option) { function indexStorage(spec, my) {
var i, j, k, m, n = 0, l = priv.indices.length, var that, priv = {};
index, key, obj, prop, found, file, label,
unique_count = 0, unique_docids = [], all_doc_response = {}, that = my.basicStorage(spec, my);
success = function (content) {
file = { value: {} }; priv.indices = spec.indices;
file.id = unique_docids[n]; priv.sub_storage = spec.sub_storage;
file.key = unique_docids[n];
file.doc = content; // Overrides
all_doc_response.rows.push(file);
// async counter, must be in callback that.specToStore = function () {
n += 1; return {
if (n === unique_count) { "indices": priv.indices,
that.success(all_doc_response); "sub_storage": priv.sub_storage
}
},
error = function () {
that.error({
"status": 404,
"statusText": "Not Found",
"error": "not_found",
"message": "Cannot find the document",
"reason": "Cannot get a document from substorage"
});
return;
}; };
};
// loop indices /**
for (i = 0; i < l; i += 1) { * Return the similarity percentage (1 >= p >= 0) between two index lists.
index = {}; *
index.reference = priv.indices[i]; * @method similarityPercentage
index.reference_size = index.reference.fields.length; * @param {Array} list_a An index list
index.current = indices[index.reference.name]; * @param {Array} list_b Another index list
* @return {Number} The similarity percentage
// a lot of loops, not sure this is the fastest way */
// loop index fields priv.similarityPercentage = function (list_a, list_b) {
for (j = 0; j < index.reference_size; j += 1) { var ai, bi, count = 0;
label = index.reference.fields[j]; for (ai = 0; ai < list_a.length; ai += 1) {
index.current_field = index.current[label]; for (bi = 0; bi < list_b.length; bi += 1) {
index.current_size = priv.getObjectSize(index.current_field); if (list_a[ai] === list_b[bi]) {
count += 1;
// loop field id array
for (j = 0; j < index.current_size; j += 1) {
for (key in index.current_field) {
if (index.current_field.hasOwnProperty(key)) {
obj = index.current_field[key];
for (prop in obj) {
if (obj.hasOwnProperty(prop)) {
for (k = 0; k < unique_docids.length; k += 1) {
if (obj[prop] === unique_docids[k]) {
found = true;
break;
}
}
if (!found) {
unique_docids.push(obj[prop]);
unique_count += 1;
}
}
}
}
} }
} }
} }
} return count / (list_a.length > list_b.length ?
list_a.length : list_b.length);
};
// construct allDocs response /**
all_doc_response.total_rows = unique_count; * Select the good index to use according to a select list.
all_doc_response.rows = []; *
for (m = 0; m < unique_count; m += 1) { * @method selectIndex
// include_docs * @param {Array} select_list An array of strings
if (include_docs) { * @return {Number} The index index
that.addJob( */
"get", priv.selectIndex = function (select_list) {
priv.substorage, var i, tmp, selector = {"index": 0, "similarity": 0};
unique_docids[m], for (i = 0; i < priv.indices.length; i += 1) {
option, tmp = priv.similarityPercentage(select_list,
success, priv.indices[i].index);
error if (tmp > selector.similarity) {
); selector.index = i;
} else { selector.similarity = tmp;
file = { value: {} };
file.id = unique_docids[m];
file.key = unique_docids[m];
all_doc_response.rows.push(file);
if (m === (unique_count - 1)) {
return all_doc_response;
} }
} }
} return selector.index;
}; };
/** /**
* Post document to substorage and create/update index file(s) * Get a database
* @method post *
* @param {object} command The JIO command * @method getIndexDatabase
* @param {string} source The source of the function call * @param {Object} option The command option
*/ * @param {Number} number The location in priv.indices
priv.postOrPut = function (command, source) { * @param {Function} callback The callback
var f = {}, indices, doc; */
doc = command.cloneDoc(); priv.getIndexDatabase = function (option, number, callback) {
if (typeof doc._id !== "string") {
doc._id = priv.generateUuid();
}
f.getIndices = function () {
var option = command.cloneOption();
that.addJob( that.addJob(
"get", "get",
priv.substorage, priv.indices[number].sub_storage || priv.sub_storage,
{"_id": priv.index_suffix}, {"_id": priv.indices[number].id},
option, option,
function (response) { function (response) {
indices = response; callback(new JSONIndex(response));
f.postDocument("put");
}, },
function (err) { function (err) {
switch (err.status) { if (err.status === 404) {
case 404: callback(new JSONIndex({
if (source !== 'PUTATTACHMENT') { "_id": priv.indices[number].id,
indices = priv.createEmptyIndexArray(); "indexing": priv.indices[number].index
f.postDocument("post"); }));
} else { return;
that.error({
"status": 404,
"statusText": "Not Found",
"error": "not found",
"message": "Document not found",
"reason": "Document not found"
});
return;
}
break;
default:
err.message = "Cannot retrieve index array";
that.error(err);
break;
} }
err.message = "Unable to get index database.";
that.error(err);
} }
); );
}; };
f.postDocument = function (index_update_method) {
if (priv.isDocidInIndex(indices, doc) && source === 'POST') { /**
// POST the document already exists * Gets a list containing all the databases set in the storage description.
that.error({ *
"status": 409, * @method getIndexDatabaseList
"statusText": "Conflicts", * @param {Object} option The command option
"error": "conflicts", * @param {Function} callback The result callback(database_list)
"message": "Cannot create a new document", */
"reason": "Document already exists" priv.getIndexDatabaseList = function (option, callback) {
}); var i, count = 0, callbacks = {}, response_list = [];
return; callbacks.error = function (index) {
return function (err) {
if (err.status === 404) {
response_list[index] = new JSONIndex({
"_id": priv.indices[index].id,
"indexing": priv.indices[index].index
});
count += 1;
if (count === priv.indices.length) {
callback(response_list);
}
return;
}
err.message = "Unable to get index database.";
that.error(err);
};
};
callbacks.success = function (index) {
return function (response) {
response_list[index] = new JSONIndex(response);
count += 1;
if (count === priv.indices.length) {
callback(response_list);
}
};
};
for (i = 0; i < priv.indices.length; i += 1) {
that.addJob(
"get",
priv.indices[i].sub_storage || priv.sub_storage,
{"_id": priv.indices[i].id},
option,
callbacks.success(i),
callbacks.error(i)
);
} }
if (source !== 'PUTATTACHMENT') { };
indices = priv.updateIndices(indices, doc);
/**
* Saves all the databases to the remote(s).
*
* @method storeIndexDatabaseList
* @param {Array} database_list The database list
* @param {Object} option The command option
* @param {Function} callback The result callback(err, response)
*/
priv.storeIndexDatabaseList = function (database_list, option, callback) {
var i, count = 0, onResponse, onError;
onResponse = function (response) {
count += 1;
if (count === priv.indices.length) {
callback({"ok": true});
}
};
onError = function (err) {
err.message = "Unable to store index database.";
that.error(err);
};
for (i = 0; i < priv.indices.length; i += 1) {
that.addJob(
"put",
priv.indices[i].sub_storage || priv.sub_storage,
database_list[i].serialized(),
option,
onResponse,
onError
);
} }
};
/**
* A generic request method which delegates the request to the sub storage.
* On response, it will index the document from the request and update all
* the databases.
*
* @method genericRequest
* @param {Command} command The JIO command
* @param {Function} method The request method
*/
priv.genericRequest = function (command, method) {
var doc = command.cloneDoc(), option = command.cloneOption();
that.addJob( that.addJob(
source === 'PUTATTACHMENT' ? "putAttachment" : "post", method,
priv.substorage, priv.sub_storage,
doc, doc,
command.cloneOption(), option,
function () { function (response) {
if (source !== 'PUTATTACHMENT') { switch (method) {
f.sendIndices(index_update_method); case "post":
} else { case "put":
that.success({ case "remove":
"ok": true, doc._id = response.id;
"id": doc._id, priv.getIndexDatabaseList(option, function (database_list) {
"attachment": doc._attachment var i;
}); switch (method) {
} case "post":
}, case "put":
function (err) { for (i = 0; i < database_list.length; i += 1) {
switch (err.status) { database_list[i].put(doc);
case 409: }
// file already exists break;
if (source !== 'PUTATTACHMENT') { case "remove":
f.sendIndices(index_update_method); for (i = 0; i < database_list.length; i += 1) {
} else { database_list[i].remove(doc);
that.success({ }
"ok": true, break;
"id": doc._id default:
break;
}
priv.storeIndexDatabaseList(database_list, option, function () {
that.success({"ok": true, "id": doc._id});
}); });
} });
break; break;
default: default:
err.message = "Cannot upload document"; that.success(response);
that.error(err);
break; break;
} }
}
);
};
f.sendIndices = function (method) {
indices._id = priv.index_suffix;
that.addJob(
method,
priv.substorage,
indices,
command.cloneOption(),
function () {
that.success({
"ok": true,
"id": doc._id
});
}, },
function (err) { function (err) {
// xxx do we try to delete the posted document ? return that.error(err);
err.message = "Cannot save index file";
that.error(err);
} }
); );
}; };
f.getIndices();
};
/** /**
* Update the document metadata and update the index * Post the document metadata and update the index
* @method put * @method post
* @param {object} command The JIO command * @param {object} command The JIO command
*/ */
that.post = function (command) { that.post = function (command) {
priv.postOrPut(command, 'POST'); priv.genericRequest(command, 'post');
}; };
/**
* Update the document metadata and update the index
* @method put
* @param {object} command The JIO command
*/
that.put = function (command) {
priv.postOrPut(command, 'PUT');
};
/**
* Add an attachment to a document (no index modification)
* @method putAttachment
* @param {object} command The JIO command
*/
that.putAttachment = function (command) {
priv.postOrPut(command, 'PUTATTACHMENT');
};
/**
* Get the document metadata
* @method get
* @param {object} command The JIO command
*/
that.get = function (command) {
that.addJob(
"get",
priv.substorage,
command.cloneDoc(),
command.cloneOption(),
function (response) {
that.success(response);
},
function (err) {
that.error(err);
}
);
};
/** /**
* Get the attachment. * Update the document metadata and update the index
* @method getAttachment * @method put
* @param {object} command The JIO command * @param {object} command The JIO command
*/ */
that.getAttachment = function (command) { that.put = function (command) {
that.addJob( priv.genericRequest(command, 'put');
"getAttachment", };
priv.substorage,
command.cloneDoc(),
command.cloneOption(),
function (response) {
that.success(response);
},
function (err) {
that.error(err);
}
);
};
/** /**
* Remove document - removing documents updates index!. * Add an attachment to a document (no index modification)
* @method remove * @method putAttachment
* @param {object} command The JIO command * @param {object} command The JIO command
*/ */
that.remove = function (command) { that.putAttachment = function (command) {
var f = {}, indices, doc, docid, option; priv.genericRequest(command, 'putAttachment');
};
doc = command.cloneDoc(); /**
option = command.cloneOption(); * Get the document metadata
* @method get
* @param {object} command The JIO command
*/
that.get = function (command) {
priv.genericRequest(command, 'get');
};
f.removeDocument = function (type) { /**
that.addJob( * Get the attachment.
"remove", * @method getAttachment
priv.substorage, * @param {object} command The JIO command
doc, */
option, that.getAttachment = function (command) {
function (response) { priv.genericRequest(command, 'getAttachment');
that.success(response);
},
function () {
that.error({
"status": 409,
"statusText": "Conflict",
"error": "conflict",
"message": "Document Update Conflict",
"reason": "Could not delete document or attachment"
});
}
);
}; };
f.getIndices = function () {
that.addJob( /**
"get", * Remove document - removing documents updates index!.
priv.substorage, * @method remove
{"_id": priv.index_suffix}, * @param {object} command The JIO command
option, */
function (response) { that.remove = function (command) {
// if deleting an attachment priv.genericRequest(command, 'remove');
if (typeof command.getAttachmentId() === 'string') {
f.removeDocument('attachment');
} else {
indices = priv.cleanIndices(response, doc);
// store update index file
that.addJob(
"put",
priv.substorage,
indices,
command.cloneOption(),
function () {
// remove actual document
f.removeDocument('doc');
},
function (err) {
err.message = "Cannot save index file";
that.error(err);
}
);
}
},
function () {
that.error({
"status": 404,
"statusText": "Not Found",
"error": "not_found",
"message": "Document index not found, please check document ID",
"reason": "Incorrect document ID"
});
return;
}
);
}; };
f.getIndices();
};
/** /**
* Remove document - removing documents updates index!. * Remove attachment
* @method remove * @method removeAttachment
* @param {object} command The JIO command * @param {object} command The JIO command
*/ */
that.removeAttachment = function (command) { that.removeAttachment = function (command) {
var f = {}, indices, doc, docid, option; priv.genericRequest(command, 'removeAttachment');
doc = command.cloneDoc();
option = command.cloneOption();
f.removeDocument = function (type) {
that.addJob(
"removeAttachment",
priv.substorage,
doc,
option,
that.success,
that.error
);
}; };
f.getIndices = function () {
that.addJob( /**
"get", * Gets a document list from the substorage
priv.substorage, * Options:
{"_id": priv.index_suffix}, * - {boolean} include_docs Also retrieve the actual document content.
option, * @method allDocs
function (response) { * @param {object} command The JIO command
// if deleting an attachment */
if (typeof command.getAttachmentId() === 'string') { that.allDocs = function (command) {
f.removeDocument('attachment'); var option = command.cloneOption(),
} else { index = priv.selectIndex(option.select_list || []);
indices = priv.cleanIndices(response, doc); // Include docs option is ignored, if you want to get all the document,
// store update index file // don't use index storage!
that.addJob(
"put", option.select_list = option.select_list || [];
priv.substorage, option.select_list.push("_id");
indices, priv.getIndexDatabase(option, index, function (db) {
command.cloneOption(), var i, id;
function () { db = db._database;
// remove actual document complex_queries.QueryFactory.create(option.query || '').
f.removeDocument('doc'); exec(db, option);
}, for (i = 0; i < db.length; i += 1) {
function (err) { id = db[i]._id;
err.message = "Cannot save index file"; delete db[i]._id;
that.error(err); db[i] = {
} "id": id,
); "key": id,
} "value": db[i],
}, };
function (err) {
that.error(err);
} }
); that.success({"total_rows": db.length, "rows": db});
});
}; };
f.getIndices();
};
/** // that.repair = function (command) {
* Gets a document list from the substorage // todo: repair
* Options: // easy but don't have time
* - {boolean} include_docs Also retrieve the actual document content. // if _id is an index id, then repair the index by doing an
* @method allDocs // allDocs and recreating the database from scratch. end.
* @param {object} command The JIO command // };
*/
//{ return that;
// "total_rows": 4, }
// "rows": [
// {
// "id": "otherdoc", if (typeof exports === "object") {
// "key": "otherdoc", // nodejs export module
// "value": { Object.defineProperty(exports, "indexStorage", {
// "rev": "1-3753476B70A49EA4D8C9039E7B04254C" configurable: false,
// } enumerable: true,
// },{...} writable: false,
// ] value: indexStorage
//} });
that.allDocs = function (command) { } else if (typeof define === "function" && define.amd) {
var f = {}, option, all_docs_response, query_object, query_syntax, // requirejs export
query_response; define(indexStorage);
option = command.cloneOption(); } else {
// classical browser and web workers JIO export
f.getIndices = function () { jIO.addStorageType("indexed", indexStorage);
that.addJob( }
"get", }());
priv.substorage,
{"_id": priv.index_suffix},
option,
function (response) {
query_syntax = command.getOption('query');
if (query_syntax !== undefined) {
// build complex query object
query_object = priv.constructQueryObject(response, query_syntax);
if (query_object.length === 0) {
that.addJob(
"allDocs",
priv.substorage,
undefined,
option,
that.success,
that.error
);
} else {
// we can use index, run query on index
query_response =
jIO.ComplexQueries.query(query_syntax, query_object);
that.success(query_response);
}
} else if (command.getOption('include_docs')) {
priv.allDocsResponseFromIndex(response, true, option);
} else {
all_docs_response =
priv.allDocsResponseFromIndex(response, false, option);
that.success(all_docs_response);
}
},
that.error
);
};
f.getIndices();
};
return that;
});
test/jiotests.js
View file @ 4c418987
...@@ -4325,56 +4325,62 @@ test ("Post", function () { ...@@ -4325,56 +4325,62 @@ test ("Post", function () {
var o = generateTools(this); var o = generateTools(this);
o.jio = JIO.newJio({ o.jio = JIO.newJio({
"type": "indexed", "type": "indexed",
"indices": [ "indices": [
{"name":"indexA", "fields":["findMeA"]}, {"id": "A", "index": ["title"]},
{"name":"indexAB", "fields":["findMeA","findMeB"]} {"id": "B", "index": ["title", "year"]}
], ],
"field_types": { "sub_storage": {
"findMeA": "string", "type": "local",
"findMeB": "string" "username": "ipost",
}, "application_name": "ipost"
"sub_storage": { }
"type": "local",
"username": "ipost",
"application_name": "ipost"
}
}); });
// post without id // post without id
o.spy (o, "status", undefined, "Post without id"); o.spy (o, "jobstatus", "done", "Post without id");
o.jio.post({}, o.f); o.jio.post({}, function (err, response) {
o.id = (response || {}).id;
o.f(err, response);
});
o.tick(o); o.tick(o);
// post non empty document // post non empty document
o.doc = {"_id": "some_id", "title": "myPost1", o.doc = {"_id": "some_id", "title": "My Title",
"findMeA":"keyword_abc", "findMeB":"keyword_def" "year": 2000, "hey": "def"};
};
o.spy (o, "value", {"ok": true, "id": "some_id"}, "Post document"); o.spy (o, "value", {"ok": true, "id": "some_id"}, "Post document");
o.jio.post(o.doc, o.f); o.jio.post(o.doc, o.f);
o.tick(o); o.tick(o);
// check document // check document
o.fakeIndex = { o.fakeIndexA = {
"_id": "ipost_indices.json", "_id": "A",
"indexAB": { "indexing": ["title"],
"findMeA": { "free": [],
"keyword_abc":["some_id"] "location": {
}, "some_id": 0
"findMeB": {
"keyword_def":["some_id"]
}
}, },
"indexA": { "database": [
"findMeA": { {"_id": "some_id", "title": "My Title"}
"keyword_abc":["some_id"] ]
}
}
}; };
o.jio.get({"_id": "ipost_indices.json"}, function (err, response) { o.spy(o, "value", o.fakeIndexA, "Check index file");
o.actualIndex = response; o.jio.get({"_id": "A"}, o.f);
deepEqual(o.actualIndex, o.fakeIndex, "Check index file"); o.tick(o);
});
o.fakeIndexB = {
"_id": "B",
"indexing": ["title", "year"],
"free": [],
"location": {
"some_id": 0
},
"database": [
{"_id": "some_id", "title": "My Title", "year": 2000}
]
};
o.spy(o, "value", o.fakeIndexB, "Check index file");
o.jio.get({"_id": "B"}, o.f);
o.tick(o); o.tick(o);
// post with escapable characters // post with escapable characters
...@@ -4402,21 +4408,17 @@ test ("Put", function(){ ...@@ -4402,21 +4408,17 @@ test ("Put", function(){
var o = generateTools(this); var o = generateTools(this);
o.jio = JIO.newJio({ o.jio = JIO.newJio({
"type": "indexed", "type": "indexed",
"indices": [ "indices": [
{"name":"indexA", "fields":["author"]}, {"id": "A", "index": ["author"]},
{"name":"indexAB", "fields":["author","year"]} {"id": "B", "index": ["year"]}
], ],
"field_types": { "sub_storage": {
"author": "string", "type": "local",
"year": "number" "username": "iput",
}, "application_name": "iput"
"sub_storage": { }
"type": "local", });
"username": "iput",
"application_name": "iput"
}
});
// put without id // put without id
// error 20 -> document id required // error 20 -> document id required
...@@ -4425,146 +4427,94 @@ test ("Put", function(){ ...@@ -4425,146 +4427,94 @@ test ("Put", function(){
o.tick(o); o.tick(o);
// put non empty document // put non empty document
o.doc = {"_id": "put1", "title": "myPut1", "author":"John Doe"}; o.doc = {"_id": "put1", "title": "myPut1", "author": "John Doe"};
o.spy (o, "value", {"ok": true, "id": "put1"}, "Put-create document"); o.spy (o, "value", {"ok": true, "id": "put1"}, "Put-create document");
o.jio.put(o.doc, o.f); o.jio.put(o.doc, o.f);
o.tick(o); o.tick(o);
// check index file // check index file
o.fakeIndex = { o.fakeIndexA = {
"indexA": { "_id": "A",
"author": { "indexing": ["author"],
"John Doe": ["put1"] "free": [],
} "location": {
}, "put1": 0
"indexAB": {
"author": {
"John Doe": ["put1"]
},
"year": {}
}, },
"_id": "iput_indices.json" "database": [{"_id": "put1", "author": "John Doe"}]
}; };
o.jio.get({"_id": "iput_indices.json"}, function (err, response) { o.spy(o, "value", o.fakeIndexA, "Check index file");
o.actualIndex = response; o.jio.get({"_id": "A"}, o.f);
deepEqual(o.actualIndex, o.fakeIndex, "Check index file"); o.tick(o);
});
o.fakeIndexB = {
"_id": "B",
"indexing": ["year"],
"free": [],
"location": {},
"database": []
};
o.spy(o, "value", o.fakeIndexB, "Check index file");
o.jio.get({"_id": "B"}, o.f);
o.tick(o); o.tick(o);
// modify document - modify keyword on index! // modify document - modify keyword on index!
o.doc = {"_id": "put1", "title": "myPuttter1", "author":"Jane Doe"}; o.doc = {"_id": "put1", "title": "myPuttter1", "author": "Jane Doe"};
o.spy (o, "value", {"ok": true, "id": "put1"}, "Modify existing document"); o.spy (o, "value", {"ok": true, "id": "put1"}, "Modify existing document");
o.jio.put(o.doc, o.f); o.jio.put(o.doc, o.f);
o.tick(o); o.tick(o);
// check index file // check index file
o.fakeIndex = { o.fakeIndexA.database[0].author = "Jane Doe";
"indexA": { o.spy(o, "value", o.fakeIndexA, "Check index file");
"author": { o.jio.get({"_id": "A"}, o.f);
"Jane Doe": ["put1"]
}
},
"indexAB": {
"author": {
"Jane Doe": ["put1"]
},
"year": {}
},
"_id": "iput_indices.json"
};
o.jio.get({"_id": "iput_indices.json"}, function (err, response) {
o.actualIndex = response;
deepEqual(o.actualIndex, o.fakeIndex, "Check index file");
});
o.tick(o); o.tick(o);
// add new document with same keyword! // add new document with same keyword!
o.doc = {"_id": "new_doc", "title": "myPut2", "author":"Jane Doe"}; o.doc = {"_id": "new_doc", "title": "myPut2", "author": "Jane Doe"};
o.spy (o, "value", {"ok": true, "id": "new_doc"}, o.spy (o, "value", {"ok": true, "id": "new_doc"},
"Add new document with same keyword"); "Add new document with same keyword");
o.jio.put(o.doc, o.f); o.jio.put(o.doc, o.f);
o.tick(o); o.tick(o);
// check index file // check index file
o.fakeIndex = { o.fakeIndexA.location.new_doc = 1;
"indexA": { o.fakeIndexA.database.push({"_id": "new_doc", "author": "Jane Doe"});
"author": { o.spy(o, "value", o.fakeIndexA, "Check index file");
"Jane Doe": ["put1", "new_doc"] o.jio.get({"_id": "A"}, o.f);
}
},
"indexAB": {
"author": {
"Jane Doe": ["put1", "new_doc"]
},
"year": {}
},
"_id": "iput_indices.json"
};
o.jio.get({"_id": "iput_indices.json"}, function (err, response) {
o.actualIndex = response;
deepEqual(o.actualIndex, o.fakeIndex, "Check index file");
});
o.tick(o); o.tick(o);
// add second keyword to index file // add second keyword to index file
o.doc = {"_id": "put1", "title": "myPut2", "author":"Jane Doe", o.doc = {"_id": "put1", "title": "myPut2", "author": "Jane Doe",
"year":"1912"}; "year":"1912"};
o.spy (o, "value", {"ok": true, "id": "put1"}, o.spy (o, "value", {"ok": true, "id": "put1"},
"add second keyword to index file"); "add second keyword to index file");
o.jio.put(o.doc, o.f); o.jio.put(o.doc, o.f);
o.tick(o); o.tick(o);
// check index file // check index file
o.fakeIndex = { o.spy(o, "value", o.fakeIndexA, "Check index file");
"indexA": { o.jio.get({"_id": "A"}, o.f);
"author": { o.tick(o);
"Jane Doe": ["put1"]
} o.fakeIndexB.location.put1 = 0;
}, o.fakeIndexB.database.push({"_id": "put1", "year": "1912"});
"indexAB": { o.spy(o, "value", o.fakeIndexB, "Check index file");
"author": { o.jio.get({"_id": "B"}, o.f);
"Jane Doe": ["put1"]
},
"year": {
"1912": ["put1"]
}
},
"_id": "iput_indices.json"
};
o.jio.get({"_id": "iput_indices.json"}, function (err, response) {
o.actualIndex = response;
deepEqual(o.actualIndex, o.fakeIndex, "Check index file");
});
o.tick(o); o.tick(o);
// remove a keyword from an existing document // remove a keyword from an existing document
o.doc = {"_id": "new_doc", "title": "myPut2"}; o.doc = {"_id": "new_doc", "title": "myPut2"};
o.spy (o, "value", {"ok": true, "id": "new_doc"}, o.spy (o, "value", {"ok": true, "id": "new_doc"},
"Remove keyword from existing document"); "Remove keyword from existing document");
o.jio.put(o.doc, o.f); o.jio.put(o.doc, o.f);
o.tick(o); o.tick(o);
// check index file // check index file
o.fakeIndex = { delete o.fakeIndexA.location.new_doc;
"indexA": { o.fakeIndexA.database[1] = null;
"author": { o.fakeIndexA.free.push(1);
"Jane Doe": ["put1"] o.spy(o, "value", o.fakeIndexA, "Check index file");
} o.jio.get({"_id": "A"}, o.f);
},
"indexAB": {
"author": {
"Jane Doe": ["put1"]
},
"year": {
"1912": ["put1"]
}
},
"_id": "iput_indices.json"
};
o.jio.get({"_id": "iput_indices.json"}, function (err, response) {
o.actualIndex = response;
deepEqual(o.actualIndex, o.fakeIndex, "Check index file");
});
o.tick(o); o.tick(o);
o.jio.stop(); o.jio.stop();
...@@ -4578,21 +4528,17 @@ test ("PutAttachment", function(){ ...@@ -4578,21 +4528,17 @@ test ("PutAttachment", function(){
var o = generateTools(this); var o = generateTools(this);
o.jio = JIO.newJio({ o.jio = JIO.newJio({
"type": "indexed", "type": "indexed",
"indices": [ "indices": [
{"name":"indexA", "fields":["author"]}, {"id": "A", "index": ["author"]},
{"name":"indexAB", "fields":["author","year"]} {"id": "B", "index": ["year"]}
], ],
"field_types": { "sub_storage": {
"author": "string", "type": "local",
"year": "number" "username": "iputatt",
}, "application_name": "iputatt"
"sub_storage": { }
"type": "local", });
"username": "iputatt",
"application_name": "iputatt"
}
});
// putAttachment without doc id // putAttachment without doc id
// error 20 -> document id required // error 20 -> document id required
...@@ -4615,7 +4561,7 @@ test ("PutAttachment", function(){ ...@@ -4615,7 +4561,7 @@ test ("PutAttachment", function(){
// putAttachment with document // putAttachment with document
o.doc = {"_id": "putattmt1","title": "myPutAttmt1"}; o.doc = {"_id": "putattmt1","title": "myPutAttmt1"};
o.spy (o, "value", {"ok": true, "id": "putattmt1"}, o.spy (o, "value", {"ok": true, "id": "putattmt1"},
"Put underlying document"); "Put underlying document");
o.jio.put(o.doc, o.f); o.jio.put(o.doc, o.f);
o.tick(o); o.tick(o);
...@@ -4695,21 +4641,17 @@ test ("Get", function(){ ...@@ -4695,21 +4641,17 @@ test ("Get", function(){
var o = generateTools(this); var o = generateTools(this);
o.jio = JIO.newJio({ o.jio = JIO.newJio({
"type": "indexed", "type": "indexed",
"indices": [ "indices": [
{"name":"indexA", "fields":["author"]}, {"id": "A", "index": ["author"]},
{"name":"indexAB", "fields":["author","year"]} {"id": "B", "index": ["year"]}
], ],
"field_types": { "sub_storage": {
"author": "string", "type": "local",
"year": "number" "username": "iget",
}, "application_name": "iget"
"sub_storage": { }
"type": "local", });
"username": "iget",
"application_name": "iget"
}
});
// get inexistent document // get inexistent document
o.spy(o, "status", 404, "Get inexistent document"); o.spy(o, "status", 404, "Get inexistent document");
...@@ -4767,13 +4709,9 @@ test ("Remove", function(){ ...@@ -4767,13 +4709,9 @@ test ("Remove", function(){
o.jio = JIO.newJio({ o.jio = JIO.newJio({
"type": "indexed", "type": "indexed",
"indices": [ "indices": [
{"name":"indexA", "fields":["author"]}, {"id": "A", "index": ["author"]},
{"name":"indexAB", "fields":["author","year"]} {"id": "B", "index": ["year"]}
], ],
"field_types": {
"author": "string",
"year": "number"
},
"sub_storage": { "sub_storage": {
"type": "local", "type": "local",
"username": "irem", "username": "irem",
...@@ -4809,26 +4747,30 @@ test ("Remove", function(){ ...@@ -4809,26 +4747,30 @@ test ("Remove", function(){
o.tick(o); o.tick(o);
// check index // check index
o.fakeIndex = { o.fakeIndexA = {
"_id": "irem_indices.json", "_id": "A",
"indexA": { "indexing": ["author"],
"author": { "free": [0],
"Martin Mustermann": ["removeAlso"] "location": {
} "removeAlso": 1
}, },
"indexAB": { "database": [null, {"_id": "removeAlso", "author": "Martin Mustermann"}]
"year": {
"2525": ["removeAlso"]
},
"author": {
"Martin Mustermann": ["removeAlso"]
}
}
}; };
o.jio.get({"_id": "irem_indices.json"},function(err, response){ o.spy(o, "value", o.fakeIndexA, "Check index file");
o.actualIndex = response; o.jio.get({"_id": "A"}, o.f);
deepEqual(o.actualIndex, o.fakeIndex, "Check index file"); o.tick(o);
});
o.fakeIndexB = {
"_id": "B",
"indexing": ["year"],
"free": [0],
"location": {
"removeAlso": 1
},
"database": [null, {"_id": "removeAlso", "year": "2525"}]
};
o.spy(o, "value", o.fakeIndexB, "Check index file");
o.jio.get({"_id": "B"}, o.f);
o.tick(o); o.tick(o);
// check document // check document
...@@ -4868,29 +4810,18 @@ test ("Remove", function(){ ...@@ -4868,29 +4810,18 @@ test ("Remove", function(){
o.tick(o); o.tick(o);
// check index // check index
o.fakeIndex = { o.fakeIndexA.free = [];
"_id": "irem_indices.json", o.fakeIndexA.location.remove3 = 0;
"indexA": { o.fakeIndexA.database[0] = {"_id": "remove3", "author": "Mrs Sunshine"};
"author":{ o.spy(o, "value", o.fakeIndexA, "Check index file");
"Martin Mustermann": ["removeAlso"], o.jio.get({"_id": "A"}, o.f);
"Mrs Sunshine": ["remove3"] o.tick(o);
}
}, o.fakeIndexB.free = [];
"indexAB": { o.fakeIndexB.location.remove3 = 0;
"year": { o.fakeIndexB.database[0] = {"_id": "remove3", "year": "1234"};
"1234": ["remove3"], o.spy(o, "value", o.fakeIndexB, "Check index file");
"2525": ["removeAlso"] o.jio.get({"_id": "B"}, o.f);
},
"author": {
"Martin Mustermann": ["removeAlso"],
"Mrs Sunshine": ["remove3"]
}
}
};
o.jio.get({"_id": "irem_indices.json"}, function (err, response) {
o.actualIndex = response;
deepEqual(o.actualIndex, o.fakeIndex, "Check index file");
});
o.tick(o); o.tick(o);
// remove document and attachment together // remove document and attachment together
...@@ -4900,26 +4831,18 @@ test ("Remove", function(){ ...@@ -4900,26 +4831,18 @@ test ("Remove", function(){
o.tick(o); o.tick(o);
// check index // check index
o.fakeIndex = { o.fakeIndexA.free = [0];
"_id": "irem_indices.json", delete o.fakeIndexA.location.remove3;
"indexA": { o.fakeIndexA.database[0] = null;
"author": { o.spy(o, "value", o.fakeIndexA, "Check index file");
"Martin Mustermann": ["removeAlso"] o.jio.get({"_id": "A"}, o.f);
} o.tick(o);
},
"indexAB": { o.fakeIndexB.free = [0];
"year": { delete o.fakeIndexB.location.remove3;
"2525": ["removeAlso"] o.fakeIndexB.database[0] = null;
}, o.spy(o, "value", o.fakeIndexB, "Check index file");
"author": { o.jio.get({"_id": "B"}, o.f);
"Martin Mustermann": ["removeAlso"]
}
}
};
o.jio.get({"_id": "irem_indices.json"}, function (err, response) {
o.actualIndex = response;
deepEqual(o.actualIndex, o.fakeIndex, "Check index file");
});
o.tick(o); o.tick(o);
// check attachment // check attachment
...@@ -4939,22 +4862,18 @@ test ("AllDocs", function () { ...@@ -4939,22 +4862,18 @@ test ("AllDocs", function () {
var o = generateTools(this); var o = generateTools(this);
o.jio = JIO.newJio({ o.jio = JIO.newJio({
"type": "indexed", "type": "indexed",
"indices": [ "indices": [
{"name":"indexA", "fields":["author"]}, {"id": "A", "index": ["author"]},
{"name":"indexAB", "fields":["author","year"]} {"id": "B", "index": ["year"]}
], ],
"field_types": { "sub_storage": {
"author": "string", "type": "local",
"year": "number" "username": "iall",
}, "application_name": "iall"
"sub_storage": { }
"type": "local", });
"username": "iall",
"application_name": "iall"
}
});
// adding documents // adding documents
o.all1 = { "_id": "dragon.doc", o.all1 = { "_id": "dragon.doc",
...@@ -4983,34 +4902,25 @@ test ("AllDocs", function () { ...@@ -4983,34 +4902,25 @@ test ("AllDocs", function () {
o.tick(o); o.tick(o);
// check index // check index
o.fakeIndex = { o.fakeIndexA = {
"_id": "iall_indices.json", "_id": "A",
"indexA": { "indexing": ["author"],
"author": { "free": [],
"Dr. No": ["dragon.doc"], "location": {
"Dr. Who": ["timemachine"], "dragon.doc": 0,
"Dr. Snuggles": ["rocket.ppt"], "timemachine": 1,
"Dr. House":["stick.jpg"] "rocket.ppt": 2,
} "stick.jpg": 3
}, },
"indexAB": { "database": [
"author": { {"_id": "dragon.doc", "author": "Dr. No"},
"Dr. No": ["dragon.doc"], {"_id": "timemachine", "author": "Dr. Who"},
"Dr. Who": ["timemachine"], {"_id": "rocket.ppt", "author": "Dr. Snuggles"},
"Dr. Snuggles": ["rocket.ppt"], {"_id": "stick.jpg", "author": "Dr. House"}
"Dr. House":["stick.jpg"] ]
},
"year": {
"1968": ["dragon.doc", "timemachine"],
"1985": ["rocket.ppt"],
"2005":["stick.jpg"]
}
}
}; };
o.jio.get({"_id": "iall_indices.json"}, function (err, response) { o.spy(o, "value", o.fakeIndexA, "Check index file");
o.actualIndex = response; o.jio.get({"_id": "A"}, o.f);
deepEqual(o.actualIndex, o.fakeIndex, "Check index file");
});
o.tick(o); o.tick(o);
o.thisShouldBeTheAnswer = { o.thisShouldBeTheAnswer = {
...@@ -5026,19 +4936,6 @@ test ("AllDocs", function () { ...@@ -5026,19 +4936,6 @@ test ("AllDocs", function () {
o.jio.allDocs(o.f); o.jio.allDocs(o.f);
o.tick(o); o.tick(o);
o.thisShouldBeTheAnswer2 = {
"rows": [
{"id": "dragon.doc", "key": "dragon.doc", "value": {}, "doc": o.all1 },
{"id": "timemachine", "key": "timemachine", "value": {}, "doc": o.all2 },
{"id": "rocket.ppt", "key": "rocket.ppt", "value": {}, "doc": o.all3 },
{"id": "stick.jpg", "key": "stick.jpg", "value": {}, "doc": o.all4 }
],
"total_rows": 4
}
o.spy(o, "value", o.thisShouldBeTheAnswer2, "allDocs (include_docs)");
o.jio.allDocs({"include_docs":true}, o.f);
o.tick(o);
o.jio.stop(); o.jio.stop();
}); });
...@@ -5049,16 +4946,11 @@ test ("AllDocs Complex Queries", function () { ...@@ -5049,16 +4946,11 @@ test ("AllDocs Complex Queries", function () {
o.jio = JIO.newJio({ o.jio = JIO.newJio({
"type": "indexed", "type": "indexed",
"indices": [ "indices": [
{"name":"indexA", "fields":["director"]}, {"id":"A", "index": ["director"]},
{"name":"indexAB", "fields":["title","year"]} {"id":"B", "index": ["title", "year"]}
//, //,
//{"name":"indexABC", "fields":["title","year","director"]} //{"name":"indexABC", "fields":["title","year","director"]}
], ],
"field_types": {
"director": "string",
"title": "string",
"year": "number"
},
"sub_storage": { "sub_storage": {
"type": "local", "type": "local",
"username": "icomplex", "username": "icomplex",
...@@ -5083,7 +4975,7 @@ test ("AllDocs Complex Queries", function () { ...@@ -5083,7 +4975,7 @@ test ("AllDocs Complex Queries", function () {
"Sidney Lumet", "Christopher Nolan", "Steven Spielberg", "Sidney Lumet", "Christopher Nolan", "Steven Spielberg",
"Peter Jackson", "David Fincher", "Irvin Kershner", "Peter Jackson", "Peter Jackson", "David Fincher", "Irvin Kershner", "Peter Jackson",
"Milos Forman", "Christopher Nolan", " Martin Scorsese" "Milos Forman", "Christopher Nolan", " Martin Scorsese"
] ];
for (i = 0; i < m; i += 1) { for (i = 0; i < m; i += 1) {
o.fakeDoc = {}; o.fakeDoc = {};
...@@ -5094,6 +4986,7 @@ test ("AllDocs Complex Queries", function () { ...@@ -5094,6 +4986,7 @@ test ("AllDocs Complex Queries", function () {
o.jio.put(o.fakeDoc); o.jio.put(o.fakeDoc);
o.clock.tick(1000); o.clock.tick(1000);
} }
// o.clock.tick(1000);
// response // response
o.allDocsResponse = {}; o.allDocsResponse = {};
...@@ -5101,123 +4994,90 @@ test ("AllDocs Complex Queries", function () { ...@@ -5101,123 +4994,90 @@ test ("AllDocs Complex Queries", function () {
o.allDocsResponse.total_rows = 15; o.allDocsResponse.total_rows = 15;
for (i = 0; i < m; i += 1) { for (i = 0; i < m; i += 1) {
o.allDocsResponse.rows.push({ o.allDocsResponse.rows.push({
"id": ""+i,
"key": ""+i,
"value": {}
});
};
// alldocs
o.jio.allDocs(function (e, r) {
var x = r.rows.sort(o.sortArrayById('id', true, parseInt));
deepEqual(
{"total_rows":r.total_rows,"rows":x}, o.allDocsResponse,
"AllDocs response generated from index"
);
});
o.clock.tick(1000);
// include docs
o.allDocsResponse2 = {};
o.allDocsResponse2.rows = [];
o.allDocsResponse2.total_rows = 15;
for (i = 0; i < m; i += 1) {
o.allDocsResponse2.rows.push({
"id": ""+i, "id": ""+i,
"key": ""+i, "key": ""+i,
"value": {}, "value": {},
"doc": localstorage.getItem(o.localpath+"/"+i) "doc": {
"_id": ""+i,
"title": o.titles[i],
"year": o.years[i],
"director": o.director[i]
}
}); });
}; }
o.response = JSON.parse(JSON.stringify(o.allDocsResponse));
for (i = 0; i < o.response.rows.length; i += 1) {
delete o.response.rows[i].doc;
}
// alldocs // alldocs
o.jio.allDocs({"include_docs":true}, function(e,r) { o.spy(o, "value", o.response, "AllDocs response generated from index");
var x = r.rows.sort(o.sortArrayById('id', true, parseInt)); o.jio.allDocs(o.f);
deepEqual( o.tick(o, 1000);
{"total_rows":r.total_rows,"rows":x}, o.allDocsResponse2,
"AllDocs response generated from index (include docs)"
);
});
o.clock.tick(1000);
// complex queries // complex queries
o.thisShouldBeTheAnswer4 = [ o.response = JSON.parse(JSON.stringify(o.allDocsResponse));
{"title": "Inception", "year": 2010}, i = 0;
{"title": "The Dark Knight", "year": 2008}, while (i < o.response.rows.length) {
{"title": "Lord of the Rings - Return of the King", "year": 2003}, if (o.response.rows[i].year < 1980) {
{"title": "Lord Of the Rings - Fellowship of the Ring", "year": 2001}, o.response.rows.splice(i, 1);
{"title": "Fight Club", "year": 1999} } else {
]; o.response.rows[i].value = {
o.spy(o, "value", o.thisShouldBeTheAnswer4, "year": o.response.rows[i].doc.year,
"title": o.response.rows[i].doc.title
}
delete o.response.rows[i].doc;
i += 1;
}
}
o.response.rows.sort(function (a, b) {
return a.value.year > b.value.year ? -1 :
a.value.year < b.value.year ? 1 : 0;
});
o.response.rows.length = 5;
o.response.total_rows = 5;
o.spy(o, "value", o.response,
"allDocs (complex queries year >= 1980, index used to do query)"); "allDocs (complex queries year >= 1980, index used to do query)");
o.jio.allDocs({ o.jio.allDocs({
"query":{ // "query":'(year: >= "1980" AND year: < "2000")',
// "query":'(year: >= "1980" AND year: < "2000")', "query": '(year: >= "1980")',
"query":'(year: >= "1980")', "limit": [0, 5],
"filter": { "sort_on": [['year', 'descending']],
"limit":[0,5], "select_list": ['title', 'year']
"sort_on":[['year','descending']],
"select_list":['title','year']
},
"wildcard_character":'%'
}
}, o.f); }, o.f);
o.tick(o); o.tick(o);
// complex queries // complex queries
o.thisShouldBeTheAnswer5 = [ o.spy(o, "value", {"total_rows": 0, "rows": []},
{"director": "Christopher Nolan", "year": 2010},
{"director": "Christopher Nolan", "year": 2008},
{"director": "Peter Jackson", "year": 2003},
{"director": "Peter Jackson", "year": 2001},
{"director": "David Fincher", "year": 1999}
];
o.spy(o, "value", o.thisShouldBeTheAnswer5,
"allDocs (complex queries year >= 1980, can't use index)"); "allDocs (complex queries year >= 1980, can't use index)");
o.jio.allDocs({ o.jio.allDocs({
"query":{ // "query":'(year: >= "1980" AND year: < "2000")',
// "query":'(year: >= "1980" AND year: < "2000")', "query": '(year: >= "1980")',
"query":'(year: >= "1980")', "limit": [0, 5],
"filter": { "sort_on": [['year','descending']],
"limit":[0,5], "select_list": ['director', 'year']
"sort_on":[['year','descending']],
"select_list":['director','year']
},
"wildcard_character":'%'
}
}, o.f); }, o.f);
o.tick(o); o.tick(o);
// empty query returns all // empty query returns all
o.thisShouldBeTheAnswer6 = [ o.response = JSON.parse(JSON.stringify(o.allDocsResponse));
{"title": "The Good, The Bad and The Ugly"}, i = 0;
{"title": "The Dark Knight"}, while (i < o.response.rows.length) {
{"title": "Star Wars Episode V"}, o.response.rows[i].value.title =
{"title": "Shawshank Redemption"}, o.response.rows[i].doc.title;
{"title": "Schindlers List"}, delete o.response.rows[i].doc;
{"title": "Pulp Fiction"}, i += 1;
{"title": "One flew over the Cuckoo's Nest"}, }
{"title": "Lord of the Rings - Return of the King"}, o.response.rows.sort(function (a, b) {
{"title": "Lord Of the Rings - Fellowship of the Ring"}, return a.value.title > b.value.title ? -1 :
{"title": "Inception"}, a.value.title < b.value.title ? 1 : 0;
{"title": "Godfellas"}, });
{"title": "Godfather 2"}, o.spy(o, "value", o.response,
{"title": "Godfather"},
{"title": "Fight Club"},
{"title": "12 Angry Men"}
];
o.spy(o, "value", o.thisShouldBeTheAnswer6,
"allDocs (empty query in complex query)"); "allDocs (empty query in complex query)");
o.jio.allDocs({ o.jio.allDocs({
"query":{ "sort_on":[['title','descending']],
"filter": { "select_list":['title']
"sort_on":[['title','descending']],
"select_list":['title']
},
"wildcard_character":'%'
}
}, o.f); }, o.f);
o.tick(o); o.tick(o);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7