Skip to content

J
jio
Commit 555c5b3d authored by Sven Franck's avatar Sven Franck
Browse files
Options

indexStorage: fixed ALLDOCS complex queries, added QUnit tests

parent d2c42f76
Show whitespace changes
Inline Side-by-side
Showing with 365 additions and 79 deletions
src/jio.storage/indexstorage.js
View file @ 555c5b3d
......@@ -10,6 +10,10 @@
* {"indexA",["field_A"]},
* {"indexAB",["field_A","field_B"]}
* ],
* "field_types": {
* "field_A": "dateTime",
* "field_B": "string"
* },
* "storage": [
* <sub storage description>,
* ...
......@@ -17,7 +21,7 @@
* }
* Index file will contain
* {
* "_id": "ipost_indices.json",
* "_id": "app-name_indices.json",
* "indexA":
* "fieldA": {
* "keyword_abc": ["some_id","some_other_id",...]
......@@ -54,6 +58,7 @@ jIO.addStorageType('indexed', function (spec, my) {
that = my.basicStorage(spec, my);
priv.indices = spec.indices;
priv.field_types = spec.field_types;
priv.substorage_key = "sub_storage";
priv.substorage = spec[priv.substorage_key];
priv.index_indicator = spec.sub_storage.application_name || "index";
......@@ -164,8 +169,8 @@ jIO.addStorageType('indexed', function (spec, my) {
* @returns {number} i Position of element in array
*/
priv.getPositionInArray = function (element, array) {
var i;
for (i = 0; i < array.length; i += 1) {
var i, l = array.length;
for (i = 0; i < l; i += 1) {
if (array[i] === element) {
return i;
}
......@@ -291,36 +296,45 @@ jIO.addStorageType('indexed', function (spec, my) {
};
/**
* Check available indices to find the best one. This index must have
* all "id" parameters of the query and if it also contains all values from
* the select-list, it can be used to run the whole query plus return results
* Check available indices to find the best one.
* TODOS: NOT NICE, redo
* @method findBestIndexForQuery
* @param {object} indices The index file
* @param {object} syntax of query
* @returns {object} response The query object constructed from Index file
*/
priv.findBestIndexForQuery = function (indices, syntax) {
var i, j, k, l, m, n, o, p,
search_ids = [],
select_ids = syntax.filter.select_list,
index, query_param, search_param, use_index = [];
// array of necessary query ids
if (syntax.query.query_list === undefined) {
search_ids.push(syntax.query.id);
} else {
for (j = 0; j < syntax.query.query_list.length; j += 1) {
search_ids.push(syntax.query.query_list[j].id);
}
}
priv.findBestIndexForQuery = function (syntax) {
var i, j, k, l, n, p, o, element, key, block,
search_ids, use_index = [], select_ids = {}, index, query_param,
// need to parse into object
current_query = jIO.ComplexQueries.parse(syntax.query);
// loop indices
for (i = 0; i < priv.indices.length; i += 1) {
search_ids = [];
block = false;
index = {};
index.reference = priv.indices[i];
index.reference_size = index.reference.fields.length;
o = search_ids.length;
for (k = 0; k < o; k += 1) {
// always check the first element in the array because it's spliced
// rebuild search_ids for iteration
if (current_query.query_list === undefined) {
search_ids.push(current_query.id);
} else {
for (j = 0; j < current_query.query_list.length; j += 1) {
if (priv.getPositionInArray(current_query.query_list[j].id,
search_ids) === null) {
search_ids.push(current_query.query_list[j].id);
}
}
}
// rebuild select_ids
for (o = 0; o < syntax.filter.select_list.length; o += 1) {
element = syntax.filter.select_list[o];
select_ids[element] = true;
}
// loop search ids and find matches in index
for (k = 0; k < search_ids.length; k += 1) {
query_param = search_ids[0];
for (l = 0; l < index.reference_size; l += 1) {
if (query_param === index.reference.fields[l]) {
......@@ -332,40 +346,34 @@ jIO.addStorageType('indexed', function (spec, my) {
}
}
// empty search_ids = index can be used, now check results
// search_ids empty = all needed search fields found on index
if (search_ids.length === 0) {
// can't return results if empty select_list (all fields)
if (select_ids.length === 0) {
p = priv.getObjectSize(select_ids);
if (p === 0) {
use_index.push({
"name": index.reference.name,
"search": true,
"results": false
});
} else {
p = select_ids.length;
for (m = 0; m < p; m += 1) {
search_param = select_ids[0];
for (n = 0; n < index.reference_size; n += 1) {
if (search_param === index.reference.fields[n]) {
select_ids.splice(
priv.getPositionInArray(search_param, select_ids),
1
);
}
delete select_ids[index.reference.fields[n]];
}
}
// empty select_list = index can do do query and results!
if (select_ids.length === 0) {
for (key in select_ids) {
if (select_ids.hasOwnProperty(key)) {
use_index.push({
"name": index.reference.name,
"search": true,
"results": true
"results": false
});
} else {
block = true;
}
}
if (block === false) {
use_index.push({
"name": index.reference.name,
"search": true,
"results": false
"results": true
});
}
}
......@@ -376,13 +384,78 @@ jIO.addStorageType('indexed', function (spec, my) {
/**
* Converts the indices file into an object usable by complex queries
* @method convertIndicesToQueryObject
* @method constructQueryObject
* @param {object} indices The index file
* @returns {object} response The query object constructed from Index file
*/
priv.convertIndicesToQueryObject = function (indices, query_syntax) {
var use_index = priv.findBestIndexForQuery(indices, query_syntax);
return indices;
priv.constructQueryObject = function (indices, query_syntax) {
var j, k, l, m, n, use_index, index,
index_name, field_names, field, key, element,
query_index, query_object = [], field_name,
entry;
// returns index-to-use|can-do-query|can-do-query-and-results
use_index = priv.findBestIndexForQuery(query_syntax);
if (use_index.length > 0) {
for (j = 0; j < use_index.length; j += 1) {
index = use_index[j];
// NOTED: the index could be used to:
// (a) get all document ids matching query
// (b) get all document ids and results (= run complex query on index)
// right now, only (b) is supported, because the complex query is
// a single step process. If it was possible to first get the
// relevant document ids, then get the results, the index could be
// used to do the first step plus use GET on the returned documents
if (index.search && index.results) {
index_name = use_index[j].name;
query_index = indices[index_name];
// get fieldnames from this index
for (k = 0; k < priv.indices.length; k += 1) {
if (priv.indices[k].name === use_index[j].name) {
field_names = priv.indices[k].fields;
}
}
for (l = 0; l < field_names.length; l += 1) {
field_name = field_names[l];
// loop entries for this field name
field = query_index[field_name];
for (key in field) {
if (field.hasOwnProperty(key)) {
element = field[key];
// key can be "string" or "number" right now
if (priv.field_types[field_name] === "number") {
key = +key;
}
for (m = 0; m < element.length; m += 1) {
if (priv.searchIndexByValue(
query_object,
element[m],
"bool"
)) {
// loop object
for (n = 0; n < query_object.length; n += 1) {
entry = query_object[n];
if (entry.id === element[m]) {
entry[field_name] = key;
}
}
} else {
entry = {};
entry.id = element[m];
entry[field_name] = key;
query_object.push(entry);
}
}
}
}
}
}
}
}
return query_object;
};
/**
* Build the alldocs response from the index file (overriding substorage)
......@@ -490,7 +563,7 @@ jIO.addStorageType('indexed', function (spec, my) {
* @param {object} command The JIO command
* @param {string} source The source of the function call
*/
priv.postOrput = function (command, source) {
priv.postOrPut = function (command, source) {
var f = {}, indices, doc, docid;
doc = command.cloneDoc();
docid = command.getDocId();
......@@ -618,7 +691,7 @@ jIO.addStorageType('indexed', function (spec, my) {
* @param {object} command The JIO command
*/
that.post = function (command) {
priv.postOrput(command, 'POST');
priv.postOrPut(command, 'POST');
};
/**
......@@ -627,7 +700,7 @@ jIO.addStorageType('indexed', function (spec, my) {
* @param {object} command The JIO command
*/
that.put = function (command) {
priv.postOrput(command, 'PUT');
priv.postOrPut(command, 'PUT');
};
/**
......@@ -636,7 +709,7 @@ jIO.addStorageType('indexed', function (spec, my) {
* @param {object} command The JIO command
*/
that.putAttachment = function (command) {
priv.postOrput(command, 'PUTATTACHMENT');
priv.postOrPut(command, 'PUTATTACHMENT');
};
/**
......@@ -782,7 +855,8 @@ jIO.addStorageType('indexed', function (spec, my) {
// ]
//}
that.allDocs = function (command) {
var f = {}, option, all_docs_response, query_object, query_syntax;
var f = {}, option, all_docs_response, query_object, query_syntax,
query_response;
option = command.cloneOption();
if (option.max_retry === 0) {
option.max_retry = 3;
......@@ -795,14 +869,61 @@ jIO.addStorageType('indexed', function (spec, my) {
priv.index_suffix,
option,
function (response) {
// should include_docs be possible besides complex queries?
query_syntax = command.getOption('query');
if (query_syntax !== undefined) {
// check to see if index can do the job
query_object = priv.convertIndicesToQueryObject(
response,
query_syntax
// build complex query object
query_object = priv.constructQueryObject(response, query_syntax);
if (query_object.length === 0) {
that.addJob(
"allDocs",
priv.substorage,
undefined,
option,
function (data) {
that.success(data);
},
function (err) {
switch (err.status) {
case 405:
that.error({
"status": 405,
"statusText": "Method Not Allowed",
"error": "method_not_allowed",
"message": "Method not allowed",
"reason": "Could not run AllDocs on this storage"
});
break;
default:
that.error({
"status": 404,
"statusText": "Not Found",
"error": "not_found",
"message": "Could not run allDocs command",
"reason": "There are no documents in the storage"
});
break;
}
}
);
} else {
// we can use index, run query on index
query_response =
jIO.ComplexQueries.query({
query: query_syntax.query,
filter: {
sort_on: query_syntax.filter.sort_on,
limit: query_syntax.filter.limit,
select_list: query_syntax.filter.select_list
},
wildcard_character: query_syntax.wildcard_character
},
query_object
);
that.success(query_response);
}
} else if (command.getOption('include_docs')) {
priv.allDocsResponseFromIndex(response, true, option);
} else {
......
test/jiotests.js
View file @ 555c5b3d
......@@ -244,7 +244,15 @@ generateTools = function (sinon) {
o.server.respondWith(method, url,
[status, { "Content-Type": 'application/xml' }, response]
);
};
o.sortArrayById = function(field, reverse, primer){
var key = function (x) {return primer ? primer(x[field]) : x[field]};
return function (a,b) {
var A = key(a), B = key(b);
return ( (A < B) ? -1 : ((A > B) ? 1 : 0) ) * [-1,1][+!!reverse];
}
};
return o;
},
......@@ -1097,7 +1105,7 @@ test ("AllDocs", function(){
"Peter Jackson", "David Fincher", "Irvin Kershner", "Peter Jackson",
"Milos Forman", "Christopher Nolan", " Martin Scorsese"
]
// set documents
for (i = 0; i < m; i += 1) {
o.fakeDoc = {};
o.fakeDoc._id = "doc_"+i;
......@@ -2790,6 +2798,10 @@ test ("Post", function () {
{"name":"indexA", "fields":["findMeA"]},
{"name":"indexAB", "fields":["findMeA","findMeB"]}
],
"field_types": {
"findMeA": "string",
"findMeB": "string"
},
"sub_storage": {
"type": "local",
"username": "ipost",
......@@ -2863,6 +2875,10 @@ test ("Put", function(){
{"name":"indexA", "fields":["author"]},
{"name":"indexAB", "fields":["author","year"]}
],
"field_types": {
"author": "string",
"year": "number"
},
"sub_storage": {
"type": "local",
"username": "iput",
......@@ -3035,6 +3051,10 @@ test ("PutAttachment", function(){
{"name":"indexA", "fields":["author"]},
{"name":"indexAB", "fields":["author","year"]}
],
"field_types": {
"author": "string",
"year": "number"
},
"sub_storage": {
"type": "local",
"username": "iputatt",
......@@ -3142,6 +3162,10 @@ test ("Get", function(){
{"name":"indexA", "fields":["author"]},
{"name":"indexAB", "fields":["author","year"]}
],
"field_types": {
"author": "string",
"year": "number"
},
"sub_storage": {
"type": "local",
"username": "iget",
......@@ -3208,6 +3232,10 @@ test ("Remove", function(){
{"name":"indexA", "fields":["author"]},
{"name":"indexAB", "fields":["author","year"]}
],
"field_types": {
"author": "string",
"year": "number"
},
"sub_storage": {
"type": "local",
"username": "irem",
......@@ -3372,6 +3400,10 @@ test ("AllDocs", function () {
{"name":"indexA", "fields":["author"]},
{"name":"indexAB", "fields":["author","year"]}
],
"field_types": {
"author": "string",
"year": "number"
},
"sub_storage": {
"type": "local",
"username": "iall",
......@@ -3461,24 +3493,157 @@ test ("AllDocs", function () {
o.spy(o, "value", o.thisShouldBeTheAnswer2, "allDocs (include_docs)");
o.jio.allDocs({"include_docs":true}, o.f);
o.tick(o);
/*
o.jio.stop();
});
test ("AllDocs Complex Queries", function () {
var o = generateTools(this), i, m = 15;
o.jio = JIO.newJio({
"type": "indexed",
"indices": [
{"name":"indexA", "fields":["director"]},
{"name":"indexAB", "fields":["title","year"]}
//,
//{"name":"indexABC", "fields":["title","year","director"]}
],
"field_types": {
"director": "string",
"title": "string",
"year": "number"
},
"sub_storage": {
"type": "local",
"username": "icomplex",
"application_name": "acomplex"
}
});
o.localpath = "jio/localstorage/icomplex/acomplex";
// sample data
o.titles = ["Shawshank Redemption", "Godfather", "Godfather 2",
"Pulp Fiction", "The Good, The Bad and The Ugly", "12 Angry Men",
"The Dark Knight", "Schindlers List",
"Lord of the Rings - Return of the King", "Fight Club",
"Star Wars Episode V", "Lord Of the Rings - Fellowship of the Ring",
"One flew over the Cuckoo's Nest", "Inception", "Godfellas"
];
o.years = [1994,1972,1974,1994,1966,1957,2008,1993,2003,1999,1980,2001,
1975,2010,1990
];
o.director = ["Frank Darabont", "Francis Ford Coppola",
"Francis Ford Coppola", "Quentin Tarantino", "Sergio Leone",
"Sidney Lumet", "Christopher Nolan", "Steven Spielberg",
"Peter Jackson", "David Fincher", "Irvin Kershner", "Peter Jackson",
"Milos Forman", "Christopher Nolan", " Martin Scorsese"
]
for (i = 0; i < m; i += 1) {
o.fakeDoc = {};
o.fakeDoc._id = ""+i;
o.fakeDoc.title = o.titles[i];
o.fakeDoc.year = o.years[i];
o.fakeDoc.director = o.director[i];
o.jio.put(o.fakeDoc);
o.clock.tick(1000);
}
// response
o.allDocsResponse = {};
o.allDocsResponse.rows = [];
o.allDocsResponse.total_rows = 15;
for (i = 0; i < m; i += 1) {
o.allDocsResponse.rows.push({
"id": ""+i,
"key": ""+i,
"value": {}
});
};
// alldocs
o.jio.allDocs(function (e, r) {
var x = r.rows.sort(o.sortArrayById('id', true, parseInt));
deepEqual(
{"total_rows":r.total_rows,"rows":x}, o.allDocsResponse,
"AllDocs response generated from index"
);
});
o.clock.tick(1000);
// include docs
o.allDocsResponse2 = {};
o.allDocsResponse2.rows = [];
o.allDocsResponse2.total_rows = 15;
for (i = 0; i < m; i += 1) {
o.allDocsResponse2.rows.push({
"id": ""+i,
"key": ""+i,
"value": {},
"doc": localstorage.getItem(o.localpath+"/"+i)
});
};
// alldocs
o.jio.allDocs({"include_docs":true}, function(e,r) {
var x = r.rows.sort(o.sortArrayById('id', true, parseInt));
deepEqual(
{"total_rows":r.total_rows,"rows":x}, o.allDocsResponse2,
"AllDocs response generated from index (include docs)"
);
});
o.clock.tick(1000);
// complex queries
o.thisShouldBeTheAnswer4 = [
{"title": "Inceptions", "year": 2010},
{"title": "The Dark Knight", "year": 2008},
{"title": "Lord of the Rings - Return of the King", "year": 2003},
{"title": "Lord Of the Rings - Fellowship of the Ring", "year": 2001},
{"title": "Fight Club", "year": 1999}
];
o.spy(o, "value", o.thisShouldBeTheAnswer4,
"allDocs (complex queries year >= 1980, index used to do query)");
o.jio.allDocs({
"query":{
// "query":'(year: >= "1980" AND year: < "2000")',
"query":'(year: >= "1980")',
"filter": {
"limit":[0,5],
"sort_on":[['year','descending']],
"select_list":['title','year']
},
"wildcard_character":'%'
}
}, o.f);
o.tick(o);
// complex queries
o.thisShouldBeTheAnswer3 = {"nothing here":"yet"}
o.spy(o, "value", o.thisShouldBeTheAnswer3,
"allDocs (complex queries year >= 1985)");
o.thisShouldBeTheAnswer5 = [
{"director": "Christopher Nolan", "year": 2010},
{"director": "Christopher Nolan", "year": 2008},
{"director": "Peter Jackson", "year": 2003},
{"director": "Peter Jackson", "year": 2001},
{"director": "David Fincher", "year": 1999}
];
o.spy(o, "value", o.thisShouldBeTheAnswer5,
"allDocs (complex queries year >= 1980, can't use index)");
o.jio.allDocs({
"query":{
"query":jIO.ComplexQueries.parse('(year: >= "1985" AND author:"D%")'),
// "query":'(year: >= "1980" AND year: < "2000")',
"query":'(year: >= "1980")',
"filter": {
"limit":[0,2],
"sort_on":[['key','descending']],
"select_list":['author','year']
"limit":[0,5],
"sort_on":[['year','descending']],
"select_list":['director','year']
},
"wildcard_character":'%'
}
}, o.f);
o.tick(o);
*/
o.jio.stop();
});
/*
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7