feat(storage): add elasticlunr storage filter on querystorage

Makefile

@@ -103,6 +103,7 @@ ${JIOVERSION}: ${EXTERNALDIR}/URI.js \
 	${EXTERNALDIR}/uritemplate.js \
 	${EXTERNALDIR}/lz-string.js \
 	${EXTERNALDIR}/moment.js \
+	${EXTERNALDIR}/elasticlunr.js \
 	${SRCDIR}/queries/parser-begin.js \
 	${SRCDIR}/queries/build/parser.js \
 	${SRCDIR}/queries/parser-end.js \
@@ -130,7 +131,8 @@ ${JIOVERSION}: ${EXTERNALDIR}/URI.js \
 	${SRCDIR}/jio.storage/cryptstorage.js \
 	${SRCDIR}/jio.storage/websqlstorage.js \
 	${SRCDIR}/jio.storage/fbstorage.js \
-	${SRCDIR}/jio.storage/cloudooostorage.js
+	${SRCDIR}/jio.storage/cloudooostorage.js \
+	${SRCDIR}/jio.storage/elasticlunrstorage.js
 	@mkdir -p $(@D)
 	cat $^ > $@
 

examples/elasticlunr-dropbox.scenario.js

+/*jslint nomen: true */
+/*global jIO, QUnit, sinon */
+(function (jIO, QUnit, sinon) {
+  "use strict";
+  var test = QUnit.test,
+    stop = QUnit.stop,
+    start = QUnit.start,
+    ok = QUnit.ok,
+    deepEqual = QUnit.deepEqual,
+    module = QUnit.module;
+
+  /////////////////////////////////////////////////////////////////
+  // Custom test substorage definition
+  /////////////////////////////////////////////////////////////////
+
+  function Storage200() {
+    return this;
+  }
+
+  jIO.addStorage('elasticlunr200', Storage200);
+
+  function Index200() {
+    return this;
+  }
+
+  jIO.addStorage('index200', Index200);
+
+  /////////////////////////////////////////////////////////////////
+  // ElasticlunrStorage.buildQuery
+  /////////////////////////////////////////////////////////////////
+  module("ElasticlunrStorage.buildQuery", {
+    setup: function () {
+      var context = this;
+
+      Index200.prototype.putAttachment = function () {
+        return true;
+      };
+
+      Index200.prototype.getAttachment = function () {
+        return true;
+      };
+
+      this.jio = jIO.createJIO({
+        type: "elasticlunr",
+        index_fields: [
+          "title"
+        ],
+        index_sub_storage: {
+          type: "index200"
+        },
+        sub_storage: {
+          type: "drivetojiomapping",
+          sub_storage: {
+            type: "dropbox",
+            access_token: "sample_token"
+          }
+        }
+      });
+      this.jio.__storage._resetIndex("id", [
+        "title"
+      ]);
+      this.documents = {};
+
+      Storage200.prototype.hasCapacity = function () {
+        return true;
+      };
+
+      Storage200.prototype.buildQuery = function (options) {
+        return new RSVP.Queue()
+          .push(function () {
+            // capacity query
+            return Object.keys(context.documents).map(function (id) {
+              return {
+                id: id,
+                value: context.documents[id]
+              };
+            });
+          })
+          .push(function (docs) {
+            // capacity filter
+            if (options.ids) {
+              return docs.filter(function (doc) {
+                return options.ids.indexOf(doc.id) >= 0;
+              });
+            }
+
+            return docs;
+          });
+      };
+
+      this.server = sinon.fakeServer.create();
+      this.server.autoRespond = true;
+      this.server.autoRespondAfter = 5;
+    },
+    teardown: function () {
+      this.server.restore();
+      delete this.server;
+    }
+  });
+
+  /////////////////////////////////////////////////////////////////
+  // Dropbox tests
+  /////////////////////////////////////////////////////////////////
+
+  test("dropbox: search by title", function () {
+    var context = this,
+      server = this.server,
+      doc = {
+        title: "foo",
+        subtitle: "bar",
+        desc: "empty"
+      };
+
+    server.respondWith(
+      "POST",
+      "https://content.dropboxapi.com/2/files/upload",
+      [204, {
+        "Content-Type": "text/xml"
+      }, ""]
+    );
+    server.respondWith(
+      "POST",
+      "https://content.dropboxapi.com/2/files/download",
+      [200, {
+        "Content-Type": "application/json"
+      }, JSON.stringify(doc)]
+    );
+
+    stop();
+
+    RSVP.all([
+      context.jio.put("1", doc),
+      context.jio.put("2", {
+        title: "bar",
+        subtitle: "bar",
+        desc: "empty"
+      })
+    ])
+      .then(function () {
+        return context.jio.allDocs({
+          query: 'title: "foo"'
+        });
+      })
+      .then(function (result) {
+        deepEqual(result, {
+          data: {
+            rows: [{
+              id: "1",
+              value: {}
+            }],
+            total_rows: 1
+          }
+        });
+      })
+      .fail(function (error) {
+        ok(false, error);
+      })
+      .always(function () {
+        start();
+      });
+  });
+}(jIO, QUnit, sinon));

examples/scenario.html

@@ -30,7 +30,10 @@ See https://www.nexedi.com/licensing for rationale and options.
 
   <link rel="stylesheet" href="../external/qunit.css" type="text/css" media="screen"/>
   <script src="../external/qunit.js" type="text/javascript"></script>
+  <script src="../external/sinon.js" type="text/javascript"></script>
+
   <script src="scenario.js"></script>
+  <script src="elasticlunr-dropbox.scenario.js"></script>
 </head>
 
 <body>

src/jio.storage/elasticlunrstorage.js

+/*
+ * Copyright 2018, Nexedi SA
+ *
+ * This program is free software: you can Use, Study, Modify and Redistribute
+ * it under the terms of the GNU General Public License version 3, or (at your
+ * option) any later version, as published by the Free Software Foundation.
+ *
+ * You can also Link and Combine this program with other software covered by
+ * the terms of any of the Free Software licenses or any of the Open Source
+ * Initiative approved licenses and Convey the resulting work. Corresponding
+ * source of such a combination shall include the source code for all other
+ * software used.
+ *
+ * This program is distributed WITHOUT ANY WARRANTY; without even the implied
+ * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * See COPYING file for full licensing terms.
+ * See https://www.nexedi.com/licensing for rationale and options.
+ */
+
+/*jslint sloppy: true, nomen: true */
+/*global jIO, RSVP, Blob, Query, elasticlunr */
+(function (jIO, RSVP, Blob, Query, elasticlunr) {
+  'use strict';
+
+  var elasticlunrStorageKey = 'jio_elasticlunr';
+
+  function findDuplicates(array) {
+    var sorted = array.slice().sort(),
+      results = [],
+      i;
+
+    for (i = 0; i < sorted.length - 1; i += 1) {
+      if (sorted[i + 1] === sorted[i]) {
+        if (results.indexOf(sorted[i]) === -1) {
+          results.push(sorted[i]);
+        }
+      }
+    }
+
+    return results;
+  }
+
+  function initIndex(id, indexFields) {
+    var index = elasticlunr();
+    indexFields.forEach(function (field) {
+      index.addField(field);
+    });
+    index.setRef(id);
+    // do not store the documents in the index
+    index.saveDocument(false);
+    return index;
+  }
+
+  function loadIndex(attachmentKey, storage, id, indexFields) {
+    var index = null;
+
+    return storage
+      .getAttachment(attachmentKey, attachmentKey, {
+        format: 'text'
+      })
+      .push(function (data) {
+        index = elasticlunr.Index.load(JSON.parse(data));
+      }, function () {
+        index = initIndex(id, indexFields);
+      })
+      .push(function () {
+        return index;
+      });
+  }
+
+  function searchQuery(index, indexedFields, key, value) {
+    var config = {
+      boolean: "OR",
+      expand: true,
+      fields: {}
+    };
+
+    if (indexedFields.indexOf(key) >= 0) {
+      config.fields[key] = {
+        boost: 1,
+        bool: 'AND'
+      };
+
+      // we can only do a single-string search, so we can
+      // stop on the first indexed field we find
+      return index.search(value, config).map(function (result) {
+        return result.ref;
+      });
+    }
+
+    return null;
+  }
+
+  function recursiveIndexQuery(index, indexedFields, query) {
+    var ids = null,
+      subquery,
+      i,
+      subids;
+
+    if (query.query_list) {
+      for (i = query.query_list.length - 1; i >= 0; i -= 1) {
+        subquery = query.query_list[i];
+
+        subids = recursiveIndexQuery(index, indexedFields, subquery);
+        if (subids !== null) {
+          query.query_list.splice(i, 1);
+          if (ids === null) {
+            ids = subids;
+          } else {
+            ids = findDuplicates(ids.concat(subids));
+          }
+        }
+      }
+
+      return ids;
+    }
+
+    return searchQuery(index, indexedFields, query.key, query.value);
+  }
+
+  /**
+   * The jIO Elasticlunr extension
+   *
+   * @class ElasticlunrStorage
+   * @constructor
+   */
+  function ElasticlunrStorage(spec) {
+    if (!spec.index_sub_storage) {
+      throw new TypeError(
+        "Elasticlunr 'index_sub_storage' must be provided."
+      );
+    }
+    this._index_sub_storage = jIO.createJIO(spec.index_sub_storage);
+    if (!this._index_sub_storage.hasCapacity('getAttachment')) {
+      throw new TypeError(
+        "Elasticlunr 'index_sub_storage' must have getAttachment capacity."
+      );
+    }
+
+    if (!spec.sub_storage) {
+      throw new TypeError(
+        "Elasticlunr 'sub_storage' must be provided."
+      );
+    }
+    this._sub_storage = jIO.createJIO(spec.sub_storage);
+
+    this._index_sub_storage_key = elasticlunrStorageKey + '_' +
+      this._sub_storage.__type;
+    this._index_id = spec.id || 'id';
+    this._index_fields = spec.index_fields || [];
+  }
+
+  ElasticlunrStorage.prototype._getIndex = function () {
+    var context = this;
+
+    if (this._index) {
+      return new RSVP.Queue().push(function () {
+        return context._index;
+      });
+    }
+
+    return loadIndex(
+      this._index_sub_storage_key,
+      this._index_sub_storage,
+      this._index_id,
+      this._index_fields
+    ).push(function (index) {
+      context._index = index;
+      return context._index;
+    });
+  };
+
+  ElasticlunrStorage.prototype._resetIndex = function (id, indexFields) {
+    this._index_id = 'id';
+    this._index_fields = indexFields;
+    this._index = initIndex(id, indexFields);
+  };
+
+  ElasticlunrStorage.prototype._saveIndex = function () {
+    var context = this;
+
+    return this._getIndex()
+      .push(function (index) {
+        var data = JSON.stringify(index);
+        return context._index_sub_storage.putAttachment(
+          context._index_sub_storage_key,
+          context._index_sub_storage_key,
+          new Blob([data])
+        );
+      });
+  };
+
+  ElasticlunrStorage.prototype.get = function () {
+    return this._sub_storage.get.apply(this._sub_storage, arguments);
+  };
+
+  ElasticlunrStorage.prototype.allAttachments = function () {
+    return this._sub_storage.allAttachments.apply(
+      this._sub_storage,
+      arguments
+    );
+  };
+
+  ElasticlunrStorage.prototype.post = function (doc) {
+    var context = this;
+
+    return this._sub_storage.post.apply(this._sub_storage, arguments)
+      .push(function (id) {
+        var data = JSON.parse(JSON.stringify(doc));
+        data.id = id.toString();
+
+        return context._getIndex().push(function (index) {
+          index.addDoc(data);
+          return context._saveIndex();
+        });
+      });
+  };
+
+  ElasticlunrStorage.prototype.put = function (id, doc) {
+    var context = this;
+
+    return this._sub_storage.put.apply(this._sub_storage, arguments)
+      .push(function () {
+        var data = JSON.parse(JSON.stringify(doc));
+        data.id = id.toString();
+
+        return context._getIndex().push(function (index) {
+          index.updateDoc(data);
+          return context._saveIndex();
+        });
+      });
+  };
+
+  ElasticlunrStorage.prototype.remove = function (id) {
+    var context = this;
+
+    // need to get the full document to remove every data from indexes
+    return this._sub_storage.get(id)
+      .push(function (doc) {
+        return context._sub_storage.remove(id)
+          .push(function () {
+            return context._getIndex();
+          })
+          .push(function (index) {
+            index.removeDoc(doc);
+            return context._saveIndex();
+          });
+      });
+  };
+
+  ElasticlunrStorage.prototype.getAttachment = function () {
+    return this._sub_storage.getAttachment.apply(
+      this._sub_storage,
+      arguments
+    );
+  };
+
+  ElasticlunrStorage.prototype.putAttachment = function () {
+    return this._sub_storage.putAttachment.apply(
+      this._sub_storage,
+      arguments
+    );
+  };
+
+  ElasticlunrStorage.prototype.removeAttachment = function () {
+    return this._sub_storage.removeAttachment.apply(
+      this._sub_storage,
+      arguments
+    );
+  };
+
+  ElasticlunrStorage.prototype.repair = function () {
+    // rebuild index?
+    return this._sub_storage.repair.apply(this._sub_storage, arguments);
+  };
+
+  ElasticlunrStorage.prototype.hasCapacity = function (name) {
+    var capacityList = [
+      'limit', 'sort', 'select', 'query'
+    ];
+
+    if (capacityList.indexOf(name) !== -1) {
+      return true;
+    }
+    if (name === 'index') {
+      return true;
+    }
+    return this._sub_storage.hasCapacity(name);
+  };
+
+  ElasticlunrStorage.prototype.buildQuery = function (options) {
+    var context = this,
+      indexedFields = this._index_fields,
+      runSubstorageQuery = options.select_list || options.include_docs,
+      parsedQuery;
+
+    if (options.query && options.query.indexOf('OR') === -1) {
+      parsedQuery = jIO.QueryFactory.create(options.query);
+
+      return context._getIndex()
+        .push(function (index) {
+          return recursiveIndexQuery(index, indexedFields, parsedQuery);
+        })
+        .push(function (ids) {
+          try {
+            if (context._sub_storage.hasCapacity('query_filtered')) {
+              // simple query with found matches, just exec a simple list
+              if ((ids || []).length && parsedQuery.type === 'simple') {
+                delete options.query;
+              } else {
+                options.query = Query.objectToSearchText(parsedQuery);
+              }
+
+              options.ids = ids;
+              return context._sub_storage.buildQuery(options);
+            }
+          } catch (ignore) {}
+
+          // run query with substorage if we want to retrieve the documents
+          if (runSubstorageQuery) {
+            return context._sub_storage.buildQuery(options);
+          }
+
+          return (ids || []).map(function (id) {
+            return {
+              id: id,
+              value: {}
+            };
+          });
+        });
+    }
+
+    return this._sub_storage.buildQuery(options);
+  };
+
+  jIO.addStorage('elasticlunr', ElasticlunrStorage);
+}(jIO, RSVP, Blob, Query, elasticlunr));

test/tests.html

@@ -76,6 +76,7 @@ See https://www.nexedi.com/licensing for rationale and options.
   <script src="jio.storage/websqlstorage.tests.js"></script>
   <script src="jio.storage/fbstorage.tests.js"></script>
   <script src="jio.storage/httpstorage.tests.js"></script>
+  <script src="jio.storage/elasticlunrstorage.tests.js"></script>
   <!--script src="../src/jio.storage/xwikistorage.js"></script>
   <script src="jio.storage/xwikistorage.tests.js"></script-->