[indexeddb] Support database indexes handling

Thanks to Preet Batth for his work on this topic.

src/jio.storage/indexeddbstorage.js

@@ -43,14 +43,15 @@
 
 /*jslint nomen: true */
 /*global indexedDB, jIO, RSVP, Blob, Math, IDBKeyRange, IDBOpenDBRequest,
-        DOMError, Event*/
+        DOMError, Set*/
 
 (function (indexedDB, jIO, RSVP, Blob, Math, IDBKeyRange, IDBOpenDBRequest,
-           DOMError) {
+           DOMError, Set) {
   "use strict";
 
   // Read only as changing it can lead to data corruption
-  var UNITE = 2000000;
+  var UNITE = 2000000,
+    INDEX_PREFIX = 'doc.';
 
   function IndexedDBStorage(description) {
     if (typeof description.database !== "string" ||
@@ -59,6 +60,8 @@
                           "must be a non-empty string");
     }
     this._database_name = "jio:" + description.database;
+    this._version = description.version;
+    this._index_key_list = description.index_key_list || [];
   }
 
   IndexedDBStorage.prototype.hasCapacity = function (name) {
@@ -69,10 +72,15 @@
     return key_list.join("_");
   }
 
-  function handleUpgradeNeeded(evt) {
+  function handleUpgradeNeeded(evt, index_key_list) {
     var db = evt.target.result,
-      store;
+      store,
+      current_store_list = Array.from(db.objectStoreNames),
+      current_index_list,
+      i,
+      index_key;
 
+    if (current_store_list.indexOf("metadata") === -1) {
       store = db.createObjectStore("metadata", {
         keyPath: "_id",
         autoIncrement: false
@@ -80,13 +88,36 @@
       // It is not possible to use openKeyCursor on keypath directly
       // https://www.w3.org/Bugs/Public/show_bug.cgi?id=19955
       store.createIndex("_id", "_id", {unique: true});
+    } else {
+      store = evt.target.transaction.objectStore("metadata");
+    }
+
+    current_index_list = new Set(store.indexNames);
+    current_index_list.delete("_id");
+    for (i = 0; i < index_key_list.length; i += 1) {
+      // Prefix the index name to prevent conflict with _id
+      index_key = INDEX_PREFIX + index_key_list[i];
+      if (current_index_list.has(index_key)) {
+        current_index_list.delete(index_key);
+      } else {
+        store.createIndex(index_key, index_key,
+                          {unique: false});
+      }
+    }
+    current_index_list = Array.from(current_index_list);
+    for (i = 0; i < current_index_list.length; i += 1) {
+      store.deleteIndex(current_index_list[i]);
+    }
 
+    if (current_store_list.indexOf("attachment") === -1) {
       store = db.createObjectStore("attachment", {
         keyPath: "_key_path",
         autoIncrement: false
       });
       store.createIndex("_id", "_id", {unique: false});
+    }
 
+    if (current_store_list.indexOf("blob") === -1) {
       store = db.createObjectStore("blob", {
         keyPath: "_key_path",
         autoIncrement: false
@@ -95,9 +126,11 @@
                         ["_id", "_attachment"], {unique: false});
       store.createIndex("_id", "_id", {unique: false});
     }
+  }
 
-  function waitForOpenIndexedDB(db_name, callback) {
-    var request;
+  function waitForOpenIndexedDB(storage, callback) {
+    var request,
+      db_name = storage._database_name;
 
     function canceller() {
       if ((request !== undefined) && (request.result !== undefined)) {
@@ -107,7 +140,7 @@
 
     function resolver(resolve, reject) {
       // Open DB //
-      request = indexedDB.open(db_name);
+      request = indexedDB.open(db_name, storage._version);
       request.onerror = function (error) {
         canceller();
         if ((error !== undefined) &&
@@ -135,7 +168,9 @@
       };
 
       // Create DB if necessary //
-      request.onupgradeneeded = handleUpgradeNeeded;
+      request.onupgradeneeded = function (evt) {
+        handleUpgradeNeeded(evt, storage._index_key_list);
+      };
 
       request.onversionchange = function () {
         canceller();
@@ -233,7 +268,7 @@
 
     function pushIncludedMetadata(cursor) {
       result_list.push({
-        "id": cursor.key,
+        "id": cursor.primaryKey,
         "value": {},
         "doc": cursor.value.doc
       });
@@ -241,24 +276,25 @@
 
     function pushMetadata(cursor) {
       result_list.push({
-        "id": cursor.key,
+        "id": cursor.primaryKey,
         "value": {}
       });
     }
 
     return new RSVP.Queue()
       .push(function () {
-        return waitForOpenIndexedDB(context._database_name, function (db) {
+        return waitForOpenIndexedDB(context, function (db) {
           return waitForTransaction(db, ["metadata"], "readonly",
                                     function (tx) {
+              var key = "_id";
               if (options.include_docs === true) {
                 return waitForAllSynchronousCursor(
-                  tx.objectStore("metadata").index("_id").openCursor(),
+                  tx.objectStore("metadata").index(key).openCursor(),
                   pushIncludedMetadata
                 );
               }
               return waitForAllSynchronousCursor(
-                tx.objectStore("metadata").index("_id").openKeyCursor(),
+                tx.objectStore("metadata").index(key).openKeyCursor(),
                 pushMetadata
               );
             });
@@ -273,7 +309,7 @@
     var context = this;
     return new RSVP.Queue()
       .push(function () {
-        return waitForOpenIndexedDB(context._database_name, function (db) {
+        return waitForOpenIndexedDB(context, function (db) {
           return waitForTransaction(db, ["metadata"], "readonly",
                                     function (tx) {
               return waitForIDBRequest(tx.objectStore("metadata").get(id));
@@ -301,7 +337,7 @@
 
     return new RSVP.Queue()
       .push(function () {
-        return waitForOpenIndexedDB(context._database_name, function (db) {
+        return waitForOpenIndexedDB(context, function (db) {
           return waitForTransaction(db, ["metadata", "attachment"], "readonly",
                                     function (tx) {
               return RSVP.all([
@@ -330,7 +366,7 @@
   };
 
   IndexedDBStorage.prototype.put = function (id, metadata) {
-    return waitForOpenIndexedDB(this._database_name, function (db) {
+    return waitForOpenIndexedDB(this, function (db) {
       return waitForTransaction(db, ["metadata"], "readwrite",
                                 function (tx) {
           return waitForIDBRequest(tx.objectStore("metadata").put({
@@ -342,7 +378,7 @@
   };
 
   IndexedDBStorage.prototype.remove = function (id) {
-    return waitForOpenIndexedDB(this._database_name, function (db) {
+    return waitForOpenIndexedDB(this, function (db) {
       return waitForTransaction(db, ["metadata", "attachment", "blob"],
                                 "readwrite", function (tx) {
 
@@ -386,10 +422,10 @@
     if (options === undefined) {
       options = {};
     }
-    var db_name = this._database_name,
-      start,
+    var start,
       end,
-      array_buffer_list = [];
+      array_buffer_list = [],
+      context = this;
 
     start = options.start || 0;
     end = options.end;
@@ -410,7 +446,7 @@
 
       return new RSVP.Queue()
         .push(function () {
-          return waitForOpenIndexedDB(db_name, function (db) {
+          return waitForOpenIndexedDB(context, function (db) {
             return waitForTransaction(db, ["blob"], "readonly",
                                       function (tx) {
                 var key_path = buildKeyPath([id, name]),
@@ -488,7 +524,7 @@
     // Request the full blob
     return new RSVP.Queue()
       .push(function () {
-        return waitForOpenIndexedDB(db_name, function (db) {
+        return waitForOpenIndexedDB(context, function (db) {
           return waitForTransaction(db, ["attachment", "blob"], "readonly",
                                     function (tx) {
               var key_path = buildKeyPath([id, name]),
@@ -555,7 +591,7 @@
   };
 
   IndexedDBStorage.prototype.putAttachment = function (id, name, blob) {
-    var db_name = this._database_name;
+    var context = this;
     return new RSVP.Queue()
       .push(function () {
         // Split the blob first
@@ -573,7 +609,7 @@
           handled_size += UNITE;
         }
 
-        return waitForOpenIndexedDB(db_name, function (db) {
+        return waitForOpenIndexedDB(context, function (db) {
           return waitForTransaction(db, ["attachment", "blob"], "readwrite",
                                     function (tx) {
               var blob_store,
@@ -640,7 +676,7 @@
   };
 
   IndexedDBStorage.prototype.removeAttachment = function (id, name) {
-    return waitForOpenIndexedDB(this._database_name, function (db) {
+    return waitForOpenIndexedDB(this, function (db) {
       return waitForTransaction(db, ["attachment", "blob"], "readwrite",
                                 function (tx) {
           var promise_list = [],
@@ -672,4 +708,5 @@
   };
 
   jIO.addStorage("indexeddb", IndexedDBStorage);
-}(indexedDB, jIO, RSVP, Blob, Math, IDBKeyRange, IDBOpenDBRequest, DOMError));
+}(indexedDB, jIO, RSVP, Blob, Math, IDBKeyRange, IDBOpenDBRequest, DOMError,
+  Set));

test/jio.storage/indexeddbstorage.tests.js

@@ -20,10 +20,10 @@
 /*jslint nomen: true */
 /*global indexedDB, Blob, sinon, IDBDatabase,
          IDBTransaction, IDBIndex, IDBObjectStore, IDBCursor, IDBKeyRange,
-         Rusha*/
+         DOMException, Rusha*/
 (function (jIO, QUnit, indexedDB, Blob, sinon, IDBDatabase,
            IDBTransaction, IDBIndex, IDBObjectStore, IDBCursor, IDBKeyRange,
-           Rusha) {
+           DOMException, Rusha) {
   "use strict";
   var test = QUnit.test,
     stop = QUnit.stop,
@@ -33,6 +33,7 @@
     deepEqual = QUnit.deepEqual,
     equal = QUnit.equal,
     module = QUnit.module,
+    throws = QUnit.throws,
     big_string = "";
 
   big_string = new Array(3000000).fill('a').join('');
@@ -53,6 +54,7 @@
   /////////////////////////////////////////////////////////////////
   module("indexeddbStorage.constructor");
   test("default unite value", function () {
+    expect(4);
     var jio = jIO.createJIO({
       type: "indexeddb",
       database: "qunit"
@@ -60,6 +62,271 @@
 
     equal(jio.__type, "indexeddb");
     deepEqual(jio.__storage._database_name, "jio:qunit");
+    deepEqual(jio.__storage._index_key_list, []);
+    deepEqual(jio.__storage._version, undefined);
+  });
+
+  test("config", function () {
+    expect(4);
+    var jio = jIO.createJIO({
+      type: "indexeddb",
+      database: "qunit",
+      version: 1,
+      index_key_list: ['a']
+    });
+
+    equal(jio.__type, "indexeddb");
+    deepEqual(jio.__storage._database_name, "jio:qunit");
+    deepEqual(jio.__storage._index_key_list, ['a']);
+    deepEqual(jio.__storage._version, 1);
+  });
+
+  /////////////////////////////////////////////////////////////////
+  // indexeddbStorage DB migration
+  /////////////////////////////////////////////////////////////////
+  module("indexeddbStorage.upgradeDB");
+
+  function setupDBMigrationTest(test, old_jio_kw, new_jio_kw, check_callback) {
+    // Create a IDB with one document
+    // Migrate it to a new version
+    // Spy IDB behaviour while getting the previous document
+    // Check that doument is still there
+    stop();
+    old_jio_kw.type = "indexeddb";
+    old_jio_kw.database = "qunit";
+    new_jio_kw.type = "indexeddb";
+    new_jio_kw.database = "qunit";
+    test.jio = jIO.createJIO(old_jio_kw);
+
+    return deleteIndexedDB(test.jio)
+      .then(function () {
+        return test.jio.put('foo', {'a': 1});
+      })
+      .then(function () {
+        test.jio = jIO.createJIO(new_jio_kw);
+
+        test.spy_open = sinon.spy(indexedDB, "open");
+        test.spy_create_store = sinon.spy(IDBDatabase.prototype,
+                                          "createObjectStore");
+        test.spy_transaction = sinon.spy(IDBDatabase.prototype, "transaction");
+        test.spy_store = sinon.spy(IDBTransaction.prototype, "objectStore");
+        test.spy_index = sinon.spy(IDBObjectStore.prototype, "index");
+        test.spy_create_index = sinon.spy(IDBObjectStore.prototype,
+                                          "createIndex");
+        test.spy_delete_index = sinon.spy(IDBObjectStore.prototype,
+                                          "deleteIndex");
+        return test.jio.get('foo');
+      })
+      .then(function (result) {
+        deepEqual(result, {'a': 1});
+        ok(test.spy_transaction.calledOnce, "transaction count " +
+           test.spy_transaction.callCount);
+        deepEqual(test.spy_transaction.firstCall.args[0], ["metadata"],
+                  "transaction first argument");
+        equal(test.spy_transaction.firstCall.args[1], "readonly",
+              "transaction second argument");
+      })
+      .always(function (param) {
+        return check_callback(param);
+      })
+      .fail(function (error) {
+        ok(false, error);
+      })
+      .always(function () {
+        test.spy_open.restore();
+        delete test.spy_open;
+        test.spy_create_store.restore();
+        delete test.spy_create_store;
+        test.spy_transaction.restore();
+        delete test.spy_transaction;
+        test.spy_store.restore();
+        delete test.spy_store;
+        test.spy_index.restore();
+        delete test.spy_index;
+        test.spy_create_index.restore();
+        delete test.spy_create_index;
+        test.spy_delete_index.restore();
+        delete test.spy_delete_index;
+
+        start();
+      });
+  }
+
+  test("no change", function () {
+    var context = this;
+    expect(10);
+
+    return setupDBMigrationTest(context, {}, {}, function () {
+      ok(context.spy_open.calledOnce, "open count " +
+         context.spy_open.callCount);
+      equal(context.spy_open.firstCall.args[0], "jio:qunit",
+            "open first argument");
+
+      equal(context.spy_create_store.callCount, 0,
+            "createObjectStore count");
+      equal(context.spy_store.callCount, 1,
+            "objectStore count");
+      equal(context.spy_create_index.callCount, 0, "createIndex count");
+      equal(context.spy_delete_index.callCount, 0, "deleteIndex count");
+    });
+  });
+
+  test("version update, no key change", function () {
+    var context = this;
+    expect(10);
+
+    return setupDBMigrationTest(context, {}, {version: 2}, function () {
+      ok(context.spy_open.calledOnce, "open count " +
+         context.spy_open.callCount);
+      equal(context.spy_open.firstCall.args[0], "jio:qunit",
+            "open first argument");
+
+      equal(context.spy_create_store.callCount, 0,
+            "createObjectStore count");
+      equal(context.spy_store.callCount, 2,
+            "objectStore count");
+      equal(context.spy_create_index.callCount, 0, "createIndex count");
+      equal(context.spy_delete_index.callCount, 0, "deleteIndex count");
+    });
+  });
+
+  test("version decrease", function () {
+    var context = this;
+    expect(8);
+
+    return setupDBMigrationTest(context, {version: 3},
+                                {version: 2}, function (evt) {
+        ok(evt.target.error instanceof DOMException);
+        equal(evt.target.error.name, 'VersionError');
+
+        ok(context.spy_open.calledOnce, "open count " +
+           context.spy_open.callCount);
+        equal(context.spy_open.firstCall.args[0], "jio:qunit",
+              "open first argument");
+
+        equal(context.spy_create_store.callCount, 0,
+              "createObjectStore count");
+        equal(context.spy_store.callCount, 0,
+              "objectStore count");
+        equal(context.spy_create_index.callCount, 0, "createIndex count");
+        equal(context.spy_delete_index.callCount, 0, "deleteIndex count");
+      });
+  });
+
+  test("version increase, key added", function () {
+    var context = this;
+    expect(13);
+
+    return setupDBMigrationTest(context, {version: 1, index_key_list: ['a']},
+                                {version: 2, index_key_list: ['a', 'b']},
+                                function () {
+        ok(context.spy_open.calledOnce, "open count " +
+           context.spy_open.callCount);
+        equal(context.spy_open.firstCall.args[0], "jio:qunit",
+              "open first argument");
+
+        equal(context.spy_create_store.callCount, 0,
+              "createObjectStore count");
+        equal(context.spy_store.callCount, 2,
+              "objectStore count");
+        equal(context.spy_create_index.callCount, 1, "createIndex count");
+        equal(context.spy_create_index.firstCall.args[0], "doc.b",
+              "first createIndex first argument");
+        equal(context.spy_create_index.firstCall.args[1], "doc.b",
+              "first createIndex second argument");
+        deepEqual(context.spy_create_index.firstCall.args[2], {unique: false},
+                  "first createIndex third argument");
+
+        equal(context.spy_delete_index.callCount, 0, "deleteIndex count");
+      });
+  });
+
+  test("version increase, key removed", function () {
+    var context = this;
+    expect(11);
+
+    return setupDBMigrationTest(context,
+                                {version: 1, index_key_list: ['a', 'b']},
+                                {version: 2, index_key_list: ['b']},
+                                function () {
+        ok(context.spy_open.calledOnce, "open count " +
+           context.spy_open.callCount);
+        equal(context.spy_open.firstCall.args[0], "jio:qunit",
+              "open first argument");
+
+        equal(context.spy_create_store.callCount, 0,
+              "createObjectStore count");
+        equal(context.spy_store.callCount, 2,
+              "objectStore count");
+        equal(context.spy_create_index.callCount, 0, "createIndex count");
+
+        equal(context.spy_delete_index.callCount, 1, "deleteIndex count");
+        equal(context.spy_delete_index.firstCall.args[0], "doc.a",
+              "first deleteIndex first argument");
+      });
+  });
+
+  test("version increase, keys added and removed", function () {
+    var context = this;
+    expect(18);
+
+    return setupDBMigrationTest(context,
+                                {version: 1, index_key_list: ['a', 'b', 'c']},
+                                {version: 2, index_key_list: ['e', 'b', 'f']},
+                                function () {
+        ok(context.spy_open.calledOnce, "open count " +
+           context.spy_open.callCount);
+        equal(context.spy_open.firstCall.args[0], "jio:qunit",
+              "open first argument");
+
+        equal(context.spy_create_store.callCount, 0,
+              "createObjectStore count");
+        equal(context.spy_store.callCount, 2,
+              "objectStore count");
+        equal(context.spy_create_index.callCount, 2, "createIndex count");
+        equal(context.spy_create_index.firstCall.args[0], "doc.e",
+              "first createIndex first argument");
+        equal(context.spy_create_index.firstCall.args[1], "doc.e",
+              "first createIndex second argument");
+        deepEqual(context.spy_create_index.firstCall.args[2], {unique: false},
+                  "first createIndex third argument");
+
+        equal(context.spy_create_index.secondCall.args[0], "doc.f",
+              "second createIndex first argument");
+        equal(context.spy_create_index.secondCall.args[1], "doc.f",
+              "second createIndex second argument");
+        deepEqual(context.spy_create_index.secondCall.args[2], {unique: false},
+                  "second createIndex third argument");
+
+        equal(context.spy_delete_index.callCount, 2, "deleteIndex count");
+        equal(context.spy_delete_index.firstCall.args[0], "doc.a",
+              "first deleteIndex first argument");
+        equal(context.spy_delete_index.secondCall.args[0], "doc.c",
+              "second deleteIndex first argument");
+      });
+  });
+
+
+  test("version idem, keys added and removed", function () {
+    var context = this;
+    expect(10);
+
+    return setupDBMigrationTest(context,
+                                {version: 1, index_key_list: ['a', 'b', 'c']},
+                                {version: 1, index_key_list: ['e', 'b', 'f']},
+                                function () {
+        ok(context.spy_open.calledOnce, "open count " +
+           context.spy_open.callCount);
+        equal(context.spy_open.firstCall.args[0], "jio:qunit",
+              "open first argument");
+
+        equal(context.spy_create_store.callCount, 0,
+              "createObjectStore count");
+        equal(context.spy_store.callCount, 1,
+              "objectStore count");
+        equal(context.spy_create_index.callCount, 0, "createIndex count");
+        equal(context.spy_delete_index.callCount, 0, "deleteIndex count");
+      });
   });
 
   /////////////////////////////////////////////////////////////////
@@ -67,12 +334,35 @@
   /////////////////////////////////////////////////////////////////
   module("indexeddbStorage.hasCapacity");
   test("can list documents", function () {
+    expect(2);
     var jio = jIO.createJIO({
       type: "indexeddb",
       database: "qunit"
     });
 
     ok(jio.hasCapacity("list"));
+    ok(jio.hasCapacity("include"));
+  });
+
+  test("can not search documents", function () {
+    expect(4);
+    var jio = jIO.createJIO({
+      type: "indexeddb",
+      database: "qunit"
+    });
+
+    throws(
+      function () {
+        jio.hasCapacity("query");
+      },
+      function (error) {
+        ok(error instanceof jIO.util.jIOError);
+        equal(error.status_code, 501);
+        equal(error.message,
+              "Capacity 'query' is not implemented on 'indexeddb'");
+        return true;
+      }
+    );
   });
 
   /////////////////////////////////////////////////////////////////
@@ -1764,4 +2054,4 @@
 
 }(jIO, QUnit, indexedDB, Blob, sinon, IDBDatabase,
   IDBTransaction, IDBIndex, IDBObjectStore, IDBCursor, IDBKeyRange,
-  Rusha));
+  DOMException, Rusha));