Get and allDocs can now be used to access older revisions without changes to...

Get and allDocs can now be used to access older revisions without changes to the API.  The approach to doing this with allDocs is very costly right now, but is being fixed for the next commit.

src/jio.storage/bryanstorage.js

@@ -28,18 +28,15 @@
     });
   }
 
-  BryanStorage.prototype.get = function (id_in, steps) {
+  BryanStorage.prototype.get = function (id_in) {
 
-    if (steps === undefined) {
-      steps = 0;
-    }
 
     // Query to get the last edit made to this document
     var substorage = this._sub_storage,
       options = {
         query: "doc_id: " + id_in,
         sort_on: [["timestamp", "descending"]],
-        limit: [steps, 1]
+        limit: [0, 1]
       };
 
     return substorage.allDocs(options)
@@ -52,7 +49,6 @@
           404
         );
       })
-
       .push(function (result) {
         if (result.op === "put") {
           return result.doc;
@@ -61,6 +57,48 @@
           "bryanstorage: cannot find object '" + id_in + "' (removed)",
           404
         );
+
+      // If no documents returned in first query, check if the id is encoding
+      // revision information
+      }, function () {
+        var steps,
+          steps_loc = id_in.lastIndexOf("_-");
+        // If revision signature is not in id_in, than return 404, since id
+        // is not found
+        if (steps_loc === -1) {
+          throw new jIO.util.jIOError(
+            "bryanstorage: cannot find object '" + id_in + "'",
+            404
+          );
+        }
+
+        // If revision signature is found, query storage based on this
+        steps = Number(id_in.slice(steps_loc + 2));
+        id_in = id_in.slice(0, steps_loc);
+        options = {
+          query: "doc_id: " + id_in,
+          sort_on: [["timestamp", "descending"]],
+          limit: [steps, 1]
+        };
+        return substorage.allDocs(options)
+          .push(function (results) {
+            if (results.data.rows.length > 0) {
+              return substorage.get(results.data.rows[0].id);
+            }
+            throw new jIO.util.jIOError(
+              "bryanstorage: cannot find object '" + id_in + "'",
+              404
+            );
+          })
+          .push(function (result) {
+            if (result.op === "put") {
+              return result.doc;
+            }
+            throw new jIO.util.jIOError(
+              "bryanstorage: cannot find object '" + id_in + "' (removed)",
+              404
+            );
+          });
       });
   };
 
@@ -77,7 +115,6 @@
         doc: data,
         op: "put"
       };
-    this._lastseen = timestamp;
     return this._sub_storage.put(timestamp, metadata);
   };
 
@@ -152,34 +189,49 @@
   };
 
   BryanStorage.prototype.buildQuery = function (options) {
+
     if (options === undefined) {
       options = {};
     }
-    if (options.sort_on === undefined) {
-      options.sort_on = [];
-    }
-    options.sort_on.push(["timestamp", "descending"]);
-    if (options.limit === undefined) {
-      options.limit = [0, -1];
+    if (options.query === undefined) {
+      options.query = "";
     }
-
-    // Default behavior is to return only the latest revision of each document
-    if (options.revision_limit === undefined) {
-      options.revision_limit = [0, 1];
-    }
-
+    options.query = jIO.QueryFactory.create(options.query);
     var meta_options = {
-      // XXX: I don't believe it's currently possible to query on sub-attributes
-      // so for now, we just use the inputted query, which obviously will fail
-      query: options.query,
-
-      // XXX: same here, we cannot sort correctly because we cannot access
-      // attributes of doc
-      sort_on: options.sort_on
-    },
+        // XXX: I don't believe it's currently possible to query on 
+        // sub-attributes so for now, we just use the inputted query, which 
+        // obviously will fail
+        query: "",
+
+        // XXX: same here, we cannot sort correctly because we cannot access
+        // attributes of doc
+        sort_on: [["timestamp", "descending"]]
+      },
       substorage = this._sub_storage,
-      max_num_docs = options.limit[1],
-      first_doc = options.limit[0];
+
+    // Check if query involved _REVISION.  If not, we will later place a 
+    // (*) AND (_REVISION: =0) as the default handling of revisions
+      rev_query = false,
+      query_obj = options.query,
+      query_stack = [],
+      ind;
+
+    if (query_obj.hasOwnProperty("query_list")) {
+      query_stack.push(query_obj);
+    } else {
+      rev_query = (query_obj.key === "_REVISION");
+    }
+    while (query_stack.length > 0 && (!rev_query)) {
+      query_obj = query_stack.pop();
+      for (ind = 0; ind < query_obj.query_list.length; ind += 1) {
+        if (query_obj.query_list[ind].hasOwnProperty("query_list")) {
+          query_stack.push(query_obj.query_list[ind]);
+        } else if (query_obj.query_list[ind].key === "_REVISION") {
+          rev_query = true;
+          break;
+        }
+      }
+    }
 
     return this._sub_storage.allDocs(meta_options)
 
@@ -191,56 +243,66 @@
         return RSVP.all(promises);
       })
 
-      .push(function (results_array) {
-        var clean_data = [],
-          ind,
-          seen_docs = {},
-          current_doc,
-          counter = 0;
-
-        // Default behavior is to not limit the number of documents returned
-        if (max_num_docs === -1) {
-          max_num_docs = results_array.length;
-        }
-        for (ind = 0; ind < results_array.length; ind += 1) {
-          current_doc = results_array[ind];
-
-          // Initialize count of revisions
-          if (!seen_docs.hasOwnProperty(current_doc.doc_id)) {
-            seen_docs[current_doc.doc_id] = 0;
+      .push(function (results) {
+        // Label all documents with their current revision status
+        var doc,
+          revision_tracker = {},
+          promises;
+        for (ind = 0; ind < results.length; ind += 1) {
+          doc = results[ind];
+          if (revision_tracker.hasOwnProperty(doc.doc_id)) {
+            revision_tracker[doc.doc_id] += 1;
+          } else {
+            revision_tracker[doc.doc_id] = 0;
           }
+          doc._REVISION = revision_tracker[doc.doc_id];
+        }
 
-          // If the latest version of this document has not yet been 
-          // included in query result
-          if (options.revision_limit[0] <= seen_docs[current_doc.doc_id] &&
-              seen_docs[current_doc.doc_id] < options.revision_limit[0] +
-              options.revision_limit[1]) {
-
-            // If the latest edit was a put operation, add it to query
-            // results
-            if (current_doc.op === "put") {
-              if (counter >= first_doc) {
-
-                // Note the rev attribute added to the output data.  
-                // This guarantees that `this.get(id, rev) === doc`
-                clean_data.push({
-                  doc: current_doc.doc,
-                  value: {},
-                  id: current_doc.doc_id,
-                  rev: seen_docs[current_doc.doc_id]
-                });
-                if (clean_data.length === max_num_docs) {
-                  return clean_data;
-                }
-              }
-              counter += 1;
-            }
+        // There must be a faster way
+        promises = results.map(function (data) {
+          return substorage.put(data.timestamp, data);
+        });
+        return RSVP.all(promises);
+      })
+      .push(function () {
+        var latest_rev_query;
+        latest_rev_query = jIO.QueryFactory.create(
+          "(_REVISION: >= 0) AND (NOT op: remove)"
+        );
+        if (rev_query) {
+          latest_rev_query.query_list[0] = options.query;
+        } else {
+          latest_rev_query.query_list[0] = jIO.QueryFactory.create(
+            "(_REVISION: =0)"
+          );
+          if (options.query.type === "simple" ||
+              options.query.type === "complex") {
+            latest_rev_query.query_list.push(options.query);
           }
-          // Keep track of how many times this doc_id has been seen
-          seen_docs[current_doc.doc_id] += 1;
         }
-        // In passing results back to allDocs, formatting of query is handled
-        return clean_data;
+        // Build a query for final push
+        options.query = latest_rev_query;
+        if (options.sort_on === undefined) {
+          options.sort_on = [];
+        }
+        options.sort_on.push(["timestamp", "descending"]);
+        return substorage.allDocs(options);
+      })
+      .push(function (results) {
+        var promises = results.data.rows.map(function (data) {
+          return substorage.get(data.id);
+        });
+        return RSVP.all(promises);
+      })
+      .push(function (results) {
+        return results
+          .map(function (current_doc) {
+            return {
+              doc: current_doc.doc,
+              value: {},
+              id: current_doc.doc_id
+            };
+          });
       });
   };
 

test/jio.storage/bryanstorage.tests.js

@@ -287,7 +287,7 @@
   test("Testing proper retrieval of older revisions of documents",
     function () {
       stop();
-      expect(12);
+      expect(18);
 
       // create storage of type "bryan" with memory as substorage
       var jio = jIO.createJIO({
@@ -314,7 +314,7 @@
           });
         })
         .push(function () {
-          return jio.get("doc", 0);
+          return jio.get("doc_-0");
         })
         .push(function (result) {
           deepEqual(result, {
@@ -325,7 +325,7 @@
           return jio.put("doc", {"k1": "v1"});
         })
         .push(function () {
-          return jio.get("doc", 0);
+          return jio.get("doc_-0");
         })
         .push(function (result) {
           deepEqual(result, {
@@ -333,7 +333,7 @@
           });
         })
         .push(function () {
-          return jio.get("doc", 1);
+          return jio.get("doc_-1");
         })
         .push(function (result) {
           deepEqual(result, {
@@ -359,19 +359,19 @@
           deepEqual(result,
             {"k4": "v4"},
             "By default, .get returns latest revision");
-          return jio.get("doc", 0);
+          return jio.get("doc");
         })
         .push(function (result) {
           deepEqual(result,
             {"k4": "v4"},
             ".get returns latest revision with second input = 0");
-          return jio.get("doc", 1);
+          return jio.get("doc_-1");
         })
         .push(function (result) {
           deepEqual(result,
             {"k3": "v3"},
             "Walk back one revision with second input = 1");
-          return jio.get("doc", 2);
+          return jio.get("doc_-2");
         })
         .push(function () {
           ok(false, "This query should have thrown a 404 error");
@@ -380,25 +380,25 @@
             deepEqual(error.status_code,
               404,
               "Current state of document is 'removed'.");
-            return jio.get("doc", 3);
+            return jio.get("doc_-3");
           })
         .push(function (result) {
           deepEqual(result,
             {"k2": "v2"},
             "Walk back three revisions with second input = 3");
-          return jio.get("doc", 4);
+          return jio.get("doc_-4");
         })
         .push(function (result) {
           deepEqual(result,
             {"k1": "v1"},
             "Walk back four revisions with second input = 4");
-          return jio.get("doc", 5);
+          return jio.get("doc_-5");
         })
         .push(function (result) {
           deepEqual(result,
             {"k0": "v0"},
             "Walk back five revisions with second input = 5");
-          return jio.get("doc", 6);
+          return jio.get("doc_-6");
         })
         .push(function () {
           ok(false, "This query should have thrown a 404 error");
@@ -408,6 +408,65 @@
               404,
               "There are only 5 previous states of this document");
           })
+
+        // Adding documents with problematic doc_id's
+        .push(function () {
+          return jio.put("doc_-name", {
+            "key": "val0"
+          });
+        })
+        .push(function () {
+          return jio.put("document_-0", {
+            "key": "and val0"
+          });
+        })
+        .push(function () {
+          return jio.put("doc_-name", {
+            "key": "val1"
+          });
+        })
+
+        .push(function () {
+          return jio.get("doc_-name");
+        })
+        .push(function (result) {
+          deepEqual(result, {
+            "key": "val1"
+          });
+          return jio.get("doc_-name_-0");
+        })
+        .push(function (result) {
+          deepEqual(result, {
+            "key": "val1"
+          });
+          return jio.get("doc_-name_-1");
+        })
+        .push(function (result) {
+          deepEqual(result, {
+            "key": "val0"
+          });
+          return jio.get("document_-0");
+        })
+        .push(function (result) {
+          deepEqual(result, {
+            "key": "and val0"
+          });
+          return jio.get("document_-0_-0");
+        })
+        .push(function (result) {
+          deepEqual(result, {
+            "key": "and val0"
+          });
+          return jio.get("document_-0_-1");
+        })
+        .push(function () {
+          ok(false, "This query should have thrown a 404 error");
+        },
+          function (error) {
+            deepEqual(error.status_code,
+              404,
+              "Document does not have this many revisions.");
+          })
         .fail(function (error) {
           //console.log(error);
           ok(false, error);
@@ -416,27 +475,25 @@
     });
 
   /////////////////////////////////////////////////////////////////
-  // Accessing older revisions with two users
+  // Querying older revisions
   /////////////////////////////////////////////////////////////////
 
   module("bryanStorage.querying_old_revisions");
   test("Testing retrieval of older revisions via allDocs calls",
     function () {
       stop();
-      expect(47);
+      expect(37);
 
       // create storage of type "bryan" with memory as substorage
-      var dbname = "db-" + Date.now(),
-        jio = jIO.createJIO({
-          type: "bryan",
+      var jio = jIO.createJIO({
+        type: "bryan",
+        sub_storage: {
+          type: "uuid",
           sub_storage: {
-            type: "uuid",
-            sub_storage: {
-              type: "indexeddb",
-              database: dbname
-            }
+            type: "memory"
           }
-        });
+        }
+      });
       jio.put("doc", {
         "k": "v0"
       })
@@ -457,8 +514,7 @@
         })
         .push(function () {
           return jio.allDocs({
-            query: "",
-            revision_limit: [0, 1]
+            query: "_REVISION : 0"
           });
         })
         .push(function (results) {
@@ -474,8 +530,7 @@
         })
         .push(function () {
           return jio.allDocs({
-            query: "",
-            revision_limit: [1, 1]
+            query: "_REVISION : =1"
           });
         })
         .push(function (results) {
@@ -486,8 +541,7 @@
             "k": "v2"
           });
           return jio.allDocs({
-            query: "",
-            revision_limit: [2, 1]
+            query: "_REVISION : =2"
           });
         })
         .push(function (results) {
@@ -498,8 +552,7 @@
             "k": "v1"
           });
           return jio.allDocs({
-            query: "",
-            revision_limit: [3, 1]
+            query: "_REVISION : =3"
           });
         })
         .push(function (results) {
@@ -510,8 +563,7 @@
             "k": "v0"
           });
           return jio.allDocs({
-            query: "",
-            revision_limit: [4, 1]
+            query: "_REVISION : =4"
           });
         })
         .push(function (results) {
@@ -519,7 +571,7 @@
         })
         .push(function () {
           return jio.allDocs({
-            revision_limit: [0, 2]
+            query: "_REVISION: <= 1"
           });
         })
         .push(function (results) {
@@ -536,7 +588,7 @@
         })
         .push(function () {
           return jio.allDocs({
-            query: "",
+            query: "NOT (_REVISION: >= 1)",
             revision_limit: [0, 1]
           });
         })
@@ -546,8 +598,7 @@
         })
         .push(function () {
           return jio.allDocs({
-            query: "",
-            revision_limit: [1, 3]
+            query: "(_REVISION: >= 1) AND (_REVISION: <= 3)"
           });
         })
         .push(function (results) {
@@ -569,7 +620,7 @@
         })
         .push(function () {
           return jio.allDocs({
-            revision_limit: [1, 3]
+            query: "(_REVISION: >0) AND (_REVISION: <= 3)"
           });
         })
         .push(function (results) {
@@ -586,7 +637,7 @@
         })
         .push(function () {
           return jio.allDocs({
-            revision_limit: [0, 2]
+            query: "(_REVISION: = 0) OR (_REVISION: = 1)"
           });
         })
         .push(function (results) {
@@ -609,7 +660,6 @@
         .push(function (results) {
           equal(results.data.rows.length, 1,
             "There is only one non-removed doc");
-          equal(results.data.rows[0].rev, 0);
           deepEqual(results.data.rows[0].doc, {
             "k2": "w1"
           });
@@ -619,16 +669,13 @@
         })
         .push(function () {
           return jio.allDocs({
-            revision_limit: [0, 4]
+            query:
+              "_REVISION: 0 OR _REVISION: 1 OR " +
+              "(_REVISION: >= 2 AND _REVISION: <= 3)"
           });
         })
         .push(function (results) {
           equal(results.data.rows.length, 5);
-          equal(results.data.rows[0].rev, 1, "Rev parameter is correct");
-          equal(results.data.rows[1].rev, 2, "Rev parameter is correct");
-          equal(results.data.rows[2].rev, 1, "Rev parameter is correct");
-          equal(results.data.rows[3].rev, 2, "Rev parameter is correct");
-          equal(results.data.rows[4].rev, 3, "Rev parameter is correct");
           deepEqual(results.data.rows[0].doc, {
             "k2": "w1"
           });
@@ -647,17 +694,13 @@
         })
         .push(function () {
           return jio.allDocs({
-            limit: [1, 4],
-            revision_limit: [0, 4]
+            query: "_REVISION: <= 3",
+            limit: [1, 4]
           });
         })
         .push(function (results) {
           equal(results.data.rows.length, 4,
-            "Correct number of results with options.limit set");
-          equal(results.data.rows[0].rev, 2, "Rev parameter is correct");
-          equal(results.data.rows[1].rev, 1, "Rev parameter is correct");
-          equal(results.data.rows[2].rev, 2, "Rev parameter is correct");
-          equal(results.data.rows[3].rev, 3, "Rev parameter is correct");
+            "Correct number of results with optins.limit set");
           deepEqual(results.data.rows[0].doc, {
             "k2": "w0"
           }, "Correct results with options.limit set");