Merge branch 'romain_wip' into nodejs

Conflicts:
	Gruntfile.js
	package.json
	src/jio.storage/erp5storage.js
	test/tests.html

Gruntfile.js

@@ -170,8 +170,7 @@ module.exports = function (grunt) {
           'lib/uri/URI.js',
           'node_modules/uritemplate/bin/uritemplate.js',
           'node_modules/lz-string/libs/lz-string.js',
-//           'node_modules/moment/moment.js',
-          'lib/moment/moment-2.13.0.js',
+          'node_modules/moment/moment.js',
 
           // queries
           'src/queries/parser-begin.js',
@@ -190,6 +189,8 @@ module.exports = function (grunt) {
           'src/jio.storage/uuidstorage.js',
           'src/jio.storage/memorystorage.js',
           'src/jio.storage/zipstorage.js',
+          'src/jio.storage/parserstorage.js',
+          'src/jio.storage/httpstorage.js',
           'src/jio.storage/dropboxstorage.js',
           'src/jio.storage/davstorage.js',
           'src/jio.storage/gdrivestorage.js',
@@ -203,6 +204,7 @@ module.exports = function (grunt) {
           'src/jio.storage/cryptstorage.js',
           'src/jio.storage/websqlstorage.js',
           'src/jio.storage/mappingstorage.js'
+          'src/jio.storage/fbstorage.js'
         ],
         dest: 'dist/<%= pkg.name %>-<%= pkg.version %>.js'
 //         dest: 'jio.js'

README.md

-## Javascript Input/Output
+# jIO
 
-**jIO is a client-side JavaScript library to manage documents across multiple
-  storages.**
+jIO is a promised-based JavaScript library that offers connectors to many storages (Dropbox, webdav, IndexedDB, GDrive, ...) using a single API. jIO supports offline use, replication, encryption and synchronization as well as querying of stored documents and attachments allowing to build complex client-side centric web applications.
 
-### Getting Started
+### jIO Documentation
 
-To set up jIO you should include jio.js, dependencies and the connectors for the storages
-you want to use in the HTML page header (note that more dependencies may be required
-depending on type of storages being used):
+The documentation can be found on [https://jio.nexedi.com/](https://jio.nexedi.com/)
 
-```html
-<script src="RSVP.js"></script>
-<script src="jio-latest.js"></script>
-```
-
-Then create your jIO instance like this:
-
-```javascript
-// create a new jio
-var jio_instance = jIO.createJIO({
-  type: "query",
-  sub_storage: {
-    type: "uuid",
-    sub_storage: {
-      "type": "indexeddb",
-      "database": "test"
-    }
-  }
-});
-```
-
-### Documents and Methods
-
-Documents are JSON strings that contain *metadata* (properties, like a filename)
-and *attachments* (optional content, for example *image.jpg*).
-
-jIO exposes the following methods to *create*, *read*, *update* and *delete* documents
-(for more information, including revision management and available options for
-each method, please refer to the documentation):
-
-```javascript
-// create and store new document
-jio_instance.post({"title": "some title"})
-  .then(function (new_id) {
-    ...
-  });
-
-// create or update an existing document
-jio_instance.put(new_id, {"title": "New Title"})
-  .then(function () 
-    // console.log("Document stored");
-  });
-
-// add an attachement to a document
-jio_instance.putAttachment(document_id, attachment_id, new Blob())
-  .then(function () {
-    // console.log("Blob stored");
-  });
-
-// read a document
-jio_instance.get(document_id)
-  .then(function (document) {
-    // console.log(document);
-    // {
-    //   "title": "New Title",
-    // }
-  });
-
-// read an attachement
-jio_instance.getAttachment(document_id, attachment_id)
-  .then(function (blob) {
-    // console.log(blob);
-  });
-
-// delete a document and its attachment(s)
-jio_instance.remove(document_id)
-  .then(function () {
-    // console.log("Document deleted");
-  });
-
-// delete an attachement
-jio_instance.removeAttachment(document_id, attachment_id)
-  .then(function () {
-    // console.log("Attachment deleted");
-  });
-
-// get all documents
-jio_instance.allDocs().then(function (response) {
-  // console.log(response);
-  // {
-  //   "data": {
-  //     "total_rows": 1,
-  //     "rows": [{
-  //       "id": "my_document",
-  //       "value": {}
-  //     }]
-  //   }
-  // }
-});
-```
-
-
-### Example
-
-This is an example of how to store a video file with one attachment in local
-storage. Note that attachments should be added after document creation.
-
-```javascript
-// create a new localStorage
-var jio_instance = jIO.createJIO({
-  "type": "local",
-});
-
-var my_video_blob = new Blob([my_video_binary_string], {
-  "type": "video/ogg"
-});
-
-// post the document
-jio_instance.put("myVideo", {
-  "title"       : "My Video",
-  "format"      : ["video/ogg", "vorbis", "HD"],
-  "language"    : "en",
-  "description" : "Images Compilation"
-}).then(function (response) {
-
-  // add video attachment
-  return jio_instance.putAttachment(
-    "myVideo",
-    "video.ogv",
-    my_video_blob
-  });
-
-}).then(function (response) {
-
-  alert('Video Stored');
-
-}, function (err) {
-
-  alert('Error when attaching the video');
-
-}, function (progression) {
-
-  console.log(progression);
-
-});
-```
-
-### Storage Locations
-
-jIO allows to build "storage trees" consisting of connectors to multiple
-storages (webDav, xWiki, S3, localStorage) and use type-storages to add features
-like revision management or indices to a child storage (sub_storage).
-
-The following storages are currently supported:
-
-- LocalStorage (browser local storage)
-- IndexedDB
-- ERP5Storage
-- DAVStorage (connect to webDAV, more information on the
-  [documentation](https://www.j-io.org/documentation/jio-documentation/))
-
-For more information on the specific storages including guidelines on how to
-create your own connector, please also refer to the [documentation](https://www.j-io.org/documentation/jio-documentation).
-
-### jIO Query
-
-jIO can use queries, which can be run in the allDocs() method to query document
-lists. A sample query would look like this (note that not all storages support
-allDocs and jio queries, and that pre-querying of documents on distant storages
-should best be done server-side):
-
-```javascript
-// run allDocs with query option on an existing jIO
-jio_instance.allDocs({
-  "query": '(fieldX: >= "string" AND fieldY: < "string")',
-  // records to display ("from to")
-  "limit": [0, 5],
-  // sort by
-  "sort_on": [[<string A>, 'descending']],
-  // fields to return in response
-  "select_list": [<string A>, <string B>]
-}).then(function (response) {
-  // console.log(response);
-  // {
-  //   "total_rows": 1,
-  //   "rows": [{
-  //     "id": <string>,
-  //     "value": {
-  //       <string A>: <string>,
-  //       <string B>: <string>
-  //     }
-  //   }, { .. }]
-  // }
-});
-```
-
-To find out more about queries, please refer to the documentation.
-
-### Authors
-
-- Francois Billioud
-- Tristan Cavelier
-- Sven Franck
-- Romain Courteaud
-
-### Copyright and license
-
-jIO is an open-source library and is licensed under the LGPL license. More
-information on LGPL can be found
-[here](http://en.wikipedia.org/wiki/GNU_Lesser_General_Public_License).
-
-### Contribute
-
-Get development environment:
-
-
-    git clone https://lab.nexedi.com/nexedi/jio.git jio.git
-    cd jio.git
+### jIO Quickstart
+    git clone https://lab.nexedi.com/nexedi/jio.git
     npm install
-    alias grunt="./node_modules/grunt-cli/bin/grunt"
-    grunt
-
-
-Run tests:
-
-
     grunt server
 
 
-and open http://127.0.0.1:9000/test/tests.html
+### jIO Code
+
+RenderJS source code is hosted on Gitlab at [https://lab.nexedi.com/nexedi/jio](https://lab.nexedi.com/nexedi/jio) (Github [mirror](https://github.com/nexedi/jio/) - please use the issue tracker on Gitlab)
 
-Submit merge requests on lab.nexedi.com.
+### jIO Test
+You can run tests after installing and building jIO by opening the */test/* folder

examples/scenario_officejs.html

+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <title>OfficeJS Coverage Scenario</title>
+
+  <script src="../node_modules/rsvp/dist/rsvp-2.0.4.js"></script>
+  <script src="../dist/jio-latest.js"></script>
+
+  <link rel="stylesheet" href="../node_modules/grunt-contrib-qunit/test/libs/qunit.css" type="text/css" media="screen"/>
+  <script src="../node_modules/grunt-contrib-qunit/test/libs/qunit.js" type="text/javascript"></script>
+  <script src="scenario_officejs.js"></script>
+</head>
+
+<body>
+  <h1 id="qunit-header">OfficeJS Coverage Scenario</h1>
+  <h2 id="qunit-banner"></h2>
+  <div id="qunit-testrunner-toolbar"></div>
+  <h2 id="qunit-userAgent"></h2>
+  <ol id="qunit-tests"></ol>
+  <div id="qunit-fixture">test markup, will be hidden</div>
+</body>
+</html>

package.json

 {
   "name": "jio",
-  "version": "v3.16.0",
+  "version": "v3.27.0",
   "license": "LGPLv3",
   "author": "Nexedi SA",
   "contributors": [
@@ -29,9 +29,8 @@
     "cloud"
   ],
   "dependencies": {
-    "rsvp": "git+https://lab.nexedi.com/nexedi/rsvp.js.git",
+    "rsvp": "git+https://lab.nexedi.com/romain/rsvp.js.git#wip",
     "uritemplate": "git+https://lab.nexedi.com/nexedi/uritemplate-js.git",
-    "moment": "2.13.0",
     "rusha": "0.8.2",
     "urijs": "^1.18.9",
     "xhr2": "^0.1.4",
@@ -43,9 +42,11 @@
     "stream-buffers": "^3.0.1",
     "mockdoc": "^0.0.2",
     "sinon": "~2.1.0"
+    "moment": "2.21.0",
+    "rusha": "0.8.2"
   },
   "devDependencies": {
-    "renderjs": "git+https://lab.nexedi.com/nexedi/renderjs.git",
+    "renderjs": "git+https://lab.nexedi.com/romain/renderjs.git#wip",
     "grunt": "0.4.x",
     "grunt-cli": "~0.1.11",
     "grunt-contrib-concat": "0.3.x",

src/jio.js

@@ -40,6 +40,7 @@
    * @param  {String} [param.dataType=""] The data type to retrieve
    * @param  {String} param.url The url
    * @param  {Any} [param.data] The data to send
+   * @param  {Number} param.timeout The request timeout value
    * @param  {Function} [param.beforeSend] A function called just before the
    *    send request. The first parameter of this function is the XHR object.
    * @return {Promise} The promise
@@ -72,6 +73,12 @@
           }
         }
       }
+      if (param.timeout !== undefined && param.timeout !== 0) {
+        xhr.timeout = param.timeout;
+        xhr.ontimeout = function () {
+          return reject(new jIO.util.jIOError("Gateway Timeout", 504));
+        };
+      }
       if (typeof param.beforeSend === 'function') {
         param.beforeSend(xhr);
       }
@@ -132,6 +139,9 @@
     if (obj === undefined) {
       return undefined;
     }
+    if (obj === null) {
+      return 'null';
+    }
     if (obj.constructor === Object) {
       key_list = Object.keys(obj).sort();
       result_list = [];

src/jio.storage/cryptstorage.js

@@ -10,24 +10,29 @@
 (function (jIO, RSVP, DOMException, Blob, crypto, Uint8Array, ArrayBuffer) {
   "use strict";
 
+  /*
+  The cryptography system used by this storage is AES-GCM.
+  Here is an example of how to generate a key to the json format:
+
+  return new RSVP.Queue()
+    .push(function () {
+      return crypto.subtle.generateKey({name: "AES-GCM", length: 256},
+                                       true, ["encrypt", "decrypt"]);
+    })
+    .push(function (key) {
+      return crypto.subtle.exportKey("jwk", key);
+    })
+    .push(function (json_key) {
+      var jio = jIO.createJIO({
+        type: "crypt",
+        key: json_key,
+        sub_storage: {storage_definition}
+      });
+    });
 
-  // you the cryptography system used by this storage is AES-GCM.
-  // here is an example of how to generate a key to the json format.
-
-  // var key,
-  //     jsonKey;
-  // crypto.subtle.generateKey({name: "AES-GCM",length: 256},
-  //                           (true), ["encrypt", "decrypt"])
-  // .then(function(res){key = res;});
-  //
-  // window.crypto.subtle.exportKey("jwk", key)
-  // .then(function(res){jsonKey = val})
-  //
-  //var storage = jIO.createJIO({type: "crypt", key: jsonKey,
-  //                             sub_storage: {...}});
-
-  // find more informations about this cryptography system on
-  // https://github.com/diafygi/webcrypto-examples#aes-gcm
+  Find more informations about this cryptography system on
+  https://github.com/diafygi/webcrypto-examples#aes-gcm
+  */
 
   /**
    * The JIO Cryptography Storage extension
@@ -105,12 +110,12 @@
       })
       .push(function (dataURL) {
         //string->arraybuffer
-        var strLen = dataURL.currentTarget.result.length,
+        var strLen = dataURL.target.result.length,
           buf = new ArrayBuffer(strLen),
           bufView = new Uint8Array(buf),
           i;
 
-        dataURL = dataURL.currentTarget.result;
+        dataURL = dataURL.target.result;
         for (i = 0; i < strLen; i += 1) {
           bufView[i] = dataURL.charCodeAt(i);
         }
@@ -147,7 +152,7 @@
           .push(function (coded) {
             var initializaton_vector;
 
-            coded = coded.currentTarget.result;
+            coded = coded.target.result;
             initializaton_vector = new Uint8Array(coded.slice(0, 12));
             return new RSVP.Queue()
               .push(function () {

src/jio.storage/documentstorage.js

 /*jslint nomen: true*/
-/*global Blob, atob, btoa, RSVP*/
-(function (jIO, Blob, atob, btoa, RSVP) {
+/*global Blob, RSVP, unescape, escape*/
+(function (jIO, Blob, RSVP, unescape, escape) {
   "use strict";
-
   /**
    * The jIO DocumentStorage extension
    *
@@ -18,7 +17,13 @@
   var DOCUMENT_EXTENSION = ".json",
     DOCUMENT_REGEXP = new RegExp("^jio_document/([\\w=]+)" +
                                  DOCUMENT_EXTENSION + "$"),
-    ATTACHMENT_REGEXP = new RegExp("^jio_attachment/([\\w=]+)/([\\w=]+)$");
+    ATTACHMENT_REGEXP = new RegExp("^jio_attachment/([\\w=]+)/([\\w=]+)$"),
+    btoa = function (str) {
+      return window.btoa(unescape(encodeURIComponent(str)));
+    },
+    atob = function (str) {
+      return decodeURIComponent(escape(window.atob(str)));
+    };
 
   function getSubAttachmentIdFromParam(id, name) {
     if (name === undefined) {
@@ -225,4 +230,4 @@
 
   jIO.addStorage('document', DocumentStorage);
 
-}(jIO, Blob, atob, btoa, RSVP));
+}(jIO, Blob, RSVP, unescape, escape));

src/jio.storage/erp5storage.js

@@ -457,17 +457,31 @@
   ERP5Storage.prototype.hasCapacity = function (name) {
     return ((name === "list") || (name === "query") ||
             (name === "select") || (name === "limit") ||
-            (name === "sort")) || (name === "bulk_get");
+            (name === "sort"));
   };
 
   function isSingleLocalRoles(parsed_query) {
     if ((parsed_query instanceof SimpleQuery) &&
+        (parsed_query.operator === undefined) &&
         (parsed_query.key === 'local_roles')) {
       // local_roles:"Assignee"
       return parsed_query.value;
     }
   }
 
+  function isSingleDomain(parsed_query) {
+    if ((parsed_query instanceof SimpleQuery) &&
+        (parsed_query.operator === undefined) &&
+        (parsed_query.key !== undefined) &&
+        (parsed_query.key.indexOf('selection_domain_') === 0)) {
+      // domain_region:"europe/france"
+      var result = {};
+      result[parsed_query.key.slice('selection_domain_'.length)] =
+        parsed_query.value;
+      return result;
+    }
+  }
+
   function isMultipleLocalRoles(parsed_query) {
     var i,
       sub_query,
@@ -479,6 +493,7 @@
       for (i = 0; i < parsed_query.query_list.length; i += 1) {
         sub_query = parsed_query.query_list[i];
         if ((sub_query instanceof SimpleQuery) &&
+            (sub_query.key !== undefined) &&
             (sub_query.key === 'local_roles')) {
           local_role_list.push(sub_query.value);
         } else {
@@ -503,49 +518,76 @@
       .push(function (site_hal) {
         var query = options.query,
           i,
+          key,
           parsed_query,
           sub_query,
           result_list,
           local_roles,
+          local_role_found = false,
+          selection_domain,
           sort_list = [];
         if (options.query) {
           parsed_query = jIO.QueryFactory.create(options.query);
-
           result_list = isSingleLocalRoles(parsed_query);
           if (result_list) {
             query = undefined;
             local_roles = result_list;
           } else {
-
-            result_list = isMultipleLocalRoles(parsed_query);
+            result_list = isSingleDomain(parsed_query);
             if (result_list) {
               query = undefined;
-              local_roles = result_list;
-            } else if ((parsed_query instanceof ComplexQuery) &&
-                       (parsed_query.operator === 'AND')) {
-
-              // portal_type:"Person" AND local_roles:"Assignee"
-              for (i = 0; i < parsed_query.query_list.length; i += 1) {
-                sub_query = parsed_query.query_list[i];
-
-                result_list = isSingleLocalRoles(sub_query);
-                if (result_list) {
-                  local_roles = result_list;
-                  parsed_query.query_list.splice(i, 1);
-                  query = jIO.Query.objectToSearchText(parsed_query);
-                  i = parsed_query.query_list.length;
-                } else {
-                  result_list = isMultipleLocalRoles(sub_query);
+              selection_domain = result_list;
+            } else {
+
+              result_list = isMultipleLocalRoles(parsed_query);
+              if (result_list) {
+                query = undefined;
+                local_roles = result_list;
+              } else if ((parsed_query instanceof ComplexQuery) &&
+                         (parsed_query.operator === 'AND')) {
+
+                // portal_type:"Person" AND local_roles:"Assignee"
+                // AND selection_domain_region:"europe/france"
+                for (i = 0; i < parsed_query.query_list.length; i += 1) {
+                  sub_query = parsed_query.query_list[i];
+
+                  if (!local_role_found) {
+                    result_list = isSingleLocalRoles(sub_query);
+                    if (result_list) {
+                      local_roles = result_list;
+                      parsed_query.query_list.splice(i, 1);
+                      query = jIO.Query.objectToSearchText(parsed_query);
+                      local_role_found = true;
+                    } else {
+                      result_list = isMultipleLocalRoles(sub_query);
+                      if (result_list) {
+                        local_roles = result_list;
+                        parsed_query.query_list.splice(i, 1);
+                        query = jIO.Query.objectToSearchText(parsed_query);
+                        local_role_found = true;
+                      }
+                    }
+                  }
+
+                  result_list = isSingleDomain(sub_query);
                   if (result_list) {
-                    local_roles = result_list;
                     parsed_query.query_list.splice(i, 1);
                     query = jIO.Query.objectToSearchText(parsed_query);
-                    i = parsed_query.query_list.length;
+                    if (selection_domain) {
+                      for (key in result_list) {
+                        if (result_list.hasOwnProperty(key)) {
+                          selection_domain[key] = result_list[key];
+                        }
+                      }
+                    } else {
+                      selection_domain = result_list;
+                    }
+                    i -= 1;
                   }
+
                 }
               }
             }
-
           }
         }
 
@@ -555,6 +597,10 @@
           }
         }
 
+        if (selection_domain) {
+          selection_domain = JSON.stringify(selection_domain);
+        }
+
         return jIO.util.ajax({
           "type": "GET",
           "url": UriTemplate.parse(site_hal._links.raw_search.href)
@@ -564,7 +610,8 @@
               select_list: options.select_list || ["title", "reference"],
               limit: options.limit,
               sort_on: sort_list,
-              local_roles: local_roles
+              local_roles: local_roles,
+              selection_domain: selection_domain
             }),
           "xhrFields": {
             withCredentials: storage._thisCredentials

src/jio.storage/fbstorage.js

+/*jslint nomen: true */
+/*global RSVP, UriTemplate*/
+(function (jIO, RSVP, UriTemplate) {
+  "use strict";
+
+  var GET_POST_URL = "https://graph.facebook.com/v2.9/{+post_id}" +
+      "?fields={+fields}&access_token={+access_token}",
+    get_post_template = UriTemplate.parse(GET_POST_URL),
+    GET_FEED_URL = "https://graph.facebook.com/v2.9/{+user_id}/feed" +
+        "?fields={+fields}&limit={+limit}&since={+since}&access_token=" +
+        "{+access_token}",
+    get_feed_template = UriTemplate.parse(GET_FEED_URL);
+
+  function FBStorage(spec) {
+    if (typeof spec.access_token !== 'string' || !spec.access_token) {
+      throw new TypeError("Access Token must be a string " +
+                          "which contains more than one character.");
+    }
+    if (typeof spec.user_id !== 'string' || !spec.user_id) {
+      throw new TypeError("User ID must be a string " +
+                          "which contains more than one character.");
+    }
+    this._access_token = spec.access_token;
+    this._user_id = spec.user_id;
+    this._default_field_list = spec.default_field_list || [];
+    this._default_limit = spec.default_limit || 500;
+  }
+
+  FBStorage.prototype.get = function (id) {
+    var that = this;
+    return new RSVP.Queue()
+      .push(function () {
+        return jIO.util.ajax({
+          type: "GET",
+          url: get_post_template.expand({post_id: id,
+            fields: that._default_field_list, access_token: that._access_token})
+        });
+      })
+      .push(function (result) {
+        return JSON.parse(result.target.responseText);
+      });
+  };
+
+  function paginateResult(url, result, select_list) {
+    return new RSVP.Queue()
+      .push(function () {
+        return jIO.util.ajax({
+          type: "GET",
+          url: url
+        });
+      })
+      .push(function (response) {
+        return JSON.parse(response.target.responseText);
+      },
+        function (err) {
+          throw new jIO.util.jIOError("Getting feed failed " + err.toString(),
+            err.target.status);
+        })
+      .push(function (response) {
+        if (response.data.length === 0) {
+          return result;
+        }
+        var i, j, obj = {};
+        for (i = 0; i < response.data.length; i += 1) {
+          obj.id = response.data[i].id;
+          obj.value = {};
+          for (j = 0; j < select_list.length; j += 1) {
+            obj.value[select_list[j]] = response.data[i][select_list[j]];
+          }
+          result.push(obj);
+          obj = {};
+        }
+        return paginateResult(response.paging.next, result, select_list);
+      });
+  }
+
+  FBStorage.prototype.buildQuery = function (query) {
+    var that = this, fields = [], limit = this._default_limit,
+      template_argument = {
+        user_id: this._user_id,
+        limit: limit,
+        access_token: this._access_token
+      };
+    if (query.include_docs) {
+      fields = fields.concat(that._default_field_list);
+    }
+    if (query.select_list) {
+      fields = fields.concat(query.select_list);
+    }
+    if (query.limit) {
+      limit = query.limit[1];
+    }
+    template_argument.fields = fields;
+    template_argument.limit = limit;
+    return paginateResult(get_feed_template.expand(template_argument), [],
+      fields)
+      .push(function (result) {
+        if (!query.limit) {
+          return result;
+        }
+        return result.slice(query.limit[0], query.limit[1]);
+      });
+  };
+
+  FBStorage.prototype.hasCapacity = function (name) {
+    var this_storage_capacity_list = ["list", "select", "include", "limit"];
+    if (this_storage_capacity_list.indexOf(name) !== -1) {
+      return true;
+    }
+  };
+
+  jIO.addStorage('facebook', FBStorage);
+
+}(jIO, RSVP, UriTemplate));
\ No newline at end of file

src/jio.storage/httpstorage.js

+/*global RSVP, Blob*/
+/*jslint nomen: true*/
+(function (jIO, RSVP, Blob) {
+  "use strict";
+
+  function HttpStorage(spec) {
+    if (spec.hasOwnProperty('catch_error')) {
+      this._catch_error = spec.catch_error;
+    } else {
+      this._catch_error = false;
+    }
+    // If timeout not set, use 0 for no timeout value
+    this._timeout = spec.timeout || 0;
+  }
+
+  HttpStorage.prototype.get = function (id) {
+    var context = this;
+    return new RSVP.Queue()
+      .push(function () {
+        return jIO.util.ajax({
+          type: 'HEAD',
+          url: id,
+          timeout: context._timeout
+        });
+      })
+      .push(undefined, function (error) {
+        if (context._catch_error) {
+          return error;
+        }
+        if ((error.target !== undefined) &&
+            (error.target.status === 404)) {
+          throw new jIO.util.jIOError("Cannot find url " + id, 404);
+        }
+        throw error;
+      })
+      .push(function (response) {
+
+        var key_list = ["Content-Disposition", "Content-Type", "Date",
+                        "Last-Modified", "Vary", "Cache-Control", "Etag",
+                        "Accept-Ranges", "Content-Range"],
+          i,
+          key,
+          value,
+          result = {};
+        result.Status = response.target.status;
+        for (i = 0; i < key_list.length; i += 1) {
+          key = key_list[i];
+          value = response.target.getResponseHeader(key);
+          if (value !== null) {
+            result[key] = value;
+          }
+        }
+        return result;
+      });
+  };
+
+  HttpStorage.prototype.allAttachments = function () {
+    return {enclosure: {}};
+  };
+
+  HttpStorage.prototype.getAttachment = function (id, name) {
+    var context = this;
+    if (name !== 'enclosure') {
+      throw new jIO.util.jIOError("Forbidden attachment: "
+                                  + id + " , " + name,
+                                  400);
+    }
+    return new RSVP.Queue()
+      .push(function () {
+        return jIO.util.ajax({
+          type: 'GET',
+          url: id,
+          dataType: "blob",
+          timeout: context._timeout
+        });
+      })
+      .push(undefined, function (error) {
+        if (context._catch_error) {
+          return error;
+        }
+        if ((error.target !== undefined) &&
+            (error.target.status === 404)) {
+          throw new jIO.util.jIOError("Cannot find url " + id, 404);
+        }
+        throw error;
+      })
+      .push(function (response) {
+        return new Blob(
+          [response.target.response || response.target.responseText],
+          {"type": response.target.getResponseHeader('Content-Type') ||
+                   "application/octet-stream"}
+        );
+      });
+  };
+
+  jIO.addStorage('http', HttpStorage);
+
+}(jIO, RSVP, Blob));
\ No newline at end of file

src/jio.storage/parserstorage.js

+/*jslint nomen: true*/
+/*global jIO, DOMParser, Node */
+(function (jIO, DOMParser, Node) {
+  "use strict";
+
+  /////////////////////////////////////////////////////////////
+  // OPML Parser
+  /////////////////////////////////////////////////////////////
+  function OPMLParser(txt) {
+    this._dom_parser = new DOMParser().parseFromString(txt, 'text/xml');
+  }
+
+  OPMLParser.prototype.parseHead = function () {
+    // fetch all children instead
+    var channel_element = this._dom_parser.querySelector("opml > head"),
+      tag_element,
+      i,
+      result = {};
+
+    for (i = channel_element.childNodes.length - 1; i >= 0; i -= 1) {
+      tag_element = channel_element.childNodes[i];
+      if (tag_element.nodeType === Node.ELEMENT_NODE) {
+        result[tag_element.tagName] = tag_element.textContent;
+      }
+    }
+    return result;
+  };
+
+  OPMLParser.prototype.parseOutline = function (result_list, outline_element,
+                                                prefix, include, id) {
+    var attribute,
+      i,
+      child,
+      result = {};
+
+    if ((id === prefix) || (id === undefined)) {
+      result_list.push({
+        id: prefix,
+        value: {}
+      });
+      if (include) {
+        for (i = outline_element.attributes.length - 1; i >= 0; i -= 1) {
+          attribute = outline_element.attributes[i];
+          if (attribute.value) {
+            result[attribute.name] = attribute.value;
+          }
+        }
+        result_list[result_list.length - 1].doc = result;
+      }
+    }
+
+    for (i = outline_element.childNodes.length - 1; i >= 0; i -= 1) {
+      child = outline_element.childNodes[i];
+      if (child.tagName === 'outline') {
+        this.parseOutline(result_list, child, prefix + '/' + i, include, id);
+      }
+    }
+  };
+
+  OPMLParser.prototype.getDocumentList = function (include, id) {
+    var result_list,
+      item_list = this._dom_parser.querySelectorAll("body > outline"),
+      i;
+
+    if ((id === '/0') || (id === undefined)) {
+      result_list = [{
+        id: '/0',
+        value: {}
+      }];
+      if (include) {
+        result_list[0].doc = this.parseHead();
+      }
+    } else {
+      result_list = [];
+    }
+
+    for (i = 0; i < item_list.length; i += 1) {
+      this.parseOutline(result_list, item_list[i], '/1/' + i, include, id);
+    }
+    return result_list;
+  };
+
+  /////////////////////////////////////////////////////////////
+  // RSS Parser
+  /////////////////////////////////////////////////////////////
+  function RSSParser(txt) {
+    this._dom_parser = new DOMParser().parseFromString(txt, 'text/xml');
+  }
+
+  RSSParser.prototype.parseElement = function (element) {
+    var tag_element,
+      i,
+      j,
+      attribute,
+      result = {};
+
+    for (i = element.childNodes.length - 1; i >= 0; i -= 1) {
+      tag_element = element.childNodes[i];
+      if ((tag_element.nodeType === Node.ELEMENT_NODE) &&
+          (tag_element.tagName !== 'item')) {
+        result[tag_element.tagName] = tag_element.textContent;
+
+        for (j = tag_element.attributes.length - 1; j >= 0; j -= 1) {
+          attribute = tag_element.attributes[j];
+          if (attribute.value) {
+            result[tag_element.tagName + '_' + attribute.name] =
+              attribute.value;
+          }
+        }
+      }
+    }
+    return result;
+  };
+
+  RSSParser.prototype.getDocumentList = function (include, id) {
+    var result_list,
+      item_list = this._dom_parser.querySelectorAll("rss > channel > item"),
+      i;
+
+    if ((id === '/0') || (id === undefined)) {
+      result_list = [{
+        id: '/0',
+        value: {}
+      }];
+      if (include) {
+        result_list[0].doc = this.parseElement(
+          this._dom_parser.querySelector("rss > channel")
+        );
+      }
+    } else {
+      result_list = [];
+    }
+
+    for (i = 0; i < item_list.length; i += 1) {
+      if ((id === '/0/' + i) || (id === undefined)) {
+        result_list.push({
+          id: '/0/' + i,
+          value: {}
+        });
+        if (include) {
+          result_list[result_list.length - 1].doc =
+            this.parseElement(item_list[i]);
+        }
+      }
+    }
+    return result_list;
+  };
+
+  /////////////////////////////////////////////////////////////
+  // Helpers
+  /////////////////////////////////////////////////////////////
+  var parser_dict = {
+    'rss': RSSParser,
+    'opml': OPMLParser
+  };
+
+  function getParser(storage) {
+    return storage._sub_storage.getAttachment(storage._document_id,
+                                              storage._attachment_id,
+                                              {format: 'text'})
+      .push(function (txt) {
+        return new parser_dict[storage._parser_name](txt);
+      });
+  }
+
+  /////////////////////////////////////////////////////////////
+  // Storage
+  /////////////////////////////////////////////////////////////
+  function ParserStorage(spec) {
+    this._attachment_id = spec.attachment_id;
+    this._document_id = spec.document_id;
+    this._parser_name = spec.parser;
+    this._sub_storage = jIO.createJIO(spec.sub_storage);
+  }
+
+  ParserStorage.prototype.hasCapacity = function (capacity) {
+    return (capacity === "list") || (capacity === 'include');
+  };
+
+  ParserStorage.prototype.buildQuery = function (options) {
+    if (options === undefined) {
+      options = {};
+    }
+    return getParser(this)
+      .push(function (parser) {
+        return parser.getDocumentList((options.include_docs || false));
+      });
+  };
+
+  ParserStorage.prototype.get = function (id) {
+    return getParser(this)
+      .push(function (parser) {
+        var result_list = parser.getDocumentList(true, id);
+        if (result_list.length) {
+          return result_list[0].doc;
+        }
+        throw new jIO.util.jIOError(
+          "Cannot find parsed document: " + id,
+          404
+        );
+      });
+  };
+
+  jIO.addStorage('parser', ParserStorage);
+
+}(jIO, DOMParser, Node));
\ No newline at end of file

src/jio.storage/querystorage.js

 /*jslint nomen: true*/
-/*global RSVP*/
-(function (jIO, RSVP) {
+/*global RSVP, jiodate*/
+(function (jIO, RSVP, jiodate) {
   "use strict";
 
+  function dateType(str) {
+    return jiodate.JIODate(new Date(str).toISOString());
+  }
+
+  function initKeySchema(storage, spec) {
+    var property;
+    for (property in spec.schema) {
+      if (spec.schema.hasOwnProperty(property)) {
+        if (spec.schema[property].type === "string" &&
+            spec.schema[property].format === "date-time") {
+          storage._key_schema.key_set[property] = {
+            read_from: property,
+            cast_to: "dateType"
+          };
+          if (storage._key_schema.cast_lookup.dateType === undefined) {
+            storage._key_schema.cast_lookup.dateType = dateType;
+          }
+        } else {
+          throw new jIO.util.jIOError(
+            "Wrong schema for property: " + property,
+            400
+          );
+        }
+      }
+    }
+  }
+
   /**
    * The jIO QueryStorage extension
    *
@@ -11,7 +38,8 @@
    */
   function QueryStorage(spec) {
     this._sub_storage = jIO.createJIO(spec.sub_storage);
-    this._key_schema = spec.key_schema;
+    this._key_schema = {key_set: {}, cast_lookup: {}};
+    initKeySchema(this, spec);
   }
 
   QueryStorage.prototype.get = function () {
@@ -211,4 +239,4 @@
 
   jIO.addStorage('query', QueryStorage);
 
-}(jIO, RSVP));
+}(jIO, RSVP, jiodate));

src/jio.storage/websqlstorage.js

@@ -179,7 +179,7 @@
       return jIO.util.readBlobAsDataURL(blob);
     })
       .push(function (strBlob) {
-        argument_list[index + 2].push(strBlob.currentTarget.result);
+        argument_list[index + 2].push(strBlob.target.result);
         return;
       });
   }

src/queries/build/parser.js

@@ -90,7 +90,7 @@ case 5: case 8: case 11: case 14: case 16:
  this.$ = $$[$0]; 
 break;
 case 6:
- this.$ = mkComplexQuery('OR', [$$[$0-1], $$[$0]]); 
+ this.$ = mkComplexQuery('AND', [$$[$0-1], $$[$0]]); 
 break;
 case 7:
  this.$ = mkComplexQuery('OR', [$$[$0-2], $$[$0]]); 

src/queries/core/parser.par

@@ -45,7 +45,7 @@ end
 
 search_text
     : and_expression                { $$ = $1; }
-    | and_expression search_text    { $$ = mkComplexQuery('OR', [$1, $2]); }
+    | and_expression search_text    { $$ = mkComplexQuery('AND', [$1, $2]); }
     | and_expression OR search_text { $$ = mkComplexQuery('OR', [$1, $3]); }
     ;
 

src/queries/query.js

@@ -40,44 +40,92 @@
   /**
    * A sort function to sort items by key
    *
-   * @param  {String} key The key to sort on
-   * @param  {String} [way="ascending"] 'ascending' or 'descending'
+   * @param  {Array} sort_list List of couples [key, direction]
    * @return {Function} The sort function
    */
-  function sortFunction(key, way) {
-    var result;
-    if (way === 'descending') {
-      result = 1;
-    } else if (way === 'ascending') {
-      result = -1;
-    } else {
-      throw new TypeError("Query.sortFunction(): " +
-                          "Argument 2 must be 'ascending' or 'descending'");
-    }
-    return function (a, b) {
+  function generateSortFunction(key_schema, sort_list) {
+    return function sortByMultipleIndex(a, b) {
+      var result,
+        cast_to,
+        key = sort_list[0][0],
+        way = sort_list[0][1],
+        i,
+        l,
+        a_string_array,
+        b_string_array,
+        f_a,
+        f_b,
+        tmp;
+
+      if (way === 'descending') {
+        result = 1;
+      } else if (way === 'ascending') {
+        result = -1;
+      } else {
+        throw new TypeError("Query.sortFunction(): " +
+                            "Argument 2 must be 'ascending' or 'descending'");
+      }
+
+      if (key_schema !== undefined &&
+          key_schema.key_set !== undefined &&
+          key_schema.key_set[key] !== undefined &&
+          key_schema.key_set[key].cast_to !== undefined) {
+        if (typeof key_schema.key_set[key].cast_to === "string") {
+          cast_to = key_schema.cast_lookup[key_schema.key_set[key].cast_to];
+        } else {
+          cast_to = key_schema.key_set[key].cast_to;
+        }
+        f_a = cast_to(a[key]);
+        f_b = cast_to(b[key]);
+        if (typeof f_b.cmp === 'function') {
+          tmp = result * f_b.cmp(f_a);
+          if (tmp !== 0) {
+            return tmp;
+          }
+          if (sort_list.length > 1) {
+            return generateSortFunction(key_schema, sort_list.slice(1))(a, b);
+          }
+          return tmp;
+        }
+        if (f_a > f_b) {
+          return -result;
+        }
+        if (f_a < f_b) {
+          return result;
+        }
+        if (sort_list.length > 1) {
+          return generateSortFunction(key_schema, sort_list.slice(1))(a, b);
+        }
+        return 0;
+      }
+
       // this comparison is 5 times faster than json comparison
-      var i, l;
-      a = metadataValueToStringArray(a[key]) || [];
-      b = metadataValueToStringArray(b[key]) || [];
-      l = a.length > b.length ? a.length : b.length;
+      a_string_array = metadataValueToStringArray(a[key]) || [];
+      b_string_array = metadataValueToStringArray(b[key]) || [];
+      l = Math.max(a_string_array.length, b_string_array.length);
       for (i = 0; i < l; i += 1) {
-        if (a[i] === undefined) {
+        if (a_string_array[i] === undefined) {
           return result;
         }
-        if (b[i] === undefined) {
+        if (b_string_array[i] === undefined) {
           return -result;
         }
-        if (a[i] > b[i]) {
+        if (a_string_array[i] > b_string_array[i]) {
           return -result;
         }
-        if (a[i] < b[i]) {
+        if (a_string_array[i] < b_string_array[i]) {
           return result;
         }
       }
+      if (sort_list.length > 1) {
+        return generateSortFunction(key_schema, sort_list.slice(1))(a, b);
+      }
       return 0;
+
     };
   }
 
+
   /**
    * Sort a list of items, according to keys and directions.
    *
@@ -85,19 +133,15 @@
    * @param  {Array} list The item list to sort
    * @return {Array} The filtered list
    */
-  function sortOn(sort_on_option, list) {
-    var sort_index;
+  function sortOn(sort_on_option, list, key_schema) {
     if (!Array.isArray(sort_on_option)) {
       throw new TypeError("jioquery.sortOn(): " +
                           "Argument 1 is not of type 'array'");
     }
-    for (sort_index = sort_on_option.length - 1; sort_index >= 0;
-         sort_index -= 1) {
-      list.sort(sortFunction(
-        sort_on_option[sort_index][0],
-        sort_on_option[sort_index][1]
-      ));
-    }
+    list.sort(generateSortFunction(
+      key_schema,
+      sort_on_option
+    ));
     return list;
   }
 
@@ -158,6 +202,35 @@
     return list;
   }
 
+  function checkKeySchema(key_schema) {
+    var prop;
+
+    if (key_schema !== undefined) {
+      if (typeof key_schema !== 'object') {
+        throw new TypeError("Query().create(): " +
+                            "key_schema is not of type 'object'");
+      }
+      // key_set is mandatory
+      if (key_schema.key_set === undefined) {
+        throw new TypeError("Query().create(): " +
+                            "key_schema has no 'key_set' property");
+      }
+      for (prop in key_schema) {
+        if (key_schema.hasOwnProperty(prop)) {
+          switch (prop) {
+          case 'key_set':
+          case 'cast_lookup':
+          case 'match_lookup':
+            break;
+          default:
+            throw new TypeError("Query().create(): " +
+                               "key_schema has unknown property '" + prop + "'");
+          }
+        }
+      }
+    }
+  }
+
   /**
    * The query to use to filter a list of objects.
    * This is an abstract class.
@@ -165,7 +238,10 @@
    * @class Query
    * @constructor
    */
-  function Query() {
+  function Query(key_schema) {
+
+    checkKeySchema(key_schema);
+    this._key_schema = key_schema || {};
 
     /**
      * Called before parsing the query. Must be overridden!
@@ -238,7 +314,7 @@
     }
 
     if (option.sort_on) {
-      sortOn(option.sort_on, item_list);
+      sortOn(option.sort_on, item_list, this._key_schema);
     }
 
     if (option.limit) {
@@ -415,8 +491,8 @@
       return new RegExp("^" + stringEscapeRegexpCharacters(string) + "$");
     }
     return new RegExp("^" + stringEscapeRegexpCharacters(string)
-      .replace(regexp_percent, '.*')
-      .replace(regexp_underscore, '.') + "$");
+      .replace(regexp_percent, '[\\s\\S]*')
+      .replace(regexp_underscore, '.') + "$", "i");
   }
 
   /**
@@ -577,7 +653,7 @@
    */
   QueryFactory.create = function (object, key_schema) {
     if (object === "") {
-      return new Query();
+      return new Query(key_schema);
     }
     if (typeof object === "string") {
       object = parseStringToObject(object);
@@ -609,35 +685,6 @@
     throw new TypeError("This object is not a query");
   }
 
-  function checkKeySchema(key_schema) {
-    var prop;
-
-    if (key_schema !== undefined) {
-      if (typeof key_schema !== 'object') {
-        throw new TypeError("SimpleQuery().create(): " +
-                            "key_schema is not of type 'object'");
-      }
-      // key_set is mandatory
-      if (key_schema.key_set === undefined) {
-        throw new TypeError("SimpleQuery().create(): " +
-                            "key_schema has no 'key_set' property");
-      }
-      for (prop in key_schema) {
-        if (key_schema.hasOwnProperty(prop)) {
-          switch (prop) {
-          case 'key_set':
-          case 'cast_lookup':
-          case 'match_lookup':
-            break;
-          default:
-            throw new TypeError("SimpleQuery().create(): " +
-                               "key_schema has unknown property '" + prop + "'");
-          }
-        }
-      }
-    }
-  }
-
   /**
    * The SimpleQuery inherits from Query, and compares one metadata value
    *
@@ -649,11 +696,7 @@
    * @param  {String} spec.value The value of the metadata to compare
    */
   function SimpleQuery(spec, key_schema) {
-    Query.call(this);
-
-    checkKeySchema(key_schema);
-
-    this._key_schema = key_schema || {};
+    Query.call(this, key_schema);
 
     /**
      * Operator to use to compare object values
@@ -717,7 +760,8 @@
       matchMethod = null,
       operator = this.operator,
       value = null,
-      key = this.key;
+      key = this.key,
+      k;
 
     if (!(regexp_comparaison.test(operator))) {
       // `operator` is not correct, we have to change it to "like" or "="
@@ -736,6 +780,22 @@
       key = this._key_schema.key_set[key];
     }
 
+    // match with all the fields if key is empty
+    if (key === '') {
+      matchMethod = this.like;
+      value = '%' + this.value + '%';
+      for (k in item) {
+        if (item.hasOwnProperty(k)) {
+          if (k !== '__id' && item[k]) {
+            if (matchMethod(item[k], value) === true) {
+              return true;
+            }
+          }
+        }
+      }
+      return false;
+    }
+
     if (typeof key === 'object') {
       checkKey(key);
       object_value = item[key.read_from];

test/jio.storage/cryptstorage.tests.js

@@ -458,7 +458,7 @@
       .push(function (coded) {
         var iv;
 
-        coded = coded.currentTarget.result;
+        coded = coded.target.result;
         iv = new Uint8Array(coded.slice(0, 12));
         return crypto.subtle.decrypt({name : "AES-GCM", iv : iv},
                                      decryptKey, coded.slice(12));

test/jio.storage/documentstorage.tests.js

@@ -236,6 +236,46 @@
       });
   });
 
+  test("with special utf-8 char", function () {
+    stop();
+    expect(5);
+
+    Storage200.prototype.putAttachment = function (id, name, blob) {
+      equal(blob.type, "application/json", "Blob type is OK");
+      equal(id, "foo", "putAttachment 200 called");
+      equal(
+        name,
+        "jio_document/Zm9vw6kKYmFy5rWL6K+V5Zub8J+YiA==.json",
+        "putAttachment 200 called"
+      );
+
+      return jIO.util.readBlobAsText(blob)
+        .then(function (result) {
+          deepEqual(JSON.parse(result.target.result),
+                    {"title": "fooé\nbar测试四😈"},
+                    "JSON is in blob");
+          return id;
+        });
+    };
+    var jio = jIO.createJIO({
+      type: "document",
+      document_id: "foo",
+      sub_storage: {
+        type: "documentstorage200"
+      }
+    });
+
+    jio.put("fooé\nbar测试四😈", {"title": "fooé\nbar测试四😈"})
+      .then(function (result) {
+        equal(result, "fooé\nbar测试四😈");
+      })
+      .fail(function (error) {
+        ok(false, error);
+      })
+      .always(function () {
+        start();
+      });
+  });
   /////////////////////////////////////////////////////////////////
   // documentStorage.remove
   /////////////////////////////////////////////////////////////////

test/jio.storage/indexeddbstorage.tests.js

@@ -493,7 +493,10 @@
       })
       .fail(function (error) {
         ok(error instanceof jIO.util.jIOError);
-        equal(error.message, "Cannot find document");
+        equal(
+          error.message,
+          "IndexedDB: cannot find object 'inexistent' in the 'metadata' store"
+        );
         equal(error.status_code, 404);
       })
       .fail(function (error) {
@@ -679,7 +682,10 @@
       })
       .fail(function (error) {
         ok(error instanceof jIO.util.jIOError);
-        equal(error.message, "Cannot find document");
+        equal(
+          error.message,
+          "IndexedDB: cannot find object 'inexistent' in the 'metadata' store"
+        );
         equal(error.status_code, 404);
       })
       .fail(function (error) {
@@ -1351,6 +1357,33 @@
       });
   });
 
+  test("retrieve empty blob", function () {
+    var context = this,
+      attachment = "attachment",
+      blob = new Blob();
+    stop();
+    expect(1);
+
+    deleteIndexedDB(context.jio)
+      .then(function () {
+        return context.jio.put("foo", {"title": "bar"});
+      })
+      .then(function () {
+        return context.jio.putAttachment("foo", attachment, blob);
+      })
+      .then(function () {
+        return context.jio.getAttachment("foo", attachment);
+      })
+      .then(function (result) {
+        deepEqual(result, blob, "check empty blob");
+      })
+      .fail(function (error) {
+        ok(false, error);
+      })
+      .always(function () {
+        start();
+      });
+  });
   /////////////////////////////////////////////////////////////////
   // indexeddbStorage.removeAttachment
   /////////////////////////////////////////////////////////////////

test/queries/key-typechecks.tests.js

@@ -54,7 +54,7 @@
     } catch (e) {
       equal(e.name, 'TypeError', 'wrong exception type');
       equal(e.message,
-         "SimpleQuery().create(): key_schema is not of type 'object'",
+         "Query().create(): key_schema is not of type 'object'",
          'wrong exception message');
     }
 
@@ -64,7 +64,7 @@
     } catch (e) {
       equal(e.name, 'TypeError', 'wrong exception type');
       equal(e.message,
-         "SimpleQuery().create(): key_schema has no 'key_set' property",
+         "Query().create(): key_schema has no 'key_set' property",
          'wrong exception message');
     }
 
@@ -76,7 +76,7 @@
     } catch (e) {
       equal(e.name, 'TypeError', 'wrong exception type');
       equal(e.message,
-         "SimpleQuery().create(): key_schema has unknown property 'foobar'",
+         "Query().create(): key_schema has unknown property 'foobar'",
          'wrong exception message');
     }