Skip to content

jio
jio
Commit bf8de85a authored by Aurel's avatar Aurel
Browse files
Options

Merge branch 'master' into nodejs

parents 5b8c0c8e 754e5df9
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 27634 additions and 2909 deletions
dist/jio-latest.js
View file @ bf8de85a
......@@ -8618,10 +8618,12 @@ return new Parser;
queue
.push(function () {
if (argument_list.length > 0) {
var argument_array = argument_list.shift();
argument_array[0] = queue;
var argument_array = argument_list.shift(),
sub_queue = new RSVP.Queue();
argument_array[0] = sub_queue;
function_used.apply(context, argument_array);
pushAndExecute(queue);
return sub_queue;
}
});
}
......@@ -8754,8 +8756,8 @@ return new Parser;
});
}
function checkAttachmentSignatureDifference(skip_attachment_dict,
queue, source,
function checkAttachmentSignatureDifference(queue, skip_attachment_dict,
source,
destination, id, name,
conflict_force,
conflict_revert,
......@@ -8805,8 +8807,8 @@ return new Parser;
});
}
function checkAttachmentLocalDeletion(skip_attachment_dict,
queue, destination, id, name, source,
function checkAttachmentLocalDeletion(queue, skip_attachment_dict,
destination, id, name, source,
conflict_force, conflict_revert,
conflict_ignore) {
var status_hash;
......@@ -8827,7 +8829,9 @@ return new Parser;
function pushDocumentAttachment(skip_attachment_dict, id, source,
destination, options) {
var queue = new RSVP.Queue();
var queue = new RSVP.Queue(),
local_dict = {},
signature_dict = {};
return queue
.push(function () {
......@@ -8851,11 +8855,10 @@ return new Parser;
]);
})
.push(function (result_list) {
var local_dict = {},
signature_dict = {},
is_modification,
var is_modification,
is_creation,
key;
key,
argument_list = [];
for (key in result_list[0]) {
if (result_list[0].hasOwnProperty(key)) {
if (!skip_attachment_dict.hasOwnProperty(key)) {
......@@ -8878,30 +8881,45 @@ return new Parser;
is_creation = !signature_dict.hasOwnProperty(key)
&& options.check_creation;
if (is_modification === true || is_creation === true) {
checkAttachmentSignatureDifference(skip_attachment_dict,
queue, source,
destination, id, key,
options.conflict_force,
options.conflict_revert,
options.conflict_ignore,
is_creation,
is_modification);
argument_list.push([undefined,
skip_attachment_dict,
source,
destination, id, key,
options.conflict_force,
options.conflict_revert,
options.conflict_ignore,
is_creation,
is_modification]);
}
}
}
return dispatchQueue(
checkAttachmentSignatureDifference,
argument_list,
context._parallel_operation_attachment_amount
);
})
.push(function () {
var key, argument_list = [];
if (options.check_deletion === true) {
for (key in signature_dict) {
if (signature_dict.hasOwnProperty(key)) {
if (!local_dict.hasOwnProperty(key)) {
checkAttachmentLocalDeletion(skip_attachment_dict,
queue, destination, id, key,
argument_list.push([undefined,
skip_attachment_dict,
destination, id, key,
source,
options.conflict_force,
options.conflict_revert,
options.conflict_ignore);
options.conflict_ignore]);
}
}
}
return dispatchQueue(
checkAttachmentLocalDeletion,
argument_list,
context._parallel_operation_attachment_amount
);
}
});
}
......@@ -9474,7 +9492,7 @@ return new Parser;
return dispatchQueue(
repairDocument,
argument_list,
context._parallel_operation_attachment_amount
context._parallel_operation_amount
);
});
}
......@@ -11401,6 +11419,7 @@ return new Parser;
"type": "POST",
"url": name,
"data": data,
"dataType": "blob",
"xhrFields": {
withCredentials: true
}
......
dist/jio-latest.min.js
View file @ bf8de85a
This source diff could not be displayed because it is too large. You can view the blob instead.
dist/jio-v3.15.1.js 0 → 100644
View file @ bf8de85a
This diff is collapsed.
dist/jio-v3.15.1.min.js 0 → 100644
View file @ bf8de85a
This source diff could not be displayed because it is too large. You can view the blob instead.
dist/jio-v3.16.0.js 0 → 100644
View file @ bf8de85a
This diff is collapsed.
dist/jio-v3.16.0.min.js 0 → 100644
View file @ bf8de85a
This source diff could not be displayed because it is too large. You can view the blob instead.
docs/_templates/layout.html
View file @ bf8de85a
{%- extends "basic/layout.html" %}
{%- block extrahead %}
<link rel="stylesheet" type="text/css" href="_static/jiodocs.css">
<script type="text/javascript" src="_static/jiodocs.js"></script>
{{ super() }}
{% if theme_touch_icon %}
<link rel="apple-touch-icon" href="{{ pathto('_static/' ~ theme_touch_icon, 1) }}" />
......@@ -9,12 +8,3 @@
<meta name="viewport" content="width=device-width, initial-scale=0.9, maximum-scale=0.9">
{% endblock %}
{%- block relbar2 %}{% endblock %}
{%- block footer %}
<div class="footer">
&copy; Copyright {{ copyright }}.
<a href="https://github.com/nexedi/jio" class="github">
<img style="position: absolute; top: 0; right: 0; border: 0;"
src="https://s3.amazonaws.com/github/ribbons/forkme_right_green_007200.png"
alt="Fork me on GitHub">
</a>
{%- endblock %}
docs/_templates/side-downloads.html deleted 100644 → 0
View file @ 5b8c0c8e
<h4><a href="./getting_started.html#download-fork" style="color:#008800">Downloads</a></h4>
docs/_templates/side-gettingstarted.html deleted 100644 → 0
View file @ 5b8c0c8e
<h4><a href="./getting_started.html">Getting Started</a></h4>
docs/_templates/side-informed.html deleted 100644 → 0
View file @ 5b8c0c8e
<h3>Stay Informed</h3>
You can follow our <a href="http://www.j-io.org/blog">blog</a> to get notified about new releases.
docs/_templates/side-support.html deleted 100644 → 0
View file @ 5b8c0c8e
<h3>Support</h3>
<p>
We have a <a href="http://www.j-io.org/forum">forum</a> and an <a href="https://github.com/nexedi/jio/issues">issue tracker</a>
on GitHub.
</p>
docs/_templates/side-top.html deleted 100644 → 0
View file @ 5b8c0c8e
<p>
<iframe src="http://ghbtns.com/github-btn.html?user=nexedi&repo=jio&type=watch&count=true&size=large"
allowtransparency="true" frameborder="0" scrolling="0" width="200px" height="35px"></iframe>
</p>
docs/authors.rst deleted 100644 → 0
View file @ 5b8c0c8e
Authors
=======
* Francois Billioud
* Tristan Cavelier
* Sven Franck
* Romain Courteaud
docs/available_storages.rst deleted 100644 → 0
View file @ 5b8c0c8e
This diff is collapsed.
docs/conf.py
View file @ bf8de85a
......@@ -134,8 +134,7 @@ html_static_path = ['_static']
# Custom sidebar templates, maps document names to template names.
html_sidebars = {
'index': ['side-top.html', 'side-gettingstarted.html', 'side-downloads.html', 'side-support.html', 'side-informed.html', 'searchbox.html'],
'**': ['side-top.html', 'localtoc.html', 'relations.html', 'searchbox.html']
'index': []
}
......
docs/developers.rst deleted 100644 → 0
View file @ 5b8c0c8e
For developers
==============
Quick start
-----------
The source repository includes ready-to-use files, so in case you do
not want to build jIO yourself, just use *sha256.amd.js*, *rsvp.js*, *jio.js*
plus the storages and dependencies you need and you will be good to go.
If you want to modify or build jIO yourself, you need to
* Clone from a repository
``$ git clone https://lab.nexedi.com/nexedi/jio.git``
* Install `NodeJS <http://nodejs.org/>`_ (including ``npm``)
* Install the Grunt command line with ``npm``.
``# npm -g install grunt-cli``
* Install the dependencies.
``$ npm install``
* Compile the JS/CC parser.
``$ make`` (until we find out how to compile it with grunt)
* Run build.
``$ grunt``
Naming Conventions
------------------
All the code follows this :ref:`JavaScript Style Guide <style-guide>`.
How to design your own jIO Storage Library
------------------------------------------
Create a constructor:
.. code-block:: javascript
function MyStorage(storage_description) {
this._value = storage_description.value;
if (typeof this._value !== 'string') {
throw new TypeError("'value' description property " +
"is not a string");
}
}
Your storage must be added to jIO. This is done
with the ``jIO.addStorage()`` method, which requires two parameters: the storage
type (string) and a constructor (function). It is called like this:
.. code-block:: javascript
// add custom storage to jIO
jIO.addStorage('mystoragetype', MyStorage);
Please refer to *localstorage.js* implementation for a good example on how to
setup a storage and what methods are required.
docs/getting_started.rst deleted 100644 → 0
View file @ 5b8c0c8e
Getting started
===============
#. :ref:`Download <download-fork>` the core jIO, the storages you need and the
dependencies required for them.
#. Add the scripts to your HTML page in the following order:
.. code-block:: html
<!-- jio core + dependencies -->
<script src="rsvp.js"></script>
<script src="jio.js"></script>
<script ...>
#. jIO connects to a number of storages and allows adding handlers (or
functions) to specific storages.
You can use both handlers and available storages to build a storage
tree across which all documents will be maintained and managed by jIO.
.. code-block:: javascript
// create your jio instance
var my_jio = jIO.createJIO(storage_description);
You have to provide a ``storage_description`` object, providing location
and credentials.
Its format depends on the type of storage,
see :ref:`List of Available Storages <list-of-available-storages>`.
#. The jIO API provides ten main methods to manage documents across the storage(s) specified in your jIO storage tree.
For details on the ``document`` and ``attachment`` objects, see :ref:`What is a document? <what-is-a-document>`
======================= ======================================================
Method Example
======================= ======================================================
``.post()`` | ``my_jio.post(document, [options]);``
| Creates a new document
``.put()`` | ``my_jio.put(document, [options]);``
| Creates/Updates a document
``.putAttachment()`` | ``my_jio.putAttachement(attachment, [options]);```
| Updates/Adds an attachment to a document
``.get()`` | ``my_jio.get(document, [options]);``
| Reads a document
``.getAttachment()`` | ``my_jio.getAttachment(attachment, [options]);``
| Reads a document attachment
``.remove()`` | ``my_jio.remove(document, [options]);``
| Deletes a document and its attachments
``.removeAttachment()`` | ``my_jio.removeAttachment(attachment, [options]);``
| Deletes a document's attachment
``.allDocs()`` | ``my_jio.allDocs([options]);``
| Retrieves a list of existing documents
``.repair()`` | ``my_jio.repair(document, [options]);``
| Repairs the document
======================= ======================================================
.. _download-fork:
Download & Fork
---------------
You can get latest jIO release by using on of those links:
`Full download <https://lab.nexedi.com/nexedi/jio/raw/master/dist/jio-latest.js>`_
`Minified download <https://lab.nexedi.com/nexedi/jio/raw/master/dist/jio-latest.min.js>`_
You can get latest RSVP release by using on of those links:
`Full download <https://lab.nexedi.com/nexedi/rsvp.js/raw/master/dist/rsvp-2.0.4.js>`_
`Minified download <https://lab.nexedi.com/nexedi/rsvp.js/raw/master/dist/rsvp-2.0.4.min.js>`_
Fork jIO
^^^^^^^^
| Feel free to use the Gitlab repository:
| `GitLab <https://lab.nexedi.com/nexedi/jio.git>`_: ``git clone https://lab.nexedi.com/nexedi/jio.git``
docs/index.rst
View file @ bf8de85a
Welcome to jIO
==============
jIO is a JavaScript library that allows to manage JSON documents on local or
remote storages in asynchronous fashion. jIO is an abstracted API mapped after
CouchDB, that offers connectors to multiple storages, special handlers to
enhance functionality (replication, revisions, indexing) and a query module to
retrieve documents and specific information across storage trees.
How does it work?
-----------------
jIO is composed of two parts - jIO core and storage libraries. The core
makes use of storage libraries (connectors) to interact with the associated remote
storage servers. Some queries can be used on top of the ``.allDocs()`` method to
query documents based on defined criteria.
jIO uses a job management system, so each method call adds a job into a
queue. The queue is copied in the browser's local storage (by default), so it
can be restored in case of browser crash. Jobs are invoked
asynchronously and ongoing jobs are not able to re-trigger to prevent
conflicts.
Copyright and license
---------------------
jIO is open source and is licensed under the `LGPL <http://en.wikipedia.org/wiki/GNU_Lesser_General_Public_License>`_ license.
jIO documentation
-----------------
.. toctree::
:maxdepth: 2
getting_started
manage_documents
replicate_storage
available_storages
query
keys
metadata
developers
style_guide
authors
jIO documentation moved to `jio.nexedi.com <https://jio.nexedi.com/>`_.
docs/keys.rst deleted 100644 → 0
View file @ 5b8c0c8e
Search Keys
===========
Features like case insensitive, accent-removing, full-text searches and more can be implemented
by customizing jIO's query behavior.
Let's start with a simple search:
.. code-block:: javascript
var query = {
type: 'simple',
key: 'someproperty',
value: comparison_value,
operator: '='
}
Each of the ``.someproperty`` attribute in objects' metadata is compared with
``comparison_value`` through a function defined by the '=' operator.
You can provide your own function to be used as '=' operator:
.. code-block:: javascript
var strictEqual = function (object_value, comparison_value) {
return comparison_value === object_value;
};
var query = {
type: 'simple',
key: {
read_from: 'someproperty',
equal_match: strictEqual
},
value: comparison_value
}
Inside ``equal_match``, you can decide to interpret the wildcard character ``%``
or just ignore it, as in this case.
If you need to convert or preprocess the values before comparison, you can provide
a conversion function:
.. code-block:: javascript
var numberType = function (obj) {
return parseFloat('3.14');
};
var query = {
type: 'simple',
key: {
read_from: 'someproperty',
cast_to: numberType
},
value: comparison_value
}
In this case, the operator is still the default '=' that works with strings.
You can combine ``cast_to`` and ``equal_match``:
.. code-block:: javascript
var query = {
type: 'simple',
key: {
read_from: 'someproperty',
cast_to: numberType,
equal_match: strictEqual
},
value: comparison_value
}
Now the query returns all objects for which the following is true:
.. code-block:: javascript
strictEqual(numberType(metadata.someproperty),
numberType(comparison_value))
For a more useful example, the following function removes the accents
from any string:
.. code-block:: javascript
var accentFold = function (s) {
var map = [
[new RegExp('[àáâãäå]', 'gi'), 'a'],
[new RegExp('æ', 'gi'), 'ae'],
[new RegExp('ç', 'gi'), 'c'],
[new RegExp('[èéêë]', 'gi'), 'e'],
[new RegExp('[ìíîï]', 'gi'), 'i'],
[new RegExp('ñ', 'gi'), 'n'],
[new RegExp('[òóôõö]', 'gi'), 'o'],
[new RegExp('œ', 'gi'), 'oe'],
[new RegExp('[ùúûü]', 'gi'), 'u'],
[new RegExp('[ýÿ]', 'gi'), 'y']
];
map.forEach(function (o) {
var rep = function (match) {
if (match.toUpperCase() === match) {
return o[1].toUpperCase();
}
return o[1];
};
s = s.replace(o[0], rep);
});
return s;
};
...
cast_to: accentFold
...
A more robust solution to manage diacritics is recommended for production
environments, with unicode normalization, like (untested):
https://github.com/walling/unorm/
Overriding operators and sorting
--------------------------------
The advantage of providing an ``equal_match`` function is that it can work with basic types;
you can keep the values as strings or, if you use a ``cast_to`` function, it can return strings,
numbers, arrays... and that's fine if all you need is the '=' operator.
It's also possible to customize the behavior of the other operators: <, >, !=...
To do that, the object returned by ``cast_to`` must contain a ``.cmp``
property, that behaves like the ``compareFunction`` described in
`Array.prototype.sort() <https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/sort>`_:
.. code-block:: javascript
function myType (...) {
...
return {
...
'cmp': function (b) {
if (a < b) {
return -1;
}
if (a > b) {
return +1;
}
return 0;
}
};
}
...
cast_to: myType
...
If the < or > comparison makes no sense for the objects, the function should return ``undefined``.
The ``.cmp()`` property is also used, if present, by the sorting feature of queries.
Partial Date/Time match
-----------------------
As a real life example, consider a list of documents that have a *start_task* property.
The value of ``start_task`` can be an `ISO 8601 <http://en.wikipedia.org/wiki/ISO_8601>`_ string
with date and time information including fractions of a second. Which is, honestly, a bit too
much for most queries.
By using a ``cast_to`` function with custom operators, it is possible to perform queries like
"start_task > 2010-06", or "start_task != 2011". Partial time can be used as well, so
we can ask for projects started after noon of a given day: ``start_task = "2011-04-05" AND start_task > "2011-04-05 12"``
The JIODate type has been implemented on top of the `Moment.js <http://momentjs.com/>`_ library, which
has a rich API with support for multiple languages and timezones. No special support for timezones
is present (yet) in JIODate.
To use JIODate, include the ``jiodate.js`` and ``moment.js`` files in your
application, then set ``cast_to = jiodate.JIODate``.
Key Schemas
-----------
Instead of providing the key object for each attribute you want to filter,
you can group all of them in a schema object for reuse:
.. code-block:: javascript
var key_schema = {
key_set: {
date_day: {
read_from: 'date',
cast_to: 'dateType',
equal_match: 'sameDay'
},
date_month: {
read_from: 'date',
cast_to: 'dateType',
equal_match: 'sameMonth'
}
},
cast_lookup: {
dateType: function (str) {
return new Date(str);
}
},
match_lookup: {
sameDay: function (a, b) {
return (
(a.getFullYear() === b.getFullYear()) &&
(a.getMonth() === b.getMonth()) &&
(a.getDate() === b.getDate())
);
},
sameMonth: function (a, b) {
return (
(a.getFullYear() === b.getFullYear()) &&
(a.getMonth() === b.getMonth())
);
}
}
}
With this schema, we have created two 'virtual' metadata attributes,
``date_day`` and ``date_month``. When queried, they match values that
happen to be in the same day, ignoring the time, or the same month, ignoring
both time and day.
A key_schema object can have three properties:
* ``key_set`` - required.
* ``cast_lookup`` - optional, an object of the form ``{name: function}`` that is
used if cast_to is a string. If cast_lookup is not provided,
then cast_to must be a function.
* ``match_lookup`` - optional, an object of the form ``{name: function}`` that is
used if ``equal_match`` is a string. If match_lookup is not provided,
then ``equal_match`` must be a function.
Using a schema
--------------
A schema can be used:
* In a query constructor. The same schema will be applied to all the sub-queries:
.. code-block:: javascript
jIO.QueryFactory.create({...}, key_schema).exec(...);
* In the ``jIO.createJIO()`` method. The same schema will be used
by all the queries created with the ``.allDocs()`` method:
.. code-block:: javascript
var jio = jIO.createJIO({
type: 'local',
username: '...',
application_name: '...',
key_schema: key_schema
});
docs/manage_documents.rst deleted 100644 → 0
View file @ 5b8c0c8e
How to manage documents?
========================
jIO is mapped after the CouchDB APIs and extends them to provide unified, scalable
and high performance access via JavaScript to a wide variety of storage backends.
If you are not familiar with `Apache CouchDB <http://couchdb.apache.org/>`_:
it is a scalable, fault-tolerant, and schema-free document-oriented database.
It is used in large and small organizations for a variety of applications where
traditional SQL databases are not the best solution for the problem at hand.
CouchDB provides a RESTful HTTP/JSON API accessible by many programming
libraries and tools (like `curl <http://curl.haxx.se/>`_ or `Pouchdb <http://pouchdb.com/>`_)
and has its own conflict management system.
.. _what-is-a-document:
What is a document?
-------------------
A document is an association of metadata and attachment(s). The metadata is the
set of properties of the document and the attachments are binary (or text) objects
that represent the content of the document.
In jIO, the metadata is a dictionary with keys and values (a JSON object), and
attachments are simple strings.
.. code-block:: javascript
{
// document metadata
title: 'A Title!',
creator: 'Mr.Author'
}
:ref:`Here <metadata-head>` is a draft about metadata to use with jIO.
Basic Methods
-------------
Below you can see examples of the main jIO methods.
.. code-block:: javascript
// Create a new jIO instance
var jio_instance = jIO.createJIO(storage_description);
// create and store new document
jio_instance.post({title: 'my document'}).
then(function (response) {
// console.log(response);
});
// create or update an existing document
jio_instance.put('document_name', {title: 'another document'}).
then(function (response) {
// console.log(response);
});
// add an attachment to a document
jio_instance.putAttachment('document_name',
'attachment_name',
new Blob([data], {'type' : data_mimetype});
).
then(function (response) {
// console.log(response);
});
// read a document
jio_instance.get('document_name').
then(function (response) {
// console.log(response);
});
// read an attachment
jio_instance.getAttachment('document_name',
'attachment_name').
then(function (response) {
// console.log(response);
});
// delete a document and its attachment(s)
jio_instance.remove('document_name').
then(function (response) {
// console.log(response);
});
// delete an attachment
jio_instance.removeAttachment('document_name',
'attachment_name').
then(function (response) {
// console.log(response);
});
// get all documents
jio_instance.allDocs().then(function (response) {
// console.log(response);
});
Promises
--------
Each jIO method (with the exception of ``.createJIO()``) returns a Promise object, which allows us to get responses into
callback parameters and to chain callbacks with other returned values.
jIO uses a custom version of `RSVP.js <https://lab.nexedi.com/nexedi/rsvp.js>`_, adding canceler and progression features.
You can read more about promises:
* `RSVP.js <https://lab.nexedi.com/nexedi/rsvp.js>`_ on GitHub
* `Promises/A+ <http://promisesaplus.com/>`_
* `CommonJS Promises <http://wiki.commonjs.org/wiki/Promises>`_
Method Options and Callback Responses
-------------------------------------
To retrieve jIO responses, you have to provide callbacks like this:
.. code-block:: javascript
jio_instance.post(metadata, [options]).
then([responseCallback], [errorCallback], [progressionCallback]);
* On command success, ``responseCallback`` is called with the jIO response as first parameter.
* On command error, ``errorCallback`` is called with the jIO error as first parameter.
* On command notification, ``progressionCallback`` is called with the storage notification.
Here is a list of responses returned by jIO according to methods and options:
============================================== ================== ===============================================
Available for Option Response (Callback first parameter)
============================================== ================== ===============================================
``.post()``, ``.put()``, ``.remove()`` Any id of the document affected (string)
``.putAttachment()``, ``.removeAttachment()`` Any no specific value
``.get()`` Any document_metadata (object)
``.getAttachment()`` Any .. code-block:: javascript
new Blob([data], {"type": content_type})
``.allDocs()`` No option .. code-block:: javascript
{
total_rows: 1,
rows: [{
id: 'mydoc',
value: {},
}]
}
``.allDocs()`` include_docs: true .. code-block:: javascript
{
total_rows: 1,
rows: [{
id: 'mydoc',
value: {
// Here, 'mydoc' metadata
}
}]
}
============================================== ================== ===============================================
In case of error, the ``errorCallback`` first parameter looks like:
.. code-block:: javascript
{
status_code: 404,
message: 'Unable to get the requested document'
}
How to store binary data
------------------------
The following example creates a new jIO in localStorage and then posts a document with two attachments.
.. code-block:: javascript
// create a new jIO
var jio_instance = jIO.createJIO({type: 'indexeddb'});
// post the document 'myVideo'
jio_instance.put( 'metadata', {
title : 'My Video',
type : 'MovingImage',
format : 'video/ogg',
description : 'Images Compilation'
})
.push(undefined, function(err) {
// pushes error handler with RSVP.Queue push method
// nothing to do with JIO itself
return alert('Error posting the document metadata');
});
// post a thumbnail attachment
jio_instance.putAttachment('metadata',
'thumbnail',
new Blob([my_image], {type: 'image/jpeg'})
).push(undefined, function(err) {
return alert('Error attaching thumbnail');
});
// post video attachment
jio_instance.putAttachment('metadata',
'video',
new Blob([my_video], {type: 'video/ogg'})
).push(undefined, function(err) {
return alert('Error attaching video');
});
alert('Video Stored');
indexedDB Storage now contains:
.. code-block:: javascript
{
"/myVideo/": {
"title": "My Video",
"type": "MovingImage",
"format": "video/ogg",
"description": "Images Compilation",
"_attachments":{
"thumbnail":{
"digest": "md5-3ue...",
"content_type": "image/jpeg",
"length": 17863
},
"video":{
"digest": "md5-0oe...",
"content_type": "video/ogg",
"length": 2840824
}
}
},
"/myVideo/thumbnail": "...",
"/myVideo/video": "..."
}
docs/metadata.rst deleted 100644 → 0
View file @ 5b8c0c8e
This diff is collapsed.
docs/query.rst deleted 100644 → 0
View file @ 5b8c0c8e
jIO Query
=========
What are Queries?
-----------------
In jIO, a query can ask a storage server to select, filter, sort, or limit a
document list before sending it back. If the server is not able to do so, the
jio query tool can do the filtering by itself on the client. Only the
``.allDocs()`` method can use jio queries.
A query can either be a string (using a specific language useful for writing
queries), or it can be a tree of objects (useful to browse queries). To handle
queries, jIO uses a parsed grammar file which is compiled using `JSCC
<http://jscc.phorward-software.com/>`_.
Why use JIO Queries?
--------------------
JIO queries can be used like database queries, for tasks such as:
* search a specific document
* sort a list of documents in a certain order
* avoid retrieving a list of ten thousand documents
* limit the list to show only N documents per page
For some storages (like localStorage), jio queries can be a powerful tool to
query accessible documents. When querying documents on a distant storage, some
server-side logic should be run to avoid returning too many documents to the
client. If distant storages are static, an alternative would be to use an
indexStorage with appropriate indices as jio queries will always try to run the
query on the index before querying documents itself.
How to use Queries with jIO?
----------------------------
Queries can be triggered by including the option named **query** in the ``.allDocs()`` method call.
Example:
.. code-block:: javascript
var options = {};
// search text query
options.query = '(creator:"John Doe") AND (format:"pdf")';
// OR query tree
options.query = {
type: 'complex',
operator: 'AND',
query_list: [{
type: 'simple',
key: 'creator',
value: 'John Doe'
}, {
type: 'simple',
key: 'format',
value: 'pdf'
}]
};
// FULL example using filtering criteria
options = {
query: '(creator:"% Doe") AND (format:"pdf")',
limit: [0, 100],
sort_on: [
['last_modified', 'descending'],
['creation_date', 'descending']
],
select_list: ['title']
};
// execution
jio_instance.allDocs(options, callback);
How to use Queries outside jIO?
-------------------------------
Refer to the `JIO Query sample page <https://lab.nexedi.com/nexedi/jio/blob/master/examples/example-queries.html>`_
for how to use these methods, in and outside jIO. The module provides:
.. code-block:: javascript
jIO: {
QueryFactory: { [Function: QueryFactory] create: [Function] },
Query: { [Function: Query],
parseStringToObject: [Function],
stringEscapeRegexpCharacters: [Function],
select: [Function],
sortOn: [Function],
limit: [Function],
searchTextToRegExp: [Function],
}
SimpleQuery: {
[Function: SimpleQuery] super_: [Function: Query]
},
ComplexQuery: {
[Function: ComplexQuery] super_: [Function: Query]
}
}
(Reference API coming soon.)
Basic example:
.. code-block:: javascript
// object list (generated from documents in storage or index)
var object_list = [
{"title": "Document number 1", "creator": "John Doe"},
{"title": "Document number 2", "creator": "James Bond"}
];
// the query to run
var query = 'title: "Document number 1"';
// running the query
var result = jIO.QueryFactory.create(query).exec(object_list);
// console.log(result);
// [ { "title": "Document number 1", "creator": "John Doe"} ]
Other example:
.. code-block:: javascript
var result = jIO.QueryFactory.create(query).exec(
object_list,
{
"select": ['title', 'year'],
"limit": [20, 20], // from 20th to 40th document
"sort_on": [['title', 'ascending'], ['year', 'descending']],
"other_keys_and_values": "are_ignored"
}
);
// this case is equal to:
var result = jIO.QueryFactory.
create(query).exec(object_list);
jIO.Query.sortOn([
['title', 'ascending'],
['year', 'descending']
], result);
jIO.Query.limit([20, 20], result);
jIO.Query.select(['title', 'year'], result);
Query in storage connectors
^^^^^^^^^^^^^^^^^^^^^^^^^^^
The query exec method must only be used if the server is not able to pre-select
documents. As mentioned before, you could use an indexStorage to maintain
indices with key information on all documents in a storage. This index file
will then be used to run queries, if all of the fields required in the query answer
are available in the index.
Matching properties
^^^^^^^^^^^^^^^^^^^
Queries select items which exactly match the value given in the query
but you can also use wildcards (``%``). If you don't want to use a wildcard,
just set the operator to ``=``.
.. code-block:: javascript
var option = {
query: 'creator:"% Doe"' // use wildcard
};
var option = {
query: 'creator:="25%"' // don't use wildcard
};
Should default search types be defined in jIO or in user interface components?
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Default search types should be defined in the application's user interface
components because criteria like filters will be changed frequently by the
component (change ``limit: [0, 10]`` to ``limit: [10, 10]`` or ``sort_on: [['title',
'ascending']]`` to ``sort_on: [['creator', 'ascending']]``) and each component must
have its own default properties to keep their own behavior.
Query into another type
^^^^^^^^^^^^^^^^^^^^^^^
Example, convert Query object into a human readable string:
.. code-block:: javascript
var query = jIO.QueryFactory.
create('year: < 2000 OR title: "%a"'),
option = {
limit: [0, 10]
},
human_read = {
"": "matches ",
"<": "is lower than ",
"<=": "is lower or equal than ",
">": "is greater than ",
">=": "is greater or equal than ",
"=": "is equal to ",
"!=": "is different than "
};
query.onParseStart = function (object, option) {
object.start = "We need only the " +
option.limit[1] +
" elements from the number " +
option.limit[0] + ". ";
};
query.onParseSimpleQuery = function (object, option) {
object.parsed = object.parsed.key +
" " + human_read[object.parsed.operator || ""] +
object.parsed.value;
};
query.onParseComplexQuery = function (object, option) {
object.parsed = "I want all document where " +
object.parsed.query_list.join(
" " + object.parsed.operator.toLowerCase() + " "
) + ". ";
};
query.onParseEnd = function (object, option) {
object.parsed = object.start + object.parsed + "Thank you!";
};
console.log(query.parse(option));
// logged: "We need only the 10 elements from the number 0. I want all
// document where year is lower than 2000 or title matches %a. Thank you!"
JSON Schemas and Grammar
------------------------
Below you can find schemas for constructing queries.
* Complex Query JSON Schema:
.. code-block:: javascript
{
"id": "ComplexQuery",
"properties": {
"type": {
"type": "string",
"format": "complex",
"default": "complex",
"description": "Type is used to recognize the query type"
},
"operator": {
"type": "string",
"format": "(AND|OR|NOT)",
"required": true,
"description": "Can be 'AND', 'OR' or 'NOT'."
},
"query_list": {
"type": "array",
"items": {
"type": "object"
},
"required": true,
"default": [],
"description": "query_list is a list of queries which " +
"can be in serialized format " +
"or in object format."
}
}
}
* Simple Query JSON Schema:
.. code-block:: javascript
{
"id": "SimpleQuery",
"properties": {
"type": {
"type": "string",
"format": "simple",
"default": "simple",
"description": "Type is used to recognize the query type."
},
"operator": {
"type": "string",
"default": "",
"format": "(>=?|<=?|!?=|)",
"description": "The operator used to compare."
},
"id": {
"type": "string",
"default": "",
"description": "The column id."
},
"value": {
"type": "string",
"default": "",
"description": "The value we want to search."
}
}
}
* JIO Query Grammar::
search_text
: and_expression
| and_expression search_text
| and_expression OR search_text
and_expression
: boolean_expression
| boolean_expression AND and_expression
boolean_expression
: NOT expression
| expression
expression
: ( search_text )
| COLUMN expression
| value
value
: OPERATOR string
| string
string
: WORD
| STRING
terminal:
OR -> /OR[ ]/
AND -> /AND[ ]/
NOT -> /NOT[ ]/
COLUMN -> /[^><!= :\(\)"][^ :\(\)"]*:/
STRING -> /"(\\.|[^\\"])*"/
WORD -> /[^><!= :\(\)"][^ :\(\)"]*/
OPERATOR -> /(>=?|<=?|!?=)/
LEFT_PARENTHESE -> /\(/
RIGHT_PARENTHESE -> /\)/
ignore: " "
docs/replicate_storage.rst deleted 100644 → 0
View file @ 5b8c0c8e
.. _replicate-storage-conflicts-and-resolution:
Replicate Storage: Conflicts and Resolution
===========================================
Why Conflicts can Occur
-----------------------
Using jIO you can store documents in multiple locations. With an
increasing number of users working on a document and some storages not being
available or responding too slow, conflicts are more likely to occur. jIO
defines a conflict as multiple versions of a document existing in a storage
tree and a user trying to save on a version that does not match the latest
version of the document.
To keep track of document versions a replicate storage must be used. When doing
so, jIO creates a document tree file for every document. This file contains all
existing versions and their status and is modified whenever a version is
added/updated/removed or when storages are being synchronized.
How conflicts are handled
-------------------------
The RemoteStorage takes in parameter two substorages, one "local" and one "remote".
The "local" storage can be remote, but it will be used for all the requests
like **get()**, **getAttachment()**, **allDocs()**...
Using the document tree, jIO tries to make every version of a document
available on the two storages. When multiple versions of a document exist,
Jio will follow the rule set by the conflict_handling option, given at storage creation.
This option can one of the following numbers:
* 0: no conflict resolution (throws an error when conflict is occuring)
* 1: keep the local state. (overwrites the remote document with local content)
* 2: keep the remote state. (overwrites the local document with remote content)
* 3: keep both copies (leave documents untouched, no signature update)
Simple Conflict Example
-----------------------
.. TODO this is a little confusing
You are keeping a namecard file on your PC updating from your smartphone. Your
smartphone ran out of battery and is offline when you update your namecard on
your PC with your new email adress. Someone else changes this email from your PC
and once your smartphone is recharged, you go back online and the previous
update is executed.
#. Set up the replicate storage:
.. code-block:: javascript
var jio_instance = jIO.createJIO({
// replicate storage
type: 'replicate',
local_sub_storage : {
type: 'local',
...
}
remote_sub_storage: {
type: 'dav',
...
}
conflict_handling: ...
});
#. 1) Create the namecard on your smartphone:
.. code-block:: javascript
jio_instance.put('myNameCard', {
email: 'jb@td.com'
}).then(function (response) {
// response -> 'myNameCard'
});
This will create the document on your WebDAV and local storage
#. 2) Someone else updates your shared namecard on WebDAV:
.. code-block:: javascript
jio_instance.put('myNameCard', {
email: 'kyle@td.com',
}).then(function (response) {
// response -> 'myNameCard'
});
Your smartphone is offline, so now you will have one version on
your smartphone and another version on WebDAV on your PC.
#. 3) Later, your smartphone is online and you modify your email:
.. code-block:: javascript
jio_instance.get('myNameCard').then(function (response) {
// response.email -> 'jb@td.com'
// the get() method checks only on your local storage
// and doesn't warn you about remote modifications.
return jio_instance.put('myNameCard', {
email: 'jack@td.com'
})
.then(function (response) {
// response -> 'myNameCard'
});
| Your latest modification of the email is: "jack@td.com"
| The modification from the other user is: "kyle@td.com"
If your conflict_handling option was:
* | 0: the email is:
| -"kyle@td.com" on WebDAV
| -"jack@td.com" on your local storage
| The storage rejects your latest modification,
| you get an error because local and remote documents are desynchronized.
| The documents in local and remote state are left untouched.
* | 1: the email is: "jack@td.com" on both storages
| The storage pushes the local modification, which is yours.
* | 2: the email is: "kyle@td.com" on both storages
| The storage keeps the remote modification, which is from the other user.
| Your local storage is modified to fit the state of the remote storage.
* | 3: the email is: "jack@td.com" on both storages
| The storage doesn't do synchronization, and pushes your modification
| without checking if the remote storage has been changed or not
docs/style_guide.rst deleted 100644 → 0
View file @ 5b8c0c8e
This diff is collapsed.
package.json
View file @ bf8de85a
{
"name": "jio",
"version": "v3.15.0",
"version": "v3.16.0",
"license": "LGPLv3",
"author": "Nexedi SA",
"contributors": [
......
src/jio.storage/erp5storage.js
View file @ bf8de85a
......@@ -445,6 +445,7 @@
"type": "POST",
"url": name,
"data": data,
"dataType": "blob",
"xhrFields": {
withCredentials: storage._thisCredentials
},
......
src/jio.storage/replicatestorage.js
View file @ bf8de85a
......@@ -218,10 +218,12 @@
queue
.push(function () {
if (argument_list.length > 0) {
var argument_array = argument_list.shift();
argument_array[0] = queue;
var argument_array = argument_list.shift(),
sub_queue = new RSVP.Queue();
argument_array[0] = sub_queue;
function_used.apply(context, argument_array);
pushAndExecute(queue);
return sub_queue;
}
});
}
......@@ -354,8 +356,8 @@
});
}
function checkAttachmentSignatureDifference(skip_attachment_dict,
queue, source,
function checkAttachmentSignatureDifference(queue, skip_attachment_dict,
source,
destination, id, name,
conflict_force,
conflict_revert,
......@@ -405,8 +407,8 @@
});
}
function checkAttachmentLocalDeletion(skip_attachment_dict,
queue, destination, id, name, source,
function checkAttachmentLocalDeletion(queue, skip_attachment_dict,
destination, id, name, source,
conflict_force, conflict_revert,
conflict_ignore) {
var status_hash;
......@@ -427,7 +429,9 @@
function pushDocumentAttachment(skip_attachment_dict, id, source,
destination, options) {
var queue = new RSVP.Queue();
var queue = new RSVP.Queue(),
local_dict = {},
signature_dict = {};
return queue
.push(function () {
......@@ -451,11 +455,10 @@
]);
})
.push(function (result_list) {
var local_dict = {},
signature_dict = {},
is_modification,
var is_modification,
is_creation,
key;
key,
argument_list = [];
for (key in result_list[0]) {
if (result_list[0].hasOwnProperty(key)) {
if (!skip_attachment_dict.hasOwnProperty(key)) {
......@@ -478,30 +481,45 @@
is_creation = !signature_dict.hasOwnProperty(key)
&& options.check_creation;
if (is_modification === true || is_creation === true) {
checkAttachmentSignatureDifference(skip_attachment_dict,
queue, source,
destination, id, key,
options.conflict_force,
options.conflict_revert,
options.conflict_ignore,
is_creation,
is_modification);
argument_list.push([undefined,
skip_attachment_dict,
source,
destination, id, key,
options.conflict_force,
options.conflict_revert,
options.conflict_ignore,
is_creation,
is_modification]);
}
}
}
return dispatchQueue(
checkAttachmentSignatureDifference,
argument_list,
context._parallel_operation_attachment_amount
);
})
.push(function () {
var key, argument_list = [];
if (options.check_deletion === true) {
for (key in signature_dict) {
if (signature_dict.hasOwnProperty(key)) {
if (!local_dict.hasOwnProperty(key)) {
checkAttachmentLocalDeletion(skip_attachment_dict,
queue, destination, id, key,
argument_list.push([undefined,
skip_attachment_dict,
destination, id, key,
source,
options.conflict_force,
options.conflict_revert,
options.conflict_ignore);
options.conflict_ignore]);
}
}
}
return dispatchQueue(
checkAttachmentLocalDeletion,
argument_list,
context._parallel_operation_attachment_amount
);
}
});
}
......@@ -1074,7 +1092,7 @@
return dispatchQueue(
repairDocument,
argument_list,
context._parallel_operation_attachment_amount
context._parallel_operation_amount
);
});
}
......
test/jio.storage/erp5storage.tests.js
View file @ bf8de85a
......@@ -490,7 +490,7 @@
}, ""]);
stop();
expect(11);
expect(12);
this.jio.putAttachment(
id,
......@@ -502,6 +502,7 @@
equal(server.requests[0].method, "POST");
equal(server.requests[0].url, submit_url);
equal(server.requests[0].status, 204);
equal(server.requests[0].responseType, "blob");
ok(server.requests[0].requestBody instanceof FormData);
ok(context.spy.calledTwice, "FormData.append count " +
......@@ -536,7 +537,7 @@
}, ""]);
stop();
expect(11);
expect(12);
this.jio.putAttachment(
id,
......@@ -548,6 +549,7 @@
equal(server.requests[0].method, "POST");
equal(server.requests[0].url, submit_url);
equal(server.requests[0].status, 204);
equal(server.requests[0].responseType, "blob");
ok(server.requests[0].requestBody instanceof FormData);
ok(context.spy.calledTwice, "FormData.append count " +
......@@ -586,7 +588,7 @@
}, ""]);
stop();
expect(12);
expect(13);
this.jio.putAttachment(
id,
......@@ -598,6 +600,7 @@
equal(server.requests[0].method, "POST");
equal(server.requests[0].url, submit_url);
equal(server.requests[0].status, 204);
equal(server.requests[0].responseType, "blob");
ok(server.requests[0].requestBody instanceof FormData);
ok(context.spy.calledOnce, "FormData.append count " +
......
test/jio.storage/replicatestorage.tests.js
View file @ bf8de85a
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7