Skip to content

J
jio
Commit 374c1b07 authored by Junming Liu's avatar Junming Liu
Browse files
Options

QiniuStorage : The latest version of QiniuStorage 

This is the lastest QiniuStorage file. Because of Qiniu not support resource manage operations, so it's only have putAttachment and getAttachment.
The method to upload or download file are using URiTemplate via Ajax. Besides, i use CryptoJS to do HMAC-SHA1 algorithm.
parent cfea757e
Hide whitespace changes
Inline Side-by-side
Showing with 171 additions and 313 deletions
examples/scenario.js
View file @ 374c1b07
......@@ -37,18 +37,18 @@
///////////////////////////
// Memory storage
///////////////////////////
return g.run({
type: "query",
sub_storage: {
type: "uuid",
sub_storage: {
type: "union",
storage_list: [{
type: "memory"
}]
}
}
});
// return g.run({
// type: "query",
// sub_storage: {
// type: "uuid",
// sub_storage: {
// type: "union",
// storage_list: [{
// type: "memory"
// }]
// }
// }
// });
///////////////////////////
// IndexedDB storage
......@@ -85,18 +85,18 @@
///////////////////////////
// Qiniu storage
///////////////////////////
// return g.run({
// type: "query",
// sub_storage: {
// type: "uuid",
// sub_storage: {
// "type": "qiniu",
// "bucket": "BUCKET",
// "access_key": "ACCESSKEY",
// "secret_key": "SECRETKEY"
// }
// }
// });
return g.run({
type: "query",
sub_storage: {
type: "uuid",
sub_storage: {
"type": "qiniu",
"bucket": "7xn150.com1.z0.glb.clouddn.com",
"access_key": "s90kGV3JYDDQPivaPVpwxrHMi9RCpncLgLctGDJQ",
"secret_key": "hfqndzXIfqP6aMpTOdgT_UjiUjARkiFXz98Cthjx"
}
}
});
///////////////////////////
// Replicate storage
......@@ -159,7 +159,10 @@
deepEqual(doc, {"title": "I don't have ID éà&\n"},
"Document correctly fetched");
// Remove the doc
return jio.remove(doc_id);
return jio.remove(doc_id)
.fail(function (error) {
console.log("remove error", error);
});
})
.then(function (doc_id) {
ok(doc_id, "Document removed");
......@@ -249,7 +252,6 @@
.then(function (result) {
equal(result.target.result, "fooé\nbar", "Attachment correctly fetched");
return jio.get("test.txt");
})
.then(function (doc) {
deepEqual(doc, {}, "Document correctly fetched");
......
src/jio.storage/qiniustorage.js
View file @ 374c1b07
/*
* Copyright 2013, Nexedi SA
* Copyright 2015, Nexedi SA
* Released under the LGPL license.
* http://www.gnu.org/licenses/lgpl.html
*/
......@@ -7,22 +7,20 @@
/**
* JIO Qiniu Storage. Type = "qiniu".
* Qiniu "database" storage.
* Qiniu storage not support resource manage operation,
* so remove,removeAttachment,allDocs,allAttachment isn't working
*/
/*global FormData, btoa, Blob, CryptoJS, define, jIO */
/*jslint indent: 2, maxlen: 80, nomen: true, unparam: true, bitwise: true */
(function (dependencies, module) {
"use strict";
if (typeof define === 'function' && define.amd) {
return define(dependencies, module);
}
// if (typeof exports === 'object') {
// return module(exports, require('jio'));
// }
module(jIO);
}([
'jio'
], function (jIO) {
/*global JSON, FormData, btoa, Blob, CryptoJS, define,
jIO, RSVP, console, UriTemplate */
/*jslint indent: 2, maxlen: 80, nomen: true, bitwise: true */
(function (jIO, RSVP, Blob, UriTemplate) {
"use strict";
var METADATA_URL = "http://{+bucket}/{+key}?e=" +
"{+DEADLINE}&token={+access_key}:{+token}",
metadata_template = UriTemplate.parse(METADATA_URL),
UPLOAD_URL = "http://up.qiniu.com/",
DEADLINE = 2451491200;
function urlsafe_base64_encode(string) {
return string
......@@ -55,8 +53,6 @@
return urlsafe_base64_encode(btoa(encodedString));
}
var UPLOAD_URL = "http://up.qiniu.com/",
DEADLINE = 2451491200;
/**
* The JIO QiniuStorage extension
......@@ -93,15 +89,16 @@
data = new FormData();
if (update === true) {
put_policy = JSON.stringify({
"scope": this._bucket + ':' + key,
"scope": "bucket" + ':' + key,
"deadline": DEADLINE
});
} else {
put_policy = JSON.stringify({
"scope": this._bucket,
"scope": "bucket",
"deadline": DEADLINE
});
}
encoded = btoa(put_policy);
encode_signed = b64_hmac_sha1(this._secret_key, encoded);
upload_token = this._access_key + ":" + encode_signed + ":" + encoded;
......@@ -110,10 +107,7 @@
data.append("token", upload_token);
data.append(
"file",
// new Blob([JSON.stringify(doc)], {type: "application/json"}),
// new Blob([doc], {type: "application/json"}),
blob,
// new Blob([], {type: "application/octet-stream"}),
key
);
......@@ -126,247 +120,101 @@
};
/**
* Create a document.
*
* @method post
* @param {Object} command The JIO command
* @param {Object} metadata The metadata to store
*/
QiniuStorage.prototype.post = function (command, metadata) {
var doc = jIO.util.deepClone(metadata),
doc_id = metadata._id;
if (!doc_id) {
doc_id = jIO.util.generateUuid();
doc._id = doc_id;
}
return this._put(
doc_id,
new Blob([JSON.stringify(doc)], {type: "application/json"})
).then(function (doc) {
if (doc !== null) {
command.success({"id": doc_id});
} else {
command.error(
"not_found",
"missing",
"Cannot find document"
);
}
}).fail(function (event) {
command.error(
event.target.status,
event.target.statusText,
"Unable to post doc"
);
});
};
/**
* Update/create a document.
* Add an attachment to a document
*
* @method put
* @param {Object} command The JIO command
* @param {Object} metadata The metadata to store
* @method putAttachment
* @id {Object} Document id
* @param {Object} param The given parameters
* @blob {Object} attachment packaged into a blob
*/
QiniuStorage.prototype.put = function (command, metadata) {
return this._put(
metadata._id,
new Blob([JSON.stringify(metadata)], {type: "application/json"}),
true
).then(function (doc) {
if (doc !== null) {
command.success({"data": doc});
} else {
command.error(
"not_found",
"missing",
"Cannot find document"
QiniuStorage.prototype.putAttachment = function (id, param, blob) {
var gadget = this;
return new RSVP.Queue()
.push(function () {
return gadget._put(
id + "/" + param,
blob,
true
);
}
}).fail(function (event) {
command.error(
event.target.status,
event.target.statusText,
"Unable to put doc"
);
});
});
};
QiniuStorage.prototype._get = function (key) {
var download_url = 'http://' + this._bucket + '.u.qiniudn.com/' + key
// var download_url = 'http://' + this._bucket + '.dn.qbox.me/' + key
+ '?e=' + DEADLINE,
token = b64_hmac_sha1(this._secret_key, download_url);
return jIO.util.ajax({
"type": "GET",
"url": download_url + "&token=" + this._access_key + ':' + token
// "dataType": "blob"
});
};
/**
* Get a document or attachment
* @method get
* @param {object} command The JIO command
**/
QiniuStorage.prototype.get = function (command, param) {
return this._get(param._id)
.then(function (doc) {
if (doc.target.responseText !== undefined) {
command.success({"data": JSON.parse(doc.target.responseText)});
} else {
command.error(
"not_found",
"missing",
"Cannot find document"
);
}
}).fail(function (event) {
command.error(
event.target.status,
event.target.statusText,
"Unable to get doc"
);
var download_url = 'http://' + this._bucket + '/' + key
+ '?e=' + DEADLINE,
token = b64_hmac_sha1(this._secret_key, download_url),
gadget = this;
return new RSVP.Queue()
.push(function () {
return jIO.util.ajax({
type: "GET",
url: metadata_template.expand({
bucket: gadget._bucket,
key: key,
DEADLINE: DEADLINE,
access_key: gadget._access_key,
token: token
})
});
});
};
/**
* Get an attachment
* Get an attaURITemplatechment
*
* @method getAttachment
* @param {Object} command The JIO command
* @param {Object} param The given parameters
* @param {Object} options The command options
* @param {Object} attachment attachment name
*/
QiniuStorage.prototype.getAttachment = function (command, param) {
return this._get(param._id + "/" + param._attachment)
.then(function (doc) {
if (doc.target.response) {
command.success({"data": doc.target.response});
} else {
// XXX Handle error
command.error(
"not_found",
"missing",
"Cannot find document"
);
QiniuStorage.prototype.getAttachment = function (param, attachment) {
var gadget = this;
return new RSVP.Queue()
.push(function () {
return gadget._get(param + "/" + attachment);
})
.push(function (doc) {
if (doc.target.response !== undefined) {
return new Blob([doc.target.response]);
}
}).fail(function (event) {
command.error(
event.target.status,
event.target.statusText,
"Unable to get attachment"
);
if (doc.target !== undefined) {
doc.id = param;
doc.target.attachment = attachment;
return new Blob([JSON.stringify(doc)], {type: "application/json"});
}
}).push(undefined, function (error) {
if ((error.target !== undefined) &&
(error.target.status === 404)) {
throw new jIO.util.jIOError("Cannot find attachment: "
+ param._id + " , " + param._attachment,
404);
}
throw error;
});
};
/**
* Add an attachment to a document
*
* @method putAttachment
* @param {Object} command The JIO command
* @id {Object} Document id
* @param {Object} param The given parameters
* @param {Object} options The command options
* @blob {Object} attachment packaged into a blob
*/
QiniuStorage.prototype.putAttachment = function (command, param) {
return this._put(
param._id + "/" + param._attachment,
param._blob,
true
).then(function (doc) {
if (doc !== null) {
command.success({"data": doc});
} else {
command.error(
"not_found",
"missing",
"Cannot find document"
QiniuStorage.prototype.putAttachment = function (id, param, blob) {
var gadget = this;
return new RSVP.Queue()
.push(function () {
return gadget._put(
id + "/" + param,
blob,
true
);
}
}).fail(function (event) {
command.error(
event.target.status,
event.target.statusText,
"Unable to put attachment"
);
});
};
/**
* Remove a document
*
* @method remove
* @param {Object} command The JIO command
* @param {Object} param The given parameters
*/
QiniuStorage.prototype.remove = function (command, param) {
var DELETE_HOST = "http://rs.qiniu.com",
DELETE_PREFIX = "/delete/",
encoded_entry_uri = urlsafe_base64_encode(btoa(
this._bucket + ':' + param._id
)),
delete_url = DELETE_HOST + DELETE_PREFIX + encoded_entry_uri,
data = DELETE_PREFIX + encoded_entry_uri + '\n',
token = b64_hmac_sha1(this._secret_key, data);
jIO.util.ajax({
"type": "POST",
"url": delete_url,
"headers": {
Authorization: "QBox " + this._access_key + ':' + token,
"Content-Type": 'application/x-www-form-urlencoded'
}
}).then(
command.success
).fail(function (error) {
command.error(
"not_found",
"missing",
"Unable to delete doc"
);
});
};
QiniuStorage.prototype.allDocs = function (command, param, options) {
var LIST_HOST = "http://rsf.qiniu.com",
LIST_PREFIX = "/list?bucket=" + this._bucket,
list_url = LIST_HOST + LIST_PREFIX,
token = b64_hmac_sha1(this._secret_key, LIST_PREFIX + '\n');
jIO.util.ajax({
"type": "POST",
"url": list_url,
"headers": {
Authorization: "QBox " + this._access_key + ':' + token,
"Content-Type": 'application/x-www-form-urlencoded'
}
}).then(function (response) {
var data = JSON.parse(response.target.responseText),
count = data.items.length,
result = [],
item,
i;
for (i = 0; i < count; i += 1) {
item = data.items[i];
result.push({
id: item.key,
key: item.key,
doc: {},
value: {}
});
}
command.success({"data": {"rows": result, "total_rows": count}});
}).fail(function (error) {
command.error(
"error",
"did not work as expected",
"Unable to call allDocs"
);
});
});
};
jIO.addStorage('qiniu', QiniuStorage);
}));
}(jIO, RSVP, Blob, UriTemplate));
test/jio.storage/qiniustorage.tests.js
View file @ 374c1b07
/*global define, module, test_util, RSVP, jIO, test, ok,
deepEqual, sinon, expect, stop, start, Blob, equal, define */
/*global define, module, RSVP, jIO, test, ok,
deepEqual, sinon, expect, stop, start, Blob, equal, define, console */
/*jslint indent: 2 */
(function (dependencies, module) {
"use strict";
if (typeof define === 'function' && define.amd) {
return define(dependencies, module);
}
module(test_util, RSVP, jIO);
module(RSVP, jIO);
}([
'test_util',
'rsvp',
'jio',
'hmacsha1',
......@@ -21,12 +19,12 @@
var qiniu_spec = {
"type": "qiniu",
"bucket": "uth6nied",
"access_key": "Imh9CFmpVZ5L1TE04Pjt-UmR_Ccr2cW9-KjSmvSA",
"secret_key": "vFkNUlI2U4B7G1sz8UL_Z25kYHozfz82z4vMWPgo"
"bucket": "7xn150.com1.z0.glb.clouddn.com",
"access_key": "s90kGV3JYDDQPivaPVpwxrHMi9RCpncLgLctGDJQ",
"secret_key": "hfqndzXIfqP6aMpTOdgT_UjiUjARkiFXz98Cthjx"
};
module("QiniuStorage", {
module("QiniuStorage ", {
setup: function () {
this.server = sinon.fakeServer.create();
......@@ -45,9 +43,12 @@
test('get', function () {
var key = "foobar12345",
server = this.server,
download_url = 'http://uth6nied.u.qiniudn.com/foobar12345?' +
/* download_url = 'http://uth6nied.u.qiniudn.com/foobar12345?' +
'e=2451491200&token=Imh9CFmpVZ5L1TE04Pjt-UmR_Ccr2cW9-KjSmvSA:' +
'hISFzrC4dQvdOR8A_MozNsB5cME=',
'hISFzrC4dQvdOR8A_MozNsB5cME=', */
download_url = 'http://7xn150.com1.z0.glb.clouddn.com/foobar12345?' +
'e=2451491200&token=s90kGV3JYDDQPivaPVpwxrHMi9RCpncLgLctGDJQ:' +
'lWIEfRXIf6tbwWjuX381DHTSxnU=',
data = {
"_id": key,
"foo": "bar"
......@@ -58,15 +59,12 @@
}, JSON.stringify(data)]);
stop();
this.jio_storage.get({"_id": key})
this.jio_storage.get(key)
.then(function (result) {
console.log("result", result);
deepEqual(result, {
"data": data,
"id": key,
"method": "get",
"result": "success",
"status": 200,
"statusText": "Ok"
"_id": key,
"foo": "bar"
});
})
.fail(function (error) {
......@@ -79,11 +77,14 @@
test('getAttachment', function () {
var key = "foobar12345",
object,
resultdata,
attachment = "barfoo54321",
server = this.server,
download_url = 'http://uth6nied.u.qiniudn.com/foobar12345/barfoo54321' +
'?e=2451491200&token=Imh9CFmpVZ5L1TE04Pjt-UmR_Ccr2cW9-KjSmvSA:' +
'L88mHkZkfjr11DqPUqb5gsDjHFY=',
download_url = 'http://7xn150.com1.z0.glb.clouddn.com/foobar12345' +
'/barfoo54321?e=2451491200&token=s90kGV3JYDDQPivaPVpwxrHMi9RCpncL' +
'gLctGDJQ:l_yS8PFderOhMyqHN0FN41tdJOM=',
data = {
"_id": key,
"foo": "bar",
......@@ -95,24 +96,33 @@
}, JSON.stringify(data)]);
stop();
this.jio_storage.getAttachment({"_id": key, "_attachment": attachment})
this.jio_storage.getAttachment(key, attachment)
.then(function (result) {
return jIO.util.readBlobAsText(result.data).then(function (e) {
return jIO.util.readBlobAsText(result).then(function (e) {
object = JSON.parse(e.target.result);
return {
"result": result,
"text": e.target.result
"result": object,
"text": object.target.responseText
};
});
}).then(function (result) {
console.log("result", result);
resultdata = {
"data": result.result.target.responseText,
"id": result.result.id,
"attachment": result.result.target.attachment,
"method": result.result.target.method,
"status": result.result.target.status,
"statusText": result.result.target.statusText
};
result.result.data = result.text;
deepEqual(result.result, {
deepEqual(resultdata, {
"data": JSON.stringify(data),
"id": key,
"attachment": attachment,
"method": "getAttachment",
"result": "success",
"method": "GET",
"status": 200,
"statusText": "Ok"
"statusText": "OK"
});
})
.fail(function (error) {
......@@ -122,7 +132,7 @@
start();
});
});
/*
test('post', function () {
var key = "foobar12345",
server = this.server,
......@@ -155,10 +165,10 @@
start();
});
});
*/
test('put', function () {
var key = "foobar12345",
server = this.server,
test('QiniuStorage.put', function () {
var server = this.server,
upload_url = 'http://up.qiniu.com/',
data = {"ok": "excellent"};
......@@ -167,9 +177,9 @@
}, JSON.stringify(data)]);
stop();
this.jio_storage.put({"_id": key})
.then(function () {
throw new Error("Not implemented");
this.jio_storage.put("bar", {"title": "foo"})
.then(function (result) {
equal(result, "bar");
})
.fail(function (error) {
ok(false, error);
......@@ -179,25 +189,23 @@
});
});
test('putAttachment', function () {
var key = "foobar12345",
attachment = "barfoo54321",
server = this.server,
upload_url = 'http://up.qiniu.com/',
data = {"ok": "excellent"};
test('QiniuStorage putAttachment', function () {
var server = this.server,
blob = new Blob(["foo"]),
upload_url = 'http://up.qiniu.com/';
server.respondWith("POST", upload_url, [200, {
"Content-Type": "application/json"
}, JSON.stringify(data)]);
}, "foo"]);
stop();
this.jio_storage.putAttachment({
"_id": key,
"_attachment": attachment,
"_blob": "bar"
})
this.jio_storage.putAttachment("bar", "foo", blob)
.then(function () {
throw new Error("Not implemented");
equal(server.requests.length, 1);
equal(server.requests[0].method, "POST");
equal(server.requests[0].url, upload_url);
equal(server.requests[0].status, "200");
equal(server.requests[0].responseText, "foo");
})
.fail(function (error) {
ok(false, error);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7