Skip to content

M
mynij
Commit 175c9113 authored by Alain Takoudjou's avatar Alain Takoudjou
Browse files
Options

erp5_web_mynij_search: Rework index torrent seed and download 

Torrent are compressed before seed
torrent blob is saved as attachment and can be reloaded quickly without rebuild everything
There are more details on download and seed progress
parent 8fc8f5c6
Hide whitespace changes
Inline Side-by-side
Showing with 273 additions and 132 deletions
bt5/erp5_web_mynij_search/PathTemplateItem/web_page_module/gadget_erp5_mynij_webtorrent_html.html
View file @ 175c9113
...@@ -16,6 +16,7 @@ ...@@ -16,6 +16,7 @@
<script src="jiodev.js" type="text/javascript"></script> <script src="jiodev.js" type="text/javascript"></script>
<script src="handlebars.js" type="text/javascript"></script> <script src="handlebars.js" type="text/javascript"></script>
<script src="webtorrent.min.js" type="text/javascript"></script> <script src="webtorrent.min.js" type="text/javascript"></script>
<script src="jszip.min.js" type="text/javascript"></script>
<script src="gadget_erp5_mynij_webtorrent.js" type="text/javascript"></script> <script src="gadget_erp5_mynij_webtorrent.js" type="text/javascript"></script>
<script id="torrent-info-template" type="text/x-handlebars-template"> <script id="torrent-info-template" type="text/x-handlebars-template">
...@@ -41,12 +42,17 @@ ...@@ -41,12 +42,17 @@
{{/if}} {{/if}}
</div> </div>
<div> <div>
<code class="downloaded"></code> {{#if is_seed}}
of <code class="total"></code> &#x2197;<code class="uploadSpeed">0 b/s</code>
— <span class="remaining"></span><br/> / <code class="total">{{total}}</code>
{{else}}
&#x2198;<code class="downloadSpeed">0 b/s</code> &#x2198;<code class="downloadSpeed">0 b/s</code>
/ &#x2197;<code class="uploadSpeed">0 b/s</code> / <code class="downloaded">0 B</code>
of <code class="total">{{total}}</code>
— <span class="remaining"></span><br/>
{{/if}}
</div> </div>
<div class="message"></div>
</div> </div>
</div> </div>
</div> </div>
......
bt5/erp5_web_mynij_search/PathTemplateItem/web_page_module/gadget_erp5_mynij_webtorrent_html.xml
View file @ 175c9113
...@@ -246,7 +246,7 @@ ...@@ -246,7 +246,7 @@
</item> </item>
<item> <item>
<key> <string>serial</string> </key> <key> <string>serial</string> </key>
<value> <string>988.48319.53521.12885</string> </value> <value> <string>989.27292.54989.38297</string> </value>
</item> </item>
<item> <item>
<key> <string>state</string> </key> <key> <string>state</string> </key>
...@@ -264,7 +264,7 @@ ...@@ -264,7 +264,7 @@
</tuple> </tuple>
<state> <state>
<tuple> <tuple>
<float>1608296295.72</float> <float>1610966584.34</float>
<string>UTC</string> <string>UTC</string>
</tuple> </tuple>
</state> </state>
......
bt5/erp5_web_mynij_search/PathTemplateItem/web_page_module/gadget_erp5_mynij_webtorrent_js.js
View file @ 175c9113
/*jslint nomen: true, indent: 2, maxerr: 3, maxlen: 80*/ /*jslint nomen: true, indent: 2, maxerr: 3, maxlen: 80*/
/*global window, RSVP, rJS, WebTorrent*/ /*global window, RSVP, rJS, WebTorrent, JSZip*/
(function (window, RSVP, rJS, document, crypto, WebTorrent) { (function (window, RSVP, rJS, document, crypto, WebTorrent, JSZip) {
"use strict"; "use strict";
var gadget_klass = rJS(window), var gadget_klass = rJS(window),
...@@ -45,8 +45,12 @@ ...@@ -45,8 +45,12 @@
// Progress // Progress
var percent = Math.round(torrent.progress * 100 * 100) / 100; var percent = Math.round(torrent.progress * 100 * 100) / 100;
progressBar.style.width = percent + '%'; progressBar.style.width = percent + '%';
downloaded.innerHTML = prettyBytes(torrent.downloaded);
total.innerHTML = prettyBytes(torrent.length); total.innerHTML = prettyBytes(torrent.length);
if (!downloaded) {
uploadSpeed.innerHTML = prettyBytes(torrent.uploadSpeed) + '/s';
return;
}
downloaded.innerHTML = prettyBytes(torrent.downloaded);
// Remaining time // Remaining time
if (torrent.done) { if (torrent.done) {
...@@ -61,7 +65,13 @@ ...@@ -61,7 +65,13 @@
// Speed rates // Speed rates
downloadSpeed.innerHTML = prettyBytes(torrent.downloadSpeed) + '/s'; downloadSpeed.innerHTML = prettyBytes(torrent.downloadSpeed) + '/s';
uploadSpeed.innerHTML = prettyBytes(torrent.uploadSpeed) + '/s'; }
function torrentHtmlElement(infohash, qclass) {
var element = document.getElementById(infohash);
if (qclass && element)
return element.querySelector('.' + qclass);
return element;
} }
gadget_klass gadget_klass
...@@ -110,10 +120,34 @@ ...@@ -110,10 +120,34 @@
}) })
.declareMethod("index_to_file", function (index_name) { .declareMethod("index_to_file", function (index_name) {
var gadget = this,
index_id = "index-" + index_name;
return gadget.state.model_gadget.getAttachment(
index_id,
"zipped",
{"format": "blob"})
.push(function (result) {
return new File([result], index_id + ".zip", {type: 'blob'});;
})
.push(undefined, function () {
console.log("Building index blob");
return gadget.notifySubmitted({
message: "Building '" + index_name + "' torrent files...",
status: "success"
})
.push(function () {
return gadget.buildIndexBlob(index_name, index_id);
});
});
})
.declareMethod("buildIndexBlob", function (index_name, index_id) {
var gadget = this, var gadget = this,
ids, ids,
index_id = "index-" + index_name, file_list = [],
file_list = []; zipped_file,
zip = new JSZip(),
name = index_id + ".zip";
return gadget.state.model_gadget.getAttachment( return gadget.state.model_gadget.getAttachment(
index_id, index_id,
...@@ -121,22 +155,14 @@ ...@@ -121,22 +155,14 @@
{"format": "json"}) {"format": "json"})
.push(function (result) { .push(function (result) {
ids = result; ids = result;
file_list.push(new File( zip.file(index_id + '.ids', JSON.stringify(result));
[new Blob([JSON.stringify(result)], {type : "application/json"})],
index_id + '.ids',
{type: 'application/json'})
);
return gadget.state.model_gadget.getAttachment( return gadget.state.model_gadget.getAttachment(
index_id, index_id,
"map", "map",
{"format": "text"}); {"format": "text"});
}) })
.push(function (result) { .push(function (result) {
file_list.push(new File( zip.file(index_id + '.map', result);
[new Blob([result], {type : "application/json"})],
index_id + '.map',
{type: 'application/json'})
);
return gadget.state.model_gadget.getAttachment( return gadget.state.model_gadget.getAttachment(
index_id, index_id,
"ctx", "ctx",
...@@ -153,19 +179,11 @@ ...@@ -153,19 +179,11 @@
return gadget.state.model_gadget.get_page(page_id); return gadget.state.model_gadget.get_page(page_id);
}) })
.push(function (page) { .push(function (page) {
file_list.push(new File( zip.file(page_id + '.json', JSON.stringify(page));
[new Blob([JSON.stringify(page)], {type : "application/json"})],
page_id + '.page',
{type: 'application/json'})
);
}); });
} }
file_list.push(new File( zip.file(index_id + '.ctx', result);
[new Blob([result], {type : "application/json"})],
index_id + '.ctx',
{type: 'application/json'})
);
// get all page of this index // get all page of this index
for (i = 0; i < ids.length; i += 1) { for (i = 0; i < ids.length; i += 1) {
get_page(ids[i].slice(1)); get_page(ids[i].slice(1));
...@@ -173,7 +191,25 @@ ...@@ -173,7 +191,25 @@
return page_queue return page_queue
}) })
.push(function () { .push(function () {
return file_list; return new RSVP.Queue()
.push(function () {
return zip.generateAsync({
type : "blob",
compression: "DEFLATE",
compressionOptions: {level: 9}
})
.then(function (zipped) {
zipped_file = new File([zipped], name, {type: 'blob'});
return gadget.state.model_gadget.putAttachment(
index_id,
"zipped",
zipped
);
});
})
.push(function () {
return zipped_file;
});
}); });
}) })
...@@ -183,7 +219,8 @@ ...@@ -183,7 +219,8 @@
blob_list = [], blob_list = [],
name_list = [], name_list = [],
failed = false, failed = false,
file_list = torrent.files; file_list = torrent.files,
msg_html;
//quick check of file consitency here before insert //quick check of file consitency here before insert
extract_blob = function (err, blob) { extract_blob = function (err, blob) {
...@@ -201,7 +238,8 @@ ...@@ -201,7 +238,8 @@
file = file_list.shift(); file = file_list.shift();
name_list.push(file.name); name_list.push(file.name);
// extract next blob file // extract next blob file
console.log("Extracting ... " + file.name); if (msg_html)
msg_html.innerText = "Extracting file " + file.name + " ...";
return file.getBlob(extract_blob); return file.getBlob(extract_blob);
} }
else { else {
...@@ -210,111 +248,129 @@ ...@@ -210,111 +248,129 @@
console.log("Torrent extraction failed!!!"); console.log("Torrent extraction failed!!!");
return; return;
} }
return gadget.saveExtractedIndex(torrent, name_list, blob_list) return gadget.saveExtractedIndex(torrent,
name_list,
blob_list)
} }
} }
// extract all blob from files // extract all blob from files
msg_html = torrentHtmlElement(torrent.infoHash, 'message')
return extract_blob(); return extract_blob();
}) })
.declareMethod("saveExtractedIndex", function (torrent, .declareMethod("saveExtractedIndex", function (torrent,
name_list, ext_list,
blob_list) { blob_list) {
var gadget = this, var gadget = this,
zip = new JSZip(),
insert_data,
zipped_blob,
doc_blob,
index_name,
doc,
ext,
i, i,
insert_queue = new RSVP.Queue(),
insert_blob, insert_blob,
insert_page, msg_html;
insert_doc,
current,
type,
index_name,
id;
insert_doc = function (blob) {
insert_queue
.push(function () {
return jIO.util.readBlobAsText(blob);
})
.push(function (evt) {
var doc = JSON.parse(evt.target.result);
return gadget.state.model_gadget.createIndexDoc({
title: doc.title,
links: doc.links,
magnet_uri: torrent.magnetURI,
status: "built"
});
})
}
insert_blob = function (index_id, type, blob) { insert_blob = function (index_id, type, data) {
insert_queue return gadget.state.model_gadget.putAttachment(
.push(function () { index_id,
return gadget.state.model_gadget.putAttachment( type,
index_id, new Blob([data], {type : "application/json"}));
type, };
blob); insert_data = function (name, data) {
}) var type,
} id;
if (name.endsWith('json')) {
insert_page = function (page_id, blob) { type = 'json';
insert_queue id = name.slice(0,-5);
.push(function () {
return jIO.util.readBlobAsText(blob);
})
.push(function (evt) {
return gadget.state.model_gadget.put(
page_id, JSON.parse(evt.target.result));
})
}
for (i = 0; i < blob_list.length; i += 1) {
current = name_list[i];
if (current.endsWith('page')) {
type = 'page';
id = current.slice(0,-5);
} else { } else {
type = current.slice(-3); type = name.slice(-3);
id = current.slice(0,-4); id = name.slice(0,-4);
} }
if (type === 'page') { if (type === 'json') {
insert_page(id, blob_list[i]); return gadget.state.model_gadget.put(id, JSON.parse(data));
} else if (type === 'ctx') { } else if (type === 'ctx') {
insert_blob(id, type, blob_list[i]); insert_blob(id, type, data);
} else if (type === 'map') { } else if (type === 'map') {
insert_blob(id, type, blob_list[i]); insert_blob(id, type, data);
} else if (type === 'ids') { } else if (type === 'ids') {
index_name = id.split("index-")[1] index_name = id.split("index-")[1]
insert_blob(id, type, blob_list[i]); insert_blob(id, type, data);
} else if (type === 'doc') {
insert_doc(blob_list[i]);
} }
} }
return insert_queue
.push(function () { msg_html = torrentHtmlElement(torrent.infoHash, 'message');
return gadget.state.model_gadget.pushIndex(index_name); // There is 2 files in the torrent, index doc and zipped data
}) for (i = 0; i < ext_list.length; i += 1) {
.push(function () { ext = ext_list[i].slice(-3);
gadget.notifySubmitted({ if (ext === "doc") {
message: "Index '" + index_name + "' Saved.", doc_blob = blob_list[i];
status: "success" }
}) else if (ext === "zip")
}) zipped_blob = blob_list[i];
.push(function () { }
// stop seeding the torrent return new RSVP.Queue()
gadget.state.client.destroy(function (err) { .push(function () {
if (err) { return jIO.util.readBlobAsText(doc_blob);
console.error(err.stack || err.message || err); })
.push(function (evt) {
doc = JSON.parse(evt.target.result);
if (msg_html)
msg_html.innerText = "Uncompressing index data...";
// extract zipped data here
return zip.loadAsync(zipped_blob)
.then(function (data) {
var promise_list = [],
get_zip;
get_zip = function (name) {
return zip.file(name).async('string')
.then(function(data) {
if (msg_html)
msg_html.innerText = "Extracting file " + name + "...";
return insert_data(name, data);
});
} }
Object.keys(data.files).forEach(function (filename) {
promise_list.push(get_zip(filename));
});
return RSVP.all(promise_list);
}); });
if (gadget.state.redirect) { })
return gadget.redirect({command: 'display', options: { .push(function () {
page: gadget.state.redirect if (msg_html)
}}); msg_html.innerText = "Creating index document...";
return gadget.state.model_gadget.createIndexDoc({
title: doc.title,
links: doc.links,
magnet_uri: torrent.magnetURI,
status: "built"
});
})
.push(function () {
return gadget.state.model_gadget.pushIndex(index_name);
})
.push(function () {
gadget.notifySubmitted({
message: "Index '" + index_name + "' Saved.",
status: "success"
})
})
.push(function () {
// stop seeding the torrent
gadget.state.client.destroy(function (err) {
if (err) {
console.error(err.stack || err.message || err);
} }
gadget.state.redirect = "";
}); });
if (gadget.state.redirect) {
return gadget.redirect({command: 'display', options: {
page: gadget.state.redirect
}});
}
gadget.state.redirect = "";
});
}) })
.onLoop(function () { .onLoop(function () {
...@@ -336,7 +392,7 @@ ...@@ -336,7 +392,7 @@
if (!torrent.ready) { if (!torrent.ready) {
continue; continue;
} }
if (document.getElementById(torrent.infoHash)) { if (torrentHtmlElement(torrent.infoHash)) {
// already present // already present
continue; continue;
} }
...@@ -349,6 +405,7 @@ ...@@ -349,6 +405,7 @@
magnet_uri: torrent.magnetURI, magnet_uri: torrent.magnetURI,
torrent_link: torrent.torrentFileBlobURL, torrent_link: torrent.torrentFileBlobURL,
infohash: torrent.infoHash, infohash: torrent.infoHash,
total: prettyBytes(torrent.length),
is_seed: true is_seed: true
}); });
container.innerHTML += torrent_template; container.innerHTML += torrent_template;
...@@ -380,7 +437,6 @@ ...@@ -380,7 +437,6 @@
} }
on_torrent = function (torrent) { on_torrent = function (torrent) {
//gadget.addSeedContent(torrent);
gadget.state.seed_index[id] = torrent; gadget.state.seed_index[id] = torrent;
torrent.on('warning', function (err) { torrent.on('warning', function (err) {
console.error(err.stack || err.message || err) console.error(err.stack || err.message || err)
...@@ -395,7 +451,7 @@ ...@@ -395,7 +451,7 @@
}); });
torrent.on('upload', function (bytes) { torrent.on('upload', function (bytes) {
var content = document.getElementById(torrent.infoHash); var content = torrentHtmlElement(torrent.infoHash);
if (content) { if (content) {
onProgress(gadget, torrent, content); onProgress(gadget, torrent, content);
} }
...@@ -427,8 +483,9 @@ ...@@ -427,8 +483,9 @@
torrent_dict[id] = name; torrent_dict[id] = name;
return gadget.index_to_file(index_doc.title); return gadget.index_to_file(index_doc.title);
}) })
.push(function (index_file_list) { .push(function (index_blob) {
var doc = {title: index_doc.title, links: index_doc.links}; var doc = {title: index_doc.title, links: index_doc.links},
index_file_list = [index_blob];
// Add index document to blob // Add index document to blob
index_file_list.push(new File( index_file_list.push(new File(
[new Blob([JSON.stringify(doc)], [new Blob([JSON.stringify(doc)],
...@@ -441,9 +498,7 @@ ...@@ -441,9 +498,7 @@
createdBy: "Nexedi SA", createdBy: "Nexedi SA",
private: false, private: false,
announceList: [ announceList: [
['wss://tracker.fastcast.nz'],
['wss://tracker.openwebtorrent.com'], ['wss://tracker.openwebtorrent.com'],
['wss://tracker.btorrent.xyz'],
['wss://softinst140244.host.vifib.net'] ['wss://softinst140244.host.vifib.net']
], ],
comment: index_doc.title comment: index_doc.title
...@@ -508,8 +563,6 @@ ...@@ -508,8 +563,6 @@
function onDone () { function onDone () {
clearInterval(gadget.state.interval); clearInterval(gadget.state.interval);
gadget.element.querySelector(".torrentBoddy")
.classList.add("is-seed");
onProgress(gadget, torrent, element); onProgress(gadget, torrent, element);
return gadget.file_to_index(torrent); return gadget.file_to_index(torrent);
} }
...@@ -522,11 +575,12 @@ ...@@ -522,11 +575,12 @@
magnet_link: torrent.name, magnet_link: torrent.name,
torrent_link: torrent.torrentFileBlobURL, torrent_link: torrent.torrentFileBlobURL,
infohash: torrent.infoHash, infohash: torrent.infoHash,
total: prettyBytes(torrent.length),
is_seed: false is_seed: false
}); });
container.innerHTML = torrent_template; container.innerHTML = torrent_template;
element = document.getElementById(torrent.infoHash); element = gadget.element.querySelector(".torrentBoddy");
torrent.on('error', function (err) { torrent.on('error', function (err) {
clearInterval(gadget.state.interval); clearInterval(gadget.state.interval);
console.error(err.stack || err.message || err); console.error(err.stack || err.message || err);
...@@ -547,4 +601,4 @@ ...@@ -547,4 +601,4 @@
}); });
}); });
}(window, RSVP, rJS, document, crypto, WebTorrent)); }(window, RSVP, rJS, document, crypto, WebTorrent, JSZip));
\ No newline at end of file \ No newline at end of file
bt5/erp5_web_mynij_search/PathTemplateItem/web_page_module/gadget_erp5_mynij_webtorrent_js.xml
View file @ 175c9113
...@@ -242,7 +242,7 @@ ...@@ -242,7 +242,7 @@
</item> </item>
<item> <item>
<key> <string>serial</string> </key> <key> <string>serial</string> </key>
<value> <string>989.18558.11562.17885</string> </value> <value> <string>989.27539.50696.19524</string> </value>
</item> </item>
<item> <item>
<key> <string>state</string> </key> <key> <string>state</string> </key>
...@@ -260,7 +260,7 @@ ...@@ -260,7 +260,7 @@
</tuple> </tuple>
<state> <state>
<tuple> <tuple>
<float>1610442308.5</float> <float>1610982586.19</float>
<string>UTC</string> <string>UTC</string>
</tuple> </tuple>
</state> </state>
......
bt5/erp5_web_mynij_search/PathTemplateItem/web_page_module/gadget_officejs_mynij_search_router.html.xml
View file @ 175c9113
...@@ -242,7 +242,7 @@ ...@@ -242,7 +242,7 @@
</item> </item>
<item> <item>
<key> <string>serial</string> </key> <key> <string>serial</string> </key>
<value> <string>989.8720.22549.10018</string> </value> <value> <string>989.8910.36396.20462</string> </value>
</item> </item>
<item> <item>
<key> <string>state</string> </key> <key> <string>state</string> </key>
...@@ -260,7 +260,7 @@ ...@@ -260,7 +260,7 @@
</tuple> </tuple>
<state> <state>
<tuple> <tuple>
<float>1609863273.31</float> <float>1610722711.08</float>
<string>UTC</string> <string>UTC</string>
</tuple> </tuple>
</state> </state>
......
bt5/erp5_web_mynij_search/SkinTemplateItem/portal_skins/erp5_mynij/action_mynij_search_import_export_index.js.js
View file @ 175c9113
...@@ -128,7 +128,7 @@ ...@@ -128,7 +128,7 @@
zip_file = new JSZip(); zip_file = new JSZip();
zip_file.file(index_id + ".json", JSON.stringify(file_dict)); zip_file.file(index_id + ".json", JSON.stringify(file_dict));
zip_file.generateAsync({ return zip_file.generateAsync({
type : "blob", type : "blob",
compression: "DEFLATE", compression: "DEFLATE",
compressionOptions: {level: 9} compressionOptions: {level: 9}
......
bt5/erp5_web_mynij_search/SkinTemplateItem/portal_skins/erp5_mynij/gadget_mynij_html_link_index.html.html 0 → 100644
View file @ 175c9113
<!doctype html>
<html>
<head>
<meta http-equiv="Content-type" content="text/html; charset=utf-8" />
<meta name="viewport" content="width=device-width" />
<!-- renderjs -->
<script src="rsvp.js" type="text/javascript"></script>
<script src="renderjs.js" type="text/javascript"></script>
<script src="flexsearch.js" type="text/javascript"></script>
<script src="gadget_mynij_html_link_index.js" type="text/javascript"></script>
</head>
<body>
<a href="https://softinst132402.host.vifib.net/erp5/portal_skins/erp5_mynij/gadget_mynij_html_link_index.html">Link to click</a>
</body>
</html>
\ No newline at end of file
bt5/erp5_web_mynij_search/SkinTemplateItem/portal_skins/erp5_mynij/gadget_mynij_html_link_index.html.xml 0 → 100644
View file @ 175c9113
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="File" module="OFS.Image"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>__name__</string> </key>
<value> <string>gadget_mynij_html_link_index.html</string> </value>
</item>
<item>
<key> <string>content_type</string> </key>
<value> <string>text/html</string> </value>
</item>
<item>
<key> <string>precondition</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>action_mynij_search_build_index.html</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_web_mynij_search/SkinTemplateItem/portal_skins/erp5_mynij/gadget_mynij_html_link_index.js.js 0 → 100644
View file @ 175c9113
(function (window, rJS, RSVP) {
"use strict";
rJS(window)
.declareMethod("render", function (options, extra) {
console.log(options);
console.log(extra);
});
}(window, rJS, RSVP));
\ No newline at end of file
bt5/erp5_web_mynij_search/SkinTemplateItem/portal_skins/erp5_mynij/gadget_mynij_html_link_index.js.xml 0 → 100644
View file @ 175c9113
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="File" module="OFS.Image"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>__name__</string> </key>
<value> <string>gadget_mynij_html_link_index.js</string> </value>
</item>
<item>
<key> <string>content_type</string> </key>
<value> <string>application/javascript</string> </value>
</item>
<item>
<key> <string>precondition</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>action_mynij_search_build_index.js</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7