erp5_web_mynij_search: Rework index torrent seed and download

Torrent are compressed before seed
torrent blob is saved as attachment and can be reloaded quickly without rebuild everything
There are more details on download and seed progress

bt5/erp5_web_mynij_search/PathTemplateItem/web_page_module/gadget_erp5_mynij_webtorrent_html.html

@@ -16,6 +16,7 @@
     <script src="jiodev.js" type="text/javascript"></script>
     <script src="handlebars.js" type="text/javascript"></script>
     <script src="webtorrent.min.js" type="text/javascript"></script>
+    <script src="jszip.min.js" type="text/javascript"></script>
     <script src="gadget_erp5_mynij_webtorrent.js" type="text/javascript"></script>
 
     <script id="torrent-info-template" type="text/x-handlebars-template">
@@ -41,12 +42,17 @@
               {{/if}}
             </div>
             <div>
-              <code class="downloaded"></code>
-              of <code class="total"></code>
-              — <span class="remaining"></span><br/>
+              {{#if is_seed}}
+              &#x2197;<code class="uploadSpeed">0 b/s</code>
+              / <code class="total">{{total}}</code>
+              {{else}}
               &#x2198;<code class="downloadSpeed">0 b/s</code>
-              / &#x2197;<code class="uploadSpeed">0 b/s</code>
+              / <code class="downloaded">0 B</code>
+              of <code class="total">{{total}}</code>
+              — <span class="remaining"></span><br/>
+              {{/if}}
             </div>
+            <div class="message"></div>
           </div>
         </div>
       </div>

bt5/erp5_web_mynij_search/PathTemplateItem/web_page_module/gadget_erp5_mynij_webtorrent_html.xml

@@ -246,7 +246,7 @@
                   </item>
                   <item>
                       <key> <string>serial</string> </key>
-                      <value> <string>988.48319.53521.12885</string> </value>
+                      <value> <string>989.27292.54989.38297</string> </value>
                   </item>
                   <item>
                       <key> <string>state</string> </key>
@@ -264,7 +264,7 @@
                           </tuple>
                           <state>
                             <tuple>
-                              <float>1608296295.72</float>
+                              <float>1610966584.34</float>
                               <string>UTC</string>
                             </tuple>
                           </state>

bt5/erp5_web_mynij_search/PathTemplateItem/web_page_module/gadget_erp5_mynij_webtorrent_js.js

 /*jslint nomen: true, indent: 2, maxerr: 3, maxlen: 80*/
-/*global window, RSVP, rJS, WebTorrent*/
-(function (window, RSVP, rJS, document, crypto, WebTorrent) {
+/*global window, RSVP, rJS, WebTorrent, JSZip*/
+(function (window, RSVP, rJS, document, crypto, WebTorrent, JSZip) {
   "use strict";
 
   var gadget_klass = rJS(window),
@@ -45,8 +45,12 @@
     // Progress
     var percent = Math.round(torrent.progress * 100 * 100) / 100;
     progressBar.style.width = percent + '%';
-    downloaded.innerHTML = prettyBytes(torrent.downloaded);
     total.innerHTML = prettyBytes(torrent.length);
+    if (!downloaded) {
+      uploadSpeed.innerHTML = prettyBytes(torrent.uploadSpeed) + '/s';
+      return;
+    }
+    downloaded.innerHTML = prettyBytes(torrent.downloaded);
 
     // Remaining time
     if (torrent.done) {
@@ -61,7 +65,13 @@
 
     // Speed rates
     downloadSpeed.innerHTML = prettyBytes(torrent.downloadSpeed) + '/s';
-    uploadSpeed.innerHTML = prettyBytes(torrent.uploadSpeed) + '/s';
+  }
+
+  function torrentHtmlElement(infohash, qclass) {
+    var element = document.getElementById(infohash);
+    if (qclass && element)
+      return element.querySelector('.' + qclass);
+    return element;
   }
 
   gadget_klass
@@ -110,10 +120,34 @@
     })
 
     .declareMethod("index_to_file", function (index_name) {
+      var gadget = this,
+          index_id = "index-" + index_name;
+      return gadget.state.model_gadget.getAttachment(
+        index_id,
+        "zipped",
+        {"format": "blob"})
+      .push(function (result) {
+        return new File([result], index_id + ".zip", {type: 'blob'});;
+      })
+      .push(undefined, function () {
+        console.log("Building index blob");
+        return gadget.notifySubmitted({
+          message: "Building '" + index_name + "' torrent files...",
+          status: "success"
+        })
+          .push(function () {
+            return gadget.buildIndexBlob(index_name, index_id);
+          });
+      });
+    })
+
+    .declareMethod("buildIndexBlob", function (index_name, index_id) {
       var gadget = this,
           ids,
-          index_id = "index-" + index_name,
-          file_list = [];
+          file_list = [],
+          zipped_file,
+          zip = new JSZip(),
+          name = index_id + ".zip";
 
       return gadget.state.model_gadget.getAttachment(
         index_id,
@@ -121,22 +155,14 @@
         {"format": "json"})
       .push(function (result) {
         ids = result;
-        file_list.push(new File(
-          [new Blob([JSON.stringify(result)], {type : "application/json"})],
-          index_id + '.ids',
-          {type: 'application/json'})
-        );
+        zip.file(index_id + '.ids', JSON.stringify(result));
         return gadget.state.model_gadget.getAttachment(
           index_id,
           "map",
           {"format": "text"});
       })
       .push(function (result) {
-        file_list.push(new File(
-          [new Blob([result], {type : "application/json"})],
-          index_id + '.map',
-          {type: 'application/json'})
-        );
+        zip.file(index_id + '.map', result);
         return gadget.state.model_gadget.getAttachment(
           index_id,
           "ctx",
@@ -153,19 +179,11 @@
               return gadget.state.model_gadget.get_page(page_id);
             })
             .push(function (page) {
-              file_list.push(new File(
-                [new Blob([JSON.stringify(page)], {type : "application/json"})],
-                page_id + '.page',
-                {type: 'application/json'})
-              );
+              zip.file(page_id + '.json', JSON.stringify(page));
             });
         }
 
-        file_list.push(new File(
-          [new Blob([result], {type : "application/json"})],
-          index_id + '.ctx',
-          {type: 'application/json'})
-        );
+        zip.file(index_id + '.ctx', result);
         // get all page of this index
         for (i = 0; i < ids.length; i += 1) {
           get_page(ids[i].slice(1));
@@ -173,7 +191,25 @@
         return page_queue
       })
       .push(function () {
-        return file_list;
+        return new RSVP.Queue()
+        .push(function () {
+          return zip.generateAsync({
+            type : "blob",
+            compression: "DEFLATE",
+            compressionOptions: {level: 9}
+          })
+            .then(function (zipped) {
+              zipped_file = new File([zipped], name, {type: 'blob'});
+              return gadget.state.model_gadget.putAttachment(
+                index_id,
+                "zipped",
+                zipped
+              );
+            });
+        })
+        .push(function () {
+          return zipped_file;
+        });
       });
     })
 
@@ -183,7 +219,8 @@
           blob_list = [],
           name_list = [],
           failed = false,
-          file_list = torrent.files;
+          file_list = torrent.files,
+          msg_html;
 
       //quick check of file consitency here before insert
       extract_blob = function (err, blob) {
@@ -201,7 +238,8 @@
           file = file_list.shift();
           name_list.push(file.name);
           // extract next blob file
-          console.log("Extracting ... " + file.name);
+          if (msg_html)
+            msg_html.innerText = "Extracting file " + file.name + " ...";
           return file.getBlob(extract_blob);
         }
         else {
@@ -210,111 +248,129 @@
             console.log("Torrent extraction failed!!!");
             return;
           }
-          return gadget.saveExtractedIndex(torrent, name_list, blob_list)
+          return gadget.saveExtractedIndex(torrent,
+                                           name_list,
+                                           blob_list)
         }
       }
       // extract all blob from files
+      msg_html = torrentHtmlElement(torrent.infoHash, 'message')
       return extract_blob();
     })
 
     .declareMethod("saveExtractedIndex", function (torrent,
-                                                   name_list,
+                                                   ext_list,
                                                    blob_list) {
       var gadget = this,
+          zip = new JSZip(),
+          insert_data,
+          zipped_blob,
+          doc_blob,
+          index_name,
+          doc,
+          ext,
           i,
-          insert_queue = new RSVP.Queue(),
           insert_blob,
-          insert_page,
-          insert_doc,
-          current,
-          type,
-          index_name,
-          id;
-
-      insert_doc = function (blob) {
-        insert_queue
-          .push(function () {
-            return jIO.util.readBlobAsText(blob);
-          })
-          .push(function (evt) {
-            var doc = JSON.parse(evt.target.result);
-            return gadget.state.model_gadget.createIndexDoc({
-              title: doc.title,
-              links: doc.links,
-              magnet_uri: torrent.magnetURI,
-              status: "built"
-            });
-          })
-      }
+          msg_html;
 
-      insert_blob = function (index_id, type, blob) {
-        insert_queue
-          .push(function () {
-            return gadget.state.model_gadget.putAttachment(
-              index_id,
-              type,
-              blob);
-          })
-      }
-
-      insert_page = function (page_id, blob) {
-        insert_queue
-          .push(function () {
-            return jIO.util.readBlobAsText(blob);
-          })
-          .push(function (evt) {
-            return gadget.state.model_gadget.put(
-              page_id, JSON.parse(evt.target.result));
-          })
-
-      }
-
-      for (i = 0; i < blob_list.length;  i += 1) {
-        current = name_list[i];
-        if (current.endsWith('page')) {
-          type = 'page';
-          id = current.slice(0,-5);
+      insert_blob = function (index_id, type, data) {
+        return gadget.state.model_gadget.putAttachment(
+          index_id,
+          type,
+          new Blob([data], {type : "application/json"}));
+      };
+      insert_data = function (name, data) {
+        var type,
+            id;
+        if (name.endsWith('json')) {
+          type = 'json';
+          id = name.slice(0,-5);
         } else {
-          type = current.slice(-3);
-          id = current.slice(0,-4);
+          type = name.slice(-3);
+          id = name.slice(0,-4);
         }
-        if (type === 'page') {
-          insert_page(id, blob_list[i]);
+        if (type === 'json') {
+          return gadget.state.model_gadget.put(id, JSON.parse(data));
         } else if (type === 'ctx') {
-          insert_blob(id, type, blob_list[i]);
+          insert_blob(id, type, data);
         } else if (type === 'map') {
-          insert_blob(id, type, blob_list[i]);
+          insert_blob(id, type, data);
         } else if (type === 'ids') {
           index_name = id.split("index-")[1]
-          insert_blob(id, type, blob_list[i]);
-        } else if (type === 'doc') {
-          insert_doc(blob_list[i]);
+          insert_blob(id, type, data);
         }
       }
-      return insert_queue
-        .push(function () {
-            return gadget.state.model_gadget.pushIndex(index_name);
-        })
-        .push(function () {
-          gadget.notifySubmitted({
-            message: "Index '" + index_name + "' Saved.",
-            status: "success"
-          })
-        })
-        .push(function () {
-          // stop seeding the torrent
-          gadget.state.client.destroy(function (err) {
-            if (err) {
-              console.error(err.stack || err.message || err);
+
+      msg_html = torrentHtmlElement(torrent.infoHash, 'message');
+      // There is 2 files in the torrent, index doc and zipped data
+      for (i = 0; i < ext_list.length; i += 1) {
+        ext = ext_list[i].slice(-3);
+        if (ext === "doc") {
+          doc_blob = blob_list[i];
+        }
+        else if (ext === "zip")
+          zipped_blob = blob_list[i];
+      }
+      return new RSVP.Queue()
+      .push(function () {
+        return jIO.util.readBlobAsText(doc_blob);
+      })
+      .push(function (evt) {
+        doc = JSON.parse(evt.target.result);
+        if (msg_html)
+          msg_html.innerText = "Uncompressing index data...";
+        // extract zipped data here
+        return zip.loadAsync(zipped_blob)
+          .then(function (data) {
+            var promise_list = [],
+                get_zip;
+            get_zip = function (name) {
+              return zip.file(name).async('string')
+                .then(function(data) {
+                  if (msg_html)
+                    msg_html.innerText = "Extracting file " + name + "...";
+                  return insert_data(name, data);
+                });
             }
+            Object.keys(data.files).forEach(function (filename) {
+                promise_list.push(get_zip(filename));
+            });
+            return RSVP.all(promise_list);
           });
-          if (gadget.state.redirect) {
-            return gadget.redirect({command: 'display', options: {
-              page: gadget.state.redirect
-            }});
+      })
+      .push(function () {
+        if (msg_html)
+          msg_html.innerText = "Creating index document...";
+        return gadget.state.model_gadget.createIndexDoc({
+          title: doc.title,
+          links: doc.links,
+          magnet_uri: torrent.magnetURI,
+          status: "built"
+        });
+      })
+      .push(function () {
+          return gadget.state.model_gadget.pushIndex(index_name);
+      })
+      .push(function () {
+        gadget.notifySubmitted({
+          message: "Index '" + index_name + "' Saved.",
+          status: "success"
+        })
+      })
+      .push(function () {
+        // stop seeding the torrent
+        gadget.state.client.destroy(function (err) {
+          if (err) {
+            console.error(err.stack || err.message || err);
           }
-          gadget.state.redirect = "";
         });
+        if (gadget.state.redirect) {
+          return gadget.redirect({command: 'display', options: {
+            page: gadget.state.redirect
+          }});
+        }
+        gadget.state.redirect = "";
+      });
     })
 
     .onLoop(function () {
@@ -336,7 +392,7 @@
         if (!torrent.ready) {
           continue;
         }
-        if (document.getElementById(torrent.infoHash)) {
+        if (torrentHtmlElement(torrent.infoHash)) {
           // already present
           continue;
         }
@@ -349,6 +405,7 @@
           magnet_uri: torrent.magnetURI,
           torrent_link: torrent.torrentFileBlobURL,
           infohash: torrent.infoHash,
+          total: prettyBytes(torrent.length),
           is_seed: true
         });
         container.innerHTML += torrent_template;
@@ -380,7 +437,6 @@
       }
 
       on_torrent = function (torrent) {
-        //gadget.addSeedContent(torrent);
         gadget.state.seed_index[id] = torrent;
         torrent.on('warning', function (err) {
           console.error(err.stack || err.message || err)
@@ -395,7 +451,7 @@
         });
 
         torrent.on('upload', function (bytes) {
-          var content = document.getElementById(torrent.infoHash);
+          var content = torrentHtmlElement(torrent.infoHash);
           if (content) {
             onProgress(gadget, torrent, content);
           }
@@ -427,8 +483,9 @@
           torrent_dict[id] = name;
           return gadget.index_to_file(index_doc.title);
         })
-        .push(function (index_file_list) {
-          var doc = {title: index_doc.title, links: index_doc.links};
+        .push(function (index_blob) {
+          var doc = {title: index_doc.title, links: index_doc.links},
+              index_file_list = [index_blob];
           // Add index document to blob
           index_file_list.push(new File(
             [new Blob([JSON.stringify(doc)],
@@ -441,9 +498,7 @@
             createdBy: "Nexedi SA",
             private: false,
             announceList: [
-              ['wss://tracker.fastcast.nz'],
               ['wss://tracker.openwebtorrent.com'],
-              ['wss://tracker.btorrent.xyz'],
               ['wss://softinst140244.host.vifib.net']
             ],
             comment: index_doc.title
@@ -508,8 +563,6 @@
 
           function onDone () {
             clearInterval(gadget.state.interval);
-            gadget.element.querySelector(".torrentBoddy")
-              .classList.add("is-seed");
             onProgress(gadget, torrent, element);
             return gadget.file_to_index(torrent);
           }
@@ -522,11 +575,12 @@
             magnet_link: torrent.name,
             torrent_link: torrent.torrentFileBlobURL,
             infohash: torrent.infoHash,
+            total: prettyBytes(torrent.length),
             is_seed: false
           });
           container.innerHTML = torrent_template;
 
-          element = document.getElementById(torrent.infoHash);
+          element = gadget.element.querySelector(".torrentBoddy");
           torrent.on('error', function (err) {
             clearInterval(gadget.state.interval);
             console.error(err.stack || err.message || err);
@@ -547,4 +601,4 @@
       });
     });
 
-}(window, RSVP, rJS, document, crypto, WebTorrent));
\ No newline at end of file
+}(window, RSVP, rJS, document, crypto, WebTorrent, JSZip));
\ No newline at end of file

bt5/erp5_web_mynij_search/PathTemplateItem/web_page_module/gadget_erp5_mynij_webtorrent_js.xml

@@ -242,7 +242,7 @@
                   </item>
                   <item>
                       <key> <string>serial</string> </key>
-                      <value> <string>989.18558.11562.17885</string> </value>
+                      <value> <string>989.27539.50696.19524</string> </value>
                   </item>
                   <item>
                       <key> <string>state</string> </key>
@@ -260,7 +260,7 @@
                           </tuple>
                           <state>
                             <tuple>
-                              <float>1610442308.5</float>
+                              <float>1610982586.19</float>
                               <string>UTC</string>
                             </tuple>
                           </state>

bt5/erp5_web_mynij_search/PathTemplateItem/web_page_module/gadget_officejs_mynij_search_router.html.xml

@@ -242,7 +242,7 @@
                   </item>
                   <item>
                       <key> <string>serial</string> </key>
-                      <value> <string>989.8720.22549.10018</string> </value>
+                      <value> <string>989.8910.36396.20462</string> </value>
                   </item>
                   <item>
                       <key> <string>state</string> </key>
@@ -260,7 +260,7 @@
                           </tuple>
                           <state>
                             <tuple>
-                              <float>1609863273.31</float>
+                              <float>1610722711.08</float>
                               <string>UTC</string>
                             </tuple>
                           </state>

bt5/erp5_web_mynij_search/SkinTemplateItem/portal_skins/erp5_mynij/action_mynij_search_import_export_index.js.js

@@ -128,7 +128,7 @@
 
         zip_file = new JSZip();
         zip_file.file(index_id + ".json", JSON.stringify(file_dict));
-        zip_file.generateAsync({
+        return zip_file.generateAsync({
           type : "blob",
           compression: "DEFLATE",
           compressionOptions: {level: 9}

bt5/erp5_web_mynij_search/SkinTemplateItem/portal_skins/erp5_mynij/gadget_mynij_html_link_index.html.html

+<!doctype html>
+<html>
+  <head>
+    <meta http-equiv="Content-type" content="text/html; charset=utf-8" />
+    <meta name="viewport" content="width=device-width" />
+    <!-- renderjs -->
+    <script src="rsvp.js" type="text/javascript"></script>
+    <script src="renderjs.js" type="text/javascript"></script>
+    <script src="flexsearch.js" type="text/javascript"></script>
+    <script src="gadget_mynij_html_link_index.js" type="text/javascript"></script>
+  </head>
+<body>
+  <a href="https://softinst132402.host.vifib.net/erp5/portal_skins/erp5_mynij/gadget_mynij_html_link_index.html">Link to click</a>
+</body>
+</html>
\ No newline at end of file

bt5/erp5_web_mynij_search/SkinTemplateItem/portal_skins/erp5_mynij/gadget_mynij_html_link_index.html.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="File" module="OFS.Image"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>__name__</string> </key>
+            <value> <string>gadget_mynij_html_link_index.html</string> </value>
+        </item>
+        <item>
+            <key> <string>content_type</string> </key>
+            <value> <string>text/html</string> </value>
+        </item>
+        <item>
+            <key> <string>precondition</string> </key>
+            <value> <string></string> </value>
+        </item>
+        <item>
+            <key> <string>title</string> </key>
+            <value> <string>action_mynij_search_build_index.html</string> </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+</ZopeData>

bt5/erp5_web_mynij_search/SkinTemplateItem/portal_skins/erp5_mynij/gadget_mynij_html_link_index.js.js

+(function (window, rJS, RSVP) {
+    "use strict";
+    rJS(window)
+
+    .declareMethod("render", function (options, extra) {
+        console.log(options);
+        console.log(extra);
+    });
+
+}(window, rJS, RSVP));
\ No newline at end of file

bt5/erp5_web_mynij_search/SkinTemplateItem/portal_skins/erp5_mynij/gadget_mynij_html_link_index.js.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="File" module="OFS.Image"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>__name__</string> </key>
+            <value> <string>gadget_mynij_html_link_index.js</string> </value>
+        </item>
+        <item>
+            <key> <string>content_type</string> </key>
+            <value> <string>application/javascript</string> </value>
+        </item>
+        <item>
+            <key> <string>precondition</string> </key>
+            <value> <string></string> </value>
+        </item>
+        <item>
+            <key> <string>title</string> </key>
+            <value> <string>action_mynij_search_build_index.js</string> </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+</ZopeData>