big update

jio @ 96b399eb

-Subproject commit b08e246c91d99293f685c85e990174f204964ca4
+Subproject commit 96b399ebc2d5db74a340be1ce4ab486cda607485

src/js/gadget_loader.js

@@ -18,30 +18,30 @@
   
   .setState({
     to_load: [
-      //"44_svt.xml", //135 urls
-      //"allemandfacile.xml", //650 urls
-      //"anglaisfacile.xml", //567 urls
-      "codeacademy.xml", //28 urls
-      //"francaisfacile.xml", //1119 urls
-      //"hgeo_college.xml", //227 urls
+      // "44_svt.xml", //135 urls
+      // "allemandfacile.xml", //650 urls
+      // "anglaisfacile.xml", //567 urls
+      // "codeacademy.xml", //28 urls
+      // "francaisfacile.xml", //1119 urls
+      // "hgeo_college.xml", //227 urls
       // "histoirencours.xml", //1415 urls
       // "italienfacile.xml", //1478 urls
       // "jerevise.xml", //919 urls
       // "junior_science_et_vie.xml", //532 urls
-      //"kmusic.xml", //107 urls
+      // "kmusic.xml", //107 urls
       // "larousse.xml", //4563 urls
-      // //"letudiant.xml", //41649 urls
+      // "letudiant.xml", //41649 urls
       // "lewebpedagogique.xml", //298 urls
-      // //"livrespourtous.xml", //12061 urls
+      // "livrespourtous.xml", //12061 urls
       // "mathovore.xml", //2222 urls
-      //"monanneeaucollege.xml", //121 urls
+      // "monanneeaucollege.xml", //121 urls
       // "nosdevoirs.xml", //462 urls
-      //"physagreg.xml", //150 urls
-      //"physique_chimie_college.xml", //282 urls
-      //"reviser_brevet.xml", //229 urls
+      // "physagreg.xml", //150 urls
+      // "physique_chimie_college.xml", //282 urls
+      // "reviser_brevet.xml", //229 urls
       // "soutien67.xml", //1604 urls
-      // //"superprof.xml", //12296 urls
-      //"technologieaucollege27.xml", //129 urls
+      // "superprof.xml", //12296 urls
+      // "technologieaucollege27.xml", //129 urls
       // "espagnolfacile.xml", //3352 urls
       // "vivelessvt.xml", //1257 urls
       // // TEST SITEMAPS TO FILL INDEX
@@ -68,15 +68,15 @@
       // "mirror.xml", //3528 urls
       // "abcnews.xml", //971 urls
       // "lequipe.xml", //3455 urls
-      //"rugbyrama.xml", //1817 urls
+      // "rugbyrama.xml", //1817 urls
       // "elle.xml", //3532 urls
       // "figaro.xml", //2965 urls
       // "lepoint.xml", //3747 urls
       // "telerama.xml", //2593 urls
       // "liberation.xml", //819 urls
       // "lemonde.xml", //3517 urls
-      //"leparisien.xml", //2189 urls
-      //"latribune.xml" //3190 urls
+      // "leparisien.xml", //2189 urls
+      // "latribune.xml" //3190 urls
      ]
   })
   

src/js/gadget_model.js

@@ -47,9 +47,6 @@
     tmp.portal_type = "page";
     return gadget.state.db.put(page_info.link, tmp)
       .push(function(){
-        /*var defer = RSVP.defer();
-        gadget.state.index.add(page_info.link, page_info.title + " " + page_info.item, defer.resolve.bind(defer));
-        return defer.promise;*/
         return gadget.state.index.add(page_info.link, page_info.title + " " + page_info.item);
       });
  })
@@ -77,77 +74,19 @@
  })
  
  .declareMethod("_save_index", function(){
-/*    var gadget = this;
+    var gadget = this;
     console.log("getting index");
     var serialized = this.state.index.export_test();
     console.log("Index received. Saving");
     return this.state.db.put("index", this.state.index.info())
       .push(function(){
-        console.log(1);
-        //return gadget.state.db.putAttachment("index", "ids", new Blob([serialized.ids], {type : "application/json"}));
-        return gadget.state.db.putAttachment("index", "ids", new Blob([serialized.ids], {type : "text/plain"}));
+        return gadget.state.db.putAttachment("index", "ids", new Blob([serialized.ids], {type : "application/json"}));
       })
       .push(function(){
-        console.log(2);
-        //return gadget.state.db.putAttachment("index", "map", new Blob([serialized.map], {type : "application/json"}));
-        var shaObj = new jsSHA("SHA-1", "TEXT");
-        shaObj.update(serialized.map);
-        console.log("map sha1 : " + shaObj.getHash("HEX"));
-        return gadget.state.db.putAttachment("index", "map", new Blob([serialized.map], {type : "text/plain"}));
+        return gadget.state.db.putAttachment("index", "map", new Blob([serialized.map], {type : "application/json"}));
       })
       .push(function(){
-        console.log(3);
-        //return gadget.state.db.putAttachment("index", "ctx", new Blob([serialized.ctx], {type : "application/json"}));
-        return gadget.state.db.putAttachment("index", "ctx", new Blob([serialized.ctx], {type : "text/plain"}));
-      })
-      .push(function(){
-        console.log("index saved");
-      })
-      .push(undefined, function (my_error) {
-        throw my_error;
-        //console.log(my_error);
-      });*/
-      
-    var gadget = this,
-      serialized,
-      i,
-      j = 0,
-      promise_list = [];
-    console.log("getting index");
-    serialized = this.state.index.export_test();
-    console.log("Index received. Saving");
-    return this.state.db.put("index_map", {})
-      .push(function(){
-        console.log(serialized.map.slice(0, 1000));
-        for (i = 0; i < serialized.map.length; i += 10000){
-          promise_list.push(gadget.state.db.putAttachment("index_map", j+"", new Blob([serialized.map.slice(i, i+10000)], {type : "application/json"})));
-          j+=1;
-        }
-        j = 0;
-        return RSVP.all(promise_list);
-      })
-      .push(function(){
-        return gadget.state.db.put("index_ids", {});
-      })
-      .push(function(){
-        promise_list = [];
-        for (i = 0; i < serialized.map.length; i += 10000){
-          promise_list.push(gadget.state.db.putAttachment("index_ids", j+"", new Blob([serialized.ids.slice(i, i+10000)], {type : "application/json"})));
-          j+=1;
-        }
-        j = 0;
-        return RSVP.all(promise_list);
-      })
-      .push(function(){
-        return gadget.state.db.put("index_ctx", {});
-      })
-      .push(function(){
-        promise_list = [];
-        for (i = 0; i < serialized.ctx.length; i += 10000){
-          promise_list.push(gadget.state.db.putAttachment("index_ctx", j+"", new Blob([serialized.map.slice(i, i+10000)], {type : "application/json"})));
-          j+=1;
-        }
-        return RSVP.all(promise_list);
+        return gadget.state.db.putAttachment("index", "ctx", new Blob([serialized.ctx], {type : "application/json"}));
       })
       .push(function(){
         console.log("index saved");
@@ -155,7 +94,6 @@
       .push(undefined, function (my_error) {
         console.log(my_error);
       });
-    
  })
  
  .declareMethod("search", function(query){
@@ -163,7 +101,7 @@
  })
  
  .declareMethod("_load_index", function(msgpack){
-/*    var gadget = this,
+   var gadget = this,
         ids,
         map;
     return gadget.state.db.getAttachment("index", "ids", {"format": "text"})
@@ -173,76 +111,13 @@
       })
       .push(function(result){
         map = result;
-        var shaObj = new jsSHA("SHA-1", "TEXT");
-        shaObj.update(map);
-        console.log("map sha1 : " + shaObj.getHash("HEX"));
         return gadget.state.db.getAttachment("index", "ctx", {"format": "text"});
       })
       .push(function(result){
         gadget.state.index.import_test(ids, map, result);
         console.log("index imported from memory");
       })
-      .push(undefined, function (my_error) {console.log(my_error)});*/
-    var gadget = this,
-      ids,
-      map,
-      ctx,
-      promise_list = [],
-      i;
-    return gadget.state.db.allAttachments("index_ids")
-      .push(function(result){
-        if (Object.keys(result).length !== 0){
-          for (i = 0; i < Object.keys(result).length; i+=1){
-            promise_list.push(gadget.state.db.getAttachment("index_ids", i+"", {format : "text"}));
-          }
-          return RSVP.all(promise_list);
-        } else {
-          return null;
-        }
-      })
-      .push(function(result){
-        if (result === null) ids = "";
-        else ids = result.join("");
-        return gadget.state.db.allAttachments("index_map");
-      })
-      .push(function(result){
-        if (Object.keys(result).length !== 0){
-          for (i = 0; i < Object.keys(result).length; i+=1){
-            promise_list.push(gadget.state.db.getAttachment("index_map", i+"", {format : "text"}));
-          }
-          return RSVP.all(promise_list);
-        } else {
-          return null;
-        }
-      })
-      .push(function(result){
-        console.log(result[0]);
-        if (result === null) map = "";
-        else map = result.join("");
-        console.log(map.slice(0, 1000));
-        return gadget.state.db.allAttachments("index_ctx");
-      })
-      .push(function(result){
-        if (Object.keys(result).length !== 0){
-          for (i = 0; i < Object.keys(result).length; i+=1){
-            promise_list.push(gadget.state.db.getAttachment("index_ctx", i+"", {format : "text"}));
-          }
-          return RSVP.all(promise_list);
-        } else {
-          return null;
-        }
-      })
-      .push(function(result){
-        if (result === null) ctx = "";
-        else ctx = result.join("");
-        return gadget.add_index(ids, map, ctx);
-      })
-      .push(function(result){
-        console.log("index imported from memory");
-      })
-      .push(undefined, function (my_error) {
-        console.log(my_error);
-      });
+      .push(undefined, function (my_error) {console.log(my_error)});
   })
     
   .declareMethod("is_empty", function(){

src/js/gadget_parser.js

@@ -43,7 +43,7 @@
   .declareAcquiredMethod("add", "add")
   .declareAcquiredMethod("add_file", "add_file")
 
-  .declareMethod("concurrent_parse", function(links_file, file_name){
+  .declareMethod("concurrent_parse", function(links_file){
     var gadget = this,
       links = new DOMParser().parseFromString(links_file, "text/xml").getElementsByTagName("url"),
       links_modified = [],

src/js/search.js

@@ -90,7 +90,6 @@
           reader = new FileReader(),
           zip = new JSZip();
       reader.onload = function(event){
-        //return gadget.state.model_gadget.add_index(event.target.result);
         var view = new Uint8Array(event.target.result),
           load = function(file_name){
             return zip.loadAsync(view)

test.html

+<!doctype html>
+<html>
+ <head>
+ <title>Test</title>
+  <script src="./external/jio/external/rsvp-2.0.4.js"></script>
+  <script src="./external/jio/dist/jio-latest.js"></script>
+  <script src="./external/renderjs/dist/renderjs-latest.js"></script>
+  <script src="./external/rusha/dist/rusha.js"></script>  
+ <script src="test.js"></script>
+ </head>
+ <body>
+ </body>
+</html>
\ No newline at end of file

test.js

+/*jslint nomen: true, indent: 2, maxerr: 3, maxlen: 80*/
+/*global window, RSVP, rJS, jIO*/
+(function (window, document, RSVP, rJS, jIO, Rusha) {
+ "use strict";
+ rJS(window)
+ 
+ .ready(function(){
+    var db,
+      small_test_map,
+      small_test_map_array,
+      received_small_test_map,
+      small_p_sent,
+      small_p_received,
+      big_test_map,
+      big_test_map_array,
+      received_big_test_map,
+      big_p_sent,
+      big_p_received,
+      string_to_array,
+      show_results,
+      rusha = new Rusha();
+    
+    db = jIO.createJIO({
+      type : "indexeddb",
+      database : "test"
+    });
+   
+    show_results = function(){
+      small_p_sent = document.createElement('p');
+      small_p_sent.innerHTML = "Sent small test map SHA1 : " + rusha.digestFromArrayBuffer(small_test_map_array);
+      document.body.appendChild(small_p_sent);
+      
+      small_p_received = document.createElement('p');
+      small_p_received.innerHTML = "Received small test map SHA1 : " + rusha.digestFromArrayBuffer(received_small_test_map);
+      document.body.appendChild(small_p_received);
+      
+      big_p_sent = document.createElement('p');
+      big_p_sent.innerHTML = "Sent big test map SHA1 : " + rusha.digestFromArrayBuffer(big_test_map_array);
+      document.body.appendChild(big_p_sent);
+      
+      big_p_received = document.createElement('p');
+      big_p_received.innerHTML = "Received big test map SHA1 : " + rusha.digestFromArrayBuffer(received_big_test_map);
+      document.body.appendChild(big_p_received);
+    };
+   
+    new RSVP.Queue()
+      .push(function(){
+        return jIO.util.ajax({url : "small_test_map"});
+      })
+      .push(function(file_small){
+        small_test_map = file_small.currentTarget.responseText;
+        return jIO.util.ajax({url : "big_test_map"});
+      })
+      .push(function(file_big){
+        big_test_map = file_big.currentTarget.responseText;
+        console.log(big_test_map.length);
+        return db.put("test", {});
+      })
+      .push(function(){
+        return jIO.util.readBlobAsArrayBuffer(new Blob([small_test_map]));
+      })
+      .push(function(small_array){
+        small_test_map_array = small_array.target.result;
+        return jIO.util.readBlobAsArrayBuffer(new Blob([big_test_map]));
+      })
+      .push(function(big_array){
+        big_test_map_array = big_array.target.result;
+        return db.putAttachment("test", "small", new Blob([small_test_map], {type : "text/plain"}));
+      })
+      .push(function(){
+        return db.putAttachment("test", "big", new Blob([big_test_map], {type : "text/plain"}));
+      })
+      .push(function(){
+        return db.getAttachment("test", "small");
+      })
+      .push(function(received_small_blob){
+        console.log("received small");
+        return jIO.util.readBlobAsArrayBuffer(received_small_blob);
+      })
+      .push(function(received_small_array){
+        received_small_test_map = received_small_array.target.result;
+        return db.getAttachment("test", "big");
+      })
+      .push(function(received_big_blob){
+        console.log("received big");
+        return jIO.util.readBlobAsArrayBuffer(received_big_blob);
+      })
+      .push(function(received_big_array){
+        received_big_test_map = received_big_array.target.result;
+        show_results();
+      })
+      .push(undefined, function(error){console.log(error)});
+ });
+ 
+}(window, document, RSVP, rJS, jIO, Rusha));
\ No newline at end of file