erp5_web_mynij_search: do not query whole indexeddb to get available indexes

speedup search page by storing indexes name in settings.

bt5/erp5_web_mynij_search/PathTemplateItem/web_page_module/gadget_erp5_page_mynij_search_js.js

@@ -12,6 +12,7 @@
   .declareAcquiredMethod("getUrlParameter", "getUrlParameter")
   .declareAcquiredMethod("getUrlFor", "getUrlFor")
   .declareAcquiredMethod("redirect", "redirect")
+  .declareAcquiredMethod("getSetting", "getSetting")
 
 
   .ready(function () {
@@ -55,7 +56,8 @@
         });
       })
       .push(function () {
-        return gadget.state.model_gadget.get_available_indices();
+        //return gadget.state.model_gadget.get_available_indices();
+        return gadget.getSetting('index_list', []);
       })
       .push(function (results) {
         index_list = results;

bt5/erp5_web_mynij_search/PathTemplateItem/web_page_module/gadget_erp5_page_mynij_search_js.xml

@@ -242,7 +242,7 @@
                   </item>
                   <item>
                       <key> <string>serial</string> </key>
-                      <value> <string>987.30048.36447.4505</string> </value>
+                      <value> <string>987.50625.31910.7697</string> </value>
                   </item>
                   <item>
                       <key> <string>state</string> </key>
@@ -260,7 +260,7 @@
                           </tuple>
                           <state>
                             <tuple>
-                              <float>1603353786.26</float>
+                              <float>1604588336.04</float>
                               <string>UTC</string>
                             </tuple>
                           </state>

bt5/erp5_web_mynij_search/PathTemplateItem/web_page_module/gadget_mynij_model.js.xml

@@ -242,7 +242,7 @@
                   </item>
                   <item>
                       <key> <string>serial</string> </key>
-                      <value> <string>987.34699.59880.60962</string> </value>
+                      <value> <string>987.50881.18269.62924</string> </value>
                   </item>
                   <item>
                       <key> <string>state</string> </key>
@@ -260,7 +260,7 @@
                           </tuple>
                           <state>
                             <tuple>
-                              <float>1603632793.79</float>
+                              <float>1605004238.18</float>
                               <string>UTC</string>
                             </tuple>
                           </state>

bt5/erp5_web_mynij_search/PathTemplateItem/web_page_module/gadget_mynij_parser.js.js

@@ -40,6 +40,16 @@
     .declareAcquiredMethod("save_index", "save_index")
     .declareAcquiredMethod("getParentElement", "getParentElement")
 
+    .declareMethod("pushIndex", function (index_name) {
+      var gadget = this;
+      return gadget.getSetting('index_list', [])
+        .push(function (index_list) {
+          if (index_list.indexOf(index_name) === -1) {
+            index_list.push(index_name);
+            return gadget.setSetting('index_list', index_list);
+          }
+        });
+    })
     //declare service to build index in background
     .declareService(function () {
       var gadget = this,
@@ -97,6 +107,9 @@
                     current.shift();
                     return gadget.setSetting('build_list', current);
                   }
+                })
+                .push(function () {
+                  return gadget.pushIndex(entry.index_name);
                 });
             }
           })
@@ -296,7 +309,15 @@
         .match(/<body[^>]*>((.|[\n\r])*)<\/body>/gim);
       if (regex) {
         tmp_div = document.createElement("div");
-        tmp_div.innerHTML = regex[0];
+        tmp_div.innerHTML = regex[0]
+          // remove all <script> sections
+          .replace(/(<script[^>]*>[^>]*.*<\/script>)/gm, '');
+        // remove footers, headers
+        tmp_div.querySelectorAll('link, style, script, footer, header')
+          .forEach(function (item) {
+          let parent = item.parentNode;
+          parent.remove(item);
+        });
       }
       if (!tmp_div)
         return;
@@ -306,6 +327,10 @@
         title : title,
         content : tmp_div.textContent || tmp_div.innerText || ""
       };
+      // remove all useless character
+      result.content = result.content
+        .replace(/[\t\r\n]+/g, '')
+        .replace(/ +(?= )/g, '');
       return gadget.add_to_index(result, index_name)
         .push(function () {
           while (tmp_div.firstChild) {

bt5/erp5_web_mynij_search/PathTemplateItem/web_page_module/gadget_mynij_parser.js.xml

@@ -242,7 +242,7 @@
                   </item>
                   <item>
                       <key> <string>serial</string> </key>
-                      <value> <string>987.37780.16896.38656</string> </value>
+                      <value> <string>987.50963.40839.11502</string> </value>
                   </item>
                   <item>
                       <key> <string>state</string> </key>
@@ -260,7 +260,7 @@
                           </tuple>
                           <state>
                             <tuple>
-                              <float>1603817569.05</float>
+                              <float>1604610243.41</float>
                               <string>UTC</string>
                             </tuple>
                           </state>

bt5/erp5_web_mynij_search/PathTemplateItem/web_page_module/gadget_mynij_result.js.js

@@ -208,8 +208,6 @@
           end,
           count = 0,
           i;
-      description = description.replace(/[\t\r\n]+/g, '')
-        .replace(/ +(?= )/g, '');
       if (description.length <= max_length)
         return description;
       regex = new RegExp('(' + search_key + ')', 'i').exec(description);

bt5/erp5_web_mynij_search/PathTemplateItem/web_page_module/gadget_mynij_result.js.xml

@@ -242,7 +242,7 @@
                   </item>
                   <item>
                       <key> <string>serial</string> </key>
-                      <value> <string>987.31604.11529.7168</string> </value>
+                      <value> <string>987.31610.30930.31436</string> </value>
                   </item>
                   <item>
                       <key> <string>state</string> </key>
@@ -260,7 +260,7 @@
                           </tuple>
                           <state>
                             <tuple>
-                              <float>1603447348.31</float>
+                              <float>1604431247.07</float>
                               <string>UTC</string>
                             </tuple>
                           </state>

bt5/erp5_web_mynij_search/bt/template_path_list

@@ -6,5 +6,6 @@ web_page_module/gadget_mynij_**
 web_page_module/gadget_officejs_mynij_**
 web_page_module/mynij-**
 web_page_module/mynij_**
+web_page_module/rss-parser.min.js.map
 web_site_module/mynij_search
 web_site_module/mynij_search/**
\ No newline at end of file