erp5_web_mynij_search: Prevent failing on all sitemap urls if there is a problem on network

bt5/erp5_web_mynij_search/PathTemplateItem/web_page_module/gadget_mynij_parser.js.js

@@ -22,7 +22,7 @@
     }
 
     function showError(error_message) {
-      console.log(error_message);
+      console.error(error_message);
     }
 
     gadget_klass
@@ -36,7 +36,8 @@
       return this.getDeclaredGadget("model")
       .push(function (model) {
         return gadget.changeState({
-          model_gadget: model
+          model_gadget: model,
+          failurelimit: 10
         });
       });
     })
@@ -96,6 +97,7 @@
               return RSVP.delay(5000);
             else {
               gadget.state.current_build = entry;
+              gadget.state.failedcount = 0;
               link_list = entry.links.split("\n").filter(l => l.length > 0);
               return gadget.build_index(link_list, entry.index_name)
                 .push(function (build_failed) {
@@ -224,7 +226,7 @@
             return gadget.get_and_handle(link, name);
           })
           .push(undefined, function (error) {
-            console.log(error);
+            console.error(error);
             failed = true;
           });
       }
@@ -233,7 +235,8 @@
       }
       return build_queue
         .push(function () {
-          return failed;
+          return failed ||
+            gadget.state.failedcount >= gadget.state.failurelimit;
         });
     })
 
@@ -356,6 +359,10 @@
     .declareMethod("get_sitemap_item", function (link, index_name, attempt) {
       var gadget = this;
 
+      if (gadget.state.failedcount >= gadget.state.failurelimit) {
+        // too much successive failure, prevent to loop
+        return;
+      }
       return new RSVP.Queue()
         .push(function () {
           var rng = Math.floor(Math.random() * Math.floor(gadget.state.proxies.length)),
@@ -363,21 +370,18 @@
           return jIO.util.ajax({url : proxy + "?url=" + link});
         })
         .push(function (page) {
+          // reset failed counter
+          gadget.state.failedcount = 0;
           if (page !== undefined) {
             return gadget.parse_sitemap_item(page, index_name);
           }
           else
             return null;
         })
-        .push(undefined, function (my_error) {
-          if (!my_error.target) {
-            showError(my_error);
-          }  else if (my_error.target.status === 500 && attempt <= 3) {
-            return gadget.get_sitemap_item(link, index_name, attempt + 1);
-            console.log("Retry " + (attempt + 1) + " on url: " + link);
-          }
-          else
-            showError(my_error);
+        .push(undefined, function (error) {
+          gadget.state.failedcount += 1;
+          showError("Failed to get url: " + link);
+          showError(error);
         });
     })
 

bt5/erp5_web_mynij_search/PathTemplateItem/web_page_module/gadget_mynij_parser.js.xml

@@ -242,7 +242,7 @@
                   </item>
                   <item>
                       <key> <string>serial</string> </key>
-                      <value> <string>994.61246.30369.5734</string> </value>
+                      <value> <string>994.62669.7856.4232</string> </value>
                   </item>
                   <item>
                       <key> <string>state</string> </key>
@@ -260,7 +260,7 @@
                           </tuple>
                           <state>
                             <tuple>
-                              <float>1632233029.49</float>
+                              <float>1632319215.7</float>
                               <string>UTC</string>
                             </tuple>
                           </state>