slapgrid: try 3 times to upload archive to binary cache

See merge request nexedi/slapos.core!185

slapos/grid/SlapObject.py

@@ -337,20 +337,24 @@ class Software(object):
       tar.add(self.software_path, arcname=self.software_url_hash)
     finally:
       tar.close()
-    self.logger.info("Trying to upload archive of software release...")
-    upload_network_cached(
-        self.software_root,
-        self.url, self.software_url_hash,
-        self.upload_binary_cache_url,
-        self.upload_binary_dir_url,
-        tarpath, self.logger,
-        self.signature_private_key_file,
-        self.shacache_ca_file,
-        self.shacache_cert_file,
-        self.shacache_key_file,
-        self.shadir_ca_file,
-        self.shadir_cert_file,
-        self.shadir_key_file)
+    for i in range(3):
+      self.logger.info("Trying to upload archive of software release (try %d)..." % (i))
+      if upload_network_cached(
+          self.software_root,
+          self.url, self.software_url_hash,
+          self.upload_binary_cache_url,
+          self.upload_binary_dir_url,
+          tarpath, self.logger,
+          self.signature_private_key_file,
+          self.shacache_ca_file,
+          self.shacache_cert_file,
+          self.shacache_key_file,
+          self.shadir_ca_file,
+          self.shadir_cert_file,
+          self.shadir_key_file):
+        break
+      # let shacache process the upload before retrying
+      time.sleep(1800)
 
   def destroy(self):
     """Removes software release."""