Add network cache upload blacklist.

slapos.cfg.example

@@ -160,8 +160,13 @@ signature-certificate-list =
   5pW18Ry5Ie7iFK4cQMerZwWPxBodEbAteYlRsI6kePV7Gf735Y1RpuN8qZ2sYL6e
   x2IMeSwJ82BpdEI5niXxB+iT0HxhmR+XaMI=
   -----END CERTIFICATE-----
-# List of URL(s) which shouldn't be installed from binary cache, separated by
+# List of URL(s) which shouldn't be download from binary cache, separated by
 # commas. Any URL beginning by a blacklisted URL will be blacklisted as well.
-binary-cache-url-blacklist =
+download-from-binary-cache-url-blacklist =
+  http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD
+  http://git.erp5.org/gitweb/slapos.core.git/blob_plain/refs/heads
+# List of URL(s) which shouldn't be upload to binary cache, separated by
+# commas. Any URL beginning by a blacklisted URL will be blacklisted as well.
+upload-to-binary-cache-url-blacklist =
   http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD
   http://git.erp5.org/gitweb/slapos.core.git/blob_plain/refs/heads

slapos/grid/SlapObject.py

@@ -55,7 +55,8 @@ class Software(object):
       shacache_key_file=None, shadir_cert_file=None, shadir_key_file=None,
       download_binary_cache_url=None, upload_binary_cache_url=None,
       download_binary_dir_url=None, upload_binary_dir_url=None,
-      binary_cache_url_blacklist = []):
+      download_from_binary_cache_url_blacklist = [],
+      upload_to_binary_cache_url_blacklist = []):
     """Initialisation of class parameters
     """
     self.url = url
@@ -78,7 +79,10 @@ class Software(object):
     self.upload_binary_cache_url = upload_binary_cache_url
     self.download_binary_dir_url = download_binary_dir_url
     self.upload_binary_dir_url = upload_binary_dir_url
-    self.binary_cache_url_blacklist = binary_cache_url_blacklist
+    self.download_from_binary_cache_url_blacklist = \
+        download_from_binary_cache_url_blacklist
+    self.upload_to_binary_cache_url_blacklist = \
+        upload_to_binary_cache_url_blacklist
 
   def install(self):
     """ Fetches binary cache if possible.
@@ -97,7 +101,7 @@ class Software(object):
             self.software_url_hash,
             tarpath, self.logger,
             self.signature_certificate_list,
-            self.binary_cache_url_blacklist):
+            self.download_from_binary_cache_url_blacklist):
       tar = tarfile.open(tarpath)
       try:
         self.logger.info("Extracting archive of cached software release...")
@@ -106,9 +110,18 @@ class Software(object):
         tar.close()
     else:
       self._install_from_buildout()
+
+      # Upload to binary cache if possible
+      blacklisted = False
+      for url in self.upload_to_binary_cache_url_blacklist:
+        if self.url.startswith(url):
+          blacklisted = True
+          self.logger.debug("Can't download from binary cache: "
+              "Software Release URL is blacklisted.")
       if (self.software_root and self.url and self.software_url_hash \
                              and self.upload_binary_cache_url \
-                             and self.upload_binary_dir_url):
+                             and self.upload_binary_dir_url \
+                             and not blacklisted):
         self.logger.info("Creating archive of software release...")
         tar = tarfile.open(tarpath, "w:gz")
         try:

slapos/grid/networkcache.py

@@ -50,7 +50,7 @@ def fallback_call(function):
 @fallback_call
 def download_network_cached(cache_url, dir_url, software_url, software_root,
                             key, path, logger, signature_certificate_list,
-                            binary_cache_url_blacklist=None):
+                            download_from_binary_cache_url_blacklist=None):
     """Downloads from a network cache provider
 
     return True if download succeeded.
@@ -61,7 +61,7 @@ def download_network_cached(cache_url, dir_url, software_url, software_root,
     if not(cache_url and dir_url and software_url and software_root):
         return False
 
-    for url in binary_cache_url_blacklist:
+    for url in download_from_binary_cache_url_blacklist:
       if software_url.startswith(url):
         return False
 

slapos/grid/slapgrid.py

@@ -238,11 +238,18 @@ def parseArgumentTupleAndReturnSlapgridObject(*argument_tuple):
   else:
     signature_certificate_list = None
 
-
-  # Parse cache / binary options
-  option_dict["binary-cache-url-blacklist"] = [
-      url.strip() for url in option_dict.get("binary-cache-url-blacklist", ""
-          ).split('\n') if url]
+  # Parse cache / binary cache options
+  # Backward compatibility about "binary-cache-url-blacklist" deprecated option
+  if option_dict.get("binary-cache-url-blacklist") and not \
+      option_dict.get("download-from-binary-cache-url-blacklist"):
+    option_dict["download-from-binary-cache-url-blacklist"] = \
+        option_dict["binary-cache-url-blacklist"]
+  option_dict["download-from-binary-cache-url-blacklist"] = [
+      url.strip() for url in option_dict.get(
+          "download-from-binary-cache-url-blacklist", "").split('\n') if url]
+  option_dict["upload-to-binary-cache-url-blacklist"] = [
+      url.strip() for url in option_dict.get(
+          "upload-to-binary-cache-url-blacklist", "").split('\n') if url]
 
   # Sleep for a random time to avoid SlapOS Master being DDOSed by an army of
   # SlapOS Nodes configured with cron.
@@ -274,8 +281,10 @@ def parseArgumentTupleAndReturnSlapgridObject(*argument_tuple):
               option_dict.get('download-binary-cache-url', None),
             upload_binary_cache_url=\
               option_dict.get('upload-binary-cache-url', None),
-            binary_cache_url_blacklist=\
-                option_dict.get('binary-cache-url-blacklist', []),
+            download_from_binary_cache_url_blacklist=\
+                option_dict.get('download-from-binary-cache-url-blacklist', []),
+            upload_to_binary_cache_url_blacklist=\
+                option_dict.get('upload-to-binary-cache-url-blacklist', []),
             upload_cache_url=option_dict.get('upload-cache-url', None),
             download_binary_dir_url=\
               option_dict.get('download-binary-dir-url', None),
@@ -370,7 +379,8 @@ class Slapgrid(object):
                signature_certificate_list=None,
                download_binary_cache_url=None,
                upload_binary_cache_url=None,
-               binary_cache_url_blacklist=None,
+               download_from_binary_cache_url_blacklist=None,
+               upload_to_binary_cache_url_blacklist=None,
                upload_cache_url=None,
                download_binary_dir_url=None,
                upload_binary_dir_url=None,
@@ -403,7 +413,10 @@ class Slapgrid(object):
     self.signature_certificate_list = signature_certificate_list
     self.download_binary_cache_url = download_binary_cache_url
     self.upload_binary_cache_url = upload_binary_cache_url
-    self.binary_cache_url_blacklist = binary_cache_url_blacklist
+    self.download_from_binary_cache_url_blacklist = \
+        download_from_binary_cache_url_blacklist
+    self.upload_to_binary_cache_url_blacklist = \
+        upload_to_binary_cache_url_blacklist
     self.upload_cache_url = upload_cache_url
     self.download_binary_dir_url = download_binary_dir_url
     self.upload_binary_dir_url = upload_binary_dir_url
@@ -505,7 +518,10 @@ class Slapgrid(object):
             signature_certificate_list=self.signature_certificate_list,
             download_binary_cache_url=self.download_binary_cache_url,
             upload_binary_cache_url=self.upload_binary_cache_url,
-            binary_cache_url_blacklist=self.binary_cache_url_blacklist,
+            download_from_binary_cache_url_blacklist=\
+                self.download_from_binary_cache_url_blacklist,
+            upload_to_binary_cache_url_blacklist=\
+                self.upload_to_binary_cache_url_blacklist,
             upload_cache_url=self.upload_cache_url,
             download_binary_dir_url=self.download_binary_dir_url,
             upload_binary_dir_url=self.upload_binary_dir_url,