grid: Implement forcing binary cache

By using option networkcache:download-from-binary-cache-force-url-list
it is possible now to force some URLs to be only downloaded from binary
cache.

Thanks to this some servers can be configured to never try to compile
selected softwares by themselves, and instead relay on binary cache.

slapos.cfg.example

@@ -162,3 +162,7 @@ upload-to-binary-cache-url-blacklist =
   http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD
   http://git.erp5.org/gitweb/slapos.git/blob_plain/refs/heads
   /
+
+# List of URL(s) which are forced to be downloaded from binary cache
+#download-from-binary-cache-force-url-list =
+#  https://lab.node.vifib.com/nexedi/slapos/raw/1.0.

slapos/grid/SlapObject.py

@@ -112,6 +112,7 @@ class Software(object):
                download_binary_dir_url=None, upload_binary_dir_url=None,
                download_from_binary_cache_url_blacklist=None,
                upload_to_binary_cache_url_blacklist=None,
+               download_from_binary_cache_force_url_list=None,
                software_min_free_space=None,
                buildout_debug=False,
                shared_part_list=''):
@@ -124,6 +125,9 @@ class Software(object):
     if upload_to_binary_cache_url_blacklist is None:
       upload_to_binary_cache_url_blacklist = []
 
+    if download_from_binary_cache_force_url_list is None:
+      download_from_binary_cache_force_url_list = []
+
     self.url = url
     self.software_root = software_root
     self.software_url_hash = md5digest(self.url)
@@ -151,6 +155,8 @@ class Software(object):
         download_from_binary_cache_url_blacklist
     self.upload_to_binary_cache_url_blacklist = \
         upload_to_binary_cache_url_blacklist
+    self.download_from_binary_cache_force_url_list = \
+        download_from_binary_cache_force_url_list
     self.software_min_free_space = software_min_free_space
 
   def check_free_space(self):
@@ -175,6 +181,12 @@ class Software(object):
     try:
       tarpath = os.path.join(cache_dir, self.software_url_hash)
       # Check if we can download from cache
+      force_binary_cache = False
+      for url_match in self.download_from_binary_cache_force_url_list:
+        if self.url.startswith(url_match):
+          force_binary_cache = True
+          self.logger.debug('Binary cache forced for %r because of match %r', self.url, url_match)
+          break
       if (not os.path.exists(self.software_path)) \
           and download_network_cached(
               self.download_binary_cache_url,
@@ -190,6 +202,10 @@ class Software(object):
           tar.extractall(path=self.software_root)
         finally:
           tar.close()
+      elif force_binary_cache:
+        message = 'Binary cache forced for %r, but failed to download, will retry again' % (self.url,)
+        self.logger.error(message)
+        raise BuildoutFailedError(message)
       else:
         self._install_from_buildout()
         # Upload to binary cache if possible and allowed

slapos/grid/slapgrid.py

@@ -188,6 +188,9 @@ def merged_options(args, configp):
   options["upload-to-binary-cache-url-blacklist"] = [
       url.strip() for url in options.get(
           "upload-to-binary-cache-url-blacklist", "").split('\n') if url]
+  options["download-from-binary-cache-force-url-list"] = [
+      url.strip() for url in options.get(
+          "download-from-binary-cache-force-url-list", "").split('\n') if url]
 
   options['firewall'] = {}
   if configp.has_section('firewall'):
@@ -259,6 +262,8 @@ def create_slapgrid_object(options, logger):
                       op.get('download-from-binary-cache-url-blacklist', []),
                   upload_to_binary_cache_url_blacklist=
                       op.get('upload-to-binary-cache-url-blacklist', []),
+                  download_from_binary_cache_force_url_list=
+                      op.get('download-from-binary-cache-force-url-list', []),
                   upload_cache_url=op.get('upload-cache-url'),
                   download_binary_dir_url=op.get('download-binary-dir-url'),
                   upload_binary_dir_url=op.get('upload-binary-dir-url'),
@@ -317,6 +322,7 @@ class Slapgrid(object):
                upload_binary_cache_url=None,
                download_from_binary_cache_url_blacklist=None,
                upload_to_binary_cache_url_blacklist=None,
+               download_from_binary_cache_force_url_list=None,
                upload_cache_url=None,
                download_binary_dir_url=None,
                upload_binary_dir_url=None,
@@ -362,6 +368,8 @@ class Slapgrid(object):
         download_from_binary_cache_url_blacklist
     self.upload_to_binary_cache_url_blacklist = \
         upload_to_binary_cache_url_blacklist
+    self.download_from_binary_cache_force_url_list = \
+        download_from_binary_cache_force_url_list
     self.upload_cache_url = upload_cache_url
     self.download_binary_dir_url = download_binary_dir_url
     self.upload_binary_dir_url = upload_binary_dir_url
@@ -555,6 +563,8 @@ stderr_logfile_backups=1
                 self.download_from_binary_cache_url_blacklist,
             upload_to_binary_cache_url_blacklist=
                 self.upload_to_binary_cache_url_blacklist,
+            download_from_binary_cache_force_url_list=
+                self.download_from_binary_cache_force_url_list,
             upload_cache_url=self.upload_cache_url,
             download_binary_dir_url=self.download_binary_dir_url,
             upload_binary_dir_url=self.upload_binary_dir_url,