From 56b0f14ea5451ff4388dc02377b6cbb9aa8f8857 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?C=C3=A9dric=20de=20Saint=20Martin?= <cedric.dsm@tiolive.com>
Date: Fri, 24 Aug 2012 14:48:47 +0200
Subject: [PATCH] Add network cache upload blacklist.

---
 slapos.cfg.example          |  9 +++++++--
 slapos/grid/SlapObject.py   | 21 +++++++++++++++++----
 slapos/grid/networkcache.py |  4 ++--
 slapos/grid/slapgrid.py     | 36 ++++++++++++++++++++++++++----------
 4 files changed, 52 insertions(+), 18 deletions(-)

diff --git a/slapos.cfg.example b/slapos.cfg.example
index dbb349a22..819e7e2ce 100644
--- a/slapos.cfg.example
+++ b/slapos.cfg.example
@@ -160,8 +160,13 @@ signature-certificate-list =
   5pW18Ry5Ie7iFK4cQMerZwWPxBodEbAteYlRsI6kePV7Gf735Y1RpuN8qZ2sYL6e
   x2IMeSwJ82BpdEI5niXxB+iT0HxhmR+XaMI=
   -----END CERTIFICATE-----
-# List of URL(s) which shouldn't be installed from binary cache, separated by
+# List of URL(s) which shouldn't be download from binary cache, separated by
 # commas. Any URL beginning by a blacklisted URL will be blacklisted as well.
-binary-cache-url-blacklist =
+download-from-binary-cache-url-blacklist =
+  http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD
+  http://git.erp5.org/gitweb/slapos.core.git/blob_plain/refs/heads
+# List of URL(s) which shouldn't be upload to binary cache, separated by
+# commas. Any URL beginning by a blacklisted URL will be blacklisted as well.
+upload-to-binary-cache-url-blacklist =
   http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD
   http://git.erp5.org/gitweb/slapos.core.git/blob_plain/refs/heads
diff --git a/slapos/grid/SlapObject.py b/slapos/grid/SlapObject.py
index 51417ad33..96be60a12 100644
--- a/slapos/grid/SlapObject.py
+++ b/slapos/grid/SlapObject.py
@@ -55,7 +55,8 @@ class Software(object):
       shacache_key_file=None, shadir_cert_file=None, shadir_key_file=None,
       download_binary_cache_url=None, upload_binary_cache_url=None,
       download_binary_dir_url=None, upload_binary_dir_url=None,
-      binary_cache_url_blacklist = []):
+      download_from_binary_cache_url_blacklist = [],
+      upload_to_binary_cache_url_blacklist = []):
     """Initialisation of class parameters
     """
     self.url = url
@@ -78,7 +79,10 @@ class Software(object):
     self.upload_binary_cache_url = upload_binary_cache_url
     self.download_binary_dir_url = download_binary_dir_url
     self.upload_binary_dir_url = upload_binary_dir_url
-    self.binary_cache_url_blacklist = binary_cache_url_blacklist
+    self.download_from_binary_cache_url_blacklist = \
+        download_from_binary_cache_url_blacklist
+    self.upload_to_binary_cache_url_blacklist = \
+        upload_to_binary_cache_url_blacklist
 
   def install(self):
     """ Fetches binary cache if possible.
@@ -97,7 +101,7 @@ class Software(object):
             self.software_url_hash,
             tarpath, self.logger,
             self.signature_certificate_list,
-            self.binary_cache_url_blacklist):
+            self.download_from_binary_cache_url_blacklist):
       tar = tarfile.open(tarpath)
       try:
         self.logger.info("Extracting archive of cached software release...")
@@ -106,9 +110,18 @@ class Software(object):
         tar.close()
     else:
       self._install_from_buildout()
+
+      # Upload to binary cache if possible
+      blacklisted = False
+      for url in self.upload_to_binary_cache_url_blacklist:
+        if self.url.startswith(url):
+          blacklisted = True
+          self.logger.debug("Can't download from binary cache: "
+              "Software Release URL is blacklisted.")
       if (self.software_root and self.url and self.software_url_hash \
                              and self.upload_binary_cache_url \
-                             and self.upload_binary_dir_url):
+                             and self.upload_binary_dir_url \
+                             and not blacklisted):
         self.logger.info("Creating archive of software release...")
         tar = tarfile.open(tarpath, "w:gz")
         try:
diff --git a/slapos/grid/networkcache.py b/slapos/grid/networkcache.py
index 1cd08a97a..4458dca6d 100644
--- a/slapos/grid/networkcache.py
+++ b/slapos/grid/networkcache.py
@@ -50,7 +50,7 @@ def fallback_call(function):
 @fallback_call
 def download_network_cached(cache_url, dir_url, software_url, software_root,
                             key, path, logger, signature_certificate_list,
-                            binary_cache_url_blacklist=None):
+                            download_from_binary_cache_url_blacklist=None):
     """Downloads from a network cache provider
 
     return True if download succeeded.
@@ -61,7 +61,7 @@ def download_network_cached(cache_url, dir_url, software_url, software_root,
     if not(cache_url and dir_url and software_url and software_root):
         return False
 
-    for url in binary_cache_url_blacklist:
+    for url in download_from_binary_cache_url_blacklist:
       if software_url.startswith(url):
         return False
 
diff --git a/slapos/grid/slapgrid.py b/slapos/grid/slapgrid.py
index 7e59c5501..a1db9e170 100644
--- a/slapos/grid/slapgrid.py
+++ b/slapos/grid/slapgrid.py
@@ -238,11 +238,18 @@ def parseArgumentTupleAndReturnSlapgridObject(*argument_tuple):
   else:
     signature_certificate_list = None
 
-
-  # Parse cache / binary options
-  option_dict["binary-cache-url-blacklist"] = [
-      url.strip() for url in option_dict.get("binary-cache-url-blacklist", ""
-          ).split('\n') if url]
+  # Parse cache / binary cache options
+  # Backward compatibility about "binary-cache-url-blacklist" deprecated option
+  if option_dict.get("binary-cache-url-blacklist") and not \
+      option_dict.get("download-from-binary-cache-url-blacklist"):
+    option_dict["download-from-binary-cache-url-blacklist"] = \
+        option_dict["binary-cache-url-blacklist"]
+  option_dict["download-from-binary-cache-url-blacklist"] = [
+      url.strip() for url in option_dict.get(
+          "download-from-binary-cache-url-blacklist", "").split('\n') if url]
+  option_dict["upload-to-binary-cache-url-blacklist"] = [
+      url.strip() for url in option_dict.get(
+          "upload-to-binary-cache-url-blacklist", "").split('\n') if url]
 
   # Sleep for a random time to avoid SlapOS Master being DDOSed by an army of
   # SlapOS Nodes configured with cron.
@@ -274,8 +281,10 @@ def parseArgumentTupleAndReturnSlapgridObject(*argument_tuple):
               option_dict.get('download-binary-cache-url', None),
             upload_binary_cache_url=\
               option_dict.get('upload-binary-cache-url', None),
-            binary_cache_url_blacklist=\
-                option_dict.get('binary-cache-url-blacklist', []),
+            download_from_binary_cache_url_blacklist=\
+                option_dict.get('download-from-binary-cache-url-blacklist', []),
+            upload_to_binary_cache_url_blacklist=\
+                option_dict.get('upload-to-binary-cache-url-blacklist', []),
             upload_cache_url=option_dict.get('upload-cache-url', None),
             download_binary_dir_url=\
               option_dict.get('download-binary-dir-url', None),
@@ -370,7 +379,8 @@ class Slapgrid(object):
                signature_certificate_list=None,
                download_binary_cache_url=None,
                upload_binary_cache_url=None,
-               binary_cache_url_blacklist=None,
+               download_from_binary_cache_url_blacklist=None,
+               upload_to_binary_cache_url_blacklist=None,
                upload_cache_url=None,
                download_binary_dir_url=None,
                upload_binary_dir_url=None,
@@ -403,7 +413,10 @@ class Slapgrid(object):
     self.signature_certificate_list = signature_certificate_list
     self.download_binary_cache_url = download_binary_cache_url
     self.upload_binary_cache_url = upload_binary_cache_url
-    self.binary_cache_url_blacklist = binary_cache_url_blacklist
+    self.download_from_binary_cache_url_blacklist = \
+        download_from_binary_cache_url_blacklist
+    self.upload_to_binary_cache_url_blacklist = \
+        upload_to_binary_cache_url_blacklist
     self.upload_cache_url = upload_cache_url
     self.download_binary_dir_url = download_binary_dir_url
     self.upload_binary_dir_url = upload_binary_dir_url
@@ -505,7 +518,10 @@ class Slapgrid(object):
             signature_certificate_list=self.signature_certificate_list,
             download_binary_cache_url=self.download_binary_cache_url,
             upload_binary_cache_url=self.upload_binary_cache_url,
-            binary_cache_url_blacklist=self.binary_cache_url_blacklist,
+            download_from_binary_cache_url_blacklist=\
+                self.download_from_binary_cache_url_blacklist,
+            upload_to_binary_cache_url_blacklist=\
+                self.upload_to_binary_cache_url_blacklist,
             upload_cache_url=self.upload_cache_url,
             download_binary_dir_url=self.download_binary_dir_url,
             upload_binary_dir_url=self.upload_binary_dir_url,
-- 
2.30.9