From 56b0f14ea5451ff4388dc02377b6cbb9aa8f8857 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20de=20Saint=20Martin?= <cedric.dsm@tiolive.com> Date: Fri, 24 Aug 2012 14:48:47 +0200 Subject: [PATCH] Add network cache upload blacklist. --- slapos.cfg.example | 9 +++++++-- slapos/grid/SlapObject.py | 21 +++++++++++++++++---- slapos/grid/networkcache.py | 4 ++-- slapos/grid/slapgrid.py | 36 ++++++++++++++++++++++++++---------- 4 files changed, 52 insertions(+), 18 deletions(-) diff --git a/slapos.cfg.example b/slapos.cfg.example index dbb349a22..819e7e2ce 100644 --- a/slapos.cfg.example +++ b/slapos.cfg.example @@ -160,8 +160,13 @@ signature-certificate-list = 5pW18Ry5Ie7iFK4cQMerZwWPxBodEbAteYlRsI6kePV7Gf735Y1RpuN8qZ2sYL6e x2IMeSwJ82BpdEI5niXxB+iT0HxhmR+XaMI= -----END CERTIFICATE----- -# List of URL(s) which shouldn't be installed from binary cache, separated by +# List of URL(s) which shouldn't be download from binary cache, separated by # commas. Any URL beginning by a blacklisted URL will be blacklisted as well. -binary-cache-url-blacklist = +download-from-binary-cache-url-blacklist = + http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD + http://git.erp5.org/gitweb/slapos.core.git/blob_plain/refs/heads +# List of URL(s) which shouldn't be upload to binary cache, separated by +# commas. Any URL beginning by a blacklisted URL will be blacklisted as well. +upload-to-binary-cache-url-blacklist = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD http://git.erp5.org/gitweb/slapos.core.git/blob_plain/refs/heads diff --git a/slapos/grid/SlapObject.py b/slapos/grid/SlapObject.py index 51417ad33..96be60a12 100644 --- a/slapos/grid/SlapObject.py +++ b/slapos/grid/SlapObject.py @@ -55,7 +55,8 @@ class Software(object): shacache_key_file=None, shadir_cert_file=None, shadir_key_file=None, download_binary_cache_url=None, upload_binary_cache_url=None, download_binary_dir_url=None, upload_binary_dir_url=None, - binary_cache_url_blacklist = []): + download_from_binary_cache_url_blacklist = [], + upload_to_binary_cache_url_blacklist = []): """Initialisation of class parameters """ self.url = url @@ -78,7 +79,10 @@ class Software(object): self.upload_binary_cache_url = upload_binary_cache_url self.download_binary_dir_url = download_binary_dir_url self.upload_binary_dir_url = upload_binary_dir_url - self.binary_cache_url_blacklist = binary_cache_url_blacklist + self.download_from_binary_cache_url_blacklist = \ + download_from_binary_cache_url_blacklist + self.upload_to_binary_cache_url_blacklist = \ + upload_to_binary_cache_url_blacklist def install(self): """ Fetches binary cache if possible. @@ -97,7 +101,7 @@ class Software(object): self.software_url_hash, tarpath, self.logger, self.signature_certificate_list, - self.binary_cache_url_blacklist): + self.download_from_binary_cache_url_blacklist): tar = tarfile.open(tarpath) try: self.logger.info("Extracting archive of cached software release...") @@ -106,9 +110,18 @@ class Software(object): tar.close() else: self._install_from_buildout() + + # Upload to binary cache if possible + blacklisted = False + for url in self.upload_to_binary_cache_url_blacklist: + if self.url.startswith(url): + blacklisted = True + self.logger.debug("Can't download from binary cache: " + "Software Release URL is blacklisted.") if (self.software_root and self.url and self.software_url_hash \ and self.upload_binary_cache_url \ - and self.upload_binary_dir_url): + and self.upload_binary_dir_url \ + and not blacklisted): self.logger.info("Creating archive of software release...") tar = tarfile.open(tarpath, "w:gz") try: diff --git a/slapos/grid/networkcache.py b/slapos/grid/networkcache.py index 1cd08a97a..4458dca6d 100644 --- a/slapos/grid/networkcache.py +++ b/slapos/grid/networkcache.py @@ -50,7 +50,7 @@ def fallback_call(function): @fallback_call def download_network_cached(cache_url, dir_url, software_url, software_root, key, path, logger, signature_certificate_list, - binary_cache_url_blacklist=None): + download_from_binary_cache_url_blacklist=None): """Downloads from a network cache provider return True if download succeeded. @@ -61,7 +61,7 @@ def download_network_cached(cache_url, dir_url, software_url, software_root, if not(cache_url and dir_url and software_url and software_root): return False - for url in binary_cache_url_blacklist: + for url in download_from_binary_cache_url_blacklist: if software_url.startswith(url): return False diff --git a/slapos/grid/slapgrid.py b/slapos/grid/slapgrid.py index 7e59c5501..a1db9e170 100644 --- a/slapos/grid/slapgrid.py +++ b/slapos/grid/slapgrid.py @@ -238,11 +238,18 @@ def parseArgumentTupleAndReturnSlapgridObject(*argument_tuple): else: signature_certificate_list = None - - # Parse cache / binary options - option_dict["binary-cache-url-blacklist"] = [ - url.strip() for url in option_dict.get("binary-cache-url-blacklist", "" - ).split('\n') if url] + # Parse cache / binary cache options + # Backward compatibility about "binary-cache-url-blacklist" deprecated option + if option_dict.get("binary-cache-url-blacklist") and not \ + option_dict.get("download-from-binary-cache-url-blacklist"): + option_dict["download-from-binary-cache-url-blacklist"] = \ + option_dict["binary-cache-url-blacklist"] + option_dict["download-from-binary-cache-url-blacklist"] = [ + url.strip() for url in option_dict.get( + "download-from-binary-cache-url-blacklist", "").split('\n') if url] + option_dict["upload-to-binary-cache-url-blacklist"] = [ + url.strip() for url in option_dict.get( + "upload-to-binary-cache-url-blacklist", "").split('\n') if url] # Sleep for a random time to avoid SlapOS Master being DDOSed by an army of # SlapOS Nodes configured with cron. @@ -274,8 +281,10 @@ def parseArgumentTupleAndReturnSlapgridObject(*argument_tuple): option_dict.get('download-binary-cache-url', None), upload_binary_cache_url=\ option_dict.get('upload-binary-cache-url', None), - binary_cache_url_blacklist=\ - option_dict.get('binary-cache-url-blacklist', []), + download_from_binary_cache_url_blacklist=\ + option_dict.get('download-from-binary-cache-url-blacklist', []), + upload_to_binary_cache_url_blacklist=\ + option_dict.get('upload-to-binary-cache-url-blacklist', []), upload_cache_url=option_dict.get('upload-cache-url', None), download_binary_dir_url=\ option_dict.get('download-binary-dir-url', None), @@ -370,7 +379,8 @@ class Slapgrid(object): signature_certificate_list=None, download_binary_cache_url=None, upload_binary_cache_url=None, - binary_cache_url_blacklist=None, + download_from_binary_cache_url_blacklist=None, + upload_to_binary_cache_url_blacklist=None, upload_cache_url=None, download_binary_dir_url=None, upload_binary_dir_url=None, @@ -403,7 +413,10 @@ class Slapgrid(object): self.signature_certificate_list = signature_certificate_list self.download_binary_cache_url = download_binary_cache_url self.upload_binary_cache_url = upload_binary_cache_url - self.binary_cache_url_blacklist = binary_cache_url_blacklist + self.download_from_binary_cache_url_blacklist = \ + download_from_binary_cache_url_blacklist + self.upload_to_binary_cache_url_blacklist = \ + upload_to_binary_cache_url_blacklist self.upload_cache_url = upload_cache_url self.download_binary_dir_url = download_binary_dir_url self.upload_binary_dir_url = upload_binary_dir_url @@ -505,7 +518,10 @@ class Slapgrid(object): signature_certificate_list=self.signature_certificate_list, download_binary_cache_url=self.download_binary_cache_url, upload_binary_cache_url=self.upload_binary_cache_url, - binary_cache_url_blacklist=self.binary_cache_url_blacklist, + download_from_binary_cache_url_blacklist=\ + self.download_from_binary_cache_url_blacklist, + upload_to_binary_cache_url_blacklist=\ + self.upload_to_binary_cache_url_blacklist, upload_cache_url=self.upload_cache_url, download_binary_dir_url=self.download_binary_dir_url, upload_binary_dir_url=self.upload_binary_dir_url, -- 2.30.9