Revert "Remove NetworkCache class in favor of standard NetworkcacheClient class"
This reverts commit abba1a33. We revert temporarily this commit until the slapcache.cfg file has been updated on all machines (remove https from download-cache-url)
Showing
| ... | @@ -40,6 +40,110 @@ from random import choice | ... | @@ -40,6 +40,110 @@ from random import choice |
| from string import ascii_lowercase | from string import ascii_lowercase | ||
| from slapos.libnetworkcache import NetworkcacheClient | from slapos.libnetworkcache import NetworkcacheClient | ||
| from slapos.networkcachehelper import helper_download_network_cached | |||
|
|||
| class NetworkCache: | |||
| def __init__(self, configuration_path): | |||
| if not os.path.exists(configuration_path): | |||
| raise ValueError("You need configuration file") | |||
| self.configuration = configuration_path | |||
| self._load() | |||
| def _load(self): | |||
| network_cache_info = configparser.RawConfigParser() | |||
| network_cache_info.read(self.configuration) | |||
| network_cache_info_dict = dict(network_cache_info.items('networkcache')) | |||
| def get_(name): | |||
| return network_cache_info_dict.get(name) | |||
| self.download_binary_cache_url = get_('download-binary-cache-url') | |||
| self.download_cache_url = get_('download-cache-url') | |||
| self.download_binary_dir_url = get_('download-binary-dir-url') | |||
| self.signature_certificate_list = get_('signature-certificate-list') | |||
| # Not mandatory | |||
| self.dir_url = get_('upload-dir-url') | |||
| self.cache_url = get_('upload-cache-url') | |||
| key_file_key = 'signature-private-key-file' | |||
| self.signature_private_key_file = get_(key_file_key) | |||
| key_file_old_key = 'signature_private_key_file' | |||
| if key_file_old_key in network_cache_info_dict: | |||
| raise ValueError( | |||
| '%s is not supported anymore, use %s' % (key_file_old_key, key_file_key)) | |||
| self.shacache_ca_file = get_('shacache-ca-file') | |||
| self.shacache_cert_file = get_('shacache-cert-file') | |||
| self.shacache_key_file = get_('shacache-key-file') | |||
| self.shadir_cert_file = get_('shadir-cert-file') | |||
| self.shadir_key_file = get_('shadir-key-file') | |||
| self.shadir_ca_file = get_('shadir-ca-file') | |||
| if network_cache_info.has_section('shacache'): | |||
| self.key = network_cache_info.get('shacache', 'key') | |||
| else: | |||
| self.key = "slapos-upgrade-testing-key" | |||
| def upload(self, path, metadata_dict, is_sha256file=False): | |||
| """ | |||
| Upload an existing file, using a file_descriptor. | |||
| """ | |||
| if is_sha256file: | |||
| key = self.key + "-sha256-content" | |||
| else: | |||
| key = self.key | |||
| file_descriptor = open(path, 'rb') | |||
| if not (self.dir_url and self.cache_url): | |||
| raise ValueError("upload-dir-url and/or upload-cache-url is not defined") | |||
| # backward compatibility | |||
| metadata_dict.setdefault('file', 'notused') | |||
| metadata_dict.setdefault('urlmd5', 'notused') | |||
| # convert '' into None in order to call nc nicely | |||
| with NetworkcacheClient(self.cache_url, self.dir_url, | |||
| signature_private_key_file=self.signature_private_key_file or None, | |||
| signature_certificate_list=self.signature_certificate_list or None, | |||
| shacache_ca_file=self.shacache_ca_file or None, | |||
| shacache_cert_file=self.shacache_cert_file or None, | |||
| shacache_key_file=self.shacache_key_file or None, | |||
| shadir_cert_file=self.shadir_cert_file or None, | |||
| shadir_key_file=self.shadir_key_file or None, | |||
| shadir_ca_file=self.shadir_ca_file or None, | |||
| ) as nc: | |||
| return nc.upload(file_descriptor, key, **metadata_dict) | |||
| def download(self, path, wanted_metadata_dict={}, | |||
| required_key_list=[], strategy=None, is_sha256file=False): | |||
| if is_sha256file: | |||
| key = self.key + "-sha256-content" | |||
| else: | |||
| key = self.key | |||
| result = helper_download_network_cached( | |||
| self.download_binary_dir_url, | |||
| self.download_binary_cache_url, | |||
| self.signature_certificate_list, | |||
| key, wanted_metadata_dict, | |||
| required_key_list, strategy) | |||
| if result: | |||
| # XXX check if nc filters signature_certificate_list! | |||
| # Creates a file with content to desired path. | |||
| file_descriptor, metadata_dict = result | |||
| f = open(path, 'w+b') | |||
| try: | |||
| shutil.copyfileobj(file_descriptor, f) | |||
| # XXX method should check MD5. | |||
| return metadata_dict | |||
| finally: | |||
| f.close() | |||
| file_descriptor.close() | |||
| return False | |||
| def strategy(entry_list): | def strategy(entry_list): | ||
| ... | @@ -51,20 +155,13 @@ def strategy(entry_list): | ... | @@ -51,20 +155,13 @@ def strategy(entry_list): |
| best_entry = entry | best_entry = entry | ||
| timestamp = entry['timestamp'] | timestamp = entry['timestamp'] | ||
| return best_entry | return best_entry | ||
| class Signature: | class Signature: | ||
| def __init__(self, config, logger=None): | def __init__(self, config, logger=None): | ||
| self.config = config | self.config = config | ||
| self.logger = logger | self.logger = logger | ||
| self.shacache = NetworkcacheClient(open(self.config.slapos_configuration, 'r')) | |||
| network_cache_info = configparser.RawConfigParser() | |||
| network_cache_info.read(self.config.slapos_configuration) | |||
| if network_cache_info.has_section('shacache'): | |||
| self.key = network_cache_info.get('shacache', 'key') | |||
| else: | |||
| self.key = "slapos-upgrade-testing-key" | |||
| def log(self, message, *args): | def log(self, message, *args): | ||
| if self.logger is not None: | if self.logger is not None: | ||
| ... | @@ -82,66 +179,32 @@ class Signature: | ... | @@ -82,66 +179,32 @@ class Signature: |
| return base | return base | ||
| def save_file_hash(self, path, destination): | def save_file_hash(self, path, destination): | ||
| base = self.get_file_hash(path) | base = self.get_file_hash(path) | ||
| with open(destination, "wb") as f: | with open(destination, "wb") as f: | ||
| f.write(base) | f.write(base) | ||
| def _download_once(self, path, wanted_metadata_dict={}, | |||
| required_key_list=[], is_sha256file=False): | |||
| if is_sha256file: | |||
| key = self.key + "-sha256-content" | |||
| else: | |||
| key = self.key | |||
| self.log('Downloading %s...', key) | |||
| result = self.shacache.select(key, wanted_metadata_dict, required_key_list) | |||
| entry = None | |||
| result = list(result) | |||
| if result: | |||
| entry = strategy(result) | |||
| if not entry: # XXX: this should be the choice of 'strategy' function | |||
| self.log("Can't find best entry matching strategy, selecting " | |||
| "random one between acceptable ones.") | |||
| entry = result[0] | |||
| if not entry: | |||
| self.log('No entry matching key %s', key) | |||
| else: | |||
| # XXX check if nc filters signature_certificate_list! | |||
| # Creates a file with content to desired path. | |||
| f = open(path, 'w+b') | |||
| fd_download = self.shacache.download(entry['sha512']) | |||
| try: | |||
| shutil.copyfileobj(fd_download, f) | |||
| # XXX method should check MD5. | |||
| return entry | |||
| finally: | |||
| f.close() | |||
| fd_download.close() | |||
| return False | |||
| def _download(self, path): | def _download(self, path): | ||
| """ | """ | ||
| Download a tar of the repository from cache, and untar it. | Download a tar of the repository from cache, and untar it. | ||
| """ | """ | ||
| try: | shacache = NetworkCache(self.config.slapos_configuration) | ||
| download_metadata_dict = self._download_once(path=path, | |||
| required_key_list=['timestamp']) | if shacache.signature_certificate_list is None: | ||
| except: | raise ValueError("You need at least one valid signature for download") | ||
| return False | |||
| download_metadata_dict = shacache.download(path=path, | |||
| required_key_list=['timestamp'], strategy=strategy) | |||
| if download_metadata_dict: | if download_metadata_dict: | ||
| self.log('File downloaded in %s', path) | self.log('File downloaded in %s', path) | ||
| current_sha256 = self.get_file_hash(path) | current_sha256 = self.get_file_hash(path) | ||
| with tempfile.NamedTemporaryFile() as f_256: | with tempfile.NamedTemporaryFile() as f_256: | ||
| sha256path = f_256.name | sha256path = f_256.name | ||
| try: | if shacache.download(path=sha256path, required_key_list=['timestamp'], | ||
| sha256sum_present = self._download_once(path=sha256path, required_key_list=['timestamp'], is_sha256file=True) | strategy=strategy, is_sha256file=True): | ||
| self.log('sha 256 downloaded in %s', sha256path) | self.log('sha 256 downloaded in %s', sha256path) | ||
| except: | |||
| sha256sum_present = False | |||
| if sha256sum_present: | |||
| expected_sha256 = f_256.read() | expected_sha256 = f_256.read() | ||
| if current_sha256 == expected_sha256: | if current_sha256 == expected_sha256: | ||
| return True | return True | ||
| else: | else: | ||
| ... | @@ -167,32 +230,29 @@ class Signature: | ... | @@ -167,32 +230,29 @@ class Signature: |
| """ | """ | ||
| Creates uploads repository to cache. | Creates uploads repository to cache. | ||
| """ | """ | ||
| shacache = NetworkCache(self.config.slapos_configuration) | |||
| sha256path = path + ".sha256" | sha256path = path + ".sha256" | ||
| self.save_file_hash(path, sha256path) | self.save_file_hash(path, sha256path) | ||
| metadata_dict = { | metadata_dict = { | ||
| # XXX: we set date from client side. It can be potentially dangerous | # XXX: we set date from client side. It can be potentially dangerous | ||
| # as it can be badly configured. | # as it can be badly configured. | ||
| 'timestamp': time.time(), | 'timestamp': time.time(), | ||
| 'token': ''.join([choice(ascii_lowercase) for _ in range(128)]), | 'token': ''.join([choice(ascii_lowercase) for _ in range(128)]) | ||
| # backward compatibility | |||
| 'file': 'notused', | |||
| 'urlmd5': 'notused' | |||
| } | } | ||
| try: | try: | ||
| sha512sum = self.shacache.upload(open(path, 'rb'), self.key, **metadata_dict) | sha512sum = shacache.upload(path=path, metadata_dict=metadata_dict) | ||
| if sha512sum: | if sha512sum: | ||
| self.log( | self.log( | ||
| 'Uploaded %s to cache (using %s key) with SHA512 %s.', path, | 'Uploaded %s to cache (using %s key) with SHA512 %s.', path, | ||
| self.key, sha512sum) | shacache.key, sha512sum) | ||
| sha512sum_path = self.shacache.upload( | sha512sum_path = shacache.upload( | ||
| open(sha256path, 'rb'), | path=sha256path, metadata_dict=metadata_dict, is_sha256file=True) | ||
| self.key + "-sha256-content", | |||
| **metadata_dict) | |||
| if sha512sum_path: | if sha512sum_path: | ||
| self.log( | self.log( | ||
| 'Uploaded %s to cache (using %s key) with SHA512 %s.', sha256path, | 'Uploaded %s to cache (using %s key) with SHA512 %s.', sha256path, | ||
| self.key, sha512sum_path) | shacache.key, sha512sum_path) | ||
| else: | else: | ||
| self.log('Fail to upload sha256file file to cache.') | self.log('Fail to upload sha256file file to cache.') | ||
| else: | else: | ||
| ... | @@ -203,7 +263,7 @@ class Signature: | ... | @@ -203,7 +263,7 @@ class Signature: |
| def upload(self): | def upload(self): | ||
| self._upload(self.config.file) | self._upload(self.config.file) | ||
| # Class containing all parameters needed for configuration | # Class containing all parameters needed for configuration | ||
| class Config: | class Config: | ||
| def __init__(self, option_dict=None): | def __init__(self, option_dict=None): | ||
| ... | ... | ||
