Commit 89f2e14d authored by Julien Muchembled's avatar Julien Muchembled

Provide a simple way to use network cache safely

To simplify things, DirectoryNotFound & UploadError are merged in
NetworkcacheException.
parent 439a4a53
...@@ -93,6 +93,21 @@ class NetworkcacheClient(object): ...@@ -93,6 +93,21 @@ class NetworkcacheClient(object):
crypto.load_certificate(crypto.FILETYPE_PEM, certificate) crypto.load_certificate(crypto.FILETYPE_PEM, certificate)
for certificate in signature_certificate_list or ()] for certificate in signature_certificate_list or ()]
# NetworkcacheClient context manager catches all exceptions and logs them
# with INFO severity. This provides a easy way to use a networkcache safely
# since most of the time, failures are not fatal.
def __enter__(self):
return self
def __exit__(self, t, v, tb):
if isinstance(v, Exception):
if isinstance(v, NetworkcacheException):
logger.info(*v.args)
else:
logger.info("ignored unhandled exception", exc_info=(t, v, tb))
return True
def _request(self, where, name=None, data=None, headers=None): def _request(self, where, name=None, data=None, headers=None):
if data is None: if data is None:
method = 'GET' method = 'GET'
...@@ -189,9 +204,9 @@ class NetworkcacheClient(object): ...@@ -189,9 +204,9 @@ class NetworkcacheClient(object):
'Content-Type': 'application/octet-stream'}) 'Content-Type': 'application/octet-stream'})
data = result.read() data = result.read()
if result.status != 201 or data != sha512sum: if result.status != 201 or data != sha512sum:
raise UploadError('Failed to upload the file to SHACACHE Server.' raise NetworkcacheException(
'Response code: %s. Response data: %s' 'Failed to upload data to SHACACHE Server.'
% (result.status, data)) ' Response code: %s. Response data: %s' % (result.status, data))
finally: finally:
f is None or f.close() f is None or f.close()
...@@ -199,8 +214,8 @@ class NetworkcacheClient(object): ...@@ -199,8 +214,8 @@ class NetworkcacheClient(object):
kw['sha512'] = sha512sum # always update sha512sum kw['sha512'] = sha512sum # always update sha512sum
file_name = kw.pop('file', file_name) file_name = kw.pop('file', file_name)
if file_name is None or urlmd5 is None: if file_name is None or urlmd5 is None:
raise ValueError('file_name and urlmd5 are required' raise NetworkcacheException(
' for non-generic upload') 'file_name and urlmd5 are required for non-generic upload')
if valid_until is not None: if valid_until is not None:
kw['valid-until'] = valid_until kw['valid-until'] = valid_until
if architecture is not None: if architecture is not None:
...@@ -217,9 +232,9 @@ class NetworkcacheClient(object): ...@@ -217,9 +232,9 @@ class NetworkcacheClient(object):
result = self._request('dir', key, json.dumps(data), { result = self._request('dir', key, json.dumps(data), {
'Content-Type': 'application/json'}) 'Content-Type': 'application/json'})
if result.status != 201: if result.status != 201:
raise UploadError('Failed to upload data to SHADIR Server.' raise NetworkcacheException('Failed to upload data to SHADIR Server.'
'Response code: %s. Response data: %s' ' Response code: %s. Response data: %s'
% (status, result.read())) % (status, result.read()))
def download(self, sha512sum): def download(self, sha512sum):
''' Download the file. ''' Download the file.
...@@ -231,16 +246,13 @@ class NetworkcacheClient(object): ...@@ -231,16 +246,13 @@ class NetworkcacheClient(object):
'''Return an iterator over shadir entries that match given criteria '''Return an iterator over shadir entries that match given criteria
''' '''
required_key_test = frozenset(required_key_list).issubset required_key_test = frozenset(required_key_list).issubset
try: data_list = self.select_generic(key, self.signature_certificate_list)
data_list = self.select_generic(key, self.signature_certificate_list)
except (urllib2.HTTPError, DirectoryNotFound), e:
logger.info(str(e))
return
for information_json, signature in data_list: for information_json, signature in data_list:
try: try:
information_dict = json.loads(information_json) information_dict = json.loads(information_json)
except Exception: except Exception:
logger.info('Failed to parse json-in-json response', exc_info=1) logger.info('Failed to parse json-in-json response (%r)',
information_json)
continue continue
try: try:
len(information_dict['sha512']) len(information_dict['sha512'])
...@@ -262,8 +274,7 @@ class NetworkcacheClient(object): ...@@ -262,8 +274,7 @@ class NetworkcacheClient(object):
try: try:
data_list = json.loads(data) data_list = json.loads(data)
except Exception: except Exception:
raise DirectoryNotFound('Failed to parse json response:\n%s' raise NetworkcacheException('Failed to parse json response (%r)' % data)
% traceback.format_exc())
if filter: if filter:
return (data for data in data_list return (data for data in data_list
if self._verifySignatureInCertificateList(*data)) if self._verifySignatureInCertificateList(*data))
...@@ -292,12 +303,10 @@ class NetworkcacheClient(object): ...@@ -292,12 +303,10 @@ class NetworkcacheClient(object):
return False return False
class DirectoryNotFound(Exception): class NetworkcacheException(Exception):
pass pass
DirectoryNotFound = UploadError = NetworkcacheException # BBB
class UploadError(Exception):
pass
def _newArgumentParser(): def _newArgumentParser():
......
...@@ -193,6 +193,9 @@ class OnlineMixin: ...@@ -193,6 +193,9 @@ class OnlineMixin:
slapos.libnetworkcache.logger.removeHandler(self.handler) slapos.libnetworkcache.logger.removeHandler(self.handler)
del self.handler del self.handler
def assertRaised(self, exc_type):
self.assertEqual(exc_type, self.handler.buffer.pop(0).exc_info[0])
def assertLog(self, msg=None): def assertLog(self, msg=None):
try: try:
self.assertTrue(self.handler.buffer.pop(0).message.startswith(msg)) self.assertTrue(self.handler.buffer.pop(0).message.startswith(msg))
...@@ -365,15 +368,16 @@ class OnlineTest(OnlineMixin, unittest.TestCase): ...@@ -365,15 +368,16 @@ class OnlineTest(OnlineMixin, unittest.TestCase):
urlmd5 = str(random.random()) urlmd5 = str(random.random())
key = 'somekey' + str(random.random()) key = 'somekey' + str(random.random())
nc.upload(self.test_data, key, urlmd5=urlmd5, file_name='my file') nc.upload(self.test_data, key, urlmd5=urlmd5, file_name='my file')
self.select(nc, 'key_another_key' + str(random.random()), with nc:
'HTTP Error 404: Not Found') nc.select('key_another_key' + str(random.random())).next()
self.assertRaised(urllib2.HTTPError)
def test_upload_shadir_no_filename(self): def test_upload_shadir_no_filename(self):
"""Check scenario with shadir used, but not filename passed""" """Check scenario with shadir used, but not filename passed"""
nc = slapos.libnetworkcache.NetworkcacheClient(self.shacache, self.shadir) with slapos.libnetworkcache.NetworkcacheClient(self.shacache,
urlmd5 = str(random.random()) self.shadir) as nc:
self.assertRaises(ValueError, nc.upload, self.test_data, 'somekey', nc.upload(self.test_data, 'somekey', str(random.random()))
urlmd5) self.assertLog('file_name and urlmd5 are required for non-generic upload')
def test_upload_twice_same(self): def test_upload_twice_same(self):
nc = slapos.libnetworkcache.NetworkcacheClient(self.shacache, self.shadir) nc = slapos.libnetworkcache.NetworkcacheClient(self.shacache, self.shadir)
...@@ -467,7 +471,9 @@ class OnlineTest(OnlineMixin, unittest.TestCase): ...@@ -467,7 +471,9 @@ class OnlineTest(OnlineMixin, unittest.TestCase):
with open(os.path.join(self.tree, 'shadir', key), 'w') as f: with open(os.path.join(self.tree, 'shadir', key), 'w') as f:
# now remove the entry from shacache # now remove the entry from shacache
f.write('This is not a json.') f.write('This is not a json.')
self.select(nc, key, 'Failed to parse json response') with nc:
nc.select(key).next()
self.assertLog('Failed to parse json response')
def test_select_json_no_in_json_response(self): def test_select_json_no_in_json_response(self):
key = 'somekey' + str(random.random()) key = 'somekey' + str(random.random())
...@@ -602,7 +608,7 @@ class OnlineTestPOST200(OnlineMixin, unittest.TestCase): ...@@ -602,7 +608,7 @@ class OnlineTestPOST200(OnlineMixin, unittest.TestCase):
def test_upload_wrong_return_code(self): def test_upload_wrong_return_code(self):
"""Check reaction on HTTP return code different then 201""" """Check reaction on HTTP return code different then 201"""
nc = slapos.libnetworkcache.NetworkcacheClient(self.shacache, self.shadir) nc = slapos.libnetworkcache.NetworkcacheClient(self.shacache, self.shadir)
self.assertRaises(slapos.libnetworkcache.UploadError, nc.upload, self.assertRaises(slapos.libnetworkcache.NetworkcacheException, nc.upload,
self.test_data) self.test_data)
...@@ -612,7 +618,7 @@ class OnlineTestWrongChecksum(OnlineMixin, unittest.TestCase): ...@@ -612,7 +618,7 @@ class OnlineTestWrongChecksum(OnlineMixin, unittest.TestCase):
def test_upload_wrong_return_sha(self): def test_upload_wrong_return_sha(self):
"""Check reaction in case of wrong sha returned""" """Check reaction in case of wrong sha returned"""
nc = slapos.libnetworkcache.NetworkcacheClient(self.shacache, self.shadir) nc = slapos.libnetworkcache.NetworkcacheClient(self.shacache, self.shadir)
self.assertRaises(slapos.libnetworkcache.UploadError, nc.upload, self.assertRaises(slapos.libnetworkcache.NetworkcacheException, nc.upload,
self.test_data) self.test_data)
......
...@@ -17,9 +17,7 @@ ...@@ -17,9 +17,7 @@
import os import os
import shutil import shutil
import urllib2 from slapos.libnetworkcache import NetworkcacheClient, logger
from slapos.libnetworkcache import NetworkcacheClient, UploadError, \
DirectoryNotFound, logger
def __upload_network_cached(dir_url, cache_url, def __upload_network_cached(dir_url, cache_url,
file_descriptor, directory_key, file_descriptor, directory_key,
...@@ -41,28 +39,14 @@ def __upload_network_cached(dir_url, cache_url, ...@@ -41,28 +39,14 @@ def __upload_network_cached(dir_url, cache_url,
metadata_dict.setdefault('urlmd5', 'notused') metadata_dict.setdefault('urlmd5', 'notused')
# convert '' into None in order to call nc nicely # convert '' into None in order to call nc nicely
if not signature_private_key_file: with NetworkcacheClient(cache_url, dir_url,
signature_private_key_file = None signature_private_key_file=signature_private_key_file or None,
if not shacache_cert_file: shacache_cert_file=shacache_cert_file or None,
shacache_cert_file = None shacache_key_file=shacache_key_file or None,
if not shacache_key_file: shadir_cert_file=shadir_cert_file or None,
shacache_key_file = None shadir_key_file=shadir_key_file or None,
if not shadir_cert_file: ) as nc:
shadir_cert_file = None
if not shadir_key_file:
shadir_key_file = None
nc = NetworkcacheClient(cache_url, dir_url,
signature_private_key_file=signature_private_key_file,
shacache_cert_file=shacache_cert_file,
shacache_key_file=shacache_key_file,
shadir_cert_file=shadir_cert_file,
shadir_key_file=shadir_key_file)
try:
return nc.upload(file_descriptor, directory_key, **metadata_dict) return nc.upload(file_descriptor, directory_key, **metadata_dict)
except (IOError, UploadError), e:
logger.info('Failed to upload file. %s' % str(e))
return False
# BBB: slapos.buildout (1.6.0-dev-SlapOS-011) imports it without using it # BBB: slapos.buildout (1.6.0-dev-SlapOS-011) imports it without using it
helper_upload_network_cached_from_file = None helper_upload_network_cached_from_file = None
...@@ -100,9 +84,10 @@ def helper_download_network_cached(dir_url, cache_url, ...@@ -100,9 +84,10 @@ def helper_download_network_cached(dir_url, cache_url,
return (file_descriptor, metadata) if succeeded, False otherwise. return (file_descriptor, metadata) if succeeded, False otherwise.
""" """
if dir_url and cache_url: if not (dir_url and cache_url):
nc = NetworkcacheClient(cache_url, dir_url, return
signature_certificate_list=signature_certificate_list) with NetworkcacheClient(cache_url, dir_url,
signature_certificate_list=signature_certificate_list) as nc:
logger.info('Downloading %s...', directory_key) logger.info('Downloading %s...', directory_key)
result = nc.select(directory_key, wanted_metadata_dict, required_key_list) result = nc.select(directory_key, wanted_metadata_dict, required_key_list)
if strategy: if strategy:
...@@ -117,10 +102,7 @@ def helper_download_network_cached(dir_url, cache_url, ...@@ -117,10 +102,7 @@ def helper_download_network_cached(dir_url, cache_url,
else: else:
entry = next(result, None) entry = next(result, None)
if entry: if entry:
try: return nc.download(entry['sha512']), entry
return nc.download(entry['sha512']), entry
except urllib2.HTTPError, e:
logger.warning('Failed to download %s: %s', directory_key, e)
else: else:
logger.info('No matching entry to download %s', directory_key) logger.info('No matching entry to download %s', directory_key)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment