cli: add signature verification

Use libnetworkcache to list only binaries with verified signature

slapos/cli/cache.py

@@ -31,7 +31,6 @@ import ast
 import hashlib
 import json
 import re
-import requests
 import sys
 
 import prettytable
@@ -63,8 +62,12 @@ class CacheLookupCommand(ConfigCommand):
     def take_action(self, args):
         configp = self.fetch_config(args)
         cache_dir = configp.get('networkcache', 'download-binary-dir-url')
+        cache_url = configp.get('networkcache', 'download-binary-cache-url')
+        signature_certificate_list = configp.get('networkcache', 'signature-certificate-list')
+
         sys.exit(
-          do_lookup(self.app.log, cache_dir, args.software_url))
+          do_lookup(self.app.log, cache_dir, cache_url,
+                    signature_certificate_list, args.software_url,))
 
 def looks_like_md5(s):
     """
@@ -74,38 +77,29 @@ def looks_like_md5(s):
     return re.match('[0-9a-f]{32}', s)
 
 
-def ostuple(jsondict):
-    srdict = json.loads(jsondict)
-    return (srdict['machine'],) + ast.literal_eval(srdict['os'])
+def ostuple(info_dict):
+    return (info_dict['machine'],) + ast.literal_eval(info_dict['os'])
 
-def do_lookup(logger, cache_dir, software_url):
+def do_lookup(logger, cache_dir, cache_url, signature_certificate_list,
+              software_url):
     if looks_like_md5(software_url):
         md5 = software_url
     else:
         md5 = hashlib.md5(str2bytes(software_url)).hexdigest()
     try:
-        url = '%s/%s' % (cache_dir, md5)
-        logger.debug('Connecting to %s', url)
-        req = requests.get(url, timeout=5)
-    except (requests.Timeout, requests.ConnectionError):
-        logger.critical('Cannot connect to cache server at %s', url)
-        return FAILURE_EXIT_CODE
-
-    if not req.ok:
-        if req.status_code == 404:
-            logger.critical('Object not in cache: %s', software_url)
-        else:
-            logger.critical('Error while looking object %s: %s', software_url, req.reason)
+        entries = list(
+            networkcache.download_entry_list(cache_url, cache_dir, md5, logger,
+                signature_certificate_list, software_url))
+    except Exception:
+        logger.critical('Error while looking object %s', software_url,
+            exc_info=True)
         return FAILURE_EXIT_CODE
 
-    entries = req.json()
-
     if not entries:
         logger.info('Object found in cache, but has no binary entries.')
         return 0
 
-    ostable = sorted(ostuple(entry[0]) for entry in entries)
-
+    ostable = sorted(ostuple(json.loads(entry[0])) for entry in entries)
     pt = prettytable.PrettyTable(['machine', 'distribution', 'version', 'id', 'compatible?'])
 
     for os in ostable:

slapos/grid/networkcache.py

@@ -57,6 +57,13 @@ def is_compatible(machine, os):
     return machine == platform.machine() and os_matches(os, distribution_tuple())
 
 
+def download_entry_list(cache_url, dir_url, key, logger,
+                        signature_certificate_list, software_url):
+    nc = NetworkcacheClient(cache_url, dir_url,
+        signature_certificate_list=signature_certificate_list or None)
+    return nc.select_generic(key)
+
+
 @fallback_call
 def download_network_cached(cache_url, dir_url, software_url, software_root,
                             key, path, logger, signature_certificate_list,
@@ -172,7 +179,7 @@ def upload_network_cached(software_root, software_url, cached_key,
     try:
         return nc.upload_generic(f, cached_key, **kw)
     except (IOError, UploadError) as e:
-        logger.info('Failed to upload file. %s' % (str(e)))
+        logger.info('Failed to upload file. %s' % str(e))
         return False
     finally:
       f.close()

slapos/tests/test_cli.py

@@ -63,6 +63,21 @@ import slapos.slap
 
 import supervisor.supervisorctl
 
+signature_certificate_list = """-----BEGIN CERTIFICATE-----     
+MIIB9jCCAV+gAwIBAgIJAKRvzcy7OH0UMA0GCSqGSIb3DQEBBQUAMBMxETAPBgNV
+BAMMCENPTVAtNzcyMCAXDTEyMDgxMDE1NDI1MVoYDzIxMTIwNzE3MTU0MjUxWjAT
+MREwDwYDVQQDDAhDT01QLTc3MjCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA    
+o7aipd6MbnuGDeR1UJUjuMLQUariAyQ2l2ZDS6TfOwjHiPw/mhzkielgk73kqN7A                                 
+sUREx41eTcYCXzTq3WP3xCLE4LxLg1eIhd4nwNHj8H18xR9aP0AGjo4UFl5BOMa1
+mwoyBt3VtfGtUmb8whpeJgHhqrPPxLoON+i6fIbXDaUCAwEAAaNQME4wHQYDVR0O
+BBYEFEfjy3OopT2lOksKmKBNHTJE2hFlMB8GA1UdIwQYMBaAFEfjy3OopT2lOksK
+mKBNHTJE2hFlMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEAaNRx6YN2                             
+M/p3R8/xS6zvH1EqJ3FFD7XeAQ52WuQnKSREzuw0dsw12ClxjcHiQEFioyTiTtjs
+5pW18Ry5Ie7iFK4cQMerZwWPxBodEbAteYlRsI6kePV7Gf735Y1RpuN8qZ2sYL6e
+x2IMeSwJ82BpdEI5niXxB+iT0HxhmR+XaMI=
+-----END CERTIFICATE-----
+"""
+
 def raiseNotFoundError(*args, **kwargs):
   raise slapos.slap.NotFoundError()
 
@@ -72,6 +87,7 @@ class CliMixin(unittest.TestCase):
     self.logger = create_autospec(logging.Logger)
     self.local = {'slap': slap, 'product': SoftwareProductCollection(self.logger, slap)}
     self.conf = create_autospec(ClientConfig)
+    self.sign_cert_list = signature_certificate_list
 
 class TestCliCache(CliMixin):
 
@@ -80,7 +96,9 @@ class TestCliCache(CliMixin):
     self.assertEqual(0, cache_do_lookup(
         self.logger,
         cache_dir="http://dir.shacache.org",
-        software_url=self.test_url))
+        cache_url="http://shacache.org",
+        software_url=self.test_url,
+        signature_certificate_list=self.sign_cert_list))
 
     self.logger.info.assert_any_call('Software URL: %s', 
             u'https://lab.nexedi.com/nexedi/slapos/raw/1.0.102/software/slaprunner/software.cfg')
@@ -97,19 +115,25 @@ class TestCliCache(CliMixin):
     self.assertEqual(10, cache_do_lookup(
         self.logger,
         cache_dir="http://dir.shacache.org",
-        software_url="this_is_uncached_url"))
+        cache_url="http://shacache.org",
+        software_url="this_is_uncached_url",
+        signature_certificate_list=self.sign_cert_list))
 
-    self.logger.critical.assert_any_call('Object not in cache: %s', 'this_is_uncached_url') 
+    self.logger.critical.assert_any_call(
+        'Error while looking object %s', 'this_is_uncached_url', exc_info=True)
 
   def test_bad_cache_dir(self):
     self.assertEqual(10, cache_do_lookup(
         self.logger,
         cache_dir="http://xxx.shacache.org",
-        software_url=self.test_url))
+        cache_url="http://shacache.org",
+        software_url=self.test_url,
+        signature_certificate_list=self.sign_cert_list))
 
     self.logger.critical.assert_any_call(
-      'Cannot connect to cache server at %s', 
-      'http://xxx.shacache.org/cccdc51a07e8c575c880f2d70dd4d458')
+        'Error while looking object %s',
+        'https://lab.nexedi.com/nexedi/slapos/raw/1.0.102/software/slaprunner/software.cfg',
+        exc_info=True)
 
 
 class TestCliCacheSource(CliMixin):