Refactor select/select_generic and make select_generic filtering lazily

slapos/libnetworkcache.py

@@ -22,8 +22,6 @@ import urllib2
 import urlparse
 from OpenSSL import crypto
 
-# XXX: code between select/select_generic must be factored
-
 # Timeout here is about timeout to CONNECT to the server (socket initialization then server answers actual data), not to retrieve/send informations.
 # To be clear: it is NOT about uploading/downloading data, but about time to connect to the server, then time that server takes to start answering.
 TIMEOUT = 60
@@ -242,38 +240,24 @@ class NetworkcacheClient(object):
     ''' Download a file from shacache by selecting the entry in shadir
     Raise DirectoryNotFound if no trustable file is found.
     '''
-    url = urljoin(self.shadir_url, key)
-    request = urllib2.Request(url=url, data=None,
-      headers=self.shadir_header_dict)
-    data = urllib2.urlopen(request, timeout=TIMEOUT).read()
-    # Filtering...
-    try:
-      data_list = json.loads(data)
-    except Exception:
-      raise DirectoryNotFound('It was impossible to parse json response:\n%s'%
-        traceback.format_exc())
-    if self.signature_certificate_list:
-      data_list = [data for data in data_list
-                        if self._verifySignatureInCertificateList(*data)]
-
-    if not data_list:
+    for information_json, signature in self.select_generic(key,
+                                              self.signature_certificate_list):
+      break
+    else:
       raise DirectoryNotFound('Could not find a trustable entry.')
-
-    information_json, signature = data_list[0]
     try:
       information_dict = json.loads(information_json)
     except Exception:
-      raise DirectoryNotFound('It was impossible to parse json-in-json '
-        'response:\n%s' % traceback.format_exc())
+      raise DirectoryNotFound('Failed to parse json-in-json response:\n%s'
+                              % traceback.format_exc())
     try:
-      sha512 = information_dict.get('sha512')
+      sha512 = information_dict['sha512']
     except Exception:
-      raise DirectoryNotFound('It was impossible to fetch sha512 from '
-        'directory response (%r):\n%s' % (information_dict,
-          traceback.format_exc()))
+      raise DirectoryNotFound('No sha512 in directory response (%r):\n%s'
+                              % (information_dict, traceback.format_exc()))
     return self.download(sha512)
 
-  def select_generic(self, key):
+  def select_generic(self, key, filter=True):
     ''' Select trustable entries from shadir.
     '''
     url = urljoin(self.shadir_url, key)
@@ -283,10 +267,12 @@ class NetworkcacheClient(object):
     try:
       data_list = json.loads(data)
     except Exception:
-      raise DirectoryNotFound('It was impossible to parse json response:\n%s' %
-        traceback.format_exc())
-    return [data for data in data_list
-                 if self._verifySignatureInCertificateList(*data)]
+      raise DirectoryNotFound('Failed to parse json response:\n%s'
+                              % traceback.format_exc())
+    if filter:
+      return (data for data in data_list
+                   if self._verifySignatureInCertificateList(*data))
+    return data_list
 
   def _getSignatureString(self, content):
     """

slapos/libnetworkcachetests.py

@@ -477,7 +477,7 @@ class OnlineTest(OnlineMixin, unittest.TestCase):
     with open(os.path.join(self.tree, 'shadir', key), 'w') as f:
       # now remove the entry from shacache
       f.write('This is not a json.')
-    self.assertDirectoryNotFound('It was impossible to parse json response',
+    self.assertDirectoryNotFound('Failed to parse json response',
                                  nc.select, key)
 
   def test_select_json_no_in_json_response(self):
@@ -491,9 +491,8 @@ class OnlineTest(OnlineMixin, unittest.TestCase):
     with open(os.path.join(self.tree, 'shadir', key), 'w') as f:
       # now remove the entry from shacache
       f.write(json.dumps([['This is not a json.', 'signature']]))
-    self.assertDirectoryNotFound(
-      'It was impossible to parse json-in-json response',
-      nc.select, key)
+    self.assertDirectoryNotFound('Failed to parse json-in-json response',
+                                 nc.select, key)
 
   def test_select_json_in_json_no_dict(self):
     key = 'somekey' + str(random.random())
@@ -506,9 +505,8 @@ class OnlineTest(OnlineMixin, unittest.TestCase):
     with open(os.path.join(self.tree, 'shadir', key), 'w') as f:
       # now remove the entry from shacache
       f.write(json.dumps([[json.dumps('This is a string'), 'signature']]))
-    self.assertDirectoryNotFound(
-      'It was impossible to fetch sha512 from directory response',
-      nc.select, key)
+    self.assertDirectoryNotFound('No sha512 in directory response',
+                                 nc.select, key)
 
   def test_select_signed_content_server_hacked(self):
     key = 'somekey' + str(random.random())