Merge upload_generic() into upload() and make it return the sha512

If people don't write stupid code like 'if result is True:',
this small API change should not break anything.

Add new index() method to add an entry to shadir when the data is known to be
already in shacache.

slapos/libnetworkcache.py

@@ -25,8 +25,7 @@ import traceback
 import urllib2
 import urlparse
 
-# XXX: code between select/select_generic and upload/upload_generic should be
-# factored
+# XXX: code between select/select_generic must be factored
 
 # Timeout here is about timeout to CONNECT to the server (socket initialization then server answers actual data), not to retrieve/send informations.
 # To be clear: it is NOT about uploading/downloading data, but about time to connect to the server, then time that server takes to start answering.
@@ -165,9 +164,9 @@ class NetworkcacheClient(object):
     self.shadir_cert_file = shadir_cert_file
 
   def upload(self, file_descriptor, key=None, urlmd5=None, file_name=None,
-    valid_until=None, architecture=None):
+                                    valid_until=None, architecture=None, **kw):
     ''' Upload the file to the server.
-    If urlmd5 is None it must only upload to SHACACHE.
+    If key is None it must only upload to SHACACHE.
     Otherwise, it must create a new entry on SHADIR.
     '''
     # do not trust, go to beginning of opened file
@@ -182,12 +181,9 @@ class NetworkcacheClient(object):
         self.shacache_port, timeout=UPLOAD_TIMEOUT)
     try:
       shacache_connection.request('POST', self.shacache_path, file_descriptor,
-                                                 self.shacache_header_dict)
-      print 'uploaded'
+                                      self.shacache_header_dict)
       result = shacache_connection.getresponse()
-      print 'answered'
       sha512sum = result.read()
-      print 'read'
     finally:
       shacache_connection.close()
 
@@ -197,104 +193,46 @@ class NetworkcacheClient(object):
                         % (self.shacache_host, result.status, sha512sum))
 
     if key is not None:
+      kw['sha512'] = sha512sum # always update sha512sum
       if file_name is None or urlmd5 is None:
-        raise ValueError('In case if key is given file_name and urlmd5 '
-          'are required.')
-      kw = dict()
-      kw['file'] = file_name
-      kw['urlmd5'] = urlmd5
-      kw['sha512'] = sha512sum
-
+        raise ValueError('file_name and urlmd5 are required'
+                         ' for non-generic upload')
       if valid_until is not None:
         kw['valid-until'] = valid_until
       if architecture is not None:
         kw['architecture'] = architecture
+      self.index(key, file=file_name, urlmd5=urlmd5, **kw)
 
-      sha_entry = json.dumps(kw)
-      try:
-        signature = self._getSignatureString(sha_entry)
-      except Exception:
-        raise UploadError('Impossible to sign content, error:\n%s' %
-          traceback.format_exc())
-      data = [sha_entry, signature]
-
-      if self.shadir_scheme == 'https':
-        shadir_connection = httplib.HTTPSConnection(self.shadir_host,
-          self.shadir_port, key_file=self.shadir_key_file,
-          cert_file=self.shadir_cert_file, timeout=UPLOAD_TIMEOUT)
-      else:
-        shadir_connection = httplib.HTTPConnection(self.shadir_host,
-          self.shadir_port, timeout=UPLOAD_TIMEOUT)
-      try:
-        shadir_connection.request('PUT', '/'.join([self.shadir_path, key]),
-          json.dumps(data), self.shadir_header_dict)
-        result = shadir_connection.getresponse()
-        data = result.read()
-      finally:
-        shadir_connection.close()
+    return sha512sum
 
-      if result.status != 201:
-        raise UploadError('Failed to upload data to SHADIR Server.' \
-                          'URL: %s. Response code: %s. Response data: %s' % \
-                                     (self.shacache_host, result.status, data))
-    return True
+  upload_generic = upload # BBB
 
-  def upload_generic(self, file_descriptor, key=None, **kw):
-    ''' Upload the file to the server.
-    If key is None, it must only upload to SHACACHE.
-    Otherwise, it must create a new entry on SHADIR.
-    '''
-    file_descriptor.seek(0)
-    file_descriptor = hashing_file(file_descriptor)
-    if self.shacache_scheme == 'https':
-      shacache_connection = httplib.HTTPSConnection(self.shacache_host,
-        self.shacache_port, key_file = self.shacache_key_file,
-        cert_file = self.shacache_cert_file, timeout=UPLOAD_TIMEOUT)
+  def index(self, key, **kw):
+    data = json.dumps(kw)
+    try:
+      data = [data, self._getSignatureString(data)]
+    except Exception:
+      raise UploadError('Impossible to sign content, error:\n%s'
+                        % traceback.format_exc())
+    if self.shadir_scheme == 'https':
+      shadir_connection = httplib.HTTPSConnection(self.shadir_host,
+          self.shadir_port, key_file=self.shadir_key_file,
+          cert_file=self.shadir_cert_file, timeout=UPLOAD_TIMEOUT)
     else:
-      shacache_connection = httplib.HTTPConnection(self.shacache_host,
-        self.shacache_port, timeout=UPLOAD_TIMEOUT)
+      shadir_connection = httplib.HTTPConnection(self.shadir_host,
+          self.shadir_port, timeout=UPLOAD_TIMEOUT)
     try:
-      shacache_connection.request('POST', self.shacache_path, file_descriptor,
-                                      self.shacache_header_dict)
-      result = shacache_connection.getresponse()
-      sha512sum = result.read()
+      shadir_connection.request('PUT', '/'.join([self.shadir_path, key]),
+          json.dumps(data), self.shadir_header_dict)
+      result = shadir_connection.getresponse()
+      data = result.read()
     finally:
-      shacache_connection.close()
+      shadir_connection.close()
 
-    if result.status != 201 or sha512sum != file_descriptor.hexdigest():
-      raise UploadError('Failed to upload the file to SHACACHE Server.'
+    if result.status != 201:
+      raise UploadError('Failed to upload data to SHADIR Server.'
                         'URL: %s. Response code: %s. Response data: %s'
-                        % (self.shacache_host, result.status, sha512sum))
-
-    if key is not None:
-      kw['sha512'] = sha512sum # always update sha512sum
-      sha_entry = json.dumps(kw)
-      try:
-        signature = self._getSignatureString(sha_entry)
-      except Exception:
-        raise UploadError('Impossible to sign content, error:\n%s' %
-          traceback.format_exc())
-      data = [sha_entry, signature]
-      if self.shadir_scheme == 'https':
-        shadir_connection = httplib.HTTPSConnection(self.shadir_host,
-          self.shadir_port, key_file = self.shadir_key_file,
-          cert_file = self.shadir_cert_file, timeout=UPLOAD_TIMEOUT)
-      else:
-        shadir_connection = httplib.HTTPConnection(self.shadir_host,
-          self.shadir_port, timeout=UPLOAD_TIMEOUT)
-      try:
-        shadir_connection.request('PUT', '/'.join([self.shadir_path, key]),
-          json.dumps(data), self.shadir_header_dict)
-        result = shadir_connection.getresponse()
-        data = result.read()
-      finally:
-        shadir_connection.close()
-      if result.status != 201:
-        raise UploadError('Failed to upload data to SHADIR Server.' \
-                          'URL: %s. Response code: %s. Response data: %s' % \
-                              (self.shadir_host, result.status, data))
-
-    return True
+                        % (self.shadir_host, result.status, data))
 
   def download(self, sha512sum):
     ''' Download the file.

slapos/libnetworkcachetests.py

@@ -493,8 +493,8 @@ class OnlineTest(OnlineMixin, unittest.TestCase):
     key_file.flush()
     signed_nc = slapos.libnetworkcache.NetworkcacheClient(
       self.shacache, self.shadir, key_file.name, [self.certificate])
-    self.assertEqual(True, signed_nc.upload(self.test_data, key,
-        urlmd5=urlmd5, file_name=file_name))
+    self.assertEqual(self.test_shasum, signed_nc.upload(self.test_data,
+        key, urlmd5=urlmd5, file_name=file_name))
     signed_nc.openssl = '/doesnotexists'
     self.assertDirectoryNotFound('Could not find a trustable entry.',
                                  signed_nc.select, key)
@@ -508,8 +508,8 @@ class OnlineTest(OnlineMixin, unittest.TestCase):
     key_file.flush()
     signed_nc = slapos.libnetworkcache.NetworkcacheClient(
       self.shacache, self.shadir, key_file.name, [self.certificate])
-    self.assertEqual(True, signed_nc.upload(self.test_data, key,
-        urlmd5=urlmd5, file_name=file_name))
+    self.assertEqual(self.test_shasum, signed_nc.upload(self.test_data,
+        key, urlmd5=urlmd5, file_name=file_name))
     signed_nc.openssl = sys.executable
     self.assertDirectoryNotFound('Could not find a trustable entry.',
                                  signed_nc.select, key)
@@ -520,8 +520,8 @@ class OnlineTest(OnlineMixin, unittest.TestCase):
     file_name = 'my file'
     nc = slapos.libnetworkcache.NetworkcacheClient(
       self.shacache, self.shadir)
-    self.assertEqual(True, nc.upload(self.test_data, key, urlmd5=urlmd5,
-        file_name=file_name))
+    self.assertEqual(self.test_shasum, nc.upload(self.test_data,
+        key, urlmd5=urlmd5, file_name=file_name))
     f = os.path.join(self.tree, 'shadir', key)
     # now remove the entry from shacache
     open(f, 'w').write(json.dumps([]))
@@ -534,8 +534,8 @@ class OnlineTest(OnlineMixin, unittest.TestCase):
     file_name = 'my file'
     nc = slapos.libnetworkcache.NetworkcacheClient(
       self.shacache, self.shadir)
-    self.assertEqual(True, nc.upload(self.test_data, key, urlmd5=urlmd5,
-        file_name=file_name))
+    self.assertEqual(self.test_shasum, nc.upload(self.test_data,
+        key, urlmd5=urlmd5, file_name=file_name))
     with open(os.path.join(self.tree, 'shadir', key), 'w') as f:
       # now remove the entry from shacache
       f.write('This is not a json.')
@@ -548,8 +548,8 @@ class OnlineTest(OnlineMixin, unittest.TestCase):
     file_name = 'my file'
     nc = slapos.libnetworkcache.NetworkcacheClient(
       self.shacache, self.shadir)
-    self.assertEqual(True, nc.upload(self.test_data, key, urlmd5=urlmd5,
-        file_name=file_name))
+    self.assertEqual(self.test_shasum, nc.upload(self.test_data,
+        key, urlmd5=urlmd5, file_name=file_name))
     with open(os.path.join(self.tree, 'shadir', key), 'w') as f:
       # now remove the entry from shacache
       f.write(json.dumps([['This is not a json.', 'signature']]))
@@ -563,8 +563,8 @@ class OnlineTest(OnlineMixin, unittest.TestCase):
     file_name = 'my file'
     nc = slapos.libnetworkcache.NetworkcacheClient(
       self.shacache, self.shadir)
-    self.assertEqual(True, nc.upload(self.test_data, key, urlmd5=urlmd5,
-        file_name=file_name))
+    self.assertEqual(self.test_shasum, nc.upload(self.test_data,
+        key, urlmd5=urlmd5, file_name=file_name))
     with open(os.path.join(self.tree, 'shadir', key), 'w') as f:
       # now remove the entry from shacache
       f.write(json.dumps([[json.dumps('This is a string'), 'signature']]))

slapos/networkcachehelper.py

@@ -53,10 +53,8 @@ def helper_upload_network_cached(dir_url, cache_url,
     return False
 
   # backward compatibility
-  if not metadata_dict.get('file'):
-    metadata_dict['file'] = 'notused'
-  if not metadata_dict.get('urlmd5'):
-    metadata_dict['urlmd5'] = 'notused'
+  metadata_dict.setdefault('file', 'notused')
+  metadata_dict.setdefault('urlmd5', 'notused')
 
   # convert '' into None in order to call nc nicely
   if not signature_private_key_file:
@@ -81,7 +79,7 @@ def helper_upload_network_cached(dir_url, cache_url,
     return False
 
   try:
-    return nc.upload_generic(file_descriptor, directory_key, **metadata_dict)
+    return nc.upload(file_descriptor, directory_key, **metadata_dict)
   except (IOError, UploadError), e:
     logger.info('Failed to upload file. %s' % str(e))
     return False