sr-test-suite: use generateHashFromFiles from slapos library

Remove method `generateHashFromFiles` in all file `test.py`, import it from `slapos.recipe.librecipe`.

/reviewed-on nexedi/slapos!617

slapos/recipe/librecipe/__init__.py

@@ -40,7 +40,7 @@ import json
 # Use to do from slapos.recipe.librecipe import GenericBaseRecipe
 from .generic import GenericBaseRecipe
 from .genericslap import GenericSlapRecipe
-from .filehash import filehash
+from .filehash import filehash, generateHashFromFiles
 
 # Utility functions to (de)serialise live python objects in order to send them
 # to master.

slapos/recipe/librecipe/filehash.py

@@ -51,6 +51,15 @@ def filehash(filename, type_=DEFAULT_HASH):
     shutil.copyfileobj(file_, digest)
   return digest.read()
 
+def generateHashFromFiles(file_list):
+  hasher = hashlib.md5()
+  for path in file_list:
+    with open(path, 'rb') as afile:
+      buf = afile.read()
+    hasher.update(b"%u\n" % len(buf))
+    hasher.update(buf)
+  return hasher.hexdigest()
+
 # Home made hashdeep <http://md5deep.sourceforge.net/>
 def dirhash(dirname, type_=DEFAULT_HASH):
   """Walk into a directory an return a unique hash for

slapos/recipe/wrapper.py

@@ -27,7 +27,7 @@
 
 import shlex
 
-from slapos.recipe.librecipe import GenericBaseRecipe
+from slapos.recipe.librecipe import GenericBaseRecipe, generateHashFromFiles
 
 class Recipe(GenericBaseRecipe):
     """Recipe to create a script from given command and options.
@@ -67,18 +67,8 @@ class Recipe(GenericBaseRecipe):
           kw['reserve_cpu'] = True
         if hash_files:
           hash_file_list = hash_files.split()
-          hash = self.generateHashFromFiles(hash_file_list)
+          hash = generateHashFromFiles(hash_file_list)
           wrapper_path = "%s-%s" % (wrapper_path, hash)
 
         return self.createWrapper(wrapper_path, args, environment, **kw)
 
-    def generateHashFromFiles(self, file_list):
-      import hashlib
-      hasher = hashlib.md5()
-      for path in file_list:
-        with open(path, 'rb') as afile:
-          buf = afile.read()
-        hasher.update(b"%u\n" % len(buf))
-        hasher.update(buf)
-      hash = hasher.hexdigest()
-      return hash

software/caddy-frontend/test/setup.py

@@ -38,6 +38,7 @@ setup(name=name,
       packages=find_packages(),
       install_requires=[
         'slapos.core',
+        'slapos.cookbook',
         'slapos.libnetworkcache',
         'erp5.util',
         'requests >= 2.20.0',  # needed for recent SSL certificate fixes

software/caddy-frontend/test/test.py

@@ -46,6 +46,7 @@ import StringIO
 import gzip
 import base64
 import re
+from slapos.recipe.librecipe import generateHashFromFiles
 
 
 try:
@@ -283,18 +284,6 @@ class TestDataMixin(object):
     except Exception as e:
       self.fail(e)
 
-  @staticmethod
-  def generateHashFromFiles(file_list):
-    import hashlib
-    hasher = hashlib.md5()
-    for path in file_list:
-      with open(path, 'r') as afile:
-        buf = afile.read()
-      hasher.update("%s\n" % len(buf))
-      hasher.update(buf)
-    hash = hasher.hexdigest()
-    return hash
-
   def getTrimmedProcessInfo(self):
     return '\n'.join(sorted([
       '%(group)s:%(name)s %(statename)s' % q for q
@@ -383,13 +372,13 @@ class TestDataMixin(object):
     hash_file_list = [os.path.join(
         self.computer_partition_root_path, 'software_release/buildout.cfg')]
     hash_value_dict = {
-      'generic': self.generateHashFromFiles(hash_file_list),
+      'generic': generateHashFromFiles(hash_file_list),
     }
     for caddy_wrapper_path in glob.glob(os.path.join(
       self.instance_path, '*', 'bin', 'caddy-wrapper')):
       partition_id = caddy_wrapper_path.split('/')[-3]
       hash_value_dict[
-        'caddy-%s' % (partition_id)] = self.generateHashFromFiles(
+        'caddy-%s' % (partition_id)] = generateHashFromFiles(
         hash_file_list + [caddy_wrapper_path]
       )
     for rejected_slave_publish_path in glob.glob(os.path.join(
@@ -399,7 +388,7 @@ class TestDataMixin(object):
         self.instance_path, partition_id, 'etc', 'rejected-slave.pem')
       hash_value_dict[
         'rejected-slave-publish'
-      ] = self.generateHashFromFiles(
+      ] = generateHashFromFiles(
         hash_file_list + [rejected_slave_publish_path, rejected_slave_pem_path]
       )
 

software/kvm/test/setup.py

@@ -43,6 +43,7 @@ setup(name=name,
       packages=find_packages(),
       install_requires=[
           'slapos.core',
+          'slapos.cookbook',
           'slapos.libnetworkcache',
           'erp5.util',
           'supervisor',

software/kvm/test/test.py

@@ -35,9 +35,8 @@ import StringIO
 import subprocess
 import json
 
-import psutil
-
 import utils
+from slapos.recipe.librecipe import generateHashFromFiles
 
 # for development: debugging logs and install Ctrl+C handler
 if os.environ.get('SLAPOS_TEST_DEBUG'):
@@ -53,17 +52,6 @@ class InstanceTestCase(utils.SlapOSInstanceTestCase):
     return (os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'software.cfg')), )
 
 class ServicesTestCase(InstanceTestCase):
-  @staticmethod
-  def generateHashFromFiles(file_list):
-    import hashlib
-    hasher = hashlib.md5()
-    for path in file_list:
-      with open(path, 'r') as afile:
-        buf = afile.read()
-      hasher.update("%s\n" % len(buf))
-      hasher.update(buf)
-    hash = hasher.hexdigest()
-    return hash
 
   def test_hashes(self):
     hash_files = [
@@ -87,7 +75,7 @@ class ServicesTestCase(InstanceTestCase):
                   for path in hash_files]
 
     for name in expected_process_names:
-      h = ServicesTestCase.generateHashFromFiles(hash_files)
+      h = generateHashFromFiles(hash_files)
       expected_process_name = name.format(hash=h)
 
       self.assertIn(expected_process_name, process_names)

software/monitor/test/setup.py

@@ -43,6 +43,7 @@ setup(name=name,
       packages=find_packages(),
       install_requires=[
           'slapos.core',
+          'slapos.cookbook',
           'slapos.libnetworkcache',
           'erp5.util',
           'supervisor',

software/monitor/test/test.py

@@ -35,9 +35,8 @@ import StringIO
 import subprocess
 import json
 
-import psutil
-
 import utils
+from slapos.recipe.librecipe import generateHashFromFiles
 
 # for development: debugging logs and install Ctrl+C handler
 if os.environ.get('SLAPOS_TEST_DEBUG'):
@@ -54,17 +53,6 @@ class InstanceTestCase(utils.SlapOSInstanceTestCase):
 
 
 class ServicesTestCase(InstanceTestCase):
-  @staticmethod
-  def generateHashFromFiles(file_list):
-    import hashlib
-    hasher = hashlib.md5()
-    for path in file_list:
-      with open(path, 'r') as afile:
-        buf = afile.read()
-      hasher.update("%s\n" % len(buf))
-      hasher.update(buf)
-    hash = hasher.hexdigest()
-    return hash
 
   def test_hashes(self):
     hash_files = [
@@ -83,7 +71,7 @@ class ServicesTestCase(InstanceTestCase):
                   for path in hash_files]
 
     for name in expected_process_names:
-      h = ServicesTestCase.generateHashFromFiles(hash_files)
+      h = generateHashFromFiles(hash_files)
       expected_process_name = name.format(hash=h)
 
       self.assertIn(expected_process_name, process_names)

software/nextcloud/test/setup.py

@@ -43,6 +43,7 @@ setup(name=name,
       packages=find_packages(),
       install_requires=[
           'slapos.core',
+          'slapos.cookbook',
           'slapos.libnetworkcache',
           'erp5.util',
           'supervisor',

software/nextcloud/test/test.py

@@ -32,6 +32,7 @@ import glob
 import re
 
 import utils
+from slapos.recipe.librecipe import generateHashFromFiles
 
 # for development: debugging logs and install Ctrl+C handler
 if os.environ.get('SLAPOS_TEST_DEBUG'):
@@ -166,18 +167,6 @@ class InstanceTestCase(utils.SlapOSInstanceTestCase):
 
 class ServicesTestCase(InstanceTestCase):
 
-  @staticmethod
-  def generateHash(file_list):
-    import hashlib
-    hasher = hashlib.md5()
-    for path in file_list:
-      with open(path, 'r') as afile:
-        buf = afile.read()
-      hasher.update("%s\n" % len(buf))
-      hasher.update(buf)
-    hash = hasher.hexdigest()
-    return hash
-
   def test_process_list(self):
     hash_list = [
       'software_release/buildout.cfg',
@@ -204,7 +193,7 @@ class ServicesTestCase(InstanceTestCase):
                       for path in hash_list]
 
     for name in expected_process_names:
-      h = ServicesTestCase.generateHash(hash_file_list)
+      h = generateHashFromFiles(hash_file_list)
       expected_process_name = name.format(hash=h)
 
       self.assertIn(expected_process_name, process_name_list)

software/plantuml/test/setup.py

@@ -43,6 +43,7 @@ setup(name=name,
       packages=find_packages(),
       install_requires=[
         'slapos.core',
+        'slapos.cookbook',
         'slapos.libnetworkcache',
         'erp5.util',
         'supervisor',

software/plantuml/test/test.py

@@ -36,6 +36,7 @@ import requests
 import plantuml
 
 import utils
+from slapos.recipe.librecipe import generateHashFromFiles
 
 # for development: debugging logs and install Ctrl+C handler
 if os.environ.get('SLAPOS_TEST_DEBUG'):
@@ -154,16 +155,6 @@ class TestSimpleDiagram(PlantUMLTestCase):
 
 
 class ServicesTestCase(PlantUMLTestCase):
-  @staticmethod
-  def generateHashFromFiles(file_list):
-    hasher = hashlib.md5()
-    for path in file_list:
-      with open(path, 'r') as afile:
-        buf = afile.read()
-      hasher.update("%s\n" % len(buf))
-      hasher.update(buf)
-    hash = hasher.hexdigest()
-    return hash
 
   def test_hashes(self):
     hash_files = [
@@ -182,7 +173,7 @@ class ServicesTestCase(PlantUMLTestCase):
                   for path in hash_files]
 
     for name in expected_process_names:
-      h = ServicesTestCase.generateHashFromFiles(hash_files)
+      h = generateHashFromFiles(hash_files)
       expected_process_name = name.format(hash=h)
 
       self.assertIn(expected_process_name, process_names)

software/powerdns/test/setup.py

@@ -43,6 +43,7 @@ setup(name=name,
       packages=find_packages(),
       install_requires=[
           'slapos.core',
+          'slapos.cookbook',
           'slapos.libnetworkcache',
           'erp5.util',
           'supervisor',

software/powerdns/test/test.py

@@ -35,9 +35,8 @@ import StringIO
 import subprocess
 import json
 
-import psutil
-
 import utils
+from slapos.recipe.librecipe import generateHashFromFiles
 
 # for development: debugging logs and install Ctrl+C handler
 if os.environ.get('SLAPOS_TEST_DEBUG'):
@@ -54,17 +53,6 @@ class InstanceTestCase(utils.SlapOSInstanceTestCase):
 
 
 class ServicesTestCase(InstanceTestCase):
-  @staticmethod
-  def generateHashFromFiles(file_list):
-    import hashlib
-    hasher = hashlib.md5()
-    for path in file_list:
-      with open(path, 'r') as afile:
-        buf = afile.read()
-      hasher.update("%s\n" % len(buf))
-      hasher.update(buf)
-    hash = hasher.hexdigest()
-    return hash
 
   def test_hashes(self):
     hash_files = [
@@ -82,7 +70,7 @@ class ServicesTestCase(InstanceTestCase):
                   for path in hash_files]
 
     for name in expected_process_names:
-      h = ServicesTestCase.generateHashFromFiles(hash_files)
+      h = generateHashFromFiles(hash_files)
       expected_process_name = name.format(hash=h)
 
       self.assertIn(expected_process_name, process_names)

software/re6stnet/test/setup.py

@@ -43,6 +43,7 @@ setup(name=name,
       packages=find_packages(),
       install_requires=[
         'slapos.core',
+        'slapos.cookbook',
         'slapos.libnetworkcache',
         'erp5.util',
         'supervisor',

software/re6stnet/test/test.py

@@ -35,9 +35,8 @@ import StringIO
 import subprocess
 import json
 
-import psutil
-
 import utils
+from slapos.recipe.librecipe import generateHashFromFiles
 
 SLAPOS_TEST_IPV4 = os.environ['SLAPOS_TEST_IPV4']
 SLAPOS_TEST_IPV6 = os.environ['SLAPOS_TEST_IPV6']
@@ -83,17 +82,6 @@ class TestPortRedirection(Re6stnetTestCase):
     }, portredir_config[0])
 
 class ServicesTestCase(Re6stnetTestCase):
-  @staticmethod
-  def generateHashFromFiles(file_list):
-    import hashlib
-    hasher = hashlib.md5()
-    for path in file_list:
-      with open(path, 'r') as afile:
-        buf = afile.read()
-      hasher.update("%s\n" % len(buf))
-      hasher.update(buf)
-    hash = hasher.hexdigest()
-    return hash
 
   @classmethod
   def getInstanceParameterDict(cls):
@@ -115,7 +103,7 @@ class ServicesTestCase(Re6stnetTestCase):
                   for path in hash_files]
 
     for name in expected_process_names:
-      h = ServicesTestCase.generateHashFromFiles(hash_files)
+      h = generateHashFromFiles(hash_files)
       expected_process_name = name.format(hash=h)
 
       self.assertIn(expected_process_name, process_names)

software/slaprunner/test/setup.py

@@ -43,6 +43,7 @@ setup(name=name,
       packages=find_packages(),
       install_requires=[
           'slapos.core',
+          'slapos.cookbook',
           'slapos.libnetworkcache',
           'erp5.util',
           'supervisor',

software/slaprunner/test/test.py

@@ -38,6 +38,7 @@ import json
 import psutil
 
 import utils
+from slapos.recipe.librecipe import generateHashFromFiles
 
 # for development: debugging logs and install Ctrl+C handler
 if os.environ.get('SLAPOS_TEST_DEBUG'):
@@ -53,17 +54,6 @@ class InstanceTestCase(utils.SlapOSInstanceTestCase):
     return (os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'software.cfg')), )
 
 class ServicesTestCase(InstanceTestCase):
-  @staticmethod
-  def generateHashFromFiles(file_list):
-    import hashlib
-    hasher = hashlib.md5()
-    for path in file_list:
-      with open(path, 'r') as afile:
-        buf = afile.read()
-      hasher.update("%s\n" % len(buf))
-      hasher.update(buf)
-    hash = hasher.hexdigest()
-    return hash
 
   def test_hashes(self):
     hash_files = [
@@ -89,7 +79,7 @@ class ServicesTestCase(InstanceTestCase):
                   for path in hash_files]
 
     for name in expected_process_names:
-      h = ServicesTestCase.generateHashFromFiles(hash_files)
+      h = generateHashFromFiles(hash_files)
       expected_process_name = name.format(hash=h)
 
       self.assertIn(expected_process_name, process_names)

software/turnserver/test/setup.py

@@ -43,6 +43,7 @@ setup(name=name,
       packages=find_packages(),
       install_requires=[
           'slapos.core',
+          'slapos.cookbook',
           'slapos.libnetworkcache',
           'erp5.util',
           'supervisor',

software/turnserver/test/test.py

@@ -32,6 +32,7 @@ import glob
 import ConfigParser
 
 import utils
+from slapos.recipe.librecipe import generateHashFromFiles
 
 # for development: debugging logs and install Ctrl+C handler
 if os.environ.get('SLAPOS_TEST_DEBUG'):
@@ -58,18 +59,6 @@ class InstanceTestCase(utils.SlapOSInstanceTestCase):
 
 class ServicesTestCase(InstanceTestCase):
 
-  @staticmethod
-  def generateHash(file_list):
-    import hashlib
-    hasher = hashlib.md5()
-    for path in file_list:
-      with open(path, 'r') as afile:
-        buf = afile.read()
-      hasher.update("%s\n" % len(buf))
-      hasher.update(buf)
-    hash = hasher.hexdigest()
-    return hash
-
   def test_process_list(self):
     hash_list = [
       'software_release/buildout.cfg',
@@ -91,7 +80,7 @@ class ServicesTestCase(InstanceTestCase):
                       for path in hash_list]
 
     for name in expected_process_names:
-      h = ServicesTestCase.generateHash(hash_file_list)
+      h = generateHashFromFiles(hash_file_list)
       expected_process_name = name.format(hash=h)
 
       self.assertIn(expected_process_name, process_name_list)