proftpd/test: use slapos.testing

software/proftpd/test/setup.py

@@ -25,30 +25,30 @@
 #
 ##############################################################################
 from setuptools import setup, find_packages
-import glob
-import os
 
 version = '0.0.1.dev0'
 name = 'slapos.test.proftpd'
-long_description = open("README.md").read()
+with open("README.md") as f:
+  long_description = f.read()
 
-setup(name=name,
-      version=version,
-      description="Test for SlapOS' ProFTPd",
-      long_description=long_description,
-      long_description_content_type='text/markdown',
-      maintainer="Nexedi",
-      maintainer_email="info@nexedi.com",
-      url="https://lab.nexedi.com/nexedi/slapos",
-      packages=find_packages(),
-      install_requires=[
+setup(
+    name=name,
+    version=version,
+    description="Test for SlapOS' ProFTPd",
+    long_description=long_description,
+    long_description_content_type='text/markdown',
+    maintainer="Nexedi",
+    maintainer_email="info@nexedi.com",
+    url="https://lab.nexedi.com/nexedi/slapos",
+    packages=find_packages(),
+    install_requires=[
         'slapos.core',
         'slapos.libnetworkcache',
         'erp5.util',
         'pysftp',
         'supervisor',
         'psutil',
-        ],
-      zip_safe=True,
-      test_suite='test',
-    )
+    ],
+    zip_safe=True,
+    test_suite='test',
+)

software/proftpd/test/test.py

@@ -37,22 +37,15 @@ import psutil
 from paramiko.ssh_exception import SSHException
 from paramiko.ssh_exception import AuthenticationException
 
-import utils
+from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
+from slapos.testing.utils import findFreeTCPPort
 
+setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
+    os.path.abspath(
+        os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
 
-# for development: debugging logs and install Ctrl+C handler
-if os.environ.get('SLAPOS_TEST_DEBUG'):
-  import logging
-  logging.basicConfig(level=logging.DEBUG)
-  import unittest
-  unittest.installHandler()
-
-
-class ProFTPdTestCase(utils.SlapOSInstanceTestCase):
-  @classmethod
-  def getSoftwareURLList(cls):
-    return (os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'software.cfg')), )
 
+class ProFTPdTestCase(SlapOSInstanceTestCase):
   def _getConnection(self, username=None, password=None, hostname=None):
     """Returns a pysftp connection connected to the SFTP
 
@@ -77,7 +70,7 @@ class ProFTPdTestCase(utils.SlapOSInstanceTestCase):
 
 class TestSFTPListen(ProFTPdTestCase):
   def test_listen_on_ipv4(self):
-    self.assertTrue(self._getConnection(hostname=self.config['ipv4_address']))
+    self.assertTrue(self._getConnection(hostname=self._ipv4_address))
 
   def test_does_not_listen_on_all_ip(self):
     with self.assertRaises(SSHException):
@@ -115,20 +108,22 @@ class TestSFTPOperations(ProFTPdTestCase):
 
       # download the file again, it should have same content
       tempdir = tempfile.mkdtemp()
-      self.addCleanup(lambda : shutil.rmtree(tempdir))
+      self.addCleanup(lambda: shutil.rmtree(tempdir))
       local_file = os.path.join(tempdir, 'testfile')
-      retrieve_same_file = sftp.get('testfile', local_file)
+      sftp.get('testfile', local_file)
       with open(local_file) as f:
         self.assertEqual(f.read(), "Hello FTP !")
 
   def test_uploaded_file_not_visible_until_fully_uploaded(self):
     test_self = self
+
     class PartialFile(StringIO.StringIO):
       def read(self, *args):
         # file is not visible yet
         test_self.assertNotIn('destination', os.listdir(test_self.upload_dir))
         # it's just a hidden file
-        test_self.assertEqual(['.in.destination.'], os.listdir(test_self.upload_dir))
+        test_self.assertEqual(
+            ['.in.destination.'], os.listdir(test_self.upload_dir))
         return StringIO.StringIO.read(self, *args)
 
     with self._getConnection() as sftp:
@@ -140,13 +135,16 @@ class TestSFTPOperations(ProFTPdTestCase):
   def test_partial_upload_are_deleted(self):
     test_self = self
     with self._getConnection() as sftp:
+
       class ErrorFile(StringIO.StringIO):
         def read(self, *args):
           # at this point, file is already created on server
-          test_self.assertEqual(['.in.destination.'], os.listdir(test_self.upload_dir))
+          test_self.assertEqual(
+              ['.in.destination.'], os.listdir(test_self.upload_dir))
           # simulate a connection closed
           sftp.sftp_client.close()
           return "something that will not be sent to server"
+
       with self.assertRaises(IOError):
         sftp.sftp_client.putfo(ErrorFile(), "destination")
     # no half uploaded file is kept
@@ -164,12 +162,14 @@ class TestSFTPOperations(ProFTPdTestCase):
 
 class TestUserManagement(ProFTPdTestCase):
   def test_user_can_be_added_from_script(self):
-    with self.assertRaisesRegexp(AuthenticationException, 'Authentication failed'):
+    with self.assertRaisesRegexp(AuthenticationException,
+                                 'Authentication failed'):
       self._getConnection(username='bob', password='secret')
 
     subprocess.check_call(
-      'echo secret | %s/bin/ftpasswd --name=bob --stdin' % self.computer_partition_root_path,
-      shell=True)
+        'echo secret | %s/bin/ftpasswd --name=bob --stdin' %
+        self.computer_partition_root_path,
+        shell=True)
     self.assertTrue(self._getConnection(username='bob', password='secret'))
 
 
@@ -177,7 +177,8 @@ class TestBan(ProFTPdTestCase):
   def test_client_are_banned_after_5_wrong_passwords(self):
     # Simulate failed 5 login attempts
     for i in range(5):
-      with self.assertRaisesRegexp(AuthenticationException, 'Authentication failed'):
+      with self.assertRaisesRegexp(AuthenticationException,
+                                   'Authentication failed'):
         self._getConnection(password='wrong')
 
     # after that, even with a valid password we cannot connect
@@ -185,17 +186,19 @@ class TestBan(ProFTPdTestCase):
       self._getConnection()
 
     # ban event is logged
-    with open(os.path.join(
-        self.computer_partition_root_path, 'var', 'log', 'proftpd-ban.log')) as ban_log_file:
+    with open(os.path.join(self.computer_partition_root_path,
+                           'var',
+                           'log',
+                           'proftpd-ban.log')) as ban_log_file:
       self.assertRegexpMatches(
-        ban_log_file.readlines()[-1],
-        'login from host .* denied due to host ban')
+          ban_log_file.readlines()[-1],
+          'login from host .* denied due to host ban')
 
 
 class TestInstanceParameterPort(ProFTPdTestCase):
   @classmethod
   def getInstanceParameterDict(cls):
-    cls.free_port = utils.findFreeTCPPort(cls.config['ipv4_address'])
+    cls.free_port = findFreeTCPPort(cls._ipv4_address)
     return {'port': cls.free_port}
 
   def test_instance_parameter_port(self):
@@ -211,23 +214,26 @@ class TestFilesAndSocketsInInstanceDir(ProFTPdTestCase):
   def setUp(self):
     """sets `self.proftpdProcess` to `psutil.Process` for the running proftpd in the instance.
     """
-    all_process_info = self.getSupervisorRPCServer().supervisor.getAllProcessInfo()
+    with self.slap.instance_supervisor_rpc as supervisor:
+      all_process_info = supervisor.getAllProcessInfo()
     # there is only one process in this instance
     process_info, = [p for p in all_process_info if p['name'] != 'watchdog']
     process = psutil.Process(process_info['pid'])
-    self.assertEqual('proftpd', process.name()) # sanity check
+    self.assertEqual('proftpd', process.name())  # sanity check
     self.proftpdProcess = process
 
   def test_only_write_file_in_instance_dir(self):
     self.assertEqual(
         [],
-        [f for f in self.proftpdProcess.open_files()
-         if f.mode != 'r'
-         if not f.path.startswith(self.computer_partition_root_path)])
+        [
+            f for f in self.proftpdProcess.open_files() if f.mode != 'r'
+            if not f.path.startswith(self.computer_partition_root_path)
+        ])
 
   def test_only_unix_socket_in_instance_dir(self):
     self.assertEqual(
         [],
-        [s for s in self.proftpdProcess.connections('unix')
-         if not s.laddr.startswith(self.computer_partition_root_path)])
-
+        [
+            s for s in self.proftpdProcess.connections('unix')
+            if not s.laddr.startswith(self.computer_partition_root_path)
+        ])

software/proftpd/test/utils.py

-##############################################################################
-#
-# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
-#
-# WARNING: This program as such is intended to be used by professional
-# programmers who take the whole responsibility of assessing all potential
-# consequences resulting from its eventual inadequacies and bugs
-# End users who are looking for a ready-to-use solution with commercial
-# guarantees and support are strongly adviced to contract a Free Software
-# Service Company
-#
-# This program is Free Software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 3
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
-#
-##############################################################################
-
-import unittest
-import os
-import socket
-from contextlib import closing
-import logging
-import StringIO
-import xmlrpclib
-
-import supervisor.xmlrpc
-from erp5.util.testnode.SlapOSControler import SlapOSControler
-from erp5.util.testnode.ProcessManager import ProcessManager
-
-
-# Utility functions
-def findFreeTCPPort(ip=''):
-  """Find a free TCP port to listen to.
-  """
-  family = socket.AF_INET6 if ':' in ip else socket.AF_INET
-  with closing(socket.socket(family, socket.SOCK_STREAM)) as s:
-    s.bind((ip, 0))
-    return s.getsockname()[1]
-
-
-# TODO:
-#  - allow requesting multiple instances ?
-
-class SlapOSInstanceTestCase(unittest.TestCase):
-  """Install one slapos instance.
-
-  This test case install software(s) and request one instance during `setUpClass`
-  and destroy the instance during `tearDownClass`.
-
-  Software Release URL, Instance Software Type and Instance Parameters can be defined
-  on the class.
-
-  All tests from the test class will run with the same instance.
-
-  The following class attributes are available:
-
-    * `computer_partition`:  the computer partition instance, implementing
-    `slapos.slap.interface.slap.IComputerPartition`.
-
-    * `computer_partition_root_path`: the path of the instance root directory.
-
-  """
-
-  # Methods to be defined by subclasses.
-  @classmethod
-  def getSoftwareURLList(cls):
-    """Return URL of software releases to install.
-
-    To be defined by subclasses.
-    """
-    raise NotImplementedError()
-
-  @classmethod
-  def getInstanceParameterDict(cls):
-    """Return instance parameters
-
-    To be defined by subclasses if they need to request instance with specific
-    parameters.
-    """
-    return {}
-
-  @classmethod
-  def getInstanceSoftwareType(cls):
-    """Return software type for instance, default "default"
-
-    To be defined by subclasses if they need to request instance with specific
-    software type.
-    """
-    return "default"
-
-  # Utility methods.
-  def getSupervisorRPCServer(self):
-    """Returns a XML-RPC connection to the supervisor used by slapos node
-
-    Refer to http://supervisord.org/api.html for details of available methods.
-    """
-    # xmlrpc over unix socket https://stackoverflow.com/a/11746051/7294664
-    return xmlrpclib.ServerProxy(
-       'http://slapos-supervisor',
-       transport=supervisor.xmlrpc.SupervisorTransport(
-           None,
-           None,
-           # XXX hardcoded socket path
-           serverurl="unix://{working_directory}/inst/supervisord.socket".format(
-             **self.config)))
-
-  # Unittest methods
-  @classmethod
-  def setUpClass(cls):
-    """Setup the class, build software and request an instance.
-
-    If you have to override this method, do not forget to call this method on
-    parent class.
-    """
-    try:
-      cls.setUpWorkingDirectory()
-      cls.setUpConfig()
-      cls.setUpSlapOSController()
-
-      cls.runSoftwareRelease()
-      # XXX instead of "runSoftwareRelease", it would be better to be closer to slapos usage:
-      # cls.supplySoftwares()
-      # cls.installSoftwares()
-
-      cls.runComputerPartition()
-      # XXX instead of "runComputerPartition", it would be better to be closer to slapos usage:
-      # cls.requestInstances()
-      # cls.createInstances()
-      # cls.requestInstances()
-
-    except Exception:
-      cls.stopSlapOSProcesses()
-      raise
-
-  @classmethod
-  def tearDownClass(cls):
-    """Tear down class, stop the processes and destroy instance.
-    """
-    cls.stopSlapOSProcesses()
-
-  # Implementation
-  @classmethod
-  def stopSlapOSProcesses(cls):
-    if hasattr(cls, '_process_manager'):
-      cls._process_manager.killPreviousRun()
-
-  @classmethod
-  def setUpWorkingDirectory(cls):
-    """Initialise the directories"""
-    cls.working_directory = os.environ.get(
-        'SLAPOS_TEST_WORKING_DIR',
-        os.path.join(os.path.dirname(__file__), '.slapos'))
-    # To prevent error: Cannot open an HTTP server: socket.error reported
-    # AF_UNIX path too long This `working_directory` should not be too deep.
-    # Socket path is 108 char max on linux
-    # https://github.com/torvalds/linux/blob/3848ec5/net/unix/af_unix.c#L234-L238
-    # Supervisord socket name contains the pid number, which is why we add
-    # .xxxxxxx in this check.
-    if len(cls.working_directory + '/inst/supervisord.socket.xxxxxxx') > 108:
-      raise RuntimeError('working directory ( {} ) is too deep, try setting '
-              'SLAPOS_TEST_WORKING_DIR'.format(cls.working_directory))
-
-    if not os.path.exists(cls.working_directory):
-      os.mkdir(cls.working_directory)
-
-  @classmethod
-  def setUpConfig(cls):
-    """Create slapos configuration"""
-    cls.config = {
-      "working_directory": cls.working_directory,
-      "slapos_directory": cls.working_directory,
-      "log_directory": cls.working_directory,
-      "computer_id": 'slapos.test',  # XXX
-      'proxy_database': os.path.join(cls.working_directory, 'proxy.db'),
-      'partition_reference': cls.__name__,
-      # "proper" slapos command must be in $PATH
-      'slapos_binary': 'slapos',
-      'node_quantity': '3',
-    }
-    # Some tests are expecting that local IP is not set to 127.0.0.1
-    ipv4_address = os.environ.get('SLAPOS_TEST_IPV4', '127.0.1.1')
-    ipv6_address = os.environ['SLAPOS_TEST_IPV6']
-
-    cls.config['proxy_host'] = cls.config['ipv4_address'] = ipv4_address
-    cls.config['ipv6_address'] = ipv6_address
-    cls.config['proxy_port'] = findFreeTCPPort(ipv4_address)
-    cls.config['master_url'] = 'http://{proxy_host}:{proxy_port}'.format(
-      **cls.config)
-
-  @classmethod
-  def setUpSlapOSController(cls):
-    """Create the a "slapos controller" and supply softwares from `getSoftwareURLList`.
-
-    This is equivalent to:
-
-    slapos proxy start
-    for sr in getSoftwareURLList; do
-      slapos supply $SR $COMP
-    done
-    """
-    cls._process_manager = ProcessManager()
-
-    # XXX this code is copied from testnode code
-    cls.slapos_controler = SlapOSControler(
-        cls.working_directory,
-        cls.config
-    )
-
-    slapproxy_log = os.path.join(cls.config['log_directory'], 'slapproxy.log')
-    logger = logging.getLogger(__name__)
-    logger.debug('Configured slapproxy log to %r', slapproxy_log)
-
-    cls.software_url_list = cls.getSoftwareURLList()
-    cls.slapos_controler.initializeSlapOSControler(
-        slapproxy_log=slapproxy_log,
-        process_manager=cls._process_manager,
-        reset_software=False,
-        software_path_list=cls.software_url_list)
-
-    # XXX we should check *earlier* if that pidfile exist and if supervisord
-    # process still running, because if developer started supervisord (or bugs?)
-    # then another supervisord will start and starting services a second time
-    # will fail.
-    cls._process_manager.supervisord_pid_file = os.path.join(
-      cls.slapos_controler.instance_root, 'var', 'run', 'supervisord.pid')
-
-  @classmethod
-  def runSoftwareRelease(cls):
-    """Run all the software releases that were supplied before.
-
-    This is the equivalent of `slapos node software`.
-
-    The tests will be marked file if software building fail.
-    """
-    logger = logging.getLogger()
-    logger.level = logging.DEBUG
-    stream = StringIO.StringIO()
-    stream_handler = logging.StreamHandler(stream)
-    logger.addHandler(stream_handler)
-
-    try:
-      cls.software_status_dict = cls.slapos_controler.runSoftwareRelease(
-        cls.config, environment=os.environ)
-      stream.seek(0)
-      stream.flush()
-      message = ''.join(stream.readlines()[-100:])
-      assert cls.software_status_dict['status_code'] == 0, message
-    finally:
-      logger.removeHandler(stream_handler)
-      del stream
-
-
-  @classmethod
-  def runComputerPartition(cls):
-    """Instanciate the software.
-
-    This is the equivalent of doing:
-
-    slapos request --type=getInstanceSoftwareType --parameters=getInstanceParameterDict
-    slapos node instance
-
-    and return the slapos request instance parameters.
-
-    This can be called by tests to simulate re-request with different parameters.
-    """
-    logger = logging.getLogger()
-    logger.level = logging.DEBUG
-    stream = StringIO.StringIO()
-    stream_handler = logging.StreamHandler(stream)
-    logger.addHandler(stream_handler)
-
-    if cls.getInstanceSoftwareType() != 'default':
-      raise NotImplementedError
-
-    instance_parameter_dict = cls.getInstanceParameterDict()
-    try:
-      cls.instance_status_dict = cls.slapos_controler.runComputerPartition(
-        cls.config,
-        cluster_configuration=instance_parameter_dict,
-        environment=os.environ)
-      stream.seek(0)
-      stream.flush()
-      message = ''.join(stream.readlines()[-100:])
-      assert cls.instance_status_dict['status_code'] == 0, message
-    finally:
-      logger.removeHandler(stream_handler)
-      del stream
-
-    # FIXME: similar to test node, only one (root) partition is really
-    #        supported for now.
-    computer_partition_list = []
-    for i in range(len(cls.software_url_list)):
-      computer_partition_list.append(
-          cls.slapos_controler.slap.registerOpenOrder().request(
-            cls.software_url_list[i],
-            # This is how testnode's SlapOSControler name created partitions
-            partition_reference='testing partition {i}'.format(
-              i=i, **cls.config),
-            partition_parameter_kw=instance_parameter_dict))
-
-    # expose some class attributes so that tests can use them:
-    # the ComputerPartition instances, to getInstanceParameterDict
-    cls.computer_partition = computer_partition_list[0]
-
-    # the path of the instance on the filesystem, for low level inspection
-    cls.computer_partition_root_path = os.path.join(
-        cls.config['working_directory'],
-        'inst',
-        cls.computer_partition.getId())
-
-
-