Reduce verbosity of tests

SlapOS test is too verbose, see http://community.slapos.org/test_result_module/20181211-426F13FE/6 as an example. The main goal of this work is to make this easier to read.

Other improvements:
 * introduce `subprocess32` on python2 to use new subprocess features on python2
 * honor `SLAPOS_TEST_DEBUG` environment variable to output logs and also enable debugging utilities ( unittest's `Ctrl+C` handler )


/reviewed-on nexedi/slapos.core!73

setup.py

@@ -2,6 +2,7 @@ from setuptools import setup, find_packages
 from shutil import copyfile
 import glob
 import os
+import sys
 
 from slapos.version import version
 name = 'slapos.core'
@@ -27,6 +28,9 @@ try:
 except ImportError:
   additional_install_requires.append('argparse')
 
+if sys.version_info[0] < 3:
+  additional_install_requires.append('subprocess32')
+
 setup(name=name,
       version=version,
       description="SlapOS core.",

slapos/proxy/views.py

@@ -28,8 +28,6 @@
 #
 ##############################################################################
 
-from __future__ import print_function
-
 from lxml import etree
 import random
 import string
@@ -121,14 +119,13 @@ def partitiondict2partition(partition):
   return slap_partition
 
 
-def execute_db(table, query, args=(), one=False, db_version=None, log=False, db=None):
+def execute_db(table, query, args=(), one=False, db_version=None, db=None):
   if not db:
     db = g.db
   if not db_version:
     db_version = DB_VERSION
   query = query % (table + db_version,)
-  if log:
-    print(query)
+  app.logger.debug(query)
   try:
     cur = db.execute(query, args)
   except:
@@ -189,7 +186,7 @@ def _upgradeDatabaseIfNeeded():
       columns = ', '.join(row.keys())
       placeholders = ':'+', :'.join(row.keys())
       query = 'INSERT INTO %s (%s) VALUES (%s)' % ('%s', columns, placeholders)
-      execute_db(table, query, row, log=True)
+      execute_db(table, query, row)
 
   g.db.commit()
 

slapos/tests/__init__.py

+##############################################################################
+#
+# Copyright (c) 2018 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import unittest
+import logging
+import os
+
+# When running with debug enabled install Ctrl+C handler and output more logs,
+# otherwise disable log output.
+if os.environ.get('SLAPOS_TEST_DEBUG'):
+  unittest.installHandler()
+  logging.basicConfig(level=logging.DEBUG)
+else:
+  logging.basicConfig(filename=os.devnull)

slapos/tests/promise.py

@@ -93,7 +93,7 @@ class TestSlapOSPromiseMixin(unittest.TestCase):
 
     self.launcher = PromiseLauncher(
       config=parameter_dict,
-      logger=None,
+      logger=logging.getLogger('slapos.test.promise'),
       dry_run=dry_run
     )
     if save_method:
@@ -116,13 +116,12 @@ class TestSlapOSPromiseMixin(unittest.TestCase):
 
   def createPromiseProcess(self, promise_name, check_anomaly=False, wrap=False):
 
-    logging.basicConfig()
     promise_path = os.path.join(self.plugin_dir, promise_name)
     return PromiseProcess(
       self.partition_dir,
       promise_name,
       promise_path,
-      logger=logging.getLogger('grid.promise'),
+      logger=logging.getLogger('slapos.test.promise'),
       argument_dict=self.genPromiseConfigDict(promise_name),
       check_anomaly=check_anomaly,
       wrap=wrap,
@@ -1091,13 +1090,11 @@ class TestSlapOSGenericPromise(TestSlapOSPromiseMixin):
     }
 
   def createPromiseProcess(self, check_anomaly=False, wrap=False):
-
-    logging.basicConfig()
     return PromiseProcess(
       self.partition_dir,
       self.promise_name,
       self.promise_path,
-      logger=logging.getLogger('grid.promise'),
+      logger=logging.getLogger('slapos.test.promise'),
       argument_dict=self.promise_config,
       check_anomaly=check_anomaly,
       wrap=wrap,

slapos/tests/slap.py

@@ -25,13 +25,12 @@
 #
 ##############################################################################
 
-from __future__ import print_function
-
 import logging
 import os
 import unittest
 from six.moves.urllib import parse
 import tempfile
+import logging
 
 import httmock
 
@@ -43,6 +42,9 @@ class UndefinedYetException(Exception):
   """To catch exceptions which are not yet defined"""
 
 
+logger = logging.getLogger('slapos.tests.slap')
+
+
 class SlapMixin(unittest.TestCase):
   """
   Useful methods for slap tests
@@ -53,7 +55,7 @@ class SlapMixin(unittest.TestCase):
       self.server_url = 'http://localhost/'
     else:
       self.server_url = self._server_url
-    print('Testing against SLAP server %r' % self.server_url)
+    logger.debug('Testing against SLAP server %r', self.server_url)
     self.slap = slapos.slap.slap()
     self.partition_id = 'PARTITION_01'
     os.environ.pop('SLAPGRID_INSTANCE_ROOT', None)
@@ -71,6 +73,9 @@ class SlapMixin(unittest.TestCase):
 class TestSlap(SlapMixin):
   """
   Test slap against slap server
+
+  This test can be used to test a running SLAP server by setting
+  TEST_SLAP_SERVER_URL environment variable to the URL of this server.
   """
 
   def test_slap_initialisation(self):
@@ -1205,8 +1210,3 @@ class TestSoftwareProductCollection(SlapMixin):
     )
     self.assertEqual(self.product_collection.foo, '0')
 
-if __name__ == '__main__':
-  print('You can point to any SLAP server by setting TEST_SLAP_SERVER_URL'
-        ' environment variable')
-  unittest.main()
-

slapos/tests/slapproxy/__init__.py

@@ -34,7 +34,10 @@ import os
 import logging
 import shutil
 import socket
-import subprocess
+try:
+  import subprocess32 as subprocess
+except ImportError:
+  import subprocess
 import sys
 import tempfile
 import time
@@ -1029,6 +1032,8 @@ database_uri = %(tempdir)s/lib/external_proxy.db
             sys.executable, '%s/../cli/entry.py' % os.path.dirname(slapos.tests.__file__),
             'proxy', 'start', '--cfg', self.external_slapproxy_configuration_file_location
         ],
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.DEVNULL,
         env={"PYTHONPATH": ':'.join(sys.path)}
     )
     # Wait a bit for proxy to be started