erp5.util: add support for Python 3

erp5/tests/testERP5TestNode.py

@@ -23,7 +23,6 @@ import sys
 import tempfile
 import json
 import time
-import types
 import re
 
 @contextmanager
@@ -144,7 +143,8 @@ class ERP5TestNode(TestCase):
         self.__dict__.update(**kw)
 
       def __call__(self, command):
-        return subprocess.check_output(command, **self.__dict__)
+        return subprocess.check_output(command, universal_newlines=True,
+                                       **self.__dict__)
     return Caller(**kw)
 
   def generateTestRepositoryList(self, add_third_repository=False):
@@ -338,7 +338,7 @@ shared = true
     vcs_repository_info['branch'] = 'master'
     rev_list = self.getAndUpdateFullRevisionList(test_node, node_test_suite)
     output = call("git branch".split()).strip()
-    print output
+    print(output)
     self.assertTrue("* master" in output.split('\n'))
     # Add a third branch on remote, make sure we could switch to it
     remote_call = self.getCaller(cwd=self.remote_repository2)
@@ -489,35 +489,21 @@ shared = true
     test_node.purgeOldTestSuite(test_suite_data)
     self.assertEquals(['foo'], os.listdir(self.working_directory))
 
-  def test_purgeOldTestSuiteChmodNonWriteable(self):
+  def test_purgeOldTestSuiteChmod(self):
     """Old test suites can be deleted even when some files/directories have
-    been chmod'd to make non-writeable  """
+    been chmod'd to make read only.  """
     test_node = self.getTestNode()
     test_suite_data = self.getTestSuiteData(add_third_repository=True)
     os.mkdir(os.path.join(self.working_directory, 'bar'))
     non_writable_file = open(os.path.join(self.working_directory, 'bar', 'non-writable-file'), 'w')
     non_writable_file.close()
-
-    os.chmod(os.path.join(self.working_directory, 'bar', 'non-writable-file'), 0o400) # -r--------
-    os.chmod(os.path.join(self.working_directory, 'bar'), 0o500) # dr-x------
+    # make this file and directory non writeable
+    os.chmod(os.path.join(self.working_directory, 'bar', 'non-writable-file'), 0o000)
+    os.chmod(os.path.join(self.working_directory, 'bar'), 0o000)
 
     test_node.purgeOldTestSuite(test_suite_data) # should not fail
     self.assertEqual([], os.listdir(self.working_directory))
 
-  def test_purgeOldTestSuiteChmodNonWriteableNonReadable(self):
-    """Old test suites can be deleted even when some files/directories have
-    been chmod'd to make them non readable and non writeable. """
-    test_node = self.getTestNode()
-    test_suite_data = self.getTestSuiteData(add_third_repository=True)
-    os.mkdir(os.path.join(self.working_directory, 'bar'))
-    non_writable_file = open(os.path.join(self.working_directory, 'bar', 'non-writable-file'), 'w')
-    non_writable_file.close()
-
-    os.chmod(os.path.join(self.working_directory, 'bar', 'non-writable-file'), 0o000) # ----------
-    os.chmod(os.path.join(self.working_directory, 'bar'), 0o000) # d---------
-
-    test_node.purgeOldTestSuite(test_suite_data) # should not fail
-    self.assertEqual([], os.listdir(self.working_directory))
 
   def test_09_runTestSuite(self, my_test_type='UnitTest'):
     """
@@ -986,7 +972,7 @@ shared = true
       self.assertRaises(SubprocessError, callPrepareSlapOS)
 
     self.assertEquals(node_test_suite.retry_software_count, 0)
-    for x in xrange(0,11):
+    for x in range(11):
       callRaisingPrepareSlapos()
     self.assertEquals(len(init_call_kw_list), 11)
     self.assertEquals(init_call_kw_list[-1]['reset_software'], False)

erp5/util/scalability/requestUrl.py

@@ -27,7 +27,7 @@ def main():
   if error_message_set:
     exit_status = 1
     for error in error_message_set:
-      print error
+      print(error)
   elif result:
-    print result
+    print(result)
   sys.exit(exit_status)

erp5/util/scalability/runScalabilityTestSuite.py

@@ -5,23 +5,15 @@ import os
 import shutil
 import time
 import sys
-import multiprocessing
-import signal
-import errno
 import json
 import logging
 import logging.handlers
 import glob
-import urlparse
-import httplib
-import base64
 import threading
-from erp5.util.benchmark.argument import ArgumentType
-from erp5.util.benchmark.performance_tester import PerformanceTester
 from erp5.util.benchmark.thread import TestThread, TestMetricThread
 from erp5.util import taskdistribution
 from erp5.util.testnode import Utils
-from erp5.util.testnode.ProcessManager import SubprocessError, ProcessManager, CancellationError
+from erp5.util.testnode.ProcessManager import ProcessManager
 import datetime
 
 MAX_INSTALLATION_TIME = 60*50
@@ -179,8 +171,8 @@ class ScalabilityLauncher(object):
     """
     data_array = self.__argumentNamespace.current_test_data.split(',')
     data = json.dumps({"count": data_array[0], "title": data_array[1], "relative_path": data_array[2]})
-    decoded_data = Utils.deunicodeData(json.loads(data))
-    return ScalabilityTest(decoded_data, self.test_result)
+    encoded_data = Utils.deunicodeData(json.loads(data))
+    return ScalabilityTest(encoded_data, self.test_result)
 
   def clearUsersFile(self, user_file_path):
     self.log("Clearing users file: %s" % user_file_path)

erp5/util/taskdistribution/__init__.py

@@ -40,11 +40,15 @@ Example use:
       test_line.stop()
 """
 from __future__ import print_function
-import httplib
+import six
+from six.moves import (
+  map,
+  http_client as httplib,
+  xmlrpc_client as xmlrpclib,
+)
 import socket
 import threading
 import time
-import xmlrpclib
 
 __all__ = ['TaskDistributor', 'TestResultProxy', 'TestResultLineProxy', 'patchRPCParser']
 
@@ -89,11 +93,17 @@ def patchRPCParser(error_handler):
     def verbose_feed(self, data):
         try:
             return original_feed(self, data)
-        except Exception, exc:
+        except Exception as exc:
             if not error_handler(data, exc):
                 raise
     parser_klass.feed = verbose_feed
 
+try: # PY3
+  basestring
+except NameError:
+  basestring = bytes, str
+  unicode = str
+
 def binarize_args(arg):
   # Converts recursively basestring arg into xmlrpclib.Binary, as they can
   # contain non-XML allowed characters
@@ -102,9 +112,9 @@ def binarize_args(arg):
       arg = arg.encode('utf-8')
     return xmlrpclib.Binary(arg)
   if isinstance(arg, (list, tuple, set)):
-    return map(binarize_args, arg)
+    return list(map(binarize_args, arg))
   if isinstance(arg, dict):
-    return {k: binarize_args(v) for k, v in arg.iteritems()}
+    return {k: binarize_args(v) for k, v in six.iteritems(arg)}
   return arg
 
 class RPCRetry(object):
@@ -349,7 +359,7 @@ class TestResultProxy(RPCRetry):
             caption_list = []
             append = caption_list.append
             for name, (stream, max_history_bytes) in \
-                    self._watcher_dict.iteritems():
+                    six.iteritems(self._watcher_dict):
                 append('==> %s <==' % (name, ))
                 start = stream.tell()
                 stream.seek(0, 2)

erp5/util/testnode/NodeTestSuite.py

@@ -32,6 +32,8 @@ import string
 import random
 from .Utils import createFolder
 
+from six.moves import range
+
 class SlapOSInstance(object):
   """
   Base of an software instance,
@@ -68,14 +70,14 @@ class NodeTestSuite(SlapOSInstance):
 
   def createSuiteLog(self):
     # /srv/slapgrid/slappartXX/srv/var/log/testnode/az-D27KqX7FxJ/suite.log
-    alphabets = string.digits + string.letters
+    alphabets = string.digits + string.ascii_letters
     while 1:
       log_folder_name = '%s-%s' % (self.reference,
-        ''.join(random.choice(alphabets) for i in xrange(10)))
+        ''.join(random.choice(alphabets) for i in range(10)))
       log_folder_path = os.path.join(self.log_directory, log_folder_name)
       try:
         os.makedirs(log_folder_path)
-      except OSError, e:
+      except OSError as e:
         if e.errno != errno.EEXIST:
           raise
       else:

erp5/util/testnode/ProcessManager.py

@@ -33,6 +33,7 @@ import signal
 import sys
 import time
 from . import logger
+from slapos.util import bytes2str
 
 MAX_TIMEOUT = 3600 * 4
 
@@ -79,7 +80,7 @@ def subprocess_capture(p, log_prefix, get_output=True):
         break
       if get_output:
         buffer.append(data)
-      log(log_prefix + data.rstrip('\n'))
+      log(log_prefix + bytes2str(data).rstrip('\n'))
   if p.stdout:
     stdout = []
     stdout_thread = threading.Thread(target=readerthread,
@@ -97,8 +98,8 @@ def subprocess_capture(p, log_prefix, get_output=True):
     stdout_thread.join()
   if p.stderr:
     stderr_thread.join()
-  return (p.stdout and ''.join(stdout),
-          p.stderr and ''.join(stderr))
+  return (p.stdout and b''.join(stdout),
+          p.stderr and b''.join(stderr))
 
 def killCommand(pid):
   """
@@ -109,7 +110,7 @@ def killCommand(pid):
   try:
     process = psutil.Process(pid)
     process.suspend()
-  except psutil.Error, e:
+  except psutil.Error as e:
     return
   process_list = [process]
   new_list = process.children(recursive=True)
@@ -118,19 +119,19 @@ def killCommand(pid):
     for child in new_list:
       try:
         child.suspend()
-      except psutil.Error, e:
+      except psutil.Error as e:
         logger.debug("killCommand/suspend: %s", e)
     time.sleep(1)
     new_list = set(process.children(recursive=True)).difference(process_list)
   for process in process_list:
     try:
       process.kill()
-    except psutil.Error, e:
+    except psutil.Error as e:
       logger.debug("killCommand/kill: %s", e)
 
 class ProcessManager(object):
 
-  stdin = file(os.devnull)
+  stdin = open(os.devnull)
 
   def __init__(self, max_timeout=MAX_TIMEOUT):
     self.process_pid_set = set()

erp5/util/testnode/ScalabilityTestRunner.py

@@ -30,30 +30,31 @@ import subprocess
 import sys
 import time
 import glob
-import SlapOSControler
-import SlapOSMasterCommunicator
+from . import SlapOSControler, SlapOSMasterCommunicator
 import json
 import time
 import shutil
 import logging
 import string
 import random
-import urlparse
+from six.moves.urllib import parse
 import base64
-import httplib
-import Utils
+from six.moves import http_client as httplib
+from . import Utils
 import requests
 import slapos.slap
-import cPickle as pickle
-from ProcessManager import SubprocessError, ProcessManager, CancellationError
+from six.moves import cPickle as pickle
+from .ProcessManager import SubprocessError, ProcessManager, CancellationError
 from subprocess import CalledProcessError
-from Updater import Updater
+from .Updater import Updater
 from erp5.util import taskdistribution
 from erp5.util.benchmark.thread import TestThread
 # for dummy slapos answer
 import signal
 from . import logger
 
+from six.moves import range
+
 # max time to generate frontend instance: 1.5 hour
 MAX_FRONTEND_TIME = 60*90
 # max time to register instance to slapOSMaster: 5 minutes
@@ -333,7 +334,7 @@ Require valid-user
     htaccess_file.close()
     password_path = testsuite_directory + PASSWORD_FILE
     with open(password_path, "w") as password_file:
-      password = ''.join(random.choice(string.digits + string.letters) for i in xrange(PASSWORD_LENGTH))
+      password = ''.join(random.choice(string.digits + string.letters) for i in range(PASSWORD_LENGTH))
       password_file.write(password)
     user = TESTNODE_USER
     command = [apache_htpasswd, "-bc", testsuite_directory + HTPASSWD, user, password]
@@ -363,7 +364,7 @@ Require valid-user
     user, password = self.generateProfilePasswordAccess()
     logger.info("Software Profile password: %s" % password)
     self.reachable_profile = "https://%s:%s@%s" % (user, password,
-      os.path.join(urlparse.urlparse(self.testnode.config['frontend_url']).netloc,
+      os.path.join(parse.urlparse(self.testnode.config['frontend_url']).netloc,
                    "software", self.randomized_path, "software.cfg"))
 
   def prepareSlapOSForTestSuite(self, node_test_suite):
@@ -526,7 +527,7 @@ Require valid-user
     if not self.launchable:
       return {'status_code' : 1, 'error_message': "Current test_suite is not actually launchable." }
     configuration_list = node_test_suite.configuration_list
-    test_list = range(0, len(configuration_list))
+    test_list = list(range(len(configuration_list)))
     try:
       test_result_proxy = self.testnode.taskdistribution.createTestResult(
                           node_test_suite.revision, test_list,

erp5/util/testnode/SlapOSControler.py

@@ -35,6 +35,8 @@ from slapos import client
 from . import logger
 from .Utils import createFolder
 
+from six.moves import range
+
 MAX_PARTITIONS = 10
 MAX_SR_RETRIES = 3
 
@@ -245,7 +247,7 @@ class SlapOSControler(object):
         computer = slap.registerComputer(config['computer_id'])
         # Call a method to ensure connection to master can be established
         computer.getComputerPartitionList()
-      except slapos.slap.ConnectionError, e:
+      except slapos.slap.ConnectionError as e:
         retries += 1
         if retries >= 60:
           raise
@@ -272,7 +274,7 @@ class SlapOSControler(object):
     # MySQL DB content) from previous runs. To support changes of partition
     # naming scheme (which already happened), do this at instance_root level.
     createFolder(instance_root, True)
-    for i in xrange(MAX_PARTITIONS):
+    for i in range(MAX_PARTITIONS):
       # create partition and configure computer
       # XXX: at the moment all partitions do share same virtual interface address
       # this is not a problem as usually all services are on different ports
@@ -280,7 +282,7 @@ class SlapOSControler(object):
       partition_path = os.path.join(instance_root, partition_reference)
       if not(os.path.exists(partition_path)):
         os.mkdir(partition_path)
-      os.chmod(partition_path, 0750)
+      os.chmod(partition_path, 0o750)
       computer.updateConfiguration(xml_marshaller.xml_marshaller.dumps({
            'address': config['ipv4_address'],
            'instance_root': instance_root,
@@ -320,7 +322,7 @@ class SlapOSControler(object):
     os.environ['PATH'] = environment['PATH']
     # a SR may fail for number of reasons (incl. network failures)
     # so be tolerant and run it a few times before giving up
-    for _ in xrange(MAX_SR_RETRIES):
+    for _ in range(MAX_SR_RETRIES):
       status_dict = self.spawn(config['slapos_binary'],
                  'node', 'software', '--all', 
                  '--pidfile', os.path.join(self.software_root, 'slapos-node.pid'),
@@ -348,7 +350,7 @@ class SlapOSControler(object):
     # try to run for all partitions as one partition may in theory request another one 
     # this not always is required but curently no way to know how "tree" of partitions
     # may "expand"
-    for _ in xrange(max_quantity):
+    for _ in range(max_quantity):
       status_dict = self.spawn(config['slapos_binary'], 'node', 'instance', 
                  '--pidfile', os.path.join(self.instance_root, 'slapos-node.pid'),
                  '--cfg', self.slapos_config, raise_error_if_fail=False,

erp5/util/testnode/SlapOSMasterCommunicator.py

+from __future__ import print_function
+
 import datetime
 import json
 import traceback
@@ -9,9 +11,11 @@ from uritemplate import expand
 import slapos.slap
 from slapos.slap import SoftwareProductCollection
 from requests.exceptions import HTTPError
-from ..taskdistribution import SAFE_RPC_EXCEPTION_LIST
+from erp5.util.taskdistribution import SAFE_RPC_EXCEPTION_LIST
 from . import logger
 
+import six
+
 # max time to instance changing state: 3 hour
 MAX_INSTANCE_TIME = 60*60*3
 
@@ -52,7 +56,7 @@ def retryOnNetworkFailure(func,
       except _except_list:
         traceback.print_exc()
 
-      print 'Network failure. Retry method %s in %i seconds' % (func, retry_time)
+      print('Network failure. Retry method %s in %i seconds' % (func, retry_time))
       time.sleep(retry_time)
       retry_time = min(retry_time*1.5, 640)
 
@@ -92,8 +96,7 @@ class SlapOSMasterCommunicator(object):
     if instance_title is not None:
       self.name = instance_title 
     if request_kw is not None:
-      if isinstance(request_kw, basestring) or \
-        isinstance(request_kw, unicode):
+      if isinstance(request_kw, (six.binary_type, six.text_type)):
         self.request_kw = json.loads(request_kw)
       else:
         self.request_kw = request_kw
@@ -214,7 +217,7 @@ class SlapOSMasterCommunicator(object):
     result = self.hateoas_navigator.GET(url)
     result = json.loads(result)
     if result['_links'].get('action_object_slap', None) is None:
-      print result['links']
+      print(result['links'])
       return None
 
     object_link = self.hateoas_navigator.hateoasGetLinkFromLinks(
@@ -385,8 +388,7 @@ class SlapOSTester(SlapOSMasterCommunicator):
     self.name = name
     self.computer_guid = computer_guid
 
-    if isinstance(request_kw, str) or \
-      isinstance(request_kw, unicode):
+    if isinstance(request_kw, (six.binary_type, six.text_type)):
       self.request_kw = json.loads(request_kw)
     else:
       self.request_kw = request_kw

erp5/util/testnode/Updater.py

@@ -30,6 +30,7 @@ import re
 from . import logger
 from .ProcessManager import SubprocessError
 from .Utils import rmtree
+from slapos.util import bytes2str, str2bytes
 
 SVN_UP_REV = re.compile(r'^(?:At|Updated to) revision (\d+).$')
 SVN_CHANGED_REV = re.compile(r'^Last Changed Rev.*:\s*(\d+)', re.MULTILINE)
@@ -82,7 +83,7 @@ class Updater(object):
           # allow several processes clean the same folder at the same time
           try:
             os.remove(os.path.join(path, file))
-          except OSError, e:
+          except OSError as e:
             if e.errno != errno.ENOENT:
               raise
 
@@ -96,7 +97,7 @@ class Updater(object):
                                       **kw)
 
   def _git(self, *args, **kw):
-    return self.spawn(self.git_binary, *args, **kw)['stdout'].strip()
+    return bytes2str(self.spawn(self.git_binary, *args, **kw)['stdout'].strip())
 
   def git_update_server_info(self):
     return self._git('update-server-info', '-f')
@@ -219,7 +220,7 @@ class Updater(object):
         self.deletePycFiles(path)
         try:
           status_dict = self.spawn(*args)
-        except SubprocessError, e:
+        except SubprocessError as e:
           if 'cleanup' not in e.stderr:
             raise
           self.spawn('svn', 'cleanup', path)

erp5/util/testnode/Utils.py

@@ -3,6 +3,9 @@ import stat
 import shutil
 import errno
 
+import six
+from six.moves import map
+
 def rmtree(path):
   """Delete a path recursively.
 
@@ -11,14 +14,22 @@ def rmtree(path):
   def chmod_retry(func, failed_path, exc_info):
     """Make sure the directories are executable and writable.
     """
+    # Depending on the Python version, the following items differ.
+    if six.PY3:
+      expected_error_type = PermissionError
+      expected_func = os.lstat
+    else:
+      expected_error_type = OSError
+      expected_func = os.listdir
+
     e = exc_info[1]
-    if isinstance(e, OSError):
+    if isinstance(e, expected_error_type):
       if e.errno == errno.ENOENT:
         # because we are calling again rmtree on listdir errors, this path might
         # have been already deleted by the recursive call to rmtree.
         return
       if e.errno == errno.EACCES:
-        if func is os.listdir:
+        if func is expected_func:
           os.chmod(failed_path, 0o700)
           # corner case to handle errors in listing directories.
           # https://bugs.python.org/issue8523
@@ -39,12 +50,16 @@ def createFolder(folder, clean=False):
     rmtree(folder)
   os.mkdir(folder)
 
-def deunicodeData(data):
-  if isinstance(data, list):
-    return map(deunicodeData, data)
-  if isinstance(data, unicode):
-    return data.encode('utf8')
-  if isinstance(data, dict):
-    return {deunicodeData(key): deunicodeData(value)
-            for key, value in data.iteritems()}
-  return data
+if six.PY3:
+  def deunicodeData(data):
+    return data
+else:
+  def deunicodeData(data):
+    if isinstance(data, list):
+      return list(map(deunicodeData, data))
+    if isinstance(data, unicode):
+      return data.encode('utf8')
+    if isinstance(data, dict):
+      return {deunicodeData(key): deunicodeData(value)
+              for key, value in six.iteritems(data)}
+    return data

erp5/util/testnode/__init__.py

@@ -24,7 +24,7 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
-import ConfigParser
+from six.moves import configparser
 import argparse
 import logging
 import logging.handlers
@@ -64,7 +64,7 @@ def main(*args):
   CONFIG = {
     'partition_reference': 'test0',
   }
-  config = ConfigParser.SafeConfigParser()
+  config = configparser.SafeConfigParser()
   # do not change case of option keys
   config.optionxform = str
   config.readfp(parsed_argument.configuration_file[0])

erp5/util/testnode/testnode.py

@@ -39,7 +39,7 @@ from .ScalabilityTestRunner import ScalabilityTestRunner
 from .UnitTestRunner import UnitTestRunner
 from .Utils import deunicodeData
 from .Utils import rmtree
-from .. import taskdistribution
+from erp5.util import taskdistribution
 
 MAX_LOG_TIME = 15 # time in days we should keep logs that we can see through
                   # httd

erp5/util/testsuite/__init__.py

+from __future__ import print_function
+
 import argparse
 import re, os, shlex, glob
 import sys, threading, subprocess
 import traceback
 import errno
+import pprint
+import six
+from six.moves import range
 
 from erp5.util import taskdistribution
-from pprint import pprint
 
+if six.PY3:
+  stdbin = lambda x: x.buffer
+else:
+  stdbin = lambda x: x
+
+# PY3: use shlex.quote
 _format_command_search = re.compile("[[\\s $({?*\\`#~';<>&|]").search
 _format_command_escape = lambda s: "'%s'" % r"'\''".join(s.split("'"))
 def format_command(*args, **kw):
@@ -31,7 +41,7 @@ def subprocess_capture(p, quiet=False):
       buffer.append(data)
   if p.stdout:
     stdout = []
-    output = quiet and (lambda data: None) or sys.stdout.write
+    output = (lambda data: None) if quiet else stdbin(sys.stdout).write
     stdout_thread = threading.Thread(target=readerthread,
                                      args=(p.stdout, output, stdout))
     stdout_thread.setDaemon(True)
@@ -39,7 +49,7 @@ def subprocess_capture(p, quiet=False):
   if p.stderr:
     stderr = []
     stderr_thread = threading.Thread(target=readerthread,
-                                     args=(p.stderr, sys.stderr.write, stderr))
+        args=(p.stderr, stdbin(sys.stderr).write, stderr))
     stderr_thread.setDaemon(True)
     stderr_thread.start()
   if p.stdout:
@@ -47,8 +57,8 @@ def subprocess_capture(p, quiet=False):
   if p.stderr:
     stderr_thread.join()
   p.wait()
-  return (p.stdout and ''.join(stdout),
-          p.stderr and ''.join(stderr))
+  return (p.stdout and b''.join(stdout),
+          p.stderr and b''.join(stderr))
 
 class SubprocessError(EnvironmentError):
   def __init__(self, status_dict):
@@ -72,15 +82,15 @@ class Persistent(object):
   def __getattr__(self, attr):
     if attr == '_db':
       try:
-        db = file(self._filename, 'r+')
-      except IOError, e:
+        db = open(self._filename, 'r+')
+      except IOError as e:
         if e.errno != errno.ENOENT:
           raise
-        db = file(self._filename, 'w+')
+        db = open(self._filename, 'w+')
       else:
         try:
           self.__dict__.update(eval(db.read()))
-        except StandardError:
+        except Exception:
           pass
       self._db = db
       return db
@@ -89,7 +99,7 @@ class Persistent(object):
 
   def sync(self):
     self._db.seek(0)
-    db = dict(x for x in self.__dict__.iteritems() if x[0][:1] != '_')
+    db = dict(x for x in six.iteritems(self.__dict__) if x[0][:1] != '_')
     pprint.pprint(db, self._db)
     self._db.truncate()
 
@@ -103,10 +113,10 @@ class TestSuite(object):
   """
 
   RUN_RE = re.compile(
-    r'Ran (?P<all_tests>\d+) tests? in (?P<seconds>\d+\.\d+)s',
+    br'Ran (?P<all_tests>\d+) tests? in (?P<seconds>\d+\.\d+)s',
     re.DOTALL)
 
-  STATUS_RE = re.compile(r"""
+  STATUS_RE = re.compile(br"""
     (OK|FAILED)\s+\(
       (failures=(?P<failures>\d+),?\s*)?
       (errors=(?P<errors>\d+),?\s*)?
@@ -117,7 +127,7 @@ class TestSuite(object):
     """, re.DOTALL | re.VERBOSE)
 
   SUB_STATUS_RE = re.compile(
-      r"""SUB\s+RESULT:\s+(?P<all_tests>\d+)\s+Tests,\s+
+      br"""SUB\s+RESULT:\s+(?P<all_tests>\d+)\s+Tests,\s+
       (?P<failures>\d+)\s+Failures\s*
       \(?
         (skipped=(?P<skips>\d+),?\s*)?
@@ -130,7 +140,10 @@ class TestSuite(object):
   mysql_db_count = 1
   allow_restart = False
   realtime_output = True
-  stdin = file(os.devnull)
+  try: # PY3
+    stdin = subprocess.DEVNULL
+  except AttributeError:
+    stdin = open(os.devnull, 'rb')
 
   def __init__(self, max_instance_count, **kw):
     self.__dict__.update(kw)
@@ -139,8 +152,8 @@ class TestSuite(object):
     self.acquire = pool.acquire
     self.release = pool.release
     self._instance = threading.local()
-    self._pool = max_instance_count == 1 and [None] or \
-                 range(1, max_instance_count + 1)
+    self._pool = [None] if max_instance_count == 1 else \
+                 list(range(1, max_instance_count + 1))
     self._ready = set()
     self.running = {}
     if max_instance_count != 1:
@@ -185,13 +198,14 @@ class TestSuite(object):
 
   def spawn(self, *args, **kw):
     quiet = kw.pop('quiet', False)
+    cwd = kw.pop('cwd', None)
     env = kw and dict(os.environ, **kw) or None
     command = format_command(*args, **kw)
-    print '\n$ ' + command
+    print('\n$ ' + command)
     sys.stdout.flush()
     try:
       p = subprocess.Popen(args, stdin=self.stdin, stdout=subprocess.PIPE,
-                           stderr=subprocess.PIPE, env=env)
+                           stderr=subprocess.PIPE, env=env, cwd=cwd)
     except Exception:
       # Catch any exception here, to warn user instead of beeing silent,
       # by generating fake error result
@@ -229,20 +243,11 @@ class EggTestSuite(TestSuite):
   The python interpreter is ``python_interpreter``
   """
   def run(self, test):
-    print test
-    original_dir = os.getcwd()
-    try:
-      os.chdir(self.egg_test_path_dict[test])
-      return self.runUnitTest(test)
-    finally:
-      os.chdir(original_dir)
-
-  def runUnitTest(self, *args, **kw):
+    print(test)
     try:
-      runUnitTest = "{python} setup.py test".format(python=self.python_interpreter)
-      args = tuple(shlex.split(runUnitTest))
-      status_dict = self.spawn(*args, **kw)
-    except SubprocessError, e:
+      status_dict = self.spawn(self.python_interpreter, 'setup.py', 'test',
+                               cwd=self.egg_test_path_dict[test])
+    except SubprocessError as e:
       status_dict = e.status_dict
     test_log = status_dict['stderr']
     search = self.RUN_RE.search(test_log)
@@ -270,7 +275,7 @@ class EggTestSuite(TestSuite):
     return status_dict
 
   def getTestList(self):
-    return self.egg_test_path_dict.keys()
+    return list(self.egg_test_path_dict)
 
 def runTestSuite():
   parser = argparse.ArgumentParser(description='Run a test suite.')
@@ -327,7 +332,7 @@ def runTestSuite():
   if test_result is not None:
     assert revision == test_result.revision, (revision, test_result.revision)
     while suite.acquire():
-      test = test_result.start(suite.running.keys())
+      test = test_result.start(list(suite.running))
       if test is not None:
         suite.start(test.name, lambda status_dict, __test=test:
           __test.stop(**status_dict))

erp5/util/timinglogparser/__init__.py

@@ -27,6 +27,8 @@
 #
 ##############################################################################
 
+from __future__ import print_function
+
 import os
 import sys
 import imp
@@ -126,7 +128,7 @@ def parseFile(filename, measure_dict):
       sys.stderr.flush()
     match_list = LINE_PATTERN.findall(line)
     if len(match_list) != 1:
-      print >>sys.stderr, 'Unparseable line: %s:%i %r' % (filename, line_number, line)
+      print('Unparseable line: %s:%i %r' % (filename, line_number, line), file=sys.stderr)
     else:
       result, filter_id, date, duration = processLine(match_list[0], filename, line_number)
       # Possible result values & meaning:
@@ -135,20 +137,21 @@ def parseFile(filename, measure_dict):
       #  (string): use & skip to next line
       if result is False:
         if debug:
-          print >>sys.stderr, '? %s:%i %r' % (filename, line_number, match_list[0])
+          print('? %s:%i %r' % (filename, line_number, match_list[0]), file=sys.stderr)
       elif result is True:
         if debug:
-          print >>sys.stderr, '- %s:%i %r' % (filename, line_number, match_list[0])
+          print('- %s:%i %r' % (filename, line_number, match_list[0]), file=sys.stderr)
         skip_count += 1
       else:
         measure_dict.setdefault(filter_id, {}).setdefault(result, {}).setdefault(date, []).append(int(duration))
         match_count += 1
     line = logfile.readline()
-  print >>sys.stderr, '%i' % (line_number, )
+  print('%i' % (line_number, ), file=sys.stderr)
   if line_number > 0:
     duration = time() - begin
-    print >>sys.stderr, "Matched %i lines (%.2f%%), %i skipped (%.2f%%), %i unmatched (%.2f%%) in %.2fs (%i lines per second)." % \
-      (match_count, (float(match_count) / line_number) * 100, skip_count, (float(skip_count) / line_number) * 100, (line_number - match_count - skip_count), (1 - (float(match_count + skip_count) / line_number)) * 100, duration, line_number / duration)
+    print("Matched %i lines (%.2f%%), %i skipped (%.2f%%), %i unmatched (%.2f%%) in %.2fs (%i lines per second)." % \
+      (match_count, (float(match_count) / line_number) * 100, skip_count, (float(skip_count) / line_number) * 100, (line_number - match_count - skip_count), (1 - (float(match_count + skip_count) / line_number)) * 100, duration, line_number / duration),
+      file=sys.stderr)
 
 debug = False
 outfile_prefix = None
@@ -161,9 +164,9 @@ decimate_count = 1
 
 try:
   opts, file_list = getopt.getopt(sys.argv[1:], '', ['debug', 'config=', 'prefix=', 'no-average', 'sum', 'load=', 'save=', 'decimate='])
-except Exception, reason:
-  print >>sys.stderr, reason
-  print >>sys.stderr, usage
+except Exception as reason:
+  print(reason, file=sys.stderr)
+  print(usage, file=sys.stderr)
   sys.exit(1)
 
 for name, value in opts:
@@ -185,7 +188,7 @@ for name, value in opts:
     decimate_count = int(value)
 
 if configuration is None:
-  raise ValueError, '--config is mandatory'
+  raise ValueError('--config is mandatory')
 
 config_file = os.path.splitext(os.path.basename(configuration))[0]
 config_path = [os.path.dirname(os.path.abspath(configuration))] + sys.path
@@ -211,18 +214,18 @@ if len(load_file_name_list):
       for result, date_dict in result_dict.iteritems():
         for date, duration_list in date_dict.iteritems():
           measure_dict.setdefault(filter_id, {}).setdefault(result, {}).setdefault(date, []).extend(duration_list)
-    print >>sys.stderr, 'Previous processing result restored from %r' % (load_file_name, )
+    print('Previous processing result restored from %r' % (load_file_name, ), file=sys.stderr)
 
 for filename in file_list:
   file_number += 1
-  print >>sys.stderr, 'Processing %s [%i/%i]...' % (filename, file_number, file_count)
+  print('Processing %s [%i/%i]...' % (filename, file_number, file_count), file=sys.stderr)
   parseFile(filename, measure_dict)
 
 if save_file_name is not None:
   save_file = open(save_file_name, 'w')
   save_file.write(repr(measure_dict))
   save_file.close()
-  print >>sys.stderr, 'Processing result saved to %r' % (save_file_name, )
+  print('Processing result saved to %r' % (save_file_name, ), file=sys.stderr)
 
 if outfile_prefix is not None:
   ## Generate a list of all measures and a 2-levels dictionnary with date as key and measure dictionnary as value
@@ -252,9 +255,9 @@ if outfile_prefix is not None:
   def renderOutput(data_format, filename_suffix):
     for sheet_id, sheet_column_list in sheet_dict.iteritems():
       outfile_name = '%s_%s_%s.csv' % (outfile_prefix, sheet_id, filename_suffix)
-      print >>sys.stderr, 'Writing to %r...' % (outfile_name, )
+      print('Writing to %r...' % (outfile_name, ), file=sys.stderr)
       outfile = open(outfile_name, 'w')
-      print >>outfile, '"date",%s' % (','.join(['"%s"' % (x[0], ) for x in sheet_column_list]), )
+      print('"date",%s' % (','.join(['"%s"' % (x[0], ) for x in sheet_column_list]), ), file=outfile)
       decimate_dict = {}
       decimate = 0
       for date in date_list:
@@ -262,11 +265,11 @@ if outfile_prefix is not None:
           decimate_dict.setdefault(key, []).extend(value)
         decimate += 1
         if decimate == decimate_count:
-          print >>outfile, '"%s",%s' % (date, ','.join([render_cell(decimate_dict.get(x[1], ''), data_format) for x in sheet_column_list]))
+          print('"%s",%s' % (date, ','.join([render_cell(decimate_dict.get(x[1], ''), data_format) for x in sheet_column_list])), file=outfile)
           decimate_dict = {}
           decimate = 0
       if len(decimate_dict):
-        print >>outfile, '"%s",%s' % (date, ','.join([render_cell(decimate_dict.get(x[1], ''), data_format) for x in sheet_column_list]))
+        print('"%s",%s' % (date, ','.join([render_cell(decimate_dict.get(x[1], ''), data_format) for x in sheet_column_list])), file=outfile)
 
   if do_average:
     renderOutput('=%(sum)i/%(count)i', 'avg')

erp5/util/timinglogplotter/__init__.py

@@ -27,6 +27,8 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
+from __future__ import print_function
+
 from datetime import date
 from os import path
 import rpy2.robjects as robjects
@@ -121,7 +123,7 @@ def main():
 
   current_dir = os.getcwd()
   for file_name in file_name_list:
-    print 'Loading %s...' % (file_name, )
+    print('Loading %s...' % (file_name, ))
     file = CSVFile(file_name)
 
     date_string_list = file.getColumn(0)
@@ -183,13 +185,13 @@ def main():
           y_data.append(value)
         i += 1
       if len(x_data) == 0:
-        print 'Nothing to plot for %s...' % (out_file_name, )
+        print('Nothing to plot for %s...' % (out_file_name, ))
         continue
       if options.minimal_non_empty_ratio is not None:
         column_len = len(column)
         if column_len:
           if float(len(x_data))/column_len < options.minimal_non_empty_ratio:
-            print 'Not enough values to plot for %s...' % (out_file_name, )
+            print('Not enough values to plot for %s...' % (out_file_name, ))
             continue
       r_y_data = robjects.FloatVector(y_data)
       r_x_data = robjects.FloatVector(x_data)
@@ -220,7 +222,7 @@ def main():
       # stop changing the out-type file
       r("""dev.off()""")
 
-      print 'Saving %s...' % (out_file_name, )
+      print('Saving %s...' % (out_file_name, ))
 
 if __name__ == '__main__':
   main()

erp5/util/webchecker/__init__.py

@@ -26,6 +26,8 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
+from __future__ import print_function
+
 import os
 import shutil
 import sys
@@ -543,7 +545,7 @@ class HTTPCacheCheckerTestSuite(object):
 
 
 from optparse import OptionParser
-import ConfigParser
+from six.moves import configparser
 
 def _formatConfiguration(configuration):
   """ format the configuration"""
@@ -559,11 +561,11 @@ def web_checker_utility():
 
   (options, args) = parser.parse_args()
   if len(args) != 1 :
-    print parser.print_help()
+    print(parser.print_help())
     parser.error('incorrect number of arguments')
   config_path = args[0]
 
-  config = ConfigParser.RawConfigParser()
+  config = configparser.RawConfigParser()
   config.read(config_path)
   working_directory = config.get('web_checker', 'working_directory')
   url = config.get('web_checker', 'url')
@@ -619,6 +621,6 @@ def web_checker_utility():
     file_object.write(result)
     file_object.close()
   else:
-    print result
+    print(result)
 
 

product/ERP5/bin/genbt5list

@@ -35,12 +35,15 @@ import posixpath
 import tarfile
 import os
 import sys
-import cgi
+try:
+  from html import escape
+except ImportError:
+  from cgi import escape
 from base64 import b64encode
-from cStringIO import StringIO
+from six import BytesIO
 from hashlib import sha1
-from urllib import unquote
-
+from six.moves.urllib.parse import unquote
+import six
 
 # Order is important for installation
 # We want to have:
@@ -107,11 +110,11 @@ item_name_list = tuple('_%s_item' % x for x in item_name_list)
 class BusinessTemplateRevision(list):
 
   def hash(self, path, text):
-    self.append((path, sha1(text).digest()))
+    self.append((path.encode('utf-8'), sha1(text).digest()))
 
   def digest(self):
     self.sort()
-    return b64encode(sha1('\0'.join(h + p for (h, p) in self)).digest())
+    return b64encode(sha1(b'\0'.join(h + p for (h, p) in self)).digest())
 
 
 class BusinessTemplate(dict):
@@ -149,7 +152,7 @@ force_install
 
   def __iter__(self):
     self['revision'] = self.revision.digest()
-    return iter(sorted(self.iteritems()))
+    return iter(sorted(six.iteritems(self)))
 
   @classmethod
   def fromTar(cls, tar):
@@ -177,8 +180,8 @@ force_install
     return iter(self)
 
 def generateInformation(dir, info=id, err=None):
-  xml = StringIO()
-  xml.write('<?xml version="1.0"?>\n<repository>\n')
+  xml = BytesIO()
+  xml.write(b'<?xml version="1.0"?>\n<repository>\n')
   for name in sorted(os.listdir(dir)):
     path = os.path.join(dir, name)
     if name.endswith('.bt5'):
@@ -199,13 +202,14 @@ def generateInformation(dir, info=id, err=None):
       property_list = BusinessTemplate.fromDir(path)
     else:
       continue
-    xml.write('  <template id="%s">\n' % name)
+    xml.write(b'  <template id="%s">\n' % name.encode('utf-8'))
     for k, v in property_list:
-      for v in (v,) if type(v) is str else v:
-        xml.write('    <%s>%s</%s>\n' % (k, cgi.escape(v), k))
-    xml.write('  </template>\n')
+      k = k.encode('utf-8')
+      for v in (v,) if type(v) is bytes else v:
+        xml.write(b'    <%s>%s</%s>\n' % (k, escape(v.decode('utf-8')).encode('utf-8'), k))
+    xml.write(b'  </template>\n')
     info('done\n')
-  xml.write('</repository>\n')
+  xml.write(b'</repository>\n')
   return xml
 
 def main(dir_list=None, **kw):