erp5.util: add support for Python 3

/reviewed-on nexedi/erp5!830

erp5/util/benchmark/performance_tester.py

@@ -28,6 +28,7 @@
 #
 ##############################################################################
 
+from __future__ import print_function
 import argparse
 import os
 import sys
@@ -264,7 +265,7 @@ class PerformanceTester(object):
         error_message = exit_msg_queue.get()
 
       except KeyboardInterrupt, e:
-        print >>sys.stderr, "\nInterrupted by user, stopping gracefully..."
+        print("\nInterrupted by user, stopping gracefully...", file=sys.stderr)
         exit_status = 2
 
       # An IOError may be raised  when receiving a SIGINT which interrupts the
@@ -337,7 +338,7 @@ class PerformanceTester(object):
 def main():
   error_message_set, exit_status = PerformanceTester().run()
   for error_message in error_message_set:
-    print >>sys.stderr, "ERROR: %s" % error_message
+    print("ERROR: %s" % error_message, file=sys.stderr)
 
   sys.exit(exit_status)
 

erp5/util/benchmark/report.py

@@ -31,6 +31,7 @@
 #
 ##############################################################################
 
+from __future__ import print_function
 import argparse
 import re
 
@@ -537,7 +538,7 @@ def generateReport():
   for filename in filename_iter:
     # There may be no results at all in case of errors
     if not os.stat(filename).st_size:
-      print >>sys.stderr, "Ignoring empty file %s" % filename
+      print("Ignoring empty file %s" % filename, file=sys.stderr)
       continue
 
     report_dict = per_nb_users_report_dict.setdefault(
@@ -546,10 +547,8 @@ def generateReport():
     report_dict['filename'].append(filename)
 
   if not per_nb_users_report_dict:
-    print >>sys.stderr, "ERROR: No result file found, perhaps "\
-        "``--filename-prefix'' should be specified?"
-
-    sys.exit(1)
+    sys.exit("ERROR: No result file found, perhaps ``--filename-prefix'' should"
+             "be specified?")
 
   pdf = PdfPages(argument_namespace.output_filename)
 

erp5/util/benchmark/scalability_tester.py

@@ -28,6 +28,7 @@
 #
 ##############################################################################
 
+from __future__ import print_function
 from .result import CSVBenchmarkResult, NothingFlushedException
 
 class CSVScalabilityBenchmarkResult(CSVBenchmarkResult):
@@ -60,7 +61,7 @@ class ScalabilityTester(PerformanceTester):
                       urllib.urlencode({'error_message_set': '|'.join(error_message_set)})).close()
 
     except:
-      print >>sys.stderr, "ERROR: %s" % Formatter().formatException(sys.exc_info())
+      print("ERROR: %s" % Formatter().formatException(sys.exc_info()), file=sys.stderr)
 
   def getResultClass(self):
     if not self._argument_namespace.erp5_publish_url:

erp5/util/scalability/requestUrl.py

@@ -27,7 +27,7 @@ def main():
   if error_message_set:
     exit_status = 1
     for error in error_message_set:
-      print error
+      print(error)
   elif result:
-    print result
+    print(result)
   sys.exit(exit_status)

erp5/util/scalability/runScalabilityTestSuite.py

@@ -5,23 +5,15 @@ import os
 import shutil
 import time
 import sys
-import multiprocessing
-import signal
-import errno
 import json
 import logging
 import logging.handlers
 import glob
-import urlparse
-import httplib
-import base64
 import threading
-from erp5.util.benchmark.argument import ArgumentType
-from erp5.util.benchmark.performance_tester import PerformanceTester
 from erp5.util.benchmark.thread import TestThread, TestMetricThread
 from erp5.util import taskdistribution
 from erp5.util.testnode import Utils
-from erp5.util.testnode.ProcessManager import SubprocessError, ProcessManager, CancellationError
+from erp5.util.testnode.ProcessManager import ProcessManager
 import datetime
 
 MAX_INSTALLATION_TIME = 60*50
@@ -179,31 +171,28 @@ class ScalabilityLauncher(object):
     """
     data_array = self.__argumentNamespace.current_test_data.split(',')
     data = json.dumps({"count": data_array[0], "title": data_array[1], "relative_path": data_array[2]})
-    decoded_data = Utils.deunicodeData(json.loads(data))
-    return ScalabilityTest(decoded_data, self.test_result)
+    encoded_data = Utils.deunicodeData(json.loads(data))
+    return ScalabilityTest(encoded_data, self.test_result)
 
   def clearUsersFile(self, user_file_path):
     self.log("Clearing users file: %s" % user_file_path)
     os.remove(user_file_path)
-    users_file = open(user_file_path, "w")
-    for line in self.users_file_original_content:
-      users_file.write(line)
-    users_file.close()
+    with open(user_file_path, "w") as users_file:
+      for line in self.users_file_original_content:
+        users_file.write(line)
 
   def updateUsersFile(self, user_quantity, password, user_file_path):
     self.log("Updating users file: %s" % user_file_path)
-    users_file = open(user_file_path, "r")
-    file_content = users_file.readlines()
+    with open(user_file_path, "r") as users_file:
+      file_content = users_file.readlines()
     self.users_file_original_content = file_content
     new_file_content = []
     for line in file_content:
       new_file_content.append(line.replace('<password>', password).replace('<user_quantity>', str(user_quantity)))
-    users_file.close()
     os.remove(user_file_path)
-    users_file = open(user_file_path, "w")
-    for line in new_file_content:
-      users_file.write(line)
-    users_file.close()
+    with open(user_file_path, "w") as users_file:
+      for line in new_file_content:
+        users_file.write(line)
 
   def run(self):
     self.log("Scalability Launcher started, with:")

erp5/util/taskdistribution/__init__.py

@@ -40,11 +40,15 @@ Example use:
       test_line.stop()
 """
 from __future__ import print_function
-import httplib
+import six
+from six.moves import (
+  map,
+  http_client as httplib,
+  xmlrpc_client as xmlrpclib,
+)
 import socket
 import threading
 import time
-import xmlrpclib
 
 __all__ = ['TaskDistributor', 'TestResultProxy', 'TestResultLineProxy', 'patchRPCParser']
 
@@ -89,11 +93,17 @@ def patchRPCParser(error_handler):
     def verbose_feed(self, data):
         try:
             return original_feed(self, data)
-        except Exception, exc:
+        except Exception as exc:
             if not error_handler(data, exc):
                 raise
     parser_klass.feed = verbose_feed
 
+try: # PY3
+  basestring
+except NameError:
+  basestring = bytes, str
+  unicode = str
+
 def binarize_args(arg):
   # Converts recursively basestring arg into xmlrpclib.Binary, as they can
   # contain non-XML allowed characters
@@ -102,9 +112,9 @@ def binarize_args(arg):
       arg = arg.encode('utf-8')
     return xmlrpclib.Binary(arg)
   if isinstance(arg, (list, tuple, set)):
-    return map(binarize_args, arg)
+    return list(map(binarize_args, arg))
   if isinstance(arg, dict):
-    return {k: binarize_args(v) for k, v in arg.iteritems()}
+    return {k: binarize_args(v) for k, v in six.iteritems(arg)}
   return arg
 
 class RPCRetry(object):
@@ -350,7 +360,7 @@ class TestResultProxy(RPCRetry):
             caption_list = []
             append = caption_list.append
             for name, (stream, max_history_bytes) in \
-                    self._watcher_dict.iteritems():
+                    six.iteritems(self._watcher_dict):
                 append('==> %s <==' % (name, ))
                 start = stream.tell()
                 stream.seek(0, 2)

erp5/util/testbrowser/examples/createERP5User.py

@@ -8,6 +8,7 @@
 #
 # TODO: There must be a better way than the code below to do that though...
 
+from __future__ import print_function
 import sys
 
 from erp5.util.testbrowser.browser import Browser
@@ -19,11 +20,8 @@ try:
   user_nbr = int(user_nbr)
 
 except ValueError:
-  print >>sys.stderr, "ERROR: Missing arguments: %s URL USERNAME " \
-      "PASSWORD NUMBER_OF_USERS NEW_USERNAME_PREFIX NEW_USERS_PASSWORD" % \
-      sys.argv[0]
-
-  sys.exit(1)
+  sys.exit("ERROR: Missing arguments: %s URL USERNAME PASSWORD NUMBER_OF_USERS "
+           "NEW_USERNAME_PREFIX NEW_USERS_PASSWORD" % sys.argv[0])
 
 # Create a browser instance
 browser = Browser(url, username, password)

erp5/util/testbrowser/examples/createPerson.py

 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
+from __future__ import division, print_function
 from erp5.util.testbrowser.browser import Browser
 
 ITERATION = 20
@@ -89,4 +90,4 @@ if __name__ == '__main__':
     counter += 1
 
   for title, time_list in result_dict.iteritems():
-    print "%s: %.4fs" % (title, float(sum(time_list)) / ITERATION)
+    print("%s: %.4fs" % (title, sum(time_list) / ITERATION))

erp5/util/testnode/NodeTestSuite.py

@@ -32,6 +32,8 @@ import string
 import random
 from .Utils import createFolder
 
+from six.moves import range
+
 class SlapOSInstance(object):
   """
   Base of an software instance,
@@ -69,14 +71,14 @@ class NodeTestSuite(SlapOSInstance):
 
   def createSuiteLog(self):
     # /srv/slapgrid/slappartXX/srv/var/log/testnode/az-D27KqX7FxJ/suite.log
-    alphabets = string.digits + string.letters
+    alphabets = string.digits + string.ascii_letters
     while 1:
       log_folder_name = '%s-%s' % (self.reference,
-        ''.join(random.choice(alphabets) for i in xrange(10)))
+        ''.join(random.choice(alphabets) for i in range(10)))
       log_folder_path = os.path.join(self.log_directory, log_folder_name)
       try:
         os.makedirs(log_folder_path)
-      except OSError, e:
+      except OSError as e:
         if e.errno != errno.EEXIST:
           raise
       else:

erp5/util/testnode/ProcessManager.py

@@ -79,7 +79,8 @@ def subprocess_capture(p, log_prefix, get_output=True):
         break
       if get_output:
         buffer.append(data)
-      log(log_prefix + data.rstrip('\n'))
+      log(log_prefix + (data if str is bytes else
+                        data.decode('utf-8', errors='replace')).rstrip('\n'))
   if p.stdout:
     stdout = []
     stdout_thread = threading.Thread(target=readerthread,
@@ -97,8 +98,8 @@ def subprocess_capture(p, log_prefix, get_output=True):
     stdout_thread.join()
   if p.stderr:
     stderr_thread.join()
-  return (p.stdout and ''.join(stdout),
-          p.stderr and ''.join(stderr))
+  return (p.stdout and b''.join(stdout),
+          p.stderr and b''.join(stderr))
 
 def killCommand(pid):
   """
@@ -109,7 +110,7 @@ def killCommand(pid):
   try:
     process = psutil.Process(pid)
     process.suspend()
-  except psutil.Error, e:
+  except psutil.Error as e:
     return
   process_list = [process]
   new_list = process.children(recursive=True)
@@ -118,19 +119,19 @@ def killCommand(pid):
     for child in new_list:
       try:
         child.suspend()
-      except psutil.Error, e:
+      except psutil.Error as e:
         logger.debug("killCommand/suspend: %s", e)
     time.sleep(1)
     new_list = set(process.children(recursive=True)).difference(process_list)
   for process in process_list:
     try:
       process.kill()
-    except psutil.Error, e:
+    except psutil.Error as e:
       logger.debug("killCommand/kill: %s", e)
 
 class ProcessManager(object):
 
-  stdin = file(os.devnull)
+  stdin = open(os.devnull)
 
   def __init__(self, max_timeout=MAX_TIMEOUT):
     self.process_pid_set = set()

erp5/util/testnode/ScalabilityTestRunner.py

@@ -30,30 +30,31 @@ import subprocess
 import sys
 import time
 import glob
-import SlapOSControler
-import SlapOSMasterCommunicator
+from . import SlapOSControler, SlapOSMasterCommunicator
 import json
 import time
 import shutil
 import logging
 import string
 import random
-import urlparse
+from six.moves.urllib.parse import urlparse
 import base64
-import httplib
-import Utils
+from six.moves import http_client as httplib
+from . import Utils
 import requests
 import slapos.slap
-import cPickle as pickle
-from ProcessManager import SubprocessError, ProcessManager, CancellationError
+from six.moves import cPickle as pickle
+from .ProcessManager import SubprocessError, ProcessManager, CancellationError
 from subprocess import CalledProcessError
-from Updater import Updater
+from .Updater import Updater
 from erp5.util import taskdistribution
 from erp5.util.benchmark.thread import TestThread
 # for dummy slapos answer
 import signal
 from . import logger
 
+from six.moves import range
+
 # max time to generate frontend instance: 1.5 hour
 MAX_FRONTEND_TIME = 60*90
 # max time to register instance to slapOSMaster: 5 minutes
@@ -322,18 +323,16 @@ ces or already launched.")
     software_hash_directory = self.testnode.config['slapos_binary'].rsplit("bin/slapos", 1)[0]
     apache_htpasswd = software_hash_directory + "parts/apache/bin/htpasswd"
     testsuite_directory = self.testnode.config['repository_path_list'][0].rsplit('/', 1)[0]
-    htaccess_file = open(testsuite_directory + HTACCESS, "w")
-    file_content = """
+    with open(testsuite_directory + HTACCESS, "w") as htaccess_file:
+      htaccess_file.write("""
 AuthType Basic
 AuthName "Password Protected Area"
 AuthUserFile "%s%s"
 Require valid-user
-""" % (testsuite_directory, HTPASSWD)
-    htaccess_file.write(file_content)
-    htaccess_file.close()
+""" % (testsuite_directory, HTPASSWD))
     password_path = testsuite_directory + PASSWORD_FILE
     with open(password_path, "w") as password_file:
-      password = ''.join(random.choice(string.digits + string.letters) for i in xrange(PASSWORD_LENGTH))
+      password = ''.join(random.choice(string.digits + string.ascii_letters) for i in range(PASSWORD_LENGTH))
       password_file.write(password)
     user = TESTNODE_USER
     command = [apache_htpasswd, "-bc", testsuite_directory + HTPASSWD, user, password]
@@ -363,7 +362,7 @@ Require valid-user
     user, password = self.generateProfilePasswordAccess()
     logger.info("Software Profile password: %s" % password)
     self.reachable_profile = "https://%s:%s@%s" % (user, password,
-      os.path.join(urlparse.urlparse(self.testnode.config['frontend_url']).netloc,
+      os.path.join(urlparse(self.testnode.config['frontend_url']).netloc,
                    "software", self.randomized_path, "software.cfg"))
 
   def prepareSlapOSForTestSuite(self, node_test_suite):
@@ -526,7 +525,7 @@ Require valid-user
     if not self.launchable:
       return {'status_code' : 1, 'error_message': "Current test_suite is not actually launchable." }
     configuration_list = node_test_suite.configuration_list
-    test_list = range(0, len(configuration_list))
+    test_list = list(range(len(configuration_list)))
     try:
       test_result_proxy = self.testnode.taskdistribution.createTestResult(
                           node_test_suite.revision, test_list,

erp5/util/testnode/SlapOSControler.py

@@ -35,6 +35,8 @@ from slapos import client
 from . import logger
 from .Utils import createFolder
 
+from six.moves import range
+
 MAX_PARTITIONS = 10
 MAX_SR_RETRIES = 3
 
@@ -243,7 +245,7 @@ class SlapOSControler(object):
         computer = slap.registerComputer(config['computer_id'])
         # Call a method to ensure connection to master can be established
         computer.getComputerPartitionList()
-      except slapos.slap.ConnectionError, e:
+      except slapos.slap.ConnectionError as e:
         retries += 1
         if retries >= 60:
           raise
@@ -270,7 +272,7 @@ class SlapOSControler(object):
     # MySQL DB content) from previous runs. To support changes of partition
     # naming scheme (which already happened), do this at instance_root level.
     createFolder(instance_root, True)
-    for i in xrange(MAX_PARTITIONS):
+    for i in range(MAX_PARTITIONS):
       # create partition and configure computer
       # XXX: at the moment all partitions do share same virtual interface address
       # this is not a problem as usually all services are on different ports
@@ -278,7 +280,7 @@ class SlapOSControler(object):
       partition_path = os.path.join(instance_root, partition_reference)
       if not(os.path.exists(partition_path)):
         os.mkdir(partition_path)
-      os.chmod(partition_path, 0750)
+      os.chmod(partition_path, 0o750)
       computer.updateConfiguration(xml_marshaller.xml_marshaller.dumps({
            'address': config['ipv4_address'],
            'instance_root': instance_root,
@@ -318,7 +320,7 @@ class SlapOSControler(object):
     os.environ['PATH'] = environment['PATH']
     # a SR may fail for number of reasons (incl. network failures)
     # so be tolerant and run it a few times before giving up
-    for _ in xrange(MAX_SR_RETRIES):
+    for _ in range(MAX_SR_RETRIES):
       status_dict = self.spawn(config['slapos_binary'],
                  'node', 'software', '--all', 
                  '--pidfile', os.path.join(self.software_root, 'slapos-node.pid'),
@@ -346,7 +348,7 @@ class SlapOSControler(object):
     # try to run for all partitions as one partition may in theory request another one 
     # this not always is required but curently no way to know how "tree" of partitions
     # may "expand"
-    for _ in xrange(max_quantity):
+    for _ in range(max_quantity):
       status_dict = self.spawn(config['slapos_binary'], 'node', 'instance', 
                  '--pidfile', os.path.join(self.instance_root, 'slapos-node.pid'),
                  '--cfg', self.slapos_config, raise_error_if_fail=False,

erp5/util/testnode/SlapOSMasterCommunicator.py

+from __future__ import print_function
+
 import datetime
 import json
 import traceback
@@ -12,6 +14,8 @@ from requests.exceptions import HTTPError
 from ..taskdistribution import SAFE_RPC_EXCEPTION_LIST
 from . import logger
 
+import six
+
 # max time to instance changing state: 3 hour
 MAX_INSTANCE_TIME = 60*60*3
 
@@ -52,7 +56,7 @@ def retryOnNetworkFailure(func,
       except _except_list:
         traceback.print_exc()
 
-      print 'Network failure. Retry method %s in %i seconds' % (func, retry_time)
+      print('Network failure. Retry method %s in %i seconds' % (func, retry_time))
       time.sleep(retry_time)
       retry_time = min(retry_time*1.5, 640)
 
@@ -92,8 +96,9 @@ class SlapOSMasterCommunicator(object):
     if instance_title is not None:
       self.name = instance_title 
     if request_kw is not None:
-      if isinstance(request_kw, basestring) or \
-        isinstance(request_kw, unicode):
+      if isinstance(request_kw, bytes):
+        self.request_kw = json.loads(request_kw.decode('utf-8'))
+      elif isinstance(request_kw, six.text_type):
         self.request_kw = json.loads(request_kw)
       else:
         self.request_kw = request_kw
@@ -214,7 +219,7 @@ class SlapOSMasterCommunicator(object):
     result = self.hateoas_navigator.GET(url)
     result = json.loads(result)
     if result['_links'].get('action_object_slap', None) is None:
-      print result['links']
+      print(result['links'])
       return None
 
     object_link = self.hateoas_navigator.hateoasGetLinkFromLinks(
@@ -385,8 +390,9 @@ class SlapOSTester(SlapOSMasterCommunicator):
     self.name = name
     self.computer_guid = computer_guid
 
-    if isinstance(request_kw, str) or \
-      isinstance(request_kw, unicode):
+    if isinstance(request_kw, bytes):
+      self.request_kw = json.loads(request_kw.decode('utf-8'))
+    elif isinstance(request_kw, six.text_type):
       self.request_kw = json.loads(request_kw)
     else:
       self.request_kw = request_kw

erp5/util/testnode/Updater.py

@@ -30,6 +30,7 @@ import re
 from . import logger
 from .ProcessManager import SubprocessError
 from .Utils import rmtree
+from slapos.util import bytes2str, str2bytes
 
 SVN_UP_REV = re.compile(r'^(?:At|Updated to) revision (\d+).$')
 SVN_CHANGED_REV = re.compile(r'^Last Changed Rev.*:\s*(\d+)', re.MULTILINE)
@@ -82,7 +83,7 @@ class Updater(object):
           # allow several processes clean the same folder at the same time
           try:
             os.remove(os.path.join(path, file))
-          except OSError, e:
+          except OSError as e:
             if e.errno != errno.ENOENT:
               raise
 
@@ -96,7 +97,7 @@ class Updater(object):
                                       **kw)
 
   def _git(self, *args, **kw):
-    return self.spawn(self.git_binary, *args, **kw)['stdout'].strip()
+    return bytes2str(self.spawn(self.git_binary, *args, **kw)['stdout'].strip())
 
   def git_update_server_info(self):
     return self._git('update-server-info', '-f')
@@ -219,7 +220,7 @@ class Updater(object):
         self.deletePycFiles(path)
         try:
           status_dict = self.spawn(*args)
-        except SubprocessError, e:
+        except SubprocessError as e:
           if 'cleanup' not in e.stderr:
             raise
           self.spawn('svn', 'cleanup', path)

erp5/util/testnode/Utils.py

@@ -3,6 +3,9 @@ import stat
 import shutil
 import errno
 
+import six
+from six.moves import map
+
 def rmtree(path):
   """Delete a path recursively.
 
@@ -11,14 +14,22 @@ def rmtree(path):
   def chmod_retry(func, failed_path, exc_info):
     """Make sure the directories are executable and writable.
     """
+    # Depending on the Python version, the following items differ.
+    if six.PY3:
+      expected_error_type = PermissionError
+      expected_func = os.lstat
+    else:
+      expected_error_type = OSError
+      expected_func = os.listdir
+
     e = exc_info[1]
-    if isinstance(e, OSError):
+    if isinstance(e, expected_error_type):
       if e.errno == errno.ENOENT:
         # because we are calling again rmtree on listdir errors, this path might
         # have been already deleted by the recursive call to rmtree.
         return
       if e.errno == errno.EACCES:
-        if func is os.listdir:
+        if func is expected_func:
           os.chmod(failed_path, 0o700)
           # corner case to handle errors in listing directories.
           # https://bugs.python.org/issue8523
@@ -39,12 +50,16 @@ def createFolder(folder, clean=False):
     rmtree(folder)
   os.mkdir(folder)
 
-def deunicodeData(data):
-  if isinstance(data, list):
-    return map(deunicodeData, data)
-  if isinstance(data, unicode):
-    return data.encode('utf8')
-  if isinstance(data, dict):
-    return {deunicodeData(key): deunicodeData(value)
-            for key, value in data.iteritems()}
-  return data
+if six.PY3:
+  def deunicodeData(data):
+    return data
+else:
+  def deunicodeData(data):
+    if isinstance(data, list):
+      return list(map(deunicodeData, data))
+    if isinstance(data, unicode):
+      return data.encode('utf8')
+    if isinstance(data, dict):
+      return {deunicodeData(key): deunicodeData(value)
+              for key, value in six.iteritems(data)}
+    return data

erp5/util/testnode/__init__.py

@@ -24,7 +24,7 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
-import ConfigParser
+from six.moves import configparser
 import argparse
 import logging
 import logging.handlers
@@ -64,7 +64,7 @@ def main(*args):
   CONFIG = {
     'partition_reference': 'test0',
   }
-  config = ConfigParser.SafeConfigParser()
+  config = configparser.SafeConfigParser()
   # do not change case of option keys
   config.optionxform = str
   config.readfp(parsed_argument.configuration_file[0])

erp5/util/testnode/testnode.py

@@ -171,7 +171,7 @@ shared = true
       # only limit to particular error, if we run that code for all errors,
       # then if server having most repositories is down for some time, we would
       # erase all repositories and facing later hours of downloads
-      if getattr(error, 'stderr', '').find('index') >= 0:
+      if b'index' in getattr(error, 'stderr', b''):
         rmtree(repository_path)
       logger.warning("Error while getting repository, ignoring this test suite",
                      exc_info=1)

erp5/util/testsuite/__init__.py

+from __future__ import print_function
+
 import argparse
 import re, os, shlex, glob
 import sys, threading, subprocess
 import traceback
 import errno
+import pprint
+import six
+from six.moves import range
 
 from erp5.util import taskdistribution
-from pprint import pprint
 
+if six.PY3:
+  stdbin = lambda x: x.buffer
+else:
+  stdbin = lambda x: x
+
+# PY3: use shlex.quote
 _format_command_search = re.compile("[[\\s $({?*\\`#~';<>&|]").search
 _format_command_escape = lambda s: "'%s'" % r"'\''".join(s.split("'"))
 def format_command(*args, **kw):
@@ -31,7 +41,7 @@ def subprocess_capture(p, quiet=False):
       buffer.append(data)
   if p.stdout:
     stdout = []
-    output = quiet and (lambda data: None) or sys.stdout.write
+    output = (lambda data: None) if quiet else stdbin(sys.stdout).write
     stdout_thread = threading.Thread(target=readerthread,
                                      args=(p.stdout, output, stdout))
     stdout_thread.setDaemon(True)
@@ -39,7 +49,7 @@ def subprocess_capture(p, quiet=False):
   if p.stderr:
     stderr = []
     stderr_thread = threading.Thread(target=readerthread,
-                                     args=(p.stderr, sys.stderr.write, stderr))
+        args=(p.stderr, stdbin(sys.stderr).write, stderr))
     stderr_thread.setDaemon(True)
     stderr_thread.start()
   if p.stdout:
@@ -47,8 +57,8 @@ def subprocess_capture(p, quiet=False):
   if p.stderr:
     stderr_thread.join()
   p.wait()
-  return (p.stdout and ''.join(stdout),
-          p.stderr and ''.join(stderr))
+  return (p.stdout and b''.join(stdout),
+          p.stderr and b''.join(stderr))
 
 class SubprocessError(EnvironmentError):
   def __init__(self, status_dict):
@@ -72,15 +82,15 @@ class Persistent(object):
   def __getattr__(self, attr):
     if attr == '_db':
       try:
-        db = file(self._filename, 'r+')
-      except IOError, e:
+        db = open(self._filename, 'r+')
+      except IOError as e:
         if e.errno != errno.ENOENT:
           raise
-        db = file(self._filename, 'w+')
+        db = open(self._filename, 'w+')
       else:
         try:
           self.__dict__.update(eval(db.read()))
-        except StandardError:
+        except Exception:
           pass
       self._db = db
       return db
@@ -89,7 +99,7 @@ class Persistent(object):
 
   def sync(self):
     self._db.seek(0)
-    db = dict(x for x in self.__dict__.iteritems() if x[0][:1] != '_')
+    db = dict(x for x in six.iteritems(self.__dict__) if x[0][:1] != '_')
     pprint.pprint(db, self._db)
     self._db.truncate()
 
@@ -103,10 +113,10 @@ class TestSuite(object):
   """
 
   RUN_RE = re.compile(
-    r'Ran (?P<all_tests>\d+) tests? in (?P<seconds>\d+\.\d+)s',
+    br'Ran (?P<all_tests>\d+) tests? in (?P<seconds>\d+\.\d+)s',
     re.DOTALL)
 
-  STATUS_RE = re.compile(r"""
+  STATUS_RE = re.compile(br"""
     (OK|FAILED)\s+\(
       (failures=(?P<failures>\d+),?\s*)?
       (errors=(?P<errors>\d+),?\s*)?
@@ -117,7 +127,7 @@ class TestSuite(object):
     """, re.DOTALL | re.VERBOSE)
 
   SUB_STATUS_RE = re.compile(
-      r"""SUB\s+RESULT:\s+(?P<all_tests>\d+)\s+Tests,\s+
+      br"""SUB\s+RESULT:\s+(?P<all_tests>\d+)\s+Tests,\s+
       (?P<failures>\d+)\s+Failures\s*
       \(?
         (skipped=(?P<skips>\d+),?\s*)?
@@ -130,7 +140,10 @@ class TestSuite(object):
   mysql_db_count = 1
   allow_restart = False
   realtime_output = True
-  stdin = file(os.devnull)
+  try: # PY3
+    stdin = subprocess.DEVNULL
+  except AttributeError:
+    stdin = open(os.devnull, 'rb')
 
   def __init__(self, max_instance_count, **kw):
     self.__dict__.update(kw)
@@ -139,8 +152,8 @@ class TestSuite(object):
     self.acquire = pool.acquire
     self.release = pool.release
     self._instance = threading.local()
-    self._pool = max_instance_count == 1 and [None] or \
-                 range(1, max_instance_count + 1)
+    self._pool = [None] if max_instance_count == 1 else \
+                 list(range(1, max_instance_count + 1))
     self._ready = set()
     self.running = {}
     if max_instance_count != 1:
@@ -185,13 +198,14 @@ class TestSuite(object):
 
   def spawn(self, *args, **kw):
     quiet = kw.pop('quiet', False)
+    cwd = kw.pop('cwd', None)
     env = kw and dict(os.environ, **kw) or None
     command = format_command(*args, **kw)
-    print '\n$ ' + command
+    print('\n$ ' + command)
     sys.stdout.flush()
     try:
       p = subprocess.Popen(args, stdin=self.stdin, stdout=subprocess.PIPE,
-                           stderr=subprocess.PIPE, env=env)
+                           stderr=subprocess.PIPE, env=env, cwd=cwd)
     except Exception:
       # Catch any exception here, to warn user instead of beeing silent,
       # by generating fake error result
@@ -229,20 +243,11 @@ class EggTestSuite(TestSuite):
   The python interpreter is ``python_interpreter``
   """
   def run(self, test):
-    print test
-    original_dir = os.getcwd()
-    try:
-      os.chdir(self.egg_test_path_dict[test])
-      return self.runUnitTest(test)
-    finally:
-      os.chdir(original_dir)
-
-  def runUnitTest(self, *args, **kw):
+    print(test)
     try:
-      runUnitTest = "{python} setup.py test".format(python=self.python_interpreter)
-      args = tuple(shlex.split(runUnitTest))
-      status_dict = self.spawn(*args, **kw)
-    except SubprocessError, e:
+      status_dict = self.spawn(self.python_interpreter, 'setup.py', 'test',
+                               cwd=self.egg_test_path_dict[test])
+    except SubprocessError as e:
       status_dict = e.status_dict
     test_log = status_dict['stderr']
     search = self.RUN_RE.search(test_log)
@@ -270,7 +275,7 @@ class EggTestSuite(TestSuite):
     return status_dict
 
   def getTestList(self):
-    return self.egg_test_path_dict.keys()
+    return list(self.egg_test_path_dict)
 
 def runTestSuite():
   parser = argparse.ArgumentParser(description='Run a test suite.')
@@ -327,7 +332,7 @@ def runTestSuite():
   if test_result is not None:
     assert revision == test_result.revision, (revision, test_result.revision)
     while suite.acquire():
-      test = test_result.start(suite.running.keys())
+      test = test_result.start(list(suite.running))
       if test is not None:
         suite.start(test.name, lambda status_dict, __test=test:
           __test.stop(**status_dict))

erp5/util/timinglogparser/__init__.py

@@ -27,6 +27,8 @@
 #
 ##############################################################################
 
+from __future__ import division, print_function
+
 import os
 import sys
 import imp
@@ -126,7 +128,7 @@ def parseFile(filename, measure_dict):
       sys.stderr.flush()
     match_list = LINE_PATTERN.findall(line)
     if len(match_list) != 1:
-      print >>sys.stderr, 'Unparseable line: %s:%i %r' % (filename, line_number, line)
+      print('Unparseable line: %s:%i %r' % (filename, line_number, line), file=sys.stderr)
     else:
       result, filter_id, date, duration = processLine(match_list[0], filename, line_number)
       # Possible result values & meaning:
@@ -135,20 +137,21 @@ def parseFile(filename, measure_dict):
       #  (string): use & skip to next line
       if result is False:
         if debug:
-          print >>sys.stderr, '? %s:%i %r' % (filename, line_number, match_list[0])
+          print('? %s:%i %r' % (filename, line_number, match_list[0]), file=sys.stderr)
       elif result is True:
         if debug:
-          print >>sys.stderr, '- %s:%i %r' % (filename, line_number, match_list[0])
+          print('- %s:%i %r' % (filename, line_number, match_list[0]), file=sys.stderr)
         skip_count += 1
       else:
         measure_dict.setdefault(filter_id, {}).setdefault(result, {}).setdefault(date, []).append(int(duration))
         match_count += 1
     line = logfile.readline()
-  print >>sys.stderr, '%i' % (line_number, )
+  print('%i' % (line_number, ), file=sys.stderr)
   if line_number > 0:
     duration = time() - begin
-    print >>sys.stderr, "Matched %i lines (%.2f%%), %i skipped (%.2f%%), %i unmatched (%.2f%%) in %.2fs (%i lines per second)." % \
-      (match_count, (float(match_count) / line_number) * 100, skip_count, (float(skip_count) / line_number) * 100, (line_number - match_count - skip_count), (1 - (float(match_count + skip_count) / line_number)) * 100, duration, line_number / duration)
+    print("Matched %i lines (%.2f%%), %i skipped (%.2f%%), %i unmatched (%.2f%%) in %.2fs (%i lines per second)." % \
+      (match_count, (match_count / line_number) * 100, skip_count, (skip_count / line_number) * 100, (line_number - match_count - skip_count), (1 - (match_count + skip_count) / line_number)) * 100, duration, line_number // duration),
+      file=sys.stderr)
 
 debug = False
 outfile_prefix = None
@@ -161,9 +164,9 @@ decimate_count = 1
 
 try:
   opts, file_list = getopt.getopt(sys.argv[1:], '', ['debug', 'config=', 'prefix=', 'no-average', 'sum', 'load=', 'save=', 'decimate='])
-except Exception, reason:
-  print >>sys.stderr, reason
-  print >>sys.stderr, usage
+except Exception as reason:
+  print(reason, file=sys.stderr)
+  print(usage, file=sys.stderr)
   sys.exit(1)
 
 for name, value in opts:
@@ -185,7 +188,7 @@ for name, value in opts:
     decimate_count = int(value)
 
 if configuration is None:
-  raise ValueError, '--config is mandatory'
+  raise ValueError('--config is mandatory')
 
 config_file = os.path.splitext(os.path.basename(configuration))[0]
 config_path = [os.path.dirname(os.path.abspath(configuration))] + sys.path
@@ -203,26 +206,24 @@ file_number = 0
 measure_dict = {}
 if len(load_file_name_list):
   for load_file_name in load_file_name_list:
-    load_file = open(load_file_name)
-    temp_measure_dict = eval(load_file.read(), {})
-    load_file.close()
+    with open(load_file_name) as load_file:
+      temp_measure_dict = eval(load_file.read(), {})
     assert isinstance(measure_dict, dict)
     for filter_id, result_dict in temp_measure_dict.iteritems():
       for result, date_dict in result_dict.iteritems():
         for date, duration_list in date_dict.iteritems():
           measure_dict.setdefault(filter_id, {}).setdefault(result, {}).setdefault(date, []).extend(duration_list)
-    print >>sys.stderr, 'Previous processing result restored from %r' % (load_file_name, )
+    print('Previous processing result restored from %r' % (load_file_name, ), file=sys.stderr)
 
 for filename in file_list:
   file_number += 1
-  print >>sys.stderr, 'Processing %s [%i/%i]...' % (filename, file_number, file_count)
+  print('Processing %s [%i/%i]...' % (filename, file_number, file_count), file=sys.stderr)
   parseFile(filename, measure_dict)
 
 if save_file_name is not None:
-  save_file = open(save_file_name, 'w')
-  save_file.write(repr(measure_dict))
-  save_file.close()
-  print >>sys.stderr, 'Processing result saved to %r' % (save_file_name, )
+  with open(save_file_name, 'w') as save_file:
+    save_file.write(repr(measure_dict))
+  print('Processing result saved to %r' % (save_file_name, ), file=sys.stderr)
 
 if outfile_prefix is not None:
   ## Generate a list of all measures and a 2-levels dictionnary with date as key and measure dictionnary as value
@@ -252,21 +253,21 @@ if outfile_prefix is not None:
   def renderOutput(data_format, filename_suffix):
     for sheet_id, sheet_column_list in sheet_dict.iteritems():
       outfile_name = '%s_%s_%s.csv' % (outfile_prefix, sheet_id, filename_suffix)
-      print >>sys.stderr, 'Writing to %r...' % (outfile_name, )
-      outfile = open(outfile_name, 'w')
-      print >>outfile, '"date",%s' % (','.join(['"%s"' % (x[0], ) for x in sheet_column_list]), )
-      decimate_dict = {}
-      decimate = 0
-      for date in date_list:
-        for key, value in line_dict[date].iteritems():
-          decimate_dict.setdefault(key, []).extend(value)
-        decimate += 1
-        if decimate == decimate_count:
-          print >>outfile, '"%s",%s' % (date, ','.join([render_cell(decimate_dict.get(x[1], ''), data_format) for x in sheet_column_list]))
-          decimate_dict = {}
-          decimate = 0
-      if len(decimate_dict):
-        print >>outfile, '"%s",%s' % (date, ','.join([render_cell(decimate_dict.get(x[1], ''), data_format) for x in sheet_column_list]))
+      print('Writing to %r...' % (outfile_name, ), file=sys.stderr)
+      with open(outfile_name, 'w') as outfile:
+        print('"date",%s' % (','.join(['"%s"' % (x[0], ) for x in sheet_column_list]), ), file=outfile)
+        decimate_dict = {}
+        decimate = 0
+        for date in date_list:
+          for key, value in line_dict[date].iteritems():
+            decimate_dict.setdefault(key, []).extend(value)
+          decimate += 1
+          if decimate == decimate_count:
+            print('"%s",%s' % (date, ','.join([render_cell(decimate_dict.get(x[1], ''), data_format) for x in sheet_column_list])), file=outfile)
+            decimate_dict = {}
+            decimate = 0
+        if len(decimate_dict):
+          print('"%s",%s' % (date, ','.join([render_cell(decimate_dict.get(x[1], ''), data_format) for x in sheet_column_list])), file=outfile)
 
   if do_average:
     renderOutput('=%(sum)i/%(count)i', 'avg')

erp5/util/timinglogplotter/__init__.py

@@ -27,6 +27,8 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
+from __future__ import division, print_function
+
 from datetime import date
 from os import path
 import rpy2.robjects as robjects
@@ -82,9 +84,9 @@ class CSVFile(object):
             if cell > value_max.get(key, 0):
               value_max[key] = cell
         column_dict[key].append(cell)
-    line_num = float(line_num) / 100
+    line_num = line_num / 100
     for key in ratio_dict:
-      ratio_dict[key] /= line_num
+      ratio_dict[key] //= line_num
 
   def getColumn(self, column_id):
     return self.column_dict[self.column_list[column_id]]
@@ -101,7 +103,7 @@ def computeExpr(expr):
   if expr:
     assert expr[0] == '='
     num, denom = expr[1:].split('/')
-    result = float(int(num)) / int(denom)
+    result = int(num) / int(denom)
   else:
     result = None
   return result
@@ -121,7 +123,7 @@ def main():
 
   current_dir = os.getcwd()
   for file_name in file_name_list:
-    print 'Loading %s...' % (file_name, )
+    print('Loading %s...' % (file_name, ))
     file = CSVFile(file_name)
 
     date_string_list = file.getColumn(0)
@@ -134,7 +136,7 @@ def main():
     # date_list will be like ['2009/07/01', '2009/07/05', '2009/07/10', ...]
     factor = 1
     if len(date_string_list) > 20:
-      factor = int(len(date_string_list) / 20)
+      factor = int(len(date_string_list) // 20)
     i = 0
     for date_string in date_string_list:
       if i % factor == 0:
@@ -183,13 +185,13 @@ def main():
           y_data.append(value)
         i += 1
       if len(x_data) == 0:
-        print 'Nothing to plot for %s...' % (out_file_name, )
+        print('Nothing to plot for %s...' % (out_file_name, ))
         continue
       if options.minimal_non_empty_ratio is not None:
         column_len = len(column)
         if column_len:
-          if float(len(x_data))/column_len < options.minimal_non_empty_ratio:
-            print 'Not enough values to plot for %s...' % (out_file_name, )
+          if len(x_data) / column_len < options.minimal_non_empty_ratio:
+            print('Not enough values to plot for %s...' % (out_file_name, ))
             continue
       r_y_data = robjects.FloatVector(y_data)
       r_x_data = robjects.FloatVector(x_data)
@@ -220,7 +222,7 @@ def main():
       # stop changing the out-type file
       r("""dev.off()""")
 
-      print 'Saving %s...' % (out_file_name, )
+      print('Saving %s...' % (out_file_name, ))
 
 if __name__ == '__main__':
   main()

erp5/util/webchecker/__init__.py

@@ -26,6 +26,8 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
+from __future__ import print_function
+
 import os
 import shutil
 import sys
@@ -543,7 +545,7 @@ class HTTPCacheCheckerTestSuite(object):
 
 
 from optparse import OptionParser
-import ConfigParser
+from six.moves import configparser
 
 def _formatConfiguration(configuration):
   """ format the configuration"""
@@ -559,11 +561,11 @@ def web_checker_utility():
 
   (options, args) = parser.parse_args()
   if len(args) != 1 :
-    print parser.print_help()
+    print(parser.print_help())
     parser.error('incorrect number of arguments')
   config_path = args[0]
 
-  config = ConfigParser.RawConfigParser()
+  config = configparser.RawConfigParser()
   config.read(config_path)
   working_directory = config.get('web_checker', 'working_directory')
   url = config.get('web_checker', 'url')
@@ -615,10 +617,9 @@ def web_checker_utility():
   result = instance.start(prohibited_file_name_list=prohibited_file_name_list,
                        prohibited_folder_name_list=prohibited_folder_name_list)
   if options.output_file:
-    file_object = open(options.output_file, 'w')
-    file_object.write(result)
-    file_object.close()
+    with open(options.output_file, 'w') as file_object:
+      file_object.write(result)
   else:
-    print result
+    print(result)
 
 

product/ERP5/bin/genbt5list

@@ -35,12 +35,24 @@ import posixpath
 import tarfile
 import os
 import sys
-import cgi
+try:
+  from html import escape
+except ImportError:
+  from cgi import escape # Deprecated since version 3.2
 from base64 import b64encode
-from cStringIO import StringIO
+from io import BytesIO
 from hashlib import sha1
-from urllib import unquote
-
+try:
+  from urllib.parse import unquote
+except ImportError:
+  from urllib import unquote
+
+if sys.version_info[0] == 3:
+  def iteritems(d):
+    return iter(d.items())
+else:
+  def iteritems(d):
+    return d.iteritems()
 
 # Order is important for installation
 # We want to have:
@@ -109,11 +121,11 @@ item_name_list = tuple('_%s_item' % x for x in item_name_list)
 class BusinessTemplateRevision(list):
 
   def hash(self, path, text):
-    self.append((path, sha1(text).digest()))
+    self.append((path.encode('utf-8'), sha1(text).digest()))
 
   def digest(self):
     self.sort()
-    return b64encode(sha1('\0'.join(h + p for (h, p) in self)).digest())
+    return b64encode(sha1(b'\0'.join(h + p for (h, p) in self)).digest())
 
 
 class BusinessTemplate(dict):
@@ -151,7 +163,7 @@ force_install
 
   def __iter__(self):
     self['revision'] = self.revision.digest()
-    return iter(sorted(self.iteritems()))
+    return iter(sorted(iteritems(self)))
 
   @classmethod
   def fromTar(cls, tar):
@@ -179,8 +191,8 @@ force_install
     return iter(self)
 
 def generateInformation(dir, info=id, err=None):
-  xml = StringIO()
-  xml.write('<?xml version="1.0"?>\n<repository>\n')
+  xml = BytesIO()
+  xml.write(b'<?xml version="1.0"?>\n<repository>\n')
   for name in sorted(os.listdir(dir)):
     path = os.path.join(dir, name)
     if name.endswith('.bt5'):
@@ -201,13 +213,16 @@ def generateInformation(dir, info=id, err=None):
       property_list = BusinessTemplate.fromDir(path)
     else:
       continue
-    xml.write('  <template id="%s">\n' % name)
+    xml.write(b'  <template id="%s">\n' % name.encode())
     for k, v in property_list:
-      for v in (v,) if type(v) is str else v:
-        xml.write('    <%s>%s</%s>\n' % (k, cgi.escape(v), k))
-    xml.write('  </template>\n')
+      if str is not bytes:
+        k = k.encode()
+      for v in (v,) if type(v) is bytes else v:
+        xml.write(b'    <%s>%s</%s>\n' % (k, escape(v) if str is bytes else
+                                             escape(v.decode()).encode(), k))
+    xml.write(b'  </template>\n')
     info('done\n')
-  xml.write('</repository>\n')
+  xml.write(b'</repository>\n')
   return xml
 
 def main(dir_list=None, **kw):