fixup! py2/py3: Make Products code compatible with both python2 and python3.

product/ERP5/Document/BusinessTemplate.py

@@ -29,7 +29,7 @@
 
 import six
 from six import string_types as basestring
-from Products.ERP5Type.Utils import ensure_list, bytes2str
+from Products.ERP5Type.Utils import ensure_list, bytes2str, str2bytes
 import fnmatch, gc, glob, imp, os, re, shutil, sys, time, tarfile
 from collections import defaultdict
 from Shared.DC.ZRDB import Aqueduct
@@ -350,6 +350,8 @@ class BusinessTemplateArchive(object):
       self._writeString(obj, path)
     else:
       if isinstance(obj, str):
+        obj = str2bytes(obj)
+      if isinstance(obj, bytes):
         self.revision.hash(path, obj)
         obj = BytesIO(obj)
       else:
@@ -825,7 +827,7 @@ class ObjectTemplateItem(BaseTemplateItem):
             obj = obj._getCopy(context)
             data = getattr(aq_base(obj), record_id, None)
             if unicode_data:
-              if not isinstance(data, six.text_type):
+              if not (six.PY2 and isinstance(data, six.text_type)):
                 break
               try:
                 data = data.encode(aq_base(obj).output_encoding)
@@ -3523,7 +3525,7 @@ class PortalTypeRolesTemplateItem(BaseTemplateItem):
     path = self.__class__.__name__
     for key in self._objects:
       xml_data = self.generateXml(key)
-      if isinstance(xml_data, six.text_type):
+      if six.PY2 and isinstance(xml_data, six.text_type):
         xml_data = xml_data.encode('utf-8')
       name = key.split('/', 1)[1]
       bta.addObject(xml_data, name=name, path=path)
@@ -3537,7 +3539,7 @@ class PortalTypeRolesTemplateItem(BaseTemplateItem):
     xml_type_roles_list = xml.findall('role')
     for role in xml_type_roles_list:
       id = role.get('id')
-      if isinstance(id, six.text_type):
+      if six.PY2 and isinstance(id, six.text_type):
         id = id.encode('utf_8', 'backslashreplace')
       type_role_property_dict = {'id': id}
       # uniq
@@ -3546,7 +3548,7 @@ class PortalTypeRolesTemplateItem(BaseTemplateItem):
         property_id = property_node.get('id')
         if property_node.text:
           value = property_node.text
-          if isinstance(value, six.text_type):
+          if six.PY2 and isinstance(value, six.text_type):
             value = value.encode('utf_8', 'backslashreplace')
           type_role_property_dict[property_id] = value
       # multi
@@ -3555,7 +3557,7 @@ class PortalTypeRolesTemplateItem(BaseTemplateItem):
         property_id = property_node.get('id')
         if property_node.text:
           value = property_node.text
-          if isinstance(value, six.text_type):
+          if six.PY2 and isinstance(value, six.text_type):
             value = value.encode('utf_8', 'backslashreplace')
           type_role_property_dict.setdefault(property_id, []).append(value)
       type_roles_list.append(type_role_property_dict)
@@ -4964,7 +4966,7 @@ class LocalRolesTemplateItem(BaseTemplateItem):
       xml_data += '\n </local_role_group_ids>'
 
     xml_data += '\n</local_roles_item>'
-    if isinstance(xml_data, six.text_type):
+    if six.PY2 and isinstance(xml_data, six.text_type):
       xml_data = xml_data.encode('utf8')
     return xml_data
 

product/ERP5/Tool/TemplateTool.py

@@ -1080,7 +1080,7 @@ class TemplateTool (BaseTool):
                               installed_revision=installed_revision,
                               repository=repository,
                               **property_dict)
-        obj.setUid(uid)
+        obj.setUid(bytes2str(uid))
         result_list.append(obj)
       result_list.sort(key=lambda x: x.getTitle())
       return result_list

product/ERP5/bootstrap/erp5_core/MixinTemplateItem/portal_components/mixin.erp5.SimulableMixin.py

@@ -34,6 +34,7 @@ from Products.ERP5Type.Globals import InitializeClass
 from Products.ERP5Type.Base import Base
 from Products.ERP5Type.TransactionalVariable import getTransactionalVariable
 from Products.ERP5Type.Errors import SimulationError
+from Products.ERP5Type.Utils import ensure_list
 
 class SimulableMixin(Base):
   security = ClassSecurityInfo()
@@ -75,7 +76,7 @@ class SimulableMixin(Base):
         ignore.update(kw)
         tv[ignore_key] = ignore
       transaction.get().addBeforeCommitHook(before_commit)
-    for k, v in item_list:
+    for k, v in ensure_list(item_list):
       if not v:
         ignore.add(k)
       elif k not in ignore:

product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/Base_translateString.py

@@ -12,8 +12,10 @@ if translation_service is not None :
     if not encoding:
       return translation_service.translate(catalog, msg, lang=lang, **kw)
     msg = translation_service.translate(catalog, msg, lang=lang, **kw)
-    if isinstance(msg, six.text_type):
+    if six.PY2 and isinstance(msg, six.text_type):
       msg = msg.encode(encoding)
+    elif six.PY3 and isinstance(msg, six.binary_type):
+      msg = msg.decode(encoding)
     return msg
   except AttributeError: # This happens in unit testing, because it is not able to find something with get_context()
     pass

product/ERP5Form/Tool/SelectionTool.py

@@ -32,6 +32,7 @@
 """
 
 from OFS.SimpleItem import SimpleItem
+from Products.ERP5Type.Utils import ensure_list
 from Products.ERP5Type.Globals import InitializeClass, DTMLFile, PersistentMapping, get_request
 from AccessControl import ClassSecurityInfo
 from ZTUtils import make_query
@@ -434,7 +435,7 @@ class SelectionTool( BaseTool, SimpleItem ):
             selection_uid_dict[int(uid)] = 1
           except (ValueError, TypeError):
             selection_uid_dict[uid] = 1
-        self.setSelectionCheckedUidsFor(list_selection_name, selection_uid_dict.keys(), REQUEST=REQUEST)
+        self.setSelectionCheckedUidsFor(list_selection_name, ensure_list(selection_uid_dict.keys()), REQUEST=REQUEST)
       if REQUEST is not None:
         return self._redirectToOriginalForm(REQUEST=REQUEST, form_id=form_id,
                                             query_string=query_string, no_reset=True)

product/ERP5Security/ERP5KeyAuthPlugin.py

@@ -46,6 +46,7 @@ from Products.PluggableAuthService.plugins.CookieAuthHelper import CookieAuthHel
 
 from Products.ERP5Type.Cache import CachingMethod
 from Products.ERP5Type.UnrestrictedMethod import UnrestrictedMethod
+from Products.ERP5Type.Utils import bytes2str
 from Products.ERP5Security.ERP5UserManager import ERP5UserManager, \
                                                   _AuthenticationFailure
 from Products import ERP5Security
@@ -64,7 +65,7 @@ class AESCipher:
   def encrypt(self, login):
     iv = Random.new().read(AES.block_size)
     encryptor = AES.new(self.encryption_key, self.mode, IV=iv)
-    return urlsafe_b64encode(iv + encryptor.encrypt(login.ljust(((len(login)-1)/16+1)*16)))
+    return urlsafe_b64encode(iv + encryptor.encrypt(login.ljust(((len(login)-1)//16+1)*16)))
 
   def decrypt(self, crypted_login):
     decoded_crypted_login = urlsafe_b64decode(crypted_login)
@@ -217,13 +218,13 @@ class ERP5KeyAuthPlugin(ERP5UserManager, CookieAuthHelper):
   def encrypt(self, login):
     """Encrypt the login"""
     cipher = globals()['%sCipher' % self._getCipher()](self.encryption_key)
-    return cipher.encrypt(login)
+    return bytes2str(cipher.encrypt(login))
 
   security.declarePrivate('decrypt')
   def decrypt(self, crypted_login):
     """Decrypt string and return the login"""
     cipher = globals()['%sCipher' % self._getCipher()](self.encryption_key)
-    return cipher.decrypt(crypted_login)
+    return bytes2str(cipher.decrypt(crypted_login))
 
   ####################################
   #ILoginPasswordHostExtractionPlugin#

product/ERP5Security/tests/testERP5Security.py

@@ -34,9 +34,11 @@ import mock
 import itertools
 import transaction
 import unittest
-import urlparse
+import six
+from six.moves.urllib.parse import urlparse, parse_qs
 from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase
 from Products.ERP5Type.tests.utils import createZODBPythonScript
+from Products.ERP5Type.Utils import bytes2str
 from AccessControl.SecurityManagement import newSecurityManager
 from AccessControl.SecurityManagement import getSecurityManager
 from AccessControl import SpecialUsers
@@ -57,8 +59,10 @@ class UserManagementTestCase(ERP5TypeTestCase):
   """TestCase for user manement, with utilities to create users and helpers
   assertion methods.
   """
-  _login_generator = itertools.count().next
-
+  if six.PY2:
+    _login_generator = itertools.count().next
+  else:
+    _login_generator = itertools.count().__next__
 
   def getBusinessTemplateList(self):
     """List of BT to install. """
@@ -719,12 +723,12 @@ class TestPreferences(UserManagementTestCase):
       current_password='bad' + password,
       new_password=new_password,
     )
-    parsed_url = urlparse.urlparse(result)
+    parsed_url = urlparse(result)
     self.assertEqual(
         parsed_url.path.split('/')[-2:],
         ['portal_preferences', 'PreferenceTool_viewChangePasswordDialog'])
     self.assertEqual(
-        urlparse.parse_qs(parsed_url.query),
+        parse_qs(parsed_url.query),
         {'portal_status_message': ['Current password is wrong.'], 'portal_status_level': ['error']})
 
     self.login()
@@ -1179,8 +1183,8 @@ class TestUserManagementExternalAuthentication(TestUserManagement):
     # view front page we should be logged in if we use authentication key
     response = self.publish(base_url, env={self.user_id_key.replace('-', '_').upper(): login})
     self.assertEqual(response.getStatus(), 200)
-    self.assertIn('Logged In', response.getBody())
-    self.assertIn(login, response.getBody())
+    self.assertIn('Logged In', bytes2str(response.getBody()))
+    self.assertIn(login, bytes2str(response.getBody()))
 
 
 class TestLocalRoleManagement(RoleManagementTestCase):
@@ -1271,7 +1275,7 @@ class TestLocalRoleManagement(RoleManagementTestCase):
     # check if assignment change is effective immediately
     self.login()
     res = viewSecurity()
-    self.assertEqual([x for x in res.body.splitlines() if x.startswith('-->')],
+    self.assertEqual([x for x in bytes2str(res.body).splitlines() if x.startswith('-->')],
                      ["--> ['F1_G1_S1']"], res.body)
     assignment = self.person.newContent( portal_type='Assignment',
                                   group='subcat',
@@ -1279,11 +1283,11 @@ class TestLocalRoleManagement(RoleManagementTestCase):
                                   function='another_subcat' )
     assignment.open()
     res = viewSecurity()
-    self.assertEqual([x for x in res.body.splitlines() if x.startswith('-->')],
+    self.assertEqual([x for x in bytes2str(res.body).splitlines() if x.startswith('-->')],
                      ["--> ['F1_G1_S1']", "--> ['F2_G1_S1']"], res.body)
     assignment.setGroup('another_subcat')
     res = viewSecurity()
-    self.assertEqual([x for x in res.body.splitlines() if x.startswith('-->')],
+    self.assertEqual([x for x in bytes2str(res.body).splitlines() if x.startswith('-->')],
                      ["--> ['F1_G1_S1']", "--> ['F2_G2_S1']"], res.body)
     self.abort()
 
@@ -1499,7 +1503,7 @@ class TestKeyAuthentication(RoleManagementTestCase):
     # view front page we should be logged in if we use authentication key
     response = self.publish('%s?__ac_key=%s' %(base_url, key))
     self.assertEqual(response.getStatus(), 200)
-    self.assertIn(reference, response.getBody())
+    self.assertIn(reference, bytes2str(response.getBody()))
 
     # check if key authentication works other page than front page
     person_module = portal.person_module
@@ -1510,7 +1514,7 @@ class TestKeyAuthentication(RoleManagementTestCase):
     self.assertTrue('%s/login_form?came_from=' % portal.getId(), response.headers['location'])
     response = self.publish('%s?__ac_key=%s' %(base_url, key))
     self.assertEqual(response.getStatus(), 200)
-    self.assertIn(reference, response.getBody())
+    self.assertIn(reference, bytes2str(response.getBody()))
 
     # check if key authentication works with web_mode too
     web_site = portal.web_site_module.newContent(portal_type='Web Site')

product/ERP5Type/Message.py

@@ -114,7 +114,7 @@ class Message(Persistent):
     if self.domain is None:
       # Map the translated string with given parameters
       if type(self.mapping) is dict:
-        if isinstance(message, six.text_type) :
+        if six.PY2 and isinstance(message, six.text_type) :
           message = message.encode('utf-8')
         message = Template(message).substitute(self.mapping)
     else:
@@ -124,7 +124,7 @@ class Message(Persistent):
       if self.mapping:
         unicode_mapping = {}
         for k, v in six.iteritems(self.mapping):
-          if isinstance(v, str):
+          if six.PY2 and isinstance(v, str):
             v = v.decode('utf-8')
           unicode_mapping[k] = v
       else:
@@ -139,9 +139,9 @@ class Message(Persistent):
         message = translated_message
 
     if isinstance(self.message, str):
-      if isinstance(message, six.text_type):
+      if six.PY2 and isinstance(message, six.text_type):
         message = message.encode('utf-8')
-    elif isinstance(message, str):
+    elif six.PY2 and isinstance(message, str):
       message = message.decode('utf-8')
 
     return message
@@ -154,7 +154,7 @@ class Message(Persistent):
     Return the translated message as a string object.
     """
     message = self.translate()
-    if isinstance(message, six.text_type):
+    if six.PY2 and isinstance(message, six.text_type):
       message = message.encode('utf-8')
     return message
 
@@ -163,7 +163,7 @@ class Message(Persistent):
     Return the translated message as a unicode object.
     """
     message = self.translate()
-    if isinstance(message, str):
+    if six.PY2 and isinstance(message, str):
       message = message.decode('utf-8')
     return message
 

product/ERP5Type/patches/DA.py

@@ -219,7 +219,7 @@ def DA__call__(self, REQUEST=None, __ick__=None, src__=0, test__=0, **kw):
     if src__: return query
 
     if self.cache_time_ > 0 and self.max_cache_ > 0:
-        result=self._cached_result(DB__, query, self.max_rows_, c)
+        result=self._cached_result(DB__, str2bytes(query), self.max_rows_, c)
     else:
       try:
         result=DB__.query(query, self.max_rows_)

product/ERP5Type/tests/ERP5TypeTestCase.py

@@ -9,7 +9,7 @@ __version__ = '0.3.0'
 
 import base64
 import errno
-import httplib
+from six.moves import http_client
 import os
 import random
 import re
@@ -19,12 +19,12 @@ import sys
 import time
 import traceback
 import urllib
-import ConfigParser
+from six.moves import configparser
 from contextlib import contextmanager
 from io import BytesIO
 from functools import partial
 from six.moves.urllib.parse import unquote_to_bytes
-from cPickle import dumps
+from six.moves.cPickle import dumps
 from glob import glob
 from hashlib import md5
 from warnings import warn
@@ -32,7 +32,6 @@ from DateTime import DateTime
 import mock
 import Products.ZMySQLDA.DA
 from Products.ZMySQLDA.DA import Connection as ZMySQLDA_Connection
-from zope.globalrequest import clearRequest
 from zope.globalrequest import getRequest
 from zope.globalrequest import setRequest
 import six
@@ -154,7 +153,7 @@ def _createTestPromiseConfigurationFile(promise_path, bt5_repository_path_list=N
                              _getVolatileMemcachedServerDict()
   cloudooo_url_list = _getConversionServerUrlList()
 
-  promise_config = ConfigParser.RawConfigParser()
+  promise_config = configparser.RawConfigParser()
   promise_config.add_section('external_service')
   promise_config.set('external_service', 'cloudooo_url_list', cloudooo_url_list)
   promise_config.set('external_service', 'memcached_url',memcached_url)

product/ERP5Type/tests/ProcessingNodeTestCase.py

@@ -2,7 +2,10 @@
 import errno, logging, mock, os, socket, time
 import itertools
 from threading import Thread
-from UserDict import IterableUserDict
+try: # six.PY2
+  from UserDict import IterableUserDict as UserDict
+except ImportError:
+  from collections import UserDict
 import transaction
 from Testing import ZopeTestCase
 from zope.globalrequest import setRequest
@@ -15,7 +18,7 @@ from Products.ERP5Type.tests.utils import \
 from Products.CMFActivity.ActivityTool import getCurrentNode
 
 
-class DictPersistentWrapper(IterableUserDict, object):
+class DictPersistentWrapper(UserDict, object):
 
   def __metaclass__(name, base, d):
     def wrap(attr):

product/ERP5Type/tests/utils.py

@@ -55,8 +55,12 @@ from Products.ERP5Type.Utils import simple_decorator
 from Products.ZSQLCatalog.SQLCatalog import Catalog
 import pytz
 import six
+if six.PY2:
+  FileIO = file
+else:
+  from io import FileIO
 
-class FileUpload(file):
+class FileUpload(FileIO):
   """Act as an uploaded file.
   """
   __allow_access_to_unprotected_subobjects__ = 1

product/ZSQLCatalog/Query/AutoQuery.py

@@ -76,17 +76,17 @@ class AutoQuery(Query):
       # Recreate value as a dict and pass it to buildSingleQuery.
       range = kw.pop('range')
       assert len(kw) == 1, repr(kw)
-      key, value = kw.items()[0]
+      key, value = list(kw.items())[0]
       query = sql_catalog.buildSingleQuery(key, {'query': value,
                                                  'range': range})
     elif operator == 'in':
       # 'in' is a *comparison* operator, not a logical operator.
       # Transform kw into the proper form.
       assert len(kw) == 1, repr(kw)
-      key, value = kw.items()[0]
+      key, value = list(kw.items())[0]
       query = sql_catalog.buildSingleQuery(key, {'query': value,
                                                  'operator': operator})
-    elif len(kw) == 1 and isinstance(kw.values()[0], (tuple, list)) and \
+    elif len(kw) == 1 and isinstance(list(kw.values())[0], (tuple, list)) and \
        operator in ('and', 'or'):
       # If there is only one parameter, and operator was given and is a
       # known logical operator, then operator will apply to it.
@@ -94,7 +94,7 @@ class AutoQuery(Query):
       #  kw = {'portal_type': ['!=a', '!=b'], 'operator': 'AND'}
       #  In such case, expected result is
       #  "portal_type!='a' AND portal_type!='b'"
-      key, value = kw.items()[0]
+      key, value = list(kw.items())[0]
       query = sql_catalog.buildSingleQuery(key, value, logical_operator=operator)
     else:
       # Otherwise, the operator will apply to the relationship between
@@ -102,7 +102,7 @@ class AutoQuery(Query):
       if operator is None:
         operator = 'and'
       if self.search_key is not None:
-        key, value = kw.items()[0]
+        key, value = list(kw.items())[0]
         kw = {key: {'query': value, 'key': self.search_key}}
       query = sql_catalog.buildQuery(kw, operator=operator, ignore_empty_string=self.ignore_empty_string)
     if self.table_alias_list is not None:

product/ZSQLCatalog/SearchText/lexer.py

@@ -58,7 +58,10 @@ class lexer(object):
                             write_tables=False)
     sys.stdout, sys.stderr = sys.__stdout__, sys.__stderr__
     # Emit all logs with regular Zope logging
-    for line in output.getvalue().split('\n'):
+    value = output.getvalue()
+    if six.PY3:
+      value = value.decode()
+    for line in value.split('\n'):
       if len(line):
         LOG('lexer', 0, line)
 

product/ZSQLCatalog/tests/testSQLCatalog.py

@@ -39,7 +39,9 @@ from Products.ZSQLCatalog.Query.RelatedQuery import RelatedQuery
 from DateTime import DateTime
 from Products.ZSQLCatalog.SQLExpression import MergeConflictError
 from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase
+from Products.ERP5Type.Utils import ensure_list
 import six
+from six.moves import xrange
 
 class MatchList(list):
   def __repr__(self):
@@ -99,7 +101,7 @@ class ReferenceQuery:
       else:
         self.args.append(arg)
     if len(kw) == 1:
-      self.column, value = kw.items()[0]
+      self.column, value = ensure_list(kw.items())[0]
       if not isinstance(value, MatchList):
         value = MatchList([value])
       self.value = value