WIP-encode-decode: fixup! py2/py3: Make Products code compatible with both python2 and python3.

bt5/erp5_accounting_l10n_fr/SkinTemplateItem/portal_skins/erp5_accounting_l10n_fr/AccountingTransactionModule_aggregateFrenchAccountingTransactionFile.py

@@ -4,6 +4,7 @@ import six
 from io import BytesIO
 import zipfile
 from Products.ERP5Type.Message import translateString
+from Products.ERP5Type.Utils import ensure_ascii
 
 portal = context.getPortalObject()
 active_process = portal.restrictedTraverse(active_process)
@@ -19,11 +20,9 @@ if test_compta_demat_compatibility:
   # some "important" characters such as €
   # https://github.com/DGFiP/Test-Compta-Demat/issues/37
   # https://github.com/DGFiP/Test-Compta-Demat/issues/39
-  fec_file = unicodedata.normalize(
+  fec_file = ensure_ascii(unicodedata.normalize(
     'NFKD', fec_file.replace(u"€", "EUR")
-  ).encode(
-    'ascii', 'ignore'
-  ).decode('ascii')
+  ), 'ignore')
 
 zipbuffer = BytesIO()
 zipfilename = at_date.strftime('FEC-%Y%m%d.zip')

bt5/erp5_core_test/TestTemplateItem/portal_components/test.erp5.testERP5Core.py

@@ -721,7 +721,7 @@ class TestERP5Core(ERP5TypeTestCase, ZopeTestCase.Functional):
     self.assertEqual("""Path,Id,Title,Short Title,Reference,Codification,Int Index,Description
 *,bar,Bar,SBar,,,3,desc
 *,foo,Foo,,Rfoo,CFoo,,
-""", csv_data.decode())
+""", bytes2str(csv_data))
 
   def test_ERP5Site_reindexLatestIndexedObjects(self):
     module = self.portal.newContent(portal_type='Folder', id='test_folder')

bt5/erp5_core_test/TestTemplateItem/portal_components/test.erp5.testNotificationTool.py

@@ -82,7 +82,7 @@ def decode_email(file_):
       payload = part.get_payload(decode=True)
       #LOG('CMFMailIn -> ',0,'charset: %s, payload: %s' % (charset,payload))
       if charset:
-        payload = payload.decode(charset)
+        payload = six.text_type(payload, charset)
       if six.PY2:
         payload = unicode2str(payload)
       if body_found:

bt5/erp5_corporate_identity/SkinTemplateItem/portal_skins/erp5_corporate_identity/WebPage_viewAsBook.py

@@ -189,17 +189,17 @@ if book_include_reference_table:
     book_table_list=book_table_list
   )
   #if book_format == 'html' or book_format == 'mhtml':
-  #  book_references = book_references.encode('UTF-8').strip()
+  #  book_references = unicode2str(book_references).strip()
 
   # backcompat for manual history tables at the beginning of documents
   # NOTE: assumes <section>s are not used elsewhere!
   #if len(book_history_section_list) > 0:
-  #  book_content = book_content.replace(book_history_section_list[-1], (book_history_section_list[-1] + book_references.encode('UTF-8').strip()))
+  #  book_content = book_content.replace(book_history_section_list[-1], (book_history_section_list[-1] + unicode2str(book_references).strip()))
   #else:
-  #  book_content = book_content.replace("${WebPage_insertTableOfReferences}", book_references.encode('UTF-8').strip())
+  #  book_content = book_content.replace("${WebPage_insertTableOfReferences}", unicode2str(book_references).strip())
   book_references = book.Base_unescape(book_references)
   if six.PY2:
-    book_references = book_references.encode('utf-8')
+    book_references = unicode2str(book_references)
   book_content = book_content.replace("${WebPage_insertTableOfReferences}", book_references.strip())
 else:
   book_content = book_content.replace("${WebPage_insertTableOfReferences}", blank)
@@ -371,7 +371,7 @@ elif book_format == "pdf":
     )
   #if book_include_reference_table:
   #  after_toc_data_list.append(
-  #    b64encode(str2bytes(book.Base_convertHtmlToSingleFile(book_references, allow_script=True))).decode()
+  #    bytes2str(b64encode(str2bytes(book.Base_convertHtmlToSingleFile(book_references, allow_script=True))))
   #  )
   xsl_style_sheet_data = str2bytes(book_table_of_content)
   embedded_html_data = str2bytes(book.Base_convertHtmlToSingleFile(book_content, allow_script=True))

bt5/erp5_credential/TestTemplateItem/portal_components/test.erp5.testERP5Credential.py

@@ -207,7 +207,7 @@ class TestERP5Credential(ERP5TypeTestCase):
         payload = part.get_payload(decode=True)
         #LOG('CMFMailIn -> ',0,'charset: %s, payload: %s' % (charset,payload))
         if charset:
-          payload = payload.decode(charset)
+          payload = six.text_type(payload, charset)
         if six.PY2:
           payload = unicode2str(payload)
         if body_found:

bt5/erp5_simplified_invoicing/TestTemplateItem/portal_components/test.erp5.testInvoice.py

@@ -43,7 +43,7 @@ from Products.ERP5Type.tests.Sequence import SequenceList
 from erp5.component.test.testPackingList import TestPackingListMixin
 from Products.ERP5.tests.utils import newSimulationExpectedFailure
 from erp5.component.module.TestInvoiceMixin import TestInvoiceMixin, TestSaleInvoiceMixin
-from Products.ERP5Type.Utils import bytes2str
+from Products.ERP5Type.Utils import ensure_ascii
 
 class TestInvoice(TestInvoiceMixin):
   """Test methods for sale and purchase invoice.
@@ -567,7 +567,7 @@ class TestInvoice(TestInvoiceMixin):
     output = BytesIO()
     output.write(odt)
     m = OpenDocumentTextFile(output)
-    text_content=bytes2str(m.toString().encode('ascii','replace'))
+    text_content=ensure_ascii(m.toString(), 'replace')
     if text_content.find('Resource Tax') != -1 :
       self.fail('fail to delete the tax line in product line')
     if text_content.find('Tax Code') == -1 :

product/ERP5/bootstrap/erp5_core/MixinTemplateItem/portal_components/mixin.erp5.DownloadableMixin.py

@@ -31,7 +31,7 @@ from six.moves.urllib.parse import quote
 from AccessControl import ClassSecurityInfo
 from Products.ERP5Type.Globals import InitializeClass
 from Products.ERP5Type import Permissions
-from Products.ERP5Type.Utils import fill_args_from_request, str2bytes, str2unicode, unicode2str
+from Products.ERP5Type.Utils import fill_args_from_request, str2bytes, bytes2str, str2unicode, unicode2str
 from Products.CMFCore.utils import getToolByName, _checkConditionalGET, _setCacheHeaders,\
     _ViewEmulator
 import warnings
@@ -57,7 +57,7 @@ except ImportError:
       if six.PY2:
         quoted_file_name = quote(unicode2str(file_name))
       else:
-        encoded_file_name = encoded_file_name.decode('us-ascii')
+        encoded_file_name = bytes2str(encoded_file_name, 'us-ascii')
         quoted_file_name = quote(file_name)
 
       return '{disposition}; '\

product/ERP5Form/ListBox.py

@@ -38,7 +38,7 @@ from Products.Formulator.Field import ZMIField
 from Products.Formulator.Errors import FormValidationError, ValidationError
 from .Selection import Selection, DomainSelection
 from .Tool.SelectionTool import createFolderMixInPageSelectionMethod
-from Products.ERP5Type.Utils import UpperCase, ensure_list, getPath, str2bytes, bytes2str
+from Products.ERP5Type.Utils import UpperCase, ensure_list, getPath, str2bytes, bytes2str, unicode2str
 from Products.ERP5Type.Document import newTempBase
 from Products.CMFCore.utils import getToolByName
 from Products.ZSQLCatalog.zsqlbrain import ZSQLBrain
@@ -2716,10 +2716,7 @@ class ListBoxListRenderer(ListBoxRenderer):
     title_listboxline = ListBoxLine()
     title_listboxline.markTitleLine()
     for c in self.getSelectedColumnList():
-      if six.PY2:
-        title_listboxline.addColumn(c[0], c[1].encode(self.getEncoding()))
-      else:
-        title_listboxline.addColumn(c[0], c[1])
+      title_listboxline.addColumn(c[0], unicode2str(c[1], self.getEncoding()))
     listboxline_list.append(title_listboxline)
 
     # Obtain the list of lines.

product/ERP5OOo/OOoTemplate.py

@@ -48,7 +48,7 @@ from io import BytesIO
 import re
 import itertools
 import six
-from Products.ERP5Type.Utils import bytes2str, str2bytes
+from Products.ERP5Type.Utils import bytes2str, str2bytes, unicode2str
 
 try:
   from zExceptions import ResourceLockedError
@@ -103,7 +103,7 @@ class OOoTemplateStringIO(FasterStringIO):
   def write(self, s):
     return FasterStringIO.write(
         self,
-        str2bytes(convert_to_xml_compatible_string(s)))
+        str2bytes(unicode2str(convert_to_xml_compatible_string(s))))
 
 from Products.PageTemplates.Expressions import ZopeContext, createZopeEngine
 
@@ -346,10 +346,7 @@ class OOoTemplate(ZopePageTemplate):
                                       ('style', 'draw:style-name', 'fr1')):
         options_dict.setdefault(name, options_dict.pop(old_name, default))
 
-      if six.PY2:
-        picture = self._resolvePath(options_dict.pop('path').encode())
-      else:
-        picture = self._resolvePath(options_dict.pop('path'))
+      picture = self._resolvePath(options_dict.pop('path'))
 
       # If this is not a File, build a new file with this content
       if not isinstance(picture, File):

product/ERP5Type/__init__.py

@@ -188,7 +188,8 @@ ModuleSecurityInfo('Products.ERP5Type.Utils').declarePublic(
     'int2letter', 'getMessageIdWithContext', 'getTranslationStringWithContext',
     'Email_parseAddressHeader', 'guessEncodingFromText',
     'isValidTALESExpression',
-    'ensure_list', 'bytes2str', 'str2bytes', 'str2unicode', 'unicode2str',
+    'ensure_list', 'ensure_ascii',
+    'bytes2str', 'str2bytes', 'str2unicode', 'unicode2str',
 )
 
 allow_module('Products.ERP5Type.Message')