py2/py3: unicode->str conversion in Python 2 only.

bt5/cloudooo_web/SkinTemplateItem/portal_skins/cloudooo_erp5_ingestion/Document_getPropertyDictFromContent.py

@@ -8,7 +8,7 @@ to use re) that would analyze text content of the doc
 and return a dictionary of properties.
 """
 #Proxify to allow discover of metadata when publishing document
-
+import six
 information = context.getContentInformation()
 
 result = {}
@@ -16,7 +16,7 @@ property_id_list = context.propertyIds()
 for k, v in information.items():
   key = k.lower()
   if v:
-    if isinstance(v, unicode): v = v.encode('utf-8')
+    if six.PY2 and isinstance(v, six.text_type): v = v.encode('utf-8')
     if key in property_id_list:
       if key == 'reference':
         pass # XXX - We can not trust reference on getContentInformation

bt5/erp5_ingestion/SkinTemplateItem/portal_skins/erp5_ingestion/Document_getPropertyDictFromContent.py

@@ -7,6 +7,7 @@ To use, write your own method (probably External Method, since it is most likely
 to use re) that would analyze text content of the doc
 and return a dictionary of properties.
 """
+import six
 information = context.getContentInformation()
 
 result = {}
@@ -14,7 +15,7 @@ property_id_list = context.propertyIds()
 for k, v in information.items():
   key = k.lower()
   if v:
-    if isinstance(v, unicode): v = v.encode('utf-8')
+    if six.PY2 and isinstance(v, six.text_type): v = v.encode('utf-8')
     if key in property_id_list:
       if key == 'reference':
         pass # XXX - We can not trust reference on getContentInformation
@@ -26,7 +27,7 @@ for k, v in information.items():
         result['contributor'] = p.getRelativeUrl()
     elif key == 'keywords':
       if isinstance(v, (list, tuple)):
-        v = [isinstance(x, unicode) and x.encode('utf-8') or x for x in v]
+        v = [x.encode('utf-8') if six.PY2 and isinstance(x, text_type) else x for x in v]
       else:
         v = v.split()
       result['subject_list'] = v

product/ERP5/bootstrap/erp5_core/DocumentTemplateItem/portal_components/document.erp5.TextDocument.py

@@ -46,6 +46,7 @@ from Products.ERP5Type.Utils import guessEncodingFromText
 
 from lxml import html as etree_html
 from lxml import etree
+import six
 
 class TextDocument(CachedConvertableMixin, BaseConvertableFileMixin, TextContentHistoryMixin,
                                                             TextContent, File):
@@ -91,16 +92,20 @@ class TextDocument(CachedConvertableMixin, BaseConvertableFileMixin, TextContent
       mapping = method(**kw)
 
       is_str = isinstance(text, str)
-      if is_str:
+      if six.PY2 and is_str:
         text = text.decode('utf-8')
 
       class UnicodeMapping:
         def __getitem__(self, item):
           v = mapping[item]
-          if isinstance(v, str):
-            v = v.decode('utf-8')
-          elif not isinstance(v, unicode):
-            v = str(v).decode('utf-8')
+          if six.PY2:
+            if isinstance(v, str):
+              v = v.decode('utf-8')
+            elif not isinstance(v, six.text_type):
+              v = str(v).decode('utf-8')
+          else:
+            if not isinstance(v, str):
+              v = str(v)
           return v
       unicode_mapping = UnicodeMapping()
 

product/ERP5/bootstrap/erp5_core/MixinTemplateItem/portal_components/mixin.erp5.CrawlableMixin.py

@@ -111,7 +111,7 @@ class CrawlableMixin:
       # For now take into acount only a and img tags
       if attribute_name not in ('href',):
         continue
-      if isinstance(link, six.text_type):
+      if six.PY2 and isinstance(link, six.text_type):
         link = link.encode('utf-8')
       href_list.append(link)
     return href_list
@@ -129,7 +129,7 @@ class CrawlableMixin:
     path_part = '/'.join(path_part.split('/')[:-1])
     base_url = urlunsplit((splitted_url[0], splitted_url[1], path_part, None,
                            None))
-    if isinstance(base_url, six.text_type):
+    if six.PY2 and isinstance(base_url, six.text_type):
       base_url = base_url.encode('utf-8')
     return base_url
 
@@ -145,7 +145,7 @@ class CrawlableMixin:
     # in www.example.com or www.3.example.com
     # keep only the example.com part
     reference_domain = ''.join(reference_domain.split('.')[-2:])
-    if isinstance(reference_domain, six.text_type):
+    if six.PY2 and isinstance(reference_domain, six.text_type):
       reference_domain = reference_domain.encode('utf-8')
     url_list = []
     base_url = self.getContentBaseURL()
@@ -159,7 +159,7 @@ class CrawlableMixin:
       if not url:
         continue
       url_domain = urlsplit(url)[1]
-      if isinstance(url_domain, six.text_type):
+      if six.PY2 and isinstance(url_domain, six.text_type):
         url_domain = url_domain.encode('utf-8')
       if url_domain and ''.join(url_domain.split('.')[-2:]) != reference_domain:
         continue

product/ERP5Form/ListBox.py

@@ -2712,7 +2712,10 @@ class ListBoxListRenderer(ListBoxRenderer):
     title_listboxline = ListBoxLine()
     title_listboxline.markTitleLine()
     for c in self.getSelectedColumnList():
-      title_listboxline.addColumn(c[0], c[1].encode(self.getEncoding()))
+      if six.PY2:
+        title_listboxline.addColumn(c[0], c[1].encode(self.getEncoding()))
+      else:
+        title_listboxline.addColumn(c[0], c[1])
     listboxline_list.append(title_listboxline)
 
     # Obtain the list of lines.
@@ -2738,7 +2741,7 @@ class ListBoxListRenderer(ListBoxRenderer):
         listboxline.checkLine(uid in checked_uid_set)
 
       for (original_value, processed_value), (sql, title) in zip(line.getValueList(), self.getSelectedColumnList()):
-        if isinstance(original_value, six.text_type):
+        if six.PY2 and isinstance(original_value, six.text_type):
           value = original_value.encode(self.getEncoding())
         else:
           value = original_value
@@ -2756,7 +2759,7 @@ class ListBoxListRenderer(ListBoxRenderer):
       stat_listboxline.markStatLine()
 
       for (original_value, processed_value), (sql, title) in zip(self.getStatValueList(), self.getSelectedColumnList()):
-        if isinstance(original_value, six.text_type):
+        if six.PY2 and isinstance(original_value, six.text_type):
           value = original_value.encode(self.getEncoding())
         else:
           value = original_value

product/ERP5OOo/OOoTemplate.py

@@ -345,7 +345,10 @@ class OOoTemplate(ZopePageTemplate):
                                       ('style', 'draw:style-name', 'fr1')):
         options_dict.setdefault(name, options_dict.pop(old_name, default))
 
-      picture = self._resolvePath(options_dict.pop('path').encode())
+      if six.PY2:
+        picture = self._resolvePath(options_dict.pop('path').encode())
+      else:
+        picture = self._resolvePath(options_dict.pop('path'))
 
       # If this is not a File, build a new file with this content
       if not isinstance(picture, File):

product/ERP5Type/Utils.py

@@ -1757,7 +1757,7 @@ def legacyNormalizeUrl(url, base_url=None):
   # Remove trailing '?'
   # http://www.example.com/? -> http://www.example.com/
   url = re_cleanup_tail.sub('', url)
-  if isinstance(url, six.text_type):
+  if six.PY2 and isinstance(url, six.text_type):
     url = url.encode('utf-8')
   return url
 
@@ -1779,7 +1779,7 @@ def urlnormNormaliseUrl(url, base_url=None):
   if base_url and not (url_protocol or url_domain):
     # Make relative URL absolute
     url = urljoin(base_url, url)
-  if isinstance(url, six.text_type):
+  if six.PY2 and isinstance(url, six.text_type):
     url = url.encode('utf-8')
   return url
 

product/Formulator/FormToXML.py

@@ -20,7 +20,10 @@ def formToXML(form, prologue=1):
         else:
           value = 'false'
       sub_element = SubElement(form_as_xml, id)
-      sub_element.text = str(value).decode(encoding)
+      if six.PY2:
+        sub_element.text = str(value).decode(encoding)
+      else:
+        sub_element.text = str(value)
     groups = SubElement(form_as_xml, 'groups')
     # export form groups
     for group in form.get_groups(include_empty=1):
@@ -36,8 +39,7 @@ def formToXML(form, prologue=1):
 
         fields.append(field_element)
         values_element = SubElement(field_element, 'values')
-        items = field.values.items()
-        items.sort()
+        items = sorted(field.values.items())
         for key, value in items:
           if value is None:
             continue
@@ -57,19 +59,27 @@ def formToXML(form, prologue=1):
             if not isinstance(value, six.string_types):
               value = str(value)
             value_element = SubElement(values_element, key)
-          value_element.text = str(value).decode(encoding)
+          if six.PY2:
+            value_element.text = str(value).decode(encoding)
+          else:
+            value_element.text = str(value)
 
         tales_element = SubElement(field_element, 'tales')
-        items = field.tales.items()
-        items.sort()
+        items = sorted(field.tales.items())
         for key, value in items:
           if value:
             tale_element = SubElement(tales_element, key)
-            tale_element.text = str(value._text).decode(encoding)
+            if six.PY2:
+              tale_element.text = str(value._text).decode(encoding)
+            else:
+              tale_element.text = str(value._text)
         messages = SubElement(field_element, 'messages')
         for message_key in field.get_error_names():
           message_element = SubElement(messages, 'message', name=message_key)
-          message_element.text = field.get_error_message(message_key).decode(encoding)
+          if six.PY2:
+            message_element.text = field.get_error_message(message_key).decode(encoding)
+          else:
+            message_element.text = field.get_error_message(message_key)
         # Special attribute for ProxyFields *delegated_list*
         delegated_list = getattr(field, 'delegated_list', [])
         if delegated_list:
@@ -78,8 +88,12 @@ def formToXML(form, prologue=1):
           [SubElement(delegated_list_element, delegated) for delegated in delegated_list]
 
     if form.unicode_mode:
-      return etree.tostring(form_as_xml, encoding='utf-8',
+      xml = etree.tostring(form_as_xml, encoding='utf-8',
                                     xml_declaration=True, pretty_print=True)
     else:
-      return etree.tostring(form_as_xml, encoding=form.stored_encoding,
+      xml = etree.tostring(form_as_xml, encoding=form.stored_encoding,
                                     xml_declaration=True, pretty_print=True)
+    if six.PY2:
+      return xml
+    else:
+      return xml.decode()

product/Formulator/Validator.py

@@ -517,7 +517,7 @@ class SelectionValidator(StringBaseValidator):
         # will remain integers.
         # XXX it is impossible with the UI currently to fill in unicode
         # items, but it's possible to do it with the TALES tab
-        if field.get_value('unicode') and isinstance(item_value, six.text_type):
+        if six.PY2 and field.get_value('unicode') and isinstance(item_value, six.text_type):
           str_value = item_value.encode(field.get_form_encoding())
         else:
           str_value = str(item_value)
@@ -908,10 +908,15 @@ fullwidth_minus_character_list = (
     )
 def normalizeFullWidthNumber(value):
   try:
-    value = unicodedata.normalize('NFKD', value.decode('UTF8'))
+    if six.PY2:
+      value = unicodedata.normalize('NFKD', value.decode('UTF8'))
+    else:
+      value = unicodedata.normalize('NFKD', value)
     if value[0] in fullwidth_minus_character_list:
       value = u'-' + value[1:]
     value = value.encode('ASCII', 'ignore')
+    if six.PY3:
+      value = value.decode()
   except UnicodeDecodeError:
     pass
   return value

product/Formulator/Widget.py

@@ -41,7 +41,7 @@ def convert_to_xml_compatible_string(value):
   """
   if not value:
     return ''
-  if isinstance(value, str):
+  if six.PY2 and isinstance(value, str):
     value = value.decode('utf-8')
 
   # remove control characters as described in the example from
@@ -260,7 +260,7 @@ class Widget:
     """
     if attr_dict is None:
       attr_dict = {}
-    if isinstance(value, str):
+    if six.PY2 and isinstance(value, str):
       #required by lxml
       value = value.decode('utf-8')
     if value is None:
@@ -287,7 +287,7 @@ class Widget:
     if attr_dict is None:
       attr_dict = {}
     attr_dict['{%s}value-type' % OFFICE_URI] = 'string'
-    if isinstance(value, str):
+    if six.PY2 and isinstance(value, str):
       #required by lxml
       value = value.decode('utf-8')
     if value is None:
@@ -330,7 +330,7 @@ class Widget:
     """
     if attr_dict is None:
       attr_dict = {}
-    if isinstance(value, str):
+    if six.PY2 and isinstance(value, str):
       #required by lxml
       value = value.decode('utf-8')
     if value is None:
@@ -618,7 +618,7 @@ class CheckBoxWidget(Widget):
       attr_dict = {}
     if isinstance(value, int):
       value = str(value)
-    if isinstance(value, str):
+    if six.PY2 and isinstance(value, str):
       #required by lxml
       value = value.decode('utf-8')
     text_node = Element('{%s}%s' % (TEXT_URI, local_name), nsmap=NSMAP)
@@ -723,7 +723,7 @@ class TextAreaWidget(Widget):
         render_prefix, attr_dict, local_name):
         if attr_dict is None:
             attr_dict = {}
-        if isinstance(value, str):
+        if six.PY2 and isinstance(value, str):
             #required by lxml
             value = value.decode('utf-8')
         text_node = Element('{%s}%s' % (TEXT_URI, local_name), nsmap=NSMAP)
@@ -995,7 +995,7 @@ class SingleItemsWidget(ItemsWidget):
 
     if attr_dict is None:
       attr_dict = {}
-    if isinstance(value, str):
+    if six.PY2 and isinstance(value, str):
       #required by lxml
       value = value.decode('utf-8')
     text_node = Element('{%s}%s' % (TEXT_URI, local_name), nsmap=NSMAP)
@@ -1146,7 +1146,10 @@ class MultiItemsWidget(ItemsWidget):
     if value is None:
       return None
     value_list = self.render_items_odf(field, value, REQUEST)
-    value = ', '.join(value_list).decode('utf-8')
+    if six.PY2:
+      value = ', '.join(value_list).decode('utf-8')
+    else:
+      value = ', '.join(value_list)
     return Widget.render_odg(self, field, value, as_string, ooo_builder,
                              REQUEST, render_prefix, attr_dict, local_name)
 
@@ -1178,7 +1181,7 @@ class MultiItemsWidget(ItemsWidget):
 
     if attr_dict is None:
       attr_dict = {}
-    if isinstance(value, str):
+    if six.PY2 and isinstance(value, str):
       #required by lxml
       value = value.decode('utf-8')
     text_node = Element('{%s}%s' % (TEXT_URI, local_name), nsmap=NSMAP)
@@ -1712,7 +1715,10 @@ class DateTimeWidget(Widget):
     # get the field value
     if not value and field.get_value('default_now'):
       value = DateTime()
-    text_node.text = self.format_value(field, value, mode='pdf').decode('utf-8')
+    if six.PY2:
+      text_node.text = self.format_value(field, value, mode='pdf').decode('utf-8')
+    else:
+      text_node.text = self.format_value(field, value, mode='pdf')
     text_node.attrib.update(attr_dict)
     if as_string:
       return etree.tostring(text_node)
@@ -1754,7 +1760,10 @@ class DateTimeWidget(Widget):
     """
     if not value and field.get_value('default_now'):
       value = DateTime()
-    value_as_text = self.format_value(field, value, mode='pdf').decode('utf-8')
+    if six.PY2:
+      value_as_text = self.format_value(field, value, mode='pdf').decode('utf-8')
+    else:
+      value_as_text = self.format_value(field, value, mode='pdf')
     return Widget.render_odg_view(self, field, value_as_text, as_string,
                                       ooo_builder, REQUEST, render_prefix,
                                       attr_dict, local_name)
@@ -1928,7 +1937,7 @@ class IntegerWidget(TextWidget) :
     if attr_dict is None:
       attr_dict = {}
     attr_dict['{%s}value-type' % OFFICE_URI] = 'float'
-    if isinstance(value, str):
+    if six.PY2 and isinstance(value, str):
       #required by lxml
       value = value.decode('utf-8')
     text_node = Element('{%s}%s' % (TEXT_URI, local_name), nsmap=NSMAP)
@@ -2128,7 +2137,10 @@ class FloatWidget(TextWidget):
     if attr_dict is None:
       attr_dict = {}
     text_node = Element('{%s}%s' % (TEXT_URI, local_name), nsmap=NSMAP)
-    text_node.text = self.format_value(field, value).decode('utf-8')
+    if six.PY2:
+      text_node.text = self.format_value(field, value).decode('utf-8')
+    else:
+      text_node.text = self.format_value(field, value)
     text_node.attrib.update(attr_dict)
     if as_string:
       return etree.tostring(text_node)
@@ -2146,7 +2158,7 @@ class FloatWidget(TextWidget):
     if attr_dict is None:
       attr_dict = {}
     attr_dict['{%s}value-type' % OFFICE_URI] = 'float'
-    if isinstance(value, str):
+    if six.PY2 and isinstance(value, str):
       #required by lxml
       value = value.decode('utf-8')
     text_node = Element('{%s}%s' % (TEXT_URI, local_name), nsmap=NSMAP)

product/Formulator/XMLToForm.py

@@ -3,6 +3,7 @@ from __future__ import absolute_import
 from . import XMLObjects
 from Products.Formulator.TALESField import TALESMethod
 from Products.Formulator.MethodField import Method
+import six
 
 def XMLToForm(s, form, override_encoding=None):
     """Takes an xml string and changes formulator form accordingly.
@@ -127,7 +128,7 @@ def XMLToForm(s, form, override_encoding=None):
         form.remove_group('Default')
 
 def encode(text, encoding):
-    if encoding is None:
+    if six.PY3 or encoding is None:
         return text
     else:
         return text.encode(encoding)

product/Formulator/tests/testFormValidator.py

@@ -56,11 +56,11 @@ class StringValidatorTestCase(ValidatorTestCase):
         self.assertEqual('<html>', result)
 
     def test_encoding(self):
-        utf8_string = 'M\303\274ller' # this is a M&uuml;ller
-        unicode_string = unicode(utf8_string, 'utf-8')
+        utf8_bytes = b'M\303\274ller' # this is a M&uuml;ller
+        unicode_string = utf8_bytes.decode('utf-8')
         result = self.v.validate(
             TestField('f', max_length=0, truncate=0, required=0, unicode=1),
-            'f', {'f' : utf8_string})
+            'f', {'f' : utf8_bytes})
         self.assertEqual(unicode_string, result)
 
     def test_strip_whitespace(self):