Update from upstream/master

bt5/erp5_accounting/SkinTemplateItem/portal_skins/erp5_accounting/AccountingTransaction_createReversalTransaction.py

@@ -64,11 +64,11 @@ line_list = context.AccountingTransaction_getAccountingTransactionLineList(
 if not cancellation_amount:
   line_list.reverse()
 
-# guess portal_type to create lines
 if line_list:
+  # guess portal_type to create lines
   line_portal_type = line_list[0].getPortalType()
 
-for line in line_list:
+  for line in line_list:
     new_line = reversal.newContent( portal_type=line_portal_type )
     new_line.edit(
       source=line.getSource(portal_type='Account'),

bt5/erp5_advanced_ecommerce/SkinTemplateItem/portal_skins/erp5_advanced_ecommerce/Resource_addToShoppingCart.py

@@ -55,6 +55,7 @@ shopping_cart_items = context.SaleOrder_getShoppingCartItemList()
 
 # get category like size and variation
 category = request.form.get('field_variation_box_your_category', '')
+base_category = ''
 if category:
   [base_category, category] = category.split('/', 1)
 variation = request.form.get('field_variation_box_your_variation', None)

bt5/erp5_advanced_ecommerce/SkinTemplateItem/portal_skins/erp5_advanced_ecommerce/Resource_getPriceCalculationDefaultContext.py

@@ -37,8 +37,7 @@ else:
   variation_dict["start_date"] = date
   variation_dict["stop_date"] = date+0.00001
 
-if web_site_value is not None:
-  price_currency_value = web_site_value.WebSite_getShoppingCartDefaultCurrency()
+price_currency_value = web_site_value.WebSite_getShoppingCartDefaultCurrency()
 
 movement = context.newContent(
     temp_object=True,

bt5/erp5_authentication_policy/TestTemplateItem/portal_components/test.erp5.testAuthenticationPolicy.py

@@ -263,7 +263,7 @@ class TestAuthenticationPolicy(ERP5TypeTestCase):
                      [x.getPassword() for x in self._getPasswordEventList(login)])
 
     # other methods (edit)...
-    login.edit(password = '123456789-4')
+    login.edit(password='123456789-4')
     self.tic()
     old_password5 = login.getPassword()
     self.assertSameSet([old_password5, old_password4, old_password3, old_password2, old_password1, old_password], \

bt5/erp5_base/DocumentTemplateItem/portal_components/document.erp5.OOoDocument.py

@@ -27,6 +27,7 @@
 #
 ##############################################################################
 
+import contextlib
 import re, zipfile
 from io import BytesIO
 from warnings import warn
@@ -178,7 +179,8 @@ class OOoDocument(OOoDocumentExtensibleTraversableMixin, BaseConvertableFileMixi
         warn('Your oood version is too old, using old method '
             'getAllowedTargets instead of getAllowedTargetList',
              DeprecationWarning)
-
+      finally:
+        server_proxy.close()
       # tuple order is reversed to be compatible with ERP5 Form
       return [(y, x) for x, y in allowed]
 
@@ -208,8 +210,8 @@ class OOoDocument(OOoDocumentExtensibleTraversableMixin, BaseConvertableFileMixi
       cs.close()
       z.close()
       return 'text/plain', s
-    server_proxy = DocumentConversionServerProxy(self)
     orig_format = self.getBaseContentType()
+    with contextlib.closing(DocumentConversionServerProxy(self)) as server_proxy:
       generate_result = server_proxy.run_generate(self.getId(),
                                        bytes2str(enc(bytes(self.getBaseData()))),
                                        None,
@@ -382,7 +384,7 @@ class OOoDocument(OOoDocumentExtensibleTraversableMixin, BaseConvertableFileMixi
       by invoking the conversion server. Store the result
       on the object. Update metadata information.
     """
-    server_proxy = DocumentConversionServerProxy(self)
+    with contextlib.closing(DocumentConversionServerProxy(self)) as server_proxy:
       response_code, response_dict, response_message = server_proxy.run_convert(
                                       self.getFilename() or self.getId(),
                                       bytes2str(enc(bytes(self.getData()))),
@@ -421,7 +423,7 @@ class OOoDocument(OOoDocumentExtensibleTraversableMixin, BaseConvertableFileMixi
       # XXX please pass a meaningful description of error as argument
       raise NotConvertedError()
 
-    server_proxy = DocumentConversionServerProxy(self)
+    with contextlib.closing(DocumentConversionServerProxy(self)) as server_proxy:
       response_code, response_dict, response_message = \
           server_proxy.run_setmetadata(self.getId(),
                                        bytes2str(enc(bytes(self.getBaseData()))),

bt5/erp5_base/MixinTemplateItem/portal_components/mixin.erp5.EncryptedPasswordMixin.py

@@ -36,10 +36,12 @@ from Acquisition import aq_base
 from Products.ERP5Type import Permissions
 from erp5.component.interface.IEncryptedPassword import IEncryptedPassword
 from Products.ERP5Type.Globals import PersistentMapping
+from Products.ERP5Type.Utils import bytes2str
 from Products.CMFCore.utils import _checkPermission
 from Products.CMFCore.exceptions import AccessControl_Unauthorized
 from six import string_types as basestring
 
+
 @zope.interface.implementer(IEncryptedPassword,)
 class EncryptedPasswordMixin(object):
   """Encrypted Password Mixin
@@ -103,7 +105,7 @@ class EncryptedPasswordMixin(object):
     # workflows on this method.
     self.password = PersistentMapping()
     if value:
-      self._setEncodedPassword(pw_encrypt(value))
+      self._setEncodedPassword(bytes2str(pw_encrypt(value)))
 
   def _setPassword(self, value):
     self.checkPasswordValueAcceptable(value)

bt5/erp5_base/SkinTemplateItem/portal_skins/erp5_base/Person_getPersonDetailedContributionList.py

@@ -28,6 +28,8 @@ elif aggregation_level == "week":
   sql_format = "%Y-%u"
 elif aggregation_level == "day":
   sql_format = "%Y-%m-%d"
+else:
+  raise ValueError("Unsupported aggregation level %s" % aggregation_level)
 if to_date is not None:
   to_date = atTheEndOfPeriod(to_date, period=aggregation_level)
 count_kw = {}

bt5/erp5_big_file/DocumentTemplateItem/portal_components/document.erp5.BigFile.py

@@ -30,7 +30,9 @@ import re
 import six
 
 if six.PY3:
-  long = int  # pylint:disable=redefined-builtin
+  long_or_int = int
+else:
+  long_or_int = long  # pylint:disable=undefined-variable
 
 class BigFile(File):
   """
@@ -189,13 +191,13 @@ class BigFile(File):
         else:
           # Date
           date = if_range.split( ';')[0]
-          try: mod_since=long(DateTime(date).timeTime())
+          try: mod_since=long_or_int(DateTime(date).timeTime())
           except Exception: mod_since=None
           if mod_since is not None:
             last_mod = self._data_mtime()
             if last_mod is None:
               last_mod = 0
-            last_mod = long(last_mod)
+            last_mod = long_or_int(last_mod)
             if last_mod > mod_since:
               # Modified, so send a normal response. We delete
               # the ranges, which causes us to skip to the 200

bt5/erp5_configurator/DocumentTemplateItem/portal_components/document.erp5.CategoriesSpreadsheetConfiguratorItem.py

@@ -81,6 +81,8 @@ class CategoriesSpreadsheetConfiguratorItem(ConfiguratorItemMixin, XMLObject):
           "Base Category %s should be created" % bc_id))
         if fixit:
           bc = ctool.newContent(id=bc_id)
+        else:
+          continue
 
       if fixit:
         for category_info in category_list:

bt5/erp5_configurator/ToolComponentTemplateItem/portal_components/tool.erp5.ConfiguratorTool.py

@@ -226,6 +226,7 @@ class ConfiguratorTool(BaseTool):
         return response
 
     ## show next form in transitions
+    html = None
     rendered = False
     while rendered is False:
       if need_validation == 1:

bt5/erp5_configurator_standard_invoicing_template/PathTemplateItem/portal_rules/new_trade_model_simulation_rule/quantity_tester.xml

@@ -55,7 +55,7 @@
         </item>
         <item>
             <key> <string>decimal_rounding_option</string> </key>
-            <value> <string>ROUND_DOWN</string> </value>
+            <value> <string>ROUND_HALF_UP</string> </value>
         </item>
         <item>
             <key> <string>id</string> </key>

bt5/erp5_core_test/TestTemplateItem/portal_components/test.erp5.testDateUtils.py

@@ -208,6 +208,7 @@ class TestPinDateTime(ERP5TypeTestCase):
 
     self.pinDateTime(datetime)
     self.assertEqual(DateTime(), datetime)
+    self.assertEqual(DateTime(None), datetime)
     self.assertEqual(DateTime('2002/02/02 02:02:02 GMT+2'), fixed_date_with_timezone)
 
     self.unpinDateTime()
@@ -221,6 +222,24 @@ class TestPinDateTime(ERP5TypeTestCase):
       self.assertEqual(DateTime(), datetime)
     self.assertGreaterEqual(DateTime(), actual_begin_date)
 
+  def test_pinDateTime_date_time_methods(self):
+    with self.pinDateTime(DateTime('2001/01/01 01:01:01')):
+      self.assertTrue(DateTime('2000').isPast())
+      self.assertTrue(DateTime('2002').isFuture())
+      self.assertTrue(DateTime('2001').isCurrentYear())
+      self.assertTrue(DateTime('2001/01/02 01:01:01').isCurrentMonth())
+      self.assertTrue(DateTime('2001/01/01 02:01:01').isCurrentDay())
+      self.assertTrue(DateTime().strftime('%Y'), '2001')
+      self.assertTrue(DateTime('2002').strftime('%Y'), '2001')
+
+  def test_pinDateTime_timezone(self):
+    with self.pinDateTime(DateTime('2001/01/01 01:01:01 GMT+9')):
+      self.assertEqual(DateTime().timezone(), 'GMT+9')
+      self.assertEqual(DateTime('2001/01/01 01:01:01 GMT+4').timezone(), 'GMT+4')
+    with self.pinDateTime(DateTime('2001/01/01 01:01:01 Europe/Paris')):
+      self.assertEqual(DateTime().timezone(), 'Europe/Paris')
+      self.assertEqual(DateTime('2001/01/01 01:01:01 GMT+4').timezone(), 'GMT+4')
+
 
 class TestTimeZoneContext(ERP5TypeTestCase):
   def afterSetUp(self):

bt5/erp5_core_test/TestTemplateItem/portal_components/test.erp5.testERP5TypeInterfaces.py

@@ -58,18 +58,16 @@ class TestERP5TypeInterfaces(unittest.TestCase):
 def makeTestMethod(import_tuple, interface):
   """Common method which checks if documents implements interface"""
   def testMethod(self):
-    Klass = getattr(
-      __import__(import_tuple[0], globals(), locals(), [import_tuple[0]]),
-      import_tuple[1])
+    from importlib import import_module
+
+    Klass = getattr(import_module(import_tuple[0]), import_tuple[1])
 
     import Products.ERP5Type.interfaces
     try:
       Interface = getattr(Products.ERP5Type.interfaces, interface)
     except AttributeError:
       InterfaceModuleName = 'erp5.component.interface.%s' % interface
-      Interface = getattr(
-        __import__(InterfaceModuleName, globals(), locals(), [InterfaceModuleName]),
-        interface)
+      Interface = getattr(import_module(InterfaceModuleName), interface)
 
     verifyClass(Interface, Klass)
 

bt5/erp5_corporate_identity/SkinTemplateItem/portal_skins/erp5_corporate_identity/WebPage_validateLink.py

@@ -11,6 +11,7 @@ Upgrade link for the specific type of display
 import re
 
 link_href = re.findall("href=['\"](.*?)['\"]", link_string)[0]
+link_title = ''
 
 # XXX flag if broken link
 if link_href.find("http") == -1:

bt5/erp5_corporate_identity/SkinTemplateItem/portal_skins/erp5_corporate_identity/WebPage_viewAsSlideshow.py

@@ -29,6 +29,7 @@ from Products.PythonScripts.standard import html_quote
 from base64 import b64encode
 
 blank = ''
+slide_content = blank
 flags = re.MULTILINE|re.DOTALL|re.IGNORECASE
 details_separator = '</section><section class="ci-notes-continue"><section><h1>cont.</h1></section>'
 pref = context.getPortalObject().portal_preferences

bt5/erp5_crm/SkinTemplateItem/portal_skins/erp5_crm/Base_getEntityListFromFromHeader.py

@@ -2,12 +2,12 @@ getResultValue = context.portal_catalog.getResultValue
 
 from Products.ERP5Type.Utils import Email_parseAddressHeader
 
-result = []
-for _, recipient in Email_parseAddressHeader(text):
+result = set()
+for _, recipient in set(Email_parseAddressHeader(text)):
   if recipient:
     email = getResultValue(url_string={'query':recipient, 'key':'ExactMatch'}, portal_type='Email', parent_portal_type='Person')
     if email is None:
       email = getResultValue(url_string={'query':recipient, 'key':'ExactMatch'}, portal_type='Email', parent_portal_type='Organisation')
     if email is not None:
-      result.append(email.getParentValue())
-return result
+      result.add(email.getParentValue())
+return list(result)

bt5/erp5_crm/TestTemplateItem/portal_components/test.erp5.testCRM.py

@@ -735,6 +735,8 @@ class TestCRMMailIngestion(BaseTestCRM):
       ('me@erp5.org', ['person_module/me']),
       ('me@erp5.org, he@erp5.org', ['person_module/me', 'person_module/he']),
       ('Sender <sender@customer.com>', ['person_module/sender']),
+      # title is also an email, it should return the person once, not twice
+      ('sender@customer.com <sender@customer.com>', ['person_module/sender']),
       # tricks to confuse the e-mail parser:
       # a comma in the name
       ('"Sender," <sender@customer.com>, he@erp5.org', ['person_module/sender',
@@ -750,9 +752,10 @@ class TestCRMMailIngestion(BaseTestCRM):
     for header, expected_paths in expected_values:
       paths = [entity.getRelativeUrl()
                for entity in portal.Base_getEntityListFromFromHeader(header)]
-      self.assertEqual(paths, expected_paths,
-                        '%r should return %r, but returned %r' %
-                        (header, expected_paths, paths))
+      self.assertEqual(
+        sorted(paths), sorted(expected_paths),
+        '%r should return %r, but returned %r' % (header, expected_paths, paths)
+      )
 
   def test_document_creation(self):
     # CRM email ingestion creates a Mail Message in event_module

bt5/erp5_dms_ui_test/SkinTemplateItem/portal_skins/erp5_dms_ui_test/PDF_getContentPassword.py

-# type: () -> bytes
+# type: () -> str
 if context.getId() == 'test_ERP5_Logo_Encrypted_PDF':
   return 'secret'
 return context.skinSuper('erp5_dms_ui_test', 'PDF_getContentPassword')(REQUEST=REQUEST)

bt5/erp5_hal_json_style/ExtensionTemplateItem/portal_components/extension.erp5.HalStyle.py

@@ -22,6 +22,7 @@ def Listbox_getBrainValue(self, brain, obj, select, can_check_local_property, ed
   ListBox.py / getValueList
   """
   tales = False
+  default_field_value = None
 
   # Use a widget, if any.
   if editable_field is not None:

bt5/erp5_hal_json_style/SkinTemplateItem/portal_skins/erp5_hal_json_style/Base_doAction.py

@@ -4,8 +4,7 @@ Base_translateString = portal.Base_translateString
 preserved_parameter_dict = {}
 
 Base_doAction = select_action.split()
-if len(Base_doAction) != 0:
-  doAction0 = Base_doAction[0]
+doAction0 = Base_doAction[0]
 
 kw['keep_items'] = preserved_parameter_dict
 

bt5/erp5_hal_json_style/SkinTemplateItem/portal_skins/erp5_hal_json_style/ERP5Document_getHateoas.py

@@ -202,7 +202,7 @@ def generateDomainTreeList(url_tool, domain_tool, domain, depth, domain_list):
 def getDomainSelection(domain_list):
   root_dict = {}
 
-  if len(domain_list) > 0:
+  if domain_list:
     category_tool = portal.portal_categories
     domain_tool = portal.portal_domains
     preference_tool = portal.portal_preferences
@@ -2169,7 +2169,7 @@ def calculateHateoas(is_portal=None, is_site_root=None, traversed_document=None,
               # XXX If only available on brains, maybe better to call on aq_self
               getBrainListItemUrlDict = getattr(brain, 'getListItemUrlDict', None)
               is_getListItemUrlDict_calculated = True
-            if getBrainListItemUrlDict is not None:
+            if getBrainListItemUrlDict is not None:  # pylint:disable=possibly-used-before-assignment
               # Check if we can get URL result from the brain
               try:
                 url_parameter_dict = getBrainListItemUrlDict(

bt5/erp5_item/SkinTemplateItem/portal_skins/erp5_item/ItemModule_createDeliveryLine.py

@@ -44,7 +44,6 @@ delivery_count = len(source_section_list)
 for item in object_list:
   source_section = item.Item_getCurrentOwnerValue()
   if source_section is not None:
-    if source_section.getUid() is not None:
     pl_value = pl_dict[str(source_section.getUid())]
   else:
     pl_value = pl_dict['UID']

bt5/erp5_oauth/SkinTemplateItem/portal_skins/erp5_oauth/OAuthTool_viewOAuthTokenList/listbox.xml

@@ -72,6 +72,10 @@
                           <string>reference</string>
                           <string>Reference</string>
                         </tuple>
+                        <tuple>
+                          <string>translated_portal_type</string>
+                          <string>Type</string>
+                        </tuple>
                         <tuple>
                           <string>translated_validation_state_title</string>
                           <string>State</string>

bt5/erp5_oauth_facebook_login/SkinTemplateItem/portal_skins/erp5_oauth_facebook_login/FacebookConnector_view.xml

@@ -81,6 +81,7 @@
                     <key> <string>right</string> </key>
                     <value>
                       <list>
+                        <string>my_translated_portal_type</string>
                         <string>my_reference</string>
                         <string>my_translated_validation_state_title</string>
                       </list>

bt5/erp5_oauth_facebook_login/SkinTemplateItem/portal_skins/erp5_oauth_facebook_login/FacebookConnector_view/my_translated_portal_type.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="ProxyField" module="Products.ERP5Form.ProxyField"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>delegated_list</string> </key>
+            <value>
+              <list>
+                <string>title</string>
+              </list>
+            </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>my_translated_portal_type</string> </value>
+        </item>
+        <item>
+            <key> <string>message_values</string> </key>
+            <value>
+              <dictionary>
+                <item>
+                    <key> <string>external_validator_failed</string> </key>
+                    <value> <string>The input failed the external validator.</string> </value>
+                </item>
+              </dictionary>
+            </value>
+        </item>
+        <item>
+            <key> <string>overrides</string> </key>
+            <value>
+              <dictionary>
+                <item>
+                    <key> <string>field_id</string> </key>
+                    <value> <string></string> </value>
+                </item>
+                <item>
+                    <key> <string>form_id</string> </key>
+                    <value> <string></string> </value>
+                </item>
+              </dictionary>
+            </value>
+        </item>
+        <item>
+            <key> <string>tales</string> </key>
+            <value>
+              <dictionary>
+                <item>
+                    <key> <string>field_id</string> </key>
+                    <value> <string></string> </value>
+                </item>
+                <item>
+                    <key> <string>form_id</string> </key>
+                    <value> <string></string> </value>
+                </item>
+              </dictionary>
+            </value>
+        </item>
+        <item>
+            <key> <string>values</string> </key>
+            <value>
+              <dictionary>
+                <item>
+                    <key> <string>field_id</string> </key>
+                    <value> <string>my_translated_workflow_state_title</string> </value>
+                </item>
+                <item>
+                    <key> <string>form_id</string> </key>
+                    <value> <string>Base_viewFieldLibrary</string> </value>
+                </item>
+                <item>
+                    <key> <string>title</string> </key>
+                    <value> <string>Type</string> </value>
+                </item>
+              </dictionary>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+</ZopeData>

bt5/erp5_oauth_google_login/SkinTemplateItem/portal_skins/erp5_oauth_google_login/GoogleConnector_view.xml

@@ -81,6 +81,7 @@
                     <key> <string>right</string> </key>
                     <value>
                       <list>
+                        <string>my_translated_portal_type</string>
                         <string>my_reference</string>
                         <string>my_translated_validation_state_title</string>
                       </list>

bt5/erp5_oauth_google_login/SkinTemplateItem/portal_skins/erp5_oauth_google_login/GoogleConnector_view/my_translated_portal_type.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="ProxyField" module="Products.ERP5Form.ProxyField"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>delegated_list</string> </key>
+            <value>
+              <list>
+                <string>title</string>
+              </list>
+            </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>my_translated_portal_type</string> </value>
+        </item>
+        <item>
+            <key> <string>message_values</string> </key>
+            <value>
+              <dictionary>
+                <item>
+                    <key> <string>external_validator_failed</string> </key>
+                    <value> <string>The input failed the external validator.</string> </value>
+                </item>
+              </dictionary>
+            </value>
+        </item>
+        <item>
+            <key> <string>overrides</string> </key>
+            <value>
+              <dictionary>
+                <item>
+                    <key> <string>field_id</string> </key>
+                    <value> <string></string> </value>
+                </item>
+                <item>
+                    <key> <string>form_id</string> </key>
+                    <value> <string></string> </value>
+                </item>
+              </dictionary>
+            </value>
+        </item>
+        <item>
+            <key> <string>tales</string> </key>
+            <value>
+              <dictionary>
+                <item>
+                    <key> <string>field_id</string> </key>
+                    <value> <string></string> </value>
+                </item>
+                <item>
+                    <key> <string>form_id</string> </key>
+                    <value> <string></string> </value>
+                </item>
+              </dictionary>
+            </value>
+        </item>
+        <item>
+            <key> <string>values</string> </key>
+            <value>
+              <dictionary>
+                <item>
+                    <key> <string>field_id</string> </key>
+                    <value> <string>my_translated_workflow_state_title</string> </value>
+                </item>
+                <item>
+                    <key> <string>form_id</string> </key>
+                    <value> <string>Base_viewFieldLibrary</string> </value>
+                </item>
+                <item>
+                    <key> <string>title</string> </key>
+                    <value> <string>Type</string> </value>
+                </item>
+              </dictionary>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+</ZopeData>

bt5/erp5_officejs_support_request_ui/PathTemplateItem/web_page_module/gadget_supportrequest_panel_js.js

@@ -184,7 +184,14 @@
             return RSVP.hash({
               url_list: gadget.getUrlForList([
                 {command: 'display'},
-                {command: 'display', options: {jio_key: "support_request_module"}},
+                {
+                  command: 'display_stored_state',
+                  options: {
+                    jio_key: "support_request_module",
+                    page: "form",
+                    view: "view"
+                  }
+                },
                 {command: 'display', options: {page: "supportrequest_preference"}},
                 {command: 'display', options: {page: "logout"}}
               ]),

bt5/erp5_officejs_support_request_ui/PathTemplateItem/web_page_module/gadget_supportrequest_panel_js.xml

@@ -228,7 +228,7 @@
                   </item>
                   <item>
                       <key> <string>actor</string> </key>
-                      <value> <string>supertitouan</string> </value>
+                      <value> <unicode>zope</unicode> </value>
                   </item>
                   <item>
                       <key> <string>comment</string> </key>
@@ -242,7 +242,7 @@
                   </item>
                   <item>
                       <key> <string>serial</string> </key>
-                      <value> <string>1017.970.56971.14218</string> </value>
+                      <value> <string>1017.64209.22264.37956</string> </value>
                   </item>
                   <item>
                       <key> <string>state</string> </key>
@@ -262,7 +262,7 @@
                           </tuple>
                           <state>
                             <tuple>
-                              <float>1717413839.14</float>
+                              <float>1721121747.19</float>
                               <string>UTC</string>
                             </tuple>
                           </state>

bt5/erp5_payroll/SkinTemplateItem/portal_skins/erp5_payroll/PaySheetTransaction_viewFieldLibrary/my_price.xml

@@ -10,9 +10,9 @@
             <key> <string>delegated_list</string> </key>
             <value>
               <list>
-                <string>title</string>
                 <string>description</string>
                 <string>precision</string>
+                <string>title</string>
               </list>
             </value>
         </item>
@@ -91,7 +91,7 @@
                 </item>
                 <item>
                     <key> <string>precision</string> </key>
-                    <value> <string></string> </value>
+                    <value> <int>2</int> </value>
                 </item>
                 <item>
                     <key> <string>target</string> </key>

bt5/erp5_payroll_l10n_fr/SkinTemplateItem/portal_skins/erp5_payroll_l10n_fr/PaySheetTransaction_getPayslipData.py

@@ -70,7 +70,7 @@ def groupSameReportSectionLine(line_to_group_list):
       tmp2_base_dict[new_key]['employee_total_price'] = tmp2_base_dict[new_key]['employee_total_price'] + value['employee_total_price']
   new_value_list = []
   # recalculate for rounding issue
-  for value_dict in tmp2_base_dict.values():
+  for _, value_dict in sorted(tmp2_base_dict.items()):
     value_dict['employer_total_price'] = value_dict['base'] * value_dict['employer_price']
     value_dict['employee_total_price'] = value_dict['base'] * value_dict['employee_price']
     new_value_list.append(value_dict)

bt5/erp5_project/SkinTemplateItem/portal_skins/erp5_project/Requirement_generateRequirements.py

@@ -52,12 +52,10 @@ for requirement_item in requirements_items:
     has_1st_level_requirement = True
     new_1st_level_requirement = []
     new_1st_level_requirement_title = requirement_item['title']
+    description_dict[new_1st_level_requirement_title] = ''
   else:
     has_1st_level_requirement = False
 
-  if has_1st_level_requirement:
-    description_dict[new_1st_level_requirement_title] = ''
-
   # the item has a second level requirement, built it
   if requirement_item['sub_title'] not in ('', None):
     has_2nd_level_requirement = True

bt5/erp5_security_uid_innodb_catalog_test/SkinTemplateItem/portal_skins/erp5_security_uid_innodb_catalog_test/Base_zCreateWorklistTable.sql

@@ -2,8 +2,8 @@ DROP TABLE IF EXISTS worklist_cache
 <dtml-var sql_delimiter>
 CREATE TABLE `worklist_cache` (
   `count` INT UNSIGNED NOT NULL,
-  `owner` VARCHAR(32) DEFAULT '',
-  `viewable_owner` VARCHAR(32) NOT NULL DEFAULT '',
+  `owner` VARCHAR(255) binary DEFAULT '',
+  `viewable_owner` VARCHAR(255) binary NOT NULL DEFAULT '',
   `security_uid` INT UNSIGNED NOT NULL,
   `alternate_security_uid` INT UNSIGNED,
   `other_security_uid` INT UNSIGNED,

bt5/erp5_syncml/ModuleComponentTemplateItem/portal_components/module.erp5.ERP5Conduit.py

@@ -477,6 +477,7 @@ class ERP5Conduit(XMLSyncUtilsMixin):
     if get_target_parent:
       result_list = result_list[:-1]
     first_object = True
+    sub_context = None
     while result_list:
       object_block = result_list[0][0]
       sub_context_id = result_list[0][3]

bt5/erp5_syncml/ModuleComponentTemplateItem/portal_components/module.erp5.XMLSyncUtils.py

@@ -128,18 +128,19 @@ def getConduitByName(conduit_name):
   Conduit can also be defined as Extension to have it editable through the web, in this
   case its definition must be Extensions.<Conduit Module>
   """
+  from importlib import import_module
   if conduit_name.startswith('Products'):
     path = conduit_name
     conduit_name = conduit_name.split('.')[-1]
-    conduit_module = __import__(path, globals(), locals(), [''])
+    conduit_module = import_module(path)
   elif conduit_name.startswith('Extensions'):
-    conduit_module = __import__(conduit_name, globals(), locals(), [''])
+    conduit_module = import_module(conduit_name)
     conduit_name = conduit_name.split('.')[-1]
   elif conduit_name.startswith('extension.'):
-    conduit_module = __import__("erp5.component."+conduit_name, globals(), locals(), [''])
+    conduit_module = import_module("erp5.component." + conduit_name)
     conduit_name = conduit_name.split('.')[-1]
   else:
-    conduit_module = __import__('erp5.component.module.'+conduit_name, globals(), locals(), [''])
+    conduit_module = import_module('erp5.component.module.' + conduit_name)
   conduit_instance = getattr(conduit_module, conduit_name)()
   return conduit_instance
 

bt5/erp5_trade/SkinTemplateItem/portal_skins/erp5_trade/Delivery_getFastInputLineList.py

@@ -21,7 +21,7 @@ if line_portal_type in portal.getPortalSaleTypeList():
   section_uid = context.getSourceSectionUid()
 elif line_portal_type in portal.getPortalPurchaseTypeList():
   section_uid = context.getDestinationSectionUid()
-elif line_portal_type in portal.getPortalInternalTypeList() + portal.getPortalInventoryMovementTypeList():
+else:
   section_uid = None
 len_line_list = len(line_list)
 used_id = [] # list use to make sure we do not generate two line with same id/uid

bt5/erp5_trade/SkinTemplateItem/portal_skins/erp5_trade/Delivery_setFastInputLineList.py

@@ -3,6 +3,8 @@
   input information.It should take into account any trade document line
   which were already created so that they are not duplicated.
 """
+# pylint:disable=possibly-used-before-assignment
+
 from Products.ERP5Type.Message import translateString
 portal = context.getPortalObject()
 

bt5/erp5_trade/SkinTemplateItem/portal_skins/erp5_trade/Delivery_updateFastInputLineList.py

@@ -43,6 +43,7 @@ elif line_portal_type in portal.getPortalInternalTypeList():
 elif line_portal_type in portal.getPortalInventoryMovementTypeList():
   section_uid = None
   no_inventory = True
+  supply_cell_portal_type = supply_line_id = None
   use_list = portal.portal_preferences.getPreferredPurchaseUseList() \
              + portal.portal_preferences.getPreferredSaleUseList()
 else:

bt5/erp5_trade/SkinTemplateItem/portal_skins/erp5_trade/OrderModule_activateGetOrderStatList.py

@@ -51,6 +51,8 @@ elif aggregation_level == "week":
   date_format = "%Y-%U"
 elif aggregation_level == "day":
   date_format = "%Y-%m-%d"
+else:
+  raise ValueError("Unsupported aggregation_level: %s" % aggregation_level)
 
 if from_date is not None and at_date is not None:
   catalog_params['delivery.start_date'] = {

bt5/erp5_web/SkinTemplateItem/portal_skins/erp5_web/Base_convertHtmlToSingleFile.py

@@ -247,6 +247,7 @@ if force_base_url:
   root_url = "/".join(base_url.split("/", 3)[:3])
   if root_url != base_url:
     base_url = "/".join(base_url.split("/")[:-1])
+  request_protocol = 'https:'
 else:
   request_protocol = context.REQUEST.SERVER_URL.split(":", 1)[0] + ":"
   root_url = base_url_root_object.absolute_url()

bt5/erp5_web_service/ToolComponentTemplateItem/portal_components/tool.erp5.WebServiceTool.py

@@ -55,15 +55,14 @@ handler_module_dict = {
   'sql' : "SQLConnection",
   'document' : "DocumentConnection",
 }
+from importlib import import_module
 for handler_id, module_id in handler_module_dict.iteritems():
   # Ignore non-functionnal plugins.
   # This is done to avoid adding strict dependencies.
   # Code relying on the presence of a plugin will fail upon
   # WebServiceTool.connect .
   try:
-    module = __import__(
-      'erp5.component.module.%s' % (module_id, ),
-      globals(), {}, [module_id])
+    module = import_module('erp5.component.module.' + module_id)
   except ImportError:
     LOG('WebServiceTool', WARNING,
         'Unable to import module %r. %r transport will not be available.' % \

bt5/erp5_worklist_sql/SkinTemplateItem/portal_skins/erp5_worklist_sql/Base_zCreateWorklistTable.sql

@@ -2,8 +2,8 @@ DROP TABLE IF EXISTS worklist_cache
 <dtml-var sql_delimiter>
 CREATE TABLE `worklist_cache` (
   `count` INT UNSIGNED NOT NULL,
-  `owner` VARCHAR(32) DEFAULT '',
-  `viewable_owner` VARCHAR(32) NOT NULL DEFAULT '',
+  `owner` VARCHAR(255) binary DEFAULT '',
+  `viewable_owner` VARCHAR(255) binary NOT NULL DEFAULT '',
   `security_uid` INT UNSIGNED NOT NULL,
   `portal_type` VARCHAR(255) NOT NULL,
   `validation_state` VARCHAR(255) NULL,

product/ERP5/Document/BusinessTemplate.py

@@ -90,6 +90,7 @@ from xml.sax.saxutils import escape
 from Products.CMFCore.Expression import Expression
 from six.moves.urllib.parse import quote, unquote, urlparse
 from difflib import unified_diff
+from importlib import import_module
 import posixpath
 import transaction
 import inspect
@@ -6858,9 +6859,7 @@ Business Template is a set of definitions, such as skins, portal types and categ
             if component_portal_type in ('Document Component',
                                          'Tool Component'):
               try:
-                klass = getattr(
-                  __import__(source_reference, {}, {}, [source_reference]),
-                  subsubmodule_name)
+                klass = getattr(import_module(source_reference), subsubmodule_name)
               except ImportError as e:
                 LOG("BusinessTemplate", WARNING,
                     "Skipping %s: Cannot be imported (%s)" % (filepath, e),
@@ -6890,7 +6889,7 @@ Business Template is a set of definitions, such as skins, portal types and categ
               # Generally: foo_bar.py => IFooBar, but to avoid quirks (such as
               # 'sql_foo.py' => 'ISQLFoo'), get the Interface class __name__
               try:
-                interface_module = __import__(source_reference, {}, {}, source_reference)
+                interface_module = import_module(source_reference)
               except ImportError as e:
                 LOG("BusinessTemplate", WARNING,
                     "Skipping %s: Cannot be imported (%s)" % (filepath, e),
@@ -6919,7 +6918,7 @@ Business Template is a set of definitions, such as skins, portal types and categ
             # TODO-arnau: Refactor with 'Interface Component'
             elif component_portal_type == 'Mixin Component':
               try:
-                mixin_module = __import__(source_reference, {}, {}, source_reference)
+                mixin_module = import_module(source_reference)
               except ImportError as e:
                 LOG("BusinessTemplate", WARNING,
                     "Skipping %s: Cannot be imported (%s)" % (filepath, e),

product/ERP5/Tool/TrashTool.py

@@ -132,6 +132,8 @@ class TrashTool(BaseTool):
           LOG("Trash Tool backupObject", WARNING,
               "Can't backup object %s" % object_path)
           return {}
+        finally:
+          copy.close()
 
     subobjects_dict = {}
 

product/ERP5/bootstrap/erp5_core/DocumentTemplateItem/portal_components/document.erp5.Document.py

@@ -27,8 +27,9 @@
 #
 ##############################################################################
 
+import logging
 import re
-from zLOG import LOG, WARNING
+from zLOG import LOG
 from AccessControl import ClassSecurityInfo
 from Acquisition import aq_base
 from Products.ERP5Type.Accessor.Constant import PropertyGetter as ConstantGetter
@@ -102,6 +103,7 @@ class DocumentConversionServerProxy():
   """
   def __init__(self, context):
     self._serverproxy_list = []
+    self._logger = logging.getLogger(__name__)
     preference_tool = getToolByName(context, 'portal_preferences')
     self._ooo_server_retry = (
       preference_tool.getPreferredDocumentConversionServerRetry() or
@@ -113,9 +115,9 @@ class DocumentConversionServerProxy():
       if not (address and port):
         raise ConversionError('OOoDocument: cannot proceed with conversion:'
               ' conversion server url is not defined in preferences')
-
-      LOG('Document', WARNING, 'PreferredOoodocServer{Address,PortNumber}' + \
-          ' are DEPRECATED please use PreferredDocumentServerUrl instead', error=True)
+      self._logger.warning(
+        'PreferredOoodocServer{Address,PortNumber} are DEPRECATED '
+        'please use PreferredDocumentServerUrl instead')
 
       uri_list =  ['%s://%s:%s' % ('http', address, port)]
 
@@ -209,6 +211,18 @@ class DocumentConversionServerProxy():
   def __getattr__(self, attr):
     return partial(self._proxy_function, attr)
 
+  def close(self):
+    error_list = []
+    for server_addr, proxy in self._serverproxy_list:
+      try:
+        proxy.__call__('close')()
+      except Exception as e:
+        self._logger.exception('Error closing %s', server_addr)
+        error_list.append(e)
+    for e in error_list:
+      raise e
+
+
 from erp5.component.mixin.DocumentExtensibleTraversableMixin import DocumentExtensibleTraversableMixin
 from erp5.component.interface.IConvertable import IConvertable
 from erp5.component.interface.ITextConvertable import ITextConvertable

product/ERP5/bootstrap/erp5_core/DocumentTemplateItem/portal_components/document.erp5.ImmobilisableItem.py

@@ -989,7 +989,8 @@ class ImmobilisableItem(Item, Amount):
       raw_annuity_price = annuity_start_price * current_ratio
     elif price_calculation_basis == "period recalculated start price":
       raw_annuity_price = local_period_start_price * current_ratio
-
+    else:
+      raise ValueError("Unsupported price_calculation_basis: %s" % price_calculation_basis)
     # Apply the prorata temporis on the raw annuity value
     if annuity_number and \
         price_calculation_basis == 'period recalculated start price' and \

product/ERP5/bootstrap/erp5_core/DocumentTemplateItem/portal_components/document.erp5.Movement.py

@@ -422,7 +422,7 @@ class Movement(XMLObject, Amount, CompositionMixin, AmountGeneratorMixin):
     quantity = self.getQuantity()
     if quantity :
       source_asset_price = self.getSourceAssetPrice()
-      if source_asset_price :
+      if source_asset_price is not None:
         return source_asset_price * - quantity
     return None
 
@@ -466,7 +466,7 @@ class Movement(XMLObject, Amount, CompositionMixin, AmountGeneratorMixin):
     quantity = self.getQuantity()
     if quantity :
       destination_asset_price = self.getDestinationAssetPrice()
-      if destination_asset_price :
+      if destination_asset_price is not None:
         return destination_asset_price * quantity
     return None
 
@@ -520,7 +520,7 @@ class Movement(XMLObject, Amount, CompositionMixin, AmountGeneratorMixin):
 
   def _getAssetPrice(self,section,date):
     price = self.getPrice()
-    if section is None or not price or getattr(
+    if section is None or price is None or getattr(
       section.aq_base, 'getPriceCurrencyValue', None
     ) is None:
       return price

product/ERP5/bootstrap/erp5_core/MixinTemplateItem/portal_components/mixin.erp5.AmountGeneratorMixin.py

@@ -271,6 +271,7 @@ class AmountGeneratorMixin:
     - is rounding really well supported (ie. before and after aggregation)
       very likely not - proxying before or after must be decided
     """
+    # pylint:disable=self-cls-assignment,possibly-used-before-assignment
     # It is the only place where we can import this
     portal = self.getPortalObject()
     getRoundingProxy = portal.portal_roundings.getRoundingProxy

product/ERP5/bootstrap/erp5_core/ModuleComponentTemplateItem/portal_components/module.erp5.ExpandPolicy.py

@@ -85,7 +85,7 @@ class _Policy(object):
     if attr == 'merge_parent':
       self.merge_parent = value = self.context.getRootAppliedRule().getPath()
     else:
-      object.__getattribute__(self, attr)
+      value = object.__getattribute__(self, attr)
     return value
 
   def deferAll(self):

product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_auto_logout/setAuthCookie.py

@@ -10,13 +10,15 @@ else:
   now = DateTime()
   kw['expires'] = (now + expire_interval).toZone('GMT').rfc822()
   ac_renew = (now + expire_interval / 2).millis()
-portal.portal_sessions[
+
+cookie_authentication = getattr(portal, 'cookie_authentication', None)
+if cookie_authentication is not None \
+    and cookie_authentication.getProperty('auth_cookie') == cookie_name:
+  portal.portal_sessions[
     portal.Base_getAutoLogoutSessionKey(
-    username=portal.Base_getUsernameFromAuthenticationCookie(
-      cookie_value,
-    )
+      username=portal.Base_getUsernameFromAuthenticationCookie(cookie_value)
     )
-]['ac_renew'] = ac_renew
+  ]['ac_renew'] = ac_renew
 
 REQUEST = portal.REQUEST
 parse_dict = urlparse(REQUEST.other.get('ACTUAL_URL'))

product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_auto_logout/twiddleAuthCookie.py

 portal = context.getPortalObject()
-if DateTime().millis() >= portal.portal_sessions[
+
+cookie_authentication = getattr(portal, 'cookie_authentication', None)
+if cookie_authentication is not None \
+    and cookie_authentication.getProperty('auth_cookie') == cookie_name \
+    and DateTime().millis() >= portal.portal_sessions[
         portal.Base_getAutoLogoutSessionKey(
           username=portal.Base_getUsernameFromAuthenticationCookie(
-          cookie_value,
+            cookie_value
           )
         )
       ].get('ac_renew', 0):

product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/Base_getCategoriesSpreadSheetMapping.py

@@ -25,6 +25,7 @@ The returned mapping has the following structure:
 This scripts guarantees that the list of category info is sorted in such a
 way that parent always precedes their children.
 """
+import six
 from Products.ERP5Type.Message import translateString
 from Products.ERP5OOo.OOoUtils import OOoParser
 parser = OOoParser()
@@ -41,41 +42,7 @@ if invalid_spreadsheet_error_handler is None:
 property_id_set = portal.portal_types.Category.getInstancePropertySet()
 property_id_set.update(getattr(portal.portal_types, 'Base Category').getInstancePropertySet())
 
-def getIDFromString(string=None):
-  """
-    This function transform a string to a safe and beautiful ID.
-    It is used here to create a safe category ID from a string.
-    But the code is not really clever...
-  """
-  if string is None:
-    return None
-  clean_id = ''
-  translation_map = { 'a'  : [u'\xe0', u'\xe3']
-                    , 'e'  : [u'\xe9', u'\xe8']
-                    , 'i'  : [u'\xed']
-                    , 'u'  : [u'\xf9']
-                    , '_'  : [' ', '+']
-                    , '-'  : ['-', u'\u2013']
-                    , 'and': ['&']
-                    }
-  # Replace odd chars by safe ascii
-  string = string.lower()
-  string = string.strip()
-  for (safe_char, char_list) in translation_map.items():
-    for char in char_list:
-      string = string.replace(char, safe_char)
-  # Exclude all non alphanumeric chars
-  for char in string:
-    if char.isalnum() or char in translation_map.keys():
-      clean_id += char
-  # Delete leading and trailing char which are not alpha-numerics
-  # This prevent having IDs with starting underscores
-  while len(clean_id) > 0 and not clean_id[0].isalnum():
-    clean_id = clean_id[1:]
-  while len(clean_id) > 0 and not clean_id[-1].isalnum():
-    clean_id = clean_id[:-1]
-
-  return clean_id
+getIDFromString = portal.Base_getSafeIdFromString
 
 # if the file is not an open office format, try to convert it using oood
 # FIXME: use portal_transforms
@@ -129,7 +96,7 @@ for table_name in spreadsheet_list.keys():
     else:
       # If there is a new column with a header and the path definition has
       # started, that seems the path definition has ended
-      property_map[column_index] = column_id.encode('utf8')
+      property_map[column_index] = column_id.encode('utf8') if six.PY2 else column_id
     column_index += 1
 
   # Construct categories data (with absolute path) from table lines
@@ -137,9 +104,9 @@ for table_name in spreadsheet_list.keys():
   # 1 table = 1 base category
   base_category_name = table_name
   base_category_id = getIDFromString(base_category_name)
-  if same_type(base_category_name, u''):
+  if six.PY2 and isinstance(base_category_name, unicode):
     base_category_name = base_category_name.encode('utf8')
-  if same_type(base_category_id, u''):
+  if six.PY2 and isinstance(base_category_id, unicode):
     base_category_id = base_category_id.encode('utf8')
   category_list = category_list_spreadsheet_mapping.setdefault(base_category_id, [])
   category_list.append({ 'path' : base_category_id
@@ -198,7 +165,7 @@ for table_name in spreadsheet_list.keys():
       if cell_id not in ('', None):
         # Handle normal properties
         if not property_id.startswith('path_'):
-          if same_type(cell_data, u''):
+          if six.PY2 and same_type(cell_data, u''):
             cell_data = cell_data.encode('utf8')
           category_property_list[property_id] = cell_data
         # Handle 'path' property
@@ -218,7 +185,7 @@ for table_name in spreadsheet_list.keys():
                 # Get the next depth
                 break
           path = '/'.join([base_category_id,] + absolute_path_element_list[::-1])
-          if same_type(path, u''):
+          if six.PY2 and same_type(path, u''):
             path = path.encode('utf8')
           category_property_list['path'] = path
 

product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/Base_getSafeIdFromString.py

@@ -2,27 +2,32 @@
   This function transform a string to a safe id.
   It is used here to create a safe category id from a string.
 """
-
-translation_map = { "a": ['\xe0']
-                  , "e": ['\xe9', '\xe8']
-                  }
-
-clean_id = ''
 if s is None:
   return None
+clean_id = ''
+translation_map = { 'a'  : [u'\xe0', u'\xe3']
+                  , 'e'  : [u'\xe9', u'\xe8']
+                  , 'i'  : [u'\xed']
+                  , 'u'  : [u'\xf9']
+                  , '_'  : [' ', '+']
+                  , '-'  : ['-', u'\u2013']
+                  , 'and': ['&']
+                  }
+# Replace odd chars by safe ascii
 s = s.lower()
 s = s.strip()
-# oocalc inserts some strange chars when you press - key in a text cell.
-# Following line is a workaround for this, because \u2013 does not exist in latin1
-s = s.replace(u'\u2013', '-')
-for char in s.encode('iso8859_1'):
-  if char == '_' or char.isalnum():
+for (safe_char, char_list) in translation_map.items():
+  for char in char_list:
+    s = s.replace(char, safe_char)
+# Exclude all non alphanumeric chars
+for char in s:
+  if char.isalnum() or char in translation_map.keys():
     clean_id += char
-  elif char.isspace() or char in ('+', '-'):
-    clean_id += '_'
-  else:
-    for (safe_char, char_list) in translation_map.items():
-      if char in char_list:
-        clean_id += safe_char
-        break
+# Delete leading and trailing char which are not alpha-numerics
+# This prevent having IDs with starting underscores
+while len(clean_id) > 0 and not clean_id[0].isalnum():
+  clean_id = clean_id[1:]
+while len(clean_id) > 0 and not clean_id[-1].isalnum():
+  clean_id = clean_id[:-1]
+
 return clean_id

product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/Base_validateRelation.py

@@ -8,6 +8,7 @@ request=context.REQUEST
 # We stop doing this
 #base_category = context.getBaseCategoryId()
 base_category = None
+redirect_url = None
 
 o = context.restrictedTraverse(object_path)
 

product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/BusinessTemplate_getModifiableFieldList.py

@@ -42,7 +42,7 @@ skin_id_list = context.getTemplateSkinIdList()
 if skin_id_list:
   if bt_title in skin_id_list:
     main_skin_id = bt_title
-  elif skin_id_list:
+  else:
     main_skin_id = skin_id_list[0]
   form_path = '%s/%s' % (main_skin_id, field_library_id)
   form = getForm(portal.portal_skins[main_skin_id], field_library_id)

product/ERP5/bootstrap/erp5_core/ToolComponentTemplateItem/portal_components/tool.erp5.DiffTool.py

@@ -27,6 +27,13 @@
 #
 ##############################################################################
 
+import six
+# pylint:disable=import-error,no-name-in-module
+if six.PY3:
+  from collections.abc import Set
+else:
+  from collections import Set
+# pylint:enable=import-error,no-name-in-module
 import difflib
 import warnings
 try:
@@ -144,7 +151,7 @@ class PortalPatch(Explicit):
 
     # Flatten the list of DiffValues
     for key, subset in tree_diff.items():
-      if isinstance(subset, set):
+      if isinstance(subset, Set):
         sublist = list(subset)
         for item in sublist:
           # XXX: This is important as the subsets with iterable item removed
@@ -192,9 +199,9 @@ class PortalPatch(Explicit):
       else:
         old_value = val.t1
         new_value = val.t2
-        if (val.t1 == None) or isinstance(val.t1, deepdiff.helper.NotPresent):
+        if (val.t1 is None) or isinstance(val.t1, deepdiff.helper.NotPresent):
           old_value = ''
-        if (val.t2 == None) or isinstance(val.t2, deepdiff.helper.NotPresent):
+        if (val.t2 is None) or isinstance(val.t2, deepdiff.helper.NotPresent):
           new_value = ''
 
         # Deepdiff doesn't creates diff for anything other than string, thus for all other cases,

product/ERP5/bootstrap/erp5_core/ToolComponentTemplateItem/portal_components/tool.erp5.SimulationTool.py

@@ -1568,8 +1568,8 @@ class SimulationTool(BaseTool):
       try:
         # We must copy the path so that getObject works
         setattr(result, 'path', line_a.path)
-      except ValueError: # XXX: ValueError ? really ?
-        # getInventory return no object, so no path available
+      except (AttributeError, ValueError):
+        # getInventory returned no object, so no path available
         pass
       if parent is not None:
         result = result.__of__(parent)

product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z_create_catalog.sql

@@ -5,8 +5,8 @@
 CREATE TABLE `catalog` (
   `uid` BIGINT UNSIGNED NOT NULL,
   `security_uid` INT UNSIGNED,
-  `owner` varbinary(255) NOT NULL default '',
-  `viewable_owner` varbinary(255) NOT NULL default '',
+  `owner` varchar(255) binary NOT NULL default '',
+  `viewable_owner` varchar(255) binary NOT NULL default '',
   `path` varchar(255) NOT NULL default '',
   `relative_url` varchar(255) NOT NULL default '',
   `parent_uid` BIGINT UNSIGNED default '0',

product/ERP5/tests/testERP5Callable.py

@@ -26,6 +26,7 @@
 #
 ##############################################################################
 
+import six
 import sys
 import traceback
 
@@ -84,6 +85,11 @@ class TestERP5PythonScript(ERP5TypeTestCase):
     self.script.setBody('return "Hello " + who')
     self.assertEqual(self.script("world"), "Hello world")
 
+    if six.PY2:
+      filename = 'ERP5 Python Script'
+    else:
+      filename = 'ERP5 Python Script:%s' % self.script.getPath()
+
     try:
       self.script(123)
     except TypeError:
@@ -91,8 +97,8 @@ class TestERP5PythonScript(ERP5TypeTestCase):
       # python script code is visible in traceback
       self.assertEqual(
         traceback.format_tb(tb)[-1],
-        '  File "ERP5 Python Script", line 1, in %s\n'
-        '    return "Hello " + who\n' % self.id()
+        '  File "%s", line 1, in %s\n'
+        '    return "Hello " + who\n' % (filename, self.id())
       )
     else:
       self.fail('Exception not raised')
@@ -126,6 +132,11 @@ class TestERP5WorkflowScript(ERP5TypeTestCase):
     self.script.setBody('return "Hello " + state_change')
     self.assertEqual(self.script("world"), "Hello world")
 
+    if six.PY2:
+      filename = 'ERP5 Workflow Script'
+    else:
+      filename = 'ERP5 Workflow Script:%s' % self.script.getPath()
+
     try:
       self.script(123)
     except TypeError:
@@ -133,8 +144,8 @@ class TestERP5WorkflowScript(ERP5TypeTestCase):
       # python script code is visible in traceback
       self.assertEqual(
         traceback.format_tb(tb)[-1],
-        '  File "ERP5 Workflow Script", line 1, in script_test_script\n'
-        '    return "Hello " + state_change\n'
+        ('  File "%s", line 1, in script_test_script\n'
+        '    return "Hello " + state_change\n') % filename
       )
     else:
       self.fail('Exception not raised')

product/ERP5/tests/testInventoryAPI.py

@@ -1708,7 +1708,7 @@ class TestMovementHistoryList(InventoryAPITestCase):
     # default is an empty list
     self.assertEqual(0, len(mvt_history_list))
 
-  def testDefault0(self):
+  def testDefaultNone(self):
     self._makeMovement()
     getMovementHistoryList = self.getSimulationTool().getMovementHistoryList
     mvt_history_list = getMovementHistoryList(
@@ -1718,6 +1718,32 @@ class TestMovementHistoryList(InventoryAPITestCase):
     # If a movement have no price, None is returned
     self.assertEqual(None, mvt_history_list[0].total_price)
 
+  def testPriceZero(self):
+    self._makeMovement(quantity=1, price=0)
+    getMovementHistoryList = self.getSimulationTool().getMovementHistoryList
+    self.assertEqual(
+      [(b.total_quantity, b.total_price) for b in getMovementHistoryList(section_uid=self.section.getUid())],
+      [(1, 0), ]
+    )
+    self.assertEqual(
+      [(b.total_quantity, b.total_price) for b in getMovementHistoryList(section_uid=self.mirror_section.getUid())],
+      [(-1, 0), ]
+    )
+
+  def testPriceNone(self):
+    self._makeMovement(quantity=1, price=None)
+    getMovementHistoryList = self.getSimulationTool().getMovementHistoryList
+    mvt_history_list = getMovementHistoryList(
+      section_uid=self.section.getUid(),)
+    self.assertEqual(
+      [(b.total_quantity, b.total_price) for b in getMovementHistoryList(section_uid=self.section.getUid())],
+      [(1, None), ]
+    )
+    self.assertEqual(
+      [(b.total_quantity, b.total_price) for b in getMovementHistoryList(section_uid=self.mirror_section.getUid())],
+      [(-1, None), ]
+    )
+
   def testMovementBothSides(self):
     """Movement History List returns movement from both sides"""
     getMovementHistoryList = self.getSimulationTool().getMovementHistoryList

product/ERP5Type/Core/Folder.py

@@ -667,6 +667,12 @@ class OFSFolder2(OFSFolder):
     except AttributeError as exc:
       raise KeyError(exc.args)
 
+  def _cleanup(self):
+    # Keep compatibility with BTreeFolder2 API, despite it is not reaquired.
+    # This api is required by the [check,fix]Consistency implementation.
+    LOG("OFSFolder2._cleanup", WARNING, "This folder class do not implement _cleanup, skip.")
+    return True
+
 OFS_HANDLER = 0
 BTREE_HANDLER = 1
 HBTREE_HANDLER = 2

product/ERP5Type/Utils.py

@@ -1076,7 +1076,8 @@ def importLocalDocument(class_id, path=None, class_path=None):
   if class_path:
     assert path is None
     module_path = class_path.rsplit('.', 1)[0]
-    module = __import__(module_path, {}, {}, (module_path,))
+    from importlib import import_module
+    module = import_module(module_path)
     try:
       klass = getattr(module, class_id)
     except AttributeError:

product/ERP5Type/XMLExportImport/__init__.py

@@ -393,10 +393,12 @@ def save_record(parser, tag, data):
 
 import xml.parsers.expat
 def importXML(jar, file, clue=''):
-    if type(file) is str:
-        file=open(file, 'rb')
-    outfile=TemporaryFile()
-    data=file.read()
+    if isinstance(file, str):
+        with open(file, 'rb') as f:
+          data = f.read()
+    else:
+        data = file.read()
+    with TemporaryFile() as outfile:
         F=ppml.xmlPickler()
         F.end_handlers['record'] = save_record
         F.end_handlers['ZopeData'] = save_zopedata
@@ -416,7 +418,7 @@ def importXML(jar, file, clue=''):
         p.EndElementHandler=F.unknown_endtag
         r=p.Parse(data)
         outfile.seek(0)
-    return jar.importFile(outfile,clue)
+        return jar.importFile(outfile, clue)
 
 customImporters = {
   magic: importXML

product/ERP5Type/dynamic/component_package.py

@@ -30,6 +30,8 @@
 # There is absolutely no reason to use relative imports when loading a Component
 from __future__ import absolute_import
 
+import errno
+import os
 import six
 import sys
 import imp
@@ -222,6 +224,15 @@ class ComponentDynamicPackage(ModuleType):
       if import_lock_held:
         imp.acquire_lock()
 
+  def find_spec(self, name, path=None, target=None):
+    """PEP-0451
+    """
+    assert six.PY3
+    if self.find_module(name, path) is None:
+      return None
+    import importlib.util
+    return importlib.util.spec_from_loader(name, self)
+
   def _getVersionPackage(self, version):
     """
     Get the version package (NAMESPACE.VERSION_version) for the given version
@@ -321,11 +332,27 @@ class ComponentDynamicPackage(ModuleType):
 
     component = getattr(site.portal_components, component_id)
     relative_url = component.getRelativeUrl()
+    if six.PY2:
+      module_file = '<' + relative_url + '>'
+    else:
+      module_file = 'erp5://' + relative_url
 
     module_fullname = '%s.%s_version.%s' % (self._namespace, version, name)
     module = ModuleType(module_fullname, component.getDescription())
 
     source_code_str = component.getTextContent(validated_only=True)
+    for override_path in os.environ.get('ERP5_COMPONENT_OVERRIDE_PATH', '').split(os.pathsep):
+      try:
+        local_override_path = os.path.join(override_path, component.getId() + '.py')
+        with open(local_override_path) as f:
+          source_code_str = f.read()
+        module_file = local_override_path
+        LOG("component_package", WARNING, "Using local override %s" % local_override_path)
+        break
+      except IOError as e:
+        if e.errno != errno.ENOENT:
+          raise
+
     version_package = self._getVersionPackage(version)
 
     # All the required objects have been loaded, acquire import lock to modify
@@ -341,7 +368,7 @@ class ComponentDynamicPackage(ModuleType):
         sys.modules[module_fullname_filesystem] = module
 
       # This must be set for imports at least (see PEP 302)
-      module.__file__ = '<' + relative_url + '>'
+      module.__file__ = module_file
       if coverage.Coverage.current():
         if hasattr(component, '_erp5_coverage_filename'):
           module.__file__ = component._erp5_coverage_filename

product/ERP5Type/dynamic/dynamic_module.py

@@ -41,6 +41,17 @@ class PackageType(ModuleType):
   """
   __path__ = []
 
+  def __init__(self, name, doc=None):
+    super(PackageType, self).__init__(name=name, doc=doc)
+    if six.PY3:
+      # PEP-0451
+      import importlib.machinery
+      self.__spec__ = importlib.machinery.ModuleSpec(
+        name=self.__name__,
+        loader=None,
+      )
+
+
 class RefManager(dict):
   """
   self[ComponentTool.last_sync] = (HTTP_REQUEST_WEAKSET,

product/ERP5Type/dynamic/lazy_class.py

@@ -22,18 +22,13 @@ from . import persistent_migration
 from ZODB.POSException import ConflictError
 import six
 
-class ERP5BaseBroken(Broken, ERP5Base, PersistentBroken):
-  # PersistentBroken can't be reused directly
-  # because its « layout differs from 'GhostPortalType' »
 
-  # This prevents serialize (ZODB) from reloading the class during commit
-  # (which would look for __Broken_newargs__ which is not present)
-  __getnewargs__ = None
-
-  def __metaclass__(name, base, d):
+class PersistentBrokenMetaClass(type):
+  def __new__(cls, name, bases, d):
     d = dict(PersistentBroken.__dict__, **d)
-    for x in '__dict__', '__metaclass__', '__weakref__':
-      del d[x]
+    del d['__dict__']
+    del d['__weakref__']
+
     def get(x):
       def get(self):
         d = self.__dict__
@@ -44,7 +39,17 @@ class ERP5BaseBroken(Broken, ERP5Base, PersistentBroken):
       return property(get)
     for x in 'id', 'title':
       d[x] = get(x)
-    return type(name, base, d)
+    return type(name, bases, d)
+
+
+@six.add_metaclass(PersistentBrokenMetaClass)
+class ERP5BaseBroken(Broken, ERP5Base, PersistentBroken):
+  # PersistentBroken can't be reused directly
+  # because its « layout differs from 'GhostPortalType' »
+
+  # This prevents serialize (ZODB) from reloading the class during commit
+  # (which would look for __Broken_newargs__ which is not present)
+  __getnewargs__ = None
 
   def __getattr__(self, name):
     try:

product/ERP5Type/dynamic/portal_type_class.py

@@ -63,9 +63,10 @@ ACQUIRE_LOCAL_ROLE_GETTER_DICT = {
 
 
 def _importFilesystemClass(classpath):
+  from importlib import import_module
   try:
     module_path, class_name = classpath.rsplit('.', 1)
-    module = __import__(module_path, {}, {}, (module_path,))
+    module = import_module(module_path)
     klass = getattr(module, class_name)
 
     # XXX is this required? (here?)

product/ERP5Type/mixin/component.py

@@ -380,9 +380,9 @@ class ComponentMixin(with_metaclass(RecordablePropertyMetaClass, type('NewBase',
     if source_reference is None or not source_reference.startswith('Products'):
       path = os.path.join(cls._getFilesystemPath(), reference + '.py')
     else:
+      from importlib import import_module
+      module_obj = import_module(source_reference)
       import inspect
-      module_obj = __import__(source_reference, globals(), {},
-                              level=0, fromlist=[source_reference])
       path = inspect.getsourcefile(module_obj)
 
     with open(path) as f:

product/ERP5Type/patches/DA.py

@@ -277,11 +277,9 @@ def getObjectMeta(original_function):
   def getObject(module, name, reload=0):
     # Modified version that ignore errors as long as the module can be be
     # imported, which is enough to use a ZODB Extension as a brain.
+    from importlib import import_module
     try:
-      m = __import__('erp5.component.extension.%s' % module, globals(),
-                     {}, 'erp5.component.extension')
-
-      o = getattr(m, name, None)
+      o = getattr(import_module('erp5.component.extension.%s' % module), name, None)
       if o is None:
         raise ImportError(
           "Cannot get %s from erp5.component.extension.%s" % (name, module))

product/ERP5Type/patches/OFSImage.py

@@ -22,14 +22,14 @@ from io import BytesIO
 from zExceptions import Forbidden
 
 def getImageInfo_with_svg_fix(data):
-    data = str(data)
+    data = bytes(data)
     size = len(data)
     height = -1
     width = -1
     content_type = ''
 
     # handle GIFs
-    if (size >= 10) and data[:6] in ('GIF87a', 'GIF89a'):
+    if (size >= 10) and data[:6] in (b'GIF87a', b'GIF89a'):
         # Check to see if content_type is correct
         content_type = 'image/gif'
         w, h = struct.unpack("<HH", data[6:10])
@@ -39,15 +39,16 @@ def getImageInfo_with_svg_fix(data):
     # See PNG v1.2 spec (http://www.cdrom.com/pub/png/spec/)
     # Bytes 0-7 are below, 4-byte chunk length, then 'IHDR'
     # and finally the 4-byte width, height
-    elif ((size >= 24) and (data[:8] == '\211PNG\r\n\032\n')
-          and (data[12:16] == 'IHDR')):
+    elif (size >= 24
+          and data[:8] == b'\211PNG\r\n\032\n'
+          and data[12:16] == b'IHDR'):
         content_type = 'image/png'
         w, h = struct.unpack(">LL", data[16:24])
         width = int(w)
         height = int(h)
 
     # Maybe this is for an older PNG version.
-    elif (size >= 16) and (data[:8] == '\211PNG\r\n\032\n'):
+    elif (size >= 16) and (data[:8] == b'\211PNG\r\n\032\n'):
         # Check to see if we have the right content type
         content_type = 'image/png'
         w, h = struct.unpack(">LL", data[8:16])
@@ -55,29 +56,32 @@ def getImageInfo_with_svg_fix(data):
         height = int(h)
 
     # handle JPEGs
-    elif (size >= 2) and (data[:2] == '\377\330'):
+    elif (size >= 2) and (data[:2] == b'\377\330'):
         content_type = 'image/jpeg'
         jpeg = BytesIO(data)
         jpeg.read(2)
         b = jpeg.read(1)
         try:
             while (b and ord(b) != 0xDA):
-                while (ord(b) != 0xFF): b = jpeg.read(1)
-                while (ord(b) == 0xFF): b = jpeg.read(1)
+                while (ord(b) != 0xFF):
+                    b = jpeg.read(1)
+                while (ord(b) == 0xFF):
+                    b = jpeg.read(1)
                 if (ord(b) >= 0xC0 and ord(b) <= 0xC3):
                     jpeg.read(3)
                     h, w = struct.unpack(">HH", jpeg.read(4))
                     break
                 else:
-                    jpeg.read(int(struct.unpack(">H", jpeg.read(2))[0])-2)
+                    jpeg.read(int(struct.unpack(">H", jpeg.read(2))[0]) - 2)
                 b = jpeg.read(1)
             width = int(w)
             height = int(h)
-        except: pass
+        except Exception:
+            pass
 
     # MONKEY PATCH START HERE
     # Handle SVG
-    elif ("</svg>" in data):
+    elif (b"</svg>" in data):
         content_type = 'image/svg+xml'
     # MONKEY PATCH ENDS HERE
 

product/ERP5Type/patches/Restricted.py

@@ -124,8 +124,14 @@ class TypeAccessChecker:
     as "a method which returing a method" because we can not know what is the
     type until it is actually called. So the three ways are simulated the
     function returned by this method.
+
+    We don't return a simple function, but a class instance with a __bool__ method
+    to accomodate the two cases where this is called by SecurityManager.validate when
+    checking access on the class (then only the bool is used) or by guarded_getattr
+    when checking access on the instance (the __call__ is used).
     """
-    def factory(inst, name):
+    class _AccessChecker:
+      def __call__(self, inst, name):
         """
         Check function used with ContainerAssertions checked by cAccessControl.
         """
@@ -143,7 +149,12 @@ class TypeAccessChecker:
           # fallback to default security
           aq_acquire(inst, name, aq_validate, getSecurityManager().validate)
         return v
-    return factory
+
+      def __bool__(self):
+        return False
+      __nonzero__ = __bool__ # six.PY2
+
+    return _AccessChecker()
 
   def __bool__(self):
     # If Containers(type(x)) is true, ZopeGuard checks will short circuit,
@@ -177,8 +188,9 @@ import past.builtins # six.PY2
 allow_module('past.builtins')
 ModuleSecurityInfo('past.builtins').declarePublic('cmp')
 
-def guarded_sorted(seq, cmp=None, key=None, reverse=False):
-    if cmp is not None: # six.PY2
+if six.PY2:
+    def guarded_sorted(seq, cmp=None, key=None, reverse=False):
+        if cmp is not None:
             from functools import cmp_to_key
             key = cmp_to_key(cmp)
 
@@ -186,7 +198,11 @@ def guarded_sorted(seq, cmp=None, key=None, reverse=False):
             for i, x in enumerate(seq):
                 guard(seq, x, i)
         return sorted(seq, key=key, reverse=reverse)
-safe_builtins['sorted'] = guarded_sorted
+    safe_builtins['sorted'] = guarded_sorted
+
+def guarded_enumerate(seq, start=0):
+    return NullIter(enumerate(guarded_iter(seq), start=start))
+safe_builtins['enumerate'] = guarded_enumerate
 
 def guarded_reversed(seq):
     return SafeIter(reversed(seq))
@@ -195,9 +211,6 @@ ContainerAssertions[reversed] = 1
 # listreverseiterator is a special type, returned by list.__reversed__
 ContainerAssertions[type(reversed([]))] = 1
 
-def guarded_enumerate(seq, start=0):
-    return NullIter(enumerate(guarded_iter(seq), start=start))
-safe_builtins['enumerate'] = guarded_enumerate
 
 def get_set_pop(s, name):
     def guarded_pop():
@@ -357,6 +370,8 @@ allow_class_attribute(datetime.tzinfo)
 # This prevents both importing _strptime with level=0, and accessing __doc__,
 # when calling datetime.datetime.strptime().
 import _strptime
+# on python3 it seems we actually need to call strptime for this.
+datetime.datetime.strptime('', '')
 
 # Allow dict.fromkeys, Only this method is a class method in dict module.
 allow_class_attribute(dict, {'fromkeys': 1})
@@ -432,6 +447,9 @@ except ImportError:
   import_default_level = -1
 def guarded_import(mname, globals=None, locals=None, fromlist=None,
     level=import_default_level):
+  # XXX workaround C-code calling PyImport_Import
+  if mname in ('numpy.core._dtype',):
+    return __import__(mname, globals, locals, fromlist)
   for fromname in fromlist or ():
     if fromname[:1] == '_':
       raise Unauthorized(fromname)
@@ -494,6 +512,7 @@ for dtype in ('int8', 'int16', 'int32', 'int64', \
               'uint8', 'uint16', 'uint32', 'uint64', \
               'float16', 'float32', 'float64', \
               'complex64', 'complex128'):
+  allow_type(type(np.dtype(dtype)))
   z = np.array([0,], dtype = dtype)
   allow_type(type(z[0]))
   allow_type(type(z))
@@ -509,7 +528,6 @@ for dtype in ('int8', 'int16', 'int32', 'int64', \
 allow_type(np.dtype)
 allow_type(np.timedelta64)
 allow_type(type(np.c_))
-allow_type(type(np.dtype('int16')))
 sz = np.array([('2017-07-12T12:30:20',)], dtype=[('date', 'M8[s]')])
 allow_type(type(sz[0]['date']))
 
@@ -556,20 +574,26 @@ else:
 
   allow_class(pd.DataFrame)
 
-  # Note: These black_list methods are for pandas 0.19.2
+  # Note: These black_list methods are for pandas 0.19.2 on PY2 and 1.4.0 on PY3
   series_black_list = ('to_csv', 'to_json', 'to_pickle', 'to_hdf',
-                       'to_sql', 'to_msgpack')
+                       'to_sql',)
+  if six.PY2:
+    series_black_list += ('to_msgpack', )
   ContainerAssertions[pd.Series] = _check_access_wrapper(
     pd.Series, dict.fromkeys(series_black_list, restrictedMethod))
 
   pandas_black_list = ('read_pickle', 'read_hdf',
-                       'read_excel', 'read_html', 'read_msgpack',
+                       'read_excel', 'read_html',
                        'read_gbq', 'read_sas', 'read_stata')
+  if six.PY2:
+    pandas_black_list += ('read_msgpack', )
   ModuleSecurityInfo(MNAME_MAP['pandas']).declarePrivate(*pandas_black_list)
 
   dataframe_black_list = ('to_csv', 'to_json', 'to_pickle', 'to_hdf',
-                          'to_excel', 'to_html', 'to_sql', 'to_msgpack',
+                          'to_excel', 'to_html', 'to_sql',
                           'to_latex', 'to_gbq', 'to_stata')
+  if six.PY2:
+    dataframe_black_list += ('to_msgpack', )
   ContainerAssertions[pd.DataFrame] = _check_access_wrapper(
     pd.DataFrame, dict.fromkeys(dataframe_black_list, restrictedMethod))
 

product/ERP5Type/patches/memcache_client.py

 # -*- coding: utf-8 -*-
-# Code based on python-memcached-1.53
+# Code based on python-memcached-1.58
 try:
     from memcache import _Host, Client, _Error
 except ImportError:
@@ -15,9 +15,11 @@ else:
             pass
     Client.MemcachedConnectionError = _ConnectionDeadError
 
+    import six
     import socket
     def _get(self, cmd, key):
-        if getattr(self, 'do_check_key', True):
+        key = self._encode_key(key)
+        if self.do_check_key:
             self.check_key(key)
         server, key = self._get_server(key)
         if not server:
@@ -29,25 +31,22 @@ else:
             self._statlog(cmd)
 
             try:
-                server.send_cmd("%s %s" % (cmd, key))
+                cmd_bytes = cmd.encode('utf-8') if six.PY3 else cmd
+                fullcmd = b''.join((cmd_bytes, b' ', key))
+                server.send_cmd(fullcmd)
                 rkey = flags = rlen = cas_id = None
 
                 if cmd == 'gets':
-                    try:
-                        rkey, flags, rlen, cas_id, = self._expect_cas_value(server,
-                                raise_exception=True)
-                    except TypeError:
-                        # BBB
-                        rkey, flags, rlen, cas_id, = self._expect_cas_value(server)
+                    rkey, flags, rlen, cas_id, = self._expect_cas_value(
+                        server, raise_exception=True
+                    )
                     if rkey and self.cache_cas:
                         self.cas_ids[rkey] = cas_id
                 else:
-                    try:
-                        rkey, flags, rlen, = self._expectvalue(server,
-                                raise_exception=True)
-                    except TypeError:
-                        # BBB
-                        rkey, flags, rlen, = self._expectvalue(server)
+                    rkey, flags, rlen, = self._expectvalue(
+                        server, raise_exception=True
+                    )
+
                 if not rkey:
                     # (patch)
                     # return None
@@ -55,13 +54,10 @@ else:
                 try:
                     value = self._recv_value(server, flags, rlen)
                 finally:
-                    try:
-                        server.expect("END", raise_exception=True)
-                    except TypeError:
-                        # BBB
-                        server.expect("END")
+                    server.expect(b"END", raise_exception=True)
             except (_Error, socket.error) as msg:
-                if isinstance(msg, tuple): msg = msg[1]
+                if isinstance(msg, tuple):
+                    msg = msg[1]
                 server.mark_dead(msg)
                 # (patch)
                 # return None

product/ERP5Type/patches/pylint.py

@@ -23,6 +23,7 @@ import six
 import sys
 import types
 import warnings
+import importlib
 from Products.ERP5Type import IS_ZOPE2
 
 # TODO: make sure that trying to use it does not import isort, because the
@@ -245,10 +246,7 @@ def _getattr(self, name, *args, **kw):
                 # XXX actually maybe we don't need this branch at all on py3
             ):
             raise
-        real_module = __import__(
-            self.name,
-            fromlist=[self.name] if six.PY2 else [name],
-            level=0)
+        real_module = importlib.import_module(self.name)
         try:
             attr = getattr(real_module, name)
         except AttributeError:
@@ -462,7 +460,7 @@ def fail_hook_BTrees(modname):
     if modname not in _inspected_modules:
         try:
             modcode = build_stub(
-              __import__(modname, {}, {}, [modname], level=0),
+              importlib.import_module(modname),
               # Exclude all classes ending with 'Py' (no reason to not call the
               # C version and not part of public API anyway)
               identifier_re=r'^[A-Za-z_]\w*(?<!Py)$')
@@ -507,7 +505,7 @@ for filename in os.listdir(os.path.dirname(lxml.__file__)):
         module_name = 'lxml.' + filename.split('.', 1)[0]
         _register_module_extender_from_live_module(
             module_name,
-            __import__(module_name, fromlist=[module_name], level=0))
+            importlib.import_module(module_name))
 
 # Wendelin and XLTE are special namespace packages which pylint fails to recognize, and so
 # complains about things like `from wendelin.bigarray.array_zodb import ZBigArray`
@@ -528,7 +526,6 @@ def register_xpkg(pkgname):
                 return m
             MANAGER.register_transform(Module, xpkg_transform, lambda node: node.name == pkgname)
         else:
-            import importlib
             def fail_hook_xpkg(modname):
                 if modname.split('.')[0] == pkgname:
                     return MANAGER.ast_from_module(importlib.import_module(modname))

product/ERP5Type/patches/python.py

@@ -155,9 +155,10 @@ def patch_linecache():
             data = get_source(name)
           except (ImportError, AttributeError):
             pass
-
         return data.splitlines(True) if data is not None else ()
 
+      if module_globals is not None:
+        # in-ZODB python scripts
         if basename(filename) in ('Script (Python)', 'ERP5 Python Script', 'ERP5 Workflow Script'):
           try:
             script = module_globals['script']
@@ -166,6 +167,8 @@ def patch_linecache():
           except Exception:
             pass
           return ()
+
+      # TALES expressions
       x = expr_search(filename)
       if x:
         return x.groups()
@@ -173,4 +176,5 @@ def patch_linecache():
 
   linecache.getlines = getlines
 
-patch_linecache()
+if sys.version_info[:3] < (3, ):
+  patch_linecache()

product/ERP5Type/tests/ERP5TypeTestCase.py

@@ -189,6 +189,7 @@ def profile_if_environ(environment_var_name):
 assert getattr(DateTime, '_original_parse_args', None) is None
 DateTime._original_parse_args = DateTime._parse_args
 
+_datetime_system_time_patcher = None
 _pinned_date_time = None
 
 def _parse_args(self, *args, **kw):
@@ -368,26 +369,38 @@ class ERP5TypeTestCaseMixin(ProcessingNodeTestCase, PortalTestCase, functional.F
       if not uf.getUserById(user_name):
         uf._doAddUser(user_name, self.newPassword(), ['Member'], [])
 
-    def pinDateTime(self, date_time):
+    @classmethod
+    def pinDateTime(cls, date_time):
       # pretend time has stopped at a certain date (i.e. the test runs
       # infinitely fast), for example to avoid errors on tests that are started
       # just before midnight.
-      # This can be used as a context manager, otherwise use unpinDateTime to
+      # This is best used as a context manager, otherwise use unpinDateTime to
       # reset.
-      global _pinned_date_time
+      global _pinned_date_time, _datetime_system_time_patcher
       assert date_time is None or isinstance(date_time, DateTime)
       _pinned_date_time = date_time
 
-      unpinDateTime = self.unpinDateTime
+      if _datetime_system_time_patcher is not None:
+        _datetime_system_time_patcher.stop()
+      if date_time is not None:
+        _datetime_system_time_patcher = mock.patch.object(
+          sys.modules['DateTime.DateTime'],
+          '_system_time',
+          return_value=date_time.timeTime())
+        _datetime_system_time_patcher.start()
+
+      unpinDateTime = cls.unpinDateTime
       class UnpinContextManager(object):
         def __enter__(self):
           return self
         def __exit__(self, *args):
           unpinDateTime()
+          _datetime_system_time_patcher.stop()
       return UnpinContextManager()
 
-    def unpinDateTime(self):
-      self.pinDateTime(None)
+    @classmethod
+    def unpinDateTime(cls):
+      cls.pinDateTime(None)
 
     def setTimeZoneToUTC(self):
       # Deprecated, prefer using `timeZoneContext` context manager instead.

product/ERP5Type/tests/ERP5TypeTestSuite.py

@@ -62,6 +62,11 @@ class ERP5TypeTestSuite(TestSuite):
       assert len(marker_connection_string) == len(actual_connection_string)
       with open(os.path.join(instance_home, 'var', 'Data.fs'), 'rb') as f:
         data_fs = f.read()
+      # XXX adjust FileStorage "magic" number so that python3 ZODB accepts reading a
+      # ZODB for python2, we'll handle the data migration ourselves.
+      from ZODB._compat import FILESTORAGE_MAGIC
+      data_fs = FILESTORAGE_MAGIC + data_fs[len(FILESTORAGE_MAGIC):]
+
       with open(os.path.join(instance_home, 'var', 'Data.fs'), 'wb') as f:
         f.write(data_fs.replace(marker_connection_string, actual_connection_string))
 

product/ERP5Type/tests/runUnitTest.py

 #!/usr/bin/env python2.7
+from __future__ import absolute_import
 from __future__ import print_function
 import os
 import sys
@@ -298,10 +299,9 @@ class ERP5TypeTestLoader(unittest.TestLoader):
         self._loading_packages = set()
 
   def _importZodbTestComponent(self, name):
+    from importlib import import_module
     import erp5.component.test
-    module = __import__('erp5.component.test.' + name,
-                        fromlist=['erp5.component.test'],
-                        level=0)
+    module = import_module('erp5.component.test.' + name)
     try:
       self._test_component_ref_list.append(module)
     except AttributeError:

product/ERP5Type/tests/testDynamicClassGeneration.py

@@ -28,6 +28,7 @@
 # 02110-1301, USA.
 #
 ##############################################################################
+from __future__ import absolute_import
 
 import gc
 import os
@@ -37,6 +38,7 @@ import unittest
 import warnings
 import re
 import sys
+from importlib import import_module
 
 import transaction
 from persistent import Persistent
@@ -1449,8 +1451,7 @@ class TestZodbModuleComponent(SecurityTestCase):
 
   def afterSetUp(self):
     self._component_tool = self.portal.portal_components
-    self._module = __import__(self._document_class._getDynamicModuleNamespace(),
-                              fromlist=['erp5.component'])
+    self._module = import_module(self._document_class._getDynamicModuleNamespace())
     self._component_tool.reset(force=True,
                                reset_portal_type_at_transaction_boundary=True)
 
@@ -1520,7 +1521,7 @@ class TestZodbModuleComponent(SecurityTestCase):
 
     if expected_default_version is not None:
       top_module_name = self._document_class._getDynamicModuleNamespace()
-      top_module = __import__(top_module_name, level=0, fromlist=[top_module_name])
+      top_module = import_module(top_module_name)
 
       # The module must be available in its default version
       self.assertHasAttribute(top_module, expected_default_version)
@@ -1549,10 +1550,7 @@ class TestZodbModuleComponent(SecurityTestCase):
 
   def _importModule(self, module_name):
     module_name = self._getComponentFullModuleName(module_name)
-    module = __import__(
-      module_name,
-      fromlist=[self._document_class._getDynamicModuleNamespace()],
-      level=0)
+    module = import_module(module_name)
     self.assertIn(module_name, sys.modules)
     return module
 
@@ -2043,8 +2041,7 @@ def bar(*args, **kwargs):
     # later that the module has not been added to the top-level package
     self.assertModuleImportable('erp5_version.%s' % imported_reference)
 
-    top_module = __import__(top_module_name, level=0,
-                            fromlist=[top_module_name])
+    top_module = import_module(top_module_name)
 
     self._importModule('erp5_version.%s' % imported_reference)
 
@@ -2106,8 +2103,7 @@ def function_foo(*args, **kwargs):
       self.failIfModuleImportable('foo_version.%s' % reference)
 
       top_module_name = self._document_class._getDynamicModuleNamespace()
-      top_module = __import__(top_module_name, level=0,
-                              fromlist=[top_module_name])
+      top_module = import_module(top_module_name)
 
       self._importModule(reference)
       module = getattr(top_module, reference)
@@ -3401,13 +3397,18 @@ break_at_import()
       return self._component_tool.readTestOutput()
 
     output = runLiveTest('testRunLiveTestImportError')
+    relative_url = 'portal_components/test.erp5.testRunLiveTestImportError'
+    if six.PY2:
+      module_file = '<' + relative_url + '>'
+    else:
+      module_file = 'erp5://' + relative_url
     self.assertIn('''
-  File "<portal_components/test.erp5.testRunLiveTestImportError>", line 4, in <module>
+  File "%(module_file)s", line 4, in <module>
     break_at_import()
-  File "<portal_components/test.erp5.testRunLiveTestImportError>", line 3, in break_at_import
+  File "%(module_file)s", line 3, in break_at_import
     import non.existing.module # pylint:disable=import-error
 ImportError: No module named non.existing.module
-''', output)
+''' % dict(module_file=module_file), output)
 
     output = runLiveTest('testDoesNotExist_import_error_because_module_does_not_exist')
     self.assertIn(

product/PortalTransforms/Transform.py

 # -*- coding: utf-8 -*-
 from zLOG import ERROR
 from six.moves import UserDict
+from importlib import import_module
 
 from zope.interface import implementer
 
@@ -21,7 +22,7 @@ from Products.PortalTransforms.transforms.broken import BrokenTransform
 
 def import_from_name(module_name):
     """ import and return a module by its name """
-    return __import__(module_name, {}, {}, module_name)
+    return import_module(module_name)
 
 def make_config_persistent(kwargs):
     """ iterates on the given dictionnary and replace list by persistent list,

product/PortalTransforms/transforms/safe_html.py

 # -*- coding: utf-8 -*-
 from six import unichr
 from zLOG import ERROR
-from six.moves.html_parser import HTMLParser, HTMLParseError
+from six.moves.html_parser import HTMLParser
 import re
 from Products.PythonScripts.standard import html_quote
 import codecs
@@ -17,6 +17,11 @@ from lxml.etree import HTMLParser as LHTMLParser
 from lxml.html import tostring
 import six
 
+if six.PY2:
+  from six.moves.html_parser import HTMLParseError
+else:
+  HTMLParseError = AssertionError
+
 try:
   from lxml.html.soupparser import fromstring as soupfromstring
 except ImportError:
@@ -365,7 +370,7 @@ def scrubHTML(html, valid=VALID_TAGS, nasty=NASTY_TAGS,
     # As suggested by python developpers:
     # "Python 3.0 implicitly rejects non-unicode strings"
     # We try to decode strings against provided codec first
-    if isinstance(html, str):
+    if isinstance(html, bytes):
       try:
         html = html.decode(default_encoding)
       except UnicodeDecodeError:

product/ZSQLCatalog/Extensions/zsqlbrain.py

@@ -33,7 +33,11 @@ class ZSQLBrain(Acquisition.Implicit):
     """
     if name.startswith('__') :
       return None
-    return getattr(self.getObject(), name, None)
+    try:
+      obj = self.getObject()
+    except ValueError:
+      return None
+    return getattr(obj, name, None)
 
   def getURL(self):
     return self.path

tests/__init__.py

@@ -4,6 +4,7 @@ import os, subprocess, re
 # test_suite is provided by 'run_test_suite'
 from test_suite import ERP5TypeTestSuite
 import sys
+import six
 from itertools import chain
 
 HERE = os.path.dirname(__file__)
@@ -46,6 +47,14 @@ class _ERP5(ERP5TypeTestSuite):
                                component_re_match.group(2))
       else:
         test_case = test_path.split(os.sep)[-1][:-3] # remove .py
+      if six.PY3:
+        # disable tests that are not compatible with Python 3.
+        if test_case in (
+          # using legacy workflow
+          'erp5_workflow_test:testWorkflowAndDCWorkflow',
+          'testUpgradeInstanceWithOldDataFsLegacyWorkflow'
+        ):
+          continue
       product = test_path.split(os.sep)[-3]
       # don't test 3rd party products
       if product in ('PortalTransforms', 'MailTemplates', 'Zelenium'):
@@ -231,15 +240,24 @@ class ERP5BusinessTemplateCodingStyleTestSuite(_ERP5):
   """Run coding style test on all business templates.
   """
   def getTestList(self):
+    def skip_business_template(path):
+      # we skip coding style check for business templates having this marker
+      # property. Since the property is not exported (on purpose), modified business templates
+      # will be candidate for coding style test again.
+      if os.path.exists(path + '/bt/skip_coding_style_test'):
+        return True
+      if six.PY3 and os.path.basename(path) in (
+          'erp5_workflow_test',  # uses legacy DCWorkflow
+        ):
+          return True
+      return False
+
     test_list = [
       os.path.basename(path)
       for path in chain(
         glob(HERE + '/../bt5/*'),
         glob(HERE + '/../product/ERP5/bootstrap/*'))
-      # we skip coding style check for business templates having this marker
-      # property. Since the property is not exported (on purpose), modified business templates
-      # will be candidate for coding style test again.
-      if not os.path.exists(path + '/bt/skip_coding_style_test') and os.path.isdir(path)
+      if os.path.isdir(path) and not skip_business_template(path)
     ]
     for path in chain(glob(HERE + '/../product/*'),
                       glob(HERE + '/../bt5')):