patchs/pylint: update for python3 support

this works with astroid 3.2.0 and pylint 3.2.0

product/ERP5Type/Tool/ComponentTool.py

@@ -159,14 +159,13 @@ class ComponentTool(BaseTool):
       erp5.component.filesystem_import_dict = None
       erp5.component.ref_manager.gc()
 
-      # Clear pylint cache
-      try:
+      # Clear astroid (pylint) cache
+      if six.PY2:
         from astroid.builder import MANAGER
-      except ImportError:
-        pass
       else:
+        from astroid.astroid_manager import MANAGER
       astroid_cache = MANAGER.astroid_cache
-        for k in astroid_cache.keys():
+      for k in list(astroid_cache.keys()):
         if k.startswith('erp5.component.') and k not in component_package_list:
           del astroid_cache[k]
 

product/ERP5Type/Utils.py

@@ -419,15 +419,11 @@ def fill_args_from_request(*optional_args):
   return decorator
 
 _pylint_message_re = re.compile(
-  '^(?P<type>[CRWEF]):\s*(?P<row>\d+),\s*(?P<column>\d+):\s*(?P<message>.*)$')
+  r'^(?P<type>[CRWEF]):\s*(?P<row>\d+),\s*(?P<column>\d+):\s*(?P<message>.*)$')
 
 def checkPythonSourceCode(source_code_str, portal_type=None):
   """
   Check source code with pylint or compile() builtin if not available.
-
-  TODO-arnau: Get rid of NamedTemporaryFile (require a patch on pylint to
-              allow passing a string) and this should probably return a proper
-              ERP5 object rather than a dict...
   """
   if not source_code_str:
     return []
@@ -462,8 +458,11 @@ def checkPythonSourceCode(source_code_str, portal_type=None):
   message_list = []
   output_file = StringIO()
   try:
-    with tempfile.NamedTemporaryFile(prefix='checkPythonSourceCode',
-                                     suffix='.py') as input_file:
+    with tempfile.NamedTemporaryFile(
+        prefix='checkPythonSourceCode',
+        suffix='.py',
+        mode='w',
+      ) as input_file:
       input_file.write(source_code_str)
       input_file.flush()
 
@@ -493,6 +492,8 @@ def checkPythonSourceCode(source_code_str, portal_type=None):
            # TODO-arnau: Enable it properly would require inspection API
            # '%s %r has no %r member'
            '--disable=E1101,E1103',
+           # XXX duplicate-bases causes too many false positives
+           '--disable=duplicate-bases',
            # map and filter should not be considered bad as in some cases
            # map is faster than its recommended replacement (list
            # comprehension)
@@ -508,7 +509,26 @@ def checkPythonSourceCode(source_code_str, portal_type=None):
            # unused variables
            '--dummy-variables-rgx=_$|dummy|__traceback_info__|__traceback_supplement__',
       ]
-
+      if six.PY3:
+        args.extend(
+          (
+            "--msg-template='{C}: {line},{column}: {msg} ({symbol})'",
+            '--load-plugins=pylint.extensions.bad_builtin',
+            # BBB until we drop compatibility with PY2
+            '--disable=redundant-u-string-prefix,raise-missing-from,keyword-arg-before-vararg',
+            # XXX acceptable to ignore in the context of ERP5
+            '--disable=unspecified-encoding',
+            # XXX to many errors for now
+            '--disable=arguments-differ,arguments-renamed',
+            '--disable=duplicate-bases,inconsistent-mro',
+          )
+        )
+      else:
+        args.extend(
+          (
+            '--load-plugins=Products.ERP5Type.patches.pylint_compatibility_disable',
+          )
+        )
       if portal_type == 'Interface Component':
         # __init__ method from base class %r is not called
         args.append('--disable=W0231')
@@ -521,19 +541,18 @@ def checkPythonSourceCode(source_code_str, portal_type=None):
         # Method should have "self" as first argument (no-self-argument)
         args.append('--disable=E0213')
 
-      try:
-        from pylint.extensions.bad_builtin import __name__ as ext
-        args.append('--load-plugins=' + ext)
-      except ImportError:
-        pass
       try:
         # Note that we don't run pylint as a subprocess, but directly from
         # ERP5 process, so that pylint can access the code from ERP5Type
         # dynamic modules from ZODB.
         Run(args, reporter=TextReporter(output_file), exit=False)
       finally:
+        if six.PY2:
           from astroid.builder import MANAGER
-        MANAGER.astroid_cache.pop(
+        else:
+          from astroid.astroid_manager import MANAGER
+        astroid_cache = MANAGER.astroid_cache
+        astroid_cache.pop(
           os.path.splitext(os.path.basename(input_file.name))[0],
           None)
 

product/ERP5Type/__init__.py

@@ -39,8 +39,7 @@ if  getZopeVersion()[0] == 2: # BBB Zope2
 else:
   IS_ZOPE2 = False
 import six
-if six.PY2:
-  from .patches import pylint
+from .patches import pylint
 from zLOG import LOG, INFO
 DISPLAY_BOOT_PROCESS = False
 

product/ERP5Type/patches/pylint_compatibility_disable.py

+"""A dummy checker to register messages from pylint3, to be able to
+disable the messages on python2 without causing bad-option-value
+"""
+from __future__ import absolute_import
+from pylint import checkers, interfaces
+
+class CompatibilityDisableChecker(checkers.BaseChecker):
+
+    name = "compatibility-disable"
+    msgs = {
+        "E9999": (
+            "not-an-iterable",
+            "not-an-iterable",
+            "",
+        ),
+        "E9998": (
+            "misplaced-bare-raise",
+            "misplaced-bare-raise",
+            "",
+        ),
+        "E9997": (
+            "unused-private-member",
+            "unused-private-member",
+            "",
+        ),
+        "E9996": (
+            "using-constant-test",
+            "using-constant-test",
+            ""
+        ),
+        "E9995": (
+            "modified-iterating-list",
+            "modified-iterating-list",
+            "",
+        ),
+        "E9994": (
+            "unsubscriptable-object",
+            "unsubscriptable-object",
+            "",
+        ),
+        "E9993": (
+            "invalid-unary-operand-type",
+            "invalid-unary-operand-type",
+            "",
+        ),
+        "E9992": (
+            "unbalanced-dict-unpacking",
+            "unbalanced-dict-unpacking",
+            "",
+        ),
+        "E9991": (
+            "self-cls-assignment",
+            "self-cls-assignment",
+            "",
+        ),
+        "E9990": (
+            "deprecated-class",
+            "deprecated-class",
+            "",
+        ),
+        "E9989": (
+            "possibly-used-before-assignment",
+            "possibly-used-before-assignment",
+            ""
+        )
+    }
+
+
+def register(linter):
+    linter.register_checker(CompatibilityDisableChecker(linter))

product/ERP5Type/tests/testDynamicClassGeneration.py

@@ -1903,11 +1903,18 @@ class TestZodbModuleComponent(SecurityTestCase):
     component.setTextContent("""import unexistent_module
 """ + valid_code)
     self.tic()
+    if six.PY2:
       self.assertEqual(
         [m.getMessage().translate() for m in component.checkConsistency()],
         ["Error in Source Code: F:  1,  0: Unable to import 'unexistent_module' (import-error)"])
       self.assertEqual(component.getTextContentErrorMessageList(),
                         ["F:  1,  0: Unable to import 'unexistent_module' (import-error)"])
+    else:
+      self.assertEqual(
+        [m.getMessage().translate() for m in component.checkConsistency()],
+        ["Error in Source Code: E:  1,  0: Unable to import 'unexistent_module' (import-error)"])
+      self.assertEqual(component.getTextContentErrorMessageList(),
+                        ["E:  1,  0: Unable to import 'unexistent_module' (import-error)"])
     self.assertEqual(component.getTextContentWarningMessageList(),
                       ["W:  1,  0: Unused import unexistent_module (unused-import)"])
 
@@ -1939,11 +1946,10 @@ class TestZodbModuleComponent(SecurityTestCase):
        [ComponentMixin._message_text_content_not_set],
        [],
        []),
-      ("""def foobar(*args, **kwargs)
-  return 42
+      ("""None()
 """ + valid_code,
-       ["Error in Source Code: E:  1,  0: invalid syntax (syntax-error)"],
-       ["E:  1,  0: invalid syntax (syntax-error)"],
+       ["Error in Source Code: E:  1,  0: None is not callable (not-callable)"],
+       ["E:  1,  0: None is not callable (not-callable)"],
        []),
       # Make sure that foobar NameError is at the end to make sure that after
       # defining foobar function, it is not available at all
@@ -2213,7 +2219,10 @@ def function_foo(*args, **kwargs):
   def _assertAstroidCacheContent(self,
                                  must_be_in_cache_set,
                                  must_not_be_in_cache_set):
+    if six.PY2:
       from astroid.builder import MANAGER
+    else:
+      from astroid.astroid_manager import MANAGER
     should_not_be_in_cache_list = []
     for modname in MANAGER.astroid_cache:
       if (modname.startswith('checkPythonSourceCode') or
@@ -2269,10 +2278,10 @@ def hoge():
       """# -*- coding: utf-8 -*-
 # Source code with non-ASCII character should not fail: éàホゲ
 from %(namespace)s import %(reference1)s
-from %(namespace)s.erp5_version import %(reference1)s
+from %(namespace)s.erp5_version import %(reference1)s  # pylint:disable=reimported
 
 from %(module2)s import hoge
-from %(module2_with_version)s import hoge
+from %(module2_with_version)s import hoge  # pylint:disable=reimported
 
 import %(module2)s
 import %(module2_with_version)s
@@ -2297,7 +2306,8 @@ from AccessControl.PermissionRole import rolesForPermissionOn, PermissionRole, i
 
 # Monkey patch of astroid 1.3.8: it raised 'no-name-in-module' because
 # Shared.DC was not considered a namespace package
-from Shared.DC.ZRDB.Results import Results # pylint: disable=unused-import
+from Shared.DC.ZRDB.Results import Results
+_ = Results
 
 import lxml.etree
 lxml.etree.Element('test')
@@ -2344,17 +2354,21 @@ _ = ZBigArray
             module2_with_version=imported_module2_with_version)) +
       component.getTextContent())
 
+    must_be_in_cache_set = set()
+    if six.PY2:
+      must_be_in_cache_set.add(namespace)
     self._assertAstroidCacheContent(
-      must_be_in_cache_set={'%s' % namespace},
+      must_be_in_cache_set=must_be_in_cache_set,
       must_not_be_in_cache_set={'%s.erp5_version' % namespace,
                                 imported_module1,
                                 imported_module1_with_version,
                                 imported_module2,
                                 imported_module2_with_version})
     component.checkSourceCode()
+    if six.PY2:
+      must_be_in_cache_set.add('%s.erp5_version' % namespace)
     self._assertAstroidCacheContent(
-      must_be_in_cache_set={'%s' % namespace,
-                            '%s.erp5_version' % namespace},
+      must_be_in_cache_set=must_be_in_cache_set,
       must_not_be_in_cache_set={imported_module1,
                                 imported_module1_with_version,
                                 imported_module2,
@@ -2362,9 +2376,11 @@ _ = ZBigArray
 
     self.tic()
     self.assertEqual(component.getValidationState(), 'modified')
+    if six.PY2:
       self.assertEqual(
         component.getTextContentErrorMessageList(),
-      ["E:  3,  0: No name '%s' in module '%s' (no-name-in-module)" %
+        [
+          "E:  3,  0: No name '%s' in module '%s' (no-name-in-module)" %
           (imported_reference1, namespace),
           "E:  4,  0: No name '%s' in module '%s.erp5_version' (no-name-in-module)" %
           (imported_reference1, namespace),
@@ -2388,9 +2404,41 @@ _ = ZBigArray
           "E: 10,  0: No name '%s' in module '%s.erp5_version' (no-name-in-module)" %
           (imported_reference2, namespace),
           "F: 10,  0: Unable to import '%s' (import-error)" %
-       imported_module2_with_version])
+          imported_module2_with_version,
+        ],
+      )
+    else:
+      self.assertEqual(
+        component.getTextContentErrorMessageList(),
+        [
+          "E:  3,  0: No name '%s' in module '%s' (no-name-in-module)" %
+          (imported_reference1, namespace),
+          "E:  4,  0: No name '%s' in module '%s.erp5_version' (no-name-in-module)" %
+          (imported_reference1, namespace),
+          "E:  6,  0: Unable to import '%s.%s' (import-error)" %
+          (namespace, imported_reference2),
+          # Spurious message but same as filesystem modules: 2 errors raised
+          # (no-name-in-module and import-error)
+          "E:  6,  0: No name '%s' in module '%s' (no-name-in-module)" %
+          (imported_reference2, namespace),
+          "E:  7,  0: Unable to import '%s' (import-error)" %
+          imported_module2_with_version,
+          # Spurious message (see above comment)
+          "E:  7,  0: No name '%s' in module '%s.erp5_version' (no-name-in-module)" %
+          (imported_reference2, namespace),
+          "E:  9,  0: Unable to import '%s.%s' (import-error)" %
+          (namespace, imported_reference2),
+          # Spurious message (see above comment)
+          "E:  9,  0: No name '%s' in module '%s' (no-name-in-module)" %
+          (imported_reference2, namespace),
+          "E: 10,  0: Unable to import '%s' (import-error)" %
+          imported_module2_with_version,
+          # Spurious message (see above comment)
+          "E: 10,  0: No name '%s' in module '%s.erp5_version' (no-name-in-module)" %
+          (imported_reference2, namespace),
+        ],
+      )
     self.assertEqual(component.getTextContentWarningMessageList(), [])
-
     ## Simulate user:
     # 1) First check and validate 'imported' Components
     self.portal.portal_workflow.doActionFor(imported_component1, 'validate_action')
@@ -2401,20 +2449,29 @@ _ = ZBigArray
 
     message_list = component.checkSourceCode()
     self.assertEqual(message_list, [])
-    self._assertAstroidCacheContent(
-      must_be_in_cache_set={'%s' % namespace,
-                            '%s.erp5_version' % namespace,
+    must_be_in_cache_set = {
       imported_module1,
       imported_module1_with_version,
       imported_module2,
-                            imported_module2_with_version},
+      imported_module2_with_version,
+    }
+    if six.PY2:
+      must_be_in_cache_set.update({
+        '%s' % namespace,
+        '%s.erp5_version' % namespace,
+      })
+    self._assertAstroidCacheContent(
+      must_be_in_cache_set=must_be_in_cache_set,
       must_not_be_in_cache_set=set())
 
     # 2) Then modify the main one so that it automatically 'validate'
     component.setTextContent(component.getTextContent() + '\n')
     self.tic()
+    must_be_in_cache_set = set()
+    if six.PY2:
+      must_be_in_cache_set.add(namespace)
     self._assertAstroidCacheContent(
-      must_be_in_cache_set={'%s' % namespace},
+      must_be_in_cache_set=must_be_in_cache_set,
       must_not_be_in_cache_set={'%s.erp5_version' % namespace,
                                 imported_module1,
                                 imported_module1_with_version,
@@ -2427,10 +2484,11 @@ _ = ZBigArray
     component.setTextContent(
       """# -*- coding: utf-8 -*-
 from %(module)s import undefined
-from %(module_with_version)s import undefined
+from %(module_with_version)s import undefined2
 
 # To avoid 'unused-import' warning...
 undefined()
+undefined2()
 
 """ % (dict(module=imported_module2,
             module_with_version=imported_module2_with_version)) +
@@ -2441,7 +2499,7 @@ undefined()
       component.getTextContentErrorMessageList(),
       ["E:  2,  0: No name 'undefined' in module '%s' (no-name-in-module)" %
        imported_module2_with_version,
-       "E:  3,  0: No name 'undefined' in module '%s' (no-name-in-module)" %
+       "E:  3,  0: No name 'undefined2' in module '%s' (no-name-in-module)" %
        imported_module2_with_version])
     self.assertEqual(component.getTextContentWarningMessageList(), [])
 
@@ -2477,8 +2535,8 @@ def hoge():
       component = self._newComponent(reference)
       component.setTextContent(component.getTextContent() + """
 from %(namespace)s import %(reference)s
-from %(namespace)s.bar_version import %(reference)s
-from %(namespace)s.erp5_version import %(reference)s
+from %(namespace)s.bar_version import %(reference)s  # pylint:disable=reimported
+from %(namespace)s.erp5_version import %(reference)s  # pylint:disable=reimported
 
 # To avoid 'unused-import' warning...
 %(reference)s.hoge()
@@ -2486,7 +2544,10 @@ from %(namespace)s.erp5_version import %(reference)s
            reference=imported_reference))
 
       component.checkSourceCode()
+      if six.PY2:
         from astroid.builder import MANAGER
+      else:
+        from astroid.astroid_manager import MANAGER
       imported_module = self._getComponentFullModuleName(imported_reference)
       self.assertEqual(
         MANAGER.astroid_cache[self._getComponentFullModuleName(imported_reference, version='bar')],