Commit 1cb2dc4c authored by Arnaud Fontaine's avatar Arnaud Fontaine

ZODB Components: Enable checking of imports with pylint.

Until now it was disabled which means that if Component A imports Component B
and the latter is not in {validated,modified} state, Component A could be
validated without raising any error and failed at execution time.

As this relies on Pylint transform/hook and avoiding monkey patch as much
as possible, make Products.ERP5Type.patches.pylint available for Python3 (not
actually tested with ERP5 but no AttributeError nor ImportError with Python3
with this code).

Also, allow developer to call validate action from 'modified' state to handle
import use case:
  1. Edit A which raises an error stating that B.
  2. Fix B and validate it.
  3. Validate again A without requiring a modification of {reference,version,text_content}.
parent 88a03532
state_change['object'].Base_checkConsistency()
obj = state_change['object']
# User explicitely wants to validate again (example use case: import-error
# on ZODB Components which have now been fixed)
if obj.getValidationState() == 'modified':
obj.checkSourceCode()
obj.Base_checkConsistency()
......@@ -26,6 +26,7 @@
<string>invalidate_action</string>
<string>modify</string>
<string>validate</string>
<string>validate_action</string>
</tuple>
</value>
</item>
......
obj = state_change['object']
error_list = []
warning_list = []
for message_dict in obj.checkSourceCode():
message = '%s:%3d,%3d: %s' % (message_dict['type'],
message_dict['row'],
message_dict['column'],
message_dict['text'])
if message_dict['type'] in ('F', 'E'):
error_list.append(message)
else:
warning_list.append(message)
obj.setTextContentWarningMessageList(warning_list)
obj.setTextContentErrorMessageList(error_list)
if not error_list and obj.getValidationState() == 'modified':
obj.checkSourceCode()
if not obj.getTextContentErrorMessageList() and obj.getValidationState() == 'modified':
obj.checkConsistencyAndValidate()
......@@ -498,8 +498,6 @@ def checkPythonSourceCode(source_code_str, portal_type=None):
# TODO-arnau: Enable it properly would require inspection API
# '%s %r has no %r member'
'--disable=E1101,E1103',
# 'No name %r in module %r'
'--disable=E0611',
# map and filter should not be considered bad as in some cases
# map is faster than its recommended replacement (list
# comprehension)
......
......@@ -414,6 +414,14 @@ class ComponentDynamicPackage(ModuleType):
delattr(package, name)
# Clear pylint cache
try:
from astroid.builder import MANAGER
except ImportError:
pass
else:
MANAGER.astroid_cache.pop(module_name, None)
class ToolComponentDynamicPackage(ComponentDynamicPackage):
def reset(self, *args, **kw):
"""
......
......@@ -309,7 +309,22 @@ class ComponentMixin(PropertyRecordableMixin, Base):
Check Component source code through Pylint or compile() builtin if not
available
"""
return checkPythonSourceCode(self.getTextContent(), self.getPortalType())
error_list = []
warning_list = []
for message_dict in checkPythonSourceCode(self.getTextContent(),
self.getPortalType()):
message = '%s:%3d,%3d: %s' % (message_dict['type'],
message_dict['row'],
message_dict['column'],
message_dict['text'])
if message_dict['type'] in ('F', 'E'):
error_list.append(message)
else:
warning_list.append(message)
self.setTextContentWarningMessageList(warning_list)
self.setTextContentErrorMessageList(error_list)
security.declareProtected(Permissions.ModifyPortalContent, 'PUT')
def PUT(self, REQUEST, RESPONSE):
......
......@@ -19,74 +19,195 @@
# 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
from __future__ import absolute_import
from inspect import getargspec
import sys
# TODO: make sure that trying to use it does not import isort, because the
# latter hacks Python in order to execute:
# sys.setdefaultencoding('utf-8')
# This changes the behaviour of some ERP5 code.
sys.modules.setdefault('isort', None)
## All arguments are passed as arguments and this needlessly outputs a 'No
## config file found, using default configuration' message on stderr.
try:
# TODO: Add support for newer versions pylint. Meanwhile, make sure that
# trying to use it does not import isort, because the latter hacks
# Python in order to execute:
# sys.setdefaultencoding('utf-8')
# This changes the behaviour of some ERP5 code.
sys.modules.setdefault('isort', None)
from pylint.checkers.imports import ImportsChecker
import astroid
ImportsChecker.get_imported_module
except (AttributeError, ImportError):
pass
else:
def _get_imported_module(self, importnode, modname):
from logilab.common.configuration import OptionsManagerMixIn
except ImportError:
# pylint 2.x (python3)
from pylint.config import OptionsManagerMixIn
OptionsManagerMixIn.read_config_file = lambda *args, **kw: None
## Pylint transforms and plugin to generate AST for ZODB Components
from astroid.builder import AstroidBuilder
from astroid.exceptions import AstroidBuildingException
from astroid import MANAGER
try:
from astroid.builder import _guess_encoding
except ImportError:
# XXX: With python3, tokenize.detect_encoding() is used instead. This
# should do the same instead of copying/pasting legacy code...
import re
_ENCODING_RGX = re.compile(r"\s*#+.*coding[:=]\s*([-\w.]+)")
def _guess_encoding(string):
"""get encoding from a python file as string or return None if not found"""
# check for UTF-8 byte-order mark
if string.startswith('\xef\xbb\xbf'):
return 'UTF-8'
for line in string.split('\n', 2)[:2]:
# check for encoding declaration
match = _ENCODING_RGX.match(line)
if match is not None:
return match.group(1)
def string_build(self, data, modname='', path=None):
"""
build astroid from source code string and return rebuilded astroid
Monkey patched to check encoding properly, as `file_build()` does:
# `data` can only be an str (not unicode)
module = self._data_build(data, modname, path)
# But here it tries to `encode()` which will fails if there is any
# non-ASCII character...
module.file_bytes = data.encode('utf-8')
return self._post_build(module, 'utf-8')
"""
encoding = _guess_encoding(data)
if encoding is None:
# Encoding not defined in the source file, assuming utf-8...
encoding = 'utf-8'
try:
return importnode.do_import_module(modname)
except astroid.InferenceError, ex:
# BEGIN
# XXX-arnau: Ignore ERP5 dynamic modules, hackish but required
# until proper introspection is implemented because otherwise it
# is impossible to validate Components importing other Components
# and as it is static analysis, the module should not be loaded
# anyway
if modname.startswith('erp5'):
return
# BytesIO() does not handle unicode:
# TypeError: 'unicode' does not have the buffer interface
if isinstance(data, unicode):
data = data.encode(encoding)
else:
# Just to avoid error later on...
data.decode(encoding)
except Exception as exc:
from zLOG import LOG, WARNING
LOG("Products.ERP5Type.patches.pylint", WARNING,
"%s: Considered as not importable: Wrong encoding? (%r)" %
(modname, exc))
raise AstroidBuildingException(exc)
module = self._data_build(data, modname, path)
module.file_bytes = data
return self._post_build(module, encoding)
AstroidBuilder.string_build = string_build
# Handle ImportError try/except checking for missing module before
# falling back to code handling such case (#9386)
pnode = importnode.parent
if pnode and isinstance(pnode, astroid.TryExcept):
for handler in pnode.handlers:
# Handling except:
if not handler.type:
return
from astroid import nodes
def erp5_package_transform(node):
"""'
erp5/' directory on the filesystem is different from 'erp5' module when
running ERP5, so replace entirely this node completely to avoid pylint
checking erp5/ directory structure for module and returns errors...
"""
# Cannot call string_build() as this would be called again and again
erp5_package_node = nodes.Module('erp5', None)
erp5_package_node.package = True
erp5_package_node._absolute_import_activated = True
return erp5_package_node
MANAGER.register_transform(nodes.Module,
erp5_package_transform,
lambda n: n.name == 'erp5')
# Handling ImportError and its Exception base classes
for klass in ImportError.mro():
if klass is object:
break
elif klass.__name__ == handler.type.name:
return
# END
def _buildAstroidModuleFromComponentModuleName(modname):
from Products.ERP5.ERP5Site import getSite
from Acquisition import aq_base
portal = getSite()
component_tool = aq_base(portal.portal_components)
component_obj = None
component_id = modname[len('erp5.component.'):]
if '_version' in modname:
try:
obj = getattr(component_tool,
component_id.replace('_version', '', 1))
except AttributeError:
raise AstroidBuildingException
if obj.getValidationState() in ('modified', 'validated'):
component_obj = obj
else:
raise AstroidBuildingException
if str(ex) != modname:
args = '%r (%s)' % (modname, ex)
else:
args = repr(modname)
self.add_message("F0401", args=args, node=importnode)
try:
package, reference = component_id.split('.', 1)
except ValueError:
raise AstroidBuildingException
for version in portal.getVersionPriorityNameList():
try:
obj = getattr(component_tool,
'%s.%s.%s' % (package, version, reference))
except AttributeError:
continue
if obj.getValidationState() in ('modified', 'validated'):
component_obj = obj
break
if component_obj is None:
raise AstroidBuildingException
# module_build() could also be used but this requires importing
# the ZODB Component and also monkey-patch it to support PEP-302
# for __file__ starting with '<'
module = AstroidBuilder(MANAGER).string_build(
component_obj.getTextContent(validated_only=True),
modname)
return module
if 'modnode' in getargspec(ImportsChecker.get_imported_module).args:
# BBB for pylint < 1.4.0
def get_imported_module(self, modnode, importnode, modname):
return _get_imported_module(self, importnode, modname)
def fail_hook_erp5_component(modname):
if not modname.startswith('erp5.'):
raise AstroidBuildingException
if (modname in ('erp5.portal_type',
'erp5.component',
'erp5.component.module',
'erp5.component.extension',
'erp5.component.document',
'erp5.component.tool',
'erp5.component.interface',
'erp5.component.mixin',
'erp5.component.test') or
(modname.startswith('erp5.component.') and modname.endswith('_version'))):
module = AstroidBuilder(MANAGER).string_build('', modname)
if modname.startswith('erp5.component'):
module.package = True
else:
get_imported_module = _get_imported_module
module = _buildAstroidModuleFromComponentModuleName(modname)
ImportsChecker.get_imported_module = get_imported_module
module._absolute_import_activated = True
return module
MANAGER.register_failed_import_hook(fail_hook_erp5_component)
# All arguments are passed as arguments and this needlessly outputs a 'No
# config file found, using default configuration' message on stderr.
from logilab.common.configuration import OptionsManagerMixIn
OptionsManagerMixIn.read_config_file = lambda *args, **kw: None
## Patch to handle 'no-name-in-module' for attributes added by monkey
## patches in Products/XXX/patches.
##
## Instead of monkey patching, an alternative would be to use Pylint
## transforms but this would require either checking dynamically which
## attributes has been added (much more complex than the current approach)
## or listing them statically (inconvenient).
from pylint.checkers import BaseChecker
from pylint.interfaces import UNDEFINED
BaseChecker_add_message = BaseChecker.add_message
def add_message(self, msg_descr, line=None, node=None, args=None,
confidence=UNDEFINED):
"""
Monkey patched to dynamically ignore some error/warning messages
"""
if msg_descr == 'no-name-in-module':
name, module_name = args
if not module_name.startswith('erp5.'):
# Do not call __import__ as this may load ZODB Component which
# should use 'version' and not use monkey patches...
try:
getattr(sys.modules[module_name], name)
except (KeyError, AttributeError):
pass
else:
# Do nothing as this does exist
return
BaseChecker_add_message(self, msg_descr, line=line, node=node,
args=args, confidence=confidence)
BaseChecker.add_message = add_message
finally:
if sys.modules['isort'] is None:
if sys.modules['isort'] is None:
del sys.modules['isort']
# -*- coding: utf-8 -*-
##############################################################################
#
# Copyright (c) 2010 Nexedi SARL and Contributors. All Rights Reserved.
......@@ -1454,8 +1455,13 @@ class _TestZodbComponent(SecurityTestCase):
text_content=text_content,
portal_type=self._portal_type)
def _getComponentFullModuleName(self, module_name):
return self._document_class._getDynamicModuleNamespace() + '.' + module_name
def _getComponentFullModuleName(self, module_name, version=None):
if version is None:
return (self._document_class._getDynamicModuleNamespace() + '.' +
module_name)
else:
return (self._document_class._getDynamicModuleNamespace() + '.' +
version + '_version' + '.' + module_name)
def failIfModuleImportable(self, module_name):
"""
......@@ -2137,6 +2143,170 @@ def function_foo(*args, **kwargs):
self.assertUserCanModifyDocument(user_id, component)
self.assertUserCanDeleteDocument(user_id, component)
def _assertAstroidCacheContent(self,
must_be_in_cache_set,
must_not_be_in_cache_set):
from astroid.builder import MANAGER
should_not_be_in_cache_list = []
for modname in MANAGER.astroid_cache:
if modname in must_not_be_in_cache_set:
should_not_be_in_cache_list.append(modname)
if modname in must_be_in_cache_set:
must_be_in_cache_set.remove(modname)
self.assertEqual(should_not_be_in_cache_list, [])
self.assertEqual(must_be_in_cache_set, set())
def testPylint(self):
# One 'imported' Component for each use case:
# from erp5.component.module import M1
# from erp5.component.module.M1 import hoge
imported_reference1 = self._generateReference('TestPylintImported1')
imported_component1 = self._newComponent(imported_reference1)
imported_component1.setTextContent(imported_component1.getTextContent() + """
def hoge():
return 'OK'
""")
imported_reference2 = self._generateReference('TestPylintImported2')
imported_component2 = self._newComponent(imported_reference2)
imported_component2.setTextContent(imported_component2.getTextContent() + """
def hoge():
return 'OK'
""")
reference = self._generateReference('TestPylint')
component = self._newComponent(reference)
self.portal.portal_workflow.doActionFor(component, 'validate_action')
self.tic()
self.assertEqual(component.getValidationState(), 'validated')
self.assertEqual(component.getTextContentErrorMessageList(), [])
self.assertEqual(component.getTextContentWarningMessageList(), [])
self.assertModuleImportable(reference)
namespace = self._document_class._getDynamicModuleNamespace()
imported_module1 = self._getComponentFullModuleName(
imported_reference1)
imported_module1_with_version = self._getComponentFullModuleName(
imported_reference1, version='erp5')
imported_module2 = self._getComponentFullModuleName(
imported_reference2)
imported_module2_with_version = self._getComponentFullModuleName(
imported_reference2, version='erp5')
component.setTextContent(
"""# -*- coding: utf-8 -*-
# Source code with non-ASCII character should not fail: éàホゲ
from %(namespace)s import %(reference1)s
from %(namespace)s.erp5_version import %(reference1)s
from %(module2)s import hoge
from %(module2_with_version)s import hoge
import %(module2)s
import %(module2_with_version)s
# To avoid 'unused-import' warnings...
%(reference1)s.hoge()
hoge()
%(module2)s.hoge()
%(module2_with_version)s.hoge()
# Attributes added through Products.XXX.patches: Must not raise error
from Products.DCWorkflow.DCWorkflow import ValidationFailed
# To avoid 'unused-import' warnings...
ValidationFailed('anything')
""" % (dict(namespace=namespace,
reference1=imported_reference1,
module2=imported_module2,
module2_with_version=imported_module2_with_version)) +
component.getTextContent())
self.tic()
self._assertAstroidCacheContent(
must_be_in_cache_set={'%s' % namespace,
'%s.erp5_version' % namespace},
must_not_be_in_cache_set={imported_module1,
imported_module1_with_version,
imported_module2,
imported_module2_with_version})
self.assertEqual(component.getValidationState(), 'modified')
self.assertEqual(
component.getTextContentErrorMessageList(),
["E: 3, 0: No name '%s' in module '%s' (no-name-in-module)" %
(imported_reference1, namespace),
"E: 4, 0: No name '%s' in module '%s.erp5_version' (no-name-in-module)" %
(imported_reference1, namespace),
# Spurious message but same as filesystem modules: 2 errors raised
# (no-name-in-module and import-error)
"E: 6, 0: No name '%s' in module '%s' (no-name-in-module)" %
(imported_reference2, namespace),
"F: 6, 0: Unable to import '%s' (import-error)" %
imported_module2,
# Spurious message (see above comment)
"E: 7, 0: No name '%s' in module '%s.erp5_version' (no-name-in-module)" %
(imported_reference2, namespace),
"F: 7, 0: Unable to import '%s' (import-error)" %
imported_module2_with_version,
# Spurious message (see above comment)
"E: 9, 0: No name '%s' in module '%s' (no-name-in-module)" %
(imported_reference2, namespace),
"F: 9, 0: Unable to import '%s' (import-error)" %
imported_module2,
# Spurious message (see above comment)
"E: 10, 0: No name '%s' in module '%s.erp5_version' (no-name-in-module)" %
(imported_reference2, namespace),
"F: 10, 0: Unable to import '%s' (import-error)" %
imported_module2_with_version])
self.assertEqual(component.getTextContentWarningMessageList(), [])
## Simulate user:
# 1) First check and validate 'imported' Components
self.portal.portal_workflow.doActionFor(imported_component1, 'validate_action')
self.portal.portal_workflow.doActionFor(imported_component2, 'validate_action')
self.tic()
self.assertEqual(imported_component1.getValidationState(), 'validated')
self.assertEqual(imported_component2.getValidationState(), 'validated')
# 2) Then validate again the main one
self.portal.portal_workflow.doActionFor(component, 'validate_action')
self.tic()
self._assertAstroidCacheContent(
must_be_in_cache_set={'%s' % namespace,
'%s.erp5_version' % namespace,
imported_module1,
imported_module1_with_version,
imported_module2,
imported_module2_with_version},
must_not_be_in_cache_set=set())
self.assertEqual(component.getValidationState(), 'validated')
self.assertEqual(component.getTextContentErrorMessageList(), [])
self.assertEqual(component.getTextContentWarningMessageList(), [])
component.setTextContent(
"""# -*- coding: utf-8 -*-
from %(module)s import undefined
from %(module_with_version)s import undefined
# To avoid 'unused-import' warning...
undefined()
""" % (dict(module=imported_module2,
module_with_version=imported_module2_with_version)) +
component.getTextContent())
self.tic()
self.assertEqual(component.getValidationState(), 'modified')
self.assertEqual(
component.getTextContentErrorMessageList(),
["E: 2, 0: No name 'undefined' in module '%s' (no-name-in-module)" %
imported_module2,
"E: 3, 0: No name 'undefined' in module '%s' (no-name-in-module)" %
imported_module2_with_version])
self.assertEqual(component.getTextContentWarningMessageList(), [])
from Products.ERP5Type.Core.ExtensionComponent import ExtensionComponent
class TestZodbExtensionComponent(_TestZodbComponent):
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment