Define erp5.component.XXX as a packages and allow import of dynamic modules.

This is especially relevant for Document Components where the source code may
perform import of another Components. This is also the first step towards
migration ERP5 Products to ZODB (the missing bit is only the code to import
them from filesystem).

product/ERP5Type/Tool/ComponentTool.py

@@ -30,6 +30,7 @@
 from types import ModuleType
 
 import transaction
+import sys
 
 from AccessControl import ClassSecurityInfo
 from Products.ERP5Type import Permissions
@@ -96,9 +97,13 @@ class ComponentTool(BaseTool):
         else:
           for name, klass in module.__dict__.items():
             if name[0] != '_' and isinstance(klass, ModuleType):
+              full_module_name = "erp5.component.%s.%s" % (module_name, name)
+
               LOG("ERP5Type.Tool.ComponentTool", INFO,
-                  "Resetting erp5.component.%s.%s" % (module_name, name))
+                  "Resetting " + full_module_name)
 
+              # The module must be deleted first
+              del sys.modules[full_module_name]
               delattr(module, name)
 
     type_tool.resetDynamicDocumentsOnceAtTransactionBoundary()

product/ERP5Type/dynamic/component_class.py

@@ -26,24 +26,133 @@
 #
 ##############################################################################
 
+import sys
+
+from Products.ERP5Type.dynamic.dynamic_module import DynamicModule
 from Products.ERP5.ERP5Site import getSite
 from types import ModuleType
 from zLOG import LOG, INFO
 
-def generateComponentClassWrapper(namespace, portal_type):
-  def generateComponentClass(component_name):
+class ComponentDynamicPackage(DynamicModule):
+  """
+  A top-level component is a package as it contains modules, this is required
+  to be able to add import hooks (as described in PEP 302) when a in the
+  source code of a Component, another Component is imported.
+
+  A Component can be loaded in two different ways:
+
+   1/ When erp5.component.extension.XXX is accessed (for example for External
+      Method as per ExternalMethod patch), thus ending up calling __getattr__
+      (DynamicModule) which then load the Component through __call__();
+
+   2/ Upon import, for example in a Document Component with ``import
+      erp5.component.XXX.YYY'', through the Importer Protocol (PEP 302), by
+      adding an instance of this class to sys.meta_path and through
+      find_module() and load_module methods. After that, this is the same as
+      1/.
+
+      This is required because Component classes do not have any physical
+      location on the filesystem, however extra care must be taken for
+      performances because load_module() will be called each time an import is
+      done, therefore the loader should be added to sys.meta_path as late as
+      possible to keep startup time to the minimum.
+  """
+  # Necessary otherwise imports will fail because an object is considered a
+  # package only if __path__ is defined
+  __path__ = []
+
+  def __init__(self, namespace, portal_type):
+    super(ComponentDynamicPackage, self).__init__(namespace, self)
+
+    self._namespace = namespace
+    self._namespace_prefix = namespace + '.'
+    self._portal_type = portal_type
+
+    # Add this module to sys.path for future imports
+    sys.modules[namespace] = self
+
+    # Add the import hook
+    sys.meta_path.append(self)
+
+  def find_module(self, fullname, path=None):
+    # Ignore any absolute imports which does not start with this package
+    # prefix, None there means that "normal" sys.path will be used
+    if not fullname.startswith(self._namespace_prefix):
+      return None
+
+    # __import__ will first try a relative import, for example
+    # erp5.component.XXX.YYY.ZZZ where erp5.component.XXX.YYY is the current
+    # Component where an import is done
+    name = fullname.replace(self._namespace_prefix, '')
+    if '.' in name:
+      return None
+
+    # Skip components not available, otherwise Products for example could be
+    # wrongly considered as importable and thus the actual filesystem class
+    # ignored
+    #
+    # XXX-arnau: This must use reference rather than ID
+    site = getSite()
+    component = getattr(site.portal_components.aq_explicit, fullname, None)
+    if not (component and
+            component.getValidationState() in ('modified', 'validated')):
+      return None
+
+    # XXX-arnau: Using the Catalog should be preferred however it is not
+    # really possible for two reasons: 1/ the Catalog lags behind the ZODB
+    # thus immediately after adding/removing a Component, it will fail to load
+    # a Component because of reindexing 2/ this is unsurprisingly really slow
+    # compared to a ZODB access.
+    #
+    # site = getSite()
+    # found = list(site.portal_catalog.unrestrictedSearchResults(
+    #   reference=name,
+    #   portal_type=self._portal_type,
+    #   parent_uid=site.portal_components.getUid(),
+    #   validation_state=('validated', 'modified')))
+    # if not found:
+    #   return None
+
+    return self
+
+  def load_module(self, fullname):
+    """
+    Load a module with given fullname (see PEP 302)
+    """
+    if not fullname.startswith(self._namespace_prefix):
+      return None
+
+    module = sys.modules.get(fullname, None)
+    if module is not None:
+      return module
+
+    # Load the module by trying to access it
+    name = fullname.replace(self._namespace_prefix, '')
+    try:
+      module = getattr(self, name)
+    except AttributeError, e:
+      return None
+
+    module.__loader__ = self
+    return module
+
+  def __call__(self, component_name):
     site = getSite()
 
     # XXX-arnau: erp5.component.extension.VERSION.REFERENCE perhaps but there
     # should be a a way to specify priorities such as portal_skins maybe?
-    component_id = '%s.%s' % (namespace, component_name)
+    component_id = '%s.%s' % (self._namespace, component_name)
     try:
-      # XXX-arnau: Performances?
+      # XXX-arnau: Performances (~ 200x slower than direct access to ZODB) and
+      # also lag behind the ZODB (e.g. reindexing), so this is certainly not a
+      # good solution
       component = site.portal_catalog.unrestrictedSearchResults(
         parent_uid=site.portal_components.getUid(),
         reference=component_name,
         validation_state=('validated', 'modified'),
-        portal_type=portal_type)[0].getObject()
+        portal_type=self._portal_type)[0].getObject()
+
+      # component = getattr(site.portal_components, component_id)
     except IndexError:
       LOG("ERP5Type.dynamic", INFO,
           "Could not find %s, perhaps it has not been migrated yet?" % \
@@ -53,8 +162,19 @@ def generateComponentClassWrapper(namespace, portal_type):
                              component_id)
     else:
       new_module = ModuleType(component_id, component.getDescription())
+
+      # The module *must* be in sys.modules before executing the code in case
+      # the module code imports (directly or indirectly) itself (see PEP 302)
+      sys.modules[component_id] = new_module
+
+      # This must be set for imports at least (see PEP 302)
+      new_module.__file__ = "<%s>" % component_name
+
+      try:
         component.load(new_module.__dict__, validated_only=True)
-      LOG("ERP5Type.dynamic", INFO, "Loaded successfully %s" % component_id)
-      return new_module
+      except Exception, e:
+        del sys.modules[component_id]
+        raise
 
-  return generateComponentClass
+      new_module.__path__ = []
+      return new_module

product/ERP5Type/dynamic/dynamic_module.py

@@ -122,17 +122,13 @@ def initializeDynamicModules():
                                                 loadTempPortalTypeClass)
 
   # Components
-  from component_class import generateComponentClassWrapper
-
   erp5.component = ModuleType("erp5.component")
   sys.modules["erp5.component"] = erp5.component
 
-  erp5.component.extension = registerDynamicModule(
-    'erp5.component.extension',
-    generateComponentClassWrapper('erp5.component.extension',
-                                  'Extension Component'))
+  from component_class import ComponentDynamicPackage
+
+  erp5.component.extension = ComponentDynamicPackage('erp5.component.extension',
+                                                     'Extension Component')
 
-  erp5.component.document = registerDynamicModule(
-    'erp5.component.document',
-    generateComponentClassWrapper('erp5.component.document',
-                                  'Document Component'))
+  erp5.component.document = ComponentDynamicPackage('erp5.component.document',
+                                                    'Document Component')

product/ERP5Type/dynamic/portal_type_class.py

@@ -86,6 +86,12 @@ core_portal_type_class_dict = {
   'Types Tool':   {'type_class': 'TypesTool',
                    'generating': False},
   'Solver Tool':  {'type_class': 'SolverTool',
+                   'generating': False},
+  # Needed to load Components
+  #
+  # XXX-arnau: only for now as the Catalog is being used (parent_uid
+  # especially), but it will later be replaced anyway...
+  'Category Tool': {'type_class': 'CategoryTool',
                      'generating': False}
   }
 
@@ -186,15 +192,18 @@ def generatePortalTypeClass(site, portal_type_name):
   if '.' in type_class:
     type_class_path = type_class
   else:
-    type_class_path = document_class_registry.get(type_class, None)
-
-    # XXX-arnau: hardcoded but this must be improved anyway when Products will
-    # be in ZODB, for now this should be enough to only care of bt5 Documents
-    if type_class_path is None or type_class_path.startswith('erp5.document'):
+    # Skip any document within ERP5Type Product as it is needed for
+    # bootstrapping anyway
+    type_class_namespace = document_class_registry.get(type_class, '')
+    if not (type_class_namespace.startswith('Products.ERP5Type') or
+            portal_type_name in core_portal_type_class_dict):
       import erp5.component.document
       module = getattr(erp5.component.document, type_class, None)
       klass = module and getattr(module, type_class, None) or None
 
+    if klass is None:
+      type_class_path = document_class_registry.get(type_class, None)
+
     if klass is None and type_class_path is None:
       raise AttributeError('Document class %s has not been registered:'
                            ' cannot import it as base of Portal Type %s'