Optimize Component loading by using a registry, similar to ERP5Type.document_class_registry.

Upon startup, a registry (dict) is initialized for each Component
module. Then, everytime a Component is validated/invalidated, this registry is
updated.

This avoids looking whether a Component exists in ZODB/Catalog and is
validated in Component Tool on every imports or when loading a Component,
instead the registry is used. Moreover, the Catalog was used before as the
Component are matched by their reference and their validation state, thus
making bootstrap tricky.

product/ERP5/bootstrap/erp5_core/WorkflowTemplateItem/portal_workflow/component_validation_workflow/scripts/addToRegistry.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>Script_magic</string> </key>
+            <value> <int>3</int> </value>
+        </item>
+        <item>
+            <key> <string>_bind_names</string> </key>
+            <value>
+              <object>
+                <klass>
+                  <global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
+                </klass>
+                <tuple/>
+                <state>
+                  <dictionary>
+                    <item>
+                        <key> <string>_asgns</string> </key>
+                        <value>
+                          <dictionary>
+                            <item>
+                                <key> <string>name_container</string> </key>
+                                <value> <string>container</string> </value>
+                            </item>
+                            <item>
+                                <key> <string>name_context</string> </key>
+                                <value> <string>context</string> </value>
+                            </item>
+                            <item>
+                                <key> <string>name_m_self</string> </key>
+                                <value> <string>script</string> </value>
+                            </item>
+                            <item>
+                                <key> <string>name_subpath</string> </key>
+                                <value> <string>traverse_subpath</string> </value>
+                            </item>
+                          </dictionary>
+                        </value>
+                    </item>
+                  </dictionary>
+                </state>
+              </object>
+            </value>
+        </item>
+        <item>
+            <key> <string>_body</string> </key>
+            <value> <string>state_change[\'object\'].addToRegistry()\n
+</string> </value>
+        </item>
+        <item>
+            <key> <string>_params</string> </key>
+            <value> <string>state_change</string> </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>addToRegistry</string> </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+</ZopeData>

product/ERP5/bootstrap/erp5_core/WorkflowTemplateItem/portal_workflow/component_validation_workflow/scripts/checkConsistency.xml

@@ -50,8 +50,7 @@
         </item>
         <item>
             <key> <string>_body</string> </key>
-            <value> <string>object = state_change[\'object\']\n
-object.Base_checkConsistency()\n
+            <value> <string>state_change[\'object\'].Base_checkConsistency()\n
 </string> </value>
         </item>
         <item>

product/ERP5/bootstrap/erp5_core/WorkflowTemplateItem/portal_workflow/component_validation_workflow/scripts/deleteFromRegistry.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>Script_magic</string> </key>
+            <value> <int>3</int> </value>
+        </item>
+        <item>
+            <key> <string>_bind_names</string> </key>
+            <value>
+              <object>
+                <klass>
+                  <global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
+                </klass>
+                <tuple/>
+                <state>
+                  <dictionary>
+                    <item>
+                        <key> <string>_asgns</string> </key>
+                        <value>
+                          <dictionary>
+                            <item>
+                                <key> <string>name_container</string> </key>
+                                <value> <string>container</string> </value>
+                            </item>
+                            <item>
+                                <key> <string>name_context</string> </key>
+                                <value> <string>context</string> </value>
+                            </item>
+                            <item>
+                                <key> <string>name_m_self</string> </key>
+                                <value> <string>script</string> </value>
+                            </item>
+                            <item>
+                                <key> <string>name_subpath</string> </key>
+                                <value> <string>traverse_subpath</string> </value>
+                            </item>
+                          </dictionary>
+                        </value>
+                    </item>
+                  </dictionary>
+                </state>
+              </object>
+            </value>
+        </item>
+        <item>
+            <key> <string>_body</string> </key>
+            <value> <string>state_change[\'object\'].deleteFromRegistry()\n
+</string> </value>
+        </item>
+        <item>
+            <key> <string>_params</string> </key>
+            <value> <string>state_change</string> </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>deleteFromRegistry</string> </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+</ZopeData>

product/ERP5/bootstrap/erp5_core/WorkflowTemplateItem/portal_workflow/component_validation_workflow/transitions/invalidate.xml

@@ -10,6 +10,10 @@
             <key> <string>actbox_category</string> </key>
             <value> <string>workflow</string> </value>
         </item>
+        <item>
+            <key> <string>actbox_icon</string> </key>
+            <value> <string></string> </value>
+        </item>
         <item>
             <key> <string>actbox_name</string> </key>
             <value> <string></string> </value>
@@ -42,7 +46,7 @@
         </item>
         <item>
             <key> <string>script_name</string> </key>
-            <value> <string></string> </value>
+            <value> <string>deleteFromRegistry</string> </value>
         </item>
         <item>
             <key> <string>title</string> </key>

product/ERP5/bootstrap/erp5_core/WorkflowTemplateItem/portal_workflow/component_validation_workflow/transitions/validate.xml

@@ -10,6 +10,10 @@
             <key> <string>actbox_category</string> </key>
             <value> <string>workflow</string> </value>
         </item>
+        <item>
+            <key> <string>actbox_icon</string> </key>
+            <value> <string></string> </value>
+        </item>
         <item>
             <key> <string>actbox_name</string> </key>
             <value> <string></string> </value>
@@ -42,7 +46,7 @@
         </item>
         <item>
             <key> <string>script_name</string> </key>
-            <value> <string></string> </value>
+            <value> <string>addToRegistry</string> </value>
         </item>
         <item>
             <key> <string>title</string> </key>

product/ERP5/bootstrap/erp5_core/bt/change_log

+2012-02-11 arnaud.fontaine
+* Update component module registry upon validate/invalidate.
+
 2012-02-07 arnaud.fontaine
 * Use resetOnceAtTransactionBoundary when a Component has been modified.
 

product/ERP5/bootstrap/erp5_core/bt/revision

-41000
+41001
\ No newline at end of file

product/ERP5Type/Core/Component.py

@@ -87,6 +87,29 @@ class Component(Base):
 
     return []
 
+  def addToRegistry(self):
+    """
+    Add the Component to its appropriate module registry
+    """
+    namespace_fullname = self._getDynamicModuleNamespace() 
+    namespace_module = __import__(namespace_fullname, {}, {},
+                                  fromlist=[namespace_fullname])
+
+    reference = self.getReference()
+    namespace_module._registry_dict[reference] = {
+      'component': self,
+      'module_name': "%s.%s" % (namespace_fullname, reference)}
+
+  def deleteFromRegistry(self):
+    """
+    Delete the Component from its appropriate module registry
+    """
+    namespace_fullname = self._getDynamicModuleNamespace() 
+    namespace_module = __import__(namespace_fullname, {}, {},
+                                  fromlist=[namespace_fullname])
+
+    del namespace_module._registry_dict[self.getReference()]
+
   def _setTextContent(self, text_content):
     """
     When the validation state is already 'validated', set the new value to
@@ -236,7 +259,6 @@ class Component(Base):
     exec source_code in namespace_dict
 
     return context.newContent(id=id,
-                              # XXX-arnau: useless field?
                               reference=class_name,
                               text_content=source_code,
                               portal_type=cls.portal_type)

product/ERP5Type/Tool/TypesTool.py

@@ -206,11 +206,8 @@ class TypesTool(TypeProvider):
     from Products.ERP5Type import document_class_registry
     document_type_set = set(document_class_registry)
 
-    # XXX-arnau: should be cached and reference?
-    component_tool = self.getPortalObject().portal_components
-    for obj in component_tool.searchFolder(portal_type='Document Component',
-                                           validation_state='validated'):
-      document_type_set.add(obj.getReference())
+    import erp5.component.document
+    document_type_set.update(erp5.component.document._registry_dict)
 
     return sorted(document_type_set)
 

product/ERP5Type/dynamic/component_class.py

@@ -54,6 +54,7 @@ class ComponentDynamicPackage(ModuleType):
   # Necessary otherwise imports will fail because an object is considered a
   # package only if __path__ is defined
   __path__ = []
+  __registry_dict = None
 
   def __init__(self, namespace, portal_type):
     super(ComponentDynamicPackage, self).__init__(namespace)
@@ -70,6 +71,43 @@ class ComponentDynamicPackage(ModuleType):
     # Add the import hook
     sys.meta_path.append(self)
 
+  @property
+  def _registry_dict(self):
+    """
+    Create the component registry, this is very similar to
+    Products.ERP5Type.document_class_registry and avoids checking whether a
+    Component exists at each call at the expense to increase startup
+    time. Moreover, it allows to handle reference easily.
+
+    XXX-arnau: handle different versions of a Component, perhaps something
+    like erp5.component.extension.VERSION.REFERENCE perhaps but there should
+    be a a way to specify priorities such as portal_skins maybe?
+    """
+    if self.__registry_dict is None:
+      try:
+        component_tool = getSite().portal_components
+      # XXX-arnau: When installing ERP5 site, erp5_core_components has not
+      # been installed yet, thus this will obviously failed...
+      except AttributeError:
+        return {}
+
+      self.__registry_dict = {}
+      # XXX-arnau: contentValues should not be used as there may be a large
+      # number of objects, but as this is done only once, that should perhaps
+      # not be a problem after all, and using the Catalog is too risky as it
+      # lags behind and depends upon objects being reindexed
+      for component in component_tool.contentValues(portal_type=self._portal_type):
+        # Only consider modified or validated states as state transition will
+        # be handled by component_validation_workflow which will take care of
+        # updating the registry
+        if component.getValidationState() in ('modified', 'validated'):
+          reference = component.getReference()
+          self.__registry_dict[reference] = {
+            'component': component,
+            'module_name': self._namespace_prefix + reference}
+
+    return self.__registry_dict
+
   def find_module(self, fullname, path=None):
     # Ignore any absolute imports which does not start with this package
     # prefix, None there means that "normal" sys.path will be used
@@ -86,29 +124,9 @@ class ComponentDynamicPackage(ModuleType):
     # Skip components not available, otherwise Products for example could be
     # wrongly considered as importable and thus the actual filesystem class
     # ignored
-    #
-    # XXX-arnau: This must use reference rather than ID
-    site = getSite()
-    component = getattr(site.portal_components.aq_explicit, fullname, None)
-    if not (component and
-            component.getValidationState() in ('modified', 'validated')):
+    if name not in self._registry_dict:
       return None
 
-    # XXX-arnau: Using the Catalog should be preferred however it is not
-    # really possible for two reasons: 1/ the Catalog lags behind the ZODB
-    # thus immediately after adding/removing a Component, it will fail to load
-    # a Component because of reindexing 2/ this is unsurprisingly really slow
-    # compared to a ZODB access.
-    #
-    # site = getSite()
-    # found = list(site.portal_catalog.unrestrictedSearchResults(
-    #   reference=name,
-    #   portal_type=self._portal_type,
-    #   parent_uid=site.portal_components.getUid(),
-    #   validation_state=('validated', 'modified')))
-    # if not found:
-    #   return None
-
     return self
 
   def load_module(self, fullname):
@@ -124,22 +142,11 @@ class ComponentDynamicPackage(ModuleType):
 
     site = getSite()
 
-    # XXX-arnau: erp5.component.extension.VERSION.REFERENCE perhaps but there
-    # should be a a way to specify priorities such as portal_skins maybe?
     component_name = fullname.replace(self._namespace_prefix, '')
     component_id = '%s.%s' % (self._namespace, component_name)
     try:
-      # XXX-arnau: Performances (~ 200x slower than direct access to ZODB) and
-      # also lag behind the ZODB (e.g. reindexing), so this is certainly not a
-      # good solution
-      component = site.portal_catalog.unrestrictedSearchResults(
-        parent_uid=site.portal_components.getUid(),
-        reference=component_name,
-        validation_state=('validated', 'modified'),
-        portal_type=self._portal_type)[0].getObject()
-
-      # component = getattr(site.portal_components, component_id)
-    except IndexError:
+      component = self._registry_dict[component_name]['component']
+    except KeyError:
       LOG("ERP5Type.dynamic", INFO,
           "Could not find %s or it has not been validated or it has not been "
           "migrated yet?" % component_id)

product/ERP5Type/dynamic/portal_type_class.py

@@ -42,15 +42,16 @@ from Products.ERP5Type.TransactionalVariable import TransactionalResource
 
 from zLOG import LOG, ERROR, INFO, WARNING, PANIC
 
-def _importClass(classpath):
+def _importClass(classpath, is_zodb_document=False):
   try:
     module_path, class_name = classpath.rsplit('.', 1)
     module = __import__(module_path, {}, {}, (module_path,))
     klass = getattr(module, class_name)
 
-    # XXX is this required? (here?)
-    setDefaultClassProperties(klass)
-    InitializeClass(klass)
+    if not is_zodb_document:
+      # XXX is this required? (here?)
+      setDefaultClassProperties(klass)
+      InitializeClass(klass)
 
     return klass
   except StandardError:
@@ -87,12 +88,6 @@ core_portal_type_class_dict = {
                    'generating': False},
   'Solver Tool':  {'type_class': 'SolverTool',
                    'generating': False},
-  # Needed to load Components
-  #
-  # XXX-arnau: only for now as the Catalog is being used (parent_uid
-  # especially), but it will later be replaced anyway...
-  'Category Tool': {'type_class': 'CategoryTool',
-                     'generating': False}
   }
 
 def generatePortalTypeClass(site, portal_type_name):
@@ -188,34 +183,44 @@ def generatePortalTypeClass(site, portal_type_name):
     raise AttributeError('Document class is not defined on Portal Type %s' \
             % portal_type_name)
 
+  is_zodb_document = False
   klass = None
   if '.' in type_class:
     type_class_path = type_class
   else:
+    type_class_path = None
+
     # Skip any document within ERP5Type Product as it is needed for
     # bootstrapping anyway
     type_class_namespace = document_class_registry.get(type_class, '')
     if not (type_class_namespace.startswith('Products.ERP5Type') or
             portal_type_name in core_portal_type_class_dict):
-      try:
-        klass = getattr(__import__('erp5.component.document.' + type_class,
-                                   fromlist=['erp5.component.document'],
-                                   level=0),
-                        type_class)
-
-      except (ImportError, AttributeError):
-        pass
-
-    if klass is None:
+      import erp5.component.document
+      module_info_dict = erp5.component.document._registry_dict.get(type_class,
+                                                                    None)
+      if module_info_dict:
+        type_class_path = "%s.%s" % (module_info_dict['module_name'], type_class)
+        is_zodb_document = True
+
+    if type_class_path is None:
       type_class_path = document_class_registry.get(type_class, None)
+      if type_class_path is None:
+        raise AttributeError('Document class %s has not been registered:'
+                             ' cannot import it as base of Portal Type %s'
+                             % (type_class, portal_type_name))
 
-    if klass is None and type_class_path is None:
-      raise AttributeError('Document class %s has not been registered:'
-                           ' cannot import it as base of Portal Type %s'
-                           % (type_class, portal_type_name))
+  try:
+    klass = _importClass(type_class_path, is_zodb_document)
+  except ImportError:
+    # A Document Component should always have a class matching its reference,
+    # so this should never happen...
+    if is_zodb_document:
+      type_class_path = document_class_registry.get(type_class, None)
+      if type_class_path is not None:
+        klass = _importClass(type_class_path)
 
-  if klass is None:
-    klass = _importClass(type_class_path)
+    if klass is None:
+      raise
 
   global property_sheet_generating_portal_type_set