ZODB Components: Products Documents for a given bt5 can now be migrated from filesystem.

Until now, only bt5 Extension/Test/Document could be migrated from
filesystem.

Determine which Products Documents are to be migrated by looking at the mro()
of the bt5 Portal Types classes. For now, this only considers
Products.ERP5.Document.* but this may be extended later on.

XXX-BEFORE-MERGE: Add Unit Test.

product/ERP5/Document/BusinessTemplate.py

@@ -6293,6 +6293,31 @@ Business Template is a set of definitions, such as skins, portal types and categ
       setattr(self, 'template_portal_type_base_category', ())
       return
 
+    @staticmethod
+    def __getAllFilesystemModuleFromPortalTypeIdList(portal_type_id_list):
+      import erp5.portal_type
+      import inspect
+
+      seen_cls_set = set()
+      for portal_type in portal_type_id_list:
+        portal_type_cls = getattr(erp5.portal_type, portal_type)
+        # Calling mro() would not load the class...
+        portal_type_cls.loadClass()
+
+        for cls in portal_type_cls.mro():
+          if cls in seen_cls_set:
+            continue
+
+          seen_cls_set.add(cls)
+          try:
+            cls_path = inspect.getfile(cls)
+          except TypeError:
+            pass
+          else:
+            cls_name = cls.__name__
+            cls_module = cls.__module__
+            yield cls_name, cls_module, cls_path
+
     security.declareProtected(Permissions.ManagePortal,
                               'getMigratableSourceCodeFromFilesystemList')
     def getMigratableSourceCodeFromFilesystemList(self, *args, **kwargs):
@@ -6318,6 +6343,28 @@ Business Template is a set of definitions, such as skins, portal types and categ
 
             migratable_component_list.append(obj)
 
+      # Inspect Portal Types classes mro() of this Business Template to find
+      # Products Documents to migrate
+      import Products.ERP5Type
+      import inspect
+      product_base_path = inspect.getfile(Products.ERP5Type).rsplit('/', 2)[0]
+      import erp5.component.document
+      for cls_name, cls_module, cls_path in self.__getAllFilesystemModuleFromPortalTypeIdList(self.getTemplatePortalTypeIdList()):
+        if (cls_path.startswith(product_base_path) and
+            # XXX: Only migrate Documents in ERP5 for the moment...
+            cls_module.startswith('Products.ERP5.Document') and
+            erp5.component.document.find_load_module(cls_name) is None):
+          existing_component = getattr(component_tool, cls_name, None)
+          if (existing_component is None or
+              existing_component.getPortalType() != 'Document Component'):
+            obj = component_tool.newContent(id="tmp_source_code_migration_%s" % cls_name,
+                                            portal_type='Document Component',
+                                            reference=cls_name,
+                                            source_reference=cls_module,
+                                            temp_object=1)
+
+            migratable_component_list.append(obj)
+
       return sorted(migratable_component_list, key=lambda o: o.getReference())
 
     security.declareProtected(Permissions.ManagePortal,
@@ -6334,16 +6381,19 @@ Business Template is a set of definitions, such as skins, portal types and categ
       """
       component_tool = self.getPortalObject().portal_components
       failed_import_dict = {}
+      migrated_product_module_set = set()
 
       template_document_id_list = self.getTemplateDocumentIdList()
       template_extension_id_list = self.getTemplateExtensionIdList()
       template_test_id_list = self.getTemplateTestIdList()
 
       for temp_obj in self.getMigratableSourceCodeFromFilesystemList():
+        source_reference = temp_obj.getSourceReference()
         try:
           obj = temp_obj.importFromFilesystem(component_tool,
                                               temp_obj.getReference(),
                                               version,
+                                              source_reference,
                                               erase_existing=erase_existing)
         except Exception, e:
           failed_import_dict[temp_obj.getReference()] = str(e)
@@ -6356,6 +6406,8 @@ Business Template is a set of definitions, such as skins, portal types and categ
           # 'Document Component'
           else:
             template_document_id_list.append(obj.getId())
+            if source_reference:
+              migrated_product_module_set.add(source_reference)
 
       if failed_import_dict:
         message = (
@@ -6374,6 +6426,27 @@ Business Template is a set of definitions, such as skins, portal types and categ
       self.setTemplateExtensionIdList(sorted(template_extension_id_list))
       self.setTemplateTestIdList(sorted(template_test_id_list))
 
+      # This will trigger a reset so that Portal Types mro() can be checked
+      # after migration for filesystem Products modules still being used
+      import transaction
+      transaction.commit()
+
+      still_used_list_dict = {}
+      for _, cls_module, _ in self.__getAllFilesystemModuleFromPortalTypeIdList(self.getPortalObject().portal_types.objectIds()):
+        if cls_module in migrated_product_module_set:
+          package, module = cls_module.rsplit('.', 1)
+          still_used_list_dict.setdefault(package, []).append(module)
+
+      if still_used_list_dict:
+        module_still_used_message = ', '.join(
+            [ "%s.{%s}" % (package, ','.join(sorted(module_list)))
+              for package, module_list in still_used_list_dict.iteritems() ])
+
+        LOG('BusinessTemplate',
+            WARNING,
+            "The following Documents are still being imported so code need to "
+            "be updated: " + module_still_used_message)
+
       if REQUEST is not None:
         message = (
           "All components were successfully imported from filesystem to ZODB. "

product/ERP5/bootstrap/erp5_core/ActionTemplateItem/portal_types/Business%20Template/migrate_component_from_filesystem.xml

@@ -92,7 +92,7 @@
       <dictionary>
         <item>
             <key> <string>text</string> </key>
-            <value> <string>python: portal.Base_checkPermission(\'portal_components\', \'Add portal content\') and context.getInstallationState() != \'installed\' and [id_ for id_ in (context.getTemplateExtensionIdList() + context.getTemplateDocumentIdList() + context.getTemplateTestIdList()) if not id_.startswith(\'extension.\') and not id_.startswith(\'document.\') and not id_.startswith(\'test.\') ]</string> </value>
+            <value> <string>python: portal.Base_checkPermission(\'portal_components\', \'Add portal content\') and context.getInstallationState() != \'installed\'</string> </value>
         </item>
       </dictionary>
     </pickle>

product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/BusinessTemplate_viewMigrateSourceCodeFromFilesystemDialog/listbox.xml

@@ -404,6 +404,8 @@
                           <string>Name</string>
                         </tuple>
                         <tuple>
+                          <string>source_reference</string>
+                          <string>Module</string>
                         </tuple>
                         <tuple>
                           <string>portal_type</string>

product/ERP5Type/interfaces/component.py

@@ -68,7 +68,7 @@ class IComponent(Interface):
     Return the ID prefix for Component objects
     """
 
-  def importFromFilesystem(cls, context, reference, version,
+  def importFromFilesystem(cls, context, reference, module, version,
                            erase_existing=False):
     """
     Import a Component from the filesystem into ZODB after checking that the

product/ERP5Type/mixin/component.py

@@ -346,7 +346,7 @@ class ComponentMixin(PropertyRecordableMixin, Base):
                             'importFromFilesystem')
   @classmethod
   def importFromFilesystem(cls, context, reference, version,
-                           erase_existing=False):
+                           module=None, erase_existing=False):
     """
     Import a Component from the filesystem into ZODB and validate it so it can
     be loaded straightaway provided validate() does not raise any error of
@@ -365,7 +365,13 @@ class ComponentMixin(PropertyRecordableMixin, Base):
       context.deleteContent(object_id)
 
     import os.path
-    path = os.path.join(cls._getFilesystemPath(), reference + '.py')
+    if module is None:
+      path = os.path.join(cls._getFilesystemPath(), reference + '.py')
+    else:
+      import sys
+      import inspect
+      path = inspect.getsourcefile(sys.modules[module])
+
     with open(path) as f:
       source_code = f.read()
 
@@ -375,6 +381,7 @@ class ComponentMixin(PropertyRecordableMixin, Base):
     # one depending upon the former...
     new_component = context.newContent(id=object_id,
                                        reference=reference,
+                                       source_reference=module,
                                        version=version,
                                        text_content=source_code,
                                        portal_type=cls.portal_type)