CategoryTool_importCategoryFile: Refactor.

Factorise code paths between simulation and "real" modes,
fixing occasional non-representativeness of simulation mode.
Factorise code (down from 300 to 200 lines).
Simplify code (ex: use set instead of None-value dict).
Merge consecutive loops:
- delete/expire immediately instead of building a list to process later
- process each Base Category entirely (create, edit, delete/expire/keep)
before going to the next
This merging should overall reduce memory usage significantly.

product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/CategoryTool_importCategoryFile.xml

@@ -52,43 +52,33 @@
             <key> <string>_body</string> </key>
             <value> <string>from Products.ERP5Type.Message import translateString\n
 from Products.ERP5Type.Document import newTempBase\n
-from zExceptions import Redirect\n
-portal = context.getPortalObject()\n
 \n
-# Initialise some general variables\n
+# XXX: allow simulation_mode without detailed_report ?\n
+detailed_report |= simulation_mode\n
+\n
+portal = context.getPortalObject()\n
+REQUEST = portal.REQUEST\n
+base_category_property_id_set = portal.portal_types[\'Base Category\'].getInstancePropertySet()\n
+category_property_id_set = portal.portal_types.Category.getInstancePropertySet()\n
+portal_categories = portal.portal_categories\n
+resolveCategory = portal_categories.resolveCategory\n
+getRelatedValueList = portal_categories.getRelatedValueList\n
+isTransitionPossible = portal.portal_workflow.isTransitionPossible\n
 detailed_report_result = []\n
 detailed_report_append = detailed_report_result.append\n
-base_category_id_list = []\n
-category_path_dict = {}\n
-simulation_new_category_id_list = []\n
-\n
-base_category_property_id_set = getattr(context.portal_types, \'Base Category\').getInstancePropertySet()\n
-category_property_id_set = context.portal_types.Category.getInstancePropertySet()\n
-\n
-getRelatedValueList = portal.portal_categories.getRelatedValueList\n
-def Object_hasRelation(obj):\n
-  # Check if there is some related objets.\n
-  result = 0\n
-  for o in obj.getIndexableChildValueList():\n
-    for related in getRelatedValueList(o):\n
-      related_url = related.getRelativeUrl()\n
-      if related_url.startswith(obj.getRelativeUrl()):\n
-        continue\n
-      elif related_url.startswith(\'portal_trash\'):\n
-        continue\n
-      else:\n
-        result = 1\n
-        break\n
-  return result\n
-\n
-\n
-def isValidID(id, is_base_category):\n
-  if is_base_category:\n
-    return id not in base_category_property_id_set\n
-  else:\n
-    return id not in category_property_id_set\n
-\n
-# Some statistics\n
+def report(field_type, message, mapping=None, field_category=\'\', level=None):\n
+  if level and level not in displayed_report:\n
+    return\n
+  detailed_report_append(newTempBase(\n
+    folder=context,\n
+    id=\'item\',\n
+    field_type=field_type,\n
+    field_category=field_category,\n
+    field_message=translateString(\n
+      message,\n
+      mapping=mapping,\n
+    ),\n
+  ))\n
 new_category_counter = 0\n
 updated_category_counter = 0\n
 total_category_counter = 0\n
@@ -97,258 +87,188 @@ deleted_category_counter = 0\n
 kept_category_counter = 0\n
 expired_category_counter = 0\n
 \n
-filename = getattr(import_file, \'filename\', \'?\')\n
-\n
-# The list of error from simulation mode\n
-error_list = []\n
-if simulation_mode:\n
-  def invalid_category_spreadsheet_handler(message):\n
-    error = newTempBase(context, \'item\')\n
-    error.edit(field_type=\'Error\',\n
-               field_message=message)\n
-    error_list.append(error)\n
-    return True # means continue processing the rest of the document\n
-else:\n
-  def invalid_category_spreadsheet_handler(message):\n
-    # action taken when an invalid spreadsheet is provided.\n
-    # we *raise* a Redirect, because we don\'t want the transaction to succeed\n
-    # note, we could make a dialog parameter to allow import invalid spreadsheet:\n
-    raise Redirect(\'%s/view?portal_status_message=%s\' % (\n
-                           context.portal_categories.absolute_url(),\n
-                           message))\n
-\n
-category_list_spreadsheet_mapping = context.Base_getCategoriesSpreadSheetMapping(import_file,\n
-                              invalid_spreadsheet_error_handler=invalid_category_spreadsheet_handler)\n
+def hasRelation(obj):\n
+  # Tests if there is any sensible related objet.\n
+  for o in obj.getIndexableChildValueList():\n
+    for related in getRelatedValueList(o):\n
+      related_url = related.getRelativeUrl()\n
+      if not related_url.startswith(obj.getRelativeUrl()) and not related_url.startswith(\'portal_trash\'):\n
+        return True\n
+  return False\n
 \n
-if simulation_mode and error_list:\n
-  context.REQUEST.other[\'category_import_report\'] = error_list\n
-  return context.CategoryTool_viewImportReport()\n
+def invalid_category_spreadsheet_handler(message):\n
+  report(\n
+    field_type=\'Error\',\n
+    message=message,\n
+  )\n
+  return True\n
+category_list_spreadsheet_dict = context.Base_getCategoriesSpreadSheetMapping(\n
+  import_file,\n
+  invalid_spreadsheet_error_handler=invalid_category_spreadsheet_handler,\n
+)\n
+if detailed_report_result:\n
+  REQUEST.RESPONSE.write(portal_categories.CategoryTool_viewImportReport())\n
+  raise Exception(\'Spreadsheet contains errors\')\n
 \n
-for base_category, category_list in \\\n
-            category_list_spreadsheet_mapping.items():\n
-  base_category_id_list.append(base_category)\n
-  # Create categories\n
+for base_category, category_list in category_list_spreadsheet_dict.iteritems():\n
   total_category_counter += len(category_list)\n
+  category_path_set = set()\n
   for category in category_list:\n
     is_new_category = False\n
-    key_list = category.keys()\n
-    if \'path\' in key_list:\n
-      base_path_obj = context.portal_categories\n
-      category_id_list = category[\'path\'].split(\'/\')\n
-      parent_path = category[\'path\'].split(\'/\')[0]+\'/\'\n
+    category_path = category.pop(\'path\')\n
+    category.pop(\'id\', None)\n
+    try:\n
+      container_path, category_id = category_path.rsplit(\'/\', 1)\n
+    except ValueError:\n
+      category_id = category_path\n
+      container = portal_categories\n
       is_base_category = True\n
-      is_valid_category = True\n
-      for category_id in category_id_list:\n
-        category_id = str(category_id)\n
-        # The current category is not existing\n
-        if not base_path_obj.hasContent(category_id) and (not {\'category\':category_id,\'path\':parent_path} in simulation_new_category_id_list):\n
-          # Create the category\n
-          if is_base_category:\n
-            category_type = \'Base Category\'\n
-          else:\n
-            category_type = \'Category\'\n
-          if isValidID(category_id, is_base_category):\n
-            if simulation_mode:\n
-              simulation_new_category_id_list.append({\'category\':category_id,\'path\':parent_path})\n
-              if \'created\' in displayed_report:\n
-                report_line = newTempBase(context, \'item\')\n
-                report_line.edit(field_type = \'Creation\', field_category = category[\'path\'].split(category_id)[0]+category_id,  field_message = translateString("Will be created new ${type}", mapping=dict(type=category_type)))\n
-                detailed_report_append(report_line)\n
-            else:\n
-              base_path_obj.newContent( portal_type       = category_type\n
-                                    , id                = category_id\n
-                                    , effective_date = effective_date\n
-                                    )\n
-              if \'created\' in displayed_report:\n
-                report_line = newTempBase(context, \'item\')\n
-                report_line.edit(field_type = \'Creation\', field_category = base_path_obj[category_id].getRelativeUrl(), field_message = translateString("Created new ${type}", mapping=dict(type=category_type)))\n
-                detailed_report_append(report_line)\n
-          else:\n
-            # The ID is invalid, we must break the loop\n
-            invalid_category_id_counter += 1\n
-            is_valid_category = False\n
-            if \'warning\' in displayed_report:\n
-              report_line = newTempBase(context, \'item\')\n
-              report_line.edit(field_type = \'WARNING\', field_category = \'\', field_message = translateString("found invalid ID ${id} ", mapping=dict(id=category_id)))\n
-              detailed_report_append(report_line)\n
-            break\n
-          is_new_category = True\n
-          new_category_counter += 1\n
-        if {\'category\':category_id,\'path\':parent_path} not in simulation_new_category_id_list:\n
-          base_path_obj = base_path_obj[category_id]\n
-          category_path_dict[base_path_obj.getRelativeUrl()] = None\n
-        is_base_category = False\n
-        parent_path = category[\'path\'].split(category_id)[0] +category_id\n
-\n
-      property_id_list = base_path_obj.propertyIds() # XXX could be cached\n
-      if is_valid_category:\n
-        # Only try to update a valid category\n
-        new_category = base_path_obj\n
-        # Set the category properties\n
-        category_update_dict = {}\n
-        first_update_reported = True\n
-        for key in key_list:\n
-          key = str(key)\n
-          if key not in [\'path\', \'id\']:\n
-            value = category[key]\n
-\n
-            if not create_local_property and key not in property_id_list:\n
-              # if we do not create local properties, then we skip properties\n
-              # that are not in category ids\n
-              report_line = newTempBase(context, \'item\')\n
-              report_line.edit(field_type=\'Update\',\n
-                  field_category=new_category.getRelativeUrl(),\n
-                  field_message=translateString("Ignoring local property ${key} with value ${value}",\n
-                                                mapping=dict(key=key, value=value)))\n
-              detailed_report_append(report_line)\n
-              continue\n
-\n
-            if is_new_category:\n
-              # Always update properties if this a new category\n
-              category_update_dict[key] = value\n
-            elif update_existing_property:\n
-              # Update if update existing property and the property is not the same\n
-              if new_category.hasProperty(key):\n
-                if str(new_category.getProperty(key)) != value :\n
-                  category_update_dict[key] = value\n
-                  if not is_new_category and \'updated\' in displayed_report:\n
-                    if first_update_reported:\n
-                      field_type = \'Update\'\n
-                      field_category = new_category.getRelativeUrl()\n
-                      first_update_reported = False\n
-                    else:\n
-                      field_type = \'\'\n
-                      field_category = \'\'\n
-                    report_line = newTempBase(context, \'item\')\n
-                    report_line.edit(field_type=field_type, field_category=field_category, field_message=translateString("Updated ${key} with value ${value} ", mapping=dict(key=key, value=value)))\n
-                    detailed_report_append(report_line)\n
-              else:\n
-                if value not in (\'\', None) :\n
-                  category_update_dict[key] = value\n
-                  if not is_new_category and \'updated\' in displayed_report:\n
-                    if first_update_reported:\n
-                      field_type = \'Update\'\n
-                      field_category = new_category.getRelativeUrl()\n
-                      first_update_reported = False\n
-                    else:\n
-                      field_type = \'\'\n
-                      field_category = \'\'\n
-                    report_line = newTempBase(context, \'item\')\n
-                    report_line.edit(field_type=field_type, field_category=field_category, field_message=translateString("Updated ${key} with value ${value} ", mapping=dict(key=key, value=value)))\n
-                    detailed_report_append(report_line)\n
-            elif value not in (\'\', None) and not new_category.hasProperty(key):\n
-              # Only set properties which are not already defined\n
-              category_update_dict[key] = value\n
-              if \'updated\' in displayed_report:\n
-                if first_update_reported:\n
-                  field_type = \'Update\'\n
-                  field_category = new_category.getRelativeUrl()\n
-                  first_update_reported = False\n
-                else:\n
-                  field_type = \'\'\n
-                  field_category = \'\'\n
-                report_line = newTempBase(context, \'item\')\n
-                report_line.edit(field_type=field_type, field_category=field_category, field_message=translateString("Updated ${key} with value ${value} ", mapping=dict(key=key, value=value)))\n
-                detailed_report_append(report_line)\n
-        if not is_new_category and category_update_dict:\n
-          updated_category_counter += 1\n
-\n
-        if not simulation_mode:\n
-          # force_update=1 is required here because\n
-          # edit(short_title=\'foo\', title=\'foo\') only stores short_title property.\n
-          new_category.edit(force_update=1, **category_update_dict)\n
+      category_type = \'Base Category\'\n
+      category_type_property_id_set = base_category_property_id_set\n
     else:\n
-      raise KeyError, \'path was not defined for a category, this should never happen.\'\n
-\n
-# Find categories to delete\n
-category_to_delete_list = []\n
-for base_category_id in base_category_id_list:\n
-  base_category = context.portal_categories[base_category_id]\n
-  for category in base_category.getCategoryChildValueList(is_self_excluded=False):\n
-    if not category_path_dict.has_key(category.getRelativeUrl()):\n
-      if existing_category_list == \'keep\' or Object_hasRelation(category):\n
-        if Object_hasRelation(category):\n
-          # TODO: add a dialog parameter allowing to delete this path\n
-          if \'warning\' in displayed_report:\n
-            report_line = newTempBase(context, \'item\')\n
-            report_line.edit(field_type=\'Warning\',\n
-              field_category=category.getRelativeUrl(),\n
-              field_message=translateString("Category is used and can not be deleted or expired "))\n
-            detailed_report_append(report_line)\n
-        else:\n
-          if existing_category_list == \'keep\' and \'kept\' in displayed_report:\n
-            report_line = newTempBase(context, \'item\')\n
-            report_line.edit(field_type=\'Keep\',\n
-              field_category=category.getRelativeUrl(),\n
-              field_message=translateString("Kept category"))\n
-            detailed_report_append(report_line)\n
-        kept_category_counter += 1\n
-      else:\n
-        if existing_category_list == \'delete\' and \'deleted\' in displayed_report:\n
-          report_line = newTempBase(context, \'item\')\n
-          report_line.edit(field_type=\'Delete\',\n
-            field_category=category.getRelativeUrl(),\n
-            field_message=translateString("Deleted category"))\n
-          detailed_report_append(report_line)\n
-          deleted_category_counter += 1\n
-        if existing_category_list == \'expire\' and \'expired\' in displayed_report:\n
-          report_line = newTempBase(context, \'item\')\n
-          report_line.edit(field_type=\'Expire\',\n
-            field_category=category.getRelativeUrl(),\n
-            field_message=translateString("Expired category"))\n
-          detailed_report_append(report_line)\n
-          expired_category_counter += 1\n
-        category_to_delete_list.append(category.getRelativeUrl())\n
-\n
-if not simulation_mode:\n
-  # Delete unused categories\n
-  if existing_category_list == \'delete\':\n
-    for category in category_to_delete_list:\n
-      category = context.portal_categories.resolveCategory(category)\n
-      if category is not None:\n
-        parent = category.getParentValue()\n
-        category_id = category.getId()\n
-        parent.deleteContent(category_id)\n
-  if existing_category_list == \'expire\':\n
-    if expiration_date:\n
-      for category in category_to_delete_list:\n
-        category = context.portal_categories.resolveCategory(category)\n
-        if category is not None:\n
-          category.edit(**{\'expiration_date\':expiration_date})\n
-    else:\n
-      isTransitionPossible = context.getPortalObject().portal_workflow \\\n
-          .isTransitionPossible\n
-      for category in category_to_delete_list:\n
-        category = context.portal_categories.resolveCategory(category)\n
-        if category is not None \\\n
-            and isTransitionPossible(category, \'expire\'):\n
-          category.expire()\n
-\n
+      container = resolveCategory(container_path)\n
+      is_base_category = False\n
+      category_type = \'Category\'\n
+      category_type_property_id_set = category_property_id_set\n
+    try:\n
+      category_value = container[category_id]\n
+    except KeyError:\n
+      if category_id in category_type_property_id_set:\n
+        report(\n
+          level=\'warning\',\n
+          field_type=\'WARNING\',\n
+          message="found invalid ID ${id} ",\n
+          mapping={\'id\':category_id},\n
+        )\n
+        invalid_category_id_counter += 1\n
+        continue\n
+      new_category_counter += 1\n
+      category_value = container.newContent(\n
+        portal_type=category_type,\n
+        id=category_id,\n
+        effective_date=effective_date,\n
+      )\n
+      report(\n
+        level=\'created\',\n
+        field_type=\'Creation\',\n
+        field_category=category_value.getRelativeUrl(),\n
+        message="Created new ${type}",\n
+        mapping={\'type\': category_type},\n
+      )\n
+      is_new_category = True\n
+    category_path_set.add(category_value.getRelativeUrl())\n
 \n
+    category_update_dict = {}\n
+    for key, value in category.iteritems():\n
+      if not create_local_property and key not in category_type_property_id_set:\n
+        report(\n
+          field_type=\'Update\',\n
+          field_category=category_value.getRelativeUrl(),\n
+          message="Ignoring local property ${key} with value ${value}",\n
+          mapping={\'key\': key, \'value\': value},\n
+        )\n
+      elif is_new_category or (\n
+            value not in (\'\', None) and\n
+            not category_value.hasProperty(key)\n
+          ) or (\n
+            update_existing_property and\n
+            str(category_value.getProperty(key)) != value\n
+          ):\n
+        category_update_dict[key] = value\n
+        if not is_new_category:\n
+          report(\n
+            level=\'updated\',\n
+            field_type=\'Update\',\n
+            field_category=category_value.getRelativeUrl(),\n
+            message="Updated ${key} with value ${value} ",\n
+            mapping={\'key\': key, \'value\': value},\n
+          )\n
+    if category_update_dict:\n
+      if not is_new_category:\n
+        updated_category_counter += 1\n
+      # force_update=1 is required here because\n
+      # edit(short_title=\'foo\', title=\'foo\') only stores short_title property.\n
+      category_value.edit(force_update=1, **category_update_dict)\n
 \n
-  context.portal_caches.clearAllCache()\n
+  to_do_list = [portal_categories[base_category]]\n
+  while to_do_list:\n
+    category = to_do_list.pop()\n
+    recurse = True\n
+    if category.getRelativeUrl() in category_path_set:\n
+      pass\n
+    elif existing_category_list == \'keep\':\n
+      report(\n
+        level=\'kept\',\n
+        field_type=\'Keep\',\n
+        field_category=category.getRelativeUrl(),\n
+        message="Kept category",\n
+      )\n
+      kept_category_counter += 1\n
+    elif hasRelation(category):\n
+      # TODO: add a dialog parameter allowing to delete this path\n
+      report(\n
+        level=\'warning\',\n
+        field_type=\'Warning\',\n
+        field_category=category.getRelativeUrl(),\n
+        message="Category is used and can not be deleted or expired ",\n
+      )\n
+    elif existing_category_list == \'delete\':\n
+      recurse = False\n
+      deleted_category_counter += 1\n
+      report(\n
+        level=\'deleted\',\n
+        field_type=\'Delete\',\n
+        field_category=category.getRelativeUrl(),\n
+        message="Deleted category",\n
+      )\n
+      category.getParentValue().deleteContent(category.getId())\n
+    elif existing_category_list == \'expire\':\n
+      report(\n
+        level=\'expired\',\n
+        field_type=\'Expire\',\n
+        field_category=category.getRelativeUrl(),\n
+        message="Expired category",\n
+      )\n
+      if expiration_date:\n
+        expired_category_counter += 1\n
+        category.edit(expiration_date=expiration_date)\n
+      elif isTransitionPossible(category, \'expire\'):\n
+        expired_category_counter += 1\n
+        category.expire()\n
+      # Report failure otherwise ?\n
+    # Report failure on unexpected value ?\n
+    if recurse:\n
+      to_do_list.extend(category.objectValues())\n
 \n
-if (detailed_report or simulation_mode) and detailed_report_result:\n
-  # Return a detailed report if requested\n
-  context.REQUEST.other[\'category_import_report\'] = detailed_report_result\n
-  return context.CategoryTool_viewImportReport()\n
+portal.portal_caches.clearAllCache()\n
 \n
-# Import is a success, go back to the portal_categories tool\n
-return context.REQUEST.RESPONSE.redirect(\n
-    # TODO translate\n
-  "%s/view?portal_status_message=%s+categories+found+in+%s:+%s+created,+%s+updated,+%s+untouched,+%s+invalid+ID.+%s+existing+categories:+%s+deleted,+%s+expired,+%s+kept."\n
-  % ( context.portal_categories.absolute_url()\n
-    , total_category_counter\n
-    , filename\n
-    , new_category_counter\n
-    , updated_category_counter\n
-    , total_category_counter - new_category_counter - updated_category_counter\n
-    , invalid_category_id_counter\n
-    , deleted_category_counter + kept_category_counter + expired_category_counter\n
-    , deleted_category_counter\n
-    , expired_category_counter\n
-    , kept_category_counter\n
-    )\n
+# TODO: translate\n
+portal_status_message = \'%s categories found in %s: %s created, %s updated, %s untouched, %s invalid ID. %s existing categories: %s deleted, %s expired, %s kept.%s\' % (\n
+  total_category_counter,\n
+  getattr(import_file, \'filename\', \'?\'),\n
+  new_category_counter,\n
+  updated_category_counter,\n
+  total_category_counter - new_category_counter - updated_category_counter,\n
+  invalid_category_id_counter,\n
+  deleted_category_counter + kept_category_counter + expired_category_counter,\n
+  deleted_category_counter,\n
+  expired_category_counter,\n
+  kept_category_counter,\n
+  \' (nothing done, simulation mode enabled)\' if simulation_mode else \'\',\n
+)\n
+if detailed_report:\n
+  REQUEST.other[\'portal_status_message\'] = portal_status_message\n
+  REQUEST.other[\'category_import_report\'] = detailed_report_result\n
+  result = portal_categories.CategoryTool_viewImportReport().encode(\'utf-8\')\n
+  if simulation_mode:\n
+    REQUEST.RESPONSE.write(result)\n
+    raise Exception(\'Dry run\')  \n
+  return result\n
+portal_categories.Base_redirect(\n
+  keep_items={\n
+    \'portal_status_message\': portal_status_message,\n
+  },\n
+  abort_transaction=simulation_mode,\n
 )\n
 </string> </value>
         </item>