Skip to content

erp5
erp5
Commit 2b9a5e82 authored by Nicolas Dumazet's avatar Nicolas Dumazet
Browse files
Options

Split Transformation indexation in several activate()'ed chunks 

Since the number of possible variations for a resource can be very big,
it is not reasonable to process the indexation of all those lines in
a single big transaction.
Split the process and use activities; one activity per (resource, variation)
couple.


git-svn-id: https://svn.erp5.org/repos/public/erp5/trunk@33435 20353a03-c40f-0410-a6d1-a30d3c3de9de
parent fbcedc58
Hide whitespace changes
Inline Side-by-side
Showing with 355 additions and 61 deletions
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/SQLCatalog_catalogTransformation.xml 0 → 100644
View file @ 2b9a5e82
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<tuple>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
<tuple/>
</tuple>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_body</string> </key>
<value> <string encoding="cdata"><![CDATA[
# This script indexes the preferred Transformation to produce\n
# a variation of a product.\n
# Only the transformation is passed as a parameter, the resource being\n
# transformation.getResourceValue()\n
\n
from Products.ERP5Type.Document import newTempMovement\n
\n
transformation = context.restrictedTraverse(transformation_relative_url)\n
resource = transformation.getResourceValue()\n
\n
row_dict_list = []\n
movement = newTempMovement(resource, \'temp\',\n
variation_category_list=variation_list,\n
resource=resource.getRelativeUrl(),\n
quantity=1.0)\n
\n
base_row = dict(uid=resource.getUid(), variation_text=movement.getVariationText())\n
\n
for amount in transformation.getAggregatedAmountList(movement):\n
transformed_resource_uid = amount.getResourceUid()\n
quantity = amount.getQuantity()\n
if transformed_resource_uid is not None and quantity is not None:\n
row_dict = base_row.copy()\n
row_dict.update(transformed_uid=transformed_resource_uid,\n
transformed_variation_text=amount.getVariationText(),\n
quantity=quantity)\n
row_dict_list.append(row_dict)\n
\n
if len(row_dict_list) > 0:\n
context.z_catalog_transformation(row_dict_list=row_dict_list)\n
]]></string> </value>
</item>
<item>
<key> <string>_code</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>transformation_relative_url, variation_list</string> </value>
</item>
<item>
<key> <string>errors</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>func_code</string> </key>
<value>
<object>
<klass>
<global name="FuncCode" module="Shared.DC.Scripts.Signature"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>co_argcount</string> </key>
<value> <int>2</int> </value>
</item>
<item>
<key> <string>co_varnames</string> </key>
<value>
<tuple>
<string>transformation_relative_url</string>
<string>variation_list</string>
<string>Products.ERP5Type.Document</string>
<string>newTempMovement</string>
<string>_getattr_</string>
<string>context</string>
<string>transformation</string>
<string>resource</string>
<string>row_dict_list</string>
<string>movement</string>
<string>dict</string>
<string>base_row</string>
<string>_getiter_</string>
<string>amount</string>
<string>transformed_resource_uid</string>
<string>quantity</string>
<string>None</string>
<string>row_dict</string>
<string>len</string>
</tuple>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>func_defaults</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>SQLCatalog_catalogTransformation</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Index Transformation required to produce a specific variation of a Resource</string> </value>
</item>
<item>
<key> <string>warnings</string> </key>
<value>
<tuple/>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/SQLCatalog_catalogTransformationList.xml 0 → 100644
View file @ 2b9a5e82
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<tuple>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
<tuple/>
</tuple>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_body</string> </key>
<value> <string># Indexing all transformation lines for all possible variations of a Resource can be very costly.\n
# Avoid doing this in a single transaction, and split the operation.\n
#\n
# One activity is generated per (resource, variation_category_list) possibility\n
\n
for i, transformation in enumerate(getDefaultConversionTransformationValue):\n
if transformation is None:\n
continue\n
transformation_relative_url = transformation.getRelativeUrl()\n
for variation_category_list in getTransformationVariationCategoryCartesianProduct[i]:\n
context.activate(activity=\'SQLQueue\').SQLCatalog_catalogTransformation(transformation_relative_url,\n
variation_category_list)\n
</string> </value>
</item>
<item>
<key> <string>_code</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>getDefaultConversionTransformationValue, getTransformationVariationCategoryCartesianProduct</string> </value>
</item>
<item>
<key> <string>errors</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>func_code</string> </key>
<value>
<object>
<klass>
<global name="FuncCode" module="Shared.DC.Scripts.Signature"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>co_argcount</string> </key>
<value> <int>2</int> </value>
</item>
<item>
<key> <string>co_varnames</string> </key>
<value>
<tuple>
<string>getDefaultConversionTransformationValue</string>
<string>getTransformationVariationCategoryCartesianProduct</string>
<string>_getiter_</string>
<string>enumerate</string>
<string>i</string>
<string>transformation</string>
<string>None</string>
<string>_getattr_</string>
<string>transformation_relative_url</string>
<string>_getitem_</string>
<string>variation_category_list</string>
<string>context</string>
</tuple>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>func_defaults</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>SQLCatalog_catalogTransformationList</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Catalog the ways to produce all variations of a Resource</string> </value>
</item>
<item>
<key> <string>warnings</string> </key>
<value>
<tuple/>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z_catalog_transformation_list.xml → product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z_catalog_transformation.xml
View file @ 2b9a5e82
...@@ -24,13 +24,7 @@ ...@@ -24,13 +24,7 @@
<value> <value>
<dictionary> <dictionary>
<item> <item>
<key> <string>getTransformationRowList</string> </key> <key> <string>row_dict_list</string> </key>
<value>
<dictionary/>
</value>
</item>
<item>
<key> <string>uid</string> </key>
<value> <value>
<dictionary/> <dictionary/>
</value> </value>
...@@ -42,8 +36,7 @@ ...@@ -42,8 +36,7 @@
<key> <string>_keys</string> </key> <key> <string>_keys</string> </key>
<value> <value>
<list> <list>
<string>uid</string> <string>row_dict_list</string>
<string>getTransformationRowList</string>
</list> </list>
</value> </value>
</item> </item>
...@@ -60,8 +53,7 @@ ...@@ -60,8 +53,7 @@
</item> </item>
<item> <item>
<key> <string>arguments_src</string> </key> <key> <string>arguments_src</string> </key>
<value> <string>uid\r\n <value> <string>row_dict_list</string> </value>
getTransformationRowList</string> </value>
</item> </item>
<item> <item>
<key> <string>cache_time_</string> </key> <key> <string>cache_time_</string> </key>
...@@ -87,7 +79,7 @@ getTransformationRowList</string> </value> ...@@ -87,7 +79,7 @@ getTransformationRowList</string> </value>
</item> </item>
<item> <item>
<key> <string>id</string> </key> <key> <string>id</string> </key>
<value> <string>z_catalog_transformation_list</string> </value> <value> <string>z_catalog_transformation</string> </value>
</item> </item>
<item> <item>
<key> <string>max_cache_</string> </key> <key> <string>max_cache_</string> </key>
...@@ -102,19 +94,14 @@ getTransformationRowList</string> </value> ...@@ -102,19 +94,14 @@ getTransformationRowList</string> </value>
<value> <string encoding="cdata"><![CDATA[ <value> <string encoding="cdata"><![CDATA[
DELETE FROM `transformation` WHERE\n DELETE FROM `transformation` WHERE\n
<dtml-sqltest uid type="int" multiple>;\n <dtml-sqltest expr="row_dict_list[0][\'uid\']" column="uid" type="int">\n
\n AND\n
<dtml-let transformation_list="[]">\n <dtml-sqltest expr="row_dict_list[0][\'variation_text\']" column="variation_text" type="string">;\n
<dtml-in getTransformationRowList prefix="loop">\n <dtml-var sql_delimiter>\n
<dtml-call expr="loop_item and transformation_list.extend(loop_item)">\n
</dtml-in>\n
\n
<dtml-if transformation_list>\n
<dtml-var sql_delimiter>\n
\n \n
INSERT INTO `transformation`\n INSERT INTO `transformation`\n
VALUES\n VALUES\n
<dtml-in transformation_list prefix="loop">\n <dtml-in row_dict_list prefix="loop">\n
(\n (\n
<dtml-sqlvar expr="loop_item[\'uid\']" type="int">,\n <dtml-sqlvar expr="loop_item[\'uid\']" type="int">,\n
<dtml-sqlvar expr="loop_item[\'variation_text\']" type="string">,\n <dtml-sqlvar expr="loop_item[\'variation_text\']" type="string">,\n
...@@ -123,11 +110,7 @@ VALUES\n ...@@ -123,11 +110,7 @@ VALUES\n
<dtml-sqlvar expr="loop_item[\'quantity\']" type="float">\n <dtml-sqlvar expr="loop_item[\'quantity\']" type="float">\n
)\n )\n
<dtml-unless sequence-end>,</dtml-unless>\n <dtml-unless sequence-end>,</dtml-unless>\n
</dtml-in>\n </dtml-in>
\n
</dtml-if>\n
</dtml-let>\n
]]></string> </value> ]]></string> </value>
</item> </item>
...@@ -168,19 +151,14 @@ VALUES\n ...@@ -168,19 +151,14 @@ VALUES\n
<value> <string encoding="cdata"><![CDATA[ <value> <string encoding="cdata"><![CDATA[
DELETE FROM `transformation` WHERE\n DELETE FROM `transformation` WHERE\n
<dtml-sqltest uid type="int" multiple>;\n <dtml-sqltest expr="row_dict_list[0][\'uid\']" column="uid" type="int">\n
\n AND\n
<dtml-let transformation_list="[]">\n <dtml-sqltest expr="row_dict_list[0][\'variation_text\']" column="variation_text" type="string">;\n
<dtml-in getTransformationRowList prefix="loop">\n <dtml-var sql_delimiter>\n
<dtml-call expr="loop_item and transformation_list.extend(loop_item)">\n
</dtml-in>\n
\n
<dtml-if transformation_list>\n
<dtml-var sql_delimiter>\n
\n \n
INSERT INTO `transformation`\n INSERT INTO `transformation`\n
VALUES\n VALUES\n
<dtml-in transformation_list prefix="loop">\n <dtml-in row_dict_list prefix="loop">\n
(\n (\n
<dtml-sqlvar expr="loop_item[\'uid\']" type="int">,\n <dtml-sqlvar expr="loop_item[\'uid\']" type="int">,\n
<dtml-sqlvar expr="loop_item[\'variation_text\']" type="string">,\n <dtml-sqlvar expr="loop_item[\'variation_text\']" type="string">,\n
...@@ -189,11 +167,7 @@ VALUES\n ...@@ -189,11 +167,7 @@ VALUES\n
<dtml-sqlvar expr="loop_item[\'quantity\']" type="float">\n <dtml-sqlvar expr="loop_item[\'quantity\']" type="float">\n
)\n )\n
<dtml-unless sequence-end>,</dtml-unless>\n <dtml-unless sequence-end>,</dtml-unless>\n
</dtml-in>\n </dtml-in>
\n
</dtml-if>\n
</dtml-let>\n
]]></string> </value> ]]></string> </value>
</item> </item>
......
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z_catalog_transformation_list.catalog_keys.xml deleted 100644 → 0
View file @ fbcedc58
<catalog_method>
<item key="sql_catalog_object_list" type="int">
<value>1</value>
</item>
<item key="_is_filtered_archive" type="int">
<value>1</value>
</item>
<item key="_filter_expression_archive" type="str">
<value>python: context.isResourceType()</value>
</item>
<item key="_filter_expression_cache_key_archive" type="tuple">
<value>portal_type</value>
</item>
</catalog_method>
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z_create_transformation.xml
View file @ 2b9a5e82
...@@ -66,7 +66,7 @@ ...@@ -66,7 +66,7 @@
KEY `transformed_resource` (`transformed_uid`, `transformed_variation_text`)\n KEY `transformed_resource` (`transformed_uid`, `transformed_variation_text`)\n
)\n )\n
\n \n
</string> </value> </string> </value>
</item> </item>
<item> <item>
<key> <string>template</string> </key> <key> <string>template</string> </key>
...@@ -112,7 +112,7 @@ ...@@ -112,7 +112,7 @@
KEY `transformed_resource` (`transformed_uid`, `transformed_variation_text`)\n KEY `transformed_resource` (`transformed_uid`, `transformed_variation_text`)\n
)\n )\n
\n \n
</string> </value> </string> </value>
</item> </item>
</dictionary> </dictionary>
</state> </state>
......
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/bt/revision
View file @ 2b9a5e82
180 181
\ No newline at end of file \ No newline at end of file
product/ERP5/bootstrap/erp5_mysql_innodb_catalog/bt/template_catalog_method_id_list
View file @ 2b9a5e82
erp5_mysql_innodb/SQLCatalog_catalogTransformation
erp5_mysql_innodb/SQLCatalog_catalogTransformationList
erp5_mysql_innodb/SQLCatalog_makeQuickSearchQuery erp5_mysql_innodb/SQLCatalog_makeQuickSearchQuery
erp5_mysql_innodb/z0_drop_alarm erp5_mysql_innodb/z0_drop_alarm
erp5_mysql_innodb/z0_drop_catalog erp5_mysql_innodb/z0_drop_catalog
...@@ -46,7 +48,7 @@ erp5_mysql_innodb/z_catalog_predicate_category_list ...@@ -46,7 +48,7 @@ erp5_mysql_innodb/z_catalog_predicate_category_list
erp5_mysql_innodb/z_catalog_predicate_list erp5_mysql_innodb/z_catalog_predicate_list
erp5_mysql_innodb/z_catalog_roles_and_users_list erp5_mysql_innodb/z_catalog_roles_and_users_list
erp5_mysql_innodb/z_catalog_stock_list erp5_mysql_innodb/z_catalog_stock_list
erp5_mysql_innodb/z_catalog_transformation_list erp5_mysql_innodb/z_catalog_transformation
erp5_mysql_innodb/z_catalog_translation_list erp5_mysql_innodb/z_catalog_translation_list
erp5_mysql_innodb/z_catalog_versioning_list erp5_mysql_innodb/z_catalog_versioning_list
erp5_mysql_innodb/z_clear_reserved erp5_mysql_innodb/z_clear_reserved
...@@ -125,4 +127,4 @@ erp5_mysql_innodb/z_show_index ...@@ -125,4 +127,4 @@ erp5_mysql_innodb/z_show_index
erp5_mysql_innodb/z_show_tables erp5_mysql_innodb/z_show_tables
erp5_mysql_innodb/z_uncatalog_delivery erp5_mysql_innodb/z_uncatalog_delivery
erp5_mysql_innodb/z_uncatalog_object erp5_mysql_innodb/z_uncatalog_object
erp5_mysql_innodb/z_unique_values erp5_mysql_innodb/z_unique_values
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7