Skip to content

erp5
erp5
Commit 031408c6 authored by Kazuhiko Shiozaki's avatar Kazuhiko Shiozaki
4 Browse files
Options

erp5_full_text_mroonga_catalog: defer fulltext index and unindex using dedicated catalog keys.

parent c21b4004
Hide whitespace changes
Inline Side-by-side
Showing with 607 additions and 35 deletions
bt5/erp5_core_test/TestTemplateItem/portal_components/test.erp5.testCopySupport.py
View file @ 031408c6
......@@ -134,8 +134,11 @@ class TestCopySupport(ERP5TypeTestCase):
self.tic()
finally:
ActivityTool.invokeGroup = ActivityTool_invokeGroup
self.assertEqual(invokeGroup_list,
['portal_catalog/uncatalogObjectList', uid_list, 'SQLQueue', False])
catalog_relative_url = self.portal.portal_catalog.getSQLCatalog().getRelativeUrl()
self.assertEqual(invokeGroup_list, [
'portal_catalog/uncatalogObjectList', uid_list, 'SQLQueue', False,
catalog_relative_url + '/SQLCatalog_deferUncatalogObjectActivity', uid_list, 'SQLQueue', False,
])
self.assertEqual(len(search_catalog(uid=uid_list)), 0)
......
bt5/erp5_full_text_mroonga_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/SQLCatalog_deferFullTextIndex.py
View file @ 031408c6
# This script is called to defer fulltext indexing.
METHOD_ID = script.id + 'Activity'
GROUP_METHOD_ID = context.getPath() + '/' + METHOD_ID
GROUP_METHOD_ID = context.getRelativeUrl() + '/' + METHOD_ID
activateObject = context.getPortalObject().portal_activities.activateObject
try:
priority = context.getActivityRuntimeEnvironment().getPriority()
......
bt5/erp5_full_text_mroonga_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/SQLCatalog_deferFullTextIndexActivity.py
View file @ 031408c6
......@@ -3,31 +3,31 @@ from ZODB.POSException import ConflictError
from zExceptions import Unauthorized
import six
method = context.z_catalog_fulltext_list
property_list = method.arguments_src.split()
parameter_dict = {x: [] for x in property_list}
for group_object in object_list:
tmp_dict = {}
try:
obj = group_object.object
for property in property_list:
getter = getattr(obj, property, None)
if callable(getter):
value = getter()
else:
value = getattr(obj, 'get%s' % UpperCase(property))()
tmp_dict[property] = value
except ConflictError:
raise
except Unauthorized: # should happen in tricky testERP5Catalog tests only
# Fake activity success: if indexation cannot View document, ignore it.
group_object.result = None
except Exception:
group_object.raised()
else:
for property, value in six.iteritems(tmp_dict):
parameter_dict[property].append(value)
group_object.result = None
if parameter_dict:
return method(**parameter_dict)
for method_id in context.getSqlDeferredCatalogObjectListList():
method = getattr(context, method_id)
property_list = method.arguments_src.split()
parameter_dict = {x: [] for x in property_list}
for group_object in object_list:
tmp_dict = {}
try:
obj = group_object.object
for property in property_list:
getter = getattr(obj, property, None)
if callable(getter):
value = getter()
else:
value = getattr(obj, 'get%s' % UpperCase(property))()
tmp_dict[property] = value
except ConflictError:
raise
except Unauthorized: # should happen in tricky testERP5Catalog tests only
# Fake activity success: if indexation cannot View document, ignore it.
group_object.result = None
except Exception:
group_object.raised()
else:
for property, value in six.iteritems(tmp_dict):
parameter_dict[property].append(value)
group_object.result = None
if parameter_dict:
method(**parameter_dict)
bt5/erp5_full_text_mroonga_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/SQLCatalog_deferUncatalogObject.catalog_keys.xml 0 → 100644
View file @ 031408c6
<catalog_method>
<item key="sql_uncatalog_object" type="int">
<value>1</value>
</item>
</catalog_method>
bt5/erp5_full_text_mroonga_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/SQLCatalog_deferUncatalogObject.py 0 → 100644
View file @ 031408c6
# This script is called to defer fulltext unindexing.
METHOD_ID = script.id + 'Activity'
GROUP_METHOD_ID = context.getRelativeUrl() + '/' + METHOD_ID
try:
priority = context.getActivityRuntimeEnvironment().getPriority()
except KeyError:
# called outside of an activity, could be an immediate reindexation
# XXX: duplicates default priority for sake of simplicity and speed.
# Strictly, this could also look-up default activate parameters, but on
# which document ? Traversing is expensive. So keep things fast by default.
priority = 1
# Optimise cache usage by reducing the likelyhood of a processing node
# including activities spawned by others into its activity group.
# Use unindexObject as dummy method id for compatibility with
# after_method_id=('unindexObject', ...)
context.activate(
activity='SQLQueue',
priority=priority,
node='same',
group_method_id=GROUP_METHOD_ID,
).unindexObject(uid=uid)
bt5/erp5_full_text_mroonga_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/SQLCatalog_deferUncatalogObject.xml 0 → 100644
View file @ 031408c6
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Python Script" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="_reconstructor" module="copy_reg"/>
</klass>
<tuple>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
<global name="object" module="__builtin__"/>
<none/>
</tuple>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>uid</string> </value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>SQLCatalog_deferUncatalogObject</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Python Script</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_full_text_mroonga_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/SQLCatalog_deferUncatalogObjectActivity.py 0 → 100644
View file @ 031408c6
uid_list = []
for grouped_message in grouped_message_list:
assert not grouped_message.args
assert grouped_message.kw.keys() == ['uid']
uid_list.append(grouped_message.kw['uid'])
for method_id in context.getSqlDeferredUncatalogObjectList():
method = getattr(context, method_id)
method(uid=uid_list)
for grouped_message in grouped_message_list:
grouped_message.result = None
bt5/erp5_full_text_mroonga_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/SQLCatalog_deferUncatalogObjectActivity.xml 0 → 100644
View file @ 031408c6
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Python Script" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="_reconstructor" module="copy_reg"/>
</klass>
<tuple>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
<global name="object" module="__builtin__"/>
<none/>
</tuple>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>grouped_message_list</string> </value>
</item>
<item>
<key> <string>_proxy_roles</string> </key>
<value>
<tuple>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>SQLCatalog_deferUncatalogObjectActivity</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Python Script</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_full_text_mroonga_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z0_uncatalog_catalog_fulltext.catalog_keys.xml
View file @ 031408c6
<catalog_method>
<item key="sql_uncatalog_object" type="int">
<item key="sql_deferred_uncatalog_object" type="int">
<value>1</value>
</item>
</catalog_method>
bt5/erp5_full_text_mroonga_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z0_uncatalog_catalog_fulltext.sql
View file @ 031408c6
DELETE FROM catalog_full_text WHERE <dtml-sqltest uid op=eq type=int>
\ No newline at end of file
DELETE FROM catalog_full_text WHERE <dtml-sqltest uid op=eq type=int multiple>
\ No newline at end of file
bt5/erp5_full_text_mroonga_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z0_uncatalog_fulltext.catalog_keys.xml
View file @ 031408c6
<catalog_method>
<item key="sql_uncatalog_object" type="int">
<item key="sql_deferred_uncatalog_object" type="int">
<value>1</value>
</item>
</catalog_method>
bt5/erp5_full_text_mroonga_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z0_uncatalog_fulltext.sql
View file @ 031408c6
DELETE FROM full_text WHERE <dtml-sqltest uid op=eq type=int>
\ No newline at end of file
DELETE FROM full_text WHERE <dtml-sqltest uid op=eq type=int multiple>
\ No newline at end of file
bt5/erp5_full_text_mroonga_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z_catalog_fulltext_list.catalog_keys.xml
View file @ 031408c6
<catalog_method>
<item key="sql_deferred_catalog_object_list" type="int">
<value>1</value>
</item>
</catalog_method>
bt5/erp5_full_text_mroonga_catalog/bt/template_catalog_method_id_list
View file @ 031408c6
erp5_mysql_innodb/SQLCatalog_deferFullTextIndex
erp5_mysql_innodb/SQLCatalog_deferFullTextIndexActivity
erp5_mysql_innodb/SQLCatalog_deferUncatalogObject
erp5_mysql_innodb/SQLCatalog_deferUncatalogObjectActivity
erp5_mysql_innodb/SQLCatalog_makeFullTextQuery
erp5_mysql_innodb/z0_drop_catalog_fulltext
erp5_mysql_innodb/z0_drop_content_translation
......
product/ERP5/bootstrap/erp5_core/PropertySheetTemplateItem/portal_property_sheets/Catalog/sql_deferred_catalog_object_list_property.xml 0 → 100644
View file @ 031408c6
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Standard Property" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_local_properties</string> </key>
<value>
<tuple>
<dictionary>
<item>
<key> <string>id</string> </key>
<value> <string>select_variable</string> </value>
</item>
<item>
<key> <string>type</string> </key>
<value> <string>string</string> </value>
</item>
</dictionary>
<dictionary>
<item>
<key> <string>id</string> </key>
<value> <string>mode</string> </value>
</item>
<item>
<key> <string>type</string> </key>
<value> <string>string</string> </value>
</item>
</dictionary>
</tuple>
</value>
</item>
<item>
<key> <string>categories</string> </key>
<value>
<tuple>
<string>elementary_type/multiple selection</string>
</tuple>
</value>
</item>
<item>
<key> <string>description</string> </key>
<value> <string>Methods to be called to catalog the list of objects with a deferred connection</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>sql_deferred_catalog_object_list_property</string> </value>
</item>
<item>
<key> <string>mode</string> </key>
<value> <string>w</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Standard Property</string> </value>
</item>
<item>
<key> <string>select_variable</string> </key>
<value> <string>getCatalogMethodIds</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
product/ERP5/bootstrap/erp5_core/PropertySheetTemplateItem/portal_property_sheets/Catalog/sql_deferred_uncatalog_object_property.xml 0 → 100644
View file @ 031408c6
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Standard Property" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_local_properties</string> </key>
<value>
<tuple>
<dictionary>
<item>
<key> <string>id</string> </key>
<value> <string>mode</string> </value>
</item>
<item>
<key> <string>type</string> </key>
<value> <string>string</string> </value>
</item>
</dictionary>
</tuple>
</value>
</item>
<item>
<key> <string>categories</string> </key>
<value>
<tuple>
<string>elementary_type/multiple selection</string>
</tuple>
</value>
</item>
<item>
<key> <string>description</string> </key>
<value> <string>Methods to be called to uncatalog an object with a deferred connection</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>sql_deferred_uncatalog_object_property</string> </value>
</item>
<item>
<key> <string>mode</string> </key>
<value> <string>w</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Standard Property</string> </value>
</item>
<item>
<key> <string>select_variable</string> </key>
<value> <string>getCatalogMethodIds</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/Catalog_viewPropertyList.xml
View file @ 031408c6
......@@ -118,7 +118,9 @@
<value>
<list>
<string>my_sql_catalog_object_list_list</string>
<string>my_sql_deferred_catalog_object_list_list</string>
<string>my_sql_uncatalog_object_list</string>
<string>my_sql_deferred_uncatalog_object_list</string>
<string>my_sql_search_tables_list</string>
<string>my_sql_catalog_datetime_search_keys_list</string>
</list>
......
product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/Catalog_viewPropertyList/my_sql_deferred_catalog_object_list_list.xml 0 → 100644
View file @ 031408c6
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="ProxyField" module="Products.ERP5Form.ProxyField"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>delegated_list</string> </key>
<value>
<list>
<string>items</string>
<string>title</string>
</list>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>my_sql_deferred_catalog_object_list_list</string> </value>
</item>
<item>
<key> <string>message_values</string> </key>
<value>
<dictionary>
<item>
<key> <string>external_validator_failed</string> </key>
<value> <string>The input failed the external validator.</string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>overrides</string> </key>
<value>
<dictionary>
<item>
<key> <string>field_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>tales</string> </key>
<value>
<dictionary>
<item>
<key> <string>field_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>items</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
</value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>values</string> </key>
<value>
<dictionary>
<item>
<key> <string>field_id</string> </key>
<value> <string>my_multi_list_field</string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string>Base_viewFieldLibrary</string> </value>
</item>
<item>
<key> <string>items</string> </key>
<value>
<list/>
</value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Deferred Catalog Objects</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="2" aka="AAAAAAAAAAI=">
<pickle>
<global name="TALESMethod" module="Products.Formulator.TALESField"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_text</string> </key>
<value> <string>python:[(\'\', \'\')]+here.getCatalogMethodIds()</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/Catalog_viewPropertyList/my_sql_deferred_uncatalog_object_list.xml 0 → 100644
View file @ 031408c6
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="ProxyField" module="Products.ERP5Form.ProxyField"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>delegated_list</string> </key>
<value>
<list>
<string>items</string>
<string>title</string>
</list>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>my_sql_deferred_uncatalog_object_list</string> </value>
</item>
<item>
<key> <string>message_values</string> </key>
<value>
<dictionary>
<item>
<key> <string>external_validator_failed</string> </key>
<value> <string>The input failed the external validator.</string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>overrides</string> </key>
<value>
<dictionary>
<item>
<key> <string>field_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>tales</string> </key>
<value>
<dictionary>
<item>
<key> <string>field_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>items</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
</value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>values</string> </key>
<value>
<dictionary>
<item>
<key> <string>field_id</string> </key>
<value> <string>my_multi_list_field</string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string>Base_viewFieldLibrary</string> </value>
</item>
<item>
<key> <string>items</string> </key>
<value>
<list/>
</value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Deferred Uncatalog Object</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="2" aka="AAAAAAAAAAI=">
<pickle>
<global name="TALESMethod" module="Products.Formulator.TALESField"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_text</string> </key>
<value> <string>python:[(\'\', \'\')]+here.getCatalogMethodIds()</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
product/ZSQLCatalog/SQLCatalog.py
View file @ 031408c6
......@@ -397,11 +397,21 @@ class Catalog(Folder,
'type' : 'multiple selection',
'select_variable' : 'getCatalogMethodIds',
'mode' : 'w' },
{ 'id' : 'sql_deferred_catalog_object_list',
'description' : 'Methods to be called to catalog the list of objects with a deferred connection',
'type' : 'multiple selection',
'select_variable' : 'getCatalogMethodIds',
'mode' : 'w' },
{ 'id' : 'sql_uncatalog_object',
'description' : 'Methods to be called to uncatalog an object',
'type' : 'multiple selection',
'select_variable' : 'getCatalogMethodIds',
'mode' : 'w' },
{ 'id' : 'sql_deferred_uncatalog_object',
'description' : 'Methods to be called to uncatalog an object with a deferred connection',
'type' : 'multiple selection',
'select_variable' : 'getCatalogMethodIds',
'mode' : 'w' },
{ 'id' : 'sql_catalog_translation_list',
'description' : 'Methods to be called to catalog the list of translation objects',
'type' : 'selection',
......@@ -582,7 +592,9 @@ class Catalog(Folder,
sql_catalog_delete_uid = ''
sql_catalog_clear_reserved = ''
sql_catalog_object_list = ()
sql_deferred_catalog_object_list = ()
sql_uncatalog_object = ()
sql_deferred_uncatalog_object = ()
sql_clear_catalog = ()
sql_catalog_translation_list = ''
sql_delete_translation_list = ''
......@@ -1275,6 +1287,9 @@ class Catalog(Folder,
def getSqlCatalogObjectListList(self):
return self.sql_catalog_object_list
def getSqlDeferredCatalogObjectListList(self):
return self.sql_deferred_catalog_object_list
def _catalogObjectList(self, object_list, method_id_list=None,
disable_cache=0, check_uid=1, idxs=None):
"""This is the real method to catalog objects."""
......@@ -1490,6 +1505,9 @@ class Catalog(Folder,
def getSqlUncatalogObjectList(self):
return self.sql_uncatalog_object
def getSqlDeferredUncatalogObjectList(self):
return self.sql_deferred_uncatalog_object
security.declarePrivate('uncatalogObject')
def uncatalogObject(self, path=None, uid=None):
"""
......@@ -2457,7 +2475,9 @@ class Catalog(Folder,
if withCMF:
method_id_set.update(
self.getSqlCatalogObjectListList() +
self.getSqlDeferredCatalogObjectListList() +
self.getSqlUncatalogObjectList() +
self.getSqlDeferredUncatalogObjectList() +
self.getSqlUpdateObjectList()
)
return [
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7