Commit 91385efb authored by Kazuhiko Shiozaki's avatar Kazuhiko Shiozaki

defer catalog / uncatalog for deferred connection

See merge request !2043
parents 3770c698 7f610066
<catalog_method>
<item key="sql_uncatalog_object" type="int">
<item key="sql_deferred_uncatalog_object" type="int">
<value>1</value>
</item>
</catalog_method>
DELETE FROM content_translation WHERE <dtml-sqltest uid op=eq type=int>
\ No newline at end of file
DELETE FROM content_translation WHERE <dtml-sqltest uid op=eq type=int multiple>
\ No newline at end of file
<catalog_method>
<item key="sql_catalog_object_list" type="int">
<item key="sql_deferred_catalog_object_list" type="int">
<value>1</value>
</item>
</catalog_method>
......@@ -134,8 +134,11 @@ class TestCopySupport(ERP5TypeTestCase):
self.tic()
finally:
ActivityTool.invokeGroup = ActivityTool_invokeGroup
self.assertEqual(invokeGroup_list,
['portal_catalog/uncatalogObjectList', uid_list, 'SQLQueue', False])
catalog_relative_url = self.portal.portal_catalog.getSQLCatalog().getRelativeUrl()
self.assertEqual(invokeGroup_list, [
'portal_catalog/uncatalogObjectList', uid_list, 'SQLQueue', False,
catalog_relative_url + '/SQLCatalog_deferUncatalogObjectActivity', uid_list, 'SQLQueue', False,
])
self.assertEqual(len(search_catalog(uid=uid_list)), 0)
......
# This script is called to defer fulltext indexing.
METHOD_ID = script.id + 'Activity'
GROUP_METHOD_ID = context.getPath() + '/' + METHOD_ID
GROUP_METHOD_ID = context.getRelativeUrl() + '/' + METHOD_ID
activateObject = context.getPortalObject().portal_activities.activateObject
try:
priority = context.getActivityRuntimeEnvironment().getPriority()
......
......@@ -3,31 +3,31 @@ from ZODB.POSException import ConflictError
from zExceptions import Unauthorized
import six
method = context.z_catalog_fulltext_list
property_list = method.arguments_src.split()
parameter_dict = {x: [] for x in property_list}
for group_object in object_list:
tmp_dict = {}
try:
obj = group_object.object
for property in property_list:
getter = getattr(obj, property, None)
if callable(getter):
value = getter()
else:
value = getattr(obj, 'get%s' % UpperCase(property))()
tmp_dict[property] = value
except ConflictError:
raise
except Unauthorized: # should happen in tricky testERP5Catalog tests only
# Fake activity success: if indexation cannot View document, ignore it.
group_object.result = None
except Exception:
group_object.raised()
else:
for property, value in six.iteritems(tmp_dict):
parameter_dict[property].append(value)
group_object.result = None
if parameter_dict:
return method(**parameter_dict)
for method_id in context.getSqlDeferredCatalogObjectListList():
method = getattr(context, method_id)
property_list = method.arguments_src.split()
parameter_dict = {x: [] for x in property_list}
for group_object in object_list:
tmp_dict = {}
try:
obj = group_object.object
for property in property_list:
getter = getattr(obj, property, None)
if callable(getter):
value = getter()
else:
value = getattr(obj, 'get%s' % UpperCase(property))()
tmp_dict[property] = value
except ConflictError:
raise
except Unauthorized: # should happen in tricky testERP5Catalog tests only
# Fake activity success: if indexation cannot View document, ignore it.
group_object.result = None
except Exception:
group_object.raised()
else:
for property, value in six.iteritems(tmp_dict):
parameter_dict[property].append(value)
group_object.result = None
if parameter_dict:
method(**parameter_dict)
<catalog_method>
<item key="sql_uncatalog_object" type="int">
<value>1</value>
</item>
</catalog_method>
# This script is called to defer fulltext unindexing.
METHOD_ID = script.id + 'Activity'
GROUP_METHOD_ID = context.getRelativeUrl() + '/' + METHOD_ID
try:
priority = context.getActivityRuntimeEnvironment().getPriority()
except KeyError:
# called outside of an activity, could be an immediate reindexation
# XXX: duplicates default priority for sake of simplicity and speed.
# Strictly, this could also look-up default activate parameters, but on
# which document ? Traversing is expensive. So keep things fast by default.
priority = 1
# Optimise cache usage by reducing the likelyhood of a processing node
# including activities spawned by others into its activity group.
# Use unindexObject as dummy method id for compatibility with
# after_method_id=('unindexObject', ...)
context.activate(
activity='SQLQueue',
priority=priority,
node='same',
group_method_id=GROUP_METHOD_ID,
).unindexObject(uid=uid)
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Python Script" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="_reconstructor" module="copy_reg"/>
</klass>
<tuple>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
<global name="object" module="__builtin__"/>
<none/>
</tuple>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>uid</string> </value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>SQLCatalog_deferUncatalogObject</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Python Script</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
uid_list = []
for grouped_message in grouped_message_list:
assert not grouped_message.args
assert grouped_message.kw.keys() == ['uid']
uid_list.append(grouped_message.kw['uid'])
for method_id in context.getSqlDeferredUncatalogObjectList():
method = getattr(context, method_id)
method(uid=uid_list)
for grouped_message in grouped_message_list:
grouped_message.result = None
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Python Script" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="_reconstructor" module="copy_reg"/>
</klass>
<tuple>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
<global name="object" module="__builtin__"/>
<none/>
</tuple>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>grouped_message_list</string> </value>
</item>
<item>
<key> <string>_proxy_roles</string> </key>
<value>
<tuple>
<string>Manager</string>
</tuple>
</value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>SQLCatalog_deferUncatalogObjectActivity</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Python Script</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
DELETE FROM catalog_full_text WHERE <dtml-sqltest uid op=eq type=int>
\ No newline at end of file
DELETE FROM catalog_full_text WHERE <dtml-sqltest uid op=eq type=int multiple>
\ No newline at end of file
<catalog_method>
<item key="sql_uncatalog_object" type="int">
<item key="sql_deferred_uncatalog_object" type="int">
<value>1</value>
</item>
</catalog_method>
DELETE FROM full_text WHERE <dtml-sqltest uid op=eq type=int>
\ No newline at end of file
DELETE FROM full_text WHERE <dtml-sqltest uid op=eq type=int multiple>
\ No newline at end of file
<catalog_method>
<item key="sql_deferred_catalog_object_list" type="int">
<value>1</value>
</item>
</catalog_method>
erp5_mysql_innodb/SQLCatalog_deferFullTextIndex
erp5_mysql_innodb/SQLCatalog_deferFullTextIndexActivity
erp5_mysql_innodb/SQLCatalog_deferUncatalogObject
erp5_mysql_innodb/SQLCatalog_deferUncatalogObjectActivity
erp5_mysql_innodb/SQLCatalog_makeFullTextQuery
erp5_mysql_innodb/z0_drop_catalog_fulltext
erp5_mysql_innodb/z0_drop_content_translation
......
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Standard Property" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_local_properties</string> </key>
<value>
<tuple>
<dictionary>
<item>
<key> <string>id</string> </key>
<value> <string>select_variable</string> </value>
</item>
<item>
<key> <string>type</string> </key>
<value> <string>string</string> </value>
</item>
</dictionary>
<dictionary>
<item>
<key> <string>id</string> </key>
<value> <string>mode</string> </value>
</item>
<item>
<key> <string>type</string> </key>
<value> <string>string</string> </value>
</item>
</dictionary>
</tuple>
</value>
</item>
<item>
<key> <string>categories</string> </key>
<value>
<tuple>
<string>elementary_type/multiple selection</string>
</tuple>
</value>
</item>
<item>
<key> <string>description</string> </key>
<value> <string>Methods to be called to catalog the list of objects with a deferred connection</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>sql_deferred_catalog_object_list_property</string> </value>
</item>
<item>
<key> <string>mode</string> </key>
<value> <string>w</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Standard Property</string> </value>
</item>
<item>
<key> <string>select_variable</string> </key>
<value> <string>getCatalogMethodIds</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Standard Property" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_local_properties</string> </key>
<value>
<tuple>
<dictionary>
<item>
<key> <string>id</string> </key>
<value> <string>mode</string> </value>
</item>
<item>
<key> <string>type</string> </key>
<value> <string>string</string> </value>
</item>
</dictionary>
</tuple>
</value>
</item>
<item>
<key> <string>categories</string> </key>
<value>
<tuple>
<string>elementary_type/multiple selection</string>
</tuple>
</value>
</item>
<item>
<key> <string>description</string> </key>
<value> <string>Methods to be called to uncatalog an object with a deferred connection</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>sql_deferred_uncatalog_object_property</string> </value>
</item>
<item>
<key> <string>mode</string> </key>
<value> <string>w</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Standard Property</string> </value>
</item>
<item>
<key> <string>select_variable</string> </key>
<value> <string>getCatalogMethodIds</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
......@@ -118,7 +118,9 @@
<value>
<list>
<string>my_sql_catalog_object_list_list</string>
<string>my_sql_deferred_catalog_object_list_list</string>
<string>my_sql_uncatalog_object_list</string>
<string>my_sql_deferred_uncatalog_object_list</string>
<string>my_sql_search_tables_list</string>
<string>my_sql_catalog_datetime_search_keys_list</string>
</list>
......
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="ProxyField" module="Products.ERP5Form.ProxyField"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>delegated_list</string> </key>
<value>
<list>
<string>items</string>
<string>title</string>
</list>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>my_sql_deferred_catalog_object_list_list</string> </value>
</item>
<item>
<key> <string>message_values</string> </key>
<value>
<dictionary>
<item>
<key> <string>external_validator_failed</string> </key>
<value> <string>The input failed the external validator.</string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>overrides</string> </key>
<value>
<dictionary>
<item>
<key> <string>field_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>tales</string> </key>
<value>
<dictionary>
<item>
<key> <string>field_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>items</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
</value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>values</string> </key>
<value>
<dictionary>
<item>
<key> <string>field_id</string> </key>
<value> <string>my_multi_list_field</string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string>Base_viewFieldLibrary</string> </value>
</item>
<item>
<key> <string>items</string> </key>
<value>
<list/>
</value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Deferred Catalog Objects</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="2" aka="AAAAAAAAAAI=">
<pickle>
<global name="TALESMethod" module="Products.Formulator.TALESField"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_text</string> </key>
<value> <string>python:[(\'\', \'\')]+here.getCatalogMethodIds()</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="ProxyField" module="Products.ERP5Form.ProxyField"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>delegated_list</string> </key>
<value>
<list>
<string>items</string>
<string>title</string>
</list>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>my_sql_deferred_uncatalog_object_list</string> </value>
</item>
<item>
<key> <string>message_values</string> </key>
<value>
<dictionary>
<item>
<key> <string>external_validator_failed</string> </key>
<value> <string>The input failed the external validator.</string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>overrides</string> </key>
<value>
<dictionary>
<item>
<key> <string>field_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>tales</string> </key>
<value>
<dictionary>
<item>
<key> <string>field_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>items</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
</value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>values</string> </key>
<value>
<dictionary>
<item>
<key> <string>field_id</string> </key>
<value> <string>my_multi_list_field</string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string>Base_viewFieldLibrary</string> </value>
</item>
<item>
<key> <string>items</string> </key>
<value>
<list/>
</value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Deferred Uncatalog Object</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="2" aka="AAAAAAAAAAI=">
<pickle>
<global name="TALESMethod" module="Products.Formulator.TALESField"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_text</string> </key>
<value> <string>python:[(\'\', \'\')]+here.getCatalogMethodIds()</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
......@@ -29,7 +29,6 @@ from OFS.Folder import Folder
from AccessControl import ClassSecurityInfo
from AccessControl.Permissions import (
access_contents_information,
import_export_objects,
manage_zcatalog_entries,
)
from AccessControl.SimpleObjectPolicies import ContainerAssertions
......@@ -54,10 +53,6 @@ import pprint
import re
import warnings
from contextlib import contextmanager
from xml.dom.minidom import parse
from xml.sax.saxutils import escape, quoteattr
import os
from hashlib import md5
from .interfaces.query_catalog import ISearchKeyCatalog
from zope.interface.verify import verifyClass
......@@ -402,11 +397,21 @@ class Catalog(Folder,
'type' : 'multiple selection',
'select_variable' : 'getCatalogMethodIds',
'mode' : 'w' },
{ 'id' : 'sql_deferred_catalog_object_list',
'description' : 'Methods to be called to catalog the list of objects with a deferred connection',
'type' : 'multiple selection',
'select_variable' : 'getCatalogMethodIds',
'mode' : 'w' },
{ 'id' : 'sql_uncatalog_object',
'description' : 'Methods to be called to uncatalog an object',
'type' : 'multiple selection',
'select_variable' : 'getCatalogMethodIds',
'mode' : 'w' },
{ 'id' : 'sql_deferred_uncatalog_object',
'description' : 'Methods to be called to uncatalog an object with a deferred connection',
'type' : 'multiple selection',
'select_variable' : 'getCatalogMethodIds',
'mode' : 'w' },
{ 'id' : 'sql_catalog_translation_list',
'description' : 'Methods to be called to catalog the list of translation objects',
'type' : 'selection',
......@@ -587,7 +592,9 @@ class Catalog(Folder,
sql_catalog_delete_uid = ''
sql_catalog_clear_reserved = ''
sql_catalog_object_list = ()
sql_deferred_catalog_object_list = ()
sql_uncatalog_object = ()
sql_deferred_uncatalog_object = ()
sql_clear_catalog = ()
sql_catalog_translation_list = ''
sql_delete_translation_list = ''
......@@ -1217,11 +1224,10 @@ class Catalog(Folder,
elapse = time.time()
c_elapse = time.clock()
words = 0
obj = REQUEST.PARENTS[1]
path = '/'.join(obj.getPhysicalPath())
results = self.aq_parent.ZopeFindAndApply(obj,
self.aq_parent.ZopeFindAndApply(obj,
obj_metatypes=obj_metatypes,
obj_ids=obj_ids,
obj_searchterm=obj_searchterm,
......@@ -1281,6 +1287,9 @@ class Catalog(Folder,
def getSqlCatalogObjectListList(self):
return self.sql_catalog_object_list
def getSqlDeferredCatalogObjectListList(self):
return self.sql_deferred_catalog_object_list
def _catalogObjectList(self, object_list, method_id_list=None,
disable_cache=0, check_uid=1, idxs=None):
"""This is the real method to catalog objects."""
......@@ -1496,6 +1505,9 @@ class Catalog(Folder,
def getSqlUncatalogObjectList(self):
return self.sql_uncatalog_object
def getSqlDeferredUncatalogObjectList(self):
return self.sql_deferred_uncatalog_object
security.declarePrivate('uncatalogObject')
def uncatalogObject(self, path=None, uid=None):
"""
......@@ -1874,9 +1886,7 @@ class Catalog(Folder,
"""
search_key, related_key_definition = self.getColumnSearchKey(key,
search_key_name=search_key_name)
if search_key is None:
result = None
else:
if search_key is not None:
if related_key_definition is not None:
search_key = search_key.getSearchKey(sql_catalog=self,
related_key_definition=related_key_definition,
......@@ -2450,30 +2460,12 @@ class Catalog(Folder,
if method.meta_type in ('Z SQL Method', 'ERP5 SQL Method') and ('deferred' in method.connection_id) == deferred:
return method.connection_id
def getSqlCatalogObjectList(self):
try:
return self.sql_catalog_object
except AttributeError:
return ()
def getSqlUncatalogObjectList(self):
try:
return self.sql_uncatalog_object
except AttributeError:
return ()
def getSqlUpdateObjectList(self):
try:
return self.sql_update_object
except AttributeError:
return ()
def getSqlCatalogObjectListList(self):
try:
return self.sql_catalog_object_list
except AttributeError:
return ()
security.declarePrivate('getFilterableMethodList')
def getFilterableMethodList(self):
"""
......@@ -2482,10 +2474,11 @@ class Catalog(Folder,
method_id_set = set()
if withCMF:
method_id_set.update(
self.getSqlCatalogObjectList() +
self.getSqlCatalogObjectListList() +
self.getSqlDeferredCatalogObjectListList() +
self.getSqlUncatalogObjectList() +
self.getSqlUpdateObjectList() +
self.getSqlCatalogObjectListList()
self.getSqlDeferredUncatalogObjectList() +
self.getSqlUpdateObjectList()
)
return [
method
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment