Commit 57e877e1 authored by Sebastien Robin's avatar Sebastien Robin

allow to do hot reindex in unit test


git-svn-id: https://svn.erp5.org/repos/public/erp5/trunk@2080 20353a03-c40f-0410-a6d1-a30d3c3de9de
parent 7b87aedf
...@@ -540,6 +540,7 @@ class Catalog(Folder, Persistent, Acquisition.Implicit, ExtensionClass.Base): ...@@ -540,6 +540,7 @@ class Catalog(Folder, Persistent, Acquisition.Implicit, ExtensionClass.Base):
keys[c.Field] = 1 keys[c.Field] = 1
keys['%s.%s' % (table, c.Field)] = 1 # Is this inconsistent ? keys['%s.%s' % (table, c.Field)] = 1 # Is this inconsistent ?
except: except:
LOG('WARNING SQLCatalog.getColumnIds, exception with method',0,method)
pass pass
keys = keys.keys() keys = keys.keys()
keys.sort() keys.sort()
...@@ -819,7 +820,7 @@ class Catalog(Folder, Persistent, Acquisition.Implicit, ExtensionClass.Base): ...@@ -819,7 +820,7 @@ class Catalog(Folder, Persistent, Acquisition.Implicit, ExtensionClass.Base):
value = value() value = value()
kw[arg] = value kw[arg] = value
except: except:
#LOG("SQLCatalog Warning: Callable value could not be called",0,str((path, arg, method_name))) LOG("SQLCatalog Warning: Callable value could not be called",0,str((path, arg, method_name)))
kw[arg] = None kw[arg] = None
method = aq_base(method).__of__(object.__of__(self)) # Use method in the context of object method = aq_base(method).__of__(object.__of__(self)) # Use method in the context of object
# Generate UID # Generate UID
...@@ -833,7 +834,7 @@ class Catalog(Folder, Persistent, Acquisition.Implicit, ExtensionClass.Base): ...@@ -833,7 +834,7 @@ class Catalog(Folder, Persistent, Acquisition.Implicit, ExtensionClass.Base):
#LOG("Call SQL Method %s with args:" % method_name,0, str(kw)) #LOG("Call SQL Method %s with args:" % method_name,0, str(kw))
try: try:
if root_indexable: if root_indexable:
#LOG("Call SQL Method %s with args:" % method_name,0, str(kw)) LOG("Call SQL Method %s with args:" % method_name,0, str(kw))
method(**kw) method(**kw)
except: except:
LOG("SQLCatalog Warning: could not catalog object with method %s" % method_name,100, str(path)) LOG("SQLCatalog Warning: could not catalog object with method %s" % method_name,100, str(path))
...@@ -901,7 +902,11 @@ class Catalog(Folder, Persistent, Acquisition.Implicit, ExtensionClass.Base): ...@@ -901,7 +902,11 @@ class Catalog(Folder, Persistent, Acquisition.Implicit, ExtensionClass.Base):
except: except:
#LOG("SQLCatalog Warning: Callable value could not be called",0,str((path, arg, method_name))) #LOG("SQLCatalog Warning: Callable value could not be called",0,str((path, arg, method_name)))
kw[arg] = None kw[arg] = None
method = aq_base(method).__of__(object.__of__(self)) # Use method in the context of object try:
method = aq_base(method).__of__(object.__of__(self)) # Use method in the context of object
except AttributeError:
LOG('SQLCatalog Warning, can not find method for object',0,(method,object))
continue
# Generate UID # Generate UID
kw['path'] = path kw['path'] = path
kw['uid'] = index kw['uid'] = index
...@@ -909,7 +914,7 @@ class Catalog(Folder, Persistent, Acquisition.Implicit, ExtensionClass.Base): ...@@ -909,7 +914,7 @@ class Catalog(Folder, Persistent, Acquisition.Implicit, ExtensionClass.Base):
# Alter/Create row # Alter/Create row
try: try:
if root_indexable: if root_indexable:
#LOG("Call SQL Method %s with args:" % method_name,0, str(kw)) LOG("No Index, Call SQL Method %s with args:" % method_name,0, str(kw))
method(**kw) method(**kw)
except: except:
LOG("SQLCatalog Warning: could not catalog object with method %s" % method_name,100, str(path)) LOG("SQLCatalog Warning: could not catalog object with method %s" % method_name,100, str(path))
...@@ -943,10 +948,17 @@ class Catalog(Folder, Persistent, Acquisition.Implicit, ExtensionClass.Base): ...@@ -943,10 +948,17 @@ class Catalog(Folder, Persistent, Acquisition.Implicit, ExtensionClass.Base):
if not root_indexable: if not root_indexable:
return return
for object in object_list: temp_list = object_list
if not hasattr(aq_base(object), 'uid'): object_list = []
for object in temp_list:
if getattr(aq_base(object), 'uid',None)==None:
# XXX should do something better # XXX should do something better
raise RuntimeError, '%r does not have uid' % (object,) LOG('SQLCatalog.catalogObjectList, will catalog:',0,object.getPhysicalPath())
object.immediateReindexObject()
#pass
#raise RuntimeError, '%r does not have uid' % (object,)
else:
object_list.append(object)
methods = self.sql_catalog_object_list methods = self.sql_catalog_object_list
for method_name in methods: for method_name in methods:
...@@ -984,6 +996,8 @@ class Catalog(Folder, Persistent, Acquisition.Implicit, ExtensionClass.Base): ...@@ -984,6 +996,8 @@ class Catalog(Folder, Persistent, Acquisition.Implicit, ExtensionClass.Base):
for arg in split(arguments): for arg in split(arguments):
value_list = [] value_list = []
for object in catalogged_object_list: for object in catalogged_object_list:
LOG('catalog_object_list: object.uid',0,getattr(object,'uid',None))
LOG('catalog_object_list: object.path',0,object.getPhysicalPath())
try: try:
value = getattr(object, arg) value = getattr(object, arg)
if callable(value): if callable(value):
...@@ -1271,6 +1285,10 @@ class Catalog(Folder, Persistent, Acquisition.Implicit, ExtensionClass.Base): ...@@ -1271,6 +1285,10 @@ class Catalog(Folder, Persistent, Acquisition.Implicit, ExtensionClass.Base):
value='' value=''
if '%' in value: if '%' in value:
where_expression += ["%s LIKE '%s'" % (key, value)] where_expression += ["%s LIKE '%s'" % (key, value)]
elif value[0:2] == '>=':
where_expression += ["%s >= '%s'" % (key, value[2:])]
elif value[0:2] == '<=':
where_expression += ["%s <= '%s'" % (key, value[2:])]
elif value[0] == '>': elif value[0] == '>':
where_expression += ["%s > '%s'" % (key, value[1:])] where_expression += ["%s > '%s'" % (key, value[1:])]
elif value[0] == '<': elif value[0] == '<':
...@@ -1481,7 +1499,7 @@ class Catalog(Folder, Persistent, Acquisition.Implicit, ExtensionClass.Base): ...@@ -1481,7 +1499,7 @@ class Catalog(Folder, Persistent, Acquisition.Implicit, ExtensionClass.Base):
method = getattr(self, self.sql_record_uncatalog_object) method = getattr(self, self.sql_record_uncatalog_object)
method(path=path) method(path=path)
def deleteRecoredObjectList(self, path): def deleteRecordedObjectList(self, path):
""" """
Delete all objects which contain any path. Delete all objects which contain any path.
""" """
......
...@@ -280,7 +280,11 @@ class ZCatalog(Folder, Persistent, Implicit): ...@@ -280,7 +280,11 @@ class ZCatalog(Folder, Persistent, Implicit):
object = self.resolve_path(o.path) object = self.resolve_path(o.path)
if o.catalog: if o.catalog:
if object is not None: if object is not None:
self.activate(passive_commit=1, priority=5).reindexObject(object, sql_catalog_id=sql_catalog_id) LOG('playBackRecordedObjectList, will reindexObject:',0,object.getPhysicalPath())
#self.activate(passive_commit=1, priority=5).reindexObject(object, sql_catalog_id=sql_catalog_id)
# XXX We can't call reindexObject has an activate message, because we can not
# do pickle with acquisition wrappers
self.reindexObject(object, sql_catalog_id=sql_catalog_id) # Ask YO, we must find something activated
# Make sure that this object has no extra reference. # Make sure that this object has no extra reference.
object = None object = None
else: else:
...@@ -310,7 +314,7 @@ class ZCatalog(Folder, Persistent, Implicit): ...@@ -310,7 +314,7 @@ class ZCatalog(Folder, Persistent, Implicit):
new_selection = tuple(skin_selection_dict[skin_name]) new_selection = tuple(skin_selection_dict[skin_name])
self.portal_skins.manage_skinLayers(skinpath = new_selection, skinname = skin_name, add_skin = 1) self.portal_skins.manage_skinLayers(skinpath = new_selection, skinname = skin_name, add_skin = 1)
if sql_connection_id_dict: if sql_connection_id_dict is not None:
def changeSQLConnectionIds(folder): def changeSQLConnectionIds(folder):
if folder.meta_type == 'Z SQL Method': if folder.meta_type == 'Z SQL Method':
connection_id = folder.connection_id connection_id = folder.connection_id
...@@ -393,7 +397,9 @@ class ZCatalog(Folder, Persistent, Implicit): ...@@ -393,7 +397,9 @@ class ZCatalog(Folder, Persistent, Implicit):
object = self.resolve_path(path) object = self.resolve_path(path)
if object is None: continue if object is None: continue
id_list = object.objectIds() id_list = object.objectIds()
# object.activate(passive_commit=1, priority=5).queueCataloggedObject(sql_catalog_id=destination_sql_catalog_id) #LOG('will hotReindex this object:',0,object.getPhysicalPath())
#LOG('will hotReindex this object uid:',0,getattr(object,'uid',None))
object.activate(passive_commit=1, priority=5).queueCataloggedObject(sql_catalog_id=destination_sql_catalog_id)
object = None object = None
get_transaction().commit() # Should not have references to objects too long. get_transaction().commit() # Should not have references to objects too long.
...@@ -418,12 +424,14 @@ class ZCatalog(Folder, Persistent, Implicit): ...@@ -418,12 +424,14 @@ class ZCatalog(Folder, Persistent, Implicit):
destination_sql_catalog_id=destination_sql_catalog_id) destination_sql_catalog_id=destination_sql_catalog_id)
# Play back records. # Play back records.
LOG('hotReindexObjectList', 0, 'Playing back records') if source_sql_catalog_id != destination_sql_catalog_id:
self.activate(passive_commit=1, after_method_id='setHotReindexingState', priority=5).playBackRecordedObjectList(sql_catalog_id=destination_sql_catalog_id) # If equals, then this does not make sense to reindex again.
LOG('hotReindexObjectList', 0, 'Playing back records')
self.activate(passive_commit=1, after_method_id='setHotReindexingState', priority=5).playBackRecordedObjectList(sql_catalog_id=destination_sql_catalog_id)
# Exchange the databases. # Exchange the databases.
LOG('hotReindexObjectList', 0, 'Exchanging databases') LOG('hotReindexObjectList', 0, 'Exchanging databases')
self.activate(after_method_id=('reindexObject', 'playBackRecordedObjectList')).exchangeDatabases(source_sql_catalog_id, destination_sql_catalog_id, skin_selection_dict, sql_connection_id_dict) self.activate(after_method_id=('reindexObject', 'playBackRecordedObjectList')).exchangeDatabases(source_sql_catalog_id, destination_sql_catalog_id, skin_selection_dict, sql_connection_id_dict) # XXX Never called by activity tool, why ??? XXX
finally: finally:
# Finish. # Finish.
LOG('hotReindexObjectList', 0, 'Finishing hot reindexing') LOG('hotReindexObjectList', 0, 'Finishing hot reindexing')
...@@ -580,34 +588,36 @@ class ZCatalog(Folder, Persistent, Implicit): ...@@ -580,34 +588,36 @@ class ZCatalog(Folder, Persistent, Implicit):
""" """
return object return object
def catalog_object(self, obj, uid=None, idxs=[], is_object_moved=0, sql_catalog_id=None): def catalog_object(self, obj, url=None, idxs=[], is_object_moved=0, sql_catalog_id=None):
""" wrapper around catalog """ """ wrapper around catalog """
if uid is None: if url is None:
try: uid = obj.getPhysicalPath try: url = obj.getPhysicalPath
except AttributeError: except AttributeError:
raise CatalogError( raise CatalogError(
"A cataloged object must support the 'getPhysicalPath' " "A cataloged object must support the 'getPhysicalPath' "
"method if no unique id is provided when cataloging" "method if no unique id is provided when cataloging"
) )
else: uid=string.join(uid(), '/') else: url=string.join(url(), '/')
elif not isinstance(uid, types.StringType): elif not isinstance(url, types.StringType):
raise CatalogError('The object unique id must be a string.') raise CatalogError('The object unique id must be a string.')
obj = self.wrapObject(obj, sql_catalog_id=sql_catalog_id) obj = self.wrapObject(obj, sql_catalog_id=sql_catalog_id)
catalog = self.getSQLCatalog(sql_catalog_id) catalog = self.getSQLCatalog(sql_catalog_id)
if catalog is not None: if catalog is not None:
catalog.catalogObject(obj, uid, is_object_moved=is_object_moved) LOG('ZSQLCatalog.catalog_object, object:',0,obj.getPhysicalPath())
catalog.catalogObject(obj, url, is_object_moved=is_object_moved) # support obj, not uid
#catalog.reindexObject(obj, is_object_moved=is_object_moved) # support obj, not uid
if self.hot_reindexing_state is not None and self.source_sql_catalog_id == catalog.id: if self.hot_reindexing_state is not None and self.source_sql_catalog_id == catalog.id:
destination_catalog = self.getSQLCatalog(self.destination_sql_catalog_id) destination_catalog = self.getSQLCatalog(self.destination_sql_catalog_id)
if self.hot_reindexing_state == 'recording': if self.hot_reindexing_state == 'recording':
destination_catalog.recordCatalogObject(uid) destination_catalog.recordCatalogObject(url)
else: else:
destination_catalog.deleteRecordedObjectList([uid]) # Prevent this object from being replayed. destination_catalog.deleteRecordedObjectList([url]) # Prevent this object from being replayed.
destination_catalog.catalogObject(obj, uid, is_object_moved=is_object_moved, LOG('ZSQLCatalog.catalog_object, and hot reindex : object:',0,obj.getPhysicalPath())
sql_catalog_id=self.destination_sql_catalog_id) destination_catalog.catalogObject(obj, url, is_object_moved=is_object_moved)
security.declarePrivate('queueCataloggedObject') security.declarePrivate('queueCataloggedObject')
def queueCataloggedObject(self, object, sql_catalog_id=None): def queueCataloggedObject(self, object, sql_catalog_id=None):
...@@ -644,6 +654,7 @@ class ZCatalog(Folder, Persistent, Implicit): ...@@ -644,6 +654,7 @@ class ZCatalog(Folder, Persistent, Implicit):
if object is not None: if object is not None:
object = self.wrapObject(object, sql_catalog_id=sql_catalog_id) object = self.wrapObject(object, sql_catalog_id=sql_catalog_id)
object_list.append(object) object_list.append(object)
LOG('flushQueuedObjectList, object_list',0,[x.getPhysicalPath() for x in object_list])
if len(object_list) > 0: if len(object_list) > 0:
catalog = self.getSQLCatalog(sql_catalog_id) catalog = self.getSQLCatalog(sql_catalog_id)
catalog.catalogObjectList(object_list) catalog.catalogObjectList(object_list)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment