ZSQLCatalog.py 53.1 KB
Newer Older
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
##############################################################################
#
# Copyright (c) 2002 Nexedi SARL. All Rights Reserved.
# Copyright (c) 2001 Zope Corporation and Contributors. All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""ZCatalog product"""

16 17 18
from App.special_dtml import DTMLFile
from App.Dialogs import MessageDialog
from App.class_init import default__class_init__ as InitializeClass
Jean-Paul Smets's avatar
Jean-Paul Smets committed
19 20 21

from OFS.Folder import Folder
from DateTime import DateTime
22
from Acquisition import Implicit, aq_base
Jean-Paul Smets's avatar
Jean-Paul Smets committed
23 24 25 26
from Persistence import Persistent
from DocumentTemplate.DT_Util import InstanceDict, TemplateDict
from DocumentTemplate.DT_Util import Eval
from AccessControl.Permission import name_trans
27 28
from AccessControl.Permissions import import_export_objects, \
    manage_zcatalog_entries
29 30
from SQLCatalog import CatalogError
from AccessControl import ClassSecurityInfo
31
from DocumentTemplate.security import RestrictedDTML
32
from Products.CMFCore.utils import getToolByName
33
from Products.ERP5Type.Cache import clearCache
34
import string, sys
35
import time
Yoshinori Okuji's avatar
Yoshinori Okuji committed
36
import urllib
37
from ZODB.POSException import ConflictError
Jean-Paul Smets's avatar
Jean-Paul Smets committed
38

39
from zLOG import LOG, ERROR, INFO
Jean-Paul Smets's avatar
Jean-Paul Smets committed
40

41 42
_marker = object()

43
manage_addZSQLCatalogForm=DTMLFile('dtml/addZSQLCatalog',globals())
Jean-Paul Smets's avatar
Jean-Paul Smets committed
44

45 46 47 48
HOT_REINDEXING_FINISHED_STATE = 'finished'
HOT_REINDEXING_RECORDING_STATE = 'recording'
HOT_REINDEXING_DOUBLE_INDEXING_STATE = 'double indexing'

49
def manage_addZSQLCatalog(self, id, title,
Jean-Paul Smets's avatar
Jean-Paul Smets committed
50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87
             vocab_id='create_default_catalog_',
             REQUEST=None):
  """Add a ZCatalog object
  """
  id=str(id)
  title=str(title)
  vocab_id=str(vocab_id)
  if vocab_id == 'create_default_catalog_':
    vocab_id = None

  c=ZCatalog(id, title, self)
  self._setObject(id, c)
  if REQUEST is not None:
    return self.manage_main(self, REQUEST,update_menu=1)


class ZCatalog(Folder, Persistent, Implicit):
  """ZCatalog object

  A ZCatalog contains arbirary index like references to Zope
  objects.  ZCatalog's can index either 'Field' values of object, or
  'Text' values.

  ZCatalog does not store references to the objects themselves, but
  rather to a unique identifier that defines how to get to the
  object.  In Zope, this unique idenfier is the object's relative
  path to the ZCatalog (since two Zope object's cannot have the same
  URL, this is an excellent unique qualifier in Zope).

  Most of the dirty work is done in the _catalog object, which is an
  instance of the Catalog class.  An interesting feature of this
  class is that it is not Zope specific.  You can use it in any
  Python program to catalog objects.

  """

  meta_type = "ZSQLCatalog"
  icon='misc_/ZCatalog/ZCatalog.gif'
88
  security = ClassSecurityInfo()
Jean-Paul Smets's avatar
Jean-Paul Smets committed
89 90 91 92 93 94 95 96

  manage_options = (
    {'label': 'Contents',       # TAB: Contents
     'action': 'manage_main',
     'help': ('OFSP','ObjectManager_Contents.stx')},
    {'label': 'Catalog',      # TAB: Cataloged Objects
     'action': 'manage_catalogView',
     'help':('ZCatalog','ZCatalog_Cataloged-Objects.stx')},
97 98
    {'label' : 'Filter',        # TAB: Filter
     'action' : 'manage_catalogFilter' },
Jean-Paul Smets's avatar
Jean-Paul Smets committed
99 100 101 102 103 104 105 106 107
    {'label': 'Properties',     # TAB: Properties
     'action': 'manage_propertiesForm',
     'help': ('OFSP','Properties.stx')},
    {'label': 'Find Objects',     # TAB: Find Objects
     'action': 'manage_catalogFind',
     'help':('ZCatalog','ZCatalog_Find-Items-to-ZCatalog.stx')},
    {'label': 'Advanced',       # TAB: Advanced
     'action': 'manage_catalogAdvanced',
     'help':('ZCatalog','ZCatalog_Advanced.stx')},
108 109
    {'label': 'Hot Reindexing',       # TAB: Hot Reindex
     'action': 'manage_catalogHotReindexing',
110
     },
Jean-Paul Smets's avatar
Jean-Paul Smets committed
111 112 113 114 115 116 117 118 119 120 121 122 123 124
    {'label': 'Undo',         # TAB: Undo
     'action': 'manage_UndoForm',
     'help': ('OFSP','Undo.stx')},
    {'label': 'Security',       # TAB: Security
     'action': 'manage_access',
     'help': ('OFSP','Security.stx')},
    {'label': 'Ownership',      # TAB: Ownership
     'action': 'manage_owner',
     'help': ('OFSP','Ownership.stx'),}
    )

  __ac_permissions__=(

    ('Manage ZCatalog Entries',
125
     ['manage_catalogView', 'manage_catalogFind',
126
      'manage_catalogSchema', 'manage_catalogFilter',
Jean-Paul Smets's avatar
Jean-Paul Smets committed
127
      'manage_catalogAdvanced', 'manage_objectInformation',
128
      'manage_catalogHotReindexing',
129
      'manage_main',],
Jean-Paul Smets's avatar
Jean-Paul Smets committed
130 131 132 133
     ['Manager']),

    ('Search ZCatalog',
     ['searchResults', '__call__', 'uniqueValuesFor',
Yoshinori Okuji's avatar
Yoshinori Okuji committed
134
      'getpath', 'schema', 'names', 'indexes',
Jean-Paul Smets's avatar
Jean-Paul Smets committed
135
      'all_meta_types', 'valid_roles', 'resolve_url',
Jean-Paul Smets's avatar
Jean-Paul Smets committed
136
      'getobject', 'getObject', 'getObjectList', 'getCatalogSearchTableIds',
137
      'getCatalogSearchResultKeys', 'getFilterableMethodList', ],
Jean-Paul Smets's avatar
Jean-Paul Smets committed
138
     ['Anonymous', 'Manager']),
139

Jean-Paul Smets's avatar
Jean-Paul Smets committed
140 141 142
    )

  _properties = (
Jean-Paul Smets's avatar
Jean-Paul Smets committed
143
    { 'id'      : 'title',
Jean-Paul Smets's avatar
Jean-Paul Smets committed
144 145 146
      'description' : 'The title of this catalog',
      'type'    : 'string',
      'mode'    : 'w' },
147 148
    { 'id'      : 'default_sql_catalog_id',
      'description' : 'The id of the default SQL Catalog',
Jean-Paul Smets's avatar
Jean-Paul Smets committed
149
      'type'    : 'selection',
150
      'select_variable'    : 'getSQLCatalogIdList',
151
      'mode'    : 'w' },
Jean-Paul Smets's avatar
Jean-Paul Smets committed
152

153 154 155 156 157 158 159 160 161 162 163 164 165
    # Hot Reindexing
    { 'id'      : 'source_sql_catalog_id',
      'description' : 'The id of a source SQL Catalog for hot reindexing',
      'type'    : 'string',
      'mode'    : '' },
    { 'id'      : 'destination_sql_catalog_id',
      'description' : 'The id of a destination SQL Catalog for hot reindexing',
      'type'    : 'string',
      'mode'    : '' },
    { 'id'      : 'hot_reindexing_state',
      'description' : 'The state of hot reindexing',
      'type'    : 'string',
      'mode'    : '' },
166 167 168 169
    { 'id'      : 'archive_path',
      'description' : 'The path of the archive which is create',
      'type'    : 'string',
      'mode'    : '' },
170 171

  )
Jean-Paul Smets's avatar
Jean-Paul Smets committed
172

173 174 175 176
  source_sql_catalog_id = None
  destination_sql_catalog_id = None
  hot_reindexing_state = None
  default_sql_catalog_id = None
177
  archive_path = None
178

Jean-Paul Smets's avatar
Jean-Paul Smets committed
179
  manage_catalogAddRowForm = DTMLFile('dtml/catalogAddRowForm', globals())
180
  manage_catalogFilter = DTMLFile( 'dtml/catalogFilter', globals() )
Jean-Paul Smets's avatar
Jean-Paul Smets committed
181 182 183 184 185
  manage_catalogView = DTMLFile('dtml/catalogView',globals())
  manage_catalogFind = DTMLFile('dtml/catalogFind',globals())
  manage_catalogSchema = DTMLFile('dtml/catalogSchema', globals())
  manage_catalogIndexes = DTMLFile('dtml/catalogIndexes', globals())
  manage_catalogAdvanced = DTMLFile('dtml/catalogAdvanced', globals())
186
  manage_catalogHotReindexing = DTMLFile('dtml/catalogHotReindexing', globals())
Jean-Paul Smets's avatar
Jean-Paul Smets committed
187 188 189 190 191 192 193 194
  manage_objectInformation = DTMLFile('dtml/catalogObjectInformation',
                                                              globals())

  def __init__(self, id, title='', container=None):
    if container is not None:
      self=self.__of__(container)
    self.id=id
    self.title=title
195

196
  security.declarePrivate('getSQLCatalogIdList')
197 198 199
  def getSQLCatalogIdList(self):
    return self.objectIds(spec=('SQLCatalog',))

200
  security.declarePublic('getSQLCatalog')
201 202 203 204 205 206 207 208 209 210 211 212 213
  def getSQLCatalog(self, id=None, default_value=None):
    """
      Get the default SQL Catalog.
    """
    if id is None:
      if not self.default_sql_catalog_id:
        id_list = self.getSQLCatalogIdList()
        if len(id_list) > 0:
          self.default_sql_catalog_id = id_list[0]
        else:
          return default_value
      id = self.default_sql_catalog_id

214
    return self._getOb(id, default_value)
215

216
  security.declareProtected(import_export_objects, 'manage_catalogExportProperties')
217 218 219
  def manage_catalogExportProperties(self, REQUEST=None, RESPONSE=None, sql_catalog_id=None):
    """
      Export properties to an XML file.
Jean-Paul Smets's avatar
Jean-Paul Smets committed
220
    """
221 222 223
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      return catalog.manage_exportProperties(REQUEST=REQUEST, RESPONSE=RESPONSE)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
224

225
  security.declareProtected(import_export_objects, 'manage_catalogImportProperties')
226 227 228 229 230 231 232
  def manage_catalogImportProperties(self, file, sql_catalog_id=None):
    """
      Import properties from an XML file.
    """
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      return catalog.manage_importProperties(file)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
233

234 235 236 237 238 239
  def __len__(self):
    catalog = self.getSQLCatalog()
    if catalog is None:
      return 0
    return len(catalog)

240
  security.declarePrivate('getHotReindexingState')
241 242 243 244 245 246 247 248 249
  def getHotReindexingState(self):
    """
      Return the current hot reindexing state.
    """
    value = getattr(self, 'hot_reindexing_state', None)
    if value is None:
      return HOT_REINDEXING_FINISHED_STATE
    return value

250
  def _setHotReindexingState(self, state='', source_sql_catalog_id=None, destination_sql_catalog_id=None, archive_path=None):
251 252
    """
      Set the state of hot reindexing.
Jean-Paul Smets's avatar
Jean-Paul Smets committed
253

254 255 256
      Do not use setProperty because the state should not modified from the ZMI directly.
      It must be maintained very carefully.
    """
257
    #LOG("_setHotReindexingState call", 300, state)
258 259 260
    if source_sql_catalog_id is None:
      source_sql_catalog_id = self.default_sql_catalog_id

261
    if state == HOT_REINDEXING_FINISHED_STATE:
262 263 264
      self.hot_reindexing_state = None
      self.source_sql_catalog_id = None
      self.destination_sql_catalog_id = None
265
      self.archive_path = None
266 267
    elif state == HOT_REINDEXING_RECORDING_STATE or \
         state == HOT_REINDEXING_DOUBLE_INDEXING_STATE:
268 269 270
      self.hot_reindexing_state = state
      self.source_sql_catalog_id = source_sql_catalog_id
      self.destination_sql_catalog_id = destination_sql_catalog_id
271
      self.archive_path = archive_path
272 273
    else:
      raise CatalogError, 'unknown hot reindexing state %s' % state
Jean-Paul Smets's avatar
Jean-Paul Smets committed
274

275
  def _finishHotReindexing(self, source_sql_catalog_id,
276 277
                          destination_sql_catalog_id, skin_selection_dict,
                          sql_connection_id_dict):
278
    """
279
      Exchange databases and finish reindexing in the same transaction.
280
    """
281 282 283 284 285
    if self.archive_path is not None  and \
           getattr(self, "portal_archives", None) is not None:
      current_archive = self.portal_archives.getCurrentArchive()
    else:
      current_archive = None
Aurel's avatar
Aurel committed
286
    default_catalog_id = self.default_sql_catalog_id
287
    self._exchangeDatabases(source_sql_catalog_id=source_sql_catalog_id,
288 289 290
                           destination_sql_catalog_id=destination_sql_catalog_id,
                           skin_selection_dict=skin_selection_dict,
                           sql_connection_id_dict=sql_connection_id_dict)
291 292 293
    # cancel archive use as current catalog before archiving
    if current_archive is not None:
      current_archive.cancel()
294
    self._setHotReindexingState(state=HOT_REINDEXING_FINISHED_STATE)
295
    clearCache(cache_factory_list=('erp5_content_short',))
296

297
  security.declarePrivate('cancelHotReindexing')
298
  def cancelHotReindexing(self):
299
    """
300 301 302 303 304
      Cancel a hot reindexing.
      Remove the hot reindexing state and flush related activities.

      TODO: Find a safe way to remove activities started by
            ERP5Site_reindexAll.
305
    """
306 307 308 309
    if self.getHotReindexingState() == HOT_REINDEXING_FINISHED_STATE:
      raise Exception, 'cancelHotReindexing called while no Hot Reindexing '\
                       'was runing. Nothing done.'
    # Remove hot reindexing state
310
    self._setHotReindexingState(HOT_REINDEXING_FINISHED_STATE)
311 312 313 314 315
    portal_activities = getToolByName(self, 'portal_activities')
    if portal_activities is not None:
      object_path = self.getPhysicalPath()
      # Activities must be removed in the reverse order they were inserted
      # to make sure removing one does not accidntaly trigger the next one.
316
      method_id_list = ('_finishHotReindexing', 'runInventoryMethod',
317
                        'playBackRecordedObjectList', 'InventoryModule_reindexMovementList'
318
                        '_setHotReindexingState')
319 320 321
      for method_id in method_id_list:
        portal_activities.flush(object_path, method_id=method_id)

322
  security.declarePrivate('playBackRecordedObjectList')
323 324 325 326
  def playBackRecordedObjectList(self, sql_catalog_id, catalog=0):
    """
      Play back the actions scheduled while hot reindexing was in "record"
      state.
327

328 329 330
      sql_catalog_id   Id of the catalog on which the actions will be played.
      catalog          0 : play unindex actions
                       1 : play index actions
331

332 333 334 335 336 337 338 339 340 341 342
      This function schedules itself for later execution.
      This is done in order to avoid accessing "too many" objects in the same
      transaction.
    """
    if self.getHotReindexingState() != HOT_REINDEXING_DOUBLE_INDEXING_STATE:
      raise Exception, 'playBackRecordedObjectList was called while '\
                       'hot_reindexing_state was not "%s". Playback aborted.' \
                       % (HOT_REINDEXING_DOUBLE_INDEXING_STATE, )
    catalog_object = self.getSQLCatalog(sql_catalog_id)
    result = catalog_object.readRecordedObjectList(catalog=catalog)
    if len(result):
343
      for o in result:
Vincent Pelletier's avatar
Vincent Pelletier committed
344
        if catalog == 0:
345
          self.uncatalog_object(uid=o.path, sql_catalog_id=sql_catalog_id)
Vincent Pelletier's avatar
Vincent Pelletier committed
346
        elif catalog == 1:
347
          obj = self.resolve_path(o.path)
348
          if obj is not None:
349
            obj.reindexObject(sql_catalog_id=sql_catalog_id)
350 351
        else:
          raise ValueError, '%s is not a valid value for "catalog".' % (catalog, )
352 353 354 355
      catalog_object.deleteRecordedObjectList(uid_list=[o.uid for o in result])
      # Re-schedule the same action in case there are remaining rows in the
      # table. This can happen if the database connector limits the number
      # of rows in the result.
356
      self.activate(priority=5).\
357 358 359
          playBackRecordedObjectList(sql_catalog_id=sql_catalog_id,
                                     catalog=catalog)
    else:
Vincent Pelletier's avatar
Vincent Pelletier committed
360
      # If there is nothing to do, go to next step.
361 362
      if catalog == 0:
        # If we were replaying unindex actions, time to replay index actions.
363
        self.activate(priority=5).\
364 365 366
            playBackRecordedObjectList(sql_catalog_id=sql_catalog_id,
                                       catalog=1)
      # If we were replaying index actions, there is nothing else to do.
367

368
  security.declarePrivate('changeSQLConnectionIds')
369 370 371 372 373 374
  def changeSQLConnectionIds(self, folder, sql_connection_id_dict):
    if sql_connection_id_dict is not None:
      if folder.meta_type == 'Z SQL Method':
        connection_id = folder.connection_id
        if connection_id in sql_connection_id_dict:
          folder.connection_id = sql_connection_id_dict[connection_id]
375
      elif getattr(aq_base(folder), 'objectValues', _marker) is not _marker:
376 377 378
        for object in folder.objectValues():
          self.changeSQLConnectionIds(object,sql_connection_id_dict)

379
  def _exchangeDatabases(self, source_sql_catalog_id, destination_sql_catalog_id,
380 381 382 383 384 385
                        skin_selection_dict, sql_connection_id_dict):
    """
      Exchange two databases.
    """
    if self.default_sql_catalog_id == source_sql_catalog_id:
      self.default_sql_catalog_id = destination_sql_catalog_id
386 387 388 389 390 391
      id_tool = getattr(self.getPortalObject(), 'portal_ids', None)
      if id_tool is None:
        # Insert the latest generated uid.
        # This must be done just before swaping the catalogs in case there were
        # generated uids since destination catalog was created.
        self[destination_sql_catalog_id].insertMaxUid()
392

393
    LOG('_exchangeDatabases skin_selection_dict:',0,skin_selection_dict)
394
    if skin_selection_dict is not None:
395
      #LOG('_exchangeDatabases skin_selection_dict:',0,'we will do manage_skinLayers')
396 397 398 399
      for skin_name, selection in self.portal_skins.getSkinPaths():
        if skin_name in skin_selection_dict:
          new_selection = tuple(skin_selection_dict[skin_name])
          self.portal_skins.manage_skinLayers(skinpath = new_selection, skinname = skin_name, add_skin = 1)
400

401
    LOG('_exchangeDatabases sql_connection_id_dict :',0,sql_connection_id_dict)
402
    if sql_connection_id_dict is not None:
403
      self.changeSQLConnectionIds(self.portal_skins, sql_connection_id_dict)
404

405
  security.declareProtected(manage_zcatalog_entries, 'manage_hotReindexAll')
406
  def manage_hotReindexAll(self, source_sql_catalog_id,
407
                           destination_sql_catalog_id,
408
                           archive_path=None,
409 410 411 412 413
                           source_sql_connection_id_list=None,
                           destination_sql_connection_id_list=None,
                           skin_name_list=None,
                           skin_selection_list=None,
                           update_destination_sql_catalog=None,
414
                           base_priority=5,
415
                           REQUEST=None, RESPONSE=None):
416
    """
417 418
      Starts a hot reindexing.

Vincent Pelletier's avatar
Vincent Pelletier committed
419 420 421
      Hot reindexing reindexes all documents using destination_sql_catalog_id
      with low priority (so site can keep working during hot reindexation).

422 423 424 425 426 427 428 429 430 431
      Once done, both catalogs will be swapped so that current catalog will
      not be used any more and destination catalog will get used "for real".

      source_catalog_id
        Id of the SQLCatalog object to use as the source catalog.
        WARNING: it is not considered normal to specify a catalog which is not
                 the current default one.
                 The feature is still provided, but you'll be on your own if
                 you try it.

432
      destination_sql_catalog_id
433 434 435 436 437 438 439 440 441 442 443 444
        Id of the SQLCatalog object to use as the new catalog.

      source_sql_connection_id_list
      destination_sql_connection_id_list
        SQL Methods in portal_skins using source_sql_connection_id_list[n]
        connection will use destination_sql_connection_id_list[n] connection
        once hot reindexing is over.

      skin_name_list
      skin_selection_list
        For each skin_name_list[n], skin_selection_list[n] will be set to
        replace the existing skin selection on portal_skins.
Jean-Paul Smets's avatar
Jean-Paul Smets committed
445
    """
446 447
    # Hot reindexing can only be runing once at a time on a system.
    if self.hot_reindexing_state is not None:
448
      raise CatalogError, 'hot reindexing process is already running %s -%s' %(self, self.hot_reindexing_state)
449 450 451 452 453 454 455 456 457 458 459 460 461

    if source_sql_catalog_id == destination_sql_catalog_id:
      raise CatalogError, 'Hot reindexing cannot be done with the same '\
                          'catalog as both source and destination. What'\
                          ' you want to do is a "clear catalog" and an '\
                          '"ERP5Site_reindexAll".'

    if source_sql_catalog_id != self.default_sql_catalog_id:
      LOG('ZSQLCatalog', 0, 'Warning : Hot reindexing is started with a '\
                            'source catalog which is not the default one.')

    # Construct a mapping for skin selections. It will be used during the
    # final hot reindexing step.
462 463 464 465 466 467 468 469 470 471 472
    skin_selection_dict = None
    if skin_name_list is not None and skin_selection_list is not None:
      skin_selection_dict = {}
      for name, selection_list in zip(skin_name_list, skin_selection_list):
        # Make sure that there is no extra space.
        new_selection_list = []
        for selection in selection_list:
          new_selection = selection.strip()
          if len(new_selection) > 0:
            new_selection_list.append(new_selection)
        skin_selection_dict[name] = new_selection_list
473

474 475
    # Construct a mapping for connection ids. It will be used during the
    # final hot reindexing step.
476
    sql_connection_id_dict = None
477 478
    if source_sql_connection_id_list is not None and \
       destination_sql_connection_id_list is not None:
479
      sql_connection_id_dict = {}
480 481 482
      for source_sql_connection_id, destination_sql_connection_id in \
          zip(source_sql_connection_id_list,
              destination_sql_connection_id_list):
483
        if source_sql_connection_id != destination_sql_connection_id:
484 485
          sql_connection_id_dict[source_sql_connection_id] = \
              destination_sql_connection_id
486

487 488 489 490 491
    destination_sql_catalog = getattr(self,destination_sql_catalog_id)
    if update_destination_sql_catalog:
      self.changeSQLConnectionIds(destination_sql_catalog,
                                  sql_connection_id_dict)

492 493
    # First of all, make sure that all root objects have uids.
    # XXX This is a workaround for tools (such as portal_simulation).
494 495
    portal = self.getPortalObject()
    for id in portal.objectIds():
496
      getUid = getattr(portal[id], 'getUid', None)
497 498
      if getUid is not None and id != "portal_uidhandler":
        # XXX check adviced by yo, getUid is different for this tool
499 500 501 502 503
        getUid() # Trigger the uid generation if none is set.

    # Mark the hot reindex as begun. Each object indexed in the still-current
    # catalog will be scheduled for reindex in the future catalog.
    LOG('hotReindexObjectList', 0, 'Starting recording')
504
    self._setHotReindexingState(HOT_REINDEXING_RECORDING_STATE,
505
                               source_sql_catalog_id=source_sql_catalog_id,
506
                               destination_sql_catalog_id=destination_sql_catalog_id,
507
                               archive_path=archive_path)
508 509
    # Clear the future catalog and start reindexing the site in it.
    final_activity_tag = 'hot_reindex_last_ERP5Site_reindexAll_tag'
510
    self.ERP5Site_reindexAll(sql_catalog_id=destination_sql_catalog_id,
511 512
                             final_activity_tag=final_activity_tag,
                             clear_catalog=1,
513
                             additional_priority=base_priority)
514 515
    # Once reindexing is finished, change the hot reindexing state so that
    # new catalog changes are applied in both catalogs.
516
    self.activate(after_tag=final_activity_tag,
517
                  priority=base_priority)._setHotReindexingState(HOT_REINDEXING_DOUBLE_INDEXING_STATE,
518
                      source_sql_catalog_id=source_sql_catalog_id,
519 520
                      destination_sql_catalog_id=destination_sql_catalog_id,
                      archive_path=archive_path)
521
    # Once in double-indexing mode, planned reindex can be replayed.
522
    self.activate(after_method_id='_setHotReindexingState',
523
                  priority=base_priority).playBackRecordedObjectList(
524 525 526
                      sql_catalog_id=destination_sql_catalog_id)
    # Once there is nothing to replay, databases are sync'ed, so the new
    # catalog can become current.
527
    self.activate(after_method_id=('playBackRecordedObjectList',
528 529
                                   'InventoryModule_reindexMovementList'),
                  after_tag='InventoryModule_reindexMovementList',
530
                  priority=base_priority)._finishHotReindexing(
531 532 533 534
                      source_sql_catalog_id=source_sql_catalog_id,
                      destination_sql_catalog_id=destination_sql_catalog_id,
                      skin_selection_dict=skin_selection_dict,
                      sql_connection_id_dict=sql_connection_id_dict)
535 536
    if RESPONSE is not None:
      URL1 = REQUEST.get('URL1')
537
      RESPONSE.redirect(URL1 + '/manage_catalogHotReindexing?manage_tabs_message=HotReindexing%20Started')
Jean-Paul Smets's avatar
Jean-Paul Smets committed
538

539
  security.declareProtected(manage_zcatalog_entries, 'manage_edit')
Jean-Paul Smets's avatar
Jean-Paul Smets committed
540 541 542 543 544 545 546 547 548
  def manage_edit(self, RESPONSE, URL1, threshold=1000, REQUEST=None):
    """ edit the catalog """
    if type(threshold) is not type(1):
      threshold=string.atoi(threshold)
    self.threshold = threshold

    RESPONSE.redirect(URL1 + '/manage_main?manage_tabs_message=Catalog%20Changed')


549
  security.declareProtected(manage_zcatalog_entries, 'manage_catalogObject')
550
  def manage_catalogObject(self, REQUEST, RESPONSE, URL1, urls=None, sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
551
    """ index Zope object(s) that 'urls' point to """
552 553
    if sql_catalog_id is None:
      sql_catalog_id = REQUEST.get('sql_catalog_id', None)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
554

555 556 557
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      catalog.manage_catalogObject(REQUEST, RESPONSE, URL1, urls=urls)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
558 559


560
  security.declareProtected(manage_zcatalog_entries, 'manage_uncatalogObject')
561
  def manage_uncatalogObject(self, REQUEST, RESPONSE, URL1, urls=None, sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
562
    """ removes Zope object(s) 'urls' from catalog """
563 564
    if sql_catalog_id is None:
      sql_catalog_id = REQUEST.get('sql_catalog_id', None)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
565

566 567 568
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      catalog.manage_uncatalogObject(REQUEST, RESPONSE, URL1, urls=urls)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
569 570


571
  security.declareProtected(manage_zcatalog_entries, 'manage_catalogReindex')
572
  def manage_catalogReindex(self, REQUEST, RESPONSE, URL1, urls=None, sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
573
    """ clear the catalog, then re-index everything """
574 575
    if sql_catalog_id is None:
      sql_catalog_id = REQUEST.get('sql_catalog_id', None)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
576

577 578 579
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      catalog.manage_catalogReindex(REQUEST, RESPONSE, URL1, urls=urls)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
580

581
  security.declareProtected(manage_zcatalog_entries, 'refreshCatalog')
582
  def refreshCatalog(self, clear=0, sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
583 584
    """ re-index everything we can find """

585 586 587 588
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      paths = catalog.getPaths()
      if clear:
589
        catalog._clear()
Jean-Paul Smets's avatar
Jean-Paul Smets committed
590

591 592 593 594 595 596
      for p in paths:
        obj = self.resolve_path(p.path)
        if not obj:
          obj = self.resolve_url(p.path, self.REQUEST)
        if obj is not None:
          self.catalog_object(obj, p.path, sql_catalog_id=sql_catalog_id)
597

598
  security.declareProtected(manage_zcatalog_entries, 'manage_catalogClear')
599
  def manage_catalogClear(self, REQUEST=None, RESPONSE=None, URL1=None, sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
600
    """ clears the whole enchilada """
601 602
    if REQUEST is not None and sql_catalog_id is None:
      sql_catalog_id = REQUEST.get('sql_catalog_id', None)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
603

604 605 606
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      catalog.manage_catalogClear(REQUEST=REQUEST, RESPONSE=RESPONSE, URL1=URL1)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
607

608
  security.declareProtected(manage_zcatalog_entries, 'manage_catalogClearReserved')
609
  def manage_catalogClearReserved(self, REQUEST=None, RESPONSE=None, URL1=None, sql_catalog_id=None):
610
    """ clears the whole enchilada """
611 612
    if REQUEST is not None and sql_catalog_id is None:
      sql_catalog_id = REQUEST.get('sql_catalog_id', None)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
613

614 615 616
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      catalog.manage_catalogClearReserved(REQUEST=REQUEST, RESPONSE=RESPONSE, URL1=URL1)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
617

618
  security.declareProtected(manage_zcatalog_entries, 'manage_catalogFoundItems')
Jean-Paul Smets's avatar
Jean-Paul Smets committed
619 620 621 622 623
  def manage_catalogFoundItems(self, REQUEST, RESPONSE, URL2, URL1,
                 obj_metatypes=None,
                 obj_ids=None, obj_searchterm=None,
                 obj_expr=None, obj_mtime=None,
                 obj_mspec=None, obj_roles=None,
624 625
                 obj_permission=None,
                 sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
626 627
    """ Find object according to search criteria and Catalog them
    """
628 629 630 631 632 633 634 635 636 637
    if sql_catalog_id is None:
      sql_catalog_id = REQUEST.get('sql_catalog_id', None)

    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      catalog.manage_catalogFoundItems(REQUEST, RESPONSE, URL2, URL1,
                                       obj_metatypes=obj_metatypes, obj_ids=obj_ids,
                                       obj_searchterm=obj_searchterm, obj_expr=obj_expr,
                                       obj_mtime=obj_mtime, obj_mspec=obj_mspec,
                                       obj_roles=obj_roles, obj_permission=obj_permission)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658

    elapse = time.time()
    c_elapse = time.clock()

    words = 0
    obj = REQUEST.PARENTS[1]
    path = string.join(obj.getPhysicalPath(), '/')


    results = self.ZopeFindAndApply(obj,
                    obj_metatypes=obj_metatypes,
                    obj_ids=obj_ids,
                    obj_searchterm=obj_searchterm,
                    obj_expr=obj_expr,
                    obj_mtime=obj_mtime,
                    obj_mspec=obj_mspec,
                    obj_permission=obj_permission,
                    obj_roles=obj_roles,
                    search_sub=1,
                    REQUEST=REQUEST,
                    apply_func=self.catalog_object,
659 660
                    apply_path=path,
                    sql_catalog_id=sql_catalog_id)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
661 662 663 664 665 666 667

    elapse = time.time() - elapse
    c_elapse = time.clock() - c_elapse

    RESPONSE.redirect(URL1 + '/manage_catalogView?manage_tabs_message=' +
              urllib.quote('Catalog Updated<br>Total time: %s<br>Total CPU time: %s' % (`elapse`, `c_elapse`)))

668
  security.declareProtected(manage_zcatalog_entries, 'manage_editSchema')
669
  def manage_editSchema(self, names, REQUEST=None, RESPONSE=None, URL1=None, sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
670
    """ add a column """
671 672 673 674
    if REQUEST is not None and sql_catalog_id is None:
      sql_catalog_id = REQUEST.get('sql_catalog_id', None)

    self.editSchema(names, sql_catalog_id=sql_catalog_id)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
675 676 677 678

    if REQUEST and RESPONSE:
      RESPONSE.redirect(URL1 + '/manage_catalogSchema?manage_tabs_message=Schema%20Saved')

679
  security.declarePrivate('newUid')
680
  def newUid(self, sql_catalog_id=None):
681 682 683
    """
        Allocates a new uid value.
    """
684 685 686
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      return catalog.newUid()
687

688
  security.declarePrivate('getDynamicRelatedKeyList')
689 690 691 692 693 694
  def getDynamicRelatedKeyList(self, sql_catalog_id=None,**kw):
    """
    Return the list of dynamic related keys.
    """
    return []

695 696
  security.declarePrivate('wrapObjectList')
  def wrapObjectList(self, object_value_list, catalog_value):
697
    """
698
      Return a list of wrapped objects for reindexing.
699 700 701

      This method should be overridden if necessary.
    """
702
    return object_value_list
703

704
  security.declareProtected(manage_zcatalog_entries, 'catalog_object')
Yoshinori Okuji's avatar
Yoshinori Okuji committed
705
  def catalog_object(self, obj, url=None, idxs=[], is_object_moved=0, sql_catalog_id=None, **kw):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
706
    """ wrapper around catalog """
707
    self.catalogObjectList([obj], sql_catalog_id=sql_catalog_id)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
708

709
  security.declarePrivate('catalogObjectList')
710 711
  def catalogObjectList(self, object_list, sql_catalog_id=None, disable_archive=0,
                        immediate_reindex_archive=1, **kw):
712 713
    """Catalog a list of objects.
    """
714 715 716 717
    catalog = self.getSQLCatalog(sql_catalog_id)
    hot_reindexing = (self.hot_reindexing_state is not None) and \
                     (catalog is not None) and \
                     (self.source_sql_catalog_id == catalog.id)
718
    archiving = self.archive_path is not None
719
    failed_object_list = []
720
    url_list = []
721
    archive_list = []
722 723
    portal_archives = getattr(self, 'portal_archives', None)
    if portal_archives is not None:
724 725
      if len(portal_archives):
        archive_list = portal_archives.getArchiveList()
726 727 728

    catalog_dict = {}

Aurel's avatar
Aurel committed
729
    # Create archive object list if necessary
730
    if archiving:
731
      # while archiving only test with the archive we used, do not care
Aurel's avatar
Aurel committed
732
      # of other as they must already be ok
733 734
      archive = self.unrestrictedTraverse(self.archive_path)
      archive_obj_list = [archive,]
735 736 737 738 739 740 741 742
      for archive_path in archive_list:
        try:
          archive = self.unrestrictedTraverse(archive_path)
        except KeyError:
          continue
        if archive.getCatalogId() == self.destination_sql_catalog_id:
          archive_obj_list.append(archive)
    else:
Aurel's avatar
Aurel committed
743
      # otherwise take all archive in use to know where object must go
744 745 746 747 748 749 750
      archive_obj_list = []
      for archive_path in archive_list:
        try:
          archive = self.unrestrictedTraverse(archive_path)
        except KeyError:
          continue
        archive_obj_list.append(archive)
751 752 753 754 755 756

    archive_enabled = (not disable_archive) \
            and (archiving or (archive_obj_list and sql_catalog_id is None))
    if archive_enabled:
      default_catalog = self.getSQLCatalog()

757
    # Construct list of object to catalogged
758
    current_catalog_object_list = []
759 760
    for obj in object_list:
      if hot_reindexing:
761
        try:
762 763 764 765 766 767
          url = obj.getPhysicalPath
        except AttributeError:
          raise CatalogError(
            "A cataloged object must support the 'getPhysicalPath' "
            "method if no unique id is provided when cataloging"
            )
768
        url = '/'.join(url())
769
        url_list.append(url)
770

771
      # either we are doing archiving, either we have used archive without a catalog specified
772 773
      if archive_enabled:
        goto_current_catalog = 0
774
        # check in which archive object must go if we defined archive
775 776 777 778
        catalog_id = None
        for archive in archive_obj_list:
          if archive.test(obj) is True:
            catalog_id = archive.getCatalogId()
779 780 781 782
            # if current catalog, no need to construct dict as it will be reindex now
            if catalog_id in (default_catalog.id, self.source_sql_catalog_id):
              goto_current_catalog = 1
              continue
783 784 785 786 787
            priority = archive.getPriority()
            if catalog_dict.has_key(catalog_id):
              catalog_dict[catalog_id]['obj'].append(obj)
            else:
              catalog_dict[catalog_id] = {'priority' : priority, 'obj' : [obj,]}
Aurel's avatar
Aurel committed
788
        if catalog_id is None and not archiving:
789
          # at least put object in current catalog if no archive match
Aurel's avatar
Aurel committed
790
          # and not doing archive
791
          goto_current_catalog = 1
792
      else:
793
        goto_current_catalog = 1
794

795
      if goto_current_catalog:
796
        current_catalog_object_list.append(obj)
797

798
    # run activity or execute for each archive depending on priority
799
    if catalog_dict:
800
      for catalog_id in catalog_dict.keys():
801 802 803
        if goto_current_catalog and catalog_id == default_catalog.id:
          # if we reindex in current catalog, do not relaunch an activity for this
          continue
804
        d = catalog_dict[catalog_id]
805 806 807
        # hot_reindexing is True when creating an object during a hot reindex, in this case, we don't want
        # to reindex it in destination catalog, it will be recorded an play only once
        if not hot_reindexing and self.hot_reindexing_state != HOT_REINDEXING_DOUBLE_INDEXING_STATE and \
808
               self.destination_sql_catalog_id == catalog_id:
809 810
          destination_catalog = self.getSQLCatalog(self.destination_sql_catalog_id)
          # reindex objects in destination catalog
811 812 813 814 815 816 817
          destination_catalog.catalogObjectList(
            self.wrapObjectList(
              object_value_list=d['obj'],
              catalog_value=destination_catalog,
            ),
            **kw
          )
818
        else:
819 820
          archive_catalog = self.getSQLCatalog(catalog_id)
          if immediate_reindex_archive:
821 822 823 824 825 826 827
            archive_catalog.catalogObjectList(
              self.wrapObjectList(
                object_value_list=d['obj'],
                catalog_value=archive_catalog,
              ),
              **kw
            )
828 829 830 831
          else:
            for obj in d['obj']:
              obj._reindexObject(sql_catalog_id=catalog_id, activate_kw = \
                                 {'priority': d['priority']}, disable_archive=1, **kw)
832

833
    if catalog is not None:
834 835 836 837 838 839 840 841
      if current_catalog_object_list:
        catalog.catalogObjectList(
          self.wrapObjectList(
            object_value_list=current_catalog_object_list,
            catalog_value=catalog,
          ),
          **kw
        )
842
      if hot_reindexing:
843
        destination_catalog = self.getSQLCatalog(self.destination_sql_catalog_id)
844
        if destination_catalog.id != catalog.id:
845
          if self.hot_reindexing_state == HOT_REINDEXING_RECORDING_STATE:
846
            destination_catalog.recordObjectList(url_list, 1)
847 848 849 850 851 852 853 854
          elif object_list:
            destination_catalog.catalogObjectList(
              self.wrapObjectList(
                object_value_list=object_list,
                catalog_value=destination_catalog,
              ),
              **kw
            )
855

856
    object_list[:] = failed_object_list
857

858
  security.declareProtected(manage_zcatalog_entries, 'uncatalog_object')
859
  def uncatalog_object(self, uid=None,path=None, sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
860
    """ wrapper around catalog """
861 862
    if uid is None:
      raise TypeError, "sorry uncatalog_object supports only uid"
863
    default_catalog = self.getSQLCatalog()
864

865 866 867 868 869
    if sql_catalog_id is None:
      archive_list = []
      if getattr(self, "portal_archives", None) is not None:
        if len(self.portal_archives):
          archive_list = self.portal_archives.getArchiveList()
870

871 872 873 874 875 876 877 878 879 880 881 882
      if len(archive_list):
        for archive_path in archive_list:
          try:
            archive = self.unrestrictedTraverse(archive_path)
          except KeyError:
            continue
          catalog_id = archive.getCatalogId()
          if catalog_id != default_catalog.id:
            # only launch activity when not in current catalog
            self.activate(activity="SQLQueue", round_robin_scheduling=1,
                          priority=archive.getPriority()).uncatalog_object(uid=uid,path=path,
                                                                           sql_catalog_id=catalog_id)
883

884
    catalog = self.getSQLCatalog(sql_catalog_id)
885
    if catalog is not None:
886
      catalog.uncatalogObject(uid=uid,path=path)
887 888 889
      if self.hot_reindexing_state is not None and self.source_sql_catalog_id == catalog.id:
        destination_catalog = self.getSQLCatalog(self.destination_sql_catalog_id)
        if destination_catalog.id != catalog.id:
890
          if self.hot_reindexing_state == HOT_REINDEXING_RECORDING_STATE:
891 892
            destination_catalog.recordObjectList([uid], 0)
          else:
893
            destination_catalog.uncatalogObject(uid=uid)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
894

895

896
  security.declarePrivate('beforeUncatalogObject')
897 898 899 900 901 902
  def beforeUncatalogObject(self, uid=None,path=None, sql_catalog_id=None):
    """ wrapper around catalog """
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      catalog.beforeUncatalogObject(uid=uid,path=path)

903
  security.declarePrivate('beforeCatalogClear')
904 905 906 907
  def beforeCatalogClear(self):
    """ allow to override this method """
    pass

908
  security.declarePrivate('catalogTranslationList')
909 910 911 912 913 914 915
  def catalogTranslationList(self, object_list, sql_catalog_id=None):
    """Catalog translations.
    """
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      catalog.catalogTranslationList(object_list)

916
  security.declarePrivate('deleteTranslationList')
917 918 919 920 921 922 923
  def deleteTranslationList(self, sql_catalog_id=None):
    """Delete translations.
    """
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      catalog.deleteTranslationList()

924
  security.declarePrivate('uniqueValuesFor')
925
  def uniqueValuesFor(self, name, sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
926
    """ returns the unique values for a given FieldIndex """
927 928 929
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      return catalog.uniqueValuesFor(name)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
930

931 932
    return ()

933
  security.declarePrivate('getpath')
934
  def getpath(self, uid, sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
935 936 937
    """
    Return the path to a cataloged object given its uid
    """
938 939
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
940 941 942
      record = catalog.getRecordForUid(uid)
      if record is not None:
        return record.path
943 944
      else:
        return None
Jean-Paul Smets's avatar
Jean-Paul Smets committed
945 946
  getPath = getpath

947
  security.declarePrivate('hasPath')
948
  def hasPath(self, path, sql_catalog_id=None):
Sebastien Robin's avatar
Sebastien Robin committed
949
    """
950
    Checks if path is catalogued
Sebastien Robin's avatar
Sebastien Robin committed
951
    """
952 953 954
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      return catalog.hasPath(path)
Sebastien Robin's avatar
Sebastien Robin committed
955

956
  security.declarePrivate('getobject')
957
  def getobject(self, uid, REQUEST=None, sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
958 959 960
    """
    Return a cataloged object given its uid
    """
961 962 963
    if REQUEST is not None and sql_catalog_id is None:
      sql_catalog_id = REQUEST.get('sql_catalog_id', None)

964 965
    path = self.getpath(uid, sql_catalog_id=sql_catalog_id)
    obj = self.aq_parent.unrestrictedTraverse(path)
966
    if obj is None:
Jean-Paul Smets's avatar
Jean-Paul Smets committed
967 968
      if REQUEST is None:
        REQUEST=self.REQUEST
969
      obj = self.resolve_url(path, REQUEST)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
970 971 972
    return obj
  getObject = getobject

973
  security.declarePrivate('getObjectList')
974
  def getObjectList(self, uid_list, REQUEST=None, sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
975 976 977 978 979
    """
    Return a cataloged object given its uid
    """
    obj_list = []
    for uid in uid_list:
980
      obj_list.append(self.getObject(uid, REQUEST, sql_catalog_id=sql_catalog_id))
Jean-Paul Smets's avatar
Jean-Paul Smets committed
981 982
    return obj_list

983
  security.declarePrivate('getMetadataForUid')
984
  def getMetadataForUid(self, rid, sql_catalog_id=None):
985 986
    # !!! do not use docstring here (CVE-2011-0720).
    # return the correct metadata for the cataloged uid
987 988 989 990
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      return catalog.getMetadataForUid(int(rid))
    return {}
Jean-Paul Smets's avatar
Jean-Paul Smets committed
991

992
  security.declarePrivate('getIndexDataForUid')
993
  def getIndexDataForUid(self, rid, sql_catalog_id=None):
994 995
    # !!! do not use docstring here (CVE-2011-0720).
    # return the current index contents for the specific uid
996 997 998 999
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      return catalog.getIndexDataForUid(rid)
    return {}
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1000 1001 1002 1003 1004

  # Aliases
  getMetadataForRID = getMetadataForUid
  getIndexDataForRID = getIndexDataForUid

1005
  security.declarePrivate('schema')
1006 1007
  def schema(self, sql_catalog_id=None):
    return self.getColumnIds(sql_catalog_id=sql_catalog_id)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1008

1009
  security.declarePrivate('indexes')
1010 1011
  def indexes(self, sql_catalog_id=None):
    return self.getColumnIds(sql_catalog_id=sql_catalog_id)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1012

1013
  security.declarePrivate('names')
1014 1015 1016 1017 1018
  def names(self, sql_catalog_id=None):
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      return catalog.names
    return {}
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1019

1020
  security.declarePrivate('getColumnIds')
1021 1022 1023 1024 1025
  def getColumnIds(self, sql_catalog_id=None):
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      return catalog.getColumnIds()
    return []
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1026

1027
  security.declarePublic('hasColumn')
1028 1029 1030 1031 1032 1033
  def hasColumn(self, column, sql_catalog_id=None):
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      return catalog.hasColumn(column)
    return False

1034
  security.declarePrivate('getAttributesForColumn')
1035
  def getAttributesForColumn(self, column, sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1036 1037 1038
    """
      Return the attribute names as a single string
    """
1039
    return string.join(self.names(sql_catalog_id=sql_catalog_id).get(column, ('',)),' ')
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1040

1041 1042 1043 1044 1045
  def _searchable_arguments(self, sql_catalog_id=None):
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      return catalog.getColumnIds(sql_catalog_id=sql_catalog_id)
    return []
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1046

1047
  security.declarePrivate('editSchema')
1048 1049 1050 1051
  def editSchema(self,names, sql_catalog_id=None):
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      catalog.editSchema(names)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1052

1053
  def _searchable_result_columns(self, sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1054
    r = []
1055 1056 1057 1058 1059 1060 1061 1062 1063
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      for name in catalog.getColumnIds():
        i = {}
        i['name'] = name
        i['type'] = 's'
        i['parser'] = str
        i['width'] = 8
        r.append(i)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1064 1065 1066 1067 1068 1069
    r.append({'name': 'data_record_id_',
          'type': 's',
          'parser': str,
          'width': 8})
    return r

1070
  security.declarePublic('buildSQLQuery')
1071
  def buildSQLQuery(self, REQUEST=None, query_table='catalog', sql_catalog_id=None, **kw):
1072 1073 1074 1075
    """
      Build a SQL query from keywords.
      If query_table is specified, it is used as the table name instead of 'catalog'.
    """
1076 1077 1078 1079
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      return catalog.buildSQLQuery(REQUEST=REQUEST, query_table=query_table, **kw)
    return ''
1080

1081 1082 1083 1084
  # Compatibility SQL Sql
  security.declarePublic('buildSqlQuery')
  buildSqlQuery = buildSQLQuery

1085
  security.declarePublic('searchResults')
1086
  def searchResults(self, REQUEST=None, sql_catalog_id=None, **kw):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1087 1088 1089 1090 1091
    """
    Search the catalog according to the ZTables search interface.
    Search terms can be passed in the REQUEST or as keyword
    arguments.
    """
1092 1093
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
1094
      return catalog.searchResults(REQUEST, **kw)
1095
    return []
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1096 1097 1098

  __call__=searchResults

1099
  security.declarePublic('countResults')
1100
  def countResults(self, REQUEST=None, sql_catalog_id=None, **kw):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1101 1102 1103
    """
    Counts the number of items which satisfy the query defined in kw.
    """
1104 1105
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
1106
      return catalog.countResults(REQUEST, **kw)
1107
    return []
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1108 1109 1110 1111 1112

## this stuff is so the find machinery works

  meta_types=() # Sub-object types that are specific to this object

1113
  security.declarePrivate('valid_roles')
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133
  def valid_roles(self):
    "Return list of valid roles"
    obj=self
    dict={}
    dup =dict.has_key
    x=0
    while x < 100:
      if hasattr(obj, '__ac_roles__'):
        roles=obj.__ac_roles__
        for role in roles:
          if not dup(role):
            dict[role]=1
      if not hasattr(obj, 'aq_parent'):
        break
      obj=obj.aq_parent
      x=x+1
    roles=dict.keys()
    roles.sort()
    return roles

1134
  security.declarePrivate('ZopeFindAndApply')
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1135 1136 1137 1138 1139 1140
  def ZopeFindAndApply(self, obj, obj_ids=None, obj_metatypes=None,
             obj_searchterm=None, obj_expr=None,
             obj_mtime=None, obj_mspec=None,
             obj_permission=None, obj_roles=None,
             search_sub=0,
             REQUEST=None, result=None, pre='',
1141 1142
             apply_func=None, apply_path='',
             sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181
    """Zope Find interface and apply

    This is a *great* hack.  Zope find just doesn't do what we
    need here; the ability to apply a method to all the objects
    *as they're found* and the need to pass the object's path into
    that method.

    """

    if result is None:
      result=[]

      if obj_metatypes and 'all' in obj_metatypes:
        obj_metatypes=None

      if obj_mtime and type(obj_mtime)==type('s'):
        obj_mtime=DateTime(obj_mtime).timeTime()

      if obj_permission:
        obj_permission=p_name(obj_permission)

      if obj_roles and type(obj_roles) is type('s'):
        obj_roles=[obj_roles]

      if obj_expr:
        # Setup expr machinations
        md=td()
        obj_expr=(Eval(obj_expr), md, md._push, md._pop)

    base=obj
    if hasattr(obj, 'aq_base'):
      base=obj.aq_base

    if not hasattr(base, 'objectItems'):
      return result
    try:  items=obj.objectItems()
    except: return result

    try: add_result=result.append
Yoshinori Okuji's avatar
Yoshinori Okuji committed
1182
    except AttributeError:
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1183 1184 1185 1186 1187
      raise AttributeError, `result`

    for id, ob in items:
      if pre: p="%s/%s" % (pre, id)
      else:   p=id
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1188

Jean-Paul Smets's avatar
Jean-Paul Smets committed
1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216
      dflag=0
      if hasattr(ob, '_p_changed') and (ob._p_changed == None):
        dflag=1

      if hasattr(ob, 'aq_base'):
        bs=ob.aq_base
      else: bs=ob

      if (
        (not obj_ids or absattr(bs.id) in obj_ids)
        and
        (not obj_metatypes or (hasattr(bs, 'meta_type') and
         bs.meta_type in obj_metatypes))
        and
        (not obj_searchterm or
         (hasattr(ob, 'PrincipiaSearchSource') and
          string.find(ob.PrincipiaSearchSource(), obj_searchterm) >= 0
          ))
        and
        (not obj_expr or expr_match(ob, obj_expr))
        and
        (not obj_mtime or mtime_match(ob, obj_mtime, obj_mspec))
        and
        ( (not obj_permission or not obj_roles) or \
           role_match(ob, obj_permission, obj_roles)
        )
        ):
        if apply_func:
1217
          apply_func(ob, (apply_path+'/'+p), sql_catalog_id=sql_catalog_id)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228
        else:
          add_result((p, ob))
          dflag=0

      if search_sub and hasattr(bs, 'objectItems'):
        self.ZopeFindAndApply(ob, obj_ids, obj_metatypes,
                    obj_searchterm, obj_expr,
                    obj_mtime, obj_mspec,
                    obj_permission, obj_roles,
                    search_sub,
                    REQUEST, result, p,
1229 1230
                    apply_func, apply_path,
                    sql_catalog_id)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1231 1232 1233 1234
      if dflag: ob._p_deactivate()

    return result

1235
  security.declarePrivate('resolve_url')
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1236 1237 1238 1239 1240 1241 1242 1243 1244 1245
  def resolve_url(self, path, REQUEST):
    """
    Attempt to resolve a url into an object in the Zope
    namespace. The url may be absolute or a catalog path
    style url. If no object is found, None is returned.
    No exceptions are raised.
    """
    script=REQUEST.script
    if string.find(path, script) != 0:
      path='%s/%s' % (script, path)
1246
    try:
Yoshinori Okuji's avatar
Yoshinori Okuji committed
1247
      return REQUEST.resolve_url(path)
1248
    except ConflictError:
Yoshinori Okuji's avatar
Yoshinori Okuji committed
1249
      raise
1250
    except:
Yoshinori Okuji's avatar
Yoshinori Okuji committed
1251
      pass
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1252

1253
  security.declarePrivate('resolve_path')
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1254
  def resolve_path(self, path):
1255 1256 1257 1258 1259
    # !!! do not use docstring here (CVE-2011-0720).
    # Attempt to resolve a url into an object in the Zope
    # namespace. The url may be absolute or a catalog path
    # style url. If no object is found, None is returned.
    # No exceptions are raised.
1260
    try:
Yoshinori Okuji's avatar
Yoshinori Okuji committed
1261 1262 1263
      return self.unrestrictedTraverse(path)
    except ConflictError:
      raise
1264
    except:
Yoshinori Okuji's avatar
Yoshinori Okuji committed
1265
      pass
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1266

1267
  security.declarePrivate('manage_normalize_paths')
1268
  def manage_normalize_paths(self, REQUEST, sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1269 1270 1271 1272 1273
    """Ensure that all catalog paths are full physical paths

    This should only be used with ZCatalogs in which all paths can
    be resolved with unrestrictedTraverse."""

1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288
    if sql_catalog_id is None:
      sql_catalog_id = REQUEST.get('sql_catalog_id', None)

    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      paths = catalog.paths
      uids = catalog.uids
      unchanged = 0
      fixed = []
      removed = []

      for path, rid in uids.items():
        ob = None
        if path[:1] == '/':
          ob = self.resolve_url(path[1:],REQUEST)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1289
        if ob is None:
1290 1291 1292 1293 1294 1295 1296 1297 1298
          ob = self.resolve_url(path, REQUEST)
          if ob is None:
            removed.append(path)
            continue
        ppath = string.join(ob.getPhysicalPath(), '/')
        if path != ppath:
          fixed.append((path, ppath))
        else:
          unchanged = unchanged + 1
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1299

1300 1301 1302 1303 1304 1305 1306
      for path, ppath in fixed:
        rid = uids[path]
        del uids[path]
        paths[rid] = ppath
        uids[ppath] = rid
      for path in removed:
        self.uncatalog_object(path, sql_catalog_id=sql_catalog_id)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1307 1308 1309 1310 1311 1312

    return MessageDialog(title='Done Normalizing Paths',
      message='%s paths normalized, %s paths removed, and '
          '%s unchanged.' % (len(fixed), len(removed), unchanged),
      action='./manage_main')

1313
  security.declarePrivate('getTableIds')
1314
  def getTableIds(self, sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1315 1316
    """Returns all tables of this catalog
    """
1317 1318 1319 1320
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      return catalog.getTableIds()
    return []
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1321

1322
  security.declarePrivate('getCatalogSearchResultKeys')
1323
  def getCatalogSearchResultKeys(self, sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1324 1325 1326
    """Return selected tables of catalog which are used in JOIN.
       catalaog is always first
    """
1327 1328 1329 1330
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      return catalog.sql_search_result_keys
    return []
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1331

1332
  security.declarePrivate('getCatalogSearchTableIds')
1333
  def getCatalogSearchTableIds(self, sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1334 1335 1336
    """Return selected tables of catalog which are used in JOIN.
       catalaog is always first
    """
1337 1338 1339 1340
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      return catalog.getCatalogSearchTableIds()
    return []
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1341

1342
  security.declarePrivate('getResultColumnIds')
1343
  def getResultColumnIds(self, sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1344 1345 1346
    """Return selected tables of catalog which are used
       as metadata
    """
1347 1348 1349 1350
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      return catalog.getResultColumnIds()
    return []
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1351

1352
  security.declarePrivate('getCatalogMethodIds')
1353
  def getCatalogMethodIds(self, sql_catalog_id=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1354 1355 1356
    """Find Z SQL methods in the current folder and above
    This function return a list of ids.
    """
1357 1358 1359 1360 1361
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      return catalog.getCatalogMethodIds()
    return {}

1362
  security.declareProtected(manage_zcatalog_entries, 'manage_editFilter')
1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375
  def manage_editFilter(self, REQUEST=None, RESPONSE=None, URL1=None, sql_catalog_id=None):
    """
    This methods allows to set a filter on each zsql method called,
    so we can test if we should or not call a zsql method, so we can
    increase a lot the speed.
    """
    if REQUEST is not None and sql_catalog_id is None:
      sql_catalog_id = REQUEST.get('sql_catalog_id', None)

    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      catalog.manage_editFilter(REQUEST=REQUEST, RESPONSE=RESPONSE, URL1=URL1)

1376
  security.declarePrivate('getFilterableMethodList')
1377 1378 1379 1380 1381 1382 1383 1384
  def getFilterableMethodList(self, sql_catalog_id=None):
    """
    Returns only zsql methods wich catalog or uncatalog objets
    """
    catalog = self.getSQLCatalog(sql_catalog_id)
    if catalog is not None:
      return catalog.getFilterableMethodList()
    return []
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1385 1386


1387
InitializeClass(ZCatalog)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416


def p_name(name):
  return '_' + string.translate(name, name_trans) + '_Permission'

def absattr(attr):
  if callable(attr): return attr()
  return attr


class td(RestrictedDTML, TemplateDict):
  pass

def expr_match(ob, ed, c=InstanceDict, r=0):
  e, md, push, pop=ed
  push(c(ob, md))
  try: r=e.eval(md)
  finally:
    pop()
    return r

def mtime_match(ob, t, q, fn=hasattr):
  if not fn(ob, '_p_mtime'):
    return 0
  return q=='<' and (ob._p_mtime < t) or (ob._p_mtime > t)

def role_match(ob, permission, roles, lt=type([]), tt=type(())):
  pr=[]
  fn=pr.append
1417

Jean-Paul Smets's avatar
Jean-Paul Smets committed
1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442
  while 1:
    if hasattr(ob, permission):
      p=getattr(ob, permission)
      if type(p) is lt:
        map(fn, p)
        if hasattr(ob, 'aq_parent'):
          ob=ob.aq_parent
          continue
        break
      if type(p) is tt:
        map(fn, p)
        break
      if p is None:
        map(fn, ('Manager', 'Anonymous'))
        break

    if hasattr(ob, 'aq_parent'):
      ob=ob.aq_parent
      continue
    break

  for role in roles:
    if not (role in pr):
      return 0
  return 1