erp5_core: add ERP5Site_resynchroniseCatalogSince script.

bt5/erp5_core_test/TestTemplateItem/portal_components/test.erp5.testERP5Core.py

@@ -28,6 +28,7 @@
 ##############################################################################
 
 import collections
+import csv
 import httplib
 import urlparse
 import base64
@@ -38,9 +39,11 @@ from AccessControl.SecurityManagement import newSecurityManager
 from DateTime import DateTime
 from Testing import ZopeTestCase
 
+from Products.ERP5Type.Utils import bytes2str, str2unicode
 from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase
 from Products.ERP5Type.tests.utils import DummyTranslationService
 
+from io import StringIO
 from zExceptions import Unauthorized
 
 if 1: # BBB
@@ -120,6 +123,11 @@ class TestERP5Core(ERP5TypeTestCase, ZopeTestCase.Functional):
     if 'test_folder' in self.portal.objectIds():
       self.portal.manage_delObjects(['test_folder'])
     self.portal.portal_selections.setSelectionFor('test_selection', None)
+    person_module = self.portal.person_module
+    person_id_list = list(person_module.objectIds())
+    if person_id_list:
+      person_module.manage_delObjects(ids=person_id_list)
+    self.portal.portal_caches.clearCache()
     self.tic()
 
   def test_01_ERP5Site_createModule(self, quiet=quiet, run=run_all_test):
@@ -800,3 +808,61 @@ class TestERP5Core(ERP5TypeTestCase, ZopeTestCase.Functional):
       len(self.portal.z_get_deleted_path_list(timestamp=DateTime() - 1)),
       len(person_list) - 5,
     )
+
+  def test_ERP5Site_resynchroniseCatalogSince(self):
+    person = self.portal.person_module.newContent(
+      portal_type='Person',
+      title='test1',
+    )
+    person2 = self.portal.person_module.newContent(
+      portal_type='Person',
+      title='test2',
+    )
+    self.tic()
+    query = self.portal.erp5_sql_connection().query
+    query('UPDATE catalog SET title="test1bis" WHERE uid=%s' % person.getUid())
+    self.assertEqual(
+      0,
+      len(self.portal.portal_catalog(portal_type='Person', title='test1')),
+    )
+    # simulate a document being deleted and then the database being
+    # truncated before that deletion.
+    self.portal.portal_catalog.beforeUncatalogObject(
+      uid=person2.getUid(),
+      path=person2.getPath(),
+    )
+    self.assertEqual(
+      0,
+      len(self.portal.portal_catalog(uid=person2.getUid())),
+    )
+    response = self.publish(
+      '/%s/ERP5Site_resynchroniseCatalogSince?from_date=%s' % (
+        self.portal.getId(), DateTime() - 1
+      ),
+      self.auth,
+    )
+    self.assertEqual(response.getStatus(), 200)
+    io_ = StringIO(str2unicode(bytes2str(response.getBody())))
+    response_dict = {x['path']:x for x in csv.DictReader(io_)}
+    person_row = response_dict[person.getPath()]
+    self.assertEqual(person_row['status'], 'present')
+    self.assertEqual(person_row['catalog title'], 'test1bis')
+    self.assertEqual(person_row['zodb title'], 'test1')
+    person2_row = response_dict[person2.getPath()]
+    self.assertEqual(person2_row['status'], 'present')
+    self.assertEqual(person2_row['catalog title'], '')
+    self.assertEqual(person2_row['zodb title'], 'test2')
+    self.portal.ERP5Site_resynchroniseCatalogSince(
+      RESPONSE=self.portal.REQUEST.RESPONSE,
+      from_date=DateTime() - 1,
+      dry=False,
+    )
+    self.tic()
+    self.assertEqual(
+      1,
+      len(self.portal.portal_catalog(portal_type='Person', title='test1')),
+    )
+    self.assertEqual(
+      1,
+      len(self.portal.portal_catalog(uid=person2.getUid())),
+    )

product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/ERP5Site_resynchroniseCatalogSince.py

+# Guards: role: Manager, as it is meaningless (and potentially very expensive) to use this script when one cannot typically see all documents.
+import csv
+import six
+from collections import OrderedDict
+from io import BytesIO, StringIO
+from erp5.component.module.Log import log
+
+if not isinstance(from_date, DateTime):
+  from_date = DateTime(from_date)
+assert from_date < DateTime(), from_date
+def always(document_value):
+  return True
+def if_has_workflow(document_value):
+  return hasattr(document_value, 'workflow_history')
+def unity(value):
+  return value
+def strftime(value):
+  if value is None:
+    return None
+  value = value.toZone('UTC')
+  return DateTime(
+    value.year(),
+    value.month(),
+    value.day(),
+    value.hour(),
+    value.minute(),
+    int(value.second()),
+    'UTC'
+  )
+select_dict = OrderedDict((
+  # column                getter                only if getter present ?  convert zodb  test zodb
+  ('uid',               ('getUid',              False,                    unity,        always)),
+  ('title',             ('getTitle',            False,                    unity,        always)),
+  ('portal_type',       ('getPortalType',       False,                    unity,        always)),
+  ('creation_date',     ('getCreationDate',     False,                    strftime,     always)),
+  ('modification_date', ('getModificationDate', False,                    strftime,     if_has_workflow)),
+  ('validation_state',  ('getValidationState',  True,                     unity,        always)),
+  ('simulation_state',  ('getSimulationState',  True,                     unity,        always)),
+))
+column_list = select_dict.keys()
+
+portal = context.getPortalObject()
+traverse = portal.restrictedTraverse
+portal_catalog = portal.portal_catalog
+# XXX: abusing CMFActivity's privilege elevation: restricted python is not allowed to call unindexObject
+unindexObject = portal_catalog.activate(activity='SQLQueue',
+  group_method_id='portal_catalog/uncatalogObjectList',
+).unindexObject
+
+if six.PY2:
+  io_ = BytesIO()
+else:
+  io_ = StringIO()
+csv_writer = csv.writer(io_)
+
+row_list = portal_catalog(
+  select_list=[e for e in column_list if e != 'uid'],
+  indexation_timestamp={
+    'query': from_date,
+    'range': '>=',
+  },
+).dictionaries()
+row_list.extend(
+  portal.z_get_deleted_path_list(timestamp=from_date).dictionaries()
+)
+log('Processing %i rows...' % len(row_list))
+
+column_title_list = ['status', 'has difference', 'path']
+for column in column_list:
+  column_title_list.append('catalog ' + column)
+  column_title_list.append('zodb ' + column)
+csv_writer.writerow(column_title_list)
+
+row_count = len(row_list)
+for i, row in enumerate(row_list):
+  zodb_property_dict = {}
+  has_difference = False
+  try:
+    document_value = traverse(row['path'])
+    __traceback_info__ = (row['path'], document_value)
+  except KeyError:
+    status = 'missing'
+    has_difference = True
+    unindexObject(uid=row['uid'])
+  else:
+    status = 'present'
+    if document_value.getUid() != row['uid']:
+      unindexObject(uid=row['uid'])
+    for column_name, (getter_name, may_be_missing, zodb_filter, document_filter) in select_dict.items():
+      if (
+        (not may_be_missing or hasattr(document_value, getter_name)) and
+        document_filter(document_value)
+      ):
+        value_from_document = zodb_filter(getattr(document_value, getter_name)())
+        zodb_property_dict[column_name] = value_from_document
+        has_difference |= value_from_document != row.get(column_name)
+    # Reindex even if no difference was found
+    document_value.reindexObject()
+  output_value_list = [
+    status,
+    int(has_difference), # integers are easier to manage than "True" or "False" in libreoffice
+    row['path'],
+  ]
+  output_value_list_append = output_value_list.append
+  for column in column_list:
+    output_value_list_append(row.get(column))
+    output_value_list_append(zodb_property_dict.get(column))
+  csv_writer.writerow(output_value_list)
+  if i % 1000 == 0:
+    log('processed %i/%i lines' % (i, row_count))
+
+io_.seek(0)
+result = io_.getvalue()
+if dry:
+  result += 'dry run'
+  if six.PY3:
+    result = result.encode()
+  RESPONSE.write(result)
+  raise Exception('dry run')
+log(result)
+return result

product/ERP5/bootstrap/erp5_core/SkinTemplateItem/portal_skins/erp5_core/ERP5Site_resynchroniseCatalogSince.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>_bind_names</string> </key>
+            <value>
+              <object>
+                <klass>
+                  <global name="_reconstructor" module="copy_reg"/>
+                </klass>
+                <tuple>
+                  <global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
+                  <global name="object" module="__builtin__"/>
+                  <none/>
+                </tuple>
+                <state>
+                  <dictionary>
+                    <item>
+                        <key> <string>_asgns</string> </key>
+                        <value>
+                          <dictionary>
+                            <item>
+                                <key> <string>name_container</string> </key>
+                                <value> <string>container</string> </value>
+                            </item>
+                            <item>
+                                <key> <string>name_context</string> </key>
+                                <value> <string>context</string> </value>
+                            </item>
+                            <item>
+                                <key> <string>name_m_self</string> </key>
+                                <value> <string>script</string> </value>
+                            </item>
+                            <item>
+                                <key> <string>name_subpath</string> </key>
+                                <value> <string>traverse_subpath</string> </value>
+                            </item>
+                          </dictionary>
+                        </value>
+                    </item>
+                  </dictionary>
+                </state>
+              </object>
+            </value>
+        </item>
+        <item>
+            <key> <string>_params</string> </key>
+            <value> <string>RESPONSE, from_date, dry=1</string> </value>
+        </item>
+        <item>
+            <key> <string>guard</string> </key>
+            <value>
+              <persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
+            </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>ERP5Site_resynchroniseCatalogSince</string> </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="2" aka="AAAAAAAAAAI=">
+    <pickle>
+      <global name="Guard" module="Products.DCWorkflow.Guard"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>roles</string> </key>
+            <value>
+              <tuple>
+                <string>Manager</string>
+              </tuple>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+</ZopeData>