Skip to content

S
slapos.core
Commit 1822be3b authored by Rafael Monnerat's avatar Rafael Monnerat
Browse files
Options

slapos_mysql_innodb_catalog: Index consistency_error 

    We index now for very few portal types the bool(checkConsistency()) in order to search inconsistent documents.

    The indexation is limited to the scope we must monitor rather them all documents for 2 major reasons:

        - Too much useless constrants on erp5 code base, not applicable for our project (ie.: On person)
        - Performance on indexation is penalized from this extra cost, specially when index large trees (Deliveries or open orders)**

    ** Not so much, however, on a scale of millions of documents it can make a major difference, so this commits keeps up to the minimal implementation.
parent 9661e882
Hide whitespace changes
Inline Side-by-side
Showing with 685 additions and 1 deletion
master/bt5/slapos_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z0_drop_consistency.catalog_keys.xml 0 → 100644
View file @ 1822be3b
<catalog_method>
<item key="sql_clear_catalog" type="int">
<value>1</value>
</item>
</catalog_method>
master/bt5/slapos_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z0_drop_consistency.xml 0 → 100644
View file @ 1822be3b
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="SQL Method" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_col</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>allow_simple_one_argument_traversal</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>arguments_src</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>cache_time_</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>class_file_</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>class_name_</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>connection_hook</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>connection_id</string> </key>
<value> <string>erp5_sql_connection</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>z0_drop_consistency</string> </value>
</item>
<item>
<key> <string>max_cache_</string> </key>
<value> <int>100</int> </value>
</item>
<item>
<key> <string>max_rows_</string> </key>
<value> <int>1000</int> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>SQL Method</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>z0_drop_consistency</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
master/bt5/slapos_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z0_uncatalog_consistency.catalog_keys.xml 0 → 100644
View file @ 1822be3b
<catalog_method>
<item key="sql_uncatalog_object" type="int">
<value>1</value>
</item>
</catalog_method>
master/bt5/slapos_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z0_uncatalog_consistency.xml 0 → 100644
View file @ 1822be3b
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="SQL Method" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>allow_simple_one_argument_traversal</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>arguments_src</string> </key>
<value> <string>uid</string> </value>
</item>
<item>
<key> <string>cache_time_</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>class_file_</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>class_name_</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>connection_hook</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>connection_id</string> </key>
<value> <string>erp5_sql_connection</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>z0_uncatalog_consistency</string> </value>
</item>
<item>
<key> <string>max_cache_</string> </key>
<value> <int>100</int> </value>
</item>
<item>
<key> <string>max_rows_</string> </key>
<value> <int>1000</int> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>SQL Method</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>z0_uncatalog_consistency</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
master/bt5/slapos_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z_catalog_consistency.catalog_keys.xml 0 → 100644
View file @ 1822be3b
<catalog_method>
<item key="sql_catalog_object_list" type="int">
<value>1</value>
</item>
</catalog_method>
master/bt5/slapos_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z_catalog_consistency.sql 0 → 100644
View file @ 1822be3b
DELETE FROM
consistency
WHERE
<dtml-in uid>
uid=<dtml-sqlvar sequence-item type="int"><dtml-if sequence-end><dtml-else> OR </dtml-if>
</dtml-in>
;
<dtml-var "'\0'">
INSERT INTO
consistency
VALUES
<dtml-in prefix="loop" expr="_.range(_.len(uid))">
(
<dtml-sqlvar expr="uid[loop_item]" type="int">,
<dtml-sqlvar expr="int(len(checkConsistency[loop_item]) > 0)" type="int">
)
<dtml-if sequence-end><dtml-else>,</dtml-if>
</dtml-in>
master/bt5/slapos_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z_catalog_consistency.xml 0 → 100644
View file @ 1822be3b
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="SQL Method" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>allow_simple_one_argument_traversal</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>arguments_src</string> </key>
<value> <string>uid\n
checkConsistency</string> </value>
</item>
<item>
<key> <string>cache_time_</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>class_file_</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>class_name_</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>connection_hook</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>connection_id</string> </key>
<value> <string>erp5_sql_connection</string> </value>
</item>
<item>
<key> <string>expression</string> </key>
<value> <string>python: context.getPortalType() in ["Sale Invoice Transaction", "Subscription Request"]</string> </value>
</item>
<item>
<key> <string>expression_cache_key</string> </key>
<value>
<tuple>
<string>portal_type</string>
</tuple>
</value>
</item>
<item>
<key> <string>filtered</string> </key>
<value> <int>1</int> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>z_catalog_consistency</string> </value>
</item>
<item>
<key> <string>max_cache_</string> </key>
<value> <int>100</int> </value>
</item>
<item>
<key> <string>max_rows_</string> </key>
<value> <int>1000</int> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>SQL Method</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>z_catalog_consistency</string> </value>
</item>
<item>
<key> <string>type</string> </key>
<value>
<tuple/>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
master/bt5/slapos_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z_create_consistency.catalog_keys.xml 0 → 100644
View file @ 1822be3b
<catalog_method>
<item key="sql_clear_catalog" type="int">
<value>1</value>
</item>
</catalog_method>
master/bt5/slapos_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z_create_consistency.sql 0 → 100644
View file @ 1822be3b
CREATE TABLE `consistency` (
`uid` BIGINT UNSIGNED NOT NULL,
`consistency_error` BOOL DEFAULT 0,
PRIMARY KEY (`uid`, `consistency_error`)
) ENGINE=InnoDB;
master/bt5/slapos_mysql_innodb_catalog/CatalogMethodTemplateItem/portal_catalog/erp5_mysql_innodb/z_create_consistency.xml 0 → 100644
View file @ 1822be3b
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="SQL Method" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_col</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>allow_simple_one_argument_traversal</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>arguments_src</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>cache_time_</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>class_file_</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>class_name_</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>connection_hook</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>connection_id</string> </key>
<value> <string>erp5_sql_connection</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>z_create_consistency</string> </value>
</item>
<item>
<key> <string>max_cache_</string> </key>
<value> <int>100</int> </value>
</item>
<item>
<key> <string>max_rows_</string> </key>
<value> <int>1000</int> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>SQL Method</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>z_create_consistency</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
master/bt5/slapos_mysql_innodb_catalog/CatalogResultTableTemplateItem/result_table_list.xml
View file @ 1822be3b
<key_list> <key_list>
<key>compute_partition</key> <key>compute_partition</key>
<key>consistency</key>
<key>slapos_item</key> <key>slapos_item</key>
<key>software_instance_tree</key> <key>software_instance_tree</key>
</key_list> </key_list>
\ No newline at end of file
master/bt5/slapos_mysql_innodb_catalog/TestTemplateItem/portal_components/test.erp5.testSlapOSCatalogConsistency.py 0 → 100644
View file @ 1822be3b
# -*- coding: utf-8 -*-
##############################################################################
#
# Copyright (C) 2013-2019 Nexedi SA and Contributors.
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
#
##############################################################################
from erp5.component.test.SlapOSTestCaseMixin import SlapOSTestCaseMixin
import transaction
def checkConsistencyWithError(self, **kw):
return ["Inconsistent"]
def checkConsistencyWithoutError(self, **kw):
return []
class TestSlapOSCatalogConsistency(SlapOSTestCaseMixin):
def afterSetUp(self):
SlapOSTestCaseMixin.afterSetUp(self)
def testSaleInvoiceTransaction(self):
from Products.ERP5Type.Base import Base
sale_invoice_transaction = self.portal.accounting_module.newContent(
portal_type="Sale Invoice Transaction"
)
original_checkConsistency = Base.checkConsistency
Base.checkConsistency = checkConsistencyWithoutError
try:
transaction.commit()
sale_invoice_transaction.immediateReindexObject()
self.tic()
document = self.portal.portal_catalog.getResultValue(
portal_type="Sale Invoice Transaction",
consistency_error=0,
uid=sale_invoice_transaction.getUid()
)
self.assertEqual(document, sale_invoice_transaction)
document = self.portal.portal_catalog.getResultValue(
portal_type="Sale Invoice Transaction",
consistency_error=1,
uid=sale_invoice_transaction.getUid()
)
self.assertEqual(document, None)
Base.checkConsistency = checkConsistencyWithError
transaction.commit()
sale_invoice_transaction.immediateReindexObject()
self.tic()
document = self.portal.portal_catalog.getResultValue(
portal_type="Sale Invoice Transaction",
consistency_error=1,
uid=sale_invoice_transaction.getUid()
)
self.assertEqual(document, sale_invoice_transaction)
document = self.portal.portal_catalog.getResultValue(
portal_type="Sale Invoice Transaction",
consistency_error=0,
uid=sale_invoice_transaction.getUid(),
)
self.assertEqual(document, None)
finally:
Base.checkConsistency = original_checkConsistency
def testSubscriptionRequest(self):
from Products.ERP5Type.Base import Base
subscription_request = self.portal.subscription_request_module.newContent(
portal_type="Subscription Request"
)
original_checkConsistency = Base.checkConsistency
Base.checkConsistency = checkConsistencyWithoutError
try:
transaction.commit()
subscription_request.immediateReindexObject()
self.tic()
document = self.portal.portal_catalog.getResultValue(
portal_type="Subscription Request",
consistency_error=0,
uid=subscription_request.getUid()
)
self.assertEqual(document, subscription_request)
document = self.portal.portal_catalog.getResultValue(
portal_type="Subscription Request",
consistency_error=1,
uid=subscription_request.getUid()
)
self.assertEqual(document, None)
Base.checkConsistency = checkConsistencyWithError
transaction.commit()
subscription_request.immediateReindexObject()
self.tic()
document = self.portal.portal_catalog.getResultValue(
portal_type="Subscription Request",
consistency_error=1,
uid=subscription_request.getUid()
)
self.assertEqual(document, subscription_request)
document = self.portal.portal_catalog.getResultValue(
portal_type="Subscription Request",
consistency_error=0,
uid=subscription_request.getUid(),
)
self.assertEqual(document, None)
finally:
Base.checkConsistency = original_checkConsistency
def testNotIndexedOnConsistencyTable(self):
from Products.ERP5Type.Base import Base
person = self.portal.person_module.newContent(
portal_type="Person"
)
original_checkConsistency = Base.checkConsistency
Base.checkConsistency = checkConsistencyWithoutError
try:
transaction.commit()
person.immediateReindexObject()
self.tic()
document = self.portal.portal_catalog.getResultValue(
portal_type="Person",
consistency_error=0,
uid=person.getUid()
)
self.assertEqual(document, None)
document = self.portal.portal_catalog.getResultValue(
portal_type="Person",
consistency_error=1,
uid=person.getUid()
)
self.assertEqual(document, None)
Base.checkConsistency = checkConsistencyWithError
transaction.commit()
person.immediateReindexObject()
self.tic()
document = self.portal.portal_catalog.getResultValue(
portal_type="Person",
consistency_error=1,
uid=person.getUid()
)
self.assertEqual(document, None)
document = self.portal.portal_catalog.getResultValue(
portal_type="Person",
consistency_error=0,
uid=person.getUid(),
)
self.assertEqual(document, None)
finally:
Base.checkConsistency = original_checkConsistency
\ No newline at end of file
master/bt5/slapos_mysql_innodb_catalog/TestTemplateItem/portal_components/test.erp5.testSlapOSCatalogConsistency.xml 0 → 100644
View file @ 1822be3b
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Test Component" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_recorded_property_dict</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
</value>
</item>
<item>
<key> <string>default_reference</string> </key>
<value> <string>testSlapOSCatalogConsistency</string> </value>
</item>
<item>
<key> <string>default_source_reference</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>test.erp5.testSlapOSCatalogConsistency</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Test Component</string> </value>
</item>
<item>
<key> <string>sid</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>text_content_error_message</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>text_content_warning_message</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>version</string> </key>
<value> <string>erp5</string> </value>
</item>
<item>
<key> <string>workflow_history</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="2" aka="AAAAAAAAAAI=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary/>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="3" aka="AAAAAAAAAAM=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary>
<item>
<key> <string>component_validation_workflow</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAQ=</string> </persistent>
</value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="4" aka="AAAAAAAAAAQ=">
<pickle>
<global name="WorkflowHistoryList" module="Products.ERP5Type.Workflow"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_log</string> </key>
<value>
<list>
<dictionary>
<item>
<key> <string>action</string> </key>
<value> <string>validate</string> </value>
</item>
<item>
<key> <string>validation_state</string> </key>
<value> <string>validated</string> </value>
</item>
</dictionary>
</list>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
master/bt5/slapos_mysql_innodb_catalog/bt/template_catalog_method_id_list
View file @ 1822be3b
erp5_mysql_innodb/z0_drop_compute_partition erp5_mysql_innodb/z0_drop_compute_partition
erp5_mysql_innodb/z0_drop_consistency
erp5_mysql_innodb/z0_drop_slapos_item erp5_mysql_innodb/z0_drop_slapos_item
erp5_mysql_innodb/z0_drop_software_instance_tree erp5_mysql_innodb/z0_drop_software_instance_tree
erp5_mysql_innodb/z0_uncatalog_compute_partition erp5_mysql_innodb/z0_uncatalog_compute_partition
erp5_mysql_innodb/z0_uncatalog_consistency
erp5_mysql_innodb/z0_uncatalog_email erp5_mysql_innodb/z0_uncatalog_email
erp5_mysql_innodb/z0_uncatalog_slapos_item erp5_mysql_innodb/z0_uncatalog_slapos_item
erp5_mysql_innodb/z0_uncatalog_software_instance erp5_mysql_innodb/z0_uncatalog_software_instance
erp5_mysql_innodb/z_catalog_compute_partition_list erp5_mysql_innodb/z_catalog_compute_partition_list
erp5_mysql_innodb/z_catalog_consistency
erp5_mysql_innodb/z_catalog_email_list erp5_mysql_innodb/z_catalog_email_list
erp5_mysql_innodb/z_catalog_item_list erp5_mysql_innodb/z_catalog_item_list
erp5_mysql_innodb/z_catalog_object_list erp5_mysql_innodb/z_catalog_object_list
...@@ -13,6 +16,7 @@ erp5_mysql_innodb/z_catalog_slapos_item_list ...@@ -13,6 +16,7 @@ erp5_mysql_innodb/z_catalog_slapos_item_list
erp5_mysql_innodb/z_catalog_software_instance_list erp5_mysql_innodb/z_catalog_software_instance_list
erp5_mysql_innodb/z_create_catalog erp5_mysql_innodb/z_create_catalog
erp5_mysql_innodb/z_create_compute_partition erp5_mysql_innodb/z_create_compute_partition
erp5_mysql_innodb/z_create_consistency
erp5_mysql_innodb/z_create_item erp5_mysql_innodb/z_create_item
erp5_mysql_innodb/z_create_slapos_item erp5_mysql_innodb/z_create_slapos_item
erp5_mysql_innodb/z_create_software_instance_tree erp5_mysql_innodb/z_create_software_instance_tree
......
master/bt5/slapos_mysql_innodb_catalog/bt/template_catalog_result_table_list
View file @ 1822be3b
compute_partition compute_partition
software_instance_tree software_instance_tree
slapos_item slapos_item
\ No newline at end of file consistency
\ No newline at end of file
master/bt5/slapos_mysql_innodb_catalog/bt/template_test_id_list 0 → 100644
View file @ 1822be3b
test.erp5.testSlapOSCatalogConsistency
\ No newline at end of file
master/tests/__init__.py
View file @ 1822be3b
...@@ -18,6 +18,7 @@ slapos_bt_list = [ ...@@ -18,6 +18,7 @@ slapos_bt_list = [
'slapos_payzen', 'slapos_payzen',
'slapos_wechat', 'slapos_wechat',
'slapos_configurator', 'slapos_configurator',
'slapos_mysql_innodb_catalog',
'slapos_jio', 'slapos_jio',
'slapos_jio_ui_test' 'slapos_jio_ui_test'
] ]
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7