Commit 471a61d4 authored by Roque Porchetto's avatar Roque Porchetto

erp5_wendelin_telecom_ingestion: optimization of queries

parent d2008ad7
......@@ -31,7 +31,7 @@ for data_stream in portal_catalog(**kw_dict):
# invalidate old (more than 10min) pending ingestions (e.g. split ingestions that were canceled/interrumped)
# invalidate old (more than 10 min) pending ingestions (e.g. split ingestions that were canceled/interrumped)
from DateTime import DateTime
now = DateTime()
now_minus_max = now - 1.0/24/60*9999
......@@ -40,21 +40,20 @@ now_minus_10 = now - 1.0/24/60*10
catalog_kw = {'creation_date': {'query': (now_minus_max, now_minus_10), 'range': 'minmax'}, 'simulation_state': 'planned', 'portal_type': 'Data Ingestion'}
for data_ingestion in portal_catalog(**catalog_kw):
related_split_ingestions = portal_catalog(portal_type = "Data Ingestion",
simulation_state = "planned",
reference = data_ingestion.getReference())
# search related data ingestions that are not old yet (less than 10 min)
catalog_kw = {'creation_date': {'query': (now_minus_10, DateTime()), 'range': 'minmax'},
'simulation_state': 'planned',
'portal_type': 'Data Ingestion',
'reference': data_ingestion.getReference()}
invalidate = True
for related_ingestion in related_split_ingestions:
# only invalidate if all related ingestion are old
if len(portal_catalog(id=related_ingestion.getId(), **catalog_kw)) == 0:
if len(portal_catalog(**catalog_kw)) > 0:
context.logEntry("Data Ingestion %s is old, but it has related ingestion that are not. So it won't be invalidated yet." % data_ingestion.getId())
invalidate = False
break
if invalidate:
# invalidate related Data Stream
kw_dict = {"portal_type": "Data Stream",
"reference": data_ingestion.getReference()}
"id": data_ingestion.getId()}
for data_stream in portal_catalog(**kw_dict):
if not data_stream.getReference().endswith("_invalid"):
data_stream.setReference(data_stream.getReference() + "_invalid")
......
from Products.ERP5Type.Log import log
from Products.ZSQLCatalog.SQLCatalog import Query, SimpleQuery
# sort manually beacause query doesn't sort
def sorted_by_date(results):
A = [ x for x in results]
for j in range(1,len(A)):
key = A[j]
i = j-1
while (i > -1) and key.getId()[-3:] < A[i].getId()[-3:]:
A[i+1]=A[i]
i=i-1
A[i+1] = key
return A
portal = context.getPortalObject()
portal_catalog = portal.portal_catalog
......@@ -34,10 +22,9 @@ for data_ingestion in portal_catalog(portal_type = "Data Ingestion",
elif len(related_split_ingestions) > 1:
try:
query = Query(portal_type="Data Stream", reference=data_ingestion.getReference())
result_list = portal_catalog(query=query, sort_on=(('creation_date', 'DESC', 'date'),))
result_list = portal_catalog(query=query, sort_on=(('creation_date', 'ascending'),))
index = 1
# for some reason, the sort query doesn't work (random order)
for data_stream in sorted_by_date(result_list):
for data_stream in result_list:
if index == 1:
full_data_stream = data_stream
else:
......
......@@ -12,7 +12,7 @@ if object.getPortalType() == "Data Set":
kw_dict = {"portal_type": "Data Stream",
"query": reference_query}
for data_stream in portal_catalog(**kw_dict):
if data_stream.getReference().startswith(data_set.getReference()+'/'):
if data_stream.getReference().startswith(data_set.getReference()+'/') and not data_stream.getReference().endswith("_invalid"):
portal.ERP5Site_invalidateIngestionObjects(data_stream.getReference())
data_set.setReference(data_set.getReference() + "_invalid")
context.logEntry("Data set '%s' invalidated." % data_set.getReference())
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment