Commit 5eaaf6b8 authored by Ophélie Gagnard's avatar Ophélie Gagnard

slapos_metadata_transform_agent: prevent to process ongoing data array

parent 3d5e9676
"""
This script is used in Ingestion Policy objects to parse provided by fluentd / ebulk reference.
In this case it expects a reference in this format for example:
"""
context.log(reference)
if not reference:
raise ValueError('reference is None')
......
......@@ -107,21 +107,27 @@ def create_ndarray(uid_list):
progress_indicator = in_stream["Progress Indicator"]
in_data_stream = in_stream["Data Stream"]
out_data_array = out_array["Data Array"]
if out_data_array.getSimulationState() == 'converted':
return
if out_data_array.getSimulationState() != 'converting':
out_data_array.transformFile()
exclude_file_list = []
out_data_array.setPublicationSectionList(in_data_stream.getPublicationSectionList())
if 'file_system_image/reference_image' in in_data_stream.getPublicationSectionList():
if out_data_array.getValidationState() == 'draft':
out_data_array.validate()
if not out_data_array.getCausality():
ingestion_line = in_data_stream.getAggregateRelatedValue(portal_type='Data Ingestion Line')
resource = ingestion_line.getResource()
resource_value = ingestion_line.getResourceValue()
exclude_file_list = ingestion_line.getResourceValue().DataProduct_getExcludeFileList()
out_data_array.edit(causality=resource)
out_data_array.edit(causality_value=resource_value)
in_data_stream.setPublicationSectionList(resource_value.getPublicationSectionList())
out_data_array.setPublicationSectionList(resource_value.getPublicationSectionList())
start = progress_indicator.getIntOffsetIndex()
end = in_data_stream.getSize()
if start >= end:
out_data_array.convertFile()
return
end, json_string_list, is_end_of_scan = get_end_and_json_list(start, in_data_stream)
triplet_list = get_triplet_list(json_string_list, is_end_of_scan)
......@@ -145,3 +151,5 @@ if end > start:
# if we did not reach end of stream
if end < in_data_stream.getSize():
return 1
out_data_array.convertFile()
......@@ -149,7 +149,7 @@
</item>
<item>
<key> <string>serial</string> </key>
<value> <string>997.41933.21687.42990</string> </value>
<value> <string>997.59106.5861.41472</string> </value>
</item>
<item>
<key> <string>state</string> </key>
......@@ -167,7 +167,7 @@
</tuple>
<state>
<tuple>
<float>1643288912.88</float>
<float>1643727633.97</float>
<string>UTC</string>
</tuple>
</state>
......
......@@ -62,7 +62,7 @@
</item>
<item>
<key> <string>data_operation_script_id</string> </key>
<value> <string>MetadataIngestionPolicy_getIngestionOperationAndParameterDict</string> </value>
<value> <string>IngestionPolicy_getIngestionOperationAndParameterDict</string> </value>
</item>
<item>
<key> <string>default_reference</string> </key>
......@@ -162,7 +162,7 @@
</item>
<item>
<key> <string>serial</string> </key>
<value> <string>995.40067.2143.6297</string> </value>
<value> <string>997.57747.29368.10086</string> </value>
</item>
<item>
<key> <string>state</string> </key>
......@@ -180,7 +180,7 @@
</tuple>
<state>
<tuple>
<float>1634807225.76</float>
<float>1643727573.35</float>
<string>UTC</string>
</tuple>
</state>
......
<workflow_chain>
<chain>
<type>Data Array</type>
<workflow>data_converting_workflow</workflow>
</chain>
</workflow_chain>
\ No newline at end of file
......@@ -5,7 +5,7 @@ portal = context.getPortalObject()
portal.portal_catalog.searchAndActivate(
portal_type="Data Array",
publication_section_relative_url="publication_section/file_system_image/node_image",
validation_state="draft",
simulation_state="converted",
method_id='DataArray_generateDiffND',
activate_kw={'tag': tag},
)
......
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="ERP5 Form" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_objects</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>action</string> </key>
<value> <string>Base_doSelect</string> </value>
</item>
<item>
<key> <string>description</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>edit_order</string> </key>
<value>
<list/>
</value>
</item>
<item>
<key> <string>encoding</string> </key>
<value> <string>UTF-8</string> </value>
</item>
<item>
<key> <string>enctype</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>group_list</string> </key>
<value>
<list>
<string>left</string>
<string>right</string>
<string>center</string>
<string>bottom</string>
<string>hidden</string>
</list>
</value>
</item>
<item>
<key> <string>groups</string> </key>
<value>
<dictionary>
<item>
<key> <string>bottom</string> </key>
<value>
<list>
<string>listbox</string>
</list>
</value>
</item>
<item>
<key> <string>center</string> </key>
<value>
<list/>
</value>
</item>
<item>
<key> <string>hidden</string> </key>
<value>
<list>
<string>listbox_shape</string>
<string>listbox_size</string>
<string>listbox_dtype</string>
</list>
</value>
</item>
<item>
<key> <string>left</string> </key>
<value>
<list/>
</value>
</item>
<item>
<key> <string>right</string> </key>
<value>
<list/>
</value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>DataArrayModule_viewDataArrayList</string> </value>
</item>
<item>
<key> <string>method</string> </key>
<value> <string>POST</string> </value>
</item>
<item>
<key> <string>name</string> </key>
<value> <string>DataArrayModule_viewDataArrayList</string> </value>
</item>
<item>
<key> <string>pt</string> </key>
<value> <string>form_list</string> </value>
</item>
<item>
<key> <string>row_length</string> </key>
<value> <int>4</int> </value>
</item>
<item>
<key> <string>stored_encoding</string> </key>
<value> <string>UTF-8</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Data Arrays</string> </value>
</item>
<item>
<key> <string>unicode_mode</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>update_action</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>update_action_title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="ProxyField" module="Products.ERP5Form.ProxyField"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>delegated_list</string> </key>
<value>
<list>
<string>all_columns</string>
<string>columns</string>
<string>domain_root_list</string>
<string>domain_tree</string>
<string>editable_columns</string>
<string>title</string>
</list>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>listbox</string> </value>
</item>
<item>
<key> <string>message_values</string> </key>
<value>
<dictionary>
<item>
<key> <string>external_validator_failed</string> </key>
<value> <string>The input failed the external validator.</string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>overrides</string> </key>
<value>
<dictionary>
<item>
<key> <string>field_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>target</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>tales</string> </key>
<value>
<dictionary>
<item>
<key> <string>field_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>target</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>values</string> </key>
<value>
<dictionary>
<item>
<key> <string>all_columns</string> </key>
<value>
<list>
<tuple>
<string>size</string>
<string>Size</string>
</tuple>
<tuple>
<string>shape</string>
<string>Shape</string>
</tuple>
<tuple>
<string>dtype</string>
<string>Dtype</string>
</tuple>
<tuple>
<string>modification_date</string>
<string>Modification Date</string>
</tuple>
<tuple>
<string>creation_date</string>
<string>Creation Date</string>
</tuple>
</list>
</value>
</item>
<item>
<key> <string>columns</string> </key>
<value>
<list>
<tuple>
<string>title</string>
<string>Title</string>
</tuple>
<tuple>
<string>reference</string>
<string>Reference</string>
</tuple>
<tuple>
<string>version</string>
<string>Version</string>
</tuple>
<tuple>
<string>description</string>
<string>Description</string>
</tuple>
<tuple>
<string>shape</string>
<string>Shape</string>
</tuple>
<tuple>
<string>dtype</string>
<string>Dtype</string>
</tuple>
<tuple>
<string>translated_simulation_state_title</string>
<string>Processing State</string>
</tuple>
</list>
</value>
</item>
<item>
<key> <string>domain_root_list</string> </key>
<value>
<list>
<tuple>
<string>publication_section</string>
<string>Publication Section</string>
</tuple>
<tuple>
<string>region</string>
<string>Region</string>
</tuple>
<tuple>
<string>function</string>
<string>Function</string>
</tuple>
<tuple>
<string>site</string>
<string>Site</string>
</tuple>
</list>
</value>
</item>
<item>
<key> <string>domain_tree</string> </key>
<value> <int>1</int> </value>
</item>
<item>
<key> <string>editable_columns</string> </key>
<value>
<list>
<tuple>
<string>shape</string>
<string>Shape</string>
</tuple>
<tuple>
<string>size</string>
<string>Size</string>
</tuple>
<tuple>
<string>dtype</string>
<string>Dtype</string>
</tuple>
</list>
</value>
</item>
<item>
<key> <string>field_id</string> </key>
<value> <string>my_list_mode_listbox</string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string>Base_viewFieldLibrary</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value>
<list>
<tuple>
<string>Data Array</string>
<string>Data Array</string>
</tuple>
</list>
</value>
</item>
<item>
<key> <string>target</string> </key>
<value> <string>Click to edit the target</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Data Arrays</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="ProxyField" module="Products.ERP5Form.ProxyField"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>delegated_list</string> </key>
<value>
<list>
<string>default</string>
<string>editable</string>
<string>title</string>
</list>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>listbox_dtype</string> </value>
</item>
<item>
<key> <string>message_values</string> </key>
<value>
<dictionary>
<item>
<key> <string>external_validator_failed</string> </key>
<value> <string>The input failed the external validator.</string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>overrides</string> </key>
<value>
<dictionary>
<item>
<key> <string>field_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>tales</string> </key>
<value>
<dictionary>
<item>
<key> <string>default</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
</value>
</item>
<item>
<key> <string>editable</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>field_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>values</string> </key>
<value>
<dictionary>
<item>
<key> <string>default</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>editable</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>field_id</string> </key>
<value> <string>my_string_field</string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string>Base_viewFieldLibrary</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Dtype of array</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="2" aka="AAAAAAAAAAI=">
<pickle>
<global name="TALESMethod" module="Products.Formulator.TALESField"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_text</string> </key>
<value> <string>python: \'%s\' %str(cell.getObject().getArrayDtype())</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="ProxyField" module="Products.ERP5Form.ProxyField"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>delegated_list</string> </key>
<value>
<list>
<string>default</string>
<string>editable</string>
<string>title</string>
</list>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>listbox_shape</string> </value>
</item>
<item>
<key> <string>message_values</string> </key>
<value>
<dictionary>
<item>
<key> <string>external_validator_failed</string> </key>
<value> <string>The input failed the external validator.</string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>overrides</string> </key>
<value>
<dictionary>
<item>
<key> <string>field_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>tales</string> </key>
<value>
<dictionary>
<item>
<key> <string>default</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
</value>
</item>
<item>
<key> <string>editable</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>field_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>values</string> </key>
<value>
<dictionary>
<item>
<key> <string>default</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>editable</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>field_id</string> </key>
<value> <string>my_string_field</string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string>Base_viewFieldLibrary</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Shape of array</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="2" aka="AAAAAAAAAAI=">
<pickle>
<global name="TALESMethod" module="Products.Formulator.TALESField"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_text</string> </key>
<value> <string>python: \'%s\' %str(cell.getObject().getArrayShape())</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="ProxyField" module="Products.ERP5Form.ProxyField"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>delegated_list</string> </key>
<value>
<list>
<string>default</string>
<string>editable</string>
<string>title</string>
</list>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>listbox_size</string> </value>
</item>
<item>
<key> <string>message_values</string> </key>
<value>
<dictionary>
<item>
<key> <string>external_validator_failed</string> </key>
<value> <string>The input failed the external validator.</string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>overrides</string> </key>
<value>
<dictionary>
<item>
<key> <string>field_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>tales</string> </key>
<value>
<dictionary>
<item>
<key> <string>default</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
</value>
</item>
<item>
<key> <string>editable</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>field_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>values</string> </key>
<value>
<dictionary>
<item>
<key> <string>default</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>editable</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>field_id</string> </key>
<value> <string>my_string_field</string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string>Base_viewFieldLibrary</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Shape of array</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="2" aka="AAAAAAAAAAI=">
<pickle>
<global name="TALESMethod" module="Products.Formulator.TALESField"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_text</string> </key>
<value> <string>python: \'%s\' %str(cell.getObject().getSize())</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
......@@ -20,10 +20,10 @@ for predecessor in context.getPredecessorValueList(
index = distribution_list.index(distribution_property) + 1
if index >= len(distribution_list):
context.setPublicationSectionList(context.getPublicationSectionList() + ['publication_section/file_system_image/diff_end'])
context.validate()
causality = context.getCausalityValue(portal_type='Data Product')
if causality and context.portal_workflow.isTransitionPossible(causality, 'invalidate'):
causality.invalidate(comment='Server has file modified')
context.processFile()
return
reference_distribution = distribution_list[index].getRelativeUrl()
......@@ -32,7 +32,7 @@ if not reference_distribution:
query = AndQuery(
Query(portal_type = ["Data Array"]),
Query(validation_state = "validated"),
Query(simulation_state = "converted"),
AndQuery(Query(publication_section_relative_url = 'publication_section/file_system_image/reference_image'),
Query(publication_section_relative_url = reference_distribution)))
......@@ -52,6 +52,7 @@ if reference_image:
publication_section="file_system_image/node_image",
causality = context.getCausality()
)
new_data_array.convertFile()
else:
causality = context.getCausalityValue(portal_type='Data Product')
if causality and context.portal_workflow.isTransitionPossible(causality, 'validate'):
......@@ -61,4 +62,4 @@ else:
if causality and context.portal_workflow.isTransitionPossible(causality, 'invalidate'):
causality.invalidate(comment='Server has file modified')
context.validate()
context.processFile()
......@@ -92,7 +92,7 @@
<string>my_start_date</string>
<string>my_predecessor_title_list</string>
<string>my_causality_title</string>
<string>my_translated_validation_state_title</string>
<string>my_translated_simulation_state_title</string>
</list>
</value>
</item>
......
......@@ -16,7 +16,7 @@
</item>
<item>
<key> <string>id</string> </key>
<value> <string>my_translated_validation_state_title</string> </value>
<value> <string>my_translated_simulation_state_title</string> </value>
</item>
<item>
<key> <string>message_values</string> </key>
......@@ -85,7 +85,7 @@
</item>
<item>
<key> <string>title</string> </key>
<value> <string>State</string> </value>
<value> <string>Processing State</string> </value>
</item>
</dictionary>
</value>
......
from DateTime import DateTime
from Products.ZSQLCatalog.SQLCatalog import AndQuery, OrQuery, Query
from Products.ERP5Type.Errors import UnsupportedWorkflowMethod
portal = context.getPortalObject()
portal_catalog = portal.portal_catalog
now = DateTime()
if not include_delivered:
batch_simulation_state = "stopped"
stream_simulation_state = "started"
else:
batch_simulation_state = ["stopped", "delivered"]
stream_simulation_state = ["started", "stopped", "delivered"]
query = AndQuery(
Query(portal_type = ["Data Ingestion Line", "Data Analysis Line"]),
Query(**{"stock.quantity": "!=0"}),
Query(resource_portal_type = "Data Product"),
# Should be improved to support mor than one analysis per ingestion
#SimpleQuery(parent_causality_related_relative_url = None),
OrQuery(Query(simulation_state = batch_simulation_state,
use_relative_url = "use/big_data/ingestion/batch"),
Query(simulation_state = stream_simulation_state,
use_relative_url = "use/big_data/ingestion/stream")))
for movement in portal_catalog(query = query):
if movement.getQuantity() <= 0:
continue
if movement.DataIngestionLine_hasMissingRequiredItem():
raise ValueError("Transformation requires movement to have " +
"aggregated data ingestion batch %s" % movement.getRelativeUrl())
delivery = movement.getParentValue()
data_supply = delivery.getSpecialiseValue(portal_type="Data Supply")
data_supply_list = delivery.getSpecialiseValueList(portal_type="Data Supply")
composed_data_supply = data_supply.asComposedDocument()
# Get applicable transformation
transformation_list = []
for transformation in composed_data_supply.getSpecialiseValueList(portal_type="Data Transformation"):
for line in transformation.objectValues():
if line.getResourceValue() == movement.getResourceValue() and line.getQuantity() < 0:
transformation_list.append(transformation)
break
for transformation in portal.portal_catalog(
portal_type = "Data Transformation",
validation_state = "validated",
resource_relative_url = movement.getResource()):
if transformation.getVariationCategoryList() == movement.getVariationCategoryList():
transformation_list.append(transformation)
for transformation in transformation_list:
context.log(transformation.getRelativeUrl())
context.log(delivery.getRelativeUrl())
is_shared_data_analysis = False
# Check if analysis already exists
data_analysis = portal_catalog.getResultValue(
portal_type="Data Analysis",
specialise_relative_url = transformation.getRelativeUrl(),
causality_relative_url = delivery.getRelativeUrl())
if data_analysis is not None:
continue
# for first level analysis check if same kind of data analysis with same project and same source already exists
# If yes, then later add additional input lines to this shared data analysis
context.log(delivery.getPortalType())
context.log(delivery.getRelativeUrl())
if delivery.getPortalType() == "Data Ingestion":
data_analysis = portal_catalog.getResultValue(
portal_type="Data Analysis",
specialise_relative_url = transformation.getRelativeUrl(),
source_relative_url = delivery.getSource(),
destination_project_relative_url = delivery.getDestinationProject())
if data_analysis is not None:
data_analysis.setDefaultCausalityValue(delivery)
data_analysis.setSpecialiseValueSet(data_analysis.getSpecialiseValueList() + data_supply_list)
is_shared_data_analysis = True
else:
# Create Analysis
data_analysis = portal.data_analysis_module.newContent(
portal_type = "Data Analysis",
title = transformation.getTitle(),
reference = delivery.getReference(),
start_date = delivery.getStartDate(),
stop_date = delivery.getStopDate(),
specialise_value_list = [transformation] + data_supply_list,
causality_value = delivery,
source = delivery.getSource(),
source_section = delivery.getSourceSection(),
source_project = delivery.getSourceProject(),
destination = delivery.getDestination(),
destination_section = delivery.getDestinationSection(),
destination_project = delivery.getDestinationProject())
data_analysis.checkConsistency(fixit=True)
# create input and output lines
for transformation_line in transformation.objectValues(
portal_type=["Data Transformation Resource Line",
"Data Transformation Operation Line"]):
resource = transformation_line.getResourceValue()
quantity = transformation_line.getQuantity()
if isinstance(quantity, tuple):
quantity = quantity[0]
# In case of shared data anylsis only add additional input lines
if is_shared_data_analysis and quantity > -1:
continue
aggregate_set = set()
# manually add device to every line
aggregate_set.add(movement.getAggregateDevice())
if transformation_line.getPortalType() == \
"Data Transformation Resource Line":
# at the moment, we only check for positive or negative quantity
if quantity < 0:
# it is an input line
# aggregate transformed item from data ingestion batch related to our
# movement. If it is an input resource line, then we search for an
# ingestion line with the same resource. If it is an operation line
# then we search for an ingestion line with resource portal type
# Data Product
batch_relative_url = movement.getAggregateDataIngestionBatch()
if batch_relative_url is not None:
related_movement_list = portal_catalog(
portal_type="Data Ingestion Line",
aggregate_relative_url=batch_relative_url,
resource_relative_url = resource.getRelativeUrl())
else:
# get related movements only from current data ingestion
related_movement_list = movement.getParentValue().searchFolder(
portal_type=["Data Ingestion Line", "Data Analysis Line"],
resource_relative_url = resource.getRelativeUrl())
for related_movement in related_movement_list:
aggregate_set.update(related_movement.getAggregateSet())
if related_movement.getUse() == "big_data/ingestion/batch":
related_movement.getParentValue().deliver()
# create new item based on item_type if it is not already aggregated
aggregate_type_set = set(
[portal.restrictedTraverse(a).getPortalType() for a in aggregate_set])
for item_type in transformation_line.getAggregatedPortalTypeList():
# create item if it does note exist yet.
# Except if it is a Data Array Line, then it is currently created by
# data operation itself (probably this exception is inconsistent)
if item_type not in aggregate_type_set and item_type != "Data Array Line":
item_query_dict = dict(
portal_type=item_type,
validation_state="validated",
item_variation_text=transformation_line.getVariationText(),
item_device_relative_url=movement.getAggregateDevice(),
item_resource_uid=resource.getUid(),
item_source_relative_url=data_analysis.getSource())
if data_analysis.getDestinationProjectValue() is not None:
item_query_dict["item_project_relative_url"] = data_analysis.getDestinationProject()
item = portal.portal_catalog.getResultValue(**item_query_dict)
if item is None:
module = portal.getDefaultModule(item_type)
item = module.newContent(portal_type = item_type,
title = transformation.getTitle(),
reference = "%s-%s" %(transformation.getTitle(),
delivery.getReference()),
version = '001')
aggregate_set.add(item.getRelativeUrl())
# find other items such as device configuration and data configuration
# from data ingestion and data supply
composed = data_analysis.asComposedDocument()
line_list = [l for l in delivery.objectValues(portal_type="Data Ingestion Line")]
line_list += [l for l in composed.objectValues(portal_type="Data Supply Line")]
for line in line_list:
if line.getResourceValue().getPortalType() == "Data Operation":
aggregate_set.update(line.getAggregateList())
data_analysis_line = data_analysis.newContent(
portal_type = "Data Analysis Line",
title = transformation_line.getTitle(),
reference = transformation_line.getReference(),
int_index = transformation_line.getIntIndex(),
resource_value = resource,
variation_category_list = transformation_line.getVariationCategoryList(),
quantity = quantity,
quantity_unit = transformation_line.getQuantityUnit(),
use = transformation_line.getUse(),
aggregate_set = aggregate_set)
# for intput lines of first level analysis set causality and specialise
if quantity < 0 and delivery.getPortalType() == "Data Ingestion":
data_analysis_line.edit(
causality_value = delivery,
specialise_value_list = data_supply_list)
data_analysis.checkConsistency(fixit=True)
try:
data_analysis.start()
except UnsupportedWorkflowMethod:
pass
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>include_delivered=False</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>ERP5Site_createDataAnalysisList</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
from DateTime import DateTime
now = DateTime()
today_string = now.strftime('%Y%m%d')
portal = context.getPortalObject()
portal_catalog = portal.portal_catalog
data_ingestion_reference = movement_dict.get('reference', reference)
resource_reference = movement_dict.get('resource_reference', None)
specialise_reference = movement_dict.get('specialise_reference', None)
data_ingestion_id = "%s-%s" %(today_string, data_ingestion_reference)
data_ingestion_query_kw = dict(
portal_type = 'Data Ingestion',
simulation_state = ['started', 'stopped'],
reference = data_ingestion_reference)
# first search for applicable data ingestion
data_ingestion = portal_catalog.getResultValue(**data_ingestion_query_kw)
def init_input_line(input_line, operation_line):
# copy device and configuration from operation line to input line
input_line.setAggregateSet(
input_line.getAggregateList() + operation_line.getAggregateList())
# Check if we have a batch referece
data_ingestion_batch_reference = movement_dict.get(
'aggregate_data_ingestion_batch_reference',
None)
data_ingestion_batch_id = "%s-%s" %(today_string,
data_ingestion_batch_reference)
data_sink_type_list = []
data_sink_list = []
data_product = None
if data_ingestion_batch_reference is not None:
data_ingestion_batch = portal_catalog.getResultValue(
portal_type = "Data Ingestion Batch",
reference = data_ingestion_batch_reference)
if data_ingestion_batch is None:
data_ingestion_batch = portal.data_ingestion_batch_module.get(
data_ingestion_batch_id)
if data_ingestion_batch is None:
data_ingestion_batch = portal.data_ingestion_batch_module.newContent(
id = data_ingestion_batch_id,
portal_type = "Data Ingestion Batch",
reference = data_ingestion_batch_reference)
else:
previous_data_ingestion_line = portal_catalog.getResultValue(
portal_type = "Data Ingestion Line",
resource_reference = resource_reference,
aggregate_uid = data_ingestion_batch.getUid())
if previous_data_ingestion_line is not None:
data_product = previous_data_ingestion_line.getResourceValue()
data_sink_type_list = data_product.getAggregatedPortalTypeList()
data_sink_list = previous_data_ingestion_line\
.getAggregateValueList(portal_type=data_sink_type_list)
input_line.setDefaultAggregateValue(data_ingestion_batch)
if not data_sink_list:
if not data_sink_type_list:
if data_product is None:
data_product = portal.portal_catalog.getResultValue(
portal_type = "Data Product",
reference = resource_reference)
data_sink_type_list = data_product.getAggregatedPortalTypeList()
for data_sink_type in data_sink_type_list:
# This should be more generic
if data_sink_type not in ("Progress Indicator", "Data Ingestion Batch"):
# first try to find existing validated data sink
# with same device, project, resource (but can be different source)
data_sink = portal.portal_catalog.getResultValue(
portal_type=data_sink_type,
validation_state="validated",
item_device_relative_url=operation_line.getAggregateDevice(),
item_project_relative_url=input_line.getDestinationProject(),
item_resource_uid=input_line.getResourceUid())
if data_sink is None:
data_sink = portal.getDefaultModule(data_sink_type).newContent(
portal_type = data_sink_type,
reference = "%s-%s" %(data_ingestion_reference, resource_reference))
data_sink.validate()
if data_sink_type == 'Data Stream':
data_sink.setPublicationSectionList(data_product.getPublicationSectionList())
data_sink_list.append(data_sink)
input_line.setAggregateValueList(
input_line.getAggregateValueList() + data_sink_list)
input_line.setQuantity(1)
if data_ingestion is None:
document = portal.data_ingestion_module.get(data_ingestion_id)
if (document is not None) and document.getSimulationState() in ('started', 'stopped'):
data_ingestion = document
if modify and data_ingestion is None:
specialise_query_kw = dict(portal_type = 'Data Supply',
reference = specialise_reference,
validation_state = 'validated')
specialise_list = [x.getRelativeUrl() for x in portal_catalog(**specialise_query_kw)]
# if we do not find a validated data supply, we look for a default data supply
if not specialise_list:
specialise_query_kw["validation_state"] = 'default'
specialise_list = [x.getRelativeUrl() for x in portal_catalog(**specialise_query_kw)]
# create a new data ingestion
data_ingestion = portal.ERP5Site_createDataIngestion(specialise_list,
data_ingestion_reference,
data_ingestion_id)
# find ingestion line for current resource
for line in data_ingestion.objectValues(portal_type="Data Ingestion Line"):
if line.getResourceReference() == resource_reference:
input_line = line
elif line.getResourceValue().getPortalType() == "Data Operation":
operation_line = line
if modify and input_line.getQuantity() == 0:
init_input_line(input_line, operation_line)
data_operation = operation_line.getResourceValue()
parameter_dict = {
input_line.getReference(): \
{v.getPortalType(): v for v in input_line.getAggregateValueList()}}
bucket_reference = movement_dict.get('bucket_reference', None)
if bucket_reference is not None:
parameter_dict['bucket_reference'] = bucket_reference
return data_operation, parameter_dict
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Workflow" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_count</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
</value>
</item>
<item>
<key> <string>_mt_index</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
</value>
</item>
<item>
<key> <string>_tree</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAQ=</string> </persistent>
</value>
</item>
<item>
<key> <string>categories</string> </key>
<value>
<tuple>
<string>source/portal_workflow/data_converting_workflow/state_draft</string>
</tuple>
</value>
</item>
<item>
<key> <string>comment</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>default_reference</string> </key>
<value> <string>processing_status_workflow</string> </value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>data_converting_workflow</string> </value>
</item>
<item>
<key> <string>manager_bypass</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Workflow</string> </value>
</item>
<item>
<key> <string>state_variable</string> </key>
<value> <string>simulation_state</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Data Converting Workflow</string> </value>
</item>
<item>
<key> <string>workflow_managed_permission</string> </key>
<value>
<tuple/>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="2" aka="AAAAAAAAAAI=">
<pickle>
<global name="Length" module="BTrees.Length"/>
</pickle>
<pickle> <int>0</int> </pickle>
</record>
<record id="3" aka="AAAAAAAAAAM=">
<pickle>
<global name="OOBTree" module="BTrees.OOBTree"/>
</pickle>
<pickle>
<none/>
</pickle>
</record>
<record id="4" aka="AAAAAAAAAAQ=">
<pickle>
<global name="OOBTree" module="BTrees.OOBTree"/>
</pickle>
<pickle>
<none/>
</pickle>
</record>
</ZopeData>
document = state_change['object']
if document.isSupportBaseDataConversion() and not document.hasBaseData():
document.processFile()
document.activate().convertToBaseFormat()
......@@ -2,7 +2,7 @@
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
<global name="Workflow Script" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
......@@ -50,11 +50,21 @@
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>movement_dict, reference, modify=True</string> </value>
<value> <string>state_change</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>MetadataIngestionPolicy_getIngestionOperationAndParameterDict</string> </value>
<value> <string>script_Document_convertToBaseFormat</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Workflow Script</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value>
<none/>
</value>
</item>
</dictionary>
</pickle>
......
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Workflow State" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>acquire_permission</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>categories</string> </key>
<value>
<tuple>
<string>destination/portal_workflow/data_converting_workflow/transition_process_file</string>
</tuple>
</value>
</item>
<item>
<key> <string>comment</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>description</string> </key>
<value> <string>State of the document when converted to an open document</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>state_converted</string> </value>
</item>
<item>
<key> <string>index</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
</value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Workflow State</string> </value>
</item>
<item>
<key> <string>state_permission_role_list_dict</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
</value>
</item>
<item>
<key> <string>state_type</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Converted</string> </value>
</item>
</dictionary>
</pickle>
</record>
<record id="2" aka="AAAAAAAAAAI=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary>
<item>
<key> <string>cell</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAQ=</string> </persistent>
</value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="3" aka="AAAAAAAAAAM=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary/>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="4" aka="AAAAAAAAAAQ=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary>
<item>
<key> <int>0</int> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAU=</string> </persistent>
</value>
</item>
<item>
<key> <int>1</int> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAY=</string> </persistent>
</value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="5" aka="AAAAAAAAAAU=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary/>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="6" aka="AAAAAAAAAAY=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary>
<item>
<key> <string>Anonymous</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>Assignee</string> </key>
<value> <int>1</int> </value>
</item>
<item>
<key> <string>Assignor</string> </key>
<value> <int>2</int> </value>
</item>
<item>
<key> <string>Associate</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>Auditor</string> </key>
<value> <int>4</int> </value>
</item>
<item>
<key> <string>Authenticated</string> </key>
<value> <int>5</int> </value>
</item>
<item>
<key> <string>Author</string> </key>
<value> <int>6</int> </value>
</item>
<item>
<key> <string>Manager</string> </key>
<value> <int>7</int> </value>
</item>
<item>
<key> <string>Member</string> </key>
<value> <int>8</int> </value>
</item>
<item>
<key> <string>Owner</string> </key>
<value> <int>9</int> </value>
</item>
<item>
<key> <string>Reviewer</string> </key>
<value> <int>10</int> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Workflow State" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>acquire_permission</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>categories</string> </key>
<value>
<tuple>
<string>destination/portal_workflow/data_converting_workflow/transition_convert_file</string>
</tuple>
</value>
</item>
<item>
<key> <string>comment</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>description</string> </key>
<value> <string>Indicates that the conversion of document to an open document is in process</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>state_converting</string> </value>
</item>
<item>
<key> <string>index</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
</value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Workflow State</string> </value>
</item>
<item>
<key> <string>state_permission_role_list_dict</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
</value>
</item>
<item>
<key> <string>state_type</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Converting</string> </value>
</item>
</dictionary>
</pickle>
</record>
<record id="2" aka="AAAAAAAAAAI=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary>
<item>
<key> <string>cell</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAQ=</string> </persistent>
</value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="3" aka="AAAAAAAAAAM=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary/>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="4" aka="AAAAAAAAAAQ=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary>
<item>
<key> <int>0</int> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAU=</string> </persistent>
</value>
</item>
<item>
<key> <int>1</int> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAY=</string> </persistent>
</value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="5" aka="AAAAAAAAAAU=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary/>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="6" aka="AAAAAAAAAAY=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary>
<item>
<key> <string>Anonymous</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>Assignee</string> </key>
<value> <int>1</int> </value>
</item>
<item>
<key> <string>Assignor</string> </key>
<value> <int>2</int> </value>
</item>
<item>
<key> <string>Associate</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>Auditor</string> </key>
<value> <int>4</int> </value>
</item>
<item>
<key> <string>Authenticated</string> </key>
<value> <int>5</int> </value>
</item>
<item>
<key> <string>Author</string> </key>
<value> <int>6</int> </value>
</item>
<item>
<key> <string>Manager</string> </key>
<value> <int>7</int> </value>
</item>
<item>
<key> <string>Member</string> </key>
<value> <int>8</int> </value>
</item>
<item>
<key> <string>Owner</string> </key>
<value> <int>9</int> </value>
</item>
<item>
<key> <string>Reviewer</string> </key>
<value> <int>10</int> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Workflow State" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>acquire_permission</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>categories</string> </key>
<value>
<tuple>
<string>destination/portal_workflow/data_converting_workflow/transition_convert_file</string>
<string>destination/portal_workflow/data_converting_workflow/transition_transform_file</string>
</tuple>
</value>
</item>
<item>
<key> <string>comment</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>description</string> </key>
<value> <string>default state of the document in the converting process</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>state_draft</string> </value>
</item>
<item>
<key> <string>index</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
</value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Workflow State</string> </value>
</item>
<item>
<key> <string>state_permission_role_list_dict</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
</value>
</item>
<item>
<key> <string>state_type</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Draft</string> </value>
</item>
</dictionary>
</pickle>
</record>
<record id="2" aka="AAAAAAAAAAI=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary>
<item>
<key> <string>cell</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAQ=</string> </persistent>
</value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="3" aka="AAAAAAAAAAM=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary/>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="4" aka="AAAAAAAAAAQ=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary>
<item>
<key> <int>0</int> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAU=</string> </persistent>
</value>
</item>
<item>
<key> <int>1</int> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAY=</string> </persistent>
</value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="5" aka="AAAAAAAAAAU=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary/>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="6" aka="AAAAAAAAAAY=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary>
<item>
<key> <string>Anonymous</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>Assignee</string> </key>
<value> <int>1</int> </value>
</item>
<item>
<key> <string>Assignor</string> </key>
<value> <int>2</int> </value>
</item>
<item>
<key> <string>Associate</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>Auditor</string> </key>
<value> <int>4</int> </value>
</item>
<item>
<key> <string>Authenticated</string> </key>
<value> <int>5</int> </value>
</item>
<item>
<key> <string>Author</string> </key>
<value> <int>6</int> </value>
</item>
<item>
<key> <string>Manager</string> </key>
<value> <int>7</int> </value>
</item>
<item>
<key> <string>Member</string> </key>
<value> <int>8</int> </value>
</item>
<item>
<key> <string>Owner</string> </key>
<value> <int>9</int> </value>
</item>
<item>
<key> <string>Reviewer</string> </key>
<value> <int>10</int> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Workflow State" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>acquire_permission</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>categories</string> </key>
<value>
<tuple>
<string>destination/portal_workflow/data_converting_workflow/transition_transform_file</string>
</tuple>
</value>
</item>
<item>
<key> <string>comment</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>description</string> </key>
<value> <string>Conversion process failed from some reason</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>state_processed</string> </value>
</item>
<item>
<key> <string>index</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
</value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Workflow State</string> </value>
</item>
<item>
<key> <string>state_permission_role_list_dict</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
</value>
</item>
<item>
<key> <string>state_type</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Processed</string> </value>
</item>
</dictionary>
</pickle>
</record>
<record id="2" aka="AAAAAAAAAAI=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary>
<item>
<key> <string>cell</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAQ=</string> </persistent>
</value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="3" aka="AAAAAAAAAAM=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary/>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="4" aka="AAAAAAAAAAQ=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary>
<item>
<key> <int>0</int> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAU=</string> </persistent>
</value>
</item>
<item>
<key> <int>1</int> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAY=</string> </persistent>
</value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="5" aka="AAAAAAAAAAU=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary/>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="6" aka="AAAAAAAAAAY=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary>
<item>
<key> <string>Anonymous</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>Assignee</string> </key>
<value> <int>1</int> </value>
</item>
<item>
<key> <string>Assignor</string> </key>
<value> <int>2</int> </value>
</item>
<item>
<key> <string>Associate</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>Auditor</string> </key>
<value> <int>4</int> </value>
</item>
<item>
<key> <string>Authenticated</string> </key>
<value> <int>5</int> </value>
</item>
<item>
<key> <string>Author</string> </key>
<value> <int>6</int> </value>
</item>
<item>
<key> <string>Manager</string> </key>
<value> <int>7</int> </value>
</item>
<item>
<key> <string>Member</string> </key>
<value> <int>8</int> </value>
</item>
<item>
<key> <string>Owner</string> </key>
<value> <int>9</int> </value>
</item>
<item>
<key> <string>Reviewer</string> </key>
<value> <int>10</int> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Workflow Transition" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>action</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>action_name</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>categories</string> </key>
<value>
<tuple>
<string>action_type/workflow</string>
<string>destination/portal_workflow/data_converting_workflow/state_converted</string>
</tuple>
</value>
</item>
<item>
<key> <string>description</string> </key>
<value> <string>This workflow method is called to convert a file to a certain format</string> </value>
</item>
<item>
<key> <string>guard_group</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>guard_permission</string> </key>
<value>
<tuple>
<string>Modify portal content</string>
</tuple>
</value>
</item>
<item>
<key> <string>guard_role</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>icon</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>transition_convert_file</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Workflow Transition</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Convert File</string> </value>
</item>
<item>
<key> <string>trigger_type</string> </key>
<value> <int>2</int> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Workflow Transition" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>action</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>action_name</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>categories</string> </key>
<value>
<tuple>
<string>action_type/workflow</string>
<string>destination/portal_workflow/data_converting_workflow/state_processed</string>
</tuple>
</value>
</item>
<item>
<key> <string>description</string> </key>
<value> <string>This workflow method is called to convert a file to a certain format</string> </value>
</item>
<item>
<key> <string>guard_group</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>guard_permission</string> </key>
<value>
<tuple>
<string>Modify portal content</string>
</tuple>
</value>
</item>
<item>
<key> <string>guard_role</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>icon</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>transition_process_file</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Workflow Transition</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Process File</string> </value>
</item>
<item>
<key> <string>trigger_type</string> </key>
<value> <int>2</int> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Workflow Transition" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>action</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>action_name</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>categories</string> </key>
<value>
<tuple>
<string>action_type/workflow</string>
<string>destination/portal_workflow/data_converting_workflow/state_converting</string>
</tuple>
</value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>guard_expression</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>guard_group</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>guard_permission</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>guard_role</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>icon</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>transition_transform_file</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Workflow Transition</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Transform File</string> </value>
</item>
<item>
<key> <string>trigger_type</string> </key>
<value> <int>2</int> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Workflow Variable" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>automatic_update</string> </key>
<value> <int>1</int> </value>
</item>
<item>
<key> <string>description</string> </key>
<value> <string>The last transition</string> </value>
</item>
<item>
<key> <string>for_catalog</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>variable_action</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Workflow Variable</string> </value>
</item>
<item>
<key> <string>status_included</string> </key>
<value> <int>1</int> </value>
</item>
<item>
<key> <string>title</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>variable_default_expression</string> </key>
<value> <string>transition/getReference|nothing</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Workflow Variable" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>automatic_update</string> </key>
<value> <int>1</int> </value>
</item>
<item>
<key> <string>description</string> </key>
<value> <string>The name of the user who performed the last transition</string> </value>
</item>
<item>
<key> <string>for_catalog</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>variable_actor</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Workflow Variable</string> </value>
</item>
<item>
<key> <string>status_included</string> </key>
<value> <int>1</int> </value>
</item>
<item>
<key> <string>title</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>variable_default_expression</string> </key>
<value> <string>user/getIdOrUserName</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Workflow Variable" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>automatic_update</string> </key>
<value> <int>1</int> </value>
</item>
<item>
<key> <string>description</string> </key>
<value> <string>Comments about the last transition</string> </value>
</item>
<item>
<key> <string>for_catalog</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>variable_comment</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Workflow Variable</string> </value>
</item>
<item>
<key> <string>status_included</string> </key>
<value> <int>1</int> </value>
</item>
<item>
<key> <string>title</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>variable_default_expression</string> </key>
<value> <string>python:state_change.kwargs.get(\'comment\', \'\')</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Workflow Variable" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>automatic_update</string> </key>
<value> <int>1</int> </value>
</item>
<item>
<key> <string>description</string> </key>
<value> <string>Error message if validation failed</string> </value>
</item>
<item>
<key> <string>for_catalog</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>variable_error_message</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Workflow Variable</string> </value>
</item>
<item>
<key> <string>status_included</string> </key>
<value> <int>1</int> </value>
</item>
<item>
<key> <string>title</string> </key>
<value>
<none/>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Workflow Variable" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>automatic_update</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>description</string> </key>
<value> <string>Provides access to workflow history</string> </value>
</item>
<item>
<key> <string>for_catalog</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>variable_history</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Workflow Variable</string> </value>
</item>
<item>
<key> <string>status_included</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>title</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>variable_default_expression</string> </key>
<value> <string>state_change/getHistory</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Workflow Variable" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>automatic_update</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>description</string> </key>
<value> <string>portal type (use as filter for worklists)</string> </value>
</item>
<item>
<key> <string>for_catalog</string> </key>
<value> <int>1</int> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>variable_portal_type</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Workflow Variable</string> </value>
</item>
<item>
<key> <string>status_included</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>title</string> </key>
<value>
<none/>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Workflow Variable" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>automatic_update</string> </key>
<value> <int>1</int> </value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>for_catalog</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>variable_serial</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Workflow Variable</string> </value>
</item>
<item>
<key> <string>status_included</string> </key>
<value> <int>1</int> </value>
</item>
<item>
<key> <string>title</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>variable_default_expression</string> </key>
<value> <string>state_change/object/getHistorySerial | nothing</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Workflow Variable" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>automatic_update</string> </key>
<value> <int>1</int> </value>
</item>
<item>
<key> <string>description</string> </key>
<value> <string>Time of the last transition</string> </value>
</item>
<item>
<key> <string>for_catalog</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>variable_time</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Workflow Variable</string> </value>
</item>
<item>
<key> <string>status_included</string> </key>
<value> <int>1</int> </value>
</item>
<item>
<key> <string>title</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>variable_default_expression</string> </key>
<value> <string>state_change/getDateTime</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
Data Array | data_converting_workflow
\ No newline at end of file
data_converting_workflow
\ No newline at end of file
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment