Skip to content

S
slapos.core
Commit a44366d8 authored by Ophélie Gagnard's avatar Ophélie Gagnard
Browse files
Options

slapos_metadata_transform_agent: Add comparison script. 

The comparison script generate the difference between a server node's
array and the reference database's array. It iterates over several
database types.
parent e98b32cf
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 2567 additions and 140 deletions
master/bt5/slapos_metadata_transform_agent/PortalTypeBaseCategoryTemplateItem/base_category_list.xml 0 → 100644
View file @ a44366d8
<base_category_list>
<portal_type id="Data Array">
<item>causality</item>
<item>predecessor</item>
</portal_type>
<portal_type id="Data Product">
<item>publication_section</item>
</portal_type>
</base_category_list>
\ No newline at end of file
master/bt5/slapos_metadata_transform_agent/SkinTemplateItem/portal_skins/slapos_metadata_transform_agent/DataArray_generateDiffND.py 0 → 100644
View file @ a44366d8
import numpy as np
from Products.ZSQLCatalog.SQLCatalog import AndQuery, Query
# Do nothing for reference image, just change state
if 'file_system_image/reference_image' in context.getPublicationSectionList():
context.processFile()
return
for publication_section in context.getPublicationSectionList():
if 'distribution' in publication_section:
current_node_distribution = publication_section
reference_distribution = None
for predecessor in context.getPredecessorValueList(
portal_type='Data Array'
):
publication_section_list = predecessor.getPublicationSectionList()
if 'file_system_image/reference_image' in publication_section_list:
for publication_section in publication_section_list:
if 'distribution' in publication_section:
distribution_property = context.portal_categories.restrictedTraverse('publication_section/%s' % publication_section)
parent_distribution = distribution_property.getParentValue()
distribution_list = [x for x in parent_distribution.objectValues(sort_on='int_index')]
index = distribution_list.index(distribution_property) + 1
if index >= len(distribution_list):
context.setPublicationSectionList(context.getPublicationSectionList() + ['publication_section/file_system_image/diff_end'])
causality = context.getCausalityValue(portal_type='Data Product')
if causality and context.portal_workflow.isTransitionPossible(causality, 'invalidate'):
causality.invalidate(comment='Server has file modified')
context.processFile()
return
reference_distribution = distribution_list[index].getRelativeUrl()
if not reference_distribution:
reference_distribution = 'publication_section/%s' % current_node_distribution
query = AndQuery(
Query(portal_type = ["Data Array"]),
Query(simulation_state = ("converted", "processed")),
AndQuery(Query(publication_section_relative_url = 'publication_section/file_system_image/reference_image'),
Query(publication_section_relative_url = reference_distribution)))
reference_image = context.portal_catalog.getResultValue(query=query)
def getDiff(a1, a2):
return np.setdiff1d(a1[:], a2[:])
if reference_image:
diff_array = getDiff(context.getArray(), reference_image.getArray())
if len(diff_array) != 0:
new_data_array = context.data_array_module.newContent(portal_type='Data Array')
new_data_array.setArray(diff_array)
new_data_array.edit(
predecessor_value_list=[context, reference_image],
publication_section="file_system_image/node_image",
causality = context.getCausality()
)
new_data_array.convertFile()
else:
causality = context.getCausalityValue(portal_type='Data Product')
if causality and context.portal_workflow.isTransitionPossible(causality, 'validate'):
causality.validate(comment='Server is ok')
else:
causality = context.getCausalityValue(portal_type='Data Product')
if causality and context.portal_workflow.isTransitionPossible(causality, 'invalidate'):
causality.invalidate(comment='Server has file modified')
context.processFile()
master/bt5/slapos_metadata_transform_agent/SkinTemplateItem/portal_skins/slapos_metadata_transform_agent/MetadataIngestionPolicy_getIngestionOperationAndParameterDict.xml → master/bt5/slapos_metadata_transform_agent/SkinTemplateItem/portal_skins/slapos_metadata_transform_agent/DataArray_generateDiffND.xml
View file @ a44366d8
......@@ -50,11 +50,11 @@
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>movement_dict, reference, modify=True</string> </value>
<value> <string>**kw</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>MetadataIngestionPolicy_getIngestionOperationAndParameterDict</string> </value>
<value> <string>DataArray_generateDiffND</string> </value>
</item>
</dictionary>
</pickle>
......
master/bt5/slapos_metadata_transform_agent/SkinTemplateItem/portal_skins/slapos_metadata_transform_agent/DataArray_getArrayFileInfoList.py 0 → 100644
View file @ a44366d8
from Products.ERP5Type.Document import newTempBase
if context.getArray() is None:
return []
class SequenceSliceMap():
def __init__(self, sequence_slice, usual_slice_length, total_length):
self.sequence_slice = sequence_slice
self.length = usual_slice_length
self.total_length = total_length
def __repr__(self):
return repr(list(self))
def __len__(self):
return self.total_length
def __getitem__(self, index):
return self.sequence_slice[index % self.length]
tmp_data_stream = context.data_stream_module.newContent(portal_type='Data Stream', temp_object=True)
def createTempBase(nr, row):
def getElementFromArray(array, index):
return tmp_data_stream.get_path_from_uid(array[index])
def getElementFromScalar(scalar, index=None):
return tmp_data_stream.get_path_from_uid(scalar)
column_list = [col for col in context.DataArray_getArrayColumnList() if col[0] != 'index']
column_iterator = enumerate(column_list)
if len(column_list) == 1:
getElement = getElementFromScalar
else:
getElement = getElementFromArray
return newTempBase(context.getPortalObject(),
str(id(row)),
index = nr,
**{col[0]: str(getElement(row, i)) for i, col in column_iterator})
length = context.getArrayShape()[0]
# never access more than 1000 lines at once
list_lines = min(list_lines, limit, 1000)
if context.REQUEST.has_key("limit"):
list_start = limit[0]
list_lines = limit[1] - limit[0]
orig_list_start = list_start
if orig_list_start + list_lines > length:
orig_list_start = length - length - (length % list_lines)
list_start = max(length - list_start - list_lines, 0)
if abs(orig_list_start) < list_lines and orig_list_start != 0:
list_end = abs(orig_list_start)
else:
list_end = max(list_start + list_lines, 0)
#list_end = max(list_start + list_lines, 0)
if list_start == list_end:
array_slice = [context.getArrayIndex(list_start)]
else:
array_slice = context.getArraySlice(list_start, list_end)
temp_base_list = list(reversed([createTempBase(nr + list_start, row) for nr, row in enumerate(array_slice)]))
# return lazy sequence of temp objects
return SequenceSliceMap(temp_base_list, list_lines, length)
master/bt5/slapos_metadata_transform_agent/SkinTemplateItem/portal_skins/slapos_metadata_transform_agent/DataArray_getArrayFileInfoList.xml 0 → 100644
View file @ a44366d8
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>Script_magic</string> </key>
<value> <int>3</int> </value>
</item>
<item>
<key> <string>_bind_names</string> </key>
<value>
<object>
<klass>
<global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
</klass>
<tuple/>
<state>
<dictionary>
<item>
<key> <string>_asgns</string> </key>
<value>
<dictionary>
<item>
<key> <string>name_container</string> </key>
<value> <string>container</string> </value>
</item>
<item>
<key> <string>name_context</string> </key>
<value> <string>context</string> </value>
</item>
<item>
<key> <string>name_m_self</string> </key>
<value> <string>script</string> </value>
</item>
<item>
<key> <string>name_subpath</string> </key>
<value> <string>traverse_subpath</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</state>
</object>
</value>
</item>
<item>
<key> <string>_params</string> </key>
<value> <string>list_start=0, list_lines=15, limit=1000, **kw</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>DataArray_getArrayFileInfoList</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
master/bt5/slapos_metadata_transform_agent/SkinTemplateItem/portal_skins/slapos_metadata_transform_agent/DataArray_view.xml 0 → 100644
View file @ a44366d8
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="ERP5 Form" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_objects</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>action</string> </key>
<value> <string>Base_edit</string> </value>
</item>
<item>
<key> <string>description</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>edit_order</string> </key>
<value>
<list/>
</value>
</item>
<item>
<key> <string>encoding</string> </key>
<value> <string>UTF-8</string> </value>
</item>
<item>
<key> <string>enctype</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>group_list</string> </key>
<value>
<list>
<string>left</string>
<string>right</string>
<string>center</string>
<string>bottom</string>
<string>hidden</string>
</list>
</value>
</item>
<item>
<key> <string>groups</string> </key>
<value>
<dictionary>
<item>
<key> <string>bottom</string> </key>
<value>
<list>
<string>listbox</string>
</list>
</value>
</item>
<item>
<key> <string>center</string> </key>
<value>
<list>
<string>my_description</string>
</list>
</value>
</item>
<item>
<key> <string>hidden</string> </key>
<value>
<list/>
</value>
</item>
<item>
<key> <string>left</string> </key>
<value>
<list>
<string>my_title</string>
<string>my_reference</string>
<string>my_version</string>
<string>my_publication_section_list</string>
<string>my_array_dtype</string>
<string>my_array_shape</string>
</list>
</value>
</item>
<item>
<key> <string>right</string> </key>
<value>
<list>
<string>my_start_date</string>
<string>my_predecessor_title_list</string>
<string>my_causality_title</string>
<string>my_translated_simulation_state_title</string>
</list>
</value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>DataArray_view</string> </value>
</item>
<item>
<key> <string>method</string> </key>
<value> <string>POST</string> </value>
</item>
<item>
<key> <string>name</string> </key>
<value> <string>DataArray_view</string> </value>
</item>
<item>
<key> <string>pt</string> </key>
<value> <string>form_view</string> </value>
</item>
<item>
<key> <string>row_length</string> </key>
<value> <int>4</int> </value>
</item>
<item>
<key> <string>stored_encoding</string> </key>
<value> <string>UTF-8</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Data Array</string> </value>
</item>
<item>
<key> <string>unicode_mode</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>update_action</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>update_action_title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
master/bt5/slapos_metadata_transform_agent/SkinTemplateItem/portal_skins/slapos_metadata_transform_agent/DataArray_view/listbox.xml 0 → 100644
View file @ a44366d8
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="ProxyField" module="Products.ERP5Form.ProxyField"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>delegated_list</string> </key>
<value>
<list>
<string>columns</string>
<string>list_method</string>
<string>portal_types</string>
<string>selection_name</string>
<string>title</string>
</list>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>listbox</string> </value>
</item>
<item>
<key> <string>message_values</string> </key>
<value>
<dictionary>
<item>
<key> <string>external_validator_failed</string> </key>
<value> <string>The input failed the external validator.</string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>overrides</string> </key>
<value>
<dictionary>
<item>
<key> <string>field_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>target</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>tales</string> </key>
<value>
<dictionary>
<item>
<key> <string>field_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>target</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>values</string> </key>
<value>
<dictionary>
<item>
<key> <string>columns</string> </key>
<value>
<list>
<tuple>
<string>reference</string>
<string>Reference</string>
</tuple>
<tuple>
<string>dtype</string>
<string>Dtype</string>
</tuple>
<tuple>
<string>index_expression</string>
<string>Index Expression</string>
</tuple>
</list>
</value>
</item>
<item>
<key> <string>field_id</string> </key>
<value> <string>my_view_mode_listbox</string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string>Base_viewFieldLibrary</string> </value>
</item>
<item>
<key> <string>list_method</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
</value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value>
<list>
<tuple>
<string>Data Acquisition Sensor</string>
<string>Data Acquisition Sensor</string>
</tuple>
</list>
</value>
</item>
<item>
<key> <string>portal_types</string> </key>
<value>
<list>
<tuple>
<string>Data Array Line</string>
<string>Data Array Line</string>
</tuple>
</list>
</value>
</item>
<item>
<key> <string>selection_name</string> </key>
<value> <string>data_array_line</string> </value>
</item>
<item>
<key> <string>target</string> </key>
<value> <string>Click to edit the target</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Data Array Lines</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="2" aka="AAAAAAAAAAI=">
<pickle>
<global name="Method" module="Products.Formulator.MethodField"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>method_name</string> </key>
<value> <string>objectValues</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
master/bt5/slapos_metadata_transform_agent/SkinTemplateItem/portal_skins/slapos_metadata_transform_agent/DataArray_view/my_causality_title.xml 0 → 100644
View file @ a44366d8
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="ProxyField" module="Products.ERP5Form.ProxyField"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>delegated_list</string> </key>
<value>
<list>
<string>portal_type</string>
<string>title</string>
</list>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>my_causality_title</string> </value>
</item>
<item>
<key> <string>message_values</string> </key>
<value>
<dictionary>
<item>
<key> <string>external_validator_failed</string> </key>
<value> <string>The input failed the external validator.</string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>overrides</string> </key>
<value>
<dictionary>
<item>
<key> <string>field_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>tales</string> </key>
<value>
<dictionary>
<item>
<key> <string>field_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>values</string> </key>
<value>
<dictionary>
<item>
<key> <string>field_id</string> </key>
<value> <string>my_relation_field</string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string>Base_viewFieldLibrary</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value>
<list>
<tuple>
<string>Data Product</string>
<string>Data Product</string>
</tuple>
</list>
</value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Data Product</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
master/bt5/slapos_metadata_transform_agent/SkinTemplateItem/portal_skins/slapos_metadata_transform_agent/DataArray_view/my_description.xml 0 → 100644
View file @ a44366d8
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="ProxyField" module="Products.ERP5Form.ProxyField"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>delegated_list</string> </key>
<value>
<list/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>my_description</string> </value>
</item>
<item>
<key> <string>message_values</string> </key>
<value>
<dictionary>
<item>
<key> <string>external_validator_failed</string> </key>
<value> <string>The input failed the external validator.</string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>overrides</string> </key>
<value>
<dictionary>
<item>
<key> <string>alternate_name</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>css_class</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>default</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>description</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>editable</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>enabled</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>external_validator</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>extra_context</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>field_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>hidden</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>tales</string> </key>
<value>
<dictionary>
<item>
<key> <string>extra_context</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>field_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>values</string> </key>
<value>
<dictionary>
<item>
<key> <string>extra_context</string> </key>
<value>
<list/>
</value>
</item>
<item>
<key> <string>field_id</string> </key>
<value> <string>my_description</string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string>Base_viewPDMFieldLibrary</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
master/bt5/slapos_metadata_transform_agent/SkinTemplateItem/portal_skins/slapos_metadata_transform_agent/DataArray_view/my_predecessor_title_list.xml 0 → 100644
View file @ a44366d8
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="ProxyField" module="Products.ERP5Form.ProxyField"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>delegated_list</string> </key>
<value>
<list>
<string>height</string>
<string>portal_type</string>
<string>title</string>
<string>view_separator</string>
</list>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>my_predecessor_title_list</string> </value>
</item>
<item>
<key> <string>message_values</string> </key>
<value>
<dictionary>
<item>
<key> <string>external_validator_failed</string> </key>
<value> <string>The input failed the external validator.</string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>overrides</string> </key>
<value>
<dictionary>
<item>
<key> <string>field_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>tales</string> </key>
<value>
<dictionary>
<item>
<key> <string>field_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>values</string> </key>
<value>
<dictionary>
<item>
<key> <string>field_id</string> </key>
<value> <string>my_multi_relation_field</string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string>Base_viewFieldLibrary</string> </value>
</item>
<item>
<key> <string>height</string> </key>
<value> <int>5</int> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value>
<list>
<tuple>
<string>Data Array</string>
<string>Data Array</string>
</tuple>
</list>
</value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Predecessors</string> </value>
</item>
<item>
<key> <string>view_separator</string> </key>
<value> <string encoding="cdata"><![CDATA[
<br />
]]></string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
master/bt5/slapos_metadata_transform_agent/SkinTemplateItem/portal_skins/slapos_metadata_transform_agent/DataArray_view/my_publication_section_list.xml 0 → 100644
View file @ a44366d8
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="ProxyField" module="Products.ERP5Form.ProxyField"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>delegated_list</string> </key>
<value>
<list/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>my_publication_section_list</string> </value>
</item>
<item>
<key> <string>message_values</string> </key>
<value>
<dictionary>
<item>
<key> <string>external_validator_failed</string> </key>
<value> <string>The input failed the external validator.</string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>overrides</string> </key>
<value>
<dictionary>
<item>
<key> <string>field_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>target</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>tales</string> </key>
<value>
<dictionary>
<item>
<key> <string>field_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>target</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>values</string> </key>
<value>
<dictionary>
<item>
<key> <string>field_id</string> </key>
<value> <string>my_view_mode_publication_section_list</string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string>Base_viewDMSFieldLibrary</string> </value>
</item>
<item>
<key> <string>target</string> </key>
<value> <string>Click to edit the target</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
master/bt5/slapos_metadata_transform_agent/SkinTemplateItem/portal_skins/slapos_metadata_transform_agent/DataArray_view/my_reference.xml 0 → 100644
View file @ a44366d8
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="ProxyField" module="Products.ERP5Form.ProxyField"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>delegated_list</string> </key>
<value>
<list/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>my_reference</string> </value>
</item>
<item>
<key> <string>message_values</string> </key>
<value>
<dictionary>
<item>
<key> <string>external_validator_failed</string> </key>
<value> <string>The input failed the external validator.</string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>overrides</string> </key>
<value>
<dictionary>
<item>
<key> <string>field_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>target</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>tales</string> </key>
<value>
<dictionary>
<item>
<key> <string>field_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>target</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>values</string> </key>
<value>
<dictionary>
<item>
<key> <string>field_id</string> </key>
<value> <string>my_view_mode_reference</string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string>Base_viewDMSFieldLibrary</string> </value>
</item>
<item>
<key> <string>target</string> </key>
<value> <string>Click to edit the target</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
master/bt5/slapos_metadata_transform_agent/SkinTemplateItem/portal_skins/slapos_metadata_transform_agent/DataArray_view/my_title.xml 0 → 100644
View file @ a44366d8
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="ProxyField" module="Products.ERP5Form.ProxyField"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>delegated_list</string> </key>
<value>
<list/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>my_title</string> </value>
</item>
<item>
<key> <string>message_values</string> </key>
<value>
<dictionary>
<item>
<key> <string>external_validator_failed</string> </key>
<value> <string>The input failed the external validator.</string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>overrides</string> </key>
<value>
<dictionary>
<item>
<key> <string>field_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>target</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>tales</string> </key>
<value>
<dictionary>
<item>
<key> <string>field_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>target</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>values</string> </key>
<value>
<dictionary>
<item>
<key> <string>field_id</string> </key>
<value> <string>my_view_mode_title</string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string>Base_viewFieldLibrary</string> </value>
</item>
<item>
<key> <string>target</string> </key>
<value> <string>Click to edit the target</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
master/bt5/slapos_metadata_transform_agent/SkinTemplateItem/portal_skins/slapos_metadata_transform_agent/DataArray_view/my_version.xml 0 → 100644
View file @ a44366d8
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="ProxyField" module="Products.ERP5Form.ProxyField"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>delegated_list</string> </key>
<value>
<list>
<string>description</string>
</list>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>my_version</string> </value>
</item>
<item>
<key> <string>message_values</string> </key>
<value>
<dictionary>
<item>
<key> <string>external_validator_failed</string> </key>
<value> <string>The input failed the external validator.</string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>overrides</string> </key>
<value>
<dictionary>
<item>
<key> <string>field_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>target</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>tales</string> </key>
<value>
<dictionary>
<item>
<key> <string>field_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>target</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>values</string> </key>
<value>
<dictionary>
<item>
<key> <string>description</string> </key>
<value> <string>Version of the web page</string> </value>
</item>
<item>
<key> <string>field_id</string> </key>
<value> <string>my_view_mode_version</string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string>Base_viewFieldLibrary</string> </value>
</item>
<item>
<key> <string>target</string> </key>
<value> <string>Click to edit the target</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
master/bt5/slapos_metadata_transform_agent/SkinTemplateItem/portal_skins/slapos_metadata_transform_agent/DataArray_viewFileMeta.xml 0 → 100644
View file @ a44366d8
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="ERP5 Form" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_objects</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>action</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>action_title</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>description</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>edit_order</string> </key>
<value>
<list/>
</value>
</item>
<item>
<key> <string>encoding</string> </key>
<value> <string>UTF-8</string> </value>
</item>
<item>
<key> <string>enctype</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>group_list</string> </key>
<value>
<list>
<string>left</string>
<string>right</string>
<string>center</string>
<string>bottom</string>
<string>hidden</string>
</list>
</value>
</item>
<item>
<key> <string>groups</string> </key>
<value>
<dictionary>
<item>
<key> <string>bottom</string> </key>
<value>
<list>
<string>listbox</string>
</list>
</value>
</item>
<item>
<key> <string>center</string> </key>
<value>
<list/>
</value>
</item>
<item>
<key> <string>hidden</string> </key>
<value>
<list/>
</value>
</item>
<item>
<key> <string>left</string> </key>
<value>
<list>
<string>my_title</string>
</list>
</value>
</item>
<item>
<key> <string>right</string> </key>
<value>
<list/>
</value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>DataArray_viewFileMeta</string> </value>
</item>
<item>
<key> <string>method</string> </key>
<value> <string>POST</string> </value>
</item>
<item>
<key> <string>name</string> </key>
<value> <string>DataArray_viewFileMeta</string> </value>
</item>
<item>
<key> <string>pt</string> </key>
<value> <string>form_view</string> </value>
</item>
<item>
<key> <string>row_length</string> </key>
<value> <int>4</int> </value>
</item>
<item>
<key> <string>stored_encoding</string> </key>
<value> <string>UTF-8</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>File Meta</string> </value>
</item>
<item>
<key> <string>unicode_mode</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>update_action</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>update_action_title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
master/bt5/slapos_metadata_transform_agent/SkinTemplateItem/portal_skins/slapos_metadata_transform_agent/DataArray_viewFileMeta/listbox.xml 0 → 100644
View file @ a44366d8
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="ProxyField" module="Products.ERP5Form.ProxyField"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>delegated_list</string> </key>
<value>
<list>
<string>columns</string>
<string>editable</string>
<string>hidden</string>
<string>list_method</string>
<string>title</string>
<string>url_columns</string>
</list>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>listbox</string> </value>
</item>
<item>
<key> <string>message_values</string> </key>
<value>
<dictionary>
<item>
<key> <string>external_validator_failed</string> </key>
<value> <string>The input failed the external validator.</string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>overrides</string> </key>
<value>
<dictionary>
<item>
<key> <string>field_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>target</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>tales</string> </key>
<value>
<dictionary>
<item>
<key> <string>columns</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
</value>
</item>
<item>
<key> <string>editable</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>field_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>hidden</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
</value>
</item>
<item>
<key> <string>list_method</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>target</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>url_columns</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>values</string> </key>
<value>
<dictionary>
<item>
<key> <string>columns</string> </key>
<value>
<list>
<tuple>
<string>title</string>
<string>title</string>
</tuple>
</list>
</value>
</item>
<item>
<key> <string>editable</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>field_id</string> </key>
<value> <string>my_view_mode_listbox</string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string>Base_viewFieldLibrary</string> </value>
</item>
<item>
<key> <string>hidden</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>list_method</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAQ=</string> </persistent>
</value>
</item>
<item>
<key> <string>target</string> </key>
<value> <string>Click to edit the target</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Preview</string> </value>
</item>
<item>
<key> <string>url_columns</string> </key>
<value>
<list/>
</value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="2" aka="AAAAAAAAAAI=">
<pickle>
<global name="TALESMethod" module="Products.Formulator.TALESField"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_text</string> </key>
<value> <string>python: here.DataArray_getArrayColumnList()</string> </value>
</item>
</dictionary>
</pickle>
</record>
<record id="3" aka="AAAAAAAAAAM=">
<pickle>
<global name="TALESMethod" module="Products.Formulator.TALESField"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_text</string> </key>
<value> <string>python: context.getArrayShape()[0] == 0</string> </value>
</item>
</dictionary>
</pickle>
</record>
<record id="4" aka="AAAAAAAAAAQ=">
<pickle>
<global name="Method" module="Products.Formulator.MethodField"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>method_name</string> </key>
<value> <string>DataArray_getArrayFileInfoList</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
master/bt5/slapos_metadata_transform_agent/SkinTemplateItem/portal_skins/slapos_metadata_transform_agent/DataArray_viewFileMeta/my_title.xml 0 → 100644
View file @ a44366d8
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="ProxyField" module="Products.ERP5Form.ProxyField"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>delegated_list</string> </key>
<value>
<list>
<string>editable</string>
</list>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>my_title</string> </value>
</item>
<item>
<key> <string>message_values</string> </key>
<value>
<dictionary>
<item>
<key> <string>external_validator_failed</string> </key>
<value> <string>The input failed the external validator.</string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>overrides</string> </key>
<value>
<dictionary>
<item>
<key> <string>field_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>target</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>tales</string> </key>
<value>
<dictionary>
<item>
<key> <string>field_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>target</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>values</string> </key>
<value>
<dictionary>
<item>
<key> <string>editable</string> </key>
<value> <int>0</int> </value>
</item>
<item>
<key> <string>field_id</string> </key>
<value> <string>my_view_mode_title</string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string>Base_viewFieldLibrary</string> </value>
</item>
<item>
<key> <string>target</string> </key>
<value> <string>Click to edit the target</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
master/bt5/slapos_metadata_transform_agent/SkinTemplateItem/portal_skins/slapos_metadata_transform_agent/MetadataIngestionPolicy_getIngestionOperationAndParameterDict.py deleted 100644 → 0
View file @ e98b32cf
from DateTime import DateTime
now = DateTime()
today_string = now.strftime('%Y%m%d')
portal = context.getPortalObject()
portal_catalog = portal.portal_catalog
data_ingestion_reference = movement_dict.get('reference', reference)
resource_reference = movement_dict.get('resource_reference', None)
specialise_reference = movement_dict.get('specialise_reference', None)
data_ingestion_id = "%s-%s" %(today_string, data_ingestion_reference)
data_ingestion_query_kw = dict(
portal_type = 'Data Ingestion',
simulation_state = ['started', 'stopped'],
reference = data_ingestion_reference)
# first search for applicable data ingestion
data_ingestion = portal_catalog.getResultValue(**data_ingestion_query_kw)
def init_input_line(input_line, operation_line):
# copy device and configuration from operation line to input line
input_line.setAggregateSet(
input_line.getAggregateList() + operation_line.getAggregateList())
# Check if we have a batch referece
data_ingestion_batch_reference = movement_dict.get(
'aggregate_data_ingestion_batch_reference',
None)
data_ingestion_batch_id = "%s-%s" %(today_string,
data_ingestion_batch_reference)
data_sink_type_list = []
data_sink_list = []
data_product = None
if data_ingestion_batch_reference is not None:
data_ingestion_batch = portal_catalog.getResultValue(
portal_type = "Data Ingestion Batch",
reference = data_ingestion_batch_reference)
if data_ingestion_batch is None:
data_ingestion_batch = portal.data_ingestion_batch_module.get(
data_ingestion_batch_id)
if data_ingestion_batch is None:
data_ingestion_batch = portal.data_ingestion_batch_module.newContent(
id = data_ingestion_batch_id,
portal_type = "Data Ingestion Batch",
reference = data_ingestion_batch_reference)
else:
previous_data_ingestion_line = portal_catalog.getResultValue(
portal_type = "Data Ingestion Line",
resource_reference = resource_reference,
aggregate_uid = data_ingestion_batch.getUid())
if previous_data_ingestion_line is not None:
data_product = previous_data_ingestion_line.getResourceValue()
data_sink_type_list = data_product.getAggregatedPortalTypeList()
data_sink_list = previous_data_ingestion_line\
.getAggregateValueList(portal_type=data_sink_type_list)
input_line.setDefaultAggregateValue(data_ingestion_batch)
if not data_sink_list:
if not data_sink_type_list:
if data_product is None:
data_product = portal.portal_catalog.getResultValue(
portal_type = "Data Product",
reference = resource_reference)
data_sink_type_list = data_product.getAggregatedPortalTypeList()
for data_sink_type in data_sink_type_list:
# This should be more generic
if data_sink_type not in ("Progress Indicator", "Data Ingestion Batch"):
# first try to find existing validated data sink
# with same device, project, resource (but can be different source)
data_sink = portal.portal_catalog.getResultValue(
portal_type=data_sink_type,
validation_state="validated",
item_device_relative_url=operation_line.getAggregateDevice(),
item_project_relative_url=input_line.getDestinationProject(),
item_resource_uid=input_line.getResourceUid())
if data_sink is None:
data_sink = portal.getDefaultModule(data_sink_type).newContent(
portal_type = data_sink_type,
reference = "%s-%s" %(data_ingestion_reference, resource_reference))
data_sink.validate()
if data_sink_type == 'Data Stream':
data_sink.setPublicationSectionList(data_product.getPublicationSectionList())
data_sink_list.append(data_sink)
input_line.setAggregateValueList(
input_line.getAggregateValueList() + data_sink_list)
input_line.setQuantity(1)
if data_ingestion is None:
document = portal.data_ingestion_module.get(data_ingestion_id)
if (document is not None) and document.getSimulationState() in ('started', 'stopped'):
data_ingestion = document
if modify and data_ingestion is None:
specialise_query_kw = dict(portal_type = 'Data Supply',
reference = specialise_reference,
validation_state = 'validated')
specialise_list = [x.getRelativeUrl() for x in portal_catalog(**specialise_query_kw)]
# if we do not find a validated data supply, we look for a default data supply
if not specialise_list:
specialise_query_kw["validation_state"] = 'default'
specialise_list = [x.getRelativeUrl() for x in portal_catalog(**specialise_query_kw)]
# create a new data ingestion
data_ingestion = portal.ERP5Site_createDataIngestion(specialise_list,
data_ingestion_reference,
data_ingestion_id)
# find ingestion line for current resource
for line in data_ingestion.objectValues(portal_type="Data Ingestion Line"):
if line.getResourceReference() == resource_reference:
input_line = line
elif line.getResourceValue().getPortalType() == "Data Operation":
operation_line = line
if modify and input_line.getQuantity() == 0:
init_input_line(input_line, operation_line)
data_operation = operation_line.getResourceValue()
parameter_dict = {
input_line.getReference(): \
{v.getPortalType(): v for v in input_line.getAggregateValueList()}}
bucket_reference = movement_dict.get('bucket_reference', None)
if bucket_reference is not None:
parameter_dict['bucket_reference'] = bucket_reference
return data_operation, parameter_dict
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7