Roque data lake

See merge request !66

bt5/erp5_wendelin_data_lake_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_data_lake/ERP5Site_getDataStreamList.py

@@ -17,7 +17,7 @@ except Exception as e: # fails because unauthorized access
 
 data_stream_dict = {}
 for stream in data_set.DataSet_getDataStreamList():
-  if not portal.ERP5Site_checkReferenceInvalidated(stream) and stream.getValidationState() != "draft":
+  if stream and not portal.ERP5Site_checkReferenceInvalidated(stream) and stream.getValidationState() != "draft":
     data_stream_info_dict = { 'id': 'data_stream_module/'+stream.getId(),
                               'size': stream.getSize(),
                               'hash': stream.getVersion() }

bt5/erp5_wendelin_data_lake_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_data_lake/ERP5Site_invalidateOldDatasets.py

+"""This script invalidate all data sets (and corresponding ingestion objects) older than wendelin.io release (<= 2019)"""
+
+portal = context.getPortalObject()
+portal_catalog = portal.portal_catalog
+
+from DateTime import DateTime
+old_date = DateTime(2019, 12, 31)
+catalog_kw = {'modification_date': {'query': old_date, 'range': '<='}}
+
+print "Following Dataset were invalidated:"
+
+for data_set in portal_catalog(portal_type="Data Set", **catalog_kw):
+  print
+  print "DATASET: " + data_set.getReference()
+  print "state: " + data_set.getValidationState()
+  print "date: " + str(data_set.getModificationDate())
+  print "len of datastream list: " + str(len(data_set.DataSet_getDataStreamList()))
+  for data_stream in data_set.DataSet_getDataStreamList():
+    if data_stream is not None:
+      portal.ERP5Site_invalidateIngestionObjects(data_stream.getReference())
+      try:
+        data_stream.invalidate()
+      except:
+        pass # fails if it's already invalidated, draft or if it doens't allow invalidation (e.g. DI)
+  portal.ERP5Site_invalidateReference(data_set)
+  try:
+    data_set.invalidate()
+  except:
+    pass # fails if it's already invalidated, draft or if it doens't allow invalidation (e.g. DI)
+return printed

bt5/erp5_wendelin_data_lake_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_data_lake/ERP5Site_invalidateOldDatasets.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>Script_magic</string> </key>
+            <value> <int>3</int> </value>
+        </item>
+        <item>
+            <key> <string>_bind_names</string> </key>
+            <value>
+              <object>
+                <klass>
+                  <global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
+                </klass>
+                <tuple/>
+                <state>
+                  <dictionary>
+                    <item>
+                        <key> <string>_asgns</string> </key>
+                        <value>
+                          <dictionary>
+                            <item>
+                                <key> <string>name_container</string> </key>
+                                <value> <string>container</string> </value>
+                            </item>
+                            <item>
+                                <key> <string>name_context</string> </key>
+                                <value> <string>context</string> </value>
+                            </item>
+                            <item>
+                                <key> <string>name_m_self</string> </key>
+                                <value> <string>script</string> </value>
+                            </item>
+                            <item>
+                                <key> <string>name_subpath</string> </key>
+                                <value> <string>traverse_subpath</string> </value>
+                            </item>
+                          </dictionary>
+                        </value>
+                    </item>
+                  </dictionary>
+                </state>
+              </object>
+            </value>
+        </item>
+        <item>
+            <key> <string>_params</string> </key>
+            <value> <string></string> </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>ERP5Site_invalidateOldDatasets</string> </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+</ZopeData>

bt5/erp5_wendelin_data_lake_ingestion/WorkflowTemplateItem/portal_workflow/data_set_interaction_workflow/scripts/DataSet_publishDataStreamList.py

@@ -3,4 +3,5 @@
 """
 data_set = state_change['object']
 for data_stream in data_set.DataSet_getDataStreamList():
-  data_stream.activate().publish()
+  if data_stream and not context.getPortalObject().ERP5Site_checkReferenceInvalidated(data_stream) and data_stream.getValidationState() != 'draft':
+    data_stream.activate().publish()