Cleanup and minor changes in scripts.

bt5/erp5_wendelin_telecom_ingestion/PathTemplateItem/portal_callables/DataIngestionLine_writeEmbulkIngestionToDataStream.py

@@ -7,5 +7,6 @@ log("Decoded data_chunk size: " + str(len(decoded)))
 
 log("FROM SCRIPT %s: appending to data stream: %s." % (script.getId(), data_stream))
 
-
 data_stream.appendData(decoded)
+
+log("FROM SCRIPT %s: ingested data successfully appended." % (script.getId()))

bt5/erp5_wendelin_telecom_ingestion/PathTemplateItem/portal_callables/DataTransformation_transformFIFDataToDataArray.py

@@ -5,4 +5,4 @@ log("Processing raw data from Data Stream " + str(input_stream_data.getReference
 result = str(context.processRawData(input_stream_data, output_array, output_descriptor))
 log(result)
 
-log("Metadata stored in Data Descriptor " + str(output_descriptor.getReference()))
+log("Metadata stored in Data Descriptor " + str(output_descriptor))#.getReference()))

bt5/erp5_wendelin_telecom_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_telecom/IngestionPolicy_getIngestionOperationAndParameterDict.py

@@ -44,8 +44,8 @@ if data_ingestion is None:
   if not specialise_value_list:
     specialise_value_list = [x.getObject() for x in portal_catalog.searchResults(
       portal_type = 'Data Supply',
-      reference = specialise_reference,
-      validation_state = 'default')]
+      reference = 'default',
+      validation_state = 'validated')]
 
   log("Data Suppliers list: " + str(specialise_value_list))
 
@@ -85,15 +85,11 @@ if data_ingestion is None:
       reference = supply_line.getReference(),
       resource = supply_line.getResource(),
     )
-    #log("CURRENT LINE: " + str(current_line) + " - Resource: " + str(current_line.getResourceReference()))
     if current_line.getResourceReference() == resource_reference:
-      #log("CURRENT LINE is an INPUT LINE")
       input_line = current_line
     elif current_line.getResourceValue().getPortalType() == "Data Operation":
-      #log("CURRENT LINE is an D.OPERATION LINE")
       operation_line = current_line
     else:
-      #log("CURRENT LINE nor INPUT or D.OP LINE, SET QUANTITY TO 0")
       # we set quantity=0 for the empty line
       current_line.setQuantity(0)
 
@@ -118,7 +114,6 @@ if data_ingestion is None:
     data_set = portal_catalog.getResultValue(
       portal_type = "Data Set",
       reference = dataset_reference)
-    log("Data Set found for dataset reference: " + dataset_reference)
     if data_set is None:
       log("Creating a new Data Set")
       data_set = portal.data_set_module.newContent(
@@ -129,10 +124,10 @@ if data_ingestion is None:
         version = "001"
       )
       data_set.validate()
-
+    else:
+      log("Data Set found for dataset reference: " + data_set.getReference())
     input_line.setDefaultAggregateValue(data_set)
 
-
 else:
   # find ingestion line for current resource
   for line in data_ingestion.objectValues(portal_type="Data Ingestion Line"):