erp5_wendelin_drone: convert data if data line was deleted

bt5/erp5_wendelin_drone/PathTemplateItem/portal_callables/DataAnalysisLine_convertRawDroneDataToArray.py

 import pandas as pd
 import numpy as np
-from zExceptions import BadRequest
 
 # Function to remove non-ASCII characters from a string, because I can not be bothered to make to_records with utf8 right now
 def remove_non_ascii(text):
@@ -8,36 +7,34 @@ def remove_non_ascii(text):
 
 
 
-progress_indicator = in_stream["Progress Indicator"]
 in_data_stream = in_stream["Data Bucket Stream"]
-out_data_array = out_array["Data Array"]
-
 keys = in_data_stream.getKeyList()
-
-end = progress_indicator.getStringOffsetIndex()
-if end is None:
-  end = ""
-
 if len(keys) == 0:
   context.log("No Keys found")
   return
 
-
-dtype= {'timestamp (ms)': '<f8',
-'latitude ()': '<f8',
-'longitude ()': '<f8',
-'AMSL (m)': '<f8',
-'rel altitude (m)': '<f8',
-'yaw ()': '<f8',
-'ground speed (m/s)': '<f8',
-'climb rate (m/s)': '<f8'}
-
-if len([x for x in keys if x not in end]) == 0:
-  context.log("No new keys found")
-  return
-
-new_end = ""
-for key in [x for x in keys if x not in end]:
+out_data_array = out_array["Data Array"]
+if out_data_array.getArray() is not None:
+  key_list = [key for key in keys if not out_data_array.hasContent(key)]
+else:
+  key_list = keys
+
+if len(key_list) == 0:
+    context.log("No new keys found")
+    return
+
+dtype= {
+  'timestamp (ms)': '<f8',
+  'latitude ()': '<f8',
+  'longitude ()': '<f8',
+  'AMSL (m)': '<f8',
+  'rel altitude (m)': '<f8',
+  'yaw ()': '<f8',
+  'ground speed (m/s)': '<f8',
+  'climb rate (m/s)': '<f8',
+}
+
+for key in key_list:
   try:
     log = in_data_stream.getBucketByKey(key)
     df = pd.read_csv(log, sep=';', dtype=dtype)
@@ -55,25 +52,16 @@ for key in [x for x in keys if x not in end]:
     ndarray = df.to_records(index = False) #column_dtypes does not work here for some reasone, even if it is an actuall parameter
 
     zbigarray = out_data_array.getArray()
-
     if zbigarray is None:
       zbigarray = out_data_array.initArray(shape=(0,), dtype=ndarray.dtype.fields)
     start_array = zbigarray.shape[0]
     zbigarray.append(ndarray)
 
-    try:
-      data_array_line = out_array.get(key)
-      if data_array_line is None:
-        data_array_line = out_data_array.newContent(id=key,
-                                             portal_type="Data Array Line")
-      data_array_line.edit(reference=key,
-           index_expression="%s:%s" %(start_array, zbigarray.shape[0])
-        )
-    except BadRequest:
-      context.log("Data Array Line already exists")
+    data_array_line = out_data_array.newContent(id=key,
+                                                portal_type="Data Array Line")
+    data_array_line.edit(
+      reference=key,
+      index_expression="%s:%s" %(start_array, zbigarray.shape[0]),
+    )
   except ValueError:
     context.log("File "+str(key)+ " is not well formated")
-  new_end = new_end + str(key)
-
-
-progress_indicator.setStringOffsetIndex(end + new_end)