erp5_wendelin_telecom_ingestion: metadata for tsv files

bt5/erp5_wendelin_telecom_ingestion/ExtensionTemplateItem/portal_components/extension.erp5.external_process_raw_data.py

@@ -70,15 +70,17 @@ def getMNEReportJSON(file_name, raw):
 
 def getRawData(file_name):
   raw = None
-  try:
-    raw = mne.io.read_raw_fif(file_name, preload=False, verbose=None)
-  except:
-    pass
-  if raw is None:
+  raw_functions = [mne.io.read_raw_fif, mne.io.read_raw_edf, mne.io.read_events_eeglab, mne.io.read_annotations_eeglab, mne.io.read_raw_eeglab,
+                   mne.io.read_raw_artemis123, mne.io.read_raw_bti, mne.io.read_raw_cnt, mne.io.read_raw_ctf, mne.io.read_raw_brainvision,
+                   mne.io.read_raw_kit, mne.read_epochs_kit, mne.io.read_raw_nicolet, mne.io.read_raw_eeglab, mne.io.read_raw_brainvision,
+                   mne.io.read_raw_egi, mne.io.read_info, mne.io.read_epochs_eeglab]
+  for function in raw_functions:
     try:
-      raw = mne.io.read_raw_edf(file_name, preload=False, verbose=None)
-    except:
-      pass
+      raw = function(file_name)
+      print("Function %s could read file %s " % (str(function), file_name))
+      break
+    except Exception as e:
+      print("Exception reading raw data with function %s: %s" % (str(function), str(e)))
   if raw is None: raise StandardError("the file does not contain raw data.")
   return raw
 
@@ -117,7 +119,7 @@ def processTextData(file_name, data_array, data_descriptor):
   except Exception as e:
     log("Error handling Data Descriptor content: " + str(e))
 
-def processCsvData(file_name, data_array, data_descriptor):
+def processCsvData(file_name, data_array, data_descriptor, delimiter=","):
   def gen_csv_chunks(reader, chunksize=CHUNK_SIZE_CSV):
     chunk = []
     for index, line in enumerate(reader):
@@ -149,7 +151,7 @@ def processCsvData(file_name, data_array, data_descriptor):
 
   try:
     with open(file_name, 'r') as csv_file:
-      reader = csv.reader(csv_file)
+      reader = csv.reader(csv_file, delimiter=delimiter)
       csv_file.seek(0)
       for i, chunk in enumerate(gen_csv_chunks(reader)):
         if i==0:
@@ -174,6 +176,21 @@ def processCsvData(file_name, data_array, data_descriptor):
   except Exception as e:
     log("Error handling csv Data Descriptor content: " + str(e))
 
+def processTsvData(file_name, data_array, data_descriptor):
+  processCsvData(file_name, data_array, data_descriptor, delimiter="\t")
+
+def processNiiData(file_name, data_array, data_descriptor):
+  data = {}
+  data["File content sample: "] = "nifti files data processing not available yet."
+  json_data = json.dumps(data)
+  data_descriptor.setTextContent(json_data)
+
+def processGZData(file_name, data_array, data_descriptor):
+  if data_array.getReference().endswith(".nii/gz"):
+    processNiiData(file_name, data_array, data_descriptor)
+  else:
+    raise KeyError("gz")
+
 def processRawData(data_stream, data_array, data_descriptor, reference_extension):
   import time
   start = time.time()
@@ -187,8 +204,12 @@ def processRawData(data_stream, data_array, data_descriptor, reference_extension
     return "Error while processing raw data - saving file: " + str(e)
 
   options = {"fif" : processFifData,
+             "nii" : processNiiData,
+             "mgz" : processNiiData,
              "txt" : processTextData,
              "csv" : processCsvData,
+             "tsv" : processTsvData,
+             "gz"  : processGZData,
              "default" : processTextData,
             }
   try:

bt5/erp5_wendelin_telecom_ingestion/ExtensionTemplateItem/portal_components/extension.erp5.external_process_raw_data.xml

@@ -52,11 +52,9 @@
             <key> <string>text_content_warning_message</string> </key>
             <value>
               <tuple>
-                <string>W: 75,  2: No exception type(s) specified (bare-except)</string>
-                <string>W: 80,  4: No exception type(s) specified (bare-except)</string>
-                <string>W: 98,  8: Unused variable \'times\' (unused-variable)</string>
-                <string>W:135,  6: No exception type(s) specified (bare-except)</string>
-                <string>W:171,  8: Unreachable code (unreachable)</string>
+                <string>W:100,  8: Unused variable \'times\' (unused-variable)</string>
+                <string>W:137,  6: No exception type(s) specified (bare-except)</string>
+                <string>W:173,  8: Unreachable code (unreachable)</string>
               </tuple>
             </value>
         </item>