erp5_wendelin_telecom_ingestion: skip default metadata processing on big files

bt5/erp5_wendelin_telecom_ingestion/ExtensionTemplateItem/portal_components/extension.erp5.external_process_raw_data.py

@@ -11,6 +11,7 @@ from lxml.html import parse
 CHUNK_SIZE = 200000
 CHUNK_SIZE_TXT = 50000
 CHUNK_SIZE_CSV = 25
+BIG_FILE = 5000000000 #5GB
 
 def saveRawFile(data_stream, file_name):
   data_stream_chunk = None
@@ -213,6 +214,9 @@ def processRawData(data_stream, data_array, data_descriptor, reference_extension
   if data_stream.getSize() == 0:
     data_descriptor.setTextContent(json.dumps(content))
     return "Empty Data Stream"
+  if data_stream.getSize() > BIG_FILE:
+    log("Ingested file bigger than 5GB, get metadata process skiped")
+    return "File bigger than 5GB"
 
   file_name = "temporal_file_%s" % DateTime().strftime('%Y%m%d-%H%M%S')
   try:

bt5/erp5_wendelin_telecom_ingestion/ExtensionTemplateItem/portal_components/extension.erp5.external_process_raw_data.xml

@@ -52,13 +52,13 @@
             <key> <string>text_content_warning_message</string> </key>
             <value>
               <tuple>
-                <string>W: 73,  2: No exception type(s) specified (bare-except)</string>
-                <string>W: 78,  4: No exception type(s) specified (bare-except)</string>
-                <string>W: 94,  8: Unused variable \'times\' (unused-variable)</string>
-                <string>W:130,  6: No exception type(s) specified (bare-except)</string>
-                <string>W:165,  8: Unreachable code (unreachable)</string>
-                <string>W:183, 11: Using type() instead of isinstance() for a typecheck. (unidiomatic-typecheck)</string>
-                <string>W:187, 10: No exception type(s) specified (bare-except)</string>
+                <string>W: 74,  2: No exception type(s) specified (bare-except)</string>
+                <string>W: 79,  4: No exception type(s) specified (bare-except)</string>
+                <string>W: 95,  8: Unused variable \'times\' (unused-variable)</string>
+                <string>W:131,  6: No exception type(s) specified (bare-except)</string>
+                <string>W:166,  8: Unreachable code (unreachable)</string>
+                <string>W:184, 11: Using type() instead of isinstance() for a typecheck. (unidiomatic-typecheck)</string>
+                <string>W:188, 10: No exception type(s) specified (bare-except)</string>
               </tuple>
             </value>
         </item>