Revert "Changes in 'erp5_wendelin_data_lake' ..."

This reverts commit 004be34a.

bt5/erp5_wendelin_configurator/PathTemplateItem/business_configuration_module/wendelin_configuration.xml

@@ -115,9 +115,9 @@
             <key> <string>categories</string> </key>
             <value>
               <tuple>
-                <string>specialise/portal_templates/54</string>
                 <string>resource/portal_workflow/erp5_wendelin_workflow</string>
                 <string>current_state/portal_workflow/erp5_wendelin_workflow/state_start</string>
+                <string>specialise/portal_templates/54</string>
               </tuple>
             </value>
         </item>
@@ -136,9 +136,11 @@
             <value>
               <object>
                 <klass>
-                  <global id="1.1" name="DateTime" module="DateTime.DateTime"/>
+                  <global id="1.1" name="_reconstructor" module="copy_reg"/>
                 </klass>
                 <tuple>
+                  <global id="1.2" name="DateTime" module="DateTime.DateTime"/>
+                  <global id="1.3" name="object" module="__builtin__"/>
                   <none/>
                 </tuple>
                 <state>
@@ -170,6 +172,8 @@
               <object>
                 <klass> <reference id="1.1"/> </klass>
                 <tuple>
+                  <reference id="1.2"/>
+                  <reference id="1.3"/>
                   <none/>
                 </tuple>
                 <state>

bt5/erp5_wendelin_configurator/SkinTemplateItem/portal_skins/erp5_configurator_wendelin/BusinessConfiguration_afterWendelinConfiguration.xml

@@ -6,18 +6,18 @@
     </pickle>
     <pickle>
       <dictionary>
-        <item>
-            <key> <string>Script_magic</string> </key>
-            <value> <int>3</int> </value>
-        </item>
         <item>
             <key> <string>_bind_names</string> </key>
             <value>
               <object>
                 <klass>
-                  <global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
+                  <global name="_reconstructor" module="copy_reg"/>
                 </klass>
-                <tuple/>
+                <tuple>
+                  <global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
+                  <global name="object" module="__builtin__"/>
+                  <none/>
+                </tuple>
                 <state>
                   <dictionary>
                     <item>

bt5/erp5_wendelin_configurator/SkinTemplateItem/portal_skins/erp5_configurator_wendelin/BusinessConfiguration_configureWendelinCategories.xml

@@ -6,18 +6,18 @@
     </pickle>
     <pickle>
       <dictionary>
-        <item>
-            <key> <string>Script_magic</string> </key>
-            <value> <int>3</int> </value>
-        </item>
         <item>
             <key> <string>_bind_names</string> </key>
             <value>
               <object>
                 <klass>
-                  <global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
+                  <global name="_reconstructor" module="copy_reg"/>
                 </klass>
-                <tuple/>
+                <tuple>
+                  <global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
+                  <global name="object" module="__builtin__"/>
+                  <none/>
+                </tuple>
                 <state>
                   <dictionary>
                     <item>

bt5/erp5_wendelin_configurator/SkinTemplateItem/portal_skins/erp5_configurator_wendelin/BusinessConfiguration_configureWendelinOrganisation.xml

@@ -6,18 +6,18 @@
     </pickle>
     <pickle>
       <dictionary>
-        <item>
-            <key> <string>Script_magic</string> </key>
-            <value> <int>3</int> </value>
-        </item>
         <item>
             <key> <string>_bind_names</string> </key>
             <value>
               <object>
                 <klass>
-                  <global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
+                  <global name="_reconstructor" module="copy_reg"/>
                 </klass>
-                <tuple/>
+                <tuple>
+                  <global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
+                  <global name="object" module="__builtin__"/>
+                  <none/>
+                </tuple>
                 <state>
                   <dictionary>
                     <item>

bt5/erp5_wendelin_configurator/SkinTemplateItem/portal_skins/erp5_configurator_wendelin/BusinessConfiguration_configureWendelinSetupDataNotebook.xml

@@ -6,18 +6,18 @@
     </pickle>
     <pickle>
       <dictionary>
-        <item>
-            <key> <string>Script_magic</string> </key>
-            <value> <int>3</int> </value>
-        </item>
         <item>
             <key> <string>_bind_names</string> </key>
             <value>
               <object>
                 <klass>
-                  <global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
+                  <global name="_reconstructor" module="copy_reg"/>
                 </klass>
-                <tuple/>
+                <tuple>
+                  <global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
+                  <global name="object" module="__builtin__"/>
+                  <none/>
+                </tuple>
                 <state>
                   <dictionary>
                     <item>

bt5/erp5_wendelin_configurator/SkinTemplateItem/portal_skins/erp5_configurator_wendelin/BusinessConfiguration_displayWendelinDownloadForm.xml

@@ -11,9 +11,13 @@
             <value>
               <object>
                 <klass>
-                  <global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
+                  <global name="_reconstructor" module="copy_reg"/>
                 </klass>
-                <tuple/>
+                <tuple>
+                  <global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
+                  <global name="object" module="__builtin__"/>
+                  <none/>
+                </tuple>
                 <state>
                   <dictionary>
                     <item>

bt5/erp5_wendelin_configurator/SkinTemplateItem/portal_skins/erp5_configurator_wendelin/BusinessConfiguration_getPrettyCategoriesSpreadsheetConfiguratorItem.xml

@@ -6,18 +6,18 @@
     </pickle>
     <pickle>
       <dictionary>
-        <item>
-            <key> <string>Script_magic</string> </key>
-            <value> <int>3</int> </value>
-        </item>
         <item>
             <key> <string>_bind_names</string> </key>
             <value>
               <object>
                 <klass>
-                  <global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
+                  <global name="_reconstructor" module="copy_reg"/>
                 </klass>
-                <tuple/>
+                <tuple>
+                  <global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
+                  <global name="object" module="__builtin__"/>
+                  <none/>
+                </tuple>
                 <state>
                   <dictionary>
                     <item>

bt5/erp5_wendelin_configurator/SkinTemplateItem/portal_skins/erp5_configurator_wendelin/BusinessConfiguration_setupWendelinStandardBT5.xml

@@ -6,18 +6,18 @@
     </pickle>
     <pickle>
       <dictionary>
-        <item>
-            <key> <string>Script_magic</string> </key>
-            <value> <int>3</int> </value>
-        </item>
         <item>
             <key> <string>_bind_names</string> </key>
             <value>
               <object>
                 <klass>
-                  <global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
+                  <global name="_reconstructor" module="copy_reg"/>
                 </klass>
-                <tuple/>
+                <tuple>
+                  <global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
+                  <global name="object" module="__builtin__"/>
+                  <none/>
+                </tuple>
                 <state>
                   <dictionary>
                     <item>

bt5/erp5_wendelin_configurator/SkinTemplateItem/portal_skins/erp5_configurator_wendelin/BusinessConfiguration_viewConfigureWendelinCategoriesDialog.xml

@@ -11,9 +11,13 @@
             <value>
               <object>
                 <klass>
-                  <global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
+                  <global name="_reconstructor" module="copy_reg"/>
                 </klass>
-                <tuple/>
+                <tuple>
+                  <global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
+                  <global name="object" module="__builtin__"/>
+                  <none/>
+                </tuple>
                 <state>
                   <dictionary>
                     <item>

bt5/erp5_wendelin_configurator/SkinTemplateItem/portal_skins/erp5_configurator_wendelin/BusinessConfiguration_viewWendelinConfigureOrganisationDialog.xml

@@ -11,9 +11,13 @@
             <value>
               <object>
                 <klass>
-                  <global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
+                  <global name="_reconstructor" module="copy_reg"/>
                 </klass>
-                <tuple/>
+                <tuple>
+                  <global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
+                  <global name="object" module="__builtin__"/>
+                  <none/>
+                </tuple>
                 <state>
                   <dictionary>
                     <item>

bt5/erp5_wendelin_configurator/SkinTemplateItem/portal_skins/erp5_configurator_wendelin/ERP5Site_createDefaultKnowledgeBox.xml

@@ -6,18 +6,18 @@
     </pickle>
     <pickle>
       <dictionary>
-        <item>
-            <key> <string>Script_magic</string> </key>
-            <value> <int>3</int> </value>
-        </item>
         <item>
             <key> <string>_bind_names</string> </key>
             <value>
               <object>
                 <klass>
-                  <global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
+                  <global name="_reconstructor" module="copy_reg"/>
                 </klass>
-                <tuple/>
+                <tuple>
+                  <global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
+                  <global name="object" module="__builtin__"/>
+                  <none/>
+                </tuple>
                 <state>
                   <dictionary>
                     <item>

bt5/erp5_wendelin_data_lake_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_data_lake/DataLake_stopIngestionList.py

 from erp5.component.module.Log import log
-from Products.ZSQLCatalog.SQLCatalog import Query, SimpleQuery
+from Products.ZSQLCatalog.SQLCatalog import Query
 import hashlib
 
 CHUNK_SIZE = 200000
@@ -14,7 +14,7 @@ def getHash(data_stream):
     end_offset = n_chunk*chunk_size+chunk_size
     try:
       data_stream_chunk = ''.join(data_stream.readChunkList(start_offset, end_offset))
-    except:
+    except Exception:
       # data stream is empty
       data_stream_chunk = ""
     hash_md5.update(data_stream_chunk)
@@ -24,7 +24,6 @@ def getHash(data_stream):
 
 def isInterruptedAbandonedSplitIngestion(reference):
   from DateTime import DateTime
-  now = DateTime()
   day_hours = 1.0/24/60*60*24
   # started split data ingestions for reference
   catalog_kw = {'portal_type': 'Data Ingestion',
@@ -90,8 +89,8 @@ for data_ingestion in portal_catalog(portal_type = "Data Ingestion",
               portal.data_stream_module.deleteContent(data_stream.getId())
         if last_data_stream_id.endswith(reference_end_split):
           portal.ERP5Site_invalidateSplitIngestions(data_ingestion.getReference(), success=True)
-          hash = getHash(full_data_stream)
-          full_data_stream.setVersion(hash)
+          full_data_stream_hash = getHash(full_data_stream)
+          full_data_stream.setVersion(full_data_stream_hash)
           if full_data_stream.getValidationState() != "validated":
             full_data_stream.validate()
           related_split_ingestions = portal_catalog(portal_type = "Data Ingestion",

bt5/erp5_wendelin_data_lake_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_data_lake/ERP5Site_checkIngestionReferenceExists.py

@@ -44,7 +44,7 @@ if data_ingestion != None:
         # previous ingestion was interrumped
         log(''.join(["[WARNING] User has restarted an interrumpted ingestion for reference ", data_ingestion.getReference(), ". Previous split ingestions will be discarted and full ingestion restarted."]))
         portal.ERP5Site_invalidateSplitIngestions(data_ingestion.getReference(), success=False)
-  except:
+  except Exception:
     pass
   # the ingestion attemp corresponds to a split ingestion in course, accept
   return FALSE

bt5/erp5_wendelin_data_lake_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_data_lake/DataLake_createDataAnalysisList.py → bt5/erp5_wendelin_data_lake_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_data_lake/ERP5Site_createDataAnalysisList.py

 from DateTime import DateTime
 portal = context.getPortalObject()
 portal_catalog = portal.portal_catalog
-from erp5.component.module.Log import log
+
 now = DateTime()
 query_dict = {
   "portal_type": "Data Ingestion Line",

bt5/erp5_wendelin_data_lake_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_data_lake/ERP5_getDescriptorHTMLContent.py → bt5/erp5_wendelin_data_lake_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_data_lake/ERP5Site_getDescriptorHTMLContent.py

-from Products.ZSQLCatalog.SQLCatalog import Query, SimpleQuery, ComplexQuery
+from Products.ZSQLCatalog.SQLCatalog import Query, ComplexQuery
 
 portal = context.getPortalObject()
 portal_catalog = portal.portal_catalog

bt5/erp5_wendelin_data_lake_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_data_lake/ERP5Site_getIngestionConstantsJson.py

 import json
 portal = context.getPortalObject()
-dict = {'invalid_suffix':portal.ERP5Site_getIngestionReferenceDictionary()['invalid_suffix'],
+dictionary = {'invalid_suffix':portal.ERP5Site_getIngestionReferenceDictionary()['invalid_suffix'],
         'split_end_suffix':portal.ERP5Site_getIngestionReferenceDictionary()['split_end_suffix'],
         'single_end_suffix':portal.ERP5Site_getIngestionReferenceDictionary()['single_end_suffix'],
         'split_first_suffix':portal.ERP5Site_getIngestionReferenceDictionary()['split_first_suffix'],
@@ -10,4 +10,4 @@ dict = {'invalid_suffix':portal.ERP5Site_getIngestionReferenceDictionary()['inva
         'reference_length':portal.ERP5Site_getIngestionReferenceDictionary()['reference_length'],
         'invalid_chars':portal.ERP5Site_getIngestionReferenceDictionary()['invalid_chars'],
         }
-return json.dumps(dict)
+return json.dumps(dictionary)

bt5/erp5_wendelin_data_lake_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_data_lake/ERP5Site_increaseDatasetVersion.py

 portal = context.getPortalObject()
-portal_catalog = portal.portal_catalog
 
 data_set = portal.data_set_module.get(reference)
 if data_set is not None:

bt5/erp5_wendelin_data_lake_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_data_lake/ERP5Site_invalidateIngestionObjects.py

-from Products.ZSQLCatalog.SQLCatalog import Query, SimpleQuery, ComplexQuery
+from Products.ZSQLCatalog.SQLCatalog import Query, ComplexQuery
 
 portal = context.getPortalObject()
 portal_catalog = portal.portal_catalog
@@ -16,5 +16,5 @@ for document in portal_catalog(**kw_dict):
   portal.ERP5Site_invalidateReference(document)
   try:
     document.invalidate()
-  except:
+  except Exception:
     pass # fails if it's already invalidated, draft or if it doens't allow invalidation (e.g. DI)

bt5/erp5_wendelin_data_lake_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_data_lake/ERP5Site_invalidateOldDatasets.py

@@ -20,11 +20,11 @@ for data_set in portal_catalog(portal_type="Data Set", **catalog_kw):
       portal.ERP5Site_invalidateIngestionObjects(data_stream.getReference())
       try:
         data_stream.invalidate()
-      except:
+      except Exception:
         pass # fails if it's already invalidated, draft or if it doens't allow invalidation (e.g. DI)
   portal.ERP5Site_invalidateReference(data_set)
   try:
     data_set.invalidate()
-  except:
+  except Exception:
     pass # fails if it's already invalidated, draft or if it doens't allow invalidation (e.g. DI)
 return printed

bt5/erp5_wendelin_data_lake_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_data_lake/ERP5Site_setDatasetDescription.py

@@ -2,7 +2,6 @@ import base64
 import json
 
 portal = context.getPortalObject()
-portal_catalog = portal.portal_catalog
 
 request = context.REQUEST
 data_chunk = request.get('data_chunk')
@@ -11,10 +10,10 @@ data_set = portal.data_set_module.get(dataset)
 if data_set is not None:
   decoded = base64.b64decode(data_chunk)
   data_set.setDescription(decoded)
-  dict = { 'status_code': 0, 'message': 'Dataset description successfully set.' }
+  response = { 'status_code': 0, 'message': 'Dataset description successfully set.' }
 else:
   message = "No remote dataset found for reference '%s'" % (dataset)
-  dict = { 'status_code': 1, 'message': message }
+  response = { 'status_code': 1, 'message': message }
   context.logEntry(message)
 
-return json.dumps(dict)
+return json.dumps(response)

bt5/erp5_wendelin_data_lake_ingestion/TestTemplateItem/portal_components/test.erp5.testDataLakeIngestion.py

@@ -103,7 +103,7 @@ class TestDataIngestion(SecurityTestCase):
   def stepIngest(self, extension, delimiter, randomize_ingestion_reference=False, data_set_reference=False):
     file_name = "file_name.csv"
     reference = self.getRandomReference()
-    array = [[random.random() for i in range(self.CHUNK_SIZE_CSV + 10)] for j in range(self.CHUNK_SIZE_CSV + 10)]
+    array = [[random.random() for _ in range(self.CHUNK_SIZE_CSV + 10)] for _ in range(self.CHUNK_SIZE_CSV + 10)]
     np.savetxt(file_name, array, delimiter=delimiter)
     chunk = []
     with open(file_name, 'r') as csv_file:

bt5/erp5_wendelin_data_lake_ingestion/WorkflowTemplateItem/portal_workflow/data_validation_workflow/scripts/checkConsistency.py

-object = state_change['object']
-object.Base_checkConsistency()
+state_change_object = state_change["object"]
+state_change_object.Base_checkConsistency()

bt5/erp5_wendelin_examples_keras/ExtensionTemplateItem/portal_components/keras_save_load.py

 import warnings
 import numpy as np
-from keras import backend as K
-from keras import __version__ as keras_version
-from keras.models import Sequential
-from keras.models import model_from_config
-from keras.optimizers import optimizer_from_config
-from keras import optimizers
+from keras import backend as K # pylint:disable=import-error
+from keras import __version__ as keras_version # pylint:disable=import-error
+from keras.models import Sequential # pylint:disable=import-error
+from keras.models import model_from_config # pylint:disable=import-error
+from keras.optimizers import optimizer_from_config # pylint:disable=import-error
+from keras import optimizers # pylint:disable=import-error
 
 def save_model(model, model_store=None):
   data = {}
@@ -179,7 +179,6 @@ def load_model(data):
     else:
       model._make_train_function()
     optimizer_weights_dict = data['optimizer_weights']
-    optimizer_weight_names = optimizer_weights_dict['weight_names']
     optimizer_weight_values = optimizer_weights_dict['weight_values']
     model.optimizer.set_weights(optimizer_weight_values)
   return model
\ No newline at end of file

bt5/erp5_wendelin_examples_keras/ExtensionTemplateItem/portal_components/keras_train_model.py

 import numpy as np
 import time
-import sys
 import transaction
 
 class Progbar(object):
@@ -27,7 +26,7 @@ class Progbar(object):
     self.verbose = verbose
     self.output1 = output
 
-    def update(self, current, values=[], force=False):
+  def update(self, current, values=None, force=False):
     """Updates the progress bar.
 
     # Arguments
@@ -36,6 +35,10 @@ class Progbar(object):
             The progress bar will display averages for these values.
         force: Whether to force visual progress update.
     """
+
+    if values in None:
+      values = []
+
     for k, v in values:
       if k not in self.sum_values:
         self.sum_values[k] = [v * (current - self.seen_so_far),
@@ -115,11 +118,14 @@ class Progbar(object):
 
     self.last_update = now
 
-    def add(self, n, values=[]):
+  def add(self, n, values=None):
+    if values is None:
+      values = []
+
     self.update(self.seen_so_far + n, values)
 
 
-from keras.callbacks import ProgbarLogger as OriginalProgbarLogger
+from keras.callbacks import ProgbarLogger as OriginalProgbarLogger # pylint:disable=import-error
 
 class ProgbarLogger(OriginalProgbarLogger):
 
@@ -162,10 +168,10 @@ def train(portal):
   # 1. you can use keras.
   # 2. you can save trained model.
   # 3. you can load trained model.
-  from cStringIO import StringIO
-  import tensorflow as tf
+  # from cStringIO import StringIO
+  import tensorflow as tf # pylint:disable=import-error
   sess = tf.Session()
-  from keras import backend as K
+  from keras import backend as K # pylint:disable=import-error
   K.set_session(sess)
 
   stream = portal.data_stream_module.wendelin_examples_keras_log
@@ -176,8 +182,8 @@ def train(portal):
   if saved_model_data is not None:
     model = portal.keras_load_model(saved_model_data)
   else:
-    from keras.models import Sequential
-    from keras.layers import Dense
+    from keras.models import Sequential # pylint:disable=import-error
+    from keras.layers import Dense # pylint:disable=import-error
     model = Sequential()
     model.add(Dense(12, input_dim=8, init='uniform', activation='relu'))
     model.add(Dense(8, init='uniform', activation='relu'))

bt5/erp5_wendelin_examples_keras/ExtensionTemplateItem/portal_components/keras_vgg16_predict.py

-from keras.applications.vgg16 import VGG16, preprocess_input, decode_predictions
-from keras.preprocessing import image
+from keras.applications.vgg16 import VGG16, preprocess_input, decode_predictions # pylint:disable=import-error
+from keras.preprocessing import image # pylint:disable=import-error
 import numpy as np
 from cStringIO import StringIO
 import PIL.Image