wendelin-scalability: add a scenario for ZBigArray processing

software/wendelin-scalability/test-neo.cfg

@@ -10,12 +10,15 @@
 #   $x 8.787 0
 #   0.100036621094
 #   6556
+#
+# The 'start_process' command is similar but by growing a ZBigArray object.
+# The random data has a compression ratio of 10%.
 
 [buildout]
 extends = test-common.cfg
-parts += start_ingest
+parts += start_ingest start_process
 
-[start_ingest]
+[start-script-common]
 recipe = slapos.recipe.template:jinja2
 rendered = ${buildout:bin-directory}/${:_buildout_section_name_}
 mode = 0755
@@ -27,20 +30,35 @@ template =
   _('--site-id', default='erp5')
   _('hostport', metavar='host[:port]', help='Zope address')
   _('password', help="'zope' user password")
-  _('reference', help='Data Stream reference')
-  _('mu', type=float)
-  _('sigma', type=float)
-  _('chunks_per_transaction', nargs='?', type=int, help='default: 128 (8 MiB)')
+  options = []
+  def option(name, **kw):
+    _(name, **kw)
+    options.append(name)
+  ${:options}
   args = parser.parse_args()
   qs = []
-  for k in 'reference', 'mu', 'sigma', 'chunks_per_transaction':
+  for k in options:
     v = getattr(args, k)
     if v is not None:
       t = type(v)
       qs.append('%s=%s' % (k if t is str else k + ':' + t.__name__, v))
   c = httplib.HTTPConnection(args.hostport)
-  c.putrequest('GET', '/%s/ERP5Site_simulateFluentdIngestion?%s'
-                      % (args.site_id, '&'.join(qs)))
+  c.putrequest('GET', '/%s/${:script}?%s' % (args.site_id, '&'.join(qs)))
   c.putheader('Authorization',
     'Basic ' + base64.b64encode('zope:'+args.password))
   c.endheaders()
+
+[start_ingest]
+<= start-script-common
+options =
+  option('id', help='Data Stream id')
+  option('mu', type=float)
+  option('sigma', type=float)
+  option('chunks_per_transaction', nargs='?', type=int, help='default: 128 (8 MiB)')
+script = ERP5Site_simulateFluentdIngestion
+
+[start_process]
+<= start-script-common
+options =
+  option('id', help='Data Array id')
+script = ERP5Site_dummyZBigArrayProcessing

software/wendelin-scalability/test_scalability_fluentd/ExtensionTemplateItem/portal_components/extension.erp5.ScalabilityFluentd.py

@@ -5,7 +5,7 @@ from random import lognormvariate
 
 bigfile_chunk_size = 65536
 
-def simulateFluentdIngestion(self, reference, mu, sigma,
+def simulateFluentdIngestion(self, id, mu, sigma,
                              chunks_per_transaction=128):
   from time import time
   import transaction
@@ -13,9 +13,9 @@ def simulateFluentdIngestion(self, reference, mu, sigma,
           + '/ingest')
   module = self['data_stream_module']
   try:
-    data_stream = module[reference]
+    data_stream = module[id]
   except KeyError:
-    data_stream = module.newContent(reference, 'Data Stream')
+    data_stream = module.newContent(id, 'Data Stream')
     transaction.commit()
 
   pack = struct.Struct('!d').pack

software/wendelin-scalability/test_scalability_fluentd/ExtensionTemplateItem/portal_components/extension.erp5.ScalabilityZBigArray.py

+def dummyZBigArrayProcessing(self, id):
+  import numpy as np
+  from random import randrange, sample
+  import transaction
+  module = self['data_array_module']
+  try:
+    array = module[id]
+  except KeyError:
+    array = module.newContent(id, 'Data Array')
+    array.initArray(shape=(0, 64), dtype=np.int32)
+    transaction.commit()
+  note = array.getPath() + '/new_data'
+  array = array.getArray()
+  rows, cols = array.shape
+
+  y = xrange(cols)
+  n = 10 * (2<<20) // (cols*4)
+  z = np.ndarray(shape=(n, cols), dtype=array.dtype)
+  for row in z:
+    for i in sample(y, 8):
+      row[i] = randrange(0, 1000)
+
+  while 1:
+    txn = transaction.begin()
+    np.random.shuffle(z)
+    rows += n
+    array.resize((rows, cols))
+    array[-n:] = z
+    txn.note(note)
+    txn.commit()

software/wendelin-scalability/test_scalability_fluentd/ExtensionTemplateItem/portal_components/extension.erp5.ScalabilityZBigArray.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="Extension Component" module="erp5.portal_type"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>_recorded_property_dict</string> </key>
+            <value>
+              <persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
+            </value>
+        </item>
+        <item>
+            <key> <string>default_reference</string> </key>
+            <value> <string>ScalabilityZBigArray</string> </value>
+        </item>
+        <item>
+            <key> <string>description</string> </key>
+            <value>
+              <none/>
+            </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>extension.erp5.ScalabilityZBigArray</string> </value>
+        </item>
+        <item>
+            <key> <string>portal_type</string> </key>
+            <value> <string>Extension Component</string> </value>
+        </item>
+        <item>
+            <key> <string>sid</string> </key>
+            <value>
+              <none/>
+            </value>
+        </item>
+        <item>
+            <key> <string>text_content_error_message</string> </key>
+            <value>
+              <tuple/>
+            </value>
+        </item>
+        <item>
+            <key> <string>text_content_warning_message</string> </key>
+            <value>
+              <tuple/>
+            </value>
+        </item>
+        <item>
+            <key> <string>version</string> </key>
+            <value> <string>erp5</string> </value>
+        </item>
+        <item>
+            <key> <string>workflow_history</string> </key>
+            <value>
+              <persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="2" aka="AAAAAAAAAAI=">
+    <pickle>
+      <global name="PersistentMapping" module="Persistence.mapping"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>data</string> </key>
+            <value>
+              <dictionary/>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="3" aka="AAAAAAAAAAM=">
+    <pickle>
+      <global name="PersistentMapping" module="Persistence.mapping"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>data</string> </key>
+            <value>
+              <dictionary>
+                <item>
+                    <key> <string>component_validation_workflow</string> </key>
+                    <value>
+                      <persistent> <string encoding="base64">AAAAAAAAAAQ=</string> </persistent>
+                    </value>
+                </item>
+              </dictionary>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="4" aka="AAAAAAAAAAQ=">
+    <pickle>
+      <global name="WorkflowHistoryList" module="Products.ERP5Type.patches.WorkflowTool"/>
+    </pickle>
+    <pickle>
+      <tuple>
+        <none/>
+        <list>
+          <dictionary>
+            <item>
+                <key> <string>action</string> </key>
+                <value> <string>validate</string> </value>
+            </item>
+            <item>
+                <key> <string>validation_state</string> </key>
+                <value> <string>validated</string> </value>
+            </item>
+          </dictionary>
+        </list>
+      </tuple>
+    </pickle>
+  </record>
+</ZopeData>

software/wendelin-scalability/test_scalability_fluentd/PathTemplateItem/portal_skins/custom/ERP5Site_dummyZBigArrayProcessing.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="ExternalMethod" module="Products.ExternalMethod.ExternalMethod"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>_function</string> </key>
+            <value> <string>dummyZBigArrayProcessing</string> </value>
+        </item>
+        <item>
+            <key> <string>_module</string> </key>
+            <value> <string>ScalabilityZBigArray</string> </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>ERP5Site_dummyZBigArrayProcessing</string> </value>
+        </item>
+        <item>
+            <key> <string>title</string> </key>
+            <value> <string></string> </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+</ZopeData>

software/wendelin-scalability/test_scalability_fluentd/PathTemplateItem/portal_skins/custom/DataStreamModule_getTotalSize.xml → software/wendelin-scalability/test_scalability_fluentd/PathTemplateItem/portal_skins/custom/Module_getTotalSize.xml

@@ -54,7 +54,7 @@
         </item>
         <item>
             <key> <string>id</string> </key>
-            <value> <string>DataStreamModule_getTotalSize</string> </value>
+            <value> <string>Module_getTotalSize</string> </value>
         </item>
       </dictionary>
     </pickle>

software/wendelin-scalability/test_scalability_fluentd/bt/template_extension_id_list

 extension.erp5.ScalabilityFluentd
+extension.erp5.ScalabilityZBigArray
\ No newline at end of file

software/wendelin-scalability/test_scalability_fluentd/bt/template_path_list

 portal_ingestion_policies/scalability_test_*
-portal_skins/custom/DataStreamModule_getTotalSize
+portal_skins/custom/ERP5Site_dummyZBigArrayProcessing
 portal_skins/custom/ERP5Site_handleRawDataFluentdIngestion
 portal_skins/custom/ERP5Site_simulateFluentdIngestion
+portal_skins/custom/Module_getTotalSize
\ No newline at end of file