Commit 1a1a159e authored by Ivan Tyagov's avatar Ivan Tyagov

Do test by transforming only part (tail) of appended data.

parent 501d29e9
...@@ -57,8 +57,6 @@ ...@@ -57,8 +57,6 @@
* transform_script_id - the script which will transform data\n * transform_script_id - the script which will transform data\n
* chunk_length - the length of a chunk\n * chunk_length - the length of a chunk\n
"""\n """\n
data_length = context.getSize()\n
\n
start = 0\n start = 0\n
end = chunk_length\n end = chunk_length\n
context.activate().DataStream_readChunkListAndTransform( \\\n context.activate().DataStream_readChunkListAndTransform( \\\n
...@@ -67,8 +65,6 @@ context.activate().DataStream_readChunkListAndTransform( \\\n ...@@ -67,8 +65,6 @@ context.activate().DataStream_readChunkListAndTransform( \\\n
chunk_length, \\\n chunk_length, \\\n
transform_script_id, \\\n transform_script_id, \\\n
data_array_reference)\n data_array_reference)\n
\n
return data_length\n
</string> </value> </string> </value>
</item> </item>
<item> <item>
......
...@@ -58,7 +58,6 @@ class Test(ERP5TypeTestCase): ...@@ -58,7 +58,6 @@ class Test(ERP5TypeTestCase):
# here, you can create the categories and objects your test will depend on # here, you can create the categories and objects your test will depend on
pass pass
def stepSetupIngestion(self, reference): def stepSetupIngestion(self, reference):
""" """
Generic step. Generic step.
...@@ -168,32 +167,48 @@ class Test(ERP5TypeTestCase): ...@@ -168,32 +167,48 @@ class Test(ERP5TypeTestCase):
reference = getRandomString() reference = getRandomString()
number_string_list = [] number_string_list = []
for my_list in list(chunks(range(0, 100001), 10)): for my_list in list(chunks(range(0, 10001), 10)):
number_string_list.append(','.join([str(x) for x in my_list])) number_string_list.append(','.join([str(x) for x in my_list]))
real_data = '\n'.join(number_string_list) real_data = '\n'.join(number_string_list)
# make sure real_data tail is also a full line # make sure real_data tail is also a full line
real_data += '\n' real_data += '\n'
ingestion_policy, data_supply, data_stream, data_array = self.stepSetupIngestion(reference) ingestion_policy, data_supply, data_stream, data_array = self.stepSetupIngestion(reference)
data_stream.appendData(real_data)
self.tic()
self.assertEqual(None, data_array.getArray())
# override DataStream_transformTail to actually do transformation on appenData # override DataStream_transformTail to actually do transformation on appenData
start = data_stream.getSize()
script_id = 'DataStream_transformTail' script_id = 'DataStream_transformTail'
script_content_list = ['**kw', """ script_content_list = ['', """
# created by testWendelin.test_01_1_IngestionTail # created by testWendelin.test_01_1_IngestionTail
context.DataStream_transform(\ start = %s
chunk_length = 10450, \ end = %s
transform_script_id = 'DataStream_copyCSVToDataArray', context.activate().DataStream_readChunkListAndTransform( \
data_array_reference = context.getReference())"""] start, \
end, \
%s, \
transform_script_id = 'DataStream_copyCSVToDataArray', \
data_array_reference=context.getReference())""" %(start, start + 10450, 10450)]
createZODBPythonScript(portal.portal_skins.custom, script_id, *script_content_list) createZODBPythonScript(portal.portal_skins.custom, script_id, *script_content_list)
# append data to Data Stream and check array. number_string_list = []
for my_list in list(chunks(range(10001, 200001), 10)):
number_string_list.append(','.join([str(x) for x in my_list]))
real_data = '\n'.join(number_string_list)
# make sure real_data tail is also a full line
real_data += '\n'
# append data to Data Stream and check array which should be feed now.
data_stream.appendData(real_data) data_stream.appendData(real_data)
self.tic() self.tic()
# test that extracted array contains same values as input CSV # test that extracted array contains same values as input CSV
zarray = data_array.getArray() zarray = data_array.getArray()
self.assertEqual(np.average(zarray), np.average(np.arange(100001))) self.assertEqual(np.average(zarray), np.average(np.arange(10001, 200001)))
self.assertTrue(np.array_equal(zarray, np.arange(100001))) self.assertTrue(np.array_equal(zarray, np.arange(10001, 200001)))
# clean up script # clean up script
portal.portal_skins.custom.manage_delObjects([script_id,]) portal.portal_skins.custom.manage_delObjects([script_id,])
...@@ -235,14 +250,12 @@ context.DataStream_transform(\ ...@@ -235,14 +250,12 @@ context.DataStream_transform(\
new_array = np.arange(1,17).reshape((4,4)) new_array = np.arange(1,17).reshape((4,4))
persistent_zbig_array[:,:] = new_array persistent_zbig_array[:,:] = new_array
self.assertEquals(new_array.shape, persistent_zbig_array.shape) self.assertEquals(new_array.shape, persistent_zbig_array.shape)
# (enable when new wendelin.core released as it can kill system)
self.assertTrue(np.array_equal(new_array, persistent_zbig_array)) self.assertTrue(np.array_equal(new_array, persistent_zbig_array))
# test set element in zbig array # test set element in zbig array
persistent_zbig_array[:2, 2] = 0 persistent_zbig_array[:2, 2] = 0
self.assertFalse(np.array_equal(new_array, persistent_zbig_array)) self.assertFalse(np.array_equal(new_array, persistent_zbig_array))
# resize Zbig Array (enable when new wendelin.core released as it can kill system) # resize Zbig Array
persistent_zbig_array = np.resize(persistent_zbig_array, (100,100)) persistent_zbig_array = np.resize(persistent_zbig_array, (100,100))
self.assertNotEquals(pure_numpy_array.shape, persistent_zbig_array.shape) self.assertNotEquals(pure_numpy_array.shape, persistent_zbig_array.shape)
\ No newline at end of file
...@@ -47,12 +47,12 @@ ...@@ -47,12 +47,12 @@
<value> <value>
<tuple> <tuple>
<string>W: 39, 8: Unused variable \'n\' (unused-variable)</string> <string>W: 39, 8: Unused variable \'n\' (unused-variable)</string>
<string>W:117, 4: Unused variable \'scipy\' (unused-variable)</string> <string>W:116, 4: Unused variable \'scipy\' (unused-variable)</string>
<string>W:119, 4: Unused variable \'pandas\' (unused-variable)</string> <string>W:118, 4: Unused variable \'pandas\' (unused-variable)</string>
<string>W:118, 4: Unused variable \'sklearn\' (unused-variable)</string> <string>W:117, 4: Unused variable \'sklearn\' (unused-variable)</string>
<string>W:137, 22: Unused variable \'data_supply\' (unused-variable)</string> <string>W:136, 22: Unused variable \'data_supply\' (unused-variable)</string>
<string>W:177, 4: Unused variable \'ingestion_policy\' (unused-variable)</string> <string>W:176, 4: Unused variable \'ingestion_policy\' (unused-variable)</string>
<string>W:177, 22: Unused variable \'data_supply\' (unused-variable)</string> <string>W:176, 22: Unused variable \'data_supply\' (unused-variable)</string>
</tuple> </tuple>
</value> </value>
</item> </item>
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment