Skip to content

E
erp5
Commit 61307a1c authored by Sebastien Robin's avatar Sebastien Robin
Browse files
Options

ERP5ProjectDistributor: allow to configure the quantity of test suite per test node on distributor

parent c9770e5e
Pipeline #2398 skipped
Hide whitespace changes
Inline Side-by-side
Showing with 294 additions and 6 deletions
bt5/erp5_test_result/PortalTypePropertySheetTemplateItem/property_sheet_list.xml
View file @ 61307a1c
...@@ -7,6 +7,9 @@ ...@@ -7,6 +7,9 @@
<portal_type id="Benchmark Result Line"> <portal_type id="Benchmark Result Line">
<item>Task</item> <item>Task</item>
</portal_type> </portal_type>
<portal_type id="ERP5 Project Unit Test Distributor">
<item>UnitTestDistributor</item>
</portal_type>
<portal_type id="ERP5 Scalability Distributor"> <portal_type id="ERP5 Scalability Distributor">
<item>ScalabilityDistributor</item> <item>ScalabilityDistributor</item>
</portal_type> </portal_type>
......
bt5/erp5_test_result/PropertySheetTemplateItem/portal_property_sheets/UnitTestDistributor.xml 0 → 100644
View file @ 61307a1c
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Property Sheet" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_count</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
</value>
</item>
<item>
<key> <string>_mt_index</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
</value>
</item>
<item>
<key> <string>_tree</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAQ=</string> </persistent>
</value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>UnitTestDistributor</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Property Sheet</string> </value>
</item>
</dictionary>
</pickle>
</record>
<record id="2" aka="AAAAAAAAAAI=">
<pickle>
<global name="Length" module="BTrees.Length"/>
</pickle>
<pickle> <int>0</int> </pickle>
</record>
<record id="3" aka="AAAAAAAAAAM=">
<pickle>
<global name="OOBTree" module="BTrees.OOBTree"/>
</pickle>
<pickle>
<none/>
</pickle>
</record>
<record id="4" aka="AAAAAAAAAAQ=">
<pickle>
<global name="OOBTree" module="BTrees.OOBTree"/>
</pickle>
<pickle>
<none/>
</pickle>
</record>
</ZopeData>
bt5/erp5_test_result/PropertySheetTemplateItem/portal_property_sheets/UnitTestDistributor/max_test_suite_property.xml 0 → 100644
View file @ 61307a1c
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Standard Property" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>categories</string> </key>
<value>
<tuple>
<string>elementary_type/int</string>
</tuple>
</value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>max_test_suite_property</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Standard Property</string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_test_result/SkinTemplateItem/portal_skins/erp5_test_result/Base_viewTestResultFieldLibrary.xml
View file @ 61307a1c
...@@ -84,6 +84,7 @@ ...@@ -84,6 +84,7 @@
<value> <value>
<list> <list>
<string>colored_integer</string> <string>colored_integer</string>
<string>my_view_mode_max_test_suite</string>
</list> </list>
</value> </value>
</item> </item>
......
bt5/erp5_test_result/SkinTemplateItem/portal_skins/erp5_test_result/Base_viewTestResultFieldLibrary/my_view_mode_max_test_suite.xml 0 → 100644
View file @ 61307a1c
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="ProxyField" module="Products.ERP5Form.ProxyField"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>delegated_list</string> </key>
<value>
<list>
<string>title</string>
</list>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>my_view_mode_max_test_suite</string> </value>
</item>
<item>
<key> <string>message_values</string> </key>
<value>
<dictionary>
<item>
<key> <string>external_validator_failed</string> </key>
<value> <string>The input failed the external validator.</string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>overrides</string> </key>
<value>
<dictionary>
<item>
<key> <string>field_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>tales</string> </key>
<value>
<dictionary>
<item>
<key> <string>field_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>values</string> </key>
<value>
<dictionary>
<item>
<key> <string>field_id</string> </key>
<value> <string>my_integer_field</string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string>Base_viewFieldLibrary</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string>Max Test Suite</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_test_result/SkinTemplateItem/portal_skins/erp5_test_result/ERP5ProjectUnitTestDistributor_view.xml
View file @ 61307a1c
...@@ -103,6 +103,7 @@ ...@@ -103,6 +103,7 @@
<value> <value>
<list> <list>
<string>my_id</string> <string>my_id</string>
<string>my_max_test_suite</string>
</list> </list>
</value> </value>
</item> </item>
......
bt5/erp5_test_result/SkinTemplateItem/portal_skins/erp5_test_result/ERP5ProjectUnitTestDistributor_view/my_max_test_suite.xml 0 → 100644
View file @ 61307a1c
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="ProxyField" module="Products.ERP5Form.ProxyField"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>id</string> </key>
<value> <string>my_max_test_suite</string> </value>
</item>
<item>
<key> <string>message_values</string> </key>
<value>
<dictionary>
<item>
<key> <string>external_validator_failed</string> </key>
<value> <string>The input failed the external validator.</string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>overrides</string> </key>
<value>
<dictionary>
<item>
<key> <string>field_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>tales</string> </key>
<value>
<dictionary>
<item>
<key> <string>field_id</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</value>
</item>
<item>
<key> <string>values</string> </key>
<value>
<dictionary>
<item>
<key> <string>field_id</string> </key>
<value> <string>my_view_mode_max_test_suite</string> </value>
</item>
<item>
<key> <string>form_id</string> </key>
<value> <string>Base_viewTestResultFieldLibrary</string> </value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
bt5/erp5_test_result/TestTemplateItem/portal_components/test.erp5.testTaskDistribution.py
View file @ 61307a1c
...@@ -14,6 +14,7 @@ class TestTaskDistribution(ERP5TypeTestCase): ...@@ -14,6 +14,7 @@ class TestTaskDistribution(ERP5TypeTestCase):
if getattr(tool, "TestTaskDistribution", None) is None: if getattr(tool, "TestTaskDistribution", None) is None:
tool.newContent(id="TestTaskDistribution", tool.newContent(id="TestTaskDistribution",
portal_type="ERP5 Project Unit Test Distributor") portal_type="ERP5 Project Unit Test Distributor")
tool.TestTaskDistribution.setMaxTestSuite(None)
if getattr(tool, "TestPerformanceTaskDistribution", None) is None: if getattr(tool, "TestPerformanceTaskDistribution", None) is None:
tool.newContent(id="TestPerformanceTaskDistribution", tool.newContent(id="TestPerformanceTaskDistribution",
portal_type="Cloud Performance Unit Test Distributor") portal_type="Cloud Performance Unit Test Distributor")
...@@ -614,6 +615,26 @@ class TestTaskDistribution(ERP5TypeTestCase): ...@@ -614,6 +615,26 @@ class TestTaskDistribution(ERP5TypeTestCase):
[test_node_six, ["six", "seven"]], [test_node_six, ["six", "seven"]],
[test_node_seven, ["four", "six"]]) [test_node_seven, ["four", "six"]])
def test_11b_checkERP5ProjectDistributionWithCustomMaxQuantity(self):
"""
Check that the property max_test_suite on the distributor could
be used to customize the quantity of test suite affected per test node
"""
test_node, = self._createTestNode(quantity=1)
test_suite_list = self._createTestSuite(quantity=5)
self.tool.TestTaskDistribution.setMaxTestSuite(None)
self.tic()
self._callOptimizeAlarm()
self.assertEqual(4, len(set(test_node.getAggregateList())))
self.tool.TestTaskDistribution.setMaxTestSuite(1)
self._callOptimizeAlarm()
self.assertEqual(1, len(set(test_node.getAggregateList())))
self.tool.TestTaskDistribution.setMaxTestSuite(10)
self._callOptimizeAlarm()
self.assertEqual(5, len(set(test_node.getAggregateList())))
self.assertEqual(set(test_node.getAggregateList()),
set([x.getRelativeUrl() for x in test_suite_list]))
def test_12_checkCloudPerformanceOptimizationIsStable(self): def test_12_checkCloudPerformanceOptimizationIsStable(self):
""" """
When we have two test suites and we have two test nodes, we should have When we have two test suites and we have two test nodes, we should have
......
bt5/erp5_test_result/bt/template_portal_type_property_sheet_list
View file @ 61307a1c
...@@ -2,6 +2,7 @@ Benchmark Result Line | Task ...@@ -2,6 +2,7 @@ Benchmark Result Line | Task
Benchmark Result | Comment Benchmark Result | Comment
Benchmark Result | SortIndex Benchmark Result | SortIndex
Benchmark Result | Task Benchmark Result | Task
ERP5 Project Unit Test Distributor | UnitTestDistributor
ERP5 Scalability Distributor | ScalabilityDistributor ERP5 Scalability Distributor | ScalabilityDistributor
Scalability Test Suite | Arrow Scalability Test Suite | Arrow
Scalability Test Suite | ScalabilityTestSuite Scalability Test Suite | ScalabilityTestSuite
......
bt5/erp5_test_result/bt/template_property_sheet_id_list
View file @ 61307a1c
...@@ -6,4 +6,5 @@ TestSuiteConstraint ...@@ -6,4 +6,5 @@ TestSuiteConstraint
ScalabilityTestSuite ScalabilityTestSuite
TestNode TestNode
ScalabilityDistributor ScalabilityDistributor
SlapOSSoftwareReleaseUnitTest SlapOSSoftwareReleaseUnitTest
\ No newline at end of file UnitTestDistributor
\ No newline at end of file
product/ERP5/Document/ERP5ProjectUnitTestDistributor.py
View file @ 61307a1c
...@@ -91,7 +91,7 @@ class ERP5ProjectUnitTestDistributor(XMLObject): ...@@ -91,7 +91,7 @@ class ERP5ProjectUnitTestDistributor(XMLObject):
else: else:
break break
def _checkCurrentConfiguration(self,test_node_list, test_suite_list_to_add): def _checkCurrentConfiguration(self,test_node_list, test_suite_list_to_add, max_test_suite):
""" """
We look at what is already installed and then we remove from the list We look at what is already installed and then we remove from the list
of test suite list to add what is already installed. of test suite list to add what is already installed.
...@@ -100,7 +100,10 @@ class ERP5ProjectUnitTestDistributor(XMLObject): ...@@ -100,7 +100,10 @@ class ERP5ProjectUnitTestDistributor(XMLObject):
test_suite_list_to_remove = [] test_suite_list_to_remove = []
for test_node in test_node_list: for test_node in test_node_list:
test_suite_list = test_node.getAggregateList() test_suite_list = test_node.getAggregateList()
for test_suite in test_suite_list: for index, test_suite in enumerate(test_suite_list, 1):
if index > max_test_suite:
test_suite_list_to_remove.append(test_suite)
continue
try: try:
test_suite_list_to_add.remove(test_suite) test_suite_list_to_add.remove(test_suite)
except ValueError: except ValueError:
...@@ -123,9 +126,10 @@ class ERP5ProjectUnitTestDistributor(XMLObject): ...@@ -123,9 +126,10 @@ class ERP5ProjectUnitTestDistributor(XMLObject):
specialise_uid=self.getUid(), sort_on=[('title','ascending')])] specialise_uid=self.getUid(), sort_on=[('title','ascending')])]
test_node_list_len = len(test_node_list) test_node_list_len = len(test_node_list)
max_test_suite = self.getMaxTestSuite(TEST_SUITE_MAX)
def _optimizeConfiguration(test_suite_list_to_add, level=0, def _optimizeConfiguration(test_suite_list_to_add, level=0,
test_node_list_to_optimize=None, test_node_list_to_optimize=None,
test_suite_max=TEST_SUITE_MAX): test_suite_max=max_test_suite):
if test_node_list_to_optimize is None: if test_node_list_to_optimize is None:
test_node_list_to_optimize = [x for x in test_node_list] test_node_list_to_optimize = [x for x in test_node_list]
if test_suite_list_to_add: if test_suite_list_to_add:
...@@ -154,7 +158,7 @@ class ERP5ProjectUnitTestDistributor(XMLObject): ...@@ -154,7 +158,7 @@ class ERP5ProjectUnitTestDistributor(XMLObject):
test_suite_score, test_suite_list_to_add = self._getSortedNodeTestSuiteList() test_suite_score, test_suite_list_to_add = self._getSortedNodeTestSuiteList()
average_quantity = float(len(test_suite_list_to_add)) / (test_node_list_len or 1) average_quantity = float(len(test_suite_list_to_add)) / (test_node_list_len or 1)
test_suite_list_to_remove = self._checkCurrentConfiguration(test_node_list, test_suite_list_to_remove = self._checkCurrentConfiguration(test_node_list,
test_suite_list_to_add) test_suite_list_to_add, max_test_suite)
self._cleanupTestNodeList(test_node_list, test_suite_list_to_remove) self._cleanupTestNodeList(test_node_list, test_suite_list_to_remove)
_optimizeConfiguration(test_suite_list_to_add) _optimizeConfiguration(test_suite_list_to_add)
# once we removed useless test suite and added needed ones, # once we removed useless test suite and added needed ones,
...@@ -172,7 +176,7 @@ class ERP5ProjectUnitTestDistributor(XMLObject): ...@@ -172,7 +176,7 @@ class ERP5ProjectUnitTestDistributor(XMLObject):
lazy_test_node_list.append(test_node) lazy_test_node_list.append(test_node)
# check on most overloaded test nodes first if we can move some work to lazy # check on most overloaded test nodes first if we can move some work to lazy
# test nodes # test nodes
for aggregate_quantity in range(TEST_SUITE_MAX, int_average_quantity, -1): for aggregate_quantity in range(max_test_suite, int_average_quantity, -1):
if len(lazy_test_node_list) == 0: if len(lazy_test_node_list) == 0:
break break
overloaded_test_node_list = [x for x in test_node_list if len(x.getAggregateList()) == aggregate_quantity] overloaded_test_node_list = [x for x in test_node_list if len(x.getAggregateList()) == aggregate_quantity]
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7