Commit 316f23c0 authored by Jérome Perrin's avatar Jérome Perrin

testnode: SlapOS shared parts support

Shared parts speed up compilation time and is becoming the standard in
SlapOS software installations, so it makes sense to use it in our test
nodes, as it also gives one more opportunity to test this feature.

erp5testnode configuration file supports a new shared_part_list option,
that can be set to a \n separated list of paths to use for shared parts,
following the same rules as slapos.core and slapos.recipe.cmmi (ie. the
first ones are read-only and the last one is read-write).

This shared_part_list option will be set in slapos.cfg used to compile
both the "software for testnode" (ie. selenium-runner) and later the
softwares under tests.
The software under tests will also use a local directory for each test
suite to install shared suite.

The directory structure is now:

  srv/
    shared/
      (shared parts to install selenium runner)
    slapos/
      soft/
        (selenium-runner software)
    testnode/
      foo/ # test suite with reference foo
        inst/
          (partitions of tested software)
        shared/
          (shared parts to install tested software)
        soft/
          (tested software)

and in the configuration srv/shared will be set as initial
shared_part_list.

When installing selenium-runner, srv/shared/ is used to write shared
parts. These shared parts are never removed.

When installing software under test, srv/shared/ and
srv/testnode/foo/shared/ are used. If parts are found in srv/shared they
are used, if they are not found, they are installed in
srv/testnode/foo/shared/.

In practice, this should mean that the shared parts installed by
selenium-runner will be reused for all tested softwares and this should
speed up initial installation of these softwares.

Currently, nothing is implemented regarding removal of unused shared
parts, but in our case:
 - srv/testnode/foo/shared/ will be removed when "foo" is removed.
 - srv/shared/ should be used only when installing selenium-runner.

If this starts to use too much disk space, one quick and dirty
workaround could be to destroy the test node instance and re-create it.
parent 2bc8f31b
...@@ -25,6 +25,7 @@ import tempfile ...@@ -25,6 +25,7 @@ import tempfile
import json import json
import time import time
import re import re
from six.moves.configparser import ConfigParser
try: try:
from unittest import mock from unittest import mock
except ImportError: except ImportError:
...@@ -100,6 +101,7 @@ class ERP5TestNode(TestCase): ...@@ -100,6 +101,7 @@ class ERP5TestNode(TestCase):
config["ipv6_address"] = "::1" config["ipv6_address"] = "::1"
config["slapos_binary"] = "/opt/slapgrid/HASH/bin/slapos" config["slapos_binary"] = "/opt/slapgrid/HASH/bin/slapos"
config["srv_directory"] = "srv_directory" config["srv_directory"] = "srv_directory"
config["shared_part_list"] = "/not/exists\n /not/exists_either"
testnode = TestNode(config) testnode = TestNode(config)
# By default, keep suite logs to stdout for easier debugging # By default, keep suite logs to stdout for easier debugging
...@@ -621,47 +623,72 @@ shared = true ...@@ -621,47 +623,72 @@ shared = true
test_node_slapos = SlapOSInstance(self.slapos_directory) test_node_slapos = SlapOSInstance(self.slapos_directory)
runner = test_type_registry[my_test_type](test_node) runner = test_type_registry[my_test_type](test_node)
node_test_suite = test_node.getNodeTestSuite('foo') node_test_suite = test_node.getNodeTestSuite('foo')
status_dict = {"status_code" : 0}
global call_list with mock.patch(
call_list = [] 'erp5.util.testnode.SlapOSControler.SlapOSControler.runSoftwareRelease',
class Patch: return_value={"status_code": 0}
def __init__(self, method_name, status_code=0): ) as runSoftwareRelease,\
self.method_name = method_name mock.patch(
self.status_code = status_code 'erp5.util.testnode.SlapOSControler.SlapOSControler.runComputerPartition',
def __call__(self, *args, **kw): return_value={"status_code": 0}
global call_list ) as runComputerPartition,\
call_list.append({"method_name": self.method_name, mock.patch('erp5.util.testnode.SlapOSControler.slapos.slap'),\
"args": [x for x in args], mock.patch('subprocess.Popen'):
"kw": kw})
return {"status_code": self.status_code}
original_SlapOSControler_initializeSlapOSControler = SlapOSControler.initializeSlapOSControler
original_SlapOSControler_runSoftwareRelease = SlapOSControler.runSoftwareRelease
original_SlapOSControler_runComputerPartition = SlapOSControler.runComputerPartition
try:
SlapOSControler.initializeSlapOSControler = Patch("initializeSlapOSControler")
SlapOSControler.runSoftwareRelease = Patch("runSoftwareRelease")
SlapOSControler.runComputerPartition = Patch("runComputerPartition")
method_list_for_prepareSlapOSForTestNode = ["initializeSlapOSControler",
"runSoftwareRelease"]
method_list_for_prepareSlapOSForTestSuite = ["initializeSlapOSControler",
"runSoftwareRelease", "runComputerPartition"]
runner.prepareSlapOSForTestNode(test_node_slapos) runner.prepareSlapOSForTestNode(test_node_slapos)
self.assertEqual(method_list_for_prepareSlapOSForTestNode, self.assertEqual(1, runSoftwareRelease.call_count)
[x["method_name"] for x in call_list]) self.assertEqual(0, runComputerPartition.call_count)
call_list = []
with mock.patch(
'erp5.util.testnode.SlapOSControler.SlapOSControler.runSoftwareRelease',
return_value={"status_code": 0}
) as runSoftwareRelease,\
mock.patch(
'erp5.util.testnode.SlapOSControler.SlapOSControler.runComputerPartition',
return_value={"status_code": 0}
) as runComputerPartition,\
mock.patch('erp5.util.testnode.SlapOSControler.slapos.slap'),\
mock.patch('subprocess.Popen'):
runner.prepareSlapOSForTestSuite(node_test_suite) runner.prepareSlapOSForTestSuite(node_test_suite)
self.assertEqual(method_list_for_prepareSlapOSForTestSuite, self.assertEqual(1, runSoftwareRelease.call_count)
[x["method_name"] for x in call_list]) self.assertEqual(1, runComputerPartition.call_count)
call_list = []
SlapOSControler.runSoftwareRelease = Patch("runSoftwareRelease", status_code=1) # test node slapos slapos uses the shared parts defined in config
# TODO : write a test for scalability case cfg_parser = ConfigParser()
self.assertRaises(SubprocessError, runner.prepareSlapOSForTestSuite, with open(os.path.join(test_node_slapos.working_directory, 'slapos.cfg')) as f:
node_test_suite) cfg_parser.readfp(f)
finally: self.assertEqual(
SlapOSControler.initializeSlapOSControler = original_SlapOSControler_initializeSlapOSControler '/not/exists\n/not/exists_either',
SlapOSControler.runSoftwareRelease = original_SlapOSControler_runSoftwareRelease cfg_parser.get('slapos', 'shared_part_list'))
SlapOSControler.runComputerPartition = original_SlapOSControler_runComputerPartition
# test suite slapos uses the shared parts from the config, plus
# a "local" folder for used as shared when installing tested
# softwares.
cfg_parser = ConfigParser()
with open(os.path.join(node_test_suite.working_directory, 'slapos.cfg')) as f:
cfg_parser.readfp(f)
self.assertEqual(
'/not/exists\n/not/exists_either\n%s/shared' % node_test_suite.working_directory,
cfg_parser.get('slapos', 'shared_part_list'))
# If running software has status_code 1 we have an error
with mock.patch(
'erp5.util.testnode.SlapOSControler.SlapOSControler.runSoftwareRelease',
return_value={"status_code": 1}
) as runSoftwareRelease,\
mock.patch(
'erp5.util.testnode.SlapOSControler.SlapOSControler.runComputerPartition',
return_value={"status_code": 0}
) as runComputerPartition,\
mock.patch('erp5.util.testnode.SlapOSControler.slapos.slap'),\
mock.patch('subprocess.Popen'):
self.assertRaises(
SubprocessError,
runner.prepareSlapOSForTestSuite,
node_test_suite)
def test_11_run(self, my_test_type='UnitTest', grade='master'): def test_11_run(self, my_test_type='UnitTest', grade='master'):
def doNothing(self, *args, **kw): def doNothing(self, *args, **kw):
......
...@@ -42,9 +42,17 @@ MAX_SR_RETRIES = 3 ...@@ -42,9 +42,17 @@ MAX_SR_RETRIES = 3
class SlapOSControler(object): class SlapOSControler(object):
def __init__(self, working_directory, config): def __init__(self, working_directory, config, use_local_shared_part=False):
self.config = config self.config = config
self.software_root = os.path.join(working_directory, 'soft') self.software_root = os.path.join(working_directory, 'soft')
self.shared_part_list = [
path.strip() for path in config['shared_part_list'].splitlines()
]
if use_local_shared_part:
shared = os.path.join(working_directory, 'shared')
createFolder(shared)
self.shared_part_list = self.shared_part_list + [shared]
self.instance_root = os.path.join(working_directory, 'inst') self.instance_root = os.path.join(working_directory, 'inst')
self.slapos_config = os.path.join(working_directory, 'slapos.cfg') self.slapos_config = os.path.join(working_directory, 'slapos.cfg')
self.proxy_database = os.path.join(working_directory, 'proxy.db') self.proxy_database = os.path.join(working_directory, 'proxy.db')
...@@ -217,7 +225,9 @@ class SlapOSControler(object): ...@@ -217,7 +225,9 @@ class SlapOSControler(object):
slapos_config_dict = config.copy() slapos_config_dict = config.copy()
slapos_config_dict.update(software_root=self.software_root, slapos_config_dict.update(software_root=self.software_root,
instance_root=self.instance_root, instance_root=self.instance_root,
proxy_database=self.proxy_database) proxy_database=self.proxy_database,
shared_part_list='\n '.join(self.shared_part_list))
with open(self.slapos_config, 'w') as f: with open(self.slapos_config, 'w') as f:
f.write(pkg_resources.resource_string( f.write(pkg_resources.resource_string(
'erp5.util.testnode', 'template/slapos.cfg.in').decode() % 'erp5.util.testnode', 'template/slapos.cfg.in').decode() %
......
...@@ -44,16 +44,17 @@ class UnitTestRunner(object): ...@@ -44,16 +44,17 @@ class UnitTestRunner(object):
def __init__(self, testnode): def __init__(self, testnode):
self.testnode = testnode self.testnode = testnode
def _getSlapOSControler(self, working_directory): def _getSlapOSControler(self, working_directory, use_local_shared_part):
""" """
Create a SlapOSControler Create a SlapOSControler
""" """
return SlapOSControler( return SlapOSControler(
working_directory, working_directory,
self.testnode.config) self.testnode.config,
use_local_shared_part=use_local_shared_part)
def _prepareSlapOS(self, working_directory, slapos_instance, def _prepareSlapOS(self, working_directory, slapos_instance,
create_partition=1, software_path_list=None,**kw): create_partition=1, software_path_list=None, use_local_shared_part=False, **kw):
""" """
Launch slapos to build software and partitions Launch slapos to build software and partitions
""" """
...@@ -67,8 +68,10 @@ class UnitTestRunner(object): ...@@ -67,8 +68,10 @@ class UnitTestRunner(object):
slapos_instance.retry_software_count) slapos_instance.retry_software_count)
# XXX Create a new controler because working_directory can be # XXX Create a new controler because working_directory can be
# Diferent depending of the preparation # Different depending of the preparation
slapos_controler = self._getSlapOSControler(working_directory) slapos_controler = self._getSlapOSControler(
working_directory,
use_local_shared_part)
slapos_controler.initializeSlapOSControler(slapproxy_log=slapproxy_log, slapos_controler.initializeSlapOSControler(slapproxy_log=slapproxy_log,
process_manager=self.testnode.process_manager, reset_software=reset_software, process_manager=self.testnode.process_manager, reset_software=reset_software,
...@@ -113,16 +116,17 @@ class UnitTestRunner(object): ...@@ -113,16 +116,17 @@ class UnitTestRunner(object):
def prepareSlapOSForTestSuite(self, node_test_suite): def prepareSlapOSForTestSuite(self, node_test_suite):
""" """
Build softwares needed by testsuites Build softwares needed by testsuites.
""" """
return self._prepareSlapOS(node_test_suite.working_directory, return self._prepareSlapOS(node_test_suite.working_directory,
node_test_suite, node_test_suite,
software_path_list=[node_test_suite.custom_profile_path], software_path_list=[node_test_suite.custom_profile_path],
cluster_configuration={'_': json.dumps(node_test_suite.cluster_configuration)}) cluster_configuration={'_': json.dumps(node_test_suite.cluster_configuration)},
use_local_shared_part=True)
def getInstanceRoot(self, node_test_suite): def getInstanceRoot(self, node_test_suite):
return self._getSlapOSControler( return self._getSlapOSControler(
node_test_suite.working_directory).instance_root node_test_suite.working_directory, True).instance_root
def runTestSuite(self, node_test_suite, portal_url): def runTestSuite(self, node_test_suite, portal_url):
config = self.testnode.config config = self.testnode.config
......
...@@ -74,7 +74,7 @@ def main(*args): ...@@ -74,7 +74,7 @@ def main(*args):
'proxy_port', 'git_binary','zip_binary','node_quantity', 'proxy_port', 'git_binary','zip_binary','node_quantity',
'test_node_title', 'ipv4_address','ipv6_address','test_suite_master_url', 'test_node_title', 'ipv4_address','ipv6_address','test_suite_master_url',
'slapos_binary', 'httpd_ip', 'httpd_port', 'httpd_software_access_port', 'slapos_binary', 'httpd_ip', 'httpd_port', 'httpd_software_access_port',
'computer_id', 'server_url'): 'computer_id', 'server_url', 'shared_part_list'):
CONFIG[key] = config.get('testnode',key) CONFIG[key] = config.get('testnode',key)
for key in ('slapos_directory', 'working_directory', 'test_suite_directory', for key in ('slapos_directory', 'working_directory', 'test_suite_directory',
......
[slapos] [slapos]
software_root = %(software_root)s software_root = %(software_root)s
instance_root = %(instance_root)s instance_root = %(instance_root)s
shared_part_list = %(shared_part_list)s
master_url = %(master_url)s master_url = %(master_url)s
computer_id = %(computer_id)s computer_id = %(computer_id)s
root_check = False root_check = False
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment