diff --git a/slapos/recipe/erp5testnode/__init__.py b/slapos/recipe/erp5testnode/__init__.py index e9b6ca32491c1ff48fc1a9d4cf72992093fce6db..3ee6bb85821b0b1d5b1de83e9b376f62b081e9e6 100644 --- a/slapos/recipe/erp5testnode/__init__.py +++ b/slapos/recipe/erp5testnode/__init__.py @@ -83,36 +83,24 @@ class Recipe(BaseSlapRecipe): git_binary=self.options['git_binary'], software_root=CONFIG['software_root'], working_directory=CONFIG['working_directory'], - vcs_repository=self.parameter_dict.get('vcs_repository'), + vcs_repository_list=eval(self.parameter_dict.get('vcs_repository_list'),), node_quantity=self.parameter_dict.get('node_quantity', '1'), - branch=self.parameter_dict.get('branch', None), test_suite_master_url=self.parameter_dict.get( 'test_suite_master_url', None), test_suite_name=self.parameter_dict.get('test_suite_name'), test_suite_title=self.parameter_dict.get('test_suite_title'), + project_title=self.parameter_dict.get('project_title'), bin_directory=self.bin_directory, - foo='bar', # botenvironemnt is splittable string of key=value to substitute # environment of running bot bot_environment=self.parameter_dict.get('bot_environment', ''), partition_reference=CONFIG['partition_reference'], environment=dict(PATH=os.environ['PATH']), + vcs_authentication_list=self.parameter_dict.get( + 'vcs_authentication_list') ) ])) - def installLocalSvn(self): - svn_dict = dict(svn_binary = self.options['svn_binary']) - svn_dict.update(self.parameter_dict) - self._writeExecutable(os.path.join(self.bin_directory, 'svn'), """\ -#!/bin/sh -%(svn_binary)s --username %(svn_username)s --password %(svn_password)s \ ---non-interactive --trust-server-cert --no-auth-cache "$@" """% svn_dict) - - svnversion = os.path.join(self.bin_directory, 'svnversion') - if os.path.lexists(svnversion): - os.unlink(svnversion) - os.symlink(self.options['svnversion_binary'], svnversion) - def installLocalGit(self): git_dict = dict(git_binary = self.options['git_binary']) git_dict.update(self.parameter_dict) @@ -122,19 +110,18 @@ class Recipe(BaseSlapRecipe): home_directory = os.path.join(*os.path.split(self.bin_directory)[0:-1]) print "home_directory : %r" % home_directory git_dict.setdefault("git_server_name", "git.erp5.org") - if git_dict.get('vcs_username', None) is not None: + if git_dict.get('vcs_authentication_list', None) is not None: netrc_file = open(os.path.join(home_directory, '.netrc'), 'w') - netrc_file.write(""" - machine %(git_server_name)s - login %(vcs_username)s - password %(vcs_password)s""" % git_dict) + for vcs_authentication_dict in git_dict['vcs_authentication_list']: + netrc_file.write(""" +machine %(host)s +login %(user_name)s +password %(password)s +""" % vcs_authentication_dict) netrc_file.close() def installLocalRepository(self): - if self.parameter_dict.get('vcs_repository').endswith('git'): - self.installLocalGit() - else: - self.installLocalSvn() + self.installLocalGit() def installLocalZip(self): zip = os.path.join(self.bin_directory, 'zip') diff --git a/slapos/recipe/erp5testnode/testnode.py b/slapos/recipe/erp5testnode/testnode.py index 932c683e2fb2c73c055c7367de28c11d751a03eb..3bcdf8491bb855e0eb3460da946c9ae9a71c0870 100644 --- a/slapos/recipe/erp5testnode/testnode.py +++ b/slapos/recipe/erp5testnode/testnode.py @@ -60,39 +60,48 @@ slapos_controler = None def run(args): config = args[0] slapgrid = None - branch = config.get('branch', None) supervisord_pid_file = os.path.join(config['instance_root'], 'var', 'run', 'supervisord.pid') subprocess.check_call([config['git_binary'], "config", "--global", "http.sslVerify", "false"]) previous_revision = None - run_software = True - # find what will be the path of the repository - repository_name = config['vcs_repository'].split('/')[-1].split('.')[0] - repository_path = os.path.join(config['working_directory'],repository_name) - config['repository_path'] = repository_path - sys.path.append(repository_path) + run_software = True # Write our own software.cfg to use the local repository custom_profile_path = os.path.join(config['working_directory'], 'software.cfg') config['custom_profile_path'] = custom_profile_path - - # create a profile in order to use the repository we already have - custom_profile = open(custom_profile_path, 'w') - profile_content = """ + vcs_repository_list = config['vcs_repository_list'] + profile_content = None + assert len(vcs_repository_list), "we must have at least one repository" + for vcs_repository in vcs_repository_list: + url = vcs_repository['url'] + buildout_section_id = vcs_repository.get('buildout_section_id', None) + repository_id = buildout_section_id or \ + url.split('/')[-1].split('.')[0] + repository_path = os.path.join(config['working_directory'],repository_id) + vcs_repository['repository_id'] = repository_id + vcs_repository['repository_path'] = repository_path + if profile_content is None: + profile_content = """ [buildout] extends = %(software_config_path)s - -[%(repository_name)s] +""" % {'software_config_path': os.path.join(repository_path, + config['profile_path'])} + if not(buildout_section_id is None): + profile_content += """ +[%(buildout_section_id)s] repository = %(repository_path)s -""" % {'software_config_path': os.path.join(repository_path, - config['profile_path']), - 'repository_name': repository_name, - 'repository_path' : repository_path} - if branch is not None: - profile_content += "\nbranch = %s" % branch +branch = %(branch)s +""" % {'buildout_section_id': buildout_section_id, + 'repository_path' : repository_path, + 'branch' : vcs_repository.get('branch','')} + + custom_profile = open(custom_profile_path, 'w') custom_profile.write(profile_content) custom_profile.close() + config['repository_path'] = repository_path + sys.path.append(repository_path) + retry_software = False try: while True: @@ -104,19 +113,24 @@ repository = %(repository_path)s except: pass process_group_pid_set.clear() + full_revision_list = [] # Make sure we have local repository - if not os.path.exists(repository_path): - parameter_list = [config['git_binary'], 'clone', - config['vcs_repository']] - if branch is not None: - parameter_list.extend(['-b',branch]) - parameter_list.append(repository_path) - subprocess.check_call(parameter_list) - # XXX this looks like to not wait the end of the command - # Make sure we have local repository - updater = Updater(repository_path, git_binary=config['git_binary']) - updater.checkout() - revision = updater.getRevision() + for vcs_repository in vcs_repository_list: + repository_path = vcs_repository['repository_path'] + repository_id = vcs_repository['repository_id'] + if not os.path.exists(repository_path): + parameter_list = [config['git_binary'], 'clone', + vcs_repository['url']] + if vcs_repository.get('branch') is not None: + parameter_list.extend(['-b',vcs_repository.get('branch')]) + parameter_list.append(repository_path) + subprocess.check_call(parameter_list) + # Make sure we have local repository + updater = Updater(repository_path, git_binary=config['git_binary']) + updater.checkout() + revision = "-".join(updater.getRevision()) + full_revision_list.append('%s=%s' % (repository_id, revision)) + revision = ','.join(full_revision_list) if previous_revision == revision: time.sleep(120) if not(retry_software): @@ -138,16 +152,19 @@ repository = %(repository_path)s assert master.getProtocolRevision() == 1 test_result = safeRpcCall(master.createTestResult, config['test_suite_name'], revision, [], - False, config['test_suite_title']) + False, config['test_suite_title'], config['project_title']) print "testnode, test_result : %r" % (test_result,) if test_result: test_result_path, test_revision = test_result if revision != test_revision: - # other testnodes on other boxes are already ready to test another - # revision - updater = Updater(repository_path, git_binary=config['git_binary'], - revision=test_revision) - updater.checkout() + for i, repository_revision in enumerate(test_revision.split(',')): + vcs_repository = vcs_repository_list[i] + repository_path = vcs_repository['repository_path'] + # other testnodes on other boxes are already ready to test another + # revision + updater = Updater(repository_path, git_binary=config['git_binary'], + revision=repository_revision.split('-')[1]) + updater.checkout() # Now prepare the installation of SlapOS slapos_controler = SlapOSControler(config, @@ -169,24 +186,21 @@ repository = %(repository_path)s # update repositories downloaded by buildout. Later we should get # from master a list of repositories - repository_path_list = glob(os.path.join(config['software_root'], - '*', 'parts', 'git_repository', '*')) - assert len(repository_path_list) >= 0 - for repository_path in repository_path_list: - updater = Updater(repository_path, git_binary=config['git_binary']) - updater.checkout() - if os.path.split(repository_path)[-1] == repository_name: - # redo checkout with good revision, the previous one is used - # to pull last code - updater = Updater(repository_path, git_binary=config['git_binary'], - revision=revision) - updater.checkout() - # calling dist/externals is only there for backward compatibility, - # the code will be removed soon - if os.path.exists(os.path.join(repository_path, 'dist/externals.py')): - process = subprocess.Popen(['dist/externals.py'], - cwd=repository_path) - process.wait() + + # The section below seems useless since we override buildout + # configuration to use local checkouts + #repository_path_list = glob(os.path.join(config['software_root'], + #'*', 'parts', 'git_repository', '*')) + #assert len(repository_path_list) >= 0 + #for repository_path in repository_path_list: + #updater = Updater(repository_path, git_binary=config['git_binary']) + #updater.checkout() + #if os.path.split(repository_path)[-1] == repository_name: + ## redo checkout with good revision, the previous one is used + ## to pull last code + #updater = Updater(repository_path, git_binary=config['git_binary'], + #revision=revision) + #updater.checkout() partition_path = os.path.join(config['instance_root'], config['partition_reference'])