From 233a6c84855e9a27c104feac5ebe82cb41e66c24 Mon Sep 17 00:00:00 2001 From: Alain Takoudjou <talino@tiolive.com> Date: Thu, 6 Sep 2012 13:15:24 +0200 Subject: [PATCH] Update boinc recipe and add boinc-app recipe --- setup.py | 1 + slapos/recipe/boinc/__init__.py | 85 ++++++++++++- slapos/recipe/boinc/configure.py | 136 ++++++++++++++++++++- slapos/recipe/boinc/template/sed_update.in | 14 +++ 4 files changed, 232 insertions(+), 4 deletions(-) create mode 100644 slapos/recipe/boinc/template/sed_update.in diff --git a/setup.py b/setup.py index 6ba984ee7..ebf85c9fc 100755 --- a/setup.py +++ b/setup.py @@ -139,6 +139,7 @@ setup(name=name, 'signalwrapper= slapos.recipe.signal_wrapper:Recipe', 'condor = slapos.recipe.condor:Recipe', 'boinc = slapos.recipe.boinc:Recipe', + 'boinc.app = slapos.recipe.boinc:App', ], 'slapos.recipe.nosqltestbed.plugin': [ 'kumo = slapos.recipe.nosqltestbed.kumo:KumoTestBed', diff --git a/slapos/recipe/boinc/__init__.py b/slapos/recipe/boinc/__init__.py index a7bb3c80d..5d8014591 100644 --- a/slapos/recipe/boinc/__init__.py +++ b/slapos/recipe/boinc/__init__.py @@ -51,6 +51,8 @@ class Recipe(GenericBaseRecipe): self.sourcedir = options['source'].strip() self.home = options['home'].strip() self.project = options['project'].strip() + self.fullname = options['fullname'].strip() + self.copyright = options['copyright'].strip() self.project_config = options['project-config'].strip() self.installroot = options['installroot'].strip() self.boinc_egg = os.path.join(self.package, 'lib/python2.7/site-packages') @@ -182,7 +184,9 @@ class Recipe(GenericBaseRecipe): xadd=os.path.join(self.installroot, 'bin/xadd'), environment=environment, service_status=service_status, - project=niceprojectname + project=niceprojectname, + fullname=self.fullname, + copyright=self.copyright ) start_service = self.createPythonScript( os.path.join(self.wrapperdir, 'config_project'), @@ -190,7 +194,7 @@ class Recipe(GenericBaseRecipe): ) path_list.append(start_service) - #Generate Boinc project wrapper + #Generate Boinc start project wrapper start_args = [os.path.join(self.installroot, 'bin/start')] start_wrapper = self.createPythonScript(os.path.join(self.wrapperdir, 'start_project'), @@ -201,4 +205,79 @@ class Recipe(GenericBaseRecipe): return path_list - update=install + update = install + + +class App(GenericBaseRecipe): + """This recipe allow to deploy an scientific applications using boinc + Note that recipe use depend on boinc-server parameter""" + + + def install(self): + + path_list = [] + package = self.options['boinc'].strip() + #Define environment variable here + boinc_egg = os.path.join(package, 'lib/python2.7/site-packages') + developegg = self.options['develop-egg'].strip() + python_path = boinc_egg + ":" + os.environ['PYTHONPATH'] + home = self.options['home'].strip() + perl = self.options['perl-binary'].strip() + svn = self.options['svn-binary'].strip() + pythonbin = self.options['python-binary'].strip() + for f in os.listdir(developegg): + dir = os.path.join(developegg, f) + if os.path.isdir(dir): + python_path += ":" + dir + bin_dir = os.path.join(home, 'bin') + environment = dict( + PATH=svn + ':' + bin_dir + ':' + perl + ':' + os.environ['PATH'], + PYTHONPATH=python_path, + PYTHON=pythonbin + ) + + #generate project.xml and config.xml script updater + bash = os.path.join(home, 'bin', 'update_config.sh') + sh_script = self.createFile(bash, + self.substituteTemplate(self.getTemplateFilename('sed_update.in'), + dict(dash=self.options['dash'].strip())) + ) + path_list.append(sh_script) + os.chmod(bash , 0700) + + service_status = os.path.join(home, '.start_service') + installroot = self.options['installroot'].strip() + version = self.options['version'].strip() + platform = self.options['platform'].strip() + apps_dir = os.path.join(installroot, 'apps') + appname = self.options['app-name'].strip() + bin_name = appname +"_"+ version +"_"+ \ + platform + self.options['extension'].strip() + application = os.path.join(apps_dir, appname, version, platform) + wrapperdir = self.options['wrapper-dir'].strip() + project = self.options['project'].strip() + + parameter = dict(installroot=installroot, project=project, + appname=appname, binary_name=bin_name, + version=version, platform=platform, + application=application, environment=environment, + service_status=service_status, + wu_name=self.options['wu-name'].strip(), + wu_number=self.options['wu-number'].strip(), + t_result=self.options['template-result'].strip(), + t_wu=self.options['template-wu'].strip(), + t_input=self.options['input-file'].strip(), + binary=self.options['binary'].strip(), + bash=bash, + ) + deploy_app = self.createPythonScript( + os.path.join(wrapperdir, appname), + '%s.configure.deployApp' % __name__, parameter + ) + path_list.append(deploy_app) + + return path_list + + update = install + + \ No newline at end of file diff --git a/slapos/recipe/boinc/configure.py b/slapos/recipe/boinc/configure.py index a70854836..7767b81bc 100644 --- a/slapos/recipe/boinc/configure.py +++ b/slapos/recipe/boinc/configure.py @@ -29,6 +29,7 @@ import os import sys import subprocess import time +import shutil def checkMysql(args): sys.path += args['python_path'].split(':') @@ -91,6 +92,8 @@ def services(args): languages = os.path.join(args['installroot'], 'html/languages') compiled = os.path.join(args['installroot'], 'html/languages/compiled') user_profile = os.path.join(args['installroot'], 'html/user_profile') + forum_file = os.path.join(args['installroot'], 'html/ops/create_forums.php') + project_inc = os.path.join(args['installroot'], 'html/project/project.inc') cmd = "chmod 02770 -R %s %s, %s %s %s" % (upload, inc, languages, compiled, user_profile) os.system("chmod g+w -R " + args['installroot']) @@ -99,8 +102,139 @@ def services(args): os.system("chmod -R o+r " + inc) os.system("chmod o+x " + languages) os.system("chmod o+x " + compiled) + os.system("sed -i '/remove the die/d' %s" % forum_file) + subprocess.Popen(["sed -i 's#REPLACE WITH PROJECT NAME#%s#' %s" % (args['fullname'], + project_inc)], shell=True, stdout=subprocess.PIPE).communicate()[0] + subprocess.Popen(["sed -i 's#REPLACE WITH COPYRIGHT HOLDER#%s#' %s" % (args['copyright'], + project_inc)], shell=True, stdout=subprocess.PIPE).communicate()[0] + + #Execute php create_forum.php... + print "Boinc Forum: Execute php create_forum.php..." + p_forum = subprocess.Popen(["php", forum_file], stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, env=env, cwd=os.path.join(args['installroot'], + 'html/ops')) + result = p_forum.communicate()[0] + if p_forum.returncode is None or p_forum.returncode != 0: + print "Failed to execute bin/xadd.\nThe error was: %s" % result + return status = open(args['service_status'], "w") status.write("started") status.close() - \ No newline at end of file + +def deployApp(args): + print "Cheking if needed to install %s..." % args['appname'] + if os.path.exists(os.path.join(args['installroot'], "."+args['appname'])): + print args['appname'] + " is already installed in this Boinc instance... skipped" + return + #Sleep until file .start_service exist (Mark the end of boinc configuration) + while True: + print "Search for file %s..." % args['service_status'] + if not os.path.exists(args['service_status']): + print "File not found... sleep for 3 secondes" + time.sleep(3) + else: + break + + print "sleeps for 30 seconds while waiting for the end of the execution of boinc_start" + time.sleep(30) + + print "setup directories..." + args['inputfile'] = os.path.join(args['installroot'], 'download', + args['appname']+'_input') + base_app = os.path.join(args['installroot'], 'apps', args['appname']) + base_app_version = os.path.join(base_app, args['version']) + args['templates'] = os.path.join(args['installroot'], 'templates') + if not os.path.exists(base_app): + os.mkdir(base_app) + if os.path.exists(base_app_version): + shutil.rmtree(base_app_version) + os.mkdir(base_app_version) + os.mkdir(args['application']) + if not os.path.exists(args['templates']): + os.mkdir(args['templates']) + shutil.copy(args['t_result'], os.path.join(args['templates'], + args['appname']+'_result')) + shutil.copy(args['t_wu'], os.path.join(args['templates'], + args['appname']+'_wu')) + shutil.copy(args['t_input'], args['inputfile']) + shutil.copy(args['binary'], os.path.join(args['application'], + args['binary_name'])) + + print "Adding '" + args['appname'] + "' to project.xml..." + print "Adding deamon for application to config.xml..." + project_xml = os.path.join(args['installroot'], 'project.xml') + config_xml = os.path.join(args['installroot'], 'config.xml') + sed_args = [args['bash'], args['appname'], args['installroot']] + sed = subprocess.Popen(sed_args, stderr=subprocess.STDOUT, + stdout=subprocess.PIPE) + result = sed.communicate()[0] + print result + + print "Running xadd script..." + env = os.environ + env['PATH'] = args['environment']['PATH'] + env['PYTHONPATH'] = args['environment']['PYTHONPATH'] + env['PYTHON'] = args['environment']['PYTHON'] + p_xadd = subprocess.Popen([os.path.join(args['installroot'], 'bin/xadd')], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env) + result = p_xadd.communicate()[0] + if p_xadd.returncode is None or p_xadd.returncode != 0: + print "Failed to execute bin/xadd.\nThe error was: %s" % result + return + + print "Sign the application binary..." + sign = os.path.join(args['installroot'], 'bin/sign_executable') + privateKeyFile = os.path.join(args['installroot'], 'keys/code_sign_private') + output = open(os.path.join(args['application'], args['binary_name']+'.sig'), 'w') + p_sign = subprocess.Popen([sign, os.path.join(args['application'], + args['binary_name']), privateKeyFile], stdout=output, + stderr=subprocess.STDOUT) + result = p_sign.communicate()[0] + if p_sign.returncode is None or p_sign.returncode != 0: + print "Failed to execute bin/sign_executable.\nThe error was: %s" % result + return + output.close() + + print "Running script bin/update_versions..." + updt_version = os.path.join(args['installroot'], 'bin/update_versions') + p_version = subprocess.Popen([updt_version], stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, stdin=subprocess.PIPE, env=env, + cwd=args['installroot']) + p_version.stdin.write('y\ny\n') + result = p_version.communicate()[0] + p_version.stdin.close() + if p_version.returncode is None or p_version.returncode != 0: + print "Failed to execute bin/update_versions.\nThe error was: %s" % result + return + + print "Fill the database... calling bin/create_work..." + create_wu(args, env) + + print "Restart Boinc..." + binstart = os.path.join(args['installroot'], 'bin/start') + binstop = os.path.join(args['installroot'], 'bin/stop') + os.system(binstop) + os.system(binstart) + + print "Boinc Application deployment is done... writing end signal file..." + sfile = open(os.path.join(args['installroot'], "."+args['appname']), 'w') + sfile.write("done") + sfile.close() + +def create_wu(args, env): + count = int(args['wu_number']) + launch_args = [os.path.join(args['installroot'], 'bin/create_work'), + '--appname', args['appname'], '--wu_name', args['wu_name'], + '--wu_template', os.path.join(args['templates'], args['appname'] + '_wu'), + '--result_template', os.path.join(args['templates'], args['appname'] + '_result'), + args['inputfile']] + for i in range(count - 1): + print "Creating project wroker num %s..." % args['wu_number'] + launch_args[4] = args['wu_name']+str(i+1) + process = subprocess.Popen(launch_args, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, env=env, + cwd=args['installroot']) + process.communicate()[0] + + \ No newline at end of file diff --git a/slapos/recipe/boinc/template/sed_update.in b/slapos/recipe/boinc/template/sed_update.in new file mode 100644 index 000000000..3886c1339 --- /dev/null +++ b/slapos/recipe/boinc/template/sed_update.in @@ -0,0 +1,14 @@ +#!%(dash)s +if [ $# -ne 2 ] +then + echo not enough argument. + echo ARGS: appname installroot + +else + +sed -i.old -e "/<\/boinc>/i\<app>\n<name>$1</name>\n<user_friendly_name>$1</user_friendly_name>\n</app>" $2/project.xml + +sed -i.old -e "/<\/daemons>/i\<daemon>\n<cmd>\nsample_bitwise_validator -d 3 -app $1\n</cmd>\n</daemon>" $2/config.xml +sed -i.old -e "/<\/daemons>/i\<daemon>\n<cmd>\nsample_assimilator -d 3 -app $1\n</cmd>\n</daemon>" $2/config.xml + +fi \ No newline at end of file -- 2.30.9