Skip to content

slapos-caddy
slapos-caddy
Commit 38cd519a authored by Alain Takoudjou's avatar Alain Takoudjou
Browse files
Options

Update BOINC, allows to install application with slapparameters

parent b56390cd
Show whitespace changes
Inline Side-by-side
Showing with 109 additions and 46 deletions
slapos/recipe/boinc/__init__.py
View file @ 38cd519a
......@@ -29,6 +29,7 @@ import os
import subprocess
import pwd
import signal
import zc.buildout
class Recipe(GenericBaseRecipe):
"""Deploy a fully operational boinc architecture."""
......@@ -208,11 +209,51 @@ class App(GenericBaseRecipe):
"""This recipe allow to deploy an scientific applications using boinc
Note that recipe use depend on boinc-server parameter"""
def downloadFiles(self):
"""This is used to download app files if necessary and update options values"""
for key in ('template-result', 'template-wu', 'input-file', 'binary'):
option = self.options[key].strip()
if option and (option.startswith('http') or option.startswith('ftp')):
#download the specified file
cache = os.path.join(self.options['home'].strip(), 'temp')
downloader = zc.buildout.download.Download(self.buildout['buildout'],
hash_name=True, cache=cache)
path, _ = downloader(option, md5sum=None)
mode = 0600
if key == 'binary':
mode = 0700
os.chmod(path, mode)
self.options[key] = path
def checkOptions(self):
"""Check if parameter send is valid to install or update application"""
if not self.options['app-name'].strip() or \
not self.options['version'].strip():
return False
self.appname = self.options['app-name'].strip()
self.version = self.options['version'].strip()
#for non exist application, check if parameter is complete
appdir = os.path.join(self.options['installroot'].strip(), 'apps',
self.options['app-name'].strip(),
self.options['version'].strip())
if not os.path.exists(appdir):
if not self.options['template-result'].strip() or not self.options['binary'].strip() \
or not self.options['input-file'].strip() or not self.options['template-wu'].strip() \
or not self.options['wu-number'].strip() or not self.options['platform'].strip():
print "Invalid argement values...operation cancelled"
return False
#write application to install
toInstall = open(os.path.join(self.options['home'].strip(),
'.install_' + self.appname + self.version), 'w')
toInstall.write('install or update')
toInstall.close()
return True
def install(self):
if self.options['app-name'].strip() == '' or \
self.options['version'].strip() == '':
#don't deploy empty application...skipped
self.appname = ''
self.version = ''
if not self.checkOptions():
#don't deploy empty or invalid application...skipped
return []
path_list = []
package = self.options['boinc'].strip()
......@@ -221,6 +262,7 @@ class App(GenericBaseRecipe):
developegg = self.options['develop-egg'].strip()
python_path = boinc_egg + ":" + os.environ['PYTHONPATH']
home = self.options['home'].strip()
user = pwd.getpwuid(os.stat(home).st_uid)[0]
perl = self.options['perl-binary'].strip()
svn = self.options['svn-binary'].strip()
for f in os.listdir(developegg):
......@@ -239,26 +281,26 @@ class App(GenericBaseRecipe):
self.substituteTemplate(self.getTemplateFilename('sed_update.in'),
dict(dash=self.options['dash'].strip(),
uldl_pid=self.options['apache-pid'].strip(),
user=self.options['user']))
user=user))
)
path_list.append(sh_script)
os.chmod(bash , 0700)
#If useful, download necessary files and update options path
self.downloadFiles()
start_boinc = os.path.join(home, '.start_boinc')
installroot = self.options['installroot'].strip()
version = self.options['version'].strip()
platform = self.options['platform'].strip()
apps_dir = os.path.join(installroot, 'apps')
appname = self.options['app-name'].strip()
bin_name = appname +"_"+ version +"_"+ \
bin_name = self.appname +"_"+ self.version +"_"+ \
platform + self.options['extension'].strip()
application = os.path.join(apps_dir, appname, version, platform)
application = os.path.join(apps_dir, self.appname, self.version, platform)
wrapperdir = self.options['wrapper-dir'].strip()
project = self.options['project'].strip()
parameter = dict(installroot=installroot, project=project,
appname=appname, binary_name=bin_name,
version=version, platform=platform,
appname=self.appname, binary_name=bin_name,
version=self.version, platform=platform,
application=application, environment=environment,
start_boinc=start_boinc,
wu_number=int(self.options['wu-number'].strip()),
......@@ -266,10 +308,10 @@ class App(GenericBaseRecipe):
t_wu=self.options['template-wu'].strip(),
t_input=self.options['input-file'].strip(),
binary=self.options['binary'].strip(),
bash=bash,
bash=bash, home_dir=home,
)
deploy_app = self.createPythonScript(
os.path.join(wrapperdir, appname),
os.path.join(wrapperdir, self.appname),
'%s.configure.deployApp' % __name__, parameter
)
path_list.append(deploy_app)
......
slapos/recipe/boinc/configure.py
View file @ 38cd519a
......@@ -53,6 +53,7 @@ def checkMysql(args):
print "Could not connect to MySQL database... sleep for 2 secondes"
time.sleep(2)
def checkFile(file, stime):
"""Loop until 'file' is created (exist)"""
while True:
......@@ -63,6 +64,7 @@ def checkFile(file, stime):
else:
break
def services(args):
"""This function configure a new installed boinc project instance"""
print "Checking if needed to install or reinstall Boinc-server..."
......@@ -115,6 +117,7 @@ def services(args):
writeFile(args['service_status'], "started")
def restart_boinc(args):
"""Stop (if currently is running state) and start all Boinc service"""
if args['drop_install']:
......@@ -130,54 +133,68 @@ def restart_boinc(args):
writeFile(args['start_boinc'], "started")
print "Done."
def deployApp(args):
"""Fully deploy or redeploy or update a BOINC application using existing BOINC instance"""
print "Cheking if needed to install %s..." % args['appname']
install_request_file = os.path.join(args['home_dir'],
'.install_' + args['appname'] + args['version'])
if not os.path.exists(install_request_file):
print "No install or update request for %s version %s..." % (
args['appname'], args['version'])
return
os.unlink(install_request_file)
token = os.path.join(args['installroot'], "." + args['appname'] + args['version'])
dropapp = False
newInstall = False
if os.path.exists(token):
args['previous_wu'] = int(open(token, 'r').read().strip())
if args['previous_wu'] >= args['wu_number']:
print args['appname'] + " version " + args['version'] + " is already installed in this Boinc instance... skipped"
return
else:
if args['previous_wu'] < args['wu_number']:
print args['appname'] + " Work units will be updated from %s to %s" % (
args['previous_wu'], args['wu_number'])
else:
args['previous_wu'] = 0
dropapp = True
newInstall = True
#Sleep until file .start_boinc exist (File indicate that BOINC has been started)
checkFile(args['start_boinc'], 3)
print "setup directories..."
args['inputfile'] = os.path.join(args['installroot'], 'download',
args['appname'] + '_input')
args['appname'] + args['version'] + '_input')
base_app = os.path.join(args['installroot'], 'apps', args['appname'])
base_app_version = os.path.join(base_app, args['version'])
args['templates'] = os.path.join(args['installroot'], 'templates')
t_result = os.path.join(args['templates'], args['appname']+'_result')
t_wu = os.path.join(args['templates'], args['appname']+'_wu')
t_result = os.path.join(args['templates'],
args['appname'] + args['version'] + '_result')
t_wu = os.path.join(args['templates'],
args['appname'] + args['version'] + '_wu')
binary = os.path.join(args['application'], args['binary_name'])
if not os.path.exists(base_app):
os.mkdir(base_app)
if dropapp:
if newInstall:
if os.path.exists(base_app_version):
shutil.rmtree(base_app_version)
os.mkdir(base_app_version)
os.mkdir(args['application'])
if not os.path.exists(args['templates']):
os.mkdir(args['templates'])
else:
if args['t_result']:
if os.path.exists(t_result):
os.unlink(t_result)
if os.path.exists(t_result):
os.unlink(t_wu)
shutil.copy(args['t_result'], t_result)
if args['t_wu']:
if os.path.exists(t_wu):
os.unlink(t_wu)
shutil.copy(args['t_wu'], t_wu)
if not os.path.exists(args['inputfile']):
if args['t_input']:
if os.path.exists(args['inputfile']):
os.unlink(args['inputfile'])
os.symlink(args['t_input'], args['inputfile'])
shutil.copy(args['binary'], os.path.join(args['application'],
args['binary_name']))
if args['binary'] and args['platform']:
if os.path.exists(binary):
os.unlink(binary)
shutil.copy(args['binary'], binary)
if newInstall:
print "Adding '" + args['appname'] + "' to project.xml..."
print "Adding deamon for application to config.xml..."
project_xml = os.path.join(args['installroot'], 'project.xml')
......@@ -185,19 +202,18 @@ def deployApp(args):
sed_args = [args['bash'], args['appname'], args['installroot']]
startProcess(sed_args)
if args['binary'] and args['platform']:
print "Sign the application binary..."
sign = os.path.join(args['installroot'], 'bin/sign_executable')
privateKeyFile = os.path.join(args['installroot'], 'keys/code_sign_private')
output = open(os.path.join(args['application'], args['binary_name']+'.sig'), 'w')
p_sign = subprocess.Popen([sign, os.path.join(args['application'],
args['binary_name']), privateKeyFile], stdout=output,
output = open(binary + '.sig', 'w')
p_sign = subprocess.Popen([sign, binary, privateKeyFile], stdout=output,
stderr=subprocess.STDOUT)
result = p_sign.communicate()[0]
if p_sign.returncode is None or p_sign.returncode != 0:
print "Failed to execute bin/sign_executable.\nThe error was: %s" % result
return
output.close()
#END if drop-app HERE
print "Running xadd script..."
env = os.environ
......@@ -231,19 +247,21 @@ def deployApp(args):
print "Boinc Application deployment is done... writing end signal file..."
writeFile(token, str(args['wu_number']))
def create_wu(args, env):
t_result = "templates/" + args['appname'] + '_result'
t_wu = "templates/" + args['appname'] + '_wu'
t_result = "templates/" + args['appname'] + args['version'] + '_result'
t_wu = "templates/" + args['appname'] + args['version'] + '_wu'
launch_args = [os.path.join(args['installroot'], 'bin/create_work'),
'--appname', args['appname'], '--wu_name', '',
'--wu_template', t_wu, '--result_template', t_result,
'--min_quorum', '1', '--target_nresults', '1',
args['appname']+'_input']
args['appname'] + args['version'] + '_input']
for i in range(args['previous_wu'], args['wu_number']):
print "Creating project wroker %s..." % str(i+1)
launch_args[4] = args['appname'] + str(i+1) + args['version'] + '_nodelete'
startProcess(launch_args, env, args['installroot'])
def startProcess(launch_args, env=None, cwd=None, stdout=subprocess.PIPE):
process = subprocess.Popen(launch_args, stdout=stdout,
stderr=subprocess.STDOUT, env=env,
......@@ -254,13 +272,14 @@ def startProcess(launch_args, env=None, cwd=None, stdout=subprocess.PIPE):
return False
return True
def runCmd(args):
"""Wait for Boinc Client started and run boinc cmd"""
client_config = os.path.join(args['installdir'], 'client_state.xml')
checkFile(client_config, 5)
time.sleep(10)
#Scan client state xml to find client ipv4 adress
host = result = re.search("<ip_addr>([\w\d\.:]+)</ip_addr>",
host = re.search("<ip_addr>([\w\d\.:]+)</ip_addr>",
open(client_config, 'r').read()).group(1)
args['base_cmd'][2] = host + ':' + args['base_cmd'][2]
print "Run boinccmd with host at %s " % args['base_cmd'][2]
......@@ -271,6 +290,7 @@ def runCmd(args):
#Load or reload cc_config file
startProcess(args['base_cmd'] + [args['cc_cmd']], cwd=args['installdir'])
def writeFile(file, content):
f = open(file, 'w')
f.write(content)
......
software/boinc/software.cfg
View file @ 38cd519a
......@@ -38,7 +38,7 @@ download-only = true
filename = upper_case
#Application configuration
app-name = upper_case
version = 1.0
version = 1.00
exec-extension =
#Please read Boinc platform before update platform value: http://boinc.berkeley.edu/trac/wiki/BoincPlatforms
platform = x86_64-pc-linux-gnu
......
stack/boinc/buildout.cfg
View file @ 38cd519a
......@@ -41,7 +41,7 @@ eggs =
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-boinc.cfg
output = ${buildout:directory}/template-boinc.cfg
md5sum = b85d32e9509e960459c2ba1e47741838
md5sum = 003bff525faa1e63913fa5f38c18becd
mode = 0644
#Template for deploying MySQL Database Server
......@@ -57,7 +57,7 @@ recipe = slapos.recipe.download
url = ${:_profile_base_location_}/template/${:filename}
mode = 0644
filename = apache.in
md5sum = 2d3efa25bf1cf6a673fb2cef13ae0529
md5sum = 0b3825a4a0ec82e279609d1f9dc72da4
location = ${buildout:parts-directory}/${:_buildout_section_name_}
# Local development
......
stack/boinc/instance-boinc.cfg
View file @ 38cd519a
......@@ -218,7 +218,8 @@ mysql-port = $${stunnel:local-port}
[slap-application]
<= boinc-server
recipe = slapos.cookbook:boinc.app
#appname and version is require to update wu-number
#appname and version is require to update any existing application
#otherwise, the recipe would try to install a new one
app-name = $${slap-parameter:app-name}
version = $${slap-parameter:version}
wu-number = $${slap-parameter:wu-number}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7