Commit ed459ed1 authored by Alain Takoudjou's avatar Alain Takoudjou

Merge branch 'master' into 1.0

parents 3c897682 7bdaefa1
......@@ -20,9 +20,11 @@ md5sum = 0284ea239083f04c8b874e08e1aca243
# in order have all patches working.
url = http://matt.ucc.asn.au/dropbear/releases/dropbear-0.53.1.tar.bz2
# NOTE DEFAULT_RECV_WINDOW and RECV_MAX_PAYLOAD_LEN are tweaked to support
# faster network throughput compared to dropbear defaults.
configure-options =
--with-zlib=${zlib:location}
CFLAGS="-DENABLE_SINGLEUSER -D__DIRTY_NO_SHELL_CHECKING"
CFLAGS="-DENABLE_SINGLEUSER -D__DIRTY_NO_SHELL_CHECKING -DDEFAULT_RECV_WINDOW=1048576 -DRECV_MAX_PAYLOAD_LEN=524288"
environment =
CPPFLAGS =-I${zlib:location}/include
......
......@@ -182,15 +182,12 @@ def keysplit(s):
class AddAuthorizedKey(GenericBaseRecipe):
def install(self):
path_list = []
ssh = self.createDirectory(self.options['home'], '.ssh')
path_list.append(ssh)
authorized_keys = AuthorizedKeysFile(os.path.join(ssh, 'authorized_keys'))
for key in keysplit(self.options['key']):
# XXX key might actually be the string 'None' or 'null'
authorized_keys.append(key)
return path_list
return [authorized_keys.filename]
......@@ -134,23 +134,25 @@ class GenericBaseRecipe(object):
pidfile=None
):
"""
Creates a very simple (one command) shell script for process replacement.
Creates a shell script for process replacement.
Takes care of quoting.
Takes care of #! line limitation when the wrapped command is a script.
if pidfile parameter is specified, then it will make the wrapper a singleton,
accepting to run only if no other instance is running.
"""
lines = [ '#!/bin/sh' ]
for comment in comments:
lines.append('# %s' % comment)
if comments:
lines += '# ', '\n# '.join(comments), '\n'
if environment:
for key in environment:
lines.append('export %s=%s' % (key, environment[key]))
lines.append('COMMAND=' + shlex.quote(command))
for key in environment or ():
lines.append('export %s=%s' % (key, environment[key]))
if pidfile:
lines.append(dedent("""\
lines.append(dedent("""
# Check for other instances
pidfile=%s
if [ -e $pidfile ]; then
......@@ -162,24 +164,31 @@ class GenericBaseRecipe(object):
rm $pidfile
fi
fi
echo $$ > $pidfile""" % (pidfile, command)))
echo $$ > $pidfile""" % (shlex.quote(pidfile), command)))
lines.append(dedent('''
# If the wrapped command uses a shebang, execute the referenced
# executable passing the script path as first argument.
# This is to workaround the limitation of 127 characters in #!
[ ! -f "$COMMAND" ] || {
[ "`head -c2`" != "#!" ] || read -r EXE ARG
} < "$COMMAND"
lines.append('exec %s' % shlex.quote(command))
exec $EXE ${ARG:+"$ARG"} "$COMMAND"'''))
parameters = map(shlex.quote, parameters)
if parameters_extra:
# pass-through further parameters
parameters.append('"$@"')
for param in parameters:
if len(lines[-1]) < 40:
lines[-1] += ' ' + shlex.quote(param)
lines[-1] += ' ' + param
else:
lines[-1] += ' \\'
lines.append('\t' + shlex.quote(param))
if parameters_extra:
# pass-through further parameters
lines[-1] += ' \\'
lines.append('\t"$@"')
lines.append('\t' + param)
content = '\n'.join(lines) + '\n'
return self.createFile(name, content, 0700)
lines.append('')
return self.createFile(name, '\n'.join(lines), 0700)
def createDirectory(self, parent, name, mode=0700):
path = os.path.join(parent, name)
......
......@@ -54,8 +54,16 @@ class Recipe(GenericBaseRecipe):
if environment is not None:
environment = dict((k.strip(), v.strip()) for k, v in [
line.split('=') for line in environment.splitlines() if line.strip() ])
return [self.createPythonScript(
wrapper_path,
# We create a python script and a wrapper around the python
# script because the python script might have a too long #! line
python_script = self.createPythonScript(
wrapper_path+'.py',
'slapos.recipe.librecipe.execute.generic_exec',
(command_line, wait_files, environment,),
)]
(command_line, wait_files, environment,), )
return [python_script, self.createWrapper(
name=wrapper_path,
command=python_script,
parameters=[],
parameters_extra=parameters_extra) ]
......@@ -58,27 +58,27 @@ command =
[template]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg
md5sum = 16b22ef9b0476352e2e5b68f5966ce62
md5sum = 51752d0e4eae8c761750117c44983594
output = ${buildout:directory}/template.cfg
mode = 0644
[template-apache-frontend]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-apache-frontend.cfg
md5sum = 00361d1e4a2788863de58e0ca9b6dfcf
md5sum = 07b6f864b46cb2f7b4b8aab71da95d13
output = ${buildout:directory}/template-apache-frontend.cfg
mode = 0644
[template-apache-replicate]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/instance-apache-replicate.cfg.in
md5sum = c495395bf4cae93ff665c5e74e6d4583
md5sum = a05bc0f925902ddf0b4d12146b10c9e1
mode = 0644
[template-slave-list]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/templates/apache-custom-slave-list.cfg.in
md5sum = f09759a0b68337d820c70e72afb4dbf9
md5sum = c01c0ebb749d16bda4649bf42d36c7ea
mode = 640
[template-slave-configuration]
......
......@@ -33,28 +33,9 @@ parts =
trafficserver-storage-config
trafficserver-promise-listen-port
## Monitoring part
###Parts to add for monitoring
certificate-authority
cron-entry-monitor
cron-entry-rss
deploy-index
deploy-settings-cgi
deploy-status-cgi
deploy-status-history-cgi
setup-static-files
certificate-authority
zero-parameters
public-symlink
cgi-httpd-wrapper
cgi-httpd-graceful-wrapper
monitor-promise
monitor-instance-log-access
## Monitor for apache
monitor-current-log-access
monitor-backup-log-access
monitor-base
monitor-ats-cache-stats-wrapper
monitor-apache-configuration-verification
monitor-verify-re6st-connectivity
extends = ${monitor-template:output}
......@@ -74,9 +55,9 @@ srv = $${buildout:directory}/srv/
var = $${buildout:directory}/var/
template = $${buildout:directory}/template/
backup = $${:srv}/backup
log = $${:var}/log
run = $${:var}/run
backup = $${buildout:directory}/srv/backup
log = $${buildout:directory}/var/log
run = $${buildout:directory}/var/run
service = $${:etc}/service
etc-run = $${:etc}/run
promise = $${:etc}/promise
......@@ -181,7 +162,7 @@ extra-context =
section logrotate_dict logrotate
section frontend_configuration frontend-configuration
section apache_configuration apache-configuration
key monitor_url monitor-parameters:url
key monitor_base_url monitor-instance-parameter:monitor-base-url
[dynamic-custom-group-template-slave-list]
< = jinja2-template-base
......@@ -633,19 +614,15 @@ frontend-name =
# Monitoring sections
#
[monitor-current-log-access]
< = monitor-directory-access
source = $${directory:log}
[monitor-backup-log-access]
< = monitor-directory-access
source = $${directory:logrotate-backup}
[monitor-conf-parameters]
private-path-list +=
$${directory:logrotate-backup}
# Produce ATS Cache stats
[monitor-ats-cache-stats-wrapper]
< = jinja2-template-base
template = ${template-wrapper:output}
rendered = $${monitor-directory:monitoring-cgi}/ats-cache-stats
rendered = $${monitor-directory:report}/ats-cache-stats
mode = 0700
command = export TS_ROOT=$${buildout:directory} && echo "<pre>$(${trafficserver:location}/bin/traffic_shell $${monitor-ats-cache-stats-config:rendered})</pre>"
extra-context =
......@@ -659,19 +636,9 @@ mode = 644
context =
raw content show:cache-stats
# Display result of apache configuration check
[monitor-apache-configuration-verification]
< = jinja2-template-base
template = ${template-wrapper:output}
rendered = $${monitor-directory:monitoring-cgi}/front-httpd-configuration
mode = 0700
command = echo "<pre>$($${apache-configuration:frontend-configuration-verification})</pre>"
extra-context =
key content :command
[monitor-verify-re6st-connectivity]
recipe = slapos.cookbook:check_url_available
path = $${monitor-directory:monitor-custom-scripts}/re6st-connectivity
path = $${directory:promise}/re6st-connectivity
url = $${public:re6st-verification-url}
dash_path = ${dash:location}/bin/dash
curl_path = ${curl:location}/bin/curl
......
......@@ -80,6 +80,10 @@ context =
[replicate]
<= slap-connection
recipe = slapos.cookbook:requestoptional
config-monitor-cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }}
config-monitor-username = ${monitor-htpasswd:username}
config-monitor-password = ${monitor-htpasswd:passwd}
{% set frontend_software_url_key = "-frontend-software-release-url" %}
{% if slapparameter_dict.has_key(frontend_software_url_key) %}
software-url = {{ slapparameter_dict.pop(frontend_software_url_key) }}
......@@ -87,8 +91,7 @@ software-url = {{ slapparameter_dict.pop(frontend_software_url_key) }}
software-url = ${slap-connection:software-release-url}
{% endif %}
software-type = {{frontend_type}}
return = private-ipv4 public-ipv4 slave-instance-information-list monitor_url
connection-monitor_url =
return = private-ipv4 public-ipv4 slave-instance-information-list monitor-base-url
{% for section, frontend_request in request_dict.iteritems() %}
[{{section}}]
......@@ -117,13 +120,10 @@ slave-amount = {{ slave_instance_list | length }}
accepted-slave-amount = {{ authorized_slave_list | length }}
rejected-slave-amount = {{ rejected_slave_list | length }}
rejected-slave-list = {{ json_module.dumps(rejected_slave_list) }}
{% for frontend in frontend_section_list %}
{{ frontend }}-monitor-url = {{ '${' + frontend + ':connection-monitor_url}' }}
{% endfor -%}
{% for frontend in frontend_list -%}
#{{frontend}}-private-ipv4 = ${request-{{frontend}}:private-ipv4}
{% endfor -%}
monitor-base-url = ${monitor-conf-parameters:base-url}
monitor-url = ${:monitor-base-url}/public/feeds
monitor-user = ${monitor-instance-parameter:username}
monitor-password = ${monitor-instance-parameter:password}
#----------------------------
#--
......@@ -149,8 +149,16 @@ extensions = jinja2.ext.do
extra-context =
section slave_information slave-information
[monitor-conf-parameters]
monitor-url-list +=
{% for frontend in frontend_section_list %}
{{ ' ${' + frontend + ':connection-monitor-base-url}' }}
{% endfor -%}
[buildout]
extends = {{ template_monitor }}
parts =
monitor-base
publish-slave-information
publish-information
{% for part in part_list -%}
......
......@@ -48,3 +48,5 @@ extra-context =
raw template_publish_slave_information ${template-replicate-publish-slave-information:target}
# Must match the key id in [switch-softwaretype] which uses this section.
raw software_type RootSoftwareInstance-default-custom-personal-custom-group-replicate
raw template_monitor ${monitor2-template:rendered}
......@@ -262,7 +262,7 @@ private-ipv4 = {{ local_ipv4 }}
{% if extra_slave_instance_list -%}
slave-instance-information-list = {{ json_module.dumps(slave_instance_information_list) }}
{% endif -%}
monitor_url = {{ monitor_url }}
monitor-base-url = {{ monitor_base_url }}
[buildout]
parts +=
......
......@@ -35,14 +35,14 @@
"title": "Remove backups older than...",
"description": "Remove all the backups in PBS that are older than specified value. It should be rdiff-backup-compatible.",
"type": "string",
"default": "3B",
"default": "2W",
"optional": true
},
"resilient-clone-number": {
"title": "Amount of backup(s) to create",
"description": "Amount of backup(s) to create. Each backup consists of a Pull Backup Server and a clone.",
"type": "integer",
"default": 2,
"default": 1,
"optional": true
},
"ignore-known-hosts-file": {
......
......@@ -50,6 +50,17 @@
"description": "Software type of your instance inside the runner",
"type": "string"
},
"cpu-usage-ratio": {
"title": "CPU Usage Ratio",
"description": "Ratio of the CPU use for compilation, if value is set to n, compilation will use number-of-cpu/n of cpus (need instance restart)",
"type": "integer",
"default" : 4
},
"instance-name": {
"title": "Displayed instance name",
"description": "Name of the instance, to show in the window title",
"type": "string"
},
"custom-frontend-backend-url": {
"title": "Custom Frontend Backend URL",
"description": "return an ipv4 frontend of the given ipv6(+optional port)",
......@@ -110,22 +121,6 @@
"description": "List of cors domains separated with space. Needed for ajax query on this monitor instance from a different domain.",
"type": "string",
"default": "monitor.app.officejs.com"
},
"cpu-usage-ratio": {
"title": "CPU Usage Ratio",
"description": "Ratio of the CPU use for compilation, if value is set to n, compilation will use number-of-cpu/n of cpus (need instance restart)",
"type": "integer",
"default" : 4
},
"resilient-clone-number": {
"title": "Number of Resilient Clones",
"description": "Number of clones to be deployed when a resilient runner, if missing a single clone is provided. Its value can be 1 or 2",
"type": "integer",
"enum": [1, 2]
},
"instance-name": {
"description": "Name of the instance, to show in the window title",
"type": "string"
}
}
}
......@@ -35,14 +35,15 @@
"title": "Remove backups older than...",
"description": "Remove all the backups in PBS that are older than specified value. It should be rdiff-backup-compatible.",
"type": "string",
"default": "3B",
"default": "2W",
"optional": true
},
"resilient-clone-number": {
"title": "Amount of backup(s) to create",
"description": "Amount of backup(s) to create. Each backup consists of a Pull Backup Server and a clone.",
"type": "integer",
"default": 2,
"default": 1,
"enum": [1, 2],
"optional": true
},
"ignore-known-hosts-file": {
......
......@@ -99,7 +99,7 @@ recipe = slapos.recipe.template:jinja2
filename = template-monitor.cfg
template = ${:_profile_base_location_}/instance-monitor.cfg.jinja2.in
rendered = ${buildout:directory}/template-monitor.cfg
md5sum = eab45fd2b0f017ab97a9c8f466c8dceb
md5sum = cd7f386fe1b5066d8508758249b408d3
context =
key apache_location apache:location
key gzip_location gzip:location
......@@ -130,7 +130,7 @@ depends =
[monitor2-bin]
<= monitor-template-script
filename = monitor.py
md5sum = 222365a469f8ab08a0367d81c0b03982
md5sum = 31beec15d3c3cd7979d04ecb834c439a
[run-promise-py]
recipe = slapos.recipe.template:jinja2
......
......@@ -61,6 +61,7 @@ services = ${:etc}/service
promises = ${:etc}/promise
monitor = ${:srv}/monitor
monitor-promise = ${:etc}/monitor-promise
monitor-report = ${:etc}/monitor-report
[monitor-directory]
recipe = slapos.cookbook:mkdirectory
......@@ -69,6 +70,7 @@ etc = ${directory:etc}
run = ${directory:monitor}/run
#run = ${directory:scripts}
promises = ${directory:monitor-promise}
report = ${directory:monitor-report}
pids = ${directory:run}/monitor
cgi-bin = ${directory:monitor}/cgi-bin
webdav = ${directory:monitor}/webdav
......@@ -374,7 +376,7 @@ monitor-httpd-port = 8196
monitor-base-url = ${monitor-frontend-promise:url}
root-instance-title = ${slap-configuration:root-instance-title}
opml-url-list =
cors-domains =
cors-domains = monitor.app.officejs.com
# XXX Hard coded parameter
collector-db = /srv/slapgrid/var/data-log/collector.db
# Credentials
......
......@@ -222,13 +222,18 @@ class Monitoring(object):
def getMonitorTitleFromUrl(self, monitor_url):
# This file should be generated
if not monitor_url.startswith('https://') and not monitor_url.startswith('http://'):
return 'Unknow Instance'
return 'Unknown Instance'
if not monitor_url.endswith('/'):
monitor_url = monitor_url + '/'
context = ssl._create_unverified_context()
url = monitor_url + '/.jio_documents/monitor.global.json' # XXX Hard Coded path
try:
response = urllib2.urlopen(url, context=context)
# XXX - working here with public url
if hasattr(ssl, '_create_unverified_context'):
context = ssl._create_unverified_context()
response = urllib2.urlopen(url, context=context)
else:
response = urllib2.urlopen(url)
except urllib2.HTTPError:
return 'Unknow Instance'
else:
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment