Commit 20aad302 authored by Aurélien Vermylen's avatar Aurélien Vermylen

Merge branch 'master' into aurelien-test-wendelin

parents 444f70dc 9de422ca
......@@ -8,7 +8,7 @@ extends =
[file]
recipe = slapos.recipe.cmmi
url = ftp://ftp.astron.com/pub/file/file-5.23.tar.gz
url = http://ftp.icm.edu.pl/packages/file/file-5.23.tar.gz
md5sum = 61db35209ce71a6d576392ce6e1d2f80
configure-options =
--disable-static
......
......@@ -14,8 +14,8 @@ extends =
[groonga]
recipe = slapos.recipe.cmmi
url = http://packages.groonga.org/source/groonga/groonga-7.0.4.tar.gz
md5sum = 90c8ddf58fecdf7dba1a053adc9b9974
url = http://packages.groonga.org/source/groonga/groonga-7.0.5.tar.gz
md5sum = d42dd98c5272ed2b9ab50ded54eb818b
# temporary patch to respect more tokens in natural language mode.
patches =
${:_profile_base_location_}/groonga.patch#9ed02fbe8400402d3eab47eee149978b
......
diff --git a/mrn_mysql_compat.h b/mrn_mysql_compat.h
index d2b227ff..337de968 100644
--- a/mrn_mysql_compat.h
+++ b/mrn_mysql_compat.h
@@ -362,6 +362,7 @@
#if defined(MRN_MARIADB_P) && \
((MYSQL_VERSION_ID >= 100207) || \
+ ((MYSQL_VERSION_ID >= 100126) && (MYSQL_VERSION_ID < 100200)) || \
((MYSQL_VERSION_ID >= 50557) && (MYSQL_VERSION_ID < 100000)))
# define mrn_create_partition_name(out, \
out_length, \
......@@ -23,8 +23,8 @@ parts =
[mariadb]
recipe = slapos.recipe.cmmi
url = https://downloads.mariadb.org/f/mariadb-10.1.25/source/mariadb-10.1.25.tar.gz/from/http%3A//fr.mirror.babylon.network/mariadb/?serve
md5sum = 6349e524bc449767c4650196d4141a02
url = https://downloads.mariadb.org/f/mariadb-10.1.26/source/mariadb-10.1.26.tar.gz/from/http%3A//fr.mirror.babylon.network/mariadb/?serve
md5sum = bb88afb72434c6d567c742896dd50d41
patch-options = -p0
patches =
${:_profile_base_location_}/mariadb_10.1.21_create_system_tables__no_test.patch#3c76aa9564a162f13aced7c0a3f783b3
......@@ -73,8 +73,8 @@ post-install =
# mroonga - a storage engine for MySQL. It provides fast fulltext search feature to all MySQL users.
# http://mroonga.github.com/
recipe = slapos.recipe.cmmi
url = http://packages.groonga.org/source/mroonga/mroonga-7.04.tar.gz
md5sum = 1d889cdd0e4f3ac0a84d18d16ab920d6
url = http://packages.groonga.org/source/mroonga/mroonga-7.05.tar.gz
md5sum = d289667a60a6aa78d8bc276ab61b2996
pre-configure = set -e
rm -rf fake_mariadb_source
mkdir -p fake_mariadb_source
......@@ -89,7 +89,8 @@ configure-options =
--disable-document
patch-options = -p1
patches =
${:_profile_base_location_}/mroonga_boolean.patch#29f85fc98e85d39b31e53b1e1683d288
${:_profile_base_location_}/mroonga_boolean.patch#c818568fe35ca6a4298f18e575d962a0
${:_profile_base_location_}/1a910088ec55c61434029d8e1cc3c6192ac508d7.diff#a8b8df4dd870a1a269dfd9dc6941467b
pre-build =
sed -i -e "s,${mariadb:location}/include,$(pwd)/fake_mariadb_source/include,g" Makefile */Makefile
environment =
......
diff --git a/ha_mroonga.cpp b/ha_mroonga.cpp
index 06691fb..b59e685 100644
--- a/ha_mroonga.cpp
+++ b/ha_mroonga.cpp
@@ -8283,7 +8283,7 @@ grn_rc ha_mroonga::generic_ft_init_ext_prepare_expression_in_boolean_mode(
--- mroonga-7.05/lib/mrn_query_parser.cpp~ 2017-07-27 07:47:36.000000000 +0200
+++ mroonga-7.05/lib/mrn_query_parser.cpp 2017-08-14 11:19:25.339015061 +0200
@@ -52,7 +52,7 @@
const char *keyword = NULL;
uint keyword_length = 0;
- grn_operator default_operator = GRN_OP_OR;
+ grn_operator default_operator = GRN_OP_ADJUST;
grn_expr_flags expression_flags = 0;
generic_ft_init_ext_parse_pragma(info,
key,
const char *raw_query = NULL;
size_t raw_query_length = 0;
- grn_operator default_operator = GRN_OP_OR;
+ grn_operator default_operator = GRN_OP_ADJUST;
grn_expr_flags expression_flags = 0;
parse_pragma(query,
query_length,
......@@ -4,8 +4,8 @@ parts =
[pcre]
recipe = slapos.recipe.cmmi
url = http://ftp.csx.cam.ac.uk/pub/software/programming/pcre/pcre-8.40.tar.bz2
md5sum = 41a842bf7dcecd6634219336e2167d1d
url = https://ftp.pcre.org/pub/pcre/pcre-8.41.tar.bz2
md5sum = c160d22723b1670447341b08c58981c1
configure-options =
--disable-static
--enable-unicode-properties
......@@ -8,9 +8,9 @@ parts =
[percona-toolkit]
recipe = slapos.recipe.cmmi
version = 2.2.15
url = http://www.percona.com/redir/downloads/percona-toolkit/${:version}/tarball/percona-toolkit-${:version}.tar.gz
md5sum = 022f40dadaea9025820530ea1f986192
version = 3.0.3
url = https://www.percona.com/downloads/percona-toolkit/${:version}/source/tarball/percona-toolkit-${:version}.tar.gz
md5sum = 8af181994fdf9aa984475637861098e9
depends =
${perl-DBI:location}
${perl-DBD-mariadb:location}
......
......@@ -35,5 +35,5 @@ environment =
[postgresql92]
<= postgresql-common
url = http://ftp.postgresql.org/pub/source/v9.2.18/postgresql-9.2.18.tar.bz2
md5sum = fd175eb5f29557c6ef2eeaf340330f9a
url = http://ftp.postgresql.org/pub/source/v9.2.18/postgresql-9.2.22.tar.bz2
md5sum = c5d3fb5229baf9e94ee2287980c55321
......@@ -51,29 +51,29 @@ pycrypto = 2.6.1
pycurl = 7.43.0
slapos.recipe.download = 1.0
slapos.recipe.template = 3.0
slapos.toolbox = 0.69
slapos.toolbox = 0.70
smmap = 0.9.0
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.70
GitPython = 2.0.8
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.70
atomize = 0.2.0
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.70
feedparser = 5.2.1
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.70
lockfile = 0.12.2
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.70
paramiko = 2.0.1
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.70
rpdb = 0.1.5
......@@ -11,7 +11,7 @@ plone.recipe.command = 1.1
pycrypto = 2.6.1
rdiff-backup = 1.0.5+SlapOSPatched001
slapos.recipe.template = 3.0
slapos.toolbox = 0.69
slapos.toolbox = 0.70
smmap = 0.9.0
numpy = 1.11.2
pyasn1 = 0.2.3
......@@ -19,41 +19,41 @@ pyasn1 = 0.2.3
websockify = 0.8.0
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.70
dnspython = 1.15.0
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.70
erp5.util = 0.4.49
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.70
passlib = 1.6.5
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.70
GitPython = 2.0.8
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.70
atomize = 0.2.0
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.70
feedparser = 5.2.1
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.70
lockfile = 0.12.2
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.70
paramiko = 2.0.1
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.70
pycurl = 7.43.0
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.70
rpdb = 0.1.5
#!{{ python_executable }}
from ipykernel.kernelbase import Kernel
from ipykernel.kernelapp import IPKernelApp
from IPython.core.display import HTML
import requests
import json
import sys
# erp5_url from buildout
erp5_url = "{{ erp5_url }}"
if not erp5_url:
erp5_url = None
else:
erp5_url = "%s/erp5/Base_executeJupyter" % erp5_url
erp5_url = None
if len(sys.argv) > 1:
erp5_url = "%s/erp5/Base_executeJupyter" % (sys.argv[1],)
class MagicInfo:
"""
......
......@@ -16,7 +16,7 @@
[instance-jupyter-notebook]
filename = instance.cfg.in
md5sum = a2116d236542e139aa446a6b88db3d28
md5sum = 27bb1bcf71a2f637f539af697ed73445
[jupyter-notebook-config]
filename = jupyter_notebook_config.py.jinja
......@@ -27,13 +27,13 @@ filename = jupyter_set_password.cgi.jinja
md5sum = b8d31441780b524a7e52d1710dd78385
[erp5-kernel]
filename = ERP5kernel.py.jinja
md5sum = 8bd16cc9f35b00172e8266f1cde5956f
filename = ERP5kernel.py
md5sum = 7d5309fe79afbcb455c0d8181b42e56c
[kernel-json]
filename = kernel.json.jinja
md5sum = ab6e78ea20855e07d388b5b86d1770fe
md5sum = 33547be93a67530165e079dc3ecfdac3
[custom-js]
filename = custom.js.jinja
filename = custom.js
md5sum = 0bf9e2eb1718b14307265fe05a167018
......@@ -129,13 +129,9 @@ recipe = slapos.cookbook:publish.serialised
url = https://[${instance-parameter:host}]:${instance-parameter:port}
[erp5-kernel]
<= dynamic-jinja2-template-base
template = {{ erp5_kernel_location }}/{{ erp5_kernel_filename }}
rendered = ${directory:erp5_kernel_dir}/ERP5kernel.py
# Use ipython as executable python as we'll be needing requests library in kernel
context =
raw python_executable {{ bin_directory }}/ipython
key erp5_url slapconfiguration:configuration.erp5-url
recipe = slapos.cookbook:symbolic.link
link-binary = {{ erp5_kernel_location }}/{{ erp5_kernel_filename }}
target-directory = ${directory:erp5_kernel_dir}
[kernel-json]
<= dynamic-jinja2-template-base
......@@ -144,13 +140,13 @@ rendered = ${directory:erp5_kernel_dir}/kernel.json
# Use python2.7 executable bin file for kernel config
context =
raw python_executable {{ python_executable }}
key kernel_dir erp5-kernel:rendered
raw kernel_dir ${erp5-kernel:target-directory}/{{ erp5_kernel_filename }}
key erp5_url slapconfiguration:configuration.erp5-url
raw display_name ERP5
raw language_name python
[custom-js]
<= dynamic-jinja2-template-base
template = {{ custom_js_location }}/{{ custom_js_filename }}
rendered = ${directory:jupyter_custom_dir}/custom.js
mode = 0744
recipe = slapos.cookbook:symbolic.link
target-directory = ${directory:jupyter_custom_dir}
link-binary = {{ custom_js_location }}/custom.js
......@@ -2,6 +2,7 @@
"argv": [
"{{python_executable}}",
"{{kernel_dir}}",
"{{erp5_url}}",
"-f",
"{connection_file}"
],
......
......@@ -5,7 +5,7 @@ extends = common.cfg
# XXX - use websockify = 0.5.1 for compatibility with kvm frontend
websockify = 0.5.1
slapos.toolbox = 0.69
slapos.toolbox = 0.70
erp5.util = 0.4.49
apache-libcloud = 1.1.0
collective.recipe.environment = 0.2.0
......@@ -16,23 +16,23 @@ smmap = 0.9.0
# websockify = 0.8.0
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.70
GitPython = 2.0.8
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.70
atomize = 0.2.0
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.70
dnspython = 1.14.0
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.70
feedparser = 5.2.1
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.70
lockfile = 0.12.2
# Required by:
......@@ -40,9 +40,9 @@ lockfile = 0.12.2
numpy = 1.11.2rc1
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.70
paramiko = 2.0.2
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.70
passlib = 1.6.5
\ No newline at end of file
# NayuOS
This is a SlapOS recipe to build NayuOS. It needs to be put in the <code>/srv/slapgrid/\<part\>/srv/runner/project/slapos/software/</code> directory. The created directory is called <code>\<nayuos_build_dirname\></code> in this documentation.
This is a SlapOS recipe to build NayuOS.
## License
......@@ -8,42 +8,61 @@ GPL v2 or later
## Requirements
* sudo on the host (for now)
* some environment variables need to be authorized to be propagated when cros_sdk calls sudo:
* sudo on the host
* environment variables need to be authorized to be propagated when cros_sdk calls sudo: `: Defaults env_keep += "CROS_CACHEDIR DEPOT_TOOLS"`
* slapuser with sudo rights to execute the cros_sdk scripts (needed to access the chroot environment provided by Chromium OS)
in /etc/sudoers (replace slapuser9 by your user, and release-R48-7647.B by the release you have chosen): `: slapuser9 ALL=NOPASSWD: /srv/slapgrid/slappart9/srv/runner/instance/slappart0/parts/chromiumos/release-R48-7647.B/chromite/bin/cros_sdk, /srv/slapgrid/slappart9/srv/runner/instance/slappart0/wrapper_bin/wrapper_cros_sdk, /bin/kill`
: Defaults env_keep += "CROS_CACHEDIR DEPOT_TOOLS"
It's useful to have the right to kill cros_sdk processes, when needed. ;)
* slapuser with sudo rights to execute the cros_sdk scripts (needed to access the chroot environment provided by Chromium OS)
in /etc/sudoers (replace slapuser9 by your user, and release-R48-7647.B by the release you have chosen):
## Technical notes
: slapuser9 ALL=NOPASSWD: /srv/slapgrid/slappart9/srv/runner/instance/slappart0/parts/chromiumos/release-R48-7647.B/chromite/bin/cros_sdk, /srv/slapgrid/slappart9/srv/runner/instance/slappart0/wrapper_bin/wrapper_cros_sdk, /bin/kill
After any change to the build process it is necessary to delete (using sudo)
the building environment at `~/srv/runner/instance/slappart0/parts/chromiumos/<TAG>`.
BEWARE that the web runner is serving images for the [official website](https://nayuos.nexedi.com).
NayuOS and ChromiumOS is "just" a version of Gentoo. Thus it uses `ebuild` packages
and anything installable in Gentoo can be installed to NayuOS too. Of course only
during OS build phase and one has to count with limited space.
### Upgrading (building new image)
Please read **Requirements** section carefully. After selecting your desired
`release` from the list <https://chromium.googlesource.com/chromiumos/manifest/+refs>
it is **necessary** to add `sudo` rules for that release as shown there.
It's useful to have the right to kill cros_sdk processes, when needed. ;)
## Input
In the vifib parameters (softinst\<nb\>.host.vifib.net \> Services \> Parameters):
* board / ex: peppy, swanky, ... (choosing daisy will accept all licenses for the daisy board build only, in order to use Mali drivers, see [chromium mailing list](https://groups.google.com/a/chromium.org/forum/#!topic/chromium-os-dev/Pf9ZG2itxWM))
* branch / ex: release-R46-7390.B (you can find the release in the [Chromium OS source tree](https://chromium.googlesource.com/chromiumos/manifest/+refs))
* keep_cache / yes|no (choosing "no" saves about 15Go of disk space per board, choosing "yes" will makes next build faster and less expensive in term of needed ressources because of not rebuilding everything)
* **board** / ex: peppy, swanky, ... (choosing daisy will accept all licenses for the daisy board build only, in order to use Mali drivers, see [chromium mailing list](https://groups.google.com/a/chromium.org/forum/#!topic/chromium-os-dev/Pf9ZG2itxWM))
* **branch** / ex: release-R46-7390.B (you can find the release in the [Chromium OS source tree](https://chromium.googlesource.com/chromiumos/manifest/+refs))
* **keep_cache** / yes|no (choosing "no" saves about 15Go of disk space per board, choosing "yes" will makes next build faster and less expensive in term of needed ressources because of not rebuilding everything)
## Output
The image will be produced in:
<code>/srv/slapgrid/\<part\>/srv/runner/instance/\<inst_part\>/parts/chromiumos/images/</code>
and the logs are in:
<code>/srv/slapgrid/\<part\>/srv/runner/instance/\<inst_part\>/var/log/cros_sources_dl.log</code> and <code>/srv/slapgrid/\<part\>/srv/runner/instance/\<inst_part\>/var/log/cros_build.log</code>
The script that download the sources and build is located in
<code>/srv/slapgrid/\<part\>/srv/runner/instance/\<inst_part\>/etc/run</code>
Software release produces a build script `<instance_partition>/etc/run/cros_full_build`.
Build produces
* Image: `<instance_partition>/parts/chromiumos/images/`
* Compilation logs: `<instance_partition>/var/log/cros_sources_dl.log`
* Build logs: `<instance_partition>/var/log/cros_build.log`
## External documents
* [ <code>repo</code> command reference ](https://source.android.com/source/using-repo.html)
* [ NayuOS official website ](https://www.nayuos.com)
* [ `repo` command reference ](https://source.android.com/source/using-repo.html)
* [ NayuOS official website ](https://nayuos.nexedi.com)
* [ crouton for chroot ](https://github.com/dnschneid/crouton) ([warning about verified boot](https://github.com/dnschneid/crouton/blob/2a1fc9da380650f47e2bcf37d00962bfb68c4830/installer/main.sh#L517-L536))
## Notes for possible improvements
* [ Running virtual machines on your chromebook ](https://www.chromium.org/chromium-os/developer-information-for-chrome-os-devices/running-virtual-machines-on-your-chromebook)
* to have a more common User Agent (the one of ChromiumOS/NayuOS is quite rare and identifies the user, see [studies of the EFF](https://panopticlick.eff.org/static/browser-uniqueness.pdf)), it seems possible to change the User-Agent flag for guest mode in the getOffTheRecord function, and adding a line (key "kUserAgent" , value "some common user agent" string). Then rebuild Chromium and [add it to NayuOS](https://www.chromium.org/chromium-os/developer-guide#TOC-Making-changes-to-the-Chromium-web-).
* remove need of root priviledge for entering the chroot, maybe by using fakeroot in 'scripts/wrapper_sudo.in'?
* change more options on Chromium OS "Privacy" part by default: there are [a few options](https://support.google.com/chromebook/answer/114836) which still use Google services
* provide ChromiumOS package manager [ chromebrew ](https://skycocker.github.io/chromebrew/) by default
# Jinja2 template of a buildout file to create runnable scripts
#
# Received variables
# - software_dir: location where this repository was cloned to
# - instance_dir: location where this 'instance.cfg' is gettin rendered (same as ${buildout:directory})
[buildout]
parts =
parameters
promise-sudo-on-host
template-full-build-script
template-sudo-wrapper
template-cros-sdk-wrapper
# eggs given by software.cfg
# standard declaration of eggs directories
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
......@@ -16,15 +23,18 @@ develop-eggs-directory = {{ develop_eggs_directory }}
recipe = slapos.recipe.build:gitclone
repository = https://chromium.googlesource.com/chromium/tools/depot_tools.git
branch = master
git-executable = {{ git_path }}/bin/git
[customize-path]
# add depot tools directory (for cros_sdk binary among others) and git directory to the path
command =
export PATH="${directory:wrapper_dir}":"{{ git_path }}":"{{ curl_path }}":"${depot-tools:location}":"$PATH";
export PATH="${directory:wrapper_dir}":"{{ git_path }}/bin":"{{ curl_path }}/bin":"${depot-tools:location}":"$PATH";
[nayuos-ebuilds]
recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/nayuos-ebuilds.git
branch = master
git-executable = {{ git_path }}/bin/git
############################################################################################
......@@ -44,20 +54,25 @@ cert = ${slap_connection:cert_file}
recipe = slapos.cookbook:mkdirectory
log = ${buildout:directory}/var/log
run = ${buildout:directory}/etc/run
promise = ${buildout:directory}/etc/promise
wrapper_dir = ${buildout:directory}/wrapper_bin
cros_location = ${buildout:directory}/parts/chromiumos
ebuilds_dir = ${nayuos-ebuilds:location}
scripts_dir = {{ scripts_dir }}
logo_dir = {{ logo_dir }}
[bin]
# dummy section to hold references to frequently used binaries
wrapper_cros_sdk=${directory:wrapper_dir}/wrapper_cros_sdk
bash=/bin/bash
sudo=/usr/bin/sudo
[promise-sudo-on-host]
# assert sudo is installed, as it is required to enter the chroot 'cros_sdk'
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:promise}/${:_buildout_section_name_}
command-line = sudo -V
[template-sudo-wrapper]
recipe = slapos.recipe.template:jinja2
template = {{ scripts_dir }}/wrapper_sudo.in
template = {{ software_dir }}/scripts/wrapper_sudo.in
rendered = ${directory:wrapper_dir}/sudo
md5sum = ded5a92be4e37ec32eb9d3087d3e19bd
mode = 0700
......@@ -68,33 +83,37 @@ context =
[template-cros-sdk-wrapper]
recipe = slapos.recipe.template:jinja2
template = {{ scripts_dir }}/wrapper_cros_sdk.in
template = {{ software_dir }}/scripts/wrapper_cros_sdk.in
rendered = ${bin:wrapper_cros_sdk}
md5sum = 7159fe3d5b85a283733cf686c4ee0a74
mode = 0700
context =
# XXX bash path is the one from the host
key bash_path bin:bash
raw git_path {{ git_path }}
raw curl_path {{ curl_path }}
raw git_path {{ git_path }}/bin
raw curl_path {{ curl_path }}/bin
[template-full-build-script]
# create the rendered script in the buildout-directory/etc/run
# (use jinja for templating)
recipe = slapos.recipe.template:jinja2
template = {{ scripts_dir }}/cros_full_build.in
template = {{ software_dir }}/scripts/cros_full_build.in
rendered = ${directory:run}/cros_full_build
md5sum = 75599e6b8418a5f3756c7c7b26600399
md5sum = 2bb9dd83260ea96dd6a6602f6faa9794
mode = 0700
context =
key bash_path bin:bash
key instance_log_dir directory:log
key cros_location directory:cros_location
key export_path_cmd customize-path:command
key branch parameters:configuration.branch
key boards_list parameters:configuration.boards
key keep_cache parameters:configuration.keep_cache
key ebuilds_dir directory:ebuilds_dir
key scripts_dir directory:scripts_dir
key logo_dir directory:logo_dir
raw nayu_dev_packages net-libs/nodejs net-misc/re6stnet dev-vcs/git dev-python/flask dev-python/virtualenv sys-fs/cryptsetup
key bash_path bin:bash
key instance_log_dir directory:log
key cros_location directory:cros_location
key export_path_cmd customize-path:command
key branch parameters:configuration.branch
key boards_list parameters:configuration.boards
key keep_cache parameters:configuration.keep_cache
key ebuilds_dir nayuos-ebuilds:location
raw scripts_dir {{ software_dir }}/scripts
raw logo_dir {{ software_dir }}/logo
# packages to be copied from our overlay into ChromiuOS' overlay
# those packages will be marked as dependency of root filesystem
raw nayu_dev_rootfs_packages app-misc/nayuos-chromium-policy
# those packages will be marked as dependency of the chromium build itself
raw nayu_dev_packages net-misc/re6stnet dev-vcs/git dev-python/flask dev-python/virtualenv sys-fs/cryptsetup
This diff is collapsed.
......@@ -14,8 +14,10 @@ ORIGINAL_GRANDENET_SCRIPT=~/trunk/src/third_party/chromiumos-overlay/net-misc/re
GRANDENET_SCRIPT="usr/local/bin/grandenet"
INIT_SSH_SERVER="etc/init/openssh-server.conf"
BASHRC="etc/skel/.bashrc"
EXPECTED_ALIAS="alias git='git --exec-path=/usr/local/libexec/git-core/'"
GIT_CORE="usr/local/libexec/git-core/"
GIT_EXPECTED_EXPORT="export GIT_EXEC_PATH=/usr/local/libexec/git-core"
VIRTUALENV_BIN="usr/local/bin/virtualenv"
CHROMIUM_POLICY="etc/chromium/policies/recommended/nayuos_policy.json"
HAS_FAILED=0
......@@ -32,14 +34,17 @@ function print_result() {
# MOUNT IMAGE AND GET INFO
install -d ${MOUNTPOINT}
install -d "${MOUNTPOINT}"
./mount_gpt_image.sh --safe -f $( ./get_latest_image.sh --board=${BOARD} ) -r ${MOUNTPOINT}
echo $(ls "${MOUNTPOINT}/usr/local")
if [[ $(ls "${MOUNTPOINT}/usr/local") ]] ; then
my_diff=$(diff ${ORIGINAL_GRANDENET_SCRIPT} "${MOUNTPOINT}/${GRANDENET_SCRIPT}")
my_diff=$(diff "${ORIGINAL_GRANDENET_SCRIPT}" "${MOUNTPOINT}/${GRANDENET_SCRIPT}")
opensshd_config=$(ls "${MOUNTPOINT}/${INIT_SSH_SERVER}")
gitalias=$(grep "${EXPECTED_ALIAS}" "${MOUNTPOINT}/${BASHRC}")
gitcore=$(ls "${MOUNTPOINT}/${GIT_CORE}")
gitexport=$(grep "${GIT_EXPECTED_EXPORT}" "${MOUNTPOINT}/${BASHRC}")
virtualenvbin=$(ls "${MOUNTPOINT}/${VIRTUALENV_BIN}")
chromium_policy=$(ls "${MOUNTPOINT}/${CHROMIUM_POLICY}")
else
is_empty=1
fi
......@@ -50,7 +55,7 @@ rmdir ${MOUNTPOINT}
# PRINT RESULTS
echo "* test if /usr/local exists"
if [[ ${is_empty} == 1 ]] ; then
if [[ "${is_empty}" == "1" ]] ; then
print_result ${FAILURE} "/usr/local is empty."
else
print_result ${SUCCESS} "/usr/local is not empty."
......@@ -71,10 +76,18 @@ else
print_result ${SUCCESS} "opensshd config removed."
fi
echo "* test git quick fix for option --exec-path"
echo "* test if git core directory is not empty"
if [[ "${gitcore}" == "" ]] ; then
print_result ${FAILURE} "No git file in ${MOUNTPOINT}/${GIT_CORE}."
else
print_result ${SUCCESS} "git core directory exists and is not empty."
fi
echo "* test bashrc changes for git paths quick fix"
if [[ ${gitalias} == "" ]] ; then
print_result ${FAILURE} "Expected alias for git command not in ${MOUNTPOINT}/${BASHRC}. Should be: ${EXPECTED_ALIAS}"
if [[ "${gitexport}" == "" ]] ; then
print_result ${FAILURE} "Expected alias for git command not in ${MOUNTPOINT}/${BASHRC}. There should be: ${GIT_EXPECTED_EXPORT}"
else
print_result ${SUCCESS} "git alias is correct."
fi
......@@ -86,6 +99,14 @@ else
else
print_result ${FAILURE} "Expected virtualenv binary not in ${MOUNTPOINT}/${VIRTUALENV_BIN}."
fi
echo "* test if Chromium policies are installed"
if [[ ${chromium_policy} ]] ; then
print_result ${SUCCESS} "Chromium policies are installed."
else
print_result ${FAILURE} "Expected Chromium policies file not in ${MOUNTPOINT}/${CHROMIUM_POLICY}."
fi
fi
exit ${HAS_FAILED}
......@@ -4,46 +4,30 @@ extends =
../../stack/slapos.cfg
parts +=
# use stack/slapos.cfg
slapos-cookbook
verify-packages-exist-on-host
template-instance
install-eggs-for-the-instance
git
curl
versions = versions
[versions]
slapos.recipe.template = 3.0
[verify-packages-exist-on-host]
recipe = plone.recipe.command
stop-on-error = true
# run the same command when installing and on updates
update-command = ${:command}
command = sudo -V
[template-instance]
# create the instance.cfg file in the buildout directory
# (use jinja for templating)
# jinja2 render instance.cfg file in the buildout directory
recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/instance.cfg
rendered = ${buildout:directory}/instance.cfg
md5sum = be6bd665c0b356a84090d200ea14b33c
mode = 0644
scripts_dir = ${:_profile_base_location_}/scripts
logo_dir = ${:_profile_base_location_}/logo
curl_path = ${curl:location}/bin
git_path = ${git:location}/bin
md5sum = 154882944c6a58642629c17d47b4f6fc
context =
# for access to the eggs from the instance
key eggs_directory buildout:eggs-directory
key develop_eggs_directory buildout:develop-eggs-directory
key scripts_dir :scripts_dir
key logo_dir :logo_dir
key curl_path :curl_path
key git_path :git_path
key software_dir :_profile_base_location_
key instance_dir buildout:directory
key curl_path curl:location
key git_path git:location
key eggs_directory buildout:eggs-directory
key develop_eggs_directory buildout:develop-eggs-directory
[install-eggs-for-the-instance]
# after installation of the eggs, the recipes
# will be available (added to sys.path)
recipe = zc.recipe.egg
eggs = slapos.recipe.build
plone.recipe.command
......@@ -118,39 +118,39 @@ persistent = 4.2.3
pycrypto = 2.6.1
pycurl = 7.43.0
slapos.recipe.template = 3.0
slapos.toolbox = 0.69
slapos.toolbox = 0.70
smmap2 = 2.0.1
transaction = 1.7.0
zodbpickle = 0.6.0
zodbtools = 0.0.0.dev3
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.70
GitPython = 2.1.3
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.70
PyRSS2Gen = 1.1
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.70
atomize = 0.2.0
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.70
dnspython = 1.14.0
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.70
feedparser = 5.2.1
# slapos.toolbox==0.69
# slapos.toolbox==0.70
lockfile = 0.12.2
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.70
paramiko = 2.1.2
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.70
passlib = 1.7.1
......@@ -46,4 +46,4 @@ mode = 0644
[versions]
slapos.recipe.template = 3.0
slapos.toolbox = 0.69
slapos.toolbox = 0.70
......@@ -112,15 +112,15 @@ plone.recipe.command = 1.1
pycrypto = 2.6.1
pycurl = 7.43.0
slapos.recipe.template = 3.0
slapos.toolbox = 0.69
slapos.toolbox = 0.70
smmap = 0.9.0
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.70
GitPython = 2.0.8
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.70
atomize = 0.2.0
# Required by:
......@@ -128,11 +128,11 @@ atomize = 0.2.0
backports.ssl-match-hostname = 3.4.0.2
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.70
feedparser = 5.1.3
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.70
lockfile = 0.12.2
# Required by:
......@@ -140,10 +140,10 @@ lockfile = 0.12.2
miniupnpc = 1.9
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.70
paramiko = 2.0.1
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.70
rpdb = 0.1.5
......@@ -15,7 +15,7 @@
# not need these here).
[template-erp5]
filename = instance-erp5.cfg.in
md5sum = 649137323f971026a6c62e357ce3f513
md5sum = aa10af616493b97d2b630c46d869a9d7
[template-balancer]
filename = instance-balancer.cfg.in
......
......@@ -9,6 +9,7 @@
{% set has_jupyter = jupyter_dict.get('enable', jupyter_enable_default.lower() in ('true', 'yes')) -%}
{% set jupyter_zope_family = jupyter_dict.get('zope-family', '') -%}
{% set monitor_base_url_dict = {} -%}
{% set monitor_dict = slapparameter_dict.get('monitor', {}) %}
[request-common]
<= request-common-base
config-use-ipv6 = {{ dumps(slapparameter_dict.get('use-ipv6', False)) }}
......@@ -42,7 +43,7 @@ config-name = {{ name }}
{{ request('memcached-persistent', 'kumofs', 'kumofs', {'tcpv4-port': 2000}, {'url': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
{{ request('memcached-volatile', 'kumofs', 'memcached', {'tcpv4-port': 2010, 'ram-storage-size': 64}, {'url': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
{{ request('cloudooo', 'cloudooo', 'cloudooo', {'tcpv4-port': 2020}, {'url': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
{{ request('mariadb', 'mariadb', 'mariadb', {'tcpv4-port': 2099}, {'database-list': True, 'test-database-list': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
{{ request('mariadb', 'mariadb', 'mariadb', {'tcpv4-port': 2099, 'max-slowqueries-threshold': monitor_dict.get('max-slowqueries-threshold', 1000), 'slowest-query-threshold': monitor_dict.get('slowest-query-threshold', '') }, {'database-list': True, 'test-database-list': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
{% if has_posftix -%}
{{ request('smtp', 'postfix', 'smtp', {'tcpv4-port': 2025, 'smtpd-sasl-user': 'erp5@nowhere'}, key_config={'smtpd-sasl-password': 'publish-early:smtpd-sasl-password'}) }}
{%- else %}
......@@ -313,7 +314,9 @@ config-ssl-authentication-dict = {{ dumps(ssl_authentication_dict) }}
config-shared-certificate-authority-path = ${directory:ca-dir}
config-monitor-passwd = ${monitor-htpasswd:passwd}
config-name = ${:name}
config-apachedex-promise-threshold = {{ dumps(monitor_dict.get('apachedex-promise-threshold', 70)) }}
config-apachedex-configuration = {{ dumps(monitor_dict.get('apachedex-configuration',
'--erp5-base "/erp5(/|$|/\?)" --skip-user-agent Zabbix --error-detail --js-embed --quiet')) }}
[request-frontend-base]
{% if has_frontend -%}
......
......@@ -17,41 +17,41 @@ gunicorn = 19.4.5
prettytable = 0.7.2
pycurl = 7.43.0
slapos.recipe.template = 3.0
slapos.toolbox = 0.69
slapos.toolbox = 0.70
smmap = 0.9.0
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.70
GitPython = 2.0.8
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.70
PyRSS2Gen = 1.1
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.70
atomize = 0.2.0
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.70
dnspython = 1.14.0
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.70
erp5.util = 0.4.49
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.70
feedparser = 5.2.1
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.70
lockfile = 0.12.2
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.70
paramiko = 2.0.2
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.70
passlib = 1.6.5
......@@ -95,7 +95,7 @@ futures = 3.1.1
gitdb2 = 2.0.2
gunicorn = 19.7.1
slapos.recipe.template = 3.0
slapos.toolbox = 0.69
slapos.toolbox = 0.70
smmap2 = 2.0.3
# Required by:
......@@ -124,11 +124,11 @@ Flask-Script = 2.0.5
Flask-WTF = 0.14.2
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.70
GitPython = 2.1.5
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.70
PyRSS2Gen = 1.1
# Required by:
......@@ -144,7 +144,7 @@ Unipath = 1.1
WTForms = 2.1
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.70
atomize = 0.2.0
# Required by:
......@@ -152,23 +152,23 @@ atomize = 0.2.0
blinker = 1.4
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.70
dnspython = 1.15.0
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.70
erp5.util = 0.4.49
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.70
feedparser = 5.2.1
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.70
lockfile = 0.12.2
# Required by:
# slapos.toolbox==0.69
# slapos.toolbox==0.70
passlib = 1.7.1
# Required by:
......
......@@ -52,6 +52,7 @@ extends =
../../component/coreutils/buildout.cfg
../../component/grep/buildout.cfg
../../component/dash/buildout.cfg
../../component/bash/buildout.cfg
../../component/wget/buildout.cfg
../../component/aspell/buildout.cfg
../../component/cloudooo/buildout.cfg
......@@ -100,6 +101,7 @@ parts +=
percona-toolkit
zabbix-agent
dash
bash
wget
userhosts
postfix
......@@ -168,6 +170,10 @@ command = grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link
<= download-base
mode = 755
[mariadb-slowquery-check-script]
<= download-base
mode = 755
[mariadb-slow-query-report-script]
<= download-base
mode = 755
......@@ -232,6 +238,8 @@ context =
key mariadb_link_binary template-mariadb:link-binary
key zope_link_binary template-zope:link-binary
key apache_location apache:location
key apdex_result_check_script apdex-result-check-script:target
key mariadb_slowquery_check_script mariadb-slowquery-check-script:target
key aspell_location aspell:location
key bin_directory buildout:bin-directory
key buildout_bin_directory buildout:bin-directory
......@@ -242,6 +250,7 @@ context =
key curl_location curl:location
key cyrus_sasl_location cyrus-sasl:location
key dash_location dash:location
key bash_location bash:location
key dbus_glib_location dbus-glib:location
key dbus_location dbus:location
key dcron_location dcron:location
......@@ -351,6 +360,9 @@ link-binary =
[template-balancer]
<= download-base
[apdex-result-check-script]
<= download-base
[template-haproxy-cfg]
<= download-base
......
......@@ -19,11 +19,15 @@ md5sum = 844d62cd6f9d6e3d1d78d52de2b72a49
[mariadb-slow-query-report-script]
filename = mysql-querydigest.sh.in
md5sum = dc974bd74cf967ae6250d81322629c44
md5sum = cfe6ab8ae54a521ecb269e9d9762cbeb
[mariadb-slowquery-check-script]
filename = instance-mariadb-check-slowquery-result.sh.in
md5sum = 356e0e2db1da0e8b479908fb739e5cc0
[template-mariadb]
filename = instance-mariadb.cfg.in
md5sum = 2126e64bc70893500dc24c29770e84c8
md5sum = 7ee2e801dda1181d1b42281e6466fc4d
[template-kumofs]
filename = instance-kumofs.cfg.in
......@@ -75,7 +79,7 @@ md5sum = 0969fbb25b05c02ef3c2d437b2f4e1a0
[template]
filename = instance.cfg.in
md5sum = e364ea67bfe786b6b6ebd6c4f0cd628a
md5sum = cbe1df5dbd9b79ed7adc027fd183e186
[monitor-template-dummy]
filename = dummy.cfg
......@@ -83,7 +87,7 @@ md5sum = d41d8cd98f00b204e9800998ecf8427e
[template-erp5]
filename = instance-erp5.cfg.in
md5sum = 64fe4800dc9430aac8e325ba4b3d97ae
md5sum = 8d9420da8f22dd41d5f076d7506a6620
[template-zeo]
filename = instance-zeo.cfg.in
......@@ -95,7 +99,11 @@ md5sum = a2377d5c53fd2a441ea713b428e4844b
[template-balancer]
filename = instance-balancer.cfg.in
md5sum = d2a339d0bc2f05f3f1f78defffae6ab6
md5sum = d14ee7f13e2bd815cc96e28101e59670
[apdex-result-check-script]
filename = instance-balancer-check-apachedex-result.sh.in
md5sum = 421c68c97cadc49911382cd3185288a1
[template-haproxy-cfg]
filename = haproxy.cfg.in
......
#!{{ bash }}
set -e
APACHEDEX_FILE='{{ apdex_file }}/ApacheDex-'$(date +%Y-%m-%d)'.html'
APACHEDEX_REPORT_JSON_FILE={{ apdex_status_file }}
DESIRED_THRESHOLD={{ user_threshold }}
# Check if the file is there
if [ ! -s "$APACHEDEX_FILE" ]; then
# If file doesn't exists create one
# If it is empty check for modification time
if [ ! -f "$APACHEDEX_FILE" ]; then
touch $APACHEDEX_FILE
else
MODIFIED_DATE=`stat -c '%Z' $APACHEDEX_FILE`
CURRENT_DATE=`date +%s`
if [[ `echo "$CURRENT_DATE - $MODIFIED_DATE" | bc` -gt 108000 ]]
then
echo "File modification date is greater than 30 hours"
JSON_CONTENT=`cat $APACHEDEX_REPORT_STATUS_FILE`
MESSAGE=`echo $JSON_CONTENT | python -c 'import json,sys;obj=json.load(sys.stdin);print obj["message"]'`
echo $MESSAGE
exit 2
else
echo "File is empty for now"
fi
fi
else
# Check if the result exists
{
REGEX="Overall<\/h2><table .*><tr>[[:space:]]<th>apdex<\/th><th>.*?<tr>[[:space:]]<td [^<]*>(.*?)%<\/td>"
FILE_CONTENT=`cat $APACHEDEX_FILE`
if [[ $FILE_CONTENT =~ $REGEX ]]
then
RESULT=${BASH_REMATCH[1]}
RESULT=${RESULT:-0}
if [[ `echo "$RESULT > $DESIRED_THRESHOLD" | bc` -eq 1 ]]
then
echo "Your score is $RESULT %, Thanks for keeping it all clean"
else
echo "Threshold is lower than exptected: Expected was $DESIRED_THRESHOLD % and we current is $RESULT %"
exit 2
fi
else
echo "No threshold found in the result"
fi
} || {
echo "Cannot parse the apdex result"
}
fi
......@@ -219,7 +219,7 @@ path = ${directory:promise}/apache
hostname = {{ ipv4 }}
port = {{ apache_dict.values()[0][0] }}
[publish]
[{{ section('publish') }}]
recipe = slapos.cookbook:publish.serialised
{% for family_name, (apache_port, scheme, _, _) in apache_dict.items() -%}
{{ family_name ~ '-v6' }} = {% if ipv6_set %}{{ scheme ~ '://[' ~ ipv6 ~ ']:' ~ apache_port }}{% endif %}
......@@ -260,7 +260,7 @@ cert =
crl =
{%- endif %}
[logrotate-apache]
[{{ section('logrotate-apache') }}]
< = logrotate-entry-base
name = apache
log = ${apache-conf-parameter-dict:error-log} ${apache-conf-parameter-dict:access-log}
......@@ -271,7 +271,7 @@ recipe = slapos.cookbook:mkdirectory
apache-conf = ${:etc}/apache
bin = ${buildout:directory}/bin
etc = ${buildout:directory}/etc
promise = ${directory:etc}/promise
promise = ${:etc}/promise
services = ${:etc}/run
var = ${buildout:directory}/var
run = ${:var}/run
......@@ -284,39 +284,39 @@ newcerts = ${:ca-dir}/newcerts
crl = ${:ca-dir}/crl
apachedex = ${monitor-directory:private}/apachedex
[monitor-generate-apachedex-report]
[{{ section('monitor-generate-apachedex-report') }}]
recipe = slapos.cookbook:wrapper
wrapper-path = ${monitor-directory:reports}/${:command}
command-line = "{{ parameter_dict['run-apachedex-location'] }}" "{{ parameter_dict['apachedex-location'] }}" "${directory:apachedex}" --default "${apachedex-parameters:default}" --apache-log-list "${apachedex-parameters:apache-log-list}" --base-list "${apachedex-parameters:base-list}" --skip-base-list "${apachedex-parameters:skip-base-list}" --erp5-base-list "${apachedex-parameters:erp5-base-list}"
command-line = "{{ parameter_dict['run-apachedex-location'] }}" "{{ parameter_dict['apachedex-location'] }}" "${directory:apachedex}" ${monitor-publish-parameters:monitor-base-url}/private/apachedex --apache-log-list "${apachedex-parameters:apache-log-list}" --config "${apachedex-parameters:configuration}"
command = apachedex_every_3_hour
[apachedex-parameters]
default_parameter =
# XXX - Sample log file with curent date: apache_access.log-%(date)s.gz
# which will be equivalent to apache_access.log-20150112.gz if the date is 2015-01-12
apache-log-list = ${apache-conf-parameter-dict:access-log}
default = ${monitor-directory:etc}/apdex_default
base-list = ${monitor-directory:etc}/apdex_base_list
skip-base-list = ${monitor-directory:etc}/apdex_skip_base_list
erp5-base-list = ${monitor-directory:etc}/apdex_erp5_base_list
configuration = {{ slapparameter_dict['apachedex-configuration'] }}
promise-threshold = {{ slapparameter_dict['apachedex-promise-threshold'] }}
[{{ section('monitor-promise-apachedex-result') }}]
recipe = slapos.recipe.template:jinja2
template = {{ parameter_dict['apdex-result-check-script'] }}
rendered = ${monitor-directory:promises}/check-apachedex-result
status-file = ${monitor-directory:private}/apachedex.report.json
context =
raw bash {{ parameter_dict['bash'] }}/bin/bash
raw user_threshold ${apachedex-parameters:promise-threshold}
key apdex_file directory:apachedex
key apdex_status_file :status-file
[monitor-instance-parameter]
monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
monitor-httpd-port = {{ next_port() }}
monitor-title = {{ slapparameter_dict['name'] }}
password = {{ slapparameter_dict['monitor-passwd'] }}
instance-configuration =
file apachedex-default ${apachedex-parameters:default}
file apachedex-base-list ${apachedex-parameters:base-list}
file apachedex-skip-base-list ${apachedex-parameters:skip-base-list}
file apachedex-erp5-base-list ${apachedex-parameters:erp5-base-list}
[buildout]
extends =
{{ logrotate_cfg }}
{{ parameter_dict['template-monitor'] }}
parts +=
publish
logrotate-apache
monitor-generate-apachedex-report
{{ part_list | join('\n ') }}
{% import "root_common" as root_common with context %}
{% import "root_common" as root_common with context -%}
{% set frontend_dict = slapparameter_dict.get('frontend', {}) -%}
{% set has_frontend = frontend_dict.get('software-url', '') != '' -%}
{% set site_id = slapparameter_dict.get('site-id', 'erp5') -%}
......@@ -10,6 +10,7 @@
{% set jupyter_zope_family = jupyter_dict.get('zope-family', '') -%}
{% set monitor_base_url_dict = {} -%}
{% set caucase_url = slapparameter_dict.get('caucase', {}).pop('url', '') -%}
{% set monitor_dict = slapparameter_dict.get('monitor', {}) %}
{% set crl_update_period = slapparameter_dict.get('caucase', {}).pop('crl-update-periodicity', 'daily') -%}
[request-common]
<= request-common-base
......@@ -44,7 +45,7 @@ config-name = {{ name }}
{{ request('memcached-persistent', 'kumofs', 'kumofs', {'tcpv4-port': 2000}, {'url': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
{{ request('memcached-volatile', 'kumofs', 'memcached', {'tcpv4-port': 2010, 'ram-storage-size': 64}, {'url': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
{{ request('cloudooo', 'cloudooo', 'cloudooo', {'tcpv4-port': 2020}, {'url': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
{{ request('mariadb', 'mariadb', 'mariadb', {'tcpv4-port': 2099}, {'database-list': True, 'test-database-list': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
{{ request('mariadb', 'mariadb', 'mariadb', {'tcpv4-port': 2099, 'max-slowqueries-threshold': monitor_dict.get('max-slowqueries-threshold', 1000), 'slowest-query-threshold': monitor_dict.get('slowest-query-threshold', '') }, {'database-list': True, 'test-database-list': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
{% if has_posftix -%}
{{ request('smtp', 'postfix', 'smtp', {'tcpv4-port': 2025, 'smtpd-sasl-user': 'erp5@nowhere'}, key_config={'smtpd-sasl-password': 'publish-early:smtpd-sasl-password'}) }}
{%- else %}
......@@ -281,6 +282,9 @@ config-caucase-url = ${request-caucase:connection-http-url}
config-crl-update-periodicity = {{ crl_update_period }}
config-backend-path-dict = {{ dumps(zope_backend_path_dict) }}
config-ssl-authentication-dict = {{ dumps(ssl_authentication_dict) }}
config-apachedex-promise-threshold = {{ dumps(monitor_dict.get('apachedex-promise-threshold', 70)) }}
config-apachedex-configuration = {{ dumps(monitor_dict.get('apachedex-configuration',
'--erp5-base "/erp5(/|$|/\?)" --skip-user-agent Zabbix --error-detail --js-embed --quiet')) }}
[request-frontend-base]
{% if has_frontend -%}
......
#!{{ bash }}
set -e
DIGEST_FILE='{{ slow_query_digest }}/slowquery_digest.txt'
SLOW_QUERY_STATUS_FILE='{{ slow_query_status }}'
DESIRED_MAX_QUERY_THRESHOLD={{ max_queries_threshold }}
DESIRED_SLOW_QUERY_THRESHOLD={{ slowest_queries_threshold }}
# Check if the file is there
if [ ! -s "$DIGEST_FILE" ]; then
# If file doesn't exists create one
# If it is empty check for modification time
if [ ! -f "$DIGEST_FILE" ]; then
touch $DIGEST_FILE
else
MODIFIED_DATE=`stat -c '%Z' $DIGEST_FILE`
CURRENT_DATE=`date +%s`
if [[ `echo "$CURRENT_DATE - $MODIFIED_DATE" | bc` -gt 108000 ]]
then
echo "File modification date is greater than 30 hours"
JSON_CONTENT=`cat $SLOW_QUERY_STATUS_FILE`
MESSAGE=`echo $JSON_CONTENT | python -c 'import json,sys;obj=json.load(sys.stdin);print obj["message"]'`
echo $MESSAGE
exit 2
else
echo "File is empty for now"
fi
fi
else
# Check if the result exists
{
# get the total number of queries ran and the max time
# TODO: improve regex
# TODO: improve the parameters (currently we are using threshold on queries and max execute time)
# # Overall: (.*) total,(?:.*\n){4}# Exec time(?: *\d*m?s){2} *(.*?)m?s
REGEX="# Overall: (.*) total,.*# Exec time *[[:digit:]]*m?s *[[:digit:]]*m?s *([[:digit:]]*)m?s"
FILE_CONTENT=`cat $DIGEST_FILE`
if [[ $FILE_CONTENT =~ $REGEX ]]
then
TOTAL_QUERIES_EXEC=${BASH_REMATCH[1]}
SLOWEST_QUERY_TIME=${BASH_REMATCH[2]}
HAS_K="${TOTAL_QUERIES_EXEC: -1}"
if [[ "$HAS_K" == "k" ]]
then
PRE="${TOTAL_QUERIES_EXEC::-1}"
TOTAL_QUERIES_EXEC=$(echo "scale=4; ${PRE:-0}*1000" | bc)
else
TOTAL_QUERIES_EXEC=${TOTAL_QUERIES_EXEC:-0}
fi
# TODO: support ms
SLOWEST_QUERY_TIME="${SLOWEST_QUERY_TIME:-0}"
if [[ `echo "$TOTAL_QUERIES_EXEC < $DESIRED_MAX_QUERY_THRESHOLD" | bc` -eq 1 && `echo "$SLOWEST_QUERY_TIME < $DESIRED_SLOW_QUERY_THRESHOLD" | bc` -eq 1 ]]
then
echo "Total number of slow queries are: $TOTAL_QUERIES_EXEC"
echo "Time taken by slowest query is: $SLOWEST_QUERY_TIME"
echo "Thanks for keeping it all clean"
else
echo "Ops! One of the two expected parameters did not meet"
echo "Time taken by slowest query is: $SLOWEST_QUERY_TIME s and required maximum is $DESIRED_SLOW_QUERY_THRESHOLD s"
echo "Total slow queries are $TOTAL_QUERIES_EXEC and expected maximum value is $DESIRED_MAX_QUERY_THRESHOLD"
exit 2
fi
else
echo "No threshold found in the result"
fi
} || {
echo "Cannot parse the result"
}
fi
......@@ -282,14 +282,30 @@ context =
recipe = slapos.recipe.template:jinja2
template = {{ parameter_dict['mariadb-slow-query-report-script'] }}
rendered = ${monitor-directory:reports}/${:filename}
output-folder = ${directory:srv}/monitor/
filename = mariadb_slow_query_every_23_hour
mode = 755
context =
raw slow_query_path ${directory:srv}/backup/logrotate/mariadb_slowquery.log
raw pt_query_exec ${binary-wrap-pt-digest:wrapper-path}
raw dash {{ parameter_dict['dash-location'] }}/bin/dash
key output_folder :output-folder
key output_folder monitor-directory:private
[slow-query-digest-parameters]
max_queries_threshold = {{ slapparameter_dict['max-slowqueries-threshold'] }}
slowest_queries_threshold = {{ slapparameter_dict['slowest-query-threshold'] }}
[{{ section('monitor-promise-slowquery-result') }}]
recipe = slapos.recipe.template:jinja2
template = {{ parameter_dict['mariadb-slowquery-check-script'] }}
rendered = ${monitor-directory:promises}/mariadb-slow-queries-result
status-file = ${monitor-directory:private}/mariadb_slow_query.report.json
context =
raw default_threshold 4000
raw bash {{ parameter_dict['bash'] }}/bin/bash
key slow_query_digest monitor-directory:private
key slow_query_status :status-file
key max_queries_threshold slow-query-digest-parameters:max_queries_threshold
key slowest_queries_threshold slow-query-digest-parameters:slowest_queries_threshold
[{{ section('promise') }}]
recipe = slapos.cookbook:wrapper
......
......@@ -93,6 +93,7 @@ openssl-location = {{ openssl_location }}
[dynamic-template-balancer-parameters]
apache = {{ apache_location }}
apdex-result-check-script = {{ apdex_result_check_script }}
openssl = {{ openssl_location }}
haproxy = {{ haproxy_location }}
bin-directory = {{ bin_directory }}
......@@ -101,6 +102,7 @@ run-apachedex-location = {{ bin_directory }}/runApacheDex
6tunnel = {{ sixtunnel_location }}
curl-location = {{ curl_location }}
dash = {{ dash_location }}
bash = {{ bash_location }}
template-haproxy-cfg = {{ template_haproxy_cfg }}
template-apache-conf = {{ template_apache_conf }}
template-monitor = {{ dumps(template_monitor) }}
......@@ -169,6 +171,7 @@ extra-context =
section parameter_dict dynamic-template-kumofs-parameters
[dynamic-template-mariadb-parameters]
bash = {{ bash_location }}
coreutils-location = {{ coreutils_location }}
dash-location = {{ dash_location }}
findutils-location = {{ findutils_location }}
......@@ -180,6 +183,7 @@ link-binary = {{ dumps(mariadb_link_binary) }}
bin-directory = {{ bin_directory }}
mariadb-resiliency-after-import-script = {{ mariadb_resiliency_after_import_script }}
mariadb-slow-query-report-script = {{ mariadb_slow_query_report_script }}
mariadb-slowquery-check-script = {{ mariadb_slowquery_check_script}}
percona-tools-location = {{ percona_toolkit_location }}
template-monitor = {{ template_monitor }}
......
......@@ -10,6 +10,8 @@ if [ ! -d "$OUTPUT_FOLDER" ]; then
exit 0
fi
OUTPUT_FILE=${OUTPUT_FOLDER}/slowquery_digest.txt
TODAY=`date +%Y%m%d`
SLOW_LOG=$SLOW_QUERY_PATH-$TODAY
......@@ -19,4 +21,5 @@ if [ ! -f "$SLOW_LOG" ]; then
exit 1
fi
eval $PT_QUERY_EXEC $SLOW_LOG
$PT_QUERY_EXEC $SLOW_LOG > $OUTPUT_FILE
echo "ok"
......@@ -131,5 +131,5 @@ depends =
PyRSS2Gen = 1.1
cns.recipe.symlink = 0.2.3
pycurl = 7.43.0
slapos.toolbox = 0.69
slapos.toolbox = 0.70
......@@ -182,7 +182,7 @@ pycparser = 2.17
# Required by:
# slapos.core==1.3.18
supervisor = 3.3.1
supervisor = 3.3.3
# Required by:
# slapos.core==1.3.18
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment