Skip to content

S
slapos
Commit c4ed790d authored by Kirill Smelkov's avatar Kirill Smelkov
Browse files
Options

Merge remote-tracking branch 'origin/master' into x/gitlab 

* origin/master:
  version up : slapos.core 1.3.11, slapos.libnetworkcache 0.14.5 and zc.buildout 1.7.1.post12.
  version up : slapos.libnetworkcache 0.14.4.
  Monitor: Implement log split and upload to wendelin on fluentd
  monitor: Include Access-Control-Expose-Headers to make improve crawling
  fixup! erp5: make mariadb monitor friendly
  erp5: make mariadb monitor friendly
  ERP5: fix support for NEO
  *: lab.nexedi.cn -> lab.nexedi.com
parents df146701 5f2822ef
Show whitespace changes
Inline Side-by-side
Showing with 194 additions and 56 deletions
component/babeld/buildout.cfg
View file @ c4ed790d
......@@ -6,7 +6,7 @@ parts = babeld
[babeld]
recipe = slapos.recipe.cmmi
url = https://lab.nexedi.cn/rafael/babeld/repository/archive.tar.gz?ref=v1.6.0-nxd1
url = https://lab.nexedi.com/rafael/babeld/repository/archive.tar.gz?ref=v1.6.0-nxd1
md5sum = 1f269d01321103873b8d245df19984f0
configure-command =
echo "No configure.."
......
component/wendelin.core/buildout.cfg
View file @ c4ed790d
......@@ -24,7 +24,7 @@ PATH = ${git:location}/bin:%(PATH)s
[wendelin.core-repository]
recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.cn/nexedi/wendelin.core.git
repository = https://lab.nexedi.com/nexedi/wendelin.core.git
# dir is pretty name as top-level -dev recipe
location = ${buildout:parts-directory}/wendelin.core-dev
git-executable = ${git:location}/bin/git
......
software/kvm/development.cfg
View file @ c4ed790d
......@@ -24,7 +24,7 @@ git-executable = ${git:location}/bin/git
[slapos.core-repository]
recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.cn/nexedi/slapos.core.git
repository = https://lab.nexedi.com/nexedi/slapos.core.git
branch = master
git-executable = ${git:location}/bin/git
......
software/monitor/cgi-httpd.conf.in
View file @ c4ed790d
......@@ -34,7 +34,7 @@ Header always set Access-Control-Allow-Origin %{AccessControlAllowOrigin}e env=A
Header set Access-Control-Allow-Credentials "true"
Header set Access-Control-Allow-Methods "PROPFIND, PROPPATCH, COPY, MOVE, DELETE, MKCOL, LOCK, UNLOCK, PUT, GETLIB, VERSION-CONTROL, CHECKIN, CHECKOUT, UNCHECKOUT, REPORT, UPDATE, CANCELUPLOAD, HEAD, OPTIONS, GET, POST"
Header set Access-Control-Allow-Headers "Overwrite, Destination, Content-Type, Depth, User-Agent, X-File-Size, X-Requested-With, If-Modified-Since, X-File-Name, Cache-Control, Authorization"
Header set Access-Control-Expose-Headers "Content-Disposition, Content-Type, Content-Length, WWW-Authenticate, WWW-Logout, Location, Date, Last-Modified, ETag, Accept-Ranges, Content-Range"
# SSL Configuration
......
software/monitor/fluentd-agent.conf.jinja2.in
View file @ c4ed790d
......@@ -18,15 +18,93 @@
type stdout
</match>
<source>
type tail
path {{ crawl_log_directory }}/*/*ping.log.20*
pos_file {{ crawl_log_directory }}/tail_in_ping.pos
tag slapos.monitor.networktest.ping.ipv4
format /^(?<time>[^;]*);(?<logtype>[^;]*);(?<computer_name>[^;]*);(?<type>[^;]*);(?<name_or_ip>[^;]*);(?<code>[^;]*);(?<average>[^;]*);(?<packet_lost>[^;]*);(?<extra>[^;]*)$/
read_from_head true
</source>
## SOURCE
<source>
type forward
type tail
path {{ crawl_log_directory }}/*/*ping6.log.20*
pos_file {{ crawl_log_directory }}/tail_in_ping6.pos
tag slapos.monitor.networktest.ping.ipv6
format /^(?<time>[^;]*);(?<logtype>[^;]*);(?<computer_name>[^;]*);(?<type>[^;]*);(?<name_or_ip>[^;]*);(?<code>[^;]*);(?<average>[^;]*);(?<packet_lost>[^;]*);(?<extra>[^;]*)$/
read_from_head true
</source>
<match slapos.monitor.networktest.ping.*>
type copy
{% for slave_instance in slave_instance_list -%}
{% if slave_instance.get("ping_ip_list") -%}
<store>
type grep
regexp1 name_or_ip ^{{ slave_instance.get("ping_ip_list") }}$
add_tag_prefix {{ slave_instance.get("slave_reference") }}
</store>
{% endif -%}
{% if slave_instance.get("ping6_ip_list") -%}
<store>
type grep
regexp1 name_or_ip ^{{ slave_instance.get("ping6_ip_list") }}$
add_tag_prefix {{ slave_instance.get("slave_reference") }}
</store>
{% endif -%}
{% endfor -%}
</match>
{% for slave_instance in slave_instance_list -%}
<match {{ slave_instance.get("slave_reference") }}.slapos.monitor.networktest.ping.ipv6>
type file
path {{ network_user_logs }}/{{ slave_instance.get("slave_reference") }}/ping6/log
append true
include_time_key true
include_tag_key true
format json
</match>
<match {{ slave_instance.get("slave_reference") }}.slapos.monitor.networktest.ping.ipv4>
type file
path {{ network_user_logs }}/{{ slave_instance.get("slave_reference") }}/ping/log
append true
include_time_key true
include_tag_key true
format json
</match>
{% endfor -%}
<source>
type http
port 8888
type tail
path {{ network_user_logs }}/*/ping*/log/*.log
pos_file {{ crawl_log_directory }}/tail_in_ping6.pos
tag slapos.wendelin.networktest
format json
read_from_head true
</source>
{% set wendelin_streamtool_uri = slapparameter_dict.get('wendelin-streamtool-uri', '') -%}
{% set wendelin_password = slapparameter_dict.get('wendelin-password', '') -%}
{% set wendelin_user = slapparameter_dict.get('wendelin-user', '') -%}
{% if wendelin_streamtool_uri and wendelin_password and wendelin_user -%}
<match slapos.wendelin.networktest>
@type wendelin
@id wendelin_out
streamtool_uri {{ wendelin_streamtool_uri }}
user {{ wendelin_user }}
password {{ wendelin_password }}
buffer_type memory
flush_interval 20s
</match>
{% endif -%}
## SOURCE
<source>
type forward
</source>
## live debugging agent
......
software/monitor/instance-monitor-distributor.cfg.jinja2
View file @ c4ed790d
[buildout]
extends = {{ instance_base_monitor }}
parts +=
slave-test-configuration
[jinja2-template-base]
recipe = slapos.recipe.template:jinja2
......@@ -20,16 +15,11 @@ extensions = jinja2.ext.do
extra-context =
section slave_information slap-parameter
[monitor-directory]
fluentd-log = ${:log}/fluentd
network-user-logs = ${:private-directory}/network-user-logs/
{% for slave_instance in slave_instance_list -%}
user-log-{{ slave_instance.get('slave_reference') }}-folder = ${:private-directory}/network-user-logs/{{ slave_instance.get('slave_reference') }}
{% endfor -%}
{% set part_list = [] -%}
# Publish information for each slave
{% set directory_list = [] -%}
{% for slave_instance in slave_instance_list -%}
{% set publish_section_title = 'publish-%s' % slave_instance.get('slave_reference') -%}
{% do part_list.append(publish_section_title) -%}
......@@ -40,8 +30,50 @@ log-access-url = ${monitor-frontend:connection-site_url}/{{ slave_instance.get('
log-access-url-v6 = ${monitor-parameters:url}/{{ slave_instance.get('slave_reference') }}
{% endfor %}
{% set data_source_dict = slapparameter_dict.get('data-source', None) -%}
{% set cron_min_count = 0 -%}
{% if data_source_dict %}
{% for entry in data_source_dict -%}
{% set cron_min_count = cron_min_count + 1 -%}
{% set cron_min = cron_min_count%60 -%}
{% do part_list.append('cron-crawl-' + entry) -%}
{% do directory_list.append(entry) -%}
[cron-crawl-{{ entry }}]
<= cron
recipe = slapos.cookbook:cron.d
name = cron-crawler-{{ entry }}
frequency = * * * * *
command = cd ${monitor-directory:crawl-log}/{{ entry }} && ${crawler-bin:wrapper-path} {{ data_source_dict.get(entry) }}
{% endfor %}
{% endif %}
[monitor-directory]
fluentd-log = ${:log}/fluentd
crawl-log = ${:srv}/crawlog
network-user-logs = ${:private-directory}/network-user-logs/
{% for slave_instance in slave_instance_list -%}
user-log-{{ slave_instance.get('slave_reference') }}-folder = ${:private-directory}/network-user-logs/{{ slave_instance.get('slave_reference') }}
user-log-{{ slave_instance.get('slave_reference') }}-ping-folder = ${:private-directory}/network-user-logs/{{ slave_instance.get('slave_reference') }}/ping
user-log-{{ slave_instance.get('slave_reference') }}-ping6-folder = ${:private-directory}/network-user-logs/{{ slave_instance.get('slave_reference') }}/ping6
{% endfor -%}
{% for directory in directory_list %}
{{ '%s = ${:crawl-log}/%s' % (directory, directory) }}
{% endfor %}
[crawler-bin]
recipe = slapos.cookbook:wrapper
command-line =
{{ wget_bin }} --no-check-certificate -l1 -r -nd --timestamp
wrapper-path = ${monitor-directory:bin}/log-crawler
parameters-extra = true
[buildout]
extends = {{ instance_base_monitor }}
parts +=
slave-test-configuration
fluentd-wrapper
{% for part in part_list %}
{{ ' %s' % part }}
......@@ -60,5 +92,17 @@ template = {{ fluent_conf_output }}
rendered = ${monitor-directory:etc}/fluentd.cfg
mode = 0744
context =
key slapparameter_dict slap-parameters:configuration
key slapparameter_dict slap-configuration:configuration
key slave_instance_list slap-parameters:slave-instance-list
key fluentd_log_directory monitor-directory:fluentd-log
key crawl_log_directory monitor-directory:crawl-log
key network_user_logs monitor-directory:network-user-logs
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
software/monitor/instance.cfg
View file @ c4ed790d
......@@ -19,8 +19,8 @@ context = key buildout buildout:bin-directory
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
key slapparameter_dict slap-configuration:configuration
key monitor_template_output slap-configuration:monitor_template_output
key network_benck_cfg_output slap-configuration:network_benck_cfg_output
raw monitor_template_output ${monitor-template:output}
raw network_benck_cfg_output ${network-bench-cfg:output}
mode = 0644
[instance-base-distributor]
......@@ -35,20 +35,16 @@ context = import json_module json
key slapparameter_dict slap-configuration:configuration
key instance_base_monitor instance-base-monitor:rendered
key slave_instance_list slap-configuration:slave-instance-list
key template_json_distributor_test slap-configuration:json_test_template
key fluent_conf_output slap-configuration:fluentd_conf_output
raw template_json_distributor_test ${json-test-template:destination}
raw fluent_conf_output ${fluentd-agent-conf:output}
raw fluentd_location ${fluentd:location}
raw wget_bin ${wget:location}/bin/wget
mode = 0644
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration
recipe = slapos.cookbook:slapconfiguration.serialised
computer = $${slap-connection:computer-id}
partition = $${slap-connection:partition-id}
url = $${slap-connection:server-url}
key = $${slap-connection:key-file}
cert = $${slap-connection:cert-file}
monitor_template_output = ${monitor-template:output}
network_benck_cfg_output = ${network-bench-cfg:output}
json_test_template = ${json-test-template:destination}
fluentd_conf_output = ${fluentd-agent-conf:output}
software/monitor/software.cfg
View file @ c4ed790d
......@@ -2,11 +2,13 @@
extends =
../../component/pycurl/buildout.cfg
../../component/wget/buildout.cfg
../../stack/monitor/buildout.cfg
../../component/fluentd/buildout.cfg
../../stack/slapos.cfg
parts =
wget
slapos-cookbook
slapos-toolbox
network-bench-cfg
......@@ -22,7 +24,7 @@ parts =
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg
output = ${buildout:directory}/template.cfg
md5sum = d3a2cae2823f5f5ce2f7c293018223f3
md5sum = b38c5da7dd4165154a8817f76015e374
mode = 0644
[template-monitor]
......@@ -36,7 +38,7 @@ mode = 0644
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/instance-monitor-distributor.cfg.jinja2
destination = ${buildout:directory}/template-monitor-distributor.cfg
md5sum = 2f11edbc57eb95f9d5901d183065d166
md5sum = d1253cf11c2b335b8bec5807e821d537
mode = 0644
[json-test-template]
......@@ -54,7 +56,7 @@ md5sum = 876f18b159fbd9325332d0f42e9172ac
[monitor-httpd-template]
url = ${:_profile_base_location_}/${:filename}
md5sum = 25b5bfc7f9c2891c4a0923d6e08568a8
md5sum = 22133e9aa5f52d7818b2fd1fd0415a00
[network-bench-cfg]
recipe = slapos.recipe.template
......@@ -63,11 +65,10 @@ md5sum = cfcbf2002b8eff5153e2bf68ed24b720
output = ${buildout:directory}/template-network-bench-cfg.in
mode = 0644
[fluentd-agent-conf]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/fluentd-agent.conf.jinja2.in
md5sum = 0098143de65630f78be402c65424beb4
md5sum = c9da9f22a1cd1fedcf7d659cba916400
output = ${buildout:directory}/fluentd-agent.conf.jinja2.in
mode = 0644
......@@ -85,6 +86,7 @@ scripts =
[fluentd]
gems +=
fluent-plugin-wendelin==0.1.alpha1
fluent-plugin-grep==0.3.4
[eggs]
recipe = z3c.recipe.scripts
......@@ -135,4 +137,3 @@ lockfile = 0.10.2
# slapos.toolbox==0.52
rpdb = 0.1.5
software/slapos-master/software.cfg
View file @ c4ed790d
......@@ -29,7 +29,7 @@ git-executable = ${git:location}/bin/git
[vifib]
recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.cn/nexedi/slapos.core.git
repository = https://lab.nexedi.com/nexedi/slapos.core.git
branch = master
git-executable = ${git:location}/bin/git
......
software/slapos-testing/software.cfg
View file @ c4ed790d
......@@ -50,7 +50,7 @@ repository = http://git.erp5.org/repos/slapos.git
[slapos.core-repository]
<= git-clone-repository
repository = https://lab.nexedi.cn/nexedi/slapos.core.git
repository = https://lab.nexedi.com/nexedi/slapos.core.git
[slapos.package-repository]
<= git-clone-repository
......
software/slaprunner/development.cfg
View file @ c4ed790d
......@@ -16,7 +16,7 @@ parts =
[slapos.toolbox-repository]
recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.cn/nexedi/slapos.toolbox.git
repository = https://lab.nexedi.com/nexedi/slapos.toolbox.git
branch = slaprunner-paas
git-executable = ${git:location}/bin/git
develop = true
......@@ -53,7 +53,7 @@ setup = ${erp5.util-repository:location}
[slapos.core-repository]
recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.cn/nexedi/slapos.core.git
repository = https://lab.nexedi.com/nexedi/slapos.core.git
branch = master
git-executable = ${git:location}/bin/git
develop = true
......
software/unstable/lamp-generic/development.cfg
View file @ c4ed790d
......@@ -34,7 +34,7 @@ git-executable = ${git:location}/bin/git
[slapos.core-repository]
recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.cn/nexedi/slapos.core.git
repository = https://lab.nexedi.com/nexedi/slapos.core.git
branch = master
git-executable = ${git:location}/bin/git
......
software/unstable/maarch/development.cfg
View file @ c4ed790d
......@@ -40,7 +40,7 @@ git-executable = ${git:location}/bin/git
[slapos.core-repository]
recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.cn/nexedi/slapos.core.git
repository = https://lab.nexedi.com/nexedi/slapos.core.git
branch = master
git-executable = ${git:location}/bin/git
......
software/unstable/tt-rss/development.cfg
View file @ c4ed790d
......@@ -37,7 +37,7 @@ git-executable = ${git:location}/bin/git
[slapos.core-repository]
recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.cn/nexedi/slapos.core.git
repository = https://lab.nexedi.com/nexedi/slapos.core.git
branch = master
git-executable = ${git:location}/bin/git
......
software/unstable/vifib/software.cfg
View file @ c4ed790d
......@@ -37,7 +37,7 @@ branch = interaction-drop
[vifib]
<= erp5
branch = master
repository = https://lab.nexedi.cn/nexedi/slapos.core.git
repository = https://lab.nexedi.com/nexedi/slapos.core.git
revision =
[local-bt5-repository]
......
software/wendelin/software.cfg
View file @ c4ed790d
......@@ -47,7 +47,7 @@ bt5_list = erp5_full_text_myisam_catalog erp5_configurator_standard erp5_configu
[wendelin]
<= erp5
repository = https://lab.nexedi.cn/nexedi/wendelin.git
repository = https://lab.nexedi.com/nexedi/wendelin.git
branch = master
[versions]
......
software/wordpress/development.cfg
View file @ c4ed790d
......@@ -38,7 +38,7 @@ git-executable = ${git:location}/bin/git
[slapos.core-repository]
recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.cn/nexedi/slapos.core.git
repository = https://lab.nexedi.com/nexedi/slapos.core.git
branch = master
git-executable = ${git:location}/bin/git
......
stack/erp5/buildout.cfg
View file @ c4ed790d
......@@ -52,8 +52,9 @@ extends =
../../component/6tunnel/buildout.cfg
../../component/findutils/buildout.cfg
../../component/userhosts/buildout.cfg
../../software/neoppod/software-common.cfg
../../component/postfix/buildout.cfg
../../software/neoppod/software-common.cfg
# keep neoppod extends last
parts +=
erp5-util-develop
......@@ -146,7 +147,7 @@ mode = 755
[template-mariadb]
<= download-base
filename = instance-mariadb.cfg.in
md5sum = 4d46ab5aa651a39c4021de6f647e8be9
md5sum = fbc39d333bf70894f6f9d094515a2a4a
link-binary =
${coreutils:location}/bin/basename
${coreutils:location}/bin/cat
......@@ -224,7 +225,7 @@ recipe = slapos.recipe.template:jinja2
# XXX: "template.cfg" is hardcoded in instanciation recipe
rendered = ${buildout:directory}/template.cfg
template = ${:_profile_base_location_}/instance.cfg.in
md5sum = 4d043c96d70b35d1fbbd8120d8edee7c
md5sum = 708a721558f9bbe8856482e60891b4c7
mode = 640
context =
key mariadb_link_binary template-mariadb:link-binary
......@@ -290,6 +291,7 @@ context =
key template_kumofs template-kumofs:target
key template_mariadb template-mariadb:target
key template_mariadb_initial_setup template-mariadb-initial-setup:target
key template_monitor monitor-template:rendered
key template_my_cnf template-my-cnf:target
key template_postfix template-postfix:target
key template_postfix_aliases template-postfix-aliases:target
......@@ -305,6 +307,15 @@ context =
key xfixes_location xfixes:location
key zlib_location zlib:location
[monitor-template-dummy]
<= download-base
# This is a placeholder, to be overriden by extending monitor SR
filename = dummy.cfg
md5sum = d41d8cd98f00b204e9800998ecf8427e
[monitor-template]
rendered = ${monitor-template-dummy:target}
[template-erp5]
<= download-base
filename = instance-erp5.cfg.in
......
stack/erp5/instance-mariadb.cfg.in
View file @ c4ed790d
......@@ -17,7 +17,7 @@
{% set ip = (ipv4_set | list)[0] -%}
{% endif -%}
[publish-mariadb-url]
[publish]
recipe = slapos.cookbook:publish.serialised
{% macro render_database_list(database_list) -%}
{% set publish_database_list = [] -%}
......@@ -232,10 +232,17 @@ command-line = "{{ parameter_dict['bin-directory'] }}/is-local-tcp-port-opened"
wrapper-path = ${directory:promise}/mariadb
parameters-extra = true
[monitor-instance-parameter]
monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
monitor-httpd-port = {{ port + 1 }}
monitor-title = Mariadb monitor
[buildout]
extends = {{ logrotate_cfg }}
extends =
{{ logrotate_cfg }}
{{ parameter_dict['template-monitor'] }}
parts +=
publish-mariadb-url
publish
logrotate-entry-mariadb
binary-link
update-mysql
......
stack/erp5/instance.cfg.in
View file @ c4ed790d
......@@ -150,6 +150,7 @@ template-mariadb-initial-setup = {{ template_mariadb_initial_setup }}
link-binary = {{ dumps(mariadb_link_binary) }}
bin-directory = {{ bin_directory }}
mariadb-resiliency-after-import-script = {{ mariadb_resiliency_after_import_script }}
template-monitor = {{ template_monitor }}
[dynamic-template-mariadb]
<= jinja2-template-base
......
stack/slapos.cfg
View file @ c4ed790d
......@@ -107,7 +107,7 @@ eggs =
[versions]
# Use SlapOS patched zc.buildout
zc.buildout = 1.7.1.post11
zc.buildout = 1.7.1.post12
# Use SlapOS patched zc.recipe.egg (zc.recipe.egg 2.x is for Buildout 2)
zc.recipe.egg = 1.3.2.post4
# Use own version of h.r.download to be able to open .xz and .lz archives
......@@ -139,9 +139,9 @@ setuptools = 18.1
simplejson = 3.8.0
six = 1.9.0
slapos.cookbook = 1.0.9
slapos.core = 1.3.10
slapos.core = 1.3.11
slapos.extension.strip = 0.1
slapos.libnetworkcache = 0.14.3
slapos.libnetworkcache = 0.14.5
slapos.recipe.build = 0.21
slapos.recipe.cmmi = 0.2
stevedore = 1.8.0
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7