Skip to content

slapos
slapos
Commit 80825610 authored by Rafael Monnerat's avatar Rafael Monnerat
Browse files
Options

software/monitor: Refactor Monitor software release 

  Follow up changes on monitor stack refectoring.
  Include a standard fluend to upload collected data to wendelin.
parent e0dbf25a
Hide whitespace changes
Inline Side-by-side
Showing with 427 additions and 323 deletions
software/monitor/cgi-httpd.conf.in deleted 100644 → 0
View file @ e0dbf25a
PidFile "{{ httpd_configuration.get('pid-file') }}"
ServerName example.com
ServerAdmin someone@email
<IfDefine !MonitorPort>
Listen [{{ httpd_configuration.get('listening-ip') }}]:{{ monitor_parameters.get('port') }}
Define MonitorPort
</IfDefine>
DocumentRoot "{{ directory.get('www') }}"
ErrorLog "{{ httpd_configuration.get('error-log') }}"
LoadModule unixd_module modules/mod_unixd.so
LoadModule access_compat_module modules/mod_access_compat.so
LoadModule authz_core_module modules/mod_authz_core.so
LoadModule authn_core_module modules/mod_authn_core.so
LoadModule authz_host_module modules/mod_authz_host.so
LoadModule mime_module modules/mod_mime.so
LoadModule cgid_module modules/mod_cgid.so
LoadModule dir_module modules/mod_dir.so
LoadModule ssl_module modules/mod_ssl.so
LoadModule alias_module modules/mod_alias.so
LoadModule autoindex_module modules/mod_autoindex.so
LoadModule auth_basic_module modules/mod_auth_basic.so
LoadModule authz_user_module modules/mod_authz_user.so
LoadModule authn_file_module modules/mod_authn_file.so
LoadModule proxy_module modules/mod_proxy.so
LoadModule proxy_http_module modules/mod_proxy_http.so
LoadModule rewrite_module modules/mod_rewrite.so
LoadModule setenvif_module modules/mod_setenvif.so
LoadModule headers_module modules/mod_headers.so
LoadModule env_module modules/mod_env.so
LoadModule socache_shmcb_module modules/mod_socache_shmcb.so
SetEnvIf Origin "http(s)?://(www\.)?(.*)$" AccessControlAllowOrigin=$0
Header always set Access-Control-Allow-Origin %{AccessControlAllowOrigin}e env=AccessControlAllowOrigin
Header set Access-Control-Allow-Credentials "true"
Header set Access-Control-Allow-Methods "PROPFIND, PROPPATCH, COPY, MOVE, DELETE, MKCOL, LOCK, UNLOCK, PUT, GETLIB, VERSION-CONTROL, CHECKIN, CHECKOUT, UNCHECKOUT, REPORT, UPDATE, CANCELUPLOAD, HEAD, OPTIONS, GET, POST"
Header set Access-Control-Allow-Headers "Overwrite, Destination, Content-Type, Depth, User-Agent, X-File-Size, X-Requested-With, If-Modified-Since, X-File-Name, Cache-Control, Authorization"
Header set Access-Control-Expose-Headers "Content-Disposition, Content-Type, Content-Length, WWW-Authenticate, WWW-Logout, Location, Date, Last-Modified, ETag, Accept-Ranges, Content-Range"
# SSL Configuration
<IfDefine !SSLConfigured>
Define SSLConfigured
SSLCertificateFile {{ httpd_configuration.get('certificate') }}
SSLCertificateKeyFile {{ httpd_configuration.get('key') }}
SSLRandomSeed startup builtin
SSLRandomSeed connect builtin
SSLRandomSeed startup /dev/urandom 256
SSLRandomSeed connect builtin
SSLProtocol all -SSLv2 -SSLv3
SSLCipherSuite ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:HIGH:!aNULL:!MD5
SSLHonorCipherOrder on
SSLSessionCache shmcb:/{{ directory.get("mod-ssl") }}/ssl_scache(512000)
SSLSessionCacheTimeout 300
</IfDefine>
SSLEngine On
ScriptSock {{ httpd_configuration.get('cgid-pid-file') }}
<Directory {{ directory.get('www') }}>
SSLVerifyDepth 1
SSLRequireSSL
SSLOptions +StrictRequire
# XXX: security????
Options +ExecCGI
AddHandler cgi-script .cgi
DirectoryIndex {{ monitor_parameters.get('index-filename') }}
</Directory>
Alias /{{ monitor_private_hash }} {{ directory.get('private-directory') }}/
<Directory {{ directory.get('private-directory') }}>
Order Allow,Deny
Allow from all
AllowOverride All
Satisfy Any
Options Indexes FollowSymLinks
DirectoryIndex index.html
<Files .htaccess>
order allow,deny
deny from all
</Files>
</Directory>
{% set slave_list = json_module.loads(slave_information.get('slave_instance_list', '{}')) -%}
{% for slave_instance in slave_list -%}
alias /{{ slave_instance.get('slave_reference') }} {{ directory.get('private-directory') }}/network-user-logs/{{ slave_instance.get('slave_reference') }}
<Directory {{ directory.get('private-directory') }}/network-user-logs/{{ slave_instance.get('slave_reference') }}>
Order Allow,Deny
Allow from all
AllowOverride All
Satisfy Any
Options Indexes FollowSymLinks
DirectoryIndex index.html
<Files .htaccess>
order allow,deny
deny from all
</Files>
</Directory>
{% endfor -%}
<Location /rewrite>
AuthType Basic
AuthName "Private access"
AuthUserFile "{{ monitor_parameters.get('htaccess-file') }}"
Require valid-user
</Location>
ProxyVia On
RewriteEngine On
{% for key, value in monitor_rewrite_rule.iteritems() %}
RewriteRule ^/rewrite/{{ key }}($|/.*) {{ value }}/$1 [P,L]
{% endfor %}
software/monitor/fluentd-agent.conf.jinja2.in
View file @ 80825610
...@@ -13,104 +13,135 @@ ...@@ -13,104 +13,135 @@
</secondary> </secondary>
</match> </match>
## match tag=debug.** and dump to console {% set wendelin_streamtool_uri = slapparameter_dict.get('wendelin-streamtool-uri', '') -%}
<match debug.**> {% set wendelin_password = slapparameter_dict.get('wendelin-password', '') -%}
type stdout {% set wendelin_user = slapparameter_dict.get('wendelin-user', '') -%}
{% set wendelin_uid = slapparameter_dict.get('wendelin-uid', 'UNKNOWN') -%}
{% if wendelin_streamtool_uri and wendelin_password and wendelin_user -%}
<source>
type syslog
port 42185
bind 127.0.0.1
tag {{ wendelin_uid }}.system
</source>
<match {{ wendelin_uid }}.system.**>
@type wendelin
@id wendelin_out
streamtool_uri {{ wendelin_streamtool_uri }}_syslog
user {{ wendelin_user }}
password {{ wendelin_password }}
buffer_type memory
flush_interval 20s
</match> </match>
<source> <source>
type tail type tail
path {{ crawl_log_directory }}/*/*ping.log.20* path {{ fluentd_log_directory }}/consumption/*/dump_user.csv
pos_file {{ crawl_log_directory }}/tail_in_ping.pos pos_file {{ fluentd_log_directory }}/consumption/dump_user.pos
tag slapos.monitor.networktest.ping.ipv4 tag slapos.monitor.usage.user.{{ wendelin_uid }}
format /^(?<time>[^;]*);(?<computer_name>[^;]*);(?<type>[^;]*);(?<name_or_ip>[^;]*);(?<code>[^;]*);(?<average>[^;]*);(?<packet_lost>[^;]*);(?<extra>[^;]*)$/ format /^(?<partition>[^,]*),(?<pid>[^,]*),(?<process>[^,]*),(?<cpu_percent>[^,]*),(?<cpu_time>[^,]*),(?<cpu_num_threads>[^,]*),(?<memory_percent>[^,]*),(?<memory_rss>[^,]*),(?<io_rw_counter>[^,]*),(?<io_cycles_counter>[^,]*),(?<date>[^,]*),(?<time>[^,]*),(?<reported>[^,]*)$/
read_from_head true read_from_head true
</source> </source>
<match slapos.monitor.usage.user.{{ wendelin_uid }}>
@type wendelin
@id wendelin_out
streamtool_uri {{ wendelin_streamtool_uri }}_user
user {{ wendelin_user }}
password {{ wendelin_password }}
buffer_type memory
flush_interval 20s
</match>
<source> <source>
type tail type tail
path {{ crawl_log_directory }}/*/*ping6.log.20* path {{ fluentd_log_directory }}/consumption/*/dump_system.csv
pos_file {{ crawl_log_directory }}/tail_in_ping6.pos pos_file {{ fluentd_log_directory }}/consumption/dump_system.pos
tag slapos.monitor.networktest.ping.ipv6 tag slapos.monitor.usage.system.{{ wendelin_uid }}
format /^(?<time>[^;]*);(?<computer_name>[^;]*);(?<type>[^;]*);(?<name_or_ip>[^;]*);(?<code>[^;]*);(?<average>[^;]*);(?<packet_lost>[^;]*);(?<extra>[^;]*)$/ format /^(?<loadavg>[^,]*),(?<cpu_percent>[^,]*),(?<memory_used>[^,]*),(?<memory_free>[^,]*),(?<net_in_bytes>[^,]*),(?<net_in_errors>[^,]*),(?<net_in_dropped>[^,]*),(?<net_out_bytes>[^,]*),(?<net_out_errors>[^,]*),(?<net_out_dropped>[^,]*),(?<date>[^,]*),(?<time>[^,]*),(?<reported>[^,]*)$/
read_from_head true read_from_head true
</source> </source>
<match slapos.monitor.networktest.ping.*> <match slapos.monitor.usage.system.{{ wendelin_uid }}>
type copy @type wendelin
{% for slave_instance in slave_instance_list -%} @id wendelin_out
{% if slave_instance.get("ping_ip_list") -%}
<store>
type grep
regexp1 name_or_ip ^{{ slave_instance.get("ping_ip_list") }}$
add_tag_prefix {{ slave_instance.get("slave_reference") }}
</store>
{% endif -%}
{% if slave_instance.get("ping6_ip_list") -%}
<store>
type grep
regexp1 name_or_ip ^{{ slave_instance.get("ping6_ip_list") }}$
add_tag_prefix {{ slave_instance.get("slave_reference") }}
</store>
{% endif -%}
{% endfor -%}
</match>
{% for slave_instance in slave_instance_list -%} streamtool_uri {{ wendelin_streamtool_uri }}_system
<match {{ slave_instance.get("slave_reference") }}.slapos.monitor.networktest.ping.ipv6> user {{ wendelin_user }}
type file password {{ wendelin_password }}
path {{ network_user_logs }}/{{ slave_instance.get("slave_reference") }}/ping6/log
append true buffer_type memory
include_time_key true flush_interval 20s
include_tag_key true
format json
</match>
<match {{ slave_instance.get("slave_reference") }}.slapos.monitor.networktest.ping.ipv4>
type file
path {{ network_user_logs }}/{{ slave_instance.get("slave_reference") }}/ping/log
append true
include_time_key true
include_tag_key true
format json
</match> </match>
{% endfor -%}
<source> <source>
type tail type tail
path {{ network_user_logs }}/*/ping*/log/*.log path {{ fluentd_log_directory }}/consumption/*/dump_computer.csv
pos_file {{ crawl_log_directory }}/tail_in_ping6.pos pos_file {{ fluentd_log_directory }}/consumption/dump_computer.pos
tag slapos.wendelin.networktest tag slapos.monitor.usage.computer.{{ wendelin_uid }}
format json format /^(?<cpu_num_core>[^,]*),(?<cpu_frequency>[^,]*),(?<cpu_type>[^,]*),(?<memory_size>[^,]*),(?<memory_type>[^,]*),(?<partition_list>[^,]*),(?<date>[^,]*),(?<time>[^,]*),(?<reported>[^,]*)$/
read_from_head true read_from_head true
</source> </source>
{% set wendelin_streamtool_uri = slapparameter_dict.get('wendelin-streamtool-uri', '') -%} <match slapos.monitor.usage.computer.{{ wendelin_uid }}>
{% set wendelin_password = slapparameter_dict.get('wendelin-password', '') -%}
{% set wendelin_user = slapparameter_dict.get('wendelin-user', '') -%}
{% if wendelin_streamtool_uri and wendelin_password and wendelin_user -%}
<match slapos.wendelin.networktest>
@type wendelin @type wendelin
@id wendelin_out @id wendelin_out
streamtool_uri {{ wendelin_streamtool_uri }} streamtool_uri {{ wendelin_streamtool_uri }}_computer
user {{ wendelin_user }} user {{ wendelin_user }}
password {{ wendelin_password }} password {{ wendelin_password }}
buffer_type memory buffer_type memory
flush_interval 20s flush_interval 20s
</match> </match>
{% endif -%}
## SOURCE
<source> <source>
type forward type tail
path {{ fluentd_log_directory }}/consumption/*/dump_disk.csv
pos_file {{ fluentd_log_directory }}/consumption/dump_disk.pos
tag slapos.monitor.usage.disk.{{ wendelin_uid }}
format /^(?<partition>[^,]*),(?<used>[^,]*),(?<free>[^,]*),(?<mountpoint>[^,]*),(?<date>[^,]*),(?<time>[^,]*),(?<reported>[^,]*)$/
read_from_head true
</source> </source>
## live debugging agent <match slapos.monitor.usage.disk.{{ wendelin_uid }}>
@type wendelin
@id wendelin_out
streamtool_uri {{ wendelin_streamtool_uri }}_disk
user {{ wendelin_user }}
password {{ wendelin_password }}
buffer_type memory
flush_interval 20s
</match>
<source> <source>
type debug_agent type tail
bind 127.0.0.1 path {{ fluentd_log_directory }}/consumption/*/dump_temperature.csv
port 24230 pos_file {{ fluentd_log_directory }}/consumption/dump_temperature.pos
tag slapos.monitor.usage.temperature.{{ wendelin_uid }}
format /^(?<sensor_id>[^,]*),(?<temperature>[^,]*),(?<alarm>[^,]*),(?<date>[^,]*),(?<time>[^,]*),(?<reported>[^,]*)$/
read_from_head true
</source> </source>
<match slapos.monitor.usage.temperature.{{ wendelin_uid }}>
@type wendelin
@id wendelin_out
streamtool_uri {{ wendelin_streamtool_uri }}_temperature
user {{ wendelin_user }}
password {{ wendelin_password }}
buffer_type memory
flush_interval 20s
</match>
{% endif -%}
software/monitor/fluentd-distributor-agent.conf.jinja2.in 0 → 100644
View file @ 80825610
# DESTINATION
<match td.*.*>
type tdlog
apikey YOUR_API_KEY
auto_create_table
buffer_type file
buffer_path {{ fluentd_log_directory }}/td-agent/buffer/td
<secondary>
type file
path {{ fluentd_log_directory }}/failed_records
</secondary>
</match>
## match tag=debug.** and dump to console
<match debug.**>
type stdout
</match>
<source>
type tail
path {{ crawl_log_directory }}/*/*ping.log.20*
pos_file {{ crawl_log_directory }}/tail_in_ping.pos
tag slapos.monitor.networktest.ping.ipv4
format /^(?<time>[^;]*);(?<computer_name>[^;]*);(?<type>[^;]*);(?<name_or_ip>[^;]*);(?<code>[^;]*);(?<average>[^;]*);(?<packet_lost>[^;]*);(?<extra>[^;]*)$/
read_from_head true
</source>
<source>
type tail
path {{ crawl_log_directory }}/*/*ping6.log.20*
pos_file {{ crawl_log_directory }}/tail_in_ping6.pos
tag slapos.monitor.networktest.ping.ipv6
format /^(?<time>[^;]*);(?<computer_name>[^;]*);(?<type>[^;]*);(?<name_or_ip>[^;]*);(?<code>[^;]*);(?<average>[^;]*);(?<packet_lost>[^;]*);(?<extra>[^;]*)$/
read_from_head true
</source>
<match slapos.monitor.networktest.ping.*>
type copy
{% for slave_instance in slave_instance_list -%}
{% if slave_instance.get("ping_ip_list") -%}
<store>
type grep
regexp1 name_or_ip ^{{ slave_instance.get("ping_ip_list") }}$
add_tag_prefix {{ slave_instance.get("slave_reference") }}
</store>
{% endif -%}
{% if slave_instance.get("ping6_ip_list") -%}
<store>
type grep
regexp1 name_or_ip ^{{ slave_instance.get("ping6_ip_list") }}$
add_tag_prefix {{ slave_instance.get("slave_reference") }}
</store>
{% endif -%}
{% endfor -%}
</match>
{% for slave_instance in slave_instance_list -%}
<match {{ slave_instance.get("slave_reference") }}.slapos.monitor.networktest.ping.ipv6>
type file
path {{ network_user_logs }}/{{ slave_instance.get("slave_reference") }}/ping6/log
append true
include_time_key true
include_tag_key true
format json
</match>
<match {{ slave_instance.get("slave_reference") }}.slapos.monitor.networktest.ping.ipv4>
type file
path {{ network_user_logs }}/{{ slave_instance.get("slave_reference") }}/ping/log
append true
include_time_key true
include_tag_key true
format json
</match>
{% endfor -%}
<source>
type tail
path {{ network_user_logs }}/*/ping*/log/*.log
pos_file {{ crawl_log_directory }}/tail_in_ping6.pos
tag slapos.wendelin.networktest
format json
read_from_head true
</source>
{% set wendelin_streamtool_uri = slapparameter_dict.get('wendelin-streamtool-uri', '') -%}
{% set wendelin_password = slapparameter_dict.get('wendelin-password', '') -%}
{% set wendelin_user = slapparameter_dict.get('wendelin-user', '') -%}
{% if wendelin_streamtool_uri and wendelin_password and wendelin_user -%}
<match slapos.wendelin.networktest>
@type wendelin
@id wendelin_out
streamtool_uri {{ wendelin_streamtool_uri }}
user {{ wendelin_user }}
password {{ wendelin_password }}
buffer_type memory
flush_interval 20s
</match>
{% endif -%}
## SOURCE
<source>
type forward
</source>
## live debugging agent
<source>
type debug_agent
bind 127.0.0.1
port 24230
</source>
software/monitor/index.cgi.in deleted 100755 → 0
View file @ e0dbf25a
#!{{ extra_eggs_interpreter }}
# Beginning of response
print "Content-Type: text/plain"
print
print "OK"
software/monitor/instance-monitor-distributor.cfg.jinja2
View file @ 80825610
...@@ -10,10 +10,10 @@ context = ...@@ -10,10 +10,10 @@ context =
[slave-test-configuration] [slave-test-configuration]
<=jinja2-template-base <=jinja2-template-base
template = {{ template_json_distributor_test }} template = {{ template_json_distributor_test }}
filename = srv/monitor-private/test.json filename = srv/monitor/private/test.json
extensions = jinja2.ext.do extensions = jinja2.ext.do
extra-context = extra-context =
section slave_information slap-parameter section slave_information slap-configuration
{% set part_list = [] -%} {% set part_list = [] -%}
# Publish information for each slave # Publish information for each slave
...@@ -27,7 +27,7 @@ extra-context = ...@@ -27,7 +27,7 @@ extra-context =
recipe = slapos.cookbook:publish recipe = slapos.cookbook:publish
-slave-reference = {{ slave_instance.get('slave_reference') }} -slave-reference = {{ slave_instance.get('slave_reference') }}
log-access-url = ${monitor-frontend:connection-site_url}/{{ slave_instance.get('slave_reference') }} log-access-url = ${monitor-frontend:connection-site_url}/{{ slave_instance.get('slave_reference') }}
log-access-url-v6 = ${monitor-parameters:url}/{{ slave_instance.get('slave_reference') }} log-access-url-v6 = ${monitor-httpd-conf-parameter:url}/{{ slave_instance.get('slave_reference') }}
{% endfor %} {% endfor %}
...@@ -49,13 +49,12 @@ command = cd ${monitor-directory:crawl-log}/{{ entry }} && ${crawler-bin:wrapper ...@@ -49,13 +49,12 @@ command = cd ${monitor-directory:crawl-log}/{{ entry }} && ${crawler-bin:wrapper
{% endif %} {% endif %}
[monitor-directory] [monitor-directory]
fluentd-log = ${:log}/fluentd
crawl-log = ${:srv}/crawlog crawl-log = ${:srv}/crawlog
network-user-logs = ${:private-directory}/network-user-logs/ network-user-logs = ${:private}/network-user-logs/
{% for slave_instance in slave_instance_list -%} {% for slave_instance in slave_instance_list -%}
user-log-{{ slave_instance.get('slave_reference') }}-folder = ${:private-directory}/network-user-logs/{{ slave_instance.get('slave_reference') }} user-log-{{ slave_instance.get('slave_reference') }}-folder = ${:private}/network-user-logs/{{ slave_instance.get('slave_reference') }}
user-log-{{ slave_instance.get('slave_reference') }}-ping-folder = ${:private-directory}/network-user-logs/{{ slave_instance.get('slave_reference') }}/ping user-log-{{ slave_instance.get('slave_reference') }}-ping-folder = ${:private}/network-user-logs/{{ slave_instance.get('slave_reference') }}/ping
user-log-{{ slave_instance.get('slave_reference') }}-ping6-folder = ${:private-directory}/network-user-logs/{{ slave_instance.get('slave_reference') }}/ping6 user-log-{{ slave_instance.get('slave_reference') }}-ping6-folder = ${:private}/network-user-logs/{{ slave_instance.get('slave_reference') }}/ping6
{% endfor -%} {% endfor -%}
{% for directory in directory_list %} {% for directory in directory_list %}
...@@ -74,35 +73,26 @@ parameters-extra = true ...@@ -74,35 +73,26 @@ parameters-extra = true
extends = {{ instance_base_monitor }} extends = {{ instance_base_monitor }}
parts += parts +=
slave-test-configuration slave-test-configuration
fluentd-wrapper fluentd-distributor-wrapper
{% for part in part_list %} {% for part in part_list %}
{{ ' %s' % part }} {{ ' %s' % part }}
{% endfor %} {% endfor %}
[fluentd-wrapper] [fluentd-distributor-wrapper]
recipe = slapos.cookbook:wrapper <=fluentd-wrapper
command-line = {{ fluentd_location }}/bin/fluentd -l ${monitor-directory:log}/fluend.log -c ${fluentd-conf-configuration:rendered} command-line = {{ fluentd_location }}/bin/fluentd -l ${monitor-directory:log}/fluend-distributor.log -c ${fluentd-distributor-conf-configuration:rendered}
wrapper-path = ${monitor-directory:service}/fluentd wrapper-path = ${monitor-directory:service}/fluentd-distributor
environment =
GEM_PATH={{ fluentd_location }}/lib/ruby/gems/1.8/
[fluentd-conf-configuration] [fluentd-distributor-conf-configuration]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
template = {{ fluent_conf_output }} template = {{ fluent_distributor_conf_output }}
rendered = ${monitor-directory:etc}/fluentd.cfg rendered = ${monitor-directory:etc}/fluentd-distributor.cfg
mode = 0744 mode = 0744
context = context =
key slapparameter_dict slap-configuration:configuration key slapparameter_dict slap-configuration:configuration
key slave_instance_list slap-parameters:slave-instance-list key slave_instance_list slap-configuration:slave-instance-list
key fluentd_log_directory monitor-directory:fluentd-log key fluentd_log_directory monitor-directory:fluentd-log
key crawl_log_directory monitor-directory:crawl-log key crawl_log_directory monitor-directory:crawl-log
key network_user_logs monitor-directory:network-user-logs key network_user_logs monitor-directory:network-user-logs
key computer_id slap-configuration:computer
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
software/monitor/instance-monitor.cfg.jinja2
View file @ 80825610
[buildout] [buildout]
extends = {{ monitor_template_output }}
parts = parts =
publish-connection-informations
cron cron
certificate-authority
cron-entry-monitor
cron-entry-rss
deploy-index
deploy-settings-cgi
deploy-status-cgi
deploy-status-history-cgi
setup-static-files
certificate-authority
zero-parameters
public-symlink
cgi-httpd-wrapper
cgi-httpd-graceful-wrapper
monitor-promise
monitor-instance-log-access
cron-network-bench cron-network-bench
symlink-re6st-logs symlink-re6st-logs
symlink-collected-logs symlink-collected-logs
monitor-collect-csv-wrapper
extends = {{ monitor_template_output }} fluentd-wrapper
monitor-base
publish-connection-information
eggs-directory = {{ eggs_directory }} eggs-directory = {{ eggs_directory }}
...@@ -52,8 +40,7 @@ template = {{ network_benck_cfg_output }} ...@@ -52,8 +40,7 @@ template = {{ network_benck_cfg_output }}
rendered = ${monitor-directory:etc}/network_bench.cfg rendered = ${monitor-directory:etc}/network_bench.cfg
mode = 0744 mode = 0744
context = context =
key slapparameter_dict slap-parameters:configuration key slapparameter_dict slap-configuration:configuration
[pwgen] [pwgen]
recipe = slapos.cookbook:generate.password recipe = slapos.cookbook:generate.password
...@@ -66,38 +53,16 @@ user = admin ...@@ -66,38 +53,16 @@ user = admin
bytes = 16 bytes = 16
[monitor-directory] [monitor-directory]
server-log = ${:private-directory}/server-log service = ${buildout:directory}/etc/service
monitor-log = ${:private-directory}/monitor-log var = ${buildout:directory}/var
srv = ${buildout:directory}/srv
server-log = ${:private}/server-log
monitor-log = ${:private}/monitor-log
cache = ${:var}/cache cache = ${:var}/cache
mod-ssl = ${:cache}/httpd_mod_ssl mod-ssl = ${:cache}/httpd_mod_ssl
system-log = ${:private-directory}/system-log system-log = ${:private}/system-log
fluentd-log = ${:log}/fluentd
[slap-parameter] consumption = ${:fluentd-log}/consumption
private-hash = ${pwgen:passwd}${pwgen32:passwd}
frontend-domain =
[monitor-frontend]
recipe = slapos.cookbook:requestoptional
name = Monitor Frontend
# XXX We have hardcoded SR URL here.
software-url = product.frontend
software-type = custom-personal
slave = true
config-url = ${monitor-parameters:url}
config-domain = ${slap-parameter:frontend-domain}
return = site_url domain
server-url = ${slap-connection:server-url}
key-file = ${slap-connection:key-file}
cert-file = ${slap-connection:cert-file}
computer-id = ${slap-connection:computer-id}
partition-id = ${slap-connection:partition-id}
[publish-connection-informations]
recipe = slapos.cookbook:publish
monitor_url = ${monitor-parameters:url}
server_log_url = ${:url}${slap-parameter:private-hash}/
url = ${monitor-frontend:connection-site_url}
[monitor-httpd-configuration-file] [monitor-httpd-configuration-file]
context = context =
...@@ -106,6 +71,47 @@ context = ...@@ -106,6 +71,47 @@ context =
section monitor_parameters monitor-parameters section monitor_parameters monitor-parameters
section httpd_configuration monitor-httpd-configuration section httpd_configuration monitor-httpd-configuration
section monitor_rewrite_rule monitor-rewrite-rule section monitor_rewrite_rule monitor-rewrite-rule
section slave_information slap-parameter section slave_information slap-configuration
key monitor_private_hash slap-parameter:private-hash key monitor_private_hash slap-configuration:private-hash
[fluentd-wrapper]
recipe = slapos.cookbook:wrapper
command-line = {{ fluentd_location }}/bin/fluentd -l ${monitor-directory:log}/fluend.log -c ${fluentd-conf-configuration:rendered}
wrapper-path = ${monitor-directory:service}/fluentd
environment =
GEM_PATH={{ fluentd_location }}/lib/ruby/gems/1.8/
[fluentd-conf-configuration]
recipe = slapos.recipe.template:jinja2
template = {{ fluent_conf_output }}
rendered = ${monitor-directory:etc}/fluentd.cfg
mode = 0744
context =
key slapparameter_dict slap-configuration:configuration
key slave_instance_list slap-configuration:slave-instance-list
key fluentd_log_directory monitor-directory:fluentd-log
key computer_id slap-connection:computer-id
[monitor-collect-csv-wrapper]
recipe = slapos.cookbook:wrapper
command-line =
${monitor-directory:bin}/python {{ monitor_collect_csv_dump }} --output_folder ${monitor-directory:fluentd-log}/consumption/
wrapper-path = ${monitor-directory:reports}/monitor-collect-csv-dump
parameters-extra = true
[publish-connection-information]
recipe = slapos.cookbook:publish
monitor-setup-url = https://monitor.app.officejs.com/#page=settings_configurator&url=${publish:monitor-url}&username=${publish:monitor-user}&password=${publish:monitor-password}
server_log_url = ${publish:monitor-base-url}/${slap-configuration:private-hash}/
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
private-hash = ${pwgen:passwd}${pwgen32:passwd}
frontend-domain =
software/monitor/instance.cfg
View file @ 80825610
...@@ -13,7 +13,7 @@ distributor = $${instance-base-distributor:rendered} ...@@ -13,7 +13,7 @@ distributor = $${instance-base-distributor:rendered}
[instance-base-monitor] [instance-base-monitor]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
template = ${template-monitor:destination} template = ${template-monitor:destination}
rendered = $${buildout:directory}/template-monitor.cfg rendered = $${buildout:directory}/template-base-monitor.cfg
extensions = jinja2.ext.do extensions = jinja2.ext.do
context = key buildout buildout:bin-directory context = key buildout buildout:bin-directory
key develop_eggs_directory buildout:develop-eggs-directory key develop_eggs_directory buildout:develop-eggs-directory
...@@ -21,12 +21,15 @@ context = key buildout buildout:bin-directory ...@@ -21,12 +21,15 @@ context = key buildout buildout:bin-directory
key slapparameter_dict slap-configuration:configuration key slapparameter_dict slap-configuration:configuration
raw monitor_template_output ${monitor-template:output} raw monitor_template_output ${monitor-template:output}
raw network_benck_cfg_output ${network-bench-cfg:output} raw network_benck_cfg_output ${network-bench-cfg:output}
raw fluentd_location ${fluentd:location}
raw fluent_conf_output ${fluentd-agent-conf:output}
raw monitor_collect_csv_dump ${monitor-collect-csv-dump:output}
mode = 0644 mode = 0644
[instance-base-distributor] [instance-base-distributor]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
template = ${template-monitor-distributor:destination} template = ${template-monitor-distributor:destination}
rendered = $${buildout:directory}/template-monitor-distributor.cfg rendered = $${buildout:directory}/template-monitor-base-distributor.cfg
extensions = jinja2.ext.do extensions = jinja2.ext.do
context = import json_module json context = import json_module json
key buildout buildout:bin-directory key buildout buildout:bin-directory
...@@ -35,9 +38,9 @@ context = import json_module json ...@@ -35,9 +38,9 @@ context = import json_module json
key slapparameter_dict slap-configuration:configuration key slapparameter_dict slap-configuration:configuration
key instance_base_monitor instance-base-monitor:rendered key instance_base_monitor instance-base-monitor:rendered
key slave_instance_list slap-configuration:slave-instance-list key slave_instance_list slap-configuration:slave-instance-list
raw template_json_distributor_test ${json-test-template:destination} raw template_json_distributor_test ${json-test-template:destination}
raw fluent_conf_output ${fluentd-agent-conf:output}
raw fluentd_location ${fluentd:location} raw fluentd_location ${fluentd:location}
raw fluent_distributor_conf_output ${fluentd-agent-distributor-conf:output}
raw wget_bin ${wget:location}/bin/wget raw wget_bin ${wget:location}/bin/wget
mode = 0644 mode = 0644
...@@ -48,3 +51,4 @@ partition = $${slap-connection:partition-id} ...@@ -48,3 +51,4 @@ partition = $${slap-connection:partition-id}
url = $${slap-connection:server-url} url = $${slap-connection:server-url}
key = $${slap-connection:key-file} key = $${slap-connection:key-file}
cert = $${slap-connection:cert-file} cert = $${slap-connection:cert-file}
software/monitor/script/collect_csv_dump.py 0 → 100644
View file @ 80825610
# -*- coding: utf-8 -*-
##############################################################################
#
# Copyright (c) 2010-2016 Vifib SARL and Contributors.
# All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public License
# as published by the Free Software Foundation; either version 2.1
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import os
import argparse
import csv
from slapos.util import mkdir_p
from slapos.collect.db import Database
def skip_bootstrap(self):
return
Database._bootstrap = skip_bootstrap
def parseArguments():
"""
Parse arguments for monitor collector instance.
"""
parser = argparse.ArgumentParser()
parser.add_argument('--output_folder',
help='Path of the folder where output files should be written.')
parser.add_argument('--collector_db',
default='/srv/slapgrid/var/data-log/',
help='The path of slapos collect database is located.')
return parser.parse_args()
def writeFile(name, folder, date_scope, rows):
if os.path.exists(
os.path.join(folder, "%s/dump_%s.csv" % (date_scope, name))):
# File already exists, no reason to recreate it.
return
mkdir_p(os.path.join(folder, date_scope), 0o755)
file_io = open(os.path.join(folder, "%s/dump_%s.csv" % (date_scope, name)), "w")
csv_output = csv.writer(file_io)
csv_output.writerows(rows)
file_io.close()
def dump_table_into_csv(db, folder):
db.connect()
table_list = db.getTableList()
# Save all dates first, as db.selector may switch the cursor
date_list = [(date_scope, _) \
for date_scope, _ in db.getDateScopeList(reported=1)]
for date_scope, amount in date_list:
for table in table_list:
if os.path.exists(
os.path.join(folder, "%s/dump_%s.csv" % (date_scope, table))):
# File already exists, no reason to recreate it.
continue
writeFile(table, folder, date_scope,
db.select(table, date_scope))
db.close()
if __name__ == "__main__":
parser = parseArguments()
if parser.output_folder is None:
raise Exception("Invalid ouput folder: %s" % parser.output_folder)
if parser.collector_db is None:
raise Exception("Invalid collector database folder: %s" % parser.collector_db)
if not os.path.exists(parser.output_folder) and \
os.path.isdir(parser.output_folder):
raise Exception("Invalid ouput folder: %s" % parser.output_folder)
if not os.path.exists(parser.collector_db):
print "Collector database not found..."
dump_table_into_csv(Database(parser.collector_db), parser.output_folder)
software/monitor/software.cfg
View file @ 80825610
...@@ -11,35 +11,35 @@ extends = ...@@ -11,35 +11,35 @@ extends =
parts = parts =
wget wget
slapos-cookbook slapos-cookbook
slapos-toolbox
network-bench-cfg network-bench-cfg
json-test-template json-test-template
template template
template-monitor-distributor template-monitor-distributor
template-monitor template-monitor
eggs
fluentd fluentd
fluentd-agent-conf fluentd-agent-conf
fluentd-agent-distributor-conf
monitor-collect-csv-dump
[template] [template]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg url = ${:_profile_base_location_}/instance.cfg
output = ${buildout:directory}/template.cfg output = ${buildout:directory}/template.cfg
md5sum = b38c5da7dd4165154a8817f76015e374 md5sum = 0c5e94beede86a91d7b01f61a7290f86
mode = 0644 mode = 0644
[template-monitor] [template-monitor]
recipe = slapos.recipe.build:download recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/instance-monitor.cfg.jinja2 url = ${:_profile_base_location_}/instance-monitor.cfg.jinja2
destination = ${buildout:directory}/template-monitor.cfg destination = ${buildout:directory}/template-base-monitor.cfg
md5sum = 810ed8199682068e27b62659d7fa101f md5sum = 4649ce8bc496cf7560c2e994f0b332ee
mode = 0644 mode = 0644
[template-monitor-distributor] [template-monitor-distributor]
recipe = slapos.recipe.build:download recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/instance-monitor-distributor.cfg.jinja2 url = ${:_profile_base_location_}/instance-monitor-distributor.cfg.jinja2
destination = ${buildout:directory}/template-monitor-distributor.cfg destination = ${buildout:directory}/template-monitor-base-distributor.cfg
md5sum = d1253cf11c2b335b8bec5807e821d537 md5sum = f579fe67fa1670b46d522b1e5f35c49c
mode = 0644 mode = 0644
[json-test-template] [json-test-template]
...@@ -49,16 +49,6 @@ destination = ${buildout:directory}/json-test-template.json.in.jinja2 ...@@ -49,16 +49,6 @@ destination = ${buildout:directory}/json-test-template.json.in.jinja2
md5sum = 2eb5596544d9c341acf653d4f7ce2680 md5sum = 2eb5596544d9c341acf653d4f7ce2680
mode = 0644 mode = 0644
# stupify index for now
[index]
url = ${:_profile_base_location_}/${:filename}
md5sum = 876f18b159fbd9325332d0f42e9172ac
[monitor-httpd-template]
url = ${:_profile_base_location_}/${:filename}
md5sum = e89b66a90409bb8e9099aa197803d337
[network-bench-cfg] [network-bench-cfg]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/network_bench.cfg.in url = ${:_profile_base_location_}/network_bench.cfg.in
...@@ -69,17 +59,33 @@ mode = 0644 ...@@ -69,17 +59,33 @@ mode = 0644
[fluentd-agent-conf] [fluentd-agent-conf]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/fluentd-agent.conf.jinja2.in url = ${:_profile_base_location_}/fluentd-agent.conf.jinja2.in
md5sum = 4b81ddcbe2f16d23013caac37151d396 md5sum = 3ea59906937eab7aeef78f46c4994ecd
output = ${buildout:directory}/fluentd-agent.conf.jinja2.in output = ${buildout:directory}/fluentd-agent.conf.jinja2.in
mode = 0644 mode = 0644
[slapos-toolbox] [fluentd-agent-distributor-conf]
recipe = zc.recipe.egg recipe = slapos.recipe.template
eggs = url = ${:_profile_base_location_}/fluentd-distributor-agent.conf.jinja2.in
md5sum = 4b81ddcbe2f16d23013caac37151d396
output = ${buildout:directory}/fluentd-agent-distributor.conf.jinja2.in
mode = 0644
[monitor-collect-csv-dump]
<= monitor-template-script
url = ${:_profile_base_location_}/script/${:filename}
filename = collect_csv_dump.py
output = ${:destination}/${:filename}
md5sum = cad2402bbd21907cfed6bc5af8c5d3ab
[extra-eggs]
<= monitor-eggs
interpreter = pythonwitheggs
eggs +=
${lxml-python:egg} ${lxml-python:egg}
${pycurl:egg} ${pycurl:egg}
${python-cryptography:egg} ${python-cryptography:egg}
slapos.toolbox slapos.toolbox
slapos.core
scripts = scripts =
networkbench networkbench
...@@ -89,43 +95,11 @@ scripts = ...@@ -89,43 +95,11 @@ scripts =
gems += gems +=
fluent-plugin-wendelin==0.1.alpha1 fluent-plugin-wendelin==0.1.alpha1
fluent-plugin-grep==0.3.4 fluent-plugin-grep==0.3.4
fluent-plugin-bin==0.1
[eggs] [monitor-eggs]
recipe = zc.recipe.egg eggs +=
eggs =
cns.recipe.symlink
slapos.cookbook slapos.cookbook
slapos.recipe.cmmi slapos.recipe.cmmi
hexagonit.recipe.download hexagonit.recipe.download
plone.recipe.command plone.recipe.command
[versions]
PyRSS2Gen = 1.1
apache-libcloud = 1.0.0rc2
cns.recipe.symlink = 0.2.3
gitdb = 0.6.4
pycurl = 7.43.0
rubygemsrecipe = 0.2.2
slapos.recipe.template = 2.9
slapos.toolbox = 0.55
smmap = 0.9.0
# Required by:
# slapos.toolbox==0.55
GitPython = 2.0.5
# Required by:
# slapos.toolbox==0.55
atomize = 0.2.0
# Required by:
# slapos.toolbox==0.55
feedparser = 5.2.1
# Required by:
# slapos.toolbox==0.55
lockfile = 0.12.2
# Required by:
# slapos.toolbox==0.55
paramiko = 2.0.0
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7