Skip to content

slapos
slapos
Commit 5d1d3c72 authored by Alain Takoudjou's avatar Alain Takoudjou
Browse files
Options

monitor: remove fluentd

parent 8551ef97
Show whitespace changes
Inline Side-by-side
Showing with 13 additions and 340 deletions
software/monitor/fluentd-agent.conf.jinja2.in deleted 100644 → 0
View file @ 8551ef97
# DESTINATION
<match td.*.*>
type tdlog
apikey YOUR_API_KEY
auto_create_table
buffer_type file
buffer_path {{ fluentd_log_directory }}/td-agent/buffer/td
<secondary>
type file
path {{ fluentd_log_directory }}/failed_records
</secondary>
</match>
{% set wendelin_streamtool_uri = slapparameter_dict.get('wendelin-streamtool-uri', '') -%}
{% set wendelin_password = slapparameter_dict.get('wendelin-password', '') -%}
{% set wendelin_user = slapparameter_dict.get('wendelin-user', '') -%}
{% set wendelin_uid = slapparameter_dict.get('wendelin-uid', 'UNKNOWN') -%}
{% if wendelin_streamtool_uri and wendelin_password and wendelin_user -%}
<source>
type syslog
port 42185
bind 127.0.0.1
tag {{ wendelin_uid }}.system
</source>
<match {{ wendelin_uid }}.system.**>
@type wendelin
@id wendelin_out
streamtool_uri {{ wendelin_streamtool_uri }}_syslog
user {{ wendelin_user }}
password {{ wendelin_password }}
buffer_type memory
flush_interval 20s
</match>
<source>
type tail
path {{ fluentd_log_directory }}/consumption/*/dump_user.csv
pos_file {{ fluentd_log_directory }}/consumption/dump_user.pos
tag slapos.monitor.usage.user.{{ wendelin_uid }}
format /^(?<partition>[^,]*),(?<pid>[^,]*),(?<process>[^,]*),(?<cpu_percent>[^,]*),(?<cpu_time>[^,]*),(?<cpu_num_threads>[^,]*),(?<memory_percent>[^,]*),(?<memory_rss>[^,]*),(?<io_rw_counter>[^,]*),(?<io_cycles_counter>[^,]*),(?<date>[^,]*),(?<time>[^,]*),(?<reported>[^,]*)$/
read_from_head true
</source>
<match slapos.monitor.usage.user.{{ wendelin_uid }}>
@type wendelin
@id wendelin_out
streamtool_uri {{ wendelin_streamtool_uri }}_user
user {{ wendelin_user }}
password {{ wendelin_password }}
buffer_type memory
flush_interval 20s
</match>
<source>
type tail
path {{ fluentd_log_directory }}/consumption/*/dump_system.csv
pos_file {{ fluentd_log_directory }}/consumption/dump_system.pos
tag slapos.monitor.usage.system.{{ wendelin_uid }}
format /^(?<loadavg>[^,]*),(?<cpu_percent>[^,]*),(?<memory_used>[^,]*),(?<memory_free>[^,]*),(?<net_in_bytes>[^,]*),(?<net_in_errors>[^,]*),(?<net_in_dropped>[^,]*),(?<net_out_bytes>[^,]*),(?<net_out_errors>[^,]*),(?<net_out_dropped>[^,]*),(?<date>[^,]*),(?<time>[^,]*),(?<reported>[^,]*)$/
read_from_head true
</source>
<match slapos.monitor.usage.system.{{ wendelin_uid }}>
@type wendelin
@id wendelin_out
streamtool_uri {{ wendelin_streamtool_uri }}_system
user {{ wendelin_user }}
password {{ wendelin_password }}
buffer_type memory
flush_interval 20s
</match>
<source>
type tail
path {{ fluentd_log_directory }}/consumption/*/dump_computer.csv
pos_file {{ fluentd_log_directory }}/consumption/dump_computer.pos
tag slapos.monitor.usage.computer.{{ wendelin_uid }}
format /^(?<cpu_num_core>[^,]*),(?<cpu_frequency>[^,]*),(?<cpu_type>[^,]*),(?<memory_size>[^,]*),(?<memory_type>[^,]*),(?<partition_list>[^,]*),(?<date>[^,]*),(?<time>[^,]*),(?<reported>[^,]*)$/
read_from_head true
</source>
<match slapos.monitor.usage.computer.{{ wendelin_uid }}>
@type wendelin
@id wendelin_out
streamtool_uri {{ wendelin_streamtool_uri }}_computer
user {{ wendelin_user }}
password {{ wendelin_password }}
buffer_type memory
flush_interval 20s
</match>
<source>
type tail
path {{ fluentd_log_directory }}/consumption/*/dump_disk.csv
pos_file {{ fluentd_log_directory }}/consumption/dump_disk.pos
tag slapos.monitor.usage.disk.{{ wendelin_uid }}
format /^(?<partition>[^,]*),(?<used>[^,]*),(?<free>[^,]*),(?<mountpoint>[^,]*),(?<date>[^,]*),(?<time>[^,]*),(?<reported>[^,]*)$/
read_from_head true
</source>
<match slapos.monitor.usage.disk.{{ wendelin_uid }}>
@type wendelin
@id wendelin_out
streamtool_uri {{ wendelin_streamtool_uri }}_disk
user {{ wendelin_user }}
password {{ wendelin_password }}
buffer_type memory
flush_interval 20s
</match>
<source>
type tail
path {{ fluentd_log_directory }}/consumption/*/dump_temperature.csv
pos_file {{ fluentd_log_directory }}/consumption/dump_temperature.pos
tag slapos.monitor.usage.temperature.{{ wendelin_uid }}
format /^(?<sensor_id>[^,]*),(?<temperature>[^,]*),(?<alarm>[^,]*),(?<date>[^,]*),(?<time>[^,]*),(?<reported>[^,]*)$/
read_from_head true
</source>
<match slapos.monitor.usage.temperature.{{ wendelin_uid }}>
@type wendelin
@id wendelin_out
streamtool_uri {{ wendelin_streamtool_uri }}_temperature
user {{ wendelin_user }}
password {{ wendelin_password }}
buffer_type memory
flush_interval 20s
</match>
{% endif -%}
software/monitor/fluentd-distributor-agent.conf.jinja2.in deleted 100644 → 0
View file @ 8551ef97
# DESTINATION
<match td.*.*>
type tdlog
apikey YOUR_API_KEY
auto_create_table
buffer_type file
buffer_path {{ fluentd_log_directory }}/td-agent/buffer/td
<secondary>
type file
path {{ fluentd_log_directory }}/failed_records
</secondary>
</match>
## match tag=debug.** and dump to console
<match debug.**>
type stdout
</match>
<source>
type tail
path {{ crawl_log_directory }}/*/*ping.log.20*
pos_file {{ crawl_log_directory }}/tail_in_ping.pos
tag slapos.monitor.networktest.ping.ipv4
format /^(?<time>[^;]*);(?<computer_name>[^;]*);(?<type>[^;]*);(?<name_or_ip>[^;]*);(?<code>[^;]*);(?<average>[^;]*);(?<packet_lost>[^;]*);(?<extra>[^;]*)$/
read_from_head true
</source>
<source>
type tail
path {{ crawl_log_directory }}/*/*ping6.log.20*
pos_file {{ crawl_log_directory }}/tail_in_ping6.pos
tag slapos.monitor.networktest.ping.ipv6
format /^(?<time>[^;]*);(?<computer_name>[^;]*);(?<type>[^;]*);(?<name_or_ip>[^;]*);(?<code>[^;]*);(?<average>[^;]*);(?<packet_lost>[^;]*);(?<extra>[^;]*)$/
read_from_head true
</source>
<match slapos.monitor.networktest.ping.*>
type copy
{% for slave_instance in slave_instance_list -%}
{% if slave_instance.get("ping_ip_list") -%}
<store>
type grep
regexp1 name_or_ip ^{{ slave_instance.get("ping_ip_list") }}$
add_tag_prefix {{ slave_instance.get("slave_reference") }}
</store>
{% endif -%}
{% if slave_instance.get("ping6_ip_list") -%}
<store>
type grep
regexp1 name_or_ip ^{{ slave_instance.get("ping6_ip_list") }}$
add_tag_prefix {{ slave_instance.get("slave_reference") }}
</store>
{% endif -%}
{% endfor -%}
</match>
{% for slave_instance in slave_instance_list -%}
<match {{ slave_instance.get("slave_reference") }}.slapos.monitor.networktest.ping.ipv6>
type file
path {{ network_user_logs }}/{{ slave_instance.get("slave_reference") }}/ping6/log
append true
include_time_key true
include_tag_key true
format json
</match>
<match {{ slave_instance.get("slave_reference") }}.slapos.monitor.networktest.ping.ipv4>
type file
path {{ network_user_logs }}/{{ slave_instance.get("slave_reference") }}/ping/log
append true
include_time_key true
include_tag_key true
format json
</match>
{% endfor -%}
<source>
type tail
path {{ network_user_logs }}/*/ping*/log/*.log
pos_file {{ crawl_log_directory }}/tail_in_ping6.pos
tag slapos.wendelin.networktest
format json
read_from_head true
</source>
{% set wendelin_streamtool_uri = slapparameter_dict.get('wendelin-streamtool-uri', '') -%}
{% set wendelin_password = slapparameter_dict.get('wendelin-password', '') -%}
{% set wendelin_user = slapparameter_dict.get('wendelin-user', '') -%}
{% if wendelin_streamtool_uri and wendelin_password and wendelin_user -%}
<match slapos.wendelin.networktest>
@type wendelin
@id wendelin_out
streamtool_uri {{ wendelin_streamtool_uri }}
user {{ wendelin_user }}
password {{ wendelin_password }}
buffer_type memory
flush_interval 20s
</match>
{% endif -%}
## SOURCE
<source>
type forward
</source>
## live debugging agent
<source>
type debug_agent
bind 127.0.0.1
port 24230
</source>
software/monitor/instance-monitor-distributor.cfg.jinja2
View file @ 5d1d3c72
...@@ -72,26 +72,7 @@ wrapper-path = ${monitor-directory:bin}/log-crawler ...@@ -72,26 +72,7 @@ wrapper-path = ${monitor-directory:bin}/log-crawler
extends = {{ instance_base_monitor }} extends = {{ instance_base_monitor }}
parts += parts +=
slave-test-configuration slave-test-configuration
fluentd-distributor-wrapper
{% for part in part_list %} {% for part in part_list %}
{{ ' %s' % part }} {{ ' %s' % part }}
{% endfor %} {% endfor %}
[fluentd-distributor-wrapper]
<=fluentd-wrapper
command-line = {{ fluentd_location }}/bin/fluentd -l ${monitor-directory:log}/fluend-distributor.log -c ${fluentd-distributor-conf-configuration:rendered}
wrapper-path = ${monitor-directory:service}/fluentd-distributor
[fluentd-distributor-conf-configuration]
recipe = slapos.recipe.template:jinja2
template = {{ fluent_distributor_conf_output }}
rendered = ${monitor-directory:etc}/fluentd-distributor.cfg
mode = 0744
context =
key slapparameter_dict slap-configuration:configuration
key slave_instance_list slap-configuration:slave-instance-list
key fluentd_log_directory monitor-directory:fluentd-log
key crawl_log_directory monitor-directory:crawl-log
key network_user_logs monitor-directory:network-user-logs
key computer_id slap-configuration:computer
software/monitor/instance-monitor.cfg.jinja2
View file @ 5d1d3c72
...@@ -8,7 +8,6 @@ parts = ...@@ -8,7 +8,6 @@ parts =
symlink-re6st-logs symlink-re6st-logs
symlink-collected-logs symlink-collected-logs
monitor-collect-csv-wrapper monitor-collect-csv-wrapper
fluentd-wrapper
monitor-base monitor-base
monitor-check-memory-usage monitor-check-memory-usage
monitor-check-cpu-usage monitor-check-cpu-usage
...@@ -24,7 +23,10 @@ offline = true ...@@ -24,7 +23,10 @@ offline = true
recipe = slapos.cookbook:cron.d recipe = slapos.cookbook:cron.d
name = network-bench-test name = network-bench-test
frequency = */10 * * * * frequency = */10 * * * *
command = {{ buildout }}/networkbench ${network-bench-configuration:rendered} ${monitor-directory:monitor-log} # skip to not fill cron log file.
# command = {{ buildout_bin }}/networkbench ${network-bench-configuration:rendered} ${monitor-directory:monitor-log}
command = true
[symlink-re6st-logs] [symlink-re6st-logs]
recipe = cns.recipe.symlink recipe = cns.recipe.symlink
...@@ -63,8 +65,7 @@ monitor-log = ${:private}/monitor-log ...@@ -63,8 +65,7 @@ monitor-log = ${:private}/monitor-log
cache = ${:var}/cache cache = ${:var}/cache
mod-ssl = ${:cache}/httpd_mod_ssl mod-ssl = ${:cache}/httpd_mod_ssl
system-log = ${:private}/system-log system-log = ${:private}/system-log
fluentd-log = ${:log}/fluentd consumption = ${:log}/consumption
consumption = ${:fluentd-log}/consumption
[monitor-httpd-configuration-file] [monitor-httpd-configuration-file]
context = context =
...@@ -76,28 +77,10 @@ context = ...@@ -76,28 +77,10 @@ context =
section slave_information slap-configuration section slave_information slap-configuration
key monitor_private_hash slap-configuration:private-hash key monitor_private_hash slap-configuration:private-hash
[fluentd-wrapper]
recipe = slapos.cookbook:wrapper
command-line = {{ fluentd_location }}/bin/fluentd -qq -l ${monitor-directory:log}/fluend.log -c ${fluentd-conf-configuration:rendered}
wrapper-path = ${monitor-directory:service}/fluentd
environment =
GEM_PATH={{ fluentd_location }}/lib/ruby/gems/1.8/
[fluentd-conf-configuration]
recipe = slapos.recipe.template:jinja2
template = {{ fluent_conf_output }}
rendered = ${monitor-directory:etc}/fluentd.cfg
mode = 0744
context =
key slapparameter_dict slap-configuration:configuration
key slave_instance_list slap-configuration:slave-instance-list
key fluentd_log_directory monitor-directory:fluentd-log
key computer_id slap-connection:computer-id
[monitor-collect-csv-wrapper] [monitor-collect-csv-wrapper]
recipe = slapos.cookbook:wrapper recipe = slapos.cookbook:wrapper
command-line = command-line =
${monitor-directory:bin}/python {{ monitor_collect_csv_dump }} --output_folder ${monitor-directory:fluentd-log}/consumption/ ${monitor-directory:bin}/python {{ monitor_collect_csv_dump }} --output_folder ${monitor-directory:consumption}
wrapper-path = ${monitor-directory:reports}/monitor-collect-csv-dump wrapper-path = ${monitor-directory:reports}/monitor-collect-csv-dump
[monitor-check-cpu-usage] [monitor-check-cpu-usage]
......
software/monitor/instance.cfg
View file @ 5d1d3c72
...@@ -15,14 +15,12 @@ recipe = slapos.recipe.template:jinja2 ...@@ -15,14 +15,12 @@ recipe = slapos.recipe.template:jinja2
template = ${template-monitor:destination} template = ${template-monitor:destination}
rendered = $${buildout:directory}/template-base-monitor.cfg rendered = $${buildout:directory}/template-base-monitor.cfg
extensions = jinja2.ext.do extensions = jinja2.ext.do
context = key buildout buildout:bin-directory context = key develop_eggs_directory buildout:develop-eggs-directory
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory key eggs_directory buildout:eggs-directory
key slapparameter_dict slap-configuration:configuration key slapparameter_dict slap-configuration:configuration
raw buildout_bin ${buildout:bin-directory}
raw monitor_template_output ${monitor-template:output} raw monitor_template_output ${monitor-template:output}
raw network_benck_cfg_output ${network-bench-cfg:output} raw network_benck_cfg_output ${network-bench-cfg:output}
raw fluentd_location ${fluentd:location}
raw fluent_conf_output ${fluentd-agent-conf:output}
raw monitor_collect_csv_dump ${monitor-collect-csv-dump:output} raw monitor_collect_csv_dump ${monitor-collect-csv-dump:output}
raw monitor_check_system_health ${monitor-system-health:output} raw monitor_check_system_health ${monitor-system-health:output}
mode = 0644 mode = 0644
...@@ -33,15 +31,13 @@ template = ${template-monitor-distributor:destination} ...@@ -33,15 +31,13 @@ template = ${template-monitor-distributor:destination}
rendered = $${buildout:directory}/template-monitor-base-distributor.cfg rendered = $${buildout:directory}/template-monitor-base-distributor.cfg
extensions = jinja2.ext.do extensions = jinja2.ext.do
context = import json_module json context = import json_module json
key buildout buildout:bin-directory
key develop_eggs_directory buildout:develop-eggs-directory key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory key eggs_directory buildout:eggs-directory
key slapparameter_dict slap-configuration:configuration key slapparameter_dict slap-configuration:configuration
key instance_base_monitor instance-base-monitor:rendered key instance_base_monitor instance-base-monitor:rendered
key slave_instance_list slap-configuration:slave-instance-list key slave_instance_list slap-configuration:slave-instance-list
raw buildout_bin ${buildout:bin-directory}
raw template_json_distributor_test ${json-test-template:destination} raw template_json_distributor_test ${json-test-template:destination}
raw fluentd_location ${fluentd:location}
raw fluent_distributor_conf_output ${fluentd-agent-distributor-conf:output}
raw wget_bin ${wget:location}/bin/wget raw wget_bin ${wget:location}/bin/wget
mode = 0644 mode = 0644
......
software/monitor/software.cfg
View file @ 5d1d3c72
...@@ -5,7 +5,6 @@ extends = ...@@ -5,7 +5,6 @@ extends =
../../component/python-cryptography/buildout.cfg ../../component/python-cryptography/buildout.cfg
../../component/wget/buildout.cfg ../../component/wget/buildout.cfg
../../stack/monitor/buildout.cfg ../../stack/monitor/buildout.cfg
../../component/fluentd/buildout.cfg
../../stack/slapos.cfg ../../stack/slapos.cfg
parts = parts =
...@@ -16,30 +15,27 @@ parts = ...@@ -16,30 +15,27 @@ parts =
template template
template-monitor-distributor template-monitor-distributor
template-monitor template-monitor
fluentd
fluentd-agent-conf
fluentd-agent-distributor-conf
monitor-collect-csv-dump monitor-collect-csv-dump
[template] [template]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg url = ${:_profile_base_location_}/instance.cfg
output = ${buildout:directory}/template.cfg output = ${buildout:directory}/template.cfg
md5sum = 3ff5fb2710bf0ea84632c6d6d3894dd9 md5sum = 641c5916739f78171c616af00fe974a2
mode = 0644 mode = 0644
[template-monitor] [template-monitor]
recipe = slapos.recipe.build:download recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/instance-monitor.cfg.jinja2 url = ${:_profile_base_location_}/instance-monitor.cfg.jinja2
destination = ${buildout:directory}/template-base-monitor.cfg destination = ${buildout:directory}/template-base-monitor.cfg
md5sum = 5def53c8faa0dfca313e53ebcc3229fa md5sum = 5cd3fe3edb4a7fbbc9671f051845d752
mode = 0644 mode = 0644
[template-monitor-distributor] [template-monitor-distributor]
recipe = slapos.recipe.build:download recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/instance-monitor-distributor.cfg.jinja2 url = ${:_profile_base_location_}/instance-monitor-distributor.cfg.jinja2
destination = ${buildout:directory}/template-monitor-base-distributor.cfg destination = ${buildout:directory}/template-monitor-base-distributor.cfg
md5sum = 0ab9a86df18125335ed365bb673bbcf3 md5sum = 61c0bfdfc0a2b51ba15fe4a49baf6091
mode = 0644 mode = 0644
[json-test-template] [json-test-template]
...@@ -56,20 +52,6 @@ md5sum = cfcbf2002b8eff5153e2bf68ed24b720 ...@@ -56,20 +52,6 @@ md5sum = cfcbf2002b8eff5153e2bf68ed24b720
output = ${buildout:directory}/template-network-bench-cfg.in output = ${buildout:directory}/template-network-bench-cfg.in
mode = 0644 mode = 0644
[fluentd-agent-conf]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/fluentd-agent.conf.jinja2.in
md5sum = 3ea59906937eab7aeef78f46c4994ecd
output = ${buildout:directory}/fluentd-agent.conf.jinja2.in
mode = 0644
[fluentd-agent-distributor-conf]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/fluentd-distributor-agent.conf.jinja2.in
md5sum = 4b81ddcbe2f16d23013caac37151d396
output = ${buildout:directory}/fluentd-agent-distributor.conf.jinja2.in
mode = 0644
[monitor-collect-csv-dump] [monitor-collect-csv-dump]
<= monitor-template-script <= monitor-template-script
url = ${:_profile_base_location_}/script/${:filename} url = ${:_profile_base_location_}/script/${:filename}
...@@ -104,12 +86,6 @@ scripts = ...@@ -104,12 +86,6 @@ scripts =
monitor.configwrite monitor.configwrite
is-process-older-than-dependency-set is-process-older-than-dependency-set
[fluentd]
gems +=
fluent-plugin-wendelin==0.1.alpha1
fluent-plugin-grep==0.3.4
fluent-plugin-bin==0.1
[monitor-eggs] [monitor-eggs]
eggs += eggs +=
slapos.cookbook slapos.cookbook
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7