From 5d1d3c7218406e0ab4d5b6b3b5a52ba49141a711 Mon Sep 17 00:00:00 2001 From: Alain Takoudjou <alain.takoudjou@nexedi.com> Date: Mon, 11 Dec 2017 14:05:22 +0100 Subject: [PATCH] monitor: remove fluentd --- software/monitor/fluentd-agent.conf.jinja2.in | 147 ------------------ .../fluentd-distributor-agent.conf.jinja2.in | 116 -------------- .../instance-monitor-distributor.cfg.jinja2 | 19 --- software/monitor/instance-monitor.cfg.jinja2 | 29 +--- software/monitor/instance.cfg | 12 +- software/monitor/software.cfg | 30 +--- 6 files changed, 13 insertions(+), 340 deletions(-) delete mode 100644 software/monitor/fluentd-agent.conf.jinja2.in delete mode 100644 software/monitor/fluentd-distributor-agent.conf.jinja2.in diff --git a/software/monitor/fluentd-agent.conf.jinja2.in b/software/monitor/fluentd-agent.conf.jinja2.in deleted file mode 100644 index b4410e9b9..000000000 --- a/software/monitor/fluentd-agent.conf.jinja2.in +++ /dev/null @@ -1,147 +0,0 @@ -# DESTINATION -<match td.*.*> - type tdlog - apikey YOUR_API_KEY - - auto_create_table - buffer_type file - buffer_path {{ fluentd_log_directory }}/td-agent/buffer/td - - <secondary> - type file - path {{ fluentd_log_directory }}/failed_records - </secondary> -</match> - -{% set wendelin_streamtool_uri = slapparameter_dict.get('wendelin-streamtool-uri', '') -%} -{% set wendelin_password = slapparameter_dict.get('wendelin-password', '') -%} -{% set wendelin_user = slapparameter_dict.get('wendelin-user', '') -%} -{% set wendelin_uid = slapparameter_dict.get('wendelin-uid', 'UNKNOWN') -%} -{% if wendelin_streamtool_uri and wendelin_password and wendelin_user -%} -<source> - type syslog - port 42185 - bind 127.0.0.1 - tag {{ wendelin_uid }}.system -</source> - -<match {{ wendelin_uid }}.system.**> - @type wendelin - @id wendelin_out - - streamtool_uri {{ wendelin_streamtool_uri }}_syslog - user {{ wendelin_user }} - password {{ wendelin_password }} - - buffer_type memory - flush_interval 20s -</match> - -<source> - type tail - path {{ fluentd_log_directory }}/consumption/*/dump_user.csv - pos_file {{ fluentd_log_directory }}/consumption/dump_user.pos - tag slapos.monitor.usage.user.{{ wendelin_uid }} - format /^(?<partition>[^,]*),(?<pid>[^,]*),(?<process>[^,]*),(?<cpu_percent>[^,]*),(?<cpu_time>[^,]*),(?<cpu_num_threads>[^,]*),(?<memory_percent>[^,]*),(?<memory_rss>[^,]*),(?<io_rw_counter>[^,]*),(?<io_cycles_counter>[^,]*),(?<date>[^,]*),(?<time>[^,]*),(?<reported>[^,]*)$/ - read_from_head true -</source> - -<match slapos.monitor.usage.user.{{ wendelin_uid }}> - @type wendelin - @id wendelin_out - - streamtool_uri {{ wendelin_streamtool_uri }}_user - user {{ wendelin_user }} - password {{ wendelin_password }} - - buffer_type memory - flush_interval 20s -</match> - -<source> - type tail - path {{ fluentd_log_directory }}/consumption/*/dump_system.csv - pos_file {{ fluentd_log_directory }}/consumption/dump_system.pos - tag slapos.monitor.usage.system.{{ wendelin_uid }} - format /^(?<loadavg>[^,]*),(?<cpu_percent>[^,]*),(?<memory_used>[^,]*),(?<memory_free>[^,]*),(?<net_in_bytes>[^,]*),(?<net_in_errors>[^,]*),(?<net_in_dropped>[^,]*),(?<net_out_bytes>[^,]*),(?<net_out_errors>[^,]*),(?<net_out_dropped>[^,]*),(?<date>[^,]*),(?<time>[^,]*),(?<reported>[^,]*)$/ - read_from_head true -</source> - -<match slapos.monitor.usage.system.{{ wendelin_uid }}> - @type wendelin - @id wendelin_out - - streamtool_uri {{ wendelin_streamtool_uri }}_system - user {{ wendelin_user }} - password {{ wendelin_password }} - - buffer_type memory - flush_interval 20s -</match> - - -<source> - type tail - path {{ fluentd_log_directory }}/consumption/*/dump_computer.csv - pos_file {{ fluentd_log_directory }}/consumption/dump_computer.pos - tag slapos.monitor.usage.computer.{{ wendelin_uid }} - format /^(?<cpu_num_core>[^,]*),(?<cpu_frequency>[^,]*),(?<cpu_type>[^,]*),(?<memory_size>[^,]*),(?<memory_type>[^,]*),(?<partition_list>[^,]*),(?<date>[^,]*),(?<time>[^,]*),(?<reported>[^,]*)$/ - - read_from_head true -</source> - -<match slapos.monitor.usage.computer.{{ wendelin_uid }}> - @type wendelin - @id wendelin_out - - streamtool_uri {{ wendelin_streamtool_uri }}_computer - user {{ wendelin_user }} - password {{ wendelin_password }} - - buffer_type memory - flush_interval 20s -</match> - -<source> - type tail - path {{ fluentd_log_directory }}/consumption/*/dump_disk.csv - pos_file {{ fluentd_log_directory }}/consumption/dump_disk.pos - tag slapos.monitor.usage.disk.{{ wendelin_uid }} - format /^(?<partition>[^,]*),(?<used>[^,]*),(?<free>[^,]*),(?<mountpoint>[^,]*),(?<date>[^,]*),(?<time>[^,]*),(?<reported>[^,]*)$/ - read_from_head true -</source> - -<match slapos.monitor.usage.disk.{{ wendelin_uid }}> - @type wendelin - @id wendelin_out - - streamtool_uri {{ wendelin_streamtool_uri }}_disk - user {{ wendelin_user }} - password {{ wendelin_password }} - - buffer_type memory - flush_interval 20s -</match> - -<source> - type tail - path {{ fluentd_log_directory }}/consumption/*/dump_temperature.csv - pos_file {{ fluentd_log_directory }}/consumption/dump_temperature.pos - tag slapos.monitor.usage.temperature.{{ wendelin_uid }} - format /^(?<sensor_id>[^,]*),(?<temperature>[^,]*),(?<alarm>[^,]*),(?<date>[^,]*),(?<time>[^,]*),(?<reported>[^,]*)$/ - read_from_head true -</source> - -<match slapos.monitor.usage.temperature.{{ wendelin_uid }}> - @type wendelin - @id wendelin_out - - streamtool_uri {{ wendelin_streamtool_uri }}_temperature - user {{ wendelin_user }} - password {{ wendelin_password }} - - buffer_type memory - flush_interval 20s -</match> - -{% endif -%} diff --git a/software/monitor/fluentd-distributor-agent.conf.jinja2.in b/software/monitor/fluentd-distributor-agent.conf.jinja2.in deleted file mode 100644 index f40cdaddd..000000000 --- a/software/monitor/fluentd-distributor-agent.conf.jinja2.in +++ /dev/null @@ -1,116 +0,0 @@ -# DESTINATION -<match td.*.*> - type tdlog - apikey YOUR_API_KEY - - auto_create_table - buffer_type file - buffer_path {{ fluentd_log_directory }}/td-agent/buffer/td - - <secondary> - type file - path {{ fluentd_log_directory }}/failed_records - </secondary> -</match> - -## match tag=debug.** and dump to console -<match debug.**> - type stdout -</match> - -<source> - type tail - path {{ crawl_log_directory }}/*/*ping.log.20* - pos_file {{ crawl_log_directory }}/tail_in_ping.pos - tag slapos.monitor.networktest.ping.ipv4 - format /^(?<time>[^;]*);(?<computer_name>[^;]*);(?<type>[^;]*);(?<name_or_ip>[^;]*);(?<code>[^;]*);(?<average>[^;]*);(?<packet_lost>[^;]*);(?<extra>[^;]*)$/ - read_from_head true -</source> - -<source> - type tail - path {{ crawl_log_directory }}/*/*ping6.log.20* - pos_file {{ crawl_log_directory }}/tail_in_ping6.pos - tag slapos.monitor.networktest.ping.ipv6 - format /^(?<time>[^;]*);(?<computer_name>[^;]*);(?<type>[^;]*);(?<name_or_ip>[^;]*);(?<code>[^;]*);(?<average>[^;]*);(?<packet_lost>[^;]*);(?<extra>[^;]*)$/ - read_from_head true -</source> - -<match slapos.monitor.networktest.ping.*> - type copy -{% for slave_instance in slave_instance_list -%} -{% if slave_instance.get("ping_ip_list") -%} - <store> - type grep - regexp1 name_or_ip ^{{ slave_instance.get("ping_ip_list") }}$ - add_tag_prefix {{ slave_instance.get("slave_reference") }} - </store> -{% endif -%} -{% if slave_instance.get("ping6_ip_list") -%} - <store> - type grep - regexp1 name_or_ip ^{{ slave_instance.get("ping6_ip_list") }}$ - add_tag_prefix {{ slave_instance.get("slave_reference") }} - </store> -{% endif -%} -{% endfor -%} -</match> - -{% for slave_instance in slave_instance_list -%} -<match {{ slave_instance.get("slave_reference") }}.slapos.monitor.networktest.ping.ipv6> - type file - path {{ network_user_logs }}/{{ slave_instance.get("slave_reference") }}/ping6/log - append true - include_time_key true - include_tag_key true - format json -</match> -<match {{ slave_instance.get("slave_reference") }}.slapos.monitor.networktest.ping.ipv4> - type file - path {{ network_user_logs }}/{{ slave_instance.get("slave_reference") }}/ping/log - append true - include_time_key true - include_tag_key true - format json -</match> -{% endfor -%} - -<source> - type tail - path {{ network_user_logs }}/*/ping*/log/*.log - pos_file {{ crawl_log_directory }}/tail_in_ping6.pos - tag slapos.wendelin.networktest - format json - read_from_head true -</source> - -{% set wendelin_streamtool_uri = slapparameter_dict.get('wendelin-streamtool-uri', '') -%} -{% set wendelin_password = slapparameter_dict.get('wendelin-password', '') -%} -{% set wendelin_user = slapparameter_dict.get('wendelin-user', '') -%} -{% if wendelin_streamtool_uri and wendelin_password and wendelin_user -%} -<match slapos.wendelin.networktest> - - @type wendelin - @id wendelin_out - - streamtool_uri {{ wendelin_streamtool_uri }} - user {{ wendelin_user }} - password {{ wendelin_password }} - - buffer_type memory - flush_interval 20s -</match> -{% endif -%} - -## SOURCE -<source> - type forward -</source> - -## live debugging agent -<source> - type debug_agent - bind 127.0.0.1 - port 24230 -</source> - diff --git a/software/monitor/instance-monitor-distributor.cfg.jinja2 b/software/monitor/instance-monitor-distributor.cfg.jinja2 index 77026a605..3cf2c3c01 100644 --- a/software/monitor/instance-monitor-distributor.cfg.jinja2 +++ b/software/monitor/instance-monitor-distributor.cfg.jinja2 @@ -72,26 +72,7 @@ wrapper-path = ${monitor-directory:bin}/log-crawler extends = {{ instance_base_monitor }} parts += slave-test-configuration - fluentd-distributor-wrapper {% for part in part_list %} {{ ' %s' % part }} {% endfor %} -[fluentd-distributor-wrapper] -<=fluentd-wrapper -command-line = {{ fluentd_location }}/bin/fluentd -l ${monitor-directory:log}/fluend-distributor.log -c ${fluentd-distributor-conf-configuration:rendered} -wrapper-path = ${monitor-directory:service}/fluentd-distributor - -[fluentd-distributor-conf-configuration] -recipe = slapos.recipe.template:jinja2 -template = {{ fluent_distributor_conf_output }} -rendered = ${monitor-directory:etc}/fluentd-distributor.cfg -mode = 0744 -context = - key slapparameter_dict slap-configuration:configuration - key slave_instance_list slap-configuration:slave-instance-list - key fluentd_log_directory monitor-directory:fluentd-log - key crawl_log_directory monitor-directory:crawl-log - key network_user_logs monitor-directory:network-user-logs - key computer_id slap-configuration:computer - diff --git a/software/monitor/instance-monitor.cfg.jinja2 b/software/monitor/instance-monitor.cfg.jinja2 index abedcc610..52ae91837 100644 --- a/software/monitor/instance-monitor.cfg.jinja2 +++ b/software/monitor/instance-monitor.cfg.jinja2 @@ -8,7 +8,6 @@ parts = symlink-re6st-logs symlink-collected-logs monitor-collect-csv-wrapper - fluentd-wrapper monitor-base monitor-check-memory-usage monitor-check-cpu-usage @@ -24,7 +23,10 @@ offline = true recipe = slapos.cookbook:cron.d name = network-bench-test frequency = */10 * * * * -command = {{ buildout }}/networkbench ${network-bench-configuration:rendered} ${monitor-directory:monitor-log} +# skip to not fill cron log file. +# command = {{ buildout_bin }}/networkbench ${network-bench-configuration:rendered} ${monitor-directory:monitor-log} +command = true + [symlink-re6st-logs] recipe = cns.recipe.symlink @@ -63,8 +65,7 @@ monitor-log = ${:private}/monitor-log cache = ${:var}/cache mod-ssl = ${:cache}/httpd_mod_ssl system-log = ${:private}/system-log -fluentd-log = ${:log}/fluentd -consumption = ${:fluentd-log}/consumption +consumption = ${:log}/consumption [monitor-httpd-configuration-file] context = @@ -76,28 +77,10 @@ context = section slave_information slap-configuration key monitor_private_hash slap-configuration:private-hash -[fluentd-wrapper] -recipe = slapos.cookbook:wrapper -command-line = {{ fluentd_location }}/bin/fluentd -qq -l ${monitor-directory:log}/fluend.log -c ${fluentd-conf-configuration:rendered} -wrapper-path = ${monitor-directory:service}/fluentd -environment = - GEM_PATH={{ fluentd_location }}/lib/ruby/gems/1.8/ - -[fluentd-conf-configuration] -recipe = slapos.recipe.template:jinja2 -template = {{ fluent_conf_output }} -rendered = ${monitor-directory:etc}/fluentd.cfg -mode = 0744 -context = - key slapparameter_dict slap-configuration:configuration - key slave_instance_list slap-configuration:slave-instance-list - key fluentd_log_directory monitor-directory:fluentd-log - key computer_id slap-connection:computer-id - [monitor-collect-csv-wrapper] recipe = slapos.cookbook:wrapper command-line = - ${monitor-directory:bin}/python {{ monitor_collect_csv_dump }} --output_folder ${monitor-directory:fluentd-log}/consumption/ + ${monitor-directory:bin}/python {{ monitor_collect_csv_dump }} --output_folder ${monitor-directory:consumption} wrapper-path = ${monitor-directory:reports}/monitor-collect-csv-dump [monitor-check-cpu-usage] diff --git a/software/monitor/instance.cfg b/software/monitor/instance.cfg index f2e829748..6eadb8445 100644 --- a/software/monitor/instance.cfg +++ b/software/monitor/instance.cfg @@ -15,14 +15,12 @@ recipe = slapos.recipe.template:jinja2 template = ${template-monitor:destination} rendered = $${buildout:directory}/template-base-monitor.cfg extensions = jinja2.ext.do -context = key buildout buildout:bin-directory - key develop_eggs_directory buildout:develop-eggs-directory +context = key develop_eggs_directory buildout:develop-eggs-directory key eggs_directory buildout:eggs-directory key slapparameter_dict slap-configuration:configuration + raw buildout_bin ${buildout:bin-directory} raw monitor_template_output ${monitor-template:output} - raw network_benck_cfg_output ${network-bench-cfg:output} - raw fluentd_location ${fluentd:location} - raw fluent_conf_output ${fluentd-agent-conf:output} + raw network_benck_cfg_output ${network-bench-cfg:output} raw monitor_collect_csv_dump ${monitor-collect-csv-dump:output} raw monitor_check_system_health ${monitor-system-health:output} mode = 0644 @@ -33,15 +31,13 @@ template = ${template-monitor-distributor:destination} rendered = $${buildout:directory}/template-monitor-base-distributor.cfg extensions = jinja2.ext.do context = import json_module json - key buildout buildout:bin-directory key develop_eggs_directory buildout:develop-eggs-directory key eggs_directory buildout:eggs-directory key slapparameter_dict slap-configuration:configuration key instance_base_monitor instance-base-monitor:rendered key slave_instance_list slap-configuration:slave-instance-list + raw buildout_bin ${buildout:bin-directory} raw template_json_distributor_test ${json-test-template:destination} - raw fluentd_location ${fluentd:location} - raw fluent_distributor_conf_output ${fluentd-agent-distributor-conf:output} raw wget_bin ${wget:location}/bin/wget mode = 0644 diff --git a/software/monitor/software.cfg b/software/monitor/software.cfg index 91f9d0d15..072d61553 100644 --- a/software/monitor/software.cfg +++ b/software/monitor/software.cfg @@ -5,7 +5,6 @@ extends = ../../component/python-cryptography/buildout.cfg ../../component/wget/buildout.cfg ../../stack/monitor/buildout.cfg - ../../component/fluentd/buildout.cfg ../../stack/slapos.cfg parts = @@ -16,30 +15,27 @@ parts = template template-monitor-distributor template-monitor - fluentd - fluentd-agent-conf - fluentd-agent-distributor-conf monitor-collect-csv-dump [template] recipe = slapos.recipe.template url = ${:_profile_base_location_}/instance.cfg output = ${buildout:directory}/template.cfg -md5sum = 3ff5fb2710bf0ea84632c6d6d3894dd9 +md5sum = 641c5916739f78171c616af00fe974a2 mode = 0644 [template-monitor] recipe = slapos.recipe.build:download url = ${:_profile_base_location_}/instance-monitor.cfg.jinja2 destination = ${buildout:directory}/template-base-monitor.cfg -md5sum = 5def53c8faa0dfca313e53ebcc3229fa +md5sum = 5cd3fe3edb4a7fbbc9671f051845d752 mode = 0644 [template-monitor-distributor] recipe = slapos.recipe.build:download url = ${:_profile_base_location_}/instance-monitor-distributor.cfg.jinja2 destination = ${buildout:directory}/template-monitor-base-distributor.cfg -md5sum = 0ab9a86df18125335ed365bb673bbcf3 +md5sum = 61c0bfdfc0a2b51ba15fe4a49baf6091 mode = 0644 [json-test-template] @@ -56,20 +52,6 @@ md5sum = cfcbf2002b8eff5153e2bf68ed24b720 output = ${buildout:directory}/template-network-bench-cfg.in mode = 0644 -[fluentd-agent-conf] -recipe = slapos.recipe.template -url = ${:_profile_base_location_}/fluentd-agent.conf.jinja2.in -md5sum = 3ea59906937eab7aeef78f46c4994ecd -output = ${buildout:directory}/fluentd-agent.conf.jinja2.in -mode = 0644 - -[fluentd-agent-distributor-conf] -recipe = slapos.recipe.template -url = ${:_profile_base_location_}/fluentd-distributor-agent.conf.jinja2.in -md5sum = 4b81ddcbe2f16d23013caac37151d396 -output = ${buildout:directory}/fluentd-agent-distributor.conf.jinja2.in -mode = 0644 - [monitor-collect-csv-dump] <= monitor-template-script url = ${:_profile_base_location_}/script/${:filename} @@ -104,12 +86,6 @@ scripts = monitor.configwrite is-process-older-than-dependency-set -[fluentd] -gems += - fluent-plugin-wendelin==0.1.alpha1 - fluent-plugin-grep==0.3.4 - fluent-plugin-bin==0.1 - [monitor-eggs] eggs += slapos.cookbook -- 2.30.9