Monitor: Implement log split and upload to wendelin on fluentd

software/monitor/fluentd-agent.conf.jinja2.in

@@ -18,15 +18,93 @@
   type stdout
 </match>
 
+<source>
+  type tail
+  path {{ crawl_log_directory }}/*/*ping.log.20*
+  pos_file {{ crawl_log_directory }}/tail_in_ping.pos
+  tag slapos.monitor.networktest.ping.ipv4
+  format /^(?<time>[^;]*);(?<logtype>[^;]*);(?<computer_name>[^;]*);(?<type>[^;]*);(?<name_or_ip>[^;]*);(?<code>[^;]*);(?<average>[^;]*);(?<packet_lost>[^;]*);(?<extra>[^;]*)$/
+  read_from_head true
+</source>
 
-## SOURCE
 <source>
-  type forward
+  type tail
+  path {{ crawl_log_directory }}/*/*ping6.log.20*
+  pos_file {{ crawl_log_directory }}/tail_in_ping6.pos
+  tag slapos.monitor.networktest.ping.ipv6
+  format /^(?<time>[^;]*);(?<logtype>[^;]*);(?<computer_name>[^;]*);(?<type>[^;]*);(?<name_or_ip>[^;]*);(?<code>[^;]*);(?<average>[^;]*);(?<packet_lost>[^;]*);(?<extra>[^;]*)$/
+  read_from_head true
+</source>
+
+<match slapos.monitor.networktest.ping.*>
+  type copy
+{% for slave_instance in slave_instance_list -%}
+{% if  slave_instance.get("ping_ip_list") -%} 
+  <store>
+    type grep
+    regexp1 name_or_ip ^{{ slave_instance.get("ping_ip_list") }}$
+    add_tag_prefix {{ slave_instance.get("slave_reference") }}
+  </store>
+{% endif -%}
+{% if  slave_instance.get("ping6_ip_list") -%}
+  <store>
+    type grep
+    regexp1 name_or_ip ^{{ slave_instance.get("ping6_ip_list") }}$
+    add_tag_prefix {{ slave_instance.get("slave_reference") }}
+  </store>
+{% endif -%}
+{% endfor -%}
+</match>
+
+{% for slave_instance in slave_instance_list -%}
+<match {{ slave_instance.get("slave_reference") }}.slapos.monitor.networktest.ping.ipv6>
+  type file
+  path {{ network_user_logs }}/{{ slave_instance.get("slave_reference") }}/ping6/log
+  append true
+  include_time_key true
+  include_tag_key true
+  format json
+</match>
+<match {{ slave_instance.get("slave_reference") }}.slapos.monitor.networktest.ping.ipv4>
+  type file
+  path {{ network_user_logs }}/{{ slave_instance.get("slave_reference") }}/ping/log
+  append true
+  include_time_key true
+  include_tag_key true
+  format json
+</match>
+{% endfor -%}
+
+<source>
+  type tail
+  path {{ network_user_logs }}/*/ping*/log/*.log
+  pos_file {{ crawl_log_directory }}/tail_in_ping6.pos
+  tag slapos.wendelin.networktest
+  format json 
+  read_from_head true
 </source>
 
+{% set wendelin_streamtool_uri = slapparameter_dict.get('wendelin-streamtool-uri', '')  -%}
+{% set wendelin_password = slapparameter_dict.get('wendelin-password', '')  -%}
+{% set wendelin_user = slapparameter_dict.get('wendelin-user', '')  -%}
+{% if wendelin_streamtool_uri and wendelin_password and wendelin_user -%}
+<match slapos.wendelin.networktest>
+  
+  @type wendelin
+  @id wendelin_out
+
+  streamtool_uri {{ wendelin_streamtool_uri }} 
+  user      {{ wendelin_user }}
+  password  {{ wendelin_password }}
+
+  buffer_type       memory
+  flush_interval    20s
+</match>
+{% endif -%}
+
+## SOURCE
 <source>
-  type http
-  port 8888
+  type forward
 </source>
 
 ## live debugging agent

software/monitor/instance-monitor-distributor.cfg.jinja2

-[buildout]
-
-extends = {{ instance_base_monitor }}
-parts += 
-  slave-test-configuration
 
 [jinja2-template-base]
 recipe = slapos.recipe.template:jinja2
@@ -20,16 +15,11 @@ extensions = jinja2.ext.do
 extra-context = 
   section slave_information slap-parameter
 
-[monitor-directory]
-fluentd-log = ${:log}/fluentd
-network-user-logs = ${:private-directory}/network-user-logs/
-{% for slave_instance in slave_instance_list -%}
-user-log-{{ slave_instance.get('slave_reference') }}-folder = ${:private-directory}/network-user-logs/{{ slave_instance.get('slave_reference') }} 
-{% endfor -%}
-
 {% set part_list = [] -%}
 # Publish information for each slave
 
+{% set directory_list = [] -%}
+
 {% for slave_instance in slave_instance_list -%}
 {%   set publish_section_title = 'publish-%s' % slave_instance.get('slave_reference')  -%}
 {%   do part_list.append(publish_section_title) -%}
@@ -40,8 +30,50 @@ log-access-url = ${monitor-frontend:connection-site_url}/{{ slave_instance.get('
 log-access-url-v6 = ${monitor-parameters:url}/{{ slave_instance.get('slave_reference') }}
 {% endfor %}
 
+
+{% set data_source_dict = slapparameter_dict.get('data-source', None)  -%}
+{% set cron_min_count = 0 -%}
+{% if data_source_dict %}
+{% for entry in data_source_dict -%}
+{% set cron_min_count = cron_min_count + 1 -%}
+{% set cron_min = cron_min_count%60 -%}
+{% do part_list.append('cron-crawl-' + entry) -%}
+{% do directory_list.append(entry) -%}
+[cron-crawl-{{ entry }}]
+<= cron
+recipe = slapos.cookbook:cron.d
+name = cron-crawler-{{ entry }}
+frequency = * * * * *
+command = cd ${monitor-directory:crawl-log}/{{ entry }} && ${crawler-bin:wrapper-path} {{ data_source_dict.get(entry) }} 
+{% endfor %}
+{% endif %}
+
+[monitor-directory]
+fluentd-log = ${:log}/fluentd
+crawl-log = ${:srv}/crawlog
+network-user-logs = ${:private-directory}/network-user-logs/
+{% for slave_instance in slave_instance_list -%}
+user-log-{{ slave_instance.get('slave_reference') }}-folder = ${:private-directory}/network-user-logs/{{ slave_instance.get('slave_reference') }}
+user-log-{{ slave_instance.get('slave_reference') }}-ping-folder = ${:private-directory}/network-user-logs/{{ slave_instance.get('slave_reference') }}/ping
+user-log-{{ slave_instance.get('slave_reference') }}-ping6-folder = ${:private-directory}/network-user-logs/{{ slave_instance.get('slave_reference') }}/ping6
+
+{% endfor -%}
+{% for directory in directory_list %}
+{{ '%s = ${:crawl-log}/%s' % (directory, directory) }}
+{% endfor %}
+
+
+[crawler-bin]
+recipe = slapos.cookbook:wrapper
+command-line = 
+  {{ wget_bin }} --no-check-certificate -l1 -r -nd --timestamp  
+wrapper-path = ${monitor-directory:bin}/log-crawler
+parameters-extra = true
+
 [buildout]
+extends = {{ instance_base_monitor }}
 parts +=
+  slave-test-configuration
   fluentd-wrapper
 {% for part in part_list %}
 {{ '  %s' % part }}
@@ -60,5 +92,17 @@ template = {{ fluent_conf_output }}
 rendered = ${monitor-directory:etc}/fluentd.cfg
 mode = 0744
 context =
-  key slapparameter_dict slap-parameters:configuration
+  key slapparameter_dict slap-configuration:configuration
+  key slave_instance_list slap-parameters:slave-instance-list
   key fluentd_log_directory monitor-directory:fluentd-log
+  key crawl_log_directory monitor-directory:crawl-log
+  key network_user_logs monitor-directory:network-user-logs
+
+[slap-configuration]
+recipe = slapos.cookbook:slapconfiguration.serialised
+computer = ${slap-connection:computer-id}
+partition = ${slap-connection:partition-id}
+url = ${slap-connection:server-url}
+key = ${slap-connection:key-file}
+cert = ${slap-connection:cert-file}
+

software/monitor/instance.cfg

@@ -19,8 +19,8 @@ context = key buildout buildout:bin-directory
           key develop_eggs_directory buildout:develop-eggs-directory
           key eggs_directory buildout:eggs-directory
           key slapparameter_dict slap-configuration:configuration
-          key monitor_template_output slap-configuration:monitor_template_output
-          key network_benck_cfg_output slap-configuration:network_benck_cfg_output 
+          raw monitor_template_output ${monitor-template:output}
+          raw network_benck_cfg_output ${network-bench-cfg:output} 
 mode = 0644
 
 [instance-base-distributor]
@@ -35,20 +35,16 @@ context = import json_module json
           key slapparameter_dict slap-configuration:configuration
           key instance_base_monitor instance-base-monitor:rendered
           key slave_instance_list slap-configuration:slave-instance-list 
-          key template_json_distributor_test slap-configuration:json_test_template
-          key fluent_conf_output slap-configuration:fluentd_conf_output
+          raw template_json_distributor_test ${json-test-template:destination} 
+          raw fluent_conf_output ${fluentd-agent-conf:output}
           raw fluentd_location ${fluentd:location}
+          raw wget_bin ${wget:location}/bin/wget
 mode = 0644
 
 [slap-configuration]
-recipe = slapos.cookbook:slapconfiguration
+recipe = slapos.cookbook:slapconfiguration.serialised
 computer = $${slap-connection:computer-id}
 partition = $${slap-connection:partition-id}
 url = $${slap-connection:server-url}
 key = $${slap-connection:key-file}
 cert = $${slap-connection:cert-file}
-monitor_template_output = ${monitor-template:output}
-network_benck_cfg_output = ${network-bench-cfg:output}
-json_test_template = ${json-test-template:destination}
-fluentd_conf_output = ${fluentd-agent-conf:output}
-

software/monitor/software.cfg

@@ -2,11 +2,13 @@
 
 extends =
   ../../component/pycurl/buildout.cfg
+  ../../component/wget/buildout.cfg
   ../../stack/monitor/buildout.cfg
   ../../component/fluentd/buildout.cfg
   ../../stack/slapos.cfg
 
 parts = 
+  wget
   slapos-cookbook
   slapos-toolbox
   network-bench-cfg
@@ -22,7 +24,7 @@ parts =
 recipe = slapos.recipe.template 
 url = ${:_profile_base_location_}/instance.cfg
 output = ${buildout:directory}/template.cfg
-md5sum = d3a2cae2823f5f5ce2f7c293018223f3 
+md5sum = b38c5da7dd4165154a8817f76015e374
 mode = 0644
 
 [template-monitor]
@@ -36,7 +38,7 @@ mode = 0644
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/instance-monitor-distributor.cfg.jinja2
 destination = ${buildout:directory}/template-monitor-distributor.cfg
-md5sum = 2f11edbc57eb95f9d5901d183065d166
+md5sum = d1253cf11c2b335b8bec5807e821d537 
 mode = 0644
 
 [json-test-template]
@@ -54,7 +56,7 @@ md5sum = 876f18b159fbd9325332d0f42e9172ac
 
 [monitor-httpd-template]
 url = ${:_profile_base_location_}/${:filename}
-md5sum = 25b5bfc7f9c2891c4a0923d6e08568a8 
+md5sum = 22133e9aa5f52d7818b2fd1fd0415a00 
 
 [network-bench-cfg]
 recipe = slapos.recipe.template
@@ -63,11 +65,10 @@ md5sum  = cfcbf2002b8eff5153e2bf68ed24b720
 output = ${buildout:directory}/template-network-bench-cfg.in
 mode = 0644
 
-
 [fluentd-agent-conf]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/fluentd-agent.conf.jinja2.in
-md5sum = 0098143de65630f78be402c65424beb4 
+md5sum = c9da9f22a1cd1fedcf7d659cba916400 
 output = ${buildout:directory}/fluentd-agent.conf.jinja2.in
 mode = 0644
 
@@ -85,6 +86,7 @@ scripts =
 [fluentd]
 gems += 
   fluent-plugin-wendelin==0.1.alpha1
+  fluent-plugin-grep==0.3.4
 
 [eggs]
 recipe = z3c.recipe.scripts
@@ -135,4 +137,3 @@ lockfile = 0.10.2
 # slapos.toolbox==0.52
 rpdb = 0.1.5
 
-