fluentd-distributor-agent.conf.jinja2.in 3.11 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116
# DESTINATION
<match td.*.*>
  type tdlog
  apikey YOUR_API_KEY

  auto_create_table
  buffer_type file
  buffer_path {{ fluentd_log_directory }}/td-agent/buffer/td

  <secondary>
    type file
    path {{ fluentd_log_directory }}/failed_records
  </secondary>
</match>

## match tag=debug.** and dump to console
<match debug.**>
  type stdout
</match>

<source>
  type tail
  path {{ crawl_log_directory }}/*/*ping.log.20*
  pos_file {{ crawl_log_directory }}/tail_in_ping.pos
  tag slapos.monitor.networktest.ping.ipv4
  format /^(?<time>[^;]*);(?<computer_name>[^;]*);(?<type>[^;]*);(?<name_or_ip>[^;]*);(?<code>[^;]*);(?<average>[^;]*);(?<packet_lost>[^;]*);(?<extra>[^;]*)$/
  read_from_head true
</source>

<source>
  type tail
  path {{ crawl_log_directory }}/*/*ping6.log.20*
  pos_file {{ crawl_log_directory }}/tail_in_ping6.pos
  tag slapos.monitor.networktest.ping.ipv6
  format /^(?<time>[^;]*);(?<computer_name>[^;]*);(?<type>[^;]*);(?<name_or_ip>[^;]*);(?<code>[^;]*);(?<average>[^;]*);(?<packet_lost>[^;]*);(?<extra>[^;]*)$/
  read_from_head true
</source>

<match slapos.monitor.networktest.ping.*>
  type copy
{% for slave_instance in slave_instance_list -%}
{% if  slave_instance.get("ping_ip_list") -%} 
  <store>
    type grep
    regexp1 name_or_ip ^{{ slave_instance.get("ping_ip_list") }}$
    add_tag_prefix {{ slave_instance.get("slave_reference") }}
  </store>
{% endif -%}
{% if  slave_instance.get("ping6_ip_list") -%}
  <store>
    type grep
    regexp1 name_or_ip ^{{ slave_instance.get("ping6_ip_list") }}$
    add_tag_prefix {{ slave_instance.get("slave_reference") }}
  </store>
{% endif -%}
{% endfor -%}
</match>

{% for slave_instance in slave_instance_list -%}
<match {{ slave_instance.get("slave_reference") }}.slapos.monitor.networktest.ping.ipv6>
  type file
  path {{ network_user_logs }}/{{ slave_instance.get("slave_reference") }}/ping6/log
  append true
  include_time_key true
  include_tag_key true
  format json
</match>
<match {{ slave_instance.get("slave_reference") }}.slapos.monitor.networktest.ping.ipv4>
  type file
  path {{ network_user_logs }}/{{ slave_instance.get("slave_reference") }}/ping/log
  append true
  include_time_key true
  include_tag_key true
  format json
</match>
{% endfor -%}

<source>
  type tail
  path {{ network_user_logs }}/*/ping*/log/*.log
  pos_file {{ crawl_log_directory }}/tail_in_ping6.pos
  tag slapos.wendelin.networktest
  format json 
  read_from_head true
</source>

{% set wendelin_streamtool_uri = slapparameter_dict.get('wendelin-streamtool-uri', '')  -%}
{% set wendelin_password = slapparameter_dict.get('wendelin-password', '')  -%}
{% set wendelin_user = slapparameter_dict.get('wendelin-user', '')  -%}
{% if wendelin_streamtool_uri and wendelin_password and wendelin_user -%}
<match slapos.wendelin.networktest>
  
  @type wendelin
  @id wendelin_out

  streamtool_uri {{ wendelin_streamtool_uri }} 
  user      {{ wendelin_user }}
  password  {{ wendelin_password }}

  buffer_type       memory
  flush_interval    20s
</match>
{% endif -%}

## SOURCE
<source>
  type forward
</source>

## live debugging agent
<source>
  type debug_agent
  bind 127.0.0.1
  port 24230
</source>