software/grafana: WIP generate telegraf and loki config

software/grafana/buildout.hash.cfg

@@ -15,7 +15,7 @@
 
 [instance-profile]
 filename = instance.cfg.in
-md5sum = 3ccdd2299e759488545b62368c7a0b91
+md5sum = 09a24413839d930420f0ac0298ef3297
 
 [influxdb-config-file]
 filename = influxdb-config-file.cfg.in
@@ -23,20 +23,20 @@ md5sum = a28972ced3e0f4aa776e43a9c44717c0
 
 [telegraf-config-file]
 filename = telegraf-config-file.cfg.in
-md5sum = a1a9c22c2a7829c66a49fc2504604d21
+md5sum = 6de1faa34842e1eda095a51edecc2083
 
 [grafana-config-file]
 filename = grafana-config-file.cfg.in
-md5sum = e255dcca466f5de51698d24cbd114577
+md5sum = 83a8445858eab21a12f1769c23424bea
 
-[grafana-provisioning-config-file]
-filename = grafana-provisioning-config-file.cfg.in
+[grafana-provisioning-datasources-config-file]
+filename = grafana-provisioning-datasources-config-file.cfg.in
 md5sum = 3aa0f1ed752b2a59ea2b5e7c1733daf3
 
+[grafana-provisioning-dashboards-config-file]
+filename = grafana-provisioning-dashboards-config-file.cfg.in
+md5sum = 5616679a9c5c2757540175ead3f5500a
+
 [loki-config-file]
 filename = loki-config-file.cfg.in
 md5sum = ad2baf4599a937d7352034a41fa24814
-
-[promtail-config-file]
-filename = promtail-config-file.cfg.in
-md5sum = c8c9d815dd7b427788c066f041f04573

software/grafana/grafana-config-file.cfg.in

@@ -154,7 +154,7 @@ reporting_enabled = true
 # in some UI views to notify that grafana or plugin update exists
 # This option does not cause any auto updates, nor send any information
 # only a GET request to https://grafana.com to get latest versions
-check_for_updates = true
+check_for_updates = false
 
 # Google Analytics universal tracking code, only enabled if you specify an id here
 google_analytics_ua_id =
@@ -345,11 +345,8 @@ user = {{ slapparameter_dict.get('smtp-username', '') }}
 password = {{ slapparameter_dict.get('smtp-password', '') and '"""%s"""' % slapparameter_dict['smtp-password'] or ""}}
 cert_file =
 key_file =
-#skip_verify = false
-skip_verify = {{ slapparameter_dict.get('smtp-verify-ssl', 'true').lower() == 'true' and 'false' or 'true' }}
-#from_address = admin@grafana.localhost
+skip_verify = {{ slapparameter_dict.get('smtp-verify-ssl') and 'true' or 'false' }}
 from_address = {{ slapparameter_dict.get('email-from-address', '') }}
-#from_name = Grafana
 from_name = {{ slapparameter_dict.get('email-from-name', 'Grafana') }}
 ehlo_identity =
 

software/grafana/grafana-provisioning-dashboards-config-file.cfg.in

+# https://grafana.com/docs/grafana/latest/administration/provisioning/#dashboards
+
+apiVersion: 1
+
+providers:
+  - name: SlapOS
+    folder: ''
+    updateIntervalSeconds: 10
+    allowUiUpdates: false
+    options:
+      path: {{ dashboards_dir }}

software/grafana/instance-input-schema.json

 {
-  "$schema": "http://json-schema.org/draft-04/schema#",
+  "$schema": "http://json-schema.org/draft-07/schema#",
   "description": "Parameters to instantiate Grafana",
   "type": "object",
   "additionalProperties": false,
@@ -18,11 +18,7 @@
     },
     "smtp-verify-ssl": {
       "description": "Verify SSL certificate of SMTP server",
-      "type": "string",
-      "enum": [
-        "true",
-        "false"
-      ]
+      "type": "boolean"
     },
     "email-from-address": {
       "description": "Email address used in From: header of emails",
@@ -33,6 +29,133 @@
       "default": "Grafana",
       "type": "string"
     },
+    "applications": {
+      "description": "Applications to monitor",
+      "type": "array",
+      "items": {
+        "type": "object",
+        "required": [
+          "name",
+          "instance-root",
+          "partitions"
+        ],
+        "properties": {
+          "name": {
+            "description": "Name of this application",
+            "type": "string"
+          },
+          "instance-root": {
+            "description": "Directory containing SlapOS partitions.",
+            "type": "string"
+          },
+          "urls": {
+            "description": "URLs to monitor for availability and certificate lifetime",
+            "type": "array",
+            "items": {
+              "type": "string"
+            }
+          },
+          "partitions": {
+            "description": "SlapOS partitions to monitor",
+            "type": "array",
+            "items": {
+              "type": "object",
+              "properties": {
+                "name": {
+                  "type": "string",
+                  "description": "Friendly name of the partition",
+                  "examples": [
+                    "mariadb",
+                    "zope-activity"
+                  ]
+                },
+                "reference": {
+                  "type": "string",
+                  "description": "Reference of the partition",
+                  "examples": [
+                    "slappart1",
+                    "slappart2"
+                  ]
+                },
+                "type": {
+                  "type": "string",
+                  "description": "Type of the partition. Known types have extra metrics and logs collected",
+                  "enum": [
+                    "erp5/mariadb",
+                    "erp5/balancer",
+                    "erp5/zope-activity",
+                    "erp5/zope-front",
+                    "erp5/zeo",
+                    "mariadb",
+                    "default"
+                  ]
+                },
+                "file-path": {
+                  "type": "string",
+                  "description": "Glob for the files to watch. This mostly makes sense for `default` type"
+                },
+                "static-tags": {
+                  "type": "object",
+                  "description": "Static tags for this partition",
+                  "examples": [
+                    {
+                      "region": "eu",
+                      "data-center": "abc123"
+                    }
+                  ]
+                }
+              },
+              "anyOf": [
+                {
+                  "properties": {
+                    "type": {
+                      "const": "default"
+                    }
+                  },
+                  "required": [
+                    "name",
+                    "file-path"
+                  ]
+                },
+                {
+                  "properties": {
+                    "type": {
+                      "not": {
+                        "const": "default"
+                      }
+                    }
+                  },
+                  "required": [
+                    "name",
+                    "reference"
+                  ]
+                }
+              ],
+              "examples": [
+                {
+                  "name": "zope-backoffice",
+                  "type": "erp5/zope-front",
+                  "reference": "slappart1",
+                  "static-tags": {
+                    "instance": "instance-name"
+                  }
+                },
+                {
+                  "name": "mariadb",
+                  "type": "erp5/mariadb",
+                  "reference": "slappart2"
+                },
+                {
+                  "name": "syslog",
+                  "type": "default",
+                  "file-path": "/var/log/syslog"
+                }
+              ]
+            }
+          }
+        }
+      }
+    },
     "promtail-extra-scrape-config": {
       "description": "Raw promtail config (experimental parameter, see https://github.com/grafana/loki/blob/v0.3.0/docs/promtail.md#scrape-configs for detail)",
       "default": "",

software/grafana/promtail-config-file.cfg.in

-
-server:
-  http_listen_address: {{ promtail['ip'] }}
-  http_listen_port: {{ promtail['http-port'] }}
-  grpc_listen_address: {{ promtail['ip'] }}
-  grpc_listen_port: {{ promtail['grpc-port'] }}
-  external_url: {{ promtail['url'] }}
-
-positions:
-  filename: {{ promtail['dir'] }}/positions.yaml
-
-clients:
-  - url: {{ loki['url'] }}/api/prom/push
-
-scrape_configs:
-- job_name: test
-  static_configs:
-  - targets:
-      - localhost
-    labels:
-      job: grafanalogs
-      __path__: ./var/log/*log
-{{ slapparameter_dict.get('promtail-extra-scrape-config', '') }}

software/grafana/software.cfg

@@ -9,7 +9,6 @@ extends =
   ../../component/dash/buildout.cfg
   buildout.hash.cfg
 
-versions = versions
 parts =
   slapos-cookbook
   instance-profile
@@ -17,9 +16,9 @@ parts =
   influxdb-config-file
   telegraf-config-file
   grafana-config-file
-  grafana-provisioning-config-file
+  grafana-provisioning-datasources-config-file
+  grafana-provisioning-dashboards-config-file
   loki-config-file
-  promtail-config-file
 
 [python]
 part = python3
@@ -75,6 +74,7 @@ environment =
   CGO_ENABLED = 0
 
 telegraf-bin = ${:bin}/telegraf
+telegraf-input-slapos-bin = ${:bin}/telegraf-input-slapos
 influx-bin = ${:bin}/influx
 influxd-bin = ${:bin}/influxd
 grafana-bin = ${:bin}/grafana-server
@@ -115,15 +115,21 @@ url = ${:_profile_base_location_}/${:filename}
 [grafana-config-file]
 <= download-file-base
 
-[grafana-provisioning-config-file]
+[grafana-provisioning-datasources-config-file]
 <= download-file-base
 
-[loki-config-file]
+[grafana-provisioning-dashboards-config-file]
 <= download-file-base
 
-[promtail-config-file]
+[loki-config-file]
 <= download-file-base
 
+[instance-eggs]
+recipe = zc.recipe.egg
+eggs =
+  ${python-PyYAML:egg}
+  toml
+
 [instance-profile]
 recipe = slapos.recipe.template:jinja2
 template = ${:_profile_base_location_}/${:filename}
@@ -134,6 +140,7 @@ context =
   section buildout buildout
   key openssl_bin openssl-output:openssl
   key telegraf_bin gowork:telegraf-bin
+  key telegraf_input_slapos_bin gowork:telegraf-input-slapos-bin
   key influxd_bin gowork:influxd-bin
   key influx_bin gowork:influx-bin
   key grafana_bin gowork:grafana-bin
@@ -144,6 +151,8 @@ context =
   key dash_bin :dash-bin
 curl-bin = ${curl:location}/bin/curl
 dash-bin = ${dash:location}/bin/dash
+depends = ${instance-eggs:eggs}
 
 [versions]
 inotifyx = 0.2.2
+toml = 0.10.2

software/grafana/software.cfg.json

 {
   "name": "Grafana",
   "description": "Grafana, Telegraf and Influxdb",
-  "serialisation": "xml",
+  "serialisation": "json-in-xml",
   "software-type": {
     "default": {
       "title": "Default",

software/grafana/telegraf-config-file.cfg.in

@@ -55,9 +55,6 @@
 [outputs.influxdb]
   # The full HTTP or UDP endpoint URL for your InfluxDB instance
   # Multiple urls can be specified for InfluxDB cluster support.
-  # urls = ["udp://localhost:8089"] # UDP endpoint example
-  # XXX XXX XXX
-  #urls = ["http://localhost:8086"] # required
   urls = ["{{ influxdb['url'] }}"]
   insecure_skip_verify = true # because we are using a self signed certificate
   # The target database for metrics (telegraf will create it if not exists)
@@ -100,32 +97,9 @@
 
 [system]
 
+{{ extra['extra-config'] }}
 
 ###############################################################################
-#                           ERP5 - PLUGINS                                    #
-###############################################################################
-#
-# Left here as example, don't edit this file directly, but place your config
+# To add ad-hoc config, don't edit this file directly, but place your config
 # files in {{ telegraf['extra-config-dir'] }}
-# 
-
-#[mysql]
-#  servers = ["root@unix(/srv/slapgrid/slappart12/srv/runner/instance/slappart1/var/run/mariadb.sock)/erp5"]
-
-#[memcached]
-#  # XXX kumofs does not support memcached's stat command
-#  servers = ["10.0.248.233:2013", "10.0.248.233:2003"]
-
-#[haproxy]
-#  servers = ["http://10.0.121.162:2150/haproxy", "http://10.0.121.162:2152/haproxy"]
-
-#[[inputs.exec]]
-#  commands = ["/srv/slapgrid/slappart0/bin/slapsensor  /srv/slapgrid/slappart0/srv/runner/instance/etc/supervisord.conf"]
-#  name_suffix = "_slapos"
-#  interval = "5s"
-
-
-
-###############################################################################
-#                              SERVICE PLUGINS                                #
 ###############################################################################

software/grafana/test/test.py

@@ -32,6 +32,7 @@ import os
 import tempfile
 import textwrap
 import time
+import json
 
 import psutil
 import requests
@@ -185,8 +186,93 @@ class TestTelegraf(GrafanaTestCase):
 
 
 class TestLoki(GrafanaTestCase):
+  instance_max_retry = 2
   @classmethod
   def getInstanceParameterDict(cls):
+    cls._logfile = tempfile.NamedTemporaryFile(suffix='log')
+    parameter_dict = {
+        "applications": [
+    {
+      "name": "ERP5",
+      "instance-root": "/srv/slapgrid/slappart4/srv/slapos/inst/",
+      "urls": [
+        "https://softinst12345-erp5.host.vifib.net/",
+      ],
+      "partitions": [
+        {
+          "name": "jerome-dev-mariadb",
+          "reference": "slappart6",
+          "type": "erp5/mariadb",
+          #"static-tags": {
+          #  "XXX": "needed?"
+          #}
+        },
+        {
+          "name": "jerome-dev-zodb",
+          "reference": "slappart7",
+          "type": "erp5/zeo",
+          #"static-tags": {
+          #  "XXX": "needed?"
+          #}
+        },
+        {
+          "name": "jerome-dev-balancer",
+          "reference": "slappart9",
+          "type": "erp5/balancer",
+          #"static-tags": {
+          #  "XXX": "needed?"
+          #}
+        },
+        {
+          "name": "jerome-dev-zope-front",
+          "reference": "slappart8",
+          "type": "erp5/zope-front",
+          #"static-tags": {
+          #  "XXX": "needed?"
+          #}
+        },
+        {
+          "name": "jerome-dev-zope-front",
+          "reference": "slappart13",
+          "type": "erp5/zope-activity",
+          #"static-tags": {
+          #  "XXX": "needed?"
+          #}
+        }
+      ]
+    }
+  ],
+  # TODO: drop this
+        'promtail-extra-scrape-config':
+        textwrap.dedent(r'''
+                - job_name: {cls.__name__}
+                  pipeline_stages:
+                    - match:
+                        selector: '{{job="{cls.__name__}"}}'
+                        stages:
+                          - multiline:
+                              firstline: '^\d{{4}}-\d{{2}}-\d{{2}}\s\d{{1,2}}\:\d{{2}}\:\d{{2}}\,\d{{3}}'
+                              max_wait_time: 3s
+                          - regex:
+                              expression: '^(?P<timestamp>.*) - (?P<name>\S+) - (?P<level>\S+) - (?P<message>.*)'
+                          - timestamp:
+                              format: 2006-01-02T15:04:05Z00:00
+                              source: timestamp
+                          - labels:
+                              level:
+                              name:
+                  static_configs:
+                    - targets:
+                        - localhost
+                      labels:
+                        job: {cls.__name__}
+                        __path__: {cls._logfile.name}
+            ''').format(**locals())
+    }
+    return {'_': json.dumps(parameter_dict)}
+
+
+  def xgetInstanceParameterDict(cls):
     cls._logfile = tempfile.NamedTemporaryFile(suffix='log')
     return {
         'promtail-extra-scrape-config':
@@ -227,6 +313,7 @@ class TestLoki(GrafanaTestCase):
     )['loki-url']
 
   def test_loki_available(self):
+
     self.assertEqual(
         requests.codes.ok,
         requests.get('{self.loki_url}/ready'.format(**locals()),