Skip to content

S
slapos
Commit 31e44cc5 authored by Jérome Perrin's avatar Jérome Perrin
Browse files
Options

grafana

parent aa18f32e
Show whitespace changes
Inline Side-by-side
Showing with 253 additions and 176 deletions
software/grafana/buildout.hash.cfg
View file @ 31e44cc5
...@@ -15,7 +15,7 @@ ...@@ -15,7 +15,7 @@
[instance-profile] [instance-profile]
filename = instance.cfg.in filename = instance.cfg.in
md5sum = 33e6510d84c83a46a2edf217b4f1beb5 md5sum = 6ff69aef5a7185f880359a78965e072e
[influxdb-config-file] [influxdb-config-file]
filename = influxdb-config-file.cfg.in filename = influxdb-config-file.cfg.in
......
software/grafana/instance-input-schema.json
View file @ 31e44cc5
{ {
"$schema": "http://json-schema.org/draft-07/schema#", "$schema": "https://json-schema.org/draft/2020-12/schema",
"description": "Parameters to instantiate Grafana", "description": "Parameters to instantiate Grafana",
"type": "object", "type": "object",
"additionalProperties": false, "additionalProperties": false,
"$defs": {
"type": {
"description": "Type of the application. With `SlapOS` type, some metrics are collected from supervisor and from some known partition types (for example: ERP5's mariadb or ERP5's zopes). With `system` type, only log files are ingested.",
"type": "string",
"default": "SlapOS",
"enum": [
"SlapOS",
"system"
]
},
"name": {
"description": "Name of this application",
"type": "string"
},
"urls": {
"description": "URLs to monitor for availability and certificate lifetime",
"type": "array",
"items": {
"type": "string"
}
},
"log-file-patterns": {
"type": "array",
"items": {
"type": "string"
},
"description": "Glob patterns for watched log files."
},
"static-tags": {
"type": "object",
"description": "Static tags for this partition",
"examples": [
{
"region": "eu",
"data-center": "abc123"
}
]
}
},
"properties": { "properties": {
"smtp-server": { "smtp-server": {
"description": "SMTP server used by grafana to send emails (in host:port format). Leaving this empty will disable email sending.", "description": "SMTP server used by Grafana to send emails (in host:port format). Leaving this empty will disable email sending.",
"type": "string" "type": "string"
}, },
"smtp-username": { "smtp-username": {
...@@ -33,33 +72,41 @@ ...@@ -33,33 +72,41 @@
"description": "Applications to monitor", "description": "Applications to monitor",
"type": "array", "type": "array",
"items": { "items": {
"oneOf": [
{
"type": "object", "type": "object",
"additionalProperties": false,
"description": "Configuration for SlapOS type application",
"required": [ "required": [
"type",
"name", "name",
"instance-root", "instance-root",
"partitions" "partitions"
], ],
"properties": { "properties": {
"type": {
"$ref": "#/$defs/type",
"const": "SlapOS"
},
"name": { "name": {
"description": "Name of this application", "$ref": "#/$defs/name"
"type": "string" },
"urls": {
"$ref": "#/$defs/urls"
}, },
"instance-root": { "instance-root": {
"description": "Directory containing SlapOS partitions.", "description": "Directory containing SlapOS partitions.",
"type": "string" "type": "string"
}, },
"urls": {
"description": "URLs to monitor for availability and certificate lifetime",
"type": "array",
"items": {
"type": "string"
}
},
"partitions": { "partitions": {
"description": "SlapOS partitions to monitor", "description": "SlapOS partitions to monitor",
"type": "array", "type": "array",
"items": { "items": {
"type": "object", "type": "object",
"required": [
"name",
"reference"
],
"properties": { "properties": {
"name": { "name": {
"type": "string", "type": "string",
...@@ -79,7 +126,7 @@ ...@@ -79,7 +126,7 @@
}, },
"type": { "type": {
"type": "string", "type": "string",
"description": "Type of the partition. Known types have extra metrics and logs collected", "description": "Type of the partition. Known types have metrics and logs collected",
"enum": [ "enum": [
"erp5/mariadb", "erp5/mariadb",
"erp5/balancer", "erp5/balancer",
...@@ -88,78 +135,108 @@ ...@@ -88,78 +135,108 @@
"erp5/zeo", "erp5/zeo",
"mariadb", "mariadb",
"default" "default"
] ],
"default": "default"
}, },
"file-path": { "log-file-patterns": {
"type": "string", "$refs": "#/$defs/log-file-patterns",
"description": "Glob for the files to watch. This mostly makes sense for `default` type" "description": "Glob pattern for log files to watch. This mostly makes sense for `default` partition type"
}, },
"static-tags": { "static-tags": {
"type": "object", "$refs": "#/$defs/static-tags"
"description": "Static tags for this partition", }
},
"examples": [ "examples": [
{ {
"region": "eu", "name": "zope-backoffice",
"data-center": "abc123" "type": "erp5/zope-front",
} "reference": "slappart1",
] "static-tags": {
"instance": "instance-name"
} }
}, },
"anyOf": [
{ {
"properties": { "name": "mariadb",
"type": { "type": "erp5/mariadb",
"const": "default" "reference": "slappart2"
}
}, },
"required": [ {
"name", "name": "Theia",
"file-path" "type": "default",
"log-file-patterns": [
".slappart*log"
]
}
] ]
}
}
}
}, },
{ {
"type": "object",
"description": "Configuration for `system` type application",
"required": [
"type",
"name"
],
"properties": { "properties": {
"type": { "type": {
"not": { "$ref": "#/$defs/type",
"const": "default" "const": "system"
}
}
}, },
"required": [ "name": {
"name", "$ref": "#/$defs/name"
"reference" },
"urls": {
"$ref": "#/$defs/urls"
},
"partitions": {
"type": "array",
"items": {
"type": "object",
"properties": {
"name": {
"type": "string",
"description": "Friendly name of the partition",
"examples": [
"syslog",
"email"
] ]
},
"log-file-patterns": {
"$refs": "#/$defs/log-file-patterns"
},
"static-tags": {
"$refs": "#/$defs/static-tags"
} }
], },
"examples": [ "examples": [
{ {
"name": "zope-backoffice", "name": "syslog",
"type": "erp5/zope-front", "log-file-patterns": [
"reference": "slappart1", "/var/log/syslog"
"static-tags": { ]
"instance": "instance-name"
}
}, },
{ {
"name": "mariadb", "name": "kernel",
"type": "erp5/mariadb", "log-file-patterns": [
"reference": "slappart2" "/var/log/kern.log",
"/var/log/messages"
]
}, },
{ {
"name": "syslog", "name": "re6stnet",
"type": "default", "log-file-patterns": [
"file-path": "/var/log/syslog" "/var/log/re6stnet/*.log"
]
} }
] ]
} }
} }
} }
} }
}, ]
"promtail-extra-scrape-config": { }
"description": "Raw promtail config (experimental parameter, see https://github.com/grafana/loki/blob/v0.3.0/docs/promtail.md#scrape-configs for detail)",
"default": "",
"type": "string"
} }
} }
} }
software/grafana/instance.cfg.in
View file @ 31e44cc5
...@@ -251,7 +251,7 @@ init = ...@@ -251,7 +251,7 @@ init =
inputs = collections.defaultdict(list) inputs = collections.defaultdict(list)
processors = collections.defaultdict(list) processors = collections.defaultdict(list)
slapparameter_dict = self.options["slapparameter-dict"] slapparameter_dict = self.options["slapparameter-dict"]
for application in slapparameter_dict.get('applications', []): for application in slapparameter_dict.get("applications", []):
partition_mapping = {} partition_mapping = {}
for partition in application.get("partitions", []): for partition in application.get("partitions", []):
partition.setdefault("type", "default") partition.setdefault("type", "default")
...@@ -351,15 +351,14 @@ init = ...@@ -351,15 +351,14 @@ init =
"tags": {"app": application["name"]}, "tags": {"app": application["name"]},
}) })
if application.get("type") == "SlapOS":
# TODO: don't run more than one per instance_root
telegraf_slapos_input_config_file = os.path.join( telegraf_slapos_input_config_file = os.path.join(
self.options['location'], self.options['location'],
f"telegraf-input-slapos-{application['name']}.cfg") f"telegraf-input-slapos-{application['name']}.cfg")
self._config_files[telegraf_slapos_input_config_file] = toml.dumps({ self._config_files[telegraf_slapos_input_config_file] = toml.dumps({
"inputs": { "inputs": {
"slapos": [{ "slapos": [{
"instance_root": application['instance-root']}]}}) "instance_root": application["instance-root"]}]}})
# TODO: supervisor process finder for # TODO: supervisor process finder for
# https://github.com/influxdata/telegraf/tree/master/plugins/inputs/procstat ? # https://github.com/influxdata/telegraf/tree/master/plugins/inputs/procstat ?
...@@ -390,7 +389,7 @@ init = ...@@ -390,7 +389,7 @@ init =
processors["enum"].append({ processors["enum"].append({
"namepass": [ f"{application['name']}-processes"], "namepass": [ f"{application['name']}-processes"],
"mapping": [{ "mapping": [{
# "tag": "group", # TODO: rename this in input plugin # "tag": "group", # TODO: rename this in input plugin # XXX I don't remember what this means
"tag": "slappart", "tag": "slappart",
"dest": "partition", "dest": "partition",
"value_mappings": partition_mapping, "value_mappings": partition_mapping,
...@@ -404,6 +403,7 @@ init = ...@@ -404,6 +403,7 @@ init =
# - [ ] alert dashboard # - [ ] alert dashboard
# - [ ] inclu "jerome-dev" partout ??? # - [ ] inclu "jerome-dev" partout ???
# - [ ] apdex # - [ ] apdex
# - [ ] "job" is bad name in Explore
options["extra-config"] = toml.dumps({ options["extra-config"] = toml.dumps({
"inputs": inputs, "inputs": inputs,
...@@ -497,8 +497,6 @@ command-line = ...@@ -497,8 +497,6 @@ command-line =
wrapper-path = ${directory:service}/${:_buildout_section_name_} wrapper-path = ${directory:service}/${:_buildout_section_name_}
extra-command-line = extra-command-line =
# level=error ts=2022-09-24T14:40:13.636615531Z caller=scheduler_processor.go:182 org_id=fake msg="error notifying frontend about finished query" err="rpc error: code = ResourceExhausted desc = grpc: received message larger than max (4200411 vs. 4194304)" frontend=10.0.44.65:9099
[loki-listen-promise-macro] [loki-listen-promise-macro]
<= check-url-available-promise <= check-url-available-promise
...@@ -703,6 +701,7 @@ install = ...@@ -703,6 +701,7 @@ install =
}, },
{ {
"regex": { "regex": {
# TODO don't include the ----
"expression": "^------\\n(?P<timestamp>\\d{4}-\\d{2}-\\d{2}\\s\\d{1,2}\\:\\d{2}\\:\\d{2}\\,\\d{3}) (?P<level>\\S+) (?P<component>\\S+) (?P<message>.*)" "expression": "^------\\n(?P<timestamp>\\d{4}-\\d{2}-\\d{2}\\s\\d{1,2}\\:\\d{2}\\:\\d{2}\\,\\d{3}) (?P<level>\\S+) (?P<component>\\S+) (?P<message>.*)"
} }
}, },
...@@ -714,9 +713,7 @@ install = ...@@ -714,9 +713,7 @@ install =
}, },
{ {
"labels": { "labels": {
"level": None, "level": None
# XXX do we really want `component` ? it may cause lots of cardinality
# "component": None
} }
} }
] ]
...@@ -873,7 +870,6 @@ install = ...@@ -873,7 +870,6 @@ install =
{ {
"labels": { "labels": {
"level": None, "level": None,
"component": None
} }
} }
] ]
......
software/grafana/software.cfg
View file @ 31e44cc5
...@@ -22,12 +22,11 @@ parts = ...@@ -22,12 +22,11 @@ parts =
loki-config-file loki-config-file
loki-nginx-config-file loki-nginx-config-file
[nodejs] ; [nodejs]
<= nodejs-16.14.0 ; <= nodejs-16.19.0
[gowork] [gowork]
golang = ${golang1.19:location}
# XXX speed up development cycle by not rebuilding workspace on every software run # XXX speed up development cycle by not rebuilding workspace on every software run
# XXX does not work ? # XXX does not work ?
update-command = update-command =
...@@ -36,14 +35,13 @@ update-command = ...@@ -36,14 +35,13 @@ update-command =
<= go-git-package <= go-git-package
go.importpath = github.com/grafana/grafana go.importpath = github.com/grafana/grafana
repository = https://github.com/grafana/grafana repository = https://github.com/grafana/grafana
revision = v9.1.5-0-gdf015a9301 revision = v10.1.2-0-g8e428858dd
[go_github.com_grafana_loki] [go_github.com_grafana_loki]
<= go-git-package <= go-git-package
go.importpath = github.com/grafana/loki go.importpath = github.com/grafana/loki
repository = https://github.com/grafana/loki repository = https://github.com/grafana/loki
revision = v2.1.0-2075-gafd63c598 revision = v2.9.1-0-gd9d5ed4a1
# tag helm-loki-3.1.0 which supports golang 1.19
[go_github.com_influxdata_influxdb] [go_github.com_influxdata_influxdb]
<= go-git-package <= go-git-package
...@@ -55,7 +53,7 @@ revision = v1.8.4-0-gbc8ec4384e ...@@ -55,7 +53,7 @@ revision = v1.8.4-0-gbc8ec4384e
<= go-git-package <= go-git-package
go.importpath = github.com/influxdata/telegraf go.importpath = github.com/influxdata/telegraf
repository = https://github.com/influxdata/telegraf repository = https://github.com/influxdata/telegraf
revision = v1.24.0-0-g3c4a6516e revision = v1.28.1-0-g3ea9ffbe2
[go_github.com_perrinjerome_slapos_telegraf_input] [go_github.com_perrinjerome_slapos_telegraf_input]
<= go-git-package <= go-git-package
...@@ -63,6 +61,13 @@ go.importpath = github.com/perrinjerome/telegraf-input-slapos ...@@ -63,6 +61,13 @@ go.importpath = github.com/perrinjerome/telegraf-input-slapos
repository = https://github.com/perrinjerome/telegraf-input-slapos repository = https://github.com/perrinjerome/telegraf-input-slapos
revision = v0.0.1-0-gf8981f3 revision = v0.0.1-0-gf8981f3
[go_github.com_prometheus_prometheus]
<= go-git-package
go.importpath = github.com/prometheus/prometheus
repository = https://github.com/prometheus/prometheus
revision = v0.41.0-0-gc0d8a56c6
# [go_github.com_jaegertracking_jaeger] # [go_github.com_jaegertracking_jaeger]
# <= go-git-package # <= go-git-package
# go.importpath = github.com/jaegertracking/jaeger # go.importpath = github.com/jaegertracking/jaeger
...@@ -70,8 +75,6 @@ revision = v0.0.1-0-gf8981f3 ...@@ -70,8 +75,6 @@ revision = v0.0.1-0-gf8981f3
# revision = v1.20.0-623-gcac21f82 # revision = v1.20.0-623-gcac21f82
[gowork] [gowork]
# Fails with current default golang1.18
golang = ${golang1.17:location}
install = install =
${go_github.com_grafana_loki:location}:./cmd/loki ${go_github.com_grafana_loki:location}:./cmd/loki
${go_github.com_grafana_loki:location}:./clients/cmd/promtail ${go_github.com_grafana_loki:location}:./clients/cmd/promtail
...@@ -79,6 +82,7 @@ install = ...@@ -79,6 +82,7 @@ install =
${go_github.com_influxdata_telegraf:location}:./cmd/... ${go_github.com_influxdata_telegraf:location}:./cmd/...
${go_github.com_influxdata_influxdb:location}:./cmd/... ${go_github.com_influxdata_influxdb:location}:./cmd/...
${go_github.com_perrinjerome_slapos_telegraf_input:location}:./... ${go_github.com_perrinjerome_slapos_telegraf_input:location}:./...
${go_github.com_prometheus_prometheus:location}:./cmd/...
# disable cgo, to prevent loki/promtail from using go-systemd # disable cgo, to prevent loki/promtail from using go-systemd
environment = environment =
...@@ -101,7 +105,7 @@ command = bash -ce " ...@@ -101,7 +105,7 @@ command = bash -ce "
. ${gowork:env.sh} && \ . ${gowork:env.sh} && \
go install github.com/google/wire/cmd/wire@v0.5.0 && \ go install github.com/google/wire/cmd/wire@v0.5.0 && \
wire gen -tags oss ./pkg/server ./pkg/cmd/grafana-cli/runner && \ wire gen -tags oss ./pkg/server ./pkg/cmd/grafana-cli/runner && \
# Unlike the loki, grafana _needs_ CGO_ENABLED, so we override here # Unlike loki, grafana _needs_ CGO_ENABLED, so we override here
export CGO_ENABLED=1 && \ export CGO_ENABLED=1 && \
go run build.go setup && \ go run build.go setup && \
go run build.go build && \ go run build.go build && \
......
software/grafana/test/test.py
View file @ 31e44cc5
...@@ -82,7 +82,7 @@ class TestGrafana(GrafanaTestCase): ...@@ -82,7 +82,7 @@ class TestGrafana(GrafanaTestCase):
self.assertEqual(requests.codes.ok, resp.status_code) self.assertEqual(requests.codes.ok, resp.status_code)
self.assertEqual(1, resp.json()['id']) self.assertEqual(1, resp.json()['id'])
def test_grafana_datasource_povisinonned(self): def test_grafana_datasource_provisioned(self):
# data sources are provisionned # data sources are provisionned
connection_params = self.computer_partition.getConnectionParameterDict() connection_params = self.computer_partition.getConnectionParameterDict()
resp = requests.get( resp = requests.get(
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7