Update Release Candidate

f6a30dcc · Rafael Monnerat · dc7ae622 · 7ce498dd · f6a30dcc · f6a30dcc
Commit f6a30dcc authored Feb 05, 2020 by Rafael Monnerat
26 changed files
--- a/component/caddy/buildout.cfg
+++ b/component/caddy/buildout.cfg
@@ -8,6 +8,8 @@ parts =
 [gowork]
 # Caddy 1.x+ uses go modules, for which gowork does not work yet
 golang  = ${golang1.12:location}
+gcc-bin-directory  = ${golang1.12:gcc-bin-directory}
+
 install =

 [gowork.goinstall]

--- a/component/golang/buildout.cfg
+++ b/component/golang/buildout.cfg
@@ -17,10 +17,12 @@ configure-command = :
 location = @@LOCATION@@
 make-binary =
 make-targets= cd src && ./all.bash && cp -alf .. ${:location}
+# gcc version to use
+gcc-bin-directory = ${gcc-8.2:location}/bin
 # some testdata files have an issue with slapos.extension.strip.
 post-install = ${findutils:location}/bin/find ${:location}/src -type d -name testdata -exec rm -rf {} \; || true
 environment =
-  PATH=${gcc-8.2:location}/bin:%(PATH)s
+  PATH=${:gcc-bin-directory}:%(PATH)s
  GOROOT_FINAL=${:location}
  ${:environment-extra}

@@ -104,6 +106,9 @@ depends = ${gowork.goinstall:recipe}
 # go version used for the workspace (possible to override in applications)
 golang  = ${golang1.10:location}

+# gcc version must be compatible with go version selected
+gcc-bin-directory = ${golang1.10:gcc-bin-directory}
+
 # no special build flags by default
 buildflags =

@@ -123,7 +128,7 @@ recipe	= slapos.recipe.template
 url     = ${:_profile_base_location_}/goenv.sh.in
 output	= ${gowork:directory}/env.sh
 depends = ${gowork.mkdir:recipe}
-md5sum	= 7a067a3974c446c3eaa0e82818ba1adb
+md5sum	= 7eaad1f9aabd3cfad554975098c5d4c3

 [gowork.mkdir]
 # NOTE do not use slapos.cookbook:mkdirectory here - if anything in software (not instance)

--- a/component/golang/goenv.sh.in
+++ b/component/golang/goenv.sh.in
@@ -3,7 +3,7 @@

 # ---- 8< ---- (buildout substitution here)
 # PATH so that go & friends work out of the box
-export PATH=${gowork:golang}/bin:${git:location}/bin:${pkgconfig:location}/bin:${buildout:bin-directory}:$PATH
+export PATH=${gowork:golang}/bin:${git:location}/bin:${pkgconfig:location}/bin:${buildout:bin-directory}:${gowork:gcc-bin-directory}:$PATH
 X=${gowork:directory}

 export PKG_CONFIG_PATH=$(echo -n "${gowork:cpkgpath}" |tr '\n' ':'):$PKG_CONFIG_PATH

--- a/component/groonga/buildout.cfg
+++ b/component/groonga/buildout.cfg
@@ -15,8 +15,8 @@ extends =
 [groonga]
 recipe = slapos.recipe.cmmi
 shared = false
-url = https://packages.groonga.org/source/groonga/groonga-9.1.0.tar.gz
-md5sum = 81722e0bc8cc0cfdddcafde2087b71df
+url = https://packages.groonga.org/source/groonga/groonga-9.1.2.tar.gz
+md5sum = 5266c49b758dde744854cb8cfe025812
 # temporary patch to respect more tokens in natural language mode.
 patches =
  ${:_profile_base_location_}/groonga.patch#9ed02fbe8400402d3eab47eee149978b

--- a/component/helloweb/buildout.cfg
+++ b/component/helloweb/buildout.cfg
@@ -26,7 +26,7 @@ install =
    lab.nexedi.com/nexedi/helloweb/go/...

 golang  = ${golang1.12:location}
-
+gcc-bin-directory  = ${golang1.12:gcc-bin-directory}

 # -*- go -*-
 [helloweb-go]

--- a/component/mariadb/buildout.cfg
+++ b/component/mariadb/buildout.cfg
@@ -29,8 +29,8 @@ parts =
 [mariadb]
 recipe = slapos.recipe.cmmi
 url = https://downloads.mariadb.org/f/mariadb-${:version}/source/mariadb-${:version}.tar.gz/from/http%3A//fr.mirror.babylon.network/mariadb/?serve
-version = 10.3.21
-md5sum = c5b9ed98f2f57f7b809d4ec9c1515107
+version = 10.3.22
+md5sum = f712a5e6fde038d0c9c6d2a2cd88b84e
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
 pre-configure =
  set -e '\bSET(PLUGIN_AUTH_PAM YES)' cmake/build_configurations/mysql_release.cmake
@@ -84,8 +84,8 @@ post-install =
 # mroonga - a storage engine for MySQL. It provides fast fulltext search feature to all MySQL users.
 # http://mroonga.github.com/
 recipe = slapos.recipe.cmmi
-url = https://packages.groonga.org/source/mroonga/mroonga-9.10.tar.gz
-md5sum = 30a3010fd5ef11a8bd440053f1624bdb
+url = https://packages.groonga.org/source/mroonga/mroonga-9.12.tar.gz
+md5sum = d0af673f1bad3b9ccf33870bb2344a25
 pre-configure = set -e
  rm -rf fake_mariadb_source
  mkdir -p fake_mariadb_source

--- a/component/openssl/buildout.cfg
+++ b/component/openssl/buildout.cfg
@@ -57,8 +57,8 @@ openssl = ${openssl:location}/bin/openssl
 [openssl-1.0]
 recipe = slapos.recipe.cmmi
 shared = true
-url = https://www.openssl.org/source/openssl-1.0.2t.tar.gz
-md5sum = ef66581b80f06eae42f5268bc0b50c6d
+url = https://www.openssl.org/source/openssl-1.0.2u.tar.gz
+md5sum = cdc2638f789ecc2db2c91488265686c1
 location = @@LOCATION@@
 # 'prefix' option to override --openssldir/--prefix (which is useful
 # when combined with INSTALL_PREFIX). Used by slapos.package.git/obs

--- a/setup.py
+++ b/setup.py
@@ -36,8 +36,14 @@ long_description = open("README.rst").read() + "\n" + \
 for f in sorted(glob.glob(os.path.join('slapos', 'recipe', 'README.*.rst'))):
  long_description += '\n' + open(f).read() + '\n'

-# extras_requires are not used because of
-#   https://bugs.launchpad.net/zc.buildout/+bug/85604
+extras_require = {
+    'test': (
+        'jsonschema',
+        'mock',
+        'testfixtures',
+    ),
+}
+
 setup(name=name,
      version=version,
      description="SlapOS recipes.",
@@ -201,10 +207,7 @@ setup(name=name,
          'kumo = slapos.recipe.nosqltestbed.kumo:KumoTestBed',
        ],
      },
+      extras_require=extras_require,
      test_suite='slapos.test',
-      tests_require=[
-        'jsonschema',
-        'mock',
-        'testfixtures',
-      ],
+      tests_require=extras_require['test'],
    )
--- a/software/grafana/README.md
+++ b/software/grafana/README.md
 # grafana / telegraf / influxdb

- 
+This is an experimental integration, mainly to evaluate these solutions.
+
 ## Custom telegraf plugins


 See https://github.com/influxdata/telegraf to learn about plugins.

 Useful plugins in this context are probably
-[exec](https://github.com/influxdata/telegraf/tree/1.5.1/plugins/inputs/exec)
+[exec](https://github.com/influxdata/telegraf/tree/1.11.1/plugins/inputs/exec),
+[logparser](https://github.com/influxdata/telegraf/tree/1.11.1/plugins/inputs/logparser)
 or
-[httpjson](https://github.com/influxdata/telegraf/tree/1.5.1/plugins/inputs/httpjson).
+[http](https://github.com/influxdata/telegraf/tree/1.11.1/plugins/inputs/http).

 Telegraf will save in the `telegraf` database from the embedded influxdb server.


 ## Grafana

-You'll have to add yourself the influxdb data source in grafana, using the
-parameters published by the slapos instance.
-
-http://docs.grafana.org/features/datasources/influxdb/
+A default user is created, username and password are published as connection
+parameters. You can add more users in grafana interface.

-When adding datasource, use *proxy* option, otherwise Grafana makes your
-browser query influxdb directly, which also uses a self signed certificate.
-One workaround is to configure your browser to also accept influxdb certificate
-before using grafana, but using proxy seems easier.
+Datasources should be automatically added.

 ## Influxdb

@@ -34,8 +31,32 @@ One important thing to notice is that the backup protocol is enabled on ipv4
 provided by slapos, so make sure this ip is not reachable from untrusted
 sources.

-## TODO
+# Ingesting/Visualizing logs
+
+Eventhough main feature is visualizing metrics, Grafana has a feature called "Explore" to view logs for a time frame.
+The following backend can be used:
+
+## Loki
+
+See `TestLoki` in test for an example.
+
+## Influxdb
+
+Influxdb logs only have tags and there does not seem to be a way to search (except than tag and time frame).
+
+To inject log files containing:
+```
+INFO the message
+WARN another message
+```
+
+use config like:
+
+```
+[[inputs.logparser]]
+  files = ["/tmp/x*.log", "/tmp/aaa.log"]

-* influxdb and telegraf runs with very low priority, this could become an option
-* make one partition for each service and use switch software type
-* make it easier to add custom configuration (how ?)
+  [inputs.logparser.grok]
+  measurement = "logs"
+  patterns = ['^%{WORD:level:tag} %{GREEDYDATA:message:string}']
+```
--- a/software/grafana/buildout.hash.cfg
+++ b/software/grafana/buildout.hash.cfg
@@ -15,7 +15,7 @@

 [instance-profile]
 filename = instance.cfg.in
-md5sum = 461d515da03de5e422e6f75189d09184
+md5sum = 214a756a6a9f73455411760b2daa25b7

 [influxdb-config-file]
 filename = influxdb-config-file.cfg.in
@@ -28,3 +28,15 @@ md5sum = a1a9c22c2a7829c66a49fc2504604d21
 [grafana-config-file]
 filename = grafana-config-file.cfg.in
 md5sum = 8244d430905b968795c7946049bed9e3
+
+[grafana-provisioning-config-file]
+filename = grafana-provisioning-config-file.cfg.in
+md5sum = 3aa0f1ed752b2a59ea2b5e7c1733daf3
+
+[loki-config-file]
+filename = loki-config-file.cfg.in
+md5sum = 02ba5acf23fcf88f5594919f46838533
+
+[promtail-config-file]
+filename = promtail-config-file.cfg.in
+md5sum = c77788d0a3cc654ad9393eb4b1f31e94
--- a/software/grafana/gowork.cfg
+++ b/software/grafana/gowork.cfg
@@ -3,844 +3,39 @@
 # list of go git repositories to fetch
 [gowork.goinstall]
 depends_gitfetch  =
-    ${go_collectd.org:recipe}
-    ${go_github.com_BurntSushi_toml:recipe}
-    ${go_github.com_Microsoft_go-winio:recipe}
-    ${go_github.com_RoaringBitmap_roaring:recipe}
-    ${go_github.com_Shopify_sarama:recipe}
-    ${go_github.com_Sirupsen_logrus:recipe}
-    ${go_github.com_StackExchange_wmi:recipe}
-    ${go_github.com_aerospike_aerospike-client-go:recipe}
-    ${go_github.com_amir_raidman:recipe}
-    ${go_github.com_apache_thrift:recipe}
-    ${go_github.com_aws_aws-sdk-go:recipe}
-    ${go_github.com_beorn7_perks:recipe}
-    ${go_github.com_bmizerany_pat:recipe}
-    ${go_github.com_boltdb_bolt:recipe}
-    ${go_github.com_bsm_sarama-cluster:recipe}
-    ${go_github.com_cenkalti_backoff:recipe}
-    ${go_github.com_cespare_xxhash:recipe}
-    ${go_github.com_couchbase_go-couchbase:recipe}
-    ${go_github.com_couchbase_gomemcached:recipe}
-    ${go_github.com_couchbase_goutils:recipe}
-    ${go_github.com_davecgh_go-spew:recipe}
-    ${go_github.com_dgrijalva_jwt-go:recipe}
-    ${go_github.com_dgryski_go-bitstream:recipe}
-    ${go_github.com_docker_docker:recipe}
-    ${go_github.com_docker_go-connections:recipe}
-    ${go_github.com_eapache_go-resiliency:recipe}
-    ${go_github.com_eapache_go-xerial-snappy:recipe}
-    ${go_github.com_eapache_queue:recipe}
-    ${go_github.com_eclipse_paho.mqtt.golang:recipe}
-    ${go_github.com_glycerine_go-unsnap-stream:recipe}
-    ${go_github.com_go-ini_ini:recipe}
-    ${go_github.com_go-logfmt_logfmt:recipe}
-    ${go_github.com_go-ole_go-ole:recipe}
-    ${go_github.com_go-sql-driver_mysql:recipe}
-    ${go_github.com_gobwas_glob:recipe}
-    ${go_github.com_gogo_protobuf:recipe}
-    ${go_github.com_golang_protobuf:recipe}
-    ${go_github.com_golang_snappy:recipe}
-    ${go_github.com_google_go-cmp:recipe}
-    ${go_github.com_gorilla_mux:recipe}
+    ${go_github.com_golang_dep:recipe}
    ${go_github.com_grafana_grafana:recipe}
-    ${go_github.com_hailocab_go-hostpool:recipe}
-    ${go_github.com_hashicorp_consul:recipe}
+    ${go_github.com_grafana_loki:recipe}
    ${go_github.com_influxdata_influxdb:recipe}
-    ${go_github.com_influxdata_influxql:recipe}
-    ${go_github.com_influxdata_tail:recipe}
    ${go_github.com_influxdata_telegraf:recipe}
-    ${go_github.com_influxdata_toml:recipe}
-    ${go_github.com_influxdata_usage-client:recipe}
-    ${go_github.com_influxdata_wlog:recipe}
-    ${go_github.com_influxdata_yamux:recipe}
-    ${go_github.com_influxdata_yarpc:recipe}
-    ${go_github.com_jackc_pgx:recipe}
-    ${go_github.com_jmespath_go-jmespath:recipe}
-    ${go_github.com_jwilder_encoding:recipe}
-    ${go_github.com_kardianos_govendor:recipe}
-    ${go_github.com_kardianos_osext:recipe}
-    ${go_github.com_kardianos_service:recipe}
-    ${go_github.com_kballard_go-shellquote:recipe}
-    ${go_github.com_matttproud_golang_protobuf_extensions:recipe}
-    ${go_github.com_miekg_dns:recipe}
-    ${go_github.com_mitchellh_mapstructure:recipe}
-    ${go_github.com_multiplay_go-ts3:recipe}
-    ${go_github.com_naoina_go-stringutil:recipe}
-    ${go_github.com_nats-io_go-nats:recipe}
-    ${go_github.com_nats-io_nats:recipe}
-    ${go_github.com_nats-io_nuid:recipe}
-    ${go_github.com_nsqio_go-nsq:recipe}
-    ${go_github.com_opencontainers_runc:recipe}
-    ${go_github.com_opentracing-contrib_go-observer:recipe}
-    ${go_github.com_opentracing_opentracing-go:recipe}
-    ${go_github.com_openzipkin_zipkin-go-opentracing:recipe}
-    ${go_github.com_peterh_liner:recipe}
-    ${go_github.com_philhofer_fwd:recipe}
-    ${go_github.com_pierrec_lz4:recipe}
-    ${go_github.com_pierrec_xxHash:recipe}
-    ${go_github.com_pkg_errors:recipe}
-    ${go_github.com_pmezard_go-difflib:recipe}
-    ${go_github.com_prometheus_client_golang:recipe}
-    ${go_github.com_prometheus_client_model:recipe}
-    ${go_github.com_prometheus_common:recipe}
-    ${go_github.com_prometheus_procfs:recipe}
-    ${go_github.com_rcrowley_go-metrics:recipe}
-    ${go_github.com_retailnext_hllpp:recipe}
-    ${go_github.com_samuel_go-zookeeper:recipe}
-    ${go_github.com_satori_go.uuid:recipe}
-    ${go_github.com_shirou_gopsutil:recipe}
-    ${go_github.com_shirou_w32:recipe}
-    ${go_github.com_soniah_gosnmp:recipe}
-    ${go_github.com_sparrc_gdm:recipe}
-    ${go_github.com_streadway_amqp:recipe}
-    ${go_github.com_stretchr_objx:recipe}
-    ${go_github.com_stretchr_testify:recipe}
-    ${go_github.com_tidwall_gjson:recipe}
-    ${go_github.com_tidwall_match:recipe}
-    ${go_github.com_tinylib_msgp:recipe}
-    ${go_github.com_vjeantet_grok:recipe}
-    ${go_github.com_wvanbergen_kafka:recipe}
-    ${go_github.com_wvanbergen_kazoo-go:recipe}
-    ${go_github.com_xlab_treeprint:recipe}
-    ${go_github.com_yuin_gopher-lua:recipe}
-    ${go_github.com_zensqlmonitor_go-mssqldb:recipe}
-    ${go_go.uber.org_atomic:recipe}
-    ${go_go.uber.org_multierr:recipe}
-    ${go_go.uber.org_zap:recipe}
-    ${go_golang.org_x_crypto:recipe}
-    ${go_golang.org_x_net:recipe}
-    ${go_golang.org_x_sys:recipe}
-    ${go_golang.org_x_text:recipe}
-    ${go_golang.org_x_time:recipe}
-    ${go_golang.org_x_tools:recipe}
-    ${go_gopkg.in_asn1-ber.v1:recipe}
-    ${go_gopkg.in_fatih_pool.v2:recipe}
-    ${go_gopkg.in_fsnotify.v1:recipe}
-    ${go_gopkg.in_gorethink_gorethink.v3:recipe}
-    ${go_gopkg.in_ldap.v2:recipe}
-    ${go_gopkg.in_mgo.v2:recipe}
-    ${go_gopkg.in_olivere_elastic.v5:recipe}
-    ${go_gopkg.in_tomb.v1:recipe}
-    ${go_gopkg.in_yaml.v2:recipe}


-[go_collectd.org]
+[go_github.com_golang_dep]
 <= go-git-package
-go.importpath = collectd.org
-repository    = https://github.com/collectd/go-collectd
-revision      = v0.3.0-17-g606bd390f3
-
-[go_github.com_BurntSushi_toml]
-<= go-git-package
-go.importpath = github.com/BurntSushi/toml
-repository    = https://github.com/BurntSushi/toml
-revision      = v0.2.0-45-ga368813c5e
-
-[go_github.com_Microsoft_go-winio]
-<= go-git-package
-go.importpath = github.com/Microsoft/go-winio
-repository    = https://github.com/Microsoft/go-winio
-revision      = ce2922f643
-
-[go_github.com_RoaringBitmap_roaring]
-<= go-git-package
-go.importpath = github.com/RoaringBitmap/roaring
-repository    = https://github.com/RoaringBitmap/roaring
-revision      = v0.2.8-174-g0a6691af7c
-
-[go_github.com_Shopify_sarama]
-<= go-git-package
-go.importpath = github.com/Shopify/sarama
-repository    = https://github.com/Shopify/sarama
-revision      = 3b1b38866a
-
-[go_github.com_Sirupsen_logrus]
-<= go-git-package
-go.importpath = github.com/Sirupsen/logrus
-repository    = https://github.com/Sirupsen/logrus
-revision      = 61e43dc76f
-
-[go_github.com_StackExchange_wmi]
-<= go-git-package
-go.importpath = github.com/StackExchange/wmi
-repository    = https://github.com/StackExchange/wmi
-revision      = f3e2bae1e0
-
-[go_github.com_aerospike_aerospike-client-go]
-<= go-git-package
-go.importpath = github.com/aerospike/aerospike-client-go
-repository    = https://github.com/aerospike/aerospike-client-go
-revision      = v1.6.4-277-g95e1ad7791
-
-[go_github.com_amir_raidman]
-<= go-git-package
-go.importpath = github.com/amir/raidman
-repository    = https://github.com/amir/raidman
-revision      = c74861fe6a
-
-[go_github.com_apache_thrift]
-<= go-git-package
-go.importpath = github.com/apache/thrift
-repository    = https://github.com/apache/thrift
-revision      = 4aaa92ece8
-
-[go_github.com_aws_aws-sdk-go]
-<= go-git-package
-go.importpath = github.com/aws/aws-sdk-go
-repository    = https://github.com/aws/aws-sdk-go
-revision      = c861d27d03
-
-[go_github.com_beorn7_perks]
-<= go-git-package
-go.importpath = github.com/beorn7/perks
-repository    = https://github.com/beorn7/perks
-revision      = 4c0e84591b
-
-[go_github.com_bmizerany_pat]
-<= go-git-package
-go.importpath = github.com/bmizerany/pat
-repository    = https://github.com/bmizerany/pat
-revision      = 6226ea591a
-
-[go_github.com_boltdb_bolt]
-<= go-git-package
-go.importpath = github.com/boltdb/bolt
-repository    = https://github.com/boltdb/bolt
-revision      = 9da3174536
-
-[go_github.com_bsm_sarama-cluster]
-<= go-git-package
-go.importpath = github.com/bsm/sarama-cluster
-repository    = https://github.com/bsm/sarama-cluster
-revision      = v1.0.0-164-gabf039439f
-
-[go_github.com_cenkalti_backoff]
-<= go-git-package
-go.importpath = github.com/cenkalti/backoff
-repository    = https://github.com/cenkalti/backoff
-revision      = b02f2bbce1
-
-[go_github.com_cespare_xxhash]
-<= go-git-package
-go.importpath = github.com/cespare/xxhash
-repository    = https://github.com/cespare/xxhash
-revision      = e4e2bd419c
-
-[go_github.com_couchbase_go-couchbase]
-<= go-git-package
-go.importpath = github.com/couchbase/go-couchbase
-repository    = https://github.com/couchbase/go-couchbase
-revision      = bfe555a140
-
-[go_github.com_couchbase_gomemcached]
-<= go-git-package
-go.importpath = github.com/couchbase/gomemcached
-repository    = https://github.com/couchbase/gomemcached
-revision      = 4a25d2f4e1
-
-[go_github.com_couchbase_goutils]
-<= go-git-package
-go.importpath = github.com/couchbase/goutils
-repository    = https://github.com/couchbase/goutils
-revision      = 5823a0cbaa
-
-[go_github.com_davecgh_go-spew]
-<= go-git-package
-go.importpath = github.com/davecgh/go-spew
-repository    = https://github.com/davecgh/go-spew
-revision      = v1.1.0-9-gecdeabc654
-
-[go_github.com_dgrijalva_jwt-go]
-<= go-git-package
-go.importpath = github.com/dgrijalva/jwt-go
-repository    = https://github.com/dgrijalva/jwt-go
-revision      = dbeaa9332f
-
-[go_github.com_dgryski_go-bitstream]
-<= go-git-package
-go.importpath = github.com/dgryski/go-bitstream
-repository    = https://github.com/dgryski/go-bitstream
-revision      = 7d46cd22db
-
-[go_github.com_docker_docker]
-<= go-git-package
-go.importpath = github.com/docker/docker
-repository    = https://github.com/docker/docker
-revision      = v17.03.2-ce-0-gf5ec1e2936
-
-[go_github.com_docker_go-connections]
-<= go-git-package
-go.importpath = github.com/docker/go-connections
-repository    = https://github.com/docker/go-connections
-revision      = 990a1a1a70
-
-[go_github.com_eapache_go-resiliency]
-<= go-git-package
-go.importpath = github.com/eapache/go-resiliency
-repository    = https://github.com/eapache/go-resiliency
-revision      = b86b1ec0dd
-
-[go_github.com_eapache_go-xerial-snappy]
-<= go-git-package
-go.importpath = github.com/eapache/go-xerial-snappy
-repository    = https://github.com/eapache/go-xerial-snappy
-revision      = bb955e01b9
-
-[go_github.com_eapache_queue]
-<= go-git-package
-go.importpath = github.com/eapache/queue
-repository    = https://github.com/eapache/queue
-revision      = 44cc805cf1
-
-[go_github.com_eclipse_paho.mqtt.golang]
-<= go-git-package
-go.importpath = github.com/eclipse/paho.mqtt.golang
-repository    = https://github.com/eclipse/paho.mqtt.golang
-revision      = d4f545eb10
-
-[go_github.com_glycerine_go-unsnap-stream]
-<= go-git-package
-go.importpath = github.com/glycerine/go-unsnap-stream
-repository    = https://github.com/glycerine/go-unsnap-stream
-revision      = 62a9a9eb44
-
-[go_github.com_go-ini_ini]
-<= go-git-package
-go.importpath = github.com/go-ini/ini
-repository    = https://github.com/go-ini/ini
-revision      = 9144852efb
-
-[go_github.com_go-logfmt_logfmt]
-<= go-git-package
-go.importpath = github.com/go-logfmt/logfmt
-repository    = https://github.com/go-logfmt/logfmt
-revision      = v0.3.0-0-g390ab7935e
-
-[go_github.com_go-ole_go-ole]
-<= go-git-package
-go.importpath = github.com/go-ole/go-ole
-repository    = https://github.com/go-ole/go-ole
-revision      = be49f7c077
-
-[go_github.com_go-sql-driver_mysql]
-<= go-git-package
-go.importpath = github.com/go-sql-driver/mysql
-repository    = https://github.com/go-sql-driver/mysql
-revision      = v1.0-470-g2e00b5cd70
-
-[go_github.com_gobwas_glob]
-<= go-git-package
-go.importpath = github.com/gobwas/glob
-repository    = https://github.com/gobwas/glob
-revision      = v0.2.2-0-gbea32b9cd2
-
-[go_github.com_gogo_protobuf]
-<= go-git-package
-go.importpath = github.com/gogo/protobuf
-repository    = https://github.com/gogo/protobuf
-revision      = 160de10b25
-
-[go_github.com_golang_protobuf]
-<= go-git-package
-go.importpath = github.com/golang/protobuf
-repository    = https://github.com/golang/protobuf
-revision      = 1e59b77b52
-
-[go_github.com_golang_snappy]
-<= go-git-package
-go.importpath = github.com/golang/snappy
-repository    = https://github.com/golang/snappy
-revision      = 553a641470
-
-[go_github.com_google_go-cmp]
-<= go-git-package
-go.importpath = github.com/google/go-cmp
-repository    = https://github.com/google/go-cmp
-revision      = f94e52cad9
-
-[go_github.com_gorilla_mux]
-<= go-git-package
-go.importpath = github.com/gorilla/mux
-repository    = https://github.com/gorilla/mux
-revision      = 392c28fe23
+go.importpath = github.com/golang/dep
+repository    = https://github.com/golang/dep
+revision      = 1f7c19e5f5

 [go_github.com_grafana_grafana]
 <= go-git-package
 go.importpath = github.com/grafana/grafana
 repository    = https://github.com/grafana/grafana
-revision      = v4.6.0-beta1-1360-g9606a34e0a
-
-[go_github.com_hailocab_go-hostpool]
-<= go-git-package
-go.importpath = github.com/hailocab/go-hostpool
-repository    = https://github.com/hailocab/go-hostpool
-revision      = e80d13ce29
+revision      = v6.3.0-0-g830da0fda0

-[go_github.com_hashicorp_consul]
+[go_github.com_grafana_loki]
 <= go-git-package
-go.importpath = github.com/hashicorp/consul
-repository    = https://github.com/hashicorp/consul
-revision      = v0.7.3-35-g63d2fc6823
+go.importpath = github.com/grafana/loki
+repository    = https://github.com/grafana/loki
+revision      = v0.3.0-0-gc673380bb3

 [go_github.com_influxdata_influxdb]
 <= go-git-package
 go.importpath = github.com/influxdata/influxdb
 repository    = https://github.com/influxdata/influxdb
-revision      = v1.4.0rc0-328-g938db68198
-
-[go_github.com_influxdata_influxql]
-<= go-git-package
-go.importpath = github.com/influxdata/influxql
-repository    = https://github.com/influxdata/influxql
-revision      = 851636b092
-
-[go_github.com_influxdata_tail]
-<= go-git-package
-go.importpath = github.com/influxdata/tail
-repository    = https://github.com/influxdata/tail
-revision      = v0-95-ga395bf99fe
+revision      = v1.7.8rc1-0-ga7afa2ecf1

 [go_github.com_influxdata_telegraf]
 <= go-git-package
 go.importpath = github.com/influxdata/telegraf
 repository    = https://github.com/influxdata/telegraf
-revision      = 1.5.0-rc1-73-g90b6b760d1
-
-[go_github.com_influxdata_toml]
-<= go-git-package
-go.importpath = github.com/influxdata/toml
-repository    = https://github.com/influxdata/toml
-revision      = 5d1d907f22
-
-[go_github.com_influxdata_usage-client]
-<= go-git-package
-go.importpath = github.com/influxdata/usage-client
-repository    = https://github.com/influxdata/usage-client
-revision      = 6d38953763
-
-[go_github.com_influxdata_wlog]
-<= go-git-package
-go.importpath = github.com/influxdata/wlog
-repository    = https://github.com/influxdata/wlog
-revision      = 7c63b0a71e
-
-[go_github.com_influxdata_yamux]
-<= go-git-package
-go.importpath = github.com/influxdata/yamux
-repository    = https://github.com/influxdata/yamux
-revision      = 1f58ded512
-
-[go_github.com_influxdata_yarpc]
-<= go-git-package
-go.importpath = github.com/influxdata/yarpc
-repository    = https://github.com/influxdata/yarpc
-revision      = fdd7e84bf3
-
-[go_github.com_jackc_pgx]
-<= go-git-package
-go.importpath = github.com/jackc/pgx
-repository    = https://github.com/jackc/pgx
-revision      = 63f58fd32e
-
-[go_github.com_jmespath_go-jmespath]
-<= go-git-package
-go.importpath = github.com/jmespath/go-jmespath
-repository    = https://github.com/jmespath/go-jmespath
-revision      = 0.2.2-14-gbd40a432e4
-
-[go_github.com_jwilder_encoding]
-<= go-git-package
-go.importpath = github.com/jwilder/encoding
-repository    = https://github.com/jwilder/encoding
-revision      = b4e1701a28
-
-[go_github.com_kardianos_govendor]
-<= go-git-package
-go.importpath = github.com/kardianos/govendor
-repository    = https://github.com/kardianos/govendor
-revision      = 274337c49c
-
-[go_github.com_kardianos_osext]
-<= go-git-package
-go.importpath = github.com/kardianos/osext
-repository    = https://github.com/kardianos/osext
-revision      = c2c54e542f
-
-[go_github.com_kardianos_service]
-<= go-git-package
-go.importpath = github.com/kardianos/service
-repository    = https://github.com/kardianos/service
-revision      = 6d3a0ee7d3
-
-[go_github.com_kballard_go-shellquote]
-<= go-git-package
-go.importpath = github.com/kballard/go-shellquote
-repository    = https://github.com/kballard/go-shellquote
-revision      = d8ec1a69a2
-
-[go_github.com_matttproud_golang_protobuf_extensions]
-<= go-git-package
-go.importpath = github.com/matttproud/golang_protobuf_extensions
-repository    = https://github.com/matttproud/golang_protobuf_extensions
-revision      = v1.0.0-2-gc12348ce28
-
-[go_github.com_miekg_dns]
-<= go-git-package
-go.importpath = github.com/miekg/dns
-repository    = https://github.com/miekg/dns
-revision      = 99f84ae56e
-
-[go_github.com_mitchellh_mapstructure]
-<= go-git-package
-go.importpath = github.com/mitchellh/mapstructure
-repository    = https://github.com/mitchellh/mapstructure
-revision      = d0303fe809
-
-[go_github.com_multiplay_go-ts3]
-<= go-git-package
-go.importpath = github.com/multiplay/go-ts3
-repository    = https://github.com/multiplay/go-ts3
-revision      = 07477f49b8
-
-[go_github.com_naoina_go-stringutil]
-<= go-git-package
-go.importpath = github.com/naoina/go-stringutil
-repository    = https://github.com/naoina/go-stringutil
-revision      = 6b638e95a3
-
-[go_github.com_nats-io_go-nats]
-<= go-git-package
-go.importpath = github.com/nats-io/go-nats
-repository    = https://github.com/nats-io/go-nats
-revision      = v1.2.0-73-gea9585611a
-
-[go_github.com_nats-io_nats]
-<= go-git-package
-go.importpath = github.com/nats-io/nats
-repository    = https://github.com/nats-io/nats
-revision      = v1.2.0-73-gea9585611a
-
-[go_github.com_nats-io_nuid]
-<= go-git-package
-go.importpath = github.com/nats-io/nuid
-repository    = https://github.com/nats-io/nuid
-revision      = 289cccf02c
-
-[go_github.com_nsqio_go-nsq]
-<= go-git-package
-go.importpath = github.com/nsqio/go-nsq
-repository    = https://github.com/nsqio/go-nsq
-revision      = v1.0.7-0-geee57a3ac4
-
-[go_github.com_opencontainers_runc]
-<= go-git-package
-go.importpath = github.com/opencontainers/runc
-repository    = https://github.com/opencontainers/runc
-revision      = v0.0.5-426-g89ab7f2ccc
-
-[go_github.com_opentracing-contrib_go-observer]
-<= go-git-package
-go.importpath = github.com/opentracing-contrib/go-observer
-repository    = https://github.com/opentracing-contrib/go-observer
-revision      = a52f234244
-
-[go_github.com_opentracing_opentracing-go]
-<= go-git-package
-go.importpath = github.com/opentracing/opentracing-go
-repository    = https://github.com/opentracing/opentracing-go
-revision      = 1361b9cd60
-
-[go_github.com_openzipkin_zipkin-go-opentracing]
-<= go-git-package
-go.importpath = github.com/openzipkin/zipkin-go-opentracing
-repository    = https://github.com/openzipkin/zipkin-go-opentracing
-revision      = 1cafbdfde9
-
-[go_github.com_peterh_liner]
-<= go-git-package
-go.importpath = github.com/peterh/liner
-repository    = https://github.com/peterh/liner
-revision      = 3681c2a912
-
-[go_github.com_philhofer_fwd]
-<= go-git-package
-go.importpath = github.com/philhofer/fwd
-repository    = https://github.com/philhofer/fwd
-revision      = bb6d471dc9
-
-[go_github.com_pierrec_lz4]
-<= go-git-package
-go.importpath = github.com/pierrec/lz4
-repository    = https://github.com/pierrec/lz4
-revision      = v1.0-0-g5c9560bfa9
-
-[go_github.com_pierrec_xxHash]
-<= go-git-package
-go.importpath = github.com/pierrec/xxHash
-repository    = https://github.com/pierrec/xxHash
-revision      = 5a004441f8
-
-[go_github.com_pkg_errors]
-<= go-git-package
-go.importpath = github.com/pkg/errors
-repository    = https://github.com/pkg/errors
-revision      = v0.8.0-0-g645ef00459
-
-[go_github.com_pmezard_go-difflib]
-<= go-git-package
-go.importpath = github.com/pmezard/go-difflib
-repository    = https://github.com/pmezard/go-difflib
-revision      = v1.0.0-0-g792786c740
-
-[go_github.com_prometheus_client_golang]
-<= go-git-package
-go.importpath = github.com/prometheus/client_golang
-repository    = https://github.com/prometheus/client_golang
-revision      = v0.9.0-pre1-21-g180b8fdc22
-
-[go_github.com_prometheus_client_model]
-<= go-git-package
-go.importpath = github.com/prometheus/client_model
-repository    = https://github.com/prometheus/client_model
-revision      = model-0.0.2-16-g99fa1f4be8
-
-[go_github.com_prometheus_common]
-<= go-git-package
-go.importpath = github.com/prometheus/common
-repository    = https://github.com/prometheus/common
-revision      = 89604d1970
-
-[go_github.com_prometheus_procfs]
-<= go-git-package
-go.importpath = github.com/prometheus/procfs
-repository    = https://github.com/prometheus/procfs
-revision      = b15cd069a8
-
-[go_github.com_rcrowley_go-metrics]
-<= go-git-package
-go.importpath = github.com/rcrowley/go-metrics
-repository    = https://github.com/rcrowley/go-metrics
-revision      = 1f30fe9094
-
-[go_github.com_retailnext_hllpp]
-<= go-git-package
-go.importpath = github.com/retailnext/hllpp
-repository    = https://github.com/retailnext/hllpp
-revision      = v1.0.0-2-g6e8b6d3944
-
-[go_github.com_samuel_go-zookeeper]
-<= go-git-package
-go.importpath = github.com/samuel/go-zookeeper
-repository    = https://github.com/samuel/go-zookeeper
-revision      = 1d7be4effb
-
-[go_github.com_satori_go.uuid]
-<= go-git-package
-go.importpath = github.com/satori/go.uuid
-repository    = https://github.com/satori/go.uuid
-revision      = v1.1.0-8-g5bf94b69c6
-
-[go_github.com_shirou_gopsutil]
-<= go-git-package
-go.importpath = github.com/shirou/gopsutil
-repository    = https://github.com/shirou/gopsutil
-revision      = v2.16.10-243-g384a55110a
-
-[go_github.com_shirou_w32]
-<= go-git-package
-go.importpath = github.com/shirou/w32
-repository    = https://github.com/shirou/w32
-revision      = 3c9377fc67
-
-[go_github.com_soniah_gosnmp]
-<= go-git-package
-go.importpath = github.com/soniah/gosnmp
-repository    = https://github.com/soniah/gosnmp
-revision      = v1.0-204-g5ad50dc75a
-
-[go_github.com_sparrc_gdm]
-<= go-git-package
-go.importpath = github.com/sparrc/gdm
-repository    = https://github.com/sparrc/gdm
-revision      = 1.4-4-g81089dabfa
-
-[go_github.com_streadway_amqp]
-<= go-git-package
-go.importpath = github.com/streadway/amqp
-repository    = https://github.com/streadway/amqp
-revision      = 63795daa9a
-
-[go_github.com_stretchr_objx]
-<= go-git-package
-go.importpath = github.com/stretchr/objx
-repository    = https://github.com/stretchr/objx
-revision      = 1a9d0bb9f5
-
-[go_github.com_stretchr_testify]
-<= go-git-package
-go.importpath = github.com/stretchr/testify
-repository    = https://github.com/stretchr/testify
-revision      = v1.0-187-g4d4bfba8f1
-
-[go_github.com_tidwall_gjson]
-<= go-git-package
-go.importpath = github.com/tidwall/gjson
-repository    = https://github.com/tidwall/gjson
-revision      = 0623bd8fbd
-
-[go_github.com_tidwall_match]
-<= go-git-package
-go.importpath = github.com/tidwall/match
-repository    = https://github.com/tidwall/match
-revision      = 173748da73
-
-[go_github.com_tinylib_msgp]
-<= go-git-package
-go.importpath = github.com/tinylib/msgp
-repository    = https://github.com/tinylib/msgp
-revision      = 428e467e72
-
-[go_github.com_vjeantet_grok]
-<= go-git-package
-go.importpath = github.com/vjeantet/grok
-repository    = https://github.com/vjeantet/grok
-revision      = d73e972b60
-
-[go_github.com_wvanbergen_kafka]
-<= go-git-package
-go.importpath = github.com/wvanbergen/kafka
-repository    = https://github.com/wvanbergen/kafka
-revision      = bc265fedb9
-
-[go_github.com_wvanbergen_kazoo-go]
-<= go-git-package
-go.importpath = github.com/wvanbergen/kazoo-go
-repository    = https://github.com/wvanbergen/kazoo-go
-revision      = 9689573521
-
-[go_github.com_xlab_treeprint]
-<= go-git-package
-go.importpath = github.com/xlab/treeprint
-repository    = https://github.com/xlab/treeprint
-revision      = 06dfc6fa17
-
-[go_github.com_yuin_gopher-lua]
-<= go-git-package
-go.importpath = github.com/yuin/gopher-lua
-repository    = https://github.com/yuin/gopher-lua
-revision      = 66c871e454
-
-[go_github.com_zensqlmonitor_go-mssqldb]
-<= go-git-package
-go.importpath = github.com/zensqlmonitor/go-mssqldb
-repository    = https://github.com/zensqlmonitor/go-mssqldb
-revision      = ffe5510c6f
-
-[go_go.uber.org_atomic]
-<= go-git-package
-go.importpath = go.uber.org/atomic
-repository    = https://github.com/uber-go/atomic
-revision      = v1.3.1-0-g8474b86a5a
-
-[go_go.uber.org_multierr]
-<= go-git-package
-go.importpath = go.uber.org/multierr
-repository    = https://github.com/uber-go/multierr
-revision      = v1.1.0-1-gfb7d312c2c
-
-[go_go.uber.org_zap]
-<= go-git-package
-go.importpath = go.uber.org/zap
-repository    = https://github.com/uber-go/zap
-revision      = v1.7.1-11-gf85c78b1dd
-
-[go_golang.org_x_crypto]
-<= go-git-package
-go.importpath = golang.org/x/crypto
-repository    = https://go.googlesource.com/crypto
-revision      = b3c9a1d25c
-
-[go_golang.org_x_net]
-<= go-git-package
-go.importpath = golang.org/x/net
-repository    = https://go.googlesource.com/net
-revision      = ab555f366c
-
-[go_golang.org_x_sys]
-<= go-git-package
-go.importpath = golang.org/x/sys
-repository    = https://go.googlesource.com/sys
-revision      = 810d700034
-
-[go_golang.org_x_text]
-<= go-git-package
-go.importpath = golang.org/x/text
-repository    = https://go.googlesource.com/text
-revision      = e19ae14969
-
-[go_golang.org_x_time]
-<= go-git-package
-go.importpath = golang.org/x/time
-repository    = https://go.googlesource.com/time
-revision      = 6dc17368e0
-
-[go_golang.org_x_tools]
-<= go-git-package
-go.importpath = golang.org/x/tools
-repository    = https://go.googlesource.com/tools
-revision      = fbec762f83
-
-[go_gopkg.in_asn1-ber.v1]
-<= go-git-package
-go.importpath = gopkg.in/asn1-ber.v1
-repository    = https://gopkg.in/asn1-ber.v1
-revision      = 4e86f43671
-
-[go_gopkg.in_fatih_pool.v2]
-<= go-git-package
-go.importpath = gopkg.in/fatih/pool.v2
-repository    = https://gopkg.in/fatih/pool.v2
-revision      = 6e328e6789
-
-[go_gopkg.in_fsnotify.v1]
-<= go-git-package
-go.importpath = gopkg.in/fsnotify.v1
-repository    = https://gopkg.in/fsnotify.v1
-revision      = a8a77c9133
-
-[go_gopkg.in_gorethink_gorethink.v3]
-<= go-git-package
-go.importpath = gopkg.in/gorethink/gorethink.v3
-repository    = https://gopkg.in/gorethink/gorethink.v3
-revision      = v2.2.1-35-g7ab832f7b6
-
-[go_gopkg.in_ldap.v2]
-<= go-git-package
-go.importpath = gopkg.in/ldap.v2
-repository    = https://gopkg.in/ldap.v2
-revision      = 8168ee085e
-
-[go_gopkg.in_mgo.v2]
-<= go-git-package
-go.importpath = gopkg.in/mgo.v2
-repository    = https://gopkg.in/mgo.v2
-revision      = 3f83fa5005
-
-[go_gopkg.in_olivere_elastic.v5]
-<= go-git-package
-go.importpath = gopkg.in/olivere/elastic.v5
-repository    = https://gopkg.in/olivere/elastic.v5
-revision      = v5.0.41-0-g3113f9b9ad
-
-[go_gopkg.in_tomb.v1]
-<= go-git-package
-go.importpath = gopkg.in/tomb.v1
-repository    = https://gopkg.in/tomb.v1
-revision      = dd632973f1
-
-[go_gopkg.in_yaml.v2]
-<= go-git-package
-go.importpath = gopkg.in/yaml.v2
-repository    = https://gopkg.in/yaml.v2
-revision      = 4c78c975fe
+revision      = 1.11.1-0-gf91de60d37
--- a/software/grafana/grafana-provisioning-config-file.cfg.in
+++ b/software/grafana/grafana-provisioning-config-file.cfg.in
+# https://grafana.com/docs/administration/provisioning/#example-datasource-config-file
+
+apiVersion: 1
+
+datasources:
+- name: telegraf
+  type: influxdb
+  access: proxy
+  url: {{ influxdb['url'] }}
+  user: {{ influxdb['auth-username'] }}
+  database: telegraf
+  isDefault: true
+  jsonData:
+    tlsSkipVerify: true
+  secureJsonData:
+    password: {{ influxdb['auth-password'] }}
+  version: 1
+  editable: false
+- name: loki
+  type: loki
+  access: proxy
+  url: {{ loki['url'] }}
+  version: 1
+  editable: false
--- a/software/grafana/instance-input-schema.json
+++ b/software/grafana/instance-input-schema.json
@@ -28,6 +28,11 @@
      "description": "Name used in From: header of emails",
      "default": "Grafana",
      "type": "string"
+    },
+    "promtail-extra-scrape-config": {
+      "description": "Raw promtail config (experimental parameter, see https://github.com/grafana/loki/blob/v0.3.0/docs/promtail.md#scrape-configs for detail)",
+      "default": "",
+      "type": "string"
    }
  }
 }
--- a/software/grafana/instance.cfg.in
+++ b/software/grafana/instance.cfg.in
@@ -3,8 +3,6 @@ parts =
  promises
  publish-connection-parameter

-extends = {{ monitor_template }}
-
 eggs-directory = {{ buildout['eggs-directory'] }}
 develop-eggs-directory = {{ buildout['develop-eggs-directory'] }}
 offline = true
@@ -35,17 +33,21 @@ etc = ${:home}/etc
 var = ${:home}/var
 srv = ${:home}/srv
 service = ${:etc}/service
+promise = ${:etc}/promise
 influxdb-data-dir = ${:srv}/influxdb
 grafana-dir = ${:srv}/grafana
 grafana-data-dir = ${:grafana-dir}/data
 grafana-logs-dir = ${:var}/log
 grafana-plugins-dir = ${:grafana-dir}/plugins
 grafana-provisioning-config-dir = ${:grafana-dir}/provisioning-config
-grafana-provisioning-datasources = ${:grafana-provisioning-config-dir}/datasources
-grafana-provisioning-dashboards = ${:grafana-provisioning-config-dir}/dashboards
+grafana-provisioning-datasources-dir = ${:grafana-provisioning-config-dir}/datasources
+grafana-provisioning-dashboards-dir = ${:grafana-provisioning-config-dir}/dashboards
 telegraf-dir = ${:srv}/telegraf
 telegraf-extra-config-dir = ${:telegraf-dir}/extra-config
-
+loki-dir = ${:srv}/loki
+loki-storage-boltdb-dir = ${:loki-dir}/index/
+loki-storage-filesystem-dir = ${:loki-dir}/chunks/
+promtail-dir = ${:srv}/promtail

 # macros
 [generate-certificate]
@@ -71,11 +73,14 @@ mode = 0644
 extensions = jinja2.ext.do

 [check-port-listening-promise]
-<= monitor-base-promise
-module = check_port_listening
-name = ${:_buildout_section_name_}.py
-
+recipe = slapos.cookbook:check_port_listening
+path = ${directory:promise}/${:_buildout_section_name_}

+[check-url-available-promise]
+recipe = slapos.cookbook:check_url_available
+path = ${directory:promise}/${:_buildout_section_name_}
+dash_path = {{ dash_bin }}
+curl_path = {{ curl_bin }}

 [influxdb]
 ipv6 = ${instance-parameter:ipv6-random}
@@ -112,27 +117,27 @@ username = influxdb

 [influxdb-listen-promise]
 <= check-port-listening-promise
-config-hostname = ${influxdb:ipv6}
-config-port = ${influxdb:http-port}
+hostname = ${influxdb:ipv6}
+port = ${influxdb:http-port}

 [influxdb-password-promise]
-<= monitor-base-promise
-module = check_command_execute
-name = ${:_buildout_section_name_}.py
-config-command =
+recipe = slapos.cookbook:wrapper
+command-line =
  {{ influx_bin }} -username ${influxdb:auth-username} -password ${influxdb:auth-password} -socket ${influxdb:unix-socket} -execute "CREATE USER ${influxdb:auth-username} WITH PASSWORD '${influxdb:auth-password}' WITH ALL PRIVILEGES"
+wrapper-path = ${directory:promise}/${:_buildout_section_name_}



 [grafana]
 ipv6 = ${instance-parameter:ipv6-random}
-port = 8080
+port = 8180
 url = https://[${:ipv6}]:${:port}

 data-dir = ${directory:grafana-data-dir}
 logs-dir = ${directory:grafana-logs-dir}
 plugins-dir = ${directory:grafana-plugins-dir}
 provisioning-config-dir = ${directory:grafana-provisioning-config-dir}
+provisioning-datasources-dir = ${directory:grafana-provisioning-datasources-dir}
 admin-user = ${grafana-password:username}
 admin-password = ${grafana-password:passwd}
 secret-key = ${grafana-secret-key:passwd}
@@ -160,11 +165,20 @@ context =
  section grafana grafana
  section apache_frontend apache-frontend
  key slapparameter_dict slap-configuration:configuration
+depends =
+  ${grafana-provisioning-config-file:rendered}
+
+[grafana-provisioning-config-file]
+<= config-file
+rendered = ${grafana:provisioning-datasources-dir}/datasource.yaml
+context =
+  section influxdb influxdb
+  section loki loki

 [grafana-listen-promise]
 <= check-port-listening-promise
-config-hostname= ${grafana:ipv6}
-config-port = ${grafana:port}
+hostname= ${grafana:ipv6}
+port = ${grafana:port}



@@ -183,6 +197,53 @@ context =
  section telegraf telegraf


+[loki]
+recipe = slapos.cookbook:wrapper
+command-line =
+   bash -c 'nice -19 chrt --idle 0 ionice -c3 {{ loki_bin }} -config.file=${loki-config-file:rendered}'
+wrapper-path = ${directory:service}/loki
+
+storage-boltdb-dir = ${directory:loki-storage-boltdb-dir}
+storage-filesystem-dir = ${directory:loki-storage-filesystem-dir}
+ip = ${instance-parameter:ipv4-random}
+port = 3100
+url = http://${:ip}:${:port}
+
+
+[loki-config-file]
+<= config-file
+context =
+  section loki loki
+
+[loki-listen-promise]
+<= check-url-available-promise
+url = ${loki:url}/ready
+
+[promtail]
+recipe = slapos.cookbook:wrapper
+command-line =
+   bash -c 'nice -19 chrt --idle 0 ionice -c3 {{ promtail_bin }} -config.file=${promtail-config-file:rendered}'
+wrapper-path = ${directory:service}/promtail
+
+dir = ${directory:promtail-dir}
+http_port = 19080
+ip = ${instance-parameter:ipv4-random}
+url = http://${:ip}:${:http_port}
+
+[promtail-config-file]
+<= config-file
+context =
+  section promtail promtail
+  section loki loki
+  key slapparameter_dict slap-configuration:configuration
+
+[promtail-listen-promise]
+<= check-port-listening-promise
+hostname= ${promtail:ip}
+port = ${promtail:http_port}
+
+
+
 [apache-frontend]
 <= slap-connection
 recipe = slapos.cookbook:requestoptional
@@ -201,6 +262,8 @@ instance-promises =
  ${influxdb-listen-promise:path}
  ${influxdb-password-promise:wrapper-path}
  ${grafana-listen-promise:path}
+  ${loki-listen-promise:path}
+  ${promtail-listen-promise:path}


 [publish-connection-parameter]
@@ -213,4 +276,6 @@ telegraf-extra-config-dir = ${telegraf:extra-config-dir}
 grafana-url = ${grafana:url}
 grafana-username = ${grafana:admin-user}
 grafana-password = ${grafana:admin-password}
+loki-url = ${loki:url}
+promtail-url = ${promtail:url}
 url = ${apache-frontend:connection-secure_access}
--- a/software/grafana/loki-config-file.cfg.in
+++ b/software/grafana/loki-config-file.cfg.in
+auth_enabled: false
+
+server:
+  http_listen_port: {{ loki['port'] }}
+
+ingester:
+  lifecycler:
+    address: {{ loki['ip'] }}
+    ring:
+      kvstore:
+        store: inmemory
+      replication_factor: 1
+  chunk_idle_period: 15m
+
+schema_config:
+  configs:
+  - from: 2018-04-15
+    store: boltdb
+    object_store: filesystem
+    schema: v9
+    index:
+      prefix: index_
+      period: 168h
+
+storage_config:
+  boltdb:
+    directory: {{ loki['storage-boltdb-dir'] }}
+
+  filesystem:
+    directory: {{ loki['storage-filesystem-dir'] }}
+
+limits_config:
+  enforce_metric_name: false
+  reject_old_samples: true
+  reject_old_samples_max_age: 168h
+
+chunk_store_config:
+  max_look_back_period: 0
+
+table_manager:
+  chunk_tables_provisioning:
+    inactive_read_throughput: 0
+    inactive_write_throughput: 0
+    provisioned_read_throughput: 0
+    provisioned_write_throughput: 0
+  index_tables_provisioning:
+    inactive_read_throughput: 0
+    inactive_write_throughput: 0
+    provisioned_read_throughput: 0
+    provisioned_write_throughput: 0
+  retention_deletes_enabled: false
+  retention_period: 0
--- a/software/grafana/promtail-config-file.cfg.in
+++ b/software/grafana/promtail-config-file.cfg.in
+# https://github.com/grafana/loki/blob/master/docs/logentry/processing-log-lines.md
+
+server:
+  http_listen_port: {{ promtail['http_port'] }}
+  # XXX this external_url does not work ... promtail still listen on all IPs
+  external_url: {{ promtail['url'] }}
+  grpc_listen_port: 0
+
+positions:
+  filename: {{ promtail['dir'] }}/positions.yaml
+
+clients:
+  - url: {{ loki['url'] }}/api/prom/push
+
+scrape_configs:
+- job_name: test
+  static_configs:
+  - targets:
+      - localhost
+    labels:
+      job: grafanalogs
+      __path__: ./var/log/*log
+{{ slapparameter_dict.get('promtail-extra-scrape-config', '') }}
--- a/software/grafana/software.cfg
+++ b/software/grafana/software.cfg
@@ -2,10 +2,11 @@
 extends =
  ../../stack/slapos.cfg
  ../../stack/nodejs.cfg
-  ../../stack/monitor/buildout.cfg
  ../../component/make/buildout.cfg
  ../../component/golang/buildout.cfg
  ../../component/openssl/buildout.cfg
+  ../../component/curl/buildout.cfg
+  ../../component/dash/buildout.cfg
  buildout.hash.cfg
  gowork.cfg

@@ -17,21 +18,24 @@ parts =
  influxdb-config-file
  telegraf-config-file
  grafana-config-file
+  grafana-provisioning-config-file
+  loki-config-file
+  promtail-config-file


 [nodejs]
-<= nodejs-8.6.0
+<= nodejs-10.6.0

 [yarn]
 # this could become a component, but it needs to be invoked from nodejs explicitly,
 # otherwise it uses system's nodejs
 recipe = slapos.recipe.build:download-unpacked
-url = https://github.com/yarnpkg/yarn/releases/download/v1.3.2/yarn-v1.3.2.tar.gz
-md5sum = db82fa09c996e9318f2f1d2ab99228f9
+url = https://github.com/yarnpkg/yarn/releases/download/v1.16.0/yarn-v1.16.0.tar.gz
+md5sum = 46790033c23803387890f545e4040690


 [gowork]
-# All the softwares installed in the go work have "non standard" installation
+# All the softwares installed in the go workspace have "non standard" installation
 # methods, so we install them in specific parts with custom commands.
 # They will be installed because they are dependencies of ${gowork.goinstall}
 install =
@@ -41,24 +45,35 @@ influx-bin = ${:bin}/influx
 influxd-bin = ${:bin}/influxd
 grafana-bin = ${:bin}/grafana-server
 grafana-homepath = ${go_github.com_grafana_grafana:location}
+loki-bin = ${:bin}/loki
+promtail-bin = ${:bin}/promtail
+
+# use recent go
+golang  = ${golang1.12:location}
+gcc-bin-directory  = ${golang1.12:gcc-bin-directory}

 [gowork.goinstall]
 command = :
-depends = 
+depends =
   ${influxdb-install:recipe}
   ${telegraf-install:recipe}
   ${grafana-install:recipe}
+   ${loki-install:recipe}
+   ${promtail-install:recipe}

 [influxdb-install]
 <= gowork.goinstall
 command = bash -c ". ${gowork:env.sh} && \
+   go install -v github.com/golang/dep/cmd/dep && \
   cd ${gowork:directory}/src/github.com/influxdata/influxdb && \
+   dep ensure && \
   go install ./cmd/..."
 update-command =

 [telegraf-install]
 <= gowork.goinstall
 command = bash -c ". ${gowork:env.sh} && \
+   go install -v github.com/golang/dep/cmd/dep && \
   cd ${gowork:directory}/src/github.com/influxdata/telegraf && \
   ${make:location}/bin/make &&
   cp telegraf ${gowork:bin}"
@@ -70,12 +85,32 @@ update-command =
 command = bash -c "export PATH=${nodejs:location}/bin/:$PATH && \
   . ${gowork:env.sh} && \
   cd ${gowork:directory}/src/github.com/grafana/grafana && \
-   ${gowork:golang}/bin/go run build.go setup && \
-   ${gowork:golang}/bin/go run build.go build && \
+   go run build.go setup && \
+   go run build.go build && \
   ${yarn:location}/bin/yarn install --pure-lockfile && \
   ${nodejs:location}/bin/npm run build"
 update-command =

+[loki-install]
+<= gowork.goinstall
+# loki also uses nodejs
+command = bash -c "export PATH=${nodejs:location}/bin/:$PATH && \
+   . ${gowork:env.sh} && \
+   go install -v github.com/golang/dep/cmd/dep && \
+   cd ${gowork:directory}/src/github.com/grafana/loki && \
+   go install ./cmd/loki"
+update-command =
+
+[promtail-install]
+<= gowork.goinstall
+# CGO_ENABLED is to disable systemd support (did not compile in my case)
+command = bash -c "export CGO_ENABLED=0 && \
+   . ${gowork:env.sh} && \
+   go install -v github.com/golang/dep/cmd/dep && \
+   cd ${gowork:directory}/src/github.com/grafana/loki && \
+   go install ./cmd/promtail"
+update-command =
+

 [download-file-base]
 recipe = slapos.recipe.build:download
@@ -92,6 +127,15 @@ mode = 0644
 [grafana-config-file]
 <= download-file-base

+[grafana-provisioning-config-file]
+<= download-file-base
+
+[loki-config-file]
+<= download-file-base
+
+[promtail-config-file]
+<= download-file-base
+
 [instance-profile]
 recipe = slapos.recipe.template:jinja2
 template = ${:_profile_base_location_}/${:filename}
@@ -106,10 +150,13 @@ context =
  key influx_bin gowork:influx-bin
  key grafana_bin gowork:grafana-bin
  key grafana_homepath gowork:grafana-homepath
-  key monitor_template monitor2-template:rendered
-
+  key loki_bin gowork:loki-bin
+  key promtail_bin gowork:promtail-bin
+  key curl_bin :curl-bin
+  key dash_bin :dash-bin
+curl-bin = ${curl:location}/bin/curl
+dash-bin = ${dash:location}/bin/dash

 [versions]
 slapos.recipe.template = 4.2
 inotifyx = 0.2.2
-
--- a/software/grafana/test/README.md
+++ b/software/grafana/test/README.md
+Tests for Grafana software release
--- a/software/grafana/test/setup.py
+++ b/software/grafana/test/setup.py
+##############################################################################
+#
+# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from setuptools import setup, find_packages
+
+version = '0.0.1.dev0'
+name = 'slapos.test.grafana'
+long_description = open("README.md").read()
+
+setup(
+    name=name,
+    version=version,
+    description="Test for SlapOS' Grafana",
+    long_description=long_description,
+    long_description_content_type='text/markdown',
+    maintainer="Nexedi",
+    maintainer_email="info@nexedi.com",
+    url="https://lab.nexedi.com/nexedi/slapos",
+    packages=find_packages(),
+    install_requires=[
+        'slapos.core',
+        'slapos.libnetworkcache',
+        'erp5.util',
+        'requests',
+        'supervisor',
+        'psutil',
+    ],
+    zip_safe=True,
+    test_suite='test',
+)
--- a/software/grafana/test/test.py
+++ b/software/grafana/test/test.py
+##############################################################################
+#
+# Copyright (c) 2019 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import os
+import textwrap
+import logging
+import tempfile
+import time
+
+import requests
+
+from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
+
+
+setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
+    os.path.abspath(
+        os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
+
+
+class GrafanaTestCase(SlapOSInstanceTestCase):
+  """Base test case for grafana.
+
+  Since the instances takes timte to start and stop,
+  we increate as lot the number of retries.
+  """
+  report_max_retry = 30
+  instance_max_retry = 30
+
+
+class TestGrafana(GrafanaTestCase):
+  def setUp(self):
+    self.grafana_url = self.computer_partition.getConnectionParameterDict(
+    )['grafana-url']
+
+  def test_grafana_available(self):
+    resp = requests.get(self.grafana_url, verify=False)
+    self.assertEqual(requests.codes.ok, resp.status_code)
+
+  def test_grafana_api(self):
+    # check API is usable
+    api_org_url = '{self.grafana_url}/api/org'.format(**locals())
+    resp = requests.get(api_org_url, verify=False)
+    self.assertEqual(requests.codes.unauthorized, resp.status_code)
+
+    connection_params = self.computer_partition.getConnectionParameterDict()
+    resp = requests.get(
+        api_org_url,
+        verify=False,
+        auth=requests.auth.HTTPBasicAuth(
+            connection_params['grafana-username'],
+            connection_params['grafana-password'],
+        ))
+    self.assertEqual(requests.codes.ok, resp.status_code)
+    self.assertEqual(1, resp.json()['id'])
+
+  def test_grafana_datasource_povisinonned(self):
+    # data sources are provisionned
+    connection_params = self.computer_partition.getConnectionParameterDict()
+    resp = requests.get(
+        '{self.grafana_url}/api/datasources'.format(**locals()),
+        verify=False,
+        auth=requests.auth.HTTPBasicAuth(
+            connection_params['grafana-username'],
+            connection_params['grafana-password'],
+        ))
+    self.assertEqual(requests.codes.ok, resp.status_code)
+    self.assertEqual(
+        sorted(['influxdb', 'loki']),
+        sorted([ds['type'] for ds in resp.json()]))
+
+
+class TestInfluxDb(GrafanaTestCase):
+  def setUp(self):
+    self.influxdb_url = self.computer_partition.getConnectionParameterDict(
+    )['influxdb-url']
+
+  def test_influxdb_available(self):
+    ping_url = '{self.influxdb_url}/ping'.format(**locals())
+    resp = requests.get(ping_url, verify=False)
+    self.assertEqual(requests.codes.no_content, resp.status_code)
+
+  def test_influxdb_api(self):
+    query_url = '{self.influxdb_url}/query'.format(**locals())
+    connection_params = self.computer_partition.getConnectionParameterDict()
+
+    for i in range(10):
+      # retry, as it may take a little delay to create databases
+      resp = requests.get(
+          query_url,
+          verify=False,
+          params=dict(
+              q='SHOW DATABASES',
+              u=connection_params['influxdb-username'],
+              p=connection_params['influxdb-password']))
+      self.assertEqual(requests.codes.ok, resp.status_code)
+      result, = resp.json()['results']
+      if result['series'] and 'values' in result['series'][0]:
+        break
+      time.sleep(0.5 * i)
+
+    self.assertIn(
+        [connection_params['influxdb-database']], result['series'][0]['values'])
+
+
+class TestTelegraf(GrafanaTestCase):
+  def test_telegraf_running(self):
+    with self.slap.instance_supervisor_rpc as supervisor:
+      all_process_info = supervisor.getAllProcessInfo()
+    process_info, = [p for p in all_process_info if 'telegraf' in p['name']]
+    self.assertEqual('RUNNING', process_info['statename'])
+
+
+class TestLoki(GrafanaTestCase):
+  @classmethod
+  def getInstanceParameterDict(cls):
+    cls._logfile = tempfile.NamedTemporaryFile(suffix='log')
+    return {
+        'promtail-extra-scrape-config':
+            textwrap.dedent(
+                r'''
+                - job_name: {cls.__name__}
+                  pipeline_stages:
+                    - regex:
+                        expression: "^(?P<timestamp>.*) - (?P<name>\\S+) - (?P<level>\\S+) - (?P<message>.*)"
+                    - timestamp:
+                        format: 2006-01-02T15:04:05Z00:00
+                        source: timestamp
+                    - labels:
+                        level:
+                        name:
+                  static_configs:
+                  - targets:
+                      - localhost
+                    labels:
+                      job: {cls.__name__}
+                      __path__: {cls._logfile.name}
+                ''').format(**locals())
+    }
+
+  @classmethod
+  def tearDownClass(cls):
+    cls._logfile.close()
+    super(TestLoki, cls).tearDownClass()
+
+  def setUp(self):
+    self.loki_url = self.computer_partition.getConnectionParameterDict(
+    )['loki-url']
+
+  def test_loki_available(self):
+    self.assertEqual(
+        requests.codes.ok,
+        requests.get('{self.loki_url}/ready'.format(**locals()),
+                     verify=False).status_code)
+
+  def test_log_ingested(self):
+    # create a logger logging to the file that we have
+    # configured in instance parameter.
+    test_logger = logging.getLogger(self.id())
+    test_logger.setLevel(logging.INFO)
+    test_handler = logging.FileHandler(filename=self._logfile.name)
+    test_handler.setFormatter(
+        logging.Formatter(
+            '%(asctime)s - %(name)s - %(levelname)s - %(message)s'))
+    test_logger.addHandler(test_handler)
+    test_logger.info("testing message")
+    test_logger.info("testing another message")
+    test_logger.warning("testing warn")
+
+    # Check our messages have been ingested
+    # we retry a few times, because there's a short delay until messages are
+    # ingested and returned.
+    for i in range(10):
+      resp = requests.get(
+          '{self.loki_url}/api/prom/query?query={{job="TestLoki"}}'.format(
+              **locals()),
+          verify=False).json()
+      if not resp:
+        time.sleep(0.5 * i)
+        continue
+
+    warn_stream_list = [stream for stream in resp['streams'] if 'level="WARNING"' in stream['labels']]
+    self.assertEqual(1, len(warn_stream_list), resp['streams'])
+    warn_stream, = warn_stream_list
+    self.assertIn("testing warn", warn_stream['entries'][0]['line'])
+
+    info_stream_list = [stream for stream in resp['streams'] if 'level="INFO"' in stream['labels']]
+    self.assertEqual(1, len(info_stream_list), resp['streams'])
+    info_stream, = info_stream_list
+    self.assertTrue(
+        [
+            line for line in info_stream['entries']
+            if "testing message" in line['line']
+        ])
+    self.assertTrue(
+        [
+            line for line in info_stream['entries']
+            if "testing another message" in line['line']
+        ])
+    # The labels we have configued are also available
+    resp = requests.get(
+        '{self.loki_url}/api/prom/label'.format(**locals()),
+        verify=False).json()
+    self.assertIn('level', resp['values'])
+    self.assertIn('name', resp['values'])
--- a/software/slapos-sr-testing/buildout.hash.cfg
+++ b/software/slapos-sr-testing/buildout.hash.cfg
@@ -15,4 +15,4 @@

 [template]
 filename = instance.cfg
-md5sum = 4664f7dae66d3f582e34cec2ca627501
+md5sum = 1cbab58e896ff63575f6a67db530d183
--- a/software/slapos-sr-testing/instance.cfg
+++ b/software/slapos-sr-testing/instance.cfg
@@ -28,7 +28,7 @@ bin = $${buildout:directory}/bin
 working-dir = $${buildout:directory}/tmp

 [test-list]
-path_list = ${slapos.cookbook-setup:setup},${slapos.test.caddy-frontend-setup:setup},${slapos.test.erp5-setup:setup},${slapos.test.slapos-master-setup:setup},${slapos.test.kvm-setup:setup},${slapos.test.monitor-setup:setup},${slapos.test.plantuml-setup:setup},${slapos.test.powerdns-setup:setup},${slapos.test.proftpd-setup:setup},${slapos.test.re6stnet-setup:setup},${slapos.test.seleniumserver-setup:setup},${slapos.test.slaprunner-setup:setup},${slapos.test.helloworld-setup:setup},${slapos.test.jupyter-setup:setup},${slapos.test.nextcloud-setup:setup},${slapos.test.turnserver-setup:setup},${slapos.test.theia-setup:setup}
+path_list = ${slapos.cookbook-setup:setup},${slapos.test.caddy-frontend-setup:setup},${slapos.test.erp5-setup:setup},${slapos.test.slapos-master-setup:setup},${slapos.test.kvm-setup:setup},${slapos.test.monitor-setup:setup},${slapos.test.plantuml-setup:setup},${slapos.test.powerdns-setup:setup},${slapos.test.proftpd-setup:setup},${slapos.test.re6stnet-setup:setup},${slapos.test.seleniumserver-setup:setup},${slapos.test.slaprunner-setup:setup},${slapos.test.helloworld-setup:setup},${slapos.test.jupyter-setup:setup},${slapos.test.nextcloud-setup:setup},${slapos.test.turnserver-setup:setup},${slapos.test.theia-setup:setup},${slapos.test.grafana-setup:setup}

 [slapos-test-runner]
 recipe = slapos.cookbook:wrapper

--- a/software/slapos-sr-testing/software.cfg
+++ b/software/slapos-sr-testing/software.cfg
@@ -24,7 +24,7 @@ recipe = zc.recipe.egg:develop

 [slapos.cookbook-setup]
 <= setup-develop-egg
-egg = slapos.cookbook
+egg = slapos.cookbook[test]
 setup = ${slapos-repository:location}/

 [slapos.test.caddy-frontend-setup]
@@ -107,6 +107,11 @@ setup = ${slapos-repository:location}/software/turnserver/test/
 egg = slapos.test.theia
 setup = ${slapos-repository:location}/software/theia/test/

+[slapos.test.grafana-setup]
+<= setup-develop-egg
+egg = slapos.test.grafana
+setup = ${slapos-repository:location}/software/grafana/test/
+
 [slapos.core-repository]
 <= git-clone-repository
 repository = https://lab.nexedi.com/nexedi/slapos.core.git
@@ -210,3 +215,7 @@ Django = 1.11
 urllib3 = 1.24.1

 backports.lzma = 0.0.13
+
+mock = 2.0.0
+testfixtures = 6.11
+funcsigs = 1.0.2
--- a/software/slapos-testing/software.cfg
+++ b/software/slapos-testing/software.cfg
@@ -55,14 +55,13 @@ depends = ${slapos.core-setup:egg}

 [slapos.cookbook-setup]
 <= setup-develop-egg
-# XXX slapos.cookbook does not have `test` extra require, `mock` is only listed in `tests_require` and is listed explicitly
-egg = slapos.cookbook
+egg = slapos.cookbook[test]
 setup = ${slapos.cookbook-repository:location}
 depends = ${slapos.core-setup:egg}

 [slapos.core-setup]
 <= setup-develop-egg
-# XXX slapos.cookbook does not have `test` extra require, `mock`, `pyflakes` and `httmock` are only listed in `tests_require` and are listed explicitly
+# XXX slapos.core does not have `test` extra require, `mock`, `pyflakes` and `httmock` are only listed in `tests_require` and are listed explicitly
 egg = slapos.core
 setup = ${slapos.core-repository:location}

@@ -201,12 +200,13 @@ gitdb2 = 2.0.4
 httmock = 1.2.6
 manuel = 1.9.0
 mock = 2.0.0
+testfixtures = 6.11.0
 pem = 18.1.0
 pycurl = 7.43.0.2
 pyflakes = 2.0.0
 smmap2 = 2.0.4
 zope.testing = 4.6.2
-urllib3 = 1.24.1 
+urllib3 = 1.24.1
 # Required by:
 # caucase
 PyJWT = 1.6.4
--- a/stack/erp5/buildout.cfg
+++ b/stack/erp5/buildout.cfg
@@ -5,12 +5,8 @@ extends =
  ztk-versions.cfg
  zope-versions.cfg
  buildout.hash.cfg
-  ../../component/cups/buildout.cfg
-  ../../component/dbus/buildout.cfg
-  ../../component/file/buildout.cfg
  ../../component/fonts/buildout.cfg
  ../../component/gcc/buildout.cfg
-  ../../component/ghostscript/buildout.cfg
  ../../component/git/buildout.cfg
  ../../component/graphviz/buildout.cfg
  ../../component/gzip/buildout.cfg
@@ -19,13 +15,9 @@ extends =
  ../../component/findutils/buildout.cfg
  ../../component/librsvg/buildout.cfg
  ../../component/imagemagick/buildout.cfg
-  ../../component/inkscape/buildout.cfg
  ../../component/kumo/buildout.cfg
  ../../component/libdmtx/buildout.cfg
-  ../../component/libffi/buildout.cfg
-  ../../component/libpng/buildout.cfg
  ../../component/matplotlib/buildout.cfg
-  ../../component/mesa/buildout.cfg
  ../../component/numpy/buildout.cfg
  ../../component/statsmodels/buildout.cfg
  ../../component/h5py/buildout.cfg
@@ -44,7 +36,6 @@ extends =
  ../../component/subversion/buildout.cfg
  ../../component/tesseract/buildout.cfg
  ../../component/w3m/buildout.cfg
-  ../../component/xorg/buildout.cfg
  ../../component/poppler/buildout.cfg
  ../../component/zabbix/buildout.cfg
  ../../component/sed/buildout.cfg
@@ -56,11 +47,9 @@ extends =
  ../../component/aspell/buildout.cfg
  ../../component/jsl/buildout.cfg
  ../../component/6tunnel/buildout.cfg
-  ../../component/findutils/buildout.cfg
  ../../component/userhosts/buildout.cfg
  ../../component/postfix/buildout.cfg
  ../../component/zbarlight/buildout.cfg
-  ../../component/perl/buildout.cfg
  ../../component/pylint/buildout.cfg
  ../../stack/caucase/buildout.cfg
  ../../software/jupyter/software.cfg
@@ -71,52 +60,20 @@ parts +=
  erp5-util-develop
  slapos-cookbook
  rdiff-backup
-  aspell
  aspell-en-dictionary
-  apache
  apache-antiloris
-  file
-  findutils
-  graphviz
-  haproxy
-  jsl
-  w3m
-  poppler
-  libpng
-  ghostscript
  mroonga-mariadb
-  imagemagick
-  inkscape
-  libdmtx
-  dmtx-utils
-  kumo
  tesseract
  tesseract-eng-traineddata
  tesseract-osd-traineddata
-  scipy
-  ocropy
  hookbox
-  percona-toolkit
  zabbix-agent
-  dash
-  bash
-  wget
-  userhosts
-  postfix

 # Buildoutish
-  eggs
  eggs-all-scripts
  testrunner
  test_suite_runner

-# basic Xorg
-  libXdmcp
-  libXext
-  libXau
-  libSM
-  libXrender
-
 # fonts
  liberation-fonts
  ipaex-fonts
@@ -124,9 +81,7 @@ parts +=
  ocrb-fonts
  android-fonts

-
 # get git repositories
-  erp5
  genbt5list

 # some additional utils
@@ -136,9 +91,7 @@ parts +=
  template

 # jupyter
-  jupyter
  jupyter-notebook-initialized-scripts
-  instance-jupyter-notebook

 # override python2.7 to add SlapOS libstdc++ in RPATH.
 [python2.7]
@@ -225,47 +178,24 @@ context =
    key mariadb_link_binary template-mariadb:link-binary
    key zope_link_binary template-zope:link-binary
    key apache_location apache:location
-    key aspell_location aspell:location
    key bin_directory buildout:bin-directory
    key buildout_bin_directory buildout:bin-directory
-    key cairo_location cairo:location
    key caucase_jinja2_library caucase-jinja2-library:target
    key coreutils_location coreutils:location
-    key cups_location cups:location
    key curl_location curl:location
    key cyrus_sasl_location cyrus-sasl:location
    key dash_location dash:location
    key bash_location bash:location
-    key dbus_glib_location dbus-glib:location
-    key dbus_location dbus:location
    key dcron_location dcron:location
    key default_cloudooo_url erp5-defaults:cloudooo-connection-url
    key erp5_location erp5:location
-    key file_location file:location
    key findutils_location findutils:location
-    key fontconfig_location fontconfig:location
-    key fonts_location fonts:location
-    key freetype_location freetype:location
-    key glib_location glib:location
-    key glu_location glu:location
    key gzip_location gzip:location
    key haproxy_location haproxy:location
-    key imagemagick_location imagemagick:location
    key instance_common_cfg instance-common:rendered
    key jsl_location jsl:location
    key jupyter_enable_default erp5-defaults:jupyter-enable-default
    key kumo_location kumo:location
-    key libICE_location libICE:location
-    key libSM_location libSM:location
-    key libX11_location libX11:location
-    key libXau_location libXau:location
-    key libXdmcp_location libXdmcp:location
-    key libXext_location libXext:location
-    key libXrender_location libXrender:location
-    key libexpat_location libexpat:location
-    key libffi_location libffi:location
-    key librsvg_location librsvg:location
-    key libxcb_location libxcb:location
    key local_bt5_repository local-bt5-repository:list
    key logrotate_location logrotate:location
    key mariadb_location mariadb:location
@@ -273,13 +203,10 @@ context =
    key mariadb_slow_query_report_script mariadb-slow-query-report-script:target
    key mariadb_start_clone_from_backup mariadb-start-clone-from-backup:target
    key matplotlibrc_location matplotlibrc:location
-    key mesa_location mesa:location
    key parts_directory buildout:parts-directory
    key openssl_location openssl:location
    key percona_toolkit_location percona-toolkit:location
    key perl_dbd_mariadb_path perl-DBD-mariadb:perl-PATH
-    key pixman_location pixman:location
-    key poppler_location poppler:location
    key postfix_location postfix:location
    key root_common root-common:target
    key site_zcml site-zcml:target
@@ -306,9 +233,6 @@ context =
    key userhosts_location userhosts:location
    key unixodbc_location unixodbc:location
    key wget_location wget:location
-    key xdamage_location xdamage:location
-    key xfixes_location xfixes:location
-    key zlib_location zlib:location
    key extra_path_list eggs:extra-paths

 [template-erp5]

--- a/stack/slapos.cfg
+++ b/stack/slapos.cfg
@@ -138,7 +138,7 @@ pytz = 2016.10
 requests = 2.13.0
 six = 1.12.0
 slapos.cookbook = 1.0.124
-slapos.core = 1.5.7
+slapos.core = 1.5.8
 slapos.extension.strip = 0.4
 slapos.extension.shared = 1.0
 slapos.libnetworkcache = 0.20