Commit 66507954 authored by Nick Thomas's avatar Nick Thomas

Merge branch 'prometheus-instrumentation' into 'master'

Prometheus instrumentation

See merge request gitlab-org/gitlab-workhorse!279
parents 36697e98 f0c9ec54
...@@ -4,10 +4,8 @@ import ( ...@@ -4,10 +4,8 @@ import (
"bufio" "bufio"
"net" "net"
"net/http" "net/http"
"strconv"
"time" "time"
"github.com/prometheus/client_golang/prometheus"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
logging "gitlab.com/gitlab-org/gitlab-workhorse/internal/log" logging "gitlab.com/gitlab-org/gitlab-workhorse/internal/log"
...@@ -15,35 +13,13 @@ import ( ...@@ -15,35 +13,13 @@ import (
var ( var (
accessLogEntry *log.Entry accessLogEntry *log.Entry
sessionsActive = prometheus.NewGauge(prometheus.GaugeOpts{
Name: "gitlab_workhorse_http_sessions_active",
Help: "Number of HTTP request-response cycles currently being handled by gitlab-workhorse.",
})
requestsTotal = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "gitlab_workhorse_http_requests_total",
Help: "How many HTTP requests have been processed by gitlab-workhorse, partitioned by status code and HTTP method.",
},
[]string{"code", "method"},
)
) )
func init() {
registerPrometheusMetrics()
}
// SetAccessLoggerEntry sets the access logger used in the system // SetAccessLoggerEntry sets the access logger used in the system
func SetAccessLoggerEntry(logEntry *log.Entry) { func SetAccessLoggerEntry(logEntry *log.Entry) {
accessLogEntry = logEntry accessLogEntry = logEntry
} }
func registerPrometheusMetrics() {
prometheus.MustRegister(sessionsActive)
prometheus.MustRegister(requestsTotal)
}
type LoggingResponseWriter interface { type LoggingResponseWriter interface {
http.ResponseWriter http.ResponseWriter
...@@ -63,7 +39,6 @@ type hijackingResponseWriter struct { ...@@ -63,7 +39,6 @@ type hijackingResponseWriter struct {
} }
func NewStatsCollectingResponseWriter(rw http.ResponseWriter) LoggingResponseWriter { func NewStatsCollectingResponseWriter(rw http.ResponseWriter) LoggingResponseWriter {
sessionsActive.Inc()
out := statsCollectingResponseWriter{ out := statsCollectingResponseWriter{
rw: rw, rw: rw,
started: time.Now(), started: time.Now(),
...@@ -135,7 +110,4 @@ func (l *statsCollectingResponseWriter) accessLogFields(r *http.Request) log.Fie ...@@ -135,7 +110,4 @@ func (l *statsCollectingResponseWriter) accessLogFields(r *http.Request) log.Fie
func (l *statsCollectingResponseWriter) RequestFinished(r *http.Request) { func (l *statsCollectingResponseWriter) RequestFinished(r *http.Request) {
l.writeAccessLog(r) l.writeAccessLog(r)
sessionsActive.Dec()
requestsTotal.WithLabelValues(strconv.Itoa(l.status), r.Method).Inc()
} }
package upstream
import (
"net/http"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
)
const (
namespace = "gitlab_workhorse"
httpSubsystem = "http"
)
func secondsDurationBuckets() []float64 {
return []float64{
0.005, /* 5ms */
0.025, /* 25ms */
0.1, /* 100ms */
0.5, /* 500ms */
1.0, /* 1s */
10.0, /* 10s */
30.0, /* 30s */
60.0, /* 1m */
300.0, /* 10m */
}
}
func byteSizeBuckets() []float64 {
return []float64{
10,
64,
256,
1024, /* 1kB */
64 * 1024, /* 64kB */
256 * 1024, /* 256kB */
1024 * 1024, /* 1mB */
64 * 1024 * 1024, /* 64mB */
}
}
var (
httpInFlightRequests = prometheus.NewGauge(prometheus.GaugeOpts{
Namespace: namespace,
Subsystem: httpSubsystem,
Name: "in_flight_requests",
Help: "A gauge of requests currently being served by workhorse.",
})
httpRequestsTotal = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: namespace,
Subsystem: httpSubsystem,
Name: "requests_total",
Help: "A counter for requests to workhorse.",
},
[]string{"code", "method", "route"},
)
httpRequestDurationSeconds = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Namespace: namespace,
Subsystem: httpSubsystem,
Name: "request_duration_seconds",
Help: "A histogram of latencies for requests to workhorse.",
Buckets: secondsDurationBuckets(),
},
[]string{"code", "method", "route"},
)
httpRequestSizeBytes = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Namespace: namespace,
Subsystem: httpSubsystem,
Name: "request_size_bytes",
Help: "A histogram of sizes of requests to workhorse.",
Buckets: byteSizeBuckets(),
},
[]string{"code", "method", "route"},
)
httpResponseSizeBytes = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Namespace: namespace,
Subsystem: httpSubsystem,
Name: "response_size_bytes",
Help: "A histogram of response sizes for requests to workhorse.",
Buckets: byteSizeBuckets(),
},
[]string{"code", "method", "route"},
)
httpTimeToWriteHeaderSeconds = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Namespace: namespace,
Subsystem: httpSubsystem,
Name: "time_to_write_header_seconds",
Help: "A histogram of request durations until the response headers are written.",
Buckets: secondsDurationBuckets(),
},
[]string{"code", "method", "route"},
)
)
func init() {
prometheus.MustRegister(httpInFlightRequests)
prometheus.MustRegister(httpRequestsTotal)
prometheus.MustRegister(httpRequestDurationSeconds)
prometheus.MustRegister(httpRequestSizeBytes)
prometheus.MustRegister(httpTimeToWriteHeaderSeconds)
}
func instrumentRoute(next http.Handler, method string, regexpStr string) http.Handler {
var handler http.Handler
handler = next
handler = promhttp.InstrumentHandlerCounter(httpRequestsTotal.MustCurryWith(map[string]string{"route": regexpStr}), handler)
handler = promhttp.InstrumentHandlerDuration(httpRequestDurationSeconds.MustCurryWith(map[string]string{"route": regexpStr}), handler)
handler = promhttp.InstrumentHandlerInFlight(httpInFlightRequests, handler)
handler = promhttp.InstrumentHandlerRequestSize(httpRequestSizeBytes.MustCurryWith(map[string]string{"route": regexpStr}), handler)
handler = promhttp.InstrumentHandlerResponseSize(httpResponseSizeBytes.MustCurryWith(map[string]string{"route": regexpStr}), handler)
handler = promhttp.InstrumentHandlerTimeToWriteHeader(httpTimeToWriteHeaderSeconds.MustCurryWith(map[string]string{"route": regexpStr}), handler)
return handler
}
...@@ -4,10 +4,8 @@ import ( ...@@ -4,10 +4,8 @@ import (
"net/http" "net/http"
"path" "path"
"regexp" "regexp"
"time"
"github.com/gorilla/websocket" "github.com/gorilla/websocket"
"github.com/prometheus/client_golang/prometheus"
apipkg "gitlab.com/gitlab-org/gitlab-workhorse/internal/api" apipkg "gitlab.com/gitlab-org/gitlab-workhorse/internal/api"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/artifacts" "gitlab.com/gitlab-org/gitlab-workhorse/internal/artifacts"
...@@ -42,20 +40,6 @@ const ( ...@@ -42,20 +40,6 @@ const (
projectPattern = `^/([^/]+/){1,}[^/]+/` projectPattern = `^/([^/]+/){1,}[^/]+/`
) )
var (
routeRequestDurations = prometheus.NewHistogramVec(prometheus.HistogramOpts{
Name: "gitlab_workhorse_request_duration_seconds",
Help: "A histogram of request times in seconds",
Buckets: prometheus.ExponentialBuckets(0.01, 2.5, 10),
},
[]string{"method", "route"},
)
)
func init() {
prometheus.MustRegister(routeRequestDurations)
}
func compileRegexp(regexpStr string) *regexp.Regexp { func compileRegexp(regexpStr string) *regexp.Regexp {
if len(regexpStr) == 0 { if len(regexpStr) == 0 {
return nil return nil
...@@ -64,19 +48,11 @@ func compileRegexp(regexpStr string) *regexp.Regexp { ...@@ -64,19 +48,11 @@ func compileRegexp(regexpStr string) *regexp.Regexp {
return regexp.MustCompile(regexpStr) return regexp.MustCompile(regexpStr)
} }
func instrumentDuration(h http.Handler, method string, regexpStr string) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
start := time.Now()
h.ServeHTTP(w, r)
routeRequestDurations.WithLabelValues(method, regexpStr).Observe(time.Since(start).Seconds())
})
}
func route(method, regexpStr string, handler http.Handler, matchers ...matcherFunc) routeEntry { func route(method, regexpStr string, handler http.Handler, matchers ...matcherFunc) routeEntry {
return routeEntry{ return routeEntry{
method: method, method: method,
regex: compileRegexp(regexpStr), regex: compileRegexp(regexpStr),
handler: instrumentDuration(denyWebsocket(handler), method, regexpStr), handler: instrumentRoute(denyWebsocket(handler), method, regexpStr),
matchers: matchers, matchers: matchers,
} }
} }
...@@ -85,7 +61,7 @@ func wsRoute(regexpStr string, handler http.Handler, matchers ...matcherFunc) ro ...@@ -85,7 +61,7 @@ func wsRoute(regexpStr string, handler http.Handler, matchers ...matcherFunc) ro
return routeEntry{ return routeEntry{
method: "GET", method: "GET",
regex: compileRegexp(regexpStr), regex: compileRegexp(regexpStr),
handler: instrumentDuration(handler, "GET", regexpStr), handler: instrumentRoute(handler, "GET", regexpStr),
matchers: append(matchers, websocket.IsWebSocketUpgrade), matchers: append(matchers, websocket.IsWebSocketUpgrade),
} }
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment