Commit ca4fd5ea authored by Andrew Newdigate's avatar Andrew Newdigate

Adds X-Request-Id response header for Workhorse

This change configures workhorse to set the Correlation-ID value in
the response header. This is a common technique which allows errors to
be traced more easily.

This can be done through several methods: 1) users can review their
browser headers and report the value 2) nginx error pages could be
configured to report the correlation-id on the error page, allowing
users to easily report it.

Once the correlation-id has been reported, support engineers or
production staff can use this to further investigate issues using
logging infrastructure, sentry or distributed tracing.
parent 50c3937e
...@@ -152,7 +152,12 @@ func main() { ...@@ -152,7 +152,12 @@ func main() {
} }
} }
up := wrapRaven(correlation.InjectCorrelationID(upstream.NewUpstream(cfg))) up := wrapRaven(
correlation.InjectCorrelationID(
upstream.NewUpstream(cfg),
correlation.WithSetResponseHeader(),
),
)
logger.Fatal(http.Serve(listener, up)) logger.Fatal(http.Serve(listener, up))
} }
...@@ -11,7 +11,6 @@ import ( ...@@ -11,7 +11,6 @@ import (
// Whether the Correlation-ID is generated or propagated, once inside this handler the request context // Whether the Correlation-ID is generated or propagated, once inside this handler the request context
// will have a Correlation-ID associated with it. // will have a Correlation-ID associated with it.
func InjectCorrelationID(h http.Handler, opts ...InboundHandlerOption) http.Handler { func InjectCorrelationID(h http.Handler, opts ...InboundHandlerOption) http.Handler {
// Currently we don't use any of the options available
config := applyInboundHandlerOptions(opts) config := applyInboundHandlerOptions(opts)
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
...@@ -26,6 +25,10 @@ func InjectCorrelationID(h http.Handler, opts ...InboundHandlerOption) http.Hand ...@@ -26,6 +25,10 @@ func InjectCorrelationID(h http.Handler, opts ...InboundHandlerOption) http.Hand
correlationID = generateRandomCorrelationIDWithFallback(r) correlationID = generateRandomCorrelationIDWithFallback(r)
} }
if config.sendResponseHeader {
setResponseHeader(w, correlationID)
}
ctx := ContextWithCorrelation(parent, correlationID) ctx := ContextWithCorrelation(parent, correlationID)
h.ServeHTTP(w, r.WithContext(ctx)) h.ServeHTTP(w, r.WithContext(ctx))
}) })
...@@ -34,3 +37,7 @@ func InjectCorrelationID(h http.Handler, opts ...InboundHandlerOption) http.Hand ...@@ -34,3 +37,7 @@ func InjectCorrelationID(h http.Handler, opts ...InboundHandlerOption) http.Hand
func extractFromRequest(r *http.Request) string { func extractFromRequest(r *http.Request) string {
return r.Header.Get(propagationHeader) return r.Header.Get(propagationHeader)
} }
func setResponseHeader(w http.ResponseWriter, correlationID string) {
w.Header().Set(propagationHeader, correlationID)
}
...@@ -3,6 +3,7 @@ package correlation ...@@ -3,6 +3,7 @@ package correlation
// The configuration for InjectCorrelationID // The configuration for InjectCorrelationID
type inboundHandlerConfig struct { type inboundHandlerConfig struct {
propagation bool propagation bool
sendResponseHeader bool
} }
// InboundHandlerOption will configure a correlation handler // InboundHandlerOption will configure a correlation handler
...@@ -29,3 +30,11 @@ func WithPropagation() InboundHandlerOption { ...@@ -29,3 +30,11 @@ func WithPropagation() InboundHandlerOption {
config.propagation = true config.propagation = true
} }
} }
// WithSetResponseHeader will configure the handler to set the correlation_id
// in the http response headers
func WithSetResponseHeader() InboundHandlerOption {
return func(config *inboundHandlerConfig) {
config.sendResponseHeader = true
}
}
/*
Package `tracing` is the primary entrypoint into LabKit's distributed tracing functionality.
(This documentation assumes some minimal knowledge of Distributed Tracing, and uses
tracing terminology without providing definitions. Please review
https://opentracing.io/docs/overview/what-is-tracing/ for an broad overview of distributed
tracing if you are not familiar with the technology)
Internally the `tracing` package relies on Opentracing, but avoids leaking this abstraction.
In theory, LabKit could replace Opentracing with another distributed tracing interface, such
as Zipkin or OpenCensus, without needing to make changes to the application (other than vendoring
in a new version of LabKit, of course).
This design decision is deliberate: the package should not leak the underlying tracing implementation.
The package provides three primary exports:
* `tracing.Initialize()` for initializing the global tracer using the `GITLAB_TRACING` environment variable.
* An HTTP Handler middleware, `tracing.Handler()`, for instrumenting incoming HTTP requests.
* An HTTP RoundTripper, `tracing.NewRoundTripper()` for instrumenting outbound HTTP requests to other services.
The provided example in `example_test.go` demonstrates usage of both the HTTP Middleware and the HTTP RoundTripper.
*Initializing the global tracer*
Opentracing makes use of a global tracer. Opentracing ships with a default NoOp tracer which does
nothing at all. This is always configured, meaning that, without initialization, Opentracing does nothing and
has a very low overhead.
LabKit's tracing is configured through an environment variable, `GITLAB_TRACING`. This environment variable contains
a "connection string"-like configuration, such as:
* `opentracing://jaeger?udp_endpoint=localhost:6831`
* `opentracing://datadog`
* `opentracing://lightstep`
The parameters for these connection-strings are implementation specific.
This configuration is identical to the one used to configure GitLab's ruby tracing libraries in the `Gitlab::Tracing`
package. Having a consistent configuration makes it easy to configure multiple processes at the same time. For example,
in GitLab Development Kit, tracing can be configured with a single environment variable, `GITLAB_TRACING=... gdk run`,
since `GITLAB_TRACING` will configure Workhorse (written in Go), Gitaly (written in Go) and GitLab's rails components,
using the same configuration.
*Compiling applications with Tracing support*
Go's Opentracing interface does not allow tracing implementations to be loaded dynamically; implementations need to be
compiled into the application. With LabKit, this is done conditionally, using build tags. Two build tags need to be
specified:
* `tracer_static` - this compiles in the static plugin registry support
* `tracer_static_[DRIVER_NAME]` - this compile in support for the given driver.
For example, to compile support for Jaeger, compile your Go app with `tracer_static,tracer_static_jaeger`
Note that multiple (or all) drivers can be compiled in alongside one another: using the tags:
`tracer_static,tracer_static_jaeger,tracer_static_lightstep,tracer_static_datadog`
If the `GITLAB_TRACING` environment variable references an unknown or unregistered driver, it will log a message
and continue without tracing. This is a deliberate decision: the risk of bringing down a cluster during a rollout
with a misconfigured tracer configuration is greater than the risk of an operator loosing some time because
their application was not compiled with the correct tracers.
*Using the HTTP Handler middleware to instrument incoming HTTP requests*
When an incoming HTTP request arrives on the server, it may already include Distributed Tracing headers,
propagated from an upstream service.
The tracing middleware will attempt to extract the tracing information from the headers (the exact headers used are
tracing implementation specific), set up a span and pass the information through the request context.
It is up to the Opentracing implementation to decide whether the span will be sent to the tracing infrastructure.
This will be implementation-specific, but generally relies on server load, sampler configuration, whether an
error occurred, whether certain spans took an anomalous amount of time, etc.
*Using the HTTP RoundTripper to instrument outgoing HTTP requests*
The RoundTripper should be added to the HTTP client RoundTripper stack (see the example). When an outbound
HTTP request is sent from the HTTP client, the RoundTripper will determine whether there is an active span
and if so, will inject headers into the outgoing HTTP request identifying the span. The details of these
headers is implementation specific.
It is important to ensure that the context is passed into the outgoing request, using `req.WithContext(ctx)`
so that the correct span information can be injected into the request headers.
*/
package tracing
...@@ -603,58 +603,58 @@ ...@@ -603,58 +603,58 @@
"versionExact": "v0.71.0" "versionExact": "v0.71.0"
}, },
{ {
"checksumSHA1": "R6fNN36q3UydLODupK0vdx9h/CY=", "checksumSHA1": "WMOuBgCyclqy+Mqunb0NbykaC4Y=",
"path": "gitlab.com/gitlab-org/labkit/correlation", "path": "gitlab.com/gitlab-org/labkit/correlation",
"revision": "2a3e1f5415a890402696dcbb2c31b5a79c17b579", "revision": "4b0e4d5df664c0f0680f3479ce7cc3378f04d40e",
"revisionTime": "2019-01-08T10:46:58Z", "revisionTime": "2019-02-11T18:00:21Z",
"version": "master", "version": "master",
"versionExact": "master" "versionExact": "master"
}, },
{ {
"checksumSHA1": "UFBFulprWZHuL9GHhjCKoHXm+Ww=", "checksumSHA1": "UFBFulprWZHuL9GHhjCKoHXm+Ww=",
"path": "gitlab.com/gitlab-org/labkit/correlation/grpc", "path": "gitlab.com/gitlab-org/labkit/correlation/grpc",
"revision": "2a3e1f5415a890402696dcbb2c31b5a79c17b579", "revision": "4b0e4d5df664c0f0680f3479ce7cc3378f04d40e",
"revisionTime": "2019-01-08T10:46:58Z", "revisionTime": "2019-02-11T18:00:21Z",
"version": "master", "version": "master",
"versionExact": "master" "versionExact": "master"
}, },
{ {
"checksumSHA1": "YyeJ2t/STDZj3X4xetPzTC0I9hU=", "checksumSHA1": "YyeJ2t/STDZj3X4xetPzTC0I9hU=",
"path": "gitlab.com/gitlab-org/labkit/correlation/raven", "path": "gitlab.com/gitlab-org/labkit/correlation/raven",
"revision": "2a3e1f5415a890402696dcbb2c31b5a79c17b579", "revision": "4b0e4d5df664c0f0680f3479ce7cc3378f04d40e",
"revisionTime": "2019-01-08T10:46:58Z", "revisionTime": "2019-02-11T18:00:21Z",
"version": "master", "version": "master",
"versionExact": "master" "versionExact": "master"
}, },
{ {
"checksumSHA1": "JAe9JIwWcJvTUxCbdOPAZxaXIz8=", "checksumSHA1": "27mCUL3y0j+3NB/+YLWxQLTiSIU=",
"path": "gitlab.com/gitlab-org/labkit/tracing", "path": "gitlab.com/gitlab-org/labkit/tracing",
"revision": "2a3e1f5415a890402696dcbb2c31b5a79c17b579", "revision": "4b0e4d5df664c0f0680f3479ce7cc3378f04d40e",
"revisionTime": "2019-01-08T10:46:58Z", "revisionTime": "2019-02-11T18:00:21Z",
"version": "master", "version": "master",
"versionExact": "master" "versionExact": "master"
}, },
{ {
"checksumSHA1": "uIvjqXAsMQK/Y5FgWRaGydYGbYs=", "checksumSHA1": "uIvjqXAsMQK/Y5FgWRaGydYGbYs=",
"path": "gitlab.com/gitlab-org/labkit/tracing/connstr", "path": "gitlab.com/gitlab-org/labkit/tracing/connstr",
"revision": "2a3e1f5415a890402696dcbb2c31b5a79c17b579", "revision": "4b0e4d5df664c0f0680f3479ce7cc3378f04d40e",
"revisionTime": "2019-01-08T10:46:58Z", "revisionTime": "2019-02-11T18:00:21Z",
"version": "master", "version": "master",
"versionExact": "master" "versionExact": "master"
}, },
{ {
"checksumSHA1": "IE38In/zPKpmKqvWAAThyaufQak=", "checksumSHA1": "IE38In/zPKpmKqvWAAThyaufQak=",
"path": "gitlab.com/gitlab-org/labkit/tracing/grpc", "path": "gitlab.com/gitlab-org/labkit/tracing/grpc",
"revision": "2a3e1f5415a890402696dcbb2c31b5a79c17b579", "revision": "4b0e4d5df664c0f0680f3479ce7cc3378f04d40e",
"revisionTime": "2019-01-08T10:46:58Z", "revisionTime": "2019-02-11T18:00:21Z",
"version": "master", "version": "master",
"versionExact": "master" "versionExact": "master"
}, },
{ {
"checksumSHA1": "hB59Es/WTWfBPLSAheQaRyHGSXA=", "checksumSHA1": "hB59Es/WTWfBPLSAheQaRyHGSXA=",
"path": "gitlab.com/gitlab-org/labkit/tracing/impl", "path": "gitlab.com/gitlab-org/labkit/tracing/impl",
"revision": "2a3e1f5415a890402696dcbb2c31b5a79c17b579", "revision": "4b0e4d5df664c0f0680f3479ce7cc3378f04d40e",
"revisionTime": "2019-01-08T10:46:58Z", "revisionTime": "2019-02-11T18:00:21Z",
"version": "master", "version": "master",
"versionExact": "master" "versionExact": "master"
}, },
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment