Commit f5c3749a authored by Andrew Newdigate's avatar Andrew Newdigate Committed by Jacob Vosmaer

Move correlation libraries out of Workhorse and into LabKit

parent 1c05820b
......@@ -2,6 +2,10 @@
Formerly known as 'gitlab-git-http-server'.
UNRELEASED
- Extract correlation code out to the LabKit project !323
v 7.2.0
- Update CI matrix to go1.10 + go1.11 and fix ResponseWriter bugs !309
......
package correlation
import (
"fmt"
"testing"
"github.com/stretchr/testify/require"
)
func TestReverseBase62Conversion(t *testing.T) {
tests := []struct {
n int64
expected string
}{
{n: 0, expected: "0"},
{n: 5, expected: "5"},
{n: 10, expected: "a"},
{n: 62, expected: "01"},
{n: 620, expected: "0a"},
{n: 6200, expected: "0C1"},
}
for _, test := range tests {
t.Run(fmt.Sprintf("%d_to_%s", test.n, test.expected), func(t *testing.T) {
require.Equal(t, test.expected, encodeReverseBase62(test.n))
})
}
}
package correlation
import (
"context"
"testing"
)
func TestExtractFromContext(t *testing.T) {
tests := []struct {
name string
ctx context.Context
want string
}{
{"nil", nil, ""},
{"missing", context.Background(), ""},
{"set", context.WithValue(context.Background(), keyCorrelationID, "CORRELATION_ID"), "CORRELATION_ID"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := ExtractFromContext(tt.ctx); got != tt.want {
t.Errorf("ExtractFromContext() = %v, want %v", got, tt.want)
}
})
}
}
func TestContextWithCorrelation(t *testing.T) {
tests := []struct {
name string
ctx context.Context
correlationID string
wantValue string
}{
{
name: "nil with value",
ctx: nil,
correlationID: "CORRELATION_ID",
wantValue: "CORRELATION_ID",
},
{
name: "nil with empty string",
ctx: nil,
correlationID: "",
wantValue: "",
},
{
name: "value",
ctx: context.Background(),
correlationID: "CORRELATION_ID",
wantValue: "CORRELATION_ID",
},
{
name: "empty",
ctx: context.Background(),
correlationID: "",
wantValue: "",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := ContextWithCorrelation(tt.ctx, tt.correlationID)
gotValue := got.Value(keyCorrelationID)
if gotValue != tt.wantValue {
t.Errorf("ContextWithCorrelation().Value() = %v, want %v", gotValue, tt.wantValue)
}
})
}
}
package correlation
import (
"net/http"
"strings"
"testing"
"github.com/stretchr/testify/require"
)
func Test_generateRandomCorrelationID(t *testing.T) {
require := require.New(t)
got, err := generateRandomCorrelationID()
require.NoError(err)
require.NotEqual(got, "", "Expected a non-empty string response")
}
func Test_generatePseudorandomCorrelationID(t *testing.T) {
require := require.New(t)
req, err := http.NewRequest("GET", "http://example.com", nil)
require.NoError(err)
got := generatePseudorandomCorrelationID(req)
require.NotEqual(got, "", "Expected a non-empty string response")
require.True(strings.HasPrefix(got, "E:"), "Expected the psuedorandom correlator to have an `E:` prefix")
}
func Test_generateRandomCorrelationIDWithFallback(t *testing.T) {
require := require.New(t)
req, err := http.NewRequest("GET", "http://example.com", nil)
require.NoError(err)
got := generateRandomCorrelationIDWithFallback(req)
require.NotEqual(got, "", "Expected a non-empty string response")
require.False(strings.HasPrefix(got, "E:"), "Not expecting fallback to pseudorandom correlationID")
}
package correlation
import (
"bytes"
"crypto/rand"
"io"
"net/http"
"net/http/httptest"
"testing"
"github.com/stretchr/testify/require"
)
func TestInjectCorrelationID(t *testing.T) {
tests := []struct {
name string
randSource io.Reader
}{
{name: "Entropy Available", randSource: rand.Reader},
{name: "No Entropy", randSource: &bytes.Buffer{}},
}
defer func() {
randSource = rand.Reader
}()
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
invoked := false
randSource = test.randSource
h := InjectCorrelationID(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) {
invoked = true
ctx := r.Context()
correlationID := ExtractFromContext(ctx)
require.NotNil(t, correlationID, "CorrelationID is missing")
require.NotEmpty(t, correlationID, "CorrelationID is missing")
}))
r := httptest.NewRequest("GET", "http://example.com", nil)
h.ServeHTTP(nil, r)
require.True(t, invoked, "handler not executed")
})
}
}
package correlation
import (
"context"
"errors"
"net/http"
"testing"
"github.com/stretchr/testify/require"
)
var httpCorrelationTests = []struct {
name string
ctx context.Context
correlationID string
hasHeader bool
}{
{
name: "nil with value",
ctx: nil,
correlationID: "CORRELATION_ID",
hasHeader: true,
},
{
name: "nil without value",
ctx: nil,
correlationID: "",
hasHeader: false,
},
{
name: "context with value",
ctx: context.Background(),
correlationID: "CORRELATION_ID",
hasHeader: true,
},
{
name: "context without value",
ctx: context.Background(),
correlationID: "",
hasHeader: false,
},
}
func Test_injectRequest(t *testing.T) {
for _, tt := range httpCorrelationTests {
t.Run(tt.name, func(t *testing.T) {
require := require.New(t)
ctx := context.WithValue(tt.ctx, keyCorrelationID, tt.correlationID)
req, err := http.NewRequest("GET", "http://example.com", nil)
require.NoError(err)
req = req.WithContext(ctx)
injectRequest(req)
value := req.Header.Get(propagationHeader)
require.True(tt.hasHeader == (value != ""), "Expected header existence %v. Instead got header %v", tt.hasHeader, value)
if tt.hasHeader {
require.Equal(tt.correlationID, value, "Expected header value %v, got %v", tt.correlationID, value)
}
})
}
}
type delegatedRoundTripper struct {
delegate func(req *http.Request) (*http.Response, error)
}
func (c delegatedRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
return c.delegate(req)
}
func roundTripperFunc(delegate func(req *http.Request) (*http.Response, error)) http.RoundTripper {
return &delegatedRoundTripper{delegate}
}
func TestInstrumentedRoundTripper(t *testing.T) {
for _, tt := range httpCorrelationTests {
t.Run(tt.name, func(t *testing.T) {
require := require.New(t)
response := &http.Response{}
mockTransport := roundTripperFunc(func(req *http.Request) (*http.Response, error) {
value := req.Header.Get(propagationHeader)
require.True(tt.hasHeader == (value != ""), "Expected header existence %v. Instead got header %v", tt.hasHeader, value)
if tt.hasHeader {
require.Equal(tt.correlationID, value, "Expected header value %v, got %v", tt.correlationID, value)
}
return response, nil
})
client := &http.Client{
Transport: NewInstrumentedRoundTripper(mockTransport),
}
ctx := context.WithValue(tt.ctx, keyCorrelationID, tt.correlationID)
req, err := http.NewRequest("GET", "http://example.com", nil)
require.NoError(err)
req = req.WithContext(ctx)
res, err := client.Do(req)
require.NoError(err)
require.Equal(response, res)
})
}
}
func TestInstrumentedRoundTripperFailures(t *testing.T) {
for _, tt := range httpCorrelationTests {
t.Run(tt.name+" - with errors", func(t *testing.T) {
require := require.New(t)
testErr := errors.New("test")
mockTransport := roundTripperFunc(func(req *http.Request) (*http.Response, error) {
value := req.Header.Get(propagationHeader)
require.True(tt.hasHeader == (value != ""), "Expected header existence %v. Instead got header %v", tt.hasHeader, value)
if tt.hasHeader {
require.Equal(tt.correlationID, value, "Expected header value %v, got %v", tt.correlationID, value)
}
return nil, testErr
})
client := &http.Client{
Transport: NewInstrumentedRoundTripper(mockTransport),
}
ctx := context.WithValue(tt.ctx, keyCorrelationID, tt.correlationID)
req, err := http.NewRequest("GET", "http://example.com", nil)
require.NoError(err)
req = req.WithContext(ctx)
res, err := client.Do(req)
require.Error(err)
require.Nil(res)
})
}
}
func TestInstrumentedRoundTripperWithoutContext(t *testing.T) {
require := require.New(t)
response := &http.Response{}
mockTransport := roundTripperFunc(func(req *http.Request) (*http.Response, error) {
return response, nil
})
client := &http.Client{
Transport: NewInstrumentedRoundTripper(mockTransport),
}
req, err := http.NewRequest("GET", "http://example.com", nil)
require.NoError(err)
res, err := client.Do(req)
require.NoError(err)
require.Equal(response, res)
}
package raven
import (
"context"
"reflect"
"testing"
raven "github.com/getsentry/raven-go"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/correlation"
)
func TestSetExtra(t *testing.T) {
tests := []struct {
name string
ctx context.Context
extra raven.Extra
want raven.Extra
}{
{
name: "trivial",
ctx: nil,
extra: nil,
want: raven.Extra{},
},
{
name: "no_context",
ctx: nil,
extra: map[string]interface{}{
"key": "value",
},
want: map[string]interface{}{
"key": "value",
},
},
{
name: "context",
ctx: correlation.ContextWithCorrelation(context.Background(), "C001"),
extra: map[string]interface{}{
"key": "value",
},
want: map[string]interface{}{
"key": "value",
ravenSentryExtraKey: "C001",
},
},
{
name: "no_injected_extras",
ctx: correlation.ContextWithCorrelation(context.Background(), "C001"),
extra: nil,
want: map[string]interface{}{
ravenSentryExtraKey: "C001",
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := SetExtra(tt.ctx, tt.extra); !reflect.DeepEqual(got, tt.want) {
t.Errorf("SetExtra() = %v, want %v", got, tt.want)
}
})
}
}
......@@ -10,7 +10,7 @@ import (
gitalyclient "gitlab.com/gitlab-org/gitaly/client"
"google.golang.org/grpc"
grpccorrelation "gitlab.com/gitlab-org/gitlab-workhorse/internal/correlation/grpc"
grpccorrelation "gitlab.com/gitlab-org/labkit/correlation/grpc"
)
type Server struct {
......@@ -114,14 +114,14 @@ func newConnection(server Server) (*grpc.ClientConn, error) {
grpc.WithStreamInterceptor(
grpc_middleware.ChainStreamClient(
grpc_prometheus.StreamClientInterceptor,
grpccorrelation.StreamClientCorrelationInterceptor,
grpccorrelation.StreamClientCorrelationInterceptor(),
),
),
grpc.WithUnaryInterceptor(
grpc_middleware.ChainUnaryClient(
grpc_prometheus.UnaryClientInterceptor,
grpccorrelation.UnaryClientCorrelationInterceptor,
grpccorrelation.UnaryClientCorrelationInterceptor(),
),
),
)
......
......@@ -6,7 +6,7 @@ import (
"github.com/getsentry/raven-go"
correlation "gitlab.com/gitlab-org/gitlab-workhorse/internal/correlation/raven"
correlation "gitlab.com/gitlab-org/labkit/correlation/raven"
)
var ravenHeaderBlacklist = []string{
......
......@@ -5,7 +5,7 @@ import (
"github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/correlation"
"gitlab.com/gitlab-org/labkit/correlation"
)
// Fields type, an helper to avoid importing logrus.Fields
......
......@@ -8,7 +8,8 @@ import (
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/require"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/correlation"
"gitlab.com/gitlab-org/labkit/correlation"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/log"
)
......
......@@ -9,7 +9,8 @@ import (
"net/http"
"time"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/correlation"
"gitlab.com/gitlab-org/labkit/correlation"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
)
......
......@@ -9,7 +9,8 @@ import (
"github.com/prometheus/client_golang/prometheus"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/correlation"
"gitlab.com/gitlab-org/labkit/correlation"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/log"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/senddata"
......
......@@ -8,8 +8,9 @@ import (
"net/url"
"time"
"gitlab.com/gitlab-org/labkit/correlation"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/badgateway"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/correlation"
)
func mustParseAddress(address, scheme string) string {
......
......@@ -13,7 +13,8 @@ import (
"github.com/jfbus/httprs"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/correlation"
"gitlab.com/gitlab-org/labkit/correlation"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
)
......
......@@ -25,8 +25,9 @@ import (
"github.com/prometheus/client_golang/prometheus/promhttp"
"gitlab.com/gitlab-org/labkit/correlation"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/config"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/correlation"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/log"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/queueing"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/redis"
......
The MIT License (MIT)
Copyright (c) 2016-2017 GitLab B.V.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
......@@ -6,11 +6,9 @@ import (
"google.golang.org/grpc"
"google.golang.org/grpc/metadata"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/correlation"
"gitlab.com/gitlab-org/labkit/correlation"
)
const metadataCorrelatorKey = "X-GitLab-Correlation-ID"
func injectFromContext(ctx context.Context) context.Context {
correlationID := correlation.ExtractFromContext(ctx)
if correlationID != "" {
......@@ -21,13 +19,17 @@ func injectFromContext(ctx context.Context) context.Context {
}
// UnaryClientCorrelationInterceptor propagates Correlation-IDs downstream
func UnaryClientCorrelationInterceptor(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
ctx = injectFromContext(ctx)
return invoker(ctx, method, req, reply, cc, opts...)
func UnaryClientCorrelationInterceptor() grpc.UnaryClientInterceptor {
return func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
ctx = injectFromContext(ctx)
return invoker(ctx, method, req, reply, cc, opts...)
}
}
// StreamClientCorrelationInterceptor propagates Correlation-IDs downstream
func StreamClientCorrelationInterceptor(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) {
ctx = injectFromContext(ctx)
return streamer(ctx, desc, cc, method, opts...)
func StreamClientCorrelationInterceptor() grpc.StreamClientInterceptor {
return func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) {
ctx = injectFromContext(ctx)
return streamer(ctx, desc, cc, method, opts...)
}
}
package grpccorrelation
const metadataCorrelatorKey = "X-GitLab-Correlation-ID"
package grpccorrelation
import (
"context"
"github.com/grpc-ecosystem/go-grpc-middleware"
"gitlab.com/gitlab-org/labkit/correlation"
"google.golang.org/grpc"
"google.golang.org/grpc/metadata"
)
func extractFromContext(ctx context.Context) context.Context {
md, ok := metadata.FromIncomingContext(ctx)
if !ok {
return ctx
}
values := md.Get(metadataCorrelatorKey)
if len(values) < 1 {
return ctx
}
return correlation.ContextWithCorrelation(ctx, values[0])
}
// UnaryServerCorrelationInterceptor propagates Correlation-IDs from incoming upstream services
func UnaryServerCorrelationInterceptor() grpc.UnaryServerInterceptor {
return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp interface{}, err error) {
ctx = extractFromContext(ctx)
return handler(ctx, req)
}
}
// StreamServerCorrelationInterceptor propagates Correlation-IDs from incoming upstream services
func StreamServerCorrelationInterceptor() grpc.StreamServerInterceptor {
return func(srv interface{}, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
wrapped := grpc_middleware.WrapServerStream(ss)
wrapped.WrappedContext = extractFromContext(ss.Context())
return handler(srv, wrapped)
}
}
......@@ -5,7 +5,10 @@ import (
)
// InjectCorrelationID middleware will propagate or create a Correlation-ID for the incoming request
func InjectCorrelationID(h http.Handler) http.Handler {
func InjectCorrelationID(h http.Handler, opts ...InboundHandlerOption) http.Handler {
// Currently we don't use any of the options available
applyInboundHandlerOptions(opts)
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
parent := r.Context()
......
package correlation
// The configuration for InjectCorrelationID
type inboundHandlerConfig struct {
}
// InboundHandlerOption will configure a correlation handler
// currently there are no options, but this gives us the option
// to extend the interface in a backwards compatible way
type InboundHandlerOption func(*inboundHandlerConfig)
func applyInboundHandlerOptions(opts []InboundHandlerOption) inboundHandlerConfig {
config := inboundHandlerConfig{}
for _, v := range opts {
v(&config)
}
return config
}
......@@ -27,6 +27,9 @@ func (c instrumentedRoundTripper) RoundTrip(req *http.Request) (res *http.Respon
// NewInstrumentedRoundTripper acts as a "client-middleware" for outbound http requests
// adding instrumentation to the outbound request and then delegating to the underlying
// transport
func NewInstrumentedRoundTripper(delegate http.RoundTripper) http.RoundTripper {
func NewInstrumentedRoundTripper(delegate http.RoundTripper, opts ...InstrumentedRoundTripperOption) http.RoundTripper {
// Currently we don't use any of the options available
applyInstrumentedRoundTripperOptions(opts)
return &instrumentedRoundTripper{delegate: delegate}
}
package correlation
// The configuration for InjectCorrelationID
type instrumentedRoundTripperConfig struct {
}
// InstrumentedRoundTripperOption will configure a correlation handler
// currently there are no options, but this gives us the option
// to extend the interface in a backwards compatible way
type InstrumentedRoundTripperOption func(*instrumentedRoundTripperConfig)
func applyInstrumentedRoundTripperOptions(opts []InstrumentedRoundTripperOption) instrumentedRoundTripperConfig {
config := instrumentedRoundTripperConfig{}
for _, v := range opts {
v(&config)
}
return config
}
......@@ -5,7 +5,7 @@ import (
raven "github.com/getsentry/raven-go"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/correlation"
"gitlab.com/gitlab-org/labkit/correlation"
)
const ravenSentryExtraKey = "gitlab.CorrelationID"
......
......@@ -414,6 +414,30 @@
"version": "v0.71.0",
"versionExact": "v0.71.0"
},
{
"checksumSHA1": "S9x46Eq79I1EkoXI8woeIU6w/wY=",
"path": "gitlab.com/gitlab-org/labkit/correlation",
"revision": "fb2eeb11fc358f8f538692d1c0f6e57d7f268691",
"revisionTime": "2018-11-21T15:01:09Z",
"version": "master",
"versionExact": "master"
},
{
"checksumSHA1": "UFBFulprWZHuL9GHhjCKoHXm+Ww=",
"path": "gitlab.com/gitlab-org/labkit/correlation/grpc",
"revision": "fb2eeb11fc358f8f538692d1c0f6e57d7f268691",
"revisionTime": "2018-11-21T15:01:09Z",
"version": "master",
"versionExact": "master"
},
{
"checksumSHA1": "YyeJ2t/STDZj3X4xetPzTC0I9hU=",
"path": "gitlab.com/gitlab-org/labkit/correlation/raven",
"revision": "fb2eeb11fc358f8f538692d1c0f6e57d7f268691",
"revisionTime": "2018-11-21T15:01:09Z",
"version": "master",
"versionExact": "master"
},
{
"checksumSHA1": "BGm8lKZmvJbf/YOJLeL1rw2WVjA=",
"path": "golang.org/x/crypto/ssh/terminal",
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment