Commit 1775891c authored by Andrew Newdigate's avatar Andrew Newdigate Committed by Nick Thomas

Add distributed tracing with LabKit

parent 254f37c7
......@@ -48,15 +48,15 @@ $(TARGET_SETUP):
gitlab-zip-cat: $(TARGET_SETUP) $(shell find cmd/gitlab-zip-cat/ -name '*.go')
$(call message,Building $@)
$(GOBUILD) -o $(BUILD_DIR)/$@ $(PKG)/cmd/$@
$(GOBUILD) -tags "$(BUILD_TAGS)" -o $(BUILD_DIR)/$@ $(PKG)/cmd/$@
gitlab-zip-metadata: $(TARGET_SETUP) $(shell find cmd/gitlab-zip-metadata/ -name '*.go')
$(call message,Building $@)
$(GOBUILD) -o $(BUILD_DIR)/$@ $(PKG)/cmd/$@
$(GOBUILD) -tags "$(BUILD_TAGS)" -o $(BUILD_DIR)/$@ $(PKG)/cmd/$@
gitlab-workhorse: $(TARGET_SETUP) $(shell find . -name '*.go' | grep -v '^\./_')
$(call message,Building $@)
$(GOBUILD) -o $(BUILD_DIR)/$@ $(PKG)
$(GOBUILD) -tags "$(BUILD_TAGS)" -o $(BUILD_DIR)/$@ $(PKG)
.PHONY: install
install: gitlab-workhorse gitlab-zip-cat gitlab-zip-metadata
......@@ -67,13 +67,13 @@ install: gitlab-workhorse gitlab-zip-cat gitlab-zip-metadata
.PHONY: test
test: $(TARGET_SETUP) prepare-tests
$(call message,$@)
@go test $(LOCAL_PACKAGES)
@go test -tags "$(BUILD_TAGS)" $(LOCAL_PACKAGES)
@echo SUCCESS
.PHONY: coverage
coverage: $(TARGET_SETUP) prepare-tests
$(call message,$@)
@go test -cover -coverprofile=test.coverage $(LOCAL_PACKAGES)
@go test -tags "$(BUILD_TAGS)" -cover -coverprofile=test.coverage $(LOCAL_PACKAGES)
go tool cover -html=test.coverage -o coverage.html
rm -f test.coverage
......
......@@ -204,6 +204,26 @@ the first priority during development.
It is OK if a feature is only covered by integration tests.
## Distributed Tracing
Workhorse supports distributed tracing through [LabKit](https://gitlab.com/gitlab-org/labkit/) using [OpenTracing APIs](https://opentracing.io).
By default, no tracing implementation is linked into the binary, but different OpenTracing providers can be linked in using [build tags](https://golang.org/pkg/go/build/#hdr-Build_Constraints)/[build constraints](https://golang.org/pkg/go/build/#hdr-Build_Constraints). This can be done by setting the `BUILD_TAGS` make variable.
For more details of the supported providers, see LabKit, but as an example, for Jaeger tracing support, include the tags: `BUILD_TAGS="tracer_static tracer_static_jaeger"`.
```shell
make BUILD_TAGS="tracer_static tracer_static_jaeger"
```
Once Workhorse is compiled with an opentracing provider, the tracing configuration is configured via the `GITLAB_TRACING` environment variable.
For example:
```shell
GITLAB_TRACING=opentracing://jaeger ./gitlab-workhorse
```
## License
This code is distributed under the MIT license, see the LICENSE file.
......
......@@ -11,6 +11,7 @@ import (
"google.golang.org/grpc"
grpccorrelation "gitlab.com/gitlab-org/labkit/correlation/grpc"
grpctracing "gitlab.com/gitlab-org/labkit/tracing/grpc"
)
type Server struct {
......@@ -113,6 +114,7 @@ func newConnection(server Server) (*grpc.ClientConn, error) {
grpc.WithPerRPCCredentials(gitalyauth.RPCCredentialsV2(server.Token)),
grpc.WithStreamInterceptor(
grpc_middleware.ChainStreamClient(
grpctracing.StreamClientTracingInterceptor(),
grpc_prometheus.StreamClientInterceptor,
grpccorrelation.StreamClientCorrelationInterceptor(),
),
......@@ -120,6 +122,7 @@ func newConnection(server Server) (*grpc.ClientConn, error) {
grpc.WithUnaryInterceptor(
grpc_middleware.ChainUnaryClient(
grpctracing.UnaryClientTracingInterceptor(),
grpc_prometheus.UnaryClientInterceptor,
grpccorrelation.UnaryClientCorrelationInterceptor(),
),
......
......@@ -10,6 +10,7 @@ import (
"time"
"gitlab.com/gitlab-org/labkit/correlation"
"gitlab.com/gitlab-org/labkit/tracing"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
)
......@@ -18,7 +19,7 @@ import (
// that are more restrictive than for http.DefaultTransport,
// they define shorter TLS Handshake, and more agressive connection closing
// to prevent the connection hanging and reduce FD usage
var httpTransport = correlation.NewInstrumentedRoundTripper(&http.Transport{
var httpTransport = tracing.NewRoundTripper(correlation.NewInstrumentedRoundTripper(&http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{
Timeout: 30 * time.Second,
......@@ -29,7 +30,7 @@ var httpTransport = correlation.NewInstrumentedRoundTripper(&http.Transport{
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 10 * time.Second,
ResponseHeaderTimeout: 30 * time.Second,
})
}))
var httpClient = &http.Client{
Transport: httpTransport,
......
......@@ -10,6 +10,7 @@ import (
"github.com/prometheus/client_golang/prometheus"
"gitlab.com/gitlab-org/labkit/correlation"
"gitlab.com/gitlab-org/labkit/tracing"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/log"
......@@ -38,7 +39,7 @@ var rangeHeaderKeys = []string{
// that are more restrictive than for http.DefaultTransport,
// they define shorter TLS Handshake, and more agressive connection closing
// to prevent the connection hanging and reduce FD usage
var httpTransport = correlation.NewInstrumentedRoundTripper(&http.Transport{
var httpTransport = tracing.NewRoundTripper(correlation.NewInstrumentedRoundTripper(&http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{
Timeout: 30 * time.Second,
......@@ -49,7 +50,7 @@ var httpTransport = correlation.NewInstrumentedRoundTripper(&http.Transport{
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 10 * time.Second,
ResponseHeaderTimeout: 30 * time.Second,
})
}))
var httpClient = &http.Client{
Transport: httpTransport,
......
......@@ -9,6 +9,7 @@ import (
"time"
"gitlab.com/gitlab-org/labkit/correlation"
"gitlab.com/gitlab-org/labkit/tracing"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/badgateway"
)
......@@ -47,7 +48,11 @@ func NewBackendRoundTripper(backend *url.URL, socket string, proxyHeadersTimeout
panic("backend is nil and socket is empty")
}
return correlation.NewInstrumentedRoundTripper(badgateway.NewRoundTripper(developmentMode, transport))
return tracing.NewRoundTripper(
correlation.NewInstrumentedRoundTripper(
badgateway.NewRoundTripper(developmentMode, transport),
),
)
}
// NewTestBackendRoundTripper sets up a RoundTripper for testing purposes
......
......@@ -7,6 +7,8 @@ import (
"github.com/gorilla/websocket"
"gitlab.com/gitlab-org/labkit/tracing"
apipkg "gitlab.com/gitlab-org/gitlab-workhorse/internal/api"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/artifacts"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/builds"
......@@ -78,6 +80,10 @@ func route(method, regexpStr string, handler http.Handler, opts ...func(*routeOp
handler = denyWebsocket(handler) // Disallow websockets
handler = instrumentRoute(handler, method, regexpStr) // Add prometheus metrics
if options.tracing {
// Add distributed tracing
handler = tracing.Handler(handler)
}
return routeEntry{
method: method,
......
......@@ -14,6 +14,7 @@ import (
"github.com/jfbus/httprs"
"gitlab.com/gitlab-org/labkit/correlation"
"gitlab.com/gitlab-org/labkit/tracing"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/helper"
)
......@@ -25,7 +26,7 @@ var ErrNotAZip = errors.New("not a zip")
var ErrArchiveNotFound = errors.New("archive not found")
var httpClient = &http.Client{
Transport: correlation.NewInstrumentedRoundTripper(&http.Transport{
Transport: tracing.NewRoundTripper(correlation.NewInstrumentedRoundTripper(&http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{
Timeout: 30 * time.Second,
......@@ -35,7 +36,7 @@ var httpClient = &http.Client{
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 10 * time.Second,
ResponseHeaderTimeout: 30 * time.Second,
}),
})),
}
// OpenArchive will open a zip.Reader from a local path or a remote object store URL
......
......@@ -26,6 +26,7 @@ import (
"github.com/prometheus/client_golang/prometheus/promhttp"
"gitlab.com/gitlab-org/labkit/correlation"
"gitlab.com/gitlab-org/labkit/tracing"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/config"
"gitlab.com/gitlab-org/gitlab-workhorse/internal/log"
......@@ -81,6 +82,8 @@ func main() {
startLogging(logConfig)
logger := log.NoContext()
tracing.Initialize(tracing.WithServiceName("gitlab-workhorse"))
backendURL, err := parseAuthBackend(*authBackend)
if err != nil {
logger.WithError(err).Fatal("invalid authBackend")
......
This source diff could not be displayed because it is too large. You can view the blob instead.
# grpc_opentracing
`import "github.com/grpc-ecosystem/go-grpc-middleware/tracing/opentracing"`
* [Overview](#pkg-overview)
* [Imported Packages](#pkg-imports)
* [Index](#pkg-index)
## <a name="pkg-overview">Overview</a>
`grpc_opentracing` adds OpenTracing
### OpenTracing Interceptors
These are both client-side and server-side interceptors for OpenTracing. They are a provider-agnostic, with backends
such as Zipkin, or Google Stackdriver Trace.
For a service that sends out requests and receives requests, you *need* to use both, otherwise downstream requests will
not have the appropriate requests propagated.
All server-side spans are tagged with grpc_ctxtags information.
For more information see:
<a href="http://opentracing.io/documentation/">http://opentracing.io/documentation/</a>
<a href="https://github.com/opentracing/specification/blob/master/semantic_conventions.md">https://github.com/opentracing/specification/blob/master/semantic_conventions.md</a>
## <a name="pkg-imports">Imported Packages</a>
- [github.com/grpc-ecosystem/go-grpc-middleware](./../..)
- [github.com/grpc-ecosystem/go-grpc-middleware/tags](./../../tags)
- [github.com/grpc-ecosystem/go-grpc-middleware/util/metautils](./../../util/metautils)
- [github.com/opentracing/opentracing-go](https://godoc.org/github.com/opentracing/opentracing-go)
- [github.com/opentracing/opentracing-go/ext](https://godoc.org/github.com/opentracing/opentracing-go/ext)
- [github.com/opentracing/opentracing-go/log](https://godoc.org/github.com/opentracing/opentracing-go/log)
- [golang.org/x/net/context](https://godoc.org/golang.org/x/net/context)
- [google.golang.org/grpc](https://godoc.org/google.golang.org/grpc)
- [google.golang.org/grpc/grpclog](https://godoc.org/google.golang.org/grpc/grpclog)
- [google.golang.org/grpc/metadata](https://godoc.org/google.golang.org/grpc/metadata)
## <a name="pkg-index">Index</a>
* [Constants](#pkg-constants)
* [func ClientAddContextTags(ctx context.Context, tags opentracing.Tags) context.Context](#ClientAddContextTags)
* [func StreamClientInterceptor(opts ...Option) grpc.StreamClientInterceptor](#StreamClientInterceptor)
* [func StreamServerInterceptor(opts ...Option) grpc.StreamServerInterceptor](#StreamServerInterceptor)
* [func UnaryClientInterceptor(opts ...Option) grpc.UnaryClientInterceptor](#UnaryClientInterceptor)
* [func UnaryServerInterceptor(opts ...Option) grpc.UnaryServerInterceptor](#UnaryServerInterceptor)
* [type FilterFunc](#FilterFunc)
* [type Option](#Option)
* [func WithFilterFunc(f FilterFunc) Option](#WithFilterFunc)
* [func WithTracer(tracer opentracing.Tracer) Option](#WithTracer)
#### <a name="pkg-files">Package files</a>
[client_interceptors.go](./client_interceptors.go) [doc.go](./doc.go) [id_extract.go](./id_extract.go) [metadata.go](./metadata.go) [options.go](./options.go) [server_interceptors.go](./server_interceptors.go)
## <a name="pkg-constants">Constants</a>
``` go
const (
TagTraceId = "trace.traceid"
TagSpanId = "trace.spanid"
)
```
## <a name="ClientAddContextTags">func</a> [ClientAddContextTags](./client_interceptors.go#L105)
``` go
func ClientAddContextTags(ctx context.Context, tags opentracing.Tags) context.Context
```
ClientAddContextTags returns a context with specified opentracing tags, which
are used by UnaryClientInterceptor/StreamClientInterceptor when creating a
new span.
## <a name="StreamClientInterceptor">func</a> [StreamClientInterceptor](./client_interceptors.go#L35)
``` go
func StreamClientInterceptor(opts ...Option) grpc.StreamClientInterceptor
```
StreamClientInterceptor returns a new streaming client interceptor for OpenTracing.
## <a name="StreamServerInterceptor">func</a> [StreamServerInterceptor](./server_interceptors.go#L37)
``` go
func StreamServerInterceptor(opts ...Option) grpc.StreamServerInterceptor
```
StreamServerInterceptor returns a new streaming server interceptor for OpenTracing.
## <a name="UnaryClientInterceptor">func</a> [UnaryClientInterceptor](./client_interceptors.go#L21)
``` go
func UnaryClientInterceptor(opts ...Option) grpc.UnaryClientInterceptor
```
UnaryClientInterceptor returns a new unary client interceptor for OpenTracing.
## <a name="UnaryServerInterceptor">func</a> [UnaryServerInterceptor](./server_interceptors.go#L23)
``` go
func UnaryServerInterceptor(opts ...Option) grpc.UnaryServerInterceptor
```
UnaryServerInterceptor returns a new unary server interceptor for OpenTracing.
## <a name="FilterFunc">type</a> [FilterFunc](./options.go#L22)
``` go
type FilterFunc func(ctx context.Context, fullMethodName string) bool
```
FilterFunc allows users to provide a function that filters out certain methods from being traced.
If it returns false, the given request will not be traced.
## <a name="Option">type</a> [Option](./options.go#L41)
``` go
type Option func(*options)
```
### <a name="WithFilterFunc">func</a> [WithFilterFunc](./options.go#L44)
``` go
func WithFilterFunc(f FilterFunc) Option
```
WithFilterFunc customizes the function used for deciding whether a given call is traced or not.
### <a name="WithTracer">func</a> [WithTracer](./options.go#L51)
``` go
func WithTracer(tracer opentracing.Tracer) Option
```
WithTracer sets a custom tracer to be used for this middleware, otherwise the opentracing.GlobalTracer is used.
- - -
Generated by [godoc2ghmd](https://github.com/GandalfUK/godoc2ghmd)
\ No newline at end of file
# grpc_opentracing
`import "github.com/grpc-ecosystem/go-grpc-middleware/tracing/opentracing"`
* [Overview](#pkg-overview)
* [Imported Packages](#pkg-imports)
* [Index](#pkg-index)
## <a name="pkg-overview">Overview</a>
`grpc_opentracing` adds OpenTracing
### OpenTracing Interceptors
These are both client-side and server-side interceptors for OpenTracing. They are a provider-agnostic, with backends
such as Zipkin, or Google Stackdriver Trace.
For a service that sends out requests and receives requests, you *need* to use both, otherwise downstream requests will
not have the appropriate requests propagated.
All server-side spans are tagged with grpc_ctxtags information.
For more information see:
<a href="http://opentracing.io/documentation/">http://opentracing.io/documentation/</a>
<a href="https://github.com/opentracing/specification/blob/master/semantic_conventions.md">https://github.com/opentracing/specification/blob/master/semantic_conventions.md</a>
## <a name="pkg-imports">Imported Packages</a>
- [github.com/grpc-ecosystem/go-grpc-middleware](./../..)
- [github.com/grpc-ecosystem/go-grpc-middleware/tags](./../../tags)
- [github.com/grpc-ecosystem/go-grpc-middleware/util/metautils](./../../util/metautils)
- [github.com/opentracing/opentracing-go](https://godoc.org/github.com/opentracing/opentracing-go)
- [github.com/opentracing/opentracing-go/ext](https://godoc.org/github.com/opentracing/opentracing-go/ext)
- [github.com/opentracing/opentracing-go/log](https://godoc.org/github.com/opentracing/opentracing-go/log)
- [golang.org/x/net/context](https://godoc.org/golang.org/x/net/context)
- [google.golang.org/grpc](https://godoc.org/google.golang.org/grpc)
- [google.golang.org/grpc/grpclog](https://godoc.org/google.golang.org/grpc/grpclog)
- [google.golang.org/grpc/metadata](https://godoc.org/google.golang.org/grpc/metadata)
## <a name="pkg-index">Index</a>
* [Constants](#pkg-constants)
* [func ClientAddContextTags(ctx context.Context, tags opentracing.Tags) context.Context](#ClientAddContextTags)
* [func StreamClientInterceptor(opts ...Option) grpc.StreamClientInterceptor](#StreamClientInterceptor)
* [func StreamServerInterceptor(opts ...Option) grpc.StreamServerInterceptor](#StreamServerInterceptor)
* [func UnaryClientInterceptor(opts ...Option) grpc.UnaryClientInterceptor](#UnaryClientInterceptor)
* [func UnaryServerInterceptor(opts ...Option) grpc.UnaryServerInterceptor](#UnaryServerInterceptor)
* [type FilterFunc](#FilterFunc)
* [type Option](#Option)
* [func WithFilterFunc(f FilterFunc) Option](#WithFilterFunc)
* [func WithTracer(tracer opentracing.Tracer) Option](#WithTracer)
#### <a name="pkg-files">Package files</a>
[client_interceptors.go](./client_interceptors.go) [doc.go](./doc.go) [id_extract.go](./id_extract.go) [metadata.go](./metadata.go) [options.go](./options.go) [server_interceptors.go](./server_interceptors.go)
## <a name="pkg-constants">Constants</a>
``` go
const (
TagTraceId = "trace.traceid"
TagSpanId = "trace.spanid"
)
```
## <a name="ClientAddContextTags">func</a> [ClientAddContextTags](./client_interceptors.go#L105)
``` go
func ClientAddContextTags(ctx context.Context, tags opentracing.Tags) context.Context
```
ClientAddContextTags returns a context with specified opentracing tags, which
are used by UnaryClientInterceptor/StreamClientInterceptor when creating a
new span.
## <a name="StreamClientInterceptor">func</a> [StreamClientInterceptor](./client_interceptors.go#L35)
``` go
func StreamClientInterceptor(opts ...Option) grpc.StreamClientInterceptor
```
StreamClientInterceptor returns a new streaming client interceptor for OpenTracing.
## <a name="StreamServerInterceptor">func</a> [StreamServerInterceptor](./server_interceptors.go#L37)
``` go
func StreamServerInterceptor(opts ...Option) grpc.StreamServerInterceptor
```
StreamServerInterceptor returns a new streaming server interceptor for OpenTracing.
## <a name="UnaryClientInterceptor">func</a> [UnaryClientInterceptor](./client_interceptors.go#L21)
``` go
func UnaryClientInterceptor(opts ...Option) grpc.UnaryClientInterceptor
```
UnaryClientInterceptor returns a new unary client interceptor for OpenTracing.
## <a name="UnaryServerInterceptor">func</a> [UnaryServerInterceptor](./server_interceptors.go#L23)
``` go
func UnaryServerInterceptor(opts ...Option) grpc.UnaryServerInterceptor
```
UnaryServerInterceptor returns a new unary server interceptor for OpenTracing.
## <a name="FilterFunc">type</a> [FilterFunc](./options.go#L22)
``` go
type FilterFunc func(ctx context.Context, fullMethodName string) bool
```
FilterFunc allows users to provide a function that filters out certain methods from being traced.
If it returns false, the given request will not be traced.
## <a name="Option">type</a> [Option](./options.go#L41)
``` go
type Option func(*options)
```
### <a name="WithFilterFunc">func</a> [WithFilterFunc](./options.go#L44)
``` go
func WithFilterFunc(f FilterFunc) Option
```
WithFilterFunc customizes the function used for deciding whether a given call is traced or not.
### <a name="WithTracer">func</a> [WithTracer](./options.go#L51)
``` go
func WithTracer(tracer opentracing.Tracer) Option
```
WithTracer sets a custom tracer to be used for this middleware, otherwise the opentracing.GlobalTracer is used.
- - -
Generated by [godoc2ghmd](https://github.com/GandalfUK/godoc2ghmd)
\ No newline at end of file
// Copyright 2017 Michal Witkowski. All Rights Reserved.
// See LICENSE for licensing terms.
package grpc_opentracing
import (
"io"
"sync"
"github.com/grpc-ecosystem/go-grpc-middleware/util/metautils"
opentracing "github.com/opentracing/opentracing-go"
"github.com/opentracing/opentracing-go/ext"
"github.com/opentracing/opentracing-go/log"
"golang.org/x/net/context"
"google.golang.org/grpc"
"google.golang.org/grpc/grpclog"
"google.golang.org/grpc/metadata"
)
// UnaryClientInterceptor returns a new unary client interceptor for OpenTracing.
func UnaryClientInterceptor(opts ...Option) grpc.UnaryClientInterceptor {
o := evaluateOptions(opts)
return func(parentCtx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
if o.filterOutFunc != nil && !o.filterOutFunc(parentCtx, method) {
return invoker(parentCtx, method, req, reply, cc, opts...)
}
newCtx, clientSpan := newClientSpanFromContext(parentCtx, o.tracer, method)
err := invoker(newCtx, method, req, reply, cc, opts...)
finishClientSpan(clientSpan, err)
return err
}
}
// StreamClientInterceptor returns a new streaming client interceptor for OpenTracing.
func StreamClientInterceptor(opts ...Option) grpc.StreamClientInterceptor {
o := evaluateOptions(opts)
return func(parentCtx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) {
if o.filterOutFunc != nil && !o.filterOutFunc(parentCtx, method) {
return streamer(parentCtx, desc, cc, method, opts...)
}
newCtx, clientSpan := newClientSpanFromContext(parentCtx, o.tracer, method)
clientStream, err := streamer(newCtx, desc, cc, method, opts...)
if err != nil {
finishClientSpan(clientSpan, err)
return nil, err
}
return &tracedClientStream{ClientStream: clientStream, clientSpan: clientSpan}, nil
}
}
// type serverStreamingRetryingStream is the implementation of grpc.ClientStream that acts as a
// proxy to the underlying call. If any of the RecvMsg() calls fail, it will try to reestablish
// a new ClientStream according to the retry policy.
type tracedClientStream struct {
grpc.ClientStream
mu sync.Mutex
alreadyFinished bool
clientSpan opentracing.Span
}
func (s *tracedClientStream) Header() (metadata.MD, error) {
h, err := s.ClientStream.Header()
if err != nil {
s.finishClientSpan(err)
}
return h, err
}
func (s *tracedClientStream) SendMsg(m interface{}) error {
err := s.ClientStream.SendMsg(m)
if err != nil {
s.finishClientSpan(err)
}
return err
}
func (s *tracedClientStream) CloseSend() error {
err := s.ClientStream.CloseSend()
if err != nil {
s.finishClientSpan(err)
}
return err
}
func (s *tracedClientStream) RecvMsg(m interface{}) error {
err := s.ClientStream.RecvMsg(m)
if err != nil {
s.finishClientSpan(err)
}
return err
}
func (s *tracedClientStream) finishClientSpan(err error) {
s.mu.Lock()
defer s.mu.Unlock()
if !s.alreadyFinished {
finishClientSpan(s.clientSpan, err)
s.alreadyFinished = true
}
}
// ClientAddContextTags returns a context with specified opentracing tags, which
// are used by UnaryClientInterceptor/StreamClientInterceptor when creating a
// new span.
func ClientAddContextTags(ctx context.Context, tags opentracing.Tags) context.Context {
return context.WithValue(ctx, clientSpanTagKey{}, tags)
}
type clientSpanTagKey struct{}
func newClientSpanFromContext(ctx context.Context, tracer opentracing.Tracer, fullMethodName string) (context.Context, opentracing.Span) {
var parentSpanCtx opentracing.SpanContext
if parent := opentracing.SpanFromContext(ctx); parent != nil {
parentSpanCtx = parent.Context()
}
opts := []opentracing.StartSpanOption{
opentracing.ChildOf(parentSpanCtx),
ext.SpanKindRPCClient,
grpcTag,
}
if tagx := ctx.Value(clientSpanTagKey{}); tagx != nil {
if opt, ok := tagx.(opentracing.StartSpanOption); ok {
opts = append(opts, opt)
}
}
clientSpan := tracer.StartSpan(fullMethodName, opts...)
// Make sure we add this to the metadata of the call, so it gets propagated:
md := metautils.ExtractOutgoing(ctx).Clone()
if err := tracer.Inject(clientSpan.Context(), opentracing.HTTPHeaders, metadataTextMap(md)); err != nil {
grpclog.Printf("grpc_opentracing: failed serializing trace information: %v", err)
}
ctxWithMetadata := md.ToOutgoing(ctx)
return opentracing.ContextWithSpan(ctxWithMetadata, clientSpan), clientSpan
}
func finishClientSpan(clientSpan opentracing.Span, err error) {
if err != nil && err != io.EOF {
ext.Error.Set(clientSpan, true)
clientSpan.LogFields(log.String("event", "error"), log.String("message", err.Error()))
}
clientSpan.Finish()
}
// Copyright 2017 Michal Witkowski. All Rights Reserved.
// See LICENSE for licensing terms.
/*
`grpc_opentracing` adds OpenTracing
OpenTracing Interceptors
These are both client-side and server-side interceptors for OpenTracing. They are a provider-agnostic, with backends
such as Zipkin, or Google Stackdriver Trace.
For a service that sends out requests and receives requests, you *need* to use both, otherwise downstream requests will
not have the appropriate requests propagated.
All server-side spans are tagged with grpc_ctxtags information.
For more information see:
http://opentracing.io/documentation/
https://github.com/opentracing/specification/blob/master/semantic_conventions.md
*/
package grpc_opentracing
// Copyright 2017 Michal Witkowski. All Rights Reserved.
// See LICENSE for licensing terms.
package grpc_opentracing
import (
"strings"
"github.com/grpc-ecosystem/go-grpc-middleware/tags"
"github.com/opentracing/opentracing-go"
"google.golang.org/grpc/grpclog"
)
const (
TagTraceId = "trace.traceid"
TagSpanId = "trace.spanid"
)
// hackyInjectOpentracingIdsToTags writes the given context to the ctxtags.
// This is done in an incredibly hacky way, because the public-facing interface of opentracing doesn't give access to
// the TraceId and SpanId of the SpanContext. Only the Tracer's Inject/Extract methods know what these are.
// Most tracers have them encoded as keys with 'traceid' and 'spanid':
// https://github.com/openzipkin/zipkin-go-opentracing/blob/594640b9ef7e5c994e8d9499359d693c032d738c/propagation_ot.go#L29
// https://github.com/opentracing/basictracer-go/blob/1b32af207119a14b1b231d451df3ed04a72efebf/propagation_ot.go#L26
func hackyInjectOpentracingIdsToTags(span opentracing.Span, tags grpc_ctxtags.Tags) {
if err := span.Tracer().Inject(span.Context(), opentracing.HTTPHeaders, &hackyTagsCarrier{tags}); err != nil {
grpclog.Printf("grpc_opentracing: failed extracting trace info into ctx %v", err)
}
}
// hackyTagsCarrier is a really hacky way of
type hackyTagsCarrier struct {
grpc_ctxtags.Tags
}
func (t *hackyTagsCarrier) Set(key, val string) {
if strings.Contains(key, "traceid") || strings.Contains(strings.ToLower(key), "traceid") {
t.Tags.Set(TagTraceId, val) // this will most likely be base-16 (hex) encoded
} else if (strings.Contains(key, "spanid") && !strings.Contains(key, "parent")) || (strings.Contains(strings.ToLower(key), "spanid") && !strings.Contains(strings.ToLower(key), "parent")) {
t.Tags.Set(TagSpanId, val) // this will most likely be base-16 (hex) encoded
}
}
// Copyright 2017 Michal Witkowski. All Rights Reserved.
// See LICENSE for licensing terms.
package grpc_opentracing
import (
"encoding/base64"
"strings"
"fmt"
"google.golang.org/grpc/metadata"
)
const (
binHdrSuffix = "-bin"
)
// metadataTextMap extends a metadata.MD to be an opentracing textmap
type metadataTextMap metadata.MD
// Set is a opentracing.TextMapReader interface that extracts values.
func (m metadataTextMap) Set(key, val string) {
// gRPC allows for complex binary values to be written.
encodedKey, encodedVal := encodeKeyValue(key, val)
// The metadata object is a multimap, and previous values may exist, but for opentracing headers, we do not append
// we just override.
m[encodedKey] = []string{encodedVal}
}
// ForeachKey is a opentracing.TextMapReader interface that extracts values.
func (m metadataTextMap) ForeachKey(callback func(key, val string) error) error {
for k, vv := range m {
for _, v := range vv {
if decodedKey, decodedVal, err := metadata.DecodeKeyValue(k, v); err == nil {
if err = callback(decodedKey, decodedVal); err != nil {
return err
}
} else {
return fmt.Errorf("failed decoding opentracing from gRPC metadata: %v", err)
}
}
}
return nil
}
// encodeKeyValue encodes key and value qualified for transmission via gRPC.
// note: copy pasted from private values of grpc.metadata
func encodeKeyValue(k, v string) (string, string) {
k = strings.ToLower(k)
if strings.HasSuffix(k, binHdrSuffix) {
val := base64.StdEncoding.EncodeToString([]byte(v))
v = string(val)
}
return k, v
}
// Copyright 2017 Michal Witkowski. All Rights Reserved.
// See LICENSE for licensing terms.
package grpc_opentracing
import (
"context"
"github.com/opentracing/opentracing-go"
)
var (
defaultOptions = &options{
filterOutFunc: nil,
tracer: nil,
}
)
// FilterFunc allows users to provide a function that filters out certain methods from being traced.
//
// If it returns false, the given request will not be traced.
type FilterFunc func(ctx context.Context, fullMethodName string) bool
type options struct {
filterOutFunc FilterFunc
tracer opentracing.Tracer
}
func evaluateOptions(opts []Option) *options {
optCopy := &options{}
*optCopy = *defaultOptions
for _, o := range opts {
o(optCopy)
}
if optCopy.tracer == nil {
optCopy.tracer = opentracing.GlobalTracer()
}
return optCopy
}
type Option func(*options)
// WithFilterFunc customizes the function used for deciding whether a given call is traced or not.
func WithFilterFunc(f FilterFunc) Option {
return func(o *options) {
o.filterOutFunc = f
}
}
// WithTracer sets a custom tracer to be used for this middleware, otherwise the opentracing.GlobalTracer is used.
func WithTracer(tracer opentracing.Tracer) Option {
return func(o *options) {
o.tracer = tracer
}
}
// Copyright 2017 Michal Witkowski. All Rights Reserved.
// See LICENSE for licensing terms.
package grpc_opentracing
import (
"github.com/grpc-ecosystem/go-grpc-middleware"
"github.com/grpc-ecosystem/go-grpc-middleware/tags"
"github.com/grpc-ecosystem/go-grpc-middleware/util/metautils"
"github.com/opentracing/opentracing-go"
"github.com/opentracing/opentracing-go/ext"
"github.com/opentracing/opentracing-go/log"
"golang.org/x/net/context"
"google.golang.org/grpc"
"google.golang.org/grpc/grpclog"
)
var (
grpcTag = opentracing.Tag{Key: string(ext.Component), Value: "gRPC"}
)
// UnaryServerInterceptor returns a new unary server interceptor for OpenTracing.
func UnaryServerInterceptor(opts ...Option) grpc.UnaryServerInterceptor {
o := evaluateOptions(opts)
return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
if o.filterOutFunc != nil && !o.filterOutFunc(ctx, info.FullMethod) {
return handler(ctx, req)
}
newCtx, serverSpan := newServerSpanFromInbound(ctx, o.tracer, info.FullMethod)
resp, err := handler(newCtx, req)
finishServerSpan(ctx, serverSpan, err)
return resp, err
}
}
// StreamServerInterceptor returns a new streaming server interceptor for OpenTracing.
func StreamServerInterceptor(opts ...Option) grpc.StreamServerInterceptor {
o := evaluateOptions(opts)
return func(srv interface{}, stream grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
if o.filterOutFunc != nil && !o.filterOutFunc(stream.Context(), info.FullMethod) {
return handler(srv, stream)
}
newCtx, serverSpan := newServerSpanFromInbound(stream.Context(), o.tracer, info.FullMethod)
wrappedStream := grpc_middleware.WrapServerStream(stream)
wrappedStream.WrappedContext = newCtx
err := handler(srv, wrappedStream)
finishServerSpan(newCtx, serverSpan, err)
return err
}
}
func newServerSpanFromInbound(ctx context.Context, tracer opentracing.Tracer, fullMethodName string) (context.Context, opentracing.Span) {
md := metautils.ExtractIncoming(ctx)
parentSpanContext, err := tracer.Extract(opentracing.HTTPHeaders, metadataTextMap(md))
if err != nil && err != opentracing.ErrSpanContextNotFound {
grpclog.Printf("grpc_opentracing: failed parsing trace information: %v", err)
}
serverSpan := tracer.StartSpan(
fullMethodName,
// this is magical, it attaches the new span to the parent parentSpanContext, and creates an unparented one if empty.
ext.RPCServerOption(parentSpanContext),
grpcTag,
)
hackyInjectOpentracingIdsToTags(serverSpan, grpc_ctxtags.Extract(ctx))
return opentracing.ContextWithSpan(ctx, serverSpan), serverSpan
}
func finishServerSpan(ctx context.Context, serverSpan opentracing.Span, err error) {
// Log context information
tags := grpc_ctxtags.Extract(ctx)
for k, v := range tags.Values() {
// Don't tag errors, log them instead.
if vErr, ok := v.(error); ok {
serverSpan.LogKV(k, vErr.Error())
} else {
serverSpan.SetTag(k, v)
}
}
if err != nil {
ext.Error.Set(serverSpan, true)
serverSpan.LogFields(log.String("event", "error"), log.String("message", err.Error()))
}
serverSpan.Finish()
}
# Changelog
## [Pending Release](https://github.com/lightstep/lightstep-tracer-go/compare/v0.15.6...HEAD)
* Requires go >= 1.7
* Imports `context` via the standard library instead of `golang.org/x/net/context`
* Fixes [#182](https://github.com/lightstep/lightstep-tracer-go/issues/182), so that `StartSpan` can now take `SpanReference`s to non-LightStep `SpanContext`s use
## [v0.15.6](https://github.com/lightstep/lightstep-tracer-go/compare/v0.15.5...v0.15.6)
* Minor update to `sendspan` to make it easier to pick which transport protocol to use.
* Add a new field to Options: DialOptions. These allow setting custom grpc dial options when using grpc.
* This is necessary to have customer balancers or interceptors.
* DialOptions shouldn't be set unless it is needed, it will default correctly.
* Added a new field to Endpoint: Scheme. Scheme can be used to override the default schemes (http/https) or set a custom scheme (for grpc).
* This is necessary for using custom grpc resolvers.
* If callers are using struct construction without field names (i.e. Endpoint{"host", port, ...}), they will need to add a new field (scheme = "").
* Scheme shouldn't be set unless it needs to overridden, it will default correctly.
## [v0.15.5](https://github.com/lightstep/lightstep-tracer-go/compare/v0.15.4...v0.15.5)
* Internal performance optimizations and a bug fix for issue [#161](https://github.com/lightstep/lightstep-tracer-go/issues/161)
## [v0.15.4](https://github.com/lightstep/lightstep-tracer-go/compare/v0.15.3...v0.15.4)
* This change affects LightStep's internal testing, not a functional change.
## [v0.15.3](https://github.com/lightstep/lightstep-tracer-go/compare/v0.15.2...v0.15.3)
* Adds compatibility for io.Writer and io.Reader in Inject/Extract, as required by Open Tracing.
## [v0.15.2](https://github.com/lightstep/lightstep-tracer-go/compare/v0.15.1...v0.15.2)
* Adds lightstep.GetLightStepReporterID.
## [v0.15.1](https://github.com/lightstep/lightstep-tracer-go/compare/v0.15.0...v0.15.1)
* Adds Gopkg.toml
## [v0.15.0](https://github.com/lightstep/lightstep-tracer-go/compare/v0.14.0...v0.15.0)
* We are replacing the internal diagnostic logging with a more flexible “Event” framework. This enables you to track and understand tracer problems with metrics and logging tools of your choice.
* We are also changed the types of the Close() and Flush() methods to take a context parameter to support cancellation. These changes are *not* backwards compatible and *you will need to update your instrumentation*. We are providing a NewTracerv0_14() method that is a drop-in replacement for the previous version.
## [v0.14.0](https://github.com/lightstep/lightstep-tracer-go/compare/v0.13.0...v0.14.0)
* Flush buffer syncronously on Close.
* Flush twice if a flush is already in flight.
* Remove gogo in favor of golang/protobuf.
* Requires grpc-go >= 1.4.0.
## [v0.13.0](https://github.com/lightstep/lightstep-tracer-go/compare/v0.12.0...v0.13.0)
* BasicTracer has been removed.
* Tracer now takes a SpanRecorder as an option.
* Tracer interface now includes Close and Flush.
* Tests redone with ginkgo/gomega.
## [v0.12.0](https://github.com/lightstep/lightstep-tracer-go/compare/v0.11.0...v0.12.0)
* Added CloseTracer function to flush and close a lightstep recorder.
## [v0.11.0](https://github.com/lightstep/lightstep-tracer-go/compare/v0.10.0...v0.11.0)
* Thrift transport is now deprecated, gRPC is the default.
# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
[[projects]]
branch = "master"
digest = "1:3b760d3b93f994df8eb1d9ebfad17d3e9e37edcb7f7efaa15b427c0d7a64f4e4"
name = "github.com/golang/protobuf"
packages = [
"proto",
"protoc-gen-go/descriptor",
"ptypes",
"ptypes/any",
"ptypes/duration",
"ptypes/timestamp",
]
pruneopts = ""
revision = "1e59b77b52bf8e4b449a57e6f79f21226d571845"
[[projects]]
branch = "master"
digest = "1:b486888585e9070ef97f0c9b2df560e007dc2eafb8ace96b13a2cb85ccec3df1"
name = "github.com/lightstep/thrift"
packages = ["lib/go/thrift"]
pruneopts = ""
revision = "de6c3c01b88c1fc8ae76cec4a30f0441ef55a4a4"
[[projects]]
digest = "1:32b27072cd55bd2fb7244de0425943d125da6a552ae2b6517cdd965a662baf18"
name = "github.com/onsi/ginkgo"
packages = [
".",
"config",
"internal/codelocation",
"internal/containernode",
"internal/failer",
"internal/leafnodes",
"internal/remote",
"internal/spec",
"internal/spec_iterator",
"internal/specrunner",
"internal/suite",
"internal/testingtproxy",
"internal/writer",
"reporters",
"reporters/stenographer",
"reporters/stenographer/support/go-colorable",
"reporters/stenographer/support/go-isatty",
"types",
]
pruneopts = ""
revision = "9eda700730cba42af70d53180f9dcce9266bc2bc"
version = "v1.4.0"
[[projects]]
digest = "1:a4e59d0b2821c983b58c317f141cd77df20570979632da8a7a352e5d12698de7"
name = "github.com/onsi/gomega"
packages = [
".",
"format",
"internal/assertion",
"internal/asyncassertion",
"internal/oraclematcher",
"internal/testingtsupport",
"matchers",
"matchers/support/goraph/bipartitegraph",
"matchers/support/goraph/edge",
"matchers/support/goraph/node",
"matchers/support/goraph/util",
"types",
]
pruneopts = ""
revision = "c893efa28eb45626cdaa76c9f653b62488858837"
version = "v1.2.0"
[[projects]]
digest = "1:78fb99d6011c2ae6c72f3293a83951311147b12b06a5ffa43abf750c4fab6ac5"
name = "github.com/opentracing/opentracing-go"
packages = [
".",
"log",
]
pruneopts = ""
revision = "1949ddbfd147afd4d964a9f00b24eb291e0e7c38"
version = "v1.0.2"
[[projects]]
branch = "master"
digest = "1:950b672f2ee80d0fc4c95a15a976ba9ee573a6fb8ede8a777770b2230776367e"
name = "golang.org/x/net"
packages = [
"context",
"html",
"html/atom",
"html/charset",
"http2",
"http2/hpack",
"idna",
"internal/timeseries",
"lex/httplex",
"trace",
]
pruneopts = ""
revision = "dc871a5d77e227f5bbf6545176ef3eeebf87e76e"
[[projects]]
branch = "master"
digest = "1:5da11ab130476e2736f32140f3c1aed2a3a96e9ba8963711a7d38db783d042bd"
name = "golang.org/x/sys"
packages = ["unix"]
pruneopts = ""
revision = "a0f4589a76f1f83070cb9e5613809e1d07b97c13"
[[projects]]
branch = "master"
digest = "1:1c70f7bb89783a026dc32920575a3feef48e065ef6e170ad227903e8194d7a36"
name = "golang.org/x/text"
packages = [
"collate",
"collate/build",
"encoding",
"encoding/charmap",
"encoding/htmlindex",
"encoding/internal",
"encoding/internal/identifier",
"encoding/japanese",
"encoding/korean",
"encoding/simplifiedchinese",
"encoding/traditionalchinese",
"encoding/unicode",
"internal/colltab",
"internal/gen",
"internal/tag",
"internal/triegen",
"internal/ucd",
"internal/utf8internal",
"language",
"runes",
"secure/bidirule",
"transform",
"unicode/bidi",
"unicode/cldr",
"unicode/norm",
"unicode/rangetable",
]
pruneopts = ""
revision = "be25de41fadfae372d6470bda81ca6beb55ef551"
[[projects]]
branch = "master"
digest = "1:6c15114fafeac4c833544476dec4207a3476799d50ab5165af79eb97806884cf"
name = "google.golang.org/genproto"
packages = [
"googleapis/api/annotations",
"googleapis/rpc/status",
]
pruneopts = ""
revision = "7f0da29060c682909f650ad8ed4e515bd74fa12a"
[[projects]]
digest = "1:6c00b4702c146631d30396090d40bfc0486f8b3af5c976d6f0daf2bc737cd7b2"
name = "google.golang.org/grpc"
packages = [
".",
"balancer",
"balancer/roundrobin",
"codes",
"connectivity",
"credentials",
"encoding",
"grpclb/grpc_lb_v1/messages",
"grpclog",
"internal",
"keepalive",
"metadata",
"naming",
"peer",
"resolver",
"resolver/dns",
"resolver/manual",
"resolver/passthrough",
"stats",
"status",
"tap",
"transport",
]
pruneopts = ""
revision = "be077907e29fdb945d351e4284eb5361e7f8924e"
version = "v1.8.1"
[[projects]]
branch = "v2"
digest = "1:f769ed60e075e4221612c2f4162fccc9d3795ef358fa463425e3b3d7a5debb27"
name = "gopkg.in/yaml.v2"
packages = ["."]
pruneopts = ""
revision = "287cf08546ab5e7e37d55a84f7ed3fd1db036de5"
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
input-imports = [
"github.com/golang/protobuf/proto",
"github.com/golang/protobuf/ptypes/timestamp",
"github.com/lightstep/thrift/lib/go/thrift",
"github.com/onsi/ginkgo",
"github.com/onsi/gomega",
"github.com/onsi/gomega/types",
"github.com/opentracing/opentracing-go",
"github.com/opentracing/opentracing-go/log",
"golang.org/x/net/context",
"golang.org/x/net/http2",
"google.golang.org/genproto/googleapis/api/annotations",
"google.golang.org/grpc",
"google.golang.org/grpc/credentials",
]
solver-name = "gps-cdcl"
solver-version = 1
[[constraint]]
name = "github.com/onsi/ginkgo"
version = "1.4.0"
[[constraint]]
name = "github.com/onsi/gomega"
version = "1.2.0"
[[constraint]]
name = "github.com/opentracing/opentracing-go"
version = "1.0.2"
[[constraint]]
branch = "master"
name = "golang.org/x/net"
[[constraint]]
name = "google.golang.org/grpc"
version = "1.4.3"
[[constraint]]
name = "github.com/lightstep/thrift"
branch = "master"
The MIT License (MIT)
Copyright (c) 2016
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
# tools
GO=go
DOCKER_PRESENT = $(shell command -v docker 2> /dev/null)
LOCAL_GOPATH = $(PWD)/../../../../
default: build
.PHONY: default build test
# generate_fake: runs counterfeiter in docker container to generate fake classes
# $(1) output file path
# $(2) input file path
# $(3) class name
define generate_fake
docker run --rm -v $(LOCAL_GOPATH):/usergo \
lightstep/gobuild:latest /bin/bash -c "\
cd /usergo/src/github.com/lightstep/lightstep-tracer-go; \
counterfeiter -o $(1) $(2) $(3)"
endef
# Thrift
ifeq (,$(wildcard $(LIGHTSTEP_REPO_ROOT)/go/src/github.com/lightstep/common-go/crouton.thrift))
lightstep_thrift/constants.go:
else
# LightStep-specific: rebuilds the LightStep thrift protocol files.
# Assumes the command is run within the LightStep development
# environment (LIGHTSTEP_REPO_ROOT is set).
lightstep_thrift/constants.go: $(LIGHTSTEP_REPO_ROOT)/go/src/github.com/lightstep/common-go/crouton.thrift
docker run --rm -v "$(LIGHTSTEP_REPO_ROOT)/go/src/github.com/lightstep/common-go:/data" -v "$(PWD):/out" thrift:0.9.2 \
thrift --gen go:package_prefix='github.com/lightstep/lightstep-tracer-go/',thrift_import='github.com/lightstep/lightstep-tracer-go/thrift_0_9_2/lib/go/thrift' -out /out /data/crouton.thrift
rm -rf lightstep_thrift/reporting_service-remote
endif
lightstepfakes/fake_recorder.go: options.go
$(call generate_fake,lightstepfakes/fake_recorder.go,options.go,SpanRecorder)
lightstep_thrift/lightstep_thriftfakes/fake_reporting_service.go: lightstep_thrift/reportingservice.go
$(call generate_fake,lightstep_thrift/lightstep_thriftfakes/fake_reporting_service.go,lightstep_thrift/reportingservice.go,ReportingService)
collectorpb/collectorpbfakes/fake_collector_service_client.go: collectorpb/collector.pb.go
$(call generate_fake,collectorpb/collectorpbfakes/fake_collector_service_client.go,collectorpb/collector.pb.go,CollectorServiceClient)
# gRPC
ifeq (,$(wildcard lightstep-tracer-common/collector.proto))
collectorpb/collector.pb.go:
else
collectorpb/collector.pb.go: lightstep-tracer-common/collector.proto
docker run --rm -v $(shell pwd)/lightstep-tracer-common:/input:ro -v $(shell pwd)/collectorpb:/output \
lightstep/grpc-gateway:latest \
protoc -I/root/go/src/tmp/vendor/github.com/grpc-ecosystem/grpc-gateway/third_party/googleapis --go_out=plugins=grpc:/output --proto_path=/input /input/collector.proto
endif
# gRPC
ifeq (,$(wildcard lightstep-tracer-common/collector.proto))
lightsteppb/lightstep.pb.go:
else
lightsteppb/lightstep.pb.go: lightstep-tracer-common/lightstep.proto
docker run --rm -v $(shell pwd)/lightstep-tracer-common:/input:ro -v $(shell pwd)/lightsteppb:/output \
lightstep/protoc:latest \
protoc --go_out=plugins=grpc:/output --proto_path=/input /input/lightstep.proto
endif
test: lightstep_thrift/constants.go collectorpb/collector.pb.go lightsteppb/lightstep.pb.go \
collectorpb/collectorpbfakes/fake_collector_service_client.go \
lightstep_thrift/lightstep_thriftfakes/fake_reporting_service.go lightstepfakes/fake_recorder.go
ifeq ($(DOCKER_PRESENT),)
$(error "docker not found. Please install from https://www.docker.com/")
endif
docker run --rm -v $(LOCAL_GOPATH):/usergo lightstep/gobuild:latest \
ginkgo -race -p /usergo/src/github.com/lightstep/lightstep-tracer-go
bash -c "! git grep -q '[g]ithub.com/golang/glog'"
build: lightstep_thrift/constants.go collectorpb/collector.pb.go lightsteppb/lightstep.pb.go \
collectorpb/collectorpbfakes/fake_collector_service_client.go version.go \
lightstep_thrift/lightstep_thriftfakes/fake_reporting_service.go lightstepfakes/fake_recorder.go
ifeq ($(DOCKER_PRESENT),)
$(error "docker not found. Please install from https://www.docker.com/")
endif
${GO} build github.com/lightstep/lightstep-tracer-go
# When releasing significant changes, make sure to update the semantic
# version number in `./VERSION`, merge changes, then run `make release_tag`.
version.go: VERSION
./tag_version.sh
release_tag:
git tag -a v`cat ./VERSION`
git push origin v`cat ./VERSION`
# lightstep-tracer-go
[![Circle CI](https://circleci.com/gh/lightstep/lightstep-tracer-go.svg?style=shield)](https://circleci.com/gh/lightstep/lightstep-tracer-go)
[![MIT license](http://img.shields.io/badge/license-MIT-blue.svg)](http://opensource.org/licenses/MIT)
[![GoDoc](https://godoc.org/github.com/lightstep/lightstep-tracer-go?status.svg)](https://godoc.org/github.com/lightstep/lightstep-tracer-go)
The LightStep distributed tracing library for Go.
## Installation
```
$ go get 'github.com/lightstep/lightstep-tracer-go'
```
## API Documentation
Godoc: https://godoc.org/github.com/lightstep/lightstep-tracer-go
## Initialization: Starting a new tracer
To initialize a tracer, configure it with a valid Access Token and optional tuning parameters. Register the tracer as the OpenTracing global tracer so that it will become available to your installed intstrumentations libraries.
```go
import (
"github.com/opentracing/opentracing-go"
"github.com/lightstep/lightstep-tracer-go"
)
func main() {
lightstepTracer := lightstep.NewTracer(lightstep.Options{
AccessToken: "YourAccessToken",
})
opentracing.SetGlobalTracer(lightstepTracer)
}
```
## Instrumenting Code: Using the OpenTracing API
All instrumentation should be done through the OpenTracing API, rather than using the lightstep tracer type directly. For API documentation and advice on instrumentation in general, see the opentracing godocs and the opentracing website.
- https://godoc.org/github.com/opentracing/opentracing-go
- http://opentracing.io
## Flushing and Closing: Managing the tracer lifecycle
As part of managaing your application lifecycle, the lightstep tracer extends the `opentracing.Tracer` interface with methods for manual flushing and closing. To access these methods, you can take the global tracer and typecast it to a `lightstep.Tracer`. As a convenience, the lightstep package provides static methods which perform the typecasting.
```go
import (
"context"
"github.com/opentracing/opentracing-go"
"github.com/lightstep/lightstep-tracer-go"
)
func shutdown(ctx context.Context) {
// access the running tracer
tracer := opentracing.GlobalTracer()
// typecast from opentracing.Tracer to lightstep.Tracer
lsTracer, ok := tracer.(lightstep.Tracer)
if (!ok) {
return
}
lsTracer.Close(ctx)
// or use static methods
lightstep.Close(ctx, tracer)
}
```
## Event Handling: Observing the LightStep tracer
In order to connect diagnostic information from the lightstep tracer into an application's logging and metrics systems, inject an event handler using the `OnEvent` static method. Events may be typecast to check for errors or specific events such as status reports.
```go
import (
"example/logger"
"example/metrics"
"github.com/lightstep/lightstep-tracer-go"
)
logAndMetricsHandler := func(event lightstep.Event){
switch event := event.(type) {
case EventStatusReport:
metrics.Count("tracer.dropped_spans", event.DroppedSpans())
case ErrorEvent:
logger.Error("LS Tracer error: %s", event)
default:
logger.Info("LS Tracer info: %s", event)
}
}
func main() {
// setup event handler first to catch startup errors
lightstep.SetGlobalEventHandler(logAndMetricsHandler)
lightstepTracer := lightstep.NewTracer(lightstep.Options{
AccessToken: "YourAccessToken",
})
opentracing.SetGlobalTracer(lightstepTracer)
}
```
Event handlers will receive events from any active tracers, as well as errors in static functions. It is suggested that you set up event handling before initializing your tracer to catch any errors on initialization.
## Advanced Configuration: Transport and Serialization Protocols
By default, the tracer will send information to LightStep using GRPC and Protocol Buffers which is the recommended configuration. If there are no specific transport protocol needs you have, there is no need to change this default.
There are three total options for transport protocols:
- [Protocol Buffers](https://developers.google.com/protocol-buffers/) over [GRPC](https://grpc.io/) - The recommended, default, and most performant solution.
- [Thrift](https://thrift.apache.org/) over HTTP - A legacy implementation not recommended for new deployments.
- \[ EXPERIMENTAL \] [Protocol Buffers](https://developers.google.com/protocol-buffers/) over HTTP - New transport protocol supported for use cases where GRPC isn't an option. In order to enable HTTP you will need to configure the LightStep collectors receiving the data to accept HTTP traffic. Reach out to LightStep for support in this.
You can configure which transport protocol the tracer uses using the `UseGRPC`, `UseThrift`, and `UseHttp` flags in the options.
\ No newline at end of file
package lightstep
import (
"context"
"io"
"net/http"
cpb "github.com/lightstep/lightstep-tracer-go/collectorpb"
"github.com/lightstep/lightstep-tracer-go/lightstep_thrift"
)
// Connection describes a closable connection. Exposed for testing.
type Connection interface {
io.Closer
}
// ConnectorFactory is for testing purposes.
type ConnectorFactory func() (interface{}, Connection, error)
// collectorResponse encapsulates internal thrift/grpc responses.
type collectorResponse interface {
GetErrors() []string
Disable() bool
}
type reportRequest struct {
thriftRequest *lightstep_thrift.ReportRequest
protoRequest *cpb.ReportRequest
httpRequest *http.Request
}
// collectorClient encapsulates internal thrift/grpc transports.
type collectorClient interface {
Report(context.Context, reportRequest) (collectorResponse, error)
Translate(context.Context, *reportBuffer) (reportRequest, error)
ConnectClient() (Connection, error)
ShouldReconnect() bool
}
func newCollectorClient(opts Options, reporterID uint64, attributes map[string]string) (collectorClient, error) {
if opts.UseThrift {
return newThriftCollectorClient(opts, reporterID, attributes), nil
}
if opts.UseHttp {
return newHTTPCollectorClient(opts, reporterID, attributes)
}
if opts.UseGRPC {
return newGrpcCollectorClient(opts, reporterID, attributes), nil
}
// No transport specified, defaulting to GRPC
return newGrpcCollectorClient(opts, reporterID, attributes), nil
}
package lightstep
import (
"context"
"fmt"
"reflect"
"time"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials"
// N.B.(jmacd): Do not use google.golang.org/glog in this package.
cpb "github.com/lightstep/lightstep-tracer-go/collectorpb"
)
const (
spansDropped = "spans.dropped"
logEncoderErrors = "log_encoder.errors"
)
var (
intType = reflect.TypeOf(int64(0))
)
// grpcCollectorClient specifies how to send reports back to a LightStep
// collector via grpc.
type grpcCollectorClient struct {
// auth and runtime information
attributes map[string]string
reporterID uint64
// accessToken is the access token used for explicit trace collection requests.
accessToken string
maxReportingPeriod time.Duration // set by GrpcOptions.MaxReportingPeriod
reconnectPeriod time.Duration // set by GrpcOptions.ReconnectPeriod
reportingTimeout time.Duration // set by GrpcOptions.ReportTimeout
// Remote service that will receive reports.
address string
grpcClient cpb.CollectorServiceClient
connTimestamp time.Time
dialOptions []grpc.DialOption
// converters
converter *protoConverter
// For testing purposes only
grpcConnectorFactory ConnectorFactory
}
func newGrpcCollectorClient(opts Options, reporterID uint64, attributes map[string]string) *grpcCollectorClient {
rec := &grpcCollectorClient{
attributes: attributes,
reporterID: reporterID,
accessToken: opts.AccessToken,
maxReportingPeriod: opts.ReportingPeriod,
reconnectPeriod: opts.ReconnectPeriod,
reportingTimeout: opts.ReportTimeout,
dialOptions: opts.DialOptions,
converter: newProtoConverter(opts),
grpcConnectorFactory: opts.ConnFactory,
}
if len(opts.Collector.Scheme) > 0 {
rec.address = opts.Collector.urlWithoutPath()
} else {
rec.address = opts.Collector.SocketAddress()
}
rec.dialOptions = append(rec.dialOptions, grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(opts.GRPCMaxCallSendMsgSizeBytes)))
if opts.Collector.Plaintext {
rec.dialOptions = append(rec.dialOptions, grpc.WithInsecure())
} else {
rec.dialOptions = append(rec.dialOptions, grpc.WithTransportCredentials(credentials.NewClientTLSFromCert(nil, "")))
}
return rec
}
func (client *grpcCollectorClient) ConnectClient() (Connection, error) {
now := time.Now()
var conn Connection
if client.grpcConnectorFactory != nil {
uncheckedClient, transport, err := client.grpcConnectorFactory()
if err != nil {
return nil, err
}
grpcClient, ok := uncheckedClient.(cpb.CollectorServiceClient)
if !ok {
return nil, fmt.Errorf("gRPC connector factory did not provide valid client")
}
conn = transport
client.grpcClient = grpcClient
} else {
transport, err := grpc.Dial(client.address, client.dialOptions...)
if err != nil {
return nil, err
}
conn = transport
client.grpcClient = cpb.NewCollectorServiceClient(transport)
}
client.connTimestamp = now
return conn, nil
}
func (client *grpcCollectorClient) ShouldReconnect() bool {
return time.Since(client.connTimestamp) > client.reconnectPeriod
}
func (client *grpcCollectorClient) Report(ctx context.Context, req reportRequest) (collectorResponse, error) {
if req.protoRequest == nil {
return nil, fmt.Errorf("protoRequest cannot be null")
}
resp, err := client.grpcClient.Report(ctx, req.protoRequest)
if err != nil {
return nil, err
}
return resp, nil
}
func (client *grpcCollectorClient) Translate(ctx context.Context, buffer *reportBuffer) (reportRequest, error) {
req := client.converter.toReportRequest(
client.reporterID,
client.attributes,
client.accessToken,
buffer,
)
return reportRequest{
protoRequest: req,
}, nil
}
package lightstep
import (
"bytes"
"context"
"fmt"
"io/ioutil"
"net"
"net/http"
"net/url"
"time"
"github.com/golang/protobuf/proto"
"github.com/lightstep/lightstep-tracer-go/collectorpb"
)
var (
acceptHeader = http.CanonicalHeaderKey("Accept")
contentTypeHeader = http.CanonicalHeaderKey("Content-Type")
)
const (
collectorHTTPMethod = "POST"
collectorHTTPPath = "/api/v2/reports"
protoContentType = "application/octet-stream"
)
// grpcCollectorClient specifies how to send reports back to a LightStep
// collector via grpc.
type httpCollectorClient struct {
// auth and runtime information
reporterID uint64
accessToken string // accessToken is the access token used for explicit trace collection requests.
attributes map[string]string
reportTimeout time.Duration
reportingPeriod time.Duration
// Remote service that will receive reports.
url *url.URL
client *http.Client
// converters
converter *protoConverter
}
type transportCloser struct {
*http.Transport
}
func (closer transportCloser) Close() error {
closer.CloseIdleConnections()
return nil
}
func newHTTPCollectorClient(
opts Options,
reporterID uint64,
attributes map[string]string,
) (*httpCollectorClient, error) {
url, err := url.Parse(opts.Collector.URL())
if err != nil {
fmt.Println("collector config does not produce valid url", err)
return nil, err
}
url.Path = collectorHTTPPath
return &httpCollectorClient{
reporterID: reporterID,
accessToken: opts.AccessToken,
attributes: attributes,
reportTimeout: opts.ReportTimeout,
reportingPeriod: opts.ReportingPeriod,
url: url,
converter: newProtoConverter(opts),
}, nil
}
func (client *httpCollectorClient) ConnectClient() (Connection, error) {
// Use a transport independent from http.DefaultTransport to provide sane
// defaults that make sense in the context of the lightstep client. The
// differences are mostly on setting timeouts based on the report timeout
// and period.
transport := &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{
Timeout: client.reportTimeout / 2,
DualStack: true,
}).DialContext,
// The collector responses are very small, there is no point asking for
// a compressed payload, explicitly disabling it.
DisableCompression: true,
IdleConnTimeout: 2 * client.reportingPeriod,
TLSHandshakeTimeout: client.reportTimeout / 2,
ResponseHeaderTimeout: client.reportTimeout,
ExpectContinueTimeout: client.reportTimeout,
MaxResponseHeaderBytes: 64 * 1024, // 64 KB, just a safeguard
}
client.client = &http.Client{
Transport: transport,
Timeout: client.reportTimeout,
}
return transportCloser{transport}, nil
}
func (client *httpCollectorClient) ShouldReconnect() bool {
// http.Transport will handle connection reuse under the hood
return false
}
func (client *httpCollectorClient) Report(context context.Context, req reportRequest) (collectorResponse, error) {
if req.httpRequest == nil {
return nil, fmt.Errorf("httpRequest cannot be null")
}
httpResponse, err := client.client.Do(req.httpRequest.WithContext(context))
if err != nil {
return nil, err
}
defer httpResponse.Body.Close()
response, err := client.toResponse(httpResponse)
if err != nil {
return nil, err
}
return response, nil
}
func (client *httpCollectorClient) Translate(ctx context.Context, buffer *reportBuffer) (reportRequest, error) {
httpRequest, err := client.toRequest(ctx, buffer)
if err != nil {
return reportRequest{}, err
}
return reportRequest{
httpRequest: httpRequest,
}, nil
}
func (client *httpCollectorClient) toRequest(
context context.Context,
buffer *reportBuffer,
) (*http.Request, error) {
protoRequest := client.converter.toReportRequest(
client.reporterID,
client.attributes,
client.accessToken,
buffer,
)
buf, err := proto.Marshal(protoRequest)
if err != nil {
return nil, err
}
requestBody := bytes.NewReader(buf)
request, err := http.NewRequest(collectorHTTPMethod, client.url.String(), requestBody)
if err != nil {
return nil, err
}
request = request.WithContext(context)
request.Header.Set(contentTypeHeader, protoContentType)
request.Header.Set(acceptHeader, protoContentType)
return request, nil
}
func (client *httpCollectorClient) toResponse(response *http.Response) (collectorResponse, error) {
if response.StatusCode != http.StatusOK {
return nil, fmt.Errorf("status code (%d) is not ok", response.StatusCode)
}
body, err := ioutil.ReadAll(response.Body)
if err != nil {
return nil, err
}
protoResponse := &collectorpb.ReportResponse{}
if err := proto.Unmarshal(body, protoResponse); err != nil {
return nil, err
}
return protoResponse, nil
}
package lightstep
import (
"context"
"fmt"
"strconv"
"strings"
"time"
"github.com/lightstep/lightstep-tracer-go/lightstep_thrift"
"github.com/lightstep/thrift/lib/go/thrift"
)
// thriftCollectorClient specifies how to send reports back to a LightStep
// collector via thrift
type thriftCollectorClient struct {
// auth and runtime information
auth *lightstep_thrift.Auth
attributes map[string]string
startTime time.Time
reporterID uint64
maxReportingPeriod time.Duration
// Remote service that will receive reports
thriftClient lightstep_thrift.ReportingService
// apiURL is the base URL of the LightStep web API, used for
// explicit trace collection requests.
collectorURL string
// AccessToken is the access token used for explicit trace
// collection requests.
AccessToken string
verbose bool
// flags replacement
maxLogMessageLen int
maxLogKeyLen int
reportTimeout time.Duration
thriftConnectorFactory ConnectorFactory
}
func newThriftCollectorClient(opts Options, guid uint64, attributes map[string]string) *thriftCollectorClient {
reportTimeout := 60 * time.Second
if opts.ReportTimeout > 0 {
reportTimeout = opts.ReportTimeout
}
now := time.Now()
rec := &thriftCollectorClient{
auth: &lightstep_thrift.Auth{
AccessToken: thrift.StringPtr(opts.AccessToken),
},
attributes: attributes,
startTime: now,
maxReportingPeriod: opts.ReportingPeriod,
verbose: opts.Verbose,
collectorURL: opts.Collector.URL(),
AccessToken: opts.AccessToken,
maxLogMessageLen: opts.MaxLogValueLen,
maxLogKeyLen: opts.MaxLogKeyLen,
reportTimeout: reportTimeout,
thriftConnectorFactory: opts.ConnFactory,
reporterID: guid,
}
return rec
}
func (client *thriftCollectorClient) ConnectClient() (Connection, error) {
var conn Connection
if client.thriftConnectorFactory != nil {
uncheckedClient, transport, err := client.thriftConnectorFactory()
if err != nil {
return nil, err
}
thriftClient, ok := uncheckedClient.(lightstep_thrift.ReportingService)
if !ok {
return nil, fmt.Errorf("thrift connector factory did not provide valid client")
}
conn = transport
client.thriftClient = thriftClient
} else {
transport, err := thrift.NewTHttpPostClient(client.collectorURL, client.reportTimeout)
if err != nil {
return nil, err
}
conn = transport
client.thriftClient = lightstep_thrift.NewReportingServiceClientFactory(
transport,
thrift.NewTBinaryProtocolFactoryDefault(),
)
}
return conn, nil
}
func (*thriftCollectorClient) ShouldReconnect() bool {
return false
}
func (client *thriftCollectorClient) Report(_ context.Context, req reportRequest) (collectorResponse, error) {
if req.thriftRequest == nil {
return nil, fmt.Errorf("thriftRequest cannot be null")
}
resp, err := client.thriftClient.Report(client.auth, req.thriftRequest)
if err != nil {
return nil, err
}
return resp, err
}
func (client *thriftCollectorClient) Translate(_ context.Context, buffer *reportBuffer) (reportRequest, error) {
rawSpans := buffer.rawSpans
// Convert them to thrift.
recs := make([]*lightstep_thrift.SpanRecord, len(rawSpans))
// TODO: could pool lightstep_thrift.SpanRecords
for i, raw := range rawSpans {
var joinIds []*lightstep_thrift.TraceJoinId
var attributes []*lightstep_thrift.KeyValue
for key, value := range raw.Tags {
// Note: the gRPC tracer uses Sprintf("%#v") for non-scalar non-string
// values, differs from the treatment here:
if strings.HasPrefix(key, "join:") {
joinIds = append(joinIds, &lightstep_thrift.TraceJoinId{TraceKey: key, Value: fmt.Sprint(value)})
} else {
attributes = append(attributes, &lightstep_thrift.KeyValue{Key: key, Value: fmt.Sprint(value)})
}
}
logs := make([]*lightstep_thrift.LogRecord, len(raw.Logs))
for j, log := range raw.Logs {
thriftLogRecord := &lightstep_thrift.LogRecord{
TimestampMicros: thrift.Int64Ptr(log.Timestamp.UnixNano() / 1000),
}
// In the deprecated thrift case, we can reuse a single "field"
// encoder across all of the N log fields.
lfe := thriftLogFieldEncoder{thriftLogRecord, client}
for _, f := range log.Fields {
f.Marshal(&lfe)
}
logs[j] = thriftLogRecord
}
// TODO implement baggage
if raw.ParentSpanID != 0 {
attributes = append(attributes, &lightstep_thrift.KeyValue{Key: ParentSpanGUIDKey,
Value: strconv.FormatUint(raw.ParentSpanID, 16)})
}
recs[i] = &lightstep_thrift.SpanRecord{
SpanGuid: thrift.StringPtr(strconv.FormatUint(raw.Context.SpanID, 16)),
TraceGuid: thrift.StringPtr(strconv.FormatUint(raw.Context.TraceID, 16)),
SpanName: thrift.StringPtr(raw.Operation),
JoinIds: joinIds,
OldestMicros: thrift.Int64Ptr(raw.Start.UnixNano() / 1000),
YoungestMicros: thrift.Int64Ptr(raw.Start.Add(raw.Duration).UnixNano() / 1000),
Attributes: attributes,
LogRecords: logs,
}
}
metrics := lightstep_thrift.Metrics{
Counts: []*lightstep_thrift.MetricsSample{
&lightstep_thrift.MetricsSample{
Name: "spans.dropped",
Int64Value: &buffer.droppedSpanCount,
},
},
}
req := &lightstep_thrift.ReportRequest{
OldestMicros: thrift.Int64Ptr(buffer.reportEnd.UnixNano() / 1000),
YoungestMicros: thrift.Int64Ptr(buffer.reportStart.UnixNano() / 1000),
Runtime: client.thriftRuntime(),
SpanRecords: recs,
InternalMetrics: &metrics,
}
return reportRequest{
thriftRequest: req,
}, nil
}
// caller must hold r.lock
func (client *thriftCollectorClient) thriftRuntime() *lightstep_thrift.Runtime {
guid := strconv.FormatUint(client.reporterID, 10)
runtimeAttrs := []*lightstep_thrift.KeyValue{}
for k, v := range client.attributes {
runtimeAttrs = append(runtimeAttrs, &lightstep_thrift.KeyValue{Key: k, Value: v})
}
return &lightstep_thrift.Runtime{
StartMicros: thrift.Int64Ptr(client.startTime.UnixNano() / 1000),
Attrs: runtimeAttrs,
Guid: &guid,
}
}
package collectorpb
func (res *ReportResponse) Disable() bool {
for _, command := range res.GetCommands() {
if command.Disable {
return true
}
}
return false
}
package lightstep
import (
"log"
"sync"
"sync/atomic"
)
func init() {
SetGlobalEventHandler(NewEventLogOneError())
}
var eventHandler atomic.Value
// An EventHandler can be registered with SetGlobalEventHandler to
type EventHandler func(Event)
// emitEvent is a thread-safe function for emiting tracer events.
func emitEvent(event Event) {
handler := eventHandler.Load().(EventHandler)
handler(event)
}
// SetGlobalEventHandler sets a global handler to receive tracer events as they occur. Events
// may be emitted by the tracer, or by calls to static functions in this package.
// It is suggested that you set your EventHandler before starting your tracer,
// but it is safe to set a new handler at any time. Setting a new handler removes
// the previous one.
//
// The EventHandler is called synchronously – do not block in your handler as it
// may interfere with the tracer. If no EventHandler handler is set, a
// LogOnceOnError handler is set by default.
//
// NOTE: Event handling is for reporting purposes only. It is not intended as a
// mechanism for controling or restarting the tracer. Connection issues, retry
// logic, and other transient errors are handled internally by the tracer.
func SetGlobalEventHandler(handler EventHandler) {
eventHandler.Store(handler)
}
/*
SetGlobalEventHandler Handlers
*/
// NewEventLogger logs events using the standard go logger.
func NewEventLogger() EventHandler {
return logOnEvent
}
func logOnEvent(event Event) {
switch event := event.(type) {
case ErrorEvent:
log.Println("LS Tracer error: ", event)
default:
log.Println("LS Tracer event: ", event)
}
}
// NewEventLogOneError logs the first error event that occurs.
func NewEventLogOneError() EventHandler {
logger := logOneError{}
return logger.OnEvent
}
type logOneError struct {
sync.Once
}
func (l *logOneError) OnEvent(event Event) {
switch event := event.(type) {
case ErrorEvent:
l.Once.Do(func() {
log.Printf("LS Tracer error: (%s). NOTE: Set the SetGlobalEventHandler handler to log events.\n", event.Error())
})
}
}
// NewEventChannel returns an SetGlobalEventHandler callback handler, and a channel that
// produces the errors. When the channel buffer is full, subsequent errors will
// be dropped. A buffer size of less than one is incorrect, and will be adjusted
// to a buffer size of one.
func NewEventChannel(buffer int) (EventHandler, <-chan Event) {
if buffer < 1 {
buffer = 1
}
eventChan := make(chan Event, buffer)
handler := func(event Event) {
select {
case eventChan <- event:
default:
}
}
return handler, eventChan
}
package lightstep
import (
"errors"
"fmt"
"reflect"
"time"
opentracing "github.com/opentracing/opentracing-go"
)
// An Event is emitted by the LightStep tracer as a reporting mechanism. They are
// handled by registering an EventHandler callback via SetGlobalEventHandler. The
// emitted events may be cast to specific event types in order access additional
// information.
//
// NOTE: To ensure that events can be accurately identified, each event type contains
// a sentinel method matching the name of the type. This method is a no-op, it is
// only used for type coersion.
type Event interface {
Event()
String() string
}
// The ErrorEvent type can be used to filter events. The `Err` method
// retuns the underlying error.
type ErrorEvent interface {
Event
error
Err() error
}
// EventStartError occurs if the Options passed to NewTracer are invalid, and
// the Tracer has failed to start.
type EventStartError interface {
ErrorEvent
EventStartError()
}
type eventStartError struct {
err error
}
func newEventStartError(err error) *eventStartError {
return &eventStartError{err: err}
}
func (*eventStartError) Event() {}
func (*eventStartError) EventStartError() {}
func (e *eventStartError) String() string {
return e.err.Error()
}
func (e *eventStartError) Error() string {
return e.err.Error()
}
func (e *eventStartError) Err() error {
return e.err
}
// EventFlushErrorState lists the possible causes for a flush to fail.
type EventFlushErrorState string
// Constant strings corresponding to flush errors
const (
FlushErrorTracerClosed EventFlushErrorState = "flush failed, the tracer is closed."
FlushErrorTracerDisabled EventFlushErrorState = "flush failed, the tracer is disabled."
FlushErrorTransport EventFlushErrorState = "flush failed, could not send report to Collector"
FlushErrorReport EventFlushErrorState = "flush failed, report contained errors"
FlushErrorTranslate EventFlushErrorState = "flush failed, could not translate report"
)
var (
errFlushFailedTracerClosed = errors.New(string(FlushErrorTracerClosed))
)
// EventFlushError occurs when a flush fails to send. Call the `State` method to
// determine the type of error.
type EventFlushError interface {
ErrorEvent
EventFlushError()
State() EventFlushErrorState
}
type eventFlushError struct {
err error
state EventFlushErrorState
}
func newEventFlushError(err error, state EventFlushErrorState) *eventFlushError {
return &eventFlushError{err: err, state: state}
}
func (*eventFlushError) Event() {}
func (*eventFlushError) EventFlushError() {}
func (e *eventFlushError) State() EventFlushErrorState {
return e.state
}
func (e *eventFlushError) String() string {
return e.err.Error()
}
func (e *eventFlushError) Error() string {
return e.err.Error()
}
func (e *eventFlushError) Err() error {
return e.err
}
// EventConnectionError occurs when the tracer fails to maintain it's connection
// with the Collector.
type EventConnectionError interface {
ErrorEvent
EventConnectionError()
}
type eventConnectionError struct {
err error
}
func newEventConnectionError(err error) *eventConnectionError {
return &eventConnectionError{err: err}
}
func (*eventConnectionError) Event() {}
func (*eventConnectionError) EventConnectionError() {}
func (e *eventConnectionError) String() string {
return e.err.Error()
}
func (e *eventConnectionError) Error() string {
return e.err.Error()
}
func (e *eventConnectionError) Err() error {
return e.err
}
// EventStatusReport occurs on every successful flush. It contains all metrics
// collected since the previous succesful flush.
type EventStatusReport interface {
Event
EventStatusReport()
StartTime() time.Time
FinishTime() time.Time
Duration() time.Duration
SentSpans() int
DroppedSpans() int
EncodingErrors() int
}
type eventStatusReport struct {
startTime time.Time
finishTime time.Time
sentSpans int
droppedSpans int
encodingErrors int
}
func newEventStatusReport(
startTime, finishTime time.Time,
sentSpans, droppedSpans, encodingErrors int,
) *eventStatusReport {
return &eventStatusReport{
startTime: startTime,
finishTime: finishTime,
sentSpans: sentSpans,
droppedSpans: droppedSpans,
encodingErrors: encodingErrors,
}
}
func (*eventStatusReport) Event() {}
func (*eventStatusReport) EventStatusReport() {}
func (s *eventStatusReport) SetSentSpans(sent int) {
s.sentSpans = sent
}
func (s *eventStatusReport) StartTime() time.Time {
return s.startTime
}
func (s *eventStatusReport) FinishTime() time.Time {
return s.finishTime
}
func (s *eventStatusReport) Duration() time.Duration {
return s.finishTime.Sub(s.startTime)
}
func (s *eventStatusReport) SentSpans() int {
return s.sentSpans
}
func (s *eventStatusReport) DroppedSpans() int {
return s.droppedSpans
}
func (s *eventStatusReport) EncodingErrors() int {
return s.encodingErrors
}
func (s *eventStatusReport) String() string {
return fmt.Sprint(
"STATUS REPORT start: ", s.startTime,
", end: ", s.finishTime,
", dropped spans: ", s.droppedSpans,
", encoding errors: ", s.encodingErrors,
)
}
// EventUnsupportedTracer occurs when a tracer being passed to a helper function
// fails to typecast as a LightStep tracer.
type EventUnsupportedTracer interface {
ErrorEvent
EventUnsupportedTracer()
Tracer() opentracing.Tracer
}
type eventUnsupportedTracer struct {
tracer opentracing.Tracer
err error
}
func newEventUnsupportedTracer(tracer opentracing.Tracer) EventUnsupportedTracer {
return &eventUnsupportedTracer{
tracer: tracer,
err: fmt.Errorf("unsupported tracer type: %v", reflect.TypeOf(tracer)),
}
}
func (e *eventUnsupportedTracer) Event() {}
func (e *eventUnsupportedTracer) EventUnsupportedTracer() {}
func (e *eventUnsupportedTracer) Tracer() opentracing.Tracer {
return e.tracer
}
func (e *eventUnsupportedTracer) String() string {
return e.err.Error()
}
func (e *eventUnsupportedTracer) Error() string {
return e.err.Error()
}
func (e *eventUnsupportedTracer) Err() error {
return e.err
}
// EventUnsupportedValue occurs when a tracer encounters an unserializable tag
// or log field.
type EventUnsupportedValue interface {
ErrorEvent
EventUnsupportedValue()
Key() string
Value() interface{}
}
type eventUnsupportedValue struct {
key string
value interface{}
err error
}
func newEventUnsupportedValue(key string, value interface{}, err error) EventUnsupportedValue {
if err == nil {
err = fmt.Errorf(
"value `%v` of type `%T` for key `%s` is an unsupported type",
value, value, key,
)
}
return &eventUnsupportedValue{
key: key,
value: value,
err: err,
}
}
func (e *eventUnsupportedValue) Event() {}
func (e *eventUnsupportedValue) EventUnsupportedValue() {}
func (e *eventUnsupportedValue) Key() string {
return e.key
}
func (e *eventUnsupportedValue) Value() interface{} {
return e.value
}
func (e *eventUnsupportedValue) String() string {
return e.err.Error()
}
func (e *eventUnsupportedValue) Error() string {
return e.err.Error()
}
func (e *eventUnsupportedValue) Err() error {
return e.err
}
const tracerDisabled = "the tracer has been disabled"
// EventTracerDisabled occurs when a tracer is disabled by either the user or
// the collector.
type EventTracerDisabled interface {
Event
EventTracerDisabled()
}
type eventTracerDisabled struct{}
func newEventTracerDisabled() EventTracerDisabled {
return eventTracerDisabled{}
}
func (eventTracerDisabled) Event() {}
func (eventTracerDisabled) EventTracerDisabled() {}
func (eventTracerDisabled) String() string {
return tracerDisabled
}
package rand
import (
"math/rand"
"sync"
)
// LockedRand implements NumberGenerator and embeds Rand that is safe for concurrent use.
type LockedRand struct {
lk sync.Mutex
r *rand.Rand
}
// NewLockedRand creates a new LockedRand that implements all Rand functions that is safe
// for concurrent use.
func NewLockedRand(seed int64) *LockedRand {
return &LockedRand{
r: rand.New(rand.NewSource(seed)),
}
}
// Seed uses the provided seed value to initialize the generator to a deterministic state.
// Seed should not be called concurrently with any other Rand method.
func (lr *LockedRand) Seed(seed int64) {
lr.lk.Lock()
lr.r.Seed(seed)
lr.lk.Unlock()
}
// TwoInt63 generates 2 random int64 without locking twice.
func (lr *LockedRand) TwoInt63() (n1, n2 int64) {
lr.lk.Lock()
n1 = lr.r.Int63()
n2 = lr.r.Int63()
lr.lk.Unlock()
return
}
// Int63 returns a non-negative pseudo-random 63-bit integer as an int64.
func (lr *LockedRand) Int63() (n int64) {
lr.lk.Lock()
n = lr.r.Int63()
lr.lk.Unlock()
return
}
// Uint32 returns a pseudo-random 32-bit value as a uint32.
func (lr *LockedRand) Uint32() (n uint32) {
lr.lk.Lock()
n = lr.r.Uint32()
lr.lk.Unlock()
return
}
// Uint64 returns a pseudo-random 64-bit value as a uint64.
func (lr *LockedRand) Uint64() (n uint64) {
lr.lk.Lock()
n = lr.r.Uint64()
lr.lk.Unlock()
return
}
// Int31 returns a non-negative pseudo-random 31-bit integer as an int32.
func (lr *LockedRand) Int31() (n int32) {
lr.lk.Lock()
n = lr.r.Int31()
lr.lk.Unlock()
return
}
// Int returns a non-negative pseudo-random int.
func (lr *LockedRand) Int() (n int) {
lr.lk.Lock()
n = lr.r.Int()
lr.lk.Unlock()
return
}
// Int63n returns, as an int64, a non-negative pseudo-random number in [0,n).
// It panics if n <= 0.
func (lr *LockedRand) Int63n(n int64) (r int64) {
lr.lk.Lock()
r = lr.r.Int63n(n)
lr.lk.Unlock()
return
}
// Int31n returns, as an int32, a non-negative pseudo-random number in [0,n).
// It panics if n <= 0.
func (lr *LockedRand) Int31n(n int32) (r int32) {
lr.lk.Lock()
r = lr.r.Int31n(n)
lr.lk.Unlock()
return
}
// Intn returns, as an int, a non-negative pseudo-random number in [0,n).
// It panics if n <= 0.
func (lr *LockedRand) Intn(n int) (r int) {
lr.lk.Lock()
r = lr.r.Intn(n)
lr.lk.Unlock()
return
}
// Float64 returns, as a float64, a pseudo-random number in [0.0,1.0).
func (lr *LockedRand) Float64() (n float64) {
lr.lk.Lock()
n = lr.r.Float64()
lr.lk.Unlock()
return
}
// Float32 returns, as a float32, a pseudo-random number in [0.0,1.0).
func (lr *LockedRand) Float32() (n float32) {
lr.lk.Lock()
n = lr.r.Float32()
lr.lk.Unlock()
return
}
// Perm returns, as a slice of n ints, a pseudo-random permutation of the integers [0,n).
func (lr *LockedRand) Perm(n int) (r []int) {
lr.lk.Lock()
r = lr.r.Perm(n)
lr.lk.Unlock()
return
}
// Read generates len(p) random bytes and writes them into p. It
// always returns len(p) and a nil error.
// Read should not be called concurrently with any other Rand method.
func (lr *LockedRand) Read(p []byte) (n int, err error) {
lr.lk.Lock()
n, err = lr.r.Read(p)
lr.lk.Unlock()
return
}
package rand
// NumberGenerator defines an interface to generate numbers.
type NumberGenerator interface {
Int63() int64
TwoInt63() (int64, int64)
}
package rand
import (
"math/rand"
"sync/atomic"
)
// Pool represents a pool of random number generators.
// To generate a random id, round robin through the source pool with atomic increment.
// With more and more goroutines, Pool improves the performance of Random vs naive global random
// mutex exponentially.
// Try tests with 20000 goroutines and 500 calls to observe the difference
type Pool struct {
sources []NumberGenerator
counter uint64 // used for round robin
size uint64
}
// see bit twiddling hacks: https://graphics.stanford.edu/~seander/bithacks.html#RoundUpPowerOf2
func nextNearestPow2uint64(v uint64) uint64 {
v--
v |= v >> 1
v |= v >> 2
v |= v >> 4
v |= v >> 8
v |= v >> 16
v |= v >> 32
v++
return v
}
// NewPool takes in a size and creates a pool of random id generators with size equal to next closest power of 2.
// eg: NewPool(10) returns a pool with 2^4 = 16 random sources.
func NewPool(seed int64, size uint64) *Pool {
groupsize := nextNearestPow2uint64(size)
pool := &Pool{
size: groupsize,
sources: make([]NumberGenerator, groupsize),
}
// seed the pool
pool.seed(seed)
return pool
}
// seed initializes the pool using a randomized sequence with given seed.
func (r *Pool) seed(seed int64) {
// init a random sequence to seed all sources
seedRan := rand.NewSource(seed)
for i := uint64(0); i < r.size; i++ {
r.sources[i] = NewLockedRand(seedRan.Int63())
}
}
// Pick returns a NumberGenerator from a pool of NumberGenerators
func (r *Pool) Pick() NumberGenerator {
// use round robin with fast modulus of pow2 numbers
selection := atomic.AddUint64(&r.counter, 1) & (r.size - 1)
return r.sources[selection]
}
// Autogenerated by Thrift Compiler (0.9.2)
// DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
package lightstep_thrift
import (
"bytes"
"fmt"
"github.com/lightstep/thrift/lib/go/thrift"
)
// (needed to ensure safety because of naive import list construction.)
var _ = thrift.ZERO
var _ = fmt.Printf
var _ = bytes.Equal
func init() {
}
package lightstep_thrift
func (res *ReportResponse) Disable() bool {
for _, command := range res.GetCommands() {
if *command.Disable {
return true
}
}
return false
}
// Code generated by protoc-gen-go. DO NOT EDIT.
// source: lightstep.proto
/*
Package lightsteppb is a generated protocol buffer package.
It is generated from these files:
lightstep.proto
It has these top-level messages:
BinaryCarrier
BasicTracerCarrier
*/
package lightsteppb
import proto "github.com/golang/protobuf/proto"
import fmt "fmt"
import math "math"
// Reference imports to suppress errors if they are not otherwise used.
var _ = proto.Marshal
var _ = fmt.Errorf
var _ = math.Inf
// This is a compile-time assertion to ensure that this generated file
// is compatible with the proto package it is being compiled against.
// A compilation error at this line likely means your copy of the
// proto package needs to be updated.
const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
// The standard carrier for binary context propagation into LightStep.
type BinaryCarrier struct {
// "text_ctx" was deprecated following lightstep-tracer-cpp-0.36
DeprecatedTextCtx [][]byte `protobuf:"bytes,1,rep,name=deprecated_text_ctx,json=deprecatedTextCtx,proto3" json:"deprecated_text_ctx,omitempty"`
// The Opentracing "basictracer" proto.
BasicCtx *BasicTracerCarrier `protobuf:"bytes,2,opt,name=basic_ctx,json=basicCtx" json:"basic_ctx,omitempty"`
}
func (m *BinaryCarrier) Reset() { *m = BinaryCarrier{} }
func (m *BinaryCarrier) String() string { return proto.CompactTextString(m) }
func (*BinaryCarrier) ProtoMessage() {}
func (*BinaryCarrier) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} }
func (m *BinaryCarrier) GetDeprecatedTextCtx() [][]byte {
if m != nil {
return m.DeprecatedTextCtx
}
return nil
}
func (m *BinaryCarrier) GetBasicCtx() *BasicTracerCarrier {
if m != nil {
return m.BasicCtx
}
return nil
}
// Copy of https://github.com/opentracing/basictracer-go/blob/master/wire/wire.proto
type BasicTracerCarrier struct {
TraceId uint64 `protobuf:"fixed64,1,opt,name=trace_id,json=traceId" json:"trace_id,omitempty"`
SpanId uint64 `protobuf:"fixed64,2,opt,name=span_id,json=spanId" json:"span_id,omitempty"`
Sampled bool `protobuf:"varint,3,opt,name=sampled" json:"sampled,omitempty"`
BaggageItems map[string]string `protobuf:"bytes,4,rep,name=baggage_items,json=baggageItems" json:"baggage_items,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
}
func (m *BasicTracerCarrier) Reset() { *m = BasicTracerCarrier{} }
func (m *BasicTracerCarrier) String() string { return proto.CompactTextString(m) }
func (*BasicTracerCarrier) ProtoMessage() {}
func (*BasicTracerCarrier) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} }
func (m *BasicTracerCarrier) GetTraceId() uint64 {
if m != nil {
return m.TraceId
}
return 0
}
func (m *BasicTracerCarrier) GetSpanId() uint64 {
if m != nil {
return m.SpanId
}
return 0
}
func (m *BasicTracerCarrier) GetSampled() bool {
if m != nil {
return m.Sampled
}
return false
}
func (m *BasicTracerCarrier) GetBaggageItems() map[string]string {
if m != nil {
return m.BaggageItems
}
return nil
}
func init() {
proto.RegisterType((*BinaryCarrier)(nil), "lightstep.BinaryCarrier")
proto.RegisterType((*BasicTracerCarrier)(nil), "lightstep.BasicTracerCarrier")
}
func init() { proto.RegisterFile("lightstep.proto", fileDescriptor0) }
var fileDescriptor0 = []byte{
// 290 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x91, 0xcf, 0x4b, 0xf3, 0x30,
0x18, 0xc7, 0xc9, 0xfa, 0xbe, 0xdb, 0xfa, 0x6c, 0x43, 0x17, 0x05, 0xab, 0x20, 0x94, 0x9d, 0x7a,
0xaa, 0x30, 0x2f, 0xb2, 0x8b, 0xd0, 0xe1, 0x61, 0xd7, 0xb0, 0x93, 0x97, 0x92, 0x36, 0x0f, 0x35,
0xb8, 0x75, 0x21, 0x7d, 0x94, 0x16, 0xff, 0x72, 0x6f, 0x92, 0x4c, 0x37, 0x61, 0xe0, 0x2d, 0xdf,
0x1f, 0x0f, 0xdf, 0x0f, 0x04, 0xce, 0x36, 0xba, 0x7a, 0xa1, 0x86, 0xd0, 0xa4, 0xc6, 0xee, 0x68,
0xc7, 0xc3, 0x83, 0x31, 0xfb, 0x80, 0x49, 0xa6, 0x6b, 0x69, 0xbb, 0xa5, 0xb4, 0x56, 0xa3, 0xe5,
0x29, 0x5c, 0x28, 0x34, 0x16, 0x4b, 0x49, 0xa8, 0x72, 0xc2, 0x96, 0xf2, 0x92, 0xda, 0x88, 0xc5,
0x41, 0x32, 0x16, 0xd3, 0x63, 0xb4, 0xc6, 0x96, 0x96, 0xd4, 0xf2, 0x05, 0x84, 0x85, 0x6c, 0x74,
0xe9, 0x5b, 0xbd, 0x98, 0x25, 0xa3, 0xf9, 0x6d, 0x7a, 0x1c, 0xcc, 0x5c, 0xb6, 0xb6, 0xb2, 0x44,
0xfb, 0xbd, 0x20, 0x86, 0xbe, 0xbf, 0xa4, 0x76, 0xf6, 0xc9, 0x80, 0x9f, 0x16, 0xf8, 0x35, 0x0c,
0xc9, 0x19, 0xb9, 0x56, 0x11, 0x8b, 0x59, 0xd2, 0x17, 0x03, 0xaf, 0x57, 0x8a, 0x5f, 0xc1, 0xa0,
0x31, 0xb2, 0x76, 0x49, 0xcf, 0x27, 0x7d, 0x27, 0x57, 0x8a, 0x47, 0x30, 0x68, 0xe4, 0xd6, 0x6c,
0x50, 0x45, 0x41, 0xcc, 0x92, 0xa1, 0xf8, 0x91, 0x7c, 0x0d, 0x93, 0x42, 0x56, 0x95, 0xac, 0x30,
0xd7, 0x84, 0xdb, 0x26, 0xfa, 0x17, 0x07, 0xc9, 0x68, 0x7e, 0xf7, 0x27, 0x64, 0x9a, 0xed, 0x4f,
0x56, 0xee, 0xe2, 0xa9, 0x26, 0xdb, 0x89, 0x71, 0xf1, 0xcb, 0xba, 0x79, 0x84, 0xe9, 0x49, 0x85,
0x9f, 0x43, 0xf0, 0x8a, 0x9d, 0x67, 0x0e, 0x85, 0x7b, 0xf2, 0x4b, 0xf8, 0xff, 0x2e, 0x37, 0x6f,
0xe8, 0x69, 0x43, 0xb1, 0x17, 0x8b, 0xde, 0x03, 0xcb, 0x26, 0xcf, 0xa3, 0x03, 0x80, 0x29, 0x8a,
0xbe, 0xff, 0x99, 0xfb, 0xaf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x7a, 0xf7, 0xd5, 0xf5, 0xac, 0x01,
0x00, 0x00,
}
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
#!/bin/sh
VERSION=$(cat ./VERSION)
cat > version.go <<EOF
package lightstep
// TracerVersionValue provides the current version of the lightstep-tracer-go release
const TracerVersionValue = "$VERSION"
EOF
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
package lightstep
// TracerVersionValue provides the current version of the lightstep-tracer-go release
const TracerVersionValue = "0.15.6"
This diff is collapsed.
Apache Thrift
Copyright 2006-2010 The Apache Software Foundation.
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
\ No newline at end of file
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment