Merge branch 'cat-secondary-proxy-workhorse-metrics' into 'master'

Add Workhorse metrics for secondary proxy requests

See merge request gitlab-org/gitlab!75936

workhorse/internal/upstream/metrics.go

@@ -101,6 +101,16 @@ var (
 		},
 		[]string{"code", "method", "route"},
 	)
+
+	httpGeoProxiedRequestsTotal = promauto.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: namespace,
+			Subsystem: httpSubsystem,
+			Name:      "geo_proxied_requests_total",
+			Help:      "A counter for Geo proxied requests through workhorse.",
+		},
+		[]string{"code", "method", "route"},
+	)
 )
 
 func instrumentRoute(next http.Handler, method string, regexpStr string) http.Handler {
@@ -115,3 +125,7 @@ func instrumentRoute(next http.Handler, method string, regexpStr string) http.Ha
 
 	return handler
 }
+
+func instrumentGeoProxyRoute(next http.Handler, method string, regexpStr string) http.Handler {
+	return promhttp.InstrumentHandlerCounter(httpGeoProxiedRequestsTotal.MustCurryWith(map[string]string{"route": regexpStr}), next)
+}

workhorse/internal/upstream/metrics_test.go

+package upstream
+
+import (
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus/testutil"
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/require"
+
+	"gitlab.com/gitlab-org/gitlab/workhorse/internal/config"
+)
+
+func TestInstrumentGeoProxyRoute(t *testing.T) {
+	const (
+		remote = `\A/remote\z`
+		local  = `\A/local\z`
+		main   = ""
+	)
+
+	u := newUpstream(config.Config{}, logrus.StandardLogger(), func(u *upstream) {
+		u.Routes = []routeEntry{
+			handleRouteWithMatchers(u, remote, withGeoProxy()),
+			handleRouteWithMatchers(u, local),
+			handleRouteWithMatchers(u, main),
+		}
+	})
+	ts := httptest.NewServer(u)
+	defer ts.Close()
+
+	testCases := []testCase{
+		{"remote", "/remote", remote},
+		{"local", "/local", local},
+		{"main", "/", main},
+	}
+
+	httpGeoProxiedRequestsTotal.Reset()
+
+	runTestCases(t, ts, testCases)
+
+	require.Equal(t, 1, testutil.CollectAndCount(httpGeoProxiedRequestsTotal))
+	require.InDelta(t, 1, testutil.ToFloat64(httpGeoProxiedRequestsTotal.WithLabelValues("200", "get", remote)), 0.1)
+	require.InDelta(t, 0, testutil.ToFloat64(httpGeoProxiedRequestsTotal.WithLabelValues("200", "get", local)), 0.1)
+	require.InDelta(t, 0, testutil.ToFloat64(httpGeoProxiedRequestsTotal.WithLabelValues("200", "get", main)), 0.1)
+}
+
+func handleRouteWithMatchers(u *upstream, regex string, matchers ...func(*routeOptions)) routeEntry {
+	handler := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		io.WriteString(w, regex)
+	})
+	return u.route("", regex, handler, matchers...)
+}

workhorse/internal/upstream/routes.go

@@ -42,8 +42,9 @@ type routeEntry struct {
 }
 
 type routeOptions struct {
-	tracing  bool
-	matchers []matcherFunc
+	tracing         bool
+	isGeoProxyRoute bool
+	matchers        []matcherFunc
 }
 
 type uploadPreparers struct {
@@ -94,7 +95,13 @@ func withoutTracing() func(*routeOptions) {
 	}
 }
 
-func (u *upstream) observabilityMiddlewares(handler http.Handler, method string, regexpStr string) http.Handler {
+func withGeoProxy() func(*routeOptions) {
+	return func(options *routeOptions) {
+		options.isGeoProxyRoute = true
+	}
+}
+
+func (u *upstream) observabilityMiddlewares(handler http.Handler, method string, regexpStr string, opts *routeOptions) http.Handler {
 	handler = log.AccessLogger(
 		handler,
 		log.WithAccessLogger(u.accessLogger),
@@ -106,6 +113,11 @@ func (u *upstream) observabilityMiddlewares(handler http.Handler, method string,
 	)
 
 	handler = instrumentRoute(handler, method, regexpStr) // Add prometheus metrics
+
+	if opts != nil && opts.isGeoProxyRoute {
+		handler = instrumentGeoProxyRoute(handler, method, regexpStr) // Add Geo prometheus metrics
+	}
+
 	return handler
 }
 
@@ -119,7 +131,7 @@ func (u *upstream) route(method, regexpStr string, handler http.Handler, opts ..
 		f(&options)
 	}
 
-	handler = u.observabilityMiddlewares(handler, method, regexpStr)
+	handler = u.observabilityMiddlewares(handler, method, regexpStr, &options)
 	handler = denyWebsocket(handler) // Disallow websockets
 	if options.tracing {
 		// Add distributed tracing
@@ -136,7 +148,7 @@ func (u *upstream) route(method, regexpStr string, handler http.Handler, opts ..
 
 func (u *upstream) wsRoute(regexpStr string, handler http.Handler, matchers ...matcherFunc) routeEntry {
 	method := "GET"
-	handler = u.observabilityMiddlewares(handler, method, regexpStr)
+	handler = u.observabilityMiddlewares(handler, method, regexpStr, nil)
 
 	return routeEntry{
 		method:   method,

workhorse/internal/upstream/upstream.go

@@ -239,5 +239,5 @@ func (u *upstream) updateGeoProxyFields(geoProxyURL *url.URL) {
 	geoProxyRoundTripper := roundtripper.NewBackendRoundTripper(u.geoProxyBackend, "", u.ProxyHeadersTimeout, u.DevelopmentMode)
 	geoProxyUpstream := proxypkg.NewProxy(u.geoProxyBackend, u.Version, geoProxyRoundTripper)
 	u.geoProxyCableRoute = u.wsRoute(`^/-/cable\z`, geoProxyUpstream)
-	u.geoProxyRoute = u.route("", "", geoProxyUpstream)
+	u.geoProxyRoute = u.route("", "", geoProxyUpstream, withGeoProxy())
 }