Merge branch 'sh-s3-workhorse-custom-endpoint' into 'master'

Ensure Workhorse AWS endpoint is only used for S3

See merge request gitlab-org/gitlab!83695

workhorse/internal/upload/destination/objectstore/s3_session.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/endpoints"
 	"github.com/aws/aws-sdk-go/aws/session"
 
 	"gitlab.com/gitlab-org/gitlab/workhorse/internal/config"
@@ -70,7 +71,23 @@ func setupS3Session(s3Credentials config.S3Credentials, s3Config config.S3Config
 	}
 
 	if s3Config.Endpoint != "" {
-		cfg.Endpoint = aws.String(s3Config.Endpoint)
+		// The administrator has configured an S3 endpoint override,
+		// e.g. to make use of S3 IPv6 support or S3 FIPS mode. We
+		// need to configure a custom resolver to make sure that
+		// the custom endpoint is only used for S3 API calls, and not
+		// for STS API calls.
+		s3CustomResolver := func(service, region string, optFns ...func(*endpoints.Options)) (endpoints.ResolvedEndpoint, error) {
+			if service == endpoints.S3ServiceID {
+				return endpoints.ResolvedEndpoint{
+					URL:           s3Config.Endpoint,
+					SigningRegion: region,
+				}, nil
+			}
+
+			return endpoints.DefaultResolver().EndpointFor(service, region, optFns...)
+		}
+
+		cfg.EndpointResolver = endpoints.ResolverFunc(s3CustomResolver)
 	}
 
 	sess, err := session.NewSession(cfg)

workhorse/internal/upload/destination/objectstore/s3_session_test.go

@@ -5,6 +5,7 @@ import (
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/endpoints"
 	"github.com/stretchr/testify/require"
 
 	"gitlab.com/gitlab-org/gitlab/workhorse/internal/config"
@@ -17,7 +18,9 @@ func TestS3SessionSetup(t *testing.T) {
 	sess, err := setupS3Session(credentials, cfg)
 	require.NoError(t, err)
 
-	require.Equal(t, aws.StringValue(sess.Config.Region), "us-west-1")
+	s3Config := sess.ClientConfig(endpoints.S3ServiceID)
+	require.Equal(t, "https://s3.us-west-1.amazonaws.com", s3Config.Endpoint)
+	require.Equal(t, "us-west-1", s3Config.SigningRegion)
 	require.True(t, aws.BoolValue(sess.Config.S3ForcePathStyle))
 
 	require.Equal(t, len(sessionCache.sessions), 1)
@@ -29,6 +32,26 @@ func TestS3SessionSetup(t *testing.T) {
 	ResetS3Session(cfg)
 }
 
+func TestS3SessionEndpointSetup(t *testing.T) {
+	credentials := config.S3Credentials{}
+	const customS3Endpoint = "https://example.com"
+	const region = "us-west-2"
+	cfg := config.S3Config{Region: region, PathStyle: true, Endpoint: customS3Endpoint}
+
+	sess, err := setupS3Session(credentials, cfg)
+	require.NoError(t, err)
+
+	// ClientConfig is what is ultimately used by an S3 client
+	s3Config := sess.ClientConfig(endpoints.S3ServiceID)
+	require.Equal(t, customS3Endpoint, s3Config.Endpoint)
+	require.Equal(t, region, s3Config.SigningRegion)
+
+	stsConfig := sess.ClientConfig(endpoints.StsServiceID)
+	require.Equal(t, "https://sts.amazonaws.com", stsConfig.Endpoint, "STS should use default endpoint")
+
+	ResetS3Session(cfg)
+}
+
 func TestS3SessionExpiry(t *testing.T) {
 	credentials := config.S3Credentials{}
 	cfg := config.S3Config{Region: "us-west-1", PathStyle: true}