Support S3 server side encryption in CI cloud native job logs

Previously if S3 server side encryption were required to store CI
artifacts via a bucket policy, the final migration to object storage
would fail, leaving the job log unmigrated.

To fix this, we need to send the right headers
(e.g. `x-amz-server-side-encryption`) to the object storage backend via
Fog.

Relates to https://gitlab.com/gitlab-org/gitlab/-/issues/273498

app/models/ci/build_trace_chunks/fog.rb

@@ -4,7 +4,7 @@ module Ci
   module BuildTraceChunks
     class Fog
       def available?
-        object_store.enabled
+        object_store_config.enabled?
       end
 
       def data(model)
@@ -14,11 +14,7 @@ module Ci
       end
 
       def set_data(model, new_data)
-        # TODO: Support AWS S3 server side encryption
-        files.create({
-          key: key(model),
-          body: new_data
-        })
+        files.create(create_attributes(model, new_data))
       end
 
       def append_data(model, new_data, offset)
@@ -57,6 +53,13 @@ module Ci
         key_raw(model.build_id, model.chunk_index)
       end
 
+      def create_attributes(model, new_data)
+        {
+          key: key(model),
+          body: new_data
+        }.merge(object_store_config.fog_attributes)
+      end
+
       def key_raw(build_id, chunk_index)
         "tmp/builds/#{build_id.to_i}/chunks/#{chunk_index.to_i}.log"
       end
@@ -64,13 +67,13 @@ module Ci
       def bucket_name
         return unless available?
 
-        object_store.remote_directory
+        object_store_config.bucket
       end
 
       def connection
         return unless available?
 
-        @connection ||= ::Fog::Storage.new(object_store.connection.to_hash.deep_symbolize_keys)
+        @connection ||= ::Fog::Storage.new(object_store_config.credentials)
       end
 
       def fog_directory
@@ -81,9 +84,13 @@ module Ci
         @files ||= fog_directory.files
       end
 
-      def object_store
+      def object_store_raw_config
         Gitlab.config.artifacts.object_store
       end
+
+      def object_store_config
+        @object_store_config ||= ::ObjectStorage::Config.new(object_store_raw_config)
+      end
     end
   end
 end

changelogs/unreleased/sh-support-sse-encryption-ci-live-trace.yml

+---
+title: Support S3 server side encryption in CI cloud native job logs
+merge_request: 47536
+author:
+type: fixed

lib/object_storage/config.rb

@@ -93,6 +93,11 @@ module ObjectStorage
 
     private
 
+    # This returns a Hash of HTTP encryption headers to send along to S3.
+    #
+    # They can also be passed in as Fog::AWS::Storage::File attributes, since there
+    # are aliases defined for them:
+    # https://github.com/fog/fog-aws/blob/ab288f29a0974d64fd8290db41080e5578be9651/lib/fog/aws/models/storage/file.rb#L24-L25
     def aws_server_side_encryption_headers
       {
         'x-amz-server-side-encryption' => server_side_encryption,

spec/models/ci/build_trace_chunks/fog_spec.rb

@@ -74,6 +74,31 @@ RSpec.describe Ci::BuildTraceChunks::Fog do
 
         expect(data_store.data(model)).to eq new_data
       end
+
+      context 'when S3 server side encryption is enabled' do
+        before do
+          config = Gitlab.config.artifacts.object_store.to_h
+          config[:storage_options] = { server_side_encryption: 'AES256' }
+          allow(data_store).to receive(:object_store_raw_config).and_return(config)
+        end
+
+        it 'creates a file with attributes' do
+          expect_next_instance_of(Fog::AWS::Storage::Files) do |files|
+            expect(files).to receive(:create).with(
+              hash_including(
+                key: anything,
+                body: new_data,
+                'x-amz-server-side-encryption' => 'AES256')
+            ).and_call_original
+          end
+
+          expect(data_store.data(model)).to be_nil
+
+          data_store.set_data(model, new_data)
+
+          expect(data_store.data(model)).to eq new_data
+        end
+      end
     end
   end
 

spec/support/helpers/stub_object_storage.rb

@@ -22,6 +22,16 @@ module StubObjectStorage
         background_upload: false,
         direct_upload: false
   )
+    new_config = config.to_h.deep_symbolize_keys.merge({
+      enabled: enabled,
+      proxy_download: proxy_download,
+      background_upload: background_upload,
+      direct_upload: direct_upload
+    })
+
+    # Needed for ObjectStorage::Config compatibility
+    allow(config).to receive(:to_hash).and_return(new_config)
+    allow(config).to receive(:to_h).and_return(new_config)
     allow(config).to receive(:enabled) { enabled }
     allow(config).to receive(:proxy_download) { proxy_download }
     allow(config).to receive(:background_upload) { background_upload }