Commit 71c61c33 authored by Stan Hu's avatar Stan Hu

Support S3 server side encryption in CI cloud native job logs

Previously if S3 server side encryption were required to store CI
artifacts via a bucket policy, the final migration to object storage
would fail, leaving the job log unmigrated.

To fix this, we need to send the right headers
(e.g. `x-amz-server-side-encryption`) to the object storage backend via
Fog.

Relates to https://gitlab.com/gitlab-org/gitlab/-/issues/273498
parent 61f92c32
......@@ -4,7 +4,7 @@ module Ci
module BuildTraceChunks
class Fog
def available?
object_store.enabled
object_store_config.enabled?
end
def data(model)
......@@ -14,11 +14,7 @@ module Ci
end
def set_data(model, new_data)
# TODO: Support AWS S3 server side encryption
files.create({
key: key(model),
body: new_data
})
files.create(create_attributes(model, new_data))
end
def append_data(model, new_data, offset)
......@@ -57,6 +53,13 @@ module Ci
key_raw(model.build_id, model.chunk_index)
end
def create_attributes(model, new_data)
{
key: key(model),
body: new_data
}.merge(object_store_config.fog_attributes)
end
def key_raw(build_id, chunk_index)
"tmp/builds/#{build_id.to_i}/chunks/#{chunk_index.to_i}.log"
end
......@@ -64,13 +67,13 @@ module Ci
def bucket_name
return unless available?
object_store.remote_directory
object_store_config.bucket
end
def connection
return unless available?
@connection ||= ::Fog::Storage.new(object_store.connection.to_hash.deep_symbolize_keys)
@connection ||= ::Fog::Storage.new(object_store_config.credentials)
end
def fog_directory
......@@ -81,9 +84,13 @@ module Ci
@files ||= fog_directory.files
end
def object_store
def object_store_raw_config
Gitlab.config.artifacts.object_store
end
def object_store_config
@object_store_config ||= ::ObjectStorage::Config.new(object_store_raw_config)
end
end
end
end
---
title: Support S3 server side encryption in CI cloud native job logs
merge_request: 47536
author:
type: fixed
......@@ -93,6 +93,11 @@ module ObjectStorage
private
# This returns a Hash of HTTP encryption headers to send along to S3.
#
# They can also be passed in as Fog::AWS::Storage::File attributes, since there
# are aliases defined for them:
# https://github.com/fog/fog-aws/blob/ab288f29a0974d64fd8290db41080e5578be9651/lib/fog/aws/models/storage/file.rb#L24-L25
def aws_server_side_encryption_headers
{
'x-amz-server-side-encryption' => server_side_encryption,
......
......@@ -74,6 +74,31 @@ RSpec.describe Ci::BuildTraceChunks::Fog do
expect(data_store.data(model)).to eq new_data
end
context 'when S3 server side encryption is enabled' do
before do
config = Gitlab.config.artifacts.object_store.to_h
config[:storage_options] = { server_side_encryption: 'AES256' }
allow(data_store).to receive(:object_store_raw_config).and_return(config)
end
it 'creates a file with attributes' do
expect_next_instance_of(Fog::AWS::Storage::Files) do |files|
expect(files).to receive(:create).with(
hash_including(
key: anything,
body: new_data,
'x-amz-server-side-encryption' => 'AES256')
).and_call_original
end
expect(data_store.data(model)).to be_nil
data_store.set_data(model, new_data)
expect(data_store.data(model)).to eq new_data
end
end
end
end
......
......@@ -22,6 +22,16 @@ module StubObjectStorage
background_upload: false,
direct_upload: false
)
new_config = config.to_h.deep_symbolize_keys.merge({
enabled: enabled,
proxy_download: proxy_download,
background_upload: background_upload,
direct_upload: direct_upload
})
# Needed for ObjectStorage::Config compatibility
allow(config).to receive(:to_hash).and_return(new_config)
allow(config).to receive(:to_h).and_return(new_config)
allow(config).to receive(:enabled) { enabled }
allow(config).to receive(:proxy_download) { proxy_download }
allow(config).to receive(:background_upload) { background_upload }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment