Merge branch 'feature/gb/add-ci-artifacts-exclude-config' into 'master'

Add support for artifacts/exclude CI configuration

See merge request gitlab-org/gitlab!30708

app/models/ci/build.rb

@@ -25,7 +25,8 @@ module Ci
 
     RUNNER_FEATURES = {
       upload_multiple_artifacts: -> (build) { build.publishes_artifacts_reports? },
-      refspecs: -> (build) { build.merge_request_ref? }
+      refspecs: -> (build) { build.merge_request_ref? },
+      artifacts_exclude: -> (build) { build.supports_artifacts_exclude? }
     }.freeze
 
     DEFAULT_RETRIES = {
@@ -920,6 +921,11 @@ module Ci
         failure_reason: :data_integrity_failure)
     end
 
+    def supports_artifacts_exclude?
+      options&.dig(:artifacts, :exclude)&.any? &&
+        Gitlab::Ci::Features.artifacts_exclude_enabled?
+    end
+
     def degradation_threshold
       var = yaml_variables.find { |v| v[:key] == DEGRADATION_THRESHOLD_VARIABLE_NAME }
       var[:value]&.to_i if var

app/presenters/ci/build_runner_presenter.rb

@@ -52,7 +52,7 @@ module Ci
     def create_archive(artifacts)
       return unless artifacts[:untracked] || artifacts[:paths]
 
-      {
+      archive = {
         artifact_type: :archive,
         artifact_format: :zip,
         name: artifacts[:name],
@@ -61,6 +61,12 @@ module Ci
         when: artifacts[:when],
         expire_in: artifacts[:expire_in]
       }
+
+      if artifacts.dig(:exclude).present? && ::Gitlab::Ci::Features.artifacts_exclude_enabled?
+        archive.merge(exclude: artifacts[:exclude])
+      else
+        archive
+      end
     end
 
     def create_reports(reports, expire_in:)

lib/api/entities/job_request/artifacts.rb

@@ -7,6 +7,7 @@ module API
         expose :name
         expose :untracked
         expose :paths
+        expose :exclude, expose_nil: false
         expose :when
         expose :expire_in
         expose :artifact_type

lib/gitlab/ci/config/entry/artifacts.rb

@@ -12,7 +12,7 @@ module Gitlab
           include ::Gitlab::Config::Entry::Validatable
           include ::Gitlab::Config::Entry::Attributable
 
-          ALLOWED_KEYS = %i[name untracked paths reports when expire_in expose_as].freeze
+          ALLOWED_KEYS = %i[name untracked paths reports when expire_in expose_as exclude].freeze
           EXPOSE_AS_REGEX = /\A\w[-\w ]*\z/.freeze
           EXPOSE_AS_ERROR_MESSAGE = "can contain only letters, digits, '-', '_' and spaces"
 
@@ -35,6 +35,8 @@ module Gitlab
               }, if: :expose_as_present?
               validates :expose_as, type: String, length: { maximum: 100 }, if: :expose_as_present?
               validates :expose_as, format: { with: EXPOSE_AS_REGEX, message: EXPOSE_AS_ERROR_MESSAGE }, if: :expose_as_present?
+              validates :exclude, array_of_strings: true, if: :exclude_enabled?
+              validates :exclude, absence: { message: 'feature is disabled' }, unless: :exclude_enabled?
               validates :reports, type: Hash
               validates :when,
                 inclusion: { in: %w[on_success on_failure always],
@@ -57,6 +59,10 @@ module Gitlab
 
             !@config[:expose_as].nil?
           end
+
+          def exclude_enabled?
+            ::Gitlab::Ci::Features.artifacts_exclude_enabled?
+          end
         end
       end
     end

lib/gitlab/ci/features.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    ##
+    # Ci::Features is a class that aggregates all CI/CD feature flags in one place.
+    #
+    module Features
+      def self.artifacts_exclude_enabled?
+        ::Feature.enabled?(:ci_artifacts_exclude, default_enabled: false)
+      end
+    end
+  end
+end

spec/lib/gitlab/ci/config/entry/artifacts_spec.rb

@@ -124,5 +124,55 @@ describe Gitlab::Ci::Config::Entry::Artifacts do
         end
       end
     end
+
+    describe 'excluded artifacts' do
+      context 'when configuration is valid and the feature is enabled' do
+        before do
+          stub_feature_flags(ci_artifacts_exclude: true)
+        end
+
+        context 'when configuration is valid' do
+          let(:config) { { untracked: true, exclude: ['some/directory/'] } }
+
+          it 'correctly parses the configuration' do
+            expect(entry).to be_valid
+            expect(entry.value).to eq config
+          end
+        end
+
+        context 'when configuration is not valid' do
+          let(:config) { { untracked: true, exclude: 1234 } }
+
+          it 'returns an error' do
+            expect(entry).not_to be_valid
+            expect(entry.errors)
+              .to include 'artifacts exclude should be an array of strings'
+          end
+        end
+      end
+
+      context 'when artifacts/exclude feature is disabled' do
+        before do
+          stub_feature_flags(ci_artifacts_exclude: false)
+        end
+
+        context 'when configuration has been provided' do
+          let(:config) { { untracked: true, exclude: ['some/directory/'] } }
+
+          it 'returns an error' do
+            expect(entry).not_to be_valid
+            expect(entry.errors).to include 'artifacts exclude feature is disabled'
+          end
+        end
+
+        context 'when configuration is not present' do
+          let(:config) { { untracked: true } }
+
+          it 'is a valid configuration' do
+            expect(entry).to be_valid
+          end
+        end
+      end
+    end
   end
 end

spec/lib/gitlab/ci/yaml_processor_spec.rb

@@ -1364,6 +1364,24 @@ module Gitlab
 
           expect { described_class.new(config) }.to raise_error(described_class::ValidationError)
         end
+
+        it 'populates a build options with complete artifacts configuration' do
+          stub_feature_flags(ci_artifacts_exclude: true)
+
+          config = <<~YAML
+            test:
+              script: echo "Hello World"
+              artifacts:
+                paths:
+                  - my/test
+                exclude:
+                  - my/test/something
+          YAML
+
+          attributes = Gitlab::Ci::YamlProcessor.new(config).build_attributes('test')
+
+          expect(attributes.dig(*%i[options artifacts exclude])).to eq(%w[my/test/something])
+        end
       end
 
       describe "release" do

spec/models/ci/build_spec.rb

@@ -4088,6 +4088,28 @@ describe Ci::Build do
 
       it { is_expected.to include(:upload_multiple_artifacts) }
     end
+
+    context 'when artifacts exclude is defined and the is feature enabled' do
+      let(:options) do
+        { artifacts: { exclude: %w[something] } }
+      end
+
+      context 'when a feature flag is enabled' do
+        before do
+          stub_feature_flags(ci_artifacts_exclude: true)
+        end
+
+        it { is_expected.to include(:artifacts_exclude) }
+      end
+
+      context 'when a feature flag is disabled' do
+        before do
+          stub_feature_flags(ci_artifacts_exclude: false)
+        end
+
+        it { is_expected.not_to include(:artifacts_exclude) }
+      end
+    end
   end
 
   describe '#supported_runner?' do

spec/presenters/ci/build_runner_presenter_spec.rb

@@ -38,6 +38,47 @@ describe Ci::BuildRunnerPresenter do
           expect(presenter.artifacts).to be_empty
         end
       end
+
+      context 'when artifacts exclude is defined' do
+        let(:build) do
+          create(:ci_build, options: { artifacts: { paths: %w[abc], exclude: %w[cde] } })
+        end
+
+        context 'when the feature is enabled' do
+          before do
+            stub_feature_flags(ci_artifacts_exclude: true)
+          end
+
+          it 'includes the list of excluded paths' do
+            expect(presenter.artifacts.first).to include(
+              artifact_type: :archive,
+              artifact_format: :zip,
+              paths: %w[abc],
+              exclude: %w[cde]
+            )
+          end
+        end
+
+        context 'when the feature is disabled' do
+          before do
+            stub_feature_flags(ci_artifacts_exclude: false)
+          end
+
+          it 'does not include the list of excluded paths' do
+            expect(presenter.artifacts.first).not_to have_key(:exclude)
+          end
+        end
+      end
+
+      context 'when artifacts exclude is not defined' do
+        let(:build) do
+          create(:ci_build, options: { artifacts: { paths: %w[abc] } })
+        end
+
+        it 'does not include an empty list of excluded paths' do
+          expect(presenter.artifacts.first).not_to have_key(:exclude)
+        end
+      end
     end
 
     context "with reports" do

spec/requests/api/runner_spec.rb

@@ -998,6 +998,53 @@ describe API::Runner, :clean_gitlab_redis_shared_state do
           end
         end
 
+        describe 'a job with excluded artifacts' do
+          context 'when excluded paths are defined' do
+            let(:job) do
+              create(:ci_build, pipeline: pipeline, token: 'test-job-token', name: 'test',
+                                stage: 'deploy', stage_idx: 1,
+                                options: { artifacts: { paths: ['abc'], exclude: ['cde'] } })
+            end
+
+            context 'when a runner supports this feature' do
+              it 'exposes excluded paths when the feature is enabled' do
+                stub_feature_flags(ci_artifacts_exclude: true)
+
+                request_job info: { features: { artifacts_exclude: true } }
+
+                expect(response).to have_gitlab_http_status(:created)
+                expect(json_response.dig('artifacts').first).to include('exclude' => ['cde'])
+              end
+
+              it 'does not expose excluded paths when the feature is disabled' do
+                stub_feature_flags(ci_artifacts_exclude: false)
+
+                request_job info: { features: { artifacts_exclude: true } }
+
+                expect(response).to have_gitlab_http_status(:created)
+                expect(json_response.dig('artifacts').first).not_to have_key('exclude')
+              end
+            end
+
+            context 'when a runner does not support this feature' do
+              it 'does not expose the build at all' do
+                stub_feature_flags(ci_artifacts_exclude: true)
+
+                request_job
+
+                expect(response).to have_gitlab_http_status(:no_content)
+              end
+            end
+          end
+
+          it 'does not expose excluded paths when these are empty' do
+            request_job
+
+            expect(response).to have_gitlab_http_status(:created)
+            expect(json_response.dig('artifacts').first).not_to have_key('exclude')
+          end
+        end
+
         def request_job(token = runner.token, **params)
           new_params = params.merge(token: token, last_update: last_update)
           post api('/jobs/request'), params: new_params, headers: { 'User-Agent' => user_agent }