Merge branch 'ee-43949-verify-job-artifacts' into 'master'

Implement foreground verification of CI artifacts (EE)

See merge request gitlab-org/gitlab-ee!4863

changelogs/unreleased/43949-verify-job-artifacts.yml

+---
+title: Implement foreground verification of CI artifacts
+merge_request: 17578
+author:
+type: added

doc/administration/raketasks/check.md

@@ -84,12 +84,14 @@ checks using those checksums can be run. These checks also detect missing files.
 
 Currently, integrity checks are supported for the following types of file:
 
+* CI artifacts
 * LFS objects
 * User uploads
 
 **Omnibus Installation**
 
 ```
+sudo gitlab-rake gitlab:artifacts:check
 sudo gitlab-rake gitlab:lfs:check
 sudo gitlab-rake gitlab:uploads:check
 ```
@@ -97,6 +99,7 @@ sudo gitlab-rake gitlab:uploads:check
 **Source Installation**
 
 ```bash
+sudo -u git -H bundle exec rake gitlab:artifacts:check RAILS_ENV=production
 sudo -u git -H bundle exec rake gitlab:lfs:check RAILS_ENV=production
 sudo -u git -H bundle exec rake gitlab:uploads:check RAILS_ENV=production
 ```
@@ -112,6 +115,7 @@ Variable  | Type    | Description
 `VERBOSE` | boolean | Causes failures to be listed individually, rather than being summarized.
 
 ```bash
+sudo gitlab-rake gitlab:artifacts:check BATCH=100 ID_FROM=50 ID_TO=250
 sudo gitlab-rake gitlab:lfs:check BATCH=100 ID_FROM=50 ID_TO=250
 sudo gitlab-rake gitlab:uploads:check BATCH=100 ID_FROM=50 ID_TO=250
 ```

ee/lib/ee/gitlab/verify/job_artifacts.rb

+module EE
+  module Gitlab
+    module Verify
+      module JobArtifacts
+        extend ::Gitlab::Utils::Override
+
+        private
+
+        override :relation
+        def relation
+          super.with_files_stored_locally
+        end
+      end
+    end
+  end
+end

ee/spec/lib/ee/gitlab/verify/job_artifacts_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Verify::JobArtifacts do
+  before do
+    stub_artifacts_object_storage
+  end
+
+  it 'skips LFS objects in object storage' do
+    local_failure = create(:ci_job_artifact)
+    create(:ci_job_artifact, :remote_store)
+
+    failures = {}
+    described_class.new(batch_size: 10).run_batches { |_, failed| failures.merge!(failed) }
+
+    expect(failures.keys).to contain_exactly(local_failure)
+  end
+end

ee/spec/tasks/gitlab/artifacts_rake_spec.rb → ee/spec/tasks/gitlab/artifacts/migrate_rake_spec.rb

@@ -2,7 +2,7 @@ require 'rake_helper'
 
 describe 'gitlab:artifacts namespace rake task' do
   before(:context) do
-    Rake.application.rake_require 'tasks/gitlab/artifacts'
+    Rake.application.rake_require 'tasks/gitlab/artifacts/migrate'
   end
 
   let(:object_storage_enabled) { false }

lib/gitlab/verify/job_artifacts.rb

+module Gitlab
+  module Verify
+    class JobArtifacts < BatchVerifier
+      prepend ::EE::Gitlab::Verify::JobArtifacts
+
+      def name
+        'Job artifacts'
+      end
+
+      def describe(object)
+        "Job artifact: #{object.id}"
+      end
+
+      private
+
+      def relation
+        ::Ci::JobArtifact.all
+      end
+
+      def expected_checksum(artifact)
+        artifact.file_sha256
+      end
+
+      def actual_checksum(artifact)
+        Digest::SHA256.file(artifact.file.path).hexdigest
+      end
+    end
+  end
+end

lib/tasks/gitlab/artifacts/check.rake

+namespace :gitlab do
+  namespace :artifacts do
+    desc 'GitLab | Artifacts | Check integrity of uploaded job artifacts'
+    task check: :environment do
+      Gitlab::Verify::RakeTask.run!(Gitlab::Verify::JobArtifacts)
+    end
+  end
+end

spec/factories/ci/job_artifacts.rb

@@ -39,5 +39,11 @@ FactoryBot.define do
           Rails.root.join('spec/fixtures/trace/sample_trace'), 'text/plain')
       end
     end
+
+    trait :correct_checksum do
+      after(:build) do |artifact, evaluator|
+        artifact.file_sha256 = Digest::SHA256.file(artifact.file.path).hexdigest
+      end
+    end
   end
 end

spec/lib/gitlab/verify/job_artifacts_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Verify::JobArtifacts do
+  include GitlabVerifyHelpers
+
+  it_behaves_like 'Gitlab::Verify::BatchVerifier subclass' do
+    let!(:objects) { create_list(:ci_job_artifact, 3, :archive) }
+  end
+
+  describe '#run_batches' do
+    let(:failures) { collect_failures }
+    let(:failure) { failures[artifact] }
+
+    let!(:artifact) { create(:ci_job_artifact, :archive, :correct_checksum) }
+
+    it 'passes artifacts with the correct file' do
+      expect(failures).to eq({})
+    end
+
+    it 'fails artifacts with a missing file' do
+      FileUtils.rm_f(artifact.file.path)
+
+      expect(failures.keys).to contain_exactly(artifact)
+      expect(failure).to be_a(Errno::ENOENT)
+      expect(failure.to_s).to include(artifact.file.path)
+    end
+
+    it 'fails artifacts with a mismatched checksum' do
+      File.truncate(artifact.file.path, 0)
+
+      expect(failures.keys).to contain_exactly(artifact)
+      expect(failure.to_s).to include('Checksum mismatch')
+    end
+  end
+end

spec/tasks/gitlab/artifacts/check_rake_spec.rb

+require 'rake_helper'
+
+describe 'gitlab:artifacts rake tasks' do
+  describe 'check' do
+    let!(:artifact) { create(:ci_job_artifact, :archive, :correct_checksum) }
+
+    before do
+      Rake.application.rake_require('tasks/gitlab/artifacts/check')
+      stub_env('VERBOSE' => 'true')
+    end
+
+    it 'outputs the integrity check for each batch' do
+      expect { run_rake_task('gitlab:artifacts:check') }.to output(/Failures: 0/).to_stdout
+    end
+
+    it 'errors out about missing files on the file system' do
+      FileUtils.rm_f(artifact.file.path)
+
+      expect { run_rake_task('gitlab:artifacts:check') }.to output(/No such file.*#{Regexp.quote(artifact.file.path)}/).to_stdout
+    end
+
+    it 'errors out about invalid checksum' do
+      artifact.update_column(:file_sha256, 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855')
+
+      expect { run_rake_task('gitlab:artifacts:check') }.to output(/Checksum mismatch/).to_stdout
+    end
+
+    it 'errors out about missing checksum' do
+      artifact.update_column(:file_sha256, nil)
+
+      expect { run_rake_task('gitlab:artifacts:check') }.to output(/Checksum missing/).to_stdout
+    end
+  end
+end