Merge branch 'cleanup-leagcy-artifact-migration' into 'master'

Cleanup legacy artifact background migration

Closes #46866

See merge request gitlab-org/gitlab-ce!24144

changelogs/unreleased/cleanup-leagcy-artifact-migration.yml

+---
+title: Cleanup legacy artifact background migration
+merge_request: 24144
+author:
+type: other

db/migrate/20190104182041_cleanup_legacy_artifact_migration.rb

+# frozen_string_literal: true
+
+class CleanupLegacyArtifactMigration < ActiveRecord::Migration[5.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  class Build < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'ci_builds'
+    self.inheritance_column = :_type_disabled
+
+    scope :with_legacy_artifacts, -> { where("artifacts_file <> ''") }
+  end
+
+  def up
+    Gitlab::BackgroundMigration.steal('MigrateLegacyArtifacts')
+
+    CleanupLegacyArtifactMigration::Build
+      .with_legacy_artifacts
+      .each_batch(of: 100) do |batch|
+      range = batch.pluck('MIN(id)', 'MAX(id)').first
+
+      Gitlab::BackgroundMigration::MigrateLegacyArtifacts.new.perform(*range)
+    end
+  end
+
+  def down
+    # no-op
+  end
+end

db/migrate/20190108192941_remove_partial_index_from_ci_builds_artifacts_file.rb

+# frozen_string_literal: true
+
+class RemovePartialIndexFromCiBuildsArtifactsFile < ActiveRecord::Migration[5.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  INDEX_NAME = 'partial_index_ci_builds_on_id_with_legacy_artifacts'.freeze
+
+  disable_ddl_transaction!
+
+  def up
+    remove_concurrent_index_by_name(:ci_builds, INDEX_NAME)
+  end
+
+  def down
+    add_concurrent_index(:ci_builds, :id, where: "artifacts_file <> ''", name: INDEX_NAME)
+  end
+end

db/schema.rb

@@ -354,7 +354,6 @@ ActiveRecord::Schema.define(version: 20190115054216) do
     t.index ["commit_id", "status", "type"], name: "index_ci_builds_on_commit_id_and_status_and_type", using: :btree
     t.index ["commit_id", "type", "name", "ref"], name: "index_ci_builds_on_commit_id_and_type_and_name_and_ref", using: :btree
     t.index ["commit_id", "type", "ref"], name: "index_ci_builds_on_commit_id_and_type_and_ref", using: :btree
-    t.index ["id"], name: "partial_index_ci_builds_on_id_with_legacy_artifacts", where: "(artifacts_file <> ''::text)", using: :btree
     t.index ["project_id", "id"], name: "index_ci_builds_on_project_id_and_id", using: :btree
     t.index ["project_id", "status"], name: "index_ci_builds_project_id_and_status_for_live_jobs_partial2", where: "(((type)::text = 'Ci::Build'::text) AND ((status)::text = ANY (ARRAY[('running'::character varying)::text, ('pending'::character varying)::text, ('created'::character varying)::text])))", using: :btree
     t.index ["protected"], name: "index_ci_builds_on_protected", using: :btree

spec/migrations/cleanup_legacy_artifact_migration_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require Rails.root.join('db', 'migrate', '20190104182041_cleanup_legacy_artifact_migration.rb')
+
+describe CleanupLegacyArtifactMigration, :migration, :sidekiq, :redis do
+  let(:migration) { spy('migration') }
+
+  context 'when still legacy artifacts exist' do
+    let(:namespaces) { table(:namespaces) }
+    let(:projects) { table(:projects) }
+    let(:pipelines) { table(:ci_pipelines) }
+    let(:jobs) { table(:ci_builds) }
+    let(:job_artifacts) { table(:ci_job_artifacts) }
+    let(:namespace) { namespaces.create!(name: 'gitlab', path: 'gitlab-org') }
+    let(:project) { projects.create!(name: 'gitlab', path: 'gitlab-ce', namespace_id: namespace.id) }
+    let(:pipeline) { pipelines.create!(project_id: project.id, ref: 'master', sha: 'adf43c3a') }
+    let(:archive_file_type) { Gitlab::BackgroundMigration::MigrateLegacyArtifacts::ARCHIVE_FILE_TYPE }
+    let(:metadata_file_type) { Gitlab::BackgroundMigration::MigrateLegacyArtifacts::METADATA_FILE_TYPE }
+    let(:local_store) { ::ObjectStorage::Store::LOCAL }
+    let(:remote_store) { ::ObjectStorage::Store::REMOTE }
+    let(:legacy_location) { Gitlab::BackgroundMigration::MigrateLegacyArtifacts::LEGACY_PATH_FILE_LOCATION }
+
+    before do
+      jobs.create!(id: 1, commit_id: pipeline.id, project_id: project.id, status: :success, artifacts_file: 'archive.zip')
+      jobs.create!(id: 2, commit_id: pipeline.id, project_id: project.id, status: :failed, artifacts_metadata: 'metadata.gz')
+      jobs.create!(id: 3, commit_id: pipeline.id, project_id: project.id, status: :failed, artifacts_file: 'archive.zip', artifacts_metadata: 'metadata.gz')
+      jobs.create!(id: 4, commit_id: pipeline.id, project_id: project.id, status: :running)
+      jobs.create!(id: 5, commit_id: pipeline.id, project_id: project.id, status: :success, artifacts_file: 'archive.zip', artifacts_file_store: remote_store, artifacts_metadata: 'metadata.gz')
+      jobs.create!(id: 6, commit_id: pipeline.id, project_id: project.id, status: :failed, artifacts_file: 'archive.zip', artifacts_metadata: 'metadata.gz')
+    end
+
+    it 'steals sidekiq jobs from MigrateLegacyArtifacts background migration' do
+      expect(Gitlab::BackgroundMigration).to receive(:steal).with('MigrateLegacyArtifacts')
+
+      migrate!
+    end
+
+    it 'migrates legacy artifacts to ci_job_artifacts table' do
+      migrate!
+
+      expect(job_artifacts.order(:job_id, :file_type).pluck('project_id, job_id, file_type, file_store, size, expire_at, file, file_sha256, file_location'))
+        .to eq([[project.id, 1, archive_file_type, local_store, nil, nil, 'archive.zip', nil, legacy_location],
+                [project.id, 3, archive_file_type, local_store, nil, nil, 'archive.zip', nil, legacy_location],
+                [project.id, 3, metadata_file_type, local_store, nil, nil, 'metadata.gz', nil, legacy_location],
+                [project.id, 5, archive_file_type, remote_store, nil, nil, 'archive.zip', nil, legacy_location],
+                [project.id, 5, metadata_file_type, local_store, nil, nil, 'metadata.gz', nil, legacy_location],
+                [project.id, 6, archive_file_type, local_store, nil, nil, 'archive.zip', nil, legacy_location],
+                [project.id, 6, metadata_file_type, local_store, nil, nil, 'metadata.gz', nil, legacy_location]])
+    end
+  end
+end