Merge branch '327078-refactor-migration-rakes' into 'master'

Refactor Pages deployment and  artifact migrations

See merge request gitlab-org/gitlab!60597

lib/gitlab/ci/artifacts/migration_helper.rb

-# frozen_string_literal: true
-
-module Gitlab
-  module Ci
-    module Artifacts
-      class MigrationHelper
-        def migrate_to_remote_storage(&block)
-          artifacts = ::Ci::JobArtifact.with_files_stored_locally
-          migrate(artifacts, ObjectStorage::Store::REMOTE, &block)
-        end
-
-        def migrate_to_local_storage(&block)
-          artifacts = ::Ci::JobArtifact.with_files_stored_remotely
-          migrate(artifacts, ObjectStorage::Store::LOCAL, &block)
-        end
-
-        private
-
-        def batch_size
-          ENV.fetch('MIGRATION_BATCH_SIZE', 10).to_i
-        end
-
-        def migrate(artifacts, store, &block)
-          artifacts.find_each(batch_size: batch_size) do |artifact| # rubocop:disable CodeReuse/ActiveRecord
-            artifact.file.migrate!(store)
-
-            yield artifact if block
-          rescue StandardError => e
-            raise StandardError.new("Failed to transfer artifact of type #{artifact.file_type} and ID #{artifact.id} with error: #{e.message}")
-          end
-        end
-      end
-    end
-  end
-end

lib/gitlab/local_and_remote_storage_migration/artifact_migrater.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module LocalAndRemoteStorageMigration
+    class ArtifactMigrater < Gitlab::LocalAndRemoteStorageMigration::BaseMigrater
+      private
+
+      def items_with_files_stored_locally
+        ::Ci::JobArtifact.with_files_stored_locally
+      end
+
+      def items_with_files_stored_remotely
+        ::Ci::JobArtifact.with_files_stored_remotely
+      end
+    end
+  end
+end

lib/gitlab/pages/migration_helper.rb → lib/gitlab/local_and_remote_storage_migration/base_migrater.rb

 # frozen_string_literal: true
 
 module Gitlab
-  module Pages
-    class MigrationHelper
+  module LocalAndRemoteStorageMigration
+    class BaseMigrater
       def initialize(logger = nil)
         @logger = logger
       end
 
       def migrate_to_remote_storage
-        deployments = ::PagesDeployment.with_files_stored_locally
-        migrate(deployments, ObjectStorage::Store::REMOTE)
+        logger.info('Starting transfer to remote storage')
+
+        migrate(items_with_files_stored_locally, ObjectStorage::Store::REMOTE)
       end
 
       def migrate_to_local_storage
-        deployments = ::PagesDeployment.with_files_stored_remotely
-        migrate(deployments, ObjectStorage::Store::LOCAL)
+        logger.info('Starting transfer to local storage')
+
+        migrate(items_with_files_stored_remotely, ObjectStorage::Store::LOCAL)
       end
 
       private
 
+      attr_reader :logger
+
       def batch_size
         ENV.fetch('MIGRATION_BATCH_SIZE', 10).to_i
       end
 
-      def migrate(deployments, store)
-        deployments.find_each(batch_size: batch_size) do |deployment| # rubocop:disable CodeReuse/ActiveRecord
-          deployment.file.migrate!(store)
+      def migrate(items, store)
+        items.find_each(batch_size: batch_size) do |item| # rubocop:disable CodeReuse/ActiveRecord
+          item.file.migrate!(store)
 
-          log_success(deployment, store)
+          log_success(item, store)
         rescue StandardError => e
-          log_error(e, deployment)
+          log_error(e, item)
         end
       end
 
-      def log_success(deployment, store)
-        logger.info("Transferred deployment ID #{deployment.id} of type #{deployment.file_type} with size #{deployment.size} to #{storage_label(store)} storage")
+      def log_success(item, store)
+        logger.info("Transferred #{item.class.name} ID #{item.id} of type #{item.file_type} with size #{item.size} to #{storage_label(store)} storage")
       end
 
-      def log_error(err, deployment)
-        logger.warn("Failed to transfer deployment of type #{deployment.file_type} and ID #{deployment.id} with error: #{err.message}")
+      def log_error(err, item)
+        logger.warn("Failed to transfer #{item.class.name} of type #{item.file_type} and ID #{item.id} with error: #{err.message}")
       end
 
       def storage_label(store)

lib/gitlab/local_and_remote_storage_migration/pages_deployment_migrater.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module LocalAndRemoteStorageMigration
+    class PagesDeploymentMigrater < Gitlab::LocalAndRemoteStorageMigration::BaseMigrater
+      private
+
+      def items_with_files_stored_locally
+        ::PagesDeployment.with_files_stored_locally
+      end
+
+      def items_with_files_stored_remotely
+        ::PagesDeployment.with_files_stored_remotely
+      end
+    end
+  end
+end

lib/tasks/gitlab/artifacts/migrate.rake

@@ -8,14 +8,11 @@ namespace :gitlab do
   namespace :artifacts do
     task migrate: :environment do
       logger = Logger.new(STDOUT)
-      logger.info('Starting transfer of artifacts to remote storage')
 
-      helper = Gitlab::Ci::Artifacts::MigrationHelper.new
+      helper = Gitlab::LocalAndRemoteStorageMigration::ArtifactMigrater.new(logger)
 
       begin
-        helper.migrate_to_remote_storage do |artifact|
-          logger.info("Transferred artifact ID #{artifact.id} of type #{artifact.file_type} with size #{artifact.size} to object storage")
-        end
+        helper.migrate_to_remote_storage
       rescue StandardError => e
         logger.error(e.message)
       end
@@ -23,14 +20,11 @@ namespace :gitlab do
 
     task migrate_to_local: :environment do
       logger = Logger.new(STDOUT)
-      logger.info('Starting transfer of artifacts to local storage')
 
-      helper = Gitlab::Ci::Artifacts::MigrationHelper.new
+      helper = Gitlab::LocalAndRemoteStorageMigration::ArtifactMigrater.new(logger)
 
       begin
-        helper.migrate_to_local_storage do |artifact|
-          logger.info("Transferred artifact ID #{artifact.id} of type #{artifact.file_type} with size #{artifact.size} to local storage")
-        end
+        helper.migrate_to_local_storage
       rescue StandardError => e
         logger.error(e.message)
       end

lib/tasks/gitlab/pages.rake

@@ -61,9 +61,8 @@ namespace :gitlab do
     namespace :deployments do
       task migrate_to_object_storage: :gitlab_environment do
         logger = Logger.new(STDOUT)
-        logger.info('Starting transfer of pages deployments to remote storage')
 
-        helper = Gitlab::Pages::MigrationHelper.new(logger)
+        helper = Gitlab::LocalAndRemoteStorageMigration::PagesDeploymentMigrater.new(logger)
 
         begin
           helper.migrate_to_remote_storage
@@ -74,9 +73,8 @@ namespace :gitlab do
 
       task migrate_to_local: :gitlab_environment do
         logger = Logger.new(STDOUT)
-        logger.info('Starting transfer of Pages deployments to local storage')
 
-        helper = Gitlab::Pages::MigrationHelper.new(logger)
+        helper = Gitlab::LocalAndRemoteStorageMigration::PagesDeploymentMigrater.new(logger)
 
         begin
           helper.migrate_to_local_storage