Merge branch 'backup_tasks' into 'master'

Extract backup tasks

See merge request gitlab-org/gitlab!79809

ee/spec/lib/ee/backup/repositories_spec.rb

@@ -6,7 +6,7 @@ RSpec.describe Backup::Repositories do
   let(:progress) { spy(:stdout) }
   let(:strategy) { spy(:strategy) }
 
-  subject { described_class.new(progress, strategy: strategy) }
+  subject { described_class.new(progress, max_concurrency: 1, max_storage_concurrency: 1, strategy: strategy) }
 
   describe '#dump' do
     context 'hashed storage' do
@@ -16,7 +16,7 @@ RSpec.describe Backup::Repositories do
       it 'calls enqueue for each repository type', :aggregate_failures do
         create(:wiki_page, container: group)
 
-        subject.dump(max_concurrency: 1, max_storage_concurrency: 1)
+        subject.dump
 
         expect(strategy).to have_received(:start).with(:create)
         expect(strategy).to have_received(:enqueue).with(project, Gitlab::GlRepository::PROJECT)
@@ -37,32 +37,32 @@ RSpec.describe Backup::Repositories do
         end
         expect(strategy).to receive(:finish!)
 
-        subject.dump(max_concurrency: 1, max_storage_concurrency: 1)
+        subject.dump
       end
 
       describe 'command failure' do
         it 'enqueue_group raises an error' do
           allow(strategy).to receive(:enqueue).with(anything, Gitlab::GlRepository::WIKI).and_raise(IOError)
 
-          expect { subject.dump(max_concurrency: 1, max_storage_concurrency: 1) }.to raise_error(IOError)
+          expect { subject.dump }.to raise_error(IOError)
         end
 
         it 'group query raises an error' do
           allow(Group).to receive_message_chain(:includes, :find_each).and_raise(ActiveRecord::StatementTimeout)
 
-          expect { subject.dump(max_concurrency: 1, max_storage_concurrency: 1) }.to raise_error(ActiveRecord::StatementTimeout)
+          expect { subject.dump }.to raise_error(ActiveRecord::StatementTimeout)
         end
       end
 
       it 'avoids N+1 database queries' do
         control_count = ActiveRecord::QueryRecorder.new do
-          subject.dump(max_concurrency: 1, max_storage_concurrency: 1)
+          subject.dump
         end.count
 
         create_list(:group, 2, :wiki_repo)
 
         expect do
-          subject.dump(max_concurrency: 1, max_storage_concurrency: 1)
+          subject.dump
         end.not_to exceed_query_limit(control_count)
       end
     end

lib/backup/artifacts.rb

@@ -9,5 +9,9 @@ module Backup
 
       super('artifacts', JobArtifactUploader.root, excludes: ['tmp'])
     end
+
+    def human_name
+      _('artifacts')
+    end
   end
 end

lib/backup/builds.rb

@@ -9,5 +9,9 @@ module Backup
 
       super('builds', Settings.gitlab_ci.builds_path)
     end
+
+    def human_name
+      _('builds')
+    end
   end
 end

lib/backup/database.rb

@@ -90,7 +90,26 @@ module Backup
       report_success(success)
       raise Backup::Error, 'Restore failed' unless success
 
-      errors
+      if errors.present?
+        warning = <<~MSG
+            There were errors in restoring the schema. This may cause
+            issues if this results in missing indexes, constraints, or
+            columns. Please record the errors above and contact GitLab
+            Support if you have questions:
+            https://about.gitlab.com/support/
+        MSG
+
+        warn warning.color(:red)
+        Gitlab::TaskHelpers.ask_to_continue
+      end
+    end
+
+    def enabled
+      true
+    end
+
+    def human_name
+      _('database')
     end
 
     protected

lib/backup/files.rb

@@ -63,6 +63,10 @@ module Backup
       end
     end
 
+    def enabled
+      true
+    end
+
     def tar
       if system(*%w[gtar --version], out: '/dev/null')
         # It looks like we can get GNU tar by running 'gtar'

lib/backup/lfs.rb

@@ -9,5 +9,9 @@ module Backup
 
       super('lfs', Settings.lfs.storage_path)
     end
+
+    def human_name
+      _('lfs objects')
+    end
   end
 end

lib/backup/manager.rb

@@ -10,6 +10,137 @@ module Backup
 
     def initialize(progress)
       @progress = progress
+
+      max_concurrency = ENV.fetch('GITLAB_BACKUP_MAX_CONCURRENCY', 1).to_i
+      max_storage_concurrency = ENV.fetch('GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY', 1).to_i
+
+      @tasks = {
+        'db' => Database.new(progress),
+        'repositories' => Repositories.new(progress,
+                                           strategy: repository_backup_strategy,
+                                           max_concurrency: max_concurrency,
+                                           max_storage_concurrency: max_storage_concurrency),
+        'uploads' => Uploads.new(progress),
+        'builds' => Builds.new(progress),
+        'artifacts' => Artifacts.new(progress),
+        'pages' => Pages.new(progress),
+        'lfs' => Lfs.new(progress),
+        'terraform_state' => TerraformState.new(progress),
+        'registry' => Registry.new(progress),
+        'packages' => Packages.new(progress)
+      }.freeze
+    end
+
+    def create
+      @tasks.keys.each do |task_name|
+        run_create_task(task_name)
+      end
+
+      write_info
+
+      if ENV['SKIP'] && ENV['SKIP'].include?('tar')
+        upload
+      else
+        pack
+        upload
+        cleanup
+        remove_old
+      end
+
+      progress.puts "Warning: Your gitlab.rb and gitlab-secrets.json files contain sensitive data \n" \
+           "and are not included in this backup. You will need these files to restore a backup.\n" \
+           "Please back them up manually.".color(:red)
+      progress.puts "Backup task is done."
+    end
+
+    def run_create_task(task_name)
+      task = @tasks[task_name]
+
+      puts_time "Dumping #{task.human_name} ... ".color(:blue)
+
+      unless task.enabled
+        puts_time "[DISABLED]".color(:cyan)
+        return
+      end
+
+      if ENV["SKIP"] && ENV["SKIP"].include?(task_name)
+        puts_time "[SKIPPED]".color(:cyan)
+        return
+      end
+
+      task.dump
+      puts_time "done".color(:green)
+
+    rescue Backup::DatabaseBackupError, Backup::FileBackupError => e
+      progress.puts "#{e.message}"
+    end
+
+    def restore
+      cleanup_required = unpack
+      verify_backup_version
+
+      unless skipped?('db')
+        begin
+          unless ENV['force'] == 'yes'
+            warning = <<-MSG.strip_heredoc
+              Be sure to stop Puma, Sidekiq, and any other process that
+              connects to the database before proceeding. For Omnibus
+              installs, see the following link for more information:
+              https://docs.gitlab.com/ee/raketasks/backup_restore.html#restore-for-omnibus-gitlab-installations
+
+              Before restoring the database, we will remove all existing
+              tables to avoid future upgrade problems. Be aware that if you have
+              custom tables in the GitLab database these tables and all data will be
+              removed.
+            MSG
+            puts warning.color(:red)
+            Gitlab::TaskHelpers.ask_to_continue
+            puts 'Removing all tables. Press `Ctrl-C` within 5 seconds to abort'.color(:yellow)
+            sleep(5)
+          end
+
+          # Drop all tables Load the schema to ensure we don't have any newer tables
+          # hanging out from a failed upgrade
+          puts_time 'Cleaning the database ... '.color(:blue)
+          Rake::Task['gitlab:db:drop_tables'].invoke
+          puts_time 'done'.color(:green)
+          run_restore_task('db')
+        rescue Gitlab::TaskAbortedByUserError
+          puts "Quitting...".color(:red)
+          exit 1
+        end
+      end
+
+      @tasks.except('db').keys.each do |task_name|
+        run_restore_task(task_name) unless skipped?(task_name)
+      end
+
+      Rake::Task['gitlab:shell:setup'].invoke
+      Rake::Task['cache:clear'].invoke
+
+      if cleanup_required
+        cleanup
+      end
+
+      remove_tmp
+
+      puts "Warning: Your gitlab.rb and gitlab-secrets.json files contain sensitive data \n" \
+           "and are not included in this backup. You will need to restore these files manually.".color(:red)
+      puts "Restore task is done."
+    end
+
+    def run_restore_task(task_name)
+      task = @tasks[task_name]
+
+      puts_time "Restoring #{task.human_name} ... ".color(:blue)
+
+      unless task.enabled
+        puts_time "[DISABLED]".color(:cyan)
+        return
+      end
+
+      task.restore
+      puts_time "done".color(:green)
     end
 
     def write_info
@@ -188,11 +319,15 @@ module Backup
     end
 
     def skipped?(item)
-      settings[:skipped] && settings[:skipped].include?(item) || disabled_features.include?(item)
+      settings[:skipped] && settings[:skipped].include?(item) || !enabled_task?(item)
     end
 
     private
 
+    def enabled_task?(task_name)
+      @tasks[task_name].enabled
+    end
+
     def backup_file?(file)
       file.match(/^(\d{10})(?:_\d{4}_\d{2}_\d{2}(_\d+\.\d+\.\d+((-|\.)(pre|rc\d))?(-ee)?)?)?_gitlab_backup\.tar$/)
     end
@@ -256,12 +391,6 @@ module Backup
       FOLDERS_TO_BACKUP.select { |name| !skipped?(name) && Dir.exist?(File.join(backup_path, name)) }
     end
 
-    def disabled_features
-      features = []
-      features << 'registry' unless Gitlab.config.registry.enabled
-      features
-    end
-
     def settings
       @settings ||= YAML.load_file("backup_information.yml")
     end
@@ -317,6 +446,21 @@ module Backup
     def google_provider?
       Gitlab.config.backup.upload.connection&.provider&.downcase == 'google'
     end
+
+    def repository_backup_strategy
+      if Feature.enabled?(:gitaly_backup, default_enabled: :yaml)
+        max_concurrency = ENV['GITLAB_BACKUP_MAX_CONCURRENCY'].presence
+        max_storage_concurrency = ENV['GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY'].presence
+        Backup::GitalyBackup.new(progress, max_parallelism: max_concurrency, storage_parallelism: max_storage_concurrency)
+      else
+        Backup::GitalyRpcBackup.new(progress)
+      end
+    end
+
+    def puts_time(msg)
+      progress.puts "#{Time.now} -- #{msg}"
+      Gitlab::BackupLogger.info(message: "#{Rainbow.uncolor(msg)}")
+    end
   end
 end
 

lib/backup/packages.rb

@@ -9,5 +9,9 @@ module Backup
 
       super('packages', Settings.packages.storage_path, excludes: ['tmp'])
     end
+
+    def human_name
+      _('packages')
+    end
   end
 end

lib/backup/pages.rb

@@ -13,5 +13,9 @@ module Backup
 
       super('pages', Gitlab.config.pages.path, excludes: [LEGACY_PAGES_TMP_PATH])
     end
+
+    def human_name
+      _('pages')
+    end
   end
 end

lib/backup/registry.rb

@@ -9,5 +9,13 @@ module Backup
 
       super('registry', Settings.registry.path)
     end
+
+    def human_name
+      _('container registry images')
+    end
+
+    def enabled
+      Gitlab.config.registry.enabled
+    end
   end
 end

lib/backup/repositories.rb

@@ -4,12 +4,14 @@ require 'yaml'
 
 module Backup
   class Repositories
-    def initialize(progress, strategy:)
+    def initialize(progress, strategy:, max_concurrency: 1, max_storage_concurrency: 1)
       @progress = progress
       @strategy = strategy
+      @max_concurrency = max_concurrency
+      @max_storage_concurrency = max_storage_concurrency
     end
 
-    def dump(max_concurrency:, max_storage_concurrency:)
+    def dump
       strategy.start(:create)
 
       # gitaly-backup is designed to handle concurrency on its own. So we want
@@ -19,6 +21,11 @@ module Backup
         return enqueue_consecutive
       end
 
+      if max_concurrency < 1 || max_storage_concurrency < 1
+        puts "GITLAB_BACKUP_MAX_CONCURRENCY and GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY must have a value of at least 1".color(:red)
+        exit 1
+      end
+
       check_valid_storages!
 
       semaphore = Concurrent::Semaphore.new(max_concurrency)
@@ -54,9 +61,17 @@ module Backup
       restore_object_pools
     end
 
+    def enabled
+      true
+    end
+
+    def human_name
+      _('repositories')
+    end
+
     private
 
-    attr_reader :progress, :strategy
+    attr_reader :progress, :strategy, :max_concurrency, :max_storage_concurrency
 
     def check_valid_storages!
       repository_storage_klasses.each do |klass|

lib/backup/terraform_state.rb

@@ -9,5 +9,9 @@ module Backup
 
       super('terraform_state', Settings.terraform_state.storage_path, excludes: ['tmp'])
     end
+
+    def human_name
+      _('terraform states')
+    end
   end
 end

lib/backup/uploads.rb

@@ -9,5 +9,9 @@ module Backup
 
       super('uploads', File.join(Gitlab.config.uploads.storage_path, "uploads"), excludes: ['tmp'])
     end
+
+    def human_name
+      _('uploads')
+    end
   end
 end

lib/tasks/gitlab/backup.rake

@@ -9,26 +9,7 @@ namespace :gitlab do
     task create: :gitlab_environment do
       warn_user_is_not_gitlab
 
-      %w(db repo uploads builds artifacts pages lfs terraform_state registry packages).each do |type|
-        Rake::Task["gitlab:backup:#{type}:create"].invoke
-      end
-
-      backup = Backup::Manager.new(progress)
-      backup.write_info
-
-      if ENV['SKIP'] && ENV['SKIP'].include?('tar')
-        backup.upload
-      else
-        backup.pack
-        backup.upload
-        backup.cleanup
-        backup.remove_old
-      end
-
-      progress.puts "Warning: Your gitlab.rb and gitlab-secrets.json files contain sensitive data \n" \
-           "and are not included in this backup. You will need these files to restore a backup.\n" \
-           "Please back them up manually.".color(:red)
-      progress.puts "Backup task is done."
+      Backup::Manager.new(progress).create
     end
 
     # Restore backup of GitLab system
@@ -36,320 +17,109 @@ namespace :gitlab do
     task restore: :gitlab_environment do
       warn_user_is_not_gitlab
 
-      backup = Backup::Manager.new(progress)
-      cleanup_required = backup.unpack
-      backup.verify_backup_version
-
-      unless backup.skipped?('db')
-        begin
-          unless ENV['force'] == 'yes'
-            warning = <<-MSG.strip_heredoc
-              Be sure to stop Puma, Sidekiq, and any other process that
-              connects to the database before proceeding. For Omnibus
-              installs, see the following link for more information:
-              https://docs.gitlab.com/ee/raketasks/backup_restore.html#restore-for-omnibus-gitlab-installations
-
-              Before restoring the database, we will remove all existing
-              tables to avoid future upgrade problems. Be aware that if you have
-              custom tables in the GitLab database these tables and all data will be
-              removed.
-            MSG
-            puts warning.color(:red)
-            ask_to_continue
-            puts 'Removing all tables. Press `Ctrl-C` within 5 seconds to abort'.color(:yellow)
-            sleep(5)
-          end
-
-          # Drop all tables Load the schema to ensure we don't have any newer tables
-          # hanging out from a failed upgrade
-          puts_time 'Cleaning the database ... '.color(:blue)
-          Rake::Task['gitlab:db:drop_tables'].invoke
-          puts_time 'done'.color(:green)
-          Rake::Task['gitlab:backup:db:restore'].invoke
-        rescue Gitlab::TaskAbortedByUserError
-          puts "Quitting...".color(:red)
-          exit 1
-        end
-      end
-
-      Rake::Task['gitlab:backup:repo:restore'].invoke unless backup.skipped?('repositories')
-      Rake::Task['gitlab:backup:uploads:restore'].invoke unless backup.skipped?('uploads')
-      Rake::Task['gitlab:backup:builds:restore'].invoke unless backup.skipped?('builds')
-      Rake::Task['gitlab:backup:artifacts:restore'].invoke unless backup.skipped?('artifacts')
-      Rake::Task['gitlab:backup:pages:restore'].invoke unless backup.skipped?('pages')
-      Rake::Task['gitlab:backup:lfs:restore'].invoke unless backup.skipped?('lfs')
-      Rake::Task['gitlab:backup:terraform_state:restore'].invoke unless backup.skipped?('terraform_state')
-      Rake::Task['gitlab:backup:registry:restore'].invoke unless backup.skipped?('registry')
-      Rake::Task['gitlab:backup:packages:restore'].invoke unless backup.skipped?('packages')
-      Rake::Task['gitlab:shell:setup'].invoke
-      Rake::Task['cache:clear'].invoke
-
-      if cleanup_required
-        backup.cleanup
-      end
-
-      backup.remove_tmp
-
-      puts "Warning: Your gitlab.rb and gitlab-secrets.json files contain sensitive data \n" \
-           "and are not included in this backup. You will need to restore these files manually.".color(:red)
-      puts "Restore task is done."
+      Backup::Manager.new(progress).restore
     end
 
     namespace :repo do
       task create: :gitlab_environment do
-        puts_time "Dumping repositories ...".color(:blue)
-
-        max_concurrency = ENV.fetch('GITLAB_BACKUP_MAX_CONCURRENCY', 1).to_i
-        max_storage_concurrency = ENV.fetch('GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY', 1).to_i
-
-        if ENV["SKIP"] && ENV["SKIP"].include?("repositories")
-          puts_time "[SKIPPED]".color(:cyan)
-        elsif max_concurrency < 1 || max_storage_concurrency < 1
-          puts "GITLAB_BACKUP_MAX_CONCURRENCY and GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY must have a value of at least 1".color(:red)
-          exit 1
-        else
-          Backup::Repositories.new(progress, strategy: repository_backup_strategy).dump(
-            max_concurrency: max_concurrency,
-            max_storage_concurrency: max_storage_concurrency
-          )
-          puts_time "done".color(:green)
-        end
+        Backup::Manager.new(progress).run_create_task('repositories')
       end
 
       task restore: :gitlab_environment do
-        puts_time "Restoring repositories ...".color(:blue)
-        Backup::Repositories.new(progress, strategy: repository_backup_strategy).restore
-        puts_time "done".color(:green)
+        Backup::Manager.new(progress).run_restore_task('repositories')
       end
     end
 
     namespace :db do
       task create: :gitlab_environment do
-        puts_time "Dumping database ... ".color(:blue)
-
-        if ENV["SKIP"] && ENV["SKIP"].include?("db")
-          puts_time "[SKIPPED]".color(:cyan)
-        else
-          begin
-            Backup::Database.new(progress).dump
-            puts_time "done".color(:green)
-          rescue Backup::DatabaseBackupError => e
-            progress.puts "#{e.message}"
-          end
-        end
+        Backup::Manager.new(progress).run_create_task('db')
       end
 
       task restore: :gitlab_environment do
-        puts_time "Restoring database ... ".color(:blue)
-        errors = Backup::Database.new(progress).restore
-
-        if errors.present?
-          warning = <<~MSG
-            There were errors in restoring the schema. This may cause
-            issues if this results in missing indexes, constraints, or
-            columns. Please record the errors above and contact GitLab
-            Support if you have questions:
-            https://about.gitlab.com/support/
-          MSG
-
-          warn warning.color(:red)
-          ask_to_continue
-        end
-
-        puts_time "done".color(:green)
+        Backup::Manager.new(progress).run_restore_task('db')
       end
     end
 
     namespace :builds do
       task create: :gitlab_environment do
-        puts_time "Dumping builds ... ".color(:blue)
-
-        if ENV["SKIP"] && ENV["SKIP"].include?("builds")
-          puts_time "[SKIPPED]".color(:cyan)
-        else
-          begin
-            Backup::Builds.new(progress).dump
-            puts_time "done".color(:green)
-          rescue Backup::FileBackupError => e
-            progress.puts "#{e.message}"
-          end
-        end
+        Backup::Manager.new(progress).run_create_task('builds')
       end
 
       task restore: :gitlab_environment do
-        puts_time "Restoring builds ... ".color(:blue)
-        Backup::Builds.new(progress).restore
-        puts_time "done".color(:green)
+        Backup::Manager.new(progress).run_restore_task('builds')
       end
     end
 
     namespace :uploads do
       task create: :gitlab_environment do
-        puts_time "Dumping uploads ... ".color(:blue)
-
-        if ENV["SKIP"] && ENV["SKIP"].include?("uploads")
-          puts_time "[SKIPPED]".color(:cyan)
-        else
-          begin
-            Backup::Uploads.new(progress).dump
-            puts_time "done".color(:green)
-          rescue Backup::FileBackupError => e
-            progress.puts "#{e.message}"
-          end
-        end
+        Backup::Manager.new(progress).run_create_task('uploads')
       end
 
       task restore: :gitlab_environment do
-        puts_time "Restoring uploads ... ".color(:blue)
-        Backup::Uploads.new(progress).restore
-        puts_time "done".color(:green)
+        Backup::Manager.new(progress).run_restore_task('uploads')
       end
     end
 
     namespace :artifacts do
       task create: :gitlab_environment do
-        puts_time "Dumping artifacts ... ".color(:blue)
-
-        if ENV["SKIP"] && ENV["SKIP"].include?("artifacts")
-          puts_time "[SKIPPED]".color(:cyan)
-        else
-          begin
-            Backup::Artifacts.new(progress).dump
-            puts_time "done".color(:green)
-          rescue Backup::FileBackupError => e
-            progress.puts "#{e.message}"
-          end
-        end
+        Backup::Manager.new(progress).run_create_task('artifacts')
       end
 
       task restore: :gitlab_environment do
-        puts_time "Restoring artifacts ... ".color(:blue)
-        Backup::Artifacts.new(progress).restore
-        puts_time "done".color(:green)
+        Backup::Manager.new(progress).run_restore_task('artifacts')
       end
     end
 
     namespace :pages do
       task create: :gitlab_environment do
-        puts_time "Dumping pages ... ".color(:blue)
-
-        if ENV["SKIP"] && ENV["SKIP"].include?("pages")
-          puts_time "[SKIPPED]".color(:cyan)
-        else
-          begin
-            Backup::Pages.new(progress).dump
-            puts_time "done".color(:green)
-          rescue Backup::FileBackupError => e
-            progress.puts "#{e.message}"
-          end
-        end
+        Backup::Manager.new(progress).run_create_task('pages')
       end
 
       task restore: :gitlab_environment do
-        puts_time "Restoring pages ... ".color(:blue)
-        Backup::Pages.new(progress).restore
-        puts_time "done".color(:green)
+        Backup::Manager.new(progress).run_restore_task('pages')
       end
     end
 
     namespace :lfs do
       task create: :gitlab_environment do
-        puts_time "Dumping lfs objects ... ".color(:blue)
-
-        if ENV["SKIP"] && ENV["SKIP"].include?("lfs")
-          puts_time "[SKIPPED]".color(:cyan)
-        else
-          begin
-            Backup::Lfs.new(progress).dump
-            puts_time "done".color(:green)
-          rescue Backup::FileBackupError => e
-            progress.puts "#{e.message}"
-          end
-        end
+        Backup::Manager.new(progress).run_create_task('lfs')
       end
 
       task restore: :gitlab_environment do
-        puts_time "Restoring lfs objects ... ".color(:blue)
-        Backup::Lfs.new(progress).restore
-        puts_time "done".color(:green)
+        Backup::Manager.new(progress).run_restore_task('lfs')
       end
     end
 
     namespace :terraform_state do
       task create: :gitlab_environment do
-        puts_time "Dumping terraform states ... ".color(:blue)
-
-        if ENV["SKIP"] && ENV["SKIP"].include?("terraform_state")
-          puts_time "[SKIPPED]".color(:cyan)
-        else
-          Backup::TerraformState.new(progress).dump
-          puts_time "done".color(:green)
-        end
+        Backup::Manager.new(progress).run_create_task('terraform_state')
       end
 
       task restore: :gitlab_environment do
-        puts_time "Restoring terraform states ... ".color(:blue)
-        Backup::TerraformState.new(progress).restore
-        puts_time "done".color(:green)
+        Backup::Manager.new(progress).run_restore_task('terraform_state')
       end
     end
 
     namespace :registry do
       task create: :gitlab_environment do
-        puts_time "Dumping container registry images ... ".color(:blue)
-
-        if Gitlab.config.registry.enabled
-          if ENV["SKIP"] && ENV["SKIP"].include?("registry")
-            puts_time "[SKIPPED]".color(:cyan)
-          else
-            begin
-              Backup::Registry.new(progress).dump
-              puts_time "done".color(:green)
-            rescue Backup::FileBackupError => e
-              progress.puts "#{e.message}"
-            end
-          end
-        else
-          puts_time "[DISABLED]".color(:cyan)
-        end
+        Backup::Manager.new(progress).run_create_task('registry')
       end
 
       task restore: :gitlab_environment do
-        puts_time "Restoring container registry images ... ".color(:blue)
-
-        if Gitlab.config.registry.enabled
-          Backup::Registry.new(progress).restore
-          puts_time "done".color(:green)
-        else
-          puts_time "[DISABLED]".color(:cyan)
-        end
+        Backup::Manager.new(progress).run_restore_task('registry')
       end
     end
 
     namespace :packages do
       task create: :gitlab_environment do
-        puts_time "Dumping packages ... ".color(:blue)
-
-        if ENV['SKIP'] && ENV['SKIP'].include?('packages')
-          puts_time "[SKIPPED]".color(:cyan)
-        else
-          Backup::Packages.new(progress).dump
-          puts_time "done".color(:green)
-        end
+        Backup::Manager.new(progress).run_create_task('packages')
       end
 
       task restore: :gitlab_environment do
-        puts_time "Restoring packages ...".color(:blue)
-        Backup::Packages.new(progress).restore
-        puts_time "done".color(:green)
+        Backup::Manager.new(progress).run_restore_task('packages')
       end
     end
 
-    def puts_time(msg)
-      progress.puts "#{Time.now} -- #{msg}"
-      Gitlab::BackupLogger.info(message: "#{Rainbow.uncolor(msg)}")
-    end
-
     def progress
       if ENV['CRON']
         # We need an object we can say 'puts' and 'print' to; let's use a
@@ -360,16 +130,6 @@ namespace :gitlab do
         $stdout
       end
     end
-
-    def repository_backup_strategy
-      if Feature.enabled?(:gitaly_backup, default_enabled: :yaml)
-        max_concurrency = ENV['GITLAB_BACKUP_MAX_CONCURRENCY'].presence
-        max_storage_concurrency = ENV['GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY'].presence
-        Backup::GitalyBackup.new(progress, max_parallelism: max_concurrency, storage_parallelism: max_storage_concurrency)
-      else
-        Backup::GitalyRpcBackup.new(progress)
-      end
-    end
   end
   # namespace end: backup
 end

locale/gitlab.pot

@@ -42490,6 +42490,9 @@ msgstr ""
 msgid "archived:"
 msgstr ""
 
+msgid "artifacts"
+msgstr ""
+
 msgid "assign yourself"
 msgstr ""
 
@@ -42519,6 +42522,9 @@ msgstr[1] ""
 msgid "branch name"
 msgstr ""
 
+msgid "builds"
+msgstr ""
+
 msgid "by"
 msgstr ""
 
@@ -42942,6 +42948,9 @@ msgstr ""
 msgid "contact with same email already exists in group hierarchy"
 msgstr ""
 
+msgid "container registry images"
+msgstr ""
+
 msgid "container_name can contain only lowercase letters, digits, '-', and '.' and must start and end with an alphanumeric character"
 msgstr ""
 
@@ -42981,6 +42990,9 @@ msgstr ""
 msgid "data"
 msgstr ""
 
+msgid "database"
+msgstr ""
+
 msgid "date must not be after 9999-12-31"
 msgstr ""
 
@@ -43354,6 +43366,9 @@ msgstr ""
 msgid "level: %{level}"
 msgstr ""
 
+msgid "lfs objects"
+msgstr ""
+
 msgid "limit of %{project_limit} reached"
 msgstr ""
 
@@ -43851,6 +43866,12 @@ msgid_plural "out of %d total tests"
 msgstr[0] ""
 msgstr[1] ""
 
+msgid "packages"
+msgstr ""
+
+msgid "pages"
+msgstr ""
+
 msgid "parent"
 msgid_plural "parents"
 msgstr[0] ""
@@ -44000,6 +44021,9 @@ msgid_plural "replies"
 msgstr[0] ""
 msgstr[1] ""
 
+msgid "repositories"
+msgstr ""
+
 msgid "repository:"
 msgstr ""
 
@@ -44120,6 +44144,9 @@ msgstr ""
 msgid "tag name"
 msgstr ""
 
+msgid "terraform states"
+msgstr ""
+
 msgid "the correct format."
 msgstr ""
 

spec/lib/backup/database_spec.rb

@@ -6,6 +6,10 @@ RSpec.describe Backup::Database do
   let(:progress) { StringIO.new }
   let(:output) { progress.string }
 
+  before do
+    allow(Gitlab::TaskHelpers).to receive(:ask_to_continue)
+  end
+
   describe '#restore' do
     let(:cmd) { %W[#{Gem.ruby} -e $stdout.puts(1)] }
     let(:data) { Rails.root.join("spec/fixtures/pages_empty.tar.gz").to_s }
@@ -20,7 +24,7 @@ RSpec.describe Backup::Database do
       let(:data) { Rails.root.join("spec/fixtures/pages_empty.tar.gz").to_s }
 
       it 'returns successfully' do
-        expect(subject.restore).to eq([])
+        subject.restore
 
         expect(output).to include("Restoring PostgreSQL database")
         expect(output).to include("[DONE]")
@@ -42,7 +46,8 @@ RSpec.describe Backup::Database do
       let(:cmd) { %W[#{Gem.ruby} -e $stderr.write("#{noise}#{visible_error}")] }
 
       it 'filters out noise from errors' do
-        expect(subject.restore).to eq([visible_error])
+        subject.restore
+
         expect(output).to include("ERRORS")
         expect(output).not_to include(noise)
         expect(output).to include(visible_error)

spec/lib/backup/manager_spec.rb

@@ -12,6 +12,11 @@ RSpec.describe Backup::Manager do
   before do
     allow(progress).to receive(:puts)
     allow(progress).to receive(:print)
+    FileUtils.mkdir_p('tmp/tests/public/uploads')
+  end
+
+  after do
+    FileUtils.rm_rf('tmp/tests/public/uploads', secure: true)
   end
 
   describe '#pack' do

spec/lib/backup/repositories_spec.rb

@@ -6,8 +6,17 @@ RSpec.describe Backup::Repositories do
   let(:progress) { spy(:stdout) }
   let(:parallel_enqueue) { true }
   let(:strategy) { spy(:strategy, parallel_enqueue?: parallel_enqueue) }
-
-  subject { described_class.new(progress, strategy: strategy) }
+  let(:max_concurrency) { 1 }
+  let(:max_storage_concurrency) { 1 }
+
+  subject do
+    described_class.new(
+      progress,
+      strategy: strategy,
+      max_concurrency: max_concurrency,
+      max_storage_concurrency: max_storage_concurrency
+    )
+  end
 
   describe '#dump' do
     let_it_be(:projects) { create_list(:project, 5, :repository) }
@@ -17,7 +26,7 @@ RSpec.describe Backup::Repositories do
         project_snippet = create(:project_snippet, :repository, project: project)
         personal_snippet = create(:personal_snippet, :repository, author: project.first_owner)
 
-        subject.dump(max_concurrency: 1, max_storage_concurrency: 1)
+        subject.dump
 
         expect(strategy).to have_received(:start).with(:create)
         expect(strategy).to have_received(:enqueue).with(project, Gitlab::GlRepository::PROJECT)
@@ -51,38 +60,40 @@ RSpec.describe Backup::Repositories do
         end
         expect(strategy).to receive(:finish!)
 
-        subject.dump(max_concurrency: 1, max_storage_concurrency: 1)
+        subject.dump
       end
 
       describe 'command failure' do
         it 'enqueue_project raises an error' do
           allow(strategy).to receive(:enqueue).with(anything, Gitlab::GlRepository::PROJECT).and_raise(IOError)
 
-          expect { subject.dump(max_concurrency: 1, max_storage_concurrency: 1) }.to raise_error(IOError)
+          expect { subject.dump }.to raise_error(IOError)
         end
 
         it 'project query raises an error' do
           allow(Project).to receive_message_chain(:includes, :find_each).and_raise(ActiveRecord::StatementTimeout)
 
-          expect { subject.dump(max_concurrency: 1, max_storage_concurrency: 1) }.to raise_error(ActiveRecord::StatementTimeout)
+          expect { subject.dump }.to raise_error(ActiveRecord::StatementTimeout)
         end
       end
 
       it 'avoids N+1 database queries' do
         control_count = ActiveRecord::QueryRecorder.new do
-          subject.dump(max_concurrency: 1, max_storage_concurrency: 1)
+          subject.dump
         end.count
 
         create_list(:project, 2, :repository)
 
         expect do
-          subject.dump(max_concurrency: 1, max_storage_concurrency: 1)
+          subject.dump
         end.not_to exceed_query_limit(control_count)
       end
     end
 
     context 'concurrency with a strategy without parallel enqueueing support' do
       let(:parallel_enqueue) { false }
+      let(:max_concurrency) { 2 }
+      let(:max_storage_concurrency) { 2 }
 
       it 'enqueues all projects sequentially' do
         expect(Thread).not_to receive(:new)
@@ -93,13 +104,14 @@ RSpec.describe Backup::Repositories do
         end
         expect(strategy).to receive(:finish!)
 
-        subject.dump(max_concurrency: 2, max_storage_concurrency: 2)
+        subject.dump
       end
     end
 
     [4, 10].each do |max_storage_concurrency|
       context "max_storage_concurrency #{max_storage_concurrency}", quarantine: 'https://gitlab.com/gitlab-org/gitlab/-/issues/241701' do
         let(:storage_keys) { %w[default test_second_storage] }
+        let(:max_storage_concurrency) { max_storage_concurrency }
 
         before do
           allow(Gitlab.config.repositories.storages).to receive(:keys).and_return(storage_keys)
@@ -116,54 +128,58 @@ RSpec.describe Backup::Repositories do
           end
           expect(strategy).to receive(:finish!)
 
-          subject.dump(max_concurrency: 1, max_storage_concurrency: max_storage_concurrency)
+          subject.dump
         end
 
-        it 'creates the expected number of threads with extra max concurrency' do
-          expect(Thread).to receive(:new)
-            .exactly(storage_keys.length * (max_storage_concurrency + 1)).times
-            .and_call_original
+        context 'with extra max concurrency' do
+          let(:max_concurrency) { 3 }
 
-          expect(strategy).to receive(:start).with(:create)
-          projects.each do |project|
-            expect(strategy).to receive(:enqueue).with(project, Gitlab::GlRepository::PROJECT)
-          end
-          expect(strategy).to receive(:finish!)
+          it 'creates the expected number of threads' do
+            expect(Thread).to receive(:new)
+              .exactly(storage_keys.length * (max_storage_concurrency + 1)).times
+              .and_call_original
 
-          subject.dump(max_concurrency: 3, max_storage_concurrency: max_storage_concurrency)
+            expect(strategy).to receive(:start).with(:create)
+            projects.each do |project|
+              expect(strategy).to receive(:enqueue).with(project, Gitlab::GlRepository::PROJECT)
+            end
+            expect(strategy).to receive(:finish!)
+
+            subject.dump
+          end
         end
 
         describe 'command failure' do
           it 'enqueue_project raises an error' do
             allow(strategy).to receive(:enqueue).and_raise(IOError)
 
-            expect { subject.dump(max_concurrency: 1, max_storage_concurrency: max_storage_concurrency) }.to raise_error(IOError)
+            expect { subject.dump }.to raise_error(IOError)
           end
 
           it 'project query raises an error' do
             allow(Project).to receive_message_chain(:for_repository_storage, :includes, :find_each).and_raise(ActiveRecord::StatementTimeout)
 
-            expect { subject.dump(max_concurrency: 1, max_storage_concurrency: max_storage_concurrency) }.to raise_error(ActiveRecord::StatementTimeout)
+            expect { subject.dump }.to raise_error(ActiveRecord::StatementTimeout)
           end
 
           context 'misconfigured storages' do
             let(:storage_keys) { %w[test_second_storage] }
 
             it 'raises an error' do
-              expect { subject.dump(max_concurrency: 1, max_storage_concurrency: max_storage_concurrency) }.to raise_error(Backup::Error, 'repositories.storages in gitlab.yml is misconfigured')
+              expect { subject.dump }.to raise_error(Backup::Error, 'repositories.storages in gitlab.yml is misconfigured')
             end
           end
         end
 
         it 'avoids N+1 database queries' do
           control_count = ActiveRecord::QueryRecorder.new do
-            subject.dump(max_concurrency: 1, max_storage_concurrency: max_storage_concurrency)
+            subject.dump
           end.count
 
           create_list(:project, 2, :repository)
 
           expect do
-            subject.dump(max_concurrency: 1, max_storage_concurrency: max_storage_concurrency)
+            subject.dump
           end.not_to exceed_query_limit(control_count)
         end
       end

spec/tasks/gitlab/backup_rake_spec.rb

@@ -4,7 +4,8 @@ require 'rake_helper'
 
 RSpec.describe 'gitlab:app namespace rake task', :delete do
   let(:enable_registry) { true }
-  let(:backup_types) { %w{db repo uploads builds artifacts pages lfs terraform_state registry packages} }
+  let(:backup_tasks) { %w{db repo uploads builds artifacts pages lfs terraform_state registry packages} }
+  let(:backup_types) { %w{db repositories uploads builds artifacts pages lfs terraform_state registry packages} }
 
   def tars_glob
     Dir.glob(File.join(Gitlab.config.backup.path, '*_gitlab_backup.tar'))
@@ -48,7 +49,7 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
   end
 
   def reenable_backup_sub_tasks
-    backup_types.each do |subtask|
+    backup_tasks.each do |subtask|
       Rake::Task["gitlab:backup:#{subtask}:create"].reenable
     end
   end
@@ -72,8 +73,11 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
           allow(YAML).to receive(:load_file)
             .and_return({ gitlab_version: gitlab_version })
           expect(Rake::Task['gitlab:db:drop_tables']).to receive(:invoke)
-          backup_types.each do |subtask|
-            expect(Rake::Task["gitlab:backup:#{subtask}:restore"]).to receive(:invoke)
+          expect_next_instance_of(::Backup::Manager) do |instance|
+            backup_types.each do |subtask|
+              expect(instance).to receive(:run_restore_task).with(subtask).ordered
+            end
+            expect(instance).not_to receive(:run_restore_task)
           end
           expect(Rake::Task['gitlab:shell:setup']).to receive(:invoke)
         end
@@ -128,16 +132,14 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
           .and_return({ gitlab_version: Gitlab::VERSION })
 
         expect(Rake::Task['gitlab:db:drop_tables']).to receive(:invoke)
-        expect(Rake::Task['gitlab:backup:db:restore']).to receive(:invoke)
-        expect(Rake::Task['gitlab:backup:repo:restore']).to receive(:invoke)
-        expect(Rake::Task['gitlab:backup:builds:restore']).to receive(:invoke)
-        expect(Rake::Task['gitlab:backup:uploads:restore']).to receive(:invoke)
-        expect(Rake::Task['gitlab:backup:artifacts:restore']).to receive(:invoke)
-        expect(Rake::Task['gitlab:backup:pages:restore']).to receive(:invoke)
-        expect(Rake::Task['gitlab:backup:lfs:restore']).to receive(:invoke)
-        expect(Rake::Task['gitlab:backup:terraform_state:restore']).to receive(:invoke)
-        expect(Rake::Task['gitlab:backup:registry:restore']).to receive(:invoke)
-        expect(Rake::Task['gitlab:backup:packages:restore']).to receive(:invoke)
+
+        expect_next_instance_of(::Backup::Manager) do |instance|
+          backup_types.each do |subtask|
+            expect(instance).to receive(:run_restore_task).with(subtask).ordered
+          end
+          expect(instance).not_to receive(:run_restore_task)
+        end
+
         expect(Rake::Task['gitlab:shell:setup']).to receive(:invoke)
       end
 
@@ -198,7 +200,7 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
 
       context 'specific backup tasks' do
         it 'prints a progress message to stdout' do
-          backup_types.each do |task|
+          backup_tasks.each do |task|
             expect { run_rake_task("gitlab:backup:#{task}:create") }.to output(/Dumping /).to_stdout_from_any_process
           end
         end
@@ -206,7 +208,7 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
         it 'logs the progress to log file' do
           expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping database ... ")
           expect(Gitlab::BackupLogger).to receive(:info).with(message: "[SKIPPED]")
-          expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping repositories ...")
+          expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping repositories ... ")
           expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping uploads ... ")
           expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping builds ... ")
           expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping artifacts ... ")
@@ -217,7 +219,7 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
           expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping packages ... ")
           expect(Gitlab::BackupLogger).to receive(:info).with(message: "done").exactly(9).times
 
-          backup_types.each do |task|
+          backup_tasks.each do |task|
             run_rake_task("gitlab:backup:#{task}:create")
           end
         end
@@ -414,11 +416,9 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
       end
 
       it 'has defaults' do
-        expect_next_instance_of(::Backup::Repositories) do |instance|
-          expect(instance).to receive(:dump)
-            .with(max_concurrency: 1, max_storage_concurrency: 1)
-            .and_call_original
-        end
+        expect(::Backup::Repositories).to receive(:new)
+          .with(anything, strategy: anything, max_concurrency: 1, max_storage_concurrency: 1)
+          .and_call_original
 
         expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout_from_any_process
       end
@@ -432,11 +432,9 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
         stub_env('GITLAB_BACKUP_MAX_CONCURRENCY', 5)
         stub_env('GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY', 2)
 
-        expect_next_instance_of(::Backup::Repositories) do |instance|
-          expect(instance).to receive(:dump)
-            .with(max_concurrency: 5, max_storage_concurrency: 2)
-            .and_call_original
-        end
+        expect(::Backup::Repositories).to receive(:new)
+          .with(anything, strategy: anything, max_concurrency: 5, max_storage_concurrency: 2)
+          .and_call_original
         expect(::Backup::GitalyBackup).to receive(:new).with(anything, max_parallelism: 5, storage_parallelism: 2).and_call_original
 
         expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout_from_any_process
@@ -489,16 +487,12 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
         .to receive(:invoke).and_return(true)
 
       expect(Rake::Task['gitlab:db:drop_tables']).to receive :invoke
-      expect(Rake::Task['gitlab:backup:db:restore']).to receive :invoke
-      expect(Rake::Task['gitlab:backup:repo:restore']).not_to receive :invoke
-      expect(Rake::Task['gitlab:backup:uploads:restore']).not_to receive :invoke
-      expect(Rake::Task['gitlab:backup:builds:restore']).to receive :invoke
-      expect(Rake::Task['gitlab:backup:artifacts:restore']).to receive :invoke
-      expect(Rake::Task['gitlab:backup:pages:restore']).to receive :invoke
-      expect(Rake::Task['gitlab:backup:lfs:restore']).to receive :invoke
-      expect(Rake::Task['gitlab:backup:terraform_state:restore']).to receive :invoke
-      expect(Rake::Task['gitlab:backup:registry:restore']).to receive :invoke
-      expect(Rake::Task['gitlab:backup:packages:restore']).to receive :invoke
+      expect_next_instance_of(::Backup::Manager) do |instance|
+        (backup_types - %w{repositories uploads}).each do |subtask|
+          expect(instance).to receive(:run_restore_task).with(subtask).ordered
+        end
+        expect(instance).not_to receive(:run_restore_task)
+      end
       expect(Rake::Task['gitlab:shell:setup']).to receive :invoke
       expect { run_rake_task('gitlab:backup:restore') }.to output.to_stdout_from_any_process
     end
@@ -538,8 +532,11 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
         .to receive(:invoke).and_return(true)
 
       expect(Rake::Task['gitlab:db:drop_tables']).to receive :invoke
-      backup_types.each do |subtask|
-        expect(Rake::Task["gitlab:backup:#{subtask}:restore"]).to receive :invoke
+      expect_next_instance_of(::Backup::Manager) do |instance|
+        backup_types.each do |subtask|
+          expect(instance).to receive(:run_restore_task).with(subtask).ordered
+        end
+        expect(instance).not_to receive(:run_restore_task)
       end
       expect(Rake::Task['gitlab:shell:setup']).to receive :invoke
       expect { run_rake_task("gitlab:backup:restore") }.to output.to_stdout_from_any_process