Merge branch 'gitaly_backup_parallel_storage' into 'master'

Configure gitaly-backup for maximum concurrency per storage

See merge request gitlab-org/gitlab!65614

lib/backup/gitaly_backup.rb

@@ -3,9 +3,10 @@
 module Backup
   # Backup and restores repositories using gitaly-backup
   class GitalyBackup
-    def initialize(progress, parallel: nil)
+    def initialize(progress, parallel: nil, parallel_storage: nil)
       @progress = progress
       @parallel = parallel
+      @parallel_storage = parallel_storage
     end
 
     def start(type)
@@ -22,6 +23,7 @@ module Backup
 
       args = []
       args += ['-parallel', @parallel.to_s] if type == :create && @parallel
+      args += ['-parallel-storage', @parallel_storage.to_s] if type == :create && @parallel_storage
 
       @read_io, @write_io = IO.pipe
       @pid = Process.spawn(bin_path, command, '-path', backup_repos_path, *args, in: @read_io, out: @progress)

lib/tasks/gitlab/backup.rake

@@ -299,7 +299,8 @@ namespace :gitlab do
     def repository_backup_strategy
       if Feature.enabled?(:gitaly_backup)
         max_concurrency = ENV['GITLAB_BACKUP_MAX_CONCURRENCY'].presence
-        Backup::GitalyBackup.new(progress, parallel: max_concurrency)
+        max_storage_concurrency = ENV['GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY'].presence
+        Backup::GitalyBackup.new(progress, parallel: max_concurrency, parallel_storage: max_storage_concurrency)
       else
         Backup::GitalyRpcBackup.new(progress)
       end

spec/lib/backup/gitaly_backup_spec.rb

@@ -4,6 +4,7 @@ require 'spec_helper'
 
 RSpec.describe Backup::GitalyBackup do
   let(:parallel) { nil }
+  let(:parallel_storage) { nil }
   let(:progress) do
     Tempfile.new('progress').tap do |progress|
       progress.unlink
@@ -14,7 +15,7 @@ RSpec.describe Backup::GitalyBackup do
     progress.close
   end
 
-  subject { described_class.new(progress, parallel: parallel) }
+  subject { described_class.new(progress, parallel: parallel, parallel_storage: parallel_storage) }
 
   context 'unknown' do
     it 'fails to start unknown' do
@@ -59,6 +60,17 @@ RSpec.describe Backup::GitalyBackup do
         end
       end
 
+      context 'parallel_storage option set' do
+        let(:parallel_storage) { 3 }
+
+        it 'passes parallel option through' do
+          expect(Process).to receive(:spawn).with(anything, 'create', '-path', anything, '-parallel-storage', '3', { in: anything, out: progress }).and_call_original
+
+          subject.start(:create)
+          subject.wait
+        end
+      end
+
       it 'raises when the exit code not zero' do
         expect(subject).to receive(:bin_path).and_return(Gitlab::Utils.which('false'))
 

spec/tasks/gitlab/backup_rake_spec.rb

@@ -407,7 +407,7 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
             .with(max_concurrency: 5, max_storage_concurrency: 2)
             .and_call_original
         end
-        expect(::Backup::GitalyBackup).to receive(:new).with(anything, parallel: 5).and_call_original
+        expect(::Backup::GitalyBackup).to receive(:new).with(anything, parallel: 5, parallel_storage: 2).and_call_original
 
         expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout_from_any_process
       end