Merge branch 'patch-31' into 'master'

Backport API support to move between repository storages/shards

Closes #23663

See merge request gitlab-org/gitlab!18721

app/models/project.rb

@@ -2039,6 +2039,16 @@ class Project < ApplicationRecord
     end
   end
 
+  def change_repository_storage(new_repository_storage_key)
+    return if repository_read_only?
+    return if repository_storage == new_repository_storage_key
+
+    raise ArgumentError unless ::Gitlab.config.repositories.storages.key?(new_repository_storage_key)
+
+    run_after_commit { ProjectUpdateRepositoryStorageWorker.perform_async(id, new_repository_storage_key) }
+    self.repository_read_only = true
+  end
+
   def pushes_since_gc
     Gitlab::Redis::SharedState.with { |redis| redis.get(pushes_since_gc_redis_shared_state_key).to_i }
   end

app/policies/project_policy.rb

@@ -469,6 +469,8 @@ class ProjectPolicy < BasePolicy
     prevent :create_pipeline
   end
 
+  rule { admin }.enable :change_repository_storage
+
   private
 
   def team_member?

ee/app/services/projects/update_repository_storage_service.rb → app/services/projects/update_repository_storage_service.rb

@@ -17,17 +17,7 @@ module Projects
       # exception.
       raise RepositoryAlreadyMoved if project.repository_storage == new_repository_storage_key
 
-      result = mirror_repository(new_repository_storage_key)
-
-      if project.wiki.repository_exists?
-        result &&= mirror_repository(new_repository_storage_key, type: Gitlab::GlRepository::WIKI)
-      end
-
-      if project.design_repository.exists?
-        result &&= mirror_repository(new_repository_storage_key, type: Gitlab::GlRepository::DESIGN)
-      end
-
-      if result
+      if mirror_repositories(new_repository_storage_key)
         mark_old_paths_for_archive
 
         project.update(repository_storage: new_repository_storage_key, repository_read_only: false)
@@ -42,6 +32,16 @@ module Projects
 
     private
 
+    def mirror_repositories(new_repository_storage_key)
+      result = mirror_repository(new_repository_storage_key)
+
+      if project.wiki.repository_exists?
+        result &&= mirror_repository(new_repository_storage_key, type: Gitlab::GlRepository::WIKI)
+      end
+
+      result
+    end
+
     def mirror_repository(new_storage_key, type: Gitlab::GlRepository::PROJECT)
       return false unless wait_for_pushes(type)
 
@@ -77,13 +77,6 @@ module Projects
                                           wiki.disk_path,
                                           "#{new_project_path}.wiki")
         end
-
-        if design_repository.exists?
-          GitlabShellWorker.perform_async(:mv_repository,
-                                          old_repository_storage,
-                                          design_repository.disk_path,
-                                          "#{new_project_path}.design")
-        end
       end
     end
 
@@ -118,3 +111,5 @@ module Projects
     end
   end
 end
+
+Projects::UpdateRepositoryStorageService.prepend_if_ee('EE::Projects::UpdateRepositoryStorageService')

app/services/projects/update_service.rb

@@ -13,6 +13,10 @@ module Projects
 
       ensure_wiki_exists if enabling_wiki?
 
+      if changing_storage_size?
+        project.change_repository_storage(params.delete(:repository_storage))
+      end
+
       yield if block_given?
 
       validate_classification_label(project, :external_authorization_classification_label)
@@ -140,6 +144,13 @@ module Projects
     def changing_pages_https_only?
       project.previous_changes.include?(:pages_https_only)
     end
+
+    def changing_storage_size?
+      new_repository_storage = params[:repository_storage]
+
+      new_repository_storage && project.repository.exists? &&
+        can?(current_user, :change_repository_storage, project)
+    end
   end
 end
 

app/workers/all_queues.yml

@@ -1151,6 +1151,13 @@
   :resource_boundary: :unknown
   :weight: 1
   :idempotent: 
+- :name: project_update_repository_storage
+  :feature_category: :source_code_management
+  :has_external_dependencies: 
+  :urgency: :default
+  :resource_boundary: :unknown
+  :weight: 1
+  :idempotent: 
 - :name: propagate_service_template
   :feature_category: :source_code_management
   :has_external_dependencies: 

changelogs/unreleased/move-storage-shards.yml

+---
+title: "Backport API support to move between repository storages/shards"
+merge_request: 18721
+author: Ben Bodenmiller
+type: added

doc/api/projects.md

@@ -1041,7 +1041,7 @@ POST /projects
 | `ci_config_path` | string | no | The path to CI config file |
 | `auto_devops_enabled` | boolean | no | Enable Auto DevOps for this project |
 | `auto_devops_deploy_strategy` | string | no | Auto Deploy strategy (`continuous`, `manual` or `timed_incremental`) |
-| `repository_storage` | string | no | **(STARTER ONLY)** Which storage shard the repository is on. Available only to admins |
+| `repository_storage` | string | no | Which storage shard the repository is on. Available only to admins |
 | `approvals_before_merge` | integer | no | **(STARTER)** How many approvers should approve merge requests by default |
 | `external_authorization_classification_label` | string | no | **(PREMIUM)** The classification label for the project |
 | `mirror` | boolean | no | **(STARTER)** Enables pull mirroring in a project |
@@ -1109,7 +1109,7 @@ POST /projects/user/:user_id
 | `ci_config_path` | string | no | The path to CI config file |
 | `auto_devops_enabled` | boolean | no | Enable Auto DevOps for this project |
 | `auto_devops_deploy_strategy` | string | no | Auto Deploy strategy (`continuous`, `manual` or `timed_incremental`) |
-| `repository_storage` | string | no | **(STARTER ONLY)** Which storage shard the repository is on. Available only to admins |
+| `repository_storage` | string | no | Which storage shard the repository is on. Available only to admins |
 | `approvals_before_merge` | integer | no | **(STARTER)** How many approvers should approve merge requests by default |
 | `external_authorization_classification_label` | string | no | **(PREMIUM)** The classification label for the project |
 | `mirror` | boolean | no | **(STARTER)** Enables pull mirroring in a project |
@@ -1177,7 +1177,7 @@ PUT /projects/:id
 | `ci_default_git_depth` | integer | no | Default number of revisions for [shallow cloning](../user/project/pipelines/settings.md#git-shallow-clone) |
 | `auto_devops_enabled` | boolean | no | Enable Auto DevOps for this project |
 | `auto_devops_deploy_strategy` | string | no | Auto Deploy strategy (`continuous`, `manual` or `timed_incremental`) |
-| `repository_storage` | string | no | **(STARTER ONLY)** Which storage shard the repository is on. Available only to admins |
+| `repository_storage` | string | no | Which storage shard the repository is on. Available only to admins |
 | `approvals_before_merge` | integer | no | **(STARTER)** How many approvers should approve merge request by default |
 | `external_authorization_classification_label` | string | no | **(PREMIUM)** The classification label for the project |
 | `mirror` | boolean | no | **(STARTER)** Enables pull mirroring in a project |

ee/app/models/ee/project.rb

@@ -491,16 +491,6 @@ module EE
       username_only_import_url
     end
 
-    def change_repository_storage(new_repository_storage_key)
-      return if repository_read_only?
-      return if repository_storage == new_repository_storage_key
-
-      raise ArgumentError unless ::Gitlab.config.repositories.storages.key?(new_repository_storage_key)
-
-      run_after_commit { ProjectUpdateRepositoryStorageWorker.perform_async(id, new_repository_storage_key) }
-      self.repository_read_only = true
-    end
-
     def repository_and_lfs_size
       statistics.total_repository_size
     end

ee/app/policies/ee/project_policy.rb

@@ -125,8 +125,6 @@ module EE
         @subject.feature_available?(:group_timelogs)
       end
 
-      rule { admin }.enable :change_repository_storage
-
       rule { support_bot }.enable :guest_access
       rule { support_bot & ~service_desk_enabled }.policy do
         prevent :create_note

ee/app/services/ee/projects/update_repository_storage_service.rb

+# frozen_string_literal: true
+
+module EE
+  module Projects
+    module UpdateRepositoryStorageService
+      extend ::Gitlab::Utils::Override
+
+      override :mirror_repositories
+      def mirror_repositories(new_repository_storage_key)
+        result = super
+
+        if project.design_repository.exists?
+          result &&= mirror_repository(new_repository_storage_key, type: Gitlab::GlRepository::DESIGN)
+        end
+
+        result
+      end
+
+      override :mark_old_paths_for_archive
+      def mark_old_paths_for_archive
+        super
+
+        old_repository_storage = project.repository_storage
+        new_project_path = moved_path(project.disk_path)
+
+        project.run_after_commit do
+          if design_repository.exists?
+            GitlabShellWorker.perform_async(:mv_repository,
+                                            old_repository_storage,
+                                            design_repository.disk_path,
+                                            "#{new_project_path}.design")
+          end
+        end
+      end
+    end
+  end
+end

ee/app/services/ee/projects/update_service.rb

@@ -21,10 +21,6 @@ module EE
         result = super do
           # Repository size limit comes as MB from the view
           project.repository_size_limit = ::Gitlab::Utils.try_megabytes_to_bytes(limit) if limit
-
-          if changing_storage_size?
-            project.change_repository_storage(params.delete(:repository_storage))
-          end
         end
 
         if result[:status] == :success
@@ -41,13 +37,6 @@ module EE
         result
       end
 
-      def changing_storage_size?
-        new_repository_storage = params[:repository_storage]
-
-        new_repository_storage && project.repository.exists? &&
-          can?(current_user, :change_repository_storage, project)
-      end
-
       private
 
       def valid_mirror_user?

ee/app/workers/all_queues.yml

@@ -528,13 +528,6 @@
   :resource_boundary: :unknown
   :weight: 1
   :idempotent: 
-- :name: project_update_repository_storage
-  :feature_category: :source_code_management
-  :has_external_dependencies: 
-  :urgency: :default
-  :resource_boundary: :unknown
-  :weight: 1
-  :idempotent: 
 - :name: refresh_license_compliance_checks
   :feature_category: :license_compliance
   :has_external_dependencies: 

ee/lib/ee/api/entities.rb

@@ -38,7 +38,6 @@ module EE
         extend ActiveSupport::Concern
 
         prepended do
-          expose :repository_storage, if: ->(_project, options) { options[:current_user].try(:admin?) }
           expose :approvals_before_merge, if: ->(project, _) { project.feature_available?(:merge_request_approvers) }
           expose :mirror, if: ->(project, _) { project.feature_available?(:repository_mirrors) }
           expose :mirror_user_id, if: ->(project, _) { project.mirror? }

ee/lib/ee/api/helpers/projects_helpers.rb

@@ -16,7 +16,6 @@ module EE
           end
 
           params :optional_project_params_ee do
-            optional :repository_storage, type: String, desc: 'Which storage shard the repository is on. Available only to admins'
             optional :approvals_before_merge, type: Integer, desc: 'How many approvers should approve merge request by default'
             optional :mirror, type: Grape::API::Boolean, desc: 'Enables pull mirroring in a project'
             optional :mirror_trigger_builds, type: Grape::API::Boolean, desc: 'Pull mirroring triggers builds'
@@ -50,7 +49,6 @@ module EE
           def update_params_at_least_one_of
             super.concat [
               :approvals_before_merge,
-              :repository_storage,
               :external_authorization_classification_label,
               :import_url,
               :packages_enabled,

ee/lib/ee/api/projects.rb

@@ -38,16 +38,9 @@ module EE
           def verify_update_project_attrs!(project, attrs)
             super
 
-            verify_storage_attrs!(attrs)
             verify_mirror_attrs!(project, attrs)
           end
 
-          def verify_storage_attrs!(attrs)
-            unless current_user.admin?
-              attrs.delete(:repository_storage)
-            end
-          end
-
           def verify_mirror_attrs!(project, attrs)
             unless can?(current_user, :admin_mirror, project)
               attrs.delete(:mirror)

ee/spec/models/project_spec.rb

@@ -2243,49 +2243,6 @@ describe Project do
     end
   end
 
-  describe '#change_repository_storage' do
-    let(:project) { create(:project, :repository) }
-    let(:read_only_project) { create(:project, :repository, repository_read_only: true) }
-
-    before do
-      FileUtils.mkdir('tmp/tests/extra_storage')
-      stub_storage_settings('extra' => { 'path' => 'tmp/tests/extra_storage' })
-    end
-
-    after do
-      FileUtils.rm_rf('tmp/tests/extra_storage')
-    end
-
-    it 'schedule the transfer of the repository to the new storage and locks the project' do
-      expect(ProjectUpdateRepositoryStorageWorker).to receive(:perform_async).with(project.id, 'extra')
-
-      project.change_repository_storage('extra')
-      project.save
-
-      expect(project).to be_repository_read_only
-    end
-
-    it "doesn't schedule the transfer if the repository is already read-only" do
-      expect(ProjectUpdateRepositoryStorageWorker).not_to receive(:perform_async)
-
-      read_only_project.change_repository_storage('extra')
-      read_only_project.save
-    end
-
-    it "doesn't lock or schedule the transfer if the storage hasn't changed" do
-      expect(ProjectUpdateRepositoryStorageWorker).not_to receive(:perform_async)
-
-      project.change_repository_storage(project.repository_storage)
-      project.save
-
-      expect(project).not_to be_repository_read_only
-    end
-
-    it 'throws an error if an invalid repository storage is provided' do
-      expect { project.change_repository_storage('unknown') }.to raise_error(ArgumentError)
-    end
-  end
-
   describe '#repository_and_lfs_size' do
     let(:project) { create(:project, :repository) }
     let(:size) { 50 }

ee/spec/requests/api/projects_spec.rb

@@ -197,27 +197,6 @@ describe API::Projects do
         expect(json_response).not_to have_key 'marked_for_deletion_at'
       end
     end
-
-    describe 'repository_storage attribute' do
-      context 'when authenticated as an admin' do
-        let(:admin) { create(:admin) }
-
-        it 'returns repository_storage attribute' do
-          get api("/projects/#{project.id}", admin)
-
-          expect(response).to have_gitlab_http_status(:ok)
-          expect(json_response['repository_storage']).to eq(project.repository_storage)
-        end
-      end
-
-      context 'when authenticated as a regular user' do
-        it 'does not return repository_storage attribute' do
-          get api("/projects/#{project.id}", user)
-
-          expect(json_response).not_to have_key('repository_storage')
-        end
-      end
-    end
   end
 
   # Assumes the following variables are defined:
@@ -488,50 +467,6 @@ describe API::Projects do
       end
     end
 
-    context 'when updating repository storage' do
-      let(:unknown_storage) { 'new-storage' }
-      let(:new_project) { create(:project, :repository, namespace: user.namespace) }
-
-      context 'as a user' do
-        it 'returns 200 but does not change repository_storage' do
-          expect do
-            Sidekiq::Testing.fake! do
-              put(api("/projects/#{new_project.id}", user), params: { repository_storage: unknown_storage, issues_enabled: false })
-            end
-          end.not_to change(ProjectUpdateRepositoryStorageWorker.jobs, :size)
-
-          expect(response).to have_gitlab_http_status(:ok)
-          expect(json_response['issues_enabled']).to eq(false)
-          expect(new_project.reload.repository.storage).to eq('default')
-        end
-      end
-
-      context 'as an admin' do
-        include_context 'custom session'
-
-        let(:admin) { create(:admin) }
-
-        it 'returns 500 when repository storage is unknown' do
-          put(api("/projects/#{new_project.id}", admin), params: { repository_storage: unknown_storage })
-
-          expect(response).to have_gitlab_http_status(:internal_server_error)
-          expect(json_response['message']).to match('ArgumentError')
-        end
-
-        it 'returns 200 when repository storage has changed' do
-          stub_storage_settings('extra' => { 'path' => 'tmp/tests/extra_storage' })
-
-          expect do
-            Sidekiq::Testing.fake! do
-              put(api("/projects/#{new_project.id}", admin), params: { repository_storage: 'extra' })
-            end
-          end.to change(ProjectUpdateRepositoryStorageWorker.jobs, :size).by(1)
-
-          expect(response).to have_gitlab_http_status(:ok)
-        end
-      end
-    end
-
     context 'when updating mirror related attributes' do
       let(:import_url) { generate(:url) }
       let(:mirror_params) do

ee/spec/services/projects/fork_service_spec.rb

-# frozen_string_literal: true
-require 'spec_helper'
-
-# This spec lives in `ee/` since moving shards is an EE-only feature.
-describe Projects::ForkService do
-  include ProjectForksHelper
-
-  context 'when a project is already forked' do
-    it 'creates a new poolresository after the project is moved to a new shard' do
-      project = create(:project, :public, :repository)
-      fork_before_move = fork_project(project)
-
-      # Stub everything required to move a project to a Gitaly shard that does not exist
-      allow(Gitlab.config.repositories.storages).to receive(:keys).and_return(%w(default test_second_storage))
-      allow_any_instance_of(Gitlab::Git::Repository).to receive(:fetch_repository_as_mirror).and_return(true)
-
-      Projects::UpdateRepositoryStorageService.new(project).execute('test_second_storage')
-      fork_after_move = fork_project(project)
-      pool_repository_before_move = PoolRepository.joins(:shard)
-                                      .where(source_project: project, shards: { name: 'default' }).first
-      pool_repository_after_move = PoolRepository.joins(:shard)
-                                     .where(source_project: project, shards: { name: 'test_second_storage' }).first
-
-      expect(fork_before_move.pool_repository).to eq(pool_repository_before_move)
-      expect(fork_after_move.pool_repository).to eq(pool_repository_after_move)
-    end
-  end
-end

ee/spec/services/projects/update_repository_storage_service_spec.rb

@@ -8,165 +8,6 @@ describe Projects::UpdateRepositoryStorageService do
   subject { described_class.new(project) }
 
   describe "#execute" do
-    let(:time) { Time.now }
-
-    before do
-      allow(Time).to receive(:now).and_return(time)
-    end
-
-    context 'without wiki and design repository' do
-      let(:project) { create(:project, :repository, repository_read_only: true, wiki_enabled: false) }
-
-      context 'when the move succeeds' do
-        it 'moves the repository to the new storage and unmarks the repository as read only' do
-          old_path = Gitlab::GitalyClient::StorageSettings.allow_disk_access do
-            project.repository.path_to_repo
-          end
-
-          expect_any_instance_of(Gitlab::Git::Repository).to receive(:fetch_repository_as_mirror)
-            .with(project.repository.raw).and_return(true)
-
-          subject.execute('test_second_storage')
-          expect(project).not_to be_repository_read_only
-          expect(project.repository_storage).to eq('test_second_storage')
-          expect(gitlab_shell.repository_exists?('default', old_path)).to be(false)
-          expect(project.project_repository.shard_name).to eq('test_second_storage')
-        end
-      end
-
-      context 'when the project is already on the target storage' do
-        it 'bails out and does nothing' do
-          expect do
-            subject.execute(project.repository_storage)
-          end.to raise_error(described_class::RepositoryAlreadyMoved)
-        end
-      end
-
-      context 'when the move fails' do
-        it 'unmarks the repository as read-only without updating the repository storage' do
-          expect_any_instance_of(Gitlab::Git::Repository).to receive(:fetch_repository_as_mirror)
-            .with(project.repository.raw).and_return(false)
-          expect(GitlabShellWorker).not_to receive(:perform_async)
-
-          subject.execute('test_second_storage')
-
-          expect(project).not_to be_repository_read_only
-          expect(project.repository_storage).to eq('default')
-        end
-      end
-    end
-
-    shared_examples 'moves repository to another storage' do |repository_type|
-      let(:project_repository_double) { double(:repository) }
-      let(:repository_double) { double(:repository) }
-
-      before do
-        # Default stub for non-specified params
-        allow(Gitlab::Git::Repository).to receive(:new).and_call_original
-
-        allow(Gitlab::Git::Repository).to receive(:new)
-          .with('test_second_storage', project.repository.raw.relative_path, project.repository.gl_repository, project.repository.full_path)
-          .and_return(project_repository_double)
-
-        allow(Gitlab::Git::Repository).to receive(:new)
-          .with('test_second_storage', repository.raw.relative_path, repository.gl_repository, repository.full_path)
-          .and_return(repository_double)
-      end
-
-      context 'when the move succeeds', :clean_gitlab_redis_shared_state do
-        before do
-          allow(project_repository_double)
-            .to receive(:fetch_repository_as_mirror)
-            .with(project.repository.raw)
-            .and_return(true)
-
-          allow(repository_double)
-            .to receive(:fetch_repository_as_mirror)
-            .with(repository.raw)
-            .and_return(true)
-        end
-
-        it "moves the project and its #{repository_type} repository to the new storage and unmarks the repository as read only" do
-          old_project_repository_path = Gitlab::GitalyClient::StorageSettings.allow_disk_access do
-            project.repository.path_to_repo
-          end
-
-          old_repository_path = repository.full_path
-
-          subject.execute('test_second_storage')
-
-          expect(project).not_to be_repository_read_only
-          expect(project.repository_storage).to eq('test_second_storage')
-          expect(gitlab_shell.repository_exists?('default', old_project_repository_path)).to be(false)
-          expect(gitlab_shell.repository_exists?('default', old_repository_path)).to be(false)
-        end
-
-        context ':repack_after_shard_migration feature flag disabled' do
-          before do
-            stub_feature_flags(repack_after_shard_migration: false)
-          end
-
-          it 'does not enqueue a GC run' do
-            expect { subject.execute('test_second_storage') }
-              .not_to change(GitGarbageCollectWorker.jobs, :count)
-          end
-        end
-
-        context ':repack_after_shard_migration feature flag enabled' do
-          before do
-            stub_feature_flags(repack_after_shard_migration: true)
-          end
-
-          it 'does not enqueue a GC run if housekeeping is disabled' do
-            stub_application_setting(housekeeping_enabled: false)
-
-            expect { subject.execute('test_second_storage') }
-              .not_to change(GitGarbageCollectWorker.jobs, :count)
-          end
-
-          it 'enqueues a GC run' do
-            expect { subject.execute('test_second_storage') }
-              .to change(GitGarbageCollectWorker.jobs, :count).by(1)
-          end
-        end
-      end
-
-      context 'when the project is already on the target storage' do
-        it 'bails out and does nothing' do
-          expect do
-            subject.execute(project.repository_storage)
-          end.to raise_error(described_class::RepositoryAlreadyMoved)
-        end
-      end
-
-      context "when the move of the #{repository_type} repository fails" do
-        it 'unmarks the repository as read-only without updating the repository storage' do
-          allow(project_repository_double).to receive(:fetch_repository_as_mirror)
-            .with(project.repository.raw).and_return(true)
-          allow(repository_double).to receive(:fetch_repository_as_mirror)
-            .with(repository.raw).and_return(false)
-
-          expect(GitlabShellWorker).not_to receive(:perform_async)
-
-          subject.execute('test_second_storage')
-
-          expect(project).not_to be_repository_read_only
-          expect(project.repository_storage).to eq('default')
-        end
-      end
-    end
-
-    context 'with wiki repository' do
-      include_examples 'moves repository to another storage', 'wiki' do
-        let(:project) { create(:project, :repository, repository_read_only: true, wiki_enabled: true) }
-        let(:repository) { project.wiki.repository }
-
-        before do
-          project.create_wiki
-        end
-      end
-    end
-
     context 'with design repository' do
       include_examples 'moves repository to another storage', 'design' do
         let(:project) { create(:project, :repository, repository_read_only: true) }
@@ -177,18 +18,5 @@ describe Projects::UpdateRepositoryStorageService do
         end
       end
     end
-
-    context 'when a object pool was joined' do
-      let(:project) { create(:project, :repository, wiki_enabled: false, repository_read_only: true) }
-      let(:pool) { create(:pool_repository, :ready, source_project: project) }
-
-      it 'leaves the pool' do
-        allow_any_instance_of(Gitlab::Git::Repository).to receive(:fetch_repository_as_mirror).and_return(true)
-
-        subject.execute('test_second_storage')
-
-        expect(project.reload_pool_repository).to be_nil
-      end
-    end
   end
 end

ee/spec/services/projects/update_service_spec.rb

@@ -179,39 +179,6 @@ describe Projects::UpdateService, '#execute' do
     end
   end
 
-  describe 'repository_storage' do
-    let(:admin_user) { create(:user, admin: true) }
-    let(:user) { create(:user) }
-    let(:project) { create(:project, :repository) }
-    let(:opts) { { repository_storage: 'b' } }
-
-    before do
-      FileUtils.mkdir('tmp/tests/storage_b')
-
-      storages = {
-          'default' => Gitlab.config.repositories.storages.default,
-          'b' => { 'path' => 'tmp/tests/storage_b' }
-      }
-      stub_storage_settings(storages)
-    end
-
-    after do
-      FileUtils.rm_rf('tmp/tests/storage_b')
-    end
-
-    it 'calls the change repository storage method if the storage changed' do
-      expect(project).to receive(:change_repository_storage).with('b')
-
-      update_project(project, admin_user, opts).inspect
-    end
-
-    it "doesn't call the change repository storage for non-admin users" do
-      expect(project).not_to receive(:change_repository_storage)
-
-      update_project(project, user, opts).inspect
-    end
-  end
-
   context 'repository_size_limit assignment as Bytes' do
     let(:admin_user) { create(:user, admin: true) }
     let(:project) { create(:project, repository_size_limit: 0) }

lib/api/entities/project.rb

@@ -106,6 +106,9 @@ module API
         project.auto_devops.nil? ? 'continuous' : project.auto_devops.deploy_strategy
       end
       expose :autoclose_referenced_issues
+      expose :repository_storage, if: ->(project, options) {
+        Ability.allowed?(options[:current_user], :change_repository_storage, project)
+      }
 
       # rubocop: disable CodeReuse/ActiveRecord
       def self.preload_relation(projects_relation, options = {})

lib/api/helpers/projects_helpers.rb

@@ -54,6 +54,7 @@ module API
         optional :auto_devops_enabled, type: Boolean, desc: 'Flag indication if Auto DevOps is enabled'
         optional :auto_devops_deploy_strategy, type: String, values: %w(continuous manual timed_incremental), desc: 'Auto Deploy strategy'
         optional :autoclose_referenced_issues, type: Boolean, desc: 'Flag indication if referenced issues auto-closing is enabled'
+        optional :repository_storage, type: String, desc: 'Which storage shard the repository is on. Available only to admins'
       end
 
       params :optional_project_params_ee do
@@ -125,6 +126,7 @@ module API
           :wiki_access_level,
           :avatar,
           :suggestion_commit_message,
+          :repository_storage,
 
           # TODO: remove in API v5, replaced by *_access_level
           :issues_enabled,

lib/api/projects.rb

@@ -25,6 +25,7 @@ module API
       end
 
       def verify_update_project_attrs!(project, attrs)
+        attrs.delete(:repository_storage) unless can?(current_user, :change_repository_storage, project)
       end
 
       def delete_project(user_project)

spec/models/project_spec.rb

@@ -2822,6 +2822,44 @@ describe Project do
     end
   end
 
+  describe '#change_repository_storage' do
+    let(:project) { create(:project, :repository) }
+    let(:read_only_project) { create(:project, :repository, repository_read_only: true) }
+
+    before do
+      stub_storage_settings('test_second_storage' => { 'path' => 'tmp/tests/extra_storage' })
+    end
+
+    it 'schedules the transfer of the repository to the new storage and locks the project' do
+      expect(ProjectUpdateRepositoryStorageWorker).to receive(:perform_async).with(project.id, 'test_second_storage')
+
+      project.change_repository_storage('test_second_storage')
+      project.save!
+
+      expect(project).to be_repository_read_only
+    end
+
+    it "doesn't schedule the transfer if the repository is already read-only" do
+      expect(ProjectUpdateRepositoryStorageWorker).not_to receive(:perform_async)
+
+      read_only_project.change_repository_storage('test_second_storage')
+      read_only_project.save!
+    end
+
+    it "doesn't lock or schedule the transfer if the storage hasn't changed" do
+      expect(ProjectUpdateRepositoryStorageWorker).not_to receive(:perform_async)
+
+      project.change_repository_storage(project.repository_storage)
+      project.save!
+
+      expect(project).not_to be_repository_read_only
+    end
+
+    it 'throws an error if an invalid repository storage is provided' do
+      expect { project.change_repository_storage('unknown') }.to raise_error(ArgumentError)
+    end
+  end
+
   describe '#pushes_since_gc' do
     let(:project) { create(:project) }
 

spec/requests/api/projects_spec.rb

@@ -1751,6 +1751,27 @@ describe API::Projects do
 
       subject { get api("/projects/#{project.id}", user) }
     end
+
+    describe 'repository_storage attribute' do
+      before do
+        get api("/projects/#{project.id}", user)
+      end
+
+      context 'when authenticated as an admin' do
+        let(:user) { create(:admin) }
+
+        it 'returns repository_storage attribute' do
+          expect(response).to have_gitlab_http_status(:ok)
+          expect(json_response['repository_storage']).to eq(project.repository_storage)
+        end
+      end
+
+      context 'when authenticated as a regular user' do
+        it 'does not return repository_storage attribute' do
+          expect(json_response).not_to have_key('repository_storage')
+        end
+      end
+    end
   end
 
   describe 'GET /projects/:id/users' do
@@ -2402,6 +2423,50 @@ describe API::Projects do
         expect(response).to have_gitlab_http_status(:forbidden)
       end
     end
+
+    context 'when updating repository storage' do
+      let(:unknown_storage) { 'new-storage' }
+      let(:new_project) { create(:project, :repository, namespace: user.namespace) }
+
+      context 'as a user' do
+        it 'returns 200 but does not change repository_storage' do
+          expect do
+            Sidekiq::Testing.fake! do
+              put(api("/projects/#{new_project.id}", user), params: { repository_storage: unknown_storage, issues_enabled: false })
+            end
+          end.not_to change(ProjectUpdateRepositoryStorageWorker.jobs, :size)
+
+          expect(response).to have_gitlab_http_status(:ok)
+          expect(json_response['issues_enabled']).to eq(false)
+          expect(new_project.reload.repository.storage).to eq('default')
+        end
+      end
+
+      context 'as an admin' do
+        include_context 'custom session'
+
+        let(:admin) { create(:admin) }
+
+        it 'returns 500 when repository storage is unknown' do
+          put(api("/projects/#{new_project.id}", admin), params: { repository_storage: unknown_storage })
+
+          expect(response).to have_gitlab_http_status(:internal_server_error)
+          expect(json_response['message']).to match('ArgumentError')
+        end
+
+        it 'returns 200 when repository storage has changed' do
+          stub_storage_settings('test_second_storage' => { 'path' => 'tmp/tests/second_storage' })
+
+          expect do
+            Sidekiq::Testing.fake! do
+              put(api("/projects/#{new_project.id}", admin), params: { repository_storage: 'test_second_storage' })
+            end
+          end.to change(ProjectUpdateRepositoryStorageWorker.jobs, :size).by(1)
+
+          expect(response).to have_gitlab_http_status(:ok)
+        end
+      end
+    end
   end
 
   describe 'POST /projects/:id/archive' do

spec/services/projects/fork_service_spec.rb

@@ -307,6 +307,27 @@ describe Projects::ForkService do
     end
   end
 
+  context 'when a project is already forked' do
+    it 'creates a new poolresository after the project is moved to a new shard' do
+      project = create(:project, :public, :repository)
+      fork_before_move = fork_project(project)
+
+      # Stub everything required to move a project to a Gitaly shard that does not exist
+      stub_storage_settings('test_second_storage' => { 'path' => 'tmp/tests/second_storage' })
+      allow_any_instance_of(Gitlab::Git::Repository).to receive(:fetch_repository_as_mirror).and_return(true)
+
+      Projects::UpdateRepositoryStorageService.new(project).execute('test_second_storage')
+      fork_after_move = fork_project(project)
+      pool_repository_before_move = PoolRepository.joins(:shard)
+                                      .where(source_project: project, shards: { name: 'default' }).first
+      pool_repository_after_move = PoolRepository.joins(:shard)
+                                     .where(source_project: project, shards: { name: 'test_second_storage' }).first
+
+      expect(fork_before_move.pool_repository).to eq(pool_repository_before_move)
+      expect(fork_after_move.pool_repository).to eq(pool_repository_after_move)
+    end
+  end
+
   context 'when forking with object pools' do
     let(:fork_from_project) { create(:project, :public) }
     let(:forker) { create(:user) }

spec/services/projects/update_repository_storage_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Projects::UpdateRepositoryStorageService do
+  include Gitlab::ShellAdapter
+
+  subject { described_class.new(project) }
+
+  describe "#execute" do
+    let(:time) { Time.now }
+
+    before do
+      allow(Time).to receive(:now).and_return(time)
+    end
+
+    context 'without wiki and design repository' do
+      let(:project) { create(:project, :repository, repository_read_only: true, wiki_enabled: false) }
+
+      context 'when the move succeeds' do
+        it 'moves the repository to the new storage and unmarks the repository as read only' do
+          old_path = Gitlab::GitalyClient::StorageSettings.allow_disk_access do
+            project.repository.path_to_repo
+          end
+
+          expect_any_instance_of(Gitlab::Git::Repository).to receive(:fetch_repository_as_mirror)
+            .with(project.repository.raw).and_return(true)
+
+          subject.execute('test_second_storage')
+          expect(project).not_to be_repository_read_only
+          expect(project.repository_storage).to eq('test_second_storage')
+          expect(gitlab_shell.repository_exists?('default', old_path)).to be(false)
+          expect(project.project_repository.shard_name).to eq('test_second_storage')
+        end
+      end
+
+      context 'when the project is already on the target storage' do
+        it 'bails out and does nothing' do
+          expect do
+            subject.execute(project.repository_storage)
+          end.to raise_error(described_class::RepositoryAlreadyMoved)
+        end
+      end
+
+      context 'when the move fails' do
+        it 'unmarks the repository as read-only without updating the repository storage' do
+          expect_any_instance_of(Gitlab::Git::Repository).to receive(:fetch_repository_as_mirror)
+            .with(project.repository.raw).and_return(false)
+          expect(GitlabShellWorker).not_to receive(:perform_async)
+
+          subject.execute('test_second_storage')
+
+          expect(project).not_to be_repository_read_only
+          expect(project.repository_storage).to eq('default')
+        end
+      end
+    end
+
+    context 'with wiki repository' do
+      include_examples 'moves repository to another storage', 'wiki' do
+        let(:project) { create(:project, :repository, repository_read_only: true, wiki_enabled: true) }
+        let(:repository) { project.wiki.repository }
+
+        before do
+          project.create_wiki
+        end
+      end
+    end
+
+    context 'when a object pool was joined' do
+      let(:project) { create(:project, :repository, wiki_enabled: false, repository_read_only: true) }
+      let(:pool) { create(:pool_repository, :ready, source_project: project) }
+
+      it 'leaves the pool' do
+        allow_any_instance_of(Gitlab::Git::Repository).to receive(:fetch_repository_as_mirror).and_return(true)
+
+        subject.execute('test_second_storage')
+
+        expect(project.reload_pool_repository).to be_nil
+      end
+    end
+  end
+end

spec/services/projects/update_service_spec.rb

@@ -613,6 +613,25 @@ describe Projects::UpdateService do
     end
   end
 
+  describe 'repository_storage' do
+    let(:admin) { create(:admin) }
+    let(:user) { create(:user) }
+    let(:project) { create(:project, :repository) }
+    let(:opts) { { repository_storage: 'test_second_storage' } }
+
+    it 'calls the change repository storage method if the storage changed' do
+      expect(project).to receive(:change_repository_storage).with('test_second_storage')
+
+      update_project(project, admin, opts).inspect
+    end
+
+    it "doesn't call the change repository storage for non-admin users" do
+      expect(project).not_to receive(:change_repository_storage)
+
+      update_project(project, user, opts).inspect
+    end
+  end
+
   def update_project(project, user, opts)
     described_class.new(project, user, opts).execute
   end

spec/support/shared_examples/services/projects/update_repository_storage_service_shared_examples.rb

+# frozen_string_literal: true
+
+RSpec.shared_examples 'moves repository to another storage' do |repository_type|
+  let(:project_repository_double) { double(:repository) }
+  let(:repository_double) { double(:repository) }
+
+  before do
+    # Default stub for non-specified params
+    allow(Gitlab::Git::Repository).to receive(:new).and_call_original
+
+    allow(Gitlab::Git::Repository).to receive(:new)
+      .with('test_second_storage', project.repository.raw.relative_path, project.repository.gl_repository, project.repository.full_path)
+      .and_return(project_repository_double)
+
+    allow(Gitlab::Git::Repository).to receive(:new)
+      .with('test_second_storage', repository.raw.relative_path, repository.gl_repository, repository.full_path)
+      .and_return(repository_double)
+  end
+
+  context 'when the move succeeds', :clean_gitlab_redis_shared_state do
+    before do
+      allow(project_repository_double)
+        .to receive(:fetch_repository_as_mirror)
+        .with(project.repository.raw)
+        .and_return(true)
+
+      allow(repository_double)
+        .to receive(:fetch_repository_as_mirror)
+        .with(repository.raw)
+        .and_return(true)
+    end
+
+    it "moves the project and its #{repository_type} repository to the new storage and unmarks the repository as read only" do
+      old_project_repository_path = Gitlab::GitalyClient::StorageSettings.allow_disk_access do
+        project.repository.path_to_repo
+      end
+
+      old_repository_path = repository.full_path
+
+      subject.execute('test_second_storage')
+
+      expect(project).not_to be_repository_read_only
+      expect(project.repository_storage).to eq('test_second_storage')
+      expect(gitlab_shell.repository_exists?('default', old_project_repository_path)).to be(false)
+      expect(gitlab_shell.repository_exists?('default', old_repository_path)).to be(false)
+    end
+
+    context ':repack_after_shard_migration feature flag disabled' do
+      before do
+        stub_feature_flags(repack_after_shard_migration: false)
+      end
+
+      it 'does not enqueue a GC run' do
+        expect { subject.execute('test_second_storage') }
+          .not_to change(GitGarbageCollectWorker.jobs, :count)
+      end
+    end
+
+    context ':repack_after_shard_migration feature flag enabled' do
+      before do
+        stub_feature_flags(repack_after_shard_migration: true)
+      end
+
+      it 'does not enqueue a GC run if housekeeping is disabled' do
+        stub_application_setting(housekeeping_enabled: false)
+
+        expect { subject.execute('test_second_storage') }
+          .not_to change(GitGarbageCollectWorker.jobs, :count)
+      end
+
+      it 'enqueues a GC run' do
+        expect { subject.execute('test_second_storage') }
+          .to change(GitGarbageCollectWorker.jobs, :count).by(1)
+      end
+    end
+  end
+
+  context 'when the project is already on the target storage' do
+    it 'bails out and does nothing' do
+      expect do
+        subject.execute(project.repository_storage)
+      end.to raise_error(described_class::RepositoryAlreadyMoved)
+    end
+  end
+
+  context "when the move of the #{repository_type} repository fails" do
+    it 'unmarks the repository as read-only without updating the repository storage' do
+      allow(project_repository_double).to receive(:fetch_repository_as_mirror)
+        .with(project.repository.raw).and_return(true)
+      allow(repository_double).to receive(:fetch_repository_as_mirror)
+        .with(repository.raw).and_return(false)
+
+      expect(GitlabShellWorker).not_to receive(:perform_async)
+
+      subject.execute('test_second_storage')
+
+      expect(project).not_to be_repository_read_only
+      expect(project.repository_storage).to eq('default')
+    end
+  end
+end