Merge branch 'georgekoltsov/update-project-overwrite-service' into 'master'

Update Import's OverwriteProjectService to remove cross-db transaction

See merge request gitlab-org/gitlab!79630

app/services/projects/overwrite_project_service.rb

@@ -6,30 +6,34 @@ module Projects
       return unless source_project && source_project.namespace_id == @project.namespace_id
 
       start_time = ::Gitlab::Metrics::System.monotonic_time
+      original_source_name = source_project.name
+      original_source_path = source_project.path
+      tmp_source_name, tmp_source_path = tmp_source_project_name(source_project)
 
-      Project.transaction do
-        move_before_destroy_relationships(source_project)
-        # Reset is required in order to get the proper
-        # uncached fork network method calls value.
-        ::Gitlab::Database::QueryAnalyzers::PreventCrossDatabaseModification.allow_cross_database_modification_within_transaction(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/340256') do
-          destroy_old_project(source_project.reset)
-        end
-        rename_project(source_project.name, source_project.path)
-
-        @project
+      move_relationships_between(source_project, @project)
+
+      source_project_rename = rename_project(source_project, tmp_source_name, tmp_source_path)
+
+      if source_project_rename[:status] == :error
+        raise 'Source project rename failed during project overwrite'
       end
-    # Projects::DestroyService can raise Exceptions, but we don't want
-    # to pass that kind of exception to the caller. Instead, we change it
-    # for a StandardError exception
-    rescue Exception => e # rubocop:disable Lint/RescueException
-      attempt_restore_repositories(source_project)
-
-      if e.instance_of?(Exception)
-        raise StandardError, e.message
-      else
-        raise
+
+      new_project_rename = rename_project(@project, original_source_name, original_source_path)
+
+      if new_project_rename[:status] == :error
+        rename_project(source_project, original_source_name, original_source_path)
+
+        raise 'New project rename failed during project overwrite'
       end
 
+      schedule_source_project_deletion(source_project)
+
+      @project
+    rescue StandardError => e
+      move_relationships_between(@project, source_project)
+      remove_source_project_from_fork_network(source_project)
+
+      raise e
     ensure
       track_service(start_time, source_project, e)
     end
@@ -48,45 +52,63 @@ module Projects
                                  error: exception.class.name)
     end
 
-    def move_before_destroy_relationships(source_project)
+    def move_relationships_between(source_project, target_project)
       options = { remove_remaining_elements: false }
 
-      ::Projects::MoveUsersStarProjectsService.new(@project, @current_user).execute(source_project, **options)
-      ::Projects::MoveAccessService.new(@project, @current_user).execute(source_project, **options)
-      ::Projects::MoveDeployKeysProjectsService.new(@project, @current_user).execute(source_project, **options)
-      ::Projects::MoveNotificationSettingsService.new(@project, @current_user).execute(source_project, **options)
-      ::Projects::MoveForksService.new(@project, @current_user).execute(source_project, **options)
-      ::Projects::MoveLfsObjectsProjectsService.new(@project, @current_user).execute(source_project, **options)
-      add_source_project_to_fork_network(source_project)
-    end
-
-    def destroy_old_project(source_project)
-      # Delete previous project (synchronously) and unlink relations
-      ::Projects::DestroyService.new(source_project, @current_user).execute
+      Project.transaction do
+        ::Projects::MoveUsersStarProjectsService.new(target_project, @current_user).execute(source_project, **options)
+        ::Projects::MoveAccessService.new(target_project, @current_user).execute(source_project, **options)
+        ::Projects::MoveDeployKeysProjectsService.new(target_project, @current_user).execute(source_project, **options)
+        ::Projects::MoveNotificationSettingsService.new(target_project, @current_user).execute(source_project, **options)
+        ::Projects::MoveForksService.new(target_project, @current_user).execute(source_project, **options)
+        ::Projects::MoveLfsObjectsProjectsService.new(target_project, @current_user).execute(source_project, **options)
+
+        add_source_project_to_fork_network(source_project)
+      end
     end
 
-    def rename_project(name, path)
-      # Update de project's name and path to the original name/path
-      ::Projects::UpdateService.new(@project,
-                                    @current_user,
-                                    { name: name, path: path })
-                               .execute
+    def schedule_source_project_deletion(source_project)
+      ::Projects::DestroyService.new(source_project, @current_user).async_execute
     end
 
-    def attempt_restore_repositories(project)
-      ::Projects::DestroyRollbackService.new(project, @current_user).execute
+    def rename_project(target_project, name, path)
+      ::Projects::UpdateService.new(target_project, @current_user, { name: name, path: path }).execute
     end
 
     def add_source_project_to_fork_network(source_project)
-      return unless @project.fork_network
+      return if source_project == @project
+      return unless fork_network
 
       # Because they have moved all references in the fork network from the source_project
       # we won't be able to query the database (only through its cached data),
       # for its former relationships. That's why we're adding it to the network
       # as a fork of the target project
-      ForkNetworkMember.create!(fork_network: @project.fork_network,
+      ForkNetworkMember.create!(fork_network: fork_network,
                                 project: source_project,
                                 forked_from_project: @project)
     end
+
+    def remove_source_project_from_fork_network(source_project)
+      return unless fork_network
+
+      fork_member = ForkNetworkMember.find_by( # rubocop: disable CodeReuse/ActiveRecord
+        fork_network: fork_network,
+        project: source_project,
+        forked_from_project: @project)
+
+      fork_member&.destroy
+    end
+
+    def tmp_source_project_name(source_project)
+      random_string = SecureRandom.hex
+      tmp_name = "#{source_project.name}-old-#{random_string}"
+      tmp_path = "#{source_project.path}-old-#{random_string}"
+
+      [tmp_name, tmp_path]
+    end
+
+    def fork_network
+      @project.fork_network_member&.fork_network
+    end
   end
 end

spec/services/projects/overwrite_project_service_spec.rb

@@ -81,16 +81,58 @@ RSpec.describe Projects::OverwriteProjectService do
         end
       end
 
-      it 'removes the original project' do
-        subject.execute(project_from)
+      it 'schedules original project for deletion' do
+        expect_next_instance_of(Projects::DestroyService) do |service|
+          expect(service).to receive(:async_execute)
+        end
 
-        expect { Project.find(project_from.id) }.to raise_error(ActiveRecord::RecordNotFound)
+        subject.execute(project_from)
       end
 
       it 'renames the project' do
+        original_path = project_from.full_path
+
         subject.execute(project_from)
 
-        expect(project_to.full_path).to eq project_from.full_path
+        expect(project_to.full_path).to eq(original_path)
+      end
+
+      it 'renames source project to temp name' do
+        allow(SecureRandom).to receive(:hex).and_return('test')
+
+        subject.execute(project_from)
+
+        expect(project_from.full_path).to include('-old-test')
+      end
+
+      context 'when project rename fails' do
+        before do
+          expect(subject).to receive(:move_relationships_between).with(project_from, project_to)
+          expect(subject).to receive(:move_relationships_between).with(project_to, project_from)
+        end
+
+        context 'source rename' do
+          it 'moves relations back to source project and raises an exception' do
+            allow(subject).to receive(:rename_project).and_return(status: :error)
+
+            expect { subject.execute(project_from) }.to raise_error(StandardError, 'Source project rename failed during project overwrite')
+          end
+        end
+
+        context 'new project rename' do
+          it 'moves relations back, renames source project back to original name and raises' do
+            name = project_from.name
+            path = project_from.path
+
+            allow(subject).to receive(:rename_project).and_call_original
+            allow(subject).to receive(:rename_project).with(project_to, name, path).and_return(status: :error)
+
+            expect { subject.execute(project_from) }.to raise_error(StandardError, 'New project rename failed during project overwrite')
+
+            expect(project_from.name).to eq(name)
+            expect(project_from.path).to eq(path)
+          end
+        end
       end
     end
 
@@ -121,7 +163,7 @@ RSpec.describe Projects::OverwriteProjectService do
       end
     end
 
-    context 'forks' do
+    context 'forks', :sidekiq_inline do
       context 'when moving a root forked project' do
         it 'moves the descendant forks' do
           expect(project_from.forks.count).to eq 2
@@ -147,6 +189,7 @@ RSpec.describe Projects::OverwriteProjectService do
           expect(project_to.fork_network.fork_network_members.map(&:project)).not_to include project_from
         end
       end
+
       context 'when moving a intermediate forked project' do
         let(:project_to) { create(:project, namespace: lvl1_forked_project_1.namespace) }
 
@@ -180,22 +223,26 @@ RSpec.describe Projects::OverwriteProjectService do
     end
 
     context 'if an exception is raised' do
+      before do
+        allow(subject).to receive(:rename_project).and_raise(StandardError)
+      end
+
       it 'rollbacks changes' do
         updated_at = project_from.updated_at
 
-        allow(subject).to receive(:rename_project).and_raise(StandardError)
-
         expect { subject.execute(project_from) }.to raise_error(StandardError)
         expect(Project.find(project_from.id)).not_to be_nil
         expect(project_from.reload.updated_at.change(usec: 0)).to eq updated_at.change(usec: 0)
       end
 
-      it 'tries to restore the original project repositories' do
-        allow(subject).to receive(:rename_project).and_raise(StandardError)
-
-        expect(subject).to receive(:attempt_restore_repositories).with(project_from)
+      it 'removes fork network member' do
+        expect(ForkNetworkMember).to receive(:create!)
+        expect(ForkNetworkMember).to receive(:find_by)
+        expect(subject).to receive(:remove_source_project_from_fork_network).and_call_original
 
         expect { subject.execute(project_from) }.to raise_error(StandardError)
+
+        expect(project_from.fork_network_member).to be_nil
       end
     end
   end