Merge branch 'tc-clean-pending-delete-projects' into 'master'

Post-deploy to clean `pending_delete` projects

Closes #20984

See merge request !11044

app/workers/namespaceless_project_destroy_worker.rb

+# Worker to destroy projects that do not have a namespace
+#
+# It destroys everything it can without having the info about the namespace it
+# used to belong to. Projects in this state should be rare.
+# The worker will reject doing anything for projects that *do* have a
+# namespace. For those use ProjectDestroyWorker instead.
+class NamespacelessProjectDestroyWorker
+  include Sidekiq::Worker
+  include DedicatedSidekiqQueue
+
+  def self.bulk_perform_async(args_list)
+    Sidekiq::Client.push_bulk('class' => self, 'queue' => sidekiq_options['queue'], 'args' => args_list)
+  end
+
+  def perform(project_id)
+    begin
+      project = Project.unscoped.find(project_id)
+    rescue ActiveRecord::RecordNotFound
+      return
+    end
+    return unless project.namespace_id.nil?  # Reject doing anything for projects that *do* have a namespace
+
+    project.team.truncate
+
+    unlink_fork(project) if project.forked?
+
+    # Override Project#remove_pages for this instance so it doesn't do anything
+    def project.remove_pages
+    end
+
+    project.destroy!
+  end
+
+  private
+
+  def unlink_fork(project)
+    merge_requests = project.forked_from_project.merge_requests.opened.from_project(project)
+
+    merge_requests.update_all(state: 'closed')
+
+    project.forked_project_link.destroy
+  end
+end

changelogs/unreleased/tc-clean-pending-delete-projects.yml

+---
+title: Add post-deploy migration to clean up projects in `pending_delete` state
+merge_request: 11044
+author:

config/sidekiq_queues.yml

@@ -40,6 +40,7 @@
   - [expire_build_instance_artifacts, 1]
   - [group_destroy, 1]
   - [irker, 1]
+  - [namespaceless_project_destroy, 1]
   - [project_cache, 1]
   - [project_destroy, 1]
   - [project_export, 1]

db/post_migrate/20170502101023_cleanup_namespaceless_pending_delete_projects.rb

+# This is the counterpart of RequeuePendingDeleteProjects and cleans all
+# projects with `pending_delete = true` and that do not have a namespace.
+class CleanupNamespacelessPendingDeleteProjects < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    @offset = 0
+
+    loop do
+      ids = pending_delete_batch
+
+      break if ids.empty?
+
+      args = ids.map { |id| Array(id) }
+
+      NamespacelessProjectDestroyWorker.bulk_perform_async(args)
+
+      @offset += 1
+    end
+  end
+
+  def down
+    # noop
+  end
+
+  private
+
+  def pending_delete_batch
+    connection.exec_query(find_batch).map{ |row| row['id'] }
+  end
+
+  BATCH_SIZE = 5000
+
+  def find_batch
+    projects = Arel::Table.new(:projects)
+    projects.project(projects[:id]).
+      where(projects[:pending_delete].eq(true)).
+      where(projects[:namespace_id].eq(nil)).
+      skip(@offset * BATCH_SIZE).
+      take(BATCH_SIZE).
+      to_sql
+  end
+end

spec/migrations/cleanup_namespaceless_pending_delete_projects_spec.rb

+require 'spec_helper'
+require Rails.root.join('db', 'post_migrate', '20170502101023_cleanup_namespaceless_pending_delete_projects.rb')
+
+describe CleanupNamespacelessPendingDeleteProjects do
+  before do
+    # Stub after_save callbacks that will fail when Project has no namespace
+    allow_any_instance_of(Project).to receive(:ensure_dir_exist).and_return(nil)
+    allow_any_instance_of(Project).to receive(:update_project_statistics).and_return(nil)
+  end
+
+  describe '#up' do
+    it 'only cleans up pending delete projects' do
+      create(:empty_project)
+      create(:empty_project, pending_delete: true)
+      project = build(:empty_project, pending_delete: true, namespace_id: nil)
+      project.save(validate: false)
+
+      expect(NamespacelessProjectDestroyWorker).to receive(:bulk_perform_async).with([[project.id.to_s]])
+
+      described_class.new.up
+    end
+
+    it 'does nothing when no pending delete projects without namespace found' do
+      create(:empty_project)
+      create(:empty_project, pending_delete: true)
+
+      expect(NamespacelessProjectDestroyWorker).not_to receive(:bulk_perform_async)
+
+      described_class.new.up
+    end
+  end
+end

spec/workers/namespaceless_project_destroy_worker_spec.rb

+require 'spec_helper'
+
+describe NamespacelessProjectDestroyWorker do
+  subject { described_class.new }
+
+  before do
+    # Stub after_save callbacks that will fail when Project has no namespace
+    allow_any_instance_of(Project).to receive(:ensure_dir_exist).and_return(nil)
+    allow_any_instance_of(Project).to receive(:update_project_statistics).and_return(nil)
+  end
+
+  describe '#perform' do
+    context 'project has namespace' do
+      it 'does not do anything' do
+        project = create(:empty_project)
+
+        subject.perform(project.id)
+
+        expect(Project.unscoped.all).to include(project)
+      end
+    end
+
+    context 'project has no namespace' do
+      let!(:project) do
+        project = build(:empty_project, namespace_id: nil)
+        project.save(validate: false)
+        project
+      end
+
+      context 'project not a fork of another project' do
+        it "truncates the project's team" do
+          expect_any_instance_of(ProjectTeam).to receive(:truncate)
+
+          subject.perform(project.id)
+        end
+
+        it 'deletes the project' do
+          subject.perform(project.id)
+
+          expect(Project.unscoped.all).not_to include(project)
+        end
+
+        it 'does not call unlink_fork' do
+          is_expected.not_to receive(:unlink_fork)
+
+          subject.perform(project.id)
+        end
+
+        it 'does not do anything in Project#remove_pages method' do
+          expect(Gitlab::PagesTransfer).not_to receive(:new)
+
+          subject.perform(project.id)
+        end
+      end
+
+      context 'project forked from another' do
+        let!(:parent_project) { create(:empty_project) }
+
+        before do
+          create(:forked_project_link, forked_to_project: project, forked_from_project: parent_project)
+        end
+
+        it 'closes open merge requests' do
+          merge_request = create(:merge_request, source_project: project, target_project: parent_project)
+
+          subject.perform(project.id)
+
+          expect(merge_request.reload).to be_closed
+        end
+
+        it 'destroys the link' do
+          subject.perform(project.id)
+
+          expect(parent_project.forked_project_links).to be_empty
+        end
+      end
+    end
+  end
+end