Merge branch 'jc-remove-cleanup-rake-tasks' into 'master'

Removing cleanup:repo, cleanup:dirs

Closes #24633

See merge request gitlab-org/gitlab!18087

changelogs/unreleased/jc-remove-cleanup-rake-tasks.yml

+---
+title: Removing cleanup:repo, cleanup:dirs
+merge_request: 18087
+author:
+type: deprecated

doc/raketasks/cleanup.md

@@ -2,46 +2,6 @@
 
 ## Remove garbage from filesystem
 
-DANGER: **Danger:**
-The commands below will remove data permanently from your GitLab instance. Only use
-these commands if you are 100% certain that it is safe to delete this data.
-
-Remove namespaces(dirs) from all repository storage paths if they don't exist in GitLab database.
-
-```
-# omnibus-gitlab
-sudo gitlab-rake gitlab:cleanup:dirs
-
-# installation from source
-bundle exec rake gitlab:cleanup:dirs RAILS_ENV=production
-```
-
-DANGER: **Danger:**
-The following task does not currently work as expected.
-The use will probably mark more existing repositories as orphaned.
-For more information, see the [issue](https://gitlab.com/gitlab-org/gitlab/issues/24633).
-
-Rename repositories from all repository storage paths if they don't exist in GitLab database.
-The repositories get a `+orphaned+TIMESTAMP` suffix so that they cannot block new repositories from being created.
-
-```
-# omnibus-gitlab
-sudo gitlab-rake gitlab:cleanup:repos
-
-# installation from source
-bundle exec rake gitlab:cleanup:repos RAILS_ENV=production
-```
-
-Remove old repository copies from repositories moved to another storage.
-
-```
-# omnibus-gitlab
-sudo gitlab-rake gitlab:cleanup:moved
-
-# installation from source
-bundle exec rake gitlab:cleanup:moved RAILS_ENV=production
-```
-
 Clean up local project upload files if they don't exist in GitLab database. The
 task attempts to fix the file if it can find its project, otherwise it moves the
 file to a lost and found directory.

lib/tasks/gitlab/cleanup.rake

@@ -3,69 +3,6 @@ require 'set'
 
 namespace :gitlab do
   namespace :cleanup do
-    desc "GitLab | Cleanup | Clean namespaces"
-    task dirs: :gitlab_environment do
-      namespaces = Set.new(Namespace.pluck(:path))
-      namespaces << Storage::HashedProject::REPOSITORY_PATH_PREFIX
-
-      Gitaly::Server.all.each do |server|
-        all_dirs = Gitlab::GitalyClient::StorageService
-          .new(server.storage)
-          .list_directories(depth: 0)
-          .reject { |dir| dir.ends_with?('.git') || namespaces.include?(File.basename(dir)) }
-
-        puts "Looking for directories to remove... "
-        all_dirs.each do |dir_path|
-          if remove?
-            begin
-              Gitlab::GitalyClient::NamespaceService.new(server.storage)
-                .remove(dir_path)
-
-              puts "Removed...#{dir_path}"
-            rescue StandardError => e
-              puts "Cannot remove #{dir_path}: #{e.message}".color(:red)
-            end
-          else
-            puts "Can be removed: #{dir_path}".color(:red)
-          end
-        end
-      end
-
-      unless remove?
-        puts "To cleanup this directories run this command with REMOVE=true".color(:yellow)
-      end
-    end
-
-    desc "GitLab | Cleanup | Clean repositories"
-    task repos: :gitlab_environment do
-      move_suffix = "+orphaned+#{Time.now.to_i}"
-
-      Gitaly::Server.all.each do |server|
-        Gitlab::GitalyClient::StorageService
-          .new(server.storage)
-          .list_directories
-          .each do |path|
-          repo_with_namespace = path.chomp('.git').chomp('.wiki')
-
-          # TODO ignoring hashed repositories for now.  But revisit to fully support
-          # possible orphaned hashed repos
-          next if repo_with_namespace.start_with?(Storage::HashedProject::REPOSITORY_PATH_PREFIX)
-          next if Project.find_by_full_path(repo_with_namespace)
-
-          new_path = path + move_suffix
-          puts path.inspect + ' -> ' + new_path.inspect
-
-          begin
-            Gitlab::GitalyClient::NamespaceService
-              .new(server.storage)
-              .rename(path, new_path)
-          rescue StandardError => e
-            puts "Error occurred while moving the repository: #{e.message}".color(:red)
-          end
-        end
-      end
-    end
-
     desc "GitLab | Cleanup | Block users that have been removed in LDAP"
     task block_removed_ldap_users: :gitlab_environment do
       warn_user_is_not_gitlab

spec/tasks/gitlab/cleanup_rake_spec.rb

@@ -5,74 +5,6 @@ describe 'gitlab:cleanup rake tasks' do
     Rake.application.rake_require 'tasks/gitlab/cleanup'
   end
 
-  describe 'cleanup namespaces and repos' do
-    let(:gitlab_shell) { Gitlab::Shell.new }
-    let(:storage) { storages.keys.first }
-    let(:storages) do
-      {
-        'default' => Gitlab::GitalyClient::StorageSettings.new(@default_storage_hash.merge('path' => 'tmp/tests/default_storage'))
-      }
-    end
-
-    before(:all) do
-      @default_storage_hash = Gitlab.config.repositories.storages.default.to_h
-    end
-
-    before do
-      allow(Gitlab.config.repositories).to receive(:storages).and_return(storages)
-    end
-
-    after do
-      Gitlab::GitalyClient::StorageService.new(storage).delete_all_repositories
-    end
-
-    describe 'cleanup:repos' do
-      before do
-        gitlab_shell.add_namespace(storage, 'broken/project.git')
-        gitlab_shell.add_namespace(storage, '@hashed/12/34/5678.git')
-      end
-
-      it 'moves it to an orphaned path' do
-        now = Time.now
-
-        Timecop.freeze(now) do
-          run_rake_task('gitlab:cleanup:repos')
-          repo_list = Gitlab::GitalyClient::StorageService.new(storage).list_directories(depth: 0)
-
-          expect(repo_list.last).to include("broken+orphaned+#{now.to_i}")
-        end
-      end
-
-      it 'ignores @hashed repos' do
-        run_rake_task('gitlab:cleanup:repos')
-
-        expect(gitlab_shell.exists?(storage, '@hashed/12/34/5678.git')).to be(true)
-      end
-    end
-
-    describe 'cleanup:dirs' do
-      it 'removes missing namespaces' do
-        gitlab_shell.add_namespace(storage, "namespace_1/project.git")
-        gitlab_shell.add_namespace(storage, "namespace_2/project.git")
-        allow(Namespace).to receive(:pluck).and_return(['namespace_1'])
-
-        stub_env('REMOVE', 'true')
-        run_rake_task('gitlab:cleanup:dirs')
-
-        expect(gitlab_shell.exists?(storage, 'namespace_1')).to be(true)
-        expect(gitlab_shell.exists?(storage, 'namespace_2')).to be(false)
-      end
-
-      it 'ignores @hashed directory' do
-        gitlab_shell.add_namespace(storage, '@hashed/12/34/5678.git')
-
-        run_rake_task('gitlab:cleanup:dirs')
-
-        expect(gitlab_shell.exists?(storage, '@hashed/12/34/5678.git')).to be(true)
-      end
-    end
-  end
-
   # A single integration test that is redundant with one part of the
   # Gitlab::Cleanup::ProjectUploads spec.
   #