cleanup.rake 4.94 KB
Newer Older
1 2 3
# frozen_string_literal: true
require 'set'

Dmitriy Zaporozhets's avatar
Dmitriy Zaporozhets committed
4 5
namespace :gitlab do
  namespace :cleanup do
6
    desc "GitLab | Cleanup | Clean namespaces"
7
    task dirs: :gitlab_environment do
8
      namespaces = Set.new(Namespace.pluck(:path))
9
      namespaces << Storage::HashedProject::REPOSITORY_PATH_PREFIX
Dmitriy Zaporozhets's avatar
Dmitriy Zaporozhets committed
10

11 12 13 14 15
      Gitaly::Server.all.each do |server|
        all_dirs = Gitlab::GitalyClient::StorageService
          .new(server.storage)
          .list_directories(depth: 0)
          .reject { |dir| dir.ends_with?('.git') || namespaces.include?(File.basename(dir)) }
Dmitriy Zaporozhets's avatar
Dmitriy Zaporozhets committed
16

17 18
        puts "Looking for directories to remove... "
        all_dirs.each do |dir_path|
19
          if remove?
20 21 22 23 24 25 26
            begin
              Gitlab::GitalyClient::NamespaceService.new(server.storage)
                .remove(dir_path)

              puts "Removed...#{dir_path}"
            rescue StandardError => e
              puts "Cannot remove #{dir_path}: #{e.message}".color(:red)
27
            end
Dmitriy Zaporozhets's avatar
Dmitriy Zaporozhets committed
28
          else
29
            puts "Can be removed: #{dir_path}".color(:red)
Dmitriy Zaporozhets's avatar
Dmitriy Zaporozhets committed
30 31 32 33
          end
        end
      end

34
      unless remove?
35
        puts "To cleanup this directories run this command with REMOVE=true".color(:yellow)
Dmitriy Zaporozhets's avatar
Dmitriy Zaporozhets committed
36 37 38
      end
    end

39
    desc "GitLab | Cleanup | Clean repositories"
40
    task repos: :gitlab_environment do
Jacob Vosmaer's avatar
Jacob Vosmaer committed
41
      move_suffix = "+orphaned+#{Time.now.to_i}"
42

43 44 45 46 47 48 49 50 51
      Gitaly::Server.all.each do |server|
        Gitlab::GitalyClient::StorageService
          .new(server.storage)
          .list_directories
          .each do |path|
          repo_with_namespace = path.chomp('.git').chomp('.wiki')

          # TODO ignoring hashed repositories for now.  But revisit to fully support
          # possible orphaned hashed repos
52
          next if repo_with_namespace.start_with?(Storage::HashedProject::REPOSITORY_PATH_PREFIX)
53
          next if Project.find_by_full_path(repo_with_namespace)
54

55 56
          new_path = path + move_suffix
          puts path.inspect + ' -> ' + new_path.inspect
57

58 59 60 61 62
          begin
            Gitlab::GitalyClient::NamespaceService
              .new(server.storage)
              .rename(path, new_path)
          rescue StandardError => e
63
            puts "Error occurred while moving the repository: #{e.message}".color(:red)
64
          end
Dmitriy Zaporozhets's avatar
Dmitriy Zaporozhets committed
65 66 67
        end
      end
    end
68

69
    desc "GitLab | Cleanup | Block users that have been removed in LDAP"
70
    task block_removed_ldap_users: :gitlab_environment do
71 72 73
      warn_user_is_not_gitlab
      block_flag = ENV['BLOCK']

74 75
      User.find_each do |user|
        next unless user.ldap_user?
76

77
        print "#{user.name} (#{user.ldap_identity.extern_uid}) ..."
78

79
        if Gitlab::Auth::LDAP::Access.allowed?(user)
80
          puts " [OK]".color(:green)
81 82
        else
          if block_flag
83
            user.block! unless user.blocked?
84
            puts " [BLOCKED]".color(:red)
85
          else
86
            puts " [NOT IN LDAP]".color(:yellow)
87 88 89 90 91
          end
        end
      end

      unless block_flag
92
        puts "To block these users run this command with BLOCK=true".color(:yellow)
93 94
      end
    end
95 96 97 98 99 100 101 102 103 104 105 106 107

    desc "GitLab | Cleanup | Clean orphaned project uploads"
    task project_uploads: :gitlab_environment do
      warn_user_is_not_gitlab

      cleaner = Gitlab::Cleanup::ProjectUploads.new(logger: logger)
      cleaner.run!(dry_run: dry_run?)

      if dry_run?
        logger.info "To clean up these files run this command with DRY_RUN=false".color(:yellow)
      end
    end

108 109 110 111 112 113 114 115 116 117
    desc 'GitLab | Cleanup | Clean orphan remote upload files that do not exist in the db'
    task remote_upload_files: :environment do
      cleaner = Gitlab::Cleanup::RemoteUploads.new(logger: logger)
      cleaner.run!(dry_run: dry_run?)

      if dry_run?
        logger.info "To cleanup these files run this command with DRY_RUN=false".color(:yellow)
      end
    end

118 119 120 121 122 123 124 125 126 127 128 129
    desc 'GitLab | Cleanup | Clean orphan job artifact files'
    task orphan_job_artifact_files: :gitlab_environment do
      warn_user_is_not_gitlab

      cleaner = Gitlab::Cleanup::OrphanJobArtifactFiles.new(limit: limit, dry_run: dry_run?, niceness: niceness, logger: logger)
      cleaner.run!

      if dry_run?
        logger.info "To clean up these files run this command with DRY_RUN=false".color(:yellow)
      end
    end

130 131 132 133 134 135 136 137
    def remove?
      ENV['REMOVE'] == 'true'
    end

    def dry_run?
      ENV['DRY_RUN'] != 'false'
    end

138 139 140 141 142 143 144 145 146 147 148 149
    def debug?
      ENV['DEBUG'].present?
    end

    def limit
      ENV['LIMIT']&.to_i
    end

    def niceness
      ENV['NICENESS'].presence
    end

150
    # rubocop:disable Gitlab/RailsLogger
151 152 153 154 155 156
    def logger
      return @logger if defined?(@logger)

      @logger = if Rails.env.development? || Rails.env.production?
                  Logger.new(STDOUT).tap do |stdout_logger|
                    stdout_logger.extend(ActiveSupport::Logger.broadcast(Rails.logger))
157
                    stdout_logger.level = debug? ? Logger::DEBUG : Logger::INFO
158 159 160 161 162
                  end
                else
                  Rails.logger
                end
    end
163
    # rubocop:enable Gitlab/RailsLogger
Dmitriy Zaporozhets's avatar
Dmitriy Zaporozhets committed
164 165
  end
end