Add rake task to cleanup description templates cache

Add a way to cleanup multiple cache keys in batches. This makes
use of redis scan to match keys by a given pattern and remove the
matched keys.

The batches are set to 1000 keys.

changelogs/unreleased/rake-task-to-clear-description-templates-cache.yml

+---
+title: Add rake task to cleanup description templates cache in batches
+merge_request: 54706
+author:
+type: added

doc/administration/raketasks/maintenance.md

@@ -203,6 +203,51 @@ You will lose any data stored in authorized_keys file.
 Do you want to continue (yes/no)? yes
 ```
 
+## Clear issue and merge request description template names cache
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/54706) in GitLab 13.10
+
+If for some reason the issue or merge request description template names in the dropdown
+do not reflect the actual description template names in the repository, you may want to
+consider clearing the Redis cache that stores the template names information. To do so, you
+need to run:
+
+### Clear cache for all issue and merge request template names
+
+In cases where you want to refresh issue and merge request templates for all projects in your installation:
+
+**Omnibus Installation**
+
+```shell
+sudo gitlab-rake cache:clear:description_templates
+```
+
+**Source Installation**
+
+```shell
+cd /home/git/gitlab
+sudo -u git -H bundle exec rake cache:clear:description_templates RAILS_ENV=production
+```
+
+### Clear cache for issue and merge request template names in specific projects
+
+In cases where you want to refresh issue and merge request templates for specific projects
+
+Provide a comma separate list of ids as `project_ids` parameter to the Rake task.
+
+**Omnibus Installation**
+
+```shell
+sudo gitlab-rake cache:clear:description_templates project_ids=10,25,35
+```
+
+**Source Installation**
+
+```shell
+cd /home/git/gitlab
+sudo -u git -H bundle exec rake cache:clear:description_templates project_ids=10,25,35 RAILS_ENV=production
+```
+
 ## Clear Redis cache
 
 If for some reason the dashboard displays the wrong information, you might want to

lib/gitlab/cache/ci/project_pipeline_status.rb

@@ -9,6 +9,8 @@ module Gitlab
       class ProjectPipelineStatus
         include Gitlab::Utils::StrongMemoize
 
+        ALL_PIPELINES_STATUS_PATTERN = 'projects/*/pipeline_status'
+
         attr_accessor :sha, :status, :ref, :project, :loaded
 
         def self.load_for_project(project)

lib/gitlab/cleanup/redis/batch_delete_by_pattern.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Cleanup
+    module Redis
+      class BatchDeleteByPattern
+        REDIS_CLEAR_BATCH_SIZE = 1000 # There seems to be no speedup when pushing beyond 1,000
+        REDIS_SCAN_START_STOP = '0'.freeze # Magic value, see http://redis.io/commands/scan
+
+        attr_reader :patterns
+
+        def initialize(patterns)
+          raise ArgumentError.new('Argument should be an Array of patterns') unless patterns.is_a?(Array)
+
+          @patterns = patterns
+        end
+
+        def execute
+          return if patterns.blank?
+
+          batch_delete_cache_keys
+        end
+
+        private
+
+        def batch_delete_cache_keys
+          Gitlab::Redis::Cache.with do |redis|
+            patterns.each do |match|
+              cursor = REDIS_SCAN_START_STOP
+              loop do
+                cursor, keys = redis.scan(
+                  cursor,
+                  match: match,
+                  count: REDIS_CLEAR_BATCH_SIZE
+                )
+
+                Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do
+                  redis.del(*keys) if keys.any?
+                end
+
+                break if cursor == REDIS_SCAN_START_STOP
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/cleanup/redis/description_templates_cache_keys_pattern_builder.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Cleanup
+    module Redis
+      class DescriptionTemplatesCacheKeysPatternBuilder
+        # project_ids - a list of project_ids for which to compute description templates cache keys or `:all` to compute
+        # a pattern that cover all description templates cache keys.
+        #
+        # Example
+        # * ::Gitlab::Cleanup::Redis::BatchDeleteDescriptionTemplates.new(:all).execute - to get 2
+        # patterns for all issue and merge request description templates cache keys.
+        #
+        # * ::Gitlab::Cleanup::Redis::BatchDeleteDescriptionTemplates.new([1,2,3,4]).execute - to get an array of
+        # patterns for each project's issue and merge request description templates cache keys.
+        def initialize(project_ids)
+          raise ArgumentError.new('project_ids can either be an array of project IDs or :all') if project_ids != :all && !project_ids.is_a?(Array)
+
+          @project_ids = parse_project_ids(project_ids)
+        end
+
+        def execute
+          case project_ids
+          when :all
+            all_instance_patterns
+          else
+            project_patterns
+          end
+        end
+
+        private
+
+        attr_reader :project_ids
+
+        def parse_project_ids(project_ids)
+          return project_ids if project_ids == :all
+
+          project_ids.map { |id| Integer(id) }
+        rescue ArgumentError
+          raise ArgumentError.new('Invalid Project ID. Please ensure all passed in project ids values are valid integer project ids.')
+        end
+
+        def project_patterns
+          cache_key_patterns = []
+          Project.id_in(project_ids).each_batch do |batch|
+            cache_key_patterns << batch.map do |pr|
+              next unless pr.repository.exists?
+
+              cache = Gitlab::RepositoryCache.new(pr.repository)
+
+              [repo_issue_templates_cache_key(cache), repo_merge_request_templates_cache_key(cache)]
+            end
+          end
+
+          cache_key_patterns.flatten.compact
+        end
+
+        def all_instance_patterns
+          [all_issue_templates_cache_key, all_merge_request_templates_cache_key]
+        end
+
+        def issue_templates_cache_key
+          Repository::METHOD_CACHES_FOR_FILE_TYPES[:issue_template]
+        end
+
+        def merge_request_templates_cache_key
+          Repository::METHOD_CACHES_FOR_FILE_TYPES[:merge_request_template]
+        end
+
+        def all_issue_templates_cache_key
+          "#{Gitlab::Redis::Cache::CACHE_NAMESPACE}:#{issue_templates_cache_key}:*"
+        end
+
+        def all_merge_request_templates_cache_key
+          "#{Gitlab::Redis::Cache::CACHE_NAMESPACE}:#{merge_request_templates_cache_key}:*"
+        end
+
+        def repo_issue_templates_cache_key(cache)
+          "#{Gitlab::Redis::Cache::CACHE_NAMESPACE}:#{cache.cache_key(issue_templates_cache_key)}"
+        end
+
+        def repo_merge_request_templates_cache_key(cache)
+          "#{Gitlab::Redis::Cache::CACHE_NAMESPACE}:#{cache.cache_key(merge_request_templates_cache_key)}"
+        end
+      end
+    end
+  end
+end

lib/tasks/cache.rake

@@ -2,32 +2,22 @@
 
 namespace :cache do
   namespace :clear do
-    REDIS_CLEAR_BATCH_SIZE = 1000 # There seems to be no speedup when pushing beyond 1,000
-    REDIS_SCAN_START_STOP = '0'.freeze # Magic value, see http://redis.io/commands/scan
-
     desc "GitLab | Cache | Clear redis cache"
     task redis: :environment do
-      Gitlab::Redis::Cache.with do |redis|
-        cache_key_pattern = %W[#{Gitlab::Redis::Cache::CACHE_NAMESPACE}*
-                               projects/*/pipeline_status]
-
-        cache_key_pattern.each do |match|
-          cursor = REDIS_SCAN_START_STOP
-          loop do
-            cursor, keys = redis.scan(
-              cursor,
-              match: match,
-              count: REDIS_CLEAR_BATCH_SIZE
-            )
+      cache_key_patterns = %W[
+        #{Gitlab::Redis::Cache::CACHE_NAMESPACE}*
+        #{Gitlab::Cache::Ci::ProjectPipelineStatus::ALL_PIPELINES_STATUS_PATTERN}
+      ]
 
-            Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do
-              redis.del(*keys) if keys.any?
+      ::Gitlab::Cleanup::Redis::BatchDeleteByPattern.new(cache_key_patterns).execute
     end
 
-            break if cursor == REDIS_SCAN_START_STOP
-          end
-        end
-      end
+    desc "GitLab | Cache | Clear description templates redis cache"
+    task description_templates: :environment do
+      project_ids = Array(ENV['project_ids']&.split(',')).map!(&:squish)
+
+      cache_key_patterns = ::Gitlab::Cleanup::Redis::DescriptionTemplatesCacheKeysPatternBuilder.new(project_ids).execute
+      ::Gitlab::Cleanup::Redis::BatchDeleteByPattern.new(cache_key_patterns).execute
     end
 
     task all: [:redis]

spec/lib/gitlab/cleanup/redis/batch_delete_by_pattern_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Cleanup::Redis::BatchDeleteByPattern, :clean_gitlab_redis_cache do
+  subject { described_class.new(patterns) }
+
+  describe 'execute' do
+    context 'when no patterns are passed' do
+      before do
+        expect(Gitlab::Redis::Cache).not_to receive(:with)
+      end
+
+      context 'with nil patterns' do
+        let(:patterns) { nil }
+
+        specify { expect { subject }.to raise_error(ArgumentError, 'Argument should be an Array of patterns') }
+      end
+
+      context 'with empty array patterns' do
+        let(:patterns) { [] }
+
+        specify { subject.execute }
+      end
+    end
+
+    context 'with patterns' do
+      context 'when key is not found' do
+        let(:patterns) { ['key'] }
+
+        before do
+          expect_any_instance_of(Redis).not_to receive(:del) # rubocop:disable RSpec/AnyInstanceOf
+        end
+
+        specify { subject.execute }
+      end
+
+      context 'with cache data' do
+        let(:cache_keys) { %w[key-test1 key-test2 key-test3 key-test4] }
+
+        before do
+          stub_const("#{described_class}::REDIS_CLEAR_BATCH_SIZE", 2)
+
+          write_to_cache
+        end
+
+        context 'with one key' do
+          let(:patterns) { ['key-test1'] }
+
+          it 'deletes the key' do
+            expect_any_instance_of(Redis).to receive(:del).with(patterns.first).once # rubocop:disable RSpec/AnyInstanceOf
+
+            subject.execute
+          end
+        end
+
+        context 'with many keys' do
+          let(:patterns) { %w[key-test1 key-test2] }
+
+          it 'deletes keys for each pattern separatelly' do
+            expect_any_instance_of(Redis).to receive(:del).with(patterns.first).once # rubocop:disable RSpec/AnyInstanceOf
+            expect_any_instance_of(Redis).to receive(:del).with(patterns.last).once # rubocop:disable RSpec/AnyInstanceOf
+
+            subject.execute
+          end
+        end
+
+        context 'with cache_keys over batch size' do
+          let(:patterns) { %w[key-test*] }
+
+          it 'deletes matched keys in batches' do
+            # redis scan returns the values in random order so just checking it is being called twice meaning
+            # scan returned results in 2 batches, which is what we expect
+            key_like = start_with('key-test')
+            expect_any_instance_of(Redis).to receive(:del).with(key_like, key_like).twice # rubocop:disable RSpec/AnyInstanceOf
+
+            subject.execute
+          end
+        end
+      end
+    end
+  end
+end
+
+def write_to_cache
+  Gitlab::Redis::Cache.with do |redis|
+    cache_keys.each_with_index do |cache_key, index|
+      redis.set(cache_key, index)
+    end
+  end
+end

spec/lib/gitlab/cleanup/redis/description_templates_cache_keys_pattern_builder_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Cleanup::Redis::DescriptionTemplatesCacheKeysPatternBuilder, :clean_gitlab_redis_cache do
+  subject { described_class.new(project_ids).execute }
+
+  describe 'execute' do
+    context 'when build pattern for all description templates' do
+      RSpec.shared_examples 'all issue and merge request templates pattern' do
+        it 'builds pattern to remove all issue and merge request templates keys' do
+          expect(subject.count).to eq(2)
+          expect(subject).to match_array(%W[
+            #{Gitlab::Redis::Cache::CACHE_NAMESPACE}:issue_template_names_hash:*
+            #{Gitlab::Redis::Cache::CACHE_NAMESPACE}:merge_request_template_names_hash:*
+          ])
+        end
+      end
+
+      context 'with project_ids == :all' do
+        let(:project_ids) { :all }
+
+        it_behaves_like 'all issue and merge request templates pattern'
+      end
+    end
+
+    context 'with project_ids' do
+      let_it_be(:project1) { create(:project, :repository) }
+      let_it_be(:project2) { create(:project, :repository) }
+
+      context 'with nil project_ids' do
+        let(:project_ids) { nil }
+
+        specify { expect { subject }.to raise_error(ArgumentError, 'project_ids can either be an array of project IDs or :all') }
+      end
+
+      context 'with project_ids as string' do
+        let(:project_ids) { '1' }
+
+        specify { expect { subject }.to raise_error(ArgumentError, 'project_ids can either be an array of project IDs or :all') }
+      end
+
+      context 'with invalid project_ids as array of strings' do
+        let(:project_ids) { %w[a b] }
+
+        specify { expect { subject }.to raise_error(ArgumentError, 'Invalid Project ID. Please ensure all passed in project ids values are valid integer project ids.') }
+      end
+
+      context 'with non existent project id' do
+        let(:project_ids) { [non_existing_record_id] }
+
+        it 'no patterns are built' do
+          expect(subject.count).to eq(0)
+        end
+      end
+
+      context 'with one project_id' do
+        let(:project_ids) { [project1.id] }
+
+        it 'builds patterns for the project' do
+          expect(subject.count).to eq(2)
+          expect(subject).to match_array(%W[
+            #{Gitlab::Redis::Cache::CACHE_NAMESPACE}:issue_template_names_hash:#{project1.full_path}:#{project1.id}
+            #{Gitlab::Redis::Cache::CACHE_NAMESPACE}:merge_request_template_names_hash:#{project1.full_path}:#{project1.id}
+          ])
+        end
+      end
+
+      context 'with many project_ids' do
+        let(:project_ids) { [project1.id, project2.id] }
+
+        RSpec.shared_examples 'builds patterns for the given projects' do
+          it 'builds patterns for the given projects' do
+            expect(subject.count).to eq(4)
+            expect(subject).to match_array(%W[
+            #{Gitlab::Redis::Cache::CACHE_NAMESPACE}:issue_template_names_hash:#{project1.full_path}:#{project1.id}
+            #{Gitlab::Redis::Cache::CACHE_NAMESPACE}:merge_request_template_names_hash:#{project1.full_path}:#{project1.id}
+            #{Gitlab::Redis::Cache::CACHE_NAMESPACE}:issue_template_names_hash:#{project2.full_path}:#{project2.id}
+            #{Gitlab::Redis::Cache::CACHE_NAMESPACE}:merge_request_template_names_hash:#{project2.full_path}:#{project2.id}
+            ])
+          end
+        end
+
+        it_behaves_like 'builds patterns for the given projects'
+
+        context 'with project_ids as string' do
+          let(:project_ids) { [project1.id.to_s, project2.id.to_s] }
+
+          it_behaves_like 'builds patterns for the given projects'
+        end
+      end
+    end
+  end
+end

spec/tasks/cache/clear/redis_spec.rb

@@ -2,7 +2,7 @@
 
 require 'rake_helper'
 
-RSpec.describe 'clearing redis cache' do
+RSpec.describe 'clearing redis cache', :clean_gitlab_redis_cache do
   before do
     Rake.application.rake_require 'tasks/cache'
   end
@@ -21,4 +21,27 @@ RSpec.describe 'clearing redis cache' do
       expect { run_rake_task('cache:clear:redis') }.to change { pipeline_status.has_cache? }
     end
   end
+
+  describe 'invoking clear description templates cache rake task' do
+    using RSpec::Parameterized::TableSyntax
+
+    before do
+      stub_env('project_ids', project_ids) if project_ids
+      service = double(:service, execute: true)
+
+      expect(Gitlab::Cleanup::Redis::DescriptionTemplatesCacheKeysPatternBuilder).to receive(:new).with(expected_project_ids).and_return(service)
+      expect(Gitlab::Cleanup::Redis::BatchDeleteByPattern).to receive(:new).and_return(service)
+    end
+
+    where(:project_ids, :expected_project_ids) do
+      nil                    | [] # this acts as no argument is being passed
+      '1'                    | %w[1]
+      '1, 2, 3'              | %w[1 2 3]
+      '1, 2, some-string, 3' | %w[1 2 some-string 3]
+    end
+
+    with_them do
+      specify { run_rake_task('cache:clear:description_templates') }
+    end
+  end
 end