Merge branch '220329-reduce-redis-calls-projects-api' into 'master'

Preload repository cache using MGET

See merge request gitlab-org/gitlab!69627

app/models/repository.rb

@@ -1132,6 +1132,10 @@ class Repository
     end
   end
 
+  def cache
+    @cache ||= Gitlab::RepositoryCache.new(self)
+  end
+
   private
 
   # TODO Genericize finder, later split this on finders by Ref or Oid
@@ -1146,10 +1150,6 @@ class Repository
     ::Commit.new(commit, container) if commit
   end
 
-  def cache
-    @cache ||= Gitlab::RepositoryCache.new(self)
-  end
-
   def redis_set_cache
     @redis_set_cache ||= Gitlab::RepositorySetCache.new(self)
   end

config/feature_flags/development/preload_repo_cache.yml

+---
+name: preload_repo_cache
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/69627
+rollout_issue_url:
+milestone: '14.3'
+type: development
+group: group::project management
+default_enabled: false

lib/api/entities/project.rb

@@ -152,6 +152,10 @@ module API
 
         super
       end
+
+      def self.repositories_for_preload(projects_relation)
+        super + projects_relation.map(&:forked_from_project).compact.map(&:repository)
+      end
     end
   end
 end

lib/api/projects_relation_builder.rb

@@ -10,6 +10,8 @@ module API
 
         execute_batch_counting(projects_relation)
 
+        preload_repository_cache(projects_relation)
+
         projects_relation
       end
 
@@ -23,6 +25,20 @@ module API
       # batch load certain counts
       def execute_batch_counting(projects_relation)
       end
+
+      def preload_repository_cache(projects_relation)
+        return unless Feature.enabled?(:preload_repo_cache, default_enabled: :yaml)
+
+        repositories = repositories_for_preload(projects_relation)
+
+        Gitlab::RepositoryCache::Preloader.new(repositories).preload( # rubocop:disable CodeReuse/ActiveRecord
+          %i[exists? root_ref has_visible_content? avatar readme_path]
+        )
+      end
+
+      def repositories_for_preload(projects_relation)
+        projects_relation.map(&:repository)
+      end
     end
   end
 end

lib/gitlab/repository_cache/preloader.rb

+# frozen_string_literal: true
+
+module Gitlab
+  class RepositoryCache
+    class Preloader
+      def initialize(repositories)
+        @repositories = repositories
+      end
+
+      def preload(methods)
+        return if @repositories.empty?
+
+        cache_keys = []
+
+        sources_by_cache_key = @repositories.each_with_object({}) do |repository, hash|
+          methods.each do |method|
+            cache_key = repository.cache.cache_key(method)
+
+            hash[cache_key] = { repository: repository, method: method }
+            cache_keys << cache_key
+          end
+        end
+
+        Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do
+          backend.read_multi(*cache_keys).each do |cache_key, value|
+            source = sources_by_cache_key[cache_key]
+
+            source[:repository].memoize_method_cache_value(source[:method], value)
+          end
+        end
+      end
+
+      private
+
+      def backend
+        @repositories.first.cache.backend
+      end
+    end
+  end
+end

lib/gitlab/repository_cache_adapter.rb

@@ -217,6 +217,10 @@ module Gitlab
       fallback
     end
 
+    def memoize_method_cache_value(method, value)
+      strong_memoize(memoizable_name(method)) { value }
+    end
+
     # Expires the caches of a specific set of methods
     def expire_method_caches(methods)
       methods.each do |name|

spec/lib/gitlab/repository_cache/preloader_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::RepositoryCache::Preloader, :use_clean_rails_redis_caching do
+  let(:projects) { create_list(:project, 2, :repository) }
+  let(:repositories) { projects.map(&:repository) }
+
+  describe '#preload' do
+    context 'when the values are already cached' do
+      before do
+        # Warm the cache but use a different model so they are not memoized
+        repos = Project.id_in(projects).order(:id).map(&:repository)
+
+        allow(repos[0].head_tree).to receive(:readme_path).and_return('README.txt')
+        allow(repos[1].head_tree).to receive(:readme_path).and_return('README.md')
+
+        repos.map(&:exists?)
+        repos.map(&:readme_path)
+      end
+
+      it 'prevents individual cache reads for cached methods' do
+        expect(Rails.cache).to receive(:read_multi).once.and_call_original
+
+        described_class.new(repositories).preload(
+          %i[exists? readme_path]
+        )
+
+        expect(Rails.cache).not_to receive(:read)
+        expect(Rails.cache).not_to receive(:write)
+
+        expect(repositories[0].exists?).to eq(true)
+        expect(repositories[0].readme_path).to eq('README.txt')
+
+        expect(repositories[1].exists?).to eq(true)
+        expect(repositories[1].readme_path).to eq('README.md')
+      end
+    end
+
+    context 'when values are not cached' do
+      it 'reads and writes from cache individually' do
+        described_class.new(repositories).preload(
+          %i[exists? has_visible_content?]
+        )
+
+        expect(Rails.cache).to receive(:read).exactly(4).times
+        expect(Rails.cache).to receive(:write).exactly(4).times
+
+        repositories.each(&:exists?)
+        repositories.each(&:has_visible_content?)
+      end
+    end
+  end
+end