Merge branch 'banzai-issue-filter-queries' into 'master'

Reduce SQL query counts in IssueReferenceFilter

## What does this MR do?

This MR adds a preparation phase for reference filters that allows them to prepare/create data structures used while iterating over HTML nodes. In this particular case the preparation phase is used for issue references to greatly cut down the amount of queries executed to get projects/issues for Markdown references.

## Are there points in the code the reviewer needs to double check?

No.

## Why was this MR needed?

Rendering Markdown containing issue references would run at most two queries for every issue reference: one to get the project and one to get the issue from said project. When rendering Markdown with lots of issue references this would result in _a lot_ of queries being executed.

## What are the relevant issue numbers?

#18042

See merge request !4410

lib/banzai/filter/abstract_reference_filter.rb

@@ -103,7 +103,7 @@ module Banzai
         ref_pattern = object_class.reference_pattern
         link_pattern = object_class.link_reference_pattern
 
-        each_node do |node|
+        nodes.each do |node|
           if text_node?(node) && ref_pattern
             replace_text_when_pattern_matches(node, ref_pattern) do |content|
               object_link_filter(content, ref_pattern)
@@ -206,6 +206,55 @@ module Banzai
         text
       end
 
+      # Returns a Hash containing all object references (e.g. issue IDs) per the
+      # project they belong to.
+      def references_per_project
+        @references_per_project ||= begin
+          refs = Hash.new { |hash, key| hash[key] = Set.new }
+
+          regex = Regexp.union(object_class.reference_pattern,
+                               object_class.link_reference_pattern)
+
+          nodes.each do |node|
+            node.to_html.scan(regex) do
+              project = $~[:project] || current_project_path
+
+              refs[project] << $~[object_sym]
+            end
+          end
+
+          refs
+        end
+      end
+
+      # Returns a Hash containing referenced projects grouped per their full
+      # path.
+      def projects_per_reference
+        @projects_per_reference ||= begin
+          hash = {}
+          refs = Set.new
+
+          references_per_project.each do |project_ref, _|
+            refs << project_ref
+          end
+
+          find_projects_for_paths(refs.to_a).each do |project|
+            hash[project.path_with_namespace] = project
+          end
+
+          hash
+        end
+      end
+
+      # Returns the projects for the given paths.
+      def find_projects_for_paths(paths)
+        Project.where_paths_in(paths).includes(:namespace)
+      end
+
+      def current_project_path
+        @current_project_path ||= project.path_with_namespace
+      end
+
       private
 
       def project_refs_cache

lib/banzai/filter/issue_reference_filter.rb

@@ -11,13 +11,40 @@ module Banzai
         Issue
       end
 
-      def find_object(project, id)
-        project.get_issue(id)
+      def find_object(project, iid)
+        issues_per_project[project][iid]
       end
 
       def url_for_object(issue, project)
         IssuesHelper.url_for_issue(issue.iid, project, only_path: context[:only_path])
       end
+
+      def project_from_ref(ref)
+        projects_per_reference[ref || current_project_path]
+      end
+
+      # Returns a Hash containing the issues per Project instance.
+      def issues_per_project
+        @issues_per_project ||= begin
+          hash = Hash.new { |h, k| h[k] = {} }
+
+          projects_per_reference.each do |path, project|
+            issue_ids = references_per_project[path]
+
+            next unless project.default_issues_tracker?
+
+            project.issues.where(iid: issue_ids.to_a).each do |issue|
+              hash[project][issue.iid] = issue
+            end
+          end
+
+          hash
+        end
+      end
+
+      def find_projects_for_paths(paths)
+        super(paths).includes(:gitlab_issue_tracker_service)
+      end
     end
   end
 end

spec/lib/banzai/filter/abstract_link_filter_spec.rb

+require 'spec_helper'
+
+describe Banzai::Filter::AbstractReferenceFilter do
+  let(:project) { create(:empty_project) }
+
+  describe '#references_per_project' do
+    it 'returns a Hash containing references grouped per project paths' do
+      doc = Nokogiri::HTML.fragment("#1 #{project.to_reference}#2")
+      filter = described_class.new(doc, project: project)
+
+      expect(filter).to receive(:object_class).twice.and_return(Issue)
+      expect(filter).to receive(:object_sym).twice.and_return(:issue)
+
+      refs = filter.references_per_project
+
+      expect(refs).to be_an_instance_of(Hash)
+      expect(refs[project.to_reference]).to eq(Set.new(%w[1 2]))
+    end
+  end
+
+  describe '#projects_per_reference' do
+    it 'returns a Hash containing projects grouped per project paths' do
+      doc = Nokogiri::HTML.fragment('')
+      filter = described_class.new(doc, project: project)
+
+      expect(filter).to receive(:references_per_project).
+        and_return({ project.path_with_namespace => Set.new(%w[1]) })
+
+      expect(filter.projects_per_reference).
+        to eq({ project.path_with_namespace => project })
+    end
+  end
+
+  describe '#find_projects_for_paths' do
+    it 'returns a list of Projects for a list of paths' do
+      doc = Nokogiri::HTML.fragment('')
+      filter = described_class.new(doc, project: project)
+
+      expect(filter.find_projects_for_paths([project.path_with_namespace])).
+        to eq([project])
+    end
+  end
+
+  describe '#current_project_path' do
+    it 'returns the path of the current project' do
+      doc = Nokogiri::HTML.fragment('')
+      filter = described_class.new(doc, project: project)
+
+      expect(filter.current_project_path).to eq(project.path_with_namespace)
+    end
+  end
+end

spec/lib/banzai/filter/issue_reference_filter_spec.rb

@@ -25,7 +25,9 @@ describe Banzai::Filter::IssueReferenceFilter, lib: true do
     let(:reference) { issue.to_reference }
 
     it 'ignores valid references when using non-default tracker' do
-      expect(project).to receive(:get_issue).with(issue.iid).and_return(nil)
+      expect_any_instance_of(described_class).to receive(:find_object).
+        with(project, issue.iid).
+        and_return(nil)
 
       exp = act = "Issue #{reference}"
       expect(reference_filter(act).to_html).to eq exp
@@ -107,8 +109,9 @@ describe Banzai::Filter::IssueReferenceFilter, lib: true do
     let(:reference) { issue.to_reference(project) }
 
     it 'ignores valid references when cross-reference project uses external tracker' do
-      expect_any_instance_of(Project).to receive(:get_issue).
-        with(issue.iid).and_return(nil)
+      expect_any_instance_of(described_class).to receive(:find_object).
+        with(project2, issue.iid).
+        and_return(nil)
 
       exp = act = "Issue #{reference}"
       expect(reference_filter(act).to_html).to eq exp

spec/services/git_push_service_spec.rb

@@ -312,7 +312,8 @@ describe GitPushService, services: true do
       end
 
       it "doesn't close issues when external issue tracker is in use" do
-        allow(project).to receive(:default_issues_tracker?).and_return(false)
+        allow_any_instance_of(Project).to receive(:default_issues_tracker?).
+          and_return(false)
 
         # The push still shouldn't create cross-reference notes.
         expect do