Enable milestone reference caching

to reduce SQL queries

Changelog: performance

lib/banzai/filter/references/milestone_reference_filter.rb

@@ -10,21 +10,63 @@ module Banzai
         self.reference_type = :milestone
         self.object_class   = Milestone
 
+        def parent_records(parent, ids)
+          return Milestone.none unless valid_context?(parent)
+
+          milestones_by_iid  = find_milestones(parent, true)
+          milestones_by_name = find_milestones(parent, false)
+          milestone_iids     = ids.map {|y| y[:milestone_iid]}.compact
+          milestone_names    = ids.map {|y| y[:milestone_name]}.compact
+          iid_relation       = milestones_by_iid.where(iid: milestone_iids)
+          milestone_relation = milestones_by_name.where(title: milestone_names)
+
+          Milestone.from_union([iid_relation, milestone_relation]).includes(:project, :group)
+        end
+
         # Links to project milestones contain the IID, but when we're handling
         # 'regular' references, we need to use the global ID to disambiguate
         # between group and project milestones.
-        def find_object(parent, id)
-          return unless valid_context?(parent)
+        def find_object(parent_object, id)
+          key = reference_cache.records_per_parent[parent_object].keys.find do |k|
+            k[:milestone_iid] == id[:milestone_iid] || k[:milestone_name] == id[:milestone_name]
+          end
 
-          find_milestone_with_finder(parent, id: id)
+          reference_cache.records_per_parent[parent_object][key] if key
         end
 
-        def find_object_from_link(parent, iid)
-          return unless valid_context?(parent)
+        # Transform a symbol extracted from the text to a meaningful value
+        #
+        # This method has the contract that if a string `ref` refers to a
+        # record `record`, then `parse_symbol(ref) == record_identifier(record)`.
+        #
+        # This contract is slightly broken here, as we only have either the label_id
+        # or the label_name, but not both.  But below, we have both pieces of information.
+        # But it's accounted for in `find_object`
+        def parse_symbol(symbol, match_data)
+          if symbol
+            # when parsing links, there is no `match_data[:milestone_iid]`, but `symbol`
+            # holds the iid
+            { milestone_iid: symbol.to_i, milestone_name: nil }
+          else
+            { milestone_iid: match_data[:milestone_iid]&.to_i, milestone_name: match_data[:milestone_name]&.tr('"', '') }
+          end
+        end
 
-          find_milestone_with_finder(parent, iid: iid)
+        # We assume that most classes are identifying records by ID.
+        #
+        # This method has the contract that if a string `ref` refers to a
+        # record `record`, then `class.parse_symbol(ref) == record_identifier(record)`.
+        # See note in `parse_symbol` above
+        def record_identifier(record)
+          { milestone_iid: record.iid, milestone_name: record.title }
         end
 
+        # def find_object_from_link(parent, iid)
+        #   return unless valid_context?(parent)
+        #
+        #   find_milestone_with_finder(parent, iid: iid)
+        # end
+
         def valid_context?(parent)
           strong_memoize(:valid_context) do
             group_context?(parent) || project_context?(parent)
@@ -50,12 +92,14 @@ module Banzai
           return super(text, pattern) if pattern != Milestone.reference_pattern
 
           milestones = {}
-          unescaped_html = unescape_html_entities(text).gsub(pattern) do |match|
-            milestone = find_milestone($~[:project], $~[:namespace], $~[:milestone_iid], $~[:milestone_name])
 
-            if milestone
-              milestones[milestone.id] = yield match, milestone.id, $~[:project], $~[:namespace], $~
-              "#{REFERENCE_PLACEHOLDER}#{milestone.id}"
+          unescaped_html = unescape_html_entities(text).gsub(pattern).with_index do |match, index|
+            ident = identifier($~)
+            milestone = yield match, ident, $~[:project], $~[:namespace], $~
+
+            if milestone != match
+              milestones[index] = milestone
+              "#{REFERENCE_PLACEHOLDER}#{index}"
             else
               match
             end
@@ -66,31 +110,10 @@ module Banzai
           escape_with_placeholders(unescaped_html, milestones)
         end
 
-        def find_milestone(project_ref, namespace_ref, milestone_id, milestone_name)
-          project_path = reference_cache.full_project_path(namespace_ref, project_ref)
-
-          # Returns group if project is not found by path
-          parent = parent_from_ref(project_path)
-
-          return unless parent
-
-          milestone_params = milestone_params(milestone_id, milestone_name)
-
-          find_milestone_with_finder(parent, milestone_params)
-        end
-
-        def milestone_params(iid, name)
-          if name
-            { name: name.tr('"', '') }
-          else
-            { iid: iid.to_i }
-          end
-        end
-
-        def find_milestone_with_finder(parent, params)
-          finder_params = milestone_finder_params(parent, params[:iid].present?)
+        def find_milestones(parent, find_by_iid = false)
+          finder_params = milestone_finder_params(parent, find_by_iid)
 
-          MilestonesFinder.new(finder_params).find_by(params)
+          MilestonesFinder.new(finder_params).execute
         end
 
         def milestone_finder_params(parent, find_by_iid)
@@ -131,6 +154,14 @@ module Banzai
         def object_link_title(object, matches)
           nil
         end
+
+        def parent
+          project || group
+        end
+
+        def requires_unescaping?
+          true
+        end
       end
     end
   end

spec/lib/banzai/filter/references/milestone_reference_filter_spec.rb

@@ -327,6 +327,7 @@ RSpec.describe Banzai::Filter::References::MilestoneReferenceFilter do
     it_behaves_like 'String-based single-word references'
     it_behaves_like 'String-based multi-word references in quotes'
     it_behaves_like 'referencing a milestone in a link href'
+    it_behaves_like 'linking to a milestone as the entire link'
     it_behaves_like 'cross-project / cross-namespace complete reference'
     it_behaves_like 'cross-project / same-namespace complete reference'
     it_behaves_like 'cross project shorthand reference'
@@ -460,4 +461,71 @@ RSpec.describe Banzai::Filter::References::MilestoneReferenceFilter do
       include_context 'group milestones'
     end
   end
+
+  context 'checking N+1' do
+    let_it_be(:group)              { create(:group) }
+    let_it_be(:group2)             { create(:group) }
+    let_it_be(:project)            { create(:project, :public, namespace: group) }
+    let_it_be(:project2)           { create(:project, :public, namespace: group2) }
+    let_it_be(:project3)           { create(:project, :public) }
+    let_it_be(:project_milestone)  { create(:milestone, project: project) }
+    let_it_be(:project_milestone2) { create(:milestone, project: project) }
+    let_it_be(:project2_milestone) { create(:milestone, project: project2) }
+    let_it_be(:group2_milestone)   { create(:milestone, group: group2) }
+    let_it_be(:project_reference)  { "#{project_milestone.to_reference}" }
+    let_it_be(:project_reference2) { "#{project_milestone2.to_reference}" }
+    let_it_be(:project2_reference) { "#{project2_milestone.to_reference}" }
+    let_it_be(:group2_reference)   { "#{project2.full_path}%#{group2_milestone.name}" }
+
+    it 'does not have N+1 per multiple references per project', :use_sql_query_cache do
+      markdown = "#{project_reference}"
+      control_count = 4
+
+      expect do
+        reference_filter(markdown)
+      end.not_to exceed_all_query_limit(control_count)
+
+      markdown = "#{project_reference} %qwert %werty %ertyu %rtyui #{project_reference2}"
+
+      expect do
+        reference_filter(markdown)
+      end.not_to exceed_all_query_limit(control_count)
+    end
+
+    it 'has N+1 for multiple unique project/group references', :use_sql_query_cache do
+      markdown = "#{project_reference}"
+      control_count = 4
+
+      expect do
+        reference_filter(markdown, project: project)
+      end.not_to exceed_all_query_limit(control_count)
+
+      # Since we're not batching label queries across projects/groups,
+      # queries increase when a new project/group is added.
+      # TODO: https://gitlab.com/gitlab-org/gitlab/-/issues/330359
+      # first reference to already loaded project (1),
+      # second reference requires project and namespace (2), and milestone (1)
+      markdown = "#{project_reference} #{group2_reference}"
+      max_count = control_count + 3
+
+      expect do
+        reference_filter(markdown)
+      end.not_to exceed_all_query_limit(max_count)
+
+      # third reference to already queried project/namespace, nothing extra (no N+1 here)
+      markdown = "#{project_reference} #{group2_reference} #{project2_reference}"
+
+      expect do
+        reference_filter(markdown)
+      end.not_to exceed_all_query_limit(max_count)
+
+      # last reference needs another namespace and label query (2)
+      markdown = "#{project_reference} #{group2_reference} #{project2_reference} #{project3.full_path}%test_milestone"
+      max_count += 2
+
+      expect do
+        reference_filter(markdown)
+      end.not_to exceed_all_query_limit(max_count)
+    end
+  end
 end