Merge branch 'adam-finish-5993-closed-issuable' into 'master'

Add indication for closed or merged issuables in GFM

Closes #1369

See merge request !9462

changelogs/unreleased/adam-finish-5993-closed-issuable.yml

+---
+title: Add indication for closed or merged issuables in GFM
+merge_request: 9462
+author: Adam Buckland

lib/banzai/filter/issuable_state_filter.rb

+module Banzai
+  module Filter
+    # HTML filter that appends state information to issuable links.
+    # Runs as a post-process filter as issuable state might change whilst
+    # Markdown is in the cache.
+    #
+    # This filter supports cross-project references.
+    class IssuableStateFilter < HTML::Pipeline::Filter
+      VISIBLE_STATES = %w(closed merged).freeze
+
+      def call
+        extractor = Banzai::IssuableExtractor.new(project, current_user)
+        issuables = extractor.extract([doc])
+
+        issuables.each do |node, issuable|
+          if VISIBLE_STATES.include?(issuable.state)
+            node.children.last.content += " [#{issuable.state}]"
+          end
+        end
+
+        doc
+      end
+
+      private
+
+      def current_user
+        context[:current_user]
+      end
+
+      def project
+        context[:project]
+      end
+    end
+  end
+end

lib/banzai/filter/redactor_filter.rb

@@ -7,7 +7,7 @@ module Banzai
     #
     class RedactorFilter < HTML::Pipeline::Filter
       def call
-        Redactor.new(project, current_user).redact([doc])
+        Redactor.new(project, current_user).redact([doc]) unless context[:skip_redaction]
 
         doc
       end

lib/banzai/issuable_extractor.rb

+module Banzai
+  # Extract references to issuables from multiple documents
+
+  # This populates RequestStore cache used in Banzai::ReferenceParser::IssueParser
+  # and Banzai::ReferenceParser::MergeRequestParser
+  # Populating the cache should happen before processing documents one-by-one
+  # so we can avoid N+1 queries problem
+
+  class IssuableExtractor
+    QUERY = %q(
+      descendant-or-self::a[contains(concat(" ", @class, " "), " gfm ")]
+      [@data-reference-type="issue" or @data-reference-type="merge_request"]
+    ).freeze
+
+    attr_reader :project, :user
+
+    def initialize(project, user)
+      @project = project
+      @user = user
+    end
+
+    # Returns Hash in the form { node => issuable_instance }
+    def extract(documents)
+      nodes = documents.flat_map do |document|
+        document.xpath(QUERY)
+      end
+
+      issue_parser = Banzai::ReferenceParser::IssueParser.new(project, user)
+      merge_request_parser = Banzai::ReferenceParser::MergeRequestParser.new(project, user)
+
+      issue_parser.issues_for_nodes(nodes).merge(
+        merge_request_parser.merge_requests_for_nodes(nodes)
+      )
+    end
+  end
+end

lib/banzai/object_renderer.rb

@@ -31,7 +31,8 @@ module Banzai
     #
     # Returns the same input objects.
     def render(objects, attribute)
-      documents = render_objects(objects, attribute)
+      documents = render_documents(objects, attribute)
+      documents = post_process_documents(documents, objects, attribute)
       redacted = redact_documents(documents)
 
       objects.each_with_index do |object, index|
@@ -41,9 +42,24 @@ module Banzai
       end
     end
 
-    # Renders the attribute of every given object.
-    def render_objects(objects, attribute)
-      render_attributes(objects, attribute)
+    private
+
+    def render_documents(objects, attribute)
+      pipeline = HTML::Pipeline.new([])
+
+      objects.map do |object|
+        pipeline.to_document(Banzai.render_field(object, attribute))
+      end
+    end
+
+    def post_process_documents(documents, objects, attribute)
+      # Called here to populate cache, refer to IssuableExtractor docs
+      IssuableExtractor.new(project, user).extract(documents)
+
+      documents.zip(objects).map do |document, object|
+        context = context_for(object, attribute)
+        Banzai::Pipeline[:post_process].to_document(document, context)
+      end
     end
 
     # Redacts the list of documents.
@@ -57,25 +73,15 @@ module Banzai
 
     # Returns a Banzai context for the given object and attribute.
     def context_for(object, attribute)
-      context = base_context.dup
-      context = context.merge(object.banzai_render_context(attribute))
-      context
-    end
-
-    # Renders the attributes of a set of objects.
-    #
-    # Returns an Array of `Nokogiri::HTML::Document`.
-    def render_attributes(objects, attribute)
-      objects.map do |object|
-        string = Banzai.render_field(object, attribute)
-        context = context_for(object, attribute)
-
-        Banzai::Pipeline[:relative_link].to_document(string, context)
-      end
+      base_context.merge(object.banzai_render_context(attribute))
     end
 
     def base_context
-      @base_context ||= @redaction_context.merge(current_user: user, project: project)
+      @base_context ||= @redaction_context.merge(
+        current_user: user,
+        project: project,
+        skip_redaction: true
+      )
     end
   end
 end

lib/banzai/pipeline/post_process_pipeline.rb

@@ -4,6 +4,7 @@ module Banzai
       def self.filters
         FilterArray[
           Filter::RelativeLinkFilter,
+          Filter::IssuableStateFilter,
           Filter::RedactorFilter
         ]
       end

lib/banzai/reference_parser/base_parser.rb

@@ -62,8 +62,7 @@ module Banzai
 
         nodes.select do |node|
           if node.has_attribute?(project_attr)
-            node_id = node.attr(project_attr).to_i
-            can_read_reference?(user, projects[node_id])
+            can_read_reference?(user, projects[node])
           else
             true
           end
@@ -112,12 +111,12 @@ module Banzai
         per_project
       end
 
-      # Returns a Hash containing objects for an attribute grouped per their
-      # IDs.
+      # Returns a Hash containing objects for an attribute grouped per the
+      # nodes that reference them.
       #
       # The returned Hash uses the following format:
       #
-      #     { id value => row }
+      #     { node => row }
       #
       # nodes - An Array of HTML nodes to process.
       #
@@ -132,9 +131,14 @@ module Banzai
         return {} if nodes.empty?
 
         ids = unique_attribute_values(nodes, attribute)
-        rows = collection_objects_for_ids(collection, ids)
+        collection_objects = collection_objects_for_ids(collection, ids)
+        objects_by_id = collection_objects.index_by(&:id)
 
-        rows.index_by(&:id)
+        nodes.each_with_object({}) do |node, hash|
+          if node.has_attribute?(attribute)
+            hash[node] = objects_by_id[node.attr(attribute).to_i]
+          end
+        end
       end
 
       # Returns an Array containing all unique values of an attribute of the
@@ -201,7 +205,7 @@ module Banzai
       #
       # The returned Hash uses the following format:
       #
-      #     { project ID => project }
+      #     { node => project }
       #
       def projects_for_nodes(nodes)
         @projects_for_nodes ||=

lib/banzai/reference_parser/issue_parser.rb

@@ -13,14 +13,14 @@ module Banzai
           issues_readable_by_user(issues.values, user).to_set
 
         nodes.select do |node|
-          readable_issues.include?(issue_for_node(issues, node))
+          readable_issues.include?(issues[node])
         end
       end
 
       def referenced_by(nodes)
         issues = issues_for_nodes(nodes)
 
-        nodes.map { |node| issue_for_node(issues, node) }.uniq
+        nodes.map { |node| issues[node] }.compact.uniq
       end
 
       def issues_for_nodes(nodes)
@@ -44,12 +44,6 @@ module Banzai
           self.class.data_attribute
         )
       end
-
-      private
-
-      def issue_for_node(issues, node)
-        issues[node.attr(self.class.data_attribute).to_i]
-      end
     end
   end
 end

lib/banzai/reference_parser/merge_request_parser.rb

@@ -3,6 +3,14 @@ module Banzai
     class MergeRequestParser < BaseParser
       self.reference_type = :merge_request
 
+      def merge_requests_for_nodes(nodes)
+        @merge_requests_for_nodes ||= grouped_objects_for_nodes(
+          nodes,
+          MergeRequest.all,
+          self.class.data_attribute
+        )
+      end
+
       def references_relation
         MergeRequest.includes(:author, :assignee, :target_project)
       end

lib/banzai/reference_parser/user_parser.rb

@@ -49,7 +49,7 @@ module Banzai
       # Check if project belongs to a group which
       # user can read.
       def can_read_group_reference?(node, user, groups)
-        node_group = groups[node.attr('data-group').to_i]
+        node_group = groups[node]
 
         node_group && can?(user, :read_group, node_group)
       end
@@ -74,8 +74,8 @@ module Banzai
           if project && project_id && project.id == project_id.to_i
             true
           elsif project_id && user_id
-            project = projects[project_id.to_i]
-            user = users[user_id.to_i]
+            project = projects[node]
+            user = users[node]
 
             project && user ? project.team.member?(user) : false
           else

spec/factories/merge_requests.rb

@@ -44,6 +44,10 @@ FactoryGirl.define do
       state :reopened
     end
 
+    trait :locked do
+      state :locked
+    end
+
     trait :simple do
       source_branch "feature"
       target_branch "master"

spec/lib/banzai/filter/issuable_state_filter_spec.rb

+require 'spec_helper'
+
+describe Banzai::Filter::IssuableStateFilter, lib: true do
+  include ActionView::Helpers::UrlHelper
+  include FilterSpecHelper
+
+  let(:user) { create(:user) }
+
+  def create_link(data)
+    link_to('text', '', class: 'gfm has-tooltip', data: data)
+  end
+
+  it 'ignores non-GFM links' do
+    html = %(See <a href="https://google.com/">Google</a>)
+    doc = filter(html, current_user: user)
+
+    expect(doc.css('a').last.text).to eq('Google')
+  end
+
+  it 'ignores non-issuable links' do
+    project = create(:empty_project, :public)
+    link = create_link(project: project, reference_type: 'issue')
+    doc = filter(link, current_user: user)
+
+    expect(doc.css('a').last.text).to eq('text')
+  end
+
+  context 'for issue references' do
+    it 'ignores open issue references' do
+      issue = create(:issue)
+      link = create_link(issue: issue.id, reference_type: 'issue')
+      doc = filter(link, current_user: user)
+
+      expect(doc.css('a').last.text).to eq('text')
+    end
+
+    it 'ignores reopened issue references' do
+      reopened_issue = create(:issue, :reopened)
+      link = create_link(issue: reopened_issue.id, reference_type: 'issue')
+      doc = filter(link, current_user: user)
+
+      expect(doc.css('a').last.text).to eq('text')
+    end
+
+    it 'appends [closed] to closed issue references' do
+      closed_issue = create(:issue, :closed)
+      link = create_link(issue: closed_issue.id, reference_type: 'issue')
+      doc = filter(link, current_user: user)
+
+      expect(doc.css('a').last.text).to eq('text [closed]')
+    end
+  end
+
+  context 'for merge request references' do
+    it 'ignores open merge request references' do
+      mr = create(:merge_request)
+      link = create_link(merge_request: mr.id, reference_type: 'merge_request')
+      doc = filter(link, current_user: user)
+
+      expect(doc.css('a').last.text).to eq('text')
+    end
+
+    it 'ignores reopened merge request references' do
+      mr = create(:merge_request, :reopened)
+      link = create_link(merge_request: mr.id, reference_type: 'merge_request')
+      doc = filter(link, current_user: user)
+
+      expect(doc.css('a').last.text).to eq('text')
+    end
+
+    it 'ignores locked merge request references' do
+      mr = create(:merge_request, :locked)
+      link = create_link(merge_request: mr.id, reference_type: 'merge_request')
+      doc = filter(link, current_user: user)
+
+      expect(doc.css('a').last.text).to eq('text')
+    end
+
+    it 'appends [closed] to closed merge request references' do
+      mr = create(:merge_request, :closed)
+      link = create_link(merge_request: mr.id, reference_type: 'merge_request')
+      doc = filter(link, current_user: user)
+
+      expect(doc.css('a').last.text).to eq('text [closed]')
+    end
+
+    it 'appends [merged] to merged merge request references' do
+      mr = create(:merge_request, :merged)
+      link = create_link(merge_request: mr.id, reference_type: 'merge_request')
+      doc = filter(link, current_user: user)
+
+      expect(doc.css('a').last.text).to eq('text [merged]')
+    end
+  end
+end

spec/lib/banzai/filter/redactor_filter_spec.rb

@@ -15,6 +15,16 @@ describe Banzai::Filter::RedactorFilter, lib: true do
     link_to('text', '', class: 'gfm', data: data)
   end
 
+  it 'skips when the skip_redaction flag is set' do
+    user = create(:user)
+    project = create(:empty_project)
+
+    link = reference_link(project: project.id, reference_type: 'test')
+    doc = filter(link, current_user: user, skip_redaction: true)
+
+    expect(doc.css('a').length).to eq 1
+  end
+
   context 'with data-project' do
     let(:parser_class) do
       Class.new(Banzai::ReferenceParser::BaseParser) do

spec/lib/banzai/issuable_extractor_spec.rb

+require 'spec_helper'
+
+describe Banzai::IssuableExtractor, lib: true do
+  let(:project) { create(:empty_project) }
+  let(:user) { create(:user) }
+  let(:extractor) { described_class.new(project, user) }
+  let(:issue) { create(:issue, project: project) }
+  let(:merge_request) { create(:merge_request, source_project: project) }
+  let(:issue_link) do
+    html_to_node(
+      "<a href='' data-issue='#{issue.id}' data-reference-type='issue' class='gfm'>text</a>"
+    )
+  end
+  let(:merge_request_link) do
+    html_to_node(
+      "<a href='' data-merge-request='#{merge_request.id}' data-reference-type='merge_request' class='gfm'>text</a>"
+    )
+  end
+
+  def html_to_node(html)
+    Nokogiri::HTML.fragment(
+      html
+    ).children[0]
+  end
+
+  it 'returns instances of issuables for nodes with references' do
+    result = extractor.extract([issue_link, merge_request_link])
+
+    expect(result).to eq(issue_link => issue, merge_request_link => merge_request)
+  end
+
+  describe 'caching' do
+    before do
+      RequestStore.begin!
+    end
+
+    after do
+      RequestStore.end!
+      RequestStore.clear!
+    end
+
+    it 'saves records to cache' do
+      extractor.extract([issue_link, merge_request_link])
+
+      second_call_queries = ActiveRecord::QueryRecorder.new do
+        extractor.extract([issue_link, merge_request_link])
+      end.count
+
+      expect(second_call_queries).to eq 0
+    end
+  end
+end

spec/lib/banzai/object_renderer_spec.rb

@@ -3,128 +3,51 @@ require 'spec_helper'
 describe Banzai::ObjectRenderer do
   let(:project) { create(:empty_project) }
   let(:user) { project.owner }
-
-  def fake_object(attrs = {})
-    object = double(attrs.merge("new_record?" => true, "destroyed?" => true))
-    allow(object).to receive(:markdown_cache_field_for).with(:note).and_return(:note_html)
-    allow(object).to receive(:banzai_render_context).with(:note).and_return(project: nil, author: nil)
-    allow(object).to receive(:update_column).with(:note_html, anything).and_return(true)
-    object
-  end
+  let(:renderer) { described_class.new(project, user, custom_value: 'value') }
+  let(:object) { Note.new(note: 'hello', note_html: '<p>hello</p>') }
 
   describe '#render' do
     it 'renders and redacts an Array of objects' do
-      renderer = described_class.new(project, user)
-      object = fake_object(note: 'hello', note_html: nil)
-
-      expect(renderer).to receive(:render_objects).with([object], :note).
-        and_call_original
-
-      expect(renderer).to receive(:redact_documents).
-        with(an_instance_of(Array)).
-        and_call_original
-
-      expect(object).to receive(:redacted_note_html=).with('<p dir="auto">hello</p>')
-      expect(object).to receive(:user_visible_reference_count=).with(0)
-
       renderer.render([object], :note)
-    end
-  end
-
-  describe '#render_objects' do
-    it 'renders an Array of objects' do
-      object = fake_object(note: 'hello', note_html: nil)
-
-      renderer = described_class.new(project, user)
 
-      expect(renderer).to receive(:render_attributes).with([object], :note).
-        and_call_original
-
-      rendered = renderer.render_objects([object], :note)
-
-      expect(rendered).to be_an_instance_of(Array)
-      expect(rendered[0]).to be_an_instance_of(Nokogiri::HTML::DocumentFragment)
-    end
-  end
-
-  describe '#redact_documents' do
-    it 'redacts a set of documents and returns them as an Array of Hashes' do
-      doc = Nokogiri::HTML.fragment('<p>hello</p>')
-      renderer = described_class.new(project, user)
-
-      expect_any_instance_of(Banzai::Redactor).to receive(:redact).
-        with([doc]).
-        and_call_original
-
-      redacted = renderer.redact_documents([doc])
-
-      expect(redacted.count).to eq(1)
-      expect(redacted.first[:visible_reference_count]).to eq(0)
-      expect(redacted.first[:document].to_html).to eq('<p>hello</p>')
+      expect(object.redacted_note_html).to eq '<p>hello</p>'
+      expect(object.user_visible_reference_count).to eq 0
     end
-  end
 
-  describe '#context_for' do
-    let(:object) { fake_object(note: 'hello') }
-    let(:renderer) { described_class.new(project, user) }
+    it 'calls Banzai::Redactor to perform redaction' do
+      expect_any_instance_of(Banzai::Redactor).to receive(:redact).and_call_original
 
-    it 'returns a Hash' do
-      expect(renderer.context_for(object, :note)).to be_an_instance_of(Hash)
-    end
-
-    it 'includes the banzai render context for the object' do
-      expect(object).to receive(:banzai_render_context).with(:note).and_return(foo: :bar)
-      context = renderer.context_for(object, :note)
-      expect(context).to have_key(:foo)
-      expect(context[:foo]).to eq(:bar)
-    end
-  end
-
-  describe '#render_attributes' do
-    it 'renders the attribute of a list of objects' do
-      objects = [fake_object(note: 'hello', note_html: nil), fake_object(note: 'bye', note_html: nil)]
-      renderer = described_class.new(project, user)
-
-      objects.each do |object|
-        expect(Banzai).to receive(:render_field).with(object, :note).and_call_original
-      end
-
-      docs = renderer.render_attributes(objects, :note)
-
-      expect(docs[0]).to be_an_instance_of(Nokogiri::HTML::DocumentFragment)
-      expect(docs[0].to_html).to eq('<p dir="auto">hello</p>')
-
-      expect(docs[1]).to be_an_instance_of(Nokogiri::HTML::DocumentFragment)
-      expect(docs[1].to_html).to eq('<p dir="auto">bye</p>')
-    end
-
-    it 'returns when no objects to render' do
-      objects = []
-      renderer = described_class.new(project, user, pipeline: :note)
-
-      expect(renderer.render_attributes(objects, :note)).to eq([])
+      renderer.render([object], :note)
     end
-  end
 
-  describe '#base_context' do
-    let(:context) do
-      described_class.new(project, user, foo: :bar).base_context
-    end
+    it 'retrieves field content using Banzai.render_field' do
+      expect(Banzai).to receive(:render_field).with(object, :note).and_call_original
 
-    it 'returns a Hash' do
-      expect(context).to be_an_instance_of(Hash)
-    end
-
-    it 'includes the custom attributes' do
-      expect(context[:foo]).to eq(:bar)
+      renderer.render([object], :note)
     end
 
-    it 'includes the current user' do
-      expect(context[:current_user]).to eq(user)
-    end
+    it 'passes context to PostProcessPipeline' do
+      another_user = create(:user)
+      another_project = create(:empty_project)
+      object = Note.new(
+        note: 'hello',
+        note_html: 'hello',
+        author: another_user,
+        project: another_project
+      )
+
+      expect(Banzai::Pipeline::PostProcessPipeline).to receive(:to_document).with(
+        anything,
+        hash_including(
+          skip_redaction: true,
+          current_user: user,
+          project: another_project,
+          author: another_user,
+          custom_value: 'value'
+        )
+      ).and_call_original
 
-    it 'includes the current project' do
-      expect(context[:project]).to eq(project)
+      renderer.render([object], :note)
     end
   end
 end

spec/lib/banzai/reference_parser/base_parser_spec.rb

@@ -92,16 +92,26 @@ describe Banzai::ReferenceParser::BaseParser, lib: true do
   end
 
   describe '#grouped_objects_for_nodes' do
-    it 'returns a Hash grouping objects per ID' do
-      nodes = [double(:node)]
+    it 'returns a Hash grouping objects per node' do
+      link = double(:link)
+
+      expect(link).to receive(:has_attribute?).
+        with('data-user').
+        and_return(true)
+
+      expect(link).to receive(:attr).
+        with('data-user').
+        and_return(user.id.to_s)
+
+      nodes = [link]
 
       expect(subject).to receive(:unique_attribute_values).
         with(nodes, 'data-user').
-        and_return([user.id])
+        and_return([user.id.to_s])
 
       hash = subject.grouped_objects_for_nodes(nodes, User, 'data-user')
 
-      expect(hash).to eq({ user.id => user })
+      expect(hash).to eq({ link => user })
     end
 
     it 'returns an empty Hash when the list of nodes is empty' do

spec/lib/banzai/reference_parser/issue_parser_spec.rb

@@ -67,6 +67,16 @@ describe Banzai::ReferenceParser::IssueParser, lib: true do
           expect(subject.referenced_by([])).to eq([])
         end
       end
+
+      context 'when issue with given ID does not exist' do
+        before do
+          link['data-issue'] = '-1'
+        end
+
+        it 'returns an empty Array' do
+          expect(subject.referenced_by([link])).to eq([])
+        end
+      end
     end
   end
 
@@ -75,7 +85,7 @@ describe Banzai::ReferenceParser::IssueParser, lib: true do
       link['data-issue'] = issue.id.to_s
       nodes = [link]
 
-      expect(subject.issues_for_nodes(nodes)).to eq({ issue.id => issue })
+      expect(subject.issues_for_nodes(nodes)).to eq({ link => issue })
     end
   end
 end

spec/lib/banzai/reference_parser/user_parser_spec.rb

@@ -180,6 +180,15 @@ describe Banzai::ReferenceParser::UserParser, lib: true do
 
         expect(subject.nodes_user_can_reference(user, [link])).to eq([])
       end
+
+      it 'returns the nodes if the project attribute value equals the current project ID' do
+        other_user = create(:user)
+
+        link['data-project'] = project.id.to_s
+        link['data-author'] = other_user.id.to_s
+
+        expect(subject.nodes_user_can_reference(user, [link])).to eq([link])
+      end
     end
 
     context 'when the link does not have a data-author attribute' do