Merge branch '20099-markdown-with-yaml-rendering-doesn-t-handle-utf8-bom' into 'master'

Normalize markdown source by removing UTF8 BOM character

See merge request gitlab-org/gitlab!46062

changelogs/unreleased/20099-markdown-with-yaml-rendering-doesn-t-handle-utf8-bom.yml

+---
+title: Tolerate UTF8 BOM character during frontmatter rendering
+merge_request: 46062
+author:
+type: fixed

lib/banzai/filter/normalize_source_filter.rb

+# frozen_string_literal: true
+
+module Banzai
+  module Filter
+    class NormalizeSourceFilter < HTML::Pipeline::Filter
+      UTF8_BOM = "\xEF\xBB\xBF"
+
+      def call
+        # Remove UTF8_BOM from beginning of source text
+        html.delete_prefix(UTF8_BOM)
+      end
+    end
+  end
+end

lib/banzai/pipeline/pre_process_pipeline.rb

@@ -5,6 +5,7 @@ module Banzai
     class PreProcessPipeline < BasePipeline
       def self.filters
         FilterArray[
+          Filter::NormalizeSourceFilter,
           Filter::FrontMatterFilter,
           Filter::BlockquoteFenceFilter,
         ]

spec/lib/banzai/filter/normalize_source_filter_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Banzai::Filter::NormalizeSourceFilter do
+  include FilterSpecHelper
+
+  it 'removes the UTF8 BOM from the beginning of the text' do
+    content = "\xEF\xBB\xBF---"
+
+    output = filter(content)
+
+    expect(output).to match '---'
+  end
+
+  it 'does not remove those characters from anywhere else in the text' do
+    content = <<~MD
+      \xEF\xBB\xBF---
+      \xEF\xBB\xBF---
+    MD
+
+    output = filter(content)
+
+    expect(output).to match "---\n\xEF\xBB\xBF---\n"
+  end
+end

spec/lib/banzai/pipeline/pre_process_pipeline_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Banzai::Pipeline::PreProcessPipeline do
+  it 'pre-processes the source text' do
+    markdown = <<~MD
+      \xEF\xBB\xBF---
+      foo: :foo_symbol
+      bar: :bar_symbol
+      ---
+
+      >>>
+      blockquote
+      >>>
+    MD
+
+    result = described_class.call(markdown, {})
+
+    aggregate_failures do
+      expect(result[:output]).not_to include "\xEF\xBB\xBF"
+      expect(result[:output]).not_to include '---'
+      expect(result[:output]).to include "```yaml\nfoo: :foo_symbol\n"
+      expect(result[:output]).to include "> blockquote\n"
+    end
+  end
+end