resolves #241744 add Kroki to support more diagrams in AsciiDoc and Markdown

app/helpers/application_settings_helper.rb

@@ -254,6 +254,8 @@ module ApplicationSettingsHelper
       :password_authentication_enabled_for_git,
       :performance_bar_allowed_group_path,
       :performance_bar_enabled,
+      :kroki_enabled,
+      :kroki_url,
       :plantuml_enabled,
       :plantuml_url,
       :polling_interval_multiplier,

app/models/application_setting.rb

@@ -128,6 +128,10 @@ class ApplicationSetting < ApplicationRecord
             presence: true,
             if: :unique_ips_limit_enabled
 
+  validates :kroki_url,
+            presence: true,
+            if: :kroki_enabled
+
   validates :plantuml_url,
             presence: true,
             if: :plantuml_enabled

app/views/admin/application_settings/_kroki.html.haml

+- expanded = integration_expanded?('kroki_')
+%section.settings.as-kroki.no-animate#js-kroki-settings{ class: ('expanded' if expanded) }
+  .settings-header
+    %h4
+      = _('Kroki')
+    %button.btn.btn-default.js-settings-toggle{ type: 'button' }
+      = expanded ? _('Collapse') : _('Expand')
+    %p
+      = _('Allow rendering of diagrams in AsciiDoc and Markdown documents using %{link}.').html_safe % { link: link_to('Kroki', 'https://kroki.io', target: '_blank') }
+  .settings-content
+    = form_for @application_setting, url: general_admin_application_settings_path(anchor: 'js-kroki-settings'), html: { class: 'fieldset-form' } do |f|
+      = form_errors(@application_setting) if expanded
+
+      %fieldset
+        .form-group
+          .form-check
+            = f.check_box :kroki_enabled, class: 'form-check-input'
+            = f.label :kroki_enabled, _('Enable Kroki'), class: 'form-check-label'
+        .form-group
+          = f.label :kroki_url, 'Kroki URL', class: 'label-bold'
+          = f.text_field :kroki_url, class: 'form-control', placeholder: 'http://kroki-instance:8080'
+          .form-text.text-muted
+            = (_('When Kroki is enabled, GitLab sends diagrams to an instance of Kroki to display them as images. You can use the free public cloud instance %{kroki_public_url} or you can %{install_link} on your own infrastructure. Once you\'ve installed Kroki, make sure to update the server URL to point to your instance.') % { kroki_public_url: '<code>https://kroki.io</code>', install_link: link_to('install Kroki', 'https://docs.kroki.io/kroki/setup/install/', target: '_blank') }).html_safe
+
+      = f.submit _('Save changes'), class: "btn btn-success"

app/views/admin/application_settings/general.html.haml

@@ -117,6 +117,7 @@
 
 = render_if_exists 'admin/application_settings/elasticsearch_form'
 = render 'admin/application_settings/gitpod'
+= render 'admin/application_settings/kroki'
 = render 'admin/application_settings/plantuml'
 = render 'admin/application_settings/sourcegraph'
 = render_if_exists 'admin/application_settings/slack'

changelogs/unreleased/241744-support-kroki.yml

+---
+title: "Add Kroki to support more diagrams in AsciiDoc and Markdown"
+merge_request: 44851
+author: Guillaume Grossetie
+type: added

db/migrate/20200913115700_add_kroki_application_settings.rb

+# frozen_string_literal: true
+
+class AddKrokiApplicationSettings < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  # rubocop:disable Migration/AddLimitToTextColumns
+  # limit is added in 20201011005400_add_text_limit_to_application_settings_kroki_url.rb
+  #
+  def change
+    add_column :application_settings, :kroki_url, :text
+    add_column :application_settings, :kroki_enabled, :boolean, default: false, null: false
+  end
+  # rubocop:enable Migration/AddLimitToTextColumns
+end

db/migrate/20201011005400_add_text_limit_to_application_settings_kroki_url.rb

+# frozen_string_literal: true
+
+class AddTextLimitToApplicationSettingsKrokiUrl < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_text_limit :application_settings, :kroki_url, 1024
+  end
+
+  def down
+    # Down is required as `add_text_limit` is not reversible
+    #
+    remove_text_limit :application_settings, :kroki_url
+  end
+end

db/schema_migrations/20200913115700

+dd2ada53f01debcc91070525e4386db959b91881a8945e9082d0b3318ceb35cf
\ No newline at end of file

db/schema_migrations/20201011005400

+07bfc8e9a684ae64b7d78c9d867f9bafebd46678f6f168aa87d2ad7f0e85d75e
\ No newline at end of file

db/structure.sql

@@ -9320,6 +9320,8 @@ CREATE TABLE application_settings (
     elasticsearch_indexed_file_size_limit_kb integer DEFAULT 1024 NOT NULL,
     enforce_namespace_storage_limit boolean DEFAULT false NOT NULL,
     container_registry_delete_tags_service_timeout integer DEFAULT 250 NOT NULL,
+    kroki_url character varying,
+    kroki_enabled boolean,
     elasticsearch_client_request_timeout integer DEFAULT 0 NOT NULL,
     gitpod_enabled boolean DEFAULT false NOT NULL,
     gitpod_url text DEFAULT 'https://gitpod.io/'::text,
@@ -9347,6 +9349,7 @@ CREATE TABLE application_settings (
     secret_detection_revocation_token_types_url text,
     cloud_license_enabled boolean DEFAULT false NOT NULL,
     CONSTRAINT app_settings_registry_exp_policies_worker_capacity_positive CHECK ((container_registry_expiration_policies_worker_capacity >= 0)),
+    CONSTRAINT check_17d9558205 CHECK ((char_length((kroki_url)::text) <= 1024)),
     CONSTRAINT check_2dba05b802 CHECK ((char_length(gitpod_url) <= 255)),
     CONSTRAINT check_51700b31b5 CHECK ((char_length(default_branch_name) <= 255)),
     CONSTRAINT check_57123c9593 CHECK ((char_length(help_page_documentation_base_url) <= 255)),

lib/api/settings.rb

@@ -102,6 +102,10 @@ module API
       optional :performance_bar_allowed_group_id, type: String, desc: 'Deprecated: Use :performance_bar_allowed_group_path instead. Path of the group that is allowed to toggle the performance bar.' # support legacy names, can be removed in v6
       optional :performance_bar_allowed_group_path, type: String, desc: 'Path of the group that is allowed to toggle the performance bar.'
       optional :performance_bar_enabled, type: String, desc: 'Deprecated: Pass `performance_bar_allowed_group_path: nil` instead. Allow enabling the performance.' # support legacy names, can be removed in v6
+      optional :kroki_enabled, type: Boolean, desc: 'Enable Kroki'
+      given kroki_enabled: ->(val) { val } do
+        requires :kroki_url, type: String, desc: 'The Kroki server URL'
+      end
       optional :plantuml_enabled, type: Boolean, desc: 'Enable PlantUML'
       given plantuml_enabled: ->(val) { val } do
         requires :plantuml_url, type: String, desc: 'The PlantUML server URL'

lib/banzai/filter/base_sanitization_filter.rb

@@ -25,7 +25,7 @@ module Banzai
 
           # Allow data-math-style attribute in order to support LaTeX formatting
           whitelist[:attributes]['code'] = %w(data-math-style)
-          whitelist[:attributes]['pre'] = %w(data-math-style data-mermaid-style)
+          whitelist[:attributes]['pre'] = %w(data-math-style data-mermaid-style data-kroki-style)
 
           # Allow html5 details/summary elements
           whitelist[:elements].push('details')

lib/banzai/filter/kroki_filter.rb

+# frozen_string_literal: true
+
+require "nokogiri"
+require "zlib"
+require "base64"
+
+module Banzai
+  module Filter
+    # HTML that replaces all diagrams supported by Kroki with the corresponding img tags.
+    #
+    class KrokiFilter < HTML::Pipeline::Filter
+      DIAGRAM_SELECTORS = ::Gitlab::Kroki::DIAGRAM_TYPES.map do |diagram_type|
+        %(pre[lang="#{diagram_type}"] > code)
+      end.join(', ')
+
+      def call
+        # QUESTION: should we make Kroki and PlantUML mutually exclusive?
+        # Potentially, Kroki and PlantUML could work side by side.
+        # In fact, if both PlantUML and Kroki are enabled, PlantUML could still render PlantUML diagrams and Kroki could render the other diagrams?
+        # Having said that, since Kroki can render PlantUML diagrams, maybe it will be confusing...
+        #
+        # What about Mermaid? should we keep client side rendering for Mermaid?
+        return doc unless settings.kroki_enabled && doc.at(DIAGRAM_SELECTORS)
+
+        diagram_format = "svg"
+        doc.css(DIAGRAM_SELECTORS).each do |node|
+          diagram_type = node.parent['lang']
+          img_tag = Nokogiri::HTML::DocumentFragment.parse(%(<img src="#{create_image_src(diagram_type, diagram_format, node.content)}"/>))
+          node.parent.replace(img_tag)
+        end
+
+        doc
+      end
+
+      private
+
+      # QUESTION: should should we use the asciidoctor-kroki gem to delegate this logic?
+      def create_image_src(type, format, text)
+        data = Base64.urlsafe_encode64(Zlib::Deflate.deflate(text, 9))
+        "#{settings.kroki_url}/#{type}/#{format}/#{data}"
+      end
+
+      def settings
+        Gitlab::CurrentSettings.current_application_settings
+      end
+    end
+  end
+end

lib/banzai/filter/syntax_highlight_filter.rb

@@ -14,7 +14,7 @@ module Banzai
       LANG_PARAMS_ATTR = 'data-lang-params'
 
       def call
-        doc.search('pre:not([data-math-style]):not([data-mermaid-style]) > code').each do |node|
+        doc.search('pre:not([data-math-style]):not([data-mermaid-style]):not([data-kroki-style]) > code').each do |node|
           highlight_node(node)
         end
 
@@ -86,7 +86,7 @@ module Banzai
       end
 
       def use_rouge?(language)
-        %w(math mermaid plantuml suggestion).exclude?(language)
+        (%w(math suggestion) + ::Gitlab::Kroki::DIAGRAM_TYPES).exclude?(language)
       end
     end
   end

lib/banzai/pipeline/ascii_doc_pipeline.rb

@@ -9,6 +9,7 @@ module Banzai
           Filter::AssetProxyFilter,
           Filter::SyntaxHighlightFilter,
           Filter::ExternalLinkFilter,
+          Filter::KrokiFilter,
           Filter::PlantumlFilter,
           Filter::ColorFilter,
           Filter::ImageLazyLoadFilter,

lib/banzai/pipeline/gfm_pipeline.rb

@@ -19,6 +19,7 @@ module Banzai
           Filter::SyntaxHighlightFilter,
           Filter::MathFilter,
           Filter::ColorFilter,
+          Filter::KrokiFilter,
           Filter::MermaidFilter,
           Filter::VideoLinkFilter,
           Filter::AudioLinkFilter,

lib/gitlab/asciidoc.rb

@@ -48,6 +48,12 @@ module Gitlab
       extensions = proc do
         include_processor ::Gitlab::Asciidoc::IncludeProcessor.new(context)
         block ::Gitlab::Asciidoc::MermaidBlockProcessor
+
+        if Gitlab::CurrentSettings.kroki_enabled
+          ::Gitlab::Kroki::DIAGRAM_TYPES.each do |name|
+            block ::Gitlab::Asciidoc::KrokiBlockProcessor, name
+          end
+        end
       end
 
       extra_attrs = path_attrs(context[:requested_path])

lib/gitlab/asciidoc/kroki_block_processor.rb

+# frozen_string_literal: true
+
+require 'asciidoctor'
+
+module Gitlab
+  module Asciidoc
+    # Kroki BlockProcessor
+    #
+    class KrokiBlockProcessor < ::Asciidoctor::Extensions::BlockProcessor
+      use_dsl
+
+      on_context :literal, :listing
+      parse_content_as :simple
+
+      def process(parent, reader, attrs)
+        diagram_type = @name
+        diagram_text = reader.string
+        create_kroki_source_block(parent, diagram_type, diagram_text, attrs)
+      end
+
+      private
+
+      def create_kroki_source_block(parent, diagram_type, diagram_text, attrs)
+        # If "subs" attribute is specified, substitute accordingly.
+        # Be careful not to specify "specialcharacters" or your diagram code won't be valid anymore!
+        subs = attrs['subs']
+        diagram_text = parent.apply_subs(diagram_text, parent.resolve_subs(subs)) if subs
+        html = %(<div><pre data-kroki-style="display" lang="#{diagram_type}"><code>#{CGI.escape_html(diagram_text)}</code></pre></div>)
+        ::Asciidoctor::Block.new(parent, :pass, {
+          content_model: :raw,
+          source: html,
+          subs: :default
+        }.merge(attrs))
+      end
+    end
+  end
+end

lib/gitlab/kroki.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Kroki
+    # QUESTION: should we use the asciidoctor-kroki gem?
+    DIAGRAM_TYPES = %w(plantuml ditaa graphviz blockdiag seqdiag actdiag nwdiag packetdiag rackdiag c4plantuml erd mermaid nomnoml svgbob umlet vega vegalite wavedrom).freeze
+  end
+end

locale/gitlab.pot

@@ -2849,6 +2849,9 @@ msgstr ""
 msgid "Allow rendering of PlantUML diagrams in Asciidoc documents."
 msgstr ""
 
+msgid "Allow rendering of diagrams in AsciiDoc and Markdown documents using %{link}."
+msgstr ""
+
 msgid "Allow repository mirroring to be configured by project maintainers"
 msgstr ""
 
@@ -10244,6 +10247,9 @@ msgstr ""
 msgid "Enable Incident Management inbound alert limit"
 msgstr ""
 
+msgid "Enable Kroki"
+msgstr ""
+
 msgid "Enable PlantUML"
 msgstr ""
 
@@ -15700,6 +15706,9 @@ msgstr ""
 msgid "Ki"
 msgstr ""
 
+msgid "Kroki"
+msgstr ""
+
 msgid "Kubernetes"
 msgstr ""
 
@@ -30674,6 +30683,9 @@ msgstr ""
 msgid "What’s your experience level?"
 msgstr ""
 
+msgid "When Kroki is enabled, GitLab sends diagrams to an instance of Kroki to display them as images. You can use the free public cloud instance %{kroki_public_url} or you can %{install_link} on your own infrastructure. Once you've installed Kroki, make sure to update the server URL to point to your instance."
+msgstr ""
+
 msgid "When a deployment job is successful, skip older deployment jobs that are still pending"
 msgstr ""
 

spec/lib/banzai/filter/kroki_filter_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Banzai::Filter::KrokiFilter do
+  include FilterSpecHelper
+
+  it 'replaces nomnoml pre tag with img tag' do
+    stub_application_setting(kroki_enabled: true, kroki_url: "http://localhost:8000")
+    doc = filter("<pre lang='nomnoml'><code>[Pirate|eyeCount: Int|raid();pillage()|\n  [beard]--[parrot]\n  [beard]-:>[foul mouth]\n]</code></pre>")
+
+    expect(doc.to_s).to eq '<img src="http://localhost:8000/nomnoml/svg/eNqLDsgsSixJrUmtTHXOL80rsVLwzCupKUrMTNHQtC7IzMlJTE_V0KzhUlCITkpNLEqJ1dWNLkgsKsoviUUSs7KLTssvzVHIzS8tyYjligUAMhEd0g==">'
+  end
+
+  it 'does not replace nomnoml pre tag with img tag if disabled' do
+    stub_application_setting(kroki_enabled: false)
+    doc = filter("<pre lang='nomnoml'><code>[Pirate|eyeCount: Int|raid();pillage()|\n  [beard]--[parrot]\n  [beard]-:>[foul mouth]\n]</code></pre>")
+
+    expect(doc.to_s).to eq "<pre lang=\"nomnoml\"><code>[Pirate|eyeCount: Int|raid();pillage()|\n  [beard]--[parrot]\n  [beard]-:&gt;[foul mouth]\n]</code></pre>"
+  end
+end

spec/lib/gitlab/asciidoc_spec.rb

@@ -462,6 +462,29 @@ module Gitlab
           expect(render(input, context)).to include(output.strip)
         end
       end
+
+      context 'with Kroki enabled' do
+        before do
+          allow_any_instance_of(ApplicationSetting).to receive(:kroki_enabled).and_return(true)
+          allow_any_instance_of(ApplicationSetting).to receive(:kroki_url).and_return('https://kroki.io')
+        end
+        it 'converts a graphviz diagram to image' do
+          input = <<~ADOC
+            [graphviz]
+            ....
+            digraph G {
+              Hello->World
+            }
+            ....
+          ADOC
+
+          output = <<~HTML
+            <div><a class="no-attachment-icon" href="https://kroki.io/graphviz/svg/eNpLyUwvSizIUHBXqOZSUPBIzcnJ17ULzy_KSeGqBQCEzQka" target="_blank" rel="noopener noreferrer"><img src="data:image/gif;base64,R0lGODlhAQABAAAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" class="lazy" data-src="https://kroki.io/graphviz/svg/eNpLyUwvSizIUHBXqOZSUPBIzcnJ17ULzy_KSeGqBQCEzQka"></a></div>
+          HTML
+
+          expect(render(input, context)).to include(output.strip)
+        end
+      end
     end
 
     context 'with project' do