Datadog integration for Gitlab pipelines v1

app/controllers/concerns/service_params.rb

@@ -23,6 +23,9 @@ module ServiceParams
     :comment_detail,
     :confidential_issues_events,
     :confluence_url,
+    :datadog_site,
+    :datadog_env,
+    :datadog_service,
     :default_irc_uri,
     :device,
     :disable_diffs,

app/models/project.rb

@@ -145,6 +145,7 @@ class Project < ApplicationRecord
   # Project services
   has_one :alerts_service
   has_one :campfire_service
+  has_one :datadog_service
   has_one :discord_service
   has_one :drone_ci_service
   has_one :emails_on_push_service
@@ -1350,6 +1351,8 @@ class Project < ApplicationRecord
   end
 
   def disabled_services
+    return ['datadog'] unless Feature.enabled?(:datadog_ci_integration, self)
+
     []
   end
 

app/models/project_services/datadog_service.rb

+# frozen_string_literal: true
+
+class DatadogService < Service
+  DEFAULT_SITE = 'datadoghq.com'.freeze
+  URL_TEMPLATE = 'https://webhooks-http-intake.logs.%{datadog_site}/v1/input/'.freeze
+  URL_TEMPLATE_API_KEYS = 'https://app.%{datadog_site}/account/settings#api'.freeze
+  URL_API_KEYS_DOCS = "https://docs.#{DEFAULT_SITE}/account_management/api-app-keys/".freeze
+
+  SUPPORTED_EVENTS = %w[
+    pipeline job
+  ].freeze
+
+  prop_accessor :datadog_site, :api_url, :api_key, :datadog_service, :datadog_env
+
+  with_options presence: true, if: :activated? do
+    validates :api_key, format: { with: /\A\w+\z/ }
+    validates :datadog_site, format: { with: /\A[\w\.]+\z/ }, unless: :api_url
+    validates :api_url, public_url: true, unless: :datadog_site
+  end
+
+  after_save :compose_service_hook, if: :activated?
+
+  def self.supported_events
+    SUPPORTED_EVENTS
+  end
+
+  def self.default_test_event
+    'pipeline'
+  end
+
+  def configurable_events
+    [] # do not allow to opt out of required hooks
+  end
+
+  def title
+    'Datadog'
+  end
+
+  def description
+    'Trace your GitLab pipelines with Datadog'
+  end
+
+  def help
+    nil
+    # Maybe adding something in the future
+    # We could link to static help pages as well
+    # [More information](#{Gitlab::Routing.url_helpers.help_page_url('integration/datadog')})"
+  end
+
+  def self.to_param
+    'datadog'
+  end
+
+  def fields
+    [
+      {
+        type: 'text', name: 'datadog_site',
+        placeholder: DEFAULT_SITE, default: DEFAULT_SITE,
+        help: 'Choose the Datadog site to send data to. Set to "datadoghq.eu" to send data to the EU site',
+        required: false
+      },
+      {
+        type: 'text', name: 'api_url', title: 'Custom URL',
+        help: '(Advanced) Define the full URL for your Datadog site directly',
+        required: false
+      },
+      {
+        type: 'password', name: 'api_key', title: 'API key',
+        help: "<a href=\"#{api_keys_url}\" target=\"_blank\">API key</a> used for authentication with Datadog",
+        required: true
+      },
+      {
+        type: 'text', name: 'datadog_service', title: 'Service', placeholder: 'gitlab-ci',
+        help: 'Name of this GitLab instance that all data will be tagged with'
+      },
+      {
+        type: 'text', name: 'datadog_env', title: 'Env',
+        help: 'The environment tag that traces will be tagged with'
+      }
+    ]
+  end
+
+  def compose_service_hook
+    hook = service_hook || build_service_hook
+    hook.url = hook_url
+    hook.save
+  end
+
+  def hook_url
+    url = api_url.presence || sprintf(URL_TEMPLATE, datadog_site: datadog_site)
+    url = URI.parse(url)
+    url.path = File.join(url.path || '/', api_key)
+    query = { service: datadog_service, env: datadog_env }.compact
+    url.query = query.to_query unless query.empty?
+    url.to_s
+  end
+
+  def api_keys_url
+    return URL_API_KEYS_DOCS unless datadog_site.presence
+
+    sprintf(URL_TEMPLATE_API_KEYS, datadog_site: datadog_site)
+  end
+
+  def execute(data)
+    return if project.disabled_services.include?(to_param)
+
+    object_kind = data[:object_kind]
+    object_kind = 'job' if object_kind == 'build'
+    return unless supported_events.include?(object_kind)
+
+    service_hook.execute(data, "#{object_kind} hook")
+  end
+
+  def test(data)
+    begin
+      result = execute(data)
+      return { success: false, result: result[:message] } if result[:http_status] != 200
+    rescue StandardError => error
+      return { success: false, result: error }
+    end
+
+    { success: true, result: result[:message] }
+  end
+end

app/models/project_services/pipelines_email_service.rb

@@ -40,6 +40,10 @@ class PipelinesEmailService < Service
     %w[pipeline]
   end
 
+  def self.default_test_event
+    'pipeline'
+  end
+
   def execute(data, force: false)
     return unless supported_events.include?(data[:object_kind])
     return unless force || should_pipeline_be_notified?(data)

app/models/service.rb

@@ -11,7 +11,7 @@ class Service < ApplicationRecord
   include EachBatch
 
   SERVICE_NAMES = %w[
-    alerts asana assembla bamboo bugzilla buildkite campfire confluence custom_issue_tracker discord
+    alerts asana assembla bamboo bugzilla buildkite campfire confluence custom_issue_tracker datadog discord
     drone_ci emails_on_push ewm external_wiki flowdock hangouts_chat hipchat irker jira
     mattermost mattermost_slash_commands microsoft_teams packagist pipelines_email
     pivotaltracker prometheus pushover redmine slack slack_slash_commands teamcity unify_circuit webex_teams youtrack
@@ -147,6 +147,10 @@ class Service < ApplicationRecord
     %w[commit push tag_push issue confidential_issue merge_request wiki_page]
   end
 
+  def self.default_test_event
+    'push'
+  end
+
   def self.event_description(event)
     ServicesHelper.service_event_description(event)
   end
@@ -386,6 +390,10 @@ class Service < ApplicationRecord
     self.class.supported_events
   end
 
+  def default_test_event
+    self.class.default_test_event
+  end
+
   def execute(data)
     # implement inside child
   end

app/services/integrations/test/project_service.rb

@@ -16,9 +16,7 @@ module Integrations
 
       def data
         strong_memoize(:data) do
-          next pipeline_events_data if integration.is_a?(::PipelinesEmailService)
-
-          case event
+          case event || integration.default_test_event
           when 'push', 'tag_push'
             push_events_data
           when 'note', 'confidential_note'
@@ -37,8 +35,6 @@ module Integrations
             deployment_events_data
           when 'release'
             releases_events_data
-          else
-            push_events_data
           end
         end
       end

config/feature_flags/development/datadog_ci_integration.yml

+---
+name: datadog_ci_integration
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/46564
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/284088
+type: development
+group: group::ecosystem
+default_enabled: false

doc/api/graphql/reference/gitlab_schema.graphql

@@ -20290,6 +20290,7 @@ enum ServiceType {
   CAMPFIRE_SERVICE
   CONFLUENCE_SERVICE
   CUSTOM_ISSUE_TRACKER_SERVICE
+  DATADOG_SERVICE
   DISCORD_SERVICE
   DRONE_CI_SERVICE
   EMAILS_ON_PUSH_SERVICE

doc/api/graphql/reference/gitlab_schema.json

@@ -58806,6 +58806,12 @@
               "isDeprecated": false,
               "deprecationReason": null
             },
+            {
+              "name": "DATADOG_SERVICE",
+              "description": null,
+              "isDeprecated": false,
+              "deprecationReason": null
+            },
             {
               "name": "DISCORD_SERVICE",
               "description": null,

doc/api/graphql/reference/index.md

@@ -4353,6 +4353,7 @@ State of a Sentry error.
 | `CAMPFIRE_SERVICE` |  |
 | `CONFLUENCE_SERVICE` |  |
 | `CUSTOM_ISSUE_TRACKER_SERVICE` |  |
+| `DATADOG_SERVICE` |  |
 | `DISCORD_SERVICE` |  |
 | `DRONE_CI_SERVICE` |  |
 | `EMAILS_ON_PUSH_SERVICE` |  |

lib/api/helpers/services_helpers.rb

@@ -304,6 +304,38 @@ module API
               desc: 'Project URL'
             }
           ],
+          'datadog' => [
+            {
+              required: true,
+              name: :api_key,
+              type: String,
+              desc: 'API key used for authentication with Datadog'
+            },
+            {
+              required: false,
+              name: :datadog_site,
+              type: String,
+              desc: 'Choose the Datadog site to send data to. Set to "datadoghq.eu" to send data to the EU site'
+            },
+            {
+              required: false,
+              name: :api_url,
+              type: String,
+              desc: '(Advanced) Define the full URL for your Datadog site directly'
+            },
+            {
+              required: false,
+              name: :datadog_service,
+              type: String,
+              desc: 'Name of this GitLab instance that all data will be tagged with'
+            },
+            {
+              required: false,
+              name: :datadog_env,
+              type: String,
+              desc: 'The environment tag that traces will be tagged with'
+            }
+          ],
           'discord' => [
             {
               required: true,
@@ -758,6 +790,7 @@ module API
           ::ConfluenceService,
           ::CampfireService,
           ::CustomIssueTrackerService,
+          ::DatadogService,
           ::DiscordService,
           ::DroneCiService,
           ::EmailsOnPushService,

spec/lib/gitlab/import_export/all_models.yml

@@ -350,6 +350,7 @@ project:
 - services
 - campfire_service
 - confluence_service
+- datadog_service
 - discord_service
 - drone_ci_service
 - emails_on_push_service

spec/models/project_services/datadog_service_spec.rb

+# frozen_string_literal: true
+require 'securerandom'
+
+require 'spec_helper'
+
+RSpec.describe DatadogService, :model do
+  let_it_be(:project) { create(:project) }
+  let_it_be(:pipeline) { create(:ci_pipeline, project: project) }
+  let_it_be(:build) { create(:ci_build, project: project) }
+
+  let(:active) { true }
+  let(:dd_site) { 'datadoghq.com' }
+  let(:default_url) { 'https://webhooks-http-intake.logs.datadoghq.com/v1/input/' }
+  let(:api_url) { nil }
+  let(:api_key) { SecureRandom.hex(32) }
+  let(:dd_env) { 'ci' }
+  let(:dd_service) { 'awesome-gitlab' }
+
+  let(:expected_hook_url) { default_url + api_key + "?env=#{dd_env}&service=#{dd_service}" }
+
+  let(:instance) do
+    described_class.new(
+      active: active,
+      project: project,
+      properties: {
+        datadog_site: dd_site,
+        api_url: api_url,
+        api_key: api_key,
+        datadog_env: dd_env,
+        datadog_service: dd_service
+      }
+    )
+  end
+
+  let(:saved_instance) do
+    instance.save!
+    instance
+  end
+
+  let(:pipeline_data) { Gitlab::DataBuilder::Pipeline.build(pipeline) }
+  let(:build_data) { Gitlab::DataBuilder::Build.build(build) }
+
+  describe 'associations' do
+    it { is_expected.to belong_to(:project) }
+    it { is_expected.to have_one(:service_hook) }
+  end
+
+  describe 'validations' do
+    subject { instance }
+
+    context 'when service is active' do
+      let(:active) { true }
+
+      it { is_expected.to validate_presence_of(:api_key) }
+      it { is_expected.to allow_value(api_key).for(:api_key) }
+      it { is_expected.not_to allow_value('87dab2403c9d462 87aec4d9214edb1e').for(:api_key) }
+      it { is_expected.not_to allow_value('................................').for(:api_key) }
+
+      context 'when selecting site' do
+        let(:dd_site) { 'datadoghq.com' }
+        let(:api_url) { nil }
+
+        it { is_expected.to validate_presence_of(:datadog_site) }
+        it { is_expected.not_to validate_presence_of(:api_url) }
+        it { is_expected.not_to allow_value('datadog hq.com').for(:datadog_site) }
+      end
+
+      context 'with custom api_url' do
+        let(:dd_site) { nil }
+        let(:api_url) { 'https://webhooks-http-intake.logs.datad0g.com/v1/input/' }
+
+        it { is_expected.not_to validate_presence_of(:datadog_site) }
+        it { is_expected.to validate_presence_of(:api_url) }
+        it { is_expected.to allow_value(api_url).for(:api_url) }
+        it { is_expected.not_to allow_value('example.com').for(:api_url) }
+      end
+
+      context 'when missing site and api_url' do
+        let(:dd_site) { nil }
+        let(:api_url) { nil }
+
+        it { is_expected.not_to be_valid }
+        it { is_expected.to validate_presence_of(:datadog_site) }
+        it { is_expected.to validate_presence_of(:api_url) }
+      end
+    end
+
+    context 'when service is not active' do
+      let(:active) { false }
+
+      it { is_expected.to be_valid }
+      it { is_expected.not_to validate_presence_of(:api_key) }
+    end
+  end
+
+  describe '#hook_url' do
+    subject { instance.hook_url }
+
+    context 'with standard site URL' do
+      it { is_expected.to eq(expected_hook_url) }
+    end
+
+    context 'with custom URL' do
+      let(:api_url) { 'https://webhooks-http-intake.logs.datad0g.com/v1/input/' }
+
+      it { is_expected.to eq(api_url + api_key + "?env=#{dd_env}&service=#{dd_service}") }
+
+      context 'blank' do
+        let(:api_url) { '' }
+
+        it { is_expected.to eq(expected_hook_url) }
+      end
+    end
+
+    context 'without optional params' do
+      let(:dd_service) { nil }
+      let(:dd_env) { nil }
+
+      it { is_expected.to eq(default_url + api_key) }
+    end
+  end
+
+  describe '#api_keys_url' do
+    subject { instance.api_keys_url }
+
+    it { is_expected.to eq("https://app.#{dd_site}/account/settings#api") }
+
+    context 'with unset datadog_site' do
+      let(:dd_site) { nil }
+
+      it { is_expected.to eq("https://docs.datadoghq.com/account_management/api-app-keys/") }
+    end
+  end
+
+  describe '#test' do
+    context 'when request is succesful' do
+      subject { saved_instance.test(pipeline_data) }
+
+      before do
+        stub_request(:post, expected_hook_url).to_return(body: 'OK')
+      end
+      it { is_expected.to eq({ success: true, result: 'OK' }) }
+    end
+
+    context 'when request fails' do
+      subject { saved_instance.test(pipeline_data) }
+
+      before do
+        stub_request(:post, expected_hook_url).to_return(body: 'CRASH!!!', status: 500)
+      end
+      it { is_expected.to eq({ success: false, result: 'CRASH!!!' }) }
+    end
+  end
+
+  describe '#execute' do
+    before do
+      stub_request(:post, expected_hook_url)
+      saved_instance.execute(data)
+    end
+
+    context 'with pipeline data' do
+      let(:data) { pipeline_data }
+      let(:expected_headers) do
+        { WebHookService::GITLAB_EVENT_HEADER => 'Pipeline Hook' }
+      end
+
+      it { expect(a_request(:post, expected_hook_url).with(headers: expected_headers)).to have_been_made }
+    end
+
+    context 'with job data' do
+      let(:data) { build_data }
+      let(:expected_headers) do
+        { WebHookService::GITLAB_EVENT_HEADER => 'Job Hook' }
+      end
+
+      it { expect(a_request(:post, expected_hook_url).with(headers: expected_headers)).to have_been_made }
+    end
+  end
+end

spec/models/project_spec.rb

@@ -5565,6 +5565,26 @@ RSpec.describe Project, factory_default: :keep do
     end
   end
 
+  describe '#disabled_services' do
+    subject { build(:project).disabled_services }
+
+    context 'without datadog_ci_integration' do
+      before do
+        stub_feature_flags(datadog_ci_integration: false)
+      end
+
+      it { is_expected.to include('datadog') }
+    end
+
+    context 'with datadog_ci_integration' do
+      before do
+        stub_feature_flags(datadog_ci_integration: true)
+      end
+
+      it { is_expected.not_to include('datadog') }
+    end
+  end
+
   describe '#find_or_initialize_service' do
     it 'avoids N+1 database queries' do
       allow(Service).to receive(:available_services_names).and_return(%w[prometheus pushover])

spec/requests/api/services_spec.rb

@@ -76,7 +76,9 @@ RSpec.describe API::Services do
 
         required_attributes = service_attrs_list.select do |attr|
           service_klass.validators_on(attr).any? do |v|
-            v.class == ActiveRecord::Validations::PresenceValidator
+            v.class == ActiveRecord::Validations::PresenceValidator &&
+            # exclude presence validators with conditional since those are not really required
+            ![:if, :unless].any? { |cond| v.options.include?(cond) }
           end
         end
 

spec/support/shared_contexts/services_shared_context.rb

@@ -16,6 +16,8 @@ Service.available_services_names.each do |service|
           hash.merge!(k => 'secrettoken')
         elsif service == 'confluence' && k == :confluence_url
           hash.merge!(k => 'https://example.atlassian.net/wiki')
+        elsif service == 'datadog' && k == :datadog_site
+          hash.merge!(k => 'datadoghq.com')
         elsif k =~ /^(.*_url|url|webhook)/
           hash.merge!(k => "http://example.com")
         elsif service_klass.method_defined?("#{k}?")