Merge branch 'link-prometheus-alerts-and-issues' into 'master'

Link Gitlab managed Prometheus alerts and issues

See merge request gitlab-org/gitlab!17477

db/migrate/20190905022045_add_issues_prometheus_alert_event_join_table.rb

+# frozen_string_literal: true
+
+class AddIssuesPrometheusAlertEventJoinTable < ActiveRecord::Migration[5.2]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    create_table :issues_prometheus_alert_events, id: false do |t|
+      t.references :issue, null: false,
+        index: false, # Uses the index below
+        foreign_key: { on_delete: :cascade }
+      t.references :prometheus_alert_event, null: false,
+        index: { name: 'issue_id_issues_prometheus_alert_events_index' },
+        foreign_key: { on_delete: :cascade }
+
+      t.timestamps_with_timezone
+      t.index [:issue_id, :prometheus_alert_event_id],
+        unique: true, name: 'issue_id_prometheus_alert_event_id_index'
+    end
+  end
+end

db/schema.rb

@@ -1877,6 +1877,15 @@ ActiveRecord::Schema.define(version: 2019_09_29_180827) do
     t.index ["updated_by_id"], name: "index_issues_on_updated_by_id", where: "(updated_by_id IS NOT NULL)"
   end
 
+  create_table "issues_prometheus_alert_events", id: false, force: :cascade do |t|
+    t.bigint "issue_id", null: false
+    t.bigint "prometheus_alert_event_id", null: false
+    t.datetime_with_timezone "created_at", null: false
+    t.datetime_with_timezone "updated_at", null: false
+    t.index ["issue_id", "prometheus_alert_event_id"], name: "issue_id_prometheus_alert_event_id_index", unique: true
+    t.index ["prometheus_alert_event_id"], name: "issue_id_issues_prometheus_alert_events_index"
+  end
+
   create_table "jira_connect_installations", force: :cascade do |t|
     t.string "client_key"
     t.string "encrypted_shared_secret"
@@ -4071,6 +4080,8 @@ ActiveRecord::Schema.define(version: 2019_09_29_180827) do
   add_foreign_key "issues", "users", column: "author_id", name: "fk_05f1e72feb", on_delete: :nullify
   add_foreign_key "issues", "users", column: "closed_by_id", name: "fk_c63cbf6c25", on_delete: :nullify
   add_foreign_key "issues", "users", column: "updated_by_id", name: "fk_ffed080f01", on_delete: :nullify
+  add_foreign_key "issues_prometheus_alert_events", "issues", on_delete: :cascade
+  add_foreign_key "issues_prometheus_alert_events", "prometheus_alert_events", on_delete: :cascade
   add_foreign_key "jira_connect_subscriptions", "jira_connect_installations", on_delete: :cascade
   add_foreign_key "jira_connect_subscriptions", "namespaces", on_delete: :cascade
   add_foreign_key "jira_tracker_data", "services", on_delete: :cascade

ee/app/models/ee/issue.rb

@@ -27,6 +27,9 @@ module EE
         end
       end
 
+      has_and_belongs_to_many :prometheus_alert_events, join_table: :issues_prometheus_alert_events
+      has_many :prometheus_alerts, through: :prometheus_alert_events
+
       validates :weight, allow_nil: true, numericality: { greater_than_or_equal_to: 0 }
     end
 

ee/app/models/prometheus_alert.rb

@@ -14,6 +14,7 @@ class PrometheusAlert < ApplicationRecord
   belongs_to :prometheus_metric, required: true, validate: true, inverse_of: :prometheus_alerts
 
   has_many :prometheus_alert_events, inverse_of: :prometheus_alert
+  has_many :related_issues, through: :prometheus_alert_events
 
   after_save :clear_prometheus_adapter_cache!
   after_destroy :clear_prometheus_adapter_cache!

ee/app/models/prometheus_alert_event.rb

@@ -3,6 +3,7 @@
 class PrometheusAlertEvent < ApplicationRecord
   belongs_to :project, required: true, validate: true, inverse_of: :prometheus_alert_events
   belongs_to :prometheus_alert, required: true, validate: true, inverse_of: :prometheus_alert_events
+  has_and_belongs_to_many :related_issues, class_name: 'Issue', join_table: :issues_prometheus_alert_events
 
   validates :payload_key, uniqueness: { scope: :prometheus_alert_id }
 
@@ -63,6 +64,10 @@ class PrometheusAlertEvent < ApplicationRecord
     find_or_initialize_by(project: project, prometheus_alert: alert, payload_key: payload_key)
   end
 
+  def self.find_by_payload_key(payload_key)
+    find_by(payload_key: payload_key)
+  end
+
   def self.status_value_for(name)
     state_machines[:status].states[name].value
   end

ee/app/services/projects/prometheus/alerts/notify_service.rb

@@ -10,9 +10,9 @@ module Projects
           return false unless valid_version?
           return false unless valid_alert_manager_token?(token)
 
+          persist_events
           send_alert_email if send_email?
           process_incident_issues if create_issue?
-          persist_events
 
           true
         end
@@ -129,7 +129,7 @@ module Projects
 
         def process_incident_issues
           firings.each do |alert|
-            IncidentManagement::ProcessAlertWorker
+            IncidentManagement::ProcessPrometheusAlertWorker
               .perform_async(project.id, alert.to_h)
           end
         end

ee/app/workers/all_queues.yml

@@ -52,6 +52,7 @@
 - pipeline_default:ci_pipeline_bridge_status
 
 - incident_management:incident_management_process_alert
+- incident_management:incident_management_process_prometheus_alert
 
 - jira_connect:jira_connect_sync_branch
 - jira_connect:jira_connect_sync_merge_request

ee/app/workers/incident_management/process_prometheus_alert_worker.rb

+# frozen_string_literal: true
+
+module IncidentManagement
+  class ProcessPrometheusAlertWorker
+    include ApplicationWorker
+
+    queue_namespace :incident_management
+
+    def perform(project_id, alert_hash)
+      project = find_project(project_id)
+      return unless project
+
+      event = find_prometheus_alert_event(alert_hash)
+      issue = create_issue(project, alert_hash)
+
+      relate_issue_to_event(event, issue)
+    end
+
+    private
+
+    def find_project(project_id)
+      Project.find_by_id(project_id)
+    end
+
+    def find_prometheus_alert_event(alert_hash)
+      started_at = alert_hash.dig('startsAt')
+      gitlab_alert_id = alert_hash.dig('labels', 'gitlab_alert_id')
+      payload_key = PrometheusAlertEvent.payload_key_for(gitlab_alert_id, started_at)
+
+      PrometheusAlertEvent.find_by_payload_key(payload_key)
+    end
+
+    def create_issue(project, alert)
+      IncidentManagement::CreateIssueService
+        .new(project, alert)
+        .execute
+        .dig(:issue)
+    end
+
+    def relate_issue_to_event(event, issue)
+      return unless event && issue
+
+      if event.related_issues.exclude?(issue)
+        event.related_issues << issue
+      end
+    end
+  end
+end

ee/changelogs/unreleased/link-prometheus-alerts-and-issues.yml

+---
+title: Link Gitlab managed Prometheus alerts and issues
+merge_request: 17477
+author:
+type: added

ee/spec/models/issue_spec.rb

@@ -45,6 +45,8 @@ describe Issue do
   describe 'relations' do
     it { is_expected.to have_many(:designs) }
     it { is_expected.to have_many(:design_versions) }
+    it { is_expected.to have_and_belong_to_many(:prometheus_alert_events) }
+    it { is_expected.to have_many(:prometheus_alerts) }
 
     describe 'versions.most_recent' do
       it 'returns the most recent version' do

ee/spec/services/projects/prometheus/alerts/notify_service_spec.rb

@@ -33,7 +33,7 @@ describe Projects::Prometheus::Alerts::NotifyService do
     let(:create_incident_service) { spy }
 
     it 'processes issues', :sidekiq do
-      expect(IncidentManagement::ProcessAlertWorker)
+      expect(IncidentManagement::ProcessPrometheusAlertWorker)
         .to receive(:perform_async)
         .with(project.id, kind_of(Hash))
         .exactly(amount).times
@@ -46,7 +46,7 @@ describe Projects::Prometheus::Alerts::NotifyService do
 
   shared_examples 'does not process incident issues' do
     it 'does not process issues' do
-      expect(IncidentManagement::ProcessAlertWorker)
+      expect(IncidentManagement::ProcessPrometheusAlertWorker)
         .not_to receive(:perform_async)
 
       expect(subject).to eq(true)

ee/spec/workers/incident_management/process_prometheus_alert_worker_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe IncidentManagement::ProcessPrometheusAlertWorker do
+  describe '#perform' do
+    let_it_be(:project) { create(:project) }
+    let_it_be(:prometheus_alert) { create(:prometheus_alert, project: project) }
+
+    before_all do
+      payload_key = PrometheusAlertEvent.payload_key_for(prometheus_alert.id, prometheus_alert.created_at.rfc3339)
+      create(:prometheus_alert_event, prometheus_alert: prometheus_alert, payload_key: payload_key)
+    end
+
+    let(:alert_params) do
+      {
+        startsAt: prometheus_alert.created_at.rfc3339,
+        labels: {
+          gitlab_alert_id: prometheus_alert.id
+        }
+      }.with_indifferent_access
+    end
+
+    it 'creates an issue' do
+      expect { subject.perform(project.id, alert_params) }
+        .to change(Issue, :count)
+        .by(1)
+    end
+
+    it 'relates issue to an event' do
+      expect { subject.perform(project.id, alert_params) }
+        .to change(prometheus_alert.related_issues, :count).from(0).to(1)
+    end
+
+    context 'when project could not be found' do
+      it 'does not create an issue' do
+        expect { subject.perform('1234', alert_params) }
+          .not_to change(Issue, :count)
+      end
+
+      it 'does not relate issue to an event' do
+        expect { subject.perform('1234', alert_params) }
+          .not_to change(prometheus_alert.related_issues, :count)
+      end
+    end
+
+    context 'when event could not be found' do
+      before do
+        alert_params[:labels][:gitlab_alert_id] = '1234'
+      end
+
+      it 'does not create an issue' do
+        expect { subject.perform(project.id, alert_params) }
+          .not_to change(Issue, :count)
+      end
+
+      it 'does not relate issue to an event' do
+        expect { subject.perform(project.id, alert_params) }
+          .not_to change(Issue, :count)
+      end
+    end
+
+    context 'when issue could not be created' do
+      before do
+        allow_any_instance_of(IncidentManagement::CreateIssueService)
+          .to receive(:execute)
+          .and_return( { error: true } )
+      end
+
+      it 'does not relate issue to an event' do
+        expect { subject.perform(project.id, alert_params) }
+          .not_to change(prometheus_alert.related_issues, :count)
+      end
+    end
+  end
+end

spec/lib/gitlab/import_export/all_models.yml

@@ -25,6 +25,8 @@ issues:
 - epic
 - designs
 - design_versions
+- prometheus_alerts
+- prometheus_alert_events
 events:
 - author
 - project