Merge branch '13402-close-alert-when-end-time-given' into 'master'

Resolve alert when end_time is given

Closes #217049 and #13402

See merge request gitlab-org/gitlab!41648

app/services/projects/alerting/notify_service.rb

@@ -42,11 +42,37 @@ module Projects
       end
 
       def process_existing_alert(alert)
-        alert.register_new_event!
+        if am_alert_params[:ended_at].present?
+          process_resolved_alert(alert)
+        else
+          alert.register_new_event!
+        end
+
+        alert
+      end
+
+      def process_resolved_alert(alert)
+        return unless auto_close_incident?
+
+        if alert.resolve(am_alert_params[:ended_at])
+          close_issue(alert.issue)
+        end
+
+        alert
+      end
+
+      def close_issue(issue)
+        return if issue.blank? || issue.closed?
+
+        ::Issues::CloseService
+          .new(project, User.alert_bot)
+          .execute(issue, system_note: false)
+
+        SystemNoteService.auto_resolve_prometheus_alert(issue, project, User.alert_bot) if issue.reset.closed?
       end
 
       def create_alert
-        alert = AlertManagement::Alert.create(am_alert_params)
+        alert = AlertManagement::Alert.create(am_alert_params.except(:ended_at))
         alert.execute_services if alert.persisted?
         SystemNoteService.create_new_alert(alert, 'Generic Alert Endpoint')
 

changelogs/unreleased/13402-close-alert-when-end-time-given.yml

+---
+title: Automatically resolve alert when receiving end time
+merge_request: 41648
+author:
+type: added

doc/operations/incident_management/generic_alerts.md

@@ -43,6 +43,7 @@ You can customize the payload by sending the following parameters. All fields ot
 | `title` | String | The title of the incident. Required. |
 | `description` | String | A high-level summary of the problem. |
 | `start_time` | DateTime | The time of the incident. If none is provided, a timestamp of the issue will be used. |
+| `end_time` | DateTime | For existing alerts only. When provided, the alert is resolved and the associated incident is closed. |
 | `service` | String | The affected service. |
 | `monitoring_tool` | String |  The name of the associated monitoring tool. |
 | `hosts` | String or Array | One or more hosts, as to where this incident occurred. |

lib/gitlab/alert_management/alert_params.rb

@@ -20,6 +20,7 @@ module Gitlab
           hosts: Array(annotations[:hosts]),
           payload: payload,
           started_at: parsed_payload['startsAt'],
+          ended_at: parsed_payload['endsAt'],
           severity: annotations[:severity],
           fingerprint: annotations[:fingerprint],
           environment: annotations[:environment]

lib/gitlab/alerting/notification_payload_parser.rb

@@ -20,7 +20,8 @@ module Gitlab
       def call
         {
           'annotations' => annotations,
-          'startsAt' => starts_at
+          'startsAt' => starts_at,
+          'endsAt' => ends_at
         }.compact
       end
 
@@ -74,6 +75,12 @@ module Gitlab
         current_time
       end
 
+      def ends_at
+        Time.parse(payload[:end_time].to_s).rfc3339
+      rescue ArgumentError
+        nil
+      end
+
       def environment
         environment_name = payload[:gitlab_environment_name]
 
@@ -85,7 +92,7 @@ module Gitlab
       end
 
       def secondary_params
-        payload.except(:start_time)
+        payload.except(:start_time, :end_time)
       end
 
       def flatten_secondary_params

spec/lib/gitlab/alert_management/alert_params_spec.rb

@@ -34,6 +34,7 @@ RSpec.describe Gitlab::AlertManagement::AlertParams do
         hosts: ['gitlab.com'],
         payload: payload,
         started_at: started_at,
+        ended_at: nil,
         fingerprint: nil,
         environment: nil
       )

spec/lib/gitlab/alerting/notification_payload_parser_spec.rb

@@ -7,10 +7,12 @@ RSpec.describe Gitlab::Alerting::NotificationPayloadParser do
 
   describe '.call' do
     let(:starts_at) { Time.current.change(usec: 0) }
+    let(:ends_at) { Time.current.change(usec: 0) }
     let(:payload) do
       {
         'title' => 'alert title',
         'start_time' => starts_at.rfc3339,
+        'end_time' => ends_at.rfc3339,
         'description' => 'Description',
         'monitoring_tool' => 'Monitoring tool name',
         'service' => 'Service',
@@ -32,7 +34,8 @@ RSpec.describe Gitlab::Alerting::NotificationPayloadParser do
             'hosts' => ['gitlab.com'],
             'severity' => 'low'
           },
-          'startsAt' => starts_at.rfc3339
+          'startsAt' => starts_at.rfc3339,
+          'endsAt' => ends_at.rfc3339
         }
       )
     end
@@ -141,6 +144,7 @@ RSpec.describe Gitlab::Alerting::NotificationPayloadParser do
         {
           'title' => '',
           'start_time' => '',
+          'end_time' => '',
           'description' => '',
           'monitoring_tool' => '',
           'service' => '',

spec/services/projects/alerting/notify_service_spec.rb

@@ -15,10 +15,12 @@ RSpec.describe Projects::Alerting::NotifyService do
     let(:fingerprint) { 'testing' }
     let(:service) { described_class.new(project, nil, payload) }
     let(:environment) { create(:environment, project: project) }
+    let(:ended_at) { nil }
     let(:payload_raw) do
       {
         title: 'alert title',
         start_time: starts_at.rfc3339,
+        end_time: ended_at&.rfc3339,
         severity: 'low',
         monitoring_tool: 'GitLab RSpec',
         service: 'GitLab Test Suite',
@@ -38,9 +40,10 @@ RSpec.describe Projects::Alerting::NotifyService do
 
       context 'with valid token' do
         let(:token) { alerts_service.token }
-        let(:incident_management_setting) { double(send_email?: email_enabled, create_issue?: issue_enabled) }
+        let(:incident_management_setting) { double(send_email?: email_enabled, create_issue?: issue_enabled, auto_close_incident?: auto_close_enabled) }
         let(:email_enabled) { false }
         let(:issue_enabled) { false }
+        let(:auto_close_enabled) { false }
 
         before do
           allow(service)
@@ -93,6 +96,57 @@ RSpec.describe Projects::Alerting::NotifyService do
 
             it_behaves_like 'adds an alert management alert event'
 
+            context 'end time given' do
+              let(:ended_at) { Time.current.change(nsec: 0) }
+
+              it 'does not resolve the alert' do
+                expect { subject }.not_to change { alert.reload.status }
+              end
+
+              it 'does not set the ended at' do
+                subject
+
+                expect(alert.reload.ended_at).to be_nil
+              end
+
+              it_behaves_like 'does not an create alert management alert'
+
+              context 'auto_close_enabled setting enabled' do
+                let(:auto_close_enabled) { true }
+
+                it 'resolves the alert and sets the end time', :aggregate_failures do
+                  subject
+                  alert.reload
+
+                  expect(alert.resolved?).to eq(true)
+                  expect(alert.ended_at).to eql(ended_at)
+                end
+
+                context 'related issue exists' do
+                  let(:alert) { create(:alert_management_alert, :with_issue, project: project, fingerprint: fingerprint_sha) }
+                  let(:issue) { alert.issue }
+
+                  context 'state_tracking is enabled' do
+                    before do
+                      stub_feature_flags(track_resource_state_change_events: true)
+                    end
+
+                    it { expect { subject }.to change { issue.reload.state }.from('opened').to('closed') }
+                    it { expect { subject }.to change(ResourceStateEvent, :count).by(1) }
+                  end
+
+                  context 'state_tracking is disabled' do
+                    before do
+                      stub_feature_flags(track_resource_state_change_events: false)
+                    end
+
+                    it { expect { subject }.to change { issue.reload.state }.from('opened').to('closed') }
+                    it { expect { subject }.to change(Note, :count).by(1) }
+                  end
+                end
+              end
+            end
+
             context 'existing alert is resolved' do
               let!(:alert) { create(:alert_management_alert, :resolved, project: project, fingerprint: fingerprint_sha) }
 
@@ -114,6 +168,13 @@ RSpec.describe Projects::Alerting::NotifyService do
             end
           end
 
+          context 'end time given' do
+            let(:ended_at) { Time.current }
+
+            it_behaves_like 'creates an alert management alert'
+            it_behaves_like 'assigns the alert properties'
+          end
+
           context 'with a minimal payload' do
             let(:payload_raw) do
               {