Move escalation creation into background job

- Add new worker
- update queue files

config/sidekiq_queues.yml

@@ -184,6 +184,8 @@
   - 1
 - - incident_management_pending_escalations_alert_check
   - 1
+- - incident_management_pending_escalations_alert_create
+  - 1
 - - invalid_gpg_signature_update
   - 2
 - - irker

ee/app/services/concerns/ee/alert_management/alert_processing.rb

@@ -45,7 +45,7 @@ module EE
       end
 
       def create_pending_escalations
-        ::IncidentManagement::PendingEscalations::CreateService.new(alert).execute
+        ::IncidentManagement::PendingEscalations::AlertCreateWorker.perform_async(alert.id)
       end
     end
   end

ee/app/services/ee/alert_management/alerts/update_service.rb

@@ -25,7 +25,7 @@ module EE
         end
 
         def create_pending_escalations
-          ::IncidentManagement::PendingEscalations::CreateService.new(alert).execute
+          ::IncidentManagement::PendingEscalations::AlertCreateWorker.perform_async(alert.id)
         end
       end
     end

ee/app/services/incident_management/pending_escalations/create_service.rb

@@ -34,9 +34,6 @@ module IncidentManagement
         end
 
         process_escalations(escalation_ids)
-
-      rescue StandardError => e
-        Gitlab::ErrorTracking.track_exception(e, target_type: target.class.to_s, target_id: target.id)
       end
 
       def create_escalation(rule)

ee/app/services/incident_management/pending_escalations/process_service.rb

@@ -12,6 +12,7 @@ module IncidentManagement
 
       def execute
         return unless ::Gitlab::IncidentManagement.escalation_policies_available?(project)
+        return if too_early_to_process?
         return if target_already_resolved?
         return if target_status_exceeded_rule?
 
@@ -33,6 +34,10 @@ module IncidentManagement
         target.status >= escalation.status_before_type_cast
       end
 
+      def too_early_to_process?
+        Time.current < escalation.process_at
+      end
+
       def notify_recipients
         NotificationService
           .new

ee/app/workers/all_queues.yml

@@ -1072,6 +1072,15 @@
   :weight: 1
   :idempotent: true
   :tags: []
+- :name: incident_management_pending_escalations_alert_create
+  :worker_name: IncidentManagement::PendingEscalations::AlertCreateWorker
+  :feature_category: :incident_management
+  :has_external_dependencies:
+  :urgency: :high
+  :resource_boundary: :unknown
+  :weight: 1
+  :idempotent: true
+  :tags: []
 - :name: ldap_group_sync
   :worker_name: LdapGroupSyncWorker
   :feature_category: :authentication_and_authorization

ee/app/workers/incident_management/pending_escalations/alert_create_worker.rb

+# frozen_string_literal: true
+
+module IncidentManagement
+  module PendingEscalations
+    class AlertCreateWorker
+      include ApplicationWorker
+
+      urgency :high
+
+      idempotent!
+      feature_category :incident_management
+
+      def perform(alert_id)
+        alert = ::AlertManagement::Alert.find(alert_id)
+
+        ::IncidentManagement::PendingEscalations::CreateService.new(alert).execute
+      end
+    end
+  end
+end

ee/spec/services/ee/alert_management/alerts/update_service_spec.rb

@@ -33,10 +33,7 @@ RSpec.describe AlertManagement::Alerts::UpdateService do
 
         let(:new_status) { :triggered }
 
-        it 'creates an escalation' do
-          expect { execute }.to change { IncidentManagement::PendingEscalations::Alert.count }.by(1)
-          expect(IncidentManagement::PendingEscalations::Alert.last.alert).to eq(alert)
-        end
+        it_behaves_like 'creates an escalation'
       end
 
       context 'moving from an open status to closed status' do

ee/spec/services/incident_management/pending_escalations/process_service_spec.rb

@@ -14,7 +14,8 @@ RSpec.describe IncidentManagement::PendingEscalations::ProcessService do
   let(:alert_params) { { status: AlertManagement::Alert::STATUSES[:triggered] } }
 
   let(:target) { alert }
-  let(:escalation) { create(:incident_management_pending_alert_escalation, rule: escalation_rule, oncall_schedule: schedule_1, target: target, status: IncidentManagement::EscalationRule.statuses[:acknowledged]) }
+  let(:process_at) { 5.minutes.ago }
+  let(:escalation) { create(:incident_management_pending_alert_escalation, rule: escalation_rule, oncall_schedule: schedule_1, target: target, status: IncidentManagement::EscalationRule.statuses[:acknowledged], process_at: process_at) }
 
   let(:service) { described_class.new(escalation) }
 
@@ -70,5 +71,11 @@ RSpec.describe IncidentManagement::PendingEscalations::ProcessService do
 
       it_behaves_like 'it does not escalate'
     end
+
+    context 'escalation is not ready to be processed' do
+      let(:process_at) { 5.minutes.from_now }
+
+      it_behaves_like 'it does not escalate'
+    end
   end
 end

ee/spec/services/projects/alerting/notify_service_spec.rb

@@ -105,7 +105,7 @@ RSpec.describe Projects::Alerting::NotifyService do
         end
 
         it_behaves_like 'does not send on-call notification'
-        include_examples 'creates an escalation', 1
+        include_examples 'creates an escalation'
 
         context 'existing alert with same payload fingerprint' do
           let_it_be(:alert) { create(:alert_management_alert, fingerprint: gitlab_fingerprint, project: project) }

ee/spec/support/shared_examples/services/alert_management/alert_processing/escalation_policies_shared_examples.rb

 # frozen_string_literal: true
 
-RSpec.shared_examples 'creates an escalation' do |count|
-  let(:count) { count }
+RSpec.shared_examples 'creates an escalation' do
   specify do
-    expect(IncidentManagement::PendingEscalations::Alert).to receive(:create!)
-      .with(target: a_kind_of(AlertManagement::Alert), rule: a_kind_of(IncidentManagement::EscalationRule), schedule_id: a_kind_of(Integer), status: a_kind_of(String), process_at: a_kind_of(ActiveSupport::TimeWithZone))
-      .exactly(count).times
-      .and_call_original
-
-    expected_args = []
-    count.times { expected_args << [a_kind_of(Integer)]}
-
-    expect(IncidentManagement::PendingEscalations::AlertCheckWorker)
-      .to receive(:bulk_perform_async)
-      .with(expected_args)
+    expect(IncidentManagement::PendingEscalations::AlertCreateWorker)
+      .to receive(:perform_async)
+      .with(a_kind_of(Integer))
 
     subject
   end