Limit the amount of time ChatNotificationWorker waits for build trace

See https://gitlab.com/gitlab-org/gitlab/issues/119198

app/workers/chat_notification_worker.rb

@@ -11,18 +11,21 @@ class ChatNotificationWorker
   # worker_has_external_dependencies!
 
   RESCHEDULE_INTERVAL = 2.seconds
+  RESCHEDULE_TIMEOUT = 5.minutes
 
   # rubocop: disable CodeReuse/ActiveRecord
-  def perform(build_id)
+  def perform(build_id, reschedule_count = 0)
     Ci::Build.find_by(id: build_id).try do |build|
       send_response(build)
     end
   rescue Gitlab::Chat::Output::MissingBuildSectionError
+    return if timeout_exceeded?(reschedule_count)
+
     # The creation of traces and sections appears to be eventually consistent.
     # As a result it's possible for us to run the above code before the trace
     # sections are present. To better handle such cases we'll just reschedule
     # the job instead of producing an error.
-    self.class.perform_in(RESCHEDULE_INTERVAL, build_id)
+    self.class.perform_in(RESCHEDULE_INTERVAL, build_id, reschedule_count + 1)
   end
   # rubocop: enable CodeReuse/ActiveRecord
 
@@ -37,4 +40,10 @@ class ChatNotificationWorker
       end
     end
   end
+
+  private
+
+  def timeout_exceeded?(reschedule_count)
+    (reschedule_count * RESCHEDULE_INTERVAL) >= RESCHEDULE_TIMEOUT
+  end
 end

changelogs/unreleased/119198-chat_notification-sidekiq-job-latency-has-increased.yml

+---
+title: Limit the amount of time ChatNotificationWorker waits for the build trace
+merge_request: 22132
+author:
+type: fixed

spec/workers/chat_notification_worker_spec.rb

@@ -23,17 +23,32 @@ describe ChatNotificationWorker do
       worker.perform(chat_build.id)
     end
 
-    it 'reschedules the job if the trace sections could not be found' do
+    context 'when the trace sections could not be found' do
+      it 'reschedules the job' do
         expect(worker)
           .to receive(:send_response)
           .and_raise(Gitlab::Chat::Output::MissingBuildSectionError)
 
         expect(described_class)
           .to receive(:perform_in)
-        .with(described_class::RESCHEDULE_INTERVAL, chat_build.id)
+          .with(described_class::RESCHEDULE_INTERVAL, chat_build.id, 1)
 
         worker.perform(chat_build.id)
       end
+
+      it "stops rescheduling the job after #{described_class::RESCHEDULE_TIMEOUT} seconds" do
+        allow(described_class).to receive(:new).and_return(worker)
+        allow(worker).to receive(:send_response).and_raise(Gitlab::Chat::Output::MissingBuildSectionError)
+
+        worker.perform(chat_build.id)
+
+        described_class.drain
+
+        max_reschedules = described_class::RESCHEDULE_TIMEOUT / described_class::RESCHEDULE_INTERVAL
+
+        expect(worker).to have_received(:send_response).exactly(max_reschedules + 1).times
+      end
+    end
   end
 
   describe '#send_response' do