Delay build trace archive operation by two minutes

This commit adds a delay to build trace archive operation because we
want to validate efficacy of adding locking to resolve race condition
between this operation and flushing chunks to object storage.

app/workers/build_finished_worker.rb

@@ -9,6 +9,8 @@ class BuildFinishedWorker # rubocop:disable Scalability/IdempotentWorker
   worker_resource_boundary :cpu
   tags :requires_disk_io
 
+  ARCHIVE_TRACES_IN = 2.minutes.freeze
+
   # rubocop: disable CodeReuse/ActiveRecord
   def perform(build_id)
     Ci::Build.find_by(id: build_id).try do |build|
@@ -33,9 +35,22 @@ class BuildFinishedWorker # rubocop:disable Scalability/IdempotentWorker
 
     # We execute these async as these are independent operations.
     BuildHooksWorker.perform_async(build.id)
-    ArchiveTraceWorker.perform_async(build.id)
     ExpirePipelineCacheWorker.perform_async(build.pipeline_id) if build.pipeline.cacheable?
     ChatNotificationWorker.perform_async(build.id) if build.pipeline.chat?
+
+    ##
+    # We want to delay sending a build trace to object storage operation to
+    # validate that this fixes a race condition between this and flushing live
+    # trace chunks and chunks being removed after consolidation and putting
+    # them into object storage archive.
+    #
+    # TODO This is temporary fix we should improve later, after we validate
+    # that this is indeed the culprit.
+    #
+    # See https://gitlab.com/gitlab-org/gitlab/-/issues/267112 for more
+    # details.
+    #
+    ArchiveTraceWorker.perform_in(ARCHIVE_TRACES_IN, build.id)
   end
 end
 

spec/workers/build_finished_worker_spec.rb

@@ -20,10 +20,10 @@ RSpec.describe BuildFinishedWorker do
         expect_any_instance_of(BuildTraceSectionsWorker).to receive(:perform)
         expect_any_instance_of(BuildCoverageWorker).to receive(:perform)
         expect(BuildHooksWorker).to receive(:perform_async)
-        expect(ArchiveTraceWorker).to receive(:perform_async)
         expect(ExpirePipelineCacheWorker).to receive(:perform_async)
         expect(ChatNotificationWorker).not_to receive(:perform_async)
         expect(Ci::BuildReportResultWorker).not_to receive(:perform)
+        expect(ArchiveTraceWorker).to receive(:perform_in)
 
         subject
       end