Read archived build logs with database reads consistency

This ensures that we avoid race condition between viewing a build log
and archiving a build log.

config/feature_flags/development/gitlab_ci_archived_trace_consistent_reads.yml

+---
+name: gitlab_ci_archived_trace_consistent_reads
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/53716
+rollout_issue_url: 
+milestone: '13.9'
+type: development
+group: group::continuous integration
+default_enabled: false

ee/lib/ee/gitlab/ci/trace.rb

+# frozen_string_literal: true
+
+module EE
+  module Gitlab
+    module Ci
+      module Trace
+        extend ::Gitlab::Utils::Override
+
+        override :destroy_stream
+        def destroy_stream(build)
+          if consistent_archived_trace?(build)
+            ::Gitlab::Database::LoadBalancing::Sticking
+              .stick('ci/build/trace', build.id)
+          end
+
+          yield
+        end
+
+        override :read_trace_artifact
+        def read_trace_artifact(build)
+          if consistent_archived_trace?(build)
+            ::Gitlab::Database::LoadBalancing::Sticking
+              .unstick_or_continue_sticking('ci/build/trace', build.id)
+          end
+
+          yield
+        end
+
+        def consistent_archived_trace?(build)
+          ::Feature.enabled?(:gitlab_ci_archived_trace_consistent_reads, build.project, default_enabled: false)
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/trace.rb

@@ -182,7 +182,7 @@ module Gitlab
         if job.trace_chunks.any?
           Gitlab::Ci::Trace::ChunkedIO.new(job) do |stream|
             archive_stream!(stream)
-            stream.destroy!
+            destroy_stream(job) { stream.destroy! }
           end
         elsif current_path
           File.open(current_path) do |stream|
@@ -268,7 +268,21 @@ module Gitlab
       end
 
       def trace_artifact
-        job.job_artifacts_trace
+        read_trace_artifact(job) { job.job_artifacts_trace }
+      end
+
+      ##
+      # Overridden in EE
+      #
+      def destroy_stream(job)
+        yield
+      end
+
+      ##
+      # Overriden in EE
+      #
+      def read_trace_artifact(job)
+        yield
       end
 
       def being_watched_cache_key
@@ -277,3 +291,5 @@ module Gitlab
     end
   end
 end
+
+::Gitlab::Ci::Trace.prepend_if_ee('EE::Gitlab::Ci::Trace')