trace as artifacts

app/models/ci/build.rb

@@ -24,6 +24,7 @@ module Ci
     has_many :job_artifacts, class_name: 'Ci::JobArtifact', foreign_key: :job_id, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
     has_one :job_artifacts_archive, -> { where(file_type: Ci::JobArtifact.file_types[:archive]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id
     has_one :job_artifacts_metadata, -> { where(file_type: Ci::JobArtifact.file_types[:metadata]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id
+    has_one :job_artifacts_trace, -> { where(file_type: Ci::JobArtifact.file_types[:trace]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id
 
     # The "environment" field for builds is a String, and is the unexpanded name
     def persisted_environment

app/models/ci/job_artifact.rb

@@ -17,9 +17,12 @@ module Ci
       end
     end
 
+    delegate :open, :exists?, to: :file
+
     enum file_type: {
       archive: 1,
-      metadata: 2
+      metadata: 2,
+      trace: 3
     }
 
     def self.artifacts_size_for(project)

app/models/concerns/artifact_migratable.rb

@@ -39,7 +39,6 @@ module ArtifactMigratable
   end
 
   def artifacts_size
-    read_attribute(:artifacts_size).to_i +
-      job_artifacts_archive&.size.to_i + job_artifacts_metadata&.size.to_i
+    read_attribute(:artifacts_size).to_i + job_artifacts.sum(:size).to_i
   end
 end

app/uploaders/job_artifact_uploader.rb

@@ -14,6 +14,14 @@ class JobArtifactUploader < GitlabUploader
     dynamic_segment
   end
 
+  def open
+    if file_storage?
+      File.open(path, "rb")
+    else
+      raise 'Only File System is supported'
+    end
+  end
+
   private
 
   def dynamic_segment

lib/api/runner.rb

@@ -120,7 +120,16 @@ module API
       put '/:id' do
         job = authenticate_job!
 
-        job.trace.set(params[:trace]) if params[:trace]
+        if params[:trace]
+          # Overwrite live-trace by full-trace
+          job.trace.set(params[:trace])
+
+          # Move full-trace to JobArtifactUploader#default_path
+          job.build_job_artifacts_trace(
+            project: job.project,
+            file_type: :trace,
+            file: UploadedFile.new(job.trace.current_path, 'trace.log'))
+        end
 
         Gitlab::Metrics.add_event(:update_build,
                                   project: job.project.full_path)

lib/gitlab/ci/trace.rb

+##
+# Current status of paths
+# Era 1: Live/Full traces in database (ci_builds.trace)
+# Era 2: Live/Full traces in `setting_root/YYYY_MM/project_ci_id/job_id.log`
+# Era 3: Live/Full traces in `setting_root/YYYY_MM/project_id/job_id.log`
+# Era 4: Live traces in `setting_root/live_trace/job_id.log`. Full traces in JobArtifactUploader#legacy_default_path.
+#
+# The legacy paths are to be migrated to the latest era.
 module Gitlab
   module Ci
     class Trace
@@ -52,12 +60,14 @@ module Gitlab
       end
 
       def exist?
-        current_path.present? || old_trace.present?
+        trace_artifact&.exists? || current_path.present? || old_trace.present?
       end
 
       def read
         stream = Gitlab::Ci::Trace::Stream.new do
-          if current_path
+          if trace_artifact&.exists?
+            trace_artifact.open
+          elsif current_path
             File.open(current_path, "rb")
           elsif old_trace
             StringIO.new(old_trace)
@@ -82,6 +92,8 @@ module Gitlab
       end
 
       def erase!
+        trace_artifact&.destory
+
         paths.each do |trace_path|
           FileUtils.rm(trace_path, force: true)
         end
@@ -89,44 +101,56 @@ module Gitlab
         job.erase_old_trace!
       end
 
+      def current_path
+        @current_path ||= paths.find do |trace_path|
+          File.exist?(trace_path)
+        end
+      end
+
       private
 
       def ensure_path
         return current_path if current_path
 
         ensure_directory
-        default_path
+        live_trace_default_path
       end
 
       def ensure_directory
-        unless Dir.exist?(default_directory)
-          FileUtils.mkdir_p(default_directory)
-        end
-      end
-
-      def current_path
-        @current_path ||= paths.find do |trace_path|
-          File.exist?(trace_path)
+        unless Dir.exist?(live_trace_default_directory)
+          FileUtils.mkdir_p(live_trace_default_directory)
         end
       end
 
+      ##
+      # This method doesn't include the latest path, which is JobArtifactUploader#default_path,
+      # Because, in EE, traces can be moved to ObjectStorage, so checking paths in Filestorage doesn't make sense.
+      # All legacy paths (`legacy_default_path` and `deprecated_path`) are to be migrated to JobArtifactUploader#default_path
       def paths
         [
-          default_path,
+          live_trace_default_path,
+          legacy_default_path,
           deprecated_path
         ].compact
       end
 
-      def default_directory
+      def live_trace_default_directory
         File.join(
           Settings.gitlab_ci.builds_path,
-          job.created_at.utc.strftime("%Y_%m"),
-          job.project_id.to_s
+          'live_trace'
         )
       end
 
-      def default_path
-        File.join(default_directory, "#{job.id}.log")
+      def live_trace_default_path
+        File.join(live_trace_default_directory, "#{job.id}.log")
+      end
+
+      def legacy_default_path
+        File.join(
+          Settings.gitlab_ci.builds_path,
+          job.created_at.utc.strftime("%Y_%m"),
+          job.project_id.to_s,
+          "#{job.id}.log")
       end
 
       def deprecated_path
@@ -137,6 +161,10 @@ module Gitlab
           "#{job.id}.log"
         ) if job.project&.ci_id
       end
+
+      def trace_artifact
+        job.job_artifacts_trace
+      end
     end
   end
 end