Added flush-to-db process for the cron worker. Rename to RescueStaleLiveTraceWorker.

app/workers/all_queues.yml

@@ -17,7 +17,7 @@
 - cronjob:stuck_ci_jobs
 - cronjob:stuck_import_jobs
 - cronjob:stuck_merge_jobs
-- cronjob:fource_archive_stale_live_trace
+- cronjob:rescue_stale_live_trace
 - cronjob:trending_projects
 - cronjob:issue_due_scheduler
 

app/workers/fource_archive_stale_live_trace_worker.rb

-class FourceArchiveStaleLiveTraceWorker
-  include ApplicationWorker
-  include CronjobQueue
-
-  def perform
-    # Find jobs with the following condition
-    # - Finished 4 hours ago (Jobs finished 4 hours ago should have an archived trace)
-    # - Not archived yet (Because ArchiveTraceWorker failed by some reason)
-    Ci::Build.finished
-      .where('finished_at < ?', 4.hours.ago)
-      .where('NOT EXISTS (?)', Ci::JobArtifact.select(1).trace.where('ci_builds.id = ci_job_artifacts.job_id'))
-      .find_each(batch_size: 1000) do |job|
-        begin
-          job.trace.archive!
-        rescue => e
-          Rails.logger.error "#{job.id}: Failed to archive stable live trace: #{e.message}"
-        end
-
-        Rails.logger.warning "#{job.id}: Live trace was force archived because it was considered as stale"
-      end
-    end
-  end
-end

app/workers/rescue_stale_live_trace_worker.rb

+class RescueStaleLiveTraceWorker
+  include ApplicationWorker
+  include CronjobQueue
+
+  def perform
+    # Reschedule to archive live traces
+    #
+    # The target jobs are with the following conditions
+    # - Finished 4 hours ago, but it's not archived yet
+    #   Jobs finished 4 hours ago should have an archived trace. Probably ArchiveTraceWorker failed by Sidekiq's inconsistancy
+    Ci::Build.finished
+      .where('finished_at BETWEEN ? AND ?', 1.week.ago, 4.hours.ago)
+      .where('NOT EXISTS (?)', Ci::JobArtifact.select(1).trace.where('ci_builds.id = ci_job_artifacts.job_id'))
+      .find_in_batch(batch_size: 1000) do |jobs|
+        job_ids = jobs.map { |job| [job.id] }
+
+        ArchiveTraceWorker.bulk_perform_async(job_ids)
+
+        Rails.logger.warning "Scheduled to archive stale live traces from #{job_ids.min} to #{job_ids.max}"
+      end
+
+    # Schedule to flush redis-chunk to database
+    #
+    # The target build_trace_chunks are with the following conditions
+    # - The last patching of the trace was 1 hour ago
+    # - The job is still running
+    Ci::BuildTraceChunk.redis
+      .joins(:build)
+      .where('ci_builds.update_at < ?', 1.hour.ago)
+      .where('ci_builds.status = ?', 'running')
+      .find_in_batch(batch_size: 1000) do |build_trace_chunks|
+        build_trace_chunk_ids = build_trace_chunks.map { |build_trace_chunk| [build_trace_chunk.id] }
+
+        BuildTraceChunkFlushToDBWorker.bulk_perform_async(build_trace_chunk_ids)
+
+        Rails.logger.warning "Scheduled to flush stale live traces to database from #{build_trace_chunk_ids.min} to #{build_trace_chunk_ids.max}"
+      end
+  end
+end

config/initializers/1_settings.rb

@@ -289,9 +289,9 @@ Settings.cron_jobs['repository_archive_cache_worker']['job_class'] = 'Repository
 Settings.cron_jobs['import_export_project_cleanup_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['import_export_project_cleanup_worker']['cron'] ||= '0 * * * *'
 Settings.cron_jobs['import_export_project_cleanup_worker']['job_class'] = 'ImportExportProjectCleanupWorker'
-Settings.cron_jobs['fource_archive_stale_live_trace_worker'] ||= Settingslogic.new({})
-Settings.cron_jobs['fource_archive_stale_live_trace_worker']['cron'] ||= '*/4 * * * *'
-Settings.cron_jobs['fource_archive_stale_live_trace_worker']['job_class'] = 'BuildTraceChunkArchiveStaleObjectsWorker'
+Settings.cron_jobs['rescue_stale_live_trace_worker'] ||= Settingslogic.new({})
+Settings.cron_jobs['rescue_stale_live_trace_worker']['cron'] ||= '*/1 * * * *'
+Settings.cron_jobs['rescue_stale_live_trace_worker']['job_class'] = 'BuildTraceChunkArchiveStaleObjectsWorker'
 Settings.cron_jobs['requests_profiles_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['requests_profiles_worker']['cron'] ||= '0 0 * * *'
 Settings.cron_jobs['requests_profiles_worker']['job_class'] = 'RequestsProfilesWorker'