Commit 72cbaea7 authored by Robert Speicher's avatar Robert Speicher

Merge branch 'eb-clean-up-unit-test-tables' into 'master'

Add cron worker for cleaning up unit test tables

See merge request gitlab-org/gitlab!61463
parents 6467ddd2 ed3e3ff1
...@@ -14,6 +14,7 @@ module Ci ...@@ -14,6 +14,7 @@ module Ci
belongs_to :project belongs_to :project
scope :by_project_and_keys, -> (project, keys) { where(project_id: project.id, key_hash: keys) } scope :by_project_and_keys, -> (project, keys) { where(project_id: project.id, key_hash: keys) }
scope :deletable, -> { where('NOT EXISTS (?)', Ci::UnitTestFailure.select(1).where("#{Ci::UnitTestFailure.table_name}.unit_test_id = #{table_name}.id")) }
class << self class << self
def find_or_create_by_batch(project, unit_test_attrs) def find_or_create_by_batch(project, unit_test_attrs)
......
...@@ -11,6 +11,8 @@ module Ci ...@@ -11,6 +11,8 @@ module Ci
belongs_to :unit_test, class_name: "Ci::UnitTest", foreign_key: :unit_test_id belongs_to :unit_test, class_name: "Ci::UnitTest", foreign_key: :unit_test_id
belongs_to :build, class_name: "Ci::Build", foreign_key: :build_id belongs_to :build, class_name: "Ci::Build", foreign_key: :build_id
scope :deletable, -> { where('failed_at < ?', REPORT_WINDOW.ago) }
def self.recent_failures_count(project:, unit_test_keys:, date_range: REPORT_WINDOW.ago..Time.current) def self.recent_failures_count(project:, unit_test_keys:, date_range: REPORT_WINDOW.ago..Time.current)
joins(:unit_test) joins(:unit_test)
.where( .where(
......
# frozen_string_literal: true
module Ci
class DeleteUnitTestsService
include EachBatch
BATCH_SIZE = 100
def execute
purge_data!(Ci::UnitTestFailure)
purge_data!(Ci::UnitTest)
end
private
def purge_data!(klass)
loop do
break unless delete_batch!(klass)
end
end
# rubocop: disable CodeReuse/ActiveRecord
def delete_batch!(klass)
deleted = 0
ActiveRecord::Base.transaction do
ids = klass.deletable.lock('FOR UPDATE SKIP LOCKED').limit(BATCH_SIZE).pluck(:id)
break if ids.empty?
deleted = klass.where(id: ids).delete_all
end
deleted > 0
end
# rubocop: enable CodeReuse/ActiveRecord
end
end
...@@ -182,6 +182,15 @@ ...@@ -182,6 +182,15 @@
:weight: 1 :weight: 1
:idempotent: :idempotent:
:tags: [] :tags: []
- :name: cronjob:ci_delete_unit_tests
:worker_name: Ci::DeleteUnitTestsWorker
:feature_category: :continuous_integration
:has_external_dependencies:
:urgency: :low
:resource_boundary: :unknown
:weight: 1
:idempotent: true
:tags: []
- :name: cronjob:ci_pipeline_artifacts_expire_artifacts - :name: cronjob:ci_pipeline_artifacts_expire_artifacts
:worker_name: Ci::PipelineArtifacts::ExpireArtifactsWorker :worker_name: Ci::PipelineArtifacts::ExpireArtifactsWorker
:feature_category: :continuous_integration :feature_category: :continuous_integration
......
# frozen_string_literal: true
module Ci
class DeleteUnitTestsWorker
include ApplicationWorker
# rubocop:disable Scalability/CronWorkerContext
# This worker does not perform work scoped to a context
include CronjobQueue
# rubocop:enable Scalability/CronWorkerContext
feature_category :continuous_integration
idempotent!
def perform
Ci::DeleteUnitTestsService.new.execute
end
end
end
---
title: Add cron worker for cleaning up unit test tables
merge_request: 61463
author:
type: added
...@@ -572,6 +572,9 @@ Settings.cron_jobs['ssh_keys_expiring_soon_notification_worker']['job_class'] = ...@@ -572,6 +572,9 @@ Settings.cron_jobs['ssh_keys_expiring_soon_notification_worker']['job_class'] =
Settings.cron_jobs['users_deactivate_dormant_users_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['users_deactivate_dormant_users_worker'] ||= Settingslogic.new({})
Settings.cron_jobs['users_deactivate_dormant_users_worker']['cron'] ||= '21,42 0-4 * * *' Settings.cron_jobs['users_deactivate_dormant_users_worker']['cron'] ||= '21,42 0-4 * * *'
Settings.cron_jobs['users_deactivate_dormant_users_worker']['job_class'] = 'Users::DeactivateDormantUsersWorker' Settings.cron_jobs['users_deactivate_dormant_users_worker']['job_class'] = 'Users::DeactivateDormantUsersWorker'
Settings.cron_jobs['ci_delete_unit_tests_worker'] ||= Settingslogic.new({})
Settings.cron_jobs['ci_delete_unit_tests_worker']['cron'] ||= '0 0 * * *'
Settings.cron_jobs['ci_delete_unit_tests_worker']['job_class'] = 'Ci::DeleteUnitTestsWorker'
Gitlab.com do Gitlab.com do
Settings.cron_jobs['batched_background_migrations_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['batched_background_migrations_worker'] ||= Settingslogic.new({})
......
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Ci::DeleteUnitTestsService do
describe '#execute' do
let!(:unit_test_1) { create(:ci_unit_test) }
let!(:unit_test_2) { create(:ci_unit_test) }
let!(:unit_test_3) { create(:ci_unit_test) }
let!(:unit_test_4) { create(:ci_unit_test) }
let!(:unit_test_1_recent_failure) { create(:ci_unit_test_failure, unit_test: unit_test_1) }
let!(:unit_test_1_old_failure) { create(:ci_unit_test_failure, unit_test: unit_test_1, failed_at: 15.days.ago) }
let!(:unit_test_2_old_failure) { create(:ci_unit_test_failure, unit_test: unit_test_2, failed_at: 15.days.ago) }
let!(:unit_test_3_old_failure) { create(:ci_unit_test_failure, unit_test: unit_test_3, failed_at: 15.days.ago) }
let!(:unit_test_4_old_failure) { create(:ci_unit_test_failure, unit_test: unit_test_4, failed_at: 15.days.ago) }
before do
stub_const("#{described_class.name}::BATCH_SIZE", 2)
described_class.new.execute
end
it 'does not delete unit test failures not older than 14 days' do
expect(unit_test_1_recent_failure.reload).to be_persisted
end
it 'deletes unit test failures older than 14 days' do
ids = [
unit_test_1_old_failure,
unit_test_2_old_failure,
unit_test_3_old_failure,
unit_test_4_old_failure
].map(&:id)
result = Ci::UnitTestFailure.where(id: ids)
expect(result).to be_empty
end
it 'deletes unit tests that have no more associated unit test failures' do
ids = [
unit_test_2,
unit_test_3,
unit_test_4
].map(&:id)
result = Ci::UnitTest.where(id: ids)
expect(result).to be_empty
end
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Ci::DeleteUnitTestsWorker do
let(:worker) { described_class.new }
describe '#perform' do
it 'executes a service' do
expect_next_instance_of(Ci::DeleteUnitTestsService) do |instance|
expect(instance).to receive(:execute)
end
worker.perform
end
end
it_behaves_like 'an idempotent worker' do
let!(:unit_test_1) { create(:ci_unit_test) }
let!(:unit_test_2) { create(:ci_unit_test) }
let!(:unit_test_1_recent_failure) { create(:ci_unit_test_failure, unit_test: unit_test_1) }
let!(:unit_test_2_old_failure) { create(:ci_unit_test_failure, unit_test: unit_test_2, failed_at: 15.days.ago) }
it 'only deletes old unit tests and their failures' do
subject
expect(unit_test_1.reload).to be_persisted
expect(unit_test_1_recent_failure.reload).to be_persisted
expect(Ci::UnitTest.find_by(id: unit_test_2.id)).to be_nil
expect(Ci::UnitTestFailure.find_by(id: unit_test_2_old_failure.id)).to be_nil
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment