ContainerExpirationPolicy recurring jobs

Add the cron job and workers to run the
container_expiration_policies based on the cadence
value.

These jobs will lead to CleanupContainerRepositoryWorker
being called for each valid policy.

app/models/ci/pipeline_schedule.rb

@@ -62,6 +62,7 @@ module Ci
                          end
     end
 
+    # To be extracted with technical debt issue https://gitlab.com/gitlab-org/gitlab/issues/191331
     def schedule_next_run!
       save! # with set_next_run_at
     rescue ActiveRecord::RecordInvalid

app/models/container_expiration_policy.rb

@@ -3,12 +3,20 @@
 class ContainerExpirationPolicy < ApplicationRecord
   belongs_to :project, inverse_of: :container_expiration_policy
 
+  delegate :container_repositories, to: :project
+
   validates :project, presence: true
   validates :enabled, inclusion: { in: [true, false] }
   validates :cadence, presence: true, inclusion: { in: ->(_) { self.cadence_options.stringify_keys } }
   validates :older_than, inclusion: { in: ->(_) { self.older_than_options.stringify_keys } }, allow_nil: true
   validates :keep_n, inclusion: { in: ->(_) { self.keep_n_options.keys } }, allow_nil: true
 
+  scope :enabled, -> { where(enabled: true) }
+  scope :runnable_schedules, -> { enabled.where("next_run_at < ?", Time.zone.now) }
+  scope :preloaded, -> { preload(:project) }
+
+  before_save :set_next_run_at
+
   def self.keep_n_options
     {
       1 => _('%{tags} tag per image name') % { tags: 1 },
@@ -38,4 +46,15 @@ class ContainerExpirationPolicy < ApplicationRecord
       '90d': _('%{days} days until tags are automatically removed') % { days: 90 }
     }
   end
+
+  def set_next_run_at
+    self.next_run_at = Time.zone.now + ChronicDuration.parse(cadence).seconds
+  end
+
+  # To be extracted with technical debt issue https://gitlab.com/gitlab-org/gitlab/issues/191331
+  def schedule_next_run!
+    save! # executes set_next_run_at
+  rescue ActiveRecord::RecordInvalid
+    update_column(:next_run_at, nil) # update without validation
+  end
 end

app/services/container_expiration_policy_service.rb

+# frozen_string_literal: true
+
+class ContainerExpirationPolicyService < BaseService
+  def execute(container_expiration_policy)
+    container_expiration_policy.schedule_next_run!
+
+    container_expiration_policy.container_repositories.find_each do |container_repository|
+      CleanupContainerRepositoryWorker.perform_async(
+        current_user.id,
+        container_repository.id,
+        container_expiration_policy.attributes.except("created_at", "updated_at")
+      )
+    end
+  end
+end

app/workers/all_queues.yml

@@ -10,6 +10,7 @@
 - chaos:chaos_sleep
 
 - cronjob:admin_email
+- cronjob:container_expiration_policy
 - cronjob:expire_build_artifacts
 - cronjob:gitlab_usage_ping
 - cronjob:import_export_project_cleanup

app/workers/container_expiration_policy_worker.rb

+# frozen_string_literal: true
+
+class ContainerExpirationPolicyWorker
+  include ApplicationWorker
+  include CronjobQueue
+
+  feature_category :container_registry
+
+  def perform
+    ContainerExpirationPolicy.runnable_schedules.preloaded.find_each do |container_expiration_policy|
+      ContainerExpirationPolicyService.new(
+        container_expiration_policy.project, container_expiration_policy.project.owner
+      ).execute(container_expiration_policy)
+    end
+  end
+end

changelogs/unreleased/15398-recurring-job.yml

+---
+title: Add a cron job and worker to run the Container Expiration Policies
+merge_request: 21593
+author:
+type: added

config/initializers/1_settings.rb

@@ -467,6 +467,9 @@ Settings.cron_jobs['schedule_migrate_external_diffs_worker']['job_class'] = 'Sch
 Settings.cron_jobs['namespaces_prune_aggregation_schedules_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['namespaces_prune_aggregation_schedules_worker']['cron'] ||= '5 1 * * *'
 Settings.cron_jobs['namespaces_prune_aggregation_schedules_worker']['job_class'] = 'Namespaces::PruneAggregationSchedulesWorker'
+Settings.cron_jobs['container_expiration_policy_worker'] ||= Settingslogic.new({})
+Settings.cron_jobs['container_expiration_policy_worker']['cron'] ||= '50 * * * *'
+Settings.cron_jobs['container_expiration_policy_worker']['job_class'] = 'ContainerExpirationPolicyWorker'
 
 Gitlab.ee do
   Settings.cron_jobs['adjourned_group_deletion_worker'] ||= Settingslogic.new({})

spec/factories/container_expiration_policies.rb

+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :container_expiration_policy, class: ContainerExpirationPolicy do
+    association :project, factory: [:project, :without_container_expiration_policy]
+    cadence { '1d' }
+    enabled { true }
+
+    trait :runnable do
+      after(:create) do |policy|
+        # next_run_at will be set before_save to Time.now + cadence, so this ensures the policy is active
+        policy.update_column(:next_run_at, Time.zone.now - 1.day)
+      end
+    end
+
+    trait :disabled do
+      enabled { false }
+    end
+  end
+end

spec/factories/projects.rb

@@ -137,6 +137,12 @@ FactoryBot.define do
       end
     end
 
+    trait :without_container_expiration_policy do
+      after(:build) do |project|
+        project.class.skip_callback(:create, :after, :create_container_expiration_policy, raise: false)
+      end
+    end
+
     # Build a custom repository by specifying a hash of `filename => content` in
     # the transient `files` attribute. Each file will be created in its own
     # commit, operating against the master branch. So, the following call:

spec/models/container_expiration_policy_spec.rb

@@ -38,4 +38,39 @@ RSpec.describe ContainerExpirationPolicy, type: :model do
       it { is_expected.not_to allow_value('foo').for(:keep_n) }
     end
   end
+
+  describe '.preloaded' do
+    subject { described_class.preloaded }
+
+    before do
+      # container_expiration_policies are created for every new project
+      create_list(:project, 3)
+    end
+
+    it 'preloads the associations' do
+      subject
+
+      query = ActiveRecord::QueryRecorder.new { subject.each(&:project) }
+
+      expect(query.count).to eq(2)
+    end
+  end
+
+  describe '.runnable_schedules' do
+    subject { described_class.runnable_schedules }
+
+    let!(:policy) { create(:container_expiration_policy, :runnable) }
+
+    it 'returns the runnable schedule' do
+      is_expected.to eq([policy])
+    end
+
+    context 'when there are no runnable schedules' do
+      let!(:policy) { }
+
+      it 'returns an empty array' do
+        is_expected.to be_empty
+      end
+    end
+  end
 end

spec/services/container_expiration_policy_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe ContainerExpirationPolicyService do
+  let_it_be(:user) { create(:user) }
+  let_it_be(:container_expiration_policy) { create(:container_expiration_policy, :runnable) }
+  let(:project) { container_expiration_policy.project }
+  let(:container_repository) { create(:container_repository, project: project) }
+
+  before do
+    project.add_maintainer(user)
+  end
+
+  describe '#execute' do
+    subject { described_class.new(project, user).execute(container_expiration_policy) }
+
+    it 'kicks off a cleanup worker for the container repository' do
+      expect(CleanupContainerRepositoryWorker).to receive(:perform_async)
+        .with(user.id, container_repository.id, anything)
+
+      subject
+    end
+
+    it 'sets next_run_at on the container_expiration_policy' do
+      subject
+
+      expect(container_expiration_policy.next_run_at).to be > Time.zone.now
+    end
+  end
+end

spec/workers/container_expiration_policy_worker_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe ContainerExpirationPolicyWorker do
+  include ExclusiveLeaseHelpers
+
+  subject { described_class.new.perform }
+
+  context 'With no container expiration policies' do
+    it 'Does not execute any policies' do
+      expect(ContainerExpirationPolicyService).not_to receive(:new)
+
+      subject
+    end
+  end
+
+  context 'With container expiration policies' do
+    context 'a valid policy' do
+      let!(:container_expiration_policy) { create(:container_expiration_policy, :runnable) }
+      let(:user) { container_expiration_policy.project.owner }
+
+      it 'runs the policy' do
+        service = instance_double(ContainerExpirationPolicyService, execute: true)
+
+        expect(ContainerExpirationPolicyService)
+          .to receive(:new).with(container_expiration_policy.project, user).and_return(service)
+
+        subject
+      end
+    end
+
+    context 'a disabled policy' do
+      let!(:container_expiration_policy) { create(:container_expiration_policy, :runnable, :disabled) }
+      let(:user) {container_expiration_policy.project.owner }
+
+      it 'does not run the policy' do
+        expect(ContainerExpirationPolicyService)
+          .not_to receive(:new).with(container_expiration_policy, user)
+
+        subject
+      end
+    end
+
+    context 'a policy that is not due for a run' do
+      let!(:container_expiration_policy) { create(:container_expiration_policy) }
+      let(:user) {container_expiration_policy.project.owner }
+
+      it 'does not run the policy' do
+        expect(ContainerExpirationPolicyService)
+          .not_to receive(:new).with(container_expiration_policy, user)
+
+        subject
+      end
+    end
+  end
+end