Merge branch '346271-state-change-history-batchedjob-2' into 'master'

Resolve "Add a table to store batched jobs state changes"

See merge request gitlab-org/gitlab!75113

config/initializers/postgres_partitioning.rb

@@ -3,7 +3,8 @@
 Gitlab::Database::Partitioning.register_models([
   AuditEvent,
   WebHookLog,
-  LooseForeignKeys::DeletedRecord
+  LooseForeignKeys::DeletedRecord,
+  Gitlab::Database::BackgroundMigration::BatchedJobTransitionLog
 ])
 
 if Gitlab.ee?

db/migrate/20211123135255_create_batched_background_migration_job_transition_logs.rb

+# frozen_string_literal: true
+
+class CreateBatchedBackgroundMigrationJobTransitionLogs < Gitlab::Database::Migration[1.0]
+  include Gitlab::Database::PartitioningMigrationHelpers
+
+  def up
+    execute(<<~SQL)
+      CREATE TABLE batched_background_migration_job_transition_logs (
+        id bigserial NOT NULL,
+        batched_background_migration_job_id bigint NOT NULL,
+        created_at timestamp with time zone NOT NULL,
+        updated_at timestamp with time zone NOT NULL,
+        previous_status smallint NOT NULL,
+        next_status smallint NOT NULL,
+        exception_class text,
+        exception_message text,
+        CONSTRAINT check_50e580811a CHECK ((char_length(exception_message) <= 1000)),
+        CONSTRAINT check_76e202c37a CHECK ((char_length(exception_class) <= 100)),
+        PRIMARY KEY (id, created_at)
+      ) PARTITION BY RANGE (created_at);
+
+      CREATE INDEX i_batched_background_migration_job_transition_logs_on_job_id
+        ON batched_background_migration_job_transition_logs USING btree (batched_background_migration_job_id);
+
+      ALTER TABLE batched_background_migration_job_transition_logs ADD CONSTRAINT fk_rails_b7523a175b
+        FOREIGN KEY (batched_background_migration_job_id) REFERENCES batched_background_migration_jobs(id) ON DELETE CASCADE;
+    SQL
+
+    min_date = Date.today
+    max_date = Date.today + 6.months
+
+    create_daterange_partitions('batched_background_migration_job_transition_logs', 'created_at', min_date, max_date)
+  end
+
+  def down
+    drop_table :batched_background_migration_job_transition_logs
+  end
+end

db/schema_migrations/20211123135255

+cf6b9bb5711b6a097e399e79fdabe01a237581d99de7fed3c2b69c65ffd23a06
\ No newline at end of file

db/structure.sql

@@ -127,6 +127,20 @@ CREATE TABLE audit_events (
 )
 PARTITION BY RANGE (created_at);
 
+CREATE TABLE batched_background_migration_job_transition_logs (
+    id bigint NOT NULL,
+    batched_background_migration_job_id bigint NOT NULL,
+    created_at timestamp with time zone NOT NULL,
+    updated_at timestamp with time zone NOT NULL,
+    previous_status smallint NOT NULL,
+    next_status smallint NOT NULL,
+    exception_class text,
+    exception_message text,
+    CONSTRAINT check_50e580811a CHECK ((char_length(exception_message) <= 1000)),
+    CONSTRAINT check_76e202c37a CHECK ((char_length(exception_class) <= 100))
+)
+PARTITION BY RANGE (created_at);
+
 CREATE TABLE incident_management_pending_alert_escalations (
     id bigint NOT NULL,
     rule_id bigint NOT NULL,
@@ -10894,6 +10908,15 @@ CREATE TABLE banned_users (
     user_id bigint NOT NULL
 );
 
+CREATE SEQUENCE batched_background_migration_job_transition_logs_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE batched_background_migration_job_transition_logs_id_seq OWNED BY batched_background_migration_job_transition_logs.id;
+
 CREATE TABLE batched_background_migration_jobs (
     id bigint NOT NULL,
     created_at timestamp with time zone NOT NULL,
@@ -21417,6 +21440,8 @@ ALTER TABLE ONLY background_migration_jobs ALTER COLUMN id SET DEFAULT nextval('
 
 ALTER TABLE ONLY badges ALTER COLUMN id SET DEFAULT nextval('badges_id_seq'::regclass);
 
+ALTER TABLE ONLY batched_background_migration_job_transition_logs ALTER COLUMN id SET DEFAULT nextval('batched_background_migration_job_transition_logs_id_seq'::regclass);
+
 ALTER TABLE ONLY batched_background_migration_jobs ALTER COLUMN id SET DEFAULT nextval('batched_background_migration_jobs_id_seq'::regclass);
 
 ALTER TABLE ONLY batched_background_migrations ALTER COLUMN id SET DEFAULT nextval('batched_background_migrations_id_seq'::regclass);
@@ -22833,6 +22858,9 @@ ALTER TABLE ONLY badges
 ALTER TABLE ONLY banned_users
     ADD CONSTRAINT banned_users_pkey PRIMARY KEY (user_id);
 
+ALTER TABLE ONLY batched_background_migration_job_transition_logs
+    ADD CONSTRAINT batched_background_migration_job_transition_logs_pkey PRIMARY KEY (id, created_at);
+
 ALTER TABLE ONLY batched_background_migration_jobs
     ADD CONSTRAINT batched_background_migration_jobs_pkey PRIMARY KEY (id);
 
@@ -25075,6 +25103,8 @@ CREATE UNIQUE INDEX finding_link_url_idx ON vulnerability_finding_links USING bt
 
 CREATE INDEX finding_links_on_vulnerability_occurrence_id ON vulnerability_finding_links USING btree (vulnerability_occurrence_id);
 
+CREATE INDEX i_batched_background_migration_job_transition_logs_on_job_id ON ONLY batched_background_migration_job_transition_logs USING btree (batched_background_migration_job_id);
+
 CREATE UNIQUE INDEX i_ci_job_token_project_scope_links_on_source_and_target_project ON ci_job_token_project_scope_links USING btree (source_project_id, target_project_id);
 
 CREATE INDEX idx_analytics_devops_adoption_segments_on_namespace_id ON analytics_devops_adoption_segments USING btree (namespace_id);
@@ -31083,6 +31113,9 @@ ALTER TABLE ONLY packages_debian_project_component_files
 ALTER TABLE ONLY namespace_aggregation_schedules
     ADD CONSTRAINT fk_rails_b565c8d16c FOREIGN KEY (namespace_id) REFERENCES namespaces(id) ON DELETE CASCADE;
 
+ALTER TABLE batched_background_migration_job_transition_logs
+    ADD CONSTRAINT fk_rails_b7523a175b FOREIGN KEY (batched_background_migration_job_id) REFERENCES batched_background_migration_jobs(id) ON DELETE CASCADE;
+
 ALTER TABLE ONLY approval_project_rules_protected_branches
     ADD CONSTRAINT fk_rails_b7567b031b FOREIGN KEY (protected_branch_id) REFERENCES protected_branches(id) ON DELETE CASCADE;
 

lib/gitlab/database/background_migration/batched_job.rb

@@ -20,6 +20,7 @@ module Gitlab
         }
 
         belongs_to :batched_migration, foreign_key: :batched_background_migration_id
+        has_many :batched_job_transition_logs, foreign_key: :batched_background_migration_job_id
 
         scope :active, -> { where(status: [:pending, :running]) }
         scope :stuck, -> { active.where('updated_at <= ?', STUCK_JOBS_TIMEOUT.ago) }

lib/gitlab/database/background_migration/batched_job_transition_log.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    module BackgroundMigration
+      class BatchedJobTransitionLog < ApplicationRecord
+        include PartitionedTable
+
+        self.table_name = :batched_background_migration_job_transition_logs
+
+        partitioned_by :created_at, strategy: :monthly, retain_for: 6.months
+
+        belongs_to :batched_job, foreign_key: :batched_background_migration_job_id
+
+        validates :previous_status, :next_status, :batched_job, presence: true
+
+        validates :exception_class, length: { maximum: 100 }
+        validates :exception_message, length: { maximum: 1000 }
+
+        enum previous_status: BatchedJob.statuses, _prefix: true
+        enum next_status: BatchedJob.statuses, _prefix: true
+      end
+    end
+  end
+end

lib/gitlab/database/gitlab_schemas.yml

@@ -552,3 +552,4 @@ x509_commit_signatures: :gitlab_main
 x509_issuers: :gitlab_main
 zentao_tracker_data: :gitlab_main
 zoom_meetings: :gitlab_main
+batched_background_migration_job_transition_logs: :gitlab_main

spec/lib/gitlab/database/background_migration/batched_job_spec.rb

@@ -7,6 +7,7 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedJob, type: :model d
 
   describe 'associations' do
     it { is_expected.to belong_to(:batched_migration).with_foreign_key(:batched_background_migration_id) }
+    it { is_expected.to have_many(:batched_job_transition_logs).with_foreign_key(:batched_background_migration_job_id) }
   end
 
   describe 'scopes' do

spec/lib/gitlab/database/background_migration/batched_job_transition_log_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Database::BackgroundMigration::BatchedJobTransitionLog, type: :model do
+  describe 'associations' do
+    it { is_expected.to belong_to(:batched_job).with_foreign_key(:batched_background_migration_job_id) }
+  end
+
+  describe 'validations' do
+    it { is_expected.to validate_presence_of(:previous_status) }
+    it { is_expected.to validate_presence_of(:next_status) }
+    it { is_expected.to validate_presence_of(:batched_job) }
+    it { is_expected.to validate_length_of(:exception_class).is_at_most(100) }
+    it { is_expected.to validate_length_of(:exception_message).is_at_most(1000) }
+    it { is_expected.to define_enum_for(:previous_status).with_values(%i(pending running failed succeeded)).with_prefix }
+    it { is_expected.to define_enum_for(:next_status).with_values(%i(pending running failed succeeded)).with_prefix }
+  end
+end