Merge branch 'registry-phase-2-rails-db-migrations' into 'master'

Database prep for phase 2 of container registry import

See merge request gitlab-org/gitlab!77725

app/helpers/application_settings_helper.rb

@@ -404,6 +404,12 @@ module ApplicationSettingsHelper
       :rate_limiting_response_text,
       :container_registry_expiration_policies_worker_capacity,
       :container_registry_cleanup_tags_service_max_list_size,
+      :container_registry_import_max_tags_count,
+      :container_registry_import_max_retries,
+      :container_registry_import_start_max_retries,
+      :container_registry_import_max_step_duration,
+      :container_registry_import_target_plan,
+      :container_registry_import_created_before,
       :keep_latest_artifact,
       :whats_new_variant,
       :user_deactivation_emails_enabled,

app/models/application_setting.rb

@@ -357,13 +357,19 @@ class ApplicationSetting < ApplicationRecord
   validates :hashed_storage_enabled, inclusion: { in: [true], message: _("Hashed storage can't be disabled anymore for new projects") }
 
   validates :container_registry_delete_tags_service_timeout,
+            :container_registry_cleanup_tags_service_max_list_size,
+            :container_registry_expiration_policies_worker_capacity,
             numericality: { only_integer: true, greater_than_or_equal_to: 0 }
 
-  validates :container_registry_cleanup_tags_service_max_list_size,
+  validates :container_registry_import_max_tags_count,
+            :container_registry_import_max_retries,
+            :container_registry_import_start_max_retries,
+            :container_registry_import_max_step_duration,
+            allow_nil: false,
             numericality: { only_integer: true, greater_than_or_equal_to: 0 }
 
-  validates :container_registry_expiration_policies_worker_capacity,
-            numericality: { only_integer: true, greater_than_or_equal_to: 0 }
+  validates :container_registry_import_target_plan, presence: true
+  validates :container_registry_import_created_before, presence: true
 
   validates :dependency_proxy_ttl_group_policy_worker_capacity,
             allow_nil: false,

app/models/application_setting_implementation.rb

@@ -217,6 +217,12 @@ module ApplicationSettingImplementation
         wiki_page_max_content_bytes: 50.megabytes,
         container_registry_delete_tags_service_timeout: 250,
         container_registry_expiration_policies_worker_capacity: 0,
+        container_registry_import_max_tags_count: 100,
+        container_registry_import_max_retries: 3,
+        container_registry_import_start_max_retries: 50,
+        container_registry_import_max_step_duration: 5.minutes,
+        container_registry_import_target_plan: 'free',
+        container_registry_import_created_before: '2022-01-23 00:00:00',
         kroki_enabled: false,
         kroki_url: nil,
         kroki_formats: { blockdiag: false, bpmn: false, excalidraw: false },

app/models/container_repository.rb

@@ -13,9 +13,15 @@ class ContainerRepository < ApplicationRecord
 
   validates :name, length: { minimum: 0, allow_nil: false }
   validates :name, uniqueness: { scope: :project_id }
+  validates :migration_state, presence: true
+
+  validates :migration_retries_count, presence: true,
+                                      numericality: { greater_than_or_equal_to: 0 },
+                                      allow_nil: false
 
   enum status: { delete_scheduled: 0, delete_failed: 1 }
   enum expiration_policy_cleanup_status: { cleanup_unscheduled: 0, cleanup_scheduled: 1, cleanup_unfinished: 2, cleanup_ongoing: 3 }
+  enum migration_skipped_reason: { not_in_plan: 0, too_many_retries: 1, too_many_tags: 2, root_namespace_in_deny_list: 3 }
 
   delegate :client, to: :registry
 

db/migrate/20220106230629_add_registry_migration_application_settings.rb

+# frozen_string_literal: true
+
+class AddRegistryMigrationApplicationSettings < Gitlab::Database::Migration[1.0]
+  # rubocop:disable Migration/AddLimitToTextColumns
+  # limit is added in 20220118141950_add_text_limit_to_container_registry_import_target_plan.rb
+  def change
+    add_column :application_settings, :container_registry_import_max_tags_count, :integer, default: 100, null: false
+    add_column :application_settings, :container_registry_import_max_retries, :integer, default: 3, null: false
+    add_column :application_settings, :container_registry_import_start_max_retries, :integer, default: 50, null: false
+    add_column :application_settings, :container_registry_import_max_step_duration, :integer, default: 5.minutes, null: false
+    add_column :application_settings, :container_registry_import_target_plan, :text, default: 'free', null: false
+    add_column :application_settings, :container_registry_import_created_before, :datetime_with_timezone, default: '2022-01-23 00:00:00', null: false
+  end
+  # rubocop:enable Migration/AddLimitToTextColumns
+end

db/migrate/20220106230712_add_migration_columns_to_container_repositories.rb

+# frozen_string_literal: true
+
+class AddMigrationColumnsToContainerRepositories < Gitlab::Database::Migration[1.0]
+  # rubocop:disable Migration/AddLimitToTextColumns
+  # limit is added in 20220117225936_add_text_limits_to_container_repositories_migration_columns.rb
+  def change
+    add_column :container_repositories, :migration_pre_import_started_at, :datetime_with_timezone
+    add_column :container_repositories, :migration_pre_import_done_at, :datetime_with_timezone
+    add_column :container_repositories, :migration_import_started_at, :datetime_with_timezone
+    add_column :container_repositories, :migration_import_done_at, :datetime_with_timezone
+    add_column :container_repositories, :migration_aborted_at, :datetime_with_timezone
+    add_column :container_repositories, :migration_skipped_at, :datetime_with_timezone
+    add_column :container_repositories, :migration_retries_count, :integer, default: 0, null: false
+    add_column :container_repositories, :migration_skipped_reason, :smallint
+    add_column :container_repositories, :migration_state, :text, default: 'default', null: false
+    add_column :container_repositories, :migration_aborted_in_state, :text
+  end
+  # rubocop:enable Migration/AddLimitToTextColumns
+end

db/migrate/20220117225936_add_text_limits_to_container_repositories_migration_columns.rb

+# frozen_string_literal: true
+
+class AddTextLimitsToContainerRepositoriesMigrationColumns < Gitlab::Database::Migration[1.0]
+  disable_ddl_transaction!
+
+  def up
+    add_text_limit :container_repositories, :migration_state, 255
+    add_text_limit :container_repositories, :migration_aborted_in_state, 255
+  end
+
+  def down
+    remove_text_limit :container_repositories, :migration_state
+    remove_text_limit :container_repositories, :migration_aborted_in_state
+  end
+end

db/migrate/20220118141950_add_text_limit_to_container_registry_import_target_plan.rb

+# frozen_string_literal: true
+
+class AddTextLimitToContainerRegistryImportTargetPlan < Gitlab::Database::Migration[1.0]
+  disable_ddl_transaction!
+
+  def up
+    add_text_limit :application_settings, :container_registry_import_target_plan, 255
+  end
+
+  def down
+    remove_text_limit :application_settings, :container_registry_import_target_plan
+  end
+end

db/schema_migrations/20220106230629

+675d8f7bf77ddb860e707c25811d4eaaac1173c9fe62ce96c2708f0bbd0f4d48
\ No newline at end of file

db/schema_migrations/20220106230712

+672b51ca014d208f971efe698edb8a8b32f541bf9d21a7f555c53f749ee936a4
\ No newline at end of file

db/schema_migrations/20220117225936

+484eaf2ce1df1e2915b7ea8a5c9f4e044957c25d1ccf5841f24c75791d1a1a13
\ No newline at end of file

db/schema_migrations/20220118141950

+a4131f86bc415f0c1897e3b975494806ffc5a834dca2b39c998c6a406e695e15
\ No newline at end of file

db/structure.sql

@@ -10483,6 +10483,12 @@ CREATE TABLE application_settings (
     future_subscriptions jsonb DEFAULT '[]'::jsonb NOT NULL,
     user_email_lookup_limit integer DEFAULT 60 NOT NULL,
     packages_cleanup_package_file_worker_capacity smallint DEFAULT 2 NOT NULL,
+    container_registry_import_max_tags_count integer DEFAULT 100 NOT NULL,
+    container_registry_import_max_retries integer DEFAULT 3 NOT NULL,
+    container_registry_import_start_max_retries integer DEFAULT 50 NOT NULL,
+    container_registry_import_max_step_duration integer DEFAULT 300 NOT NULL,
+    container_registry_import_target_plan text DEFAULT 'free'::text NOT NULL,
+    container_registry_import_created_before timestamp with time zone DEFAULT '2022-01-23 00:00:00+00'::timestamp with time zone NOT NULL,
     runner_token_expiration_interval integer,
     group_runner_token_expiration_interval integer,
     project_runner_token_expiration_interval integer,
@@ -10496,6 +10502,7 @@ CREATE TABLE application_settings (
     CONSTRAINT check_17d9558205 CHECK ((char_length((kroki_url)::text) <= 1024)),
     CONSTRAINT check_2dba05b802 CHECK ((char_length(gitpod_url) <= 255)),
     CONSTRAINT check_32710817e9 CHECK ((char_length(static_objects_external_storage_auth_token_encrypted) <= 255)),
+    CONSTRAINT check_3559645ae5 CHECK ((char_length(container_registry_import_target_plan) <= 255)),
     CONSTRAINT check_3def0f1829 CHECK ((char_length(sentry_clientside_dsn) <= 255)),
     CONSTRAINT check_4f8b811780 CHECK ((char_length(sentry_dsn) <= 255)),
     CONSTRAINT check_51700b31b5 CHECK ((char_length(default_branch_name) <= 255)),
@@ -12910,7 +12917,19 @@ CREATE TABLE container_repositories (
     status smallint,
     expiration_policy_started_at timestamp with time zone,
     expiration_policy_cleanup_status smallint DEFAULT 0 NOT NULL,
-    expiration_policy_completed_at timestamp with time zone
+    expiration_policy_completed_at timestamp with time zone,
+    migration_pre_import_started_at timestamp with time zone,
+    migration_pre_import_done_at timestamp with time zone,
+    migration_import_started_at timestamp with time zone,
+    migration_import_done_at timestamp with time zone,
+    migration_aborted_at timestamp with time zone,
+    migration_skipped_at timestamp with time zone,
+    migration_retries_count integer DEFAULT 0 NOT NULL,
+    migration_skipped_reason smallint,
+    migration_state text DEFAULT 'default'::text NOT NULL,
+    migration_aborted_in_state text,
+    CONSTRAINT check_13c58fe73a CHECK ((char_length(migration_state) <= 255)),
+    CONSTRAINT check_97f0249439 CHECK ((char_length(migration_aborted_in_state) <= 255))
 );
 
 CREATE SEQUENCE container_repositories_id_seq

spec/models/application_setting_spec.rb

@@ -77,6 +77,18 @@ RSpec.describe ApplicationSetting do
     it { is_expected.to validate_numericality_of(:container_registry_cleanup_tags_service_max_list_size).only_integer.is_greater_than_or_equal_to(0) }
     it { is_expected.to validate_numericality_of(:container_registry_expiration_policies_worker_capacity).only_integer.is_greater_than_or_equal_to(0) }
 
+    it { is_expected.to validate_numericality_of(:container_registry_import_max_tags_count).only_integer.is_greater_than_or_equal_to(0) }
+    it { is_expected.to validate_numericality_of(:container_registry_import_max_retries).only_integer.is_greater_than_or_equal_to(0) }
+    it { is_expected.to validate_numericality_of(:container_registry_import_start_max_retries).only_integer.is_greater_than_or_equal_to(0) }
+    it { is_expected.to validate_numericality_of(:container_registry_import_max_step_duration).only_integer.is_greater_than_or_equal_to(0) }
+    it { is_expected.not_to allow_value(nil).for(:container_registry_import_max_tags_count) }
+    it { is_expected.not_to allow_value(nil).for(:container_registry_import_max_retries) }
+    it { is_expected.not_to allow_value(nil).for(:container_registry_import_start_max_retries) }
+    it { is_expected.not_to allow_value(nil).for(:container_registry_import_max_step_duration) }
+
+    it { is_expected.to validate_presence_of(:container_registry_import_target_plan) }
+    it { is_expected.to validate_presence_of(:container_registry_import_created_before) }
+
     it { is_expected.to validate_numericality_of(:dependency_proxy_ttl_group_policy_worker_capacity).only_integer.is_greater_than_or_equal_to(0) }
     it { is_expected.not_to allow_value(nil).for(:dependency_proxy_ttl_group_policy_worker_capacity) }
 

spec/models/container_repository_spec.rb

@@ -25,12 +25,20 @@ RSpec.describe ContainerRepository do
         headers: { 'Content-Type' => 'application/json' })
   end
 
+  it_behaves_like 'having unique enum values'
+
   describe 'associations' do
     it 'belongs to the project' do
       expect(repository).to belong_to(:project)
     end
   end
 
+  describe 'validations' do
+    it { is_expected.to validate_presence_of(:migration_retries_count) }
+    it { is_expected.to validate_numericality_of(:migration_retries_count).is_greater_than_or_equal_to(0) }
+    it { is_expected.to validate_presence_of(:migration_state) }
+  end
+
   describe '#tag' do
     it 'has a test tag' do
       expect(repository.tag('test')).not_to be_nil