Merge branch '208735-container-expiration-policy-app-setting' into 'master'

Application setting for container policies on past projects

See merge request gitlab-org/gitlab!28479

app/helpers/application_settings_helper.rb

@@ -176,6 +176,7 @@ module ApplicationSettingsHelper
       :authorized_keys_enabled,
       :auto_devops_enabled,
       :auto_devops_domain,
+      :container_expiration_policies_enable_historic_entries,
       :container_registry_token_expire_delay,
       :default_artifacts_expire_in,
       :default_branch_protection,

app/models/application_setting.rb

@@ -142,6 +142,9 @@ class ApplicationSetting < ApplicationRecord
 
   validates :default_artifacts_expire_in, presence: true, duration: true
 
+  validates :container_expiration_policies_enable_historic_entries,
+             inclusion: { in: [true, false], message: 'must be a boolean value' }
+
   validates :container_registry_token_expire_delay,
             presence: true,
             numericality: { only_integer: true, greater_than: 0 }

app/models/application_setting_implementation.rb

@@ -42,6 +42,7 @@ module ApplicationSettingImplementation
         asset_proxy_enabled: false,
         authorized_keys_enabled: true, # TODO default to false if the instance is configured to use AuthorizedKeysCommand
         commit_email_hostname: default_commit_email_hostname,
+        container_expiration_policies_enable_historic_entries: false,
         container_registry_token_expire_delay: 5,
         default_artifacts_expire_in: '30 days',
         default_branch_protection: Settings.gitlab['default_branch_protection'],

app/views/admin/application_settings/_registry.html.haml

@@ -5,5 +5,14 @@
     .form-group
       = f.label :container_registry_token_expire_delay, 'Authorization token duration (minutes)', class: 'label-bold'
       = f.number_field :container_registry_token_expire_delay, class: 'form-control'
+    .form-group
+      .form-check
+        = f.check_box :container_expiration_policies_enable_historic_entries, class: 'form-check-input'
+        = f.label :container_expiration_policies_enable_historic_entries, class: 'form-check-label' do
+          = _("Enable container expiration and retention policies for projects created earlier than GitLab 12.7.")
+          = link_to icon('question-circle'), help_page_path('user/packages/container_registry/index', anchor: 'expiration-policy')
+        .form-text.text-muted
+          = _("Existing projects will be able to use expiration policies. Avoid enabling this if an external Container Registry is being used, as there is a performance risk if many images exist on one project.")
+          = link_to icon('question-circle'), help_page_path('user/packages/container_registry/index', anchor: 'use-with-external-container-registries')
 
   = f.submit 'Save changes', class: "btn btn-success"

changelogs/unreleased/208735-container-expiration-policy-app-setting.yml

+---
+title: Add application setting to enable container expiration and retention policies
+  on pre 12.8 projects
+merge_request: 28479
+author:
+type: added

db/migrate/20200331195952_add_container_expiration_policies_enable_historic_entries_to_application_settings.rb

+# frozen_string_literal: true
+
+class AddContainerExpirationPoliciesEnableHistoricEntriesToApplicationSettings < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_column_with_default(:application_settings,
+                            :container_expiration_policies_enable_historic_entries,
+                            :boolean,
+                            default: false,
+                            allow_null: false)
+  end
+
+  def down
+    remove_column(:application_settings,
+                  :container_expiration_policies_enable_historic_entries)
+  end
+end

db/structure.sql

@@ -397,7 +397,8 @@ CREATE TABLE public.application_settings (
     email_restrictions text,
     npm_package_requests_forwarding boolean DEFAULT true NOT NULL,
     namespace_storage_size_limit bigint DEFAULT 0 NOT NULL,
-    seat_link_enabled boolean DEFAULT true NOT NULL
+    seat_link_enabled boolean DEFAULT true NOT NULL,
+    container_expiration_policies_enable_historic_entries boolean DEFAULT false NOT NULL
 );
 
 CREATE SEQUENCE public.application_settings_id_seq
@@ -13001,6 +13002,7 @@ COPY "schema_migrations" (version) FROM STDIN;
 20200330121000
 20200330123739
 20200330132913
+20200331195952
 20200331220930
 20200402123926
 20200402135250

doc/administration/packages/container_registry.md

@@ -516,6 +516,10 @@ on how to achieve that.
 
 ## Use an external container registry with GitLab as an auth endpoint
 
+NOTE: **Note:**
+In using an external container registry, some features associated with the
+container registry may be unavailable or have [inherant risks](./../../user/packages/container_registry/index.md#use-with-external-container-registries)
+
 **Omnibus GitLab**
 
 You can use GitLab as an auth endpoint with an external container registry.

doc/api/settings.md

@@ -45,6 +45,7 @@ Example response:
    "default_group_visibility" : "private",
    "gravatar_enabled" : true,
    "sign_in_text" : null,
+   "container_expiration_policies_enable_historic_entries": true,
    "container_registry_token_expire_delay": 5,
    "repository_storages": ["default"],
    "plantuml_enabled": false,

doc/user/admin_area/settings/index.md

@@ -61,7 +61,7 @@ Access the default page for admin area settings by navigating to
 | ------ | ----------- |
 | [Continuous Integration and Deployment](continuous_integration.md) | Auto DevOps, runners and job artifacts. |
 | [Required pipeline configuration](continuous_integration.md#required-pipeline-configuration-premium-only) **(PREMIUM ONLY)** | Set an instance-wide auto included [pipeline configuration](../../../ci/yaml/README.md). This pipeline configuration will be run after the project's own configuration. |
-| [Package Registry](continuous_integration.md#package-registry-configuration-premium-only) **(PREMIUM ONLY)**| Settings related to the use and experience of using GitLab's Package Registry. |
+| [Package Registry](continuous_integration.md#package-registry-configuration-premium-only) **(PREMIUM ONLY)**| Settings related to the use and experience of using GitLab's Package Registry. Note there are [risks involved](./../../packages/container_registry/index.md#use-with-external-container-registries) in enabling some of these settings. |
 
 ## Reporting
 

doc/user/packages/container_registry/index.md

@@ -488,7 +488,9 @@ older tags and images are regularly removed from the Container Registry.
 > [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/15398) in GitLab 12.8.
 
 NOTE: **Note:**
-Expiration policies are only available for projects created in GitLab 12.8 and later.
+Expiration policies for projects created before GitLab 12.8 may be enabled by an
+admin in the [CI/CD Package Registry settings](./../../admin_area/settings/index.md#cicd).
+Note the inherant [risks involved](./index.md#use-with-external-container-registries).
 
 It is possible to create a per-project expiration policy, so that you can make sure that
 older tags and images are regularly removed from the Container Registry.
@@ -539,6 +541,15 @@ Examples:
 
 See the API documentation for further details: [Edit project](../../../api/projects.md#edit-project).
 
+### Use with external container registries
+
+When using an [external container registry](./../../../administration/packages/container_registry.md#use-an-external-container-registry-with-gitlab-as-an-auth-endpoint),
+running an experation policy on a project may have some performance risks. If a project is going to run
+a policy that will remove large quantities of tags (in the thousands), the GitLab background jobs that
+run the policy may get backed up or fail completely. It is recommended you only enable container expiration
+policies for projects that were created before GitLab 12.8 if you are confident the amount of tags
+being cleaned up will be minimal.
+
 ## Limitations
 
 Moving or renaming existing Container Registry repositories is not supported

locale/gitlab.pot

@@ -7538,6 +7538,9 @@ msgstr ""
 msgid "Enable classification control using an external service"
 msgstr ""
 
+msgid "Enable container expiration and retention policies for projects created earlier than GitLab 12.7."
+msgstr ""
+
 msgid "Enable email restrictions for sign ups"
 msgstr ""
 
@@ -8357,6 +8360,9 @@ msgstr ""
 msgid "Existing members and groups"
 msgstr ""
 
+msgid "Existing projects will be able to use expiration policies. Avoid enabling this if an external Container Registry is being used, as there is a performance risk if many images exist on one project."
+msgstr ""
+
 msgid "Existing shares"
 msgstr ""
 

spec/models/application_setting_spec.rb

@@ -34,6 +34,10 @@ describe ApplicationSetting do
     it { is_expected.to allow_value("dev.gitlab.com").for(:commit_email_hostname) }
     it { is_expected.not_to allow_value("@dev.gitlab").for(:commit_email_hostname) }
 
+    it { is_expected.to allow_value(true).for(:container_expiration_policies_enable_historic_entries) }
+    it { is_expected.to allow_value(false).for(:container_expiration_policies_enable_historic_entries) }
+    it { is_expected.not_to allow_value(nil).for(:container_expiration_policies_enable_historic_entries) }
+
     it { is_expected.to allow_value("myemail@gitlab.com").for(:lets_encrypt_notification_email) }
     it { is_expected.to allow_value(nil).for(:lets_encrypt_notification_email) }
     it { is_expected.not_to allow_value("notanemail").for(:lets_encrypt_notification_email) }