Commit 94fc0619 authored by Alessio Caiazza's avatar Alessio Caiazza

Add timed incremental rollout to Auto DevOps

Auto DevOps deployment strategies now supports timed incremental
rollout. We are deprecating the usage of INCREMENTAL_ROLLOUT_ENABLED
environment variable in Auto DevOps template.

The new behavior will be driven by the INCREMENTAL_ROLLOUT_MODE variable
that can either be manual (same as INCREMENTAL_ROLLOUT_ENABLED) or
timed.

Rollout deployments will be executed using a 5 minute delay between each
job.
parent 1a90632c
...@@ -5,7 +5,8 @@ class ProjectAutoDevops < ActiveRecord::Base ...@@ -5,7 +5,8 @@ class ProjectAutoDevops < ActiveRecord::Base
enum deploy_strategy: { enum deploy_strategy: {
continuous: 0, continuous: 0,
manual: 1 manual: 1,
timed_incremental: 2
} }
scope :enabled, -> { where(enabled: true) } scope :enabled, -> { where(enabled: true) }
...@@ -30,10 +31,7 @@ class ProjectAutoDevops < ActiveRecord::Base ...@@ -30,10 +31,7 @@ class ProjectAutoDevops < ActiveRecord::Base
value: domain.presence || instance_domain) value: domain.presence || instance_domain)
end end
if manual? variables.concat(deployment_strategy_default_variables)
variables.append(key: 'STAGING_ENABLED', value: '1')
variables.append(key: 'INCREMENTAL_ROLLOUT_ENABLED', value: '1')
end
end end
end end
...@@ -51,4 +49,16 @@ class ProjectAutoDevops < ActiveRecord::Base ...@@ -51,4 +49,16 @@ class ProjectAutoDevops < ActiveRecord::Base
!project.public? && !project.public? &&
!project.deploy_tokens.find_by(name: DeployToken::GITLAB_DEPLOY_TOKEN_NAME).present? !project.deploy_tokens.find_by(name: DeployToken::GITLAB_DEPLOY_TOKEN_NAME).present?
end end
def deployment_strategy_default_variables
Gitlab::Ci::Variables::Collection.new.tap do |variables|
if manual?
variables.append(key: 'STAGING_ENABLED', value: '1')
variables.append(key: 'INCREMENTAL_ROLLOUT_ENABLED', value: '1') # deprecated
variables.append(key: 'INCREMENTAL_ROLLOUT_MODE', value: 'manual')
elsif timed_incremental?
variables.append(key: 'INCREMENTAL_ROLLOUT_MODE', value: 'timed')
end
end
end
end end
...@@ -39,10 +39,17 @@ ...@@ -39,10 +39,17 @@
= form.label :deploy_strategy_continuous, class: 'form-check-label' do = form.label :deploy_strategy_continuous, class: 'form-check-label' do
= s_('CICD|Continuous deployment to production') = s_('CICD|Continuous deployment to production')
= link_to icon('question-circle'), help_page_path('topics/autodevops/index.md', anchor: 'auto-deploy'), target: '_blank' = link_to icon('question-circle'), help_page_path('topics/autodevops/index.md', anchor: 'auto-deploy'), target: '_blank'
.form-check
= form.radio_button :deploy_strategy, 'timed_incremental', class: 'form-check-input'
= form.label :deploy_strategy_timed_incremental, class: 'form-check-label' do
= s_('CICD|Continuous deployment to production using timed incremental rollout')
= link_to icon('question-circle'), help_page_path('topics/autodevops/index.md', anchor: 'timed-incremental-rollout-to-production'), target: '_blank'
.form-check .form-check
= form.radio_button :deploy_strategy, 'manual', class: 'form-check-input' = form.radio_button :deploy_strategy, 'manual', class: 'form-check-input'
= form.label :deploy_strategy_manual, class: 'form-check-label' do = form.label :deploy_strategy_manual, class: 'form-check-label' do
= s_('CICD|Automatic deployment to staging, manual deployment to production') = s_('CICD|Automatic deployment to staging, manual deployment to production')
= link_to icon('question-circle'), help_page_path('ci/environments.md', anchor: 'manually-deploying-to-environments'), target: '_blank' = link_to icon('question-circle'), help_page_path('topics/autodevops/index.md', anchor: 'incremental-rollout-to-production'), target: '_blank'
= f.submit _('Save changes'), class: "btn btn-success prepend-top-15" = f.submit _('Save changes'), class: "btn btn-success prepend-top-15"
---
title: Add timed incremental rollout to Auto DevOps
merge_request: 22023
author:
type: added
...@@ -239,14 +239,19 @@ project's **Settings > CI/CD > Auto DevOps**. ...@@ -239,14 +239,19 @@ project's **Settings > CI/CD > Auto DevOps**.
The available options are: The available options are:
- **Continuous deployment to production** - enables [Auto Deploy](#auto-deploy) - **Continuous deployment to production**: Enables [Auto Deploy](#auto-deploy)
by setting the [`STAGING_ENABLED`](#deploy-policy-for-staging-and-production-environments) and with `master` branch directly deployed to production.
[`INCREMENTAL_ROLLOUT_ENABLED`](#incremental-rollout-to-production) variables - **Continuous deployment to production using timed incremental rollout**: Sets the
to false. [`INCREMENTAL_ROLLOUT_MODE`](#timed-incremental-rollout-to-production) variable
- **Automatic deployment to staging, manual deployment to production** - sets the to `timed`, and production deployment will be executed with a 5 minute delay between
each increment in rollout.
- **Automatic deployment to staging, manual deployment to production**: Sets the
[`STAGING_ENABLED`](#deploy-policy-for-staging-and-production-environments) and [`STAGING_ENABLED`](#deploy-policy-for-staging-and-production-environments) and
[`INCREMENTAL_ROLLOUT_ENABLED`](#incremental-rollout-to-production) variables [`INCREMENTAL_ROLLOUT_MODE`](#incremental-rollout-to-production) variables
to true, and the user is responsible for manually deploying to staging and production. to `1` and `manual`. This means:
- `master` branch is directly deployed to staging.
- Manual actions are provided for incremental rollout to production.
## Stages of Auto DevOps ## Stages of Auto DevOps
...@@ -609,7 +614,7 @@ also be customized, and you can easily use a [custom buildpack](#custom-buildpac ...@@ -609,7 +614,7 @@ also be customized, and you can easily use a [custom buildpack](#custom-buildpac
| `DB_MIGRATE` | From GitLab 11.4, this variable can be used to specify the command to run to migrate the application's PostgreSQL database. It runs inside the application pod. | | `DB_MIGRATE` | From GitLab 11.4, this variable can be used to specify the command to run to migrate the application's PostgreSQL database. It runs inside the application pod. |
| `STAGING_ENABLED` | From GitLab 10.8, this variable can be used to define a [deploy policy for staging and production environments](#deploy-policy-for-staging-and-production-environments). | | `STAGING_ENABLED` | From GitLab 10.8, this variable can be used to define a [deploy policy for staging and production environments](#deploy-policy-for-staging-and-production-environments). |
| `CANARY_ENABLED` | From GitLab 11.0, this variable can be used to define a [deploy policy for canary environments](#deploy-policy-for-canary-environments). | | `CANARY_ENABLED` | From GitLab 11.0, this variable can be used to define a [deploy policy for canary environments](#deploy-policy-for-canary-environments). |
| `INCREMENTAL_ROLLOUT_ENABLED`| From GitLab 10.8, this variable can be used to enable an [incremental rollout](#incremental-rollout-to-production) of your application for the production environment. | | `INCREMENTAL_ROLLOUT_MODE`| From GitLab 11.4, this variable, if present, can be used to enable an [incremental rollout](#incremental-rollout-to-production) of your application for the production environment.<br/>Set to: <ul><li>`manual`, for manual deployment jobs.</li><li>`timed`, for automatic rollout deployments with a 5 minute delay each one.</li></ul> |
| `TEST_DISABLED` | From GitLab 11.0, this variable can be used to disable the `test` job. If the variable is present, the job will not be created. | | `TEST_DISABLED` | From GitLab 11.0, this variable can be used to disable the `test` job. If the variable is present, the job will not be created. |
| `CODE_QUALITY_DISABLED` | From GitLab 11.0, this variable can be used to disable the `codequality` job. If the variable is present, the job will not be created. | | `CODE_QUALITY_DISABLED` | From GitLab 11.0, this variable can be used to disable the `codequality` job. If the variable is present, the job will not be created. |
| `SAST_DISABLED` | From GitLab 11.0, this variable can be used to disable the `sast` job. If the variable is present, the job will not be created. | | `SAST_DISABLED` | From GitLab 11.0, this variable can be used to disable the `sast` job. If the variable is present, the job will not be created. |
...@@ -730,9 +735,8 @@ to use an incremental rollout to replace just a few pods with the latest code. ...@@ -730,9 +735,8 @@ to use an incremental rollout to replace just a few pods with the latest code.
This will allow you to first check how the app is behaving, and later manually This will allow you to first check how the app is behaving, and later manually
increasing the rollout up to 100%. increasing the rollout up to 100%.
If `INCREMENTAL_ROLLOUT_ENABLED` is defined in your project (e.g., set If `INCREMENTAL_ROLLOUT_MODE` is set to `manual` in your project, then instead
`INCREMENTAL_ROLLOUT_ENABLED` to `1` as a secret variable), then instead of the of the standard `production` job, 4 different
standard `production` job, 4 different
[manual jobs](../../ci/pipelines.md#manual-actions-from-the-pipeline-graph) [manual jobs](../../ci/pipelines.md#manual-actions-from-the-pipeline-graph)
will be created: will be created:
...@@ -756,21 +760,45 @@ environment page. ...@@ -756,21 +760,45 @@ environment page.
Below, you can see how the pipeline will look if the rollout or staging Below, you can see how the pipeline will look if the rollout or staging
variables are defined. variables are defined.
- **Without `INCREMENTAL_ROLLOUT_ENABLED` and without `STAGING_ENABLED`** Without `INCREMENTAL_ROLLOUT_MODE` and without `STAGING_ENABLED`:
![Staging and rollout disabled](img/rollout_staging_disabled.png)
Without `INCREMENTAL_ROLLOUT_MODE` and with `STAGING_ENABLED`:
![Staging and rollout disabled](img/rollout_staging_disabled.png) ![Staging enabled](img/staging_enabled.png)
- **Without `INCREMENTAL_ROLLOUT_ENABLED` and with `STAGING_ENABLED`** With `INCREMENTAL_ROLLOUT_MODE` set to `manual` and without `STAGING_ENABLED`:
![Staging enabled](img/staging_enabled.png) ![Rollout enabled](img/rollout_enabled.png)
- **With `INCREMENTAL_ROLLOUT_ENABLED` and without `STAGING_ENABLED`** With `INCREMENTAL_ROLLOUT_MODE` set to `manual` and with `STAGING_ENABLED`
![Rollout and staging enabled](img/rollout_staging_enabled.png)
CAUTION: **Caution:**
Before GitLab 11.4 this feature was enabled by the presence of the
`INCREMENTAL_ROLLOUT_ENABLED` environment variable.
This configuration is deprecated and will be removed in the future.
#### Timed incremental rollout to production **[PREMIUM]**
> [Introduced](https://gitlab.com/gitlab-org/gitlab-ee/issues/7545) in GitLab 11.4.
TIP: **Tip:**
You can also set this inside your [project's settings](#deployment-strategy).
![Rollout enabled](img/rollout_enabled.png) This configuration based on
[incremental rollout to production](#incremental-rollout-to-production).
- **With `INCREMENTAL_ROLLOUT_ENABLED` and with `STAGING_ENABLED`** Everything behaves the same way, except:
![Rollout and staging enabled](img/rollout_staging_enabled.png) - It's enabled by setting the `INCREMENTAL_ROLLOUT_MODE` variable to `timed`.
- Instead of the standard `production` job, the following jobs with a 5 minute delay between each are created:
1. `timed rollout 10%`
1. `timed rollout 25%`
1. `timed rollout 50%`
1. `timed rollout 100%`
## Currently supported languages ## Currently supported languages
......
...@@ -25,8 +25,9 @@ ...@@ -25,8 +25,9 @@
# level, or manually added below. # level, or manually added below.
# #
# Continuous deployment to production is enabled by default. # Continuous deployment to production is enabled by default.
# If you want to deploy to staging first, or enable incremental rollouts, # If you want to deploy to staging first, set STAGING_ENABLED environment variable.
# set STAGING_ENABLED or INCREMENTAL_ROLLOUT_ENABLED environment variables. # If you want to enable incremental rollout, either manual or time based,
# set INCREMENTAL_ROLLOUT_TYPE environment variable to "manual" or "timed".
# If you want to use canary deployments, set CANARY_ENABLED environment variable. # If you want to use canary deployments, set CANARY_ENABLED environment variable.
# #
# If Auto DevOps fails to detect the proper buildpack, or if you want to # If Auto DevOps fails to detect the proper buildpack, or if you want to
...@@ -61,6 +62,10 @@ stages: ...@@ -61,6 +62,10 @@ stages:
- staging - staging
- canary - canary
- production - production
- incremental rollout 10%
- incremental rollout 25%
- incremental rollout 50%
- incremental rollout 100%
- performance - performance
- cleanup - cleanup
...@@ -282,11 +287,6 @@ stop_review: ...@@ -282,11 +287,6 @@ stop_review:
variables: variables:
- $REVIEW_DISABLED - $REVIEW_DISABLED
# Keys that start with a dot (.) will not be processed by GitLab CI.
# Staging and canary jobs are disabled by default, to enable them
# remove the dot (.) before the job name.
# https://docs.gitlab.com/ee/ci/yaml/README.html#hidden-keys
# Staging deploys are disabled by default since # Staging deploys are disabled by default since
# continuous deployment to production is enabled by default # continuous deployment to production is enabled by default
# If you prefer to automatically deploy to staging and # If you prefer to automatically deploy to staging and
...@@ -368,6 +368,7 @@ production: ...@@ -368,6 +368,7 @@ production:
- $STAGING_ENABLED - $STAGING_ENABLED
- $CANARY_ENABLED - $CANARY_ENABLED
- $INCREMENTAL_ROLLOUT_ENABLED - $INCREMENTAL_ROLLOUT_ENABLED
- $INCREMENTAL_ROLLOUT_MODE
production_manual: production_manual:
<<: *production_template <<: *production_template
...@@ -383,11 +384,11 @@ production_manual: ...@@ -383,11 +384,11 @@ production_manual:
except: except:
variables: variables:
- $INCREMENTAL_ROLLOUT_ENABLED - $INCREMENTAL_ROLLOUT_ENABLED
- $INCREMENTAL_ROLLOUT_MODE
# This job implements incremental rollout on for every push to `master`. # This job implements incremental rollout on for every push to `master`.
.rollout: &rollout_template .rollout: &rollout_template
stage: production
script: script:
- check_kube_domain - check_kube_domain
- install_dependencies - install_dependencies
...@@ -405,52 +406,77 @@ production_manual: ...@@ -405,52 +406,77 @@ production_manual:
artifacts: artifacts:
paths: [environment_url.txt] paths: [environment_url.txt]
rollout 10%: .manual_rollout_template: &manual_rollout_template
<<: *rollout_template <<: *rollout_template
variables: stage: production
ROLLOUT_PERCENTAGE: 10
when: manual when: manual
# This selectors are backward compatible mode with $INCREMENTAL_ROLLOUT_ENABLED (before 11.4)
only: only:
refs: refs:
- master - master
kubernetes: active kubernetes: active
variables: variables:
- $INCREMENTAL_ROLLOUT_MODE == "manual"
- $INCREMENTAL_ROLLOUT_ENABLED - $INCREMENTAL_ROLLOUT_ENABLED
except:
variables:
- $INCREMENTAL_ROLLOUT_MODE == "timed"
rollout 25%: .timed_rollout_template: &timed_rollout_template
<<: *rollout_template <<: *rollout_template
variables: when: delayed
ROLLOUT_PERCENTAGE: 25 start_in: 5 minutes
when: manual
only: only:
refs: refs:
- master - master
kubernetes: active kubernetes: active
variables: variables:
- $INCREMENTAL_ROLLOUT_ENABLED - $INCREMENTAL_ROLLOUT_MODE == "timed"
rollout 50%: timed rollout 10%:
<<: *rollout_template <<: *timed_rollout_template
stage: incremental rollout 10%
variables:
ROLLOUT_PERCENTAGE: 10
timed rollout 25%:
<<: *timed_rollout_template
stage: incremental rollout 25%
variables:
ROLLOUT_PERCENTAGE: 25
timed rollout 50%:
<<: *timed_rollout_template
stage: incremental rollout 50%
variables: variables:
ROLLOUT_PERCENTAGE: 50 ROLLOUT_PERCENTAGE: 50
when: manual
only: timed rollout 100%:
refs: <<: *timed_rollout_template
- master <<: *production_template
kubernetes: active stage: incremental rollout 100%
variables: variables:
- $INCREMENTAL_ROLLOUT_ENABLED ROLLOUT_PERCENTAGE: 100
rollout 10%:
<<: *manual_rollout_template
variables:
ROLLOUT_PERCENTAGE: 10
rollout 25%:
<<: *manual_rollout_template
variables:
ROLLOUT_PERCENTAGE: 25
rollout 50%:
<<: *manual_rollout_template
variables:
ROLLOUT_PERCENTAGE: 50
rollout 100%: rollout 100%:
<<: *manual_rollout_template
<<: *production_template <<: *production_template
when: manual
allow_failure: false allow_failure: false
only:
refs:
- master
kubernetes: active
variables:
- $INCREMENTAL_ROLLOUT_ENABLED
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
......
...@@ -1081,6 +1081,9 @@ msgstr "" ...@@ -1081,6 +1081,9 @@ msgstr ""
msgid "CICD|Continuous deployment to production" msgid "CICD|Continuous deployment to production"
msgstr "" msgstr ""
msgid "CICD|Continuous deployment to production using timed incremental rollout"
msgstr ""
msgid "CICD|Default to Auto DevOps pipeline" msgid "CICD|Default to Auto DevOps pipeline"
msgstr "" msgstr ""
......
...@@ -5,8 +5,16 @@ FactoryBot.define do ...@@ -5,8 +5,16 @@ FactoryBot.define do
domain "example.com" domain "example.com"
deploy_strategy :continuous deploy_strategy :continuous
trait :manual do trait :continuous_deployment do
deploy_strategy :manual deploy_strategy ProjectAutoDevops.deploy_strategies[:continuous] # rubocop:disable FactoryBot/DynamicAttributeDefinedStatically
end
trait :manual_deployment do
deploy_strategy ProjectAutoDevops.deploy_strategies[:manual] # rubocop:disable FactoryBot/DynamicAttributeDefinedStatically
end
trait :timed_incremental_deployment do
deploy_strategy ProjectAutoDevops.deploy_strategies[:timed_incremental] # rubocop:disable FactoryBot/DynamicAttributeDefinedStatically
end end
trait :disabled do trait :disabled do
......
...@@ -70,24 +70,31 @@ describe ProjectAutoDevops do ...@@ -70,24 +70,31 @@ describe ProjectAutoDevops do
end end
context 'when deploy_strategy is manual' do context 'when deploy_strategy is manual' do
let(:domain) { 'example.com' } let(:auto_devops) { build_stubbed(:project_auto_devops, :manual_deployment, project: project) }
let(:expected_variables) do
[
{ key: 'INCREMENTAL_ROLLOUT_MODE', value: 'manual' },
{ key: 'STAGING_ENABLED', value: '1' },
{ key: 'INCREMENTAL_ROLLOUT_ENABLED', value: '1' }
]
end
before do it { expect(auto_devops.predefined_variables).to include(*expected_variables) }
auto_devops.deploy_strategy = 'manual'
end end
context 'when deploy_strategy is continuous' do
let(:auto_devops) { build_stubbed(:project_auto_devops, :continuous_deployment, project: project) }
it do it do
expect(auto_devops.predefined_variables.map { |var| var[:key] }) expect(auto_devops.predefined_variables.map { |var| var[:key] })
.to include("STAGING_ENABLED", "INCREMENTAL_ROLLOUT_ENABLED") .not_to include("STAGING_ENABLED", "INCREMENTAL_ROLLOUT_ENABLED")
end end
end end
context 'when deploy_strategy is continuous' do context 'when deploy_strategy is timed_incremental' do
let(:domain) { 'example.com' } let(:auto_devops) { build_stubbed(:project_auto_devops, :timed_incremental_deployment, project: project) }
before do it { expect(auto_devops.predefined_variables).to include(key: 'INCREMENTAL_ROLLOUT_MODE', value: 'timed') }
auto_devops.deploy_strategy = 'continuous'
end
it do it do
expect(auto_devops.predefined_variables.map { |var| var[:key] }) expect(auto_devops.predefined_variables.map { |var| var[:key] })
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment