Skip to content

G
gitlab-ce
Commit 2a3ced54 authored by Lin Jen-Shin's avatar Lin Jen-Shin
Browse files
Options

Merge branch '216730-decrease-compile-assets-pull-cache-duration-from-9-17-minutes' into 'master' 

Simplify/optimize the assets compilation jobs and update caches on 2-hourly scheduled master pipelines [UPDATE CACHE]

Closes #218440

See merge request gitlab-org/gitlab!32171
parents a240ab20 862d74e1
Hide whitespace changes
Inline Side-by-side
Showing with 445 additions and 447 deletions
.gitlab-ci.yml
View file @ 2a3ced54
......@@ -15,7 +15,7 @@ stages:
# in cases where jobs require Docker-in-Docker, the job
# definition must be extended with `.use-docker-in-docker`
default:
image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.6-golang-1.14-git-2.26-lfs-2.9-chrome-73.0-node-12.x-yarn-1.21-postgresql-9.6-graphicsmagick-1.3.34"
image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.6-golang-1.14-git-2.26-lfs-2.9-chrome-73.0-node-12.x-yarn-1.21-postgresql-11-graphicsmagick-1.3.34"
tags:
- gitlab-org
# All jobs are interruptible by default
......
.gitlab/ci/docs.gitlab-ci.yml
View file @ 2a3ced54
......@@ -62,7 +62,7 @@ docs lint:
graphql-reference-verify:
extends:
- .default-retry
- .default-cache
- .rails-cache
- .default-before_script
- .docs:rules:graphql-reference-verify
- .use-pg11
......
.gitlab/ci/frontend.gitlab-ci.yml
View file @ 2a3ced54
.assets-compile-cache:
cache:
paths:
- vendor/ruby/
- public/assets/webpack/
- assets-hash.txt
- .yarn-cache/
- tmp/cache/assets/sprockets
- tmp/cache/babel-loader
- tmp/cache/vue-loader
- tmp/cache/webpack-dlls
.gitlab:assets:compile-metadata:
.frontend-base:
extends:
- .default-retry
- .default-before_script
- .assets-compile-cache
image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.6-git-2.26-lfs-2.9-chrome-73.0-node-12.x-yarn-1.21-graphicsmagick-1.3.34-docker-19.03.1
variables:
SETUP_DB: "false"
# we override the max_old_space_size to prevent OOM errors
NODE_OPTIONS: --max_old_space_size=3584
WEBPACK_VENDOR_DLL: "true"
.compile-assets-base:
extends: .frontend-base
image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.6-git-2.26-lfs-2.9-node-12.x-yarn-1.21-graphicsmagick-1.3.34
stage: prepare
script:
- node --version
- run_timed_command "retry yarn install --frozen-lockfile"
- free -m
- run_timed_command "bin/rake gitlab:assets:compile > assets-compile.log 2>&1"
- run_timed_command "scripts/clean-old-cached-assets"
compile-production-assets:
extends:
- .compile-assets-base
- .frontend:rules:compile-production-assets
variables:
NODE_ENV: "production"
RAILS_ENV: "production"
SETUP_DB: "false"
WEBPACK_REPORT: "true"
# we override the max_old_space_size to prevent OOM errors
NODE_OPTIONS: --max_old_space_size=3584
cache:
key: "assets-compile:production:v1"
artifacts:
name: webpack-report
expire_in: 31d
paths:
- webpack-report/
- assets-compile.log
# These assets are used in multiple locations:
# - in `build-assets-image` job to create assets image for packaging systems
# - GitLab UI for integration tests: https://gitlab.com/gitlab-org/gitlab-ui/-/blob/e88493b3c855aea30bf60baee692a64606b0eb1e/.storybook/preview-head.pug#L1
- public/assets
- public/assets/
- webpack-report/
when: always
script:
- node --version
- retry yarn install --frozen-lockfile --production --cache-folder .yarn-cache --prefer-offline
- free -m
- time bin/rake gitlab:assets:compile > assets-compile.log 2>&1
- scripts/clean-old-cached-assets
after_script:
- rm -f /etc/apt/sources.list.d/google*.list # We don't need to update Chrome here
gitlab:assets:compile pull-push-cache:
compile-test-assets:
extends:
- .gitlab:assets:compile-metadata
- .frontend:rules:gitlab-assets-compile-pull-push-cache
cache:
policy: pull-push
gitlab:assets:compile pull-cache:
extends:
- .gitlab:assets:compile-metadata
- .frontend:rules:gitlab-assets-compile-pull-cache
cache:
policy: pull
build-assets-image:
extends:
- .use-kaniko
- .frontend:rules:gitlab-assets-compile-pull-cache
stage: build-images
needs: ["gitlab:assets:compile pull-cache"]
variables:
GIT_DEPTH: "1"
script:
# TODO: Change the image tag to be the MD5 of assets files and skip image building if the image exists
# We'll also need to pass GITLAB_ASSETS_TAG to the trigerred omnibus-gitlab pipeline similarly to how we do it for trigerred CNG pipelines
# https://gitlab.com/gitlab-org/gitlab/issues/208389
- scripts/build_assets_image
.compile-assets-metadata:
extends:
- .default-retry
- .default-before_script
- .assets-compile-cache
stage: prepare
script:
- node --version
- retry yarn install --frozen-lockfile --cache-folder .yarn-cache --prefer-offline
- free -m
- time bin/rake gitlab:assets:compile > assets-compile.log 2>&1
- scripts/clean-old-cached-assets
variables:
SETUP_DB: "false"
# we override the max_old_space_size to prevent OOM errors
NODE_OPTIONS: --max_old_space_size=3584
WEBPACK_VENDOR_DLL: "true"
cache:
key: "assets-compile:test:v1"
- .compile-assets-base
- .frontend:rules:compile-test-assets
artifacts:
expire_in: 7d
paths:
- node_modules
- public/assets
- assets-compile.log
- public/assets/
- node_modules/@gitlab/svgs/dist/icons.json # app/helpers/icons_helper.rb uses this file
when: always
compile-assets pull-push-cache:
compile-test-assets as-if-foss:
extends:
- .compile-assets-metadata
- .frontend:rules:compile-assets-pull-push-cache
cache:
policy: pull-push
- compile-test-assets
- .frontend:rules:compile-test-assets-as-if-foss
- .as-if-foss
compile-assets pull-push-cache as-if-foss:
update-assets-compile-production-cache:
extends:
- .compile-assets-metadata
- .frontend:rules:compile-assets-pull-push-cache-as-if-foss
- .as-if-foss
- compile-production-assets
- .shared:rules:update-cache
stage: prepare
artifacts: {} # This job's purpose is only to update the cache.
cache:
policy: pull-push
key: "assets-compile:test:as-if-foss:v1"
policy: push # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.
compile-assets pull-cache:
update-assets-compile-test-cache:
extends:
- .compile-assets-metadata
- .frontend:rules:compile-assets-pull-cache
- compile-test-assets
- .shared:rules:update-cache
stage: prepare
artifacts: {} # This job's purpose is only to update the cache.
cache:
policy: pull
policy: push # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.
compile-assets pull-cache as-if-foss:
update-yarn-cache:
extends:
- .compile-assets-metadata
- .frontend:rules:compile-assets-pull-cache-as-if-foss
- .as-if-foss
- .default-retry
- .yarn-cache
- .shared:rules:update-cache
stage: prepare
script:
- source scripts/utils.sh
- run_timed_command "retry yarn install --frozen-lockfile"
cache:
policy: pull
key: "assets-compile:test:as-if-foss:v1"
policy: push
build-assets-image:
extends:
- .use-kaniko
- .frontend:rules:compile-production-assets
stage: build-images
needs: ["compile-production-assets"]
variables:
GIT_DEPTH: "1"
script:
# TODO: Change the image tag to be the MD5 of assets files and skip image building if the image exists
# We'll also need to pass GITLAB_ASSETS_TAG to the trigerred omnibus-gitlab pipeline similarly to how we do it for trigerred CNG pipelines
# https://gitlab.com/gitlab-org/gitlab/issues/208389
- run_timed_command "scripts/build_assets_image"
.frontend-fixtures-base:
extends:
- .default-retry
- .frontend-base
- .rails-cache
- .default-before_script
- .use-pg11
stage: fixtures
needs: ["setup-test-env", "compile-assets pull-cache"]
needs: ["setup-test-env", "compile-test-assets"]
variables:
SETUP_DB: "true"
script:
- run_timed_command "scripts/gitaly-test-build"
- run_timed_command "scripts/gitaly-test-spawn"
- run_timed_command "bundle exec rake frontend:fixtures"
- run_timed_command "bin/rake frontend:fixtures"
artifacts:
name: frontend-fixtures
expire_in: 31d
when: always
paths:
- node_modules
- public/assets
- tmp/tests/frontend/
frontend-fixtures:
......@@ -164,25 +135,27 @@ frontend-fixtures-as-if-foss:
- .frontend:rules:default-frontend-jobs-as-if-foss
- .as-if-foss
.frontend-job-base:
.frontend-test-base:
extends:
- .default-retry
- .default-cache
- .default-before_script
- .yarn-cache
variables:
USE_BUNDLE_INSTALL: "false"
SETUP_DB: "false"
stage: test
before_script:
- source scripts/utils.sh
.karma-base:
extends: .frontend-job-base
extends: .frontend-test-base
variables:
# we override the max_old_space_size to prevent OOM errors
NODE_OPTIONS: --max_old_space_size=3584
script:
- source scripts/utils.sh
- export BABEL_ENV=coverage CHROME_LOG_FILE=chrome_debug.log
- date
- yarn karma
- run_timed_command "retry yarn install --frozen-lockfile"
- run_timed_command "yarn karma"
karma:
extends:
......@@ -209,15 +182,11 @@ karma-as-if-foss:
needs: ["frontend-fixtures-as-if-foss"]
.jest-base:
extends: .frontend-job-base
extends: .frontend-test-base
script:
- date
- yarn jest --ci --coverage --testSequencer ./scripts/frontend/parallel_ci_sequencer.js
cache:
key: jest
paths:
- tmp/cache/jest/
policy: pull-push
- source scripts/utils.sh
- run_timed_command "retry yarn install --frozen-lockfile"
- run_timed_command "yarn jest --ci --coverage --testSequencer ./scripts/frontend/parallel_ci_sequencer.js"
jest:
extends:
......@@ -238,17 +207,13 @@ jest:
jest-integration:
extends:
- .frontend-job-base
- .frontend-test-base
- .frontend:rules:default-frontend-jobs
script:
- date
- yarn jest:integration --ci
- source scripts/utils.sh
- run_timed_command "retry yarn install --frozen-lockfile"
- run_timed_command "yarn jest:integration --ci"
needs: ["frontend-fixtures"]
cache:
key: jest-integration
paths:
- tmp/cache/jest/
policy: pull-push
jest-as-if-foss:
extends:
......@@ -256,8 +221,6 @@ jest-as-if-foss:
- .frontend:rules:default-frontend-jobs-as-if-foss
- .as-if-foss
needs: ["frontend-fixtures-as-if-foss"]
cache:
policy: pull
coverage-frontend:
extends:
......@@ -268,33 +231,26 @@ coverage-frontend:
stage: post-test
before_script:
- source scripts/utils.sh
- retry yarn install --frozen-lockfile
- run_timed_command "retry yarn install --frozen-lockfile"
script:
- yarn node scripts/frontend/merge_coverage_frontend.js
- run_timed_command "yarn node scripts/frontend/merge_coverage_frontend.js"
artifacts:
name: coverage-frontend
expire_in: 31d
paths:
- coverage-frontend/
cache:
policy: pull
.qa-frontend-node:
extends:
- .default-retry
- .yarn-cache
- .frontend:rules:qa-frontend-node
stage: test
dependencies: []
cache:
key: "$CI_JOB_NAME"
paths:
- .yarn-cache/
policy: pull-push
script:
- date
- yarn install --frozen-lockfile --cache-folder .yarn-cache --prefer-offline
- date
- yarn run webpack-prod
- source scripts/utils.sh
- run_timed_command "yarn install --frozen-lockfile"
- run_timed_command "yarn run webpack-prod"
qa-frontend-node:10:
extends: .qa-frontend-node
......@@ -309,25 +265,18 @@ qa-frontend-node:latest:
webpack-dev-server:
extends:
- .default-retry
- .yarn-cache
- .frontend:rules:default-frontend-jobs
stage: test
needs: []
variables:
WEBPACK_MEMORY_TEST: "true"
WEBPACK_VENDOR_DLL: "true"
cache:
key:
files:
- yarn.lock
prefix: "v1"
paths:
- node_modules/
- tmp/cache/webpack-dlls/
script:
- source scripts/utils.sh
- retry yarn install --frozen-lockfile
- retry yarn webpack-vendor
- node --expose-gc node_modules/.bin/webpack-dev-server --config config/webpack.config.js
- run_timed_command "retry yarn install --frozen-lockfile"
- run_timed_command "retry yarn webpack-vendor"
- run_timed_command "node --expose-gc node_modules/.bin/webpack-dev-server --config config/webpack.config.js"
artifacts:
name: webpack-dev-server
expire_in: 31d
......@@ -340,7 +289,7 @@ bundle-size-review:
- .frontend:rules:bundle-size-review
image: registry.gitlab.com/gitlab-org/gitlab-build-images:danger
stage: test
needs: ["gitlab:assets:compile pull-cache"]
needs: ["compile-production-assets"]
script:
- mkdir -p bundle-size-review
- cp webpack-report/index.html bundle-size-review/bundle-report.html
......
.gitlab/ci/global.gitlab-ci.yml
View file @ 2a3ced54
......@@ -10,46 +10,58 @@
.default-before_script:
before_script:
- date
- '[ "$FOSS_ONLY" = "1" ] && rm -rf ee/ qa/spec/ee/ qa/qa/specs/features/ee/ qa/qa/ee/ qa/qa/ee.rb'
- export GOPATH=$CI_PROJECT_DIR/.go
- mkdir -p $GOPATH
- source scripts/utils.sh
- source scripts/prepare_build.sh
- date
# Jobs that only need to pull cache
.default-cache:
.rails-cache:
cache:
key: "debian-stretch-ruby-2.6.6-pg11-node-12.x"
key: "rails-v1"
paths:
- .go/pkg/mod
- vendor/ruby
- .yarn-cache/
- vendor/gitaly-ruby
- vendor/ruby/
- vendor/gitaly-ruby/
- .go/pkg/mod/
policy: pull
.rails-cache:
.static-analysis-cache:
cache:
key: "static-analysis-v1"
paths:
- vendor/ruby/
- node_modules/
- tmp/rubocop_cache/
policy: pull
.qa-cache:
cache:
key:
files:
- Gemfile.lock
- GITALY_SERVER_VERSION
prefix: "ruby-go-cache-v1"
key: "qa-v1"
paths:
- vendor/ruby
- vendor/gitaly-ruby
- .go/pkg/mod
- qa/vendor/ruby/
policy: pull
.yarn-cache:
cache:
key:
files:
- yarn.lock
prefix: "v1"
key: "yarn-v1"
paths:
- node_modules/
- tmp/cache/webpack-dlls/
policy: pull
.assets-compile-cache:
cache:
key: "assets-compile-${NODE_ENV}-v1"
paths:
- vendor/ruby/
- node_modules/
- assets-hash.txt
- public/assets/webpack/
- tmp/cache/assets/sprockets/
- tmp/cache/babel-loader/
- tmp/cache/vue-loader/
- tmp/cache/webpack-dlls/
policy: pull
.use-pg11:
image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.6-golang-1.14-git-2.26-lfs-2.9-chrome-73.0-node-12.x-yarn-1.21-postgresql-11-graphicsmagick-1.3.34"
......@@ -75,6 +87,7 @@
name: gcr.io/kaniko-project/executor:debug-v0.20.0
entrypoint: [""]
before_script:
- source scripts/utils.sh
- mkdir -p /kaniko/.docker
- echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json
......
.gitlab/ci/memory.gitlab-ci.yml
View file @ 2a3ced54
.only-code-memory-job-base:
extends:
- .default-retry
- .default-cache
- .rails-cache
- .default-before_script
- .memory:rules
......@@ -39,7 +39,7 @@ memory-on-boot:
- .only-code-memory-job-base
- .use-pg11
stage: test
needs: ["setup-test-env", "compile-assets pull-cache"]
needs: ["setup-test-env", "compile-test-assets"]
variables:
NODE_ENV: "production"
RAILS_ENV: "production"
......
.gitlab/ci/pages.gitlab-ci.yml
View file @ 2a3ced54
......@@ -7,7 +7,7 @@ pages:
- rspec:coverage
- coverage-frontend
- karma
- gitlab:assets:compile pull-cache
- compile-production-assets
script:
- mv public/ .public/
- mkdir public/
......
.gitlab/ci/qa.gitlab-ci.yml
View file @ 2a3ced54
.qa-job-base:
extends:
- .default-retry
- .qa-cache
stage: test
needs: []
cache:
key: "qa-framework-jobs:v1"
paths:
- vendor/ruby
before_script:
- '[ "$FOSS_ONLY" = "1" ] && rm -rf ee/ qa/spec/ee/ qa/qa/specs/features/ee/ qa/qa/ee/ qa/qa/ee.rb'
- cd qa/
......@@ -22,11 +19,9 @@ qa:internal:
qa:internal-as-if-foss:
extends:
- .qa-job-base
- qa:internal
- .qa:rules:as-if-foss
- .as-if-foss
script:
- bundle exec rspec
qa:selectors:
extends:
......@@ -41,6 +36,16 @@ qa:selectors-as-if-foss:
- .qa:rules:as-if-foss
- .as-if-foss
update-qa-cache:
extends:
- .qa-job-base
- .shared:rules:update-cache
stage: prepare
script:
- echo "Cache has been updated and ready to be uploaded."
cache:
policy: push # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.
.package-and-qa-base:
image: ruby:2.6-alpine
stage: qa
......
.gitlab/ci/rails.gitlab-ci.yml
View file @ 2a3ced54
.rails:needs:setup-and-assets:
needs: ["setup-test-env", "compile-assets pull-cache"]
.rails-job-base:
extends:
- .default-retry
......@@ -35,33 +32,54 @@ setup-test-env:
- tmp/tests/repositories
- tmp/tests/second_storage
when: always
update-rails-cache:
extends:
- setup-test-env
- .shared:rules:update-cache
artifacts: {} # This job's purpose is only to update the cache.
cache:
policy: push # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.
.static-analysis-base:
extends:
- .default-retry
- .default-before_script
- .static-analysis-cache
needs: []
variables:
SETUP_DB: "false"
ENABLE_SPRING: "1"
update-static-analysis-cache:
extends:
- .static-analysis-base
- .shared:rules:update-cache
stage: prepare
script:
- rm -rf ./node_modules # We remove node_modules because there's no mechanism to remove stall entries.
- run_timed_command "retry yarn install --frozen-lockfile"
- bundle exec rubocop --parallel # For the moment we only cache `vendor/ruby/`, `node_modules/`, and `tmp/rubocop_cache` so we don't need to run all the tasks,
cache:
# We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up but RuboCop has a mechanism
# for keeping only the N latest cache files, so we take advantage of it with `pull-push` and removing `node_modules` at the start of the job.
policy: pull-push
static-analysis:
extends:
- .rails-job-base
- .static-analysis-base
- .rails:rules:default-refs-code-backstage-qa
- .rails:needs:setup-and-assets
stage: test
variables:
SETUP_DB: "false"
ENABLE_SPRING: "1"
parallel: 4
script:
- run_timed_command "retry yarn install --frozen-lockfile"
- scripts/static-analysis
cache:
key: "ruby-2.6.6-pg11-rubocop"
paths:
- vendor/ruby
- tmp/rubocop_cache
policy: pull-push
downtime_check:
extends:
- .rails-job-base
- .rails:rules:downtime_check
needs: ["setup-test-env"]
needs: []
stage: test
variables:
SETUP_DB: "false"
......@@ -71,7 +89,7 @@ downtime_check:
.rspec-base:
extends: .rails-job-base
stage: test
needs: ["setup-test-env", "retrieve-tests-metadata", "compile-assets pull-cache"]
needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets"]
script:
- run_timed_command "scripts/gitaly-test-build"
- run_timed_command "scripts/gitaly-test-spawn"
......@@ -219,8 +237,6 @@ rspec:coverage:
- memory-on-boot
variables:
SETUP_DB: "false"
cache:
policy: pull
script:
- bundle exec scripts/merge-simplecov
- bundle exec scripts/gather-test-memory-data
......@@ -248,7 +264,7 @@ rspec:coverage:
- .rails:rules:as-if-foss
- .as-if-foss
- .use-pg11
needs: ["setup-test-env", "retrieve-tests-metadata", "compile-assets pull-cache as-if-foss"]
needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets as-if-foss"]
.rspec-ee-base-pg11:
extends:
......
.gitlab/ci/review.gitlab-ci.yml
View file @ 2a3ced54
......@@ -38,7 +38,7 @@ review-build-cng:
- install_api_client_dependencies_with_apk
- install_gitlab_gem
needs:
- job: gitlab:assets:compile pull-cache
- job: compile-production-assets
artifacts: false
script:
- BUILD_TRIGGER_TOKEN=$REVIEW_APPS_BUILD_TRIGGER_TOKEN ./scripts/trigger-build cng
......@@ -238,5 +238,3 @@ danger-review:
- source scripts/utils.sh
- retry yarn install --frozen-lockfile
- danger --fail-on-errors=true --verbose
cache:
policy: pull
.gitlab/ci/rules.gitlab-ci.yml
View file @ 2a3ced54
......@@ -37,6 +37,9 @@
.if-merge-request-title-as-if-foss: &if-merge-request-title-as-if-foss
if: '$CI_MERGE_REQUEST_TITLE =~ /RUN AS-IF-FOSS/'
.if-merge-request-title-update-caches: &if-merge-request-title-update-caches
if: '$CI_MERGE_REQUEST_TITLE =~ /UPDATE CACHE/'
.if-security-merge-request: &if-security-merge-request
if: '$CI_PROJECT_NAMESPACE == "gitlab-org/security" && $CI_MERGE_REQUEST_IID'
......@@ -183,6 +186,14 @@
- ".dockerignore"
- "qa/**/*"
################
# Shared rules #
################
.shared:rules:update-cache:
rules:
- <<: *if-master-schedule-2-hourly
- <<: *if-merge-request-title-update-caches
####################
# Cache repo rules #
####################
......@@ -244,45 +255,18 @@
##################
# Frontend rules #
##################
# This job only runs on `master` since it pushes to the cache.
.frontend:rules:gitlab-assets-compile-pull-push-cache:
rules:
- <<: *if-not-canonical-namespace
when: never
- <<: *if-master-refs
changes: *code-backstage-qa-patterns
when: on_success
.frontend:rules:gitlab-assets-compile-pull-cache:
.frontend:rules:compile-production-assets:
rules:
- <<: *if-not-canonical-namespace
when: never
- <<: *if-default-refs
changes: *code-backstage-qa-patterns
when: on_success
.frontend:rules:compile-assets-pull-push-cache:
rules:
- <<: *if-master-refs
changes: *code-backstage-qa-patterns
when: on_success
# This job only runs on `master` since it pushes to the cache.
.frontend:rules:compile-assets-pull-push-cache-as-if-foss:
rules:
- <<: *if-not-ee
when: never
- <<: *if-master-push
changes: *code-backstage-qa-patterns
- <<: *if-master-schedule-2-hourly
.frontend:rules:compile-assets-pull-cache:
.frontend:rules:compile-test-assets:
rules:
- <<: *if-default-refs
changes: *code-backstage-qa-patterns
when: on_success
- changes: *code-backstage-qa-patterns
.frontend:rules:compile-assets-pull-cache-as-if-foss:
.frontend:rules:compile-test-assets-as-if-foss:
rules:
- <<: *if-not-ee
when: never
......@@ -299,7 +283,6 @@
rules:
- <<: *if-default-refs
changes: *code-backstage-patterns
when: on_success
.frontend:rules:default-frontend-jobs-as-if-foss:
rules:
......@@ -327,10 +310,8 @@
rules:
- <<: *if-master-refs
changes: *frontend-dependency-patterns
when: on_success
- <<: *if-merge-request
changes: *frontend-dependency-patterns
when: on_success
.frontend:rules:qa-frontend-node-latest:
rules:
......
.gitlab/ci/setup.gitlab-ci.yml
View file @ 2a3ced54
......@@ -3,7 +3,7 @@
cache gems:
extends:
- .default-retry
- .default-cache
- .rails-cache
- .default-before_script
- .setup:rules:cache-gems
stage: test
......
.gitlab/ci/test-metadata.gitlab-ci.yml
View file @ 2a3ced54
......@@ -3,11 +3,6 @@
TESTS_METADATA_S3_BUCKET: "gitlab-ce-cache"
before_script:
- source scripts/utils.sh
cache:
key: tests_metadata
paths:
- knapsack/
- rspec_flaky/
artifacts:
expire_in: 31d
paths:
......@@ -20,8 +15,6 @@ retrieve-tests-metadata:
- .tests-metadata-state
- .test-metadata:rules:retrieve-tests-metadata
stage: prepare
cache:
policy: pull
script:
- source scripts/rspec_helpers.sh
- retrieve_tests_metadata
......@@ -44,8 +37,6 @@ update-tests-metadata:
- rspec-ee unit pg11 geo
- rspec-ee integration pg11 geo
- rspec-ee system pg11 geo
cache:
policy: push
script:
- retry gem install fog-aws mime-types activesupport rspec_profiling postgres-copy --no-document
- source scripts/rspec_helpers.sh
......
.rubocop.yml
View file @ 2a3ced54
......@@ -27,6 +27,7 @@ AllCops:
- 'plugins/**/*'
- 'file_hooks/**/*'
CacheRootDirectory: tmp
MaxFilesInCache: 18000
Cop/AvoidKeywordArgumentsInSidekiqWorkers:
Enabled: true
......
config/initializers/01_secret_token.rb
View file @ 2a3ced54
......@@ -61,6 +61,8 @@ def generate_new_rsa_private_key
end
def warn_missing_secret(secret)
return if Rails.env.test?
warn "Missing Rails.application.secrets.#{secret} for #{Rails.env} environment. The secret will be generated and stored in config/secrets.yml."
end
......
doc/development/pipelines.md
View file @ 2a3ced54
......@@ -11,173 +11,17 @@ We're striving to [dogfood](https://about.gitlab.com/handbook/engineering/#dogfo
GitLab [CI/CD features and best-practices](../ci/yaml/README.md)
as much as possible.
## Stages
## Overview
The current stages are:
- `sync`: This stage is used to synchronize changes from <https://gitlab.com/gitlab-org/gitlab> to
<https://gitlab.com/gitlab-org/gitlab-foss>.
- `prepare`: This stage includes jobs that prepare artifacts that are needed by
jobs in subsequent stages.
- `build-images`: This stage includes jobs that prepare Docker images
that are needed by jobs in subsequent stages or downstream pipelines.
- `fixtures`: This stage includes jobs that prepare fixtures needed by frontend tests.
- `test`: This stage includes most of the tests, DB/migration jobs, and static analysis jobs.
- `post-test`: This stage includes jobs that build reports or gather data from
the `test` stage's jobs (e.g. coverage, Knapsack metadata etc.).
- `review-prepare`: This stage includes a job that build the CNG images that are
later used by the (Helm) Review App deployment (see
[Review Apps](testing_guide/review_apps.md) for details).
- `review`: This stage includes jobs that deploy the GitLab and Docs Review Apps.
- `qa`: This stage includes jobs that perform QA tasks against the Review App
that is deployed in the previous stage.
- `post-qa`: This stage includes jobs that build reports or gather data from
the `qa` stage's jobs (e.g. Review App performance report).
- `pages`: This stage includes a job that deploys the various reports as
GitLab Pages (e.g. <https://gitlab-org.gitlab.io/gitlab/coverage-ruby/>,
<https://gitlab-org.gitlab.io/gitlab/coverage-javascript/>,
<https://gitlab-org.gitlab.io/gitlab/webpack-report/>).
## Default image
The default image is defined in <https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab-ci.yml>.
It includes Ruby, Go, Git, Git LFS, Chrome, Node, Yarn, PostgreSQL, and Graphics Magick.
The images used in our pipelines are configured in the
[`gitlab-org/gitlab-build-images`](https://gitlab.com/gitlab-org/gitlab-build-images)
project, which is push-mirrored to <https://dev.gitlab.org/gitlab/gitlab-build-images>
for redundancy.
The current version of the build images can be found in the
["Used by GitLab section"](https://gitlab.com/gitlab-org/gitlab-build-images/blob/master/.gitlab-ci.yml).
## Default variables
In addition to the [predefined variables](../ci/variables/predefined_variables.md),
each pipeline includes default variables defined in
<https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab-ci.yml>.
## Common job definitions
Most of the jobs [extend from a few CI definitions](../ci/yaml/README.md#extends)
defined in [`.gitlab/ci/global.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab/ci/global.gitlab-ci.yml)
that are scoped to a single [configuration parameter](../ci/yaml/README.md#configuration-parameters).
| Job definitions | Description |
|------------------|-------------|
| `.default-tags` | Ensures a job has the `gitlab-org` tag to ensure it's using our dedicated runners. |
| `.default-retry` | Allows a job to [retry](../ci/yaml/README.md#retry) upon `unknown_failure`, `api_failure`, `runner_system_failure`, `job_execution_timeout`, or `stuck_or_timeout_failure`. |
| `.default-before_script` | Allows a job to use a default `before_script` definition suitable for Ruby/Rails tasks that may need a database running (e.g. tests). |
| `.default-cache` | Allows a job to use a default `cache` definition suitable for Ruby/Rails and frontend tasks. |
| `.use-pg11` | Allows a job to use the `postgres:11.6` and `redis:alpine` services. |
| `.use-pg11-ee` | Same as `.use-pg11` but also use the `docker.elastic.co/elasticsearch/elasticsearch:6.4.2` services. |
| `.use-kaniko` | Allows a job to use the `kaniko` tool to build Docker images. |
| `.as-if-foss` | Simulate the FOSS project by setting the `FOSS_ONLY='1'` environment variable. |
## `workflow:rules`
We're using the [`workflow:rules` keyword](../ci/yaml/README.md#workflowrules) to
define default rules to determine whether or not a pipeline is created.
These rules are defined in <https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab-ci.yml>
and are as follows:
1. If `$FORCE_GITLAB_CI` is set, create a pipeline.
1. For merge requests, create a pipeline.
1. For `master` branch, create a pipeline (this includes on schedules, pushes, merges, etc.).
1. For tags, create a pipeline.
1. If `$GITLAB_INTERNAL` isn't set, don't create a pipeline.
1. For stable, auto-deploy, and security branches, create a pipeline.
1. For any other cases (e.g. when pushing a branch with no MR for it), no pipeline is created.
## `rules`, `if:` conditions and `changes:` patterns
We're using the [`rules` keyword](../ci/yaml/README.md#rules) extensively.
All `rules` definitions are defined in
<https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/rules.gitlab-ci.yml>,
then included in individual jobs via [`extends`](../ci/yaml/README.md#extends).
The `rules` definitions are composed of `if:` conditions and `changes:` patterns,
which are also defined in
<https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/rules.gitlab-ci.yml>
and included in `rules` definitions via [YAML anchors](../ci/yaml/README.md#anchors)
### `if:` conditions
| `if:` conditions | Description | Notes |
|------------------|-------------|-------|
| `if-not-canonical-namespace` | Matches if the project isn't in the canonical (`gitlab-org/`) or security (`gitlab-org/security`) namespace. | Use to create a job for forks (by using `when: on_success\|manual`), or **not** create a job for forks (by using `when: never`). |
| `if-not-ee` | Matches if the project isn't EE (i.e. project name isn't `gitlab` or `gitlab-ee`). | Use to create a job only in the FOSS project (by using `when: on_success|manual`), or **not** create a job if the project is EE (by using `when: never`). |
| `if-not-foss` | Matches if the project isn't FOSS (i.e. project name isn't `gitlab-foss`, `gitlab-ce`, or `gitlabhq`). | Use to create a job only in the EE project (by using `when: on_success|manual`), or **not** create a job if the project is FOSS (by using `when: never`). |
| `if-default-refs` | Matches if the pipeline is for `master`, `/^[\d-]+-stable(-ee)?$/` (stable branches), `/^\d+-\d+-auto-deploy-\d+$/` (auto-deploy branches), `/^security\//` (security branches), merge requests, and tags. | Note that jobs won't be created for branches with this default configuration. |
| `if-master-refs` | Matches if the current branch is `master`. | |
| `if-master-or-tag` | Matches if the pipeline is for the `master` branch or for a tag. | |
| `if-merge-request` | Matches if the pipeline is for a merge request. | |
| `if-nightly-master-schedule` | Matches if the pipeline is for a `master` scheduled pipeline with `$NIGHTLY` set. | |
| `if-dot-com-gitlab-org-schedule` | Limits jobs creation to scheduled pipelines for the `gitlab-org` group on GitLab.com. | |
| `if-dot-com-gitlab-org-master` | Limits jobs creation to the `master` branch for the `gitlab-org` group on GitLab.com. | |
| `if-dot-com-gitlab-org-merge-request` | Limits jobs creation to merge requests for the `gitlab-org` group on GitLab.com. | |
| `if-dot-com-gitlab-org-and-security-tag` | Limits job creation to tags for the `gitlab-org` and `gitlab-org/security` groups on GitLab.com. | |
| `if-dot-com-gitlab-org-and-security-merge-request` | Limit jobs creation to merge requests for the `gitlab-org` and `gitlab-org/security` groups on GitLab.com. | |
| `if-dot-com-ee-schedule` | Limits jobs to scheduled pipelines for the `gitlab-org/gitlab` project on GitLab.com. | |
| `if-cache-credentials-schedule` | Limits jobs to scheduled pipelines with the `$CI_REPO_CACHE_CREDENTIALS` variable set. | |
### `changes:` patterns
| `changes:` patterns | Description |
|------------------------------|--------------------------------------------------------------------------|
| `ci-patterns` | Only create job for CI config-related changes. |
| `yaml-patterns` | Only create job for YAML-related changes. |
| `docs-patterns` | Only create job for docs-related changes. |
| `frontend-dependency-patterns` | Only create job when frontend dependencies are updated (i.e. `package.json`, and `yarn.lock`). changes. |
| `frontend-patterns` | Only create job for frontend-related changes. |
| `backstage-patterns` | Only create job for backstage-related changes (i.e. Danger, fixtures, RuboCop, specs). |
| `code-patterns` | Only create job for code-related changes. |
| `qa-patterns` | Only create job for QA-related changes. |
| `code-backstage-patterns` | Combination of `code-patterns` and `backstage-patterns`. |
| `code-qa-patterns` | Combination of `code-patterns` and `qa-patterns`. |
| `code-backstage-qa-patterns` | Combination of `code-patterns`, `backstage-patterns`, and `qa-patterns`. |
## Interruptible jobs pipelines
By default, all jobs are [interruptible](../ci/yaml/README.md#interruptible), except the
`dont-interrupt-me` job which runs automatically on `master`, and is `manual`
otherwise.
If you want a running pipeline to finish even if you push new commits to a merge
request, be sure to start the `dont-interrupt-me` job before pushing.
## PostgreSQL versions testing
### Current versions testing
| Where? | PostgreSQL version |
| ------ | ------ |
| MRs | 11 |
| `master` (non-scheduled pipelines) | 11 |
| 2-hourly scheduled pipelines | 11 |
### Long-term plan
We follow the [PostgreSQL versions shipped with Omnibus GitLab](https://docs.gitlab.com/omnibus/package-information/postgresql_versions.html):
| PostgreSQL version | 12.10 (April 2020) | 13.0 (May 2020) | 13.1 (June 2020) | 13.2 (July 2020) | 13.3 (August 2020) | 13.4, 13.5 | 13.6 (November 2020) | 14.0 (May 2021?) |
| ------ | ------------------ | --------------- | ---------------- | ---------------- | ------------------ | ------------ | -------------------- | ---------------- |
| PG9.6 | MRs/`master`/`2-hour`/`nightly` | - | - | - | - | - | - | - |
| PG10 | `nightly` | - | - | - | - | - | - | - |
| PG11 | `master`/`2-hour` | MRs/`master`/`2-hour`/`nightly` | MRs/`master`/`2-hour`/`nightly` | MRs/`master`/`2-hour`/`nightly` | MRs/`master`/`2-hour`/`nightly` | MRs/`master`/`2-hour`/`nightly` | `nightly` | - |
| PG12 | - | - | - | - | `master`/`2-hour` | `master`/`2-hour` | MRs/`master`/`2-hour`/`nightly` | `master`/`2-hour` |
| PG13 | - | - | - | - | - | - | - | MRs/`master`/`2-hour`/`nightly` |
## Pipeline types
### Pipeline types
Since we use the [`rules:`](../ci/yaml/README.md#rules) and [`needs:`](../ci/yaml/README.md#needs) keywords extensively,
we have four main pipeline types which are described below. Note that an MR that includes multiple types of changes would
have a pipelines that include jobs from multiple types (e.g. a combination of docs-only and code-only pipelines).
### Docs-only MR pipeline
#### Docs-only MR pipeline
<details><summary>Click to view the graph</summary>
Reference pipeline: <https://gitlab.com/gitlab-org/gitlab/pipelines/135236627>
......@@ -191,7 +35,11 @@ graph LR
end
```
### Code-only MR pipeline
</details>
#### Code-only MR pipeline
<details><summary>Click to view the graph</summary>
Reference pipeline: <https://gitlab.com/gitlab-org/gitlab/pipelines/136295694>
......@@ -204,11 +52,11 @@ graph RL;
click 1-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8100542&udv=0"
1-2["build-qa-image (3.4 minutes)"];
click 1-2 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914325&udv=0"
1-3["compile-assets pull-cache (9.06 minutes)"];
1-3["compile-test-assets (9.06 minutes)"];
click 1-3 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914317&udv=0"
1-4["compile-assets pull-cache as-if-foss (8.35 minutes)"];
1-4["compile-test-assets as-if-foss (8.35 minutes)"];
click 1-4 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8356616&udv=0"
1-5["gitlab:assets:compile pull-cache (22 minutes)"];
1-5["compile-production-assets (22 minutes)"];
click 1-5 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914312&udv=0"
1-6["setup-test-env (8.22 minutes)"];
click 1-6 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914315&udv=0"
......@@ -224,6 +72,8 @@ graph RL;
1-18["kubesec-sast"];
1-19["nodejs-scan-sast"];
1-20["secrets-sast"];
1-21["static-analysis (17 minutes)"];
click 1-21 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914471&udv=0"
class 1-3 criticalPath;
class 1-6 criticalPath;
......@@ -241,8 +91,6 @@ graph RL;
2_1-1 & 2_1-2 & 2_1-3 & 2_1-4 --> 1-6;
end
2_2-1["static-analysis (17 minutes)"];
click 2_2-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914471&udv=0"
2_2-2["frontend-fixtures (17.2 minutes)"];
class 2_2-2 criticalPath;
click 2_2-2 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=7910143&udv=0"
......@@ -252,13 +100,13 @@ graph RL;
click 2_2-4 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8356727&udv=0"
2_2-5["webpack-dev-server (6.1 minutes)"];
click 2_2-5 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8404303&udv=0"
subgraph "Needs `setup-test-env` & `compile-assets`";
2_2-1 & 2_2-2 & 2_2-4 & 2_2-5 --> 1-6 & 1-3;
subgraph "Needs `setup-test-env` & `compile-test-assets`";
2_2-2 & 2_2-4 & 2_2-5 --> 1-6 & 1-3;
2_2-3 --> 1-6 & 1-4;
end
2_3-1["build-assets-image (2.5 minutes)"];
subgraph "Needs `gitlab:assets:compile`";
subgraph "Needs `compile-production-assets`";
2_3-1 --> 1-5
end
......@@ -269,7 +117,7 @@ graph RL;
end
2_5-1["rspec & db jobs (12-22 minutes)"];
subgraph "Needs `compile-assets`, `setup-test-env`, & `retrieve-tests-metadata`";
subgraph "Needs `compile-test-assets`, `setup-test-env`, & `retrieve-tests-metadata`";
2_5-1 --> 1-3 & 1-6 & 1-14;
class 2_5-1 criticalPath;
click 2_5-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations"
......@@ -304,7 +152,11 @@ graph RL;
end
```
### Frontend-only MR pipeline
</details>
#### Frontend-only MR pipeline
<details><summary>Click to view the graph</summary>
Reference pipeline: <https://gitlab.com/gitlab-org/gitlab/pipelines/134661039>
......@@ -317,11 +169,11 @@ graph RL;
click 1-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8100542&udv=0"
1-2["build-qa-image (3.4 minutes)"];
click 1-2 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914325&udv=0"
1-3["compile-assets pull-cache (9.06 minutes)"];
1-3["compile-test-assets (9.06 minutes)"];
click 1-3 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914317&udv=0"
1-4["compile-assets pull-cache as-if-foss (8.35 minutes)"];
1-4["compile-test-assets as-if-foss (8.35 minutes)"];
click 1-4 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8356616&udv=0"
1-5["gitlab:assets:compile pull-cache (22 minutes)"];
1-5["compile-production-assets (22 minutes)"];
click 1-5 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914312&udv=0"
1-6["setup-test-env (8.22 minutes)"];
click 1-6 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914315&udv=0"
......@@ -337,6 +189,8 @@ graph RL;
1-18["kubesec-sast"];
1-19["nodejs-scan-sast"];
1-20["secrets-sast"];
1-21["static-analysis (17 minutes)"];
click 1-21 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914471&udv=0"
class 1-3 criticalPath;
class 1-5 criticalPath;
......@@ -355,8 +209,6 @@ graph RL;
2_1-1 & 2_1-2 & 2_1-3 & 2_1-4 --> 1-6;
end
2_2-1["static-analysis (17 minutes)"];
click 2_2-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914471&udv=0"
2_2-2["frontend-fixtures (17.2 minutes)"];
class 2_2-2 criticalPath;
click 2_2-2 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=7910143&udv=0"
......@@ -366,14 +218,14 @@ graph RL;
click 2_2-4 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8356727&udv=0"
2_2-5["webpack-dev-server (6.1 minutes)"];
click 2_2-5 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8404303&udv=0"
subgraph "Needs `setup-test-env` & `compile-assets`";
2_2-1 & 2_2-2 & 2_2-4 & 2_2-5 --> 1-6 & 1-3;
subgraph "Needs `setup-test-env` & `compile-test-assets`";
2_2-2 & 2_2-4 & 2_2-5 --> 1-6 & 1-3;
2_2-3 --> 1-6 & 1-4;
end
2_3-1["build-assets-image (2.5 minutes)"];
class 2_3-1 criticalPath;
subgraph "Needs `gitlab:assets:compile`";
subgraph "Needs `compile-production-assets`";
2_3-1 --> 1-5
end
......@@ -384,7 +236,7 @@ graph RL;
end
2_5-1["rspec & db jobs (12-22 minutes)"];
subgraph "Needs `compile-assets`, `setup-test-env, & `retrieve-tests-metadata`";
subgraph "Needs `compile-test-assets`, `setup-test-env, & `retrieve-tests-metadata`";
2_5-1 --> 1-3 & 1-6 & 1-14;
class 2_5-1 criticalPath;
click 2_5-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations"
......@@ -444,7 +296,11 @@ graph RL;
end
```
### QA-only MR pipeline
</details>
#### QA-only MR pipeline
<details><summary>Click to view the graph</summary>
Reference pipeline: <https://gitlab.com/gitlab-org/gitlab/pipelines/134645109>
......@@ -457,11 +313,11 @@ graph RL;
click 1-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8100542&udv=0"
1-2["build-qa-image (3.4 minutes)"];
click 1-2 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914325&udv=0"
1-3["compile-assets pull-cache (9.06 minutes)"];
1-3["compile-test-assets (9.06 minutes)"];
click 1-3 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914317&udv=0"
1-4["compile-assets pull-cache as-if-foss (8.35 minutes)"];
1-4["compile-test-assets as-if-foss (8.35 minutes)"];
click 1-4 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=8356616&udv=0"
1-5["gitlab:assets:compile pull-cache (22 minutes)"];
1-5["compile-production-assets (22 minutes)"];
click 1-5 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914312&udv=0"
1-6["setup-test-env (8.22 minutes)"];
click 1-6 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914315&udv=0"
......@@ -477,6 +333,8 @@ graph RL;
1-18["kubesec-sast"];
1-19["nodejs-scan-sast"];
1-20["secrets-sast"];
1-21["static-analysis (17 minutes)"];
click 1-21 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914471&udv=0"
class 1-5 criticalPath;
end
......@@ -487,14 +345,8 @@ graph RL;
2_1-1 --> 1-6;
end
2_2-1["static-analysis (17 minutes)"];
click 2_2-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914471&udv=0"
subgraph "Needs `setup-test-env` & `compile-assets`";
2_2-1 --> 1-6 & 1-3;
end
2_3-1["build-assets-image (2.5 minutes)"];
subgraph "Needs `gitlab:assets:compile`";
subgraph "Needs `compile-production-assets`";
2_3-1 --> 1-5
class 2_3-1 criticalPath;
end
......@@ -507,16 +359,56 @@ graph RL;
end
```
## Test jobs
</details>
### `workflow:rules`
We're using the [`workflow:rules` keyword](../ci/yaml/README.md#workflowrules) to
define default rules to determine whether or not a pipeline is created.
These rules are defined in <https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab-ci.yml>
and are as follows:
1. If `$FORCE_GITLAB_CI` is set, create a pipeline.
1. For merge requests, create a pipeline.
1. For `master` branch, create a pipeline (this includes on schedules, pushes, merges, etc.).
1. For tags, create a pipeline.
1. If `$GITLAB_INTERNAL` isn't set, don't create a pipeline.
1. For stable, auto-deploy, and security branches, create a pipeline.
1. For any other cases (e.g. when pushing a branch with no MR for it), no pipeline is created.
### PostgreSQL versions testing
#### Current versions testing
| Where? | PostgreSQL version |
| ------ | ------ |
| MRs | 11 |
| `master` (non-scheduled pipelines) | 11 |
| 2-hourly scheduled pipelines | 11 |
#### Long-term plan
We follow the [PostgreSQL versions shipped with Omnibus GitLab](https://docs.gitlab.com/omnibus/package-information/postgresql_versions.html):
| PostgreSQL version | 12.10 (April 2020) | 13.0 (May 2020) | 13.1 (June 2020) | 13.2 (July 2020) | 13.3 (August 2020) | 13.4, 13.5 | 13.6 (November 2020) | 14.0 (May 2021?) |
| ------ | ------------------ | --------------- | ---------------- | ---------------- | ------------------ | ------------ | -------------------- | ---------------- |
| PG9.6 | MRs/`master`/`2-hour`/`nightly` | - | - | - | - | - | - | - |
| PG10 | `nightly` | - | - | - | - | - | - | - |
| PG11 | `master`/`2-hour` | MRs/`master`/`2-hour`/`nightly` | MRs/`master`/`2-hour`/`nightly` | MRs/`master`/`2-hour`/`nightly` | MRs/`master`/`2-hour`/`nightly` | MRs/`master`/`2-hour`/`nightly` | `nightly` | - |
| PG12 | - | - | - | - | `master`/`2-hour` | `master`/`2-hour` | MRs/`master`/`2-hour`/`nightly` | `master`/`2-hour` |
| PG13 | - | - | - | - | - | - | - | MRs/`master`/`2-hour`/`nightly` |
### Test jobs
Consult [GitLab tests in the Continuous Integration (CI) context](testing_guide/ci.md)
for more information.
## Review app jobs
### Review app jobs
Consult the [Review Apps](testing_guide/review_apps.md) dedicated page for more information.
## As-if-FOSS jobs
### As-if-FOSS jobs
The `* as-if-foss` jobs allows to run GitLab's test suite "as-if-FOSS", meaning as if the jobs would run in the context
of the `gitlab-org/gitlab-foss` project. These jobs are only created in the following cases:
......@@ -532,7 +424,39 @@ folders removed before the tests start running.
The intent is to ensure that a change won't introduce a failure once the `gitlab-org/gitlab` project will be synced to
the `gitlab-org/gitlab-foss` project.
## Pre-clone step
## Performance
### Interruptible pipelines
By default, all jobs are [interruptible](../ci/yaml/README.md#interruptible), except the
`dont-interrupt-me` job which runs automatically on `master`, and is `manual`
otherwise.
If you want a running pipeline to finish even if you push new commits to a merge
request, be sure to start the `dont-interrupt-me` job before pushing.
### Caching strategy
1. All jobs must only pull caches by default.
1. All jobs must be able to pass with an empty cache. In other words, caches are only there to speed up jobs.
1. We currently have 6 different caches defined in
[`.gitlab/ci/global.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab/ci/global.gitlab-ci.yml),
with fixed keys:
- `.rails-cache`.
- `.static-analysis-cache`.
- `.qa-cache`
- `.yarn-cache`.
- `.assets-compile-cache` (the key includes `${NODE_ENV}` so it's actually two different caches).
1. Only 6 specific jobs, running in 2-hourly scheduled pipelines, are pushing (i.e. updating) to the caches:
- `update-rails-cache`, defined in [`.gitlab/ci/rails.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab/ci/rails.gitlab-ci.yml).
- `update-static-analysis-cache`, defined in [`.gitlab/ci/rails.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab/ci/rails.gitlab-ci.yml).
- `update-qa-cache`, defined in [`.gitlab/ci/qa.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab/ci/qa.gitlab-ci.yml).
- `update-assets-compile-production-cache`, defined in [`.gitlab/ci/frontend.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab/ci/frontend.gitlab-ci.yml).
- `update-assets-compile-test-cache`, defined in [`.gitlab/ci/frontend.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab/ci/frontend.gitlab-ci.yml).
- `update-yarn-cache`, defined in [`.gitlab/ci/frontend.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab/ci/frontend.gitlab-ci.yml).
1. These jobs will run in merge requests whose title include `UPDATE CACHE`.
### Pre-clone step
The `gitlab-org/gitlab` project on GitLab.com uses a [pre-clone step](https://gitlab.com/gitlab-org/gitlab/-/issues/39134)
to seed the project with a recent archive of the repository. This is done for
......@@ -597,6 +521,124 @@ credentials are stored in the 1Password GitLab.com Production vault.
Note that this bucket should be located in the same continent as the
runner, or [network egress charges will apply](https://cloud.google.com/storage/pricing).
## CI configuration internals
### Stages
The current stages are:
- `sync`: This stage is used to synchronize changes from <https://gitlab.com/gitlab-org/gitlab> to
<https://gitlab.com/gitlab-org/gitlab-foss>.
- `prepare`: This stage includes jobs that prepare artifacts that are needed by
jobs in subsequent stages.
- `build-images`: This stage includes jobs that prepare Docker images
that are needed by jobs in subsequent stages or downstream pipelines.
- `fixtures`: This stage includes jobs that prepare fixtures needed by frontend tests.
- `test`: This stage includes most of the tests, DB/migration jobs, and static analysis jobs.
- `post-test`: This stage includes jobs that build reports or gather data from
the `test` stage's jobs (e.g. coverage, Knapsack metadata etc.).
- `review-prepare`: This stage includes a job that build the CNG images that are
later used by the (Helm) Review App deployment (see
[Review Apps](testing_guide/review_apps.md) for details).
- `review`: This stage includes jobs that deploy the GitLab and Docs Review Apps.
- `qa`: This stage includes jobs that perform QA tasks against the Review App
that is deployed in the previous stage.
- `post-qa`: This stage includes jobs that build reports or gather data from
the `qa` stage's jobs (e.g. Review App performance report).
- `pages`: This stage includes a job that deploys the various reports as
GitLab Pages (e.g. <https://gitlab-org.gitlab.io/gitlab/coverage-ruby/>,
<https://gitlab-org.gitlab.io/gitlab/coverage-javascript/>,
<https://gitlab-org.gitlab.io/gitlab/webpack-report/>).
### Default image
The default image is defined in <https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab-ci.yml>.
It includes Ruby, Go, Git, Git LFS, Chrome, Node, Yarn, PostgreSQL, and Graphics Magick.
The images used in our pipelines are configured in the
[`gitlab-org/gitlab-build-images`](https://gitlab.com/gitlab-org/gitlab-build-images)
project, which is push-mirrored to <https://dev.gitlab.org/gitlab/gitlab-build-images>
for redundancy.
The current version of the build images can be found in the
["Used by GitLab section"](https://gitlab.com/gitlab-org/gitlab-build-images/blob/master/.gitlab-ci.yml).
### Default variables
In addition to the [predefined variables](../ci/variables/predefined_variables.md),
each pipeline includes default variables defined in
<https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab-ci.yml>.
### Common job definitions
Most of the jobs [extend from a few CI definitions](../ci/yaml/README.md#extends)
defined in [`.gitlab/ci/global.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab/ci/global.gitlab-ci.yml)
that are scoped to a single [configuration parameter](../ci/yaml/README.md#configuration-parameters).
| Job definitions | Description |
|------------------|-------------|
| `.default-tags` | Ensures a job has the `gitlab-org` tag to ensure it's using our dedicated runners. |
| `.default-retry` | Allows a job to [retry](../ci/yaml/README.md#retry) upon `unknown_failure`, `api_failure`, `runner_system_failure`, `job_execution_timeout`, or `stuck_or_timeout_failure`. |
| `.default-before_script` | Allows a job to use a default `before_script` definition suitable for Ruby/Rails tasks that may need a database running (e.g. tests). |
| `.rails-cache` | Allows a job to use a default `cache` definition suitable for Ruby/Rails tasks. |
| `.static-analysis-cache` | Allows a job to use a default `cache` definition suitable for static analysis tasks. |
| `.yarn-cache` | Allows a job to use a default `cache` definition suitable for frontend jobs that do a `yarn install`. |
| `.assets-compile-cache` | Allows a job to use a default `cache` definition suitable for frontend jobs that compile assets. |
| `.use-pg11` | Allows a job to use the `postgres:11.6` and `redis:alpine` services. |
| `.use-pg11-ee` | Same as `.use-pg11` but also use the `docker.elastic.co/elasticsearch/elasticsearch:6.4.2` services. |
| `.use-kaniko` | Allows a job to use the `kaniko` tool to build Docker images. |
| `.as-if-foss` | Simulate the FOSS project by setting the `FOSS_ONLY='1'` environment variable. |
### `rules`, `if:` conditions and `changes:` patterns
We're using the [`rules` keyword](../ci/yaml/README.md#rules) extensively.
All `rules` definitions are defined in
<https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/rules.gitlab-ci.yml>,
then included in individual jobs via [`extends`](../ci/yaml/README.md#extends).
The `rules` definitions are composed of `if:` conditions and `changes:` patterns,
which are also defined in
<https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/rules.gitlab-ci.yml>
and included in `rules` definitions via [YAML anchors](../ci/yaml/README.md#anchors)
#### `if:` conditions
| `if:` conditions | Description | Notes |
|------------------|-------------|-------|
| `if-not-canonical-namespace` | Matches if the project isn't in the canonical (`gitlab-org/`) or security (`gitlab-org/security`) namespace. | Use to create a job for forks (by using `when: on_success\|manual`), or **not** create a job for forks (by using `when: never`). |
| `if-not-ee` | Matches if the project isn't EE (i.e. project name isn't `gitlab` or `gitlab-ee`). | Use to create a job only in the FOSS project (by using `when: on_success|manual`), or **not** create a job if the project is EE (by using `when: never`). |
| `if-not-foss` | Matches if the project isn't FOSS (i.e. project name isn't `gitlab-foss`, `gitlab-ce`, or `gitlabhq`). | Use to create a job only in the EE project (by using `when: on_success|manual`), or **not** create a job if the project is FOSS (by using `when: never`). |
| `if-default-refs` | Matches if the pipeline is for `master`, `/^[\d-]+-stable(-ee)?$/` (stable branches), `/^\d+-\d+-auto-deploy-\d+$/` (auto-deploy branches), `/^security\//` (security branches), merge requests, and tags. | Note that jobs won't be created for branches with this default configuration. |
| `if-master-refs` | Matches if the current branch is `master`. | |
| `if-master-or-tag` | Matches if the pipeline is for the `master` branch or for a tag. | |
| `if-merge-request` | Matches if the pipeline is for a merge request. | |
| `if-nightly-master-schedule` | Matches if the pipeline is for a `master` scheduled pipeline with `$NIGHTLY` set. | |
| `if-dot-com-gitlab-org-schedule` | Limits jobs creation to scheduled pipelines for the `gitlab-org` group on GitLab.com. | |
| `if-dot-com-gitlab-org-master` | Limits jobs creation to the `master` branch for the `gitlab-org` group on GitLab.com. | |
| `if-dot-com-gitlab-org-merge-request` | Limits jobs creation to merge requests for the `gitlab-org` group on GitLab.com. | |
| `if-dot-com-gitlab-org-and-security-tag` | Limits job creation to tags for the `gitlab-org` and `gitlab-org/security` groups on GitLab.com. | |
| `if-dot-com-gitlab-org-and-security-merge-request` | Limit jobs creation to merge requests for the `gitlab-org` and `gitlab-org/security` groups on GitLab.com. | |
| `if-dot-com-ee-schedule` | Limits jobs to scheduled pipelines for the `gitlab-org/gitlab` project on GitLab.com. | |
| `if-cache-credentials-schedule` | Limits jobs to scheduled pipelines with the `$CI_REPO_CACHE_CREDENTIALS` variable set. | |
#### `changes:` patterns
| `changes:` patterns | Description |
|------------------------------|--------------------------------------------------------------------------|
| `ci-patterns` | Only create job for CI config-related changes. |
| `yaml-patterns` | Only create job for YAML-related changes. |
| `docs-patterns` | Only create job for docs-related changes. |
| `frontend-dependency-patterns` | Only create job when frontend dependencies are updated (i.e. `package.json`, and `yarn.lock`). changes. |
| `frontend-patterns` | Only create job for frontend-related changes. |
| `backstage-patterns` | Only create job for backstage-related changes (i.e. Danger, fixtures, RuboCop, specs). |
| `code-patterns` | Only create job for code-related changes. |
| `qa-patterns` | Only create job for QA-related changes. |
| `code-backstage-patterns` | Combination of `code-patterns` and `backstage-patterns`. |
| `code-qa-patterns` | Combination of `code-patterns` and `qa-patterns`. |
| `code-backstage-qa-patterns` | Combination of `code-patterns`, `backstage-patterns`, and `qa-patterns`. |
---
[Return to Development documentation](README.md)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7