[EE] Use merge request pipelines with change detection

Signed-off-by: Rémy Coutable <remy@rymai.me>

.gitlab-ci.yml

 image: "dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.6.3-golang-1.11-git-2.22-chrome-73.0-node-12.x-yarn-1.16-postgresql-9.6-graphicsmagick-1.3.33"
 
 stages:
-  - build
   - prepare
   - quick-test
   - test

.gitlab/ci/cng.gitlab-ci.yml

@@ -11,5 +11,6 @@ cloud-native-image:
     - CNG_PROJECT_PATH="gitlab-org/build/CNG" BUILD_TRIGGER_TOKEN=$CI_JOB_TOKEN ./scripts/trigger-build cng
   only:
     refs:
-      - tags@gitlab-org/gitlab-foss
-      - tags@gitlab-org/gitlab
+      - tags
+    variables:
+      - $CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE == "gitlab-org"

.gitlab/ci/docs.gitlab-ci.yml

@@ -2,6 +2,12 @@
   extends:
     - .default-tags
     - .default-retry
+    - .only-docs-changes
+  only:
+    refs:
+      - merge_requests
+    variables:
+      - $CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE == "gitlab-org"
   image: ruby:2.6-alpine
   stage: review
   dependencies: []
@@ -19,55 +25,32 @@
     - apk add --update openssl
     - wget $CI_PROJECT_URL/raw/$CI_COMMIT_SHA/scripts/trigger-build-docs
     - chmod 755 trigger-build-docs
-
-# Trigger a manual docs build in gitlab-docs only on non docs-only branches.
-# Useful to preview the docs changes live.
-review-docs-deploy-manual:
-  extends:
-    - .review-docs
-    - .except-docs-qa
-  script:
     - gem install gitlab --no-document
-    - ./trigger-build-docs deploy
-  when: manual
-  only:
-    - branches@gitlab-org/gitlab-foss
-    - branches@gitlab-org/gitlab
 
 # Always trigger a docs build in gitlab-docs only on docs-only branches.
 # Useful to preview the docs changes live.
 review-docs-deploy:
-  extends:
-    - .review-docs
-    - .except-qa
+  extends: .review-docs
   script:
-    - gem install gitlab --no-document
     - ./trigger-build-docs deploy
-  only:
-    - /(^docs[\/-].+|.+-docs$)/@gitlab-org/gitlab-foss
-    - /(^docs[\/-].+|.+-docs$)/@gitlab-org/gitlab
+  when: manual
 
 # Cleanup remote environment of gitlab-docs
 review-docs-cleanup:
-  extends:
-    - .review-docs
-    - .except-qa
+  extends: .review-docs
   environment:
     name: review-docs/$CI_COMMIT_REF_SLUG
     action: stop
   script:
-    - gem install gitlab --no-document
     - ./trigger-build-docs cleanup
   when: manual
-  only:
-    - branches@gitlab-org/gitlab-foss
-    - branches@gitlab-org/gitlab
 
 docs lint:
   extends:
     - .default-tags
     - .default-retry
-    - .except-qa
+    - .default-only
+    - .only-docs-changes
   image: "registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-docs-lint"
   stage: test
   dependencies: []

.gitlab/ci/frontend.gitlab-ci.yml

@@ -9,9 +9,10 @@
   extends:
     - .default-tags
     - .default-retry
-    - .assets-compile-cache
+    - .default-only
     - .default-before_script
-    - .except-docs
+    - .assets-compile-cache
+    - .only-code-qa-changes
   image: dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.6.3-git-2.22-chrome-73.0-node-12.x-yarn-1.16-graphicsmagick-1.3.33-docker-18.06.1
   stage: test
   dependencies: ["setup-test-env"]
@@ -45,10 +46,9 @@
     - scripts/clean-old-cached-assets
     - rm -f /etc/apt/sources.list.d/google*.list  # We don't need to update Chrome here
   only:
-    - /.+/@gitlab-org/gitlab-foss
-    - /.+/@gitlab-org/gitlab
-    - /.+/@gitlab/gitlabhq
-    - /.+/@gitlab/gitlab-ee
+    variables:
+      - $CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE == "gitlab-org"
+      - $CI_SERVER_HOST == "dev.gitlab.org"
   tags:
     - gitlab-org
     - docker
@@ -57,8 +57,7 @@ gitlab:assets:compile:
   extends: .gitlab:assets:compile-metadata
   only:
     refs:
-      - master@gitlab-org/gitlab-foss
-      - master@gitlab-org/gitlab
+      - master
   cache:
     policy: pull-push
 
@@ -66,9 +65,7 @@ gitlab:assets:compile pull-cache:
   extends: .gitlab:assets:compile-metadata
   except:
     refs:
-      - master@gitlab-org/gitlab-foss
-      - master@gitlab-org/gitlab
-      - /(^docs[\/-].+|.+-docs$)/
+      - master
   cache:
     policy: pull
 
@@ -76,8 +73,10 @@ gitlab:assets:compile pull-cache:
   extends:
     - .default-tags
     - .default-retry
-    - .assets-compile-cache
+    - .default-only
     - .default-before_script
+    - .assets-compile-cache
+    - .only-code-qa-changes
     - .use-pg
   stage: prepare
   script:
@@ -98,12 +97,10 @@ gitlab:assets:compile pull-cache:
       - public/assets
 
 compile-assets:
-  extends:
-    - .compile-assets-metadata
+  extends: .compile-assets-metadata
   only:
     refs:
-      - master@gitlab-org/gitlab-foss
-      - master@gitlab-org/gitlab
+      - master
   cache:
     policy: pull-push
 
@@ -111,21 +108,23 @@ compile-assets pull-cache:
   extends: .compile-assets-metadata
   except:
     refs:
-      - master@gitlab-org/gitlab-foss
-      - master@gitlab-org/gitlab
-      - /(^docs[\/-].+|.+-docs$)/
+      - master
   cache:
     policy: pull
 
-karma:
+.only-code-frontend-job-base:
   extends:
     - .default-tags
     - .default-retry
     - .default-cache
+    - .default-only
     - .default-before_script
+    - .only-code-changes
     - .use-pg
-    - .except-docs
   dependencies: ["compile-assets", "compile-assets pull-cache", "setup-test-env"]
+
+karma:
+  extends: .only-code-frontend-job-base
   variables:
     # we override the max_old_space_size to prevent OOM errors
     NODE_OPTIONS: --max_old_space_size=3584
@@ -148,14 +147,7 @@ karma:
       junit: junit_karma.xml
 
 jest:
-  extends:
-    - .default-tags
-    - .default-retry
-    - .default-cache
-    - .default-before_script
-    - .use-pg
-    - .except-docs-qa
-  dependencies: ["compile-assets", "compile-assets pull-cache", "setup-test-env"]
+  extends: .only-code-frontend-job-base
   script:
     - scripts/gitaly-test-spawn
     - date
@@ -178,27 +170,26 @@ jest:
       - tmp/jest/jest/
     policy: pull-push
 
-.qa:
+.qa-job-base:
   extends:
     - .default-tags
     - .default-retry
     - .default-cache
-    - .except-docs
+    - .default-only
+    - .only-code-qa-changes
   dependencies: []
   stage: test
-  variables:
-    SETUP_DB: "false"
   before_script:
     - cd qa/
     - bundle install
 
 qa:internal:
-  extends: .qa
+  extends: .qa-job-base
   script:
     - bundle exec rspec
 
 qa:selectors:
-  extends: .qa
+  extends: .qa-job-base
   script:
     - bundle exec bin/qa Test::Sanity::Selectors
 
@@ -207,7 +198,8 @@ qa:selectors:
     - .default-tags
     - .default-retry
     - .default-cache
-    - .except-docs
+    - .default-only
+    - .only-code-changes
   dependencies: []
   cache:
     key: "$CI_JOB_NAME"
@@ -238,10 +230,9 @@ webpack-dev-server:
     - .default-tags
     - .default-retry
     - .default-cache
-    - .except-docs-qa
-  dependencies: ["compile-assets", "compile-assets pull-cache", "setup-test-env"]
+    - .only-code-changes
+  dependencies: ["setup-test-env", "compile-assets", "compile-assets pull-cache"]
   variables:
-    SETUP_DB: "false"
     WEBPACK_MEMORY_TEST: "true"
   script:
     - node --version

.gitlab/ci/global.gitlab-ci.yml

@@ -27,40 +27,83 @@
       - vendor/gitaly-ruby
     policy: pull
 
-.except-docs:
-  except:
+.default-only:
+  only:
     refs:
-      - /(^docs[\/-].+|.+-docs$)/
+      - master
+      - /^[\d-]+-stable(-ee)?$/
+      - /^\d+-\d+-auto-deploy-\d+$/
+      - merge_requests
+      - tags
 
-.except-qa:
-  except:
-    refs:
-      - /(^qa[\/-].*|.*-qa$)/
+.only-code-changes:
+  only:
+    changes:
+      - ".gitlab/ci/**/*"
+      - ".{eslintignore,gitattributes,nvmrc,prettierrc,stylelintrc,yamllint}"
+      - ".{codeclimate,eslintrc,gitlab-ci,haml-lint,haml-lint_todo,rubocop,rubocop_todo,scss-lint}.yml"
+      - ".csscomb.json"
+      - "Dangerfile"
+      - "Dockerfile.assets"
+      - "*_VERSION"
+      - "Gemfile{,.lock}"
+      - "Rakefile"
+      - "{babel.config,jest.config}.js"
+      - "config.ru"
+      - "{package.json,yarn.lock}"
+      - "{app,bin,config,danger,db,ee,fixtures,haml_lint,lib,public,rubocop,scripts,spec,symbol,vendor}/**/*"
+      - "doc/README.md"  # Some RSpec test rely on this file
 
-.except-docs-qa:
-  except:
-    refs:
-      - /(^docs[\/-].+|.+-docs$)/
-      - /(^qa[\/-].*|.*-qa$)/
+.only-qa-changes:
+  only:
+    changes:
+      - ".dockerignore"
+      - "qa/**/*"
 
-.except-docs-qa-geo:
-  except:
-    refs:
-      - /(^docs[\/-].+|.+-docs$)/
-      - /(^qa[\/-].*|.*-qa$)/
-      - /(^geo[\/-].*|.*-geo$)/
+.only-docs-changes:
+  only:
+    changes:
+      - ".gitlab/route-map.yml"
+      - "doc/**/*"
+      - ".markdownlint.json"
 
-.review-only:
+.only-code-qa-changes:
   only:
-    refs:
-      - branches@gitlab-org/gitlab-foss
-      - branches@gitlab-org/gitlab
+    changes:
+      - ".gitlab/ci/**/*"
+      - ".{eslintignore,gitattributes,nvmrc,prettierrc,stylelintrc,yamllint}"
+      - ".{codeclimate,eslintrc,gitlab-ci,haml-lint,haml-lint_todo,rubocop,rubocop_todo,scss-lint}.yml"
+      - ".csscomb.json"
+      - "Dangerfile"
+      - "Dockerfile.assets"
+      - "*_VERSION"
+      - "Gemfile{,.lock}"
+      - "Rakefile"
+      - "{babel.config,jest.config}.js"
+      - "config.ru"
+      - "{package.json,yarn.lock}"
+      - "{app,bin,config,danger,db,ee,fixtures,haml_lint,lib,public,rubocop,scripts,spec,symbol,vendor}/**/*"
+      - "doc/README.md"  # Some RSpec test rely on this file
+      - ".dockerignore"
+      - "qa/**/*"
+
+.only-review:
+  only:
+    variables:
+      - $CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE == "gitlab-org"
     kubernetes: active
   except:
     refs:
       - master
       - /^\d+-\d+-auto-deploy-\d+$/
-      - /(^docs[\/-].+|.+-docs$)/
+
+.only-review-schedules:
+  only:
+    refs:
+      - schedules
+    variables:
+      - $REVIEW_APP_CLEANUP && $CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE == "gitlab-org"
+    kubernetes: active
 
 .use-pg:
   services:
@@ -74,3 +117,9 @@
     - name: postgres:10.9
       command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
     - name: redis:alpine
+
+.only-ee:
+  only:
+    variables:
+      - $CI_PROJECT_NAME == "gitlab-ee"
+      - $CI_PROJECT_NAME == "gitlab"  # New name of gitlab-ee after the single codebase migration

.gitlab/ci/memory.gitlab-ci.yml

-memory-static:
+.only-code-memory-job-base:
   extends:
     - .default-tags
     - .default-retry
     - .default-cache
+    - .default-only
     - .default-before_script
-    - .except-docs
+    - .only-code-changes
+
+memory-static:
+  extends: .only-code-memory-job-base
   variables:
     SETUP_DB: "false"
   script:
@@ -31,12 +35,8 @@ memory-static:
 # All tests are run without a webserver (directly using Rack::Mock by default).
 memory-on-boot:
   extends:
-    - .default-tags
-    - .default-retry
-    - .default-cache
-    - .default-before_script
+    - .only-code-memory-job-base
     - .use-pg-10
-    - .except-docs-qa
   variables:
     NODE_ENV: "production"
     RAILS_ENV: "production"

.gitlab/ci/pages.gitlab-ci.yml

@@ -3,11 +3,12 @@ pages:
     - .default-tags
     - .default-retry
     - .default-cache
-    - .except-docs
+    - .default-only
   only:
     refs:
-      - master@gitlab-org/gitlab-foss
-      - master@gitlab-org/gitlab
+      - master
+    variables:
+      - $CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE == "gitlab-org"
   stage: pages
   dependencies: ["coverage", "karma", "gitlab:assets:compile"]
   script:

.gitlab/ci/qa.gitlab-ci.yml

 .package-and-qa-base:
+  extends: .default-only
   image: ruby:2.6-alpine
   stage: qa
   dependencies: []
@@ -10,17 +11,16 @@
     - install_gitlab_gem
     - ./scripts/trigger-build omnibus
   only:
-    refs:
-      - branches@gitlab-org/gitlab-foss
-      - branches@gitlab-org/gitlab
+    variables:
+      - $CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE == "gitlab-org"
 
 package-and-qa-manual:
-  extends: .package-and-qa-base
+  extends:
+    - .package-and-qa-base
+    - .only-code-changes
   except:
     refs:
       - master
-      - /(^docs[\/-].+|.+-docs$)/
-      - /(^qa[\/-].*|.*-qa$)/
   when: manual
   needs: ["build-qa-image", "gitlab:assets:compile pull-cache"]
 
@@ -34,10 +34,11 @@ package-and-qa-manual:master:
   needs: ["build-qa-image", "gitlab:assets:compile"]
 
 package-and-qa:
-  extends: .package-and-qa-base
-  only:
+  extends:
+    - .package-and-qa-base
+    - .only-qa-changes
+  except:
     refs:
-      - /(^qa[\/-].*|.*-qa$)/@gitlab-org/gitlab-foss
-      - /(^qa[\/-].*|.*-qa$)/@gitlab-org/gitlab
+      - master
   needs: ["build-qa-image", "gitlab:assets:compile pull-cache"]
   allow_failure: true

.gitlab/ci/rails.gitlab-ci.yml

-.only-schedules-master:
+.only-master:
   only:
     refs:
-      - schedules@gitlab-org/gitlab-foss
-      - schedules@gitlab-org/gitlab
-      - master@gitlab-org/gitlab-foss
-      - master@gitlab-org/gitlab
-      - master@gitlab/gitlabhq
-      - master@gitlab/gitlab-ee
-
-.only-gitlab-ee:
-  only:
-    - branches@gitlab-org/gitlab
-    - tags@gitlab-org/gitlab
+      - master
 
 .rake-exec:
   extends:
     - .default-tags
     - .default-retry
     - .default-cache
+    - .default-only
     - .default-before_script
   variables:
     SETUP_DB: "false"
   script:
     - bundle exec rake $CI_JOB_NAME
 
-.rspec-base:
+.only-code-rails-job-base:
+  extends:
+    - .default-tags
+    - .default-retry
+    - .default-cache
+    - .default-only
+    - .default-before_script
+    - .only-code-changes
+
+.only-code-qa-rails-job-base:
   extends:
     - .default-tags
     - .default-retry
     - .default-cache
+    - .default-only
     - .default-before_script
-    - .except-docs-qa
+    - .only-code-qa-changes
+
+.rspec-base:
+  extends: .only-code-rails-job-base
   stage: test
   script:
     - JOB_NAME=( $CI_JOB_NAME )
@@ -79,12 +83,8 @@
 
 setup-test-env:
   extends:
-    - .default-tags
-    - .default-retry
-    - .default-cache
-    - .default-before_script
+    - .only-code-qa-rails-job-base
     - .use-pg
-    - .except-docs
   stage: prepare
   script:
     - bundle exec ruby -Ispec -e 'require "spec_helper" ; TestEnv.init'
@@ -111,19 +111,19 @@ rspec system pg:
 rspec unit pg-10:
   extends:
     - .rspec-base-pg-10
-    - .only-schedules-master
+    - .only-master
   parallel: 20
 
 rspec integration pg-10:
   extends:
     - .rspec-base-pg-10
-    - .only-schedules-master
+    - .only-master
   parallel: 6
 
 rspec system pg-10:
   extends:
     - .rspec-base-pg-10
-    - .only-schedules-master
+    - .only-master
   parallel: 24
 
 rspec-fast-spec-helper:
@@ -133,9 +133,8 @@ rspec-fast-spec-helper:
 
 rspec quarantine pg:
   extends:
-    - .default-before_script
     - .rspec-base-pg
-    - .only-schedules-master
+    - .only-master
   script:
     - export NO_KNAPSACK=1 CACHE_CLASSES=true
     - scripts/gitaly-test-spawn
@@ -143,12 +142,7 @@ rspec quarantine pg:
   allow_failure: true
 
 static-analysis:
-  extends:
-    - .default-tags
-    - .default-retry
-    - .default-cache
-    - .default-before_script
-    - .except-docs
+  extends: .only-code-qa-rails-job-base
   dependencies: ["setup-test-env", "compile-assets", "compile-assets pull-cache"]
   variables:
     SETUP_DB: "false"
@@ -162,81 +156,48 @@ static-analysis:
     policy: pull-push
 
 downtime_check:
-  extends: .rake-exec
+  extends:
+    - .rake-exec
+    - .only-code-changes
   except:
     refs:
       - master
       - tags
-      - /^[\d-]+-stable(-ee)?$/
-      - /(^docs[\/-].+|.+-docs$)/
-      - /(^qa[\/-].*|.*-qa$)/
+    variables:
+      - $CI_COMMIT_REF_NAME =~ /^[\d-]+-stable(-ee)?$/
   stage: test
   dependencies: ["setup-test-env"]
   needs: ["setup-test-env"]
 
-ee_compat_check:
-  extends: .rake-exec
-  dependencies: []
-  except:
-    refs:
-      - master
-      - tags
-      - branches@gitlab-org/gitlab
-      - branches@gitlab/gitlab-ee
-      - /^[\d-]+-stable(-ee)?$/
-      - /(^docs[\/-].+|.+-docs$)/
-      - /^security-/
-  artifacts:
-    name: "${CI_JOB_NAME}_${CI_COMIT_REF_NAME}_${CI_COMMIT_SHA}"
-    when: always
-    expire_in: 10d
-    paths:
-      - ee_compat_check/patches/*.patch
-
-# DB migration, rollback, and seed jobs
-db:migrate:reset:
+.db-job-base:
   extends:
-    - .default-tags
-    - .default-retry
-    - .default-cache
-    - .default-before_script
+    - .only-code-rails-job-base
     - .use-pg
-    - .except-docs-qa
   stage: test
   dependencies: ["setup-test-env"]
   needs: ["setup-test-env"]
+
+# DB migration, rollback, and seed jobs
+db:migrate:reset:
+  extends: .db-job-base
   script:
     - bundle exec rake db:migrate:reset
 
 db:check-schema:
-  extends:
-    - .default-tags
-    - .default-retry
-    - .default-cache
-    - .default-before_script
-    - .use-pg
-    - .except-docs-qa
-  stage: test
-  dependencies: ["setup-test-env"]
-  needs: ["setup-test-env"]
+  extends: .db-job-base
   script:
     - source scripts/schema_changed.sh
 
 db:migrate-from-v11.11.0:
-  extends:
-    - .default-tags
-    - .default-retry
-    - .default-cache
-    - .default-before_script
-    - .use-pg
-    - .except-docs-qa
-  stage: test
-  dependencies: ["setup-test-env"]
-  needs: ["setup-test-env"]
+  extends: .db-job-base
   variables:
     SETUP_DB: "false"
   script:
-    - git fetch https://gitlab.com/gitlab-org/gitlab.git v11.11.0-ee
+    - export PROJECT_TO_CHECKOUT="gitlab-foss"
+    - export TAG_TO_CHECKOUT="v11.11.0"
+    - '[[ ! -d "ee/" ]] || export PROJECT_TO_CHECKOUT="gitlab"'
+    - '[[ ! -d "ee/" ]] || export TAG_TO_CHECKOUT="v11.11.0-ee"'
+    - git fetch https://gitlab.com/gitlab-org/$PROJECT_TO_CHECKOUT.git $TAG_TO_CHECKOUT
     - git checkout -f FETCH_HEAD
     - sed -i "s/gem 'oj', '~> 2.17.4'//" Gemfile
     - sed -i "s/gem 'bootsnap', '~> 1.0.0'/gem 'bootsnap'/" Gemfile
@@ -254,31 +215,13 @@ db:migrate-from-v11.11.0:
     - bundle exec rake db:migrate
 
 db:rollback:
-  extends:
-    - .default-tags
-    - .default-retry
-    - .default-cache
-    - .default-before_script
-    - .use-pg
-    - .except-docs-qa
-  stage: test
-  dependencies: ["setup-test-env"]
-  needs: ["setup-test-env"]
+  extends: .db-job-base
   script:
     - bundle exec rake db:migrate VERSION=20180101160629
     - bundle exec rake db:migrate SKIP_SCHEMA_VERSION_CHECK=true
 
 gitlab:setup:
-  extends:
-    - .default-tags
-    - .default-retry
-    - .default-cache
-    - .default-before_script
-    - .use-pg
-    - .except-docs-qa
-  stage: test
-  dependencies: ["setup-test-env"]
-  needs: ["setup-test-env"]
+  extends: .db-job-base
   variables:
     SETUP_DB: "false"
   script:
@@ -295,14 +238,7 @@ gitlab:setup:
       - log/development.log
 
 coverage:
-  # Don't include dedicated-no-docs-no-db-pull-cache-job here since we need to
-  # download artifacts from all the rspec jobs instead of from setup-test-env only
-  extends:
-    - .default-tags
-    - .default-retry
-    - .default-cache
-    - .default-before_script
-    - .except-docs-qa
+  extends: .only-code-rails-job-base
   cache:
     policy: pull
   variables:
@@ -324,7 +260,7 @@ coverage:
 .rspec-base-ee:
   extends:
     - .rspec-base
-    - .only-gitlab-ee
+    - .only-ee
   script:
     - JOB_NAME=( $CI_JOB_NAME )
     - TEST_TOOL=${JOB_NAME[0]}
@@ -362,10 +298,9 @@ rspec system pg ee:
   extends: .rspec-base-pg-ee
   parallel: 5
 
-.rspec-base-pg-geo:
-  extends:
-    - .rspec-base
-    - .only-gitlab-ee
+.rspec-base-geo:
+  extends: .rspec-base-ee
+  parallel: 3
   script:
     - JOB_NAME=( $CI_JOB_NAME )
     - TEST_TOOL=${JOB_NAME[0]}
@@ -382,33 +317,37 @@ rspec system pg ee:
 
 rspec geo pg ee:
   extends:
-    - .rspec-base-pg-geo
+    - .rspec-base-geo
     - .use-pg
-    - .except-docs-qa-geo
-  parallel: 3
+  except:
+    variables:
+      - $CI_COMMIT_REF_NAME =~ /(^geo[\/-].*|.*-geo$)/
 
 rspec geo pg-10 ee:
   extends:
-    - .rspec-base-pg-geo
+    - .rspec-base-geo
     - .use-pg-10
-    - .except-docs-qa-geo
-  parallel: 3
+  except:
+    variables:
+      - $CI_COMMIT_REF_NAME =~ /(^geo[\/-].*|.*-geo$)/
 
 quick-rspec geo pg ee:
   extends:
-    - .rspec-base-pg-geo
+    - .rspec-base-geo
     - .use-pg
   stage: quick-test
   only:
-    - /(^geo[\/-].*|.*-geo$)/
+    variables:
+      - $CI_COMMIT_REF_NAME =~ /(^geo[\/-].*|.*-geo$)/
 
 quick-rspec geo pg-10 ee:
   extends:
-    - .rspec-base-pg-geo
+    - .rspec-base-geo
     - .use-pg-10
   stage: quick-test
   only:
-    - /(^geo[\/-].*|.*-geo$)/
+    variables:
+      - $CI_COMMIT_REF_NAME =~ /(^geo[\/-].*|.*-geo$)/
 
 rspec quarantine pg ee:
   extends: rspec quarantine pg
@@ -417,35 +356,10 @@ rspec quarantine pg ee:
     - scripts/gitaly-test-spawn
     - bin/rspec --color --format documentation --format RspecJunitFormatter --out junit_rspec.xml --tag quarantine -- ee/spec/
 
-migration:upgrade-pg-ce-to-ee:
-  extends:
-    - .default-tags
-    - .default-retry
-    - .default-cache
-    - .default-before_script
-    - .use-pg
-    - .except-docs-qa
-  dependencies: ["setup-test-env"]
-  variables:
-    SETUP_DB: "false"
-  script:
-    - ruby -r./scripts/ee_specific_check/ee_specific_check -e'EESpecificCheck.fetch_remote_ce_branch'
-    - git checkout -f FETCH_HEAD
-    - . scripts/utils.sh
-    - . scripts/prepare_build.sh
-    - date
-    - setup_db
-    - date
-    - git checkout -f $CI_COMMIT_SHA
-    - date
-    - . scripts/prepare_build.sh
-    - date
-    - bundle exec rake db:migrate
-
 db:rollback geo:
   extends:
     - db:rollback
-    - .only-gitlab-ee
+    - .only-ee
   script:
     - bundle exec rake geo:db:migrate VERSION=20170627195211
     - bundle exec rake geo:db:migrate

.gitlab/ci/reports.gitlab-ci.yml

-include:
-  - template: Code-Quality.gitlab-ci.yml
-  - template: Security/SAST.gitlab-ci.yml
-  - template: Security/Dependency-Scanning.gitlab-ci.yml
-  - template: Security/DAST.gitlab-ci.yml
+# include:
+#   - template: Code-Quality.gitlab-ci.yml
+#   - template: Security/SAST.gitlab-ci.yml
+#   - template: Security/Dependency-Scanning.gitlab-ci.yml
+#   - template: Security/DAST.gitlab-ci.yml
 
-.reports:
+# We need to duplicate this job's definition because it seems it's impossible to
+# override an included `only.refs`.
+# See https://gitlab.com/gitlab-org/gitlab/issues/31371.
+code_quality:
   extends:
     - .default-retry
-    - .except-docs
-
-code_quality:
-  extends: .reports
+    - .default-only
+    - .only-code-changes
+  stage: test
+  image: docker:stable
+  allow_failure: true
+  services:
+    - docker:stable-dind
+  variables:
+    DOCKER_DRIVER: overlay2
+    DOCKER_TLS_CERTDIR: ""
+  script:
+    - |
+      if ! docker info &>/dev/null; then
+        if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then
+          export DOCKER_HOST='tcp://localhost:2375'
+        fi
+      fi
+    - docker run
+        --env SOURCE_CODE="$PWD"
+        --volume "$PWD":/code
+        --volume /var/run/docker.sock:/var/run/docker.sock
+        "registry.gitlab.com/gitlab-org/security-products/codequality:12-0-stable" /code
+  artifacts:
+    reports:
+      codequality: gl-code-quality-report.json
+    expire_in: 1 week
+  dependencies: []
+  except:
+    variables:
+      - $CODE_QUALITY_DISABLED
 
+# We need to duplicate this job's definition because it seems it's impossible to
+# override an included `only.refs`.
+# See https://gitlab.com/gitlab-org/gitlab/issues/31371.
 sast:
-  extends: .reports
+  extends:
+    - .default-retry
+    - .default-only
+    - .only-code-changes
+  stage: test
+  image: docker:stable
   variables:
+    DOCKER_DRIVER: overlay2
+    DOCKER_TLS_CERTDIR: ""
     SAST_BRAKEMAN_LEVEL: 2
-    SAST_EXCLUDED_PATHS: qa,spec,doc
+    SAST_EXCLUDED_PATHS: qa,spec,doc,ee/spec
+  allow_failure: true
+  services:
+    - docker:stable-dind
+  script:
+    - export SAST_VERSION=${SP_VERSION:-$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')}
+    - |
+      if ! docker info &>/dev/null; then
+        if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then
+          export DOCKER_HOST='tcp://localhost:2375'
+        fi
+      fi
+    - | # this is required to avoid undesirable reset of Docker image ENV variables being set on build stage
+      function propagate_env_vars() {
+        CURRENT_ENV=$(printenv)
+
+        for VAR_NAME; do
+          echo $CURRENT_ENV | grep "${VAR_NAME}=" > /dev/null && echo "--env $VAR_NAME "
+        done
+      }
+    - |
+      docker run \
+        $(propagate_env_vars \
+          SAST_BANDIT_EXCLUDED_PATHS \
+          SAST_ANALYZER_IMAGES \
+          SAST_ANALYZER_IMAGE_PREFIX \
+          SAST_ANALYZER_IMAGE_TAG \
+          SAST_DEFAULT_ANALYZERS \
+          SAST_PULL_ANALYZER_IMAGES \
+          SAST_BRAKEMAN_LEVEL \
+          SAST_FLAWFINDER_LEVEL \
+          SAST_GITLEAKS_ENTROPY_LEVEL \
+          SAST_GOSEC_LEVEL \
+          SAST_EXCLUDED_PATHS \
+          SAST_DOCKER_CLIENT_NEGOTIATION_TIMEOUT \
+          SAST_PULL_ANALYZER_IMAGE_TIMEOUT \
+          SAST_RUN_ANALYZER_TIMEOUT \
+          SAST_JAVA_VERSION \
+          ANT_HOME \
+          ANT_PATH \
+          GRADLE_PATH \
+          JAVA_OPTS \
+          JAVA_PATH \
+          JAVA_8_VERSION \
+          JAVA_11_VERSION \
+          MAVEN_CLI_OPTS \
+          MAVEN_PATH \
+          MAVEN_REPO_PATH \
+          SBT_PATH \
+          FAIL_NEVER \
+        ) \
+        --volume "$PWD:/code" \
+        --volume /var/run/docker.sock:/var/run/docker.sock \
+        "registry.gitlab.com/gitlab-org/security-products/sast:$SAST_VERSION" /app/bin/run /code
   artifacts:
     expire_in: 7 days
     paths:
       - gl-sast-report.json
+    reports:
+      sast: gl-sast-report.json
+  dependencies: []
+  only:
+    variables:
+      - $GITLAB_FEATURES =~ /\bsast\b/
+  except:
+    variables:
+      - $SAST_DISABLED
 
+# We need to duplicate this job's definition because it seems it's impossible to
+# override an included `only.refs`.
+# See https://gitlab.com/gitlab-org/gitlab/issues/31371.
 dependency_scanning:
-  extends: .reports
+  extends:
+    - .default-retry
+    - .default-only
+    - .only-code-changes
+  stage: test
+  image: docker:stable
+  variables:
+    DOCKER_DRIVER: overlay2
+    DOCKER_TLS_CERTDIR: ""
+  allow_failure: true
+  services:
+    - docker:stable-dind
+  script:
+    - export DS_VERSION=${SP_VERSION:-$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')}
+    - |
+      if ! docker info &>/dev/null; then
+        if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then
+          export DOCKER_HOST='tcp://localhost:2375'
+        fi
+      fi
+    - | # this is required to avoid undesirable reset of Docker image ENV variables being set on build stage
+      function propagate_env_vars() {
+        CURRENT_ENV=$(printenv)
 
+        for VAR_NAME; do
+          echo $CURRENT_ENV | grep "${VAR_NAME}=" > /dev/null && echo "--env $VAR_NAME "
+        done
+      }
+    - |
+      docker run \
+        $(propagate_env_vars \
+          DS_ANALYZER_IMAGES \
+          DS_ANALYZER_IMAGE_PREFIX \
+          DS_ANALYZER_IMAGE_TAG \
+          DS_DEFAULT_ANALYZERS \
+          DS_EXCLUDED_PATHS \
+          DEP_SCAN_DISABLE_REMOTE_CHECKS \
+          DS_DOCKER_CLIENT_NEGOTIATION_TIMEOUT \
+          DS_PULL_ANALYZER_IMAGE_TIMEOUT \
+          DS_RUN_ANALYZER_TIMEOUT \
+          DS_PYTHON_VERSION \
+          DS_PIP_DEPENDENCY_PATH \
+          PIP_INDEX_URL \
+          PIP_EXTRA_INDEX_URL \
+        ) \
+        --volume "$PWD:/code" \
+        --volume /var/run/docker.sock:/var/run/docker.sock \
+        "registry.gitlab.com/gitlab-org/security-products/dependency-scanning:$DS_VERSION" /code
+  artifacts:
+    reports:
+      dependency_scanning: gl-dependency-scanning-report.json
+  dependencies: []
+  only:
+    variables:
+      - $GITLAB_FEATURES =~ /\bdependency_scanning\b/
+  except:
+    variables:
+      - $DEPENDENCY_SCANNING_DISABLED
+
+# We need to duplicate this job's definition because it seems it's impossible to
+# override an included `only.refs`.
+# See https://gitlab.com/gitlab-org/gitlab/issues/31371.
 dast:
   extends:
-    - .reports
-    - .review-only
+    - .default-retry
+    - .default-only
+    - .only-code-qa-changes
+    - .only-review
   stage: qa
   dependencies: ["review-deploy"]
   before_script:
     - export DAST_WEBSITE="$(cat review_app_url.txt)"
+  image:
+    name: "registry.gitlab.com/gitlab-org/security-products/dast:$CI_SERVER_VERSION_MAJOR-$CI_SERVER_VERSION_MINOR-stable"
+  variables:
+  # URL to scan:
+  # DAST_WEBSITE: https://example.com/
+  #
+  # Time limit for target availability (scan is attempted even when timeout):
+  # DAST_TARGET_AVAILABILITY_TIMEOUT: 60
+  #
+  # Set these variables to scan with an authenticated user:
+  # DAST_AUTH_URL: https://example.com/sign-in
+  # DAST_USERNAME: john.doe@example.com
+  # DAST_PASSWORD: john-doe-password
+  # DAST_USERNAME_FIELD: session[user] # the name of username field at the sign-in HTML form
+  # DAST_PASSWORD_FIELD: session[password] # the name of password field at the sign-in HTML form
+  # DAST_AUTH_EXCLUDE_URLS: http://example.com/sign-out,http://example.com/sign-out-2 # optional: URLs to skip during the authenticated scan; comma-separated, no spaces in between
+  #
+  # Perform ZAP Full Scan, which includes both passive and active scanning:
+  # DAST_FULL_SCAN_ENABLED: "true"
+  allow_failure: true
+  script:
+    - export DAST_WEBSITE=${DAST_WEBSITE:-$(cat environment_url.txt)}
+    - /analyze -t $DAST_WEBSITE
   artifacts:
     expire_in: 7 days
     paths:
       - gl-dast-report.json
+    reports:
+      dast: gl-dast-report.json
+  only:
+    variables:
+      - $GITLAB_FEATURES =~ /\bdast\b/
+  except:
+    variables:
+      - $DAST_DISABLED

.gitlab/ci/review.gitlab-ci.yml

-.review-schedules-only:
-  only:
-    refs:
-      - schedules@gitlab-org/gitlab-foss
-      - schedules@gitlab-org/gitlab
-    kubernetes: active
-    variables:
-      - $REVIEW_APP_CLEANUP
-  except:
-    refs:
-      - tags
-      - /(^docs[\/-].+|.+-docs$)/
-
 .review-base:
   extends:
     - .default-tags
     - .default-retry
-    - .review-only
+    - .default-only
+    - .only-review
+    - .only-code-qa-changes
   image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base
   dependencies: []
   before_script:
@@ -25,6 +14,7 @@
   extends:
     - .default-tags
     - .default-retry
+    - .default-only
   image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-qa-alpine
   services:
     - docker:19.03.0-dind
@@ -39,12 +29,11 @@
 build-qa-image:
   extends:
     - .review-docker
-    - .except-docs
+    - .only-code-qa-changes
   only:
-    refs:
-      - branches@gitlab-org/gitlab-foss
-      - branches@gitlab-org/gitlab
-  stage: test
+    variables:
+      - $CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE == "gitlab-org"
+  stage: prepare
   script:
     - '[[ ! -d "ee/" ]] || export GITLAB_EDITION="ee"'
     - export QA_IMAGE="${CI_REGISTRY}/${CI_PROJECT_PATH}/gitlab/gitlab-${GITLAB_EDITION}-qa:${CI_COMMIT_REF_SLUG}"
@@ -53,6 +42,9 @@ build-qa-image:
     - time docker push ${QA_IMAGE}
 
 .review-build-cng-base:
+  extends:
+    - .default-only
+    - .only-code-qa-changes
   image: ruby:2.6-alpine
   stage: review-prepare
   before_script:
@@ -66,13 +58,13 @@ build-qa-image:
 review-build-cng:
   extends:
     - .review-build-cng-base
-    - .review-only
+    - .only-review
   needs: ["gitlab:assets:compile pull-cache"]
 
 schedule:review-build-cng:
   extends:
     - .review-build-cng-base
-    - .review-schedules-only
+    - .only-review-schedules
   needs: ["gitlab:assets:compile"]
 
 .review-deploy-base:
@@ -118,7 +110,7 @@ review-deploy:
 schedule:review-deploy:
   extends:
     - .review-deploy-base
-    - .review-schedules-only
+    - .only-review-schedules
   needs: ["schedule:review-build-cng"]
 
 review-stop:
@@ -153,7 +145,8 @@ review-cleanup-failed-deployment:
 .review-qa-base:
   extends:
     - .review-docker
-    - .review-only
+    - .only-review
+    - .only-code-qa-changes
   stage: qa
   variables:
     QA_ARTIFACTS_DIR: "${CI_PROJECT_DIR}/qa"
@@ -200,7 +193,9 @@ review-qa-all:
 parallel-spec-reports:
   extends:
     - .default-tags
-    - .except-docs
+    - .default-only
+    - .only-code-qa-changes
+    - .only-review
   image: ruby:2.6-alpine
   stage: post-test
   dependencies: ["review-qa-all"]
@@ -247,14 +242,14 @@ review-performance:
 schedule:review-performance:
   extends:
     - review-performance
-    - .review-schedules-only
+    - .only-review-schedules
   dependencies: ["schedule:review-deploy"]
 
 schedule:review-cleanup:
   extends:
     - .review-base
-    - .review-schedules-only
-  stage: build
+    - .only-review-schedules
+  stage: prepare
   allow_failure: true
   environment:
     name: review/auto-cleanup
@@ -270,6 +265,7 @@ danger-review:
     - .default-tags
     - .default-retry
     - .default-cache
+    - .default-only
   image: registry.gitlab.com/gitlab-org/gitlab-build-images:danger
   stage: test
   dependencies: []
@@ -279,10 +275,9 @@ danger-review:
   except:
     refs:
       - master
-      - /^\d+-\d+-auto-deploy-\d+$/
-      - /^[\d-]+-stable(-ee)?$/
-      - /^ce-to-ee-.*/
-      - /.*-stable(-ee)?-prepare-.*/
+    variables:
+      - $CI_COMMIT_REF_NAME =~ /^\d+-\d+-auto-deploy-\d+$/
+      - $CI_COMMIT_REF_NAME =~ /^[\d-]+-stable(-ee)?$/
   script:
     - git version
     - node --version

.gitlab/ci/setup.gitlab-ci.yml

@@ -6,7 +6,6 @@ cache gems:
     - .default-retry
     - .default-cache
     - .default-before_script
-    - .except-docs
   stage: test
   dependencies: ["setup-test-env"]
   needs: ["setup-test-env"]
@@ -19,15 +18,17 @@ cache gems:
       - vendor/cache
   only:
     refs:
-      - master@gitlab-org/gitlab-foss
-      - master@gitlab-org/gitlab
+      - master
       - tags
+    variables:
+      - $CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE == "gitlab-org"
 
 .minimal-job:
   extends:
     - .default-tags
     - .default-retry
-    - .except-docs-qa
+    - .default-only
+    - .only-code-changes
   dependencies: []
 
 gitlab_git_test:
@@ -40,5 +41,6 @@ no_ee_check:
   script:
     - scripts/no-ee-check
   only:
-    refs:
-      - branches@gitlab-org/gitlab-foss
+    variables:
+      - $CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAME == "gitlab-foss"
+      - $CI_SERVER_HOST == "dev.gitlab.org" && $CI_PROJECT_NAME == "gitlabhq"

.gitlab/ci/test-metadata.gitlab-ci.yml

 .tests-metadata-state:
+  extends: .default-only
   variables:
     TESTS_METADATA_S3_BUCKET: "gitlab-ce-cache"
   before_script:
@@ -13,7 +14,7 @@
 retrieve-tests-metadata:
   extends:
     - .tests-metadata-state
-    - .except-docs-qa
+    - .only-code-changes
   stage: prepare
   cache:
     key: tests_metadata
@@ -26,8 +27,8 @@ retrieve-tests-metadata:
     - mkdir -p rspec_profiling/
     - wget -O $FLAKY_RSPEC_SUITE_REPORT_PATH http://${TESTS_METADATA_S3_BUCKET}.s3.amazonaws.com/$FLAKY_RSPEC_SUITE_REPORT_PATH || rm $FLAKY_RSPEC_SUITE_REPORT_PATH
     - '[[ -f $FLAKY_RSPEC_SUITE_REPORT_PATH ]] || echo "{}" > ${FLAKY_RSPEC_SUITE_REPORT_PATH}'
-    - wget -O $EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH http://${TESTS_METADATA_S3_BUCKET}.s3.amazonaws.com/$EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH || rm $EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH
-    - '[[ -f $EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH ]] || echo "{}" > ${EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH}'
+    - '[[ ! -d "ee/" ]] || wget -O $EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH http://${TESTS_METADATA_S3_BUCKET}.s3.amazonaws.com/$EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH || rm $EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH'
+    - '[[ ! -d "ee/" ]] || [[ -f $EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH ]] || echo "{}" > ${EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH}'
 
 update-tests-metadata:
   extends: .tests-metadata-state
@@ -43,9 +44,9 @@ update-tests-metadata:
     - echo "{}" > ${KNAPSACK_RSPEC_SUITE_REPORT_PATH}
     - scripts/merge-reports ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/rspec_*_pg_node_*.json
     - '[[ -z ${TESTS_METADATA_S3_BUCKET} ]] || scripts/sync-reports put $TESTS_METADATA_S3_BUCKET $KNAPSACK_RSPEC_SUITE_REPORT_PATH'
-    - echo "{}" > ${EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH}
-    - scripts/merge-reports ${EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/rspec_*_pg_ee_*node_*.json
-    - '[[ -z ${TESTS_METADATA_S3_BUCKET} ]] || scripts/sync-reports put $TESTS_METADATA_S3_BUCKET $EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH'
+    - '[[ ! -d "ee/" ]] || echo "{}" > ${EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH}'
+    - '[[ ! -d "ee/" ]] || scripts/merge-reports ${EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/rspec_*_pg_ee_*node_*.json'
+    - '[[ ! -d "ee/" ]] || [[ -z ${TESTS_METADATA_S3_BUCKET} ]] || scripts/sync-reports put $TESTS_METADATA_S3_BUCKET $EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH'
     - rm -f knapsack/${CI_PROJECT_NAME}/*_node_*.json
     - scripts/merge-reports ${FLAKY_RSPEC_SUITE_REPORT_PATH} rspec_flaky/all_*_*.json
     - FLAKY_RSPEC_GENERATE_REPORT=1 scripts/prune-old-flaky-specs ${FLAKY_RSPEC_SUITE_REPORT_PATH}
@@ -54,15 +55,17 @@ update-tests-metadata:
     - scripts/insert-rspec-profiling-data
   only:
     refs:
-      - master@gitlab-org/gitlab-foss
-      - master@gitlab-org/gitlab
-      - master@gitlab/gitlabhq
-      - master@gitlab/gitlab-ee
+      - master
+    variables:
+      - $CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE == "gitlab-org"
+      - $CI_SERVER_HOST == "dev.gitlab.org"
 
 flaky-examples-check:
   extends:
     - .default-tags
     - .default-retry
+    - .default-only
+    - .only-code-changes
   image: ruby:2.6-alpine
   stage: post-test
   variables:
@@ -70,12 +73,7 @@ flaky-examples-check:
   allow_failure: true
   only:
     refs:
-      - branches
-  except:
-    refs:
-      - master
-      - /(^docs[\/-].+|.+-docs$)/
-      - /(^qa[\/-].*|.*-qa$)/
+      - merge_requests
   artifacts:
     expire_in: 30d
     paths:

.gitlab/ci/yaml.gitlab-ci.yml

@@ -4,7 +4,10 @@ lint-ci-gitlab:
   extends:
     - .default-tags
     - .default-retry
-    - .except-docs
+    - .default-only
+  only:
+    changes:
+      - "**/*.yml"
   image: sdesbure/yamllint:latest
   dependencies: []
   script: