Merge branch '326720-experiment-with-multiple-caches' into 'master'

ci: Dogfood the multiple-cache feature [UPDATE CACHE]

See merge request gitlab-org/gitlab!60647

.gitlab/ci/frontend.gitlab-ci.yml

@@ -58,31 +58,27 @@ compile-test-assets as-if-foss:
 update-assets-compile-production-cache:
   extends:
     - compile-production-assets
+    - .assets-compile-cache-push
     - .shared:rules:update-cache
   stage: prepare
   artifacts: {}  # This job's purpose is only to update the cache.
-  cache:
-    policy: push  # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.
 
 update-assets-compile-test-cache:
   extends:
     - compile-test-assets
+    - .assets-compile-cache-push
     - .shared:rules:update-cache
   stage: prepare
   artifacts: {}  # This job's purpose is only to update the cache.
-  cache:
-    policy: push  # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.
 
 update-yarn-cache:
   extends:
     - .default-retry
-    - .yarn-cache
+    - .yarn-cache-push
     - .shared:rules:update-cache
   stage: prepare
   script:
     - *yarn-install
-  cache:
-    policy: push
 
 .frontend-fixtures-base:
   extends:

.gitlab/ci/global.gitlab-ci.yml

@@ -16,75 +16,147 @@
     - source scripts/utils.sh
     - source scripts/prepare_build.sh
 
+.ruby-gems-cache: &ruby-gems-cache
+  key: "ruby-gems-v1"
+  paths:
+    - vendor/ruby/
+  policy: pull
+
+.ruby-gems-cache-push: &ruby-gems-cache-push
+  <<: *ruby-gems-cache
+  policy: push  # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.
+
+.gitaly-ruby-gems-cache: &gitaly-ruby-gems-cache
+  key: "gitaly-ruby-gems-v1"
+  paths:
+    - vendor/gitaly-ruby/
+  policy: pull
+
+.gitaly-ruby-gems-cache-push: &gitaly-ruby-gems-cache-push
+  <<: *gitaly-ruby-gems-cache
+  policy: push  # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.
+
+.go-pkg-cache: &go-pkg-cache
+  key: "go-pkg-v1"
+  paths:
+    - .go/pkg/mod/
+  policy: pull
+
+.go-pkg-cache-push: &go-pkg-cache-push
+  <<: *go-pkg-cache
+  policy: push  # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.
+
+.node-modules-cache: &node-modules-cache
+  key: "node-modules-${NODE_ENV}-v1"
+  paths:
+    - node_modules/
+    - tmp/cache/webpack-dlls/
+  policy: pull
+
+.node-modules-cache-push: &node-modules-cache-push
+  <<: *node-modules-cache
+  policy: push  # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.
+
+.assets-cache: &assets-cache
+  key: "assets-${NODE_ENV}-v1"
+  paths:
+    - assets-hash.txt
+    - public/assets/webpack/
+    - tmp/cache/assets/sprockets/
+    - tmp/cache/babel-loader/
+    - tmp/cache/vue-loader/
+  policy: pull
+
+.assets-cache-push: &assets-cache-push
+  <<: *assets-cache
+  policy: push  # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.
+
+.rubocop-cache: &rubocop-cache
+  key: "rubocop-v1"
+  paths:
+    - tmp/rubocop_cache/
+  policy: pull
+
+.rubocop-cache-push: &rubocop-cache-push
+  <<: *rubocop-cache
+  # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up but RuboCop has a mechanism
+  # for keeping only the N latest cache files, so we take advantage of it with `pull-push`.
+  policy: pull-push
+
+.qa-ruby-gems-cache: &qa-ruby-gems-cache
+  key: "qa-ruby-gems-v1"
+  paths:
+    - qa/vendor/ruby/
+  policy: pull
+
+.qa-ruby-gems-cache-push: &qa-ruby-gems-cache-push
+  <<: *qa-ruby-gems-cache
+  policy: push  # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.
+
 .setup-test-env-cache:
   cache:
-    key: "setup-test-env-v1"
-    paths:
-      - vendor/ruby/
-      - vendor/gitaly-ruby/
-      - .go/pkg/mod/
-    policy: pull
+    - *ruby-gems-cache
+    - *gitaly-ruby-gems-cache
+    - *go-pkg-cache
+
+.setup-test-env-cache-push:
+  cache:
+    - *ruby-gems-cache-push
+    - *gitaly-ruby-gems-cache-push
+    - *go-pkg-cache-push
 
 .rails-cache:
   cache:
-    key: "rails-v5"
-    paths:
-      - vendor/ruby/
-      - vendor/gitaly-ruby/
-    policy: pull
+    - *ruby-gems-cache
+    - *gitaly-ruby-gems-cache
 
 .static-analysis-cache:
   cache:
-    key: "static-analysis-v2"
-    paths:
-      - vendor/ruby/
-      - node_modules/
-      - tmp/rubocop_cache/
-    policy: pull
+    - *ruby-gems-cache
+    - *node-modules-cache
+    - *rubocop-cache
+
+.static-analysis-cache-push:
+  cache:
+    - *ruby-gems-cache  # We don't push this cache as it's already rebuilt by `update-setup-test-env-cache`
+    - *rubocop-cache-push
 
 .coverage-cache:
   cache:
-    key: "coverage-cache-v1"
-    paths:
-      - vendor/ruby/
-    policy: pull
+    - *ruby-gems-cache
 
 .danger-review-cache:
   cache:
-    key: "danger-review-v1"
-    paths:
-      - vendor/ruby/
-      - node_modules/
-    policy: pull
+    - *ruby-gems-cache
+    - *node-modules-cache
 
 .qa-cache:
   cache:
-    key: "qa-v2"
-    paths:
-      - qa/vendor/ruby/
-    policy: pull
+    - *qa-ruby-gems-cache
+
+.qa-cache-push:
+  cache:
+    - *qa-ruby-gems-cache-push
 
 .yarn-cache:
   cache:
-    key: "yarn-v1"
-    paths:
-      - node_modules/
-      - tmp/cache/webpack-dlls/
-    policy: pull
+    - *node-modules-cache
+
+.yarn-cache-push:
+  cache:
+    - *node-modules-cache-push
 
 .assets-compile-cache:
   cache:
-    key: "assets-compile-${NODE_ENV}-v1"
-    paths:
-      - vendor/ruby/
-      - node_modules/
-      - assets-hash.txt
-      - public/assets/webpack/
-      - tmp/cache/assets/sprockets/
-      - tmp/cache/babel-loader/
-      - tmp/cache/vue-loader/
-      - tmp/cache/webpack-dlls/
-    policy: pull
+    - *ruby-gems-cache
+    - *node-modules-cache
+    - *assets-cache
+
+.assets-compile-cache-push:
+  cache:
+    - *ruby-gems-cache  # We don't push this cache as it's already rebuilt by `update-setup-test-env-cache`
+    - *node-modules-cache-push
+    - *assets-cache-push
 
 .use-pg11:
   image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.7.2.patched-golang-1.14-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-11-graphicsmagick-1.3.36"

.gitlab/ci/qa.gitlab-ci.yml

@@ -41,12 +41,11 @@ qa:selectors-as-if-foss:
 update-qa-cache:
   extends:
     - .qa-job-base
+    - .qa-cache-push
     - .shared:rules:update-cache
   stage: prepare
   script:
     - echo "Cache has been updated and ready to be uploaded."
-  cache:
-    policy: push  # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.
 
 .package-and-qa-base:
   image: ${GITLAB_DEPENDENCY_PROXY}ruby:2.7-alpine

.gitlab/ci/rails.gitlab-ci.yml

@@ -187,18 +187,10 @@ setup-test-env:
 update-setup-test-env-cache:
   extends:
     - setup-test-env
+    - .setup-test-env-cache-push
     - .shared:rules:update-cache
   artifacts:
     paths: []  # This job's purpose is only to update the cache.
-  cache:
-    policy: push  # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.
-
-update-rails-cache:
-  extends:
-    - update-setup-test-env-cache
-    - .rails-cache
-  cache:
-    policy: push  # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.
 
 .coverage-base:
   extends:
@@ -209,16 +201,6 @@ update-rails-cache:
     SETUP_DB: "false"
     USE_BUNDLE_INSTALL: "false"
 
-update-coverage-cache:
-  extends:
-    - .coverage-base
-    - .shared:rules:update-cache
-  stage: prepare
-  script:
-    - !reference [.minimal-bundle-install, script]
-  cache:
-    policy: push  # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.
-
 .static-analysis-base:
   extends:
     - .default-retry
@@ -232,16 +214,11 @@ update-coverage-cache:
 update-static-analysis-cache:
   extends:
     - .static-analysis-base
+    - .static-analysis-cache-push
     - .shared:rules:update-cache
   stage: prepare
   script:
-    - rm -rf ./node_modules  # We remove node_modules because there's no mechanism to remove stall entries.
-    - run_timed_command "retry yarn install --frozen-lockfile"
-    - run_timed_command "bundle exec rubocop --parallel"  # For the moment we only cache `vendor/ruby/`, `node_modules/`, and `tmp/rubocop_cache` so we don't need to run all the tasks,
-  cache:
-    # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up but RuboCop has a mechanism
-    # for keeping only the N latest cache files, so we take advantage of it with `pull-push` and removing `node_modules` at the start of the job.
-    policy: pull-push
+    - run_timed_command "bundle exec rubocop --parallel"  # For the moment we only cache `tmp/rubocop_cache` so we don't need to run all the tasks.
 
 static-analysis:
   extends:

.gitlab/ci/review.gitlab-ci.yml

@@ -230,12 +230,3 @@ danger-review:
       else
         run_timed_command "bundle exec danger --fail-on-errors=true --verbose"
       fi
-
-update-danger-review-cache:
-  extends:
-    - danger-review
-    - .shared:rules:update-cache
-  stage: prepare
-  script: echo 'Cache is fresh!'
-  cache:
-    policy: push  # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.

doc/development/pipelines.md

@@ -548,7 +548,7 @@ request, be sure to start the `dont-interrupt-me` job before pushing.
 
 1. All jobs must only pull caches by default.
 1. All jobs must be able to pass with an empty cache. In other words, caches are only there to speed up jobs.
-1. We currently have several different caches defined in
+1. We currently have several different cache definitions defined in
    [`.gitlab/ci/global.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab/ci/global.gitlab-ci.yml),
    with fixed keys:
    - `.setup-test-env-cache`
@@ -559,17 +559,15 @@ request, be sure to start the `dont-interrupt-me` job before pushing.
    - `.qa-cache`
    - `.yarn-cache`
    - `.assets-compile-cache` (the key includes `${NODE_ENV}` so it's actually two different caches).
+1. These cache definitions are composed of [multiple atomic caches](../ci/yaml/README.md#multiple-caches).
 1. Only 6 specific jobs, running in 2-hourly scheduled pipelines, are pushing (i.e. updating) to the caches:
    - `update-setup-test-env-cache`, defined in [`.gitlab/ci/rails.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab/ci/rails.gitlab-ci.yml).
-   - `update-rails-cache`, defined in [`.gitlab/ci/rails.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab/ci/rails.gitlab-ci.yml).
    - `update-static-analysis-cache`, defined in [`.gitlab/ci/rails.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab/ci/rails.gitlab-ci.yml).
-   - `update-coverage-cache`, defined in [`.gitlab/ci/rails.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab/ci/rails.gitlab-ci.yml).
-   - `update-danger-review-cache`, defined in [`.gitlab/ci/review.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab/ci/review.gitlab-ci.yml).
    - `update-qa-cache`, defined in [`.gitlab/ci/qa.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab/ci/qa.gitlab-ci.yml).
    - `update-assets-compile-production-cache`, defined in [`.gitlab/ci/frontend.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab/ci/frontend.gitlab-ci.yml).
    - `update-assets-compile-test-cache`, defined in [`.gitlab/ci/frontend.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab/ci/frontend.gitlab-ci.yml).
    - `update-yarn-cache`, defined in [`.gitlab/ci/frontend.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab/ci/frontend.gitlab-ci.yml).
-1. These jobs run in merge requests whose title include `UPDATE CACHE`.
+1. These jobs can also be forced to run in merge requests whose title include `UPDATE CACHE` (this can be useful to warm the caches in a MR that updates the cache keys).
 
 ### Artifacts strategy
 
@@ -593,12 +591,12 @@ The `CI_PRE_CLONE_SCRIPT` is currently defined as a project CI/CD variable:
 (
   echo "Downloading archived master..."
   wget -O /tmp/gitlab.tar.gz https://storage.googleapis.com/gitlab-ci-git-repo-cache/project-278964/gitlab-master-shallow.tar.gz
-  
+
   if [ ! -f /tmp/gitlab.tar.gz ]; then
       echo "Repository cache not available, cloning a new directory..."
       exit
   fi
-  
+
   rm -rf $CI_PROJECT_DIR
   echo "Extracting tarball into $CI_PROJECT_DIR..."
   mkdir -p $CI_PROJECT_DIR