Merge branch '216730-decrease-compile-assets-pull-cache-duration-from-9-17-minutes' into 'master'

Simplify/optimize the assets compilation jobs and update caches on 2-hourly scheduled master pipelines [UPDATE CACHE]

Closes #218440

See merge request gitlab-org/gitlab!32171

.gitlab-ci.yml

@@ -15,7 +15,7 @@ stages:
 # in cases where jobs require Docker-in-Docker, the job
 # definition must be extended with `.use-docker-in-docker`
 default:
-  image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.6-golang-1.14-git-2.26-lfs-2.9-chrome-73.0-node-12.x-yarn-1.21-postgresql-9.6-graphicsmagick-1.3.34"
+  image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.6-golang-1.14-git-2.26-lfs-2.9-chrome-73.0-node-12.x-yarn-1.21-postgresql-11-graphicsmagick-1.3.34"
   tags:
     - gitlab-org
   # All jobs are interruptible by default

.gitlab/ci/docs.gitlab-ci.yml

@@ -62,7 +62,7 @@ docs lint:
 graphql-reference-verify:
   extends:
     - .default-retry
-    - .default-cache
+    - .rails-cache
     - .default-before_script
     - .docs:rules:graphql-reference-verify
     - .use-pg11

.gitlab/ci/global.gitlab-ci.yml

@@ -10,46 +10,58 @@
 
 .default-before_script:
   before_script:
-    - date
     - '[ "$FOSS_ONLY" = "1" ] && rm -rf ee/ qa/spec/ee/ qa/qa/specs/features/ee/ qa/qa/ee/ qa/qa/ee.rb'
     - export GOPATH=$CI_PROJECT_DIR/.go
     - mkdir -p $GOPATH
     - source scripts/utils.sh
     - source scripts/prepare_build.sh
-    - date
 
-# Jobs that only need to pull cache
-.default-cache:
+.rails-cache:
   cache:
-    key: "debian-stretch-ruby-2.6.6-pg11-node-12.x"
+    key: "rails-v1"
     paths:
-      - .go/pkg/mod
-      - vendor/ruby
-      - .yarn-cache/
-      - vendor/gitaly-ruby
+      - vendor/ruby/
+      - vendor/gitaly-ruby/
+      - .go/pkg/mod/
     policy: pull
 
-.rails-cache:
+.static-analysis-cache:
+  cache:
+    key: "static-analysis-v1"
+    paths:
+      - vendor/ruby/
+      - node_modules/
+      - tmp/rubocop_cache/
+    policy: pull
+
+.qa-cache:
   cache:
-    key:
-      files:
-        - Gemfile.lock
-        - GITALY_SERVER_VERSION
-      prefix: "ruby-go-cache-v1"
+    key: "qa-v1"
     paths:
-      - vendor/ruby
-      - vendor/gitaly-ruby
-      - .go/pkg/mod
+      - qa/vendor/ruby/
     policy: pull
 
 .yarn-cache:
   cache:
-    key:
-      files:
-        - yarn.lock
-      prefix: "v1"
+    key: "yarn-v1"
     paths:
       - node_modules/
+      - tmp/cache/webpack-dlls/
+    policy: pull
+
+.assets-compile-cache:
+  cache:
+    key: "assets-compile-${NODE_ENV}-v1"
+    paths:
+      - vendor/ruby/
+      - node_modules/
+      - assets-hash.txt
+      - public/assets/webpack/
+      - tmp/cache/assets/sprockets/
+      - tmp/cache/babel-loader/
+      - tmp/cache/vue-loader/
+      - tmp/cache/webpack-dlls/
+    policy: pull
 
 .use-pg11:
   image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.6-golang-1.14-git-2.26-lfs-2.9-chrome-73.0-node-12.x-yarn-1.21-postgresql-11-graphicsmagick-1.3.34"
@@ -75,6 +87,7 @@
     name: gcr.io/kaniko-project/executor:debug-v0.20.0
     entrypoint: [""]
   before_script:
+    - source scripts/utils.sh
     - mkdir -p /kaniko/.docker
     - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json
 

.gitlab/ci/memory.gitlab-ci.yml

 .only-code-memory-job-base:
   extends:
     - .default-retry
-    - .default-cache
+    - .rails-cache
     - .default-before_script
     - .memory:rules
 
@@ -39,7 +39,7 @@ memory-on-boot:
     - .only-code-memory-job-base
     - .use-pg11
   stage: test
-  needs: ["setup-test-env", "compile-assets pull-cache"]
+  needs: ["setup-test-env", "compile-test-assets"]
   variables:
     NODE_ENV: "production"
     RAILS_ENV: "production"

.gitlab/ci/pages.gitlab-ci.yml

@@ -7,7 +7,7 @@ pages:
     - rspec:coverage
     - coverage-frontend
     - karma
-    - gitlab:assets:compile pull-cache
+    - compile-production-assets
   script:
     - mv public/ .public/
     - mkdir public/

.gitlab/ci/qa.gitlab-ci.yml

 .qa-job-base:
   extends:
     - .default-retry
+    - .qa-cache
   stage: test
   needs: []
-  cache:
-    key: "qa-framework-jobs:v1"
-    paths:
-      - vendor/ruby
   before_script:
     - '[ "$FOSS_ONLY" = "1" ] && rm -rf ee/ qa/spec/ee/ qa/qa/specs/features/ee/ qa/qa/ee/ qa/qa/ee.rb'
     - cd qa/
@@ -22,11 +19,9 @@ qa:internal:
 
 qa:internal-as-if-foss:
   extends:
-    - .qa-job-base
+    - qa:internal
     - .qa:rules:as-if-foss
     - .as-if-foss
-  script:
-    - bundle exec rspec
 
 qa:selectors:
   extends:
@@ -41,6 +36,16 @@ qa:selectors-as-if-foss:
     - .qa:rules:as-if-foss
     - .as-if-foss
 
+update-qa-cache:
+  extends:
+    - .qa-job-base
+    - .shared:rules:update-cache
+  stage: prepare
+  script:
+    - echo "Cache has been updated and ready to be uploaded."
+  cache:
+    policy: push  # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.
+
 .package-and-qa-base:
   image: ruby:2.6-alpine
   stage: qa

.gitlab/ci/rails.gitlab-ci.yml

-.rails:needs:setup-and-assets:
-  needs: ["setup-test-env", "compile-assets pull-cache"]
-
 .rails-job-base:
   extends:
     - .default-retry
@@ -35,33 +32,54 @@ setup-test-env:
       - tmp/tests/repositories
       - tmp/tests/second_storage
     when: always
+
+update-rails-cache:
+  extends:
+    - setup-test-env
+    - .shared:rules:update-cache
+  artifacts: {}  # This job's purpose is only to update the cache.
   cache:
+    policy: push  # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.
+
+.static-analysis-base:
+  extends:
+    - .default-retry
+    - .default-before_script
+    - .static-analysis-cache
+  needs: []
+  variables:
+    SETUP_DB: "false"
+    ENABLE_SPRING: "1"
+
+update-static-analysis-cache:
+  extends:
+    - .static-analysis-base
+    - .shared:rules:update-cache
+  stage: prepare
+  script:
+    - rm -rf ./node_modules  # We remove node_modules because there's no mechanism to remove stall entries.
+    - run_timed_command "retry yarn install --frozen-lockfile"
+    - bundle exec rubocop --parallel  # For the moment we only cache `vendor/ruby/`, `node_modules/`, and `tmp/rubocop_cache` so we don't need to run all the tasks,
+  cache:
+    # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up but RuboCop has a mechanism
+    # for keeping only the N latest cache files, so we take advantage of it with `pull-push` and removing `node_modules` at the start of the job.
     policy: pull-push
 
 static-analysis:
   extends:
-    - .rails-job-base
+    - .static-analysis-base
     - .rails:rules:default-refs-code-backstage-qa
-    - .rails:needs:setup-and-assets
   stage: test
-  variables:
-    SETUP_DB: "false"
-    ENABLE_SPRING: "1"
   parallel: 4
   script:
+    - run_timed_command "retry yarn install --frozen-lockfile"
     - scripts/static-analysis
-  cache:
-    key: "ruby-2.6.6-pg11-rubocop"
-    paths:
-      - vendor/ruby
-      - tmp/rubocop_cache
-    policy: pull-push
 
 downtime_check:
   extends:
     - .rails-job-base
     - .rails:rules:downtime_check
-  needs: ["setup-test-env"]
+  needs: []
   stage: test
   variables:
     SETUP_DB: "false"
@@ -71,7 +89,7 @@ downtime_check:
 .rspec-base:
   extends: .rails-job-base
   stage: test
-  needs: ["setup-test-env", "retrieve-tests-metadata", "compile-assets pull-cache"]
+  needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets"]
   script:
     - run_timed_command "scripts/gitaly-test-build"
     - run_timed_command "scripts/gitaly-test-spawn"
@@ -219,8 +237,6 @@ rspec:coverage:
     - memory-on-boot
   variables:
     SETUP_DB: "false"
-  cache:
-    policy: pull
   script:
     - bundle exec scripts/merge-simplecov
     - bundle exec scripts/gather-test-memory-data
@@ -248,7 +264,7 @@ rspec:coverage:
     - .rails:rules:as-if-foss
     - .as-if-foss
     - .use-pg11
-  needs: ["setup-test-env", "retrieve-tests-metadata", "compile-assets pull-cache as-if-foss"]
+  needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets as-if-foss"]
 
 .rspec-ee-base-pg11:
   extends:

.gitlab/ci/review.gitlab-ci.yml

@@ -38,7 +38,7 @@ review-build-cng:
     - install_api_client_dependencies_with_apk
     - install_gitlab_gem
   needs:
-    - job: gitlab:assets:compile pull-cache
+    - job: compile-production-assets
       artifacts: false
   script:
     - BUILD_TRIGGER_TOKEN=$REVIEW_APPS_BUILD_TRIGGER_TOKEN ./scripts/trigger-build cng
@@ -238,5 +238,3 @@ danger-review:
     - source scripts/utils.sh
     - retry yarn install --frozen-lockfile
     - danger --fail-on-errors=true --verbose
-  cache:
-    policy: pull

.gitlab/ci/rules.gitlab-ci.yml

@@ -37,6 +37,9 @@
 .if-merge-request-title-as-if-foss: &if-merge-request-title-as-if-foss
   if: '$CI_MERGE_REQUEST_TITLE =~ /RUN AS-IF-FOSS/'
 
+.if-merge-request-title-update-caches: &if-merge-request-title-update-caches
+  if: '$CI_MERGE_REQUEST_TITLE =~ /UPDATE CACHE/'
+
 .if-security-merge-request: &if-security-merge-request
   if: '$CI_PROJECT_NAMESPACE == "gitlab-org/security" && $CI_MERGE_REQUEST_IID'
 
@@ -183,6 +186,14 @@
   - ".dockerignore"
   - "qa/**/*"
 
+################
+# Shared rules #
+################
+.shared:rules:update-cache:
+  rules:
+    - <<: *if-master-schedule-2-hourly
+    - <<: *if-merge-request-title-update-caches
+
 ####################
 # Cache repo rules #
 ####################
@@ -244,45 +255,18 @@
 ##################
 # Frontend rules #
 ##################
-# This job only runs on `master` since it pushes to the cache.
-.frontend:rules:gitlab-assets-compile-pull-push-cache:
-  rules:
-    - <<: *if-not-canonical-namespace
-      when: never
-    - <<: *if-master-refs
-      changes: *code-backstage-qa-patterns
-      when: on_success
-
-.frontend:rules:gitlab-assets-compile-pull-cache:
+.frontend:rules:compile-production-assets:
   rules:
     - <<: *if-not-canonical-namespace
       when: never
     - <<: *if-default-refs
       changes: *code-backstage-qa-patterns
-      when: on_success
-
-.frontend:rules:compile-assets-pull-push-cache:
-  rules:
-    - <<: *if-master-refs
-      changes: *code-backstage-qa-patterns
-      when: on_success
-
-# This job only runs on `master` since it pushes to the cache.
-.frontend:rules:compile-assets-pull-push-cache-as-if-foss:
-  rules:
-    - <<: *if-not-ee
-      when: never
-    - <<: *if-master-push
-      changes: *code-backstage-qa-patterns
-    - <<: *if-master-schedule-2-hourly
 
-.frontend:rules:compile-assets-pull-cache:
+.frontend:rules:compile-test-assets:
   rules:
-    - <<: *if-default-refs
-      changes: *code-backstage-qa-patterns
-      when: on_success
+    - changes: *code-backstage-qa-patterns
 
-.frontend:rules:compile-assets-pull-cache-as-if-foss:
+.frontend:rules:compile-test-assets-as-if-foss:
   rules:
     - <<: *if-not-ee
       when: never
@@ -299,7 +283,6 @@
   rules:
     - <<: *if-default-refs
       changes: *code-backstage-patterns
-      when: on_success
 
 .frontend:rules:default-frontend-jobs-as-if-foss:
   rules:
@@ -327,10 +310,8 @@
   rules:
     - <<: *if-master-refs
       changes: *frontend-dependency-patterns
-      when: on_success
     - <<: *if-merge-request
       changes: *frontend-dependency-patterns
-      when: on_success
 
 .frontend:rules:qa-frontend-node-latest:
   rules:

.gitlab/ci/setup.gitlab-ci.yml

@@ -3,7 +3,7 @@
 cache gems:
   extends:
     - .default-retry
-    - .default-cache
+    - .rails-cache
     - .default-before_script
     - .setup:rules:cache-gems
   stage: test

.gitlab/ci/test-metadata.gitlab-ci.yml

@@ -3,11 +3,6 @@
     TESTS_METADATA_S3_BUCKET: "gitlab-ce-cache"
   before_script:
     - source scripts/utils.sh
-  cache:
-    key: tests_metadata
-    paths:
-      - knapsack/
-      - rspec_flaky/
   artifacts:
     expire_in: 31d
     paths:
@@ -20,8 +15,6 @@ retrieve-tests-metadata:
     - .tests-metadata-state
     - .test-metadata:rules:retrieve-tests-metadata
   stage: prepare
-  cache:
-    policy: pull
   script:
     - source scripts/rspec_helpers.sh
     - retrieve_tests_metadata
@@ -44,8 +37,6 @@ update-tests-metadata:
     - rspec-ee unit pg11 geo
     - rspec-ee integration pg11 geo
     - rspec-ee system pg11 geo
-  cache:
-    policy: push
   script:
     - retry gem install fog-aws mime-types activesupport rspec_profiling postgres-copy --no-document
     - source scripts/rspec_helpers.sh

.rubocop.yml

@@ -27,6 +27,7 @@ AllCops:
     - 'plugins/**/*'
     - 'file_hooks/**/*'
   CacheRootDirectory: tmp
+  MaxFilesInCache: 18000
 
 Cop/AvoidKeywordArgumentsInSidekiqWorkers:
   Enabled: true

config/initializers/01_secret_token.rb

@@ -61,6 +61,8 @@ def generate_new_rsa_private_key
 end
 
 def warn_missing_secret(secret)
+  return if Rails.env.test?
+
   warn "Missing Rails.application.secrets.#{secret} for #{Rails.env} environment. The secret will be generated and stored in config/secrets.yml."
 end