Merge branch '327257-run-rubocop-in-a-dedicated-job' into 'master'

ci: Cache 25000 RuboCop files and add debugging for RuboCop

See merge request gitlab-org/gitlab!58926

.gitlab/ci/global.gitlab-ci.yml

@@ -81,7 +81,7 @@
   <<: *rubocop-cache
   # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up but RuboCop has a mechanism
   # for keeping only the N latest cache files, so we take advantage of it with `pull-push`.
-  policy: pull-push
+  policy: push
 
 .qa-ruby-gems-cache: &qa-ruby-gems-cache
   key: "qa-ruby-gems-v1"

.gitlab/ci/rails.gitlab-ci.yml

@@ -218,6 +218,13 @@ update-static-analysis-cache:
     - .shared:rules:update-cache
   stage: prepare
   script:
+    - git log -n 1 --pretty=format:%H -- .rubocop.yml
+    - git log -n 1 --pretty=format:%H -- .rubocop_manual_todo.yml
+    - git log -n 1 --pretty=format:%H -- .rubocop_todo.yml
+    - git log -n 1 --pretty=format:%H -- rubocop/rubocop-migrations.yml
+    - git log -n 1 --pretty=format:%H -- rubocop/rubocop-usage-data.yml
+    - git log -n 1 --pretty=format:%H -- rubocop/rubocop-code_reuse.yml
+    - bundle exec scripts/debug-rubocop spec/factories/namespace/aggregation_schedules.rb
     - run_timed_command "bundle exec rubocop --parallel"  # For the moment we only cache `tmp/rubocop_cache` so we don't need to run all the tasks.
 
 static-analysis:
@@ -228,6 +235,13 @@ static-analysis:
   parallel: 4
   script:
     - run_timed_command "retry yarn install --frozen-lockfile"
+    - git log -n 1 --pretty=format:%H -- .rubocop.yml
+    - git log -n 1 --pretty=format:%H -- .rubocop_manual_todo.yml
+    - git log -n 1 --pretty=format:%H -- .rubocop_todo.yml
+    - git log -n 1 --pretty=format:%H -- rubocop/rubocop-migrations.yml
+    - git log -n 1 --pretty=format:%H -- rubocop/rubocop-usage-data.yml
+    - git log -n 1 --pretty=format:%H -- rubocop/rubocop-code_reuse.yml
+    - bundle exec scripts/debug-rubocop spec/factories/namespace/aggregation_schedules.rb
     - scripts/static-analysis
 
 static-analysis as-if-foss:

.rubocop.yml

@@ -36,8 +36,9 @@ AllCops:
     - 'plugins/**/*'
     - 'file_hooks/**/*'
     - 'workhorse/**/*'
+    - 'spec/support/*.git/**/*'  # e.g. spec/support/gitlab-git-test.git
   CacheRootDirectory: tmp
-  MaxFilesInCache: 18000
+  MaxFilesInCache: 25000
 
 Cop/AvoidKeywordArgumentsInSidekiqWorkers:
   Enabled: true

scripts/debug-rubocop

+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'rubocop'
+
+file = ARGV[0]
+
+unless file
+  warn('Error: missing file, please supply one')
+  abort
+end
+
+# Taken from https://github.com/rubocop/rubocop/blob/v0.93.1/lib/rubocop/result_cache.rb#L159-L171
+def file_checksum(file, config_store)
+  digester = Digest::SHA1.new
+  mode = File.stat(file).mode
+
+  puts "mode of #{file} is #{mode}"
+  puts "signature of #{file} is #{config_store.for_file(file).signature}"
+  puts "config is:"
+  puts config_store.for_file(file).to_h
+
+  digester.update(
+    "#{file}#{mode}#{config_store.for_file(file).signature}"
+  )
+  digester.file(file)
+  digester.hexdigest
+rescue Errno::ENOENT
+  # Spurious files that come and go should not cause a crash, at least not
+  # here.
+  '_'
+end
+
+config_store = RuboCop::ConfigStore.new
+checksum = file_checksum(file, config_store)
+puts "File checksum for #{file} is #{checksum}"

scripts/static-analysis

@@ -20,25 +20,25 @@ class StaticAnalysis
   # contain values that a FOSS installation won't find. To work
   # around this we will only enable this task on EE installations.
   TASKS_BY_DURATIONS_SECONDS_DESC = {
-    %w[bin/rake lint:haml] => 338,
-    (Gitlab.ee? ? %w[bin/rake gettext:updated_check] : nil) => 308,
+    %w[bin/rake lint:haml] => 488,
+    (Gitlab.ee? ? %w[bin/rake gettext:updated_check] : nil) => 410,
     # Most of the time, RuboCop finishes in 30 seconds, but sometimes it can take around 1200 seconds so we set a
     # duration of 300 to lower the likelihood that it will run in the same job as another long task...
     %w[bundle exec rubocop --parallel] => 300,
-    %w[yarn run lint:eslint:all] => 197,
-    %w[yarn run lint:prettier] => 124,
-    %w[bin/rake gettext:lint] => 96,
+    %w[yarn run lint:eslint:all] => 264,
+    %w[yarn run lint:prettier] => 134,
+    %w[bin/rake gettext:lint] => 81,
     %w[bundle exec license_finder] => 49,
-    %w[bin/rake lint:static_verification] => 22,
-    %w[bin/rake gitlab:sidekiq:all_queues_yml:check] => 13,
-    (Gitlab.ee? ? %w[bin/rake gitlab:sidekiq:sidekiq_queues_yml:check] : nil) => 13,
+    %w[bin/rake lint:static_verification] => 24,
+    %w[bin/rake gitlab:sidekiq:all_queues_yml:check] => 12,
+    (Gitlab.ee? ? %w[bin/rake gitlab:sidekiq:sidekiq_queues_yml:check] : nil) => 11,
     %w[bin/rake config_lint] => 11,
-    %w[yarn run internal:stylelint] => 9,
-    %w[scripts/lint-conflicts.sh] => 0.59,
-    %w[yarn run block-dependencies] => 0.35,
-    %w[scripts/lint-rugged] => 0.23,
-    %w[scripts/gemfile_lock_changed.sh] => 0.02,
-    %w[scripts/frontend/check_no_partial_karma_jest.sh] => 0.01
+    %w[yarn run internal:stylelint] => 8,
+    %w[scripts/lint-conflicts.sh] => 1,
+    %w[yarn run block-dependencies] => 1,
+    %w[scripts/lint-rugged] => 1,
+    %w[scripts/gemfile_lock_changed.sh] => 1,
+    %w[scripts/frontend/check_no_partial_karma_jest.sh] => 1
   }.reject { |k| k.nil? }.sort_by { |a| -a[1] }.to_h.keys.freeze
 
   def run_tasks!