Automatically exclude flaky tests from RSpec jobs

.gitlab/ci/rails.gitlab-ci.yml

@@ -534,6 +534,50 @@ rspec:feature-flags:
         run_timed_command "bundle exec scripts/used-feature-flags";
       fi
 
+rspec:skipped-flaky-tests-report:
+  extends:
+    - .default-retry
+    - .rails:rules:skipped-flaky-tests-report
+  image: ruby:2.7-alpine
+  stage: post-test
+  # We cannot use needs since it would mean needing 84 jobs (since most are parallelized)
+  # so we use `dependencies` here.
+  dependencies:
+    # FOSS/EE jobs
+    - rspec migration pg12
+    - rspec unit pg12
+    - rspec integration pg12
+    - rspec system pg12
+    # FOSS/EE minimal jobs
+    - rspec migration pg12 minimal
+    - rspec unit pg12 minimal
+    - rspec integration pg12 minimal
+    - rspec system pg12 minimal
+    # EE jobs
+    - rspec-ee migration pg12
+    - rspec-ee unit pg12
+    - rspec-ee integration pg12
+    - rspec-ee system pg12
+    # EE minimal jobs
+    - rspec-ee migration pg12 minimal
+    - rspec-ee unit pg12 minimal
+    - rspec-ee integration pg12 minimal
+    - rspec-ee system pg12 minimal
+    # Geo jobs
+    - rspec-ee unit pg12 geo
+    - rspec-ee integration pg12 geo
+    - rspec-ee system pg12 geo
+    # Geo minimal jobs
+    - rspec-ee unit pg12 geo minimal
+    - rspec-ee integration pg12 geo minimal
+    - rspec-ee system pg12 geo minimal
+  script:
+    - cat rspec_flaky/skipped_flaky_tests_*_report.txt >> skipped_flaky_tests_report.txt
+  artifacts:
+    expire_in: 31d
+    paths:
+      - skipped_flaky_tests_report.txt
+
 # EE/FOSS: default refs (MRs, default branch, schedules) jobs #
 #######################################################
 

.gitlab/ci/rules.gitlab-ci.yml

@@ -1352,6 +1352,13 @@
       when: never
     - changes: *code-backstage-patterns
 
+.rails:rules:skipped-flaky-tests-report:
+  rules:
+    - <<: *if-not-ee
+      when: never
+    - if: '$SKIP_FLAKY_TESTS_AUTOMATICALLY == "true"'
+      changes: *code-backstage-patterns
+
 #########################
 # Static analysis rules #
 #########################

doc/development/pipelines.md

@@ -166,6 +166,13 @@ Our current RSpec tests parallelization setup is as follows:
 
 After that, the next pipeline uses the up-to-date `knapsack/report-master.json` file.
 
+### Flaky tests
+
+Tests that are [known to be flaky](testing_guide/flaky_tests.md#automatic-retries-and-flaky-tests-detection) are:
+
+- skipped if the `$SKIP_FLAKY_TESTS_AUTOMATICALLY` variable is set to `true` (`false` by default)
+- run if `$SKIP_FLAKY_TESTS_AUTOMATICALLY` variable is not set to `true` or if the `~"pipeline:run-flaky-tests"` label is set on the MR
+
 ### Monitoring
 
 The GitLab test suite is [monitored](performance.md#rspec-profiling) for the `main` branch, and any branch

scripts/rspec_helpers.sh

@@ -166,6 +166,7 @@ function rspec_paralellized_job() {
     export SUITE_FLAKY_RSPEC_REPORT_PATH="${FLAKY_RSPEC_SUITE_REPORT_PATH}"
     export FLAKY_RSPEC_REPORT_PATH="rspec_flaky/all_${report_name}_report.json"
     export NEW_FLAKY_RSPEC_REPORT_PATH="rspec_flaky/new_${report_name}_report.json"
+    export SKIPPED_FLAKY_TESTS_REPORT_PATH="rspec_flaky/skipped_flaky_tests_${report_name}_report.txt"
 
     if [[ ! -f $FLAKY_RSPEC_REPORT_PATH ]]; then
       echo "{}" > "${FLAKY_RSPEC_REPORT_PATH}"

spec/spec_helper.rb

@@ -107,9 +107,7 @@ RSpec.configure do |config|
         warn `curl -s -o log/goroutines.log http://localhost:9236/debug/pprof/goroutine?debug=2`
       end
     end
-  end
-
-  unless ENV['CI']
+  else
     # Allow running `:focus` examples locally,
     # falling back to all tests when there is no `:focus` example.
     config.filter_run focus: true

spec/support/flaky_tests.rb

+# frozen_string_literal: true
+
+return unless ENV['CI']
+return unless ENV['SKIP_FLAKY_TESTS_AUTOMATICALLY'] == "true"
+return if ENV['CI_MERGE_REQUEST_LABELS'].include?(/pipeline:run-flaky-tests/)
+
+require_relative '../tooling/rspec_flaky/report'
+
+RSpec.configure do |config|
+  $flaky_test_example_ids = begin # rubocop:disable Style/GlobalVars
+    raise "$SUITE_FLAKY_RSPEC_REPORT_PATH is empty." if ENV['SUITE_FLAKY_RSPEC_REPORT_PATH'].to_s.empty?
+    raise "#{ENV['SUITE_FLAKY_RSPEC_REPORT_PATH']} doesn't exist" unless File.exist?(ENV['SUITE_FLAKY_RSPEC_REPORT_PATH'])
+
+    RspecFlaky::Report.load(ENV['SUITE_FLAKY_RSPEC_REPORT_PATH']).map { |_, flaky_test_data| flaky_test_data["example_id"] }
+  rescue => e # rubocop:disable Style/RescueStandardError
+    puts e
+    []
+  end
+  $skipped_flaky_tests_report = [] # rubocop:disable Style/GlobalVars
+
+  config.around do |example|
+    # Skip flaky tests automatically
+    if $flaky_test_example_ids.include?(example.id) # rubocop:disable Style/GlobalVars
+      puts "Skipping #{example.id} '#{example.full_description}' because it's flaky."
+      $skipped_flaky_tests_report << example.id # rubocop:disable Style/GlobalVars
+    else
+      example.run
+    end
+  end
+
+  config.after(:suite) do
+    next unless ENV['SKIPPED_FLAKY_TESTS_REPORT_PATH']
+
+    File.write(ENV['SKIPPED_FLAKY_TESTS_REPORT_PATH'], "#{$skipped_flaky_tests_report.join("\n")}\n") # rubocop:disable Style/GlobalVars
+  end
+end