Create fail-fast mechanism on MR pipeline

Add rspec fail-fast job
- This job runs rspec on test files detected by
  the job `detect-tests`.

Add fail-pipeline-early job
- This job cancels the current pipeline in order
  to provide fast feedback upon test failure.
  This happens when the `rspec fail-fast` jobs has failed.

Add toggle to enable `rspec fail-fast`
- The rspec-fail-fast and fail-pipeline-early jobs
  will only be created if the RSPEC_FAIL_FAST_ENABLED variable
  is "true".
- Using this flag, we can experiment with this pipeline
  change with more control.

Add option to skip rspec fail-fast
- Using MR title SKIP RSPEC FAIL-FAST
- Using dont-interrupt-me job

.gitlab/ci/rails.gitlab-ci.yml

@@ -505,6 +505,22 @@ rspec-ee system pg12 geo:
 
 ##################################################
 # EE: Canonical MR pipelines
+rspec fail-fast:
+  extends:
+    - .rspec-base-pg11
+    - .rails:rules:rspec fail-fast
+  stage: test
+  needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets", "detect-tests"]
+  script:
+    - run_timed_command "scripts/gitaly-test-build"
+    - run_timed_command "scripts/gitaly-test-spawn"
+    - source scripts/rspec_helpers.sh
+    - rspec_fail_fast tmp/matching_tests.txt "--tag ~quarantine"
+  artifacts:
+    expire_in: 7d
+    paths:
+      - tmp/capybara/
+
 rspec foss-impact:
   extends:
     - .rspec-base-pg11-as-if-foss
@@ -520,5 +536,19 @@ rspec foss-impact:
     paths:
       - tmp/capybara/
 
+fail-pipeline-early:
+  extends:
+    - .rails:rules:fail-pipeline-early
+  stage: post-test
+  needs:
+    - job: rspec fail-fast
+      artifacts: false
+  variables:
+    GIT_DEPTH: 1
+  before_script:
+    - source scripts/utils.sh
+    - install_api_client_dependencies_with_apt
+  script:
+    - fail_pipeline_early
 # EE: Canonical MR pipelines
 ##################################################

.gitlab/ci/rules.gitlab-ci.yml

@@ -67,6 +67,12 @@
 .if-cache-credentials-schedule: &if-cache-credentials-schedule
   if: '$CI_REPO_CACHE_CREDENTIALS && $CI_PIPELINE_SOURCE == "schedule"'
 
+.if-rspec-fail-fast-disabled: &if-rspec-fail-fast-disabled
+  if: '$RSPEC_FAIL_FAST_ENABLED != "true"'
+
+.if-rspec-fail-fast-skipped: &if-rspec-fail-fast-skipped
+  if: '$CI_MERGE_REQUEST_TITLE =~ /SKIP RSPEC FAIL-FAST/'
+
 ####################
 # Changes patterns #
 ####################
@@ -579,6 +585,34 @@
     - <<: *if-dot-com-gitlab-org-merge-request
       changes: *code-backstage-patterns
 
+.rails:rules:rspec fail-fast:
+  rules:
+    - <<: *if-rspec-fail-fast-disabled
+      when: never
+    - <<: *if-rspec-fail-fast-skipped
+      when: never
+    - <<: *if-not-ee
+      when: never
+    - <<: *if-security-merge-request
+      changes: *code-backstage-patterns
+    - <<: *if-dot-com-gitlab-org-merge-request
+      changes: *code-backstage-patterns
+
+.rails:rules:fail-pipeline-early:
+  rules:
+    - <<: *if-rspec-fail-fast-disabled
+      when: never
+    - <<: *if-rspec-fail-fast-skipped
+      when: never
+    - <<: *if-not-ee
+      when: never
+    - <<: *if-security-merge-request
+      changes: *code-backstage-patterns
+      when: on_failure
+    - <<: *if-dot-com-gitlab-org-merge-request
+      changes: *code-backstage-patterns
+      when: on_failure
+
 .rails:rules:downtime_check:
   rules:
     - <<: *if-merge-request

doc/development/pipelines.md

@@ -357,6 +357,65 @@ graph RL;
   end
 ```
 
+### Fail-fast pipeline in Merge Requests
+
+To provide faster feedback when a Merge Request breaks existing tests, we are experimenting with a
+fail-fast mechanism.
+
+An `rspec fail-fast` job is added in parallel to all other `rspec` jobs in a Merge
+Request pipeline. This job runs the tests that are directly related to the changes
+in the Merge Request.
+
+If any of these tests fail, the `rspec fail-fast` job fails, triggering a
+`fail-pipeline-early` job to run. The `fail-pipeline-early` job:
+
+- Cancels the currently running pipeline and all in-progress jobs.
+- Sets pipeline to have status `failed`.
+
+For example:
+
+```mermaid
+graph LR
+    subgraph "prepare stage";
+        A["detect-tests"]
+    end
+
+    subgraph "test stage";
+        B["jest"];
+        C["rspec migration"];
+        D["rspec unit"];
+        E["rspec integration"];
+        F["rspec system"];
+        G["rspec fail-fast"];
+    end
+
+    subgraph "post-test stage";
+        Z["fail-pipeline-early"];
+    end
+    
+    A --"artifact: list of test files"--> G
+    G --"on failure"--> Z
+```
+
+A Merge Request author may choose to opt-out of the fail fast mechanism by doing one of the following:
+
+- Including `[SKIP RSPEC FAIL-FAST]` in the Merge Request title.
+- Starting the `dont-interrupt-me` job found in the `sync` stage of a Merge Request pipeline.
+
+The `rspec fail-fast` is a no-op if there are more than 10 test files related to the
+Merge Request. This prevents `rspec fail-fast` duration from exceeding the average
+`rspec` job duration and defeating its purpose.
+
+This number can be overridden by setting a CI variable named `RSPEC_FAIL_FAST_TEST_FILE_COUNT_THRESHOLD`.
+
+NOTE: **Note:**
+This experiment is only enabled when the CI variable `RSPEC_FAIL_FAST_ENABLED=true` is set.
+
+#### Determining related test files in a Merge Request
+
+The test files related to the Merge Request are determined using the [`test_file_finder`](https://gitlab.com/gitlab-org/ci-cd/test_file_finder) gem.
+We are using a custom mapping between source file to test files, maintained in the `tests.yml` file.
+
 ### PostgreSQL versions testing
 
 #### Current versions testing

scripts/rspec_helpers.sh

@@ -111,6 +111,27 @@ function rspec_paralellized_job() {
   date
 }
 
+function rspec_fail_fast() {
+  local test_file_count_threshold=${RSPEC_FAIL_FAST_TEST_FILE_COUNT_THRESHOLD:-10}
+  local matching_tests_file=${1}
+  local rspec_opts=${2}
+  local test_files="$(cat "${matching_tests_file}")"
+  local test_file_count=$(wc -w "${matching_tests_file}" | awk {'print $1'})
+
+  if [[ "${test_file_count}" -gt "${test_file_count_threshold}" ]]; then
+    echo "This job is intentionally skipped because there are more than ${test_file_count_threshold} test files matched,"
+    echo "which would take too long to run in this job."
+    echo "All the tests would be run in other rspec jobs."
+    exit 0
+  fi
+
+  if [[ -n $test_files ]]; then
+    rspec_simple_job "${rspec_opts} ${test_files}"
+  else
+    echo "No rspec fail-fast tests to run"
+  fi
+}
+
 function rspec_matched_foss_tests() {
   local test_file_count_threshold=20
   local matching_tests_file=${1}
@@ -131,6 +152,6 @@ function rspec_matched_foss_tests() {
   if [[ -n $test_files ]]; then
     rspec_simple_job "${rspec_opts} ${test_files}"
   else
-    echo "No FOSS test files to run"
+    echo "No impacted FOSS rspec tests to run"
   fi
 }

scripts/utils.sh

@@ -137,3 +137,15 @@ function play_job() {
   job_url=$(curl --silent --show-error --request POST --header "PRIVATE-TOKEN: ${api_token}" "${url}" | jq ".web_url")
   echoinfo "Manual job '${job_name}' started at: ${job_url}"
 }
+
+function fail_pipeline_early() {
+  local dont_interrupt_me_job_id
+  dont_interrupt_me_job_id=$(get_job_id 'dont-interrupt-me' 'scope=success')
+
+  if [[ "${dont_interrupt_me_job_id}" != "" ]]; then
+    echoinfo "This pipeline cannot be interrupted due to \`dont-interrupt-me\` job ${dont_interrupt_me_job_id}"
+  else
+    echoinfo "Failing pipeline early for fast feedback due to test failures in rspec fail-fast."
+    curl --request POST --header "PRIVATE-TOKEN: ${GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN}" "https://${CI_SERVER_HOST}/api/v4/projects/${CI_PROJECT_ID}/pipelines/${CI_PIPELINE_ID}/cancel"
+  fi
+}