Disallow most failure when CI config are changed

It happened that changing a CI config led to broken jobs but since those
jobs were allowed to fail we didn't realize they were broken before
getting reports from other engineers (and potentially from downstream
projects).

Fixes https://gitlab.com/gitlab-org/gitlab/-/issues/215846.
Signed-off-by: Rémy Coutable <remy@rymai.me>

.gitlab/ci/cache-repo.gitlab-ci.yml

@@ -21,7 +21,6 @@ cache-repo:
   extends: .cache-repo:rules
   image: gcr.io/google.com/cloudsdktool/cloud-sdk:alpine
   stage: sync
-  allow_failure: true
   variables:
     GIT_STRATEGY: none
     TAR_FILENAME: /tmp/gitlab-master.tar

.gitlab/ci/cng.gitlab-ci.yml

@@ -3,7 +3,6 @@ cloud-native-image:
   image: ruby:2.6-alpine
   dependencies: []
   stage: post-test
-  allow_failure: true
   variables:
     GIT_DEPTH: "1"
   script:

.gitlab/ci/docs.gitlab-ci.yml

@@ -2,7 +2,6 @@
   extends:
     - .default-retry
     - .docs:rules:review-docs
-  allow_failure: true
   image: ruby:2.6-alpine
   stage: review
   dependencies: []

.gitlab/ci/frontend.gitlab-ci.yml

@@ -288,9 +288,10 @@ qa-frontend-node:10:
   image: node:dubnium
 
 qa-frontend-node:latest:
-  extends: .qa-frontend-node
+  extends:
+    - .qa-frontend-node
+    - .frontend:rules:qa-frontend-node-latest
   image: node:latest
-  allow_failure: true
 
 webpack-dev-server:
   extends:

.gitlab/ci/qa.gitlab-ci.yml

@@ -60,4 +60,3 @@ package-and-qa:
       artifacts: false
     - job: gitlab:assets:compile pull-cache
       artifacts: false
-  allow_failure: true

.gitlab/ci/reports.gitlab-ci.yml

@@ -14,7 +14,6 @@ code_quality:
     - .use-docker-in-docker
   stage: test
   needs: []
-  allow_failure: true
   variables:
     CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.9"
   script:
@@ -49,7 +48,6 @@ code_quality:
   # `needs: []` starts the job immediately in the pipeline
   # https://docs.gitlab.com/ee/ci/yaml/README.html#needs
   needs: []
-  allow_failure: true
   artifacts:
     paths:
       - gl-sast-report.json  # GitLab-specific
@@ -79,10 +77,11 @@ eslint-sast:
   image:
     name: "$SAST_ANALYZER_IMAGE_PREFIX/eslint:$SAST_ANALYZER_IMAGE_TAG"
 
-nodejs-scan-sast:
-  extends: .sast
-  image:
-    name: "$SAST_ANALYZER_IMAGE_PREFIX/nodejs-scan:$SAST_ANALYZER_IMAGE_TAG"
+# Temporary disabled as it's constantly failing. See https://gitlab.com/gitlab-org/gitlab/-/issues/213769.
+# nodejs-scan-sast:
+#   extends: .sast
+#   image:
+#     name: "$SAST_ANALYZER_IMAGE_PREFIX/nodejs-scan:$SAST_ANALYZER_IMAGE_TAG"
 
 secrets-sast:
   extends: .sast
@@ -101,7 +100,6 @@ dependency_scanning:
   needs: []
   variables:
     DS_EXCLUDED_PATHS: "qa/qa/ee/fixtures/secure_premade_reports,spec,ee/spec"  # GitLab-specific
-  allow_failure: true
   script:
     - export DS_VERSION=${SP_VERSION:-$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')}
     - |
@@ -172,7 +170,6 @@ dast:
     # DAST_USERNAME_FIELD: "user[login]"
     # DAST_PASSWORD_FIELD: "user[passowrd]"
     DAST_VERSION: 1
-  allow_failure: true
   script:
     - 'export DAST_WEBSITE="${DAST_WEBSITE:-$(cat environment_url.txt)}"'
     # To be done in a later iteration

.gitlab/ci/review.gitlab-ci.yml

@@ -2,7 +2,7 @@ build-qa-image:
   extends:
     - .use-kaniko
     - .default-retry
-    - .review:rules:mr-and-schedule-auto
+    - .review:rules:build-qa-image
   stage: build-images
   needs: []
   script:
@@ -27,12 +27,11 @@ review-cleanup:
   script:
     - ruby -rrubygems scripts/review_apps/automated_cleanup.rb
     - gcp_cleanup
-  allow_failure: true
 
 review-build-cng:
   extends:
     - .default-retry
-    - .review:rules:mr-and-schedule-auto-if-frontend-manual-otherwise
+    - .review:rules:review-build-cng
   image: ruby:2.6-alpine
   stage: review-prepare
   before_script:
@@ -69,7 +68,6 @@ review-deploy:
   stage: review
   dependencies: []
   resource_group: "review/${CI_COMMIT_REF_NAME}"
-  allow_failure: true
   before_script:
     - '[[ -d "ee/" ]] || export GITLAB_EDITION="ce"'
     - export GITLAB_SHELL_VERSION=$(<GITLAB_SHELL_VERSION)
@@ -113,7 +111,7 @@ review-deploy:
 review-stop-failed-deployment:
   extends:
     - .review-stop-base
-    - .review:rules:mr-only-auto
+    - .review:rules:review-stop-failed-deployment
   stage: prepare
   script:
     - delete_failed_release
@@ -123,7 +121,6 @@ review-stop:
     - .review-stop-base
     - .review:rules:mr-only-manual
   stage: review
-  allow_failure: true
   script:
     - delete_release
 
@@ -136,7 +133,6 @@ review-stop:
   # This is needed so that manual jobs with needs don't block the pipeline.
   # See https://gitlab.com/gitlab-org/gitlab/-/issues/199979.
   dependencies: ["review-deploy"]
-  allow_failure: true
   variables:
     QA_ARTIFACTS_DIR: "${CI_PROJECT_DIR}/qa"
     QA_CAN_TEST_GIT_PROTOCOL_V2: "false"
@@ -165,7 +161,7 @@ review-stop:
 review-qa-smoke:
   extends:
     - .review-qa-base
-    - .review:rules:mr-only-auto-if-frontend-manual-otherwise
+    - .review:rules:review-qa-smoke
   script:
     - gitlab-qa Test::Instance::Smoke "${QA_IMAGE}" "${CI_ENVIRONMENT_URL}"
 
@@ -190,7 +186,6 @@ review-performance:
   # This is needed so that manual jobs with needs don't block the pipeline.
   # See https://gitlab.com/gitlab-org/gitlab/-/issues/199979.
   dependencies: ["review-deploy"]
-  allow_failure: true
   before_script:
     - export CI_ENVIRONMENT_URL="$(cat environment_url.txt)"
     - echo "${CI_ENVIRONMENT_URL}"
@@ -213,7 +208,6 @@ parallel-spec-reports:
   image: ruby:2.6-alpine
   stage: post-qa
   dependencies: ["review-qa-all"]
-  allow_failure: true
   variables:
     NEW_PARALLEL_SPECS_REPORT: qa/report-new.html
     BASE_ARTIFACT_URL: "${CI_PROJECT_URL}/-/jobs/${CI_JOB_ID}/artifacts/file/qa/"

.gitlab/ci/rules.gitlab-ci.yml

@@ -58,6 +58,9 @@
 ####################
 # Changes patterns #
 ####################
+.ci-patterns: &ci-patterns
+  - ".gitlab/ci/**/*"
+
 .yaml-patterns: &yaml-patterns
   - "**/*.yml"
 
@@ -179,7 +182,7 @@
 .cache-repo:rules:
   rules:
     - <<: *if-cache-credentials-schedule
-      when: on_success
+      allow_failure: true
 
 #############
 # CNG rules #
@@ -188,6 +191,7 @@
   rules:
     - <<: *if-dot-com-gitlab-org-and-security-tag
       when: manual
+      allow_failure: true
 
 ######################
 # Dev fixtures rules #
@@ -214,6 +218,7 @@
     - <<: *if-dot-com-gitlab-org-merge-request
       changes: *docs-patterns
       when: manual
+      allow_failure: true
 
 .docs:rules:docs-lint:
   rules:
@@ -308,6 +313,15 @@
       changes: *frontend-dependency-patterns
       when: on_success
 
+.frontend:rules:qa-frontend-node-latest:
+  rules:
+    - <<: *if-master-refs
+      changes: *frontend-dependency-patterns
+      allow_failure: true
+    - <<: *if-merge-request
+      changes: *frontend-dependency-patterns
+      allow_failure: true
+
 ################
 # Memory rules #
 ################
@@ -347,14 +361,18 @@
 
 .qa:rules:package-and-qa:
   rules:
+    - <<: *if-dot-com-gitlab-org-merge-request
+      changes: *ci-patterns
+      allow_failure: true
     - <<: *if-dot-com-gitlab-org-merge-request
       changes: *qa-patterns
-      when: on_success
+      allow_failure: true
     - <<: *if-dot-com-gitlab-org-merge-request
       changes: *code-patterns
       when: manual
+      allow_failure: true
     - <<: *if-dot-com-gitlab-org-schedule
-      when: on_success
+      allow_failure: true
 
 ###############
 # Rails rules #
@@ -433,6 +451,7 @@
     # - <<: *if-master-refs  # To be done in a later iteration: https://gitlab.com/gitlab-org/gitlab/issues/31160#note_278188255
     - <<: *if-default-refs
       changes: *code-backstage-patterns
+      allow_failure: true
 
 .reports:rules:sast:
   rules:
@@ -441,6 +460,7 @@
     # - <<: *if-master-refs  # To be done in a later iteration: https://gitlab.com/gitlab-org/gitlab/issues/31160#note_278188255
     - <<: *if-default-refs
       changes: *code-backstage-qa-patterns
+      allow_failure: true
 
 .reports:rules:dependency_scanning:
   rules:
@@ -449,6 +469,7 @@
     # - <<: *if-master-refs  # To be done in a later iteration: https://gitlab.com/gitlab-org/gitlab/issues/31160#note_278188255
     - <<: *if-default-refs
       changes: *code-backstage-qa-patterns
+      allow_failure: true
 
 .reports:rules:dast:
   rules:
@@ -456,10 +477,11 @@
       when: never
     - <<: *if-dot-com-gitlab-org-merge-request
       changes: *frontend-patterns
-      when: on_success
+      allow_failure: true
     - <<: *if-dot-com-gitlab-org-merge-request
       changes: *code-qa-patterns
       when: manual
+      allow_failure: true
 
 .reports:schedule-dast:
   rules:
@@ -470,59 +492,74 @@
 ################
 # Review rules #
 ################
-.review:rules:mr-and-schedule-auto:
+.review:rules:build-qa-image:
   rules:
     - <<: *if-dot-com-gitlab-org-merge-request
       changes: *code-qa-patterns
-      when: on_success
     - <<: *if-dot-com-gitlab-org-schedule
-      when: on_success
+
+.review:rules:review-build-cng:
+  rules:
+    - <<: *if-dot-com-gitlab-org-merge-request
+      changes: *ci-patterns
+    - <<: *if-dot-com-gitlab-org-merge-request
+      changes: *frontend-patterns
+    - <<: *if-dot-com-gitlab-org-merge-request
+      changes: *code-qa-patterns
+      when: manual
+      allow_failure: true
+    - <<: *if-dot-com-gitlab-org-schedule
 
 .review:rules:mr-and-schedule-auto-if-frontend-manual-otherwise:
   rules:
+    - <<: *if-dot-com-gitlab-org-merge-request
+      changes: *ci-patterns
     - <<: *if-dot-com-gitlab-org-merge-request
       changes: *frontend-patterns
-      when: on_success
+      allow_failure: true
     - <<: *if-dot-com-gitlab-org-merge-request
       changes: *code-qa-patterns
       when: manual
       allow_failure: true
     - <<: *if-dot-com-gitlab-org-schedule
-      when: on_success
+      allow_failure: true
 
-.review:rules:mr-only-auto:
+.review:rules:review-stop-failed-deployment:
   rules:
     - <<: *if-dot-com-gitlab-org-merge-request
       changes: *code-qa-patterns
-      when: on_success
 
-.review:rules:mr-only-auto-if-frontend-manual-otherwise:
+.review:rules:review-qa-smoke:
   rules:
+    - <<: *if-dot-com-gitlab-org-merge-request
+      changes: *ci-patterns
     - <<: *if-dot-com-gitlab-org-merge-request
       changes: *frontend-patterns
-      when: on_success
+      allow_failure: true
     - <<: *if-dot-com-gitlab-org-merge-request
       changes: *code-qa-patterns
       when: manual
+      allow_failure: true
 
 .review:rules:mr-only-manual:
   rules:
     - <<: *if-dot-com-gitlab-org-merge-request
       changes: *code-qa-patterns
       when: manual
+      allow_failure: true
 
 .review:rules:review-cleanup:
   rules:
     - <<: *if-dot-com-gitlab-org-merge-request
       changes: *code-qa-patterns
       when: manual
+      allow_failure: true
     - <<: *if-dot-com-gitlab-org-schedule
-      when: on_success
+      allow_failure: true
 
 .review:rules:danger:
   rules:
     - if: '$DANGER_GITLAB_API_TOKEN && $CI_MERGE_REQUEST_IID'
-      when: on_success
 
 ###############
 # Setup rules #
@@ -538,10 +575,11 @@
 .setup:rules:dont-interrupt-me:
   rules:
     - <<: *if-master-or-tag
-      when: on_success
+      allow_failure: true
     - <<: *if-auto-deploy-branches
-      when: on_success
+      allow_failure: true
     - when: manual
+      allow_failure: true
 
 .setup:rules:gitlab_git_test:
   rules:

.gitlab/ci/setup.gitlab-ci.yml

@@ -26,7 +26,6 @@ dont-interrupt-me:
   stage: sync
   image: alpine:edge
   interruptible: false
-  allow_failure: true
   variables:
     GIT_STRATEGY: none
   script:

doc/development/pipelines.md

@@ -132,6 +132,7 @@ and included in `rules` definitions via [YAML anchors](../ci/yaml/README.md#anch
 
 | `changes:` patterns          | Description                                                              |
 |------------------------------|--------------------------------------------------------------------------|
+| `ci-patterns`                | Only create job for CI config-related changes.                           |
 | `yaml-patterns`              | Only create job for YAML-related changes.                                |
 | `docs-patterns`              | Only create job for docs-related changes.                                |
 | `frontend-dependency-patterns` | Only create job when frontend dependencies are updated (i.e. `package.json`, and `yarn.lock`). changes.                                |
@@ -384,7 +385,7 @@ graph RL;
   subgraph "Needs `gitlab:assets:compile`";
     2_3-1 --> 1-5
   end
-  
+
   subgraph "Needs `build-qa-image` & `build-assets-image`";
     2_4-1["package-and-qa (manual)"] --> 1-2 & 2_3-1;
     click 2_4-1 "https://app.periscopedata.com/app/gitlab/652085/Engineering-Productivity---Pipeline-Build-Durations?widget=6914305&udv=0"