Rearrange jobs in rails.gitlab-ci.yml

Group jobs into:
- job base specs: e.g `*-base:`
- parallel specs: e.g `*-parallel:`
- scheduling specs: actual jobs with scheduling rules

.gitlab/ci/global.gitlab-ci.yml

@@ -72,6 +72,15 @@
   variables:
     POSTGRES_HOST_AUTH_METHOD: trust
 
+.use-pg12:
+  image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.6-golang-1.14-git-2.27-lfs-2.9-chrome-83-node-12.x-yarn-1.21-postgresql-12-graphicsmagick-1.3.34"
+  services:
+    - name: postgres:12
+      command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
+    - name: redis:alpine
+  variables:
+    POSTGRES_HOST_AUTH_METHOD: trust
+
 .use-pg11-ee:
   image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.6-golang-1.14-git-2.27-lfs-2.9-chrome-83-node-12.x-yarn-1.21-postgresql-11-graphicsmagick-1.3.34"
   services:
@@ -82,6 +91,16 @@
   variables:
     POSTGRES_HOST_AUTH_METHOD: trust
 
+.use-pg12-ee:
+  image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.6-golang-1.14-git-2.27-lfs-2.9-chrome-83-node-12.x-yarn-1.21-postgresql-12-graphicsmagick-1.3.34"
+  services:
+    - name: postgres:12
+      command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
+    - name: redis:alpine
+    - name: elasticsearch:6.4.2
+  variables:
+    POSTGRES_HOST_AUTH_METHOD: trust
+
 .use-kaniko:
   image:
     name: gcr.io/kaniko-project/executor:debug-v0.20.0

.gitlab/ci/rails.gitlab-ci.yml

+######################
+# rspec job base specs
 .rails-job-base:
   extends:
     - .default-retry
     - .default-before_script
     - .rails-cache
 
+.rspec-base:
+  extends: .rails-job-base
+  stage: test
+  needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets"]
+  script:
+    - run_timed_command "scripts/gitaly-test-build"
+    - run_timed_command "scripts/gitaly-test-spawn"
+    - source scripts/rspec_helpers.sh
+    - rspec_paralellized_job "--tag ~quarantine --tag ~geo --tag ~level:migration"
+  artifacts:
+    expire_in: 31d
+    when: always
+    paths:
+      - coverage/
+      - knapsack/
+      - rspec_flaky/
+      - rspec_profiling/
+      - tmp/capybara/
+      - tmp/memory_test/
+      - log/*.log
+    reports:
+      junit: junit_rspec.xml
+
+.rspec-base-migration:
+  extends: .rails:rules:ee-and-foss-migration
+  script:
+    - run_timed_command "scripts/gitaly-test-build"
+    - run_timed_command "scripts/gitaly-test-spawn"
+    - source scripts/rspec_helpers.sh
+    - rspec_paralellized_job "--tag ~quarantine --tag ~geo --tag level:migration"
+
+.rspec-base-pg11:
+  extends:
+    - .rspec-base
+    - .use-pg11
+
+.rspec-base-pg12:
+  extends:
+    - .rspec-base
+    - .use-pg12
+
+.rspec-base-pg11-as-if-foss:
+  extends:
+    - .rspec-base
+    - .as-if-foss
+    - .use-pg11
+  needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets as-if-foss"]
+
+.rspec-ee-base-pg11:
+  extends:
+    - .rspec-base
+    - .use-pg11-ee
+
+.rspec-ee-base-pg12:
+  extends:
+    - .rspec-base
+    - .use-pg12-ee
+
+.rspec-ee-base-geo:
+  extends: .rspec-base
+  script:
+    - run_timed_command "scripts/gitaly-test-build"
+    - run_timed_command "scripts/gitaly-test-spawn"
+    - source scripts/rspec_helpers.sh
+    - scripts/prepare_postgres_fdw.sh
+    - rspec_paralellized_job "--tag ~quarantine --tag geo"
+
+.rspec-ee-base-geo-pg11:
+  extends:
+    - .rspec-ee-base-geo
+    - .use-pg11-ee
+
+.rspec-ee-base-geo-pg12:
+  extends:
+    - .rspec-ee-base-geo
+    - .use-pg12-ee
+
+.db-job-base:
+  extends:
+    - .rails-job-base
+    - .rails:rules:ee-and-foss-migration
+    - .use-pg11
+  stage: test
+  needs: ["setup-test-env"]
+# rspec job base specs
+######################
+
+############################
+# rspec job parallel configs
+.rspec-migration-parallel:
+  parallel: 5
+
+.rspec-ee-migration-parallel:
+  parallel: 2
+
+.rspec-unit-parallel:
+  parallel: 20
+
+.rspec-ee-unit-parallel:
+  parallel: 10
+
+.rspec-ee-unit-geo-parallel:
+  parallel: 2
+
+.rspec-integration-parallel:
+  parallel: 8
+
+.rspec-ee-integration-parallel:
+  parallel: 4
+
+.rspec-system-parallel:
+  parallel: 24
+
+.rspec-ee-system-parallel:
+  parallel: 6
+# rspec job parallel configs
+############################
+
 #######################################################
 # EE/FOSS: default refs (MRs, master, schedules) jobs #
 setup-test-env:
@@ -86,65 +206,29 @@ downtime_check:
   script:
     - bundle exec rake downtime_check
 
-.rspec-base:
-  extends: .rails-job-base
-  stage: test
-  needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets"]
-  script:
-    - run_timed_command "scripts/gitaly-test-build"
-    - run_timed_command "scripts/gitaly-test-spawn"
-    - source scripts/rspec_helpers.sh
-    - rspec_paralellized_job "--tag ~quarantine --tag ~geo --tag ~level:migration"
-  artifacts:
-    expire_in: 31d
-    when: always
-    paths:
-      - coverage/
-      - knapsack/
-      - rspec_flaky/
-      - rspec_profiling/
-      - tmp/capybara/
-      - tmp/memory_test/
-      - log/*.log
-    reports:
-      junit: junit_rspec.xml
-
-.rspec-base-pg11:
-  extends:
-    - .rspec-base
-    - .use-pg11
-
-.rspec-base-migration:
-  extends: .rails:rules:ee-and-foss-migration
-  script:
-    - run_timed_command "scripts/gitaly-test-build"
-    - run_timed_command "scripts/gitaly-test-spawn"
-    - source scripts/rspec_helpers.sh
-    - rspec_paralellized_job "--tag ~quarantine --tag ~geo --tag level:migration"
-
 rspec migration pg11:
   extends:
     - .rspec-base-pg11
     - .rspec-base-migration
-  parallel: 5
+    - .rspec-migration-parallel
 
 rspec unit pg11:
   extends:
     - .rspec-base-pg11
     - .rails:rules:ee-and-foss-unit
-  parallel: 20
+    - .rspec-unit-parallel
 
 rspec integration pg11:
   extends:
     - .rspec-base-pg11
     - .rails:rules:ee-and-foss-integration
-  parallel: 8
+    - .rspec-integration-parallel
 
 rspec system pg11:
   extends:
     - .rspec-base-pg11
     - .rails:rules:ee-and-foss-system
-  parallel: 24
+    - .rspec-system-parallel
 
 rspec fast_spec_helper:
   extends:
@@ -153,14 +237,6 @@ rspec fast_spec_helper:
   script:
     - bin/rspec spec/fast_spec_helper.rb
 
-.db-job-base:
-  extends:
-    - .rails-job-base
-    - .rails:rules:ee-and-foss-migration
-    - .use-pg11
-  stage: test
-  needs: ["setup-test-env"]
-
 db:migrate:reset:
   extends: .db-job-base
   script:
@@ -263,87 +339,61 @@ rspec:coverage:
 
 ##################################################
 # EE: default refs (MRs, master, schedules) jobs #
-.rspec-base-pg11-as-if-foss:
-  extends:
-    - .rspec-base
-    - .as-if-foss
-    - .use-pg11
-  needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets as-if-foss"]
-
 rspec migration pg11-as-if-foss:
   extends:
     - .rspec-base-pg11-as-if-foss
     - .rspec-base-migration
     - .rails:rules:as-if-foss-migration
-  parallel: 5
+    - .rspec-migration-parallel
 
 rspec unit pg11-as-if-foss:
   extends:
     - .rspec-base-pg11-as-if-foss
     - .rails:rules:as-if-foss-unit
-  parallel: 20
+    - .rspec-unit-parallel
 
 rspec integration pg11-as-if-foss:
   extends:
     - .rspec-base-pg11-as-if-foss
     - .rails:rules:as-if-foss-integration
-  parallel: 8
+    - .rspec-integration-parallel
 
 rspec system pg11-as-if-foss:
   extends:
     - .rspec-base-pg11-as-if-foss
     - .rails:rules:as-if-foss-system
-  parallel: 24
-
-.rspec-ee-base-pg11:
-  extends:
-    - .rspec-base
-    - .use-pg11-ee
+    - .rspec-system-parallel
 
 rspec-ee migration pg11:
   extends:
     - .rspec-ee-base-pg11
     - .rspec-base-migration
     - .rails:rules:ee-only-migration
-  parallel: 2
+    - .rspec-ee-migration-parallel
 
 rspec-ee unit pg11:
   extends:
     - .rspec-ee-base-pg11
     - .rails:rules:ee-only-unit
-  parallel: 10
+    - .rspec-ee-unit-parallel
 
 rspec-ee integration pg11:
   extends:
     - .rspec-ee-base-pg11
     - .rails:rules:ee-only-integration
-  parallel: 4
+    - .rspec-ee-integration-parallel
 
 rspec-ee system pg11:
   extends:
     - .rspec-ee-base-pg11
     - .rails:rules:ee-only-system
-  parallel: 6
-
-.rspec-ee-base-geo:
-  extends: .rspec-base
-  script:
-    - run_timed_command "scripts/gitaly-test-build"
-    - run_timed_command "scripts/gitaly-test-spawn"
-    - source scripts/rspec_helpers.sh
-    - scripts/prepare_postgres_fdw.sh
-    - rspec_paralellized_job "--tag ~quarantine --tag geo"
-
-.rspec-ee-base-geo-pg11:
-  extends:
-    - .rspec-ee-base-geo
-    - .use-pg11-ee
+    - .rspec-ee-system-parallel
 
 rspec-ee unit pg11 geo:
   extends:
     - .rspec-ee-base-geo-pg11
     - .rails:rules:ee-only-unit
-  parallel: 2
+    - .rspec-ee-unit-geo-parallel
 
 rspec-ee integration pg11 geo:
   extends:
@@ -365,6 +415,80 @@ db:rollback geo:
 # EE: default refs (MRs, master, schedules) jobs #
 ##################################################
 
+##########################################
+# EE/FOSS: master nightly scheduled jobs #
+rspec migration pg12:
+  extends:
+    - .rspec-base-pg12
+    - .rspec-base-migration
+    - .rails:rules:master-schedule-nightly--code-backstage
+    - .rspec-migration-parallel
+
+rspec unit pg12:
+  extends:
+    - .rspec-base-pg12
+    - .rails:rules:master-schedule-nightly--code-backstage
+    - .rspec-unit-parallel
+
+rspec integration pg12:
+  extends:
+    - .rspec-base-pg12
+    - .rails:rules:master-schedule-nightly--code-backstage
+    - .rspec-integration-parallel
+
+rspec system pg12:
+  extends:
+    - .rspec-base-pg12
+    - .rails:rules:master-schedule-nightly--code-backstage
+    - .rspec-system-parallel
+# EE/FOSS: master nightly scheduled jobs #
+##########################################
+
+#####################################
+# EE: master nightly scheduled jobs #
+rspec-ee migration pg12:
+  extends:
+    - .rspec-ee-base-pg12
+    - .rspec-base-migration
+    - .rails:rules:master-schedule-nightly--code-backstage-ee-only
+    - .rspec-ee-migration-parallel
+
+rspec-ee unit pg12:
+  extends:
+    - .rspec-ee-base-pg12
+    - .rails:rules:master-schedule-nightly--code-backstage-ee-only
+    - .rspec-ee-unit-parallel
+
+rspec-ee integration pg12:
+  extends:
+    - .rspec-ee-base-pg12
+    - .rails:rules:master-schedule-nightly--code-backstage-ee-only
+    - .rspec-ee-integration-parallel
+
+rspec-ee system pg12:
+  extends:
+    - .rspec-ee-base-pg12
+    - .rails:rules:master-schedule-nightly--code-backstage-ee-only
+    - .rspec-ee-system-parallel
+
+rspec-ee unit pg12 geo:
+  extends:
+    - .rspec-ee-base-geo-pg12
+    - .rails:rules:master-schedule-nightly--code-backstage-ee-only
+    - .rspec-ee-unit-geo-parallel
+
+rspec-ee integration pg12 geo:
+  extends:
+    - .rspec-ee-base-geo-pg12
+    - .rails:rules:master-schedule-nightly--code-backstage-ee-only
+
+rspec-ee system pg12 geo:
+  extends:
+    - .rspec-ee-base-geo-pg12
+    - .rails:rules:master-schedule-nightly--code-backstage-ee-only
+# EE: master nightly scheduled jobs #
+#####################################
+
 ##################################################
 # EE: Canonical MR pipelines
 rspec foss-impact:
@@ -383,5 +507,5 @@ rspec foss-impact:
     paths:
       - tmp/matching_foss_tests.txt
       - tmp/capybara/
-# EE: Merge Request pipelines
+# EE: Canonical MR pipelines
 ##################################################

.gitlab/ci/rules.gitlab-ci.yml

@@ -116,6 +116,8 @@
   - "{,ee/}{app/channels,app/controllers,app/finders,app/graphql,app/helpers,app/mailers,app/models,app/policies,app/presenters,app/serializers,app/services,app/uploaders,app/validators,app/views,app/workers}/**/*"
   - "{,ee/}{bin,cable,config,db,lib}/**/*"
   - "{,ee/}spec/**/*.rb"
+  - ".gitlab-ci.yml"
+  - ".gitlab/ci/**/*"
 
 .db-patterns: &db-patterns
   - "{,ee/}{,spec/}{db,migrations}/**/*"
@@ -546,6 +548,20 @@
       when: never
     - <<: *if-master-schedule-2-hourly
 
+.rails:rules:master-schedule-nightly--code-backstage:
+  rules:
+    - <<: *if-master-schedule-nightly
+    - <<: *if-merge-request
+      changes: [".gitlab/ci/rails.gitlab-ci.yml"]
+
+.rails:rules:master-schedule-nightly--code-backstage-ee-only:
+  rules:
+    - <<: *if-not-ee
+      when: never
+    - <<: *if-master-schedule-nightly
+    - <<: *if-merge-request
+      changes: [".gitlab/ci/rails.gitlab-ci.yml"]
+
 ##################
 # Releases rules #
 ##################

doc/development/pipelines.md

@@ -370,18 +370,17 @@ and are as follows:
 | MRs | 11 |
 | `master` (non-scheduled pipelines) | 11 |
 | 2-hourly scheduled pipelines | 11 |
+| `nightly` scheduled pipelines | 11, 12 |
 
 #### Long-term plan
 
 We follow the [PostgreSQL versions shipped with Omnibus GitLab](https://docs.gitlab.com/omnibus/package-information/postgresql_versions.html):
 
-| PostgreSQL version | 12.10 (April 2020) | 13.0 (May 2020) | 13.1 (June 2020) | 13.2 (July 2020) | 13.3 (August 2020) | 13.4, 13.5   | 13.6 (November 2020) | 14.0 (May 2021?) |
-| ------ | ------------------ | --------------- | ---------------- | ---------------- | ------------------ | ------------ | -------------------- | ---------------- |
-| PG9.6  | MRs/`master`/`2-hour`/`nightly` | -               | -                | -                | -                  | -            | -                    | -                |
-| PG10   | `nightly`           | -               | -                | -                | -                  | -            | -                    | -                |
-| PG11   | `master`/`2-hour` | MRs/`master`/`2-hour`/`nightly` | MRs/`master`/`2-hour`/`nightly` | MRs/`master`/`2-hour`/`nightly` | MRs/`master`/`2-hour`/`nightly` | MRs/`master`/`2-hour`/`nightly` | `nightly`              | -                |
-| PG12   | -                  | -               | -                | -                | `master`/`2-hour` | `master`/`2-hour` | MRs/`master`/`2-hour`/`nightly`         | `master`/`2-hour` |
-| PG13   | -                  | -               | -                | -                | -                  | -            | -                    | MRs/`master`/`2-hour`/`nightly`     |
+| PostgreSQL version | 13.0 (May 2020) | 13.1 (June 2020) | 13.2 (July 2020) | 13.3 (August 2020) | 13.4, 13.5   | 13.6 (November 2020) | 14.0 (May 2021?) |
+| ------ | --------------- | ---------------- | ---------------- | ------------------ | ------------ | -------------------- | ---------------- |
+| PG11   | MRs/`master`/`2-hour`/`nightly` | MRs/`master`/`2-hour`/`nightly` | MRs/`master`/`2-hour`/`nightly` | MRs/`master`/`2-hour`/`nightly` | MRs/`master`/`2-hour`/`nightly` | `nightly` | - |
+| PG12   | - | - | `nightly` | `2-hour`/`nightly` | `2-hour`/`nightly` | MRs/`2-hour`/`nightly` | `2-hour`/`nightly`     |
+| PG13   | - | - | -         | -                  | -                  | -                      | MRs/`2-hour`/`nightly` |
 
 ### Test jobs
 

scripts/prepare_build.sh

@@ -11,7 +11,7 @@ if [ "$USE_BUNDLE_INSTALL" != "false" ]; then
   # When we test multiple versions of PG in the same pipeline, we have a single `setup-test-env`
   # job but the `pg` gem needs to be rebuilt since it includes extensions (https://guides.rubygems.org/gems-with-extensions).
   # Uncomment the following line if multiple versions of PG are tested in the same pipeline.
-  # run_timed_command "bundle pristine pg"
+  run_timed_command "bundle pristine pg"
 fi
 
 # Only install knapsack after bundle install! Otherwise oddly some native

spec/support/helpers/trigger_helpers.rb

@@ -27,7 +27,10 @@ module TriggerHelpers
     expected_timing, expected_events = fires_on.first
     expect(timing).to eq(expected_timing.to_s)
     expect(events).to match_array(Array.wrap(expected_events))
-    expect(definition).to eq("execute procedure #{fn_name}()")
+
+    # TODO: Update CREATE TRIGGER syntax to use EXECUTE FUNCTION
+    # https://gitlab.com/gitlab-org/gitlab/-/issues/227089
+    expect(definition).to match(%r{execute (?:procedure|function) #{fn_name}()})
   end
 
   private