Skip to content

G
gitlab-ce
Commit c76094e3 authored by Albert Salim's avatar Albert Salim Committed by Rémy Coutable
Browse files
Options

Rearrange jobs in rails.gitlab-ci.yml 

Group jobs into:
- job base specs: e.g `*-base:`
- parallel specs: e.g `*-parallel:`
- scheduling specs: actual jobs with scheduling rules
parent 6a9c2b82
Hide whitespace changes
Inline Side-by-side
Showing with 254 additions and 93 deletions
.gitlab/ci/global.gitlab-ci.yml
View file @ c76094e3
......@@ -72,6 +72,15 @@
variables:
POSTGRES_HOST_AUTH_METHOD: trust
.use-pg12:
image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.6-golang-1.14-git-2.27-lfs-2.9-chrome-83-node-12.x-yarn-1.21-postgresql-12-graphicsmagick-1.3.34"
services:
- name: postgres:12
command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
- name: redis:alpine
variables:
POSTGRES_HOST_AUTH_METHOD: trust
.use-pg11-ee:
image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.6-golang-1.14-git-2.27-lfs-2.9-chrome-83-node-12.x-yarn-1.21-postgresql-11-graphicsmagick-1.3.34"
services:
......@@ -82,6 +91,16 @@
variables:
POSTGRES_HOST_AUTH_METHOD: trust
.use-pg12-ee:
image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.6-golang-1.14-git-2.27-lfs-2.9-chrome-83-node-12.x-yarn-1.21-postgresql-12-graphicsmagick-1.3.34"
services:
- name: postgres:12
command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
- name: redis:alpine
- name: elasticsearch:6.4.2
variables:
POSTGRES_HOST_AUTH_METHOD: trust
.use-kaniko:
image:
name: gcr.io/kaniko-project/executor:debug-v0.20.0
......
.gitlab/ci/rails.gitlab-ci.yml
View file @ c76094e3
######################
# rspec job base specs
.rails-job-base:
extends:
- .default-retry
- .default-before_script
- .rails-cache
.rspec-base:
extends: .rails-job-base
stage: test
needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets"]
script:
- run_timed_command "scripts/gitaly-test-build"
- run_timed_command "scripts/gitaly-test-spawn"
- source scripts/rspec_helpers.sh
- rspec_paralellized_job "--tag ~quarantine --tag ~geo --tag ~level:migration"
artifacts:
expire_in: 31d
when: always
paths:
- coverage/
- knapsack/
- rspec_flaky/
- rspec_profiling/
- tmp/capybara/
- tmp/memory_test/
- log/*.log
reports:
junit: junit_rspec.xml
.rspec-base-migration:
extends: .rails:rules:ee-and-foss-migration
script:
- run_timed_command "scripts/gitaly-test-build"
- run_timed_command "scripts/gitaly-test-spawn"
- source scripts/rspec_helpers.sh
- rspec_paralellized_job "--tag ~quarantine --tag ~geo --tag level:migration"
.rspec-base-pg11:
extends:
- .rspec-base
- .use-pg11
.rspec-base-pg12:
extends:
- .rspec-base
- .use-pg12
.rspec-base-pg11-as-if-foss:
extends:
- .rspec-base
- .as-if-foss
- .use-pg11
needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets as-if-foss"]
.rspec-ee-base-pg11:
extends:
- .rspec-base
- .use-pg11-ee
.rspec-ee-base-pg12:
extends:
- .rspec-base
- .use-pg12-ee
.rspec-ee-base-geo:
extends: .rspec-base
script:
- run_timed_command "scripts/gitaly-test-build"
- run_timed_command "scripts/gitaly-test-spawn"
- source scripts/rspec_helpers.sh
- scripts/prepare_postgres_fdw.sh
- rspec_paralellized_job "--tag ~quarantine --tag geo"
.rspec-ee-base-geo-pg11:
extends:
- .rspec-ee-base-geo
- .use-pg11-ee
.rspec-ee-base-geo-pg12:
extends:
- .rspec-ee-base-geo
- .use-pg12-ee
.db-job-base:
extends:
- .rails-job-base
- .rails:rules:ee-and-foss-migration
- .use-pg11
stage: test
needs: ["setup-test-env"]
# rspec job base specs
######################
############################
# rspec job parallel configs
.rspec-migration-parallel:
parallel: 5
.rspec-ee-migration-parallel:
parallel: 2
.rspec-unit-parallel:
parallel: 20
.rspec-ee-unit-parallel:
parallel: 10
.rspec-ee-unit-geo-parallel:
parallel: 2
.rspec-integration-parallel:
parallel: 8
.rspec-ee-integration-parallel:
parallel: 4
.rspec-system-parallel:
parallel: 24
.rspec-ee-system-parallel:
parallel: 6
# rspec job parallel configs
############################
#######################################################
# EE/FOSS: default refs (MRs, master, schedules) jobs #
setup-test-env:
......@@ -86,65 +206,29 @@ downtime_check:
script:
- bundle exec rake downtime_check
.rspec-base:
extends: .rails-job-base
stage: test
needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets"]
script:
- run_timed_command "scripts/gitaly-test-build"
- run_timed_command "scripts/gitaly-test-spawn"
- source scripts/rspec_helpers.sh
- rspec_paralellized_job "--tag ~quarantine --tag ~geo --tag ~level:migration"
artifacts:
expire_in: 31d
when: always
paths:
- coverage/
- knapsack/
- rspec_flaky/
- rspec_profiling/
- tmp/capybara/
- tmp/memory_test/
- log/*.log
reports:
junit: junit_rspec.xml
.rspec-base-pg11:
extends:
- .rspec-base
- .use-pg11
.rspec-base-migration:
extends: .rails:rules:ee-and-foss-migration
script:
- run_timed_command "scripts/gitaly-test-build"
- run_timed_command "scripts/gitaly-test-spawn"
- source scripts/rspec_helpers.sh
- rspec_paralellized_job "--tag ~quarantine --tag ~geo --tag level:migration"
rspec migration pg11:
extends:
- .rspec-base-pg11
- .rspec-base-migration
parallel: 5
- .rspec-migration-parallel
rspec unit pg11:
extends:
- .rspec-base-pg11
- .rails:rules:ee-and-foss-unit
parallel: 20
- .rspec-unit-parallel
rspec integration pg11:
extends:
- .rspec-base-pg11
- .rails:rules:ee-and-foss-integration
parallel: 8
- .rspec-integration-parallel
rspec system pg11:
extends:
- .rspec-base-pg11
- .rails:rules:ee-and-foss-system
parallel: 24
- .rspec-system-parallel
rspec fast_spec_helper:
extends:
......@@ -153,14 +237,6 @@ rspec fast_spec_helper:
script:
- bin/rspec spec/fast_spec_helper.rb
.db-job-base:
extends:
- .rails-job-base
- .rails:rules:ee-and-foss-migration
- .use-pg11
stage: test
needs: ["setup-test-env"]
db:migrate:reset:
extends: .db-job-base
script:
......@@ -263,87 +339,61 @@ rspec:coverage:
##################################################
# EE: default refs (MRs, master, schedules) jobs #
.rspec-base-pg11-as-if-foss:
extends:
- .rspec-base
- .as-if-foss
- .use-pg11
needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets as-if-foss"]
rspec migration pg11-as-if-foss:
extends:
- .rspec-base-pg11-as-if-foss
- .rspec-base-migration
- .rails:rules:as-if-foss-migration
parallel: 5
- .rspec-migration-parallel
rspec unit pg11-as-if-foss:
extends:
- .rspec-base-pg11-as-if-foss
- .rails:rules:as-if-foss-unit
parallel: 20
- .rspec-unit-parallel
rspec integration pg11-as-if-foss:
extends:
- .rspec-base-pg11-as-if-foss
- .rails:rules:as-if-foss-integration
parallel: 8
- .rspec-integration-parallel
rspec system pg11-as-if-foss:
extends:
- .rspec-base-pg11-as-if-foss
- .rails:rules:as-if-foss-system
parallel: 24
.rspec-ee-base-pg11:
extends:
- .rspec-base
- .use-pg11-ee
- .rspec-system-parallel
rspec-ee migration pg11:
extends:
- .rspec-ee-base-pg11
- .rspec-base-migration
- .rails:rules:ee-only-migration
parallel: 2
- .rspec-ee-migration-parallel
rspec-ee unit pg11:
extends:
- .rspec-ee-base-pg11
- .rails:rules:ee-only-unit
parallel: 10
- .rspec-ee-unit-parallel
rspec-ee integration pg11:
extends:
- .rspec-ee-base-pg11
- .rails:rules:ee-only-integration
parallel: 4
- .rspec-ee-integration-parallel
rspec-ee system pg11:
extends:
- .rspec-ee-base-pg11
- .rails:rules:ee-only-system
parallel: 6
.rspec-ee-base-geo:
extends: .rspec-base
script:
- run_timed_command "scripts/gitaly-test-build"
- run_timed_command "scripts/gitaly-test-spawn"
- source scripts/rspec_helpers.sh
- scripts/prepare_postgres_fdw.sh
- rspec_paralellized_job "--tag ~quarantine --tag geo"
.rspec-ee-base-geo-pg11:
extends:
- .rspec-ee-base-geo
- .use-pg11-ee
- .rspec-ee-system-parallel
rspec-ee unit pg11 geo:
extends:
- .rspec-ee-base-geo-pg11
- .rails:rules:ee-only-unit
parallel: 2
- .rspec-ee-unit-geo-parallel
rspec-ee integration pg11 geo:
extends:
......@@ -365,6 +415,80 @@ db:rollback geo:
# EE: default refs (MRs, master, schedules) jobs #
##################################################
##########################################
# EE/FOSS: master nightly scheduled jobs #
rspec migration pg12:
extends:
- .rspec-base-pg12
- .rspec-base-migration
- .rails:rules:master-schedule-nightly--code-backstage
- .rspec-migration-parallel
rspec unit pg12:
extends:
- .rspec-base-pg12
- .rails:rules:master-schedule-nightly--code-backstage
- .rspec-unit-parallel
rspec integration pg12:
extends:
- .rspec-base-pg12
- .rails:rules:master-schedule-nightly--code-backstage
- .rspec-integration-parallel
rspec system pg12:
extends:
- .rspec-base-pg12
- .rails:rules:master-schedule-nightly--code-backstage
- .rspec-system-parallel
# EE/FOSS: master nightly scheduled jobs #
##########################################
#####################################
# EE: master nightly scheduled jobs #
rspec-ee migration pg12:
extends:
- .rspec-ee-base-pg12
- .rspec-base-migration
- .rails:rules:master-schedule-nightly--code-backstage-ee-only
- .rspec-ee-migration-parallel
rspec-ee unit pg12:
extends:
- .rspec-ee-base-pg12
- .rails:rules:master-schedule-nightly--code-backstage-ee-only
- .rspec-ee-unit-parallel
rspec-ee integration pg12:
extends:
- .rspec-ee-base-pg12
- .rails:rules:master-schedule-nightly--code-backstage-ee-only
- .rspec-ee-integration-parallel
rspec-ee system pg12:
extends:
- .rspec-ee-base-pg12
- .rails:rules:master-schedule-nightly--code-backstage-ee-only
- .rspec-ee-system-parallel
rspec-ee unit pg12 geo:
extends:
- .rspec-ee-base-geo-pg12
- .rails:rules:master-schedule-nightly--code-backstage-ee-only
- .rspec-ee-unit-geo-parallel
rspec-ee integration pg12 geo:
extends:
- .rspec-ee-base-geo-pg12
- .rails:rules:master-schedule-nightly--code-backstage-ee-only
rspec-ee system pg12 geo:
extends:
- .rspec-ee-base-geo-pg12
- .rails:rules:master-schedule-nightly--code-backstage-ee-only
# EE: master nightly scheduled jobs #
#####################################
##################################################
# EE: Canonical MR pipelines
rspec foss-impact:
......@@ -383,5 +507,5 @@ rspec foss-impact:
paths:
- tmp/matching_foss_tests.txt
- tmp/capybara/
# EE: Merge Request pipelines
# EE: Canonical MR pipelines
##################################################
.gitlab/ci/rules.gitlab-ci.yml
View file @ c76094e3
......@@ -116,6 +116,8 @@
- "{,ee/}{app/channels,app/controllers,app/finders,app/graphql,app/helpers,app/mailers,app/models,app/policies,app/presenters,app/serializers,app/services,app/uploaders,app/validators,app/views,app/workers}/**/*"
- "{,ee/}{bin,cable,config,db,lib}/**/*"
- "{,ee/}spec/**/*.rb"
- ".gitlab-ci.yml"
- ".gitlab/ci/**/*"
.db-patterns: &db-patterns
- "{,ee/}{,spec/}{db,migrations}/**/*"
......@@ -546,6 +548,20 @@
when: never
- <<: *if-master-schedule-2-hourly
.rails:rules:master-schedule-nightly--code-backstage:
rules:
- <<: *if-master-schedule-nightly
- <<: *if-merge-request
changes: [".gitlab/ci/rails.gitlab-ci.yml"]
.rails:rules:master-schedule-nightly--code-backstage-ee-only:
rules:
- <<: *if-not-ee
when: never
- <<: *if-master-schedule-nightly
- <<: *if-merge-request
changes: [".gitlab/ci/rails.gitlab-ci.yml"]
##################
# Releases rules #
##################
......
doc/development/pipelines.md
View file @ c76094e3
......@@ -370,18 +370,17 @@ and are as follows:
| MRs | 11 |
| `master` (non-scheduled pipelines) | 11 |
| 2-hourly scheduled pipelines | 11 |
| `nightly` scheduled pipelines | 11, 12 |
#### Long-term plan
We follow the [PostgreSQL versions shipped with Omnibus GitLab](https://docs.gitlab.com/omnibus/package-information/postgresql_versions.html):
| PostgreSQL version | 12.10 (April 2020) | 13.0 (May 2020) | 13.1 (June 2020) | 13.2 (July 2020) | 13.3 (August 2020) | 13.4, 13.5 | 13.6 (November 2020) | 14.0 (May 2021?) |
| ------ | ------------------ | --------------- | ---------------- | ---------------- | ------------------ | ------------ | -------------------- | ---------------- |
| PG9.6 | MRs/`master`/`2-hour`/`nightly` | - | - | - | - | - | - | - |
| PG10 | `nightly` | - | - | - | - | - | - | - |
| PG11 | `master`/`2-hour` | MRs/`master`/`2-hour`/`nightly` | MRs/`master`/`2-hour`/`nightly` | MRs/`master`/`2-hour`/`nightly` | MRs/`master`/`2-hour`/`nightly` | MRs/`master`/`2-hour`/`nightly` | `nightly` | - |
| PG12 | - | - | - | - | `master`/`2-hour` | `master`/`2-hour` | MRs/`master`/`2-hour`/`nightly` | `master`/`2-hour` |
| PG13 | - | - | - | - | - | - | - | MRs/`master`/`2-hour`/`nightly` |
| PostgreSQL version | 13.0 (May 2020) | 13.1 (June 2020) | 13.2 (July 2020) | 13.3 (August 2020) | 13.4, 13.5 | 13.6 (November 2020) | 14.0 (May 2021?) |
| ------ | --------------- | ---------------- | ---------------- | ------------------ | ------------ | -------------------- | ---------------- |
| PG11 | MRs/`master`/`2-hour`/`nightly` | MRs/`master`/`2-hour`/`nightly` | MRs/`master`/`2-hour`/`nightly` | MRs/`master`/`2-hour`/`nightly` | MRs/`master`/`2-hour`/`nightly` | `nightly` | - |
| PG12 | - | - | `nightly` | `2-hour`/`nightly` | `2-hour`/`nightly` | MRs/`2-hour`/`nightly` | `2-hour`/`nightly` |
| PG13 | - | - | - | - | - | - | MRs/`2-hour`/`nightly` |
### Test jobs
......
scripts/prepare_build.sh
View file @ c76094e3
......@@ -11,7 +11,7 @@ if [ "$USE_BUNDLE_INSTALL" != "false" ]; then
# When we test multiple versions of PG in the same pipeline, we have a single `setup-test-env`
# job but the `pg` gem needs to be rebuilt since it includes extensions (https://guides.rubygems.org/gems-with-extensions).
# Uncomment the following line if multiple versions of PG are tested in the same pipeline.
# run_timed_command "bundle pristine pg"
run_timed_command "bundle pristine pg"
fi
# Only install knapsack after bundle install! Otherwise oddly some native
......
spec/support/helpers/trigger_helpers.rb
View file @ c76094e3
......@@ -27,7 +27,10 @@ module TriggerHelpers
expected_timing, expected_events = fires_on.first
expect(timing).to eq(expected_timing.to_s)
expect(events).to match_array(Array.wrap(expected_events))
expect(definition).to eq("execute procedure #{fn_name}()")
# TODO: Update CREATE TRIGGER syntax to use EXECUTE FUNCTION
# https://gitlab.com/gitlab-org/gitlab/-/issues/227089
expect(definition).to match(%r{execute (?:procedure|function) #{fn_name}()})
end
private
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7