Skip to content

G
gitlab-ce
Commit d2d9a11e authored by Rémy Coutable's avatar Rémy Coutable
Browse files
Options

Merge branch 'test-pg' into 'master' 

Test both PostgreSQL and MySQL for the win

Closes #2582

See merge request !10156
parents df65e437 075d2bac
Hide whitespace changes
Inline Side-by-side
Showing with 212 additions and 85 deletions
.gitlab-ci.yml
View file @ d2d9a11e
image: "dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.3.3-golang-1.8-git-2.7-phantomjs-2.1-node-7.1" image: "dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.3.3-golang-1.8-git-2.7-phantomjs-2.1-node-7.1-postgresql-9.6"
cache: cache:
key: "ruby-233" key: "ruby-233"
...@@ -10,8 +10,6 @@ variables: ...@@ -10,8 +10,6 @@ variables:
RAILS_ENV: "test" RAILS_ENV: "test"
NODE_ENV: "test" NODE_ENV: "test"
SIMPLECOV: "true" SIMPLECOV: "true"
SETUP_DB: "true"
USE_BUNDLE_INSTALL: "true"
GIT_DEPTH: "20" GIT_DEPTH: "20"
PHANTOMJS_VERSION: "2.1.1" PHANTOMJS_VERSION: "2.1.1"
GET_SOURCES_ATTEMPTS: "3" GET_SOURCES_ATTEMPTS: "3"
...@@ -19,12 +17,9 @@ variables: ...@@ -19,12 +17,9 @@ variables:
KNAPSACK_SPINACH_SUITE_REPORT_PATH: knapsack/${CI_PROJECT_NAME}/spinach_report-master.json KNAPSACK_SPINACH_SUITE_REPORT_PATH: knapsack/${CI_PROJECT_NAME}/spinach_report-master.json
before_script: before_script:
- source ./scripts/prepare_build.sh
- cp config/gitlab.yml.example config/gitlab.yml
- bundle --version - bundle --version
- '[ "$USE_BUNDLE_INSTALL" != "true" ] || retry bundle install --without postgres production --jobs $(nproc) --clean $FLAGS' - . scripts/utils.sh
- retry gem install knapsack fog-aws mime-types - ./scripts/prepare_build.sh
- '[ "$SETUP_DB" != "true" ] || bundle exec rake db:drop db:create db:schema:load db:migrate add_limits_mysql'
stages: stages:
- prepare - prepare
...@@ -52,20 +47,34 @@ stages: ...@@ -52,20 +47,34 @@ stages:
paths: paths:
- knapsack/ - knapsack/
.use-db: &use-db .use-pg: &use-pg
services:
- postgres:latest
- redis:alpine
.use-mysql: &use-mysql
services: services:
- mysql:latest - mysql:latest
- redis:alpine - redis:alpine
.only-master-and-ee-or-mysql: &only-master-and-ee-or-mysql
only:
- /\-(?i)mysql$/
- master@gitlab-org/gitlab-ce
- master@gitlab/gitlabhq
- tags@gitlab-org/gitlab-ce
- tags@gitlab/gitlabhq
- //@gitlab-org/gitlab-ee
- //@gitlab/gitlab-ee
.rspec-knapsack: &rspec-knapsack .rspec-knapsack: &rspec-knapsack
stage: test stage: test
<<: *dedicated-runner <<: *dedicated-runner
<<: *use-db
script: script:
- JOB_NAME=( $CI_JOB_NAME ) - JOB_NAME=( $CI_JOB_NAME )
- export CI_NODE_INDEX=${JOB_NAME[1]} - export CI_NODE_INDEX=${JOB_NAME[-2]}
- export CI_NODE_TOTAL=${JOB_NAME[2]} - export CI_NODE_TOTAL=${JOB_NAME[-1]}
- export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${JOB_NAME[0]}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${JOB_NAME[0]}_${JOB_NAME[1]}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
- export KNAPSACK_GENERATE_REPORT=true - export KNAPSACK_GENERATE_REPORT=true
- export CACHE_CLASSES=true - export CACHE_CLASSES=true
- cp ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH} - cp ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH}
...@@ -78,15 +87,23 @@ stages: ...@@ -78,15 +87,23 @@ stages:
- knapsack/ - knapsack/
- tmp/capybara/ - tmp/capybara/
.rspec-knapsack-pg: &rspec-knapsack-pg
<<: *rspec-knapsack
<<: *use-pg
.rspec-knapsack-mysql: &rspec-knapsack-mysql
<<: *rspec-knapsack
<<: *use-mysql
<<: *only-master-and-ee-or-mysql
.spinach-knapsack: &spinach-knapsack .spinach-knapsack: &spinach-knapsack
stage: test stage: test
<<: *dedicated-runner <<: *dedicated-runner
<<: *use-db
script: script:
- JOB_NAME=( $CI_JOB_NAME ) - JOB_NAME=( $CI_JOB_NAME )
- export CI_NODE_INDEX=${JOB_NAME[1]} - export CI_NODE_INDEX=${JOB_NAME[-2]}
- export CI_NODE_TOTAL=${JOB_NAME[2]} - export CI_NODE_TOTAL=${JOB_NAME[-1]}
- export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${JOB_NAME[0]}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${JOB_NAME[0]}_${JOB_NAME[1]}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
- export KNAPSACK_GENERATE_REPORT=true - export KNAPSACK_GENERATE_REPORT=true
- export CACHE_CLASSES=true - export CACHE_CLASSES=true
- cp ${KNAPSACK_SPINACH_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH} - cp ${KNAPSACK_SPINACH_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH}
...@@ -99,6 +116,15 @@ stages: ...@@ -99,6 +116,15 @@ stages:
- knapsack/ - knapsack/
- tmp/capybara/ - tmp/capybara/
.spinach-knapsack-pg: &spinach-knapsack-pg
<<: *spinach-knapsack
<<: *use-pg
.spinach-knapsack-mysql: &spinach-knapsack-mysql
<<: *spinach-knapsack
<<: *use-mysql
<<: *only-master-and-ee-or-mysql
# Prepare and merge knapsack tests # Prepare and merge knapsack tests
knapsack: knapsack:
<<: *knapsack-state <<: *knapsack-state
...@@ -116,8 +142,8 @@ update-knapsack: ...@@ -116,8 +142,8 @@ update-knapsack:
<<: *dedicated-runner <<: *dedicated-runner
stage: post-test stage: post-test
script: script:
- scripts/merge-reports ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/rspec_node_*.json - scripts/merge-reports ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/rspec_pg_node_*.json
- scripts/merge-reports ${KNAPSACK_SPINACH_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/spinach_node_*.json - scripts/merge-reports ${KNAPSACK_SPINACH_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/spinach_pg_node_*.json
- '[[ -z ${KNAPSACK_S3_BUCKET} ]] || scripts/sync-reports put $KNAPSACK_S3_BUCKET $KNAPSACK_RSPEC_SUITE_REPORT_PATH $KNAPSACK_SPINACH_SUITE_REPORT_PATH' - '[[ -z ${KNAPSACK_S3_BUCKET} ]] || scripts/sync-reports put $KNAPSACK_S3_BUCKET $KNAPSACK_RSPEC_SUITE_REPORT_PATH $KNAPSACK_SPINACH_SUITE_REPORT_PATH'
- rm -f knapsack/${CI_PROJECT_NAME}/*_node_*.json - rm -f knapsack/${CI_PROJECT_NAME}/*_node_*.json
only: only:
...@@ -127,7 +153,7 @@ update-knapsack: ...@@ -127,7 +153,7 @@ update-knapsack:
- master@gitlab/gitlab-ee - master@gitlab/gitlab-ee
setup-test-env: setup-test-env:
<<: *use-db <<: *use-pg
<<: *dedicated-runner <<: *dedicated-runner
stage: prepare stage: prepare
script: script:
...@@ -142,37 +168,69 @@ setup-test-env: ...@@ -142,37 +168,69 @@ setup-test-env:
- public/assets - public/assets
- tmp/tests - tmp/tests
rspec 0 20: *rspec-knapsack rspec pg 0 20: *rspec-knapsack-pg
rspec 1 20: *rspec-knapsack rspec pg 1 20: *rspec-knapsack-pg
rspec 2 20: *rspec-knapsack rspec pg 2 20: *rspec-knapsack-pg
rspec 3 20: *rspec-knapsack rspec pg 3 20: *rspec-knapsack-pg
rspec 4 20: *rspec-knapsack rspec pg 4 20: *rspec-knapsack-pg
rspec 5 20: *rspec-knapsack rspec pg 5 20: *rspec-knapsack-pg
rspec 6 20: *rspec-knapsack rspec pg 6 20: *rspec-knapsack-pg
rspec 7 20: *rspec-knapsack rspec pg 7 20: *rspec-knapsack-pg
rspec 8 20: *rspec-knapsack rspec pg 8 20: *rspec-knapsack-pg
rspec 9 20: *rspec-knapsack rspec pg 9 20: *rspec-knapsack-pg
rspec 10 20: *rspec-knapsack rspec pg 10 20: *rspec-knapsack-pg
rspec 11 20: *rspec-knapsack rspec pg 11 20: *rspec-knapsack-pg
rspec 12 20: *rspec-knapsack rspec pg 12 20: *rspec-knapsack-pg
rspec 13 20: *rspec-knapsack rspec pg 13 20: *rspec-knapsack-pg
rspec 14 20: *rspec-knapsack rspec pg 14 20: *rspec-knapsack-pg
rspec 15 20: *rspec-knapsack rspec pg 15 20: *rspec-knapsack-pg
rspec 16 20: *rspec-knapsack rspec pg 16 20: *rspec-knapsack-pg
rspec 17 20: *rspec-knapsack rspec pg 17 20: *rspec-knapsack-pg
rspec 18 20: *rspec-knapsack rspec pg 18 20: *rspec-knapsack-pg
rspec 19 20: *rspec-knapsack rspec pg 19 20: *rspec-knapsack-pg
spinach 0 10: *spinach-knapsack rspec mysql 0 20: *rspec-knapsack-mysql
spinach 1 10: *spinach-knapsack rspec mysql 1 20: *rspec-knapsack-mysql
spinach 2 10: *spinach-knapsack rspec mysql 2 20: *rspec-knapsack-mysql
spinach 3 10: *spinach-knapsack rspec mysql 3 20: *rspec-knapsack-mysql
spinach 4 10: *spinach-knapsack rspec mysql 4 20: *rspec-knapsack-mysql
spinach 5 10: *spinach-knapsack rspec mysql 5 20: *rspec-knapsack-mysql
spinach 6 10: *spinach-knapsack rspec mysql 6 20: *rspec-knapsack-mysql
spinach 7 10: *spinach-knapsack rspec mysql 7 20: *rspec-knapsack-mysql
spinach 8 10: *spinach-knapsack rspec mysql 8 20: *rspec-knapsack-mysql
spinach 9 10: *spinach-knapsack rspec mysql 9 20: *rspec-knapsack-mysql
rspec mysql 10 20: *rspec-knapsack-mysql
rspec mysql 11 20: *rspec-knapsack-mysql
rspec mysql 12 20: *rspec-knapsack-mysql
rspec mysql 13 20: *rspec-knapsack-mysql
rspec mysql 14 20: *rspec-knapsack-mysql
rspec mysql 15 20: *rspec-knapsack-mysql
rspec mysql 16 20: *rspec-knapsack-mysql
rspec mysql 17 20: *rspec-knapsack-mysql
rspec mysql 18 20: *rspec-knapsack-mysql
rspec mysql 19 20: *rspec-knapsack-mysql
spinach pg 0 10: *spinach-knapsack-pg
spinach pg 1 10: *spinach-knapsack-pg
spinach pg 2 10: *spinach-knapsack-pg
spinach pg 3 10: *spinach-knapsack-pg
spinach pg 4 10: *spinach-knapsack-pg
spinach pg 5 10: *spinach-knapsack-pg
spinach pg 6 10: *spinach-knapsack-pg
spinach pg 7 10: *spinach-knapsack-pg
spinach pg 8 10: *spinach-knapsack-pg
spinach pg 9 10: *spinach-knapsack-pg
spinach mysql 0 10: *spinach-knapsack-mysql
spinach mysql 1 10: *spinach-knapsack-mysql
spinach mysql 2 10: *spinach-knapsack-mysql
spinach mysql 3 10: *spinach-knapsack-mysql
spinach mysql 4 10: *spinach-knapsack-mysql
spinach mysql 5 10: *spinach-knapsack-mysql
spinach mysql 6 10: *spinach-knapsack-mysql
spinach mysql 7 10: *spinach-knapsack-mysql
spinach mysql 8 10: *spinach-knapsack-mysql
spinach mysql 9 10: *spinach-knapsack-mysql
# Other generic tests # Other generic tests
.ruby-static-analysis: &ruby-static-analysis .ruby-static-analysis: &ruby-static-analysis
...@@ -228,24 +286,37 @@ rake ee_compat_check: ...@@ -228,24 +286,37 @@ rake ee_compat_check:
paths: paths:
- ee_compat_check/patches/*.patch - ee_compat_check/patches/*.patch
rake db:migrate:reset: .db-migrate-reset: &db-migrate-reset
stage: test stage: test
<<: *use-db
<<: *dedicated-runner <<: *dedicated-runner
script: script:
- bundle exec rake db:migrate:reset - bundle exec rake db:migrate:reset
rake db:rollback: rake pg db:migrate:reset:
<<: *db-migrate-reset
<<: *use-pg
rake mysql db:migrate:reset:
<<: *db-migrate-reset
<<: *use-mysql
.db-rollback: &db-rollback
stage: test stage: test
<<: *use-db
<<: *dedicated-runner <<: *dedicated-runner
script: script:
- bundle exec rake db:rollback STEP=120 - bundle exec rake db:rollback STEP=120
- bundle exec rake db:migrate - bundle exec rake db:migrate
rake db:seed_fu: rake pg db:rollback:
<<: *db-rollback
<<: *use-pg
rake mysql db:rollback:
<<: *db-rollback
<<: *use-mysql
.db-seed_fu: &db-seed_fu
stage: test stage: test
<<: *use-db
<<: *dedicated-runner <<: *dedicated-runner
variables: variables:
SIZE: "1" SIZE: "1"
...@@ -261,6 +332,14 @@ rake db:seed_fu: ...@@ -261,6 +332,14 @@ rake db:seed_fu:
paths: paths:
- log/development.log - log/development.log
rake pg db:seed_fu:
<<: *db-seed_fu
<<: *use-pg
rake mysql db:seed_fu:
<<: *db-seed_fu
<<: *use-mysql
rake gitlab:assets:compile: rake gitlab:assets:compile:
stage: test stage: test
<<: *dedicated-runner <<: *dedicated-runner
...@@ -285,7 +364,7 @@ rake karma: ...@@ -285,7 +364,7 @@ rake karma:
paths: paths:
- vendor/ruby - vendor/ruby
stage: test stage: test
<<: *use-db <<: *use-pg
<<: *dedicated-runner <<: *dedicated-runner
variables: variables:
BABEL_ENV: "coverage" BABEL_ENV: "coverage"
...@@ -342,9 +421,8 @@ bundler:audit: ...@@ -342,9 +421,8 @@ bundler:audit:
script: script:
- "bundle exec bundle-audit check --update --ignore CVE-2016-4658" - "bundle exec bundle-audit check --update --ignore CVE-2016-4658"
migration paths: .migration-paths: &migration-paths
stage: test stage: test
<<: *use-db
<<: *dedicated-runner <<: *dedicated-runner
variables: variables:
SETUP_DB: "false" SETUP_DB: "false"
...@@ -356,13 +434,21 @@ migration paths: ...@@ -356,13 +434,21 @@ migration paths:
script: script:
- git fetch origin v8.14.10 - git fetch origin v8.14.10
- git checkout -f FETCH_HEAD - git checkout -f FETCH_HEAD
- bundle install --without postgres production --jobs $(nproc) $FLAGS --retry=3 - bundle install $BUNDLE_INSTALL_FLAGS
- bundle exec rake db:drop db:create db:schema:load db:seed_fu - bundle exec rake db:drop db:create db:schema:load db:seed_fu
- git checkout $CI_COMMIT_SHA - git checkout $CI_COMMIT_SHA
- bundle install --without postgres production --jobs $(nproc) $FLAGS --retry=3 - bundle install $BUNDLE_INSTALL_FLAGS
- source scripts/prepare_build.sh - . scripts/prepare_build.sh
- bundle exec rake db:migrate - bundle exec rake db:migrate
migration pg paths:
<<: *migration-paths
<<: *use-pg
migration mysql paths:
<<: *migration-paths
<<: *use-mysql
coverage: coverage:
stage: post-test stage: post-test
services: [] services: []
...@@ -409,7 +495,7 @@ trigger_docs: ...@@ -409,7 +495,7 @@ trigger_docs:
before_script: before_script:
- apk update && apk add curl - apk update && apk add curl
variables: variables:
GIT_STRATEGY: none GIT_STRATEGY: "none"
cache: {} cache: {}
artifacts: {} artifacts: {}
script: script:
......
config/database.yml.mysql
View file @ d2d9a11e
...@@ -25,6 +25,7 @@ development: ...@@ -25,6 +25,7 @@ development:
pool: 5 pool: 5
username: root username: root
password: "secure password" password: "secure password"
# host: localhost
# socket: /tmp/mysql.sock # socket: /tmp/mysql.sock
# Warning: The database defined as "test" will be erased and # Warning: The database defined as "test" will be erased and
...@@ -39,4 +40,5 @@ test: &test ...@@ -39,4 +40,5 @@ test: &test
pool: 5 pool: 5
username: root username: root
password: password:
# host: localhost
# socket: /tmp/mysql.sock # socket: /tmp/mysql.sock
config/database.yml.postgresql
View file @ d2d9a11e
...@@ -9,7 +9,7 @@ production: ...@@ -9,7 +9,7 @@ production:
# username: git # username: git
# password: # password:
# host: localhost # host: localhost
# port: 5432 # port: 5432
# #
# Development specific # Development specific
...@@ -21,6 +21,7 @@ development: ...@@ -21,6 +21,7 @@ development:
pool: 5 pool: 5
username: postgres username: postgres
password: password:
# host: localhost
# #
# Staging specific # Staging specific
...@@ -32,6 +33,7 @@ staging: ...@@ -32,6 +33,7 @@ staging:
pool: 5 pool: 5
username: postgres username: postgres
password: password:
# host: localhost
# Warning: The database defined as "test" will be erased and # Warning: The database defined as "test" will be erased and
# re-generated from your development database when you run "rake". # re-generated from your development database when you run "rake".
...@@ -43,3 +45,4 @@ test: &test ...@@ -43,3 +45,4 @@ test: &test
pool: 5 pool: 5
username: postgres username: postgres
password: password:
# host: localhost
config/initializers/rspec_profiling.rb
View file @ d2d9a11e
...@@ -38,7 +38,7 @@ if Rails.env.test? ...@@ -38,7 +38,7 @@ if Rails.env.test?
end end
end end
if ENV.has_key?('CI') if ENV.has_key?('CI') && ENV['GITLAB_DATABASE'] == 'postgresql'
RspecProfiling::VCS::Git.prepend(RspecProfilingExt::Git) RspecProfiling::VCS::Git.prepend(RspecProfilingExt::Git)
RspecProfiling::Run.prepend(RspecProfilingExt::Run) RspecProfiling::Run.prepend(RspecProfilingExt::Run)
end end
......
scripts/prepare_build.sh
View file @ d2d9a11e
#!/bin/sh #!/bin/sh
retry() { . scripts/utils.sh
if eval "$@"; then
return 0 export SETUP_DB=${SETUP_DB:-true}
fi export USE_BUNDLE_INSTALL=${USE_BUNDLE_INSTALL:-true}
export BUNDLE_INSTALL_FLAGS="--without production --jobs $(nproc) --path vendor --retry 3 --quiet"
# Determine the database by looking at the job name.
# For example, we'll get pg if the job is `rspec pg 19 20`
export GITLAB_DATABASE=$(echo $CI_JOB_NAME | cut -f2 -d' ')
# This would make the default database postgresql, and we could also use
# pg to mean postgresql.
if [ "$GITLAB_DATABASE" != 'mysql' ]; then
export GITLAB_DATABASE='postgresql'
fi
cp config/database.yml.$GITLAB_DATABASE config/database.yml
for i in 2 1; do if [ "$GITLAB_DATABASE" = 'postgresql' ]; then
sleep 3s sed -i 's/# host:.*/host: postgres/g' config/database.yml
echo "Retrying $i..." else # Assume it's mysql
if eval "$@"; then sed -i 's/username:.*/username: root/g' config/database.yml
return 0 sed -i 's/password:.*/password:/g' config/database.yml
fi sed -i 's/# host:.*/host: mysql/g' config/database.yml
done fi
return 1
}
cp config/database.yml.mysql config/database.yml
sed -i 's/username:.*/username: root/g' config/database.yml
sed -i 's/password:.*/password:/g' config/database.yml
sed -i 's/# socket:.*/host: mysql/g' config/database.yml
cp config/resque.yml.example config/resque.yml cp config/resque.yml.example config/resque.yml
sed -i 's/localhost/redis/g' config/resque.yml sed -i 's/localhost/redis/g' config/resque.yml
export FLAGS="--path vendor --retry 3 --quiet" cp config/gitlab.yml.example config/gitlab.yml
if [ "$USE_BUNDLE_INSTALL" != "false" ]; then
retry bundle install --clean $BUNDLE_INSTALL_FLAGS
fi
# Only install knapsack after bundle install! Otherwise oddly some native
# gems could not be found under some circumstance. No idea why, hours wasted.
retry gem install knapsack fog-aws mime-types
if [ "$SETUP_DB" != "false" ]; then
bundle exec rake db:drop db:create db:schema:load db:migrate
if [ "$GITLAB_DATABASE" = "mysql" ]; then
bundle exec rake add_limits_mysql
fi
fi
scripts/utils.sh 0 → 100644
View file @ d2d9a11e
retry() {
if eval "$@"; then
return 0
fi
for i in 2 1; do
sleep 3s
echo "Retrying $i..."
if eval "$@"; then
return 0
fi
done
return 1
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7