Commit 3fd0b364 authored by Douwe Maan's avatar Douwe Maan

Merge branch 'knapsack' into 'master'

Use Knapsack to evenly distribute specs

Try to distribute all tests evenly accross all nodes to reduce the testing time of GitLab CE.

This uses high amount of auto-scaled machines.


See merge request !4240
parents 981829f3 12dd7bd7
...@@ -2,7 +2,7 @@ image: "ruby:2.1" ...@@ -2,7 +2,7 @@ image: "ruby:2.1"
services: services:
- mysql:latest - mysql:latest
- redis:latest - redis:alpine
cache: cache:
key: "ruby21" key: "ruby21"
...@@ -13,234 +13,199 @@ variables: ...@@ -13,234 +13,199 @@ variables:
MYSQL_ALLOW_EMPTY_PASSWORD: "1" MYSQL_ALLOW_EMPTY_PASSWORD: "1"
# retry tests only in CI environment # retry tests only in CI environment
RSPEC_RETRY_RETRY_COUNT: "3" RSPEC_RETRY_RETRY_COUNT: "3"
RAILS_ENV: "test"
SIMPLECOV: "true"
USE_DB: "true"
USE_BUNDLE_INSTALL: "true"
before_script: before_script:
- source ./scripts/prepare_build.sh - source ./scripts/prepare_build.sh
- ruby -v
- which ruby
- retry gem install bundler --no-ri --no-rdoc
- cp config/gitlab.yml.example config/gitlab.yml - cp config/gitlab.yml.example config/gitlab.yml
- touch log/application.log - bundle --version
- touch log/test.log - '[ "$USE_BUNDLE_INSTALL" != "true" ] || retry bundle install --without postgres production --jobs $(nproc) "${FLAGS[@]}"'
- retry bundle install --without postgres production --jobs $(nproc) "${FLAGS[@]}" - retry gem install knapsack
- RAILS_ENV=test bundle exec rake db:drop db:create db:schema:load db:migrate - '[ "$USE_DB" != "true" ] || bundle exec rake db:drop db:create db:schema:load db:migrate'
stages: stages:
- prepare
- test - test
- notifications - post-test
spec:feature: # Prepare and merge knapsack tests
stage: test
script:
- RAILS_ENV=test bundle exec rake assets:precompile 2>/dev/null
- RAILS_ENV=test SIMPLECOV=true bundle exec rake spec:feature
spec:api:
stage: test
script:
- RAILS_ENV=test SIMPLECOV=true bundle exec rake spec:api
spec:models:
stage: test
script:
- RAILS_ENV=test SIMPLECOV=true bundle exec rake spec:models
spec:lib:
stage: test
script:
- RAILS_ENV=test SIMPLECOV=true bundle exec rake spec:lib
spec:services:
stage: test
script:
- RAILS_ENV=test SIMPLECOV=true bundle exec rake spec:services
spec:other:
stage: test
script:
- RAILS_ENV=test SIMPLECOV=true bundle exec rake spec:other
spinach:project:half:
stage: test
script:
- RAILS_ENV=test bundle exec rake assets:precompile 2>/dev/null
- RAILS_ENV=test SIMPLECOV=true bundle exec rake spinach:project:half
spinach:project:rest:
stage: test
script:
- RAILS_ENV=test bundle exec rake assets:precompile 2>/dev/null
- RAILS_ENV=test SIMPLECOV=true bundle exec rake spinach:project:rest
spinach:other: .knapsack-state: &knapsack-state
stage: test services: []
script: variables:
- RAILS_ENV=test bundle exec rake assets:precompile 2>/dev/null USE_DB: "false"
- RAILS_ENV=test SIMPLECOV=true bundle exec rake spinach:other USE_BUNDLE_INSTALL: "false"
teaspoon:
stage: test
script:
- RAILS_ENV=test bundle exec teaspoon
rubocop:
stage: test
script:
- bundle exec rubocop
scss-lint:
stage: test
script:
- bundle exec rake scss_lint
license-finder:
stage: test
script:
- bundle exec license_finder
brakeman:
stage: test
script:
- bundle exec rake brakeman
flog:
stage: test
script:
- bundle exec rake flog
flay:
stage: test
script:
- bundle exec rake flay
bundler:audit:
stage: test
only:
- master
script:
- "bundle exec bundle-audit check --update --ignore OSVDB-115941"
db-migrate-reset:
stage: test
script:
- RAILS_ENV=test bundle exec rake db:migrate:reset
# Ruby 2.2 jobs
spec:feature:ruby22:
stage: test
image: ruby:2.2
only:
- master
script:
- RAILS_ENV=test bundle exec rake assets:precompile 2>/dev/null
- RAILS_ENV=test SIMPLECOV=true bundle exec rake spec:feature
cache: cache:
key: "ruby22" key: "knapsack"
paths: paths:
- vendor - knapsack/
artifacts:
spec:api:ruby22:
stage: test
image: ruby:2.2
only:
- master
script:
- RAILS_ENV=test SIMPLECOV=true bundle exec rake spec:api
cache:
key: "ruby22"
paths: paths:
- vendor - knapsack/
spec:models:ruby22: knapsack:
stage: test <<: *knapsack-state
image: ruby:2.2 stage: prepare
only:
- master
script: script:
- RAILS_ENV=test SIMPLECOV=true bundle exec rake spec:models - mkdir -p knapsack/
cache: - '[[ -f knapsack/rspec_report.json ]] || echo "{}" > knapsack/rspec_report.json'
key: "ruby22" - '[[ -f knapsack/spinach_report.json ]] || echo "{}" > knapsack/spinach_report.json'
paths:
- vendor
spec:lib:ruby22: update-knapsack:
stage: test <<: *knapsack-state
image: ruby:2.2 stage: post-test
only:
- master
script: script:
- RAILS_ENV=test SIMPLECOV=true bundle exec rake spec:lib - scripts/merge-reports knapsack/rspec_report.json knapsack/rspec_node_*.json
cache: - scripts/merge-reports knapsack/spinach_report.json knapsack/spinach_node_*.json
key: "ruby22" - rm -f knapsack/*_node_*.json
paths:
- vendor
spec:services:ruby22: # Execute all testing suites
stage: test
image: ruby:2.2
only:
- master
script:
- RAILS_ENV=test SIMPLECOV=true bundle exec rake spec:services
cache:
key: "ruby22"
paths:
- vendor
spec:other:ruby22: .rspec-knapsack: &rspec-knapsack
stage: test stage: test
image: ruby:2.2
only:
- master
script: script:
- RAILS_ENV=test SIMPLECOV=true bundle exec rake spec:other - bundle exec rake assets:precompile 2>/dev/null
cache: - JOB_NAME=( $CI_BUILD_NAME )
key: "ruby22" - export CI_NODE_INDEX=${JOB_NAME[1]}
- export CI_NODE_TOTAL=${JOB_NAME[2]}
- export KNAPSACK_REPORT_PATH=knapsack/rspec_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
- export KNAPSACK_GENERATE_REPORT=true
- cp knapsack/rspec_report.json ${KNAPSACK_REPORT_PATH}
- knapsack rspec
artifacts:
paths: paths:
- vendor - knapsack/
spinach:project:half:ruby22: .spinach-knapsack: &spinach-knapsack
stage: test stage: test
image: ruby:2.2 script:
only: - bundle exec rake assets:precompile 2>/dev/null
- master - JOB_NAME=( $CI_BUILD_NAME )
script: - export CI_NODE_INDEX=${JOB_NAME[1]}
- RAILS_ENV=test bundle exec rake assets:precompile 2>/dev/null - export CI_NODE_TOTAL=${JOB_NAME[2]}
- RAILS_ENV=test SIMPLECOV=true bundle exec rake spinach:project:half - export KNAPSACK_REPORT_PATH=knapsack/spinach_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
cache: - export KNAPSACK_GENERATE_REPORT=true
key: "ruby22" - cp knapsack/spinach_report.json ${KNAPSACK_REPORT_PATH}
- knapsack spinach "-r rerun"
# retry failed tests 3 times
- retry '[ ! -e tmp/spinach-rerun.txt ] || bin/spinach -r rerun $(cat tmp/spinach-rerun.txt)'
artifacts:
paths: paths:
- vendor - knapsack/
spinach:project:rest:ruby22: rspec 0 20: *rspec-knapsack
stage: test rspec 1 20: *rspec-knapsack
image: ruby:2.2 rspec 2 20: *rspec-knapsack
rspec 3 20: *rspec-knapsack
rspec 4 20: *rspec-knapsack
rspec 5 20: *rspec-knapsack
rspec 6 20: *rspec-knapsack
rspec 7 20: *rspec-knapsack
rspec 8 20: *rspec-knapsack
rspec 9 20: *rspec-knapsack
rspec 10 20: *rspec-knapsack
rspec 11 20: *rspec-knapsack
rspec 12 20: *rspec-knapsack
rspec 13 20: *rspec-knapsack
rspec 14 20: *rspec-knapsack
rspec 15 20: *rspec-knapsack
rspec 16 20: *rspec-knapsack
rspec 17 20: *rspec-knapsack
rspec 18 20: *rspec-knapsack
rspec 19 20: *rspec-knapsack
spinach 0 10: *spinach-knapsack
spinach 1 10: *spinach-knapsack
spinach 2 10: *spinach-knapsack
spinach 3 10: *spinach-knapsack
spinach 4 10: *spinach-knapsack
spinach 5 10: *spinach-knapsack
spinach 6 10: *spinach-knapsack
spinach 7 10: *spinach-knapsack
spinach 8 10: *spinach-knapsack
spinach 9 10: *spinach-knapsack
# Execute all testing suites against Ruby 2.2
.ruby-22: &ruby-22
image: "ruby:2.2"
only: only:
- master - master
script:
- RAILS_ENV=test bundle exec rake assets:precompile 2>/dev/null
- RAILS_ENV=test SIMPLECOV=true bundle exec rake spinach:project:rest
cache: cache:
key: "ruby22" key: "ruby22"
paths: paths:
- vendor - vendor
spinach:other:ruby22: .rspec-knapsack-ruby22: &rspec-knapsack-ruby22
<<: *rspec-knapsack
<<: *ruby-22
.spinach-knapsack-ruby22: &spinach-knapsack-ruby22
<<: *rspec-knapsack
<<: *ruby-22
rspec 0 20 ruby22: *rspec-knapsack-ruby22
rspec 1 20 ruby22: *rspec-knapsack-ruby22
rspec 2 20 ruby22: *rspec-knapsack-ruby22
rspec 3 20 ruby22: *rspec-knapsack-ruby22
rspec 4 20 ruby22: *rspec-knapsack-ruby22
rspec 5 20 ruby22: *rspec-knapsack-ruby22
rspec 6 20 ruby22: *rspec-knapsack-ruby22
rspec 7 20 ruby22: *rspec-knapsack-ruby22
rspec 8 20 ruby22: *rspec-knapsack-ruby22
rspec 9 20 ruby22: *rspec-knapsack-ruby22
rspec 10 20 ruby22: *rspec-knapsack-ruby22
rspec 11 20 ruby22: *rspec-knapsack-ruby22
rspec 12 20 ruby22: *rspec-knapsack-ruby22
rspec 13 20 ruby22: *rspec-knapsack-ruby22
rspec 14 20 ruby22: *rspec-knapsack-ruby22
rspec 15 20 ruby22: *rspec-knapsack-ruby22
rspec 16 20 ruby22: *rspec-knapsack-ruby22
rspec 17 20 ruby22: *rspec-knapsack-ruby22
rspec 18 20 ruby22: *rspec-knapsack-ruby22
rspec 19 20 ruby22: *rspec-knapsack-ruby22
spinach 0 10 ruby22: *spinach-knapsack-ruby22
spinach 1 10 ruby22: *spinach-knapsack-ruby22
spinach 2 10 ruby22: *spinach-knapsack-ruby22
spinach 3 10 ruby22: *spinach-knapsack-ruby22
spinach 4 10 ruby22: *spinach-knapsack-ruby22
spinach 5 10 ruby22: *spinach-knapsack-ruby22
spinach 6 10 ruby22: *spinach-knapsack-ruby22
spinach 7 10 ruby22: *spinach-knapsack-ruby22
spinach 8 10 ruby22: *spinach-knapsack-ruby22
spinach 9 10 ruby22: *spinach-knapsack-ruby22
# Other generic tests
.exec: &exec
stage: test
script:
- bundle exec $CI_BUILD_NAME
teaspoon: *exec
rubocop: *exec
rake scss_lint: *exec
rake brakeman: *exec
rake flog: *exec
rake flay: *exec
rake db:migrate:reset: *exec
license_finder: *exec
bundler:audit:
stage: test stage: test
image: ruby:2.2
only: only:
- master - master
script: script:
- RAILS_ENV=test bundle exec rake assets:precompile 2>/dev/null - "bundle exec bundle-audit check --update --ignore OSVDB-115941"
- RAILS_ENV=test SIMPLECOV=true bundle exec rake spinach:other
cache: # Notify slack in the end
key: "ruby22"
paths:
- vendor
notify:slack: notify:slack:
stage: notifications stage: post-test
script: script:
- ./scripts/notify_slack.sh "#builds" "Build on \`$CI_BUILD_REF_NAME\` failed! Commit \`$(git log -1 --oneline)\` See <https://gitlab.com/gitlab-org/$(basename "$PWD")/commit/"$CI_BUILD_REF"/builds>" - ./scripts/notify_slack.sh "#builds" "Build on \`$CI_BUILD_REF_NAME\` failed! Commit \`$(git log -1 --oneline)\` See <https://gitlab.com/gitlab-org/$(basename "$PWD")/commit/"$CI_BUILD_REF"/builds>"
when: on_failure when: on_failure
......
...@@ -316,6 +316,7 @@ group :test do ...@@ -316,6 +316,7 @@ group :test do
gem 'webmock', '~> 1.21.0' gem 'webmock', '~> 1.21.0'
gem 'test_after_commit', '~> 0.4.2' gem 'test_after_commit', '~> 0.4.2'
gem 'sham_rack' gem 'sham_rack'
gem 'knapsack'
end end
group :production do group :production do
......
...@@ -358,6 +358,9 @@ GEM ...@@ -358,6 +358,9 @@ GEM
actionpack (>= 3.0.0) actionpack (>= 3.0.0)
activesupport (>= 3.0.0) activesupport (>= 3.0.0)
kgio (2.10.0) kgio (2.10.0)
knapsack (1.11.0)
rake
timecop (>= 0.1.0)
launchy (2.4.3) launchy (2.4.3)
addressable (~> 2.3) addressable (~> 2.3)
letter_opener (1.4.1) letter_opener (1.4.1)
...@@ -735,6 +738,7 @@ GEM ...@@ -735,6 +738,7 @@ GEM
thor (0.19.1) thor (0.19.1)
thread_safe (0.3.5) thread_safe (0.3.5)
tilt (2.0.2) tilt (2.0.2)
timecop (0.8.1)
timfel-krb5-auth (0.8.3) timfel-krb5-auth (0.8.3)
tinder (1.10.1) tinder (1.10.1)
eventmachine (~> 1.0) eventmachine (~> 1.0)
...@@ -882,6 +886,7 @@ DEPENDENCIES ...@@ -882,6 +886,7 @@ DEPENDENCIES
jquery-ui-rails (~> 5.0.0) jquery-ui-rails (~> 5.0.0)
jwt jwt
kaminari (~> 0.17.0) kaminari (~> 0.17.0)
knapsack
letter_opener_web (~> 1.3.0) letter_opener_web (~> 1.3.0)
license_finder license_finder
licensee (~> 8.0.0) licensee (~> 8.0.0)
......
...@@ -3,8 +3,11 @@ ...@@ -3,8 +3,11 @@
# for example lib/tasks/capistrano.rake, and they will automatically be available to Rake. # for example lib/tasks/capistrano.rake, and they will automatically be available to Rake.
require File.expand_path('../config/application', __FILE__) require File.expand_path('../config/application', __FILE__)
require 'knapsack'
relative_url_conf = File.expand_path('../config/initializers/relative_url', __FILE__) relative_url_conf = File.expand_path('../config/initializers/relative_url', __FILE__)
require relative_url_conf if File.exist?("#{relative_url_conf}.rb") require relative_url_conf if File.exist?("#{relative_url_conf}.rb")
Gitlab::Application.load_tasks Gitlab::Application.load_tasks
Knapsack.load_tasks
...@@ -11,11 +11,14 @@ ENV['RAILS_ENV'] = 'test' ...@@ -11,11 +11,14 @@ ENV['RAILS_ENV'] = 'test'
require './config/environment' require './config/environment'
require 'rspec/expectations' require 'rspec/expectations'
require 'sidekiq/testing/inline' require 'sidekiq/testing/inline'
require 'knapsack'
require_relative 'capybara' require_relative 'capybara'
require_relative 'db_cleaner' require_relative 'db_cleaner'
require_relative 'rerun' require_relative 'rerun'
Knapsack::Adapters::SpinachAdapter.bind
%w(select2_helper test_env repo_helpers).each do |f| %w(select2_helper test_env repo_helpers).each do |f|
require Rails.root.join('spec', 'support', f) require Rails.root.join('spec', 'support', f)
end end
......
#!/usr/bin/env ruby
require 'json'
require 'yaml'
main_report_file = ARGV.shift
unless main_report_file
puts 'usage: merge_reports <main-report> [extra reports...]'
exit 1
end
puts "Loading #{main_report_file}..."
main_report = JSON.parse(File.read(main_report_file))
new_report = main_report.dup
ARGV.each do |report_file|
report = JSON.parse(File.read(report_file))
# Remove existing values
updates = report.delete_if do |key, value|
main_report[key] && main_report[key] == value
end
new_report.merge!(updates)
puts "Merged #{report_file} adding #{updates.size} results."
end
File.write(main_report_file, JSON.pretty_generate(new_report))
puts "Saved #{main_report_file}."
#!/bin/bash #!/bin/bash
retry() { retry() {
for i in $(seq 1 3); do
if eval "$@"; then if eval "$@"; then
return 0 return 0
fi fi
for i in 2 1; do
sleep 3s sleep 3s
echo "Retrying..." echo "Retrying $i..."
if eval "$@"; then
return 0
fi
done done
return 1 return 1
} }
......
...@@ -15,6 +15,9 @@ require 'rspec/rails' ...@@ -15,6 +15,9 @@ require 'rspec/rails'
require 'shoulda/matchers' require 'shoulda/matchers'
require 'sidekiq/testing/inline' require 'sidekiq/testing/inline'
require 'rspec/retry' require 'rspec/retry'
require 'knapsack'
Knapsack::Adapters::RSpecAdapter.bind
# Requires supporting ruby files with custom matchers and macros, etc, # Requires supporting ruby files with custom matchers and macros, etc,
# in spec/support/ and its subdirectories. # in spec/support/ and its subdirectories.
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment