Merge branch 'simple-cov' into 'master'

Generate coverage report from whole test suite

Extends our CI plan to merge all coverage results and generate HTML report.

See merge request !5018

.gitlab-ci.yml

@@ -28,6 +28,7 @@ stages:
 - prepare
 - test
 - post-test
+- pages
 
 # Prepare and merge knapsack tests
 .knapsack-state: &knapsack-state
@@ -40,6 +41,7 @@ stages:
     paths:
     - knapsack/
   artifacts:
+    expire_in: 31d
     paths:
     - knapsack/
 
@@ -81,8 +83,10 @@ update-knapsack:
     - cp knapsack/rspec_report.json ${KNAPSACK_REPORT_PATH}
     - knapsack rspec
   artifacts:
+    expire_in: 31d
     paths:
     - knapsack/
+    - coverage/
 
 .spinach-knapsack: &spinach-knapsack
   stage: test
@@ -97,8 +101,10 @@ update-knapsack:
     - cp knapsack/spinach_report.json ${KNAPSACK_REPORT_PATH}
     - knapsack spinach "-r rerun" || retry '[ ! -e tmp/spinach-rerun.txt ] || bundle exec spinach -r rerun $(cat tmp/spinach-rerun.txt)'
   artifacts:
+    expire_in: 31d
     paths:
     - knapsack/
+    - coverage/
 
 rspec 0 20: *rspec-knapsack
 rspec 1 20: *rspec-knapsack
@@ -186,14 +192,14 @@ spinach 9 10 ruby23: *spinach-knapsack-ruby23
 
 # Other generic tests
 
-.static-analyses-variables: &static-analyses-variables
+.ruby-static-analysis: &ruby-static-analysis
   variables:
     SIMPLECOV: "false"
     USE_DB: "false"
     USE_BUNDLE_INSTALL: "true"
 
 .exec: &exec
-  <<: *static-analyses-variables
+  <<: *ruby-static-analysis
   stage: test
   script:
     - bundle exec $CI_BUILD_NAME
@@ -220,12 +226,28 @@ teaspoon:
 
 bundler:audit:
   stage: test
-  <<: *static-analyses-variables
+  <<: *ruby-static-analysis
   only:
     - master
   script:
     - "bundle exec bundle-audit check --update --ignore OSVDB-115941"
 
+coverage:
+  stage: post-test
+  services: []
+  variables:
+    USE_DB: "false"
+    USE_BUNDLE_INSTALL: "true"
+  script:
+    - bundle exec scripts/merge-simplecov
+  artifacts:
+    name: coverage
+    expire_in: 31d
+    paths:
+    - coverage/index.html
+    - coverage/assets/
+
+
 # Notify slack in the end
 
 notify:slack:
@@ -238,3 +260,18 @@ notify:slack:
     - tags@gitlab-org/gitlab-ce
     - master@gitlab-org/gitlab-ee
     - tags@gitlab-org/gitlab-ee
+
+pages:
+  before_script: []
+  stage: pages
+  dependencies:
+    - coverage
+  script:
+    - mv public/ .public/
+    - mkdir public/
+    - mv coverage public/coverage-ruby
+  artifacts:
+    paths:
+      - public
+  only:
+    - master

.simplecov

-# .simplecov
-SimpleCov.start 'rails' do
-  merge_timeout 3600
-end

Gemfile

@@ -303,7 +303,7 @@ group :development, :test do
   gem 'rubocop', '~> 0.41.2', require: false
   gem 'rubocop-rspec', '~> 1.5.0', require: false
   gem 'scss_lint', '~> 0.47.0', require: false
-  gem 'simplecov', '~> 0.11.0', require: false
+  gem 'simplecov', '0.12.0', require: false
   gem 'flog', '~> 4.3.2', require: false
   gem 'flay', '~> 2.6.1', require: false
   gem 'bundler-audit', '~> 0.5.0', require: false

Gemfile.lock

@@ -673,9 +673,9 @@ GEM
       rufus-scheduler (>= 2.0.24)
       sidekiq (>= 4.0.0)
     simple_oauth (0.1.9)
-    simplecov (0.11.2)
+    simplecov (0.12.0)
       docile (~> 1.1.0)
-      json (~> 1.8)
+      json (>= 1.8, < 3)
       simplecov-html (~> 0.10.0)
     simplecov-html (0.10.0)
     sinatra (1.4.7)
@@ -962,7 +962,7 @@ DEPENDENCIES
   shoulda-matchers (~> 2.8.0)
   sidekiq (~> 4.0)
   sidekiq-cron (~> 0.4.0)
-  simplecov (~> 0.11.0)
+  simplecov (= 0.12.0)
   sinatra (~> 1.4.4)
   six (~> 0.2.0)
   slack-notifier (~> 1.2.0)

features/support/env.rb

-if ENV['SIMPLECOV']
-  require 'simplecov'
-end
+require './spec/simplecov_env'
+SimpleCovEnv.start!
 
 ENV['RAILS_ENV'] = 'test'
 require './config/environment'

lib/tasks/test.rake

@@ -7,5 +7,5 @@ end
 
 unless Rails.env.production?
   desc "GitLab | Run all tests on CI with simplecov"
-  task test_ci: [:rubocop, :brakeman, 'teaspoon', :spinach, :spec]
+  task test_ci: [:rubocop, :brakeman, :teaspoon, :spinach, :spec]
 end

scripts/merge-simplecov

+#!/usr/bin/env ruby
+
+require_relative '../spec/simplecov_env'
+SimpleCovEnv.configure_profile
+
+module SimpleCov
+  module ResultMerger
+    class << self
+      def resultset_files
+        Dir.glob(File.join(SimpleCov.coverage_path, '*', '.resultset.json'))
+      end
+
+      def resultset_hashes
+        resultset_files.map do |path|
+          begin
+            JSON.parse(File.read(path))
+          rescue
+            {}
+          end
+        end
+      end
+
+      def resultset
+        resultset_hashes.reduce({}, :merge)
+      end
+    end
+  end
+end
+
+SimpleCov::ResultMerger.merged_result.format!

spec/simplecov_env.rb

+require 'simplecov'
+
+module SimpleCovEnv
+  extend self
+
+  def start!
+    return unless ENV['SIMPLECOV']
+
+    configure_profile
+    configure_job
+
+    SimpleCov.start
+  end
+
+  def configure_job
+    SimpleCov.configure do
+      if ENV['CI_BUILD_NAME']
+        coverage_dir "coverage/#{ENV['CI_BUILD_NAME']}"
+        command_name ENV['CI_BUILD_NAME']
+      end
+
+      if ENV['CI']
+        SimpleCov.at_exit do
+          # In CI environment don't generate formatted reports
+          # Only generate .resultset.json
+          SimpleCov.result
+        end
+      end
+    end
+  end
+
+  def configure_profile
+    SimpleCov.configure do
+      load_profile 'test_frameworks'
+      track_files '{app,lib}/**/*.rb'
+
+      add_filter '/vendor/ruby/'
+      add_filter 'config/initializers/'
+
+      add_group 'Controllers', 'app/controllers'
+      add_group 'Models', 'app/models'
+      add_group 'Mailers', 'app/mailers'
+      add_group 'Helpers', 'app/helpers'
+      add_group 'Workers', %w(app/jobs app/workers)
+      add_group 'Libraries', 'lib'
+      add_group 'Services', 'app/services'
+      add_group 'Finders', 'app/finders'
+      add_group 'Uploaders', 'app/uploaders'
+      add_group 'Validators', 'app/validators'
+
+      merge_timeout 7200
+    end
+  end
+end

spec/spec_helper.rb

-if ENV['SIMPLECOV']
-  require 'simplecov'
-  SimpleCov.start :rails
-end
+require './spec/simplecov_env'
+SimpleCovEnv.start!
 
 ENV["RAILS_ENV"] ||= 'test'