Generate knapsack report for review-qa-all

Add knapsack qa report and use it to run tests in parallel

Use the RSpec runner with knapsack

The way the Knapsack runner uses exec to start rspec seems
incompatible with the way we expect it to work. Plus, it requires
specifying KNAPSACK_TEST_DIR.

Instead, we use knapsacks AllocatorBuilder to select the spec
files to run, and then start rspec as normal, via
RSpec::Core::Runner.run

This also means we can incorporate tags.

Let the job run automatically

Include KNAPSACK_TEST_FILE_PATTERN in vars

Check all defined knapsack env vars before requiring knapsack

.gitlab/ci/review.gitlab-ci.yml

@@ -165,7 +165,10 @@ review-qa-all:
   <<: *review-qa-base
   allow_failure: true
   when: manual
+  parallel: 5
   script:
+    - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/review-qa-all_master_report.json
+    - export KNAPSACK_TEST_FILE_PATTERN=qa/specs/features/**/*_spec.rb
     - gitlab-qa Test::Instance::Any "${QA_IMAGE}" "${CI_ENVIRONMENT_URL}"
 
 .review-performance-base: &review-performance-base

qa/Gemfile

@@ -10,3 +10,4 @@ gem 'airborne', '~> 0.2.13'
 gem 'nokogiri', '~> 1.10.3'
 gem 'rspec-retry', '~> 0.6.1'
 gem 'faker', '~> 1.6', '>= 1.6.6'
+gem 'knapsack', '~> 1.17'

qa/Gemfile.lock

@@ -39,6 +39,8 @@ GEM
       domain_name (~> 0.5)
     i18n (0.9.1)
       concurrent-ruby (~> 1.0)
+    knapsack (1.17.1)
+      rake
     launchy (2.4.3)
       addressable (~> 2.3)
     method_source (0.9.0)
@@ -102,6 +104,7 @@ DEPENDENCIES
   capybara (~> 2.16.1)
   capybara-screenshot (~> 1.0.18)
   faker (~> 1.6, >= 1.6.6)
+  knapsack (~> 1.17)
   nokogiri (~> 1.10.3)
   pry-byebug (~> 3.5.1)
   rake (~> 12.3.0)

qa/knapsack/gitlab-ce/review-qa-all_master_report.json

+{
+  "qa/specs/features/browser_ui/1_manage/project/add_project_member_spec.rb": 9.697327613830566,
+  "qa/specs/features/browser_ui/3_create/merge_request/rebase_merge_request_spec.rb": 46.54227638244629,
+  "qa/specs/features/browser_ui/1_manage/project/create_project_spec.rb": 10.214765310287476,
+  "qa/specs/features/browser_ui/1_manage/project/view_project_activity_spec.rb": 7.882027864456177,
+  "qa/specs/features/api/3_create/repository/files_spec.rb": 5.015859127044678,
+  "qa/specs/features/browser_ui/2_plan/issue/filter_issue_comments_spec.rb": 12.772682905197144,
+  "qa/specs/features/browser_ui/3_create/repository/push_protected_branch_spec.rb": 29.76174831390381,
+  "qa/specs/features/browser_ui/3_create/repository/use_ssh_key_spec.rb": 22.800872802734375,
+  "qa/specs/features/browser_ui/1_manage/login/register_spec.rb": 22.320587396621704,
+  "qa/specs/features/api/1_manage/users_spec.rb": 0.6089541912078857,
+  "qa/specs/features/browser_ui/3_create/repository/clone_spec.rb": 0.9618203639984131,
+  "qa/specs/features/browser_ui/2_plan/issue/collapse_comments_in_discussions_spec.rb": 13.403101205825806,
+  "qa/specs/features/browser_ui/non_devops/performance_bar_spec.rb": 8.810423135757446,
+  "qa/specs/features/browser_ui/3_create/repository/push_over_http_spec.rb": 7.730542182922363,
+  "qa/specs/features/browser_ui/3_create/repository/add_ssh_key_spec.rb": 16.18057894706726,
+  "qa/specs/features/browser_ui/4_verify/ci_variable/add_ci_variable_spec.rb": 8.31815505027771,
+  "qa/specs/features/browser_ui/3_create/repository/push_http_private_token_spec.rb": 9.48607873916626,
+  "qa/specs/features/browser_ui/2_plan/issue/create_issue_spec.rb": 19.552733182907104,
+  "qa/specs/features/browser_ui/3_create/wiki/create_edit_clone_push_wiki_spec.rb": 17.273863554000854,
+  "qa/specs/features/browser_ui/7_configure/auto_devops/create_project_with_auto_devops_spec.rb": 8.281434059143066,
+  "qa/specs/features/browser_ui/3_create/repository/create_edit_delete_file_via_web_spec.rb": 18.047621726989746,
+  "qa/specs/features/browser_ui/6_release/deploy_token/add_deploy_token_spec.rb": 7.422840595245361,
+  "qa/specs/features/browser_ui/1_manage/login/log_in_spec.rb": 3.438166856765747,
+  "qa/specs/features/browser_ui/4_verify/runner/register_runner_spec.rb": 18.679633855819702,
+  "qa/specs/features/browser_ui/3_create/merge_request/squash_merge_request_spec.rb": 27.943300485610962,
+  "qa/specs/features/browser_ui/3_create/repository/add_list_delete_branches_spec.rb": 39.17585229873657,
+  "qa/specs/features/browser_ui/3_create/repository/add_file_template_spec.rb": 40.09336972236633,
+  "qa/specs/features/browser_ui/3_create/merge_request/view_merge_request_diff_patch_spec.rb": 3.705310821533203,
+  "qa/specs/features/browser_ui/3_create/snippet/create_snippet_spec.rb": 5.812374591827393,
+  "qa/specs/features/browser_ui/6_release/deploy_key/clone_using_deploy_key_spec.rb": 92.46774697303772,
+  "qa/specs/features/browser_ui/1_manage/project/import_github_repo_spec.rb": 100.28881478309631,
+  "qa/specs/features/browser_ui/3_create/repository/push_mirroring_over_http_spec.rb": 23.710937023162842,
+  "qa/specs/features/browser_ui/1_manage/login/login_via_oauth_spec.rb": 20.58603596687317,
+  "qa/specs/features/browser_ui/3_create/web_ide/add_file_template_spec.rb": 25.460349321365356,
+  "qa/specs/features/browser_ui/3_create/repository/push_over_http_file_size_spec.rb": 19.459370374679565,
+  "qa/specs/features/browser_ui/3_create/repository/user_views_commit_diff_patch_spec.rb": 6.731764793395996,
+  "qa/specs/features/browser_ui/6_release/deploy_key/add_deploy_key_spec.rb": 15.342933893203735,
+  "qa/specs/features/browser_ui/2_plan/issue/issue_suggestions_spec.rb": 11.280649185180664,
+  "qa/specs/features/browser_ui/3_create/merge_request/create_merge_request_spec.rb": 57.48992609977722,
+  "qa/specs/features/browser_ui/3_create/merge_request/merge_merge_request_from_fork_spec.rb": 32.5517954826355
+}
\ No newline at end of file

qa/qa/runtime/env.rb

@@ -136,6 +136,10 @@ module QA
         ENV['GITLAB_QA_PASSWORD_2']
       end
 
+      def knapsack?
+        !!(ENV['KNAPSACK_GENERATE_REPORT'] || ENV['KNAPSACK_REPORT_PATH'] || ENV['KNAPSACK_TEST_FILE_PATTERN'])
+      end
+
       def ldap_username
         @ldap_username ||= ENV['GITLAB_LDAP_USERNAME']
       end

qa/qa/specs/runner.rb

 # frozen_string_literal: true
 
 require 'rspec/core'
+require 'rspec/expectations'
+require 'knapsack'
 
 module QA
   module Specs
@@ -32,10 +34,25 @@ module QA
         end
 
         args.push(options)
-        args.push(DEFAULT_TEST_PATH_ARGS) unless options.any? { |opt| opt =~ %r{/features/} }
 
         Runtime::Browser.configure!
 
+        if Runtime::Env.knapsack?
+          allocator = Knapsack::AllocatorBuilder.new(Knapsack::Adapters::RSpecAdapter).allocator
+
+          QA::Runtime::Logger.info ''
+          QA::Runtime::Logger.info 'Report specs:'
+          QA::Runtime::Logger.info allocator.report_node_tests.join(', ')
+          QA::Runtime::Logger.info ''
+          QA::Runtime::Logger.info 'Leftover specs:'
+          QA::Runtime::Logger.info allocator.leftover_node_tests.join(', ')
+          QA::Runtime::Logger.info ''
+
+          args.push(['--', allocator.node_tests])
+        else
+          args.push(DEFAULT_TEST_PATH_ARGS) unless options.any? { |opt| opt =~ %r{/features/} }
+        end
+
         RSpec::Core::Runner.run(args.flatten, $stderr, $stdout).tap do |status|
           abort if status.nonzero?
         end

qa/spec/runtime/env_spec.rb

@@ -168,6 +168,30 @@ describe QA::Runtime::Env do
     end
   end
 
+  describe '.knapsack?' do
+    it 'returns true if KNAPSACK_GENERATE_REPORT is defined' do
+      stub_env('KNAPSACK_GENERATE_REPORT', 'true')
+
+      expect(described_class.knapsack?).to be_truthy
+    end
+
+    it 'returns true if KNAPSACK_REPORT_PATH is defined' do
+      stub_env('KNAPSACK_REPORT_PATH', '/a/path')
+
+      expect(described_class.knapsack?).to be_truthy
+    end
+
+    it 'returns true if KNAPSACK_TEST_FILE_PATTERN is defined' do
+      stub_env('KNAPSACK_TEST_FILE_PATTERN', '/a/**/pattern')
+
+      expect(described_class.knapsack?).to be_truthy
+    end
+
+    it 'returns false if neither KNAPSACK_GENERATE_REPORT nor KNAPSACK_REPORT_PATH nor KNAPSACK_TEST_FILE_PATTERN are defined' do
+      expect(described_class.knapsack?).to be_falsey
+    end
+  end
+
   describe '.require_github_access_token!' do
     it 'raises ArgumentError if GITHUB_ACCESS_TOKEN is not defined' do
       stub_env('GITHUB_ACCESS_TOKEN', nil)

qa/spec/spec_helper.rb

@@ -3,6 +3,11 @@
 require_relative '../qa'
 require 'rspec/retry'
 
+if ENV['CI'] && QA::Runtime::Env.knapsack? && !ENV['NO_KNAPSACK']
+  require 'knapsack'
+  Knapsack::Adapters::RSpecAdapter.bind
+end
+
 %w[helpers shared_examples].each do |d|
   Dir[::File.join(__dir__, d, '**', '*.rb')].each { |f| require f }
 end