Commit eb79ecdd authored by Rémy Coutable's avatar Rémy Coutable Committed by Albert Salim

ci: Fix the retrying of failing specs for `-ee` jobs

The "only-failure" feature takes the intersection of failed examples
with the list of files that would run without the `--only-failures`
flag.
In our case, the problem only appeared for `-ee` jobs since the default
RSpec pattern is `spec/**/*_spec.rb`, the intersection would be empty.
Passing the pattern from the first run makes the intersection return
the "failed" examples correctly.

Also, disable Crystalball for the retry run so that the first run report
is used.
Signed-off-by: default avatarRémy Coutable <remy@rymai.me>
parent a3bdb419
......@@ -8,6 +8,7 @@
- knapsack/
- rspec/
- crystalball/
when: always
retrieve-tests-metadata:
extends:
......
......@@ -209,6 +209,8 @@ function debug_rspec_variables() {
echoinfo "NEW_FLAKY_RSPEC_REPORT_PATH: ${NEW_FLAKY_RSPEC_REPORT_PATH}"
echoinfo "SKIPPED_FLAKY_TESTS_REPORT_PATH: ${SKIPPED_FLAKY_TESTS_REPORT_PATH}"
echoinfo "RETRIED_TESTS_REPORT_PATH: ${RETRIED_TESTS_REPORT_PATH}"
echoinfo "CRYSTALBALL: ${CRYSTALBALL}"
}
function rspec_paralellized_job() {
......@@ -245,11 +247,7 @@ function rspec_paralellized_job() {
cp "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" "${KNAPSACK_REPORT_PATH}"
if [[ -z "${KNAPSACK_TEST_FILE_PATTERN}" ]]; then
pattern=$(ruby -r./tooling/quality/test_level.rb -e "puts Quality::TestLevel.new(${spec_folder_prefixes}).pattern(:${test_level})")
export KNAPSACK_TEST_FILE_PATTERN="${pattern}"
fi
export KNAPSACK_TEST_FILE_PATTERN=$(ruby -r./tooling/quality/test_level.rb -e "puts Quality::TestLevel.new(${spec_folder_prefixes}).pattern(:${test_level})")
export FLAKY_RSPEC_REPORT_PATH="${rspec_flaky_folder_path}all_${report_name}_report.json"
export NEW_FLAKY_RSPEC_REPORT_PATH="${rspec_flaky_folder_path}new_${report_name}_report.json"
export SKIPPED_FLAKY_TESTS_REPORT_PATH="${rspec_flaky_folder_path}skipped_flaky_tests_${report_name}_report.txt"
......@@ -285,21 +283,7 @@ function rspec_paralellized_job() {
# Experiment to retry failed examples in a new RSpec process: https://gitlab.com/gitlab-org/quality/team-tasks/-/issues/1148
if [[ $rspec_run_status -ne 0 ]]; then
if [[ "${RETRY_FAILED_TESTS_IN_NEW_PROCESS}" == "true" ]]; then
# Keep track of the tests that are retried, later consolidated in a single file by the `rspec:flaky-tests-report` job
local failed_examples=$(grep " failed" ${RSPEC_LAST_RUN_RESULTS_FILE})
echo "${CI_JOB_URL}" > "${RETRIED_TESTS_REPORT_PATH}"
echo $failed_examples >> "${RETRIED_TESTS_REPORT_PATH}"
echoinfo "Retrying the failing examples in a new RSpec proces..."
install_junit_merge_gem
# Retry only the tests that failed on first try
rspec_simple_job "--only-failures" "${JUNIT_RETRY_FILE}"
rspec_run_status=$?
# Merge the JUnit report from retry into the first-try report
junit_merge "${JUNIT_RETRY_FILE}" "${JUNIT_RESULT_FILE}"
$rspec_run_status=$(retry_failed_rspec_examples)
fi
else
echosuccess "No examples to retry, congrats!"
......@@ -308,6 +292,31 @@ function rspec_paralellized_job() {
exit $rspec_run_status
}
function retry_failed_rspec_examples() {
local rspec_run_status=0
# Keep track of the tests that are retried, later consolidated in a single file by the `rspec:flaky-tests-report` job
local failed_examples=$(grep " failed" ${RSPEC_LAST_RUN_RESULTS_FILE})
echo "${CI_JOB_URL}" > "${RETRIED_TESTS_REPORT_PATH}"
echo $failed_examples >> "${RETRIED_TESTS_REPORT_PATH}"
echoinfo "Retrying the failing examples in a new RSpec proces..."
install_junit_merge_gem
# Disable Crystalball on retry to not overwrite the existing report
export CRYSTALBALL="false"
# Retry only the tests that failed on first try
rspec_simple_job "--only-failures --pattern \"${KNAPSACK_TEST_FILE_PATTERN}\"" "${JUNIT_RETRY_FILE}"
rspec_run_status=$?
# Merge the JUnit report from retry into the first-try report
junit_merge "${JUNIT_RETRY_FILE}" "${JUNIT_RESULT_FILE}"
return $rspec_run_status
}
function rspec_rerun_previous_failed_tests() {
local test_file_count_threshold=${RSPEC_PREVIOUS_FAILED_TEST_FILE_COUNT_THRESHOLD:-10}
local matching_tests_file=${1}
......
......@@ -6,7 +6,7 @@ module CrystalballEnv
extend self
def start!
return unless ENV['CRYSTALBALL']
return unless ENV['CRYSTALBALL'] == 'true'
require 'crystalball'
require_relative '../tooling/lib/tooling/crystalball/coverage_lines_execution_detector'
......
......@@ -39,11 +39,23 @@ RSpec.describe Tooling::TestMapGenerator do
YAML
end
let(:yaml3) do
<<~YAML
---
:type: Crystalball::ExecutionMap
:commit: 74056e8d9cf3773f43faa1cf5416f8779c8284c9
:timestamp: 1602671965
:version:
---
YAML
end
let(:pathname) { instance_double(Pathname) }
before do
stub_file_read('yaml1.yml', content: yaml1)
stub_file_read('yaml2.yml', content: yaml2)
stub_file_read('yaml3.yml', content: yaml3)
end
context 'with single yaml' do
......@@ -74,6 +86,10 @@ RSpec.describe Tooling::TestMapGenerator do
expect(subject.mapping[file]).to match_array(tests)
end
end
it 'displays a warning when report has no examples' do
expect { subject.parse('yaml3.yml') }.to output(%|No examples in yaml3.yml! Metadata: {:type=>"Crystalball::ExecutionMap", :commit=>"74056e8d9cf3773f43faa1cf5416f8779c8284c9", :timestamp=>1602671965, :version=>nil}\n|).to_stdout
end
end
context 'with multiple yamls' do
......
......@@ -12,7 +12,12 @@ module Tooling
def parse(yaml_files)
Array(yaml_files).each do |yaml_file|
data = File.read(yaml_file)
_metadata, example_groups = data.split("---\n").reject(&:empty?).map { |yml| YAML.safe_load(yml, [Symbol]) }
metadata, example_groups = data.split("---\n").reject(&:empty?).map { |yml| YAML.safe_load(yml, [Symbol]) }
if example_groups.nil?
puts "No examples in #{yaml_file}! Metadata: #{metadata}"
next
end
example_groups.each do |example_id, files|
files.each do |file|
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment