ci: Fix the retrying of failing specs for `-ee` jobs

The "only-failure" feature takes the intersection of failed examples
with the list of files that would run without the `--only-failures`
flag.
In our case, the problem only appeared for `-ee` jobs since the default
RSpec pattern is `spec/**/*_spec.rb`, the intersection would be empty.
Passing the pattern from the first run makes the intersection return
the "failed" examples correctly.

Also, disable Crystalball for the retry run so that the first run report
is used.
Signed-off-by: Rémy Coutable <remy@rymai.me>

.gitlab/ci/test-metadata.gitlab-ci.yml

@@ -8,6 +8,7 @@
       - knapsack/
       - rspec/
       - crystalball/
+    when: always
 
 retrieve-tests-metadata:
   extends:

scripts/rspec_helpers.sh

@@ -209,6 +209,8 @@ function debug_rspec_variables() {
   echoinfo "NEW_FLAKY_RSPEC_REPORT_PATH: ${NEW_FLAKY_RSPEC_REPORT_PATH}"
   echoinfo "SKIPPED_FLAKY_TESTS_REPORT_PATH: ${SKIPPED_FLAKY_TESTS_REPORT_PATH}"
   echoinfo "RETRIED_TESTS_REPORT_PATH: ${RETRIED_TESTS_REPORT_PATH}"
+
+  echoinfo "CRYSTALBALL: ${CRYSTALBALL}"
 }
 
 function rspec_paralellized_job() {
@@ -245,11 +247,7 @@ function rspec_paralellized_job() {
 
   cp "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" "${KNAPSACK_REPORT_PATH}"
 
-  if [[ -z "${KNAPSACK_TEST_FILE_PATTERN}" ]]; then
-    pattern=$(ruby -r./tooling/quality/test_level.rb -e "puts Quality::TestLevel.new(${spec_folder_prefixes}).pattern(:${test_level})")
-    export KNAPSACK_TEST_FILE_PATTERN="${pattern}"
-  fi
-
+  export KNAPSACK_TEST_FILE_PATTERN=$(ruby -r./tooling/quality/test_level.rb -e "puts Quality::TestLevel.new(${spec_folder_prefixes}).pattern(:${test_level})")
   export FLAKY_RSPEC_REPORT_PATH="${rspec_flaky_folder_path}all_${report_name}_report.json"
   export NEW_FLAKY_RSPEC_REPORT_PATH="${rspec_flaky_folder_path}new_${report_name}_report.json"
   export SKIPPED_FLAKY_TESTS_REPORT_PATH="${rspec_flaky_folder_path}skipped_flaky_tests_${report_name}_report.txt"
@@ -285,21 +283,7 @@ function rspec_paralellized_job() {
   # Experiment to retry failed examples in a new RSpec process: https://gitlab.com/gitlab-org/quality/team-tasks/-/issues/1148
   if [[ $rspec_run_status -ne 0 ]]; then
     if [[ "${RETRY_FAILED_TESTS_IN_NEW_PROCESS}" == "true" ]]; then
-      # Keep track of the tests that are retried, later consolidated in a single file by the `rspec:flaky-tests-report` job
-      local failed_examples=$(grep " failed" ${RSPEC_LAST_RUN_RESULTS_FILE})
-      echo "${CI_JOB_URL}" > "${RETRIED_TESTS_REPORT_PATH}"
-      echo $failed_examples >> "${RETRIED_TESTS_REPORT_PATH}"
-
-      echoinfo "Retrying the failing examples in a new RSpec proces..."
-
-      install_junit_merge_gem
-
-      # Retry only the tests that failed on first try
-      rspec_simple_job "--only-failures" "${JUNIT_RETRY_FILE}"
-      rspec_run_status=$?
-
-      # Merge the JUnit report from retry into the first-try report
-      junit_merge "${JUNIT_RETRY_FILE}" "${JUNIT_RESULT_FILE}"
+      $rspec_run_status=$(retry_failed_rspec_examples)
     fi
   else
     echosuccess "No examples to retry, congrats!"
@@ -308,6 +292,31 @@ function rspec_paralellized_job() {
   exit $rspec_run_status
 }
 
+function retry_failed_rspec_examples() {
+  local rspec_run_status=0
+
+  # Keep track of the tests that are retried, later consolidated in a single file by the `rspec:flaky-tests-report` job
+  local failed_examples=$(grep " failed" ${RSPEC_LAST_RUN_RESULTS_FILE})
+  echo "${CI_JOB_URL}" > "${RETRIED_TESTS_REPORT_PATH}"
+  echo $failed_examples >> "${RETRIED_TESTS_REPORT_PATH}"
+
+  echoinfo "Retrying the failing examples in a new RSpec proces..."
+
+  install_junit_merge_gem
+
+  # Disable Crystalball on retry to not overwrite the existing report
+  export CRYSTALBALL="false"
+
+  # Retry only the tests that failed on first try
+  rspec_simple_job "--only-failures --pattern \"${KNAPSACK_TEST_FILE_PATTERN}\"" "${JUNIT_RETRY_FILE}"
+  rspec_run_status=$?
+
+  # Merge the JUnit report from retry into the first-try report
+  junit_merge "${JUNIT_RETRY_FILE}" "${JUNIT_RESULT_FILE}"
+
+  return $rspec_run_status
+}
+
 function rspec_rerun_previous_failed_tests() {
   local test_file_count_threshold=${RSPEC_PREVIOUS_FAILED_TEST_FILE_COUNT_THRESHOLD:-10}
   local matching_tests_file=${1}

spec/crystalball_env.rb

@@ -6,7 +6,7 @@ module CrystalballEnv
   extend self
 
   def start!
-    return unless ENV['CRYSTALBALL']
+    return unless ENV['CRYSTALBALL'] == 'true'
 
     require 'crystalball'
     require_relative '../tooling/lib/tooling/crystalball/coverage_lines_execution_detector'

spec/tooling/lib/tooling/test_map_generator_spec.rb

@@ -39,11 +39,23 @@ RSpec.describe Tooling::TestMapGenerator do
       YAML
     end
 
+    let(:yaml3) do
+      <<~YAML
+        ---
+        :type: Crystalball::ExecutionMap
+        :commit: 74056e8d9cf3773f43faa1cf5416f8779c8284c9
+        :timestamp: 1602671965
+        :version:
+        ---
+      YAML
+    end
+
     let(:pathname) { instance_double(Pathname) }
 
     before do
       stub_file_read('yaml1.yml', content: yaml1)
       stub_file_read('yaml2.yml', content: yaml2)
+      stub_file_read('yaml3.yml', content: yaml3)
     end
 
     context 'with single yaml' do
@@ -74,6 +86,10 @@ RSpec.describe Tooling::TestMapGenerator do
           expect(subject.mapping[file]).to match_array(tests)
         end
       end
+
+      it 'displays a warning when report has no examples' do
+        expect { subject.parse('yaml3.yml') }.to output(%|No examples in yaml3.yml! Metadata: {:type=>"Crystalball::ExecutionMap", :commit=>"74056e8d9cf3773f43faa1cf5416f8779c8284c9", :timestamp=>1602671965, :version=>nil}\n|).to_stdout
+      end
     end
 
     context 'with multiple yamls' do

tooling/lib/tooling/test_map_generator.rb

@@ -12,7 +12,12 @@ module Tooling
     def parse(yaml_files)
       Array(yaml_files).each do |yaml_file|
         data = File.read(yaml_file)
-        _metadata, example_groups = data.split("---\n").reject(&:empty?).map { |yml| YAML.safe_load(yml, [Symbol]) }
+        metadata, example_groups = data.split("---\n").reject(&:empty?).map { |yml| YAML.safe_load(yml, [Symbol]) }
+
+        if example_groups.nil?
+          puts "No examples in #{yaml_file}! Metadata: #{metadata}"
+          next
+        end
 
         example_groups.each do |example_id, files|
           files.each do |file|