Merge branch 'acunskis-reliable-report-fix' into 'master'

E2E: Reliable spec report improvements

See merge request gitlab-org/gitlab!77343

qa/Gemfile

@@ -25,7 +25,7 @@ gem 'octokit', '~> 4.21'
 gem 'webdrivers', '~> 5.0'
 gem 'zeitwerk', '~> 2.4'
 gem 'influxdb-client', '~> 1.17'
-gem 'terminal-table', '~> 1.8', require: false
+gem 'terminal-table', '~> 3.0.0', require: false
 gem 'slack-notifier', '~> 2.4', require: false
 gem 'fog-google', '~> 1.17', require: false
 

qa/Gemfile.lock

@@ -112,12 +112,12 @@ GEM
       fog-core
       nokogiri (>= 1.5.11, < 2.0.0)
     formatador (0.3.0)
-    gitlab (4.16.1)
-      httparty (~> 0.14, >= 0.14.0)
-      terminal-table (~> 1.5, >= 1.5.1)
-    gitlab-qa (7.14.0)
+    gitlab (4.18.0)
+      httparty (~> 0.18)
+      terminal-table (>= 1.5.1)
+    gitlab-qa (7.17.1)
       activesupport (~> 6.1)
-      gitlab (~> 4.16.1)
+      gitlab (~> 4.18.0)
       http (~> 5.0)
       nokogiri (~> 1.10)
       table_print (= 1.5.7)
@@ -184,12 +184,12 @@ GEM
     memoizable (0.4.2)
       thread_safe (~> 0.3, >= 0.3.1)
     method_source (0.9.0)
-    mime-types (3.4.0)
+    mime-types (3.4.1)
       mime-types-data (~> 3.2015)
     mime-types-data (3.2021.1115)
     mini_mime (1.1.0)
     mini_portile2 (2.6.1)
-    minitest (5.14.4)
+    minitest (5.15.0)
     multi_json (1.15.0)
     multi_xml (0.6.0)
     multipart-post (2.1.1)
@@ -280,8 +280,8 @@ GEM
     slack-notifier (2.4.0)
     systemu (2.6.5)
     table_print (1.5.7)
-    terminal-table (1.8.0)
-      unicode-display_width (~> 1.1, >= 1.1.1)
+    terminal-table (3.0.2)
+      unicode-display_width (>= 1.1.1, < 3)
     thread_safe (0.3.6)
     timecop (0.9.1)
     trailblazer-option (0.1.2)
@@ -291,7 +291,7 @@ GEM
     unf (0.1.4)
       unf_ext
     unf_ext (0.0.8)
-    unicode-display_width (1.8.0)
+    unicode-display_width (2.1.0)
     unparser (0.4.7)
       abstract_type (~> 0.0.7)
       adamantium (~> 0.2.0)
@@ -312,7 +312,7 @@ GEM
     webrick (1.7.0)
     xpath (3.2.0)
       nokogiri (~> 1.8)
-    zeitwerk (2.5.1)
+    zeitwerk (2.5.2)
 
 PLATFORMS
   ruby
@@ -345,10 +345,10 @@ DEPENDENCIES
   ruby-debug-ide (~> 0.7.0)
   selenium-webdriver (~> 4.0)
   slack-notifier (~> 2.4)
-  terminal-table (~> 1.8)
+  terminal-table (~> 3.0.0)
   timecop (~> 0.9.1)
   webdrivers (~> 5.0)
   zeitwerk (~> 2.4)
 
 BUNDLED WITH
-   2.2.30
+   2.2.33

qa/qa/tools/reliable_report.rb

@@ -16,7 +16,7 @@ module QA
       PROJECT_ID = 278964
 
       def initialize(range)
-        @range = range
+        @range = range.to_i
         @influxdb_bucket = "e2e-test-stats"
         @slack_channel = "#quality-reports"
         @influxdb_url = ENV["QA_INFLUXDB_URL"] || raise("Missing QA_INFLUXDB_URL env variable")
@@ -34,7 +34,7 @@ module QA
         reporter.print_report
         reporter.report_in_issue_and_slack if report_in_issue_and_slack == "true"
       rescue StandardError => e
-        reporter.notify_failure(e)
+        reporter&.notify_failure(e)
         raise(e)
       end
 
@@ -100,67 +100,74 @@ module QA
         issue = []
         issue << "[[_TOC_]]"
         issue << "# Candidates for promotion to reliable #{execution_interval}"
-        issue << "```\n#{stable_summary_table}\n```"
-        issue << results_markdown(stable_results_tables)
+        issue << "Total amount: **#{stable_test_runs.sum { |_k, v| v.count }}**"
+        issue << stable_summary_table(markdown: true).to_s
+        issue << results_markdown(:stable)
         return issue.join("\n\n") if unstable_reliable_test_runs.empty?
 
         issue << "# Reliable specs with failures #{execution_interval}"
-        issue << "```\n#{unstable_summary_table}\n```"
-        issue << results_markdown(unstable_reliable_results_tables)
+        issue << "Total amount: **#{unstable_reliable_test_runs.sum { |_k, v| v.count }}**"
+        issue << unstable_summary_table(markdown: true).to_s
+        issue << results_markdown(:unstable)
         issue.join("\n\n")
       end
 
       # Stable spec summary table
       #
+      # @param [Boolean] markdown
       # @return [Terminal::Table]
-      def stable_summary_table
-        @stable_summary_table ||= terminal_table(
+      def stable_summary_table(markdown: false)
+        terminal_table(
           rows: stable_test_runs.map { |stage, specs| [stage, specs.length] },
           title: "Stable spec summary for past #{range} days".ljust(50),
-          headings: %w[STAGE COUNT]
+          headings: %w[STAGE COUNT],
+          markdown: markdown
         )
       end
 
       # Unstable reliable summary table
       #
+      # @param [Boolean] markdown
       # @return [Terminal::Table]
-      def unstable_summary_table
-        @unstable_summary_table ||= terminal_table(
+      def unstable_summary_table(markdown: false)
+        terminal_table(
           rows: unstable_reliable_test_runs.map { |stage, specs| [stage, specs.length] },
           title: "Unstable spec summary for past #{range} days".ljust(50),
-          headings: %w[STAGE COUNT]
+          headings: %w[STAGE COUNT],
+          markdown: markdown
         )
       end
 
       # Result tables for stable specs
       #
+      # @param [Boolean] markdown
       # @return [Hash]
-      def stable_results_tables
-        @stable_results ||= results_tables(:stable)
+      def stable_results_tables(markdown: false)
+        results_tables(:stable, markdown: markdown)
       end
 
       # Result table for unstable specs
       #
+      # @param [Boolean] markdown
       # @return [Hash]
-      def unstable_reliable_results_tables
-        @unstable_results ||= results_tables(:unstable)
+      def unstable_reliable_results_tables(markdown: false)
+        results_tables(:unstable, markdown: markdown)
       end
 
       # Markdown formatted tables
       #
-      # @param [Hash] results
+      # @param [Symbol] type result type - :stable, :unstable
       # @return [String]
-      def results_markdown(results)
-        results.map do |stage, table|
+      def results_markdown(type)
+        runs = type == :stable ? stable_test_runs : unstable_reliable_test_runs
+        results_tables(type, markdown: true).map do |stage, table|
           <<~STAGE.strip
-            ## #{stage}
+            ## #{stage} (#{runs[stage].count})
 
             <details>
             <summary>Executions table</summary>
 
-            ```
             #{table}
-            ```
 
             </details>
           STAGE
@@ -170,15 +177,19 @@ module QA
       # Results table
       #
       # @param [Symbol] type result type - :stable, :unstable
+      # @param [Boolean] markdown
       # @return [Hash<Symbol, Terminal::Table>]
-      def results_tables(type)
+      def results_tables(type, markdown: false)
         (type == :stable ? stable_test_runs : unstable_reliable_test_runs).to_h do |stage, specs|
           headings = ["name", "runs", "failures", "failure rate"]
 
           [stage, terminal_table(
-            rows: specs.map { |k, v| [name_column(k, v[:file]), *table_params(v.values)] },
             title: "Top #{type} specs in '#{stage}' stage for past #{range} days",
-            headings: headings.map(&:upcase)
+            headings: headings.map(&:upcase),
+            markdown: markdown,
+            rows: specs.map do |k, v|
+              [name_column(name: k, file: v[:file], markdown: markdown), *table_params(v.values)]
+            end
           )]
         end
       end
@@ -217,13 +228,17 @@ module QA
 
       # Terminal table for result formatting
       #
+      # @param [Array] rows
+      # @param [Array] headings
+      # @param [String] title
+      # @param [Boolean] markdown
       # @return [Terminal::Table]
-      def terminal_table(rows:, headings:, title: nil)
+      def terminal_table(rows:, headings:, title:, markdown:)
         Terminal::Table.new(
           headings: headings,
-          style: { all_separators: true },
-          title: title,
-          rows: rows
+          title: markdown ? nil : title,
+          rows: rows,
+          style: markdown ? { border: :markdown } : { all_separators: true }
         )
       end
 
@@ -235,17 +250,17 @@ module QA
         [*parameters[1..2], "#{parameters.last}%"]
       end
 
-      # Name column value
+      # Name column content
       #
       # @param [String] name
       # @param [String] file
+      # @param [Boolean] markdown
       # @return [String]
-      def name_column(name, file)
-        spec_name = name.length > 150 ? "#{name} ".scan(/.{1,150} /).map(&:strip).join("\n") : name
-        name_line = "name: '#{spec_name}'"
-        file_line = "file: '#{file}'"
+      def name_column(name:, file:, markdown: false)
+        return "**name**: #{name}<br>**file**: #{file}" if markdown
 
-        "#{name_line}\n#{file_line.ljust(160)}"
+        wrapped_name = name.length > 150 ? "#{name} ".scan(/.{1,150} /).map(&:strip).join("\n") : name
+        "name: '#{wrapped_name}'\nfile: #{file.ljust(160)}"
       end
 
       # Test executions grouped by name
@@ -258,9 +273,10 @@ module QA
         all_runs = query_api.query(query: query(reliable)).values
         all_runs.each_with_object(Hash.new { |hsh, key| hsh[key] = {} }) do |table, result|
           records = table.records
-          # skip specs that executed less time than defined by range
+          # skip specs that executed less time than defined by range or stopped executing before report date
           # offset 1 day due to how schedulers are configured and first run can be 1 day later
           next if (Date.today - Date.parse(records.first.values["_time"])).to_i < (range - 1)
+          next if (Date.today - Date.parse(records.last.values["_time"])).to_i > 1
 
           last_record = records.last.values
           name = last_record["name"]

qa/spec/tools/reliable_report_spec.rb

@@ -29,7 +29,7 @@ describe QA::Tools::ReliableReport do
         records: [
           instance_double("InfluxDB2::FluxRecord", values: values),
           instance_double("InfluxDB2::FluxRecord", values: values),
-          instance_double("InfluxDB2::FluxRecord", values: values)
+          instance_double("InfluxDB2::FluxRecord", values: values.merge({ "_time" => Time.now.to_s }))
         ]
       )
     }
@@ -49,7 +49,7 @@ describe QA::Tools::ReliableReport do
         records: [
           instance_double("InfluxDB2::FluxRecord", values: { **values, "status" => "passed" }),
           instance_double("InfluxDB2::FluxRecord", values: values),
-          instance_double("InfluxDB2::FluxRecord", values: values)
+          instance_double("InfluxDB2::FluxRecord", values: values.merge({ "_time" => Time.now.to_s }))
         ]
       )
     }
@@ -80,41 +80,34 @@ describe QA::Tools::ReliableReport do
 
   def markdown_section(summary, result, stage, type)
     <<~SECTION.strip
-      ```
-      #{summary_table(summary, type)}
-      ```
+      #{summary_table(summary, type, true)}
 
-      ## #{stage}
+      ## #{stage} (1)
 
       <details>
       <summary>Executions table</summary>
 
-      ```
-      #{table(result, ['NAME', 'RUNS', 'FAILURES', 'FAILURE RATE'], "Top #{type} specs in '#{stage}' stage for past #{range} days")}
-      ```
+      #{table(result, ['NAME', 'RUNS', 'FAILURES', 'FAILURE RATE'], "Top #{type} specs in '#{stage}' stage for past #{range} days", true)}
 
       </details>
     SECTION
   end
 
-  def summary_table(summary, type)
-    table(summary, %w[STAGE COUNT], "#{type.capitalize} spec summary for past #{range} days".ljust(50))
+  def summary_table(summary, type, markdown = false)
+    table(summary, %w[STAGE COUNT], "#{type.capitalize} spec summary for past #{range} days".ljust(50), markdown)
   end
 
-  def table(rows, headings, title)
+  def table(rows, headings, title, markdown = false)
     Terminal::Table.new(
       headings: headings,
-      style: { all_separators: true },
-      title: title,
-      rows: rows
+      title: markdown ? nil : title,
+      rows: rows,
+      style: markdown ? { border: :markdown } : { all_separators: true }
     )
   end
 
   def name_column(spec_name)
-    name = "name: '#{spec_name}'"
-    file = "file: 'spec.rb'".ljust(160)
-
-    "#{name}\n#{file}"
+    "**name**: #{spec_name}<br>**file**: spec.rb"
   end
 
   before do
@@ -151,10 +144,14 @@ describe QA::Tools::ReliableReport do
 
         # Candidates for promotion to reliable (#{Date.today - range} - #{Date.today})
 
+        Total amount: **1**
+
         #{markdown_section([['manage', 1]], [[name_column('stable spec'), 3, 0, '0%']], 'manage', 'stable')}
 
         # Reliable specs with failures (#{Date.today - range} - #{Date.today})
 
+        Total amount: **1**
+
         #{markdown_section([['create', 1]], [[name_column('unstable spec'), 3, 2, '66.67%']], 'create', 'unstable')}
       TXT
     end