Merge branch 'acunskis-reliable-report-stages' into 'master'

E2E: Improve reliable|unreliable spec report

See merge request gitlab-org/gitlab!74871

qa/qa/support/formatters/test_stats_formatter.rb

@@ -167,7 +167,7 @@ module QA
         # @param [String] location
         # @return [String, nil]
         def devops_stage(file_path)
-          file_path.match(%r{(\d{1,2}_\w+)/})&.captures&.first
+          file_path.match(%r{\d{1,2}_(\w+)/})&.captures&.first
         end
       end
     end

qa/qa/tools/reliable_report.rb

@@ -8,7 +8,7 @@ module QA
   module Tools
     class ReliableReport
       def initialize(run_type, range = 30)
-        @results = 10
+        @results = 2
         @slack_channel = "#quality-reports"
         @range = range
         @run_type = run_type
@@ -20,27 +20,17 @@ module QA
       #
       # @return [void]
       def show_top_stable
-        puts terminal_table(
-          rows: top_stable.map { |k, v| [name_column(k, v[:file]), *table_params(v.values)] },
-          title: stable_title
-        )
+        results_table(:stable).each { |table| puts "#{table}\n\n" }
       end
 
       # Post top stable spec report to slack
-      # Slice table in to multiple messages due to max char limitation
       #
       # @return [void]
       def notify_top_stable
-        tables = top_stable.each_slice(5).map do |slice|
-          terminal_table(
-            rows: slice.map { |spec| [name_column(spec[0], spec[1][:file]), *table_params(spec[1].values)] }
-          )
-        end
-
         puts "\nSending top stable spec report to #{slack_channel} slack channel"
         slack_args = { icon_emoji: ":mtg_green:", username: "Stable Spec Report" }
         notifier.post(text: "*#{stable_title}*", **slack_args)
-        tables.each { |table| notifier.post(text: "```#{table}```", **slack_args) }
+        results_table(:stable).each { |table| notifier.post(text: "```#{table}```", **slack_args) }
       end
 
       # Print top unstable specs
@@ -49,29 +39,19 @@ module QA
       def show_top_unstable
         return puts("No unstable tests present!") if top_unstable_reliable.empty?
 
-        puts terminal_table(
-          rows: top_unstable_reliable.map { |k, v| [name_column(k, v[:file]), *table_params(v.values)] },
-          title: unstable_title
-        )
+        results_table(:unstable).each { |table| puts "#{table}\n\n" }
       end
 
       # Post top unstable reliable spec report to slack
-      # Slice table in to multiple messages due to max char limitation
       #
       # @return [void]
       def notify_top_unstable
         return puts("No unstable tests present!") if top_unstable_reliable.empty?
 
-        tables = top_unstable_reliable.each_slice(5).map do |slice|
-          terminal_table(
-            rows: slice.map { |spec| [name_column(spec[0], spec[1][:file]), *table_params(spec[1].values)] }
-          )
-        end
-
         puts "\nSending top unstable reliable spec report to #{slack_channel} slack channel"
         slack_args = { icon_emoji: ":sadpanda:", username: "Unstable Spec Report" }
         notifier.post(text: "*#{unstable_title}*", **slack_args)
-        tables.each { |table| notifier.post(text: "```#{table}```", **slack_args) }
+        results_table(:unstable).each { |table| notifier.post(text: "```#{table}```", **slack_args) }
       end
 
       private
@@ -83,21 +63,42 @@ module QA
                   :stable_title,
                   :unstable_title
 
+      # Results table
+      #
+      # @param [Symbol] type result type - :stable, :unstable
+      # @return [Hash]
+      def results_table(type)
+        (type == :stable ? top_stable : top_unstable_reliable).map do |stage, specs|
+          terminal_table(
+            rows: specs.map { |k, v| [name_column(k, v[:file]), *table_params(v.values)] },
+            title: "Top #{type} specs in '#{stage}' stage"
+          )
+        end
+      end
+
       # Top stable specs
       #
       # @return [Hash]
       def top_stable
-        @top_stable ||= runs(reliable: false).sort_by { |k, v| [v[:failure_rate], -v[:runs]] }[0..results - 1].to_h
+        @top_stable ||= runs(reliable: false).transform_values do |specs|
+          specs.sort_by { |k, v| [v[:failure_rate], -v[:runs]] }[0..results - 1].to_h
+        end
       end
 
       # Top unstable reliable specs
       #
       # @return [Hash]
       def top_unstable_reliable
-        @top_unstable_reliable ||= runs(reliable: true)
-          .reject { |k, v| v[:failure_rate] == 0 }
-          .sort_by { |k, v| -v[:failure_rate] }[0..results - 1]
-          .to_h
+        @top_unstable_reliable ||= begin
+          unstable = runs(reliable: true).transform_values do |specs|
+            specs
+              .reject { |k, v| v[:failure_rate] == 0 }
+              .sort_by { |k, v| -v[:failure_rate] }[0..results - 1]
+              .to_h
+          end
+
+          unstable.reject { |k, v| v.empty? }
+        end
       end
 
       # Terminal table for result formatting
@@ -136,20 +137,23 @@ module QA
       # Test executions grouped by name
       #
       # @param [Boolean] reliable
-      # @return [Hash]
+      # @return [Hash<String, Hash>]
       def runs(reliable:)
         puts("Fetching data on #{reliable ? 'reliable ' : ''}test execution for past 30 days in '#{run_type}' runs")
         puts
 
-        query_api.query(query: query(reliable)).values.each_with_object({}) do |table, result|
+        all_runs = query_api.query(query: query(reliable)).values
+        all_runs.each_with_object(Hash.new { |hsh, key| hsh[key] = {} }) do |table, result|
           records = table.records
           name = records.last.values["name"]
           file = records.last.values["file_path"].split("/").last
+          stage = records.last.values["stage"] || "unknown"
+
           runs = records.count
           failed = records.count { |r| r.values["status"] == "failed" }
           failure_rate = (failed.to_f / runs.to_f) * 100
 
-          result[name] = {
+          result[stage][name] = {
             file: file,
             runs: runs,
             failed: failed,

qa/spec/support/formatters/test_stats_formatter_spec.rb

@@ -45,7 +45,7 @@ describe QA::Support::Formatters::TestStatsFormatter do
         job_name: "test-job",
         merge_request: "false",
         run_type: run_type,
-        stage: stage
+        stage: stage.match(%r{\d{1,2}_(\w+)}).captures.first
       },
       fields: {
         id: './spec/support/formatters/test_stats_formatter_spec.rb[1:1]',

qa/spec/tools/reliable_report_spec.rb

@@ -12,30 +12,31 @@ describe QA::Tools::ReliableReport do
   let(:slack_channel) { "#quality-reports" }
   let(:run_type) { "package-and-qa" }
   let(:range) { 30 }
-  let(:results) { 10 }
+  let(:results) { 2 }
 
   let(:runs) { { 0 => stable_spec, 1 => unstable_spec } }
 
+  let(:spec_values) { { "file_path" => "some/spec.rb", "stage" => "manage" } }
   let(:stable_spec) do
-    spec_values = { "name" => "stable spec", "status" => "passed", "file_path" => "some/spec.rb" }
+    values = { "name" => "stable spec", "status" => "passed", **spec_values }
     instance_double(
       "InfluxDB2::FluxTable",
       records: [
-        instance_double("InfluxDB2::FluxRecord", values: spec_values),
-        instance_double("InfluxDB2::FluxRecord", values: spec_values),
-        instance_double("InfluxDB2::FluxRecord", values: spec_values)
+        instance_double("InfluxDB2::FluxRecord", values: values),
+        instance_double("InfluxDB2::FluxRecord", values: values),
+        instance_double("InfluxDB2::FluxRecord", values: values)
       ]
     )
   end
 
   let(:unstable_spec) do
-    spec_values = { "name" => "unstable spec", "status" => "failed", "file_path" => "some/spec.rb" }
+    values = { "name" => "unstable spec", "status" => "failed", **spec_values }
     instance_double(
       "InfluxDB2::FluxTable",
       records: [
-        instance_double("InfluxDB2::FluxRecord", values: { **spec_values, "status" => "passed" }),
-        instance_double("InfluxDB2::FluxRecord", values: spec_values),
-        instance_double("InfluxDB2::FluxRecord", values: spec_values)
+        instance_double("InfluxDB2::FluxRecord", values: { **values, "status" => "passed" }),
+        instance_double("InfluxDB2::FluxRecord", values: values),
+        instance_double("InfluxDB2::FluxRecord", values: values)
       ]
     )
   end
@@ -86,7 +87,8 @@ describe QA::Tools::ReliableReport do
     let(:query) { flux_query(false) }
     let(:fetch_message) { "Fetching data on test execution for past #{range} days in '#{run_type}' runs" }
     let(:slack_send_message) { "Sending top stable spec report to #{slack_channel} slack channel" }
-    let(:title) { "Top #{results} stable specs for past #{range} days in '#{run_type}' runs" }
+    let(:message_title) { "Top #{results} stable specs for past #{range} days in '#{run_type}' runs" }
+    let(:table_title) { "Top stable specs in 'manage' stage" }
     let(:rows) do
       [
         [name_column("stable spec"), 3, 0, "0%"],
@@ -95,15 +97,15 @@ describe QA::Tools::ReliableReport do
     end
 
     it "prints top stable spec report to console" do
-      expect { reporter.show_top_stable }.to output("#{fetch_message}\n\n#{table(rows, title)}\n").to_stdout
+      expect { reporter.show_top_stable }.to output("#{fetch_message}\n\n#{table(rows, table_title)}\n\n").to_stdout
     end
 
     it "sends top stable spec report to slack" do
       slack_args = { icon_emoji: ":mtg_green:", username: "Stable Spec Report" }
 
-      expect { reporter.notify_top_stable }.to output("#{fetch_message}\n\n\n#{slack_send_message}\n").to_stdout
-      expect(slack_notifier).to have_received(:post).with(text: "*#{title}*", **slack_args)
-      expect(slack_notifier).to have_received(:post).with(text: "```#{table(rows)}```", **slack_args)
+      expect { reporter.notify_top_stable }.to output("\n#{slack_send_message}\n#{fetch_message}\n\n").to_stdout
+      expect(slack_notifier).to have_received(:post).with(text: "*#{message_title}*", **slack_args)
+      expect(slack_notifier).to have_received(:post).with(text: "```#{table(rows, table_title)}```", **slack_args)
     end
   end
 
@@ -111,19 +113,20 @@ describe QA::Tools::ReliableReport do
     let(:query) { flux_query(true) }
     let(:fetch_message) { "Fetching data on reliable test execution for past #{range} days in '#{run_type}' runs" }
     let(:slack_send_message) { "Sending top unstable reliable spec report to #{slack_channel} slack channel" }
-    let(:title) { "Top #{results} unstable reliable specs for past #{range} days in '#{run_type}' runs" }
+    let(:message_title) { "Top #{results} unstable reliable specs for past #{range} days in '#{run_type}' runs" }
+    let(:table_title) { "Top unstable specs in 'manage' stage" }
     let(:rows) { [[name_column("unstable spec"), 3, 2, "66.67%"]] }
 
     it "prints top unstable spec report to console" do
-      expect { reporter.show_top_unstable }.to output("#{fetch_message}\n\n#{table(rows, title)}\n").to_stdout
+      expect { reporter.show_top_unstable }.to output("#{fetch_message}\n\n#{table(rows, table_title)}\n\n").to_stdout
     end
 
     it "sends top unstable reliable spec report to slack" do
       slack_args = { icon_emoji: ":sadpanda:", username: "Unstable Spec Report" }
 
       expect { reporter.notify_top_unstable }.to output("#{fetch_message}\n\n\n#{slack_send_message}\n").to_stdout
-      expect(slack_notifier).to have_received(:post).with(text: "*#{title}*", **slack_args)
-      expect(slack_notifier).to have_received(:post).with(text: "```#{table(rows)}```", **slack_args)
+      expect(slack_notifier).to have_received(:post).with(text: "*#{message_title}*", **slack_args)
+      expect(slack_notifier).to have_received(:post).with(text: "```#{table(rows, table_title)}```", **slack_args)
     end
   end