Push test stats directly to influxdb instance

Improve logging and add filename

Log successful push, trim filepath

Use getters for values

Use job name by default for run_type

Improve exported data

Improve error handling

Check for top upstream mr iid variable

Add flag to disable metrics export

Fix quarantine key name

Remove nil safe operator

Use single timestamp for export

Use pipeline creation timestamp

Remove pipeline id tag

Add metrics exporter specs

Do not export metrics for qa unit tests

Simplify stop notification send method

Remove unused variable

Add pipeline_id value

Separate block style let var with empty line

Do not fallback on job_name if QA_RUN_TYPE is not set

.gitlab/ci/qa.gitlab-ci.yml

@@ -7,6 +7,7 @@
   variables:
     USE_BUNDLE_INSTALL: "false"
     SETUP_DB: "false"
+    QA_EXPORT_TEST_METRICS: "false"
   before_script:
     - !reference [.default-before_script, before_script]
     - cd qa/

qa/Gemfile

@@ -24,6 +24,7 @@ gem 'rspec-parameterized', '~> 0.4.2'
 gem 'octokit', '~> 4.21'
 gem 'webdrivers', '~> 4.6'
 gem 'zeitwerk', '~> 2.4'
+gem 'influxdb-client', '~> 1.17'
 
 gem 'chemlab', '~> 0.7'
 gem 'chemlab-library-www-gitlab-com', '~> 0.1'

qa/Gemfile.lock

@@ -88,6 +88,7 @@ GEM
     i18n (1.8.10)
       concurrent-ruby (~> 1.0)
     ice_nine (0.11.2)
+    influxdb-client (1.17.0)
     knapsack (1.17.1)
       rake
     launchy (2.4.3)
@@ -220,6 +221,7 @@ DEPENDENCIES
   deprecation_toolkit (~> 1.5.1)
   faker (~> 2.19, >= 2.19.0)
   gitlab-qa
+  influxdb-client (~> 1.17)
   knapsack (~> 1.17)
   octokit (~> 4.21)
   parallel (~> 1.19)

qa/qa/runtime/env.rb

@@ -403,6 +403,10 @@ module QA
         ENV['GITLAB_TLS_CERTIFICATE']
       end
 
+      def export_metrics?
+        running_in_ci? && enabled?(ENV['QA_EXPORT_TEST_METRICS'], default: true)
+      end
+
       private
 
       def remote_grid_credentials

qa/qa/specs/helpers/rspec.rb

@@ -19,13 +19,22 @@ module QA
         # expanding into the global state
         # See: https://github.com/rspec/rspec-core/issues/2603
         def describe_successfully(*args, &describe_body)
-          reporter = ::RSpec.configuration.reporter
-
-          example_group = RSpec.describe(*args, &describe_body)
+          example_group = ::RSpec.describe(*args, &describe_body)
           ran_successfully = example_group.run reporter
           expect(ran_successfully).to eq true
           example_group
         end
+
+        def send_stop_notification
+          reporter.notify(
+            :stop,
+            ::RSpec::Core::Notifications::ExamplesNotification.new(reporter)
+          )
+        end
+
+        def reporter
+          ::RSpec.configuration.reporter
+        end
       end
     end
   end

qa/qa/support/formatters/test_stats_formatter.rb

@@ -4,52 +4,114 @@ module QA
   module Support
     module Formatters
       class TestStatsFormatter < RSpec::Core::Formatters::BaseFormatter
-        STATS_DIR = 'tmp/test-stats'
-
-        RSpec::Core::Formatters.register(
-          self,
-          :start,
-          :stop
-        )
-
-        # Start test run
-        #
-        # @param [RSpec::Core::Notifications::StartNotification] _notification
-        # @return [void]
-        def start(_notification)
-          FileUtils.mkdir_p(STATS_DIR)
-        end
+        RSpec::Core::Formatters.register(self, :stop)
 
         # Finish test execution
         #
         # @param [RSpec::Core::Notifications::ExamplesNotification] notification
         # @return [void]
         def stop(notification)
-          File.open(File.join(STATS_DIR, "test-stats-#{SecureRandom.uuid}.json"), 'w') do |file|
-            specs = notification.examples.map { |example| test_stats(example) }
-            file.write(specs.to_json)
-          end
+          return log(:warn, 'Missing QA_INFLUXDB_URL, skipping metrics export!') unless influxdb_url
+          return log(:warn, 'Missing QA_INFLUXDB_TOKEN, skipping metrics export!') unless influxdb_token
+
+          data = notification.examples.map { |example| test_stats(example) }.compact
+          influx_client.create_write_api.write(data: data)
+          log(:info, "Pushed #{data.length} entries to influxdb")
+        rescue StandardError => e
+          log(:error, "Failed to push data to influxdb, error: #{e}")
         end
 
         private
 
-        # Get test stats
+        # InfluxDb client
+        #
+        # @return [InfluxDB2::Client]
+        def influx_client
+          @influx_client ||= InfluxDB2::Client.new(
+            influxdb_url,
+            influxdb_token,
+            bucket: 'e2e-test-stats',
+            org: 'gitlab-qa',
+            use_ssl: false,
+            precision: InfluxDB2::WritePrecision::NANOSECOND
+          )
+        end
+
+        # InfluxDb instance url
+        #
+        # @return [String]
+        def influxdb_url
+          @influxdb_url ||= ENV['QA_INFLUXDB_URL']
+        end
+
+        # Influxdb token
+        #
+        # @return [String]
+        def influxdb_token
+          @influxdb_token ||= ENV['QA_INFLUXDB_TOKEN']
+        end
+
+        # Transform example to influxdb compatible metrics data
+        # https://github.com/influxdata/influxdb-client-ruby#data-format
         #
         # @param [RSpec::Core::Example] example
         # @return [Hash]
         def test_stats(example)
           {
-            id: example.id,
+            name: 'test-stats',
+            time: time,
+            tags: {
               name: example.full_description,
+              file_path: example.metadata[:file_path].gsub('./qa/specs/features', ''),
               status: example.execution_result.status,
-            reliable: example.metadata.key?(:reliable),
-            quarantined: example.metadata.key?(:quarantined),
-            run_time: example.execution_result.run_time,
-            attempts: example.metadata[:retry_attempts],
-            # use allure job name since it is already present and indicates what type of run it is
-            # example: staging-full, staging-sanity etc.
-            run_type: ENV['ALLURE_JOB_NAME']
+              reliable: example.metadata.key?(:reliable).to_s,
+              quarantined: example.metadata.key?(:quarantine).to_s,
+              retried: ((example.metadata[:retry_attempts] || 0) > 0).to_s,
+              job_name: job_name,
+              merge_request: merge_request,
+              run_type: ENV['QA_RUN_TYPE']
+            },
+            fields: {
+              id: example.id,
+              run_time: (example.execution_result.run_time * 1000).round,
+              retry_attempts: example.metadata[:retry_attempts] || 0,
+              job_url: QA::Runtime::Env.ci_job_url,
+              pipeline_id: ENV['CI_PIPELINE_ID']
             }
+          }
+        rescue StandardError => e
+          log(:error, "Failed to transform example '#{example.id}', error: #{e}")
+          nil
+        end
+
+        # Single common timestamp for all exported example metrics to keep data points consistently grouped
+        #
+        # @return [Time]
+        def time
+          @time ||= DateTime.strptime(ENV['CI_PIPELINE_CREATED_AT']).to_time
+        end
+
+        # Is a merge request execution
+        #
+        # @return [String]
+        def merge_request
+          @merge_request ||= (!!ENV['CI_MERGE_REQUEST_IID'] || !!ENV['TOP_UPSTREAM_MERGE_REQUEST_IID']).to_s
+        end
+
+        # Base ci job name
+        #
+        # @return [String]
+        def job_name
+          @job_name ||= QA::Runtime::Env.ci_job_name.gsub(%r{ \d{1,2}/\d{1,2}}, '')
+        end
+
+        # Print log message
+        #
+        # @param [Symbol] level
+        # @param [String] message
+        # @return [void]
+        def log(level, message)
+          QA::Runtime::Logger.public_send(level, "influxdb exporter: #{message}")
         end
       end
     end

qa/spec/spec_helper.rb

@@ -24,7 +24,7 @@ RSpec.configure do |config|
 
   config.add_formatter QA::Support::Formatters::ContextFormatter
   config.add_formatter QA::Support::Formatters::QuarantineFormatter
-  config.add_formatter QA::Support::Formatters::TestStatsFormatter if QA::Runtime::Env.running_in_ci?
+  config.add_formatter QA::Support::Formatters::TestStatsFormatter if QA::Runtime::Env.export_metrics?
 
   config.before do |example|
     QA::Runtime::Logger.debug("\nStarting test: #{example.full_description}\n")

qa/spec/support/formatters/test_stats_formatter_spec.rb

+# frozen_string_literal: true
+
+require 'rspec/core/sandbox'
+
+describe QA::Support::Formatters::TestStatsFormatter do
+  include QA::Support::Helpers::StubEnv
+  include QA::Specs::Helpers::RSpec
+
+  let(:url) { "http://influxdb.net" }
+  let(:token) { "token" }
+  let(:ci_timestamp) { "2021-02-23T20:58:41Z" }
+  let(:ci_job_name) { "test-job 1/5" }
+  let(:ci_job_url) { "url" }
+  let(:ci_pipeline_id) { "123" }
+  let(:run_type) { 'staging-full' }
+  let(:influx_client) { instance_double('InfluxDB2::Client', create_write_api: influx_write_api) }
+  let(:influx_write_api) { instance_double('InfluxDB2::WriteApi', write: nil) }
+
+  let(:influx_client_args) do
+    {
+      bucket: 'e2e-test-stats',
+      org: 'gitlab-qa',
+      use_ssl: false,
+      precision: InfluxDB2::WritePrecision::NANOSECOND
+    }
+  end
+
+  let(:data) do
+    {
+      name: 'test-stats',
+      time: DateTime.strptime(ci_timestamp).to_time,
+      tags: {
+        name: "stats export #{spec_name}",
+        file_path: './spec/support/formatters/test_stats_formatter_spec.rb',
+        status: :passed,
+        reliable: reliable,
+        quarantined: quarantined,
+        retried: "false",
+        job_name: "test-job",
+        merge_request: "false",
+        run_type: run_type
+      },
+      fields: {
+        id: './spec/support/formatters/test_stats_formatter_spec.rb[1:1]',
+        run_time: 0,
+        retry_attempts: 0,
+        job_url: ci_job_url,
+        pipeline_id: ci_pipeline_id
+      }
+    }
+  end
+
+  def run_spec(&spec)
+    describe_successfully('stats export', &spec)
+    send_stop_notification
+  end
+
+  around do |example|
+    RSpec::Core::Sandbox.sandboxed do |config|
+      config.formatter = QA::Support::Formatters::TestStatsFormatter
+
+      config.before(:context) { RSpec.current_example = nil }
+
+      example.run
+    end
+  end
+
+  before do
+    allow(InfluxDB2::Client).to receive(:new).with(url, token, **influx_client_args) { influx_client }
+  end
+
+  context "without influxdb variables configured" do
+    it "skips export without influxdb url" do
+      stub_env('QA_INFLUXDB_URL', nil)
+      stub_env('QA_INFLUXDB_TOKEN', nil)
+
+      run_spec do
+        it('skips export') {}
+      end
+
+      expect(influx_client).not_to receive(:create_write_api)
+    end
+
+    it "skips export without influxdb token" do
+      stub_env('QA_INFLUXDB_URL', url)
+      stub_env('QA_INFLUXDB_TOKEN', nil)
+
+      run_spec do
+        it('skips export') {}
+      end
+
+      expect(influx_client).not_to receive(:create_write_api)
+    end
+  end
+
+  context 'with influxdb variables configured' do
+    let(:spec_name) { 'exports data' }
+    let(:run_type) { ci_job_name.gsub(%r{ \d{1,2}/\d{1,2}}, '') }
+
+    before do
+      stub_env('QA_INFLUXDB_URL', url)
+      stub_env('QA_INFLUXDB_TOKEN', token)
+      stub_env('CI_PIPELINE_CREATED_AT', ci_timestamp)
+      stub_env('CI_JOB_URL', ci_job_url)
+      stub_env('CI_JOB_NAME', ci_job_name)
+      stub_env('CI_PIPELINE_ID', ci_pipeline_id)
+      stub_env('CI_MERGE_REQUEST_IID', nil)
+      stub_env('TOP_UPSTREAM_MERGE_REQUEST_IID', nil)
+      stub_env('QA_RUN_TYPE', run_type)
+    end
+
+    context 'with reliable spec' do
+      let(:reliable) { 'true' }
+      let(:quarantined) { 'false' }
+
+      it 'exports data to influxdb' do
+        run_spec do
+          it('exports data', :reliable) {}
+        end
+
+        expect(influx_write_api).to have_received(:write).with(data: [data])
+      end
+    end
+
+    context 'with quarantined spec' do
+      let(:reliable) { 'false' }
+      let(:quarantined) { 'true' }
+
+      it 'exports data to influxdb' do
+        run_spec do
+          it('exports data', :quarantine) {}
+        end
+
+        expect(influx_write_api).to have_received(:write).with(data: [data])
+      end
+    end
+  end
+end