Merge branch 'acunskis-metrics-detect-run-type' into 'master'

E2E: Detect test run type for staging, canary and production environment for metrics exports

See merge request gitlab-org/gitlab!69754

qa/qa/support/formatters/test_stats_formatter.rb

@@ -41,14 +41,14 @@ module QA
         #
         # @return [String]
         def influxdb_url
-          @influxdb_url ||= ENV['QA_INFLUXDB_URL']
+          @influxdb_url ||= env('QA_INFLUXDB_URL')
         end
 
         # Influxdb token
         #
         # @return [String]
         def influxdb_token
-          @influxdb_token ||= ENV['QA_INFLUXDB_TOKEN']
+          @influxdb_token ||= env('QA_INFLUXDB_TOKEN')
         end
 
         # Transform example to influxdb compatible metrics data
@@ -69,14 +69,14 @@ module QA
               retried: ((example.metadata[:retry_attempts] || 0) > 0).to_s,
               job_name: job_name,
               merge_request: merge_request,
-              run_type: ENV['QA_RUN_TYPE']
+              run_type: env('QA_RUN_TYPE') || run_type
             },
             fields: {
               id: example.id,
               run_time: (example.execution_result.run_time * 1000).round,
               retry_attempts: example.metadata[:retry_attempts] || 0,
               job_url: QA::Runtime::Env.ci_job_url,
-              pipeline_id: ENV['CI_PIPELINE_ID']
+              pipeline_id: env('CI_PIPELINE_ID')
             }
           }
         rescue StandardError => e
@@ -84,25 +84,51 @@ module QA
           nil
         end
 
+        # Project name
+        #
+        # @return [String]
+        def project_name
+          @project_name ||= QA::Runtime::Env.ci_project_name
+        end
+
+        # Base ci job name
+        #
+        # @return [String]
+        def job_name
+          @job_name ||= QA::Runtime::Env.ci_job_name.gsub(%r{ \d{1,2}/\d{1,2}}, '')
+        end
+
         # Single common timestamp for all exported example metrics to keep data points consistently grouped
         #
         # @return [Time]
         def time
-          @time ||= DateTime.strptime(ENV['CI_PIPELINE_CREATED_AT']).to_time
+          @time ||= DateTime.strptime(env('CI_PIPELINE_CREATED_AT')).to_time
         end
 
         # Is a merge request execution
         #
         # @return [String]
         def merge_request
-          @merge_request ||= (!!ENV['CI_MERGE_REQUEST_IID'] || !!ENV['TOP_UPSTREAM_MERGE_REQUEST_IID']).to_s
+          @merge_request ||= (!!env('CI_MERGE_REQUEST_IID') || !!env('TOP_UPSTREAM_MERGE_REQUEST_IID')).to_s
         end
 
-        # Base ci job name
+        # Test run type from staging, canary or production env
         #
-        # @return [String]
-        def job_name
-          @job_name ||= QA::Runtime::Env.ci_job_name.gsub(%r{ \d{1,2}/\d{1,2}}, '')
+        # @return [String>, nil]
+        def run_type
+          return unless %w[staging canary production].include?(project_name)
+
+          @run_type ||= begin
+            test_subset = if env('NO_ADMIN') == 'true'
+                            'sanity-no-admin'
+                          elsif env('SMOKE_ONLY') == 'true'
+                            'sanity'
+                          else
+                            'full'
+                          end
+
+            "#{project_name}-#{test_subset}"
+          end
         end
 
         # Print log message
@@ -113,6 +139,16 @@ module QA
         def log(level, message)
           QA::Runtime::Logger.public_send(level, "influxdb exporter: #{message}")
         end
+
+        # Return non empty environment variable value
+        #
+        # @param [String] name
+        # @return [String, nil]
+        def env(name)
+          return unless ENV[name] && !ENV[name].empty?
+
+          ENV[name]
+        end
       end
     end
   end

qa/spec/support/formatters/test_stats_formatter_spec.rb

@@ -13,6 +13,8 @@ describe QA::Support::Formatters::TestStatsFormatter do
   let(:ci_job_url) { "url" }
   let(:ci_pipeline_id) { "123" }
   let(:run_type) { 'staging-full' }
+  let(:reliable) { 'false' }
+  let(:quarantined) { 'false' }
   let(:influx_client) { instance_double('InfluxDB2::Client', create_write_api: influx_write_api) }
   let(:influx_write_api) { instance_double('InfluxDB2::WriteApi', write: nil) }
 
@@ -30,7 +32,7 @@ describe QA::Support::Formatters::TestStatsFormatter do
       name: 'test-stats',
       time: DateTime.strptime(ci_timestamp).to_time,
       tags: {
-        name: "stats export #{spec_name}",
+        name: 'stats export spec',
         file_path: './spec/support/formatters/test_stats_formatter_spec.rb',
         status: :passed,
         reliable: reliable,
@@ -51,6 +53,8 @@ describe QA::Support::Formatters::TestStatsFormatter do
   end
 
   def run_spec(&spec)
+    spec ||= -> { it('spec') {} }
+
     describe_successfully('stats export', &spec)
     send_stop_notification
   end
@@ -74,9 +78,7 @@ describe QA::Support::Formatters::TestStatsFormatter do
       stub_env('QA_INFLUXDB_URL', nil)
       stub_env('QA_INFLUXDB_TOKEN', nil)
 
-      run_spec do
-        it('skips export') {}
-      end
+      run_spec
 
       expect(influx_client).not_to have_received(:create_write_api)
     end
@@ -85,9 +87,7 @@ describe QA::Support::Formatters::TestStatsFormatter do
       stub_env('QA_INFLUXDB_URL', url)
       stub_env('QA_INFLUXDB_TOKEN', nil)
 
-      run_spec do
-        it('skips export') {}
-      end
+      run_spec
 
       expect(influx_client).not_to have_received(:create_write_api)
     end
@@ -111,11 +111,10 @@ describe QA::Support::Formatters::TestStatsFormatter do
 
     context 'with reliable spec' do
       let(:reliable) { 'true' }
-      let(:quarantined) { 'false' }
 
-      it 'exports data to influxdb' do
+      it 'exports data to influxdb with correct reliable tag' do
         run_spec do
-          it('exports data', :reliable) {}
+          it('spec', :reliable) {}
         end
 
         expect(influx_write_api).to have_received(:write).with(data: [data])
@@ -123,14 +122,45 @@ describe QA::Support::Formatters::TestStatsFormatter do
     end
 
     context 'with quarantined spec' do
-      let(:reliable) { 'false' }
       let(:quarantined) { 'true' }
 
-      it 'exports data to influxdb' do
+      it 'exports data to influxdb with correct quarantine tag' do
         run_spec do
-          it('exports data', :quarantine) {}
+          it('spec', :quarantine) {}
+        end
+
+        expect(influx_write_api).to have_received(:write).with(data: [data])
+      end
+    end
+
+    context 'with staging full run' do
+      let(:run_type) { 'staging-full' }
+
+      before do
+        stub_env('CI_PROJECT_NAME', 'staging')
+        stub_env('QA_RUN_TYPE', nil)
+      end
+
+      it 'exports data to influxdb with correct run type' do
+        run_spec
+
+        expect(influx_write_api).to have_received(:write).with(data: [data])
+      end
     end
 
+    context 'with staging sanity no admin' do
+      let(:run_type) { 'staging-sanity-no-admin' }
+
+      before do
+        stub_env('CI_PROJECT_NAME', 'staging')
+        stub_env('NO_ADMIN', 'true')
+        stub_env('SMOKE_ONLY', 'true')
+        stub_env('QA_RUN_TYPE', nil)
+      end
+
+      it 'exports data to influxdb with correct run type' do
+        run_spec
+
         expect(influx_write_api).to have_received(:write).with(data: [data])
       end
     end