Merge branch 'acunskis-record-fabrication-time' into 'master'

E2E: Track fabrication time of resources

See merge request gitlab-org/gitlab!78587

qa/qa/resource/base.rb

@@ -31,7 +31,7 @@ module QA
           parents = options.fetch(:parents) { [] }
 
           do_fabricate!(resource: resource, prepare_block: prepare_block, parents: parents) do
-            log_fabrication(:browser_ui, resource, parents, args) { resource.fabricate!(*args) }
+            log_and_record_fabrication(:browser_ui, resource, parents, args) { resource.fabricate!(*args) }
 
             current_url
           end
@@ -47,7 +47,7 @@ module QA
           resource.eager_load_api_client!
 
           do_fabricate!(resource: resource, prepare_block: prepare_block, parents: parents) do
-            log_fabrication(:api, resource, parents, args) { resource.fabricate_via_api! }
+            log_and_record_fabrication(:api, resource, parents, args) { resource.fabricate_via_api! }
           end
         end
 
@@ -59,7 +59,7 @@ module QA
           resource.eager_load_api_client!
 
           do_fabricate!(resource: resource, prepare_block: prepare_block, parents: parents) do
-            log_fabrication(:api, resource, parents, args) { resource.remove_via_api! }
+            log_and_record_fabrication(:api, resource, parents, args) { resource.remove_via_api! }
           end
         end
 
@@ -71,36 +71,19 @@ module QA
           resource_web_url = yield
           resource.web_url = resource_web_url
 
-          QA::Tools::TestResourceDataProcessor.collect(resource, resource_identifier(resource))
-
           resource
         end
 
-        def resource_identifier(resource)
-          if resource.respond_to?(:username) && resource.username
-            "with username '#{resource.username}'"
-          elsif resource.respond_to?(:full_path) && resource.full_path
-            "with full_path '#{resource.full_path}'"
-          elsif resource.respond_to?(:name) && resource.name
-            "with name '#{resource.name}'"
-          elsif resource.respond_to?(:id) && resource.id
-            "with id '#{resource.id}'"
-          elsif resource.respond_to?(:iid) && resource.iid
-            "with iid '#{resource.iid}'"
-          end
-        rescue QA::Resource::Base::NoValueError
-          nil
-        end
-
-        def log_fabrication(method, resource, parents, args)
+        def log_and_record_fabrication(fabrication_method, resource, parents, args)
           start = Time.now
 
           Support::FabricationTracker.start_fabrication
           result = yield.tap do
             fabrication_time = Time.now - start
+            identifier = resource_identifier(resource)
 
             fabrication_http_method = if resource.api_fabrication_http_method == :get
-                                        if self.include?(Reusable)
+                                        if include?(Reusable)
                                           "Retrieved for reuse"
                                         else
                                           "Retrieved"
@@ -109,16 +92,23 @@ module QA
                                         "Built"
                                       end
 
-            Support::FabricationTracker.save_fabrication(:"#{method}_fabrication", fabrication_time)
+            Support::FabricationTracker.save_fabrication(:"#{fabrication_method}_fabrication", fabrication_time)
+            Tools::TestResourceDataProcessor.collect(
+              resource: resource,
+              info: identifier,
+              fabrication_method: fabrication_method,
+              fabrication_time: fabrication_time
+            )
+
             Runtime::Logger.debug do
               msg = ["==#{'=' * parents.size}>"]
               msg << "#{fabrication_http_method} a #{name}"
-              msg << resource_identifier(resource) if resource_identifier(resource)
+              msg << identifier
               msg << "as a dependency of #{parents.last}" if parents.any?
-              msg << "via #{method}"
+              msg << "via #{fabrication_method}"
               msg << "in #{fabrication_time} seconds"
 
-              msg.join(' ')
+              msg.compact.join(' ')
             end
           end
           Support::FabricationTracker.finish_fabrication
@@ -126,6 +116,26 @@ module QA
           result
         end
 
+        # Unique resource identifier
+        #
+        # @param [QA::Resource::Base] resource
+        # @return [String]
+        def resource_identifier(resource)
+          if resource.respond_to?(:username) && resource.username
+            "with username '#{resource.username}'"
+          elsif resource.respond_to?(:full_path) && resource.full_path
+            "with full_path '#{resource.full_path}'"
+          elsif resource.respond_to?(:name) && resource.name
+            "with name '#{resource.name}'"
+          elsif resource.respond_to?(:id) && resource.id
+            "with id '#{resource.id}'"
+          elsif resource.respond_to?(:iid) && resource.iid
+            "with iid '#{resource.iid}'"
+          end
+        rescue QA::Resource::Base::NoValueError
+          nil
+        end
+
         # Define custom attribute
         #
         # @param [Symbol] name

qa/qa/resource/fork.rb

@@ -31,6 +31,8 @@ module QA
         end
       end
 
+      delegate :path_with_namespace, to: :project
+
       def fabricate!
         populate(:upstream, :user)
 

qa/qa/resource/group.rb

@@ -50,12 +50,6 @@ module QA
         resource_web_url(api_get)
       rescue ResourceNotFoundError
         super
-
-        Support::Retrier.retry_on_exception(sleep_interval: 5) do
-          resource = resource_web_url(api_get)
-          populate(:runners_token)
-          resource
-        end
       end
 
       def api_get_path

qa/qa/resource/sandbox.rb

@@ -51,14 +51,6 @@ module QA
         resource_web_url(api_get)
       rescue ResourceNotFoundError
         super
-
-        # If the group was just created the runners token might not be
-        # available via the API immediately.
-        Support::Retrier.retry_on_exception(sleep_interval: 5) do
-          resource = resource_web_url(api_get)
-          populate(:runners_token)
-          resource
-        end
       end
 
       def api_get_path

qa/qa/specs/features/browser_ui/4_verify/pipeline/pass_dotenv_variables_to_downstream_via_bridge_spec.rb

@@ -25,7 +25,7 @@ module QA
         Resource::Runner.fabricate! do |runner|
           runner.name = executor
           runner.tags = [executor]
-          runner.token = group.sandbox.runners_token
+          runner.token = group.reload!.runners_token
         end
       end
 

qa/qa/specs/features/browser_ui/5_package/package_registry/npm/npm_instance_level_spec.rb

@@ -50,7 +50,7 @@ module QA
           runner.name = "qa-runner-#{Time.now.to_i}"
           runner.tags = ["runner-for-#{project.group.name}"]
           runner.executor = :docker
-          runner.token = project.group.runners_token
+          runner.token = project.group.reload!.runners_token
         end
       end
 

qa/qa/specs/features/browser_ui/5_package/package_registry/nuget_repository_spec.rb

@@ -54,7 +54,7 @@ module QA
           runner.name = "qa-runner-#{Time.now.to_i}"
           runner.tags = ["runner-for-#{project.group.name}"]
           runner.executor = :docker
-          runner.token = project.group.runners_token
+          runner.token = project.group.reload!.runners_token
         end
       end
 

qa/qa/specs/features/ee/browser_ui/4_verify/pipeline_subscription_with_group_owned_project_spec.rb

@@ -23,7 +23,7 @@ module QA
       let!(:runner) do
         Resource::Runner.fabricate! do |runner|
           runner.project = upstream_project
-          runner.token = upstream_project.group.sandbox.runners_token
+          runner.token = upstream_project.group.reload!.runners_token
           runner.name = executor
           runner.tags = [executor]
         end

qa/qa/specs/features/ee/browser_ui/6_release/multi-project_pipelines_spec.rb

@@ -23,7 +23,7 @@ module QA
 
       let!(:runner) do
         Resource::Runner.fabricate_via_api! do |runner|
-          runner.token = group.sandbox.runners_token
+          runner.token = group.reload!.runners_token
           runner.name = executor
           runner.tags = [executor]
         end

qa/qa/support/formatters/test_stats_formatter.rb

@@ -14,40 +14,38 @@ module QA
           return log(:warn, 'Missing QA_INFLUXDB_URL, skipping metrics export!') unless influxdb_url
           return log(:warn, 'Missing QA_INFLUXDB_TOKEN, skipping metrics export!') unless influxdb_token
 
-          data = notification.examples.map { |example| test_stats(example) }.compact
-          influx_client.create_write_api.write(data: data)
-          log(:info, "Pushed #{data.length} entries to influxdb")
-        rescue StandardError => e
-          log(:error, "Failed to push data to influxdb, error: #{e}")
+          push_test_stats(notification.examples)
+          push_fabrication_stats
         end
 
         private
 
-        # InfluxDb client
+        # Push test execution stats to influxdb
         #
-        # @return [InfluxDB2::Client]
-        def influx_client
-          @influx_client ||= InfluxDB2::Client.new(
-            influxdb_url,
-            influxdb_token,
-            bucket: 'e2e-test-stats',
-            org: 'gitlab-qa',
-            precision: InfluxDB2::WritePrecision::NANOSECOND
-          )
-        end
+        # @param [Array<RSpec::Core::Example>] examples
+        # @return [void]
+        def push_test_stats(examples)
+          data = examples.map { |example| test_stats(example) }.compact
 
-        # InfluxDb instance url
-        #
-        # @return [String]
-        def influxdb_url
-          @influxdb_url ||= env('QA_INFLUXDB_URL')
+          influx_client.write(data: data)
+          log(:debug, "Pushed #{data.length} test execution entries to influxdb")
+        rescue StandardError => e
+          log(:error, "Failed to push test execution stats to influxdb, error: #{e}")
         end
 
-        # Influxdb token
+        # Push resource fabrication stats to influxdb
         #
-        # @return [String]
-        def influxdb_token
-          @influxdb_token ||= env('QA_INFLUXDB_TOKEN')
+        # @return [void]
+        def push_fabrication_stats
+          data = Tools::TestResourceDataProcessor.resources.flat_map do |resource, values|
+            values.map { |v| fabrication_stats(resource: resource, **v) }
+          end
+          return if data.empty?
+
+          influx_client.write(data: data)
+          log(:debug, "Pushed #{data.length} resource fabrication entries to influxdb")
+        rescue StandardError => e
+          log(:error, "Failed to push fabrication stats to influxdb, error: #{e}")
         end
 
         # Transform example to influxdb compatible metrics data
@@ -93,6 +91,33 @@ module QA
           nil
         end
 
+        # Resource fabrication data point
+        #
+        # @param [String] resource
+        # @param [String] info
+        # @param [Symbol] fabrication_method
+        # @param [Symbol] http_method
+        # @param [Integer] fabrication_time
+        # @return [Hash]
+        def fabrication_stats(resource:, info:, fabrication_method:, http_method:, fabrication_time:, **)
+          {
+            name: 'fabrication-stats',
+            time: time,
+            tags: {
+              resource: resource,
+              fabrication_method: fabrication_method,
+              http_method: http_method,
+              run_type: env('QA_RUN_TYPE') || run_type,
+              merge_request: merge_request
+            },
+            fields: {
+              fabrication_time: fabrication_time,
+              info: info,
+              job_url: QA::Runtime::Env.ci_job_url
+            }
+          }
+        end
+
         # Project name
         #
         # @return [String]
@@ -150,7 +175,7 @@ module QA
         # @param [String] message
         # @return [void]
         def log(level, message)
-          QA::Runtime::Logger.public_send(level, "influxdb exporter: #{message}")
+          QA::Runtime::Logger.public_send(level, "[influxdb exporter]: #{message}")
         end
 
         # Return non empty environment variable value
@@ -170,6 +195,33 @@ module QA
         def devops_stage(file_path)
           file_path.match(%r{\d{1,2}_(\w+)/})&.captures&.first
         end
+
+        # InfluxDb client
+        #
+        # @return [InfluxDB2::WriteApi]
+        def influx_client
+          @influx_client ||= InfluxDB2::Client.new(
+            influxdb_url,
+            influxdb_token,
+            bucket: 'e2e-test-stats',
+            org: 'gitlab-qa',
+            precision: InfluxDB2::WritePrecision::NANOSECOND
+          ).create_write_api
+        end
+
+        # InfluxDb instance url
+        #
+        # @return [String]
+        def influxdb_url
+          @influxdb_url ||= env('QA_INFLUXDB_URL')
+        end
+
+        # Influxdb token
+        #
+        # @return [String]
+        def influxdb_token
+          @influxdb_token ||= env('QA_INFLUXDB_TOKEN')
+        end
       end
     end
   end

qa/qa/tools/test_resource_data_processor.rb

@@ -6,60 +6,80 @@
 module QA
   module Tools
     class TestResourceDataProcessor
-      @resources ||= Hash.new { |hsh, key| hsh[key] = [] }
+      include Singleton
+
+      def initialize
+        @resources = Hash.new { |hsh, key| hsh[key] = [] }
+      end
 
       class << self
-        # Ignoring rspec-mocks, sandbox, user and fork resources
-        # TODO: Will need to figure out which user resources can be collected, ignore for now
-        #
-        # Collecting resources created in E2E tests
-        # Data is a Hash of resources with keys as resource type (group, project, issue, etc.)
-        # Each type contains an array of resource object (hash) of the same type
-        # E.g: { "QA::Resource::Project": [ { info: 'foo', api_path: '/foo'}, {...} ] }
-        def collect(resource, info)
-          return if resource.api_response.nil? ||
-            resource.is_a?(RSpec::Mocks::Double) ||
-            resource.is_a?(Resource::Sandbox) ||
-            resource.is_a?(Resource::User) ||
-            resource.is_a?(Resource::Fork)
+        delegate :collect, :write_to_file, :resources, to: :instance
+      end
 
-          api_path = if resource.respond_to?(:api_delete_path)
-                       resource.api_delete_path.gsub('%2F', '/')
-                     elsif resource.respond_to?(:api_get_path)
-                       resource.api_get_path.gsub('%2F', '/')
-                     else
-                       'Cannot find resource API path'
-                     end
+      # @return [Hash<String, Array>]
+      attr_reader :resources
 
-          type = resource.class.name
+      # Collecting resources created in E2E tests
+      # Data is a Hash of resources with keys as resource type (group, project, issue, etc.)
+      # Each type contains an array of resource object (hash) of the same type
+      # E.g: { "QA::Resource::Project": [ { info: 'foo', api_path: '/foo'}, {...} ] }
+      #
+      # @param [QA::Resource::Base] resource fabricated resource
+      # @param [String] info resource info
+      # @param [Symbol] method fabrication method, api or browser_ui
+      # @param [Integer] time fabrication time
+      # @return [Hash]
+      def collect(resource:, info:, fabrication_method:, fabrication_time:)
+        api_path = resource_api_path(resource)
+        type = resource.class.name
 
-          @resources[type] << { info: info, api_path: api_path }
-        end
+        resources[type] << {
+          info: info,
+          api_path: api_path,
+          fabrication_method: fabrication_method,
+          fabrication_time: fabrication_time,
+          http_method: resource.api_fabrication_http_method
+        }
+      end
+
+      # If JSON file exists and not empty, read and load file content
+      # Merge what is saved in @resources into the content from file
+      # Overwrite file content with the new data hash
+      # Otherwise create file and write data hash to file for the first time
+      #
+      # @return [void]
+      def write_to_file
+        return if resources.empty?
 
-        # If JSON file exists and not empty, read and load file content
-        # Merge what is saved in @resources into the content from file
-        # Overwrite file content with the new data hash
-        # Otherwise create file and write data hash to file for the first time
-        def write_to_file
-          return if @resources.empty?
+        file = Pathname.new(Runtime::Env.test_resources_created_filepath)
+        FileUtils.mkdir_p(file.dirname)
 
-          file = Runtime::Env.test_resources_created_filepath
-          FileUtils.mkdir_p('tmp/')
-          FileUtils.touch(file)
-          data = nil
+        data = resources.deep_dup
+        # merge existing json if present
+        JSON.parse(File.read(file)).deep_merge!(data) { |key, val, other_val| val + other_val } if file.exist?
+
+        File.write(file, JSON.pretty_generate(data))
+      end
 
-          if File.zero?(file)
-            data = @resources
-          else
-            data = JSON.parse(File.read(file))
+      private
 
-            @resources.each_pair do |key, val|
-              data[key].nil? ? data[key] = val : val.each { |item| data[key] << item }
-            end
-          end
+      # Determine resource api path or return default value
+      # Some resources fabricated via UI can raise no attribute error
+      #
+      # @param [QA::Resource::Base] resource
+      # @return [String]
+      def resource_api_path(resource)
+        default = 'Cannot find resource API path'
 
-          File.open(file, 'w') { |f| f.write(JSON.pretty_generate(data.each_value(&:uniq!))) }
+        if resource.respond_to?(:api_delete_path)
+          resource.api_delete_path.gsub('%2F', '/')
+        elsif resource.respond_to?(:api_get_path)
+          resource.api_get_path.gsub('%2F', '/')
+        else
+          default
         end
+      rescue QA::Resource::Base::NoValueError
+        default
       end
     end
   end

qa/spec/resource/base_spec.rb

@@ -7,6 +7,11 @@ RSpec.describe QA::Resource::Base do
   let(:location) { 'http://location' }
   let(:log_regex) { %r{==> Built a MyResource with username 'qa' via #{method} in [\d.\-e]+ seconds+} }
 
+  before do
+    allow(QA::Tools::TestResourceDataProcessor).to receive(:collect)
+    allow(QA::Tools::TestResourceDataProcessor).to receive(:write_to_file)
+  end
+
   shared_context 'with fabrication context' do
     subject do
       Class.new(described_class) do

qa/spec/support/formatters/test_stats_formatter_spec.rb

@@ -22,6 +22,7 @@ describe QA::Support::Formatters::TestStatsFormatter do
   let(:file_path) { "./qa/specs/features/#{stage}/subfolder/some_spec.rb" }
   let(:ui_fabrication) { 0 }
   let(:api_fabrication) { 0 }
+  let(:fabrication_resources) { {} }
 
   let(:influx_client_args) do
     {
@@ -88,6 +89,7 @@ describe QA::Support::Formatters::TestStatsFormatter do
 
   before do
     allow(InfluxDB2::Client).to receive(:new).with(url, token, **influx_client_args) { influx_client }
+    allow(QA::Tools::TestResourceDataProcessor).to receive(:resources) { fabrication_resources }
   end
 
   context "without influxdb variables configured" do
@@ -135,6 +137,7 @@ describe QA::Support::Formatters::TestStatsFormatter do
           it('spec', :reliable, testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/1234') {}
         end
 
+        expect(influx_write_api).to have_received(:write).once
         expect(influx_write_api).to have_received(:write).with(data: [data])
       end
     end
@@ -147,6 +150,7 @@ describe QA::Support::Formatters::TestStatsFormatter do
           it('spec', :quarantine, testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/1234') {}
         end
 
+        expect(influx_write_api).to have_received(:write).once
         expect(influx_write_api).to have_received(:write).with(data: [data])
       end
     end
@@ -162,6 +166,7 @@ describe QA::Support::Formatters::TestStatsFormatter do
       it 'exports data to influxdb with correct run type' do
         run_spec
 
+        expect(influx_write_api).to have_received(:write).once
         expect(influx_write_api).to have_received(:write).with(data: [data])
       end
     end
@@ -179,6 +184,7 @@ describe QA::Support::Formatters::TestStatsFormatter do
       it 'exports data to influxdb with correct run type' do
         run_spec
 
+        expect(influx_write_api).to have_received(:write).once
         expect(influx_write_api).to have_received(:write).with(data: [data])
       end
     end
@@ -195,8 +201,48 @@ describe QA::Support::Formatters::TestStatsFormatter do
       it 'exports data to influxdb with fabrication times' do
         run_spec
 
+        expect(influx_write_api).to have_received(:write).once
         expect(influx_write_api).to have_received(:write).with(data: [data])
       end
     end
+
+    context 'with fabrication resources' do
+      let(:fabrication_resources) do
+        {
+          'QA::Resource::Project' => [{
+            info: "with id '1'",
+            api_path: '/project',
+            fabrication_method: :api,
+            fabrication_time: 1,
+            http_method: :post
+          }]
+        }
+      end
+
+      let(:fabrication_data) do
+        {
+          name: 'fabrication-stats',
+          time: DateTime.strptime(ci_timestamp).to_time,
+          tags: {
+            resource: 'QA::Resource::Project',
+            fabrication_method: :api,
+            http_method: :post,
+            run_type: run_type,
+            merge_request: "false"
+          },
+          fields: {
+            fabrication_time: 1,
+            info: "with id '1'",
+            job_url: ci_job_url
+          }
+        }
+      end
+
+      it 'exports fabrication stats data to influxdb' do
+        run_spec
+
+        expect(influx_write_api).to have_received(:write).with(data: [fabrication_data])
+      end
+    end
   end
 end

qa/spec/support/shared_contexts/packages_registry_shared_context.rb

@@ -32,7 +32,7 @@ module QA
         runner.name = "qa-runner-#{Time.now.to_i}"
         runner.tags = ["runner-for-#{package_project.group.name}"]
         runner.executor = :docker
-        runner.token = package_project.group.runners_token
+        runner.token = package_project.group.reload!.runners_token
       end
     end
 

qa/spec/tools/test_resources_data_processor_spec.rb

 # frozen_string_literal: true
 
 RSpec.describe QA::Tools::TestResourceDataProcessor do
+  include QA::Support::Helpers::StubEnv
+
+  subject(:processor) { Class.new(described_class).instance }
+
   let(:info) { 'information' }
-  let(:api_path) { '/foo' }
-  let(:result) { [{ info: info, api_path: api_path }] }
+  let(:api_response) { {} }
+  let(:method) { :api }
+  let(:time) { 2 }
+  let(:api_path) { resource.api_delete_path }
+  let(:resource) { QA::Resource::Project.init { |project| project.id = 1 } }
 
-  describe '.collect' do
-    context 'when resource is not restricted' do
-      let(:resource) { instance_double(QA::Resource::Project, api_delete_path: '/foo', api_response: 'foo') }
+  let(:result) do
+    {
+      'QA::Resource::Project' => [{
+        info: info,
+        api_path: api_path,
+        fabrication_method: method,
+        fabrication_time: time,
+        http_method: :post
+      }]
+    }
+  end
 
-      it 'collects resource' do
-        expect(described_class.collect(resource, info)).to eq(result)
-      end
+  before do
+    processor.collect(resource: resource, info: info, fabrication_method: method, fabrication_time: time)
+  end
+
+  describe '.collect' do
+    it 'collects and stores resource' do
+      expect(processor.resources).to eq(result)
     end
+  end
+
+  describe '.write_to_file' do
+    let(:resources_file) { Pathname.new(Faker::File.file_name(dir: 'tmp', ext: 'json')) }
 
-    context 'when resource api response is nil' do
-      let(:resource) { double(QA::Resource::Project, api_delete_path: '/foo', api_response: nil) }
+    before do
+      stub_env('QA_TEST_RESOURCES_CREATED_FILEPATH', resources_file)
 
-      it 'does not collect resource' do
-        expect(described_class.collect(resource, info)).to eq(nil)
-      end
+      allow(File).to receive(:write)
     end
 
-    context 'when resource is restricted' do
-      let(:resource) { double(QA::Resource::Sandbox, api_delete_path: '/foo', api_response: 'foo') }
+    it 'writes applicable resources to file' do
+      processor.write_to_file
 
-      it 'does not collect resource' do
-        expect(described_class.collect(resource, info)).to eq(nil)
-      end
+      expect(File).to have_received(:write).with(resources_file, JSON.pretty_generate(result))
     end
   end
 end