Filter service ping payload by category

To provide more granular control over which metrics
are reported from GitLab instances service ping payload
is filtered by 4 new categories, each of which,
has dedicated ways to control when it is included.
In case of issues in submission process, there is
a fallback to old one implemented for now.

Changelog: changed
MR: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/65944

app/services/service_ping/submit_service.rb

@@ -20,34 +20,50 @@ module ServicePing
     def execute
       return unless ServicePing::PermitDataCategoriesService.new.product_intelligence_enabled?
 
-      usage_data = Gitlab::UsageData.data(force_refresh: true)
+      begin
+        usage_data = BuildPayloadService.new.execute
+        raw_usage_data, response = submit_usage_data_payload(usage_data)
+      rescue StandardError
+        return unless Gitlab::CurrentSettings.usage_ping_enabled?
+
+        usage_data = Gitlab::UsageData.data(force_refresh: true)
+        raw_usage_data, response = submit_usage_data_payload(usage_data)
+      end
 
-      raise SubmissionError, 'Usage data is blank' if usage_data.blank?
+      version_usage_data_id = response.dig('conv_index', 'usage_data_id') || response.dig('dev_ops_score', 'usage_data_id')
 
-      raw_usage_data = save_raw_usage_data(usage_data)
+      unless version_usage_data_id.is_a?(Integer) && version_usage_data_id > 0
+        raise SubmissionError, "Invalid usage_data_id in response: #{version_usage_data_id}"
+      end
+
+      raw_usage_data.update_version_metadata!(usage_data_id: version_usage_data_id)
+
+      store_metrics(response)
+    end
+
+    private
 
-      response = Gitlab::HTTP.post(
+    def submit_payload(usage_data)
+      Gitlab::HTTP.post(
         url,
         body: usage_data.to_json,
         allow_local_requests: true,
         headers: { 'Content-type' => 'application/json' }
       )
+    end
 
-      raise SubmissionError, "Unsuccessful response code: #{response.code}" unless response.success?
+    def submit_usage_data_payload(usage_data)
+      raise SubmissionError, 'Usage data is blank' if usage_data.blank?
 
-      version_usage_data_id = response.dig('conv_index', 'usage_data_id') || response.dig('dev_ops_score', 'usage_data_id')
+      raw_usage_data = save_raw_usage_data(usage_data)
 
-      unless version_usage_data_id.is_a?(Integer) && version_usage_data_id > 0
-        raise SubmissionError, "Invalid usage_data_id in response: #{version_usage_data_id}"
-      end
+      response = submit_payload(usage_data)
 
-      raw_usage_data.update_version_metadata!(usage_data_id: version_usage_data_id)
+      raise SubmissionError, "Unsuccessful response code: #{response.code}" unless response.success?
 
-      store_metrics(response)
+      [raw_usage_data, response]
     end
 
-    private
-
     def save_raw_usage_data(usage_data)
       RawUsageData.safe_find_or_create_by(recorded_at: usage_data[:recorded_at]) do |record|
         record.payload = usage_data

ee/app/services/ee/service_ping/build_payload_service.rb

@@ -20,6 +20,8 @@ module EE
       end
 
       def filtered_usage_data(payload = raw_payload, parents = [])
+        return if payload.nil?
+
         payload.keep_if do |label, node|
           if leaf?(node)
             permitted_categories.include?(metric_category(label, parents))

spec/services/service_ping/submit_service_ping_service_spec.rb

@@ -245,11 +245,63 @@ RSpec.describe ServicePing::SubmitService do
 
     context 'and usage data is nil' do
       before do
+        allow(ServicePing::BuildPayloadService).to receive(:execute).and_return(nil)
         allow(Gitlab::UsageData).to receive(:data).and_return(nil)
       end
 
       it_behaves_like 'does not send a blank usage ping payload'
     end
+
+    context 'if payload service fails' do
+      before do
+        stub_response(body: with_dev_ops_score_params)
+        allow(ServicePing::BuildPayloadService).to receive(:execute).and_raise(described_class::SubmissionError, 'SubmissionError')
+      end
+
+      it 'calls UsageData .data method' do
+        usage_data = build_usage_data
+
+        expect(Gitlab::UsageData).to receive(:data).and_return(usage_data)
+
+        subject.execute
+      end
+    end
+
+    context 'calls BuildPayloadService first' do
+      before do
+        stub_response(body: with_dev_ops_score_params)
+      end
+
+      it 'returns usage data' do
+        usage_data = build_usage_data
+
+        expect_next_instance_of(ServicePing::BuildPayloadService) do |service|
+          expect(service).to receive(:execute).and_return(usage_data)
+        end
+
+        subject.execute
+      end
+    end
+
+    context 'if version app response fails' do
+      before do
+        stub_response(body: with_dev_ops_score_params, status: 404)
+
+        usage_data = build_usage_data
+        allow_next_instance_of(ServicePing::BuildPayloadService) do |service|
+          allow(service).to receive(:execute).and_return(usage_data)
+        end
+      end
+
+      it 'calls UsageData .data method' do
+        usage_data = build_usage_data
+
+        expect(Gitlab::UsageData).to receive(:data).and_return(usage_data)
+
+        # SubmissionError is raised as a result of 404 in response from HTTP Request
+        expect { subject.execute }.to raise_error(described_class::SubmissionError)
+      end
+    end
   end
 
   def stub_response(body:, status: 201)
@@ -260,4 +312,8 @@ RSpec.describe ServicePing::SubmitService do
         status: status
       )
   end
+
+  def build_usage_data
+    { uuid: 'uuid', recorded_at: Time.current }
+  end
 end