Merge branch '345389-track-failed-service-pings-for-self-managed-instances-2' into 'master'

Track failed Service Pings for self-managed instances

See merge request gitlab-org/gitlab!78028

app/services/service_ping/submit_service.rb

@@ -5,6 +5,7 @@ module ServicePing
     PRODUCTION_BASE_URL = 'https://version.gitlab.com'
     STAGING_BASE_URL = 'https://gitlab-services-version-gitlab-com-staging.gs-staging.gitlab.org'
     USAGE_DATA_PATH = 'usage_data'
+    ERROR_PATH = 'usage_ping_errors'
 
     SubmissionError = Class.new(StandardError)
 
@@ -15,12 +16,23 @@ module ServicePing
     def execute
       return unless ServicePing::ServicePingSettings.product_intelligence_enabled?
 
+      start = Time.current
       begin
         usage_data = BuildPayloadService.new.execute
         response = submit_usage_data_payload(usage_data)
-      rescue StandardError
+      rescue StandardError => e
         return unless Gitlab::CurrentSettings.usage_ping_enabled?
 
+        error_payload = {
+          time: Time.current,
+          uuid: Gitlab::UsageData.add_metric('UuidMetric'),
+          hostname: Gitlab::UsageData.add_metric('HostnameMetric'),
+          version: Gitlab::UsageData.alt_usage_data { Gitlab::VERSION },
+          message: e.message,
+          elapsed: (Time.current - start).round(1)
+        }
+        submit_payload({ error: error_payload }, url: error_url)
+
         usage_data = Gitlab::UsageData.data(force_refresh: true)
         response = submit_usage_data_payload(usage_data)
       end
@@ -42,12 +54,16 @@ module ServicePing
       URI.join(base_url, USAGE_DATA_PATH)
     end
 
+    def error_url
+      URI.join(base_url, ERROR_PATH)
+    end
+
     private
 
-    def submit_payload(usage_data)
+    def submit_payload(payload, url: self.url)
       Gitlab::HTTP.post(
         url,
-        body: usage_data.to_json,
+        body: payload.to_json,
         allow_local_requests: true,
         headers: { 'Content-type' => 'application/json' }
       )

doc/development/service_ping/index.md

@@ -205,6 +205,25 @@ sequenceDiagram
    If a firewall exception is needed, the required URL depends on several things. If
    the hostname is `version.gitlab.com`, the protocol is `TCP`, and the port number is `443`,
    the required URL is <https://version.gitlab.com/>.
+1. In case of an error, it will be reported to the Version application along with following pieces of information:
+
+- `uuid` - GitLab instance unique identifier
+- `hostname` - GitLab instance hostname
+- `version` - GitLab instance current versions 
+- `elapsed` - Amount of time which passed since Service Ping report process started and moment of error occurrence
+- `message` - Error message
+
+<pre>
+<code>
+{
+  "uuid"=>"02333324-1cd7-4c3b-a45b-a4993f05fb1d", 
+  "hostname"=>"127.0.0.1", 
+  "version"=>"14.7.0-pre", 
+  "elapsed"=>0.006946, 
+  "message"=>'PG::UndefinedColumn: ERROR:  column \"non_existent_attribute\" does not exist\nLINE 1: SELECT COUNT(non_existent_attribute) FROM \"issues\" /*applica...'
+}
+</code>
+</pre>
 
 ### On a Geo secondary site
 
@@ -510,7 +529,7 @@ To generate Service Ping, use [Teleport](https://goteleport.com/docs/) or a deta
 
 ### Verification (After approx 30 hours)
 
-#### Verify with a detached screen session
+#### Verify with Teleport
 
 1. Follow [the steps](https://gitlab.com/gitlab-com/runbooks/-/blob/master/docs/Teleport/Connect_to_Rails_Console_via_Teleport.md#how-to-use-teleport-to-connect-to-rails-console) to request a new access to the required environment and connect to the Rails console
 1. Check the last payload in `raw_usage_data` table: `RawUsageData.last.payload`

spec/services/service_ping/submit_service_ping_service_spec.rb

@@ -63,7 +63,7 @@ RSpec.describe ServicePing::SubmitService do
 
   shared_examples 'does not send a blank usage ping payload' do
     it do
-      expect(Gitlab::HTTP).not_to receive(:post)
+      expect(Gitlab::HTTP).not_to receive(:post).with(subject.url, any_args)
 
       expect { subject.execute }.to raise_error(described_class::SubmissionError) do |error|
         expect(error.message).to include('Usage data is blank')
@@ -129,6 +129,7 @@ RSpec.describe ServicePing::SubmitService do
       stub_usage_data_connections
       stub_database_flavor_check
       stub_application_setting(usage_ping_enabled: true)
+      stub_response(body: nil, url: subject.error_url, status: 201)
     end
 
     context 'and user requires usage stats consent' do
@@ -277,7 +278,8 @@ RSpec.describe ServicePing::SubmitService do
     context 'if payload service fails' do
       before do
         stub_response(body: with_dev_ops_score_params)
-        allow(ServicePing::BuildPayloadService).to receive(:execute).and_raise(described_class::SubmissionError, 'SubmissionError')
+        allow(ServicePing::BuildPayloadService).to receive_message_chain(:new, :execute)
+                                                     .and_raise(described_class::SubmissionError, 'SubmissionError')
       end
 
       it 'calls UsageData .data method' do
@@ -287,6 +289,15 @@ RSpec.describe ServicePing::SubmitService do
 
         subject.execute
       end
+
+      it 'submits error' do
+        expect(Gitlab::HTTP).to receive(:post).with(subject.url, any_args)
+                                              .and_call_original
+        expect(Gitlab::HTTP).to receive(:post).with(subject.error_url, any_args)
+                                              .and_call_original
+
+        subject.execute
+      end
     end
 
     context 'calls BuildPayloadService first' do