Merge branch '220840-add-machine-sysname-in-topology-usageping' into 'master'

Collect specifics about machine type and architecture

Closes #220840

See merge request gitlab-org/gitlab!34627

changelogs/unreleased/220840-add-machine-sysname-in-topology-usageping.yml

+---
+title: Add machine/sysname/release in topology usage ping
+merge_request: 34627
+author:
+type: other

doc/development/telemetry/usage_ping.md

@@ -708,6 +708,7 @@ appear to be associated to any of the services running, since they all appear to
 | `nodes`                                                   | `topology`                           | `enablement`  |                  |         | The list of server nodes on which GitLab components are running            |
 | `node_memory_total_bytes`                                 | `topology > nodes`                   | `enablement`  |                  |         | The total available memory of this node                                    |
 | `node_cpus`                                               | `topology > nodes`                   | `enablement`  |                  |         | The number of CPU cores of this node                                       |
+| `node_uname_info`                                         | `topology > nodes`                   | `enablement`  |                  |         | The basic hardware architecture and OS release information on this node    |
 | `node_services`                                           | `topology > nodes`                   | `enablement`  |                  |         | The list of GitLab services running on this node                           |
 | `name`                                                    | `topology > nodes > node_services`   | `enablement`  |                  |         | The name of the GitLab service running on this node                        |
 | `process_count`                                           | `topology > nodes > node_services`   | `enablement`  |                  |         | The number of processes running for this service                           |
@@ -920,6 +921,11 @@ The following is example content of the Usage Ping payload.
       {
         "node_memory_total_bytes": 33269903360,
         "node_cpus": 16,
+        "node_uname_info": {
+          "machine": "x86_64",
+          "sysname": "Linux",
+          "release": "4.19.76-linuxkit"
+        },
         "node_services": [
           {
             "name": "web",

lib/gitlab/usage_data/topology.rb

@@ -62,6 +62,7 @@ module Gitlab
         # node-level data
         by_instance_mem = topology_node_memory(client)
         by_instance_cpus = topology_node_cpus(client)
+        by_instance_uname_info = topology_node_uname_info(client)
         # service-level data
         by_instance_by_job_by_type_memory = topology_all_service_memory(client)
         by_instance_by_job_process_count = topology_all_service_process_count(client)
@@ -72,6 +73,7 @@ module Gitlab
           {
             node_memory_total_bytes: by_instance_mem[instance],
             node_cpus: by_instance_cpus[instance],
+            node_uname_info: by_instance_uname_info[instance],
             node_services:
               topology_node_services(
                 instance, by_instance_by_job_process_count, by_instance_by_job_by_type_memory, by_instance_by_job_server_types
@@ -92,6 +94,14 @@ module Gitlab
         end
       end
 
+      def topology_node_uname_info(client)
+        node_uname_info = query_safely('node_uname_info', 'node_uname_info', fallback: []) do |query|
+          client.query(query)
+        end
+
+        map_instance_labels(node_uname_info, %w(machine sysname release))
+      end
+
       def topology_all_service_memory(client)
         {
           rss: topology_service_memory_rss(client),
@@ -102,31 +112,31 @@ module Gitlab
 
       def topology_service_memory_rss(client)
         query_safely(
-          'gitlab_usage_ping:node_service_process_resident_memory_bytes:avg', 'service_rss', fallback: []
+          'gitlab_usage_ping:node_service_process_resident_memory_bytes:avg', 'service_rss', fallback: {}
         ) { |query| aggregate_by_labels(client, one_week_average(query)) }
       end
 
       def topology_service_memory_uss(client)
         query_safely(
-          'gitlab_usage_ping:node_service_process_unique_memory_bytes:avg', 'service_uss', fallback: []
+          'gitlab_usage_ping:node_service_process_unique_memory_bytes:avg', 'service_uss', fallback: {}
         ) { |query| aggregate_by_labels(client, one_week_average(query)) }
       end
 
       def topology_service_memory_pss(client)
         query_safely(
-          'gitlab_usage_ping:node_service_process_proportional_memory_bytes:avg', 'service_pss', fallback: []
+          'gitlab_usage_ping:node_service_process_proportional_memory_bytes:avg', 'service_pss', fallback: {}
         ) { |query| aggregate_by_labels(client, one_week_average(query)) }
       end
 
       def topology_all_service_process_count(client)
         query_safely(
-          'gitlab_usage_ping:node_service_process:count', 'service_process_count', fallback: []
+          'gitlab_usage_ping:node_service_process:count', 'service_process_count', fallback: {}
         ) { |query| aggregate_by_labels(client, one_week_average(query)) }
       end
 
       def topology_all_service_server_types(client)
         query_safely(
-          'gitlab_usage_ping:node_service_app_server_workers:sum', 'service_workers', fallback: []
+          'gitlab_usage_ping:node_service_app_server_workers:sum', 'service_workers', fallback: {}
         ) { |query| aggregate_by_labels(client, query) }
       end
 
@@ -212,6 +222,16 @@ module Gitlab
           metric
         end
       end
+
+      # Given query result vector, map instance to a hash of target labels key/value.
+      # @return [Hash] mapping instance to a hash of target labels key/value, or the empty hash if input empty vector
+      def map_instance_labels(query_result_vector, target_labels)
+        query_result_vector.to_h do |result|
+          key = drop_port(result['metric']['instance'])
+          value = result['metric'].slice(*target_labels).symbolize_keys
+          [key, value]
+        end
+      end
     end
   end
 end

spec/lib/gitlab/usage_data/topology_spec.rb

@@ -25,6 +25,7 @@ RSpec.describe Gitlab::UsageData::Topology do
             receive_app_request_volume_query,
             receive_node_memory_query,
             receive_node_cpu_count_query,
+            receive_node_uname_info_query,
             receive_node_service_memory_rss_query,
             receive_node_service_memory_uss_query,
             receive_node_service_memory_pss_query,
@@ -40,6 +41,11 @@ RSpec.describe Gitlab::UsageData::Topology do
               {
                 node_memory_total_bytes: 512,
                 node_cpus: 8,
+                node_uname_info: {
+                  machine: 'x86_64',
+                  sysname: 'Linux',
+                  release: '4.19.76-linuxkit'
+                },
                 node_services: [
                   {
                     name: 'web',
@@ -59,6 +65,11 @@ RSpec.describe Gitlab::UsageData::Topology do
               {
                 node_memory_total_bytes: 1024,
                 node_cpus: 16,
+                node_uname_info: {
+                  machine: 'x86_64',
+                  sysname: 'Linux',
+                  release: '4.15.0-101-generic'
+                },
                 node_services: [
                   {
                     name: 'sidekiq',
@@ -88,6 +99,7 @@ RSpec.describe Gitlab::UsageData::Topology do
             receive_app_request_volume_query(result: []),
             receive_node_memory_query(result: []),
             receive_node_cpu_count_query,
+            receive_node_uname_info_query,
             receive_node_service_memory_rss_query(result: []),
             receive_node_service_memory_uss_query(result: []),
             receive_node_service_memory_pss_query,
@@ -107,6 +119,11 @@ RSpec.describe Gitlab::UsageData::Topology do
             nodes: [
               {
                 node_cpus: 16,
+                node_uname_info: {
+                  machine: 'x86_64',
+                  release: '4.15.0-101-generic',
+                  sysname: 'Linux'
+                },
                 node_services: [
                   {
                     name: 'sidekiq',
@@ -121,6 +138,11 @@ RSpec.describe Gitlab::UsageData::Topology do
               },
               {
                 node_cpus: 8,
+                node_uname_info: {
+                  machine: 'x86_64',
+                  release: '4.19.76-linuxkit',
+                  sysname: 'Linux'
+                },
                 node_services: [
                   {
                     name: 'web',
@@ -150,6 +172,7 @@ RSpec.describe Gitlab::UsageData::Topology do
               { 'app_requests' => 'Gitlab::PrometheusClient::ConnectionError' },
               { 'node_memory' => 'Gitlab::PrometheusClient::ConnectionError' },
               { 'node_cpus' => 'Gitlab::PrometheusClient::ConnectionError' },
+              { 'node_uname_info' => 'Gitlab::PrometheusClient::ConnectionError' },
               { 'service_rss' => 'Gitlab::PrometheusClient::ConnectionError' },
               { 'service_uss' => 'Gitlab::PrometheusClient::ConnectionError' },
               { 'service_pss' => 'Gitlab::PrometheusClient::ConnectionError' },
@@ -228,6 +251,39 @@ RSpec.describe Gitlab::UsageData::Topology do
       ])
   end
 
+  def receive_node_uname_info_query(result: nil)
+    receive(:query)
+      .with('node_uname_info')
+      .and_return(result || [
+        {
+          "metric" => {
+            "__name__" => "node_uname_info",
+            "domainname" => "(none)",
+            "instance" => "instance1:9100",
+            "job" => "node_exporter",
+            "machine" => "x86_64",
+            "nodename" => "instance1",
+            "release" => "4.19.76-linuxkit",
+            "sysname" => "Linux"
+          },
+          "value" => [1592463033.359, "1"]
+        },
+        {
+          "metric" => {
+            "__name__" => "node_uname_info",
+            "domainname" => "(none)",
+            "instance" => "instance2:9100",
+            "job" => "node_exporter",
+            "machine" => "x86_64",
+            "nodename" => "instance2",
+            "release" => "4.15.0-101-generic",
+            "sysname" => "Linux"
+          },
+          "value" => [1592463033.359, "1"]
+        }
+      ])
+  end
+
   def receive_node_service_memory_rss_query(result: nil)
     receive(:query)
       .with(/process_resident_memory_bytes/, an_instance_of(Hash))

spec/support/helpers/usage_data_helpers.rb

@@ -228,6 +228,7 @@ module UsageDataHelpers
   def allow_prometheus_queries
     allow_next_instance_of(Gitlab::PrometheusClient) do |client|
       allow(client).to receive(:aggregate).and_return({})
+      allow(client).to receive(:query).and_return({})
     end
   end