Merge branch 'am-add-hash-metrics-to-service-ping' into 'master'

Add metric to service ping if has definition

See merge request gitlab-org/gitlab!70061

config/metrics/counts_28d/20210216183631_two-factor-via-u2f-device.yml

----
-data_category: optional
-key_path: usage_activity_by_stage_monthly.manage.user_auth_by_provider.two-factor-via-u2f-device
-description: Number of unique user logins using two factor via a U2F device
-product_section: dev
-product_stage: manage
-product_group: group::access
-product_category: authentication_and_authorization
-value_type: number
-status: active
-time_frame: 28d
-data_source: database
-distribution:
-- ce
-tier:
-- free
-- premium
-- ultimate
-milestone: "<13.9"

config/metrics/counts_28d/20210216183633_two-factor-via-webauthn-device.yml

----
-data_category: optional
-key_path: usage_activity_by_stage_monthly.manage.user_auth_by_provider.two-factor-via-webauthn-device
-description: Number of unique user logins using two factor via a WebAuthn device
-product_section: dev
-product_stage: manage
-product_group: group::access
-product_category: authentication_and_authorization
-value_type: number
-status: active
-time_frame: 28d
-data_source: database
-distribution:
-- ce
-tier:
-- free
-- premium
-- ultimate
-milestone: "<13.9"

config/metrics/counts_28d/20210216183634_standard.yml

----
-data_category: optional
-key_path: usage_activity_by_stage_monthly.manage.user_auth_by_provider.standard
-description: Number of unique user logins using password authentication
-product_section: dev
-product_stage: manage
-product_group: group::access
-product_category: authentication_and_authorization
-value_type: number
-status: active
-time_frame: 28d
-data_source: database
-distribution:
-- ce
-tier:
-- free
-- premium
-- ultimate
-performance_indicator_type: []
-milestone: "<13.9"

config/metrics/counts_28d/20210216183636_google_oauth2.yml

----
-data_category: optional
-key_path: usage_activity_by_stage_monthly.manage.user_auth_by_provider.google_oauth2
-description: Number of unique user logins using Google OAuth authentication
-product_section: dev
-product_stage: manage
-product_group: group::access
-product_category: authentication_and_authorization
-value_type: number
-status: active
-time_frame: 28d
-data_source: database
-distribution:
-- ce
-tier:
-- free
-- premium
-- ultimate
-performance_indicator_type: []
-milestone: "<13.9"

config/metrics/counts_28d/20210216183629_two-factor.yml → config/metrics/counts_28d/20210910132229_user_auth_by_provider.yml

 ---
-data_category: optional
-key_path: usage_activity_by_stage_monthly.manage.user_auth_by_provider.two-factor
-description: Number of unique user logins using two factor authentication
+key_path: usage_activity_by_stage_monthly.manage.user_auth_by_provider
+name: count_distinct_users_using_two_factor_authentication
+description: Number of unique user logins using two factor authentication for available providers
 product_section: dev
 product_stage: manage
 product_group: group::access
 product_category: authentication_and_authorization
-value_type: number
+value_type: object
 status: active
+milestone: "14.3"
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/70061
 time_frame: 28d
 data_source: database
+data_category: optional
+performance_indicator_type: []
 distribution:
 - ce
+- ee
 tier:
 - free
 - premium
 - ultimate
-milestone: "<13.9"
+value_json_schema: "config/metrics/objects_schemas/user_auth_by_provider.json"

config/metrics/counts_all/20210216183404_two-factor-via-u2f-device.yml

----
-data_category: optional
-key_path: usage_activity_by_stage.manage.user_auth_by_provider.two-factor-via-u2f-device
-description: Number of unique user logins using two factor via a U2F device
-product_section: dev
-product_stage: manage
-product_group: group::access
-product_category: authentication_and_authorization
-value_type: number
-status: active
-time_frame: all
-data_source: database
-distribution:
-- ce
-tier:
-- free
-- premium
-- ultimate
-milestone: "<13.9"

config/metrics/counts_all/20210216183406_two-factor-via-webauthn-device.yml

----
-data_category: optional
-key_path: usage_activity_by_stage.manage.user_auth_by_provider.two-factor-via-webauthn-device
-description: Number of unique user logins using two factor via a WebAuthn device
-product_section: dev
-product_stage: manage
-product_group: group::access
-product_category: authentication_and_authorization
-value_type: number
-status: active
-time_frame: all
-data_source: database
-distribution:
-- ce
-tier:
-- free
-- premium
-- ultimate
-milestone: "<13.9"

config/metrics/counts_all/20210216183408_standard.yml

----
-data_category: optional
-key_path: usage_activity_by_stage.manage.user_auth_by_provider.standard
-description: Number of unique user logins using password authentication
-product_section: dev
-product_stage: manage
-product_group: group::access
-product_category: authentication_and_authorization
-value_type: number
-status: active
-time_frame: all
-data_source: database
-distribution:
-- ce
-tier:
-- free
-- premium
-- ultimate
-milestone: "<13.9"

config/metrics/counts_all/20210216183410_google_oauth2.yml

----
-data_category: optional
-key_path: usage_activity_by_stage.manage.user_auth_by_provider.google_oauth2
-description: Number of unique user logins using Google OAuth authentication
-product_section: dev
-product_stage: manage
-product_group: group::access
-product_category: authentication_and_authorization
-value_type: number
-status: active
-time_frame: all
-data_source: database
-distribution:
-- ce
-tier:
-- free
-- premium
-- ultimate
-milestone: "<13.9"

config/metrics/counts_all/20210216183402_two-factor.yml → config/metrics/counts_all/20210910132001_user_auth_by_provider.yml

 ---
-data_category: optional
-key_path: usage_activity_by_stage.manage.user_auth_by_provider.two-factor
-description: Number of unique user logins using two factor authentication
+key_path: usage_activity_by_stage.manage.user_auth_by_provider
+name: count_distinct_users_using_two_factor_authentication
+description: Number of unique user logins using two factor authentication for available providers
 product_section: dev
 product_stage: manage
 product_group: group::access
 product_category: authentication_and_authorization
-value_type: number
+value_type: object
 status: active
+milestone: "14.3"
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/70061
 time_frame: all
 data_source: database
+data_category: optional
+performance_indicator_type: []
 distribution:
 - ce
+- ee
 tier:
 - free
 - premium
 - ultimate
-milestone: "<13.9"
+value_json_schema: "config/metrics/objects_schemas/user_auth_by_provider.json"

config/metrics/objects_schemas/user_auth_by_provider.json

+{
+  "type": "object",
+  "description": "Distinct users by provider",
+  "properties": {
+    "two-factor": {"type": "number", "description": " Number of unique user logins using two factor authentication" },
+    "two-factor-via-u2f-device": {"type": "number", "description": " Number of unique user logins using two factor authentication via U2F" },
+    "two-factor-via-webauthn-device": {"type": "number", "description": " Number of unique user logins using two factor authentication via two-factor-via-webauthn-device" },
+    "standard": {"type": "number", "description": " Number of unique user logins using password authentication" },
+    "google_oauth2": {"type": "number", "description": " Number of unique user logins using Google OAuth authentication" },
+    "twitter": {"type": "number", "description": " Number of unique user logins using Twitter authentication" },
+    "github": {"type": "number", "description": " Number of unique user logins using Github authentication" },
+    "bitbucket": {"type": "number", "description": " Number of unique user logins using Bitbucket authentication" },
+    "group_saml": {"type": "number", "description": " Number of unique user logins using group SAML authentication" },
+    "salesforce": {"type": "number", "description": " Number of unique user logins using group Salesforce authentication" }
+  }
+}

ee/app/services/ee/service_ping/build_payload_service.rb

@@ -20,13 +20,13 @@ module EE
       end
 
       def filtered_usage_data(payload = raw_payload, parents = [])
-        return if payload.nil?
+        return unless payload.is_a?(Hash)
 
         payload.keep_if do |label, node|
-          if leaf?(node)
-            permitted_categories.include?(metric_category(label, parents))
+          if has_metric_definition?(label, parents)
+            permitted_metric?(label, parents)
           else
-            filtered_usage_data(node, parents.dup << label)
+            filtered_usage_data(node, parents.dup << label) if node.is_a?(Hash)
           end
         end
       end
@@ -35,6 +35,15 @@ module EE
         @permitted_categories ||= ::ServicePing::PermitDataCategoriesService.new.execute
       end
 
+      def permitted_metric?(key, parent_keys)
+        permitted_categories.include?(metric_category(key, parent_keys))
+      end
+
+      def has_metric_definition?(key, parent_keys)
+        key_path = parent_keys.dup.append(key).join('.')
+        metric_definitions[key_path].present?
+      end
+
       def metric_category(key, parent_keys)
         key_path = parent_keys.dup.append(key).join('.')
         metric_definitions[key_path]
@@ -45,10 +54,6 @@ module EE
       def metric_definitions
         @metric_definitions ||= ::Gitlab::Usage::MetricDefinition.definitions
       end
-
-      def leaf?(node)
-        !node.is_a?(Hash)
-      end
     end
   end
 end

spec/support/shared_contexts/services/service_ping/stubbed_service_ping_metrics_definitions_shared_context.rb

@@ -6,21 +6,25 @@ RSpec.shared_context 'stubbed service ping metrics definitions' do
   let(:metrics_definitions) { standard_metrics + subscription_metrics + operational_metrics + optional_metrics }
   let(:standard_metrics) do
     [
-      metric_attributes('uuid', "standard")
+      metric_attributes('uuid', 'standard'),
+      metric_attributes('recorded_at', 'standard'),
+      metric_attributes('settings.collected_data_categories', 'standard', 'object')
     ]
   end
 
   let(:operational_metrics) do
     [
-      metric_attributes('counts.merge_requests', "operational"),
+      metric_attributes('counts.merge_requests', 'operational'),
       metric_attributes('counts.todos', "operational")
     ]
   end
 
   let(:optional_metrics) do
     [
-      metric_attributes('counts.boards', "optional"),
-      metric_attributes('gitaly.filesystems', '').except('data_category')
+      metric_attributes('counts.boards', 'optional', 'number'),
+      metric_attributes('gitaly.filesystems', '').except('data_category'),
+      metric_attributes('usage_activity_by_stage.monitor.projects_with_enabled_alert_integrations_histogram', 'optional', 'object'),
+      metric_attributes('topology', 'optional', 'object')
     ]
   end
 
@@ -34,10 +38,11 @@ RSpec.shared_context 'stubbed service ping metrics definitions' do
     )
   end
 
-  def metric_attributes(key_path, category)
+  def metric_attributes(key_path, category, value_type = 'string')
     {
       'key_path' => key_path,
-      'data_category' => category
+      'data_category' => category,
+      'value_type' => value_type
     }
   end
 end