Add CPU/Memory totals to Kubernetes system metrics

config/prometheus/additional_metrics.yml

@@ -139,29 +139,48 @@
 - group: System metrics (Kubernetes)
   priority: 5
   metrics:
-  - title: "Memory Usage"
+  - title: "Memory Usage (Total)"
+    y_label: "Total Memory Used"
+    required_metrics:
+      - container_memory_usage_bytes
+    weight: 4
+    queries:
+    - query_range: 'avg(sum(container_memory_usage_bytes{container_name!="POD",pod_name=~"^%{ci_environment_slug}-(.*)",namespace="%{kube_namespace}"}) by (job)) without (job)  /1024/1024/1024'
+      label: Total
+      unit: GB
+  - title: "Core Usage (Total)"
+    y_label: "Total Cores"
+    required_metrics:
+     - container_cpu_usage_seconds_total
+    weight: 3
+    queries:
+    - query_range: 'avg(sum(rate(container_cpu_usage_seconds_total{container_name!="POD",pod_name=~"^%{ci_environment_slug}-(.*)",namespace="%{kube_namespace}"}[15m])) by (job)) without (job)'
+      label: Total
+      unit: "cores"
+  - title: "Memory Usage (Pod Average)"
     y_label: "Memory Used per Pod"
     required_metrics:
       - container_memory_usage_bytes
-    weight: 1
+    weight: 2
     queries:
     - query_range: 'avg(sum(container_memory_usage_bytes{container_name!="POD",pod_name=~"^%{ci_environment_slug}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="%{kube_namespace}"}) by (job)) without (job) / count(avg(container_memory_usage_bytes{container_name!="POD",pod_name=~"^%{ci_environment_slug}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="%{kube_namespace}"}) without (job)) /1024/1024'
-      label: Average
+      label: Pod average
       unit: MB
     - query_range: 'avg(sum(container_memory_usage_bytes{container_name!="POD",pod_name=~"^%{ci_environment_slug}-canary-(.*)",namespace="%{kube_namespace}"}) by (job)) without (job) / count(avg(container_memory_usage_bytes{container_name!="POD",pod_name=~"^%{ci_environment_slug}-canary-(.*)",namespace="%{kube_namespace}"}) without (job)) /1024/1024'
-      label: Average
+      label: Pod average
       unit: MB
       track: canary
-  - title: "CPU Usage"
+  - title: "Core Usage (Pod Average)"
     y_label: "Cores per Pod"
     required_metrics:
      - container_cpu_usage_seconds_total
     weight: 1
     queries:
     - query_range: 'avg(sum(rate(container_cpu_usage_seconds_total{container_name!="POD",pod_name=~"^%{ci_environment_slug}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="%{kube_namespace}"}[15m])) by (job)) without (job) / count(sum(rate(container_cpu_usage_seconds_total{container_name!="POD",pod_name=~"^%{ci_environment_slug}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="%{kube_namespace}"}[15m])) by (pod_name))'
-      label: Average
+      label: Pod average
       unit: "cores"
     - query_range: 'avg(sum(rate(container_cpu_usage_seconds_total{container_name!="POD",pod_name=~"^%{ci_environment_slug}-canary-(.*)",namespace="%{kube_namespace}"}[15m])) by (job)) without (job) / count(sum(rate(container_cpu_usage_seconds_total{container_name!="POD",pod_name=~"^%{ci_environment_slug}-canary-(.*)",namespace="%{kube_namespace}"}[15m])) by (pod_name))'
-      label: Average
+      label: Pod average
       unit: "cores"
       track: canary
+    

ee/changelogs/unreleased/ee-add-cpu-mem-totals.yml

+---
+title: Add total CPU/Memory metrics, adds weighting for proper sorting
+merge_request: 5260
+author: 
+type: added