Add Kubernetes CPU/Mem usage totals

changelogs/unreleased/add-cpu-mem-totals.yml

+---
+title: Add Total CPU/Memory consumption metrics for Kubernetes
+merge_request: 17731
+author:
+type: added

config/prometheus/additional_metrics.yml

@@ -139,21 +139,39 @@
 - group: System metrics (Kubernetes)
   priority: 5
   metrics:
-  - title: "Memory Usage"
+  - title: "Memory Usage (Total)"
+    y_label: "Total Memory Used"
+    required_metrics:
+      - container_memory_usage_bytes
+    weight: 4
+    queries:
+    - query_range: 'avg(sum(container_memory_usage_bytes{container_name!="POD",pod_name=~"^%{ci_environment_slug}-(.*)",namespace="%{kube_namespace}"}) by (job)) without (job)  /1024/1024/1024'
+      label: Total
+      unit: GB
+  - title: "Core Usage (Total)"
+    y_label: "Total Cores"
+    required_metrics:
+     - container_cpu_usage_seconds_total
+    weight: 3
+    queries:
+    - query_range: 'avg(sum(rate(container_cpu_usage_seconds_total{container_name!="POD",pod_name=~"^%{ci_environment_slug}-(.*)",namespace="%{kube_namespace}"}[15m])) by (job)) without (job)'
+      label: Total
+      unit: "cores"
+  - title: "Memory Usage (Pod average)"
     y_label: "Memory Used per Pod"
     required_metrics:
       - container_memory_usage_bytes
-    weight: 1
+    weight: 2
     queries:
-    - query_range: 'avg(sum(container_memory_usage_bytes{container_name!="POD",pod_name=~"^%{ci_environment_slug}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="%{kube_namespace}"}) by (job)) without (job) / count(avg(container_memory_usage_bytes{container_name!="POD",pod_name=~"^%{ci_environment_slug}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="%{kube_namespace}"}) without (job)) /1024/1024'
+    - query_range: 'avg(sum(container_memory_usage_bytes{container_name!="POD",pod_name=~"^%{ci_environment_slug}-(.*)",namespace="%{kube_namespace}"}) by (job)) without (job) / count(avg(container_memory_usage_bytes{container_name!="POD",pod_name=~"^%{ci_environment_slug}-(.*)",namespace="%{kube_namespace}"}) without (job)) /1024/1024'
       label: Pod average
       unit: MB
-  - title: "CPU Usage"
+  - title: "Core Usage (Pod average)"
     y_label: "Cores per Pod"
     required_metrics:
      - container_cpu_usage_seconds_total
     weight: 1
     queries:
-    - query_range: 'avg(sum(rate(container_cpu_usage_seconds_total{container_name!="POD",pod_name=~"^%{ci_environment_slug}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="%{kube_namespace}"}[15m])) by (job)) without (job) / count(sum(rate(container_cpu_usage_seconds_total{container_name!="POD",pod_name=~"^%{ci_environment_slug}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="%{kube_namespace}"}[15m])) by (pod_name))'
+    - query_range: 'avg(sum(rate(container_cpu_usage_seconds_total{container_name!="POD",pod_name=~"^%{ci_environment_slug}-(.*)",namespace="%{kube_namespace}"}[15m])) by (job)) without (job) / count(sum(rate(container_cpu_usage_seconds_total{container_name!="POD",pod_name=~"^%{ci_environment_slug}-(.*)",namespace="%{kube_namespace}"}[15m])) by (pod_name))'
       label: Pod average
       unit: "cores"
\ No newline at end of file