Merge branch '195978-reduce-environment-terminals-cached-data' into 'master'

Resolve "Reduce Pod and Deployment cached data"

See merge request gitlab-org/gitlab!23179

app/models/clusters/platforms/kubernetes.rb

@@ -92,7 +92,10 @@ module Clusters
       def calculate_reactive_cache_for(environment)
         return unless enabled?
 
-        { pods: read_pods(environment.deployment_namespace) }
+        pods = read_pods(environment.deployment_namespace)
+
+        # extract_relevant_pod_data avoids uploading all the pod info into ReactiveCaching
+        { pods: extract_relevant_pod_data(pods) }
       end
 
       def terminals(environment, data)
@@ -203,6 +206,21 @@ module Clusters
       def nullify_blank_namespace
         self.namespace = nil if namespace.blank?
       end
+
+      def extract_relevant_pod_data(pods)
+        pods.map do |pod|
+          {
+            'metadata' => pod.fetch('metadata', {})
+                             .slice('name', 'generateName', 'labels', 'annotations', 'creationTimestamp'),
+            'status' => pod.fetch('status', {}).slice('phase'),
+            'spec' => {
+              'containers' => pod.fetch('spec', {})
+                                 .fetch('containers', [])
+                                 .map { |c| c.slice('name') }
+            }
+          }
+        end
+      end
     end
   end
 end

ee/app/models/ee/clusters/platforms/kubernetes.rb

@@ -5,15 +5,23 @@ module EE
     module Platforms
       module Kubernetes
         extend ActiveSupport::Concern
+        extend ::Gitlab::Utils::Override
         include ::Gitlab::Utils::StrongMemoize
 
         CACHE_KEY_GET_POD_LOG = 'get_pod_log'
 
         LOGS_LIMIT = 500.freeze
 
+        override :calculate_reactive_cache_for
         def calculate_reactive_cache_for(environment)
           result = super
-          result[:deployments] = read_deployments(environment.deployment_namespace) if result
+
+          if result
+            deployments = read_deployments(environment.deployment_namespace)
+
+            # extract_relevant_deployment_data avoids uploading all the deployment info into ReactiveCaching
+            result[:deployments] = extract_relevant_deployment_data(deployments)
+          end
 
           result
         end
@@ -172,6 +180,16 @@ module EE
         rescue Kubeclient::ResourceNotFoundError
           []
         end
+
+        def extract_relevant_deployment_data(deployments)
+          deployments.map do |deployment|
+            {
+              'metadata' => deployment.fetch('metadata', {}).slice('name', 'generation', 'labels', 'annotations'),
+              'spec' => deployment.fetch('spec', {}).slice('replicas'),
+              'status' => deployment.fetch('status', {}).slice('observedGeneration')
+            }
+          end
+        end
       end
     end
   end

ee/lib/gitlab/kubernetes/deployment.rb

@@ -46,8 +46,8 @@ module Gitlab
 
       def created_instances
         filtered_pods_by_track.map do |pod|
-          metadata = pod.fetch('metadata', {})
-          pod_name = metadata['name'] || metadata['generate_name']
+          pod_metadata = pod.fetch('metadata', {})
+          pod_name = pod_metadata['name'] || pod_metadata['generateName']
           pod_status = pod.dig('status', 'phase')
 
           deployment_instance(pod_name: pod_name, pod_status: pod_status)

ee/spec/models/ee/clusters/platforms/kubernetes_spec.rb

@@ -345,22 +345,22 @@ describe Clusters::Platforms::Kubernetes do
   describe '#calculate_reactive_cache_for' do
     let(:cluster) { create(:cluster, :project, platform_kubernetes: service) }
     let(:service) { create(:cluster_platform_kubernetes, :configured) }
-    let(:namespace) { 'app' }
+    let(:namespace) { 'project-namespace' }
     let(:environment) { instance_double(Environment, deployment_namespace: namespace) }
+    let(:expected_pod_cached_data) do
+      kube_pod.tap { |kp| kp['metadata'].delete('namespace') }
+    end
 
     subject { service.calculate_reactive_cache_for(environment) }
 
-    before do
-      allow(service).to receive(:read_pods).and_return([])
-    end
-
     context 'when kubernetes responds with valid deployments' do
       before do
+        stub_kubeclient_pods(namespace)
         stub_kubeclient_deployments(namespace)
       end
 
       shared_examples 'successful deployment request' do
-        it { is_expected.to include(deployments: [kube_deployment]) }
+        it { is_expected.to include(pods: [expected_pod_cached_data], deployments: [kube_deployment]) }
       end
 
       context 'on a project level cluster' do
@@ -384,6 +384,7 @@ describe Clusters::Platforms::Kubernetes do
 
     context 'when kubernetes responds with 500s' do
       before do
+        stub_kubeclient_pods(namespace)
         stub_kubeclient_deployments(namespace, status: 500)
       end
 
@@ -392,6 +393,7 @@ describe Clusters::Platforms::Kubernetes do
 
     context 'when kubernetes responds with 404s' do
       before do
+        stub_kubeclient_pods(namespace)
         stub_kubeclient_deployments(namespace, status: 404)
       end
 

spec/models/clusters/platforms/kubernetes_spec.rb

@@ -361,8 +361,10 @@ describe Clusters::Platforms::Kubernetes do
 
   describe '#calculate_reactive_cache_for' do
     let(:service) { create(:cluster_platform_kubernetes, :configured) }
-    let(:pod) { kube_pod }
-    let(:namespace) { pod["metadata"]["namespace"] }
+    let(:expected_pod_cached_data) do
+      kube_pod.tap { |kp| kp['metadata'].delete('namespace') }
+    end
+    let(:namespace) { "project-namespace" }
     let(:environment) { instance_double(Environment, deployment_namespace: namespace) }
 
     subject { service.calculate_reactive_cache_for(environment) }
@@ -381,7 +383,7 @@ describe Clusters::Platforms::Kubernetes do
         stub_kubeclient_deployments(namespace)
       end
 
-      it { is_expected.to include(pods: [pod]) }
+      it { is_expected.to include(pods: [expected_pod_cached_data]) }
     end
 
     context 'when kubernetes responds with 500s' do

spec/support/helpers/kubernetes_helpers.rb

@@ -494,7 +494,7 @@ module KubernetesHelpers
       "metadata" => {
         "name" => name,
         "namespace" => namespace,
-        "generate_name" => "generated-name-with-suffix",
+        "generateName" => "generated-name-with-suffix",
         "creationTimestamp" => "2016-11-25T19:55:19Z",
         "annotations" => {
           "app.gitlab.com/env" => environment_slug,
@@ -520,7 +520,7 @@ module KubernetesHelpers
       "metadata" => {
         "name" => name,
         "namespace" => namespace,
-        "generate_name" => "generated-name-with-suffix",
+        "generateName" => "generated-name-with-suffix",
         "creationTimestamp" => "2016-11-25T19:55:19Z",
         "labels" => {
           "serving.knative.dev/service" => name
@@ -551,10 +551,7 @@ module KubernetesHelpers
       },
       "spec" => { "replicas" => 3 },
       "status" => {
-        "observedGeneration" => 4,
-        "replicas" => 3,
-        "updatedReplicas" => 3,
-        "availableReplicas" => 3
+        "observedGeneration" => 4
       }
     }
   end