Generate a Kubeconfig CI variable for configured cluster agents

If a project has associated cluster agents, a KUBECONFIG CI variable
will be provided to all CI jobs for the project. Each agent receieves
a corresponding Kubernetes context, which allows a job to connect to
the agent's cluster (via KAS) without further configuration.

The variable is only added if the job is not creating a deployment
to a cluster, because this could overwrite an existing variable.

https://gitlab.com/gitlab-org/gitlab/-/merge_requests/67089

app/models/ci/build.rb

@@ -554,6 +554,7 @@ module Ci
           .concat(persisted_variables)
           .concat(dependency_proxy_variables)
           .concat(job_jwt_variables)
+          .concat(kubernetes_variables)
           .concat(scoped_variables)
           .concat(job_variables)
           .concat(persisted_environment_variables)
@@ -1172,6 +1173,10 @@ module Ci
       end
     end
 
+    def kubernetes_variables
+      [] # Overridden in EE
+    end
+
     def conditionally_allow_failure!(exit_code)
       return unless exit_code
 

config/feature_flags/development/agent_kubeconfig_ci_variable.yml

+---
+name: agent_kubeconfig_ci_variable
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/67089
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/337164
+milestone: '14.2'
+type: development
+group: group::configure
+default_enabled: false

ee/app/models/ee/ci/build.rb

@@ -225,6 +225,25 @@ module EE
         end
       end
 
+      override :kubernetes_variables
+      def kubernetes_variables
+        ::Gitlab::Ci::Variables::Collection.new.tap do |collection|
+          break collection unless ::Feature.enabled?(:agent_kubeconfig_ci_variable, project, default_enabled: :yaml)
+
+          # A cluster deployemnt may also define a KUBECONFIG variable, so to keep existing
+          # configurations working we shouldn't overwrite it here.
+          # This check will be removed when Cluster and Agent configurations are
+          # merged in https://gitlab.com/gitlab-org/gitlab/-/issues/335089
+          break collection if deployment&.deployment_cluster
+
+          template = ::Ci::GenerateKubeconfigService.new(self).execute
+
+          if template.valid?
+            collection.append(key: 'KUBECONFIG', value: template.to_yaml, public: false, file: true)
+          end
+        end
+      end
+
       def parse_security_artifact_blob(security_report, blob)
         report_clone = security_report.clone_as_blank
         parse_raw_security_artifact_blob(report_clone, blob)

ee/app/models/ee/ci/pipeline.rb

@@ -181,6 +181,12 @@ module EE
         ondemand_dast_scan? && parameter_source?
       end
 
+      def authorized_cluster_agents
+        strong_memoize(:authorized_cluster_agents) do
+          ::Clusters::DeployableAgentsFinder.new(project).execute
+        end
+      end
+
       private
 
       def has_security_reports?

ee/app/services/ci/generate_kubeconfig_service.rb

+# frozen_string_literal: true
+
+module Ci
+  class GenerateKubeconfigService
+    def initialize(build)
+      @build = build
+      @project = build.project
+      @template = Gitlab::Kubernetes::Kubeconfig::Template.new
+    end
+
+    def execute
+      template.add_cluster(
+        name: cluster_name,
+        url: Gitlab::Kas.tunnel_url
+      )
+
+      agents.each do |agent|
+        user = user_name(agent)
+
+        template.add_user(
+          name: user,
+          token: agent_token(agent)
+        )
+
+        template.add_context(
+          name: context_name(agent),
+          cluster: cluster_name,
+          user: user
+        )
+      end
+
+      template
+    end
+
+    private
+
+    attr_reader :build, :project, :template
+
+    def agents
+      build.pipeline.authorized_cluster_agents
+    end
+
+    def cluster_name
+      'gitlab'
+    end
+
+    def user_name(agent)
+      ['agent', agent.id].join(delimiter)
+    end
+
+    def context_name(agent)
+      [project.full_path, agent.name].join(delimiter)
+    end
+
+    def agent_token(agent)
+      ['ci', agent.id, build.token].join(delimiter)
+    end
+
+    def delimiter
+      ':'
+    end
+  end
+end

ee/spec/models/ci/build_spec.rb

@@ -325,6 +325,39 @@ RSpec.describe Ci::Build do
         expect(requirement_variable).to be_nil
       end
     end
+
+    describe 'kubernetes variables' do
+      let(:service) { double(execute: template) }
+      let(:template) { double(to_yaml: 'example-kubeconfig', valid?: template_valid) }
+      let(:template_valid) { true }
+
+      before do
+        allow(::Ci::GenerateKubeconfigService).to receive(:new).with(job).and_return(service)
+      end
+
+      it { is_expected.to include(key: 'KUBECONFIG', value: 'example-kubeconfig', public: false, file: true) }
+
+      context 'feature flag is disabled' do
+        before do
+          stub_feature_flags(agent_kubeconfig_ci_variable: false)
+        end
+
+        it { is_expected.not_to include(key: 'KUBECONFIG', value: 'example-kubeconfig', public: false, file: true) }
+      end
+
+      context 'job is deploying to a cluster' do
+        let(:deployment) { create(:deployment, deployment_cluster: create(:deployment_cluster)) }
+        let(:job) { create(:ci_build, pipeline: pipeline, deployment: deployment) }
+
+        it { is_expected.not_to include(key: 'KUBECONFIG', value: 'example-kubeconfig', public: false, file: true) }
+      end
+
+      context 'generated config is invalid' do
+        let(:template_valid) { false }
+
+        it { is_expected.not_to include(key: 'KUBECONFIG', value: 'example-kubeconfig', public: false, file: true) }
+      end
+    end
   end
 
   describe '#has_security_reports?' do

ee/spec/models/ci/pipeline_spec.rb

@@ -649,4 +649,18 @@ RSpec.describe Ci::Pipeline do
       it { is_expected.to eq(true) }
     end
   end
+
+  describe '#authorized_cluster_agents' do
+    let(:finder) { double(execute: agents) }
+    let(:agents) { double }
+
+    it 'retrieves agent records from the finder and caches the result' do
+      expect(Clusters::DeployableAgentsFinder).to receive(:new).once
+        .with(pipeline.project)
+        .and_return(finder)
+
+      expect(pipeline.authorized_cluster_agents).to eq(agents)
+      expect(pipeline.authorized_cluster_agents).to eq(agents) # cached
+    end
+  end
 end

ee/spec/services/ci/generate_kubeconfig_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Ci::GenerateKubeconfigService do
+  describe '#execute' do
+    let(:project) { create(:project) }
+    let(:build) { create(:ci_build, project: project) }
+    let(:agent1) { create(:cluster_agent, project: project) }
+    let(:agent2) { create(:cluster_agent, project: project) }
+
+    let(:template) { instance_double(Gitlab::Kubernetes::Kubeconfig::Template) }
+
+    subject { described_class.new(build).execute }
+
+    before do
+      expect(Gitlab::Kubernetes::Kubeconfig::Template).to receive(:new).and_return(template)
+      expect(build.pipeline).to receive(:authorized_cluster_agents).and_return([agent1, agent2])
+    end
+
+    it 'adds a cluster, and a user and context for each available agent' do
+      expect(template).to receive(:add_cluster).with(
+        name: 'gitlab',
+        url: Gitlab::Kas.tunnel_url
+      ).once
+
+      expect(template).to receive(:add_user).with(
+        name: "agent:#{agent1.id}",
+        token: "ci:#{agent1.id}:#{build.token}"
+      )
+      expect(template).to receive(:add_user).with(
+        name: "agent:#{agent2.id}",
+        token: "ci:#{agent2.id}:#{build.token}"
+      )
+
+      expect(template).to receive(:add_context).with(
+        name: "#{project.full_path}:#{agent1.name}",
+        cluster: 'gitlab',
+        user: "agent:#{agent1.id}"
+      )
+      expect(template).to receive(:add_context).with(
+        name: "#{project.full_path}:#{agent2.name}",
+        cluster: 'gitlab',
+        user: "agent:#{agent2.id}"
+      )
+
+      expect(subject).to eq(template)
+    end
+  end
+end

lib/gitlab/kas.rb

@@ -5,6 +5,7 @@ module Gitlab
     INTERNAL_API_REQUEST_HEADER = 'Gitlab-Kas-Api-Request'
     VERSION_FILE = 'GITLAB_KAS_VERSION'
     JWT_ISSUER = 'gitlab-kas'
+    K8S_PROXY_PATH = 'k8s-proxy'
 
     include JwtAuthenticatable
 
@@ -39,6 +40,10 @@ module Gitlab
         Gitlab.config.gitlab_kas.external_url
       end
 
+      def tunnel_url
+        URI.join(external_url, K8S_PROXY_PATH).to_s
+      end
+
       # Return GitLab KAS internal_url
       #
       # @return [String] internal_url

spec/lib/gitlab/kas_spec.rb

@@ -65,6 +65,12 @@ RSpec.describe Gitlab::Kas do
     end
   end
 
+  describe '.tunnel_url' do
+    it 'returns gitlab_kas external_url with proxy path appended' do
+      expect(described_class.tunnel_url).to eq(Gitlab.config.gitlab_kas.external_url + '/k8s-proxy')
+    end
+  end
+
   describe '.internal_url' do
     it 'returns gitlab_kas internal_url config' do
       expect(described_class.internal_url).to eq(Gitlab.config.gitlab_kas.internal_url)