Merge branch 'qa/adds-cluster-agent-e2e' into 'master'

QA - Adds cluster agent e2e testing

See merge request gitlab-org/gitlab!49053

qa/qa.rb

@@ -101,6 +101,11 @@ module QA
       autoload :ProjectCluster, 'qa/resource/kubernetes_cluster/project_cluster'
     end
 
+    module Clusters
+      autoload :Agent, 'qa/resource/clusters/agent.rb'
+      autoload :AgentToken, 'qa/resource/clusters/agent_token.rb'
+    end
+
     module Events
       autoload :Base, 'qa/resource/events/base'
       autoload :Project, 'qa/resource/events/project'

qa/qa/fixtures/kubernetes_agent/agentk-config.yaml.erb

+gitops:
+  manifest_projects:
+  - id: <%= project.full_path %>
\ No newline at end of file

qa/qa/fixtures/kubernetes_agent/agentk-manifest.yaml.erb

+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: gitlab-agent
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: gitlab-agent
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: gitlab-agent
+  template:
+    metadata:
+      labels:
+        app: gitlab-agent
+    spec:
+      serviceAccountName: gitlab-agent
+      containers:
+        - name: agent
+          image: "registry.gitlab.com/gitlab-org/cluster-integration/gitlab-agent/agentk:<%= Runtime::Env.gitlab_agentk_version %>"
+          args:
+            - --token-file=/config/token
+            - --kas-address
+            - "<%= kas_wss_address %>" # Use this for GitLab chart deployments
+            # - "<%= kas_grpc_address %>" # Use this for GDK
+          volumeMounts:
+            - name: token-volume
+              mountPath: /config
+      volumes:
+        - name: token-volume
+          secret:
+            secretName: gitlab-agent-token
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxSurge: 0
+      maxUnavailable: 1
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: gitlab-agent-write
+rules:
+  - resources:
+      - "*"
+    apiGroups:
+      - "*"
+    verbs:
+      - create
+      - update
+      - delete
+      - patch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: gitlab-agent-write-binding
+roleRef:
+  name: gitlab-agent-write
+  kind: ClusterRole
+  apiGroup: rbac.authorization.k8s.io
+subjects:
+  - name: gitlab-agent
+    kind: ServiceAccount
+    namespace: default
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: gitlab-agent-read
+rules:
+  - resources:
+      - "*"
+    apiGroups:
+      - "*"
+    verbs:
+      - get
+      - list
+      - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: gitlab-agent-read-binding
+roleRef:
+  name: gitlab-agent-read
+  kind: ClusterRole
+  apiGroup: rbac.authorization.k8s.io
+subjects:
+  - name: gitlab-agent
+    kind: ServiceAccount
+    namespace: default

qa/qa/fixtures/kubernetes_agent/galatic-empire-manifest.yaml

+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: "galatic-empire"
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: "imperial-starfleet"
+  namespace: "galatic-empire"
+data:
+  deathStars: "1"
+  starDestroyers: "5"
+  tieFighters: "25"

qa/qa/resource/clusters/agent.rb

+# frozen_string_literal: true
+
+module QA
+  module Resource
+    module Clusters
+      class Agent < QA::Resource::Base
+        attribute :id
+        attribute :name
+        attribute :project do
+          QA::Resource::Project.fabricate_via_api! do |project|
+            project.name = 'project-with-cluster-agent'
+          end
+        end
+
+        def initialize
+          @name = "my-agent"
+        end
+
+        def fabricate!
+          puts 'TODO: FABRICATE VIA UI'
+        end
+        # TODO
+        #
+        # The UI for this model is not yet implemented. So far it can only be
+        # created through the GraphQL API
+        # def fabricate
+        #
+        # end
+
+        def api_get_path
+          "gid://gitlab/Clusters::Agent/#{id}"
+        end
+
+        def api_post_path
+          "/graphql"
+        end
+
+        def api_post_body
+          <<~GQL
+          mutation createAgent {
+            createClusterAgent(input: { projectPath: "#{project.full_path}", name: "#{@name}" }) {
+              clusterAgent {
+                id
+                name
+              }
+              errors
+            }
+          }
+          GQL
+        end
+      end
+    end
+  end
+end

qa/qa/resource/clusters/agent_token.rb

+# frozen_string_literal: true
+
+module QA
+  module Resource
+    module Clusters
+      class AgentToken < QA::Resource::Base
+        attribute :id
+        attribute :secret
+        attribute :agent do
+          QA::Resource::Clusters::Agent.fabricate_via_api!
+        end
+
+        def fabricate!
+          puts 'TODO: FABRICATE VIA UI'
+        end
+        # TODO
+        #
+        # The UI for this model is not yet implemented. So far it can only be
+        # created through the GraphQL API
+        # def fabricate
+        #
+        # end
+
+        def api_get_path
+          "gid://gitlab/Clusters::AgentToken/#{id}"
+        end
+
+        def api_post_path
+          "/graphql"
+        end
+
+        def api_post_body
+          <<~GQL
+          mutation createToken {
+            clusterAgentTokenCreate(input: { clusterAgentId: "gid://gitlab/Clusters::Agent/#{agent.id}" }) {
+              secret # This is the value you need to use on the next step
+              token {
+                createdAt
+                id
+              }
+              errors
+            }
+          }
+          GQL
+        end
+      end
+    end
+  end
+end

qa/qa/runtime/env.rb

@@ -407,6 +407,10 @@ module QA
         QA::Runtime::Scenario.attributes.include?(:geo_secondary_address)
       end
 
+      def gitlab_agentk_version
+        ENV.fetch('GITLAB_AGENTK_VERSION', 'v13.7.0')
+      end
+
       private
 
       def remote_grid_credentials

qa/qa/service/kubernetes_cluster.rb

@@ -43,6 +43,14 @@ module QA
         cluster_name
       end
 
+      def create_secret(secret, secret_name)
+        shell("kubectl create secret generic #{secret_name} --from-literal=token='#{secret}'")
+      end
+
+      def apply_manifest(manifest)
+        shell('kubectl apply -f -', stdin_data: manifest)
+      end
+
       private
 
       def fetch_api_url

qa/qa/specs/features/ee/api/7_configure/kubernetes/kubernetes_agent_spec.rb

+# frozen_string_literal: true
+require 'erb'
+
+module QA
+  RSpec.describe 'Configure' do
+    include Service::Shellout
+
+    describe 'Kubernetes Agent', :orchestrated, :kubernetes, :requires_admin, quarantine: { issue: 'https://gitlab.com/gitlab-org/gitlab/-/issues/294177', type: :waiting_on } do
+      let!(:cluster) { Service::KubernetesCluster.new(provider_class: Service::ClusterProvider::K3s).create! }
+
+      let(:agent_token) do
+        Resource::Clusters::AgentToken.fabricate_via_api!
+      end
+
+      let(:project) do
+        agent_token.agent.project
+      end
+
+      before do
+        install_agentk(cluster, agent_token)
+
+        creates_agent_config(project)
+      end
+
+      after do
+        cluster.remove!
+
+        project.group.remove_via_api!
+      end
+
+      it 'deploys a K8s manifest file', testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/issues/1106' do
+        deploy_manifest(project)
+
+        expect(manifest_deployed?).to be_truthy
+      end
+
+      private
+
+      def manifest_deployed?
+        wait_until_shell_command_matches('kubectl get namespace --no-headers --ignore-not-found galatic-empire', /galatic-empire   Active/, sleep_interval: 5)
+      end
+
+      def install_agentk(cluster, agent_token)
+        cluster.create_secret(agent_token.secret, 'gitlab-agent-token')
+
+        uri = URI.parse(Runtime::Scenario.gitlab_address)
+
+        kas_grpc_address = "grpc://#{uri.host}:8150"
+        kas_wss_address = "wss://kas.#{uri.host}:#{uri.port}"
+        agent_manifest_template = read_agent_fixture('agentk-manifest.yaml.erb')
+        agent_manifest_yaml = ERB.new(agent_manifest_template).result(binding)
+
+        cluster.apply_manifest(agent_manifest_yaml)
+      end
+
+      def read_agent_fixture(file_name)
+        file_path = Pathname
+          .new(__dir__)
+          .join("../../../../../../fixtures/kubernetes_agent/#{file_name}")
+
+        File.read(file_path)
+      end
+
+      def creates_agent_config(project)
+        Resource::Repository::Commit.fabricate_via_api! do |commit|
+          agent_config_template = read_agent_fixture("agentk-config.yaml.erb")
+          agent_config = ERB.new(agent_config_template).result(binding)
+
+          commit.project = project
+          commit.commit_message = 'Creates agent config'
+          commit.add_files(
+            [
+              {
+                file_path: '.gitlab/agents/my-agent/config.yaml',
+                content: agent_config
+              }
+            ]
+          )
+        end
+      end
+
+      def deploy_manifest(project)
+        Resource::Repository::Commit.fabricate_via_api! do |commit|
+          galatic_empire_manifest = read_agent_fixture("galatic-empire-manifest.yaml")
+
+          commit.project = project
+          commit.commit_message = 'Deploys the Galatic Empire!'
+          commit.add_files(
+            [
+              {
+                file_path: 'manifest.yaml',
+                content: galatic_empire_manifest
+              }
+            ]
+          )
+        end
+      end
+    end
+  end
+end