Merge branch 'pl-qa-k8s' into 'master'

Refactor kubernetes setup in QA

See merge request gitlab-org/gitlab-ce!29799

qa/qa.rb

@@ -359,6 +359,13 @@ module QA
     autoload :KubernetesCluster, 'qa/service/kubernetes_cluster'
     autoload :Omnibus, 'qa/service/omnibus'
     autoload :Runner, 'qa/service/runner'
+
+    module ClusterProvider
+      autoload :Base, 'qa/service/cluster_provider/base'
+      autoload :Gcloud, 'qa/service/cluster_provider/gcloud'
+      autoload :Minikube, 'qa/service/cluster_provider/minikube'
+      autoload :K3d, 'qa/service/cluster_provider/k3d'
+    end
   end
 
   ##

qa/qa/service/cluster_provider/base.rb

+# frozen_string_literal: true
+
+module QA
+  module Service
+    module ClusterProvider
+      class Base
+        include Service::Shellout
+
+        attr_reader :rbac
+
+        def initialize(rbac:)
+          @rbac = rbac
+        end
+
+        def cluster_name
+          @cluster_name ||= "qa-cluster-#{Time.now.utc.strftime("%Y%m%d%H%M%S")}-#{SecureRandom.hex(4)}"
+        end
+
+        def set_credentials(admin_user)
+          raise NotImplementedError
+        end
+
+        def validate_dependencies
+          raise NotImplementedError
+        end
+
+        def setup
+          raise NotImplementedError
+        end
+
+        def teardown
+          raise NotImplementedError
+        end
+
+        def filter_credentials(credentials)
+          credentials
+        end
+      end
+    end
+  end
+end

qa/qa/service/cluster_provider/gcloud.rb

+# frozen_string_literal: true
+
+module QA
+  module Service
+    module ClusterProvider
+      class Gcloud < Base
+        def validate_dependencies
+          find_executable('gcloud') || raise("You must first install `gcloud` executable to run these tests.")
+        end
+
+        def set_credentials(admin_user)
+          master_auth = JSON.parse(`gcloud container clusters describe #{cluster_name} --region #{Runtime::Env.gcloud_region} --format 'json(masterAuth.username, masterAuth.password)'`)
+
+          shell <<~CMD.tr("\n", ' ')
+            kubectl config set-credentials #{admin_user}
+              --username #{master_auth['masterAuth']['username']}
+              --password #{master_auth['masterAuth']['password']}
+          CMD
+        end
+
+        def setup
+          login_if_not_already_logged_in
+          create_cluster
+        end
+
+        def teardown
+          delete_cluster
+        end
+
+        private
+
+        def login_if_not_already_logged_in
+          if Runtime::Env.has_gcloud_credentials?
+            attempt_login_with_env_vars
+          else
+            account = `gcloud auth list --filter=status:ACTIVE --format="value(account)"`
+            if account.empty?
+              raise "Failed to login to gcloud. No credentials provided in environment and no credentials found locally."
+            else
+              puts "gcloud account found. Using: #{account} for creating K8s cluster."
+            end
+          end
+        end
+
+        def attempt_login_with_env_vars
+          puts "No gcloud account. Attempting to login from env vars GCLOUD_ACCOUNT_EMAIL and GCLOUD_ACCOUNT_KEY."
+          gcloud_account_key = Tempfile.new('gcloud-account-key')
+          gcloud_account_key.write(Runtime::Env.gcloud_account_key)
+          gcloud_account_key.close
+          gcloud_account_email = Runtime::Env.gcloud_account_email
+          shell("gcloud auth activate-service-account #{gcloud_account_email} --key-file #{gcloud_account_key.path}")
+        ensure
+          gcloud_account_key && gcloud_account_key.unlink
+        end
+
+        def auth_options
+          "--enable-legacy-authorization" unless rbac
+        end
+
+        def create_cluster
+          shell <<~CMD.tr("\n", ' ')
+            gcloud container clusters
+            create #{cluster_name}
+            #{auth_options}
+            --enable-basic-auth
+            --region #{Runtime::Env.gcloud_region}
+            --disk-size 10GB
+            --num-nodes #{Runtime::Env.gcloud_num_nodes}
+            && gcloud container clusters
+            get-credentials
+            --region #{Runtime::Env.gcloud_region}
+            #{cluster_name}
+          CMD
+        end
+
+        def delete_cluster
+          shell <<~CMD.tr("\n", ' ')
+            gcloud container clusters delete
+              --region #{Runtime::Env.gcloud_region}
+              #{cluster_name}
+              --quiet --async
+          CMD
+        end
+      end
+    end
+  end
+end

qa/qa/service/cluster_provider/k3d.rb

+# frozen_string_literal: true
+
+module QA
+  module Service
+    module ClusterProvider
+      class K3d < Base
+        def validate_dependencies
+          find_executable('k3d') || raise("You must first install `k3d` executable to run these tests.")
+        end
+
+        def set_credentials(admin_user)
+        end
+
+        def setup
+          shell "k3d create --workers 1 --name #{cluster_name} --wait 0"
+
+          @old_kubeconfig = ENV['KUBECONFIG']
+          ENV['KUBECONFIG'] = fetch_kubeconfig
+          raise "Could not fetch kubeconfig" unless ENV['KUBECONFIG']
+
+          install_local_storage
+        end
+
+        def teardown
+          ENV['KUBECONFIG'] = @old_kubeconfig
+          shell "k3d delete --name #{cluster_name}"
+        end
+
+        # Fetch "real" certificate
+        # See https://github.com/rancher/k3s/issues/27
+        def filter_credentials(credentials)
+          kubeconfig = YAML.load_file(ENV['KUBECONFIG'])
+          ca_certificate = kubeconfig.dig('clusters', 0, 'cluster', 'certificate-authority-data')
+
+          credentials.merge('data' => credentials['data'].merge('ca.crt' => ca_certificate))
+        end
+
+        private
+
+        def retry_until(max_attempts: 10, wait: 1)
+          max_attempts.times do
+            result = yield
+            return result if result
+
+            sleep wait
+          end
+
+          raise "Retried #{max_attempts} times. Aborting"
+        end
+
+        def fetch_kubeconfig
+          retry_until do
+            config = `k3d get-kubeconfig --name #{cluster_name}`.chomp
+            config if config =~ /kubeconfig.yaml/
+          end
+        end
+
+        def install_local_storage
+          shell('kubectl apply -f -', stdin_data: local_storage_config)
+        end
+
+        # See https://github.com/rancher/k3d/issues/67
+        def local_storage_config
+          <<~YAML
+            ---
+            apiVersion: v1
+            kind: ServiceAccount
+            metadata:
+              name: storage-provisioner
+              namespace: kube-system
+            ---
+            apiVersion: rbac.authorization.k8s.io/v1
+            kind: ClusterRoleBinding
+            metadata:
+              name: storage-provisioner
+            roleRef:
+              apiGroup: rbac.authorization.k8s.io
+              kind: ClusterRole
+              name: system:persistent-volume-provisioner
+            subjects:
+              - kind: ServiceAccount
+                name: storage-provisioner
+                namespace: kube-system
+            ---
+            apiVersion: v1
+            kind: Pod
+            metadata:
+              name: storage-provisioner
+              namespace: kube-system
+            spec:
+              serviceAccountName: storage-provisioner
+              tolerations:
+              - effect: NoExecute
+                key: node.kubernetes.io/not-ready
+                operator: Exists
+                tolerationSeconds: 300
+              - effect: NoExecute
+                key: node.kubernetes.io/unreachable
+                operator: Exists
+                tolerationSeconds: 300
+              hostNetwork: true
+              containers:
+              - name: storage-provisioner
+                image: gcr.io/k8s-minikube/storage-provisioner:v1.8.1
+                command: ["/storage-provisioner"]
+                imagePullPolicy: IfNotPresent
+                volumeMounts:
+                - mountPath: /tmp
+                  name: tmp
+              volumes:
+              - name: tmp
+                hostPath:
+                  path: /tmp
+                  type: Directory
+            ---
+            kind: StorageClass
+            apiVersion: storage.k8s.io/v1
+            metadata:
+              name: standard
+              namespace: kube-system
+              annotations:
+                storageclass.kubernetes.io/is-default-class: "true"
+              labels:
+                addonmanager.kubernetes.io/mode: EnsureExists
+            provisioner: k8s.io/minikube-hostpath
+          YAML
+        end
+      end
+    end
+  end
+end

qa/qa/service/cluster_provider/minikube.rb

+# frozen_string_literal: true
+
+module QA
+  module Service
+    module ClusterProvider
+      class Minikube < Base
+        def validate_dependencies
+          find_executable('minikube') || raise("You must first install `minikube` executable to run these tests.")
+        end
+
+        def set_credentials(admin_user)
+        end
+
+        def setup
+          shell 'minikube stop'
+          shell "minikube profile #{cluster_name}"
+          shell 'minikube start'
+        end
+
+        def teardown
+          shell 'minikube delete'
+        end
+      end
+    end
+  end
+end

qa/qa/service/kubernetes_cluster.rb

@@ -9,90 +9,63 @@ module QA
     class KubernetesCluster
       include Service::Shellout
 
-      attr_reader :api_url, :ca_certificate, :token, :rbac
+      attr_reader :api_url, :ca_certificate, :token, :rbac, :provider
 
-      def initialize(rbac: true)
+      def initialize(rbac: true, provider_class: QA::Service::ClusterProvider::Gcloud)
         @rbac = rbac
-      end
-
-      def cluster_name
-        @cluster_name ||= "qa-cluster-#{SecureRandom.hex(4)}-#{Time.now.utc.strftime("%Y%m%d%H%M%S")}"
+        @provider = provider_class.new(rbac: rbac)
       end
 
       def create!
         validate_dependencies
-        login_if_not_already_logged_in
-
-        shell <<~CMD.tr("\n", ' ')
-          gcloud container clusters
-          create #{cluster_name}
-          #{auth_options}
-          --enable-basic-auth
-          --region #{Runtime::Env.gcloud_region}
-          --disk-size 10GB
-          --num-nodes #{Runtime::Env.gcloud_num_nodes}
-          && gcloud container clusters
-          get-credentials
-          --region #{Runtime::Env.gcloud_region}
-          #{cluster_name}
-        CMD
-
-        @api_url = `kubectl config view --minify -o jsonpath='{.clusters[].cluster.server}'`
-
-        @admin_user = "#{cluster_name}-admin"
-        master_auth = JSON.parse(`gcloud container clusters describe #{cluster_name} --region #{Runtime::Env.gcloud_region} --format 'json(masterAuth.username, masterAuth.password)'`)
-        shell <<~CMD.tr("\n", ' ')
-          kubectl config set-credentials #{@admin_user}
-          --username #{master_auth['masterAuth']['username']}
-          --password #{master_auth['masterAuth']['password']}
-        CMD
-
-        if rbac
-          create_service_account
-
-          secrets = JSON.parse(`kubectl get secrets -o json`)
-          gitlab_account = secrets['items'].find do |item|
-            item['metadata']['annotations']['kubernetes.io/service-account.name'] == 'gitlab-account'
-          end
-
-          @ca_certificate = Base64.decode64(gitlab_account['data']['ca.crt'])
-          @token = Base64.decode64(gitlab_account['data']['token'])
-        else
-          @ca_certificate = Base64.decode64(`kubectl get secrets -o jsonpath="{.items[0].data['ca\\.crt']}"`)
-          @token = Base64.decode64(`kubectl get secrets -o jsonpath='{.items[0].data.token}'`)
-        end
+
+        @provider.validate_dependencies
+        @provider.setup
+
+        @api_url = fetch_api_url
+
+        credentials = @provider.filter_credentials(fetch_credentials)
+        @ca_certificate = Base64.decode64(credentials.dig('data', 'ca.crt'))
+        @token = Base64.decode64(credentials.dig('data', 'token'))
 
         self
       end
 
       def remove!
-        shell <<~CMD.tr("\n", ' ')
-          gcloud container clusters delete
-          --region #{Runtime::Env.gcloud_region}
-          #{cluster_name}
-          --quiet --async
-        CMD
+        @provider.teardown
+      end
+
+      def cluster_name
+        @provider.cluster_name
       end
 
       private
 
-      def create_service_account
-        shell('kubectl create -f -', stdin_data: service_account)
-        shell("kubectl --user #{@admin_user} create -f -", stdin_data: service_account_role_binding)
+      def fetch_api_url
+        `kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}'`
+      end
+
+      def fetch_credentials
+        return global_credentials unless rbac
+
+        @provider.set_credentials(admin_user)
+        create_service_account(admin_user)
+        account_credentials
       end
 
-      def service_account
-        <<~YAML
+      def admin_user
+        @admin_user ||= "#{@provider.cluster_name}-admin"
+      end
+
+      def create_service_account(user)
+        service_account = <<~YAML
+          ---
           apiVersion: v1
           kind: ServiceAccount
           metadata:
             name: gitlab-account
             namespace: default
-        YAML
-      end
-
-      def service_account_role_binding
-        <<~YAML
+          ---
           kind: ClusterRoleBinding
           apiVersion: rbac.authorization.k8s.io/v1
           metadata:
@@ -106,39 +79,24 @@ module QA
             name: cluster-admin
             apiGroup: rbac.authorization.k8s.io
         YAML
-      end
 
-      def auth_options
-        "--enable-legacy-authorization" unless rbac
+        shell('kubectl apply -f -', stdin_data: service_account)
       end
 
-      def validate_dependencies
-        find_executable('gcloud') || raise("You must first install `gcloud` executable to run these tests.")
-        find_executable('kubectl') || raise("You must first install `kubectl` executable to run these tests.")
-      end
+      def account_credentials
+        secrets = JSON.parse(`kubectl get secrets -o json`)
 
-      def login_if_not_already_logged_in
-        if Runtime::Env.has_gcloud_credentials?
-          attempt_login_with_env_vars
-        else
-          account = `gcloud auth list --filter=status:ACTIVE --format="value(account)"`
-          if account.empty?
-            raise "Failed to login to gcloud. No credentials provided in environment and no credentials found locally."
-          else
-            puts "gcloud account found. Using: #{account} for creating K8s cluster."
-          end
+        secrets['items'].find do |item|
+          item['metadata']['annotations']['kubernetes.io/service-account.name'] == 'gitlab-account'
         end
       end
 
-      def attempt_login_with_env_vars
-        puts "No gcloud account. Attempting to login from env vars GCLOUD_ACCOUNT_EMAIL and GCLOUD_ACCOUNT_KEY."
-        gcloud_account_key = Tempfile.new('gcloud-account-key')
-        gcloud_account_key.write(Runtime::Env.gcloud_account_key)
-        gcloud_account_key.close
-        gcloud_account_email = Runtime::Env.gcloud_account_email
-        shell("gcloud auth activate-service-account #{gcloud_account_email} --key-file #{gcloud_account_key.path}")
-      ensure
-        gcloud_account_key && gcloud_account_key.unlink
+      def global_credentials
+        JSON.parse(`kubectl get secrets -o jsonpath='{.items[0]}'`)
+      end
+
+      def validate_dependencies
+        find_executable('kubectl') || raise("You must first install `kubectl` executable to run these tests.")
       end
     end
   end