Merge branch '46686-create-eks-cluster-from-gitlab' into 'master'

GitLab EKS Cluster backend services

See merge request gitlab-org/gitlab!16758

app/models/clusters/providers/aws.rb

@@ -3,6 +3,7 @@
 module Clusters
   module Providers
     class Aws < ApplicationRecord
+      include Gitlab::Utils::StrongMemoize
       include Clusters::Concerns::ProviderStatus
 
       self.table_name = 'cluster_providers_aws'
@@ -42,6 +43,18 @@ module Clusters
           session_token: nil
         )
       end
+
+      def api_client
+        strong_memoize(:api_client) do
+          ::Aws::CloudFormation::Client.new(credentials: credentials, region: region)
+        end
+      end
+
+      def credentials
+        strong_memoize(:credentials) do
+          ::Aws::Credentials.new(access_key_id, secret_access_key, session_token)
+        end
+      end
     end
   end
 end

app/services/clusters/aws/fetch_credentials_service.rb

+# frozen_string_literal: true
+
+module Clusters
+  module Aws
+    class FetchCredentialsService
+      attr_reader :provider
+
+      MissingRoleError = Class.new(StandardError)
+
+      def initialize(provider)
+        @provider = provider
+      end
+
+      def execute
+        raise MissingRoleError.new('AWS provisioning role not configured') unless provision_role.present?
+
+        ::Aws::AssumeRoleCredentials.new(
+          client: client,
+          role_arn: provision_role.role_arn,
+          role_session_name: session_name,
+          external_id: provision_role.role_external_id
+        ).credentials
+      end
+
+      private
+
+      def provision_role
+        provider.created_by_user.aws_role
+      end
+
+      def client
+        ::Aws::STS::Client.new(credentials: gitlab_credentials, region: provider.region)
+      end
+
+      def gitlab_credentials
+        ::Aws::Credentials.new(access_key_id, secret_access_key)
+      end
+
+      ##
+      # This setting is not yet configurable or documented as these
+      # services are not currently used. This will be addressed in
+      # https://gitlab.com/gitlab-org/gitlab/merge_requests/18307
+      def access_key_id
+        Gitlab.config.kubernetes.provisioners.aws.access_key_id
+      end
+
+      ##
+      # This setting is not yet configurable or documented as these
+      # services are not currently used. This will be addressed in
+      # https://gitlab.com/gitlab-org/gitlab/merge_requests/18307
+      def secret_access_key
+        Gitlab.config.kubernetes.provisioners.aws.secret_access_key
+      end
+
+      def session_name
+        "gitlab-eks-cluster-#{provider.cluster_id}-user-#{provider.created_by_user_id}"
+      end
+    end
+  end
+end

app/services/clusters/aws/finalize_creation_service.rb

+# frozen_string_literal: true
+
+module Clusters
+  module Aws
+    class FinalizeCreationService
+      include Gitlab::Utils::StrongMemoize
+
+      attr_reader :provider
+
+      delegate :cluster, to: :provider
+
+      def execute(provider)
+        @provider = provider
+
+        configure_provider
+        create_gitlab_service_account!
+        configure_platform_kubernetes
+        configure_node_authentication!
+
+        cluster.save!
+      rescue ::Aws::CloudFormation::Errors::ServiceError => e
+        log_service_error(e.class.name, provider.id, e.message)
+        provider.make_errored!(s_('ClusterIntegration|Failed to fetch CloudFormation stack: %{message}') % { message: e.message })
+      rescue Kubeclient::HttpError => e
+        log_service_error(e.class.name, provider.id, e.message)
+        provider.make_errored!(s_('ClusterIntegration|Failed to run Kubeclient: %{message}') % { message: e.message })
+      rescue ActiveRecord::RecordInvalid => e
+        log_service_error(e.class.name, provider.id, e.message)
+        provider.make_errored!(s_('ClusterIntegration|Failed to configure EKS provider: %{message}') % { message: e.message })
+      end
+
+      private
+
+      def create_gitlab_service_account!
+        Clusters::Kubernetes::CreateOrUpdateServiceAccountService.gitlab_creator(
+          kube_client,
+          rbac: true
+        ).execute
+      end
+
+      def configure_provider
+        provider.status_event = :make_created
+      end
+
+      def configure_platform_kubernetes
+        cluster.build_platform_kubernetes(
+          api_url: cluster_endpoint,
+          ca_cert: cluster_certificate,
+          token: request_kubernetes_token)
+      end
+
+      def request_kubernetes_token
+        Clusters::Kubernetes::FetchKubernetesTokenService.new(
+          kube_client,
+          Clusters::Kubernetes::GITLAB_ADMIN_TOKEN_NAME,
+          Clusters::Kubernetes::GITLAB_SERVICE_ACCOUNT_NAMESPACE
+        ).execute
+      end
+
+      def kube_client
+        @kube_client ||= build_kube_client!(
+          cluster_endpoint,
+          cluster_certificate
+        )
+      end
+
+      def build_kube_client!(api_url, ca_pem)
+        raise "Incomplete settings" unless api_url
+
+        Gitlab::Kubernetes::KubeClient.new(
+          api_url,
+          auth_options: kubeclient_auth_options,
+          ssl_options: kubeclient_ssl_options(ca_pem),
+          http_proxy_uri: ENV['http_proxy']
+        )
+      end
+
+      def kubeclient_auth_options
+        { bearer_token: Kubeclient::AmazonEksCredentials.token(provider.credentials, cluster.name) }
+      end
+
+      def kubeclient_ssl_options(ca_pem)
+        opts = { verify_ssl: OpenSSL::SSL::VERIFY_PEER }
+
+        if ca_pem.present?
+          opts[:cert_store] = OpenSSL::X509::Store.new
+          opts[:cert_store].add_cert(OpenSSL::X509::Certificate.new(ca_pem))
+        end
+
+        opts
+      end
+
+      def cluster_stack
+        @cluster_stack ||= provider.api_client.describe_stacks(stack_name: provider.cluster.name).stacks.first
+      end
+
+      def stack_output_value(key)
+        cluster_stack.outputs.detect { |output| output.output_key == key }.output_value
+      end
+
+      def node_instance_role_arn
+        stack_output_value('NodeInstanceRole')
+      end
+
+      def cluster_endpoint
+        strong_memoize(:cluster_endpoint) do
+          stack_output_value('ClusterEndpoint')
+        end
+      end
+
+      def cluster_certificate
+        strong_memoize(:cluster_certificate) do
+          Base64.decode64(stack_output_value('ClusterCertificate'))
+        end
+      end
+
+      def configure_node_authentication!
+        kube_client.create_config_map(node_authentication_config)
+      end
+
+      def node_authentication_config
+        Gitlab::Kubernetes::ConfigMaps::AwsNodeAuth.new(node_instance_role_arn).generate
+      end
+
+      def logger
+        @logger ||= Gitlab::Kubernetes::Logger.build
+      end
+
+      def log_service_error(exception, provider_id, message)
+        logger.error(
+          exception: exception.class.name,
+          service: self.class.name,
+          provider_id: provider_id,
+          message: message
+        )
+      end
+    end
+  end
+end

app/services/clusters/aws/provision_service.rb

+# frozen_string_literal: true
+
+module Clusters
+  module Aws
+    class ProvisionService
+      attr_reader :provider
+
+      def execute(provider)
+        @provider = provider
+
+        configure_provider_credentials
+        provision_cluster
+
+        if provider.make_creating
+          WaitForClusterCreationWorker.perform_in(
+            Clusters::Aws::VerifyProvisionStatusService::INITIAL_INTERVAL,
+            provider.cluster_id
+          )
+        else
+          provider.make_errored!("Failed to update provider record; #{provider.errors.full_messages}")
+        end
+      rescue Clusters::Aws::FetchCredentialsService::MissingRoleError
+        provider.make_errored!('Amazon role is not configured')
+      rescue ::Aws::Errors::MissingCredentialsError, Settingslogic::MissingSetting
+        provider.make_errored!('Amazon credentials are not configured')
+      rescue ::Aws::STS::Errors::ServiceError => e
+        provider.make_errored!("Amazon authentication failed; #{e.message}")
+      rescue ::Aws::CloudFormation::Errors::ServiceError => e
+        provider.make_errored!("Amazon CloudFormation request failed; #{e.message}")
+      end
+
+      private
+
+      def credentials
+        @credentials ||= Clusters::Aws::FetchCredentialsService.new(provider).execute
+      end
+
+      def configure_provider_credentials
+        provider.update!(
+          access_key_id: credentials.access_key_id,
+          secret_access_key: credentials.secret_access_key,
+          session_token: credentials.session_token
+        )
+      end
+
+      def provision_cluster
+        provider.api_client.create_stack(
+          stack_name: provider.cluster.name,
+          template_body: stack_template,
+          parameters: parameters,
+          capabilities: ["CAPABILITY_IAM"]
+        )
+      end
+
+      def parameters
+        [
+          parameter('ClusterName', provider.cluster.name),
+          parameter('ClusterRole', provider.role_arn),
+          parameter('ClusterControlPlaneSecurityGroup', provider.security_group_id),
+          parameter('VpcId', provider.vpc_id),
+          parameter('Subnets', provider.subnet_ids.join(',')),
+          parameter('NodeAutoScalingGroupDesiredCapacity', provider.num_nodes.to_s),
+          parameter('NodeInstanceType', provider.instance_type),
+          parameter('KeyName', provider.key_name)
+        ]
+      end
+
+      def parameter(key, value)
+        { parameter_key: key, parameter_value: value }
+      end
+
+      def stack_template
+        File.read(Rails.root.join('vendor', 'aws', 'cloudformation', 'eks_cluster.yaml'))
+      end
+    end
+  end
+end

app/services/clusters/aws/verify_provision_status_service.rb

+# frozen_string_literal: true
+
+module Clusters
+  module Aws
+    class VerifyProvisionStatusService
+      attr_reader :provider
+
+      INITIAL_INTERVAL = 5.minutes
+      POLL_INTERVAL = 1.minute
+      TIMEOUT = 30.minutes
+
+      def execute(provider)
+        @provider = provider
+
+        case cluster_stack.stack_status
+        when 'CREATE_IN_PROGRESS'
+          continue_creation
+        when 'CREATE_COMPLETE'
+          finalize_creation
+        else
+          provider.make_errored!("Unexpected status; #{cluster_stack.stack_status}")
+        end
+      rescue ::Aws::CloudFormation::Errors::ServiceError => e
+        provider.make_errored!("Amazon CloudFormation request failed; #{e.message}")
+      end
+
+      private
+
+      def cluster_stack
+        @cluster_stack ||= provider.api_client.describe_stacks(stack_name: provider.cluster.name).stacks.first
+      end
+
+      def continue_creation
+        if timeout_threshold.future?
+          WaitForClusterCreationWorker.perform_in(POLL_INTERVAL, provider.cluster_id)
+        else
+          provider.make_errored!(_('Kubernetes cluster creation time exceeds timeout; %{timeout}') % { timeout: TIMEOUT })
+        end
+      end
+
+      def timeout_threshold
+        cluster_stack.creation_time + TIMEOUT
+      end
+
+      def finalize_creation
+        Clusters::Aws::FinalizeCreationService.new.execute(provider)
+      end
+    end
+  end
+end

app/workers/cluster_provision_worker.rb

@@ -9,7 +9,11 @@ class ClusterProvisionWorker
   def perform(cluster_id)
     Clusters::Cluster.find_by_id(cluster_id).try do |cluster|
       cluster.provider.try do |provider|
-        Clusters::Gcp::ProvisionService.new.execute(provider) if cluster.gcp?
+        if cluster.gcp?
+          Clusters::Gcp::ProvisionService.new.execute(provider)
+        elsif cluster.aws?
+          Clusters::Aws::ProvisionService.new.execute(provider)
+        end
       end
     end
   end

app/workers/wait_for_cluster_creation_worker.rb

@@ -9,7 +9,11 @@ class WaitForClusterCreationWorker
   def perform(cluster_id)
     Clusters::Cluster.find_by_id(cluster_id).try do |cluster|
       cluster.provider.try do |provider|
-        Clusters::Gcp::VerifyProvisionStatusService.new.execute(provider) if cluster.gcp?
+        if cluster.gcp?
+          Clusters::Gcp::VerifyProvisionStatusService.new.execute(provider)
+        elsif cluster.aws?
+          Clusters::Aws::VerifyProvisionStatusService.new.execute(provider)
+        end
       end
     end
   end

doc/development/kubernetes.md

@@ -74,6 +74,50 @@ We have some Webmock stubs in
 [`KubernetesHelpers`](https://gitlab.com/gitlab-org/gitlab/blob/master/spec/support/helpers/kubernetes_helpers.rb)
 which can help with mocking out calls to Kubernetes API in your tests.
 
+### Amazon EKS integration
+
+This section outlines the process for allowing a GitLab instance to create EKS clusters.
+
+The following prerequisites are required:
+
+A `Customer` AWS account. This is the account in which the
+EKS cluster will be created. The following resources must be present:
+
+- A provisioning role that has permissions to create the cluster
+  and associated resources. It must list the `GitLab` AWS account
+  as a trusted entity.
+- A VPC, management role, security group, and subnets for use by the cluster.
+
+A `GitLab` AWS account. This is the account which performs
+the provisioning actions. The following resources must be present:
+
+- A service account with permissions to assume the provisioning
+  role in the `Customer` account above.
+- Credentials for this service account configured in GitLab via
+  the `kubernetes` section of `gitlab.yml`
+
+The process for creating a cluster is as follows:
+
+1. Using the :provision_role_external_id, GitLab assumes the role provided
+   by :provision_role_arn and stores a set of temporary credentials on the
+   provider record. By default these credentials are valid for one hour.
+1. A CloudFormation stack is created, based on the
+   [`AWS CloudFormation EKS template`](https://gitlab.com/gitlab-org/gitlab/blob/master/vendor/aws/cloudformation/eks_cluster.yaml).
+   This triggers creation of all resources required for an EKS cluster.
+1. GitLab polls the status of the stack until all resources are ready,
+   which takes somewhere between 10 and 15 minutes in most cases.
+1. When the stack is ready, GitLab stores the cluster details and generates
+   another set of temporary credentials, this time to allow connecting to
+   the cluster via Kubeclient. These credentials are valid for one minute.
+1. GitLab configures the worker nodes so that they are able to authenticate
+   to the cluster, and creates a service account for itself for future operations.
+1. Credentials that are no longer required are removed. This deletes the following
+   attributes:
+
+- `access_key_id`
+- `secret_access_key`
+- `session_token`
+
 ## Security
 
 ### SSRF

lib/gitlab/kubernetes/config_maps/aws_node_auth.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Kubernetes
+    module ConfigMaps
+      class AwsNodeAuth
+        attr_reader :node_role
+
+        def initialize(node_role)
+          @node_role = node_role
+        end
+
+        def generate
+          Kubeclient::Resource.new(
+            metadata: metadata,
+            data: data
+          )
+        end
+
+        private
+
+        def metadata
+          {
+            'name' => 'aws-auth',
+            'namespace' => 'kube-system'
+          }
+        end
+
+        def data
+          { 'mapRoles' => instance_role_config(node_role) }
+        end
+
+        def instance_role_config(role)
+          [{
+            'rolearn' => role,
+            'username' => 'system:node:{{EC2PrivateDNSName}}',
+            'groups' => [
+              'system:bootstrappers',
+              'system:nodes'
+            ]
+          }].to_yaml
+        end
+      end
+    end
+  end
+end

locale/gitlab.pot

@@ -3585,9 +3585,15 @@ msgstr ""
 msgid "ClusterIntegration|Every new Google Cloud Platform (GCP) account receives $300 in credit upon %{sign_up_link}. In partnership with Google, GitLab is able to offer an additional $200 for both new and existing GCP accounts to get started with GitLab's Google Kubernetes Engine Integration."
 msgstr ""
 
+msgid "ClusterIntegration|Failed to configure EKS provider: %{message}"
+msgstr ""
+
 msgid "ClusterIntegration|Failed to configure Google Kubernetes Engine Cluster: %{message}"
 msgstr ""
 
+msgid "ClusterIntegration|Failed to fetch CloudFormation stack: %{message}"
+msgstr ""
+
 msgid "ClusterIntegration|Failed to request to Google Cloud Platform: %{message}"
 msgstr ""
 

spec/lib/gitlab/kubernetes/config_maps/aws_node_auth_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::Kubernetes::ConfigMaps::AwsNodeAuth do
+  describe '#generate' do
+    let(:role) { 'arn:aws:iam::123456789012:role/node-instance-role' }
+
+    let(:name) { 'aws-auth' }
+    let(:namespace) { 'kube-system' }
+    let(:role_config) do
+      [{
+        'rolearn' => role,
+        'username' => 'system:node:{{EC2PrivateDNSName}}',
+        'groups' => [
+          'system:bootstrappers',
+          'system:nodes'
+        ]
+      }]
+    end
+
+    subject { described_class.new(role).generate }
+
+    it 'builds a Kubeclient Resource' do
+      expect(subject).to be_a(Kubeclient::Resource)
+
+      expect(subject.metadata.name).to eq(name)
+      expect(subject.metadata.namespace).to eq(namespace)
+
+      expect(YAML.safe_load(subject.data.mapRoles)).to eq(role_config)
+    end
+  end
+end

spec/models/clusters/providers/aws_spec.rb

@@ -64,13 +64,48 @@ describe Clusters::Providers::Aws do
     before do
       expect(provider.access_key_id).to be_present
       expect(provider.secret_access_key).to be_present
+      expect(provider.session_token).to be_present
     end
 
-    it 'removes access_key_id and secret_access_key' do
+    it 'removes access_key_id, secret_access_key and session_token' do
       subject
 
       expect(provider.access_key_id).to be_nil
       expect(provider.secret_access_key).to be_nil
+      expect(provider.session_token).to be_nil
     end
   end
+
+  describe '#api_client' do
+    let(:provider) { create(:cluster_provider_aws) }
+    let(:credentials) { double }
+    let(:client) { double }
+
+    subject { provider.api_client }
+
+    before do
+      allow(provider).to receive(:credentials).and_return(credentials)
+
+      expect(Aws::CloudFormation::Client).to receive(:new)
+        .with(credentials: credentials, region: provider.region)
+        .and_return(client)
+    end
+
+    it { is_expected.to eq client }
+  end
+
+  describe '#credentials' do
+    let(:provider) { create(:cluster_provider_aws) }
+    let(:credentials) { double }
+
+    subject { provider.credentials }
+
+    before do
+      expect(Aws::Credentials).to receive(:new)
+        .with(provider.access_key_id, provider.secret_access_key, provider.session_token)
+        .and_return(credentials)
+    end
+
+    it { is_expected.to eq credentials }
+  end
 end

spec/services/clusters/aws/fetch_credentials_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Clusters::Aws::FetchCredentialsService do
+  describe '#execute' do
+    let(:provider) { create(:cluster_provider_aws) }
+
+    let(:gitlab_access_key_id) { 'gitlab-access-key-id' }
+    let(:gitlab_secret_access_key) { 'gitlab-secret-access-key' }
+
+    let(:gitlab_credentials) { Aws::Credentials.new(gitlab_access_key_id, gitlab_secret_access_key) }
+    let(:sts_client) { Aws::STS::Client.new(credentials: gitlab_credentials, region: provider.region) }
+    let(:assumed_role) { instance_double(Aws::AssumeRoleCredentials, credentials: assumed_role_credentials) }
+
+    let(:kubernetes_provisioner_settings) do
+      {
+        aws: {
+          access_key_id: gitlab_access_key_id,
+          secret_access_key: gitlab_secret_access_key
+        }
+      }
+    end
+
+    let(:assumed_role_credentials) { double }
+
+    subject { described_class.new(provider).execute }
+
+    context 'provision role is configured' do
+      let(:provision_role) { create(:aws_role, user: provider.created_by_user) }
+
+      before do
+        stub_config(kubernetes: { provisioners: kubernetes_provisioner_settings })
+
+        expect(Aws::Credentials).to receive(:new)
+          .with(gitlab_access_key_id, gitlab_secret_access_key)
+          .and_return(gitlab_credentials)
+
+        expect(Aws::STS::Client).to receive(:new)
+          .with(credentials: gitlab_credentials, region: provider.region)
+          .and_return(sts_client)
+
+        expect(Aws::AssumeRoleCredentials).to receive(:new)
+          .with(
+            client: sts_client,
+            role_arn: provision_role.role_arn,
+            role_session_name: "gitlab-eks-cluster-#{provider.cluster_id}-user-#{provider.created_by_user_id}",
+            external_id: provision_role.role_external_id
+          ).and_return(assumed_role)
+      end
+
+      it { is_expected.to eq assumed_role_credentials }
+    end
+
+    context 'provision role is not configured' do
+      before do
+        expect(provider.created_by_user.aws_role).to be_nil
+      end
+
+      it 'raises an error' do
+        expect { subject }.to raise_error(described_class::MissingRoleError, 'AWS provisioning role not configured')
+      end
+    end
+  end
+end

spec/services/clusters/aws/finalize_creation_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Clusters::Aws::FinalizeCreationService do
+  describe '#execute' do
+    let(:provider) { create(:cluster_provider_aws, :creating) }
+    let(:platform) { provider.cluster.platform_kubernetes }
+
+    let(:create_service_account_service) { double(execute: true) }
+    let(:fetch_token_service) { double(execute: gitlab_token) }
+    let(:kube_client) { double(create_config_map: true) }
+    let(:cluster_stack) { double(outputs: [endpoint_output, cert_output, node_role_output]) }
+    let(:node_auth_config_map) { double }
+
+    let(:endpoint_output) { double(output_key: 'ClusterEndpoint', output_value: api_url) }
+    let(:cert_output) { double(output_key: 'ClusterCertificate', output_value: Base64.encode64(ca_pem)) }
+    let(:node_role_output) { double(output_key: 'NodeInstanceRole', output_value: node_role) }
+
+    let(:api_url) { 'https://kubernetes.example.com' }
+    let(:ca_pem) { File.read(Rails.root.join('spec/fixtures/clusters/sample_cert.pem')) }
+    let(:gitlab_token) { 'gitlab-token' }
+    let(:iam_token) { 'iam-token' }
+    let(:node_role) { 'arn::aws::iam::123456789012:role/node-role' }
+
+    subject { described_class.new.execute(provider) }
+
+    before do
+      allow(Clusters::Kubernetes::CreateOrUpdateServiceAccountService).to receive(:gitlab_creator)
+        .with(kube_client, rbac: true)
+        .and_return(create_service_account_service)
+
+      allow(Clusters::Kubernetes::FetchKubernetesTokenService).to receive(:new)
+        .with(
+          kube_client,
+          Clusters::Kubernetes::GITLAB_ADMIN_TOKEN_NAME,
+          Clusters::Kubernetes::GITLAB_SERVICE_ACCOUNT_NAMESPACE)
+        .and_return(fetch_token_service)
+
+      allow(Gitlab::Kubernetes::KubeClient).to receive(:new)
+        .with(
+          api_url,
+          auth_options: { bearer_token: iam_token },
+          ssl_options: {
+            verify_ssl: OpenSSL::SSL::VERIFY_PEER,
+            cert_store: instance_of(OpenSSL::X509::Store)
+          },
+          http_proxy_uri: nil
+        )
+        .and_return(kube_client)
+
+      allow(provider.api_client).to receive(:describe_stacks)
+        .with(stack_name: provider.cluster.name)
+        .and_return(double(stacks: [cluster_stack]))
+
+      allow(Kubeclient::AmazonEksCredentials).to receive(:token)
+        .with(provider.credentials, provider.cluster.name)
+        .and_return(iam_token)
+
+      allow(Gitlab::Kubernetes::ConfigMaps::AwsNodeAuth).to receive(:new)
+        .with(node_role).and_return(double(generate: node_auth_config_map))
+    end
+
+    it 'configures the provider and platform' do
+      subject
+
+      expect(provider).to be_created
+      expect(platform.api_url).to eq(api_url)
+      expect(platform.ca_pem).to eq(ca_pem)
+      expect(platform.token).to eq(gitlab_token)
+      expect(platform).to be_rbac
+    end
+
+    it 'calls the create_service_account_service' do
+      expect(create_service_account_service).to receive(:execute).once
+
+      subject
+    end
+
+    it 'configures cluster node authentication' do
+      expect(kube_client).to receive(:create_config_map).with(node_auth_config_map).once
+
+      subject
+    end
+
+    describe 'error handling' do
+      shared_examples 'provision error' do |message|
+        it "sets the status to :errored with an appropriate error message" do
+          subject
+
+          expect(provider).to be_errored
+          expect(provider.status_reason).to include(message)
+        end
+      end
+
+      context 'failed to request stack details from AWS' do
+        before do
+          allow(provider.api_client).to receive(:describe_stacks)
+            .and_raise(Aws::CloudFormation::Errors::ServiceError.new(double, "Error message"))
+        end
+
+        include_examples 'provision error', 'Failed to fetch CloudFormation stack'
+      end
+
+      context 'failed to create auth config map' do
+        before do
+          allow(kube_client).to receive(:create_config_map)
+            .and_raise(Kubeclient::HttpError.new(500, 'Error', nil))
+        end
+
+        include_examples 'provision error', 'Failed to run Kubeclient'
+      end
+
+      context 'failed to save records' do
+        before do
+          allow(provider.cluster).to receive(:save!)
+            .and_raise(ActiveRecord::RecordInvalid)
+        end
+
+        include_examples 'provision error', 'Failed to configure EKS provider'
+      end
+    end
+  end
+end

spec/services/clusters/aws/provision_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Clusters::Aws::ProvisionService do
+  describe '#execute' do
+    let(:provider) { create(:cluster_provider_aws) }
+
+    let(:client) { instance_double(Aws::CloudFormation::Client, create_stack: true) }
+    let(:cloudformation_template) { double }
+    let(:credentials) do
+      instance_double(
+        Aws::Credentials,
+        access_key_id: 'key',
+        secret_access_key: 'secret',
+        session_token: 'token'
+      )
+    end
+
+    let(:parameters) do
+      [
+        { parameter_key: 'ClusterName', parameter_value: provider.cluster.name },
+        { parameter_key: 'ClusterRole', parameter_value: provider.role_arn },
+        { parameter_key: 'ClusterControlPlaneSecurityGroup', parameter_value: provider.security_group_id },
+        { parameter_key: 'VpcId', parameter_value: provider.vpc_id },
+        { parameter_key: 'Subnets', parameter_value: provider.subnet_ids.join(',') },
+        { parameter_key: 'NodeAutoScalingGroupDesiredCapacity', parameter_value: provider.num_nodes.to_s },
+        { parameter_key: 'NodeInstanceType', parameter_value: provider.instance_type },
+        { parameter_key: 'KeyName', parameter_value: provider.key_name }
+      ]
+    end
+
+    subject { described_class.new.execute(provider) }
+
+    before do
+      allow(Clusters::Aws::FetchCredentialsService).to receive(:new)
+        .with(provider).and_return(double(execute: credentials))
+
+      allow(provider).to receive(:api_client)
+        .and_return(client)
+
+      allow(File).to receive(:read)
+        .with(Rails.root.join('vendor', 'aws', 'cloudformation', 'eks_cluster.yaml'))
+        .and_return(cloudformation_template)
+    end
+
+    it 'updates the provider status to :creating and configures the provider with credentials' do
+      subject
+
+      expect(provider).to be_creating
+      expect(provider.access_key_id).to eq 'key'
+      expect(provider.secret_access_key).to eq 'secret'
+      expect(provider.session_token).to eq 'token'
+    end
+
+    it 'creates a CloudFormation stack' do
+      expect(client).to receive(:create_stack).with(
+        stack_name: provider.cluster.name,
+        template_body: cloudformation_template,
+        parameters: parameters,
+        capabilities: ["CAPABILITY_IAM"]
+      )
+
+      subject
+    end
+
+    it 'schedules a worker to monitor creation status' do
+      expect(WaitForClusterCreationWorker).to receive(:perform_in)
+        .with(Clusters::Aws::VerifyProvisionStatusService::INITIAL_INTERVAL, provider.cluster_id)
+
+      subject
+    end
+
+    describe 'error handling' do
+      shared_examples 'provision error' do |message|
+        it "sets the status to :errored with an appropriate error message" do
+          subject
+
+          expect(provider).to be_errored
+          expect(provider.status_reason).to include(message)
+        end
+      end
+
+      context 'invalid state transition' do
+        before do
+          allow(provider).to receive(:make_creating).and_return(false)
+        end
+
+        include_examples 'provision error', 'Failed to update provider record'
+      end
+
+      context 'AWS role is not configured' do
+        before do
+          allow(Clusters::Aws::FetchCredentialsService).to receive(:new)
+            .and_raise(Clusters::Aws::FetchCredentialsService::MissingRoleError)
+        end
+
+        include_examples 'provision error', 'Amazon role is not configured'
+      end
+
+      context 'AWS credentials are not configured' do
+        before do
+          allow(Clusters::Aws::FetchCredentialsService).to receive(:new)
+            .and_raise(Aws::Errors::MissingCredentialsError)
+        end
+
+        include_examples 'provision error', 'Amazon credentials are not configured'
+      end
+
+      context 'AWS credentials are not configured' do
+        before do
+          allow(Clusters::Aws::FetchCredentialsService).to receive(:new)
+            .and_raise(Settingslogic::MissingSetting)
+        end
+
+        include_examples 'provision error', 'Amazon credentials are not configured'
+      end
+
+      context 'Authentication failure' do
+        before do
+          allow(Clusters::Aws::FetchCredentialsService).to receive(:new)
+            .and_raise(Aws::STS::Errors::ServiceError.new(double, 'Error message'))
+        end
+
+        include_examples 'provision error', 'Amazon authentication failed'
+      end
+
+      context 'CloudFormation failure' do
+        before do
+          allow(client).to receive(:create_stack)
+            .and_raise(Aws::CloudFormation::Errors::ServiceError.new(double, 'Error message'))
+        end
+
+        include_examples 'provision error', 'Amazon CloudFormation request failed'
+      end
+    end
+  end
+end

spec/services/clusters/aws/verify_provision_status_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Clusters::Aws::VerifyProvisionStatusService do
+  describe '#execute' do
+    let(:provider) { create(:cluster_provider_aws) }
+
+    let(:stack) { double(stack_status: stack_status, creation_time: creation_time) }
+    let(:creation_time) { 1.minute.ago }
+
+    subject { described_class.new.execute(provider) }
+
+    before do
+      allow(provider.api_client).to receive(:describe_stacks)
+        .with(stack_name: provider.cluster.name)
+        .and_return(double(stacks: [stack]))
+    end
+
+    shared_examples 'provision error' do |message|
+      it "sets the status to :errored with an appropriate error message" do
+        subject
+
+        expect(provider).to be_errored
+        expect(provider.status_reason).to include(message)
+      end
+    end
+
+    context 'stack creation is still in progress' do
+      let(:stack_status) { 'CREATE_IN_PROGRESS' }
+      let(:verify_service) { double(execute: true) }
+
+      it 'schedules a worker to check again later' do
+        expect(WaitForClusterCreationWorker).to receive(:perform_in)
+          .with(described_class::POLL_INTERVAL, provider.cluster_id)
+
+        subject
+      end
+
+      context 'stack creation is taking too long' do
+        let(:creation_time) { 1.hour.ago }
+
+        include_examples 'provision error', 'Kubernetes cluster creation time exceeds timeout'
+      end
+    end
+
+    context 'stack creation is complete' do
+      let(:stack_status) { 'CREATE_COMPLETE' }
+      let(:finalize_service) { double(execute: true) }
+
+      it 'finalizes creation' do
+        expect(Clusters::Aws::FinalizeCreationService).to receive(:new).and_return(finalize_service)
+        expect(finalize_service).to receive(:execute).with(provider).once
+
+        subject
+      end
+    end
+
+    context 'stack creation failed' do
+      let(:stack_status) { 'CREATE_FAILED' }
+
+      include_examples 'provision error', 'Unexpected status'
+    end
+
+    context 'error communicating with CloudFormation API' do
+      let(:stack_status) { 'CREATE_IN_PROGRESS' }
+
+      before do
+        allow(provider.api_client).to receive(:describe_stacks)
+          .and_raise(Aws::CloudFormation::Errors::ServiceError.new(double, 'Error message'))
+      end
+
+      include_examples 'provision error', 'Amazon CloudFormation request failed'
+    end
+  end
+end

spec/workers/cluster_provision_worker_spec.rb

@@ -9,7 +9,18 @@ describe ClusterProvisionWorker do
       let(:provider) { create(:cluster_provider_gcp, :scheduled) }
 
       it 'provision a cluster' do
-        expect_any_instance_of(Clusters::Gcp::ProvisionService).to receive(:execute)
+        expect_any_instance_of(Clusters::Gcp::ProvisionService).to receive(:execute).with(provider)
+
+        described_class.new.perform(cluster.id)
+      end
+    end
+
+    context 'when provider type is aws' do
+      let(:cluster) { create(:cluster, provider_type: :aws, provider_aws: provider) }
+      let(:provider) { create(:cluster_provider_aws, :scheduled) }
+
+      it 'provision a cluster' do
+        expect_any_instance_of(Clusters::Aws::ProvisionService).to receive(:execute).with(provider)
 
         described_class.new.perform(cluster.id)
       end

spec/workers/wait_for_cluster_creation_worker_spec.rb

@@ -8,8 +8,19 @@ describe WaitForClusterCreationWorker do
       let(:cluster) { create(:cluster, provider_type: :gcp, provider_gcp: provider) }
       let(:provider) { create(:cluster_provider_gcp, :creating) }
 
-      it 'provision a cluster' do
-        expect_any_instance_of(Clusters::Gcp::VerifyProvisionStatusService).to receive(:execute)
+      it 'provisions a cluster' do
+        expect_any_instance_of(Clusters::Gcp::VerifyProvisionStatusService).to receive(:execute).with(provider)
+
+        described_class.new.perform(cluster.id)
+      end
+    end
+
+    context 'when provider type is aws' do
+      let(:cluster) { create(:cluster, provider_type: :aws, provider_aws: provider) }
+      let(:provider) { create(:cluster_provider_aws, :creating) }
+
+      it 'provisions a cluster' do
+        expect_any_instance_of(Clusters::Aws::VerifyProvisionStatusService).to receive(:execute).with(provider)
 
         described_class.new.perform(cluster.id)
       end