Commit 6950f38f authored by Alessio Caiazza's avatar Alessio Caiazza

Install k8s application with helm running inside the cluster

parent 84f5aaa7
module Clusters
module Concerns
end
end
module Clusters
module Concerns
module AppStatus
extend ActiveSupport::Concern
included do
state_machine :status, initial: :scheduled do
state :errored, value: -1
state :scheduled, value: 0
state :installing, value: 1
state :installed, value: 2
event :make_installing do
transition any - [:installing] => :installing
end
event :make_installed do
transition any - [:installed] => :installed
end
event :make_errored do
transition any - [:errored] => :errored
end
before_transition any => [:errored] do |app_status, transition|
status_reason = transition.args.first
app_status.status_reason = status_reason if status_reason
end
end
end
end
end
end
module Clusters
module Kubernetes
def self.table_name_prefix
'clusters_kubernetes_'
end
def self.app(app_name)
case app_name
when HelmApp::NAME
HelmApp
else
raise ArgumentError, "Unknown app #{app_name}"
end
end
end
end
module Clusters
module Kubernetes
class HelmApp < ActiveRecord::Base
NAME = 'helm'.freeze
include ::Clusters::Concerns::AppStatus
belongs_to :kubernetes_service, class_name: 'KubernetesService', foreign_key: :service_id
default_value_for :version, Gitlab::Clusters::Helm::HELM_VERSION
alias_method :cluster, :kubernetes_service
def name
NAME
end
end
end
end
...@@ -3,6 +3,8 @@ class KubernetesService < DeploymentService ...@@ -3,6 +3,8 @@ class KubernetesService < DeploymentService
include Gitlab::Kubernetes include Gitlab::Kubernetes
include ReactiveCaching include ReactiveCaching
has_one :helm_app, class_name: 'Clusters::Kubernetes::HelmApp', foreign_key: :service_id
self.reactive_cache_key = ->(service) { [service.class.model_name.singular, service.project_id] } self.reactive_cache_key = ->(service) { [service.class.model_name.singular, service.project_id] }
# Namespace defaults to the project path, but can be overridden in case that # Namespace defaults to the project path, but can be overridden in case that
...@@ -136,6 +138,10 @@ class KubernetesService < DeploymentService ...@@ -136,6 +138,10 @@ class KubernetesService < DeploymentService
{ pods: read_pods } { pods: read_pods }
end end
def helm
Gitlab::Clusters::Helm.new(build_kubeclient!)
end
TEMPLATE_PLACEHOLDER = 'Kubernetes namespace'.freeze TEMPLATE_PLACEHOLDER = 'Kubernetes namespace'.freeze
private private
......
module Clusters
class BaseHelmService
attr_accessor :app
def initialize(app)
@app = app
end
protected
def helm
return @helm if defined?(@helm)
@helm = @app.cluster.helm
end
end
end
module Clusters
class FetchAppInstallationStatusService < BaseHelmService
def execute
return unless app.installing?
phase = helm.installation_status(app)
log = helm.installation_log(app) if phase == 'Failed'
yield(phase, log) if block_given?
rescue KubeException => ke
app.make_errored!("Kubernetes error: #{ke.message}") unless app.errored?
end
end
end
module Clusters
class FinalizeAppInstallationService < BaseHelmService
def execute
helm.delete_installation_pod!(app)
app.make_errored!('Installation aborted') if aborted?
end
private
def aborted?
app.installing? || app.scheduled?
end
end
end
module Clusters
class InstallAppService < BaseHelmService
def execute
return unless app.scheduled?
begin
helm.install(app)
if app.make_installing
ClusterWaitForAppInstallationWorker.perform_in(
ClusterWaitForAppInstallationWorker::INITIAL_INTERVAL, app.name, app.id)
else
app.make_errored!("Failed to update app record; #{app.errors}")
end
rescue KubeException => ke
app.make_errored!("Kubernetes error: #{ke.message}")
rescue StandardError => e
Rails.logger.warn(e.message)
app.make_errored!("Can't start installation process")
end
end
end
end
module Clusters
class InstallTillerService < BaseService
def execute
ensure_namespace
install
end
private
def kubernetes_service
return @kubernetes_service if defined?(@kubernetes_service)
@kubernetes_service = project&.kubernetes_service
end
def ensure_namespace
kubernetes_service&.ensure_namespace!
end
def install
kubernetes_service&.helm_client&.init!
end
end
end
class ClusterInstallAppWorker
include Sidekiq::Worker
include ClusterQueue
include ClusterApp
def perform(app_name, app_id)
find_app(app_name, app_id) do |app|
Clusters::InstallAppService.new(app).execute
end
end
end
class ClusterWaitForAppInstallationWorker
include Sidekiq::Worker
include ClusterQueue
include ClusterApp
INITIAL_INTERVAL = 30.seconds
EAGER_INTERVAL = 10.seconds
TIMEOUT = 20.minutes
def perform(app_name, app_id)
find_app(app_name, app_id) do |app|
Clusters::FetchAppInstallationStatusService.new(app).execute do |phase, log|
case phase
when 'Succeeded'
if app.make_installed
Clusters::FinalizeAppInstallationService.new(app).execute
else
app.make_errored!("Failed to update app record; #{app.errors}")
end
when 'Failed'
app.make_errored!(log || 'Installation silently failed')
Clusters::FinalizeAppInstallationService.new(app).execute
else
if Time.now.utc - app.updated_at.to_time.utc > TIMEOUT
app.make_errored!('App installation timeouted')
else
ClusterWaitForAppInstallationWorker.perform_in(EAGER_INTERVAL, app.name, app.id)
end
end
end
end
end
end
module ClusterApp
extend ActiveSupport::Concern
included do
def find_app(app_name, id)
app = Clusters::Kubernetes.app(app_name).find(id)
yield(app) if block_given?
end
end
end
class CreateClustersKubernetesHelmApps < ActiveRecord::Migration
def change
create_table :clusters_kubernetes_helm_apps do |t|
t.integer :status, null: false
t.datetime_with_timezone :created_at, null: false
t.datetime_with_timezone :updated_at, null: false
t.references :service, index: true, null: false, foreign_key: { on_delete: :cascade }
t.string :version, null: false
t.text :status_reason
end
end
end
module Gitlab
module Clusters
class Helm
Error = Class.new(StandardError)
HELM_VERSION = '2.7.0'.freeze
NAMESPACE = 'gitlab-managed-apps'.freeze
COMMAND_SCRIPT = <<-EOS.freeze
set -eo pipefail
apk add -U ca-certificates openssl >/dev/null
wget -q -O - https://kubernetes-helm.storage.googleapis.com/helm-v${HELM_VERSION}-linux-amd64.tar.gz | tar zxC /tmp >/dev/null
mv /tmp/linux-amd64/helm /usr/bin/
helm init ${HELM_INIT_OPTS} >/dev/null
[[ -z "${HELM_COMMAND+x}" ]] || helm ${HELM_COMMAND} >/dev/null
EOS
def initialize(kubeclient)
@kubeclient = kubeclient
end
def init!
ensure_namespace!
@kubeclient.create_pod(pod_resource(OpenStruct.new(name: 'helm')))
end
def install(app)
ensure_namespace!
@kubeclient.create_pod(pod_resource(app))
end
##
# Returns Pod phase
#
# https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase
#
# values: "Pending", "Running", "Succeeded", "Failed", "Unknown"
#
def installation_status(app)
@kubeclient.get_pod(pod_name(app), NAMESPACE).status.phase
end
def installation_log(app)
@kubeclient.get_pod_log(pod_name(app), NAMESPACE).body
end
def delete_installation_pod!(app)
@kubeclient.delete_pod(pod_name(app), NAMESPACE)
end
private
def pod_name(app)
"install-#{app.name}"
end
def pod_resource(app)
labels = { 'gitlab.org/action': 'install', 'gitlab.org/application': app.name }
metadata = { name: pod_name(app), namespace: NAMESPACE, labels: labels }
container = {
name: 'helm',
image: 'alpine:3.6',
env: generate_pod_env(app),
command: %w(/bin/sh),
args: %w(-c $(COMMAND_SCRIPT))
}
spec = { containers: [container], restartPolicy: 'Never' }
::Kubeclient::Resource.new(metadata: metadata, spec: spec)
end
def generate_pod_env(app)
env = {
HELM_VERSION: HELM_VERSION,
TILLER_NAMESPACE: NAMESPACE,
COMMAND_SCRIPT: COMMAND_SCRIPT
}
if app.name != 'helm'
env[:HELM_INIT_OPTS] = '--client-only'
env[:HELM_COMMAND] = helm_install_comand(app)
end
env.map { |key, value| { name: key, value: value } }
end
def helm_install_comand(app)
"install #{app.chart} --name #{app.name} --namespace #{NAMESPACE}"
end
def ensure_namespace!
begin
@kubeclient.get_namespace(NAMESPACE)
rescue KubeException => ke
raise ke unless ke.error_code == 404
namespace_resource = ::Kubeclient::Resource.new
namespace_resource.metadata = {}
namespace_resource.metadata.name = NAMESPACE
@kubeclient.create_namespace(namespace_resource)
end
end
end
end
end
require 'rails_helper'
require_relative '../kubernetes_spec'
RSpec.describe Clusters::Kubernetes::HelmApp, type: :model do
it_behaves_like 'a registered kubernetes app'
it { is_expected.to belong_to(:kubernetes_service) }
describe '#cluster' do
it 'is an alias to #kubernetes_service' do
expect(subject.method(:cluster).original_name).to eq(:kubernetes_service)
end
end
end
require 'rails_helper'
RSpec.shared_examples 'a registered kubernetes app' do
let(:name) { described_class::NAME }
it 'can be retrieved with Clusters::Kubernetes.app' do
expect(Clusters::Kubernetes.app(name)).to eq(described_class)
end
end
...@@ -7,8 +7,9 @@ describe KubernetesService, :use_clean_rails_memory_store_caching do ...@@ -7,8 +7,9 @@ describe KubernetesService, :use_clean_rails_memory_store_caching do
let(:project) { build_stubbed(:kubernetes_project) } let(:project) { build_stubbed(:kubernetes_project) }
let(:service) { project.kubernetes_service } let(:service) { project.kubernetes_service }
describe "Associations" do describe 'Associations' do
it { is_expected.to belong_to :project } it { is_expected.to belong_to :project }
it { is_expected.to have_one(:helm_app) }
end end
describe 'Validations' do describe 'Validations' do
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment