Commit c778066b authored by Albert Salim's avatar Albert Salim

Clean up stale namespaces

parent 592cabc6
......@@ -116,6 +116,12 @@ class AutomatedCleanup
delete_helm_releases(releases_to_delete)
end
def perform_stale_namespace_cleanup!(days:)
kubernetes_client = Tooling::KubernetesClient.new(namespace: nil)
kubernetes_client.cleanup_review_app_namespaces(created_before: threshold_time(days: days), wait: false)
end
def perform_stale_pvc_cleanup!(days:)
kubernetes.cleanup_by_created_at(resource_type: 'pvc', created_before: threshold_time(days: days), wait: false)
end
......@@ -203,6 +209,10 @@ timed('Helm releases cleanup') do
automated_cleanup.perform_helm_releases_cleanup!(days: 7)
end
timed('Stale Namespace cleanup') do
automated_cleanup.perform_stale_namespace_cleanup!(days: 14)
end
timed('Stale PVC cleanup') do
automated_cleanup.perform_stale_pvc_cleanup!(days: 30)
end
......
......@@ -135,6 +135,52 @@ RSpec.describe Tooling::KubernetesClient do
end
end
describe '#cleanup_review_app_namespaces' do
let(:two_days_ago) { Time.now - 3600 * 24 * 2 }
let(:namespaces) { %w[review-abc-123 review-xyz-789] }
subject { described_class.new(namespace: nil) }
before do
allow(subject).to receive(:review_app_namespaces_created_before).with(created_before: two_days_ago).and_return(namespaces)
end
shared_examples 'a kubectl command to delete namespaces older than given creation time' do
let(:wait) { true }
specify do
expect(Gitlab::Popen).to receive(:popen_with_detail)
.with(["kubectl delete namespace " +
%(--now --ignore-not-found --wait=#{wait} #{namespaces.join(' ')})])
.and_return(Gitlab::Popen::Result.new([], '', '', double(success?: true)))
# We're not verifying the output here, just silencing it
expect { subject.cleanup_review_app_namespaces(created_before: two_days_ago) }.to output.to_stdout
end
end
it_behaves_like 'a kubectl command to delete namespaces older than given creation time'
it 'raises an error if the Kubernetes command fails' do
expect(Gitlab::Popen).to receive(:popen_with_detail)
.with(["kubectl delete namespace " +
%(--now --ignore-not-found --wait=true #{namespaces.join(' ')})])
.and_return(Gitlab::Popen::Result.new([], '', '', double(success?: false)))
expect { subject.cleanup_review_app_namespaces(created_before: two_days_ago) }.to raise_error(described_class::CommandFailedError)
end
context 'with no namespaces found' do
let(:namespaces) { [] }
it 'does not call #delete_namespaces_by_exact_names' do
expect(subject).not_to receive(:delete_namespaces_by_exact_names)
subject.cleanup_review_app_namespaces(created_before: two_days_ago)
end
end
end
describe '#raw_resource_names' do
it 'calls kubectl to retrieve the resource names' do
expect(Gitlab::Popen).to receive(:popen_with_detail)
......@@ -200,4 +246,49 @@ RSpec.describe Tooling::KubernetesClient do
it_behaves_like 'a kubectl command to retrieve resource names sorted by creationTimestamp'
end
end
describe '#review_app_namespaces_created_before' do
let(:three_days_ago) { Time.now - 3600 * 24 * 3 }
let(:two_days_ago) { Time.now - 3600 * 24 * 2 }
let(:namespace_created_three_days_ago) { 'namespace-created-three-days-ago' }
let(:resource_type) { 'namespace' }
let(:raw_resources) do
{
items: [
{
apiVersion: "v1",
kind: "Namespace",
metadata: {
creationTimestamp: three_days_ago,
name: namespace_created_three_days_ago,
labels: {
tls: 'review-apps-tls'
}
}
},
{
apiVersion: "v1",
kind: "Namespace",
metadata: {
creationTimestamp: Time.now,
name: 'another-pvc',
labels: {
tls: 'review-apps-tls'
}
}
}
]
}.to_json
end
specify do
expect(Gitlab::Popen).to receive(:popen_with_detail)
.with(["kubectl get namespace " \
"-l tls=review-apps-tls " \
"--sort-by='{.metadata.creationTimestamp}' -o json"])
.and_return(Gitlab::Popen::Result.new([], raw_resources, '', double(success?: true)))
expect(subject.__send__(:review_app_namespaces_created_before, created_before: two_days_ago)).to contain_exactly(namespace_created_three_days_ago)
end
end
end
......@@ -27,6 +27,13 @@ module Tooling
delete_by_exact_names(resource_type: resource_type, resource_names: resource_names, wait: wait)
end
def cleanup_review_app_namespaces(created_before:, wait: true)
namespaces = review_app_namespaces_created_before(created_before: created_before)
return if namespaces.empty?
delete_namespaces_by_exact_names(resource_names: namespaces, wait: wait)
end
private
def delete_by_selector(release_name:, wait:)
......@@ -66,6 +73,19 @@ module Tooling
run_command(command)
end
def delete_namespaces_by_exact_names(resource_names:, wait:)
command = [
'delete',
'namespace',
'--now',
'--ignore-not-found',
%(--wait=#{wait}),
resource_names.join(' ')
]
run_command(command)
end
def delete_by_matching_name(release_name:)
resource_names = raw_resource_names
command = [
......@@ -101,9 +121,32 @@ module Tooling
]
response = run_command(command)
JSON.parse(response)['items'] # rubocop:disable Gitlab/Json
.map { |resource| resource.dig('metadata', 'name') if Time.parse(resource.dig('metadata', 'creationTimestamp')) < created_before }
.compact
resources_created_before_date(response, created_before)
end
def review_app_namespaces_created_before(created_before:)
command = [
'get',
'namespace',
"-l tls=review-apps-tls", # Get only namespaces used for review-apps
"--sort-by='{.metadata.creationTimestamp}'",
'-o json'
]
response = run_command(command)
resources_created_before_date(response, created_before)
end
def resources_created_before_date(response, date)
items = JSON.parse(response)['items'] # rubocop:disable Gitlab/Json
items.filter_map do |item|
item_created_at = Time.parse(item.dig('metadata', 'creationTimestamp'))
item.dig('metadata', 'name') if item_created_at < date
end
rescue ::JSON::ParserError => ex
puts "Ignoring this JSON parsing error: #{ex}\n\nResponse was:\n#{response}" # rubocop:disable Rails/Output
[]
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment