ci: Automate the cleanup of stale PVC

This adds automated cleanup of stale PVC that were created as part of
Review Apps but are not is use anymore.
Signed-off-by: Rémy Coutable <remy@rymai.me>

Guardfile

@@ -6,7 +6,7 @@ require "guard/rspec/dsl"
 
 cmd = ENV['GUARD_CMD'] || (ENV['SPRING'] ? 'spring rspec' : 'bundle exec rspec')
 
-directories %w(app ee lib spec)
+directories %w(app ee lib rubocop tooling spec)
 
 rspec_context_for = proc do |context_path|
   OpenStruct.new(to_s: "spec").tap do |rspec|
@@ -42,6 +42,8 @@ guard_setup = proc do |context_path|
 
   # Ruby files
   watch(%r{^#{context_path}(lib/.+)\.rb$}) { |m| rspec.spec.call(m[1]) }
+  watch(%r{^#{context_path}(rubocop/.+)\.rb$}) { |m| rspec.spec.call(m[1]) }
+  watch(%r{^#{context_path}(tooling/.+)\.rb$}) { |m| rspec.spec.call(m[1]) }
 
   # Rails files
   rails = rails_context_for.call(context_path, %w(erb haml slim))

scripts/review_apps/automated_cleanup.rb

@@ -115,6 +115,10 @@ class AutomatedCleanup
     delete_helm_releases(releases_to_delete)
   end
 
+  def perform_stale_pvc_cleanup!(days:)
+    kubernetes.cleanup_by_created_at(resource_type: 'pvc', created_before: threshold_time(days: days), wait: false)
+  end
+
   private
 
   def fetch_environment(environment)
@@ -155,7 +159,7 @@ class AutomatedCleanup
 
     releases_names = releases.map(&:name)
     helm.delete(release_name: releases_names)
-    kubernetes.cleanup(release_name: releases_names, wait: false)
+    kubernetes.cleanup_by_release(release_name: releases_names, wait: false)
 
   rescue Tooling::Helm3Client::CommandFailedError => ex
     raise ex unless ignore_exception?(ex.message, IGNORED_HELM_ERRORS)
@@ -198,4 +202,8 @@ timed('Helm releases cleanup') do
   automated_cleanup.perform_helm_releases_cleanup!(days: 7)
 end
 
+timed('Stale PVC cleanup') do
+  automated_cleanup.perform_stale_pvc_cleanup!(days: 30)
+end
+
 exit(0)

tooling/lib/tooling/helm3_client.rb

@@ -66,13 +66,15 @@ module Tooling
         %(--output json),
         *args
       ]
-      releases = JSON.parse(run_command(command)) # rubocop:disable Gitlab/Json
+
+      response = run_command(command)
+      releases = JSON.parse(response) # rubocop:disable Gitlab/Json
 
       releases.map do |release|
         Release.new(*release.values_at(*RELEASE_JSON_ATTRIBUTES))
       end
     rescue ::JSON::ParserError => ex
-      puts "Ignoring this JSON parsing error: #{ex}" # rubocop:disable Rails/Output
+      puts "Ignoring this JSON parsing error: #{ex}\n\nResponse was:\n#{response}" # rubocop:disable Rails/Output
       []
     end
 

tooling/lib/tooling/kubernetes_client.rb

 # frozen_string_literal: true
 
+require 'json'
+require 'time'
 require_relative '../../../lib/gitlab/popen' unless defined?(Gitlab::Popen)
-require_relative '../../../lib/gitlab/json' unless defined?(Gitlab::JSON)
 
 module Tooling
   class KubernetesClient
@@ -14,11 +15,16 @@ module Tooling
       @namespace = namespace
     end
 
-    def cleanup(release_name:, wait: true)
+    def cleanup_by_release(release_name:, wait: true)
       delete_by_selector(release_name: release_name, wait: wait)
       delete_by_matching_name(release_name: release_name)
     end
 
+    def cleanup_by_created_at(resource_type:, created_before:, wait: true)
+      resource_names = resource_names_created_before(resource_type: resource_type, created_before: created_before)
+      delete_by_exact_names(resource_type: resource_type, resource_names: resource_names, wait: wait)
+    end
+
     private
 
     def delete_by_selector(release_name:, wait:)
@@ -45,6 +51,21 @@ module Tooling
       run_command(command)
     end
 
+    def delete_by_exact_names(resource_names:, wait:, resource_type: nil)
+      command = [
+        'delete',
+        resource_type,
+        %(--namespace "#{namespace}"),
+        '--now',
+        '--ignore-not-found',
+        '--include-uninitialized',
+        %(--wait=#{wait}),
+        resource_names.join(' ')
+      ]
+
+      run_command(command)
+    end
+
     def delete_by_matching_name(release_name:)
       resource_names = raw_resource_names
       command = [
@@ -70,8 +91,26 @@ module Tooling
       run_command(command).lines.map(&:strip)
     end
 
+    def resource_names_created_before(resource_type:, created_before:)
+      command = [
+        'get',
+        resource_type,
+        %(--namespace "#{namespace}"),
+        "--sort-by='{.metadata.creationTimestamp}'",
+        '-o json'
+      ]
+
+      response = run_command(command)
+      JSON.parse(response)['items'] # rubocop:disable Gitlab/Json
+        .map { |resource| resource.dig('metadata', 'name') if Time.parse(resource.dig('metadata', 'creationTimestamp')) < created_before }
+        .compact
+    rescue ::JSON::ParserError => ex
+      puts "Ignoring this JSON parsing error: #{ex}\n\nResponse was:\n#{response}" # rubocop:disable Rails/Output
+      []
+    end
+
     def run_command(command)
-      final_command = ['kubectl', *command].join(' ')
+      final_command = ['kubectl', *command.compact].join(' ')
       puts "Running command: `#{final_command}`" # rubocop:disable Rails/Output
 
       result = Gitlab::Popen.popen_with_detail([final_command])