Silence unrecoverable `Kubeclient::HttpError`s in cleanup

These should not cause cluster removal to fail, because
they will never recover. And they are likely "user errors",
so they should not affect our error budgets.

See https://gitlab.com/gitlab-org/gitlab/-/issues/342923

Changelog: fixed

Add specs to Kubeclient::HttpError rescue

app/services/clusters/cleanup/project_namespace_service.rb

@@ -26,8 +26,10 @@ module Clusters
 
           begin
             kubeclient_delete_namespace(kubernetes_namespace)
-          rescue Kubeclient::HttpError
-            next
+          rescue Kubeclient::HttpError => e
+            # unauthorized, forbidden: GitLab's access has been revoked
+            # certificate verify failed: Cluster is probably gone forever
+            raise unless e.message =~ /unauthorized|forbidden|certificate verify failed/i
           end
 
           kubernetes_namespace.destroy!

app/services/clusters/cleanup/service_account_service.rb

@@ -24,6 +24,10 @@ module Clusters
         # The resources have already been deleted, possibly on a previous attempt that timed out
       rescue Gitlab::UrlBlocker::BlockedUrlError
         # User gave an invalid cluster from the start, or deleted the endpoint before this job ran
+      rescue Kubeclient::HttpError => e
+        # unauthorized, forbidden: GitLab's access has been revoked
+        # certificate verify failed: Cluster is probably gone forever
+        raise unless e.message =~ /unauthorized|forbidden|certificate verify failed/i
       end
     end
   end

spec/services/clusters/cleanup/project_namespace_service_spec.rb

@@ -95,5 +95,31 @@ RSpec.describe Clusters::Cleanup::ProjectNamespaceService do
         subject
       end
     end
+
+    context 'when there is a Kubeclient::HttpError' do
+      let(:kubeclient_instance_double) do
+        instance_double(Gitlab::Kubernetes::KubeClient)
+      end
+
+      ['Unauthorized', 'forbidden', 'Certificate verify Failed'].each do |message|
+        it 'schedules ::ServiceAccountWorker with accepted errors' do
+          allow(kubeclient_instance_double)
+            .to receive(:delete_namespace)
+            .and_raise(Kubeclient::HttpError.new(401, message, nil))
+
+          expect(Clusters::Cleanup::ServiceAccountWorker).to receive(:perform_async).with(cluster.id)
+
+          subject
+        end
+      end
+
+      it 'raises error with unaccepted errors' do
+        allow(kubeclient_instance_double)
+          .to receive(:delete_namespace)
+          .and_raise(Kubeclient::HttpError.new(401, 'unexpected message', nil))
+
+        expect { subject }.to raise_error(Kubeclient::HttpError)
+      end
+    end
   end
 end

spec/services/clusters/cleanup/service_account_service_spec.rb

@@ -52,5 +52,19 @@ RSpec.describe Clusters::Cleanup::ServiceAccountService do
         expect { subject }.to change { Clusters::Cluster.where(id: cluster.id).exists? }.from(true).to(false)
       end
     end
+
+    context 'when there is a Kubeclient::HttpError' do
+      ['Unauthorized', 'forbidden', 'Certificate verify Failed'].each do |message|
+        before do
+          allow(kubeclient_instance_double)
+            .to receive(:delete_service_account)
+            .and_raise(Kubeclient::HttpError.new(401, message, nil))
+        end
+
+        it 'destroys cluster' do
+          expect { subject }.to change { Clusters::Cluster.where(id: cluster.id).exists? }.from(true).to(false)
+        end
+      end
+    end
   end
 end