Enable Prometheus metrics and show debug info for tests

This is attempt to debug intermittent GRPC errors.

Relates to https://gitlab.com/gitlab-org/gitlab/-/issues/297359

lib/gitlab/setup_helper.rb

@@ -98,6 +98,7 @@ module Gitlab
 
           if Rails.env.test?
             socket_filename = options[:gitaly_socket] || "gitaly.socket"
+            prometheus_listen_addr = options[:prometheus_listen_addr]
 
             config = {
               # Override the set gitaly_address since Praefect is in the loop
@@ -106,8 +107,9 @@ module Gitlab
               # Compared to production, tests run in constrained environments. This
               # number is meant to grow with the number of concurrent rails requests /
               # sidekiq jobs, and concurrency will be low anyway in test.
-              git: { catfile_cache_size: 5 }
-            }
+              git: { catfile_cache_size: 5 },
+              prometheus_listen_addr: prometheus_listen_addr
+            }.compact
 
             storage_path = Rails.root.join('tmp', 'tests', 'second_storage').to_s
             storages << { name: 'test_second_storage', path: storage_path }

spec/spec_helper.rb

@@ -92,6 +92,25 @@ RSpec.configure do |config|
     config.full_backtrace = true
   end
 
+  # Attempt to troubleshoot https://gitlab.com/gitlab-org/gitlab/-/issues/297359
+  if ENV['CI']
+    config.after do |example|
+      if example.exception.is_a?(GRPC::Unavailable)
+        warn "=== gRPC unavailable detected, process list:"
+        processes = `ps -ef | grep toml`
+        warn processes
+        warn "=== free memory"
+        warn `free -m`
+        warn "=== uptime"
+        warn `uptime`
+        warn "=== Prometheus metrics:"
+        warn `curl -s http://localhost:9236/metrics`
+        warn "=== Taking goroutine dump in log/goroutines.log..."
+        warn `curl -o log/goroutines.log http://localhost:9236/debug/pprof/goroutine?debug=2`
+      end
+    end
+  end
+
   unless ENV['CI']
     # Re-run failures locally with `--only-failures`
     config.example_status_persistence_file_path = './spec/examples.txt'

spec/support/helpers/test_env.rb

@@ -170,7 +170,14 @@ module TestEnv
       install_dir: gitaly_dir,
       version: Gitlab::GitalyClient.expected_server_version,
       task: "gitlab:gitaly:install[#{install_gitaly_args}]") do
-        Gitlab::SetupHelper::Gitaly.create_configuration(gitaly_dir, { 'default' => repos_path }, force: true)
+        Gitlab::SetupHelper::Gitaly.create_configuration(
+          gitaly_dir,
+          { 'default' => repos_path },
+          force: true,
+          options: {
+            prometheus_listen_addr: ':9236'
+          }
+        )
         Gitlab::SetupHelper::Gitaly.create_configuration(
           gitaly_dir,
           { 'default' => repos_path },