Move timeout settings from Prom Client to Service

Also:
- Restore original 60s timeouts at service level
- Include duration in the error message in application logs.
- Add changlog entry

Since the client is used widely now and different features come with
different query profiles, and since we already set defaults in the http
client, we should define this closer to the caller now.

app/models/project_services/prometheus_service.rb

@@ -97,7 +97,13 @@ class PrometheusService < MonitoringService
   def prometheus_client
     return unless should_return_client?
 
-    options = { allow_local_requests: allow_local_api_url? }
+    options = {
+      allow_local_requests: allow_local_api_url?,
+      # We should choose more conservative timeouts, but some queries we run are now busting our
+      # default timeouts, which are stricter. We should make those queries faster instead.
+      # See https://gitlab.com/gitlab-org/gitlab/-/issues/233109
+      timeout: 60
+    }
 
     if behind_iap?
       # Adds the Authorization header

app/services/web_hook_service.rb

@@ -55,16 +55,17 @@ class WebHookService
       message: response.to_s
     }
   rescue SocketError, OpenSSL::SSL::SSLError, Errno::ECONNRESET, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Net::OpenTimeout, Net::ReadTimeout, Gitlab::HTTP::BlockedUrlError, Gitlab::HTTP::RedirectionTooDeep, Gitlab::Json::LimitedEncoder::LimitExceeded => e
+    execution_duration = Gitlab::Metrics::System.monotonic_time - start_time
     log_execution(
       trigger: hook_name,
       url: hook.url,
       request_data: data,
       response: InternalErrorResponse.new,
-      execution_duration: Gitlab::Metrics::System.monotonic_time - start_time,
+      execution_duration: execution_duration,
       error_message: e.to_s
     )
 
-    Gitlab::AppLogger.error("WebHook Error => #{e}")
+    Gitlab::AppLogger.error("WebHook Error after #{execution_duration.to_i.seconds}s => #{e}")
 
     {
       status: :error,

changelogs/unreleased/233109-http-timeout-handling-improvements.yml

+---
+title: Set longer Prometheus timeouts in PrometheusService
+merge_request: 39318
+author:
+type: other

lib/gitlab/prometheus_client.rb

@@ -132,7 +132,7 @@ module Gitlab
 
     def http_options
       strong_memoize(:http_options) do
-        { follow_redirects: false, open_timeout: 5, read_timeout: 10 }.merge(mapped_options)
+        { follow_redirects: false }.merge(mapped_options)
       end
     end