Compare live vs actual CI minutes consumption

doc/administration/monitoring/prometheus/gitlab_metrics.md

@@ -130,6 +130,7 @@ The following metrics are available:
 | `pipeline_graph_links_total`                                     | Histogram   | 13.9    | Number of links per graph                                                                                             |                                                           |
 | `pipeline_graph_links_per_job_ratio`                             | Histogram   | 13.9    | Ratio of links to job per graph                                                                                       |                                                           |
 | `gitlab_ci_pipeline_security_orchestration_policy_processing_duration_seconds` | Histogram   | 13.12    | Time in seconds it takes to process Security Policies in CI/CD pipeline                                                                    |                                                           |
+| `gitlab_ci_difference_live_vs_actual_minutes`                    | Histogram   | 13.12    | Difference between CI minute consumption counted while jobs were running (live) vs when jobs are complete (actual). Used to enforce CI minute consumption limits on long running jobs. | `plan` |
 
 ## Metrics controlled by a feature flag
 

ee/app/services/ci/minutes/update_build_minutes_service.rb

@@ -16,6 +16,8 @@ module Ci
         legacy_track_usage_of_monthly_minutes(consumption_in_seconds)
 
         track_usage_of_monthly_minutes(consumption)
+
+        compare_with_live_consumption(build, consumption)
       end
 
       private
@@ -40,6 +42,14 @@ module Ci
         end
       end
 
+      def compare_with_live_consumption(build, consumption)
+        live_consumption = ::Ci::Minutes::TrackLiveConsumptionService.new(build).live_consumption
+        return if live_consumption == 0
+
+        difference = consumption.to_f - live_consumption.to_f
+        observe_ci_minutes_difference(difference, plan: namespace.actual_plan_name)
+      end
+
       def namespace_statistics
         namespace.namespace_statistics || namespace.create_namespace_statistics
       end
@@ -51,6 +61,12 @@ module Ci
       def namespace
         project.shared_runners_limit_namespace
       end
+
+      def observe_ci_minutes_difference(difference, plan:)
+        ::Gitlab::Ci::Pipeline::Metrics
+          .gitlab_ci_difference_live_vs_actual_minutes
+          .observe({ plan: plan }, difference)
+      end
     end
   end
 end

ee/changelogs/unreleased/compare-live-and-persisted-minutes-consumption-with.yml

+---
+title: Compare live vs actual CI minutes consumption and record it as a Prometheus metric
+merge_request: 60949
+author:
+type: added

ee/spec/services/ci/minutes/update_build_minutes_service_spec.rb

@@ -139,6 +139,44 @@ RSpec.describe Ci::Minutes::UpdateBuildMinutesService do
             .to eq((project.statistics.reload.shared_runners_seconds.to_f / 60).round(2))
         end
       end
+
+      context 'when live tracking exists for the build', :redis do
+        before do
+          allow(Gitlab).to receive(:com?).and_return(true)
+
+          build.update!(status: :running)
+
+          freeze_time do
+            ::Ci::Minutes::TrackLiveConsumptionService.new(build).tap do |service|
+              service.time_last_tracked_consumption!((build.duration.to_i - 5.minutes).ago)
+              service.execute
+            end
+          end
+
+          build.update!(status: :success)
+        end
+
+        it 'observes the difference between actual vs live consumption' do
+          histogram = double(:histogram)
+          expect(::Gitlab::Ci::Pipeline::Metrics)
+            .to receive(:gitlab_ci_difference_live_vs_actual_minutes)
+            .and_return(histogram)
+
+          expect(histogram).to receive(:observe).with({ plan: 'free' }, 5 * cost_factor)
+
+          subject
+        end
+
+        it_behaves_like 'new tracking matches legacy tracking'
+      end
+
+      context 'when live tracking does not exist for the build' do
+        it 'does not observe the difference' do
+          expect(::Gitlab::Ci::Pipeline::Metrics).not_to receive(:gitlab_ci_difference_live_vs_actual_minutes)
+
+          subject
+        end
+      end
     end
 
     context 'for specific runner' do

lib/gitlab/ci/pipeline/metrics.rb

@@ -70,6 +70,14 @@ module Gitlab
 
           Gitlab::Metrics.counter(name, comment)
         end
+
+        def self.gitlab_ci_difference_live_vs_actual_minutes
+          name = :gitlab_ci_difference_live_vs_actual_minutes
+          comment = 'Comparison between CI minutes consumption from live tracking vs actual consumption'
+          labels = {}
+          buckets = [-120.0, -60.0, -30.0, -10.0, -5.0, -3.0, -1.0, 0.0, 1.0, 3.0, 5.0, 10.0, 30.0, 60.0, 120.0]
+          ::Gitlab::Metrics.histogram(name, comment, labels, buckets)
+        end
       end
     end
   end