Store pipeline counts by status

This change periodically stores pipeline counts by status for the
instance statistics feature.

app/graphql/types/admin/analytics/instance_statistics/measurement_identifier_enum.rb

@@ -14,6 +14,10 @@ module Types
           value 'MERGE_REQUESTS', 'Merge request count', value: :merge_requests
           value 'GROUPS', 'Group count', value: :groups
           value 'PIPELINES', 'Pipeline count', value: :pipelines
+          value 'PIPELINES_SUCCEEDED', 'Pipeline count with success status', value: :pipelines_succeeded
+          value 'PIPELINES_FAILED', 'Pipeline count with failed status', value: :pipelines_failed
+          value 'PIPELINES_CANCELED', 'Pipeline count with canceled status', value: :pipelines_canceled
+          value 'PIPELINES_SKIPPED', 'Pipeline count with skipped status', value: :pipelines_skipped
         end
       end
     end

app/models/analytics/instance_statistics/measurement.rb

@@ -3,13 +3,19 @@
 module Analytics
   module InstanceStatistics
     class Measurement < ApplicationRecord
+      EXPERIMENTAL_IDENTIFIERS = %i[pipelines_succeeded pipelines_failed pipelines_canceled pipelines_skipped].freeze
+
       enum identifier: {
         projects: 1,
         users: 2,
         issues: 3,
         merge_requests: 4,
         groups: 5,
-        pipelines: 6
+        pipelines: 6,
+        pipelines_succeeded: 7,
+        pipelines_failed: 8,
+        pipelines_canceled: 9,
+        pipelines_skipped: 10
       }
 
       IDENTIFIER_QUERY_MAPPING = {
@@ -18,7 +24,11 @@ module Analytics
         identifiers[:issues] => -> { Issue },
         identifiers[:merge_requests] => -> { MergeRequest },
         identifiers[:groups] => -> { Group },
-        identifiers[:pipelines] => -> { Ci::Pipeline }
+        identifiers[:pipelines] => -> { Ci::Pipeline },
+        identifiers[:pipelines_succeeded] => -> { Ci::Pipeline.success },
+        identifiers[:pipelines_failed] => -> { Ci::Pipeline.failed },
+        identifiers[:pipelines_canceled] => -> { Ci::Pipeline.canceled },
+        identifiers[:pipelines_skipped] => -> { Ci::Pipeline.skipped }
       }.freeze
 
       validates :recorded_at, :identifier, :count, presence: true
@@ -26,6 +36,14 @@ module Analytics
 
       scope :order_by_latest, -> { order(recorded_at: :desc) }
       scope :with_identifier, -> (identifier) { where(identifier: identifier) }
+
+      def self.measurement_identifier_values
+        if Feature.enabled?(:store_ci_pipeline_counts_by_status)
+          identifiers.values
+        else
+          identifiers.values - EXPERIMENTAL_IDENTIFIERS.map { |identifier| identifiers[identifier] }
+        end
+      end
     end
   end
 end

app/workers/analytics/instance_statistics/count_job_trigger_worker.rb

@@ -17,10 +17,9 @@ module Analytics
         return if Feature.disabled?(:store_instance_statistics_measurements, default_enabled: true)
 
         recorded_at = Time.zone.now
-        measurement_identifiers = Analytics::InstanceStatistics::Measurement.identifiers
 
         worker_arguments = Gitlab::Analytics::InstanceStatistics::WorkersArgumentBuilder.new(
-          measurement_identifiers: measurement_identifiers.values,
+          measurement_identifiers: ::Analytics::InstanceStatistics::Measurement.measurement_identifier_values,
           recorded_at: recorded_at
         ).execute
 

changelogs/unreleased/246847-store-pipeline-counts-by-status.yml

+---
+title: Store pipeline counts by status for instance statistics
+merge_request: 43027
+author:
+type: changed

config/feature_flags/development/store_ci_pipeline_counts_by_status.yml

+---
+name: store_ci_pipeline_counts_by_status
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/43027
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/254721
+type: development
+group: group::analytics
+default_enabled: false

db/post_migrate/20200922170907_change_index_on_pipeline_status.rb

+# frozen_string_literal: true
+
+class ChangeIndexOnPipelineStatus < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  OLD_INDEX_NAME = 'index_ci_pipelines_on_status'
+  NEW_INDEX_NAME = 'index_ci_pipelines_on_status_and_id'
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :ci_pipelines, [:status, :id], name: NEW_INDEX_NAME
+    remove_concurrent_index_by_name :ci_pipelines, name: OLD_INDEX_NAME
+  end
+
+  def down
+    add_concurrent_index :ci_pipelines, :status, name: OLD_INDEX_NAME
+    remove_concurrent_index_by_name :ci_pipelines, name: NEW_INDEX_NAME
+  end
+end

db/schema_migrations/20200922170907

+ab044b609a29e9a179813de79dab9770665917a8ed78db907755a64f2d4aa47c
\ No newline at end of file

db/structure.sql

@@ -19711,7 +19711,7 @@ CREATE INDEX index_ci_pipelines_on_project_id_and_user_id_and_status_and_ref ON
 
 CREATE INDEX index_ci_pipelines_on_project_idandrefandiddesc ON ci_pipelines USING btree (project_id, ref, id DESC);
 
-CREATE INDEX index_ci_pipelines_on_status ON ci_pipelines USING btree (status);
+CREATE INDEX index_ci_pipelines_on_status_and_id ON ci_pipelines USING btree (status, id);
 
 CREATE INDEX index_ci_pipelines_on_user_id_and_created_at_and_config_source ON ci_pipelines USING btree (user_id, created_at, config_source);
 

doc/api/graphql/reference/gitlab_schema.graphql

@@ -9388,6 +9388,26 @@ enum MeasurementIdentifier {
   """
   PIPELINES
 
+  """
+  Pipeline count with canceled status
+  """
+  PIPELINES_CANCELED
+
+  """
+  Pipeline count with failed status
+  """
+  PIPELINES_FAILED
+
+  """
+  Pipeline count with skipped status
+  """
+  PIPELINES_SKIPPED
+
+  """
+  Pipeline count with success status
+  """
+  PIPELINES_SUCCEEDED
+
   """
   Project count
   """

doc/api/graphql/reference/gitlab_schema.json

@@ -26010,6 +26010,30 @@
               "description": "Pipeline count",
               "isDeprecated": false,
               "deprecationReason": null
+            },
+            {
+              "name": "PIPELINES_SUCCEEDED",
+              "description": "Pipeline count with success status",
+              "isDeprecated": false,
+              "deprecationReason": null
+            },
+            {
+              "name": "PIPELINES_FAILED",
+              "description": "Pipeline count with failed status",
+              "isDeprecated": false,
+              "deprecationReason": null
+            },
+            {
+              "name": "PIPELINES_CANCELED",
+              "description": "Pipeline count with canceled status",
+              "isDeprecated": false,
+              "deprecationReason": null
+            },
+            {
+              "name": "PIPELINES_SKIPPED",
+              "description": "Pipeline count with skipped status",
+              "isDeprecated": false,
+              "deprecationReason": null
             }
           ],
           "possibleTypes": null

doc/api/graphql/reference/index.md

@@ -3224,6 +3224,10 @@ Possible identifier types for a measurement.
 | `ISSUES` | Issue count |
 | `MERGE_REQUESTS` | Merge request count |
 | `PIPELINES` | Pipeline count |
+| `PIPELINES_CANCELED` | Pipeline count with canceled status |
+| `PIPELINES_FAILED` | Pipeline count with failed status |
+| `PIPELINES_SKIPPED` | Pipeline count with skipped status |
+| `PIPELINES_SUCCEEDED` | Pipeline count with success status |
 | `PROJECTS` | Project count |
 | `USERS` | User count |
 

spec/factories/instance_statistics/measurement.rb

@@ -13,5 +13,13 @@ FactoryBot.define do
     trait :group_count do
       identifier { :groups }
     end
+
+    trait :pipelines_succeeded_count do
+      identifier { :pipelines_succeeded }
+    end
+
+    trait :pipelines_skipped_count do
+      identifier { :pipelines_skipped }
+    end
   end
 end

spec/graphql/resolvers/admin/analytics/instance_statistics/measurements_resolver_spec.rb

@@ -5,9 +5,11 @@ require 'spec_helper'
 RSpec.describe Resolvers::Admin::Analytics::InstanceStatistics::MeasurementsResolver do
   include GraphqlHelpers
 
+  let_it_be(:admin_user) { create(:user, :admin) }
+  let(:current_user) { admin_user }
+
   describe '#resolve' do
     let_it_be(:user) { create(:user) }
-    let_it_be(:admin_user) { create(:user, :admin) }
 
     let_it_be(:project_measurement_new) { create(:instance_statistics_measurement, :project_count, recorded_at: 2.days.ago) }
     let_it_be(:project_measurement_old) { create(:instance_statistics_measurement, :project_count, recorded_at: 10.days.ago) }
@@ -39,6 +41,37 @@ RSpec.describe Resolvers::Admin::Analytics::InstanceStatistics::MeasurementsReso
         end
       end
     end
+
+    context 'when requesting pipeline counts by pipeline status' do
+      let_it_be(:pipelines_succeeded_measurement) { create(:instance_statistics_measurement, :pipelines_succeeded_count, recorded_at: 2.days.ago) }
+      let_it_be(:pipelines_skipped_measurement) { create(:instance_statistics_measurement, :pipelines_skipped_count, recorded_at: 2.days.ago) }
+
+      subject { resolve_measurements({ identifier: identifier }, { current_user: current_user }) }
+
+      context 'filter for pipelines_succeeded' do
+        let(:identifier) { 'pipelines_succeeded' }
+
+        it { is_expected.to eq([pipelines_succeeded_measurement]) }
+      end
+
+      context 'filter for pipelines_skipped' do
+        let(:identifier) { 'pipelines_skipped' }
+
+        it { is_expected.to eq([pipelines_skipped_measurement]) }
+      end
+
+      context 'filter for pipelines_failed' do
+        let(:identifier) { 'pipelines_failed' }
+
+        it { is_expected.to be_empty }
+      end
+
+      context 'filter for pipelines_canceled' do
+        let(:identifier) { 'pipelines_canceled' }
+
+        it { is_expected.to be_empty }
+      end
+    end
   end
 
   def resolve_measurements(args = {}, context = {})

spec/models/analytics/instance_statistics/measurement_spec.rb

@@ -20,7 +20,11 @@ RSpec.describe Analytics::InstanceStatistics::Measurement, type: :model do
         issues: 3,
         merge_requests: 4,
         groups: 5,
-        pipelines: 6
+        pipelines: 6,
+        pipelines_succeeded: 7,
+        pipelines_failed: 8,
+        pipelines_canceled: 9,
+        pipelines_skipped: 10
       }.with_indifferent_access)
     end
   end
@@ -42,4 +46,28 @@ RSpec.describe Analytics::InstanceStatistics::Measurement, type: :model do
       it { is_expected.to match_array([measurement_1, measurement_2]) }
     end
   end
+
+  describe '#measurement_identifier_values' do
+    subject { described_class.measurement_identifier_values.count }
+
+    context 'when the `store_ci_pipeline_counts_by_status` feature flag is off' do
+      let(:expected_count) { Analytics::InstanceStatistics::Measurement.identifiers.size - Analytics::InstanceStatistics::Measurement::EXPERIMENTAL_IDENTIFIERS.size }
+
+      before do
+        stub_feature_flags(store_ci_pipeline_counts_by_status: false)
+      end
+
+      it { is_expected.to eq(expected_count) }
+    end
+
+    context 'when the `store_ci_pipeline_counts_by_status` feature flag is on' do
+      let(:expected_count) { Analytics::InstanceStatistics::Measurement.identifiers.size }
+
+      before do
+        stub_feature_flags(store_ci_pipeline_counts_by_status: true)
+      end
+
+      it { is_expected.to eq(expected_count) }
+    end
+  end
 end

spec/workers/analytics/instance_statistics/counter_job_worker_spec.rb

@@ -18,7 +18,7 @@ RSpec.describe Analytics::InstanceStatistics::CounterJobWorker do
     it 'counts a scope and stores the result' do
       subject
 
-      measurement = Analytics::InstanceStatistics::Measurement.first
+      measurement = Analytics::InstanceStatistics::Measurement.users.first
       expect(measurement.recorded_at).to be_like_time(recorded_at)
       expect(measurement.identifier).to eq('users')
       expect(measurement.count).to eq(2)
@@ -33,7 +33,7 @@ RSpec.describe Analytics::InstanceStatistics::CounterJobWorker do
     it 'sets 0 as the count' do
       subject
 
-      measurement = Analytics::InstanceStatistics::Measurement.first
+      measurement = Analytics::InstanceStatistics::Measurement.groups.first
       expect(measurement.recorded_at).to be_like_time(recorded_at)
       expect(measurement.identifier).to eq('groups')
       expect(measurement.count).to eq(0)
@@ -51,4 +51,20 @@ RSpec.describe Analytics::InstanceStatistics::CounterJobWorker do
 
     expect { subject }.not_to change { Analytics::InstanceStatistics::Measurement.count }
   end
+
+  context 'when pipelines_succeeded identifier is passed' do
+    let_it_be(:pipeline) { create(:ci_pipeline, :success) }
+
+    let(:successful_pipelines_measurement_identifier) { ::Analytics::InstanceStatistics::Measurement.identifiers.fetch(:pipelines_succeeded) }
+    let(:job_args) { [successful_pipelines_measurement_identifier, pipeline.id, pipeline.id, recorded_at] }
+
+    it 'counts successful pipelines' do
+      subject
+
+      measurement = Analytics::InstanceStatistics::Measurement.pipelines_succeeded.first
+      expect(measurement.recorded_at).to be_like_time(recorded_at)
+      expect(measurement.identifier).to eq('pipelines_succeeded')
+      expect(measurement.count).to eq(1)
+    end
+  end
 end