Merge branch '220477-harden-ci-pipelines-usage-data-queries' into 'master'

Harden  CI pipelines usage data queries

See merge request gitlab-org/gitlab!34045

changelogs/unreleased/220477-harden-ci-pipelines-usage-data-queries.yml

+---
+title: Harden CI pipelines usage data queries with an index
+merge_request: 34045
+author:
+type: performance

db/post_migrate/20200608075553_add_index_on_user_id_and_created_at_and_source_to_ci_pipelines.rb

+# frozen_string_literal: true
+
+class AddIndexOnUserIdAndCreatedAtAndSourceToCiPipelines < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :ci_pipelines, [:user_id, :created_at, :source]
+    remove_concurrent_index :ci_pipelines, [:user_id, :created_at]
+  end
+
+  def down
+    add_concurrent_index :ci_pipelines, [:user_id, :created_at]
+    remove_concurrent_index :ci_pipelines, [:user_id, :created_at, :source]
+  end
+end

db/structure.sql

@@ -9455,7 +9455,7 @@ CREATE INDEX index_ci_pipelines_on_project_idandrefandiddesc ON public.ci_pipeli
 
 CREATE INDEX index_ci_pipelines_on_status ON public.ci_pipelines USING btree (status);
 
-CREATE INDEX index_ci_pipelines_on_user_id_and_created_at ON public.ci_pipelines USING btree (user_id, created_at);
+CREATE INDEX index_ci_pipelines_on_user_id_and_created_at_and_source ON public.ci_pipelines USING btree (user_id, created_at, source);
 
 CREATE UNIQUE INDEX index_ci_refs_on_project_id_and_ref_path ON public.ci_refs USING btree (project_id, ref_path);
 
@@ -13846,5 +13846,6 @@ COPY "schema_migrations" (version) FROM STDIN;
 20200602013901
 20200603073101
 20200604143628
+20200608075553
 \.
 

ee/lib/ee/gitlab/usage_data.rb

@@ -341,12 +341,12 @@ module EE
         def usage_activity_by_stage_verify(time_period)
           {
             ci_builds: distinct_count(::Ci::Build.where(time_period), :user_id),
-            ci_external_pipelines: distinct_count(::Ci::Pipeline.external.where(time_period), :user_id),
-            ci_internal_pipelines: distinct_count(::Ci::Pipeline.internal.where(time_period), :user_id),
-            ci_pipeline_config_auto_devops: distinct_count(::Ci::Pipeline.auto_devops_source.where(time_period), :user_id),
-            ci_pipeline_config_repository: distinct_count(::Ci::Pipeline.repository_source.where(time_period), :user_id),
+            ci_external_pipelines: distinct_count(::Ci::Pipeline.external.where(time_period), :user_id, start: user_minimum_id, finish: user_maximum_id),
+            ci_internal_pipelines: distinct_count(::Ci::Pipeline.internal.where(time_period), :user_id, start: user_minimum_id, finish: user_maximum_id),
+            ci_pipeline_config_auto_devops: distinct_count(::Ci::Pipeline.auto_devops_source.where(time_period), :user_id, start: user_minimum_id, finish: user_maximum_id),
+            ci_pipeline_config_repository: distinct_count(::Ci::Pipeline.repository_source.where(time_period), :user_id, start: user_minimum_id, finish: user_maximum_id),
             ci_pipeline_schedules: distinct_count(::Ci::PipelineSchedule.where(time_period), :owner_id),
-            ci_pipelines: distinct_count(::Ci::Pipeline.where(time_period), :user_id),
+            ci_pipelines: distinct_count(::Ci::Pipeline.where(time_period), :user_id, start: user_minimum_id, finish: user_maximum_id),
             ci_triggers: distinct_count(::Ci::Trigger.where(time_period), :owner_id),
             clusters_applications_runner: cluster_applications_user_distinct_count(::Clusters::Applications::Runner, time_period),
             projects_reporting_ci_cd_back_to_github: distinct_count(::Project.with_github_service_pipeline_events.where(time_period), :creator_id)