Instrument CI template usage across projects

Uses Redis HLL counters to track the number of distinct projects that
run pipelines with include:template for a few templates of interest.

The tracking is behind a common feature flag so that it can all be
turned off at once if there are performance concerns.

See https://gitlab.com/gitlab-org/gitlab/-/issues/219457

app/services/ci/create_pipeline_service.rb

@@ -27,6 +27,7 @@ module Ci
                 Gitlab::Ci::Pipeline::Chain::Limit::JobActivity,
                 Gitlab::Ci::Pipeline::Chain::CancelPendingPipelines,
                 Gitlab::Ci::Pipeline::Chain::Metrics,
+                Gitlab::Ci::Pipeline::Chain::TemplateUsage,
                 Gitlab::Ci::Pipeline::Chain::Pipeline::Process].freeze
 
     # Create a new pipeline in the specified project.

config/feature_flags/development/usage_data_track_ci_templates_unique_projects.yml

+---
+name: usage_data_track_ci_templates_unique_projects
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/50481
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/296880
+milestone: '13.8'
+type: development
+group: group::configure
+default_enabled: false

lib/gitlab/ci/config.rb

@@ -70,6 +70,10 @@ module Gitlab
         @normalized_jobs ||= Ci::Config::Normalizer.new(jobs).normalize_jobs
       end
 
+      def included_templates
+        @context.expandset.filter_map { |i| i[:template] }
+      end
+
       private
 
       def expand_config(config)

lib/gitlab/ci/pipeline/chain/template_usage.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Pipeline
+      module Chain
+        class TemplateUsage < Chain::Base
+          def perform!
+            included_templates.each do |template|
+              track_event(template)
+            end
+          end
+
+          def break?
+            false
+          end
+
+          private
+
+          def track_event(template)
+            Gitlab::UsageDataCounters::CiTemplateUniqueCounter
+              .track_unique_project_event(project_id: pipeline.project_id, template: template)
+          end
+
+          def included_templates
+            command.yaml_processor_result.included_templates
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/yaml_processor/result.rb

@@ -53,6 +53,10 @@ module Gitlab
           @stages ||= @ci_config.stages
         end
 
+        def included_templates
+          @included_templates ||= @ci_config.included_templates
+        end
+
         def build_attributes(name)
           job = jobs.fetch(name.to_sym, {})
 

lib/gitlab/usage_data_counters/ci_template_unique_counter.rb

+# frozen_string_literal: true
+
+module Gitlab::UsageDataCounters
+  class CiTemplateUniqueCounter
+    REDIS_SLOT = 'ci_templates'.freeze
+
+    TEMPLATE_TO_EVENT = {
+      'Auto-DevOps.gitlab-ci.yml' => 'auto_devops',
+      'AWS/CF-Provision-and-Deploy-EC2.gitlab-ci.yml' => 'aws_cf_deploy_ec2',
+      'AWS/Deploy-ECS.gitlab-ci.yml' => 'aws_deploy_ecs',
+      'Jobs/Build.gitlab-ci.yml' => 'auto_devops_build',
+      'Jobs/Deploy.gitlab-ci.yml' => 'auto_devops_deploy',
+      'Jobs/Deploy.latest.gitlab-ci.yml' => 'auto_devops_deploy_latest',
+      'Security/SAST.gitlab-ci.yml' => 'security_sast',
+      'Security/Secret-Detection.gitlab-ci.yml' => 'security_secret_detection',
+      'Terraform/Base.latest.gitlab-ci.yml' => 'terraform_base_latest'
+    }.freeze
+
+    class << self
+      def track_unique_project_event(project_id:, template:)
+        return if Feature.disabled?(:usage_data_track_ci_templates_unique_projects, default_enabled: :yaml)
+
+        if event = unique_project_event(template)
+          Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event, values: project_id)
+        end
+      end
+
+      private
+
+      def unique_project_event(template)
+        if name = TEMPLATE_TO_EVENT[template]
+          "p_#{REDIS_SLOT}_#{name}"
+        end
+      end
+    end
+  end
+end

lib/gitlab/usage_data_counters/known_events/common.yml

@@ -467,3 +467,44 @@
   redis_slot: terraform
   aggregation: weekly
   feature_flag: usage_data_p_terraform_state_api_unique_users
+# CI templates
+- name: p_ci_templates_auto_devops
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+  feature_flag: usage_data_track_ci_templates_unique_projects
+- name: p_ci_templates_aws_cf_deploy_ec2
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+  feature_flag: usage_data_track_ci_templates_unique_projects
+- name: p_ci_templates_auto_devops_build
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+  feature_flag: usage_data_track_ci_templates_unique_projects
+- name: p_ci_templates_auto_devops_deploy
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+  feature_flag: usage_data_track_ci_templates_unique_projects
+- name: p_ci_templates_auto_devops_deploy_latest
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+  feature_flag: usage_data_track_ci_templates_unique_projects
+- name: p_ci_templates_security_sast
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+  feature_flag: usage_data_track_ci_templates_unique_projects
+- name: p_ci_templates_security_secret_detection
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+  feature_flag: usage_data_track_ci_templates_unique_projects
+- name: p_ci_templates_terraform_base_latest
+  category: ci_templates
+  redis_slot: ci_templates
+  aggregation: weekly
+  feature_flag: usage_data_track_ci_templates_unique_projects

spec/lib/gitlab/ci/config_spec.rb

@@ -82,6 +82,30 @@ RSpec.describe Gitlab::Ci::Config do
     end
   end
 
+  describe '#included_templates' do
+    let(:yml) do
+      <<-EOS
+        include:
+          - template: Jobs/Deploy.gitlab-ci.yml
+          - template: Jobs/Build.gitlab-ci.yml
+          - remote: https://example.com/gitlab-ci.yml
+      EOS
+    end
+
+    before do
+      stub_request(:get, 'https://example.com/gitlab-ci.yml').to_return(status: 200, body: <<-EOS)
+        test:
+          script: [ 'echo hello world' ]
+      EOS
+    end
+
+    subject(:included_templates) do
+      config.included_templates
+    end
+
+    it { is_expected.to contain_exactly('Jobs/Deploy.gitlab-ci.yml', 'Jobs/Build.gitlab-ci.yml') }
+  end
+
   context 'when using extendable hash' do
     let(:yml) do
       <<-EOS

spec/lib/gitlab/ci/pipeline/chain/template_usage_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Ci::Pipeline::Chain::TemplateUsage do
+  let_it_be(:project) { create(:project) }
+  let_it_be(:user) { create(:user) }
+  let(:pipeline) { create(:ci_pipeline, project: project) }
+
+  let(:command) do
+    Gitlab::Ci::Pipeline::Chain::Command.new(project: project, current_user: user)
+  end
+
+  let(:step) { described_class.new(pipeline, command) }
+
+  describe '#perform!' do
+    subject(:perform) { step.perform! }
+
+    it 'tracks the included templates' do
+      expect(command).to(
+        receive(:yaml_processor_result)
+          .and_return(
+            double(included_templates: %w(Template-1 Template-2))
+          )
+      )
+
+      %w(Template-1 Template-2).each do |expected_template|
+        expect(Gitlab::UsageDataCounters::CiTemplateUniqueCounter).to(
+          receive(:track_unique_project_event)
+            .with(project_id: project.id, template: expected_template)
+        )
+      end
+
+      perform
+    end
+  end
+end

spec/lib/gitlab/usage_data_counters/ci_template_unique_counter_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::UsageDataCounters::CiTemplateUniqueCounter do
+  let(:project_id) { 1 }
+
+  describe '.track_unique_project_event' do
+    described_class::TEMPLATE_TO_EVENT.keys.each do |template|
+      context "when given template #{template}" do
+        it_behaves_like 'tracking unique hll events', :usage_data_track_ci_templates_unique_projects do
+          subject(:request) { described_class.track_unique_project_event(project_id: project_id, template: template) }
+
+          let(:target_id) { "p_ci_templates_#{described_class::TEMPLATE_TO_EVENT[template]}" }
+          let(:expected_type) { instance_of(Integer) }
+        end
+      end
+    end
+
+    it 'does not track templates outside of TEMPLATE_TO_EVENT' do
+      expect(Gitlab::UsageDataCounters::HLLRedisCounter).not_to(
+        receive(:track_event)
+      )
+      Dir.glob(File.join('lib', 'gitlab', 'ci', 'templates', '**'), base: Rails.root) do |template|
+        next if described_class::TEMPLATE_TO_EVENT.key?(template)
+
+        described_class.track_unique_project_event(project_id: 1, template: template)
+      end
+    end
+  end
+end

spec/lib/gitlab/usage_data_counters/hll_redis_counter_spec.rb

@@ -38,7 +38,8 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s
         'ci_secrets_management',
         'snippets',
         'code_review',
-        'terraform'
+        'terraform',
+        'ci_templates'
       )
     end
   end

spec/services/ci/create_pipeline_service_spec.rb

@@ -91,6 +91,14 @@ RSpec.describe Ci::CreatePipelineService do
           .with({ source: 'push' }, 5)
       end
 
+      it 'tracks included template usage' do
+        expect_next_instance_of(Gitlab::Ci::Pipeline::Chain::TemplateUsage) do |instance|
+          expect(instance).to receive(:perform!)
+        end
+
+        execute_service
+      end
+
       describe 'recording a conversion event' do
         it 'schedules a record conversion event worker' do
           expect(Experiments::RecordConversionEventWorker).to receive(:perform_async).with(:ci_syntax_templates, user.id)