Merge branch 'count-security-pipelines' into 'master'

Count pipelines that have security jobs

See merge request gitlab-org/gitlab!37809

doc/development/telemetry/usage_ping.md

@@ -711,6 +711,12 @@ appear to be associated to any of the services running, since they all appear to
 | `releases`                                                | `usage_activity_by_stage`            | `release`     |                  | CE+EE   | Unique release tags in project                                             |
 | `successful_deployments`                                  | `usage_activity_by_stage`            | `release`     |                  | CE+EE   | Total successful deployments                                               |
 | `user_preferences_group_overview_security_dashboard`      | `usage_activity_by_stage`            | `secure`      |                  |         |                                                                            |
+| `sast_pipeline`                                           | `usage_activity_by_stage_monthly`    | `secure`      |                  |  EE     | Number of pipelines running sast jobs                                      |
+| `dependency_scanning_pipeline`                            | `usage_activity_by_stage_monthly`    | `secure`      |                  |  EE     | Number of pipelines running dependency jobs                                |
+| `container_scanning_pipeline`                             | `usage_activity_by_stage_monthly`    | `secure`      |                  |  EE     | Number of pipelines running container scanning jobs                        |
+| `dast_pipeline`                                           | `usage_activity_by_stage_monthly`    | `secure`      |                  |  EE     | Number of pipelines running dast jobs                                      |
+| `secret_detection_pipeline`                               | `usage_activity_by_stage_monthly`    | `secure`      |                  |  EE     | Number of pipelines running secret detection jobs                          |
+| `coverage_fuzzing_pipeline`                               | `usage_activity_by_stage_monthly`    | `secure`      |                  |  EE     | Number of pipelines running coverage jobs                                  |
 | `ci_builds`                                               | `usage_activity_by_stage`            | `verify`      |                  | CE+EE   | Unique builds in project                                                   |
 | `ci_external_pipelines`                                   | `usage_activity_by_stage`            | `verify`      |                  | CE+EE   | Total pipelines in external repositories                                   |
 | `ci_internal_pipelines`                                   | `usage_activity_by_stage`            | `verify`      |                  | CE+EE   | Total pipelines in GitLab repositories                                     |

ee/changelogs/unreleased/count-security-pipelines.yml

+---
+title: Count pipelines that have security jobs
+merge_request: 37809
+author:
+type: added

ee/lib/ee/gitlab/usage_data.rb

@@ -320,6 +320,8 @@ module EE
                                                                           finish: user_maximum_id)
           end
 
+          results.merge!(count_secure_pipelines(time_period))
+
           results[:"#{prefix}unique_users_all_secure_scanners"] = distinct_count(::Ci::Build.where(name: SECURE_PRODUCT_TYPES.keys).where(time_period), :user_id)
 
           # handle license rename https://gitlab.com/gitlab-org/gitlab/issues/8911
@@ -333,6 +335,27 @@ module EE
 
         private
 
+        # rubocop:disable CodeReuse/ActiveRecord
+        # rubocop: disable UsageData/LargeTable
+        def count_secure_pipelines(time_period)
+          return {} if time_period.blank?
+
+          start = ::Ci::Pipeline.minimum(:id)
+          finish = ::Ci::Pipeline.maximum(:id)
+          pipelines_with_secure_jobs = {}
+
+          ::Security::Scan.scan_types.each do |name, scan_type|
+            relation = ::Ci::Build.joins(:security_scans)
+                                .where(status: 'success', retried: [nil, false])
+                                .where('security_scans.scan_type = ?', scan_type)
+                                .where(time_period)
+            pipelines_with_secure_jobs["#{name}_pipeline".to_sym] = distinct_count(relation, :commit_id, start: start, finish: finish)
+          end
+
+          pipelines_with_secure_jobs
+        end
+        # rubocop: enabled UsageData/LargeTable
+
         def approval_merge_request_rule_minimum_id
           strong_memoize(:approval_merge_request_rule_minimum_id) do
             ::ApprovalMergeRequestRule.minimum(:id)

ee/spec/lib/ee/gitlab/usage_data_spec.rb

@@ -516,6 +516,55 @@ RSpec.describe Gitlab::UsageData do
             user_license_management_jobs: 1,
             user_sast_jobs: 1,
             user_secret_detection_jobs: 1,
+            sast_pipeline: 0,
+            dependency_scanning_pipeline: 0,
+            container_scanning_pipeline: 0,
+            dast_pipeline: 0,
+            secret_detection_pipeline: 0,
+            coverage_fuzzing_pipeline: 0,
+            user_unique_users_all_secure_scanners: 1
+          )
+        end
+
+        it 'counts pipelines that have security jobs' do
+          for_defined_days_back do
+            ds_build = create(:ci_build, name: 'retirejs', user: user, status: 'success')
+            ds_bundler_build = create(:ci_build, name: 'bundler-audit', user: user, commit_id: ds_build.pipeline.id, status: 'success')
+            secret_detection_build = create(:ci_build, name: 'secret', user: user, commit_id: ds_build.pipeline.id, status: 'success')
+            cs_build = create(:ci_build, name: 'klar', user: user, status: 'success')
+            sast_build = create(:ci_build, name: 'sast', user: user, status: 'success', retried: true)
+            create(:security_scan, build: ds_build, scan_type: 'dependency_scanning' )
+            create(:security_scan, build: ds_bundler_build, scan_type: 'dependency_scanning')
+            create(:security_scan, build: secret_detection_build, scan_type: 'secret_detection')
+            create(:security_scan, build: cs_build, scan_type: 'container_scanning')
+            create(:security_scan, build: sast_build, scan_type: 'sast')
+          end
+
+          expect(described_class.uncached_data[:usage_activity_by_stage][:secure]).to include(
+            user_preferences_group_overview_security_dashboard: 3,
+            user_container_scanning_jobs: 1,
+            user_dast_jobs: 1,
+            user_dependency_scanning_jobs: 1,
+            user_license_management_jobs: 1,
+            user_sast_jobs: 1,
+            user_secret_detection_jobs: 1,
+            user_unique_users_all_secure_scanners: 1
+          )
+
+          expect(described_class.uncached_data[:usage_activity_by_stage_monthly][:secure]).to include(
+            user_preferences_group_overview_security_dashboard: 3,
+            user_container_scanning_jobs: 1,
+            user_dast_jobs: 1,
+            user_dependency_scanning_jobs: 1,
+            user_license_management_jobs: 1,
+            user_sast_jobs: 1,
+            user_secret_detection_jobs: 1,
+            sast_pipeline: 0,
+            dependency_scanning_pipeline: 1,
+            container_scanning_pipeline: 1,
+            dast_pipeline: 0,
+            secret_detection_pipeline: 1,
+            coverage_fuzzing_pipeline: 0,
             user_unique_users_all_secure_scanners: 1
           )
         end
@@ -535,6 +584,12 @@ RSpec.describe Gitlab::UsageData do
             user_license_management_jobs: 1,
             user_sast_jobs: 2,
             user_secret_detection_jobs: 1,
+            sast_pipeline: 0,
+            dependency_scanning_pipeline: 0,
+            container_scanning_pipeline: 0,
+            dast_pipeline: 0,
+            secret_detection_pipeline: 0,
+            coverage_fuzzing_pipeline: 0,
             user_unique_users_all_secure_scanners: 3
           )
         end
@@ -552,6 +607,12 @@ RSpec.describe Gitlab::UsageData do
             user_license_management_jobs: 2,
             user_sast_jobs: 1,
             user_secret_detection_jobs: 1,
+            sast_pipeline: 0,
+            dependency_scanning_pipeline: 0,
+            container_scanning_pipeline: 0,
+            dast_pipeline: 0,
+            secret_detection_pipeline: 0,
+            coverage_fuzzing_pipeline: 0,
             user_unique_users_all_secure_scanners: 1
           )
         end
@@ -568,6 +629,12 @@ RSpec.describe Gitlab::UsageData do
             user_license_management_jobs: -1,
             user_sast_jobs: -1,
             user_secret_detection_jobs: -1,
+            sast_pipeline: -1,
+            dependency_scanning_pipeline: -1,
+            container_scanning_pipeline: -1,
+            dast_pipeline: -1,
+            secret_detection_pipeline: -1,
+            coverage_fuzzing_pipeline: -1,
             user_unique_users_all_secure_scanners: -1
           )
         end