Merge branch 'trace-parser' into 'master'

Parse requirements reports

See merge request gitlab-org/gitlab!33031

app/models/ci/job_artifact.rb

@@ -39,7 +39,8 @@ module Ci
       dotenv: '.env',
       cobertura: 'cobertura-coverage.xml',
       terraform: 'tfplan.json',
-      cluster_applications: 'gl-cluster-applications.json'
+      cluster_applications: 'gl-cluster-applications.json',
+      requirements: 'requirements.json'
     }.freeze
 
     INTERNAL_TYPES = {
@@ -71,7 +72,8 @@ module Ci
       license_management: :raw,
       license_scanning: :raw,
       performance: :raw,
-      terraform: :raw
+      terraform: :raw,
+      requirements: :raw
     }.freeze
 
     DOWNLOADABLE_TYPES = %w[
@@ -90,6 +92,7 @@ module Ci
       metrics
       performance
       sast
+      requirements
     ].freeze
 
     TYPE_AND_FORMAT_PAIRS = INTERNAL_TYPES.merge(REPORT_TYPES).freeze
@@ -182,7 +185,8 @@ module Ci
       terraform: 18, # Transformed json
       accessibility: 19,
       cluster_applications: 20,
-      secret_detection: 21 ## EE-specific
+      secret_detection: 21, ## EE-specific
+      requirements: 22 ## EE-specific
     }
 
     enum file_format: {

config/sidekiq_queues.yml

@@ -242,6 +242,8 @@
   - 1
 - - repository_update_remote_mirror
   - 1
+- - requirements_management_process_requirements_reports
+  - 1
 - - security_scans
   - 2
 - - self_monitoring_project_create

ee/app/models/ee/ci/build.rb

@@ -116,6 +116,16 @@ module EE
         metrics_report
       end
 
+      def collect_requirements_reports!(requirements_report)
+        return requirements_report unless project.feature_available?(:requirements)
+
+        each_report(::Ci::JobArtifact::REQUIREMENTS_REPORT_FILE_TYPES) do |file_type, blob, report_artifact|
+          ::Gitlab::Ci::Parsers.fabricate!(file_type).parse!(blob, requirements_report)
+        end
+
+        requirements_report
+      end
+
       def retryable?
         !merge_train_pipeline? && super
       end

ee/app/models/ee/ci/job_artifact.rb

@@ -19,6 +19,7 @@ module EE
       SAST_REPORT_TYPES = %w[sast].freeze
       SECRET_DETECTION_REPORT_TYPES = %w[secret_detection].freeze
       DAST_REPORT_TYPES = %w[dast].freeze
+      REQUIREMENTS_REPORT_FILE_TYPES = %w[requirements].freeze
 
       scope :project_id_in, ->(ids) { where(project_id: ids) }
       scope :with_files_stored_remotely, -> { where(file_store: ::JobArtifactUploader::Store::REMOTE) }

ee/app/models/ee/ci/pipeline.rb

@@ -47,7 +47,8 @@ module EE
           performance: %i[merge_request_performance_metrics],
           license_management: %i[license_scanning],
           license_scanning: %i[license_scanning],
-          metrics: %i[metrics_reports]
+          metrics: %i[metrics_reports],
+          requirements: %i[requirements]
         }.freeze
 
         state_machine :status do

ee/app/models/requirements_management/test_report.rb

@@ -3,6 +3,7 @@
 module RequirementsManagement
   class TestReport < ApplicationRecord
     include Sortable
+    include BulkInsertSafe
 
     belongs_to :requirement, inverse_of: :test_reports
     belongs_to :author, inverse_of: :test_reports, class_name: 'User'
@@ -14,6 +15,27 @@ module RequirementsManagement
 
     enum state: { passed: 1 }
 
+    scope :for_user_build, ->(user_id, build_id) { where(author_id: user_id, build_id: build_id) }
+
+    def self.persist_all_requirement_reports_as_passed(build)
+      reports = []
+      timestamp = Time.current
+      build.project.requirements.opened.select(:id).find_each do |requirement|
+        reports << new(
+          requirement_id: requirement.id,
+          # pipeline_reference will be removed:
+          # https://gitlab.com/gitlab-org/gitlab/-/issues/219999
+          pipeline_id: build.pipeline_id,
+          build_id: build.id,
+          author_id: build.user_id,
+          created_at: timestamp,
+          state: :passed
+        )
+      end
+
+      bulk_insert!(reports)
+    end
+
     private
 
     def validate_pipeline_reference

ee/app/policies/ee/project_policy.rb

@@ -396,6 +396,7 @@ module EE
 
       rule { requirements_available & reporter }.policy do
         enable :create_requirement
+        enable :create_requirement_test_report
         enable :admin_requirement
         enable :update_requirement
       end

ee/app/services/requirements_management/process_test_reports_service.rb

+# frozen_string_literal: true
+
+# This service collects all requirements reports from the CI job and creates a
+# series of test report resources, one for each open requirement
+
+module RequirementsManagement
+  class ProcessTestReportsService < BaseService
+    include Gitlab::Allowable
+
+    def initialize(build)
+      @build = build
+    end
+
+    def execute
+      return if test_report_already_generated?
+      return unless report.all_passed?
+
+      raise Gitlab::Access::AccessDeniedError unless can?(@build.user, :create_requirement_test_report, @build.project)
+
+      RequirementsManagement::TestReport.persist_all_requirement_reports_as_passed(@build)
+    end
+
+    private
+
+    def test_report_already_generated?
+      RequirementsManagement::TestReport.for_user_build(@build.user_id, @build.id).exists?
+    end
+
+    def report
+      ::Gitlab::Ci::Reports::RequirementsManagement::Report.new.tap do |report|
+        @build.collect_requirements_reports!(report)
+      end
+    end
+  end
+end

ee/app/workers/all_queues.yml

@@ -643,6 +643,14 @@
   :weight: 1
   :idempotent: 
   :tags: []
+- :name: requirements_management_process_requirements_reports
+  :feature_category: :requirements_management
+  :has_external_dependencies: 
+  :urgency: :low
+  :resource_boundary: :unknown
+  :weight: 1
+  :idempotent: true
+  :tags: []
 - :name: service_desk_email_receiver
   :feature_category: :issue_tracking
   :has_external_dependencies: 

ee/app/workers/ee/build_finished_worker.rb

@@ -9,6 +9,7 @@ module EE
       ::Ci::Minutes::EmailNotificationService.new(build.project.reset).execute if ::Gitlab.com?
 
       StoreSecurityScansWorker.perform_async(build.id)
+      RequirementsManagement::ProcessRequirementsReportsWorker.perform_async(build.id)
 
       super
     end

ee/app/workers/requirements_management/process_requirements_reports_worker.rb

+# frozen_string_literal: true
+
+module RequirementsManagement
+  class ProcessRequirementsReportsWorker
+    include ApplicationWorker
+
+    feature_category :requirements_management
+    idempotent!
+
+    def perform(build_id)
+      ::Ci::Build.find_by_id(build_id).try do |build|
+        RequirementsManagement::ProcessTestReportsService.new(build).execute
+      end
+    end
+  end
+end

ee/changelogs/unreleased/trace-parser.yml

+---
+title: Added CI parser for requirements reports
+merge_request: 33031
+author:
+type: added

ee/lib/ee/gitlab/ci/parsers.rb

@@ -16,7 +16,8 @@ module EE
                 dast: ::Gitlab::Ci::Parsers::Security::Dast,
                 sast: ::Gitlab::Ci::Parsers::Security::Sast,
                 secret_detection: ::Gitlab::Ci::Parsers::Security::SecretDetection,
-                metrics: ::Gitlab::Ci::Parsers::Metrics::Generic
+                metrics: ::Gitlab::Ci::Parsers::Metrics::Generic,
+                requirements: ::Gitlab::Ci::Parsers::RequirementsManagement::Requirement
             })
           end
         end

ee/lib/gitlab/ci/parsers/requirements_management/requirement.rb

+# frozen_string_literal: true
+module Gitlab
+  module Ci
+    module Parsers
+      module RequirementsManagement
+        class Requirement
+          RequirementParserError = Class.new(Gitlab::Ci::Parsers::ParserError)
+
+          def parse!(json_data, report)
+            result = Gitlab::Json.parse!(json_data)
+            raise RequirementParserError, 'Invalid report format' unless result.is_a?(Hash)
+
+            result.each { |ref, state| report.add_requirement(ref, state) }
+          rescue JSON::ParserError
+            raise RequirementParserError, 'JSON parsing failed'
+          end
+        end
+      end
+    end
+  end
+end

ee/lib/gitlab/ci/reports/requirements_management/report.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Reports
+      module RequirementsManagement
+        class Report
+          attr_reader :requirements
+
+          def initialize
+            @requirements = {}
+          end
+
+          def add_requirement(key, value)
+            @requirements[key] = value
+          end
+
+          def all_passed?
+            @requirements['*'] == 'passed'
+          end
+        end
+      end
+    end
+  end
+end

ee/spec/factories/ci/builds.rb

@@ -121,5 +121,11 @@ FactoryBot.define do
         end
       end
     end
+
+    trait :requirements_report do
+      after(:build) do |build|
+        build.job_artifacts << create(:ee_ci_job_artifact, :requirements, job: build)
+      end
+    end
   end
 end

ee/spec/factories/ci/job_artifacts.rb

@@ -332,5 +332,15 @@ FactoryBot.define do
         artifact.file = fixture_file_upload(path, 'application/json')
       end
     end
+
+    trait :requirements do
+      file_format { :raw }
+      file_type { :requirements }
+
+      after(:build) do |artifact, _|
+        artifact.file = fixture_file_upload(
+          Rails.root.join('ee/spec/fixtures/requirements_management/report.json'), 'application/json')
+      end
+    end
   end
 end

ee/spec/fixtures/requirements_management/report.json

+{
+  "*": "passed"
+}

ee/spec/models/ci/build_spec.rb

@@ -407,6 +407,40 @@ RSpec.describe Ci::Build do
     end
   end
 
+  describe '#collect_requirements_reports!' do
+    subject { job.collect_requirements_reports!(requirements_report) }
+
+    let(:requirements_report) { Gitlab::Ci::Reports::RequirementsManagement::Report.new }
+
+    context 'when there is a requirements report' do
+      before do
+        create(:ee_ci_job_artifact, :requirements, job: job, project: job.project)
+      end
+
+      context 'when requirements are available' do
+        before do
+          stub_licensed_features(requirements: true)
+        end
+
+        it 'parses blobs and adds the results to the report' do
+          expect { subject }.to change { requirements_report.requirements.count }.from(0).to(1)
+        end
+      end
+
+      context 'when requirements are not available' do
+        before do
+          stub_licensed_features(requirements: false)
+        end
+
+        it 'does not parse requirements report' do
+          subject
+
+          expect(requirements_report.requirements.count).to eq(0)
+        end
+      end
+    end
+  end
+
   describe '#retryable?' do
     subject { build.retryable? }
 

ee/spec/models/requirements_management/test_report_spec.rb

@@ -35,4 +35,41 @@ RSpec.describe RequirementsManagement::TestReport do
       end
     end
   end
+
+  describe 'scopes' do
+    describe 'for_user_build' do
+      it "returns only test reports matching build's user and pipeline" do
+        user = create(:user)
+        build = create(:ci_build)
+        report1 = create(:test_report, author: user, build: build)
+        create(:test_report, author: user)
+        create(:test_report, build: build)
+
+        expect(described_class.for_user_build(user.id, build.id)).to match_array([report1])
+      end
+    end
+  end
+
+  describe '.persist_all_requirement_reports_as_passed' do
+    let_it_be(:project) { create(:project) }
+    let_it_be(:build) { create(:ee_ci_build, :requirements_report, project: project) }
+
+    subject { described_class.persist_all_requirement_reports_as_passed(build) }
+
+    it 'creates test report with passed status for each open requirement' do
+      requirement = create(:requirement, state: :opened, project: project)
+      create(:requirement, state: :opened)
+      create(:requirement, state: :archived, project: project)
+
+      expect { subject }.to change { RequirementsManagement::TestReport.count }.by(1)
+
+      reports = RequirementsManagement::TestReport.where(pipeline: build.pipeline)
+      expect(reports.size).to eq(1)
+      expect(reports.first).to have_attributes(
+        requirement: requirement,
+        author: build.user,
+        state: 'passed'
+      )
+    end
+  end
 end

ee/spec/services/requirements_management/process_test_reports_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe RequirementsManagement::ProcessTestReportsService do
+  let_it_be(:project) { create(:project) }
+  let_it_be(:user) { create(:user) }
+  let_it_be(:build) { create(:ee_ci_build, :requirements_report, project: project, user: user) }
+
+  describe '#execute' do
+    let_it_be(:requirement1) { create(:requirement, state: :opened, project: project) }
+    let_it_be(:requirement2) { create(:requirement, state: :opened, project: project) }
+    let_it_be(:requirement3) { create(:requirement, state: :archived, project: project) }
+
+    subject { described_class.new(build).execute }
+
+    before do
+      stub_licensed_features(requirements: true)
+    end
+
+    context 'when user can create requirements test reports' do
+      before do
+        project.add_reporter(user)
+      end
+
+      it 'creates new test report for each open requirement' do
+        expect(RequirementsManagement::TestReport).to receive(:persist_all_requirement_reports_as_passed).with(build).and_call_original
+
+        expect { subject }.to change { RequirementsManagement::TestReport.count }.by(2)
+      end
+
+      it 'does not create test report for the same pipeline and user twice' do
+        expect { subject }.to change { RequirementsManagement::TestReport.count }.by(2)
+
+        expect { subject }.not_to change { RequirementsManagement::TestReport }
+      end
+
+      context 'when build does not contain any requirements report' do
+        let(:build) { create(:ee_ci_build, project: project, user: user) }
+
+        it 'does not create any test report' do
+          expect { subject }.not_to change { RequirementsManagement::TestReport }
+        end
+      end
+    end
+
+    context 'when user is not allowed to create requirements test reports' do
+      it 'raises an exception' do
+        expect { subject }.to raise_exception(Gitlab::Access::AccessDeniedError)
+      end
+    end
+  end
+end

ee/spec/support/shared_examples/policies/requirement_policy_shared_examples.rb

 # frozen_string_literal: true
 
 RSpec.shared_examples 'resource with requirement permissions' do
-  let(:all_permissions) { [:read_requirement, :create_requirement, :admin_requirement, :update_requirement, :destroy_requirement] }
+  let(:all_permissions) do
+    [:read_requirement, :create_requirement, :admin_requirement,
+     :update_requirement, :destroy_requirement,
+     :create_requirement_test_report]
+  end
+
   let(:manage_permissions) { all_permissions - [:destroy_requirement] }
   let(:non_read_permissions) { all_permissions - [:read_requirement] }
 

ee/spec/workers/build_finished_worker_spec.rb

@@ -62,5 +62,11 @@ RSpec.describe BuildFinishedWorker do
         subject
       end
     end
+
+    it 'processes requirements reports' do
+      expect(RequirementsManagement::ProcessRequirementsReportsWorker).to receive(:perform_async)
+
+      subject
+    end
   end
 end

ee/spec/workers/requirements_management/process_requirements_reports_worker_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe RequirementsManagement::ProcessRequirementsReportsWorker do
+  describe '#perform' do
+    context 'build exists' do
+      let(:build) { create(:ci_build) }
+
+      it 'processes requirements reports' do
+        service_double = instance_double(RequirementsManagement::ProcessTestReportsService, execute: true)
+        expect(RequirementsManagement::ProcessTestReportsService).to receive(:new).and_return(service_double)
+
+        described_class.new.perform(build.id)
+      end
+    end
+
+    context 'build does not exist' do
+      it 'does not store requirements reports' do
+        expect(RequirementsManagement::ProcessTestReportsService).not_to receive(:new)
+
+        described_class.new.perform(non_existing_record_id)
+      end
+    end
+  end
+end

lib/gitlab/ci/config/entry/reports.rb

@@ -14,7 +14,8 @@ module Gitlab
           ALLOWED_KEYS =
             %i[junit codequality sast secret_detection dependency_scanning container_scanning
                dast performance license_management license_scanning metrics lsif
-               dotenv cobertura terraform accessibility cluster_applications].freeze
+               dotenv cobertura terraform accessibility cluster_applications
+               requirements].freeze
 
           attributes ALLOWED_KEYS
 
@@ -40,6 +41,7 @@ module Gitlab
               validates :terraform, array_of_strings_or_string: true
               validates :accessibility, array_of_strings_or_string: true
               validates :cluster_applications, array_of_strings_or_string: true
+              validates :requirements, array_of_strings_or_string: true
             end
           end
 

spec/services/ci/retry_build_service_spec.rb

@@ -38,7 +38,8 @@ describe Ci::RetryBuildService do
        job_artifacts_codequality job_artifacts_metrics scheduled_at
        job_variables waiting_for_resource_at job_artifacts_metrics_referee
        job_artifacts_network_referee job_artifacts_dotenv
-       job_artifacts_cobertura needs job_artifacts_accessibility].freeze
+       job_artifacts_cobertura needs job_artifacts_accessibility
+       job_artifacts_requirements].freeze
 
   ignore_accessors =
     %i[type lock_version target_url base_tags trace_sections