Merge branch '247130-gather-devops-segment-snapshots-info' into 'master'

Resolve "DevOps Adoption MVC (backend)"

See merge request gitlab-org/gitlab!48684

app/models/approval.rb

 # frozen_string_literal: true
 
 class Approval < ApplicationRecord
+  include CreatedAtFilterable
+
   belongs_to :user
   belongs_to :merge_request
 

app/models/ci/build.rb

@@ -190,6 +190,8 @@ module Ci
 
     scope :with_coverage, -> { where.not(coverage: nil) }
 
+    scope :for_project, -> (project_id) { where(project_id: project_id) }
+
     acts_as_taggable
 
     add_authentication_token_field :token, encrypted: :optional

app/models/ci/pipeline.rb

@@ -287,7 +287,7 @@ module Ci
     scope :for_branch, -> (branch) { for_ref(branch).where(tag: false) }
     scope :for_id, -> (id) { where(id: id) }
     scope :for_iid, -> (iid) { where(iid: iid) }
-    scope :for_project, -> (project) { where(project: project) }
+    scope :for_project, -> (project_id) { where(project_id: project_id) }
     scope :created_after, -> (time) { where('ci_pipelines.created_at > ?', time) }
     scope :created_before_id, -> (id) { where('ci_pipelines.id < ?', id) }
     scope :before_pipeline, -> (pipeline) { created_before_id(pipeline.id).outside_pipeline_family(pipeline) }

app/models/deployment.rb

@@ -37,6 +37,7 @@ class Deployment < ApplicationRecord
   end
 
   scope :for_status, -> (status) { where(status: status) }
+  scope :for_project, -> (project_id) { where(project_id: project_id) }
 
   scope :visible, -> { where(status: %i[running success failed canceled]) }
   scope :stoppable, -> { where.not(on_stop: nil).where.not(deployable_id: nil).success }

changelogs/unreleased/247130-gather-devops-segment-snapshots-info.yml

+---
+title: Add approvals created_at index
+merge_request: 48684
+author:
+type: performance

config/gitlab.yml.example

@@ -475,6 +475,10 @@ production: &base
   # GitLab EE only jobs. These jobs are automatically enabled for an EE
   # installation, and ignored for a CE installation.
   ee_cron_jobs:
+    # Schedule snapshots for all devops adoption segments
+    analytics_devops_adoption_create_all_snapshots_worker:
+      cron: 0 0 1 * *
+
     # Snapshot active users statistics
     historical_data_worker:
       cron: "0 12 * * *"

config/initializers/1_settings.rb

@@ -541,6 +541,9 @@ Settings.cron_jobs['manage_evidence_worker']['cron'] ||= '0 * * * *'
 Settings.cron_jobs['manage_evidence_worker']['job_class'] = 'Releases::ManageEvidenceWorker'
 
 Gitlab.ee do
+  Settings.cron_jobs['analytics_devops_adoption_create_all_snapshots_worker'] ||= Settingslogic.new({})
+  Settings.cron_jobs['analytics_devops_adoption_create_all_snapshots_worker']['cron'] ||= '0 0 1 * *'
+  Settings.cron_jobs['analytics_devops_adoption_create_all_snapshots_worker']['job_class'] = 'Analytics::DevopsAdoption::CreateAllSnapshotsWorker'
   Settings.cron_jobs['active_user_count_threshold_worker'] ||= Settingslogic.new({})
   Settings.cron_jobs['active_user_count_threshold_worker']['cron'] ||= '0 12 * * *'
   Settings.cron_jobs['active_user_count_threshold_worker']['job_class'] = 'ActiveUserCountThresholdWorker'

config/sidekiq_queues.yml

@@ -30,6 +30,8 @@
   - 1
 - - analytics_code_review_metrics
   - 1
+- - analytics_devops_adoption_create_snapshot
+  - 1
 - - analytics_instance_statistics_counter_job
   - 1
 - - approve_blocked_users

db/migrate/20201208143911_add_approvals_created_at_index.rb

+# frozen_string_literal: true
+
+class AddApprovalsCreatedAtIndex < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  INDEX_NAME = 'index_approvals_on_merge_request_id_and_created_at'
+
+  def up
+    add_concurrent_index :approvals, [:merge_request_id, :created_at], name: INDEX_NAME
+  end
+
+  def down
+    remove_concurrent_index_by_name :approvals, INDEX_NAME
+  end
+end

db/schema_migrations/20201208143911

+1e55cafd8b7c5b13514a8709c05d75c8ef0bdd99ea1a5bd3d36f8d20fc0ead2b
\ No newline at end of file

db/structure.sql

@@ -20593,6 +20593,8 @@ CREATE INDEX index_approval_rules_code_owners_rule_type ON approval_merge_reques
 
 CREATE INDEX index_approvals_on_merge_request_id ON approvals USING btree (merge_request_id);
 
+CREATE INDEX index_approvals_on_merge_request_id_and_created_at ON approvals USING btree (merge_request_id, created_at);
+
 CREATE UNIQUE INDEX index_approvals_on_user_id_and_merge_request_id ON approvals USING btree (user_id, merge_request_id);
 
 CREATE INDEX index_approver_groups_on_group_id ON approver_groups USING btree (group_id);

ee/app/models/analytics/devops_adoption/segment.rb

@@ -5,6 +5,7 @@ class Analytics::DevopsAdoption::Segment < ApplicationRecord
 
   has_many :segment_selections
   has_many :groups, through: :segment_selections
+  has_many :projects, through: :segment_selections
   has_many :snapshots, inverse_of: :segment
   has_one :latest_snapshot, -> { order(recorded_at: :desc) }, inverse_of: :segment, class_name: 'Snapshot'
 

ee/app/models/security/scan.rb

@@ -2,6 +2,7 @@
 
 module Security
   class Scan < ApplicationRecord
+    include CreatedAtFilterable
     include IgnorableColumns
 
     self.table_name = 'security_scans'

ee/app/services/analytics/devops_adoption/segments/create_service.rb

@@ -31,7 +31,7 @@ module Analytics
         attr_reader :segment, :params, :current_user
 
         def response_payload
-          { segment: @segment }
+          { segment: segment }
         end
 
         def attributes

ee/app/services/analytics/devops_adoption/snapshots/calculate_and_save_service.rb

+# frozen_string_literal: true
+
+module Analytics
+  module DevopsAdoption
+    module Snapshots
+      class CalculateAndSaveService
+        attr_reader :segment, :range_end
+
+        def initialize(segment:, range_end: Time.zone.now)
+          @segment = segment
+          @range_end = range_end
+        end
+
+        def execute
+          CreateService.new(params: SnapshotCalculator.new(segment: segment, range_end: range_end).calculate).execute
+        end
+      end
+    end
+  end
+end

ee/app/services/analytics/devops_adoption/snapshots/create_service.rb

+# frozen_string_literal: true
+
+module Analytics
+  module DevopsAdoption
+    module Snapshots
+      class CreateService
+        ALLOWED_ATTRIBUTES = [
+          :segment,
+          :segment_id,
+          :recorded_at,
+          :issue_opened,
+          :merge_request_opened,
+          :merge_request_approved,
+          :runner_configured,
+          :pipeline_succeeded,
+          :deploy_succeeded,
+          :security_scan_succeeded
+        ].freeze
+
+        def initialize(snapshot: Analytics::DevopsAdoption::Snapshot.new, params: {})
+          @snapshot = snapshot
+          @params = params
+        end
+
+        def execute
+          success = false
+
+          ActiveRecord::Base.transaction do
+            snapshot.assign_attributes(attributes)
+
+            success = snapshot.save && snapshot.segment.update(last_recorded_at: snapshot.recorded_at)
+
+            raise ActiveRecord::Rollback unless success
+          end
+
+          if success
+            ServiceResponse.success(payload: response_payload)
+          else
+            ServiceResponse.error(message: 'Validation error', payload: response_payload)
+          end
+        end
+
+        private
+
+        attr_reader :snapshot, :params
+
+        def response_payload
+          { snapshot: snapshot }
+        end
+
+        def attributes
+          params.slice(*ALLOWED_ATTRIBUTES)
+        end
+      end
+    end
+  end
+end

ee/app/workers/all_queues.yml

@@ -27,6 +27,14 @@
   :weight: 1
   :idempotent:
   :tags: []
+- :name: cronjob:analytics_devops_adoption_create_all_snapshots
+  :feature_category: :devops_reports
+  :has_external_dependencies:
+  :urgency: :low
+  :resource_boundary: :unknown
+  :weight: 1
+  :idempotent: true
+  :tags: []
 - :name: cronjob:clear_shared_runners_minutes
   :feature_category: :continuous_integration
   :has_external_dependencies:
@@ -637,6 +645,14 @@
   :weight: 1
   :idempotent: true
   :tags: []
+- :name: analytics_devops_adoption_create_snapshot
+  :feature_category: :devops_reports
+  :has_external_dependencies:
+  :urgency: :low
+  :resource_boundary: :unknown
+  :weight: 1
+  :idempotent: true
+  :tags: []
 - :name: approve_blocked_users
   :feature_category: :users
   :has_external_dependencies:

ee/app/workers/analytics/devops_adoption/create_all_snapshots_worker.rb

+# frozen_string_literal: true
+
+module Analytics
+  module DevopsAdoption
+    class CreateAllSnapshotsWorker
+      include ApplicationWorker
+      # This worker does not perform work scoped to a context
+      include CronjobQueue # rubocop:disable Scalability/CronWorkerContext
+
+      feature_category :devops_reports
+      idempotent!
+
+      WORKERS_GAP = 5.seconds
+
+      # rubocop: disable CodeReuse/ActiveRecord
+      def perform
+        range_end = Time.zone.now
+        ::Analytics::DevopsAdoption::Segment.all.pluck(:id).each.with_index do |segment_id, i|
+          CreateSnapshotWorker.perform_in(i * WORKERS_GAP, segment_id, range_end)
+        end
+      end
+      # rubocop: enable CodeReuse/ActiveRecord
+    end
+  end
+end

ee/app/workers/analytics/devops_adoption/create_snapshot_worker.rb

+# frozen_string_literal: true
+
+module Analytics
+  module DevopsAdoption
+    class CreateSnapshotWorker
+      include ApplicationWorker
+
+      feature_category :devops_reports
+      idempotent!
+
+      def perform(segment_id, range_end)
+        segment = ::Analytics::DevopsAdoption::Segment.find(segment_id)
+
+        ::Analytics::DevopsAdoption::Snapshots::CalculateAndSaveService.new(segment: segment, range_end: range_end).execute
+      end
+    end
+  end
+end

ee/lib/analytics/devops_adoption/snapshot_calculator.rb

+# frozen_string_literal: true
+
+module Analytics
+  module DevopsAdoption
+    class SnapshotCalculator
+      attr_reader :segment, :range_end, :range_start
+
+      ADOPTION_FLAGS = %i[issue_opened merge_request_opened merge_request_approved runner_configured pipeline_succeeded deploy_succeeded security_scan_succeeded].freeze
+
+      def initialize(segment:, range_end: Time.zone.now)
+        @segment = segment
+        @range_end = range_end
+        @range_start = Analytics::DevopsAdoption::Snapshot.new(recorded_at: range_end).start_time
+      end
+
+      def calculate
+        params = { recorded_at: range_end, segment: segment }
+
+        ADOPTION_FLAGS.each do |flag|
+          params[flag] = send(flag) # rubocop:disable GitlabSecurity/PublicSend
+        end
+
+        params
+      end
+
+      private
+
+      def snapshot_groups
+        @snapshot_groups ||= Gitlab::ObjectHierarchy.new(segment.groups).base_and_descendants
+      end
+
+      # rubocop: disable CodeReuse/ActiveRecord
+      def snapshot_project_ids
+        @snapshot_project_ids ||= (segment.projects.pluck(:id) + Project.in_namespace(snapshot_groups).pluck(:id)).uniq
+      end
+      # rubocop: enable CodeReuse/ActiveRecord
+
+      def snapshot_merge_requests
+        @snapshot_merge_requests ||= MergeRequest.of_projects(snapshot_project_ids)
+      end
+
+      def issue_opened
+        Issue.in_projects(snapshot_project_ids).created_before(range_end).created_after(range_start).exists?
+      end
+
+      def merge_request_opened
+        snapshot_merge_requests.created_before(range_end).created_after(range_start).exists?
+      end
+
+      # rubocop: disable CodeReuse/ActiveRecord
+      def merge_request_approved
+        Approval.joins(:merge_request).merge(snapshot_merge_requests).created_before(range_end).created_after(range_start).exists?
+      end
+      # rubocop: enable CodeReuse/ActiveRecord
+
+      def runner_configured
+        Ci::Runner.active.belonging_to_group_or_project(snapshot_groups, snapshot_project_ids).exists?
+      end
+
+      def pipeline_succeeded
+        Ci::Pipeline.success.for_project(snapshot_project_ids).updated_before(range_end).updated_after(range_start).exists?
+      end
+
+      def deploy_succeeded
+        Deployment.success.for_project(snapshot_project_ids).updated_before(range_end).updated_after(range_start).exists?
+      end
+
+      # rubocop: disable CodeReuse/ActiveRecord
+      def security_scan_succeeded
+        Security::Scan
+          .joins(:build)
+          .merge(Ci::Build.for_project(snapshot_project_ids))
+          .created_before(range_end)
+          .created_after(range_start)
+          .exists?
+      end
+      # rubocop: enable CodeReuse/ActiveRecord
+    end
+  end
+end

ee/spec/lib/analytics/devops_adoption/snapshot_calculator_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Analytics::DevopsAdoption::SnapshotCalculator do
+  subject(:data) { described_class.new(segment: segment).calculate }
+
+  let_it_be(:group1) { create(:group) }
+  let_it_be(:project2) { create(:project) }
+  let_it_be(:segment) { create(:devops_adoption_segment, groups: [group1], projects: [project2]) }
+  let_it_be(:subgroup) { create(:group, parent: group1) }
+  let_it_be(:project) { create(:project, group: group1) }
+  let_it_be(:subproject) { create(:project, group: subgroup) }
+
+  describe 'issue_opened' do
+    subject { data[:issue_opened] }
+
+    let_it_be(:old_issue) { create(:issue, project: subproject, created_at: 1.year.ago) }
+
+    context 'with an issue opened within 30 days' do
+      let_it_be(:fresh_issue) { create(:issue, project: project2, created_at: 3.weeks.ago) }
+
+      it { is_expected.to eq true }
+    end
+
+    it { is_expected.to eq false }
+  end
+
+  describe 'merge_request_opened' do
+    subject { data[:merge_request_opened] }
+
+    let!(:old_merge_request) { create(:merge_request, source_project: subproject, created_at: 1.year.ago) }
+
+    context 'with a merge request opened within 30 days' do
+      let!(:fresh_merge_request) { create(:merge_request, source_project: project2, created_at: 3.weeks.ago) }
+
+      it { is_expected.to eq true }
+    end
+
+    it { is_expected.to eq false }
+  end
+
+  describe 'merge_request_approved' do
+    subject { data[:merge_request_approved] }
+
+    let!(:old_merge_request) { create(:merge_request, source_project: subproject, created_at: 1.year.ago) }
+    let!(:old_approval) { create(:approval, merge_request: old_merge_request, created_at: 6.months.ago) }
+
+    context 'with a merge request approved within 30 days' do
+      let!(:fresh_approval) { create(:approval, merge_request: old_merge_request, created_at: 3.weeks.ago) }
+
+      it { is_expected.to eq true }
+    end
+
+    it { is_expected.to eq false }
+  end
+
+  describe 'runner_configured' do
+    subject { data[:runner_configured] }
+
+    let!(:inactive_runner) { create(:ci_runner, :project, active: false) }
+    let!(:ci_runner_project) { create(:ci_runner_project, project: project, runner: inactive_runner )}
+
+    context 'with active runner present' do
+      let!(:active_runner) { create(:ci_runner, :project, active: true) }
+      let!(:ci_runner_project) { create(:ci_runner_project, project: subproject, runner: active_runner )}
+
+      it { is_expected.to eq true }
+    end
+
+    it { is_expected.to eq false }
+  end
+
+  describe 'pipeline_succeeded' do
+    subject { data[:pipeline_succeeded] }
+
+    let!(:failed_pipeline) { create(:ci_pipeline, :failed, project: project2, updated_at: 1.day.ago) }
+    let!(:old_pipeline) { create(:ci_pipeline, :success, project: project2, updated_at: 100.days.ago) }
+
+    context 'with successful pipeline in last 30 days' do
+      let!(:fresh_pipeline) { create(:ci_pipeline, :success, project: project2, updated_at: 1.week.ago) }
+
+      it { is_expected.to eq true }
+    end
+
+    it { is_expected.to eq false }
+  end
+
+  describe 'deploy_succeeded' do
+    subject { data[:deploy_succeeded] }
+
+    let!(:old_deployment) { create(:deployment, :success, updated_at: 100.days.ago) }
+    let!(:old_group) do
+      create(:group).tap do |g|
+        g.projects << old_deployment.project
+      end
+    end
+
+    let(:segment) { create(:devops_adoption_segment, groups: [old_group]) }
+
+    context 'with any deployment in last 30 days' do
+      let!(:fresh_deployment) { create(:deployment, :success, updated_at: 1.day.ago) }
+      let!(:fresh_group) do
+        create(:group).tap do |g|
+          g.projects << fresh_deployment.project
+        end
+      end
+
+      let(:segment) { create(:devops_adoption_segment, groups: [old_group, fresh_group]) }
+
+      it { is_expected.to eq true }
+    end
+
+    it { is_expected.to eq false }
+  end
+
+  describe 'security_scan_succeeded' do
+    subject { data[:security_scan_succeeded] }
+
+    let!(:old_security_scan) { create :security_scan, build: create(:ci_build, project: project2), created_at: 100.days.ago }
+
+    context 'with successful security scan in last 30 days' do
+      let!(:fresh_security_scan) { create :security_scan, build: create(:ci_build, project: project2), created_at: 10.days.ago }
+
+      it { is_expected.to eq true }
+    end
+
+    it { is_expected.to eq false }
+  end
+end

ee/spec/services/analytics/devops_adoption/snapshots/calculate_and_save_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Analytics::DevopsAdoption::Snapshots::CalculateAndSaveService do
+  let(:segment_mock) { instance_double('Analytics::DevopsAdoption::Segment') }
+
+  subject { described_class.new(segment: segment_mock) }
+
+  it 'creates a snapshot with whatever snapshot calculator returns' do
+    allow_next_instance_of(Analytics::DevopsAdoption::SnapshotCalculator) do |calc|
+      allow(calc).to receive(:calculate).and_return('calculated_data')
+    end
+
+    expect_next_instance_of(Analytics::DevopsAdoption::Snapshots::CreateService, params: 'calculated_data') do |service|
+      expect(service).to receive(:execute).and_return('create_service_response')
+    end
+
+    expect(subject.execute).to eq('create_service_response')
+  end
+end

ee/spec/services/analytics/devops_adoption/snapshots/create_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Analytics::DevopsAdoption::Snapshots::CreateService do
+  subject(:service_response) { described_class.new(params: params).execute }
+
+  let(:snapshot) { service_response.payload[:snapshot] }
+  let(:params) do
+    params = Analytics::DevopsAdoption::SnapshotCalculator::ADOPTION_FLAGS.each_with_object({}) do |attribute, result|
+      result[attribute] = rand(2).odd?
+    end
+    params[:recorded_at] = Time.zone.now
+    params[:segment] = segment
+    params
+  end
+
+  let(:segment) { create(:devops_adoption_segment, last_recorded_at: 1.year.ago) }
+
+  it 'persists the snapshot & updates segment last recorded at date' do
+    expect(subject).to be_success
+    expect(snapshot).to have_attributes(params)
+    expect(snapshot.segment.reload.last_recorded_at).to be_like_time(snapshot.recorded_at)
+  end
+
+  context 'when params are invalid' do
+    let(:params) { super().merge(recorded_at: nil) }
+
+    it 'does not persist the snapshot' do
+      expect(subject).to be_error
+      expect(subject.message).to eq('Validation error')
+      expect(snapshot).not_to be_persisted
+      expect(snapshot.segment.reload.last_recorded_at).not_to eq nil
+    end
+  end
+end

ee/spec/workers/analytics/devops_adoption/create_all_snapshots_worker_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Analytics::DevopsAdoption::CreateAllSnapshotsWorker do
+  subject(:worker) { described_class.new }
+
+  describe "#perform" do
+    let!(:segment1) { create :devops_adoption_segment }
+    let!(:segment2) { create :devops_adoption_segment }
+
+    it 'schedules workers for each individual segment' do
+      freeze_time do
+        expect(Analytics::DevopsAdoption::CreateSnapshotWorker).to receive(:perform_in).with(0, segment1.id, Time.zone.now)
+        expect(Analytics::DevopsAdoption::CreateSnapshotWorker).to receive(:perform_in).with(5, segment2.id, Time.zone.now)
+
+        worker.perform
+      end
+    end
+  end
+end

ee/spec/workers/analytics/devops_adoption/create_snapshot_worker_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Analytics::DevopsAdoption::CreateSnapshotWorker do
+  subject(:worker) { described_class.new }
+
+  describe "#perform" do
+    let!(:segment) { create :devops_adoption_segment }
+    let!(:range_end) { 1.day.ago }
+
+    it 'calls for Analytics::DevopsAdoption::Snapshots::CalculateAndSaveService service' do
+      expect_next_instance_of(::Analytics::DevopsAdoption::Snapshots::CalculateAndSaveService, segment: segment, range_end: range_end) do |instance|
+        expect(instance).to receive(:execute)
+      end
+
+      worker.perform(segment.id, range_end)
+    end
+  end
+end

spec/factories/merge_requests.rb

@@ -268,7 +268,7 @@ FactoryBot.define do
       source_project = merge_request.source_project
 
       # Fake `fetch_ref!` if we don't have repository
-      # We have too many existing tests replying on this behaviour
+      # We have too many existing tests relying on this behaviour
       unless [target_project, source_project].all?(&:repository_exists?)
         allow(merge_request).to receive(:fetch_ref!)
       end