Merge branch 'new-cycle-analytics-backend-events-with-query-customization' into 'master'

Cycle Analytics implement event query customization

See merge request gitlab-org/gitlab!16895

lib/gitlab/analytics/cycle_analytics/stage_events.rb

@@ -18,7 +18,8 @@ module Gitlab
           StageEvents::MergeRequestMerged => 104,
           StageEvents::CodeStageStart => 1_000,
           StageEvents::IssueStageEnd => 1_001,
-          StageEvents::PlanStageStart => 1_002
+          StageEvents::PlanStageStart => 1_002,
+          StageEvents::ProductionStageEnd => 1_003
         }.freeze
 
         EVENTS = ENUM_MAPPING.keys.freeze
@@ -32,7 +33,8 @@ module Gitlab
             StageEvents::MergeRequestCreated
           ],
           StageEvents::IssueCreated => [
-            StageEvents::IssueStageEnd
+            StageEvents::IssueStageEnd,
+            StageEvents::ProductionStageEnd
           ],
           StageEvents::MergeRequestCreated => [
             StageEvents::MergeRequestMerged

lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb

@@ -16,6 +16,21 @@ module Gitlab
           def object_type
             MergeRequest
           end
+
+          def timestamp_projection
+            issue_metrics_table[:first_mentioned_in_commit_at]
+          end
+
+          # rubocop: disable CodeReuse/ActiveRecord
+          def apply_query_customization(query)
+            issue_metrics_join = mr_closing_issues_table
+              .join(issue_metrics_table)
+              .on(mr_closing_issues_table[:issue_id].eq(issue_metrics_table[:issue_id]))
+              .join_sources
+
+            query.joins(:merge_requests_closing_issues).joins(issue_metrics_join)
+          end
+          # rubocop: enable CodeReuse/ActiveRecord
         end
       end
     end

lib/gitlab/analytics/cycle_analytics/stage_events/issue_created.rb

@@ -16,6 +16,10 @@ module Gitlab
           def object_type
             Issue
           end
+
+          def timestamp_projection
+            issue_table[:created_at]
+          end
         end
       end
     end

lib/gitlab/analytics/cycle_analytics/stage_events/issue_first_mentioned_in_commit.rb

@@ -16,6 +16,16 @@ module Gitlab
           def object_type
             Issue
           end
+
+          def timestamp_projection
+            issue_metrics_table[:first_mentioned_in_commit_at]
+          end
+
+          # rubocop: disable CodeReuse/ActiveRecord
+          def apply_query_customization(query)
+            query.joins(:metrics)
+          end
+          # rubocop: enable CodeReuse/ActiveRecord
         end
       end
     end

lib/gitlab/analytics/cycle_analytics/stage_events/issue_stage_end.rb

@@ -16,6 +16,19 @@ module Gitlab
           def object_type
             Issue
           end
+
+          def timestamp_projection
+            Arel::Nodes::NamedFunction.new('COALESCE', [
+              issue_metrics_table[:first_associated_with_milestone_at],
+              issue_metrics_table[:first_added_to_board_at]
+            ])
+          end
+
+          # rubocop: disable CodeReuse/ActiveRecord
+          def apply_query_customization(query)
+            query.joins(:metrics).where(issue_metrics_table[:first_added_to_board_at].not_eq(nil).or(issue_metrics_table[:first_associated_with_milestone_at].not_eq(nil)))
+          end
+          # rubocop: enable CodeReuse/ActiveRecord
         end
       end
     end

lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_created.rb

@@ -16,6 +16,10 @@ module Gitlab
           def object_type
             MergeRequest
           end
+
+          def timestamp_projection
+            mr_table[:created_at]
+          end
         end
       end
     end

lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_first_deployed_to_production.rb

@@ -16,6 +16,16 @@ module Gitlab
           def object_type
             MergeRequest
           end
+
+          def timestamp_projection
+            mr_metrics_table[:first_deployed_to_production_at]
+          end
+
+          # rubocop: disable CodeReuse/ActiveRecord
+          def apply_query_customization(query)
+            query.joins(:metrics).where(timestamp_projection.gteq(mr_table[:created_at]))
+          end
+          # rubocop: enable CodeReuse/ActiveRecord
         end
       end
     end

lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_finished.rb

@@ -16,6 +16,16 @@ module Gitlab
           def object_type
             MergeRequest
           end
+
+          def timestamp_projection
+            mr_metrics_table[:latest_build_finished_at]
+          end
+
+          # rubocop: disable CodeReuse/ActiveRecord
+          def apply_query_customization(query)
+            query.joins(:metrics)
+          end
+          # rubocop: enable CodeReuse/ActiveRecord
         end
       end
     end

lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_started.rb

@@ -16,6 +16,16 @@ module Gitlab
           def object_type
             MergeRequest
           end
+
+          def timestamp_projection
+            mr_metrics_table[:latest_build_started_at]
+          end
+
+          # rubocop: disable CodeReuse/ActiveRecord
+          def apply_query_customization(query)
+            query.joins(:metrics)
+          end
+          # rubocop: enable CodeReuse/ActiveRecord
         end
       end
     end

lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_merged.rb

@@ -16,6 +16,16 @@ module Gitlab
           def object_type
             MergeRequest
           end
+
+          def timestamp_projection
+            mr_metrics_table[:merged_at]
+          end
+
+          # rubocop: disable CodeReuse/ActiveRecord
+          def apply_query_customization(query)
+            query.joins(:metrics)
+          end
+          # rubocop: enable CodeReuse/ActiveRecord
         end
       end
     end

lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb

@@ -16,6 +16,22 @@ module Gitlab
           def object_type
             Issue
           end
+
+          def timestamp_projection
+            Arel::Nodes::NamedFunction.new('COALESCE', [
+              issue_metrics_table[:first_associated_with_milestone_at],
+              issue_metrics_table[:first_added_to_board_at]
+            ])
+          end
+
+          # rubocop: disable CodeReuse/ActiveRecord
+          def apply_query_customization(query)
+            query
+              .joins(:metrics)
+              .where(issue_metrics_table[:first_added_to_board_at].not_eq(nil).or(issue_metrics_table[:first_associated_with_milestone_at].not_eq(nil)))
+              .where(issue_metrics_table[:first_mentioned_in_commit_at].not_eq(nil))
+          end
+          # rubocop: enable CodeReuse/ActiveRecord
         end
       end
     end

lib/gitlab/analytics/cycle_analytics/stage_events/production_stage_end.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Analytics
+    module CycleAnalytics
+      module StageEvents
+        class ProductionStageEnd < SimpleStageEvent
+          def self.name
+            PlanStageStart.name
+          end
+
+          def self.identifier
+            :production_stage_end
+          end
+
+          def object_type
+            Issue
+          end
+
+          def timestamp_projection
+            mr_metrics_table[:first_deployed_to_production_at]
+          end
+
+          # rubocop: disable CodeReuse/ActiveRecord
+          def apply_query_customization(query)
+            query.joins(merge_requests_closing_issues: { merge_request: [:metrics] }).where(mr_metrics_table[:first_deployed_to_production_at].gteq(mr_table[:created_at]))
+          end
+          # rubocop: enable CodeReuse/ActiveRecord
+        end
+      end
+    end
+  end
+end

lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb

@@ -6,6 +6,8 @@ module Gitlab
       module StageEvents
         # Base class for expressing an event that can be used for a stage.
         class StageEvent
+          include Gitlab::CycleAnalytics::MetricsTables
+
           def initialize(params)
             @params = params
           end
@@ -21,6 +23,21 @@ module Gitlab
           def object_type
             raise NotImplementedError
           end
+
+          # Each StageEvent must expose a timestamp or a timestamp like expression in order to build a range query.
+          # Example: get me all the Issue records between start event end end event
+          def timestamp_projection
+            raise NotImplementedError
+          end
+
+          # Optionally a StageEvent may apply additional filtering or join other tables on the base query.
+          def apply_query_customization(query)
+            query
+          end
+
+          private
+
+          attr_reader :params
         end
       end
     end

spec/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::Analytics::CycleAnalytics::StageEvents::CodeStageStart do
+  let(:subject) { described_class.new({}) }
+  let(:project) { create(:project) }
+
+  it_behaves_like 'cycle analytics event'
+
+  it 'needs connection with an issue via merge_requests_closing_issues table' do
+    issue = create(:issue, project: project)
+    merge_request = create(:merge_request, source_project: project)
+    create(:merge_requests_closing_issues, issue: issue, merge_request: merge_request)
+
+    other_merge_request = create(:merge_request, source_project: project, source_branch: 'a', target_branch: 'master')
+
+    records = subject.apply_query_customization(MergeRequest.all)
+    expect(records).to eq([merge_request])
+    expect(records).not_to include(other_merge_request)
+  end
+end

spec/lib/gitlab/analytics/cycle_analytics/stage_events/issue_created_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::Analytics::CycleAnalytics::StageEvents::IssueCreated do
+  it_behaves_like 'cycle analytics event'
+end

spec/lib/gitlab/analytics/cycle_analytics/stage_events/issue_first_mentioned_in_commit_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::Analytics::CycleAnalytics::StageEvents::IssueFirstMentionedInCommit do
+  it_behaves_like 'cycle analytics event'
+end

spec/lib/gitlab/analytics/cycle_analytics/stage_events/issue_stage_end_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::Analytics::CycleAnalytics::StageEvents::IssueStageEnd do
+  it_behaves_like 'cycle analytics event'
+end

spec/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_created_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::Analytics::CycleAnalytics::StageEvents::MergeRequestCreated do
+  it_behaves_like 'cycle analytics event'
+end

spec/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_first_deployed_to_production_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::Analytics::CycleAnalytics::StageEvents::MergeRequestFirstDeployedToProduction do
+  it_behaves_like 'cycle analytics event'
+end

spec/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_finished_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::Analytics::CycleAnalytics::StageEvents::MergeRequestLastBuildFinished do
+  it_behaves_like 'cycle analytics event'
+end

spec/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_last_build_started_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::Analytics::CycleAnalytics::StageEvents::MergeRequestLastBuildStarted do
+  it_behaves_like 'cycle analytics event'
+end

spec/lib/gitlab/analytics/cycle_analytics/stage_events/merge_request_merged_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::Analytics::CycleAnalytics::StageEvents::MergeRequestMerged do
+  it_behaves_like 'cycle analytics event'
+end

spec/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::Analytics::CycleAnalytics::StageEvents::PlanStageStart do
+  let(:subject) { described_class.new({}) }
+  let(:project) { create(:project) }
+
+  it_behaves_like 'cycle analytics event'
+
+  it 'filters issues where first_associated_with_milestone_at or first_added_to_board_at is filled' do
+    issue1 = create(:issue, project: project)
+    issue1.metrics.update!(first_added_to_board_at: 1.month.ago, first_mentioned_in_commit_at: 2.months.ago)
+
+    issue2 = create(:issue, project: project)
+    issue2.metrics.update!(first_associated_with_milestone_at: 1.month.ago, first_mentioned_in_commit_at: 2.months.ago)
+
+    issue_without_metrics = create(:issue, project: project)
+
+    records = subject.apply_query_customization(Issue.all)
+    expect(records).to match_array([issue1, issue2])
+    expect(records).not_to include(issue_without_metrics)
+  end
+end

spec/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event_spec.rb

@@ -3,8 +3,11 @@
 require 'spec_helper'
 
 describe Gitlab::Analytics::CycleAnalytics::StageEvents::StageEvent do
+  let(:instance) { described_class.new({}) }
+
   it { expect(described_class).to respond_to(:name) }
   it { expect(described_class).to respond_to(:identifier) }
-
-  it { expect(described_class.new({})).to respond_to(:object_type) }
+  it { expect(instance).to respond_to(:object_type) }
+  it { expect(instance).to respond_to(:timestamp_projection) }
+  it { expect(instance).to respond_to(:apply_query_customization) }
 end

spec/support/shared_examples/cycle_analytics_event_shared_examples.rb

+# frozen_string_literal: true
+
+shared_examples_for 'cycle analytics event' do
+  let(:instance) { described_class.new({}) }
+
+  it { expect(described_class.name).to be_a_kind_of(String) }
+  it { expect(described_class.identifier).to be_a_kind_of(Symbol) }
+  it { expect(instance.object_type.ancestors).to include(ApplicationRecord) }
+  it { expect(instance).to respond_to(:timestamp_projection) }
+
+  describe '#apply_query_customization' do
+    it 'expects an ActiveRecord::Relation object as argument and returns a modified version of it' do
+      input_query = instance.object_type.all
+
+      output_query = instance.apply_query_customization(input_query)
+      expect(output_query).to be_a_kind_of(ActiveRecord::Relation)
+    end
+  end
+end