Remove the default value stream

With the use_vsa_aggregated_tables feature flag
enabled we should no longer return the in-memory
default value stream.

ee/app/controllers/groups/analytics/cycle_analytics/value_streams_controller.rb

@@ -97,7 +97,13 @@ class Groups::Analytics::CycleAnalytics::ValueStreamsController < Groups::Analyt
   end
 
   def value_streams
-    @group.value_streams.preload_associated_models.presence || [in_memory_default_value_stream]
+    value_streams = @group.value_streams.preload_associated_models
+
+    if Feature.enabled?(:use_vsa_aggregated_tables, @group, default_enabled: :yaml)
+      value_streams
+    else
+      value_streams.presence || [in_memory_default_value_stream]
+    end
   end
 
   def in_memory_default_value_stream

ee/app/services/ee/analytics/cycle_analytics/stages/list_service.rb

@@ -11,7 +11,11 @@ module EE
           def execute
             return forbidden unless allowed?
 
-            success(persisted_stages.presence || build_default_stages)
+            if parent.is_a?(Group) && ::Feature.enabled?(:use_vsa_aggregated_tables, parent, default_enabled: :yaml)
+              success(persisted_stages)
+            else
+              success(persisted_stages.presence || build_default_stages)
+            end
           end
 
           private

ee/spec/controllers/groups/analytics/cycle_analytics/stages_controller_spec.rb

@@ -5,6 +5,15 @@ require 'spec_helper'
 RSpec.describe Groups::Analytics::CycleAnalytics::StagesController do
   let_it_be(:user) { create(:user) }
   let_it_be(:group, refind: true) { create(:group) }
+  let_it_be(:stages) { [] }
+
+  let_it_be(:value_stream) do
+    create(:cycle_analytics_group_value_stream,
+           group: group,
+           name: 'No stage value stream',
+           stages: stages
+          )
+  end
 
   context 'when params have only group_id' do
     let(:params) { { group_id: group } }
@@ -19,6 +28,8 @@ RSpec.describe Groups::Analytics::CycleAnalytics::StagesController do
     end
 
     context 'when use_vsa_aggregated_tables FF is disabled' do
+      let_it_be(:stages) { Gitlab::Analytics::CycleAnalytics::DefaultStages.all }
+
       it_behaves_like 'Value Stream Analytics Stages controller' do
         before do
           stub_feature_flags(use_vsa_aggregated_tables: false)
@@ -28,7 +39,19 @@ RSpec.describe Groups::Analytics::CycleAnalytics::StagesController do
   end
 
   context 'when params have group_id and value_stream_id' do
-    let_it_be(:value_stream) { create(:cycle_analytics_group_value_stream, group: group) }
+    let_it_be(:stages) do
+      [
+        create(:cycle_analytics_group_stage, group: group, name: "Issue", relative_position: 1),
+        create(:cycle_analytics_group_stage, group: group, name: "Code", relative_position: 2)
+      ]
+    end
+
+    let_it_be(:value_stream) do
+      create(:cycle_analytics_group_value_stream,
+             group: group,
+             name: 'First value stream',
+             stages: stages)
+    end
 
     let(:params) { { group_id: group, value_stream_id: value_stream.id } }
     let(:parent) { group }

ee/spec/controllers/groups/analytics/cycle_analytics/value_streams_controller_spec.rb

@@ -16,14 +16,34 @@ RSpec.describe Groups::Analytics::CycleAnalytics::ValueStreamsController do
   end
 
   describe 'GET #index' do
-    it 'returns an in-memory default value stream' do
-      get :index, params: params
+    context 'when the use_vsa_aggregated_tables feature flag is off' do
+      before do
+        stub_feature_flags(use_vsa_aggregated_tables: false)
+      end
+
+      it 'returns an in-memory default value stream' do
+        get :index, params: params
+
+        expect(response).to have_gitlab_http_status(:ok)
+
+        expect(json_response.size).to eq(1)
+        expect(json_response.first['id']).to eq(Analytics::CycleAnalytics::Stages::BaseService::DEFAULT_VALUE_STREAM_NAME)
+        expect(json_response.first['name']).to eq(Analytics::CycleAnalytics::Stages::BaseService::DEFAULT_VALUE_STREAM_NAME)
+      end
+    end
+
+    context 'when the use_vsa_aggregated_tables feature flag is on' do
+      before do
+        stub_feature_flags(use_vsa_aggregated_tables: true)
+      end
 
-      expect(response).to have_gitlab_http_status(:ok)
+      it 'returns an empty array' do
+        get :index, params: params
 
-      expect(json_response.size).to eq(1)
-      expect(json_response.first['id']).to eq(Analytics::CycleAnalytics::Stages::BaseService::DEFAULT_VALUE_STREAM_NAME)
-      expect(json_response.first['name']).to eq(Analytics::CycleAnalytics::Stages::BaseService::DEFAULT_VALUE_STREAM_NAME)
+        expect(response).to have_gitlab_http_status(:ok)
+
+        expect(json_response.size).to eq(0)
+      end
     end
 
     context 'when persisted value streams present' do

ee/spec/features/groups/analytics/cycle_analytics/charts_spec.rb

@@ -12,6 +12,16 @@ RSpec.describe 'Value stream analytics charts', :js do
   let_it_be(:group_label2) { create(:group_label, group: group) }
   let_it_be(:label) { create(:group_label, group: group2) }
 
+  let_it_be(:custom_value_stream_name) { "New created value stream" }
+
+  let_it_be(:value_stream) do
+    create(:cycle_analytics_group_value_stream, group: group, name: custom_value_stream_name, stages: [
+      create(:cycle_analytics_group_stage, group: group, name: "Issue", relative_position: 1, start_event_identifier: :issue_created, end_event_identifier: :issue_closed),
+      create(:cycle_analytics_group_stage, group: group, name: "Code", relative_position: 2, start_event_identifier: :merge_request_created, end_event_identifier: :merge_request_merged),
+      create(:cycle_analytics_group_stage, group: group, name: "Milestone Plan", relative_position: 3, start_event_identifier: :issue_first_associated_with_milestone, end_event_identifier: :issue_first_added_to_board)
+    ])
+  end
+
   3.times do |i|
     let_it_be("issue_#{i}".to_sym) { create(:issue, title: "New Issue #{i}", project: project, created_at: 2.days.ago) }
   end
@@ -33,12 +43,9 @@ RSpec.describe 'Value stream analytics charts', :js do
   end
 
   context 'Duration chart' do
-    let(:custom_value_stream_name) { "New created value stream" }
-
     before do
       select_group(group)
-
-      create_custom_value_stream(custom_value_stream_name)
+      select_value_stream(custom_value_stream_name)
     end
 
     it 'displays data for all stages on the overview' do
@@ -81,6 +88,7 @@ RSpec.describe 'Value stream analytics charts', :js do
           end
 
           select_group(group)
+          select_value_stream(custom_value_stream_name)
         end
 
         it 'displays the chart' do

ee/spec/features/groups/analytics/cycle_analytics/filters_and_data_spec.rb

@@ -17,12 +17,13 @@ RSpec.describe 'Group value stream analytics filters and data', :js do
   let(:mr) { create_merge_request_closing_issue(user, project, issue, commit_message: "References #{issue.to_reference}") }
   let(:pipeline) { create(:ci_empty_pipeline, status: 'created', project: project, ref: mr.source_branch, sha: mr.source_branch_sha, head_pipeline_of: mr) }
 
-  path_nav_selector = '[data-testid="vsa-path-navigation"]'
-  filter_bar_selector = '[data-testid="vsa-filter-bar"]'
-  card_metric_selector = '[data-testid="vsa-metrics"] .gl-single-stat'
-  new_issues_count = 3
+  let(:path_nav_selector) { '[data-testid="vsa-path-navigation"]' }
+  let(:filter_bar_selector) { '[data-testid="vsa-filter-bar"]' }
+  let(:card_metric_selector) { '[data-testid="vsa-metrics"] .gl-single-stat' }
 
-  new_issues_count.times do |i|
+  let(:empty_state_selector) { '[data-testid="vsa-empty-state"]' }
+
+  3.times do |i|
     let_it_be("issue_#{i}".to_sym) { create(:issue, title: "New Issue #{i}", project: sub_group_project, created_at: 2.days.ago) }
   end
 
@@ -58,6 +59,13 @@ RSpec.describe 'Group value stream analytics filters and data', :js do
   end
 
   shared_examples 'empty state' do
+    it 'renders the empty state' do
+      expect(page).to have_selector(empty_state_selector)
+      expect(page).to have_text(s_('CycleAnalytics|Custom value streams to measure your DevSecOps lifecycle'))
+    end
+  end
+
+  shared_examples 'empty value stream stage' do
     it 'displays an empty state' do
       element = page.find('.empty-state')
 
@@ -95,8 +103,7 @@ RSpec.describe 'Group value stream analytics filters and data', :js do
 
       issue_count = page.all(card_metric_selector)[3]
 
-      expect(issue_count).to have_content(n_('New Issue', 'New Issues', 3))
-      expect(issue_count).to have_content(new_issues_count)
+      expect(issue_count).to have_content(n_('New Issue', 'New Issues', 4))
     end
 
     it 'displays time metrics' do
@@ -129,10 +136,6 @@ RSpec.describe 'Group value stream analytics filters and data', :js do
     end
 
     context 'navigation' do
-      before do
-        select_group(selected_group)
-      end
-
       it 'shows the path navigation' do
         expect(page).to have_selector(path_nav_selector)
       end
@@ -140,7 +143,7 @@ RSpec.describe 'Group value stream analytics filters and data', :js do
       it 'each stage will have median values' do
         stage_medians = page.all('.gl-path-button span').collect(&:text)
 
-        expect(stage_medians).to eq(["-"] * 7)
+        expect(stage_medians).to match_array(["-"] * 4)
       end
 
       it 'displays the default list of stages' do
@@ -148,7 +151,7 @@ RSpec.describe 'Group value stream analytics filters and data', :js do
 
         expect(path_nav).to have_content(_("Overview"))
 
-        %w[Issue Plan Code Test Review Staging].each do |item|
+        ['Issue', 'Code', 'Milestone Plan'].each do |item|
           string_id = "CycleAnalytics|#{item}"
           expect(path_nav).to have_content(s_(string_id))
         end
@@ -170,87 +173,156 @@ RSpec.describe 'Group value stream analytics filters and data', :js do
     end
   end
 
-  context 'without valid query parameters set' do
-    context 'with created_after date > created_before date' do
-      before do
-        visit "#{group_analytics_cycle_analytics_path(group)}?created_after=2019-12-31&created_before=2019-11-01"
-      end
+  context 'with no value streams' do
+    before do
+      select_group(group, empty_state_selector)
+    end
 
-      it_behaves_like 'no group available'
+    it_behaves_like 'empty state'
+  end
+
+  context 'with value streams' do
+    def vsa_stages(selected_group)
+      [
+        create(:cycle_analytics_group_stage, group: selected_group, name: "Issue", relative_position: 1, start_event_identifier: :issue_created, end_event_identifier: :issue_closed),
+        create(:cycle_analytics_group_stage, group: selected_group, name: "Code", relative_position: 2, start_event_identifier: :merge_request_created, end_event_identifier: :merge_request_merged),
+        create(:cycle_analytics_group_stage, group: selected_group, name: "Milestone Plan", relative_position: 3, start_event_identifier: :issue_first_associated_with_milestone, end_event_identifier: :issue_first_added_to_board)
+      ]
     end
 
-    context 'with fake parameters' do
-      before do
-        visit "#{group_analytics_cycle_analytics_path(group)}?beans=not-cool"
+    let_it_be(:issue) { create(:issue, project: project) }
+    let_it_be(:value_stream) { create(:cycle_analytics_group_value_stream, group: group, name: 'First value stream', stages: vsa_stages(group)) }
+    let_it_be(:subgroup_value_stream) { create(:cycle_analytics_group_value_stream, group: sub_group, name: 'First subgroup value stream', stages: vsa_stages(sub_group)) }
 
-        select_stage("Issue")
+    context 'without valid query parameters set' do
+      context 'with created_after date > created_before date' do
+        before do
+          visit "#{group_analytics_cycle_analytics_path(group)}?created_after=2019-12-31&created_before=2019-11-01"
+        end
+
+        it_behaves_like 'no group available'
       end
 
-      it_behaves_like 'empty state'
+      context 'with fake parameters' do
+        before do
+          visit "#{group_analytics_cycle_analytics_path(group)}?beans=not-cool"
+
+          select_value_stream('First value stream')
+          select_stage("Issue")
+        end
+
+        it_behaves_like 'empty value stream stage'
+      end
     end
-  end
 
-  context 'with valid query parameters set' do
-    projects_dropdown = '.js-projects-dropdown-filter'
+    context 'with valid query parameters set' do
+      projects_dropdown = '.js-projects-dropdown-filter'
 
-    context 'with project_ids set' do
-      before do
-        visit "#{group_analytics_cycle_analytics_path(group)}?project_ids[]=#{project.id}"
+      context 'with project_ids set' do
+        before do
+          visit "#{group_analytics_cycle_analytics_path(group)}?project_ids[]=#{project.id}"
+        end
+
+        it 'has the projects dropdown prepopulated' do
+          element = page.find(projects_dropdown)
+
+          expect(element).to have_content project.name
+        end
       end
 
-      it 'has the projects dropdown prepopulated' do
-        element = page.find(projects_dropdown)
+      context 'with created_before and created_after set' do
+        before do
+          visit "#{group_analytics_cycle_analytics_path(group)}?created_before=2019-12-31&created_after=2019-11-01"
+        end
+
+        it 'has the date range prepopulated' do
+          date_range = '.js-daterange-picker'
+          element = page.find(date_range)
 
-        expect(element).to have_content project.name
+          expect(element.find('.js-daterange-picker-from input').value).to eq '2019-11-01'
+          expect(element.find('.js-daterange-picker-to input').value).to eq '2019-12-31'
+          expect(page.find('.js-tasks-by-type-chart')).to have_text(_("Showing data for group '%{group_name}' from Nov 1, 2019 to Dec 31, 2019") % { group_name: group.name })
+        end
       end
     end
 
-    context 'with created_before and created_after set' do
-      date_range = '.js-daterange-picker'
+    context 'with a group' do
+      let(:selected_group) { group }
 
       before do
-        visit "#{group_analytics_cycle_analytics_path(group)}?created_before=2019-12-31&created_after=2019-11-01"
+        select_group(group)
+        select_value_stream('First value stream')
       end
 
-      it 'has the date range prepopulated' do
-        element = page.find(date_range)
+      it_behaves_like 'group value stream analytics'
 
-        expect(element.find('.js-daterange-picker-from input').value).to eq '2019-11-01'
-        expect(element.find('.js-daterange-picker-to input').value).to eq '2019-12-31'
-        expect(page.find('.js-tasks-by-type-chart')).to have_text(_("Showing data for group '%{group_name}' from Nov 1, 2019 to Dec 31, 2019") % { group_name: group.name })
-      end
+      it_behaves_like 'has overview metrics'
+
+      it_behaves_like 'has default filters'
     end
-  end
 
-  context 'with a group' do
-    let(:selected_group) { group }
+    context 'with a sub group' do
+      let(:selected_group) { sub_group }
 
-    before do
-      select_group(group)
+      before do
+        select_group(sub_group)
+        select_value_stream('First subgroup value stream')
+      end
+
+      it_behaves_like 'group value stream analytics'
+
+      it_behaves_like 'has overview metrics'
+
+      it_behaves_like 'has default filters'
     end
 
-    it_behaves_like 'group value stream analytics'
+    context 'with lots of data', :js do
+      around do |example|
+        freeze_time { example.run }
+      end
 
-    it_behaves_like 'has overview metrics'
+      before do
+        issue.update!(created_at: 5.days.ago)
+        create_cycle(user, project, issue, mr, milestone, pipeline)
+        create(:labeled_issue, created_at: 5.days.ago, project: create(:project, group: group), labels: [group_label1])
+        create(:labeled_issue, created_at: 3.days.ago, project: create(:project, group: group), labels: [group_label2])
 
-    it_behaves_like 'has default filters'
-  end
+        issue.metrics.update!(first_mentioned_in_commit_at: mr.created_at - 5.hours)
+        mr.metrics.update!(first_deployed_to_production_at: mr.created_at + 2.hours, merged_at: mr.created_at + 1.hour)
 
-  context 'with a sub group' do
-    let(:selected_group) { sub_group }
+        deploy_master(user, project, environment: 'staging')
+        deploy_master(user, project)
 
-    before do
-      select_group(sub_group)
-    end
+        aggregation = Analytics::CycleAnalytics::Aggregation.safe_create_for_group(group)
+        Analytics::CycleAnalytics::AggregatorService.new(aggregation: aggregation).execute
 
-    it_behaves_like 'group value stream analytics'
+        select_group(group)
+        select_value_stream('First value stream')
+      end
 
-    it_behaves_like 'has overview metrics'
+      stages_with_data = [
+        { title: 'Issue', description: 'Time before an issue gets scheduled', events_count: 1, time: '5d' },
+        { title: 'Code', description: 'Time until first merge request', events_count: 1, time: '1h' }
+      ]
 
-    it_behaves_like 'has default filters'
+      stages_without_data = [{ title: 'Milestone', description: 'Time before an issue starts implementation', events_count: 0, time: "-" }]
+
+      it 'each stage with events will display the stage events list when selected', :sidekiq_might_not_need_inline do
+        stages_without_data.each do |stage|
+          select_stage(stage[:title])
+          expect(page).not_to have_selector('[data-testid="vsa-stage-event"]')
+        end
+
+        stages_with_data.each do |stage|
+          select_stage(stage[:title])
+          expect(page).to have_selector('[data-testid="vsa-stage-table"]')
+          expect(page.all('[data-testid="vsa-stage-event"]').length).to eq(stage[:events_count])
+        end
+      end
+    end
   end
 
-  context 'with lots of data', :js do
+  context 'with use_vs_aggregated_table disabled', :js do
     let_it_be(:issue) { create(:issue, project: project) }
 
     around do |example|

ee/spec/features/groups/analytics/cycle_analytics/multiple_value_streams_spec.rb

@@ -18,7 +18,7 @@ RSpec.describe 'Multiple value streams', :js do
 
   let(:extended_form_fields_selector) { '[data-testid="extended-form-fields"]' }
   let(:preset_selector) { '[data-testid="vsa-preset-selector"]' }
-  let!(:default_value_stream) { create(:cycle_analytics_group_value_stream, group: group, name: 'default') }
+  let(:empty_state_selector) { '[data-testid="vsa-empty-state"]' }
 
   3.times do |i|
     let_it_be("issue_#{i}".to_sym) { create(:issue, title: "New Issue #{i}", project: project, created_at: 2.days.ago) }
@@ -198,7 +198,7 @@ RSpec.describe 'Multiple value streams', :js do
     end
   end
 
-  describe 'With a group' do
+  shared_examples 'create group value streams' do
     name = 'group value stream'
 
     before do
@@ -210,7 +210,7 @@ RSpec.describe 'Multiple value streams', :js do
     it_behaves_like 'delete a value stream', name
   end
 
-  describe 'With a sub group' do
+  shared_examples 'create sub group value streams' do
     name = 'sub group value stream'
 
     before do
@@ -221,4 +221,42 @@ RSpec.describe 'Multiple value streams', :js do
     it_behaves_like 'update a value stream', name
     it_behaves_like 'delete a value stream', name
   end
+
+  context 'use_vsa_aggregated_tables feature flag off' do
+    before do
+      stub_feature_flags(use_vsa_aggregated_tables: false)
+    end
+
+    it_behaves_like 'create group value streams'
+    it_behaves_like 'create sub group value streams'
+  end
+
+  context 'use_vsa_aggregated_tables feature flag on' do
+    context 'without a value stream' do
+      before do
+        select_group(group, empty_state_selector)
+      end
+
+      it 'renders the empty state' do
+        expect(page).to have_text(s_('CycleAnalytics|Custom value streams to measure your DevSecOps lifecycle'))
+      end
+
+      it 'can navigate to the create value stream form' do
+        page.find('[data-testid="create-value-stream-button"]').click
+
+        expect(page).to have_selector('[data-testid="value-stream-form-modal"]')
+      end
+    end
+
+    context 'with a value stream' do
+      before do
+        # ensure we have a value stream already available
+        create(:cycle_analytics_group_value_stream, group: group, name: 'default')
+        create(:cycle_analytics_group_value_stream, group: sub_group, name: 'default')
+      end
+
+      it_behaves_like 'create group value streams'
+      it_behaves_like 'create sub group value streams'
+    end
+  end
 end

ee/spec/services/analytics/cycle_analytics/stages/list_service_spec.rb

@@ -19,27 +19,43 @@ RSpec.describe Analytics::CycleAnalytics::Stages::ListService do
     stub_licensed_features(cycle_analytics_for_groups: true)
   end
 
-  it_behaves_like 'permission check for Value Stream Analytics Stage services', :cycle_analytics_for_groups
+  context 'when the use_vsa_aggregated_tables feature is enabled' do
+    before do
+      stub_feature_flags(use_vsa_aggregated_tables: true)
+    end
 
-  it 'returns only the default stages' do
-    expect(stages.size).to eq(Gitlab::Analytics::CycleAnalytics::DefaultStages.all.size)
-  end
+    it_behaves_like 'permission check for Value Stream Analytics Stage services', :cycle_analytics_for_groups
 
-  it 'provides the default stages as non-persisted objects' do
-    expect(stages.map(&:id)).to all(be_nil)
-  end
+    it 'returns empty array' do
+      expect(stages.size).to eq(0)
+    end
+
+    it 'provides the default stages as non-persisted objects' do
+      expect(stages.map(&:id)).to all(be_nil)
+    end
 
-  it 'does not persist the value stream record' do
-    expect { subject }.not_to change { Analytics::CycleAnalytics::GroupValueStream.count }
+    it 'does not persist the value stream record' do
+      expect { subject }.not_to change { Analytics::CycleAnalytics::GroupValueStream.count }
+    end
+
+    context 'when there are persisted stages' do
+      let_it_be(:stage1) { create(:cycle_analytics_group_stage, parent: group, relative_position: 2, value_stream: value_stream) }
+      let_it_be(:stage2) { create(:cycle_analytics_group_stage, parent: group, relative_position: 3, value_stream: value_stream) }
+      let_it_be(:stage3) { create(:cycle_analytics_group_stage, parent: group, relative_position: 1, value_stream: value_stream) }
+
+      it 'returns the persisted stages in order' do
+        expect(stages).to eq([stage3, stage1, stage2])
+      end
+    end
   end
 
-  context 'when there are persisted stages' do
-    let_it_be(:stage1) { create(:cycle_analytics_group_stage, parent: group, relative_position: 2, value_stream: value_stream) }
-    let_it_be(:stage2) { create(:cycle_analytics_group_stage, parent: group, relative_position: 3, value_stream: value_stream) }
-    let_it_be(:stage3) { create(:cycle_analytics_group_stage, parent: group, relative_position: 1, value_stream: value_stream) }
+  context 'when the use_vsa_aggregated_tables feature is disabled' do
+    before do
+      stub_feature_flags(use_vsa_aggregated_tables: false)
+    end
 
-    it 'returns the persisted stages in order' do
-      expect(stages).to eq([stage3, stage1, stage2])
+    it 'returns the default stages' do
+      expect(stages.size).to eq(Gitlab::Analytics::CycleAnalytics::DefaultStages.all.size)
     end
   end
 end

ee/spec/support/shared_examples/controllers/analytics/cycle_analytics/shared_stage_shared_examples.rb

@@ -22,7 +22,7 @@ RSpec.shared_examples 'Value Stream Analytics Stages controller' do
       subject
 
       response_start_events = json_response['stages'].map { |s| s['start_event_identifier'] }
-      start_events = Gitlab::Analytics::CycleAnalytics::DefaultStages.all.map { |s| s['start_event_identifier'] }
+      start_events = stages.map { |s| s['start_event_identifier'] }
 
       expect(response_start_events).to eq(start_events)
     end

spec/support/helpers/cycle_analytics_helpers.rb

@@ -86,6 +86,13 @@ module CycleAnalyticsHelpers
     wait_for_stages_to_load(ready_selector)
   end
 
+  def select_value_stream(value_stream_name)
+    toggle_value_stream_dropdown
+
+    page.find('[data-testid="dropdown-value-streams"]').all('li button').find { |item| item.text == value_stream_name.to_s }.click
+    wait_for_requests
+  end
+
   def toggle_dropdown(field)
     page.within("[data-testid*='#{field}']") do
       find('.dropdown-toggle').click