Add QA test for prometheus alerts

The configuration of alerts depends on a lot of moving parts and has
been prone to issue. This test sets up a cluster, creates a project,
connects the cluster to the project, sets up Auto Devops, deploys the
application, then performs basic CRUD operations on alerts from the
metrics dashboard. Alerts are configurated by updating Prometheus, so
a future test could build upon the assertions by ensuring that the alert
can indeed be fired.

app/assets/javascripts/monitoring/components/charts/time_series.vue

@@ -311,7 +311,10 @@ export default {
       <gl-tooltip :target="() => $refs.graphTitle" :disabled="!showTitleTooltip">
         {{ graphData.title }}
       </gl-tooltip>
-      <div class="prometheus-graph-widgets js-graph-widgets flex-fill">
+      <div
+        class="prometheus-graph-widgets js-graph-widgets flex-fill"
+        data-qa-selector="prometheus_graph_widgets"
+      >
         <slot></slot>
       </div>
     </div>

app/assets/javascripts/monitoring/components/dashboard.vue

@@ -353,7 +353,7 @@ export default {
 </script>
 
 <template>
-  <div class="prometheus-graphs">
+  <div class="prometheus-graphs" data-qa-selector="prometheus_graphs">
     <div
       v-if="showHeader"
       ref="prometheusGraphsHeader"

app/assets/javascripts/monitoring/components/panel_type.vue

@@ -138,6 +138,7 @@ export default {
         v-gl-tooltip
         class="ml-auto mx-3"
         toggle-class="btn btn-transparent border-0"
+        data-qa-selector="prometheus_widgets_dropdown"
         :right="true"
         :no-caret="true"
         :title="__('More actions')"
@@ -161,7 +162,11 @@ export default {
         >
           {{ __('Generate link to chart') }}
         </gl-dropdown-item>
-        <gl-dropdown-item v-if="alertWidgetAvailable" v-gl-modal="`alert-modal-${index}`">
+        <gl-dropdown-item
+          v-if="alertWidgetAvailable"
+          v-gl-modal="`alert-modal-${index}`"
+          data-qa-selector="alert_widget_menu_item"
+        >
           {{ __('Alerts') }}
         </gl-dropdown-item>
       </gl-dropdown>

app/assets/javascripts/pipelines/components/pipelines_table_row.vue

@@ -331,6 +331,7 @@ export default {
           :loading="isRetrying"
           :disabled="isRetrying"
           container-class="js-pipelines-retry-button btn btn-default btn-retry"
+          data-qa-selector="pipeline_retry_button"
           @click="handleRetryClick"
         >
           <icon name="repeat" />

app/views/layouts/nav/sidebar/_project.html.haml

@@ -240,7 +240,7 @@
 
             - if project_nav_tab? :environments
               = nav_link(controller: :environments, action: [:metrics, :metrics_redirect]) do
-                = link_to metrics_project_environments_path(@project), title: _('Metrics'), class: 'shortcuts-metrics' do
+                = link_to metrics_project_environments_path(@project), title: _('Metrics'), class: 'shortcuts-metrics', data: { qa_selector: 'operations_metrics_link' } do
                   %span
                     = _('Metrics')
 

ee/app/assets/javascripts/monitoring/components/alert_widget_form.vue

@@ -239,11 +239,12 @@ export default {
         <gl-dropdown
           id="alert-query-dropdown"
           :text="queryDropdownLabel"
-          toggle-class="dropdown-menu-toggle"
+          toggle-class="dropdown-menu-toggle qa-alert-query-dropdown"
         >
           <gl-dropdown-item
             v-for="query in relevantQueries"
             :key="query.metricId"
+            data-qa-selector="alert_query_option"
             @click="selectQuery(query.metricId)"
           >
             {{ query.label }}
@@ -282,6 +283,7 @@ export default {
           v-model.number="threshold"
           :disabled="formDisabled"
           type="number"
+          data-qa-selector="alert_threshold_field"
         />
       </gl-form-group>
     </div>

qa/qa.rb

@@ -289,6 +289,8 @@ module QA
           autoload :AddExisting, 'qa/page/project/operations/kubernetes/add_existing'
           autoload :Show, 'qa/page/project/operations/kubernetes/show'
         end
+
+        autoload :Metrics, 'qa/page/project/operations/metrics'
       end
 
       module Wiki

qa/qa/ee.rb

@@ -128,6 +128,8 @@ module QA
           module Kubernetes
             autoload :Show, 'qa/ee/page/project/operations/kubernetes/show'
           end
+
+          autoload :Metrics, 'qa/ee/page/project/operations/metrics'
         end
 
         module Packages

qa/qa/ee/page/project/operations/metrics.rb

+# frozen_string_literal: true
+
+module QA
+  module EE
+    module Page
+      module Project
+        module Operations
+          module Metrics
+            EXPECTED_LABEL = 'Total (GB)'
+
+            def self.prepended(page)
+              page.module_eval do
+                view 'ee/app/assets/javascripts/monitoring/components/alert_widget_form.vue' do
+                  element :alert_query_dropdown
+                  element :alert_query_option
+                  element :alert_threshold_field
+                end
+              end
+            end
+
+            def wait_for_alert(operator = '>', threshold = 0)
+              wait_until(reload: false) { has_alert?(operator, threshold) }
+            end
+
+            def has_alert?(operator = '>', threshold = 0)
+              within_element :prometheus_graphs do
+                has_text?([EXPECTED_LABEL, operator, threshold].join(' '))
+              end
+            end
+
+            def write_first_alert(operator = '>', threshold = 0)
+              open_first_alert_modal
+              click_on operator
+              fill_element :alert_threshold_field, threshold
+
+              within('.modal-content') { click_button(class: 'btn-success') }
+            end
+
+            def delete_first_alert
+              open_first_alert_modal
+
+              within('.modal-content') { click_button(class: 'btn-danger') }
+              wait_for_requests
+            end
+
+            def open_first_alert_modal
+              all_elements(:prometheus_widgets_dropdown, minimum: 1).first.click
+              click_element :alert_widget_menu_item
+
+              click_element :alert_query_dropdown unless has_element?(:alert_query_option, wait: 3)
+              all_elements(:alert_query_option, minimum: 1).first.click
+            end
+          end
+        end
+      end
+    end
+  end
+end

qa/qa/page/project/operations/metrics.rb

+# frozen_string_literal: true
+
+module QA
+  module Page
+    module Project
+      module Operations
+        class Metrics < Page::Base
+          EXPECTED_TITLE = 'Memory Usage (Total)'
+          LOADING_MESSAGE = 'Waiting for performance data'
+
+          view 'app/assets/javascripts/monitoring/components/dashboard.vue' do
+            element :prometheus_graphs
+          end
+
+          view 'app/assets/javascripts/monitoring/components/charts/time_series.vue' do
+            element :prometheus_graph_widgets
+          end
+
+          view 'app/assets/javascripts/monitoring/components/panel_type.vue' do
+            element :prometheus_widgets_dropdown
+            element :alert_widget_menu_item
+          end
+
+          def wait_for_metrics
+            wait_for_data
+            return if has_metrics?
+
+            wait_until(max_duration: 180) do
+              wait_for_data
+              has_metrics?
+            end
+          end
+
+          def wait_for_data
+            wait_until(reload: false) { !has_text?(LOADING_MESSAGE) } if has_text?(LOADING_MESSAGE)
+          end
+
+          def has_metrics?
+            within_element :prometheus_graphs do
+              has_text?(EXPECTED_TITLE)
+            end
+          end
+        end
+      end
+    end
+  end
+end
+
+QA::Page::Project::Operations::Metrics.prepend_if_ee('QA::EE::Page::Project::Operations::Metrics')

qa/qa/page/project/pipeline/index.rb

@@ -9,6 +9,7 @@ module QA::Page
 
       view 'app/assets/javascripts/pipelines/components/pipelines_table_row.vue' do
         element :pipeline_commit_status
+        element :pipeline_retry_button
       end
 
       def click_on_latest_pipeline
@@ -18,10 +19,25 @@ module QA::Page
       end
 
       def wait_for_latest_pipeline_success
+        wait_for_latest_pipeline_status { has_text?('passed') }
+      end
+
+      def wait_for_latest_pipeline_completion
+        wait_for_latest_pipeline_status { has_text?('passed') || has_text?('failed') }
+      end
+
+      def wait_for_latest_pipeline_status
         wait_until(reload: false, max_duration: 300) do
-          within_element_by_index(:pipeline_commit_status, 0) do
-            has_text?('passed')
-          end
+          within_element_by_index(:pipeline_commit_status, 0) { yield }
+        end
+      end
+
+      def wait_for_latest_pipeline_success_or_retry
+        wait_for_latest_pipeline_completion
+
+        if has_text?('failed')
+          click_element :pipeline_retry_button
+          wait_for_latest_pipeline_success
         end
       end
     end

qa/qa/page/project/sub_menus/operations.rb

@@ -12,6 +12,7 @@ module QA
               view 'app/views/layouts/nav/sidebar/_project.html.haml' do
                 element :link_operations
                 element :operations_environments_link
+                element :operations_metrics_link
               end
             end
           end
@@ -24,6 +25,14 @@ module QA
             end
           end
 
+          def go_to_operations_metrics
+            hover_operations do
+              within_submenu do
+                click_element(:operations_metrics_link)
+              end
+            end
+          end
+
           def go_to_operations_kubernetes
             hover_operations do
               within_submenu do

qa/qa/specs/features/ee/browser_ui/8_monitor/health/alerting_spec.rb

+# frozen_string_literal: true
+
+module QA
+  context 'Monitor' do
+    describe 'Alerts', :orchestrated, :kubernetes do
+      before do
+        @cluster = Service::KubernetesCluster.new.create!
+      end
+
+      after do
+        @cluster&.remove!
+      end
+
+      it 'allows configuration of alerts' do
+        Flow::Login.sign_in
+        project = create_project
+        create_kubernetes_cluster(project, @cluster)
+        push_repository(project)
+        wait_for_deployment
+
+        Page::Project::Operations::Metrics.perform do |metrics|
+          verify_metrics(metrics)
+          verify_add_alert(metrics)
+          verify_edit_alert(metrics)
+          verify_persist_alert(metrics)
+          verify_delete_alert(metrics)
+        end
+      end
+
+      private
+
+      def create_project
+        Resource::Project.fabricate_via_api! do |p|
+          p.name = 'alerts'
+          p.description = 'Project with alerting configured'
+        end
+      end
+
+      def create_kubernetes_cluster(project, cluster)
+        Resource::KubernetesCluster.fabricate_via_browser_ui! do |c|
+          c.project = project
+          c.cluster = cluster
+          c.install_helm_tiller = true
+          c.install_prometheus = true
+          c.install_runner = true
+        end
+      end
+
+      def push_repository(project)
+        Resource::Repository::ProjectPush.fabricate! do |push|
+          push.project = project
+          push.directory = Pathname
+            .new(__dir__)
+            .join('../../../../../../fixtures/monitored_auto_devops')
+          push.commit_message = 'Create Auto DevOps compatible gitlab-ci.yml'
+        end
+      end
+
+      def wait_for_deployment
+        Page::Project::Menu.perform(&:click_ci_cd_pipelines)
+        Page::Project::Pipeline::Index.perform(&:wait_for_latest_pipeline_success_or_retry)
+        Page::Project::Menu.perform(&:go_to_operations_metrics)
+      end
+
+      def verify_metrics(metrics)
+        metrics.wait_for_metrics
+
+        expect(metrics).to have_metrics
+        expect(metrics).not_to have_alert
+      end
+
+      def verify_add_alert(metrics)
+        metrics.write_first_alert('>', 0)
+
+        expect(metrics).to have_alert
+      end
+
+      def verify_edit_alert(metrics)
+        metrics.write_first_alert('<', 0)
+
+        expect(metrics).to have_alert('<')
+      end
+
+      def verify_persist_alert(metrics)
+        metrics.refresh
+        metrics.wait_for_metrics
+        metrics.wait_for_alert('<')
+
+        expect(metrics).to have_alert('<')
+      end
+
+      def verify_delete_alert(metrics)
+        metrics.delete_first_alert
+
+        expect(metrics).not_to have_alert('<')
+      end
+    end
+  end
+end