Notify owners of namespace when CI usage is reaching his limit

Mutliple levels of notifications are suppoerted.
We just send the notification when a new level has been reached.

db/schema.rb

@@ -2116,6 +2116,7 @@ ActiveRecord::Schema.define(version: 20190703130053) do
     t.boolean "auto_devops_enabled"
     t.integer "extra_shared_runners_minutes_limit"
     t.datetime_with_timezone "last_ci_minutes_notification_at"
+    t.integer "last_ci_minutes_usage_notification_level"
     t.index ["created_at"], name: "index_namespaces_on_created_at", using: :btree
     t.index ["custom_project_templates_group_id", "type"], name: "index_namespaces_on_custom_project_templates_group_id_and_type", where: "(custom_project_templates_group_id IS NOT NULL)", using: :btree
     t.index ["file_template_project_id"], name: "index_namespaces_on_file_template_project_id", using: :btree

ee/app/helpers/ee/runners_helper.rb

 # frozen_string_literal: true
 module EE
   module RunnersHelper
-    def purchase_shared_runner_minutes_link(user, project)
-      if ::Gitlab.com? && can?(user, :admin_project, project)
-        link_to(_("Click here"), EE::SUBSCRIPTIONS_PLANS_URL, target: '_blank', rel: 'noopener') + s_("Pipelines| to purchase more minutes.")
+    def ci_usage_warning_message(namespace, project)
+      message = ci_usage_base_message(namespace)
+
+      return unless message
+
+      if ::Gitlab.com? && can?(current_user, :admin_project, project)
+        message += " #{purchase_shared_runner_minutes_link}"
+      elsif namespace.shared_runners_minutes_used?
+        message += s_('Pipelines|Pipelines will not run anymore on shared Runners.')
+      end
+
+      message.html_safe
+    end
+
+    def ci_usage_warning_class(namespace)
+      if EE::Namespace::CI_USAGE_ALERT_LEVELS.min == namespace.last_ci_minutes_usage_notification_level
+        'alert-danger'
       else
-        s_("Pipelines|Pipelines will not run anymore on shared Runners.")
+        'alert-warning'
+      end
+    end
+
+    private
+
+    def purchase_shared_runner_minutes_link
+      link = link_to(_("Click here"), EE::SUBSCRIPTIONS_PLANS_URL, target: '_blank', rel: 'noopener')
+
+      link + s_("Pipelines| to purchase more minutes.")
+    end
+
+    def ci_usage_base_message(namespace)
+      if namespace.shared_runners_minutes_used?
+        s_("Pipelines|%{namespace_name} has exceeded its pipeline minutes quota.") % { namespace_name: namespace.name }
+      elsif namespace.last_ci_minutes_usage_notification_level
+        s_("Pipelines|%{namespace_name} has less than %{notification_level}%% of CI minutes available.") % { namespace_name: namespace.name, notification_level: namespace.last_ci_minutes_usage_notification_level }
       end
     end
   end

ee/app/mailers/ci_minutes_usage_mailer.rb

 # frozen_string_literal: true
 
 class CiMinutesUsageMailer < BaseMailer
-  def notify(namespace_name, contact_email)
+  def notify(namespace_name, recipients)
     @namespace_name = namespace_name
 
     mail(
-      to: contact_email,
+      bcc: recipients,
       subject: "GitLab CI Runner Minutes quota for #{namespace_name} has run out"
     )
   end
+
+  def notify_limit(namespace_name, recipients, percentage_of_available_mins)
+    @namespace_name = namespace_name
+    @percentage_of_available_mins = percentage_of_available_mins
+
+    mail(
+      bcc: recipients,
+      subject: "GitLab CI Runner Minutes quota for #{namespace_name} has \
+                less than #{percentage_of_available_mins}% available"
+    )
+  end
 end

ee/app/models/ee/namespace.rb

@@ -27,6 +27,8 @@ module EE
     LICENSE_PLANS_TO_NAMESPACE_PLANS = NAMESPACE_PLANS_TO_LICENSE_PLANS.invert.freeze
     PLANS = NAMESPACE_PLANS_TO_LICENSE_PLANS.keys.freeze
 
+    CI_USAGE_ALERT_LEVELS = [30, 5].freeze
+
     prepended do
       include EachBatch
 

ee/app/services/ci_minutes_usage_notify_service.rb

@@ -2,22 +2,53 @@
 
 class CiMinutesUsageNotifyService < BaseService
   def execute
+    notify_on_total_usage
+    notify_on_partial_usage
+  end
+
+  private
+
+  def recipients
+    namespace.user? ? [namespace.owner.email] : namespace.owners.pluck(:email) # rubocop:disable CodeReuse/ActiveRecord
+  end
+
+  def notify_on_total_usage
     return unless namespace.shared_runners_minutes_used? && namespace.last_ci_minutes_notification_at.nil?
 
     namespace.update_columns(last_ci_minutes_notification_at: Time.now)
 
-    owners.each do |user|
-      CiMinutesUsageMailer.notify(namespace.name, user.email).deliver_later
-    end
+    CiMinutesUsageMailer.notify(namespace.name, recipients).deliver_later
   end
 
-  private
+  def notify_on_partial_usage
+    return if namespace.shared_runners_minutes_used?
+    return if namespace.last_ci_minutes_usage_notification_level == current_alert_level
+    return if alert_levels.max < ci_minutes_percent_left
+
+    namespace.update_columns(last_ci_minutes_usage_notification_level: current_alert_level)
+
+    CiMinutesUsageMailer.notify_limit(namespace.name, recipients, current_alert_level).deliver_later
+  end
 
   def namespace
     @namespace ||= project.shared_runners_limit_namespace
   end
 
-  def owners
-    namespace.user? ? [namespace.owner] : namespace.owners
+  def ci_minutes_percent_left
+    quota = namespace.actual_shared_runners_minutes_limit
+    used = namespace.shared_runners_minutes.to_i
+    minutes_left = quota - used
+
+    return 0 if minutes_left <= 0
+
+    (minutes_left.to_f * 100) / quota.to_f
+  end
+
+  def alert_levels
+    @alert_levels ||= EE::Namespace::CI_USAGE_ALERT_LEVELS.sort
+  end
+
+  def current_alert_level
+    @current_alert_level ||= alert_levels.find { |level| level >= ci_minutes_percent_left }
   end
 end

ee/app/views/ci_minutes_usage_mailer/notify_limit.html.haml

+%p
+  This is an automated notification to let you know that your CI Runner Minutes quota for "#{@namespace_name}" is below #{@percentage_of_available_mins}%.
+%p
+  Click #{link_to('here', EE::SUBSCRIPTIONS_PLANS_URL)} to purchase more minutes.
+%p
+  If you need assistance, please contact #{link_to('GitLab support', 'https://support.gitlab.com')}.

ee/app/views/ci_minutes_usage_mailer/notify_limit.text.erb

+This is an automated notification to let you know that your CI Runner Minutes
+quota for "<%= @namespace_name %>" is below <%= @percentage_of_available_mins %>%.
+
+Please visit <%= EE::SUBSCRIPTIONS_PLANS_URL %> to purchase more minutes.
+
+If you need assistance, please contact GitLab support (https://support.gitlab.com).

ee/app/views/shared/_shared_runners_minutes_limit.html.haml

@@ -3,11 +3,11 @@
 - scope = (project || namespace).full_path
 - has_limit = (project || namespace).shared_runners_minutes_limit_enabled?
 - can_see_status = project.nil? || can?(current_user, :create_pipeline, project)
+- ci_warning_message = ci_usage_warning_message(namespace, project)
 
-- if cookies[:hide_shared_runner_quota_message].blank? && has_limit && namespace.shared_runners_minutes_used? && can_see_status
-  .shared-runner-quota-message.alert.alert-warning.d-none.d-sm-block{ data: { scope: scope } }
-    = namespace.name
-    has exceeded its pipeline minutes quota. #{purchase_shared_runner_minutes_link(current_user, project)}
+- if cookies[:hide_shared_runner_quota_message].blank? && has_limit && can_see_status && ci_warning_message.present?
+  .shared-runner-quota-message.alert.d-none.d-sm-block{ class: ci_usage_warning_class(namespace), data: { scope: scope } }
+    = ci_warning_message
 
     .float-right
       = link_to 'Remind later', '#', class: 'hide-shared-runner-limit-message alert-link'

ee/app/workers/clear_shared_runners_minutes_worker.rb

@@ -19,8 +19,9 @@ class ClearSharedRunnersMinutesWorker
         .update_all("extra_shared_runners_minutes_limit = #{extra_minutes_left_sql} FROM namespace_statistics")
     end
 
-    Namespace.where.not(last_ci_minutes_notification_at: nil).each_batch do |relation|
-      relation.update_all(last_ci_minutes_notification_at: nil)
+    Namespace.where('last_ci_minutes_notification_at IS NOT NULL OR last_ci_minutes_usage_notification_level IS NOT NULL')
+      .each_batch do |relation|
+      relation.update_all(last_ci_minutes_notification_at: nil, last_ci_minutes_usage_notification_level: nil)
     end
 
     NamespaceStatistics.where.not(shared_runners_seconds: 0)

ee/changelogs/unreleased/10283-notifications-for-ci-minutes-quota-limit-approaching.yml

+---
+title: Add notifications for CI Minutes quota limit approaching
+merge_request: 14328
+author:
+type: added

ee/lib/ee/api/namespaces.rb

@@ -25,6 +25,7 @@ module EE
             # Reset last_ci_minutes_notification_at if customer purchased extra CI minutes.
             if params[:extra_shared_runners_minutes_limit].present?
               update_attrs[:last_ci_minutes_notification_at] = nil
+              update_attrs[:last_ci_minutes_usage_notification_level] = nil
             end
 
             namespace.update(update_attrs)

ee/spec/features/ci_shared_runner_warnings_spec.rb

@@ -27,6 +27,24 @@ describe 'CI shared runner limits' do
     end
 
     context 'when limit is defined' do
+      before do
+        stub_const("EE::Namespace::CI_USAGE_ALERT_LEVELS", [30, 5])
+      end
+
+      context 'when usage has reached a notification level' do
+        let(:group) { create(:group, :with_build_minutes_limit, last_ci_minutes_usage_notification_level: 30) }
+
+        it 'displays a warning message on pipelines page' do
+          visit_project_pipelines
+          expect_quota_exceeded_alert("#{group.name} has less than 30% of CI minutes available.")
+        end
+
+        it 'displays a warning message on project homepage' do
+          visit_project_home
+          expect_quota_exceeded_alert("#{group.name} has less than 30% of CI minutes available.")
+        end
+      end
+
       context 'when limit is exceeded' do
         let(:group) { create(:group, :with_used_build_minutes_limit) }
 

ee/spec/helpers/ee/runners_helper_spec.rb

+# frozen_string_literal: true
+
+require "spec_helper"
+
+describe EE::RunnersHelper do
+  describe '.ci_usage_warning_message' do
+    let(:project) { create(:project, namespace: namespace) }
+    let(:minutes_used) { 0 }
+    let(:user) { create(:user) }
+
+    let(:namespace) do
+      create(:group, shared_runners_minutes_limit: 100)
+    end
+
+    let!(:statistics) do
+      create(:namespace_statistics, namespace: namespace, shared_runners_seconds: minutes_used * 60)
+    end
+
+    before do
+      allow(::Gitlab).to receive(:com?).and_return(true)
+      allow(helper).to receive(:current_user).and_return(user)
+      allow(helper).to receive(:can?).with(user, :admin_project, project) { false }
+
+      stub_const("EE::Namespace::CI_USAGE_ALERT_LEVELS", [50])
+    end
+
+    subject { helper.ci_usage_warning_message(namespace, project) }
+
+    context 'when CI minutes quota is above the warning limits' do
+      let(:minutes_used) { 40 }
+
+      it 'does not return a message' do
+        expect(subject).to be_nil
+      end
+    end
+
+    context 'when current user is an owner' do
+      before do
+        allow(helper).to receive(:can?).with(user, :admin_project, project) { true }
+      end
+
+      context 'when usage has reached first level of notification' do
+        before do
+          namespace.update_attribute(:last_ci_minutes_usage_notification_level, 50)
+        end
+
+        it 'shows the partial usage message' do
+          expect(subject).to match("#{namespace.name} has less than 50% of CI minutes available.")
+          expect(subject).to match('to purchase more minutes')
+        end
+      end
+
+      context 'when usage is above the quota' do
+        let(:minutes_used) { 120 }
+
+        it 'shows the total usage message' do
+          expect(subject).to match("#{namespace.name} has exceeded its pipeline minutes quota.")
+          expect(subject).to match('to purchase more minutes')
+        end
+      end
+    end
+
+    context 'when current user is not an owner' do
+      context 'when usage has reached first level of notification' do
+        before do
+          namespace.update_attribute(:last_ci_minutes_usage_notification_level, 50)
+        end
+
+        it 'shows the partial usage message without the purchase link' do
+          expect(subject).to match("#{namespace.name} has less than 50% of CI minutes available.")
+          expect(subject).not_to match('to purchase more minutes')
+        end
+      end
+
+      context 'when usage is above the quota' do
+        let(:minutes_used) { 120 }
+
+        it 'shows the total usage message without the purchase link' do
+          expect(subject).to match("#{namespace.name} has exceeded its pipeline minutes quota.")
+          expect(subject).not_to match('to purchase more minutes')
+        end
+      end
+    end
+  end
+end

ee/spec/requests/api/namespaces_spec.rb

@@ -157,15 +157,17 @@ describe API::Namespaces do
       end
     end
 
-    context 'when namespace has a value for last_ci_minutes_notification_at' do
+    [:last_ci_minutes_notification_at, :last_ci_minutes_usage_notification_level].each do |attr|
+      context "when namespace has a value for #{attr}" do
         before do
-        group1.update_attribute(:last_ci_minutes_notification_at, Time.now)
+          group1.update_attribute(attr, Time.now)
         end
 
         it 'resets that value when assigning extra CI minutes' do
           expect do
             put api("/namespaces/#{group1.full_path}", admin), params: { plan: 'silver', extra_shared_runners_minutes_limit: 1000 }
-        end.to change { group1.reload.last_ci_minutes_notification_at }.to(nil)
+          end.to change { group1.reload.send(attr) }.to(nil)
+        end
       end
     end
   end

ee/spec/services/ci_minutes_usage_notify_service_spec.rb

@@ -16,19 +16,13 @@ describe CiMinutesUsageNotifyService do
   shared_examples 'namespace with all CI minutes used' do
     context 'when usage is over the quote' do
       it 'sends the email to the owner' do
-        expect(CiMinutesUsageMailer).to receive(:notify).once.with(namespace.name, user.email).and_return(spy)
+        expect(CiMinutesUsageMailer).to receive(:notify).once.with(namespace.name, [user.email]).and_return(spy)
 
         subject
       end
     end
   end
 
-  describe '#execute' do
-    let(:extra_ci_minutes) { 0 }
-    let(:namespace) do
-      create(:namespace, shared_runners_minutes_limit: 2000, extra_shared_runners_minutes_limit: extra_ci_minutes)
-    end
-
   let(:project) { create(:project, namespace: namespace) }
   let(:user) { create(:user) }
   let(:user_2) { create(:user) }
@@ -38,6 +32,12 @@ describe CiMinutesUsageNotifyService do
     create(:namespace_statistics, namespace: namespace, shared_runners_seconds: ci_minutes_used * 60)
   end
 
+  describe '#execute' do
+    let(:extra_ci_minutes) { 0 }
+    let(:namespace) do
+      create(:namespace, shared_runners_minutes_limit: 2000, extra_shared_runners_minutes_limit: extra_ci_minutes)
+    end
+
     subject { described_class.new(project).execute }
 
     context 'with a personal namespace' do
@@ -99,8 +99,9 @@ describe CiMinutesUsageNotifyService do
           let(:ci_minutes_used) { 2001 }
 
           it 'sends the email to all the owners' do
-            expect(CiMinutesUsageMailer).to receive(:notify).with(namespace.name, user.email).and_return(spy)
-            expect(CiMinutesUsageMailer).to receive(:notify).with(namespace.name, user_2.email).and_return(spy)
+            expect(CiMinutesUsageMailer).to receive(:notify)
+              .with(namespace.name, [user_2.email, user.email])
+              .and_return(spy)
 
             subject
           end
@@ -120,4 +121,72 @@ describe CiMinutesUsageNotifyService do
       end
     end
   end
+
+  describe 'CI usage limit approaching' do
+    let(:namespace) { create(:group, shared_runners_minutes_limit: 2000) }
+
+    def notify_owners
+      described_class.new(project).execute
+    end
+
+    shared_examples 'no notification is sent' do
+      it 'does not notify owners' do
+        expect(CiMinutesUsageMailer).not_to receive(:notify_limit)
+
+        notify_owners
+      end
+    end
+
+    shared_examples 'notification for custom level is sent' do |minutes_used, expected_level|
+      before do
+        namespace_statistics.update_attribute(:shared_runners_seconds, minutes_used * 60)
+      end
+
+      it 'notifies the the owners about it' do
+        expect(CiMinutesUsageMailer).to receive(:notify_limit)
+          .with(namespace.name, array_including(user_2.email, user.email), expected_level)
+          .and_call_original
+
+        notify_owners
+      end
+    end
+
+    before do
+      stub_const("EE::Namespace::CI_USAGE_ALERT_LEVELS", [30, 5])
+
+      namespace.add_owner(user)
+      namespace.add_owner(user_2)
+    end
+
+    context 'when available minutes are above notification levels' do
+      let(:ci_minutes_used) { 1000 }
+
+      it_behaves_like 'no notification is sent'
+    end
+
+    context 'when available minutes have reached the first level of alert' do
+      it_behaves_like 'notification for custom level is sent', 1500, 30
+
+      context 'when other Pipeline has finished but second level of alert has not been reached' do
+        before do
+          namespace_statistics.update_attribute(:shared_runners_seconds, 1500 * 60)
+          notify_owners
+
+          namespace_statistics.update_attribute(:shared_runners_seconds, 1600 * 60)
+        end
+
+        it_behaves_like 'no notification is sent'
+      end
+    end
+
+    context 'when available minutes have reached the second level of alert' do
+      it_behaves_like 'notification for custom level is sent', 1500, 30
+
+      it_behaves_like 'notification for custom level is sent', 1980, 5
+    end
+
+    context 'when there are not available minutes to use' do
+      include_examples 'no notification is sent'
+    end
+  end
 end

ee/spec/workers/build_finished_worker_spec.rb

@@ -37,7 +37,7 @@ describe BuildFinishedWorker do
       namespace.update_attribute(:shared_runners_minutes_limit, 2000)
       namespace_stats.update_attribute(:shared_runners_seconds, 2100 * 60)
 
-      expect(CiMinutesUsageMailer).to receive(:notify).once.with(namespace.name, namespace.owner.email).and_return(spy)
+      expect(CiMinutesUsageMailer).to receive(:notify).once.with(namespace.name, [namespace.owner.email]).and_return(spy)
 
       subject
     end

ee/spec/workers/clear_shared_runners_minutes_worker_spec.rb

@@ -90,6 +90,22 @@ describe ClearSharedRunnersMinutesWorker do
           end
         end
       end
+
+      [:last_ci_minutes_notification_at, :last_ci_minutes_usage_notification_level].each do |attr|
+        context "when #{attr} is present" do
+          before do
+            namespace.update_attribute(attr, Time.now)
+          end
+
+          it 'nullifies the field' do
+            expect(namespace.send(attr)).to be_present
+
+            subject
+
+            expect(namespace.reload.send(attr)).not_to be_present
+          end
+        end
+      end
     end
   end
 end

locale/gitlab.pot

@@ -9848,6 +9848,12 @@ msgstr ""
 msgid "Pipelines| to purchase more minutes."
 msgstr ""
 
+msgid "Pipelines|%{namespace_name} has exceeded its pipeline minutes quota."
+msgstr ""
+
+msgid "Pipelines|%{namespace_name} has less than %{notification_level}%% of CI minutes available."
+msgstr ""
+
 msgid "Pipelines|API"
 msgstr ""