Merge branch '332696-db-reindeixng-grafana-app-settings' into 'master'

Add DB reindexing Grafana config as application settings

See merge request gitlab-org/gitlab!82348

app/models/application_setting.rb

@@ -609,6 +609,7 @@ class ApplicationSetting < ApplicationRecord
   attr_encrypted :cloud_license_auth_token, encryption_options_base_32_aes_256_gcm
   attr_encrypted :external_pipeline_validation_service_token, encryption_options_base_32_aes_256_gcm
   attr_encrypted :mailgun_signing_key, encryption_options_base_32_aes_256_gcm.merge(encode: false)
+  attr_encrypted :database_grafana_api_key, encryption_options_base_32_aes_256_gcm.merge(encode: false, encode_iv: false)
 
   validates :disable_feed_token,
             inclusion: { in: [true, false], message: _('must be a boolean value') }

db/migrate/20220310011530_add_database_grafana_config_to_application_settings.rb

+# frozen_string_literal: true
+
+class AddDatabaseGrafanaConfigToApplicationSettings < Gitlab::Database::Migration[1.0]
+  # rubocop:disable Migration/AddLimitToTextColumns
+  # limit is added in 20220307002607_add_text_limit_to_db_reindexing_grafana_application_settings
+  def change
+    add_column :application_settings, :encrypted_database_grafana_api_key, :binary
+    add_column :application_settings, :encrypted_database_grafana_api_key_iv, :binary
+    add_column :application_settings, :database_grafana_api_url, :text
+    add_column :application_settings, :database_grafana_tag, :text
+  end
+  # rubocop:enable Migration/AddLimitToTextColumns
+end

db/migrate/20220310011613_add_text_limit_to_database_grafana_application_settings.rb

+# frozen_string_literal: true
+
+class AddTextLimitToDatabaseGrafanaApplicationSettings < Gitlab::Database::Migration[1.0]
+  disable_ddl_transaction!
+
+  def up
+    add_text_limit :application_settings, :database_grafana_api_url, 255
+    add_text_limit :application_settings, :database_grafana_tag, 255
+  end
+
+  def down
+    remove_text_limit :application_settings, :database_grafana_tag
+    remove_text_limit :application_settings, :database_grafana_api_url
+  end
+end

db/schema_migrations/20220310011530

+a4245a3543796b48f16786e9c178f70d236b3ae4636661f021ad4e8f0c678f2c
\ No newline at end of file

db/schema_migrations/20220310011613

+ef816d9391d67a34121d11e6b6cc37de92768bd21bc301fa10c6652b1a0b66b6
\ No newline at end of file

db/structure.sql

@@ -11253,6 +11253,10 @@ CREATE TABLE application_settings (
     container_registry_expiration_policies_caching boolean DEFAULT true NOT NULL,
     search_rate_limit integer DEFAULT 30 NOT NULL,
     search_rate_limit_unauthenticated integer DEFAULT 10 NOT NULL,
+    encrypted_database_grafana_api_key bytea,
+    encrypted_database_grafana_api_key_iv bytea,
+    database_grafana_api_url text,
+    database_grafana_tag text,
     CONSTRAINT app_settings_container_reg_cleanup_tags_max_list_size_positive CHECK ((container_registry_cleanup_tags_service_max_list_size >= 0)),
     CONSTRAINT app_settings_dep_proxy_ttl_policies_worker_capacity_positive CHECK ((dependency_proxy_ttl_group_policy_worker_capacity >= 0)),
     CONSTRAINT app_settings_ext_pipeline_validation_service_url_text_limit CHECK ((char_length(external_pipeline_validation_service_url) <= 255)),
@@ -11261,8 +11265,10 @@ CREATE TABLE application_settings (
     CONSTRAINT app_settings_yaml_max_depth_positive CHECK ((max_yaml_depth > 0)),
     CONSTRAINT app_settings_yaml_max_size_positive CHECK ((max_yaml_size_bytes > 0)),
     CONSTRAINT check_17d9558205 CHECK ((char_length((kroki_url)::text) <= 1024)),
+    CONSTRAINT check_2b820eaac3 CHECK ((char_length(database_grafana_tag) <= 255)),
     CONSTRAINT check_2dba05b802 CHECK ((char_length(gitpod_url) <= 255)),
     CONSTRAINT check_32710817e9 CHECK ((char_length(static_objects_external_storage_auth_token_encrypted) <= 255)),
+    CONSTRAINT check_3455368420 CHECK ((char_length(database_grafana_api_url) <= 255)),
     CONSTRAINT check_3559645ae5 CHECK ((char_length(container_registry_import_target_plan) <= 255)),
     CONSTRAINT check_3def0f1829 CHECK ((char_length(sentry_clientside_dsn) <= 255)),
     CONSTRAINT check_4f8b811780 CHECK ((char_length(sentry_dsn) <= 255)),

lib/gitlab/database/reindexing/grafana_notifier.rb

@@ -5,10 +5,10 @@ module Gitlab
     module Reindexing
       # This can be used to send annotations for reindexing to a Grafana API
       class GrafanaNotifier
-        def initialize(api_key = ENV['GITLAB_GRAFANA_API_KEY'], api_url = ENV['GITLAB_GRAFANA_API_URL'], additional_tag = ENV['GITLAB_REINDEXING_GRAFANA_TAG'] || Rails.env)
-          @api_key = api_key
-          @api_url = api_url
-          @additional_tag = additional_tag
+        def initialize(api_key: nil, api_url: nil, additional_tag: nil)
+          @api_key = api_key || default_api_key
+          @api_url = api_url || default_api_url
+          @additional_tag = additional_tag || default_additional_tag
         end
 
         def notify_start(action)
@@ -35,10 +35,22 @@ module Gitlab
 
         private
 
+        def default_api_key
+          Gitlab::CurrentSettings.database_grafana_api_key || ENV['GITLAB_GRAFANA_API_KEY']
+        end
+
+        def default_api_url
+          Gitlab::CurrentSettings.database_grafana_api_url || ENV['GITLAB_GRAFANA_API_URL']
+        end
+
+        def default_additional_tag
+          Gitlab::CurrentSettings.database_grafana_tag || ENV['GITLAB_REINDEXING_GRAFANA_TAG'] || Rails.env
+        end
+
         def base_payload(action)
           {
             time: (action.action_start.utc.to_f * 1000).to_i,
-            tags: ['reindex', @additional_tag, action.index.tablename, action.index.name].compact
+            tags: ['reindex', @additional_tag.presence, action.index.tablename, action.index.name].compact
           }
         end
 

spec/lib/gitlab/database/reindexing/grafana_notifier_spec.rb

@@ -74,8 +74,28 @@ RSpec.describe Gitlab::Database::Reindexing::GrafanaNotifier do
   end
 
   describe '#notify_start' do
-    context 'additional tag is nil' do
-      subject { described_class.new(api_key, api_url, nil).notify_start(action) }
+    context 'when Grafana is configured using application settings' do
+      subject { described_class.new.notify_start(action) }
+
+      let(:payload) do
+        {
+          time: (action.action_start.utc.to_f * 1000).to_i,
+          tags: ['reindex', additional_tag, action.index.tablename, action.index.name],
+          text: "Started reindexing of #{action.index.name} on #{action.index.tablename}"
+        }
+      end
+
+      before do
+        stub_application_setting(database_grafana_api_key: api_key)
+        stub_application_setting(database_grafana_api_url: api_url)
+        stub_application_setting(database_grafana_tag: additional_tag)
+      end
+
+      it_behaves_like 'interacting with Grafana annotations API'
+    end
+
+    context 'when there is no additional tag' do
+      subject { described_class.new(api_key: api_key, api_url: api_url, additional_tag: '').notify_start(action) }
 
       let(:payload) do
         {
@@ -88,8 +108,8 @@ RSpec.describe Gitlab::Database::Reindexing::GrafanaNotifier do
       it_behaves_like 'interacting with Grafana annotations API'
     end
 
-    context 'additional tag is not nil' do
-      subject { described_class.new(api_key, api_url, additional_tag).notify_start(action) }
+    context 'additional tag is provided' do
+      subject { described_class.new(api_key: api_key, api_url: api_url, additional_tag: additional_tag).notify_start(action) }
 
       let(:payload) do
         {
@@ -104,8 +124,30 @@ RSpec.describe Gitlab::Database::Reindexing::GrafanaNotifier do
   end
 
   describe '#notify_end' do
-    context 'additional tag is nil' do
-      subject { described_class.new(api_key, api_url, nil).notify_end(action) }
+    context 'when Grafana is configured using application settings' do
+      subject { described_class.new.notify_end(action) }
+
+      let(:payload) do
+        {
+          time: (action.action_start.utc.to_f * 1000).to_i,
+          tags: ['reindex', additional_tag, action.index.tablename, action.index.name],
+          text: "Finished reindexing of #{action.index.name} on #{action.index.tablename} (#{action.state})",
+          timeEnd: (action.action_end.utc.to_f * 1000).to_i,
+          isRegion: true
+        }
+      end
+
+      before do
+        stub_application_setting(database_grafana_api_key: api_key)
+        stub_application_setting(database_grafana_api_url: api_url)
+        stub_application_setting(database_grafana_tag: additional_tag)
+      end
+
+      it_behaves_like 'interacting with Grafana annotations API'
+    end
+
+    context 'when there is no additional tag' do
+      subject { described_class.new(api_key: api_key, api_url: api_url, additional_tag: '').notify_end(action) }
 
       let(:payload) do
         {
@@ -120,8 +162,8 @@ RSpec.describe Gitlab::Database::Reindexing::GrafanaNotifier do
       it_behaves_like 'interacting with Grafana annotations API'
     end
 
-    context 'additional tag is not nil' do
-      subject { described_class.new(api_key, api_url, additional_tag).notify_end(action) }
+    context 'additional tag is provided' do
+      subject { described_class.new(api_key: api_key, api_url: api_url, additional_tag: additional_tag).notify_end(action) }
 
       let(:payload) do
         {

spec/models/application_setting_spec.rb

@@ -1306,4 +1306,16 @@ RSpec.describe ApplicationSetting do
       end
     end
   end
+
+  describe '#database_grafana_api_key' do
+    it 'is encrypted' do
+      subject.database_grafana_api_key = 'somesecret'
+
+      aggregate_failures do
+        expect(subject.encrypted_database_grafana_api_key).to be_present
+        expect(subject.encrypted_database_grafana_api_key_iv).to be_present
+        expect(subject.encrypted_database_grafana_api_key).not_to eq(subject.database_grafana_api_key)
+      end
+    end
+  end
 end