Merge branch '328466-add-username-password-to-es-settings' into 'master'

Add username & password fields for Advanced Search

See merge request gitlab-org/gitlab!60710

app/models/application_setting.rb

@@ -529,6 +529,7 @@ class ApplicationSetting < ApplicationRecord
   attr_encrypted :akismet_api_key, encryption_options_base_32_aes_256_gcm
   attr_encrypted :spam_check_api_key, encryption_options_base_32_aes_256_gcm.merge(encode: false)
   attr_encrypted :elasticsearch_aws_secret_access_key, encryption_options_base_32_aes_256_gcm
+  attr_encrypted :elasticsearch_password, encryption_options_base_32_aes_256_gcm.merge(encode: false)
   attr_encrypted :recaptcha_private_key, encryption_options_base_32_aes_256_gcm
   attr_encrypted :recaptcha_site_key, encryption_options_base_32_aes_256_gcm
   attr_encrypted :slack_app_secret, encryption_options_base_32_aes_256_gcm

changelogs/unreleased/328466-add-username-password-to-es-settings.yml

+---
+title: Add username and password fields for Advanced Search
+merge_request: 60710
+author:
+type: changed

config/application.rb

@@ -145,6 +145,7 @@ module Gitlab
       encrypted_key
       import_url
       elasticsearch_url
+      elasticsearch_password
       search
       jwt
       otp_attempt

db/migrate/20210430100947_add_elasticsearch_username_password_to_application_settings.rb

+# frozen_string_literal: true
+
+class AddElasticsearchUsernamePasswordToApplicationSettings < ActiveRecord::Migration[6.0]
+  def change
+    # rubocop:disable Migration/AddLimitToTextColumns
+    # limit is added in 20210505124816_add_text_limit_to_elasticsearch_username
+    add_column :application_settings, :elasticsearch_username, :text
+    # rubocop:enable Migration/AddLimitToTextColumns
+
+    add_column :application_settings, :encrypted_elasticsearch_password, :binary
+    add_column :application_settings, :encrypted_elasticsearch_password_iv, :binary
+  end
+end

db/migrate/20210505124816_add_text_limit_to_elasticsearch_username.rb

+# frozen_string_literal: true
+
+class AddTextLimitToElasticsearchUsername < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  def up
+    add_text_limit :application_settings, :elasticsearch_username, 255
+  end
+
+  def down
+    remove_text_limit :application_settings, :elasticsearch_username
+  end
+end

db/schema_migrations/20210430100947

+c8875e02134542370cc5a792bdaefc77b66f58a33a46720f5ef562c33c5b8f41
\ No newline at end of file

db/schema_migrations/20210505124816

+56aa9590f4bc37d8f8c4ed869a4b095ba39925fb06ab58500eead895d19ee336
\ No newline at end of file

db/structure.sql

@@ -9514,6 +9514,9 @@ CREATE TABLE application_settings (
     encrypted_spam_check_api_key bytea,
     encrypted_spam_check_api_key_iv bytea,
     floc_enabled boolean DEFAULT false NOT NULL,
+    elasticsearch_username text,
+    encrypted_elasticsearch_password bytea,
+    encrypted_elasticsearch_password_iv bytea,
     CONSTRAINT app_settings_container_reg_cleanup_tags_max_list_size_positive CHECK ((container_registry_cleanup_tags_service_max_list_size >= 0)),
     CONSTRAINT app_settings_ext_pipeline_validation_service_url_text_limit CHECK ((char_length(external_pipeline_validation_service_url) <= 255)),
     CONSTRAINT app_settings_registry_exp_policies_worker_capacity_positive CHECK ((container_registry_expiration_policies_worker_capacity >= 0)),
@@ -9529,6 +9532,7 @@ CREATE TABLE application_settings (
     CONSTRAINT check_a5704163cc CHECK ((char_length(secret_detection_revocation_token_types_url) <= 255)),
     CONSTRAINT check_d03919528d CHECK ((char_length(container_registry_vendor) <= 255)),
     CONSTRAINT check_d820146492 CHECK ((char_length(spam_check_endpoint_url) <= 255)),
+    CONSTRAINT check_e5024c8801 CHECK ((char_length(elasticsearch_username) <= 255)),
     CONSTRAINT check_e5aba18f02 CHECK ((char_length(container_registry_version) <= 255)),
     CONSTRAINT check_ef6176834f CHECK ((char_length(encrypted_cloud_license_auth_token_iv) <= 255))
 );

doc/api/settings.md

@@ -277,7 +277,9 @@ listed in the descriptions of the relevant settings.
 | `elasticsearch_namespace_ids`            | array of integers | no                                  | **(PREMIUM)** The namespaces to index via Elasticsearch if `elasticsearch_limit_indexing` is enabled. |
 | `elasticsearch_project_ids`              | array of integers | no                                  | **(PREMIUM)** The projects to index via Elasticsearch if `elasticsearch_limit_indexing` is enabled. |
 | `elasticsearch_search`                   | boolean          | no                                   | **(PREMIUM)** Enable Elasticsearch search. |
-| `elasticsearch_url`                      | string           | no                                   | **(PREMIUM)** The URL to use for connecting to Elasticsearch. Use a comma-separated list to support cluster (for example, `http://localhost:9200, http://localhost:9201"`). If your Elasticsearch instance is password protected, pass the `username:password` in the URL (for example, `http://<username>:<password>@<elastic_host>:9200/`). |
+| `elasticsearch_url`                      | string           | no                                   | **(PREMIUM)** The URL to use for connecting to Elasticsearch. Use a comma-separated list to support cluster (for example, `http://localhost:9200, http://localhost:9201"`). |
+| `elasticsearch_username`                 | string           | no                                   | **(PREMIUM)** The `username` of your Elasticsearch instance. |
+| `elasticsearch_password`                 | string           | no                                   | **(PREMIUM)** The password of your Elasticsearch instance. |
 | `email_additional_text`                  | string           | no                                   | **(PREMIUM)** Additional text added to the bottom of every email for legal/auditing/compliance reasons. |
 | `email_author_in_body`                   | boolean          | no                                   | Some email servers do not support overriding the email sender name. Enable this option to include the name of the author of the issue, merge request or comment in the email body instead. |
 | `enabled_git_access_protocol`            | string           | no                                   | Enabled protocols for Git access. Allowed values are: `ssh`, `http`, and `nil` to allow both protocols. |

doc/integration/elasticsearch.md

@@ -229,7 +229,9 @@ The following Elasticsearch settings are available:
 | `Elasticsearch indexing`                              | Enables or disables Elasticsearch indexing and creates an empty index if one does not already exist. You may want to enable indexing but disable search in order to give the index time to be fully completed, for example. Also, keep in mind that this option doesn't have any impact on existing data, this only enables/disables the background indexer which tracks data changes and ensures new data is indexed. |
 | `Pause Elasticsearch indexing`                        | Enables or disables temporary indexing pause. This is useful for cluster migration/reindexing. All changes are still tracked, but they are not committed to the Elasticsearch index until resumed. |
 | `Search with Elasticsearch enabled`                   | Enables or disables using Elasticsearch in search. |
-| `URL`                                                 | The URL to use for connecting to Elasticsearch. Use a comma-separated list to support clustering (e.g., `http://host1, https://host2:9200`). If your Elasticsearch instance is password protected, pass the `username:password` in the URL (e.g., `http://<username>:<password>@<elastic_host>:9200/`). Special characters in the username or password should use [percentage encoding](https://en.wikipedia.org/wiki/Percent-encoding). |
+| `URL`                                                 | The URL of your Elasticsearch instance. Use a comma-separated list to support clustering (for example, `http://host1, https://host2:9200`). If your Elasticsearch instance is password-protected, use the `Username` and `Password` fields described below. Alternatively, use inline credentials such as `http://<username>:<password>@<elastic_host>:9200/`. |
+| `Username`                                                 | The `username` of your Elasticsearch instance. |
+| `Password`                                                 | The password of your Elasticsearch instance. |
 | `Number of Elasticsearch shards`                      | Elasticsearch indexes are split into multiple shards for performance reasons. In general, you should use at least 5 shards, and indexes with tens of millions of documents need to have more shards ([see below](#guidance-on-choosing-optimal-cluster-configuration)). Changes to this value do not take effect until the index is recreated. You can read more about tradeoffs in the [Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/scalability.html). |
 | `Number of Elasticsearch replicas`                    | Each Elasticsearch shard can have a number of replicas. These are a complete copy of the shard, and can provide increased query performance or resilience against hardware failure. Increasing this value will greatly increase total disk space required by the index. |
 | `Limit namespaces and projects that can be indexed`   | Enabling this will allow you to select namespaces and projects to index. All other namespaces and projects will use database search instead. Please note that if you enable this option but do not select any namespaces or projects, none will be indexed. [Read more below](#limiting-namespaces-and-projects).

ee/app/helpers/ee/application_settings_helper.rb

@@ -40,6 +40,8 @@ module EE
         :elasticsearch_search,
         :elasticsearch_shards,
         :elasticsearch_url,
+        :elasticsearch_username,
+        :elasticsearch_password,
         :elasticsearch_limit_indexing,
         :elasticsearch_namespace_ids,
         :elasticsearch_project_ids,

ee/app/models/ee/application_setting.rb

@@ -16,6 +16,7 @@ module EE
 
       EMAIL_ADDITIONAL_TEXT_CHARACTER_LIMIT = 10_000
       DEFAULT_NUMBER_OF_DAYS_BEFORE_REMOVAL = 7
+      MASK_PASSWORD = '*****'
 
       belongs_to :file_template_project, class_name: "Project"
 
@@ -56,6 +57,9 @@ module EE
                 presence: { message: "can't be blank when indexing is enabled" },
                 if: ->(setting) { setting.elasticsearch_indexing? }
 
+      validates :elasticsearch_username, length: { maximum: 255 }
+      validates :elasticsearch_password, length: { maximum: 255 }
+
       validates :secret_detection_revocation_token_types_url,
                 presence: { message: "can't be blank when secret detection token revocation is enabled" },
                 if: ->(setting) { setting.secret_detection_token_revocation_enabled? }
@@ -140,6 +144,8 @@ module EE
           elasticsearch_max_bulk_concurrency: 10,
           elasticsearch_max_bulk_size_bytes: 10.megabytes,
           elasticsearch_url: ENV['ELASTIC_URL'] || 'http://localhost:9200',
+          elasticsearch_username: nil,
+          elasticsearch_password: nil,
           elasticsearch_client_request_timeout: 0,
           elasticsearch_analyzers_smartcn_enabled: false,
           elasticsearch_analyzers_smartcn_search: false,
@@ -304,9 +310,27 @@ module EE
       write_attribute(:elasticsearch_url, cleaned.join(','))
     end
 
+    def elasticsearch_password=(value)
+      return if value == MASK_PASSWORD
+
+      super
+    end
+
+    def elasticsearch_url_with_credentials
+      return elasticsearch_url if elasticsearch_username.blank?
+
+      elasticsearch_url.map do |url|
+        uri = URI.parse(url)
+
+        uri.user = elasticsearch_username
+        uri.password = elasticsearch_password.presence || ''
+        uri.to_s
+      end
+    end
+
     def elasticsearch_config
       {
-        url:                    elasticsearch_url,
+        url:                    elasticsearch_url_with_credentials,
         aws:                    elasticsearch_aws,
         aws_access_key:         elasticsearch_aws_access_key,
         aws_secret_access_key:  elasticsearch_aws_secret_access_key,

ee/app/views/admin/application_settings/_elasticsearch_form.html.haml

@@ -70,6 +70,13 @@
               .form-text.gl-text-gray-600.gl-mt-0
                 = _('The URL to use for connecting to Elasticsearch. Use a comma-separated list to support clustering (e.g., "http://localhost:9200, http://localhost:9201").')
 
+            .form-group
+              = f.label :elasticsearch_username, _('Username (for password-protected Elasticsearch servers)'), class: 'label-bold'
+              = f.text_field :elasticsearch_username, value: @application_setting.elasticsearch_username, class: 'form-control gl-form-input', data: { qa_selector: 'username_field' }
+            .form-group
+              = f.label :elasticsearch_password, _('Password (for password-protected Elasticsearch servers)'), class: 'label-bold'
+              = f.password_field :elasticsearch_password, value: (@application_setting.elasticsearch_password.present? ? ApplicationSetting::MASK_PASSWORD : ''), class: 'form-control gl-form-input', data: { qa_selector: 'password_field' }
+
             .form-group
               = f.label :elasticsearch_shards, _('Number of Elasticsearch shards and replicas (per index)'), class: 'gl-font-weight-bold'
 

ee/lib/ee/api/helpers/settings_helpers.rb

@@ -22,6 +22,8 @@ module EE
               optional :elasticsearch_search, type: Grape::API::Boolean, desc: 'Enable Elasticsearch search'
               optional :elasticsearch_pause_indexing, type: Grape::API::Boolean, desc: 'Pause Elasticsearch indexing'
               requires :elasticsearch_url, type: String, desc: 'The url to use for connecting to Elasticsearch. Use a comma-separated list to support clustering (e.g., "http://localhost:9200, http://localhost:9201")'
+              optional :elasticsearch_username, type: String, desc: 'The username of your Elasticsearch instance.'
+              optional :elasticsearch_password, type: String, desc: 'The password of your Elasticsearch instance.'
               optional :elasticsearch_limit_indexing, type: Grape::API::Boolean, desc: 'Limit Elasticsearch to index certain namespaces and projects'
             end
 

ee/spec/models/application_setting_spec.rb

@@ -59,6 +59,10 @@ RSpec.describe ApplicationSetting do
     it { is_expected.not_to allow_value(1.1).for(:elasticsearch_client_request_timeout) }
     it { is_expected.not_to allow_value(-1).for(:elasticsearch_client_request_timeout) }
 
+    it { is_expected.to allow_value('').for(:elasticsearch_username) }
+    it { is_expected.to allow_value('a' * 255).for(:elasticsearch_username) }
+    it { is_expected.not_to allow_value('a' * 256).for(:elasticsearch_username) }
+
     it { is_expected.to allow_value(nil).for(:required_instance_ci_template) }
     it { is_expected.not_to allow_value("").for(:required_instance_ci_template) }
     it { is_expected.not_to allow_value("  ").for(:required_instance_ci_template) }
@@ -318,10 +322,56 @@ RSpec.describe ApplicationSetting do
     end
   end
 
+  describe '#elasticsearch_url_with_credentials' do
+    it 'embeds credentials in the result' do
+      setting.elasticsearch_url = 'http://example.com,https://example2.com:9200'
+      setting.elasticsearch_username = 'foo'
+      setting.elasticsearch_password = 'bar'
+
+      expect(setting.elasticsearch_url_with_credentials).to eq(%w[http://foo:bar@example.com https://foo:bar@example2.com:9200])
+    end
+
+    it 'embeds username only' do
+      setting.elasticsearch_url = 'http://example.com,https://example2.com:9200'
+      setting.elasticsearch_username = 'foo'
+      setting.elasticsearch_password = ''
+
+      expect(setting.elasticsearch_url_with_credentials).to eq(%w[http://foo:@example.com https://foo:@example2.com:9200])
+    end
+
+    it 'overrides existing embedded credentials' do
+      setting.elasticsearch_url = 'http://username:password@example.com,https://test:test@example2.com:9200'
+      setting.elasticsearch_username = 'foo'
+      setting.elasticsearch_password = 'bar'
+
+      expect(setting.elasticsearch_url_with_credentials).to eq(%w[http://foo:bar@example.com https://foo:bar@example2.com:9200])
+    end
+
+    it 'returns original url if credentials blank' do
+      setting.elasticsearch_url = 'http://username:password@example.com,https://test:test@example2.com:9200'
+      setting.elasticsearch_username = ''
+      setting.elasticsearch_password = ''
+
+      expect(setting.elasticsearch_url_with_credentials).to eq(%w[http://username:password@example.com https://test:test@example2.com:9200])
+    end
+  end
+
+  describe '#elasticsearch_password' do
+    it 'does not modify password if it is unchanged in the form' do
+      setting.elasticsearch_password = 'foo'
+
+      setting.elasticsearch_password = ApplicationSetting::MASK_PASSWORD
+
+      expect(setting.elasticsearch_password).to eq('foo')
+    end
+  end
+
   describe '#elasticsearch_config' do
     it 'places all elasticsearch configuration values into a hash' do
       setting.update!(
         elasticsearch_url: 'http://example.com:9200',
+        elasticsearch_username: 'foo',
+        elasticsearch_password: 'bar',
         elasticsearch_aws: false,
         elasticsearch_aws_region:     'test-region',
         elasticsearch_aws_access_key: 'test-access-key',
@@ -332,7 +382,7 @@ RSpec.describe ApplicationSetting do
       )
 
       expect(setting.elasticsearch_config).to eq(
-        url: ['http://example.com:9200'],
+        url: ['http://foo:bar@example.com:9200'],
         aws: false,
         aws_region:     'test-region',
         aws_access_key: 'test-access-key',

locale/gitlab.pot

@@ -23650,6 +23650,9 @@ msgstr ""
 msgid "Password"
 msgstr ""
 
+msgid "Password (for password-protected Elasticsearch servers)"
+msgstr ""
+
 msgid "Password (optional)"
 msgstr ""
 
@@ -35462,6 +35465,9 @@ msgstr ""
 msgid "UserProfile|made a private contribution"
 msgstr ""
 
+msgid "Username (for password-protected Elasticsearch servers)"
+msgstr ""
+
 msgid "Username (optional)"
 msgstr ""