Enable elasticsearch per project or group

In order to allow us to incrementally enable ES on gitlab.com we add two
 new classes:

- ElasticsearchIndexedNamespaces
- ElasticsearchIndexedProjects

These classes are used by ApplicationSetting to enable/disable projects
and namespaces (groups) that should be indexed by elasticsearch.

We also have the application setting, `elasticsearch_limit_indexing`,
that enables/disables the new functionality

In order to be able to selectively choose projects/namespaces to use
with elasticsearch, `elasticsearch_limit_indexing` MUST be enabled under
the admin integrations options.

app/controllers/search_controller.rb

@@ -14,8 +14,6 @@ class SearchController < ApplicationController
   layout 'search'
 
   def show
-    search_service = SearchService.new(current_user, params)
-
     @project = search_service.project
     @group = search_service.group
 

app/helpers/search_helper.rb

@@ -50,6 +50,10 @@ module SearchHelper
     filename
   end
 
+  def search_service
+    @search_service ||= ::SearchService.new(current_user, params)
+  end
+
   private
 
   # Autocomplete results for various settings pages

app/services/search_service.rb

@@ -69,3 +69,5 @@ class SearchService
 
   attr_reader :current_user, :params
 end
+
+SearchService.prepend(EE::SearchService)

app/views/search/_category.html.haml

@@ -87,7 +87,7 @@
           = _("Milestones")
           %span.badge.badge-pill
             = limited_count(@search_results.limited_milestones_count)
-      - if Gitlab::CurrentSettings.elasticsearch_search?
+      - if search_service.use_elasticsearch?
         %li{ class: active_when(@scope == 'blobs') }
           = link_to search_filter_path(scope: 'blobs') do
             = _("Code")

app/views/search/_form.html.haml

@@ -13,7 +13,7 @@
     - unless params[:snippets].eql? 'true'
       = render 'filter'
     = button_tag _("Search"), class: "btn btn-success btn-search"
-  - if Gitlab::CurrentSettings.elasticsearch_search?
+  - if search_service.use_elasticsearch?
     .form-text.text-muted
-      = link_to 'Advanced search functionality', help_page_path('user/search/advanced_search_syntax.md'), target: '_blank'
+      = link_to _('Advanced search functionality'), help_page_path('user/search/advanced_search_syntax.md'), target: '_blank'
       is enabled.

config/sidekiq_queues.yml

@@ -103,6 +103,7 @@
   - [elastic_batch_project_indexer, 1]
   - [elastic_indexer, 1]
   - [elastic_commit_indexer, 1]
+  - [elastic_namespace_indexer, 1]
   - [export_csv, 1]
   - [incident_management, 2]
 

db/schema.rb

@@ -217,6 +217,7 @@ ActiveRecord::Schema.define(version: 20190322132835) do
     t.string "runners_registration_token_encrypted"
     t.integer "local_markdown_version", default: 0, null: false
     t.integer "first_day_of_week", default: 0, null: false
+    t.boolean "elasticsearch_limit_indexing", default: false, null: false
     t.index ["custom_project_templates_group_id"], name: "index_application_settings_on_custom_project_templates_group_id", using: :btree
     t.index ["file_template_project_id"], name: "index_application_settings_on_file_template_project_id", using: :btree
     t.index ["usage_stats_set_by_user_id"], name: "index_application_settings_on_usage_stats_set_by_user_id", using: :btree
@@ -1065,6 +1066,20 @@ ActiveRecord::Schema.define(version: 20190322132835) do
     t.index ["merge_request_id"], name: "index_draft_notes_on_merge_request_id", using: :btree
   end
 
+  create_table "elasticsearch_indexed_namespaces", id: false, force: :cascade do |t|
+    t.datetime_with_timezone "created_at", null: false
+    t.datetime_with_timezone "updated_at", null: false
+    t.integer "namespace_id"
+    t.index ["namespace_id"], name: "index_elasticsearch_indexed_namespaces_on_namespace_id", unique: true, using: :btree
+  end
+
+  create_table "elasticsearch_indexed_projects", id: false, force: :cascade do |t|
+    t.datetime_with_timezone "created_at", null: false
+    t.datetime_with_timezone "updated_at", null: false
+    t.integer "project_id"
+    t.index ["project_id"], name: "index_elasticsearch_indexed_projects_on_project_id", unique: true, using: :btree
+  end
+
   create_table "emails", force: :cascade do |t|
     t.integer "user_id", null: false
     t.string "email", null: false
@@ -3456,6 +3471,8 @@ ActiveRecord::Schema.define(version: 20190322132835) do
   add_foreign_key "design_management_versions", "design_management_designs", on_delete: :cascade
   add_foreign_key "draft_notes", "merge_requests", on_delete: :cascade
   add_foreign_key "draft_notes", "users", column: "author_id", on_delete: :cascade
+  add_foreign_key "elasticsearch_indexed_namespaces", "namespaces", on_delete: :cascade
+  add_foreign_key "elasticsearch_indexed_projects", "projects", on_delete: :cascade
   add_foreign_key "environments", "projects", name: "fk_d1c8c1da6a", on_delete: :cascade
   add_foreign_key "epic_issues", "epics", on_delete: :cascade
   add_foreign_key "epic_issues", "issues", on_delete: :cascade

doc/integration/elasticsearch.md

@@ -118,11 +118,32 @@ The following Elasticsearch settings are available:
 | `Use the new repository indexer (beta)` | Perform repository indexing using [GitLab Elasticsearch Indexer](https://gitlab.com/gitlab-org/gitlab-elasticsearch-indexer). |
 | `Search with Elasticsearch enabled` | Enables/disables using Elasticsearch in search. |
 | `URL`                              | The URL to use for connecting to Elasticsearch. Use a comma-separated list to support clustering (e.g., "http://host1, https://host2:9200"). If your Elasticsearch instance is password protected, pass the `username:password` in the URL (e.g., `http://<username>:<password>@<elastic_host>:9200/`). |
+| `Limit namespaces and projects that can be indexed` | Enabling this will allow you to select namespaces and projects to index. All other namespaces and projects will use database search instead. Please note that if you enable this option but do not select any namespaces or projects, none will be indexed. [Read more below](#limiting-namespaces-and-projects).
 | `Using AWS hosted Elasticsearch with IAM credentials` | Sign your Elasticsearch requests using [AWS IAM authorization](http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html) or [AWS EC2 Instance Profile Credentials](http://docs.aws.amazon.com/codedeploy/latest/userguide/getting-started-create-iam-instance-profile.html#getting-started-create-iam-instance-profile-cli). The policies must be configured to allow `es:*` actions. |
 | `AWS Region` | The AWS region your Elasticsearch service is located in. |
 | `AWS Access Key` | The AWS access key. |
 | `AWS Secret Access Key` | The AWS secret access key. |
 
+### Limiting namespaces and projects
+
+If you select `Limit namespaces and projects that can be indexed`, more options will become available
+![limit namespaces and projects options](img/limit_namespaces_projects_options.png)
+
+You can select namespaces and projects to index exclusively. Please note that if the namespace is a group it will include
+any sub-groups and projects belonging to those sub-groups to be indexed as well.
+
+You can filter the selection dropdown by writing part of the namespace or project name you're interested in.
+![limit namespace filter](img/limit_namespace_filter.png)
+
+NOTE: **Note**:
+If no namespaces or projects are selected, no Elasticsearch indexing will take place.
+
+CAUTION: **Warning**:
+If you have already indexed your instance, you will have to regenerate the index in order to delete all existing data
+for filtering to work correctly. To do this run the rake tasks `gitlab:elastic:create_empty_index` and
+`gitlab:elastic:clear_index_status` Afterwards, removing a namespace or a projeect from the list will delete the data
+from the Elasticsearch index as expected.
+
 ## Disabling Elasticsearch
 
 To disable the Elasticsearch integration:

ee/app/assets/javascripts/pages/admin/application_settings/integrations/index.js

+import 'select2/select2';
+import $ from 'jquery';
+import { s__ } from '~/locale';
+import Api from '~/api';
+
+const onLimitCheckboxChange = (checked, $limitByNamespaces, $limitByProjects) => {
+  $limitByNamespaces.find('.select2').select2('data', null);
+  $limitByNamespaces.find('.select2').select2('data', null);
+  $limitByNamespaces.toggleClass('hidden', !checked);
+  $limitByProjects.toggleClass('hidden', !checked);
+};
+
+const getDropdownConfig = (placeholder, apiPath, textProp) => ({
+  placeholder,
+  multiple: true,
+  initSelection($el, callback) {
+    callback($el.data('selected'));
+  },
+  ajax: {
+    url: Api.buildUrl(apiPath),
+    dataType: 'JSON',
+    quietMillis: 250,
+    data(search) {
+      return {
+        search,
+      };
+    },
+    results(data) {
+      return {
+        results: data.map(entity => ({
+          id: entity.id,
+          text: entity[textProp],
+        })),
+      };
+    },
+  },
+});
+
+document.addEventListener('DOMContentLoaded', () => {
+  const $container = $('#js-elasticsearch-settings');
+
+  $container
+    .find('.js-limit-checkbox')
+    .on('change', e =>
+      onLimitCheckboxChange(
+        e.currentTarget.checked,
+        $container.find('.js-limit-namespaces'),
+        $container.find('.js-limit-projects'),
+      ),
+    );
+
+  $container
+    .find('.js-elasticsearch-namespaces')
+    .select2(
+      getDropdownConfig(
+        s__('Elastic|None. Select namespaces to index.'),
+        Api.namespacesPath,
+        'full_path',
+      ),
+    );
+
+  $container
+    .find('.js-elasticsearch-projects')
+    .select2(
+      getDropdownConfig(
+        s__('Elastic|None. Select projects to index.'),
+        Api.projectsPath,
+        'name_with_namespace',
+      ),
+    );
+});

ee/app/helpers/ee/application_settings_helper.rb

@@ -62,6 +62,9 @@ module EE
         :elasticsearch_indexing,
         :elasticsearch_search,
         :elasticsearch_url,
+        :elasticsearch_limit_indexing,
+        :elasticsearch_namespace_ids,
+        :elasticsearch_project_ids,
         :geo_status_timeout,
         :help_text,
         :pseudonymizer_enabled,
@@ -78,6 +81,18 @@ module EE
       ]
     end
 
+    def elasticsearch_objects_options(objects)
+      objects.map { |g| { id: g.id, text: g.full_name } }
+    end
+
+    def elasticsearch_namespace_ids
+      ElasticsearchIndexedNamespace.namespace_ids.join(',')
+    end
+
+    def elasticsearch_project_ids
+      ElasticsearchIndexedProject.project_ids.join(',')
+    end
+
     def self.repository_mirror_attributes
       [
         :mirror_max_capacity,

ee/app/models/concerns/elastic/application_search.rb

@@ -228,7 +228,11 @@ module Elastic
 
       # Should be overridden in the models where some records should be skipped
       def searchable?
-        true
+        self.use_elasticsearch?
+      end
+
+      def use_elasticsearch?
+        self.project&.use_elasticsearch?
       end
 
       def generic_attributes

ee/app/models/concerns/elastic/projects_search.rb

@@ -15,6 +15,10 @@ module Elastic
     included do
       include ApplicationSearch
 
+      def use_elasticsearch?
+        ::Gitlab::CurrentSettings.elasticsearch_indexes_project?(self)
+      end
+
       def as_indexed_json(options = {})
         # We don't use as_json(only: ...) because it calls all virtual and serialized attributtes
         # https://gitlab.com/gitlab-org/gitlab-ee/issues/349

ee/app/models/concerns/elastic/repositories_search.rb

@@ -25,7 +25,7 @@ module Elastic
 
       def self.import
         Project.find_each do |project|
-          if project.repository.exists? && !project.repository.empty?
+          if project.repository.exists? && !project.repository.empty? && project.use_elasticsearch?
             project.repository.index_commits
             project.repository.index_blobs
           end

ee/app/models/concerns/elastic/snippets_search.rb

@@ -32,6 +32,10 @@ module Elastic
         data
       end
 
+      def use_elasticsearch?
+        ::Gitlab::CurrentSettings.elasticsearch_indexing?
+      end
+
       def self.elastic_search(query, options: {})
         query_hash = basic_query_hash(%w(title file_name), query)
 

ee/app/models/concerns/elastic/wiki_repositories_search.rb

@@ -25,7 +25,7 @@ module Elastic
 
       def self.import
         Project.with_wiki_enabled.find_each do |project|
-          unless project.wiki.empty?
+          if project.use_elasticsearch? && !project.wiki.empty?
             project.wiki.index_blobs
           end
         end

ee/app/models/ee/application_setting.rb

@@ -14,6 +14,11 @@ module EE
       EMAIL_ADDITIONAL_TEXT_CHARACTER_LIMIT = 10_000
       INSTANCE_REVIEW_MIN_USERS = 100
 
+      attr_accessor :elasticsearch_namespace_ids, :elasticsearch_project_ids
+
+      after_save -> { update_elasticsearch_containers(ElasticsearchIndexedNamespace, :namespace_id, elasticsearch_namespace_ids) }, on: [:create, :update]
+      after_save -> { update_elasticsearch_containers(ElasticsearchIndexedProject, :project_id, elasticsearch_project_ids) }, on: [:create, :update]
+
       belongs_to :file_template_project, class_name: "Project"
 
       ignore_column :minimum_mirror_sync_time
@@ -120,6 +125,59 @@ module EE
       end
     end
 
+    def update_elasticsearch_containers(klass, attribute, container_ids)
+      return unless elasticsearch_limit_indexing?
+
+      container_ids = container_ids&.split(",")
+      return unless container_ids.present?
+
+      # Destroy any containers that have been removed. This runs callbacks, etc
+      # #rubocop:disable Cop/DestroyAll
+      klass.where.not(attribute => container_ids).each_batch do |batch, _index|
+        batch.destroy_all
+      end
+      # #rubocop:enable Cop/DestroyAll
+
+      # Disregard any duplicates that are already present
+      container_ids -= klass.pluck(attribute)
+
+      # Add new containers
+      container_ids.each { |id| klass.create(attribute => id) }
+    end
+
+    def elasticsearch_indexes_project?(project)
+      return false unless elasticsearch_indexing?
+      return true unless elasticsearch_limit_indexing?
+
+      elasticsearch_limited_projects.exists?(project.id)
+    end
+
+    def elasticsearch_indexes_namespace?(namespace)
+      return false unless elasticsearch_indexing?
+      return true unless elasticsearch_limit_indexing?
+
+      elasticsearch_limited_namespaces.exists?(namespace.id)
+    end
+
+    def elasticsearch_limited_projects(ignore_namespaces = false)
+      return ::Project.where(id: ElasticsearchIndexedProject.select(:project_id)) if ignore_namespaces
+
+      union = ::Gitlab::SQL::Union.new([
+                                         ::Project.where(namespace_id: elasticsearch_limited_namespaces.select(:id)),
+                                         ::Project.where(id: ElasticsearchIndexedProject.select(:project_id))
+                                       ]).to_sql
+
+      ::Project.from("(#{union}) projects")
+    end
+
+    def elasticsearch_limited_namespaces(ignore_descendants = false)
+      namespaces = ::Namespace.where(id: ElasticsearchIndexedNamespace.select(:namespace_id))
+
+      return namespaces if ignore_descendants
+
+      ::Gitlab::ObjectHierarchy.new(namespaces).base_and_descendants
+    end
+
     def pseudonymizer_available?
       License.feature_available?(:pseudonymizer)
     end

ee/app/models/ee/issue_assignee.rb

@@ -8,7 +8,7 @@ module EE
     end
 
     def update_elasticsearch_index
-      if ::Gitlab::CurrentSettings.current_application_settings.elasticsearch_indexing?
+      if issue.project&.use_elasticsearch?
         ::ElasticIndexerWorker.perform_async(
           :update,
           'Issue',

ee/app/models/ee/namespace.rb

@@ -271,6 +271,10 @@ module EE
       actual_plan_name == GOLD_PLAN
     end
 
+    def use_elasticsearch?
+      ::Gitlab::CurrentSettings.elasticsearch_indexes_namespace?(self)
+    end
+
     private
 
     def validate_plan_name

ee/app/models/ee/note.rb

@@ -14,8 +14,9 @@ module EE
       scope :searchable, -> { where(system: false) }
     end
 
+    # Original method in Elastic::ApplicationSearch
     def searchable?
-      !system
+      !system && super
     end
 
     def for_epic?

ee/app/models/ee/project_feature.rb

@@ -6,7 +6,7 @@ module EE
 
     prepended do
       after_commit on: :update do
-        if ::Gitlab::CurrentSettings.current_application_settings.elasticsearch_indexing?
+        if project.use_elasticsearch?
           ElasticIndexerWorker.perform_async(:update, 'Project', project_id, project.es_id)
         end
       end

ee/app/models/ee/project_wiki.rb

@@ -16,7 +16,7 @@ module EE
     end
 
     def update_elastic_index
-      index_blobs if ::Gitlab::CurrentSettings.elasticsearch_indexing?
+      index_blobs if project.use_elasticsearch?
     end
 
     def path_to_repo

ee/app/models/elasticsearch_indexed_namespace.rb

+# frozen_string_literal: true
+
+class ElasticsearchIndexedNamespace < ActiveRecord::Base
+  include EachBatch
+
+  self.primary_key = 'namespace_id'
+
+  after_commit :index, on: :create
+  after_commit :delete_from_index, on: :destroy
+
+  belongs_to :namespace
+
+  validates :namespace_id, presence: true, uniqueness: true
+
+  def self.namespace_ids
+    self.pluck(:namespace_id)
+  end
+
+  private
+
+  def index
+    ElasticNamespaceIndexerWorker.perform_async(namespace_id, :index)
+  end
+
+  def delete_from_index
+    ElasticNamespaceIndexerWorker.perform_async(namespace_id, :delete)
+  end
+end

ee/app/models/elasticsearch_indexed_project.rb

+# frozen_string_literal: true
+
+class ElasticsearchIndexedProject < ActiveRecord::Base
+  include EachBatch
+
+  self.primary_key = 'project_id'
+
+  after_commit :index, on: :create
+  after_commit :delete_from_index, on: :destroy
+
+  belongs_to :project
+
+  validates :project_id, presence: true, uniqueness: true
+
+  def self.project_ids
+    self.pluck(:project_id)
+  end
+
+  private
+
+  def index
+    if Gitlab::CurrentSettings.elasticsearch_indexing? && project.searchable?
+      ElasticIndexerWorker.perform_async(:index, project.class.to_s, project.id, project.es_id)
+    end
+  end
+
+  def delete_from_index
+    if Gitlab::CurrentSettings.elasticsearch_indexing? && project.searchable?
+      ElasticIndexerWorker.perform_async(
+        :delete,
+        project.class.to_s,
+        project.id,
+        project.es_id,
+        es_parent: project.es_parent
+      )
+    end
+  end
+end

ee/app/models/index_status.rb

@@ -4,4 +4,6 @@ class IndexStatus < ApplicationRecord
   belongs_to :project
 
   validates :project_id, uniqueness: true, presence: true
+
+  scope :for_project, ->(project_id) { where(project_id: project_id) }
 end

ee/app/services/ee/git/branch_push_service.rb

@@ -9,7 +9,7 @@ module EE
 
       override :execute_related_hooks
       def execute_related_hooks
-        if ::Gitlab::CurrentSettings.elasticsearch_indexing? && default_branch? && should_index_commits?
+        if should_index_commits?
           ::ElasticCommitIndexerWorker.perform_async(project.id, params[:oldrev], params[:newrev])
         end
 
@@ -19,7 +19,9 @@ module EE
       private
 
       def should_index_commits?
-        ::Gitlab::Redis::SharedState.with { |redis| !redis.sismember(:elastic_projects_indexing, project.id) }
+        default_branch? &&
+          project.use_elasticsearch? &&
+          ::Gitlab::Redis::SharedState.with { |redis| !redis.sismember(:elastic_projects_indexing, project.id) }
       end
 
       override :pipeline_options

ee/app/services/ee/git_push_service.rb

+# frozen_string_literal: true
+
+module EE
+  module GitPushService
+    extend ::Gitlab::Utils::Override
+
+    protected
+
+    override :execute_related_hooks
+    def execute_related_hooks
+      if should_index_commits?
+        ::ElasticCommitIndexerWorker.perform_async(project.id, params[:oldrev], params[:newrev])
+      end
+
+      super
+    end
+
+    private
+
+    def should_index_commits?
+      default_branch? &&
+        project.use_elasticsearch? &&
+        ::Gitlab::Redis::SharedState.with { |redis| !redis.sismember(:elastic_projects_indexing, project.id) }
+    end
+
+    override :pipeline_options
+    def pipeline_options
+      { mirror_update: project.mirror? && project.repository.up_to_date_with_upstream?(branch_name) }
+    end
+  end
+end

ee/app/services/ee/search/global_service.rb

@@ -8,7 +8,7 @@ module EE
 
       override :execute
       def execute
-        if ::Gitlab::CurrentSettings.elasticsearch_search?
+        if use_elasticsearch?
           ::Gitlab::Elastic::SearchResults.new(current_user, params[:search],
                                                elastic_projects, projects,
                                                elastic_global)
@@ -17,6 +17,11 @@ module EE
         end
       end
 
+      def use_elasticsearch?
+        ::Gitlab::CurrentSettings.elasticsearch_search? &&
+          !::Gitlab::CurrentSettings.elasticsearch_limit_indexing?
+      end
+
       def elastic_projects
         strong_memoize(:elastic_projects) do
           if current_user&.full_private_access?

ee/app/services/ee/search/group_service.rb

@@ -5,10 +5,17 @@ module EE
     module GroupService
       extend ::Gitlab::Utils::Override
 
+      override :use_elasticsearch?
+      def use_elasticsearch?
+        group&.use_elasticsearch?
+      end
+
+      override :elastic_projects
       def elastic_projects
         @elastic_projects ||= projects.pluck(:id) # rubocop:disable CodeReuse/ActiveRecord
       end
 
+      override :elastic_global
       def elastic_global
         false
       end

ee/app/services/ee/search/project_service.rb

@@ -7,7 +7,7 @@ module EE
 
       override :execute
       def execute
-        if ::Gitlab::CurrentSettings.elasticsearch_search?
+        if use_elasticsearch?
           ::Gitlab::Elastic::ProjectSearchResults.new(current_user,
                                                     params[:search],
                                                     project.id,
@@ -16,6 +16,11 @@ module EE
           super
         end
       end
+
+      # This method is used in the top-level SearchService, so cannot be in-lined into #execute
+      def use_elasticsearch?
+        project.use_elasticsearch?
+      end
     end
   end
 end

ee/app/services/ee/search/snippet_service.rb

@@ -8,12 +8,17 @@ module EE
       override :execute
 
       def execute
-        if ::Gitlab::CurrentSettings.elasticsearch_search?
+        if use_elasticsearch?
           ::Gitlab::Elastic::SnippetSearchResults.new(current_user, params[:search])
         else
           super
         end
       end
+
+      # This method is used in the top-level SearchService, so cannot be in-lined into #execute
+      def use_elasticsearch?
+        ::Gitlab::CurrentSettings.elasticsearch_search?
+      end
     end
   end
 end

ee/app/services/ee/search_service.rb

+# frozen_string_literal: true
+
+module EE
+  module SearchService
+    # This is a proper method instead of a `delegate` in order to
+    # avoid adding unnecessary methods to Search::SnippetService
+    def use_elasticsearch?
+      search_service.use_elasticsearch?
+    end
+  end
+end

ee/app/views/admin/application_settings/_elasticsearch_form.html.haml

@@ -42,6 +42,20 @@
             .form-text.text-muted
               The url to use for connecting to Elasticsearch. Use a comma-separated list to support clustering (e.g., "http://localhost:9200, http://localhost:9201").
 
+          .form-group
+            .form-check
+              = f.check_box :elasticsearch_limit_indexing, class: 'form-check-input js-limit-checkbox'
+              = f.label :elasticsearch_limit_indexing, class: 'form-check-label' do
+                = _('Limit namespaces and projects that can be indexed')
+
+          .form-group.js-limit-namespaces{ class: ('hidden' unless @application_setting.elasticsearch_limit_indexing) }
+            = f.label :elasticsearch_namespace_ids, _('Namespaces to index'), class: 'label-bold'
+            = f.text_field :elasticsearch_namespace_ids, class: 'js-elasticsearch-namespaces', value: elasticsearch_namespace_ids, data: { selected: elasticsearch_objects_options(@application_setting.elasticsearch_limited_namespaces(true)).to_json }
+
+          .form-group.js-limit-projects{ class: ('hidden' unless @application_setting.elasticsearch_limit_indexing) }
+            = f.label :elasticsearch_project_ids, _('Projects to index'), class: 'label-bold'
+            = f.text_field :elasticsearch_project_ids, class: 'js-elasticsearch-projects', value: elasticsearch_project_ids, data: { selected: elasticsearch_objects_options(@application_setting.elasticsearch_limited_projects(true)).to_json }
+
         .sub-section
           %h4 Elasticsearch AWS IAM credentials
           .form-group

ee/app/workers/all_queues.yml

@@ -50,6 +50,7 @@
 - admin_emails
 - create_github_webhook
 - elastic_batch_project_indexer
+- elastic_namespace_indexer
 - elastic_commit_indexer
 - elastic_indexer
 - export_csv

ee/app/workers/ee/post_receive.rb

@@ -29,8 +29,7 @@ module EE
     end
 
     def update_wiki_es_indexes(post_received)
-      return unless ::Gitlab::CurrentSettings.current_application_settings
-        .elasticsearch_indexing?
+      return unless post_received.project.use_elasticsearch?
 
       post_received.project.wiki.index_blobs
     end

ee/app/workers/elastic_batch_project_indexer_worker.rb

@@ -19,6 +19,8 @@ class ElasticBatchProjectIndexerWorker
   private
 
   def run_indexer(project, update_index)
+    return unless project.use_elasticsearch?
+
     # Ensure we remove the hold on the project, no matter what, so ElasticCommitIndexerWorker can do its thing
     # We do this before the indexer starts to avoid the possibility of pushes coming in during this time not
     # being indexed.

ee/app/workers/elastic_commit_indexer_worker.rb

@@ -10,6 +10,8 @@ class ElasticCommitIndexerWorker
 
     project = Project.find(project_id)
 
+    return true unless project.use_elasticsearch?
+
     Gitlab::Elastic::Indexer.new(project).run(oldrev, newrev)
   end
 end

ee/app/workers/elastic_indexer_worker.rb

@@ -17,11 +17,9 @@ class ElasticIndexerWorker
       record = klass.find(record_id)
       record.__elasticsearch__.client = client
 
-      if klass.nested?
-        record.__elasticsearch__.__send__ "#{operation}_document", routing: record.es_parent # rubocop:disable GitlabSecurity/PublicSend
-      else
-        record.__elasticsearch__.__send__ "#{operation}_document" # rubocop:disable GitlabSecurity/PublicSend
-      end
+      import(operation, record, klass)
+
+      initial_index_project(record) if klass == Project && operation.to_s.match?(/index/)
 
       update_issue_notes(record, options["changed_fields"]) if klass == Issue
     when /delete/
@@ -57,6 +55,30 @@ class ElasticIndexerWorker
 
   def clear_project_data(record_id, es_id)
     remove_children_documents('project', record_id, es_id)
+    IndexStatus.for_project(record_id).delete_all
+  end
+
+  def initial_index_project(project)
+    {
+      Issue        => project.issues,
+      MergeRequest => project.merge_requests,
+      Snippet      => project.snippets,
+      Note         => project.notes.searchable,
+      Milestone    => project.milestones
+    }.each do |klass, objects|
+      objects.find_each { |object| import(:index, object, klass) }
+    end
+
+    # Finally, index blobs/commits/wikis
+    ElasticCommitIndexerWorker.perform_async(project.id)
+  end
+
+  def import(operation, record, klass)
+    if klass.nested?
+      record.__elasticsearch__.__send__ "#{operation}_document", routing: record.es_parent # rubocop:disable GitlabSecurity/PublicSend
+    else
+      record.__elasticsearch__.__send__ "#{operation}_document" # rubocop:disable GitlabSecurity/PublicSend
+    end
   end
 
   def remove_documents_by_project_id(record_id)

ee/app/workers/elastic_namespace_indexer_worker.rb

+# frozen_string_literal: true
+
+class ElasticNamespaceIndexerWorker
+  include ApplicationWorker
+
+  sidekiq_options retry: 2
+
+  def perform(namespace_id, operation)
+    return true unless Gitlab::CurrentSettings.elasticsearch_indexing?
+    return true unless Gitlab::CurrentSettings.elasticsearch_limit_indexing?
+
+    namespace = Namespace.find(namespace_id)
+
+    case operation.to_s
+    when /index/
+      index_projects(namespace)
+    when /delete/
+      delete_from_index(namespace)
+    end
+  end
+
+  private
+
+  def index_projects(namespace)
+    # The default of 1000 is good for us since Sidekiq documentation doesn't recommend more than 1000 per batch call
+    # https://www.rubydoc.info/github/mperham/sidekiq/Sidekiq%2FClient:push_bulk
+    namespace.all_projects.find_in_batches do |batch|
+      args = batch.map { |project| [:index, project.class.to_s, project.id, project.es_id] }
+      ElasticIndexerWorker.bulk_perform_async(args)
+    end
+  end
+
+  def delete_from_index(namespace)
+    namespace.all_projects.find_in_batches do |batch|
+      args = batch.map { |project| [:delete, project.class.to_s, project.id, project.es_id] }
+      ElasticIndexerWorker.bulk_perform_async(args)
+    end
+  end
+end

ee/changelogs/unreleased/1607-elastic_feature_flag_per_group.yml

+---
+title: Allow per-project and per-group enabling of Elasticsearch indexing
+merge_request: 9861
+author:
+type: added

ee/db/migrate/20190318020549_add_elasticsearch_limit_indexing_to_application_setting.rb

+# frozen_string_literal: true
+
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddElasticsearchLimitIndexingToApplicationSetting < ActiveRecord::Migration[5.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_column_with_default :application_settings, :elasticsearch_limit_indexing, :boolean, default: false, allow_null: false
+  end
+
+  def down
+    remove_column :application_settings, :elasticsearch_limit_indexing
+  end
+end

ee/db/migrate/20190318021429_add_elastic_namespace_link_and_elastic_project_link.rb

+# frozen_string_literal: true
+
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddElasticNamespaceLinkAndElasticProjectLink < ActiveRecord::Migration[5.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    create_table :elasticsearch_indexed_namespaces, id: false do |t|
+      t.timestamps_with_timezone null: false
+
+      t.references :namespace, nil: false, index: { unique: true }, foreign_key: { on_delete: :cascade }
+    end
+
+    create_table :elasticsearch_indexed_projects, id: false do |t|
+      t.timestamps_with_timezone null: false
+
+      t.references :project, nil: false, index: { unique: true }, foreign_key: { on_delete: :cascade }
+    end
+  end
+end

ee/lib/ee/gitlab/fake_application_settings.rb

+# frozen_string_literal: true
+
+module EE
+  module Gitlab
+    module FakeApplicationSettings
+      def elasticsearch_indexes_project?(_project)
+        false
+      end
+
+      def elasticsearch_indexes_namespace?(_namespace)
+        false
+      end
+    end
+  end
+end

ee/lib/tasks/gitlab/elastic.rake

@@ -57,7 +57,7 @@ namespace :gitlab do
       projects = apply_project_filters(Project.with_wiki_enabled)
 
       projects.find_each do |project|
-        unless project.wiki.empty?
+        if project.use_elasticsearch? && !project.wiki.empty?
           puts "Indexing wiki of #{project.full_name}..."
 
           begin

ee/spec/factories/elasticsearch_indexed_containers.rb

+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :elasticsearch_indexed_namespace do
+    namespace
+  end
+
+  factory :elasticsearch_indexed_project do
+    project
+  end
+end

ee/spec/features/admin/admin_settings_spec.rb

@@ -45,15 +45,64 @@ describe 'Admin updates EE-only settings' do
     expect(page).to have_content "Application settings saved successfully"
   end
 
-  it 'Enable elastic search indexing' do
-    visit integrations_admin_application_settings_path
-    page.within('.as-elasticsearch') do
-      check 'Elasticsearch indexing'
-      click_button 'Save changes'
+  context 'Elasticsearch settings' do
+    before do
+      visit integrations_admin_application_settings_path
     end
 
-    expect(Gitlab::CurrentSettings.elasticsearch_indexing).to be_truthy
-    expect(page).to have_content "Application settings saved successfully"
+    it 'Enable elastic search indexing' do
+      page.within('.as-elasticsearch') do
+        check 'Elasticsearch indexing'
+        click_button 'Save changes'
+      end
+
+      expect(Gitlab::CurrentSettings.elasticsearch_indexing).to be_truthy
+      expect(page).to have_content "Application settings saved successfully"
+    end
+
+    it 'Allows limiting projects and namespaces to index', :js do
+      project = create(:project)
+      namespace = create(:namespace)
+
+      page.within('.as-elasticsearch') do
+        expect(page).not_to have_content('Namespaces to index')
+        expect(page).not_to have_content('Projects to index')
+
+        check 'Limit namespaces and projects that can be indexed'
+
+        expect(page).to have_content('Namespaces to index')
+        expect(page).to have_content('Projects to index')
+
+        fill_in 'Namespaces to index', with: namespace.name
+        wait_for_requests
+      end
+
+      page.within('#select2-drop') do
+        expect(page).to have_content(namespace.full_path)
+      end
+
+      page.within('.as-elasticsearch') do
+        find('.js-limit-namespaces .select2-choices input[type=text]').native.send_keys(:enter)
+
+        fill_in 'Projects to index', with: project.name
+        wait_for_requests
+      end
+
+      page.within('#select2-drop') do
+        expect(page).to have_content(project.full_name)
+      end
+
+      page.within('.as-elasticsearch') do
+        find('.js-limit-projects .select2-choices input[type=text]').native.send_keys(:enter)
+
+        click_button 'Save changes'
+      end
+
+      expect(Gitlab::CurrentSettings.elasticsearch_limit_indexing).to be_truthy
+      expect(ElasticsearchIndexedNamespace.exists?(namespace_id: namespace.id)).to be_truthy
+      expect(ElasticsearchIndexedProject.exists?(project_id: project.id)).to be_truthy
+      expect(page).to have_content "Application settings saved successfully"
+    end
   end
 
   it 'Enable Slack application' do

ee/spec/models/application_setting_spec.rb

@@ -209,6 +209,79 @@ describe ApplicationSetting do
         aws_secret_access_key: 'test-secret-access-key'
       )
     end
+
+    context 'limiting namespaces and projects' do
+      before do
+        setting.update!(elasticsearch_indexing: true)
+        setting.update!(elasticsearch_limit_indexing: true)
+      end
+
+      context 'namespaces' do
+        let(:namespaces) { create_list(:namespace, 3) }
+
+        it 'creates ElasticsearchIndexedNamespace objects when given elasticsearch_namespace_ids' do
+          expect do
+            setting.update!(elasticsearch_namespace_ids: namespaces.map(&:id).join(','))
+          end.to change { ElasticsearchIndexedNamespace.count }.by(3)
+        end
+
+        it 'deletes ElasticsearchIndexedNamespace objects not in elasticsearch_namespace_ids' do
+          create :elasticsearch_indexed_namespace, namespace: namespaces.last
+
+          expect do
+            setting.update!(elasticsearch_namespace_ids: namespaces.first(2).map(&:id).join(','))
+          end.to change { ElasticsearchIndexedNamespace.count }.from(1).to(2)
+
+          expect(ElasticsearchIndexedNamespace.where(namespace_id: namespaces.last.id)).not_to exist
+        end
+
+        it 'tells you if a namespace is allowed to be indexed' do
+          create :elasticsearch_indexed_namespace, namespace: namespaces.last
+
+          expect(setting.elasticsearch_indexes_namespace?(namespaces.last)).to be_truthy
+          expect(setting.elasticsearch_indexes_namespace?(namespaces.first)).to be_falsey
+        end
+      end
+
+      context 'projects' do
+        let(:projects) { create_list(:project, 3) }
+
+        it 'creates ElasticsearchIndexedProject objects when given elasticsearch_project_ids' do
+          expect do
+            setting.update!(elasticsearch_project_ids: projects.map(&:id).join(','))
+          end.to change { ElasticsearchIndexedProject.count }.by(3)
+        end
+
+        it 'deletes ElasticsearchIndexedProject objects not in elasticsearch_project_ids' do
+          create :elasticsearch_indexed_project, project: projects.last
+
+          expect do
+            setting.update!(elasticsearch_project_ids: projects.first(2).map(&:id).join(','))
+          end.to change { ElasticsearchIndexedProject.count }.from(1).to(2)
+
+          expect(ElasticsearchIndexedProject.where(project_id: projects.last.id)).not_to exist
+        end
+
+        it 'tells you if a project is allowed to be indexed' do
+          create :elasticsearch_indexed_project, project: projects.last
+
+          expect(setting.elasticsearch_indexes_project?(projects.last)).to be_truthy
+          expect(setting.elasticsearch_indexes_project?(projects.first)).to be_falsey
+        end
+      end
+
+      it 'returns projects that are allowed to be indexed' do
+        project1 = create(:project)
+        projects = create_list(:project, 3)
+
+        setting.update!(
+          elasticsearch_project_ids: projects.map(&:id).join(','),
+          elasticsearch_namespace_ids: project1.namespace.id.to_s
+        )
+
+        expect(setting.elasticsearch_limited_projects).to match_array(projects << project1)
+      end
+    end
   end
 
   describe 'custom project templates' do

ee/spec/models/concerns/elastic/issue_spec.rb

@@ -7,6 +7,52 @@ describe Issue, :elastic do
 
   let(:project) { create :project }
 
+  context 'when limited indexing is on' do
+    set(:project) { create :project, name: 'test1' }
+    set(:issue) { create :issue, project: project}
+
+    before do
+      stub_ee_application_setting(elasticsearch_limit_indexing: true)
+    end
+
+    context 'when the project is not enabled specifically' do
+      context '#searchable?' do
+        it 'returns false' do
+          expect(issue.searchable?).to be_falsey
+        end
+      end
+    end
+
+    context 'when a project is enabled specifically' do
+      before do
+        create :elasticsearch_indexed_project, project: project
+      end
+
+      context '#searchable?' do
+        it 'returns true' do
+          expect(issue.searchable?).to be_truthy
+        end
+      end
+    end
+
+    context 'when a group is enabled' do
+      set(:group) { create(:group) }
+
+      before do
+        create :elasticsearch_indexed_namespace, namespace: group
+      end
+
+      context '#searchable?' do
+        it 'returns true' do
+          project = create :project, name: 'test1', group: group
+          issue = create :issue, project: project
+
+          expect(issue.searchable?).to be_truthy
+        end
+      end
+    end
+  end
+
   it "searches issues" do
     Sidekiq::Testing.inline! do
       create :issue, title: 'bla-bla term1', project: project

ee/spec/models/concerns/elastic/merge_request_spec.rb

@@ -5,6 +5,15 @@ describe MergeRequest, :elastic do
     stub_ee_application_setting(elasticsearch_search: true, elasticsearch_indexing: true)
   end
 
+  it_behaves_like 'limited indexing is enabled' do
+    set(:object) { create :merge_request, source_project: project }
+    set(:group) { create(:group) }
+    let(:group_object) do
+      project = create :project, name: 'test1', group: group
+      create :merge_request, source_project: project
+    end
+  end
+
   it "searches merge requests" do
     project = create :project, :repository
 

ee/spec/models/concerns/elastic/milestone_spec.rb

@@ -5,6 +5,15 @@ describe Milestone, :elastic do
     stub_ee_application_setting(elasticsearch_search: true, elasticsearch_indexing: true)
   end
 
+  it_behaves_like 'limited indexing is enabled' do
+    set(:object) { create :milestone, project: project }
+    set(:group) { create(:group) }
+    let(:group_object) do
+      project = create :project, name: 'test1', group: group
+      create :milestone, project: project
+    end
+  end
+
   it "searches milestones" do
     project = create :project
 

ee/spec/models/concerns/elastic/note_spec.rb

@@ -5,6 +5,35 @@ describe Note, :elastic do
     stub_ee_application_setting(elasticsearch_search: true, elasticsearch_indexing: true)
   end
 
+  it_behaves_like 'limited indexing is enabled' do
+    set(:object) { create :note, project: project }
+    set(:group) { create(:group) }
+    let(:group_object) do
+      project = create :project, name: 'test1', group: group
+      create :note, project: project
+    end
+
+    context '#searchable?' do
+      before do
+        create :elasticsearch_indexed_project, project: project
+      end
+
+      it 'also works on diff notes' do
+        notes = []
+        notes << create(:diff_note_on_merge_request, note: "term")
+        notes << create(:diff_note_on_commit, note: "term")
+        notes << create(:legacy_diff_note_on_merge_request, note: "term")
+        notes << create(:legacy_diff_note_on_commit, note: "term")
+
+        notes.each do |note|
+          create :elasticsearch_indexed_project, project: note.noteable.project
+
+          expect(note.searchable?).to be_truthy
+        end
+      end
+    end
+  end
+
   it "searches notes" do
     issue = create :issue
 

ee/spec/models/concerns/elastic/project_spec.rb

@@ -5,6 +5,93 @@ describe Project, :elastic do
     stub_ee_application_setting(elasticsearch_search: true, elasticsearch_indexing: true)
   end
 
+  context 'when limited indexing is on' do
+    set(:project) { create :project, name: 'test1' }
+
+    before do
+      stub_ee_application_setting(elasticsearch_limit_indexing: true)
+    end
+
+    context 'when the project is not enabled specifically' do
+      context '#searchable?' do
+        it 'returns false' do
+          expect(project.searchable?).to be_falsey
+        end
+      end
+
+      context '#use_elasticsearch?' do
+        it 'returns false' do
+          expect(project.use_elasticsearch?).to be_falsey
+        end
+      end
+    end
+
+    context 'when a project is enabled specifically' do
+      before do
+        create :elasticsearch_indexed_project, project: project
+      end
+
+      context '#searchable?' do
+        it 'returns true' do
+          expect(project.searchable?).to be_truthy
+        end
+      end
+
+      context '#use_elasticsearch?' do
+        it 'returns true' do
+          expect(project.use_elasticsearch?).to be_truthy
+        end
+      end
+
+      it 'only indexes enabled projects' do
+        Sidekiq::Testing.inline! do
+          # We have to trigger indexing of the previously-created project because we don't have a way to
+          # enable ES for it before it's created, at which point it won't be indexed anymore
+          ElasticIndexerWorker.perform_async(:index, project.class.to_s, project.id, project.es_id)
+          create :project, path: 'test2', description: 'awesome project'
+          create :project
+
+          Gitlab::Elastic::Helper.refresh_index
+        end
+
+        expect(described_class.elastic_search('test*', options: { project_ids: :any }).total_count).to eq(1)
+        expect(described_class.elastic_search('test2', options: { project_ids: :any }).total_count).to eq(0)
+      end
+    end
+
+    context 'when a group is enabled' do
+      set(:group) { create(:group) }
+
+      before do
+        create :elasticsearch_indexed_namespace, namespace: group
+      end
+
+      context '#searchable?' do
+        it 'returns true' do
+          project = create :project, name: 'test1', group: group
+
+          expect(project.searchable?).to be_truthy
+        end
+      end
+
+      it 'indexes only projects under the group', :nested_groups do
+        Sidekiq::Testing.inline! do
+          create :project, name: 'test1', group: create(:group, parent: group)
+          create :project, name: 'test2', description: 'awesome project'
+          create :project, name: 'test3', group: group
+          create :project, path: 'someone_elses_project', name: 'test4'
+
+          Gitlab::Elastic::Helper.refresh_index
+        end
+
+        expect(described_class.elastic_search('test*', options: { project_ids: :any }).total_count).to eq(2)
+        expect(described_class.elastic_search('test3', options: { project_ids: :any }).total_count).to eq(1)
+        expect(described_class.elastic_search('test2', options: { project_ids: :any }).total_count).to eq(0)
+        expect(described_class.elastic_search('test4', options: { project_ids: :any }).total_count).to eq(0)
+      end
+    end
+  end
+
   it "finds projects" do
     project_ids = []
 

ee/spec/models/concerns/elastic/snippet_spec.rb

@@ -5,6 +5,16 @@ describe Snippet, :elastic do
     stub_ee_application_setting(elasticsearch_search: true, elasticsearch_indexing: true)
   end
 
+  it 'always returns global result for Elasticsearch indexing for #use_elasticsearch?' do
+    snippet = create :snippet
+
+    expect(snippet.use_elasticsearch?).to eq(true)
+
+    stub_ee_application_setting(elasticsearch_indexing: false)
+
+    expect(snippet.use_elasticsearch?).to eq(false)
+  end
+
   context 'searching snippets by code' do
     let!(:author) { create(:user) }
     let!(:project) { create(:project) }

ee/spec/models/ee/namespace_spec.rb

@@ -30,4 +30,30 @@ describe Namespace do
       it_behaves_like 'plan helper', namespace_plan
     end
   end
+
+  describe '#use_elasticsearch?' do
+    let(:namespace) { create :namespace }
+
+    it 'returns false if elasticsearch indexing is disabled' do
+      stub_ee_application_setting(elasticsearch_indexing: false)
+
+      expect(namespace.use_elasticsearch?).to eq(false)
+    end
+
+    it 'returns true if elasticsearch indexing enabled but limited indexing disabled' do
+      stub_ee_application_setting(elasticsearch_indexing: true, elasticsearch_limit_indexing: false)
+
+      expect(namespace.use_elasticsearch?).to eq(true)
+    end
+
+    it 'returns true if it is enabled specifically' do
+      stub_ee_application_setting(elasticsearch_indexing: true, elasticsearch_limit_indexing: true)
+
+      expect(namespace.use_elasticsearch?).to eq(false)
+
+      ::Gitlab::CurrentSettings.update!(elasticsearch_namespace_ids: namespace.id.to_s)
+
+      expect(namespace.use_elasticsearch?).to eq(true)
+    end
+  end
 end

ee/spec/models/elasticsearch_indexed_namespace_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe ElasticsearchIndexedNamespace do
+  before do
+    stub_ee_application_setting(elasticsearch_indexing: true)
+  end
+
+  it_behaves_like 'an elasticsearch indexed container' do
+    let(:container) { :elasticsearch_indexed_namespace }
+    let(:attribute) { :namespace_id }
+    let(:index_action) do
+      expect(ElasticNamespaceIndexerWorker).to receive(:perform_async).with(subject.namespace_id, :index)
+    end
+    let(:delete_action) do
+      expect(ElasticNamespaceIndexerWorker).to receive(:perform_async).with(subject.namespace_id, :delete)
+    end
+  end
+end

ee/spec/models/elasticsearch_indexed_project_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe ElasticsearchIndexedProject do
+  before do
+    stub_ee_application_setting(elasticsearch_indexing: true)
+  end
+
+  it_behaves_like 'an elasticsearch indexed container' do
+    let(:container) { :elasticsearch_indexed_project }
+    let(:attribute) { :project_id }
+    let(:index_action) do
+      expect(ElasticIndexerWorker).to receive(:perform_async).with(:index, 'Project', subject.project_id, any_args)
+    end
+    let(:delete_action) do
+      expect(ElasticIndexerWorker).to receive(:perform_async).with(:delete, 'Project', subject.project_id, any_args)
+    end
+  end
+end

ee/spec/models/project_import_state_spec.rb

@@ -52,16 +52,18 @@ describe ProjectImportState, type: :model do
 
         context 'no index status' do
           it 'schedules a full index of the repository' do
-            expect(ElasticCommitIndexerWorker).to receive(:perform_async).with(import_state.project_id, nil)
+            expect(ElasticCommitIndexerWorker).to receive(:perform_async).with(import_state.project_id, Gitlab::Git::BLANK_SHA)
 
             import_state.finish
           end
         end
 
         context 'with index status' do
-          let!(:index_status) { import_state.project.create_index_status!(indexed_at: Time.now, last_commit: 'foo') }
+          let!(:index_status) { import_state.project.index_status }
 
           it 'schedules a progressive index of the repository' do
+            index_status.update!(indexed_at: Time.now, last_commit: 'foo')
+
             expect(ElasticCommitIndexerWorker).to receive(:perform_async).with(import_state.project_id, index_status.last_commit)
 
             import_state.finish

ee/spec/services/ee/git/branch_push_service_spec.rb

@@ -65,6 +65,47 @@ describe Git::BranchPushService do
 
         execute_service(project, user, oldrev, newrev, ref)
       end
+
+      context 'when limited indexing is on' do
+        before do
+          stub_ee_application_setting(elasticsearch_limit_indexing: true)
+        end
+
+        context 'when the project is not enabled specifically' do
+          it 'does not run ElasticCommitIndexerWorker' do
+            expect(ElasticCommitIndexerWorker).not_to receive(:perform_async)
+
+            subject.execute
+          end
+        end
+
+        context 'when a project is enabled specifically' do
+          before do
+            create :elasticsearch_indexed_project, project: project
+          end
+
+          it 'runs ElasticCommitIndexerWorker' do
+            expect(ElasticCommitIndexerWorker).to receive(:perform_async).with(project.id, oldrev, newrev)
+
+            subject.execute
+          end
+        end
+
+        context 'when a group is enabled' do
+          let(:group) { create(:group) }
+          let(:project) { create(:project, :repository, :mirror, group: group) }
+
+          before do
+            create :elasticsearch_indexed_namespace, namespace: group
+          end
+
+          it 'runs ElasticCommitIndexerWorker' do
+            expect(ElasticCommitIndexerWorker).to receive(:perform_async).with(project.id, oldrev, newrev)
+
+            subject.execute
+          end
+        end
+      end
     end
   end
 

ee/spec/support/shared_examples/models/concerns/elastic/limited_indexing.rb

+# frozen_string_literal: true
+
+shared_examples 'limited indexing is enabled' do
+  set(:project) { create :project, :repository, name: 'test1' }
+
+  before do
+    stub_ee_application_setting(elasticsearch_limit_indexing: true)
+  end
+
+  context 'when the project is not enabled specifically' do
+    context '#searchable?' do
+      it 'returns false' do
+        expect(object.searchable?).to be_falsey
+      end
+    end
+  end
+
+  context 'when a project is enabled specifically' do
+    before do
+      create :elasticsearch_indexed_project, project: project
+    end
+
+    context '#searchable?' do
+      it 'returns true' do
+        expect(object.searchable?).to be_truthy
+      end
+    end
+  end
+
+  context 'when a group is enabled' do
+    before do
+      create :elasticsearch_indexed_namespace, namespace: group
+    end
+
+    context '#searchable?' do
+      it 'returns true' do
+        expect(group_object.searchable?).to be_truthy
+      end
+    end
+  end
+end

ee/spec/support/shared_examples/models/elasticsearch_indexed_container.rb

+# frozen_string_literal: true
+
+shared_examples 'an elasticsearch indexed container' do
+  describe 'validations' do
+    subject { create container }
+
+    it 'validates uniqueness of main attribute' do
+      is_expected.to validate_uniqueness_of(attribute)
+    end
+  end
+
+  describe 'callbacks' do
+    subject { build container }
+
+    describe 'on save' do
+      it 'triggers index_project' do
+        is_expected.to receive(:index)
+
+        subject.save!
+      end
+
+      it 'performs the expected action' do
+        index_action
+
+        subject.save!
+      end
+    end
+
+    describe 'on destroy' do
+      subject { create container }
+
+      it 'triggers delete_from_index' do
+        is_expected.to receive(:delete_from_index)
+
+        subject.destroy!
+      end
+
+      it 'performs the expected action' do
+        delete_action
+
+        subject.destroy!
+      end
+    end
+  end
+end

ee/spec/workers/elastic_batch_project_indexer_worker_spec.rb

@@ -5,6 +5,26 @@ describe ElasticBatchProjectIndexerWorker do
   let(:projects) { create_list(:project, 2) }
 
   describe '#perform' do
+    before do
+      stub_ee_application_setting(elasticsearch_indexing: true)
+    end
+
+    context 'with elasticsearch only enabled for a particular project' do
+      before do
+        stub_ee_application_setting(elasticsearch_limit_indexing: true)
+        create :elasticsearch_indexed_project, project: projects.first
+      end
+
+      it 'only indexes the enabled project' do
+        projects.each { |project| expect_index(project, false).and_call_original }
+
+        expect(Gitlab::Elastic::Indexer).to receive(:new).with(projects.first).and_return(double(run: true))
+        expect(Gitlab::Elastic::Indexer).not_to receive(:new).with(projects.last)
+
+        worker.perform(projects.first.id, projects.last.id)
+      end
+    end
+
     it 'runs the indexer for projects in the batch range' do
       projects.each { |project| expect_index(project, false) }
 
@@ -32,7 +52,7 @@ describe ElasticBatchProjectIndexerWorker do
 
     context 'update_index = false' do
       it 'indexes all projects it receives even if already indexed' do
-        projects.first.build_index_status.update!(last_commit: 'foo')
+        projects.first.index_status.update!(last_commit: 'foo')
 
         expect_index(projects.first, false).and_call_original
         expect_next_instance_of(Gitlab::Elastic::Indexer) do |indexer|
@@ -45,8 +65,6 @@ describe ElasticBatchProjectIndexerWorker do
 
     context 'with update_index' do
       it 'reindexes projects that were already indexed' do
-        projects.first.create_index_status!
-
         expect_index(projects.first, true)
         expect_index(projects.last, true)
 
@@ -54,7 +72,7 @@ describe ElasticBatchProjectIndexerWorker do
       end
 
       it 'starts indexing at the last indexed commit' do
-        projects.first.create_index_status!(last_commit: 'foo')
+        projects.first.index_status.update!(last_commit: 'foo')
 
         expect_index(projects.first, true).and_call_original
         expect_any_instance_of(Gitlab::Elastic::Indexer).to receive(:run).with('foo')

ee/spec/workers/elastic_indexer_worker_spec.rb

@@ -106,4 +106,31 @@ describe ElasticIndexerWorker, :elastic do
 
     expect(Elasticsearch::Model.search('*').total_count).to be(0)
   end
+
+  it 'indexes all nested objects for a Project' do
+    # To be able to access it outside the following block
+    project = nil
+
+    Sidekiq::Testing.disable! do
+      project = create :project, :repository
+      create :issue, project: project
+      create :milestone, project: project
+      create :note, project: project
+      create :merge_request, target_project: project, source_project: project
+      create :project_snippet, project: project
+    end
+
+    expect(ElasticCommitIndexerWorker).to receive(:perform_async).with(project.id).and_call_original
+
+    # Nothing should be in the index at this point
+    expect(Elasticsearch::Model.search('*').total_count).to be(0)
+
+    Sidekiq::Testing.inline! do
+      subject.perform("index", "Project", project.id, project.es_id)
+    end
+    Gitlab::Elastic::Helper.refresh_index
+
+    ## All database objects + data from repository. The absolute value does not matter
+    expect(Elasticsearch::Model.search('*').total_count).to be > 40
+  end
 end

ee/spec/workers/elastic_namespace_indexer_worker_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe ElasticNamespaceIndexerWorker, :elastic do
+  subject { described_class.new }
+
+  before do
+    stub_ee_application_setting(elasticsearch_indexing: true)
+    stub_ee_application_setting(elasticsearch_limit_indexing: true)
+  end
+
+  it 'returns true if ES disabled' do
+    stub_ee_application_setting(elasticsearch_indexing: false)
+
+    expect(ElasticIndexerWorker).not_to receive(:perform_async)
+
+    expect(subject.perform(1, "index")).to be_truthy
+  end
+
+  it 'returns true if limited indexing is not enabled' do
+    stub_ee_application_setting(elasticsearch_limit_indexing: false)
+
+    expect(ElasticIndexerWorker).not_to receive(:perform_async)
+
+    expect(subject.perform(1, "index")).to be_truthy
+  end
+
+  describe 'indexing and deleting' do
+    set(:namespace) { create :namespace }
+    let(:projects) { create_list :project, 3, namespace: namespace }
+
+    it 'indexes all projects belonging to the namespace' do
+      args = projects.map { |project| [:index, project.class.to_s, project.id, project.es_id] }
+      expect(ElasticIndexerWorker).to receive(:bulk_perform_async).with(args)
+
+      subject.perform(namespace.id, :index)
+    end
+
+    it 'deletes all projects belonging to the namespace' do
+      args = projects.map { |project| [:delete, project.class.to_s, project.id, project.es_id] }
+      expect(ElasticIndexerWorker).to receive(:bulk_perform_async).with(args)
+
+      subject.perform(namespace.id, :delete)
+    end
+  end
+end

ee/spec/workers/post_receive_spec.rb

@@ -70,5 +70,55 @@ describe PostReceive do
 
       described_class.new.perform(gl_repository, key_id, base64_changes)
     end
+
+    context 'when limited indexing is on' do
+      before do
+        stub_ee_application_setting(
+          elasticsearch_search: true,
+          elasticsearch_indexing: true,
+          elasticsearch_limit_indexing: true
+        )
+      end
+
+      context 'when the project is not enabled specifically' do
+        it 'does not trigger wiki index update' do
+          expect(ProjectWiki).not_to receive(:new)
+
+          described_class.new.perform(gl_repository, key_id, base64_changes)
+        end
+      end
+
+      context 'when a project is enabled specifically' do
+        before do
+          create :elasticsearch_indexed_project, project: project
+        end
+
+        it 'triggers wiki index update' do
+          expect_next_instance_of(ProjectWiki) do |project_wiki|
+            expect(project_wiki).to receive(:index_blobs)
+          end
+
+          described_class.new.perform(gl_repository, key_id, base64_changes)
+        end
+      end
+
+      context 'when a group is enabled' do
+        let(:group) { create(:group) }
+        let(:project) { create(:project, group: group) }
+        let(:key) { create(:key, user: group.owner) }
+
+        before do
+          create :elasticsearch_indexed_namespace, namespace: group
+        end
+
+        it 'triggers wiki index update' do
+          expect_next_instance_of(ProjectWiki) do |project_wiki|
+            expect(project_wiki).to receive(:index_blobs)
+          end
+
+          described_class.new.perform(gl_repository, key_id, base64_changes)
+        end
+      end
+    end
   end
 end

lib/gitlab/fake_application_settings.rb

@@ -32,3 +32,5 @@ module Gitlab
     alias_method :has_attribute?, :[]
   end
 end
+
+Gitlab::FakeApplicationSettings.prepend(EE::Gitlab::FakeApplicationSettings)

locale/gitlab.pot

@@ -749,6 +749,9 @@ msgstr ""
 msgid "Advanced permissions, Large File Storage and Two-Factor authentication settings."
 msgstr ""
 
+msgid "Advanced search functionality"
+msgstr ""
+
 msgid "Advanced settings"
 msgstr ""
 
@@ -3742,6 +3745,12 @@ msgstr ""
 msgid "Elasticsearch integration. Elasticsearch AWS IAM."
 msgstr ""
 
+msgid "Elastic|None. Select namespaces to index."
+msgstr ""
+
+msgid "Elastic|None. Select projects to index."
+msgstr ""
+
 msgid "Email"
 msgstr ""
 
@@ -6327,6 +6336,9 @@ msgstr ""
 msgid "Licenses"
 msgstr ""
 
+msgid "Limit namespaces and projects that can be indexed"
+msgstr ""
+
 msgid "Limited to showing %d event at most"
 msgid_plural "Limited to showing %d events at most"
 msgstr[0] ""
@@ -6908,6 +6920,9 @@ msgstr ""
 msgid "Name:"
 msgstr ""
 
+msgid "Namespaces to index"
+msgstr ""
+
 msgid "Naming, tags, avatar"
 msgstr ""
 
@@ -8307,6 +8322,9 @@ msgstr ""
 msgid "Projects that belong to a group are prefixed with the group namespace. Existing projects may be moved into a group."
 msgstr ""
 
+msgid "Projects to index"
+msgstr ""
+
 msgid "Projects with write access"
 msgstr ""