Support elasticsearch in Search API

doc/api/search.md

 # Search API
 
+[Introduced][ce-41763] in GitLab 10.5
+
 Every API call to search must be authenticated.
 
 ## Global Search API
@@ -15,7 +17,9 @@ GET /search
 | `scope`       | string   | yes        | The scope to search in                |
 | `search`      | string   | yes        | The search query  |
 
-Search the expression within the specified scope. Currentyly these scopes are supported: projects, issues, merge_requests, milestones, snippet_titles, snippet_blobs
+Search the expression within the specified scope. Currentyly these scopes are supported: projects, issues, merge_requests, milestones, snippet_titles, snippet_blobs.
+
+If Elasticsearch is enabled additional scopes available are blobs, wiki_blobs and commits. Find more about [the feature](../integration/elasticsearch.md).
 
 The response depends on the requested scope.
 
@@ -279,6 +283,86 @@ Example response:
 ]
 ```
 
+### Scope: wiki_blobs
+
+This scope is available only if [Elasticsearch](../integration/elasticsearch.md) is enabled.
+
+```bash
+curl --request GET --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" https://gitlab.example.com/api/v4/search?scope=wiki_blobs&search=bye
+```
+
+Example response:
+
+```json
+
+[
+  {
+    "basename": "home",
+    "data": "hello\n\nand bye\n\nend",
+    "filename": "home.md",
+    "id": null,
+    "ref": "master",
+    "startline": 5
+  }
+]
+```
+
+### Scope: commits
+
+This scope is available only if [Elasticsearch](../integration/elasticsearch.md) is enabled.
+
+```bash
+curl --request GET --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" https://gitlab.example.com/api/v4/search?scope=commits&search=bye
+```
+
+Example response:
+
+```json
+
+[
+  {
+  "id": "4109c2d872d5fdb1ed057400d103766aaea97f98",
+  "short_id": "4109c2d8",
+  "title": "goodbye $.browser",
+  "created_at": "2013-02-18T22:02:54.000Z",
+  "parent_ids": [
+    "59d05353ab575bcc2aa958fe1782e93297de64c9"
+  ],
+  "message": "goodbye $.browser\n",
+  "author_name": "angus croll",
+  "author_email": "anguscroll@gmail.com",
+  "authored_date": "2013-02-18T22:02:54.000Z",
+  "committer_name": "angus croll",
+  "committer_email": "anguscroll@gmail.com",
+  "committed_date": "2013-02-18T22:02:54.000Z"
+  }
+]
+```
+
+### Scope: blobs
+
+This scope is available only if [Elasticsearch](../integration/elasticsearch.md) is enabled.
+
+```bash
+curl --request GET --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" https://gitlab.example.com/api/v4/search?scope=blobs&search=installation
+```
+
+Example response:
+
+```json
+
+[
+  {
+    "basename": "README",
+    "data": "```\n\n## Installation\n\nQuick start using the [pre-built",
+    "filename": "README.md",
+    "id": null,
+    "ref": "master",
+    "startline": 46
+  }
+]
+```
+
 
 ## Group Search API
 
@@ -296,7 +380,9 @@ GET /groups/:id/-/search
 | `scope`       | string   | yes        | The scope to search in                |
 | `search`      | string   | yes        | The search query  |
 
-Search the expression within the specified scope. Currentyly these scopes are supported: projects, issues, merge_requests, milestones
+Search the expression within the specified scope. Currentyly these scopes are supported: projects, issues, merge_requests, milestones.
+
+If Elasticsearch is enabled additional scopes available are blobs, wiki_blobs and commits. Find more about [the feature](../integration/elasticsearch.md).
 
 The response depends on the requested scope.
 
@@ -498,6 +584,87 @@ Example response:
 ]
 ```
 
+### Scope: wiki_blobs
+
+This scope is available only if [Elasticsearch](../integration/elasticsearch.md) is enabled.
+
+```bash
+curl --request GET --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" https://gitlab.example.com/api/v4/groups/6/-/search?scope=wiki_blobs&search=bye
+```
+
+Example response:
+
+```json
+
+[
+  {
+    "basename": "home",
+    "data": "hello\n\nand bye\n\nend",
+    "filename": "home.md",
+    "id": null,
+    "ref": "master",
+    "startline": 5
+  }
+]
+```
+
+### Scope: commits
+
+This scope is available only if [Elasticsearch](../integration/elasticsearch.md) is enabled.
+
+```bash
+curl --request GET --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" https://gitlab.example.com/api/v4/groups/6/-/search?scope=commits&search=bye
+```
+
+Example response:
+
+```json
+
+[
+  {
+  "id": "4109c2d872d5fdb1ed057400d103766aaea97f98",
+  "short_id": "4109c2d8",
+  "title": "goodbye $.browser",
+  "created_at": "2013-02-18T22:02:54.000Z",
+  "parent_ids": [
+    "59d05353ab575bcc2aa958fe1782e93297de64c9"
+  ],
+  "message": "goodbye $.browser\n",
+  "author_name": "angus croll",
+  "author_email": "anguscroll@gmail.com",
+  "authored_date": "2013-02-18T22:02:54.000Z",
+  "committer_name": "angus croll",
+  "committer_email": "anguscroll@gmail.com",
+  "committed_date": "2013-02-18T22:02:54.000Z"
+  }
+]
+```
+
+### Scope: blobs
+
+This scope is available only if [Elasticsearch](../integration/elasticsearch.md) is enabled.
+
+```bash
+curl --request GET --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" https://gitlab.example.com/api/v4/groups/6/-/search?scope=blobs&search=installation
+```
+
+Example response:
+
+```json
+
+[
+  {
+    "basename": "README",
+    "data": "```\n\n## Installation\n\nQuick start using the [pre-built",
+    "filename": "README.md",
+    "id": null,
+    "ref": "master",
+    "startline": 46
+  }
+]
+```
+
+
 ## Project Search API
 
 Search within the specified project.
@@ -514,7 +681,7 @@ GET /projects/:id/-/search
 | `scope`       | string   | yes        | The scope to search in                |
 | `search`      | string   | yes        | The search query  |
 
-Search the expression within the specified scope. Currentyly these scopes are supported: issues, merge_requests, milestones, notes, wiki_blobs, commits, blobs
+Search the expression within the specified scope. Currentyly these scopes are supported: issues, merge_requests, milestones, notes, wiki_blobs, commits, blobs.
 
 The response depends on the requested scope.
 
@@ -791,3 +958,5 @@ Example response:
   }
 ]
 ```
+
+[ce-41763]: https://gitlab.com/gitlab-org/gitlab-ce/issues/41763

lib/api/search.rb

@@ -18,6 +18,8 @@ module API
         snippet_blobs: Entities::Snippet
       }.freeze
 
+      ELASTICSEARCH_SCOPES = %w(wiki_blobs blobs commits).freeze
+
       def search(additional_params = {})
         search_params = {
           scope: params[:scope],
@@ -33,6 +35,12 @@ module API
       end
 
       def process_results(results)
+        return [] if results.empty?
+
+        if results.is_a?(Elasticsearch::Model::Response::Response)
+          return paginate(results).map { |blob| Gitlab::Elastic::SearchResults.parse_search_result(blob) }
+        end
+
         case params[:scope]
         when 'wiki_blobs'
           paginate(results).map { |blob| Gitlab::ProjectSearchResults.parse_search_result(blob) }
@@ -50,6 +58,16 @@ module API
       def entity
         SCOPE_ENTITY[params[:scope].to_sym]
       end
+
+      def check_elasticsearch_scope!
+        if ELASTICSEARCH_SCOPES.include?(params[:scope]) && !elasticsearch?
+          render_api_error!({ error: 'Scope not supported without Elasticsearch!' }, 400)
+        end
+      end
+
+      def elasticsearch?
+        Gitlab::CurrentSettings.current_application_settings.elasticsearch_search?
+      end
     end
 
     resource :search do
@@ -58,12 +76,18 @@ module API
       end
       params do
         requires :search, type: String, desc: 'The expression it should be searched for'
-        requires :scope, type: String, desc: 'The scope of search, available scopes:
-          projects, issues, merge_requests, milestones, snippet_titles, snippet_blobs',
-          values: %w(projects issues merge_requests milestones snippet_titles snippet_blobs)
+        requires :scope,
+          type: String,
+          desc: 'The scope of search, available scopes:
+            projects, issues, merge_requests, milestones, snippet_titles, snippet_blobs,
+            if Elasticsearch enabled: wiki_blobs, blobs, commits',
+          values: %w(projects issues merge_requests milestones snippet_titles snippet_blobs
+                     wiki_blobs blobs commits)
         use :pagination
       end
       get do
+        check_elasticsearch_scope!
+
         present search, with: entity
       end
     end
@@ -75,12 +99,16 @@ module API
       params do
         requires :id, type: String, desc: 'The ID of a group'
         requires :search, type: String, desc: 'The expression it should be searched for'
-        requires :scope, type: String, desc: 'The scope of search, available scopes:
-          projects, issues, merge_requests, milestones',
-          values: %w(projects issues merge_requests milestones)
+        requires :scope,
+          type: String,
+          desc: 'The scope of search, available scopes:
+            projects, issues, merge_requests, milestones,
+            if Elasticsearch enabled: wiki_blobs, blobs, commits',
+          values: %w(projects issues merge_requests milestones wiki_blobs blobs commits)
         use :pagination
       end
       get ':id/-/search' do
+        check_elasticsearch_scope!
         find_group!(params[:id])
 
         present search(group_id: params[:id]), with: entity
@@ -94,8 +122,10 @@ module API
       params do
         requires :id, type: String, desc: 'The ID of a project'
         requires :search, type: String, desc: 'The expression it should be searched for'
-        requires :scope, type: String, desc: 'The scope of search, available scopes:
-          issues, merge_requests, milestones, notes, wiki_blobs, commits, blobs',
+        requires :scope,
+          type: String,
+          desc: 'The scope of search, available scopes:
+            issues, merge_requests, milestones, notes, wiki_blobs, commits, blobs',
           values: %w(issues merge_requests milestones notes wiki_blobs commits blobs)
         use :pagination
       end

spec/requests/api/search_spec.rb

@@ -3,8 +3,8 @@ require 'spec_helper'
 describe API::Search do
   set(:user) { create(:user) }
   set(:group) { create(:group) }
-  set(:project) { create(:project, :public, name: 'awesome project', group: group) }
-  set(:repo_project) { create(:project, :public, :repository, group: group) }
+  let(:project) { create(:project, :public, name: 'awesome project', group: group) }
+  let(:repo_project) { create(:project, :public, :repository, group: group) }
 
   shared_examples 'response is correct' do |schema:, size: 1|
     it { expect(response).to have_gitlab_http_status(200) }
@@ -13,6 +13,74 @@ describe API::Search do
     it { expect(json_response.size).to eq(size) }
   end
 
+  shared_examples 'elasticsearch disabled' do
+    it 'returns 400 error for wiki_blobs scope' do
+      get api(endpoint, user), scope: 'wiki_blobs', search: 'awesome'
+
+      expect(response).to have_gitlab_http_status(400)
+    end
+
+    it 'returns 400 error for blobs scope' do
+      get api(endpoint, user), scope: 'blobs', search: 'monitors'
+
+      expect(response).to have_gitlab_http_status(400)
+    end
+
+    it 'returns 400 error for commits scope' do
+      get api(endpoint, user), scope: 'commits', search: 'folder'
+
+      expect(response).to have_gitlab_http_status(400)
+    end
+  end
+
+  shared_examples 'elasticsearch enabled' do
+    before do
+      stub_application_setting(elasticsearch_search: true, elasticsearch_indexing: true)
+      Gitlab::Elastic::Helper.create_empty_index
+    end
+
+    after do
+      Gitlab::Elastic::Helper.delete_index
+      stub_application_setting(elasticsearch_search: false, elasticsearch_indexing: false)
+    end
+
+    context 'for wiki_blobs scope' do
+      before do
+        wiki = create(:project_wiki, project: project)
+        create(:wiki_page, wiki: wiki, attrs: { title: 'home', content: "Awesome page" })
+
+        project.wiki.index_blobs
+        Gitlab::Elastic::Helper.refresh_index
+
+        get api(endpoint, user), scope: 'wiki_blobs', search: 'awesome'
+      end
+
+      it_behaves_like 'response is correct', schema: 'public_api/v4/blobs'
+    end
+
+    context 'for commits scope' do
+      before do
+        repo_project.repository.index_commits
+        Gitlab::Elastic::Helper.refresh_index
+
+        get api(endpoint, user), scope: 'commits', search: 'folder'
+      end
+
+      it_behaves_like 'response is correct', schema: 'public_api/v4/commits', size: 2
+    end
+
+    context 'for blobs scope' do
+      before do
+        repo_project.repository.index_blobs
+        Gitlab::Elastic::Helper.refresh_index
+
+        get api(endpoint, user), scope: 'blobs', search: 'monitors'
+      end
+
+      it_behaves_like 'response is correct', schema: 'public_api/v4/blobs'
+    end
+  end
+
   describe 'GET /search'  do
     context 'when user is not authenticated' do
       it 'returns 401 error' do
@@ -41,6 +109,8 @@ describe API::Search do
     context 'with correct params' do
       context 'for projects scope' do
         before do
+          project
+
           get api('/search', user), scope: 'projects', search: 'awesome'
         end
 
@@ -96,6 +166,18 @@ describe API::Search do
 
         it_behaves_like 'response is correct', schema: 'public_api/v4/snippets'
       end
+
+      context 'when elasticsearch is enabled' do
+        it_behaves_like 'elasticsearch disabled' do
+          let(:endpoint) { '/search' }
+        end
+      end
+
+      context 'when elasticsearch is enabled' do
+        it_behaves_like 'elasticsearch enabled' do
+          let(:endpoint) { '/search' }
+        end
+      end
     end
   end
 
@@ -145,6 +227,8 @@ describe API::Search do
     context 'with correct params' do
       context 'for projects scope' do
         before do
+          project
+
           get api("/groups/#{group.id}/-/search", user), scope: 'projects', search: 'awesome'
         end
 
@@ -180,6 +264,18 @@ describe API::Search do
 
         it_behaves_like 'response is correct', schema: 'public_api/v4/milestones'
       end
+
+      context 'when elasticsearch is enabled' do
+        it_behaves_like 'elasticsearch disabled' do
+          let(:endpoint) { "/groups/#{group.id}/-/search" }
+        end
+      end
+
+      context 'when elasticsearch is enabled' do
+        it_behaves_like 'elasticsearch enabled' do
+          let(:endpoint) { "/groups/#{group.id}/-/search" }
+        end
+      end
     end
   end