Merge branch 'elastic_update' into 'master'

Update Elasticsearch to 5.1

Closes #1253

See merge request !1059

.gitlab-ci.yml

@@ -9,7 +9,7 @@ variables:
   MYSQL_ALLOW_EMPTY_PASSWORD: "1"
   # retry tests only in CI environment
   RSPEC_RETRY_RETRY_COUNT: "3"
-  ELASTIC_HOST: "registry.gitlab.com-gitlab-org-test-elastic-image"
+  ELASTIC_HOST: "elasticsearch"
   RAILS_ENV: "test"
   SIMPLECOV: "true"
   SETUP_DB: "true"
@@ -17,6 +17,8 @@ variables:
   GIT_DEPTH: "20"
   PHANTOMJS_VERSION: "2.1.1"
   GET_SOURCES_ATTEMPTS: "3"
+  # This hack is needed to make ES not that memory hungry
+  ES_JAVA_OPTS: "-Xms600m -Xmx600m"
 
 before_script:
   - source ./scripts/prepare_build.sh
@@ -55,7 +57,7 @@ stages:
   services:
     - mysql:latest
     - redis:alpine
-    - registry.gitlab.com/gitlab-org/test-elastic-image
+    - elasticsearch:5.1
 
 .rspec-knapsack: &rspec-knapsack
   stage: test

Gemfile

@@ -104,9 +104,10 @@ gem 'unf', '~> 0.1.4'
 gem 'seed-fu', '~> 2.3.5'
 
 # Search
-gem 'elasticsearch-model'
-gem 'elasticsearch-rails'
-gem 'gitlab-elasticsearch-git', '~> 1.0.1', require: "elasticsearch/git"
+gem 'elasticsearch-model', '~> 0.1.9'
+gem 'elasticsearch-rails', '~> 0.1.9'
+gem 'elasticsearch-api',   '5.0.3'
+gem 'gitlab-elasticsearch-git', '1.1.1', require: "elasticsearch/git"
 
 # Markdown and HTML processing
 gem 'html-pipeline',        '~> 1.11.0'

Gemfile.lock

@@ -86,7 +86,7 @@ GEM
       sass (>= 3.3.4)
     brakeman (3.4.1)
     browser (2.2.0)
-    builder (3.2.2)
+    builder (3.2.3)
     bullet (5.2.0)
       activesupport (>= 3.0.0)
       uniform_notifier (~> 1.10.0)
@@ -169,17 +169,17 @@ GEM
       railties (>= 4.2)
     dropzonejs-rails (0.7.2)
       rails (> 3.1)
-    elasticsearch (1.0.15)
-      elasticsearch-api (= 1.0.15)
-      elasticsearch-transport (= 1.0.15)
-    elasticsearch-api (1.0.15)
+    elasticsearch (5.0.3)
+      elasticsearch-api (= 5.0.3)
+      elasticsearch-transport (= 5.0.3)
+    elasticsearch-api (5.0.3)
       multi_json
-    elasticsearch-model (0.1.8)
+    elasticsearch-model (0.1.9)
       activesupport (> 3)
       elasticsearch (> 0.4)
       hashie
-    elasticsearch-rails (0.1.8)
-    elasticsearch-transport (1.0.15)
+    elasticsearch-rails (0.1.9)
+    elasticsearch-transport (5.0.3)
       faraday
       multi_json
     email_reply_trimmer (0.1.6)
@@ -265,12 +265,12 @@ GEM
       mime-types (>= 1.19)
       rugged (>= 0.23.0b)
     github-markup (1.4.0)
-    gitlab-elasticsearch-git (1.0.1)
+    gitlab-elasticsearch-git (1.1.1)
       activemodel (~> 4.2)
       activesupport (~> 4.2)
       charlock_holmes (~> 0.7)
-      elasticsearch-api (~> 1.0)
-      elasticsearch-model (~> 0.1.8)
+      elasticsearch-api
+      elasticsearch-model (~> 0.1.9)
       github-linguist (~> 4.7)
       rugged (~> 0.24)
     gitlab-flowdock-git-hook (1.0.1)
@@ -353,7 +353,7 @@ GEM
       temple (~> 0.7.6)
       thor
       tilt
-    hashie (3.4.4)
+    hashie (3.5.1)
     health_check (2.2.1)
       rails (>= 4.0)
     hipchat (1.5.2)
@@ -889,8 +889,9 @@ DEPENDENCIES
   diffy (~> 3.1.0)
   doorkeeper (~> 4.2.0)
   dropzonejs-rails (~> 0.7.1)
-  elasticsearch-model
-  elasticsearch-rails
+  elasticsearch-api (= 5.0.3)
+  elasticsearch-model (~> 0.1.9)
+  elasticsearch-rails (~> 0.1.9)
   email_reply_trimmer (~> 0.1)
   email_spec (~> 1.6.0)
   factory_girl_rails (~> 4.7.0)
@@ -908,7 +909,7 @@ DEPENDENCIES
   gemnasium-gitlab-service (~> 0.2)
   gemojione (~> 3.0)
   github-linguist (~> 4.7.0)
-  gitlab-elasticsearch-git (~> 1.0.1)
+  gitlab-elasticsearch-git (= 1.1.1)
   gitlab-flowdock-git-hook (~> 1.0.1)
   gitlab-license (~> 1.0)
   gitlab-markup (~> 1.5.1)

app/models/concerns/elastic/issues_search.rb

@@ -7,14 +7,14 @@ module Elastic
 
       mappings _parent: { type: 'project' } do
         indexes :id,          type: :integer
-        indexes :iid,         type: :integer, index: :not_analyzed
-        indexes :title,       type: :string,
+        indexes :iid,         type: :integer
+        indexes :title,       type: :text,
                               index_options: 'offsets'
-        indexes :description, type: :string,
+        indexes :description, type: :text,
                               index_options: 'offsets'
         indexes :created_at,  type: :date
         indexes :updated_at,  type: :date
-        indexes :state,       type: :string
+        indexes :state,       type: :text
         indexes :project_id,  type: :integer
         indexes :author_id,   type: :integer
         indexes :assignee_id, type: :integer

app/models/concerns/elastic/merge_requests_search.rb

@@ -8,18 +8,18 @@ module Elastic
       mappings _parent: { type: 'project' } do
         indexes :id,                type: :integer
         indexes :iid,               type: :integer
-        indexes :target_branch,     type: :string,
+        indexes :target_branch,     type: :text,
                                     index_options: 'offsets'
-        indexes :source_branch,     type: :string,
+        indexes :source_branch,     type: :text,
                                     index_options: 'offsets'
-        indexes :title,             type: :string,
+        indexes :title,             type: :text,
                                     index_options: 'offsets'
-        indexes :description,       type: :string,
+        indexes :description,       type: :text,
                                     index_options: 'offsets'
         indexes :created_at,        type: :date
         indexes :updated_at,        type: :date
-        indexes :state,             type: :string
-        indexes :merge_status,      type: :string
+        indexes :state,             type: :text
+        indexes :merge_status,      type: :text
         indexes :source_project_id, type: :integer
         indexes :target_project_id, type: :integer
         indexes :author_id,         type: :integer

app/models/concerns/elastic/milestones_search.rb

@@ -7,9 +7,9 @@ module Elastic
 
       mappings _parent: { type: 'project' } do
         indexes :id,          type: :integer
-        indexes :title,       type: :string,
+        indexes :title,       type: :text,
                               index_options: 'offsets'
-        indexes :description, type: :string,
+        indexes :description, type: :text,
                               index_options: 'offsets'
         indexes :project_id,  type: :integer
         indexes :created_at,  type: :date

app/models/concerns/elastic/notes_search.rb

@@ -7,7 +7,7 @@ module Elastic
 
       mappings _parent: { type: 'project' } do
         indexes :id,          type: :integer
-        indexes :note,        type: :string,
+        indexes :note,        type: :text,
                               index_options: 'offsets'
         indexes :project_id,  type: :integer
         indexes :created_at,  type: :date

app/models/concerns/elastic/projects_search.rb

@@ -7,16 +7,16 @@ module Elastic
 
       mappings do
         indexes :id,                  type: :integer
-        indexes :name,                type: :string,
+        indexes :name,                type: :text,
                                       index_options: 'offsets'
-        indexes :path,                type: :string,
+        indexes :path,                type: :text,
                                       index_options: 'offsets'
-        indexes :name_with_namespace, type: :string,
+        indexes :name_with_namespace, type: :text,
                                       index_options: 'offsets',
                                       analyzer: :my_ngram_analyzer
-        indexes :path_with_namespace, type: :string,
+        indexes :path_with_namespace, type: :text,
                                       index_options: 'offsets'
-        indexes :description,         type: :string,
+        indexes :description,         type: :text,
                                       index_options: 'offsets'
         indexes :namespace_id,        type: :integer
         indexes :created_at,          type: :date
@@ -91,7 +91,7 @@ module Elastic
           }
         end
 
-        query_hash[:query][:bool][:filter] = { and: filters }
+        query_hash[:query][:bool][:filter] = filters
 
         query_hash[:sort] = [:_score]
 

app/models/concerns/elastic/snippets_search.rb

@@ -7,15 +7,15 @@ module Elastic
 
       mappings do
         indexes :id,               type: :integer
-        indexes :title,            type: :string,
+        indexes :title,            type: :text,
                                    index_options: 'offsets'
-        indexes :file_name,        type: :string,
+        indexes :file_name,        type: :text,
                                    index_options: 'offsets'
-        indexes :content,          type: :string,
+        indexes :content,          type: :text,
                                    index_options: 'offsets'
         indexes :created_at,       type: :date
         indexes :updated_at,       type: :date
-        indexes :state,            type: :string
+        indexes :state,            type: :text
         indexes :project_id,       type: :integer
         indexes :author_id,        type: :integer
         indexes :visibility_level, type: :integer

app/workers/elastic_indexer_worker.rb

@@ -36,7 +36,7 @@ class ElasticIndexerWorker
         client.delete index: klass.index_name, type: klass.document_type, id: record_id
       end
 
-      clear_project_indexes(record_id) if klass == Project
+      clear_project_data(record_id) if klass == Project
     end
   rescue Elasticsearch::Transport::Transport::Errors::NotFound, ActiveRecord::RecordNotFound
     # These errors can happen in several cases, including:
@@ -56,43 +56,33 @@ class ElasticIndexerWorker
     end
   end
 
-  def clear_project_indexes(record_id)
-    remove_repository_index(record_id)
-    remove_wiki_index(record_id)
-    remove_nested_content(record_id)
+  def clear_project_data(record_id)
+    remove_children_documents(Repository.document_type, record_id)
+    remove_children_documents(ProjectWiki.document_type, record_id)
+    remove_children_documents(MergeRequest.document_type, record_id)
+    remove_documents_by_project_id(record_id)
   end
 
-  def remove_repository_index(record_id)
-    client.delete_by_query({
-      index: Repository.__elasticsearch__.index_name,
-      body: {
-        query: {
-          or: [
-            { term: { "commit.rid" => record_id } },
-            { term: { "blob.rid" => record_id } }
-          ]
-        }
-      }
-    })
-  end
-
-  def remove_nested_content(record_id)
+  def remove_documents_by_project_id(record_id)
     client.delete_by_query({
       index: Project.__elasticsearch__.index_name,
       body: {
         query: {
-          term: { "_parent" => record_id }
+          term: { "project_id" => record_id }
         }
       }
     })
   end
 
-  def remove_wiki_index(record_id)
+  def remove_children_documents(document_type, parent_record_id)
     client.delete_by_query({
-      index: ProjectWiki.__elasticsearch__.index_name,
+      index: Project.__elasticsearch__.index_name,
       body: {
         query: {
-          term: { "blob.rid" => "wiki_#{record_id}" }
+          parent_id: {
+            type: document_type,
+            id: parent_record_id
+          }
         }
       }
     })

changelogs/unreleased-ee/elastic_update.yml

+---
+title: Update Elasticsearch to 5.1
+merge_request:
+author:

doc/integration/elasticsearch.md

@@ -29,12 +29,7 @@ GitLab, or on a separate server.
 
 ## Requirements
 
-These are the requirements needed for Elasticsearch to work:
-
-- GitLab 8.4+
-- Elasticsearch 2.4.x (with [Delete By Query Plugin](https://www.elastic.co/guide/en/elasticsearch/plugins/2.4/plugins-delete-by-query.html) installed)
-
-Please note that we don't support Elasticsearch 5.x at this time.
+Elasticsearch 5.1.x.
 
 ## Install Elasticsearch
 

lib/tasks/gitlab/check.rake

@@ -1082,23 +1082,15 @@ namespace :gitlab do
     client = Elasticsearch::Client.new(host: ApplicationSetting.current.elasticsearch_host,
                                        port: ApplicationSetting.current.elasticsearch_port)
 
-    print "Elasticsearch version 2.4.x? ... "
+    print "Elasticsearch version 5.1.x? ... "
 
     version = Gitlab::VersionInfo.parse(client.info["version"]["number"])
 
-    if version.major == 2 && version.minor == 4
+    if version.major == 5 && version.minor == 1
       puts "yes (#{version})".color(:green)
     else
       puts "no, you have #{version}".color(:red)
     end
-
-    print "Elasticsearch has plugin delete-by-query installed? ... "
-
-    if client.cat.plugins.include?("delete-by-query")
-      puts "yes".color(:green)
-    else
-      puts "no".color(:red)
-    end
   end
 
   def check_gitlab_geo_node(node)