Merge remote-tracking branch 'origin/master' into ce-to-ee

Signed-off-by: Rémy Coutable <remy@rymai.me>

.gitlab-ci.yml

@@ -55,13 +55,13 @@ stages:
   services:
     - postgres:latest
     - redis:alpine
-    - elasticsearch:5.1
+    - elasticsearch:5.3
 
 .use-mysql: &use-mysql
   services:
     - mysql:latest
     - redis:alpine
-    - elasticsearch:5.1
+    - elasticsearch:5.3
 
 .only-master-and-ee-or-mysql: &only-master-and-ee-or-mysql
   only:

Gemfile

@@ -113,7 +113,6 @@ gem 'seed-fu', '~> 2.3.5'
 gem 'elasticsearch-model', '~> 0.1.9'
 gem 'elasticsearch-rails', '~> 0.1.9'
 gem 'elasticsearch-api',   '5.0.3'
-gem 'gitlab-elasticsearch-git', '1.1.1', require: "elasticsearch/git"
 gem 'aws-sdk'
 gem 'faraday_middleware-aws-signers-v4'
 

Gemfile.lock

@@ -285,14 +285,6 @@ GEM
       mime-types (>= 1.19)
       rugged (>= 0.23.0b)
     github-markup (1.4.0)
-    gitlab-elasticsearch-git (1.1.1)
-      activemodel (~> 4.2)
-      activesupport (~> 4.2)
-      charlock_holmes (~> 0.7)
-      elasticsearch-api
-      elasticsearch-model (~> 0.1.9)
-      github-linguist (~> 4.7)
-      rugged (~> 0.24)
     gitlab-flowdock-git-hook (1.0.1)
       flowdock (~> 0.7)
       gitlab-grit (>= 2.4.1)
@@ -947,7 +939,6 @@ DEPENDENCIES
   gemojione (~> 3.0)
   gitaly (~> 0.5.0)
   github-linguist (~> 4.7.0)
-  gitlab-elasticsearch-git (= 1.1.1)
   gitlab-flowdock-git-hook (~> 1.0.1)
   gitlab-license (~> 1.0)
   gitlab-markup (~> 1.5.1)

app/models/concerns/elastic/application_search.rb

@@ -117,10 +117,10 @@ module Elastic
                          query: {
                            bool: {
                              must: [{
-                               multi_match: {
+                               simple_query_string: {
                                  fields: fields,
                                  query: query,
-                                 operator: :and
+                                 default_operator: :and
                                }
                              }]
                            }

app/models/concerns/elastic/notes_search.rb

@@ -50,14 +50,7 @@ module Elastic
       def self.elastic_search(query, options: {})
         options[:in] = ['note']
 
-        query_hash = {
-          query: {
-            bool: {
-              must: [{ match: { note: query } }],
-            },
-          }
-        }
-
+        query_hash = basic_query_hash(%w[note], query)
         query_hash = project_ids_filter(query_hash, options)
         query_hash = confidentiality_filter(query_hash, options[:current_user])
 

app/models/concerns/elastic/snippets_search.rb

@@ -52,23 +52,9 @@ module Elastic
       end
 
       def self.elastic_search_code(query, options: {})
-        query_hash = {
-          query: {
-            bool: {
-              must: [{ match: { content: query } }]
-            }
-          }
-        }
-
+        query_hash = basic_query_hash(%w(content), query)
         query_hash = filter(query_hash, options[:user])
 
-        query_hash[:sort] = [
-          { updated_at: { order: :desc } },
-          :_score
-        ]
-
-        query_hash[:highlight] = { fields: { content: {} } }
-
         self.__elasticsearch__.search(query_hash)
       end
 

app/views/projects/protected_tags/_create_protected_tag.html.haml

@@ -9,7 +9,7 @@
         .form-group
           = f.label :name, class: 'col-md-2 text-right' do
             Tag:
-          .col-md-10
+          .col-md-10.protected-tags-dropdown
             = render partial: "projects/protected_tags/dropdown", locals: { f: f }
             .help-block
               = link_to 'Wildcards', help_page_path('user/project/protected_tags', anchor: 'wildcard-protected-tags')

app/views/search/_form.html.haml

@@ -13,3 +13,7 @@
     - unless params[:snippets].eql? 'true'
       = render 'filter' if current_user
     = button_tag "Search", class: "btn btn-success btn-search"
+  - if current_application_settings.elasticsearch_search?
+    .help-block
+      = link_to 'Advanced search functionality', help_page_path('user/search/advanced-search-syntax.md'), target: '_blank'
+      is enabled.

bin/elastic_repo_indexer

@@ -3,12 +3,14 @@
 require 'rubygems'
 require 'bundler/setup'
 require 'json'
-require 'elasticsearch/git'
 require 'active_support'
 require 'active_support/core_ext'
 require 'benchmark'
 
-require File.expand_path('../lib/gitlab/elastic/client', File.dirname(__FILE__))
+$: << File.expand_path('../lib', File.dirname(__FILE__))
+
+require 'gitlab/elastic/client'
+require 'elasticsearch/git'
 
 Thread.abort_on_exception = true
 

changelogs/unreleased-ee/1052-advanced-search-syntax.yml

+---
+title: Support advanced search queries using elasticsearch
+merge_request: 1770
+author:

changelogs/unreleased-ee/2253-support-more-es-versions.yml

+---
+title: Support more elasticsearch versions
+merge_request: 1716
+author:

doc/articles/how_to_configure_ldap_gitlab_ee/index.md

+# How to configure LDAP with GitLab EE
+
+> **Type:** admin guide ||
+> **Level:** intermediary ||
+> **Author:** [Chris Wilson](https://gitlab.com/MrChrisW) ||
+> **Publication date:** 2017/05/03
+
+## Introduction
+
+The present article follows [How to Configure LDAP with GitLab CE](../how_to_configure_ldap_gitlab_ce/index.md). Make sure to read through it before moving forward.
+
+## GitLab Enterprise Edition - LDAP features
+
+[GitLab Enterprise Edition (EE)](https://about.gitlab.com/gitlab-ee/) has a number of advantages when it comes to integrating with Active Directory (LDAP):
+
+- [Administrator Sync](#administrator-sync): As an extension of group sync, you can automatically manage your global GitLab administrators. Specify a group CN for `admin_group` and all members of the LDAP group will be given administrator privileges.
+- [Group Sync](#group-sync): This allows GitLab group membership to be automatically updated based on LDAP group members.
+- [Multiple LDAP servers](#multiple-ldap-servers): The ability to configure multiple LDAP servers. This is useful if an organization has different LDAP servers within departments. This is not designed for failover. We're working on [supporting LDAP failover](https://gitlab.com/gitlab-org/gitlab-ee/issues/139) in GitLab.
+
+- Daily user synchronization: Once a day, GitLab will run a synchronization to check and update GitLab users against LDAP. This process updates all user details automatically.
+
+On the following section, you'll find a description for each of these features. Read through [LDAP GitLab EE docs](../../administration/auth/ldap-ee.md) for complementary information.
+
+![GitLab OU Structure](img/admin_group.png)
+
+All members of the group `Global Admins` will be given **administrator** access to GitLab, allowing them to view the `/admin` dashboard.
+
+### Group Sync
+
+Group syncing allows AD (LDAP) groups to be mapped to GitLab groups. This provides more control over per-group user management. To configure group syncing edit the `group_base` **DN** (`'OU=Global Groups,OU=GitLab INT,DC=GitLab,DC=org'`). This **OU** contains all groups that will be associated with [GitLab groups](../../workflow/groups.md).
+
+#### Creating group links - example
+
+As an example, let's suppose we have a "UKGov" GitLab group, which deals with confidential government information. Therefore, it is important that users of this group are given the correct permissions to projects contained within the group. Granular group permissions can be applied based on the AD group.
+
+**UK Developers** of our "UKGov" group are given **"developer"** permissions.
+
+_The developer permission allows the development staff to effectively manage all project code, issues, and merge requests._
+
+**UK Support** staff of our "UKGov" group are given **"reporter"** permissions.
+
+_The reporter permission allows support staff to manage issues, labels, and review project code._
+
+**US People Ops** of our "UKGov" group are given **"guest"** permissions.
+
+![Creating group links](img/group_linking.gif)
+
+> Guest permissions allows people ops staff to review and lodge new issues while allowing no read or write access to project code or [confidential issues](../../user/project/issues/confidential_issues.md#permissions-and-access-to-confidential-issues) created by other users.
+
+See the [permission list](../../user/permissions.md) for complementary info.
+
+#### Group permissions - example
+
+Considering the previous example, our staff will have
+access to our GitLab instance with the following structure:
+
+![GitLab OU Structure](img/group_link_final.png)
+
+Using this permission structure in our example allows only UK staff access to sensitive information stored in the projects code, while still allowing other teams to work effectively. As all permissions are controlled via AD groups new users can be quickly added to existing groups. New group members will then automatically inherit the required permissions.
+
+> [More information](../../administration/auth/ldap-ee.md#group-sync) on group syncing.
+
+### Updating user permissions - new feature
+
+Since GitLab [v8.15](https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/822) LDAP user permissions can now be manually overridden by an admin user. To override a user's permissions visit the groups **Members** page and select **Edit permissions**.
+
+![Setting manual permissions](img/manual_permissions.gif)
+
+### Multiple LDAP servers
+
+GitLab EE can support multiple LDAP servers. Simply configure another server in the `gitlab.rb` file within the `ldap_servers` block. In the example below we configure a new secondary server with the label **GitLab Secondary AD**. This is shown on the GitLab login screen. Large enterprises often utilize multiple LDAP servers for segregating organizational departments.
+
+![Multiple LDAP Servers Login](img/multi_login.gif)
+
+Considering the example illustrated on the image above,
+our `gitlab.rb` configuration would look like:
+
+```ruby
+gitlab_rails['ldap_enabled'] = true
+gitlab_rails['ldap_servers'] = {
+'main' => {
+  'label' => 'GitLab AD',
+  'host' =>  'ad.example.org',
+  'port' => 636,
+  'uid' => 'sAMAccountName',
+  'method' => 'ssl',
+  'bind_dn' => 'CN=GitLabSRV,CN=Users,DC=GitLab,DC=org',
+  'password' => 'Password1',
+  'active_directory' => true,
+  'base' => 'OU=GitLab INT,DC=GitLab,DC=org',
+  'group_base' => 'OU=Global Groups,OU=GitLab INT,DC=GitLab,DC=org',
+  'admin_group' => 'Global Admins'
+  },
+
+'secondary' => {
+  'label' => 'GitLab Secondary AD',
+  'host' =>  'ad-secondary.example.net',
+  'port' => 636,
+  'uid' => 'sAMAccountName',
+  'method' => 'ssl',
+  'bind_dn' => 'CN=GitLabSRV,CN=Users,DC=GitLab,DC=com',
+  'password' => 'Password1',
+  'active_directory' => true,
+  'base' => 'OU=GitLab Secondary,DC=GitLab,DC=com',
+  'group_base' => 'OU=Global Groups,OU=GitLab INT,DC=GitLab,DC=com',
+  'admin_group' => 'Global Admins'
+  }
+}
+```
+
+## Conclusion
+
+Integration of GitLab with Active Directory (LDAP) reduces the complexity of user management.
+It has the advantage of improving user permission controls, whilst easing the deployment of GitLab into an existing [IT environment](https://www.techopedia.com/definition/29199/it-infrastructure). GitLab EE offers advanced group management and multiple LDAP servers.
+
+With the assistance of the [GitLab Support](https://about.gitlab.com/support) team, setting up GitLab with an existing AD/LDAP solution will be a smooth and painless process.

doc/articles/index.md

@@ -7,6 +7,12 @@ to provide the community with guidance on specific processes to achieve certain
 They are written by members of the GitLab Team and by
 [Community Writers](https://about.gitlab.com/handbook/product/technical-writing/community-writers/).
 
+## Authentication
+
+- **LDAP**
+  - [How to configure LDAP with GitLab CE](how_to_configure_ldap_gitlab_ce/index.md)
+  - [How to configure LDAP with GitLab EE](how_to_configure_ldap_gitlab_ee/index.md)
+
 ## GitLab Pages
 
 - **GitLab Pages from A to Z**

doc/ci/pipelines.md

 # Introduction to pipelines and jobs
 
->**Note:**
-Introduced in GitLab 8.8.
+> Introduced in GitLab 8.8.
 
 ## Pipelines
 
@@ -9,11 +8,17 @@ A pipeline is a group of [jobs][] that get executed in [stages][](batches).
 All of the jobs in a stage are executed in parallel (if there are enough
 concurrent [Runners]), and if they all succeed, the pipeline moves on to the
 next stage. If one of the jobs fails, the next stage is not (usually)
-executed.
+executed. You can access the pipelines page in your project's **Pipelines** tab.
+
+In the following image you can see that the pipeline consists of four stages
+(`build`, `test`, `staging`, `production`) each one having one or more jobs.
+
+>**Note:**
+GitLab capitalizes the stages' names when shown in the [pipeline graphs](#pipeline-graphs).
 
 ![Pipelines example](img/pipelines.png)
 
-## Types of Pipelines
+## Types of pipelines
 
 There are three types of pipelines that often use the single shorthand of "pipeline". People often talk about them as if each one is "the" pipeline, but really, they're just pieces of a single, comprehensive pipeline.
 
@@ -23,7 +28,7 @@ There are three types of pipelines that often use the single shorthand of "pipel
 2. **Deploy Pipeline**: Deploy stage(s) defined in `.gitlab-ci.yml` The flow of deploying code to servers through various stages: e.g. development to staging to production
 3. **Project Pipeline**: Cross-project CI dependencies [triggered via API][triggers], particularly for micro-services, but also for complicated build dependencies: e.g. api -> front-end, ce/ee -> omnibus.
 
-## Development Workflows
+## Development workflows
 
 Pipelines accommodate several development workflows:
 
@@ -45,19 +50,142 @@ confused with a `build` job or `build` stage.
 Pipelines are defined in `.gitlab-ci.yml` by specifying [jobs] that run in
 [stages].
 
-See full [documentation](yaml/README.md#jobs).
+See the reference [documentation for jobs](yaml/README.md#jobs).
 
 ## Seeing pipeline status
 
-You can find the current and historical pipeline runs under **Pipelines** for
-your project.
+You can find the current and historical pipeline runs under your project's
+**Pipelines** tab. Clicking on a pipeline will show the jobs that were run for
+that pipeline.
+
+![Pipelines index page](img/pipelines_index.png)
 
 ## Seeing job status
 
-Clicking on a pipeline will show the jobs that were run for that pipeline.
+When you visit a single pipeline you can see the related jobs for that pipeline.
 Clicking on an individual job will show you its job trace, and allow you to
 cancel the job, retry it, or erase the job trace.
 
+![Pipelines example](img/pipelines.png)
+
+## Pipeline graphs
+
+> [Introduced][ce-5742] in GitLab 8.11.
+
+Pipelines can be complex structures with many sequential and parallel jobs.
+To make it a little easier to see what is going on, you can view a graph
+of a single pipeline and its status.
+
+A pipeline graph can be shown in two different ways depending on what page you
+are on.
+
+---
+
+The regular pipeline graph that shows the names of the jobs of each stage can
+be found when you are on a [single pipeline page](#seeing-pipeline-status).
+
+![Pipelines example](img/pipelines.png)
+
+Then, there is the pipeline mini graph which takes less space and can give you a
+quick glance if all jobs pass or something failed. The pipeline mini graph can
+be found when you visit:
+
+- the pipelines index page
+- a single commit page
+- a merge request page
+
+That way, you can see all related jobs for a single commit and the net result
+of each stage of your pipeline. This allows you to quickly see what failed and
+fix it. Stages in pipeline mini graphs are collapsible. Hover your mouse over
+them and click to expand their jobs.
+
+| **Mini graph** | **Mini graph expanded** |
+| :------------: | :---------------------: |
+| ![Pipelines mini graph](img/pipelines_mini_graph_simple.png) | ![Pipelines mini graph extended](img/pipelines_mini_graph.png) |
+
+### Grouping similar jobs in the pipeline graph
+
+> [Introduced][ce-6242] in GitLab 8.12.
+
+If you have many similar jobs, your pipeline graph becomes very long and hard
+to read. For that reason, similar jobs can automatically be grouped together.
+If the job names are formatted in certain ways, they will be collapsed into
+a single group in regular pipeline graphs (not the mini graphs).
+You'll know when a pipeline has grouped jobs if you don't see the retry or
+cancel button inside them. Hovering over them will show the number of grouped
+jobs. Click to expand them.
+
+![Grouped pipelines](img/pipelines_grouped.png)
+
+The basic requirements is that there are two numbers separated with one of
+the following (you can even use them interchangeably):
+
+- a space
+- a backslash (`/`)
+- a colon (`:`)
+
+>**Note:**
+More specifically, [it uses][regexp] this regular expression: `\d+[\s:\/\\]+\d+\s*`.
+
+The jobs will be ordered by comparing those two numbers from left to right. You
+usually want the first to be the index and the second the total.
+
+For example, the following jobs will be grouped under a job named `test`:
+
+- `test 0 3` => `test`
+- `test 1 3` => `test`
+- `test 2 3` => `test`
+
+The following jobs will be grouped under a job named `test ruby`:
+
+- `test 1:2 ruby` => `test ruby`
+- `test 2:2 ruby` => `test ruby`
+
+The following jobs will be grouped under a job named `test ruby` as well:
+
+- `1/3 test ruby` => `test ruby`
+- `2/3 test ruby` => `test ruby`
+- `3/3 test ruby` => `test ruby`
+
+### Manual actions from the pipeline graph
+
+> [Introduced][ce-7931] in GitLab 8.15.
+
+[Manual actions][manual] allow you to require manual interaction before moving
+forward with a particular job in CI. Your entire pipeline can run automatically,
+but the actual [deploy to production][env-manual] will require a click.
+
+You can do this straight from the pipeline graph. Just click on the play button
+to execute that particular job. For example, in the image below, the `production`
+stage has a job with a manual action.
+
+![Pipelines example](img/pipelines.png)
+
+### Ordering of jobs in pipeline graphs
+
+**Regular pipeline graph**
+
+In the single pipeline page, jobs are sorted by name.
+
+**Mini pipeline graph**
+
+> [Introduced][ce-9760] in GitLab 9.0.
+
+In the pipeline mini graphs, the jobs are sorted first by severity and then
+by name. The order of severity is:
+
+- failed
+- warning
+- pending
+- running
+- manual
+- canceled
+- success
+- skipped
+- created
+
+![Pipeline mini graph sorting](img/pipelines_mini_graph_sorting.png)
+
 ## How the pipeline duration is calculated
 
 Total running time for a given pipeline would exclude retries and pending
@@ -96,7 +224,14 @@ respective link in the [Pipelines settings] page.
 
 [jobs]: #jobs
 [jobs-yaml]: yaml/README.md#jobs
+[manual]: yaml/README.md#manual
+[env-manual]: environments.md#manually-deploying-to-environments
 [stages]: yaml/README.md#stages
 [runners]: runners/README.html
 [pipelines settings]: ../user/project/pipelines/settings.md
 [triggers]: triggers/README.md
+[ce-5742]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/5742
+[ce-6242]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/6242
+[ce-7931]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/7931
+[ce-9760]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/9760
+[regexp]: https://gitlab.com/gitlab-org/gitlab-ce/blob/2f3dc314f42dbd79813e6251792853bc231e69dd/app/models/commit_status.rb#L99

doc/integration/elasticsearch.md

@@ -33,7 +33,7 @@ service.
 | GitLab version | Elasticsearch version |
 | -------------- | --------------------- |
 | GitLab Enterprise Edition 8.4 - 8.17  | Elasticsearch 2.4 with [Delete By Query Plugin](https://www.elastic.co/guide/en/elasticsearch/plugins/2.4/plugins-delete-by-query.html) installed |
-| GitLab Enterprise Edition 9.0+        | Elasticsearch 5.1 |
+| GitLab Enterprise Edition 9.0+        | Elasticsearch 5.1 - 5.3 |
 
 ## Install Elasticsearch
 

doc/topics/authentication/index.md

@@ -18,6 +18,8 @@ This page gathers all the resources for the topic **Authentication** within GitL
 - [LDAP (Enterprise Edition)](https://docs.gitlab.com/ee/administration/auth/ldap-ee.html)
 - [Enforce Two-factor Authentication (2FA)](../../security/two_factor_authentication.md#enforce-two-factor-authentication-2fa)
 - **Articles:**
+  - [How to Configure LDAP with GitLab CE](../../articles/how_to_configure_ldap_gitlab_ce/index.md)
+  - [How to Configure LDAP with GitLab EE](../../articles/how_to_configure_ldap_gitlab_ee/index.md)
   - [Feature Highlight: LDAP Integration](https://about.gitlab.com/2014/07/10/feature-highlight-ldap-sync/)
   - [Debugging LDAP](https://about.gitlab.com/handbook/support/workflows/ldap/debugging_ldap.html)
 - **Integrations:**

doc/user/search/advanced-search-syntax.md

+## Advanced search syntax
+
+If your site administrator has enabled [Elasticsearch integration](../../integration/elasticsearch.md)
+then some advanced search functionality is available.
+
+Full details can be found in the
+[Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-simple-query-string-query.html#_simple_query_string_syntax)
+but here's a quick guide:
+
+* Searches look for all the words in a query, in any order - e.g.: searching
+  issues for `display bug` will return all issues matching both those words, in any order.
+* To find the exact term, use double quotes: `"display bug"`
+* To find bugs not mentioning display, use `-`: `bug -display`
+* To find a bug in display or sound, use `|`: `bug display | sound`
+* To group terms together, use parentheses: `bug | (display +sound)`
+* To match a partial word, use `*`: `bug find_by_*`
+* To find a term containing one of these symbols, use `\`: `argument \-last`
+

lib/elasticsearch/git.rb

+require "elasticsearch/git/model"
+require "elasticsearch/git/repository"
+
+module Elasticsearch
+  module Git
+  end
+end

lib/elasticsearch/git/encoder_helper.rb

+require 'active_support/concern'
+require 'charlock_holmes'
+
+module Elasticsearch
+  module Git
+    module EncoderHelper
+      extend ActiveSupport::Concern
+
+      included do
+        def encode!(message)
+          return nil unless message.respond_to? :force_encoding
+
+          # if message is utf-8 encoding, just return it
+          message.force_encoding("UTF-8")
+          return message if message.valid_encoding?
+
+          # return message if message type is binary
+          detect = CharlockHolmes::EncodingDetector.detect(message)
+          return message.force_encoding("BINARY") if detect && detect[:type] == :binary
+
+          # encoding message to detect encoding
+          if detect && detect[:encoding]
+            message.force_encoding(detect[:encoding])
+          end
+
+          # encode and clean the bad chars
+          message.replace clean(message)
+        rescue
+          encoding = detect ? detect[:encoding] : "unknown"
+          "--broken encoding: #{encoding}"
+        end
+
+        private
+
+        def clean(message)
+          message.encode("UTF-16BE", undef: :replace, invalid: :replace, replace: "")
+          .encode("UTF-8")
+          .gsub("\0".encode("UTF-8"), "")
+        end
+      end
+    end
+  end
+end

lib/elasticsearch/git/lite_blob.rb

+require 'linguist'
+require 'elasticsearch/git/encoder_helper'
+
+module Elasticsearch
+  module Git
+    class LiteBlob
+      include Linguist::BlobHelper
+      include Elasticsearch::Git::EncoderHelper
+
+      attr_accessor :id, :name, :path, :size, :mode, :commit_id
+      attr_writer :data
+
+      def initialize(repo, raw_blob_hash)
+        @id   = raw_blob_hash[:oid]
+        @blob = repo.lookup(@id)
+
+        @mode = raw_blob_hash[:mode].to_s(8)
+        @size = @blob.size
+        @path = encode!(raw_blob_hash[:path])
+        @name = @path.split('/').last
+      end
+
+      def data
+        @data ||= encode!(@blob.content)
+      end
+    end
+  end
+end

lib/elasticsearch/git/model.rb

+require 'active_support/concern'
+require 'active_model'
+require 'elasticsearch/model'
+
+module Elasticsearch
+  module Git
+    module Model
+      extend ActiveSupport::Concern
+
+      included do
+        extend ActiveModel::Naming
+        include ActiveModel::Model
+        include Elasticsearch::Model
+
+        env = if defined?(::Rails)
+                ::Rails.env.to_s
+              else
+                nil
+              end
+
+        index_name [self.name.downcase, 'index', env].compact.join('-')
+
+        settings \
+          index: {
+          analysis: {
+            analyzer: {
+              path_analyzer: {
+                type: 'custom',
+                tokenizer: 'path_tokenizer',
+                filter: %w(lowercase asciifolding)
+              },
+              sha_analyzer: {
+                type: 'custom',
+                tokenizer: 'sha_tokenizer',
+                filter: %w(lowercase asciifolding)
+              },
+              code_analyzer: {
+                type: 'custom',
+                tokenizer: 'standard',
+                filter: %w(code lowercase asciifolding),
+                char_filter: ["code_mapping"]
+              },
+              code_search_analyzer: {
+                type: 'custom',
+                tokenizer: 'standard',
+                filter: %w(lowercase asciifolding),
+                char_filter: ["code_mapping"]
+              }
+            },
+            tokenizer: {
+              sha_tokenizer: {
+                type: "edgeNGram",
+                min_gram: 5,
+                max_gram: 40,
+                token_chars: %w(letter digit)
+              },
+              path_tokenizer: {
+                type: 'path_hierarchy',
+                reverse: true
+              },
+            },
+            filter: {
+              code: {
+                type: "pattern_capture",
+                preserve_original: 1,
+                patterns: [
+                  "(\\p{Ll}+|\\p{Lu}\\p{Ll}+|\\p{Lu}+)",
+                  "(\\d+)"
+                ]
+              }
+            },
+            char_filter: {
+              code_mapping: {
+                type: "mapping",
+                mappings: [
+                  ". => ' '"
+                ]
+              }
+            },
+          }
+        }
+      end
+    end
+  end
+end

lib/gitlab/elastic/search_results.rb

@@ -10,7 +10,7 @@ module Gitlab
       def initialize(current_user, query, limit_project_ids, public_and_internal_projects = true)
         @current_user = current_user
         @limit_project_ids = limit_project_ids
-        @query = Shellwords.shellescape(query) if query.present?
+        @query = query
         @public_and_internal_projects = public_and_internal_projects
       end
 

lib/tasks/gitlab/check.rake

@@ -1078,11 +1078,11 @@ namespace :gitlab do
   def check_elasticsearch
     client = Gitlab::Elastic::Client.build(current_application_settings.elasticsearch_config)
 
-    print "Elasticsearch version 5.1.x? ... "
+    print "Elasticsearch version 5.1 - 5.3? ... "
 
     version = Gitlab::VersionInfo.parse(client.info["version"]["number"])
 
-    if version.major == 5 && version.minor == 1
+    if version.major == 5 && (1..3).cover?(version.minor)
       puts "yes (#{version})".color(:green)
     else
       puts "no, you have #{version}".color(:red)

spec/features/merge_requests/create_new_mr_spec.rb

@@ -36,7 +36,7 @@ feature 'Create New Merge Request', feature: true, js: true do
 
     first('.js-target-branch').click
     first('.dropdown-target-branch .dropdown-content')
-    first('.dropdown-target-branch .dropdown-content a', text: 'v1.1.0').click
+    find('.dropdown-target-branch .dropdown-content a', text: 'v1.1.0', match: :first).click
 
     expect(page).to have_content "b83d6e3"
   end

spec/features/protected_tags/access_control_ce_spec.rb

@@ -10,7 +10,8 @@ RSpec.shared_examples "protected tags > access control > CE" do
 
         unless allowed_to_create_button.text == access_type_name
           allowed_to_create_button.click
-          within(".dropdown.open .dropdown-menu") { click_on access_type_name }
+          find('.create_access_levels-container .dropdown-menu li', match: :first)
+          within('.create_access_levels-container .dropdown-menu') { click_on access_type_name }
         end
       end
 

spec/features/protected_tags_spec.rb

@@ -11,6 +11,7 @@ feature 'Projected Tags', feature: true, js: true do
     find(".js-protected-tag-select").click
     find(".dropdown-input-field").set(tag_name)
     click_on("Create wildcard #{tag_name}")
+    find('.protected-tags-dropdown .dropdown-menu', visible: false)
   end
 
   describe "explicit protected tags" do

spec/models/concerns/elastic/issue_spec.rb

@@ -15,19 +15,19 @@ describe Issue, elastic: true do
 
   it "searches issues" do
     Sidekiq::Testing.inline! do
-      create :issue, title: 'bla-bla term', project: project
-      create :issue, description: 'bla-bla term', project: project
+      create :issue, title: 'bla-bla term1', project: project
+      create :issue, description: 'bla-bla term2', project: project
       create :issue, project: project
 
       # The issue I have no access to
-      create :issue, title: 'bla-bla term'
+      create :issue, title: 'bla-bla term3'
 
       Gitlab::Elastic::Helper.refresh_index
     end
 
     options = { project_ids: [project.id] }
 
-    expect(described_class.elastic_search('term', options: options).total_count).to eq(2)
+    expect(described_class.elastic_search('(term1 | term2 | term3) +bla-bla', options: options).total_count).to eq(2)
   end
 
   it "returns json with all needed elements" do

spec/models/concerns/elastic/merge_request_spec.rb

@@ -15,19 +15,19 @@ describe MergeRequest, elastic: true do
     project = create :project
 
     Sidekiq::Testing.inline! do
-      create :merge_request, title: 'bla-bla term', source_project: project
-      create :merge_request, description: 'term in description', source_project: project, target_branch: "feature2"
+      create :merge_request, title: 'bla-bla term1', source_project: project
+      create :merge_request, description: 'term2 in description', source_project: project, target_branch: "feature2"
       create :merge_request, source_project: project, target_branch: "feature3"
 
       # The merge request you have no access to
-      create :merge_request, title: 'also with term'
+      create :merge_request, title: 'also with term3'
 
       Gitlab::Elastic::Helper.refresh_index
     end
 
     options = { project_ids: [project.id] }
 
-    expect(described_class.elastic_search('term', options: options).total_count).to eq(2)
+    expect(described_class.elastic_search('term1 | term2 | term3', options: options).total_count).to eq(2)
   end
 
   it "returns json with all needed elements" do

spec/models/concerns/elastic/milestone_spec.rb

@@ -15,19 +15,19 @@ describe Milestone, elastic: true do
     project = create :empty_project
 
     Sidekiq::Testing.inline! do
-      create :milestone, title: 'bla-bla term', project: project
-      create :milestone, description: 'bla-bla term', project: project
+      create :milestone, title: 'bla-bla term1', project: project
+      create :milestone, description: 'bla-bla term2', project: project
       create :milestone, project: project
 
       # The milestone you have no access to
-      create :milestone, title: 'bla-bla term'
+      create :milestone, title: 'bla-bla term3'
 
       Gitlab::Elastic::Helper.refresh_index
     end
 
     options = { project_ids: [project.id] }
 
-    expect(described_class.elastic_search('term', options: options).total_count).to eq(2)
+    expect(described_class.elastic_search('(term1 | term2 | term3) +bla-bla', options: options).total_count).to eq(2)
   end
 
   it "returns json with all needed elements" do

spec/models/concerns/elastic/note_spec.rb

@@ -15,18 +15,18 @@ describe Note, elastic: true do
     issue = create :issue
 
     Sidekiq::Testing.inline! do
-      create :note, note: 'bla-bla term', project: issue.project
+      create :note, note: 'bla-bla term1', project: issue.project
       create :note, project: issue.project
 
       # The note in the project you have no access to
-      create :note, note: 'bla-bla term'
+      create :note, note: 'bla-bla term2'
 
       Gitlab::Elastic::Helper.refresh_index
     end
 
     options = { project_ids: [issue.project.id] }
 
-    expect(described_class.elastic_search('term', options: options).total_count).to eq(1)
+    expect(described_class.elastic_search('term1 | term2', options: options).total_count).to eq(1)
   end
 
   it "indexes && searches diff notes" do

spec/models/concerns/elastic/project_spec.rb

@@ -15,8 +15,8 @@ describe Project, elastic: true do
     project_ids = []
 
     Sidekiq::Testing.inline! do
-      project = create :empty_project, name: 'test'
-      project1 = create :empty_project, path: 'test1'
+      project = create :empty_project, name: 'test1'
+      project1 = create :empty_project, path: 'test2'
       project2 = create :empty_project
       create :empty_project, path: 'someone_elses_project'
       project_ids += [project.id, project1.id, project2.id]
@@ -24,8 +24,9 @@ describe Project, elastic: true do
       Gitlab::Elastic::Helper.refresh_index
     end
 
-    expect(described_class.elastic_search('test', options: { project_ids: project_ids }).total_count).to eq(1)
     expect(described_class.elastic_search('test1', options: { project_ids: project_ids }).total_count).to eq(1)
+    expect(described_class.elastic_search('test2', options: { project_ids: project_ids }).total_count).to eq(1)
+    expect(described_class.elastic_search('test*', options: { project_ids: project_ids }).total_count).to eq(2)
     expect(described_class.elastic_search('someone_elses_project', options: { project_ids: project_ids }).total_count).to eq(0)
   end
 

spec/models/concerns/elastic/project_wiki_spec.rb

@@ -15,12 +15,14 @@ describe ProjectWiki, elastic: true do
     project = create :empty_project
 
     Sidekiq::Testing.inline! do
-      project.wiki.create_page("index_page", "Bla bla")
+      project.wiki.create_page("index_page", "Bla bla term1")
+      project.wiki.create_page("omega_page", "Bla bla term2")
       project.wiki.index_blobs
 
       Gitlab::Elastic::Helper.refresh_index
     end
 
-    expect(project.wiki.search('bla', type: :blob)[:blobs][:total_count]).to eq(1)
+    expect(project.wiki.search('term1', type: :blob)[:blobs][:total_count]).to eq(1)
+    expect(project.wiki.search('term1 | term2', type: :blob)[:blobs][:total_count]).to eq(2)
   end
 end

spec/models/concerns/elastic/repository_spec.rb

@@ -22,6 +22,7 @@ describe Repository, elastic: true do
     end
 
     expect(project.repository.search('def popen')[:blobs][:total_count]).to eq(1)
+    expect(project.repository.search('def | popen')[:blobs][:total_count] > 1).to be_truthy
     expect(project.repository.search('initial')[:commits][:total_count]).to eq(1)
   end
 

spec/models/concerns/elastic/snippet_spec.rb

@@ -19,9 +19,9 @@ describe Snippet, elastic: true do
     let!(:internal_snippet) { create(:snippet, :internal, content: 'password: XXX') }
     let!(:private_snippet)  { create(:snippet, :private, content: 'password: XXX', author: author) }
 
-    let!(:project_public_snippet)   { create(:snippet, :public, project: project, content: 'password: XXX') }
-    let!(:project_internal_snippet) { create(:snippet, :internal, project: project, content: 'password: XXX') }
-    let!(:project_private_snippet)  { create(:snippet, :private, project: project, content: 'password: XXX') }
+    let!(:project_public_snippet)   { create(:snippet, :public, project: project, content: 'password: 123') }
+    let!(:project_internal_snippet) { create(:snippet, :internal, project: project, content: 'password: 456') }
+    let!(:project_private_snippet)  { create(:snippet, :private, project: project, content: 'password: 789') }
 
     before do
       Gitlab::Elastic::Helper.refresh_index
@@ -60,6 +60,16 @@ describe Snippet, elastic: true do
       expect(result.records).to match_array [public_snippet, internal_snippet, private_snippet]
     end
 
+    it 'supports advanced search syntax' do
+      member = create(:user)
+      project.add_reporter(member)
+
+      result = described_class.elastic_search_code('password +(123 | 789)', options: { user: member })
+
+      expect(result.total_count).to eq(2)
+      expect(result.records).to match_array [project_public_snippet, project_private_snippet]
+    end
+
     [:admin, :auditor].each do |user_type|
       it "returns all snippets for #{user_type}" do
         superuser = create(user_type)