Merge remote-tracking branch 'origin/master' into ce-to-ee

Signed-off-by: Rémy Coutable <remy@rymai.me>

.gitlab-ci.yml

@@ -55,13 +55,13 @@ stages:
   services:
     - postgres:latest
     - redis:alpine
-    - elasticsearch:5.1
+    - elasticsearch:5.3
 
 .use-mysql: &use-mysql
   services:
     - mysql:latest
     - redis:alpine
-    - elasticsearch:5.1
+    - elasticsearch:5.3
 
 .only-master-and-ee-or-mysql: &only-master-and-ee-or-mysql
   only:

Gemfile

@@ -113,7 +113,6 @@ gem 'seed-fu', '~> 2.3.5'
 gem 'elasticsearch-model', '~> 0.1.9'
 gem 'elasticsearch-rails', '~> 0.1.9'
 gem 'elasticsearch-api',   '5.0.3'
-gem 'gitlab-elasticsearch-git', '1.1.1', require: "elasticsearch/git"
 gem 'aws-sdk'
 gem 'faraday_middleware-aws-signers-v4'
 

Gemfile.lock

@@ -285,14 +285,6 @@ GEM
       mime-types (>= 1.19)
       rugged (>= 0.23.0b)
     github-markup (1.4.0)
-    gitlab-elasticsearch-git (1.1.1)
-      activemodel (~> 4.2)
-      activesupport (~> 4.2)
-      charlock_holmes (~> 0.7)
-      elasticsearch-api
-      elasticsearch-model (~> 0.1.9)
-      github-linguist (~> 4.7)
-      rugged (~> 0.24)
     gitlab-flowdock-git-hook (1.0.1)
       flowdock (~> 0.7)
       gitlab-grit (>= 2.4.1)
@@ -947,7 +939,6 @@ DEPENDENCIES
   gemojione (~> 3.0)
   gitaly (~> 0.5.0)
   github-linguist (~> 4.7.0)
-  gitlab-elasticsearch-git (= 1.1.1)
   gitlab-flowdock-git-hook (~> 1.0.1)
   gitlab-license (~> 1.0)
   gitlab-markup (~> 1.5.1)

app/models/concerns/elastic/application_search.rb

@@ -117,10 +117,10 @@ module Elastic
                          query: {
                            bool: {
                              must: [{
-                               multi_match: {
+                               simple_query_string: {
                                  fields: fields,
                                  query: query,
-                                 operator: :and
+                                 default_operator: :and
                                }
                              }]
                            }

app/models/concerns/elastic/notes_search.rb

@@ -50,14 +50,7 @@ module Elastic
       def self.elastic_search(query, options: {})
         options[:in] = ['note']
 
-        query_hash = {
-          query: {
-            bool: {
-              must: [{ match: { note: query } }],
-            },
-          }
-        }
-
+        query_hash = basic_query_hash(%w[note], query)
         query_hash = project_ids_filter(query_hash, options)
         query_hash = confidentiality_filter(query_hash, options[:current_user])
 

app/models/concerns/elastic/snippets_search.rb

@@ -52,23 +52,9 @@ module Elastic
       end
 
       def self.elastic_search_code(query, options: {})
-        query_hash = {
-          query: {
-            bool: {
-              must: [{ match: { content: query } }]
-            }
-          }
-        }
-
+        query_hash = basic_query_hash(%w(content), query)
         query_hash = filter(query_hash, options[:user])
 
-        query_hash[:sort] = [
-          { updated_at: { order: :desc } },
-          :_score
-        ]
-
-        query_hash[:highlight] = { fields: { content: {} } }
-
         self.__elasticsearch__.search(query_hash)
       end
 

app/views/projects/protected_tags/_create_protected_tag.html.haml

@@ -9,7 +9,7 @@
         .form-group
           = f.label :name, class: 'col-md-2 text-right' do
             Tag:
-          .col-md-10
+          .col-md-10.protected-tags-dropdown
             = render partial: "projects/protected_tags/dropdown", locals: { f: f }
             .help-block
               = link_to 'Wildcards', help_page_path('user/project/protected_tags', anchor: 'wildcard-protected-tags')

app/views/search/_form.html.haml

@@ -13,3 +13,7 @@
     - unless params[:snippets].eql? 'true'
       = render 'filter' if current_user
     = button_tag "Search", class: "btn btn-success btn-search"
+  - if current_application_settings.elasticsearch_search?
+    .help-block
+      = link_to 'Advanced search functionality', help_page_path('user/search/advanced-search-syntax.md'), target: '_blank'
+      is enabled.

bin/elastic_repo_indexer

@@ -3,12 +3,14 @@
 require 'rubygems'
 require 'bundler/setup'
 require 'json'
-require 'elasticsearch/git'
 require 'active_support'
 require 'active_support/core_ext'
 require 'benchmark'
 
-require File.expand_path('../lib/gitlab/elastic/client', File.dirname(__FILE__))
+$: << File.expand_path('../lib', File.dirname(__FILE__))
+
+require 'gitlab/elastic/client'
+require 'elasticsearch/git'
 
 Thread.abort_on_exception = true
 

changelogs/unreleased-ee/1052-advanced-search-syntax.yml

+---
+title: Support advanced search queries using elasticsearch
+merge_request: 1770
+author:

changelogs/unreleased-ee/2253-support-more-es-versions.yml

+---
+title: Support more elasticsearch versions
+merge_request: 1716
+author:

doc/articles/how_to_configure_ldap_gitlab_ce/index.md

+# How to configure LDAP with GitLab CE
+
+> **Type:** admin guide ||
+> **Level:** intermediary ||
+> **Author:** [Chris Wilson](https://gitlab.com/MrChrisW) ||
+> **Publication date:** 2017/05/03
+
+## Introduction
+
+Managing a large number of users in GitLab can become a burden for system administrators. As an organization grows so do user accounts. Keeping these user accounts in sync across multiple enterprise applications often becomes a time consuming task.
+
+In this guide we will focus on configuring GitLab with Active Directory. [Active Directory](https://en.wikipedia.org/wiki/Active_Directory) is a popular LDAP compatible directory service provided by Microsoft, included in all modern Windows Server operating systems.
+
+GitLab has supported LDAP integration since [version 2.2](https://about.gitlab.com/2012/02/22/gitlab-version-2-2/). With GitLab LDAP [group syncing](#group-syncing-ee) being added to GitLab Enterprise Edition in [version 6.0](https://about.gitlab.com/2013/08/20/gitlab-6-dot-0-released/). LDAP integration has become one of the most popular features in GitLab.
+
+## Getting started
+
+### Choosing an LDAP Server
+
+The main reason organizations choose to utilize a LDAP server is to keep the entire organization's user base consolidated into a central repository. Users can access multiple applications and systems across the  IT environment using a single login. Because LDAP is an open, vendor-neutral, industry standard application protocol, the number of applications using LDAP authentication continues to increase.
+
+There are many commercial and open source [directory servers](https://en.wikipedia.org/wiki/Directory_service#LDAP_implementations) that support the LDAP protocol. Deciding on the right directory server highly depends on the existing IT environment in which the server will be integrated with.
+
+For example, [Active Directory](https://technet.microsoft.com/en-us/library/hh831484(v=ws.11).aspx) is generally favored in a primarily Windows environment, as this allows quick integration with existing services. Other popular directory services include:
+
+- [Oracle Internet Directory](http://www.oracle.com/technetwork/middleware/id-mgmt/overview/index-082035.html)
+- [OpenLDAP](http://www.openldap.org/)
+- [389 Directory](http://directory.fedoraproject.org/)
+- [OpenDJ](https://forgerock.org/opendj/)
+- [ApacheDS](https://directory.apache.org/)
+
+> GitLab uses the [Net::LDAP](https://rubygems.org/gems/net-ldap) library under the hood. This means it supports all [IETF](https://tools.ietf.org/html/rfc2251) compliant LDAPv3 servers.
+
+### Active Directory (AD)
+
+We won't cover the installation and configuration of Windows Server or Active Directory Domain Services in this tutorial. There are a number of resources online to guide you through this process:
+
+- Install Windows Server 2012  - (_technet.microsoft.com_) - [Installing Windows Server 2012 ](https://technet.microsoft.com/en-us/library/jj134246(v=ws.11).aspx)
+
+- Install Active Directory Domain Services (AD DS) (_technet.microsoft.com_)- [Install Active Directory Domain Services](https://technet.microsoft.com/windows-server-docs/identity/ad-ds/deploy/install-active-directory-domain-services--level-100-#BKMK_PS)
+
+> **Shortcut:** You can quickly install AD DS via PowerShell using
+`Install-WindowsFeature AD-Domain-Services -IncludeManagementTools`
+
+### Creating an AD **OU** structure
+
+Configuring organizational units (**OU**s) is an important part of setting up Active Directory. **OU**s form the base for an entire organizational structure. Using GitLab as an example we have designed the **OU** structure below using the geographic **OU** model. In the Geographic Model we separate **OU**s for different geographic regions.
+
+| GitLab **OU** Design           |  GitLab AD Structure             |
+| :----------------------------: | :------------------------------: |
+| ![GitLab OU Design][gitlab_ou] |  ![GitLab AD Structure][ldap_ou] |
+
+[gitlab_ou]: img/gitlab_ou.png
+[ldap_ou]: img/ldap_ou.gif
+
+Using PowerShell you can output the **OU** structure as a table (_all names are examples only_):
+
+```ps
+Get-ADObject -LDAPFilter "(objectClass=*)" -SearchBase 'OU=GitLab INT,DC=GitLab,DC=org' -Properties CanonicalName | Format-Table Name,CanonicalName -A
+```
+
+```
+OU                CanonicalName
+----              -------------
+GitLab INT        GitLab.org/GitLab INT
+United States     GitLab.org/GitLab INT/United States
+Developers        GitLab.org/GitLab INT/United States/Developers
+Gary Johnson      GitLab.org/GitLab INT/United States/Developers/Gary Johnson
+Ellis Matthews    GitLab.org/GitLab INT/United States/Developers/Ellis Matthews
+William Collins   GitLab.org/GitLab INT/United States/Developers/William Collins
+People Ops        GitLab.org/GitLab INT/United States/People Ops
+Margaret Baker    GitLab.org/GitLab INT/United States/People Ops/Margaret Baker
+Libby Hartzler    GitLab.org/GitLab INT/United States/People Ops/Libby Hartzler
+Victoria Ryles    GitLab.org/GitLab INT/United States/People Ops/Victoria Ryles
+The Netherlands   GitLab.org/GitLab INT/The Netherlands
+Developers        GitLab.org/GitLab INT/The Netherlands/Developers
+John Doe          GitLab.org/GitLab INT/The Netherlands/Developers/John Doe
+Jon Mealy         GitLab.org/GitLab INT/The Netherlands/Developers/Jon Mealy
+Jane Weingarten   GitLab.org/GitLab INT/The Netherlands/Developers/Jane Weingarten
+Production        GitLab.org/GitLab INT/The Netherlands/Production
+Sarah Konopka     GitLab.org/GitLab INT/The Netherlands/Production/Sarah Konopka
+Cynthia Bruno     GitLab.org/GitLab INT/The Netherlands/Production/Cynthia Bruno
+David George      GitLab.org/GitLab INT/The Netherlands/Production/David George
+United Kingdom    GitLab.org/GitLab INT/United Kingdom
+Developers        GitLab.org/GitLab INT/United Kingdom/Developers
+Leroy Fox         GitLab.org/GitLab INT/United Kingdom/Developers/Leroy Fox
+Christopher Alley GitLab.org/GitLab INT/United Kingdom/Developers/Christopher Alley
+Norris Morita     GitLab.org/GitLab INT/United Kingdom/Developers/Norris Morita
+Support           GitLab.org/GitLab INT/United Kingdom/Support
+Laura Stanley     GitLab.org/GitLab INT/United Kingdom/Support/Laura Stanley
+Nikki Schuman     GitLab.org/GitLab INT/United Kingdom/Support/Nikki Schuman
+Harriet Butcher   GitLab.org/GitLab INT/United Kingdom/Support/Harriet Butcher
+Global Groups     GitLab.org/GitLab INT/Global Groups
+DevelopersNL      GitLab.org/GitLab INT/Global Groups/DevelopersNL
+DevelopersUK      GitLab.org/GitLab INT/Global Groups/DevelopersUK
+DevelopersUS      GitLab.org/GitLab INT/Global Groups/DevelopersUS
+ProductionNL      GitLab.org/GitLab INT/Global Groups/ProductionNL
+SupportUK         GitLab.org/GitLab INT/Global Groups/SupportUK
+People Ops US     GitLab.org/GitLab INT/Global Groups/People Ops US
+Global Admins     GitLab.org/GitLab INT/Global Groups/Global Admins
+```
+
+> See [more information](https://technet.microsoft.com/en-us/library/ff730967.aspx) on searching Active Directory with Windows PowerShell from [The Scripting Guys](https://technet.microsoft.com/en-us/scriptcenter/dd901334.aspx)
+
+## GitLab LDAP configuration
+
+The initial configuration of LDAP in GitLab requires changes to the `gitlab.rb` configuration file. Below is an example of a complete configuration using an Active Directory.
+
+The two Active Directory specific values are `active_directory: true` and `uid: 'sAMAccountName'`. `sAMAccountName` is an attribute returned by Active Directory used for GitLab usernames. See the example output from `ldapsearch` for a full list of attributes a "person" object (user) has in **AD** - [`ldapsearch` example](#using-ldapsearch-unix)
+
+> Both group_base and admin_group configuration options are only available in GitLab Enterprise Edition. See [GitLab EE - LDAP Features](#gitlab-enterprise-edition---ldap-features)
+
+### Example `gitlab.rb` LDAP
+
+```
+gitlab_rails['ldap_enabled'] = true
+gitlab_rails['ldap_servers'] = {
+'main' => {
+  'label' => 'GitLab AD',
+  'host' =>  'ad.example.org',
+  'port' => 636,
+  'uid' => 'sAMAccountName',
+  'method' => 'ssl',
+  'bind_dn' => 'CN=GitLabSRV,CN=Users,DC=GitLab,DC=org',
+  'password' => 'Password1',
+  'active_directory' => true,
+  'base' => 'OU=GitLab INT,DC=GitLab,DC=org',
+  'group_base' => 'OU=Global Groups,OU=GitLab INT,DC=GitLab,DC=org',
+  'admin_group' => 'Global Admins'
+  }
+}
+```
+
+> **Note:** Remember to run  `gitlab-ctl reconfigure` after modifying `gitlab.rb`
+
+## Security improvements (LDAPS)
+
+Security is an important aspect when deploying an LDAP server. By default, LDAP traffic is transmitted unsecured. LDAP can be secured using SSL/TLS called LDAPS, or commonly "LDAP over SSL".
+
+Securing LDAP (enabling LDAPS) on Windows Server 2012 involves installing a valid SSL certificate. For full details see Microsoft's guide [How to enable LDAP over SSL with a third-party certification authority](https://support.microsoft.com/en-us/help/321051/how-to-enable-ldap-over-ssl-with-a-third-party-certification-authority)
+
+> By default a LDAP service listens for connections on TCP and UDP port 389. LDAPS (LDAP over SSL) listens on port 636
+
+### Testing you AD server
+
+#### Using **AdFind** (Windows)
+
+You can use the [`AdFind`](https://social.technet.microsoft.com/wiki/contents/articles/7535.adfind-command-examples.aspx) utility (on Windows based systems) to test that your LDAP server is accessible and authentication is working correctly. This is a freeware utility built by [Joe Richards](http://www.joeware.net/freetools/tools/adfind/index.htm).
+
+**Return all objects**
+
+You can use the filter `objectclass=*` to return all directory objects.
+
+```sh
+adfind -h ad.example.org:636 -ssl -u "CN=GitLabSRV,CN=Users,DC=GitLab,DC=org" -up Password1 -b "OU=GitLab INT,DC=GitLab,DC=org" -f (objectClass=*)
+```
+
+**Return single object using filter**
+
+You can also retrieve a single object by **specifying** the object name or full **DN**. In this example we specify the object name only `CN=Leroy Fox`.
+
+```sh
+adfind -h ad.example.org:636 -ssl -u "CN=GitLabSRV,CN=Users,DC=GitLab,DC=org" -up Password1 -b "OU=GitLab INT,DC=GitLab,DC=org" -f (&(objectcategory=person)(CN=Leroy Fox))”
+```
+
+#### Using **ldapsearch** (Unix)
+
+You can use the `ldapsearch` utility (on Unix based systems) to test that your LDAP server is accessible and authentication is working correctly. This utility is included in the [`ldap-utils`](https://wiki.debian.org/LDAP/LDAPUtils) package.
+
+**Return all objects**
+
+You can use the filter `objectclass=*` to return all directory objects.
+
+```sh
+ldapsearch -D "CN=GitLabSRV,CN=Users,DC=GitLab,DC=org" \
+-w Password1 -p 636 -h ad.example.org \
+-b "OU=GitLab INT,DC=GitLab,DC=org" -Z \
+-s sub "(objectclass=*)"
+```
+
+**Return single object using filter**
+
+You can also retrieve a single object by **specifying** the object name or full **DN**. In this example we specify the object name only `CN=Leroy Fox`.
+
+```sh
+ldapsearch -D "CN=GitLabSRV,CN=Users,DC=GitLab,DC=org" -w Password1 -p 389 -h ad.example.org -b "OU=GitLab INT,DC=GitLab,DC=org" -Z -s sub "CN=Leroy Fox"
+```
+
+**Full output of `ldapsearch` command:** - Filtering for _CN=Leroy Fox_
+
+```
+# LDAPv3
+# base <OU=GitLab INT,DC=GitLab,DC=org> with scope subtree
+# filter: CN=Leroy Fox
+# requesting: ALL
+#
+
+# Leroy Fox, Developers, United Kingdom, GitLab INT, GitLab.org
+dn: CN=Leroy Fox,OU=Developers,OU=United Kingdom,OU=GitLab INT,DC=GitLab,DC=or
+ g
+objectClass: top
+objectClass: person
+objectClass: organizationalPerson
+objectClass: user
+cn: Leroy Fox
+sn: Fox
+givenName: Leroy
+distinguishedName: CN=Leroy Fox,OU=Developers,OU=United Kingdom,OU=GitLab INT,
+ DC=GitLab,DC=org
+instanceType: 4
+whenCreated: 20170210030500.0Z
+whenChanged: 20170213050128.0Z
+displayName: Leroy Fox
+uSNCreated: 16790
+memberOf: CN=DevelopersUK,OU=Global Groups,OU=GitLab INT,DC=GitLab,DC=org
+uSNChanged: 20812
+name: Leroy Fox
+objectGUID:: rBCAo6NR6E6vfSKgzcUILg==
+userAccountControl: 512
+badPwdCount: 0
+codePage: 0
+countryCode: 0
+badPasswordTime: 0
+lastLogoff: 0
+lastLogon: 0
+pwdLastSet: 131311695009850084
+primaryGroupID: 513
+objectSid:: AQUAAAAAAAUVAAAA9GMAb7tdJZvsATf7ZwQAAA==
+accountExpires: 9223372036854775807
+logonCount: 0
+sAMAccountName: Leroyf
+sAMAccountType: 805306368
+userPrincipalName: Leroyf@GitLab.org
+objectCategory: CN=Person,CN=Schema,CN=Configuration,DC=GitLab,DC=org
+dSCorePropagationData: 16010101000000.0Z
+lastLogonTimestamp: 131314356887754250
+
+# search result
+search: 2
+result: 0 Success
+
+# numResponses: 2
+# numEntries: 1
+```
+
+## Basic user authentication
+
+After configuring LDAP, basic authentication will be available. Users can then login using their directory credentials. An extra tab is added to the GitLab login screen for the configured LDAP server (e.g "**GitLab AD**").
+
+![GitLab OU Structure](img/user_auth.gif)
+
+Users that are removed from the LDAP base group (e.g `OU=GitLab INT,DC=GitLab,DC=org`) will be **blocked** in GitLab. [More information](../../administration/auth/ldap.md#security) on LDAP security.
+
+If `allow_username_or_email_login` is enabled in the LDAP configuration, GitLab will ignore everything after the first '@' in the LDAP username used on login. Example: The username `jon.doe@example.com` is converted to `jon.doe` when authenticating with the LDAP server. Disable this setting if you use `userPrincipalName` as the `uid`.
+
+## LDAP extended features on GitLab EE
+
+With [GitLab Enterprise Edition (EE)](https://about.gitlab.com/giltab-ee/), besides everything we just described, you'll
+have extended functionalities with LDAP, such as:
+
+- Group sync
+- Group permissions
+- Updating user permissions
+- Multiple LDAP servers
+
+Read through the article on [LDAP for GitLab EE](../how_to_configure_ldap_gitlab_ee/index.md) for an overview.

doc/articles/how_to_configure_ldap_gitlab_ee/index.md

+# How to configure LDAP with GitLab EE
+
+> **Type:** admin guide ||
+> **Level:** intermediary ||
+> **Author:** [Chris Wilson](https://gitlab.com/MrChrisW) ||
+> **Publication date:** 2017/05/03
+
+## Introduction
+
+The present article follows [How to Configure LDAP with GitLab CE](../how_to_configure_ldap_gitlab_ce/index.md). Make sure to read through it before moving forward.
+
+## GitLab Enterprise Edition - LDAP features
+
+[GitLab Enterprise Edition (EE)](https://about.gitlab.com/gitlab-ee/) has a number of advantages when it comes to integrating with Active Directory (LDAP):
+
+- [Administrator Sync](#administrator-sync): As an extension of group sync, you can automatically manage your global GitLab administrators. Specify a group CN for `admin_group` and all members of the LDAP group will be given administrator privileges.
+- [Group Sync](#group-sync): This allows GitLab group membership to be automatically updated based on LDAP group members.
+- [Multiple LDAP servers](#multiple-ldap-servers): The ability to configure multiple LDAP servers. This is useful if an organization has different LDAP servers within departments. This is not designed for failover. We're working on [supporting LDAP failover](https://gitlab.com/gitlab-org/gitlab-ee/issues/139) in GitLab.
+
+- Daily user synchronization: Once a day, GitLab will run a synchronization to check and update GitLab users against LDAP. This process updates all user details automatically.
+
+On the following section, you'll find a description for each of these features. Read through [LDAP GitLab EE docs](../../administration/auth/ldap-ee.md) for complementary information.
+
+![GitLab OU Structure](img/admin_group.png)
+
+All members of the group `Global Admins` will be given **administrator** access to GitLab, allowing them to view the `/admin` dashboard.
+
+### Group Sync
+
+Group syncing allows AD (LDAP) groups to be mapped to GitLab groups. This provides more control over per-group user management. To configure group syncing edit the `group_base` **DN** (`'OU=Global Groups,OU=GitLab INT,DC=GitLab,DC=org'`). This **OU** contains all groups that will be associated with [GitLab groups](../../workflow/groups.md).
+
+#### Creating group links - example
+
+As an example, let's suppose we have a "UKGov" GitLab group, which deals with confidential government information. Therefore, it is important that users of this group are given the correct permissions to projects contained within the group. Granular group permissions can be applied based on the AD group.
+
+**UK Developers** of our "UKGov" group are given **"developer"** permissions.
+
+_The developer permission allows the development staff to effectively manage all project code, issues, and merge requests._
+
+**UK Support** staff of our "UKGov" group are given **"reporter"** permissions.
+
+_The reporter permission allows support staff to manage issues, labels, and review project code._
+
+**US People Ops** of our "UKGov" group are given **"guest"** permissions.
+
+![Creating group links](img/group_linking.gif)
+
+> Guest permissions allows people ops staff to review and lodge new issues while allowing no read or write access to project code or [confidential issues](../../user/project/issues/confidential_issues.md#permissions-and-access-to-confidential-issues) created by other users.
+
+See the [permission list](../../user/permissions.md) for complementary info.
+
+#### Group permissions - example
+
+Considering the previous example, our staff will have
+access to our GitLab instance with the following structure:
+
+![GitLab OU Structure](img/group_link_final.png)
+
+Using this permission structure in our example allows only UK staff access to sensitive information stored in the projects code, while still allowing other teams to work effectively. As all permissions are controlled via AD groups new users can be quickly added to existing groups. New group members will then automatically inherit the required permissions.
+
+> [More information](../../administration/auth/ldap-ee.md#group-sync) on group syncing.
+
+### Updating user permissions - new feature
+
+Since GitLab [v8.15](https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/822) LDAP user permissions can now be manually overridden by an admin user. To override a user's permissions visit the groups **Members** page and select **Edit permissions**.
+
+![Setting manual permissions](img/manual_permissions.gif)
+
+### Multiple LDAP servers
+
+GitLab EE can support multiple LDAP servers. Simply configure another server in the `gitlab.rb` file within the `ldap_servers` block. In the example below we configure a new secondary server with the label **GitLab Secondary AD**. This is shown on the GitLab login screen. Large enterprises often utilize multiple LDAP servers for segregating organizational departments.
+
+![Multiple LDAP Servers Login](img/multi_login.gif)
+
+Considering the example illustrated on the image above,
+our `gitlab.rb` configuration would look like:
+
+```ruby
+gitlab_rails['ldap_enabled'] = true
+gitlab_rails['ldap_servers'] = {
+'main' => {
+  'label' => 'GitLab AD',
+  'host' =>  'ad.example.org',
+  'port' => 636,
+  'uid' => 'sAMAccountName',
+  'method' => 'ssl',
+  'bind_dn' => 'CN=GitLabSRV,CN=Users,DC=GitLab,DC=org',
+  'password' => 'Password1',
+  'active_directory' => true,
+  'base' => 'OU=GitLab INT,DC=GitLab,DC=org',
+  'group_base' => 'OU=Global Groups,OU=GitLab INT,DC=GitLab,DC=org',
+  'admin_group' => 'Global Admins'
+  },
+
+'secondary' => {
+  'label' => 'GitLab Secondary AD',
+  'host' =>  'ad-secondary.example.net',
+  'port' => 636,
+  'uid' => 'sAMAccountName',
+  'method' => 'ssl',
+  'bind_dn' => 'CN=GitLabSRV,CN=Users,DC=GitLab,DC=com',
+  'password' => 'Password1',
+  'active_directory' => true,
+  'base' => 'OU=GitLab Secondary,DC=GitLab,DC=com',
+  'group_base' => 'OU=Global Groups,OU=GitLab INT,DC=GitLab,DC=com',
+  'admin_group' => 'Global Admins'
+  }
+}
+```
+
+## Conclusion
+
+Integration of GitLab with Active Directory (LDAP) reduces the complexity of user management.
+It has the advantage of improving user permission controls, whilst easing the deployment of GitLab into an existing [IT environment](https://www.techopedia.com/definition/29199/it-infrastructure). GitLab EE offers advanced group management and multiple LDAP servers.
+
+With the assistance of the [GitLab Support](https://about.gitlab.com/support) team, setting up GitLab with an existing AD/LDAP solution will be a smooth and painless process.

doc/articles/index.md

@@ -7,6 +7,12 @@ to provide the community with guidance on specific processes to achieve certain
 They are written by members of the GitLab Team and by
 [Community Writers](https://about.gitlab.com/handbook/product/technical-writing/community-writers/).
 
+## Authentication
+
+- **LDAP**
+  - [How to configure LDAP with GitLab CE](how_to_configure_ldap_gitlab_ce/index.md)
+  - [How to configure LDAP with GitLab EE](how_to_configure_ldap_gitlab_ee/index.md)
+
 ## GitLab Pages
 
 - **GitLab Pages from A to Z**

doc/ci/pipelines.md

 # Introduction to pipelines and jobs
 
->**Note:**
-Introduced in GitLab 8.8.
+> Introduced in GitLab 8.8.
 
 ## Pipelines
 
@@ -9,11 +8,17 @@ A pipeline is a group of [jobs][] that get executed in [stages][](batches).
 All of the jobs in a stage are executed in parallel (if there are enough
 concurrent [Runners]), and if they all succeed, the pipeline moves on to the
 next stage. If one of the jobs fails, the next stage is not (usually)
-executed.
+executed. You can access the pipelines page in your project's **Pipelines** tab.
+
+In the following image you can see that the pipeline consists of four stages
+(`build`, `test`, `staging`, `production`) each one having one or more jobs.
+
+>**Note:**
+GitLab capitalizes the stages' names when shown in the [pipeline graphs](#pipeline-graphs).
 
 ![Pipelines example](img/pipelines.png)
 
-## Types of Pipelines
+## Types of pipelines
 
 There are three types of pipelines that often use the single shorthand of "pipeline". People often talk about them as if each one is "the" pipeline, but really, they're just pieces of a single, comprehensive pipeline.
 
@@ -23,7 +28,7 @@ There are three types of pipelines that often use the single shorthand of "pipel
 2. **Deploy Pipeline**: Deploy stage(s) defined in `.gitlab-ci.yml` The flow of deploying code to servers through various stages: e.g. development to staging to production
 3. **Project Pipeline**: Cross-project CI dependencies [triggered via API][triggers], particularly for micro-services, but also for complicated build dependencies: e.g. api -> front-end, ce/ee -> omnibus.
 
-## Development Workflows
+## Development workflows
 
 Pipelines accommodate several development workflows:
 
@@ -45,19 +50,142 @@ confused with a `build` job or `build` stage.
 Pipelines are defined in `.gitlab-ci.yml` by specifying [jobs] that run in
 [stages].
 
-See full [documentation](yaml/README.md#jobs).
+See the reference [documentation for jobs](yaml/README.md#jobs).
 
 ## Seeing pipeline status
 
-You can find the current and historical pipeline runs under **Pipelines** for
-your project.
+You can find the current and historical pipeline runs under your project's
+**Pipelines** tab. Clicking on a pipeline will show the jobs that were run for
+that pipeline.
+
+![Pipelines index page](img/pipelines_index.png)
 
 ## Seeing job status
 
-Clicking on a pipeline will show the jobs that were run for that pipeline.
+When you visit a single pipeline you can see the related jobs for that pipeline.
 Clicking on an individual job will show you its job trace, and allow you to
 cancel the job, retry it, or erase the job trace.
 
+![Pipelines example](img/pipelines.png)
+
+## Pipeline graphs
+
+> [Introduced][ce-5742] in GitLab 8.11.
+
+Pipelines can be complex structures with many sequential and parallel jobs.
+To make it a little easier to see what is going on, you can view a graph
+of a single pipeline and its status.
+
+A pipeline graph can be shown in two different ways depending on what page you
+are on.
+
+---
+
+The regular pipeline graph that shows the names of the jobs of each stage can
+be found when you are on a [single pipeline page](#seeing-pipeline-status).
+
+![Pipelines example](img/pipelines.png)
+
+Then, there is the pipeline mini graph which takes less space and can give you a
+quick glance if all jobs pass or something failed. The pipeline mini graph can
+be found when you visit:
+
+- the pipelines index page
+- a single commit page
+- a merge request page
+
+That way, you can see all related jobs for a single commit and the net result
+of each stage of your pipeline. This allows you to quickly see what failed and
+fix it. Stages in pipeline mini graphs are collapsible. Hover your mouse over
+them and click to expand their jobs.
+
+| **Mini graph** | **Mini graph expanded** |
+| :------------: | :---------------------: |
+| ![Pipelines mini graph](img/pipelines_mini_graph_simple.png) | ![Pipelines mini graph extended](img/pipelines_mini_graph.png) |
+
+### Grouping similar jobs in the pipeline graph
+
+> [Introduced][ce-6242] in GitLab 8.12.
+
+If you have many similar jobs, your pipeline graph becomes very long and hard
+to read. For that reason, similar jobs can automatically be grouped together.
+If the job names are formatted in certain ways, they will be collapsed into
+a single group in regular pipeline graphs (not the mini graphs).
+You'll know when a pipeline has grouped jobs if you don't see the retry or
+cancel button inside them. Hovering over them will show the number of grouped
+jobs. Click to expand them.
+
+![Grouped pipelines](img/pipelines_grouped.png)
+
+The basic requirements is that there are two numbers separated with one of
+the following (you can even use them interchangeably):
+
+- a space
+- a backslash (`/`)
+- a colon (`:`)
+
+>**Note:**
+More specifically, [it uses][regexp] this regular expression: `\d+[\s:\/\\]+\d+\s*`.
+
+The jobs will be ordered by comparing those two numbers from left to right. You
+usually want the first to be the index and the second the total.
+
+For example, the following jobs will be grouped under a job named `test`:
+
+- `test 0 3` => `test`
+- `test 1 3` => `test`
+- `test 2 3` => `test`
+
+The following jobs will be grouped under a job named `test ruby`:
+
+- `test 1:2 ruby` => `test ruby`
+- `test 2:2 ruby` => `test ruby`
+
+The following jobs will be grouped under a job named `test ruby` as well:
+
+- `1/3 test ruby` => `test ruby`
+- `2/3 test ruby` => `test ruby`
+- `3/3 test ruby` => `test ruby`
+
+### Manual actions from the pipeline graph
+
+> [Introduced][ce-7931] in GitLab 8.15.
+
+[Manual actions][manual] allow you to require manual interaction before moving
+forward with a particular job in CI. Your entire pipeline can run automatically,
+but the actual [deploy to production][env-manual] will require a click.
+
+You can do this straight from the pipeline graph. Just click on the play button
+to execute that particular job. For example, in the image below, the `production`
+stage has a job with a manual action.
+
+![Pipelines example](img/pipelines.png)
+
+### Ordering of jobs in pipeline graphs
+
+**Regular pipeline graph**
+
+In the single pipeline page, jobs are sorted by name.
+
+**Mini pipeline graph**
+
+> [Introduced][ce-9760] in GitLab 9.0.
+
+In the pipeline mini graphs, the jobs are sorted first by severity and then
+by name. The order of severity is:
+
+- failed
+- warning
+- pending
+- running
+- manual
+- canceled
+- success
+- skipped
+- created
+
+![Pipeline mini graph sorting](img/pipelines_mini_graph_sorting.png)
+
 ## How the pipeline duration is calculated
 
 Total running time for a given pipeline would exclude retries and pending
@@ -96,7 +224,14 @@ respective link in the [Pipelines settings] page.
 
 [jobs]: #jobs
 [jobs-yaml]: yaml/README.md#jobs
+[manual]: yaml/README.md#manual
+[env-manual]: environments.md#manually-deploying-to-environments
 [stages]: yaml/README.md#stages
 [runners]: runners/README.html
 [pipelines settings]: ../user/project/pipelines/settings.md
 [triggers]: triggers/README.md
+[ce-5742]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/5742
+[ce-6242]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/6242
+[ce-7931]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/7931
+[ce-9760]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/9760
+[regexp]: https://gitlab.com/gitlab-org/gitlab-ce/blob/2f3dc314f42dbd79813e6251792853bc231e69dd/app/models/commit_status.rb#L99

doc/integration/elasticsearch.md

@@ -33,7 +33,7 @@ service.
 | GitLab version | Elasticsearch version |
 | -------------- | --------------------- |
 | GitLab Enterprise Edition 8.4 - 8.17  | Elasticsearch 2.4 with [Delete By Query Plugin](https://www.elastic.co/guide/en/elasticsearch/plugins/2.4/plugins-delete-by-query.html) installed |
-| GitLab Enterprise Edition 9.0+        | Elasticsearch 5.1 |
+| GitLab Enterprise Edition 9.0+        | Elasticsearch 5.1 - 5.3 |
 
 ## Install Elasticsearch
 

doc/topics/authentication/index.md

@@ -18,6 +18,8 @@ This page gathers all the resources for the topic **Authentication** within GitL
 - [LDAP (Enterprise Edition)](https://docs.gitlab.com/ee/administration/auth/ldap-ee.html)
 - [Enforce Two-factor Authentication (2FA)](../../security/two_factor_authentication.md#enforce-two-factor-authentication-2fa)
 - **Articles:**
+  - [How to Configure LDAP with GitLab CE](../../articles/how_to_configure_ldap_gitlab_ce/index.md)
+  - [How to Configure LDAP with GitLab EE](../../articles/how_to_configure_ldap_gitlab_ee/index.md)
   - [Feature Highlight: LDAP Integration](https://about.gitlab.com/2014/07/10/feature-highlight-ldap-sync/)
   - [Debugging LDAP](https://about.gitlab.com/handbook/support/workflows/ldap/debugging_ldap.html)
 - **Integrations:**

doc/user/search/advanced-search-syntax.md

+## Advanced search syntax
+
+If your site administrator has enabled [Elasticsearch integration](../../integration/elasticsearch.md)
+then some advanced search functionality is available.
+
+Full details can be found in the
+[Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-simple-query-string-query.html#_simple_query_string_syntax)
+but here's a quick guide:
+
+* Searches look for all the words in a query, in any order - e.g.: searching
+  issues for `display bug` will return all issues matching both those words, in any order.
+* To find the exact term, use double quotes: `"display bug"`
+* To find bugs not mentioning display, use `-`: `bug -display`
+* To find a bug in display or sound, use `|`: `bug display | sound`
+* To group terms together, use parentheses: `bug | (display +sound)`
+* To match a partial word, use `*`: `bug find_by_*`
+* To find a term containing one of these symbols, use `\`: `argument \-last`
+

lib/elasticsearch/git.rb

+require "elasticsearch/git/model"
+require "elasticsearch/git/repository"
+
+module Elasticsearch
+  module Git
+  end
+end

lib/elasticsearch/git/encoder_helper.rb

+require 'active_support/concern'
+require 'charlock_holmes'
+
+module Elasticsearch
+  module Git
+    module EncoderHelper
+      extend ActiveSupport::Concern
+
+      included do
+        def encode!(message)
+          return nil unless message.respond_to? :force_encoding
+
+          # if message is utf-8 encoding, just return it
+          message.force_encoding("UTF-8")
+          return message if message.valid_encoding?
+
+          # return message if message type is binary
+          detect = CharlockHolmes::EncodingDetector.detect(message)
+          return message.force_encoding("BINARY") if detect && detect[:type] == :binary
+
+          # encoding message to detect encoding
+          if detect && detect[:encoding]
+            message.force_encoding(detect[:encoding])
+          end
+
+          # encode and clean the bad chars
+          message.replace clean(message)
+        rescue
+          encoding = detect ? detect[:encoding] : "unknown"
+          "--broken encoding: #{encoding}"
+        end
+
+        private
+
+        def clean(message)
+          message.encode("UTF-16BE", undef: :replace, invalid: :replace, replace: "")
+          .encode("UTF-8")
+          .gsub("\0".encode("UTF-8"), "")
+        end
+      end
+    end
+  end
+end

lib/elasticsearch/git/lite_blob.rb

+require 'linguist'
+require 'elasticsearch/git/encoder_helper'
+
+module Elasticsearch
+  module Git
+    class LiteBlob
+      include Linguist::BlobHelper
+      include Elasticsearch::Git::EncoderHelper
+
+      attr_accessor :id, :name, :path, :size, :mode, :commit_id
+      attr_writer :data
+
+      def initialize(repo, raw_blob_hash)
+        @id   = raw_blob_hash[:oid]
+        @blob = repo.lookup(@id)
+
+        @mode = raw_blob_hash[:mode].to_s(8)
+        @size = @blob.size
+        @path = encode!(raw_blob_hash[:path])
+        @name = @path.split('/').last
+      end
+
+      def data
+        @data ||= encode!(@blob.content)
+      end
+    end
+  end
+end

lib/elasticsearch/git/model.rb

+require 'active_support/concern'
+require 'active_model'
+require 'elasticsearch/model'
+
+module Elasticsearch
+  module Git
+    module Model
+      extend ActiveSupport::Concern
+
+      included do
+        extend ActiveModel::Naming
+        include ActiveModel::Model
+        include Elasticsearch::Model
+
+        env = if defined?(::Rails)
+                ::Rails.env.to_s
+              else
+                nil
+              end
+
+        index_name [self.name.downcase, 'index', env].compact.join('-')
+
+        settings \
+          index: {
+          analysis: {
+            analyzer: {
+              path_analyzer: {
+                type: 'custom',
+                tokenizer: 'path_tokenizer',
+                filter: %w(lowercase asciifolding)
+              },
+              sha_analyzer: {
+                type: 'custom',
+                tokenizer: 'sha_tokenizer',
+                filter: %w(lowercase asciifolding)
+              },
+              code_analyzer: {
+                type: 'custom',
+                tokenizer: 'standard',
+                filter: %w(code lowercase asciifolding),
+                char_filter: ["code_mapping"]
+              },
+              code_search_analyzer: {
+                type: 'custom',
+                tokenizer: 'standard',
+                filter: %w(lowercase asciifolding),
+                char_filter: ["code_mapping"]
+              }
+            },
+            tokenizer: {
+              sha_tokenizer: {
+                type: "edgeNGram",
+                min_gram: 5,
+                max_gram: 40,
+                token_chars: %w(letter digit)
+              },
+              path_tokenizer: {
+                type: 'path_hierarchy',
+                reverse: true
+              },
+            },
+            filter: {
+              code: {
+                type: "pattern_capture",
+                preserve_original: 1,
+                patterns: [
+                  "(\\p{Ll}+|\\p{Lu}\\p{Ll}+|\\p{Lu}+)",
+                  "(\\d+)"
+                ]
+              }
+            },
+            char_filter: {
+              code_mapping: {
+                type: "mapping",
+                mappings: [
+                  ". => ' '"
+                ]
+              }
+            },
+          }
+        }
+      end
+    end
+  end
+end

lib/elasticsearch/git/repository.rb

+require 'active_support/concern'
+require 'active_model'
+require 'elasticsearch'
+require 'elasticsearch/git/model'
+require 'elasticsearch/git/encoder_helper'
+require 'elasticsearch/git/lite_blob'
+require 'rugged'
+require 'open3'
+
+module Elasticsearch
+  module Git
+    module Repository
+      CreateIndexException = Class.new(StandardError)
+
+      BLOBS_BATCH = 100
+      COMMMITS_BATCH = 500
+
+      extend ActiveSupport::Concern
+
+      included do
+        include Elasticsearch::Git::Model
+        include Elasticsearch::Git::EncoderHelper
+
+        mapping _parent: { type: 'project' } do
+          indexes :blob do
+            indexes :id,          type: :text,
+                                  index_options: 'offsets',
+                                  analyzer: :sha_analyzer
+            indexes :rid,         type: :keyword
+            indexes :oid,         type: :text,
+                                  index_options: 'offsets',
+                                  analyzer: :sha_analyzer
+            indexes :commit_sha,  type: :text,
+                                  index_options: 'offsets',
+                                  analyzer: :sha_analyzer
+            indexes :path,        type: :text,
+                                  analyzer: :path_analyzer
+            indexes :file_name,   type: :text,
+                                  analyzer: :code_analyzer,
+                                  search_analyzer: :code_search_analyzer
+            indexes :content,     type: :text,
+                                  index_options: 'offsets',
+                                  analyzer: :code_analyzer,
+                                  search_analyzer: :code_search_analyzer
+            indexes :language,    type: :keyword
+          end
+
+          indexes :commit do
+            indexes :id,          type: :text,
+                                  index_options: 'offsets',
+                                  analyzer: :sha_analyzer
+            indexes :rid,         type: :keyword
+            indexes :sha,         type: :text,
+                                  index_options: 'offsets',
+                                  analyzer: :sha_analyzer
+
+            indexes :author do
+              indexes :name,      type: :text, index_options: 'offsets'
+              indexes :email,     type: :text, index_options: 'offsets'
+              indexes :time,      type: :date, format: :basic_date_time_no_millis
+            end
+
+            indexes :commiter do
+              indexes :name,      type: :text, index_options: 'offsets'
+              indexes :email,     type: :text, index_options: 'offsets'
+              indexes :time,      type: :date, format: :basic_date_time_no_millis
+            end
+
+            indexes :message,     type: :text, index_options: 'offsets'
+          end
+        end
+
+        # Indexing all text-like blobs in repository
+        #
+        # All data stored in global index
+        # Repository can be selected by 'rid' field
+        # If you want - this field can be used for store 'project' id
+        #
+        # blob {
+        #   id - uniq id of blob from all repositories
+        #   oid - blob id in repository
+        #   content - blob content
+        #   commit_sha - last actual commit sha
+        # }
+        #
+        # For search from blobs use type 'blob'
+        def index_blobs(from_rev: nil, to_rev: repository_for_indexing.last_commit.oid)
+          from, to = parse_revs(from_rev, to_rev)
+
+          diff = repository_for_indexing.diff(from, to)
+          deltas = diff.deltas
+
+          deltas.reverse.each_slice(BLOBS_BATCH) do |slice|
+            bulk_operations = slice.map do |delta|
+              if delta.status == :deleted
+                next if delta.old_file[:mode].to_s(8) == "160000"
+                b = LiteBlob.new(repository_for_indexing, delta.old_file)
+                delete_blob(b)
+              else
+                next if delta.new_file[:mode].to_s(8) == "160000"
+                b = LiteBlob.new(repository_for_indexing, delta.new_file)
+                index_blob(b, to)
+              end
+            end
+
+            perform_bulk bulk_operations
+
+            yield slice, deltas.length if block_given?
+          end
+
+          ObjectSpace.garbage_collect
+        end
+
+        def perform_bulk(bulk_operations)
+          bulk_operations.compact!
+
+          return false if bulk_operations.empty?
+
+          client_for_indexing.bulk body: bulk_operations
+        end
+
+        def delete_blob(blob)
+          return unless blob.text?
+          {
+            delete: {
+              _index: "#{self.class.index_name}",
+              _type: self.class.name.underscore,
+              _id: "#{repository_id}_#{blob.path}",
+              _parent: project_id
+            }
+          }
+        end
+
+        def index_blob(blob, target_sha)
+          return unless can_index_blob?(blob)
+          {
+            index:  {
+              _index: "#{self.class.index_name}",
+              _type: self.class.name.underscore,
+              _id: "#{repository_id}_#{blob.path}",
+              _parent: project_id,
+              data: {
+                blob: {
+                  type: "blob",
+                  oid: blob.id,
+                  rid: repository_id,
+                  content: blob.data,
+                  commit_sha: target_sha,
+                  path: blob.path,
+
+                  # We're duplicating file_name parameter here because
+                  # we need another analyzer for it.
+                  # Ideally this should be done with copy_to: 'blob.file_name' option
+                  # but it does not work in ES v2.3.*. We're doing it so to not make users
+                  # install newest versions
+                  # https://github.com/elastic/elasticsearch-mapper-attachments/issues/124
+                  file_name: blob.path,
+
+                  language: blob.language ? blob.language.name : "Text"
+                }
+              }
+            }
+          }
+        end
+
+        # Index text-like files which size less 1.mb
+        def can_index_blob?(blob)
+          blob.text? && (blob.size && blob.size.to_i < 1048576)
+        end
+
+        # Indexing all commits in repository
+        #
+        # All data stored in global index
+        # Repository can be filtered by 'rid' field
+        # If you want - this field can be used git store 'project' id
+        #
+        # commit {
+        #  sha - commit sha
+        #  author {
+        #    name - commit author name
+        #    email - commit author email
+        #    time - commit time
+        #  }
+        #  commiter {
+        #    name - committer name
+        #    email - committer email
+        #    time - commit time
+        #  }
+        #  message - commit message
+        # }
+        #
+        # For search from commits use type 'commit'
+        def index_commits(from_rev: nil, to_rev: repository_for_indexing.last_commit.oid)
+          from, to = parse_revs(from_rev, to_rev)
+          range = [from, to].compact.join('..')
+          out, err, status = Open3.capture3("git log #{range} --format=\"%H\"", chdir: repository_for_indexing.path)
+
+          if status.success? && err.blank?
+            # TODO: use rugged walker!!!
+            commit_oids = out.split("\n")
+
+            commit_oids.each_slice(COMMMITS_BATCH) do |batch|
+              bulk_operations = batch.map do |commit|
+                index_commit(repository_for_indexing.lookup(commit))
+              end
+
+              perform_bulk bulk_operations
+
+              yield batch, commit_oids.length if block_given?
+            end
+
+            ObjectSpace.garbage_collect
+          end
+        end
+
+        def index_commit(commit)
+          author    = commit.author
+          committer = commit.committer
+
+          {
+            index:  {
+              _index: "#{self.class.index_name}",
+              _type: self.class.name.underscore,
+              _id: "#{repository_id}_#{commit.oid}",
+              _parent: project_id,
+              data: {
+                commit: {
+                  type: "commit",
+                  rid: repository_id,
+                  sha: commit.oid,
+                  author: {
+                    name: encode!(author[:name]),
+                    email: encode!(author[:email]),
+                    time: author[:time].strftime('%Y%m%dT%H%M%S%z'),
+                  },
+                  committer: {
+                    name: encode!(committer[:name]),
+                    email: encode!(committer[:email]),
+                    time: committer[:time].strftime('%Y%m%dT%H%M%S%z'),
+                  },
+                  message: encode!(commit.message)
+                }
+              }
+            }
+          }
+        end
+
+        def parse_revs(from_rev, to_rev)
+          from = if index_new_branch?(from_rev)
+                   if to_rev == repository_for_indexing.last_commit.oid
+                     nil
+                   else
+                     repository_for_indexing.merge_base(
+                       to_rev,
+                       repository_for_indexing.last_commit.oid
+                     )
+                   end
+                 else
+                   from_rev
+                 end
+
+          return from, to_rev
+        end
+
+        def index_new_branch?(from)
+          from == '0000000000000000000000000000000000000000'
+        end
+
+        # Representation of repository as indexed json
+        # Attention: It can be very very very huge hash
+        def as_indexed_json(options = {})
+          data = {}
+          data[:blobs] = index_blobs_array
+          data[:commits] = index_commits_array
+          data
+        end
+
+        # Indexing blob from current index
+        def index_blobs_array
+          result = []
+
+          target_sha = repository_for_indexing.head.target.oid
+
+          if repository_for_indexing.bare?
+            tree = repository_for_indexing.lookup(target_sha).tree
+            result.push(recurse_blobs_index_hash(tree))
+          else
+            repository_for_indexing.index.each do |blob|
+              b = LiteBlob.new(repository_for_indexing, blob)
+              result.push(
+                {
+                  type: 'blob',
+                  id: "#{target_sha}_#{b.path}",
+                  rid: repository_id,
+                  oid: b.id,
+                  content: b.data,
+                  commit_sha: target_sha
+                }
+              ) if b.text?
+            end
+          end
+
+          result
+        end
+
+        def recurse_blobs_index_hash(tree, path = "")
+          result = []
+
+          tree.each_blob do |blob|
+            blob[:path] = path + blob[:name]
+            b = LiteBlob.new(repository_for_indexing, blob)
+            result.push(
+              {
+                type: 'blob',
+                id: "#{repository_for_indexing.head.target.oid}_#{path}#{blob[:name]}",
+                rid: repository_id,
+                oid: b.id,
+                content: b.data,
+                commit_sha: repository_for_indexing.head.target.oid
+              }
+            ) if b.text?
+          end
+
+          tree.each_tree do |nested_tree|
+            result.push(recurse_blobs_index_hash(repository_for_indexing.lookup(nested_tree[:oid]), "#{nested_tree[:name]}/"))
+          end
+
+          result.flatten
+        end
+
+        # Lookup all object ids for commit objects
+        def index_commits_array
+          res = []
+
+          repository_for_indexing.each_id do |oid|
+            obj = repository_for_indexing.lookup(oid)
+            if obj.type == :commit
+              res.push(
+                {
+                  type: 'commit',
+                  sha: obj.oid,
+                  author: obj.author,
+                  committer: obj.committer,
+                  message: encode!(obj.message)
+                }
+              )
+            end
+          end
+
+          res
+        end
+
+        def search(query, type: :all, page: 1, per: 20, options: {})
+          options[:repository_id] = repository_id if options[:repository_id].nil?
+          self.class.search(query, type: type, page: page, per: per, options: options)
+        end
+
+        # Repository id used for identity data from different repositories
+        # Update this value if needed
+        def set_repository_id(id = nil)
+          @repository_id = id || path_to_repo
+        end
+
+        # For Overwrite
+        def repository_id
+          @repository_id
+        end
+
+        unless defined?(path_to_repo)
+          def path_to_repo
+            if @path_to_repo.blank?
+              raise NotImplementedError, 'Please, define "path_to_repo" method, or set "path_to_repo" via "repository_for_indexing" method'
+            else
+              @path_to_repo
+            end
+          end
+        end
+
+        def repository_for_indexing(repo_path = nil)
+          return @rugged_repo_indexer if defined? @rugged_repo_indexer
+
+          @path_to_repo ||= repo_path || path_to_repo
+
+          set_repository_id
+
+          @rugged_repo_indexer = Rugged::Repository.new(@path_to_repo)
+        end
+
+        def client_for_indexing
+          @client_for_indexing ||= Elasticsearch::Client.new retry_on_failure: 5
+        end
+      end
+
+      module ClassMethods
+        def search(query, type: :all, page: 1, per: 20, options: {})
+          results = { blobs: [], commits: [] }
+
+          case type.to_sym
+          when :all
+            results[:blobs] = search_blob(query, page: page, per: per, options: options)
+            results[:commits] = search_commit(query, page: page, per: per, options: options)
+          when :blob
+            results[:blobs] = search_blob(query, page: page, per: per, options: options)
+          when :commit
+            results[:commits] = search_commit(query, page: page, per: per, options: options)
+          end
+
+          results
+        end
+
+        def search_commit(query, page: 1, per: 20, options: {})
+          page ||= 1
+
+          fields = %w(message^10 sha^5 author.name^2 author.email^2 committer.name committer.email).map {|i| "commit.#{i}"}
+
+          query_hash = {
+            query: {
+              bool: {
+                must: [{
+                  simple_query_string: {
+                    fields: fields,
+                    query: query,
+                    default_operator: :or
+                  }
+                }]
+              }
+            },
+            size: per,
+            from: per * (page - 1)
+          }
+
+          if query.blank?
+            query_hash[:query][:bool][:must] = { match_all: {} }
+            query_hash[:track_scores] = true
+          end
+
+          if options[:repository_id]
+            query_hash[:query][:bool][:filter] = {
+              terms: {
+                'commit.rid' => [options[:repository_id]].flatten
+              }
+            }
+          end
+
+          if options[:additional_filter]
+            query_hash[:query][:bool][:filter] ||= []
+            query_hash[:query][:bool][:filter] << options[:additional_filter]
+          end
+
+          if options[:highlight]
+            es_fields = fields.map { |field| field.split('^').first }.inject({}) do |memo, field|
+              memo[field.to_sym] = {}
+              memo
+            end
+
+            query_hash[:highlight] = {
+              pre_tags: ["gitlabelasticsearch→"],
+              post_tags: ["←gitlabelasticsearch"],
+              fields: es_fields
+            }
+          end
+
+          options[:order] = :default if options[:order].blank?
+
+          query_hash[:sort] = [:_score]
+
+          res = self.__elasticsearch__.search(query_hash)
+          {
+            results: res.results,
+            total_count: res.size
+          }
+        end
+
+        def search_blob(query, type: :all, page: 1, per: 20, options: {})
+          page ||= 1
+
+          query_hash = {
+            query: {
+              bool: {
+                must: {
+                  simple_query_string: {
+                    query: query,
+                    default_operator: :and,
+                    fields: %w[blob.content blob.file_name]
+                  }
+                }
+              }
+            },
+            size: per,
+            from: per * (page - 1)
+          }
+
+          query_hash[:query][:bool][:filter] = []
+
+          if options[:repository_id]
+            query_hash[:query][:bool][:filter] << {
+              terms: {
+                'blob.rid' => [options[:repository_id]].flatten
+              }
+            }
+          end
+
+          if options[:additional_filter]
+            query_hash[:query][:bool][:filter] ||= []
+            query_hash[:query][:bool][:filter] << options[:additional_filter]
+          end
+
+          if options[:language]
+            query_hash[:query][:bool][:filter] << {
+              terms: {
+                'blob.language' => [options[:language]].flatten
+              }
+            }
+          end
+
+          options[:order] = :default if options[:order].blank?
+
+          query_hash[:sort] = [:_score]
+
+          if options[:highlight]
+            query_hash[:highlight] = {
+              pre_tags: ["gitlabelasticsearch→"],
+              post_tags: ["←gitlabelasticsearch"],
+              order: "score",
+              fields: {
+                "blob.content" => {},
+                "blob.file_name" => {},
+              }
+            }
+          end
+
+          res = self.__elasticsearch__.search(query_hash)
+
+          {
+            results: res.results,
+            total_count: res.size
+          }
+        end
+
+        def search_file_names(query, page: 1, per: 20, options: {})
+          query_hash = {
+            fields: ['blob.path'],
+            query: {
+                fuzzy: {
+                    'repository.blob.path' => { value: query }
+                },
+            },
+            filter: {
+                term: {
+                    'repository.blob.rid' => [options[:repository_id]].flatten
+                }
+            },
+            size: per,
+            from: per * (page - 1)
+          }
+
+          self.__elasticsearch__.search(query_hash)
+        end
+      end
+    end
+  end
+end

lib/gitlab/elastic/search_results.rb

@@ -10,7 +10,7 @@ module Gitlab
       def initialize(current_user, query, limit_project_ids, public_and_internal_projects = true)
         @current_user = current_user
         @limit_project_ids = limit_project_ids
-        @query = Shellwords.shellescape(query) if query.present?
+        @query = query
         @public_and_internal_projects = public_and_internal_projects
       end
 

lib/tasks/gitlab/check.rake

@@ -1078,11 +1078,11 @@ namespace :gitlab do
   def check_elasticsearch
     client = Gitlab::Elastic::Client.build(current_application_settings.elasticsearch_config)
 
-    print "Elasticsearch version 5.1.x? ... "
+    print "Elasticsearch version 5.1 - 5.3? ... "
 
     version = Gitlab::VersionInfo.parse(client.info["version"]["number"])
 
-    if version.major == 5 && version.minor == 1
+    if version.major == 5 && (1..3).cover?(version.minor)
       puts "yes (#{version})".color(:green)
     else
       puts "no, you have #{version}".color(:red)

spec/features/merge_requests/create_new_mr_spec.rb

@@ -36,7 +36,7 @@ feature 'Create New Merge Request', feature: true, js: true do
 
     first('.js-target-branch').click
     first('.dropdown-target-branch .dropdown-content')
-    first('.dropdown-target-branch .dropdown-content a', text: 'v1.1.0').click
+    find('.dropdown-target-branch .dropdown-content a', text: 'v1.1.0', match: :first).click
 
     expect(page).to have_content "b83d6e3"
   end

spec/features/protected_tags/access_control_ce_spec.rb

@@ -10,7 +10,8 @@ RSpec.shared_examples "protected tags > access control > CE" do
 
         unless allowed_to_create_button.text == access_type_name
           allowed_to_create_button.click
-          within(".dropdown.open .dropdown-menu") { click_on access_type_name }
+          find('.create_access_levels-container .dropdown-menu li', match: :first)
+          within('.create_access_levels-container .dropdown-menu') { click_on access_type_name }
         end
       end
 

spec/features/protected_tags_spec.rb

@@ -11,6 +11,7 @@ feature 'Projected Tags', feature: true, js: true do
     find(".js-protected-tag-select").click
     find(".dropdown-input-field").set(tag_name)
     click_on("Create wildcard #{tag_name}")
+    find('.protected-tags-dropdown .dropdown-menu', visible: false)
   end
 
   describe "explicit protected tags" do

spec/models/concerns/elastic/issue_spec.rb

@@ -15,19 +15,19 @@ describe Issue, elastic: true do
 
   it "searches issues" do
     Sidekiq::Testing.inline! do
-      create :issue, title: 'bla-bla term', project: project
-      create :issue, description: 'bla-bla term', project: project
+      create :issue, title: 'bla-bla term1', project: project
+      create :issue, description: 'bla-bla term2', project: project
       create :issue, project: project
 
       # The issue I have no access to
-      create :issue, title: 'bla-bla term'
+      create :issue, title: 'bla-bla term3'
 
       Gitlab::Elastic::Helper.refresh_index
     end
 
     options = { project_ids: [project.id] }
 
-    expect(described_class.elastic_search('term', options: options).total_count).to eq(2)
+    expect(described_class.elastic_search('(term1 | term2 | term3) +bla-bla', options: options).total_count).to eq(2)
   end
 
   it "returns json with all needed elements" do

spec/models/concerns/elastic/merge_request_spec.rb

@@ -15,19 +15,19 @@ describe MergeRequest, elastic: true do
     project = create :project
 
     Sidekiq::Testing.inline! do
-      create :merge_request, title: 'bla-bla term', source_project: project
-      create :merge_request, description: 'term in description', source_project: project, target_branch: "feature2"
+      create :merge_request, title: 'bla-bla term1', source_project: project
+      create :merge_request, description: 'term2 in description', source_project: project, target_branch: "feature2"
       create :merge_request, source_project: project, target_branch: "feature3"
 
       # The merge request you have no access to
-      create :merge_request, title: 'also with term'
+      create :merge_request, title: 'also with term3'
 
       Gitlab::Elastic::Helper.refresh_index
     end
 
     options = { project_ids: [project.id] }
 
-    expect(described_class.elastic_search('term', options: options).total_count).to eq(2)
+    expect(described_class.elastic_search('term1 | term2 | term3', options: options).total_count).to eq(2)
   end
 
   it "returns json with all needed elements" do

spec/models/concerns/elastic/milestone_spec.rb

@@ -15,19 +15,19 @@ describe Milestone, elastic: true do
     project = create :empty_project
 
     Sidekiq::Testing.inline! do
-      create :milestone, title: 'bla-bla term', project: project
-      create :milestone, description: 'bla-bla term', project: project
+      create :milestone, title: 'bla-bla term1', project: project
+      create :milestone, description: 'bla-bla term2', project: project
       create :milestone, project: project
 
       # The milestone you have no access to
-      create :milestone, title: 'bla-bla term'
+      create :milestone, title: 'bla-bla term3'
 
       Gitlab::Elastic::Helper.refresh_index
     end
 
     options = { project_ids: [project.id] }
 
-    expect(described_class.elastic_search('term', options: options).total_count).to eq(2)
+    expect(described_class.elastic_search('(term1 | term2 | term3) +bla-bla', options: options).total_count).to eq(2)
   end
 
   it "returns json with all needed elements" do

spec/models/concerns/elastic/note_spec.rb

@@ -15,18 +15,18 @@ describe Note, elastic: true do
     issue = create :issue
 
     Sidekiq::Testing.inline! do
-      create :note, note: 'bla-bla term', project: issue.project
+      create :note, note: 'bla-bla term1', project: issue.project
       create :note, project: issue.project
 
       # The note in the project you have no access to
-      create :note, note: 'bla-bla term'
+      create :note, note: 'bla-bla term2'
 
       Gitlab::Elastic::Helper.refresh_index
     end
 
     options = { project_ids: [issue.project.id] }
 
-    expect(described_class.elastic_search('term', options: options).total_count).to eq(1)
+    expect(described_class.elastic_search('term1 | term2', options: options).total_count).to eq(1)
   end
 
   it "indexes && searches diff notes" do

spec/models/concerns/elastic/project_spec.rb

@@ -15,8 +15,8 @@ describe Project, elastic: true do
     project_ids = []
 
     Sidekiq::Testing.inline! do
-      project = create :empty_project, name: 'test'
-      project1 = create :empty_project, path: 'test1'
+      project = create :empty_project, name: 'test1'
+      project1 = create :empty_project, path: 'test2'
       project2 = create :empty_project
       create :empty_project, path: 'someone_elses_project'
       project_ids += [project.id, project1.id, project2.id]
@@ -24,8 +24,9 @@ describe Project, elastic: true do
       Gitlab::Elastic::Helper.refresh_index
     end
 
-    expect(described_class.elastic_search('test', options: { project_ids: project_ids }).total_count).to eq(1)
     expect(described_class.elastic_search('test1', options: { project_ids: project_ids }).total_count).to eq(1)
+    expect(described_class.elastic_search('test2', options: { project_ids: project_ids }).total_count).to eq(1)
+    expect(described_class.elastic_search('test*', options: { project_ids: project_ids }).total_count).to eq(2)
     expect(described_class.elastic_search('someone_elses_project', options: { project_ids: project_ids }).total_count).to eq(0)
   end
 

spec/models/concerns/elastic/project_wiki_spec.rb

@@ -15,12 +15,14 @@ describe ProjectWiki, elastic: true do
     project = create :empty_project
 
     Sidekiq::Testing.inline! do
-      project.wiki.create_page("index_page", "Bla bla")
+      project.wiki.create_page("index_page", "Bla bla term1")
+      project.wiki.create_page("omega_page", "Bla bla term2")
       project.wiki.index_blobs
 
       Gitlab::Elastic::Helper.refresh_index
     end
 
-    expect(project.wiki.search('bla', type: :blob)[:blobs][:total_count]).to eq(1)
+    expect(project.wiki.search('term1', type: :blob)[:blobs][:total_count]).to eq(1)
+    expect(project.wiki.search('term1 | term2', type: :blob)[:blobs][:total_count]).to eq(2)
   end
 end

spec/models/concerns/elastic/repository_spec.rb

@@ -22,6 +22,7 @@ describe Repository, elastic: true do
     end
 
     expect(project.repository.search('def popen')[:blobs][:total_count]).to eq(1)
+    expect(project.repository.search('def | popen')[:blobs][:total_count] > 1).to be_truthy
     expect(project.repository.search('initial')[:commits][:total_count]).to eq(1)
   end
 

spec/models/concerns/elastic/snippet_spec.rb

@@ -19,9 +19,9 @@ describe Snippet, elastic: true do
     let!(:internal_snippet) { create(:snippet, :internal, content: 'password: XXX') }
     let!(:private_snippet)  { create(:snippet, :private, content: 'password: XXX', author: author) }
 
-    let!(:project_public_snippet)   { create(:snippet, :public, project: project, content: 'password: XXX') }
-    let!(:project_internal_snippet) { create(:snippet, :internal, project: project, content: 'password: XXX') }
-    let!(:project_private_snippet)  { create(:snippet, :private, project: project, content: 'password: XXX') }
+    let!(:project_public_snippet)   { create(:snippet, :public, project: project, content: 'password: 123') }
+    let!(:project_internal_snippet) { create(:snippet, :internal, project: project, content: 'password: 456') }
+    let!(:project_private_snippet)  { create(:snippet, :private, project: project, content: 'password: 789') }
 
     before do
       Gitlab::Elastic::Helper.refresh_index
@@ -60,6 +60,16 @@ describe Snippet, elastic: true do
       expect(result.records).to match_array [public_snippet, internal_snippet, private_snippet]
     end
 
+    it 'supports advanced search syntax' do
+      member = create(:user)
+      project.add_reporter(member)
+
+      result = described_class.elastic_search_code('password +(123 | 789)', options: { user: member })
+
+      expect(result.total_count).to eq(2)
+      expect(result.records).to match_array [project_public_snippet, project_private_snippet]
+    end
+
     [:admin, :auditor].each do |user_type|
       it "returns all snippets for #{user_type}" do
         superuser = create(user_type)