Merge branch 'master' into 8-12-stable-ee

Conflicts:
	app/assets/javascripts/protected_branch_dropdown.js.es6

CHANGELOG-EE

 Please view this file on the master branch, on stable branches it's out of date.
 v 8.12.0 (Unreleased)
+  - Reduce UPDATE queries when moving between import states on projects
+  - [ES] Instrument Elasticsearch::Git::Repository
+  - Request only the LDAP attributes we need
+  - [ES] Instrument other Gitlab::Elastic classes
+  - [ES] Fix: Elasticsearch does not find partial matches in project names
+  - [ES] Global code search
+  - [ES] Improve logging
 
 v 8.11.6
   - Exclude blocked users from potential MR approvers
+  - Add 'Sync now' to group members page !704
+
+v 8.11.6
+  - Fix mirrored projects allowing empty import urls
 
 v 8.11.5
   - API: Restore backward-compatibility for POST /projects/:id/members when membership is locked

GITLAB_WORKHORSE_VERSION

-0.8.0
+0.8.1

Gemfile

@@ -108,7 +108,7 @@ gem 'seed-fu', '~> 2.3.5'
 # Search
 gem 'elasticsearch-model'
 gem 'elasticsearch-rails'
-gem 'gitlab-elasticsearch-git', '~> 0.0.17', require: "elasticsearch/git"
+gem 'gitlab-elasticsearch-git', '~> 1.0.1', require: "elasticsearch/git"
 
 # Markdown and HTML processing
 gem 'html-pipeline', '~> 1.11.0'

Gemfile.lock

@@ -283,7 +283,7 @@ GEM
       mime-types (>= 1.19)
       rugged (>= 0.23.0b)
     github-markup (1.4.0)
-    gitlab-elasticsearch-git (0.0.17)
+    gitlab-elasticsearch-git (1.0.1)
       activemodel (~> 4.2)
       activesupport (~> 4.2)
       charlock_holmes (~> 0.7)
@@ -886,7 +886,7 @@ DEPENDENCIES
   gemojione (~> 3.0)
   github-linguist (~> 4.7.0)
   github-markup (~> 1.4)
-  gitlab-elasticsearch-git (~> 0.0.17)
+  gitlab-elasticsearch-git (~> 1.0.1)
   gitlab-flowdock-git-hook (~> 1.0.1)
   gitlab-license (~> 1.0)
   gitlab_git (~> 10.6.3)

app/assets/javascripts/protected_branch_dropdown.js.es6

@@ -43,15 +43,16 @@
       });
     }
 
-    bindEvents() {
-      this.$protectedBranch.on('click', this.onClickCreateWildcard.bind(this));
-    }
-
     onClickCreateWildcard() {
+      // Refresh the dropdown's data, which ends up calling `getProtectedBranches`
       this.$dropdown.data('glDropdown').remote.execute();
       this.$dropdown.data('glDropdown').selectRowAtIndex(0);
     } 
 
+    bindEvents() {
+      this.$protectedBranch.on('click', this.onClickCreateWildcard.bind(this));
+    }
+
     getProtectedBranches(term, callback) {
       if (this.selectedBranch) {
         callback(gon.open_branches.concat(this.selectedBranch));

app/controllers/concerns/authenticates_with_two_factor.rb

@@ -62,6 +62,7 @@ module AuthenticatesWithTwoFactor
       session.delete(:otp_user_id)
       session.delete(:challenges)
 
+      remember_me(user) if user_params[:remember_me] == '1'
       sign_in(user)
     else
       flash.now[:alert] = 'Authentication via U2F device failed.'

app/controllers/groups/ldaps_controller.rb

@@ -2,9 +2,10 @@ class Groups::LdapsController < Groups::ApplicationController
   before_action :group
   before_action :authorize_admin_group!
 
-  def reset_access
-    LdapGroupResetService.new.execute(group, current_user)
+  def sync
+    @group.pending_ldap_sync
+    LdapGroupSyncWorker.perform_async(@group.id)
 
-    redirect_to group_group_members_path(@group), notice: 'Access reset complete'
+    redirect_to group_group_members_path(@group), notice: 'The group sync has been scheduled'
   end
 end

app/controllers/search_controller.rb

@@ -38,9 +38,10 @@ class SearchController < ApplicationController
 
         Search::SnippetService.new(current_user, params).execute
       else
-        unless %w(projects issues merge_requests milestones).include?(@scope)
+        unless %w(projects issues merge_requests milestones blobs commits).include?(@scope)
           @scope = 'projects'
         end
+
         Search::GlobalService.new(current_user, params).execute
       end
 

app/helpers/search_helper.rb

@@ -30,6 +30,92 @@ module SearchHelper
     "Showing #{from} - #{to} of #{count} #{scope.humanize(capitalize: false)} for \"#{term}\""
   end
 
+  def parse_search_result(result)
+    if result.is_a?(String)
+      parse_search_result_from_grep(result)
+    else
+      parse_search_result_from_elastic(result)
+    end
+  end
+
+  def parse_search_result_from_elastic(result)
+    ref = result["_source"]["blob"]["commit_sha"]
+    filename = result["_source"]["blob"]["path"]
+    extname = File.extname(filename)
+    basename = filename.sub(/#{extname}$/, '')
+    content = result["_source"]["blob"]["content"]
+    total_lines = content.lines.size
+
+    highlighted_content = result["highlight"]["blob.content"]
+    term = highlighted_content && highlighted_content[0].match(/gitlabelasticsearch→(.*?)←gitlabelasticsearch/)[1]
+
+    found_line_number = 0
+
+    content.each_line.each_with_index do |line, index|
+      if term && line.include?(term)
+        found_line_number = index
+        break
+      end
+    end
+
+    from = if found_line_number >= 2
+             found_line_number - 2
+           else
+             found_line_number
+           end
+
+    to = if (total_lines - found_line_number) > 3
+           found_line_number + 2
+         else
+           found_line_number
+         end
+
+    data = content.lines[from..to]
+
+    OpenStruct.new(
+      filename: filename,
+      basename: basename,
+      ref: ref,
+      startline: from + 1,
+      data: data.join
+    )
+  end
+
+  def parse_search_result_from_grep(result)
+    ref = nil
+    filename = nil
+    basename = nil
+    startline = 0
+
+    result.each_line.each_with_index do |line, index|
+      if line =~ /^.*:.*:\d+:/
+        ref, filename, startline = line.split(':')
+        startline = startline.to_i - index
+        extname = Regexp.escape(File.extname(filename))
+        basename = filename.sub(/#{extname}$/, '')
+        break
+      end
+    end
+
+    data = ""
+
+    result.each_line do |line|
+      data << line.sub(ref, '').sub(filename, '').sub(/^:-\d+-/, '').sub(/^::\d+:/, '')
+    end
+
+    OpenStruct.new(
+      filename: filename,
+      basename: basename,
+      ref: ref,
+      startline: startline,
+      data: data
+    )
+  end
+
+  def find_project_for_blob(blob)
+    Project.find(blob['_parent'])
+  end
+
   private
 
   # Autocomplete results for various settings pages

app/models/application_setting.rb

@@ -154,7 +154,7 @@ class ApplicationSetting < ActiveRecord::Base
       default_project_visibility: Settings.gitlab.default_projects_features['visibility_level'],
       default_snippet_visibility: Settings.gitlab.default_projects_features['visibility_level'],
       domain_whitelist: Settings.gitlab['domain_whitelist'],
-      import_sources: %w[github bitbucket gitlab google_code fogbugz git gitlab_project],
+      import_sources: Gitlab::ImportSources.values,
       shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'],
       max_artifacts_size: Settings.artifacts['max_size'],
       require_two_factor_authentication: false,

app/models/concerns/elastic/application_search.rb

@@ -13,14 +13,26 @@ module Elastic
           analysis: {
             analyzer: {
               default: {
-                tokenizer: "standard",
-                filter: ["standard", "lowercase", "my_stemmer"]
+                tokenizer: 'standard',
+                filter: ['standard', 'lowercase', 'my_stemmer']
+              },
+              my_ngram_analyzer: {
+                tokenizer: 'my_ngram_tokenizer',
+                filter: ['lowercase']
               }
             },
             filter: {
               my_stemmer: {
-                type: "stemmer",
-                name: "light_english"
+                type: 'stemmer',
+                name: 'light_english'
+              }
+            },
+            tokenizer: {
+              my_ngram_tokenizer: {
+                type: 'nGram',
+                min_gram: 2,
+                max_gram: 3,
+                token_chars: [ 'letter', 'digit' ]
               }
             }
           }

app/models/concerns/elastic/projects_search.rb

@@ -12,7 +12,8 @@ module Elastic
         indexes :path,                type: :string,
                                       index_options: 'offsets'
         indexes :name_with_namespace, type: :string,
-                                      index_options: 'offsets'
+                                      index_options: 'offsets',
+                                      analyzer: :my_ngram_analyzer
         indexes :path_with_namespace, type: :string,
                                       index_options: 'offsets'
         indexes :description,         type: :string,

app/models/concerns/elastic/repositories_search.rb

@@ -11,8 +11,8 @@ module Elastic
         project.id
       end
 
-      def self.repositories_count
-        Project.cached_count
+      def project_id
+        project.id
       end
 
       def client_for_indexing
@@ -27,6 +27,40 @@ module Elastic
           end
         end
       end
+
+      def find_commits_by_message_with_elastic(query, page: 1, per_page: 20)
+        response = project.repository.search(query, type: :commit, page: page, per: per_page)[:commits][:results]
+
+        commits = response.map do |result|
+          commit result["_source"]["commit"]["sha"]
+        end
+
+        # Before "map" we had a paginated array so we need to recover it
+        offset = per_page * ((page || 1) - 1)
+        Kaminari.paginate_array(commits, total_count: response.total_count, limit: per_page, offset: offset)
+      end
+    end
+
+    class_methods do
+      def find_commits_by_message_with_elastic(query, page: 1, per_page: 20, options: {})
+        response = Repository.search(
+          query,
+          type: :commit,
+          page: page,
+          per: per_page,
+          options: options
+        )[:commits][:results]
+
+        commits = response.map do |result|
+          sha = result["_source"]["commit"]["sha"]
+          project = Project.find(result["_source"]["commit"]["rid"])
+          project.commit(sha)
+        end
+
+        # Before "map" we had a paginated array so we need to recover it
+        offset = per_page * ((page || 1) - 1)
+        Kaminari.paginate_array(commits, total_count: response.total_count, limit: per_page, offset: offset)
+      end
     end
   end
 end

app/models/concerns/elastic/wiki_repositories_search.rb

@@ -11,8 +11,8 @@ module Elastic
         "wiki_#{project.id}"
       end
 
-      def self.repositories_count
-        Project.with_wiki_enabled.count
+      def project_id
+        project.id
       end
 
       def client_for_indexing

app/models/ee/group.rb

@@ -10,10 +10,15 @@ module EE
       state_machine :ldap_sync_status, namespace: :ldap_sync, initial: :ready do
         state :ready
         state :started
+        state :pending
         state :failed
 
+        event :pending do
+          transition [:ready, :failed] => :pending
+        end
+
         event :start do
-          transition [:ready, :failed] => :started
+          transition [:ready, :pending, :failed] => :started
         end
 
         event :finish do

app/models/project.rb

@@ -169,7 +169,6 @@ class Project < ActiveRecord::Base
   validates_uniqueness_of :name, scope: :namespace_id
   validates_uniqueness_of :path, scope: :namespace_id
   validates :import_url, addressable_url: true, if: :external_import?
-  validates :mirror_user, presence: true, if: :mirror?
   validates :star_count, numericality: { greater_than_or_equal_to: 0 }
   validate :check_limit, on: :create
   validate :avatar_type,
@@ -183,6 +182,11 @@ class Project < ActiveRecord::Base
     presence: true,
     inclusion: { in: ->(_object) { Gitlab.config.repositories.storages.keys } }
 
+  with_options if: :mirror? do |project|
+    project.validates :import_url, presence: true
+    project.validates :mirror_user, presence: true
+  end
+
   add_authentication_token_field :runners_token
   before_save :ensure_runners_token
   before_validation :mark_remote_mirrors_for_removal
@@ -237,12 +241,11 @@ class Project < ActiveRecord::Base
 
     after_transition any => :finished, do: :reset_cache_and_import_attrs
 
-    after_transition started: :finished do |project, transaction|
+    before_transition started: :finished do |project, transaction|
       if project.mirror?
         timestamp = DateTime.now
         project.mirror_last_update_at = timestamp
         project.mirror_last_successful_update_at = timestamp
-        project.save
       end
 
       if current_application_settings.elasticsearch_indexing?
@@ -250,10 +253,8 @@ class Project < ActiveRecord::Base
       end
     end
 
-    after_transition started: :failed do |project, transaction|
-      if project.mirror?
-        project.update(mirror_last_update_at: DateTime.now)
-      end
+    before_transition started: :failed do |project, transaction|
+      project.mirror_last_update_at = DateTime.now if project.mirror?
     end
   end
 

app/models/repository.rb

@@ -131,12 +131,6 @@ class Repository
     commits
   end
 
-  def find_commits_by_message_with_elastic(query)
-    project.repository.search(query, type: :commit)[:commits][:results].map do |result|
-      commit result["_source"]["commit"]["sha"]
-    end
-  end
-
   def find_branch(name, fresh_repo: true)
     # Since the Repository object may have in-memory index changes, invalidating the memoized Repository object may
     # cause unintended side effects. Because finding a branch is a read-only operation, we can safely instantiate
@@ -1146,88 +1140,6 @@ class Repository
     Gitlab::Popen.popen(args, path_to_repo).first.scrub.split(/^--$/)
   end
 
-  def parse_search_result(result)
-    if result.is_a?(String)
-      parse_search_result_from_grep(result)
-    else
-      parse_search_result_from_elastic(result)
-    end
-  end
-
-  def parse_search_result_from_elastic(result)
-    ref = result["_source"]["blob"]["commit_sha"]
-    filename = result["_source"]["blob"]["path"]
-    extname = File.extname(filename)
-    basename = filename.sub(/#{extname}$/, '')
-    content = result["_source"]["blob"]["content"]
-    total_lines = content.lines.size
-
-    highlighted_content = result["highlight"]["blob.content"]
-    term = highlighted_content && highlighted_content[0].match(/gitlabelasticsearch→(.*?)←gitlabelasticsearch/)[1]
-
-    found_line_number = 0
-
-    content.each_line.each_with_index do |line, index|
-      if term && line.include?(term)
-        found_line_number = index
-        break
-      end
-    end
-
-    from = if found_line_number >= 2
-             found_line_number - 2
-           else
-             found_line_number
-           end
-
-    to = if (total_lines - found_line_number) > 3
-           found_line_number + 2
-         else
-           found_line_number
-         end
-
-    data = content.lines[from..to]
-
-    OpenStruct.new(
-      filename: filename,
-      basename: basename,
-      ref: ref,
-      startline: from + 1,
-      data: data.join
-    )
-  end
-
-  def parse_search_result_from_grep(result)
-    ref = nil
-    filename = nil
-    basename = nil
-    startline = 0
-
-    result.each_line.each_with_index do |line, index|
-      if line =~ /^.*:.*:\d+:/
-        ref, filename, startline = line.split(':')
-        startline = startline.to_i - index
-        extname = Regexp.escape(File.extname(filename))
-        basename = filename.sub(/#{extname}$/, '')
-        break
-      end
-    end
-
-    data = ""
-
-    result.each_line do |line|
-      data << line.sub(ref, '').sub(filename, '').sub(/^:-\d+-/, '').sub(/^::\d+:/, '')
-    end
-
-    OpenStruct.new(
-      filename: filename,
-      basename: basename,
-      ref: ref,
-      startline: startline,
-      data: data
-    )
-  end
-
   def fetch_ref(source_path, source_ref, target_ref)
     args = %W(#{Gitlab.config.git.bin_path} fetch --no-tags -f #{source_path} #{source_ref}:#{target_ref})
     Gitlab::Popen.popen(args, path_to_repo)

app/models/service.rb

@@ -12,6 +12,7 @@ class Service < ActiveRecord::Base
   default_value_for :tag_push_events, true
   default_value_for :note_events, true
   default_value_for :build_events, true
+  default_value_for :pipeline_events, true
   default_value_for :wiki_page_events, true
 
   after_initialize :initialize_properties

app/views/devise/sessions/two_factor.html.haml

@@ -18,6 +18,5 @@
             = f.submit "Verify code", class: "btn btn-save"
 
       - if @user.two_factor_u2f_enabled?
-
         %hr
-        = render "u2f/authenticate"
+        = render "u2f/authenticate", locals: { params: params, resource: resource, resource_name: resource_name }

app/views/groups/group_members/_ldap_sync.html.haml

+- if current_user && @group.ldap_synced?
+  .bs-callout.bs-callout-info
+    The members of this group are managed using LDAP and cannot be added, changed or removed here.
+    Because LDAP permissions in GitLab get updated one user at a time and because GitLab caches LDAP check results, changes on your LDAP server or in this group's LDAP sync settings may take up to #{Gitlab.config.ldap['sync_time']}s to show in the list below.
+    %ul
+      - @group.ldap_group_links.each do |ldap_group_link|
+        %li
+          People in cn
+          %code= ldap_group_link.cn
+          are given
+          %code= ldap_group_link.human_access
+          access.
+    - if can?(current_user, :admin_group, @group)
+      = render 'sync_button'

app/views/groups/group_members/_sync_button.html.haml

+- if @group.ldap_sync_started?
+  %span.btn.disabled
+    = icon("refresh spin")
+    Syncing…
+- elsif @group.ldap_sync_pending?
+  %span.btn.disabled
+    = icon("refresh spin")
+    Pending sync…
+- else
+  = link_to sync_group_ldap_path(@group), method: :put, class: 'btn' do
+    = icon("refresh")
+    Sync now
+- if @group.ldap_sync_ready? && @group.ldap_sync_last_successful_update_at
+  %p.inline.prepend-left-10
+    Successfully synced #{time_ago_with_tooltip(@group.ldap_sync_last_successful_update_at)}.

app/views/groups/group_members/index.html.haml

@@ -13,23 +13,7 @@
 
     = render 'shared/members/requests', membership_source: @group, requesters: @requesters
 
-  - if current_user && @group.ldap_synced?
-    .bs-callout.bs-callout-info
-      The members of this group are managed using LDAP and cannot be added, changed or removed here.
-      Because LDAP permissions in GitLab get updated one user at a time and because GitLab caches LDAP check results, changes on your LDAP server or in this group's LDAP sync settings may take up to #{Gitlab.config.ldap['sync_time']}s to show in the list below.
-      %ul
-        - @group.ldap_group_links.each do |ldap_group_link|
-          %li
-            People in cn
-            %code= ldap_group_link.cn
-            are given
-            %code= ldap_group_link.human_access
-            access.
-      - if can?(current_user, :admin_group_member, @group)
-        = form_tag(reset_access_group_ldap_path(@group), method: :put, class: 'inline') do
-          = button_to 'Clear LDAP permission cache', '#', class: "btn btn-remove js-confirm-danger",
-            data: { "confirm-danger-message" => clear_ldap_permission_cache_message,
-                    'warning-message' => 'If you made manual permission tweaks for some group members they will be lost.' }
+  = render 'ldap_sync'
 
   .panel.panel-default
     .panel-heading
@@ -51,5 +35,3 @@
     event.preventDefault();
     Turbolinks.visit(this.action + '?' + $(this).serialize());
   });
-
-= render 'shared/confirm_modal', phrase: 'reset'

app/views/projects/branches/_branch.html.haml

@@ -3,6 +3,7 @@
 - diverging_commit_counts = @repository.diverging_commit_counts(branch)
 - number_commits_behind = diverging_commit_counts[:behind]
 - number_commits_ahead = diverging_commit_counts[:ahead]
+- merge_project = can?(current_user, :create_merge_request, @project) ? @project : (current_user && current_user.fork_of(@project))
 %li(class="js-branch-#{branch.name}")
   %div
     = link_to namespace_project_tree_path(@project.namespace, @project, branch.name), class: 'item-title str-truncated' do
@@ -25,12 +26,12 @@
         diverged from upstream
 
     .controls.hidden-xs
-      - if create_mr_button?(@repository.root_ref, branch.name)
+      - if merge_project && create_mr_button?(@repository.root_ref, branch.name)
         = link_to create_mr_path(@repository.root_ref, branch.name), class: 'btn btn-default' do
           Merge Request
 
       - if branch.name != @repository.root_ref
-        = link_to namespace_project_compare_index_path(@project.namespace, @project, from: @repository.root_ref, to: branch.name), class: 'btn btn-default', method: :post, title: "Compare" do
+        = link_to namespace_project_compare_index_path(@project.namespace, @project, from: @repository.root_ref, to: branch.name), class: "btn btn-default #{'prepend-left-10' unless merge_project}", method: :post, title: "Compare" do
           Compare
 
       = render 'projects/buttons/download', project: @project, ref: branch.name

app/views/projects/commits/_commit.html.haml

@@ -14,6 +14,8 @@
     .commit-info-block
       .commit-row-title
         %span.item-title
+          - unless @project
+            #{project.name_with_namespace}:
           = link_to_gfm commit.title, namespace_project_commit_path(project.namespace, project, commit.id), class: "commit-row-message"
           %span.commit-row-message.visible-xs-inline
             ·

app/views/projects/new.html.haml

@@ -139,5 +139,8 @@
 
   $('.import_git').click(function( event ) {
     $projectImportUrl = $('#project_import_url')
+    $projectMirror = $('#project_mirror')
+
     $projectImportUrl.attr('disabled', !$projectImportUrl.attr('disabled'))
+    $projectMirror.attr('disabled', !$projectMirror.attr('disabled'))
   });

app/views/search/_category.html.haml

@@ -69,3 +69,14 @@
         Milestones
         %span.badge
           = @search_results.milestones_count
+    - if current_application_settings.elasticsearch_search?
+      %li{class: ("active" if @scope == 'blobs')}
+        = link_to search_filter_path(scope: 'blobs') do
+          Code
+          %span.badge
+            = @search_results.blobs_count
+      %li{class: ("active" if @scope == 'commits')}
+        = link_to search_filter_path(scope: 'commits') do
+          Commits
+          %span.badge
+            = @search_results.commits_count

app/views/search/results/_blob.html.haml

-- blob = @project.repository.parse_search_result(blob)
+- parsed_blob = parse_search_result(blob)
+- project = @project || find_project_for_blob(blob)
+- blob_link = namespace_project_blob_path(project.namespace, project, tree_join(parsed_blob.ref, parsed_blob.filename))
 .blob-result
   .file-holder
     .file-title
-      - blob_link = namespace_project_blob_path(@project.namespace, @project, tree_join(blob.ref, blob.filename))
       = link_to blob_link do
-        %i.fa.fa-file
+        = icon('fa-file')
         %strong
-          = blob.filename
+          - if @project
+            = parsed_blob.filename
+          - else
+            #{project.name_with_namespace}:
+            %i= parsed_blob.filename
+
     .file-content.code.term
-      = render 'shared/file_highlight', blob: blob, first_line_number: blob.startline, blob_link: blob_link
+      = render 'shared/file_highlight', blob: parsed_blob, first_line_number: parsed_blob.startline, blob_link: blob_link

app/views/search/results/_commit.html.haml

 .search-result-row
-  = render 'projects/commits/commit', project: @project, commit: commit
+  = render 'projects/commits/commit', project: commit.project, commit: commit

app/views/search/results/_wiki_blob.html.haml

-- wiki_blob = @project.repository.parse_search_result(wiki_blob)
+- wiki_blob = parse_search_result(wiki_blob)
 .blob-result
   .file-holder
     .file-title

app/views/shared/_import_form.html.haml

@@ -19,7 +19,7 @@
   .col-sm-offset-2.col-sm-10
     .checkbox
       = f.label :mirror do
-        = f.check_box :mirror
+        = f.check_box :mirror, disabled: true
         %strong
           Mirror repository
       .help-block

app/views/u2f/_authenticate.html.haml

@@ -20,6 +20,8 @@
   %div
     %p We heard back from your U2F device. Click this button to authenticate with the GitLab server.
     = form_tag(new_user_session_path, method: :post) do |f|
+      - resource_params = params[resource_name].presence || params
+      = hidden_field_tag 'user[remember_me]', resource_params.fetch(:remember_me, 0)
       = hidden_field_tag 'user[device_response]', nil, class: 'form-control', required: true, id: "js-device-response"
       = submit_tag "Authenticate via U2F Device", class: "btn btn-success"
 

app/workers/ldap_group_sync_worker.rb

@@ -3,9 +3,21 @@ class LdapGroupSyncWorker
 
   sidekiq_options retry: false
 
-  def perform
+  def perform(group_id = nil)
+    if group_id
+      group = Group.find_by(id: group_id)
+      unless group
+        logger.warn "Could not find group #{group_id} for LDAP group sync"
+        return
+      end
+
+      logger.info "Started LDAP group sync for group #{group.name} (#{group.id})"
+      EE::Gitlab::LDAP::Sync::Group.execute_all_providers(group)
+      logger.info "Finished LDAP group sync for group #{group.name} (#{group.id})"
+    else
       logger.info 'Started LDAP group sync'
       EE::Gitlab::LDAP::Sync::Groups.execute
       logger.info 'Finished LDAP group sync'
     end
+  end
 end

bin/elastic_repo_indexer

@@ -17,7 +17,7 @@ FROM_SHA = ENV['FROM_SHA']
 TO_SHA = ENV['TO_SHA']
 RAILS_ENV = ENV['RAILS_ENV']
 
-LOGGER.info("[ES indexer]: Has been scheduled for project #{REPO_PATH} with SHA range #{FROM_SHA}:#{TO_SHA}")
+LOGGER.info("Has been scheduled for project #{REPO_PATH} with SHA range #{FROM_SHA}:#{TO_SHA}")
 
 elastic_connection_info = JSON.parse ENV['ELASTIC_CONNECTION_INFO']
 ELASTIC_HOST = elastic_connection_info['host']
@@ -41,6 +41,10 @@ class Repository
     PROJECT_ID
   end
 
+  def project_id
+    PROJECT_ID
+  end
+
   def path_to_repo
     REPO_PATH
   end
@@ -50,22 +54,26 @@ repo = Repository.new
 
 params = { from_rev: FROM_SHA, to_rev: TO_SHA }.compact
 
-print "Indexing commits..."
+LOGGER.info("Indexing commits started")
 
 timings = Benchmark.measure do
-  repo.index_commits(params)
+  indexed = 0
+  repo.index_commits(params) do |batch, total_count|
+    indexed += batch.length
+    LOGGER.info("Indexed #{indexed}/#{total_count} commits")
+  end
 end
 
-puts "Done"
+LOGGER.info("Commits for #{REPO_PATH} are indexed. Time elapsed: #{timings.real}")
 
-LOGGER.info("[ES indexer]: Commits for #{REPO_PATH} are indexed. Time elapsed: #{timings.real}")
-
-print "Indexing blobs..."
+LOGGER.info("Indexing blobs started")
 
 timings = Benchmark.measure do
-  repo.index_blobs(params)
+  indexed = 0
+  repo.index_blobs(params) do |batch, total_count|
+    indexed += batch.length
+    LOGGER.info("Indexed #{indexed}/#{total_count} blobs")
+  end
 end
 
-puts "Done"
-
-LOGGER.info("[ES indexer]: Blobs for #{REPO_PATH} are indexed. Time elapsed: #{timings.real}")
+LOGGER.info("Blobs for #{REPO_PATH} are indexed. Time elapsed: #{timings.real}")

config/initializers/metrics.rb

@@ -149,10 +149,18 @@ if Gitlab::Metrics.enabled?
     config.instrument_methods(Gitlab::Highlight)
     config.instrument_instance_methods(Gitlab::Highlight)
 
+    config.instrument_methods(Elasticsearch::Git::Repository)
+    config.instrument_instance_methods(Elasticsearch::Git::Repository)
+
     config.instrument_instance_methods(Search::GlobalService)
     config.instrument_instance_methods(Search::ProjectService)
+
     config.instrument_instance_methods(Gitlab::Elastic::SearchResults)
     config.instrument_instance_methods(Gitlab::Elastic::ProjectSearchResults)
+    config.instrument_instance_methods(Gitlab::Elastic::Indexer)
+    config.instrument_instance_methods(Gitlab::Elastic::SnippetSearchResults)
+    config.instrument_methods(Gitlab::Elastic::Helper)
+
     config.instrument_instance_methods(Elastic::ApplicationSearch)
     config.instrument_instance_methods(Elastic::IssuesSearch)
     config.instrument_instance_methods(Elastic::MergeRequestsSearch)

config/routes.rb

@@ -463,7 +463,7 @@ Rails.application.routes.draw do
       resource :analytics, only: [:show]
       resource :ldap, only: [] do
         member do
-          put :reset_access
+          put :sync
         end
       end
 

db/migrate/20160831231325_update_mirror_when_empty_import_url_in_projects.rb

+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class UpdateMirrorWhenEmptyImportUrlInProjects < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    update_column_in_batches(:projects, :mirror, false) do |table, query|
+      query.where(table[:import_url].eq(nil).or(table[:import_url].eq('')))
+    end
+  end
+end

doc/README.md

@@ -26,6 +26,7 @@
 
 ## Administrator documentation
 
+- [Upload your GitLab License](user/admin_area/license.md) Upload the license you purchased for GitLab Enterprise Edition to unlock its features.
 - [Audit Events](administration/audit_events.md) Check how user access changed in projects and groups.
 - [Access restrictions](user/admin_area/settings/visibility_and_access_controls.md#enabled-git-access-protocols) Define which Git access protocols can be used to talk to GitLab
 - [Authentication/Authorization](administration/auth/README.md) Configure
@@ -36,7 +37,6 @@
 - [Push Rules](push_rules/push_rules.md) Advanced push rules for your project.
 - [Help message](customization/help_message.md) Set information about administrators of your GitLab instance.
 - [Install](install/README.md) Requirements, directory structures and installation from source.
-- [Installing your license](license/README.md)
 - [Integration](integration/README.md) How to integrate with systems such as JIRA, Redmine, LDAP and Twitter.
 - [Restart GitLab](administration/restart_gitlab.md) Learn how to restart GitLab and its components.
 - [Issue closing](customization/issue_closing.md) Customize how to close an issue from commit messages.

doc/install/installation.md

@@ -403,7 +403,7 @@ If you are not using Linux you may have to run `gmake` instead of
     cd /home/git
     sudo -u git -H git clone https://gitlab.com/gitlab-org/gitlab-workhorse.git
     cd gitlab-workhorse
-    sudo -u git -H git checkout v0.8.0
+    sudo -u git -H git checkout v0.8.1
     sudo -u git -H make
 
 ### Initialize Database and Activate Advanced Features

doc/integration/elasticsearch.md

@@ -31,7 +31,7 @@ is installed or on a separate server.
 These are the minimum requirements needed for Elasticsearch to work:
 
 - GitLab 8.4+
-- Elasticsearch 2.0+ (with [Delete By Query Plugin](https://www.elastic.co/guide/en/elasticsearch/plugins/2.0/plugins-delete-by-query.html) installed)
+- Elasticsearch 2.4+ (with [Delete By Query Plugin](https://www.elastic.co/guide/en/elasticsearch/plugins/2.4/plugins-delete-by-query.html) installed)
 
 ## Install Elasticsearch
 

doc/license/README.md

-# Installing your license
-
-To activate all GitLab Enterprise Edition functionality, you need to upload a
-license. Once you've received your license from GitLab Inc., you can upload it
-by signing into your GitLab instance as an admin, and navigating to
-**Admin Area > License**.
-
-If you've received a `.gitlab-license` file, you can upload it directly.
-
-![upload.png](img/upload.png)
-
-If you've received your license as plain text, you need to select the
-"Enter license key" option, copy the license and paste it into the "License key"
-field.
-
-![enter.png](img/enter.png)
-
-Once you've uploaded your license, all GitLab Enterprise Edition functionality
-will be active until the end of the license period.
-
-You can review the license details at any time in the License section of the
-Admin Area.
-
-![details.png](img/details.png)
-
-## Notification before the license expires
-
-One month before the license expires, a message informing when the expiration
-is due to will be shown to GitLab admins. Make sure that you update your license
-beforehand otherwise you will miss important features if it expires.
-
-![License expiration](img/expire_message.png)
-
-## What happens when my license expires?
-
-In case your license expires, you will not be able to push any commits to
-GitLab and creation of new issues and merge requests will be disabled.
-
-A message to inform of the locked state of GitLab will be presented to
-admins only.
-
-![No license message](img/no_license_message.png)
+This document was moved to [user/admin_area/license](../user/admin_area/license.md).

doc/update/8.11-to-8.12.md

@@ -82,7 +82,7 @@ GitLab 8.1.
 ```bash
 cd /home/git/gitlab-workhorse
 sudo -u git -H git fetch --all
-sudo -u git -H git checkout v0.8.0
+sudo -u git -H git checkout v0.8.1
 sudo -u git -H make
 ```
 
@@ -182,6 +182,35 @@ To make sure you didn't miss anything run a more thorough check:
 
 If all items are green, then congratulations, the upgrade is complete!
 
+### 11. Elasticsearch index update (if you currently use Elasticsearch)
+
+In 8.12 release we changed the index mapping and this is why the whole index should be removed and built from scratch. Also the Elasticsearch 2.3.* contains a bug that causes to fail all queries that use highlight feature and Parent Child relationship at once, so we recommend to use the version 2.4 and newer. After updating your Elasticsearch server, please re-create your index by using one of two ways listed below:
+
+1. Re-create the index. The following command is acceptable for not very big GitLab instances (storage size no more than few gigabytes).
+
+    ```
+    # Omnibus installations
+    sudo gitlab-rake gitlab:elastic:index
+
+    # Installations from source
+    bundle exec rake gitlab:elastic:index
+    ```
+
+1. For very big GitLab instances you have to remove index first. Note: Consider disabling ES search feature (**Admin > Settings**) before removing the index, in order to allow your users to use regular search while you recreate the Elasticsearch index.
+
+    ```
+    # Omnibus installations
+    sudo gitlab-rake gitlab:elastic:delete_indexes
+    sudo gitlab-rake gitlab:elastic:clear_index_status
+
+    # Installations from source
+    bundle exec rake gitlab:elastic:delete_indexes
+    bundle exec rake gitlab:elastic:clear_index_status
+    ```
+
+    Then we recommend to follow [Add GitLab's data to the Elasticsearch index](../integration/elasticsearch.md#add-gitlabs-data-to-the-elasticsearch-index).
+
+
 ## Things went south? Revert to previous version (8.11)
 
 ### 1. Revert the code to the previous version

doc/update/8.12-ce-to-ee.md

+# From Community Edition 8.12 to Enterprise Edition 8.12
+
+This guide assumes you have a correctly configured and tested installation of
+GitLab Community Edition 8.12. If you run into any trouble or if you have any
+questions please contact us at [support@gitlab.com].
+
+### 0. Backup
+
+Make a backup just in case something goes wrong:
+
+```bash
+cd /home/git/gitlab
+sudo -u git -H bundle exec rake gitlab:backup:create RAILS_ENV=production
+```
+
+For installations using MySQL, this may require granting "LOCK TABLES"
+privileges to the GitLab user on the database version.
+
+### 1. Stop server
+
+    sudo service gitlab stop
+
+### 2. Get the EE code
+
+```bash
+cd /home/git/gitlab
+sudo -u git -H git remote add -f ee https://gitlab.com/gitlab-org/gitlab-ee.git
+sudo -u git -H git checkout 8-12-stable-ee
+```
+
+### 3. Install libs, migrations, etc.
+
+```bash
+cd /home/git/gitlab
+
+# MySQL installations (note: the line below states '--without postgres')
+sudo -u git -H bundle install --without postgres development test --deployment
+
+# PostgreSQL installations (note: the line below states '--without mysql')
+sudo -u git -H bundle install --without mysql development test --deployment
+
+# Run database migrations
+sudo -u git -H bundle exec rake db:migrate RAILS_ENV=production
+
+# Clean up assets and cache
+sudo -u git -H bundle exec rake assets:clean assets:precompile cache:clear RAILS_ENV=production
+```
+
+### 4. Start application
+
+    sudo service gitlab start
+    sudo service nginx restart
+
+### 5. Check application status
+
+Check if GitLab and its environment are configured correctly:
+
+    sudo -u git -H bundle exec rake gitlab:env:info RAILS_ENV=production
+
+To make sure you didn't miss anything run a more thorough check with:
+
+    sudo -u git -H bundle exec rake gitlab:check RAILS_ENV=production
+
+If all items are green, then congratulations upgrade complete!
+
+## Things went south? Revert to previous version (Community Edition 8.12)
+
+### 1. Revert the code to the previous version
+
+```bash
+cd /home/git/gitlab
+sudo -u git -H git checkout 8-12-stable
+```
+
+### 2. Restore from the backup
+
+```bash
+cd /home/git/gitlab
+sudo -u git -H bundle exec rake gitlab:backup:restore RAILS_ENV=production
+```
+
+[support@gitlab.com]: mailto:support@gitlab.com

doc/user/admin_area/license.md

+# Activate all GitLab Enterprise Edition functionality with a license
+
+To activate all GitLab Enterprise Edition (EE) functionality, you need to upload
+a license. The license has the form of a base64 encoded ASCII text with a
+`.gitlab-license` extension and can be obtained when you [purchase one][pricing]
+or when you sign up for a [free trial]. Once you've received your license from
+GitLab Inc., you can upload it by signing into your GitLab instance as an admin.
+
+## Uploading your license
+
+The very first time you visit your GitLab EE installation, you should see a
+notice urging you to upload a license with a link that takes you straight to the
+License admin area. Otherwise, you can  navigate manually to the **Admin Area**
+by clicking the wrench icon in the upper right corner and the going to the
+**License** tab.
+
+![License admin area](img/license_admin_area.png)
+
+---
+
+If you've received a `.gitlab-license` file, you should have already downloaded
+it in your local machine. You can then upload it directly by choosing the
+license file and clicking the **Upload license** button. In the image below,
+you can see that the selected license file is named `GitLab.gitlab-license`.
+
+![Upload license](img/license_upload.png)
+
+---
+
+If you've received your license as plain text, you need to select the
+"Enter license key" option, copy the license, paste it into the "License key"
+field and click **Upload license**.
+
+![Enter license](img/license_enter.png)
+
+Once you've uploaded your license, all GitLab Enterprise Edition functionality
+will be active until the end of the license period.
+You can review the license details at any time in the License section of the
+Admin Area.
+
+![License details](img/license_details.png)
+
+## License history
+
+It's possible to upload more than one license, but only the last one will be
+taken into account.
+
+You can see your previous licenses' history at the bottom of the License page.
+
+![License history](img/license_history.png)
+
+## Notification before the license expires
+
+One month before the license expires, a message informing when the expiration
+is due to will be shown to GitLab admins. Make sure that you update your license
+beforehand otherwise you will miss important features if it expires.
+
+![License expiration](img/license_expire_message.png)
+
+## What happens when your license expires
+
+In case your license expires, you will not be able to push any commits to
+GitLab and creation of new issues and merge requests will be disabled.
+
+A message to inform of the locked state of GitLab will be presented to all
+users and admins will be able to see a link to upload a license.
+
+![No license message](img/license_no_license_message.png)
+
+[free trial]: https://about.gitlab.com/free-trial/
+[pricing]: https://about.gitlab.com/pricing/

doc/user/project/labels.md

@@ -86,7 +86,7 @@ appear on top.
 ## Subscribe to labels
 
 If you don’t want to miss issues or merge requests that are important to you,
-simply subscribe to a label. You’ll get notified whenever the label gets added
+simply subscribe to a label. You’ll get notified via email whenever the label gets added
 to an issue or merge request, making sure you don’t miss a thing.
 
 Go to your project's **Issues > Labels** area, find the label(s) you want to

lib/ee/gitlab/ldap/adapter.rb

@@ -16,7 +16,8 @@ module EE
         def groups(cn = "*", size = nil)
           options = {
             base: config.group_base,
-            filter: Net::LDAP::Filter.eq("cn", cn)
+            filter: Net::LDAP::Filter.eq("cn", cn),
+            attributes: %w(dn cn memberuid member submember uniquemember memberof)
           }
 
           options.merge!(size: size) if size
@@ -30,13 +31,13 @@ module EE
           groups(*args).first
         end
 
-        def dn_matches_filter?(dn, filter)
+        def dns_for_filter(filter)
           ldap_search(
-            base: dn,
+            base: config.base,
             filter: filter,
-            scope: Net::LDAP::SearchScope_BaseObject,
+            scope: Net::LDAP::SearchScope_WholeSubtree,
             attributes: %w{dn}
-          ).any?
+          ).map(&:dn)
         end
       end
     end

lib/ee/gitlab/ldap/person.rb

+module EE
+  module Gitlab
+    module LDAP
+      module Person
+        def ssh_keys
+          if config.sync_ssh_keys? && entry.respond_to?(config.sync_ssh_keys)
+            entry[config.sync_ssh_keys.to_sym].
+              map { |key| key[/(ssh|ecdsa)-[^ ]+ [^\s]+/] }.
+              compact
+          else
+            []
+          end
+        end
+
+        def kerberos_principal
+          # The following is only meaningful for Active Directory
+          return unless entry.respond_to?(:sAMAccountName)
+          entry[:sAMAccountName].first + '@' + windows_domain_name.upcase
+        end
+
+        def windows_domain_name
+          # The following is only meaningful for Active Directory
+          require 'net/ldap/dn'
+          dn_components = []
+          Net::LDAP::DN.new(dn).each_pair { |name, value| dn_components << { name: name, value: value } }
+          dn_components.
+            reverse.
+            take_while { |rdn| rdn[:name].casecmp('DC').zero? }. # Domain Component
+          map { |rdn| rdn[:value] }.
+            reverse.
+            join('.')
+        end
+      end
+    end
+  end
+end

lib/gitlab/elastic/project_search_results.rb

@@ -28,17 +28,12 @@ module Gitlab
         when 'wiki_blobs'
           wiki_blobs.page(page).per(per_page)
         when 'commits'
-          Kaminari.paginate_array(commits).page(page).per(per_page)
+          commits(page: page, per_page: per_page)
         else
           super
         end
       end
 
-      def total_count
-        @total_count ||= issues_count + merge_requests_count + blobs_count +
-                         notes_count + wiki_blobs_count + commits_count
-      end
-
       def blobs_count
         @blobs_count ||= blobs.total_count
       end
@@ -52,7 +47,7 @@ module Gitlab
       end
 
       def commits_count
-        @commits_count ||= commits.count
+        @commits_count ||= commits.total_count
       end
 
       private
@@ -98,17 +93,19 @@ module Gitlab
         Note.elastic_search(query, options: opt)
       end
 
-      def commits
+      def commits(page: 1, per_page: 20)
         if project.empty_repo? || query.blank?
           Kaminari.paginate_array([])
         else
           # We use elastic for default branch only
           if root_ref?
-            project.repository.find_commits_by_message_with_elastic(query)
+            project.repository.find_commits_by_message_with_elastic(
+              query,
+              page: (page || 1).to_i,
+              per_page: per_page
+            )
           else
-            Kaminari.paginate_array(
             project.repository.find_commits_by_message(query).compact
-            )
           end
         end
       end

lib/gitlab/elastic/search_results.rb

@@ -24,19 +24,27 @@ module Gitlab
           merge_requests.page(page).per(per_page).records
         when 'milestones'
           milestones.page(page).per(per_page).records
+        when 'blobs'
+          blobs.page(page).per(per_page)
+        when 'commits'
+          commits(page: page, per_page: per_page)
         else
           Kaminari.paginate_array([])
         end
       end
 
-      def total_count
-        @total_count ||= projects_count + issues_count + merge_requests_count + milestones_count
-      end
-
       def projects_count
         @projects_count ||= projects.total_count
       end
 
+      def blobs_count
+        @blobs_count ||= blobs.total_count
+      end
+
+      def commits_count
+        @commits_count ||= commits.total_count
+      end
+
       def issues_count
         @issues_count ||= issues.total_count
       end
@@ -49,10 +57,6 @@ module Gitlab
         @milestones_count ||= milestones.total_count
       end
 
-      def empty?
-        total_count.zero?
-      end
-
       private
 
       def projects
@@ -92,6 +96,64 @@ module Gitlab
         MergeRequest.elastic_search(query, options: opt)
       end
 
+      def blobs
+        if query.blank?
+          Kaminari.paginate_array([])
+        else
+          opt = {
+            additional_filter: build_filter_by_project(limit_project_ids, @public_and_internal_projects)
+          }
+
+          Repository.search(
+            query,
+            type: :blob,
+            options: opt.merge({ highlight: true })
+          )[:blobs][:results].response
+        end
+      end
+
+      def commits(page: 1, per_page: 20)
+        if query.blank?
+          Kaminari.paginate_array([])
+        else
+          options = {
+            additional_filter: build_filter_by_project(limit_project_ids, @public_and_internal_projects)
+          }
+
+          Repository.find_commits_by_message_with_elastic(
+            query,
+            page: (page || 1).to_i,
+            per_page: per_page,
+            options: options
+          )
+        end
+      end
+
+      def build_filter_by_project(project_ids, public_and_internal_projects)
+        conditions = [{ terms: { id: project_ids } }]
+
+        if public_and_internal_projects
+          conditions << {
+            term: { visibility_level: Project::PUBLIC }
+          }
+
+          conditions << {
+            term: { visibility_level: Project::INTERNAL }
+          }
+        end
+
+        {
+          has_parent: {
+            parent_type: 'project',
+            query: {
+              bool: {
+                should: conditions
+              }
+            }
+          }
+        }
+      end
+
       def default_scope
         'projects'
       end

lib/gitlab/github_import/importer.rb

@@ -133,8 +133,7 @@ module Gitlab
 
         if issue.labels.count > 0
           label_ids = issue.labels
-            .map { |raw| LabelFormatter.new(project, raw).attributes }
-            .map { |attrs| Label.find_by(attrs).try(:id) }
+            .map { |attrs| project.labels.find_by(title: attrs.name).try(:id) }
             .compact
 
           issuable.update_attribute(:label_ids, label_ids)

lib/gitlab/github_import/label_formatter.rb

@@ -13,6 +13,12 @@ module Gitlab
         Label
       end
 
+      def create!
+        project.labels.find_or_create_by!(title: title) do |label|
+          label.color = color
+        end
+      end
+
       private
 
       def color

lib/gitlab/ldap/adapter.rb

@@ -29,31 +29,7 @@ module Gitlab
       end
 
       def users(field, value, limit = nil)
-        if field.to_sym == :dn
-          options = {
-            base: value,
-            scope: Net::LDAP::SearchScope_BaseObject
-          }
-        else
-          options = {
-            base: config.base,
-            filter: Net::LDAP::Filter.eq(field, value)
-          }
-        end
-
-        if config.user_filter.present?
-          user_filter = Net::LDAP::Filter.construct(config.user_filter)
-
-          options[:filter] = if options[:filter]
-                               Net::LDAP::Filter.join(options[:filter], user_filter)
-                             else
-                               user_filter
-                             end
-        end
-
-        if limit.present?
-          options.merge!(size: limit)
-        end
+        options = user_options(field, value, limit)
 
         entries = ldap_search(options).select do |entry|
           entry.respond_to? config.uid
@@ -68,13 +44,11 @@ module Gitlab
         users(*args).first
       end
 
-      def dns_for_filter(filter)
-        ldap_search(
-          base: config.base,
+      def dn_matches_filter?(dn, filter)
+        ldap_search(base: dn,
                     filter: filter,
-          scope: Net::LDAP::SearchScope_WholeSubtree,
-          attributes: %w{dn}
-        ).map(&:dn)
+                    scope: Net::LDAP::SearchScope_BaseObject,
+                    attributes: %w{dn}).any?
       end
 
       def ldap_search(*args)
@@ -98,6 +72,38 @@ module Gitlab
         Rails.logger.warn("LDAP search timed out after #{config.timeout} seconds")
         []
       end
+
+      private
+
+      def user_options(field, value, limit)
+        options = { attributes: %W(#{config.uid} cn mail dn) }
+        options[:size] = limit if limit
+
+        if field.to_sym == :dn
+          options[:base] = value
+          options[:scope] = Net::LDAP::SearchScope_BaseObject
+          options[:filter] = user_filter
+        else
+          options[:base] = config.base
+          options[:filter] = user_filter(Net::LDAP::Filter.eq(field, value))
+        end
+
+        options
+      end
+
+      def user_filter(filter = nil)
+        if config.user_filter.present?
+          user_filter = Net::LDAP::Filter.construct(config.user_filter)
+        end
+
+        if user_filter && filter
+          Net::LDAP::Filter.join(filter, user_filter)
+        elsif user_filter
+          user_filter
+        else
+          filter
+        end
+      end
     end
   end
 end

lib/gitlab/ldap/person.rb

 module Gitlab
   module LDAP
     class Person
+      include EE::Gitlab::LDAP::Person
+
       # Active Directory-specific LDAP filter that checks if bit 2 of the
       # userAccountControl attribute is set.
       # Source: http://ctogonewild.com/2009/09/03/bitmask-searches-in-ldap/
@@ -47,35 +49,6 @@ module Gitlab
         entry.dn
       end
 
-      def ssh_keys
-        if config.sync_ssh_keys? && entry.respond_to?(config.sync_ssh_keys)
-          entry[config.sync_ssh_keys.to_sym].
-            map { |key| key[/(ssh|ecdsa)-[^ ]+ [^\s]+/] }.
-            compact
-        else
-          []
-        end
-      end
-
-      def kerberos_principal
-        # The following is only meaningful for Active Directory
-        return unless entry.respond_to?(:sAMAccountName)
-        entry[:sAMAccountName].first + '@' + windows_domain_name.upcase
-      end
-
-      def windows_domain_name
-        # The following is only meaningful for Active Directory
-        require 'net/ldap/dn'
-        dn_components = []
-        Net::LDAP::DN.new(dn).each_pair { |name, value| dn_components << { name: name, value: value } }
-        dn_components.
-          reverse.
-          take_while { |rdn| rdn[:name].casecmp('DC').zero? }. # Domain Component
-          map { |rdn| rdn[:value] }.
-          reverse.
-          join('.')
-      end
-
       private
 
       def entry

lib/gitlab/search_results.rb

@@ -69,6 +69,7 @@ module Gitlab
 
     def merge_requests
       merge_requests = MergeRequest.in_projects(project_ids_relation)
+
       if query =~ /[#!](\d+)\z/
         merge_requests = merge_requests.where(iid: $1)
       else

lib/gitlab/url_sanitizer.rb

@@ -9,6 +9,8 @@ module Gitlab
     end
 
     def self.valid?(url)
+      return false unless url
+
       Addressable::URI.parse(url.strip)
 
       true

lib/tasks/gitlab/elastic.rake

@@ -58,7 +58,7 @@ namespace :gitlab do
 
     desc "GitLab | Elasticsearch | Index wiki repositories"
     task index_wikis: :environment  do
-      projects = apply_project_filters(Project.where(wiki_enabled: true))
+      projects = apply_project_filters(Project.with_wiki_enabled)
 
       projects.find_each do |project|
         unless project.wiki.empty?

spec/controllers/sessions_controller_spec.rb

@@ -136,6 +136,29 @@ describe SessionsController do
         post(:create, { user: user_params }, { otp_user_id: user.id })
       end
 
+      context 'remember_me field' do
+        it 'sets a remember_user_token cookie when enabled' do
+          allow(U2fRegistration).to receive(:authenticate).and_return(true)
+          allow(controller).to receive(:find_user).and_return(user)
+          expect(controller).
+            to receive(:remember_me).with(user).and_call_original
+
+          authenticate_2fa_u2f(remember_me: '1', login: user.username, device_response: "{}")
+
+          expect(response.cookies['remember_user_token']).to be_present
+        end
+
+        it 'does nothing when disabled' do
+          allow(U2fRegistration).to receive(:authenticate).and_return(true)
+          allow(controller).to receive(:find_user).and_return(user)
+          expect(controller).not_to receive(:remember_me)
+
+          authenticate_2fa_u2f(remember_me: '0', login: user.username, device_response: "{}")
+
+          expect(response.cookies['remember_user_token']).to be_nil
+        end
+      end
+
       it "creates an audit log record" do
         allow(U2fRegistration).to receive(:authenticate).and_return(true)
         expect { authenticate_2fa_u2f(login: user.username, device_response: "{}") }.to change { SecurityEvent.count }.by(1)

spec/finders/pipelines_finder_spec.rb

@@ -43,9 +43,10 @@ describe PipelinesFinder do
     let(:params) { { scope: 'running' } }
 
     it 'orders in descending order on ID' do
-      create(:ci_pipeline, project: project, ref: 'feature')
+      feature_pipeline = create(:ci_pipeline, project: project, ref: 'feature')
 
-      expect(subject.map(&:id)).to eq [3, 2, 1]
+      expected_ids = [feature_pipeline.id, branch_pipeline.id, tag_pipeline.id].sort.reverse
+      expect(subject.map(&:id)).to eq expected_ids
     end
   end
 end

spec/javascripts/fixtures/u2f/authenticate.html.haml

-= render partial: "u2f/authenticate", locals: { new_user_session_path: "/users/sign_in" }
+= render partial: "u2f/authenticate", locals: { new_user_session_path: "/users/sign_in", params: {}, resource_name: "user" }