Merge branch 'master' into 'bootstrap4'

# Conflicts:
#   app/views/projects/jobs/index.html.haml

.rubocop_todo.yml

@@ -143,7 +143,7 @@ Lint/MissingCopEnableDirective:
 Lint/NestedPercentLiteral:
   Exclude:
     - 'lib/gitlab/git/repository.rb'
-    - 'spec/support/email_format_shared_examples.rb'
+    - 'spec/support/shared_examples/email_format_shared_examples.rb'
 
 # Offense count: 1
 Lint/ReturnInVoidContext:
@@ -195,8 +195,8 @@ Naming/HeredocDelimiterCase:
     - 'spec/lib/gitlab/diff/parser_spec.rb'
     - 'spec/lib/json_web_token/rsa_token_spec.rb'
     - 'spec/models/commit_spec.rb'
-    - 'spec/support/repo_helpers.rb'
-    - 'spec/support/seed_repo.rb'
+    - 'spec/support/helpers/repo_helpers.rb'
+    - 'spec/support/helpers/seed_repo.rb'
 
 # Offense count: 112
 # Configuration parameters: Blacklist.
@@ -496,7 +496,7 @@ Style/EmptyLiteral:
     - 'spec/lib/gitlab/request_context_spec.rb'
     - 'spec/lib/gitlab/workhorse_spec.rb'
     - 'spec/requests/api/jobs_spec.rb'
-    - 'spec/support/chat_slash_commands_shared_examples.rb'
+    - 'spec/support/shared_examples/chat_slash_commands_shared_examples.rb'
 
 # Offense count: 102
 # Cop supports --auto-correct.

CHANGELOG.md

@@ -2,6 +2,34 @@
 documentation](doc/development/changelog.md) for instructions on adding your own
 entry.
 
+## 10.7.1 (2018-04-23)
+
+### Fixed (11 changes)
+
+- [API] Fix URLs in the `Link` header for `GET /projects/:id/repository/contributors` when no value is passed for `order_by` or `sort`. !18393
+- Fix a case with secret variables being empty sometimes. !18400
+- Fix `Trace::HttpIO` can not render multi-byte chars. !18417
+- Fix specifying a non-default ref when requesting an archive using the legacy URL. !18468
+- Respect visibility options and description when importing project from template. !18473
+- Removes 'No Job log' message from build trace. !18523
+- Align action icons in pipeline graph.
+- Fix direct_upload when records with null file_store are used.
+- Removed alert box in IDE when redirecting to new merge request.
+- Fixed IDE not loading for sub groups.
+- Fixed IDE not showing loading state when tree is loading.
+
+### Performance (4 changes)
+
+- Validate project path prior to hitting the database. !18322
+- Add index to file_store on ci_job_artifacts. !18444
+- Fix N+1 queries when loading participants for a commit note.
+- Support Markdown rendering using multiple projects.
+
+### Added (1 change)
+
+- Add an API endpoint to download git repository snapshots. !18173
+
+
 ## 10.7.0 (2018-04-22)
 
 ### Security (6 changes, 2 of them are from the community)

CONTRIBUTING.md

@@ -26,7 +26,9 @@ _This notice should stay as the first item in the CONTRIBUTING.md file._
   - [Type labels (~"feature proposal", ~bug, ~customer, etc.)](#type-labels-feature-proposal-bug-customer-etc)
   - [Subject labels (~wiki, ~"container registry", ~ldap, ~api, etc.)](#subject-labels-wiki-container-registry-ldap-api-etc)
   - [Team labels (~"CI/CD", ~Discussion, ~Edge, ~Platform, etc.)](#team-labels-cicd-discussion-edge-platform-etc)
-  - [Priority labels (~Deliverable, ~Stretch, ~"Next Patch Release")](#priority-labels-deliverable-stretch-next-patch-release)
+  - [Milestone labels (~Deliverable, ~Stretch, ~"Next Patch Release")](#milestone-labels-deliverable-stretch-next-patch-release)
+  - [Priority labels (~Deliverable, ~Stretch, ~"Next Patch Release")](#bug-priority-labels-p1-p2-p3-etc)
+  - [Severity labels (~Deliverable, ~Stretch, ~"Next Patch Release")](#bug-severity-labels-s1-s2-s3-etc)
   - [Label for community contributors (~"Accepting Merge Requests")](#label-for-community-contributors-accepting-merge-requests)
 - [Implement design & UI elements](#implement-design-ui-elements)
 - [Issue tracker](#issue-tracker)
@@ -127,6 +129,8 @@ Most issues will have labels for at least one of the following:
 - Subject: ~wiki, ~"container registry", ~ldap, ~api, ~frontend, etc.
 - Team: ~"CI/CD", ~Discussion, ~Edge, ~Platform, etc.
 - Milestone: ~Deliverable, ~Stretch, ~"Next Patch Release"
+- Priority: ~P1, ~P2, ~P3, ~P4
+- Severity: ~S1, ~S2, ~S3, ~S4
 
 All labels, their meaning and priority are defined on the
 [labels page][labels-page].
@@ -210,7 +214,7 @@ This label documents the planned timeline & urgency which is used to measure aga
 
 | Label | Meaning         | Estimate time to fix                                             | Guidance |
 |-------|-----------------|------------------------------------------------------------------|----------|
-| ~P1   | Urgent Priority | The current release                                              |  |
+| ~P1   | Urgent Priority | The current release + potentially immediate hotfix to GitLab.com |  |
 | ~P2   | High Priority   | The next release                                                 |  |
 | ~P3   | Medium Priority | Within the next 3 releases (approx one quarter)                  |  |
 | ~P4   | Low Priority    | Anything outside the next 3 releases (approx beyond one quarter) | The issue is prominent but does not impact user workflow and a workaround is documented  |

Gemfile.lock

@@ -178,7 +178,7 @@ GEM
     docile (1.1.5)
     domain_name (0.5.20170404)
       unf (>= 0.0.5, < 1.0.0)
-    doorkeeper (4.3.1)
+    doorkeeper (4.3.2)
       railties (>= 4.2)
     doorkeeper-openid_connect (1.3.0)
       doorkeeper (~> 4.3)

VERSION

-10.7.0-pre
+10.8.0-pre

app/models/concerns/avatarable.rb

@@ -31,12 +31,13 @@ module Avatarable
 
     asset_host = ActionController::Base.asset_host
     use_asset_host = asset_host.present?
+    use_authentication = respond_to?(:public?) && !public?
 
     # Avatars for private and internal groups and projects require authentication to be viewed,
     # which means they can only be served by Rails, on the regular GitLab host.
     # If an asset host is configured, we need to return the fully qualified URL
     # instead of only the avatar path, so that Rails doesn't prefix it with the asset host.
-    if use_asset_host && respond_to?(:public?) && !public?
+    if use_asset_host && use_authentication
       use_asset_host = false
       only_path = false
     end
@@ -49,6 +50,6 @@ module Avatarable
       url_base << gitlab_config.relative_url_root
     end
 
-    url_base + avatar.url
+    url_base + avatar.local_url
   end
 end

app/models/concerns/routable.rb

@@ -102,7 +102,7 @@ module Routable
   # the route. Caching this per request ensures that even if we have multiple instances,
   # we will not have to duplicate work, avoiding N+1 queries in some cases.
   def full_path
-    return uncached_full_path unless RequestStore.active?
+    return uncached_full_path unless RequestStore.active? && persisted?
 
     RequestStore[full_path_key] ||= uncached_full_path
   end
@@ -124,6 +124,11 @@ module Routable
     end
   end
 
+  # Group would override this to check from association
+  def owned_by?(user)
+    owner == user
+  end
+
   private
 
   def set_path_errors

app/models/group.rb

@@ -125,6 +125,10 @@ class Group < Namespace
     self[:lfs_enabled]
   end
 
+  def owned_by?(user)
+    owners.include?(user)
+  end
+
   def add_users(users, access_level, current_user: nil, expires_at: nil)
     GroupMember.add_users(
       self,

app/uploaders/gitlab_uploader.rb

@@ -65,6 +65,10 @@ class GitlabUploader < CarrierWave::Uploader::Base
     !!model
   end
 
+  def local_url
+    File.join('/', self.class.base_dir, dynamic_segment, filename)
+  end
+
   private
 
   # Designed to be overridden by child uploaders that have a dynamic path

app/views/admin/application_settings/_repository_storage.html.haml

@@ -21,7 +21,7 @@
           .form-text.text-muted
             Manage repository storage paths. Learn more in the
             = succeed "." do
-              = link_to "repository storages documentation", help_page_path("administration/repository_storages")
+              = link_to "repository storages documentation", help_page_path("administration/repository_storage_paths")
     .sub-section
       %h4 Circuit breaker
       .form-group.row

app/views/projects/jobs/index.html.haml

@@ -15,7 +15,7 @@
         - unless @repository.gitlab_ci_yml
           = link_to 'Get started with Pipelines', help_page_path('ci/quick_start/README'), class: 'btn btn-info'
 
-        = link_to ci_lint_path, class: 'btn btn-secondary' do
+        = link_to project_ci_lint_path(@project), class: 'btn btn-secondary' do
           %span CI lint
 
   .content-list.builds-content-list

app/views/projects/pipelines/_with_tabs.html.haml

@@ -26,7 +26,7 @@
         %ul
           - pipeline.yaml_errors.split(",").each do |error|
             %li= error
-        You can also test your .gitlab-ci.yml in the #{link_to "Lint", ci_lint_path}
+        You can also test your .gitlab-ci.yml in the #{link_to "Lint", project_ci_lint_path(@project)}
 
     - if pipeline.project.builds_enabled? && !pipeline.ci_yaml_file
       .bs-callout.bs-callout-warning

changelogs/unreleased/39345-get-raw-archive.yml

----
-title: Add an API endpoint to download git repository snapshots
-merge_request: 18173
-author:
-type: added

changelogs/unreleased/42936-improve-ns-factory-avoid-duplicates.yml

+---
+title: Fix discussions API setting created_at for notable in a group or notable in
+  a project in a group with owners
+merge_request: 18464
+author:
+type: fixed

changelogs/unreleased/44775-avatar-on-os-fails-with-cdn.yml

+---
+title: Fixed wrong avatar URL when the avatar is on object storage.
+merge_request: 18092
+author:
+type: fixed

changelogs/unreleased/44981-http-io-trace-with-multi-byte-char.yml

----
-title: Fix `Trace::HttpIO` can not render multi-byte chars
-merge_request: 18417
-author:
-type: fixed

changelogs/unreleased/45287-align-icons.yml

----
-title: Align action icons in pipeline graph
-merge_request:
-author:
-type: fixed

changelogs/unreleased/45363-optional-params-on-api-endpoint-produce-invalid-pagination-header-links.yml

----
-title: '[API] Fix URLs in the `Link` header for `GET /projects/:id/repository/contributors`
-  when no value is passed for `order_by` or `sort`'
-merge_request: 18393
-author:
-type: fixed

changelogs/unreleased/45476-geo-statement-timeout-counting-local-job-artifacts.yml

----
-title: Add index to file_store on ci_job_artifacts
-merge_request: 18444
-author:
-type: performance

changelogs/unreleased/45507-fix-repository-archive-url.yml

----
-title: Fix specifying a non-default ref when requesting an archive using the legacy
-  URL
-merge_request: 18468
-author:
-type: fixed

changelogs/unreleased/45625-job-log-shows-no-job-log.yml

----
-title: Removes 'No Job log' message from build trace
-merge_request: 18523
-author:
-type: fixed

changelogs/unreleased/45666-project-ci-lint-links.yml

+---
+title: Update links to /ci/lint with ones to project ci/lint
+merge_request: 18539
+author: Takuya Noguchi
+type: fixed

changelogs/unreleased/ab-45247-project-lookups-validation.yml

----
-title: Validate project path prior to hitting the database.
-merge_request: 18322
-author:
-type: performance

changelogs/unreleased/fix-direct-upload-for-old-records.yml

----
-title: Fix direct_upload when records with null file_store are used
-merge_request:
-author:
-type: fixed

changelogs/unreleased/fix-gb-fix-empty-secret-variables.yml

----
-title: Fix a case with secret variables being empty sometimes
-merge_request: 18400
-author:
-type: fixed

changelogs/unreleased/fj-42601-respect-visibility-options.yml

----
-title: Respect visibility options and description when importing project from template
-merge_request: 18473
-author:
-type: fixed

changelogs/unreleased/ide-mr-changes-alert-box.yml

----
-title: Removed alert box in IDE when redirecting to new merge request
-merge_request:
-author:
-type: fixed

changelogs/unreleased/ide-subgroup-fix.yml

----
-title: Fixed IDE not loading for sub groups
-merge_request:
-author:
-type: fixed

changelogs/unreleased/ide-tree-loading-fix.yml

----
-title: Fixed IDE not showing loading state when tree is loading
-merge_request:
-author:
-type: fixed

changelogs/unreleased/rendering-markdown-multiple-projects.yml

----
-title: Support Markdown rendering using multiple projects
-merge_request:
-author:
-type: performance

changelogs/unreleased/sh-fix-award-emoji-nplus-one-participants.yml

----
-title: Fix N+1 queries when loading participants for a commit note
-merge_request:
-author:
-type: performance

changelogs/unreleased/update-doorkeeper-changelog.yml

+---
+title: Update doorkeeper to 4.3.2 to fix GitLab OAuth authentication
+merge_request: 18543
+author:
+type: fixed

config/initializers/1_settings.rb

-# rubocop:disable GitlabSecurity/PublicSend
-
-require_dependency Rails.root.join('lib/gitlab') # Load Gitlab as soon as possible
-
-class Settings < Settingslogic
-  source ENV.fetch('GITLAB_CONFIG') { "#{Rails.root}/config/gitlab.yml" }
-  namespace Rails.env
-
-  class << self
-    def gitlab_on_standard_port?
-      on_standard_port?(gitlab)
-    end
-
-    def host_without_www(url)
-      host(url).sub('www.', '')
-    end
-
-    def build_gitlab_ci_url
-      custom_port =
-        if on_standard_port?(gitlab)
-          nil
-        else
-          ":#{gitlab.port}"
-        end
-
-      [
-        gitlab.protocol,
-        "://",
-        gitlab.host,
-        custom_port,
-        gitlab.relative_url_root
-      ].join('')
-    end
-
-    def build_pages_url
-      base_url(pages).join('')
-    end
-
-    def build_gitlab_shell_ssh_path_prefix
-      user_host = "#{gitlab_shell.ssh_user}@#{gitlab_shell.ssh_host}"
-
-      if gitlab_shell.ssh_port != 22
-        "ssh://#{user_host}:#{gitlab_shell.ssh_port}/"
-      else
-        if gitlab_shell.ssh_host.include? ':'
-          "[#{user_host}]:"
-        else
-          "#{user_host}:"
-        end
-      end
-    end
-
-    def build_base_gitlab_url
-      base_url(gitlab).join('')
-    end
-
-    def build_gitlab_url
-      (base_url(gitlab) + [gitlab.relative_url_root]).join('')
-    end
-
-    # check that values in `current` (string or integer) is a contant in `modul`.
-    def verify_constant_array(modul, current, default)
-      values = default || []
-      unless current.nil?
-        values = []
-        current.each do |constant|
-          values.push(verify_constant(modul, constant, nil))
-        end
-        values.delete_if { |value| value.nil? }
-      end
-
-      values
-    end
-
-    # check that `current` (string or integer) is a contant in `modul`.
-    def verify_constant(modul, current, default)
-      constant = modul.constants.find { |name| modul.const_get(name) == current }
-      value = constant.nil? ? default : modul.const_get(constant)
-      if current.is_a? String
-        value = modul.const_get(current.upcase) rescue default
-      end
-
-      value
-    end
-
-    def absolute(path)
-      File.expand_path(path, Rails.root)
-    end
-
-    private
-
-    def base_url(config)
-      custom_port = on_standard_port?(config) ? nil : ":#{config.port}"
-
-      [
-        config.protocol,
-        "://",
-        config.host,
-        custom_port
-      ]
-    end
-
-    def on_standard_port?(config)
-      config.port.to_i == (config.https ? 443 : 80)
-    end
-
-    # Extract the host part of the given +url+.
-    def host(url)
-      url = url.downcase
-      url = "http://#{url}" unless url.start_with?('http')
-
-      # Get rid of the path so that we don't even have to encode it
-      url_without_path = url.sub(%r{(https?://[^/]+)/?.*}, '\1')
-
-      URI.parse(url_without_path).host
-    end
-
-    # Runs every minute in a random ten-minute period on Sundays, to balance the
-    # load on the server receiving these pings. The usage ping is safe to run
-    # multiple times because of a 24 hour exclusive lock.
-    def cron_for_usage_ping
-      hour = rand(24)
-      minute = rand(6)
-
-      "#{minute}0-#{minute}9 #{hour} * * 0"
-    end
-  end
-end
+require_dependency File.expand_path('../../lib/gitlab', __dir__) # Load Gitlab as soon as possible
 
 # Default settings
 Settings['ldap'] ||= Settingslogic.new({})

config/initializers/2_app.rb

-module Gitlab
-  def self.config
-    Settings
-  end
-
-  VERSION  = File.read(Rails.root.join("VERSION")).strip.freeze
-  REVISION = Gitlab::Popen.popen(%W(#{config.git.bin_path} log --pretty=format:%h -n 1)).first.chomp.freeze
-end

db/fixtures/development/17_cycle_analytics.rb

 require './spec/support/sidekiq'
-require './spec/support/test_env'
+require './spec/support/helpers/test_env'
 
 class Gitlab::Seeder::CycleAnalytics
   def initialize(project, perf: false)

doc/development/testing_guide/best_practices.md

@@ -63,6 +63,8 @@ writing one](testing_levels.md#consider-not-writing-a-system-test)!
 
 Sometimes you may need to debug Capybara tests by observing browser behavior.
 
+#### Live debug
+
 You can pause Capybara and view the website on the browser by using the
 `live_debug` method in your spec. The current page will be automatically opened
 in your default browser.
@@ -90,6 +92,54 @@ Finished in 34.51 seconds (files took 0.76702 seconds to load)
 
 Note: `live_debug` only works on javascript enabled specs.
 
+#### Run `:js` spec in a visible browser
+
+Run the spec with `CHROME_HEADLESS=0`, e.g.:
+
+```
+CHROME_HEADLESS=0 bundle exec rspec some_spec.rb
+```
+
+The test will go by quickly, but this will give you an idea of what's happening.
+
+You can also add `byebug` or `binding.pry` to pause execution and step through
+the test.
+
+#### Screenshots
+
+We use the `capybara-screenshot` gem to automatically take a screenshot on
+failure. In CI you can download these files as job artifacts.
+
+Also, you can manually take screenshots at any point in a test by adding the
+methods below. Be sure to remove them when they are no longer needed! See
+https://github.com/mattheworiordan/capybara-screenshot#manual-screenshots for
+more.
+
+Add `screenshot_and_save_page` in a `:js` spec to screenshot what Capybara
+"sees", and save the page source.
+
+Add `screenshot_and_open_image` in a `:js` spec to screenshot what Capybara
+"sees", and automatically open the image.
+
+### Fast unit tests
+
+Some classes are well-isolated from Rails and you should be able to test them
+without the overhead added by the Rails environment and Bundler's `:default`
+group's gem loading. In these cases, you can `require 'fast_spec_helper'`
+instead of `require 'spec_helper'` in your test file, and your test should run
+really fast since:
+
+- Gems loading is skipped
+- Rails app boot is skipped
+- gitlab-shell and Gitaly setup are skipped
+- Test repositories setup are skipped
+
+Note that in some cases, you might have to add some `require_dependency 'foo'`
+in your file under test since Rails autoloading is not available in these cases.
+
+This shouldn't be a problem since explicitely listing dependencies should be
+considered a good practice anyway.
+
 ### `let` variables
 
 GitLab's RSpec suite has made extensive use of `let` variables to reduce
@@ -281,14 +331,13 @@ All fixtures should be be placed under `spec/fixtures/`.
 
 RSpec config files are files that change the RSpec config (i.e.
 `RSpec.configure do |config|` blocks). They should be placed under
-`spec/support/config/`.
+`spec/support/`.
 
 Each file should be related to a specific domain, e.g.
-`spec/support/config/capybara.rb`, `spec/support/config/carrierwave.rb`, etc.
+`spec/support/capybara.rb`, `spec/support/carrierwave.rb`, etc.
 
-Helpers can be included in the `spec/support/config/rspec.rb` file. If a
-helpers module applies only to a certain kind of specs, it should add modifiers
-to the `config.include` call. For instance if
+If a helpers module applies only to a certain kind of specs, it should add
+modifiers to the `config.include` call. For instance if
 `spec/support/helpers/cycle_analytics_helpers.rb` applies to `:lib` and
 `type: :model` specs only, you would write the following:
 
@@ -299,6 +348,14 @@ RSpec.configure do |config|
 end
 ```
 
+If a config file only consists of `config.include`, you can add these
+`config.include` directly in `spec/spec_helper.rb`.
+
+For very generic helpers, consider including them in the `spec/support/rspec.rb`
+file which is used by the `spec/fast_spec_helper.rb` file. See
+[Fast unit tests](#fast-unit-tests) for more details about the
+`spec/fast_spec_helper.rb` file.
+
 ---
 
 [Return to Testing documentation](index.md)

doc/development/testing_guide/testing_rake_tasks.md

@@ -9,7 +9,7 @@ At a minimum, requiring the Rake helper will redirect `stdout`, include the
 runtime task helpers, and include the `RakeHelpers` Spec support module.
 
 The `RakeHelpers` module exposes a `run_rake_task(<task>)` method to make
-executing tasks simple. See `spec/support/rake_helpers.rb` for all available
+executing tasks simple. See `spec/support/helpers/rake_helpers.rb` for all available
 methods.
 
 Example:

doc/user/project/deploy_tokens/index.md

@@ -23,7 +23,7 @@ You can create as many deploy tokens as you like from the settings of your proje
 
 ![Personal access tokens page](img/deploy_tokens.png)
 
-## Revoking a personal access token
+## Revoking a deploy token
 
 At any time, you can revoke any deploy token by just clicking the
 respective **Revoke** button under the 'Active deploy tokens' area.

features/support/env.rb

@@ -12,7 +12,11 @@ end
 
 WebMock.enable!
 
-%w(select2_helper test_env repo_helpers wait_for_requests sidekiq project_forks_helper webmock).each do |f|
+%w(select2_helper test_env repo_helpers wait_for_requests project_forks_helper).each do |f|
+  require Rails.root.join('spec', 'support', 'helpers', f)
+end
+
+%w(sidekiq webmock).each do |f|
   require Rails.root.join('spec', 'support', f)
 end
 

lib/api/helpers/notes_helpers.rb

@@ -64,8 +64,10 @@ module API
         authorize! :create_note, noteable
 
         parent = noteable_parent(noteable)
+
         if opts[:created_at]
-          opts.delete(:created_at) unless current_user.admin? || parent.owner == current_user
+          opts.delete(:created_at) unless
+            current_user.admin? || parent.owned_by?(current_user)
         end
 
         project = parent if parent.is_a?(Project)

lib/gitlab.rb

-require_dependency 'gitlab/git'
+require_dependency 'settings'
+require_dependency 'gitlab/popen'
 
 module Gitlab
+  def self.root
+    Pathname.new(File.expand_path('..', __dir__))
+  end
+
+  def self.config
+    Settings
+  end
+
   COM_URL = 'https://gitlab.com'.freeze
   APP_DIRS_PATTERN = %r{^/?(app|config|ee|lib|spec|\(\w*\))}
   SUBDOMAIN_REGEX = %r{\Ahttps://[a-z0-9]+\.gitlab\.com\z}
+  VERSION = File.read(root.join("VERSION")).strip.freeze
+  REVISION = Gitlab::Popen.popen(%W(#{config.git.bin_path} log --pretty=format:%h -n 1)).first.chomp.freeze
 
   def self.com?
     # Check `gl_subdomain?` as well to keep parity with gitlab.com

lib/gitlab/git.rb

+require_dependency 'gitlab/encoding_helper'
+
 module Gitlab
   module Git
     # The ID of empty tree.

lib/settings.rb

+require 'settingslogic'
+
+class Settings < Settingslogic
+  source ENV.fetch('GITLAB_CONFIG') { Pathname.new(File.expand_path('..', __dir__)).join('config/gitlab.yml') }
+  namespace ENV.fetch('GITLAB_ENV') { Rails.env }
+
+  class << self
+    def gitlab_on_standard_port?
+      on_standard_port?(gitlab)
+    end
+
+    def host_without_www(url)
+      host(url).sub('www.', '')
+    end
+
+    def build_gitlab_ci_url
+      custom_port =
+        if on_standard_port?(gitlab)
+          nil
+        else
+          ":#{gitlab.port}"
+        end
+
+      [
+        gitlab.protocol,
+        "://",
+        gitlab.host,
+        custom_port,
+        gitlab.relative_url_root
+      ].join('')
+    end
+
+    def build_pages_url
+      base_url(pages).join('')
+    end
+
+    def build_gitlab_shell_ssh_path_prefix
+      user_host = "#{gitlab_shell.ssh_user}@#{gitlab_shell.ssh_host}"
+
+      if gitlab_shell.ssh_port != 22
+        "ssh://#{user_host}:#{gitlab_shell.ssh_port}/"
+      else
+        if gitlab_shell.ssh_host.include? ':'
+          "[#{user_host}]:"
+        else
+          "#{user_host}:"
+        end
+      end
+    end
+
+    def build_base_gitlab_url
+      base_url(gitlab).join('')
+    end
+
+    def build_gitlab_url
+      (base_url(gitlab) + [gitlab.relative_url_root]).join('')
+    end
+
+    # check that values in `current` (string or integer) is a contant in `modul`.
+    def verify_constant_array(modul, current, default)
+      values = default || []
+      unless current.nil?
+        values = []
+        current.each do |constant|
+          values.push(verify_constant(modul, constant, nil))
+        end
+        values.delete_if { |value| value.nil? }
+      end
+
+      values
+    end
+
+    # check that `current` (string or integer) is a contant in `modul`.
+    def verify_constant(modul, current, default)
+      constant = modul.constants.find { |name| modul.const_get(name) == current }
+      value = constant.nil? ? default : modul.const_get(constant)
+      if current.is_a? String
+        value = modul.const_get(current.upcase) rescue default
+      end
+
+      value
+    end
+
+    def absolute(path)
+      File.expand_path(path, Rails.root)
+    end
+
+    private
+
+    def base_url(config)
+      custom_port = on_standard_port?(config) ? nil : ":#{config.port}"
+
+      [
+        config.protocol,
+        "://",
+        config.host,
+        custom_port
+      ]
+    end
+
+    def on_standard_port?(config)
+      config.port.to_i == (config.https ? 443 : 80)
+    end
+
+    # Extract the host part of the given +url+.
+    def host(url)
+      url = url.downcase
+      url = "http://#{url}" unless url.start_with?('http')
+
+      # Get rid of the path so that we don't even have to encode it
+      url_without_path = url.sub(%r{(https?://[^/]+)/?.*}, '\1')
+
+      URI.parse(url_without_path).host
+    end
+
+    # Runs every minute in a random ten-minute period on Sundays, to balance the
+    # load on the server receiving these pings. The usage ping is safe to run
+    # multiple times because of a 24 hour exclusive lock.
+    def cron_for_usage_ping
+      hour = rand(24)
+      minute = rand(6)
+
+      "#{minute}0-#{minute}9 #{hour} * * 0"
+    end
+  end
+end

rubocop/spec_helpers.rb

 module RuboCop
   module SpecHelpers
-    SPEC_HELPERS = %w[spec_helper.rb rails_helper.rb].freeze
+    SPEC_HELPERS = %w[fast_spec_helper.rb rails_helper.rb spec_helper.rb].freeze
 
     # Returns true if the given node originated from the spec directory.
     def in_spec?(node)

spec/controllers/import/bitbucket_controller_spec.rb

@@ -223,11 +223,12 @@ describe Import::BitbucketController do
     end
 
     context 'user has chosen an existing nested namespace and name for the project', :postgresql do
-      let(:parent_namespace) { create(:group, name: 'foo', owner: user) }
+      let(:parent_namespace) { create(:group, name: 'foo') }
       let(:nested_namespace) { create(:group, name: 'bar', parent: parent_namespace) }
       let(:test_name) { 'test_name' }
 
       before do
+        parent_namespace.add_owner(user)
         nested_namespace.add_owner(user)
       end
 
@@ -273,7 +274,7 @@ describe Import::BitbucketController do
 
     context 'user has chosen existent and non-existent nested namespaces and name for the project', :postgresql do
       let(:test_name) { 'test_name' }
-      let!(:parent_namespace) { create(:group, name: 'foo', owner: user) }
+      let!(:parent_namespace) { create(:group, name: 'foo') }
 
       before do
         parent_namespace.add_owner(user)

spec/controllers/import/gitlab_controller_spec.rb

@@ -196,10 +196,11 @@ describe Import::GitlabController do
       end
 
       context 'user has chosen an existing nested namespace for the project', :postgresql do
-        let(:parent_namespace) { create(:group, name: 'foo', owner: user) }
+        let(:parent_namespace) { create(:group, name: 'foo') }
         let(:nested_namespace) { create(:group, name: 'bar', parent: parent_namespace) }
 
         before do
+          parent_namespace.add_owner(user)
           nested_namespace.add_owner(user)
         end
 
@@ -245,7 +246,7 @@ describe Import::GitlabController do
 
       context 'user has chosen existent and non-existent nested namespaces and name for the project', :postgresql do
         let(:test_name) { 'test_name' }
-        let!(:parent_namespace) { create(:group, name: 'foo', owner: user) }
+        let!(:parent_namespace) { create(:group, name: 'foo') }
 
         before do
           parent_namespace.add_owner(user)

spec/controllers/projects/forks_controller_spec.rb

@@ -4,7 +4,11 @@ describe Projects::ForksController do
   let(:user) { create(:user) }
   let(:project) { create(:project, :public, :repository) }
   let(:forked_project) { Projects::ForkService.new(project, user).execute }
-  let(:group) { create(:group, owner: forked_project.creator) }
+  let(:group) { create(:group) }
+
+  before do
+    group.add_owner(user)
+  end
 
   describe 'GET index' do
     def get_forks

spec/factories/commits.rb

-require_relative '../support/repo_helpers'
+require_relative '../support/helpers/repo_helpers'
 
 FactoryBot.define do
   factory :commit do

spec/factories/gpg_key_subkeys.rb

-require_relative '../support/gpg_helpers'
-
 FactoryBot.define do
   factory :gpg_key_subkey do
     gpg_key

spec/factories/gpg_keys.rb

-require_relative '../support/gpg_helpers'
+require_relative '../support/helpers/gpg_helpers'
 
 FactoryBot.define do
   factory :gpg_key do

spec/factories/gpg_signature.rb

-require_relative '../support/gpg_helpers'
-
 FactoryBot.define do
   factory :gpg_signature do
     commit_sha { Digest::SHA1.hexdigest(SecureRandom.hex) }

spec/factories/groups.rb

@@ -5,6 +5,14 @@ FactoryBot.define do
     type 'Group'
     owner nil
 
+    after(:create) do |group|
+      if group.owner
+        # We could remove this after we have proper constraint:
+        # https://gitlab.com/gitlab-org/gitlab-ce/issues/43292
+        raise "Don't set owner for groups, use `group.add_owner(user)` instead"
+      end
+    end
+
     trait :public do
       visibility_level Gitlab::VisibilityLevel::PUBLIC
     end

spec/factories/namespaces.rb

@@ -2,6 +2,22 @@ FactoryBot.define do
   factory :namespace do
     sequence(:name) { |n| "namespace#{n}" }
     path { name.downcase.gsub(/\s/, '_') }
-    owner
+
+    # This is a workaround to avoid the user creating another namespace via
+    # User#ensure_namespace_correct. We should try to remove it and then
+    # we could remove this workaround
+    association :owner, factory: :user, strategy: :build
+    before(:create) do |namespace|
+      owner = namespace.owner
+
+      if owner
+        # We're changing the username here because we want to keep our path,
+        # and User#ensure_namespace_correct would change the path based on
+        # username, so we're forced to do this otherwise we'll need to change
+        # a lot of existing tests.
+        owner.username = namespace.path
+        owner.namespace = namespace
+      end
+    end
   end
 end

spec/factories/notes.rb

-require_relative '../support/repo_helpers'
+require_relative '../support/helpers/repo_helpers'
 
 include ActionDispatch::TestProcess
 

spec/factories/projects.rb

-require_relative '../support/test_env'
+require_relative '../support/helpers/test_env'
 
 FactoryBot.define do
   # Project without repository

spec/fast_spec_helper.rb

+require 'bundler/setup'
+
+ENV['GITLAB_ENV'] = 'test'
+ENV['IN_MEMORY_APPLICATION_SETTINGS'] = 'true'
+
+unless Object.respond_to?(:require_dependency)
+  class Object
+    alias_method :require_dependency, :require
+  end
+end
+
+# Defines Gitlab and Gitlab.config which are at the center of the app
+require_relative '../lib/gitlab' unless defined?(Gitlab.config)
+
+require_relative 'support/rspec'

spec/features/projects/jobs/user_browses_jobs_spec.rb

@@ -26,7 +26,7 @@ describe 'User browses jobs' do
     page.within('.nav-controls') do
       ci_lint_tool_link = page.find_link('CI lint')
 
-      expect(ci_lint_tool_link[:href]).to end_with(ci_lint_path)
+      expect(ci_lint_tool_link[:href]).to end_with(project_ci_lint_path(project))
     end
   end
 

spec/features/projects/new_project_spec.rb

@@ -64,7 +64,7 @@ feature 'New project' do
     end
 
     context 'with group namespace' do
-      let(:group) { create(:group, :private, owner: user) }
+      let(:group) { create(:group, :private) }
 
       before do
         group.add_owner(user)
@@ -81,7 +81,7 @@ feature 'New project' do
     end
 
     context 'with subgroup namespace' do
-      let(:group) { create(:group, owner: user) }
+      let(:group) { create(:group) }
       let(:subgroup) { create(:group, parent: group) }
 
       before do

spec/lib/gitlab/bitbucket_import/project_creator_spec.rb

@@ -15,7 +15,7 @@ describe Gitlab::BitbucketImport::ProjectCreator do
            has_wiki?: false)
   end
 
-  let(:namespace) { create(:group, owner: user) }
+  let(:namespace) { create(:group) }
   let(:token) { "asdasd12345" }
   let(:secret) { "sekrettt" }
   let(:access_params) { { bitbucket_access_token: token, bitbucket_access_token_secret: secret } }

spec/lib/gitlab/gitlab_import/project_creator_spec.rb

@@ -12,7 +12,7 @@ describe Gitlab::GitlabImport::ProjectCreator do
       owner: { name: "john" }
     }.with_indifferent_access
   end
-  let(:namespace) { create(:group, owner: user) }
+  let(:namespace) { create(:group) }
   let(:token) { "asdffg" }
   let(:access_params) { { gitlab_access_token: token } }
 

spec/lib/gitlab/google_code_import/project_creator_spec.rb

@@ -9,7 +9,7 @@ describe Gitlab::GoogleCodeImport::ProjectCreator do
       "repositoryUrls" => ["https://vim.googlecode.com/git/"]
     )
   end
-  let(:namespace) { create(:group, owner: user) }
+  let(:namespace) { create(:group) }
 
   before do
     namespace.add_owner(user)

spec/lib/gitlab/legacy_github_import/project_creator_spec.rb

@@ -2,7 +2,7 @@ require 'spec_helper'
 
 describe Gitlab::LegacyGithubImport::ProjectCreator do
   let(:user) { create(:user) }
-  let(:namespace) { create(:group, owner: user) }
+  let(:namespace) { create(:group) }
 
   let(:repo) do
     OpenStruct.new(

spec/lib/gitlab_spec.rb

-require 'rails_helper'
+require 'fast_spec_helper'
+
+require_dependency 'gitlab'
 
 describe Gitlab do
+  describe '.root' do
+    it 'returns the root path of the app' do
+      expect(described_class.root).to eq(Pathname.new(File.expand_path('../..', __dir__)))
+    end
+  end
+
   describe '.com?' do
     it 'is true when on GitLab.com' do
       stub_config_setting(url: 'https://gitlab.com')

spec/models/concerns/avatarable_spec.rb

 require 'spec_helper'
 
 describe Avatarable do
-  set(:project) { create(:project, avatar: fixture_file_upload(File.join(Rails.root, 'spec/fixtures/dk.png'))) }
+  let(:project) { create(:project, :with_avatar) }
 
   let(:gitlab_host) { "https://gitlab.example.com" }
   let(:relative_url_root) { "/gitlab" }
@@ -37,11 +37,23 @@ describe Avatarable do
         project.visibility_level = visibility_level
       end
 
-      let(:avatar_path) { (avatar_path_prefix + [project.avatar.url]).join }
+      let(:avatar_path) { (avatar_path_prefix + [project.avatar.local_url]).join }
 
       it 'returns the expected avatar path' do
         expect(project.avatar_path(only_path: only_path)).to eq(avatar_path)
       end
+
+      context "when avatar is stored remotely" do
+        before do
+          stub_uploads_object_storage(AvatarUploader)
+
+          project.avatar.migrate!(ObjectStorage::Store::REMOTE)
+        end
+
+        it 'returns the expected avatar path' do
+          expect(project.avatar_url(only_path: only_path)).to eq(avatar_path)
+        end
+      end
     end
   end
 end

spec/models/project_spec.rb

@@ -325,7 +325,7 @@ describe Project do
     let(:owner)     { create(:user, name: 'Gitlab') }
     let(:namespace) { create(:namespace, path: 'sample-namespace', owner: owner) }
     let(:project)   { create(:project, path: 'sample-project', namespace: namespace) }
-    let(:group)     { create(:group, name: 'Group', path: 'sample-group', owner: owner) }
+    let(:group)     { create(:group, name: 'Group', path: 'sample-group') }
 
     context 'when nil argument' do
       it 'returns nil' do

spec/models/user_spec.rb

@@ -1164,8 +1164,12 @@ describe User do
     end
 
     context 'with a group route matching the given path' do
+      let!(:group) { create(:group, path: 'group_path') }
+
       context 'when the group namespace has an owner_id (legacy data)' do
-        let!(:group) { create(:group, path: 'group_path', owner: user) }
+        before do
+          group.update!(owner_id: user.id)
+        end
 
         it 'returns nil' do
           expect(described_class.find_by_full_path('group_path')).to eq(nil)
@@ -1173,8 +1177,6 @@ describe User do
       end
 
       context 'when the group namespace does not have an owner_id' do
-        let!(:group) { create(:group, path: 'group_path') }
-
         it 'returns nil' do
           expect(described_class.find_by_full_path('group_path')).to eq(nil)
         end

spec/services/groups/nested_create_service_spec.rb

@@ -59,8 +59,11 @@ describe Groups::NestedCreateService do
 
     describe "#execute" do
       it 'returns the group if it already existed' do
-        parent = create(:group, path: 'a-group', owner: user)
-        child = create(:group, path: 'a-sub-group', parent: parent, owner: user)
+        parent = create(:group, path: 'a-group')
+        child = create(:group, path: 'a-sub-group', parent: parent)
+
+        parent.add_owner(user)
+        child.add_owner(user)
 
         expect(service.execute).to eq(child)
       end

spec/spec_helper.rb

@@ -32,42 +32,19 @@ require 'rainbow/ext/string'
 
 # Requires supporting ruby files with custom matchers and macros, etc,
 # in spec/support/ and its subdirectories.
+# Requires helpers, and shared contexts/examples first since they're used in other support files
+Dir[Rails.root.join("spec/support/helpers/*.rb")].each { |f| require f }
+Dir[Rails.root.join("spec/support/shared_contexts/*.rb")].each { |f| require f }
+Dir[Rails.root.join("spec/support/shared_examples/*.rb")].each { |f| require f }
 Dir[Rails.root.join("spec/support/**/*.rb")].each { |f| require f }
 
 RSpec.configure do |config|
   config.use_transactional_fixtures = false
   config.use_instantiated_fixtures  = false
-  config.mock_with :rspec
 
   config.verbose_retry = true
   config.display_try_failure_messages = true
 
-  config.include Devise::Test::ControllerHelpers, type: :controller
-  config.include Devise::Test::ControllerHelpers, type: :view
-  config.include Devise::Test::IntegrationHelpers, type: :feature
-  config.include Warden::Test::Helpers, type: :request
-  config.include LoginHelpers, type: :feature
-  config.include SearchHelpers, type: :feature
-  config.include CookieHelper, :js
-  config.include InputHelper, :js
-  config.include SelectionHelper, :js
-  config.include InspectRequests, :js
-  config.include WaitForRequests, :js
-  config.include LiveDebugger, :js
-  config.include StubConfiguration
-  config.include EmailHelpers, :mailer, type: :mailer
-  config.include TestEnv
-  config.include ActiveJob::TestHelper
-  config.include ActiveSupport::Testing::TimeHelpers
-  config.include StubGitlabCalls
-  config.include StubGitlabData
-  config.include ApiHelpers, :api
-  config.include Gitlab::Routing, type: :routing
-  config.include MigrationsHelpers, :migration
-  config.include StubFeatureFlags
-  config.include StubENV
-  config.include ExpectOffense
-
   config.infer_spec_type_from_file_location!
 
   config.define_derived_metadata(file_path: %r{/spec/}) do |metadata|
@@ -82,7 +59,33 @@ RSpec.configure do |config|
     metadata[:type] = match[1].singularize.to_sym if match
   end
 
-  config.raise_errors_for_deprecations!
+  config.include ActiveJob::TestHelper
+  config.include ActiveSupport::Testing::TimeHelpers
+  config.include CycleAnalyticsHelpers
+  config.include ExpectOffense
+  config.include FactoryBot::Syntax::Methods
+  config.include FixtureHelpers
+  config.include GitlabRoutingHelper
+  config.include StubFeatureFlags
+  config.include StubGitlabCalls
+  config.include StubGitlabData
+  config.include TestEnv
+  config.include Devise::Test::ControllerHelpers, type: :controller
+  config.include Devise::Test::IntegrationHelpers, type: :feature
+  config.include LoginHelpers, type: :feature
+  config.include SearchHelpers, type: :feature
+  config.include EmailHelpers, :mailer, type: :mailer
+  config.include Warden::Test::Helpers, type: :request
+  config.include Gitlab::Routing, type: :routing
+  config.include Devise::Test::ControllerHelpers, type: :view
+  config.include ApiHelpers, :api
+  config.include CookieHelper, :js
+  config.include InputHelper, :js
+  config.include SelectionHelper, :js
+  config.include InspectRequests, :js
+  config.include WaitForRequests, :js
+  config.include LiveDebugger, :js
+  config.include MigrationsHelpers, :migration
 
   if ENV['CI']
     # This includes the first try, i.e. tests will be run 4 times before failing.

spec/support/capybara.rb

@@ -60,6 +60,8 @@ Capybara::Screenshot.register_driver(:chrome) do |driver, path|
 end
 
 RSpec.configure do |config|
+  config.include CapybaraHelpers, type: :feature
+
   config.before(:context, :js) do
     next if $capybara_server_already_started
 

spec/support/controllers/githubish_import_controller_shared_examples.rb

@@ -56,7 +56,7 @@ shared_examples 'a GitHub-ish import controller: GET status' do
   end
 
   it "assigns variables" do
-    project = create(:project, import_type: provider, creator_id: user.id)
+    project = create(:project, import_type: provider, namespace: user.namespace)
     stub_client(repos: [repo, org_repo], orgs: [org], org_repos: [org_repo])
 
     get :status
@@ -69,7 +69,7 @@ shared_examples 'a GitHub-ish import controller: GET status' do
   end
 
   it "does not show already added project" do
-    project = create(:project, import_type: provider, creator_id: user.id, import_source: 'asd/vim')
+    project = create(:project, import_type: provider, namespace: user.namespace, import_source: 'asd/vim')
     stub_client(repos: [repo], orgs: [])
 
     get :status
@@ -257,11 +257,12 @@ shared_examples 'a GitHub-ish import controller: POST create' do
     end
 
     context 'user has chosen an existing nested namespace and name for the project', :postgresql do
-      let(:parent_namespace) { create(:group, name: 'foo', owner: user) }
+      let(:parent_namespace) { create(:group, name: 'foo') }
       let(:nested_namespace) { create(:group, name: 'bar', parent: parent_namespace) }
       let(:test_name) { 'test_name' }
 
       before do
+        parent_namespace.add_owner(user)
         nested_namespace.add_owner(user)
       end
 
@@ -307,7 +308,7 @@ shared_examples 'a GitHub-ish import controller: POST create' do
 
     context 'user has chosen existent and non-existent nested namespaces and name for the project', :postgresql do
       let(:test_name) { 'test_name' }
-      let!(:parent_namespace) { create(:group, name: 'foo', owner: user) }
+      let!(:parent_namespace) { create(:group, name: 'foo') }
 
       before do
         parent_namespace.add_owner(user)

spec/support/factory_bot.rb

-RSpec.configure do |config|
-  config.include FactoryBot::Syntax::Methods
-end

spec/support/generate-seed-repo-rb

@@ -8,7 +8,7 @@
 #
 # Usage:
 #
-#   ./spec/support/generate-seed-repo-rb > spec/support/seed_repo.rb
+#   ./spec/support/generate-seed-repo-rb > spec/support/helpers/seed_repo.rb
 #
 #
 

spec/support/gitlab-git-test.git/README.md

@@ -12,5 +12,5 @@ inflate the size of the gitlab-ce repository.
 - make changes in your local clone of gitlab-git-test
 - run `git push` which will push to your local source `gitlab-ce/spec/support/gitlab-git-test.git`
 - in gitlab-ce: run `spec/support/prepare-gitlab-git-test-for-commit`
-- in gitlab-ce: `git add spec/support/seed_repo.rb spec/support/gitlab-git-test.git`
+- in gitlab-ce: `git add spec/support/helpers/seed_repo.rb spec/support/gitlab-git-test.git`
 - commit your changes in gitlab-ce

spec/support/capybara_helpers.rb → spec/support/helpers/capybara_helpers.rb

@@ -41,7 +41,3 @@ module CapybaraHelpers
     page.driver.browser.manage.delete_cookie('_gitlab_session')
   end
 end
-
-RSpec.configure do |config|
-  config.include CapybaraHelpers, type: :feature
-end

spec/support/cycle_analytics_helpers.rb → spec/support/helpers/cycle_analytics_helpers.rb

@@ -135,7 +135,3 @@ module CycleAnalyticsHelpers
     end
   end
 end
-
-RSpec.configure do |config|
-  config.include CycleAnalyticsHelpers
-end

spec/support/fixture_helpers.rb → spec/support/helpers/fixture_helpers.rb

@@ -9,7 +9,3 @@ module FixtureHelpers
     File.expand_path(Rails.root.join(dir, 'spec', 'fixtures', filename))
   end
 end
-
-RSpec.configure do |config|
-  config.include FixtureHelpers
-end

spec/support/helpers/gitlab_verify_helpers.rb

+module GitlabVerifyHelpers
+  def collect_ranges(args = {})
+    verifier = described_class.new(args.merge(batch_size: 1))
+
+    collect_results(verifier).map { |range, _| range }
+  end
+
+  def collect_failures
+    verifier = described_class.new(batch_size: 1)
+
+    out = {}
+
+    collect_results(verifier).map { |_, failures| out.merge!(failures) }
+
+    out
+  end
+
+  def collect_results(verifier)
+    out = []
+
+    verifier.run_batches { |*args| out << args }
+
+    out
+  end
+end