Add latest changes from gitlab-org/gitlab@master

app/controllers/projects/commits_controller.rb

@@ -64,10 +64,13 @@ class Projects::CommitsController < Projects::ApplicationController
     render_404 unless @path.empty? || request.format == :atom || @repository.blob_at(@commit.id, @path) || @repository.tree(@commit.id, @path).entries.present?
     @limit, @offset = (params[:limit] || 40).to_i, (params[:offset] || 0).to_i
     search = params[:search]
+    author = params[:author]
 
     @commits =
       if search.present?
         @repository.find_commits_by_message(search, @ref, @path, @limit, @offset)
+      elsif author.present?
+        @repository.commits(@ref, author: author, path: @path, limit: @limit, offset: @offset)
       else
         @repository.commits(@ref, path: @path, limit: @limit, offset: @offset)
       end

app/controllers/repositories/git_http_controller.rb

@@ -7,7 +7,7 @@ module Repositories
     before_action :access_check
     prepend_before_action :deny_head_requests, only: [:info_refs]
 
-    rescue_from Gitlab::GitAccess::UnauthorizedError, with: :render_403_with_exception
+    rescue_from Gitlab::GitAccess::ForbiddenError, with: :render_403_with_exception
     rescue_from Gitlab::GitAccess::NotFoundError, with: :render_404_with_exception
     rescue_from Gitlab::GitAccess::ProjectCreationError, with: :render_422_with_exception
     rescue_from Gitlab::GitAccess::TimeoutError, with: :render_503_with_exception

app/models/ci/pipeline.rb

@@ -599,7 +599,7 @@ module Ci
     # Manually set the notes for a Ci::Pipeline
     # There is no ActiveRecord relation between Ci::Pipeline and notes
     # as they are related to a commit sha. This method helps importing
-    # them using the +Gitlab::ImportExport::ProjectRelationFactory+ class.
+    # them using the +Gitlab::ImportExport::Project::RelationFactory+ class.
     def notes=(notes)
       notes.each do |note|
         note[:id] = nil

app/models/repository.rb

@@ -139,6 +139,7 @@ class Repository
       repo: raw_repository,
       ref: ref,
       path: opts[:path],
+      author: opts[:author],
       follow: Array(opts[:path]).length == 1,
       limit: opts[:limit],
       offset: opts[:offset],

app/services/groups/import_export/export_service.rb

@@ -18,6 +18,8 @@ module Groups
         end
 
         save!
+      ensure
+        cleanup
       end
 
       private
@@ -28,7 +30,7 @@ module Groups
         if savers.all?(&:save)
           notify_success
         else
-          cleanup_and_notify_error!
+          notify_error!
         end
       end
 
@@ -37,21 +39,19 @@ module Groups
       end
 
       def tree_exporter
-        Gitlab::ImportExport::GroupTreeSaver.new(group: @group, current_user: @current_user, shared: @shared, params: @params)
+        Gitlab::ImportExport::Group::TreeSaver.new(group: @group, current_user: @current_user, shared: @shared, params: @params)
       end
 
       def file_saver
         Gitlab::ImportExport::Saver.new(exportable: @group, shared: @shared)
       end
 
-      def cleanup_and_notify_error
-        FileUtils.rm_rf(shared.export_path)
-
-        notify_error
+      def cleanup
+        FileUtils.rm_rf(shared.archive_path) if shared&.archive_path
       end
 
-      def cleanup_and_notify_error!
-        cleanup_and_notify_error
+      def notify_error!
+        notify_error
 
         raise Gitlab::ImportExport::Error.new(shared.errors.to_sentence)
       end

app/services/groups/import_export/import_service.rb

@@ -34,7 +34,7 @@ module Groups
       end
 
       def restorer
-        @restorer ||= Gitlab::ImportExport::GroupTreeRestorer.new(user: @current_user,
+        @restorer ||= Gitlab::ImportExport::Group::TreeRestorer.new(user: @current_user,
                                                                   shared: @shared,
                                                                   group: @group,
                                                                   group_hash: nil)

app/services/lfs/lock_file_service.rb

@@ -4,13 +4,13 @@ module Lfs
   class LockFileService < BaseService
     def execute
       unless can?(current_user, :push_code, project)
-        raise Gitlab::GitAccess::UnauthorizedError, 'You have no permissions'
+        raise Gitlab::GitAccess::ForbiddenError, 'You have no permissions'
       end
 
       create_lock!
     rescue ActiveRecord::RecordNotUnique
       error('already locked', 409, current_lock)
-    rescue Gitlab::GitAccess::UnauthorizedError => ex
+    rescue Gitlab::GitAccess::ForbiddenError => ex
       error(ex.message, 403)
     rescue => ex
       error(ex.message, 500)

app/services/lfs/unlock_file_service.rb

@@ -4,11 +4,11 @@ module Lfs
   class UnlockFileService < BaseService
     def execute
       unless can?(current_user, :push_code, project)
-        raise Gitlab::GitAccess::UnauthorizedError, _('You have no permissions')
+        raise Gitlab::GitAccess::ForbiddenError, _('You have no permissions')
       end
 
       unlock_file
-    rescue Gitlab::GitAccess::UnauthorizedError => ex
+    rescue Gitlab::GitAccess::ForbiddenError => ex
       error(ex.message, 403)
     rescue ActiveRecord::RecordNotFound
       error(_('Lock not found'), 404)

app/services/projects/import_export/export_service.rb

@@ -14,6 +14,8 @@ module Projects
 
         save_all!
         execute_after_export_action(after_export_strategy)
+      ensure
+        cleanup
       end
 
       private
@@ -24,7 +26,7 @@ module Projects
         return unless after_export_strategy
 
         unless after_export_strategy.execute(current_user, project)
-          cleanup_and_notify_error
+          notify_error
         end
       end
 
@@ -33,7 +35,7 @@ module Projects
           Gitlab::ImportExport::Saver.save(exportable: project, shared: shared)
           notify_success
         else
-          cleanup_and_notify_error!
+          notify_error!
         end
       end
 
@@ -54,7 +56,7 @@ module Projects
       end
 
       def project_tree_saver
-        Gitlab::ImportExport::ProjectTreeSaver.new(project: project, current_user: current_user, shared: shared, params: params)
+        Gitlab::ImportExport::Project::TreeSaver.new(project: project, current_user: current_user, shared: shared, params: params)
       end
 
       def uploads_saver
@@ -73,16 +75,12 @@ module Projects
         Gitlab::ImportExport::LfsSaver.new(project: project, shared: shared)
       end
 
-      def cleanup_and_notify_error
-        Rails.logger.error("Import/Export - Project #{project.name} with ID: #{project.id} export error - #{shared.errors.join(', ')}") # rubocop:disable Gitlab/RailsLogger
-
-        FileUtils.rm_rf(shared.export_path)
-
-        notify_error
+      def cleanup
+        FileUtils.rm_rf(shared.archive_path) if shared&.archive_path
       end
 
-      def cleanup_and_notify_error!
-        cleanup_and_notify_error
+      def notify_error!
+        notify_error
 
         raise Gitlab::ImportExport::Error.new(shared.errors.to_sentence)
       end
@@ -92,6 +90,8 @@ module Projects
       end
 
       def notify_error
+        Rails.logger.error("Import/Export - Project #{project.name} with ID: #{project.id} export error - #{shared.errors.join(', ')}") # rubocop:disable Gitlab/RailsLogger
+
         notification_service.project_not_exported(project, current_user, shared.errors)
       end
     end

changelogs/unreleased/do-not-parse-undefined-severity-confidence.yml

+---
+title: Do not parse undefined severity and confidence from reports
+merge_request: 25884
+author:
+type: other

changelogs/unreleased/find-commits-by-author.yml

+---
+title: Filter commits by author
+merge_request: 25597
+author:
+type: added

changelogs/unreleased/fj-rename-unauthorized-error-to-forbidden-error.yml

+---
+title: Align git returned error codes
+merge_request: 25936
+author:
+type: changed

changelogs/unreleased/georgekoltsov-196188-cleanup-temp-exports.yml

+---
+title: Ensure temp export data is removed if Group/Project export failed
+merge_request: 25828
+author:
+type: fixed

danger/gemfile/Dangerfile

-GEMFILE_LOCK_NOT_UPDATED_MESSAGE_SHORT = <<~MSG.freeze
-%<gemfile>s was updated but %<gemfile_lock>s wasn't updated.
-MSG
-
-GEMFILE_LOCK_NOT_UPDATED_MESSAGE_FULL = <<~MSG.freeze
-**#{GEMFILE_LOCK_NOT_UPDATED_MESSAGE_SHORT}**
-
-Usually, when %<gemfile>s is updated, you should run
-```
-bundle install
-```
-
-or
-
-```
-bundle update <the-added-or-updated-gem>
-```
-
-and commit the %<gemfile_lock>s changes.
-MSG
-
-gemfile_modified = git.modified_files.include?("Gemfile")
-gemfile_lock_modified = git.modified_files.include?("Gemfile.lock")
-
-if gemfile_modified && !gemfile_lock_modified
-  gitlab_danger = GitlabDanger.new(helper.gitlab_helper)
-
-  format_str = gitlab_danger.ci? ? GEMFILE_LOCK_NOT_UPDATED_MESSAGE_FULL : GEMFILE_LOCK_NOT_UPDATED_MESSAGE_SHORT
-
-  message = format(format_str,
-    gemfile: gitlab_danger.html_link("Gemfile"),
-    gemfile_lock: gitlab_danger.html_link("Gemfile.lock")
-  )
-
-  warn(message)
-end

doc/administration/compliance.md

@@ -17,3 +17,4 @@ GitLab’s [security features](../security/README.md) may also help you meet rel
 |**[Audit logs](audit_events.md)**<br>To maintain the integrity of your code, GitLab Enterprise Edition Premium gives admins the ability to view any modifications made within the GitLab server in an advanced audit log system, so you can control, analyze, and track every change.|Premium+||
 |**[Auditor users](auditor_users.md)**<br>Auditor users are users who are given read-only access to all projects, groups, and other resources on the GitLab instance.|Premium+||
 |**[Credentials inventory](../user/admin_area/credentials_inventory.md)**<br>With a credentials inventory, GitLab administrators can keep track of the credentials used by all of the users in their GitLab instance. |Ultimate||
+|**Separation of Duties using [Protected branches](../user/project/protected_branches.md#protected-branches-approval-by-code-owners-premium) and [custom CI Configuration Paths](../user/project/pipelines/settings.md#custom-ci-configuration-path)**<br> GitLab Silver and Premium users can leverage GitLab's cross-project YAML configuration's to define deployers of code and developers of code. View the [Separation of Duties Deploy Project](https://gitlab.com/guided-explorations/separation-of-duties-deploy/blob/master/README.md) and [Separation of Duties Project](https://gitlab.com/guided-explorations/separation-of-duties/blob/master/README.md) to see how to use this set up to define these roles.|Premium+||

doc/administration/packages/index.md

@@ -118,7 +118,10 @@ upload packages:
      #'path_style' => false                    # If true, use 'host/bucket_name/object' instead of 'bucket_name.host/object'.
    }
    ```
-
+  
+   NOTE: **Note:**
+   Some build tools, like Gradle, must make `HEAD` requests to Amazon S3 to pull a dependency’s metadata. The `gitlab_rails['packages_object_store_proxy_download']` property must be set to `true`. Without this setting, GitLab won't act as a proxy to the Amazon S3 service, and will instead return the signed URL. This will cause a `HTTP 403 Forbidden` response, since Amazon S3 expects a signed URL.
+  
 1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure)
    for the changes to take effect.
 

doc/development/import_project.md

@@ -81,7 +81,7 @@ The last option is to import a project using a Rails console:
    sudo -u git -H bundle exec rails console RAILS_ENV=production
    ```
 
-1. Create a project and run `ProjectTreeRestorer`:
+1. Create a project and run `Project::TreeRestorer`:
 
    ```ruby
    shared_class = Struct.new(:export_path) do
@@ -98,7 +98,7 @@ The last option is to import a project using a Rails console:
    begin
      #Enable Request store
      RequestStore.begin!
-     Gitlab::ImportExport::ProjectTreeRestorer.new(user: user, shared: shared, project: project).restore
+     Gitlab::ImportExport::Project::TreeRestorer.new(user: user, shared: shared, project: project).restore
    ensure
      RequestStore.end!
      RequestStore.clear!
@@ -128,11 +128,11 @@ The last option is to import a project using a Rails console:
 For Performance testing, we should:
 
 - Import a quite large project, [`gitlabhq`](https://gitlab.com/gitlab-org/quality/performance-data#gitlab-performance-test-framework-data) should be a good example.
-- Measure the execution time of `ProjectTreeRestorer`.
+- Measure the execution time of `Project::TreeRestorer`.
 - Count the number of executed SQL queries during the restore.
 - Observe the number of GC cycles happening.
 
-You can use this [snippet](https://gitlab.com/gitlab-org/gitlab/snippets/1924954), which will restore the project, and measure the execution time of `ProjectTreeRestorer`, number of SQL queries and number of GC cycles happening.
+You can use this [snippet](https://gitlab.com/gitlab-org/gitlab/snippets/1924954), which will restore the project, and measure the execution time of `Project::TreeRestorer`, number of SQL queries and number of GC cycles happening.
 
 You can execute the script from the `gdk/gitlab` directory like this:
 

doc/user/clusters/applications.md

@@ -257,7 +257,7 @@ use an A record. If your external endpoint is a hostname, use a CNAME record.
 
 > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/21966) in GitLab 12.7.
 
-A Web Application Firewall (WAF) is able to examine traffic being sent/received
+A Web Application Firewall (WAF) examines traffic being sent or received,
 and can block malicious traffic before it reaches your application. The benefits
 of a WAF are:
 
@@ -266,7 +266,7 @@ of a WAF are:
 - Access control for your application
 - Highly configurable logging and blocking rules
 
-Out of the box, GitLab provides you with a WAF known as [`ModSecurity`](https://www.modsecurity.org/)
+Out of the box, GitLab provides you with a WAF known as [`ModSecurity`](https://www.modsecurity.org/).
 
 ModSecurity is a toolkit for real-time web application monitoring, logging,
 and access control. With GitLab's offering, the [OWASP's Core Rule Set](https://www.modsecurity.org/CRS/Documentation/),
@@ -288,9 +288,6 @@ when installing your [Ingress application](#ingress).
 If this is your first time using GitLab's WAF, we recommend you follow the
 [quick start guide](../../topics/web_application_firewall/quick_start_guide.md).
 
-There is a small performance overhead by enabling ModSecurity. However,
-if this is considered significant for your application, you can disable it.
-
 There is a small performance overhead by enabling ModSecurity. If this is
 considered significant for your application, you can disable ModSecurity's
 rule engine for your deployed application by setting
@@ -693,7 +690,7 @@ cilium:
 ```
 
 The `clusterType` variable enables the recommended Helm variables for
-a corresponding cluster type, the default value is blank. You can
+a corresponding cluster type. The default value is blank. You can
 check the recommended variables for each cluster type in the official
 documentation:
 
@@ -720,13 +717,13 @@ information.
 By default, Cilium will drop all non-whitelisted packets upon policy
 deployment. The audit mode is scheduled for release in
 [Cilium 1.8](https://github.com/cilium/cilium/pull/9970). In the audit
-mode non-whitelisted packets will not be dropped, instead audit
-notifications will be generated. GitLab provides alternative Docker
+mode, non-whitelisted packets will not be dropped, and audit
+notifications will be generated instead. GitLab provides alternative Docker
 images for Cilium with the audit patch included. You can switch to the
 custom build and enable the audit mode by adding the following to
 `.gitlab/managed-apps/cilium/values.yaml`:
 
-```yml
+```yaml
 global:
   registry: registry.gitlab.com/gitlab-org/defend/cilium
   policyAuditMode: true
@@ -737,15 +734,15 @@ agent:
 ```
 
 The Cilium monitor log for traffic is logged out by the
-`cilium-monitor` sidecar container. You can check these logs via:
+`cilium-monitor` sidecar container. You can check these logs with the following command:
 
 ```shell
 kubectl -n gitlab-managed-apps logs cilium-XXXX cilium-monitor
 ```
 
-You can disable the monitor log via `.gitlab/managed-apps/cilium/values.yaml`:
+You can disable the monitor log in `.gitlab/managed-apps/cilium/values.yaml`:
 
-```yml
+```yaml
 agent:
   monitor:
     enabled: false

doc/user/project/settings/import_export.md

@@ -89,7 +89,7 @@ The following items will NOT be exported:
 
 NOTE: **Note:**
 For more details on the specific data persisted in a project export, see the
-[`import_export.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/import_export/import_export.yml) file.
+[`import_export.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/import_export/project/import_export.yml) file.
 
 ## Exporting a project and its data
 

lib/api/internal/base.rb

@@ -49,8 +49,8 @@ module API
                            result = access_checker.check(params[:action], params[:changes])
                            @project ||= access_checker.project
                            result
-                         rescue Gitlab::GitAccess::UnauthorizedError => e
-                           return response_with_status(code: 401, success: false, message: e.message)
+                         rescue Gitlab::GitAccess::ForbiddenError => e
+                           return response_with_status(code: 403, success: false, message: e.message)
                          rescue Gitlab::GitAccess::TimeoutError => e
                            return response_with_status(code: 503, success: false, message: e.message)
                          rescue Gitlab::GitAccess::NotFoundError => e

lib/gitlab/checks/branch_check.rb

@@ -28,7 +28,7 @@ module Gitlab
 
         logger.log_timed(LOG_MESSAGES[:delete_default_branch_check]) do
           if deletion? && branch_name == project.default_branch
-            raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:delete_default_branch]
+            raise GitAccess::ForbiddenError, ERROR_MESSAGES[:delete_default_branch]
           end
         end
 
@@ -42,7 +42,7 @@ module Gitlab
           return unless ProtectedBranch.protected?(project, branch_name) # rubocop:disable Cop/AvoidReturnFromBlocks
 
           if forced_push?
-            raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:force_push_protected_branch]
+            raise GitAccess::ForbiddenError, ERROR_MESSAGES[:force_push_protected_branch]
           end
         end
 
@@ -62,15 +62,15 @@ module Gitlab
           break if user_access.can_push_to_branch?(branch_name)
 
           unless user_access.can_merge_to_branch?(branch_name)
-            raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:create_protected_branch]
+            raise GitAccess::ForbiddenError, ERROR_MESSAGES[:create_protected_branch]
           end
 
           unless safe_commit_for_new_protected_branch?
-            raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:invalid_commit_create_protected_branch]
+            raise GitAccess::ForbiddenError, ERROR_MESSAGES[:invalid_commit_create_protected_branch]
           end
 
           unless updated_from_web?
-            raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:non_web_create_protected_branch]
+            raise GitAccess::ForbiddenError, ERROR_MESSAGES[:non_web_create_protected_branch]
           end
         end
       end
@@ -78,11 +78,11 @@ module Gitlab
       def protected_branch_deletion_checks
         logger.log_timed(LOG_MESSAGES[:protected_branch_deletion_checks]) do
           unless user_access.can_delete_branch?(branch_name)
-            raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:non_master_delete_protected_branch]
+            raise GitAccess::ForbiddenError, ERROR_MESSAGES[:non_master_delete_protected_branch]
           end
 
           unless updated_from_web?
-            raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:non_web_delete_protected_branch]
+            raise GitAccess::ForbiddenError, ERROR_MESSAGES[:non_web_delete_protected_branch]
           end
         end
       end
@@ -91,11 +91,11 @@ module Gitlab
         logger.log_timed(LOG_MESSAGES[:protected_branch_push_checks]) do
           if matching_merge_request?
             unless user_access.can_merge_to_branch?(branch_name) || user_access.can_push_to_branch?(branch_name)
-              raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:merge_protected_branch]
+              raise GitAccess::ForbiddenError, ERROR_MESSAGES[:merge_protected_branch]
             end
           else
             unless user_access.can_push_to_branch?(branch_name)
-              raise GitAccess::UnauthorizedError, push_to_protected_branch_rejected_message
+              raise GitAccess::ForbiddenError, push_to_protected_branch_rejected_message
             end
           end
         end

lib/gitlab/checks/diff_check.rb

@@ -46,7 +46,7 @@ module Gitlab
       def validate_diff(diff)
         validations_for_diff.each do |validation|
           if error = validation.call(diff)
-            raise ::Gitlab::GitAccess::UnauthorizedError, error
+            raise ::Gitlab::GitAccess::ForbiddenError, error
           end
         end
       end
@@ -77,7 +77,7 @@ module Gitlab
         logger.log_timed(LOG_MESSAGES[__method__]) do
           path_validations.each do |validation|
             if error = validation.call(file_paths)
-              raise ::Gitlab::GitAccess::UnauthorizedError, error
+              raise ::Gitlab::GitAccess::ForbiddenError, error
             end
           end
         end

lib/gitlab/checks/lfs_check.rb

@@ -15,7 +15,7 @@ module Gitlab
           lfs_check = Checks::LfsIntegrity.new(project, newrev, logger.time_left)
 
           if lfs_check.objects_missing?
-            raise GitAccess::UnauthorizedError, ERROR_MESSAGE
+            raise GitAccess::ForbiddenError, ERROR_MESSAGE
           end
         end
       end

lib/gitlab/checks/push_check.rb

@@ -6,7 +6,7 @@ module Gitlab
       def validate!
         logger.log_timed("Checking if you are allowed to push...") do
           unless can_push?
-            raise GitAccess::UnauthorizedError, GitAccess::ERROR_MESSAGES[:push_code]
+            raise GitAccess::ForbiddenError, GitAccess::ERROR_MESSAGES[:push_code]
           end
         end
       end

lib/gitlab/checks/tag_check.rb

@@ -20,7 +20,7 @@ module Gitlab
 
         logger.log_timed(LOG_MESSAGES[:tag_checks]) do
           if tag_exists? && user_access.cannot_do_action?(:admin_tag)
-            raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:change_existing_tags]
+            raise GitAccess::ForbiddenError, ERROR_MESSAGES[:change_existing_tags]
           end
         end
 
@@ -33,11 +33,11 @@ module Gitlab
         logger.log_timed(LOG_MESSAGES[__method__]) do
           return unless ProtectedTag.protected?(project, tag_name) # rubocop:disable Cop/AvoidReturnFromBlocks
 
-          raise(GitAccess::UnauthorizedError, ERROR_MESSAGES[:update_protected_tag]) if update?
-          raise(GitAccess::UnauthorizedError, ERROR_MESSAGES[:delete_protected_tag]) if deletion?
+          raise(GitAccess::ForbiddenError, ERROR_MESSAGES[:update_protected_tag]) if update?
+          raise(GitAccess::ForbiddenError, ERROR_MESSAGES[:delete_protected_tag]) if deletion?
 
           unless user_access.can_create_tag?(tag_name)
-            raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:create_protected_tag]
+            raise GitAccess::ForbiddenError, ERROR_MESSAGES[:create_protected_tag]
           end
         end
       end

lib/gitlab/git/repository.rb

@@ -322,6 +322,7 @@ module Gitlab
           limit: 10,
           offset: 0,
           path: nil,
+          author: nil,
           follow: false,
           skip_merges: false,
           after: nil,

lib/gitlab/git_access.rb

@@ -6,7 +6,7 @@ module Gitlab
   class GitAccess
     include Gitlab::Utils::StrongMemoize
 
-    UnauthorizedError = Class.new(StandardError)
+    ForbiddenError = Class.new(StandardError)
     NotFoundError = Class.new(StandardError)
     ProjectCreationError = Class.new(StandardError)
     TimeoutError = Class.new(StandardError)
@@ -125,7 +125,7 @@ module Gitlab
       return unless actor.is_a?(Key)
 
       unless actor.valid?
-        raise UnauthorizedError, "Your SSH key #{actor.errors[:key].first}."
+        raise ForbiddenError, "Your SSH key #{actor.errors[:key].first}."
       end
     end
 
@@ -133,7 +133,7 @@ module Gitlab
       return if request_from_ci_build?
 
       unless protocol_allowed?
-        raise UnauthorizedError, "Git access over #{protocol.upcase} is not allowed"
+        raise ForbiddenError, "Git access over #{protocol.upcase} is not allowed"
       end
     end
 
@@ -148,7 +148,7 @@ module Gitlab
 
       unless user_access.allowed?
         message = Gitlab::Auth::UserAccessDeniedReason.new(user).rejection_message
-        raise UnauthorizedError, message
+        raise ForbiddenError, message
       end
     end
 
@@ -156,11 +156,11 @@ module Gitlab
       case cmd
       when *DOWNLOAD_COMMANDS
         unless authentication_abilities.include?(:download_code) || authentication_abilities.include?(:build_download_code)
-          raise UnauthorizedError, ERROR_MESSAGES[:auth_download]
+          raise ForbiddenError, ERROR_MESSAGES[:auth_download]
         end
       when *PUSH_COMMANDS
         unless authentication_abilities.include?(:push_code)
-          raise UnauthorizedError, ERROR_MESSAGES[:auth_upload]
+          raise ForbiddenError, ERROR_MESSAGES[:auth_upload]
         end
       end
     end
@@ -189,19 +189,19 @@ module Gitlab
 
     def check_upload_pack_disabled!
       if http? && upload_pack_disabled_over_http?
-        raise UnauthorizedError, ERROR_MESSAGES[:upload_pack_disabled_over_http]
+        raise ForbiddenError, ERROR_MESSAGES[:upload_pack_disabled_over_http]
       end
     end
 
     def check_receive_pack_disabled!
       if http? && receive_pack_disabled_over_http?
-        raise UnauthorizedError, ERROR_MESSAGES[:receive_pack_disabled_over_http]
+        raise ForbiddenError, ERROR_MESSAGES[:receive_pack_disabled_over_http]
       end
     end
 
     def check_command_existence!(cmd)
       unless ALL_COMMANDS.include?(cmd)
-        raise UnauthorizedError, ERROR_MESSAGES[:command_not_allowed]
+        raise ForbiddenError, ERROR_MESSAGES[:command_not_allowed]
       end
     end
 
@@ -209,7 +209,7 @@ module Gitlab
       return unless receive_pack?(cmd)
 
       if Gitlab::Database.read_only?
-        raise UnauthorizedError, push_to_read_only_message
+        raise ForbiddenError, push_to_read_only_message
       end
     end
 
@@ -253,23 +253,23 @@ module Gitlab
         guest_can_download_code?
 
       unless passed
-        raise UnauthorizedError, ERROR_MESSAGES[:download]
+        raise ForbiddenError, ERROR_MESSAGES[:download]
       end
     end
 
     def check_push_access!
       if project.repository_read_only?
-        raise UnauthorizedError, ERROR_MESSAGES[:read_only]
+        raise ForbiddenError, ERROR_MESSAGES[:read_only]
       end
 
       if deploy_key?
         unless deploy_key.can_push_to?(project)
-          raise UnauthorizedError, ERROR_MESSAGES[:deploy_key_upload]
+          raise ForbiddenError, ERROR_MESSAGES[:deploy_key_upload]
         end
       elsif user
         # User access is verified in check_change_access!
       else
-        raise UnauthorizedError, ERROR_MESSAGES[:upload]
+        raise ForbiddenError, ERROR_MESSAGES[:upload]
       end
 
       check_change_access!
@@ -284,7 +284,7 @@ module Gitlab
           project.any_branch_allows_collaboration?(user_access.user)
 
         unless can_push
-          raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:push_code]
+          raise ForbiddenError, ERROR_MESSAGES[:push_code]
         end
       else
         # If there are worktrees with a HEAD pointing to a non-existent object,

lib/gitlab/git_access_wiki.rb

@@ -19,11 +19,11 @@ module Gitlab
 
     def check_change_access!
       unless user_access.can_do_action?(:create_wiki)
-        raise UnauthorizedError, ERROR_MESSAGES[:write_to_wiki]
+        raise ForbiddenError, ERROR_MESSAGES[:write_to_wiki]
       end
 
       if Gitlab::Database.read_only?
-        raise UnauthorizedError, push_to_read_only_message
+        raise ForbiddenError, push_to_read_only_message
       end
 
       true

lib/gitlab/gitaly_client/commit_service.rb

@@ -324,7 +324,8 @@ module Gitlab
         request.after    = GitalyClient.timestamp(options[:after]) if options[:after]
         request.before   = GitalyClient.timestamp(options[:before]) if options[:before]
         request.revision = encode_binary(options[:ref]) if options[:ref]
-        request.order = options[:order].upcase.sub('DEFAULT', 'NONE') if options[:order].present?
+        request.author   = encode_binary(options[:author]) if options[:author]
+        request.order    = options[:order].upcase.sub('DEFAULT', 'NONE') if options[:order].present?
 
         request.paths = encode_repeated(Array(options[:path])) if options[:path].present?
 

lib/gitlab/import_export.rb

@@ -43,7 +43,7 @@ module Gitlab
     end
 
     def config_file
-      Rails.root.join('lib/gitlab/import_export/import_export.yml')
+      Rails.root.join('lib/gitlab/import_export/project/import_export.yml')
     end
 
     def version_filename
@@ -77,7 +77,7 @@ module Gitlab
     end
 
     def group_config_file
-      Rails.root.join('lib/gitlab/import_export/group_import_export.yml')
+      Rails.root.join('lib/gitlab/import_export/group/import_export.yml')
     end
   end
 end

lib/gitlab/import_export/attribute_cleaner.rb

@@ -4,8 +4,8 @@ module Gitlab
   module ImportExport
     class AttributeCleaner
       ALLOWED_REFERENCES = [
-        *ProjectRelationFactory::PROJECT_REFERENCES,
-        *ProjectRelationFactory::USER_REFERENCES,
+        *Gitlab::ImportExport::Project::RelationFactory::PROJECT_REFERENCES,
+        *Gitlab::ImportExport::Project::RelationFactory::USER_REFERENCES,
         'group_id',
         'commit_id',
         'discussion_id',

lib/gitlab/import_export/base/object_builder.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module ImportExport
+    module Base
+      # Base class for Group & Project Object Builders.
+      # This class is not intended to be used on its own but
+      # rather inherited from.
+      #
+      # Cache keeps 1000 entries at most, 1000 is chosen based on:
+      #    - one cache entry uses around 0.5K memory, 1000 items uses around 500K.
+      #      (leave some buffer it should be less than 1M). It is afforable cost for project import.
+      #    - for projects in Gitlab.com, it seems 1000 entries for labels/milestones is enough.
+      #      For example, gitlab has ~970 labels and 26 milestones.
+      LRU_CACHE_SIZE = 1000
+
+      class ObjectBuilder
+        def self.build(*args)
+          new(*args).find
+        end
+
+        def initialize(klass, attributes)
+          @klass = klass.ancestors.include?(Label) ? Label : klass
+          @attributes = attributes
+
+          if Gitlab::SafeRequestStore.active?
+            @lru_cache = cache_from_request_store
+            @cache_key = [klass, attributes]
+          end
+        end
+
+        def find
+          find_with_cache do
+            find_object || klass.create(prepare_attributes)
+          end
+        end
+
+        protected
+
+        def where_clauses
+          raise NotImplementedError
+        end
+
+        # attributes wrapped in a method to be
+        # adjusted in sub-class if needed
+        def prepare_attributes
+          attributes
+        end
+
+        private
+
+        attr_reader :klass, :attributes, :lru_cache, :cache_key
+
+        def find_with_cache
+          return yield unless lru_cache && cache_key
+
+          lru_cache[cache_key] ||= yield
+        end
+
+        def cache_from_request_store
+          Gitlab::SafeRequestStore[:lru_cache] ||= LruRedux::Cache.new(LRU_CACHE_SIZE)
+        end
+
+        def find_object
+          klass.where(where_clause).first
+        end
+
+        def where_clause
+          where_clauses.reduce(:and)
+        end
+
+        def table
+          @table ||= klass.arel_table
+        end
+
+        # Returns Arel clause:
+        # `"{table_name}"."{attrs.keys[0]}" = '{attrs.values[0]} AND {table_name}"."{attrs.keys[1]}" = '{attrs.values[1]}"`
+        # from the given Hash of attributes.
+        def attrs_to_arel(attrs)
+          attrs.map do |key, value|
+            table[key].eq(value)
+          end.reduce(:and)
+        end
+
+        # Returns Arel clause `"{table_name}"."title" = '{attributes['title']}'`
+        # if attributes has 'title key, otherwise `nil`.
+        def where_clause_for_title
+          attrs_to_arel(attributes.slice('title'))
+        end
+
+        # Returns Arel clause `"{table_name}"."description" = '{attributes['description']}'`
+        # if attributes has 'description key, otherwise `nil`.
+        def where_clause_for_description
+          attrs_to_arel(attributes.slice('description'))
+        end
+
+        # Returns Arel clause `"{table_name}"."created_at" = '{attributes['created_at']}'`
+        # if attributes has 'created_at key, otherwise `nil`.
+        def where_clause_for_created_at
+          attrs_to_arel(attributes.slice('created_at'))
+        end
+      end
+    end
+  end
+end

lib/gitlab/import_export/base_object_builder.rb

-# frozen_string_literal: true
-
-module Gitlab
-  module ImportExport
-    # Base class for Group & Project Object Builders.
-    # This class is not intended to be used on its own but
-    # rather inherited from.
-    #
-    # Cache keeps 1000 entries at most, 1000 is chosen based on:
-    #    - one cache entry uses around 0.5K memory, 1000 items uses around 500K.
-    #      (leave some buffer it should be less than 1M). It is afforable cost for project import.
-    #    - for projects in Gitlab.com, it seems 1000 entries for labels/milestones is enough.
-    #      For example, gitlab has ~970 labels and 26 milestones.
-    LRU_CACHE_SIZE = 1000
-
-    class BaseObjectBuilder
-      def self.build(*args)
-        new(*args).find
-      end
-
-      def initialize(klass, attributes)
-        @klass = klass.ancestors.include?(Label) ? Label : klass
-        @attributes = attributes
-
-        if Gitlab::SafeRequestStore.active?
-          @lru_cache = cache_from_request_store
-          @cache_key = [klass, attributes]
-        end
-      end
-
-      def find
-        find_with_cache do
-          find_object || klass.create(prepare_attributes)
-        end
-      end
-
-      protected
-
-      def where_clauses
-        raise NotImplementedError
-      end
-
-      # attributes wrapped in a method to be
-      # adjusted in sub-class if needed
-      def prepare_attributes
-        attributes
-      end
-
-      private
-
-      attr_reader :klass, :attributes, :lru_cache, :cache_key
-
-      def find_with_cache
-        return yield unless lru_cache && cache_key
-
-        lru_cache[cache_key] ||= yield
-      end
-
-      def cache_from_request_store
-        Gitlab::SafeRequestStore[:lru_cache] ||= LruRedux::Cache.new(LRU_CACHE_SIZE)
-      end
-
-      def find_object
-        klass.where(where_clause).first
-      end
-
-      def where_clause
-        where_clauses.reduce(:and)
-      end
-
-      def table
-        @table ||= klass.arel_table
-      end
-
-      # Returns Arel clause:
-      # `"{table_name}"."{attrs.keys[0]}" = '{attrs.values[0]} AND {table_name}"."{attrs.keys[1]}" = '{attrs.values[1]}"`
-      # from the given Hash of attributes.
-      def attrs_to_arel(attrs)
-        attrs.map do |key, value|
-          table[key].eq(value)
-        end.reduce(:and)
-      end
-
-      # Returns Arel clause `"{table_name}"."title" = '{attributes['title']}'`
-      # if attributes has 'title key, otherwise `nil`.
-      def where_clause_for_title
-        attrs_to_arel(attributes.slice('title'))
-      end
-
-      # Returns Arel clause `"{table_name}"."description" = '{attributes['description']}'`
-      # if attributes has 'description key, otherwise `nil`.
-      def where_clause_for_description
-        attrs_to_arel(attributes.slice('description'))
-      end
-
-      # Returns Arel clause `"{table_name}"."created_at" = '{attributes['created_at']}'`
-      # if attributes has 'created_at key, otherwise `nil`.
-      def where_clause_for_created_at
-        attrs_to_arel(attributes.slice('created_at'))
-      end
-    end
-  end
-end

lib/gitlab/import_export/group/object_builder.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module ImportExport
+    module Group
+      # Given a class, it finds or creates a new object at group level.
+      #
+      # Example:
+      #   `Group::ObjectBuilder.build(Label, label_attributes)`
+      #    finds or initializes a label with the given attributes.
+      class ObjectBuilder < Base::ObjectBuilder
+        def self.build(*args)
+          ::Group.transaction do
+            super
+          end
+        end
+
+        def initialize(klass, attributes)
+          super
+
+          @group = @attributes['group']
+
+          update_description
+        end
+
+        private
+
+        attr_reader :group
+
+        # Convert description empty string to nil
+        # due to existing object being saved with description: nil
+        # Which makes object lookup to fail since nil != ''
+        def update_description
+          attributes['description'] = nil if attributes['description'] == ''
+        end
+
+        def where_clauses
+          [
+            where_clause_base,
+            where_clause_for_title,
+            where_clause_for_description,
+            where_clause_for_created_at
+          ].compact
+        end
+
+        # Returns Arel clause `"{table_name}"."group_id" = {group.id}`
+        def where_clause_base
+          table[:group_id].in(group_and_ancestor_ids)
+        end
+
+        def group_and_ancestor_ids
+          group.ancestors.map(&:id) << group.id
+        end
+      end
+    end
+  end
+end

lib/gitlab/import_export/group/relation_factory.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module ImportExport
+    module Group
+      class RelationFactory < Base::RelationFactory
+        OVERRIDES = {
+          labels:     :group_labels,
+          priorities: :label_priorities,
+          label:      :group_label,
+          parent:     :epic
+        }.freeze
+
+        EXISTING_OBJECT_RELATIONS = %i[
+          epic
+          epics
+          milestone
+          milestones
+          label
+          labels
+          group_label
+          group_labels
+        ].freeze
+
+        private
+
+        def setup_models
+          setup_note if @relation_name == :notes
+
+          update_group_references
+        end
+
+        def update_group_references
+          return unless self.class.existing_object_relations.include?(@relation_name)
+          return unless @relation_hash['group_id']
+
+          @relation_hash['group_id'] = @importable.id
+        end
+      end
+    end
+  end
+end

lib/gitlab/import_export/group/tree_restorer.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module ImportExport
+    module Group
+      class TreeRestorer
+        attr_reader :user
+        attr_reader :shared
+        attr_reader :group
+
+        def initialize(user:, shared:, group:, group_hash:)
+          @path = File.join(shared.export_path, 'group.json')
+          @user = user
+          @shared = shared
+          @group = group
+          @group_hash = group_hash
+        end
+
+        def restore
+          @tree_hash = @group_hash || read_tree_hash
+          @group_members = @tree_hash.delete('members')
+          @children = @tree_hash.delete('children')
+
+          if members_mapper.map && restorer.restore
+            @children&.each do |group_hash|
+              group = create_group(group_hash: group_hash, parent_group: @group)
+              shared = Gitlab::ImportExport::Shared.new(group)
+
+              self.class.new(
+                user: @user,
+                shared: shared,
+                group: group,
+                group_hash: group_hash
+              ).restore
+            end
+          end
+
+          return false if @shared.errors.any?
+
+          true
+        rescue => e
+          @shared.error(e)
+          false
+        end
+
+        private
+
+        def read_tree_hash
+          json = IO.read(@path)
+          ActiveSupport::JSON.decode(json)
+        rescue => e
+          @shared.logger.error(
+            group_id:   @group.id,
+            group_name: @group.name,
+            message:    "Import/Export error: #{e.message}"
+          )
+
+          raise Gitlab::ImportExport::Error.new('Incorrect JSON format')
+        end
+
+        def restorer
+          @relation_tree_restorer ||= RelationTreeRestorer.new(
+            user:             @user,
+            shared:           @shared,
+            importable:       @group,
+            tree_hash:        @tree_hash.except('name', 'path'),
+            members_mapper:   members_mapper,
+            object_builder:   object_builder,
+            relation_factory: relation_factory,
+            reader:           reader
+          )
+        end
+
+        def create_group(group_hash:, parent_group:)
+          group_params = {
+            name:      group_hash['name'],
+            path:      group_hash['path'],
+            parent_id: parent_group&.id,
+            visibility_level: sub_group_visibility_level(group_hash, parent_group)
+          }
+
+          ::Groups::CreateService.new(@user, group_params).execute
+        end
+
+        def sub_group_visibility_level(group_hash, parent_group)
+          original_visibility_level = group_hash['visibility_level'] || Gitlab::VisibilityLevel::PRIVATE
+
+          if parent_group && parent_group.visibility_level < original_visibility_level
+            Gitlab::VisibilityLevel.closest_allowed_level(parent_group.visibility_level)
+          else
+            original_visibility_level
+          end
+        end
+
+        def members_mapper
+          @members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @group_members, user: @user, importable: @group)
+        end
+
+        def relation_factory
+          Gitlab::ImportExport::Group::RelationFactory
+        end
+
+        def object_builder
+          Gitlab::ImportExport::Group::ObjectBuilder
+        end
+
+        def reader
+          @reader ||= Gitlab::ImportExport::Reader.new(
+            shared: @shared,
+            config: Gitlab::ImportExport::Config.new(
+              config: Gitlab::ImportExport.group_config_file
+            ).to_h
+          )
+        end
+      end
+    end
+  end
+end

lib/gitlab/import_export/group/tree_saver.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module ImportExport
+    module Group
+      class TreeSaver
+        attr_reader :full_path, :shared
+
+        def initialize(group:, current_user:, shared:, params: {})
+          @params       = params
+          @current_user = current_user
+          @shared       = shared
+          @group        = group
+          @full_path    = File.join(@shared.export_path, ImportExport.group_filename)
+        end
+
+        def save
+          group_tree = serialize(@group, reader.group_tree)
+          tree_saver.save(group_tree, @shared.export_path, ImportExport.group_filename)
+
+          true
+        rescue => e
+          @shared.error(e)
+          false
+        end
+
+        private
+
+        def serialize(group, relations_tree)
+          group_tree = tree_saver.serialize(group, relations_tree)
+
+          group.children.each do |child|
+            group_tree['children'] ||= []
+            group_tree['children'] << serialize(child, relations_tree)
+          end
+
+          group_tree
+        rescue => e
+          @shared.error(e)
+        end
+
+        def reader
+          @reader ||= Gitlab::ImportExport::Reader.new(
+            shared: @shared,
+            config: Gitlab::ImportExport::Config.new(
+              config: Gitlab::ImportExport.group_config_file
+            ).to_h
+          )
+        end
+
+        def tree_saver
+          @tree_saver ||= RelationTreeSaver.new
+        end
+      end
+    end
+  end
+end

lib/gitlab/import_export/group_object_builder.rb

-# frozen_string_literal: true
-
-module Gitlab
-  module ImportExport
-    # Given a class, it finds or creates a new object at group level.
-    #
-    # Example:
-    #   `GroupObjectBuilder.build(Label, label_attributes)`
-    #    finds or initializes a label with the given attributes.
-    class GroupObjectBuilder < BaseObjectBuilder
-      def self.build(*args)
-        Group.transaction do
-          super
-        end
-      end
-
-      def initialize(klass, attributes)
-        super
-
-        @group = @attributes['group']
-
-        update_description
-      end
-
-      private
-
-      attr_reader :group
-
-      # Convert description empty string to nil
-      # due to existing object being saved with description: nil
-      # Which makes object lookup to fail since nil != ''
-      def update_description
-        attributes['description'] = nil if attributes['description'] == ''
-      end
-
-      def where_clauses
-        [
-          where_clause_base,
-          where_clause_for_title,
-          where_clause_for_description,
-          where_clause_for_created_at
-        ].compact
-      end
-
-      # Returns Arel clause `"{table_name}"."group_id" = {group.id}`
-      def where_clause_base
-        table[:group_id].in(group_and_ancestor_ids)
-      end
-
-      def group_and_ancestor_ids
-        group.ancestors.map(&:id) << group.id
-      end
-    end
-  end
-end

lib/gitlab/import_export/group_project_object_builder.rb

-# frozen_string_literal: true
-
-module Gitlab
-  module ImportExport
-    # Given a class, it finds or creates a new object
-    # (initializes in the case of Label) at group or project level.
-    # If it does not exist in the group, it creates it at project level.
-    #
-    # Example:
-    #   `GroupProjectObjectBuilder.build(Label, label_attributes)`
-    #    finds or initializes a label with the given attributes.
-    #
-    # It also adds some logic around Group Labels/Milestones for edge cases.
-    class GroupProjectObjectBuilder < BaseObjectBuilder
-      def self.build(*args)
-        Project.transaction do
-          super
-        end
-      end
-
-      def initialize(klass, attributes)
-        super
-
-        @group = @attributes['group']
-        @project = @attributes['project']
-      end
-
-      def find
-        return if epic? && group.nil?
-
-        super
-      end
-
-      private
-
-      attr_reader :group, :project
-
-      def where_clauses
-        [
-          where_clause_base,
-          where_clause_for_title,
-          where_clause_for_klass
-        ].compact
-      end
-
-      # Returns Arel clause `"{table_name}"."project_id" = {project.id}` if project is present
-      # For example: merge_request has :target_project_id, and we are searching by :iid
-      # or, if group is present:
-      # `"{table_name}"."project_id" = {project.id} OR "{table_name}"."group_id" = {group.id}`
-      def where_clause_base
-        [].tap do |clauses|
-          clauses << table[:project_id].eq(project.id) if project
-          clauses << table[:group_id].in(group.self_and_ancestors_ids) if group
-        end.reduce(:or)
-      end
-
-      # Returns Arel clause for a particular model or `nil`.
-      def where_clause_for_klass
-        attrs_to_arel(attributes.slice('iid')) if merge_request?
-      end
-
-      def prepare_attributes
-        attributes.dup.tap do |atts|
-          atts.delete('group') unless epic?
-
-          if label?
-            atts['type'] = 'ProjectLabel' # Always create project labels
-          elsif milestone?
-            if atts['group_id'] # Transform new group milestones into project ones
-              atts['iid'] = nil
-              atts.delete('group_id')
-            else
-              claim_iid
-            end
-          end
-
-          atts['importing'] = true if klass.ancestors.include?(Importable)
-        end
-      end
-
-      def label?
-        klass == Label
-      end
-
-      def milestone?
-        klass == Milestone
-      end
-
-      def merge_request?
-        klass == MergeRequest
-      end
-
-      def epic?
-        klass == Epic
-      end
-
-      # If an existing group milestone used the IID
-      # claim the IID back and set the group milestone to use one available
-      # This is necessary to fix situations like the following:
-      #  - Importing into a user namespace project with exported group milestones
-      #    where the IID of the Group milestone could conflict with a project one.
-      def claim_iid
-        # The milestone has to be a group milestone, as it's the only case where
-        # we set the IID as the maximum. The rest of them are fixed.
-        milestone = project.milestones.find_by(iid: attributes['iid'])
-
-        return unless milestone
-
-        milestone.iid = nil
-        milestone.ensure_project_iid!
-        milestone.save!
-      end
-    end
-  end
-end
-
-Gitlab::ImportExport::GroupProjectObjectBuilder.prepend_if_ee('EE::Gitlab::ImportExport::GroupProjectObjectBuilder')

lib/gitlab/import_export/group_relation_factory.rb

-# frozen_string_literal: true
-
-module Gitlab
-  module ImportExport
-    class GroupRelationFactory < BaseRelationFactory
-      OVERRIDES = {
-        labels:     :group_labels,
-        priorities: :label_priorities,
-        label:      :group_label,
-        parent:     :epic
-      }.freeze
-
-      EXISTING_OBJECT_RELATIONS = %i[
-        epic
-        epics
-        milestone
-        milestones
-        label
-        labels
-        group_label
-        group_labels
-      ].freeze
-
-      private
-
-      def setup_models
-        setup_note if @relation_name == :notes
-
-        update_group_references
-      end
-
-      def update_group_references
-        return unless self.class.existing_object_relations.include?(@relation_name)
-        return unless @relation_hash['group_id']
-
-        @relation_hash['group_id'] = @importable.id
-      end
-    end
-  end
-end

lib/gitlab/import_export/group_tree_restorer.rb

-# frozen_string_literal: true
-
-module Gitlab
-  module ImportExport
-    class GroupTreeRestorer
-      attr_reader :user
-      attr_reader :shared
-      attr_reader :group
-
-      def initialize(user:, shared:, group:, group_hash:)
-        @path = File.join(shared.export_path, 'group.json')
-        @user = user
-        @shared = shared
-        @group = group
-        @group_hash = group_hash
-      end
-
-      def restore
-        @tree_hash = @group_hash || read_tree_hash
-        @group_members = @tree_hash.delete('members')
-        @children = @tree_hash.delete('children')
-
-        if members_mapper.map && restorer.restore
-          @children&.each do |group_hash|
-            group = create_group(group_hash: group_hash, parent_group: @group)
-            shared = Gitlab::ImportExport::Shared.new(group)
-
-            self.class.new(
-              user: @user,
-              shared: shared,
-              group: group,
-              group_hash: group_hash
-            ).restore
-          end
-        end
-
-        return false if @shared.errors.any?
-
-        true
-      rescue => e
-        @shared.error(e)
-        false
-      end
-
-      private
-
-      def read_tree_hash
-        json = IO.read(@path)
-        ActiveSupport::JSON.decode(json)
-      rescue => e
-        @shared.logger.error(
-          group_id:   @group.id,
-          group_name: @group.name,
-          message:    "Import/Export error: #{e.message}"
-        )
-
-        raise Gitlab::ImportExport::Error.new('Incorrect JSON format')
-      end
-
-      def restorer
-        @relation_tree_restorer ||= RelationTreeRestorer.new(
-          user:             @user,
-          shared:           @shared,
-          importable:       @group,
-          tree_hash:        @tree_hash.except('name', 'path'),
-          members_mapper:   members_mapper,
-          object_builder:   object_builder,
-          relation_factory: relation_factory,
-          reader:           reader
-        )
-      end
-
-      def create_group(group_hash:, parent_group:)
-        group_params = {
-          name:      group_hash['name'],
-          path:      group_hash['path'],
-          parent_id: parent_group&.id,
-          visibility_level: sub_group_visibility_level(group_hash, parent_group)
-        }
-
-        ::Groups::CreateService.new(@user, group_params).execute
-      end
-
-      def sub_group_visibility_level(group_hash, parent_group)
-        original_visibility_level = group_hash['visibility_level'] || Gitlab::VisibilityLevel::PRIVATE
-
-        if parent_group && parent_group.visibility_level < original_visibility_level
-          Gitlab::VisibilityLevel.closest_allowed_level(parent_group.visibility_level)
-        else
-          original_visibility_level
-        end
-      end
-
-      def members_mapper
-        @members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @group_members, user: @user, importable: @group)
-      end
-
-      def relation_factory
-        Gitlab::ImportExport::GroupRelationFactory
-      end
-
-      def object_builder
-        Gitlab::ImportExport::GroupObjectBuilder
-      end
-
-      def reader
-        @reader ||= Gitlab::ImportExport::Reader.new(
-          shared: @shared,
-          config: Gitlab::ImportExport::Config.new(
-            config: Gitlab::ImportExport.group_config_file
-          ).to_h
-        )
-      end
-    end
-  end
-end

lib/gitlab/import_export/group_tree_saver.rb

-# frozen_string_literal: true
-
-module Gitlab
-  module ImportExport
-    class GroupTreeSaver
-      attr_reader :full_path, :shared
-
-      def initialize(group:, current_user:, shared:, params: {})
-        @params       = params
-        @current_user = current_user
-        @shared       = shared
-        @group        = group
-        @full_path    = File.join(@shared.export_path, ImportExport.group_filename)
-      end
-
-      def save
-        group_tree = serialize(@group, reader.group_tree)
-        tree_saver.save(group_tree, @shared.export_path, ImportExport.group_filename)
-
-        true
-      rescue => e
-        @shared.error(e)
-        false
-      end
-
-      private
-
-      def serialize(group, relations_tree)
-        group_tree = tree_saver.serialize(group, relations_tree)
-
-        group.children.each do |child|
-          group_tree['children'] ||= []
-          group_tree['children'] << serialize(child, relations_tree)
-        end
-
-        group_tree
-      rescue => e
-        @shared.error(e)
-      end
-
-      def reader
-        @reader ||= Gitlab::ImportExport::Reader.new(
-          shared: @shared,
-          config: Gitlab::ImportExport::Config.new(
-            config: Gitlab::ImportExport.group_config_file
-          ).to_h
-        )
-      end
-
-      def tree_saver
-        @tree_saver ||= RelationTreeSaver.new
-      end
-    end
-  end
-end

lib/gitlab/import_export/importer.rb

@@ -49,7 +49,7 @@ module Gitlab
       end
 
       def project_tree
-        @project_tree ||= Gitlab::ImportExport::ProjectTreeRestorer.new(user: current_user,
+        @project_tree ||= Gitlab::ImportExport::Project::TreeRestorer.new(user: current_user,
                                                                         shared: shared,
                                                                         project: project)
       end
@@ -125,7 +125,7 @@ module Gitlab
 
       def project_to_overwrite
         strong_memoize(:project_to_overwrite) do
-          Project.find_by_full_path("#{project.namespace.full_path}/#{original_path}")
+          ::Project.find_by_full_path("#{project.namespace.full_path}/#{original_path}")
         end
       end
     end

lib/gitlab/import_export/members_mapper.rb

@@ -91,9 +91,9 @@ module Gitlab
 
       def relation_class
         case @importable
-        when Project
+        when ::Project
           ProjectMember
-        when Group
+        when ::Group
           GroupMember
         end
       end

lib/gitlab/import_export/project/object_builder.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module ImportExport
+    module Project
+      # Given a class, it finds or creates a new object
+      # (initializes in the case of Label) at group or project level.
+      # If it does not exist in the group, it creates it at project level.
+      #
+      # Example:
+      #   `ObjectBuilder.build(Label, label_attributes)`
+      #    finds or initializes a label with the given attributes.
+      #
+      # It also adds some logic around Group Labels/Milestones for edge cases.
+      class ObjectBuilder < Base::ObjectBuilder
+        def self.build(*args)
+          ::Project.transaction do
+            super
+          end
+        end
+
+        def initialize(klass, attributes)
+          super
+
+          @group = @attributes['group']
+          @project = @attributes['project']
+        end
+
+        def find
+          return if epic? && group.nil?
+
+          super
+        end
+
+        private
+
+        attr_reader :group, :project
+
+        def where_clauses
+          [
+            where_clause_base,
+            where_clause_for_title,
+            where_clause_for_klass
+          ].compact
+        end
+
+        # Returns Arel clause `"{table_name}"."project_id" = {project.id}` if project is present
+        # For example: merge_request has :target_project_id, and we are searching by :iid
+        # or, if group is present:
+        # `"{table_name}"."project_id" = {project.id} OR "{table_name}"."group_id" = {group.id}`
+        def where_clause_base
+          [].tap do |clauses|
+            clauses << table[:project_id].eq(project.id) if project
+            clauses << table[:group_id].in(group.self_and_ancestors_ids) if group
+          end.reduce(:or)
+        end
+
+        # Returns Arel clause for a particular model or `nil`.
+        def where_clause_for_klass
+          attrs_to_arel(attributes.slice('iid')) if merge_request?
+        end
+
+        def prepare_attributes
+          attributes.dup.tap do |atts|
+            atts.delete('group') unless epic?
+
+            if label?
+              atts['type'] = 'ProjectLabel' # Always create project labels
+            elsif milestone?
+              if atts['group_id'] # Transform new group milestones into project ones
+                atts['iid'] = nil
+                atts.delete('group_id')
+              else
+                claim_iid
+              end
+            end
+
+            atts['importing'] = true if klass.ancestors.include?(Importable)
+          end
+        end
+
+        def label?
+          klass == Label
+        end
+
+        def milestone?
+          klass == Milestone
+        end
+
+        def merge_request?
+          klass == MergeRequest
+        end
+
+        def epic?
+          klass == Epic
+        end
+
+        # If an existing group milestone used the IID
+        # claim the IID back and set the group milestone to use one available
+        # This is necessary to fix situations like the following:
+        #  - Importing into a user namespace project with exported group milestones
+        #    where the IID of the Group milestone could conflict with a project one.
+        def claim_iid
+          # The milestone has to be a group milestone, as it's the only case where
+          # we set the IID as the maximum. The rest of them are fixed.
+          milestone = project.milestones.find_by(iid: attributes['iid'])
+
+          return unless milestone
+
+          milestone.iid = nil
+          milestone.ensure_project_iid!
+          milestone.save!
+        end
+      end
+    end
+  end
+end
+
+Gitlab::ImportExport::Project::ObjectBuilder.prepend_if_ee('EE::Gitlab::ImportExport::Project::ObjectBuilder')

lib/gitlab/import_export/project/relation_factory.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module ImportExport
+    module Project
+      class RelationFactory < Base::RelationFactory
+        prepend_if_ee('::EE::Gitlab::ImportExport::Project::RelationFactory') # rubocop: disable Cop/InjectEnterpriseEditionModule
+
+        OVERRIDES = { snippets: :project_snippets,
+                      ci_pipelines: 'Ci::Pipeline',
+                      pipelines: 'Ci::Pipeline',
+                      stages: 'Ci::Stage',
+                      statuses: 'commit_status',
+                      triggers: 'Ci::Trigger',
+                      pipeline_schedules: 'Ci::PipelineSchedule',
+                      builds: 'Ci::Build',
+                      runners: 'Ci::Runner',
+                      hooks: 'ProjectHook',
+                      merge_access_levels: 'ProtectedBranch::MergeAccessLevel',
+                      push_access_levels: 'ProtectedBranch::PushAccessLevel',
+                      create_access_levels: 'ProtectedTag::CreateAccessLevel',
+                      labels: :project_labels,
+                      priorities: :label_priorities,
+                      auto_devops: :project_auto_devops,
+                      label: :project_label,
+                      custom_attributes: 'ProjectCustomAttribute',
+                      project_badges: 'Badge',
+                      metrics: 'MergeRequest::Metrics',
+                      ci_cd_settings: 'ProjectCiCdSetting',
+                      error_tracking_setting: 'ErrorTracking::ProjectErrorTrackingSetting',
+                      links: 'Releases::Link',
+                      metrics_setting: 'ProjectMetricsSetting' }.freeze
+
+        BUILD_MODELS = %i[Ci::Build commit_status].freeze
+
+        GROUP_REFERENCES = %w[group_id].freeze
+
+        PROJECT_REFERENCES = %w[project_id source_project_id target_project_id].freeze
+
+        EXISTING_OBJECT_RELATIONS = %i[
+          milestone
+          milestones
+          label
+          labels
+          project_label
+          project_labels
+          group_label
+          group_labels
+          project_feature
+          merge_request
+          epic
+          ProjectCiCdSetting
+          container_expiration_policy
+        ].freeze
+
+        def create
+          @object = super
+
+          # We preload the project, user, and group to re-use objects
+          @object = preload_keys(@object, PROJECT_REFERENCES, @importable)
+          @object = preload_keys(@object, GROUP_REFERENCES, @importable.group)
+          @object = preload_keys(@object, USER_REFERENCES, @user)
+        end
+
+        private
+
+        def invalid_relation?
+          # Do not create relation if it is:
+          #   - An unknown service
+          #   - A legacy trigger
+          unknown_service? ||
+            (!Feature.enabled?(:use_legacy_pipeline_triggers, @importable) && legacy_trigger?)
+        end
+
+        def setup_models
+          case @relation_name
+          when :merge_request_diff_files then setup_diff
+          when :notes then setup_note
+          when :'Ci::Pipeline' then setup_pipeline
+          when *BUILD_MODELS then setup_build
+          end
+
+          update_project_references
+          update_group_references
+        end
+
+        def generate_imported_object
+          if @relation_name == :merge_requests
+            MergeRequestParser.new(@importable, @relation_hash.delete('diff_head_sha'), super, @relation_hash).parse!
+          else
+            super
+          end
+        end
+
+        def update_project_references
+          # If source and target are the same, populate them with the new project ID.
+          if @relation_hash['source_project_id']
+            @relation_hash['source_project_id'] = same_source_and_target? ? @relation_hash['project_id'] : MergeRequestParser::FORKED_PROJECT_ID
+          end
+
+          @relation_hash['target_project_id'] = @relation_hash['project_id'] if @relation_hash['target_project_id']
+        end
+
+        def same_source_and_target?
+          @relation_hash['target_project_id'] && @relation_hash['target_project_id'] == @relation_hash['source_project_id']
+        end
+
+        def update_group_references
+          return unless existing_object?
+          return unless @relation_hash['group_id']
+
+          @relation_hash['group_id'] = @importable.namespace_id
+        end
+
+        def setup_build
+          @relation_hash.delete('trace') # old export files have trace
+          @relation_hash.delete('token')
+          @relation_hash.delete('commands')
+          @relation_hash.delete('artifacts_file_store')
+          @relation_hash.delete('artifacts_metadata_store')
+          @relation_hash.delete('artifacts_size')
+        end
+
+        def setup_diff
+          @relation_hash['diff'] = @relation_hash.delete('utf8_diff')
+        end
+
+        def setup_pipeline
+          @relation_hash.fetch('stages', []).each do |stage|
+            stage.statuses.each do |status|
+              status.pipeline = imported_object
+            end
+          end
+        end
+
+        def unknown_service?
+          @relation_name == :services && parsed_relation_hash['type'] &&
+            !Object.const_defined?(parsed_relation_hash['type'])
+        end
+
+        def legacy_trigger?
+          @relation_name == :'Ci::Trigger' && @relation_hash['owner_id'].nil?
+        end
+
+        def preload_keys(object, references, value)
+          return object unless value
+
+          references.each do |key|
+            attribute = "#{key.delete_suffix('_id')}=".to_sym
+            next unless object.respond_to?(key) && object.respond_to?(attribute)
+
+            if object.read_attribute(key) == value&.id
+              object.public_send(attribute, value) # rubocop:disable GitlabSecurity/PublicSend
+            end
+          end
+
+          object
+        end
+      end
+    end
+  end
+end

lib/gitlab/import_export/project/tree_loader.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module ImportExport
+    module Project
+      class TreeLoader
+        def load(path, dedup_entries: false)
+          tree_hash = ActiveSupport::JSON.decode(IO.read(path))
+
+          if dedup_entries
+            dedup_tree(tree_hash)
+          else
+            tree_hash
+          end
+        end
+
+        private
+
+        # This function removes duplicate entries from the given tree recursively
+        # by caching nodes it encounters repeatedly. We only consider nodes for
+        # which there can actually be multiple equivalent instances (e.g. strings,
+        # hashes and arrays, but not `nil`s, numbers or booleans.)
+        #
+        # The algorithm uses a recursive depth-first descent with 3 cases, starting
+        # with a root node (the tree/hash itself):
+        # - a node has already been cached; in this case we return it from the cache
+        # - a node has not been cached yet but should be; descend into its children
+        # - a node is neither cached nor qualifies for caching; this is a no-op
+        def dedup_tree(node, nodes_seen = {})
+          if nodes_seen.key?(node) && distinguishable?(node)
+            yield nodes_seen[node]
+          elsif should_dedup?(node)
+            nodes_seen[node] = node
+
+            case node
+            when Array
+              node.each_index do |idx|
+                dedup_tree(node[idx], nodes_seen) do |cached_node|
+                  node[idx] = cached_node
+                end
+              end
+            when Hash
+              node.each do |k, v|
+                dedup_tree(v, nodes_seen) do |cached_node|
+                  node[k] = cached_node
+                end
+              end
+            end
+          else
+            node
+          end
+        end
+
+        # We do not need to consider nodes for which there cannot be multiple instances
+        def should_dedup?(node)
+          node && !(node.is_a?(Numeric) || node.is_a?(TrueClass) || node.is_a?(FalseClass))
+        end
+
+        # We can only safely de-dup values that are distinguishable. True value objects
+        # are always distinguishable by nature. Hashes however can represent entities,
+        # which are identified by ID, not value. We therefore disallow de-duping hashes
+        # that do not have an `id` field, since we might risk dropping entities that
+        # have equal attributes yet different identities.
+        def distinguishable?(node)
+          if node.is_a?(Hash)
+            node.key?('id')
+          else
+            true
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/import_export/project/tree_restorer.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module ImportExport
+    module Project
+      class TreeRestorer
+        LARGE_PROJECT_FILE_SIZE_BYTES = 500.megabyte
+
+        attr_reader :user
+        attr_reader :shared
+        attr_reader :project
+
+        def initialize(user:, shared:, project:)
+          @user = user
+          @shared = shared
+          @project = project
+          @tree_loader = TreeLoader.new
+        end
+
+        def restore
+          @tree_hash = read_tree_hash
+          @project_members = @tree_hash.delete('project_members')
+
+          RelationRenameService.rename(@tree_hash)
+
+          if relation_tree_restorer.restore
+            import_failure_service.with_retry(action: 'set_latest_merge_request_diff_ids!') do
+              @project.merge_requests.set_latest_merge_request_diff_ids!
+            end
+
+            true
+          else
+            false
+          end
+        rescue => e
+          @shared.error(e)
+          false
+        end
+
+        private
+
+        def large_project?(path)
+          File.size(path) >= LARGE_PROJECT_FILE_SIZE_BYTES
+        end
+
+        def read_tree_hash
+          path = File.join(@shared.export_path, 'project.json')
+          dedup_entries = large_project?(path) &&
+            Feature.enabled?(:dedup_project_import_metadata, project.group)
+
+          @tree_loader.load(path, dedup_entries: dedup_entries)
+        rescue => e
+          Rails.logger.error("Import/Export error: #{e.message}") # rubocop:disable Gitlab/RailsLogger
+          raise Gitlab::ImportExport::Error.new('Incorrect JSON format')
+        end
+
+        def relation_tree_restorer
+          @relation_tree_restorer ||= RelationTreeRestorer.new(
+            user: @user,
+            shared: @shared,
+            importable: @project,
+            tree_hash: @tree_hash,
+            object_builder: object_builder,
+            members_mapper: members_mapper,
+            relation_factory: relation_factory,
+            reader: reader
+          )
+        end
+
+        def members_mapper
+          @members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @project_members,
+                                                                      user: @user,
+                                                                      importable: @project)
+        end
+
+        def object_builder
+          Project::ObjectBuilder
+        end
+
+        def relation_factory
+          Project::RelationFactory
+        end
+
+        def reader
+          @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared)
+        end
+
+        def import_failure_service
+          @import_failure_service ||= ImportFailureService.new(@project)
+        end
+      end
+    end
+  end
+end

lib/gitlab/import_export/project/tree_saver.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module ImportExport
+    module Project
+      class TreeSaver
+        attr_reader :full_path
+
+        def initialize(project:, current_user:, shared:, params: {})
+          @params       = params
+          @project      = project
+          @current_user = current_user
+          @shared       = shared
+          @full_path    = File.join(@shared.export_path, ImportExport.project_filename)
+        end
+
+        def save
+          project_tree = tree_saver.serialize(@project, reader.project_tree)
+          fix_project_tree(project_tree)
+          tree_saver.save(project_tree, @shared.export_path, ImportExport.project_filename)
+
+          true
+        rescue => e
+          @shared.error(e)
+          false
+        end
+
+        private
+
+        # Aware that the resulting hash needs to be pure-hash and
+        # does not include any AR objects anymore, only objects that run `.to_json`
+        def fix_project_tree(project_tree)
+          if @params[:description].present?
+            project_tree['description'] = @params[:description]
+          end
+
+          project_tree['project_members'] += group_members_array
+
+          RelationRenameService.add_new_associations(project_tree)
+        end
+
+        def reader
+          @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared)
+        end
+
+        def group_members_array
+          group_members.as_json(reader.group_members_tree).each do |group_member|
+            group_member['source_type'] = 'Project' # Make group members project members of the future import
+          end
+        end
+
+        def group_members
+          return [] unless @current_user.can?(:admin_group, @project.group)
+
+          # We need `.where.not(user_id: nil)` here otherwise when a group has an
+          # invitee, it would make the following query return 0 rows since a NULL
+          # user_id would be present in the subquery
+          # See http://stackoverflow.com/questions/129077/not-in-clause-and-null-values
+          non_null_user_ids = @project.project_members.where.not(user_id: nil).select(:user_id)
+
+          GroupMembersFinder.new(@project.group).execute.where.not(user_id: non_null_user_ids)
+        end
+
+        def tree_saver
+          @tree_saver ||= RelationTreeSaver.new
+        end
+      end
+    end
+  end
+end

lib/gitlab/import_export/project_relation_factory.rb

-# frozen_string_literal: true
-
-module Gitlab
-  module ImportExport
-    class ProjectRelationFactory < BaseRelationFactory
-      prepend_if_ee('::EE::Gitlab::ImportExport::ProjectRelationFactory') # rubocop: disable Cop/InjectEnterpriseEditionModule
-
-      OVERRIDES = { snippets: :project_snippets,
-                    ci_pipelines: 'Ci::Pipeline',
-                    pipelines: 'Ci::Pipeline',
-                    stages: 'Ci::Stage',
-                    statuses: 'commit_status',
-                    triggers: 'Ci::Trigger',
-                    pipeline_schedules: 'Ci::PipelineSchedule',
-                    builds: 'Ci::Build',
-                    runners: 'Ci::Runner',
-                    hooks: 'ProjectHook',
-                    merge_access_levels: 'ProtectedBranch::MergeAccessLevel',
-                    push_access_levels: 'ProtectedBranch::PushAccessLevel',
-                    create_access_levels: 'ProtectedTag::CreateAccessLevel',
-                    labels: :project_labels,
-                    priorities: :label_priorities,
-                    auto_devops: :project_auto_devops,
-                    label: :project_label,
-                    custom_attributes: 'ProjectCustomAttribute',
-                    project_badges: 'Badge',
-                    metrics: 'MergeRequest::Metrics',
-                    ci_cd_settings: 'ProjectCiCdSetting',
-                    error_tracking_setting: 'ErrorTracking::ProjectErrorTrackingSetting',
-                    links: 'Releases::Link',
-                    metrics_setting: 'ProjectMetricsSetting' }.freeze
-
-      BUILD_MODELS = %i[Ci::Build commit_status].freeze
-
-      GROUP_REFERENCES = %w[group_id].freeze
-
-      PROJECT_REFERENCES = %w[project_id source_project_id target_project_id].freeze
-
-      EXISTING_OBJECT_RELATIONS = %i[
-        milestone
-        milestones
-        label
-        labels
-        project_label
-        project_labels
-        group_label
-        group_labels
-        project_feature
-        merge_request
-        epic
-        ProjectCiCdSetting
-        container_expiration_policy
-      ].freeze
-
-      def create
-        @object = super
-
-        # We preload the project, user, and group to re-use objects
-        @object = preload_keys(@object, PROJECT_REFERENCES, @importable)
-        @object = preload_keys(@object, GROUP_REFERENCES, @importable.group)
-        @object = preload_keys(@object, USER_REFERENCES, @user)
-      end
-
-      private
-
-      def invalid_relation?
-        # Do not create relation if it is:
-        #   - An unknown service
-        #   - A legacy trigger
-        unknown_service? ||
-          (!Feature.enabled?(:use_legacy_pipeline_triggers, @importable) && legacy_trigger?)
-      end
-
-      def setup_models
-        case @relation_name
-        when :merge_request_diff_files then setup_diff
-        when :notes then setup_note
-        when :'Ci::Pipeline' then setup_pipeline
-        when *BUILD_MODELS then setup_build
-        end
-
-        update_project_references
-        update_group_references
-      end
-
-      def generate_imported_object
-        if @relation_name == :merge_requests
-          MergeRequestParser.new(@importable, @relation_hash.delete('diff_head_sha'), super, @relation_hash).parse!
-        else
-          super
-        end
-      end
-
-      def update_project_references
-        # If source and target are the same, populate them with the new project ID.
-        if @relation_hash['source_project_id']
-          @relation_hash['source_project_id'] = same_source_and_target? ? @relation_hash['project_id'] : MergeRequestParser::FORKED_PROJECT_ID
-        end
-
-        @relation_hash['target_project_id'] = @relation_hash['project_id'] if @relation_hash['target_project_id']
-      end
-
-      def same_source_and_target?
-        @relation_hash['target_project_id'] && @relation_hash['target_project_id'] == @relation_hash['source_project_id']
-      end
-
-      def update_group_references
-        return unless existing_object?
-        return unless @relation_hash['group_id']
-
-        @relation_hash['group_id'] = @importable.namespace_id
-      end
-
-      def setup_build
-        @relation_hash.delete('trace') # old export files have trace
-        @relation_hash.delete('token')
-        @relation_hash.delete('commands')
-        @relation_hash.delete('artifacts_file_store')
-        @relation_hash.delete('artifacts_metadata_store')
-        @relation_hash.delete('artifacts_size')
-      end
-
-      def setup_diff
-        @relation_hash['diff'] = @relation_hash.delete('utf8_diff')
-      end
-
-      def setup_pipeline
-        @relation_hash.fetch('stages', []).each do |stage|
-          stage.statuses.each do |status|
-            status.pipeline = imported_object
-          end
-        end
-      end
-
-      def unknown_service?
-        @relation_name == :services && parsed_relation_hash['type'] &&
-          !Object.const_defined?(parsed_relation_hash['type'])
-      end
-
-      def legacy_trigger?
-        @relation_name == :'Ci::Trigger' && @relation_hash['owner_id'].nil?
-      end
-
-      def preload_keys(object, references, value)
-        return object unless value
-
-        references.each do |key|
-          attribute = "#{key.delete_suffix('_id')}=".to_sym
-          next unless object.respond_to?(key) && object.respond_to?(attribute)
-
-          if object.read_attribute(key) == value&.id
-            object.public_send(attribute, value) # rubocop:disable GitlabSecurity/PublicSend
-          end
-        end
-
-        object
-      end
-    end
-  end
-end

lib/gitlab/import_export/project_tree_loader.rb

-# frozen_string_literal: true
-
-module Gitlab
-  module ImportExport
-    class ProjectTreeLoader
-      def load(path, dedup_entries: false)
-        tree_hash = ActiveSupport::JSON.decode(IO.read(path))
-
-        if dedup_entries
-          dedup_tree(tree_hash)
-        else
-          tree_hash
-        end
-      end
-
-      private
-
-      # This function removes duplicate entries from the given tree recursively
-      # by caching nodes it encounters repeatedly. We only consider nodes for
-      # which there can actually be multiple equivalent instances (e.g. strings,
-      # hashes and arrays, but not `nil`s, numbers or booleans.)
-      #
-      # The algorithm uses a recursive depth-first descent with 3 cases, starting
-      # with a root node (the tree/hash itself):
-      # - a node has already been cached; in this case we return it from the cache
-      # - a node has not been cached yet but should be; descend into its children
-      # - a node is neither cached nor qualifies for caching; this is a no-op
-      def dedup_tree(node, nodes_seen = {})
-        if nodes_seen.key?(node) && distinguishable?(node)
-          yield nodes_seen[node]
-        elsif should_dedup?(node)
-          nodes_seen[node] = node
-
-          case node
-          when Array
-            node.each_index do |idx|
-              dedup_tree(node[idx], nodes_seen) do |cached_node|
-                node[idx] = cached_node
-              end
-            end
-          when Hash
-            node.each do |k, v|
-              dedup_tree(v, nodes_seen) do |cached_node|
-                node[k] = cached_node
-              end
-            end
-          end
-        else
-          node
-        end
-      end
-
-      # We do not need to consider nodes for which there cannot be multiple instances
-      def should_dedup?(node)
-        node && !(node.is_a?(Numeric) || node.is_a?(TrueClass) || node.is_a?(FalseClass))
-      end
-
-      # We can only safely de-dup values that are distinguishable. True value objects
-      # are always distinguishable by nature. Hashes however can represent entities,
-      # which are identified by ID, not value. We therefore disallow de-duping hashes
-      # that do not have an `id` field, since we might risk dropping entities that
-      # have equal attributes yet different identities.
-      def distinguishable?(node)
-        if node.is_a?(Hash)
-          node.key?('id')
-        else
-          true
-        end
-      end
-    end
-  end
-end

lib/gitlab/import_export/project_tree_restorer.rb

-# frozen_string_literal: true
-
-module Gitlab
-  module ImportExport
-    class ProjectTreeRestorer
-      LARGE_PROJECT_FILE_SIZE_BYTES = 500.megabyte
-
-      attr_reader :user
-      attr_reader :shared
-      attr_reader :project
-
-      def initialize(user:, shared:, project:)
-        @user = user
-        @shared = shared
-        @project = project
-        @tree_loader = ProjectTreeLoader.new
-      end
-
-      def restore
-        @tree_hash = read_tree_hash
-        @project_members = @tree_hash.delete('project_members')
-
-        RelationRenameService.rename(@tree_hash)
-
-        if relation_tree_restorer.restore
-          import_failure_service.with_retry(action: 'set_latest_merge_request_diff_ids!') do
-            @project.merge_requests.set_latest_merge_request_diff_ids!
-          end
-
-          true
-        else
-          false
-        end
-      rescue => e
-        @shared.error(e)
-        false
-      end
-
-      private
-
-      def large_project?(path)
-        File.size(path) >= LARGE_PROJECT_FILE_SIZE_BYTES
-      end
-
-      def read_tree_hash
-        path = File.join(@shared.export_path, 'project.json')
-        dedup_entries = large_project?(path) &&
-          Feature.enabled?(:dedup_project_import_metadata, project.group)
-
-        @tree_loader.load(path, dedup_entries: dedup_entries)
-      rescue => e
-        Rails.logger.error("Import/Export error: #{e.message}") # rubocop:disable Gitlab/RailsLogger
-        raise Gitlab::ImportExport::Error.new('Incorrect JSON format')
-      end
-
-      def relation_tree_restorer
-        @relation_tree_restorer ||= RelationTreeRestorer.new(
-          user: @user,
-          shared: @shared,
-          importable: @project,
-          tree_hash: @tree_hash,
-          object_builder: object_builder,
-          members_mapper: members_mapper,
-          relation_factory: relation_factory,
-          reader: reader
-        )
-      end
-
-      def members_mapper
-        @members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @project_members,
-                                                                    user: @user,
-                                                                    importable: @project)
-      end
-
-      def object_builder
-        Gitlab::ImportExport::GroupProjectObjectBuilder
-      end
-
-      def relation_factory
-        Gitlab::ImportExport::ProjectRelationFactory
-      end
-
-      def reader
-        @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared)
-      end
-
-      def import_failure_service
-        @import_failure_service ||= ImportFailureService.new(@project)
-      end
-    end
-  end
-end

lib/gitlab/import_export/project_tree_saver.rb

-# frozen_string_literal: true
-
-module Gitlab
-  module ImportExport
-    class ProjectTreeSaver
-      attr_reader :full_path
-
-      def initialize(project:, current_user:, shared:, params: {})
-        @params       = params
-        @project      = project
-        @current_user = current_user
-        @shared       = shared
-        @full_path    = File.join(@shared.export_path, ImportExport.project_filename)
-      end
-
-      def save
-        project_tree = tree_saver.serialize(@project, reader.project_tree)
-        fix_project_tree(project_tree)
-        tree_saver.save(project_tree, @shared.export_path, ImportExport.project_filename)
-
-        true
-      rescue => e
-        @shared.error(e)
-        false
-      end
-
-      private
-
-      # Aware that the resulting hash needs to be pure-hash and
-      # does not include any AR objects anymore, only objects that run `.to_json`
-      def fix_project_tree(project_tree)
-        if @params[:description].present?
-          project_tree['description'] = @params[:description]
-        end
-
-        project_tree['project_members'] += group_members_array
-
-        RelationRenameService.add_new_associations(project_tree)
-      end
-
-      def reader
-        @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared)
-      end
-
-      def group_members_array
-        group_members.as_json(reader.group_members_tree).each do |group_member|
-          group_member['source_type'] = 'Project' # Make group members project members of the future import
-        end
-      end
-
-      def group_members
-        return [] unless @current_user.can?(:admin_group, @project.group)
-
-        # We need `.where.not(user_id: nil)` here otherwise when a group has an
-        # invitee, it would make the following query return 0 rows since a NULL
-        # user_id would be present in the subquery
-        # See http://stackoverflow.com/questions/129077/not-in-clause-and-null-values
-        non_null_user_ids = @project.project_members.where.not(user_id: nil).select(:user_id)
-
-        GroupMembersFinder.new(@project.group).execute.where.not(user_id: non_null_user_ids)
-      end
-
-      def tree_saver
-        @tree_saver ||= RelationTreeSaver.new
-      end
-    end
-  end
-end

lib/gitlab/import_export/relation_tree_restorer.rb

@@ -69,7 +69,7 @@ module Gitlab
       def process_relation_item!(relation_key, relation_definition, relation_index, data_hash)
         relation_object = build_relation(relation_key, relation_definition, data_hash)
         return unless relation_object
-        return if importable_class == Project && group_model?(relation_object)
+        return if importable_class == ::Project && group_model?(relation_object)
 
         relation_object.assign_attributes(importable_class_sym => @importable)
 
@@ -110,7 +110,7 @@ module Gitlab
           excluded_keys:  excluded_keys_for_relation(importable_class_sym))
 
         @importable.assign_attributes(params)
-        @importable.drop_visibility_level! if importable_class == Project
+        @importable.drop_visibility_level! if importable_class == ::Project
 
         Gitlab::Timeless.timeless(@importable) do
           @importable.save!

lib/gitlab_danger.rb

@@ -3,7 +3,6 @@
 class GitlabDanger
   LOCAL_RULES ||= %w[
     changes_size
-    gemfile
     documentation
     frozen_string
     duplicate_yarn_dependencies

scripts/gemfile_lock_changed.sh

+#!/bin/sh
+
+gemfile_lock_changed() {
+  if [ -n "$(git diff --name-only -- Gemfile.lock)" ]; then
+  cat << EOF
+  Gemfile was updated but Gemfile.lock was not updated.
+
+  Usually, when Gemfile is updated, you should run
+  \`\`\`
+  bundle install
+  \`\`\`
+
+  or
+
+  \`\`\`
+  bundle update <the-added-or-updated-gem>
+  \`\`\`
+
+  and commit the Gemfile.lock changes.
+EOF
+
+    exit 1
+  fi
+}
+
+gemfile_lock_changed

scripts/static-analysis

@@ -49,7 +49,8 @@ def jobs_to_run(node_index, node_total)
     %w[scripts/lint-conflicts.sh],
     %w[scripts/lint-rugged],
     %w[scripts/frontend/check_no_partial_karma_jest.sh],
-    %w[scripts/lint-changelog-filenames]
+    %w[scripts/lint-changelog-filenames],
+    %w[scripts/gemfile_lock_changed.sh]
   ]
 
   case node_total

spec/features/admin/admin_health_check_spec.rb

@@ -4,7 +4,7 @@ require 'spec_helper'
 
 describe "Admin Health Check", :feature do
   include StubENV
-  set(:admin) { create(:admin) }
+  let_it_be(:admin) { create(:admin) }
 
   before do
     stub_env('IN_MEMORY_APPLICATION_SETTINGS', 'false')

spec/features/boards/multiple_boards_spec.rb

@@ -3,11 +3,11 @@
 require 'spec_helper'
 
 describe 'Multiple Issue Boards', :js do
-  set(:user) { create(:user) }
-  set(:project) { create(:project, :public) }
-  set(:planning) { create(:label, project: project, name: 'Planning') }
-  set(:board) { create(:board, name: 'board1', project: project) }
-  set(:board2) { create(:board, name: 'board2', project: project) }
+  let_it_be(:user) { create(:user) }
+  let_it_be(:project) { create(:project, :public) }
+  let_it_be(:planning) { create(:label, project: project, name: 'Planning') }
+  let_it_be(:board) { create(:board, name: 'board1', project: project) }
+  let_it_be(:board2) { create(:board, name: 'board2', project: project) }
   let(:parent) { project }
   let(:boards_path) { project_boards_path(project) }
 

spec/features/boards/new_issue_spec.rb

@@ -129,10 +129,10 @@ describe 'Issue Boards new issue', :js do
   end
 
   context 'group boards' do
-    set(:group) { create(:group, :public) }
-    set(:project) { create(:project, namespace: group) }
-    set(:group_board) { create(:board, group: group) }
-    set(:list) { create(:list, board: group_board, position: 0) }
+    let_it_be(:group) { create(:group, :public) }
+    let_it_be(:project) { create(:project, namespace: group) }
+    let_it_be(:group_board) { create(:board, group: group) }
+    let_it_be(:list) { create(:list, board: group_board, position: 0) }
 
     context 'for unauthorized users' do
       before do

spec/features/commits_spec.rb

@@ -3,8 +3,8 @@
 require 'spec_helper'
 
 describe 'Commits' do
-  let(:project) { create(:project, :repository) }
-  let(:user) { create(:user) }
+  let_it_be(:project) { create(:project, :repository) }
+  let_it_be(:user) { create(:user) }
 
   describe 'CI' do
     before do
@@ -183,4 +183,41 @@ describe 'Commits' do
       expect(find('.js-project-refs-dropdown')).to have_content branch_name
     end
   end
+
+  context 'viewing commits for an author' do
+    let(:author_commit) { project.repository.commits(nil, limit: 1).first }
+    let(:commits) { project.repository.commits(nil, author: author, limit: 40) }
+
+    before do
+      project.add_maintainer(user)
+      sign_in(user)
+      visit project_commits_path(project, nil, author: author)
+    end
+
+    shared_examples 'show commits by author' do
+      it "includes the author's commits" do
+        commits.each do |commit|
+          expect(page).to have_content("#{author_commit.author_name} authored #{commit.authored_date.strftime("%b %d, %Y")}")
+        end
+      end
+    end
+
+    context 'author is complete' do
+      let(:author) { "#{author_commit.author_name} <#{author_commit.author_email}>" }
+
+      it_behaves_like 'show commits by author'
+    end
+
+    context 'author is just a name' do
+      let(:author) { "#{author_commit.author_name}" }
+
+      it_behaves_like 'show commits by author'
+    end
+
+    context 'author is just an email' do
+      let(:author) { "#{author_commit.author_email}" }
+
+      it_behaves_like 'show commits by author'
+    end
+  end
 end

spec/features/dashboard/root_explore_spec.rb

@@ -3,17 +3,17 @@
 require 'spec_helper'
 
 describe 'Root explore' do
-  set(:public_project) { create(:project, :public) }
-  set(:archived_project) { create(:project, :archived) }
-  set(:internal_project) { create(:project, :internal) }
-  set(:private_project) { create(:project, :private) }
+  let_it_be(:public_project) { create(:project, :public) }
+  let_it_be(:archived_project) { create(:project, :archived) }
+  let_it_be(:internal_project) { create(:project, :internal) }
+  let_it_be(:private_project) { create(:project, :private) }
 
   before do
     allow(Gitlab).to receive(:com?).and_return(true)
   end
 
   context 'when logged in' do
-    set(:user) { create(:user) }
+    let_it_be(:user) { create(:user) }
 
     before do
       sign_in(user)

spec/features/explore/user_explores_projects_spec.rb

@@ -3,10 +3,10 @@
 require 'spec_helper'
 
 describe 'User explores projects' do
-  set(:archived_project) { create(:project, :archived) }
-  set(:internal_project) { create(:project, :internal) }
-  set(:private_project) { create(:project, :private) }
-  set(:public_project) { create(:project, :public) }
+  let_it_be(:archived_project) { create(:project, :archived) }
+  let_it_be(:internal_project) { create(:project, :internal) }
+  let_it_be(:private_project) { create(:project, :private) }
+  let_it_be(:public_project) { create(:project, :public) }
 
   context 'when not signed in' do
     context 'when viewing public projects' do
@@ -19,7 +19,7 @@ describe 'User explores projects' do
   end
 
   context 'when signed in' do
-    set(:user) { create(:user) }
+    let_it_be(:user) { create(:user) }
 
     before do
       sign_in(user)

spec/features/groups/labels/user_sees_links_to_issuables_spec.rb

@@ -3,7 +3,7 @@
 require 'spec_helper'
 
 describe 'Groups > Labels > User sees links to issuables' do
-  set(:group) { create(:group, :public) }
+  let_it_be(:group) { create(:group, :public) }
 
   before do
     create(:group_label, group: group, title: 'bug')

spec/features/issues/user_views_issues_spec.rb

@@ -7,7 +7,7 @@ describe "User views issues" do
   let!(:open_issue1) { create(:issue, project: project) }
   let!(:open_issue2) { create(:issue, project: project) }
 
-  set(:user) { create(:user) }
+  let_it_be(:user) { create(:user) }
 
   shared_examples "opens issue from list" do
     it "opens issue" do

spec/features/merge_request/user_posts_notes_spec.rb

@@ -5,8 +5,7 @@ require 'spec_helper'
 describe 'Merge request > User posts notes', :js do
   include NoteInteractionHelpers
 
-  set(:project) { create(:project, :repository) }
-
+  let_it_be(:project) { create(:project, :repository) }
   let(:user) { project.creator }
   let(:merge_request) do
     create(:merge_request, source_project: project, target_project: project)

spec/features/merge_requests/user_sorts_merge_requests_spec.rb

@@ -10,10 +10,10 @@ describe 'User sorts merge requests' do
     create(:merge_request_with_diffs, source_project: project, target_project: project, source_branch: 'merge-test')
   end
 
-  set(:user) { create(:user) }
-  set(:group) { create(:group) }
-  set(:group_member) { create(:group_member, :maintainer, user: user, group: group) }
-  set(:project) { create(:project, :public, group: group) }
+  let_it_be(:user) { create(:user) }
+  let_it_be(:group) { create(:group) }
+  let_it_be(:group_member) { create(:group_member, :maintainer, user: user, group: group) }
+  let_it_be(:project) { create(:project, :public, group: group) }
 
   before do
     sign_in(user)

spec/features/merge_requests/user_views_open_merge_requests_spec.rb

@@ -3,7 +3,7 @@
 require 'spec_helper'
 
 describe 'User views open merge requests' do
-  set(:user) { create(:user) }
+  let_it_be(:user) { create(:user) }
 
   shared_examples_for 'shows merge requests' do
     it 'shows merge requests' do
@@ -12,7 +12,7 @@ describe 'User views open merge requests' do
   end
 
   context 'when project is public' do
-    set(:project) { create(:project, :public, :repository) }
+    let_it_be(:project) { create(:project, :public, :repository) }
 
     context 'when not signed in' do
       context "when the target branch is the project's default branch" do
@@ -114,7 +114,7 @@ describe 'User views open merge requests' do
   context 'when project is internal' do
     let!(:merge_request) { create(:merge_request, source_project: project, target_project: project) }
 
-    set(:project) { create(:project, :internal, :repository) }
+    let_it_be(:project) { create(:project, :internal, :repository) }
 
     context 'when signed in' do
       before do

spec/features/milestones/user_creates_milestone_spec.rb

@@ -3,8 +3,8 @@
 require 'spec_helper'
 
 describe "User creates milestone", :js do
-  set(:user) { create(:user) }
-  set(:project) { create(:project) }
+  let_it_be(:user) { create(:user) }
+  let_it_be(:project) { create(:project) }
 
   before do
     project.add_developer(user)

spec/features/milestones/user_edits_milestone_spec.rb

@@ -3,9 +3,9 @@
 require 'spec_helper'
 
 describe "User edits milestone", :js do
-  set(:user) { create(:user) }
-  set(:project) { create(:project) }
-  set(:milestone) { create(:milestone, project: project, start_date: Date.today, due_date: 5.days.from_now) }
+  let_it_be(:user) { create(:user) }
+  let_it_be(:project) { create(:project) }
+  let_it_be(:milestone) { create(:milestone, project: project, start_date: Date.today, due_date: 5.days.from_now) }
 
   before do
     project.add_developer(user)

spec/features/milestones/user_promotes_milestone_spec.rb

@@ -3,10 +3,10 @@
 require 'spec_helper'
 
 describe 'User promotes milestone' do
-  set(:group) { create(:group) }
-  set(:user) { create(:user) }
-  set(:project) { create(:project, namespace: group) }
-  set(:milestone) { create(:milestone, project: project) }
+  let_it_be(:group) { create(:group) }
+  let_it_be(:user) { create(:user) }
+  let_it_be(:project) { create(:project, namespace: group) }
+  let_it_be(:milestone) { create(:milestone, project: project) }
 
   context 'when user can admin group milestones' do
     before do

spec/features/milestones/user_views_milestone_spec.rb

@@ -3,10 +3,10 @@
 require 'spec_helper'
 
 describe "User views milestone" do
-  set(:user) { create(:user) }
-  set(:project) { create(:project) }
-  set(:milestone) { create(:milestone, project: project) }
-  set(:labels) { create_list(:label, 2, project: project) }
+  let_it_be(:user) { create(:user) }
+  let_it_be(:project) { create(:project) }
+  let_it_be(:milestone) { create(:milestone, project: project) }
+  let_it_be(:labels) { create_list(:label, 2, project: project) }
 
   before do
     project.add_developer(user)

spec/features/milestones/user_views_milestones_spec.rb

@@ -3,9 +3,9 @@
 require 'spec_helper'
 
 describe "User views milestones" do
-  set(:user) { create(:user) }
-  set(:project) { create(:project) }
-  set(:milestone) { create(:milestone, project: project) }
+  let_it_be(:user) { create(:user) }
+  let_it_be(:project) { create(:project) }
+  let_it_be(:milestone) { create(:milestone, project: project) }
 
   before do
     project.add_developer(user)
@@ -22,8 +22,8 @@ describe "User views milestones" do
   end
 
   context "with issues" do
-    set(:issue) { create(:issue, project: project, milestone: milestone) }
-    set(:closed_issue) { create(:closed_issue, project: project, milestone: milestone) }
+    let_it_be(:issue) { create(:issue, project: project, milestone: milestone) }
+    let_it_be(:closed_issue) { create(:closed_issue, project: project, milestone: milestone) }
 
     it "opens milestone" do
       click_link(milestone.title)
@@ -38,7 +38,7 @@ describe "User views milestones" do
   end
 
   context "with associated releases" do
-    set(:first_release) { create(:release, project: project, name: "The first release", milestones: [milestone], released_at: Time.zone.parse('2019-10-07')) }
+    let_it_be(:first_release) { create(:release, project: project, name: "The first release", milestones: [milestone], released_at: Time.zone.parse('2019-10-07')) }
 
     context "with a single associated release" do
       it "shows the associated release" do
@@ -48,10 +48,10 @@ describe "User views milestones" do
     end
 
     context "with lots of associated releases" do
-      set(:second_release) { create(:release, project: project, name: "The second release", milestones: [milestone], released_at: first_release.released_at + 1.day) }
-      set(:third_release) { create(:release, project: project, name: "The third release", milestones: [milestone], released_at: second_release.released_at + 1.day) }
-      set(:fourth_release) { create(:release, project: project, name: "The fourth release", milestones: [milestone], released_at: third_release.released_at + 1.day) }
-      set(:fifth_release) { create(:release, project: project, name: "The fifth release", milestones: [milestone], released_at: fourth_release.released_at + 1.day) }
+      let_it_be(:second_release) { create(:release, project: project, name: "The second release", milestones: [milestone], released_at: first_release.released_at + 1.day) }
+      let_it_be(:third_release) { create(:release, project: project, name: "The third release", milestones: [milestone], released_at: second_release.released_at + 1.day) }
+      let_it_be(:fourth_release) { create(:release, project: project, name: "The fourth release", milestones: [milestone], released_at: third_release.released_at + 1.day) }
+      let_it_be(:fifth_release) { create(:release, project: project, name: "The fifth release", milestones: [milestone], released_at: fourth_release.released_at + 1.day) }
 
       it "shows the associated releases and the truncation text" do
         expect(page).to have_content("Releases #{fifth_release.name} • #{fourth_release.name} • #{third_release.name} • 2 more releases")
@@ -66,9 +66,9 @@ describe "User views milestones" do
 end
 
 describe "User views milestones with no MR" do
-  set(:user) { create(:user) }
-  set(:project) { create(:project, :merge_requests_disabled) }
-  set(:milestone) { create(:milestone, project: project) }
+  let_it_be(:user) { create(:user) }
+  let_it_be(:project) { create(:project, :merge_requests_disabled) }
+  let_it_be(:milestone) { create(:milestone, project: project) }
 
   before do
     project.add_developer(user)

spec/features/projects/artifacts/user_downloads_artifacts_spec.rb

@@ -3,9 +3,9 @@
 require "spec_helper"
 
 describe "User downloads artifacts" do
-  set(:project) { create(:project, :repository, :public) }
-  set(:pipeline) { create(:ci_empty_pipeline, status: :success, sha: project.commit.id, project: project) }
-  set(:job) { create(:ci_build, :artifacts, :success, pipeline: pipeline) }
+  let_it_be(:project) { create(:project, :repository, :public) }
+  let_it_be(:pipeline) { create(:ci_empty_pipeline, status: :success, sha: project.commit.id, project: project) }
+  let_it_be(:job) { create(:ci_build, :artifacts, :success, pipeline: pipeline) }
 
   shared_examples "downloading" do
     it "downloads the zip" do

spec/features/projects/badges/pipeline_badge_spec.rb

@@ -3,7 +3,7 @@
 require 'spec_helper'
 
 describe 'Pipeline Badge' do
-  set(:project) { create(:project, :repository, :public) }
+  let_it_be(:project) { create(:project, :repository, :public) }
   let(:ref) { project.default_branch }
 
   context 'when the project has a pipeline' do

spec/features/projects/branches/user_deletes_branch_spec.rb

@@ -3,7 +3,7 @@
 require "spec_helper"
 
 describe "User deletes branch", :js do
-  set(:user) { create(:user) }
+  let_it_be(:user) { create(:user) }
   let(:project) { create(:project, :repository) }
 
   before do

spec/features/projects/branches/user_views_branches_spec.rb

@@ -3,8 +3,8 @@
 require "spec_helper"
 
 describe "User views branches" do
-  set(:project) { create(:project, :repository) }
-  set(:user) { project.owner }
+  let_it_be(:project) { create(:project, :repository) }
+  let_it_be(:user) { project.owner }
 
   before do
     sign_in(user)
@@ -23,7 +23,7 @@ describe "User views branches" do
   end
 
   context "protected branches" do
-    set(:protected_branch) { create(:protected_branch, project: project) }
+    let_it_be(:protected_branch) { create(:protected_branch, project: project) }
 
     before do
       visit(project_protected_branches_path(project))

spec/features/projects/commit/user_views_user_status_on_commit_spec.rb

@@ -5,8 +5,8 @@ require 'spec_helper'
 describe 'Project > Commit > View user status' do
   include RepoHelpers
 
-  set(:project) { create(:project, :repository) }
-  set(:user) { create(:user) }
+  let_it_be(:project) { create(:project, :repository) }
+  let_it_be(:user) { create(:user) }
   let(:commit_author) { create(:user, email: sample_commit.author_email) }
 
   before do

spec/features/projects/labels/user_creates_labels_spec.rb

@@ -3,8 +3,8 @@
 require "spec_helper"
 
 describe "User creates labels" do
-  set(:project) { create(:project_empty_repo, :public) }
-  set(:user) { create(:user) }
+  let_it_be(:project) { create(:project_empty_repo, :public) }
+  let_it_be(:user) { create(:user) }
 
   shared_examples_for "label creation" do
     it "creates new label" do
@@ -66,7 +66,7 @@ describe "User creates labels" do
   end
 
   context "in another project" do
-    set(:another_project) { create(:project_empty_repo, :public) }
+    let_it_be(:another_project) { create(:project_empty_repo, :public) }
 
     before do
       create(:label, project: project, title: "bug") # Create label for `project` (not `another_project`) project.

spec/features/projects/labels/user_edits_labels_spec.rb

@@ -3,9 +3,9 @@
 require "spec_helper"
 
 describe "User edits labels" do
-  set(:project) { create(:project_empty_repo, :public) }
-  set(:label) { create(:label, project: project) }
-  set(:user) { create(:user) }
+  let_it_be(:project) { create(:project_empty_repo, :public) }
+  let_it_be(:label) { create(:label, project: project) }
+  let_it_be(:user) { create(:user) }
 
   before do
     project.add_maintainer(user)

spec/features/projects/labels/user_promotes_label_spec.rb

@@ -3,10 +3,10 @@
 require 'spec_helper'
 
 describe 'User promotes label' do
-  set(:group) { create(:group) }
-  set(:user) { create(:user) }
-  set(:project) { create(:project, namespace: group) }
-  set(:label) { create(:label, project: project) }
+  let_it_be(:group) { create(:group) }
+  let_it_be(:user) { create(:user) }
+  let_it_be(:project) { create(:project, namespace: group) }
+  let_it_be(:label) { create(:label, project: project) }
 
   context 'when user can admin group labels' do
     before do

spec/features/projects/labels/user_sees_links_to_issuables_spec.rb

@@ -3,7 +3,7 @@
 require 'spec_helper'
 
 describe 'Projects > Labels > User sees links to issuables' do
-  set(:user) { create(:user) }
+  let_it_be(:user) { create(:user) }
 
   before do
     label # creates the label
@@ -50,7 +50,7 @@ describe 'Projects > Labels > User sees links to issuables' do
   end
 
   context 'with a group label' do
-    set(:group) { create(:group) }
+    let_it_be(:group) { create(:group) }
     let(:label) { create(:group_label, group: group, title: 'bug') }
 
     context 'when merge requests and issues are enabled for the project' do

spec/features/projects/labels/user_views_labels_spec.rb

@@ -3,9 +3,8 @@
 require "spec_helper"
 
 describe "User views labels" do
-  set(:project) { create(:project_empty_repo, :public) }
-  set(:user) { create(:user) }
-
+  let_it_be(:project) { create(:project_empty_repo, :public) }
+  let_it_be(:user) { create(:user) }
   let(:label_titles) { %w[bug enhancement feature] }
   let!(:prioritized_label) { create(:label, project: project, title: 'prioritized-label-name', priority: 1) }
 

spec/features/projects/settings/project_settings_spec.rb

@@ -3,7 +3,7 @@
 require 'spec_helper'
 
 describe 'Projects settings' do
-  set(:project) { create(:project) }
+  let_it_be(:project) { create(:project) }
   let(:user) { project.owner }
   let(:panel) { find('.general-settings', match: :first) }
   let(:button) { panel.find('.btn.js-settings-toggle') }

spec/features/projects/show/user_sees_git_instructions_spec.rb

@@ -3,7 +3,7 @@
 require 'spec_helper'
 
 describe 'Projects > Show > User sees Git instructions' do
-  set(:user) { create(:user) }
+  let_it_be(:user) { create(:user) }
 
   shared_examples_for 'redirects to the sign in page' do
     it 'redirects to the sign in page' do
@@ -49,7 +49,7 @@ describe 'Projects > Show > User sees Git instructions' do
 
   context 'when project is public' do
     context 'when project has no repo' do
-      set(:project) { create(:project, :public) }
+      let_it_be(:project) { create(:project, :public) }
 
       before do
         sign_in(project.owner)
@@ -60,7 +60,7 @@ describe 'Projects > Show > User sees Git instructions' do
     end
 
     context 'when project is empty' do
-      set(:project) { create(:project_empty_repo, :public) }
+      let_it_be(:project) { create(:project_empty_repo, :public) }
 
       context 'when not signed in' do
         before do
@@ -98,7 +98,7 @@ describe 'Projects > Show > User sees Git instructions' do
     end
 
     context 'when project is not empty' do
-      set(:project) { create(:project, :public, :repository) }
+      let_it_be(:project) { create(:project, :public, :repository) }
 
       before do
         visit(project_path(project))
@@ -141,7 +141,7 @@ describe 'Projects > Show > User sees Git instructions' do
   end
 
   context 'when project is internal' do
-    set(:project) { create(:project, :internal, :repository) }
+    let_it_be(:project) { create(:project, :internal, :repository) }
 
     context 'when not signed in' do
       before do
@@ -163,7 +163,7 @@ describe 'Projects > Show > User sees Git instructions' do
   end
 
   context 'when project is private' do
-    set(:project) { create(:project, :private) }
+    let_it_be(:project) { create(:project, :private) }
 
     before do
       visit(project_path(project))

spec/features/projects/show/user_sees_last_commit_ci_status_spec.rb

@@ -3,7 +3,7 @@
 require 'spec_helper'
 
 describe 'Projects > Show > User sees last commit CI status' do
-  set(:project) { create(:project, :repository, :public) }
+  let_it_be(:project) { create(:project, :repository, :public) }
 
   it 'shows the project README', :js do
     project.enable_ci

spec/features/projects/show/user_sees_readme_spec.rb

@@ -3,9 +3,8 @@
 require 'spec_helper'
 
 describe 'Projects > Show > User sees README' do
-  set(:user) { create(:user) }
-
-  set(:project) { create(:project, :repository, :public) }
+  let_it_be(:user) { create(:user) }
+  let_it_be(:project) { create(:project, :repository, :public) }
 
   it 'shows the project README', :js do
     visit project_path(project)

spec/features/projects/user_sees_user_popover_spec.rb

@@ -3,8 +3,7 @@
 require 'spec_helper'
 
 describe 'User sees user popover', :js do
-  set(:project) { create(:project, :repository) }
-
+  let_it_be(:project) { create(:project, :repository) }
   let(:user) { project.creator }
   let(:merge_request) do
     create(:merge_request, source_project: project, target_project: project)

spec/features/projects/wiki/markdown_preview_spec.rb

@@ -3,7 +3,7 @@
 require 'spec_helper'
 
 describe 'Projects > Wiki > User previews markdown changes', :js do
-  set(:user) { create(:user) }
+  let_it_be(:user) { create(:user) }
   let(:project) { create(:project, :wiki_repo, namespace: user.namespace) }
   let(:wiki_page) { create(:wiki_page, wiki: project.wiki, attrs: { title: 'home', content: '[some link](other-page)' }) }
   let(:wiki_content) do

spec/features/projects/wiki/user_creates_wiki_page_spec.rb

@@ -3,6 +3,8 @@
 require "spec_helper"
 
 describe "User creates wiki page" do
+  include WikiHelpers
+
   let(:user) { create(:user) }
   let(:wiki) { ProjectWiki.new(project, user) }
   let(:project) { create(:project) }
@@ -14,9 +16,11 @@ describe "User creates wiki page" do
   end
 
   context "when wiki is empty" do
-    before do
+    before do |example|
       visit(project_wikis_path(project))
 
+      wait_for_svg_to_be_loaded(example)
+
       click_link "Create your first page"
     end
 
@@ -45,7 +49,7 @@ describe "User creates wiki page" do
         expect(page).to have_content("Create New Page")
       end
 
-      it "shows non-escaped link in the pages list", :quarantine do
+      it "shows non-escaped link in the pages list" do
         fill_in(:wiki_title, with: "one/two/three-test")
 
         page.within(".wiki-form") do
@@ -163,7 +167,7 @@ describe "User creates wiki page" do
         expect(page).to have_link('Link to Home', href: "/#{project.full_path}/-/wikis/home")
       end
 
-      it_behaves_like 'wiki file attachments', :quarantine
+      it_behaves_like 'wiki file attachments'
     end
 
     context "in a group namespace", :js do
@@ -175,7 +179,7 @@ describe "User creates wiki page" do
         expect(page).to have_field("wiki[message]", with: "Create home")
       end
 
-      it "creates a page from the home page", :quarantine do
+      it "creates a page from the home page" do
         page.within(".wiki-form") do
           fill_in(:wiki_content, with: "My awesome wiki!")
 

spec/features/projects/wiki/user_views_wiki_page_spec.rb

@@ -19,9 +19,12 @@ describe 'User views a wiki page' do
     sign_in(user)
   end
 
-  context 'when wiki is empty' do
+  context 'when wiki is empty', :js do
     before do
-      visit(project_wikis_path(project))
+      visit project_wikis_path(project)
+
+      wait_for_svg_to_be_loaded
+
       click_link "Create your first page"
 
       fill_in(:wiki_title, with: 'one/two/three-test')
@@ -32,7 +35,7 @@ describe 'User views a wiki page' do
       end
     end
 
-    it 'shows the history of a page that has a path', :js do
+    it 'shows the history of a page that has a path' do
       expect(current_path).to include('one/two/three-test')
 
       first(:link, text: 'three').click
@@ -45,7 +48,7 @@ describe 'User views a wiki page' do
       end
     end
 
-    it 'shows an old version of a page', :js do
+    it 'shows an old version of a page' do
       expect(current_path).to include('one/two/three-test')
       expect(find('.wiki-pages')).to have_content('three')
 
@@ -162,9 +165,12 @@ describe 'User views a wiki page' do
   end
 
   it 'opens a default wiki page', :js do
-    visit(project_path(project))
+    visit project_path(project)
 
     find('.shortcuts-wiki').click
+
+    wait_for_svg_to_be_loaded
+
     click_link "Create your first page"
 
     expect(page).to have_content('Create New Page')

spec/features/read_only_spec.rb

@@ -3,7 +3,7 @@
 require 'spec_helper'
 
 describe 'read-only message' do
-  set(:user) { create(:user) }
+  let_it_be(:user) { create(:user) }
 
   before do
     sign_in(user)

spec/features/security/project/internal_access_spec.rb

@@ -5,7 +5,7 @@ require 'spec_helper'
 describe "Internal Project Access" do
   include AccessMatchers
 
-  set(:project) { create(:project, :internal, :repository) }
+  let_it_be(:project, reload: true) { create(:project, :internal, :repository) }
 
   describe "Project should be internal" do
     describe '#internal?' do

spec/features/security/project/private_access_spec.rb

@@ -5,7 +5,7 @@ require 'spec_helper'
 describe "Private Project Access" do
   include AccessMatchers
 
-  set(:project) { create(:project, :private, :repository, public_builds: false) }
+  let_it_be(:project, reload: true) { create(:project, :private, :repository, public_builds: false) }
 
   describe "Project should be private" do
     describe '#private?' do

spec/features/security/project/public_access_spec.rb

@@ -5,7 +5,7 @@ require 'spec_helper'
 describe "Public Project Access" do
   include AccessMatchers
 
-  set(:project) { create(:project, :public, :repository) }
+  let_it_be(:project, reload: true) { create(:project, :public, :repository) }
 
   describe "Project should be public" do
     describe '#public?' do

spec/features/user_sorts_things_spec.rb

@@ -10,10 +10,10 @@ describe "User sorts things" do
   include Spec::Support::Helpers::Features::SortingHelpers
   include DashboardHelper
 
-  set(:project) { create(:project_empty_repo, :public) }
-  set(:current_user) { create(:user) } # Using `current_user` instead of just `user` because of the hardoced call in `assigned_mrs_dashboard_path` which is used below.
-  set(:issue) { create(:issue, project: project, author: current_user) }
-  set(:merge_request) { create(:merge_request, target_project: project, source_project: project, author: current_user) }
+  let_it_be(:project) { create(:project_empty_repo, :public) }
+  let_it_be(:current_user) { create(:user) } # Using `current_user` instead of just `user` because of the hardoced call in `assigned_mrs_dashboard_path` which is used below.
+  let_it_be(:issue) { create(:issue, project: project, author: current_user) }
+  let_it_be(:merge_request) { create(:merge_request, target_project: project, source_project: project, author: current_user) }
 
   before do
     project.add_developer(current_user)