Add latest changes from gitlab-org/gitlab@master

.gitlab/ci/frontend.gitlab-ci.yml

@@ -37,6 +37,8 @@
     paths:
       - webpack-report/
       - assets-compile.log
+      # We consume these files in GitLab UI for integration tests:
+      # https://gitlab.com/gitlab-org/gitlab-ui/-/blob/e88493b3c855aea30bf60baee692a64606b0eb1e/.storybook/preview-head.pug#L1
       - public/assets/application-*.css
       - public/assets/application-*.css.gz
     when: always

app/models/concerns/has_unique_internal_users.rb

+# frozen_string_literal: true
+
+module HasUniqueInternalUsers
+  extend ActiveSupport::Concern
+
+  class_methods do
+    private
+
+    def unique_internal(scope, username, email_pattern, &block)
+      scope.first || create_unique_internal(scope, username, email_pattern, &block)
+    end
+
+    def create_unique_internal(scope, username, email_pattern, &creation_block)
+      # Since we only want a single one of these in an instance, we use an
+      # exclusive lease to ensure than this block is never run concurrently.
+      lease_key = "user:unique_internal:#{username}"
+      lease = Gitlab::ExclusiveLease.new(lease_key, timeout: 1.minute.to_i)
+
+      until uuid = lease.try_obtain
+        # Keep trying until we obtain the lease. To prevent hammering Redis too
+        # much we'll wait for a bit between retries.
+        sleep(1)
+      end
+
+      # Recheck if the user is already present. One might have been
+      # added between the time we last checked (first line of this method)
+      # and the time we acquired the lock.
+      existing_user = uncached { scope.first }
+      return existing_user if existing_user.present?
+
+      uniquify = Uniquify.new
+
+      username = uniquify.string(username) { |s| User.find_by_username(s) }
+
+      email = uniquify.string(-> (n) { Kernel.sprintf(email_pattern, n) }) do |s|
+        User.find_by_email(s)
+      end
+
+      user = scope.build(
+        username: username,
+        email: email,
+        &creation_block
+      )
+
+      Users::UpdateService.new(user, user: user).execute(validate: false)
+      user
+    ensure
+      Gitlab::ExclusiveLease.cancel(lease_key, uuid)
+    end
+  end
+end

app/models/user.rb

@@ -21,6 +21,7 @@ class User < ApplicationRecord
   include OptionallySearch
   include FromUnion
   include BatchDestroyDependentAssociations
+  include HasUniqueInternalUsers
   include IgnorableColumns
 
   DEFAULT_NOTIFICATION_LEVEL = :participating
@@ -612,7 +613,7 @@ class User < ApplicationRecord
     # owns records previously belonging to deleted users.
     def ghost
       email = 'ghost%s@example.com'
-      unique_internal(where(ghost: true), 'ghost', email) do |u|
+      unique_internal(where(ghost: true, user_type: :ghost), 'ghost', email) do |u|
         u.bio = _('This is a "Ghost User", created to hold all issues authored by users that have since been deleted. This user cannot be removed.')
         u.name = 'Ghost User'
       end
@@ -650,6 +651,13 @@ class User < ApplicationRecord
     ghost? || bot?
   end
 
+  # We are transitioning from ghost boolean column to user_type
+  # so we need to read from old column for now
+  # @see https://gitlab.com/gitlab-org/gitlab/-/issues/210025
+  def ghost?
+    ghost
+  end
+
   def self.internal
     where(ghost: true).or(bots)
   end
@@ -1793,48 +1801,6 @@ class User < ApplicationRecord
     end
   end
 
-  def self.unique_internal(scope, username, email_pattern, &block)
-    scope.first || create_unique_internal(scope, username, email_pattern, &block)
-  end
-
-  def self.create_unique_internal(scope, username, email_pattern, &creation_block)
-    # Since we only want a single one of these in an instance, we use an
-    # exclusive lease to ensure than this block is never run concurrently.
-    lease_key = "user:unique_internal:#{username}"
-    lease = Gitlab::ExclusiveLease.new(lease_key, timeout: 1.minute.to_i)
-
-    until uuid = lease.try_obtain
-      # Keep trying until we obtain the lease. To prevent hammering Redis too
-      # much we'll wait for a bit between retries.
-      sleep(1)
-    end
-
-    # Recheck if the user is already present. One might have been
-    # added between the time we last checked (first line of this method)
-    # and the time we acquired the lock.
-    existing_user = uncached { scope.first }
-    return existing_user if existing_user.present?
-
-    uniquify = Uniquify.new
-
-    username = uniquify.string(username) { |s| User.find_by_username(s) }
-
-    email = uniquify.string(-> (n) { Kernel.sprintf(email_pattern, n) }) do |s|
-      User.find_by_email(s)
-    end
-
-    user = scope.build(
-      username: username,
-      email: email,
-      &creation_block
-    )
-
-    Users::UpdateService.new(user, user: user).execute(validate: false) # rubocop: disable CodeReuse/ServiceClass
-    user
-  ensure
-    Gitlab::ExclusiveLease.cancel(lease_key, uuid)
-  end
-
   def groups_with_developer_maintainer_project_access
     project_creation_levels = [::Gitlab::Access::DEVELOPER_MAINTAINER_PROJECT_ACCESS]
 

app/models/user_type_enums.rb

@@ -2,13 +2,11 @@
 
 module UserTypeEnums
   def self.types
-    bots.merge(human: nil)
+    @types ||= bots.merge(human: nil, ghost: 5)
   end
 
   def self.bots
-    {
-      alert_bot: 2
-    }.with_indifferent_access
+    @bots ||= { alert_bot: 2 }.with_indifferent_access
   end
 end
 

changelogs/unreleased/210025-migrate-ghost-to-user-type.yml

+---
+title: Fill user_type for ghost users
+merge_request: 27387
+author:
+type: other

db/migrate/20200313101649_fill_ghost_user_type.rb

+# frozen_string_literal: true
+
+class FillGhostUserType < ActiveRecord::Migration[6.0]
+  DOWNTIME = false
+
+  def up
+    execute('UPDATE users SET user_type = 5 WHERE ghost IS TRUE AND user_type IS NULL')
+  end
+
+  def down
+    execute('UPDATE users SET user_type = NULL WHERE ghost IS TRUE AND user_type IS NOT NULL')
+  end
+end

doc/administration/auth/cognito.md

@@ -26,7 +26,7 @@ The following steps enable AWS Cognito as an authentication provider:
    - **Callback URL** - `https://gitlab.example.com/users/auth/cognito/callback`
      - Substitute the URL of your GitLab instance for `gitlab.example.com`
    - **Allowed OAuth Flows** - Authorization code grant
-   - **Allowed OAuth Scopes** - `email` and `openid`
+   - **Allowed OAuth2 Scopes** - `email`, `openid`, and `profile`
 
 1. Save changes for the app client settings.
 1. Under **Domain name** include the AWS domain name for your AWS Cognito application.
@@ -54,15 +54,17 @@ Include the code block in the `/etc/gitlab/gitlab.rb` file:
        "app_id" => "CLIENT ID",
        "app_secret" => "CLIENT SECRET",
        "args" => {
+         "scope" => "openid profile email",
          client_options: {
            'site' => 'https://your_domain.auth.your_region.amazoncognito.com',
-           'authorize_url' => '/login',
+           'authorize_url' => '/oauth2/authorize',
            'token_url' => '/oauth2/token',
            'user_info_url' => '/oauth2/userInfo'
          },
          user_response_structure: {
            root_path: [],
-           attributes: { nickname: 'email'}
+           id_path: ['sub'],
+           attributes: { nickname: 'email', name: 'email', email: 'email' }
          },
          name: 'cognito',
          strategy_class: "OmniAuth::Strategies::OAuth2Generic"

doc/development/insert_into_tables_in_batches.md

@@ -184,10 +184,9 @@ simply be treated as if you had invoked `save` from outside the block.
 
 There are a few restrictions to how these APIs can be used:
 
-- `ON CONFLICT` behavior cannot currently be configured; an error will be raised on primary key conflicts.
-- `BulkInsertableAssociations` furthermore has the following restrictions:
-  - only compatible with `has_many` relations.
-  - does not support `has_many through: ...` relations.
+- `BulkInsertableAssociations`:
+  - It is currently only compatible with `has_many` relations.
+  - It does not yet support `has_many through: ...` relations.
 - Writing [`jsonb`](https://www.postgresql.org/docs/current/datatype-json.html) content is
 [not currently supported](https://gitlab.com/gitlab-org/gitlab/-/issues/210560).
 

doc/development/what_requires_downtime.md

@@ -162,7 +162,7 @@ class CleanupUsersUpdatedAtRename < ActiveRecord::Migration[4.2]
 end
 ```
 
-NOTE: **Note:** If you're renaming a large table, please carefully consider the state when the first migration has run but the second cleanup migration hasn't been run yet.
+NOTE: **Note:** If you're renaming a [large table](https://gitlab.com/gitlab-org/gitlab/-/blob/master/rubocop/migration_helpers.rb#L9), please carefully consider the state when the first migration has run but the second cleanup migration hasn't been run yet.
 With [Canary](https://about.gitlab.com/handbook/engineering/infrastructure/library/canary/) it is possible that the system runs in this state for a significant amount of time.
 
 ## Changing Column Constraints

doc/user/application_security/sast/index.md

@@ -17,10 +17,11 @@ to learn how to protect your organization.
 If you are using [GitLab CI/CD](../../../ci/README.md), you can analyze your source code for known
 vulnerabilities using Static Application Security Testing (SAST).
 
-You can take advantage of SAST by either [including the CI job](#configuration) in
-your existing `.gitlab-ci.yml` file or by implicitly using
-[Auto SAST](../../../topics/autodevops/index.md#auto-sast-ultimate)
-that is provided by [Auto DevOps](../../../topics/autodevops/index.md).
+You can take advantage of SAST by doing one of the following:
+
+- [Including the CI job](#configuration) in your existing `.gitlab-ci.yml` file.
+- Implicitly using [Auto SAST](../../../topics/autodevops/index.md#auto-sast-ultimate) provided by
+  [Auto DevOps](../../../topics/autodevops/index.md).
 
 GitLab checks the SAST report, compares the found vulnerabilities between the
 source and target branches, and shows the information right on the merge request.
@@ -91,12 +92,14 @@ The Java analyzers can also be used for variants like the
 
 ## Configuration
 
-For GitLab 11.9 and later, to enable SAST, you must
-[include](../../../ci/yaml/README.md#includetemplate) the
-[`SAST.gitlab-ci.yml` template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml)
-that's provided as a part of your GitLab installation.
-For GitLab versions earlier than 11.9, you can copy and use the job as defined
-that template.
+NOTE: **Note:**
+You don't have to configure SAST manually as shown in this section if you're using [Auto SAST](../../../topics/autodevops/index.md#auto-sast-ultimate)
+provided by [Auto DevOps](../../../topics/autodevops/index.md).
+
+For GitLab 11.9 and later, to enable SAST you must [include](../../../ci/yaml/README.md#includetemplate)
+the [`SAST.gitlab-ci.yml` template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml)
+that is provided as a part of your GitLab installation. For GitLab versions earlier than 11.9, you
+can copy and use the job as defined that template.
 
 Add the following to your `.gitlab-ci.yml` file:
 
@@ -448,8 +451,9 @@ the report JSON unless stated otherwise. Presence of optional fields depends on
 
 ## Secret detection
 
-GitLab is also able to detect secrets and credentials that have been unintentionally pushed to the repository.
-For example, an API key that allows write access to third-party deployment environments.
+GitLab is also able to detect secrets and credentials that have been unintentionally pushed to the
+repository (for example, an API key that allows write access to third-party deployment
+environments).
 
 This check is performed by a specific analyzer during the `sast` job. It runs regardless of the programming
 language of your app, and you don't need to change anything to your

doc/user/group/saml_sso/index.md

@@ -190,12 +190,8 @@ NOTE: **Note:** GitLab is unable to provide support for IdPs that are not listed
 |----------|---------------|
 | ADFS (Active Directory Federation Services) | [Create a Relying Party Trust](https://docs.microsoft.com/en-us/windows-server/identity/ad-fs/operations/create-a-relying-party-trust) |
 | Azure | [Configuring single sign-on to applications](https://docs.microsoft.com/en-us/azure/active-directory/manage-apps/configure-single-sign-on-non-gallery-applications) |
-| Auth0 | [Auth0 as Identity Provider](https://auth0.com/docs/protocols/saml/saml-idp-generic) |
-| G Suite | [Set up your own custom SAML application](https://support.google.com/a/answer/6087519?hl=en) |
-| JumpCloud | [Single Sign On (SSO) with GitLab](https://support.jumpcloud.com/support/s/article/single-sign-on-sso-with-gitlab-2019-08-21-10-36-47) |
 | Okta | [Setting up a SAML application in Okta](https://developer.okta.com/docs/guides/saml-application-setup/overview/) |
 | OneLogin | [Use the OneLogin SAML Test Connector](https://onelogin.service-now.com/support?id=kb_article&sys_id=93f95543db109700d5505eea4b96198f) |
-| Ping One for Enterprise | [Add and configure a new SAML application](https://support.pingidentity.com/s/document-item?bundleId=pingone&topicId=xsh1564020480660-1.html) |
 
 When [configuring your identify provider](#configuring-your-identity-provider), please consider the notes below for specific providers to help avoid common issues and as a guide for terminology used.
 
@@ -360,6 +356,13 @@ This can be prevented by configuring the [NameID](#nameid) to return a consisten
 
 Not a problem, the SAML standard means that a wide range of identity providers will work with GitLab. Unfortunately we aren't familiar with all of them so can only offer support configuring the [listed providers](#providers).
 
+Your identity provider may also have relevant documentation. It may be generic SAML documentation, or specifically targeted for GitLab. Examples:
+
+- [Auth0](https://auth0.com/docs/protocols/saml/saml-idp-generic)
+- [G Suite](https://support.google.com/a/answer/6087519?hl=en)
+- [JumpCloud](https://support.jumpcloud.com/support/s/article/single-sign-on-sso-with-gitlab-2019-08-21-10-36-47)
+- [OneLogin](https://onelogin.service-now.com/support?id=kb_article&sys_id=93f95543db109700d5505eea4b96198f).
+
 ### I need additional information to configure my identity provider
 
 Many SAML terms can vary between providers. It is possible that the information you are looking for is listed under another name.

package.json

@@ -66,8 +66,7 @@
     "core-js": "^3.6.4",
     "cropper": "^2.3.0",
     "css-loader": "^1.0.0",
-    "d3": "^4.13.0",
-    "d3-scale": "^1.0.7",
+    "d3-scale": "^2.2.2",
     "d3-selection": "^1.2.0",
     "dateformat": "^3.0.3",
     "deckar01-task_list": "^2.3.1",

spec/models/user_spec.rb

@@ -3134,6 +3134,8 @@ describe User, :do_not_mock_admin_mode do
       expect(ghost).to be_persisted
       expect(ghost.namespace).not_to be_nil
       expect(ghost.namespace).to be_persisted
+      expect(ghost.user_type).to eq 'ghost'
+      expect(ghost.ghost).to eq true
     end
 
     it "does not create a second ghost user if one is already present" do