Merge branch '11-11-stable-prepare-rc4' into '11-11-stable'

Prepare 11.11.0-rc4 release

See merge request gitlab-org/gitlab-ce!28479

Gemfile

@@ -407,7 +407,6 @@ gem 'health_check', '~> 2.6.0'
 # System information
 gem 'vmstat', '~> 2.3.0'
 gem 'sys-filesystem', '~> 1.1.6'
-gem 'sys-proctable', '~> 1.2'
 
 # SSH host key support
 gem 'net-ssh', '~> 5.0'

Gemfile.lock

@@ -904,8 +904,6 @@ GEM
       httpclient (>= 2.4)
     sys-filesystem (1.1.6)
       ffi
-    sys-proctable (1.2.1)
-      ffi
     sysexits (1.2.0)
     temple (0.8.0)
     test-prof (0.2.5)
@@ -1211,7 +1209,6 @@ DEPENDENCIES
   stackprof (~> 0.2.10)
   state_machines-activerecord (~> 0.5.1)
   sys-filesystem (~> 1.1.6)
-  sys-proctable (~> 1.2)
   test-prof (~> 0.2.5)
   thin (~> 1.7.0)
   timecop (~> 0.8.0)

app/services/members/create_service.rb

@@ -23,7 +23,16 @@ module Members
 
       members.each do |member|
         if member.errors.any?
-          errors << "#{member.user.username}: #{member.errors.full_messages.to_sentence}"
+          current_error =
+            # Invited users may not have an associated user
+            if member.user.present?
+              "#{member.user.username}: "
+            else
+              ""
+            end
+
+          current_error += member.errors.full_messages.to_sentence
+          errors << current_error
         else
           after_execute(member: member)
         end

app/uploaders/file_uploader.rb

@@ -109,12 +109,20 @@ class FileUploader < GitlabUploader
   def upload_path
     if file_storage?
       # Legacy path relative to project.full_path
-      File.join(dynamic_segment, identifier)
+      local_storage_path(identifier)
     else
-      File.join(store_dir, identifier)
+      remote_storage_path(identifier)
     end
   end
 
+  def local_storage_path(file_identifier)
+    File.join(dynamic_segment, file_identifier)
+  end
+
+  def remote_storage_path(file_identifier)
+    File.join(store_dir, file_identifier)
+  end
+
   def store_dirs
     {
       Store::LOCAL => File.join(base_dir, dynamic_segment),

app/uploaders/personal_file_uploader.rb

@@ -6,15 +6,12 @@ class PersonalFileUploader < FileUploader
     options.storage_path
   end
 
-  def self.base_dir(model, store = nil)
-    base_dirs(model)[store || Store::LOCAL]
-  end
-
-  def self.base_dirs(model)
-    {
-      Store::LOCAL => File.join(options.base_dir, model_path_segment(model)),
-      Store::REMOTE => model_path_segment(model)
-    }
+  def self.base_dir(model, _store = nil)
+    # base_dir is the path seen by the user when rendering Markdown, so
+    # it should be the same for both local and object storage. It is
+    # typically prefaced with uploads/-/system, but that prefix
+    # is omitted in the path stored on disk.
+    File.join(options.base_dir, model_path_segment(model))
   end
 
   def self.model_path_segment(model)
@@ -40,8 +37,61 @@ class PersonalFileUploader < FileUploader
     store_dirs[object_store]
   end
 
+  # A personal snippet path is stored using FileUploader#upload_path.
+  #
+  # The format for the path:
+  #
+  # Local storage: :random_hex/:filename.
+  # Object storage: personal_snippet/:id/:random_hex/:filename.
+  #
+  # upload_paths represent the possible paths for a given identifier,
+  # which will vary depending on whether the file is stored in local or
+  # object storage. upload_path should match an element in upload_paths.
+  #
+  # base_dir represents the path seen by the user in Markdown, and it
+  # should always be prefixed with uploads/-/system.
+  #
+  # store_dirs represent the paths that are actually used on disk. For
+  # object storage, this should omit the prefix /uploads/-/system.
+  #
+  # For example, consider the requested path /uploads/-/system/personal_snippet/172/ff4ad5c2e40b39ae57cda51577317d20/file.png.
+  #
+  # For local storage:
+  #
+  # File on disk: /opt/gitlab/embedded/service/gitlab-rails/public/uploads/-/system/personal_snippet/172/ff4ad5c2e40b39ae57cda51577317d20/file.png.
+  #
+  # base_dir: uploads/-/system/personal_snippet/172
+  # upload_path: ff4ad5c2e40b39ae57cda51577317d20/file.png
+  # upload_paths: ["ff4ad5c2e40b39ae57cda51577317d20/file.png", "personal_snippet/172/ff4ad5c2e40b39ae57cda51577317d20/file.png"].
+  # store_dirs:
+  # => {1=>"uploads/-/system/personal_snippet/172/ff4ad5c2e40b39ae57cda51577317d20", 2=>"personal_snippet/172/ff4ad5c2e40b39ae57cda51577317d20"}
+  #
+  # For object storage:
+  #
+  # upload_path: personal_snippet/172/ff4ad5c2e40b39ae57cda51577317d20/file.png
+  def upload_paths(identifier)
+    [
+      local_storage_path(identifier),
+      File.join(remote_storage_base_path, identifier)
+    ]
+  end
+
+  def store_dirs
+    {
+      Store::LOCAL => File.join(base_dir, dynamic_segment),
+      Store::REMOTE => remote_storage_base_path
+    }
+  end
+
   private
 
+  # To avoid prefacing the remote storage path with `/uploads/-/system`,
+  # we just drop that part so that the destination path will be
+  # personal_snippet/:id/:random_hex/:filename.
+  def remote_storage_base_path
+    File.join(self.class.model_path_segment(model), dynamic_segment)
+  end
+
   def secure_url
     File.join('/', base_dir, secret, file.filename)
   end

changelogs/unreleased/sh-fix-invited-members.yml

+---
+title: Fix Error 500 when inviting user already present
+merge_request: 28198
+author:
+type: fixed

changelogs/unreleased/sh-fix-personal-snippet-uploads-object-storage.yml

+---
+title: Fix incorrect prefix used in new uploads for personal snippets
+merge_request: 28337
+author:
+type: fixed

config/puma.rb.example

+# frozen_string_literal: true
+
+# Load "path" as a rackup file.
+#
+# The default is "config.ru".
+#
+rackup 'config.ru'
+pidfile '/home/git/gitlab/tmp/pids/puma.pid'
+state_path '/home/git/gitlab/tmp/pids/puma.state'
+
+stdout_redirect '/home/git/gitlab/log/puma.stdout.log',
+  '/home/git/gitlab/log/puma.stderr.log',
+  true
+
+# Configure "min" to be the minimum number of threads to use to answer
+# requests and "max" the maximum.
+#
+# The default is "0, 16".
+#
+threads 1, 16
+
+# By default, workers accept all requests and queue them to pass to handlers.
+# When false, workers accept the number of simultaneous requests configured.
+#
+# Queueing requests generally improves performance, but can cause deadlocks if
+# the app is waiting on a request to itself. See https://github.com/puma/puma/issues/612
+#
+# When set to false this may require a reverse proxy to handle slow clients and
+# queue requests before they reach puma. This is due to disabling HTTP keepalive
+queue_requests false
+
+# Bind the server to "url". "tcp://", "unix://" and "ssl://" are the only
+# accepted protocols.
+bind 'unix:///home/git/gitlab/tmp/sockets/gitlab.socket'
+
+workers 3
+
+require_relative "/home/git/gitlab/lib/gitlab/cluster/lifecycle_events"
+require_relative "/home/git/gitlab/lib/gitlab/cluster/puma_worker_killer_initializer"
+
+on_restart do
+  # Signal application hooks that we're about to restart
+  Gitlab::Cluster::LifecycleEvents.do_master_restart
+end
+
+before_fork do
+  # Signal to the puma killer
+  Gitlab::Cluster::PumaWorkerKillerInitializer.start @config.options unless ENV['DISABLE_PUMA_WORKER_KILLER']
+
+  # Signal application hooks that we're about to fork
+  Gitlab::Cluster::LifecycleEvents.do_before_fork
+end
+
+Gitlab::Cluster::LifecycleEvents.set_puma_options @config.options
+on_worker_boot do
+  # Signal application hooks of worker start
+  Gitlab::Cluster::LifecycleEvents.do_worker_start
+end
+
+# Preload the application before starting the workers; this conflicts with
+# phased restart feature. (off by default)
+preload_app!
+
+tag 'gitlab-puma-worker'
+
+# Verifies that all workers have checked in to the master process within
+# the given timeout. If not the worker process will be restarted. Default
+# value is 60 seconds.
+#
+worker_timeout 60

db/post_migrate/20190424134256_drop_projects_ci_id.rb

@@ -22,7 +22,12 @@ class DropProjectsCiId < ActiveRecord::Migration[5.1]
   end
 
   def down
+    unless column_exists?(:projects, :ci_id)
       add_column :projects, :ci_id, :integer
+    end
+
+    unless index_exists?(:projects, :ci_id)
       add_concurrent_index :projects, :ci_id
     end
+  end
 end

doc/administration/dependency_proxy.md

+# GitLab Dependency Proxy administration **[PREMIUM ONLY]**
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab-ee/issues/7934) in [GitLab Premium](https://about.gitlab.com/pricing) 11.11.
+
+GitLab can be utilized as a dependency proxy for a variety of common package managers.
+
+This is the administration documentation. If you want to learn how to use the
+dependency proxies, see the [user guide](../user/group/dependency_proxy/index.md).
+
+## Enabling the Dependency Proxy feature
+
+NOTE: **Note:**
+Dependency proxy requires the Puma web server to be enabled.
+Puma support is EXPERIMENTAL at this time.
+
+To enable the Dependency proxy feature:
+
+**Omnibus GitLab installations**
+
+1. Edit `/etc/gitlab/gitlab.rb` and add the following line:
+
+   ```ruby
+   gitlab_rails['dependency_proxy_enabled'] = true
+   ```
+
+1. Save the file and [reconfigure GitLab][] for the changes to take effect.
+1. Enable the [Puma web server](https://docs.gitlab.com/omnibus/settings/puma.html).
+
+**Installations from source**
+
+1. After the installation is complete, you will have to configure the `dependency_proxy`
+   section in `config/gitlab.yml`. Set to `true` to enable it:
+
+   ```yaml
+   dependency_proxy:
+     enabled: true
+   ```
+
+1. [Restart GitLab] for the changes to take effect.
+1. Enable the [Puma web server](../install/installation.md#using-puma).
+
+## Changing the storage path
+
+By default, the dependency proxy files are stored locally, but you can change the default
+local location or even use object storage.
+
+### Changing the local storage path
+
+The dependency proxy files for Omnibus GitLab installations are stored under
+`/var/opt/gitlab/gitlab-rails/shared/dependency_proxy/` and for source
+installations under `shared/dependency_proxy/` (relative to the git home directory).
+To change the local storage path:
+
+**Omnibus GitLab installations**
+
+1. Edit `/etc/gitlab/gitlab.rb` and add the following line:
+
+   ```ruby
+   gitlab_rails['dependency_proxy_storage_path'] = "/mnt/dependency_proxy"
+   ```
+
+1. Save the file and [reconfigure GitLab][] for the changes to take effect.
+
+**Installations from source**
+
+1. Edit the `dependency_proxy` section in `config/gitlab.yml`:
+
+   ```yaml
+   dependency_proxy:
+     enabled: true
+     storage_path: shared/dependency_proxy
+   ```
+1. [Restart GitLab] for the changes to take effect.
+
+### Using object storage
+
+Instead of relying on the local storage, you can use an object storage to
+upload the blobs of the dependency proxy:
+
+**Omnibus GitLab installations**
+
+1. Edit `/etc/gitlab/gitlab.rb` and add the following lines (uncomment where
+   necessary):
+
+   ```ruby
+   gitlab_rails['dependency_proxy_enabled'] = true
+   gitlab_rails['dependency_proxy_storage_path'] = "/var/opt/gitlab/gitlab-rails/shared/dependency_proxy"
+   gitlab_rails['dependency_proxy_object_store_enabled'] = true
+   gitlab_rails['dependency_proxy_object_store_remote_directory'] = "dependency_proxy" # The bucket name.
+   gitlab_rails['dependency_proxy_object_store_direct_upload'] = false         # Use Object Storage directly for uploads instead of background uploads if enabled (Default: false).
+   gitlab_rails['dependency_proxy_object_store_background_upload'] = true      # Temporary option to limit automatic upload (Default: true).
+   gitlab_rails['dependency_proxy_object_store_proxy_download'] = false        # Passthrough all downloads via GitLab instead of using Redirects to Object Storage.
+   gitlab_rails['dependency_proxy_object_store_connection'] = {
+     ##
+     ## If the provider is AWS S3, uncomment the following
+     ##
+     #'provider' => 'AWS',
+     #'region' => 'eu-west-1',
+     #'aws_access_key_id' => 'AWS_ACCESS_KEY_ID',
+     #'aws_secret_access_key' => 'AWS_SECRET_ACCESS_KEY',
+     ##
+     ## If the provider is other than AWS (an S3-compatible one), uncomment the following
+     ##
+     #'host' => 's3.amazonaws.com',
+     #'aws_signature_version' => 4             # For creation of signed URLs. Set to 2 if provider does not support v4.
+     #'endpoint' => 'https://s3.amazonaws.com' # Useful for S3-compliant services such as DigitalOcean Spaces.
+     #'path_style' => false                    # If true, use 'host/bucket_name/object' instead of 'bucket_name.host/object'.
+   }
+   ```
+
+1. Save the file and [reconfigure GitLab][] for the changes to take effect.
+
+**Installations from source**
+
+1. Edit the `dependency_proxy` section in `config/gitlab.yml` (uncomment where necessary):
+
+   ```yaml
+   dependency_proxy:
+     enabled: true
+     ##
+     ## The location where build dependency_proxy are stored (default: shared/dependency_proxy).
+     ##
+     #storage_path: shared/dependency_proxy
+     object_store:
+       enabled: false
+       remote_directory: dependency_proxy # The bucket name.
+       #direct_upload: false      # Use Object Storage directly for uploads instead of background uploads if enabled (Default: false).
+       #background_upload: true   # Temporary option to limit automatic upload (Default: true).
+       #proxy_download: false     # Passthrough all downloads via GitLab instead of using Redirects to Object Storage.
+       connection:
+         ##
+         ## If the provider is AWS S3, uncomment the following
+         ##
+         #provider: AWS
+         #region: us-east-1
+         #aws_access_key_id: AWS_ACCESS_KEY_ID
+         #aws_secret_access_key: AWS_SECRET_ACCESS_KEY
+         ##
+         ## If the provider is other than AWS (an S3-compatible one), uncomment the following
+         ##
+         #host: 's3.amazonaws.com'             # default: s3.amazonaws.com.
+         #aws_signature_version: 4             # For creation of signed URLs. Set to 2 if provider does not support v4.
+         #endpoint: 'https://s3.amazonaws.com' # Useful for S3-compliant services such as DigitalOcean Spaces.
+         #path_style: false                    # If true, use 'host/bucket_name/object' instead of 'bucket_name.host/object'.
+   ```
+
+1. [Restart GitLab] for the changes to take effect.
+
+[reconfigure gitlab]: restart_gitlab.md#omnibus-gitlab-reconfigure "How to reconfigure Omnibus GitLab"
+[restart gitlab]: restart_gitlab.md#omnibus-gitlab-reconfigure "How to reconfigure Omnibus GitLab"

doc/administration/monitoring/prometheus/gitlab_metrics.md

@@ -47,7 +47,6 @@ The following metrics are available:
 | successful_login_captcha_total    | Gauge | 11.0 | Counter of successful CAPTCHA attempts during login |
 | unicorn_active_connections        | Gauge | 11.0 | The number of active Unicorn connections (workers) |
 | unicorn_queued_connections        | Gauge | 11.0 | The number of queued Unicorn connections |
-| unicorn_workers                   | Gauge | 11.11 | The number of Unicorn workers |
 
 ## Sidekiq Metrics available for Geo **[PREMIUM]**
 
@@ -101,10 +100,6 @@ Some basic Ruby runtime metrics are available:
 | ruby_file_descriptors                  | Gauge     | 11.1  | File descriptors per process |
 | ruby_memory_bytes                      | Gauge     | 11.1  | Memory usage by process |
 | ruby_sampler_duration_seconds_total    | Counter   | 11.1  | Time spent collecting stats |
-| ruby_process_cpu_seconds_total         | Gauge     | 11.11 | Total amount of CPU time per process |
-| ruby_process_max_fds                   | Gauge     | 11.11 | Maximum number of open file descriptors per process |
-| ruby_process_resident_memory_bytes     | Gauge     | 11.11 | Memory usage by process, measured in bytes |
-| ruby_process_start_time_seconds        | Gauge     | 11.11 | The elapsed time between system boot and the process started, measured in seconds |
 
 [GC.stat]: https://ruby-doc.org/core-2.3.0/GC.html#method-c-stat
 

doc/install/installation.md

@@ -435,6 +435,7 @@ sudo -u git -H editor config/resque.yml
 
 CAUTION: **Caution:**
 Make sure to edit both `gitlab.yml` and `unicorn.rb` to match your setup. 
+If you want to use Puma web server, see [Using Puma](#using-puma) for the additional steps.
 
 NOTE: **Note:**
 If you want to use HTTPS, see [Using HTTPS](#using-https) for the additional steps.
@@ -875,6 +876,25 @@ You also need to change the corresponding options (e.g. `ssh_user`, `ssh_host`,
 
 Apart from the always supported markdown style, there are other rich text files that GitLab can display. But you might have to install a dependency to do so. See the [github-markup gem README](https://github.com/gitlabhq/markup#markups) for more information.
 
+### Using Puma
+
+Puma is a multi-threaded HTTP 1.1 server for Ruby applications.
+
+To use GitLab with Puma:
+
+1. Finish GitLab setup so you have it up and running.
+1. Copy the supplied example Puma config file into place:
+
+   ```sh
+   cd /home/git/gitlab
+
+   # Copy config file for the web server
+   sudo -u git -H config/puma.rb.example config/puma.rb
+   ```
+
+1. Edit the system `init.d` script to use `EXPERIMENTAL_PUMA=1` flag. If you have `/etc/default/gitlab`, then you should edit it instead.
+1. Restart GitLab.
+
 ## Troubleshooting
 
 ### "You appear to have cloned an empty repository."

doc/user/admin_area/settings/continuous_integration.md

@@ -118,7 +118,15 @@ will be synced to your Group and you can visualize it from  the
 
     ![Additional minutes](img/additional_minutes.png)
 
-NOTE: **Important note**: If you have some minutes used over your default quota, these minutes will
+Be aware that:
+
+1. If you have purchased extra CI minutes before the purchase of a paid plan,
+we will calculate a pro-rated charge for your paid plan. That means you may
+be charged for less than one year since your subscription was previously
+created with the extra CI minutes.
+1. Once the extra CI minutes has been assigned to a Group they cannot be transferred
+to a different Group.
+1. If you have some minutes used over your default quota, these minutes will
 be deducted from your Additional Minutes quota immediately after your purchase of additional
 minutes.
 

doc/user/application_security/container_scanning/index.md

@@ -122,12 +122,13 @@ container_scanning:
     ## https://docs.gitlab.com/ee/ci/variables/#predefined-environment-variables
     CI_APPLICATION_REPOSITORY: $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG
     CI_APPLICATION_TAG: $CI_COMMIT_SHA
+    CLAIR_LOCAL_SCAN_VERSION: v2.0.8_fe9b059d930314b54c78f75afe265955faf4fdc1
   allow_failure: true
   services:
     - docker:stable-dind
   script:
     - docker run -d --name db arminc/clair-db:latest
-    - docker run -p 6060:6060 --link db:postgres -d --name clair --restart on-failure arminc/clair-local-scan:v2.0.6
+    - docker run -p 6060:6060 --link db:postgres -d --name clair --restart on-failure arminc/clair-local-scan:${CLAIR_LOCAL_SCAN_VERSION}
     - apk add -U wget ca-certificates
     - docker pull ${CI_APPLICATION_REPOSITORY}:${CI_APPLICATION_TAG}
     - wget https://github.com/arminc/clair-scanner/releases/download/v8/clair-scanner_linux_amd64
@@ -164,12 +165,13 @@ container_scanning:
     ## https://docs.gitlab.com/ee/ci/variables/#predefined-environment-variables
     CI_APPLICATION_REPOSITORY: $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG
     CI_APPLICATION_TAG: $CI_COMMIT_SHA
+    CLAIR_LOCAL_SCAN_VERSION: v2.0.8_fe9b059d930314b54c78f75afe265955faf4fdc1
   allow_failure: true
   services:
     - docker:stable-dind
   script:
     - docker run -d --name db arminc/clair-db:latest
-    - docker run -p 6060:6060 --link db:postgres -d --name clair --restart on-failure arminc/clair-local-scan:v2.0.6
+    - docker run -p 6060:6060 --link db:postgres -d --name clair --restart on-failure arminc/clair-local-scan:${CLAIR_LOCAL_SCAN_VERSION}
     - apk add -U wget ca-certificates
     - docker pull ${CI_APPLICATION_REPOSITORY}:${CI_APPLICATION_TAG}
     - wget https://github.com/arminc/clair-scanner/releases/download/v8/clair-scanner_linux_amd64

doc/user/group/dependency_proxy/index.md

+# Dependency Proxy **[PREMIUM]**
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab-ee/issues/7934) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.11.
+
+NOTE: **Note:**
+This is the user guide. In order to use the dependency proxy, an administrator
+must first [configure it](../../../administration/dependency_proxy.md).
+
+For many organizations, it is desirable to have a local proxy for frequently used
+upstream images/packages. In the case of CI/CD, the proxy is responsible for
+receiving a request and returning the upstream image from a registry, acting
+as a pull-through cache.
+
+The dependency proxy is available in the group level. To access it, navigate to
+a group's **Overview > Dependency Proxy**.
+
+![Dependency Proxy group page](img/group_dependency_proxy.png)
+
+## Supported dependency proxies
+
+NOTE: **Note:**
+For a list of the upcoming additions to the proxies, visit the
+[direction page](https://about.gitlab.com/direction/package/dependency_proxy/#top-vision-items).
+
+The following dependency proxies are supported.
+
+| Dependency proxy | GitLab version |
+| ---------------- | -------------- |
+| Docker           | 11.11+         |
+
+## Using the Docker dependency proxy
+
+With the Docker dependency proxy, you can use GitLab as a source for a Docker image.
+To get a Docker image into the dependency proxy:
+
+1. Find the proxy URL on your group's page under **Overview > Dependency Proxy**,
+   for example `gitlab.com/groupname/dependency_proxy/containers`.
+1. Trigger GitLab to pull the Docker image you want (e.g., `alpine:latest` or
+   `linuxserver/nextcloud:latest`) and store it in the proxy storage by using
+   one of the following ways:
+
+   - Manually pulling the Docker image:
+
+     ```bash
+     docker pull gitlab.com/groupname/dependency_proxy/containers/alpine:latest
+     ```
+
+   - From a `Dockerfile`:
+
+     ```bash
+     FROM gitlab.com/groupname/dependency_proxy/containers/alpine:latest
+     ```
+
+   - In [`.gitlab-ci.yml`](../../../ci/yaml/README.md#image):
+
+     ```bash
+     image: gitlab.com/groupname/dependency_proxy/containers/alpine:latest
+     ```
+
+GitLab will then pull the Docker image from Docker Hub and will cache the blobs
+on the GitLab server. The next time you pull the same image, it will get the latest
+information about the image from Docker Hub but will serve the existing blobs
+from GitLab.
+
+The blobs are kept forever, and there is no hard limit on how much data can be
+stored.
+
+## Limitations
+
+The following limitations apply:
+
+- Only public groups are supported (authentication is not supported yet).
+- Only Docker Hub is supported.
+- This feature requires Docker Hub being available.

doc/user/group/index.md

@@ -368,5 +368,9 @@ and issues) performed by your group members.
 
 With [GitLab Issues Analytics](issues_analytics/index.md), in groups, you can see a bar chart of the number of issues created each month.
 
+## Dependency Proxy **[PREMIUM]**
+
+Use GitLab as a [dependency proxy](dependency_proxy/index.md) for upstream Docker images.
+
 [ee]: https://about.gitlab.com/pricing/
 [ee-2534]: https://gitlab.com/gitlab-org/gitlab-ee/issues/2534

doc/user/group/saml_sso/index.md

@@ -22,8 +22,16 @@ SAML SSO for groups is used only as a convenient way to add users and does not s
 
 ![Issuer and callback for configuring SAML identity provider with GitLab.com](img/group_saml_configuration_information.png)
 
-NOTE: **Note:**
-Partial SSO enforcement was introduced in [11.8](https://gitlab.com/gitlab-org/gitlab-ee/issues/5291). With this option enabled, users must use your group's GitLab single sign on URL to be added to the group or be added via SCIM. Users can no longer be added manually. After a user has been added to the group, GitLab does not continue to enforce the use of SSO, but we'll [add a persistent check](https://gitlab.com/gitlab-org/gitlab-ee/issues/9255) in a later version.
+### SSO enforcement
+
+SSO enforcement was:
+
+- [Introduced in GitLab 11.8](https://gitlab.com/gitlab-org/gitlab-ee/issues/5291).
+- [Improved upon in GitLab 11.11 with ongoing enforcement in the GitLab UI](https://gitlab.com/gitlab-org/gitlab-ee/issues/9255).
+
+With this option enabled, users must use your group's GitLab single sign on URL to be added to the group or be added via SCIM. Users cannot be added manually, and may only access project/group resources via the UI by signing in through the SSO URL.
+
+We intend to add a similar SSO requirement for [Git and API activity](https://gitlab.com/gitlab-org/gitlab-ee/issues/9152) in the future.
 
 ### NameID
 

lib/gitlab/metrics/samplers/ruby_sampler.rb

@@ -23,32 +23,25 @@ module Gitlab
         end
 
         def init_metrics
-          metrics = {
-            file_descriptors:               ::Gitlab::Metrics.gauge(with_prefix(:file, :descriptors), 'File descriptors used', labels, :livesum),
-            memory_bytes:                   ::Gitlab::Metrics.gauge(with_prefix(:memory, :bytes), 'Memory used', labels, :livesum),
-            process_cpu_seconds_total:      ::Gitlab::Metrics.gauge(with_prefix(:process, :cpu_seconds_total), 'Process CPU seconds total'),
-            process_max_fds:                ::Gitlab::Metrics.gauge(with_prefix(:process, :max_fds), 'Process max fds'),
-            process_resident_memory_bytes:  ::Gitlab::Metrics.gauge(with_prefix(:process, :resident_memory_bytes), 'Memory used', labels, :livesum),
-            process_start_time_seconds:     ::Gitlab::Metrics.gauge(with_prefix(:process, :start_time_seconds), 'Process start time seconds'),
-            sampler_duration:               ::Gitlab::Metrics.counter(with_prefix(:sampler, :duration_seconds_total), 'Sampler time', labels),
-            total_time:                     ::Gitlab::Metrics.counter(with_prefix(:gc, :duration_seconds_total), 'Total GC time', labels)
-          }
-
+          metrics = {}
+          metrics[:sampler_duration] = ::Gitlab::Metrics.counter(with_prefix(:sampler, :duration_seconds_total), 'Sampler time', labels)
+          metrics[:total_time] = ::Gitlab::Metrics.counter(with_prefix(:gc, :duration_seconds_total), 'Total GC time', labels)
           GC.stat.keys.each do |key|
             metrics[key] = ::Gitlab::Metrics.gauge(with_prefix(:gc_stat, key), to_doc_string(key), labels, :livesum)
           end
 
+          metrics[:memory_usage] = ::Gitlab::Metrics.gauge(with_prefix(:memory, :bytes), 'Memory used', labels, :livesum)
+          metrics[:file_descriptors] = ::Gitlab::Metrics.gauge(with_prefix(:file, :descriptors), 'File descriptors used', labels, :livesum)
+
           metrics
         end
 
         def sample
           start_time = System.monotonic_time
 
+          metrics[:memory_usage].set(labels.merge(worker_label), System.memory_usage)
           metrics[:file_descriptors].set(labels.merge(worker_label), System.file_descriptor_count)
-          metrics[:process_cpu_seconds_total].set(labels.merge(worker_label), ::Gitlab::Metrics::System.cpu_time)
-          metrics[:process_max_fds].set(labels.merge(worker_label), ::Gitlab::Metrics::System.max_open_file_descriptors)
-          metrics[:process_start_time_seconds].set(labels.merge(worker_label), ::Gitlab::Metrics::System.process_start_time)
-          set_memory_usage_metrics
+
           sample_gc
 
           metrics[:sampler_duration].increment(labels, System.monotonic_time - start_time)
@@ -68,14 +61,6 @@ module Gitlab
           metrics[:total_time].increment(labels, GC::Profiler.total_time)
         end
 
-        def set_memory_usage_metrics
-          memory_usage = System.memory_usage
-          memory_labels = labels.merge(worker_label)
-
-          metrics[:memory_bytes].set(memory_labels, memory_usage)
-          metrics[:process_resident_memory_bytes].set(memory_labels, memory_usage)
-        end
-
         def worker_label
           return {} unless defined?(Unicorn::Worker)
 

lib/gitlab/metrics/samplers/unicorn_sampler.rb

@@ -8,16 +8,12 @@ module Gitlab
           super(interval)
         end
 
-        def metrics
-          @metrics ||= init_metrics
+        def unicorn_active_connections
+          @unicorn_active_connections ||= ::Gitlab::Metrics.gauge(:unicorn_active_connections, 'Unicorn active connections', {}, :max)
         end
 
-        def init_metrics
-          {
-            unicorn_active_connections: ::Gitlab::Metrics.gauge(:unicorn_active_connections, 'Unicorn active connections', {}, :max),
-            unicorn_queued_connections: ::Gitlab::Metrics.gauge(:unicorn_queued_connections, 'Unicorn queued connections', {}, :max),
-            unicorn_workers:            ::Gitlab::Metrics.gauge(:unicorn_workers, 'Unicorn workers')
-          }
+        def unicorn_queued_connections
+          @unicorn_queued_connections ||= ::Gitlab::Metrics.gauge(:unicorn_queued_connections, 'Unicorn queued connections', {}, :max)
         end
 
         def enabled?
@@ -27,13 +23,14 @@ module Gitlab
 
         def sample
           Raindrops::Linux.tcp_listener_stats(tcp_listeners).each do |addr, stats|
-            set_unicorn_connection_metrics('tcp', addr, stats)
+            unicorn_active_connections.set({ socket_type: 'tcp', socket_address: addr }, stats.active)
+            unicorn_queued_connections.set({ socket_type: 'tcp', socket_address: addr }, stats.queued)
           end
+
           Raindrops::Linux.unix_listener_stats(unix_listeners).each do |addr, stats|
-            set_unicorn_connection_metrics('unix', addr, stats)
+            unicorn_active_connections.set({ socket_type: 'unix', socket_address: addr }, stats.active)
+            unicorn_queued_connections.set({ socket_type: 'unix', socket_address: addr }, stats.queued)
           end
-
-          metrics[:unicorn_workers].set({}, unicorn_workers_count)
         end
 
         private
@@ -42,13 +39,6 @@ module Gitlab
           @tcp_listeners ||= Unicorn.listener_names.grep(%r{\A[^/]+:\d+\z})
         end
 
-        def set_unicorn_connection_metrics(type, addr, stats)
-          labels = { socket_type: type, socket_address: addr }
-
-          metrics[:unicorn_active_connections].set(labels, stats.active)
-          metrics[:unicorn_queued_connections].set(labels, stats.queued)
-        end
-
         def unix_listeners
           @unix_listeners ||= Unicorn.listener_names - tcp_listeners
         end
@@ -56,10 +46,6 @@ module Gitlab
         def unicorn_with_listeners?
           defined?(Unicorn) && Unicorn.listener_names.any?
         end
-
-        def unicorn_workers_count
-          Sys::ProcTable.ps.select {|p| p.cmdline.match(/unicorn_rails worker.+ #{Rails.root.to_s}/)}.count
-        end
       end
     end
   end

lib/gitlab/metrics/system.rb

@@ -23,21 +23,6 @@ module Gitlab
         def self.file_descriptor_count
           Dir.glob('/proc/self/fd/*').length
         end
-
-        def self.max_open_file_descriptors
-          match = File.read('/proc/self/limits').match(/Max open files\s*(\d+)/)
-
-          return unless match && match[1]
-
-          match[1].to_i
-        end
-
-        def self.process_start_time
-          start_time_in_jiffies = Sys::ProcTable.ps(pid: Process.pid).starttime
-          return 0 unless start_time_in_jiffies
-
-          start_time_in_jiffies / 100
-        end
       else
         def self.memory_usage
           0.0
@@ -46,14 +31,6 @@ module Gitlab
         def self.file_descriptor_count
           0
         end
-
-        def self.max_open_file_descriptors
-          0
-        end
-
-        def self.process_start_time
-          0
-        end
       end
 
       # THREAD_CPUTIME is not supported on OS X

lib/support/init.d/gitlab

 #! /bin/sh
 
 # GITLAB
-# Maintainer: @randx
-# Authors: rovanion.luckey@gmail.com, @randx
+# Maintainer: @dzaporozhets
+# Authors: rovanion.luckey@gmail.com, @dzaporozhets
 
 ### BEGIN INIT INFO
 # Provides:          gitlab
@@ -26,6 +26,7 @@
 
 ### Environment variables
 RAILS_ENV="production"
+EXPERIMENTAL_PUMA=""
 
 # Script variable names should be lower-case not to conflict with
 # internal /bin/sh variables such as PATH, EDITOR or SHELL.
@@ -75,7 +76,7 @@ check_pids(){
     echo "Could not create the path $pid_path needed to store the pids."
     exit 1
   fi
-  # If there exists a file which should hold the value of the Unicorn pid: read it.
+  # If there exists a file which should hold the value of the web server pid: read it.
   if [ -f "$web_server_pid_path" ]; then
     wpid=$(cat "$web_server_pid_path")
   else
@@ -198,7 +199,7 @@ check_stale_pids(){
   # If there is a pid it is something else than 0, the service is running if
   # *_status is == 0.
   if [ "$wpid" != "0" ] && [ "$web_status" != "0" ]; then
-    echo "Removing stale Unicorn web server pid. This is most likely caused by the web server crashing the last time it ran."
+    echo "Removing stale web server pid. This is most likely caused by the web server crashing the last time it ran."
     if ! rm "$web_server_pid_path"; then
       echo "Unable to remove stale pid, exiting."
       exit 1
@@ -250,12 +251,12 @@ exit_if_not_running(){
   fi
 }
 
-## Starts Unicorn and Sidekiq if they're not running.
+## Starts web server and Sidekiq if they're not running.
 start_gitlab() {
   check_stale_pids
 
   if [ "$web_status" != "0" ]; then
-    echo "Starting GitLab Unicorn"
+    echo "Starting GitLab web server"
   fi
   if [ "$sidekiq_status" != "0" ]; then
     echo "Starting GitLab Sidekiq"
@@ -275,12 +276,12 @@ start_gitlab() {
 
   # Then check if the service is running. If it is: don't start again.
   if [ "$web_status" = "0" ]; then
-    echo "The Unicorn web server already running with pid $wpid, not restarting."
+    echo "The web server already running with pid $wpid, not restarting."
   else
     # Remove old socket if it exists
     rm -f "$rails_socket" 2>/dev/null
     # Start the web server
-    RAILS_ENV=$RAILS_ENV bin/web start
+    RAILS_ENV=$RAILS_ENV EXPERIMENTAL_PUMA=$EXPERIMENTAL_PUMA bin/web start
   fi
 
   # If sidekiq is already running, don't start it again.
@@ -336,13 +337,13 @@ start_gitlab() {
   print_status
 }
 
-## Asks Unicorn, Sidekiq and MailRoom if they would be so kind as to stop, if not kills them.
+## Asks web server, Sidekiq and MailRoom if they would be so kind as to stop, if not kills them.
 stop_gitlab() {
   exit_if_not_running
 
   if [ "$web_status" = "0" ]; then
-    echo "Shutting down GitLab Unicorn"
-    RAILS_ENV=$RAILS_ENV bin/web stop
+    echo "Shutting down GitLab web server"
+    RAILS_ENV=$RAILS_ENV EXPERIMENTAL_PUMA=$EXPERIMENTAL_PUMA bin/web stop
   fi
   if [ "$sidekiq_status" = "0" ]; then
     echo "Shutting down GitLab Sidekiq"
@@ -398,9 +399,9 @@ print_status() {
     return
   fi
   if [ "$web_status" = "0" ]; then
-      echo "The GitLab Unicorn web server with pid $wpid is running."
+      echo "The GitLab web server with pid $wpid is running."
   else
-      printf "The GitLab Unicorn web server is \033[31mnot running\033[0m.\n"
+      printf "The GitLab web server is \033[31mnot running\033[0m.\n"
   fi
   if [ "$sidekiq_status" = "0" ]; then
       echo "The GitLab Sidekiq job dispatcher with pid $spid is running."
@@ -438,15 +439,15 @@ print_status() {
   fi
 }
 
-## Tells unicorn to reload its config and Sidekiq to restart
+## Tells web server to reload its config and Sidekiq to restart
 reload_gitlab(){
   exit_if_not_running
   if [ "$wpid" = "0" ];then
-    echo "The GitLab Unicorn Web server is not running thus its configuration can't be reloaded."
+    echo "The GitLab web server Web server is not running thus its configuration can't be reloaded."
     exit 1
   fi
-  printf "Reloading GitLab Unicorn configuration... "
-  RAILS_ENV=$RAILS_ENV bin/web reload
+  printf "Reloading GitLab web server configuration... "
+  RAILS_ENV=$RAILS_ENV EXPERIMENTAL_PUMA=$EXPERIMENTAL_PUMA bin/web reload
   echo "Done."
 
   echo "Restarting GitLab Sidekiq since it isn't capable of reloading its config..."
@@ -461,7 +462,7 @@ reload_gitlab(){
   print_status
 }
 
-## Restarts Sidekiq and Unicorn.
+## Restarts Sidekiq and web server.
 restart_gitlab(){
   check_status
   if [ "$web_status" = "0" ] || [ "$sidekiq_status" = "0" ] || [ "$gitlab_workhorse" = "0" ] || { [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; } || { [ "$gitlab_pages_enabled" = true ] && [ "$gitlab_pages_status" = "0" ]; } || { [ "$gitaly_enabled" = true ] && [ "$gitaly_status" = "0" ]; }; then

lib/support/init.d/gitlab.default.example

@@ -5,6 +5,9 @@
 # Normal values are "production", "test" and "development".
 RAILS_ENV="production"
 
+# Uncomment the line below to enable Puma web server instead of Unicorn.
+# EXPERIMENTAL_PUMA=1
+
 # app_user defines the user that GitLab is run as.
 # The default is "git".
 app_user="git"

spec/lib/gitlab/metrics/samplers/ruby_sampler_spec.rb

@@ -10,20 +10,17 @@ describe Gitlab::Metrics::Samplers::RubySampler do
 
   describe '#sample' do
     it 'samples various statistics' do
-      expect(Gitlab::Metrics::System).to receive(:cpu_time)
-      expect(Gitlab::Metrics::System).to receive(:file_descriptor_count)
       expect(Gitlab::Metrics::System).to receive(:memory_usage)
-      expect(Gitlab::Metrics::System).to receive(:process_start_time)
-      expect(Gitlab::Metrics::System).to receive(:max_open_file_descriptors)
+      expect(Gitlab::Metrics::System).to receive(:file_descriptor_count)
       expect(sampler).to receive(:sample_gc)
 
       sampler.sample
     end
 
-    it 'adds a metric containing the process resident memory bytes' do
+    it 'adds a metric containing the memory usage' do
       expect(Gitlab::Metrics::System).to receive(:memory_usage).and_return(9000)
 
-      expect(sampler.metrics[:process_resident_memory_bytes]).to receive(:set).with({}, 9000)
+      expect(sampler.metrics[:memory_usage]).to receive(:set).with({}, 9000)
 
       sampler.sample
     end
@@ -37,27 +34,6 @@ describe Gitlab::Metrics::Samplers::RubySampler do
       sampler.sample
     end
 
-    it 'adds a metric containing the process total cpu time' do
-      expect(Gitlab::Metrics::System).to receive(:cpu_time).and_return(0.51)
-      expect(sampler.metrics[:process_cpu_seconds_total]).to receive(:set).with({}, 0.51)
-
-      sampler.sample
-    end
-
-    it 'adds a metric containing the process start time' do
-      expect(Gitlab::Metrics::System).to receive(:process_start_time).and_return(12345)
-      expect(sampler.metrics[:process_start_time_seconds]).to receive(:set).with({}, 12345)
-
-      sampler.sample
-    end
-
-    it 'adds a metric containing the process max file descriptors' do
-      expect(Gitlab::Metrics::System).to receive(:max_open_file_descriptors).and_return(1024)
-      expect(sampler.metrics[:process_max_fds]).to receive(:set).with({}, 1024)
-
-      sampler.sample
-    end
-
     it 'clears any GC profiles' do
       expect(GC::Profiler).to receive(:clear)
 

spec/lib/gitlab/metrics/samplers/unicorn_sampler_spec.rb

@@ -39,8 +39,8 @@ describe Gitlab::Metrics::Samplers::UnicornSampler do
         it 'updates metrics type unix and with addr' do
           labels = { socket_type: 'unix', socket_address: socket_address }
 
-          expect(subject.metrics[:unicorn_active_connections]).to receive(:set).with(labels, 'active')
-          expect(subject.metrics[:unicorn_queued_connections]).to receive(:set).with(labels, 'queued')
+          expect(subject).to receive_message_chain(:unicorn_active_connections, :set).with(labels, 'active')
+          expect(subject).to receive_message_chain(:unicorn_queued_connections, :set).with(labels, 'queued')
 
           subject.sample
         end
@@ -50,6 +50,7 @@ describe Gitlab::Metrics::Samplers::UnicornSampler do
     context 'unicorn listens on tcp sockets' do
       let(:tcp_socket_address) { '0.0.0.0:8080' }
       let(:tcp_sockets) { [tcp_socket_address] }
+
       before do
         allow(unicorn).to receive(:listener_names).and_return(tcp_sockets)
       end
@@ -70,29 +71,13 @@ describe Gitlab::Metrics::Samplers::UnicornSampler do
         it 'updates metrics type unix and with addr' do
           labels = { socket_type: 'tcp', socket_address: tcp_socket_address }
 
-          expect(subject.metrics[:unicorn_active_connections]).to receive(:set).with(labels, 'active')
-          expect(subject.metrics[:unicorn_queued_connections]).to receive(:set).with(labels, 'queued')
+          expect(subject).to receive_message_chain(:unicorn_active_connections, :set).with(labels, 'active')
+          expect(subject).to receive_message_chain(:unicorn_queued_connections, :set).with(labels, 'queued')
 
           subject.sample
         end
       end
     end
-
-    context 'additional metrics' do
-      let(:unicorn_workers) { 2 }
-
-      before do
-        allow(unicorn).to receive(:listener_names).and_return([""])
-        allow(::Gitlab::Metrics::System).to receive(:cpu_time).and_return(3.14)
-        allow(subject).to receive(:unicorn_workers_count).and_return(unicorn_workers)
-      end
-
-      it "sets additional metrics" do
-        expect(subject.metrics[:unicorn_workers]).to receive(:set).with({}, unicorn_workers)
-
-        subject.sample
-      end
-    end
   end
 
   describe '#start' do

spec/lib/gitlab/metrics/system_spec.rb

@@ -13,18 +13,6 @@ describe Gitlab::Metrics::System do
         expect(described_class.file_descriptor_count).to be > 0
       end
     end
-
-    describe '.max_open_file_descriptors' do
-      it 'returns the max allowed open file descriptors' do
-        expect(described_class.max_open_file_descriptors).to be > 0
-      end
-    end
-
-    describe '.process_start_time' do
-      it 'returns the process start time' do
-        expect(described_class.process_start_time).to be > 0
-      end
-    end
   else
     describe '.memory_usage' do
       it 'returns 0.0' do
@@ -37,18 +25,6 @@ describe Gitlab::Metrics::System do
         expect(described_class.file_descriptor_count).to eq(0)
       end
     end
-
-    describe '.max_open_file_descriptors' do
-      it 'returns 0' do
-        expect(described_class.max_open_file_descriptors).to eq(0)
-      end
-    end
-
-    describe 'process_start_time' do
-      it 'returns 0' do
-        expect(described_class.process_start_time).to eq(0)
-      end
-    end
   end
 
   describe '.cpu_time' do

spec/services/members/create_service_spec.rb

@@ -44,7 +44,18 @@ describe Members::CreateService do
     result = described_class.new(user, params).execute(project)
 
     expect(result[:status]).to eq(:error)
-    expect(result[:message]).to include(project_user.username)
+    expect(result[:message]).to include("#{project_user.username}: Access level is not included in the list")
     expect(project.users).not_to include project_user
   end
+
+  it 'does not add a member with an existing invite' do
+    invited_member = create(:project_member, :invited, project: project)
+
+    params = { user_ids: invited_member.invite_email,
+               access_level: Gitlab::Access::GUEST }
+    result = described_class.new(user, params).execute(project)
+
+    expect(result[:status]).to eq(:error)
+    expect(result[:message]).to eq('Invite email has already been taken')
+  end
 end

spec/uploaders/personal_file_uploader_spec.rb

@@ -7,33 +7,19 @@ describe PersonalFileUploader do
 
   subject { uploader }
 
-  it_behaves_like 'builds correct paths',
-                  store_dir: %r[uploads/-/system/personal_snippet/\d+],
-                  upload_path: %r[\h+/\S+],
-                  absolute_path: %r[#{CarrierWave.root}/uploads/-/system/personal_snippet\/\d+\/\h+\/\S+$]
-
-  context "object_store is REMOTE" do
+  shared_examples '#base_dir' do
     before do
-      stub_uploads_object_storage
-    end
-
-    include_context 'with storage', described_class::Store::REMOTE
-
-    it_behaves_like 'builds correct paths',
-                    store_dir: %r[\d+/\h+],
-                    upload_path: %r[^personal_snippet\/\d+\/\h+\/<filename>]
+      subject.instance_variable_set(:@secret, 'secret')
     end
 
-  describe '#upload_paths' do
-    it 'builds correct paths for both local and remote storage' do
-      paths = uploader.upload_paths('test.jpg')
+    it 'is prefixed with uploads/-/system' do
+      allow(uploader).to receive(:file).and_return(double(extension: 'txt', filename: 'file_name'))
 
-      expect(paths.first).to match(%r[\h+\/test.jpg])
-      expect(paths.second).to match(%r[^personal_snippet\/\d+\/\h+\/test.jpg])
+      expect(described_class.base_dir(model)).to eq("uploads/-/system/personal_snippet/#{model.id}")
     end
   end
 
-  describe '#to_h' do
+  shared_examples '#to_h' do
     before do
       subject.instance_variable_set(:@secret, 'secret')
     end
@@ -50,6 +36,40 @@ describe PersonalFileUploader do
     end
   end
 
+  describe '#upload_paths' do
+    it 'builds correct paths for both local and remote storage' do
+      paths = uploader.upload_paths('test.jpg')
+
+      expect(paths.first).to match(%r[\h+\/test.jpg])
+      expect(paths.second).to match(%r[^personal_snippet\/\d+\/\h+\/test.jpg])
+    end
+  end
+
+  context 'object_store is LOCAL' do
+    it_behaves_like 'builds correct paths',
+                    store_dir: %r[uploads/-/system/personal_snippet/\d+/\h+],
+                    upload_path: %r[\h+/\S+],
+                    absolute_path: %r[#{CarrierWave.root}/uploads/-/system/personal_snippet\/\d+\/\h+\/\S+$]
+
+    it_behaves_like '#base_dir'
+    it_behaves_like '#to_h'
+  end
+
+  context "object_store is REMOTE" do
+    before do
+      stub_uploads_object_storage
+    end
+
+    include_context 'with storage', described_class::Store::REMOTE
+
+    it_behaves_like 'builds correct paths',
+                    store_dir: %r[\d+/\h+],
+                    upload_path: %r[^personal_snippet\/\d+\/\h+\/<filename>]
+
+    it_behaves_like '#base_dir'
+    it_behaves_like '#to_h'
+  end
+
   describe "#migrate!" do
     before do
       uploader.store!(fixture_file_upload('spec/fixtures/doc_sample.txt'))