Add latest changes from gitlab-org/gitlab@master

GITLAB_PAGES_VERSION

-1.9.0
+1.10.0

app/controllers/application_controller.rb

@@ -288,9 +288,7 @@ class ApplicationController < ActionController::Base
   def check_password_expiration
     return if session[:impersonator_id] || !current_user&.allow_password_authentication?
 
-    password_expires_at = current_user&.password_expires_at
-
-    if password_expires_at && password_expires_at < Time.now
+    if current_user&.password_expired?
       return redirect_to new_profile_password_path
     end
   end

app/controllers/projects/lfs_api_controller.rb

@@ -2,6 +2,7 @@
 
 class Projects::LfsApiController < Projects::GitHttpClientController
   include LfsRequest
+  include Gitlab::Utils::StrongMemoize
 
   LFS_TRANSFER_CONTENT_TYPE = 'application/octet-stream'
 
@@ -81,7 +82,7 @@ class Projects::LfsApiController < Projects::GitHttpClientController
       download: {
         href: "#{project.http_url_to_repo}/gitlab-lfs/objects/#{object[:oid]}",
         header: {
-          Authorization: request.headers['Authorization']
+          Authorization: authorization_header
         }.compact
       }
     }
@@ -92,7 +93,7 @@ class Projects::LfsApiController < Projects::GitHttpClientController
       upload: {
         href: "#{project.http_url_to_repo}/gitlab-lfs/objects/#{object[:oid]}/#{object[:size]}",
         header: {
-          Authorization: request.headers['Authorization'],
+          Authorization: authorization_header,
           # git-lfs v2.5.0 sets the Content-Type based on the uploaded file. This
           # ensures that Workhorse can intercept the request.
           'Content-Type': LFS_TRANSFER_CONTENT_TYPE
@@ -122,6 +123,18 @@ class Projects::LfsApiController < Projects::GitHttpClientController
   def lfs_read_only_message
     _('You cannot write to this read-only GitLab instance.')
   end
+
+  def authorization_header
+    strong_memoize(:authorization_header) do
+      lfs_auth_header || request.headers['Authorization']
+    end
+  end
+
+  def lfs_auth_header
+    return unless user.is_a?(User)
+
+    Gitlab::LfsToken.new(user).basic_encoding
+  end
 end
 
 Projects::LfsApiController.prepend_if_ee('EE::Projects::LfsApiController')

app/models/ci/artifact_blob.rb

@@ -53,7 +53,7 @@ module Ci
       pages_config.enabled &&
         pages_config.artifacts_server &&
         EXTENSIONS_SERVED_BY_PAGES.include?(File.extname(name)) &&
-        job.project.public?
+        (pages_config.access_control || job.project.public?)
     end
 
     private

app/models/user.rb

@@ -1519,6 +1519,10 @@ class User < ApplicationRecord
     todos.find_by(target: target, state: :pending)
   end
 
+  def password_expired?
+    !!(password_expires_at && password_expires_at < Time.now)
+  end
+
   # @deprecated
   alias_method :owned_or_masters_groups, :owned_or_maintainers_groups
 

changelogs/unreleased/fj-28429-generate-lfs-token-authorization.yml

+---
+title: Generate LFS token authorization for user LFS requests
+merge_request: 17332
+author:
+type: fixed

changelogs/unreleased/preview_private_artifacts.yml

+---
+title: Enable preview of private artifacts
+merge_request: 16675
+author: Tuomo Ala-Vannesluoma
+type: added

doc/administration/high_availability/README.md

@@ -25,17 +25,17 @@ solution should balance the costs against the benefits.
 
 There are many options when choosing a highly-available GitLab architecture. We
 recommend engaging with GitLab Support to choose the best architecture for your
-use-case. This page contains some various options and guidelines based on
+use case. This page contains some various options and guidelines based on
 experience with GitLab.com and Enterprise Edition on-premises customers.
 
-For a detailed insight into how GitLab scales and configures GitLab.com, you can
+For detailed insight into how GitLab scales and configures GitLab.com, you can
 watch [this 1 hour Q&A](https://www.youtube.com/watch?v=uCU8jdYzpac)
 with [John Northrup](https://gitlab.com/northrup), and live questions coming in from some of our customers.
 
 ## GitLab Components
 
 The following components need to be considered for a scaled or highly-available
-environment. In many cases components can be combined on the same nodes to reduce
+environment. In many cases, components can be combined on the same nodes to reduce
 complexity.
 
 - Unicorn/Workhorse - Web-requests (UI, API, Git over HTTP)
@@ -57,12 +57,12 @@ infrastructure and maintenance costs of full high availability.
 ### Basic Scaling
 
 This is the simplest form of scaling and will work for the majority of
-cases. Backend components such as PostgreSQL, Redis and storage are offloaded
+cases. Backend components such as PostgreSQL, Redis, and storage are offloaded
 to their own nodes while the remaining GitLab components all run on 2 or more
 application nodes.
 
 This form of scaling also works well in a cloud environment when it is more
-cost-effective to deploy several small nodes rather than a single
+cost effective to deploy several small nodes rather than a single
 larger one.
 
 - 1 PostgreSQL node
@@ -85,11 +85,11 @@ you can continue with the next step.
 
 ### Full Scaling
 
-For very large installations it may be necessary to further split components
-for maximum scalability. In a fully-scaled architecture the application node
+For very large installations, it might be necessary to further split components
+for maximum scalability. In a fully-scaled architecture, the application node
 is split into separate Sidekiq and Unicorn/Workhorse nodes. One indication that
 this architecture is required is if Sidekiq queues begin to periodically increase
-in size, indicating that there is contention or not enough resources.
+in size, indicating that there is contention or there are not enough resources.
 
 - 1 PostgreSQL node
 - 1 Redis node
@@ -100,7 +100,7 @@ in size, indicating that there is contention or not enough resources.
 
 ## High Availability Architecture Examples
 
-When organizations require scaling *and* high availability the following
+When organizations require scaling *and* high availability, the following
 architectures can be utilized. As the introduction section at the top of this
 page mentions, there is a tradeoff between cost/complexity and uptime. Be sure
 this complexity is absolutely required before taking the step into full
@@ -108,11 +108,11 @@ high availability.
 
 For all examples below, we recommend running Consul and Redis Sentinel on
 dedicated nodes. If Consul is running on PostgreSQL nodes or Sentinel on
-Redis nodes there is a potential that high resource usage by PostgreSQL or
+Redis nodes, there is a potential that high resource usage by PostgreSQL or
 Redis could prevent communication between the other Consul and Sentinel nodes.
-This may lead to the other nodes believing a failure has occurred and automated
-failover is necessary. Isolating them from the services they monitor reduces
-the chances of split-brain.
+This may lead to the other nodes believing a failure has occurred and initiating
+automated failover. Isolating Redis and Consul from the services they monitor
+reduces the chances of a false positive that a failure has occurred.
 
 The examples below do not really address high availability of NFS. Some enterprises
 have access to NFS appliances that manage availability. This is the best case
@@ -131,7 +131,7 @@ trade-offs and limits.
 This architecture will work well for many GitLab customers. Larger customers
 may begin to notice certain events cause contention/high load - for example,
 cloning many large repositories with binary files, high API usage, a large
-number of enqueued Sidekiq jobs, etc. If this happens you should consider
+number of enqueued Sidekiq jobs, and so on. If this happens, you should consider
 moving to a hybrid or fully distributed architecture depending on what is causing
 the contention.
 
@@ -162,32 +162,11 @@ contention due to certain workloads.
 
 ![Hybrid architecture diagram](img/hybrid.png)
 
-#### Reference Architecture
-
-- **Supported Users (approximate):** 10,000
-- **Known Issues:** While validating the reference architecture, slow endpoints were discovered and are being investigated. [See issue #64335](https://gitlab.com/gitlab-org/gitlab-foss/issues/64335)
-
-The Support and Quality teams built, performance tested, and validated an
-environment that supports about 10,000 users. The specifications below are a
-representation of the work so far. The specifications may be adjusted in the
-future based on additional testing and iteration.
-
-NOTE: **Note:** The specifications here were performance tested against a specific coded workload. Your exact needs may be more, depending on your workload. Your workload is influenced by factors such as - but not limited to - how active your users are, how much automation you use, mirroring, and repo/change size.
-
-- 3 PostgreSQL - 4 CPU, 16GiB memory per node
-- 1 PgBouncer - 2 CPU, 4GiB memory
-- 2 Redis - 2 CPU, 8GiB memory per node
-- 3 Consul/Sentinel - 2 CPU, 2GiB memory per node
-- 4 Sidekiq - 4 CPU, 16GiB memory per node
-- 5 GitLab application nodes - 16 CPU, 64GiB memory per node
-- 1 Gitaly - 16 CPU, 64GiB memory
-- 1 Monitoring node - 2 CPU, 8GiB memory, 100GiB local storage
-
 ### Fully Distributed
 
 This architecture scales to hundreds of thousands of users and projects and is
 the basis of the GitLab.com architecture. While this scales well it also comes
-with the added complexity of many more nodes to configure, manage and monitor.
+with the added complexity of many more nodes to configure, manage, and monitor.
 
 - 3 PostgreSQL nodes
 - 4 or more Redis nodes (2 separate clusters for persistent and cache data)
@@ -214,3 +193,59 @@ separately:
 1. [Configure the GitLab application servers](gitlab.md)
 1. [Configure the load balancers](load_balancer.md)
 1. [Monitoring node (Prometheus and Grafana)](monitoring_node.md)
+
+## Reference Architecture Examples
+
+These reference architecture examples rely on the general rule that approximately 2 requests per second (RPS) of load is generated for every 100 users.
+
+### 10,000 User Configuration
+
+- **Supported Users (approximate):** 10,000
+- **RPS:** 200 requests per second
+- **Known Issues:** While validating the reference architecture, slow endpoints were discovered and are being investigated. [gitlab-org/gitlab-ce/issues/64335](https://gitlab.com/gitlab-org/gitlab-ce/issues/64335)
+
+The Support and Quality teams built, performance tested, and validated an
+environment that supports about 10,000 users. The specifications below are a
+representation of the work so far. The specifications may be adjusted in the
+future based on additional testing and iteration.
+
+NOTE: **Note:** The specifications here were performance tested against a specific coded workload. Your exact needs may be more, depending on your workload. Your workload is influenced by factors such as - but not limited to - how active your users are, how much automation you use, mirroring, and repo/change size.
+
+- 3 PostgreSQL - 4 CPU, 16GiB memory per node
+- 1 PgBouncer - 2 CPU, 4GiB memory
+- 2 Redis - 2 CPU, 8GiB memory per node
+- 3 Consul/Sentinel - 2 CPU, 2GiB memory per node
+- 4 Sidekiq - 4 CPU, 16GiB memory per node
+- 5 GitLab application nodes - 16 CPU, 64GiB memory per node
+- 1 Gitaly - 16 CPU, 64GiB memory
+- 1 Monitoring node - 2 CPU, 8GiB memory, 100GiB local storage
+
+### 25,000 User Configuration
+
+- **Supported Users (approximate):** 25,000
+- **RPS:** 500 requests per second
+- **Status:** Work-in-progress
+- **Related Issues:** [gitlab-org/quality/performance/issues/57](https://gitlab.com/gitlab-org/quality/performance/issues/57)
+
+The Support and Quality teams are in the process of building and performance testing
+an environment that will support about 25,000 users. The specifications below
+are a work-in-progress representation of the work so far. The Quality team will be
+certifying this environment in late 2019. The specifications may be adjusted
+prior to certification based on performance testing.
+
+TBD: Add specs
+
+### 50,000 User Configuration
+
+- **Supported Users (approximate):** 50,000
+- **RPS:** 1,000 requests per second
+- **Status:** Work-in-progress
+- **Related Issues:** [gitlab-org/quality/performance/issues/66](https://gitlab.com/gitlab-org/quality/performance/issues/66)
+
+The Support and Quality teams are in the process of building and performance testing
+an environment that will support about 50,000 users. The specifications below
+are a work-in-progress representation of the work so far. The Quality team will be
+certifying this environment in late 2019. The specifications may be adjusted
+prior to certification based on performance testing.
+
+TBD: Add specs

doc/user/project/pipelines/job_artifacts.md

@@ -56,6 +56,8 @@ For more examples on artifacts, follow the [artifacts reference in
 > directly in a new tab without the need to download them when
 > [GitLab Pages](../../../administration/pages/index.md) is enabled.
 > The same holds for textual formats (currently supported extensions: `.txt`, `.json`, and `.log`).
+> With [GitLab 12.4][gitlab-16675], also artifacts in private projects can be previewed
+> when [GitLab Pages access control](../../../administration/pages/index.md#access-control) is enabled.
 
 After a job finishes, if you visit the job's specific page, there are three
 buttons. You can download the artifacts archive or browse its contents, whereas
@@ -198,6 +200,7 @@ In order to retrieve a job artifact of a different project, you might need to us
 
 [expiry date]: ../../../ci/yaml/README.md#artifactsexpire_in
 [ce-14399]: https://gitlab.com/gitlab-org/gitlab-foss/merge_requests/14399
+[gitlab-16675]: https://gitlab.com/gitlab-org/gitlab/merge_requests/16675
 
 <!-- ## Troubleshooting
 

lib/gitlab/auth.rb

@@ -231,7 +231,7 @@ module Gitlab
 
         authentication_abilities =
           if token_handler.user?
-            full_authentication_abilities
+            read_write_project_authentication_abilities
           elsif token_handler.deploy_key_pushable?(project)
             read_write_authentication_abilities
           else
@@ -272,10 +272,21 @@ module Gitlab
         ]
       end
 
-      def read_only_authentication_abilities
+      def read_only_project_authentication_abilities
         [
           :read_project,
-          :download_code,
+          :download_code
+        ]
+      end
+
+      def read_write_project_authentication_abilities
+        read_only_project_authentication_abilities + [
+          :push_code
+        ]
+      end
+
+      def read_only_authentication_abilities
+        read_only_project_authentication_abilities + [
           :read_container_image
         ]
       end

lib/gitlab/lfs_token.rb

@@ -34,8 +34,11 @@ module Gitlab
       HMACToken.new(actor).token(DEFAULT_EXPIRE_TIME)
     end
 
+    # When the token is an lfs one and the actor
+    # is blocked or the password has been changed,
+    # the token is no longer valid
     def token_valid?(token_to_check)
-      HMACToken.new(actor).token_valid?(token_to_check)
+      HMACToken.new(actor).token_valid?(token_to_check) && valid_user?
     end
 
     def deploy_key_pushable?(project)
@@ -46,6 +49,12 @@ module Gitlab
       user? ? :lfs_token : :lfs_deploy_token
     end
 
+    def valid_user?
+      return true unless user?
+
+      !actor.blocked? && (!actor.allow_password_authentication? || !actor.password_expired?)
+    end
+
     def authentication_payload(repository_http_path)
       {
         username: actor_name,
@@ -55,6 +64,10 @@ module Gitlab
       }
     end
 
+    def basic_encoding
+      ActionController::HttpAuthentication::Basic.encode_credentials(actor_name, token)
+    end
+
     private # rubocop:disable Lint/UselessAccessModifier
 
     class HMACToken

spec/controllers/projects/artifacts_controller_spec.rb

@@ -286,6 +286,25 @@ describe Projects::ArtifactsController do
         expect(response).to render_template('projects/artifacts/file')
       end
     end
+
+    context 'when the project is private and pages access control is enabled' do
+      let(:private_project) { create(:project, :repository, :private) }
+      let(:pipeline) { create(:ci_pipeline, project: private_project) }
+      let(:job) { create(:ci_build, :success, :artifacts, pipeline: pipeline) }
+
+      before do
+        private_project.add_developer(user)
+
+        allow(Gitlab.config.pages).to receive(:access_control).and_return(true)
+        allow(Gitlab.config.pages).to receive(:artifacts_server).and_return(true)
+      end
+
+      it 'renders the file view' do
+        get :file, params: { namespace_id: private_project.namespace, project_id: private_project, job_id: job, path: 'ci_artifacts.txt' }
+
+        expect(response).to have_gitlab_http_status(302)
+      end
+    end
   end
 
   describe 'GET raw' do

spec/features/projects/artifacts/user_browses_artifacts_spec.rb

@@ -114,5 +114,24 @@ describe "User browses artifacts" do
 
       it { expect(page).to have_link("doc_sample.txt").and have_no_selector(".js-artifact-tree-external-icon") }
     end
+
+    context "when the project is private and pages access control is enabled" do
+      let!(:private_project) { create(:project, :private) }
+      let(:pipeline) { create(:ci_empty_pipeline, project: private_project) }
+      let(:job) { create(:ci_build, :artifacts, pipeline: pipeline) }
+      let(:user) { create(:user) }
+
+      before do
+        private_project.add_developer(user)
+
+        allow(Gitlab.config.pages).to receive(:access_control).and_return(true)
+
+        sign_in(user)
+
+        visit(browse_project_job_artifacts_path(private_project, job, "other_artifacts_0.1.2"))
+      end
+
+      it { expect(page).to have_link("doc_sample.txt").and have_selector(".js-artifact-tree-external-icon") }
+    end
   end
 end

spec/lib/gitlab/lfs_token_spec.rb

@@ -115,6 +115,46 @@ describe Gitlab::LfsToken, :clean_gitlab_redis_shared_state do
           expect(lfs_token.token_valid?(lfs_token.token)).to be_truthy
         end
       end
+
+      context 'when the actor is a regular user' do
+        context 'when the user is blocked' do
+          let(:actor) { create(:user, :blocked) }
+
+          it 'returns false' do
+            expect(lfs_token.token_valid?(lfs_token.token)).to be_falsey
+          end
+        end
+
+        context 'when the user password is expired' do
+          let(:actor) { create(:user, password_expires_at: 1.minute.ago) }
+
+          it 'returns false' do
+            expect(lfs_token.token_valid?(lfs_token.token)).to be_falsey
+          end
+        end
+      end
+
+      context 'when the actor is an ldap user' do
+        before do
+          allow(actor).to receive(:ldap_user?).and_return(true)
+        end
+
+        context 'when the user is blocked' do
+          let(:actor) { create(:user, :blocked) }
+
+          it 'returns false' do
+            expect(lfs_token.token_valid?(lfs_token.token)).to be_falsey
+          end
+        end
+
+        context 'when the user password is expired' do
+          let(:actor) { create(:user, password_expires_at: 1.minute.ago) }
+
+          it 'returns true' do
+            expect(lfs_token.token_valid?(lfs_token.token)).to be_truthy
+          end
+        end
+      end
     end
   end
 

spec/models/user_spec.rb

@@ -3616,4 +3616,34 @@ describe User do
       end
     end
   end
+
+  describe '#password_expired?' do
+    let(:user) { build(:user, password_expires_at: password_expires_at) }
+
+    subject { user.password_expired? }
+
+    context 'when password_expires_at is not set' do
+      let(:password_expires_at) {}
+
+      it 'returns false' do
+        is_expected.to be_falsey
+      end
+    end
+
+    context 'when password_expires_at is in the past' do
+      let(:password_expires_at) { 1.minute.ago }
+
+      it 'returns true' do
+        is_expected.to be_truthy
+      end
+    end
+
+    context 'when password_expires_at is in the future' do
+      let(:password_expires_at) { 1.minute.from_now }
+
+      it 'returns false' do
+        is_expected.to be_falsey
+      end
+    end
+  end
 end

spec/support/helpers/lfs_http_helpers.rb

+# frozen_string_literal: true
+require_relative 'workhorse_helpers'
+
+module LfsHttpHelpers
+  include WorkhorseHelpers
+
+  def authorize_ci_project
+    ActionController::HttpAuthentication::Basic.encode_credentials('gitlab-ci-token', build.token)
+  end
+
+  def authorize_user
+    ActionController::HttpAuthentication::Basic.encode_credentials(user.username, user.password)
+  end
+
+  def authorize_deploy_key
+    Gitlab::LfsToken.new(key).basic_encoding
+  end
+
+  def authorize_user_key
+    Gitlab::LfsToken.new(user).basic_encoding
+  end
+
+  def authorize_deploy_token
+    ActionController::HttpAuthentication::Basic.encode_credentials(deploy_token.username, deploy_token.token)
+  end
+
+  def post_lfs_json(url, body = nil, headers = nil)
+    params = body.try(:to_json)
+    headers = (headers || {}).merge('Content-Type' => LfsRequest::CONTENT_TYPE)
+
+    post(url, params: params, headers: headers)
+  end
+
+  def batch_url(project)
+    "#{project.http_url_to_repo}/info/lfs/objects/batch"
+  end
+
+  def objects_url(project, oid = nil, size = nil)
+    File.join(["#{project.http_url_to_repo}/gitlab-lfs/objects", oid, size].compact.map(&:to_s))
+  end
+
+  def authorize_url(project, oid, size)
+    File.join(objects_url(project, oid, size), 'authorize')
+  end
+
+  def download_body(objects)
+    request_body('download', objects)
+  end
+
+  def upload_body(objects)
+    request_body('upload', objects)
+  end
+
+  def request_body(operation, objects)
+    objects = [objects] unless objects.is_a?(Array)
+
+    {
+      'operation' => operation,
+      'objects' => objects
+    }
+  end
+end

spec/support/shared_examples/lfs_http_shared_examples.rb

+# frozen_string_literal: true
+
+shared_examples 'LFS http 200 response' do
+  it_behaves_like 'LFS http expected response code and message' do
+    let(:response_code) { 200 }
+  end
+end
+
+shared_examples 'LFS http 401 response' do
+  it_behaves_like 'LFS http expected response code and message' do
+    let(:response_code) { 401 }
+  end
+end
+
+shared_examples 'LFS http 403 response' do
+  it_behaves_like 'LFS http expected response code and message' do
+    let(:response_code) { 403 }
+    let(:message) { 'Access forbidden. Check your access level.' }
+  end
+end
+
+shared_examples 'LFS http 501 response' do
+  it_behaves_like 'LFS http expected response code and message' do
+    let(:response_code) { 501 }
+    let(:message) { 'Git LFS is not enabled on this GitLab server, contact your admin.' }
+  end
+end
+
+shared_examples 'LFS http 404 response' do
+  it_behaves_like 'LFS http expected response code and message' do
+    let(:response_code) { 404 }
+  end
+end
+
+shared_examples 'LFS http expected response code and message' do
+  let(:response_code) { }
+  let(:message) { }
+
+  it 'responds with the expected response code and message' do
+    expect(response).to have_gitlab_http_status(response_code)
+    expect(json_response['message']).to eq(message) if message
+  end
+end