Merge branch '11582-dependency-proxy-private-projects' into 'master'

Use the Dependency Proxy with private GitLab groups

See merge request gitlab-org/gitlab!46042

app/controllers/concerns/dependency_proxy/auth.rb

+# frozen_string_literal: true
+
+module DependencyProxy
+  module Auth
+    extend ActiveSupport::Concern
+
+    included do
+      # We disable `authenticate_user!` since the `DependencyProxy::Auth` performs auth using JWT token
+      skip_before_action :authenticate_user!, raise: false
+      prepend_before_action :authenticate_user_from_jwt_token!
+    end
+
+    def authenticate_user_from_jwt_token!
+      return unless dependency_proxy_for_private_groups?
+
+      authenticate_with_http_token do |token, _|
+        user = user_from_token(token)
+        sign_in(user) if user
+      end
+
+      request_bearer_token! unless current_user
+    end
+
+    private
+
+    def dependency_proxy_for_private_groups?
+      Feature.enabled?(:dependency_proxy_for_private_groups, default_enabled: false)
+    end
+
+    def request_bearer_token!
+      # unfortunately, we cannot use https://api.rubyonrails.org/classes/ActionController/HttpAuthentication/Token.html#method-i-authentication_request
+      response.headers['WWW-Authenticate'] = ::DependencyProxy::Registry.authenticate_header
+      render plain: '', status: :unauthorized
+    end
+
+    def user_from_token(token)
+      token_payload = DependencyProxy::AuthTokenService.decoded_token_payload(token)
+      User.find(token_payload['user_id'])
+    rescue JWT::DecodeError, JWT::ExpiredSignature, JWT::ImmatureSignature
+      nil
+    end
+  end
+end

app/controllers/concerns/dependency_proxy/group_access.rb

+# frozen_string_literal: true
+
+module DependencyProxy
+  module GroupAccess
+    extend ActiveSupport::Concern
+
+    included do
+      before_action :verify_dependency_proxy_enabled!
+      before_action :authorize_read_dependency_proxy!
+    end
+
+    private
+
+    def verify_dependency_proxy_enabled!
+      render_404 unless group.dependency_proxy_feature_available?
+    end
+
+    def authorize_read_dependency_proxy!
+      access_denied! unless can?(current_user, :read_dependency_proxy, group)
+    end
+
+    def authorize_admin_dependency_proxy!
+      access_denied! unless can?(current_user, :admin_dependency_proxy, group)
+    end
+  end
+end

app/controllers/concerns/dependency_proxy_access.rb

-# frozen_string_literal: true
-
-module DependencyProxyAccess
-  extend ActiveSupport::Concern
-
-  included do
-    before_action :verify_dependency_proxy_enabled!
-    before_action :authorize_read_dependency_proxy!
-  end
-
-  private
-
-  def verify_dependency_proxy_enabled!
-    render_404 unless group.dependency_proxy_feature_available?
-  end
-
-  def authorize_read_dependency_proxy!
-    access_denied! unless can?(current_user, :read_dependency_proxy, group)
-  end
-
-  def authorize_admin_dependency_proxy!
-    access_denied! unless can?(current_user, :admin_dependency_proxy, group)
-  end
-end

app/controllers/groups/dependency_proxies_controller.rb

@@ -2,7 +2,7 @@
 
 module Groups
   class DependencyProxiesController < Groups::ApplicationController
-    include DependencyProxyAccess
+    include DependencyProxy::GroupAccess
 
     before_action :authorize_admin_dependency_proxy!, only: :update
     before_action :dependency_proxy

app/controllers/groups/dependency_proxy_auth_controller.rb

+# frozen_string_literal: true
+
+class Groups::DependencyProxyAuthController < ApplicationController
+  include DependencyProxy::Auth
+
+  feature_category :dependency_proxy
+
+  def authenticate
+    render plain: '', status: :ok
+  end
+end

app/controllers/groups/dependency_proxy_for_containers_controller.rb

 # frozen_string_literal: true
 
 class Groups::DependencyProxyForContainersController < Groups::ApplicationController
-  include DependencyProxyAccess
+  include DependencyProxy::Auth
+  include DependencyProxy::GroupAccess
   include SendFileUpload
 
   before_action :ensure_token_granted!
@@ -9,7 +10,7 @@ class Groups::DependencyProxyForContainersController < Groups::ApplicationContro
 
   attr_reader :token
 
-  feature_category :package_registry
+  feature_category :dependency_proxy
 
   def manifest
     result = DependencyProxy::PullManifestService.new(image, tag, token).execute

app/controllers/jwt_controller.rb

@@ -11,7 +11,8 @@ class JwtController < ApplicationController
   feature_category :authentication_and_authorization
 
   SERVICES = {
-    Auth::ContainerRegistryAuthenticationService::AUDIENCE => Auth::ContainerRegistryAuthenticationService
+    ::Auth::ContainerRegistryAuthenticationService::AUDIENCE => ::Auth::ContainerRegistryAuthenticationService,
+    ::Auth::DependencyProxyAuthenticationService::AUDIENCE => ::Auth::DependencyProxyAuthenticationService
   }.freeze
 
   def auth

app/models/dependency_proxy/registry.rb

 # frozen_string_literal: true
 
 class DependencyProxy::Registry
-  AUTH_URL = 'https://auth.docker.io'.freeze
-  LIBRARY_URL = 'https://registry-1.docker.io/v2'.freeze
+  AUTH_URL = 'https://auth.docker.io'
+  LIBRARY_URL = 'https://registry-1.docker.io/v2'
+  PROXY_AUTH_URL = Gitlab::Utils.append_path(Gitlab.config.gitlab.url, "jwt/auth")
 
   class << self
     def auth_url(image)
@@ -17,6 +18,10 @@ class DependencyProxy::Registry
       "#{LIBRARY_URL}/#{image_path(image)}/blobs/#{blob_sha}"
     end
 
+    def authenticate_header
+      "Bearer realm=\"#{PROXY_AUTH_URL}\",service=\"#{::Auth::DependencyProxyAuthenticationService::AUDIENCE}\""
+    end
+
     private
 
     def image_path(image)

app/services/auth/dependency_proxy_authentication_service.rb

+# frozen_string_literal: true
+
+module Auth
+  class DependencyProxyAuthenticationService < BaseService
+    AUDIENCE = 'dependency_proxy'
+    HMAC_KEY = 'gitlab-dependency-proxy'
+    DEFAULT_EXPIRE_TIME = 1.minute
+
+    def execute(authentication_abilities:)
+      return error('dependency proxy not enabled', 404) unless ::Gitlab.config.dependency_proxy.enabled
+      return error('access forbidden', 403) unless current_user
+
+      { token: authorized_token.encoded }
+    end
+
+    class << self
+      include ::Gitlab::Utils::StrongMemoize
+
+      def secret
+        strong_memoize(:secret) do
+          OpenSSL::HMAC.hexdigest(
+            'sha256',
+            ::Settings.attr_encrypted_db_key_base,
+            HMAC_KEY
+          )
+        end
+      end
+
+      def token_expire_at
+        Time.current + Gitlab::CurrentSettings.container_registry_token_expire_delay.minutes
+      end
+    end
+
+    private
+
+    def authorized_token
+      JSONWebToken::HMACToken.new(self.class.secret).tap do |token|
+        token['user_id'] = current_user.id
+        token.expire_time = self.class.token_expire_at
+      end
+    end
+  end
+end

app/services/dependency_proxy/auth_token_service.rb

+# frozen_string_literal: true
+
+module DependencyProxy
+  class AuthTokenService < DependencyProxy::BaseService
+    attr_reader :token
+
+    def initialize(token)
+      @token = token
+    end
+
+    def execute
+      JSONWebToken::HMACToken.decode(token, ::Auth::DependencyProxyAuthenticationService.secret).first
+    end
+
+    class << self
+      def decoded_token_payload(token)
+        self.new(token).execute
+      end
+    end
+  end
+end

app/views/groups/dependency_proxies/show.html.haml

@@ -7,7 +7,7 @@
     - link_start = '<a href="%{url}">'.html_safe % { url: help_page_path('user/packages/dependency_proxy/index') }
     = _('Create a local proxy for storing frequently used upstream images. %{link_start}Learn more%{link_end} about dependency proxies.').html_safe % { link_start: link_start, link_end: '</a>'.html_safe }
 
-- if @group.public?
+- if Feature.enabled?(:dependency_proxy_for_private_groups, default_enabled: false) || @group.public?
   - if can?(current_user, :admin_dependency_proxy, @group)
     = form_for(@dependency_proxy, method: :put, url: group_dependency_proxy_path(@group)) do |f|
       .form-group

config/feature_flags/development/dependency_proxy_for_private_groups.yml

+---
+name: dependency_proxy_for_private_groups
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/46042
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/276777
+milestone: '13.7'
+type: development
+group: group::package
+default_enabled: false

config/routes/group.rb

@@ -125,7 +125,7 @@ end
 # Dependency proxy for containers
 # Because docker adds v2 prefix to URI this need to be outside of usual group routes
 scope format: false do
-  get 'v2', to: proc { [200, {}, ['']] } # rubocop:disable Cop/PutGroupRoutesUnderScope
+  get 'v2' => 'groups/dependency_proxy_auth#authenticate' # rubocop:disable Cop/PutGroupRoutesUnderScope
 
   constraints image: Gitlab::PathRegex.container_image_regex, sha: Gitlab::PathRegex.container_image_blob_sha_regex do
     get 'v2/*group_id/dependency_proxy/containers/*image/manifests/*tag' => 'groups/dependency_proxy_for_containers#manifest' # rubocop:todo Cop/PutGroupRoutesUnderScope

doc/user/packages/dependency_proxy/index.md

@@ -8,6 +8,8 @@ info: To determine the technical writer assigned to the Stage/Group associated w
 
 > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/7934) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.11.
 > - [Moved](https://gitlab.com/gitlab-org/gitlab/-/issues/273655) to [GitLab Core](https://about.gitlab.com/pricing/) in GitLab 13.6.
+> - [Support for private groups](https://gitlab.com/gitlab-org/gitlab/-/issues/11582) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.7.
+> - Anonymous access to images in public groups is no longer available starting in [GitLab Premium](https://about.gitlab.com/pricing/) 13.7.
 
 The GitLab Dependency Proxy is a local proxy you can use for your frequently-accessed
 upstream images.
@@ -17,9 +19,7 @@ upstream image from a registry, acting as a pull-through cache.
 
 ## Prerequisites
 
-To use the Dependency Proxy:
-
-- Your group must be public. Authentication for private groups is [not supported yet](https://gitlab.com/gitlab-org/gitlab/-/issues/11582).
+The Dependency Proxy must be [enabled by an administrator](../../../administration/packages/dependency_proxy.md).
 
 ### Supported images and packages
 
@@ -58,6 +58,56 @@ Prerequisites:
 - Docker Hub must be available. Follow [this issue](https://gitlab.com/gitlab-org/gitlab/-/issues/241639)
   for progress on accessing images when Docker Hub is down.
 
+### Authenticate with the Dependency Proxy
+
+Because the Dependency Proxy is storing Docker images in a space associated with your group,
+you must authenticate against the Dependency Proxy.
+
+Follow the [instructions for using images from a private registry](../../../ci/docker/using_docker_images.md#define-an-image-from-a-private-container-registry),
+but instead of using `registry.example.com:5000`, use your GitLab domain with no port `gitlab.example.com`.
+
+For example, to manually log in:
+
+```shell
+docker login gitlab.example.com --username my_username --password my_password
+```
+
+You can authenticate using:
+
+- Your GitLab username and password.
+- A [personal access token](../../../user/profile/personal_access_tokens.md) with the scope set to `read_registry` and `write_registry`.
+
+#### Authenticate within CI/CD
+
+To work with the Dependency Proxy in [GitLab CI/CD](../../../ci/README.md), you can use
+`CI_REGISTRY_USER` and `CI_REGISTRY_PASSWORD`.
+
+```shell
+docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" gitlab.example.com
+```
+
+You can use other [predefined variables](../../../ci/variables/predefined_variables.md)
+to further generalize your CI script. For example:
+
+```yaml
+# .gitlab-ci.yml
+
+dependency-proxy-pull-master:
+  # Official docker image.
+  image: docker:latest
+  stage: build
+  services:
+    - docker:dind
+  before_script:
+    - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_SERVER_HOST":"$CI_SERVER_PORT"
+  script:
+    - docker pull "$CI_SERVER_HOST":"$CI_SERVER_PORT"/groupname/dependency_proxy/containers/alpine:latest
+```
+
+You can also use [custom environment variables](../../../ci/variables/README.md#custom-environment-variables) to store and access your personal access token or other valid credentials.
+
+### Store a Docker image in Dependency Proxy cache
+
 To store a Docker image in Dependency Proxy storage:
 
 1. Go to your group's **Packages & Registries > Dependency Proxy**.

spec/controllers/groups/dependency_proxy_auth_controller_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Groups::DependencyProxyAuthController do
+  include DependencyProxyHelpers
+
+  describe 'GET #authenticate' do
+    subject { get :authenticate }
+
+    context 'feature flag disabled' do
+      before do
+        stub_feature_flags(dependency_proxy_for_private_groups: false)
+      end
+
+      it 'returns successfully', :aggregate_failures do
+        subject
+
+        expect(response).to have_gitlab_http_status(:success)
+      end
+    end
+
+    context 'without JWT' do
+      it 'returns unauthorized with oauth realm', :aggregate_failures do
+        subject
+
+        expect(response).to have_gitlab_http_status(:unauthorized)
+        expect(response.headers['WWW-Authenticate']).to eq DependencyProxy::Registry.authenticate_header
+      end
+    end
+
+    context 'with valid JWT' do
+      let_it_be(:user) { create(:user) }
+      let(:jwt) { build_jwt(user) }
+      let(:token_header) { "Bearer #{jwt.encoded}" }
+
+      before do
+        request.headers['HTTP_AUTHORIZATION'] = token_header
+      end
+
+      it { is_expected.to have_gitlab_http_status(:success) }
+    end
+
+    context 'with invalid JWT' do
+      context 'bad user' do
+        let(:jwt) { build_jwt(double('bad_user', id: 999)) }
+        let(:token_header) { "Bearer #{jwt.encoded}" }
+
+        before do
+          request.headers['HTTP_AUTHORIZATION'] = token_header
+        end
+
+        it { is_expected.to have_gitlab_http_status(:not_found) }
+      end
+
+      context 'token with no user id' do
+        let(:token_header) { "Bearer #{build_jwt.encoded}" }
+
+        before do
+          request.headers['HTTP_AUTHORIZATION'] = token_header
+        end
+
+        it { is_expected.to have_gitlab_http_status(:not_found) }
+      end
+
+      context 'expired token' do
+        let_it_be(:user) { create(:user) }
+        let(:jwt) { build_jwt(user, expire_time: Time.zone.now - 1.hour) }
+        let(:token_header) { "Bearer #{jwt.encoded}" }
+
+        before do
+          request.headers['HTTP_AUTHORIZATION'] = token_header
+        end
+
+        it { is_expected.to have_gitlab_http_status(:unauthorized) }
+      end
+    end
+  end
+end

spec/controllers/groups/dependency_proxy_for_containers_controller_spec.rb

@@ -3,8 +3,77 @@
 require 'spec_helper'
 
 RSpec.describe Groups::DependencyProxyForContainersController do
+  include HttpBasicAuthHelpers
+  include DependencyProxyHelpers
+
+  let_it_be(:user) { create(:user) }
   let(:group) { create(:group) }
   let(:token_response) { { status: :success, token: 'abcd1234' } }
+  let(:jwt) { build_jwt(user) }
+  let(:token_header) { "Bearer #{jwt.encoded}" }
+
+  shared_examples 'without a token' do
+    before do
+      request.headers['HTTP_AUTHORIZATION'] = nil
+    end
+
+    context 'feature flag disabled' do
+      before do
+        stub_feature_flags(dependency_proxy_for_private_groups: false)
+      end
+
+      it { is_expected.to have_gitlab_http_status(:ok) }
+    end
+
+    it { is_expected.to have_gitlab_http_status(:unauthorized) }
+  end
+
+  shared_examples 'feature flag disabled with private group' do
+    before do
+      stub_feature_flags(dependency_proxy_for_private_groups: false)
+    end
+
+    it 'redirects', :aggregate_failures do
+      group.update!(visibility_level: Gitlab::VisibilityLevel::PRIVATE)
+
+      subject
+
+      expect(response).to have_gitlab_http_status(:redirect)
+      expect(response.location).to end_with(new_user_session_path)
+    end
+  end
+
+  shared_examples 'without permission' do
+    context 'with invalid user' do
+      before do
+        user = double('bad_user', id: 999)
+        token_header = "Bearer #{build_jwt(user).encoded}"
+        request.headers['HTTP_AUTHORIZATION'] = token_header
+      end
+
+      it { is_expected.to have_gitlab_http_status(:not_found) }
+    end
+
+    context 'with valid user that does not have access' do
+      let(:group) { create(:group, :private) }
+
+      before do
+        user = double('bad_user', id: 999)
+        token_header = "Bearer #{build_jwt(user).encoded}"
+        request.headers['HTTP_AUTHORIZATION'] = token_header
+      end
+
+      it { is_expected.to have_gitlab_http_status(:not_found) }
+    end
+
+    context 'when user is not found' do
+      before do
+        allow(User).to receive(:find).and_return(nil)
+      end
+
+      it { is_expected.to have_gitlab_http_status(:unauthorized) }
+    end
+  end
 
   shared_examples 'not found when disabled' do
     context 'feature disabled' do
@@ -27,6 +96,8 @@ RSpec.describe Groups::DependencyProxyForContainersController do
     allow_next_instance_of(DependencyProxy::RequestTokenService) do |instance|
       allow(instance).to receive(:execute).and_return(token_response)
     end
+
+    request.headers['HTTP_AUTHORIZATION'] = token_header
   end
 
   describe 'GET #manifest' do
@@ -46,6 +117,10 @@ RSpec.describe Groups::DependencyProxyForContainersController do
         enable_dependency_proxy
       end
 
+      it_behaves_like 'without a token'
+      it_behaves_like 'without permission'
+      it_behaves_like 'feature flag disabled with private group'
+
       context 'remote token request fails' do
         let(:token_response) do
           {
@@ -113,6 +188,10 @@ RSpec.describe Groups::DependencyProxyForContainersController do
         enable_dependency_proxy
       end
 
+      it_behaves_like 'without a token'
+      it_behaves_like 'without permission'
+      it_behaves_like 'feature flag disabled with private group'
+
       context 'remote blob request fails' do
         let(:blob_response) do
           {

spec/features/groups/dependency_proxy_spec.rb

@@ -79,13 +79,19 @@ RSpec.describe 'Group Dependency Proxy' do
         sign_in(developer)
       end
 
-      context 'group is private' do
-        let(:group) { create(:group, :private) }
+      context 'feature flag is disabled' do
+        before do
+          stub_feature_flags(dependency_proxy_for_private_groups: false)
+        end
 
-        it 'informs user that feature is only available for public groups' do
-          visit path
+        context 'group is private' do
+          let(:group) { create(:group, :private) }
 
-          expect(page).to have_content('Dependency proxy feature is limited to public groups for now.')
+          it 'informs user that feature is only available for public groups' do
+            visit path
+
+            expect(page).to have_content('Dependency proxy feature is limited to public groups for now.')
+          end
         end
       end
 

spec/models/dependency_proxy/registry_spec.rb

@@ -54,4 +54,11 @@ RSpec.describe DependencyProxy::Registry, type: :model do
       end
     end
   end
+
+  describe '#authenticate_header' do
+    it 'returns the OAuth realm and service header' do
+      expect(described_class.authenticate_header)
+        .to eq("Bearer realm=\"#{Gitlab.config.gitlab.url}/jwt/auth\",service=\"dependency_proxy\"")
+    end
+  end
 end

spec/routing/group_routing_spec.rb

@@ -81,6 +81,10 @@ RSpec.describe "Groups", "routing" do
   end
 
   describe 'dependency proxy for containers' do
+    it 'routes to #authenticate' do
+      expect(get('/v2')).to route_to('groups/dependency_proxy_auth#authenticate')
+    end
+
     context 'image name without namespace' do
       it 'routes to #manifest' do
         expect(get('/v2/gitlabhq/dependency_proxy/containers/ruby/manifests/2.3.6'))

spec/services/auth/dependency_proxy_authentication_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Auth::DependencyProxyAuthenticationService do
+  let_it_be(:user) { create(:user) }
+  let(:service) { Auth::DependencyProxyAuthenticationService.new(nil, user) }
+
+  before do
+    stub_config(dependency_proxy: { enabled: true })
+  end
+
+  describe '#execute' do
+    subject { service.execute(authentication_abilities: nil) }
+
+    context 'dependency proxy is not enabled' do
+      before do
+        stub_config(dependency_proxy: { enabled: false })
+      end
+
+      it 'returns not found' do
+        result = subject
+
+        expect(result[:http_status]).to eq(404)
+        expect(result[:message]).to eq('dependency proxy not enabled')
+      end
+    end
+
+    context 'without a user' do
+      let(:user) { nil }
+
+      it 'returns forbidden' do
+        result = subject
+
+        expect(result[:http_status]).to eq(403)
+        expect(result[:message]).to eq('access forbidden')
+      end
+    end
+
+    context 'with a user' do
+      it 'returns a token' do
+        expect(subject[:token]).not_to be_nil
+      end
+    end
+  end
+end

spec/services/dependency_proxy/auth_token_service_spec.rb

+# frozen_string_literal: true
+require 'spec_helper'
+
+RSpec.describe DependencyProxy::AuthTokenService do
+  include DependencyProxyHelpers
+
+  describe '.decoded_token_payload' do
+    let_it_be(:user) { create(:user) }
+    let_it_be(:token) { build_jwt(user) }
+
+    subject { described_class.decoded_token_payload(token.encoded) }
+
+    it 'returns the user' do
+      result = subject
+
+      expect(result['user_id']).to eq(user.id)
+    end
+
+    it 'raises an error if the token is expired' do
+      travel_to(Time.zone.now + Auth::DependencyProxyAuthenticationService.token_expire_at + 1.minute) do
+        expect { subject }.to raise_error(JWT::ExpiredSignature)
+      end
+    end
+
+    it 'raises an error if decoding fails' do
+      allow(JWT).to receive(:decode).and_raise(JWT::DecodeError)
+
+      expect { subject }.to raise_error(JWT::DecodeError)
+    end
+
+    it 'raises an error if signature is immature' do
+      allow(JWT).to receive(:decode).and_raise(JWT::ImmatureSignature)
+
+      expect { subject }.to raise_error(JWT::ImmatureSignature)
+    end
+  end
+end

spec/support/helpers/dependency_proxy_helpers.rb

@@ -25,6 +25,13 @@ module DependencyProxyHelpers
       .to_return(status: status, body: body)
   end
 
+  def build_jwt(user = nil, expire_time: nil)
+    JSONWebToken::HMACToken.new(::Auth::DependencyProxyAuthenticationService.secret).tap do |jwt|
+      jwt['user_id'] = user.id if user
+      jwt.expire_time = expire_time || jwt.issued_at + 1.minute
+    end
+  end
+
   private
 
   def registry