Merge branch '213566-deploy-token-basic-auth' into 'master'

Deploy tokens for PyPI and NuGet using basic auth headers

See merge request gitlab-org/gitlab!31035

changelogs/unreleased/213566-deploy-token-basic-auth.yml

+---
+title: Deploy tokens can be used in the API with Basic Auth Headers enabling NuGet
+  and PyPI to be used with deploy tokens
+merge_request: 31035
+author:
+type: added

doc/user/packages/nuget_repository/index.md

@@ -68,7 +68,9 @@ You should then be able to see the **Packages & Registries** section on the left
 You will need the following:
 
 - Your GitLab username.
-- A personal access token. You can generate a [personal access token](../../../user/profile/personal_access_tokens.md) with the scope set to `api` for repository authentication.
+- A personal access token or deploy token. For repository authentication:
+  - You can generate a [personal access token](../../../user/profile/personal_access_tokens.md) with the scope set to `api`.
+  - You can generate a [deploy token](./../../project/deploy_tokens/index.md) with the scope set to `read_package_registry`, `write_package_registry`, or both.
 - A suitable name for your source.
 - Your project ID which can be found on the home page of your project.
 
@@ -83,7 +85,7 @@ You can now add a new source to NuGet with:
 To add the GitLab NuGet Repository as a source with `nuget`:
 
 ```shell
-nuget source Add -Name <source_name> -Source "https://gitlab-instance.example.com/api/v4/projects/<your_project_id>/packages/nuget/index.json" -UserName <gitlab_username> -Password <gitlab_personal_access_token>
+nuget source Add -Name <source_name> -Source "https://gitlab-instance.example.com/api/v4/projects/<your_project_id>/packages/nuget/index.json" -UserName <gitlab_username or deploy_token_username> -Password <gitlab_personal_access_token or deploy_token>
 ```
 
 Where:
@@ -107,8 +109,8 @@ nuget source Add -Name "GitLab" -Source "https//gitlab.example/api/v4/projects/1
    - **Location**: `https://gitlab.com/api/v4/projects/<your_project_id>/packages/nuget/index.json`
      - Replace `<your_project_id>` with your project ID.
      - If you have a self-managed GitLab installation, replace `gitlab.com` with your domain name.
-   - **Username**: Your GitLab username
-   - **Password**: Your personal access token
+   - **Username**: Your GitLab username or deploy token username
+   - **Password**: Your personal access token or deploy token
 
    ![Visual Studio Adding a NuGet source](img/visual_studio_adding_nuget_source.png)
 
@@ -131,8 +133,8 @@ To add the GitLab NuGet Repository as a source for .NET, create a file named `nu
     </packageSources>
     <packageSourceCredentials>
         <gitlab>
-            <add key="Username" value="<gitlab_username>" />
-            <add key="ClearTextPassword" value="<gitlab_personal_access_token>" />
+            <add key="Username" value="<gitlab_username or deploy_token_username>" />
+            <add key="ClearTextPassword" value="<gitlab_personal_access_token or deploy_token>" />
         </gitlab>
     </packageSourceCredentials>
 </configuration>

doc/user/packages/pypi_repository/index.md

@@ -150,6 +150,8 @@ Package Registry**. Before we do so, we next need to set up authentication.
 
 ## Adding the GitLab PyPi Repository as a source
 
+### Authenticating with a personal access token
+
 You will need the following:
 
 - A personal access token. You can generate a [personal access token](../../../user/profile/personal_access_tokens.md) with the scope set to `api` for repository authentication.
@@ -169,6 +171,27 @@ username = __token__
 password = <your personal access token>
 ```
 
+### Authenticating with a deploy token
+
+You will need the following:
+
+- A deploy token. You can generate a [deploy token](./../../project/deploy_tokens/index.md) with the `read_package_registry` and/or `write_package_registry` scopes for repository authentication.
+- A suitable name for your source.
+- Your project ID which can be found on the home page of your project.
+
+Edit your `~/.pypirc` file and add the following:
+
+```ini
+[distutils]
+index-servers =
+    gitlab
+
+[gitlab]
+repository = https://gitlab.com/api/v4/projects/<project_id>/packages/pypi
+username = <deploy token username>
+password = <deploy token>
+```
+
 ## Uploading packages
 
 When uploading packages, note that:

ee/lib/api/nuget_packages.rb

@@ -54,6 +54,7 @@ module API
     params do
       requires :id, type: String, desc: 'The ID of a project', regexp: POSITIVE_INTEGER_REGEX
     end
+    route_setting :authentication, deploy_token_allowed: true
     resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
       before do
         authorize_packages_feature!(authorized_user_project)
@@ -64,6 +65,7 @@ module API
         desc 'The NuGet Service Index' do
           detail 'This feature was introduced in GitLab 12.6'
         end
+        route_setting :authentication, deploy_token_allowed: true
         get 'index', format: :json do
           authorize_read_package!(authorized_user_project)
 
@@ -80,6 +82,7 @@ module API
         params do
           requires :package, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)'
         end
+        route_setting :authentication, deploy_token_allowed: true
         put do
           authorize_upload!(authorized_user_project)
 
@@ -104,6 +107,7 @@ module API
 
           forbidden!
         end
+        route_setting :authentication, deploy_token_allowed: true
         put 'authorize' do
           authorize_workhorse!(subject: authorized_user_project, has_length: false)
         end
@@ -120,6 +124,7 @@ module API
           desc 'The NuGet Metadata Service - Package name level' do
             detail 'This feature was introduced in GitLab 12.8'
           end
+          route_setting :authentication, deploy_token_allowed: true
           get 'index', format: :json do
             present ::Packages::Nuget::PackagesMetadataPresenter.new(find_packages),
                     with: EE::API::Entities::Nuget::PackagesMetadata
@@ -131,6 +136,7 @@ module API
           params do
             requires :package_version, type: String, desc: 'The NuGet package version', regexp: API::NO_SLASH_URL_PART_REGEX
           end
+          route_setting :authentication, deploy_token_allowed: true
           get '*package_version', format: :json do
             present ::Packages::Nuget::PackageMetadataPresenter.new(find_package),
                     with: EE::API::Entities::Nuget::PackageMetadata
@@ -149,6 +155,7 @@ module API
           desc 'The NuGet Content Service - index request' do
             detail 'This feature was introduced in GitLab 12.8'
           end
+          route_setting :authentication, deploy_token_allowed: true
           get 'index', format: :json do
             present ::Packages::Nuget::PackagesVersionsPresenter.new(find_packages),
                     with: EE::API::Entities::Nuget::PackagesVersions
@@ -161,6 +168,7 @@ module API
             requires :package_version, type: String, desc: 'The NuGet package version', regexp: API::NO_SLASH_URL_PART_REGEX
             requires :package_filename, type: String, desc: 'The NuGet package filename', regexp: API::NO_SLASH_URL_PART_REGEX
           end
+          route_setting :authentication, deploy_token_allowed: true
           get '*package_version/*package_filename', format: :nupkg do
             filename = "#{params[:package_filename]}.#{params[:format]}"
             package_file = ::Packages::PackageFileFinder.new(find_package, filename, with_file_name_like: true)
@@ -190,6 +198,7 @@ module API
           desc 'The NuGet Search Service' do
             detail 'This feature was introduced in GitLab 12.8'
           end
+          route_setting :authentication, deploy_token_allowed: true
           get format: :json do
             search_options = {
               include_prerelease_versions: params[:prerelease],

ee/lib/api/pypi_packages.rb

@@ -68,6 +68,7 @@ module API
           requires :sha256, type: String, desc: 'The PyPi package sha256 check sum'
         end
 
+        route_setting :authentication, deploy_token_allowed: true
         get 'files/:sha256/*file_identifier' do
           project = unauthorized_user_project!
 
@@ -88,6 +89,7 @@ module API
 
         # An Api entry point but returns an HTML file instead of JSON.
         # PyPi simple API returns the package descriptor as a simple HTML file.
+        route_setting :authentication, deploy_token_allowed: true
         get 'simple/*package_name', format: :txt do
           authorize_read_package!(authorized_user_project)
 
@@ -115,6 +117,7 @@ module API
           optional :sha256_digest, type: String
         end
 
+        route_setting :authentication, deploy_token_allowed: true
         post do
           authorize_upload!(authorized_user_project)
 
@@ -129,6 +132,7 @@ module API
           forbidden!
         end
 
+        route_setting :authentication, deploy_token_allowed: true
         post 'authorize' do
           authorize_workhorse!(subject: authorized_user_project, has_length: false)
         end

ee/spec/requests/api/nuget_packages_spec.rb

@@ -8,6 +8,8 @@ describe API::NugetPackages do
   let_it_be(:user) { create(:user) }
   let_it_be(:project, reload: true) { create(:project, :public) }
   let_it_be(:personal_access_token) { create(:personal_access_token, user: user) }
+  let_it_be(:deploy_token) { create(:deploy_token, read_package_registry: true, write_package_registry: true) }
+  let_it_be(:project_deploy_token) { create(:project_deploy_token, deploy_token: deploy_token, project: project) }
 
   describe 'GET /api/v4/projects/:id/packages/nuget' do
     let(:url) { "/projects/#{project.id}/packages/nuget/index.json" }
@@ -57,6 +59,8 @@ describe API::NugetPackages do
         end
       end
 
+      it_behaves_like 'deploy token for package GET requests'
+
       it_behaves_like 'rejects nuget access with unknown project id'
 
       it_behaves_like 'rejects nuget access with invalid project id'
@@ -115,6 +119,8 @@ describe API::NugetPackages do
         end
       end
 
+      it_behaves_like 'deploy token for package uploads'
+
       it_behaves_like 'rejects nuget access with unknown project id'
 
       it_behaves_like 'rejects nuget access with invalid project id'
@@ -186,6 +192,8 @@ describe API::NugetPackages do
         end
       end
 
+      it_behaves_like 'deploy token for package uploads'
+
       it_behaves_like 'rejects nuget access with unknown project id'
 
       it_behaves_like 'rejects nuget access with invalid project id'
@@ -244,6 +252,8 @@ describe API::NugetPackages do
           it_behaves_like params[:shared_examples_name], params[:user_role], params[:expected_status], params[:member]
         end
 
+        it_behaves_like 'deploy token for package GET requests'
+
         it_behaves_like 'rejects nuget access with unknown project id'
 
         it_behaves_like 'rejects nuget access with invalid project id'
@@ -304,6 +314,8 @@ describe API::NugetPackages do
         end
       end
 
+      it_behaves_like 'deploy token for package GET requests'
+
       context 'with invalid package name' do
         let_it_be(:package_name) { 'Unkown' }
 
@@ -364,6 +376,8 @@ describe API::NugetPackages do
         end
       end
 
+      it_behaves_like 'deploy token for package GET requests'
+
       it_behaves_like 'rejects nuget access with unknown project id'
 
       it_behaves_like 'rejects nuget access with invalid project id'
@@ -423,6 +437,8 @@ describe API::NugetPackages do
         end
       end
 
+      it_behaves_like 'deploy token for package GET requests'
+
       it_behaves_like 'rejects nuget access with unknown project id'
 
       it_behaves_like 'rejects nuget access with invalid project id'
@@ -490,6 +506,8 @@ describe API::NugetPackages do
         end
       end
 
+      it_behaves_like 'deploy token for package GET requests'
+
       it_behaves_like 'rejects nuget access with unknown project id'
 
       it_behaves_like 'rejects nuget access with invalid project id'

ee/spec/requests/api/pypi_packages_spec.rb

@@ -8,6 +8,8 @@ describe API::PypiPackages do
   let_it_be(:user) { create(:user) }
   let_it_be(:project, reload: true) { create(:project, :public) }
   let_it_be(:personal_access_token) { create(:personal_access_token, user: user) }
+  let_it_be(:deploy_token) { create(:deploy_token, read_package_registry: true, write_package_registry: true) }
+  let_it_be(:project_deploy_token) { create(:project_deploy_token, deploy_token: deploy_token, project: project) }
 
   describe 'GET /api/v4/projects/:id/packages/pypi/simple/:package_name' do
     let_it_be(:package) { create(:pypi_package, project: project) }
@@ -58,6 +60,8 @@ describe API::PypiPackages do
         end
       end
 
+      it_behaves_like 'deploy token for package GET requests'
+
       it_behaves_like 'rejects PyPI access with unknown project id'
     end
 
@@ -114,6 +118,8 @@ describe API::PypiPackages do
         end
       end
 
+      it_behaves_like 'deploy token for package uploads'
+
       it_behaves_like 'rejects PyPI access with unknown project id'
     end
 
@@ -196,6 +202,8 @@ describe API::PypiPackages do
         it_behaves_like 'returning response status', :bad_request
       end
 
+      it_behaves_like 'deploy token for package uploads'
+
       it_behaves_like 'rejects PyPI access with unknown project id'
     end
 
@@ -253,6 +261,20 @@ describe API::PypiPackages do
         end
       end
 
+      context 'with deploy token headers' do
+        let(:headers) { build_basic_auth_header(deploy_token.username, deploy_token.token) }
+
+        context 'valid token' do
+          it_behaves_like 'returning response status', :success
+        end
+
+        context 'invalid token' do
+          let(:headers) { build_basic_auth_header('foo', 'bar') }
+
+          it_behaves_like 'returning response status', :success
+        end
+      end
+
       it_behaves_like 'rejects PyPI access with unknown project id'
     end
 

ee/spec/support/shared_examples/requests/api/packages_shared_examples.rb

+# frozen_string_literal: true
+
+RSpec.shared_examples 'deploy token for package GET requests' do
+  context 'with deploy token headers' do
+    let(:headers) { build_basic_auth_header(deploy_token.username, deploy_token.token) }
+
+    subject { get api(url), headers: headers }
+
+    before do
+      project.update!(visibility_level: Gitlab::VisibilityLevel::PRIVATE)
+    end
+
+    context 'valid token' do
+      it_behaves_like 'returning response status', :success
+    end
+
+    context 'invalid token' do
+      let(:headers) { build_basic_auth_header(deploy_token.username, 'bar') }
+
+      it_behaves_like 'returning response status', :unauthorized
+    end
+  end
+end
+
+RSpec.shared_examples 'deploy token for package uploads' do
+  context 'with deploy token headers' do
+    let(:headers) { build_basic_auth_header(deploy_token.username, deploy_token.token).merge(workhorse_header) }
+
+    before do
+      project.update!(visibility_level: Gitlab::VisibilityLevel::PRIVATE)
+    end
+
+    context 'valid token' do
+      it_behaves_like 'returning response status', :success
+    end
+
+    context 'invalid token' do
+      let(:headers) { build_basic_auth_header(deploy_token.username, 'bar').merge(workhorse_header) }
+
+      it_behaves_like 'returning response status', :unauthorized
+    end
+  end
+end

lib/gitlab/auth/auth_finders.rb

@@ -104,11 +104,17 @@ module Gitlab
 
       # This returns a deploy token, not a user since a deploy token does not
       # belong to a user.
+      #
+      # deploy tokens are accepted with deploy token headers and basic auth headers
       def deploy_token_from_request
         return unless route_authentication_setting[:deploy_token_allowed]
 
         token = current_request.env[DEPLOY_TOKEN_HEADER].presence || parsed_oauth_token
 
+        if has_basic_credentials?(current_request)
+          _, token = user_name_and_password(current_request)
+        end
+
         deploy_token = DeployToken.active.find_by_token(token)
         @current_authenticated_deploy_token = deploy_token # rubocop:disable Gitlab/ModuleWithInstanceVariables
 

spec/lib/gitlab/auth/auth_finders_spec.rb

@@ -21,6 +21,13 @@ describe Gitlab::Auth::AuthFinders do
     env[key] = value
   end
 
+  def set_basic_auth_header(username, password)
+    set_header(
+      'HTTP_AUTHORIZATION',
+      ActionController::HttpAuthentication::Basic.encode_credentials(username, password)
+    )
+  end
+
   describe '#find_user_from_warden' do
     context 'with CSRF token' do
       before do
@@ -238,6 +245,24 @@ describe Gitlab::Auth::AuthFinders do
         it { is_expected.to be_nil }
       end
     end
+
+    context 'with basic auth headers' do
+      before do
+        set_basic_auth_header(deploy_token.username, deploy_token.token)
+      end
+
+      it { is_expected.to eq deploy_token }
+
+      it_behaves_like 'an unauthenticated route'
+
+      context 'with incorrect token' do
+        before do
+          set_basic_auth_header(deploy_token.username, 'invalid')
+        end
+
+        it { is_expected.to be_nil }
+      end
+    end
   end
 
   describe '#find_user_from_access_token' do
@@ -394,7 +419,7 @@ describe Gitlab::Auth::AuthFinders do
 
   describe '#find_personal_access_token_from_http_basic_auth' do
     def auth_header_with(token)
-      set_header('HTTP_AUTHORIZATION', ActionController::HttpAuthentication::Basic.encode_credentials('username', token))
+      set_basic_auth_header('username', token)
     end
 
     context 'access token is valid' do
@@ -441,14 +466,6 @@ describe Gitlab::Auth::AuthFinders do
   end
 
   describe '#find_user_from_basic_auth_job' do
-    def basic_http_auth(username, password)
-      ActionController::HttpAuthentication::Basic.encode_credentials(username, password)
-    end
-
-    def set_auth(username, password)
-      set_header('HTTP_AUTHORIZATION', basic_http_auth(username, password))
-    end
-
     subject { find_user_from_basic_auth_job }
 
     context 'when the request does not have AUTHORIZATION header' do
@@ -457,25 +474,25 @@ describe Gitlab::Auth::AuthFinders do
 
     context 'with wrong credentials' do
       it 'returns nil without user and password' do
-        set_auth(nil, nil)
+        set_basic_auth_header(nil, nil)
 
         is_expected.to be_nil
       end
 
       it 'returns nil without password' do
-        set_auth('some-user', nil)
+        set_basic_auth_header('some-user', nil)
 
         is_expected.to be_nil
       end
 
       it 'returns nil without user' do
-        set_auth(nil, 'password')
+        set_basic_auth_header(nil, 'password')
 
         is_expected.to be_nil
       end
 
       it 'returns nil without CI username' do
-        set_auth('user', 'password')
+        set_basic_auth_header('user', 'password')
 
         is_expected.to be_nil
       end
@@ -487,19 +504,19 @@ describe Gitlab::Auth::AuthFinders do
       let(:build) { create(:ci_build, user: user) }
 
       it 'returns nil without password' do
-        set_auth(username, nil)
+        set_basic_auth_header(username, nil)
 
         is_expected.to be_nil
       end
 
       it 'returns user with valid token' do
-        set_auth(username, build.token)
+        set_basic_auth_header(username, build.token)
 
         is_expected.to eq user
       end
 
       it 'raises error with invalid token' do
-        set_auth(username, 'token')
+        set_basic_auth_header(username, 'token')
 
         expect { subject }.to raise_error(Gitlab::Auth::UnauthorizedError)
       end