Merge branch '218017-nuget-size-limits-db' into 'master'

Add maximum file upload size limit for the package registry

See merge request gitlab-org/gitlab!39633

app/services/packages/npm/create_package_service.rb

@@ -7,6 +7,7 @@ module Packages
       def execute
         return error('Version is empty.', 400) if version.blank?
         return error('Package already exists.', 403) if current_package_exists?
+        return error('File is too large.', 400) if file_size_exceeded?
 
         ActiveRecord::Base.transaction { create_package! }
       end
@@ -86,6 +87,10 @@ module Packages
         _version, versions_data = params[:versions].first
         versions_data
       end
+
+      def file_size_exceeded?
+        project.actual_limits.exceeded?(:npm_max_file_size, attachment['length'].to_i)
+      end
     end
   end
 end

changelogs/unreleased/218017-nuget-size-limits-db.yml

+---
+title: Add package file size limits to plan limits
+merge_request: 39633
+author:
+type: added

db/migrate/20200818171229_add_package_max_file_size_to_plan_limits.rb

+# frozen_string_literal: true
+
+class AddPackageMaxFileSizeToPlanLimits < ActiveRecord::Migration[6.0]
+  DOWNTIME = false
+
+  def change
+    add_column(:plan_limits, :conan_max_file_size, :bigint, default: 50.megabytes, null: false)
+    add_column(:plan_limits, :maven_max_file_size, :bigint, default: 50.megabytes, null: false)
+    add_column(:plan_limits, :npm_max_file_size, :bigint, default: 50.megabytes, null: false)
+    add_column(:plan_limits, :nuget_max_file_size, :bigint, default: 50.megabytes, null: false)
+    add_column(:plan_limits, :pypi_max_file_size, :bigint, default: 50.megabytes, null: false)
+  end
+end

db/schema_migrations/20200818171229

+987f316571f41ad679cad54089bc523f62d04691c10e5cf1957cf60edd71f889
\ No newline at end of file

db/structure.sql

@@ -14103,7 +14103,12 @@ CREATE TABLE public.plan_limits (
     ci_max_artifact_size_coverage_fuzzing integer DEFAULT 0 NOT NULL,
     ci_max_artifact_size_browser_performance integer DEFAULT 0 NOT NULL,
     ci_max_artifact_size_load_performance integer DEFAULT 0 NOT NULL,
-    ci_needs_size_limit integer DEFAULT 50 NOT NULL
+    ci_needs_size_limit integer DEFAULT 50 NOT NULL,
+    conan_max_file_size bigint DEFAULT 52428800 NOT NULL,
+    maven_max_file_size bigint DEFAULT 52428800 NOT NULL,
+    npm_max_file_size bigint DEFAULT 52428800 NOT NULL,
+    nuget_max_file_size bigint DEFAULT 52428800 NOT NULL,
+    pypi_max_file_size bigint DEFAULT 52428800 NOT NULL
 );
 
 CREATE SEQUENCE public.plan_limits_id_seq

doc/administration/instance_limits.md

@@ -514,3 +514,38 @@ Total number of changes (branches or tags) in a single push to determine whether
 individual push events or bulk push event will be created.
 
 More information can be found in the [Push event activities limit and bulk push events documentation](../user/admin_area/settings/push_event_activities_limit.md).
+
+## Package Registry Limits
+
+### File Size Limits
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/218017) in GitLab 13.4.
+
+On GitLab.com, the maximum file size for a package that's uploaded to the [GitLab Package Registry](../user/packages/package_registry/index.md)
+is 50 megabytes.
+
+Limits are set per package type.
+
+To set this limit on a self-managed installation, run the following in the
+[GitLab Rails console](troubleshooting/debug.md#starting-a-rails-console-session):
+
+```ruby
+# File size limit is stored in bytes
+
+# For Conan Packages
+Plan.default.actual_limits.update!(conan_max_file_size: 100.megabytes)
+
+# For NPM Packages
+Plan.default.actual_limits.update!(npm_max_file_size: 100.megabytes)
+
+# For NuGet Packages
+Plan.default.actual_limits.update!(nuget_max_file_size: 100.megabytes)
+
+# For Maven Packages
+Plan.default.actual_limits.update!(maven_max_file_size: 100.megabytes)
+
+# For PyPI Packages
+Plan.default.actual_limits.update!(pypi_max_file_size: 100.megabytes)
+```
+
+Set the limit to `0` to allow any file size.

lib/api/conan_packages.rb

@@ -293,7 +293,7 @@ module API
           route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true
 
           put 'authorize' do
-            authorize_workhorse!(subject: project)
+            authorize_workhorse!(subject: project, maximum_size: project.actual_limits.conan_max_file_size)
           end
         end
 
@@ -320,7 +320,7 @@ module API
           route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true
 
           put 'authorize' do
-            authorize_workhorse!(subject: project)
+            authorize_workhorse!(subject: project, maximum_size: project.actual_limits.conan_max_file_size)
           end
 
           desc 'Upload package files' do

lib/api/helpers/packages/conan/api_helpers.rb

@@ -155,6 +155,7 @@ module API
 
           def upload_package_file(file_type)
             authorize_upload!(project)
+            bad_request!('File is too large') if project.actual_limits.exceeded?(:conan_max_file_size, params['file.size'].to_i)
 
             current_package = find_or_create_package
 

lib/api/maven_packages.rb

@@ -200,7 +200,7 @@ module API
 
         status 200
         content_type Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE
-        ::Packages::PackageFileUploader.workhorse_authorize(has_length: true)
+        ::Packages::PackageFileUploader.workhorse_authorize(has_length: true, maximum_size: user_project.actual_limits.maven_max_file_size)
       end
 
       desc 'Upload the maven package file' do
@@ -214,6 +214,7 @@ module API
       route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true
       put ':id/packages/maven/*path/:file_name', requirements: MAVEN_ENDPOINT_REQUIREMENTS do
         authorize_upload!
+        bad_request!('File is too large') if user_project.actual_limits.exceeded?(:maven_max_file_size, params[:file].size)
 
         file_name, format = extract_format(params[:file_name])
 

lib/api/nuget_packages.rb

@@ -92,6 +92,7 @@ module API
 
         put do
           authorize_upload!(authorized_user_project)
+          bad_request!('File is too large') if authorized_user_project.actual_limits.exceeded?(:nuget_max_file_size, params[:package].size)
 
           file_params = params.merge(
             file: params[:package],
@@ -118,7 +119,11 @@ module API
         route_setting :authentication, deploy_token_allowed: true, job_token_allowed: :basic_auth, basic_auth_personal_access_token: true
 
         put 'authorize' do
-          authorize_workhorse!(subject: authorized_user_project, has_length: false)
+          authorize_workhorse!(
+            subject: authorized_user_project,
+            has_length: false,
+            maximum_size: authorized_user_project.actual_limits.nuget_max_file_size
+          )
         end
 
         params do

lib/api/pypi_packages.rb

@@ -120,6 +120,7 @@ module API
         route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true
         post do
           authorize_upload!(authorized_user_project)
+          bad_request!('File is too large') if authorized_user_project.actual_limits.exceeded?(:pypi_max_file_size, params[:content].size)
 
           track_event('push_package')
 
@@ -136,7 +137,11 @@ module API
 
         route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true
         post 'authorize' do
-          authorize_workhorse!(subject: authorized_user_project, has_length: false)
+          authorize_workhorse!(
+            subject: authorized_user_project,
+            has_length: false,
+            maximum_size: authorized_user_project.actual_limits.pypi_max_file_size
+          )
         end
       end
     end

spec/models/packages/package_spec.rb

@@ -482,4 +482,17 @@ RSpec.describe Packages::Package, type: :model do
       it { is_expected.to contain_exactly(*tags) }
     end
   end
+
+  describe 'plan_limits' do
+    Packages::Package.package_types.keys.without('composer').each do |pt|
+      context "File size limits for #{pt}" do
+        let(:package) { create("#{pt}_package") }
+
+        it "plan_limits includes column #{pt}_max_file_size" do
+          expect { package.project.actual_limits.send("#{pt}_max_file_size") }
+            .not_to raise_error(NoMethodError)
+        end
+      end
+    end
+  end
 end

spec/requests/api/conan_packages_spec.rb

@@ -681,6 +681,18 @@ RSpec.describe API::ConanPackages do
     let(:recipe_path) { "foo/bar/#{project.full_path.tr('/', '+')}/baz"}
 
     shared_examples 'uploads a package file' do
+      context 'file size above maximum limit' do
+        before do
+          params['file.size'] = project.actual_limits.conan_max_file_size + 1
+        end
+
+        it 'handles as a local file' do
+          subject
+
+          expect(response).to have_gitlab_http_status(:bad_request)
+        end
+      end
+
       context 'with object storage disabled' do
         context 'without a file from workhorse' do
           let(:params) { { file: nil } }

spec/requests/api/maven_packages_spec.rb

@@ -528,6 +528,18 @@ RSpec.describe API::MavenPackages do
     context 'when params from workhorse are correct' do
       let(:params) { { file: file_upload } }
 
+      context 'file size is too large' do
+        it 'rejects the request' do
+          allow_next_instance_of(UploadedFile) do |uploaded_file|
+            allow(uploaded_file).to receive(:size).and_return(project.actual_limits.maven_max_file_size + 1)
+          end
+
+          upload_file_with_token(params)
+
+          expect(response).to have_gitlab_http_status(:bad_request)
+        end
+      end
+
       it 'rejects a malicious request' do
         put api("/projects/#{project.id}/packages/maven/com/example/my-app/#{version}/%2e%2e%2f.ssh%2fauthorized_keys"), params: params, headers: headers_with_token
 

spec/requests/api/nuget_packages_spec.rb

@@ -220,6 +220,18 @@ RSpec.describe API::NugetPackages do
       it_behaves_like 'rejects nuget access with unknown project id'
 
       it_behaves_like 'rejects nuget access with invalid project id'
+
+      context 'file size above maximum limit' do
+        let(:headers) { basic_auth_header(deploy_token.username, deploy_token.token).merge(workhorse_header) }
+
+        before do
+          allow_next_instance_of(UploadedFile) do |uploaded_file|
+            allow(uploaded_file).to receive(:size).and_return(project.actual_limits.nuget_max_file_size + 1)
+          end
+        end
+
+        it_behaves_like 'returning response status', :bad_request
+      end
     end
   end
 

spec/requests/api/pypi_packages_spec.rb

@@ -185,6 +185,18 @@ RSpec.describe API::PypiPackages do
     it_behaves_like 'deploy token for package uploads'
 
     it_behaves_like 'rejects PyPI access with unknown project id'
+
+    context 'file size above maximum limit' do
+      let(:headers) { basic_auth_header(deploy_token.username, deploy_token.token).merge(workhorse_header) }
+
+      before do
+        allow_next_instance_of(UploadedFile) do |uploaded_file|
+          allow(uploaded_file).to receive(:size).and_return(project.actual_limits.pypi_max_file_size + 1)
+        end
+      end
+
+      it_behaves_like 'returning response status', :bad_request
+    end
   end
 
   describe 'GET /api/v4/projects/:id/packages/pypi/files/:sha256/*file_identifier' do

spec/services/packages/npm/create_package_service_spec.rb

@@ -61,6 +61,15 @@ RSpec.describe Packages::Npm::CreatePackageService do
       it { expect(subject[:message]).to be 'Package already exists.' }
     end
 
+    context 'file size above maximum limit' do
+      before do
+        params['_attachments']["#{package_name}-#{version}.tgz"]['length'] = project.actual_limits.npm_max_file_size + 1
+      end
+
+      it { expect(subject[:http_status]).to eq 400 }
+      it { expect(subject[:message]).to be 'File is too large.' }
+    end
+
     context 'with incorrect namespace' do
       let(:package_name) { '@my_other_namespace/my-app' }