Merge branch '235490-generic-packages/generic-type' into 'master'

Add new "Generic" package type

See merge request gitlab-org/gitlab!40061

app/models/packages/package.rb

@@ -38,8 +38,12 @@ class Packages::Package < ApplicationRecord
   validates :version, format: { with: Gitlab::Regex.conan_recipe_component_regex }, if: :conan?
   validates :version, format: { with: Gitlab::Regex.maven_version_regex }, if: -> { version? && maven? }
   validates :version, format: { with: Gitlab::Regex.pypi_version_regex }, if: :pypi?
+  validates :version,
+    presence: true,
+    format: { with: Gitlab::Regex.generic_package_version_regex },
+    if: :generic?
 
-  enum package_type: { maven: 1, npm: 2, conan: 3, nuget: 4, pypi: 5, composer: 6 }
+  enum package_type: { maven: 1, npm: 2, conan: 3, nuget: 4, pypi: 5, composer: 6, generic: 7 }
 
   scope :with_name, ->(name) { where(name: name) }
   scope :with_name_like, ->(name) { where(arel_table[:name].matches(name)) }

changelogs/unreleased/235490-generic-packages-generic-type.yml

+---
+title: Add new "generic" package type
+merge_request: 40061
+author:
+type: added

db/migrate/20200821034419_add_unique_index_for_generic_packages.rb

+# frozen_string_literal: true
+
+class AddUniqueIndexForGenericPackages < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  INDEX_NAME = 'index_packages_on_project_id_name_version_unique_when_generic'
+  PACKAGE_TYPE_GENERIC = 7
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :packages_packages, [:project_id, :name, :version], unique: true, where: "package_type = #{PACKAGE_TYPE_GENERIC}", name: INDEX_NAME
+  end
+
+  def down
+    remove_concurrent_index_by_name(:packages_packages, INDEX_NAME)
+  end
+end

db/migrate/20200824045812_add_generic_package_max_file_size_to_plan_limits.rb

+# frozen_string_literal: true
+
+class AddGenericPackageMaxFileSizeToPlanLimits < ActiveRecord::Migration[6.0]
+  DOWNTIME = false
+
+  def change
+    add_column(:plan_limits, :generic_packages_max_file_size, :bigint, default: 5.gigabytes, null: false)
+  end
+end

db/schema_migrations/20200821034419

+ddf3452bb44437324d20c9db03e998f8903f5ff9732d29cf85dd5d579507952d
\ No newline at end of file

db/schema_migrations/20200824045812

+4f3528d7df6e61c8b14911644f9223ac5f6e678184d1c8370d1e9a60389cd60c
\ No newline at end of file

db/structure.sql

@@ -14195,7 +14195,8 @@ CREATE TABLE public.plan_limits (
     maven_max_file_size bigint DEFAULT 52428800 NOT NULL,
     npm_max_file_size bigint DEFAULT 52428800 NOT NULL,
     nuget_max_file_size bigint DEFAULT 52428800 NOT NULL,
-    pypi_max_file_size bigint DEFAULT 52428800 NOT NULL
+    pypi_max_file_size bigint DEFAULT 52428800 NOT NULL,
+    generic_packages_max_file_size bigint DEFAULT '5368709120'::bigint NOT NULL
 );
 
 CREATE SEQUENCE public.plan_limits_id_seq
@@ -20414,6 +20415,8 @@ CREATE INDEX index_packages_maven_metadata_on_package_id_and_path ON public.pack
 
 CREATE INDEX index_packages_nuget_dl_metadata_on_dependency_link_id ON public.packages_nuget_dependency_link_metadata USING btree (dependency_link_id);
 
+CREATE UNIQUE INDEX index_packages_on_project_id_name_version_unique_when_generic ON public.packages_packages USING btree (project_id, name, version) WHERE (package_type = 7);
+
 CREATE INDEX index_packages_package_files_file_store_is_null ON public.packages_package_files USING btree (id) WHERE (file_store IS NULL);
 
 CREATE INDEX index_packages_package_files_on_file_store ON public.packages_package_files USING btree (file_store);

doc/api/graphql/reference/gitlab_schema.graphql

@@ -10565,6 +10565,11 @@ enum PackageTypeEnum {
   """
   CONAN
 
+  """
+  Packages from the generic package manager
+  """
+  GENERIC
+
   """
   Packages from the maven package manager
   """

doc/api/graphql/reference/gitlab_schema.json

@@ -31806,6 +31806,12 @@
               "description": "Packages from the composer package manager",
               "isDeprecated": false,
               "deprecationReason": null
+            },
+            {
+              "name": "GENERIC",
+              "description": "Packages from the generic package manager",
+              "isDeprecated": false,
+              "deprecationReason": null
             }
           ],
           "possibleTypes": null

lib/gitlab/regex.rb

@@ -104,6 +104,10 @@ module Gitlab
           \b (?# word boundary)
         /ix.freeze
       end
+
+      def generic_package_version_regex
+        /\A\d+\.\d+\.\d+\z/
+      end
     end
 
     extend self

spec/factories/packages.rb

@@ -121,6 +121,12 @@ FactoryBot.define do
         conan_metadatum { build(:conan_metadatum, package: nil) }
       end
     end
+
+    factory :generic_package do
+      sequence(:name) { |n| "generic-package-#{n}" }
+      version { '1.0.0' }
+      package_type { :generic }
+    end
   end
 
   factory :composer_metadatum, class: 'Packages::Composer::Metadatum' do

spec/graphql/types/package_type_enum_spec.rb

@@ -4,6 +4,6 @@ require 'spec_helper'
 
 RSpec.describe GitlabSchema.types['PackageTypeEnum'] do
   it 'exposes all package types' do
-    expect(described_class.values.keys).to contain_exactly(*%w[MAVEN NPM CONAN NUGET PYPI COMPOSER])
+    expect(described_class.values.keys).to contain_exactly(*%w[MAVEN NPM CONAN NUGET PYPI COMPOSER GENERIC])
   end
 end

spec/lib/gitlab/regex_spec.rb

@@ -426,4 +426,21 @@ RSpec.describe Gitlab::Regex do
     it { is_expected.not_to match('1.2') }
     it { is_expected.not_to match('1./2.3') }
   end
+
+  describe '.generic_package_version_regex' do
+    subject { described_class.generic_package_version_regex }
+
+    it { is_expected.to match('1.2.3') }
+    it { is_expected.to match('1.3.350') }
+    it { is_expected.not_to match('1.3.350-20201230123456') }
+    it { is_expected.not_to match('..1.2.3') }
+    it { is_expected.not_to match('  1.2.3') }
+    it { is_expected.not_to match("1.2.3  \r\t") }
+    it { is_expected.not_to match("\r\t 1.2.3") }
+    it { is_expected.not_to match('1.2.3-4/../../') }
+    it { is_expected.not_to match('1.2.3-4%2e%2e%') }
+    it { is_expected.not_to match('../../../../../1.2.3') }
+    it { is_expected.not_to match('%2e%2e%2f1.2.3') }
+    it { is_expected.not_to match('') }
+  end
 end

spec/models/packages/package_spec.rb

@@ -236,6 +236,25 @@ RSpec.describe Packages::Package, type: :model do
         it { is_expected.not_to allow_value('%2e%2e%2f1.2.3').for(:version) }
       end
 
+      context 'generic package' do
+        subject { build_stubbed(:generic_package) }
+
+        it { is_expected.to validate_presence_of(:version) }
+        it { is_expected.to allow_value('1.2.3').for(:version) }
+        it { is_expected.to allow_value('1.3.350').for(:version) }
+        it { is_expected.not_to allow_value('1.3.350-20201230123456').for(:version) }
+        it { is_expected.not_to allow_value('..1.2.3').for(:version) }
+        it { is_expected.not_to allow_value('  1.2.3').for(:version) }
+        it { is_expected.not_to allow_value("1.2.3  \r\t").for(:version) }
+        it { is_expected.not_to allow_value("\r\t 1.2.3").for(:version) }
+        it { is_expected.not_to allow_value('1.2.3-4/../../').for(:version) }
+        it { is_expected.not_to allow_value('1.2.3-4%2e%2e%').for(:version) }
+        it { is_expected.not_to allow_value('../../../../../1.2.3').for(:version) }
+        it { is_expected.not_to allow_value('%2e%2e%2f1.2.3').for(:version) }
+        it { is_expected.not_to allow_value('').for(:version) }
+        it { is_expected.not_to allow_value(nil).for(:version) }
+      end
+
       it_behaves_like 'validating version to be SemVer compliant for', :npm_package
       it_behaves_like 'validating version to be SemVer compliant for', :nuget_package
     end
@@ -552,11 +571,17 @@ RSpec.describe Packages::Package, type: :model do
 
   describe 'plan_limits' do
     Packages::Package.package_types.keys.without('composer').each do |pt|
+      plan_limit_name = if pt == 'generic'
+                          "#{pt}_packages_max_file_size"
+                        else
+                          "#{pt}_max_file_size"
+                        end
+
       context "File size limits for #{pt}" do
         let(:package) { create("#{pt}_package") }
 
-        it "plan_limits includes column #{pt}_max_file_size" do
-          expect { package.project.actual_limits.send("#{pt}_max_file_size") }
+        it "plan_limits includes column #{plan_limit_name}" do
+          expect { package.project.actual_limits.send(plan_limit_name) }
             .not_to raise_error(NoMethodError)
         end
       end

spec/support/shared_examples/services/packages_shared_examples.rb

@@ -161,6 +161,7 @@ RSpec.shared_examples 'filters on each package_type' do |is_project: false|
   let_it_be(:package4) { create(:nuget_package, project: project) }
   let_it_be(:package5) { create(:pypi_package, project: project) }
   let_it_be(:package6) { create(:composer_package, project: project) }
+  let_it_be(:package7) { create(:generic_package, project: project) }
 
   Packages::Package.package_types.keys.each do |package_type|
     context "for package type #{package_type}" do