Merge branch '241678-python-name-normalization' into 'master'

Search pypi packages by normalized name

See merge request gitlab-org/gitlab!44807

app/models/packages/package.rb

@@ -52,6 +52,7 @@ class Packages::Package < ApplicationRecord
 
   scope :with_name, ->(name) { where(name: name) }
   scope :with_name_like, ->(name) { where(arel_table[:name].matches(name)) }
+  scope :with_normalized_pypi_name, ->(name) { where("LOWER(regexp_replace(name, '[-_.]+', '-', 'g')) = ?", name.downcase) }
   scope :search_by_name, ->(query) { fuzzy_search(query, [:name], use_minimum_char_limit: false) }
   scope :with_version, ->(version) { where(version: version) }
   scope :without_version_like, -> (version) { where.not(arel_table[:version].matches(version)) }

changelogs/unreleased/241678-python-name-normalization.yml

+---
+title: Search for python packages with normalized name to allow installs of packages
+  with periods and underscores
+merge_request: 44807
+author:
+type: changed

doc/user/packages/pypi_repository/index.md

@@ -111,6 +111,11 @@ touch setup.py
 
 This file contains all the information about our package. For more information
 about this file, see [creating setup.py](https://packaging.python.org/tutorials/packaging-projects/#creating-setup-py).
+GitLab identifies packages based on
+[Python normalized names (PEP-503)](https://www.python.org/dev/peps/pep-0503/#normalized-names),
+so ensure your package name meets these requirements.
+See the [installation section](#install-packages) for more details.
+
 For this guide, we don't need to extensively fill out this file, simply add the
 below to your `setup.py`:
 
@@ -300,6 +305,12 @@ Installing collected packages: mypypipackage
 Successfully installed mypypipackage-0.0.1
 ```
 
+GitLab looks for packages using
+[Python normalized names (PEP-503)](https://www.python.org/dev/peps/pep-0503/#normalized-names),
+so the characters `-`, `_`, and `.` are all treated the same and repeated characters are removed.
+A `pip install` request for `my.package` looks for packages that match any of
+the three characters, such as `my-package`, `my_package`, and `my....package`.
+
 ## Using GitLab CI with PyPI packages
 
 > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/202012) in GitLab 13.4.

lib/api/pypi_packages.rb

@@ -33,7 +33,7 @@ module API
 
       def find_package_versions
         packages = packages_finder
-          .with_name(params[:package_name])
+          .with_normalized_pypi_name(params[:package_name])
 
         not_found!('Package') if packages.empty?
 

spec/models/packages/package_spec.rb

@@ -516,6 +516,14 @@ RSpec.describe Packages::Package, type: :model do
 
       it { is_expected.to match_array([package1, package2]) }
     end
+
+    describe '.with_normalized_pypi_name' do
+      let_it_be(:pypi_package) { create(:pypi_package, name: 'Foo.bAr---BAZ_buz') }
+
+      subject { described_class.with_normalized_pypi_name('foo-bar-baz-buz') }
+
+      it { is_expected.to match_array([pypi_package]) }
+    end
   end
 
   describe '.select_distinct_name' do

spec/requests/api/pypi_packages_spec.rb

@@ -57,6 +57,16 @@ RSpec.describe API::PypiPackages do
       end
     end
 
+    context 'with a normalized package name' do
+      let_it_be(:package) { create(:pypi_package, project: project, name: 'my.package') }
+      let(:url) { "/projects/#{project.id}/packages/pypi/simple/my-package" }
+      let(:headers) { basic_auth_header(user.username, personal_access_token.token) }
+
+      subject { get api(url), headers: headers }
+
+      it_behaves_like 'PyPI package versions', :developer, :success
+    end
+
     it_behaves_like 'deploy token for package GET requests'
 
     it_behaves_like 'job token for package GET requests'