Skip to content

G
gitlab-ce
Commit e2806cd0 authored by Giorgenes Gelatti's avatar Giorgenes Gelatti Committed by James Lopez
Browse files
Options

PyPi package install support 

- Lists pypi package versions from the API
- Downloads package file endpoint
parent b9997292
Hide whitespace changes
Inline Side-by-side
Showing with 253 additions and 37 deletions
ee/app/models/packages/package.rb
View file @ e2806cd0
...@@ -92,6 +92,11 @@ class Packages::Package < ApplicationRecord ...@@ -92,6 +92,11 @@ class Packages::Package < ApplicationRecord
.where(packages_package_files: { file_name: file_name }).last! .where(packages_package_files: { file_name: file_name }).last!
end end
def self.by_file_name_and_sha256(file_name, sha256)
joins(:package_files)
.where(packages_package_files: { file_name: file_name, file_sha256: sha256 }).last!
end
def self.pluck_names def self.pluck_names
pluck(:name) pluck(:name)
end end
......
ee/app/presenters/packages/pypi/package_presenter.rb 0 → 100644
View file @ e2806cd0
# frozen_string_literal: true
# Display package version data acording to PyPi
# Simple API: https://warehouse.pypa.io/api-reference/legacy/#simple-project-api
module Packages
module Pypi
class PackagePresenter
include API::Helpers::RelatedResourcesHelpers
def initialize(packages, project)
@packages = packages
@project = project
end
# Returns the HTML body for PyPi simple API.
# Basically a list of package download links for a specific
# package
def body
<<-HTML
<!DOCTYPE html>
<html>
<head>
<title>Links for #{name}</title>
</head>
<body>
<h1>Links for #{name}</h1>
#{links}
</body>
</html>
HTML
end
private
def links
refs = []
@packages.map do |package|
package.package_files.each do |file|
url = build_pypi_package_path(file)
refs << package_link(url, package.pypi_metadatum.required_python, file.file_name)
end
end
refs.join
end
def package_link(url, required_python, filename)
"<a href=\"#{url}\" data-requires-python=\"#{required_python}\">#{filename}</a><br>"
end
def build_pypi_package_path(file)
expose_url(
api_v4_projects_packages_pypi_files_file_identifier_path(
{
id: @project.id,
sha256: file.file_sha256,
file_identifier: file.file_name
},
true
)
) + "#sha256=#{file.file_sha256}"
end
def name
@packages.first.name
end
end
end
end
ee/changelogs/unreleased/208747-pypi-package-installation.yml 0 → 100644
View file @ e2806cd0
---
title: Support PyPi package installation
merge_request: 27827
author:
type: added
ee/lib/api/pypi_packages.rb
View file @ e2806cd0
...@@ -22,6 +22,33 @@ module API ...@@ -22,6 +22,33 @@ module API
render_api_error!(e.message, 400) render_api_error!(e.message, 400)
end end
rescue_from ActiveRecord::RecordInvalid do |e|
render_api_error!(e.message, 400)
end
helpers do
def packages_finder(project = authorized_user_project)
project
.packages
.pypi
.has_version
.processed
end
def find_package_versions
packages = packages_finder
.with_name(params[:package_name])
not_found!('Package') if packages.empty?
packages
end
def unauthorized_user_project
@unauthorized_user_project ||= find_project(params[:id]) || not_found!
end
end
before do before do
require_packages_enabled! require_packages_enabled!
end end
...@@ -32,11 +59,11 @@ module API ...@@ -32,11 +59,11 @@ module API
resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
before do before do
unless ::Feature.enabled?(:pypi_packages, authorized_user_project) unless ::Feature.enabled?(:pypi_packages, unauthorized_user_project)
not_found! not_found!
end end
authorize_packages_feature!(authorized_user_project) authorize_packages_feature!(unauthorized_user_project)
end end
namespace ':id/packages/pypi' do namespace ':id/packages/pypi' do
...@@ -46,10 +73,17 @@ module API ...@@ -46,10 +73,17 @@ module API
params do params do
requires :file_identifier, type: String, desc: 'The PyPi package file identifier', file_path: true requires :file_identifier, type: String, desc: 'The PyPi package file identifier', file_path: true
requires :sha256, type: String, desc: 'The PyPi package sha256 check sum'
end end
get 'files/*file_identifier', :txt do get 'files/:sha256/*file_identifier' do
authorize_read_package!(authorized_user_project) project = unauthorized_user_project
filename = "#{params[:file_identifier]}.#{params[:format]}"
package = packages_finder(project).by_file_name_and_sha256(filename, params[:sha256])
package_file = ::Packages::PackageFileFinder.new(package, filename, with_file_name_like: false).execute
present_carrierwave_file!(package_file.file, supports_direct_download: true)
end end
desc 'The PyPi Simple Endpoint' do desc 'The PyPi Simple Endpoint' do
...@@ -60,8 +94,20 @@ module API ...@@ -60,8 +94,20 @@ module API
requires :package_name, type: String, file_path: true, desc: 'The PyPi package name' requires :package_name, type: String, file_path: true, desc: 'The PyPi package name'
end end
# An Api entry point but returns an HTML file instead of JSON.
# PyPi simple API returns the package descriptor as a simple HTML file.
get 'simple/*package_name', format: :txt do get 'simple/*package_name', format: :txt do
authorize_read_package!(authorized_user_project) authorize_read_package!(authorized_user_project)
packages = find_package_versions
presenter = ::Packages::Pypi::PackagePresenter.new(packages, authorized_user_project)
# Adjusts grape output format
# to be HTML
content_type "text/html; charset=utf-8"
env['api.format'] = :binary
body presenter.body
end end
desc 'The PyPi Package upload endpoint' do desc 'The PyPi Package upload endpoint' do
......
ee/spec/factories/packages.rb
View file @ e2806cd0
...@@ -50,9 +50,15 @@ FactoryBot.define do ...@@ -50,9 +50,15 @@ FactoryBot.define do
end end
factory :pypi_package do factory :pypi_package do
pypi_metadatum
sequence(:name) { |n| "pypi-package-#{n}"} sequence(:name) { |n| "pypi-package-#{n}"}
sequence(:version) { |n| "1.0.#{n}" } sequence(:version) { |n| "1.0.#{n}" }
package_type { :pypi } package_type { :pypi }
after :create do |package|
create :package_file, :pypi, package: package, file_name: "#{package.name}-#{package.version}.tar.gz"
end
end end
factory :conan_package do factory :conan_package do
...@@ -188,6 +194,16 @@ FactoryBot.define do ...@@ -188,6 +194,16 @@ FactoryBot.define do
size { 300.kilobytes } size { 300.kilobytes }
end end
trait(:pypi) do
package
file { fixture_file_upload('ee/spec/fixtures/pypi/sample-project.tar.gz') }
file_name { 'sample-project-1.0.0.tar.gz' }
file_sha1 { '2c0cfbed075d3fae226f051f0cc771b533e01aff' }
file_md5 { '0a7392d24f42f83068fa3767c5310052' }
file_sha256 { '440e5e148a25331bbd7991575f7d54933c0ebf6cc735a18ee5066ac1381bb590' }
size { 1149.bytes }
end
trait :object_storage do trait :object_storage do
file_store { Packages::PackageFileUploader::Store::REMOTE } file_store { Packages::PackageFileUploader::Store::REMOTE }
end end
...@@ -207,6 +223,11 @@ FactoryBot.define do ...@@ -207,6 +223,11 @@ FactoryBot.define do
package_channel { 'stable' } package_channel { 'stable' }
end end
factory :pypi_metadatum, class: 'Packages::PypiMetadatum' do
association :package, package_type: :pypi
required_python { '>=2.7' }
end
factory :conan_file_metadatum, class: 'Packages::ConanFileMetadatum' do factory :conan_file_metadatum, class: 'Packages::ConanFileMetadatum' do
package_file package_file
recipe_revision { '0' } recipe_revision { '0' }
......
ee/spec/presenters/packages/pypi/package_presenter_spec.rb 0 → 100644
View file @ e2806cd0
# frozen_string_literal: true
require 'spec_helper'
describe ::Packages::Pypi::PackagePresenter do
using RSpec::Parameterized::TableSyntax
let_it_be(:project) { create(:project) }
let_it_be(:package_name) { 'sample-project' }
let_it_be(:package1) { create(:pypi_package, project: project, name: package_name, version: '1.0.0') }
let_it_be(:package2) { create(:pypi_package, project: project, name: package_name, version: '2.0.0') }
let(:packages) { [package1, package2] }
let(:presenter) { described_class.new(packages, project) }
describe '#body' do
subject { presenter.body}
shared_examples_for "pypi package presenter" do
let(:file) { package.package_files.first }
let(:filename) { file.file_name }
let(:expected_file) { "<a href=\"http://localhost/api/v4/projects/#{project.id}/packages/pypi/files/#{file.file_sha256}/#{filename}#sha256=#{file.file_sha256}\" data-requires-python=\"#{package.pypi_metadatum.required_python}\">#{filename}</a><br>" }
it { is_expected.to include expected_file }
end
it_behaves_like "pypi package presenter" do
let(:package) { package1 }
end
it_behaves_like "pypi package presenter" do
let(:package) { package2 }
end
end
end
ee/spec/requests/api/pypi_packages_spec.rb
View file @ e2806cd0
...@@ -10,7 +10,8 @@ describe API::PypiPackages do ...@@ -10,7 +10,8 @@ describe API::PypiPackages do
let_it_be(:personal_access_token) { create(:personal_access_token, user: user) } let_it_be(:personal_access_token) { create(:personal_access_token, user: user) }
describe 'GET /api/v4/projects/:id/packages/pypi/simple/:package_name' do describe 'GET /api/v4/projects/:id/packages/pypi/simple/:package_name' do
let(:url) { "/projects/#{project.id}/packages/pypi/simple/sample-project" } let_it_be(:package) { create(:pypi_package, project: project) }
let(:url) { "/projects/#{project.id}/packages/pypi/simple/#{package.name}" }
subject { get api(url) } subject { get api(url) }
...@@ -23,16 +24,16 @@ describe API::PypiPackages do ...@@ -23,16 +24,16 @@ describe API::PypiPackages do
using RSpec::Parameterized::TableSyntax using RSpec::Parameterized::TableSyntax
where(:project_visibility_level, :user_role, :member, :user_token, :shared_examples_name, :expected_status) do where(:project_visibility_level, :user_role, :member, :user_token, :shared_examples_name, :expected_status) do
'PUBLIC' | :developer | true | true | 'process PyPi api request' | :success 'PUBLIC' | :developer | true | true | 'PyPi package versions' | :success
'PUBLIC' | :guest | true | true | 'process PyPi api request' | :success 'PUBLIC' | :guest | true | true | 'PyPi package versions' | :success
'PUBLIC' | :developer | true | false | 'process PyPi api request' | :success 'PUBLIC' | :developer | true | false | 'PyPi package versions' | :success
'PUBLIC' | :guest | true | false | 'process PyPi api request' | :success 'PUBLIC' | :guest | true | false | 'PyPi package versions' | :success
'PUBLIC' | :developer | false | true | 'process PyPi api request' | :success 'PUBLIC' | :developer | false | true | 'PyPi package versions' | :success
'PUBLIC' | :guest | false | true | 'process PyPi api request' | :success 'PUBLIC' | :guest | false | true | 'PyPi package versions' | :success
'PUBLIC' | :developer | false | false | 'process PyPi api request' | :success 'PUBLIC' | :developer | false | false | 'PyPi package versions' | :success
'PUBLIC' | :guest | false | false | 'process PyPi api request' | :success 'PUBLIC' | :guest | false | false | 'PyPi package versions' | :success
'PUBLIC' | :anonymous | false | true | 'process PyPi api request' | :success 'PUBLIC' | :anonymous | false | true | 'PyPi package versions' | :success
'PRIVATE' | :developer | true | true | 'process PyPi api request' | :success 'PRIVATE' | :developer | true | true | 'PyPi package versions' | :success
'PRIVATE' | :guest | true | true | 'process PyPi api request' | :forbidden 'PRIVATE' | :guest | true | true | 'process PyPi api request' | :forbidden
'PRIVATE' | :developer | true | false | 'process PyPi api request' | :unauthorized 'PRIVATE' | :developer | true | false | 'process PyPi api request' | :unauthorized
'PRIVATE' | :guest | true | false | 'process PyPi api request' | :unauthorized 'PRIVATE' | :guest | true | false | 'process PyPi api request' | :unauthorized
...@@ -201,11 +202,11 @@ describe API::PypiPackages do ...@@ -201,11 +202,11 @@ describe API::PypiPackages do
it_behaves_like 'rejects PyPI packages access with packages features disabled' it_behaves_like 'rejects PyPI packages access with packages features disabled'
end end
describe 'GET /api/v4/projects/:id/packages/pypi/files/*file_identifier' do describe 'GET /api/v4/projects/:id/packages/pypi/files/:sha256/*file_identifier' do
let_it_be(:package_name) { 'Dummy-Package' } let_it_be(:package_name) { 'Dummy-Package' }
let_it_be(:package) { create(:pypi_package, project: project, name: package_name) } let_it_be(:package) { create(:pypi_package, project: project, name: package_name, version: '1.0.0') }
let(:url) { "/projects/#{project.id}/packages/pypi/files/sample_project-1.0.0-py3-none-any.whl" } let(:url) { "/projects/#{project.id}/packages/pypi/files/#{package.package_files.first.file_sha256}/#{package_name}-1.0.0.tar.gz" }
subject { get api(url) } subject { get api(url) }
...@@ -218,24 +219,24 @@ describe API::PypiPackages do ...@@ -218,24 +219,24 @@ describe API::PypiPackages do
using RSpec::Parameterized::TableSyntax using RSpec::Parameterized::TableSyntax
where(:project_visibility_level, :user_role, :member, :user_token, :shared_examples_name, :expected_status) do where(:project_visibility_level, :user_role, :member, :user_token, :shared_examples_name, :expected_status) do
'PUBLIC' | :developer | true | true | 'process PyPi api request' | :success 'PUBLIC' | :developer | true | true | 'PyPi package download' | :success
'PUBLIC' | :guest | true | true | 'process PyPi api request' | :success 'PUBLIC' | :guest | true | true | 'PyPi package download' | :success
'PUBLIC' | :developer | true | false | 'process PyPi api request' | :success 'PUBLIC' | :developer | true | false | 'PyPi package download' | :success
'PUBLIC' | :guest | true | false | 'process PyPi api request' | :success 'PUBLIC' | :guest | true | false | 'PyPi package download' | :success
'PUBLIC' | :developer | false | true | 'process PyPi api request' | :success 'PUBLIC' | :developer | false | true | 'PyPi package download' | :success
'PUBLIC' | :guest | false | true | 'process PyPi api request' | :success 'PUBLIC' | :guest | false | true | 'PyPi package download' | :success
'PUBLIC' | :developer | false | false | 'process PyPi api request' | :success 'PUBLIC' | :developer | false | false | 'PyPi package download' | :success
'PUBLIC' | :guest | false | false | 'process PyPi api request' | :success 'PUBLIC' | :guest | false | false | 'PyPi package download' | :success
'PUBLIC' | :anonymous | false | true | 'process PyPi api request' | :success 'PUBLIC' | :anonymous | false | true | 'PyPi package download' | :success
'PRIVATE' | :developer | true | true | 'process PyPi api request' | :success 'PRIVATE' | :developer | true | true | 'PyPi package download' | :success
'PRIVATE' | :guest | true | true | 'process PyPi api request' | :forbidden 'PRIVATE' | :guest | true | true | 'PyPi package download' | :success
'PRIVATE' | :developer | true | false | 'process PyPi api request' | :unauthorized 'PRIVATE' | :developer | true | false | 'PyPi package download' | :success
'PRIVATE' | :guest | true | false | 'process PyPi api request' | :unauthorized 'PRIVATE' | :guest | true | false | 'PyPi package download' | :success
'PRIVATE' | :developer | false | true | 'process PyPi api request' | :not_found 'PRIVATE' | :developer | false | true | 'PyPi package download' | :success
'PRIVATE' | :guest | false | true | 'process PyPi api request' | :not_found 'PRIVATE' | :guest | false | true | 'PyPi package download' | :success
'PRIVATE' | :developer | false | false | 'process PyPi api request' | :unauthorized 'PRIVATE' | :developer | false | false | 'PyPi package download' | :success
'PRIVATE' | :guest | false | false | 'process PyPi api request' | :unauthorized 'PRIVATE' | :guest | false | false | 'PyPi package download' | :success
'PRIVATE' | :anonymous | false | true | 'process PyPi api request' | :unauthorized 'PRIVATE' | :anonymous | false | true | 'PyPi package download' | :success
end end
with_them do with_them do
......
ee/spec/support/shared_examples/requests/api/pypi_packages_shared_examples.rb
View file @ e2806cd0
...@@ -90,6 +90,38 @@ RSpec.shared_examples 'PyPi package creation' do |user_type, status, add_member ...@@ -90,6 +90,38 @@ RSpec.shared_examples 'PyPi package creation' do |user_type, status, add_member
end end
end end
RSpec.shared_examples 'PyPi package versions' do |user_type, status, add_member = true|
context "for user type #{user_type}" do
before do
project.send("add_#{user_type}", user) if add_member && user_type != :anonymous
end
it 'returns the package listing' do
subject
expect(response.body).to match(package.package_files.first.file_name)
end
it_behaves_like 'returning response status', status
end
end
RSpec.shared_examples 'PyPi package download' do |user_type, status, add_member = true|
context "for user type #{user_type}" do
before do
project.send("add_#{user_type}", user) if add_member && user_type != :anonymous
end
it 'returns the package listing' do
subject
expect(response.body).to eq(File.open(package.package_files.first.file.path, "rb").read)
end
it_behaves_like 'returning response status', status
end
end
RSpec.shared_examples 'process PyPi api request' do |user_type, status, add_member = true| RSpec.shared_examples 'process PyPi api request' do |user_type, status, add_member = true|
context "for user type #{user_type}" do context "for user type #{user_type}" do
before do before do
...@@ -105,7 +137,7 @@ RSpec.shared_examples 'rejects PyPI access with unknown project id' do ...@@ -105,7 +137,7 @@ RSpec.shared_examples 'rejects PyPI access with unknown project id' do
let(:project) { OpenStruct.new(id: 1234567890) } let(:project) { OpenStruct.new(id: 1234567890) }
context 'as anonymous' do context 'as anonymous' do
it_behaves_like 'process PyPi api request', :anonymous, :unauthorized it_behaves_like 'process PyPi api request', :anonymous, :not_found
end end
context 'as authenticated user' do context 'as authenticated user' do
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7