Merge branch '214731-serve-composer-packages' into 'master'

PHP Composer: Serve package files for package installation

Closes #214731

See merge request gitlab-org/gitlab!32390

db/migrate/20200615232735_add_index_to_composer_metadata.rb

+# frozen_string_literal: true
+
+class AddIndexToComposerMetadata < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index(:packages_composer_metadata, [:package_id, :target_sha], unique: true)
+  end
+
+  def down
+    remove_concurrent_index(:packages_composer_metadata, [:package_id, :target_sha])
+  end
+end

db/structure.sql

@@ -10430,6 +10430,8 @@ CREATE UNIQUE INDEX index_packages_build_infos_on_package_id ON public.packages_
 
 CREATE INDEX index_packages_build_infos_on_pipeline_id ON public.packages_build_infos USING btree (pipeline_id);
 
+CREATE UNIQUE INDEX index_packages_composer_metadata_on_package_id_and_target_sha ON public.packages_composer_metadata USING btree (package_id, target_sha);
+
 CREATE UNIQUE INDEX index_packages_conan_file_metadata_on_package_file_id ON public.packages_conan_file_metadata USING btree (package_file_id);
 
 CREATE UNIQUE INDEX index_packages_conan_metadata_on_package_id_username_channel ON public.packages_conan_metadata USING btree (package_id, package_username, package_channel);
@@ -13995,5 +13997,6 @@ COPY "schema_migrations" (version) FROM STDIN;
 20200615083635
 20200615121217
 20200615123055
+20200615232735
 \.
 

ee/app/finders/packages/composer/packages_finder.rb

+# frozen_string_literal: true
+module Packages
+  module Composer
+    class PackagesFinder < Packages::GroupPackagesFinder
+      def initialize(current_user, group, params = {})
+        @current_user = current_user
+        @group = group
+        @params = params
+      end
+
+      def execute
+        packages_for_group_projects.composer.preload_composer
+      end
+    end
+  end
+end

ee/app/models/packages/package.rb

@@ -53,6 +53,13 @@ class Packages::Package < ApplicationRecord
     joins(:conan_metadatum).where(packages_conan_metadata: { package_username: package_username })
   end
 
+  scope :with_composer_target, -> (target) do
+    includes(:composer_metadatum)
+      .joins(:composer_metadatum)
+      .where(Packages::Composer::Metadatum.table_name => { target_sha: target })
+  end
+  scope :preload_composer, -> { preload(:composer_metadatum) }
+
   scope :without_nuget_temporary_name, -> { where.not(name: Packages::Nuget::CreatePackageService::TEMPORARY_PACKAGE_NAME) }
 
   scope :has_version, -> { where.not(version: nil) }

ee/app/presenters/packages/composer/packages_presenter.rb

+# frozen_string_literal: true
+
+module Packages
+  module Composer
+    class PackagesPresenter
+      include API::Helpers::RelatedResourcesHelpers
+
+      def initialize(group, packages)
+        @group = group
+        @packages = packages
+      end
+
+      def root
+        path = api_v4_group___packages_composer_package_name_path({ id: @group.id, package_name: '%package%', format: '.json' }, true)
+        { 'packages' => [], 'provider-includes' => { 'p/%hash%.json' => { 'sha256' => provider_sha } }, 'providers-url' => path }
+      end
+
+      def provider
+        { 'providers' => providers_map }
+      end
+
+      def package_versions(packages = @packages)
+        { 'packages' => { packages.first.name => package_versions_map(packages) } }
+      end
+
+      private
+
+      def package_versions_map(packages)
+        packages.each_with_object({}) do |package, map|
+          map[package.version] = package_metadata(package)
+        end
+      end
+
+      def package_metadata(package)
+        json = package.composer_metadatum.composer_json
+
+        json.merge('dist' => package_dist(package), 'uid' => package.id, 'version' => package.version)
+      end
+
+      def package_dist(package)
+        sha = package.composer_metadatum.target_sha
+        archive_api_path = api_v4_projects_packages_composer_archives_package_name_path({ id: package.project_id, package_name: package.name, format: '.zip' }, true)
+
+        {
+          'type' => 'zip',
+          'url' => expose_url(archive_api_path) + "?sha=#{sha}",
+          'reference' => sha,
+          'shasum' => ''
+        }
+      end
+
+      def providers_map
+        map = {}
+
+        @packages.group_by(&:name).each_pair do |package_name, packages|
+          map[package_name] = { 'sha256' => package_versions_sha(packages) }
+        end
+
+        map
+      end
+
+      def package_versions_sha(packages)
+        Digest::SHA256.hexdigest(package_versions(packages).to_json)
+      end
+
+      def provider_sha
+        Digest::SHA256.hexdigest(provider.to_json)
+      end
+    end
+  end
+end

ee/lib/api/composer_packages.rb

@@ -7,6 +7,7 @@ module API
     helpers ::API::Helpers::RelatedResourcesHelpers
     helpers ::API::Helpers::Packages::BasicAuthHelpers
     include ::API::Helpers::Packages::BasicAuthHelpers::Constants
+    include ::Gitlab::Utils::StrongMemoize
 
     content_type :json, 'application/json'
     default_format :json
@@ -25,6 +26,24 @@ module API
       render_api_error!(e.message, 400)
     end
 
+    helpers do
+      def packages
+        strong_memoize(:packages) do
+          packages = ::Packages::Composer::PackagesFinder.new(current_user, user_group).execute
+
+          if params[:package_name].present?
+            packages = packages.with_name(params[:package_name])
+          end
+
+          packages
+        end
+      end
+
+      def presenter
+        @presenter ||= ::Packages::Composer::PackagesPresenter.new(user_group, packages)
+      end
+    end
+
     before do
       require_packages_enabled!
     end
@@ -47,6 +66,7 @@ module API
       route_setting :authentication, job_token_allowed: true
 
       get ':id/-/packages/composer/packages' do
+        presenter.root
       end
 
       desc 'Composer packages endpoint at group level for packages list'
@@ -58,13 +78,21 @@ module API
       route_setting :authentication, job_token_allowed: true
 
       get ':id/-/packages/composer/p/:sha' do
+        presenter.provider
       end
 
       desc 'Composer packages endpoint at group level for package versions metadata'
 
+      params do
+        requires :package_name, type: String, file_path: true, desc: 'The Composer package name'
+      end
+
       route_setting :authentication, job_token_allowed: true
 
       get ':id/-/packages/composer/*package_name', requirements: COMPOSER_ENDPOINT_REQUIREMENTS, file_path: true do
+        not_found! if packages.empty?
+
+        presenter.package_versions
       end
     end
 
@@ -83,13 +111,15 @@ module API
 
       desc 'Composer packages endpoint for registering packages'
 
-      params do
-        optional :branch, type: String, desc: 'The name of the branch'
-        optional :tag, type: String, desc: 'The name of the tag'
-        exactly_one_of :tag, :branch
-      end
-
       namespace ':id/packages/composer' do
+        route_setting :authentication, job_token_allowed: true
+
+        params do
+          optional :branch, type: String, desc: 'The name of the branch'
+          optional :tag, type: String, desc: 'The name of the tag'
+          exactly_one_of :tag, :branch
+        end
+
         post do
           authorize_create_package!(authorized_user_project)
 
@@ -107,6 +137,25 @@ module API
 
           created!
         end
+
+        params do
+          requires :sha, type: String, desc: 'Shasum of current json'
+          requires :package_name, type: String, file_path: true, desc: 'The Composer package name'
+        end
+
+        get 'archives/*package_name' do
+          metadata = unauthorized_user_project
+            .packages
+            .composer
+            .with_name(params[:package_name])
+            .with_composer_target(params[:sha])
+            .first
+            &.composer_metadatum
+
+          not_found! unless metadata
+
+          send_git_archive unauthorized_user_project.repository, ref: metadata.target_sha, format: 'zip', append_sha: true
+        end
       end
     end
   end

ee/spec/factories/packages.rb

@@ -71,6 +71,17 @@ FactoryBot.define do
       sequence(:name) { |n| "composer-package-#{n}"}
       sequence(:version) { |n| "1.0.#{n}" }
       package_type { :composer }
+
+      transient do
+        sha { project.repository.find_branch('master').target }
+        json { { name: name, version: version } }
+      end
+
+      trait(:with_metadatum) do
+        after :create do |package, evaluator|
+          create :composer_metadatum, package: package, target_sha: evaluator.sha, composer_json: evaluator.json
+        end
+      end
     end
 
     factory :conan_package do
@@ -106,6 +117,9 @@ FactoryBot.define do
     end
   end
 
+  factory :composer_metadatum, class: 'Packages::Composer::Metadatum' do
+  end
+
   factory :package_build_info, class: 'Packages::BuildInfo' do
   end
 

ee/spec/fixtures/api/schemas/packages/composer/index.json

+{
+  "type": "object",
+  "required": ["packages", "provider-includes", "providers-url"],
+  "properties": {
+    "packages": {
+      "type": "array",
+      "items": { "type": "integer" }
+    },
+    "providers-url": {
+      "type": "string"
+    },
+    "provider-includes": {
+      "type": "object",
+      "required": ["p/%hash%.json"],
+      "properties": {
+        "p/%hash%.json": {
+          "type": "object",
+          "required": ["sha256"],
+          "properties": {
+            "sha256": {
+              "type": "string"
+            }
+          }
+        }
+      }
+    }
+  },
+  "additionalProperties": false
+}

ee/spec/fixtures/api/schemas/packages/composer/package.json

+{
+  "type": "object",
+  "required": [
+    "packages"
+  ],
+  "properties": {
+    "packages": {
+      "type": "object",
+      "propertyNames": {
+        "pattern": "^[A-Za-z_]+"
+      },
+      "patternProperties": {
+        "^[A-Za-z_]+": {
+          "type": "object",
+          "propertyNames": {
+            "pattern": "^[A-Za-z_0-9.]+"
+          },
+          "patternProperties": {
+            "^[A-Za-z_0-9.]+": {
+              "type": "object",
+              "required": [
+                "dist",
+                "uid",
+                "version"
+              ],
+              "properties": {
+                "uid": {
+                  "type": "integer"
+                },
+                "version": {
+                  "type": "string"
+                },
+                "dist": {
+                  "type": "object",
+                  "required": [
+                    "type",
+                    "url",
+                    "reference",
+                    "shasum"
+                  ],
+                  "properties": {
+                    "type": {
+                      "type": "string"
+                    },
+                    "url": {
+                      "type": "string"
+                    },
+                    "reference": {
+                      "type": "string"
+                    },
+                    "shasum": {
+                      "type": "string"
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+      },
+      "additionalProperties": false
+    }
+  },
+  "additionalProperties": false
+}

ee/spec/fixtures/api/schemas/packages/composer/provider.json

+{
+  "type": "object",
+  "required": ["providers"],
+  "properties": {
+    "providers": {
+      "type": "object",
+      "propertyNames": {
+        "pattern": "^[A-Za-z_]+"
+      },
+      "patternProperties": {
+        "^[A-Za-z_]+": {
+          "type": "object",
+          "required": ["sha256"],
+          "properties": {
+            "sha256": {
+              "type": "string"
+            }
+          }
+        }
+      },
+      "additionalProperties": false
+    }
+  },
+  "additionalProperties": false
+}

ee/spec/models/packages/package_spec.rb

@@ -14,6 +14,20 @@ RSpec.describe Packages::Package, type: :model do
     it { is_expected.to have_one(:nuget_metadatum).inverse_of(:package) }
   end
 
+  describe '.with_composer_target' do
+    let!(:package1) { create(:composer_package, :with_metadatum, sha: '123') }
+    let!(:package2) { create(:composer_package, :with_metadatum, sha: '123') }
+    let!(:package3) { create(:composer_package, :with_metadatum, sha: '234') }
+
+    subject { described_class.with_composer_target('123').to_a }
+
+    it 'selects packages with the specified sha' do
+      expect(subject).to include(package1)
+      expect(subject).to include(package2)
+      expect(subject).not_to include(package3)
+    end
+  end
+
   describe '.sort_by_attribute' do
     let_it_be(:group) { create(:group, :public) }
     let_it_be(:project) { create(:project, :public, namespace: group, name: 'project A') }

ee/spec/presenters/packages/composer/packages_presenter_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe ::Packages::Composer::PackagesPresenter do
+  using RSpec::Parameterized::TableSyntax
+
+  let_it_be(:package_name) { 'sample-project' }
+  let_it_be(:json) { { 'name' => package_name } }
+  let_it_be(:group) { create(:group) }
+  let_it_be(:project) { create(:project, :custom_repo, files: { 'composer.json' => json.to_json }, group: group) }
+  let_it_be(:package1) { create(:composer_package, :with_metadatum, project: project, name: package_name, version: '1.0.0', json: json) }
+  let_it_be(:package2) { create(:composer_package, :with_metadatum, project: project, name: package_name, version: '2.0.0', json: json) }
+
+  let(:branch) { project.repository.find_branch('master') }
+
+  let(:packages) { [package1, package2] }
+  let(:presenter) { described_class.new(group, packages) }
+
+  describe '#package_versions' do
+    subject { presenter.package_versions }
+
+    def expected_json(package)
+      {
+        'dist' => {
+          'reference' => branch.target,
+          'shasum' => '',
+          'type' => 'zip',
+          'url' => "http://localhost/api/v4/projects/#{project.id}/packages/composer/archives/#{package.name}.zip?sha=#{branch.target}"
+        },
+        'name' => package.name,
+        'uid' => package.id,
+        'version' => package.version
+      }
+    end
+
+    it 'returns the packages json' do
+      packages = subject['packages'][package_name]
+
+      expect(packages['1.0.0']).to eq(expected_json(package1))
+      expect(packages['2.0.0']).to eq(expected_json(package2))
+    end
+  end
+
+  describe '#provider' do
+    subject { presenter.provider}
+
+    let(:expected_json) do
+      {
+        'providers' => {
+          package_name => {
+            'sha256' => /^\h+$/
+          }
+        }
+      }
+    end
+
+    it 'returns the provider json' do
+      expect(subject).to match(expected_json)
+    end
+  end
+
+  describe '#root' do
+    subject { presenter.root }
+
+    let(:expected_json) do
+      {
+        'packages' => [],
+        'provider-includes' => { 'p/%hash%.json' => { 'sha256' => /^\h+$/ } },
+        'providers-url' => "/api/v4/group/#{group.id}/-/packages/composer/%package%.json"
+      }
+    end
+
+    it 'returns the provider json' do
+      expect(subject).to match(expected_json)
+    end
+  end
+end

ee/spec/support/shared_examples/requests/api/composer_packages_shared_examples.rb

 # frozen_string_literal: true
 
+RSpec.shared_context 'Composer user type' do |user_type, add_member|
+  before do
+    group.send("add_#{user_type}", user) if add_member && user_type != :anonymous
+    project.send("add_#{user_type}", user) if add_member && user_type != :anonymous
+  end
+end
+
+RSpec.shared_examples 'Composer package index' do |user_type, status, add_member = true|
+  include_context 'Composer user type', user_type, add_member do
+    it 'returns the package index' do
+      subject
+
+      expect(response).to have_gitlab_http_status(status)
+      expect(response).to match_response_schema('packages/composer/index', dir: 'ee')
+    end
+  end
+end
+
+RSpec.shared_examples 'Composer empty provider index' do |user_type, status, add_member = true|
+  include_context 'Composer user type', user_type, add_member do
+    it 'returns the package index' do
+      subject
+
+      expect(response).to have_gitlab_http_status(status)
+      expect(response).to match_response_schema('packages/composer/provider', dir: 'ee')
+      expect(json_response['providers']).to eq({})
+    end
+  end
+end
+
+RSpec.shared_examples 'Composer provider index' do |user_type, status, add_member = true|
+  include_context 'Composer user type', user_type, add_member do
+    it 'returns the package index' do
+      subject
+
+      expect(response).to have_gitlab_http_status(status)
+      expect(response).to match_response_schema('packages/composer/provider', dir: 'ee')
+      expect(json_response['providers']).to include(package.name)
+    end
+  end
+end
+
+RSpec.shared_examples 'Composer package api request' do |user_type, status, add_member = true|
+  include_context 'Composer user type', user_type, add_member do
+    it 'returns the package index' do
+      subject
+
+      expect(response).to have_gitlab_http_status(status)
+      expect(response).to match_response_schema('packages/composer/package', dir: 'ee')
+      expect(json_response['packages']).to include(package.name)
+      expect(json_response['packages'][package.name]).to include(package.version)
+    end
+  end
+end
+
 RSpec.shared_examples 'Composer package creation' do |user_type, status, add_member = true|
   context "for user type #{user_type}" do
     before do
@@ -43,6 +98,7 @@ end
 RSpec.shared_context 'Composer api group access' do |project_visibility_level, user_role, user_token|
   include_context 'Composer auth headers', user_role, user_token do
     before do
+      project.update!(visibility_level: Gitlab::VisibilityLevel.const_get(project_visibility_level, false))
       group.update!(visibility_level: Gitlab::VisibilityLevel.const_get(project_visibility_level, false))
     end
   end
@@ -79,13 +135,13 @@ RSpec.shared_examples 'rejects Composer access with unknown project id' do
     let(:project) { double(id: non_existing_record_id) }
 
     context 'as anonymous' do
-      it_behaves_like 'process PyPi api request', :anonymous, :not_found
+      it_behaves_like 'process Composer api request', :anonymous, :not_found
     end
 
     context 'as authenticated user' do
       subject { get api(url), headers: build_basic_auth_header(user.username, personal_access_token.token) }
 
-      it_behaves_like 'process PyPi api request', :anonymous, :not_found
+      it_behaves_like 'process Composer api request', :anonymous, :not_found
     end
   end
 end