Commit 1dfbaaa8 authored by Steve Abrams's avatar Steve Abrams Committed by Andreas Brandl

Conan packages search API endpoint

Add conan search api endpoint and related search
services for finding conan packages using the
conan CLI 'conan search' command
parent 528d4ee7
# frozen_string_literal: true
class AddIndexPackagesOnNameTrigramToPackagesPackages < ActiveRecord::Migration[5.2]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
INDEX_NAME = 'index_packages_packages_on_name_trigram'
disable_ddl_transaction!
def up
add_concurrent_index :packages_packages, :name, name: INDEX_NAME, using: :gin, opclass: { name: :gin_trgm_ops }
end
def down
remove_concurrent_index_by_name(:packages_packages, INDEX_NAME)
end
end
...@@ -2556,6 +2556,7 @@ ActiveRecord::Schema.define(version: 2019_09_27_074328) do ...@@ -2556,6 +2556,7 @@ ActiveRecord::Schema.define(version: 2019_09_27_074328) do
t.string "name", null: false t.string "name", null: false
t.string "version" t.string "version"
t.integer "package_type", limit: 2, null: false t.integer "package_type", limit: 2, null: false
t.index ["name"], name: "index_packages_packages_on_name_trigram", opclass: :gin_trgm_ops, using: :gin
t.index ["project_id"], name: "index_packages_packages_on_project_id" t.index ["project_id"], name: "index_packages_packages_on_project_id"
end end
......
# frozen_string_literal: true
module Packages
class ConanPackageFinder
attr_reader :query, :current_user
def initialize(query, current_user)
@query = query
@current_user = current_user
end
def execute
packages_for_current_user.with_name_like(query).order_name_asc
end
private
def packages
Packages::Package.conan
end
def packages_for_current_user
packages.for_projects(projects_visible_to_current_user)
end
def projects_visible_to_current_user
::Project.public_or_visible_to_user(current_user)
end
end
end
# frozen_string_literal: true # frozen_string_literal: true
class Packages::Package < ApplicationRecord class Packages::Package < ApplicationRecord
include Sortable
belongs_to :project belongs_to :project
# package_files must be destroyed by ruby code in order to properly remove carrierwave uploads and update project statistics # package_files must be destroyed by ruby code in order to properly remove carrierwave uploads and update project statistics
has_many :package_files, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :package_files, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
...@@ -16,9 +18,10 @@ class Packages::Package < ApplicationRecord ...@@ -16,9 +18,10 @@ class Packages::Package < ApplicationRecord
validate :valid_npm_package_name, if: :npm? validate :valid_npm_package_name, if: :npm?
validate :package_already_taken, if: :npm? validate :package_already_taken, if: :npm?
enum package_type: { maven: 1, npm: 2 } enum package_type: { maven: 1, npm: 2, conan: 3 }
scope :with_name, ->(name) { where(name: name) } scope :with_name, ->(name) { where(name: name) }
scope :with_name_like, ->(name) { where(arel_table[:name].matches(name)) }
scope :with_version, ->(version) { where(version: version) } scope :with_version, ->(version) { where(version: version) }
scope :has_version, -> { where.not(version: nil) } scope :has_version, -> { where.not(version: nil) }
scope :preload_files, -> { preload(:package_files) } scope :preload_files, -> { preload(:package_files) }
...@@ -50,6 +53,10 @@ class Packages::Package < ApplicationRecord ...@@ -50,6 +53,10 @@ class Packages::Package < ApplicationRecord
.where(packages_package_files: { file_name: file_name }).last! .where(packages_package_files: { file_name: file_name }).last!
end end
def self.pluck_names
pluck(:name)
end
def self.sort_by_attribute(method) def self.sort_by_attribute(method)
case method.to_s case method.to_s
when 'created_asc' then order_created when 'created_asc' then order_created
......
# frozen_string_literal: true
module Packages
module Conan
class SearchService < BaseService
include ActiveRecord::Sanitization::ClassMethods
WILDCARD = '*'
RECIPE_SEPARATOR = '@'
def initialize(user, params)
super(nil, user, params)
end
def execute
return ServiceResponse.error(message: 'not found', http_status: :not_found) unless feature_available?
ServiceResponse.success(payload: { results: search_results })
end
private
def search_results
return [] if wildcard_query?
search_packages(build_query)
end
def wildcard_query?
params[:query] == WILDCARD
end
def build_query
sanitized_query = sanitize_sql_like(params[:query].delete(WILDCARD))
return "#{sanitized_query}%" if params[:query].end_with?(WILDCARD)
return sanitized_query if sanitized_query.include?(RECIPE_SEPARATOR)
"#{sanitized_query}/%"
end
def feature_available?
Feature.enabled?(:conan_package_registry)
end
def search_packages(query)
Packages::ConanPackageFinder.new(query, current_user).execute.pluck_names
end
end
end
end
...@@ -40,6 +40,17 @@ module API ...@@ -40,6 +40,17 @@ module API
get 'ping' do get 'ping' do
header 'X-Conan-Server-Capabilities', [].join(',') header 'X-Conan-Server-Capabilities', [].join(',')
end end
desc 'Search for packages' do
detail 'This feature was introduced in GitLab 12.3'
end
params do
requires :q, type: String, desc: 'Search query'
end
get 'conans/search' do
service = ::Packages::Conan::SearchService.new(current_user, query: params[:q]).execute
service.payload
end
end end
namespace 'packages/conan/v1/conans/*url_recipe' do namespace 'packages/conan/v1/conans/*url_recipe' do
......
...@@ -7,7 +7,7 @@ module EE ...@@ -7,7 +7,7 @@ module EE
class_methods do class_methods do
def package_name_regex def package_name_regex
@package_name_regex ||= %r{\A\@?(([\w\-\.]*)/)*([\w\-\.]*)\z}.freeze @package_name_regex ||= %r{\A\@?(([\w\-\.\+]*)\/)*([\w\-\.]+)@?(([\w\-\.\+]*)\/)*([\w\-\.]*)\z}.freeze
end end
def maven_path_regex def maven_path_regex
......
...@@ -29,6 +29,12 @@ FactoryBot.define do ...@@ -29,6 +29,12 @@ FactoryBot.define do
create :package_file, :npm, package: package create :package_file, :npm, package: package
end end
end end
factory :conan_package do
sequence(:name) { |n| "package-#{n}/1.0.0@#{project.full_path.tr('/', '+')}/stable"}
version { '1.0.0' }
package_type { 'conan' }
end
end end
factory :package_file, class: Packages::PackageFile do factory :package_file, class: Packages::PackageFile do
......
# frozen_string_literal: true
require 'spec_helper'
describe Packages::ConanPackageFinder do
describe '#execute' do
let_it_be(:user) { create(:user) }
let_it_be(:project) { create(:project, :public) }
let!(:conan_package) { create(:conan_package, project: project) }
let!(:conan_package2) { create(:conan_package, project: project) }
subject { described_class.new(query, user).execute }
context 'packages that are not visible to user' do
let!(:non_visible_project) { create(:project, :private) }
let!(:non_visible_conan_package) { create(:conan_package, project: non_visible_project) }
let(:query) { "#{conan_package.name.split('/').first[0, 3]}%" }
it { is_expected.to eq [conan_package, conan_package2] }
end
end
end
...@@ -21,8 +21,11 @@ describe Gitlab::Regex do ...@@ -21,8 +21,11 @@ describe Gitlab::Regex do
it { is_expected.to match('foo/bar') } it { is_expected.to match('foo/bar') }
it { is_expected.to match('@foo/bar') } it { is_expected.to match('@foo/bar') }
it { is_expected.to match('com/mycompany/app/my-app') } it { is_expected.to match('com/mycompany/app/my-app') }
it { is_expected.to match('my-package/1.0.0@my+project+path/beta') }
it { is_expected.not_to match('my-package/1.0.0@@@@@my+project+path/beta') }
it { is_expected.not_to match('$foo/bar') } it { is_expected.not_to match('$foo/bar') }
it { is_expected.not_to match('@foo/@/bar') } it { is_expected.not_to match('@foo/@/bar') }
it { is_expected.not_to match('@@foo/bar') }
it { is_expected.not_to match('my package name') } it { is_expected.not_to match('my package name') }
it { is_expected.not_to match('!!()()') } it { is_expected.not_to match('!!()()') }
end end
......
...@@ -3,6 +3,10 @@ require 'spec_helper' ...@@ -3,6 +3,10 @@ require 'spec_helper'
describe API::ConanPackages do describe API::ConanPackages do
let(:base_secret) { SecureRandom.base64(64) } let(:base_secret) { SecureRandom.base64(64) }
let(:personal_access_token) { create(:personal_access_token) }
let(:headers) do
{ 'HTTP_AUTHORIZATION' => ActionController::HttpAuthentication::Basic.encode_credentials('foo', personal_access_token.token) }
end
let(:jwt_secret) do let(:jwt_secret) do
OpenSSL::HMAC.hexdigest( OpenSSL::HMAC.hexdigest(
...@@ -12,11 +16,6 @@ describe API::ConanPackages do ...@@ -12,11 +16,6 @@ describe API::ConanPackages do
) )
end end
let(:personal_access_token) { create(:personal_access_token) }
let(:headers) do
{ 'HTTP_AUTHORIZATION' => ActionController::HttpAuthentication::Basic.encode_credentials('foo', personal_access_token.token) }
end
before do before do
stub_licensed_features(packages: true) stub_licensed_features(packages: true)
allow(Settings).to receive(:attr_encrypted_db_key_base).and_return(base_secret) allow(Settings).to receive(:attr_encrypted_db_key_base).and_return(base_secret)
...@@ -99,6 +98,35 @@ describe API::ConanPackages do ...@@ -99,6 +98,35 @@ describe API::ConanPackages do
end end
end end
describe 'GET /api/v4/packages/conan/v1/conans/search' do
let(:project) { create(:project, :public) }
let(:package) { create(:conan_package, project: project) }
before do
get api('/packages/conan/v1/conans/search'), headers: headers, params: params
end
subject { JSON.parse(response.body)['results'] }
context 'returns packages with a matching name' do
let(:params) { { q: package.name } }
it { is_expected.to contain_exactly(package.name) }
end
context 'returns packages using a * wildcard' do
let(:params) {{ q: "#{package.name[0, 3]}*" }}
it { is_expected.to contain_exactly(package.name) }
end
context 'does not return non-matching packages' do
let(:params) {{ q: "foo" }}
it { is_expected.to be_blank }
end
end
describe 'GET /api/v4/packages/conan/v1/users/authenticate' do describe 'GET /api/v4/packages/conan/v1/users/authenticate' do
it 'responds with 401 Unauthorized when invalid token is provided' do it 'responds with 401 Unauthorized when invalid token is provided' do
headers = { 'HTTP_AUTHORIZATION' => ActionController::HttpAuthentication::Basic.encode_credentials('foo', 'wrong-token') } headers = { 'HTTP_AUTHORIZATION' => ActionController::HttpAuthentication::Basic.encode_credentials('foo', 'wrong-token') }
......
...@@ -163,7 +163,7 @@ describe API::NpmPackages do ...@@ -163,7 +163,7 @@ describe API::NpmPackages do
end end
context 'invalid package name' do context 'invalid package name' do
let(:package_name) { "@#{group.path}/my_inv@lid_package_name" } let(:package_name) { "@#{group.path}/my_inv@@lid_package_name" }
let(:params) { upload_params(package_name) } let(:params) { upload_params(package_name) }
it 'handles an ActiveRecord::RecordInvalid exception with 400 error' do it 'handles an ActiveRecord::RecordInvalid exception with 400 error' do
......
# frozen_string_literal: true
require 'spec_helper'
describe Packages::Conan::SearchService do
let_it_be(:user) { create(:user) }
let_it_be(:project) { create(:project, :public) }
let!(:conan_package) { create(:conan_package, project: project) }
let!(:conan_package2) { create(:conan_package, project: project) }
subject { described_class.new(user, query: query) }
describe '#execute' do
context 'feature unavailable' do
let(:query) { '' }
before do
stub_feature_flags(conan_package_registry: false)
end
it 'responds with 404 not found' do
result = subject.execute
expect(result.http_status).to eq :not_found
expect(result.message).to eq('not found')
end
end
context 'with wildcard' do
let(:partial_name) { conan_package.name.first[0, 3] }
let(:query) { "#{partial_name}*" }
it 'makes a wildcard query' do
result = subject.execute
expect(result.status).to eq :success
expect(result.payload).to eq(results: [conan_package.name, conan_package2.name])
end
end
context 'with only wildcard' do
let(:query) { '*' }
it 'returns empty' do
result = subject.execute
expect(result.status).to eq :success
expect(result.payload).to eq(results: [])
end
end
context 'with no wildcard' do
let(:query) { conan_package.name.split('/').first }
it 'makes a search using the beginning of the recipe' do
result = subject.execute
expect(result.status).to eq :success
expect(result.payload).to eq(results: [conan_package.name])
end
end
context 'with full recipe match' do
let(:query) { conan_package.name }
it 'makes an exact search' do
result = subject.execute
expect(result.status).to eq :success
expect(result.payload).to eq(results: [conan_package.name])
end
end
context 'with malicious query' do
let(:query) { 'DROP TABLE foo;' }
it 'returns empty' do
result = subject.execute
expect(result.status).to eq :success
expect(result.payload).to eq(results: [])
end
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment