Merge branch 'helm_api_get_index' into 'master'

Helm chart download API endpoint [RUN ALL RSPEC] [RUN AS-IF-FOSS]

See merge request gitlab-org/gitlab!61014

app/finders/packages/helm/package_files_finder.rb

+# frozen_string_literal: true
+
+module Packages
+  module Helm
+    class PackageFilesFinder
+      def initialize(project, channel, params = {})
+        @project = project
+        @channel = channel
+        @params = params
+      end
+
+      def execute
+        package_files = Packages::PackageFile.for_helm_with_channel(@project, @channel).preload_helm_file_metadata
+        by_file_name(package_files)
+      end
+
+      private
+
+      def by_file_name(files)
+        return files unless @params[:file_name]
+
+        files.with_file_name(@params[:file_name])
+      end
+    end
+  end
+end

app/models/packages/package_file.rb

@@ -33,11 +33,18 @@ class Packages::PackageFile < ApplicationRecord
   scope :with_files_stored_locally, -> { where(file_store: ::Packages::PackageFileUploader::Store::LOCAL) }
   scope :preload_conan_file_metadata, -> { preload(:conan_file_metadatum) }
   scope :preload_debian_file_metadata, -> { preload(:debian_file_metadatum) }
+  scope :preload_helm_file_metadata, -> { preload(:helm_file_metadatum) }
 
   scope :for_rubygem_with_file_name, ->(project, file_name) do
     joins(:package).merge(project.packages.rubygems).with_file_name(file_name)
   end
 
+  scope :for_helm_with_channel, ->(project, channel) do
+    joins(:package).merge(project.packages.helm.installable)
+                   .joins(:helm_file_metadatum)
+                   .where(packages_helm_file_metadata: { channel: channel })
+  end
+
   scope :with_conan_file_type, ->(file_type) do
     joins(:conan_file_metadatum)
       .where(packages_conan_file_metadata: { conan_file_type: ::Packages::Conan::FileMetadatum.conan_file_types[file_type] })

config/feature_flags/development/helm_packages.yml

+---
+name: helm_packages
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/61014
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/331693
+milestone: '14.0'
+type: development
+group: group::package
+default_enabled: false

config/metrics/counts_28d/20210517074859_i_package_helm_deploy_token_monthly.yml

+---
+key_path: redis_hll_counters.deploy_token_packages.i_package_helm_deploy_token_monthly
+description: Distinct Helm pakages deployed in recent 28 days
+product_section: ops
+product_stage: package
+product_group: group::package
+product_category: package_registry
+value_type: number
+status: implemented
+milestone: "14.0"
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/61014
+time_frame: 28d
+data_source: redis_hll
+distribution:
+- ce
+- ee
+tier:
+- free
+- premium
+- ultimate

config/metrics/counts_28d/20210517075259_i_package_helm_user_monthly.yml

+---
+key_path: redis_hll_counters.user_packages.i_package_helm_user_monthly
+description: Distinct user count events for Helm packages in recent 28 days
+product_section: ops
+product_stage: package
+product_group: group::package
+product_category: package_registry
+value_type: number
+status: implemented
+milestone: "14.0"
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/61014
+time_frame: 28d
+data_source: redis_hll
+distribution:
+- ce
+- ee
+tier:
+- free
+- premium
+- ultimate

config/metrics/counts_7d/20210517074851_i_package_helm_deploy_token_weekly.yml

+---
+key_path: redis_hll_counters.deploy_token_packages.i_package_helm_deploy_token_weekly
+description: Distinct Helm pakages deployed in recent 7 days
+product_section: ops
+product_stage: package
+product_group: group::package
+product_category: package_registry
+value_type: number
+status: implemented
+milestone: "14.0"
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/61014
+time_frame: 7d
+data_source: redis_hll
+distribution:
+- ce
+- ee
+tier:
+- free
+- premium
+- ultimate

config/metrics/counts_7d/20210517075252_i_package_helm_user_weekly.yml

+---
+key_path: redis_hll_counters.user_packages.i_package_helm_user_weekly
+description: Distinct user count events for Helm packages in recent 7 days
+product_section: ops
+product_stage: package
+product_group: group::package
+product_category: package_registry
+value_type: number
+status: implemented
+milestone: "14.0"
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/61014
+time_frame: 7d
+data_source: redis_hll
+distribution:
+- ce
+- ee
+tier:
+- free
+- premium
+- ultimate

config/metrics/counts_all/20210517073546_package_events_i_package_helm_pull_package.yml

+---
+key_path: counts.package_events_i_package_helm_pull_package
+description: Total count of pull Helm packages events
+product_section: ops
+product_stage: package
+product_group: group::package
+product_category: package_registry
+value_type: number
+status: implemented
+milestone: "14.0"
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/61014
+time_frame: all
+data_source: redis
+distribution:
+- ce
+- ee
+tier:
+- free
+- premium
+- ultimate

doc/development/usage_ping/dictionary.md

@@ -3718,6 +3718,18 @@ Status: `data_available`
 
 Tiers: `free`
 
+### `counts.package_events_i_package_helm_pull_package`
+
+Total count of pull Helm packages events
+
+[YAML definition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/metrics/counts_all/20210517073546_package_events_i_package_helm_pull_package.yml)
+
+Group: `group::package`
+
+Status: `implemented`
+
+Tiers: `free`, `premium`, `ultimate`
+
 ### `counts.package_events_i_package_maven_delete_package`
 
 Missing description
@@ -10198,6 +10210,30 @@ Status: `data_available`
 
 Tiers:
 
+### `redis_hll_counters.deploy_token_packages.i_package_helm_deploy_token_monthly`
+
+Distinct Helm pakages deployed in recent 28 days
+
+[YAML definition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/metrics/counts_28d/20210517074859_i_package_helm_deploy_token_monthly.yml)
+
+Group: `group::package`
+
+Status: `implemented`
+
+Tiers: `free`, `premium`, `ultimate`
+
+### `redis_hll_counters.deploy_token_packages.i_package_helm_deploy_token_weekly`
+
+Distinct Helm pakages deployed in recent 7 days
+
+[YAML definition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/metrics/counts_7d/20210517074851_i_package_helm_deploy_token_weekly.yml)
+
+Group: `group::package`
+
+Status: `implemented`
+
+Tiers: `free`, `premium`, `ultimate`
+
 ### `redis_hll_counters.deploy_token_packages.i_package_maven_deploy_token_monthly`
 
 Missing description
@@ -15286,6 +15322,30 @@ Status: `data_available`
 
 Tiers:
 
+### `redis_hll_counters.user_packages.i_package_helm_user_monthly`
+
+Distinct user count events for Helm packages in recent 28 days
+
+[YAML definition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/metrics/counts_28d/20210517075259_i_package_helm_user_monthly.yml)
+
+Group: `group::package`
+
+Status: `implemented`
+
+Tiers: `free`, `premium`, `ultimate`
+
+### `redis_hll_counters.user_packages.i_package_helm_user_weekly`
+
+Distinct user count events for Helm packages in recent 7 days
+
+[YAML definition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/metrics/counts_7d/20210517075252_i_package_helm_user_weekly.yml)
+
+Group: `group::package`
+
+Status: `implemented`
+
+Tiers: `free`, `premium`, `ultimate`
+
 ### `redis_hll_counters.user_packages.i_package_maven_user_monthly`
 
 Missing description

lib/api/api.rb

@@ -224,6 +224,7 @@ module API
       mount ::API::NpmInstancePackages
       mount ::API::GenericPackages
       mount ::API::GoProxy
+      mount ::API::HelmPackages
       mount ::API::Pages
       mount ::API::PagesDomains
       mount ::API::ProjectClusters

lib/api/helm_packages.rb

+# frozen_string_literal: true
+
+###
+# API endpoints for the Helm package registry
+module API
+  class HelmPackages < ::API::Base
+    helpers ::API::Helpers::PackagesHelpers
+    helpers ::API::Helpers::Packages::BasicAuthHelpers
+    include ::API::Helpers::Authentication
+
+    feature_category :package_registry
+
+    FILE_NAME_REQUIREMENTS = {
+      file_name: API::NO_SLASH_URL_PART_REGEX
+    }.freeze
+
+    content_type :binary, 'application/octet-stream'
+
+    authenticate_with do |accept|
+      accept.token_types(:personal_access_token, :deploy_token, :job_token)
+            .sent_through(:http_basic_auth)
+    end
+
+    before do
+      require_packages_enabled!
+    end
+
+    after_validation do
+      not_found! unless Feature.enabled?(:helm_packages, authorized_user_project)
+    end
+
+    params do
+      requires :id, type: String, desc: 'The ID or full path of a project'
+    end
+    resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
+      namespace ':id/packages/helm' do
+        desc 'Download a chart' do
+          detail 'This feature was introduced in GitLab 14.0'
+        end
+        params do
+          requires :channel, type: String, desc: 'Helm channel', regexp: Gitlab::Regex.helm_channel_regex
+          requires :file_name, type: String, desc: 'Helm package file name'
+        end
+        get ":channel/charts/:file_name.tgz", requirements: FILE_NAME_REQUIREMENTS do
+          authorize_read_package!(authorized_user_project)
+
+          package_file = Packages::Helm::PackageFilesFinder.new(authorized_user_project, params[:channel], file_name: "#{params[:file_name]}.tgz").execute.last!
+
+          track_package_event('pull_package', :helm)
+
+          present_carrierwave_file!(package_file.file)
+        end
+      end
+    end
+  end
+end

lib/gitlab/usage_data_counters/counter_events/package_events.yml

@@ -21,6 +21,7 @@
 - i_package_golang_delete_package
 - i_package_golang_pull_package
 - i_package_golang_push_package
+- i_package_helm_pull_package
 - i_package_maven_delete_package
 - i_package_maven_pull_package
 - i_package_maven_push_package

lib/gitlab/usage_data_counters/known_events/package_events.yml

@@ -47,6 +47,14 @@
   category: user_packages
   aggregation: weekly
   redis_slot: package
+- name: i_package_helm_deploy_token
+  category: deploy_token_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_helm_user
+  category: user_packages
+  aggregation: weekly
+  redis_slot: package
 - name: i_package_maven_deploy_token
   category: deploy_token_packages
   aggregation: weekly

spec/finders/packages/helm/package_files_finder_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe ::Packages::Helm::PackageFilesFinder do
+  let_it_be(:project1) { create(:project) }
+  let_it_be(:project2) { create(:project) }
+  let_it_be(:helm_package) { create(:helm_package, project: project1) }
+  let_it_be(:helm_package_file) { helm_package.package_files.first }
+  let_it_be(:debian_package) { create(:debian_package, project: project1) }
+
+  describe '#execute' do
+    let(:project) { project1 }
+    let(:channel) { 'stable' }
+    let(:params) { {} }
+
+    subject { described_class.new(project, channel, params).execute }
+
+    context 'with empty params' do
+      it { is_expected.to match_array([helm_package_file]) }
+    end
+
+    context 'with another project' do
+      let(:project) { project2 }
+
+      it { is_expected.to match_array([]) }
+    end
+
+    context 'with another channel' do
+      let(:channel) { 'staging' }
+
+      it { is_expected.to match_array([]) }
+    end
+
+    context 'with file_name' do
+      let(:params) { { file_name: helm_package_file.file_name } }
+
+      it { is_expected.to match_array([helm_package_file]) }
+    end
+
+    context 'with another file_name' do
+      let(:params) { { file_name: 'foobar.tgz' } }
+
+      it { is_expected.to match_array([]) }
+    end
+  end
+end

spec/lib/gitlab/usage_data_counters/package_event_counter_spec.rb

@@ -14,7 +14,7 @@ RSpec.describe Gitlab::UsageDataCounters::PackageEventCounter, :clean_gitlab_red
   end
 
   it 'includes the right events' do
-    expect(described_class::KNOWN_EVENTS.size).to eq 51
+    expect(described_class::KNOWN_EVENTS.size).to eq 52
   end
 
   described_class::KNOWN_EVENTS.each do |event|

spec/models/packages/package_file_spec.rb

@@ -116,6 +116,22 @@ RSpec.describe Packages::PackageFile, type: :model do
     end
   end
 
+  describe '.for_helm_with_channel' do
+    let_it_be(:project) { create(:project) }
+    let_it_be(:non_helm_package) { create(:nuget_package, project: project, package_type: :nuget) }
+    let_it_be(:helm_package1) { create(:helm_package, project: project, package_type: :helm) }
+    let_it_be(:helm_package2) { create(:helm_package, project: project, package_type: :helm) }
+    let_it_be(:channel) { 'some-channel' }
+
+    let_it_be(:non_helm_file) { create(:package_file, :nuget, package: non_helm_package) }
+    let_it_be(:helm_file1) { create(:helm_package_file, package: helm_package1) }
+    let_it_be(:helm_file2) { create(:helm_package_file, package: helm_package2, channel: channel) }
+
+    it 'returns the matching file only for Helm packages' do
+      expect(described_class.for_helm_with_channel(project, channel)).to contain_exactly(helm_file2)
+    end
+  end
+
   describe '#update_file_store callback' do
     let_it_be(:package_file) { build(:package_file, :nuget, size: nil) }
 

spec/requests/api/helm_packages_spec.rb

+# frozen_string_literal: true
+require 'spec_helper'
+
+RSpec.describe API::HelmPackages do
+  include_context 'helm api setup'
+
+  using RSpec::Parameterized::TableSyntax
+
+  let_it_be_with_reload(:project) { create(:project, :public) }
+  let_it_be(:deploy_token) { create(:deploy_token, read_package_registry: true, write_package_registry: true) }
+  let_it_be(:project_deploy_token) { create(:project_deploy_token, deploy_token: deploy_token, project: project) }
+
+  describe 'GET /api/v4/projects/:id/packages/helm/:channel/charts/:file_name.tgz' do
+    let_it_be(:package) { create(:helm_package, project: project) }
+
+    let(:channel) { package.package_files.first.helm_channel }
+
+    let(:url) { "/projects/#{project.id}/packages/helm/#{channel}/charts/#{package.name}-#{package.version}.tgz" }
+
+    subject { get api(url) }
+
+    context 'with valid project' do
+      where(:visibility, :user_role, :member, :user_token, :shared_examples_name, :expected_status) do
+        :public  | :developer  | true  | true  | 'process helm download content request'   | :success
+        :public  | :guest      | true  | true  | 'process helm download content request'   | :success
+        :public  | :developer  | true  | false | 'rejects helm packages access'            | :unauthorized
+        :public  | :guest      | true  | false | 'rejects helm packages access'            | :unauthorized
+        :public  | :developer  | false | true  | 'process helm download content request'   | :success
+        :public  | :guest      | false | true  | 'process helm download content request'   | :success
+        :public  | :developer  | false | false | 'rejects helm packages access'            | :unauthorized
+        :public  | :guest      | false | false | 'rejects helm packages access'            | :unauthorized
+        :public  | :anonymous  | false | true  | 'process helm download content request'   | :success
+        :private | :developer  | true  | true  | 'process helm download content request'   | :success
+        :private | :guest      | true  | true  | 'rejects helm packages access'            | :forbidden
+        :private | :developer  | true  | false | 'rejects helm packages access'            | :unauthorized
+        :private | :guest      | true  | false | 'rejects helm packages access'            | :unauthorized
+        :private | :developer  | false | true  | 'rejects helm packages access'            | :not_found
+        :private | :guest      | false | true  | 'rejects helm packages access'            | :not_found
+        :private | :developer  | false | false | 'rejects helm packages access'            | :unauthorized
+        :private | :guest      | false | false | 'rejects helm packages access'            | :unauthorized
+        :private | :anonymous  | false | true  | 'rejects helm packages access'            | :unauthorized
+      end
+
+      with_them do
+        let(:token) { user_token ? personal_access_token.token : 'wrong' }
+        let(:headers) { user_role == :anonymous ? {} : basic_auth_header(user.username, token) }
+
+        subject { get api(url), headers: headers }
+
+        before do
+          project.update!(visibility: visibility.to_s)
+        end
+
+        it_behaves_like params[:shared_examples_name], params[:user_role], params[:expected_status], params[:member]
+      end
+    end
+
+    it_behaves_like 'deploy token for package GET requests'
+
+    it_behaves_like 'rejects helm access with unknown project id'
+  end
+end

spec/support/shared_contexts/requests/api/helm_packages_shared_context.rb

+# frozen_string_literal: true
+
+RSpec.shared_context 'helm api setup' do
+  include WorkhorseHelpers
+  include PackagesManagerApiSpecHelpers
+  include HttpBasicAuthHelpers
+
+  let_it_be(:user) { create(:user) }
+  let_it_be(:personal_access_token) { create(:personal_access_token, user: user) }
+end

spec/support/shared_examples/requests/api/helm_packages_shared_examples.rb

+# frozen_string_literal: true
+
+RSpec.shared_examples 'rejects helm packages access' do |user_type, status, add_member = true|
+  context "for user type #{user_type}" do
+    before do
+      project.send("add_#{user_type}", user) if add_member && user_type != :anonymous
+    end
+
+    it_behaves_like 'returning response status', status
+
+    if status == :unauthorized
+      it 'has the correct response header' do
+        subject
+
+        expect(response.headers['WWW-Authenticate']).to eq 'Basic realm="GitLab Packages Registry"'
+      end
+    end
+  end
+end
+
+RSpec.shared_examples 'process helm download content request' do |user_type, status, add_member = true|
+  context "for user type #{user_type}" do
+    before do
+      project.send("add_#{user_type}", user) if add_member && user_type != :anonymous
+    end
+
+    it_behaves_like 'returning response status', status
+
+    it_behaves_like 'a package tracking event', 'API::HelmPackages', 'pull_package'
+
+    it 'returns a valid package archive' do
+      subject
+
+      expect(response.media_type).to eq('application/octet-stream')
+    end
+  end
+end
+
+RSpec.shared_examples 'rejects helm access with unknown project id' do
+  context 'with an unknown project' do
+    let(:project) { OpenStruct.new(id: 1234567890) }
+
+    context 'as anonymous' do
+      it_behaves_like 'rejects helm packages access', :anonymous, :unauthorized
+    end
+
+    context 'as authenticated user' do
+      subject { get api(url), headers: basic_auth_header(user.username, personal_access_token.token) }
+
+      it_behaves_like 'rejects helm packages access', :anonymous, :not_found
+    end
+  end
+end