Merge branch '11631-dependency-proxy-purge-api' into 'master'

Endpoint for purging group dependency proxy cache

See merge request gitlab-org/gitlab!27843

changelogs/unreleased/11631-dependency-proxy-purge-api.yml

+---
+title: Add an endpoint to allow group admin users to purge the dependency proxy for a group
+merge_request: 27843
+author:
+type: added

config/sidekiq_queues.yml

@@ -66,6 +66,8 @@
   - 1
 - - delete_user
   - 1
+- - dependency_proxy
+  - 1
 - - deployment
   - 3
 - - design_management_new_version

doc/api/dependency_proxy.md

+# Dependency Proxy API **(PREMIUM)**
+
+## Purge the dependency proxy for a group
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/11631) in GitLab 12.10.
+
+Deletes the cached blobs for a group. This endpoint requires group admin access.
+
+```plaintext
+DELETE /groups/:id/dependency_proxy/cache
+```
+
+| Attribute | Type | Required | Description |
+| --------- | ---- | -------- | ----------- |
+| `id`      | integer/string | yes | The ID or [URL-encoded path of the group](README.md#namespaced-path-encoding) owned by the authenticated user |
+
+Example request:
+
+```shell
+curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/groups/5/dependency_proxy/cache"
+```

doc/user/packages/dependency_proxy/index.md

@@ -65,6 +65,13 @@ from GitLab.
 The blobs are kept forever, and there is no hard limit on how much data can be
 stored.
 
+## Clearing the cache
+
+It is possible to use the GitLab API to purge the dependency proxy cache for a
+given group to gain back disk space that may be taken up by image blobs that
+are no longer needed. See the [dependency proxy API documentation](../../../api/dependency_proxy.md)
+for more details.
+
 ## Limitations
 
 The following limitations apply:

ee/app/services/ee/groups/destroy_service.rb

@@ -7,11 +7,21 @@ module EE
 
       override :execute
       def execute
-        super.tap { |group| log_audit_event unless group&.persisted? }
+        super.tap do |group|
+          delete_dependency_proxy_blobs(group)
+
+          log_audit_event unless group&.persisted?
+        end
       end
 
       private
 
+      def delete_dependency_proxy_blobs(group)
+        # the blobs reference files that need to be destroyed that cascade delete
+        # does not remove
+        group.dependency_proxy_blobs.destroy_all # rubocop:disable Cop/DestroyAll
+      end
+
       def log_audit_event
         ::AuditEventService.new(
           current_user,

ee/app/workers/all_queues.yml

@@ -192,6 +192,13 @@
   :resource_boundary: :cpu
   :weight: 1
   :idempotent: 
+- :name: dependency_proxy:purge_dependency_proxy_cache
+  :feature_category: :dependency_proxy
+  :has_external_dependencies: 
+  :urgency: :low
+  :resource_boundary: :unknown
+  :weight: 1
+  :idempotent: true
 - :name: epics:epics_update_epics_dates
   :feature_category: :epics
   :has_external_dependencies: 

ee/app/workers/purge_dependency_proxy_cache_worker.rb

+# frozen_string_literal: true
+
+class PurgeDependencyProxyCacheWorker
+  include ApplicationWorker
+  include Gitlab::Allowable
+  idempotent!
+
+  queue_namespace :dependency_proxy
+  feature_category :dependency_proxy
+
+  def perform(current_user_id, group_id)
+    @current_user = User.find_by_id(current_user_id)
+    @group = Group.find_by_id(group_id)
+
+    return unless valid?
+
+    @group.dependency_proxy_blobs.destroy_all # rubocop:disable Cop/DestroyAll
+  end
+
+  private
+
+  def valid?
+    return unless @group
+
+    can?(@current_user, :admin_group, @group) && @group.dependency_proxy_feature_available?
+  end
+end

ee/lib/api/dependency_proxy.rb

+# frozen_string_literal: true
+
+module API
+  class DependencyProxy < Grape::API
+    helpers ::API::Helpers::PackagesHelpers
+
+    helpers do
+      def obtain_new_purge_cache_lease
+        Gitlab::ExclusiveLease
+          .new("dependency_proxy:delete_group_blobs:#{user_group.id}",
+               timeout: 1.hour)
+          .try_obtain
+      end
+    end
+
+    before do
+      authorize! :admin_group, user_group
+    end
+
+    params do
+      requires :id, type: String, desc: 'The ID of a group'
+    end
+    resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
+      desc 'Deletes all dependency_proxy_blobs for a group' do
+        detail 'This feature was introduced in GitLab 12.10'
+      end
+      delete ':id/dependency_proxy/cache' do
+        not_found! unless user_group.dependency_proxy_feature_available?
+
+        message = 'This request has already been made. It may take some time to purge the cache. You can run this at most once an hour for a given group'
+        render_api_error!(message, 409) unless obtain_new_purge_cache_lease
+
+        # rubocop:disable CodeReuse/Worker
+        PurgeDependencyProxyCacheWorker.perform_async(current_user.id, user_group.id)
+        # rubocop:enable CodeReuse/Worker
+      end
+    end
+  end
+end

ee/lib/api/helpers/packages_helpers.rb

@@ -9,6 +9,10 @@ module API
         not_found! unless ::Gitlab.config.packages.enabled
       end
 
+      def require_dependency_proxy_enabled!
+        not_found! unless ::Gitlab.config.dependency_proxy.enabled
+      end
+
       def authorize_packages_feature!(subject = user_project)
         forbidden! unless subject.feature_available?(:packages)
       end

ee/lib/ee/api/api.rb

@@ -15,6 +15,7 @@ module EE
         mount ::API::ProjectApprovalRules
         mount ::API::ProjectApprovalSettings
         mount ::API::Unleash
+        mount ::API::DependencyProxy
         mount ::API::EpicIssues
         mount ::API::EpicLinks
         mount ::API::Epics

ee/spec/requests/api/dependency_proxy_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe API::DependencyProxy, api: true do
+  include ExclusiveLeaseHelpers
+
+  let_it_be(:user) { create(:user) }
+  let_it_be(:blob) { create(:dependency_proxy_blob )}
+  let_it_be(:group) { blob.group }
+
+  before do
+    group.add_owner(user)
+    stub_config(dependency_proxy: { enabled: true })
+    stub_licensed_features(dependency_proxy: true)
+  end
+
+  describe 'DELETE /groups/:id/dependency_proxy/cache' do
+    subject { delete api("/groups/#{group.id}/dependency_proxy/cache", user) }
+
+    context 'with feature available and enabled' do
+      let_it_be(:lease_key) { "dependency_proxy:delete_group_blobs:#{group.id}" }
+
+      context 'an admin user' do
+        it 'deletes the blobs and returns no content' do
+          stub_exclusive_lease(lease_key, timeout: 1.hour)
+          expect(PurgeDependencyProxyCacheWorker).to receive(:perform_async)
+
+          subject
+
+          expect(response).to have_gitlab_http_status(:no_content)
+        end
+
+        context 'called multiple times in one hour', :clean_gitlab_redis_shared_state do
+          it 'returns 409 with an error message' do
+            stub_exclusive_lease_taken(lease_key, timeout: 1.hour)
+
+            subject
+
+            expect(response).to have_gitlab_http_status(:conflict)
+            expect(response.body).to include('This request has already been made.')
+          end
+
+          it 'executes service only for the first time' do
+            expect(PurgeDependencyProxyCacheWorker).to receive(:perform_async).once
+
+            2.times { subject }
+          end
+        end
+      end
+
+      context 'a non-admin' do
+        let(:user) { create(:user) }
+
+        before do
+          group.add_maintainer(user)
+        end
+
+        it_behaves_like 'returning response status', :forbidden
+      end
+    end
+
+    context 'depencency proxy is not enabled' do
+      before do
+        stub_config(dependency_proxy: { enabled: false })
+      end
+
+      it_behaves_like 'returning response status', :not_found
+    end
+
+    context 'dependency feature is not available' do
+      before do
+        stub_config(dependency_proxy: { enabled: true })
+        stub_licensed_features(dependency_proxy: false)
+      end
+
+      it_behaves_like 'returning response status', :not_found
+    end
+  end
+end

ee/spec/services/groups/destroy_service_spec.rb

@@ -31,4 +31,17 @@ describe Groups::DestroyService do
       end
     end
   end
+
+  context 'dependency_proxy_blobs' do
+    let_it_be(:blob) { create(:dependency_proxy_blob) }
+    let_it_be(:group) { blob.group }
+
+    before do
+      group.add_maintainer(user)
+    end
+
+    it 'destroys the dependency proxy blobs' do
+      expect { subject.execute }.to change { DependencyProxy::Blob.count }.by(-1)
+    end
+  end
 end

ee/spec/workers/purge_dependency_proxy_cache_worker_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe PurgeDependencyProxyCacheWorker do
+  let_it_be(:user) { create(:admin) }
+  let_it_be(:blob) { create(:dependency_proxy_blob )}
+  let_it_be(:group) { blob.group }
+  let_it_be(:group_id) { group.id }
+
+  subject { described_class.new.perform(user.id, group_id) }
+
+  before do
+    stub_config(dependency_proxy: { enabled: true })
+    stub_licensed_features(dependency_proxy: true)
+  end
+
+  describe '#perform' do
+    shared_examples 'returns nil' do
+      it 'returns nil' do
+        expect { subject }.not_to change { group.dependency_proxy_blobs.size }
+        expect(subject).to be_nil
+      end
+    end
+
+    context 'an admin user' do
+      include_examples 'an idempotent worker' do
+        let(:job_args) { [user.id, group_id] }
+
+        it 'deletes the blobs and returns ok' do
+          expect(group.dependency_proxy_blobs.size).to eq(1)
+
+          subject
+
+          expect(group.dependency_proxy_blobs.size).to eq(0)
+        end
+      end
+    end
+
+    context 'a non-admin user' do
+      let(:user) { create(:user) }
+
+      it_behaves_like 'returns nil'
+    end
+
+    context 'an invalid user id' do
+      let(:user) { double('User', id: 99999 ) }
+
+      it_behaves_like 'returns nil'
+    end
+
+    context 'an invalid group' do
+      let(:group_id) { 99999 }
+
+      it_behaves_like 'returns nil'
+    end
+  end
+end