Merge branch 'feature/dependency-proxy-migrate-rake' into 'master'

Add dependency proxy migrate rake task

See merge request gitlab-org/gitlab!79663

app/models/dependency_proxy/blob.rb

@@ -14,6 +14,8 @@ class DependencyProxy::Blob < ApplicationRecord
   validates :file, presence: true
   validates :file_name, presence: true
 
+  scope :with_files_stored_locally, -> { where(file_store: ::DependencyProxy::FileUploader::Store::LOCAL) }
+
   mount_file_store_uploader DependencyProxy::FileUploader
 
   def self.total_size
@@ -24,3 +26,5 @@ class DependencyProxy::Blob < ApplicationRecord
     find_or_initialize_by(file_name: file_name)
   end
 end
+
+DependencyProxy::Blob.prepend_mod_with('DependencyProxy::Blob')

app/models/dependency_proxy/manifest.rb

@@ -17,6 +17,7 @@ class DependencyProxy::Manifest < ApplicationRecord
   validates :digest, presence: true
 
   scope :order_id_desc, -> { reorder(id: :desc) }
+  scope :with_files_stored_locally, -> { where(file_store: ::DependencyProxy::FileUploader::Store::LOCAL) }
 
   mount_file_store_uploader DependencyProxy::FileUploader
 
@@ -24,3 +25,5 @@ class DependencyProxy::Manifest < ApplicationRecord
     find_by(file_name: file_name) || find_by(digest: digest)
   end
 end
+
+DependencyProxy::Manifest.prepend_mod_with('DependencyProxy::Manifest')

doc/administration/object_storage.md

@@ -526,7 +526,7 @@ To migrate existing local data to object storage see the following guides:
 - [Uploads](raketasks/uploads/migrate.md#migrate-to-object-storage)
 - [Merge request diffs](merge_request_diffs.md#using-object-storage)
 - [Packages](packages/index.md#migrating-local-packages-to-object-storage) (optional feature)
-- Dependency Proxy - [migration not yet supported](https://gitlab.com/gitlab-org/gitlab/-/issues/343064)
+- [Dependency Proxy](packages/dependency_proxy.md#migrate-local-dependency-proxy-blobs-and-manifests-to-object-storage)
 - [Terraform state files](terraform_state.md#migrate-to-object-storage)
 - [Pages content](pages/index.md#migrate-pages-deployments-to-object-storage)
 

doc/administration/packages/dependency_proxy.md

@@ -199,20 +199,52 @@ This section describes the earlier configuration format.
 
 1. [Restart GitLab](../restart_gitlab.md#installations-from-source "How to restart GitLab") for the changes to take effect.
 
-### Migrate from local storage to object storage
+#### Migrate local Dependency Proxy blobs and manifests to object storage
 
-GitLab currently [doesn't support](https://gitlab.com/gitlab-org/gitlab/-/issues/343064)
-migrating the existing cache to object storage.
+After [configuring object storage](#using-object-storage),
+use the following task to migrate existing Dependency Proxy blobs and manifests from local storage
+to remote storage. The processing is done in a background worker and requires no downtime.
 
-To switch from local storage to object storage:
+For Omnibus GitLab:
 
-1. [Reconfigure GitLab to use object storage](#using-object-storage).
-1. Purge the dependency proxy cache (on both local and object storage)
-   using the [Dependency Proxy API](../../api/dependency_proxy.md#purge-the-dependency-proxy-for-a-group).
+```shell
+sudo gitlab-rake "gitlab:dependency_proxy:migrate"
+```
+
+For installations from source:
+
+```shell
+RAILS_ENV=production sudo -u git -H bundle exec rake gitlab:dependency_proxy:migrate
+```
+
+You can optionally track progress and verify that all packages migrated successfully using the
+[PostgreSQL console](https://docs.gitlab.com/omnibus/settings/database.html#connecting-to-the-bundled-postgresql-database):
+
+- For Omnibus GitLab instances: `sudo gitlab-rails dbconsole`
+- For installations from source: `sudo -u git -H psql -d gitlabhq_production`
+
+Verify that `objectstg` (where `file_store = '2'`) has the count of all Dependency Proxy blobs and
+manifests for each respective query:
 
-Object storage now caches container images when they're requested. This also
-[reduces the size of dependency proxy storage](../../user/packages/dependency_proxy/reduce_dependency_proxy_storage.md)
-by removing unused images.
+```shell
+gitlabhq_production=# SELECT count(*) AS total, sum(case when file_store = '1' then 1 else 0 end) AS filesystem, sum(case when file_store = '2' then 1 else 0 end) AS objectstg FROM dependency_proxy_blobs;
+
+total | filesystem | objectstg
+------+------------+-----------
+ 22   |          0 |        22
+
+gitlabhq_production=# SELECT count(*) AS total, sum(case when file_store = '1' then 1 else 0 end) AS filesystem, sum(case when file_store = '2' then 1 else 0 end) AS objectstg FROM dependency_proxy_manifests;
+
+total | filesystem | objectstg
+------+------------+-----------
+ 10   |          0 |        10
+```
+
+Verify that there are no files on disk in the `packages` folder:
+
+```shell
+sudo find /var/opt/gitlab/gitlab-rails/shared/dependency_proxy -type f | grep -v tmp | wc -l
+```
 
 ## Disabling Authentication
 

ee/app/models/ee/dependency_proxy/blob.rb

+# frozen_string_literal: true
+
+module EE
+  module DependencyProxy
+    module Blob
+      extend ActiveSupport::Concern
+
+      def log_geo_deleted_event
+        # Keep empty for now. Should be addressed in future
+        # by https://gitlab.com/gitlab-org/gitlab/-/issues/259694
+      end
+    end
+  end
+end

ee/app/models/ee/dependency_proxy/manifest.rb

+# frozen_string_literal: true
+
+module EE
+  module DependencyProxy
+    module Manifest
+      extend ActiveSupport::Concern
+
+      def log_geo_deleted_event
+        # Keep empty for now. Should be addressed in future
+        # by https://gitlab.com/gitlab-org/gitlab/-/issues/259694
+      end
+    end
+  end
+end

lib/tasks/gitlab/dependency_proxy/migrate.rake

+# frozen_string_literal: true
+
+require 'logger'
+
+desc "GitLab | Dependency Proxy | Migrate dependency proxy files to remote storage"
+namespace :gitlab do
+  namespace :dependency_proxy do
+    task migrate: :environment do
+      logger = Logger.new($stdout)
+      logger.info('Starting transfer of dependency proxy files to object storage')
+
+      unless ::DependencyProxy::FileUploader.object_store_enabled?
+        raise 'Object store is disabled for dependency proxy feature'
+      end
+
+      ::DependencyProxy::Blob.with_files_stored_locally.find_each(batch_size: 10) do |blob_file|
+        blob_file.file.migrate!(::DependencyProxy::FileUploader::Store::REMOTE)
+
+        logger.info("Transferred dependency proxy blob file #{blob_file.id} of size #{blob_file.size.to_i.bytes} to object storage")
+      rescue StandardError => e
+        logger.error("Failed to transfer dependency proxy blob file #{blob_file.id} with error: #{e.message}")
+      end
+
+      ::DependencyProxy::Manifest.with_files_stored_locally.find_each(batch_size: 10) do |manifest_file|
+        manifest_file.file.migrate!(::DependencyProxy::FileUploader::Store::REMOTE)
+
+        logger.info("Transferred dependency proxy manifest file #{manifest_file.id} of size #{manifest_file.size.to_i.bytes} to object storage")
+      rescue StandardError => e
+        logger.error("Failed to transfer dependency proxy manifest file #{manifest_file.id} with error: #{e.message}")
+      end
+    end
+  end
+end

spec/tasks/gitlab/dependency_proxy/migrate_rake_spec.rb

+# frozen_string_literal: true
+
+require 'rake_helper'
+
+RSpec.describe 'gitlab:dependency_proxy namespace rake task', :silence_stdout do
+  before :all do
+    Rake.application.rake_require 'tasks/gitlab/dependency_proxy/migrate'
+  end
+
+  describe 'migrate' do
+    let(:local) { ObjectStorage::Store::LOCAL }
+    let(:remote) { ObjectStorage::Store::REMOTE }
+    let!(:blob) { create(:dependency_proxy_blob) }
+    let!(:manifest) { create(:dependency_proxy_manifest) }
+
+    def dependency_proxy_migrate
+      run_rake_task('gitlab:dependency_proxy:migrate')
+    end
+
+    context 'object storage disabled' do
+      before do
+        stub_dependency_proxy_object_storage(enabled: false)
+      end
+
+      it "doesn't migrate files" do
+        expect { dependency_proxy_migrate }.to raise_error('Object store is disabled for dependency proxy feature')
+      end
+    end
+
+    context 'object storage enabled' do
+      before do
+        stub_dependency_proxy_object_storage
+      end
+
+      it 'migrates local file to object storage' do
+        expect { dependency_proxy_migrate }.to change { blob.reload.file_store }.from(local).to(remote)
+          .and change { manifest.reload.file_store }.from(local).to(remote)
+      end
+    end
+
+    context 'an error is raised while migrating' do
+      let(:blob_error) { 'Failed to transfer dependency proxy blob file' }
+      let(:manifest_error) { 'Failed to transfer dependency proxy manifest file' }
+      let!(:blob_non_existent) { create(:dependency_proxy_blob) }
+      let!(:manifest_non_existent) { create(:dependency_proxy_manifest) }
+
+      before do
+        stub_dependency_proxy_object_storage
+        blob_non_existent.file.file.delete
+        manifest_non_existent.file.file.delete
+      end
+
+      it 'fails to migrate a local file that does not exist' do
+        expect { dependency_proxy_migrate }.to output(include(blob_error, manifest_error)).to_stdout
+      end
+    end
+  end
+end