Make Hashed Storage rollback resilient to exceptions

app/services/projects/hashed_storage/rollback_repository_service.rb

@@ -21,14 +21,32 @@ module Projects
           project.storage_version = ::Project::HASHED_STORAGE_FEATURES[:repository]
         end
 
-        project.repository_read_only = false
-        project.save!(validate: false)
-
-        if result && block_given?
-          yield
+        project.transaction do
+          project.save!(validate: false)
+          project.set_repository_writable!
         end
 
         result
+      rescue Gitlab::Git::CommandError => e
+        logger.error("Repository #{project.full_path} failed to rollback (PROJECT_ID=#{project.id}). Git operation failed: #{e.inspect}")
+
+        rollback_migration!
+
+        false
+      rescue OpenSSL::Cipher::CipherError => e
+        logger.error("Repository #{project.full_path} failed to rollback (PROJECT_ID=#{project.id}). There is a problem with encrypted attributes: #{e.inspect}")
+
+        rollback_migration!
+
+        false
+      end
+
+      private
+
+      def rollback_migration!
+        rollback_folder_move
+        project.storage_version = ::Project::HASHED_STORAGE_FEATURES[:repository]
+        project.set_repository_writable!
       end
     end
   end

changelogs/unreleased/259605-repositories-gone-after-update-broken-migration-to-hashed-storage.yml

+---
+title: "Hashed Storage: make migration and rollback resilient to exceptions"
+merge_request: 46178
+author:
+type: fixed

spec/services/projects/hashed_storage/rollback_repository_service_spec.rb

@@ -77,6 +77,42 @@ RSpec.describe Projects::HashedStorage::RollbackRepositoryService, :clean_gitlab
       end
     end
 
+    context 'when exception happens' do
+      it 'handles OpenSSL::Cipher::CipherError' do
+        expect(project).to receive(:ensure_runners_token).and_raise(OpenSSL::Cipher::CipherError)
+
+        expect { service.execute }.not_to raise_exception
+      end
+
+      it 'ensures rollback when OpenSSL::Cipher::CipherError' do
+        expect(project).to receive(:ensure_runners_token).and_raise(OpenSSL::Cipher::CipherError)
+        expect(service).to receive(:rollback_folder_move).and_call_original
+
+        service.execute
+        project.reload
+
+        expect(project.hashed_storage?(:repository)).to be_truthy
+        expect(project.repository_read_only?).to be_falsey
+      end
+
+      it 'handles Gitlab::Git::CommandError' do
+        expect(project).to receive(:write_repository_config).and_raise(Gitlab::Git::CommandError)
+
+        expect { service.execute }.not_to raise_exception
+      end
+
+      it 'ensures rollback when Gitlab::Git::CommandError' do
+        expect(project).to receive(:write_repository_config).and_raise(Gitlab::Git::CommandError)
+        expect(service).to receive(:rollback_folder_move).and_call_original
+
+        service.execute
+        project.reload
+
+        expect(project.hashed_storage?(:repository)).to be_truthy
+        expect(project.repository_read_only?).to be_falsey
+      end
+    end
+
     context 'when one move fails' do
       it 'rolls repositories back to original name' do
         allow(service).to receive(:move_repository).and_call_original