E2E test update: add env var for non-cluster storage

qa/qa/runtime/api/client.rb

@@ -6,6 +6,8 @@ module QA
       class Client
         attr_reader :address, :user
 
+        AuthorizationError = Class.new(RuntimeError)
+
         def initialize(address = :gitlab, personal_access_token: nil, is_new_session: true, user: nil, ip_limits: false)
           @address = address
           @personal_access_token = personal_access_token

qa/qa/runtime/api/repository_storage_moves.rb

@@ -10,16 +10,24 @@ module QA
         RepositoryStorageMovesError = Class.new(RuntimeError)
 
         def has_status?(project, status, destination_storage = Env.additional_repository_storage)
-          all.any? do |move|
-            move[:project][:path_with_namespace] == project.path_with_namespace &&
+          find_any do |move|
+            next unless move[:project][:path_with_namespace] == project.path_with_namespace
+
+            QA::Runtime::Logger.debug("Move data: #{move}")
+
             move[:state] == status &&
             move[:destination_storage_name] == destination_storage
           end
         end
 
-        def all
+        def find_any
           Logger.debug('Getting repository storage moves')
-          parse_body(get(Request.new(api_client, '/project_repository_storage_moves').url))
+
+          Support::Waiter.wait_until do
+            with_paginated_response_body(Request.new(api_client, '/project_repository_storage_moves', per_page: '100').url) do |page|
+              break true if page.any? { |item| yield item }
+            end
+          end
         end
 
         private

qa/qa/runtime/env.rb

@@ -61,6 +61,10 @@ module QA
         ENV['QA_ADDITIONAL_REPOSITORY_STORAGE']
       end
 
+      def non_cluster_repository_storage
+        ENV['QA_GITALY_NON_CLUSTER_STORAGE'] || 'gitaly'
+      end
+
       def praefect_repository_storage
         ENV['QA_PRAEFECT_REPOSITORY_STORAGE']
       end

qa/qa/service/praefect_manager.rb

@@ -5,6 +5,8 @@ module QA
     class PraefectManager
       include Service::Shellout
 
+      attr_accessor :gitlab
+
       def initialize
         @gitlab = 'gitlab-gitaly-ha'
         @praefect = 'praefect'
@@ -100,6 +102,14 @@ module QA
         enable_writes
       end
 
+      def verify_storage_move(source_storage, destination_storage)
+        return if QA::Runtime::Env.dot_com?
+
+        repo_path = verify_storage_move_from_gitaly(source_storage[:name])
+
+        destination_storage[:type] == :praefect ? verify_storage_move_to_praefect(repo_path, destination_storage[:name]) : verify_storage_move_to_gitaly(repo_path, destination_storage[:name])
+      end
+
       def wait_for_praefect
         wait_until_shell_command_matches(
           "docker exec #{@praefect} bash -c 'cat /var/log/gitlab/praefect/current'",
@@ -163,16 +173,48 @@ module QA
 
       private
 
-      def wait_until_shell_command(cmd)
-        Support::Waiter.wait_until do
+      def verify_storage_move_from_gitaly(storage)
+        wait_until_shell_command("docker exec #{@gitlab} bash -c 'tail -n 50 /var/log/gitlab/gitaly/current'") do |line|
+          log = JSON.parse(line)
+
+          break log['grpc.request.repoPath'] if log['grpc.method'] == 'RenameRepository' && log['grpc.request.repoStorage'] == storage && !log['grpc.request.repoPath'].include?('wiki')
+        rescue JSON::ParserError
+          # Ignore lines that can't be parsed as JSON
+        end
+      end
+
+      def verify_storage_move_to_praefect(repo_path, virtual_storage)
+        wait_until_shell_command("docker exec #{@gitlab} bash -c 'tail -n 50 /var/log/gitlab/praefect/current'") do |line|
+          log = JSON.parse(line)
+
+          log['grpc.method'] == 'ReplicateRepository' && log['virtual_storage'] == virtual_storage && log['relative_path'] == repo_path
+        rescue JSON::ParserError
+          # Ignore lines that can't be parsed as JSON
+        end
+      end
+
+      def verify_storage_move_to_gitaly(repo_path, storage)
+        wait_until_shell_command("docker exec #{@gitlab} bash -c 'tail -n 50 /var/log/gitlab/gitaly/current'") do |line|
+          log = JSON.parse(line)
+
+          log['grpc.method'] == 'ReplicateRepository' && log['grpc.request.repoStorage'] == storage && log['grpc.request.repoPath'] == repo_path
+        rescue JSON::ParserError
+          # Ignore lines that can't be parsed as JSON
+        end
+      end
+
+      def wait_until_shell_command(cmd, **kwargs)
+        sleep_interval = kwargs.delete(:sleep_interval) || 1
+
+        Support::Waiter.wait_until(sleep_interval: sleep_interval, **kwargs) do
           shell cmd do |line|
             break true if yield line
           end
         end
       end
 
-      def wait_until_shell_command_matches(cmd, regex)
-        wait_until_shell_command(cmd) do |line|
+      def wait_until_shell_command_matches(cmd, regex, **kwargs)
+        wait_until_shell_command(cmd, kwargs) do |line|
           QA::Runtime::Logger.info(line.chomp)
 
           line =~ regex

qa/qa/specs/features/api/3_create/repository/changing_repository_storage_spec.rb

@@ -3,13 +3,14 @@
 module QA
   RSpec.describe 'Create' do
     describe 'Changing Gitaly repository storage', :requires_admin do
+      praefect_manager = Service::PraefectManager.new
+      praefect_manager.gitlab = 'gitlab'
+
       shared_examples 'repository storage move' do
         it 'confirms a `finished` status after moving project repository storage' do
           expect(project).to have_file('README.md')
-
-          project.change_repository_storage(destination_storage)
-
-          expect(Runtime::API::RepositoryStorageMoves).to have_status(project, 'finished', destination_storage)
+          expect { project.change_repository_storage(destination_storage[:name]) }.not_to raise_error
+          expect { praefect_manager.verify_storage_move(source_storage, destination_storage) }.not_to raise_error
 
           Resource::Repository::ProjectPush.fabricate! do |push|
             push.project = project
@@ -25,28 +26,35 @@ module QA
       end
 
       context 'when moving from one Gitaly storage to another', :orchestrated, :repository_storage do
+        let(:source_storage) { { type: :gitaly, name: 'default' } }
+        let(:destination_storage) { { type: :gitaly, name: QA::Runtime::Env.additional_repository_storage } }
+
         let(:project) do
           Resource::Project.fabricate_via_api! do |project|
             project.name = 'repo-storage-move-status'
             project.initialize_with_readme = true
+            project.api_client = Runtime::API::Client.as_admin
           end
         end
-        let(:destination_storage) { QA::Runtime::Env.additional_repository_storage }
 
         it_behaves_like 'repository storage move'
       end
 
       # Note: This test doesn't have the :orchestrated tag because it runs in the Test::Integration::Praefect
       # scenario with other tests that aren't considered orchestrated.
-      context 'when moving from Gitaly to Gitaly Cluster', :requires_praefect, quarantine: { issue: 'https://gitlab.com/gitlab-org/gitlab/-/issues/227127', type: :investigating } do
+      # It also runs on staging using nfs-file07 as non-cluster storage and nfs-file22 as cluster/praefect storage
+      context 'when moving from Gitaly to Gitaly Cluster', :requires_praefect do
+        let(:source_storage) { { type: :gitaly, name: QA::Runtime::Env.non_cluster_repository_storage } }
+        let(:destination_storage) { { type: :praefect, name: QA::Runtime::Env.praefect_repository_storage } }
+
         let(:project) do
           Resource::Project.fabricate_via_api! do |project|
             project.name = 'repo-storage-move'
             project.initialize_with_readme = true
-            project.repository_storage = 'gitaly'
+            project.repository_storage = source_storage[:name]
+            project.api_client = Runtime::API::Client.as_admin
           end
         end
-        let(:destination_storage) { QA::Runtime::Env.praefect_repository_storage }
 
         it_behaves_like 'repository storage move'
       end

qa/qa/support/api.rb

@@ -77,6 +77,30 @@ module QA
 
         error.response
       end
+
+      def with_paginated_response_body(url)
+        loop do
+          response = get(url)
+
+          QA::Runtime::Logger.debug("Fetching page #{response.headers[:x_page]} of #{response.headers[:x_total_pages]}...")
+
+          yield parse_body(response)
+
+          next_link = pagination_links(response).find { |link| link[:rel] == 'next' }
+          break unless next_link
+
+          url = next_link[:url]
+        end
+      end
+
+      def pagination_links(response)
+        response.headers[:link].split(',').map do |link|
+          match = link.match(/\<(?<url>.*)\>\; rel=\"(?<rel>\w+)\"/)
+          break nil unless match
+
+          { url: match[:url], rel: match[:rel] }
+        end.compact
+      end
     end
   end
 end