Add E2E test of backend node recovery

Includes changes to allow Resources to return enumerable results via
the API

qa/qa/resource/project.rb

@@ -123,6 +123,10 @@ module QA
         "#{api_get_path}/runners"
       end
 
+      def api_commits_path
+        "#{api_get_path}/repository/commits"
+      end
+
       def api_repository_branches_path
         "#{api_get_path}/repository/branches"
       end
@@ -176,6 +180,10 @@ module QA
         raise Runtime::API::RepositoryStorageMoves::RepositoryStorageMovesError, 'Timed out while waiting for the repository storage move to finish'
       end
 
+      def commits
+        parse_body(get(Runtime::API::Request.new(api_client, api_commits_path).url))
+      end
+
       def import_status
         response = get Runtime::API::Request.new(api_client, "/projects/#{id}/import").url
 

qa/qa/service/shellout.rb

@@ -33,6 +33,31 @@ module QA
           end
         end
       end
+
+      def sql_to_docker_exec_cmd(sql, username, password, database, host, container)
+        <<~CMD
+          docker exec --env PGPASSWORD=#{password} #{container} \
+            bash -c "psql -U #{username} -d #{database} -h #{host} -c \\"#{sql}\\""
+        CMD
+      end
+
+      def wait_until_shell_command(cmd, **kwargs)
+        sleep_interval = kwargs.delete(:sleep_interval) || 1
+
+        Support::Waiter.wait_until(sleep_interval: sleep_interval, **kwargs) do
+          shell cmd do |line|
+            break true if yield line
+          end
+        end
+      end
+
+      def wait_until_shell_command_matches(cmd, regex, **kwargs)
+        wait_until_shell_command(cmd, kwargs) do |line|
+          QA::Runtime::Logger.debug(line.chomp)
+
+          line =~ regex
+        end
+      end
     end
   end
 end

qa/qa/specs/features/api/3_create/repository/backend_node_recovery_spec.rb

+# frozen_string_literal: true
+
+module QA
+  RSpec.describe 'Create' do
+    context 'Gitaly' do
+      describe 'Backend node recovery', :orchestrated, :gitaly_ha, :skip_live_env do
+        let(:praefect_manager) { Service::PraefectManager.new }
+        let(:project) do
+          Resource::Project.fabricate! do |project|
+            project.name = "gitaly_cluster"
+            project.initialize_with_readme = true
+          end
+        end
+
+        before do
+          # Reset the cluster in case previous tests left it in a bad state
+          praefect_manager.reset_primary_to_original
+        end
+
+        after do
+          # Leave the cluster in a suitable state for subsequent tests
+          praefect_manager.reset_primary_to_original
+        end
+
+        it 'recovers from dataloss' do
+          # Create a new project with a commit and wait for it to replicate
+          praefect_manager.wait_for_replication(project.id)
+
+          # Stop the primary node to trigger failover, and then wait
+          # for Gitaly to be ready for writes again
+          praefect_manager.trigger_failover_by_stopping_primary_node
+          praefect_manager.wait_for_new_primary
+          praefect_manager.wait_for_health_check_current_primary_node
+          praefect_manager.wait_for_gitaly_check
+
+          # Confirm that we have access to the repo after failover
+          Support::Waiter.wait_until(retry_on_exception: true, sleep_interval: 5) do
+            Resource::Repository::Commit.fabricate_via_api! do |commits|
+              commits.project = project
+              commits.sha = 'master'
+            end
+          end
+
+          # Push a commit to the new primary
+          Resource::Repository::ProjectPush.fabricate! do |push|
+            push.project = project
+            push.new_branch = false
+            push.commit_message = 'pushed after failover'
+            push.file_name = 'new_file'
+            push.file_content = 'new file'
+          end
+
+          # Start the old primary node again
+          praefect_manager.start_primary_node
+          praefect_manager.wait_for_health_check_current_primary_node
+
+          # Confirm dataloss (i.e., inconsistent nodes)
+          expect(praefect_manager.dataloss?).to be true
+          expect(praefect_manager.replicated?(project.id)).to be false
+
+          # Reconcile nodes to recover from dataloss
+          praefect_manager.reconcile_nodes
+          praefect_manager.wait_for_replication(project.id)
+
+          # Confirm that both commits are available after reconciliation
+          expect(project.commits.map { |commit| commit[:message].chomp })
+            .to include("Initial commit").and include("pushed after failover")
+        end
+      end
+    end
+  end
+end

qa/qa/specs/features/browser_ui/3_create/gitaly/high_availability_spec.rb

@@ -41,8 +41,6 @@ module QA
             expect(show).to have_file(initial_file)
           end
 
-          praefect_manager.enable_writes
-
           Resource::Repository::Commit.fabricate_via_api! do |commit|
             commit.project = project
             commit.add_files([