Add health check for postgres in gitaly E2E

qa/qa/service/praefect_manager.rb

@@ -174,7 +174,7 @@ module QA
       end
 
       def start_all_nodes
-        start_node(@postgres)
+        start_postgres
         start_node(@primary_node)
         start_node(@secondary_node)
         start_node(@tertiary_node)
@@ -183,6 +183,16 @@ module QA
         wait_for_health_check_all_nodes
       end
 
+      def start_postgres
+        start_node(@postgres)
+
+        Support::Waiter.repeat_until(max_attempts: 60, sleep_interval: 1) do
+          shell(sql_to_docker_exec_cmd("SELECT 1 as healthy_database"), fail_on_exception: false) do |line|
+            break true if line.include?("healthy_database")
+          end
+        end
+      end
+
       def verify_storage_move(source_storage, destination_storage, repo_type: :project)
         return if Specs::Helpers::ContextSelector.dot_com?
 

qa/qa/service/shellout.rb

@@ -13,23 +13,25 @@ module QA
       # TODO, make it possible to use generic QA framework classes
       # as a library - gitlab-org/gitlab-qa#94
       #
-      def shell(command, stdin_data: nil)
+      def shell(command, stdin_data: nil, fail_on_exception: true)
         puts "Executing `#{command}`"
 
         Open3.popen2e(*command) do |stdin, out, wait|
           stdin.puts(stdin_data) if stdin_data
           stdin.close if stdin_data
+          cmd_output = ''
 
           if block_given?
             out.each do |line|
+              cmd_output += line
               yield line
             end
           end
 
           out.each_char { |char| print char }
 
-          if wait.value.exited? && wait.value.exitstatus.nonzero?
-            raise CommandError, "Command `#{command}` failed!"
+          if wait.value.exited? && wait.value.exitstatus.nonzero? && fail_on_exception
+            raise CommandError, "Command failed: #{command} \nCommand Output: #{cmd_output}"
           end
         end
       end