Merge branch 'ml-wait-and-retry-if-retry-later-error-occurs' into 'master'

Workaround for `Retry later` errors during E2E tests

See merge request gitlab-org/gitlab!28625

qa/qa/page/base.rb

@@ -16,6 +16,10 @@ module QA
 
       def_delegators :evaluator, :view, :views
 
+      def initialize
+        @retry_later_backoff = QA::Support::Repeater::DEFAULT_MAX_WAIT_TIME
+      end
+
       def assert_no_element(name)
         assert_no_selector(element_selector_css(name))
       end
@@ -259,6 +263,19 @@ module QA
         visit find_element(name)['href']
       end
 
+      def wait_if_retry_later
+        return if @retry_later_backoff > QA::Support::Repeater::DEFAULT_MAX_WAIT_TIME * 5
+
+        if has_css?('body', text: 'Retry later', wait: 0)
+          QA::Runtime::Logger.warn("`Retry later` error occurred. Sleeping for #{@retry_later_backoff} seconds...")
+          sleep @retry_later_backoff
+          refresh
+          @retry_later_backoff += QA::Support::Repeater::DEFAULT_MAX_WAIT_TIME
+
+          wait_if_retry_later
+        end
+      end
+
       def self.path
         raise NotImplementedError
       end

qa/qa/page/main/login.rb

@@ -152,6 +152,8 @@ module QA
         private
 
         def sign_in_using_gitlab_credentials(user:, skip_page_validation: false)
+          wait_if_retry_later
+
           switch_to_sign_in_tab if has_sign_in_tab?
           switch_to_standard_tab if has_standard_tab?
 

qa/qa/page/main/menu.rb

@@ -77,6 +77,8 @@ module QA
 
         def sign_out
           retry_until do
+            wait_if_retry_later
+
             break true unless signed_in?
 
             within_user_menu do

qa/qa/page/validatable.rb

@@ -8,6 +8,8 @@ module QA
       def validate_elements_present!
         base_page = self.new
 
+        base_page.wait_if_retry_later
+
         elements.each do |element|
           next unless element.required?
 

qa/qa/runtime/env.rb

@@ -43,6 +43,10 @@ module QA
         ENV['GITLAB_QA_ADMIN_ACCESS_TOKEN']
       end
 
+      def ci_project_name
+        ENV['CI_PROJECT_NAME']
+      end
+
       def debug?
         enabled?(ENV['QA_DEBUG'], default: false)
       end

qa/spec/spec_helper.rb

@@ -63,8 +63,9 @@ RSpec.configure do |config|
   config.display_try_failure_messages = true
 
   if ENV['CI'] && !QA::Runtime::Env.disable_rspec_retry?
+    non_quarantine_retries = QA::Runtime::Env.ci_project_name =~ /staging|canary|production/ ? 3 : 2
     config.around do |example|
-      retry_times = example.metadata.key?(:quarantine) ? 1 : 2
+      retry_times = example.metadata.key?(:quarantine) ? 1 : non_quarantine_retries
       example.run_with_retry retry: retry_times
     end
   end