Merge branch 'qa-shl-retry-feature-flags' into 'master'

Fix flakiness in Enforce SAML SSO spec

Closes #202579 and #204734

See merge request gitlab-org/gitlab!24668

qa/qa/ee/page/group/settings/saml_sso.rb

@@ -31,11 +31,15 @@ module QA
             end
 
             def has_enforced_sso_button?
-              has_element?(:enforced_sso_toggle_button, wait: 1.0)
+              has_button = has_element?(:enforced_sso_toggle_button, wait: 5)
+              QA::Runtime::Logger.debug "has_enforced_sso_button?: #{has_button}"
+              has_button
             end
 
             def enforce_sso_enabled?
-              has_enforced_sso_button? && find_element(:enforced_sso_toggle_button)[:class].include?('is-checked')
+              enabled = has_enforced_sso_button? && find_element(:enforced_sso_toggle_button).find('input', visible: :all)[:value] == 'true'
+              QA::Runtime::Logger.debug "enforce_sso_enabled?: #{enabled}"
+              enabled
             end
 
             def enforce_sso
@@ -49,11 +53,11 @@ module QA
             end
 
             def has_group_managed_accounts_button?
-              has_element?(:group_managed_accounts_toggle_button, wait: 1.0)
+              has_element?(:group_managed_accounts_toggle_button, wait: 5)
             end
 
             def group_managed_accounts_enabled?
-              enforce_sso_enabled? && has_group_managed_accounts_button? && find_element(:group_managed_accounts_toggle_button)[:class].include?('is-checked')
+              enforce_sso_enabled? && has_group_managed_accounts_button? && find_element(:group_managed_accounts_toggle_button).find('input', visible: :all)[:value] == 'true'
             end
 
             def enable_group_managed_accounts

qa/qa/flow/saml.rb

@@ -38,7 +38,9 @@ module QA
 
       def visit_saml_sso_settings(group, direct: false)
         if direct
-          page.visit "#{group.web_url}/-/saml"
+          url = "#{group.web_url}/-/saml"
+          Runtime::Logger.debug("Visiting url \"#{url}\" directly")
+          page.visit url
         else
           group.visit!
 

qa/qa/runtime/feature.rb

@@ -48,6 +48,12 @@ module QA
         feature && feature["state"] == "on"
       end
 
+      def get_features
+        request = Runtime::API::Request.new(api_client, "/features")
+        response = get(request.url)
+        response.body
+      end
+
       private
 
       def api_client
@@ -76,12 +82,6 @@ module QA
           raise SetFeatureError, "Setting feature flag #{key} to #{value} failed with `#{response}`."
         end
       end
-
-      def get_features
-        request = Runtime::API::Request.new(api_client, "/features")
-        response = get(request.url)
-        response.body
-      end
     end
   end
 end

qa/qa/specs/features/ee/browser_ui/1_manage/group/group_saml_enforced_sso_spec.rb

 # frozen_string_literal: true
 
 module QA
-  context 'Manage', :group_saml, :orchestrated, :requires_admin do
+  context 'Manage', :group_saml, :orchestrated do
     describe 'Group SAML SSO - Enforced SSO' do
       include Support::Api
 
       before(:all) do
+        Support::Retrier.retry_on_exception do
+          Flow::Saml.remove_saml_idp_service(@saml_idp_service) if @saml_idp_service
+
           @group = Resource::Sandbox.fabricate_via_api! do |sandbox_group|
             sandbox_group.path = "saml_sso_group_#{SecureRandom.hex(8)}"
           end
@@ -18,6 +21,7 @@ module QA
 
           @managed_group_url = setup_and_enable_enforce_sso
         end
+      end
 
       before do
         Flow::Saml.logout_from_idp(@saml_idp_service)
@@ -143,18 +147,11 @@ module QA
 
       Support::Retrier.retry_on_exception do
         Flow::Saml.visit_saml_sso_settings(@group)
+        ensure_enforced_sso_button_shown
 
         managed_group_url = EE::Page::Group::Settings::SamlSSO.perform do |saml_sso|
-          # Once the feature flags are enabled, it takes some time for the toggle buttons to show on the UI.
-          # This issue does not happen manually. Only happens with the test as they are too fast.
-          Support::Retrier.retry_until(sleep_interval: 1, raise_on_failure: true) do
-            condition_met = saml_sso.has_enforced_sso_button?
-            page.refresh unless condition_met
-
-            condition_met
-          end
-
           saml_sso.enforce_sso
+
           saml_sso.set_id_provider_sso_url(@saml_idp_service.idp_sso_url)
           saml_sso.set_cert_fingerprint(@saml_idp_service.idp_certificate_fingerprint)
 
@@ -164,10 +161,29 @@ module QA
         end
 
         Flow::Saml.visit_saml_sso_settings(@group, direct: true)
-        raise "Enforced SSO not setup correctly" unless EE::Page::Group::Settings::SamlSSO.perform(&:enforce_sso_enabled?)
+        ensure_enforced_sso_button_shown
+
+        unless EE::Page::Group::Settings::SamlSSO.perform(&:enforce_sso_enabled?)
+          QA::Runtime::Logger.debug "Enforced SSO not setup correctly. About to raise failure."
+          QA::Runtime::Logger.debug Capybara::Screenshot.screenshot_and_save_page
+          QA::Runtime::Logger.debug Runtime::Feature.get_features
+
+          raise "Enforced SSO not setup correctly"
+        end
 
         managed_group_url
       end
     end
+
+    def ensure_enforced_sso_button_shown
+      # Sometimes, the toggle button for SAML SSO does not appear and only appears after a refresh
+      # This issue can only be reproduced manually if you are too quick to go to the group setting page
+      # after enabling the feature flags.
+      Support::Retrier.retry_until(sleep_interval: 1, raise_on_failure: true) do
+        condition_met = EE::Page::Group::Settings::SamlSSO.perform(&:has_enforced_sso_button?)
+        page.refresh unless condition_met
+        condition_met
+      end
+    end
   end
 end