Perform a real search and wait longer for success

It can take longer than expected for elasticsearch to be ready to index
new data. This makes sure it actually does return valid results, rather
than only accepting a query.

qa/qa/page/base.rb

@@ -132,20 +132,25 @@ module QA
         element.select value
       end
 
+      def has_active_element?(name, **kwargs)
+        has_element?(name, class: 'active', **kwargs)
+      end
+
       def has_element?(name, **kwargs)
         wait_for_requests
 
-        wait = kwargs[:wait] ? kwargs[:wait] && kwargs.delete(:wait) : Capybara.default_max_wait_time
-        text = kwargs[:text] ? kwargs[:text] && kwargs.delete(:text) : nil
+        wait = kwargs.delete(:wait) || Capybara.default_max_wait_time
+        text = kwargs.delete(:text)
+        klass = kwargs.delete(:class)
 
-        has_css?(element_selector_css(name, kwargs), text: text, wait: wait)
+        has_css?(element_selector_css(name, kwargs), text: text, wait: wait, class: klass)
       end
 
       def has_no_element?(name, **kwargs)
         wait_for_requests
 
-        wait = kwargs[:wait] ? kwargs[:wait] && kwargs.delete(:wait) : Capybara.default_max_wait_time
-        text = kwargs[:text] ? kwargs[:text] && kwargs.delete(:text) : nil
+        wait = kwargs.delete(:wait) || Capybara.default_max_wait_time
+        text = kwargs.delete(:text)
 
         has_no_css?(element_selector_css(name, kwargs), wait: wait, text: text)
       end

qa/qa/page/main/menu.rb

@@ -76,9 +76,15 @@ module QA
         end
 
         def sign_out
+          retry_until do
+            break true unless signed_in?
+
             within_user_menu do
               click_element :sign_out_link
             end
+
+            has_no_element?(:user_avatar)
+          end
         end
 
         def sign_out_if_signed_in

qa/qa/page/search/results.rb

@@ -19,11 +19,11 @@ module QA::Page
       end
 
       def switch_to_code
-        click_element(:code_tab)
+        switch_to_tab(:code_tab)
       end
 
       def switch_to_projects
-        click_element(:projects_tab)
+        switch_to_tab(:projects_tab)
       end
 
       def has_file_in_project?(file_name, project_name)
@@ -32,7 +32,7 @@ module QA::Page
 
       def has_file_with_content?(file_name, file_text)
         within_element_by_index(:result_item_content, 0) do
-          false unless has_element?(:file_title_content, text: file_name)
+          break false unless has_element?(:file_title_content, text: file_name)
 
           has_element?(:file_text_content, text: file_text)
         end
@@ -41,6 +41,15 @@ module QA::Page
       def has_project?(project_name)
         has_element?(:project, project_name: project_name)
       end
+
+      private
+
+      def switch_to_tab(tab)
+        retry_until do
+          click_element(tab)
+          has_active_element?(tab)
+        end
+      end
     end
   end
 end

qa/qa/resource/repository/commit.rb

@@ -11,6 +11,8 @@ module QA
                       :file_path,
                       :sha
 
+        attribute :short_id
+
         attribute :project do
           Project.fabricate! do |resource|
             resource.name = 'project-with-commit'

qa/qa/runtime/search.rb

 # frozen_string_literal: true
 
+require 'securerandom'
+
 module QA
   module Runtime
     module Search
@@ -8,26 +10,83 @@ module QA
 
       ElasticSearchServerError = Class.new(RuntimeError)
 
-      def elasticsearch_responding?
+      def assert_elasticsearch_responding
         QA::Runtime::Logger.debug("Attempting to search via Elasticsearch...")
 
-        QA::Support::Retrier.retry_on_exception do
-          # We don't care about the results of the search, we just need
-          # any search that uses Elasticsearch, not the native search
-          # The Elasticsearch-only scopes are blobs, wiki_blobs, and commits.
-          request = Runtime::API::Request.new(api_client, "/search?scope=blobs&search=foo")
-          response = get(request.url)
+        QA::Support::Retrier.retry_on_exception(max_attempts: 3) do
+          search_term = SecureRandom.hex(8)
 
-          unless response.code == singleton_class::HTTP_STATUS_OK
-            raise ElasticSearchServerError, "Search attempt failed. Request returned (#{response.code}): `#{response}`."
+          QA::Runtime::Logger.debug("Creating commit and project including search term '#{search_term}'...")
+
+          content = "Elasticsearch test commit #{search_term}"
+          project = Resource::Project.fabricate_via_api! do |project|
+            project.name = "project-to-search-#{search_term}"
+          end
+          commit = Resource::Repository::Commit.fabricate_via_api! do |commit|
+            commit.project = project
+            commit.commit_message = content
+            commit.add_files(
+              [
+                {
+                  file_path: 'test.txt',
+                  content: content
+                }
+              ]
+            )
+          end
+
+          find_commit(commit, "commit*#{search_term}")
+          find_project(project, "to-search*#{search_term}")
+        end
+      end
+
+      def find_code(file_name, search_term)
+        find_target_in_scope('blobs', search_term) do |record|
+          record[:filename] == file_name && record[:data].include?(search_term)
         end
 
-          true
+        QA::Runtime::Logger.debug("Found file '#{file_name} containing code '#{search_term}'")
       end
+
+      def find_commit(commit, search_term)
+        find_target_in_scope('commits', search_term) do |record|
+          record[:message] == commit.commit_message
+        end
+
+        QA::Runtime::Logger.debug("Found commit '#{commit.commit_message} (#{commit.short_id})' via '#{search_term}'")
+      end
+
+      def find_project(project, search_term)
+        find_target_in_scope('projects', search_term) do |record|
+          record[:name] == project.name
+        end
+
+        QA::Runtime::Logger.debug("Found project '#{project.name}' via '#{search_term}'")
       end
 
       private
 
+      def find_target_in_scope(scope, search_term)
+        QA::Support::Retrier.retry_until(max_attempts: 10, sleep_interval: 10, raise_on_failure: true, retry_on_exception: true) do
+          result = search(scope, search_term)
+          result && result.any? { |record| yield record }
+        end
+      end
+
+      def search(scope, term)
+        QA::Runtime::Logger.debug("Search scope '#{scope}' for '#{term}'...")
+        request = Runtime::API::Request.new(api_client, "/search?scope=#{scope}&search=#{term}")
+        response = get(request.url)
+
+        unless response.code == singleton_class::HTTP_STATUS_OK
+          msg = "Search attempt failed. Request returned (#{response.code}): `#{response}`."
+          QA::Runtime::Logger.debug(msg)
+          raise ElasticSearchServerError, msg
+        end
+
+        parse_body(response)
+      end
+
       def api_client
         @api_client ||= Runtime::API::Client.new(:gitlab)
       end

qa/qa/specs/features/ee/browser_ui/enablement/elasticsearch/advanced_global_advanced_syntax_search_spec.rb

@@ -5,11 +5,17 @@ module QA
     # Failure issue: https://gitlab.com/gitlab-org/gitlab/issues/43732
     describe 'Elasticsearch advanced global search with advanced syntax', :orchestrated, :elasticsearch, :requires_admin, :quarantine do
       let(:project_name_suffix) { SecureRandom.hex(8) }
+      let(:project_file_name) { 'elasticsearch.rb' }
+      let(:project_file_content) { "elasticsearch: #{SecureRandom.hex(8)}" }
+      let(:project) do
+        Resource::Project.fabricate_via_api! do |project|
+          project.add_name_uuid = false
+          project.name = "es-adv-global-search-#{project_name_suffix}"
+          project.description = "This is a unique project description #{project_name_suffix}"
+        end
+      end
 
-      before do
-        @project_file_name = 'elasticsearch.rb'
-        @project_file_content = "elasticsearch: #{SecureRandom.hex(8)}"
-
+      before(:context) do
         QA::EE::Resource::Settings::Elasticsearch.fabricate_via_api! do |es|
           es.user = QA::Resource::User.new.tap do |user|
             user.username = QA::Runtime::User.admin_username
@@ -18,34 +24,32 @@ module QA
           es.api_client = Runtime::API::Client.as_admin
         end
 
-        Runtime::Search.elasticsearch_responding?
-
-        @project = Resource::Project.fabricate_via_api! do |project|
-          project.add_name_uuid = false
-          project.name = "es-adv-global-search-#{project_name_suffix}1"
-          project.description = "This is a unique project description #{project_name_suffix}2"
+        Runtime::Search.assert_elasticsearch_responding
       end
 
+      before do
         Resource::Repository::ProjectPush.fabricate! do |push|
-          push.project = @project
-          push.file_name = @project_file_name
-          push.file_content = @project_file_content
+          push.project = project
+          push.file_name = project_file_name
+          push.file_content = project_file_content
         end
 
         Flow::Login.sign_in
       end
 
       context 'when searching for projects using advanced syntax' do
-        it 'searches in the project name' do
-          expect_search_to_find_project("es-adv-*#{project_name_suffix}1")
+        it 'searches in the project name', retry: 3 do
+          expect_search_to_find_project("es-adv-*#{project_name_suffix}")
         end
 
-        it 'searches in the project description' do
-          expect_search_to_find_project("unique +#{project_name_suffix}2")
+        it 'searches in the project description', retry: 3 do
+          expect_search_to_find_project("unique +#{project_name_suffix}")
         end
       end
 
       def expect_search_to_find_project(search_term)
+        expect { Runtime::Search.find_project(project, search_term) }.not_to raise_error
+
         Page::Main::Menu.perform do |menu|
           menu.search_for(search_term)
         end
@@ -53,8 +57,10 @@ module QA
         Page::Search::Results.perform do |results|
           results.switch_to_projects
 
+          results.retry_on_exception(reload: true, sleep_interval: 10) do
             expect(results).to have_content("Advanced search functionality is enabled")
-          expect(results).to have_project(@project.name)
+            expect(results).to have_project(project.name)
+          end
         end
       end
     end

qa/qa/specs/features/ee/browser_ui/enablement/elasticsearch/elasticsearch_reindexing_spec.rb

@@ -15,7 +15,7 @@ module QA
           QA::EE::Resource::Settings::Elasticsearch.fabricate_via_browser_ui!
         end
 
-        Runtime::Search.elasticsearch_responding?
+        Runtime::Search.assert_elasticsearch_responding
 
         @project = Resource::Project.fabricate_via_api! do |project|
           project.name = project_name
@@ -30,7 +30,9 @@ module QA
         end.project.visit!
       end
 
-      it 'tests reindexing after push' do
+      it 'tests reindexing after push', retry: 3 do
+        expect { Runtime::Search.find_code(@project_file_name, @project_file_content) }.not_to raise_error
+
         QA::Page::Main::Menu.perform do |menu|
           menu.search_for(@project_file_content)
         end
@@ -42,7 +44,7 @@ module QA
         end
       end
 
-      it 'tests reindexing after webIDE' do
+      it 'tests reindexing after webIDE', retry: 3 do
         template = {
             file_name: 'LICENSE',
             name: 'Mozilla Public License 2.0',
@@ -57,6 +59,8 @@ module QA
           ide.commit_changes
         end
 
+        expect { Runtime::Search.find_code(template[:file_name], content[0..33]) }.not_to raise_error
+
         Page::Main::Menu.perform(&:go_to_groups)
 
         QA::Page::Main::Menu.perform do |menu|

qa/qa/support/page/logging.rb

@@ -173,6 +173,7 @@ module QA
         def log_has_element_or_not(method, name, found, **kwargs)
           msg = ["#{method} :#{name}"]
           msg << %Q(with text "#{kwargs[:text]}") if kwargs[:text]
+          msg << "class: #{kwargs[:class]}" if kwargs[:class]
           msg << "(wait: #{kwargs[:wait] || Capybara.default_max_wait_time})"
           msg << "returned: #{found}"