Merge branch 'acunskis-automatically-attach-links-allure' into 'master'

E2E: Automatically attach testcases and issues to allure report

See merge request gitlab-org/gitlab!68351

doc/development/testing_guide/end_to_end/rspec_metadata_tests.md

@@ -44,3 +44,4 @@ This is a partial list of the [RSpec metadata](https://relishapp.com/rspec/rspec
 | `:smtp`           | The test requires a GitLab instance to be configured to use an SMTP server. Tests SMTP notification email delivery from GitLab by using MailHog. |
 | `:testcase`       | The link to the test case issue in the [Quality Test Cases project](https://gitlab.com/gitlab-org/quality/testcases/). |
 | `:transient`      | The test tests transient bugs. It is excluded by default. |
+| `:issue`, `:issue_${num}` | Optional links to issues which might be related to the spec. Helps keeping track of related issues and can also be used by tools that create test reports. Currently added automatically to `Allure` test report. Multiple tags can be used by adding optional number postfix like `issue_1`, `issue_2` etc. |

qa/Gemfile

@@ -4,7 +4,7 @@ source 'https://rubygems.org'
 
 gem 'gitlab-qa', require: 'gitlab/qa'
 gem 'activesupport', '~> 6.1.3.2' # This should stay in sync with the root's Gemfile
-gem 'allure-rspec', '~> 2.14.1'
+gem 'allure-rspec', '~> 2.14.5'
 gem 'capybara', '~> 3.35.0'
 gem 'capybara-screenshot', '~> 1.0.23'
 gem 'rake', '~> 12.3.3'

qa/Gemfile.lock

@@ -19,10 +19,10 @@ GEM
       rack-test (>= 1.1.0, < 2.0)
       rest-client (>= 2.0.2, < 3.0)
       rspec (~> 3.8)
-    allure-rspec (2.14.2)
-      allure-ruby-commons (= 2.14.2)
+    allure-rspec (2.14.5)
+      allure-ruby-commons (= 2.14.5)
       rspec-core (>= 3.8, < 4)
-    allure-ruby-commons (2.14.2)
+    allure-ruby-commons (2.14.5)
       mime-types (>= 3.3, < 4)
       oj (>= 3.10, < 4)
       require_all (>= 2, < 4)
@@ -111,7 +111,7 @@ GEM
     octokit (4.21.0)
       faraday (>= 0.9)
       sawyer (~> 0.8.0, >= 0.5.3)
-    oj (3.12.1)
+    oj (3.13.2)
     parallel (1.19.2)
     parallel_tests (2.29.0)
       parallel
@@ -212,7 +212,7 @@ PLATFORMS
 DEPENDENCIES
   activesupport (~> 6.1.3.2)
   airborne (~> 0.3.4)
-  allure-rspec (~> 2.14.1)
+  allure-rspec (~> 2.14.5)
   capybara (~> 3.35.0)
   capybara-screenshot (~> 1.0.23)
   chemlab (~> 0.7)

qa/qa/runtime/allure_report.rb

@@ -29,6 +29,13 @@ module QA
           AllureRspec.configure do |config|
             config.results_directory = 'tmp/allure-results'
             config.clean_results_directory = true
+
+            # automatically attach links to testcases and issues
+            config.tms_tag = :testcase
+            config.link_tms_pattern = '{}'
+            config.issue_tag = :issue
+            config.link_issue_pattern = '{}'
+
             config.environment_properties = environment_info if Env.running_in_ci?
 
             # Set custom environment name to separate same specs executed on different environments

qa/qa/specs/features/api/1_manage/bulk_import_group_spec.rb

@@ -71,9 +71,6 @@ module QA
         end
       end
 
-      # Non blocking issues:
-      # https://gitlab.com/gitlab-org/gitlab/-/issues/331252
-      # https://gitlab.com/gitlab-org/gitlab/-/issues/333678 <- can cause 500 when creating user and group back to back
       it(
         'imports group with subgroups and labels',
         testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/issues/1871'

qa/qa/specs/features/browser_ui/1_manage/group/bulk_import_group_spec.rb

@@ -54,10 +54,14 @@ module QA
         end
       end
 
-      # Non blocking issues:
-      # https://gitlab.com/gitlab-org/gitlab/-/issues/331252
-      # https://gitlab.com/gitlab-org/gitlab/-/issues/333678 <- can cause 500 when creating user and group back to back
-      it 'imports group from UI', testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/issues/1785' do
+      it(
+        'imports group from UI',
+        testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/issues/1785',
+        issue_1: 'https://gitlab.com/gitlab-org/gitlab/-/issues/331252',
+        issue_2: 'https://gitlab.com/gitlab-org/gitlab/-/issues/333678',
+        # mostly impacts testing as it makes small groups import slower
+        issue_3: 'https://gitlab.com/gitlab-org/gitlab/-/issues/332351'
+      ) do
         Page::Group::BulkImport.perform do |import_page|
           import_page.import_group(imported_group.path, imported_group.sandbox.path)
 

qa/qa/support/allure_metadata_formatter.rb

@@ -17,9 +17,6 @@ module QA
       def example_started(example_notification)
         example = example_notification.example
 
-        testcase = example.metadata[:testcase]
-        example.tms('Testcase', testcase) if testcase
-
         quarantine_issue = example.metadata.dig(:quarantine, :issue)
         example.issue('Quarantine issue', quarantine_issue) if quarantine_issue
 

qa/spec/support/allure_metadata_formatter_spec.rb

@@ -34,7 +34,6 @@ describe QA::Support::AllureMetadataFormatter do
     formatter.example_started(rspec_example_notification)
 
     aggregate_failures do
-      expect(rspec_example).to have_received(:tms).with('Testcase', 'testcase')
       expect(rspec_example).to have_received(:issue).with('Quarantine issue', 'issue')
       expect(rspec_example).to have_received(:add_link).with(name: "Job(#{ci_job})", url: ci_job_url)
       expect(rspec_example).to have_received(:issue).with(