Make test reports frontend handle separated errors

Display issues list when there are errors
Join errors and failures arrays for display
Add test report error result cases to specs

app/assets/javascripts/reports/components/grouped_test_reports_app.vue

@@ -62,9 +62,21 @@ export default {
       return (
         report.existing_failures.length > 0 ||
         report.new_failures.length > 0 ||
-        report.resolved_failures.length > 0
+        report.resolved_failures.length > 0 ||
+        report.existing_errors.length > 0 ||
+        report.new_errors.length > 0 ||
+        report.resolved_errors.length > 0
       );
     },
+    unresolvedIssues(report) {
+      return report.existing_failures.concat(report.existing_errors);
+    },
+    newIssues(report) {
+      return report.new_failures.concat(report.new_errors);
+    },
+    resolvedIssues(report) {
+      return report.resolved_failures.concat(report.resolved_errors);
+    },
   },
 };
 </script>
@@ -87,9 +99,9 @@ export default {
         <issues-list
           v-if="shouldRenderIssuesList(report)"
           :key="`issues-list-${i}`"
-          :unresolved-issues="report.existing_failures"
-          :new-issues="report.new_failures"
-          :resolved-issues="report.resolved_failures"
+          :unresolved-issues="unresolvedIssues(report)"
+          :new-issues="newIssues(report)"
+          :resolved-issues="resolvedIssues(report)"
           :component="$options.componentNames.TestIssueBody"
           class="report-block-group-list"
         />

app/assets/javascripts/reports/store/mutations.js

@@ -16,6 +16,7 @@ export default {
     state.summary.total = response.summary.total;
     state.summary.resolved = response.summary.resolved;
     state.summary.failed = response.summary.failed;
+    state.summary.errored = response.summary.errored;
 
     state.status = response.status;
     state.reports = response.suites;
@@ -29,6 +30,7 @@ export default {
       total: 0,
       resolved: 0,
       failed: 0,
+      errored: 0,
     };
     state.status = null;
   },

app/assets/javascripts/reports/store/state.js

@@ -13,6 +13,7 @@ export default () => ({
     total: 0,
     resolved: 0,
     failed: 0,
+    errored: 0,
   },
 
   /**
@@ -23,10 +24,14 @@ export default () => ({
    *     total: {Number},
    *     resolved: {Number},
    *     failed: {Number},
+   *     errored: {Number},
    *   },
    *   new_failures: {Array.<Object>},
    *   resolved_failures: {Array.<Object>},
    *   existing_failures: {Array.<Object>},
+   *   new_errors: {Array.<Object>},
+   *   resolved_errors: {Array.<Object>},
+   *   existing_errors: {Array.<Object>},
    * }
    */
   reports: [],

app/assets/javascripts/reports/store/utils.js

@@ -8,10 +8,11 @@ import {
 } from '../constants';
 
 const textBuilder = results => {
-  const { failed, resolved, total } = results;
+  const { failed, errored, resolved, total } = results;
 
-  const failedString = failed
-    ? n__('%d failed/error test result', '%d failed/error test results', failed)
+  const failedOrErrored = (failed || 0) + (errored || 0);
+  const failedString = failedOrErrored
+    ? n__('%d failed/error test result', '%d failed/error test results', failedOrErrored)
     : null;
   const resolvedString = resolved
     ? n__('%d fixed test result', '%d fixed test results', resolved)
@@ -20,7 +21,7 @@ const textBuilder = results => {
 
   let resultsString = s__('Reports|no changed test results');
 
-  if (failed) {
+  if (failedOrErrored) {
     if (resolved) {
       resultsString = sprintf(s__('Reports|%{failedString} and %{resolvedString}'), {
         failedString,

spec/features/merge_request/user_sees_merge_widget_spec.rb

@@ -699,6 +699,137 @@ describe 'Merge request > User sees merge widget', :js do
         end
       end
 
+      context 'when a new error exists' do
+        let(:base_reports) do
+          Gitlab::Ci::Reports::TestReports.new.tap do |reports|
+            reports.get_suite('rspec').add_test_case(create_test_case_rspec_success)
+            reports.get_suite('junit').add_test_case(create_test_case_java_success)
+          end
+        end
+
+        let(:head_reports) do
+          Gitlab::Ci::Reports::TestReports.new.tap do |reports|
+            reports.get_suite('rspec').add_test_case(create_test_case_rspec_success)
+            reports.get_suite('junit').add_test_case(create_test_case_java_error)
+          end
+        end
+
+        it 'shows test reports summary which includes the new error' do
+          within(".js-reports-container") do
+            click_button 'Expand'
+
+            expect(page).to have_content('Test summary contained 1 failed/error test result out of 2 total tests')
+            within(".js-report-section-container") do
+              expect(page).to have_content('rspec found no changed test results out of 1 total test')
+              expect(page).to have_content('junit found 1 failed/error test result out of 1 total test')
+              expect(page).to have_content('New')
+              expect(page).to have_content('addTest')
+            end
+          end
+        end
+
+        context 'when user clicks the new error' do
+          it 'shows the test report detail' do
+            within(".js-reports-container") do
+              click_button 'Expand'
+
+              within(".js-report-section-container") do
+                click_button 'addTest'
+
+                expect(page).to have_content('8.88')
+              end
+            end
+          end
+        end
+      end
+
+      context 'when an existing error exists' do
+        let(:base_reports) do
+          Gitlab::Ci::Reports::TestReports.new.tap do |reports|
+            reports.get_suite('rspec').add_test_case(create_test_case_rspec_error)
+            reports.get_suite('junit').add_test_case(create_test_case_java_success)
+          end
+        end
+
+        let(:head_reports) do
+          Gitlab::Ci::Reports::TestReports.new.tap do |reports|
+            reports.get_suite('rspec').add_test_case(create_test_case_rspec_error)
+            reports.get_suite('junit').add_test_case(create_test_case_java_success)
+          end
+        end
+
+        it 'shows test reports summary which includes the existing error' do
+          within(".js-reports-container") do
+            click_button 'Expand'
+
+            expect(page).to have_content('Test summary contained 1 failed/error test result out of 2 total tests')
+            within(".js-report-section-container") do
+              expect(page).to have_content('rspec found 1 failed/error test result out of 1 total test')
+              expect(page).to have_content('junit found no changed test results out of 1 total test')
+              expect(page).not_to have_content('New')
+              expect(page).to have_content('Test#sum when a is 4 and b is 4 returns summary')
+            end
+          end
+        end
+
+        context 'when user clicks the existing error' do
+          it 'shows test report detail of it' do
+            within(".js-reports-container") do
+              click_button 'Expand'
+
+              within(".js-report-section-container") do
+                click_button 'Test#sum when a is 4 and b is 4 returns summary'
+
+                expect(page).to have_content('4.44')
+              end
+            end
+          end
+        end
+      end
+
+      context 'when a resolved error exists' do
+        let(:base_reports) do
+          Gitlab::Ci::Reports::TestReports.new.tap do |reports|
+            reports.get_suite('rspec').add_test_case(create_test_case_rspec_success)
+            reports.get_suite('junit').add_test_case(create_test_case_java_error)
+          end
+        end
+
+        let(:head_reports) do
+          Gitlab::Ci::Reports::TestReports.new.tap do |reports|
+            reports.get_suite('rspec').add_test_case(create_test_case_rspec_success)
+            reports.get_suite('junit').add_test_case(create_test_case_java_success)
+          end
+        end
+
+        it 'shows test reports summary which includes the resolved error' do
+          within(".js-reports-container") do
+            click_button 'Expand'
+
+            expect(page).to have_content('Test summary contained 1 fixed test result out of 2 total tests')
+            within(".js-report-section-container") do
+              expect(page).to have_content('rspec found no changed test results out of 1 total test')
+              expect(page).to have_content('junit found 1 fixed test result out of 1 total test')
+              expect(page).to have_content('addTest')
+            end
+          end
+        end
+
+        context 'when user clicks the resolved error' do
+          it 'shows test report detail of it' do
+            within(".js-reports-container") do
+              click_button 'Expand'
+
+              within(".js-report-section-container") do
+                click_button 'addTest'
+
+                expect(page).to have_content('5.55')
+              end
+            end
+          end
+        end
+      end
+
       context 'properly truncates the report' do
         let(:base_reports) do
           Gitlab::Ci::Reports::TestReports.new.tap do |reports|

spec/frontend/reports/store/utils_spec.js

@@ -35,6 +35,16 @@ describe('Reports store utils', () => {
       );
     });
 
+    it('should render text for multiple errored results', () => {
+      const name = 'Test summary';
+      const data = { errored: 7, total: 10 };
+      const result = utils.summaryTextBuilder(name, data);
+
+      expect(result).toBe(
+        'Test summary contained 7 failed/error test results out of 10 total tests',
+      );
+    });
+
     it('should render text for multiple fixed results', () => {
       const name = 'Test summary';
       const data = { resolved: 4, total: 10 };
@@ -62,6 +72,27 @@ describe('Reports store utils', () => {
         'Test summary contained 1 failed/error test result and 1 fixed test result out of 10 total tests',
       );
     });
+
+    it('should render text for singular failed, errored, and fixed results', () => {
+      // these will be singular when the copy is updated
+      const name = 'Test summary';
+      const data = { failed: 1, errored: 1, resolved: 1, total: 10 };
+      const result = utils.summaryTextBuilder(name, data);
+
+      expect(result).toBe(
+        'Test summary contained 2 failed/error test results and 1 fixed test result out of 10 total tests',
+      );
+    });
+
+    it('should render text for multiple failed, errored, and fixed results', () => {
+      const name = 'Test summary';
+      const data = { failed: 2, errored: 3, resolved: 4, total: 10 };
+      const result = utils.summaryTextBuilder(name, data);
+
+      expect(result).toBe(
+        'Test summary contained 5 failed/error test results and 4 fixed test results out of 10 total tests',
+      );
+    });
   });
 
   describe('reportTextBuilder', () => {
@@ -89,6 +120,14 @@ describe('Reports store utils', () => {
       expect(result).toBe('Rspec found 3 failed/error test results out of 10 total tests');
     });
 
+    it('should render text for multiple errored results', () => {
+      const name = 'Rspec';
+      const data = { errored: 7, total: 10 };
+      const result = utils.reportTextBuilder(name, data);
+
+      expect(result).toBe('Rspec found 7 failed/error test results out of 10 total tests');
+    });
+
     it('should render text for multiple fixed results', () => {
       const name = 'Rspec';
       const data = { resolved: 4, total: 10 };
@@ -116,6 +155,27 @@ describe('Reports store utils', () => {
         'Rspec found 1 failed/error test result and 1 fixed test result out of 10 total tests',
       );
     });
+
+    it('should render text for singular failed, errored, and fixed results', () => {
+      // these will be singular when the copy is updated
+      const name = 'Rspec';
+      const data = { failed: 1, errored: 1, resolved: 1, total: 10 };
+      const result = utils.reportTextBuilder(name, data);
+
+      expect(result).toBe(
+        'Rspec found 2 failed/error test results and 1 fixed test result out of 10 total tests',
+      );
+    });
+
+    it('should render text for multiple failed, errored, and fixed results', () => {
+      const name = 'Rspec';
+      const data = { failed: 2, errored: 3, resolved: 4, total: 10 };
+      const result = utils.reportTextBuilder(name, data);
+
+      expect(result).toBe(
+        'Rspec found 5 failed/error test results and 4 fixed test results out of 10 total tests',
+      );
+    });
   });
 
   describe('statusIcon', () => {

spec/javascripts/reports/components/grouped_test_reports_app_spec.js

@@ -5,6 +5,7 @@ import state from '~/reports/store/state';
 import component from '~/reports/components/grouped_test_reports_app.vue';
 import mountComponent from '../../helpers/vue_mount_component_helper';
 import newFailedTestReports from '../mock_data/new_failures_report.json';
+import newErrorsTestReports from '../mock_data/new_errors_report.json';
 import successTestReports from '../mock_data/no_failures_report.json';
 import mixedResultsTestReports from '../mock_data/new_and_fixed_failures_report.json';
 import resolvedFailures from '../mock_data/resolved_failures.json';
@@ -99,6 +100,34 @@ describe('Grouped Test Reports App', () => {
     });
   });
 
+  describe('with new error result', () => {
+    beforeEach(() => {
+      mock.onGet('test_results.json').reply(200, newErrorsTestReports, {});
+      vm = mountComponent(Component, {
+        endpoint: 'test_results.json',
+      });
+    });
+
+    it('renders error summary text + new badge', done => {
+      setTimeout(() => {
+        expect(vm.$el.querySelector('.gl-spinner')).toBeNull();
+        expect(vm.$el.querySelector('.js-code-text').textContent.trim()).toEqual(
+          'Test summary contained 2 failed/error test results out of 11 total tests',
+        );
+
+        expect(vm.$el.textContent).toContain(
+          'karma found 2 failed/error test results out of 3 total tests',
+        );
+
+        expect(vm.$el.textContent).toContain('New');
+        expect(vm.$el.textContent).toContain(
+          'rspec:pg found no changed test results out of 8 total tests',
+        );
+        done();
+      }, 0);
+    });
+  });
+
   describe('with mixed results', () => {
     beforeEach(() => {
       mock.onGet('test_results.json').reply(200, mixedResultsTestReports, {});
@@ -127,7 +156,7 @@ describe('Grouped Test Reports App', () => {
     });
   });
 
-  describe('with resolved failures', () => {
+  describe('with resolved failures and resolved errors', () => {
     beforeEach(() => {
       mock.onGet('test_results.json').reply(200, resolvedFailures, {});
       vm = mountComponent(Component, {
@@ -139,11 +168,11 @@ describe('Grouped Test Reports App', () => {
       setTimeout(() => {
         expect(vm.$el.querySelector('.gl-spinner')).toBeNull();
         expect(vm.$el.querySelector('.js-code-text').textContent.trim()).toEqual(
-          'Test summary contained 2 fixed test results out of 11 total tests',
+          'Test summary contained 4 fixed test results out of 11 total tests',
         );
 
         expect(vm.$el.textContent).toContain(
-          'rspec:pg found 2 fixed test results out of 8 total tests',
+          'rspec:pg found 4 fixed test results out of 8 total tests',
         );
         done();
       }, 0);
@@ -161,6 +190,19 @@ describe('Grouped Test Reports App', () => {
         done();
       }, 0);
     });
+
+    it('renders resolved errors', done => {
+      setTimeout(() => {
+        expect(vm.$el.querySelector('.report-block-container').textContent).toContain(
+          resolvedFailures.suites[0].resolved_errors[0].name,
+        );
+
+        expect(vm.$el.querySelector('.report-block-container').textContent).toContain(
+          resolvedFailures.suites[0].resolved_errors[1].name,
+        );
+        done();
+      }, 0);
+    });
   });
 
   describe('with error', () => {

spec/javascripts/reports/mock_data/new_and_fixed_failures_report.json

 {
   "status": "failed",
-  "summary": { "total": 11, "resolved": 2, "failed": 2 },
+  "summary": { "total": 11, "resolved": 2, "errored": 0, "failed": 2 },
   "suites": [
     {
       "name": "rspec:pg",
       "status": "failed",
-      "summary": { "total": 8, "resolved": 2, "failed": 1 },
+      "summary": { "total": 8, "resolved": 2, "errored": 0, "failed": 1 },
       "new_failures": [
         {
           "status": "failed",
@@ -28,12 +28,15 @@
           "system_output": null
         }
       ],
-      "existing_failures": []
+      "existing_failures": [],
+      "new_errors": [],
+      "resolved_errors": [],
+      "existing_errors": []
     },
     {
       "name": "java ant",
       "status": "failed",
-      "summary": { "total": 3, "resolved": 0, "failed": 1 },
+      "summary": { "total": 3, "resolved": 0, "errored": 0, "failed": 1 },
       "new_failures": [],
       "resolved_failures": [],
       "existing_failures": [
@@ -43,7 +46,10 @@
           "execution_time": 0.004,
           "system_output": "junit.framework.AssertionFailedError: expected:<3> but was:<-1>\n\tat CalculatorTest.sumTest(Unknown Source)\n\tat java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n\tat java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\n\tat java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n"
         }
-      ]
+      ],
+      "new_errors": [],
+      "resolved_errors": [],
+      "existing_errors": []
     }
   ]
 }

spec/javascripts/reports/mock_data/new_errors_report.json

+{
+  "summary": { "total": 11, "resolved": 0, "errored": 2, "failed": 0 },
+  "suites": [
+    {
+      "name": "rspec:pg",
+      "summary": { "total": 8, "resolved": 0, "errored": 0, "failed": 0 },
+      "new_failures": [],
+      "resolved_failures": [],
+      "existing_failures": [],
+      "new_errors": [],
+      "resolved_errors": [],
+      "existing_errors": []
+    },
+    {
+      "name": "karma",
+      "summary": { "total": 3, "resolved": 0, "errored": 2, "failed": 0 },
+      "new_failures": [],
+      "resolved_failures": [],
+      "existing_failures": [],
+      "new_errors": [
+        {
+          "result": "error",
+          "name": "Test#sum when a is 1 and b is 2 returns summary",
+          "execution_time": 0.009411,
+          "system_output": "Failed: Error in render: 'TypeError: Cannot read property 'status' of undefined'"
+        },
+        {
+          "result": "error",
+          "name": "Test#sum when a is 100 and b is 200 returns summary",
+          "execution_time": 0.000162,
+          "system_output": "Failed: Error in render: 'TypeError: Cannot read property 'length' of undefined'"
+        }
+      ],
+      "resolved_errors": [],
+      "existing_errors": []
+    }
+  ]
+}

spec/javascripts/reports/mock_data/new_failures_report.json

 {
-  "summary": { "total": 11, "resolved": 0, "failed": 2 },
+  "summary": { "total": 11, "resolved": 0, "errored": 0, "failed": 2 },
   "suites": [
     {
       "name": "rspec:pg",
-      "summary": { "total": 8, "resolved": 0, "failed": 2 },
+      "summary": { "total": 8, "resolved": 0, "errored": 0, "failed": 2 },
       "new_failures": [
         {
           "result": "failure",
@@ -19,14 +19,20 @@
         }
       ],
       "resolved_failures": [],
-      "existing_failures": []
+      "existing_failures": [],
+      "new_errors": [],
+      "resolved_errors": [],
+      "existing_errors": []
     },
     {
       "name": "java ant",
-      "summary": { "total": 3, "resolved": 0, "failed": 0 },
+      "summary": { "total": 3, "resolved": 0, "errored": 0, "failed": 0 },
       "new_failures": [],
       "resolved_failures": [],
-      "existing_failures": []
+      "existing_failures": [],
+      "new_errors": [],
+      "resolved_errors": [],
+      "existing_errors": []
     }
   ]
 }

spec/javascripts/reports/mock_data/no_failures_report.json

 {
   "status": "success",
-  "summary": { "total": 11, "resolved": 0, "failed": 0 },
+  "summary": { "total": 11, "resolved": 0, "errored": 0, "failed": 0 },
   "suites": [
     {
       "name": "rspec:pg",
       "status": "success",
-      "summary": { "total": 8, "resolved": 0, "failed": 0 },
+      "summary": { "total": 8, "resolved": 0, "errored": 0, "failed": 0 },
       "new_failures": [],
       "resolved_failures": [],
-      "existing_failures": []
+      "existing_failures": [],
+      "new_errors": [],
+      "resolved_errors": [],
+      "existing_errors": []
     },
     {
       "name": "java ant",
       "status": "success",
-      "summary": { "total": 3, "resolved": 0, "failed": 0 },
+      "summary": { "total": 3, "resolved": 0, "errored": 0, "failed": 0 },
       "new_failures": [],
       "resolved_failures": [],
-      "existing_failures": []
+      "existing_failures": [],
+      "new_errors": [],
+      "resolved_errors": [],
+      "existing_errors": []
     }
   ]
 }

spec/javascripts/reports/mock_data/resolved_failures.json

 {
   "status": "success",
-  "summary": { "total": 11, "resolved": 2, "failed": 0 },
+  "summary": { "total": 11, "resolved": 4, "errored": 0, "failed": 0 },
   "suites": [
     {
       "name": "rspec:pg",
       "status": "success",
-      "summary": { "total": 8, "resolved": 2, "failed": 0 },
+      "summary": { "total": 8, "resolved": 4, "errored": 0, "failed": 0 },
       "new_failures": [],
       "resolved_failures": [
         {
@@ -23,15 +23,36 @@
           "stack_trace": null
         }
       ],
-      "existing_failures": []
+      "existing_failures": [],
+      "new_errors": [],
+      "resolved_errors": [
+        {
+          "status": "success",
+          "name": "Test#sum when a is 4 and b is 4 returns summary",
+          "execution_time": 0.00342,
+          "system_output": null,
+          "stack_trace": null
+        },
+        {
+          "status": "success",
+          "name": "Test#sum when a is 40 and b is 400 returns summary",
+          "execution_time": 0.0000231,
+          "system_output": null,
+          "stack_trace": null
+        }
+      ],
+      "existing_errors": []
     },
     {
       "name": "java ant",
       "status": "success",
-      "summary": { "total": 3, "resolved": 0, "failed": 0 },
+      "summary": { "total": 3, "resolved": 0, "errored": 0, "failed": 0 },
       "new_failures": [],
       "resolved_failures": [],
-      "existing_failures": []
+      "existing_failures": [],
+      "new_errors": [],
+      "resolved_errors": [],
+      "existing_errors": []
     }
   ]
 }