test_result: Support evaluating test result status per line

This is to support SLAPOS-EGG-TEST which runs tests for multiple
repositories from the same test suite. Evaluating each commit from
each repository with the result of the test result is problematic because
every time one test fail, all repositories are marked as failed.

Introduce a new property on test result repository to optionally define
the pattern of test result line to consider for this repository. This way,
with our SLAPOS-EGG-TEST running tests for slapos, slapos.core,
... etc and producing test result line with the same title as the repository
name, we can use something like `slapos$` (the pattern is a regular expression)
to match only slapos,  `slapos\.core` to match only slapos.core etc.

This also allow to define more complex relation if necessary, for example kedifa
and caucase seems related, maybe we want to configure so that a failure on kedifa
test suite marks the commits from both kedifa and caucase as failed (or maybe
not, but at least this way should make it possible)

/reviewed-on nexedi/erp5!964

bt5/erp5_test_result/SkinTemplateItem/portal_skins/erp5_test_result/TestResult_annotateCommit.py

 """Annotate a commit with state of the test (running, canceled, success, failed)
 """
+import re
 portal = context.getPortalObject()
 
 portal_url = portal.ERP5Site_getAbsoluteUrl()
@@ -7,6 +8,27 @@ test_result_relative_url = context.getRelativeUrl()
 test_result_relative_id = context.getId()
 test_result_reference = context.getReference()
 
+
+def getTestResultStateForTestSuiteRepository(test_result_line_pattern):
+  """Compute the state for this test repository.
+
+  This is used in test results like SLAPOS-EGG-TEST where we run in the
+  same test suite tests for multiple more-or-less independent repositories.
+  We don't want to mark commits from all repositories failed when only one
+  test as a problem.
+  On test suite repository, we define the pattern of test lines to consider,
+  this is used here to evaluate this repository only based on the result
+  of the individual test result lines.
+  """
+  if test_result_line_pattern:
+    # state of test result lines matching the pattern.
+    return 'success' if {'PASSED'} == {
+        test_result_line.getStringIndex() for test_result_line
+        in context.contentValues(portal_type='Test Result Line')
+        if re.search(test_result_line_pattern, test_result_line.getTitle())
+    } else 'failed'
+  return state # global state of the test result.
+
 test_suite_data = context.TestResult_getTestSuiteData()
 if test_suite_data:
   for repository_info in test_suite_data['repository_dict'].values():
@@ -16,7 +38,8 @@ if test_suite_data:
       connector.postCommitStatus(
           repository_info['repository_url'],
           repository_info['revision'],
-          state,
+          getTestResultStateForTestSuiteRepository(
+              repository_info['test_result_line_pattern']) if state != 'running' else state,
           connector.getTestResultUrlTemplate().format(
               portal_url=portal_url,
               test_result_relative_url=test_result_relative_url,

bt5/erp5_test_result/SkinTemplateItem/portal_skins/erp5_test_result/TestResult_getTestSuiteData.py

@@ -5,6 +5,9 @@
      revision: commit sha
      repository_url: git url of the repository
      commits_count: number of commits
+     connector_relative_url: URL of the gitlab connector
+     test_result_line_pattern: pattern of test result lines titles that needs
+                               to be considered to assess the result of this test.
 
 returns None if test suite cannot be found.
 """
@@ -42,6 +45,8 @@ for test_result_repository in test_suite.contentValues(portal_type='Test Suite R
   repository_data = repository_dict.setdefault(buildout_section_id, {})
   repository_data['repository_url'] = test_result_repository.getGitUrl()
   repository_data['connector_relative_url'] = test_result_repository.getDestination()
+  repository_data['test_result_line_pattern'] = test_result_repository.getSourceReference()
+
 
 if REQUEST:
   REQUEST.RESPONSE.setHeader('content-type', 'application/json; charset=utf-8')

bt5/erp5_test_result/SkinTemplateItem/portal_skins/erp5_test_result/TestSuiteRepository_view.xml

@@ -105,6 +105,7 @@
                     <value>
                       <list>
                         <string>my_destination_title</string>
+                        <string>my_source_reference</string>
                       </list>
                     </value>
                 </item>

bt5/erp5_test_result/SkinTemplateItem/portal_skins/erp5_test_result/TestSuiteRepository_view/my_source_reference.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="ProxyField" module="Products.ERP5Form.ProxyField"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>delegated_list</string> </key>
+            <value>
+              <list>
+                <string>description</string>
+                <string>title</string>
+              </list>
+            </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>my_source_reference</string> </value>
+        </item>
+        <item>
+            <key> <string>message_values</string> </key>
+            <value>
+              <dictionary>
+                <item>
+                    <key> <string>external_validator_failed</string> </key>
+                    <value> <string>The input failed the external validator.</string> </value>
+                </item>
+              </dictionary>
+            </value>
+        </item>
+        <item>
+            <key> <string>overrides</string> </key>
+            <value>
+              <dictionary>
+                <item>
+                    <key> <string>field_id</string> </key>
+                    <value> <string></string> </value>
+                </item>
+                <item>
+                    <key> <string>form_id</string> </key>
+                    <value> <string></string> </value>
+                </item>
+              </dictionary>
+            </value>
+        </item>
+        <item>
+            <key> <string>tales</string> </key>
+            <value>
+              <dictionary>
+                <item>
+                    <key> <string>field_id</string> </key>
+                    <value> <string></string> </value>
+                </item>
+                <item>
+                    <key> <string>form_id</string> </key>
+                    <value> <string></string> </value>
+                </item>
+              </dictionary>
+            </value>
+        </item>
+        <item>
+            <key> <string>values</string> </key>
+            <value>
+              <dictionary>
+                <item>
+                    <key> <string>description</string> </key>
+                    <value> <string>Only annotate commits based on the state of test result lines matching this regular expression.\n
+\n
+Leave this blank so that the status of test result is considered for all repositories.</string> </value>
+                </item>
+                <item>
+                    <key> <string>field_id</string> </key>
+                    <value> <string>my_string_field</string> </value>
+                </item>
+                <item>
+                    <key> <string>form_id</string> </key>
+                    <value> <string>Base_viewFieldLibrary</string> </value>
+                </item>
+                <item>
+                    <key> <string>title</string> </key>
+                    <value> <string>Test Result Line Pattern</string> </value>
+                </item>
+              </dictionary>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+</ZopeData>

bt5/erp5_test_result/TestTemplateItem/portal_components/test.erp5.testTaskDistribution.py

@@ -1557,12 +1557,14 @@ class TestGitlabRESTConnectorInterface(ERP5TypeTestCase):
                 "connector_relative_url": None,
                 "repository_url": "https://lab.nexedi.com/nexedi/erp5.git",
                 "revision": "dc7b6e2e85e9434a97694a698884b057b7d30286",
+                "test_result_line_pattern": None,
             },
             "test": {
                 "commits_count": 10,
                 "connector_relative_url": self.connector.getRelativeUrl(),
                 "repository_url": "https://lab.example.com/nexedi/test.git",
                 "revision": "cc4c79c003f7cfe0bfcbc7b302eac988110c96ae",
+                "test_result_line_pattern": None,
             },
         },
         "test_suite_relative_url": self.test_suite.getRelativeUrl()
@@ -1588,6 +1590,7 @@ class TestGitlabRESTConnectorInterface(ERP5TypeTestCase):
         "connector_relative_url": None,
         "repository_url": "https://lab.nexedi.com/nexedi/erp5.git",
         "revision": "dc7b6e2e85e9434a97694a698884b057b7d30286",
+        "test_result_line_pattern": None,
     },
     test_suite_data['repository_dict']['foo-bar'])
 
@@ -1640,6 +1643,62 @@ class TestGitlabRESTConnectorInterface(ERP5TypeTestCase):
       self.test_result.start()
       self.tic()
 
+  def test_status_per_test_result_line(self):
+    self.test_suite.test_repo.setSourceReference(
+        'test_result_line')
+    self.test_result.newContent(
+        portal_type='Test Result Line',
+        title='this test_result_line is OK',
+        all_tests=1,
+        string_index='PASSED'
+    )
+    self.test_result.newContent(
+        portal_type='Test Result Line',
+        title='another unrelated line',
+        all_tests=1,
+        string_index='FAILED'
+    )
+    self._start_test_result()
+    self.test_result.setStringIndex('FAILED')
+
+    # this test failed, but we are only interested in lines matching `test_result_line`
+    # because that line was successful, the test is OK.
+    with responses.RequestsMock() as rsps:
+      rsps.add_callback(
+          responses.POST,
+          self.post_commit_status_url,
+          self._response_callback('success'))
+      self.test_result.stop()
+      self.tic()
+
+  def test_status_per_test_result_line_multiple_matches(self):
+    self.test_suite.test_repo.setSourceReference(
+        'test_result_line')
+    self.test_result.newContent(
+        portal_type='Test Result Line',
+        title='this test_result_line is OK',
+        all_tests=1,
+        string_index='PASSED'
+    )
+    self.test_result.newContent(
+        portal_type='Test Result Line',
+        title='also test_result_line but failed',
+        all_tests=1,
+        string_index='FAILED'
+    )
+    self._start_test_result()
+    self.test_result.setStringIndex('PASSED')
+
+    # this test is marked as passed, but we are  interested in all
+    # lines matching `test_result_line` and one of them is failed.
+    with responses.RequestsMock() as rsps:
+      rsps.add_callback(
+          responses.POST,
+          self.post_commit_status_url,
+          self._response_callback('failed'))
+      self.test_result.stop()
+      self.tic()
+
   def test_commit_not_found(self):
     with responses.RequestsMock() as rsps:
       rsps.add(