Commit 53689cf4 authored by Jérome Perrin's avatar Jérome Perrin

ERP5TypeFunctionalTestCase improvements and fixes

 * expose firefox console and error logs in `geckodriver.log`
 * expose `geckodriver.log` in the folder testnodes expose on http
 * fix some cases where test were detected as "did not run" and cases where "did not run" tests were not reported as failures

See merge request nexedi/erp5!1444
parents 6f3cb199 1b1dbf60
......@@ -39,6 +39,7 @@ import logging
from ZPublisher.HTTPResponse import HTTPResponse
from zExceptions.ExceptionFormatter import format_exception
from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase
from Products.ERP5Type.tests.runUnitTest import log_directory
from Products.ERP5Type.Utils import stopProcess, PR_SET_PDEATHSIG
from lxml import etree
from lxml.html import builder as E
......@@ -207,11 +208,20 @@ class FunctionalTestRunner:
# https://bugzilla.mozilla.org/show_bug.cgi?id=1338144
options = webdriver.FirefoxOptions()
options.set_preference('dom.serviceWorkers.enabled', True)
# output javascript console and errors on stdout to help diagnosing failures
options.set_preference('devtools.console.stdout.content', True)
kw = dict(capabilities=capabilities, options=options)
firefox_bin = os.environ.get('firefox_bin')
if firefox_bin:
geckodriver = os.path.join(os.path.dirname(firefox_bin), 'geckodriver')
kw.update(firefox_binary=firefox_bin, executable_path=geckodriver)
kw.update(
firefox_binary=firefox_bin,
executable_path=geckodriver,
# BBB in selenium 3.8.0 this option was named log_path
log_path=os.path.join(log_directory, 'geckodriver.log'),
# service_log_path=os.path.join(log_directory, 'geckodriver.log'),
)
browser = webdriver.Firefox(**kw)
start_time = time.time()
logger.info("Running with browser: %s", browser)
......@@ -248,13 +258,25 @@ class FunctionalTestRunner:
)))
# XXX No idea how to wait for the iframe content to be loaded
time.sleep(5)
# Count number of test to be executed
# Count number of tests to be executed
test_count = browser.execute_script(
"return document.getElementById('testSuiteFrame').contentDocument.querySelector('tbody').children.length"
) - 1
# Wait for tests to end
WebDriverWait(browser, self.timeout).until(EC.presence_of_element_located((
By.XPATH, '//td[@id="testRuns" and contains(text(), "%i")]' % test_count
)))
# At the end of each test, updateSuiteWithResultOfPreviousTest updates
# testSuiteFrame iframe with hidden div containing the test results table.
# We will inspect these tables to know which tests have failed. First we
# need to wait a bit more, because at the end of test ( testComplete ),
# updateSuiteWithResultOfPreviousTest is called by setTimeout. We want to
# wait for the last test (which is the last td) result table to be present
browser.switch_to_frame('testSuiteFrame')
WebDriverWait(browser, 10).until(EC.presence_of_element_located((
By.XPATH, '//table/tbody/tr/td[last()]//table'
)))
browser.switch_to_default_content()
self.execution_duration = round(time.time() - start_time, 2)
html_parser = etree.HTMLParser(recover=True)
iframe = etree.fromstring(
......
......@@ -133,20 +133,21 @@ class ERP5(_ERP5):
def _updateFunctionalTestResponse(self, status_dict):
""" Convert the Unit Test output into more accurate information
related to funcional test run.
related to functional test run.
"""
# Parse relevant information to update response information
try:
summary, html_test_result = status_dict['stderr'].split("-"*79)[1:3]
except ValueError:
# In case of error when parse the file, preserve the original
# informations. This prevents we have unfinished tests.
# information. This prevents we have unfinished tests.
return status_dict
status_dict['html_test_result'] = html_test_result
search = self.FTEST_PASS_FAIL_RE.search(summary)
if search:
group_dict = search.groupdict()
status_dict['failure_count'] = int(group_dict['failures'])
status_dict['failure_count'] = int(group_dict['failures']) \
+ int(status_dict.get('failure_count', 0))
status_dict['test_count'] = int(group_dict['total'])
status_dict['skip_count'] = int(group_dict['expected_failure'])
return status_dict
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment