Commit b71beea0 authored by Xiaowu Zhang's avatar Xiaowu Zhang Committed by Romain Courteaud

[HARDCODED][stack erp5] add a runZeleniumTest script

To run zelenium tests on saucelabs
Desactivate usuable runTestSuite
parent f8dc9d41
...@@ -153,6 +153,13 @@ url = ${:_profile_base_location_}/template/instance-mariadb-resiliency-after-imp ...@@ -153,6 +153,13 @@ url = ${:_profile_base_location_}/template/instance-mariadb-resiliency-after-imp
md5sum = b32d9ee1cb85f85d8d2f2b58f58459f1 md5sum = b32d9ee1cb85f85d8d2f2b58f58459f1
mode = 755 mode = 755
[template-run-zelenium]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/template/run-zelenium-test.py.in
md5sum = b78c7f5db0e9c9de9b96ac600da675a6
output = ${buildout:directory}/run-zelenium-test.py.in
mode = 755
[template-mariadb] [template-mariadb]
<= download-base <= download-base
filename = instance-mariadb.cfg.in filename = instance-mariadb.cfg.in
...@@ -234,7 +241,7 @@ recipe = slapos.recipe.template:jinja2 ...@@ -234,7 +241,7 @@ recipe = slapos.recipe.template:jinja2
# XXX: "template.cfg" is hardcoded in instanciation recipe # XXX: "template.cfg" is hardcoded in instanciation recipe
rendered = ${buildout:directory}/template.cfg rendered = ${buildout:directory}/template.cfg
template = ${:_profile_base_location_}/instance.cfg.in template = ${:_profile_base_location_}/instance.cfg.in
md5sum = 29e60884bb09ba5461a7c4717d67877e md5sum = 90916b88414f5a02a535e35c776cbbc9
mode = 640 mode = 640
context = context =
key mariadb_link_binary template-mariadb:link-binary key mariadb_link_binary template-mariadb:link-binary
...@@ -312,6 +319,7 @@ context = ...@@ -312,6 +319,7 @@ context =
key template_zeo template-zeo:target key template_zeo template-zeo:target
key template_zope template-zope:target key template_zope template-zope:target
key template_zope_conf template-zope-conf:target key template_zope_conf template-zope-conf:target
key template_run_zelenium template-run-zelenium:output
key userhosts_location userhosts:location key userhosts_location userhosts:location
key wget_location wget:location key wget_location wget:location
key xdamage_location xdamage:location key xdamage_location xdamage:location
...@@ -331,7 +339,7 @@ rendered = ${monitor-template-dummy:target} ...@@ -331,7 +339,7 @@ rendered = ${monitor-template-dummy:target}
[template-erp5] [template-erp5]
<= download-base <= download-base
filename = instance-erp5.cfg.in filename = instance-erp5.cfg.in
md5sum = b5f49c90017684aa3389ef3f97ece509 md5sum = bd7d240bc8c1edf01cb717e5bba74a61
[template-zeo] [template-zeo]
<= download-base <= download-base
...@@ -341,7 +349,7 @@ md5sum = b0cb0ee97cddc79112a718e065806037 ...@@ -341,7 +349,7 @@ md5sum = b0cb0ee97cddc79112a718e065806037
[template-zope] [template-zope]
<= download-base <= download-base
filename = instance-zope.cfg.in filename = instance-zope.cfg.in
md5sum = 83304f1940e0d7555bd678e2cea4c738 md5sum = 49ccb2e83e0ed55b616c6d95ef5d810e
link-binary = link-binary =
${aspell:location}/bin/aspell ${aspell:location}/bin/aspell
${dmtx-utils:location}/bin/dmtxwrite ${dmtx-utils:location}/bin/dmtxwrite
...@@ -565,6 +573,7 @@ eggs = ...@@ -565,6 +573,7 @@ eggs =
# Needed for parsing .po files from our Localizer subset # Needed for parsing .po files from our Localizer subset
polib polib
selenium
# parameterizing the version of the generated python interpreter name by the # parameterizing the version of the generated python interpreter name by the
# python section version causes dependency between this egg section and the # python section version causes dependency between this egg section and the
...@@ -825,3 +834,4 @@ zope.app.dependable = 3.5.1 ...@@ -825,3 +834,4 @@ zope.app.dependable = 3.5.1
# Products.CMFCalendar==2.2.3 # Products.CMFCalendar==2.2.3
# five.formlib==1.0.4 # five.formlib==1.0.4
zope.app.form = 4.0.2 zope.app.form = 4.0.2
selenium = 2.53.1
...@@ -135,6 +135,7 @@ config-bt5-repository-url = {{ dumps(slapparameter_dict.get('bt5-repository-url' ...@@ -135,6 +135,7 @@ config-bt5-repository-url = {{ dumps(slapparameter_dict.get('bt5-repository-url'
config-cloudooo-url = ${request-cloudooo:connection-url} config-cloudooo-url = ${request-cloudooo:connection-url}
config-deadlock-debugger-password = ${publish-early:deadlock-debugger-password} config-deadlock-debugger-password = ${publish-early:deadlock-debugger-password}
config-developer-list = {{ dumps(slapparameter_dict.get('developer-list', [inituser_login])) }} config-developer-list = {{ dumps(slapparameter_dict.get('developer-list', [inituser_login])) }}
config-saucelabs-dict = {{ dumps(slapparameter_dict.get('saucelabs-dict', {})) }}
config-hosts-dict = {{ dumps(slapparameter_dict.get('hosts-dict', {})) }} config-hosts-dict = {{ dumps(slapparameter_dict.get('hosts-dict', {})) }}
config-hostalias-dict = {{ dumps(slapparameter_dict.get('hostalias-dict', {})) }} config-hostalias-dict = {{ dumps(slapparameter_dict.get('hostalias-dict', {})) }}
config-inituser-login = {{ dumps(inituser_login) }} config-inituser-login = {{ dumps(inituser_login) }}
......
...@@ -2,6 +2,7 @@ ...@@ -2,6 +2,7 @@
{% set next_port = slapparameter_dict['port-base'] -%} {% set next_port = slapparameter_dict['port-base'] -%}
{% set site_id = slapparameter_dict['site-id'] -%} {% set site_id = slapparameter_dict['site-id'] -%}
{% set zodb_dict = slapparameter_dict['zodb-dict'] -%} {% set zodb_dict = slapparameter_dict['zodb-dict'] -%}
{% set saucelabs_dict = slapparameter_dict.get('saucelabs-dict', None) -%}
{% set node_id_base = slapparameter_dict['name'] -%} {% set node_id_base = slapparameter_dict['name'] -%}
{% set part_list = [] -%} {% set part_list = [] -%}
{% set publish_list = [] -%} {% set publish_list = [] -%}
...@@ -393,6 +394,25 @@ monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }} ...@@ -393,6 +394,25 @@ monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
monitor-httpd-port = {{ next_port }} monitor-httpd-port = {{ next_port }}
monitor-title = Zope monitor monitor-title = Zope monitor
{% if saucelabs_dict -%}
[test-zelenium-runner-parameter]
configuration = {{ dumps(saucelabs_dict) }}
user = {{ dumps(slapparameter_dict['inituser-login']) }}
password = {{ dumps(slapparameter_dict['inituser-password']) }}
[{{ section('test-zelenium-runner') }}]
recipe = slapos.recipe.template:jinja2
template = {{ parameter_dict['run-zelenium-template'] }}
# XXX Surcharged normal runTestSuite brutally
rendered = ${directory:bin}/runTestSuite
extensions = jinja2.ext.do
context =
import json_module json
key configuration test-zelenium-runner-parameter:configuration
key user test-zelenium-runner-parameter:user
key password test-zelenium-runner-parameter:password
{%- endif %}
[buildout] [buildout]
extends = extends =
{{ logrotate_cfg }} {{ logrotate_cfg }}
......
...@@ -115,6 +115,7 @@ extra-context = ...@@ -115,6 +115,7 @@ extra-context =
[dynamic-template-zope-parameters] [dynamic-template-zope-parameters]
bin-directory = {{ bin_directory }} bin-directory = {{ bin_directory }}
zope-conf-template = {{ template_zope_conf }} zope-conf-template = {{ template_zope_conf }}
run-zelenium-template = {{ template_run_zelenium }}
buildout-bin-directory = {{ buildout_bin_directory }} buildout-bin-directory = {{ buildout_bin_directory }}
6tunnel = {{ sixtunnel_location }} 6tunnel = {{ sixtunnel_location }}
coreutils = {{ coreutils_location }} coreutils = {{ coreutils_location }}
......
#!${buildout:directory}/bin/${eggs:interpreter}
# BEWARE: This file is operated by slapgrid
# BEWARE: It will be overwritten automatically
import argparse, os, sys, traceback
from erp5.util import taskdistribution
import time
from lxml import etree
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time
import json
def main():
parser = argparse.ArgumentParser(description='Run a test suite.')
parser.add_argument('--test_suite', help='The test suite name')
parser.add_argument('--test_suite_title', help='The test suite title')
parser.add_argument('--test_node_title', help='The test node title')
parser.add_argument('--project_title', help='The project title')
parser.add_argument('--revision', help='The revision to test',
default='dummy_revision')
parser.add_argument('--node_quantity', help='ignored', type=int)
parser.add_argument('--master_url',
help='The Url of Master controling many suites')
parser.add_argument('--frontend_url',
help='The url of frontend of the test suite')
parser.add_argument('--target',
help='Target OS to run tests on',
type=str)
parser.add_argument('--target_version',
help='Target OS version to use',
type=str,)
parser.add_argument('--target_browser',
help='The desired browser of the target OS to be used. Example: Firefox if target is Android.',
type=str,)
parser.add_argument('--target_device',
help='The desired device running the target OS. Example: iPad Simulator, if target is iOS.',
type=str,)
parser.add_argument('--appium_server_auth',
help='Combination of user and token to access SauceLabs service. (i.e. user:token)',
type=str)
args = parser.parse_args()
parsed_parameters = json.loads('{{ json_module.dumps(configuration) }}')
if not getattr(args, 'target', None):
args.target = parsed_parameters.get('target', 'firefox')
if not getattr(args, 'test_suite', None):
args.test_suite = parsed_parameters.get('test-suite')
if not getattr(args, 'target_version', None):
args.target_version = parsed_parameters.get('target-version')
if not getattr(args, 'appium_server_auth', None):
args.appium_server_auth = parsed_parameters.get('appium-server-auth')
if not getattr(args, 'target_browser', None):
args.target_browser = parsed_parameters.get('target-browser')
if not getattr(args, 'target_device', None):
args.target_device = parsed_parameters.get('target-device')
if not getattr(args, 'frontend_url', None):
args.frontend_url = parsed_parameters.get('frontend-url')
is_browser_running = False
test_line_dict = {}
test_suite_title = args.test_suite_title or args.test_suite
test_suite = args.test_suite
revision = args.revision
# curl https://saucelabs.com/rest/v1/info/platforms/all
# https://wiki.saucelabs.com/display/DOCS/Platform+Configurator#/
if args.target in ['iOS', 'Android']:
# parameters for mobile emulators have different names then parameters for
# desktop OSes
capabilities = {
'platformName': args.target,
'platformVersion': args.target_version,
'deviceName': args.target_device,
'browserName': args.target_browser
}
elif 'Windows' in args.target or 'OS X' in args.target:
capabilities = {
'browserName': args.target_browser,
'platform': args.target,
'version': args.target_version
}
if not args.appium_server_auth:
raise RuntimeError('--appium_server_auth is required.')
if not args.frontend_url:
raise RuntimeError('--frontend_url is required.')
appium_url = "http://%s@ondemand.saucelabs.com/wd/hub" % (args.appium_server_auth)
# adjust make path to frontend
# Do not store any test result in the ZMI
url = "%s/erp5/portal_tests/core/TestRunner.html" \
"?test=../test_suite_html" \
"&auto=on" \
"&resultsUrl=../getId" \
"&__ac_name=%s" \
"&__ac_password=%s" % (args.frontend_url, "{{ user }}", "{{ password }}")
# Wait until all bt5 are installed...
time.sleep(120)
try:
browser = webdriver.Remote(appium_url, capabilities)
is_browser_running = True
agent = browser.execute_script("return navigator.userAgent")
print url
print agent
start_time = time.time()
browser.get(url)
# Wait for Zelenium to be loaded
WebDriverWait(browser, 10).until(EC.presence_of_element_located((
By.XPATH, '//iframe[@id="testSuiteFrame"]'
)))
# XXX No idea how to wait for the iframe content to be loaded
time.sleep(5)
# Count number of test to be executed
test_count = browser.execute_script(
"return document.getElementById('testSuiteFrame').contentDocument.querySelector('tbody').children.length"
) - 1 # First child is the file name
# Wait for test to be executed
erp5_zelenium_test_timeout = 90
WebDriverWait(browser, erp5_zelenium_test_timeout * (test_count + 1)).until(EC.presence_of_element_located((
By.XPATH, '//td[@id="testRuns" and contains(text(), "%i")]' % test_count
)))
execution_duration = round(time.time() - start_time, 2)
if test_count:
test_execution_duration = execution_duration / test_count
else:
test_execution_duration = 0
html_parser = etree.HTMLParser(recover=True)
# body = etree.fromstring(browser.page_source.encode('UTF-8'), html_parser)
# test_count = int(body.xpath('//td[@id="testRuns"]')[0].text)
# failed_test_count = int(body.xpath('//td[@id="testFailures"]')[0].text)
# print 'Run %i, failed %i' % (test_count, failed_test_count)
# https://github.com/appium/appium/issues/5199
# browser.switch_to.frame(browser.find_element_by_id("testSuiteFrame"))
# iframe = etree.fromstring(browser.page_source.encode('UTF-8'), html_parser)
iframe = etree.fromstring(
browser.execute_script(
"return document.getElementById('testSuiteFrame').contentDocument.querySelector('html').innerHTML"
).encode('UTF-8'),
html_parser
)
browser.quit()
is_browser_running = False
tbody = iframe.xpath('.//body/table/tbody')[0]
tr_count = 0
for tr in tbody:
if tr_count:
# First td is the main title
test_name = tr[0][0].text
skip_count = success_count = error_count = 0
if len(tr) == 1:
# Test was not executed
tr_count = 1
test_table = 'Test not executed!'
test_tbody = 'Test not executed!'
else:
test_table = tr[1].xpath('.//table')[0]
test_tbody = tr[1].xpath('.//tbody')[0]
tr_count = len(test_tbody)
for tr in test_tbody:
# print etree.tostring(tr).split('\n')[0]
status = tr.attrib.get('class')
if 'status_done' in status:
skip_count += 1
elif 'status_passed' in status:
success_count += 1
elif 'status_failed' in status:
error_count += 1
test_line_dict[test_name] = {
'test_count': tr_count,
'error_count': error_count,
'failure_count': tr_count - (skip_count + success_count + error_count),
'skip_count': skip_count,
'duration': test_execution_duration,
'command': url,
'stdout': agent,
'stderr': '',
'html_test_result': etree.tostring(test_table)
}
tr_count += 1
tool = taskdistribution.TaskDistributionTool(portal_url=args.master_url)
test_result = tool.createTestResult(revision = revision,
test_name_list = test_line_dict.keys(),
node_title = args.test_node_title,
test_title = test_suite_title,
project_title = args.project_title)
if test_result is None or not hasattr(args, 'master_url'):
return
# report test results
while 1:
test_result_line = test_result.start()
if not test_result_line:
print 'No test result anymore.'
break
print 'Submitting: "%s"' % test_result_line.name
# report status back to Nexedi ERP5
test_result_line.stop(**test_line_dict[test_result_line.name])
except:
# Catch any exception here, to warn user instead of being silent,
# by generating fake error result
print traceback.format_exc()
result = dict(status_code=-1,
command=url,
stderr=traceback.format_exc(),
stdout='')
# XXX: inform test node master of error
raise EnvironmentError(result)
finally:
if is_browser_running:
# if by any chance browser is still running due to
# traceback raised make sure we cleanup
browser.quit()
if __name__ == "__main__":
main()
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment