Commit 9535660a authored by Jason Madden's avatar Jason Madden

Rework the testrunner to group tests.

This almost cuts the test time in half on my machine.
parent 3f41248c
......@@ -25,6 +25,8 @@ from gevent.hub import Hub
from .exception import ExpectedException
class QuietHub(Hub):
_resolver = None
_threadpool = None
EXPECTED_TEST_ERROR = (ExpectedException,)
......@@ -42,3 +44,6 @@ class QuietHub(Hub):
# see handle_error
return
return Hub.print_exception(self, context, t, v, tb)
def destroy(self, destroy_loop=None):
raise AssertionError("Do not destroy the hub in a unittest")
......@@ -1247,7 +1247,7 @@ disabled_tests += [
'test_ssl.BasicSocketTests.test_openssl_version'
]
if TRAVIS and OSX:
if OSX:
disabled_tests += [
# This sometimes produces OSError: Errno 40: Message too long
......
#!/usr/bin/env python
from __future__ import print_function, absolute_import, division
import re
import sys
import os
import glob
import traceback
import importlib
from datetime import timedelta
from contextlib import contextmanager
from datetime import timedelta
from multiprocessing.pool import ThreadPool
from multiprocessing import cpu_count
from gevent._util import Lazy
from . import util
from .resources import parse_resources
from .resources import setup_resources
......@@ -244,106 +249,258 @@ class TravisFoldingRunner(object):
def __call__(self):
return self._runner()
def discover(
tests=None, ignore_files=None,
ignored=(), coverage=False,
package=None,
configured_ignore_coverage=(),
configured_test_options=None,
):
# pylint:disable=too-many-locals,too-many-branches
configured_test_options = configured_test_options or {}
olddir = os.getcwd()
ignore = set(ignored or ())
if ignore_files:
ignore_files = ignore_files.split(',')
for f in ignore_files:
ignore.update(set(load_list_from_file(f, package)))
if coverage:
ignore.update(configured_ignore_coverage)
if package:
package_dir = _dir_from_package_name(package)
# We need to glob relative names, our config is based on filenames still
os.chdir(package_dir)
if not tests:
tests = set(glob.glob('test_*.py')) - set(['test_support.py'])
else:
tests = set(tests)
if ignore:
# Always ignore the designated list, even if tests were specified
# on the command line. This fixes a nasty interaction with test__threading_vs_settrace.py
# being run under coverage when 'grep -l subprocess test*py' is used to list the tests
# to run.
tests -= ignore
tests = sorted(tests)
to_process = []
to_import = []
for filename in tests:
# Support either 'gevent.tests.foo' or 'gevent/tests/foo.py'
if filename.startswith('gevent.tests'):
# XXX: How does this interact with 'package'? Probably not well
qualified_name = module_name = filename
filename = filename[len('gevent.tests') + 1:]
filename = filename.replace('.', os.sep) + '.py'
else:
module_name = os.path.splitext(filename)[0]
qualified_name = package + '.' + module_name if package else module_name
with open(os.path.abspath(filename), 'rb') as f:
# Some of the test files (e.g., test__socket_dns) are
# UTF8 encoded. Depending on the environment, Python 3 may
# try to decode those as ASCII, which fails with UnicodeDecodeError.
# Thus, be sure to open and compare in binary mode.
# Open the absolute path to make errors more clear,
# but we can't store the absolute path, our configuration is based on
# relative file names.
contents = f.read()
if b'TESTRUNNER' in contents: # test__monkey_patching.py
# XXX: Rework this to avoid importing.
to_import.append(qualified_name)
else:
# XXX: For simple python module tests, try this with `runpy.run_module`,
# very similar to the way we run things for monkey patching.
# The idea here is that we can perform setup ahead of time (e.g., setup_resources())
# in each test without having to do it manually or force calls or modifications to those
# tests.
class Discovery(object):
package_dir = None
package = None
def __init__(
self,
tests=None,
ignore_files=None,
ignored=(),
coverage=False,
package=None,
configured_ignore_coverage=(),
configured_test_options=None,
):
# pylint:disable=too-many-locals,too-many-branches
self.configured_test_options = configured_test_options or {}
self.ignore = set(ignored or ())
self.tests = tests
self.configured_test_options = configured_test_options
if ignore_files:
ignore_files = ignore_files.split(',')
for f in ignore_files:
self.ignore.update(set(load_list_from_file(f, package)))
if coverage:
self.ignore.update(configured_ignore_coverage)
if package:
self.package = package
self.package_dir = _dir_from_package_name(package)
class Discovered(object):
def __init__(self, package, configured_test_options, ignore):
self.package = package
self.configured_test_options = configured_test_options
self.ignore = ignore
self.to_import = []
self.std_monkey_patch_files = []
self.no_monkey_patch_files = []
self.commands = []
@staticmethod
def __makes_simple_monkey_patch(
contents,
_patch_present=re.compile(br'[^#].*patch_all\(\)'),
_patch_indented=re.compile(br' .*patch_all\(\)')
):
return (
# A non-commented patch_all() call is present
bool(_patch_present.search(contents))
# that is not indented (because that implies its not at the top-level,
# so some preconditions are being set)
and not _patch_indented.search(contents)
)
@staticmethod
def __file_allows_monkey_combine(contents):
return b'testrunner-no-monkey-combine' not in contents
@staticmethod
def __file_allows_combine(contents):
return b'testrunner-no-combine' not in contents
@staticmethod
def __calls_unittest_main_toplevel(
contents,
_greentest_main=re.compile(br' greentest.main\(\)'),
_unittest_main=re.compile(br' unittest.main\(\)'),
_import_main=re.compile(br'from gevent.testing import.*main'),
_main=re.compile(br' main\(\)'),
):
# TODO: Add a check that this comes in a line directly after
# if __name__ == __main__.
return (
_greentest_main.search(contents)
or _unittest_main.search(contents)
or (_import_main.search(contents) and _main.search(contents))
)
def __can_monkey_combine(self, filename, contents):
return (
filename not in self.configured_test_options
and self.__makes_simple_monkey_patch(contents)
and self.__file_allows_monkey_combine(contents)
and self.__file_allows_combine(contents)
and self.__calls_unittest_main_toplevel(contents)
)
@staticmethod
def __makes_no_monkey_patch(contents, _patch_present=re.compile(br'[^#].*patch_\w*\(')):
return not _patch_present.search(contents)
def __can_nonmonkey_combine(self, filename, contents):
return (
filename not in self.configured_test_options
and self.__makes_no_monkey_patch(contents)
and self.__file_allows_combine(contents)
and self.__calls_unittest_main_toplevel(contents)
)
def __begin_command(self):
cmd = [sys.executable, '-u']
if PYPY and PY2:
# Doesn't seem to be an env var for this
cmd.extend(('-X', 'track-resources'))
if package:
# Using a package is the best way to work with coverage 5
# when we specify 'source = <package>'
cmd.append('-m' + qualified_name)
return cmd
def __add_test(self, qualified_name, filename, contents):
if b'TESTRUNNER' in contents: # test__monkey_patching.py
# XXX: Rework this to avoid importing.
# XXX: Rework this to allow test combining (it could write the files out and return
# them directly; we would use 'python -m gevent.monkey --module unittest ...)
self.to_import.append(qualified_name)
# TODO: I'm pretty sure combining breaks keeping particular tests standalone
elif self.__can_monkey_combine(filename, contents):
self.std_monkey_patch_files.append(qualified_name if self.package else filename)
elif self.__can_nonmonkey_combine(filename, contents):
self.no_monkey_patch_files.append(qualified_name if self.package else filename)
else:
cmd.append(filename)
options = DEFAULT_RUN_OPTIONS.copy()
options.update(configured_test_options.get(filename, {}))
to_process.append((cmd, options))
os.chdir(olddir)
# When we actually execute, do so from the original directory,
# this helps find setup.py
for qualified_name in to_import:
module = importlib.import_module(qualified_name)
for cmd, options in module.TESTRUNNER():
if remove_options(cmd)[-1] in ignore:
continue
to_process.append((cmd, options))
# XXX: For simple python module tests, try this with
# `runpy.run_module`, very similar to the way we run
# things for monkey patching. The idea here is that we
# can perform setup ahead of time (e.g.,
# setup_resources()) in each test without having to do
# it manually or force calls or modifications to those
# tests.
cmd = self.__begin_command()
if self.package:
# Using a package is the best way to work with coverage 5
# when we specify 'source = <package>'
cmd.append('-m' + qualified_name)
else:
cmd.append(filename)
options = DEFAULT_RUN_OPTIONS.copy()
options.update(self.configured_test_options.get(filename, {}))
self.commands.append((cmd, options))
@staticmethod
def __remove_options(lst):
return [x for x in lst if x and not x.startswith('-')]
def __expand_imports(self):
for qualified_name in self.to_import:
module = importlib.import_module(qualified_name)
for cmd, options in module.TESTRUNNER():
if self.__remove_options(cmd)[-1] in self.ignore:
continue
self.commands.append((cmd, options))
del self.to_import[:]
def __combine_commands(self, files, group_size=5):
if not files:
return
from itertools import groupby
cnt = [0, 0]
def make_group(_):
if cnt[0] > group_size:
cnt[0] = 0
cnt[1] += 1
cnt[0] += 1
return cnt[1]
for _, group in groupby(files, make_group):
cmd = self.__begin_command()
cmd.append('-m')
cmd.append('unittest')
cmd.append('-v')
for name in group:
cmd.append(name)
self.commands.insert(0, (cmd, DEFAULT_RUN_OPTIONS.copy()))
del files[:]
def visit_file(self, filename):
# Support either 'gevent.tests.foo' or 'gevent/tests/foo.py'
if filename.startswith('gevent.tests'):
# XXX: How does this interact with 'package'? Probably not well
qualified_name = module_name = filename
filename = filename[len('gevent.tests') + 1:]
filename = filename.replace('.', os.sep) + '.py'
else:
module_name = os.path.splitext(filename)[0]
qualified_name = self.package + '.' + module_name if self.package else module_name
with open(os.path.abspath(filename), 'rb') as f:
# Some of the test files (e.g., test__socket_dns) are
# UTF8 encoded. Depending on the environment, Python 3 may
# try to decode those as ASCII, which fails with UnicodeDecodeError.
# Thus, be sure to open and compare in binary mode.
# Open the absolute path to make errors more clear,
# but we can't store the absolute path, our configuration is based on
# relative file names.
contents = f.read()
self.__add_test(qualified_name, filename, contents)
def visit_files(self, filenames):
for filename in filenames:
self.visit_file(filename)
self.__expand_imports()
self.__combine_commands(self.std_monkey_patch_files)
self.__combine_commands(self.no_monkey_patch_files)
@contextmanager
def _in_package_dir(self):
olddir = os.getcwd()
if self.package_dir:
os.chdir(self.package_dir)
try:
yield
finally:
os.chdir(olddir)
@Lazy
def discovered(self):
tests = self.tests
discovered = self.Discovered(self.package, self.configured_test_options, self.ignore)
# We need to glob relative names, our config is based on filenames still
with self._in_package_dir():
if not tests:
tests = set(glob.glob('test_*.py')) - set(['test_support.py'])
else:
tests = set(tests)
if self.ignore:
# Always ignore the designated list, even if tests
# were specified on the command line. This fixes a
# nasty interaction with
# test__threading_vs_settrace.py being run under
# coverage when 'grep -l subprocess test*py' is used
# to list the tests to run.
tests -= self.ignore
tests = sorted(tests)
discovered.visit_files(tests)
return to_process
return discovered
def __iter__(self):
return iter(self.discovered.commands) # pylint:disable=no-member
def remove_options(lst):
return [x for x in lst if x and not x.startswith('-')]
def __len__(self):
return len(self.discovered.commands) # pylint:disable=no-member
def load_list_from_file(filename, package):
result = []
......@@ -600,7 +757,7 @@ def main():
IGNORE_COVERAGE = config['IGNORE_COVERAGE']
tests = discover(
tests = Discovery(
options.tests,
ignore_files=options.ignore,
ignored=IGNORED_TESTS,
......
......@@ -29,7 +29,8 @@ class TestTimeout(greentest.TestCase):
while True:
listener.recvfrom(10000)
gevent.spawn(reader)
greader = gevent.spawn(reader)
self._close_on_teardown(greader.kill)
r = Resolver(servers=[address[0]], timeout=0.001, tries=1,
udp_port=address[-1])
......
......@@ -158,4 +158,4 @@ class Test(greentest.TestCase):
if __name__ == '__main__':
greentest.main()
greentest.main() # pragma: testrunner-no-combine
......@@ -7,18 +7,25 @@ try:
except ImportError:
import _thread as thread
from gevent import testing as greentest
hub = gevent.get_hub()
watcher = hub.loop.async_()
class Test(greentest.TestCase):
def test(self):
hub = gevent.get_hub()
watcher = hub.loop.async_()
# BWC for <3.7: This should still be an attribute
assert hasattr(hub.loop, 'async')
# BWC for <3.7: This should still be an attribute
assert hasattr(hub.loop, 'async')
gevent.spawn_later(0.1, thread.start_new_thread, watcher.send, ())
gevent.spawn_later(0.1, thread.start_new_thread, watcher.send, ())
start = time.time()
start = time.time()
with gevent.Timeout(1.0): # Large timeout for appveyor
hub.wait(watcher)
with gevent.Timeout(1.0): # Large timeout for appveyor
hub.wait(watcher)
print('Watcher %r reacted after %.6f seconds' % (watcher, time.time() - start - 0.1))
print('Watcher %r reacted after %.6f seconds' % (watcher, time.time() - start - 0.1))
if __name__ == '__main__':
greentest.main()
import gevent
from gevent.hub import get_hub
called = []
from gevent import testing as greentest
class Test(greentest.TestCase):
def f():
called.append(1)
def test(self):
loop = get_hub().loop
called = []
def f():
called.append(1)
def main():
loop = get_hub().loop
x = loop.run_callback(f)
x = loop.run_callback(f)
assert x, x
gevent.sleep(0)
assert called == [1], called
assert not x, (x, bool(x))
assert x, x
gevent.sleep(0)
assert called == [1], called
assert not x, (x, bool(x))
x = loop.run_callback(f)
assert x, x
x.stop()
assert not x, x
gevent.sleep(0)
assert called == [1], called
assert not x, x
x = loop.run_callback(f)
assert x, x
x.stop()
assert not x, x
gevent.sleep(0)
assert called == [1], called
assert not x, x
if __name__ == '__main__':
called[:] = []
main()
greentest.main()
......@@ -48,4 +48,4 @@ class TestDestroyHub(unittest.TestCase):
hub.destroy()
if __name__ == '__main__':
unittest.main()
unittest.main() # pragma: testrunner-no-combine
from __future__ import print_function, absolute_import
from gevent import monkey; monkey.patch_all(subprocess=True)
from gevent import monkey; monkey.patch_all()
import signal
import socket
......
from gevent import monkey
monkey.patch_all(subprocess=True)
monkey.patch_all()
from gevent.server import DatagramServer
......
......@@ -21,7 +21,8 @@ import sys
import gevent
from gevent import socket
from gevent.testing import TestCase, main, tcp_listener
from gevent import testing as greentest
from gevent.testing import TestCase, tcp_listener
from gevent.testing import gc_collect_if_needed
from gevent.testing import skipOnPyPy
from gevent.testing import params
......@@ -142,4 +143,4 @@ class TestGreenIo(TestCase):
if __name__ == '__main__':
main()
greentest.main()
import gevent
from gevent import testing as greentest
def func():
pass
class Test(greentest.TestCase):
a = gevent.spawn(func)
b = gevent.spawn(func)
gevent.joinall([a, b, a])
def test(self):
def func():
pass
a = gevent.spawn(func)
b = gevent.spawn(func)
gevent.joinall([a, b, a])
if __name__ == '__main__':
greentest.main()
from gevent.core import loop
from gevent import get_hub
from gevent import testing as greentest
count = 0
class Test(greentest.TestCase):
def test(self):
count = [0]
def incr():
count[0] += 1
def incr():
global count
count += 1
loop = get_hub().loop
loop.run_callback(incr)
loop.run()
self.assertEqual(count, [1])
loop = loop()
loop.run_callback(incr)
loop.run()
assert count == 1, count
if __name__ == '__main__':
greentest.main()
import sys
import unittest
from gevent.testing import TestCase, main
from gevent.testing import TestCase
import gevent
from gevent.timeout import Timeout
......@@ -53,4 +53,4 @@ class TestQueue(TestCase): # pragma: no cover
if __name__ == '__main__':
main()
unittest.main()
from subprocess import Popen
from gevent import monkey
monkey.patch_all()
......@@ -79,6 +77,7 @@ class TestMonkey(SubscriberCleanupMixin, unittest.TestCase):
self.assertTrue(monkey.is_object_patched(modname, objname))
def test_patch_subprocess_twice(self):
Popen = monkey.get_original('subprocess', 'Popen')
self.assertNotIn('gevent', repr(Popen))
self.assertIs(Popen, monkey.get_original('subprocess', 'Popen'))
monkey.patch_subprocess()
......
......@@ -22,7 +22,7 @@ from __future__ import print_function
from gevent import monkey
monkey.patch_all(thread=False)
monkey.patch_all()
from contextlib import contextmanager
try:
......
import unittest
import gevent.testing as greentest
from gevent.testing import TestCase, main
from gevent.testing import TestCase
import gevent
from gevent.hub import get_hub, LoopExit
from gevent import util
......@@ -462,4 +462,4 @@ del AbstractGenericGetTestCase
if __name__ == '__main__':
main()
greentest.main()
......@@ -6,24 +6,29 @@ Fails with PyPy 2.2.1
"""
from __future__ import print_function
import sys
import greenlet
from gevent import testing as greentest
print('Your greenlet version: %s' % (getattr(greenlet, '__version__', None), ))
class Test(greentest.TestCase):
def test(self):
import greenlet
result = []
print('Your greenlet version: %s' % (getattr(greenlet, '__version__', None), ))
result = []
def func():
result.append(repr(sys.exc_info()))
def func():
result.append(repr(sys.exc_info()))
g = greenlet.greenlet(func)
try:
1 / 0
except ZeroDivisionError:
g.switch()
g = greenlet.greenlet(func)
try:
1 / 0
except ZeroDivisionError:
g.switch()
self.assertEqual(result, ['(None, None, None)'])
assert result == ['(None, None, None)'], result
if __name__ == '__main__':
greentest.main()
......@@ -140,7 +140,10 @@ class TestCase(greentest.TestCase):
conn.close()
ex = exc.exception
self.assertIn(ex.args[0], (errno.ECONNREFUSED, errno.EADDRNOTAVAIL), ex)
self.assertIn(ex.args[0],
(errno.ECONNREFUSED, errno.EADDRNOTAVAIL,
errno.ECONNRESET, errno.ECONNABORTED),
(ex, ex.args))
def assert500(self):
self.Settings.assert500(self)
......
......@@ -49,7 +49,7 @@ class Thread(_Thread):
class TestTCP(greentest.TestCase):
maxDiff = None
__timeout__ = None
TIMEOUT_ERROR = socket.timeout
long_data = ", ".join([str(x) for x in range(20000)])
......@@ -210,7 +210,9 @@ class TestTCP(greentest.TestCase):
if match_data is None:
match_data = self.long_data
self.assertEqual(read_data, [match_data])
read_data = read_data[0].split(b',')
match_data = match_data.split(b',')
self.assertEqual(read_data, match_data)
def test_sendall_str(self):
self._test_sendall(self.long_data)
......
"""
Tests specifically for the monkey-patched threading module.
"""
from gevent import monkey; monkey.patch_all()
from gevent import monkey; monkey.patch_all() # pragma: testrunner-no-monkey-combine
import gevent.hub
# check that the locks initialized by 'threading' did not init the hub
......
......@@ -3,7 +3,7 @@
import threading
from gevent import monkey
monkey.patch_all()
monkey.patch_all() # pragma: testrunner-no-monkey-combine
import gevent.testing as greentest
......
......@@ -5,4 +5,4 @@ import threading
# in python code, this used to throw RuntimeErro("Cannot release un-acquired lock")
# See https://github.com/gevent/gevent/issues/615
with threading.RLock():
monkey.patch_all()
monkey.patch_all() # pragma: testrunner-no-monkey-combine
......@@ -25,7 +25,7 @@ class Test(greentest.TestCase):
def target():
tcurrent = threading.current_thread()
monkey.patch_all()
monkey.patch_all() # pragma: testrunner-no-monkey-combine
tcurrent2 = threading.current_thread()
self.assertIsNot(tcurrent, current)
# We get a dummy thread now
......
......@@ -52,6 +52,6 @@ if __name__ == '__main__':
# Only patch after we're running
from gevent import monkey
monkey.patch_all()
monkey.patch_all() # pragma: testrunner-no-monkey-combine
greentest.main()
......@@ -7,7 +7,7 @@ import gevent.testing as greentest
script = """
from gevent import monkey
monkey.patch_all()
monkey.patch_all() # pragma: testrunner-no-monkey-combine
import sys, os, threading, time
......
......@@ -142,6 +142,7 @@ class TestTree(greentest.TestCase):
value = re.compile(' fileno=.').sub('', value)
value = value.replace('ref=-1', 'ref=0')
value = value.replace("type.current_tree", 'GreenletTree.current_tree')
value = value.replace('gevent.tests.__main__.MyLocal', '__main__.MyLocal')
return value
@greentest.ignores_leakcheck
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment