runner.py 16.3 KB
Newer Older
1
#!/usr/bin/env python
2
#
Julien Muchembled's avatar
Julien Muchembled committed
3
# Copyright (C) 2009-2017  Nexedi SA
4
#
5 6 7 8
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
9
#
10 11 12 13 14 15
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
16
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
17

18
import traceback
19 20
import unittest
import time
Grégory Wisniewski's avatar
Grégory Wisniewski committed
21
import sys
22
import os
23
import re
24 25
from collections import Counter, defaultdict
from cStringIO import StringIO
26
from fnmatch import fnmatchcase
27
from unittest.runner import _WritelnDecorator
28 29 30 31 32

if filter(re.compile(r'--coverage$|-\w*c').match, sys.argv[1:]):
    # Start coverage as soon as possible.
    import coverage
    coverage = coverage.Coverage()
33
    coverage.neotestrunner = []
34 35
    coverage.start()

36
from neo.lib import logging
37 38
from neo.tests import getTempDirectory, NeoTestBase, Patch, \
    __dict__ as neo_tests__dict__
39 40
from neo.tests.benchmark import BenchmarkRunner

41 42
# list of test modules
# each of them have to import its TestCase classes
43
UNIT_TEST_MODULES = [
44 45 46 47
    # generic parts
    'neo.tests.testConnection',
    'neo.tests.testHandler',
    'neo.tests.testNodes',
48
    'neo.tests.testUtil',
49 50 51 52 53
    'neo.tests.testPT',
    # master application
    'neo.tests.master.testClientHandler',
    'neo.tests.master.testMasterApp',
    'neo.tests.master.testMasterPT',
54
    'neo.tests.master.testRecovery',
55
    'neo.tests.master.testStorageHandler',
56
    'neo.tests.master.testTransactions',
57 58 59 60
    # storage application
    'neo.tests.storage.testClientHandler',
    'neo.tests.storage.testMasterHandler',
    'neo.tests.storage.testStorageApp',
61
    'neo.tests.storage.testStorage' + os.getenv('NEO_TESTS_ADAPTER', 'SQLite'),
62
    'neo.tests.storage.testTransactions',
63 64
    # client application
    'neo.tests.client.testClientApp',
65
    'neo.tests.client.testMasterHandler',
66
    'neo.tests.client.testZODBURI',
67 68
    # light functional tests
    'neo.tests.threaded.test',
69
    'neo.tests.threaded.testImporter',
70
    'neo.tests.threaded.testReplication',
Julien Muchembled's avatar
Julien Muchembled committed
71
    'neo.tests.threaded.testSSL',
72 73
]

74
FUNC_TEST_MODULES = [
75
    'neo.tests.functional.testMaster',
76
    'neo.tests.functional.testClient',
77
    'neo.tests.functional.testCluster',
78
    'neo.tests.functional.testStorage',
79 80
]

81
ZODB_TEST_MODULES = [
82 83 84 85 86
    ('neo.tests.zodb.testBasic', 'check'),
    ('neo.tests.zodb.testConflict', 'check'),
    ('neo.tests.zodb.testHistory', 'check'),
    ('neo.tests.zodb.testIterator', 'check'),
    ('neo.tests.zodb.testMT', 'check'),
87
    ('neo.tests.zodb.testPack', 'check'),
88 89 90
    ('neo.tests.zodb.testPersistent', 'check'),
    ('neo.tests.zodb.testReadOnly', 'check'),
    ('neo.tests.zodb.testRevision', 'check'),
91
    #('neo.tests.zodb.testRecovery', 'check'),
92 93 94
    ('neo.tests.zodb.testSynchronization', 'check'),
    # ('neo.tests.zodb.testVersion', 'check'),
    ('neo.tests.zodb.testUndo', 'check'),
95 96 97
    ('neo.tests.zodb.testZODB', 'check'),
]

98

99 100 101 102
class StopOnSuccess(Exception):
    pass


103
class NeoTestRunner(unittest.TextTestResult):
104 105
    """ Custom result class to build report with statistics per module """

106 107 108
    _readable_tid = ()

    def __init__(self, title, verbosity, stop_on_success, readable_tid):
109 110
        super(NeoTestRunner, self).__init__(
            _WritelnDecorator(sys.stderr), False, verbosity)
111
        self._title = title
112
        self.stop_on_success = stop_on_success
113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139
        if readable_tid:
            from neo.lib import util
            from neo.lib.util import dump, p64, u64
            from neo.master.transactions import TransactionManager
            def _nextTID(orig, tm, ttid=None, divisor=None):
                n = self._next_tid
                self._next_tid = n + 1
                n = str(n).rjust(3, '-')
                if ttid:
                    t = u64('T%s%s-' % (n, ttid[1:4]))
                    m = (u64(ttid) - t) % divisor
                    assert m < 211, (p64(t), divisor)
                    t = p64(t + m)
                else:
                    t = 'T%s----' % n
                assert tm._last_tid < t, (tm._last_tid, t)
                tm._last_tid = t
                return t
            self._readable_tid = (
                Patch(self, 1, _next_tid=0),
                Patch(TransactionManager, _nextTID=_nextTID),
                Patch(util, 1, orig_dump=type(dump)(
                    dump.__code__, dump.__globals__)),
                Patch(dump, __code__=(lambda s:
                    s if type(s) is str and s.startswith('T') else
                    orig_dump(s)).__code__),
                )
140
        self.modulesStats = {}
141
        self.failedImports = {}
142 143
        self.run_dict = defaultdict(int)
        self.time_dict = defaultdict(int)
144
        self.temp_directory = getTempDirectory()
145

146 147 148
    def wasSuccessful(self):
        return not (self.failures or self.errors or self.unexpectedSuccesses)

149
    def run(self, name, modules, only):
150
        suite = unittest.TestSuite()
151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168
        loader = unittest.TestLoader()
        if only:
            exclude = only[0] == '!'
            test_only = only[exclude + 1:]
            only = only[exclude]
            if test_only:
                def getTestCaseNames(testCaseClass):
                    tests = loader.__class__.getTestCaseNames(
                        loader, testCaseClass)
                    x = testCaseClass.__name__ + '.'
                    return [t for t in tests
                              if exclude != any(fnmatchcase(x + t, o)
                                                for o in test_only)]
                loader.getTestCaseNames = getTestCaseNames
                if not only:
                    only = '*'
        else:
            print '\n', name
169
        for test_module in modules:
170 171
            # load prefix if supplied
            if isinstance(test_module, tuple):
172
                test_module, loader.testMethodPrefix = test_module
173 174
            if only and not (exclude and test_only or
                             exclude != fnmatchcase(test_module, only)):
175
                continue
176 177 178
            try:
                test_module = __import__(test_module, globals(), locals(), ['*'])
            except ImportError, err:
179
                self.failedImports[test_module] = err
180
                print "Import of %s failed : %s" % (test_module, err)
181
                traceback.print_exc()
182 183
                continue
            suite.addTests(loader.loadTestsFromModule(test_module))
184 185 186 187 188
        try:
            suite.run(self)
        finally:
            # Workaround weird behaviour of Python.
            self._previousTestClass = None
189

190
    def startTest(self, test):
191
        super(NeoTestRunner, self).startTest(test)
192 193
        for patch in self._readable_tid:
            patch.apply()
194 195
        self.run_dict[test.__class__.__module__] += 1
        self.start_time = time.time()
196

197 198 199
    def stopTest(self, test):
        self.time_dict[test.__class__.__module__] += \
          time.time() - self.start_time
200 201
        for patch in self._readable_tid:
            patch.revert()
202
        super(NeoTestRunner, self).stopTest(test)
203 204 205 206 207 208 209 210 211
        if self.stop_on_success is not None:
            count = self.getUnexpectedCount()
            if (count < self.testsRun - len(self.skipped)
                    if self.stop_on_success else count):
                raise StopOnSuccess

    def getUnexpectedCount(self):
        return (len(self.errors) + len(self.failures)
              + len(self.unexpectedSuccesses))
212

213
    def _buildSummary(self, add_status):
214
        unexpected_count = self.getUnexpectedCount()
215 216
        expected_count = len(self.expectedFailures)
        success = self.testsRun - unexpected_count - expected_count
217
        add_status('Directory', self.temp_directory)
218
        if self.testsRun:
Vincent Pelletier's avatar
Vincent Pelletier committed
219
            add_status('Status', '%.3f%%' % (success * 100.0 / self.testsRun))
220
        for var in os.environ:
221 222
            if var.startswith('NEO_TEST'):
                add_status(var, os.environ[var])
223
        # visual
224 225 226 227
        header       = "%25s |  run  | unexpected | expected | skipped |  time    \n" % 'Test Module'
        separator    = "%25s-+-------+------------+----------+---------+----------\n" % ('-' * 25)
        format       = "%25s |  %3s  |     %3s    |    %3s   |   %3s   | %6.2fs   \n"
        group_f      = "%25s |       |            |          |         |          \n"
228
        # header
229
        s = ' ' * 30 + ' NEO TESTS REPORT\n\n' + header + separator
230
        group = None
231 232 233 234 235 236 237 238 239 240
        unexpected = Counter(x[0].__class__.__module__
                             for x in (self.errors, self.failures)
                             for x in x)
        unexpected.update(x.__class__.__module__
                          for x in self.unexpectedSuccesses)
        expected = Counter(x[0].__class__.__module__
                           for x in self.expectedFailures)
        skipped = Counter(x[0].__class__.__module__
                          for x in self.skipped)
        total_time = 0
241
        # for each test case
242
        for k, v in sorted(self.run_dict.iteritems()):
243
            # display group below its content
244
            _group, name = k.rsplit('.', 1)
245
            if _group != group:
246 247
                if group:
                    s += separator + group_f % group + separator
248
                group = _group
249 250 251 252
            t = self.time_dict[k]
            total_time += t
            s += format % (name.lstrip('test'), v, unexpected.get(k, '.'),
                           expected.get(k, '.'), skipped.get(k, '.'), t)
253 254 255
        # the last group
        s += separator  + group_f % group + separator
        # the final summary
256 257 258 259
        s += format % ("Summary", self.testsRun, unexpected_count or '.',
                       expected_count or '.', len(self.skipped) or '.',
                       total_time) + separator + '\n'
        return "%s Tests, %s Failed" % (self.testsRun, unexpected_count), s
260

261
    def buildReport(self, add_status):
262
        subject, summary = self._buildSummary(add_status)
263 264
        if self.stop_on_success:
            return subject, summary
265 266 267 268 269
        body = StringIO()
        body.write(summary)
        for test in self.unexpectedSuccesses:
            body.write("UNEXPECTED SUCCESS: %s\n" % self.getDescription(test))
        self.stream = _WritelnDecorator(body)
270 271
        self.printErrorList('ERROR', self.errors)
        self.printErrorList('FAIL', self.failures)
272
        return subject, body.getvalue()
273 274 275 276

class TestRunner(BenchmarkRunner):

    def add_options(self, parser):
277
        parser.add_option('-c', '--coverage', action='store_true',
278
            help='Enable coverage')
279 280 281
        parser.add_option('-C', '--cov-unit', action='store_true',
            help='Same as -c but output 1 file per test,'
                 ' in the temporary test directory')
282 283 284
        parser.add_option('-L', '--log', action='store_true',
            help='Force all logs to be emitted immediately and keep'
                 ' packet body in logs of successful threaded tests')
285 286
        parser.add_option('-l', '--loop', type='int', default=1,
            help='Repeat tests several times')
287 288
        parser.add_option('-f', '--functional', action='store_true',
            help='Functional tests')
289 290 291 292 293 294 295 296
        parser.add_option('-s', '--stop-on-error', action='store_false',
            dest='stop_on_success',
            help='Continue as long as tests pass successfully.'
                 ' It is usually combined with --loop, to check that tests'
                 ' do not fail randomly.')
        parser.add_option('-S', '--stop-on-success', action='store_true',
            help='Opposite of --stop-on-error: stop as soon as a test'
                 ' passes. Details about errors are not printed at exit.')
297 298 299
        parser.add_option('-r', '--readable-tid', action='store_true',
            help='Change master behaviour to generate readable TIDs for easier'
                 ' debugging (rather than from current time).')
300 301 302 303
        parser.add_option('-u', '--unit', action='store_true',
            help='Unit & threaded tests')
        parser.add_option('-z', '--zodb', action='store_true',
            help='ZODB test suite running on a NEO')
304 305
        parser.add_option('-v', '--verbose', action='store_true',
            help='Verbose output')
306
        parser.usage += " [[!] module [test...]]"
307
        parser.format_epilog = lambda _: """
308 309 310 311
Positional:
  Filter by given module/test. These arguments are shell patterns.
  This implies -ufz if none of this option is passed.

312 313 314 315 316
Environment Variables:
  NEO_TESTS_ADAPTER           Default is SQLite for threaded clusters,
                              MySQL otherwise.

  MySQL specific:
317
    NEO_DB_SOCKET             default: libmysqlclient.so default
318 319 320 321 322 323 324 325 326 327 328 329 330
    NEO_DB_PREFIX             default: %(DB_PREFIX)s
    NEO_DB_ADMIN              default: %(DB_ADMIN)s
    NEO_DB_PASSWD             default: %(DB_PASSWD)s
    NEO_DB_USER               default: %(DB_USER)s

  ZODB tests:
    NEO_TEST_ZODB_FUNCTIONAL  Clusters are threaded by default. If true,
                              they are built like in functional tests.
    NEO_TEST_ZODB_MASTERS     default: 1
    NEO_TEST_ZODB_PARTITIONS  default: 1
    NEO_TEST_ZODB_REPLICAS    default: 0
    NEO_TEST_ZODB_STORAGES    default: 1
""" % neo_tests__dict__
331 332

    def load_options(self, options, args):
333 334
        if options.coverage and options.cov_unit:
            sys.exit('-c conflicts with -C')
335 336 337 338
        if not (options.unit or options.functional or options.zodb):
            if not args:
                sys.exit('Nothing to run, please give one of -f, -u, -z')
            options.unit = options.functional = options.zodb = True
339
        return dict(
340
            log = options.log,
341
            loop = options.loop,
342 343 344
            unit = options.unit,
            functional = options.functional,
            zodb = options.zodb,
345
            verbosity = 2 if options.verbose else 1,
346
            coverage = options.coverage,
347
            cov_unit = options.cov_unit,
348
            only = args,
349
            stop_on_success = options.stop_on_success,
350
            readable_tid = options.readable_tid,
351
        )
352

353 354
    def start(self):
        config = self._config
355 356
        logging.backlog(max_packet=1<<20,
            **({'max_size': None} if config.log else {}))
357
        only = config.only
358
        # run requested tests
359
        runner = NeoTestRunner(config.title or 'Neo', config.verbosity,
360
                               config.stop_on_success, config.readable_tid)
361 362 363 364 365 366 367 368 369 370 371 372 373 374 375
        if config.cov_unit:
            from coverage import Coverage
            cov_dir = runner.temp_directory + '/coverage'
            os.mkdir(cov_dir)
            @Patch(NeoTestBase)
            def setUp(orig, self):
                orig(self)
                self.__coverage = Coverage('%s/%s' % (cov_dir, self.id()))
                self.__coverage.start()
            @Patch(NeoTestBase)
            def _tearDown(orig, self, success):
                self.__coverage.stop()
                self.__coverage.save()
                del self.__coverage
                orig(self, success)
376
        try:
377 378 379 380 381 382 383
            for _ in xrange(config.loop):
                if config.unit:
                    runner.run('Unit tests', UNIT_TEST_MODULES, only)
                if config.functional:
                    runner.run('Functional tests', FUNC_TEST_MODULES, only)
                if config.zodb:
                    runner.run('ZODB tests', ZODB_TEST_MODULES, only)
384 385 386
        except KeyboardInterrupt:
            config['mail_to'] = None
            traceback.print_exc()
387 388
        except StopOnSuccess:
            pass
389 390
        if config.coverage:
            coverage.stop()
391 392
            if coverage.neotestrunner:
                coverage.combine(coverage.neotestrunner)
393
            coverage.save()
394 395
        if runner.dots:
            print
396
        # build report
397
        if (only or config.stop_on_success) and not config.mail_to:
398 399 400
            runner._buildSummary = lambda *args: (
                runner.__class__._buildSummary(runner, *args)[0], '')
            self.build_report = str
401 402
        self._successful = runner.wasSuccessful()
        return runner.buildReport(self.add_status)
403

404
def main(args=None):
405 406
    from neo.storage.database.manager import DatabaseManager
    DatabaseManager.UNSAFE = True
407 408
    runner = TestRunner()
    runner.run()
Julien Muchembled's avatar
Julien Muchembled committed
409
    return sys.exit(not runner.was_successful())
410 411 412

if __name__ == "__main__":
    main()