racetest: Review 2

- Rename `TestGroup` -> `TestWorkGroup`. I originally named it as just
  WorkGroup similarly to sync.WorkGroup in pygolang, but later added
  "Test" prefix to highlight the difference that this class manages
  working group of threads that take part in tests, instead of group of
  arbitrary threads. However in that process I dropped the "Work" part
  which turned the name to be somewhat ambiguous: it is not clear whether
  "TestGroup" is a group of threads serving one test, or a group of
  separate tests without any relation to threading. To remove the
  ambiguity let's restore the "Work" in the name so that both "Test" and
  "WorkGroup" are there.

- Review test_racetest.py a bit:

  * make it a bit more robust by increasing "ok" timeout from 0.1s to 10s.
    in my experience 0.1s is too little and will regularly hit "timeout"
    when CI machines are overloaded. I'm still somewhat uncomfortable
    with 0.1s timeout we left in tests that exercise timeout handling,
    but let's leave it as is for now and see how it goes.

  * add test for cases when only one thread fails out of many, and when
    several threads fail too.

  * minor cosmetics.

- spellcheck.

src/ZODB/tests/racetest.py

@@ -81,7 +81,7 @@ class T2ObjectsInc:
     """T2ObjectsInc is specification with behaviour where two objects obj1
     and obj2 are incremented synchronously.
 
-    It is used in tests where bugs can be immedeately observed after the race.
+    It is used in tests where bugs can be immediately observed after the race.
 
     invariant:  obj1 == obj2
     """
@@ -207,7 +207,7 @@ class RaceTests(object):
         init()
 
         N = 500
-        tg = TestGroup(self)
+        tg = TestWorkGroup(self)
         tg.go(xrun, verify, N, name='Tverify')
         tg.go(xrun, modify, N, name='Tmodify')
         tg.wait(60)
@@ -317,7 +317,7 @@ class RaceTests(object):
         init()
 
         N = 100
-        tg = TestGroup(self)
+        tg = TestWorkGroup(self)
         for _ in range(nwork):
             tg.go(T, N)
         tg.wait(60)
@@ -325,9 +325,9 @@ class RaceTests(object):
     # verify storage for race in between client disconnect and external
     # invalidations. https://github.com/zopefoundation/ZEO/issues/209
     #
-    # This test is simlar to check_race_load_vs_external_invalidate, but
+    # This test is similar to check_race_load_vs_external_invalidate, but
     # increases the number of workers and also makes every worker to repeatedly
-    # reconnect to the storage, so that the probability of disconection is
+    # reconnect to the storage, so that the probability of disconnection is
     # high. It also uses T2ObjectsInc2Phase instead of T2ObjectsInc because if
     # an invalidation is skipped due to the disconnect/invalidation race,
     # T2ObjectsInc won't catch the bug as both objects will be either in old
@@ -408,7 +408,7 @@ class RaceTests(object):
         init()
 
         N = 100 // (2*4)  # N reduced to save time
-        tg = TestGroup(self)
+        tg = TestWorkGroup(self)
         for _ in range(nwork):
             tg.go(T, N)
         tg.wait(60)
@@ -425,7 +425,7 @@ def _state_init(db, spec):
     zconn.close()
 
 
-# `_state_invalidate_half1` invalidatates first 50% of database objects, so
+# `_state_invalidate_half1` invalidates first 50% of database objects, so
 # that the next time they are accessed, they are reloaded from the storage.
 def _state_invalidate_half1(root):
     keys = list(sorted(root.keys()))
@@ -483,9 +483,9 @@ def _state_details(root):  # -> txt
     return txt
 
 
-class TestGroup(object):
-    """TestGroup represents group of threads that run together to verify
-       somthing.
+class TestWorkGroup(object):
+    """TestWorkGroup represents group of threads that run together to verify
+       something.
 
        - .go() adds test thread to the group.
        - .wait() waits for all spawned threads to finish and reports all
@@ -498,7 +498,7 @@ class TestGroup(object):
         self.testcase = testcase
         self.failed_event = threading.Event()
         self.fail_mu = threading.Lock()
-        self.failv = []           # failures registerd by .fail
+        self.failv = []           # failures registered by .fail
         self.threadv = []         # spawned threads
         self.waitg = WaitGroup()  # to wait for spawned threads
 
@@ -509,7 +509,7 @@ class TestGroup(object):
         self.failed_event.set()
 
     def failed(self):
-        """did the thest already fail."""
+        """did the test already fail."""
         return self.failed_event.is_set()
 
     def go(self, f, *argv, **kw):

src/ZODB/tests/test_racetest.py

@@ -14,14 +14,14 @@
 from time import sleep
 from unittest import TestCase
 
-from .racetest import TestGroup
+from .racetest import TestWorkGroup
 
 
-class TestGroupTests(TestCase):
+class TestWorkGroupTests(TestCase):
     def setUp(self):
         self._failed = failed = []
         case_mockup = SimpleNamespace(fail=failed.append)
-        self.tg = TestGroup(case_mockup)
+        self.tg = TestWorkGroup(case_mockup)
 
     @property
     def failed(self):
@@ -30,19 +30,42 @@ class TestGroupTests(TestCase):
     def test_success(self):
         tg = self.tg
         tg.go(tg_test_function)
-        tg.wait(0.1)
+        tg.wait(10)
         self.assertEqual(self.failed, "")
 
-    def test_failure(self):
+    def test_failure1(self):
         tg = self.tg
         tg.go(tg_test_function, T_FAIL)
-        tg.wait(0.1)
-        self.assertEqual(self.failed, "0 failed")
+        tg.wait(10)
+        self.assertEqual(self.failed, "T0 failed")
+
+    def test_failure1_okmany(self):
+        tg = self.tg
+        tg.go(tg_test_function, T_SUCCESS)
+        tg.go(tg_test_function, T_SUCCESS)
+        tg.go(tg_test_function, T_SUCCESS)
+        tg.go(tg_test_function, T_FAIL)
+        tg.wait(10)
+        self.assertEqual(self.failed, "T3 failed")
+
+    def test_failure_many(self):
+        tg = self.tg
+        tg.go(tg_test_function, T_FAIL)
+        tg.go(tg_test_function, T_SUCCESS)
+        tg.go(tg_test_function, T_FAIL)
+        tg.go(tg_test_function, T_SUCCESS)
+        tg.go(tg_test_function, T_FAIL)
+        tg.wait(10)
+        self.assertIn("T0 failed", self.failed)
+        self.assertIn("T2 failed", self.failed)
+        self.assertIn("T4 failed", self.failed)
+        self.assertNotIn("T1 failed", self.failed)
+        self.assertNotIn("T3 failed", self.failed)
 
     def test_exception(self):
         tg = self.tg
         tg.go(tg_test_function, T_EXC)
-        tg.wait(0.1)
+        tg.wait(10)
         self.assertIn("Unhandled exception", self.failed)
         self.assertIn("in thread T0", self.failed)
 
@@ -74,10 +97,11 @@ def tg_test_function(tg, tx, mode=T_SUCCESS, waits=1, wait_time=0.2):
     if mode == T_SUCCESS:
         return
     if mode == T_FAIL:
-        tg.fail("%d failed" % tx)
+        tg.fail("T%d failed" % tx)
         return
     if mode == T_EXC:
         raise ValueError(str(tx))
+    assert mode == T_SLOW
     while waits:
         waits -= 1
         if tg.failed():