Add timeout for test runner

typon/trans/test_runner.py

@@ -42,6 +42,7 @@ class TestStatus(enum.Enum):
     RUNTIME_ERROR = 3  # returned from ALT_RUNNER
     SKIPPED = 4
     PYTHON_ERROR = 5
+    TIMEOUT = 6
 
     def ascii(self):
         color, msg = {
@@ -51,13 +52,15 @@ class TestStatus(enum.Enum):
             TestStatus.RUNTIME_ERROR: (cf.red, "Run"),
             TestStatus.SKIPPED: (cf.yellow, "Skipped"),
             TestStatus.PYTHON_ERROR: (cf.red, "Python"),
+            TestStatus.TIMEOUT: (cf.red, "Timeout"),
         }[self]
         return color("{:^7s}".format(msg))
 
-def exec_cmd(cmd, quiet):
-    stdout = subprocess.DEVNULL if quiet else sys.stdout
-    stderr = subprocess.DEVNULL if quiet else sys.stderr
-    return subprocess.run(cmd, shell=True, stdout=stdout, stderr=stderr).returncode
+def exec_cmd(cmd):
+    try:
+        return subprocess.run(cmd, shell=True, stdout=sys.stdout, stderr=sys.stderr, timeout=200).returncode
+    except subprocess.TimeoutExpired:
+        return TestStatus.TIMEOUT
 
 def run_test(path, quiet=True):
     if quiet:
@@ -79,7 +82,6 @@ def run_test(path, quiet=True):
             if not quiet:
                 raise
             return TestStatus.TYPON_ERROR
-    #print(res)
     name_cpp = path.with_suffix('.cpp')
     with open(name_cpp, "w", encoding="utf-8") as fcpp:
         fcpp.write(res)
@@ -88,10 +90,10 @@ def run_test(path, quiet=True):
         return TestStatus.SUCCESS
     execute_str = "true" if (execute and not args.generate) else "false"
     name_bin = path.with_suffix('').as_posix()
-    if exec_cmd(f'bash -c "export PYTHONPATH=stdlib; if {execute_str}; then python3 ./{path.as_posix()}; fi"', quiet) != 0:
+    if exec_cmd(f'bash -c "export PYTHONPATH=stdlib; if {execute_str}; then python3 ./{path.as_posix()}; fi"') != 0:
         return TestStatus.PYTHON_ERROR
     if compile and (alt := environ.get("ALT_RUNNER")):
-        if (code := exec_cmd(alt.format(name_bin=name_bin, name_cpp_posix=name_cpp.as_posix(), run_file=execute_str), quiet)) != 0:
+        if (code := exec_cmd(alt.format(name_bin=name_bin, name_cpp_posix=name_cpp.as_posix(), run_file=execute_str))) != 0:
             return TestStatus(code)
     else:
         print("no ALT_RUNNER")
@@ -118,10 +120,14 @@ def run_tests():
         print("Running", len(tests), "tests as", pool._max_workers, "workers")
         start = datetime.now()
         with pool as executor:
-            futures = [executor.submit(runner, path) for path in tests]
-            for future in concurrent.futures.as_completed(futures):
-                path, status, duration = future.result()
-                print(f"[{status.ascii()}] ({duration.total_seconds():2.2f}s) {path.name}")
+            futures = {path: executor.submit(runner, path) for path in tests}
+            try:
+                for future in concurrent.futures.as_completed(futures.values(), 120):
+                    path, status, duration = future.result()
+                    print(f"[{status.ascii()}] ({duration.total_seconds():2.2f}s) {path.name}")
+            except TimeoutError:
+                new_futures = {p: f for p, f in futures.items() if not f.done()}
+                print("Timeout,", len(new_futures), "left:", new_futures.keys())
         duration = datetime.now() - start
         print("Done in", duration.total_seconds(), "seconds")