Commit 2100d87c authored by Stefan Behnel's avatar Stefan Behnel

fix KeyboardInterrupt handling when running cythonize() in parallel

parent d5af12c4
...@@ -798,15 +798,24 @@ def cythonize(module_list, exclude=[], nthreads=0, aliases=None, quiet=False, fo ...@@ -798,15 +798,24 @@ def cythonize(module_list, exclude=[], nthreads=0, aliases=None, quiet=False, fo
# Requires multiprocessing (or Python >= 2.6) # Requires multiprocessing (or Python >= 2.6)
try: try:
import multiprocessing import multiprocessing
pool = multiprocessing.Pool(nthreads) pool = multiprocessing.Pool(
nthreads, initializer=_init_multiprocessing_helper)
except (ImportError, OSError): except (ImportError, OSError):
print("multiprocessing required for parallel cythonization") print("multiprocessing required for parallel cythonization")
nthreads = 0 nthreads = 0
else: else:
# This is a bit more involved than it should be, because KeyboardInterrupts
# break the multiprocessing workers when using a normal pool.map().
# See, for example:
# http://noswap.com/blog/python-multiprocessing-keyboardinterrupt
try: try:
pool.map(cythonize_one_helper, to_compile) result = pool.map_async(cythonize_one_helper, to_compile, chunksize=1)
finally:
pool.close() pool.close()
while not result.ready():
result.get(10) # seconds
except KeyboardInterrupt:
pool.terminate()
pool.join()
if not nthreads: if not nthreads:
for args in to_compile: for args in to_compile:
cythonize_one(*args[1:]) cythonize_one(*args[1:])
...@@ -942,6 +951,7 @@ def cythonize_one(pyx_file, c_file, fingerprint, quiet, options=None, raise_on_f ...@@ -942,6 +951,7 @@ def cythonize_one(pyx_file, c_file, fingerprint, quiet, options=None, raise_on_f
finally: finally:
f.close() f.close()
def cythonize_one_helper(m): def cythonize_one_helper(m):
import traceback import traceback
try: try:
...@@ -950,6 +960,13 @@ def cythonize_one_helper(m): ...@@ -950,6 +960,13 @@ def cythonize_one_helper(m):
traceback.print_exc() traceback.print_exc()
raise raise
def _init_multiprocessing_helper():
# KeyboardInterrupt kills workers, so don't let them get it
import signal
signal.signal(signal.SIGINT, signal.SIG_IGN)
def cleanup_cache(cache, target_size, ratio=.85): def cleanup_cache(cache, target_size, ratio=.85):
try: try:
p = subprocess.Popen(['du', '-s', '-k', os.path.abspath(cache)], stdout=subprocess.PIPE) p = subprocess.Popen(['du', '-s', '-k', os.path.abspath(cache)], stdout=subprocess.PIPE)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment