Skip to content

pygolang
pygolang
Commit 4d64fd0f authored by Kirill Smelkov's avatar Kirill Smelkov
Browse files
Options

X update (sync with master + ustr.translate fixes)

parent ac87a2ed
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 794 additions and 171 deletions
.gitmodules
View file @ 4d64fd0f
[submodule "3rdparty/ratas"]
path = 3rdparty/ratas
url = https://github.com/jsnell/ratas.git
[submodule "3rdparty/funchook"]
path = 3rdparty/funchook
url = https://github.com/kubo/funchook.git
......
.lsan-ignore.txt 0 → 100644
View file @ 4d64fd0f
# .lsan-ignore.txt lists memory leak events that LeakSanitizer should not
# report when running pygolang tests.
#
# Many python allocations, whose lifetime coincides with python interpreter
# lifetime, and which are not explicitly freed on python shutdown, are
# reported as leaks by default. Disable leak reporting for those to avoid
# non-pygolang related printouts.
# >>> Everything created when initializing python, e.g. sys.stderr
# #0 0x7f21e74f3bd7 in malloc .../asan_malloc_linux.cpp:69
# #1 0x555f361ff9a4 in PyThread_allocate_lock Python/thread_pthread.h:385
# #2 0x555f3623f72a in _buffered_init Modules/_io/bufferedio.c:725
# #3 0x555f3623ff7e in _io_BufferedWriter___init___impl Modules/_io/bufferedio.c:1803
# #4 0x555f3623ff7e in _io_BufferedWriter___init__ Modules/_io/clinic/bufferedio.c.h:489
# #5 0x555f3610c086 in type_call Objects/typeobject.c:1103
# #6 0x555f3609cdcc in _PyObject_MakeTpCall Objects/call.c:214
# #7 0x555f3609d6a8 in _PyObject_VectorcallTstate Include/internal/pycore_call.h:90
# #8 0x555f3609d6a8 in _PyObject_VectorcallTstate Include/internal/pycore_call.h:77
# #9 0x555f3609d6a8 in _PyObject_CallFunctionVa Objects/call.c:536
# #10 0x555f3609e89c in _PyObject_CallFunction_SizeT Objects/call.c:590
# #11 0x555f3623a0df in _io_open_impl Modules/_io/_iomodule.c:407
# #12 0x555f3623a0df in _io_open Modules/_io/clinic/_iomodule.c.h:264
# #13 0x555f360f17da in cfunction_vectorcall_FASTCALL_KEYWORDS Objects/methodobject.c:443
# #14 0x555f3609d54c in _PyObject_VectorcallTstate Include/internal/pycore_call.h:92
# #15 0x555f3609d54c in _PyObject_CallFunctionVa Objects/call.c:536
# #16 0x555f3609ec34 in callmethod Objects/call.c:608
# #17 0x555f3609ec34 in _PyObject_CallMethod Objects/call.c:677
# #18 0x555f361e60cf in create_stdio Python/pylifecycle.c:2244
# #19 0x555f361e6523 in init_sys_streams Python/pylifecycle.c:2431
# #20 0x555f361e6523 in init_interp_main Python/pylifecycle.c:1154
# #21 0x555f361e7204 in pyinit_main Python/pylifecycle.c:1230
# #22 0x555f361e85ba in Py_InitializeFromConfig Python/pylifecycle.c:1261
# #23 0x555f3621010a in pymain_init Modules/main.c:67
# #24 0x555f362113de in pymain_main Modules/main.c:701
# #25 0x555f362113de in Py_BytesMain Modules/main.c:734
leak:^pymain_init$
# >>> Everything created when importing py modules, e.g.
# #0 0x7f18c86f3bd7 in malloc .../asan_malloc_linux.cpp:69
# #1 0x55b971430acf in PyMem_RawMalloc Objects/obmalloc.c:586
# #2 0x55b971430acf in _PyObject_Malloc Objects/obmalloc.c:2003
# #3 0x55b971430acf in _PyObject_Malloc Objects/obmalloc.c:1996
# #4 0x55b971415696 in new_keys_object Objects/dictobject.c:632
# #5 0x55b971415716 in dictresize Objects/dictobject.c:1429
# #6 0x55b97141961a in insertion_resize Objects/dictobject.c:1183
# #7 0x55b97141961a in insertdict Objects/dictobject.c:1248
# #8 0x55b97143eb7b in add_subclass Objects/typeobject.c:6547
# #9 0x55b97144ca52 in type_ready_add_subclasses Objects/typeobject.c:6345
# #10 0x55b97144ca52 in type_ready Objects/typeobject.c:6476
# #11 0x55b971451a1f in PyType_Ready Objects/typeobject.c:6508
# #12 0x55b971451a1f in type_new_impl Objects/typeobject.c:3189
# #13 0x55b971451a1f in type_new Objects/typeobject.c:3323
# #14 0x55b971443014 in type_call Objects/typeobject.c:1091
# #15 0x55b9713d3dcc in _PyObject_MakeTpCall Objects/call.c:214
# #16 0x55b9713d47bd in _PyObject_FastCallDictTstate Objects/call.c:141
# #17 0x55b9713d47bd in PyObject_VectorcallDict Objects/call.c:165
# #18 0x55b9714d14c2 in builtin___build_class__ Python/bltinmodule.c:209
# #19 0x55b9714287da in cfunction_vectorcall_FASTCALL_KEYWORDS Objects/methodobject.c:443
# #20 0x55b9713d4a7b in _PyObject_VectorcallTstate Include/internal/pycore_call.h:92
# #21 0x55b9713d4a7b in PyObject_Vectorcall Objects/call.c:299
# #22 0x55b97137666e in _PyEval_EvalFrameDefault Python/ceval.c:4769
# #23 0x55b9714d7e6b in _PyEval_EvalFrame Include/internal/pycore_ceval.h:73
# #24 0x55b9714d7e6b in _PyEval_Vector Python/ceval.c:6434
# #25 0x55b9714d7e6b in PyEval_EvalCode Python/ceval.c:1148
# #26 0x55b9714d2e1f in builtin_exec_impl Python/bltinmodule.c:1077
# #27 0x55b9714d2e1f in builtin_exec Python/clinic/bltinmodule.c.h:465
# #28 0x55b9714287da in cfunction_vectorcall_FASTCALL_KEYWORDS Objects/methodobject.c:443
# #29 0x55b971376dcb in do_call_core Python/ceval.c:7349
# #30 0x55b971376dcb in _PyEval_EvalFrameDefault Python/ceval.c:5376
# #31 0x55b9714d7faf in _PyEval_EvalFrame Include/internal/pycore_ceval.h:73
# #32 0x55b9714d7faf in _PyEval_Vector Python/ceval.c:6434
# #33 0x55b9713d436e in _PyObject_VectorcallTstate Include/internal/pycore_call.h:92
# #34 0x55b9713d436e in object_vacall Objects/call.c:819
# #35 0x55b9713d63cf in PyObject_CallMethodObjArgs Objects/call.c:879
# #36 0x55b9715080e1 in import_find_and_load Python/import.c:1748
# #37 0x55b9715080e1 in PyImport_ImportModuleLevelObject Python/import.c:1847
# #38 0x55b97137de9c in import_name Python/ceval.c:7422
# #39 0x55b97137de9c in _PyEval_EvalFrameDefault Python/ceval.c:3946
# #40 0x55b9714d7e6b in _PyEval_EvalFrame Include/internal/pycore_ceval.h:73
# #41 0x55b9714d7e6b in _PyEval_Vector Python/ceval.c:6434
# #42 0x55b9714d7e6b in PyEval_EvalCode Python/ceval.c:1148
# #43 0x55b9714d2e1f in builtin_exec_impl Python/bltinmodule.c:1077
# #44 0x55b9714d2e1f in builtin_exec Python/clinic/bltinmodule.c.h:465
# #45 0x55b9714287da in cfunction_vectorcall_FASTCALL_KEYWORDS Objects/methodobject.c:443
# #46 0x55b971376dcb in do_call_core Python/ceval.c:7349
# #47 0x55b971376dcb in _PyEval_EvalFrameDefault Python/ceval.c:5376
leak:^PyImport_Import
# importlib.import_module leads to
# #0 0x7f1951ef3bd7 in malloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:69
# #1 0x55f399e8cacf in PyMem_RawMalloc Objects/obmalloc.c:586
# #2 0x55f399e8cacf in _PyObject_Malloc Objects/obmalloc.c:2003
# #3 0x55f399e8cacf in _PyObject_Malloc Objects/obmalloc.c:1996
# #4 0x55f399e86344 in PyModule_ExecDef Objects/moduleobject.c:400
# #5 0x55f399f6178a in exec_builtin_or_dynamic Python/import.c:2345
# #6 0x55f399f6178a in _imp_exec_dynamic_impl Python/import.c:2419
# #7 0x55f399f6178a in _imp_exec_dynamic Python/clinic/import.c.h:474
# #8 0x55f399e8438a in cfunction_vectorcall_O Objects/methodobject.c:514
leak:^_imp_exec_dynamic
# >>> Everything allocated at DSO initialization, e.g.
# #0 0x7f35d2af46c8 in operator new(unsigned long) .../asan_new_delete.cpp:95
# #1 0x7f35ce897e9f in __static_initialization_and_destruction_0 golang/context.cpp:61
# #2 0x7f35ce8982ef in _GLOBAL__sub_I_context.cpp golang/context.cpp:380
# #3 0x7f35d32838bd in call_init elf/dl-init.c:90
# #4 0x7f35d32838bd in call_init elf/dl-init.c:27
# #5 0x7f35d32839a3 in _dl_init elf/dl-init.c:137
# #6 0x7f35d256e023 in __GI__dl_catch_exception elf/dl-error-skeleton.c:182
# #7 0x7f35d328a09d in dl_open_worker elf/dl-open.c:808
# #8 0x7f35d256dfc9 in __GI__dl_catch_exception elf/dl-error-skeleton.c:208
# #9 0x7f35d328a437 in _dl_open elf/dl-open.c:884
# #10 0x7f35d24a4437 in dlopen_doit dlfcn/dlopen.c:56
# #11 0x7f35d256dfc9 in __GI__dl_catch_exception elf/dl-error-skeleton.c:208
# #12 0x7f35d256e07e in __GI__dl_catch_error elf/dl-error-skeleton.c:227
# #13 0x7f35d24a3f26 in _dlerror_run dlfcn/dlerror.c:138
# #14 0x7f35d24a44e8 in dlopen_implementation dlfcn/dlopen.c:71
# #15 0x7f35d24a44e8 in ___dlopen dlfcn/dlopen.c:81
# #16 0x7f35d2a77ff9 in dlopen .../sanitizer_common_interceptors.inc:6341
leak:^_GLOBAL_
# global<> does not deallocate its reference on purpose
leak:^_test_global()$
ratas @ becd5fc5
Subproject commit becd5fc5c1e9ea600cd8b3b1c24d564794fedac4
MANIFEST.in
View file @ 4d64fd0f
include COPYING README.rst CHANGELOG.rst tox.ini pyproject.toml trun .nxdtest
include COPYING README.rst CHANGELOG.rst tox.ini pyproject.toml trun .lsan-ignore.txt .nxdtest conftest.py
include golang/libgolang.h
include golang/runtime/libgolang.cpp
include golang/runtime/libpyxruntime.cpp
......
conftest.py
View file @ 4d64fd0f
# pygolang | pytest config
# Copyright (C) 2021-2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
from __future__ import print_function, absolute_import
import gc
# Do full GC before pytest exits, to avoid false positives in the leak detector.
def pytest_unconfigure():
gc.collect()
# ignore tests in distorm - else it breaks as e.g.
#
# 3rdparty/funchook/distorm/python/test_distorm3.py:15: in <module>
......
golang/_golang.pyx
View file @ 4d64fd0f
......@@ -173,11 +173,18 @@ cdef void __goviac(void *arg) nogil:
# ---- channels ----
# _frompyx indicates that a constructor is called from pyx code
cdef object _frompyx = object()
@final
cdef class pychan:
def __cinit__(pychan pych, size=0, dtype=object):
pych.dtype = parse_dtype(dtype)
pych._ch = _makechan_pyexc(dtypeRegistry[<int>pych.dtype].size, size)
if dtype is _frompyx:
pych.dtype = DTYPE_STRUCTZ # anything
pych._ch = NULL
else:
pych.dtype = parse_dtype(dtype)
pych._ch = _makechan_pyexc(dtypeRegistry[<int>pych.dtype].size, size)
# pychan.nil(X) creates new nil pychan with specified dtype.
# TODO try to avoid exposing .nil on pychan instances, and expose only pychan.nil
......@@ -370,7 +377,7 @@ cdef void pychan_asserttype(pychan pych, DType dtype) nogil:
panic("pychan: channel type mismatch")
cdef pychan pychan_from_raw(_chan *_ch, DType dtype):
cdef pychan pych = pychan.__new__(pychan)
cdef pychan pych = pychan.__new__(pychan, dtype=_frompyx)
pych.dtype = dtype
pych._ch = _ch; _chanxincref(_ch)
return pych
......@@ -626,9 +633,7 @@ cdef object c_to_py(DType dtype, const chanElemBuf *cfrom):
# mkpynil creates pychan instance that represents nil[dtype].
cdef PyObject *mkpynil(DType dtype):
cdef pychan pynil = pychan.__new__(pychan)
pynil.dtype = dtype
pynil._ch = NULL # should be already NULL
cdef pychan pynil = pychan_from_raw(NULL, dtype)
Py_INCREF(pynil)
return <PyObject *>pynil
......@@ -818,9 +823,6 @@ from libcpp.typeinfo cimport type_info
from cython.operator cimport typeid
from libc.string cimport strcmp
# _frompyx indicates that a constructor is called from pyx code
cdef object _frompyx = object()
cdef class pyerror(Exception):
# pyerror <- error
@staticmethod
......
golang/_golang_str.pyx
View file @ 4d64fd0f
......@@ -580,6 +580,7 @@ cdef class _pybstr(bytes): # https://github.com/cython/cython/issues/711
def title(self): return pyb(pyu(self).title())
def translate(self, table, delete=None):
# bytes mode (compatibility with str/py2)
# XXX isinstance(zbytes) -> isinstance(bytes) ?
if table is None or isinstance(table, zbytes) or delete is not None:
if delete is None: delete = b''
return pyb(zbytes.translate(self, table, delete))
......@@ -905,12 +906,7 @@ cdef class _pyustr(unicode):
def translate(self, table):
# unicode.translate does not accept bstr values
t = {}
for k,v in table.items():
if not isinstance(v, int): # either unicode ordinal,
v = _xpyu_coerce(v) # character or None
t[k] = v
return pyu(zunicode.translate(self, t))
return pyu(zunicode.translate(self, _pyustrTranslateTab(table)))
def upper(self): return pyu(zunicode.upper(self))
def zfill(self, width): return pyu(zunicode.zfill(self, width))
......@@ -983,6 +979,18 @@ cdef class _pyustrIter:
x = next(self.uiter)
return pyu(x)
# _pyustrTranslateTab wraps table for .translate to return bstr as unicode
# because unicode.translate does not accept bstr values.
cdef class _pyustrTranslateTab:
cdef object tab
def __init__(self, tab):
self.tab = tab
def __getitem__(self, k):
v = self.tab[k]
if not isinstance(v, int): # either unicode ordinal,
v = _xpyu_coerce(v) # character or None
return v
# _bdata/_udata retrieve raw data from bytes/unicode.
def _bdata(obj): # -> bytes
......
golang/_golang_test.pyx
View file @ 4d64fd0f
......@@ -2,7 +2,7 @@
# cython: language_level=2
# distutils: language=c++
#
# Copyright (C) 2018-2020 Nexedi SA and Contributors.
# Copyright (C) 2018-2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -344,6 +344,25 @@ cdef nogil:
pych.chan_double().close()
# verify that pychan_from_raw is not leaking C channel.
def test_pychan_from_raw_noleak():
# pychan_from_raw used to create another channel and leak it
#
# this test _implicitly_ verifies that it is no longer the case - if it is,
# LSAN will report a memory leak after running the test.
#
# TODO consider adding explicit verification effective even under regular
# builds. Possible options:
#
# * verify malloc totals before and after tested code
# see e.g. https://stackoverflow.com/q/1761125/9456786
# * hook _makechan and verify that it is not invoked from under
# pychan_from_raw. Depends on funchook integration.
cdef chan[int] ch = makechan[int]()
cdef pychan pych = pychan.from_chan_int(ch) # uses pychan_from_raw internally
# pych and ch are freed automatically
# ---- benchmarks ----
# bench_go_nogil mirrors golang_test.py:bench_go
......
golang/golang_str_test.py
View file @ 4d64fd0f
......@@ -1567,7 +1567,10 @@ def test_strings_methods():
# checkop verifies that `s.meth(*argv, **kw)` gives the same result for s,
# argv and kw being various combinations of unicode,bstr,ustr, bytes/bytearray.
def checkop(s, meth, *argv, **kw):
assert type(s) is str
if six.PY3:
assert type(s) is str
else:
assert type(s) in (str, unicode) # some tests use unicode because \u does not work in str literals
ok = kw.pop('ok')
if six.PY2:
ok = deepReplaceStr(ok, xunicode)
......@@ -1738,7 +1741,7 @@ def test_strings_methods():
_("123").isnumeric( ok=True)
_("0x123").isnumeric( ok=False)
_("мир").isprintable( ok=True, optional=True) # py3.0
_("\u2009").isspace( ok=x32(True,False)) # thin space
_(u"\u2009").isspace( ok=True) # thin space
_(" ").isspace( ok=True)
_("мир").isspace( ok=False)
_("мир").istitle( ok=False)
......@@ -1748,8 +1751,8 @@ def test_strings_methods():
_("мир").ljust(10, ok="мир ")
_("мир").ljust(10, 'ж', ok="миржжжжжжж")
_("МиР").lower( ok="мир")
_("\u2009 мир").lstrip( ok=x32("мир", "\u2009 мир"))
_("\u2009 мир\u2009 ").lstrip( ok=x32("мир\u2009 ", "\u2009 мир\u2009 "))
_(u"\u2009 мир").lstrip( ok="мир")
_(u"\u2009 мир\u2009 ").lstrip( ok=u"мир\u2009 ")
_("мммир").lstrip('ми', ok="р")
_("миру мир").partition('ру', ok=("ми", "ру", " мир"))
_("миру мир").partition('ж', ok=("миру мир", "", ""))
......@@ -1764,15 +1767,15 @@ def test_strings_methods():
_("миру мир").rpartition('ж', ok=("", "", "миру мир"))
_("мир").rsplit( ok=["мир"])
_("привет мир").rsplit( ok=["привет", "мир"])
_("привет\u2009мир").rsplit( ok=x32(["привет", "мир"], ["привет\u2009мир"]))
_(u"привет\u2009мир").rsplit( ok=["привет", "мир"])
_("привет мир").rsplit("и", ok=["пр", "вет м", "р"])
_("привет мир").rsplit("и", 1, ok=["привет м", "р"])
_("мир \u2009").rstrip( ok=x32("мир", "мир \u2009"))
_(" мир \u2009").rstrip( ok=x32(" мир", " мир \u2009"))
_(u"мир \u2009").rstrip( ok="мир")
_(u" мир \u2009").rstrip( ok=" мир")
_("мируу").rstrip('ру', ok="ми")
_("мир").split( ok=["мир"])
_("привет мир").split( ok=["привет", "мир"])
_("привет\u2009мир").split( ok=x32(['привет', 'мир'], ["привет\u2009мир"]))
_(u"привет\u2009мир").split( ok=['привет', 'мир'])
_("привет мир").split("и", ok=["пр", "вет м", "р"])
_("привет мир").split("и", 1, ok=["пр", "вет мир"])
_("мир").splitlines( ok=["мир"])
......@@ -1782,11 +1785,12 @@ def test_strings_methods():
_("мир\nтруд\nмай\n").splitlines( ok=["мир", "труд", "май"])
_("мир\nтруд\nмай\n").splitlines(True, ok=["мир\n", "труд\n", "май\n"])
# startswith - tested in test_strings_index
_("\u2009 мир \u2009").strip( ok=x32("мир", "\u2009 мир \u2009"))
_(u"\u2009 мир \u2009").strip( ok="мир")
_("миру мир").strip('мир', ok="у ")
_("МиР").swapcase( ok="мИр")
_("МиР").title( ok="Мир")
_("мир").translate({ord(u'м'):ord(u'и'), ord(u'и'):'я', ord(u'р'):None}, ok="ия")
_(u"\u0000\u0001\u0002.").translate([u'м', ord(u'и'), None], ok="ми.")
_("МиР").upper( ok="МИР")
_("мир").zfill(10, ok="0000000мир")
_("123").zfill(10, ok="0000000123")
......
golang/golang_test.py
View file @ 4d64fd0f
# -*- coding: utf-8 -*-
# Copyright (C) 2018-2023 Nexedi SA and Contributors.
# Copyright (C) 2018-2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -74,7 +74,8 @@ import_pyx_tests("golang._golang_test")
# leaked goroutine behaviour check: done in separate process because we need
# to test process termination exit there.
def test_go_leaked():
pyrun([dir_testprog + "/golang_test_goleaked.py"])
pyrun([dir_testprog + "/golang_test_goleaked.py"],
lsan=False) # there are on-purpose leaks in this test
# benchmark go+join a thread/coroutine.
# pyx/nogil mirror is in _golang_test.pyx
......@@ -1756,6 +1757,11 @@ def _pyrun(argv, stdin=None, stdout=None, stderr=None, **kw): # -> retcode, st
assert len(enc) == 1
env['PYTHONIOENCODING'] = enc.pop()
# disable LeakSanitizer if requested, e.g. when test is known to leak something on purpose
lsan = kw.pop('lsan', True)
if not lsan:
env['ASAN_OPTIONS'] = env.get('ASAN_OPTIONS', '') + ',detect_leaks=0'
p = Popen(argv, stdin=(PIPE if stdin else None), stdout=stdout, stderr=stderr, env=env, **kw)
stdout, stderr = p.communicate(stdin)
......
golang/libgolang.h
View file @ 4d64fd0f
......@@ -345,8 +345,13 @@ typedef struct _libgolang_runtime_ops {
// previously successfully allocated via sema_alloc.
void (*sema_free) (_libgolang_sema*);
// sema_acquire/sema_release should acquire/release live semaphore allocated via sema_alloc.
void (*sema_acquire)(_libgolang_sema*);
// sema_acquire should try to acquire live semaphore allocated via sema_alloc during given time.
// it returns whether acquisition succeeded or timed out.
// the timeout is specified in nanoseconds.
// UINT64_MAX means no timeout.
bool (*sema_acquire)(_libgolang_sema*, uint64_t timeout_ns);
// sema_release should release live semaphore allocated via sema_alloc.
void (*sema_release)(_libgolang_sema*);
// nanosleep should pause current goroutine for at least dt nanoseconds.
......
golang/pyx/build_test.py
View file @ 4d64fd0f
# -*- coding: utf-8 -*-
# Copyright (C) 2019 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
# Copyright (C) 2019-2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
......@@ -28,7 +28,8 @@ testprog = dirname(__file__) + "/testprog"
# verify that we can build/run external package that uses pygolang in pyx mode.
def test_pyx_build():
pyxuser = testprog + "/golang_pyx_user"
pyrun(["setup.py", "build_ext", "-i"], cwd=pyxuser)
pyrun(["setup.py", "build_ext", "-i"], cwd=pyxuser,
lsan=False) # gcc leaks
# run built test.
_ = pyout(["-c",
......@@ -44,8 +45,8 @@ def test_pyx_build():
# verify that we can build/run external dso that uses libgolang.
def test_dso_build():
dsouser = testprog + "/golang_dso_user"
pyrun(["setup.py", "build_dso", "-i"], cwd=dsouser)
pyrun(["setup.py", "build_ext", "-i"], cwd=dsouser)
pyrun(["setup.py", "build_dso", "-i"], cwd=dsouser, lsan=False) # gcc leaks
pyrun(["setup.py", "build_ext", "-i"], cwd=dsouser, lsan=False) # gcc leaks
# run built test.
_ = pyout(["-c",
......
golang/runtime/_libgolang.pxd
View file @ 4d64fd0f
# cython: language_level=2
# Copyright (C) 2019-2022 Nexedi SA and Contributors.
# Copyright (C) 2019-2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -36,7 +36,7 @@ cdef extern from "golang/libgolang.h" namespace "golang" nogil:
_libgolang_sema* (*sema_alloc) ()
void (*sema_free) (_libgolang_sema*)
void (*sema_acquire)(_libgolang_sema*)
bint (*sema_acquire)(_libgolang_sema*, uint64_t timeout_ns)
void (*sema_release)(_libgolang_sema*)
void (*nanosleep)(uint64_t)
......
golang/runtime/_runtime_gevent.pyx
View file @ 4d64fd0f
# cython: language_level=2
# Copyright (C) 2019-2023 Nexedi SA and Contributors.
# Copyright (C) 2019-2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -40,7 +40,10 @@ ELSE:
from gevent import sleep as pygsleep
from libc.stdint cimport uint64_t
from libc.stdint cimport uint8_t, uint64_t, UINT64_MAX
cdef extern from *:
ctypedef bint cbool "bool"
from cpython cimport PyObject, Py_INCREF, Py_DECREF
from cython cimport final
......@@ -95,9 +98,12 @@ cdef:
Py_DECREF(pygsema)
return True
bint _sema_acquire(_libgolang_sema *gsema):
bint _sema_acquire(_libgolang_sema *gsema, uint64_t timeout_ns, cbool* pacq):
pygsema = <PYGSema>gsema
pygsema.acquire()
timeout = None
if timeout_ns != UINT64_MAX:
timeout = float(timeout_ns) * 1e-9
pacq[0] = pygsema.acquire(timeout=timeout)
return True
bint _sema_release(_libgolang_sema *gsema):
......@@ -142,14 +148,16 @@ cdef nogil:
if not ok:
panic("pyxgo: gevent: sema: free: failed")
void sema_acquire(_libgolang_sema *gsema):
cbool sema_acquire(_libgolang_sema *gsema, uint64_t timeout_ns):
cdef PyExc exc
cdef cbool acq
with gil:
pyexc_fetch(&exc)
ok = _sema_acquire(gsema)
ok = _sema_acquire(gsema, timeout_ns, &acq)
pyexc_restore(exc)
if not ok:
panic("pyxgo: gevent: sema: acquire: failed")
return acq
void sema_release(_libgolang_sema *gsema):
cdef PyExc exc
......
golang/runtime/_runtime_thread.pyx
View file @ 4d64fd0f
# cython: language_level=2
# Copyright (C) 2019-2022 Nexedi SA and Contributors.
# Copyright (C) 2019-2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -35,7 +35,12 @@ from __future__ import print_function, absolute_import
#
# NOTE Cython declares PyThread_acquire_lock/PyThread_release_lock as nogil
from cpython.pythread cimport PyThread_acquire_lock, PyThread_release_lock, \
PyThread_type_lock, WAIT_LOCK
PyThread_type_lock, WAIT_LOCK, NOWAIT_LOCK, PyLockStatus, PY_LOCK_ACQUIRED, PY_LOCK_FAILURE
cdef extern from * nogil:
ctypedef int PY_TIMEOUT_T # long long there
PyLockStatus PyThread_acquire_lock_timed(PyThread_type_lock, PY_TIMEOUT_T timeout_us, int intr_flag)
# NOTE On Darwin, even though this is considered as POSIX, Python uses
# mutex+condition variable to implement its lock, and, as of 20190828, Py2.7
......@@ -98,6 +103,9 @@ from libc.errno cimport errno, EINTR, EBADF
from posix.fcntl cimport mode_t
from posix.stat cimport struct_stat
from posix.strings cimport bzero
cdef extern from *:
ctypedef bint cbool "bool"
IF POSIX:
from posix.time cimport clock_gettime, nanosleep as posix_nanosleep, timespec, CLOCK_REALTIME
ELSE:
......@@ -138,11 +146,46 @@ cdef nogil:
pysema = <PyThread_type_lock>gsema
PyThread_free_lock(pysema)
void sema_acquire(_libgolang_sema *gsema):
cbool sema_acquire(_libgolang_sema *gsema, uint64_t timeout_ns):
pysema = <PyThread_type_lock>gsema
ok = PyThread_acquire_lock(pysema, WAIT_LOCK)
if ok == 0:
panic("pyxgo: thread: sema_acquire: PyThread_acquire_lock failed")
IF PY3:
cdef PY_TIMEOUT_T timeout_us
ELSE:
cdef uint64_t tprev, t, tsleep
if timeout_ns == UINT64_MAX:
ok = PyThread_acquire_lock(pysema, WAIT_LOCK)
if ok == 0:
panic("pyxgo: thread: sema_acquire: PyThread_acquire_lock failed")
return 1
else:
IF PY3:
timeout_us = timeout_ns // 1000
lkok = PyThread_acquire_lock_timed(pysema, timeout_us, 0)
if lkok == PY_LOCK_FAILURE:
return 0
elif lkok == PY_LOCK_ACQUIRED:
return 1
else:
panic("pyxgo: thread: sema_acquire: PyThread_acquire_lock_timed failed")
ELSE:
# py2 misses PyThread_acquire_lock_timed - provide fallback ourselves
tprev = nanotime()
while 1:
ok = PyThread_acquire_lock(pysema, NOWAIT_LOCK)
if ok:
return 1
tsleep = min(timeout_ns, 50*1000) # poll every 50 μs = 20 Hz
if tsleep == 0:
break
nanosleep(tsleep)
t = nanotime()
if t < tprev:
break # clock skew
if t - tprev >= timeout_ns:
break
timeout_ns -= t - tprev
tprev = t
return 0
void sema_release(_libgolang_sema *gsema):
pysema = <PyThread_type_lock>gsema
......
golang/runtime/libgolang.cpp
View file @ 4d64fd0f
......@@ -131,6 +131,7 @@ using internal::_runtime;
namespace internal { namespace atomic { extern void _init(); } }
namespace os { namespace signal { extern void _init(); } }
namespace time { extern void _init(); }
void _libgolang_init(const _libgolang_runtime_ops *runtime_ops) {
if (_runtime != nil) // XXX better check atomically
panic("libgolang: double init");
......@@ -138,6 +139,7 @@ void _libgolang_init(const _libgolang_runtime_ops *runtime_ops) {
internal::atomic::_init();
os::signal::_init();
time::_init();
}
void _taskgo(void (*f)(void *), void *arg) {
......@@ -166,7 +168,15 @@ void _semafree(_sema *sema) {
}
void _semaacquire(_sema *sema) {
_runtime->sema_acquire((_libgolang_sema *)sema);
bool ok;
ok = _runtime->sema_acquire((_libgolang_sema *)sema, UINT64_MAX);
if (!ok)
panic("semaacquire: failed");
}
// NOTE not currently exposed in public API
bool _semaacquire_timed(_sema *sema, uint64_t timeout_ns) {
return _runtime->sema_acquire((_libgolang_sema *)sema, timeout_ns);
}
void _semarelease(_sema *sema) {
......
golang/time.cpp
View file @ 4d64fd0f
This diff is collapsed.
golang/time.h
View file @ 4d64fd0f
#ifndef _NXD_LIBGOLANG_TIME_H
#define _NXD_LIBGOLANG_TIME_H
// Copyright (C) 2019-2023 Nexedi SA and Contributors.
// Copyright (C) 2019-2024 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -118,6 +118,7 @@ private:
double _dt;
sync::Mutex _mu;
bool _stop;
Timer _timer;
// don't new - create only via new_ticker()
private:
......@@ -147,18 +148,12 @@ LIBGOLANG_API Timer new_timer(double dt);
struct _Timer : object {
chan<double> c;
private:
func<void()> _f;
sync::Mutex _mu;
double _dt; // +inf - stopped, otherwise - armed
int _ver; // current timer was armed by n'th reset
// don't new - create only via new_timer() & co
private:
_Timer();
~_Timer();
friend Timer _new_timer(double dt, func<void()> f);
friend class _TimerImpl;
public:
LIBGOLANG_API void decref();
......@@ -182,9 +177,6 @@ public:
//
// the timer must be either already stopped or expired.
LIBGOLANG_API void reset(double dt);
private:
void _fire(double dt, int ver);
};
......
golang/time_test.py
View file @ 4d64fd0f
# -*- coding: utf-8 -*-
# Copyright (C) 2019 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
# Copyright (C) 2019-2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
......@@ -20,9 +20,10 @@
from __future__ import print_function, absolute_import
from golang import select
from golang import time
from golang import select, func, defer
from golang import time, sync
from golang.golang_test import panics
from six.moves import range as xrange
# all timer tests operate in dt units
dt = 10*time.millisecond
......@@ -65,6 +66,7 @@ def test_ticker_time():
# test_timer verifies that Timer/Ticker fire as expected.
@func
def test_timer():
# start timers at x5, x7 and x11 intervals an verify that the timers fire
# in expected sequence. The times when the timers fire do not overlap in
......@@ -73,15 +75,15 @@ def test_timer():
tv = [] # timer events
Tstart = time.now()
t23 = time.Timer(23*dt)
t5 = time.Timer( 5*dt)
t23 = time.Timer(23*dt); defer(t23.stop)
t5 = time.Timer( 5*dt); defer(t5 .stop)
def _():
tv.append(7)
t7f.reset(7*dt)
t7f = time.Timer( 7*dt, f=_)
t7f = time.Timer( 7*dt, f=_); defer(t7f.stop)
tx11 = time.Ticker(11*dt)
tx11 = time.Ticker(11*dt); defer(tx11.stop)
while 1:
_, _rx = select(
......@@ -108,19 +110,20 @@ def test_timer():
# test_timer_misc, similarly to test_timer, verifies misc timer convenience functions.
@func
def test_timer_misc():
tv = []
Tstart = time.now()
c23 = time.after(23*dt)
c5 = time.after( 5*dt)
c23 = time.after(23*dt) # cannot stop
c5 = time.after( 5*dt) # cannot stop
def _():
tv.append(7)
t7f.reset(7*dt)
t7f = time.after_func(7*dt, _)
t7f = time.after_func(7*dt, _); defer(t7f.stop)
cx11 = time.tick(11*dt)
cx11 = time.tick(11*dt) # cannot stop
while 1:
_, _rx = select(
......@@ -148,13 +151,14 @@ def test_timer_misc():
# test_timer_stop verifies that .stop() cancels Timer or Ticker.
@func
def test_timer_stop():
tv = []
t10 = time.Timer (10*dt)
t2 = time.Timer ( 2*dt) # will fire and cancel t3, tx5
t3 = time.Timer ( 3*dt) # will be canceled
tx5 = time.Ticker( 5*dt) # will be canceled
t10 = time.Timer (10*dt); defer(t10.stop)
t2 = time.Timer ( 2*dt); defer(t2 .stop) # will fire and cancel t3, tx5
t3 = time.Timer ( 3*dt); defer(t3 .stop) # will be canceled
tx5 = time.Ticker( 5*dt); defer(tx5.stop) # will be canceled
while 1:
_, _rx = select(
......@@ -180,9 +184,10 @@ def test_timer_stop():
# test_timer_stop_drain verifies that Timer/Ticker .stop() drains timer channel.
@func
def test_timer_stop_drain():
t = time.Timer (1*dt)
tx = time.Ticker(1*dt)
t = time.Timer (1*dt); defer(t.stop)
tx = time.Ticker(1*dt); defer(tx.stop)
time.sleep(2*dt)
assert len(t.c) == 1
......@@ -195,9 +200,45 @@ def test_timer_stop_drain():
assert len(tx.c) == 0
# test_timer_stop_vs_func verifies that Timer .stop() works correctly with func-timer.
@func
def test_timer_stop_vs_func():
tv = []
def _1(): tv.append(1)
def _2(): tv.append(2)
t1 = time.after_func(1e6*dt, _1); defer(t1.stop)
t2 = time.after_func( 1*dt, _2); defer(t2.stop)
time.sleep(2*dt)
assert t1.stop() == True
assert t2.stop() == False
assert tv == [2]
# test_timer_reset_armed verifies that .reset() panics if called on armed timer.
@func
def test_timer_reset_armed():
# reset while armed
t = time.Timer(10*dt)
t = time.Timer(10*dt); defer(t.stop)
with panics("Timer.reset: the timer is armed; must be stopped or expired"):
t.reset(5*dt)
# bench_timer_arm_cancel benchmarks arming timers that do not fire.
# it shows how cheap or expensive it is to use timers to implement timeouts.
def bench_timer_arm_cancel(b):
for i in xrange(b.N):
t = time.Timer(10*time.second)
_ = t.stop()
assert _ is True
# bench_timer_arm_fire benchmarks arming timers that do fire.
# it shows what it costs to go through all steps related to timer loop and firing timers.
def bench_timer_arm_fire(b):
wg = sync.WaitGroup()
wg.add(b.N)
for i in xrange(b.N):
t = time.after_func(1*time.millisecond, wg.done)
wg.wait()
setup.py
View file @ 4d64fd0f
......@@ -188,7 +188,7 @@ class develop(XInstallGPython, _develop):
# requirements of packages under "golang." namespace
R = {
'cmd.pybench': {'pytest', 'py'},
'cmd.pybench': {'pytest', 'py ; python_version >= "3"'},
'pyx.build': {'setuptools', 'wheel', 'cython < 3', 'setuptools_dso >= 2.8'},
'x.perf.benchlib': {'numpy'},
}
......@@ -467,8 +467,11 @@ setup(
'golang/os/signal.h',
'golang/strings.h',
'golang/sync.h',
'golang/time.h'],
include_dirs = ['3rdparty/include'],
'golang/time.h',
'3rdparty/ratas/src/timer-wheel.h'],
include_dirs = [
'3rdparty/include',
'3rdparty/ratas/src'],
define_macros = [('BUILDING_LIBGOLANG', None)],
soversion = '0.1'),
......@@ -604,9 +607,6 @@ setup(
Programming Language :: Python :: 2
Programming Language :: Python :: 2.7
Programming Language :: Python :: 3
Programming Language :: Python :: 3.5
Programming Language :: Python :: 3.6
Programming Language :: Python :: 3.7
Programming Language :: Python :: 3.8
Programming Language :: Python :: 3.9
Programming Language :: Python :: 3.10
......
tox.ini
View file @ 4d64fd0f
[tox]
envlist =
{py27d,py27,py37,py38,py39d,py39,py310d,py310,py311d,py311,py312,pypy,pypy3}-{thread,gevent}
{py27d,py27,py38,py39d,py39,py310d,py310,py311d,py311,py312,pypy,pypy3}-{thread,gevent}
# ThreadSanitizer
......@@ -10,24 +10,23 @@ envlist =
# (*) PyPy locks its GIL (see RPyGilAcquire) by manually doing atomic cmpxchg
# and other games, which TSAN cannot see if PyPy itself was not compiled with
# -fsanitize=thread.
{py27d,py27,py37,py38,py39d,py39,py310d,py310,py311d,py311,py312 }-{thread }-tsan
{py27d,py27,py38,py39d,py39,py310d,py310,py311d,py311,py312 }-{thread }-tsan
# XXX py*-gevent-tsan would be nice to have, but at present TSAN is not
# effective with gevent, because it does not understand greenlet "thread"
# switching and so perceives the program as having only one thread where races
# are impossible. Disabled to save time.
# {py27d,py27,py37,py38,py39d,py39,py310d,py310,py311d,py311,py312 }-{ gevent}-tsan
# {py27d,py27,py38,py39d,py39,py310d,py310,py311d,py311,py312 }-{ gevent}-tsan
# AddressSanitizer
# XXX asan does not work with gevent: https://github.com/python-greenlet/greenlet/issues/113
{py27d,py27,py37,py38,py39d,py39,py310d,py310,py311d,py311,py312,pypy,pypy3}-{thread }-asan
{py27d,py27,py38,py39d,py39,py310d,py310,py311d,py311,py312,pypy,pypy3}-{thread }-asan
[testenv]
basepython =
py27d: python2.7-dbg
py27: python2.7
py37: python3.7
py38: python3.8
py39d: python3.9-dbg
py39: python3.9
......@@ -43,16 +42,16 @@ basepython =
setenv =
# distutils take CFLAGS for both C and C++.
# distutils use CFLAGS also at link stage -> we don't need to set LDFLAGS separately.
tsan: CFLAGS=-g -fsanitize=thread
asan: CFLAGS=-g -fsanitize=address
tsan: CFLAGS=-g -fsanitize=thread -fno-omit-frame-pointer
asan: CFLAGS=-g -fsanitize=address -fno-omit-frame-pointer
# XXX however distutils' try_link, which is used by numpy.distutils use only CC
# as linker without CFLAGS and _without_ LDFLAGS, which fails if *.o were
# compiled with -fsanitize=X and linked without that option. Work it around
# with also adjusting CC.
# XXX better arrange to pass CFLAGS to pygolang only, e.g. by adding --race or
# --sanitize=thread to `setup.py build_ext`.
tsan: CC=cc -fsanitize=thread
asan: CC=cc -fsanitize=address
tsan: CC=cc -fsanitize=thread -fno-omit-frame-pointer
asan: CC=cc -fsanitize=address -fno-omit-frame-pointer
# always compile pygolang from source and don't reuse binary pygolang wheels as
# we compile each case with different CFLAGS.
......@@ -76,3 +75,5 @@ commands=
# likewise for python debug builds.
asan,tsan,py{27,39,310,311,312}d: -s \
gpython/ golang/
allowlist_externals={toxinidir}/trun
trun
View file @ 4d64fd0f
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (C) 2019-2024 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
......@@ -34,7 +35,7 @@ trun cares to run python with LD_PRELOAD set appropriately to /path/to/libtsan.s
from __future__ import print_function, absolute_import
import os, sys, re, subprocess, types
import os, os.path, sys, re, subprocess, platform, types
PY3 = (bytes is not str)
if PY3:
from importlib import machinery as imp_machinery
......@@ -87,6 +88,7 @@ def main():
# determine if _golang.so is linked to a sanitizer, and if yes, to which
# particular sanitizer DSO. Set LD_PRELOAD appropriately.
libxsan = None
ld_preload = None
if 'linux' in sys.platform:
p = subprocess.Popen(["ldd", _golang_so.path], stdout=subprocess.PIPE)
......@@ -127,7 +129,8 @@ def main():
_ = grep1("DYLD_INSERT_LIBRARIES=(.*)$", err)
if _ is not None:
ld_preload = ("DYLD_INSERT_LIBRARIES", _.group(1))
libxsan = _.group(1)
ld_preload = ("DYLD_INSERT_LIBRARIES", libxsan)
else:
print("trun %r: `import golang` failed with unexpected error:" % sys.argv[1:], file=sys.stderr)
print(err, file=sys.stderr)
......@@ -144,7 +147,7 @@ def main():
env_prepend("TSAN_OPTIONS", "halt_on_error=1")
env_prepend("ASAN_OPTIONS", "halt_on_error=1")
# tweak TSAN/ASAN defaults:
# tweak TSAN/ASAN/LSAN defaults:
# enable TSAN deadlock detector
# (unfortunately it caughts only few _potential_ deadlocks and actually
......@@ -152,15 +155,49 @@ def main():
env_prepend("TSAN_OPTIONS", "detect_deadlocks=1")
env_prepend("TSAN_OPTIONS", "second_deadlock_stack=1")
# many python allocations, whose lifetime coincides with python interpreter
# lifetime and which are not explicitly freed on python shutdown, are
# reported as leaks. Disable leak reporting to avoid huge non-pygolang
# related printouts.
env_prepend("ASAN_OPTIONS", "detect_leaks=0")
# tune ASAN to check more aggressively by default
env_prepend("ASAN_OPTIONS", "detect_stack_use_after_return=1")
# enable ASAN/LSAN leak detector.
#
# Do it only on CPython ≥ 3.11 because on py2 and on earlier py3 versions
# there are many many python allocations, whose lifetime coincide with
# python interpreter lifetime, and which are not explicitly freed on python
# shutdown. For py3 they significantly improved this step by step and
# starting from 3.11 it becomes practical to silence some still-leaks with
# suppressions, while for earlier py3 versions and especially for py2 it
# is, unfortunately, not manageable. Do not spend engineering time with
# activating LSAN on PyPy as that is tier 2 platform and bug tail history
# of memory leaks is very long even only on cpython.
if sys.version_info < (3,11):
env_prepend("ASAN_OPTIONS", "detect_leaks=0")
if libxsan is not None:
if 'asan' in libxsan.lower():
print("W: trun %r: asan: leak detection deactivated on %s %s" % (
sys.argv[1:], platform.python_implementation(), platform.python_version()),
file=sys.stderr)
else:
env_prepend("ASAN_OPTIONS", "detect_leaks=1")
env_prepend("LSAN_OPTIONS", "suppressions=%s" % os.path.abspath(os.path.join(
os.path.dirname(__file__), ".lsan-ignore.txt")))
# do not print statistics for suppressed leaks - else it breaks tests that verify program output
env_prepend("LSAN_OPTIONS", "print_suppressions=0")
# enable DWARF-based unwinding.
# else, if python is not compiled with -fno-omit-frame-pointer, it can show
# the whole traceback as e.g. just
# Direct leak of 32 byte(s) in 1 object(s) allocated from:
# #0 0x7f88522f3bd7 in malloc ../../../../src/libsanitizer/asan/asan_malloc_linux.cpp:69
# #1 0x55f910a3d9a4 in PyThread_allocate_lock Python/thread_pthread.h:385
# and our leak suppressions won't work.
# this is slower compared to default frame-pointer based unwinding, but
# still works reasonably timely when run with just tests.
env_prepend("ASAN_OPTIONS", "fast_unwind_on_malloc=0")
# leak suppression also needs full tracebacks to work correctly, since with
# python there are many levels of call nesting at C level, and to filter-out e.g.
# top-level PyImport_Import we need to go really deep.
env_prepend("ASAN_OPTIONS", "malloc_context_size=255")
# exec `...`
os.execvp(sys.argv[1], sys.argv[1:])
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7