Commit 55d39d4d authored by Kirill Smelkov's avatar Kirill Smelkov

Windows support

Pygolang stopped to work on Windows in 2019 starting from 8fa3c15b (Start using
Cython and providing Cython/nogil API). Restore it now.
parents 3a83e8f6 d1e92fa2
......@@ -10,6 +10,8 @@ build/
*.so.*
*.dylib
*.dll
*.lib
*.exp
*.pyd
*_dsoinfo.py
......
......@@ -3,6 +3,7 @@ include golang/libgolang.h
include golang/runtime/libgolang.cpp
include golang/runtime/libpyxruntime.cpp
include golang/pyx/runtime.h
include golang/pyx/testprog/golang_dso_user/dsouser/dso.h
include golang/pyx/testprog/golang_dso_user/dsouser/dso.cpp
include golang/runtime/internal.h
include golang/runtime/internal/atomic.h
......@@ -33,6 +34,8 @@ include golang/sync_test.cpp
include golang/time.h
include golang/time.cpp
include golang/_testing.h
include golang/_compat/windows/strings.h
include golang/_compat/windows/unistd.h
recursive-include golang *.py *.pxd *.pyx *.toml *.txt*
recursive-include gpython *.py
recursive-include 3rdparty *.h
......
# -*- coding: utf-8 -*-
# Copyright (C) 2018-2022 Nexedi SA and Contributors.
# Copyright (C) 2018-2023 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -42,6 +42,9 @@ from golang._gopath import gimport # make gimport available from golang
import inspect, sys
import decorator, six
import setuptools_dso
setuptools_dso.dylink_prepare_dso('golang.runtime.libgolang')
from golang._golang import _pysys_exc_clear as _sys_exc_clear
# @func is a necessary decorator for functions for selected golang features to work.
......
#ifndef _NXD_LIBGOLANG_COMPAT_WIN_STRINGS_H
#define _NXD_LIBGOLANG_COMPAT_WIN_STRINGS_H
//
// Copyright (C) 2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
#include <string.h>
static inline void bzero(void *p, size_t n) {
memset(p, '\0', n);
}
#endif // _NXD_LIBGOLANG_COMPAT_WIN_STRINGS_H
#ifndef _NXD_LIBGOLANG_COMPAT_WIN_UNISTD_H
#define _NXD_LIBGOLANG_COMPAT_WIN_UNISTD_H
//
// Copyright (C) 2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
// stub unistd.h to be used on Windows where it is absent.
// we need this because e.g. `cimport posix.stat` forces inclusion of unistd.h
// even if we use part of posix.stat that is available everywhere.
#include <io.h>
#define O_CLOEXEC _O_NOINHERIT
#endif // _NXD_LIBGOLANG_COMPAT_WIN_UNISTD_H
// Copyright (C) 2019-2020 Nexedi SA and Contributors.
// Copyright (C) 2019-2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -48,7 +48,7 @@ string _vsprintf(const char *format, va_list argp) {
return string(buf.get(), buf.get() + nchar); // without trailing '\0'
}
string sprintf(const string &format, ...) {
string sprintf(const string format, ...) {
va_list argp;
va_start(argp, format);
string str = fmt::_vsprintf(format.c_str(), argp);
......@@ -64,7 +64,7 @@ string sprintf(const char *format, ...) {
return str;
}
error ___errorf(const string &format, ...) {
error ___errorf(const string format, ...) {
va_list argp;
va_start(argp, format);
error err = errors::New(fmt::_vsprintf(format.c_str(), argp));
......
#ifndef _NXD_LIBGOLANG_FMT_H
#define _NXD_LIBGOLANG_FMT_H
// Copyright (C) 2019-2020 Nexedi SA and Contributors.
// Copyright (C) 2019-2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -40,7 +40,7 @@ namespace golang {
namespace fmt {
// sprintf formats text into string.
LIBGOLANG_API string sprintf(const string &format, ...);
LIBGOLANG_API string sprintf(const string format, ...);
// intseq<i1, i2, ...> and intrange<n> are used by errorf to handle %w.
......@@ -75,7 +75,7 @@ namespace {
//
// format suffix ": %w" is additionally handled as in Go with
// `errorf("... : %w", ..., err)` creating error that can be unwrapped back to err.
LIBGOLANG_API error ___errorf(const string& format, ...);
LIBGOLANG_API error ___errorf(const string format, ...);
LIBGOLANG_API error ___errorfTryWrap(const string& format, error last_err, ...);
LIBGOLANG_API string ___error_str(error err);
......@@ -111,7 +111,10 @@ inline error errorf(const string& format, Argv... argv) {
// `const char *` overloads just to catch format mistakes as
// __attribute__(format) does not work with std::string.
LIBGOLANG_API string sprintf(const char *format, ...)
__attribute__ ((format (printf, 1, 2)));
#ifndef _MSC_VER
__attribute__ ((format (printf, 1, 2)))
#endif
;
// cannot use __attribute__(format) for errorf as we add %w handling.
// still `const char *` overload is useful for performance.
......
......@@ -1739,8 +1739,28 @@ def _pyrun(argv, stdin=None, stdout=None, stderr=None, **kw): # -> retcode, st
pathv.extend(envpath.split(os.pathsep))
env['PYTHONPATH'] = os.pathsep.join(pathv)
# set $PYTHONIOENCODING to encoding of stdin/stdout/stderr
# we need to do it because on Windows `python x.py | ...` runs with stdio
# encoding set to cp125X even if just `python x.py` runs with stdio
# encoding=UTF-8.
if 'PYTHONIOENCODING' not in env:
enc = set([_.encoding for _ in (sys.stdin, sys.stdout, sys.stderr)])
if None in enc: # without -s pytest uses _pytest.capture.DontReadFromInput
enc.remove(None) # with None .encoding
assert len(enc) == 1
env['PYTHONIOENCODING'] = enc.pop()
p = Popen(argv, stdin=(PIPE if stdin else None), stdout=stdout, stderr=stderr, env=env, **kw)
stdout, stderr = p.communicate(stdin)
# on windows print emits \r\n instead of just \n
# normalize that to \n in *out
if os.name == 'nt':
if stdout is not None:
stdout = stdout.replace(b'\r\n', b'\n')
if stderr is not None:
stderr = stderr.replace(b'\r\n', b'\n')
return p.returncode, stdout, stderr
# pyrun runs `sys.executable argv... <stdin`.
......@@ -1807,6 +1827,13 @@ def assertDoc(want, got):
got = got.replace(dir_pygolang, "PYGOLANG") # /home/x/.../pygolang -> PYGOLANG
got = got.replace(udir_pygolang, "PYGOLANG") # ~/.../pygolang -> PYGOLANG
# got: normalize PYGOLANG\a\b\c -> PYGOLANG/a/b/c
# a\b\c\d.py -> a/b/c/d.py
def _(m):
return m.group(0).replace(os.path.sep, '/')
got = re.sub(r"(?<=PYGOLANG)[^\s]+(?=\s)", _, got)
got = re.sub(r"([\w\\\.]+)(?=\.py)", _, got)
# want: process conditionals
# PY39(...) -> ... if py ≥ 3.9 else ø (inline)
# `... +PY39` -> ... if py ≥ 3.9 else ø (whole line)
......@@ -1862,8 +1889,10 @@ def test_fmtargspec():
# readfile returns content of file @path.
def readfile(path):
with open(path, "r") as f:
def readfile(path): # -> bytes
# on windows in text mode files are opened with encoding=locale.getdefaultlocale()
# which is CP125X instead of UTF-8. -> manually decode as 'UTF-8'
with open(path, "rb") as f:
return f.read()
# abbrev_home returns path with user home prefix abbreviated with ~.
......
......@@ -176,6 +176,12 @@
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#ifdef _MSC_VER // no mode_t on msvc
typedef int mode_t;
#endif
// DSO symbols visibility (based on https://gcc.gnu.org/wiki/Visibility)
#if defined _WIN32 || defined __CYGWIN__
#define LIBGOLANG_DSO_EXPORT __declspec(dllexport)
......@@ -829,7 +835,7 @@ struct _interface {
protected:
// don't use destructor -> use decref
~_interface();
LIBGOLANG_API ~_interface();
};
typedef refptr<_interface> interface;
......
......@@ -37,7 +37,8 @@
// GLIBC < 2.32 provides sys_siglist but not sigdescr_np in its headers
// cut this short
// (on darwing sys_siglist declaration is normally provided)
#ifndef __APPLE__
// (on windows sys_siglist is not available at all)
#if !(defined(__APPLE__) || defined(_WIN32))
extern "C" {
extern const char * const sys_siglist[];
}
......@@ -248,7 +249,7 @@ tuple<string, error> ReadFile(const string& path) {
tuple<File, File, error> Pipe() {
int vfd[2], syserr;
syserr = sys::Pipe(vfd);
syserr = sys::Pipe(vfd, 0);
if (syserr != 0)
return make_tuple(nil, nil, fmt::errorf("pipe: %w", sys::NewErrno(syserr)));
......@@ -286,8 +287,20 @@ string Signal::String() const {
const Signal& sig = *this;
const char *sigstr = nil;
#ifdef _WIN32
switch (sig.signo) {
case SIGABRT: return "Aborted";
case SIGBREAK: return "Break";
case SIGFPE: return "Floating point exception";
case SIGILL: return "Illegal instruction";
case SIGINT: return "Interrupt";
case SIGSEGV: return "Segmentation fault";
case SIGTERM: return "Terminated";
}
#else
if (0 <= sig.signo && sig.signo < NSIG)
sigstr = ::sys_siglist[sig.signo]; // might be nil as well
#endif
if (sigstr != nil)
return string(sigstr);
......
#ifndef _NXD_LIBGOLANG_OS_H
#define _NXD_LIBGOLANG_OS_H
//
// Copyright (C) 2019-2022 Nexedi SA and Contributors.
// Copyright (C) 2019-2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -61,7 +61,7 @@ private:
~_File();
friend File _newFile(_libgolang_ioh* ioh, const string& name);
public:
void decref();
LIBGOLANG_API void decref();
public:
LIBGOLANG_API string Name() const;
......@@ -95,9 +95,15 @@ private:
// Open opens file @path.
LIBGOLANG_API std::tuple<File, error> Open(const string &path, int flags = O_RDONLY,
mode_t mode = S_IRUSR | S_IWUSR | S_IXUSR |
mode_t mode =
#if !defined(_MSC_VER)
S_IRUSR | S_IWUSR | S_IXUSR |
S_IRGRP | S_IWGRP | S_IXGRP |
S_IROTH | S_IWOTH | S_IXOTH);
S_IROTH | S_IWOTH | S_IXOTH
#else
_S_IREAD | _S_IWRITE
#endif
);
// NewFile wraps OS-level file-descriptor into File.
// The ownership of sysfd is transferred to File.
......
// Copyright (C) 2021-2022 Nexedi SA and Contributors.
// Copyright (C) 2021-2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -89,6 +89,10 @@
#include <atomic>
#include <tuple>
#if defined(_WIN32)
# include <windows.h>
#endif
#define DEBUG 0
#if DEBUG
......@@ -97,6 +101,12 @@
# define debugf(format, ...) do {} while (0)
#endif
#if defined(_MSC_VER)
# define HAVE_SIGACTION 0
#else
# define HAVE_SIGACTION 1
#endif
// golang::os::signal::
namespace golang {
namespace os {
......@@ -109,13 +119,30 @@ using std::tie;
using std::vector;
using cxx::set;
#if HAVE_SIGACTION
static void xsigemptyset(sigset_t *sa_mask);
#else
// custom `struct sigaction` emulated via signal(2)
#define SA_SIGINFO 1
typedef struct {} siginfo_t;
struct sigaction {
union {
void (*sa_handler)(int);
void (*sa_sigaction)(int, siginfo_t*, void*);
};
int sa_flags;
};
static void _os_sighandler_nosigaction(int sig);
#endif
static void _os_sighandler(int sig, siginfo_t *info, void *ucontext);
static void _notify(int signo);
static void _checksig(int signo);
static void _checkActEqual(const struct sigaction *a, const struct sigaction *b);
static void _spinwaitNextQueueCycle();
static int sys_sigaction(int signo, const struct sigaction *act, struct sigaction *oldact);
static void xsys_sigaction(int signo, const struct sigaction *act, struct sigaction *oldact);
static void xsigemptyset(sigset_t *sa_mask);
static bool _sigact_equal(const struct sigaction *a, const struct sigaction *b);
......@@ -160,27 +187,31 @@ void _init() {
// create _wakerx <-> _waketx pipe; set _waketx to nonblocking mode
int vfd[2];
if (sys::Pipe(vfd) < 0)
if (sys::Pipe(vfd, O_CLOEXEC) < 0)
panic("pipe(_wakerx, _waketx)"); // TODO +syserr
if (sys::Fcntl(vfd[0], F_SETFD, FD_CLOEXEC) < 0)
panic("fcntl(_wakerx, FD_CLOEXEC)"); // TODO +syserr
error err;
tie(_wakerx, err) = os::NewFile(vfd[0], "_wakerx");
if (err != nil)
panic("os::newFile(_wakerx");
_waketx = vfd[1];
#ifndef _WIN32
if (sys::Fcntl(_waketx, F_SETFL, O_NONBLOCK) < 0)
panic("fcntl(_waketx, O_NONBLOCK)"); // TODO +syserr
if (sys::Fcntl(_waketx, F_SETFD, FD_CLOEXEC) < 0)
panic("fcntl(_waketx, FD_CLOEXEC)"); // TODO +syserr
#else
HANDLE hwaketx = (HANDLE)_get_osfhandle(_waketx);
DWORD mode = PIPE_READMODE_BYTE | PIPE_NOWAIT;
if (!SetNamedPipeHandleState(hwaketx, &mode, NULL, NULL))
panic("SetNamedPipeHandleState(hwaketx, PIPE_NOWAIT)"); // TODO +syserr
#endif
_actIgnore.sa_handler = SIG_IGN;
_actIgnore.sa_flags = 0;
xsigemptyset(&_actIgnore.sa_mask);
_actNotify.sa_sigaction = _os_sighandler;
_actNotify.sa_flags = SA_SIGINFO;
#if HAVE_SIGACTION
xsigemptyset(&_actIgnore.sa_mask);
xsigemptyset(&_actNotify.sa_mask);
#endif
}
......@@ -248,6 +279,14 @@ done:
if (sah != SIG_IGN) {
if (sah != SIG_DFL) {
sah(sig);
// handlers installed via signal usually reinstall themselves
// return it back to us
struct sigaction after;
xsys_sigaction(sig, &_actNotify, &after);
if (! (_sigact_equal(&after, &_actNotify) ||
( ((after.sa_flags & SA_SIGINFO) == 0) &&
((after.sa_handler == SIG_DFL) || (after.sa_handler == sah)))))
panic("collision detected wrt thirdparty signal usage");
}
else {
// SIG_DFL && _SigReset - reraise to die if the signal is fatal
......@@ -343,7 +382,7 @@ static int/*syserr*/ _Notify1(chan<os::Signal> ch, os::Signal sig) {
// retrieve current signal action
struct sigaction cur;
int syserr = sys::Sigaction(sig.signo, nil, &cur);
int syserr = sys_sigaction(sig.signo, nil, &cur);
if (syserr < 0) {
// TODO reenable once we can panic with any object
//return fmt::errorf("sigaction sig%d: %w", sig.signo, sys::NewErrno(syserr);
......@@ -389,7 +428,7 @@ static int/*syserr*/ _Notify1(chan<os::Signal> ch, os::Signal sig) {
// register our sigaction
struct sigaction old;
syserr = sys::Sigaction(sig.signo, &_actNotify, &old);
syserr = sys_sigaction(sig.signo, &_actNotify, &old);
if (syserr < 0) {
// TODO reenable once we can panic with any object
//return fmt::errorf("sigaction sig%d: %w", sig.signo, sys::NewErrno(syserr);
......@@ -469,7 +508,7 @@ static int/*syserr*/ _Ignore1(os::Signal sig) {
if (h == nil) {
h = new _SigHandler();
h->sigstate.store(_SigIgnoring);
int syserr = sys::Sigaction(sig.signo, nil, &h->prev_act);
int syserr = sys_sigaction(sig.signo, nil, &h->prev_act);
if (syserr < 0) {
delete h;
return syserr; // TODO errctx
......@@ -481,7 +520,7 @@ static int/*syserr*/ _Ignore1(os::Signal sig) {
h->sigstate.store(_SigIgnoring);
h->subscribers.clear();
int syserr = sys::Sigaction(sig.signo, &_actIgnore, nil);
int syserr = sys_sigaction(sig.signo, &_actIgnore, nil);
if (syserr < 0)
return syserr; // TODO errctx
......@@ -509,7 +548,7 @@ static int/*syserr*/ _Reset1(os::Signal sig) {
h->sigstate.store(_SigReset);
struct sigaction act;
int syserr = sys::Sigaction(sig.signo, &h->prev_act, &act);
int syserr = sys_sigaction(sig.signo, &h->prev_act, &act);
if (syserr < 0)
return syserr; // TODO errctx
if (sigstate == _SigNotifying)
......@@ -617,16 +656,75 @@ static void _checksig(int signo) {
}
#if HAVE_SIGACTION
static void xsigemptyset(sigset_t *sa_mask) {
if (sigemptyset(sa_mask) < 0)
panic("sigemptyset failed"); // must always succeed
}
#endif
static void xsys_sigaction(int signo, const struct sigaction *act, struct sigaction *oldact) {
int syserr = sys::Sigaction(signo, act, oldact);
int syserr = sys_sigaction(signo, act, oldact);
if (syserr != 0)
panic("sigaction failed"); // TODO add errno detail
}
static int sys_sigaction(int signo, const struct sigaction *act, struct sigaction *oldact) {
#if HAVE_SIGACTION
return sys::Sigaction(signo, act, oldact);
#else
sys::sighandler_t oldh, newh;
int syserr;
if (act == nil) {
newh = SIG_GET;
}
else if (act->sa_flags & SA_SIGINFO) {
if (act->sa_sigaction != _os_sighandler)
panic("BUG: compat sigaction: act->sa_sigaction != _os_sighandler");
newh = _os_sighandler_nosigaction;
}
else {
newh = act->sa_handler;
}
oldh = sys::Signal(signo, newh, &syserr);
if (oldh == SIG_ERR)
return syserr;
if (oldact != nil) {
if (oldh == _os_sighandler_nosigaction) {
oldact->sa_sigaction = _os_sighandler;
oldact->sa_flags = SA_SIGINFO;
}
else {
oldact->sa_handler = oldh;
oldact->sa_flags = 0;
}
}
return 0;
#endif
}
#if !HAVE_SIGACTION
static void _os_sighandler_nosigaction(int sig) {
// reinstall sighandler since on handler invocation it is reset to SIG_DFL.
// do it as fast as we can as the first thing - to minimize window of race
// while signal handler is reset from being our _os_sighandler.
//
// NOTE it is ok to reinstall before invoking _os_sighandler because
// _os_sighandler uses only atomics and pipe write and so is reentrant.
sys::sighandler_t h;
int syserr;
h = sys::Signal(sig, _os_sighandler_nosigaction, &syserr);
if (h == SIG_ERR)
panic("signal(reinstall) failed");
if ( !(h == SIG_DFL /*reset*/ ||
h == _os_sighandler_nosigaction /*concurrent sighandler*/) )
panic("collision detected wrt thirdparty signal usage");
// invoke _os_sighandler as if it was setup by sigaction.
_os_sighandler(sig, NULL, NULL);
}
#endif
static bool _sigact_equal(const struct sigaction *a, const struct sigaction *b) {
// don't compare sigaction by memcmp - it will fail because struct sigaction
......
# -*- coding: utf-8 -*-
# Copyright (C) 2021-2022 Nexedi SA and Contributors.
# Copyright (C) 2021-2023 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -30,12 +30,27 @@ from golang.golang_test import panics, _pyrun
from pytest import raises
from subprocess import PIPE
try:
from signal import raise_signal
except ImportError: # py2
from _testcapi import raise_signal
# directories
dir_os = dirname(__file__) # .../pygolang/os
dir_testprog = dir_os + "/testprog" # .../pygolang/os/testprog
# default to use SIGUSR1/SIGUSR2 in tests.
# but use SIGTERM/SIGINT if those are not available (windows).
try:
SIG1 = getattr(syscall, 'SIGUSR1')
SIG2 = getattr(syscall, 'SIGUSR2')
except AttributeError:
SIG1 = syscall.SIGTERM
SIG2 = syscall.SIGINT
N = 1000
# test_signal verifies signal delivery to channels controlled by Notify/Stop/Ignore/Reset.
......@@ -43,9 +58,9 @@ N = 1000
def test_signal():
# Notify/Stop with wrong chan dtype -> panic
_ = panics("pychan: channel type mismatch")
with _: signal.Notify(chan(2), syscall.SIGUSR1)
with _: signal.Notify(chan(2), SIG1)
with _: signal.Stop (chan(2))
with _: signal.Notify(chan(2, dtype='C.int'), syscall.SIGUSR1)
with _: signal.Notify(chan(2, dtype='C.int'), SIG1)
with _: signal.Stop (chan(2, dtype='C.int'))
# Notify/Ignore/Reset with wrong signal type
......@@ -54,109 +69,109 @@ def test_signal():
with _: signal.Ignore(None)
with _: signal.Reset(None)
# subscribe ch1(USR1), ch12(USR1,USR2) and ch2(USR2)
# subscribe ch1(SIG1), ch12(SIG1,SIG2) and ch2(SIG2)
ch1 = chan(2, dtype=gos.Signal)
ch12 = chan(2, dtype=gos.Signal)
ch2 = chan(2, dtype=gos.Signal)
signal.Notify(ch1, syscall.SIGUSR1)
signal.Notify(ch12, syscall.SIGUSR1, syscall.SIGUSR2)
signal.Notify(ch2, syscall.SIGUSR2)
signal.Notify(ch1, SIG1)
signal.Notify(ch12, SIG1, SIG2)
signal.Notify(ch2, SIG2)
def _():
signal.Reset()
defer(_)
for i in range(N):
# raise SIGUSR1 -> should be delivered to ch1 and ch12
# raise SIG1 -> should be delivered to ch1 and ch12
assert len(ch1) == 0
assert len(ch12) == 0
assert len(ch2) == 0
killme(syscall.SIGUSR1)
killme(SIG1)
waitfor(lambda: len(ch1) == 1 and len(ch12) == 1)
sig1 = ch1.recv()
sig12 = ch12.recv()
assert sig1 == syscall.SIGUSR1
assert sig12 == syscall.SIGUSR1
assert sig1 == SIG1
assert sig12 == SIG1
# raise SIGUSR2 -> should be delivered to ch12 and ch2
# raise SIG2 -> should be delivered to ch12 and ch2
assert len(ch1) == 0
assert len(ch12) == 0
assert len(ch2) == 0
killme(syscall.SIGUSR2)
killme(SIG2)
waitfor(lambda: len(ch12) == 1 and len(ch2) == 1)
sig12 = ch12.recv()
sig2 = ch2.recv()
assert sig12 == syscall.SIGUSR2
assert sig2 == syscall.SIGUSR2
# if SIGUSR2 will be eventually delivered to ch1 - it will break
# in SIGUSR1 check on next iteration.
assert sig12 == SIG2
assert sig2 == SIG2
# if SIG2 will be eventually delivered to ch1 - it will break
# in SIG1 check on next iteration.
# Stop(ch2) -> signals should not be delivered to ch2 anymore
signal.Stop(ch2)
for i in range(N):
# USR1 -> ch1, ch12
# SIG1 -> ch1, ch12
assert len(ch1) == 0
assert len(ch12) == 0
assert len(ch2) == 0
killme(syscall.SIGUSR1)
killme(SIG1)
waitfor(lambda: len(ch1) == 1 and len(ch12) == 1)
sig1 = ch1.recv()
sig12 = ch12.recv()
assert sig1 == syscall.SIGUSR1
assert sig12 == syscall.SIGUSR1
assert sig1 == SIG1
assert sig12 == SIG1
# USR2 -> ch12, !ch2
# SIG2 -> ch12, !ch2
assert len(ch1) == 0
assert len(ch12) == 0
assert len(ch2) == 0
killme(syscall.SIGUSR2)
killme(SIG2)
waitfor(lambda: len(ch12) == 1)
sig12 = ch12.recv()
assert sig12 == syscall.SIGUSR2
# if SIGUSR2 will be eventually delivered to ch2 - it will break on
assert sig12 == SIG2
# if SIG2 will be eventually delivered to ch2 - it will break on
# next iteration.
# Ignore(USR1) -> ch1 should not be delivered to anymore, ch12 should be delivered only USR2
signal.Ignore(syscall.SIGUSR1)
# Ignore(SIG1) -> ch1 should not be delivered to anymore, ch12 should be delivered only SIG2
signal.Ignore(SIG1)
for i in range(N):
# USR1 -> ø
# SIG1 -> ø
assert len(ch1) == 0
assert len(ch12) == 0
assert len(ch2) == 0
killme(syscall.SIGUSR1)
killme(SIG1)
time.sleep(1E-6)
# USR2 -> ch12
# SIG2 -> ch12
assert len(ch1) == 0
assert len(ch12) == 0
assert len(ch2) == 0
killme(syscall.SIGUSR2)
killme(SIG2)
waitfor(lambda: len(ch12) == 1)
sig12 = ch12.recv()
assert sig12 == syscall.SIGUSR2
# if SIGUSR1 or SIGUSR2 will be eventually delivered to ch1 or ch2 - it
assert sig12 == SIG2
# if SIG1 or SIG2 will be eventually delivered to ch1 or ch2 - it
# will break on next iteration.
# Notify after Ignore
signal.Notify(ch1, syscall.SIGUSR1)
signal.Notify(ch1, SIG1)
for i in range(N):
# USR1 -> ch1
# SIG1 -> ch1
assert len(ch1) == 0
assert len(ch12) == 0
assert len(ch2) == 0
killme(syscall.SIGUSR1)
killme(SIG1)
waitfor(lambda: len(ch1) == 1)
sig1 = ch1.recv()
assert sig1 == syscall.SIGUSR1
assert sig1 == SIG1
# USR2 -> ch12
# SIG2 -> ch12
assert len(ch1) == 0
assert len(ch12) == 0
assert len(ch2) == 0
killme(syscall.SIGUSR2)
killme(SIG2)
waitfor(lambda: len(ch12) == 1)
sig12 = ch12.recv()
assert sig12 == syscall.SIGUSR2
# if SIGUSR1 or SIGUSR2 will be eventually delivered to wrong place -
assert sig12 == SIG2
# if SIG1 or SIG2 will be eventually delivered to wrong place -
# it will break on next iteration.
# Reset is tested in test_stdlib_interop (it needs non-terminating default
......@@ -173,11 +188,11 @@ def test_notify_reinstall():
for i in range(N):
signal.Stop(ch)
signal.Notify(ch, syscall.SIGUSR1)
signal.Notify(ch, SIG1)
time.sleep(0.1*time.second)
assert len(ch) == 0
killme(syscall.SIGUSR1)
killme(SIG1)
time.sleep(0.1*time.second)
assert len(ch) == 1
......@@ -188,7 +203,7 @@ def test_signal_all():
assert b"ok (notify)" in out
assert b"ok (ignore)" in out
assert b"terminating ..." in out
assert retcode == -syscall.SIGTERM.signo
assert retcode == (-syscall.SIGTERM.signo if os.name != 'nt' else 3)
# test_stdlib_interop verifies that there is minimal compatibility in between
......@@ -206,23 +221,23 @@ def test_stdlib_interop():
ch1 = chan(2, dtype=object) # NOTE not gos.Signal nor 'C.os::Signal'
def _(signo, frame):
ch1.send("USR1")
pysig.signal(pysig.SIGUSR1, _)
ch1.send("SIG1")
pysig.signal(SIG1.signo, _)
def _():
pysig.signal(pysig.SIGUSR1, pysig.SIG_IGN)
pysig.signal(SIG1.signo, pysig.SIG_IGN)
defer(_)
# verify that plain pysig delivery works
for i in range(N):
assert len(ch1) == 0
killme(syscall.SIGUSR1)
killme(SIG1)
waitfor(lambda: len(ch1) == 1)
obj1 = ch1.recv()
assert obj1 == "USR1"
assert obj1 == "SIG1"
# verify that combined pysig + golang.os.signal delivery works
ch2 = chan(2, dtype=gos.Signal)
signal.Notify(ch2, syscall.SIGUSR1)
signal.Notify(ch2, SIG1)
def _():
signal.Stop(ch2)
defer(_)
......@@ -230,46 +245,46 @@ def test_stdlib_interop():
for i in range(N):
assert len(ch1) == 0
assert len(ch2) == 0
killme(syscall.SIGUSR1)
killme(SIG1)
waitfor(lambda: len(ch1) == 1 and len(ch2) == 1)
obj1 = ch1.recv()
sig2 = ch2.recv()
assert obj1 == "USR1"
assert sig2 == syscall.SIGUSR1
assert obj1 == "SIG1"
assert sig2 == SIG1
# Ignore stops delivery to both pysig and golang.os.signal
signal.Ignore(syscall.SIGUSR1)
signal.Ignore(SIG1)
for i in range(N):
assert len(ch1) == 0
assert len(ch2) == 0
killme(syscall.SIGUSR1)
killme(SIG1)
time.sleep(1E-6)
time.sleep(0.1) # just in case
assert len(ch1) == 0
assert len(ch2) == 0
# after Reset pysig delivery is restored even after Ignore
signal.Reset(syscall.SIGUSR1)
signal.Reset(SIG1)
for i in range(N):
assert len(ch1) == 0
assert len(ch2) == 0
killme(syscall.SIGUSR1)
killme(SIG1)
waitfor(lambda: len(ch1) == 1)
assert len(ch2) == 0
obj1 = ch1.recv()
assert obj1 == "USR1"
assert obj1 == "SIG1"
# Reset stops delivery to golang.os.signal and restores pysig delivery
signal.Notify(ch2, syscall.SIGUSR1)
signal.Reset(syscall.SIGUSR1)
signal.Notify(ch2, SIG1)
signal.Reset(SIG1)
for i in range(N):
assert len(ch1) == 0
assert len(ch2) == 0
killme(syscall.SIGUSR1)
killme(SIG1)
waitfor(lambda: len(ch1) == 1)
assert len(ch2) == 0
obj1 = ch1.recv()
assert obj1 == "USR1"
assert obj1 == "SIG1"
# test_stdlib_interop_KeyboardInterrupt verifies that signal.{Notify,Ignore} disable
......@@ -353,8 +368,9 @@ def test_stdlib_interop_KeyboardInterrupt():
# killme sends signal sig to own process.
def killme(sig):
mypid = os.getpid()
os.kill(mypid, sig.signo)
# use raise(sig) instead of kill(mypid, sig) so that it works on windows,
# where os.kill unconditionally terminates target process.
raise_signal(sig.signo)
# wait for waits until cond() becomes true or timeout.
def waitfor(cond):
......
#!/usr/bin/env python
# Copyright (C) 2021-2022 Nexedi SA and Contributors.
# Copyright (C) 2021-2023 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -24,7 +24,8 @@ from __future__ import print_function, absolute_import
from golang import chan
from golang import os as gos, syscall, time
from golang.os import signal
import os, sys
from golang.os.signal_test import killme
import sys
def main():
# build "all signals" list
......@@ -35,11 +36,17 @@ def main():
if sig not in allsigv: # avoid e.g. SIGCHLD/SIGCLD dups
allsigv.append(sig)
allsigv.sort(key=lambda sig: sig.signo)
allsigv.remove(syscall.SIGKILL) # SIGKILL/SIGSTOP cannot be caught
allsigv.remove(syscall.SIGSTOP)
allsigv.remove(syscall.SIGBUS) # AddressSanitizer crashes on SIGBUS/SIGFPE/SIGSEGV
allsigv.remove(syscall.SIGFPE)
allsigv.remove(syscall.SIGSEGV)
def without(signame):
sig = getattr(syscall, signame, None)
if sig is not None:
allsigv.remove(sig)
without('SIGKILL') # SIGKILL/SIGSTOP cannot be caught
without('SIGSTOP')
without('SIGBUS') # AddressSanitizer crashes on SIGBUS/SIGFPE/SIGSEGV
without('SIGFPE')
without('SIGSEGV')
without('SIGILL') # SIGILL/SIGABRT cause termination on windows
without('SIGABRT')
# Notify() -> kill * -> should be notified
ch = chan(10, dtype=gos.Signal)
......@@ -72,11 +79,6 @@ def main():
killme(syscall.SIGTERM)
raise AssertionError("not terminated")
# killme sends signal sig to own process.
def killme(sig):
mypid = os.getpid()
os.kill(mypid, sig.signo)
def emit(msg=''):
print(msg)
sys.stdout.flush()
......
# Copyright (C) 2019-2022 Nexedi SA and Contributors.
# Copyright (C) 2019-2023 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -73,8 +73,10 @@ _dso_build_ext = setuptools_dso.build_ext
class build_ext(_dso_build_ext):
def build_extension(self, ext):
# wrap _compiler <src> -> <obj> with our code
# ._compile is used on gcc/clang but not with msvc
_compile = self.compiler._compile
def _(obj, src, ext, cc_args, extra_postargs, pp_opts):
def xcompile(obj, src, ext, cc_args, extra_postargs, pp_opts):
# filter_out removes arguments that start with argprefix
cc_args = cc_args[:]
extra_postargs = extra_postargs[:]
......@@ -101,11 +103,30 @@ class build_ext(_dso_build_ext):
filter_out('-std=gnu++')
_compile(obj, src, ext, cc_args, extra_postargs, pp_opts)
self.compiler._compile = _
# msvc handles all sources directly in the loop in .compile and we can
# do per-source adjustsment only in .spawn .
spawn = self.compiler.spawn
def xspawn(argv):
c = False
for arg in argv:
if arg.startswith('/Tc'):
c = True
if c:
argv = argv[:]
for i in range(len(argv)):
if argv[i] == '/std:c++20':
argv[i] = '/std:c11'
return spawn(argv)
self.compiler._compile = xcompile
self.compiler.spawn = xspawn
try:
_dso_build_ext.build_extension(self, ext) # super doesn't work for _dso_build_ext
finally:
self.compiler._compile = _compile
self.compiler.spawn = spawn
# setup should be used instead of setuptools.setup
......@@ -128,14 +149,14 @@ def setup(**kw):
# x_dsos = [DSO('mypkg.mydso', ['mypkg/mydso.cpp'])],
# )
def DSO(name, sources, **kw):
_, kw = _with_build_defaults(kw)
_, kw = _with_build_defaults(name, kw)
dso = setuptools_dso.DSO(name, sources, **kw)
return dso
# _with_build_defaults returns copy of kw amended with build options common for
# both DSO and Extension.
def _with_build_defaults(kw): # -> (pygo, kw')
def _with_build_defaults(name, kw): # -> (pygo, kw')
# find pygolang root
gopkg = _findpkg("golang")
pygo = dirname(gopkg.path) # .../pygolang/golang -> .../pygolang
......@@ -144,15 +165,22 @@ def _with_build_defaults(kw): # -> (pygo, kw')
kw = kw.copy()
# prepend -I<pygolang> so that e.g. golang/libgolang.h is found
sysname = platform.system().lower()
win = (sysname == 'windows')
msvc = win # TODO also support mingw ?
# prepend -I<pygolang> so that e.g. golang/libgolang.h is found
# same with -I<pygolang>/golang/_compat/<os>
incv = kw.get('include_dirs', [])[:]
incv.insert(0, pygo)
incv.insert(1, join(pygo, 'golang', '_compat', sysname))
kw['include_dirs'] = incv
# link with libgolang.so
dsov = kw.get('dsos', [])[:]
dsov.insert(0, 'golang.runtime.libgolang')
kw['dsos'] = dsov
# link with libgolang.so if it is not libgolang itself
if name != 'golang.runtime.libgolang':
dsov = kw.get('dsos', [])[:]
dsov.insert(0, 'golang.runtime.libgolang')
kw['dsos'] = dsov
# default language to C++ (chan[T] & co are accessible only via C++)
lang = kw.setdefault('language', 'c++')
......@@ -160,9 +188,20 @@ def _with_build_defaults(kw): # -> (pygo, kw')
# default to C++11 (chan[T] & co require C++11 features)
ccdefault = []
if lang == 'c++':
ccdefault.append('-std=c++11')
if not msvc:
if name == 'golang.runtime.libgolang':
ccdefault.append('-std=gnu++11') # not c++11 as linux/list.h uses typeof
else:
ccdefault.append('-std=c++11')
else:
# MSVC requries C++20 for designated struct initializers
ccdefault.append('/std:c++20')
# and make exception handling: "fully conformant", so that e.g. extern "C" `panic` works
ccdefault.extend(['/EHc-', '/EHsr'])
# default to no strict-aliasing
ccdefault.append('-fno-strict-aliasing')
if not msvc:
ccdefault.append('-fno-strict-aliasing') # use only on gcc/clang - msvc does it by default
_ = kw.get('extra_compile_args', [])[:]
_[0:0] = ccdefault # if another e.g. -std=... was already there -
......@@ -188,6 +227,8 @@ def _with_build_defaults(kw): # -> (pygo, kw')
'os/signal.h',
'pyx/runtime.h',
'_testing.h',
'_compat/windows/strings.h',
'_compat/windows/unistd.h',
]])
kw['depends'] = dependv
......@@ -203,7 +244,7 @@ def _with_build_defaults(kw): # -> (pygo, kw')
# ext_modules = [Extension('mypkg.mymod', ['mypkg/mymod.pyx'])],
# )
def Extension(name, sources, **kw):
pygo, kw = _with_build_defaults(kw)
pygo, kw = _with_build_defaults(name, kw)
# some pyx-level depends to workaround a bit lack of proper dependency
# tracking in setuptools/distutils.
......
#ifndef _NXD_LIBGOLANG_PYX_RUNTIME_H
#define _NXD_LIBGOLANG_PYX_RUNTIME_H
// Copyright (C) 2018-2019 Nexedi SA and Contributors.
// Copyright (C) 2018-2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -58,8 +58,8 @@ class _PyError final : public _error, object {
private:
_PyError();
~_PyError();
friend error PyErr_Fetch();
friend void PyErr_ReRaise(PyError pyerr);
friend LIBPYXRUNTIME_API error PyErr_Fetch();
friend LIBPYXRUNTIME_API void PyErr_ReRaise(PyError pyerr);
public:
LIBPYXRUNTIME_API void incref();
LIBPYXRUNTIME_API void decref();
......
// Copyright (C) 2019 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
// Copyright (C) 2019-2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
......@@ -20,7 +20,7 @@
// Small library that uses a bit of libgolang features, mainly to verify
// that external project can build against libgolang.
#include <golang/libgolang.h>
#include "dso.h"
using namespace golang;
#include <stdio.h>
......
// Copyright (C) 2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
#ifndef _NXD_GOLANG_DSOUSER_DSO_H
#define _NXD_GOLANG_DSOUSER_DSO_H
#include <golang/libgolang.h>
#if BUILDING_DSOUSER_DSO
# define DSOUSER_DSO_API LIBGOLANG_DSO_EXPORT
#else
# define DSOUSER_DSO_API LIBGOLANG_DSO_IMPORT
#endif
DSOUSER_DSO_API void dsotest();
#endif // _NXD_GOLANG_DSOUSER_DSO_H
# cython: language_level=2
#
# Copyright (C) 2019 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
# Copyright (C) 2019-2023 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
......@@ -21,10 +21,7 @@
# Small test driver that calls dso.so .
cdef extern from * nogil:
"""
void dsotest();
"""
cdef extern from "dso.h" nogil:
void dsotest()
def main():
......
# Copyright (C) 2019 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
# Copyright (C) 2019-2023 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
......@@ -24,7 +24,9 @@ setup(
name = 'golang_dso_user',
description = 'test project that uses libgolang from a dso',
x_dsos = [DSO('dsouser.dso', ['dsouser/dso.cpp'])],
x_dsos = [DSO('dsouser.dso',
['dsouser/dso.cpp'],
define_macros = [('BUILDING_DSOUSER_DSO', None)])],
ext_modules = [Extension('dsouser.test',
['dsouser/test.pyx'],
dsos = ['dsouser.dso'])],
......
# cython: language_level=2
# Copyright (C) 2019-2022 Nexedi SA and Contributors.
# Copyright (C) 2019-2023 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -57,7 +57,7 @@ from posix.fcntl cimport mode_t, F_GETFL, F_SETFL, O_NONBLOCK, O_ACCMODE, O_RDON
from posix.stat cimport struct_stat, S_ISREG, S_ISDIR, S_ISBLK
from posix.strings cimport bzero
from gevent.fileobject import FileObjectThread, FileObjectPosix
from gevent import fileobject as gfileobj
# _goviapy & _togo serve go
......@@ -194,31 +194,39 @@ cdef nogil:
return ioh
_libgolang_ioh* _io_fdopen(int *out_syserr, int sysfd):
# check if we should enable O_NONBLOCK on this file-descriptor
# even though we could enable O_NONBLOCK for regular files, it does not
# work as expected as most unix'es report regular files as always read
# and write ready.
cdef struct_stat st
cdef int syserr = syscall.Fstat(sysfd, &st)
if syserr < 0:
out_syserr[0] = syserr
return NULL
m = st.st_mode
blocking = (S_ISREG(m) or S_ISDIR(m) or S_ISBLK(m)) # fd cannot refer to symlink
# retrieve current sysfd flags and access mode
flags = syscall.Fcntl(sysfd, F_GETFL, 0)
if flags < 0:
out_syserr[0] = flags
return NULL
acc = (flags & O_ACCMODE)
# enable O_NONBLOCK if needed
if not blocking:
syserr = syscall.Fcntl(sysfd, F_SETFL, flags | O_NONBLOCK)
IF POSIX:
# check if we should enable O_NONBLOCK on this file-descriptor
# even though we could enable O_NONBLOCK for regular files, it does not
# work as expected as most unix'es report regular files as always read
# and write ready.
cdef struct_stat st
cdef int syserr = syscall.Fstat(sysfd, &st)
if syserr < 0:
out_syserr[0] = syserr
return NULL
m = st.st_mode
blocking = (S_ISREG(m) or S_ISDIR(m) or S_ISBLK(m)) # fd cannot refer to symlink
# retrieve current sysfd flags and access mode
flags = syscall.Fcntl(sysfd, F_GETFL, 0)
if flags < 0:
out_syserr[0] = flags
return NULL
acc = (flags & O_ACCMODE)
# enable O_NONBLOCK if needed
if not blocking:
syserr = syscall.Fcntl(sysfd, F_SETFL, flags | O_NONBLOCK)
if syserr < 0:
out_syserr[0] = syserr
return NULL
ELSE: # !POSIX (windows)
cdef bint blocking = True
# FIXME acc: use GetFileInformationByHandleEx to determine whether
# HANDLE(sysfd) was opened for reading or writing.
# https://stackoverflow.com/q/9442436/9456786
cdef int acc = O_RDWR
# create IOH backed by FileObjectThread or FileObjectPosix
ioh = <IOH*>calloc(1, sizeof(IOH))
......@@ -253,9 +261,9 @@ cdef:
pygfobj = None
try:
if blocking:
pygfobj = FileObjectThread(sysfd, mode=mode, buffering=0)
pygfobj = gfileobj.FileObjectThread(sysfd, mode=mode, buffering=0)
else:
pygfobj = FileObjectPosix(sysfd, mode=mode, buffering=0)
pygfobj = gfileobj.FileObjectPosix(sysfd, mode=mode, buffering=0)
except OSError as e:
out_syserr[0] = -e.errno
else:
......@@ -338,7 +346,19 @@ cdef:
cdef uint8_t[::1] mem = <uint8_t[:count]>buf
xmem = memoryview(mem) # to avoid https://github.com/cython/cython/issues/3900 on mem[:0]=b''
try:
n = pygfobj.readinto(xmem)
# NOTE buf might be on stack, so it must not be accessed, e.g. from
# FileObjectThread, while our greenlet is parked (see STACK_DEAD_WHILE_PARKED
# for details). -> Use intermediate on-heap buffer to protect from that.
#
# Also: we cannot use pygfobj.readinto due to
# https://github.com/gevent/gevent/pull/1948
#
# TODO use .readinto() when buf is not on stack and gevent is
# recent enough or pygfobj is not FileObjectThread.
#n = pygfobj.readinto(xmem)
buf2 = pygfobj.read(count)
n = len(buf2)
xmem[:n] = buf2
except OSError as e:
n = -e.errno
out_n[0] = n
......@@ -361,8 +381,14 @@ cdef:
bint _io_write(IOH* ioh, int* out_n, const void *buf, size_t count):
pygfobj = <object>ioh.pygfobj
cdef const uint8_t[::1] mem = <const uint8_t[:count]>buf
# NOTE buf might be on stack, so it must not be accessed, e.g. from
# FileObjectThread, while our greenlet is parked (see STACK_DEAD_WHILE_PARKED
# for details). -> Use intermediate on-heap buffer to protect from that.
buf2 = bytearray(mem)
try:
n = pygfobj.write(mem)
n = pygfobj.write(buf2)
except OSError as e:
n = -e.errno
out_n[0] = n
......
// Copyright (C) 2022 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
// Copyright (C) 2022-2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
......@@ -20,7 +20,9 @@
#include "golang/runtime/internal/atomic.h"
#include "golang/libgolang.h"
#ifndef _WIN32
#include <pthread.h>
#endif
// golang::internal::atomic::
namespace golang {
......@@ -41,9 +43,12 @@ static void _forkNewEpoch() {
}
void _init() {
// there is no fork on windows
#ifndef _WIN32
int e = pthread_atfork(/*prepare*/nil, /*inparent*/nil, /*inchild*/_forkNewEpoch);
if (e != 0)
panic("pthread_atfork failed");
#endif
}
......
// Copyright (C) 2021-2022 Nexedi SA and Contributors.
// Copyright (C) 2021-2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -57,15 +57,18 @@ string _Errno::Error() {
_Errno& e = *this;
char ebuf[128];
bool ok;
#if __APPLE__
int x = ::strerror_r(-e.syserr, ebuf, sizeof(ebuf));
if (x == 0)
return string(ebuf);
return "unknown error " + std::to_string(-e.syserr);
ok = (::strerror_r(-e.syserr, ebuf, sizeof(ebuf)) == 0);
#elif defined(_WIN32)
ok = (::strerror_s(ebuf, sizeof(ebuf), -e.syserr) == 0);
#else
char *estr = ::strerror_r(-e.syserr, ebuf, sizeof(ebuf));
return string(estr);
#endif
if (ok)
return string(ebuf);
return "unknown error " + std::to_string(-e.syserr);
}
......@@ -99,6 +102,7 @@ __Errno Close(int fd) {
return err;
}
#ifndef _WIN32
__Errno Fcntl(int fd, int cmd, int arg) {
int save_errno = errno;
int err = ::fcntl(fd, cmd, arg);
......@@ -107,6 +111,7 @@ __Errno Fcntl(int fd, int cmd, int arg) {
errno = save_errno;
return err;
}
#endif
__Errno Fstat(int fd, struct ::stat *out_st) {
int save_errno = errno;
......@@ -119,6 +124,10 @@ __Errno Fstat(int fd, struct ::stat *out_st) {
int Open(const char *path, int flags, mode_t mode) {
int save_errno = errno;
#ifdef _WIN32 // default to open files in binary mode
if ((flags & (_O_TEXT | _O_BINARY)) == 0)
flags |= _O_BINARY;
#endif
int fd = ::open(path, flags, mode);
if (fd < 0)
fd = -errno;
......@@ -126,15 +135,39 @@ int Open(const char *path, int flags, mode_t mode) {
return fd;
}
__Errno Pipe(int vfd[2]) {
__Errno Pipe(int vfd[2], int flags) {
// supported flags: O_CLOEXEC
if (flags & ~(O_CLOEXEC))
return -EINVAL;
int save_errno = errno;
int err = ::pipe(vfd);
int err;
#ifdef __linux__
err = ::pipe2(vfd, flags);
#elif defined(_WIN32)
err = ::_pipe(vfd, 4096, flags | _O_BINARY);
#else
err = ::pipe(vfd);
if (err)
goto out;
if (flags & O_CLOEXEC) {
for (int i=0; i<2; i++) {
err = Fcntl(vfd[i], F_SETFD, FD_CLOEXEC);
if (err) {
Close(vfd[0]);
Close(vfd[1]);
goto out;
}
}
}
out:
#endif
if (err == -1)
err = -errno;
errno = save_errno;
return err;
}
#ifndef _WIN32
__Errno Sigaction(int signo, const struct ::sigaction *act, struct ::sigaction *oldact) {
int save_errno = errno;
int err = ::sigaction(signo, act, oldact);
......@@ -143,6 +176,17 @@ __Errno Sigaction(int signo, const struct ::sigaction *act, struct ::sigaction *
errno = save_errno;
return err;
}
#endif
sighandler_t Signal(int signo, sighandler_t handler, int *out_psyserr) {
int save_errno = errno;
sighandler_t oldh = ::signal(signo, handler);
*out_psyserr = (oldh == SIG_ERR
? (errno ? -errno : -EINVAL) // windows returns SIG_ERR/errno=0 for invalid signo
: 0);
errno = save_errno;
return oldh;
}
}}} // golang::internal::syscall::
#ifndef _NXD_LIBGOLANG_RUNTIME_INTERNAL_SYSCALL_H
#define _NXD_LIBGOLANG_RUNTIME_INTERNAL_SYSCALL_H
// Copyright (C) 2021-2022 Nexedi SA and Contributors.
// Copyright (C) 2021-2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -59,15 +59,21 @@ error NewErrno(__Errno syserr); // TODO better return Errno directly.
// system calls
int/*n|err*/ Read(int fd, void *buf, size_t count);
int/*n|err*/ Write(int fd, const void *buf, size_t count);
__Errno Close(int fd);
__Errno Fcntl(int fd, int cmd, int arg);
__Errno Fstat(int fd, struct ::stat *out_st);
int/*fd|err*/ Open(const char *path, int flags, mode_t mode);
__Errno Pipe(int vfd[2]);
__Errno Sigaction(int signo, const struct ::sigaction *act, struct ::sigaction *oldact);
LIBGOLANG_API int/*n|err*/ Read(int fd, void *buf, size_t count);
LIBGOLANG_API int/*n|err*/ Write(int fd, const void *buf, size_t count);
LIBGOLANG_API __Errno Close(int fd);
#ifndef _WIN32
LIBGOLANG_API __Errno Fcntl(int fd, int cmd, int arg);
#endif
LIBGOLANG_API __Errno Fstat(int fd, struct ::stat *out_st);
LIBGOLANG_API int/*fd|err*/ Open(const char *path, int flags, mode_t mode);
LIBGOLANG_API __Errno Pipe(int vfd[2], int flags);
#ifndef _WIN32
LIBGOLANG_API __Errno Sigaction(int signo, const struct ::sigaction *act, struct ::sigaction *oldact);
#endif
typedef void (*sighandler_t)(int);
LIBGOLANG_API sighandler_t /*sigh|SIG_ERR*/ Signal(int signo, sighandler_t handler, int *out_psyserr);
}}} // golang::internal::syscall::
......
// Copyright (C) 2018-2022 Nexedi SA and Contributors.
// Copyright (C) 2018-2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -43,12 +43,23 @@
#include <stdlib.h>
#include <string.h>
#include <strings.h>
// linux/list.h needs ARRAY_SIZE XXX -> better use c.h or ccan/array_size.h ?
#ifndef ARRAY_SIZE
# define ARRAY_SIZE(A) (sizeof(A) / sizeof((A)[0]))
#endif
#include <linux/list.h>
// MSVC does not support statement expressions and typeof
// -> redo list_entry via C++ lambda.
#ifdef _MSC_VER
# undef list_entry
# define list_entry(ptr, type, member) [&]() { \
const decltype( ((type *)0)->member ) *__mptr = (ptr); \
return (type *)( (char *)__mptr - offsetof(type,member) ); \
}()
#endif
using std::atomic;
using std::bad_alloc;
......@@ -899,6 +910,13 @@ template<> int _chanselect2</*onstack=*/true> (const _selcase *, int, const vect
template<> int _chanselect2</*onstack=*/false>(const _selcase *, int, const vector<int>&);
static int __chanselect2(const _selcase *, int, const vector<int>&, _WaitGroup*);
// PRNG for select.
// TODO consider switching to xoroshiro or wyrand if speed is an issue
// https://thompsonsed.co.uk/random-number-generators-for-c-performance-tested
// https://nullprogram.com/blog/2017/09/21/
static std::random_device _devrand;
static thread_local std::mt19937 _t_rng(_devrand());
// _chanselect executes one ready send or receive channel case.
//
// if no case is ready and default case was provided, select chooses default.
......@@ -925,7 +943,7 @@ int _chanselect(const _selcase *casev, int casec) {
vector<int> nv(casec); // n -> n(case) TODO -> caller stack-allocate nv
for (int i=0; i <casec; i++)
nv[i] = i;
std::random_shuffle(nv.begin(), nv.end());
std::shuffle(nv.begin(), nv.end(), _t_rng);
// first pass: poll all cases and bail out in the end if default was provided
int ndefault = -1;
......
#ifndef _NXD_LIBGOLANG_SYNC_H
#define _NXD_LIBGOLANG_SYNC_H
// Copyright (C) 2018-2020 Nexedi SA and Contributors.
// Copyright (C) 2018-2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -212,7 +212,7 @@ class _WorkGroup : public object {
private:
_WorkGroup();
~_WorkGroup();
friend WorkGroup NewWorkGroup(context::Context ctx);
friend LIBGOLANG_API WorkGroup NewWorkGroup(context::Context ctx);
public:
LIBGOLANG_API void decref();
......
...
_____________________________________ main _____________________________________
...________________ main ________________...
../__init__.py:...: in _
return f(*argv, **kw)
golang_test_defer_excchain.py:42: in main
......
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (C) 2022 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
# Copyright (C) 2022-2023 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
......@@ -29,8 +29,8 @@ from golang import b, u
from golang.gcompat import qq
def main():
sb = b("привет b")
su = u("привет u")
sb = b("привет αβγ b")
su = u("привет αβγ u")
print("print(qq(b)):", qq(sb))
print("print(qq(u)):", qq(su))
......
print(qq(b)): "привет b"
print(qq(u)): "привет u"
print(qq(b)): "привет αβγ b"
print(qq(u)): "привет αβγ u"
#ifndef _NXD_LIBGOLANG_TIME_H
#define _NXD_LIBGOLANG_TIME_H
// Copyright (C) 2019-2020 Nexedi SA and Contributors.
// Copyright (C) 2019-2023 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -123,7 +123,7 @@ private:
private:
_Ticker();
~_Ticker();
friend Ticker new_ticker(double dt);
friend Ticker LIBGOLANG_API new_ticker(double dt);
public:
LIBGOLANG_API void decref();
......
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (C) 2018-2021 Nexedi SA and Contributors.
# Copyright (C) 2018-2023 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -50,19 +50,27 @@ _pyopt_long = ('version',)
# init, if provided, is called after options are parsed, but before interpreter start.
def pymain(argv, init=None):
import sys
from os.path import dirname, realpath
import os
from os.path import dirname, realpath, splitext
# sys.executable
# on windows there are
# gpython-script.py
# gpython.exe
# gpython-script.py
# gpython.manifest
# and argv[0] is gpython-script.py
# with gpython-script.py sometimes embedded into gpython.exe (pip/distlib)
# and argv[0] is 'gpython-script.py' (setuptools) or 'gpython' (pip/distlib; note no .exe)
exe = realpath(argv[0])
argv = argv[1:]
if exe.endswith('-script.py'):
exe = exe[:-len('-script.py')]
exe = exe + '.exe'
if os.name == 'nt':
if exe.endswith('-script.py'):
exe = exe[:-len('-script.py')] # gpython-script.py -> gpython.exe
exe = exe + '.exe'
else:
_, ext = splitext(exe) # gpython -> gpython.exe
if not ext:
exe += '.exe'
sys._gpy_underlying_executable = sys.executable
sys.executable = exe
......@@ -70,20 +78,26 @@ def pymain(argv, init=None):
# `gpython file` will add path-to-file to sys.path[0] by itself, and
# /path/to/gpython is unnecessary and would create difference in behaviour
# in between gpython and python.
#
# on windows when gpython.exe comes with embedded __main__.py, it is
# gpython.exe that is installed into sys.path[0] .
exedir = dirname(exe)
if sys.path[0] == exedir:
if sys.path[0] in (exedir, exe):
del sys.path[0]
else:
# buildout injects `sys.path[0:0] = eggs` into python scripts.
# detect that and remove sys.path entry corresponding to exedir.
if not _is_buildout_script(exe):
raise RuntimeError('pymain: internal error: sys.path[0] was not set by underlying python to dirname(exe):'
raise RuntimeError('pymain: internal error: sys.path[0] was not set by underlying python to dirname(exe) or exe:'
'\n\n\texe:\t%s\n\tsys.path[0]:\t%s' % (exe, sys.path[0]))
else:
if exedir in sys.path:
sys.path.remove(exedir)
else:
raise RuntimeError('pymain: internal error: sys.path does not contain dirname(exe):'
ok = False
for _ in (exedir, exe):
if _ in sys.path:
sys.path.remove(_)
ok = True
if not ok:
raise RuntimeError('pymain: internal error: sys.path does not contain dirname(exe) or exe:'
'\n\n\texe:\t%s\n\tsys.path:\t%s' % (exe, sys.path))
......@@ -202,7 +216,6 @@ def pymain(argv, init=None):
#
# python -O gpython file.py
if len(reexec_with) > 0:
import os
argv = [sys._gpy_underlying_executable] + reexec_with + [sys.executable] + reexec_argv
os.execv(argv[0], argv)
......@@ -439,13 +452,13 @@ def main():
# _is_buildout_script returns whether file @path is generated as python buildout script.
def _is_buildout_script(path):
with open(path, 'r') as f:
with open(path, 'rb') as f:
src = f.read()
# buildout injects the following prologues into python scripts:
# sys.path[0:0] = [
# ...
# ]
return ('\nsys.path[0:0] = [\n' in src)
return (b'\nsys.path[0:0] = [\n' in src)
# _IGetOpt provides getopt-style incremental options parsing.
......
# -*- coding: utf-8 -*-
# Copyright (C) 2019-2021 Nexedi SA and Contributors.
# Copyright (C) 2019-2023 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -180,7 +180,7 @@ def test_pymain():
_ = pyout(['-m', 'hello', 'abc', 'def'], cwd=testdata)
# realpath rewrites e.g. `local/lib -> lib` if local/lib is symlink
hellopy = realpath(join(testdata, 'hello.py'))
assert _ == b"hello\nworld\n['%s', 'abc', 'def']\n" % b(hellopy)
assert _ == b"hello\nworld\n[%s, 'abc', 'def']\n" % b(repr(hellopy))
# -m<module>
__ = pyout(['-mhello', 'abc', 'def'], cwd=testdata)
assert __ == _
......@@ -191,8 +191,8 @@ def test_pymain():
# -i after stdin (also tests interactive mode as -i forces interactive even on non-tty)
d = {
b'hellopy': b(hellopy),
b'ps1': b'' # cpython emits prompt to stderr
b'repr(hellopy)': b(repr(hellopy)),
b'ps1': b'' # cpython emits prompt to stderr
}
if is_pypy and not is_gpython:
d[b'ps1'] = b'>>>> ' # native pypy emits prompt to stdout and >>>> instead of >>>
......@@ -208,7 +208,7 @@ def test_pymain():
assert _ == b"hello\nworld\n['-c']\n%(ps1)s'~~HELLO~~'\n%(ps1)s" % d
# -i after -m
_ = pyout(['-i', '-m', 'hello'], stdin=b'world.tag', cwd=testdata)
assert _ == b"hello\nworld\n['%(hellopy)s']\n%(ps1)s'~~WORLD~~'\n%(ps1)s" % d
assert _ == b"hello\nworld\n[%(repr(hellopy))s]\n%(ps1)s'~~WORLD~~'\n%(ps1)s" % d
# -i after file
_ = pyout(['-i', 'testdata/hello.py'], stdin=b'tag', cwd=here)
assert _ == b"hello\nworld\n['testdata/hello.py']\n%(ps1)s'~~HELLO~~'\n%(ps1)s" % d
......
# -*- coding: utf-8 -*-
# Copyright (C) 2020 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
# Copyright (C) 2020-2023 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
......@@ -61,9 +61,10 @@ def modpy_imports_from():
else:
raise AssertionError("module 'mod' is already there")
tmpd = tempfile.mkdtemp('', 'modpy_imports_from')
tmpd = tempfile.mkdtemp('', 'modpy_imports_from')
tmpd_ = tmpd + os.path.sep
try:
pymod = "%s/mod.py" % tmpd
pymod = tmpd_ + "mod.py"
with open(pymod, "w") as f:
f.write("# hello up there\n")
......@@ -73,9 +74,9 @@ def modpy_imports_from():
files = set()
for dirpath, dirnames, filenames in os.walk(tmpd):
for _ in filenames:
f = '%s/%s' % (dirpath, _)
if f.startswith(tmpd+'/'):
f = f[len(tmpd+'/'):]
f = os.path.join(dirpath, _)
if f.startswith(tmpd_):
f = f[len(tmpd_):]
files.add(f)
......
[build-system]
requires = ["setuptools", "wheel", "setuptools_dso >= 1.7", "cython", "gevent"]
requires = ["setuptools", "wheel", "setuptools_dso >= 2.7", "cython", "gevent"]
......@@ -19,28 +19,27 @@
# See https://www.nexedi.com/licensing for rationale and options.
from setuptools import find_packages
# setuptools has Library but this days it is not well supported and test for it
# has been killed https://github.com/pypa/setuptools/commit/654c26f78a30
# -> use setuptools_dso instead.
from setuptools_dso import DSO
from setuptools.command.install_scripts import install_scripts as _install_scripts
from setuptools.command.develop import develop as _develop
from distutils import sysconfig
from os.path import dirname, join
import sys, re
import sys, os, re
# read file content
def readfile(path):
with open(path, 'r') as f:
return f.read()
# reuse golang.pyx.build to build pygolang extensions.
def readfile(path): # -> str
with open(path, 'rb') as f:
data = f.read()
if not isinstance(data, str): # py3
data = data.decode('utf-8')
return data
# reuse golang.pyx.build to build pygolang dso and extensions.
# we have to be careful and inject synthetic golang package in order to be
# able to import golang.pyx.build without built/working golang.
trun = {}
exec(readfile('trun'), trun)
trun['ximport_empty_golangmod']()
from golang.pyx.build import setup, Extension as Ext
from golang.pyx.build import setup, DSO, Extension as Ext
# grep searches text for pattern.
......@@ -156,7 +155,7 @@ class develop(XInstallGPython, _develop):
# requirements of packages under "golang." namespace
R = {
'cmd.pybench': {'pytest'},
'pyx.build': {'setuptools', 'wheel', 'cython', 'setuptools_dso >= 1.7'},
'pyx.build': {'setuptools', 'wheel', 'cython', 'setuptools_dso >= 2.7'},
'x.perf.benchlib': {'numpy'},
}
# TODO generate `a.b -> a`, e.g. x.perf = join(x.perf.*); x = join(x.*)
......@@ -174,6 +173,14 @@ for k in sorted(R.keys()):
extras_require[k] = list(sorted(R[k]))
# get_python_libdir() returns path where libpython is located
def get_python_libdir():
# mimic what distutils.command.build_ext does
if os.name == 'nt':
return join(sysconfig.get_config_var('installed_platbase'), 'libs')
else:
return sysconfig.get_config_var('LIBDIR')
setup(
name = 'pygolang',
version = version,
......@@ -223,19 +230,17 @@ setup(
'golang/strings.h',
'golang/sync.h',
'golang/time.h'],
include_dirs = ['.', '3rdparty/include'],
include_dirs = ['3rdparty/include'],
define_macros = [('BUILDING_LIBGOLANG', None)],
extra_compile_args = ['-std=gnu++11'], # not c++11 as linux/list.h uses typeof
soversion = '0.1'),
DSO('golang.runtime.libpyxruntime',
['golang/runtime/libpyxruntime.cpp'],
depends = ['golang/pyx/runtime.h'],
include_dirs = ['.', sysconfig.get_python_inc()],
include_dirs = [sysconfig.get_python_inc()],
library_dirs = [get_python_libdir()],
define_macros = [('BUILDING_LIBPYXRUNTIME', None)],
extra_compile_args = ['-std=c++11'],
soversion = '0.1',
dsos = ['golang.runtime.libgolang'])],
soversion = '0.1')],
ext_modules = [
Ext('golang._golang',
......@@ -311,11 +316,12 @@ setup(
include_package_data = True,
install_requires = ['gevent', 'six', 'decorator', 'Importing;python_version<="2.7"',
# only runtime part: for dylink_prepare_dso
'setuptools_dso >= 2.7',
# pyx.build -> setuptools_dso uses multiprocessing
# FIXME geventmp fails on python2, but setuptools_dso
# uses multiprocessing only on Python3, so for now we
# are ok. https://github.com/karellen/geventmp/pull/2
'geventmp;python_version>="3"',
# setuptools_dso uses multiprocessing only on Python3, and only on systems where
# mp.get_start_method()!='fork', while geventmp does not work on windows.
'geventmp ; python_version>="3" and platform_system != "Windows" ',
],
extras_require = extras_require,
......@@ -349,6 +355,11 @@ setup(
Programming Language :: Python :: 3.11
Programming Language :: Python :: Implementation :: CPython
Programming Language :: Python :: Implementation :: PyPy
Operating System :: POSIX
Operating System :: POSIX :: Linux
Operating System :: Unix
Operating System :: MacOS
Operating System :: Microsoft :: Windows
Topic :: Software Development :: Interpreters
Topic :: Software Development :: Libraries :: Python Modules\
""".splitlines()]
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment