Skip to content

N
nemu3
Commit ba619ec3 authored by Tom Niget's avatar Tom Niget
Browse files
Options

nemu works in python 3

parent f8914e80
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 340 additions and 225 deletions
src/nemu/compat.py 0 → 100644
View file @ ba619ec3
import os
import socket as pysocket
def pipe() -> tuple[int, int]:
a, b = os.pipe2(0)
os.set_inheritable(a, True)
os.set_inheritable(b, True)
return a, b
def socket(*args, **kwargs) -> pysocket.socket:
s = pysocket.socket(*args, **kwargs)
s.set_inheritable(True)
return s
def socketpair(*args, **kwargs) -> tuple[pysocket.socket, pysocket.socket]:
a, b = pysocket.socketpair(*args, **kwargs)
a.set_inheritable(True)
b.set_inheritable(True)
return a, b
def fromfd(*args, **kwargs) -> pysocket.socket:
s = pysocket.fromfd(*args, **kwargs)
s.set_inheritable(True)
return s
def fdopen(*args, **kwargs) -> pysocket.socket:
s = os.fdopen(*args, **kwargs)
s.set_inheritable(True)
return s
\ No newline at end of file
src/nemu/environ.py
View file @ ba619ec3
......@@ -25,8 +25,12 @@ import subprocess
import sys
import syslog
from syslog import LOG_ERR, LOG_WARNING, LOG_NOTICE, LOG_INFO, LOG_DEBUG
from typing import TypeVar, Callable
__all__ = ["IP_PATH", "TC_PATH", "BRCTL_PATH", "SYSCTL_PATH", "HZ"]
from nemu import compat
__all__ += ["TCPDUMP_PATH", "NETPERF_PATH", "XAUTH_PATH", "XDPYINFO_PATH"]
__all__ += ["execute", "backticks", "eintr_wrapper"]
__all__ += ["find_listen_port"]
......@@ -35,14 +39,14 @@ __all__ += ["set_log_level", "logger"]
__all__ += ["error", "warning", "notice", "info", "debug"]
def find_bin(name, extra_path = None):
def find_bin(name, extra_path=None):
"""Try hard to find the location of needed programs."""
search = []
if "PATH" in os.environ:
search += os.environ["PATH"].split(":")
search.extend(os.path.join(x, y)
for x in ("/", "/usr/", "/usr/local/")
for y in ("bin", "sbin"))
for x in ("/", "/usr/", "/usr/local/")
for y in ("bin", "sbin"))
if extra_path:
search += extra_path
......@@ -52,16 +56,18 @@ def find_bin(name, extra_path = None):
return path
return None
def find_bin_or_die(name, extra_path = None):
def find_bin_or_die(name, extra_path=None):
"""Try hard to find the location of needed programs; raise on failure."""
res = find_bin(name, extra_path)
if not res:
raise RuntimeError("Cannot find `%s', impossible to continue." % name)
return res
IP_PATH = find_bin_or_die("ip")
TC_PATH = find_bin_or_die("tc")
BRCTL_PATH = find_bin_or_die("brctl")
IP_PATH = find_bin_or_die("ip")
TC_PATH = find_bin_or_die("tc")
BRCTL_PATH = find_bin_or_die("brctl")
SYSCTL_PATH = find_bin_or_die("sysctl")
# Optional tools
......@@ -78,21 +84,23 @@ try:
os.stat("/sys/class/net")
except:
raise RuntimeError("Sysfs does not seem to be mounted, impossible to " +
"continue.")
"continue.")
def execute(cmd):
def execute(cmd: list[str]):
"""Execute a command, if the return value is non-zero, raise an exception.
Raises:
RuntimeError: the command was unsuccessful (return code != 0).
"""
debug("execute(%s)" % cmd)
proc = subprocess.Popen(cmd, stdout = subprocess.DEVNULL, stderr = subprocess.PIPE)
proc = subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
_, err = proc.communicate()
if proc.returncode != 0:
raise RuntimeError("Error executing `%s': %s" % (" ".join(cmd), err))
def backticks(cmd):
def backticks(cmd: list[str]) -> str:
"""Execute a command and capture its output.
If the return value is non-zero, raise an exception.
......@@ -102,30 +110,35 @@ def backticks(cmd):
RuntimeError: the command was unsuccessful (return code != 0).
"""
debug("backticks(%s)" % cmd)
proc = subprocess.Popen(cmd, stdout = subprocess.PIPE,
stderr = subprocess.PIPE)
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out, err = proc.communicate()
if proc.returncode != 0:
raise RuntimeError("Error executing `%s': %s" % (" ".join(cmd), err))
return out.decode("utf-8")
def eintr_wrapper(func, *args):
T = TypeVar("T")
def eintr_wrapper(func: Callable[..., T], *args) -> T:
"Wraps some callable with a loop that retries on EINTR."
while True:
try:
return func(*args)
except OSError as ex: # pragma: no cover
except OSError as ex: # pragma: no cover
if ex.errno == errno.EINTR:
continue
raise
except IOError as ex: # pragma: no cover
except IOError as ex: # pragma: no cover
if ex.errno == errno.EINTR:
continue
raise
def find_listen_port(family = socket.AF_INET, type = socket.SOCK_STREAM,
proto = 0, addr = "127.0.0.1", min_port = 1, max_port = 65535):
sock = socket.socket(family, type, proto)
def find_listen_port(family=socket.AF_INET, type=socket.SOCK_STREAM,
proto=0, addr="127.0.0.1", min_port=1, max_port=65535):
sock = compat.socket(family, type, proto)
for port in range(min_port, max_port + 1):
try:
sock.bind((addr, port))
......@@ -134,44 +147,50 @@ def find_listen_port(family = socket.AF_INET, type = socket.SOCK_STREAM,
pass
raise RuntimeError("Cannot find an usable port in the range specified")
# Logging
_log_level = LOG_DEBUG
_log_level = LOG_WARNING
_log_use_syslog = False
_log_stream = sys.stderr
_log_syslog_opts = ()
_log_pid = os.getpid()
def set_log_level(level):
"Sets the log level for console messages, does not affect syslog logging."
global _log_level
assert level > LOG_ERR and level <= LOG_DEBUG
_log_level = level
def set_log_output(stream):
"Redirect console messages to the provided stream."
global _log_stream
assert hasattr(stream, "write") and hasattr(stream, "flush")
_log_stream = stream
def log_use_syslog(use = True, ident = None, logopt = 0,
facility = syslog.LOG_USER):
def log_use_syslog(use=True, ident=None, logopt=0,
facility=syslog.LOG_USER):
"Enable or disable the use of syslog for logging messages."
global _log_use_syslog, _log_syslog_opts
_log_syslog_opts = (ident, logopt, facility)
_log_use_syslog = use
_init_log()
def _init_log():
if not _log_use_syslog:
syslog.closelog()
return
(ident, logopt, facility) = _log_syslog_opts
(ident, logopt, facility) = _log_syslog_opts
if not ident:
#ident = os.path.basename(sys.argv[0])
# ident = os.path.basename(sys.argv[0])
ident = "nemu"
syslog.openlog("%s[%d]" % (ident, os.getpid()), logopt, facility)
info("Syslog logging started")
def logger(priority, message):
"Print a log message in syslog, console or both."
if _log_use_syslog:
......@@ -183,27 +202,37 @@ def logger(priority, message):
return
eintr_wrapper(_log_stream.write,
"[%d] %s\n" % (os.getpid(), message.rstrip()))
"[%d] %s\n" % (os.getpid(), message.rstrip()))
_log_stream.flush()
def error(message):
logger(LOG_ERR, message)
def warning(message):
logger(LOG_WARNING, message)
def notice(message):
logger(LOG_NOTICE, message)
def info(message):
logger(LOG_INFO, message)
def debug(message):
logger(LOG_DEBUG, message)
def _custom_hook(tipe, value, traceback): # pragma: no cover
def _custom_hook(tipe, value, traceback): # pragma: no cover
"""Custom exception hook, to print nested exceptions information."""
if hasattr(value, "child_traceback"):
sys.stderr.write("Nested exception, original traceback " +
"(most recent call last):\n")
"(most recent call last):\n")
sys.stderr.write(value.child_traceback + ("-" * 70) + "\n")
sys.__excepthook__(tipe, value, traceback)
sys.excepthook = _custom_hook
sys.excepthook = _custom_hook
src/nemu/interface.py
View file @ ba619ec3
......@@ -40,7 +40,7 @@ class Interface(object):
def _gen_if_name():
n = Interface._gen_next_id()
# Max 15 chars
return "NETNSif-%.4x%.3x" % (os.getpid() % 0xffff, n)
return "NETNSif-%.4x%.3x" % (os.getpid() & 0xffff, n)
def __init__(self, index):
self._idx = index
......@@ -386,7 +386,7 @@ class Switch(ExternalInterface):
def _gen_br_name():
n = Switch._gen_next_id()
# Max 15 chars
return "NETNSbr-%.4x%.3x" % (os.getpid() % 0xffff, n)
return "NETNSbr-%.4x%.3x" % (os.getpid() & 0xffff, n)
def __init__(self, **args):
"""Creates a new Switch object, which models a linux bridge device.
......
src/nemu/node.py
View file @ ba619ec3
......@@ -27,6 +27,7 @@ import weakref
import nemu.interface
import nemu.protocol
import nemu.subprocess_
from nemu import compat
from nemu.environ import *
__all__ = ['Node', 'get_nodes', 'import_if']
......@@ -195,7 +196,7 @@ class Node(object):
# Requires CAP_SYS_ADMIN privileges to run.
def _start_child(nonetns) -> (socket.socket, int):
# Create socket pair to communicate
(s0, s1) = socket.socketpair(socket.AF_UNIX, socket.SOCK_STREAM, 0)
(s0, s1) = compat.socketpair(socket.AF_UNIX, socket.SOCK_STREAM, 0)
# Spawn a child that will run in a loop
pid = os.fork()
if pid:
......
src/nemu/protocol.py
View file @ ba619ec3
This diff is collapsed.
src/nemu/subprocess_.py
View file @ ba619ec3
This diff is collapsed.
test/test_protocol.py
View file @ ba619ec3
......@@ -6,14 +6,16 @@ import nemu.protocol
import os, socket, sys, threading, unittest
import test_util
from nemu import compat
class TestServer(unittest.TestCase):
@test_util.skip("python 3 can't makefile a socket in r+")
def test_server_startup(self):
# Test the creation of the server object with different ways of passing
# the file descriptor; and check the banner.
(s0, s1) = socket.socketpair(socket.AF_UNIX, socket.SOCK_STREAM, 0)
(s2, s3) = socket.socketpair(socket.AF_UNIX, socket.SOCK_STREAM, 0)
(s0, s1) = compat.socketpair(socket.AF_UNIX, socket.SOCK_STREAM, 0)
(s2, s3) = compat.socketpair(socket.AF_UNIX, socket.SOCK_STREAM, 0)
def test_help(fd):
fd.write("HELP\n")
......@@ -34,13 +36,13 @@ class TestServer(unittest.TestCase):
t = threading.Thread(target = run_server)
t.start()
s = os.fdopen(s1.fileno(), "r+", 1)
s = os.fdopen(s1.fileno(), "r", 1)
self.assertEqual(s.readline()[0:4], "220 ")
test_help(s)
s.close()
s0.close()
s = os.fdopen(s3.fileno(), "r+", 1)
s = os.fdopen(s3.fileno(), "r", 1)
self.assertEqual(s.readline()[0:4], "220 ")
test_help(s)
s.close()
......@@ -48,7 +50,7 @@ class TestServer(unittest.TestCase):
t.join()
def test_server_clean(self):
(s0, s1) = socket.socketpair(socket.AF_UNIX, socket.SOCK_STREAM, 0)
(s0, s1) = compat.socketpair(socket.AF_UNIX, socket.SOCK_STREAM, 0)
def run_server():
nemu.protocol.Server(s0, s0).run()
......@@ -56,8 +58,9 @@ class TestServer(unittest.TestCase):
t.start()
cli = nemu.protocol.Client(s1, s1)
argv = [ '/bin/sh', '-c', 'yes' ]
pid = cli.spawn(argv, stdout = subprocess.DEVNULL)
argv = [ '/bin/sh', '-c', 'yes' ]
nullfd = open("/dev/null", "wb")
pid = cli.spawn(argv, stdout = nullfd.fileno())
self.assertTrue(os.path.exists("/proc/%d" % pid))
# try to exit while there are still processes running
cli.shutdown()
......@@ -68,7 +71,7 @@ class TestServer(unittest.TestCase):
self.assertFalse(os.path.exists("/proc/%d" % pid))
def test_spawn_recovery(self):
(s0, s1) = socket.socketpair(socket.AF_UNIX, socket.SOCK_STREAM, 0)
(s0, s1) = compat.socketpair(socket.AF_UNIX, socket.SOCK_STREAM, 0)
def run_server():
nemu.protocol.Server(s0, s0).run()
......@@ -93,7 +96,7 @@ class TestServer(unittest.TestCase):
@test_util.skip("python 3 can't makefile a socket in r+")
def test_basic_stuff(self):
(s0, s1) = socket.socketpair(socket.AF_UNIX, socket.SOCK_STREAM, 0)
(s0, s1) = compat.socketpair(socket.AF_UNIX, socket.SOCK_STREAM, 0)
srv = nemu.protocol.Server(s0, s0)
s1 = s1.makefile("r+", 1)
......
test/test_subprocess.py
View file @ ba619ec3
......@@ -6,6 +6,9 @@ import nemu, test_util
import nemu.subprocess_ as sp
import grp, os, pwd, signal, socket, sys, time, unittest
from nemu import compat
def _stat(path):
try:
return os.stat(path)
......@@ -104,7 +107,7 @@ class TestSubprocess(unittest.TestCase):
# uses a default search path
self.assertRaises(OSError, sp.spawn, 'sleep', env = {'PATH': ''})
r, w = os.pipe()
r, w = compat.pipe()
p = sp.spawn('/bin/echo', ['echo', 'hello world'], stdout = w)
os.close(w)
self.assertEqual(_readall(r), b"hello world\n")
......@@ -120,8 +123,8 @@ class TestSubprocess(unittest.TestCase):
# It cannot be wait()ed again.
self.assertRaises(OSError, sp.wait, p)
r0, w0 = os.pipe()
r1, w1 = os.pipe()
r0, w0 = compat.pipe()
r1, w1 = compat.pipe()
p = sp.spawn('/bin/cat', stdout = w0, stdin = r1, close_fds = [r0, w1])
os.close(w0)
os.close(r1)
......@@ -140,25 +143,25 @@ class TestSubprocess(unittest.TestCase):
self.assertRaises(ValueError, node.Subprocess,
['/bin/sleep', '1000'], user = self.nouid)
# Invalid CWD: it is a file
self.assertRaises(OSError, node.Subprocess,
self.assertRaises(NotADirectoryError, node.Subprocess,
'/bin/sleep', cwd = '/bin/sleep')
# Invalid CWD: does not exist
self.assertRaises(OSError, node.Subprocess,
self.assertRaises(FileNotFoundError, node.Subprocess,
'/bin/sleep', cwd = self.nofile)
# Exec failure
self.assertRaises(OSError, node.Subprocess, self.nofile)
self.assertRaises(FileNotFoundError, node.Subprocess, self.nofile)
# Test that the environment is cleared: sleep should not be found
self.assertRaises(OSError, node.Subprocess,
self.assertRaises(FileNotFoundError, node.Subprocess,
'sleep', env = {'PATH': ''})
# Argv
self.assertRaises(OSError, node.Subprocess, 'true; false')
self.assertRaises(FileNotFoundError, node.Subprocess, 'true; false')
self.assertEqual(node.Subprocess('true').wait(), 0)
self.assertEqual(node.Subprocess('true; false', shell = True).wait(),
1)
# Piping
r, w = os.pipe()
r, w = compat.pipe()
p = node.Subprocess(['echo', 'hello world'], stdout = w)
os.close(w)
self.assertEqual(_readall(r), b"hello world\n")
......@@ -166,7 +169,7 @@ class TestSubprocess(unittest.TestCase):
p.wait()
# cwd
r, w = os.pipe()
r, w = compat.pipe()
p = node.Subprocess('/bin/pwd', stdout = w, cwd = "/")
os.close(w)
self.assertEqual(_readall(r), b"/\n")
......@@ -194,7 +197,7 @@ class TestSubprocess(unittest.TestCase):
# closing stdout (so _readall finishes)
cmd = 'trap "" TERM; echo; exec sleep 100 > /dev/null'
r, w = os.pipe()
r, w = compat.pipe()
p = node.Subprocess(cmd, shell = True, stdout = w)
os.close(w)
self.assertEqual(_readall(r), b"\n") # wait for trap to be installed
......@@ -202,9 +205,10 @@ class TestSubprocess(unittest.TestCase):
pid = p.pid
os.kill(pid, 0) # verify process still there
# Avoid the warning about the process being killed
old_err = sys.stderr
with open("/dev/null", "w") as sys.stderr:
p.destroy()
sys.stderr = sys.__stderr__
sys.stderr = old_err
self.assertRaises(OSError, os.kill, pid, 0) # should be dead by now
p = node.Subprocess(['sleep', '100'])
......@@ -217,8 +221,8 @@ class TestSubprocess(unittest.TestCase):
node = nemu.Node(nonetns = True)
# repeat test with Popen interface
r0, w0 = os.pipe()
r1, w1 = os.pipe()
r0, w0 = compat.pipe()
r1, w1 = compat.pipe()
p = node.Popen('cat', stdout = w0, stdin = r1)
os.close(w0)
os.close(r1)
......@@ -228,7 +232,7 @@ class TestSubprocess(unittest.TestCase):
os.close(r0)
# now with a socketpair, not using integers
(s0, s1) = socket.socketpair(socket.AF_UNIX, socket.SOCK_STREAM, 0)
(s0, s1) = compat.socketpair(socket.AF_UNIX, socket.SOCK_STREAM, 0)
p = node.Popen('cat', stdout = s0, stdin = s0)
s0.close()
s1.send(b"hello world\n")
......@@ -255,9 +259,9 @@ class TestSubprocess(unittest.TestCase):
p = node.Popen('cat >&2', shell = True, stdin = sp.PIPE,
stderr = sp.PIPE)
p.stdin.write("hello world\n")
p.stdin.write(b"hello world\n")
p.stdin.close()
self.assertEqual(p.stderr.readlines(), ["hello world\n"])
self.assertEqual(p.stderr.readlines(), [b"hello world\n"])
self.assertEqual(p.stdout, None)
self.assertEqual(p.wait(), 0)
......@@ -268,9 +272,9 @@ class TestSubprocess(unittest.TestCase):
#
p = node.Popen(['sh', '-c', 'cat >&2'],
stdin = sp.PIPE, stdout = sp.PIPE, stderr = sp.STDOUT)
p.stdin.write("hello world\n")
p.stdin.write(b"hello world\n")
p.stdin.close()
self.assertEqual(p.stdout.readlines(), ["hello world\n"])
self.assertEqual(p.stdout.readlines(), [b"hello world\n"])
self.assertEqual(p.stderr, None)
self.assertEqual(p.wait(), 0)
......@@ -281,9 +285,9 @@ class TestSubprocess(unittest.TestCase):
#
p = node.Popen(['tee', '/dev/stderr'],
stdin = sp.PIPE, stdout = sp.PIPE, stderr = sp.STDOUT)
p.stdin.write("hello world\n")
p.stdin.write(b"hello world\n")
p.stdin.close()
self.assertEqual(p.stdout.readlines(), ["hello world\n"] * 2)
self.assertEqual(p.stdout.readlines(), [b"hello world\n"] * 2)
self.assertEqual(p.stderr, None)
self.assertEqual(p.wait(), 0)
......@@ -295,10 +299,10 @@ class TestSubprocess(unittest.TestCase):
#
p = node.Popen(['tee', '/dev/stderr'],
stdin = sp.PIPE, stdout = sp.PIPE, stderr = sp.PIPE)
p.stdin.write("hello world\n")
p.stdin.write(b"hello world\n")
p.stdin.close()
self.assertEqual(p.stdout.readlines(), ["hello world\n"])
self.assertEqual(p.stderr.readlines(), ["hello world\n"])
self.assertEqual(p.stdout.readlines(), [b"hello world\n"])
self.assertEqual(p.stderr.readlines(), [b"hello world\n"])
self.assertEqual(p.wait(), 0)
p = node.Popen(['tee', '/dev/stderr'],
......
test/test_util.py
View file @ ba619ec3
......@@ -5,7 +5,7 @@ import os, re, subprocess, sys
import nemu.subprocess_
from nemu.environ import *
def process_ipcmd(str):
def process_ipcmd(str: str):
cur = None
out = {}
for line in str.split("\n"):
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7