test: deploy re6st on net ns, and run ping test

network_build: use nemu to create net namespace, provide several network

re6st_wrap: wrap the deploy of re6st node, ease the creation of cert
file and run of the node

test_net contain the tests:
  - ping test: net segment(router), multi-net segment(demo),test after a
    reboot. Because net_demo contain a subnet UPnP, test is not stable.

ping is like ping file in demo, a script to run multiping

re6st/tests/test_network/__init__.py

+__all__ = ["test_ping"]

re6st/tests/test_network/fixnemu.py

+# -*- coding: utf-8 -*-
+
+# Copyright 2010, 2011 INRIA
+# Copyright 2011 Martín Ferrari <martin.ferrari@gmail.com>
+#
+# This file is contains patches to Nemu.
+#
+# Nemu is free software: you can redistribute it and/or modify it under the
+# terms of the GNU General Public License version 2, as published by the Free
+# Software Foundation.
+#
+# Nemu is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# Nemu.  If not, see <http://www.gnu.org/licenses/>.
+
+import re
+from new import function
+from nemu.iproute import backticks, get_if_data, route, \
+    get_addr_data, get_all_route_data, interface
+
+def _get_all_route_data():
+    ipdata = backticks([IP_PATH, "-o", "route", "list"]) # "table", "all"
+    ipdata += backticks([IP_PATH, "-o", "-f", "inet6", "route", "list"])
+
+    ifdata = get_if_data()[1]
+    ret = []
+    for line in ipdata.split("\n"):
+        if line == "":
+            continue
+        # PATCH: parse 'from'
+        # PATCH: 'dev' is missing on 'unreachable' ipv4 routes
+        match = re.match('(?:(unicast|local|broadcast|multicast|throw|'
+            r'unreachable|prohibit|blackhole|nat) )?(\S+)(?: from (\S+))?'
+            r'(?: via (\S+))?(?: dev (\S+))?.*(?: metric (\d+))?', line)
+        if not match:
+            raise RuntimeError("Invalid output from `ip route': `%s'" % line)
+        tipe = match.group(1) or "unicast"
+        prefix = match.group(2)
+        #src = match.group(3)
+        nexthop = match.group(4)
+        interface = ifdata[match.group(5) or "lo"]
+        metric = match.group(6)
+        if prefix == "default" or re.search(r'/0$', prefix):
+            prefix = None
+            prefix_len = 0
+        else:
+            match = re.match(r'([0-9a-f:.]+)(?:/(\d+))?$', prefix)
+            prefix = match.group(1)
+            prefix_len = int(match.group(2) or 32)
+        ret.append(route(tipe, prefix, prefix_len, nexthop, interface.index,
+            metric))
+    return ret
+
+get_all_route_data.func_code = _get_all_route_data.func_code
+
+interface__init__ = interface.__init__
+def __init__(self, *args, **kw):
+    interface__init__(self, *args, **kw)
+    if self.name:
+        self.name = self.name.split('@',1)[0]
+interface.__init__ = __init__
+
+get_addr_data.orig = function(get_addr_data.func_code,
+                              get_addr_data.func_globals)
+def _get_addr_data():
+    byidx, bynam = get_addr_data.orig()
+    return byidx, {name.split('@',1)[0]: a for name, a in bynam.iteritems()}
+get_addr_data.func_code = _get_addr_data.func_code

re6st/tests/test_network/miniupnpd.conf

+clean_ruleset_interval=600
+allow 1024-65535 10.0.0.0/8 1024-65535
+deny 0-65535 0.0.0.0/0 0-65535

re6st/tests/test_network/network_build.py

+import time
+import nemu
+import weakref
+import ipaddress
+import logging
+from subprocess import PIPE
+from pathlib2 import Path
+
+fix_file =  Path(__file__).parent.resolve() / "fixnemu.py"
+execfile(str(fix_file))
+IPTABLES = 'iptables-nft'
+
+class Node(nemu.Node):
+    """simple nemu.Node used for registry and nodes"""
+    def __init__(self):
+        super(Node, self).__init__()
+        self.Popen(('sysctl', '-q',
+        'net.ipv4.icmp_echo_ignore_broadcasts=0')).wait()
+
+    def _add_interface(self, iface):
+        self.iface = iface
+        iface.__dict__['node'] = weakref.proxy(self)
+        return super(Node, self)._add_interface(iface)
+
+    @property
+    def ip(self):
+        if hasattr(self, "_ip"):
+            return str(self._ip)
+
+        # return 1 ipv4 address of the one interface, reverse mode
+        for iface in self.get_interfaces()[::-1]:
+            for addr in iface.get_addresses():
+                addr = addr['address']
+                if '.' in addr:
+                    #TODO different type problem?
+                    self._ip = addr
+                    return addr
+
+
+    def connect_switch(self, switch, ip, prefix_len=24):
+        self.if_s = if_s = nemu.NodeInterface(self)
+        switch.connect(if_s)
+        if_s.up = True
+        if_s.add_v4_address(ip, prefix_len=prefix_len)
+        return if_s
+
+class NetManager(object):
+    """contain all the nemu object created, so they can live more time"""
+    def __init__(self):
+        self.object = []
+        self.registrys = {}
+
+
+def connectible_test(nm):
+    """test each node can ping to their registry
+
+    Args:
+        nm: NetManger
+
+    Raise:
+        AssertionError
+    """
+    for reg in nm.registrys:
+        for node in nm.registrys[reg]:
+            app0 = node.Popen(["ping", "-c", "1", reg.ip], stdout=PIPE)
+            ret = app0.wait()
+            assert ret == 0, "network construct failed {} to {}".format(node.ip, reg.ip)
+
+    logging.debug("each node can ping to their registry")
+
+def net_route():
+    """build a network connect by a route(bridge)
+
+    Returns:
+        a network manager contain 3 nodes
+    """
+    nm = NetManager()
+
+    switch1 = nemu.Switch()
+    switch1.up = True
+
+    registry = Node()
+    machine1 = Node()
+    machine2 = Node()
+
+    r1_if_0 = registry.connect_switch(switch1, "192.168.1.1")
+    m1_if_0 = machine1.connect_switch(switch1, "192.168.1.2")
+    m2_if_0 = machine2.connect_switch(switch1, "192.168.1.3")
+
+    nm.object.append(switch1)
+    nm.registrys[registry] = [machine1, machine2]
+
+    connectible_test(nm)
+    return nm
+
+def net_demo():
+
+    internet = Node()
+    gateway1 = Node()
+    gateway2 = Node()
+
+    registry = Node()
+    m1 = Node()
+    m2 = Node()
+    m3 = Node()
+    m4 = Node()
+    m5 = Node()
+    m6 = Node()
+    m7 = Node()
+    m8 = Node()
+
+    switch1 = nemu.Switch()
+    switch2 = nemu.Switch()
+    switch3 = nemu.Switch()
+
+    nm = NetManager()
+    nm.object = [internet, switch3, switch1, switch2, gateway1, gateway2]
+    nm.registrys = {registry: [m1, m2, m3, m4, m5, m6, m7, m8]}
+
+    # for node in [g1, m3, m4, m5]:
+    #     print "pid: {}".format(node.pid)
+
+    re_if_0, in_if_0 = nemu.P2PInterface.create_pair(registry,internet)
+    g1_if_0, in_if_1 = nemu.P2PInterface.create_pair(gateway1, internet)
+    g2_if_0, in_if_2 = nemu.P2PInterface.create_pair(gateway2, internet)
+
+    re_if_0.add_v4_address(address="10.0.0.2", prefix_len=24)
+    in_if_0.add_v4_address(address='10.0.0.1', prefix_len=24)
+    in_if_1.add_v4_address(address='10.1.0.1', prefix_len=24)
+    in_if_2.add_v4_address(address='10.2.0.1', prefix_len=24)
+    g1_if_0.add_v4_address(address='10.1.0.2', prefix_len=24)
+    g2_if_0.add_v4_address(address='10.2.0.2', prefix_len=24)
+
+    for iface in [re_if_0, in_if_0, g1_if_0, in_if_1, g2_if_0, in_if_2]:
+        nm.object.append(iface)
+        iface.up = True
+
+    ip = ipaddress.ip_address(u"10.1.1.1")
+    for i, node in enumerate([gateway1, m1, m2]):
+        iface = node.connect_switch(switch1, str(ip + i))
+        nm.object.append(iface)
+        if i: # except the first
+            node.add_route(nexthop=ip)
+
+    gateway1.Popen((IPTABLES, '-t', 'nat', '-A', 'POSTROUTING', '-o', g1_if_0.name, '-j', 'MASQUERADE')).wait()
+    gateway1.Popen((IPTABLES, '-t', 'nat', '-N', 'MINIUPNPD')).wait()
+    gateway1.Popen((IPTABLES, '-t', 'nat', '-A', 'PREROUTING', '-i', g1_if_0.name, '-j', 'MINIUPNPD')).wait()
+    gateway1.Popen((IPTABLES, '-N', 'MINIUPNPD')).wait()
+
+
+    ip = ipaddress.ip_address(u"10.2.1.1")
+    for i, node in enumerate([gateway2, m3, m4, m5]):
+        iface = node.connect_switch(switch1, str(ip + i))
+        nm.object.append(iface)
+        if i: # except the first
+            node.add_route(prefix='10.0.0.0', prefix_len=8, nexthop=ip)
+
+    ip = ipaddress.ip_address(u"10.0.1.1")
+    for i, node in enumerate([internet, m6, m7, m8]):
+        iface = node.connect_switch(switch2, str(ip + i))
+        nm.object.append(iface)
+        if i: # except the first
+            node.add_route(prefix='10.0.0.0', prefix_len=8, nexthop=ip)
+
+    registry.add_route(prefix='10.0.0.0', prefix_len=8, nexthop='10.0.0.1')
+    gateway1.add_route(prefix='10.0.0.0', prefix_len=8, nexthop='10.1.0.1')
+    gateway2.add_route(prefix='10.0.0.0', prefix_len=8, nexthop='10.2.0.1')
+    internet.add_route(prefix='10.2.0.0', prefix_len=16, nexthop='10.2.0.2')
+
+    MINIUPnP_CONF = Path(__file__).parent / 'miniupnpd.conf'
+    gateway1.proc = gateway1.Popen(['miniupnpd', '-d', '-f', MINIUPnP_CONF,
+                                    '-P', 'miniupnpd.pid','-a', gateway1.if_s.name,
+                                    '-i', g1_if_0.name],
+                                    stdout=PIPE, stderr=PIPE)
+
+    switch1.up = switch2.up = switch3.up =True
+    connectible_test(nm)
+    return nm
+
+def network_direct():
+    """one server and one client connect direct"""
+    registry = Node()
+    m0 = Node()
+    nm = NetManager()
+    nm.registrys = {registry: [m0]}
+
+    re_if_0, m_if_0 = nemu.P2PInterface.create_pair(registry, m0)
+
+    registry._ip = u"10.1.2.1"
+    re_if_0.add_v4_address(u"10.1.2.1", prefix_len=24)
+
+    m_if_0.add_v4_address(u"10.1.2.2", prefix_len=24)
+    re_if_0.up = m_if_0.up = True
+
+    for node in [m0]:
+        app0 = node.Popen(["ping", "-c", "1", "10.1.2.1"], stdout=PIPE)
+        ret = app0.wait()
+        assert ret == 0, "network construct failed"
+    return nm
+
+if __name__ == "__main__":
+    nm = network_demo()
+    time.sleep(1000000)

re6st/tests/test_network/ping.py

+# -*- coding: utf-8 -*-
+'''
+Script launched on machines from the demo with the option -p/--ping
+It uses Multiping to ping several IPs passed as arguments.
+After Re6st is stable, this script logs when it does not get response from a
+machine in a csv file stored in the directory of the machine in this format:
+time, sequence number, number of non-responding machines, ip of these machines
+'''
+import argparse, errno, socket, time, sys
+from multiping import MultiPing
+
+PING_INTERVAL = 10
+PING_TIMEOUT = 4
+
+class MultiPing(MultiPing):
+    # Patch of Multiping because it stays blocked to ipv4
+    # emission when we want to ping only ipv6 addresses.
+    # So we only keep the ipv6 part for the demo.
+    # Bug issued: https://github.com/romana/multi-ping/issues/22
+    def _read_all_from_socket(self, timeout):
+        pkts = []
+        if self._ipv6_address_present:
+            try:
+                self._sock6.settimeout(timeout)
+                while True:
+                    p = self._sock6.recv(128)
+                    pkts.append((bytearray(p), time.time()))
+                    self._sock6.settimeout(0)
+            except socket.timeout:
+                pass
+            except socket.error as e:
+                if e.errno == errno.EWOULDBLOCK:
+                    pass
+                else:
+                    raise
+        return pkts
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-a', nargs = '+', help = 'the list of addresses to ping')
+    parser.add_argument('--retry', action='store_true', help='retry ping unitl success')
+    
+    args = parser.parse_args()
+    addrs = args.a
+    retry = args.retry
+
+    while True:
+        mp = MultiPing(addrs)
+        mp.send()
+        _, no_responses = mp.receive(PING_TIMEOUT)
+        
+        if retry and no_responses:
+            continue
+        else:
+            sys.stdout.write(" ".join(no_responses))
+            return
+
+
+if __name__ == '__main__':
+    main()

re6st/tests/test_network/re6st_wrap.py

+"""wrap the deploy of re6st node, ease the creation of cert
+file and run of the node
+"""
+import json
+import shutil
+import sqlite3
+import weakref
+import ipaddress
+import time
+import re
+import tempfile
+import logging
+from subprocess import PIPE, call
+from pathlib2 import Path
+
+import re6st.tests.tools as tools
+
+WORK_DIR = Path(__file__).parent / "temp_net_test"
+DH_FILE = WORK_DIR / "dh2048.pem"
+
+RE6STNET = "re6stnet"
+RE6STNET = "python -m re6st.cli.node"
+RE6ST_REGISTRY = "re6st-registry"
+RE6ST_REGISTRY = "python -m re6st.cli.registry"
+RE6ST_CONF = "re6st-conf"
+RE6ST_CONF = "python -m re6st.cli.conf"
+
+def initial():
+    """create the workplace and dh file"""
+    if not WORK_DIR.exists():
+        WORK_DIR.mkdir()
+    if not DH_FILE.exists():
+        logging.info("create dh file")
+        call(['openssl', 'dhparam', '-out', str(DH_FILE), '2048'], stderr=PIPE)
+
+def ip_to_serial(ip6):
+    """convert ipv6 address to serial"""
+    ip6 = ipaddress.IPv6Address(u"{}".format(ip6))
+    ip6 = "1{:x}".format(int(ip6)).rstrip('0')
+    return int(ip6, 16)
+
+
+class Re6stRegistry(object):
+    """class run a re6st-registry service on a namespace"""
+    registry_seq = 0
+
+    def __init__(self, node, ip6, client_number, recreate=False):
+        self.node = node
+        # TODO need set once
+        self.ip = node.ip
+        self.ip6 = ip6
+        self.client_number = client_number
+        self.name = self.generate_name()
+
+        self.path = WORK_DIR / self.name
+        self.ca_key = self.path / "ca.key"
+        # because re6st-conf will create ca.crt so use another name
+        self.ca_crt = self.path / "ca.cert"
+        self.log = self.path / "registry.log"
+        self.db = self.path / "registry.db"
+        self.run_path = tempfile.mkdtemp()
+
+        if recreate and self.path.exists():
+            shutil.rmtree(str(self.path))
+
+        if not self.path.exists():
+            self.create_registry()
+
+        # use hash to identify the registry
+        with self.ca_key.open() as f:
+            text = f.read()
+            self.ident = hash(text)
+
+        # clear log file
+        if self.log.exists():
+            self.log.unlink()
+
+        self.clean()
+
+        self.run()
+
+        # wait the servcice started
+        p = self.node.Popen(['python', '-c', """if 1:
+        import socket, time
+        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        while True:
+            try:
+                s.connect(('localhost', 80))
+                break
+            except socket.error:
+                time.sleep(.1)
+        """])
+
+        now = time.time()
+        while time.time() - now < 10:
+            if p.poll() != None:
+                break
+            time.sleep(0.1)
+        else:
+            p.destroy()
+            logging.error("registry failed to start, %s", self.name)
+            raise Exception("registry failed to start")
+        logging.info("re6st service started")
+
+    @classmethod
+    def generate_name(cls):
+        cls.registry_seq += 1
+        return "registry_{}".format(cls.registry_seq)
+
+    @property
+    def url(self):
+        return "http://{ip}/".format(ip=self.ip)
+
+    def create_registry(self):
+        self.path.mkdir()
+        tools.create_ca_file(str(self.ca_key), str(self.ca_crt),
+                             serial=ip_to_serial(self.ip6))
+
+    def run(self):
+        cmd = ("{script} --ca {ca} --key {key} --dh {dh} --ipv4 10.42.0.0/16 8 "
+               " --logfile {log} --db {db} --run {run} --hello 4 --mailhost s "
+               "-v4 --client-count {nb}")
+        cmd = cmd.format(script=RE6ST_REGISTRY, ca=self.ca_crt,
+                         key=self.ca_key, dh=DH_FILE, log=self.log, db=self.db,
+                         run=self.run_path, nb=(self.client_number+1)//2).split()
+        logging.debug("run registry %s at ns: %s with cmd: %s",
+                     self.name, self.node.pid, " ".join(cmd))
+        self.proc = self.node.Popen(cmd, stdout=PIPE, stderr=PIPE)
+
+    def clean(self):
+        """remove the file created last time"""
+        for f in [self.log]:
+            if f.exists():
+                f.unlink()
+
+    def __del__(self):
+        try:
+            logging.debug("teminate process %s", self.proc.pid)
+            self.proc.destroy()
+        except:
+            pass
+
+
+class Re6stNode(object):
+    """class run a re6stnet service on a namespace"""
+    node_seq = 0
+
+    def __init__(self, node, registry, name=None, recreate=False):
+        """
+        node: nemu node
+        name: name for res6st node
+        """
+        self.name = name if name else self.generate_name()
+        self.node = node
+        self.registry = weakref.proxy(registry)
+
+        self.path = WORK_DIR / self.name
+        self.email = self.name + "@example.com"
+
+        if self.name == self.registry.name:
+            self.run_path = self.registry.run_path
+        else:
+            self.run_path = tempfile.mkdtemp()
+
+        self.log = self.path  / "re6stnet.log"
+        self.crt = self.path / "cert.crt"
+        self.key = self.path / 'cert.key'
+        self.console = self.run_path + "/console.sock"
+        self.data_file = self.path / "data.json" # contain data for restart node
+
+        # condition, node of the registry
+        if self.name == self.registry.name:
+            self.ip6 = self.registry.ip6
+            if not self.crt.exists():
+                self.create_node()
+        else:
+            # if ca file changed, we need recreate node file
+            if self.data_file.exists():
+                with self.data_file.open() as f:
+                    data = json.load(f)
+                    self.ip6 = data.get("ip6")
+                    recreate = not data.get('hash') == self.registry.ident
+            else:
+                recreate = True
+
+            if recreate and self.path.exists():
+                shutil.rmtree(str(self.path))
+
+            if not self.path.exists():
+                self.path.mkdir()
+                self.create_node()
+
+        logging.debug("%s's subnet is %s", self.name, self.ip6)
+
+        self.clean()
+
+
+    def __repr__(self):
+        return self.name
+
+    @classmethod
+    def generate_name(cls):
+        cls.node_seq += 1
+        return "node_{}".format(cls.node_seq)
+
+    def create_node(self):
+        """create necessary file for node"""
+        logging.info("create dir of node %s", self.name)
+        cmd = "{script} --registry {registry_url} --email {email}"
+        cmd = cmd.format(script=RE6ST_CONF, registry_url=self.registry.url,
+                         email=self.email).split()
+        p = self.node.Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE,
+                            cwd=str(self.path))
+        # read token
+        db = sqlite3.connect(str(self.registry.db), isolation_level=None)
+        count = 0
+        token = None
+        while not token:
+            time.sleep(.1)
+            token = db.execute("SELECT token FROM token WHERE email=?",
+                               (self.email,)).fetchone()
+            count += 1
+            if count > 100:
+                p.destroy()
+                raise Exception("can't connect to the Register")
+
+        out, _ = p.communicate(str(token[0]))
+        # logging.debug("re6st-conf output: {}".format(out))
+        # find the ipv6 subnet of node
+        self.ip6 = re.search('(?<=subnet: )[0-9:a-z]+', out).group(0)
+        data = {'ip6': self.ip6, 'hash': self.registry.ident}
+        with open(str(self.data_file), 'w') as f:
+            json.dump(data, f)
+        logging.info("create dir of node %s finish", self.name)
+
+    def run(self, *args):
+        """execute re6stnet"""
+        cmd = ("{script} --log {log} --run {run} --state {state}"
+               " --dh {dh} --ca {ca} --cert {cert} --key {key} -v4"
+               " --registry {registry} --console {console}"
+              )
+        cmd = cmd.format(script=RE6STNET, log=self.path, run=self.run_path,
+                         state=self.path, dh=DH_FILE, ca=self.registry.ca_crt,
+                         cert=self.crt, key=self.key, registry=self.registry.url,
+                         console=self.console).split()
+        cmd += args
+        logging.debug("run node %s at ns: %s with cmd: %s",
+                     self.name, self.node.pid, " ".join(cmd))
+        self.proc = self.node.Popen(cmd, stdout=PIPE, stderr=PIPE)
+
+    def clean(self):
+        """remove the file created last time"""
+        for name in ["re6stnet.log", "babeld.state", "cache.db", "babeld.log"]:
+            f = self.path / name
+            if f.exists():
+                f.unlink()
+
+    def stop(self):
+        """stop running re6stnet process"""
+        logging.debug("%s teminate process %s", self.name, self.proc.pid)
+        self.proc.destroy()
+
+
+    def __del__(self):
+        """teminate process and rm temp dir"""
+        try:
+            self.stop()
+        except Exception as e:
+            logging.warn("%s: %s", self.name, e)
+
+        # python2 seems auto clean the tempdir
+        # try:
+        #     shutil.rmtree(self.run_path)
+        # except Exception as e:
+        #     logging.error("{}: {}".format(self.name, e))

re6st/tests/test_network/test_net.py

+"""contain ping-test for re6set net"""
+import os
+import unittest
+import time
+import psutil
+import logging
+import sqlite3
+import random
+from binascii import b2a_hex
+from pathlib2 import Path
+import re6st_wrap
+import  network_build
+
+PING_PATH = str(Path(__file__).parent.resolve() / "ping.py")
+BABEL_HMAC = 'babel_hmac0', 'babel_hmac1', 'babel_hmac2'
+
+def deploy_re6st(nm, recreate=False):
+    net = nm.registrys
+    nodes = []
+    registrys = []
+    re6st_wrap.Re6stRegistry.registry_seq = 0
+    re6st_wrap.Re6stNode.node_seq = 0
+    for registry in net:
+        reg = re6st_wrap.Re6stRegistry(registry, "2001:db8:42::", len(net[registry]),
+                                       recreate=recreate)
+        reg_node = re6st_wrap.Re6stNode(registry, reg, name=reg.name)
+        registrys.append(reg)
+        reg_node.run("--gateway", "--disable-proto", "none", "--ip", registry.ip)
+        nodes.append(reg_node)
+        for m in net[registry]:
+            node = re6st_wrap.Re6stNode(m, reg)
+            node.run("-i" + m.iface.name)
+            nodes.append(node)
+    return nodes, registrys
+
+def wait_stable(nodes, timeout=240):
+    """try use ping6 from each node to the other until ping success to all the
+    other nodes
+    Args:
+        timeout: int, the time for wait
+
+    return:
+        True if success
+    """
+    logging.info("wait all node stable, timeout: %s", timeout)
+    now = time.time()
+    ips = {node.ip6: node.name for node in nodes}
+    # start the ping processs
+    for node in nodes:
+        sub_ips = set(ips) - {node.ip6}
+        node.ping_proc = node.node.Popen(
+            ["python", PING_PATH, '--retry', '-a'] + list(sub_ips))
+
+    # check all the node network can ping each other, in order reverse
+    unfinished = list(nodes)
+    while unfinished:
+        for i in range(len(unfinished)-1, -1, -1):
+            node = unfinished[i]
+            if node.ping_proc.poll() is not None:
+                logging.debug("%s 's network is stable", node.name)
+                unfinished.pop(i)
+        time.sleep(0.5)
+
+        if time.time() - now > timeout:
+            for node in unfinished:
+                node.ping_proc.destroy()
+            logging.warn("%s  can't ping to all the nodes", unfinished)
+            return False
+    logging.info("wait time cost: %s", time.time() - now)
+    return True
+
+@unittest.skipIf(os.geteuid() != 0, "require root or create user namespace plz")
+class TestNet(unittest.TestCase):
+    """ network test case"""
+
+    @classmethod
+    def setUpClass(cls):
+        """create work dir"""
+        logging.basicConfig(level=logging.INFO)
+        re6st_wrap.initial()
+
+    @classmethod
+    def tearDownClass(cls):
+        """watch any process leaked after tests"""
+        logging.basicConfig(level=logging.WARNING)
+        for p in psutil.Process().children():
+            logging.debug("unterminate ps, name: %s, pid: %s, status: %s, cmd: %s",
+                          p.name(), p.pid, p.status(), p.cmdline())
+            p.terminate()
+            # try:
+            #     p.kill()
+            # except:
+            #     pass
+
+    def test_ping_router(self):
+        """create a network in a net segment, test the connectivity by ping
+        """
+        nm = network_build.net_route()
+        nodes, _ = deploy_re6st(nm)
+
+        wait_stable(nodes, 40)
+        time.sleep(10)
+
+        self.assertTrue(wait_stable(nodes, 30), " ping test failed")
+
+    def test_ping_demo(self):
+        """create a network demo, test the connectivity by ping
+        wait at most 50 seconds, and test each node ping to other by ipv6 addr
+        """
+        nm = network_build.net_demo()
+        nodes, _ = deploy_re6st(nm)
+        # wait 60, if the re6stnet stable quit wait
+        wait_stable(nodes, 100)
+        time.sleep(20)
+
+        self.assertTrue(wait_stable(nodes, 100), "ping test failed")
+
+    def test_reboot_one_machine(self):
+        """create a network demo, wait the net stable, reboot on machine,
+        then test if network recover, this test seems always failed
+        """
+        nm = network_build.net_demo()
+        nodes, _ = deploy_re6st(nm)
+
+        wait_stable(nodes, 100)
+
+        # stop on machine randomly
+        index = int(random.random() * 7) + 1
+        machine = nodes[index]
+        machine.stop()
+        time.sleep(5)
+        machine.run("-i" + machine.node.iface.name)
+        logging.info("restart %s", machine.name)
+
+        self.assertTrue(wait_stable(nodes, 400), "network can't recover")
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.DEBUG, filename='test.log', filemode='w',
+                        format='%(asctime)s %(levelname)s %(message)s',
+                        datefmt='%I:%M:%S')
+    unittest.main(verbosity=3)

re6st/tests/tools.py

@@ -68,8 +68,9 @@ def create_cert_file(pkey_file, cert_file, ca, ca_key, prefix, serial):
 
 
 
-def create_ca_file(pkey_file, cert_file):
-
+def create_ca_file(pkey_file, cert_file, serial=0x120010db80042):
+    """create key and ca file with specify name
+    return key, cert in pem format """
     key = crypto.PKey()
     key.generate_key(crypto.TYPE_RSA, 2048)
     cert = crypto.X509()
@@ -82,7 +83,7 @@ def create_ca_file(pkey_file, cert_file):
     subject.O = "nexedi"
     subject.CN = "TEST-CA"
     cert.set_issuer(cert.get_subject())
-    cert.set_serial_number(10000)
+    cert.set_serial_number(serial)
     cert.set_pubkey(key)
     cert.sign(key, "sha512")
 

setup.py

@@ -95,6 +95,7 @@ setup(
     extras_require = {
         'geoip': ['geoip2'],
         'multicast': ['PyYAML'],
+        'test': ['mock', 'pathlib2', 'nemu', 'python-unshare', 'python-passfd', 'multiping']
     },
     #dependency_links = [
     #    "http://miniupnp.free.fr/files/download.php?file=miniupnpc-1.7.20120714.tar.gz#egg=miniupnpc-1.7",