some

re6st/tests/test_network/hmac.py

@@ -29,10 +29,11 @@ def checkHMAC(db, machines):
             else:
                 i = 0 if hmac[0] else 1
                 if hmac[i] != sign or hmac[i+1] != accept:
-                    logging.debug('HMAC config wrong for in %s' % args)
+                    logging.warn('HMAC config wrong for in %s' % args)
+                    logging.warn("HMAC sign: {}, accept: {}".format(sign, accept))
                     rc = False
     if rc:
-        logging.debug('All nodes use Babel with the correct HMAC configuration')
+        logging.info('All nodes use Babel with the correct HMAC configuration')
     else:
-        logging.debug('Expected config: %s' % dict(zip(BABEL_HMAC, hmac)))
+        logging.warn('Expected config: %s' % dict(zip(BABEL_HMAC, hmac)))
     return rc

re6st/tests/test_network/my_net.py

@@ -77,7 +77,7 @@ class Netns(object):
         return subprocess.Popen(['nsenter', '-t', str(self.pid), '-n'] + cmd, stdout=stdout, stderr=stderr, **kw)
 
 
-    def run(self, cmd, check=True, stdout=sys.stdout, stderr=sys.stderr,**kw):
+    def run(self, cmd, stdout=sys.stdout, stderr=sys.stderr,**kw):
         """ wrapper for subprocess.checkout"""
         subprocess.check_call(['nsenter', '-t', str(self.pid), '-n'] + cmd, stdout=stdout, stderr=stderr, **kw)
 

re6st/tests/test_network/re6st_wrap.py

@@ -84,10 +84,9 @@ class Re6stRegistry(object):
         self.clean()
 
         self.run()
-        
 
         # wait the servcice started
-        self.node.Popen(['python', '-c', """if 1:
+        p = self.node.Popen(['python', '-c', """if 1:
         import socket, time
         s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         while True:
@@ -96,8 +95,13 @@ class Re6stRegistry(object):
                 break
             except socket.error:
                 time.sleep(.1)
-        """]).wait()
-        logging.debug("re6st service started")
+        """])
+        try:
+            wait_ps(p, 5)
+        except Exception as e:
+            logging.error("{}: {}", self.name, e)
+            raise e
+        logging.info("re6st service started")
 
     @classmethod
     def generate_name(cls):
@@ -123,7 +127,7 @@ class Re6stRegistry(object):
         cmd = cmd.format(script=RE6ST_REGISTRY, ca=self.ca_crt,
                          key=self.ca_key, dh=DH_FILE, log=self.log, db=self.db,
                          run=self.run_path).split()
-        logging.info("run registry {} with cmd: {}".format(self.name, " ".join(cmd)))
+        logging.info("run registry {} at ns: {} with cmd: {}".format(self.name, self.node.pid," ".join(cmd)))
         self.proc = self.node.Popen(cmd, stdout=PIPE, stderr=PIPE)
 
     def clean(self):
@@ -199,6 +203,10 @@ class Re6stNode(object):
         self.clean()
 
 
+    def __repr__(self):
+        return self.name
+
+    
     @classmethod
     def generate_name(cls):
         cls.node_seq += 1
@@ -246,7 +254,7 @@ class Re6stNode(object):
                          cert=self.crt, key=self.key, registry=self.registry.url,
                          console=self.console).split()
         cmd += args
-        logging.info("run node {} with cmd: {}".format(self.name, " ".join(cmd)))
+        logging.info("run node {} at ns: {} with cmd: {}".format(self.name, self.node.pid," ".join(cmd)))
         self.proc = self.node.Popen(cmd, stdout=PIPE, stderr=PIPE)
 
     def clean(self):

re6st/tests/test_network/test_net.py

@@ -12,6 +12,7 @@ from pathlib2 import Path
 import re6st_wrap
 import my_net
 import hmac
+import random
 
 PING_PATH = str(Path(__file__).parent.resolve() / "ping.py")
 
@@ -54,14 +55,17 @@ def wait_stable(nodes, timeout=240):
         for i in range(len(unfinished)-1,-1,-1):
             node = unfinished[i]
             if node.ping_proc.poll() is not None:
-                logging.info("{}'s network is stable".format(node.name))
+                logging.debug("{}'s network is stable".format(node.name))
                 unfinished.pop(i)
         time.sleep(0.5)
 
         if time.time() - now > timeout:
             for node in unfinished:
                 node.ping_proc.terminate()
+            logging.warn("{} can't ping to all the nodes".format(unfinished))
             return False
+    logging.info("wait time cost: {}".format(time.time() - now))
+    return True
 
 def ping_test(nodes):
     """test Re6st node connecty """
@@ -96,7 +100,7 @@ class TestNet(unittest.TestCase):
         # logging.basicConfig(level=logging.WARNING)
         # clean all the running process
         for p in psutil.Process().children():
-            logging.warning("unterminate subprocess, name: {}, pid: {}, status: {}, cmd: {}".format(p.name(), p.pid, p.status(), p.cmdline()))
+            logging.debug("unterminate subprocess, name: {}, pid: {}, status: {}, cmd: {}".format(p.name(), p.pid, p.status(), p.cmdline()))
             # try:
             #     p.kill()
             # except:
@@ -109,10 +113,20 @@ class TestNet(unittest.TestCase):
         nm =  my_net.net_demo()
         nodes, registrys = deploy_re6st(nm)
         # wait 60, if the re6stnet stable quit wait
-        wait_stable(nodes, 50)
-        time.sleep(10)
 
-        self.assertFalse(wait_stable(nodes, 30), " ping test failed")
+        self.assertTrue(wait_stable(nodes, 80), "ping test failed")
+
+        # stop on machine randomly and reboot it, test if network recover
+        # index = int(random.random() * 7) + 1
+        # machine = nodes[index]
+
+        # machine.proc.terminate()
+        # machine.proc.wait()
+        # time.sleep(5)
+        # machine.run("-i" + machine.node.out.name)
+
+        # self.assertTrue(wait_stable(nodes, 200), "second ping test failed")
+
             
     def test_ping_router(self):
         """create a network in a net segment, test the connectivity by ping
@@ -123,12 +137,13 @@ class TestNet(unittest.TestCase):
         wait_stable(nodes, 50)
         time.sleep(10)
 
-        self.assertFalse(wait_stable(nodes, 40), " ping test failed")
+        self.assertTrue(wait_stable(nodes, 40), " ping test failed")
 
 
     def test_hmac(self):
         """create a network demo, and run hmac test, this test check hmac 3 times
         """
+
         nm =  my_net.net_demo()
         nodes, registrys = deploy_re6st(nm)
         
@@ -136,22 +151,20 @@ class TestNet(unittest.TestCase):
             "204 != urllib.urlopen('http://127.0.0.1/updateHMAC').code)"]
             
         registry = registrys[0]
-        machine1 = nodes[1]
+        machine1 = nodes[5]
         reg1_db = sqlite3.connect(str(registry.db), isolation_level=None, check_same_thread=False)
 
         # reg1_db.text_factory = str
         m_net1 = [node.name for node in nodes]
 
-        # wait net stable
-        time.sleep(100)
+        # wait net stable, wait at most 100 seconds
+        wait_stable(nodes, 100)
 
         logging.info('Check that the initial HMAC config is deployed on network 1')
         self.assertTrue(hmac.checkHMAC(reg1_db, m_net1), "first hmac check failed")
 
-        # check the update abillity
         logging.info('Test that a HMAC update works with nodes that are up')
         registry.node.run(updateHMAC)
-        logging.info('Updated HMAC (config = hmac0 & hmac1), waiting...')
         time.sleep(60)
         # Checking HMAC on machines connected to registry 1...
         self.assertTrue(hmac.checkHMAC(reg1_db, m_net1), "second hmac check failed: HMAC update don't work")
@@ -162,22 +175,19 @@ class TestNet(unittest.TestCase):
         logging.info('Re6st on {} is stopped'.format(machine1.name))
         machine1.proc.terminate()
         machine1.proc.wait()
-
         time.sleep(5)
         registry.node.run(updateHMAC)
-        logging.info('Updated HMAC on registry (config = hmac1 & hmac2), waiting...')
         time.sleep(60)
-        machine1.run()
-        logging.info( 'Started re6st on {}, waiting for it'.format(machine1.name))
-        time.sleep(100)
-        # logging.info( 'Checking HMAC on machines connected to registry 1...')
+        machine1.run("-i" + machine1.node.out.name)
+        wait_stable(nodes, 100)
         self.assertTrue(hmac.checkHMAC(reg1_db, m_net1), "third hmac check failed: machine restart failed")
         logging.info( 'Testing of HMAC done!')
         reg1_db.close()
 
 
 if __name__ == "__main__":
-    logging.basicConfig(level=logging.DEBUG, filename='test.log', filemode='w')
+    logging.basicConfig(level=logging.DEBUG, filename='test.log', filemode='w', 
+        format='%(asctime)s %(levelname)s %(message)s', datefmt='%I:%M:%S')
     unittest.main(verbosity=3)