Changed database structure, introduced address to replace ip, port, proto tuples

db.py

@@ -3,13 +3,17 @@ import utils
 
 class PeerManager:
 
-    def __init__(self, db_path, server, server_port, refresh_time, external_ip, internal_ip, port, proto, db_size):
+    # internal ip = temp arg/attribute
+    def __init__(self, db_path, server, server_port, refresh_time, address, internal_ip, prefix, manual, db_size):
         self._refresh_time = refresh_time
-        self._external_ip = external_ip
+        self._address = address
         self._internal_ip = internal_ip
-        self._external_port = port
-        self._proto = proto
+        self._prefix = prefix
+        self._server = server
+        self._server_port = server_port
         self._db_size = db_size
+        self._manual = manual
+
         self._proxy = xmlrpclib.ServerProxy('http://%s:%u' % (server, server_port))
 
         utils.log('Connectiong to peers database', 4)
@@ -30,31 +34,30 @@ class PeerManager:
         self.next_refresh = time.time() + self._refresh_time
 
     def _declare(self):
-        if self._external_ip != None:
-            utils.log('Sendin connection info to server', 3)
-            self._proxy.declare((self._internal_ip, self._external_ip, self._external_port, self._proto))
+        if self._address != None:
+            utils.log('Sending connection info to server', 3)
+            self._proxy.declare((self._internal_ip, utils.address_list(self._address)))
         else:
-            utils.log('Warning : could not send the external ip because it is unknown', 4)
+            utils.log("Warning : couldn't advertise to server, external config not known", 4)
 
     def _populate(self):
         utils.log('Populating the peers DB', 2)
         new_peer_list = self._proxy.getPeerList(self._db_size, self._internal_ip)
-        self._db.executemany("INSERT OR IGNORE INTO peers (ip, port, proto, used) VALUES (?,?,?,0)", new_peer_list)
-        if self._external_ip != None:
-            self._db.execute("DELETE FROM peers WHERE ip = ?", (self._external_ip,))
+        self._db.executemany("INSERT OR IGNORE INTO peers (prefix, address) VALUES (?,?)", new_peer_list)
+        self._db.execute("DELETE FROM peers WHERE prefix = ?", (self._prefix,))
         utils.log('New peers : %s' % ', '.join(map(str, new_peer_list)), 5)
 
     def getUnusedPeers(self, peer_count):
-        return self._db.execute("SELECT id, ip, port, proto FROM peers WHERE used = 0 "
-                "ORDER BY RANDOM() LIMIT ?", (peer_count,))
+        return self._db.execute("""SELECT prefix, address FROM peers WHERE used = 0
+                                   ORDER BY RANDOM() LIMIT ?""", (peer_count,))
 
-    def usePeer(self, id):
-        utils.log('Updating peers database : using peer ' + str(id), 5)
-        self._db.execute("UPDATE peers SET used = 1 WHERE id = ?", (id,))
+    def usePeer(self, prefix):
+        utils.log('Updating peers database : using peer ' + str(prefix), 5)
+        self._db.execute("UPDATE peers SET used = 1 WHERE prefix = ?", (prefix,))
 
-    def unusePeer(self, id):
-        utils.log('Updating peers database : unusing peer ' + str(id), 5)
-        self._db.execute("UPDATE peers SET used = 0 WHERE id = ?", (id,))
+    def unusePeer(self, prefix):
+        utils.log('Updating peers database : unusing peer ' + str(prefix), 5)
+        self._db.execute("UPDATE peers SET used = 0 WHERE id = ?", (prefix,))
 
     def handle_message(self, msg):
         script_type, arg = msg.split()
@@ -63,9 +66,13 @@ class PeerManager:
         elif script_type == 'client-disconnect':
             utils.log('%s has disconnected' % (arg,), 3)
         elif script_type == 'route-up':
-            if arg != self._external_ip:
-                self._external_ip = arg
-                utils.log('External Ip : ' + arg, 3)
+            if not self._manual:
+                external_ip, external_port = arg.split(',')
+                new_address = [[external_ip, external_port, 'udp'],
+                                 [external_ip, external_port, 'tcp-client']]
+                if self._address != new_address:
+                    self._address = new_address
+                    utils.log('Received new external configuration : %:%s' % (external_ip, external_port), 3)
                     self._declare()
         else:
             utils.log('Unknow message recieved from the openvpn pipe : ' + msg, 1)

ovpn-client

@@ -5,4 +5,4 @@ if os.environ['script_type'] == 'up':
     os.execlp('ip', 'ip', 'link', 'set', os.environ['dev'], 'up')
 
 # Write into pipe external ip address received
-os.write(int(sys.argv[1]), '%(script_type)s %(OPENVPN_external_ip)s\n' % os.environ)
+os.write(int(sys.argv[1]), '%(script_type)s %(OPENVPN_external_ip)s,%(OPENVPN_external_port)s\n' % os.environ)

ovpn-server

@@ -49,6 +49,8 @@ if script_type == 'client-connect':
     with open(sys.argv[2], 'w') as f:
         f.write('push "setenv-safe external_ip %s"\n'
                 % os.environ['trusted_ip'])
+        f.write('push "setenv-safe external_port %s"\n'
+                % os.environ['trusted_port'])
 
 # Write into pipe connect/disconnect events
 os.write(int(sys.argv[1]), '%(script_type)s %(common_name)s\n' % os.environ)

plib.py

@@ -17,8 +17,7 @@ def openvpn(hello_interval, *args, **kw):
     utils.log(str(args), 5)
     return subprocess.Popen(args, **kw)
 
-def server(server_ip, network, max_clients, dh_path, pipe_fd, port, proto, 
-        hello_interval, *args, **kw):
+def server(server_ip, network, max_clients, dh_path, pipe_fd, port, proto, hello_interval, *args, **kw):
     utils.log('Starting server', 3)
     return openvpn(hello_interval,
         '--tls-server',
@@ -32,15 +31,16 @@ def server(server_ip, network, max_clients, dh_path, pipe_fd, port, proto,
         '--proto', proto,
         *args, **kw)
 
-def client(server_ip, pipe_fd, hello_interval, *args, **kw):
+def client(server_address, pipe_fd, hello_interval, *args, **kw):
     utils.log('Starting client', 5)
-    return openvpn(hello_interval,
-        '--nobind',
+    remote= ['--nobind', 
              '--client',
-        '--remote', server_ip,
              '--up', 'ovpn-client',
-        '--route-up', 'ovpn-client ' + str(pipe_fd),
-        *args, **kw)
+             '--route-up', 'ovpn-client ' + str(pipe_fd) ]
+    for ip, port, proto in utils.address_set(server_address):
+        remote += '--remote', ip, port, proto
+    remote += args
+    return openvpn(hello_interval, *remote, **kw)
 
 def router(network, internal_ip, interface_list,
            wireless, hello_interval, **kw):
@@ -69,11 +69,3 @@ def router(network, internal_ip, interface_list,
     utils.log(str(args), 5)
     return subprocess.Popen(args, **kw)
 
-def watch(interface):
-    return ( subprocess.call(['ip6tables', '-I', 'INPUT', '-i', interface]) and
-             subprocess.call(['ip6tables', '-I', 'OUTPUT', '-o', interface]))
-
-def unwatch(interface):
-    return ( subprocess.call(['ip6tables', '-D', 'INPUT', '-i', interface]) and
-             subprocess.call(['ip6tables', '-D', 'OUTPUT', '-o', interface]))
-

registry.py

@@ -52,9 +52,7 @@ class main(object):
         self.db = sqlite3.connect(self.config.db, isolation_level=None)
         self.db.execute("""CREATE TABLE IF NOT EXISTS peers (
                         prefix text primary key not null,
-                        ip text not null,
-                        port integer not null,
-                        proto text not null,
+                        address text not null,
                         date integer default (strftime('%s','now')))""")
         self.db.execute("""CREATE TABLE IF NOT EXISTS tokens (
                         token text primary key not null,
@@ -62,22 +60,22 @@ class main(object):
                         prefix_len integer not null,
                         date integer not null)""")
         try:
-            self.db.execute("""CREATE TABLE vifib (
+            self.db.execute("""CREATE TABLE vpn (
                                prefix text primary key not null,
                                email text,
                                cert text)""")
         except sqlite3.OperationalError, e:
-            if e.args[0] != 'table vifib already exists':
+            if e.args[0] != 'table vpn already exists':
                 raise RuntimeError
         else:
-            self.db.execute("INSERT INTO vifib VALUES ('',null,null)")
+            self.db.execute("INSERT INTO vpn VALUES ('',null,null)")
 
         # Loading certificates
         with open(self.config.ca) as f:
             self.ca = crypto.load_certificate(crypto.FILETYPE_PEM, f.read())
         with open(self.config.key) as f:
             self.key = crypto.load_privatekey(crypto.FILETYPE_PEM, f.read())
-        # Get vifib network prefix
+        # Get vpn network prefix
         self.network = bin(self.ca.get_serial_number())[3:]
         print "Network prefix : %s/%u" % (self.network, len(self.network))
 
@@ -121,12 +119,12 @@ class main(object):
 
     def _getPrefix(self, prefix_len):
         assert 0 < prefix_len <= 128 - len(self.network)
-        for prefix, in self.db.execute("""SELECT prefix FROM vifib WHERE length(prefix) <= ? AND cert is null
+        for prefix, in self.db.execute("""SELECT prefix FROM vpn WHERE length(prefix) <= ? AND cert is null
                                          ORDER BY length(prefix) DESC""", (prefix_len,)):
             while len(prefix) < prefix_len:
-                self.db.execute("UPDATE vifib SET prefix = ? WHERE prefix = ?", (prefix + '1', prefix))
+                self.db.execute("UPDATE vpn SET prefix = ? WHERE prefix = ?", (prefix + '1', prefix))
                 prefix += '0'
-                self.db.execute("INSERT INTO vifib VALUES (?,null,null)", (prefix,))
+                self.db.execute("INSERT INTO vpn VALUES (?,null,null)", (prefix,))
             return prefix
         raise RuntimeError # TODO: raise better exception
 
@@ -158,7 +156,7 @@ class main(object):
             cert = crypto.dump_certificate(crypto.FILETYPE_PEM, cert)
 
             # Insert certificate into db
-            self.db.execute("UPDATE vifib SET email = ?, cert = ? WHERE prefix = ?", (email, cert, prefix) )
+            self.db.execute("UPDATE vpn SET email = ?, cert = ? WHERE prefix = ?", (email, cert, prefix) )
 
         return cert
       except:
@@ -172,19 +170,19 @@ class main(object):
         # TODO: Insert a flag column for bootstrap ready servers in peers
         # ( servers which shouldn't go down or change ip and port as opposed to servers owned by particulars )
         # that way, we also ascertain that the server sent is not the new node....
-        ip, port, proto = self.db.execute("SELECT ip, port, proto FROM peers ORDER BY random() LIMIT 1").next()
-        print "Sending bootstrap peer ( %s, %s, %s)" % (ip, port, proto)
-        return ip, port, proto
+        prefix, address = self.db.execute("SELECT prefix, address FROM peers ORDER BY random() LIMIT 1").next()
+        print "Sending bootstrap peer (%s, %s)" % (prefix, str(address))
+        return prefix, address
 
     def declare(self, handler, address):
         print "declaring new node"
-        client_address, ip, port, proto = address
+        client_address, address = address
         #client_address, _ = handler.client_address
         client_ip = utils.binFromIp(client_address)
         if client_ip.startswith(self.network):
             prefix = client_ip[len(self.network):]
-            prefix, = self.db.execute("SELECT prefix FROM vifib WHERE prefix <= ? ORDER BY prefix DESC LIMIT 1", (prefix,)).next()
-            self.db.execute("INSERT OR REPLACE INTO peers (prefix, ip, port, proto) VALUES (?,?,?,?)", (prefix, ip, port, proto))
+            prefix, = self.db.execute("SELECT prefix FROM vpn WHERE prefix <= ? ORDER BY prefix DESC LIMIT 1", (prefix,)).next()
+            self.db.execute("INSERT OR REPLACE INTO peers (prefix, address) VALUES (?,?)", (prefix, address))
             return True
         else:
             # TODO: use log + DO NOT PRINT BINARY IP
@@ -196,7 +194,7 @@ class main(object):
         client_ip = utils.binFromIp(client_address)
         if client_ip.startswith(self.network):
             print "sending peers"
-            return self.db.execute("SELECT ip, port, proto FROM peers ORDER BY random() LIMIT ?", (n,)).fetchall()
+            return self.db.execute("SELECT prefix, address FROM peers ORDER BY random() LIMIT ?", (n,)).fetchall()
         else:
             # TODO: use log + DO NOT PRINT BINARY IP
             print "Unauthorized connection from %s which does not start with %s" % (client_ip, self.network)

setup.py

@@ -18,13 +18,9 @@ def main():
             help='Port to which connect on the server')
     _('-d', '--dir', default='/etc/vifib',
             help='Directory where the key and certificate will be stored')
-    _('-r', '--req', nargs='+',
-            help='''Certificate request additional arguments. For example :
-                  --req name1 value1 name2 value2, to add attributes name1 and name2''')
+    _('-r', '--req', nargs=2, action='append',
+            help='Name and value of certificate request additional arguments')
     config = parser.parse_args()
-    if config.req and len(config.req) % 2 == 1:
-        print "Sorry, request argument was incorrect, there must be an even number of request arguments"
-        sys.exit(1)
 
     # Establish connection with server
     s = xmlrpclib.ServerProxy('http://%s:%u' % (config.server, config.port))
@@ -41,17 +37,14 @@ def main():
     db = sqlite3.connect(os.path.join(config.dir, 'peers.db'), isolation_level=None)
     try:
         db.execute("""CREATE TABLE peers (
-                   id INTEGER PRIMARY KEY AUTOINCREMENT,
-                   ip TEXT NOT NULL,
-                   port INTEGER NOT NULL,
-                   proto TEXT NOT NULL,
-                   used INTEGER NOT NULL default 0,
+                   prefix TEXT PRIMARY KEY,
+                   address TEXT NOT NULL,
+                   used INTEGER NOT NULL DEFAULT 0,
                    date INTEGER DEFAULT (strftime('%s', 'now')))""")
         db.execute("CREATE INDEX _peers_used ON peers(used)")
-        db.execute("CREATE UNIQUE INDEX _peers_address ON peers(ip, port, proto)")
         if not config.no_boot:
-            boot_ip, boot_port, boot_proto = s.getBootstrapPeer()
-            db.execute("INSERT INTO peers (ip, port, proto) VALUES (?,?,?)", (boot_ip, boot_port, boot_proto))
+            prefix, address = s.getBootstrapPeer()
+            db.execute("INSERT INTO peers (prefix, address) VALUES (?,?)", (prefix, address))
     except sqlite3.OperationalError, e:
         if e.args[0] == 'table peers already exists':
             print "Table peers already exists, leaving it as it is"
@@ -75,10 +68,8 @@ def main():
     req = crypto.X509Req()
     subj = req.get_subject()
     if config.req:
-        while len(config.req) > 1:
-            key = config.req.pop(0)
-            value = config.req.pop(0)
-            setattr(subj, key, value)
+        for arg in config.req:
+            setattr(subj, arg[0], arg[1])
     req.set_pubkey(pkey)
     req.sign(pkey, 'sha1')
     req = crypto.dump_certificate_request(crypto.FILETYPE_PEM, req)

tunnel.py

@@ -5,12 +5,12 @@ log = None
 smooth = 0.3
 
 class Connection:
-    def __init__(self, ip, write_pipe, hello, port, proto, iface, peer_id,
+    def __init__(self, address, write_pipe, hello, iface, prefix,
             ovpn_args):
-        self.process = plib.client(ip, write_pipe, hello,
+        self.process = plib.client(address, write_pipe, hello,
                 '--dev', iface, '--proto', proto, '--rport', str(port),
                 *ovpn_args, stdout=os.open(os.path.join(log,
-                'vifibnet.client.%s.log' % (peer_id,)),
+                'vifibnet.client.%s.log' % (prefix,)),
                 os.O_WRONLY|os.O_CREAT|os.O_TRUNC) )
 
         self.iface = iface
@@ -22,7 +22,7 @@ class Connection:
         # Check that the connection is alive
         if self.process.poll() != None:
             utils.log('Connection with %s has failed with return code %s' 
-                     % (id, self.process.returncode), 3)
+                     % (prefix, self.process.returncode), 3)
             return False
 
         trafic = self._getTrafic()
@@ -70,42 +70,41 @@ class TunnelManager:
         self.next_refresh = time.time() + self._refresh_time
 
     def _cleanDeads(self):
-        for id in self._connection_dict.keys():
-            if not self._connection_dict[id].refresh():
-                self._kill(id)
+        for prefix in self._connection_dict.keys():
+            if not self._connection_dict[prefix].refresh():
+                self._kill(prefix)
 
     def _removeSomeTunnels(self):
         for i in range(0, max(0, len(self._connection_dict) -
                     self._client_count + self._refresh_count)):
-            peer_id = random.choice(self._connection_dict.keys())
-            self._kill(peer_id)
+            prefix = random.choice(self._connection_dict.keys())
+            self._kill(prefix)
 
-    def _kill(self, peer_id):
-        utils.log('Killing the connection with id ' + str(peer_id), 2)
-        connection = self._connection_dict.pop(peer_id)
+    def _kill(self, prefix):
+        utils.log('Killing the connection with ' + prefix, 2)
+        connection = self._connection_dict.pop(prefix)
         try:
             connection.process.kill()
         except OSError:
             # If the process is already exited
             pass
         self.free_interface_set.add(connection.iface)
-        self._peer_db.unusePeer(peer_id)
+        self._peer_db.unusePeer(prefix)
 
     def _makeNewTunnels(self):
         utils.log('Trying to make %i new tunnels' %
-                (self._client_count - len(self._connection_dict)), 3)
+                (self._client_count - len(self._connection_dict)), 5)
         try:
-            for peer_id, ip, port, proto in self._peer_db.getUnusedPeers(
+            for prefix, address in self._peer_db.getUnusedPeers(
                     self._client_count - len(self._connection_dict)):
-                utils.log('Establishing a connection with id %s (%s:%s)'
-                        % (peer_id, ip, port), 2)
+                utils.log('Establishing a connection with %s (%s:%s)' % prefix, 2)
                 iface = self.free_interface_set.pop()
-                self._connection_dict[peer_id] = Connection(ip, 
-                        self._write_pipe, self._hello, port, proto, iface,
-                        peer_id, self._ovpn_args)
-                self._peer_db.usePeer(peer_id)
+                self._connection_dict[prefix] = Connection(address,
+                        self._write_pipe, self._hello, iface,
+                        prefix, self._ovpn_args)
+                self._peer_db.usePeer(prefix)
         except KeyError:
-            utils.log("Can't establish connection with %s"
-                    ": no available interface" % ip, 2)
+            utils.log("""Can't establish connection with %s
+                         : no available interface""" % prefix, 2)
         except Exception:
             traceback.print_exc()

utils.py

@@ -21,7 +21,7 @@ def ipFromBin(prefix):
 def ipFromPrefix(vifibnet, prefix, prefix_len):
     prefix = bin(int(prefix))[2:].rjust(prefix_len, '0')
     ip_t = (vifibnet + prefix).ljust(128, '0')
-    return ipFromBin(ip_t)
+    return ipFromBin(ip_t), prefix
 
 def networkFromCa(ca_path):
     # Get network prefix from ca.crt
@@ -37,13 +37,9 @@ def ipFromCert(network, cert_path):
         prefix, prefix_len = subject.CN.split('/')
         return ipFromPrefix(network, prefix, int(prefix_len))
 
-def ovpnArgs(optional_args, ca_path, cert_path):
-    # Treat openvpn arguments
-    if optional_args[0] == "--":
-        del optional_args[0]
-    optional_args.append('--ca')
-    optional_args.append(ca_path)
-    optional_args.append('--cert')
-    optional_args.append(cert_path)
-    return optional_args
+def address_list(address_set):
+    return ';'.join(map(','.join, address_set))
 
+def address_set(address_list):
+    return set(tuple(address.split(','))
+        for address in address_list.split(';'))

vifibnet.py

 #!/usr/bin/env python
-import argparse, errno, math, os, select, subprocess, sys, time, traceback, upnpigd
+import argparse, errno, math, os, select, subprocess, sys, time, traceback
+from argparse import ArgumentParser
 from OpenSSL import crypto
 import db, plib, upnpigd, utils, tunnel
 
+def ArgParser(ArgumentParser):
+
+    def convert_arg_line_to_args(self, arg_line):
+        for arg in ('--' + arg_line.strip()).split():
+            if arg.strip():
+                yield arg
+
+def ovpnArgs(optional_args, ca_path, cert_path):
+    # Treat openvpn arguments
+    if optional_args[0] == "--":
+        del optional_args[0]
+    optional_args.append('--ca')
+    optional_args.append(ca_path)
+    optional_args.append('--cert')
+    optional_args.append(cert_path)
+    return optional_args
+
 def getConfig():
     parser = argparse.ArgumentParser(
             description='Resilient virtual private network application')
     _ = parser.add_argument
     # Server address SHOULD be a vifib address ( else requests will be denied )
     _('--server', required=True,
-            help='Address for peer discovery server')
+            help="VPN address of the discovery peer server")
     _('--server-port', required=True, type=int,
-            help='Peer discovery server port')
+            help="VPN port of the discovery peer server")
     _('-log', '-l', default='/var/log',
             help='Path to vifibnet logs directory')
     _('--tunnel-refresh', default=300, type=int,
@@ -34,15 +52,15 @@ def getConfig():
             help='Path to the certificate authority file')
     _('--cert', required=True,
             help='Path to the certificate file')
-    _('--ip', default=None, dest='external_ip',
-            help='Ip address of the machine on the internet')
-    _('--internal-port', default=1194,
-            help='The internal port to listen on for incomming connections')
-    _('--external-port', default=1194,
-            help='The external port to advertise for other peers to connect')
-    _('--proto', default='udp',
-            help='The protocol to use for the others peers to connect')
+    ipconfig = parser.add_mutually_exclusive_group()
+    __ = ipconfig.add_argument
+    __('--ip', default=None, dest='address', action='append', nargs=3,
+            help='Ip address, port and protocol advertised to other vpn nodes')
+    __('--internal-port', default=1194,
+            help='Internal port to listen on for incomming connections')
     # args to be removed ?
+    _('--proto', default='udp',
+            help='The protocol used by other peers to connect')
     _('--connection-count', default=30, type=int,
             help='Number of client connections')
     _('--refresh-rate', default=0.05, type=float,
@@ -55,9 +73,10 @@ def getConfig():
 def main():
     # Get arguments
     config = getConfig()
+    manual = bool(config.address)
     network = utils.networkFromCa(config.ca)
-    internal_ip = utils.ipFromCert(network, config.cert)
-    openvpn_args = utils.ovpnArgs(config.openvpn_args, config.ca, config.cert)
+    internal_ip, prefix = utils.ipFromCert(network, config.cert)
+    openvpn_args = ovpnArgs(config.openvpn_args, config.ca, config.cert)
 
     # Set global variables
     tunnel.log = config.log
@@ -69,14 +88,20 @@ def main():
     read_pipe = os.fdopen(r_pipe)
 
     # Init db and tunnels
-    if config.external_ip == None:
+    if manual:
+        utils.log('Manual external configuration', 3)
+    else:
+        utils.log('Attempting automatic configuration via UPnP', 3)
         try:
-            config.external_ip, config.external_port = upnpigd.ForwardViaUPnP(config.internal_port)
+            external_ip, external_port = upnpigd.ForwardViaUPnP(config.internal_port)
+            config.address = [[external_ip, external_port, 'udp'],
+                              [external_ip, external_port, 'tcp-client']]
         except Exception:
-            utils.log('An atempt to forward a port via UPnP failed', 5)
+            utils.log('An atempt to forward a port via UPnP failed', 3)
+            raise RuntimeError
 
-    peer_db = db.PeerManager(config.db, config.server, config.server_port, config.peers_db_refresh,
-            config.external_ip, internal_ip, config.external_port, config.proto, 200)
+    peer_db = db.PeerManager(config.db, config.server, config.server_port,
+            config.peers_db_refresh, config.address, internal_ip, prefix, manual, 200)
     tunnel_manager = tunnel.TunnelManager(write_pipe, peer_db, openvpn_args, config.hello,
             config.tunnel_refresh, config.connection_count, config.refresh_rate)