Merge branch 'master' of https://git.erp5.org/repos/vifibnet

Conflicts:
	TODO
	tunnel.py

README

@@ -154,7 +154,7 @@ OPTIONS : VIFIBNET.PY
             line, vifibnet will attempt to forward a port with upnp for each
             couple port/proto given.
             Protocols should be either udp or tcp-server.
-            Default : (1194, udp)
+            Default : (1194, udp), (1194, tcp-server)
 
     --tunnel-refresh duration
             Interval in seconds between two tunnel refresh. Refreshing tunnels

TODO

-To be done :
-    use the server as a bootstrap node -> switch peer discovery to be done
-    by vifibnet directly ?
+URGENT => Name ideas :
+    resnet ( for Resiliable NET )
+    rsnet ( Resiliable Scalable NET )
 
+To be done :
     Use an algorithm to choose which connections to keep and/or establish
     instead of pure randomness
         number of routes / tunnel
@@ -9,6 +10,8 @@ To be done :
 
     Write docstrings for all class/methods/functions
 
+    Handle corrupt peers DB gracefully
+
 
 To be discussed:
     U : Babel seems to be very long to establish the routes : maybe we should

db.py

@@ -28,6 +28,11 @@ class PeerManager:
         utils.log('Preparing peers database...', 4)
         try:
             self._db.execute("UPDATE peers SET used = 0")
+            self._db.execute("""CREATE TABLE IF NOT EXISTS blacklist (
+                                prefix TEXT PRIMARY KEY,
+                                flag INTEGER NOT NULL)""")
+            self._db.execute("""CREATE INDEX IF NOT EXISTS
+                                blacklist_flag ON blacklist(flag)""")
         except sqlite3.OperationalError, e:
             if e.args[0] == 'no such table: peers':
                 raise RuntimeError
@@ -35,13 +40,22 @@ class PeerManager:
 
         self.next_refresh = time.time()
 
-    def reset_blacklist(self):
-        self._blacklist = [(self._prefix)]
+    def clear_blacklist(self, flag):
+        utils.log('Clearing blacklist from flag %u' % (flag,), 3)
+        self._db.execute("DELETE FROM blacklist WHERE flag = ?", (flag,))
+        utils.log('Blacklist cleared', 5)
 
-    def blacklist(self, prefix):
+    def blacklist(self, prefix, flag):
         utils.log('Blacklisting %s' % (prefix,), 4)
         self._db.execute("DELETE FROM peers WHERE prefix = ?", (prefix,))
-        self._blacklist = list(set(self._blacklist + [(prefix,)]))
+        self._db.execute("INSERT OR REPLACE INTO blacklist VALUES (?,?)",
+                          (prefix, flag))
+        utils.log('%s blacklisted' % (prefix,), 5)
+
+    def whitelist(self, prefix):
+        utils.log('Unblacklisting %s' % (prefix,), 4)
+        self._db.execute("DELETE FROM blacklist WHERE prefix = ?", (prefix,))
+        utils.log('%s whitelisted' % (prefix,), 5)
 
     def refresh(self):
         utils.log('Refreshing the peers DB...', 2)
@@ -74,8 +88,8 @@ class PeerManager:
                             (str(len(new_peer_list) - self._db_size),))
         self._db.executemany("""INSERT OR IGNORE INTO peers (prefix, address)
                                 VALUES (?,?)""", new_peer_list)
-        self._db.executemany("DELETE FROM peers WHERE prefix = ?",
-                              self._blacklist)
+        self._db.execute("""DELETE FROM peers WHERE prefix IN
+                            (SELECT prefix FROM blacklist)""")
         utils.log('DB populated', 3)
         utils.log('New peers : %s' % ', '.join(map(str, new_peer_list)), 5)
 
@@ -121,3 +135,4 @@ class PeerManager:
         else:
             utils.log('Unknow message recieved from the openvpn pipe : '
                     + msg, 1)
+

registry.py

@@ -55,6 +55,9 @@ class main(object):
                 help='Path to certificate key')
         _('--mailhost', required=True,
                 help='SMTP server mail host')
+        _('--bootstrap', nargs=4, action="append",
+                help='''VPN prefix, ip address, port and protocol to send as
+                        bootstrap peers, instead of random ones''')
         self.config = parser.parse_args()
 
         # Database initializing
@@ -180,8 +183,14 @@ class main(object):
         # TODO: Insert a flag column for bootstrap ready servers in peers
         # ( servers which shouldn't go down or change ip and port as opposed to servers owned by particulars )
         # that way, we also ascertain that the server sent is not the new node....
-        prefix, address = self.db.execute("SELECT prefix, address FROM peers ORDER BY random() LIMIT 1").next()
-        print "Sending bootstrap peer (%s, %s)" % (prefix, str(address))
+        if self.config.bootstrap:
+            bootpeer = random.choice(self.config.bootstrap)
+            prefix = bootpeer[0]
+            address = ','.join(bootpeer[1:])
+        else:
+            prefix, address = self.db.execute("""SELECT prefix, address
+                              FROM peers ORDER BY random() LIMIT 1""")
+        print "Sending bootstrap peer (%s, %s)" % (prefix, address)
         return prefix, address
 
     def declare(self, handler, address):

tunnel.py

@@ -7,7 +7,6 @@ smooth = 0.3     # this is used to smooth the traffic sampling. Lower value
 protected = 0.2  # ratio of the tunnels protected against kill because they are
                  # used a lot
 
-
 # Be carfull the refresh interval should let the routes be established
 
 
@@ -35,9 +34,13 @@ class Connection:
                      % (self._prefix, self.process.returncode), 3)
             return False
 
-        self._updateBandwidth()
+        # self._updateBandwidth()
         return True
 
+    # Unused for now. By killing tunnels with significantly lower trafic
+    # in comparison to other tunnels, we hope to connect to nodes with
+    # better bandwith, in order to improve connectivity with destinations
+    # we are really interested in.
     def _updateBandwidth(self):
         try:
             f_rx = open('/sys/class/net/%s/statistics/rx_bytes' %
@@ -81,7 +84,6 @@ class TunnelManager:
         self._network = network
         self._net_len = len(network)
         self._iface_list = iface_list
-        self.__indirect_connect = []
         self.free_interface_set = set(('client1', 'client2', 'client3',
                                        'client4', 'client5', 'client6',
                                        'client7', 'client8', 'client9',
@@ -154,18 +156,16 @@ class TunnelManager:
 
     def _countRoutes(self):
         utils.log('Starting to count the routes on each interface...', 3)
-        self._indirect_connect = []
+        self._peer_db.clear_blacklist(0)
         for iface in self._iface_to_prefix.keys():
             self._connection_dict[self._iface_to_prefix[iface]].routes = 0
-        f = open('/proc/net/ipv6_route', 'r')
-        for line in f:
-            ip, subnet_size, iface = struct.unpack('32s x 2s 106x %ss x'
-                % (len(line) - 142), line)
-            ip = bin(int(ip, 16))[2:].rjust(128, '0')
+        for line in open('/proc/net/ipv6_route'):
+            line = line.split()
+            ip = bin(int(line[0], 16))[2:].rjust(128, '0')
 
             if ip.startswith(self._network):
-                iface = iface.strip()
-                subnet_size = int(subnet_size, 16)
+                iface = line[-1]
+                subnet_size = int(line[1], 16)
                 utils.log('Route on iface %s detected to %s/%s'
                         % (iface, ip, subnet_size), 8)
                 if iface in self._iface_to_prefix.keys():
@@ -174,7 +174,7 @@ class TunnelManager:
                     prefix = ip[self._net_len:subnet_size]
                     utils.log('A route to %s has been discovered on the LAN'
                             % (prefix,), 3)
-                    self._peer_db.blacklist(prefix)
+                    self._peer_db.blacklist(prefix, 0)
 
         utils.log("Routes have been counted", 3)
         for p in self._connection_dict.keys():
@@ -185,3 +185,4 @@ class TunnelManager:
     def killAll(self):
         for prefix in self._connection_dict.keys():
             self._kill(prefix)
+            

vifibnet.py

@@ -85,7 +85,7 @@ def main():
     # Get arguments
     config = getConfig()
     if not config.pp:
-        config.pp = [['1194', 'udp']]
+        config.pp = [['1194', 'udp'], ['1194', 'tcp-server']]
     manual = bool(config.address)
     network = utils.networkFromCa(config.ca)
     internal_ip, prefix = utils.ipFromCert(network, config.cert)
@@ -137,7 +137,7 @@ def main():
    # Establish connections
     server_process = list(plib.server(internal_ip, len(network) + len(prefix),
         config.connection_count, config.dh, write_pipe, port,
-        proto, config.hello, '--dev', 'vifibnet', *openvpn_args,
+        proto, config.hello, '--dev', 'vifibnet-%s' % proto, *openvpn_args,
         stdout=os.open(os.path.join(config.log,
             'vifibnet.server.%s.log' % (proto,)),
             os.O_WRONLY | os.O_CREAT | os.O_TRUNC),