Merge branch 're6st-master'

slapos/recipe/re6stnet/__init__.py

@@ -71,9 +71,13 @@ class Recipe(GenericBaseRecipe):
   def generateCertificate(self):
     key_file = self.options['key-file'].strip()
     cert_file = self.options['cert-file'].strip()
+    dh_file = self.options['dh-file'].strip()
     if not os.path.exists(key_file):
       serial = self.getSerialFromIpv6(self.options['ipv6-prefix'].strip())
 
+      dh_command = [self.options['openssl-bin'], 'dhparam', '-out',
+                            '%s' % dh_file, self.options['key-size']]
+
       key_command = [self.options['openssl-bin'], 'genrsa', '-out',
                             '%s' % key_file, self.options['key-size']]
 
@@ -82,6 +86,7 @@ class Recipe(GenericBaseRecipe):
                   '-x509', '-batch', '-key', '%s' % key_file, '-set_serial',
                   '%s' % serial, '-days', '3650', '-out', '%s' % cert_file]
 
+      subprocess.check_call(dh_command)
       subprocess.check_call(key_command)
       subprocess.check_call(cert_command)
 
@@ -96,7 +101,7 @@ class Recipe(GenericBaseRecipe):
       if not reference in token_dict:
         # we generate new token
         number = reference.split('-')[1]
-        new_token = number + ''.join(random.sample(string.ascii_lowercase, 15))
+        new_token = number + ''.join(random.sample(string.ascii_lowercase, 20))
         token_dict[reference] = new_token
         to_add_dict[reference] = new_token
 
@@ -127,6 +132,17 @@ class Recipe(GenericBaseRecipe):
       return content
     return ''
 
+  def genHash(self, length):
+    hash_path = os.path.join(self.options['conf-dir'], '%s-hash' % length)
+    if not os.path.exists(hash_path):
+      pool = string.letters + string.digits
+      hash_string = ''.join(random.choice(pool) for i in xrange(length))
+      self.writeFile(hash_path, hash_string)
+    else:
+      hash_string = self.readFile(hash_path)
+
+    return hash_string
+
   def install(self):
     path_list = []
     token_save_path = os.path.join(self.options['conf-dir'], 'token.json')
@@ -156,7 +172,7 @@ class Recipe(GenericBaseRecipe):
       path = os.path.join(token_list_path, '%s.remove' % reference)
       if not os.path.exists(path):
         self.createFile(path, rm_token_dict[reference])
-        # remove request add file if exists
+        # remove request add token if exists
         add_path = os.path.join(token_list_path, '%s.add' % reference)
         if os.path.exists(add_path):
           os.unlink(add_path)
@@ -191,6 +207,12 @@ class Recipe(GenericBaseRecipe):
       )
     path_list.append(request_check)
 
+    revoke_check = self.createPythonScript(
+        self.options['revoke-service-wrapper'].strip(),
+        '%s.re6stnet.requestRevoqueCertificate' % __name__, service_dict
+      )
+    path_list.append(revoke_check)
+
     # Send connection parameters of slave instances
     if token_dict:
       self.slap.initializeConnection(self.server_url, self.key_file,
@@ -211,7 +233,7 @@ class Recipe(GenericBaseRecipe):
           computer_partition.setConnectionDict(
               {'token':token, '1_info':msg},
               slave_reference)
-        except:
+        except Exception:
           self.logger.fatal("Error while sending slave %s informations: %s",
              slave_reference, traceback.format_exc())
 

slapos/recipe/re6stnet/re6stnet.py

@@ -5,8 +5,10 @@ import os
 import time
 import sqlite3
 import slapos
+import traceback
 
-from re6st import registry
+from re6st import registry, x509
+from OpenSSL import crypto
 
 log = logging.getLogger('SLAPOS-RE6STNET')
 logging.basicConfig(level=logging.DEBUG)
@@ -50,6 +52,7 @@ def bang(args):
   partition.bang(message='Published parameters changed!')
   log.info("Bang with message 'parameters changed'...")
 
+
 def requestAddToken(args, can_bang=True):
 
   time.sleep(3)
@@ -69,12 +72,13 @@ def requestAddToken(args, can_bang=True):
     token = readFile(request_file)
     if token :
       reference = reference_key.split('.')[0]
+      # email is unique as reference is also unique
       email = '%s@slapos' % reference.lower()
       try:
         result = client.requestAddToken(token, email)
-      except Exception, e:
+      except Exception:
         log.debug('Request add token fail for %s... \n %s' % (request_file,
-                    str(e)))
+                    traceback.format_exc()))
         continue
       if result and result == token:
         # update information
@@ -97,7 +101,7 @@ def requestRemoveToken(args):
   if not path_list:
     log.info("No token to delete. Exiting...")
     return
-  
+
   client = registry.RegistryClient(args['registry_url'])
   for reference_key in path_list:
     request_file = os.path.join(base_token_path, reference_key)
@@ -106,23 +110,58 @@ def requestRemoveToken(args):
       reference = reference_key.split('.')[0]
       try:
         result = client.requestDeleteToken(token)
-      except Exception, e:
+      except Exception:
         log.debug('Request delete token fail for %s... \n %s' % (request_file,
-                    str(e)))
+                    traceback.format_exc()))
         continue
+      else:
+        # certificate is invalidated, it will be revoked
+        writeFile(os.path.join(base_token_path, '%s.revoke' % reference), '')
       if result == 'True':
         # update information
         log.info("Token deleted for slave instance %s. Clean up file status..." %
                             reference)
+      if result in ['True', 'False']:
         os.unlink(request_file)
         status_file = os.path.join(base_token_path, '%s.status' % reference)
         if os.path.exists(status_file):
           os.unlink(status_file)
-      else:
-        log.debug('Request delete token fail for %s...' % request_file)
     else:
       log.debug('Bad token. Request add token fail for %s...' % request_file)
 
+def requestRevoqueCertificate(args):
+
+  base_token_path = args['token_base_path']
+  db = getDb(args['db'])
+  path_list = [x for x in os.listdir(base_token_path) if x.endswith('.revoke')]
+  client = registry.RegistryClient(args['registry_url'])
+
+  for reference_key in path_list:
+    reference = reference_key.split('.')[0]
+    # XXX - email is always unique
+    email = '%s@slapos' % reference.lower()
+    cert_string = ''
+    try:
+      cert_string, = db.execute("SELECT cert FROM cert WHERE email = ?",
+          (email,)).next()
+    except StopIteration:
+      # Certificate was not generated yet !!!
+      pass
+
+    try:
+      if cert_string:
+        cert = crypto.load_certificate(crypto.FILETYPE_PEM, cert_string)
+        cn = x509.subnetFromCert(cert)
+        result = client.revoke(str(cn))
+        time.sleep(2)
+    except Exception:
+      log.debug('Request revoke certificate fail for %s... \n %s' % (reference,
+                  traceback.format_exc()))
+      continue
+    else:
+      os.unlink(os.path.join(base_token_path, reference_key))
+      log.info("Certificate revoked for slave instance %s." % reference)
+
 def checkService(args, can_bang=True):
   base_token_path = args['token_base_path']
   token_dict = loadJsonFile(args['token_json'])
@@ -164,7 +203,7 @@ def checkService(args, can_bang=True):
         time.sleep(1)
         writeFile(status_file, 'TOKEN_USED')
         log.info("Token status of %s updated to 'used'." % slave_reference)
-      except IOError, e:
+      except IOError:
         # XXX- this file should always exists
         log.debug('Error when writing in file %s. Clould not update status of %s...' %
                               (status_file, slave_reference))
@@ -181,3 +220,4 @@ def manage(args):
 
   # check status of all token
   checkService(args)
+

slapos/test/recipe/test_re6stnet.py

@@ -24,6 +24,7 @@ class Re6stnetTest(unittest.TestCase):
                 'openssl-bin': '/usr/bin/openssl',
                 'key-file': os.path.join(self.ssl_dir, 'cert.key'),
                 'cert-file': os.path.join(self.ssl_dir, 'cert.crt'),
+                'dh-file': os.path.join(self.ssl_dir, 'dh.pem'),
                 'key-size': '2048',
                 'conf-dir': self.conf_dir,
                 'token-dir': self.token_dir,
@@ -36,6 +37,7 @@ class Re6stnetTest(unittest.TestCase):
                 'manager-wrapper': os.path.join(self.base_dir, 'manager_wrapper'),
                 'drop-service-wrapper': os.path.join(self.base_dir, 'drop_wrapper'),
                 'check-service-wrapper': os.path.join(self.base_dir, 'check_wrapper'),
+                'revoke-service-wrapper': os.path.join(self.base_dir, 'revoke_wrapper'),
                 'slave-instance-list': '{}'
                 }
     
@@ -70,7 +72,7 @@ class Re6stnetTest(unittest.TestCase):
       options = self.options
 
       return re6stnet.Recipe(buildout=buildout, name='re6stnet', options=options)
-  
+
   def checkWrapper(self, path):
     self.assertTrue(os.path.exists(path))
     content = ""
@@ -96,7 +98,10 @@ class Re6stnetTest(unittest.TestCase):
     with open(path, 'r') as f:
         content = f.read()
     self.assertIn("@%s" % config_file, content)
-  
+
+  def fake_generateCertificates(self):
+    return
+
   def test_generateCertificates(self):
     
     self.options['ipv6-prefix'] = '2001:db8:24::/48'
@@ -106,31 +111,35 @@ class Re6stnetTest(unittest.TestCase):
     
     recipe.generateCertificate()
     
-    self.assertTrue(os.path.exists(self.options['key-file']))
-    self.assertTrue(os.path.exists(self.options['cert-file']))
+    self.assertItemsEqual(os.listdir(self.ssl_dir),
+                          ['cert.key', 'cert.crt', 'dh.pem'])
     
     last_time = time.ctime(os.stat(self.options['key-file'])[7])
-    
+
     recipe.generateCertificate()
-    
+
     self.assertTrue(os.path.exists(self.options['key-file']))
     this_time = time.ctime(os.stat(self.options['key-file'])[7])
-    
+
     self.assertEqual(last_time, this_time)
   
-  def test_generateCertificates_other_ipv6(self):
-    
-    self.options['ipv6-prefix'] = 'be28:db8:fe6a:d85:4fe:54a:ae:aea/64'
-    
+  def test_getSerialFromIpv6(self):
+
+    ipv6 = 'be28:db8:fe6a:d85:4fe:54a:ae:aea/64'
+
     recipe = self.new_recipe()
-    
-    recipe.generateCertificate()
-    
-    self.assertTrue(os.path.exists(self.options['key-file']))
-    self.assertTrue(os.path.exists(self.options['cert-file']))
+    serial = recipe.getSerialFromIpv6(ipv6)
+
+    self.assertEqual(serial, '0x1be280db8fe6a0d8504fe054a00ae0aea')
+
+    ipv6 = '2001:db8:24::/48'
+    serial = recipe.getSerialFromIpv6(ipv6)
+
+    self.assertEqual(serial, '0x120010db80024')
 
   def test_install(self):
     recipe = self.new_recipe()
+    recipe.generateCertificate = self.fake_generateCertificates
 
     recipe.options.update({
         'ipv6-prefix': '2001:db8:24::/48',
@@ -147,9 +156,6 @@ class Re6stnetTest(unittest.TestCase):
       # Recipe will raise not found error when trying to publish slave informations
       pass
     
-    self.assertItemsEqual(os.listdir(self.ssl_dir),
-                          ['cert.key', 'cert.crt'])
-    
     token_file = os.path.join(self.options['conf-dir'], 'token.json')
     self.assertTrue(os.path.exists(token_file))
     
@@ -175,6 +181,7 @@ class Re6stnetTest(unittest.TestCase):
     self.checkWrapper(os.path.join(self.base_dir, 'manager_wrapper'))
     self.checkWrapper(os.path.join(self.base_dir, 'drop_wrapper'))
     self.checkWrapper(os.path.join(self.base_dir, 'check_wrapper'))
+    self.checkWrapper(os.path.join(self.base_dir, 'revoke_wrapper'))
     self.checkRegistryWrapper()
     
     # Remove one element
@@ -198,25 +205,24 @@ class Re6stnetTest(unittest.TestCase):
   
   def test_install_empty_slave(self):
     recipe = self.new_recipe()
+    recipe.generateCertificate = self.fake_generateCertificates
 
     recipe.options.update({
         'ipv6-prefix': '2001:db8:24::/48'
         })
 
     recipe.install()
-    
-    self.assertItemsEqual(os.listdir(self.ssl_dir),
-                          ['cert.key', 'cert.crt'])
-    
+
     token_file = os.path.join(self.options['conf-dir'], 'token.json')
     self.assertTrue(os.path.exists(token_file))
     
     token_content = recipe.readFile(token_file)
     self.assertEqual(token_content, '{}')
     self.assertItemsEqual(os.listdir(self.options['token-dir']), [])
-    
+
     self.checkWrapper(os.path.join(self.base_dir, 'manager_wrapper'))
     self.checkWrapper(os.path.join(self.base_dir, 'drop_wrapper'))
     self.checkWrapper(os.path.join(self.base_dir, 'check_wrapper'))
+    self.checkWrapper(os.path.join(self.base_dir, 'revoke_wrapper'))
     self.checkRegistryWrapper()
 

software/re6stnet/instance-re6stnet.cfg.in

 
 {% set python_bin = parameter_dict['python-executable'] -%}
 {% set re6st_registry = parameter_dict['re6st-registry'] -%}
+{% set re6stnet = parameter_dict['re6stnet'] -%}
 {% set publish_dict = {} -%}
 {% set part_list = [] -%}
 {% set ipv6 = (ipv6_set | list)[0] -%}
@@ -105,6 +106,7 @@ ipv6 = {{ ipv6 }}
 db = ${re6stnet-dirs:registry}/registry.db
 ca = ${re6stnet-dirs:ssl}/re6stnet.crt
 key = ${re6stnet-dirs:ssl}/re6stnet.key
+dh = ${re6stnet-dirs:ssl}/dh.pem
 mailhost = 127.0.0.1
 prefix-length = 16
 anonymous-prefix-length = 32
@@ -119,17 +121,12 @@ context = section parameter_dict re6st-registry-conf-dict
 
 [re6st-registry]
 recipe = slapos.cookbook:re6stnet.registry
-port = ${re6st-registry-conf-dict:port}
-ipv4 = ${re6st-registry-conf-dict:ipv4}
 command = {{ re6st_registry }}
-config-file = ${re6st-registry-conf:rendered}
-db-path = ${re6st-registry-conf-dict:db}
 wrapper = ${directory:services}/re6st-registry
 manager-wrapper = ${directory:bin}/re6stManageToken
 check-service-wrapper = ${directory:bin}/re6stCheckService
 drop-service-wrapper = ${directory:bin}/re6stManageDeleteToken
-key-file = ${re6st-registry-conf-dict:key}
-cert-file = ${re6st-registry-conf-dict:ca}
+revoke-service-wrapper = ${directory:bin}/re6stRevokeCertificate
 openssl-bin = {{ openssl_bin }}/openssl
 python-bin = {{ python_bin }}
 ipv6-prefix = {{ slapparameter_dict.get('ipv6-prefix', '2001:db8:24::/48') }}
@@ -137,6 +134,15 @@ key-size = {{ slapparameter_dict.get('key-size', 2048) }}
 conf-dir = ${re6stnet-dirs:conf}
 token-dir = ${re6stnet-dirs:token}
 
+#Re6st config
+config-file = ${re6st-registry-conf:rendered}
+port = ${re6st-registry-conf-dict:port}
+ipv4 = ${re6st-registry-conf-dict:ipv4}
+db-path = ${re6st-registry-conf-dict:db}
+key-file = ${re6st-registry-conf-dict:key}
+cert-file = ${re6st-registry-conf-dict:ca}
+dh-file = ${re6st-registry-conf-dict:dh}
+
 slave-instance-list = ${slap-parameter:slave_instance_list}
 
 environment = 
@@ -154,6 +160,13 @@ name = re6stnet-check-token
 frequency = 0 */1 * * *
 command = {{ python_bin }} ${re6st-registry:check-service-wrapper}
 
+[cron-entry-re6st-revoke]
+recipe = slapos.cookbook:cron.d
+cron-entries = ${cron:cron-entries}
+name = re6stnet-revoke-cert
+frequency = */30 * * * *
+command = {{ python_bin }} ${re6st-registry:revoke-service-wrapper}
+
 [cron-entry-re6st-drop]
 recipe = slapos.cookbook:cron.d
 cron-entries = ${cron:cron-entries}
@@ -179,6 +192,7 @@ hostname = ${apache-conf:ipv6}
 port = ${apache-conf:port}
 
 {% do publish_dict.__setitem__('re6stry-url', uri_scheme ~ '://[${apache-conf:ipv6}]:${apache-conf:port}') -%}
+{% do publish_dict.__setitem__('re6stry-local-url',  'http://${re6st-registry:ipv4}:${re6st-registry:port}/') -%}
 [publish]
 recipe = slapos.cookbook:publish
 {% for name, value in publish_dict.items() -%}
@@ -197,6 +211,7 @@ parts =
   cron-entry-logrotate
   cron-entry-re6st-check
   cron-entry-re6st-drop
+  cron-entry-re6st-revoke
   apache-httpd
   publish
   

software/re6stnet/instance.cfg.in

@@ -30,7 +30,8 @@ context =
 [dynamic-template-re6stnet-parameters]
 bin-directory = {{ bin_directory }}
 python-executable  = {{ python_with_eggs }}
-re6st-registry = {{ re6stnet_registry }}
+re6st-registry = {{ bin_directory }}/re6st-registry
+re6stnet = {{ bin_directory }}/re6stnet
 template-apache-conf = {{ template_apache_conf }}
 apache-location = {{ apache_location }}
 template-re6st-registry-conf = {{ template_re6st_registry_conf }}

software/re6stnet/re6st-registry.conf.in

 port  {{ parameter_dict['port'] }}
 4  {{ parameter_dict['ipv4'] }}
-6  {{ parameter_dict['ipv6'] }}
+# 6  {{ parameter_dict['ipv6'] }}
 db  {{ parameter_dict['db'] }}
 ca  {{ parameter_dict['ca'] }}
 key  {{ parameter_dict['key'] }}
+dh   {{ parameter_dict['dh'] }}
 mailhost {{ parameter_dict['mailhost'] }}
 prefix-length  {{ parameter_dict['prefix-length'] }}
 anonymous-prefix-length  {{ parameter_dict['anonymous-prefix-length'] }}

software/re6stnet/software.cfg

@@ -78,7 +78,7 @@ context =
 < = template-jinja2-base
 filename = template.cfg
 template = ${:_profile_base_location_}/instance.cfg.in
-md5sum = 0929cf851c4883bcb5c69fc2f918eaeb
+md5sum = ded1faad7f289ffe9ac7aeee3d98413e
 extra-context =
     key apache_location apache:location
     key dash_location dash:location
@@ -89,12 +89,11 @@ extra-context =
     key template_re6st_registry_conf template-re6st-registry-conf:target
     key template_logrotate_base template-logrotate-base:rendered
     raw python_with_eggs ${buildout:directory}/bin/${extra-eggs:interpreter}
-    raw re6stnet_registry ${buildout:directory}/bin/re6st-registry
 
 [template-re6stnet]
 < = download-base
 filename = instance-re6stnet.cfg.in
-md5sum = e088fb05ea6e1ceff8a5ac00fd28bd75
+md5sum = df2a0c4f63c5e12cbd314cc02fbf23e1
 
 [template-logrotate-base]
 < = template-jinja2-base
@@ -113,7 +112,7 @@ md5sum = c220229ee37866c8cc404d602edd389d
 [template-re6st-registry-conf]
 < = download-base
 filename = re6st-registry.conf.in
-md5sum = ae910e8e154be6575bb19f6eae686a87
+md5sum = 7760a213896755e707993d67d8d980bb
 
 [check-recipe]
 recipe = plone.recipe.command