re6stnet recipe manage token and update slave instance status when token is used

setup.py

@@ -174,6 +174,7 @@ setup(name=name,
           'request.serialised = slapos.recipe.request:Serialised',
           'request.edge = slapos.recipe.request:RequestEdge',
           'requestoptional = slapos.recipe.request:RequestOptional',
+          're6stnet.registry = slapos.recipe.re6stnet:Recipe',
           'reverseproxy.nginx = slapos.recipe.reverse_proxy_nginx:Recipe',
           'seleniumrunner = slapos.recipe.seleniumrunner:Recipe',
           'sheepdogtestbed = slapos.recipe.sheepdogtestbed:SheepDogTestBed',

slapos/recipe/re6stnet/__init__.py

+##############################################################################
+#
+# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+import subprocess
+
+from slapos.recipe.librecipe import GenericBaseRecipe
+import socket
+import struct
+import os
+import string, random
+import json
+import traceback
+from slapos import slap
+
+class Recipe(GenericBaseRecipe):
+  
+  def __init__(self, buildout, name, options):
+    """Default initialisation"""
+    self.slap = slap.slap()
+
+    # SLAP related information
+    slap_connection = buildout['slap-connection']
+    self.computer_id = slap_connection['computer-id']
+    self.computer_partition_id = slap_connection['partition-id']
+    self.server_url = slap_connection['server-url']
+    self.software_release_url = slap_connection['software-release-url']
+    self.key_file = slap_connection.get('key-file')
+    self.cert_file = slap_connection.get('cert-file')
+    
+    return GenericBaseRecipe.__init__(self, buildout, name, options)
+
+  def getSerialFromIpv6(self, ipv6):
+    prefix = ipv6.split('/')[0].lower()
+    hi, lo = struct.unpack('!QQ', socket.inet_pton(socket.AF_INET6, prefix))
+    ipv6_int = (hi << 64) | lo
+    serial = '0x1%x' % ipv6_int
+
+    # delete non significant part
+    for part in prefix.split(':')[::-1]:
+      if part:
+        for i in ['0']*(4 - len(part)):
+          part = i + part
+        serial = serial.split(part)[0] + part
+        break
+
+    return serial
+
+  def generateCertificate(self):
+    key_file = self.options['key-file'].strip()
+    cert_file = self.options['cert-file'].strip()
+    if not os.path.exists(key_file):
+      serial = self.getSerialFromIpv6(self.options['ipv6-prefix'].strip())
+
+      key_command = [self.options['openssl-bin'], 'genrsa', '-out',
+                            '%s' % key_file, self.options['key-size']]
+
+      #'-config', openssl_configuration
+      cert_command = [self.options['openssl-bin'], 'req', '-nodes', '-new',
+                  '-x509', '-batch', '-key', '%s' % key_file, '-set_serial',
+                  '%s' % serial, '-days', '3650', '-out', '%s' % cert_file]
+
+      subprocess.check_call(key_command)
+      subprocess.check_call(cert_command)
+
+  def generateSlaveTokenList(self, slave_instance_list, token_file):
+    to_remove_dict = {}
+    to_add_dict = {}
+    token_dict = self.loadJsonFile(token_file)
+
+    reference_list = [slave_instance.get('slave_reference') for slave_instance
+                        in slave_instance_list]
+    for reference in reference_list:
+      if not reference in token_dict:
+        # we generate new token
+        number = reference.split('-')[1]
+        new_token = number + ''.join(random.sample(string.ascii_lowercase, 15))
+        token_dict[reference] = new_token
+        to_add_dict[reference] = new_token
+
+    for reference in token_dict.keys():
+      if not reference in reference_list:
+        # This slave instance is destroyed ?
+        to_remove_dict[reference] = token_dict.pop(reference)
+
+    return token_dict, to_add_dict, to_remove_dict
+
+  def loadJsonFile(self, path):
+    if os.path.exists(path):
+      with open(path, 'r') as f:
+        content = f.read()
+      return json.loads(content)
+    else:
+      return {}
+
+  def writeFile(self, path, data):
+    with open(path, 'w') as f:
+      f.write(data)
+    return path
+
+  def readFile(self, path):
+    if os.path.exists(path):
+      with open(path, 'r') as f:
+        content = f.read()
+      return content
+    return ''
+
+  def install(self):
+    path_list = []
+    token_save_path = os.path.join(self.options['conf-dir'], 'token.json')
+    token_list_path = self.options['token-dir']
+
+    self.generateCertificate()
+
+    wrapper = self.createWrapper(name=self.options['wrapper'],
+                            command=self.options['command'],
+                            parameters=['@%s' % self.options['config-file']])
+
+    path_list.append(wrapper)
+
+    slave_list = json.loads(self.options['slave-instance-list'])
+    registry_url = 'http://%s:%s/' % (self.options['ipv4'], self.options['port'])
+    token_dict, add_token_dict, rm_token_dict = self.generateSlaveTokenList(
+                                                    slave_list, token_save_path)
+
+    # write request add token
+    for reference in add_token_dict:
+      path = os.path.join(token_list_path, '%s.add' % reference)
+      if not os.path.exists(path):
+        self.createFile(path, add_token_dict[reference])
+
+    # write request remove token
+    for reference in rm_token_dict:
+      path = os.path.join(token_list_path, '%s.remove' % reference)
+      if not os.path.exists(path):
+        self.createFile(path, rm_token_dict[reference])
+
+    self.createFile(token_save_path, json.dumps(token_dict))
+
+    service_dict = dict(token_base_path=token_list_path,
+                        token_json=token_save_path,
+                        db=self.options['db-path'],
+                        partition_id=self.computer_partition_id,
+                        computer_id=self.computer_id,
+                        registry_url=registry_url)
+    service_dict['server_url'] = self.server_url
+    service_dict['cert_file'] = self.cert_file
+    service_dict['key_file'] = self.key_file
+
+    request_add = self.createPythonScript(
+        self.options['manager-wrapper'].strip(),
+        '%s.re6stnet.manage' % __name__, service_dict
+      )
+    path_list.append(request_add)
+
+    request_drop = self.createPythonScript(
+        self.options['drop-service-wrapper'].strip(),
+        '%s.re6stnet.requestRemoveToken' % __name__, service_dict
+      )
+    path_list.append(request_drop)
+
+    request_check = self.createPythonScript(
+        self.options['check-service-wrapper'].strip(),
+        '%s.re6stnet.checkService' % __name__, service_dict
+      )
+    path_list.append(request_check)
+
+    # Send connection parameters of slave instances
+    if token_dict:
+      self.slap.initializeConnection(self.server_url, self.key_file,
+        self.cert_file)
+      computer_partition = self.slap.registerComputerPartition(
+        self.computer_id,
+        self.computer_partition_id)
+
+      for slave_reference, token in token_dict.iteritems():
+        try:
+          status_file = os.path.join(token_list_path, '%s.status' % slave_reference)
+          status = self.readFile(status_file) or 'New token requested'
+          msg = status
+          if status == 'TOKEN_ADDED':
+            msg = 'Token is ready for use'
+          elif status == 'TOKEN_USED':
+            msg = 'Token not available, it has been used to generate re6stnet certificate.'
+          computer_partition.setConnectionDict(
+              {'token':token, '1_info':msg},
+              slave_reference)
+        except:
+          self.logger.fatal("Error while sending slave %s informations: %s",
+             slave_reference, traceback.format_exc())
+
+    return path_list
+
+

slapos/recipe/re6stnet/re6stnet.py

+# -*- coding: utf-8 -*-
+import logging
+import json
+import os
+import time
+import sqlite3
+import slapos
+
+from re6st import registry
+
+log = logging.getLogger('SLAPOS-RE6STNET')
+logging.basicConfig(level=logging.DEBUG)
+
+def loadJsonFile(path):
+  if os.path.exists(path):
+    with open(path, 'r') as f:
+      content = f.read()
+      return json.loads(content)
+  else:
+    return {}
+
+def writeFile(path, data):
+  with open(path, 'w') as f:
+    f.write(data)
+
+def readFile(path):
+  if os.path.exists(path):
+    with open(path, 'r') as f:
+      content = f.read()
+    return content
+  return ''
+
+def getDb(db_path):
+  db = sqlite3.connect(db_path, isolation_level=None,
+                                                  check_same_thread=False)
+  db.text_factory = str
+  
+  return db.cursor()
+
+def bang(args):
+  computer_guid = args['computer_id']
+  partition_id = args['partition_id']
+  slap = slapos.slap.slap()
+
+  # Redeploy instance to update published information
+  slap.initializeConnection(args['server_url'], args['key_file'],
+                                                          args['cert_file'])
+  partition = slap.registerComputerPartition(computer_guid=computer_guid,
+                                                   partition_id=partition_id)
+  partition.bang(message='Published parameters changed!')
+  log.info("Bang with message 'parameters changed'...")
+
+def requestAddToken(args, can_bang=True):
+
+  time.sleep(3)
+  registry_url = args['registry_url']
+  base_token_path = args['token_base_path']
+  path_list = [x for x in os.listdir(base_token_path) if x.endswith('.add')]
+
+  if not path_list:
+    log.info("No new token to add. Exiting...")
+    return
+
+  client = registry.RegistryClient(registry_url)
+  call_bang = False
+
+  for reference_key in path_list:
+    request_file = os.path.join(base_token_path, reference_key)
+    token = readFile(request_file)
+    if token :
+      reference = reference_key.split('.')[0]
+      email = '%s@slapos' % reference.lower()
+      try:
+        result = client.requestAddToken(token, email)
+      except Exception, e:
+        log.debug('Request add token fail for %s... \n %s' % (request_file,
+                    str(e)))
+        continue
+      if result and result == token:
+        # update information
+        log.info("New token added for slave instance %s. Updating file status..." %
+                            reference)
+        writeFile(os.path.join(base_token_path, '%s.status' % reference),
+                    'TOKEN_ADDED')
+        os.unlink(request_file)
+        call_bang = True
+    else:
+      log.debug('Bad token. Request add token fail for %s...' % request_file)
+  
+  if can_bang and call_bang:
+    bang(args)
+
+def requestRemoveToken(args):
+  base_token_path = args['token_base_path']
+  path_list = [x for x in os.listdir(base_token_path) if x.endswith('.remove')]
+  
+  if not path_list:
+    log.info("No token to delete. Exiting...")
+    return
+  
+  client = registry.RegistryClient(args['registry_url'])
+  for reference_key in path_list:
+    request_file = os.path.join(base_token_path, reference_key)
+    token = readFile(request_file)
+    if token :
+      reference = reference_key.split('.')[0]
+      try:
+        result = client.requestDeleteToken(token)
+      except Exception, e:
+        log.debug('Request delete token fail for %s... \n %s' % (request_file,
+                    str(e)))
+        continue
+      if result == 'True':
+        # update information
+        log.info("Token deleted for slave instance %s. Clean up file status..." %
+                            reference)
+        os.unlink(request_file)
+        status_file = os.path.join(base_token_path, '%s.status' % reference)
+        if os.path.exists(status_file):
+          os.unlink(status_file)
+      else:
+        log.debug('Request delete token fail for %s...' % request_file)
+    else:
+      log.debug('Bad token. Request add token fail for %s...' % request_file)
+
+def checkService(args, can_bang=True):
+  base_token_path = args['token_base_path']
+  token_dict = loadJsonFile(args['token_json'])
+
+  if not token_dict:
+    return
+
+  db = getDb(args['db'])
+  call_bang = False
+
+  computer_guid = args['computer_id']
+  partition_id = args['partition_id']
+  slap = slapos.slap.slap()
+
+  # Check token status
+  for slave_reference, token in token_dict.iteritems():
+    status_file = os.path.join(base_token_path, '%s.status' % slave_reference)
+    if not os.path.exists(status_file):
+      # This token is not added yet!
+      continue
+
+    msg = readFile(status_file)
+    if msg == 'TOKEN_USED':
+      continue
+
+    # Check if token is not in the database
+    status = False
+    try:
+        token_found, = db.execute("SELECT token FROM token WHERE token = ?",
+            (token,)).next()
+        if token_found == token:
+          status = True
+    except StopIteration:
+        pass
+    if not status:
+      # Token is used to register client
+      call_bang = True
+      try:
+        time.sleep(1)
+        writeFile(status_file, 'TOKEN_USED')
+        log.info("Token status of %s updated to 'used'." % slave_reference)
+      except IOError, e:
+        # XXX- this file should always exists
+        log.debug('Error when writing in file %s. Clould not update status of %s...' %
+                              (status_file, slave_reference))
+
+  if call_bang and can_bang:
+    bang(args)
+
+def manage(args):
+  # Request Add new tokens
+  requestAddToken(args)
+
+  # Request delete removed token
+  requestRemoveToken(args)
+
+  # check status of all token
+  checkService(args)