[slapos.package] Implement upload key, fixes and remove promise folder

  Removal of promise folder is a clean up only. This code should be reworked.

slapos/package/_autoupdate.py

-import os
-import subprocess
-import sys
-
-
-def do_update():
-    print 'Updating slapprepare'
-    subprocess.call(['easy_install', '-U', 'slapprepare'])
-
-
-def main():
-    if '--no-update' in sys.argv:
-        sys.argv.remove('--no-update')
-    else:
-        do_update()
-
-    args = [
-	os.path.join(os.path.dirname(sys.argv[0]), 'slapprepare-raw')
-	 ] + sys.argv[1:]
-
-    subprocess.call(args)
-

slapos/package/promise/__init__.py

-
-import sample
-
-promise_list = (
-  sample.Promise,
-  )

slapos/package/promise/hostname.py

-from slapos.package.base_promise import BasePromise
-import os
-
-class Promise(BasePromise):
-
-  configuration_file_path = '/etc/HOSTNAME' 
-
-  def checkConsistency(self, fixit=0, **kw):
-    is_ok = False
-    computer_id = kw["computer_id"]
-
-    self.log("Setting hostname in : %s" % self.configuration_file_path)
-    if os.path.exists(self.configuration_file_path):
-       is_ok = computer_id in open(self.configuration_file_path, 'r').read()
-
-    if not is_ok and fixit:
-      return self.fixConsistency(**kw)
-
-    return is_ok
-
-  def fixConsistency(self, **kw):
-    """Configures hostname daemon"""
-    computer_id = kw["computer_id"]
-    open(self.configuration_file_path, 'w').write("%s\n" % computer_id)
-    self._call(['hostname', '-F', self.configuration_file_path])
-    return self.checkConsistency(fixit=0, **kw)

slapos/package/promise/hosts.py

-from _util import _call
-
-SLAPOS_MARK = '# Added by SlapOS\n'
-
-class Promise:
-
-  def checkConsistency(self, fixit=0, **kw):
-    is_ok = False
-     
-    if not is_ok and fixit:
-      return self.fixConsistency(**kw)
-
-    return is_ok
-
-  def fixConsistency(self, **kw)
-    """Configures NTP daemon"""
-    server = "server pool.ntp.org"
-    old_ntp = open('/etc/ntp.conf', 'r').readlines()
-    new_ntp = open('/etc/ntp.conf', 'w')
-    for line in old_ntp:
-      if line.startswith('server'):
-        continue
-      new_ntp.write(line)
-    new_ntp.write(SLAPOS_MARK)
-    new_ntp.write(server + '\n')
-    new_ntp.close()
-    _call(['chkconfig', '--add', 'ntp'])
-    _call(['chkconfig', 'ntp', 'on'])
-    _call(['systemctl', 'enable', 'ntp.service'])
-    _call(['systemctl', 'restart', 'ntp.service'])
-    return self.checkConsistency(fixit=0, **kw)

slapos/package/promise/ntp.py

-from slapos.package.base_promise import BasePackagePromise
-
-class Promise(BasePackagePromise):
-  package_name = "ntp"
-  binary_path = '/usr/sbin/ntpd'

slapos/package/promise/ntp_conf.py

-from slapos.package.base_promise import BasePromise
-
-SLAPOS_MARK = '# Added by SlapOS\n'
-
-class Promise(BasePromise):
-
-  configuration_file_path = '/etc/ntp.conf'
-
-  def checkConsistency(self, fixit=0, **kw):
-    is_ok = False
-    server = "server pool.ntp.org"
-    old_ntp = open(self.configuration_file_path, 'r').readlines()
-    for line in old_ntp:
-      if line.startswith('server pool.ntp.org'):
-        continue
-        is_ok = True
-    
-    if not is_ok and fixit:
-      return self.fixConsistency(**kw)
-
-    return is_ok
-
-  def fixConsistency(self, **kw)
-    """Configures NTP daemon"""
-    server = "server pool.ntp.org"
-    old_ntp = open(self.configuration_file_path, 'r').readlines()
-    new_ntp = open(self.configuration_file_path, 'w')
-    for line in old_ntp:
-      if line.startswith('server'):
-        continue
-      new_ntp.write(line)
-    new_ntp.write(SLAPOS_MARK)
-    new_ntp.write(server + '\n')
-    new_ntp.close()
-    self._chkconfig('ntp', 'add')
-    self._chkconfig('ntp', 'on')
-    self._service('ntp.service', 'enable')
-    self._service['ntp.service', 'restart')
-    return self.checkConsistency(fixit=0, **kw)
-

slapos/package/promise/sample.py

-
-
-class Promise:
-  
-  def checkConsistency(self, fixit=0, **kw):
-    print "checkConsistency invoked"
-    if fixit:
-      self.fixConsistency()
-
-  def fixConsistency(self, **kw):
-    print "Fixed Consistency"
-  

slapos/package/promise/template/hosts.in

-#
-# hosts         This file describes a number of hostname-to-address
-#               mappings for the TCP/IP subsystem.  It is mostly
-#               used at boot time, when no name servers are running.
-#               On small systems, this file can be used instead of a
-#               "named" name server.
-# Syntax:
-#    
-# IP-Address  Full-Qualified-Hostname  Short-Hostname
-#
-
-127.0.0.1	%(computer_id)s localhost
-
-# special IPv6 addresses
-::1             %(computer_id)s localhost ipv6-localhost ipv6-loopback
-
-fe00::0         ipv6-localnet
-
-ff00::0         ipv6-mcastprefix
-ff02::1         ipv6-allnodes
-ff02::2         ipv6-allrouters
-ff02::3         ipv6-allhosts
-

slapos/package/signature.py

@@ -30,25 +30,43 @@
 
 import ConfigParser
 import os
+import time
+import traceback
 import tempfile
+import datetime
 
 from slapos.networkcachehelper import helper_download_network_cached_to_file
+from slapos.networkcachehelper import helper_upload_network_cached_from_file
 
 
 class NetworkCache:
-  def __init__(self, configuration):
-    if not os.path.exists(slapos_conf):
+  def __init__(self, configuration_path):
+    if not os.path.exists(configuration_path):
       raise ValueError("You need configuration file")
-    self.configuration = configuration
+    self.configuration = configuration_path
+    self._load()
  
   def _load(self):
-    
     network_cache_info = ConfigParser.RawConfigParser()
     network_cache_info.read(self.configuration)
-    self.download_binary_cache_url = network_cache_info.get('networkcache', 'download-binary-cache-url')
-    self.download_cache_url = network_cache_info.get('networkcache', 'download-cache-url')
-    self.download_binary_dir_url = network_cache_info.get('networkcache', 'download-binary-dir-url')
-    self.signature_certificate_list = network_cache_info.get('networkcache', 'signature-certificate-list')
+    network_cache_info_dict = dict(network_cache_info.items('networkcache'))
+    def get_(name):
+      return network_cache_info_dict.get(name)
+
+    self.download_binary_cache_url = get_('download-binary-cache-url')
+    self.download_cache_url = get_('download-cache-url')
+    self.download_binary_dir_url = get_('download-binary-dir-url')
+    self.signature_certificate_list = get_('signature-certificate-list')
+
+    # Not mandatory
+    self.dir_url = get_('upload-dir-url')
+    self.cache_url = get_('upload-cache-url')
+    self.signature_private_key_file = get_('signature_private_key_file')
+    self.shacache_cert_file = get_('shacache-cert-file')
+    self.shacache_key_file = get_('shacache-key-file')
+    self.shadir_cert_file = get_('shadir-cert-file')
+    self.shadir_key_file = get_('shadir-key-file')
+
 
     if network_cache_info.has_section('slapupdate'):
       self.directory_key = network_cache_info.get('slapupdate', 'upgrade_key')
@@ -60,14 +78,6 @@ class Signature:
   def __init__(self, config, logger=None):
     self.config = config
     self.logger = logger
-    self.current_state_path = config.srv_file
-
-    # Get configuration
-    self.current_state = ConfigParser.RawConfigParser()
-    self.current_state.read(config.srv_file)
-  
-    self.next_state = ConfigParser.RawConfigParser()
-    self.next_state.read(self.download())
 
   def _download(self, path):
     """
@@ -106,7 +116,57 @@ class Signature:
     else:
       raise ValueError("No result from shacache")
 
+  def _upload(self, path):
+    """
+    Creates uploads repository to cache.
+    """
+    shacache = NetworkCache(self.config.slapos_configuration)
+ 
+    metadata_dict = {
+      # XXX: we set date from client side. It can be potentially dangerous
+      # as it can be badly configured.
+      'timestamp': time.time(),
+    }
+    try:
+      if helper_upload_network_cached_from_file(
+        path=path,
+        directory_key=shacache.directory_key,
+        metadata_dict=metadata_dict,
+        # Then we give a lot of not interesting things
+        dir_url=shacache.dir_url,
+        cache_url=shacache.cache_url,
+        signature_private_key_file=shacache.signature_private_key_file,
+        shacache_cert_file=shacache.shacache_cert_file,
+        shacache_key_file=shacache.shacache_key_file,
+        shadir_cert_file=shacache.shadir_cert_file,
+        shadir_key_file=shacache.shadir_key_file,
+      ):
+        print 'Uploaded update file to cache.'
+    except Exception:
+      print 'Unable to upload to cache:\n%s.' % traceback.format_exc()
+
+  def upload(self, dry_run=0):
+    upgrade_info = ConfigParser.RawConfigParser()
+    upgrade_info.read(self.config.upgrade_file)
+
+    tomorrow = datetime.date.today() + datetime.timedelta(days=1)
+
+    if self.config.reboot:
+      upgrade_info.set('system', 'reboot', tomorrow.isoformat())
+    if self.config.upgrade:
+      upgrade_info.set('system', 'upgrade', tomorrow.isoformat())
+
+    file = open(self.config.upgrade_file, "w")
+    upgrade_info.write(file)
+    file.close()
+
+    if dry_run:
+      return
+
+    self._upload(self.config.upgrade_file)
+ 
   def update(self, reboot=None, upgrade=None):
+    self.load()
     if reboot is None and upgrade is None:
       return 
     if not self.current_state.has_section('system'):
@@ -118,7 +178,7 @@ class Signature:
     if upgrade is not None:
       self.current_state.set('system', 'upgrade', upgrade)
   
-    current_state_file = open(self.current_state_path, "w")
+    current_state_file = open(self.config.srv_file, "w")
     self.current_state.write(current_state_file)
     current_state_file.close()
 
@@ -133,6 +193,12 @@ class Signature:
     """
     Extract information from config file and server file
     """
+    self.current_state = ConfigParser.RawConfigParser()
+    self.current_state.read(self.config.srv_file)
+
+    self.next_state = ConfigParser.RawConfigParser()
+    self.next_state.read(self.download())
+
     self.reboot = self._read_state(self.next_state, "upgrade")
     self.upgrade = self._read_state(self.next_state, "upgrade")
     self.last_reboot = self._read_state(self.current_state, "reboot")

slapos/package/test/test_networkcache.py

+from slapos.package import update, signature
+import tempfile
+import unittest
+
+SIGNATURE = "-----BEGIN CERTIFICATE-----\nXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\n-----END CERTIFICATE-----"
+
+UPDATE_CFG_DATA = """
+[slapupdate]
+upgrade_key = slapos-upgrade-testing-key-with-config-file
+
+[networkcache]
+download-binary-cache-url = http://www.shacache.org/shacache
+download-cache-url = https://www.shacache.org/shacache
+download-binary-dir-url = http://www.shacache.org/shadir
+
+signature-certificate-list = -----BEGIN CERTIFICATE-----
+  XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
+  -----END CERTIFICATE-----
+
+
+"""
+
+UPDATE_CFG_WITH_UPLOAD_DATA = UPDATE_CFG_DATA + """
+signature_private_key_file = /etc/opt/slapos/signature.key
+signature_certificate_file = /etc/opt/slapos/signature.cert
+upload-cache-url = https://www.shacache.org/shacache
+shacache-cert-file = /etc/opt/slapos/shacache.crt
+shacache-key-file = /etc/opt/slapos/shacache.key
+upload-dir-url = https://www.shacache.org/shadir
+shadir-cert-file = /etc/opt/slapos/shacache.crt
+shadir-key-file = /etc/opt/slapos/shacache.key
+"""
+
+
+class NetworkCacheTestCase(unittest.TestCase):
+
+  def test_basic(self):
+    info, self.configuration_file_path = tempfile.mkstemp()
+    open(self.configuration_file_path, 'w').write(UPDATE_CFG_DATA)
+    shacache = signature.NetworkCache(self.configuration_file_path)
+    self.assertEqual(shacache.download_binary_cache_url,
+                     "http://www.shacache.org/shacache")
+    self.assertEqual(shacache.download_cache_url,
+                     "https://www.shacache.org/shacache")
+    self.assertEqual(shacache.download_binary_dir_url,
+                     "http://www.shacache.org/shadir")
+
+    self.assertEqual(shacache.signature_certificate_list,
+                     SIGNATURE)            
+
+    self.assertEqual(shacache.directory_key,
+                     'slapos-upgrade-testing-key-with-config-file')
+    # Check keys that don't exist
+    # Not mandatory
+    self.assertEqual(shacache.dir_url , None)
+    self.assertEqual(shacache.cache_url , None)
+    self.assertEqual(shacache.signature_private_key_file , None)
+    self.assertEqual(shacache.shacache_cert_file , None)
+    self.assertEqual(shacache.shacache_key_file , None)
+    self.assertEqual(shacache.shadir_cert_file , None)
+    self.assertEqual(shacache.shadir_key_file , None)
+
+
+  def test_with_upload(self):
+    info, self.configuration_file_path = tempfile.mkstemp()
+    open(self.configuration_file_path, 'w').write(UPDATE_CFG_WITH_UPLOAD_DATA)
+    shacache = signature.NetworkCache(self.configuration_file_path)
+    self.assertEqual(shacache.download_binary_cache_url,
+                     "http://www.shacache.org/shacache")
+    self.assertEqual(shacache.download_cache_url,
+                     "https://www.shacache.org/shacache")
+    self.assertEqual(shacache.download_binary_dir_url,
+                     "http://www.shacache.org/shadir")
+
+    self.assertEqual(shacache.signature_certificate_list,
+                     SIGNATURE)            
+
+    self.assertEqual(shacache.directory_key,
+                     'slapos-upgrade-testing-key-with-config-file')
+    # Check keys that don't exist
+    # Not mandatory
+    self.assertEqual(shacache.dir_url , 'https://www.shacache.org/shadir')
+    self.assertEqual(shacache.cache_url , 'https://www.shacache.org/shacache')
+    self.assertEqual(shacache.shacache_cert_file , '/etc/opt/slapos/shacache.crt')
+    self.assertEqual(shacache.shacache_key_file , '/etc/opt/slapos/shacache.key')
+    self.assertEqual(shacache.shadir_cert_file , '/etc/opt/slapos/shacache.crt')
+    self.assertEqual(shacache.shadir_key_file , '/etc/opt/slapos/shacache.key')
+
+    self.assertEqual(shacache.signature_private_key_file ,'/etc/opt/slapos/signature.key')
+
+  def test_file_dont_exist(self):
+    self.assertRaises(ValueError, signature.NetworkCache, 
+                          "/abc/123")
+
+
+
+
+

slapos/package/test/test_promise_hostname.py

-
-from slapos.package.promise import hostname
-import os
-import unittest
-
-def _fake_call(self, *args, **kw):
-  self.last_call = (args, kw)
-
-class testPromiseHostnameCase(unittest.TestCase):
-
-  def setUp(self):
-    hostname.Promise._call = _fake_call
-
-  def testHostnameCheckConsistency(self):
-    promise = hostname.Promise()
-    promise.configuration_file_path = "/tmp/hostname_for_test"
-
-    self.assertFalse(promise.checkConsistency(computer_id="TESTING"))
-
-  def testHostnameFixConsistency(self):
-    hostname.Promise._call = _fake_call
-    promise = hostname.Promise()
-    promise.hostname_path = "/tmp/hostname_for_test_fix"
-
-    if os.path.exists(promise.hostname_path):
-      os.remove(promise.hostname_path)
-
-    self.assertFalse(promise.checkConsistency(computer_id="TESTING"))
-    self.assertTrue(promise.fixConsistency(computer_id="TESTING"))
-    self.assertEqual(promise.last_call, 
-                     ((['hostname', '-F', '/tmp/hostname_for_test_fix'],), {})
-                    )

slapos/package/test/test_promise_ntp.py

-
-from slapos.package.promise import ntp
-import os
-import unittest
-
-NTP_CONTENT = """
-
-server 0.ubuntu.pool.ntp.org
-
-"""
-
-def _fake_call(self, *args, **kw):
-  self.last_call = (args, kw)
-
-class testPromiseHostnameCase(unittest.TestCase):
-
-  def setUp(self):
-    ntp.Promise._call = _fake_call
-
-  def testHostnameCheckConsistency(self):
-    promise = ntp.Promise()
-
-    self.assertFalse(promise.checkConsistency(computer_id="TESTING"))
-
-  def testHostnameFixConsistency(self):
-    hostname.Promise._call = _fake_call
-    promise = ntp.Promise()
-    promise.hostname_path = "/tmp/hostname_for_test_fix"
-
-    if os.path.exists(promise.hostname_path):
-      os.remove(promise.hostname_path)
-
-    self.assertFalse(promise.checkConsistency(computer_id="TESTING"))
-    self.assertTrue(promise.fixConsistency(computer_id="TESTING"))
-    self.assertEqual(promise.last_call, 
-                     ((['hostname', '-F', '/tmp/hostname_for_test_fix'],), {})
-                    )
-

slapos/package/update.py

@@ -36,6 +36,8 @@ import os
 import subprocess as sub
 import sys
 import tempfile
+from signature import Signature
+
 
 # create console handler and set level to warning
 ch = logging.StreamHandler()
@@ -95,7 +97,6 @@ class Upgrader:
     # add ch to logger
     self.logger.addHandler(ch)
 
-
   def checkConsistency(self, *args, **kw):
     print "CHECK CONSISTENCY %s" % ((args, kw),)
 
@@ -118,21 +119,25 @@ class Upgrader:
   
     # Check if run for first time
     if signature.last_reboot is None:
+      if not self.config.dry_run:
         signature.update(reboot=today, upgrade=today)
   
       # Purge repositories list and add new ones
-      self.checkConsistency()
+      self.checkConsistency(fixit=not self.config.dry_run)
     else:
       if signature.last_upgrade < signature.upgrade:
         # Purge repositories list and add new ones
-  
+        if not self.config.dry_run:
           signature.update(upgrade=today)
-        self.checkConcistency()
+        self.checkConsistency(fixit=not self.config.dry_run)
       else:
         logger.info("Your system is up to date")
   
       if signature.last_reboot < signature.reboot:
+        if not self.config.dry_run:
           signature.update(reboot=today)
+        else:
+          self.logger.debug("Dry run: Rebooting required.")
   
 def main():
   """Update computer and slapos"""
@@ -141,3 +146,7 @@ def main():
   upgrader = Upgrader(Parser(usage=usage).check_args())
   upgrader.run()
   sys.exit()
+
+
+if __name__ == '__main__':
+  main()

upload-update.py → slapos/package/upload_key.py

@@ -2,7 +2,7 @@
 # -*- coding: utf-8 -*-
 ##############################################################################
 #
-# Copyright (c) 2012 Vifib SARL and Contributors.
+# Copyright (c) 2012,2013 Vifib SARL and Contributors.
 # All Rights Reserved.
 #
 # WARNING: This program as such is intended to be used by professional
@@ -31,11 +31,10 @@
 import ConfigParser
 import datetime
 from optparse import OptionParser, Option
-import traceback
 import sys
-import time
 
-from slapos.networkcachehelper import helper_upload_network_cached_from_file
+from update import Config
+from signature import Signature
 
 
 class Parser(OptionParser):
@@ -48,8 +47,11 @@ class Parser(OptionParser):
     """
     OptionParser.__init__(self, usage=usage, version=version,
                       option_list=[
+        Option("--slapos-configuration",
+               default='/etc/opt/slapos/slapos.cfg',
+               help="Configuration File used to upload the key."),
         Option("--upgrade-file",
-               default='/etc/slapos-cache/slapos-upgrade',
+               default='/etc/opt/slapos/slapos-upgrade',
                help="File used as reference to upgrade."),
         Option("-u", "--upgrade",
                default=False,
@@ -82,88 +84,25 @@ def get_yes_no(prompt):
     if answer.upper() in ['N', 'NO']:
         return False
 
-
-def upload_network_cached_from_file(path, networkcache_options):
-  """
-  Creates uploads repository to cache.
-  """
-  print 'Uploading update'
-  # XXX create a file descriptor from string. (simplest way: create tmpfile and write)
-
-  metadata_dict = {
-    # XXX: we set date from client side. It can be potentially dangerous
-    # as it can be badly configured.
-    'timestamp': time.time(),
-  }
-  try:
-    if helper_upload_network_cached_from_file(
-      path=path,
-      directory_key='slapos-upgrade',
-      metadata_dict=metadata_dict,
-      # Then we give a lot of not interesting things
-      dir_url=networkcache_options.get('upload-dir-url'),
-      cache_url=networkcache_options.get('upload-cache-url'),
-      signature_private_key_file=networkcache_options.get(
-        'signature_private_key_file'),
-      shacache_cert_file=networkcache_options.get('shacache-cert-file'),
-      shacache_key_file=networkcache_options.get('shacache-key-file'),
-      shadir_cert_file=networkcache_options.get('shadir-cert-file'),
-      shadir_key_file=networkcache_options.get('shadir-key-file'),
-    ):
-      print 'Uploaded update file to cache.'
-  except Exception:
-    print 'Unable to upload to cache:\n%s.' % traceback.format_exc()
-
-
-def save_current_state(current_state, config):
-  """
-  Will save ConfigParser to config file
-  """
-  file = open(config.upgrade_file, "w")
-  current_state.write(file)
-  file.close()
-
-
-# Class containing all parameters needed for configuration
-class Config:
-  def setConfig(self, option_dict):
-    """
-    Set options given by parameters.
-    """
-    # Set options parameters
-    for option, value in option_dict.__dict__.items():
-      setattr(self, option, value)
-
-    self.today = datetime.date.today()
-    self.tomorrow = self.today + datetime.timedelta(days=1)
-
-
 def new_upgrade(config):
-  upgrade_info = ConfigParser.RawConfigParser()
-  upgrade_info.read(config.upgrade_file)
-  if config.reboot:
-    upgrade_info.set('system', 'reboot', config.tomorrow)
-  if config.upgrade:
-    upgrade_info.set('system', 'upgrade', config.tomorrow)
-  save_current_state(upgrade_info, config)
+  signature = Signature(config)
+  signature.upload(dry_run=1) 
+
   print " You will update this :"
   print open(config.upgrade_file).read()
   if not get_yes_no("Do you want to continue? "):
     sys.exit(0)
 
-  networkcache_info = ConfigParser.RawConfigParser()
-  networkcache_info.read('/etc/slapos-cache/slapos.cfg')
-  networkcache_info_dict = dict(networkcache_info.items('networkcache'))
   if not config.dry_run:
-    upload_network_cached_from_file(config.upgrade_file, networkcache_info_dict)
-
+    print "Uploading..."
+    signature.upload()
 
 def main():
   """Upload file to update computer and slapos"""
   usage = "usage: [options] "
   # Parse arguments
-  config = Config()
-  config.setConfig(Parser(usage=usage).check_args())
+  config = Config(Parser(usage=usage).check_args())
+  config.srv_file = "/srv/slapupdate"
   new_upgrade(config)
   sys.exit()