slapos: add support for Python 3

/reviewed-on !610

component/kumo/buildout.cfg

@@ -14,7 +14,6 @@ recipe = slapos.recipe.cmmi
 shared = true
 url = https://github.com/downloads/etolabo/kumofs/kumofs-0.4.13.tar.gz
 md5sum = 46148e9536222d0ad2ef36777c55714d
-pre-configure-hook = ${:_profile_base_location_}/kumo-hooks.py#958a595a02de75624728f8d65e39d800:pre_configure_hook
 patches =
   ${:_profile_base_location_}/kumofs-0.4.13_ipv6support_multiiplistenfix.patch#53af9f1f1375940841c589a6cbe11425
   ${:_profile_base_location_}/kumofs-0.4.13_fix_gcc-4.9_ftbfs.patch#c09e04c620ce11c3fdd4afc3459cd355

component/kumo/kumo-hooks.py

-import os
-import sys
-import traceback
-from shutil import copy
-from subprocess import Popen, PIPE
-
-CONFIGURE_PATH = os.path.join('configure')
-CONFIGURE_BACKUP_PATH = CONFIGURE_PATH + '_disabled'
-# Fake configure, generating a fake Makefile which will create a marker file
-# instead of actually installing anything.
-# This is needed (?) to fetch --prefix from configure parameters, so we know
-# where to tell Makefile to put the dummy file.
-FAKE_CONFIGURE = '''#!%(python)s -S
-import os
-import sys
-print 'Configuration is disabled on this host because %%s'
-print 'Original configure file available at %(backup)s'
-prefix = None
-next = False
-for arg in sys.argv:
-    if next:
-        prefix = arg
-        break
-    if arg.startswith('--prefix'):
-        if arg.startswith('--prefix='):
-            _, prefix = arg.split('=', 1)
-            break
-        next = True
-if prefix is None:
-    raise '--prefix parameter not found'
-# Generate Makefile with proper prefix
-open('Makefile', 'w').write("""all:
-\techo 'make disabled, see configure'
-
-install:
-\ttouch %%%%s""" %%%% (
-  os.path.join(prefix, 'BUILD_DISABLED_BY_BUILDOUT'),
-))
-sys.exit(0)
-''' % {
-    'backup': CONFIGURE_BACKUP_PATH,
-    'python': sys.executable,
-}
-
-def pre_configure_hook(options, buildout):
-    gcc_executable = os.getenv('CC', 'gcc')
-    try:
-        gcc = Popen([gcc_executable, '-v'], stdout=PIPE, stderr=PIPE,
-            close_fds=True)
-    except OSError, (errno, _):
-        if errno == 2:
-            # No gcc installed, nothing to check
-            pass
-        else:
-            print 'Unexpected failure trying to detect gcc version'
-            traceback.print_exc()
-    else:
-        gcc.wait()
-        # Considered innocent until proven guilty.
-        error = None
-        for line in '\n'.join((gcc.stdout.read(), gcc.stderr.read())).splitlines():
-            if line.startswith('gcc version'):
-                if '4.1.1' in line and 'prerelease' in line:
-                    # There is a bug in 4.1.1 prerelease (ie, as of mandriva
-                    # 2007.0) g++ preventing kumo compilation from succeeding.
-                    error = 'broken GCC version: %s' % (line, )
-                break
-        else:
-            print >>sys.stderr, 'GCC version could not be detected, ' \
-                'building anyway'
-        if error is not None:
-            print 'Disabling build, with reason:', error
-            # Copy to preserver permission
-            copy(CONFIGURE_PATH, CONFIGURE_BACKUP_PATH)
-            open(CONFIGURE_PATH, 'w').write(FAKE_CONFIGURE % (error, ))
-

component/perl/buildout.cfg

@@ -93,7 +93,7 @@ make-binary=
   ${:extra-env} PERL5LIB="${:inc}:${:install-inc}:${:perl-PERL5LIB}" make
 
 # this post-make-hook is same for all users of the macro.
-post-make-hook = ${:_profile_base_location_}/../../component/perl/perl-CPAN-package-create-wrapper.py#d012f7ba3300b14b2c6b173351d410be:post_make_hook
+post-make-hook = ${:_profile_base_location_}/../../component/perl/perl-CPAN-package-create-wrapper.py#f28c45a0f473ae050ca3ebaed5c39a4e:post_make_hook
 
 perl_location = ${perl:location}
 

component/perl/perl-CPAN-package-create-wrapper.py

@@ -17,7 +17,7 @@ def post_make_hook(options, buildout, environmet):
 export PERL5LIB="{site_perl}:$PERL5LIB"
 exec {perl_location}/bin/perl "$@"
 '''.format(**locals()))
-  os.chmod(perl_wrapper_path, 0755)
+  os.chmod(perl_wrapper_path, 0o755)
 
   # create a wrapper for each scripts installed in perl-bin
   for script_path in glob.glob(os.path.join(prefix, 'perl-bin', '*')):
@@ -28,4 +28,4 @@ exec {perl_location}/bin/perl "$@"
 export PERL5LIB="{site_perl}:$PERL5LIB"
 exec {perl_location}/bin/perl {script_path} "$@"
 '''.format(**locals()))
-      os.chmod(wrapper_path, 0755)
+      os.chmod(wrapper_path, 0o755)

component/phantomjs/buildout.cfg

@@ -2,7 +2,6 @@
 extends =
   ../fontconfig/buildout.cfg
   ../libexpat/buildout.cfg
-  ../dash/buildout.cfg
 parts =
   phantomjs
 
@@ -19,17 +18,19 @@ x86 = https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-1.9.7-linux-i686
 x86-64 = https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-1.9.7-linux-x86_64.tar.bz2 f278996c3edd0e8d8ec4893807f27d71
 
 script =
-  if not self.options.get('url'): self.options['url'], self.options['md5sum'] = self.options[guessPlatform()].split(' ')
-  extract_dir = self.extract(self.download(self.options['url'], self.options.get('md5sum')))
+  if not self.options.get('url'):
+    self.options['url'], self.options['md5sum'] = \
+      self.options[guessPlatform()].split(' ')
+  extract_dir = self.extract(self.download(self.options['url'],
+                             self.options.get('md5sum')))
   workdir = guessworkdir(extract_dir)
   self.copyTree(workdir, "%(location)s")
   wrapper_location = os.path.join("%(location)s", "phantomjs-slapos")
-  wrapper = open(wrapper_location, 'w')
-  wrapper.write("""#!${dash:location}/bin/dash
+  with open(wrapper_location, 'w') as wrapper:
+    wrapper.write("""#!/bin/sh
   cd %(location)s
   export LD_LIBRARY_PATH=%(location)s:${freetype:location}/lib/:${fontconfig:location}/lib/:${libexpat:location}/lib
   export PATH=${fontconfig:location}/bin:$PATH
-  exec %(location)s/bin/phantomjs $*""")
-  wrapper.flush()
-  wrapper.close()
-  os.chmod(wrapper_location, 0755)
+  exec %(location)s/bin/phantomjs "$@"
+  """)
+  os.chmod(wrapper_location, 0o755)

slapos/recipe/apachephpconfigure/__init__.py

@@ -24,6 +24,8 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
+from __future__ import print_function
+
 from slapos.recipe.librecipe import GenericBaseRecipe
 import zc.buildout
 import sys
@@ -101,17 +103,17 @@ class Recipe(GenericBaseRecipe):
 
     # TODO factor
     if delete != []:
-      print "Creating lampconfigure with 'delete' arguments"
+      print("Creating lampconfigure with 'delete' arguments")
       command = argument + delete
     if rename != []:
       for parameters in rename:
-        print "Creating lampconfigure with 'rename' arguments"
+        print("Creating lampconfigure with 'rename' arguments")
         command = argument + rename
     if chmod != []:
-      print "Creating lampconfigure with 'chmod' arguments"
+      print("Creating lampconfigure with 'chmod' arguments")
       command = argument + chmod
     if data != []:
-      print "Creating lampconfigure with 'run' arguments"
+      print("Creating lampconfigure with 'run' arguments")
       command = argument + data
 
 

slapos/recipe/bonjourgrid/__init__.py

@@ -94,7 +94,7 @@ class Recipe(GenericBaseRecipe):
       dict(ip_address=self.options['ipv6'].strip(),
             project=project,
             middleware=type)))
-    os.chmod(config_info_file, 0744)
+    os.chmod(config_info_file, 0o744)
     path_list.append(config_info)
 
   update = install

slapos/recipe/certificate_authority/__init__.py

@@ -26,11 +26,11 @@
 ##############################################################################
 import os
 import hashlib
-import ConfigParser
+from six.moves import configparser
 import tempfile
 
 from slapos.recipe.librecipe import GenericBaseRecipe
-from certificate_authority import popenCommunicate
+from .certificate_authority import popenCommunicate
 
 class Recipe(GenericBaseRecipe):
 
@@ -119,7 +119,7 @@ class Request(Recipe):
       open(certificate, 'w').write(cert_content)
       request_needed = False
     else:
-      parser = ConfigParser.RawConfigParser()
+      parser = configparser.RawConfigParser()
       parser.add_section('certificate')
       parser.set('certificate', 'name', name)
       parser.set('certificate', 'key_file', key)

slapos/recipe/certificate_authority/certificate_authority.py

+from __future__ import print_function
+
 import os
 import subprocess
 import time
-import ConfigParser
+from six.moves import configparser
 import uuid
 
 
@@ -95,12 +97,12 @@ class CertificateAuthority:
 
   def checkRequestDir(self):
     for request_file in os.listdir(self.request_dir):
-      parser = ConfigParser.RawConfigParser()
+      parser = configparser.RawConfigParser()
       parser.readfp(open(os.path.join(self.request_dir, request_file), 'r'))
       if self._checkCertificate(parser.get('certificate', 'name'),
           parser.get('certificate', 'key_file'), parser.get('certificate',
             'certificate_file')):
-        print 'Created certificate %r' % parser.get('certificate', 'name')
+        print('Created certificate %r' % parser.get('certificate', 'name'))
 
 def runCertificateAuthority(*args):
   ca = CertificateAuthority(*args)

slapos/recipe/condor/__init__.py

@@ -29,7 +29,6 @@ import os
 import subprocess
 import zc.buildout
 import filecmp
-import urlparse
 import shutil
 import re
 import json
@@ -130,9 +129,9 @@ class Recipe(GenericBaseRecipe):
 
     #create condor binary launcher for slapos
     if not os.path.exists(self.wrapper_bin):
-      os.makedirs(self.wrapper_bin, int('0744', 8))
+      os.makedirs(self.wrapper_bin, int('0o744', 8))
     if not os.path.exists(self.wrapper_sbin):
-      os.makedirs(self.wrapper_sbin, int('0744', 8))
+      os.makedirs(self.wrapper_sbin, int('0o744', 8))
     #generate script for each file in prefix/bin
     for binary in os.listdir(self.prefix+'/bin'):
       wrapper_location = os.path.join(self.wrapper_bin, binary)
@@ -153,7 +152,7 @@ class Recipe(GenericBaseRecipe):
       wrapper.write(content)
       wrapper.close()
       path_list.append(wrapper_location)
-      os.chmod(wrapper_location, 0744)
+      os.chmod(wrapper_location, 0o744)
 
     #generate script for each file in prefix/sbin
     for binary in os.listdir(self.prefix+'/sbin'):
@@ -175,7 +174,7 @@ class Recipe(GenericBaseRecipe):
       wrapper.write(content)
       wrapper.close()
       path_list.append(wrapper_location)
-      os.chmod(wrapper_location, 0744)
+      os.chmod(wrapper_location, 0o744)
 
     #generate script for start condor
     wrapper = self.createPythonScript(
@@ -228,7 +227,7 @@ class AppSubmit(GenericBaseRecipe):
         for file in file_list:
           if file and (file.startswith('http') or file.startswith('ftp')):
             file_list[file] = self.download(file_list[file])
-          os.chmod(file_list[file], 0600)
+          os.chmod(file_list[file], 0o600)
       else:
         app_list[app]['files'] = {}
 
@@ -236,11 +235,11 @@ class AppSubmit(GenericBaseRecipe):
       if executable and (executable.startswith('http') or executable.startswith('ftp')):
         app_list[app]['executable'] = self.download(executable,
                                       app_list[app]['executable-name'])
-        os.chmod(app_list[app]['executable-name'], 0700)
+        os.chmod(app_list[app]['executable-name'], 0o700)
       submit_file = app_list[app].get('description-file', '')
       if submit_file and (submit_file.startswith('http') or submit_file.startswith('ftp')):
         app_list[app]['description-file'] = self.download(submit_file, 'submit')
-        os.chmod(app_list[app]['description-file'], 0600)
+        os.chmod(app_list[app]['description-file'], 0o600)
 
     return app_list
 

slapos/recipe/davstorage/__init__.py

@@ -25,7 +25,7 @@
 #
 ##############################################################################
 import subprocess
-import httplib
+from six.moves import http_client as httplib
 import base64
 import os
 import shutil

slapos/recipe/dcron.py

@@ -29,6 +29,8 @@ import os
 from slapos.recipe.librecipe import GenericBaseRecipe
 from zc.buildout import UserError
 
+from six.moves import map
+
 class Recipe(GenericBaseRecipe):
 
   def install(self):
@@ -124,7 +126,7 @@ def systemd_to_cron(spec):
     x = spec[i]
     if x != '*':
       for x in x.split(','):
-        x = map(int, x.split('/', 1))
+        x = list(map(int, x.split('/', 1)))
         a = x[0] - y
         if 0 <= a < z:
           if len(x) == 1:

slapos/recipe/erp5_promise/__init__.py

@@ -26,7 +26,7 @@
 ##############################################################################
 
 from slapos.recipe.librecipe import GenericBaseRecipe
-import ConfigParser
+from six.moves import configparser
 
 class Recipe(GenericBaseRecipe):
   """
@@ -34,7 +34,7 @@ class Recipe(GenericBaseRecipe):
   """
 
   def install(self):
-    promise_parser = ConfigParser.RawConfigParser()
+    promise_parser = configparser.RawConfigParser()
     for section_name, option_id_list in (
           ('portal_templates', (
             ('repository', 'bt5-repository-url'),

slapos/recipe/erp5testnode/__init__.py

@@ -24,10 +24,10 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
-import ConfigParser
+from six.moves import configparser
+import io
 import json
 import os
-import StringIO
 
 from slapos.recipe.librecipe import GenericBaseRecipe
 
@@ -40,13 +40,13 @@ class Recipe(GenericBaseRecipe):
     CONFIG['PATH'] = os.environ['PATH']
 
     if self.options['instance-dict']:
-      config_instance_dict = ConfigParser.ConfigParser()
+      config_instance_dict = configparser.ConfigParser()
       config_instance_dict.add_section('instance_dict')
       instance_dict = json.loads(self.options['instance-dict'])
 
       for k ,v in instance_dict.iteritems():
         config_instance_dict.set('instance_dict', k, v)
-      value = StringIO.StringIO()
+      value = io.StringIO()
       config_instance_dict.write(value)
       CONFIG['instance_dict'] = value.getvalue()
 

slapos/recipe/free_port.py

@@ -25,7 +25,7 @@
 #
 ##############################################################################
 
-import ConfigParser
+from six.moves import configparser
 import os
 import netaddr
 import socket
@@ -48,7 +48,7 @@ class Recipe(object):
     # If this check isn't done, a new port would be picked for every upgrade
     # of the software release
     try:
-      parser = ConfigParser.RawConfigParser()
+      parser = configparser.RawConfigParser()
       if os.path.exists(buildout['buildout']['installed']):
         with open(buildout['buildout']['installed']) as config_file:
           parser.readfp(config_file)
@@ -59,7 +59,7 @@ class Recipe(object):
         if port != '0':
           self.options['port'] = port
           return
-    except (IOError, ConfigParser.NoSectionError, ConfigParser.NoOptionError):
+    except (IOError, configparser.NoSectionError, configparser.NoOptionError):
       pass
 
     # Otherwise, let's find one

slapos/recipe/generic_cloudooo/__init__.py

@@ -24,9 +24,11 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
+from functools import cmp_to_key
 import zc.buildout
 from slapos.recipe.librecipe import GenericBaseRecipe
 
+@cmp_to_key
 def compareMimetypeEntryPair(a, b):
   """
     Like comparing strings, but here the star `*` is stronger than any other
@@ -115,7 +117,7 @@ class Recipe(GenericBaseRecipe):
       if l and not l.isspace()
     ]
     mimetype_entry_list.extend(default_mimetype_entry_list)
-    mimetype_entry_list.sort(compareMimetypeEntryPair)
+    mimetype_entry_list.sort(key=compareMimetypeEntryPair)
     conversion_server_dict['MIMETYPE_ENTRY_LIST'] = \
       "\n".join(["  " + l for l in mimetype_entry_list])
     config_file = self.createFile(self.options['configuration-file'],

slapos/recipe/lamp/__init__.py

@@ -31,7 +31,7 @@ import pkg_resources
 import zc.buildout
 import sys
 import zc.recipe.egg
-import urlparse
+from six.moves.urllib.parse import urlparse
 
 # Warning : this recipe is deprecated and has been replaced by apachephp.
 
@@ -264,7 +264,7 @@ class Request(BaseRecipe):
     mysql = self.request(self.options['mariadb-software-url'],
       software_type, 'MariaDB Server', partition_parameter_kw=parameters
     ).getConnectionParameter('url')
-    mysql_parsed = urlparse.urlparse(mysql)
+    mysql_parsed = urlparse(mysql)
 
     mysql_host, mysql_port = mysql_parsed.hostname, mysql_parsed.port
     if mysql_parsed.scheme == 'mysqls': # Listen over stunnel

slapos/recipe/libcloud/__init__.py

@@ -57,7 +57,7 @@ class Recipe(BaseSlapRecipe):
       try:
         self.slave_partition_configuration_dict_list.append(
             self._installSlavePartition(slave_partition))
-      except SlavePartitionError, e:
+      except SlavePartitionError as e:
         self.logger.warning('Slave Parttion %r not installed, issue: %r'%(
           slave_partition.getId(), e))
     # Installs wrappers

slapos/recipe/librecipe/__init__.py

@@ -34,13 +34,13 @@ import stat
 import netaddr
 import time
 import re
-import urlparse
+from six.moves.urllib.parse import urlunparse
 import json
 
 # Use to do from slapos.recipe.librecipe import GenericBaseRecipe
-from generic import GenericBaseRecipe
-from genericslap import GenericSlapRecipe
-from filehash import filehash
+from .generic import GenericBaseRecipe
+from .genericslap import GenericSlapRecipe
+from .filehash import filehash
 
 # Utility functions to (de)serialise live python objects in order to send them
 # to master.
@@ -324,7 +324,7 @@ class BaseSlapRecipe:
     if port is not None:
       netloc += ':%s' % port
 
-    url = urlparse.urlunparse((scheme, netloc, path, params, query, fragment))
+    url = urlunparse((scheme, netloc, path, params, query, fragment))
 
     return url
 

slapos/recipe/librecipe/execute.py

+from __future__ import print_function
+
 import sys
 import os
 import signal
@@ -5,6 +7,8 @@ import subprocess
 from collections import defaultdict
 from inotify_simple import INotify, flags
 
+import six
+
 def _wait_files_creation(file_list):
   # Establish a list of directory and subfiles.
   # and test existence before watching, so that we don't miss an event.
@@ -14,7 +18,7 @@ def _wait_files_creation(file_list):
     directories[dirname][filename] = os.path.lexists(f)
 
   def all_files_exists():
-    return all(all(files.itervalues()) for files in directories.itervalues())
+    return all(all(six.itervalues(files)) for files in six.itervalues(directories))
 
   with INotify() as inotify:
     watchdescriptors = {inotify.add_watch(dirname,
@@ -101,7 +105,7 @@ def generic_exec(args, extra_environ=None, wait_list=None,
 child_pg = None
 
 def sig_handler(sig, frame):
-  print 'Received signal %r, killing children and exiting' % sig
+  print('Received signal %r, killing children and exiting' % sig)
   if child_pg is not None:
     os.killpg(child_pg, signal.SIGHUP)
     os.killpg(child_pg, signal.SIGTERM)
@@ -116,7 +120,7 @@ def execute_with_signal_translation(args):
   child = subprocess.Popen(args, close_fds=True, preexec_fn=os.setsid)
   child_pg = child.pid
   try:
-    print 'Process %r started' % (args, )
+    print('Process %r started' % (args, ))
     signal.pause()
   finally:
     os.killpg(child_pg, signal.SIGHUP)

slapos/recipe/librecipe/filehash.py

@@ -24,6 +24,7 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
+from __future__ import print_function
 import hashlib
 import shutil
 import os
@@ -91,6 +92,6 @@ if __name__ == '__main__':
   if len(sys.argv) == 1:
     raise ValueError("Not enough command line arguments")
   if len(sys.argv) == 2:
-    print sys.argv[1], '-', pathhash(sys.argv[1])
+    print(sys.argv[1], '-', pathhash(sys.argv[1]))
   else:
-    print sys.argv[2], '-', pathhash(sys.argv[2], sys.argv[1])
+    print(sys.argv[2], '-', pathhash(sys.argv[2], sys.argv[1]))

slapos/recipe/librecipe/generic.py

@@ -35,8 +35,12 @@ import inspect
 import re
 import shutil
 import stat
-import urllib
-import urlparse
+from six.moves.urllib.parse import quote
+import itertools
+import six
+from six.moves import map
+from six.moves.urllib.parse import urlunparse
+
 
 import pkg_resources
 import zc.buildout
@@ -90,7 +94,7 @@ class GenericBaseRecipe(object):
     """Options Hook method. This method can be overriden in child classes"""
     return
 
-  def createFile(self, name, content, mode=0600):
+  def createFile(self, name, content, mode=0o600):
     """Create a file with content
 
     The parent directory should exists, else it would raise IOError"""
@@ -117,7 +121,7 @@ class GenericBaseRecipe(object):
       f.write(content)
     return os.path.abspath(name)
 
-  def createExecutable(self, name, content, mode=0700):
+  def createExecutable(self, name, content, mode=0o700):
     return self.createFile(name, content, mode)
 
   def addLineToFile(self, filepath, line, encoding='utf8'):
@@ -148,9 +152,9 @@ class GenericBaseRecipe(object):
       module, function = function
     path, filename = os.path.split(os.path.abspath(name))
 
-    assert not isinstance(args, (basestring, dict)), args
-    args = map(repr, args)
-    args += map('%s=%r'.__mod__, kw.iteritems())
+    assert not isinstance(args, (six.string_types, dict)), args
+    args = itertools.chain(map(repr, args),
+                           map('%s=%r'.__mod__, six.iteritems(kw)))
 
     return zc.buildout.easy_install.scripts(
       [(filename, module, function)], self._ws, sys.executable,
@@ -173,12 +177,12 @@ class GenericBaseRecipe(object):
     lines = ['#!/bin/sh']
 
     if env:
-      for k, v in sorted(env.iteritems()):
+      for k, v in sorted(six.iteritems(env)):
         lines.append('export %s=%s' % (k, shlex.quote(v)))
 
     lines.append('exec')
 
-    args = map(shlex.quote, args)
+    args = list(map(shlex.quote, args))
     args.append('"$@"')
     for arg in args:
       if len(lines[-1]) < 40:
@@ -188,9 +192,9 @@ class GenericBaseRecipe(object):
         lines.append('\t' + arg)
 
     lines.append('')
-    return self.createFile(path, '\n'.join(lines), 0700)
+    return self.createFile(path, '\n'.join(lines), 0o700)
 
-  def createDirectory(self, parent, name, mode=0700):
+  def createDirectory(self, parent, name, mode=0o700):
     path = os.path.join(parent, name)
     if not os.path.exists(path):
       os.mkdir(path, mode)
@@ -240,9 +244,9 @@ class GenericBaseRecipe(object):
     netloc = ''
     if auth is not None:
       auth = tuple(auth)
-      netloc = urllib.quote(str(auth[0])) # Login
+      netloc = quote(str(auth[0])) # Login
       if len(auth) > 1:
-        netloc += ':%s' % urllib.quote(auth[1]) # Password
+        netloc += ':%s' % quote(auth[1]) # Password
       netloc += '@'
 
     # host is an ipv6 address whithout brackets
@@ -254,7 +258,7 @@ class GenericBaseRecipe(object):
     if port is not None:
       netloc += ':%s' % port
 
-    url = urlparse.urlunparse((scheme, netloc, path, params, query, fragment))
+    url = urlunparse((scheme, netloc, path, params, query, fragment))
 
     return url
 

slapos/recipe/librecipe/genericslap.py

@@ -27,7 +27,7 @@
 from slapos import slap
 import time
 
-from generic import GenericBaseRecipe
+from .generic import GenericBaseRecipe
 
 CONNECTION_CACHE = {}
 

slapos/recipe/mkdirectory.py

@@ -27,6 +27,7 @@
 import os
 
 from slapos.recipe.librecipe import GenericBaseRecipe
+import six
 
 class Recipe(GenericBaseRecipe):
 
@@ -36,7 +37,7 @@ class Recipe(GenericBaseRecipe):
     self.mode = int(self.directory.pop('mode', '0777'), 8)
 
   def install(self):
-    for path in sorted(self.directory.itervalues()):
+    for path in sorted(six.itervalues(self.directory)):
       if path and not os.path.isdir(path):
         os.makedirs(path, self.mode)
     # WARNING: This recipe is currently used to create directories that will

slapos/recipe/nosqltestbed/__init__.py

@@ -25,6 +25,8 @@
 #
 ##############################################################################
 
+from __future__ import print_function
+
 import sys
 import pkg_resources
 from logging import Formatter
@@ -41,7 +43,7 @@ class NoSQLTestBed(BaseSlapRecipe):
 
       testbed = plugin_class()
     except:
-      print Formatter().formatException(sys.exc_info())
+      print(Formatter().formatException(sys.exc_info()))
       return None
 
     software_type = self.parameter_dict.get('slap_software_type', 'default')

slapos/recipe/pbs.py

@@ -25,12 +25,14 @@
 #
 ##############################################################################
 
+from __future__ import print_function
+
 import json
 import os
 import subprocess
 import sys
 import textwrap
-import urlparse
+from six.moves.urllib.parse import urlparse
 
 from slapos.recipe.librecipe import GenericSlapRecipe
 from slapos.recipe.dropbear import KnownHostsFile
@@ -208,7 +210,7 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
       # This behavior has been removed to accelerate deployment of the
       # Software Release. The buildout, instead of failing, can process
       # other sections, which will return parameters to the main instance faster
-    parsed_url = urlparse.urlparse(url)
+    parsed_url = urlparse(url)
 
     slave_type = entry['type']
     if not slave_type in ['pull', 'push']:
@@ -216,7 +218,7 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
 
     slave_id = entry['notification-id']
 
-    print 'Processing PBS slave %s with type %s' % (slave_id, slave_type)
+    print('Processing PBS slave %s with type %s' % (slave_id, slave_type))
 
     path_list.append(self.createPythonScript(
       os.path.join(self.options['promises-directory'], "ssh-to-%s" % slave_id),

slapos/recipe/postgres/__init__.py

@@ -25,7 +25,7 @@
 #
 ##############################################################################
 
-import md5
+import hashlib
 import os
 import subprocess
 import textwrap
@@ -195,7 +195,7 @@ class Recipe(GenericBaseRecipe):
         password = self.options['password']
 
         # encrypt the password to avoid storing in the logs
-        enc_password = 'md5' + md5.md5(password+user).hexdigest()
+        enc_password = 'md5' + hashlib.md5(password+user).hexdigest()
 
         self.runPostgresCommand(cmd="""ALTER USER "%s" ENCRYPTED PASSWORD '%s'""" % (user, enc_password))
 

slapos/recipe/publish.py

@@ -24,9 +24,11 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
+from __future__ import print_function
 import zc.buildout
 from slapos.recipe.librecipe import wrap
 from slapos.recipe.librecipe import GenericSlapRecipe
+import six
 
 CONNECTION_PARAMETER_STRING = 'connection-'
 
@@ -77,9 +79,9 @@ class PublishSection(GenericSlapRecipe):
     for section in self.options['section-list'].strip().split():
       section = section.strip()
       options = self.buildout[section].copy()
-      for k, v in options.iteritems():
+      for k, v in six.iteritems(options):
         if k.startswith(CONNECTION_PARAMETER_STRING):
-          print k, v
+          print(k, v)
           publish_dict[k.lstrip(CONNECTION_PARAMETER_STRING)] = v
     self.setConnectionDict(publish_dict)
     return []

slapos/recipe/publish_early.py

@@ -27,6 +27,7 @@
 
 from collections import defaultdict
 from .librecipe import unwrap, wrap, GenericSlapRecipe
+import six
 
 def volatileOptions(options, volatile):
   def copy():
@@ -109,9 +110,9 @@ class Recipe(GenericSlapRecipe):
       publish = False
       publish_dict = {}
       try:
-        for init_section, init in init.iteritems():
+        for init_section, init in six.iteritems(init):
           override = {}
-          for k, v in init.iteritems():
+          for k, v in six.iteritems(init):
             try:
               override[v] = published_dict[k]
             except KeyError:
@@ -120,7 +121,7 @@ class Recipe(GenericSlapRecipe):
           init_section = buildout[init_section]
           assert buildout.Options is Options
           new = {}
-          for k, v in init.iteritems():
+          for k, v in six.iteritems(init):
             try:
               publish_dict[k] = new[v] = init_section.pop(v)
             except KeyError:

slapos/recipe/random.py

@@ -172,7 +172,7 @@ class Password(object):
           raise
 
       fd = os.open(self.storage_path,
-        os.O_CREAT | os.O_EXCL | os.O_WRONLY | os.O_TRUNC, 0600)
+        os.O_CREAT | os.O_EXCL | os.O_WRONLY | os.O_TRUNC, 0o600)
       try:
         os.write(fd, self.passwd)
       finally:

slapos/recipe/re6stnet/__init__.py

@@ -123,7 +123,7 @@ class Recipe(GenericBaseRecipe):
         token_dict[reference] = new_token
         to_add_dict[reference] = new_token
 
-    for reference in token_dict.keys():
+    for reference in list(token_dict):
       if not reference in reference_list:
         # This slave instance is destroyed ?
         to_remove_dict[reference] = token_dict.pop(reference)

slapos/recipe/redis/__init__.py

@@ -81,5 +81,5 @@ def promise(host, port, unixsocket):
     r = Redis(host=host, port=port, unix_socket_path=unixsocket, db=0)
     r.publish("Promise-Service","SlapOS Promise")
     r.connection_pool.disconnect()
-  except Exception, e:
+  except Exception as e:
     sys.exit(e)

slapos/recipe/request.py

@@ -33,6 +33,8 @@ from slapos.slap import SoftwareProductCollection
 import slapos.recipe.librecipe.generic as librecipe
 import traceback
 
+import six
+
 SOFTWARE_PRODUCT_NAMESPACE = "product."
 DEFAULT_SOFTWARE_TYPE = 'RootSoftwareInstance'
 
@@ -110,10 +112,10 @@ class Recipe(object):
       raise UserError("'config' & 'sla' options are obsolete."
                       " Clean up your software release.")
     filter_kw = {k[4:]: v
-      for k, v in options.iteritems()
+      for k, v in six.iteritems(options)
       if k.startswith('sla-') and v}
     partition_parameter_kw = self._filterForStorage({k[7:]: v
-      for k, v in options.iteritems()
+      for k, v in six.iteritems(options)
       if k.startswith('config-')})
     slave = options.get('slave', 'false').lower() in \
       librecipe.GenericBaseRecipe.TRUE_VALUES
@@ -196,7 +198,7 @@ class Recipe(object):
       except KeyError:
         if self.failed is None:
           self.failed = param
-      if isinstance(value, unicode):
+      if six.PY2 and isinstance(value, unicode):
         value = value.encode('UTF-8')
       options['connection-%s' % param] = value
 
@@ -310,12 +312,12 @@ class RequestEdge(Recipe):
 
       self.request_dict[country] = Recipe(buildout, name, local_options)
       # "Bubble" all connection parameters
-      for option, value in local_options.iteritems():
+      for option, value in six.iteritems(local_options):
         if option.startswith(CONNECTION_PARAMETER_STRING):
           self.options['%s-%s' % (option, country)] = value
 
   def install(self):
-    for country, request in self.request_dict.iteritems():
+    for country, request in six.iteritems(self.request_dict):
       request.install()
     return []
 

slapos/recipe/sheepdogtestbed/__init__.py

@@ -26,8 +26,6 @@
 ##############################################################################
 
 import os
-import urllib
-import urllib2
 import pkg_resources
 from slapos.recipe.librecipe import BaseSlapRecipe
 

slapos/recipe/slapconfiguration.py

@@ -31,7 +31,8 @@ import os
 
 import slapos.slap
 from slapos.recipe.librecipe import unwrap
-from ConfigParser import RawConfigParser
+import six
+from six.moves.configparser import RawConfigParser
 from netaddr import valid_ipv4, valid_ipv6
 from slapos.util import mkdir_p
 from slapos import format as slapformat
@@ -115,7 +116,7 @@ class Recipe(object):
                                       buildout['buildout']['directory'])
 
       match = self.OPTCRE_match
-      for key, value in parameter_dict.iteritems():
+      for key, value in six.iteritems(parameter_dict):
           if match(key) is not None:
               continue
           options['configuration.' + key] = value
@@ -157,11 +158,10 @@ class Recipe(object):
               options[his_key.replace('_', '-')] = value
       # Get Instance and root instance title or return UNKNOWN if not set
       options['instance-title'] = parameter_dict.pop('instance_title',
-                                            'UNKNOWN Instance').encode('UTF-8')
+                                            'UNKNOWN Instance')
       options['root-instance-title'] = parameter_dict.pop('root_instance_title',
-                                            'UNKNOWN').encode('UTF-8')
-      options['instance-guid'] = computer_partition.getInstanceGuid() \
-          .encode('UTF-8')
+                                            'UNKNOWN')
+      options['instance-guid'] = computer_partition.getInstanceGuid()
 
       ipv4_set = set()
       v4_add = ipv4_set.add
@@ -204,9 +204,9 @@ class Recipe(object):
 
       # also export single ip values for those recipes that don't support sets.
       if ipv4_set:
-          options['ipv4-random'] = list(ipv4_set)[0].encode('UTF-8')
+          options['ipv4-random'] = min(ipv4_set)
       if ipv6_set:
-          options['ipv6-random'] = list(ipv6_set)[0].encode('UTF-8')
+          options['ipv6-random'] = min(ipv6_set)
 
       storage_home = options.get('storage-home')
       storage_dict = {}
@@ -240,7 +240,7 @@ class Recipe(object):
           # be very careful with overriding master's information
           for key, value in flatten_dict(partition_params).items():
             if key not in options:
-              if isinstance(value, unicode):
+              if six.PY2 and isinstance(value, unicode):
                 value = value.encode('UTF-8')
               options[key] = value
       # print out augmented options to see what we are passing
@@ -265,8 +265,11 @@ class JsonDump(Recipe):
   def __init__(self, buildout, name, options):
     parameter_dict = self.fetch_parameter_dict(options)
     self._json_output = options['json-output']
-    with os.fdopen(os.open(self._json_output, os.O_WRONLY | os.O_CREAT, 0600), 'w') as fout:
-      fout.write(json.dumps(parameter_dict, indent=2, sort_keys=True))
+    # XXX: do not touch file if there's no change to avoid excessive IO
+    #      (see https://lab.nexedi.com/nexedi/slapos.recipe.template/commit/14d26bc8c77a1940f389026bdbd3a9b229b241f4
+    #       for an example to fix this)
+    with os.fdopen(os.open(self._json_output, os.O_WRONLY | os.O_CREAT, 0o600), 'w') as fout:
+      json.dump(parameter_dict, fout, indent=2, sort_keys=True)
 
     def install(self):
         return [self._json_output]

slapos/recipe/stunnel/__init__.py

@@ -36,7 +36,7 @@ def kill(pid_file, sig=signal.SIGUSR1):
       pid = int(f.read().strip())
     try:
       os.kill(pid, sig)
-    except OSError, e:
+    except OSError as e:
       if e.errno != errno.ESRCH: # No such process
         raise e
       os.unlink(pid_file)

slapos/recipe/wrapper.py

@@ -76,9 +76,9 @@ class Recipe(GenericBaseRecipe):
       import hashlib
       hasher = hashlib.md5()
       for path in file_list:
-        with open(path, 'r') as afile:
+        with open(path, 'rb') as afile:
           buf = afile.read()
-        hasher.update("%s\n" % len(buf))
+        hasher.update(b"%u\n" % len(buf))
         hasher.update(buf)
       hash = hasher.hexdigest()
       return hash

slapos/recipe/xwiki/__init__.py

@@ -69,7 +69,7 @@ export JAVA_OPTS="${JAVA_OPTS} -Djava.awt.headless=true"
     bindir = os.path.join(tomcat_home, 'bin')
     for f in os.listdir(bindir):
       if f.endswith('.sh'):
-        os.chmod(os.path.join(bindir, f), 0755)
+        os.chmod(os.path.join(bindir, f), 0o755)
     tomcat_wrapper = self.createRunningWrapper('xwiki', """#!/bin/sh
 export JRE_HOME=%(java_home)s
 exec %(catalina)s run

slapos/recipe/zabbixagent/__init__.py

@@ -33,7 +33,6 @@ import hashlib
 import sys
 import zc.buildout
 import zc.recipe.egg
-import ConfigParser
 
 class Recipe(BaseSlapRecipe):
   def installLogrotate(self):

slapos/test/recipe/test_pbs.py

@@ -5,6 +5,8 @@ import sys
 import tempfile
 import unittest
 
+import six
+
 
 class PBSTest(unittest.TestCase):
 
@@ -22,7 +24,7 @@ class PBSTest(unittest.TestCase):
     def test_push(self):
         recipe = self.new_recipe()
 
-        with tempfile.NamedTemporaryFile() as rdiff_wrapper:
+        with tempfile.NamedTemporaryFile('w+') as rdiff_wrapper:
             recipe.wrapper_push(remote_schema='TEST_REMOTE_SCHEMA',
                                 local_dir='TEST_LOCAL_DIR',
                                 remote_dir='TEST_REMOTE_DIR',
@@ -35,7 +37,7 @@ class PBSTest(unittest.TestCase):
     def test_pull(self):
         recipe = self.new_recipe()
 
-        with tempfile.NamedTemporaryFile() as rdiff_wrapper:
+        with tempfile.NamedTemporaryFile('w+') as rdiff_wrapper:
             recipe.wrapper_pull(remote_schema='TEST_REMOTE_SCHEMA',
                                 local_dir='TEST_LOCAL_DIR',
                                 remote_dir='TEST_REMOTE_DIR',
@@ -104,22 +106,22 @@ class PBSTest(unittest.TestCase):
 
         recipe._install()
 
-        self.assertItemsEqual(os.listdir(promises_directory),
+        six.assertCountEqual(self, os.listdir(promises_directory),
                               ['ssh-to-pulltest', 'ssh-to-pushtest'])
 
-        self.assertItemsEqual(os.listdir(wrappers_directory),
+        six.assertCountEqual(self, os.listdir(wrappers_directory),
                               ['pulltest_raw', 'pulltest', 'pushtest_raw', 'pushtest'])
 
-        self.assertItemsEqual(os.listdir(directory),
+        six.assertCountEqual(self, os.listdir(directory),
                               ['TEST_NAME'])
 
-        self.assertItemsEqual(os.listdir(feeds_directory),
+        six.assertCountEqual(self, os.listdir(feeds_directory),
                               ['pulltest', 'pushtest'])
 
-        self.assertItemsEqual(os.listdir(run_directory),
+        six.assertCountEqual(self, os.listdir(run_directory),
                               [])
 
-        self.assertItemsEqual(os.listdir(cron_directory),
+        six.assertCountEqual(self, os.listdir(cron_directory),
                               ['pulltest', 'pushtest'])
 
         shutil.rmtree(promises_directory)

slapos/test/recipe/test_plugin.py

@@ -3,6 +3,7 @@ from slapos.recipe import promise_plugin
 from slapos.test.utils import makeRecipe
 from pprint import pformat
 import stat, json
+import six
 
 class TestPromisePlugin(unittest.TestCase):
 
@@ -68,7 +69,7 @@ in multi line
     with open(self.output) as f:
       content = f.read()
     self.assertIn("from slapos.promise.plugin.check_site_available import RunPromise", content)
-    self.assertIn('extra_config_dict = { }', content)
+    self.assertIn('extra_config_dict = %s' % ('{}' if six.PY3 else '{ }'), content)
 
 
   def test_bad_parameters(self):
@@ -99,7 +100,7 @@ in multi line
     with self.assertRaises(ValueError) as p:
       recipe.install()
 
-    self.assertEqual(p.exception.message, "Import path %r is not a valid" % self.options['import'])
+    self.assertEqual(str(p.exception), "Import path %r is not a valid" % self.options['import'])
 
   def test_bad_content(self):
     self.options['content'] = 'from slapos.plugin.check_site_available import toto; print "toto"'
@@ -110,5 +111,5 @@ in multi line
     with self.assertRaises(ValueError) as p:
       recipe.install()
 
-    self.assertEqual(p.exception.message, "Promise content %r is not valid" % self.options['content'])
+    self.assertEqual(str(p.exception), "Promise content %r is not valid" % self.options['content'])
 

slapos/test/recipe/test_re6stnet.py

@@ -6,6 +6,8 @@ import tempfile
 import unittest
 from slapos.slap.slap import NotFoundError, ConnectionError
 
+import six
+
 
 class Re6stnetTest(unittest.TestCase):
 
@@ -88,7 +90,7 @@ class Re6stnetTest(unittest.TestCase):
     
     recipe.generateCertificate()
     
-    self.assertItemsEqual(os.listdir(self.ssl_dir),
+    six.assertCountEqual(self, os.listdir(self.ssl_dir),
                           ['cert.key', 'cert.crt', 'dh.pem'])
     
     last_time = time.ctime(os.stat(self.options['key-file'])[7])
@@ -143,10 +145,10 @@ class Re6stnetTest(unittest.TestCase):
     
     token_dict = recipe.loadJsonFile(token_file)
     self.assertEqual(len(token_dict), 2)
-    self.assertTrue(token_dict.has_key('SOFTINST-58770'))
-    self.assertTrue(token_dict.has_key('SOFTINST-58778'))
+    self.assertIn('SOFTINST-58770', token_dict)
+    self.assertIn('SOFTINST-58778', token_dict)
         
-    self.assertItemsEqual(os.listdir(self.token_dir),
+    six.assertCountEqual(self, os.listdir(self.token_dir),
                           ['SOFTINST-58770.add', 'SOFTINST-58778.add'])
 
     first_add = recipe.readFile(os.path.join(self.token_dir, 'SOFTINST-58770.add'))
@@ -173,7 +175,7 @@ class Re6stnetTest(unittest.TestCase):
     
     self.assertEqual(len(token_dict), 1)
     self.assertEqual(token_dict['SOFTINST-58770'], first_add)
-    self.assertItemsEqual(os.listdir(self.token_dir),
+    six.assertCountEqual(self, os.listdir(self.token_dir),
                           ['SOFTINST-58770.add', 'SOFTINST-58778.remove'])
     
     second_remove = recipe.readFile(os.path.join(self.token_dir, 'SOFTINST-58778.remove'))
@@ -193,7 +195,7 @@ class Re6stnetTest(unittest.TestCase):
     
     token_content = recipe.readFile(token_file)
     self.assertEqual(token_content, '{}')
-    self.assertItemsEqual(os.listdir(self.options['token-dir']), [])
+    six.assertCountEqual(self, os.listdir(self.options['token-dir']), [])
 
     self.checkWrapper(os.path.join(self.base_dir, 'manager_wrapper'))
 

slapos/test/utils.py

@@ -2,7 +2,7 @@
 """
 import sys
 import os.path
-from ConfigParser import ConfigParser
+from zc.buildout.configparser import parse
 
 import logging
 
@@ -52,18 +52,17 @@ def makeRecipe(recipe_class, options, name='test', slap_connection=None):
   buildout_cfg = os.path.join(base_directory, 'buildout.cfg')
 
   if os.path.exists(buildout_cfg):
-    parser = ConfigParser()
-    parser.readfp(open(buildout_cfg))
-    if parser.has_option('buildout', 'eggs-directory'):
-      # when buildout_cfg is an instance buildout (like in SLAPOS-EGG-TEST),
-      # there's a ${buildout:eggs-directory} we can use.
-      eggs_directory = parser.get('buildout', 'eggs-directory')
-      develop_eggs_directory = parser.get('buildout', 'develop-eggs-directory')
-    else:
-      # when when buildout_cfg is a software buildout, we can only guess the
-      # standard eggs directories.
-      eggs_directory = os.path.join(base_directory, 'eggs')
-      develop_eggs_directory = os.path.join(base_directory, 'develop-eggs')
+    with open(buildout_cfg) as f:
+      parsed_cfg = parse(f, buildout_cfg)
+
+    # When buildout_cfg is an instance buildout (like in SLAPOS-EGG-TEST),
+    # there's a ${buildout:eggs-directory} we can use.
+    # When buildout_cfg is a software buildout, we can only guess the
+    # standard eggs directories.
+    eggs_directory = parsed_cfg['buildout'].get(
+      'eggs-directory', os.path.join(base_directory, 'eggs'))
+    develop_eggs_directory = parsed_cfg['buildout'].get(
+      'develop-eggs-directory', os.path.join(base_directory, 'develop-eggs'))
 
     logging.getLogger(__name__).info(
         'Using eggs-directory (%s) and develop-eggs-directory (%s) from buildout at %s',

software/slapos-testing/software-py3.cfg

+[buildout]
+extends =
+  ../../component/python3/buildout.cfg
+  software.cfg
+python = python3.5
+
+[nghttp2]
+environment =
+  PATH=${autoconf:location}/bin:${automake:location}/bin:${libtool:location}/bin:${m4:location}/bin:${python3.5:location}/bin:%(PATH)s
+
+[supervisor-repository]
+<= git-clone-repository
+repository = https://github.com/Supervisor/supervisor.git
+
+[supervisor-develop]
+recipe = zc.recipe.egg:develop
+egg = supervisor
+setup = ${supervisor-repository:location}
+
+[eggs]
+recipe = zc.recipe.egg
+eggs += ${supervisor-develop:egg}
+
+[versions]
+supervisor =

stack/slapos.cfg

@@ -148,7 +148,7 @@ slapos.toolbox = 0.94
 stevedore = 1.21.0
 subprocess32 = 3.5.3
 unicodecsv = 0.14.1
-xml-marshaller = 0.9.7
+xml-marshaller = 1.0.2
 paramiko = 2.1.3
 CacheControl = 0.12.5
 msgpack = 0.6.1
@@ -228,7 +228,7 @@ lockfile = 0.12.2
 # Required by:
 # slapos.core==1.4.26
 # XXX 'slapos node format' raises an exception with netifaces 0.10.5.
-netifaces = 0.10.4
+netifaces = 0.10.7
 
 # Required by:
 # cryptography==1.8.1