all: More python3 adaptations.

What was not picked up by 2to3.

caucase/ca.py

@@ -19,6 +19,7 @@
 Caucase - Certificate Authority for Users, Certificate Authority for SErvices
 """
 from __future__ import absolute_import
+from binascii import hexlify, unhexlify
 import datetime
 import json
 import os
@@ -55,7 +56,7 @@ _SUBJECT_OID_DICT = {
   'GN': x509.oid.NameOID.GIVEN_NAME,
   # pylint: enable=bad-whitespace
 }
-_BACKUP_MAGIC = 'caucase\0'
+_BACKUP_MAGIC = b'caucase\0'
 _CONFIG_NAME_AUTO_SIGN_CSR_AMOUNT = 'auto_sign_csr_amount'
 
 def Extension(value, critical):
@@ -227,9 +228,9 @@ class CertificateAuthority(object):
     # Note: requested_amount is None when a known CSR is re-submitted
     csr_id, requested_amount = self._storage.appendCertificateSigningRequest(
       csr_pem=csr_pem,
-      key_id=x509.SubjectKeyIdentifier.from_public_key(
+      key_id=hexlify(x509.SubjectKeyIdentifier.from_public_key(
         csr.public_key(),
-      ).digest.encode('hex'),
+      ).digest),
       override_limits=override_limits,
     )
     if requested_amount is not None and \
@@ -632,8 +633,8 @@ class CertificateAuthority(object):
       current_crt_pem = utils.dump_certificate(key_pair['crt'])
       result.append(utils.wrap(
         {
-          'old_pem': previous_crt_pem,
-          'new_pem': current_crt_pem,
+          'old_pem': utils.toUnicode(previous_crt_pem),
+          'new_pem': utils.toUnicode(current_crt_pem),
         },
         previous_key,
         self.digest_list[0],
@@ -799,31 +800,31 @@ class UserCertificateAuthority(CertificateAuthority):
         continue
       public_key = crt.public_key()
       key_list.append({
-        'id': x509.SubjectKeyIdentifier.from_public_key(
-          public_key,
-        ).digest.encode('hex'),
+        'id': utils.toUnicode(hexlify(
+          x509.SubjectKeyIdentifier.from_public_key(public_key).digest,
+        )),
         'cipher': {
           'name': 'rsa_oaep_sha1_mgf1_sha1',
         },
-        'key': public_key.encrypt(
+        'key': utils.toUnicode(hexlify(public_key.encrypt(
           signing_key + symetric_key,
           OAEP(
             mgf=MGF1(algorithm=hashes.SHA1()),
             algorithm=hashes.SHA1(),
             label=None,
           ),
-        ).encode('hex'),
+        ))),
       })
     if not key_list:
       # No users yet, backup is meaningless
       return False
-    header = json.dumps({
+    header = utils.toBytes(json.dumps({
       'cipher': {
         'name': 'aes256_cbc_pkcs7_hmac_10M_sha256',
-        'parameter': iv.encode('hex'),
+        'parameter': utils.toUnicode(hexlify(iv)),
       },
       'key_list': key_list,
-    })
+    }))
     padder = padding.PKCS7(128).padder()
     write(_BACKUP_MAGIC)
     write(struct.pack('<I', len(header)))
@@ -877,11 +878,11 @@ class UserCertificateAuthority(CertificateAuthority):
     if header['cipher']['name'] != 'aes256_cbc_pkcs7_hmac_10M_sha256':
       raise ValueError('Unrecognised symetric cipher')
     private_key = utils.load_privatekey(key_pem)
-    key_id = x509.SubjectKeyIdentifier.from_public_key(
+    key_id = hexlify(x509.SubjectKeyIdentifier.from_public_key(
       private_key.public_key(),
-    ).digest.encode('hex')
+    ).digest)
     symetric_key_list = [
-      x for x in header['key_list'] if x['id'] == key_id
+      x for x in header['key_list'] if utils.toBytes(x['id']) == key_id
     ]
     if not symetric_key_list:
       raise ValueError(
@@ -891,7 +892,7 @@ class UserCertificateAuthority(CertificateAuthority):
     if symetric_key_entry['cipher']['name'] != 'rsa_oaep_sha1_mgf1_sha1':
       raise ValueError('Unrecognised asymetric cipher')
     both_keys = private_key.decrypt(
-      symetric_key_entry['key'].decode('hex'),
+      unhexlify(symetric_key_entry['key']),
       OAEP(
         mgf=MGF1(algorithm=hashes.SHA1()),
         algorithm=hashes.SHA1(),
@@ -902,7 +903,7 @@ class UserCertificateAuthority(CertificateAuthority):
       raise ValueError('Invalid key length')
     decryptor = Cipher(
       algorithms.AES(both_keys[32:]),
-      modes.CBC(header['cipher']['parameter'].decode('hex')),
+      modes.CBC(unhexlify(header['cipher']['parameter'])),
       backend=_cryptography_backend,
     ).decryptor()
     unpadder = padding.PKCS7(128).unpadder()

caucase/cli.py

@@ -20,6 +20,7 @@ Caucase - Certificate Authority for Users, Certificate Authority for SErvices
 """
 from __future__ import absolute_import, print_function
 import argparse
+from binascii import hexlify
 import datetime
 import httplib
 import json
@@ -102,7 +103,7 @@ class CLICaucaseClient(object):
     """
     for csr_id, csr_path in csr_id_path_list:
       csr_pem = self._client.getCertificateSigningRequest(int(csr_id))
-      with open(csr_path, 'a') as csr_file:
+      with open(csr_path, 'ab') as csr_file:
         csr_file.write(csr_pem)
 
   def getCRT(self, warning, error, crt_id_path_list, ca_list):
@@ -157,7 +158,7 @@ class CLICaucaseClient(object):
             )
             error = True
             continue
-        with open(crt_path, 'a') as crt_file:
+        with open(crt_path, 'ab') as crt_file:
           crt_file.write(crt_pem)
     return warning, error
 
@@ -228,11 +229,17 @@ class CLICaucaseClient(object):
         key_len=key_len,
       )
       if key_path is None:
-        with open(crt_path, 'w') as crt_file:
+        with open(crt_path, 'wb') as crt_file:
           crt_file.write(new_key_pem)
           crt_file.write(new_crt_pem)
       else:
-        with open(crt_path, 'w') as crt_file, open(key_path, 'w') as key_file:
+        with open(
+          crt_path,
+          'wb',
+        ) as crt_file, open(
+          key_path,
+          'wb',
+        ) as key_file:
           key_file.write(new_key_pem)
           crt_file.write(new_crt_pem)
       updated = True
@@ -250,7 +257,7 @@ class CLICaucaseClient(object):
       ),
     )
     for entry in self._client.getPendingCertificateRequestList():
-      csr = utils.load_certificate_request(entry['csr'])
+      csr = utils.load_certificate_request(utils.toBytes(entry['csr']))
       print(
         '%20s | %r' % (
           entry['id'],
@@ -264,7 +271,7 @@ class CLICaucaseClient(object):
     --sign-csr
     """
     for csr_id in csr_id_list:
-      self._client.createCertificate(int(csr_id))
+      self._client.createCertificate(int(utils.toUnicode(csr_id)))
 
   def signCSRWith(self, csr_id_path_list):
     """
@@ -272,7 +279,7 @@ class CLICaucaseClient(object):
     """
     for csr_id, csr_path in csr_id_path_list:
       self._client.createCertificate(
-        int(csr_id),
+        int(utils.toUnicode(csr_id)),
         template_csr=utils.getCertRequest(csr_path),
       )
 
@@ -763,7 +770,7 @@ def updater(argv=None, until=utils.until):
           # Still here ? Ok, wait a bit and try again.
           until(datetime.datetime.utcnow() + datetime.timedelta(0, 60))
         else:
-          with open(args.crt, 'a') as crt_file:
+          with open(args.crt, 'ab') as crt_file:
             crt_file.write(crt_pem)
           updated = True
           break
@@ -797,10 +804,11 @@ def updater(argv=None, until=utils.until):
       if RetryingCaucaseClient.updateCRLFile(ca_url, args.crl, ca_crt_list):
         print('Got new CRL')
         updated = True
-        next_deadline = min(
-          next_deadline,
-          utils.load_crl(open(args.crl).read(), ca_crt_list).next_update,
-        )
+        with open(args.crl, 'rb') as crl_file:
+          next_deadline = min(
+            next_deadline,
+            utils.load_crl(crli_file.read(), ca_crt_list).next_update,
+          )
       if args.crt:
         crt_pem, key_pem, key_path = utils.getKeyPair(args.crt, args.key)
         crt = utils.load_certificate(crt_pem, ca_crt_list, None)
@@ -812,16 +820,16 @@ def updater(argv=None, until=utils.until):
             key_len=args.key_len,
           )
           if key_path is None:
-            with open(args.crt, 'w') as crt_file:
+            with open(args.crt, 'wb') as crt_file:
               crt_file.write(new_key_pem)
               crt_file.write(new_crt_pem)
           else:
             with open(
               args.crt,
-              'w',
+              'wb',
             ) as crt_file, open(
               key_path,
-              'w',
+              'wb',
             ) as key_file:
               key_file.write(new_key_pem)
               crt_file.write(new_crt_pem)
@@ -894,11 +902,11 @@ def rerequest(argv=None):
   key_pem = utils.dump_privatekey(key)
   orig_umask = os.umask(0o177)
   try:
-    with open(args.key, 'w') as key_file:
+    with open(args.key, 'wb') as key_file:
       key_file.write(key_pem)
   finally:
     os.umask(orig_umask)
-  with open(args.csr, 'w') as csr_file:
+  with open(args.csr, 'wb') as csr_file:
     csr_file.write(csr_pem)
 
 def key_id(argv=None):
@@ -926,17 +934,20 @@ def key_id(argv=None):
   )
   args = parser.parse_args(argv)
   for key_path in args.private_key:
-    print(
-      key_path,
-      x509.SubjectKeyIdentifier.from_public_key(
-        utils.load_privatekey(open(key_path).read()).public_key(),
-      ).digest.encode('hex'),
-    )
+    with open(key_path, 'rb') as key_file:
+      print(
+        key_path,
+        utils.toUnicode(hexlify(
+          x509.SubjectKeyIdentifier.from_public_key(
+            utils.load_privatekey(key_file.read()).public_key(),
+          ).digest,
+        )),
+      )
   for backup_path in args.backup:
     print(backup_path)
-    with open(backup_path) as backup_file:
+    with open(backup_path, 'rb') as backup_file:
       magic = backup_file.read(8)
-      if magic != 'caucase\0':
+      if magic != b'caucase\0':
         raise ValueError('Invalid backup magic string')
       header_len, = struct.unpack(
         '<I',

caucase/client.py

@@ -69,7 +69,7 @@ class CaucaseClient(object):
     """
     if not os.path.exists(ca_crt_path):
       ca_pem = cls(ca_url=url).getCACertificate()
-      with open(ca_crt_path, 'w') as ca_crt_file:
+      with open(ca_crt_path, 'wb') as ca_crt_file:
         ca_crt_file.write(ca_pem)
       updated = True
     else:
@@ -85,8 +85,8 @@ class CaucaseClient(object):
       cls(ca_url=url, ca_crt_pem_list=ca_pem_list).getCACertificateChain(),
     )
     if ca_pem_list != loaded_ca_pem_list:
-      data = ''.join(ca_pem_list)
-      with open(ca_crt_path, 'w') as ca_crt_file:
+      data = b''.join(ca_pem_list)
+      with open(ca_crt_path, 'wb') as ca_crt_file:
         ca_crt_file.write(data)
       updated = True
     return updated
@@ -107,13 +107,13 @@ class CaucaseClient(object):
     Return whether an update happened.
     """
     if os.path.exists(crl_path):
-      my_crl = utils.load_crl(open(crl_path).read(), ca_list)
+      my_crl = utils.load_crl(open(crl_path, 'rb').read(), ca_list)
     else:
       my_crl = None
     latest_crl_pem = cls(ca_url=url).getCertificateRevocationList()
     latest_crl = utils.load_crl(latest_crl_pem, ca_list)
     if my_crl is None or latest_crl.signature != my_crl.signature:
-      with open(crl_path, 'w') as crl_file:
+      with open(crl_path, 'wb') as crl_file:
         crl_file.write(latest_crl_pem)
       return True
     return False
@@ -138,7 +138,11 @@ class CaucaseClient(object):
     ssl_context = ssl.create_default_context(
       # unicode object needed as we use PEM, otherwise create_default_context
       # expects DER.
-      cadata=''.join(http_ca_crt_pem_list).decode('ascii')  if http_ca_crt_pem_list else None,
+      cadata=(
+        utils.toUnicode(''.join(http_ca_crt_pem_list))
+        if http_ca_crt_pem_list
+        else None
+      ),
     )
     if not http_ca_crt_pem_list:
       ssl_context.check_hostname = False
@@ -191,13 +195,7 @@ class CaucaseClient(object):
     """
     [AUTHENTICATED] Retrieve all pending CSRs.
     """
-    return [
-      {
-        y.encode('ascii'): z.encode('ascii') if isinstance(z, unicode) else z
-        for y, z in x.iteritems()
-      }
-      for x in json.loads(self._https('GET', '/csr'))
-    ]
+    return json.loads(self._https('GET', '/csr'))
 
   def createCertificateSigningRequest(self, csr):
     """
@@ -254,14 +252,14 @@ class CaucaseClient(object):
         continue
       if not found:
         found = utils.load_ca_certificate(
-          payload['old_pem'].encode('ascii'),
+          utils.toBytes(payload['old_pem']),
         ) == trust_anchor
       if found:
         if utils.load_ca_certificate(
-          payload['old_pem'].encode('ascii'),
+          utils.toBytes(payload['old_pem']),
         ) != previous_ca:
           raise ValueError('CA signature chain broken')
-        new_pem = payload['new_pem'].encode('ascii')
+        new_pem = utils.toBytes(payload['new_pem'])
         result.append(new_pem)
         previous_ca = utils.load_ca_certificate(new_pem)
     return result
@@ -279,8 +277,8 @@ class CaucaseClient(object):
         json.dumps(
           utils.wrap(
             {
-              'crt_pem': utils.dump_certificate(old_crt),
-              'renew_csr_pem': utils.dump_certificate_request(
+              'crt_pem': utils.toUnicode(utils.dump_certificate(old_crt)),
+              'renew_csr_pem': utils.toUnicode(utils.dump_certificate_request(
                 x509.CertificateSigningRequestBuilder(
                 ).subject_name(
                   # Note: caucase server ignores this, but cryptography
@@ -291,7 +289,7 @@ class CaucaseClient(object):
                   algorithm=utils.DEFAULT_DIGEST_CLASS(),
                   backend=_cryptography_backend,
                 ),
-              ),
+              )),
             },
             old_key,
             utils.DEFAULT_DIGEST,
@@ -307,6 +305,7 @@ class CaucaseClient(object):
     [ANONYMOUS] if key is provided.
     [AUTHENTICATED] if key is missing.
     """
+    crt = utils.toUnicode(crt)
     if key:
       method = self._http
       data = utils.wrap(

caucase/http.py

@@ -70,7 +70,7 @@ def _createKey(path):
   """
   return os.fdopen(
     os.open(path, os.O_WRONLY | os.O_CREAT, 0o600),
-    'w',
+    'wb',
   )
 
 class ThreadingWSGIServer(ThreadingMixIn, WSGIServer):
@@ -236,7 +236,7 @@ def getSSLContext(
   # implementation cross-check would have been nice.
   #ssl_context.verify_flags = ssl.VERIFY_CRL_CHECK_LEAF
   ssl_context.load_verify_locations(
-    cadata=cau.getCACertificate().decode('ascii'),
+    cadata=utils.toUnicode(cau.getCACertificate()),
   )
   http_cas_certificate_list = http_cas.getCACertificateList()
   threshold_delta = datetime.timedelta(threshold, 0)
@@ -500,12 +500,12 @@ def main(argv=None, until=utils.until):
   )
   args = parser.parse_args(argv)
 
-  base_url = u'http://' + args.netloc.decode('ascii')
+  base_url = u'http://' + utils.toUnicode(args.netloc)
   parsed_base_url = urlparse(base_url)
   hostname = parsed_base_url.hostname
   name_constraints_permited = []
   name_constraints_excluded = []
-  hostname_dnsname = hostname.decode('ascii')
+  hostname_dnsname = utils.toUnicode(hostname)
   try:
     hostname_ip_address = ipaddress.ip_address(hostname_dnsname)
   except ValueError:
@@ -615,7 +615,7 @@ def main(argv=None, until=utils.until):
     crt_life_time=args.service_crt_validity,
   )
   if os.path.exists(args.cors_key_store):
-    with open(args.cors_key_store) as cors_key_file:
+    with open(args.cors_key_store, 'rb') as cors_key_file:
       cors_secret_list = json.load(cors_key_file)
   else:
     cors_secret_list = []
@@ -761,7 +761,7 @@ def main(argv=None, until=utils.until):
           tmp_backup_fd, tmp_backup_path = tempfile.mkstemp(
             prefix='caucase_backup_',
           )
-          with os.fdopen(tmp_backup_fd, 'w') as backup_file:
+          with os.fdopen(tmp_backup_fd, 'wb') as backup_file:
             result = cau.doBackup(backup_file.write)
           if result:
             backup_path = os.path.join(
@@ -782,6 +782,7 @@ def main(argv=None, until=utils.until):
   finally:
     sys.stderr.write('Exiting\n')
     for server in itertools.chain(http_list, https_list):
+      server.server_close()
       server.shutdown()
 
 def manage(argv=None):
@@ -820,7 +821,7 @@ def manage(argv=None):
     default=[],
     metavar='PEM_FILE',
     action='append',
-    type=argparse.FileType('r'),
+    type=argparse.FileType('rb'),
     help='Import key pairs as initial service CA certificate. '
     'May be provided multiple times to import multiple key pairs. '
     'Keys and certificates may be in separate files. '
@@ -846,7 +847,7 @@ def manage(argv=None):
     default=[],
     metavar='PEM_FILE',
     action='append',
-    type=argparse.FileType('r'),
+    type=argparse.FileType('rb'),
     help='Import service revocation list. Corresponding CA certificate must '
     'be already present in the database (including added in the same run '
     'using --import-ca).',
@@ -854,7 +855,7 @@ def manage(argv=None):
   parser.add_argument(
     '--export-ca',
     metavar='PEM_FILE',
-    type=argparse.FileType('w'),
+    type=argparse.FileType('wb'),
     help='Export all CA certificates in a PEM file. Passphrase will be '
     'prompted to protect all keys.',
   )
@@ -873,30 +874,35 @@ def manage(argv=None):
       # maybe user extracted their private key ?
       key_pem = utils.getKey(backup_key_path)
     cau_crt_life_time = args.user_crt_validity
-    with open(backup_path) as backup_file:
-      with open(backup_crt_path, 'a') as new_crt_file:
-        new_crt_file.write(
-          UserCertificateAuthority.restoreBackup(
-            db_class=SQLite3Storage,
-            db_path=db_path,
-            read=backup_file.read,
-            key_pem=key_pem,
-            csr_pem=utils.getCertRequest(backup_csr_path),
-            db_kw={
-              'table_prefix': 'cau',
-              # max_csr_amount: not needed, renewal ignores quota
-              # Effectively disables certificate expiration
-              'crt_keep_time': cau_crt_life_time,
-              'crt_read_keep_time': cau_crt_life_time,
-              'enforce_unique_key_id': True,
-            },
-            kw={
-              # Disable CA cert renewal
-              'ca_key_size': None,
-              'crt_life_time': cau_crt_life_time,
-            },
-          ),
-        )
+    with open(
+      backup_path,
+      'rb',
+    ) as backup_file, open(
+      backup_crt_path,
+      'ab',
+    ) as new_crt_file:
+      new_crt_file.write(
+        UserCertificateAuthority.restoreBackup(
+          db_class=SQLite3Storage,
+          db_path=db_path,
+          read=backup_file.read,
+          key_pem=key_pem,
+          csr_pem=utils.getCertRequest(backup_csr_path),
+          db_kw={
+            'table_prefix': 'cau',
+            # max_csr_amount: not needed, renewal ignores quota
+            # Effectively disables certificate expiration
+            'crt_keep_time': cau_crt_life_time,
+            'crt_read_keep_time': cau_crt_life_time,
+            'enforce_unique_key_id': True,
+          },
+          kw={
+            # Disable CA cert renewal
+            'ca_key_size': None,
+            'crt_life_time': cau_crt_life_time,
+          },
+        ),
+      )
   if args.import_ca:
     import_ca_dict = defaultdict(
       (lambda: {'crt': None, 'key': None, 'from': []}),

caucase/http_wsgibase.py

@@ -22,6 +22,7 @@ Separate from .http because of different-licensed code in the middle.
 """
 from __future__ import absolute_import
 from wsgiref.simple_server import ServerHandler
+from .utils import toBytes
 
 class ProxyFile(object):
   """
@@ -48,7 +49,7 @@ class ChunkedFile(ProxyFile):
     """
     Read chunked data.
     """
-    result = ''
+    result = b''
     if not self._at_eof:
       readline = self.readline
       read = self.__getattr__('read')
@@ -61,7 +62,7 @@ class ChunkedFile(ProxyFile):
           if len(chunk_header) > MAX_CHUNKED_HEADER_LENGTH:
             raise ValueError('Chunked encoding header too long')
           try:
-            chunk_length = int(chunk_header.split(';', 1)[0], 16)
+            chunk_length = int(chunk_header.split(b';', 1)[0], 16)
           except ValueError:
             raise ValueError('Invalid chunked encoding header')
           if not chunk_length:
@@ -78,7 +79,7 @@ class ChunkedFile(ProxyFile):
         if to_read != chunk_length:
           self._chunk_remaining_length = chunk_length - to_read
           break
-        if read(2) != '\r\n':
+        if read(2) != b'\r\n':
           raise ValueError('Invalid chunked encoding separator')
     return result
 
@@ -131,7 +132,7 @@ class CleanServerHandler(ServerHandler):
     """
     Emit "100 Continue" intermediate response.
     """
-    self._write('HTTP/%s 100 Continue\r\n\r\n' % (
-      self.http_version,
+    self._write(b'HTTP/%s 100 Continue\r\n\r\n' % (
+      toBytes(self.http_version),
     ))
     self._flush()

caucase/storage.py

@@ -25,6 +25,7 @@ import sqlite3
 from threading import local
 from time import time
 from .exceptions import NoStorage, NotFound, Found
+from .utils import toBytes, toUnicode
 
 __all__ = ('SQLite3Storage', )
 
@@ -207,8 +208,8 @@ class SQLite3Storage(local):
       )
       return [
         {
-          'crt_pem': x['crt'].encode('ascii'),
-          'key_pem': x['key'].encode('ascii'),
+          'crt_pem': toBytes(x['crt']),
+          'key_pem': toBytes(x['key']),
         }
         for x in db.cursor().execute(
           'SELECT key, crt FROM %sca ORDER BY expiration_date ASC' % (
@@ -326,7 +327,7 @@ class SQLite3Storage(local):
       )
       if result is None:
         raise NotFound
-      return result['csr'].encode('ascii')
+      return toBytes(result['csr'])
 
   def getCertificateSigningRequestList(self):
     """
@@ -338,7 +339,11 @@ class SQLite3Storage(local):
       return [
         {
           'id': str(x['id']),
-          'csr': x['csr'].encode('ascii'),
+          # XXX: because only call chain will end up serialising this value in
+          # json, and for some reason python3 json module refuses bytes.
+          # So rather than byte-ify (consistently with all PEM-encoded values)
+          # to then have to unicode-ify, just unicode-ify here.
+          'csr': toUnicode(x['csr']),
         }
         for x in db.cursor().execute(
           'SELECT id, csr FROM %scrt WHERE crt IS NULL' % (
@@ -401,7 +406,7 @@ class SQLite3Storage(local):
             crt_id,
           )
         )
-      return row['crt'].encode('ascii')
+      return toBytes(row['crt'])
 
   def getCertificateByKeyIdentifier(self, key_id):
     """
@@ -419,7 +424,7 @@ class SQLite3Storage(local):
       )
       if row is None:
         raise NotFound
-      return row['crt'].encode('ascii')
+      return toBytes(row['crt'])
 
   def iterCertificates(self):
     """
@@ -434,7 +439,7 @@ class SQLite3Storage(local):
         row = c.fetchone()
         if row is None:
           break
-        yield row['crt'].encode('ascii')
+        yield toBytes(row['crt'])
 
   def revoke(self, serial, expiration_date):
     """
@@ -483,7 +488,7 @@ class SQLite3Storage(local):
         (time(), )
       )
       if row is not None:
-        return row['crl'].encode('ascii')
+        return toBytes(row['crl'])
     return None
 
   def getNextCertificateRevocationListNumber(self):
@@ -547,7 +552,7 @@ class SQLite3Storage(local):
     class (so not limited to table_prefix).
     """
     for statement in self._db.iterdump():
-      yield statement.encode('utf-8') + '\0'
+      yield toBytes(statement, 'utf-8') + b'\0'
 
   @staticmethod
   def restore(db_path, restorator):
@@ -563,14 +568,14 @@ class SQLite3Storage(local):
       Produces chunks which correspond (in content, not necessarily in size)
       to what dumpIterator produces.
     """
-    buf = ''
+    buf = b''
     if os.path.exists(db_path):
       raise ValueError('%r exists, not restoring.' % (db_path, ))
     c = sqlite3.connect(db_path, isolation_level=None).cursor()
     for chunk in restorator:
-      statement_list = (buf + chunk).split('\0')
+      statement_list = (buf + chunk).split(b'\0')
       buf = statement_list.pop()
       for statement in statement_list:
-        c.execute((statement).decode('utf-8'))
+        c.execute(toUnicode(statement, 'utf-8'))
     if buf:
       raise ValueError('Short read, backup truncated ?')

caucase/utils.py

@@ -21,6 +21,7 @@ Caucase - Certificate Authority for Users, Certificate Authority for SErvices
 Small-ish functions needed in many places.
 """
 from __future__ import absolute_import
+from binascii import a2b_base64, b2a_base64
 from collections import defaultdict
 import datetime
 import json
@@ -273,19 +274,20 @@ def wrap(payload, key, digest):
   """
   Sign payload (which gets json-serialised) with key, using given digest.
   """
-  payload = json.dumps(payload).encode('utf-8')
+  payload = toBytes(json.dumps(payload), 'utf-8')
   hash_class = getattr(hashes, digest.upper())
   return {
-    'payload': payload,
+    'payload': toUnicode(payload),
     'digest': digest,
-    'signature': key.sign(
-      payload + digest + ' ',
+    # For some reason, python3 thinks that a b2a method should return bytes.
+    'signature': toUnicode(b2a_base64(key.sign(
+      payload + toBytes(digest) + b' ',
       padding.PSS(
         mgf=padding.MGF1(hash_class()),
         salt_length=padding.PSS.MAX_LENGTH,
       ),
       hash_class(),
-    ).encode('base64'),
+    ))),
   }
 
 def nullWrap(payload):
@@ -308,10 +310,10 @@ def unwrap(wrapped, getCertificate, digest_list):
   Note: does *not* verify received certificate itself (validity, issuer, ...).
   """
   # Check whether given digest is allowed
-  digest = wrapped['digest'].encode('ascii')
+  digest = wrapped['digest']
   if digest not in digest_list:
     raise cryptography.exceptions.UnsupportedAlgorithm(
-      '%r is not in allowed digest list',
+      '%r is not in allowed digest list %r' % (digest, digest_list),
     )
   hash_class = getattr(hashes, digest.upper())
   try:
@@ -319,11 +321,11 @@ def unwrap(wrapped, getCertificate, digest_list):
   except ValueError:
     raise NotJSON
   x509.load_pem_x509_certificate(
-    getCertificate(payload).encode('ascii'),
+    toBytes(getCertificate(payload)),
     _cryptography_backend,
   ).public_key().verify(
-    wrapped['signature'].encode('ascii').decode('base64'),
-    wrapped['payload'].encode('utf-8') + digest + ' ',
+    a2b_base64(toBytes(wrapped['signature'])),
+    toBytes(wrapped['payload'], 'utf-8') + toBytes(digest) + b' ',
     padding.PSS(
       mgf=padding.MGF1(hash_class()),
       salt_length=padding.PSS.MAX_LENGTH,
@@ -445,6 +447,18 @@ class SleepInterrupt(KeyboardInterrupt):
   """
   pass
 
+def toUnicode(value, encoding='ascii'):
+  """
+  Convert value to unicode object, if it is not already.
+  """
+  return value if isinstance(value, unicode) else value.decode(encoding)
+
+def toBytes(value, encoding='ascii'):
+  """
+  Convert valye to bytes object, if it is not already.
+  """
+  return value if isinstance(value, bytes) else value.encode(encoding)
+
 def interruptibleSleep(duration): # pragma: no cover
   """
   Like sleep, but raises SleepInterrupt when interrupted by KeyboardInterrupt

caucase/wsgi.py

@@ -19,11 +19,11 @@
 Caucase - Certificate Authority for Users, Certificate Authority for SErvices
 """
 from __future__ import absolute_import
-from cgi import escape
 from Cookie import SimpleCookie, CookieError
 import httplib
 import json
 import os
+import sys
 import threading
 import time
 import traceback
@@ -34,10 +34,15 @@ import jwt
 from . import utils
 from . import exceptions
 
+if sys.version_info >= (3, ): # pragma: no cover
+  from html import escape
+else: # pragma: no cover
+  from cgi import escape
+
 __all__ = ('Application', 'CORSTokenManager')
 
 # TODO: l10n
-CORS_FORM_TEMPLATE = '''\
+CORS_FORM_TEMPLATE = b'''\
 <html>
 <head>
   <title>Caucase CORS access</title>
@@ -213,11 +218,11 @@ class CORSTokenManager(object):
         key = os.urandom(32)
         secret_list.append((now + self._secret_validity_period, key))
         self._onNewKey(secret_list)
-    return jwt.encode(
+    return utils.toUnicode(jwt.encode(
       payload={'p': payload},
       key=key,
       algorithm='HS256',
-    )
+    ))
 
   def verify(self, token, default=None):
     """
@@ -571,7 +576,7 @@ class Application(object):
       except exceptions.NoStorage:
         raise InsufficientStorage
       except exceptions.NotJSON:
-        raise BadRequest('Invalid json payload')
+        raise BadRequest(b'Invalid json payload')
       except exceptions.CertificateAuthorityException as e:
         raise BadRequest(str(e))
       except Exception:
@@ -581,7 +586,7 @@ class Application(object):
     except ApplicationError as e:
       status = e.status
       header_list = e.response_headers
-      result = [str(x) for x in e.args]
+      result = [utils.toBytes(str(x)) for x in e.args]
     # Note: header_list and cors_header_list are expected to contain
     # distinct header sets. This may not always stay true for "Vary".
     header_list.extend(cors_header_list)
@@ -605,7 +610,7 @@ class Application(object):
     try:
       return int(crt_id)
     except ValueError:
-      raise BadRequest('Invalid integer')
+      raise BadRequest(b'Invalid integer')
 
   @staticmethod
   def _read(environ):
@@ -619,9 +624,9 @@ class Application(object):
     try:
       length = int(environ.get('CONTENT_LENGTH') or MAX_BODY_LENGTH)
     except ValueError:
-      raise BadRequest('Invalid Content-Length')
+      raise BadRequest(b'Invalid Content-Length')
     if length > MAX_BODY_LENGTH:
-      raise TooLarge('Content-Length limit exceeded')
+      raise TooLarge(b'Content-Length limit exceeded')
     return environ['wsgi.input'].read(length)
 
   def _authenticate(self, environ, header_list):
@@ -653,12 +658,12 @@ class Application(object):
     json decoding fails.
     """
     if environ.get('CONTENT_TYPE') != 'application/json':
-      raise BadRequest('Bad Content-Type')
+      raise BadRequest(b'Bad Content-Type')
     data = self._read(environ)
     try:
       return json.loads(data)
     except ValueError:
-      raise BadRequest('Invalid json')
+      raise BadRequest(b'Invalid json')
 
   def _createCORSCookie(self, environ, value):
     """
@@ -859,7 +864,10 @@ class Application(object):
             name = action['name']
             assert name not in hal_section_dict, name
             hal_section_dict[name] = descriptor_dict
-    return self._returnFile(json.dumps(hal), 'application/hal+json')
+    return self._returnFile(
+      utils.toBytes(json.dumps(hal)),
+      'application/hal+json',
+    )
 
   def getCORSForm(self, context, environ): # pylint: disable=unused-argument
     """
@@ -881,9 +889,9 @@ class Application(object):
       raise BadRequest
     return self._returnFile(
       CORS_FORM_TEMPLATE % {
-        'caucase': escape(self._http_url, quote=True),
-        'return_to': escape(return_to, quote=True),
-        'origin': escape(origin, quote=True),
+        b'caucase': utils.toBytes(escape(self._http_url, quote=True)),
+        b'return_to': utils.toBytes(escape(return_to, quote=True)),
+        b'origin': utils.toBytes(escape(origin, quote=True)),
       },
       'text/html',
       [
@@ -902,7 +910,7 @@ class Application(object):
     if environ['wsgi.url_scheme'] != 'https':
       raise NotFound
     if environ.get('CONTENT_TYPE') != 'application/x-www-form-urlencoded':
-      raise BadRequest('Unhandled Content-Type')
+      raise BadRequest(b'Unhandled Content-Type')
     try:
       form_dict = parse_qs(self._read(environ), strict_parsing=True)
       origin, = form_dict['origin']
@@ -961,7 +969,7 @@ class Application(object):
     header_list = []
     self._authenticate(environ, header_list)
     return self._returnFile(
-      json.dumps(context.getCertificateRequestList()),
+      utils.toBytes(json.dumps(context.getCertificateRequestList())),
       'application/json',
       header_list,
     )
@@ -973,7 +981,7 @@ class Application(object):
     try:
       csr_id = context.appendCertificateSigningRequest(self._read(environ))
     except exceptions.NotACertificateSigningRequest:
-      raise BadRequest('Not a valid certificate signing request')
+      raise BadRequest(b'Not a valid certificate signing request')
     return (STATUS_CREATED, [('Location', str(csr_id))], [])
 
   def deletePendingCertificateRequest(self, context, environ, subpath):
@@ -1013,7 +1021,7 @@ class Application(object):
     Handle GET /{context}/crt/ca.crt.json urls.
     """
     return self._returnFile(
-      json.dumps(context.getValidCACertificateChain()),
+      utils.toBytes(json.dumps(context.getValidCACertificateChain())),
       'application/json',
     )
 
@@ -1050,7 +1058,7 @@ class Application(object):
         context.digest_list,
       )
     context.revoke(
-      crt_pem=payload['revoke_crt_pem'].encode('ascii'),
+      crt_pem=utils.toBytes(payload['revoke_crt_pem']),
     )
     return (STATUS_NO_CONTENT, header_list, [])
 
@@ -1065,8 +1073,8 @@ class Application(object):
     )
     return self._returnFile(
       context.renew(
-        crt_pem=payload['crt_pem'].encode('ascii'),
-        csr_pem=payload['renew_csr_pem'].encode('ascii'),
+        crt_pem=utils.toBytes(payload['crt_pem']),
+        csr_pem=utils.toBytes(payload['renew_csr_pem']),
       ),
       'application/pkix-cert',
     )
@@ -1084,7 +1092,7 @@ class Application(object):
     elif environ.get('CONTENT_TYPE') == 'application/pkcs10':
       template_csr = utils.load_certificate_request(body)
     else:
-      raise BadRequest('Bad Content-Type')
+      raise BadRequest(b'Bad Content-Type')
     header_list = []
     self._authenticate(environ, header_list)
     context.createCertificate(