Pass text to json.loads for python3.5 compatibility

caucase/ca.py

@@ -886,7 +886,7 @@ class UserCertificateAuthority(CertificateAuthority):
       '<I',
       read(struct.calcsize('<I')),
     )
-    header = json.loads(read(header_len))
+    header = json.loads(read(header_len).decode('utf-8'))
     if header['cipher']['name'] != 'aes256_cbc_pkcs7_hmac_10M_sha256':
       raise ValueError('Unrecognised symetric cipher')
     private_key = utils.load_privatekey(key_pem)

caucase/cli.py

@@ -1002,5 +1002,5 @@ def key_id(argv=None):
         '<I',
         backup_file.read(struct.calcsize('<I')),
       )
-      for key_entry in json.loads(backup_file.read(header_len))['key_list']:
+      for key_entry in json.loads(backup_file.read(header_len).decode('utf-8'))['key_list']:
         print(' ', key_entry['id'].encode('utf-8'))

caucase/client.py

@@ -195,7 +195,7 @@ class CaucaseClient(object):
     """
     [AUTHENTICATED] Retrieve all pending CSRs.
     """
-    return json.loads(self._https('GET', '/csr'))
+    return json.loads(self._https('GET', '/csr').decode('utf-8'))
 
   def createCertificateSigningRequest(self, csr):
     """
@@ -241,7 +241,7 @@ class CaucaseClient(object):
       key=lambda x: x.not_valid_before,
     )[-1]
     result = []
-    for entry in json.loads(self._getCertificate('/ca.crt.json')):
+    for entry in json.loads(self._getCertificate('/ca.crt.json').decode('utf-8')):
       try:
         payload = utils.unwrap(
           entry,

caucase/test.py

@@ -1679,7 +1679,7 @@ class CaucaseTest(unittest.TestCase):
     self.maxDiff = None
     self.assertEqual(status, 200)
     self.assertEqual(header_dict['Content-Type'], 'application/hal+json')
-    self.assertEqual(json.loads(body), {
+    self.assertEqual(json.loads(body.decode('utf-8')), {
       u"_links": {
         u"getCAUHAL": {
           u"href": HATEOAS_HTTP_PREFIX + u"/cau",
@@ -1701,7 +1701,7 @@ class CaucaseTest(unittest.TestCase):
     })
     self.assertEqual(status, 200)
     self.assertEqual(header_dict['Content-Type'], 'application/hal+json')
-    self.assertEqual(json.loads(body), {
+    self.assertEqual(json.loads(body.decode('utf-8')), {
       u"_actions": {
         u"createCertificate": {
           u"href": HATEOAS_HTTPS_PREFIX + u"/cau/crt/{+crt_id}",

caucase/wsgi.py

@@ -670,7 +670,7 @@ class Application(object):
       raise BadRequest(b'Bad Content-Type')
     data = self._read(environ)
     try:
-      return json.loads(data)
+      return json.loads(data.decode('utf-8'))
     except ValueError:
       raise BadRequest(b'Invalid json')
 

[Vincent Pelletier]

Shouldn't backup production be changed to provide explicit encoding as well ?

[Vincent Pelletier]

Actually, it is using toBytes. So this should be using toUnicode for consistency.

[Jérome Perrin]

I thought that since json is encoded as utf-* (by json spec), let's decode as utf-8 and not just in ascii so that we don't break if there's non-ascii.

The python 3.6+ implementation auto-detect utf-8,16 or 32 when reading bytes: https://github.com/python/cpython/blob/955f96f6aae7c1a54d88c3f7a51c2e142ac7e4d4/Lib/json/init.py#L244 .

I don't know if we need encoding for the rest of the backup file content, but for this json header, it seems we can parse it as json and don't worry about encoding.

(edit to fix init link )

[Vincent Pelletier]

My point is just that doBackup already uses toBytes when writing to file, which encodes to ascii. So any non-ASCII char would have failed to even produce a backup file, and I see no reason to expect nor allow chars besides ASCII, at least for now. And if one day this is needed, both will need to be changed anyway.

So restoreBackup needs to use the symetric API, which is toUnicode, for consistency.

[Vincent Pelletier]

Should be using toUnicode.

[Vincent Pelletier]

Should be using toUnicode.

[Vincent Pelletier]

Should be using toUnicode.