Skip to content

S
slapos.libnetworkcache
Commit eb0246c2 authored by Julien Muchembled's avatar Julien Muchembled
Browse files
Options

Add support for Python 3

parent 4dee0c16
Show whitespace changes
Inline Side-by-side
Showing with 123 additions and 99 deletions
slapos/crypto.py
View file @ eb0246c2
......@@ -11,7 +11,7 @@ FILETYPE_PEM = 1
def load_privatekey(type, buffer):
r = _tmpfile()
r.write(buffer)
r.write(buffer.encode())
r.flush()
return r
......@@ -20,7 +20,7 @@ def load_certificate(type, buffer):
r = _tmpfile()
p = Popen(("openssl", "x509", "-pubkey", "-noout"),
stdin=PIPE, stdout=r, stderr=PIPE)
err = p.communicate(buffer)[1]
err = p.communicate(buffer.encode())[1]
if p.poll():
raise Error(err)
return r
......
slapos/libnetworkcache.py
View file @ eb0246c2
......@@ -13,10 +13,7 @@
##############################################################################
import argparse
import ConfigParser
import hashlib
import httplib
import inspect
import json
import logging
import os
......@@ -26,8 +23,19 @@ import sys
import tarfile
import tempfile
import traceback
import urllib2
import urlparse
from base64 import b64encode, decodestring, encodestring
try:
import configparser
from http.client import HTTPConnection, HTTPSConnection
from urllib.error import HTTPError
from urllib.parse import urlsplit
from urllib.request import urlopen
basestring = bytes, str
except ImportError:
import ConfigParser as configparser
from httplib import HTTPConnection, HTTPSConnection
from urllib2 import HTTPError, urlopen
from urlparse import urlsplit
try:
from OpenSSL import crypto
except ImportError:
......@@ -52,17 +60,21 @@ class short_exc_info(tuple):
l += traceback.format_exception_only(t, v)
return ''.join(l).rstrip()
def byteify(input):
def strify(input):
'''Transform every unicode string inside a list or dict into normal strings. '''
if isinstance(input, dict):
return dict([(byteify(key), byteify(value)) for key, value in input.iteritems()])
return dict((strify(key), strify(value)) for key, value in input.items())
elif isinstance(input, list):
return [byteify(element) for element in input]
return map(strify, input)
elif isinstance(input, unicode):
return input.encode('utf-8')
else:
return input
try:
unicode
except NameError:
def strify(input):
return input
class NetworkcacheClient(object):
'''
......@@ -100,7 +112,7 @@ class NetworkcacheClient(object):
def __new_init(self, config, signature_certificate_list=None):
if not hasattr(config, 'get'):
parser = ConfigParser.SafeConfigParser()
parser = configparser.SafeConfigParser()
parser.readfp(config)
config = dict(parser.items('networkcache'))
self.config = config
......@@ -147,12 +159,12 @@ class NetworkcacheClient(object):
method = 'PUT' if name else 'POST'
url = self.config['upload-%s-url' % where]
timeout = UPLOAD_TIMEOUT
parsed_url = urlparse.urlsplit(url.rstrip('/') + ('/' + name if name else ''))
parsed_url = urlsplit(url.rstrip('/') + ('/' + name if name else ''))
if not headers:
headers = {}
if parsed_url.username:
headers['Authorization'] = 'Basic %s' % ('%s:%s' % (
parsed_url.username, parsed_url.password)).encode('base64').strip()
headers['Authorization'] = 'Basic ' + b64encode('%s:%s' % (
parsed_url.username, parsed_url.password))
headers["Connection"] = "close"
connection_kw = {
'host': parsed_url.hostname,
......@@ -162,23 +174,21 @@ class NetworkcacheClient(object):
if parsed_url.scheme == 'https':
connection_kw['cert_file'] = self.config['sha%s-cert-file' % where]
connection_kw['key_file'] = self.config['sha%s-key-file' % where]
if 'context' in inspect.getargspec(
httplib.HTTPSConnection.__init__).args:
if hasattr(ssl, 'create_default_context'):
connection_kw['context'] = ssl.create_default_context(
cafile=self.config.get('sha%s-ca-file' % where)
)
connection = httplib.HTTPSConnection(**connection_kw)
connection = HTTPSConnection(**connection_kw)
else:
connection = httplib.HTTPConnection(**connection_kw)
connection = HTTPConnection(**connection_kw)
try:
connection.request(method, parsed_url.path, data, headers)
r = connection.getresponse()
if 200 <= r.status < 300:
return r
except:
finally:
connection.close()
raise
raise urllib2.HTTPError(url, r.status, r.reason, r.msg, r.fp)
raise HTTPError(url, r.status, r.reason, r.msg, r.fp)
@staticmethod
def archive(path):
......@@ -216,7 +226,7 @@ class NetworkcacheClient(object):
try:
try:
file_descriptor.seek(0)
except StandardError:
except Exception:
f = tempfile.TemporaryFile()
while 1:
data = file_descriptor.read(65536)
......@@ -234,14 +244,14 @@ class NetworkcacheClient(object):
sha512sum = sha512sum.hexdigest()
try:
self._request('cache', sha512sum).close()
except urllib2.HTTPError:
except HTTPError:
size = file_descriptor.tell()
file_descriptor.seek(0)
result = self._request('cache', data=file_descriptor, headers={
'Content-Length': str(size),
'Content-Type': 'application/octet-stream'})
data = result.read()
if result.status != 201 or data != sha512sum:
if result.status != 201 or data != sha512sum.encode():
raise NetworkcacheException(
'Failed to upload data to SHACACHE Server.'
' Response code: %s. Response data: %s' % (result.status, data))
......@@ -266,7 +276,7 @@ class NetworkcacheClient(object):
def index(self, key, **kw):
data = json.dumps(kw)
data = [data, self._getSignatureString(data)]
data = [data, self._getSignatureString(data.encode())]
result = self._request('dir', key, json.dumps(data), {
'Content-Type': 'application/json'})
if result.status != 201:
......@@ -287,19 +297,19 @@ class NetworkcacheClient(object):
data_list = self.select_generic(key, self.signature_certificate_list)
for information_json, signature in data_list:
try:
information_dict = byteify(json.loads(information_json))
information_dict = strify(json.loads(information_json))
except Exception:
logger.info('Failed to parse json-in-json response (%r)',
information_json)
continue
try:
len(information_dict['sha512'])
except StandardError:
except Exception:
logger.info('Bad or missing sha512 in directory response (%r)',
information_dict)
continue
if required_key_test(information_dict):
for k, v in wanted_metadata_dict.iteritems():
for k, v in wanted_metadata_dict.items():
if information_dict.get(k) != v:
break
else:
......@@ -310,7 +320,7 @@ class NetworkcacheClient(object):
'''
data = self._request('dir', key).read()
try:
data_list = json.loads(data)
data_list = json.loads(data.decode())
except Exception:
raise NetworkcacheException('Failed to parse json response (%r)' % data)
if filter:
......@@ -323,7 +333,9 @@ class NetworkcacheClient(object):
Return the signature based on certification file.
"""
k = self.signature_private_key
return '' if k is None else crypto.sign(k, content, 'sha1').encode('base64')
if k is None:
return ''
return encodestring(crypto.sign(k, content, 'sha1')).decode()
def _verifySignatureInCertificateList(self, content, signature_string):
"""
......@@ -331,8 +343,8 @@ class NetworkcacheClient(object):
find any.
"""
if signature_string:
signature = signature_string.decode('base64')
content = str(content)
signature = decodestring(signature_string.encode())
content = content.encode()
for certificate in self.signature_certificate_list:
try:
crypto.verify(certificate, signature, content, 'sha1')
......@@ -371,12 +383,12 @@ def cmd_upload(*args):
f = None
try:
if args.file:
f = open(args.file)
f = open(args.file, 'rb')
elif os.path.isdir(args.url):
f = nc.archive(args.url)
else:
f = urllib2.urlopen(args.url)
urlmd5 = hashlib.md5(args.url).hexdigest()
f = urlopen(args.url)
urlmd5 = str(hashlib.md5(args.url.encode()).hexdigest())
nc.upload(f, args.prefix_key + urlmd5 + args.suffix_key, urlmd5=urlmd5,
file_name=os.path.basename(args.url) or "file",
**dict(x.split('=', 1) for x in args.meta))
......@@ -387,5 +399,8 @@ def cmd_download(*args):
parser = _newArgumentParser("URL of data to download.")
args = parser.parse_args(args or sys.argv[1:])
nc = NetworkcacheClient(args.config)
key = args.prefix_key + hashlib.md5(args.url).hexdigest() + args.suffix_key
shutil.copyfileobj(nc.download(nc.select(key).next()['sha512']), sys.stdout)
urlmd5 = str(hashlib.md5(args.url.encode()).hexdigest())
key = args.prefix_key + urlmd5 + args.suffix_key
f = sys.stdout
shutil.copyfileobj(nc.download(next(nc.select(key))['sha512']),
getattr(f, 'buffer', f))
slapos/libnetworkcachetests.py
View file @ eb0246c2
import BaseHTTPServer
import datetime
import errno
import hashlib
import httplib
import json
import logging
import logging.handlers
import os
import urllib2
import random
import shutil
import ssl
......@@ -20,21 +17,29 @@ import unittest
import slapos.libnetworkcache
import slapos.signature
import sys
from cStringIO import StringIO
from io import BytesIO
try:
from http.server import BaseHTTPRequestHandler, HTTPServer
from http.client import HTTPConnection, NOT_FOUND
from urllib.error import HTTPError
except ImportError:
from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
from httplib import HTTPConnection, NOT_FOUND
from urllib2 import HTTPError
logging.basicConfig()
class NCHandler(BaseHTTPServer.BaseHTTPRequestHandler):
class NCHandler(BaseHTTPRequestHandler):
def __init__(self, request, address, server):
self.__server = server
self.tree = server.tree
BaseHTTPServer.BaseHTTPRequestHandler.__init__(
BaseHTTPRequestHandler.__init__(
self, request, address, server)
def handle(self):
try:
BaseHTTPServer.BaseHTTPRequestHandler.handle(self)
BaseHTTPRequestHandler.handle(self)
except Exception:
traceback.print_exc()
raise
......@@ -49,80 +54,79 @@ class NCHandler(BaseHTTPServer.BaseHTTPRequestHandler):
and
os.path.exists(path)
):
self.send_response(404, 'Not Found')
self.send_error(404)
return
self.send_response(200)
out = open(path, 'rb').read()
with open(path, 'rb') as f:
out = f.read()
self.send_header('Content-Length', len(out))
self.end_headers()
self.wfile.write(out)
def do_PUT(self):
assert 'shadir' in self.path
assert self.headers.getheader('content-type') == 'application/json'
assert self.headers.get('content-type') == 'application/json'
path = os.path.abspath(os.path.join(self.tree, *self.path.split('/')))
if not os.path.exists(os.path.dirname(path)):
os.makedirs(os.path.dirname(path))
data = self.rfile.read(int(self.headers.getheader('content-length')))
data = self.rfile.read(int(self.headers.get('content-length')))
cksum = hashlib.sha512(data).hexdigest()
if os.path.exists(path):
with open(path, 'rb') as f:
json_data_list = json.loads(f.read().strip())
with open(path) as f:
json_data_list = json.load(f)
else:
json_data_list = []
json_data_list.append(json.loads(data))
data = json.dumps(json_data_list)
json_data_list.append(json.loads(data.decode()))
open(path, 'wb').write(data)
with open(path, 'w') as f:
json.dump(json_data_list, f)
self.send_response(201)
self.send_header('Content-Length', str(len(cksum)))
self.send_header('Content-Type', 'text/html')
self.end_headers()
self.wfile.write(cksum)
return
self.wfile.write(cksum.encode())
def do_POST(self):
assert 'shacache' in self.path
assert self.headers.getheader('content-type') == 'application/octet-stream'
assert self.headers.get('content-type') == 'application/octet-stream'
path = os.path.abspath(os.path.join(self.tree, *self.path.split('/')))
if not os.path.exists(path):
os.makedirs(path)
data = self.rfile.read(int(self.headers.getheader('content-length')))
data = self.rfile.read(int(self.headers.get('content-length')))
cksum = hashlib.sha512(data).hexdigest()
path = os.path.join(path, cksum)
# Although real server would accept the request,
# clients should avoid uploading same content twice.
assert not os.path.exists(path)
open(path, 'wb').write(data)
with open(path, 'wb') as f:
f.write(data)
self.send_response(201)
self.send_header('Content-Length', str(len(cksum)))
self.send_header('Content-Type', 'text/html')
self.end_headers()
self.wfile.write(cksum)
return
self.wfile.write(cksum.encode())
class NCHandlerPOST200(NCHandler):
def do_POST(self):
self.send_response(200)
return
self.end_headers()
class NCHandlerReturnWrong(NCHandler):
def do_POST(self):
cksum = 'incorrect'
cksum = b'incorrect'
self.send_response(201)
self.send_header('Content-Length', str(len(cksum)))
self.send_header('Content-Type', 'text/html')
self.end_headers()
self.wfile.write(cksum)
return
class Server(BaseHTTPServer.HTTPServer):
class Server(HTTPServer):
def __init__(self, tree, *args):
BaseHTTPServer.HTTPServer.__init__(self, *args)
HTTPServer.__init__(self, *args)
self.tree = os.path.abspath(tree)
__run = True
......@@ -158,7 +162,7 @@ class OfflineTest(unittest.TestCase):
def test_upload_offline(self):
nc = slapos.libnetworkcache.NetworkcacheClient(self.shacache_url,
self.shadir_url)
self.assertRaises(IOError, nc.upload, StringIO())
self.assertRaises(IOError, nc.upload, BytesIO())
class OnlineMixin:
......@@ -174,18 +178,21 @@ class OnlineMixin:
if not 'TEST_SHA_CACHE' in os.environ and not 'TEST_SHA_DIR' in os.environ:
self.tree = tempfile.mkdtemp()
self.thread = Server.run(self.tree, (self.host, self.port), self.handler)
self.test_string = str(random.random())
self.test_data = StringIO(self.test_string)
self.test_string = str(random.random()).encode()
self.test_data = BytesIO(self.test_string)
self.test_shasum = hashlib.sha512(self.test_string).hexdigest()
self.handler = logging.handlers.BufferingHandler(float('inf'))
slapos.libnetworkcache.logger.addHandler(self.handler)
def tearDown(self):
if not 'TEST_SHA_CACHE' in os.environ and not 'TEST_SHA_DIR' in os.environ:
conn = HTTPConnection(self.host, self.port)
try:
httplib.HTTPConnection(self.host, self.port).request('KILL', '/')
conn.request('KILL', '/')
except Exception:
pass
finally:
conn.close()
if self.thread is not None:
self.thread.join()
......@@ -204,7 +211,7 @@ class OnlineMixin:
def select(self, nc, key, *args):
try:
return nc.download(nc.select(key).next()['sha512'])
return nc.download(next(nc.select(key))['sha512'])
except StopIteration:
for msg in args:
self.assertLog(msg)
......@@ -369,8 +376,8 @@ class OnlineTest(OnlineMixin, unittest.TestCase):
key = 'somekey' + str(random.random())
nc.upload(self.test_data, key, urlmd5=urlmd5, file_name='my file')
with nc:
nc.select('key_another_key' + str(random.random())).next()
self.assertRaised(urllib2.HTTPError)
next(nc.select('key_another_key' + str(random.random())))
self.assertRaised(HTTPError)
def test_upload_shadir_no_filename(self):
"""Check scenario with shadir used, but not filename passed"""
......@@ -403,14 +410,14 @@ class OnlineTest(OnlineMixin, unittest.TestCase):
try:
nc.download(self.test_shasum)
self.fail("HTTPError not raised")
except urllib2.HTTPError, error:
self.assertEqual(error.code, httplib.NOT_FOUND)
except HTTPError as error:
self.assertEqual(error.code, NOT_FOUND)
def test_select_signed_content(self):
key = 'somekey' + str(random.random())
urlmd5 = str(random.random())
file_name = 'my file'
key_file = tempfile.NamedTemporaryFile()
key_file = tempfile.NamedTemporaryFile('w+')
key_file.write(self.key)
key_file.flush()
signed_nc = slapos.libnetworkcache.NetworkcacheClient(
......@@ -423,7 +430,7 @@ class OnlineTest(OnlineMixin, unittest.TestCase):
key = 'somekey' + str(random.random())
urlmd5 = str(random.random())
file_name = 'my file'
key_file = tempfile.NamedTemporaryFile()
key_file = tempfile.NamedTemporaryFile('w+')
key_file.write(self.key)
key_file.flush()
signed_nc = slapos.libnetworkcache.NetworkcacheClient(
......@@ -437,7 +444,7 @@ class OnlineTest(OnlineMixin, unittest.TestCase):
key = 'somekey' + str(random.random())
urlmd5 = str(random.random())
file_name = 'my file'
key_file = tempfile.NamedTemporaryFile()
key_file = tempfile.NamedTemporaryFile('w+')
key_file.write(self.key)
key_file.flush()
signed_nc = slapos.libnetworkcache.NetworkcacheClient(
......@@ -455,9 +462,10 @@ class OnlineTest(OnlineMixin, unittest.TestCase):
self.shacache, self.shadir)
self.assertEqual(self.test_shasum, nc.upload(self.test_data,
key, urlmd5=urlmd5, file_name=file_name))
f = os.path.join(self.tree, 'shadir', key)
path = os.path.join(self.tree, 'shadir', key)
# now remove the entry from shacache
open(f, 'w').write(json.dumps([]))
with open(path, 'w') as f:
json.dump([], f)
self.assertEqual(self.select(nc, key), None)
def test_select_no_json_response(self):
......@@ -472,7 +480,7 @@ class OnlineTest(OnlineMixin, unittest.TestCase):
# now remove the entry from shacache
f.write('This is not a json.')
with nc:
nc.select(key).next()
next(nc.select(key))
self.assertLog('Failed to parse json response')
def test_select_json_no_in_json_response(self):
......@@ -505,7 +513,7 @@ class OnlineTest(OnlineMixin, unittest.TestCase):
key = 'somekey' + str(random.random())
urlmd5 = str(random.random())
file_name = 'my file'
key_file = tempfile.NamedTemporaryFile()
key_file = tempfile.NamedTemporaryFile('w+')
key_file.write(self.key)
key_file.flush()
signed_nc = slapos.libnetworkcache.NetworkcacheClient(
......@@ -513,9 +521,9 @@ class OnlineTest(OnlineMixin, unittest.TestCase):
signed_nc.upload(self.test_data, key, urlmd5=urlmd5, file_name=file_name)
# Hacker entered the server...
f = os.path.join(self.tree, 'shadir', key)
original_data = open(f).read()
original_json = json.loads(original_data)
path = os.path.join(self.tree, 'shadir', key)
with open(path) as f:
original_json = json.load(f)
hacked_entry_json = json.loads(original_json[0][0])
# ...he modifies something...
hacked_entry_json['file'] = 'hacked'
......@@ -524,11 +532,12 @@ class OnlineTest(OnlineMixin, unittest.TestCase):
hacked_json[0][0] = json.dumps(hacked_entry_json)
# ...but as he has no access to key, no way to sign data..
# hacked_json[0][1] is still a good key
open(f, 'w').write(json.dumps(hacked_json))
with open(path, 'w') as f:
json.dump(hacked_json, f)
self.assertEqual(self.select(signed_nc, key), None)
def test_DirectoryNotFound_non_trustable_entry(self):
key_file = tempfile.NamedTemporaryFile()
key_file = tempfile.NamedTemporaryFile('w+')
key_file.write(self.key)
key_file.flush()
......@@ -573,7 +582,7 @@ class OnlineTestSSLServer(OnlineMixin, unittest.TestCase):
nc = slapos.libnetworkcache.NetworkcacheClient(self.shacache, self.shadir)
try:
nc.upload(self.test_data)
except ssl.SSLError, e:
except ssl.SSLError as e:
self.assertTrue('alert handshake failure' in str(e))
def test_upload_to_ssl_auth(self):
......@@ -589,7 +598,7 @@ class OnlineTestSSLServer(OnlineMixin, unittest.TestCase):
try:
nc.upload(self.test_data, key=str(random.random()),
file_name=str(random.random()), urlmd5=str(random.random()))
except ssl.SSLError, e:
except ssl.SSLError as e:
self.assertTrue('alert handshake failure' in str(e))
def test_upload_with_key_with_ssl_auth(self):
......@@ -641,11 +650,11 @@ class GenerateSignatureScriptTest(unittest.TestCase):
self.common_name)
today = datetime.date.today()
result = subprocess.check_output(['openssl', 'x509', '-noout', '-subject',
'-in', self.certificate])
'-in', self.certificate], universal_newlines=True)
self.assertEqual('subject= /CN=%s' % self.common_name, result.strip())
result = subprocess.check_output(['openssl', 'x509', '-noout', '-enddate',
'-in', self.certificate])
self.assertTrue(' %s ' % (today.year + 100) in result)
'-in', self.certificate], universal_newlines=True)
self.assertIn(' %s ' % (today.year + 100), result)
def test_generate_key_exists(self):
with tempfile.NamedTemporaryFile() as key:
......
slapos/networkcachehelper.py
View file @ eb0246c2
......@@ -51,9 +51,6 @@ def __upload_network_cached(dir_url, cache_url,
) as nc:
return nc.upload(file_descriptor, directory_key, **metadata_dict)
# BBB: slapos.buildout (1.6.0-dev-SlapOS-011) imports it without using it
helper_upload_network_cached_from_file = None
def helper_upload_network_cached_from_directory(dir_url, cache_url,
path, directory_key, metadata_dict,
signature_private_key_file, shacache_ca_file, shacache_cert_file,
......
slapos/signature.py
View file @ eb0246c2
......@@ -12,11 +12,14 @@
#
##############################################################################
import ConfigParser
import argparse
import os
import subprocess
import sys
try:
import configparser
except ImportError:
import ConfigParser as configparser
def generateCertificate(certificate_file, key_file, common_name):
if os.path.lexists(certificate_file):
......@@ -26,8 +29,8 @@ def generateCertificate(certificate_file, key_file, common_name):
raise ValueError("Key %r exists, will not overwrite." %
key_file)
print 'Generating certificate for %r (key: %r, certficate: %r)' % (
common_name, key_file, certificate_file)
print('Generating certificate for %r (key: %r, certficate: %r)' % (
common_name, key_file, certificate_file))
subj = '/CN=%s' % common_name
subprocess.check_call(["openssl", "req", "-x509", "-nodes", "-days", "36500",
"-subj", subj, "-newkey", "rsa:1024", "-keyout", key_file, "-out",
......@@ -39,7 +42,7 @@ def run(*args):
parser.add_argument('slapos_config', type=argparse.FileType('r'),
help='SlapOS configuration file.')
config = ConfigParser.SafeConfigParser()
config = configparser.SafeConfigParser()
option = parser.parse_args(list(args) or sys.argv[1:])
config.readfp(option.slapos_config)
generateCertificate(config.get('networkcache', 'signature-certificate-file'),
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7