Commit 00c26491 authored by Julien Muchembled's avatar Julien Muchembled Committed by Xavier Thompson

[feat] Extend Download API to use an alternate URL as fallback

This retries with the alternate URL in case of HTTPError with the main
one.

Used by slapos.recipe.build:download* and slapos.recipe.cmmi recipes.
parent 648fcdd6
...@@ -20,13 +20,14 @@ except ImportError: ...@@ -20,13 +20,14 @@ except ImportError:
try: try:
# Python 3 # Python 3
from urllib.error import HTTPError
from urllib.request import Request, urlopen from urllib.request import Request, urlopen
from urllib.parse import urlparse, urlunparse from urllib.parse import urlparse, urlunparse
except ImportError: except ImportError:
# Python 2 # Python 2
from urlparse import urlparse from urlparse import urlparse
from urlparse import urlunparse from urlparse import urlunparse
from urllib2 import Request, urlopen from urllib2 import Request, urlopen, HTTPError
from zc.buildout.easy_install import realpath from zc.buildout.easy_install import realpath
from base64 import b64encode from base64 import b64encode
...@@ -104,7 +105,7 @@ class Download(object): ...@@ -104,7 +105,7 @@ class Download(object):
""" """
return self.download_cached if self.cache else self.download return self.download_cached if self.cache else self.download
def download_cached(self, url, md5sum=None, path=None): def download_cached(self, url, md5sum=None, path=None, alternate_url=None):
"""Download a file from a URL using the cache. """Download a file from a URL using the cache.
This method assumes that the cache has been configured. This method assumes that the cache has been configured.
...@@ -137,7 +138,7 @@ class Download(object): ...@@ -137,7 +138,7 @@ class Download(object):
# Don't download directly to cached_path to minimize # Don't download directly to cached_path to minimize
# the probability to alter old data if download fails. # the probability to alter old data if download fails.
try: try:
path, is_temp = self.download(url, md5sum, path) path, is_temp = self.download(url, md5sum, path, alternate_url)
except ChecksumError: except ChecksumError:
raise raise
except Exception: except Exception:
...@@ -160,11 +161,11 @@ class Download(object): ...@@ -160,11 +161,11 @@ class Download(object):
else: else:
self.logger.debug('Cache miss; will cache %s as %s' % self.logger.debug('Cache miss; will cache %s as %s' %
(url, cached_path)) (url, cached_path))
self.download(url, md5sum, cached_path) self.download(url, md5sum, cached_path, alternate_url)
return locate_at(cached_path, path), False return locate_at(cached_path, path), False
def download(self, url, md5sum=None, path=None): def download(self, url, md5sum=None, path=None, alternate_url=None):
"""Download a file from a URL to a given or temporary path. """Download a file from a URL to a given or temporary path.
An online resource is always downloaded to a temporary file and moved An online resource is always downloaded to a temporary file and moved
...@@ -193,20 +194,29 @@ class Download(object): ...@@ -193,20 +194,29 @@ class Download(object):
"Couldn't download %r in offline mode." % url) "Couldn't download %r in offline mode." % url)
self.logger.info('Downloading %s' % url) self.logger.info('Downloading %s' % url)
download_url = url
tmp_path = path tmp_path = path
cleanup = True cleanup = True
try: try:
if not path: if not path:
handle, tmp_path = tempfile.mkstemp(prefix='buildout-') handle, tmp_path = tempfile.mkstemp(prefix='buildout-')
os.close(handle) os.close(handle)
tmp_path, headers = self.urlretrieve(url, tmp_path) try:
tmp_path, headers = self.urlretrieve(url, tmp_path)
except HTTPError:
if not alternate_url:
raise
self.logger.info('using alternate URL: %s', alternate_url)
download_url = alternate_url
tmp_path, headers = self.urlretrieve(
alternate_url, tmp_path)
if not check_md5sum(tmp_path, md5sum): if not check_md5sum(tmp_path, md5sum):
raise ChecksumError( raise ChecksumError(
'MD5 checksum mismatch downloading %r' % url) 'MD5 checksum mismatch downloading %r' % download_url)
cleanup = False cleanup = False
except IOError as e: except IOError as e:
raise zc.buildout.UserError("Error downloading %s: %s" raise zc.buildout.UserError("Error downloading %s: %s"
% (url, e)) % (download_url, e))
finally: finally:
if cleanup and tmp_path: if cleanup and tmp_path:
remove(tmp_path) remove(tmp_path)
......
...@@ -63,6 +63,32 @@ When trying to access a file that doesn't exist, we'll get an exception: ...@@ -63,6 +63,32 @@ When trying to access a file that doesn't exist, we'll get an exception:
... else: print_('woops') ... else: print_('woops')
download error download error
An alternate URL can be used in case of HTTPError with the main one.
Useful when a version of a resource can only be downloaded with a temporary
URL as long as it's the last version, and this version is then moved to a
permanent place when a newer version is released. In such case, when using
a cache, it's important that the main URL (`url`) is always used as cache key.
And `alternate_url` shall be the temporary URL.
>>> path, is_temp = download(server_url+'not-there',
... alternate_url=server_url+'foo.txt')
>>> cat(path)
This is a foo text.
>>> is_temp
True
>>> remove(path)
The main URL is tried first:
>>> write(server_data, 'other.txt', 'This is some other text.')
>>> path, is_temp = download(server_url+'other.txt',
... alternate_url=server_url+'foo.txt')
>>> cat(path)
This is some other text.
>>> is_temp
True
>>> remove(path)
Downloading a local file doesn't produce a temporary file but simply returns Downloading a local file doesn't produce a temporary file but simply returns
the local file itself: the local file itself:
......
...@@ -492,9 +492,9 @@ a better solution would re-use the logging already done by the utility.) ...@@ -492,9 +492,9 @@ a better solution would re-use the logging already done by the utility.)
>>> import zc.buildout >>> import zc.buildout
>>> old_download = zc.buildout.download.Download.download >>> old_download = zc.buildout.download.Download.download
>>> def wrapper_download(self, url, md5sum=None, path=None): >>> def wrapper_download(self, url, *args, **kw):
... print_("The URL %s was downloaded." % url) ... print_("The URL %s was downloaded." % url)
... return old_download(url, md5sum, path) ... return old_download(url, *args, **kw)
>>> zc.buildout.download.Download.download = wrapper_download >>> zc.buildout.download.Download.download = wrapper_download
>>> zc.buildout.buildout.main([]) >>> zc.buildout.buildout.main([])
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment