download: add netrc file support

Like for URL that contain credentials, we still skip auth challenge
because it's faster and:
- we only support one auth scheme (basic)
- netrc provides no way to specify realms, which seem anyway to be
  less and less used (https://stackoverflow.com/q/69303610 reports
  that recent browsers don't display them anymore)

See merge request nexedi/slapos.buildout!25

src/zc/buildout/download.py

@@ -31,7 +31,9 @@ except ImportError:
 from zc.buildout.easy_install import realpath
 from base64 import b64encode
 from contextlib import closing
+import errno
 import logging
+import netrc
 import os
 import os.path
 import re
@@ -42,6 +44,23 @@ from . import bytes2str, str2bytes
 from .rmtree import rmtree
 
 
+class netrc(netrc.netrc):
+
+    def __init__(*args):
+        pass
+
+    def authenticators(self, host):
+        self.__class__, = self.__class__.__bases__
+        try:
+            self.__init__()
+        except IOError as e:
+            if e.errno != errno.ENOENT:
+                raise
+            self.__init__(os.devnull)
+        return self.authenticators(host)
+
+netrc = netrc()
+
 class ChecksumError(zc.buildout.UserError):
     pass
 
@@ -269,14 +288,20 @@ class Download(object):
     def urlretrieve(self, url, tmp_path):
         parsed_url = urlparse(url)
         req = url
-        if parsed_url.scheme in ('http', 'https'):
+        while parsed_url.scheme in ('http', 'https'): # not a loop
             auth_host = parsed_url.netloc.rsplit('@', 1)
             if len(auth_host) > 1:
                 auth = auth_host[0]
                 url = parsed_url._replace(netloc=auth_host[1]).geturl()
-                req = Request(url)
-                req.add_header("Authorization",
-                               "Basic " + bytes2str(b64encode(str2bytes(auth))))
+            else:
+                auth = netrc.authenticators(parsed_url.hostname)
+                if not auth:
+                    break
+                auth = '{0}:{2}'.format(*auth)
+            req = Request(url)
+            req.add_header("Authorization",
+                           "Basic " + bytes2str(b64encode(str2bytes(auth))))
+            break
         with closing(urlopen(req)) as src:
             with open(tmp_path, 'wb') as dst:
                 shutil.copyfileobj(src, dst)

src/zc/buildout/download.txt

@@ -166,6 +166,24 @@ True
 Traceback (most recent call last):
 UserError: Error downloading ...: HTTP Error 403: Forbidden
 
+... with netrc:
+
+>>> url = server_url + 'private/foo:bar'
+>>> download(url)
+Traceback (most recent call last):
+UserError: Error downloading ...: HTTP Error 403: Forbidden
+>>> import os, zc.buildout.download
+>>> old_home = os.environ['HOME']
+>>> home = os.environ['HOME'] = tmpdir('test-netrc')
+>>> netrc = join(home, '.netrc')
+>>> write(netrc, 'machine localhost login foo password bar')
+>>> os.chmod(netrc, 0o600)
+>>> zc.buildout.download.netrc.__init__()
+>>> path, is_temp = download(url)
+>>> is_temp; remove(path)
+True
+>>> os.environ['HOME'] = old_home
+
 Downloading using the download cache
 ------------------------------------