Fix most Deprecation and Resource warnings.

On Python 3, these made the build output very hard to read (e.g., https://travis-ci.org/buildout/buildout/jobs/394026829)

On Python 3.7, these could actually break the doctests.

Closing the files should get us closer to being able to pass the tests with PyPy.

Makefile

@@ -51,4 +51,4 @@ clean:
 	rm -rf $(BUILD_DIRS) $(PYTHON_BUILD_DIR)
 
 test:
-	$(HERE)/bin/test -1 -v
+	$(HERE)/bin/test -1 -vvv -c

setup.py

@@ -18,7 +18,8 @@ import os
 from setuptools import setup
 
 def read(*rnames):
-    return open(os.path.join(os.path.dirname(__file__), *rnames)).read()
+    with open(os.path.join(os.path.dirname(__file__), *rnames)) as f:
+        return f.read()
 
 doc_intro = """
 

src/zc/buildout/bootstrap.txt

@@ -19,7 +19,9 @@ Make sure the bootstrap script actually works::
     ... [buildout]
     ... parts =
     ... ''')
-    >>> write('bootstrap.py', open(bootstrap_py).read())
+    >>> with open(bootstrap_py) as f:
+    ...     bootstrap_py_contents = f.read()
+    >>> write('bootstrap.py', bootstrap_py_contents)
     >>> print_('X'); print_(system(
     ...     zc.buildout.easy_install._safe_arg(sys.executable)+' '+
     ...     'bootstrap.py')); print_('X') # doctest: +ELLIPSIS
@@ -52,7 +54,7 @@ By default it gets the latest version::
     >>> buildout_script = join(sample_buildout, 'bin', 'buildout')
     >>> if sys.platform.startswith('win'):
     ...     buildout_script += '-script.py'
-    >>> print_(open(buildout_script).read()) # doctest: +ELLIPSIS
+    >>> with open(buildout_script) as f: print_(f.read()) # doctest: +ELLIPSIS
     #...
     sys.path[0:0] = [
       '/sample/eggs/zc.buildout-22.0.0...egg',
@@ -83,7 +85,7 @@ Now let's try with ``2.0.0``, which happens to exist::
 
 Let's make sure the generated ``buildout`` script uses it::
 
-    >>> print_(open(buildout_script).read()) # doctest: +ELLIPSIS
+    >>> with open(buildout_script) as f: print_(f.read()) # doctest: +ELLIPSIS
     #...
     sys.path[0:0] = [
       '/sample/eggs/zc.buildout-2.0.0...egg',
@@ -103,7 +105,7 @@ Now let's try with ``31.0.0``, which happens to exist::
 
 Let's make sure the generated ``buildout`` script uses it::
 
-    >>> print_(open(buildout_script).read()) # doctest: +ELLIPSIS
+    >>> with open(buildout_script) as f: print_(f.read()) # doctest: +ELLIPSIS
     #...
     sys.path[0:0] = [
       '/sample/eggs/zc.buildout-...egg',
@@ -121,7 +123,7 @@ which happens to exist::
 
 Let's make sure the generated ``buildout`` script uses it::
 
-    >>> print_(open(buildout_script).read()) # doctest: +ELLIPSIS
+    >>> with open(buildout_script) as f: print_(f.read()) # doctest: +ELLIPSIS
     #...
     sys.path[0:0] = [
       '/sample/eggs/zc.buildout-2.0.0...egg',
@@ -153,7 +155,7 @@ specify the setuptools version, and to reuse the setuptools zipfile::
     >>> os.path.exists('setuptools-32.1.0.zip')
     True
 
-    >>> print_(open(buildout_script).read()) # doctest: +ELLIPSIS
+    >>> with open(buildout_script) as f: print_(f.read()) # doctest: +ELLIPSIS
     #...
     sys.path[0:0] = [
       '/sample/eggs/zc.buildout-2.0.0...egg',

src/zc/buildout/bootstrap_cl_settings.test

@@ -20,7 +20,8 @@ Some people pass buildout settings to bootstrap.
     ... [buildout]
     ... parts =
     ... ''')
-    >>> write('bootstrap.py', open(bootstrap_py).read())
+    >>> with open(bootstrap_py) as f: bootstrap_py_contents = f.read()
+    >>> write('bootstrap.py', bootstrap_py_contents)
     >>> print_('X'); print_(system(
     ...     zc.buildout.easy_install._safe_arg(sys.executable) +
     ...     ' bootstrap.py buildout:directory=' + top +
@@ -43,7 +44,8 @@ They might do it with init, but no worries:
     >>> os.chdir(top)
     >>> mkdir(top, 'buildout')
     >>> os.chdir(top)
-    >>> write('bootstrap.py', open(bootstrap_py).read())
+    >>> with open(bootstrap_py) as f: bootstrap_py_contents = f.read()
+    >>> write('bootstrap.py', bootstrap_py_contents)
     >>> print_('X'); print_(system(
     ...     zc.buildout.easy_install._safe_arg(sys.executable) +
     ...     ' bootstrap.py buildout:directory=' + top +

src/zc/buildout/buildout.py

@@ -586,7 +586,8 @@ class Buildout(DictMixin):
             if dist.precedence == pkg_resources.DEVELOP_DIST:
                 dest = os.path.join(options['develop-eggs-directory'],
                                     name+'.egg-link')
-                open(dest, 'w').write(dist.location)
+                with open(dest, 'w') as f:
+                  f.write(dist.location)
                 entries.append(dist.location)
             else:
                 dest = os.path.join(options['eggs-directory'],
@@ -905,7 +906,7 @@ class Buildout(DictMixin):
                     setup = self._buildout_path(setup)
                     files = glob.glob(setup)
                     if not files:
-                        self._logger.warn("Couldn't develop %r (not found)",
+                        self._logger.warning("Couldn't develop %r (not found)",
                                              setup)
                     else:
                         files.sort()
@@ -1138,7 +1139,7 @@ class Buildout(DictMixin):
         if (realpath(os.path.abspath(sys.argv[0])) != should_run):
             self._logger.debug("Running %r.", realpath(sys.argv[0]))
             self._logger.debug("Local buildout is %r.", should_run)
-            self._logger.warn("Not upgrading because not running a local "
+            self._logger.warning("Not upgrading because not running a local "
                                  "buildout command.")
             return
 
@@ -1548,7 +1549,7 @@ class Options(DictMixin):
 
     _template_split = re.compile('([$]{[^}]*})').split
     _simple = re.compile('[-a-zA-Z0-9 ._]+$').match
-    _valid = re.compile('\${[-a-zA-Z0-9 ._]*:[-a-zA-Z0-9 ._]+}$').match
+    _valid = re.compile(r'\${[-a-zA-Z0-9 ._]*:[-a-zA-Z0-9 ._]+}$').match
     def _sub(self, template, seen, last=True):
         value = self._template_split(template)
         subs = []
@@ -1652,7 +1653,7 @@ class Options(DictMixin):
                     elif os.path.isfile(p):
                         os.remove(p)
                     else:
-                        self.buildout._logger.warn("Couldn't clean up %r.", p)
+                        self.buildout._logger.warning("Couldn't clean up %r.", p)
                 raise
         finally:
             self._created = None
@@ -2037,7 +2038,7 @@ def _check_for_unused_options_in_section(buildout, section):
     unused = [option for option in sorted(options._raw)
               if option not in options._data]
     if unused:
-        buildout._logger.warn("Unused options for %s: %s."
+        buildout._logger.warning("Unused options for %s: %s."
                                  % (section, ' '.join(map(repr, unused)))
         )
 

src/zc/buildout/buildout.txt

@@ -3342,3 +3342,7 @@ We see that our extension is loaded and executed::
     ext ['buildout', 'versions']
     Develop: '/sample-bootstrapped/demo'
     unload ['buildout', 'versions']
+
+..
+
+    >>> stop_server(server_url)

src/zc/buildout/configparser.py

@@ -178,7 +178,7 @@ def parse(fp, fpname, exp_globals=dict):
                     tail = tail.replace(';', '#') if tail else ''
                     # un-escape literal # and ; . Do not use a
                     # string-escape decode
-                    expr = expression.replace(r'\x23','#').replace(r'x3b', ';')
+                    expr = expression.replace(r'\x23','#').replace(r'\x3b', ';')
                     # rebuild a valid Python expression wrapped in a list
                     expr = head + expr + tail
                     # lazily populate context only expression

src/zc/buildout/download.py

@@ -23,10 +23,20 @@ try:
     from urllib.request import FancyURLopener, URLopener, urlretrieve
     from urllib.parse import urlparse
     from urllib import request
+    import warnings
 
     class PatchedURLopener(FancyURLopener):
         http_error_default = URLopener.http_error_default
 
+        def __init__(self, *args, **kwargs):
+            with warnings.catch_warnings():
+                warnings.simplefilter('ignore')
+                # This creates a DeprecationWarning on 3.6:
+                # PatchedURLopener style of invoking requests is deprecated.
+                # Use newer urlopen functions/methods.
+                # Said warning breaks our doctests.
+                super(PatchedURLopener, self).__init__(*args, **kwargs)
+
     request._urlopener = PatchedURLopener()  # Ook! Monkey patch!
 
 except ImportError:

src/zc/buildout/easy_install.py

@@ -54,7 +54,7 @@ default_index_url = os.environ.get(
 logger = logging.getLogger('zc.buildout.easy_install')
 
 url_match = re.compile('[a-z0-9+.-]+://').match
-is_source_encoding_line = re.compile('coding[:=]\s*([-\w.]+)').search
+is_source_encoding_line = re.compile(r'coding[:=]\s*([-\w.]+)').search
 # Source encoding regex from http://www.python.org/dev/peps/pep-0263/
 
 is_win32 = sys.platform == 'win32'

src/zc/buildout/repeatable.txt

@@ -397,7 +397,7 @@ at the end.
 
 The versions file now contains the extra pin:
 
-    >>> print_(open('my_versions.cfg').read()) # doctest: +ELLIPSIS
+    >>> with open('my_versions.cfg') as f: print_(f.read()) # doctest: +ELLIPSIS
     <BLANKLINE>
     ...
     # Added by buildout at YYYY-MM-DD hh:mm:ss.dddddd
@@ -437,7 +437,7 @@ printing them to the console):
 
 The versions file contains the extra pin:
 
-    >>> print_(open('my_versions.cfg').read()) # doctest: +ELLIPSIS
+    >>> with open('my_versions.cfg') as f: print_(f.read()) # doctest: +ELLIPSIS
     <BLANKLINE>
     [versions]
     ...

src/zc/buildout/rmtree.py

@@ -40,7 +40,7 @@ def rmtree (path):
     Now create a file ...
 
     >>> foo = os.path.join (d, 'foo')
-    >>> _ = open (foo, 'w').write ('huhu')
+    >>> with open (foo, 'w') as f: _ = f.write ('huhu')
     >>> bar = os.path.join (d, 'bar')
     >>> os.symlink(bar, bar)
 
@@ -107,4 +107,3 @@ def test_suite():
 
 if "__main__" == __name__:
     doctest.testmod()
-

src/zc/buildout/testing.py

@@ -112,6 +112,19 @@ def system(command, input='', with_exit_code=False):
     # https://github.com/buildout/buildout/pull/311
     # http://bugs.python.org/issue19884
     env = dict(os.environ, TERM='dumb')
+
+    # Beginning in Python 3.4, 'U' mode to open() is deprecated.
+    # Python 3.7 changes the way deprecations are shown for main
+    # modules, and introduces $PYTHONDEVMODE which turns on warnigs in
+    # more places. If that's done, this leads many of our doctests to
+    # break; some code path through executing setup.py does this, but
+    # it's not in our code. Unfortunately, normalizing this printed
+    # line away doesn't work, it just produces a blank line. We resort
+    # to turning that warning off.
+    warnings = env.get('PYTHONWARNINGS', '')
+    env['PYTHONWARNINGS'] = "ignore:'U' mode is deprecated:DeprecationWarning::," + warnings
+
+
     p = subprocess.Popen(command,
                          shell=True,
                          stdin=subprocess.PIPE,
@@ -131,6 +144,7 @@ def system(command, input='', with_exit_code=False):
         # Use the with_exit_code=True parameter when you want to test the exit
         # code of the command you're running.
         output += 'EXIT CODE: %s' % p.wait()
+    p.wait()
     return output
 
 def get(url):
@@ -293,6 +307,7 @@ def buildoutSetUp(test):
         sdist = sdist,
         bdist_egg = bdist_egg,
         start_server = start_server,
+        stop_server = stop_server,
         buildout = os.path.join(sample, 'bin', 'buildout'),
         wait_until = wait_until,
         print_ = print_,
@@ -330,6 +345,7 @@ class Handler(BaseHTTPRequestHandler):
 
     def do_GET(self):
         if '__stop__' in self.path:
+            self.__server.server_close()
             raise SystemExit
 
         def k():
@@ -401,6 +417,7 @@ def _run(tree, port):
     server_address = ('localhost', port)
     httpd = Server(tree, server_address, Handler)
     httpd.serve_forever()
+    httpd.server_close()
 
 def get_port():
     for i in range(10):
@@ -513,10 +530,10 @@ if sys.version_info > (2, ):
         re.compile('(\n?)d  __pycache__\n'), '\\1')
 else:
     normalize___pycache__ = (
-        re.compile('(\n?)-  \S+\.pyc\n'), '\\1')
+        re.compile(r'(\n?)-  \S+\.pyc\n'), '\\1')
 
 normalize_egg_py = (
-    re.compile('-py\d[.]\d(-\S+)?.egg'),
+    re.compile(r'-py\d[.]\d(-\S+)?.egg'),
     '-pyN.N.egg',
     )
 

src/zc/buildout/windows.txt

@@ -28,7 +28,7 @@ can't delete.
     ...             os.makedirs (self.location)
     ...
     ...         name = os.path.join (self.location, 'readonly.txt')
-    ...         open (name, 'w').write ('this is a read only file')
+    ...         with open (name, 'w') as f: f.write ('this is a read only file')
     ...         os.chmod(name, 256)
     ...         return ()
     ...

tox.ini

@@ -7,7 +7,7 @@ commands =
     # buildout's dev.py wants python to not have setuptools
     pip uninstall -y zc.buildout setuptools pip
     python dev.py
-    {toxinidir}/bin/test -1 -v -c
+    {toxinidir}/bin/test -1 -vvv -c
 # since we're removing setuptools we can't possibly reuse the virtualenv
 recreate = true
 # if the user has ccache installed, PATH may contain /usr/lib/ccache that has a

zc.recipe.egg_/setup.py

@@ -20,7 +20,8 @@ import os
 from setuptools import setup, find_packages
 
 def read(*rnames):
-    return open(os.path.join(os.path.dirname(__file__), *rnames)).read()
+    with open(os.path.join(os.path.dirname(__file__), *rnames)) as f:
+        return f.read()
 
 name = "zc.recipe.egg"
 setup(

zc.recipe.egg_/src/zc/recipe/egg/egg.py

@@ -120,7 +120,7 @@ class Scripts(Eggs):
             assert relative_paths == 'false'
 
     parse_entry_point = re.compile(
-        '([^=]+)=(\w+(?:[.]\w+)*):(\w+(?:[.]\w+)*)$'
+        r'([^=]+)=(\w+(?:[.]\w+)*):(\w+(?:[.]\w+)*)$'
         ).match
     def install(self):
         reqs, ws = self.working_set()

zc.recipe.egg_/src/zc/recipe/egg/tests.py

@@ -50,9 +50,9 @@ def test_suite():
                zc.buildout.tests.normalize_bang,
                zc.buildout.tests.normalize_S,
                zc.buildout.testing.not_found,
-               (re.compile('[d-]  zc.buildout(-\S+)?[.]egg(-link)?'),
+               (re.compile(r'[d-]  zc.buildout(-\S+)?[.]egg(-link)?'),
                 'zc.buildout.egg'),
-               (re.compile('[d-]  setuptools-[^-]+-'), 'setuptools-X-'),
+               (re.compile(r'[d-]  setuptools-[^-]+-'), 'setuptools-X-'),
                (re.compile(r'eggs\\\\demo'), 'eggs/demo'),
                (re.compile(r'[a-zA-Z]:\\\\foo\\\\bar'), '/foo/bar'),
                ])
@@ -66,15 +66,15 @@ def test_suite():
                zc.buildout.testing.normalize_endings,
                zc.buildout.testing.not_found,
                (re.compile('__buildout_signature__ = '
-                           'sample-\S+\s+'
-                           'zc.recipe.egg-\S+\s+'
-                           'setuptools-\S+\s+'
-                           'zc.buildout-\S+\s*'
+                           r'sample-\S+\s+'
+                           r'zc.recipe.egg-\S+\s+'
+                           r'setuptools-\S+\s+'
+                           r'zc.buildout-\S+\s*'
                            ),
                 '__buildout_signature__ = sample- zc.recipe.egg-'),
-               (re.compile('find-links = http://localhost:\d+/'),
+               (re.compile(r'find-links = http://localhost:\d+/'),
                 'find-links = http://localhost:8080/'),
-               (re.compile('index = http://localhost:\d+/index'),
+               (re.compile(r'index = http://localhost:\d+/index'),
                 'index = http://localhost:8080/index'),
                ])
             ),
@@ -87,7 +87,7 @@ def test_suite():
                 zc.buildout.testing.normalize_endings,
                 zc.buildout.testing.not_found,
                 (re.compile("(d  ((ext)?demo(needed)?|other)"
-                            "-\d[.]\d-py)\d[.]\d(-\S+)?[.]egg"),
+                            r"-\d[.]\d-py)\d[.]\d(-\S+)?[.]egg"),
                  '\\1V.V.egg'),
                 (re.compile('extdemo.c\n.+\\\\extdemo.exp\n'), ''),
                 (re.compile(