Merge pull request #251 from buildout/lelit-171-fix-fixes

Slightly improved version of lelit's #172

CHANGES.rst

@@ -40,6 +40,14 @@ Change History
   configuration so that tests run much quicker so that buildout is easier and
   quicker to develop.
 
+- ``download-cache``, ``eggs-directory`` and ``extends-cache`` are now
+  automatically created if their parent directory exists. Also they can be
+  relative directories (relative to the location of the buildout config file
+  that defines them). Also they can now be in the form ``~/subdir``, with the
+  usual convention that the ``~`` char means the home directory of the user
+  running buildout.
+  [lelit]
+
 2.3.1 (2014-12-16)
 ==================
 

DEVELOPERS.txt

@@ -7,6 +7,10 @@ When you're developing buildout itself, you need to know two things:
   will find your already-installed setuptools, leading to test differences
   when setuptools' presence is explicitly tested.
 
+- Also the presence of ``~/.buildout/default.cfg`` may interfere with the
+  tests so you may want to temporarily rename it so that it does not get in
+  the way.
+
 - Don't bootstrap with ``python bootstrap/bootstrap.py`` but with ``python
   dev.py``.
 

src/zc/buildout/bootstrap.txt

@@ -24,9 +24,9 @@ Make sure the bootstrap script actually works::
     ...     zc.buildout.easy_install._safe_arg(sys.executable)+' '+
     ...     'bootstrap.py')); print_('X') # doctest: +ELLIPSIS
     X...
+    Creating directory '/sample/eggs'.
     Creating directory '/sample/bin'.
     Creating directory '/sample/parts'.
-    Creating directory '/sample/eggs'.
     Creating directory '/sample/develop-eggs'.
     Generated script '/sample/bin/buildout'.
     ...

src/zc/buildout/bootstrap_cl_settings.test

@@ -27,9 +27,9 @@ Some people pass buildout settings to bootstrap.
     ...     ' -c'+join('buildout', 'buildout.cfg')
     ...    )); print_('X') # doctest: +ELLIPSIS
     X...
+    Creating directory '/top/eggs'.
     Creating directory '/top/bin'.
     Creating directory '/top/parts'.
-    Creating directory '/top/eggs'.
     Creating directory '/top/develop-eggs'.
     Generated script '/top/bin/buildout'.
     ...
@@ -52,9 +52,9 @@ They might do it with init, but no worries:
     ...    )); print_('X') # doctest: +ELLIPSIS
     X...
     Creating '/top/buildout/buildout.cfg'.
+    Creating directory '/top/eggs'.
     Creating directory '/top/bin'.
     Creating directory '/top/parts'.
-    Creating directory '/top/eggs'.
     Creating directory '/top/develop-eggs'.
     Generated script '/top/bin/buildout'.
     ...

src/zc/buildout/buildout.py

@@ -253,6 +253,37 @@ class Buildout(DictMixin):
                  if k not in versions
                  ))
 
+        # Absolutize some particular directory, handling also the ~/foo form,
+        # and considering the location of the configuration file that generated
+        # the setting as the base path, falling back to the main configuration
+        # file location
+        for name in ('download-cache', 'eggs-directory', 'extends-cache'):
+            if name in data['buildout']:
+                origdir, src = data['buildout'][name]
+                if '${' in origdir:
+                    continue
+                if not os.path.isabs(origdir):
+                    if src in ('DEFAULT_VALUE',
+                               'COMPUTED_VALUE',
+                               'COMMAND_LINE_VALUE'):
+                        if 'directory' in data['buildout']:
+                            basedir = data['buildout']['directory'][0]
+                        else:
+                            basedir = self._buildout_dir
+                    else:
+                        if _isurl(src):
+                            raise zc.buildout.UserError(
+                                'Setting "%s" to a non absolute location ("%s") '
+                                'within a\n'
+                                'remote configuration file ("%s") is ambiguous.' % (
+                                    name, origdir, src))
+                        basedir = os.path.dirname(src)
+                    absdir = os.path.expanduser(origdir)
+                    if not os.path.isabs(absdir):
+                        absdir = os.path.join(basedir, absdir)
+                    absdir = os.path.abspath(absdir)
+                    data['buildout'][name] = (absdir, src)
+
         self._annotated = copy.deepcopy(data)
         self._raw = _unannotate(data)
         self._data = {}
@@ -349,18 +380,22 @@ class Buildout(DictMixin):
                                                    self.update_versions_file)
 
         download_cache = options.get('download-cache')
+        extends_cache = options.get('extends-cache')
+        eggs_cache = options.get('eggs-directory')
+        for cache in [download_cache, extends_cache, eggs_cache]:
+            if cache:
+                cache = os.path.join(options['directory'], cache)
+                if not os.path.exists(cache):
+                    # Note: os.mkdir only creates the dir if the parent
+                    # exists. This is the way we want it.
+                    self._logger.info('Creating directory %r.', cache)
+                    os.mkdir(cache)
+
         if download_cache:
-            download_cache = os.path.join(options['directory'], download_cache)
-            if not os.path.isdir(download_cache):
-                raise zc.buildout.UserError(
-                    'The specified download cache:\n'
-                    '%r\n'
-                    "Doesn't exist.\n"
-                    % download_cache)
+            # Actually, we want to use a subdirectory in there called 'dist'.
             download_cache = os.path.join(download_cache, 'dist')
-            if not os.path.isdir(download_cache):
+            if not os.path.exists(download_cache):
                 os.mkdir(download_cache)
-
             zc.buildout.easy_install.download_cache(download_cache)
 
         if bool_option(options, 'install-from-cache'):
@@ -376,11 +411,6 @@ class Buildout(DictMixin):
         for name in _buildout_default_options:
             options[name]
 
-        # Do the same for extends-cache which is not among the defaults but
-        # wasn't recognized as having been used since it was used before
-        # tracking was turned on.
-        options.get('extends-cache')
-
         os.chdir(options['directory'])
 
     def _buildout_path(self, name):
@@ -983,9 +1013,6 @@ class Buildout(DictMixin):
                 path.append(self['buildout']['eggs-directory'])
             else:
                 dest = self['buildout']['eggs-directory']
-                if not os.path.exists(dest):
-                    self._logger.info('Creating directory %r.', dest)
-                    os.mkdir(dest)
 
             zc.buildout.easy_install.install(
                 specs, dest, path=path,
@@ -1444,9 +1471,9 @@ def _default_globals():
     These default expressions are convenience defaults available when eveluating
     section headers expressions.
     NB: this is wrapped in a function so that the computing of these expressions
-    is lazy and done only if needed (ie if there is at least one section with 
-    an expression) because the computing of some of these expressions can be 
-    expensive. 
+    is lazy and done only if needed (ie if there is at least one section with
+    an expression) because the computing of some of these expressions can be
+    expensive.
     """
     # partially derived or inspired from its.py
     # Copyright (c) 2012, Kenneth Reitz All rights reserved.

src/zc/buildout/buildout.txt

@@ -810,7 +810,7 @@ COMMAND_LINE_VALUE).
         DEFAULT_VALUE
     directory= /sample-buildout
         COMPUTED_VALUE
-    eggs-directory= eggs
+    eggs-directory= /sample-buildout/eggs
         DEFAULT_VALUE
     executable= ...
         DEFAULT_VALUE
@@ -2446,9 +2446,9 @@ provide alternate locations, and even names for these directories.
     ... ))
 
     >>> print_(system(buildout), end='')
+    Creating directory '/sample-alt/basket'.
     Creating directory '/sample-alt/scripts'.
     Creating directory '/sample-alt/work'.
-    Creating directory '/sample-alt/basket'.
     Creating directory '/sample-alt/developbasket'.
     Develop: '/sample-buildout/recipes'
     Uninstalling d4.
@@ -2482,9 +2482,9 @@ You can also specify an alternate buildout directory:
     ...    ))
 
     >>> print_(system(buildout), end='')
+    Creating directory '/sample-alt/eggs'.
     Creating directory '/sample-alt/bin'.
     Creating directory '/sample-alt/parts'.
-    Creating directory '/sample-alt/eggs'.
     Creating directory '/sample-alt/develop-eggs'.
     Develop: '/sample-buildout/recipes'
 
@@ -2772,9 +2772,9 @@ local buildout scripts.
     ...              +' -c'+os.path.join(sample_bootstrapped, 'setup.cfg')
     ...              +' init'), end='')
     Creating '/sample-bootstrapped/setup.cfg'.
+    Creating directory '/sample-bootstrapped/eggs'.
     Creating directory '/sample-bootstrapped/bin'.
     Creating directory '/sample-bootstrapped/parts'.
-    Creating directory '/sample-bootstrapped/eggs'.
     Creating directory '/sample-bootstrapped/develop-eggs'.
     Generated script '/sample-bootstrapped/bin/buildout'.
 
@@ -2833,9 +2833,9 @@ if there isn't a configuration file:
     >>> print_(system(buildout
     ...              +' -c'+os.path.join(sample_bootstrapped2, 'setup.cfg')
     ...              +' bootstrap'), end='')
+    Creating directory '/sample-bootstrapped2/eggs'.
     Creating directory '/sample-bootstrapped2/bin'.
     Creating directory '/sample-bootstrapped2/parts'.
-    Creating directory '/sample-bootstrapped2/eggs'.
     Creating directory '/sample-bootstrapped2/develop-eggs'.
     Generated script '/sample-bootstrapped2/bin/buildout'.
 

src/zc/buildout/downloadcache.txt

@@ -46,7 +46,7 @@ download:
     <a href="index/">index/</a><br>
     <a href="other-1.0-py2.4.egg">other-1.0-py2.4.egg</a><br>
     </body></html>
-    
+
 
 We'll enable logging on the link server so we can see what's going on:
 
@@ -87,7 +87,7 @@ If we remove the installed eggs from eggs directory and re-run the buildout:
     >>> for  f in os.listdir('eggs'):
     ...     if f.startswith('demo'):
     ...         remove('eggs', f)
-   
+
     >>> print_(system(buildout), end='')
     GET 200 /
     Updating eggs.
@@ -139,3 +139,79 @@ install-from-cache option set to true:
     Getting distribution for 'demoneeded'.
     Got demoneeded 1.1.
     Generated script '/sample-buildout/bin/demo'.
+
+Auto-creation of download cache directory
+-----------------------------------------
+
+With zc.buildout version 2.2.2 or higher the cache directory is automatically
+created, provided it is within an already existing directory::
+
+    >>> write('buildout.cfg',
+    ... '''
+    ... [buildout]
+    ... parts =
+    ... download-cache = %(cache)s/newdir
+    ... ''' % globals())
+
+    >>> print_(system(buildout), end='')
+    Creating directory '/cache/newdir'.
+    Uninstalling eggs.
+
+    >>> ls(cache)
+    d  dist
+    d  newdir
+
+Using relative paths
+--------------------
+
+You can use a relative path for ``download-cache`` (the same logic is applied to
+``eggs-directory`` and to ``extends-cache`` too) and in such case it is considered
+relative to the location of the configuration file that sets its value.
+
+As an example, we create a ``base.cfg`` configuration in a different directory::
+
+    >>> basedir = tmpdir('basecfg')
+    >>> write(basedir, 'base.cfg',
+    ... '''
+    ... [buildout]
+    ... download-cache = cache
+    ... ''')
+
+and a ``buildout.cfg`` that extends from there::
+
+    >>> write('buildout.cfg',
+    ... '''
+    ... [buildout]
+    ... extends = %(basedir)s/base.cfg
+    ... parts =
+    ... ''' % globals())
+
+    >>> dummy = system(buildout)
+    >>> ls(basedir)
+    -  base.cfg
+    d  cache
+
+Of course this cannot be used when the base configuration is not on the local
+filesystem because it wouldn't make any sense having a remote cache::
+
+    >>> server_data = tmpdir('server_data')
+    >>> server_url = start_server(server_data)
+    >>> cd(sample_buildout)
+
+    >>> write(server_data, 'base.cfg', """\
+    ... [buildout]
+    ... download-cache = cache
+    ... """)
+
+    >>> write('buildout.cfg',
+    ... '''
+    ... [buildout]
+    ... extends = %(server_url)s/base.cfg
+    ... parts =
+    ... ''' % globals())
+
+    >>> print_(system(buildout), end='') # doctest: +ELLIPSIS
+    While:
+      Initializing.
+    Error: Setting "download-cache" to a non absolute location ("cache") within a
+    remote configuration file...

src/zc/buildout/setup.txt

@@ -30,6 +30,7 @@ We'll use the buildout script from our sample buildout:
 
     >>> print_(system(buildout+' setup'), end='')
     ... # doctest: +NORMALIZE_WHITESPACE
+    Creating directory '/sample-buildout/test/eggs'.
     Error: The setup command requires the path to a setup script or
     directory containing a setup script, and its arguments.
 

src/zc/buildout/tests.py

@@ -646,9 +646,9 @@ bootstrapping.
     >>> os.chdir(d)
     >>> print_(system(os.path.join(sample_buildout, 'bin', 'buildout')
     ...              + ' bootstrap'), end='')
+    Creating directory '/sample-bootstrap/eggs'.
     Creating directory '/sample-bootstrap/bin'.
     Creating directory '/sample-bootstrap/parts'.
-    Creating directory '/sample-bootstrap/eggs'.
     Creating directory '/sample-bootstrap/develop-eggs'.
     Generated script '/sample-bootstrap/bin/buildout'.
     """
@@ -673,9 +673,9 @@ def bug_92891_bootstrap_crashes_with_egg_recipe_in_buildout_section():
     >>> os.chdir(d)
     >>> print_(system(os.path.join(sample_buildout, 'bin', 'buildout')
     ...              + ' bootstrap'), end='')
+    Creating directory '/sample-bootstrap/eggs'.
     Creating directory '/sample-bootstrap/bin'.
     Creating directory '/sample-bootstrap/parts'.
-    Creating directory '/sample-bootstrap/eggs'.
     Creating directory '/sample-bootstrap/develop-eggs'.
     Generated script '/sample-bootstrap/bin/buildout'.
 

src/zc/buildout/update.txt

@@ -175,9 +175,9 @@ directory:
 
     >>> cd(sample_buildout2)
     >>> print_(system(buildout), end='')
+    Creating directory '/sample_buildout2/eggs'.
     Creating directory '/sample_buildout2/bin'.
     Creating directory '/sample_buildout2/parts'.
-    Creating directory '/sample_buildout2/eggs'.
     Creating directory '/sample_buildout2/develop-eggs'.
     Getting distribution for 'zc.buildout>=1.99'.
     Got zc.buildout 99.99.