Added support controlling how eggs with extensions are built.

git-svn-id: http://svn.zope.org/repos/main/zc.buildout/trunk@69383 62d5b8a3-27da-0310-9561-8e5933582275

src/zc/buildout/easy_install.py

@@ -20,10 +20,15 @@ installed.
 $Id$
 """
 
-import logging, os, re, sys
-import pkg_resources
+import logging, os, re, tempfile, sys
+import pkg_resources, setuptools.command.setopt
 import zc.buildout
 
+# XXX we could potentially speed this up quite a bit by keeping our
+# own PackageIndex to analyse whether there are newer dists.  A hitch
+# is that the package index seems to go out of its way to only handle
+# one Python version at a time. :(
+
 logger = logging.getLogger('zc.buildout.easy_install')
 
 # Include buildout and setuptools eggs in paths
@@ -131,15 +136,15 @@ def _call_easy_install(spec, dest, links=(),
         logger.debug('Running easy_install:\n%s "%s"\npath=%s\n',
                      executable, '" "'.join(args), path)
     
-    args += (dict(PYTHONPATH=path), )
+    args += (dict(os.environ, PYTHONPATH=path), )
     sys.stdout.flush() # We want any pending output first
     exit_code = os.spawnle(os.P_WAIT, executable, executable, *args)
+    assert exit_code == 0
 
     # We may overwrite distributions, so clear importer
     # cache.
     sys.path_importer_cache.clear()
 
-    assert exit_code == 0
 
 
 def _get_dist(requirement, env, ws,
@@ -235,6 +240,77 @@ def install(specs, dest,
             
     return ws
 
+
+def _editable(spec, dest, links=(), index = None, executable=sys.executable):
+    prefix = sys.exec_prefix + os.path.sep
+    path = os.pathsep.join([p for p in sys.path if not p.startswith(prefix)])
+    args = (
+        '-c', 'from setuptools.command.easy_install import main; main()',
+        '-eb', dest)
+    if links:
+        args += ('-f', ' '.join(links))
+    if index:
+        args += ('-i', index)
+    level = logger.getEffectiveLevel()
+    if level > logging.DEBUG:
+        args += ('-q', )
+    elif level < logging.DEBUG:
+        args += ('-v', )
+    
+    args += (spec, )
+
+    if level <= logging.DEBUG:
+        logger.debug('Running easy_install:\n%s "%s"\npath=%s\n',
+                     executable, '" "'.join(args), path)
+    
+    args += (dict(os.environ, PYTHONPATH=path), )
+    sys.stdout.flush() # We want any pending output first
+    exit_code = os.spawnle(os.P_WAIT, executable, executable, *args)
+    assert exit_code == 0
+
+def build(spec, dest, build_ext,
+          links=(), index=None,
+          executable=sys.executable,
+          path=None):
+
+    # XXX we're going to download and build the egg every stinking time.
+    # We need to not do that.
+
+    logger.debug('Building %r', spec)
+
+    path = path and path[:] or []
+    if dest is not None:
+        path.insert(0, dest)
+
+    path += buildout_and_setuptools_path
+
+    links = list(links) # make copy, because we may need to mutate
+    
+    # For each spec, see if it is already installed.  We create a working
+    # set to keep track of what we've collected and to make sue than the
+    # distributions assembled are consistent.
+    env = pkg_resources.Environment(path, python=_get_version(executable))
+    requirement = pkg_resources.Requirement.parse(spec)
+
+    dist = _satisfied(requirement, env)
+    if dist is not None:
+        return dist
+
+    # Get an editable version of the package to a temporary directory:
+    tmp = tempfile.mkdtemp('editable')
+    _editable(spec, tmp, links, index, executable)
+
+    setup_cfg = os.path.join(tmp, requirement.key, 'setup.cfg')
+    if not os.path.exists(setup_cfg):
+        f = open(setup_cfg, 'w')
+        f.close()
+    setuptools.command.setopt.edit_config(setup_cfg, dict(build_ext=build_ext))
+
+    # Now run easy_install for real:
+    _call_easy_install(
+        os.path.join(tmp, requirement.key),
+        dest, links, index, executable, True)
+
 def working_set(specs, executable, path):
     return install(specs, None, executable=executable, path=path)
 

src/zc/buildout/easy_install.txt

@@ -18,7 +18,10 @@ easy_install command that provides some additional semantics:
   look for additional distributions.  We always give preference to
   develop eggs.
 
-The easy_install module provides a single method, install.  The
+- Distutils options for building extensions can be passed.
+
+The easy_install module provides a method, install, for installing one
+or more packages and their dependencies.  The
 install function takes 2 positional arguments:
 
 - An iterable of setuptools requirement strings for the distributions
@@ -73,6 +76,7 @@ We have a link server that has a number of eggs:
     <a href="demoneeded-1.0-py2.4.egg">demoneeded-1.0-py2.4.egg</a><br>
     <a href="demoneeded-1.1-py2.3.egg">demoneeded-1.1-py2.3.egg</a><br>
     <a href="demoneeded-1.1-py2.4.egg">demoneeded-1.1-py2.4.egg</a><br>
+    <a href="extdemo-1.4.tar.gz">extdemo-1.4.tar.gz</a><br>
     <a href="index/">index/</a><br>
     <a href="other-1.0-py2.3.egg">other-1.0-py2.3.egg</a><br>
     <a href="other-1.0-py2.4.egg">other-1.0-py2.4.egg</a><br>
@@ -267,3 +271,94 @@ original script names to new script names.
 
     >>> print system(os.path.join(bin, 'run')),
     3 1
+
+Handling custom build options for extensions
+--------------------------------------------
+
+Sometimes, we need to control how extension modules are built.  The
+build method provides this level of control.  It takes a single
+package specification, downloads a source distribution, and builds it
+with specified custom build options.
+
+The build method takes 3 positional arguments:
+
+spec
+   A package specification
+
+dest
+   A destination directory
+
+build_ext
+   A dictionary of options to be passed to the distutils build_ext
+   command when building extensions.
+
+It supports a number of optional keyword arguments:
+
+links
+   a sequence of URLs, file names, or directories to look for
+   links to distributions,
+
+index
+   The URL of an index server, or almost any other valid URL. :)
+
+   If not specified, the Python Package Index,
+   http://cheeseshop.python.org/pypi, is used.  You can specify an
+   alternate index with this option.  If you use the links option and
+   if the links point to the needed distributions, then the index can
+   be anything and will be largely ignored.  In the examples, here,
+   we'll just point to an empty directory on our link server.  This 
+   will make our examples run a little bit faster.
+
+executable
+   A path to a Python executable.  Distributions will ne installed
+   using this executable and will be for the matching Python version.
+
+path
+   A list of additional directories to search for locally-installed
+   distributions.
+
+always_unzip
+   A flag indicating that newly-downloaded distributions should be
+   directories even if they could be installed as zip files.
+
+Our link server included a source distribution that includes a simple
+extension, extdemo.c::
+
+  #include <Python.h>
+  #include <extdemo.h>
+
+  static PyMethodDef methods[] = {};
+
+  PyMODINIT_FUNC
+  initextdemo(void)
+  {
+      PyObject *d;
+      d = Py_InitModule3("extdemo", methods, "");
+      PyDict_SetItemString(d, "val", PyInt_FromLong(EXTDEMO));    
+  }
+
+The extension depends on a system-dependnt include file, extdemo.h,
+that defines a constant, EXTDEMO, that is exposed by the extension.
+
+We'll add an include directory to our sample buildout and add the
+needed include file to it:
+
+    >>> mkdir(sample_buildout, 'include')
+    >>> open(os.path.join(sample_buildout, 'include', 'extdemo.h'), 'w').write(
+    ...    "#define EXTDEMO 42\n")
+
+Now, we can use the build function to create an egg from the source
+distribution:
+
+    >>> zc.buildout.easy_install.build(
+    ...   'extdemo', dest, 
+    ...   {'include-dirs': os.path.join(sample_buildout, 'include')},
+    ...   links=[link_server], index=link_server+'index/')
+
+Now if we look in our destination directory, we see we have an extdemo egg:
+
+    >>> ls(dest)
+    d  demo-0.3-py2.4.egg
+    d  demoneeded-1.1-py2.4.egg
+    d  extdemo-1.4-py2.3-unix-i686.egg
+

src/zc/buildout/testing.py

@@ -200,6 +200,49 @@ def multi_python(test):
     test.globs['python2_4_executable'] = p24
 
 
+
+extdemo_c = """
+#include <Python.h>
+#include <extdemo.h>
+
+static PyMethodDef methods[] = {};
+
+PyMODINIT_FUNC
+initextdemo(void)
+{
+    PyObject *d;
+    d = Py_InitModule3("extdemo", methods, "");
+    PyDict_SetItemString(d, "val", PyInt_FromLong(EXTDEMO));    
+}
+"""
+
+extdemo_setup_py = """
+from distutils.core import setup, Extension
+
+setup(name = "extdemo", version = "1.4", url="http://www.zope.org",
+      author="Demo", author_email="demo@demo.com",
+      ext_modules = [Extension('extdemo', ['extdemo.c'])],
+      )
+"""
+
+def add_source_dist(test):
+    import tarfile
+    tmp = tempfile.mkdtemp('test-sdist')
+    open(os.path.join(tmp, 'extdemo.c'), 'w').write(extdemo_c);
+    open(os.path.join(tmp, 'setup.py'), 'w').write(extdemo_setup_py);
+    open(os.path.join(tmp, 'README'), 'w').write("");
+    open(os.path.join(tmp, 'MANIFEST.in'), 'w').write("include *.c\n");
+    here = os.getcwd()
+    os.chdir(tmp)
+    status = os.spawnl(os.P_WAIT, sys.executable, sys.executable,
+                       os.path.join(tmp, 'setup.py'), '-q', 'sdist')
+    os.chdir(here)
+    assert status == 0
+    shutil.move(
+        os.path.join(tmp, 'dist', 'extdemo-1.4.tar.gz'),
+        os.path.join(test.globs['sample_eggs'], 'extdemo-1.4.tar.gz'),
+        )
+    
 def make_tree(test):
     sample_eggs = test.globs['sample_eggs']
     tree = dict(
@@ -266,6 +309,8 @@ class Handler(BaseHTTPServer.BaseHTTPRequestHandler):
             self.send_header('Content-Length', len(out))
             if name.endswith('.egg'):
                 self.send_header('Content-Type', 'application/zip')
+            elif name.endswith('.gz'):
+                self.send_header('Content-Type', 'application/x-gzip')
             else:
                 self.send_header('Content-Type', 'text/html')
         self.end_headers()

src/zc/buildout/tests.py

@@ -141,6 +141,12 @@ def linkerSetUp(test):
     zc.buildout.testing.multi_python(test)
     zc.buildout.testing.setUpServer(test, zc.buildout.testing.make_tree(test))
 
+def easy_install_SetUp(test):
+    zc.buildout.testing.buildoutSetUp(test, clear_home=False)
+    zc.buildout.testing.multi_python(test)
+    zc.buildout.testing.add_source_dist(test)
+    zc.buildout.testing.setUpServer(test, zc.buildout.testing.make_tree(test))
+
 class PythonNormalizing(renormalizing.RENormalizing):
 
     def _transform(self, want, got):
@@ -216,14 +222,15 @@ def test_suite():
         
         doctest.DocFileSuite(
             'easy_install.txt', 
-            setUp=linkerSetUp, tearDown=zc.buildout.testing.buildoutTearDown,
+            setUp=easy_install_SetUp,
+            tearDown=zc.buildout.testing.buildoutTearDown,
 
             checker=PythonNormalizing([
                (re.compile("'%(sep)s\S+sample-install%(sep)s(dist%(sep)s)?"
                            % dict(sep=os.path.sep)),
                 '/sample-eggs/'),
-               (re.compile("(-  (demo(needed)?|other)"
-                           "-\d[.]\d-py)\d[.]\d[.]egg"),
+               (re.compile("([d-]  ((ext)?demo(needed)?|other)"
+                           "-\d[.]\d-py)\d[.]\d(-[^. \t\n]+)?[.]egg"),
                 '\\1V.V.egg'),
                ]),
             ),

zc.recipe.egg_/README.txt

@@ -35,6 +35,12 @@ scripts
    disabled.  If the option isn't given at all, then all scripts
    defined by the named eggs will be generated.
 
+Custom eggs
+-----------
+
+The zc.recipe.egg:custom recipe supports building custom eggs,
+currently with specialized options for building extensions.
+
 To do
 -----
 

zc.recipe.egg_/setup.py

@@ -19,6 +19,9 @@ setup(
     install_requires = ['zc.buildout', 'setuptools'],
     tests_require = ['zope.testing'],
     test_suite = name+'.tests.test_suite',
-    entry_points = {'zc.buildout': ['default = %s:Egg' % name]},    
+    entry_points = {'zc.buildout': ['default = %s:Egg' % name,
+                                    'custom = %s:Custom' % name,
+                                    ]
+                    },    
     dependency_links = ['http://download.zope.org/distribution/'],
     )

zc.recipe.egg_/src/zc/recipe/egg/__init__.py

-from zc.recipe.egg.egg import Egg                                 
+from zc.recipe.egg.egg import Egg
+from zc.recipe.egg.custom import Custom

zc.recipe.egg_/src/zc/recipe/egg/custom.py

+##############################################################################
+#
+# Copyright (c) 2006 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Install packages as eggs
+
+$Id$
+"""
+
+import os, re, zipfile
+import zc.buildout.easy_install
+
+class Custom:
+
+    def __init__(self, buildout, name, options):
+        self.buildout = buildout
+        self.name = name
+        self.options = options
+        links = options.get('find-links',
+                            buildout['buildout'].get('find-links'))
+        if links:
+            links = links.split()
+            options['find-links'] = '\n'.join(links)
+        else:
+            links = ()
+        self.links = links
+
+        index = options.get('index', buildout['buildout'].get('index'))
+        if index is not None:
+            options['index'] = index
+        self.index = index
+
+        options['_b'] = buildout['buildout']['bin-directory']
+        options['_e'] = buildout['buildout']['eggs-directory']
+        options['_d'] = buildout['buildout']['develop-eggs-directory']
+
+        assert options.get('unzip') in ('true', 'false', None)
+
+        python = options.get('python', buildout['buildout']['python'])
+        options['executable'] = buildout[python]['executable']
+
+        build_ext = {}
+        for be_option in ('include-dirs', 'library-dirs', 'rpath'):
+            value = options.get(be_option)
+            if value is None:
+                continue
+            value = [
+                os.path.join(
+                    buildout['buildout']['directory'],
+                    v.strip()
+                    )
+                for v in value.strip().split('\n')
+                if v.strip()
+            ]
+            build_ext[be_option] = ':'.join(value)
+            options[be_option] = ':'.join(value)
+        self.build_ext = build_ext
+
+    def install(self):
+        if self.buildout['buildout'].get('offline') == 'true':
+            return
+        options = self.options
+        distribution = options.get('eggs', self.name).strip()
+        build_ext = dict([
+            (k, options[k])
+            for k in ('include-dirs', 'library-dirs', 'rpath')
+            if k in options
+            ])
+        zc.buildout.easy_install.build(
+            distribution, options['_d'], self.build_ext,
+            self.links, self.index, options['executable'], [options['_e']],
+            )
+        

zc.recipe.egg_/src/zc/recipe/egg/custom.txt

+Custon eggs
+===========
+
+Sometimes, It's necessary to provide extra control over how an egg is
+created.  This is commonly true for eggs with extension modules that
+need to access libraries or include files.
+
+The zc.recipe.egg:custom recipe can be used to define an egg with
+custom build parameters.  The currently defined parameters are:
+
+include-dirs
+   A new-line separated list of directories to search for include
+   files.
+
+library-dirs
+   A new-line separated list of directories to search for libraries
+   to link with.
+
+rpath
+   A new-line separated list of directories to search for dynamic libraries
+   at run time.
+
+In addition, the following options can be used to specify the egg:
+
+egg
+    An eggs to install given as a setuptools requirement string.
+    This defaults to the part name.
+
+find-links
+   A list of URLs, files, or directories to search for distributions.
+
+index
+   The URL of an index server, or almost any other valid URL. :)
+
+   If not specified, the Python Package Index,
+   http://cheeseshop.python.org/pypi, is used.  You can specify an
+   alternate index with this option.  If you use the links option and
+   if the links point to the needed distributions, then the index can
+   be anything and will be largely ignored.  In the examples, here,
+   we'll just point to an empty directory on our link server.  This 
+   will make our examples run a little bit faster.
+
+python
+   The name of a section to get the Python executable from.
+   If not specified, then the buildout python option is used.  The
+   Python executable is found in the executable option of the named
+   section. 
+
+To illustrate this, we'll define a buildout that builds an egg for a
+package that has a simple extension module::
+
+  #include <Python.h>
+  #include <extdemo.h>
+
+  static PyMethodDef methods[] = {};
+
+  PyMODINIT_FUNC
+  initextdemo(void)
+  {
+      PyObject *d;
+      d = Py_InitModule3("extdemo", methods, "");
+      PyDict_SetItemString(d, "val", PyInt_FromLong(EXTDEMO));    
+  }
+
+The extension depends on a system-dependnt include file, extdemo.h,
+that defines a constant, EXTDEMO, that is exposed by the extension.
+
+The extension module is available as a source distribution,
+extdemo-1.4.tar.gz, on a distribution server.
+
+We have a sample buildout that we'll add an include directory to with
+the necessary include file:
+
+    >>> mkdir(sample_buildout, 'include')
+    >>> import os
+    >>> open(os.path.join(sample_buildout, 'include', 'extdemo.h'), 'w').write(
+    ...    "#define EXTDEMO 42\n")
+
+We'll also update the buildout configuration file to define a part for
+the egg:
+
+    >>> write(sample_buildout, 'buildout.cfg',
+    ... """
+    ... [buildout]
+    ... parts = extdemo
+    ...
+    ... [extdemo]
+    ... recipe = zc.recipe.egg:custom
+    ... find-links = %(server)s
+    ... index = %(server)s/index
+    ... include-dirs = include
+    ... """ % dict(server=link_server))
+
+    >>> os.chdir(sample_buildout)
+    >>> buildout = os.path.join(sample_buildout, 'bin', 'buildout')
+
+    >>> print system(buildout),
+    zip_safe flag not set; analyzing archive contents...
+
+We got the zip_safe warning because the source distribution we used
+wasn't setuptools based and thus didn't set the option.
+
+The egg is created in the develop-eggs directory *not* the eggs
+directory because it depends on buildout-specific parameters and the
+eggs directory can be shared across multiple buildouts.
+
+    >>> ls(sample_buildout, 'develop-eggs')
+    d  extdemo-1.4-py2.4-unix-i686.egg
+    -  zc.recipe.egg.egg-link
+
+Note that no scripts or dependencies are installed.  To install
+dependencies or scripts for a custom egg, define another part and use
+the zc.recipe.egg recipe, listing the custom egg as one of the eggs to
+be installed.  The zc.recipe.egg recipe will use the installed egg.

zc.recipe.egg_/src/zc/recipe/egg/tests.py

@@ -41,6 +41,16 @@ def setUpPython(test):
 
     zc.buildout.testing.multi_python(test)
     zc.buildout.testing.setUpServer(test, zc.buildout.testing.make_tree(test))
+
+def setUpCustom(test):
+    zc.buildout.testing.buildoutSetUp(test)
+    open(os.path.join(test.globs['sample_buildout'],
+                      'develop-eggs', 'zc.recipe.egg.egg-link'),
+         'w').write(dirname(__file__, 4))
+    zc.buildout.testing.create_sample_eggs(test)
+    zc.buildout.testing.add_source_dist(test)
+    zc.buildout.testing.setUpServer(test, zc.buildout.testing.make_tree(test))
+
     
 def test_suite():
     return unittest.TestSuite((
@@ -87,7 +97,17 @@ def test_suite():
                 r'/sample-\1/\2'),
                (re.compile('\S+sample-(\w+)'), r'/sample-\1'),
                ]),
-            ),        
+            ),
+        doctest.DocFileSuite(
+            'custom.txt',
+            setUp=setUpCustom, tearDown=zc.buildout.testing.buildoutTearDown,
+            checker=renormalizing.RENormalizing([
+               (re.compile("(d  ((ext)?demo(needed)?|other)"
+                           "-\d[.]\d-py)\d[.]\d(-[^. \t\n]+)?[.]egg"),
+                '\\1V.V.egg'),
+               ]),
+            ),
+        
         ))
 
 if __name__ == '__main__':