From 4c0bd32b3a9e4555925fad010dedd006471c3acc Mon Sep 17 00:00:00 2001 From: jim <jim@62d5b8a3-27da-0310-9561-8e5933582275> Date: Wed, 9 Aug 2006 20:42:27 +0000 Subject: [PATCH] Added support controlling how eggs with extensions are built. git-svn-id: http://svn.zope.org/repos/main/zc.buildout/trunk@69383 62d5b8a3-27da-0310-9561-8e5933582275 --- src/zc/buildout/easy_install.py | 84 +++++++++++++- src/zc/buildout/easy_install.txt | 97 +++++++++++++++- src/zc/buildout/testing.py | 45 ++++++++ src/zc/buildout/tests.py | 13 ++- zc.recipe.egg_/README.txt | 6 + zc.recipe.egg_/setup.py | 5 +- zc.recipe.egg_/src/zc/recipe/egg/__init__.py | 3 +- zc.recipe.egg_/src/zc/recipe/egg/custom.py | 82 +++++++++++++ zc.recipe.egg_/src/zc/recipe/egg/custom.txt | 114 +++++++++++++++++++ zc.recipe.egg_/src/zc/recipe/egg/tests.py | 22 +++- 10 files changed, 460 insertions(+), 11 deletions(-) create mode 100644 zc.recipe.egg_/src/zc/recipe/egg/custom.py create mode 100644 zc.recipe.egg_/src/zc/recipe/egg/custom.txt diff --git a/src/zc/buildout/easy_install.py b/src/zc/buildout/easy_install.py index 4789942b..32301890 100644 --- a/src/zc/buildout/easy_install.py +++ b/src/zc/buildout/easy_install.py @@ -20,10 +20,15 @@ installed. $Id$ """ -import logging, os, re, sys -import pkg_resources +import logging, os, re, tempfile, sys +import pkg_resources, setuptools.command.setopt import zc.buildout +# XXX we could potentially speed this up quite a bit by keeping our +# own PackageIndex to analyse whether there are newer dists. A hitch +# is that the package index seems to go out of its way to only handle +# one Python version at a time. :( + logger = logging.getLogger('zc.buildout.easy_install') # Include buildout and setuptools eggs in paths @@ -131,15 +136,15 @@ def _call_easy_install(spec, dest, links=(), logger.debug('Running easy_install:\n%s "%s"\npath=%s\n', executable, '" "'.join(args), path) - args += (dict(PYTHONPATH=path), ) + args += (dict(os.environ, PYTHONPATH=path), ) sys.stdout.flush() # We want any pending output first exit_code = os.spawnle(os.P_WAIT, executable, executable, *args) + assert exit_code == 0 # We may overwrite distributions, so clear importer # cache. sys.path_importer_cache.clear() - assert exit_code == 0 def _get_dist(requirement, env, ws, @@ -235,6 +240,77 @@ def install(specs, dest, return ws + +def _editable(spec, dest, links=(), index = None, executable=sys.executable): + prefix = sys.exec_prefix + os.path.sep + path = os.pathsep.join([p for p in sys.path if not p.startswith(prefix)]) + args = ( + '-c', 'from setuptools.command.easy_install import main; main()', + '-eb', dest) + if links: + args += ('-f', ' '.join(links)) + if index: + args += ('-i', index) + level = logger.getEffectiveLevel() + if level > logging.DEBUG: + args += ('-q', ) + elif level < logging.DEBUG: + args += ('-v', ) + + args += (spec, ) + + if level <= logging.DEBUG: + logger.debug('Running easy_install:\n%s "%s"\npath=%s\n', + executable, '" "'.join(args), path) + + args += (dict(os.environ, PYTHONPATH=path), ) + sys.stdout.flush() # We want any pending output first + exit_code = os.spawnle(os.P_WAIT, executable, executable, *args) + assert exit_code == 0 + +def build(spec, dest, build_ext, + links=(), index=None, + executable=sys.executable, + path=None): + + # XXX we're going to download and build the egg every stinking time. + # We need to not do that. + + logger.debug('Building %r', spec) + + path = path and path[:] or [] + if dest is not None: + path.insert(0, dest) + + path += buildout_and_setuptools_path + + links = list(links) # make copy, because we may need to mutate + + # For each spec, see if it is already installed. We create a working + # set to keep track of what we've collected and to make sue than the + # distributions assembled are consistent. + env = pkg_resources.Environment(path, python=_get_version(executable)) + requirement = pkg_resources.Requirement.parse(spec) + + dist = _satisfied(requirement, env) + if dist is not None: + return dist + + # Get an editable version of the package to a temporary directory: + tmp = tempfile.mkdtemp('editable') + _editable(spec, tmp, links, index, executable) + + setup_cfg = os.path.join(tmp, requirement.key, 'setup.cfg') + if not os.path.exists(setup_cfg): + f = open(setup_cfg, 'w') + f.close() + setuptools.command.setopt.edit_config(setup_cfg, dict(build_ext=build_ext)) + + # Now run easy_install for real: + _call_easy_install( + os.path.join(tmp, requirement.key), + dest, links, index, executable, True) + def working_set(specs, executable, path): return install(specs, None, executable=executable, path=path) diff --git a/src/zc/buildout/easy_install.txt b/src/zc/buildout/easy_install.txt index 8ebff2ec..db206c51 100644 --- a/src/zc/buildout/easy_install.txt +++ b/src/zc/buildout/easy_install.txt @@ -18,7 +18,10 @@ easy_install command that provides some additional semantics: look for additional distributions. We always give preference to develop eggs. -The easy_install module provides a single method, install. The +- Distutils options for building extensions can be passed. + +The easy_install module provides a method, install, for installing one +or more packages and their dependencies. The install function takes 2 positional arguments: - An iterable of setuptools requirement strings for the distributions @@ -73,6 +76,7 @@ We have a link server that has a number of eggs: <a href="demoneeded-1.0-py2.4.egg">demoneeded-1.0-py2.4.egg</a><br> <a href="demoneeded-1.1-py2.3.egg">demoneeded-1.1-py2.3.egg</a><br> <a href="demoneeded-1.1-py2.4.egg">demoneeded-1.1-py2.4.egg</a><br> + <a href="extdemo-1.4.tar.gz">extdemo-1.4.tar.gz</a><br> <a href="index/">index/</a><br> <a href="other-1.0-py2.3.egg">other-1.0-py2.3.egg</a><br> <a href="other-1.0-py2.4.egg">other-1.0-py2.4.egg</a><br> @@ -267,3 +271,94 @@ original script names to new script names. >>> print system(os.path.join(bin, 'run')), 3 1 + +Handling custom build options for extensions +-------------------------------------------- + +Sometimes, we need to control how extension modules are built. The +build method provides this level of control. It takes a single +package specification, downloads a source distribution, and builds it +with specified custom build options. + +The build method takes 3 positional arguments: + +spec + A package specification + +dest + A destination directory + +build_ext + A dictionary of options to be passed to the distutils build_ext + command when building extensions. + +It supports a number of optional keyword arguments: + +links + a sequence of URLs, file names, or directories to look for + links to distributions, + +index + The URL of an index server, or almost any other valid URL. :) + + If not specified, the Python Package Index, + http://cheeseshop.python.org/pypi, is used. You can specify an + alternate index with this option. If you use the links option and + if the links point to the needed distributions, then the index can + be anything and will be largely ignored. In the examples, here, + we'll just point to an empty directory on our link server. This + will make our examples run a little bit faster. + +executable + A path to a Python executable. Distributions will ne installed + using this executable and will be for the matching Python version. + +path + A list of additional directories to search for locally-installed + distributions. + +always_unzip + A flag indicating that newly-downloaded distributions should be + directories even if they could be installed as zip files. + +Our link server included a source distribution that includes a simple +extension, extdemo.c:: + + #include <Python.h> + #include <extdemo.h> + + static PyMethodDef methods[] = {}; + + PyMODINIT_FUNC + initextdemo(void) + { + PyObject *d; + d = Py_InitModule3("extdemo", methods, ""); + PyDict_SetItemString(d, "val", PyInt_FromLong(EXTDEMO)); + } + +The extension depends on a system-dependnt include file, extdemo.h, +that defines a constant, EXTDEMO, that is exposed by the extension. + +We'll add an include directory to our sample buildout and add the +needed include file to it: + + >>> mkdir(sample_buildout, 'include') + >>> open(os.path.join(sample_buildout, 'include', 'extdemo.h'), 'w').write( + ... "#define EXTDEMO 42\n") + +Now, we can use the build function to create an egg from the source +distribution: + + >>> zc.buildout.easy_install.build( + ... 'extdemo', dest, + ... {'include-dirs': os.path.join(sample_buildout, 'include')}, + ... links=[link_server], index=link_server+'index/') + +Now if we look in our destination directory, we see we have an extdemo egg: + + >>> ls(dest) + d demo-0.3-py2.4.egg + d demoneeded-1.1-py2.4.egg + d extdemo-1.4-py2.3-unix-i686.egg + diff --git a/src/zc/buildout/testing.py b/src/zc/buildout/testing.py index 0f11596f..a57bace3 100644 --- a/src/zc/buildout/testing.py +++ b/src/zc/buildout/testing.py @@ -200,6 +200,49 @@ def multi_python(test): test.globs['python2_4_executable'] = p24 + +extdemo_c = """ +#include <Python.h> +#include <extdemo.h> + +static PyMethodDef methods[] = {}; + +PyMODINIT_FUNC +initextdemo(void) +{ + PyObject *d; + d = Py_InitModule3("extdemo", methods, ""); + PyDict_SetItemString(d, "val", PyInt_FromLong(EXTDEMO)); +} +""" + +extdemo_setup_py = """ +from distutils.core import setup, Extension + +setup(name = "extdemo", version = "1.4", url="http://www.zope.org", + author="Demo", author_email="demo@demo.com", + ext_modules = [Extension('extdemo', ['extdemo.c'])], + ) +""" + +def add_source_dist(test): + import tarfile + tmp = tempfile.mkdtemp('test-sdist') + open(os.path.join(tmp, 'extdemo.c'), 'w').write(extdemo_c); + open(os.path.join(tmp, 'setup.py'), 'w').write(extdemo_setup_py); + open(os.path.join(tmp, 'README'), 'w').write(""); + open(os.path.join(tmp, 'MANIFEST.in'), 'w').write("include *.c\n"); + here = os.getcwd() + os.chdir(tmp) + status = os.spawnl(os.P_WAIT, sys.executable, sys.executable, + os.path.join(tmp, 'setup.py'), '-q', 'sdist') + os.chdir(here) + assert status == 0 + shutil.move( + os.path.join(tmp, 'dist', 'extdemo-1.4.tar.gz'), + os.path.join(test.globs['sample_eggs'], 'extdemo-1.4.tar.gz'), + ) + def make_tree(test): sample_eggs = test.globs['sample_eggs'] tree = dict( @@ -266,6 +309,8 @@ class Handler(BaseHTTPServer.BaseHTTPRequestHandler): self.send_header('Content-Length', len(out)) if name.endswith('.egg'): self.send_header('Content-Type', 'application/zip') + elif name.endswith('.gz'): + self.send_header('Content-Type', 'application/x-gzip') else: self.send_header('Content-Type', 'text/html') self.end_headers() diff --git a/src/zc/buildout/tests.py b/src/zc/buildout/tests.py index 534eaf7b..e8570988 100644 --- a/src/zc/buildout/tests.py +++ b/src/zc/buildout/tests.py @@ -141,6 +141,12 @@ def linkerSetUp(test): zc.buildout.testing.multi_python(test) zc.buildout.testing.setUpServer(test, zc.buildout.testing.make_tree(test)) +def easy_install_SetUp(test): + zc.buildout.testing.buildoutSetUp(test, clear_home=False) + zc.buildout.testing.multi_python(test) + zc.buildout.testing.add_source_dist(test) + zc.buildout.testing.setUpServer(test, zc.buildout.testing.make_tree(test)) + class PythonNormalizing(renormalizing.RENormalizing): def _transform(self, want, got): @@ -216,14 +222,15 @@ def test_suite(): doctest.DocFileSuite( 'easy_install.txt', - setUp=linkerSetUp, tearDown=zc.buildout.testing.buildoutTearDown, + setUp=easy_install_SetUp, + tearDown=zc.buildout.testing.buildoutTearDown, checker=PythonNormalizing([ (re.compile("'%(sep)s\S+sample-install%(sep)s(dist%(sep)s)?" % dict(sep=os.path.sep)), '/sample-eggs/'), - (re.compile("(- (demo(needed)?|other)" - "-\d[.]\d-py)\d[.]\d[.]egg"), + (re.compile("([d-] ((ext)?demo(needed)?|other)" + "-\d[.]\d-py)\d[.]\d(-[^. \t\n]+)?[.]egg"), '\\1V.V.egg'), ]), ), diff --git a/zc.recipe.egg_/README.txt b/zc.recipe.egg_/README.txt index 7ed986a0..973e99c9 100644 --- a/zc.recipe.egg_/README.txt +++ b/zc.recipe.egg_/README.txt @@ -35,6 +35,12 @@ scripts disabled. If the option isn't given at all, then all scripts defined by the named eggs will be generated. +Custom eggs +----------- + +The zc.recipe.egg:custom recipe supports building custom eggs, +currently with specialized options for building extensions. + To do ----- diff --git a/zc.recipe.egg_/setup.py b/zc.recipe.egg_/setup.py index d60ea9b6..e3d0a253 100644 --- a/zc.recipe.egg_/setup.py +++ b/zc.recipe.egg_/setup.py @@ -19,6 +19,9 @@ setup( install_requires = ['zc.buildout', 'setuptools'], tests_require = ['zope.testing'], test_suite = name+'.tests.test_suite', - entry_points = {'zc.buildout': ['default = %s:Egg' % name]}, + entry_points = {'zc.buildout': ['default = %s:Egg' % name, + 'custom = %s:Custom' % name, + ] + }, dependency_links = ['http://download.zope.org/distribution/'], ) diff --git a/zc.recipe.egg_/src/zc/recipe/egg/__init__.py b/zc.recipe.egg_/src/zc/recipe/egg/__init__.py index 96248231..49ac3db8 100644 --- a/zc.recipe.egg_/src/zc/recipe/egg/__init__.py +++ b/zc.recipe.egg_/src/zc/recipe/egg/__init__.py @@ -1 +1,2 @@ -from zc.recipe.egg.egg import Egg +from zc.recipe.egg.egg import Egg +from zc.recipe.egg.custom import Custom diff --git a/zc.recipe.egg_/src/zc/recipe/egg/custom.py b/zc.recipe.egg_/src/zc/recipe/egg/custom.py new file mode 100644 index 00000000..da202afe --- /dev/null +++ b/zc.recipe.egg_/src/zc/recipe/egg/custom.py @@ -0,0 +1,82 @@ +############################################################################## +# +# Copyright (c) 2006 Zope Corporation and Contributors. +# All Rights Reserved. +# +# This software is subject to the provisions of the Zope Public License, +# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. +# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED +# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS +# FOR A PARTICULAR PURPOSE. +# +############################################################################## +"""Install packages as eggs + +$Id$ +""" + +import os, re, zipfile +import zc.buildout.easy_install + +class Custom: + + def __init__(self, buildout, name, options): + self.buildout = buildout + self.name = name + self.options = options + links = options.get('find-links', + buildout['buildout'].get('find-links')) + if links: + links = links.split() + options['find-links'] = '\n'.join(links) + else: + links = () + self.links = links + + index = options.get('index', buildout['buildout'].get('index')) + if index is not None: + options['index'] = index + self.index = index + + options['_b'] = buildout['buildout']['bin-directory'] + options['_e'] = buildout['buildout']['eggs-directory'] + options['_d'] = buildout['buildout']['develop-eggs-directory'] + + assert options.get('unzip') in ('true', 'false', None) + + python = options.get('python', buildout['buildout']['python']) + options['executable'] = buildout[python]['executable'] + + build_ext = {} + for be_option in ('include-dirs', 'library-dirs', 'rpath'): + value = options.get(be_option) + if value is None: + continue + value = [ + os.path.join( + buildout['buildout']['directory'], + v.strip() + ) + for v in value.strip().split('\n') + if v.strip() + ] + build_ext[be_option] = ':'.join(value) + options[be_option] = ':'.join(value) + self.build_ext = build_ext + + def install(self): + if self.buildout['buildout'].get('offline') == 'true': + return + options = self.options + distribution = options.get('eggs', self.name).strip() + build_ext = dict([ + (k, options[k]) + for k in ('include-dirs', 'library-dirs', 'rpath') + if k in options + ]) + zc.buildout.easy_install.build( + distribution, options['_d'], self.build_ext, + self.links, self.index, options['executable'], [options['_e']], + ) + diff --git a/zc.recipe.egg_/src/zc/recipe/egg/custom.txt b/zc.recipe.egg_/src/zc/recipe/egg/custom.txt new file mode 100644 index 00000000..e1202513 --- /dev/null +++ b/zc.recipe.egg_/src/zc/recipe/egg/custom.txt @@ -0,0 +1,114 @@ +Custon eggs +=========== + +Sometimes, It's necessary to provide extra control over how an egg is +created. This is commonly true for eggs with extension modules that +need to access libraries or include files. + +The zc.recipe.egg:custom recipe can be used to define an egg with +custom build parameters. The currently defined parameters are: + +include-dirs + A new-line separated list of directories to search for include + files. + +library-dirs + A new-line separated list of directories to search for libraries + to link with. + +rpath + A new-line separated list of directories to search for dynamic libraries + at run time. + +In addition, the following options can be used to specify the egg: + +egg + An eggs to install given as a setuptools requirement string. + This defaults to the part name. + +find-links + A list of URLs, files, or directories to search for distributions. + +index + The URL of an index server, or almost any other valid URL. :) + + If not specified, the Python Package Index, + http://cheeseshop.python.org/pypi, is used. You can specify an + alternate index with this option. If you use the links option and + if the links point to the needed distributions, then the index can + be anything and will be largely ignored. In the examples, here, + we'll just point to an empty directory on our link server. This + will make our examples run a little bit faster. + +python + The name of a section to get the Python executable from. + If not specified, then the buildout python option is used. The + Python executable is found in the executable option of the named + section. + +To illustrate this, we'll define a buildout that builds an egg for a +package that has a simple extension module:: + + #include <Python.h> + #include <extdemo.h> + + static PyMethodDef methods[] = {}; + + PyMODINIT_FUNC + initextdemo(void) + { + PyObject *d; + d = Py_InitModule3("extdemo", methods, ""); + PyDict_SetItemString(d, "val", PyInt_FromLong(EXTDEMO)); + } + +The extension depends on a system-dependnt include file, extdemo.h, +that defines a constant, EXTDEMO, that is exposed by the extension. + +The extension module is available as a source distribution, +extdemo-1.4.tar.gz, on a distribution server. + +We have a sample buildout that we'll add an include directory to with +the necessary include file: + + >>> mkdir(sample_buildout, 'include') + >>> import os + >>> open(os.path.join(sample_buildout, 'include', 'extdemo.h'), 'w').write( + ... "#define EXTDEMO 42\n") + +We'll also update the buildout configuration file to define a part for +the egg: + + >>> write(sample_buildout, 'buildout.cfg', + ... """ + ... [buildout] + ... parts = extdemo + ... + ... [extdemo] + ... recipe = zc.recipe.egg:custom + ... find-links = %(server)s + ... index = %(server)s/index + ... include-dirs = include + ... """ % dict(server=link_server)) + + >>> os.chdir(sample_buildout) + >>> buildout = os.path.join(sample_buildout, 'bin', 'buildout') + + >>> print system(buildout), + zip_safe flag not set; analyzing archive contents... + +We got the zip_safe warning because the source distribution we used +wasn't setuptools based and thus didn't set the option. + +The egg is created in the develop-eggs directory *not* the eggs +directory because it depends on buildout-specific parameters and the +eggs directory can be shared across multiple buildouts. + + >>> ls(sample_buildout, 'develop-eggs') + d extdemo-1.4-py2.4-unix-i686.egg + - zc.recipe.egg.egg-link + +Note that no scripts or dependencies are installed. To install +dependencies or scripts for a custom egg, define another part and use +the zc.recipe.egg recipe, listing the custom egg as one of the eggs to +be installed. The zc.recipe.egg recipe will use the installed egg. diff --git a/zc.recipe.egg_/src/zc/recipe/egg/tests.py b/zc.recipe.egg_/src/zc/recipe/egg/tests.py index 61cac382..15ebaa27 100644 --- a/zc.recipe.egg_/src/zc/recipe/egg/tests.py +++ b/zc.recipe.egg_/src/zc/recipe/egg/tests.py @@ -41,6 +41,16 @@ def setUpPython(test): zc.buildout.testing.multi_python(test) zc.buildout.testing.setUpServer(test, zc.buildout.testing.make_tree(test)) + +def setUpCustom(test): + zc.buildout.testing.buildoutSetUp(test) + open(os.path.join(test.globs['sample_buildout'], + 'develop-eggs', 'zc.recipe.egg.egg-link'), + 'w').write(dirname(__file__, 4)) + zc.buildout.testing.create_sample_eggs(test) + zc.buildout.testing.add_source_dist(test) + zc.buildout.testing.setUpServer(test, zc.buildout.testing.make_tree(test)) + def test_suite(): return unittest.TestSuite(( @@ -87,7 +97,17 @@ def test_suite(): r'/sample-\1/\2'), (re.compile('\S+sample-(\w+)'), r'/sample-\1'), ]), - ), + ), + doctest.DocFileSuite( + 'custom.txt', + setUp=setUpCustom, tearDown=zc.buildout.testing.buildoutTearDown, + checker=renormalizing.RENormalizing([ + (re.compile("(d ((ext)?demo(needed)?|other)" + "-\d[.]\d-py)\d[.]\d(-[^. \t\n]+)?[.]egg"), + '\\1V.V.egg'), + ]), + ), + )) if __name__ == '__main__': -- 2.30.9