merge from gary-4

CHANGES.txt

@@ -11,19 +11,17 @@ New Features:
   than zc.recipe.egg (which is still a fully supported, and simpler, way of
   generating scripts and interpreters if you are using a "clean" Python).
 
-  A hopefully slight limitation: in no cases are distributions in your
-  site-packages used to satisfy buildout dependencies.  The
-  site-packages can be used in addition to the dependencies specified in
-  your buildout, and buildout dependencies can override code in your
-  site-packages, but even if your Python's site-packages has the same
-  exact version as specified in your buildout configuration, buildout
-  will still use its own copy.
-
-- Added new function, ``zc.buildout.easy_install.generate_scripts``, to
-  generate scripts and interpreter.  It produces a full-featured
+  (Note that this branch is incomplete in its implementation of this feature:
+  if eggs are in installed in site-packages but you do not want to use
+  site-packages, the eggs will drag in site-packages even if you try to
+  exclude it.  This is addressed in subsequent branches in the series of
+  which this one is a part.)
+
+- Added new function, ``zc.buildout.easy_install.sitepackage_safe_scripts``,
+  to generate scripts and interpreter.  It produces a full-featured
   interpreter (all command-line options supported) and the ability to
-  safely let scripts include site packages.  The ``z3c.recipe.scripts``
-  recipe uses this new function.
+  safely let scripts include site packages, such as with a system
+  Python.  The ``z3c.recipe.scripts`` recipe uses this new function.
 
 - Improve bootstrap.
 

README.txt

@@ -35,12 +35,15 @@ Existing recipes include:
 `zc.recipe.egg <http://pypi.python.org/pypi/zc.recipe.egg>`_
    The egg recipe installes one or more eggs, with their
    dependencies.  It installs their console-script entry points with
-   the needed eggs included in their paths.
+   the needed eggs included in their paths.  It is suitable for use with
+   a "clean" Python: one without packages installed in site-packages.
 
 `z3c.recipe.scripts <http://pypi.python.org/pypi/z3c.recipe.scripts>`_
-  This scripts recipe builds interpreter scripts and entry point scripts
-  based on eggs.  These scripts have more features and flexibility than the
-  ones offered by zc.recipe.egg.
+  Like zc.recipe.egg, this recipe builds interpreter scripts and entry
+  point scripts based on eggs.  It can be used with a Python that has
+  packages installed in site-packages, such as a system Python.  The
+  interpreter also has more features than the one offered by
+  zc.recipe.egg.
 
 `zc.recipe.testrunner <http://pypi.python.org/pypi/zc.recipe.testrunner>`_
    The testrunner egg creates a test runner script for one or

src/zc/buildout/easy_install.py

@@ -61,15 +61,16 @@ setuptools_loc = pkg_resources.working_set.find(
     pkg_resources.Requirement.parse('setuptools')
     ).location
 
-# Include buildout and setuptools eggs in paths
-buildout_and_setuptools_path = [
-    setuptools_loc,
-    pkg_resources.working_set.find(
-        pkg_resources.Requirement.parse('zc.buildout')).location,
-    ]
+# Include buildout and setuptools eggs in paths.  We prevent dupes just to
+# keep from duplicating any log messages about them.
+buildout_loc = pkg_resources.working_set.find(
+    pkg_resources.Requirement.parse('zc.buildout')).location
+buildout_and_setuptools_path = [setuptools_loc]
+if os.path.normpath(setuptools_loc) != os.path.normpath(buildout_loc):
+    buildout_and_setuptools_path.append(buildout_loc)
 
 def _get_system_paths(executable):
-    """return lists of standard lib and site paths for executable.
+    """Return lists of standard lib and site paths for executable.
     """
     # We want to get a list of the site packages, which is not easy.
     # The canonical way to do this is to use
@@ -227,24 +228,47 @@ else:
 #
 # The namespace packages installed in site-packages with
 # --single-version-externally-managed use a mechanism that cause them to
-# be processed when site.py is imported.  Simply starting Python with -S
-# addresses the problem in Python 2.4 and 2.5, but Python 2.6's distutils
-# imports a value from the site module, so we unfortunately have to do more
-# drastic surgery in the _easy_install_cmd code below.  The changes to
-# sys.modules specifically try to only remove namespace modules installed by
-# the --single-version-externally-managed code.
+# be processed when site.py is imported  (see
+# http://mail.python.org/pipermail/distutils-sig/2009-May/011730.html
+# for another description of the problem).  Simply starting Python with
+# -S addresses the problem in Python 2.4 and 2.5, but Python 2.6's
+# distutils imports a value from the site module, so we unfortunately
+# have to do more drastic surgery in the _easy_install_cmd code below.
+#
+# Here's an example of the .pth files created by setuptools when using that
+# flag:
+#
+# import sys,new,os;
+# p = os.path.join(sys._getframe(1).f_locals['sitedir'], *('<NAMESPACE>',));
+# ie = os.path.exists(os.path.join(p,'__init__.py'));
+# m = not ie and sys.modules.setdefault('<NAMESPACE>',new.module('<NAMESPACE>'));
+# mp = (m or []) and m.__dict__.setdefault('__path__',[]);
+# (p not in mp) and mp.append(p)
+#
+# The code, below, then, runs under -S, indicating that site.py should
+# not be loaded initially.  It gets the initial sys.path under these
+# circumstances, and then imports site (because Python 2.6's distutils
+# will want it, as mentioned above). It then reinstates the old sys.path
+# value. Then it removes namespace packages (created by the setuptools
+# code above) from sys.modules.  It identifies namespace packages by
+# iterating over every loaded module.  It first looks if there is a
+# __path__, so it is a package; and then it sees if that __path__ does
+# not have an __init__.py.  (Note that PEP 382,
+# http://www.python.org/dev/peps/pep-0382, makes it possible to have a
+# namespace package that has an __init__.py, but also should make it
+# unnecessary for site.py to preprocess these packages, so it should be
+# fine, as far as can be guessed as of this writing.)  Finally, it
+# imports easy_install and runs it.
 
 _easy_install_cmd = _safe_arg('''\
-import sys; \
-p = sys.path[:]; \
-m = sys.modules.keys(); \
-import site; \
-sys.path[:] = p; \
-m_attrs = set(('__builtins__', '__file__', '__package__', '__path__')); \
-match = set(('__path__',)); \
+import sys,os;\
+p = sys.path[:];\
+import site;\
+sys.path[:] = p;\
 [sys.modules.pop(k) for k, v in sys.modules.items()\
- if k not in m and v and m_attrs.intersection(dir(v)) == match]; \
-from setuptools.command.easy_install import main; \
+ if hasattr(v, '__path__') and len(v.__path__)==1 and\
+ not os.path.exists(os.path.join(v.__path__[0],'__init__.py'))];\
+from setuptools.command.easy_install import main;\
 main()''')
 
 
@@ -1126,8 +1150,9 @@ def scripts(reqs, working_set, executable, dest,
             ):
     """Generate scripts and/or an interpreter.
 
-    See generate_scripts for a newer version with more options and a
-    different approach.
+    See sitepackage_safe_scripts for a version that can be used with a Python
+    that can be used with a Python that has code installed in site-packages.
+    It has more options and a different approach.
     """
     path = _get_path(working_set, extra_paths)
     if initialization:
@@ -1142,12 +1167,12 @@ def scripts(reqs, working_set, executable, dest,
             _pyscript(spath, sname, executable, rpsetup))
     return generated
 
-def generate_scripts(
+def sitepackage_safe_scripts(
     dest, working_set, executable, site_py_dest,
     reqs=(), scripts=None, interpreter=None, extra_paths=(),
     initialization='', add_site_packages=False, exec_sitecustomize=False,
     relative_paths=False, script_arguments='', script_initialization=''):
-    """Generate scripts and/or an interpreter.
+    """Generate scripts and/or an interpreter from a system Python.
 
     This accomplishes the same job as the ``scripts`` function, above,
     but it does so in an alternative way that allows safely including
@@ -1161,7 +1186,7 @@ def generate_scripts(
         site_py_dest, working_set, executable, extra_paths,
         add_site_packages, relative_paths))
     script_initialization = (
-        '\nimport site # imports custom buildbot-generated site.py\n%s' % (
+        '\nimport site # imports custom buildout-generated site.py\n%s' % (
             script_initialization,))
     if not script_initialization.endswith('\n'):
         script_initialization += '\n'
@@ -1175,7 +1200,7 @@ def generate_scripts(
 
 # Utilities for the script generation functions.
 
-# These are shared by both ``scripts`` and ``generate_scripts``
+# These are shared by both ``scripts`` and ``sitepackage_safe_scripts``
 
 def _get_path(working_set, extra_paths=()):
     """Given working set and extra paths, return a normalized path list."""
@@ -1442,7 +1467,7 @@ if _interactive:
     __import__("code").interact(banner="", local=globals())
 '''
 
-# These are used only by the newer ``generate_scripts`` function.
+# These are used only by the newer ``sitepackage_safe_scripts`` function.
 
 def _get_module_file(executable, name):
     """Return a module's file path.

src/zc/buildout/easy_install.txt

@@ -595,7 +595,7 @@ The easy_install module provides support for creating scripts from eggs.
 It provides two competing functions.  One, ``scripts``, is a
 well-established approach to generating reliable scripts with a "clean"
 Python--e.g., one that does not have any packages in its site-packages.
-The other, ``generate_scripts``, is newer, a bit trickier, and is
+The other, ``sitepackage_safe_scripts``, is newer, a bit trickier, and is
 designed to work with a Python that has code in its site-packages, such
 as a system Python.
 
@@ -607,10 +607,10 @@ baking a script's path into the script.  This has two advantages:
 
 - The script doesn't have to import pkg_resources because the logic that
   pkg_resources would execute at run time is executed at script-creation
-  time.  (There is an exception in ``generate_scripts`` if you want to
-  have your Python's site packages available, as discussed below, but
-  even in that case pkg_resources is only partially activated, which can
-  be a significant time savings.)
+  time.  (There is an exception in ``sitepackage_safe_scripts`` if you
+  want to have your Python's site packages available, as discussed
+  below, but even in that case pkg_resources is only partially
+  activated, which can be a significant time savings.)
 
 
 The ``scripts`` function
@@ -994,22 +994,23 @@ We specified an interpreter and its paths are adjusted too:
         del _interactive
         __import__("code").interact(banner="", local=globals())
 
-The ``generate_scripts`` function
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+The ``sitepackage_safe_scripts`` function
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-The newer function for creating scripts is ``generate_scripts``.  It has the
-same basic functionality as the ``scripts`` function: it can create scripts
-to run arbitrary entry points, and to run a Python interpreter.  The
-following are the differences from a user's perspective.
+The newer function for creating scripts is ``sitepackage_safe_scripts``.
+ It has the same basic functionality as the ``scripts`` function: it can
+create scripts to run arbitrary entry points, and to run a Python
+interpreter.  The following are the differences from a user's
+perspective.
 
 - It can be used safely with a Python that has packages installed itself,
   such as a system-installed Python.
 
 - In contrast to the interpreter generated by the ``scripts`` method, which
   supports only a small subset of the usual Python executable's options,
-  the interpreter generated by ``generate_scripts`` supports all of them.
-  This makes it possible to use as full Python replacement for scripts that
-  need the distributions specified in your buildout.
+  the interpreter generated by ``sitepackage_safe_scripts`` supports all
+  of them. This makes it possible to use as full Python replacement for
+  scripts that need the distributions specified in your buildout.
 
 - Both the interpreter and the entry point scripts allow you to include the
   site packages, and/or the sitecustomize, of the Python executable, if
@@ -1033,7 +1034,7 @@ Here's the simplest example, building an interpreter script.
     >>> ws = zc.buildout.easy_install.install(
     ...     ['demo'], join(interpreter_dir, 'eggs'), links=[link_server],
     ...     index=link_server+'index/')
-    >>> generated = zc.buildout.easy_install.generate_scripts(
+    >>> generated = zc.buildout.easy_install.sitepackage_safe_scripts(
     ...     interpreter_bin_dir, ws, sys.executable, interpreter_parts_dir,
     ...     interpreter='py')
 
@@ -1135,7 +1136,7 @@ If you provide initialization, it goes in sitecustomize.py.
     >>> initialization_string = """\
     ... import os
     ... os.environ['FOO'] = 'bar baz bing shazam'"""
-    >>> generated = zc.buildout.easy_install.generate_scripts(
+    >>> generated = zc.buildout.easy_install.sitepackage_safe_scripts(
     ...     interpreter_bin_dir, ws, sys.executable, interpreter_parts_dir,
     ...     interpreter='py', initialization=initialization_string)
     >>> cat(sitecustomize_path)
@@ -1150,7 +1151,7 @@ again the UNIX version; the Windows version uses subprocess instead of
 os.execve.)
 
     >>> reset_interpreter()
-    >>> generated = zc.buildout.easy_install.generate_scripts(
+    >>> generated = zc.buildout.easy_install.sitepackage_safe_scripts(
     ...     interpreter_bin_dir, ws, sys.executable, interpreter_parts_dir,
     ...     interpreter='py', relative_paths=interpreter_dir)
     >>> cat(py_path)
@@ -1206,7 +1207,7 @@ The ``extra_paths`` argument affects the path in site.py.  Notice that
 
     >>> reset_interpreter()
     >>> mkdir(interpreter_dir, 'other')
-    >>> generated = zc.buildout.easy_install.generate_scripts(
+    >>> generated = zc.buildout.easy_install.sitepackage_safe_scripts(
     ...     interpreter_bin_dir, ws, sys.executable, interpreter_parts_dir,
     ...     interpreter='py', extra_paths=[join(interpreter_dir, 'other')])
     >>> sys.stdout.write('#\n'); cat(site_path) # doctest: +ELLIPSIS
@@ -1232,15 +1233,15 @@ The ``extra_paths`` argument affects the path in site.py.  Notice that
      '/interpreter/other']
     <BLANKLINE>
 
-The ``generate_scripts`` function: using site-packages
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+The ``sitepackage_safe_scripts`` function: using site-packages
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-The ``generate_scripts`` function supports including site packages.  This has
-some advantages and some serious dangers.
+The ``sitepackage_safe_scripts`` function supports including site
+packages.  This has some advantages and some serious dangers.
 
 A typical reason to include site-packages is that it is easier to
 install one or more dependencies in your Python than it is with
-buildbot.  Some packages, such as lxml or Python PostgreSQL integration,
+buildout.  Some packages, such as lxml or Python PostgreSQL integration,
 have dependencies that can be much easier to build and/or install using
 other mechanisms, such as your operating system's package manager.  By
 installing some core packages into your Python's site-packages, this can
@@ -1264,7 +1265,7 @@ That explained, let's see how it works.  If you don't use namespace packages,
 this is very straightforward.
 
     >>> reset_interpreter()
-    >>> generated = zc.buildout.easy_install.generate_scripts(
+    >>> generated = zc.buildout.easy_install.sitepackage_safe_scripts(
     ...     interpreter_bin_dir, ws, sys.executable, interpreter_parts_dir,
     ...     interpreter='py', add_site_packages=True)
     >>> sys.stdout.write('#\n'); cat(site_path)
@@ -1333,7 +1334,7 @@ call to another text fixture to create.
     >>> ws = zc.buildout.easy_install.install(
     ...     ['demo', 'tellmy.fortune'], join(interpreter_dir, 'eggs'),
     ...     links=[link_server, namespace_eggs], index=link_server+'index/')
-    >>> generated = zc.buildout.easy_install.generate_scripts(
+    >>> generated = zc.buildout.easy_install.sitepackage_safe_scripts(
     ...     interpreter_bin_dir, ws, sys.executable, interpreter_parts_dir,
     ...     interpreter='py', add_site_packages=True)
     >>> sys.stdout.write('#\n'); cat(site_path)
@@ -1391,7 +1392,7 @@ include site-packages, and use relative paths.  For completeness, we'll look
 at that result.
 
     >>> reset_interpreter()
-    >>> generated = zc.buildout.easy_install.generate_scripts(
+    >>> generated = zc.buildout.easy_install.sitepackage_safe_scripts(
     ...     interpreter_bin_dir, ws, sys.executable, interpreter_parts_dir,
     ...     interpreter='py', add_site_packages=True,
     ...     relative_paths=interpreter_dir)
@@ -1448,8 +1449,8 @@ sitecustomize module in the underlying Python if you set the argument to
 True.  The z3c.recipe.scripts package sets up the full environment necessary
 to demonstrate this piece.
 
-The ``generate_scripts`` function: writing scripts for entry points
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+The ``sitepackage_safe_scripts`` function: writing scripts for entry points
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 All of the examples so far for this function have been creating
 interpreters.  The function can also write scripts for entry
@@ -1463,7 +1464,7 @@ see a simple example.
     >>> ws = zc.buildout.easy_install.install(
     ...     ['demo'], join(interpreter_dir, 'eggs'), links=[link_server],
     ...     index=link_server+'index/')
-    >>> generated = zc.buildout.easy_install.generate_scripts(
+    >>> generated = zc.buildout.easy_install.sitepackage_safe_scripts(
     ...     interpreter_bin_dir, ws, sys.executable, interpreter_parts_dir,
     ...     reqs=['demo'])
 
@@ -1497,7 +1498,7 @@ The demo script runs the entry point defined in the demo egg:
         ]
     <BLANKLINE>
     <BLANKLINE>
-    import site # imports custom buildbot-generated site.py
+    import site # imports custom buildout-generated site.py
     <BLANKLINE>
     import eggrecipedemo
     <BLANKLINE>
@@ -1524,7 +1525,7 @@ pertinent to the entry point scripts, you can use the
 Let's see ``script_arguments`` and ``script_initialization`` in action.
 
     >>> reset_interpreter()
-    >>> generated = zc.buildout.easy_install.generate_scripts(
+    >>> generated = zc.buildout.easy_install.sitepackage_safe_scripts(
     ...     interpreter_bin_dir, ws, sys.executable, interpreter_parts_dir,
     ...     reqs=['demo'], script_arguments='1, 2',
     ...    script_initialization='import os\nos.chdir("foo")')
@@ -1536,7 +1537,7 @@ Let's see ``script_arguments`` and ``script_initialization`` in action.
       '/interpreter/parts/interpreter',
       ]
     <BLANKLINE>
-    import site # imports custom buildbot-generated site.py
+    import site # imports custom buildout-generated site.py
     import os
     os.chdir("foo")
     <BLANKLINE>

src/zc/buildout/tests.py

@@ -1881,8 +1881,9 @@ def handle_namespace_package_in_both_site_packages_and_buildout_eggs():
     r"""
 If you have the same namespace package in both site-packages and in
 buildout, we need to be very careful that faux-Python-executables and
-scripts generated by easy_install.generate_scripts correctly combine the two.
-We show this with the local recipe that uses the function, z3c.recipe.scripts.
+scripts generated by easy_install.sitepackage_safe_scripts correctly
+combine the two. We show this with the local recipe that uses the
+function, z3c.recipe.scripts.
 
 To demonstrate this, we will create three packages: tellmy.version 1.0,
 tellmy.version 1.1, and tellmy.fortune 1.0.  tellmy.version 1.1 is installed.
@@ -2049,8 +2050,9 @@ Before the bugfix, running this buildout would generate this error:
     We already have: tellmy.version 1.0
     <BLANKLINE>
 
-The bugfix was simply to add Python's "-S" option when calling
-easyinstall (see zc.buildout.easy_install.Installer._call_easy_install).
+You can see the copiously commented fix for this in easy_install.py (see
+zc.buildout.easy_install.Installer._call_easy_install and particularly
+the comment leading up to zc.buildout.easy_install._easy_install_cmd).
 Now the install works correctly, as seen here.
 
     >>> print system(buildout)

src/zc/buildout/testselectingpython.py

@@ -61,7 +61,7 @@ def multi_python(test):
             ['setuptools'], executable_dir,
             index='http://www.python.org/pypi/',
             always_unzip=True, executable=other_executable)
-        zc.buildout.easy_install.generate_scripts(
+        zc.buildout.easy_install.sitepackage_safe_scripts(
             executable_dir, ws, other_executable, executable_parts,
             reqs=['setuptools'], interpreter='py')
         original_executable = other_executable

z3c.recipe.scripts_/src/z3c/recipe/scripts/README.txt

@@ -47,7 +47,7 @@ extends
 exec-sitecustomize
     Normally the Python's real sitecustomize module is not processed.
     If you want it to be processed, set this value to 'true'.  This will
-    be honored irrespective of the setting for include-site-paths.
+    be honored irrespective of the setting for add-site-packages.
 
 script-initialization
     The standard initialization code affects both an interpreter and scripts.
@@ -181,7 +181,7 @@ some advantages and some serious dangers.
 
 A typical reason to include site-packages is that it is easier to
 install one or more dependencies in your Python than it is with
-buildbot.  Some packages, such as lxml or Python PostgreSQL integration,
+buildout.  Some packages, such as lxml or Python PostgreSQL integration,
 have dependencies that can be much easier to build and/or install using
 other mechanisms, such as your operating system's package manager.  By
 installing some core packages into your Python's site-packages, this can
@@ -399,5 +399,4 @@ interpreter, so that you are not forced to use the name of the section.
     42
     <BLANKLINE>
 
-The other options all identical to the zc.recipe.egg script.  Here are some
-quick demos and discussions.
+The other options all identical to zc.recipe.egg.

z3c.recipe.scripts_/src/z3c/recipe/scripts/scripts.py

@@ -63,7 +63,7 @@ class Interpreter(Base):
         if not os.path.exists(options['parts-directory']):
             os.mkdir(options['parts-directory'])
             generated.append(options['parts-directory'])
-        generated.extend(zc.buildout.easy_install.generate_scripts(
+        generated.extend(zc.buildout.easy_install.sitepackage_safe_scripts(
             options['bin-directory'], ws, options['executable'],
             options['parts-directory'],
             interpreter=options['name'],
@@ -86,7 +86,7 @@ class Scripts(Base):
         if not os.path.exists(options['parts-directory']):
             os.mkdir(options['parts-directory'])
             generated.append(options['parts-directory'])
-        generated.extend(zc.buildout.easy_install.generate_scripts(
+        generated.extend(zc.buildout.easy_install.sitepackage_safe_scripts(
             options['bin-directory'], ws, options['executable'],
             options['parts-directory'], reqs=reqs, scripts=scripts,
             interpreter=options.get('interpreter'),

z3c.recipe.scripts_/src/z3c/recipe/scripts/tests.py

@@ -25,7 +25,7 @@ from zope.testing import doctest, renormalizing
 # all of the examples.  The README tests ``extends``,
 # ``include-site-customization`` and ``name``.  That leaves ``python``,
 # ``extra-paths``, ``initialization``, ``relative-paths``, and
-# ``include-site-packages``.
+# ``add-site-packages``.
 
 def supports_python_option():
     """