an interpreter variant that supports all interpreter flags (AFAIK). this...

an interpreter variant that supports all interpreter flags (AFAIK).  this commit contains core implementation (in easy_install module) and tests of all functionality except import_sitecustomize.  Integration into zc.recipe.egg and tests for import_sitecustomize (which will be more involved, and in tests.py) will follow.

src/zc/buildout/easy_install.py

@@ -968,17 +968,91 @@ def scripts(reqs, working_set, executable, dest,
 
     return generated
 
-import_site_snippet = '''\
-# We have to import pkg_resources before namespace
-# package .pth files are processed or else the distribution's namespace
-# packages will mask all of the egg-based packages in the same namespace
-# package.
-try:
-  import pkg_resources
-except ImportError:
+def interpreter(name, working_set, executable, dest, site_py_dest,
+                extra_paths=(),
+                initialization='',
+                relative_paths=False,
+                import_site=False,
+                import_sitecustomize=False
+                ):
+    generated = []
+    # Write sitecustomize.py.
+    sitecustomize_path = os.path.join(site_py_dest, 'sitecustomize.py')
+    sitecustomize = open(sitecustomize_path, 'w')
+    if initialization:
+        sitecustomize.write(initialization + '\n')
+    if import_sitecustomize:
+        real_sitecustomize_path = _get_module_file(
+            executable, 'sitecustomize')
+        if real_sitecustomize_path:
+            sitecustomize.write('execfile(%s)\n' % (real_sitecustomize_path,))
+    sitecustomize.close()
+    generated.append(sitecustomize_path)
+    # Write site.py.
+    path = [dist.location for dist in working_set]
+    path.extend(extra_paths)
+    path = map(realpath, path)
+    site_path = os.path.join(site_py_dest, 'site.py')
+    path_string, rpsetup = _relative_path_and_setup(
+        site_path, path, relative_paths)
+    site = open(site_path, 'w')
+    site.write(rpsetup)
+    site.write(sys_path_template % (path_string,))
+    if import_site:
+        real_site_path = _get_module_file(executable, 'site')
+        if real_site_path:
+            site.write(import_site_template % (real_site_path,))
+    else:
+        site.write('import sitecustomize\n')
+    generated.append(site_path)
+    # Write interpreter script.
+    script_name = full_name = os.path.join(dest, name)
+    site_py_dest_string, rpsetup = _relative_path_and_setup(
+        full_name, [site_py_dest], relative_paths)
+    if is_win32:
+        script_name += '-script.py'
+    contents = interpreter_template % dict(
+        python = _safe_arg(executable),
+        site_dest = site_py_dest_string,
+        relative_paths_setup = rpsetup,
+        )
+    changed = not (os.path.exists(script_name) and
+                   open(script_name).read() == contents)
+    if is_win32:
+        # Generate exe file and give the script a magic name.
+        exe = full_name + '.exe'
+        open(exe, 'wb').write(
+            pkg_resources.resource_string('setuptools', 'cli.exe')
+            )
+        generated.append(exe)
+    if changed:
+        open(script_name, 'w').write(contents)
+        try:
+            os.chmod(script_name,0755)
+        except (AttributeError, os.error):
             pass
-import site
-'''
+        logger.info("Generated interpreter %r.", name)
+    generated.append(script_name)
+    return generated
+
+def _get_module_file(executable, name):
+    cmd = [executable, "-c",
+           "import imp; fp, path, desc = imp.find_module(%r); fp.close; print path" % (name,)]
+    _proc = subprocess.Popen(
+        cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    stdout, stderr = _proc.communicate();
+    if _proc.returncode:
+        logger.info(
+            'Could not find file for module %s:\n%s', name, stderr)
+        return None
+    # else: ...
+    res = stdout.strip()
+    if res.endswith('.pyc') or res.endswith('.pyo'):
+        raise RuntimeError('Cannot find uncompiled version of %s' % (name,))
+    if not os.path.exists(res):
+        raise RuntimeError(
+            'File does not exist for module %s:\n%s' % (name, res))
+    return res
 
 def _relative_path_and_setup(sname, path, relative_paths):
     if relative_paths:
@@ -996,7 +1070,6 @@ def _relative_path_and_setup(sname, path, relative_paths):
         rpsetup = ''
     return spath, rpsetup
 
-
 def _relative_depth(common, path):
     n = 0
     while 1:
@@ -1033,7 +1106,6 @@ def _relativitize(path, script, relative_paths):
     else:
         return repr(path)
 
-
 relative_paths_setup = """
 import os
 
@@ -1090,6 +1162,25 @@ if is_jython and jython_os_name == 'linux':
 else:
     script_header = '#!%(python)s -S'
 
+_import_site_start = '''\
+# We have to import pkg_resources before namespace
+# package .pth files are processed or else the distribution's namespace
+# packages will mask all of the egg-based packages in the same namespace
+# package.
+try:
+  import pkg_resources
+except ImportError:
+  pass
+'''
+
+import_site_snippet = _import_site_start + 'import site\n'
+import_site_template = _import_site_start + 'execfile(%r)\n'
+sys_path_template = '''\
+import sys
+sys.path[0:0] = [
+  %s,
+  ]
+'''
 
 script_template = script_header + '''\
 
@@ -1106,6 +1197,20 @@ if __name__ == '__main__':
     %(module_name)s.%(attrs)s(%(arguments)s)
 '''
 
+interpreter_template = script_header + '''\
+
+%(relative_paths_setup)s
+import os
+import sys
+
+argv = [sys.executable] + sys.argv[1:]
+environ = os.environ.copy()
+path = %(site_dest)s
+if environ.get('PYTHONPATH'):
+    path = os.pathsep.join([path, environ['PYTHONPATH']])
+environ['PYTHONPATH'] = path
+os.execve(sys.executable, argv, environ)
+'''
 
 def _pyscript(path, dest, executable, rsetup, import_site):
     generated = []

src/zc/buildout/easy_install.txt

@@ -999,6 +999,233 @@ We specified an interpreter and its paths are adjusted too:
         __import__("code").interact(banner="", local=globals())
 
 
+Interpreter generation
+----------------------
+
+The interpreter created above is a script that emulates the Python
+interpreter.  Sometimes it is useful to have a full-fledged Python interpreter
+available.  The ``interpreter`` function supports this.
+
+It works by creating site.py and sitecustomize.py files that set up the
+desired paths and initialization.  These must be placed within an otherwise
+empty directory.  Typically this is in a recipe's parts directory.
+
+Here's the simplest example.
+
+    >>> interpreter_dir = tmpdir('interpreter')
+    >>> mkdir(interpreter_dir, 'bin')
+    >>> mkdir(interpreter_dir, 'eggs')
+    >>> mkdir(interpreter_dir, 'parts')
+    >>> mkdir(interpreter_dir, 'parts', 'interpreter')
+
+    >>> ws = zc.buildout.easy_install.install(
+    ...     ['demo'], join(interpreter_dir, 'eggs'), links=[link_server],
+    ...     index=link_server+'index/')
+    >>> generated = zc.buildout.easy_install.interpreter(
+    ...     'py', ws, sys.executable, join(interpreter_dir, 'bin'),
+    ...     join(interpreter_dir, 'parts', 'interpreter'))
+
+On non-Windows systems, this creates three files.
+
+    >>> len(generated)
+    3
+    >>> import pprint
+    >>> pprint.pprint(sorted(generated))
+    ['/interpreter/bin/py',
+     '/interpreter/parts/interpreter/site.py',
+     '/interpreter/parts/interpreter/sitecustomize.py']
+
+These are the three generated files.
+
+    >>> cat(interpreter_dir, 'bin', 'py')
+    #!/usr/bin/python2.4 -S
+    <BLANKLINE>
+    import os
+    import sys
+    <BLANKLINE>
+    argv = [sys.executable] + sys.argv[1:]
+    environ = os.environ.copy()
+    path = '/interpreter/parts/interpreter'
+    if environ.get('PYTHONPATH'):
+        path = os.pathsep.join([path, environ['PYTHONPATH']])
+    environ['PYTHONPATH'] = path
+    os.execve(sys.executable, argv, environ)
+    >>> ls(interpreter_dir, 'parts', 'interpreter')
+    -  site.py
+    -  sitecustomize.py
+    >>> cat(interpreter_dir, 'parts', 'interpreter', 'site.py')
+    import sys
+    sys.path[0:0] = [
+      '/interpreter/eggs/demo-0.3-py2.4.egg',
+      '/interpreter/eggs/demoneeded-1.1-py2.4.egg',
+      ]
+    import sitecustomize
+    >>> cat(interpreter_dir, 'parts', 'interpreter', 'sitecustomize.py')
+
+Here are some examples of the interpreter in use.
+
+    >>> print system(join(interpreter_dir, 'bin', 'py') + ' -c "print 16+26"')
+    42
+    <BLANKLINE>
+    >>> res = system(join(interpreter_dir, 'bin', 'py') +
+    ...              ' -c "import sys, pprint; pprint.pprint(sys.path)"')
+    >>> print res # doctest: +ELLIPSIS
+    ['',
+     '/interpreter/eggs/demo-0.3-py2.4.egg',
+     '/interpreter/eggs/demoneeded-1.1-py2.4.egg',
+     '/interpreter/parts/interpreter',
+     ...]
+    <BLANKLINE>
+    >>> clean_paths = eval(res.strip()) # This is used later for comparison.
+
+If you provide initialization, it goes in sitecustomize.py.
+
+    >>> def reset_interpreter():
+    ...     # This is necessary because, in our tests, the timestamps of the
+    ...     # .pyc files are not outdated when we want them to be.
+    ...     rmdir(interpreter_dir, 'bin')
+    ...     mkdir(interpreter_dir, 'bin')
+    ...     rmdir(interpreter_dir, 'parts', 'interpreter')
+    ...     mkdir(interpreter_dir, 'parts', 'interpreter')
+    ...
+    >>> reset_interpreter()
+
+    >>> initialization_string = """\
+    ... import os
+    ... os.environ['FOO'] = 'bar baz bing shazam'"""
+    >>> generated = zc.buildout.easy_install.interpreter(
+    ...     'py', ws, sys.executable, join(interpreter_dir, 'bin'),
+    ...     join(interpreter_dir, 'parts', 'interpreter'),
+    ...     initialization=initialization_string)
+    >>> cat(interpreter_dir, 'parts', 'interpreter', 'sitecustomize.py')
+    import os
+    os.environ['FOO'] = 'bar baz bing shazam'
+    >>> print system(join(interpreter_dir, 'bin', 'py') +
+    ...              """ -c 'import os; print os.environ["FOO"]'""")
+    bar baz bing shazam
+    <BLANKLINE>
+
+If you use relative paths, this affects the interpreter and site.py.
+
+    >>> reset_interpreter()
+    >>> generated = zc.buildout.easy_install.interpreter(
+    ...     'py', ws, sys.executable, join(interpreter_dir, 'bin'),
+    ...     join(interpreter_dir, 'parts', 'interpreter'),
+    ...     relative_paths=interpreter_dir)
+    >>> cat(interpreter_dir, 'bin', 'py')
+    #!/usr/bin/python2.4 -S
+    <BLANKLINE>
+    import os
+    <BLANKLINE>
+    join = os.path.join
+    base = os.path.dirname(os.path.abspath(os.path.realpath(__file__)))
+    base = os.path.dirname(base)
+    <BLANKLINE>
+    import os
+    import sys
+    <BLANKLINE>
+    argv = [sys.executable] + sys.argv[1:]
+    environ = os.environ.copy()
+    path = join(base, 'parts/interpreter')
+    if environ.get('PYTHONPATH'):
+        path = os.pathsep.join([path, environ['PYTHONPATH']])
+    environ['PYTHONPATH'] = path
+    os.execve(sys.executable, argv, environ)
+    >>> cat(interpreter_dir, 'parts', 'interpreter', 'site.py')
+    <BLANKLINE>
+    import os
+    <BLANKLINE>
+    join = os.path.join
+    base = os.path.dirname(os.path.abspath(os.path.realpath(__file__)))
+    base = os.path.dirname(base)
+    base = os.path.dirname(base)
+    import sys
+    sys.path[0:0] = [
+      join(base, 'eggs/demo-0.3-pyN.N.egg'),
+      join(base, 'eggs/demoneeded-1.1-pyN.N.egg'),
+      ]
+    import sitecustomize
+    >>> print system(join(interpreter_dir, 'bin', 'py') +
+    ...              ' -c "import sys, pprint; pprint.pprint(sys.path)"')
+    ... # doctest: +ELLIPSIS
+    ['',
+     '/interpreter/eggs/demo-0.3-py2.4.egg',
+     '/interpreter/eggs/demoneeded-1.1-py2.4.egg',
+     '/interpreter/parts/interpreter',
+     ...]
+    <BLANKLINE>
+
+The ``extra_paths`` argument affects the path in site.py.
+
+    >>> reset_interpreter()
+    >>> generated = zc.buildout.easy_install.interpreter(
+    ...     'py', ws, sys.executable, join(interpreter_dir, 'bin'),
+    ...     join(interpreter_dir, 'parts', 'interpreter'),
+    ...     extra_paths=[join(interpreter_dir, 'other')])
+    >>> cat(interpreter_dir, 'parts', 'interpreter', 'site.py')
+    import sys
+    sys.path[0:0] = [
+      '/interpreter/eggs/demo-0.3-py2.4.egg',
+      '/interpreter/eggs/demoneeded-1.1-py2.4.egg',
+      '/interpreter/other',
+      ]
+    import sitecustomize
+    >>> print system(join(interpreter_dir, 'bin', 'py') +
+    ...              ' -c "import sys, pprint; pprint.pprint(sys.path)"')
+    ... # doctest: +ELLIPSIS
+    ['',
+     '/interpreter/eggs/demo-0.3-py2.4.egg',
+     '/interpreter/eggs/demoneeded-1.1-py2.4.egg',
+     '/interpreter/other',
+     '/interpreter/parts/interpreter',
+     ...]
+    <BLANKLINE>
+
+The ``import_site`` argument also affects site.py.  It causes the executable's
+real site.py to be executed after our customizations have run.
+
+    >>> reset_interpreter()
+    >>> generated = zc.buildout.easy_install.interpreter(
+    ...     'py', ws, sys.executable, join(interpreter_dir, 'bin'),
+    ...     join(interpreter_dir, 'parts', 'interpreter'),
+    ...     import_site=True)
+    >>> cat(interpreter_dir, 'parts', 'interpreter', 'site.py')
+    import sys
+    sys.path[0:0] = [
+      '/interpreter/eggs/demo-0.3-py2.4.egg',
+      '/interpreter/eggs/demoneeded-1.1-py2.4.egg',
+      ]
+    # We have to import pkg_resources before namespace
+    # package .pth files are processed or else the distribution's namespace
+    # packages will mask all of the egg-based packages in the same namespace
+    # package.
+    try:
+      import pkg_resources
+    except ImportError:
+      pass
+    execfile('/usr/lib/python/site.py')
+    >>> res = system(join(interpreter_dir, 'bin', 'py') +
+    ...              ' -c "import sys, pprint; pprint.pprint(sys.path)"')
+    >>> print res # doctest: +ELLIPSIS
+    ['',
+     '/interpreter/eggs/demo-0.3-py2.4.egg',
+     '/interpreter/eggs/demoneeded-1.1-py2.4.egg',
+     '/interpreter/parts/interpreter',
+     ...]
+    <BLANKLINE>
+
+The clean_paths gathered earlier is a subset of this full list of paths.
+
+    >>> full_paths = eval(res.strip())
+    >>> len(clean_paths) < len(full_paths)
+    True
+    >>> set(os.path.normpath(p) for p in clean_paths).difference(
+    ...     os.path.normpath(p) for p in full_paths)
+    set([])
+
+The ``import_sitecustomize`` argument does the same thing for the
+sitecustomize module.
+
 Handling custom build options for extensions provided in source distributions
 -----------------------------------------------------------------------------
 

src/zc/buildout/tests.py

@@ -2933,6 +2933,8 @@ def test_suite():
                (re.compile('extdemo[.]pyd'), 'extdemo.so'),
                (re.compile('[-d]  setuptools-\S+[.]egg'), 'setuptools.egg'),
                (re.compile(r'\\[\\]?'), '/'),
+               (re.compile('''execfile\(['"].+site.py['"]\)'''),
+                "execfile('/usr/lib/python/site.py')"),
                ]+(sys.version_info < (2, 5) and [
                   (re.compile('.*No module named runpy.*', re.S), ''),
                   (re.compile('.*usage: pdb.py scriptfile .*', re.S), ''),