Skip to content

S
slapos.buildout
Commit 8e902170 authored by Gary Poster's avatar Gary Poster
Browse files
Options

fix virtualenv interaction by identfying broken virtualenv characteristic and... 

fix virtualenv interaction by identfying broken virtualenv characteristic and reverting to previous behavior in that case.
parent 67fd1da8
Hide whitespace changes
Inline Side-by-side
Showing with 378 additions and 36 deletions
bootstrap/bootstrap.py
View file @ 8e902170
......@@ -20,7 +20,7 @@ use the -c option to specify an alternate configuration file.
$Id$
"""
import os, shutil, sys, tempfile, textwrap, urllib, urllib2
import os, shutil, sys, tempfile, textwrap, urllib, urllib2, subprocess
from optparse import OptionParser
if sys.platform == 'win32':
......@@ -32,11 +32,18 @@ if sys.platform == 'win32':
else:
quote = str
# Detect https://bugs.launchpad.net/virtualenv/+bug/572545 .
proc = subprocess.Popen(
[sys.executable, '-Sc', 'import ConfigParser'],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
proc.communicate()
has_broken_dash_S = bool(proc.returncode)
# In order to be more robust in the face of system Pythons, we want to
# run without site-packages loaded. This is somewhat tricky, in
# particular because Python 2.6's distutils imports site, so starting
# with the -S flag is not sufficient. However, we'll start with that:
if 'site' in sys.modules:
if not has_broken_dash_S and 'site' in sys.modules:
# We will restart with python -S.
args = sys.argv[:]
args[0:0] = [sys.executable, '-S']
......@@ -167,6 +174,9 @@ cmd = [quote(sys.executable),
'-mqNxd',
quote(eggs_dir)]
if not has_broken_dash_S:
cmd.insert(1, '-S')
if options.download_base:
cmd.extend(['-f', quote(options.download_base)])
......
dev.py
View file @ 8e902170
......@@ -19,7 +19,7 @@ buildout egg itself is installed as a develop egg.
$Id$
"""
import os, shutil, sys, subprocess, urllib2
import os, shutil, sys, subprocess, urllib2, subprocess
from optparse import OptionParser
if sys.platform == 'win32':
......@@ -31,11 +31,15 @@ if sys.platform == 'win32':
else:
quote = str
# Detect https://bugs.launchpad.net/virtualenv/+bug/572545 .
has_broken_dash_S = subprocess.call(
[sys.executable, '-Sc', 'import ConfigParser'])
# In order to be more robust in the face of system Pythons, we want to
# run without site-packages loaded. This is somewhat tricky, in
# particular because Python 2.6's distutils imports site, so starting
# with the -S flag is not sufficient. However, we'll start with that:
if 'site' in sys.modules:
if not has_broken_dash_S and 'site' in sys.modules:
# We will restart with python -S.
args = sys.argv[:]
args[0:0] = [sys.executable, '-S']
......@@ -117,10 +121,14 @@ except ImportError:
env = os.environ.copy() # Windows needs yet-to-be-determined values from this.
env['PYTHONPATH'] = os.path.dirname(pkg_resources.__file__)
subprocess.Popen(
[sys.executable] +
['setup.py', '-q', 'develop', '-m', '-x', '-d', 'develop-eggs'],
env=env).wait()
cmd = [quote(sys.executable),
'setup.py', '-q', 'develop', '-m', '-x', '-d', 'develop-eggs']
if not has_broken_dash_S:
cmd.insert(1, '-S')
subprocess.Popen(cmd, env=env).wait()
pkg_resources.working_set.add_entry('src')
......
src/zc/buildout/buildout.py
View file @ 8e902170
......@@ -34,6 +34,7 @@ import shutil
import sys
import tempfile
import UserDict
import warnings
import zc.buildout
import zc.buildout.download
import zc.buildout.easy_install
......@@ -51,6 +52,9 @@ is_jython = sys.platform.startswith('java')
if is_jython:
import subprocess
_sys_executable_has_broken_dash_S = (
zc.buildout.easy_install._has_broken_dash_S(sys.executable))
class MissingOption(zc.buildout.UserError, KeyError):
"""A required option was missing.
"""
......@@ -359,7 +363,7 @@ class Buildout(UserDict.DictMixin):
distributions, options['executable'],
[options['develop-eggs-directory'],
options['eggs-directory']],
include_site_packages=False,
include_site_packages=_sys_executable_has_broken_dash_S,
)
else:
ws = zc.buildout.easy_install.install(
......@@ -370,7 +374,7 @@ class Buildout(UserDict.DictMixin):
path=[options['develop-eggs-directory']],
newest=self.newest,
allow_hosts=self._allow_hosts,
include_site_packages=False,
include_site_packages=_sys_executable_has_broken_dash_S,
)
# Now copy buildout and setuptools eggs, and record destination eggs:
......@@ -408,7 +412,8 @@ class Buildout(UserDict.DictMixin):
relative_paths = ''
zc.buildout.easy_install.sitepackage_safe_scripts(
options['bin-directory'], ws, options['executable'], partsdir,
reqs=['zc.buildout'], relative_paths=relative_paths)
reqs=['zc.buildout'], relative_paths=relative_paths,
include_site_packages=_sys_executable_has_broken_dash_S)
init = bootstrap
......@@ -854,7 +859,7 @@ class Buildout(UserDict.DictMixin):
index = options.get('index'),
path = [options['develop-eggs-directory']],
allow_hosts = self._allow_hosts,
include_site_packages=False
include_site_packages=_sys_executable_has_broken_dash_S
)
upgraded = []
......@@ -910,7 +915,8 @@ class Buildout(UserDict.DictMixin):
os.mkdir(partsdir)
zc.buildout.easy_install.sitepackage_safe_scripts(
options['bin-directory'], ws, sys.executable, partsdir,
reqs=['zc.buildout'])
reqs=['zc.buildout'],
include_site_packages=_sys_executable_has_broken_dash_S)
# Restart
args = map(zc.buildout.easy_install._safe_arg, sys.argv)
......@@ -951,7 +957,7 @@ class Buildout(UserDict.DictMixin):
links = self['buildout'].get('find-links', '').split(),
index = self['buildout'].get('index'),
newest=self.newest, allow_hosts=self._allow_hosts,
include_site_packages=False)
include_site_packages=_sys_executable_has_broken_dash_S)
# Clear cache because extensions might now let us read pages we
# couldn't read before.
......@@ -1069,8 +1075,7 @@ def _install_and_load(spec, group, entry, buildout):
working_set=pkg_resources.working_set,
newest=buildout.newest,
allow_hosts=buildout._allow_hosts,
include_site_packages=False,
)
include_site_packages=_sys_executable_has_broken_dash_S)
__doing__ = 'Loading %s recipe entry %s:%s.', group, spec, entry
return pkg_resources.load_entry_point(
......@@ -1577,6 +1582,11 @@ Options:
will be started. This is especially useful for debuging recipe
problems.
-s
Squelch warnings about using an executable with a broken -S
implementation.
Assignments are of the form: section:option=value and are used to
provide configuration options that override those given in the
configuration file. For example, to run the buildout in offline mode,
......@@ -1642,11 +1652,12 @@ def main(args=None):
windows_restart = False
user_defaults = True
debug = False
ignore_broken_dash_s = False
while args:
if args[0][0] == '-':
op = orig_op = args.pop(0)
op = op[1:]
while op and op[0] in 'vqhWUoOnNDA':
while op and op[0] in 'vqhWUoOnNDAs':
if op[0] == 'v':
verbosity += 10
elif op[0] == 'q':
......@@ -1665,6 +1676,8 @@ def main(args=None):
options.append(('buildout', 'newest', 'false'))
elif op[0] == 'D':
debug = True
elif op[0] == 's':
ignore_broken_dash_s = True
else:
_help()
op = op[1:]
......@@ -1708,6 +1721,17 @@ def main(args=None):
# The rest should be commands, so we'll stop here
break
if verbosity < 0 or ignore_broken_dash_s:
broken_dash_S_filter_action = 'ignore'
elif verbosity == 0: # This is the default.
broken_dash_S_filter_action = 'once'
else:
broken_dash_S_filter_action = 'default'
warnings.filterwarnings(
broken_dash_S_filter_action,
re.escape(
zc.buildout.easy_install.BROKEN_DASH_S_WARNING),
UserWarning)
if verbosity:
options.append(('buildout', 'verbosity', str(verbosity)))
......
src/zc/buildout/easy_install.py
View file @ 8e902170
......@@ -33,6 +33,7 @@ import shutil
import subprocess
import sys
import tempfile
import warnings
import zc.buildout
import zipimport
......@@ -54,6 +55,18 @@ is_jython = sys.platform.startswith('java')
is_distribute = (
pkg_resources.Requirement.parse('setuptools').key=='distribute')
BROKEN_DASH_S_WARNING = (
'Buildout has been asked to exclude or limit site-packages so that '
'builds can be repeatable when using a system Python. However, '
'the chosen Python executable has a broken implementation of -S (see '
'https://bugs.launchpad.net/virtualenv/+bug/572545 for an example '
"problem) and this breaks buildout's ability to isolate site-packages. "
"If the executable already has a clean site-packages (e.g., "
"using virtualenv's ``--no-site-packages`` option) you may be getting "
'equivalent repeatability. To silence this warning, use the -s argument '
'to the buildout script. Alternatively, use a Python executable with a '
'working -S (such as a standard Python binary).')
if is_jython:
import java.lang.System
jython_os_name = (java.lang.System.getProperties()['os.name']).lower()
......@@ -70,6 +83,14 @@ buildout_and_setuptools_path = [setuptools_loc]
if os.path.normpath(setuptools_loc) != os.path.normpath(buildout_loc):
buildout_and_setuptools_path.append(buildout_loc)
def _has_broken_dash_S(executable):
"""Detect https://bugs.launchpad.net/virtualenv/+bug/572545 ."""
proc = subprocess.Popen(
[executable, '-Sc', 'import ConfigParser'],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
proc.communicate()
return bool(proc.returncode)
def _get_system_paths(executable):
"""Return lists of standard lib and site paths for executable.
"""
......@@ -209,10 +230,10 @@ else:
_safe_arg = str
# The following string is used to run easy_install in
# Installer._call_easy_install. It is started with python -S (that is,
# don't import site at start). That flag, and all of the code in this
# snippet above the last two lines, exist to work around a relatively rare
# problem. If
# Installer._call_easy_install. It is usually started with python -S
# (that is, don't import site at start). That flag, and all of the code
# in this snippet above the last two lines, exist to work around a
# relatively rare problem. If
#
# - your buildout configuration is trying to install a package that is within
# a namespace package, and
......@@ -264,17 +285,16 @@ else:
# unnecessary for site.py to preprocess these packages, so it should be
# fine, as far as can be guessed as of this writing.) Finally, it
# imports easy_install and runs it.
_easy_install_cmd = _safe_arg('''\
_easy_install_preface = '''\
import sys,os;\
p = sys.path[:];\
import site;\
sys.path[:] = p;\
[sys.modules.pop(k) for k, v in sys.modules.items()\
if hasattr(v, '__path__') and len(v.__path__)==1 and\
not os.path.exists(os.path.join(v.__path__[0],'__init__.py'))];\
from setuptools.command.easy_install import main;\
main()''')
not os.path.exists(os.path.join(v.__path__[0],'__init__.py'))];'''
_easy_install_cmd = (
'from setuptools.command.easy_install import main;main()')
class Installer:
......@@ -321,6 +341,7 @@ class Installer:
self._index_url = index
self._executable = executable
self._has_broken_dash_S = _has_broken_dash_S(self._executable)
if always_unzip is not None:
self._always_unzip = always_unzip
path = (path and path[:] or [])
......@@ -329,6 +350,17 @@ class Installer:
if allowed_eggs_from_site_packages is not None:
self._allowed_eggs_from_site_packages = tuple(
allowed_eggs_from_site_packages)
if self._has_broken_dash_S:
if (not self._include_site_packages or
self._allowed_eggs_from_site_packages != ('*',)):
# We can't do this if the executable has a broken -S.
warnings.warn(BROKEN_DASH_S_WARNING)
self._include_site_packages = True
self._allowed_eggs_from_site_packages = ('*',)
self._easy_install_cmd = _easy_install_preface + _easy_install_cmd
else:
self._easy_install_cmd = _easy_install_cmd
self._easy_install_cmd = _safe_arg(self._easy_install_cmd)
stdlib, self._site_packages = _get_system_paths(executable)
version_info = _get_version_info(executable)
if version_info == sys.version_info:
......@@ -487,7 +519,9 @@ class Installer:
try:
path = setuptools_loc
args = ('-Sc', _easy_install_cmd, '-mUNxd', _safe_arg(tmp))
args = ('-c', self._easy_install_cmd, '-mUNxd', _safe_arg(tmp))
if not self._has_broken_dash_S:
args = ('-S',) + args
if self._always_unzip:
args += ('-Z', )
level = logger.getEffectiveLevel()
......@@ -1176,6 +1210,11 @@ def scripts(reqs, working_set, executable, dest,
_pyscript(spath, sname, executable, rpsetup))
return generated
# We need to give an alternate name to the ``scripts`` function so that it
# can be referenced within sitepackage_safe_scripts, which uses ``scripts``
# as an argument name.
_original_scripts_function = scripts
def sitepackage_safe_scripts(
dest, working_set, executable, site_py_dest,
reqs=(), scripts=None, interpreter=None, extra_paths=(),
......@@ -1188,6 +1227,12 @@ def sitepackage_safe_scripts(
Python site packages, if desired, and choosing to execute the Python's
sitecustomize.
"""
if _has_broken_dash_S(executable):
if not include_site_packages:
warnings.warn(BROKEN_DASH_S_WARNING)
return _original_scripts_function(
reqs, working_set, executable, dest, scripts, extra_paths,
script_arguments, interpreter, initialization, relative_paths)
generated = []
generated.append(_generate_sitecustomize(
site_py_dest, executable, initialization, exec_sitecustomize))
......
src/zc/buildout/testing.py
View file @ 8e902170
......@@ -596,7 +596,7 @@ else:
sep = re.escape(os.path.sep)
normalize_path = (
re.compile(
r'''[^'" \t\n\r]+%(sep)s_[Tt][Ee][Ss][Tt]_%(sep)s([^"' \t\n\r]+)'''
r'''[^'" \t\n\r!]+%(sep)s_[Tt][Ee][Ss][Tt]_%(sep)s([^"' \t\n\r]+)'''
% dict(sep=sep)),
_normalize_path,
)
......
src/zc/buildout/tests.py
View file @ 8e902170
......@@ -3923,14 +3923,29 @@ def test_suite():
setUp=easy_install_SetUp,
tearDown=zc.buildout.testing.buildoutTearDown,
checker=renormalizing.RENormalizing([
zc.buildout.testing.normalize_path,
zc.buildout.testing.normalize_endings,
zc.buildout.testing.normalize_script,
normalize_bang,
(re.compile('Downloading.*setuptools.*egg\n'), ''),
(re.compile('options:'), 'Options:'),
(re.compile('usage:'), 'Usage:'),
]),
zc.buildout.testing.normalize_path,
zc.buildout.testing.normalize_endings,
zc.buildout.testing.normalize_script,
normalize_bang,
(re.compile('Downloading.*setuptools.*egg\n'), ''),
(re.compile('options:'), 'Options:'),
(re.compile('usage:'), 'Usage:'),
]),
))
test_suite.append(doctest.DocFileSuite(
'virtualenv.txt',
setUp=easy_install_SetUp,
tearDown=zc.buildout.testing.buildoutTearDown,
checker=renormalizing.RENormalizing([
zc.buildout.testing.normalize_path,
zc.buildout.testing.normalize_endings,
zc.buildout.testing.normalize_script,
zc.buildout.testing.normalize_egg_py,
(re.compile('(setuptools|distribute)-\S+-'),
'setuptools.egg'),
(re.compile('zc.buildout-\S+-'),
'zc.buildout.egg'),
]),
))
return unittest.TestSuite(test_suite)
src/zc/buildout/virtualenv.txt 0 → 100644
View file @ 8e902170
Version 1.5.0 of buildout (and higher) provides the ability to use
buildout directly with a system Python if you use z3c.recipe.scripts or
other isolation-aware recipes that use the sitepackage_safe_scripts function.
Some people use virtualenv to provide similar functionality.
Unfortunately, a problem with the virtualenv executable as of this
writing means that -S will not work properly with it (see
https://bugs.launchpad.net/virtualenv/+bug/572545). This breaks
buildout's approach to providing isolation.
Because of this, if buildout detects an executable with a broken -S
option, it will revert to its pre-1.5.0 behavior. If buildout has been
asked to provide isolation, it will warn the user that isolation will
not be provided by buildout, but proceed. This should give full
backwards compatibility to virtualenv users.
The only minor annoyance in the future may be recipes that explicitly
use the new buildout functionality to provide isolation: as described
above, the builds will proceed, but users will receive warnings that
buildout is not providing isolation itself. The warnings themselves can
be squelched when running bin/buildout with the ``-s`` option or with a
lower verbosity than usual (e.g., one or more ``-q`` options).
For tests, then, we can examine several things. We'll focus on four.
- Running bootstrap with an executable broken in this way will not try to do
any -S tricks.
- Running sitepackage_safe_scripts with a virtualenv will create an
old-style script. This will affect the bin/buildout script that is
created, for instance. If the sitepackage_safe_scripts function is asked
to provide isolation under these circumstances, it will warn that isolation
will not be available, but still create the desired script.
- Using the easy_install Installer or install or build functions and trying
to request isolation will generate a warning and then the isolation request
will be ignored as it proceeds.
- Passing -s (or -q) to the bin/buildout script will squelch warnings.
Testing these involves first creating a Python that exhibits the same
behavior as the problematic one we care about from virtualenv. Let's do that
first.
>>> import os, sys
>>> py_path, site_packages_path = make_py()
>>> py_file = open(py_path)
>>> py_lines = py_file.readlines()
>>> py_file.close()
>>> py_file = open(py_path, 'w')
>>> extra = '''\
... new_argv = argv[:1]
... for ix, val in enumerate(argv[1:]):
... if val.startswith('--'):
... new_argv.append(val)
... if val.startswith('-') and len(val) > 1:
... if 'S' in val:
... val = val.replace('S', '')
... environ['BROKEN_DASH_S'] = 'Y'
... if val != '-':
... new_argv.append(val)
... if 'c' in val:
... new_argv.extend(argv[ix+2:])
... break
... else:
... new_argv.extend(argv[ix+1:])
... argv = new_argv
... '''
>>> for line in py_lines:
... py_file.write(line)
... if line.startswith('environ = os.environ.copy()'):
... py_file.write(extra)
... print 'Rewritten.'
...
Rewritten.
>>> py_file.close()
>>> sitecustomize_path = join(os.path.dirname(site_packages_path),
... 'parts', 'py', 'sitecustomize.py')
>>> sitecustomize_file = open(sitecustomize_path, 'a')
>>> sitecustomize_file.write('''
... import os, sys
... sys.executable = %r
... if 'BROKEN_DASH_S' in os.environ:
... class ImportHook:
... @staticmethod
... def find_module(fullname, path=None):
... if fullname == 'ConfigParser':
... raise ImportError()
...
... sys.meta_path.append(ImportHook)
... sys.modules.pop('site', None) # Keeps site out of sys.modules.
... # This will be a close-enough approximation of site not being
... # loaded for our tests--it lets us provoke the right errors when
... # the fixes are absent, and works well enough when the fixes are
... # present.
... ''' % (py_path,))
>>> sitecustomize_file.close()
>>> print call_py(
... py_path,
... "import ConfigParser")
<BLANKLINE>
>>> print 'X'; print call_py(
... py_path,
... "import ConfigParser",
... '-S') # doctest: +ELLIPSIS
X...Traceback (most recent call last):
...
ImportError: No module named ConfigParser
<BLANKLINE>
>>> from zc.buildout.easy_install import _has_broken_dash_S
>>> _has_broken_dash_S(py_path)
True
Well, that was ugly, but it seems to have done the trick. The
executable represented by py_path has the same problematic
characteristic as the virtualenv one: -S results in a Python that does
not allow the import of some packages from the standard library. We'll
test with this.
First, let's try running bootstrap.
>>> from os.path import dirname, join
>>> import zc.buildout
>>> bootstrap_py = join(
... dirname(
... dirname(
... dirname(
... dirname(zc.buildout.__file__)
... )
... )
... ),
... 'bootstrap', 'bootstrap.py')
>>> broken_S_buildout = tmpdir('broken_S')
>>> os.chdir(broken_S_buildout)
>>> write('buildout.cfg',
... '''
... [buildout]
... parts =
... ''')
>>> write('bootstrap.py', open(bootstrap_py).read())
>>> print 'X'; print system(
... zc.buildout.easy_install._safe_arg(py_path)+' '+
... 'bootstrap.py'); print 'X' # doctest: +ELLIPSIS
X...
Generated script '/broken_S/bin/buildout'.
...
If bootstrap didn't look out for a broken -S, that would have failed. Moreover,
take a look at bin/buildout:
>>> cat('bin', 'buildout')
#!/executable_buildout/bin/py
<BLANKLINE>
import sys
sys.path[0:0] = [
'/broken_S/eggs/setuptools-0.0-pyN.N.egg',
'/broken_S/eggs/zc.buildout-0.0-pyN.N.egg',
]
<BLANKLINE>
import zc.buildout.buildout
<BLANKLINE>
if __name__ == '__main__':
zc.buildout.buildout.main()
That's the old-style buildout script: no changes for users with this issue.
Of course, they don't get the new features either, presumably because
they don't need or want them. This means that if they use a recipe that
tries to use a new feature, the behavior needs to degrade gracefully.
Here's an example. We'll switch to another buildout in which it is easier to
use local dev versions of zc.buildout and z3c.recipe.scripts.
>>> os.chdir(dirname(dirname(buildout)))
>>> write('buildout.cfg',
... '''
... [buildout]
... parts = eggs
... find-links = %(link_server)s
...
... [primed_python]
... executable = %(py_path)s
...
... [eggs]
... recipe = z3c.recipe.scripts
... python = primed_python
... interpreter = py
... eggs = demo
... ''' % globals())
>>> print system(buildout) # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
Installing eggs.
Getting distribution for 'demo'.
Got demo 0.4c1.
Getting distribution for 'demoneeded'.
Got demoneeded 1.2c1.
Generated script '/sample-buildout/bin/demo'.
Generated interpreter '/sample-buildout/bin/py'.
...UserWarning: Buildout has been asked to exclude or limit site-packages
so that builds can be repeatable when using a system Python. However,
the chosen Python executable has a broken implementation of -S (see
https://bugs.launchpad.net/virtualenv/+bug/572545 for an example
problem) and this breaks buildout's ability to isolate site-packages.
If the executable already has a clean site-packages (e.g., using
virtualenv's ``--no-site-packages`` option) you may be getting
equivalent repeatability. To silence this warning, use the -s argument
to the buildout script. Alternatively, use a Python executable with a
working -S (such as a standard Python binary).
warnings.warn(BROKEN_DASH_S_WARNING)
<BLANKLINE>
So, it did what we asked as best it could, but gave a big warning. If
you don't want those warnings for those particular recipes that use the
new features, you can use the "-s" option to squelch the warnings.
>>> print system(buildout + ' -s')
Updating eggs.
<BLANKLINE>
A lower verbosity (one or more -q options) also quiets the warning.
>>> print system(buildout + ' -q')
<BLANKLINE>
Notice that, as we saw before with bin/buildout, the generated scripts
are old-style, because the new-style feature gracefully degrades to the
previous implementation when it encounters an executable with a broken
dash-S.
>>> print 'X'; cat('bin', 'py') # doctest: +ELLIPSIS
X...
<BLANKLINE>
import sys
<BLANKLINE>
sys.path[0:0] = [
'/sample-buildout/eggs/demo-0.4c1-pyN.N.egg',
'/sample-buildout/eggs/demoneeded-1.2c1-pyN.N.egg',
]
...
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7