Skip to content

S
slapos.buildout
Commit 7e825d61 authored by Gary Poster's avatar Gary Poster
Browse files
Options

simplify resulting site.py function

parent 8f1f787f
Show whitespace changes
Inline Side-by-side
Showing with 202 additions and 271 deletions
src/zc/buildout/easy_install.py
View file @ 7e825d61
......@@ -1390,69 +1390,66 @@ def _generate_site(dest, working_set, executable, extra_paths=(),
rpsetup = '\n'.join(
[(line and ' %s' % (line,) or line)
for line in rpsetup.split('\n')])
real_site_path = _get_module_file(executable, 'site')
real_site = open(real_site_path, 'r')
site = open(site_path, 'w')
extra_path_snippet = add_site_packages_snippet[add_site_packages]
extra_path_snippet_followup = add_site_packages_snippet_followup[
add_site_packages]
namespace_setup = ''
addsitedir = addsitedir_snippet
if add_site_packages:
stdlib, site_paths = _get_system_paths(executable)
extra_path_snippet = extra_path_snippet % _format_paths(
(repr(p) for p in site_paths), 2)
path_string = ''.join([
path_string,
(",\n"
" # These are the underlying Python's site-packages.\n"
" "),
_format_paths((repr(p) for p in site_paths), 2)])
distribution = working_set.find(
pkg_resources.Requirement.parse('setuptools'))
if distribution is not None:
# We need to worry about namespace packages.
namespace_setup = namespace_add_site_packages_setup % (
distribution.location,)
addsitedir = addsitedir_namespace_add_site_packages_snippet
addsitepackages_marker = 'def addsitepackages('
enableusersite_marker = 'ENABLE_USER_SITE = '
successful_rewrite = False
real_site_path = _get_module_file(executable, 'site')
real_site = open(real_site_path, 'r')
site = open(site_path, 'w')
try:
for line in real_site.readlines():
if line.startswith(enableusersite_marker):
site.write(enableusersite_marker)
site.write('False # buildout does not support user sites.\n')
elif line.startswith(addsitepackages_marker):
site.write(addsitepackages_script % (
extra_path_snippet, rpsetup, path_string,
extra_path_snippet_followup))
namespace_setup, rpsetup, path_string, addsitedir))
site.write(line[len(addsitepackages_marker):])
successful_rewrite = True
else:
site.write(line)
finally:
site.close()
real_site.close()
if not successful_rewrite:
raise RuntimeError('Buildout did not successfully rewrite site.py')
return site_path
add_site_packages_snippet = ['''
paths = []''', '''
paths = [ # These are the underlying Python's site-packages.
%s]
sys.path[0:0] = paths
known_paths.update([os.path.normcase(os.path.abspath(p)) for p in paths])
try:
import pkg_resources
except ImportError:
# No namespace packages in sys.path; no fixup needed.
pkg_resources = None''']
add_site_packages_snippet_followup = ['', '''
if pkg_resources is not None:
# There may be namespace packages in sys.path. This is much faster
# than importing pkg_resources after the sys.path has a large number
# of eggs.
for p in sys.path:
pkg_resources.fixup_namespace_packages(p)''']
namespace_add_site_packages_setup = '''
setuptools_path = %r
sys.path.append(setuptools_path)
known_paths.add(setuptools_path)
import pkg_resources'''
addsitepackages_script = '''\
def addsitepackages(known_paths):%s
%s paths[0:0] = [ # eggs
%s
]
# Process all dirs. Look for .pth files. If they exist, defer
# processing "import" varieties.
addsitedir_snippet = '''
for path in paths:
addsitedir(path, known_paths)'''
addsitedir_namespace_add_site_packages_snippet = '''
dotpth = os.extsep + "pth"
deferred = []
for path in reversed(paths):
# Duplicating addsitedir.
for path in paths:
# This duplicates addsitedir except for adding the pkg_resources call.
sitedir, sitedircase = makepath(path)
if not sitedircase in known_paths and os.path.exists(sitedir):
sys.path.insert(0, sitedir)
sys.path.append(sitedir)
pkg_resources.working_set.add_entry(sitedir)
known_paths.add(sitedircase)
try:
names = os.listdir(sitedir)
......@@ -1461,44 +1458,18 @@ def addsitepackages(known_paths):%s
names = [name for name in names if name.endswith(dotpth)]
names.sort()
for name in names:
# Duplicating addpackage.
fullname = os.path.join(sitedir, name)
try:
f = open(fullname, "rU")
except IOError:
continue
try:
for line in f:
if line.startswith("#"):
continue
if (line.startswith("import ") or
line.startswith("import\t")):
# This line is supposed to be executed. It
# might be a setuptools namespace package
# installed with a system package manager.
# Defer this so we can process egg namespace
# packages first, or else the eggs with the same
# namespace will be ignored.
deferred.append((sitedir, name, fullname, line))
continue
line = line.rstrip()
dir, dircase = makepath(sitedir, line)
if not dircase in known_paths and os.path.exists(dir):
sys.path.append(dir)
known_paths.add(dircase)
finally:
f.close()%s
# Process "import ..." .pth lines.
for sitedir, name, fullname, line in deferred:
# Note that some lines--such as the one setuptools writes for
# namespace packages--expect some or all of sitedir, name, and
# fullname to be present in the frame locals, as it is in
# ``addpackage``.
try:
exec line
except:
print "Error in %%s" %% (fullname,)
raise
addpackage(sitedir, name, known_paths)'''
addsitepackages_script = '''\
def addsitepackages(known_paths):
"""Add site packages.
This function is written by buildout. See original_addsitepackages,
below, for the original version."""%s
%s paths = [
# Eggs.
%s
]%s
global addsitepackages
addsitepackages = original_addsitepackages
return known_paths
......
src/zc/buildout/easy_install.txt
View file @ 7e825d61
......@@ -1020,66 +1020,17 @@ following shows the part that buildout inserts.
... # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
#...
def addsitepackages(known_paths):
paths = []
paths[0:0] = [ # eggs
"""Add site packages.
<BLANKLINE>
This function is written by buildout. See original_addsitepackages,
below, for the original version."""
paths = [
# Eggs.
'/interpreter/eggs/demo-0.3-pyN.N.egg',
'/interpreter/eggs/demoneeded-1.1-pyN.N.egg'
]
# Process all dirs. Look for .pth files. If they exist, defer
# processing "import" varieties.
dotpth = os.extsep + "pth"
deferred = []
for path in reversed(paths):
# Duplicating addsitedir.
sitedir, sitedircase = makepath(path)
if not sitedircase in known_paths and os.path.exists(sitedir):
sys.path.insert(0, sitedir)
known_paths.add(sitedircase)
try:
names = os.listdir(sitedir)
except os.error:
continue
names = [name for name in names if name.endswith(dotpth)]
names.sort()
for name in names:
# Duplicating addpackage.
fullname = os.path.join(sitedir, name)
try:
f = open(fullname, "rU")
except IOError:
continue
try:
for line in f:
if line.startswith("#"):
continue
if (line.startswith("import ") or
line.startswith("import ")):
# This line is supposed to be executed. It
# might be a setuptools namespace package
# installed with a system package manager.
# Defer this so we can process egg namespace
# packages first, or else the eggs with the same
# namespace will be ignored.
deferred.append((sitedir, name, fullname, line))
continue
line = line.rstrip()
dir, dircase = makepath(sitedir, line)
if not dircase in known_paths and os.path.exists(dir):
sys.path.append(dir)
known_paths.add(dircase)
finally:
f.close()
# Process "import ..." .pth lines.
for sitedir, name, fullname, line in deferred:
# Note that some lines--such as the one setuptools writes for
# namespace packages--expect some or all of sitedir, name, and
# fullname to be present in the frame locals, as it is in
# ``addpackage``.
try:
exec line
except:
print "Error in %s" % (fullname,)
raise
for path in paths:
addsitedir(path, known_paths)
global addsitepackages
addsitepackages = original_addsitepackages
return known_paths
......@@ -1088,12 +1039,6 @@ following shows the part that buildout inserts.
<BLANKLINE>
def original_addsitepackages(known_paths):...
As you can see, it manipulates the path to insert the eggs and then processes
any .pth files. The lines in the .pth files that use the "import" feature
are deferred because it is a pattern we will need in a later example, when we
show how we can add site packages, and handle competing namespace packages
in both site packages and eggs.
Here are some examples of the interpreter in use.
>>> print call_py(interpreter_path, "print 16+26")
......@@ -1102,10 +1047,10 @@ Here are some examples of the interpreter in use.
>>> res = call_py(interpreter_path, "import sys; print sys.path")
>>> print res # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
['',
'/interpreter/eggs/demo-0.3-pyN.N.egg',
'/interpreter/eggs/demoneeded-1.1-pyN.N.egg',
'/interpreter/parts/interpreter',
...]
...,
'/interpreter/eggs/demo-0.3-pyN.N.egg',
'/interpreter/eggs/demoneeded-1.1-pyN.N.egg']
<BLANKLINE>
>>> clean_paths = eval(res.strip()) # This is used later for comparison.
......@@ -1167,7 +1112,10 @@ paths join a base to a path, as with the use of this argument in the
>>> sys.stdout.write('#\n'); cat(site_path) # doctest: +ELLIPSIS
#...
def addsitepackages(known_paths):
paths = []
"""Add site packages.
<BLANKLINE>
This function is written by buildout. See original_addsitepackages,
below, for the original version."""
<BLANKLINE>
import os
<BLANKLINE>
......@@ -1175,7 +1123,8 @@ paths join a base to a path, as with the use of this argument in the
base = os.path.dirname(os.path.abspath(os.path.realpath(__file__)))
base = os.path.dirname(base)
base = os.path.dirname(base)
paths[0:0] = [ # eggs
paths = [
# Eggs.
join(base, 'eggs/demo-0.3-pyN.N.egg'),
join(base, 'eggs/demoneeded-1.1-pyN.N.egg')
]...
......@@ -1186,10 +1135,10 @@ The paths resolve in practice as you would expect.
... "import sys, pprint; pprint.pprint(sys.path)")
... # doctest: +ELLIPSIS
['',
'/interpreter/eggs/demo-0.3-py2.4.egg',
'/interpreter/eggs/demoneeded-1.1-py2.4.egg',
'/interpreter/parts/interpreter',
...]
...,
'/interpreter/eggs/demo-0.3-pyN.N.egg',
'/interpreter/eggs/demoneeded-1.1-pyN.N.egg']
<BLANKLINE>
The ``extra_paths`` argument affects the path in site.py. Notice that
......@@ -1203,8 +1152,12 @@ The ``extra_paths`` argument affects the path in site.py. Notice that
>>> sys.stdout.write('#\n'); cat(site_path) # doctest: +ELLIPSIS
#...
def addsitepackages(known_paths):
paths = []
paths[0:0] = [ # eggs
"""Add site packages.
<BLANKLINE>
This function is written by buildout. See original_addsitepackages,
below, for the original version."""
paths = [
# Eggs.
'/interpreter/eggs/demo-0.3-pyN.N.egg',
'/interpreter/eggs/demoneeded-1.1-pyN.N.egg',
'/interpreter/other'
......@@ -1214,11 +1167,11 @@ The ``extra_paths`` argument affects the path in site.py. Notice that
... "import sys, pprint; pprint.pprint(sys.path)")
... # doctest: +ELLIPSIS
['',
'/interpreter/parts/interpreter',
...,
'/interpreter/eggs/demo-0.3-pyN.N.egg',
'/interpreter/eggs/demoneeded-1.1-pyN.N.egg',
'/interpreter/other',
'/interpreter/parts/interpreter',
...]
'/interpreter/other']
<BLANKLINE>
The ``generate_scripts`` function: using site-packages
......@@ -1249,10 +1202,8 @@ and the Python you use is not managed precisely by your application (for
instance, it is a system Python), you open yourself up to these
possibilities. Don't be unaware of the dangers.
That explained, let's see how it works. Unfortunately, because of how
setuptools namespace packages are implemented differently for operating
system packages (debs or rpms) and normal installation, there's a tricky
dance.
That explained, let's see how it works. If you don't use namespace packages,
this is very straightforward.
>>> reset_interpreter()
>>> generated = zc.buildout.easy_install.generate_scripts(
......@@ -1262,28 +1213,88 @@ dance.
... # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
#...
def addsitepackages(known_paths):
paths = [ # These are the underlying Python's site-packages.
'...']
sys.path[0:0] = paths
known_paths.update([os.path.normcase(os.path.abspath(p)) for p in paths])
try:
"""Add site packages.
<BLANKLINE>
This function is written by buildout. See original_addsitepackages,
below, for the original version."""
paths = [
# Eggs.
'/interpreter/eggs/demo-0.3-pyN.N.egg',
'/interpreter/eggs/demoneeded-1.1-pyN.N.egg',
# These are the underlying Python's site-packages.
...
]...
It simply adds the site-packages after the eggs.
Here's an example of the new script in use. Other documents and tests in
this package give the feature a more thorough workout, but this should
give you an idea of the feature.
>>> res = call_py(interpreter_path, "import sys; print sys.path")
>>> print res # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
['',
'/interpreter/parts/interpreter',
...,
'/interpreter/eggs/demo-0.3-py2.4.egg',
'/interpreter/eggs/demoneeded-1.1-py2.4.egg',
...]
<BLANKLINE>
The clean_paths gathered earlier is a subset of this full list of paths.
>>> full_paths = eval(res.strip())
>>> len(clean_paths) < len(full_paths)
True
>>> set(os.path.normpath(p) for p in clean_paths).issubset(
... os.path.normpath(p) for p in full_paths)
True
Unfortunately, because of how setuptools namespace packages are implemented
differently for operating system packages (debs or rpms) as opposed to
standard setuptools installation, there's a slightly trickier dance if you
use them. To show this we'll needs some extra eggs that use namespaces.
We'll use the ``tellmy.fortune`` package, which we'll need to make an initial
call to another text fixture to create.
>>> from zc.buildout.tests import create_sample_namespace_eggs
>>> namespace_eggs = tmpdir('namespace_eggs')
>>> create_sample_namespace_eggs(namespace_eggs)
>>> ws = zc.buildout.easy_install.install(
... ['demo', 'tellmy.fortune'], join(interpreter_dir, 'eggs'),
... links=[link_server, namespace_eggs], index=link_server+'index/')
>>> generated = zc.buildout.easy_install.generate_scripts(
... interpreter_bin_dir, ws, sys.executable, interpreter_parts_dir,
... interpreter='py', add_site_packages=True)
>>> sys.stdout.write('#\n'); cat(site_path)
... # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
#...
def addsitepackages(known_paths):
"""Add site packages.
<BLANKLINE>
This function is written by buildout. See original_addsitepackages,
below, for the original version."""
setuptools_path = '...setuptools...'
sys.path.append(setuptools_path)
known_paths.add(setuptools_path)
import pkg_resources
except ImportError:
# No namespace packages in sys.path; no fixup needed.
pkg_resources = None
paths[0:0] = [ # eggs
paths = [
# Eggs.
'/interpreter/eggs/demo-0.3-pyN.N.egg',
'/interpreter/eggs/demoneeded-1.1-pyN.N.egg'
'/interpreter/eggs/tellmy.fortune-1.0-pyN.N.egg',
'...setuptools...',
'/interpreter/eggs/demoneeded-1.1-pyN.N.egg',
# These are the underlying Python's site-packages.
...
]
# Process all dirs. Look for .pth files. If they exist, defer
# processing "import" varieties.
dotpth = os.extsep + "pth"
deferred = []
for path in reversed(paths):
# Duplicating addsitedir.
for path in paths:
# This duplicates addsitedir except for adding the pkg_resources call.
sitedir, sitedircase = makepath(path)
if not sitedircase in known_paths and os.path.exists(sitedir):
sys.path.insert(0, sitedir)
sys.path.append(sitedir)
pkg_resources.working_set.add_entry(sitedir)
known_paths.add(sitedircase)
try:
names = os.listdir(sitedir)
......@@ -1292,50 +1303,7 @@ dance.
names = [name for name in names if name.endswith(dotpth)]
names.sort()
for name in names:
# Duplicating addpackage.
fullname = os.path.join(sitedir, name)
try:
f = open(fullname, "rU")
except IOError:
continue
try:
for line in f:
if line.startswith("#"):
continue
if (line.startswith("import ") or
line.startswith("import ")):
# This line is supposed to be executed. It
# might be a setuptools namespace package
# installed with a system package manager.
# Defer this so we can process egg namespace
# packages first, or else the eggs with the same
# namespace will be ignored.
deferred.append((sitedir, name, fullname, line))
continue
line = line.rstrip()
dir, dircase = makepath(sitedir, line)
if not dircase in known_paths and os.path.exists(dir):
sys.path.append(dir)
known_paths.add(dircase)
finally:
f.close()
if pkg_resources is not None:
# There may be namespace packages in sys.path. This is much faster
# than importing pkg_resources after the sys.path has a large number
# of eggs.
for p in sys.path:
pkg_resources.fixup_namespace_packages(p)
# Process "import ..." .pth lines.
for sitedir, name, fullname, line in deferred:
# Note that some lines--such as the one setuptools writes for
# namespace packages--expect some or all of sitedir, name, and
# fullname to be present in the frame locals, as it is in
# ``addpackage``.
try:
exec line
except:
print "Error in %s" % (fullname,)
raise
addpackage(sitedir, name, known_paths)
global addsitepackages
addsitepackages = original_addsitepackages
return known_paths
......@@ -1344,35 +1312,23 @@ dance.
<BLANKLINE>
def original_addsitepackages(known_paths):...
As you can see, the script now first tries to import pkg_resources. If it
exists, then we need to process egg files specially to look for namespace
packages there *before* we process process lines in .pth files that use the
"import" feature--lines that might be part of the setuptools namespace
package implementation for system packages, as mentioned above, and that
must come after processing egg namespaces.
Here's an example of the new script in use. Other documents and tests in
this package give the feature a more thorough workout, but this should
give you an idea of the feature.
>>> res = call_py(interpreter_path, "import sys; print sys.path")
>>> print res # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
>>> print call_py(interpreter_path, "import sys; print sys.path")
... # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
['',
'/interpreter/eggs/demo-0.3-py2.4.egg',
'/interpreter/eggs/demoneeded-1.1-py2.4.egg',
'...',
'/interpreter/parts/interpreter',
...,
'...setuptools...',
'/interpreter/eggs/demo-0.3-pyN.N.egg',
'/interpreter/eggs/tellmy.fortune-1.0-pyN.N.egg',
'/interpreter/eggs/demoneeded-1.1-pyN.N.egg',
...]
<BLANKLINE>
The clean_paths gathered earlier is a subset of this full list of paths.
>>> full_paths = eval(res.strip())
>>> len(clean_paths) < len(full_paths)
True
>>> set(os.path.normpath(p) for p in clean_paths).issubset(
... os.path.normpath(p) for p in full_paths)
True
As you can see, the script now first imports pkg_resources. Then we
need to process egg files specially to look for namespace packages there
*before* we process process lines in .pth files that use the "import"
feature--lines that might be part of the setuptools namespace package
implementation for system packages, as mentioned above, and that must
come after processing egg namespaces.
The ``exec_sitecustomize`` argument does the same thing for the
sitecustomize module--it allows you to include the code from the
......@@ -1392,6 +1348,9 @@ files then initialize the Python environment as we have already seen. Let's
see a simple example.
>>> reset_interpreter()
>>> ws = zc.buildout.easy_install.install(
... ['demo'], join(interpreter_dir, 'eggs'), links=[link_server],
... index=link_server+'index/')
>>> generated = zc.buildout.easy_install.generate_scripts(
... interpreter_bin_dir, ws, sys.executable, interpreter_parts_dir,
... reqs=['demo'])
......
src/zc/buildout/tests.py
View file @ 7e825d61
......@@ -2890,8 +2890,12 @@ def increment_on_command_line():
######################################################################
def make_py_with_system_install(make_py, sample_eggs):
from zc.buildout.testing import write, mkdir
py_path, site_packages_path = make_py()
create_sample_namespace_eggs(sample_eggs, site_packages_path)
return py_path
def create_sample_namespace_eggs(dest, site_packages_path=None):
from zc.buildout.testing import write, mkdir
for pkg, version in (('version', '1.0'), ('version', '1.1'),
('fortune', '1.0')):
tmp = tempfile.mkdtemp()
......@@ -2918,14 +2922,13 @@ def make_py_with_system_install(make_py, sample_eggs):
" author='bob', url='bob', author_email='bob')\n"
% locals()
)
zc.buildout.testing.sdist(tmp, sample_eggs)
if pkg == 'version' and version == '1.1':
zc.buildout.testing.sdist(tmp, dest)
if (site_packages_path and pkg == 'version' and version == '1.1'):
# We install the 1.1 version in site packages the way a
# system packaging system (debs, rpms) would do it.
zc.buildout.testing.sys_install(tmp, site_packages_path)
finally:
shutil.rmtree(tmp)
return py_path
def create_sample_eggs(test, executable=sys.executable):
write = test.globs['write']
......
z3c.recipe.scripts_/src/z3c/recipe/scripts/README.txt
View file @ 7e825d61
......@@ -168,9 +168,8 @@ provided.
Here's an example of using the generated interpreter.
>>> print system(join(sample_buildout, 'bin', 'py') +
... ' -c "import sys, pprint; pprint.pprint(sys.path[:3])"')
['',
'/sample-buildout/eggs/demo-0.2-pyN.N.egg',
... ' -c "import sys, pprint; pprint.pprint(sys.path[-2:])"')
['/sample-buildout/eggs/demo-0.2-pyN.N.egg',
'/sample-buildout/eggs/demoneeded-1.2c1-pyN.N.egg']
<BLANKLINE>
......@@ -241,13 +240,12 @@ Now let's take a look at add-site-packages.
... ''' -c "import sys, pprint; pprint.pprint(sys.path)"''')
... # doctest: +ELLIPSIS
['',
'/sample-buildout/parts/py',
...,
'/sample-buildout/eggs/demo-0.2-pyN.N.egg',
'/sample-buildout/eggs/demoneeded-1.2c1-pyN.N.egg',
'/executable_buildout/eggs/setuptools-0.6c11-pyN.N.egg',
'/executable_buildout/site-packages',
'/sample-buildout/parts/py',
'/executable_buildout/parts/py',
...]
'/executable_buildout/eggs/setuptools-X-pyN.N.egg',
'/executable_buildout/site-packages']
<BLANKLINE>
Next we will use the exec-sitecustomize option. It simply copies
......@@ -327,9 +325,8 @@ Now let's put it in action.
Generated interpreter '/sample-buildout/bin/python'.
>>> print system(join(sample_buildout, 'bin', 'python') +
... ' -c "import sys, pprint; pprint.pprint(sys.path[:3])"')
['',
'/sample-buildout/eggs/demo-0.2-pyN.N.egg',
... ' -c "import sys, pprint; pprint.pprint(sys.path[-2:])"')
['/sample-buildout/eggs/demo-0.2-pyN.N.egg',
'/sample-buildout/eggs/demoneeded-1.2c1-pyN.N.egg']
<BLANKLINE>
>>> print system(join(sample_buildout, 'bin', 'python') +
......
z3c.recipe.scripts_/src/z3c/recipe/scripts/tests.py
View file @ 7e825d61
......@@ -226,7 +226,7 @@ Let's look at the site.py that was generated:
... # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
#...
def addsitepackages(known_paths):
paths = []
"..."
<BLANKLINE>
import os
<BLANKLINE>
......@@ -234,7 +234,8 @@ Let's look at the site.py that was generated:
base = os.path.dirname(os.path.abspath(os.path.realpath(__file__)))
base = os.path.dirname(base)
base = os.path.dirname(base)
paths[0:0] = [ # eggs
paths = [
# Eggs.
'/foo/bar',
join(base, 'spam')
]...
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7