Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
S
slapos.buildout
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
6
Merge Requests
6
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
nexedi
slapos.buildout
Commits
f5a2a7c1
Commit
f5a2a7c1
authored
2 years ago
by
Julien Muchembled
Committed by
Xavier Thompson
1 month ago
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
[feat] easy_install: add slapos.libnetworkcache support
parent
c1320b90
No related merge requests found
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
91 additions
and
24 deletions
+91
-24
src/zc/buildout/easy_install.py
src/zc/buildout/easy_install.py
+89
-22
src/zc/buildout/tests/easy_install.txt
src/zc/buildout/tests/easy_install.txt
+2
-2
No files found.
src/zc/buildout/easy_install.py
View file @
f5a2a7c1
...
...
@@ -43,6 +43,8 @@ import zc.buildout.rmtree
from
zc.buildout
import
WINDOWS
from
zc.buildout
import
PY3
import
warnings
from
contextlib
import
closing
from
setuptools.package_index
import
distros_for_location
,
URL_SCHEME
import
csv
try
:
...
...
@@ -113,6 +115,9 @@ python_lib = distutils.sysconfig.get_python_lib()
FILE_SCHEME
=
re
.
compile
(
'file://'
,
re
.
I
).
match
DUNDER_FILE_PATTERN
=
re
.
compile
(
r"__file__ = '(?P<filename>.+)'$"
)
networkcache_key
=
'pypi:{}={}'
.
format
class
_Monkey
(
object
):
def
__init__
(
self
,
module
,
**
kw
):
mdict
=
self
.
_mdict
=
module
.
__dict__
...
...
@@ -466,32 +471,71 @@ class Installer(object):
finally
:
zc
.
buildout
.
rmtree
.
rmtree
(
tmp
)
def
_obtain
(
self
,
requirement
,
source
=
None
):
def
_obtain
(
self
,
requirement
,
source
=
None
,
networkcache_failed
=
False
):
# get the non-patched version
req
=
str
(
requirement
)
if
PATCH_MARKER
in
req
:
requirement
=
pkg_resources
.
Requirement
.
parse
(
re
.
sub
(
orig_versions_re
,
''
,
req
))
# initialize out index for this project:
wheel
=
getattr
(
requirement
,
'wheel'
,
False
)
def
filter_precedence
(
dist
):
return
(
dist
.
precedence
==
WHL_DIST
)
==
wheel
and
(
dist
.
precedence
==
pkg_resources
.
SOURCE_DIST
if
source
else
not
(
dist
.
precedence
==
pkg_resources
.
DEVELOP_DIST
and
{
'setup.py'
,
'pyproject.toml'
}.
isdisjoint
(
os
.
listdir
(
dist
.
location
)))
)
index
=
self
.
_index
if
not
networkcache_failed
:
try
:
(
operator
,
version
,),
=
requirement
.
specs
except
ValueError
:
pass
else
:
# Network cache is not expected to contain all versions so it
# couldn't tell whether a found version is the best existing
# one. Therefore, it's only accessed when we have a
# specification for a single version, which is anyway enough
# for our usage (picked versions not allowed).
if
operator
==
'=='
:
# But first, avoid any network access by checking local
# urls. PackageIndex.add_find_links scans them immediately.
dists
=
[
dist
for
dist
in
index
[
requirement
.
project_name
]
if
dist
in
requirement
and
filter_precedence
(
dist
)
and
(
FILE_SCHEME
(
dist
.
location
)
or
not
URL_SCHEME
(
dist
.
location
))]
if
dists
:
return
max
(
dists
)
from
.buildout
import
networkcache_client
as
nc
if
nc
:
key
=
networkcache_key
(
requirement
.
key
,
version
)
if
nc
.
tryDownload
(
key
):
with
nc
:
for
entry
in
nc
.
select
(
key
):
basename
=
entry
[
'basename'
]
for
dist
in
distros_for_location
(
entry
[
'sha512'
],
basename
):
# The version comparison is to keep
# the one that's correctly parsed by
# distros_for_location.
if
(
dist
.
version
==
version
and
self
.
_env
.
can_add
(
dist
)
and
filter_precedence
(
dist
)):
dist
.
networkcache
=
(
basename
,
requirement
,
source
)
dists
.
append
(
dist
)
if
dists
:
return
max
(
dists
)
# initialize out index for this project:
if
index
.
obtain
(
requirement
)
is
None
:
# Nothing is available.
return
None
# Filter the available dists for the requirement and source flag
wheel
=
getattr
(
requirement
,
'wheel'
,
False
)
dists
=
[
dist
for
dist
in
index
[
requirement
.
project_name
]
if
((
dist
in
requirement
)
and
(
dist
.
precedence
==
WHL_DIST
)
==
wheel
and
(
dist
.
precedence
==
pkg_resources
.
SOURCE_DIST
if
source
else
not
(
dist
.
precedence
==
pkg_resources
.
DEVELOP_DIST
and
{
'setup.py'
,
'pyproject.toml'
}.
isdisjoint
(
os
.
listdir
(
dist
.
location
))
)
)
)
]
if
dist
in
requirement
and
filter_precedence
(
dist
)]
# If we prefer final dists, filter for final and use the
# result if it is non empty.
...
...
@@ -528,18 +572,41 @@ class Installer(object):
):
return
dist
best
.
sort
()
return
best
[
-
1
]
return
max
(
best
)
def
_fetch
(
self
,
dist
,
tmp
,
download_cache
):
if
(
download_cache
and
(
realpath
(
os
.
path
.
dirname
(
dist
.
location
))
==
download_cache
)
):
logger
.
debug
(
"Download cache has %s at: %s"
,
dist
,
dist
.
location
)
return
dist
from
.buildout
import
networkcache_client
as
nc
while
hasattr
(
dist
,
'networkcache'
):
basename
,
requirement
,
source
=
dist
.
networkcache
new_location
=
os
.
path
.
join
(
tmp
,
basename
)
with
nc
,
closing
(
nc
.
download
(
dist
.
location
))
as
src
,
\
open
(
new_location
,
'wb'
)
as
dst
:
shutil
.
copyfileobj
(
src
,
dst
)
break
# Downloading content from network cache failed: let's resume index
# lookup to get a fallback url. This will respect _satisfied()
# decision because the specification is for a single version.
dist
=
self
.
_obtain
(
requirement
,
source
,
networkcache_failed
=
True
)
if
dist
is
None
:
raise
zc
.
buildout
.
UserError
(
"Couldn't find a distribution for %r."
%
str
(
requirement
))
else
:
if
(
download_cache
and
(
realpath
(
os
.
path
.
dirname
(
dist
.
location
))
==
download_cache
)
):
logger
.
debug
(
"Download cache has %s at: %s"
,
dist
,
dist
.
location
)
return
dist
logger
.
debug
(
"Fetching %s from: %s"
,
dist
,
dist
.
location
)
new_location
=
self
.
_index
.
download
(
dist
.
location
,
tmp
)
if
nc
:
key
=
networkcache_key
(
dist
.
key
,
dist
.
version
)
if
nc
.
tryUpload
(
key
):
with
nc
,
open
(
new_location
,
'rb'
)
as
f
:
nc
.
upload
(
f
,
key
,
basename
=
os
.
path
.
basename
(
new_location
))
logger
.
debug
(
"Fetching %s from: %s"
,
dist
,
dist
.
location
)
new_location
=
self
.
_index
.
download
(
dist
.
location
,
tmp
)
if
(
download_cache
and
(
realpath
(
new_location
)
==
realpath
(
dist
.
location
))
and
os
.
path
.
isfile
(
new_location
)
...
...
This diff is collapsed.
Click to expand it.
src/zc/buildout/tests/easy_install.txt
View file @
f5a2a7c1
...
...
@@ -1451,9 +1451,8 @@ Now when we install the distributions:
... ['demo==0.2'], dest,
... links=[link_server], index=link_server+'index/')
GET 200 /
GET 404 /index/demo/
GET 200 /index/
GET 404 /index/demoneeded/
GET 200 /index/
>>> zc.buildout.easy_install.build(
... 'extdemo', dest,
...
...
@@ -1475,6 +1474,7 @@ from the link server:
>>> ws = zc.buildout.easy_install.install(
... ['demo'], dest,
... links=[link_server], index=link_server+'index/')
GET 404 /index/demo/
GET 200 /demo-0.3-py2.4.egg
Normally, the download cache is the preferred source of downloads, but
...
...
This diff is collapsed.
Click to expand it.
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment