Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
S
slapos.buildout
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
6
Merge Requests
6
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
nexedi
slapos.buildout
Commits
fb3af4fc
Commit
fb3af4fc
authored
May 11, 2011
by
Lucas Carvalho
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Cleanup the code to use the libnetworkcache.py
parent
66d44d8f
Changes
1
Show whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
23 additions
and
55 deletions
+23
-55
src/zc/buildout/networkcache.py
src/zc/buildout/networkcache.py
+23
-55
No files found.
src/zc/buildout/networkcache.py
View file @
fb3af4fc
...
...
@@ -14,10 +14,9 @@
import
os
import
urllib
import
posixpath
import
base64
import
hashlib
from
libnetworkcache
import
NetworkcacheClient
def
_get_hash_from_file
(
path
,
hash_object
):
...
...
@@ -35,26 +34,16 @@ def _get_hash_from_file(path, hash_object):
f
.
close
()
def
_get_sha256_from_file
(
path
):
"""
Return the sha256sum from file.
"""
if
not
os
.
path
.
isfile
(
path
):
raise
ValueError
,
'Could not extract sha256sum.'
\
' The path is not a file: %s '
%
path
return
_get_hash_from_file
(
path
,
hashlib
.
sha256
())
def
check_sha256sum
(
path
,
sha256sum
):
"""Tell whether the SHA256sum checksum of the file at path matches
"""
return
sha256sum
==
_get_hash_from_file
(
path
,
hashlib
.
sha256
())
def
_update_network_cached_log
(
sha256sum
,
url
):
def
_update_network_cached_log
(
key
,
url
):
"""Update the networkcached.log file.
Adding the last
sha256sum
and URL which has been retrieved from networkcached server.
Adding the last
key
and URL which has been retrieved from networkcached server.
"""
# Keep the log
...
...
@@ -71,28 +60,12 @@ def _update_network_cached_log(sha256sum, url):
break
if
not
flag
:
f
.
seek
(
0
,
2
)
f
.
write
(
'%s %s
\
n
'
%
(
sha256sum
,
url
))
f
.
write
(
'%s %s
\
n
'
%
(
key
,
url
))
f
.
truncate
(
f
.
tell
())
finally
:
f
.
close
()
def
get_sha256sum_from_networkcached
(
network_cache
,
url
,
logger
):
"""
Return the sha256sum if the url exists in networkcached server.
"""
network_cached_url
=
os
.
path
.
join
(
network_cache
,
'get'
,
base64
.
encodestring
(
url
))
try
:
result
=
urllib
.
urlopen
(
network_cached_url
)
if
int
(
result
.
code
)
==
200
:
return
result
.
read
().
strip
()
logger
.
info
(
'The url is not cached yet: %s'
%
url
)
except
IOError
,
e
:
logger
.
info
(
'An error occurred to get sha256sum of url %s. %s'
%
\
(
url
,
str
(
e
)))
def
download_network_cached
(
network_cache
,
path
,
url
,
logger
):
"""Download from a network cache provider
...
...
@@ -105,16 +78,18 @@ def download_network_cached(network_cache, path, url, logger):
# Not able to use network cache
return
False
sha256sum
=
get_sha256sum_from_networkcached
(
network_cache
,
url
,
logger
)
if
sha256sum
is
None
:
return
False
network_cached_url
=
os
.
path
.
join
(
network_cache
,
sha256sum
)
key
=
hashlib
.
sha256
(
url
).
hexdigest
()
network_cached_url
=
os
.
path
.
join
(
network_cache
,
key
)
logger
.
info
(
'Downloading from network cache %s'
%
network_cached_url
)
try
:
path
,
headers
=
urllib
.
urlretrieve
(
network_cached_url
,
path
)
if
not
check_sha256sum
(
path
,
sha256sum
):
logger
.
info
(
'MD5/SHA256 checksum mismatch downloading %r'
%
\
nc
=
NetworkcacheClient
(
network_cache
)
result
,
path
=
nc
.
get
(
key
,
path
)
if
result
.
status
!=
200
:
logger
.
info
(
'Fail to download from network cache: File Not Found.'
)
return
False
if
not
check_sha256sum
(
path
,
result
.
getheader
(
'x-sha256sum'
)):
logger
.
info
(
'SHA256 checksum mismatch downloading %r'
%
\
network_cached_url
)
return
False
except
IOError
,
e
:
...
...
@@ -132,23 +107,16 @@ def upload_network_cached(network_cache, external_url, path, logger):
'Upload cache ignored, network-cache was not provided'
)
return
False
sha256sum
=
_get_sha256_from_file
(
path
)
try
:
f
=
open
(
path
,
'r'
)
data
=
f
.
read
()
url
=
os
.
path
.
join
(
network_cache
,
sha256sum
)
key
=
hashlib
.
sha256
(
external_url
).
hexdigest
()
try
:
result
=
urllib
.
urlopen
(
url
,
urllib
.
urlencode
({
'data'
:
base64
.
encodestring
(
data
),
'url'
:
base64
.
encodestring
(
external_url
)}))
if
result
.
code
==
200
:
_update_network_cached_log
(
sha256sum
,
external_url
)
nc
=
NetworkcacheClient
(
network_cache
)
result
=
nc
.
put
(
key
,
path
)
if
result
.
status
==
200
:
_update_network_cached_log
(
key
,
external_url
)
except
(
IOError
,
EOFError
),
e
:
logger
.
info
(
'Fail to upload cache on %s. %s'
%
\
(
external_url
,
str
(
e
)))
finally
:
f
.
close
()
return
True
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment