Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
S
slapos.core
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Ophélie Gagnard
slapos.core
Commits
b696881f
Commit
b696881f
authored
Oct 23, 2014
by
Marco Mariani
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
added options: software_min_free_space (def.200MB) and instance_min_free_space (def.100MB)
parent
8b829033
Changes
4
Show whitespace changes
Inline
Side-by-side
Showing
4 changed files
with
215 additions
and
4 deletions
+215
-4
slapos/grid/SlapObject.py
slapos/grid/SlapObject.py
+68
-2
slapos/grid/exception.py
slapos/grid/exception.py
+5
-0
slapos/grid/slapgrid.py
slapos/grid/slapgrid.py
+14
-2
slapos/human.py
slapos/human.py
+128
-0
No files found.
slapos/grid/SlapObject.py
View file @
b696881f
...
...
@@ -48,8 +48,9 @@ from slapos.grid import utils # for methods that could be mocked, access them t
from
slapos.slap.slap
import
NotFoundError
from
slapos.grid.svcbackend
import
getSupervisorRPC
from
slapos.grid.exception
import
(
BuildoutFailedError
,
WrongPermissionError
,
PathDoesNotExistError
)
PathDoesNotExistError
,
DiskSpaceError
)
from
slapos.grid.networkcache
import
download_network_cached
,
upload_network_cached
from
slapos.human
import
bytes2human
WATCHDOG_MARK
=
'-on-watch'
...
...
@@ -61,6 +62,38 @@ PROGRAM_PARTITION_TEMPLATE = pkg_resources.resource_stream(__name__,
'templates/program_partition_supervisord.conf.in'
).
read
()
def
free_space
(
path
,
fn
):
while
True
:
try
:
disk
=
os
.
statvfs
(
path
)
return
fn
(
disk
)
except
OSError
:
pass
if
os
.
sep
not
in
path
:
break
path
=
os
.
path
.
split
(
path
)[
0
]
def
free_space_root
(
path
):
"""
Returns free space available to the root user, in bytes.
A non-existent path can be provided, and the ancestors
will be queried instead.
"""
return
free_space
(
path
,
lambda
d
:
d
.
bsize
*
d
.
f_bfree
)
def
free_space_nonroot
(
path
):
"""
Returns free space available to non-root users, in bytes.
A non-existent path can be provided, and the ancestors
will be queried instead.
"""
return
free_space
(
path
,
lambda
d
:
d
.
f_bsize
*
d
.
f_bavail
)
class
Software
(
object
):
"""This class is responsible for installing a software release"""
...
...
@@ -73,7 +106,8 @@ class Software(object):
download_binary_cache_url
=
None
,
upload_binary_cache_url
=
None
,
download_binary_dir_url
=
None
,
upload_binary_dir_url
=
None
,
download_from_binary_cache_url_blacklist
=
None
,
upload_to_binary_cache_url_blacklist
=
None
):
upload_to_binary_cache_url_blacklist
=
None
,
software_min_free_space
=
None
):
"""Initialisation of class parameters
"""
...
...
@@ -106,6 +140,17 @@ class Software(object):
download_from_binary_cache_url_blacklist
self
.
upload_to_binary_cache_url_blacklist
=
\
upload_to_binary_cache_url_blacklist
self
.
software_min_free_space
=
software_min_free_space
def
check_free_space
(
self
):
required
=
self
.
software_min_free_space
available
=
free_space_nonroot
(
self
.
software_path
)
if
available
<
required
:
msg
=
"Not enough space for {path}: available {available}, required {required} (option 'software_min_free_space')"
raise
DiskSpaceError
(
msg
.
format
(
path
=
self
.
software_path
,
available
=
bytes2human
(
available
),
required
=
bytes2human
(
required
)))
def
install
(
self
):
""" Fetches binary cache if possible.
...
...
@@ -113,6 +158,9 @@ class Software(object):
"""
self
.
logger
.
info
(
"Installing software release %s..."
%
self
.
url
)
cache_dir
=
tempfile
.
mkdtemp
()
self
.
check_free_space
()
try
:
tarpath
=
os
.
path
.
join
(
cache_dir
,
self
.
software_url_hash
)
# Check if we can download from cache
...
...
@@ -293,6 +341,7 @@ class Partition(object):
logger
,
certificate_repository_path
=
None
,
retention_delay
=
'0'
,
instance_min_free_space
=
None
):
"""Initialisation of class parameters"""
self
.
buildout
=
buildout
...
...
@@ -333,6 +382,19 @@ class Partition(object):
self
.
instance_path
,
self
.
retention_lock_date_filename
)
self
.
instance_min_free_space
=
instance_min_free_space
def
check_free_space
(
self
):
required
=
self
.
instance_min_free_space
available
=
free_space_nonroot
(
self
.
instance_path
)
if
available
<
required
:
msg
=
"Not enough space for {path}: available {available}, required {required} (option 'instance_min_free_space')"
raise
DiskSpaceError
(
msg
.
format
(
path
=
self
.
instance_path
,
available
=
bytes2human
(
available
),
required
=
bytes2human
(
required
)))
def
_updateCertificate
(
self
):
try
:
partition_certificate
=
self
.
computer_partition
.
getCertificate
()
...
...
@@ -397,6 +459,9 @@ class Partition(object):
"""
self
.
logger
.
info
(
"Installing Computer Partition %s..."
%
self
.
computer_partition
.
getId
())
self
.
check_free_space
()
# Checks existence and permissions of Partition directory
# Note : Partitions have to be created and configured before running slapgrid
if
not
os
.
path
.
isdir
(
self
.
instance_path
):
...
...
@@ -505,6 +570,7 @@ class Partition(object):
[
'buildout:bin-directory=%s'
%
os
.
path
.
join
(
self
.
instance_path
,
'sbin'
)])
buildout_binary
=
os
.
path
.
join
(
self
.
instance_path
,
'sbin'
,
'buildout'
)
# Launches buildout
utils
.
launchBuildout
(
path
=
self
.
instance_path
,
buildout_binary
=
buildout_binary
,
...
...
slapos/grid/exception.py
View file @
b696881f
...
...
@@ -38,3 +38,8 @@ class WrongPermissionError(Exception):
class
BuildoutFailedError
(
Exception
):
pass
class
DiskSpaceError
(
Exception
):
pass
slapos/grid/slapgrid.py
View file @
b696881f
...
...
@@ -56,6 +56,7 @@ from slapos.grid.SlapObject import Software, Partition
from
slapos.grid.svcbackend
import
launchSupervisord
from
slapos.grid.utils
import
(
md5digest
,
createPrivateDirectory
,
dropPrivileges
,
SlapPopen
,
updateFile
)
from
slapos.human
import
human2bytes
import
slapos.slap
...
...
@@ -201,6 +202,9 @@ def create_slapgrid_object(options, logger):
]
op
=
options
software_min_free_space
=
human2bytes
(
op
.
get
(
'software_min_free_space'
,
'200M'
))
instance_min_free_space
=
human2bytes
(
op
.
get
(
'instance_min_free_space'
,
'100M'
))
return
Slapgrid
(
software_root
=
op
[
'software_root'
],
instance_root
=
op
[
'instance_root'
],
master_url
=
op
[
'master_url'
],
...
...
@@ -235,7 +239,9 @@ def create_slapgrid_object(options, logger):
# Try to fetch from deprecated argument
software_release_filter_list
=
op
.
get
(
'only-sr'
,
op
.
get
(
'only_sr'
)),
# Try to fetch from deprecated argument
computer_partition_filter_list
=
op
.
get
(
'only-cp'
,
op
.
get
(
'only_cp'
)))
computer_partition_filter_list
=
op
.
get
(
'only-cp'
,
op
.
get
(
'only_cp'
)),
software_min_free_space
=
software_min_free_space
,
instance_min_free_space
=
instance_min_free_space
)
def
check_required_only_partitions
(
existing
,
required
):
...
...
@@ -288,6 +294,8 @@ class Slapgrid(object):
develop
=
False
,
software_release_filter_list
=
None
,
computer_partition_filter_list
=
None
,
software_min_free_space
=
None
,
instance_min_free_space
=
None
,
):
"""Makes easy initialisation of class parameters"""
# Parses arguments
...
...
@@ -339,6 +347,8 @@ class Slapgrid(object):
self
.
computer_partition_filter_list
=
\
computer_partition_filter_list
.
split
(
","
)
self
.
maximum_periodicity
=
maximum_periodicity
self
.
software_min_free_space
=
software_min_free_space
self
.
instance_min_free_space
=
instance_min_free_space
def
getWatchdogLine
(
self
):
invocation_list
=
[
WATCHDOG_PATH
]
...
...
@@ -433,7 +443,8 @@ class Slapgrid(object):
shacache_cert_file
=
self
.
shacache_cert_file
,
shacache_key_file
=
self
.
shacache_key_file
,
shadir_cert_file
=
self
.
shadir_cert_file
,
shadir_key_file
=
self
.
shadir_key_file
)
shadir_key_file
=
self
.
shadir_key_file
,
software_min_free_space
=
self
.
software_min_free_space
)
if
state
==
'available'
:
completed_tag
=
os
.
path
.
join
(
software_path
,
'.completed'
)
if
(
self
.
develop
or
(
not
os
.
path
.
exists
(
completed_tag
)
and
...
...
@@ -670,6 +681,7 @@ class Slapgrid(object):
buildout
=
self
.
buildout
,
logger
=
self
.
logger
,
retention_delay
=
retention_delay
,
instance_min_free_space
=
self
.
instance_min_free_space
,
)
computer_partition_state
=
computer_partition
.
getState
()
...
...
slapos/human.py
0 → 100644
View file @
b696881f
#!/usr/bin/env python
import
sys
"""
Bytes-to-human / human-to-bytes converter.
Based on: http://goo.gl/kTQMs
Working with Python 2.x and 3.x.
Author: Giampaolo Rodola' <g.rodola [AT] gmail [DOT] com>
License: MIT
"""
# see: http://goo.gl/kTQMs
SYMBOLS
=
{
'customary'
:
(
'B'
,
'K'
,
'M'
,
'G'
,
'T'
,
'P'
,
'E'
,
'Z'
,
'Y'
),
'slapos'
:
(
''
,
'KB'
,
'MB'
,
'GB'
,
'TB'
,
'PB'
,
'EB'
,
'ZB'
,
'YB'
),
'customary_ext'
:
(
'byte'
,
'kilo'
,
'mega'
,
'giga'
,
'tera'
,
'peta'
,
'exa'
,
'zetta'
,
'iotta'
),
'iec'
:
(
'Bi'
,
'Ki'
,
'Mi'
,
'Gi'
,
'Ti'
,
'Pi'
,
'Ei'
,
'Zi'
,
'Yi'
),
'iec_ext'
:
(
'byte'
,
'kibi'
,
'mebi'
,
'gibi'
,
'tebi'
,
'pebi'
,
'exbi'
,
'zebi'
,
'yobi'
),
}
def
bytes2human
(
n
,
format
=
'%(value).1f %(symbol)s'
,
symbols
=
'slapos'
):
"""
Convert n bytes into a human readable string based on format.
symbols can be either "customary", "customary_ext", "iec" or "iec_ext",
see: http://goo.gl/kTQMs
>>> bytes2human(0)
'0.0 B'
>>> bytes2human(0.9)
'0.0 B'
>>> bytes2human(1)
'1.0 B'
>>> bytes2human(1.9)
'1.0 B'
>>> bytes2human(1024)
'1.0 K'
>>> bytes2human(1048576)
'1.0 M'
>>> bytes2human(1099511627776127398123789121)
'909.5 Y'
>>> bytes2human(9856, symbols="customary")
'9.6 K'
>>> bytes2human(9856, symbols="customary_ext")
'9.6 kilo'
>>> bytes2human(9856, symbols="iec")
'9.6 Ki'
>>> bytes2human(9856, symbols="iec_ext")
'9.6 kibi'
>>> bytes2human(10000, "%(value).1f %(symbol)s/sec")
'9.8 K/sec'
>>> # precision can be adjusted by playing with %f operator
>>> bytes2human(10000, format="%(value).5f %(symbol)s")
'9.76562 K'
"""
n
=
int
(
n
)
if
n
<
0
:
raise
ValueError
(
"n < 0"
)
symbols
=
SYMBOLS
[
symbols
]
prefix
=
{}
for
i
,
s
in
enumerate
(
symbols
[
1
:]):
prefix
[
s
]
=
1
<<
(
i
+
1
)
*
10
for
symbol
in
reversed
(
symbols
[
1
:]):
if
n
>=
prefix
[
symbol
]:
value
=
float
(
n
)
/
prefix
[
symbol
]
return
format
%
locals
()
return
format
%
dict
(
symbol
=
symbols
[
0
],
value
=
n
)
def
human2bytes
(
s
):
"""
Attempts to guess the string format based on default symbols
set and return the corresponding bytes as an integer.
When unable to recognize the format ValueError is raised.
>>> human2bytes('0 B')
0
>>> human2bytes('1 K')
1024
>>> human2bytes('1 M')
1048576
>>> human2bytes('1 Gi')
1073741824
>>> human2bytes('1 tera')
1099511627776
>>> human2bytes('0.5kilo')
512
>>> human2bytes('0.1 byte')
0
>>> human2bytes('1 k') # k is an alias for K
1024
>>> human2bytes('12 foo')
Traceback (most recent call last):
...
ValueError: can't interpret '12 foo'
"""
init
=
s
num
=
""
while
s
and
s
[
0
:
1
].
isdigit
()
or
s
[
0
:
1
]
==
'.'
:
num
+=
s
[
0
]
s
=
s
[
1
:]
num
=
float
(
num
)
letter
=
s
.
strip
()
for
name
,
sset
in
SYMBOLS
.
items
():
if
letter
in
sset
:
break
else
:
if
letter
==
'k'
:
# treat 'k' as an alias for 'K' as per: http://goo.gl/kTQMs
sset
=
SYMBOLS
[
'customary'
]
letter
=
letter
.
upper
()
else
:
raise
ValueError
(
"can't interpret %r"
%
init
)
prefix
=
{
sset
[
0
]:
1
}
for
i
,
s
in
enumerate
(
sset
[
1
:]):
prefix
[
s
]
=
1
<<
(
i
+
1
)
*
10
return
int
(
num
*
prefix
[
letter
])
if
__name__
==
"__main__"
:
import
doctest
doctest
.
testmod
()
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment