Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
S
slapos.core
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Cédric Le Ninivin
slapos.core
Commits
af23c67d
Commit
af23c67d
authored
Jul 20, 2022
by
Cédric Le Ninivin
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
wip: slapgrid-cp move out of slap
parent
41cdc016
Changes
2
Show whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
176 additions
and
91 deletions
+176
-91
slapos/grid/SlapObject.py
slapos/grid/SlapObject.py
+8
-8
slapos/grid/slapgrid.py
slapos/grid/slapgrid.py
+168
-83
No files found.
slapos/grid/SlapObject.py
View file @
af23c67d
...
@@ -581,7 +581,7 @@ class Partition(object):
...
@@ -581,7 +581,7 @@ class Partition(object):
installs the software partition with the help of buildout
installs the software partition with the help of buildout
"""
"""
self
.
logger
.
info
(
"Installing Computer Partition %s..."
self
.
logger
.
info
(
"Installing Computer Partition %s..."
%
self
.
computer_partition
.
get
Id
(
))
%
self
.
computer_partition
.
get
(
"compute_partition_id"
))
self
.
check_free_space
()
self
.
check_free_space
()
...
@@ -731,7 +731,7 @@ class Partition(object):
...
@@ -731,7 +731,7 @@ class Partition(object):
if
os
.
path
.
exists
(
self
.
supervisord_partition_configuration_path
):
if
os
.
path
.
exists
(
self
.
supervisord_partition_configuration_path
):
os
.
unlink
(
self
.
supervisord_partition_configuration_path
)
os
.
unlink
(
self
.
supervisord_partition_configuration_path
)
else
:
else
:
partition_id
=
self
.
computer_partition
.
get
Id
(
)
partition_id
=
self
.
computer_partition
.
get
(
"compute_partition_id"
)
group_partition_template
=
bytes2str
(
pkg_resources
.
resource_string
(
__name__
,
group_partition_template
=
bytes2str
(
pkg_resources
.
resource_string
(
__name__
,
'templates/group_partition_supervisord.conf.in'
))
'templates/group_partition_supervisord.conf.in'
))
self
.
supervisor_configuration_group
=
group_partition_template
%
{
self
.
supervisor_configuration_group
=
group_partition_template
%
{
...
@@ -766,22 +766,22 @@ class Partition(object):
...
@@ -766,22 +766,22 @@ class Partition(object):
"""Asks supervisord to start the instance. If this instance is not
"""Asks supervisord to start the instance. If this instance is not
installed, we install it.
installed, we install it.
"""
"""
partition_id
=
self
.
computer_partition
.
get
Id
(
)
partition_id
=
self
.
computer_partition
.
get
(
"compute_partition_id"
)
try
:
try
:
with
self
.
getSupervisorRPC
()
as
supervisor
:
with
self
.
getSupervisorRPC
()
as
supervisor
:
supervisor
.
startProcessGroup
(
partition_id
,
False
)
supervisor
.
startProcessGroup
(
partition_id
,
False
)
except
xmlrpclib
.
Fault
as
exc
:
except
xmlrpclib
.
Fault
as
exc
:
if
exc
.
faultString
.
startswith
(
'BAD_NAME:'
):
if
exc
.
faultString
.
startswith
(
'BAD_NAME:'
):
self
.
logger
.
info
(
"Nothing to start on %s..."
%
self
.
logger
.
info
(
"Nothing to start on %s..."
%
self
.
computer_partition
.
get
Id
(
))
self
.
computer_partition
.
get
(
"compute_partition_id"
))
else
:
else
:
raise
raise
else
:
else
:
self
.
logger
.
info
(
"Requested start of %s..."
%
self
.
computer_partition
.
get
Id
(
))
self
.
logger
.
info
(
"Requested start of %s..."
%
self
.
computer_partition
.
get
(
"compute_partition_id"
))
def
stop
(
self
):
def
stop
(
self
):
"""Asks supervisord to stop the instance."""
"""Asks supervisord to stop the instance."""
partition_id
=
self
.
computer_partition
.
get
Id
(
)
partition_id
=
self
.
computer_partition
.
get
(
"compute_partition_id"
)
try
:
try
:
with
self
.
getSupervisorRPC
()
as
supervisor
:
with
self
.
getSupervisorRPC
()
as
supervisor
:
supervisor
.
stopProcessGroup
(
partition_id
,
False
)
supervisor
.
stopProcessGroup
(
partition_id
,
False
)
...
@@ -791,13 +791,13 @@ class Partition(object):
...
@@ -791,13 +791,13 @@ class Partition(object):
else
:
else
:
raise
raise
else
:
else
:
self
.
logger
.
info
(
"Requested stop of %s..."
%
self
.
computer_partition
.
get
Id
(
))
self
.
logger
.
info
(
"Requested stop of %s..."
%
self
.
computer_partition
.
get
(
"compute_partition_id"
))
def
destroy
(
self
):
def
destroy
(
self
):
"""Destroys the partition and makes it available for subsequent use."
"""Destroys the partition and makes it available for subsequent use."
"""
"""
self
.
logger
.
info
(
"Destroying Computer Partition %s..."
self
.
logger
.
info
(
"Destroying Computer Partition %s..."
%
self
.
computer_partition
.
get
Id
(
))
%
self
.
computer_partition
.
get
(
"compute_partition_id"
))
self
.
createRetentionLockDate
()
self
.
createRetentionLockDate
()
if
not
self
.
checkRetentionIsAuthorized
():
if
not
self
.
checkRetentionIsAuthorized
():
...
...
slapos/grid/slapgrid.py
View file @
af23c67d
...
@@ -416,6 +416,7 @@ class Slapgrid(object):
...
@@ -416,6 +416,7 @@ class Slapgrid(object):
if
computer_partition_filter_list
is
not
None
:
if
computer_partition_filter_list
is
not
None
:
self
.
computer_partition_filter_list
=
\
self
.
computer_partition_filter_list
=
\
computer_partition_filter_list
.
split
(
","
)
computer_partition_filter_list
.
split
(
","
)
self
.
computer_partition_list
=
None
self
.
maximum_periodicity
=
maximum_periodicity
self
.
maximum_periodicity
=
maximum_periodicity
self
.
software_min_free_space
=
software_min_free_space
self
.
software_min_free_space
=
software_min_free_space
self
.
instance_min_free_space
=
instance_min_free_space
self
.
instance_min_free_space
=
instance_min_free_space
...
@@ -550,11 +551,50 @@ stderr_logfile_backups=1
...
@@ -550,11 +551,50 @@ stderr_logfile_backups=1
launchSupervisord
(
instance_root
=
self
.
instance_root
,
logger
=
self
.
logger
)
launchSupervisord
(
instance_root
=
self
.
instance_root
,
logger
=
self
.
logger
)
def
getComputerPartitionList
(
self
):
def
getComputerPartitionList
(
self
):
if
self
.
computer_partition_list
is
None
:
if
not
self
.
api_backward_compatibility
:
self
.
computer_partition_list
=
self
.
slap
.
jio_api_connector
.
allDocs
({
"portal_type"
:
"Software Instance"
,
"compute_node_id"
:
self
.
computer_id
,
})
else
:
try
:
try
:
return
self
.
computer
.
getComputerPartitionList
()
slap_partition_list
=
self
.
computer
.
getComputerPartitionList
()
except
socket
.
error
as
exc
:
except
socket
.
error
as
exc
:
self
.
logger
.
fatal
(
exc
)
self
.
logger
.
fatal
(
exc
)
raise
raise
self
.
computer_partition_list
=
[]
for
partition
in
slap_partition_list
:
try
:
software_release_uri
=
partition
.
getSoftwareRelease
().
getURI
()
except
(
NotFoundError
,
TypeError
,
NameError
):
software_release_uri
=
None
self
.
computer_partition_list
.
append
({
"reference"
:
partition
.
_instance_guid
,
"portal_type"
:
"Software Instance"
,
"compute_partition_id"
:
partition
.
getId
(),
"state"
:
partition
.
getState
(),
"software_type"
:
self
.
getInstanceParameterDict
().
get
(
'slap_software_type'
,
None
),
"parameters"
:
partition
.
getInstanceParameterDict
(),
"instance_processing_timestamp"
:
partition
.
getInstanceParameterDict
().
get
(
"timestamp"
),
"slap_partition"
:
partition
,
"access_status_message"
:
partition
.
getAccessStatus
(),
"software_release_uri"
:
software_release_uri
,
})
return
self
.
computer_partition_list
def
sendPartitionError
(
self
,
partition
,
error_message
,
logger
=
None
):
if
not
self
.
api_backward_compatibility
:
self
.
slap
.
jio_api_connector
.
put
({
"portal_type"
:
"Software Instance"
,
"reported_state"
:
"error"
,
"status_message"
:
str
(
error_message
),
"reference"
:
partition
.
get
(
"reference"
)
})
else
:
partition
[
"slap_partition"
].
error
(
error_message
,
logger
=
logger
)
def
processSoftwareReleaseList
(
self
):
def
processSoftwareReleaseList
(
self
):
"""Will process each Software Release.
"""Will process each Software Release.
...
@@ -764,7 +804,7 @@ stderr_logfile_backups=1
...
@@ -764,7 +804,7 @@ stderr_logfile_backups=1
return
promise_checker
.
run
()
return
promise_checker
.
run
()
def
_endInstallationTransaction
(
self
,
computer_partition
):
def
_endInstallationTransaction
(
self
,
computer_partition
):
partition_id
=
computer_partition
.
get
Id
(
)
partition_id
=
computer_partition
.
get
(
"compute_partition_id"
)
transaction_file_name
=
COMPUTER_PARTITION_REQUEST_LIST_TEMPLATE_FILENAME
%
partition_id
transaction_file_name
=
COMPUTER_PARTITION_REQUEST_LIST_TEMPLATE_FILENAME
%
partition_id
transaction_file_path
=
os
.
path
.
join
(
self
.
instance_root
,
transaction_file_path
=
os
.
path
.
join
(
self
.
instance_root
,
partition_id
,
partition_id
,
...
@@ -773,6 +813,7 @@ stderr_logfile_backups=1
...
@@ -773,6 +813,7 @@ stderr_logfile_backups=1
if
os
.
path
.
exists
(
transaction_file_path
):
if
os
.
path
.
exists
(
transaction_file_path
):
with
open
(
transaction_file_path
,
'r'
)
as
tf
:
with
open
(
transaction_file_path
,
'r'
)
as
tf
:
try
:
try
:
# XXX CLN
computer_partition
.
setComputerPartitionRelatedInstanceList
(
computer_partition
.
setComputerPartitionRelatedInstanceList
(
[
reference
for
reference
in
tf
.
read
().
split
(
'
\
n
'
)
if
reference
]
[
reference
for
reference
in
tf
.
read
().
split
(
'
\
n
'
)
if
reference
]
)
)
...
@@ -986,6 +1027,7 @@ stderr_logfile_backups=1
...
@@ -986,6 +1027,7 @@ stderr_logfile_backups=1
elif
valid_ipv6
(
ip
):
elif
valid_ipv6
(
ip
):
ipv6_list
.
append
(
ip
)
ipv6_list
.
append
(
ip
)
# XXX CLN
hosting_ip_list
=
computer_partition
.
getFullHostingIpAddressList
()
hosting_ip_list
=
computer_partition
.
getFullHostingIpAddressList
()
for
iface
,
ip
in
hosting_ip_list
:
for
iface
,
ip
in
hosting_ip_list
:
if
valid_ipv4
(
ip
):
if
valid_ipv4
(
ip
):
...
@@ -1015,11 +1057,11 @@ stderr_logfile_backups=1
...
@@ -1015,11 +1057,11 @@ stderr_logfile_backups=1
for
ip
in
ipv4_list
:
for
ip
in
ipv4_list
:
cmd_list
=
getFirewallRules
(
ip
,
hosting_ipv4_list
,
cmd_list
=
getFirewallRules
(
ip
,
hosting_ipv4_list
,
source_ipv4_list
,
ip_type
=
'ipv4'
)
source_ipv4_list
,
ip_type
=
'ipv4'
)
self
.
_checkAddFirewallRules
(
computer_partition
.
get
Id
(
),
self
.
_checkAddFirewallRules
(
computer_partition
.
get
(
"compute_partition_id"
),
cmd_list
,
add
=
add_rules
)
cmd_list
,
add
=
add_rules
)
def
_checkPromiseAnomaly
(
self
,
local_partition
,
computer_partition
):
def
_checkPromiseAnomaly
(
self
,
local_partition
,
computer_partition
):
partition_access_status
=
computer_partition
.
get
AccessStatus
(
)
partition_access_status
=
computer_partition
.
get
(
"access_status_message"
,
""
)
status_error
=
False
status_error
=
False
if
partition_access_status
and
partition_access_status
.
startswith
(
"#error"
):
if
partition_access_status
and
partition_access_status
.
startswith
(
"#error"
):
status_error
=
True
status_error
=
True
...
@@ -1031,17 +1073,24 @@ stderr_logfile_backups=1
...
@@ -1031,17 +1073,24 @@ stderr_logfile_backups=1
self
.
logger
.
error
(
e
)
self
.
logger
.
error
(
e
)
if
partition_access_status
is
None
or
not
status_error
:
if
partition_access_status
is
None
or
not
status_error
:
local_partition
.
_updateCertificate
()
local_partition
.
_updateCertificate
()
computer_partition
.
error
(
e
,
logger
=
self
.
logger
)
self
.
sendPartitionError
(
computer_partition
,
e
,
logger
=
self
.
logger
)
else
:
else
:
if
partition_access_status
is
None
or
status_error
:
if
partition_access_status
is
None
or
status_error
:
local_partition
.
_updateCertificate
()
local_partition
.
_updateCertificate
()
computer_partition
.
started
()
if
not
self
.
api_backward_compatibility
:
self
.
slap
.
jio_api_connector
({
"portal_type"
:
"Software Instance"
,
"reference"
:
computer_partition
.
get
(
"reference"
),
"reported_state"
:
"started"
})
else
:
computer_partition
[
"slap_partition"
].
started
()
def
processPromise
(
self
,
computer_partition
):
def
processPromise
(
self
,
computer_partition
):
"""
"""
Process the promises from a given Computer Partition, depending on its state
Process the promises from a given Computer Partition, depending on its state
"""
"""
computer_partition_id
=
computer_partition
.
get
Id
(
)
computer_partition_id
=
computer_partition
.
get
(
"compute_partition_id"
)
# Sanity checks before processing
# Sanity checks before processing
# Those values should not be None or empty string or any falsy value
# Those values should not be None or empty string or any falsy value
...
@@ -1056,12 +1105,7 @@ stderr_logfile_backups=1
...
@@ -1056,12 +1105,7 @@ stderr_logfile_backups=1
instance_path
=
os
.
path
.
join
(
self
.
instance_root
,
computer_partition_id
)
instance_path
=
os
.
path
.
join
(
self
.
instance_root
,
computer_partition_id
)
os
.
environ
[
'SLAPGRID_INSTANCE_ROOT'
]
=
self
.
instance_root
os
.
environ
[
'SLAPGRID_INSTANCE_ROOT'
]
=
self
.
instance_root
try
:
software_url
=
computer_partition
.
get
(
"software_release_uri"
)
software_url
=
computer_partition
.
getSoftwareRelease
().
getURI
()
except
NotFoundError
:
# Problem with instance: SR URI not set.
# Try to process it anyway, it may need to be deleted.
software_url
=
None
try
:
try
:
software_path
=
os
.
path
.
join
(
self
.
software_root
,
md5digest
(
software_url
))
software_path
=
os
.
path
.
join
(
self
.
software_root
,
md5digest
(
software_url
))
...
@@ -1070,7 +1114,7 @@ stderr_logfile_backups=1
...
@@ -1070,7 +1114,7 @@ stderr_logfile_backups=1
# Try to process it anyway, it may need to be deleted.
# Try to process it anyway, it may need to be deleted.
software_path
=
None
software_path
=
None
computer_partition_state
=
computer_partition
.
get
State
(
)
computer_partition_state
=
computer_partition
.
get
(
"state"
)
local_partition
=
Partition
(
local_partition
=
Partition
(
software_path
=
software_path
,
software_path
=
software_path
,
...
@@ -1107,7 +1151,7 @@ stderr_logfile_backups=1
...
@@ -1107,7 +1151,7 @@ stderr_logfile_backups=1
"""
"""
Process a Computer Partition, depending on its state
Process a Computer Partition, depending on its state
"""
"""
computer_partition_id
=
computer_partition
.
get
Id
(
)
computer_partition_id
=
computer_partition
.
get
(
"compute_partition_id"
)
# Sanity checks before processing
# Sanity checks before processing
# Those values should not be None or empty string or any falsy value
# Those values should not be None or empty string or any falsy value
...
@@ -1137,20 +1181,15 @@ stderr_logfile_backups=1
...
@@ -1137,20 +1181,15 @@ stderr_logfile_backups=1
instance_path
,
instance_path
,
COMPUTER_PARTITION_TIMESTAMP_FILENAME
COMPUTER_PARTITION_TIMESTAMP_FILENAME
)
)
parameter_dict
=
computer_partition
.
get
InstanceParameterDict
(
)
parameter_dict
=
computer_partition
.
get
(
"parameters"
,
{}
)
timestamp
=
parameter_dict
.
get
(
'timestamp'
)
timestamp
=
computer_partition
.
get
(
"processing_timestamp"
)
error_output_file
=
os
.
path
.
join
(
error_output_file
=
os
.
path
.
join
(
instance_path
,
instance_path
,
COMPUTER_PARTITION_INSTALL_ERROR_FILENAME
%
computer_partition_id
COMPUTER_PARTITION_INSTALL_ERROR_FILENAME
%
computer_partition_id
)
)
try
:
software_url
=
computer_partition
.
get
(
"software_release_uri"
)
software_url
=
computer_partition
.
getSoftwareRelease
().
getURI
()
except
NotFoundError
:
# Problem with instance: SR URI not set.
# Try to process it anyway, it may need to be deleted.
software_url
=
None
try
:
try
:
software_path
=
os
.
path
.
join
(
self
.
software_root
,
md5digest
(
software_url
))
software_path
=
os
.
path
.
join
(
self
.
software_root
,
md5digest
(
software_url
))
except
TypeError
:
except
TypeError
:
...
@@ -1158,7 +1197,7 @@ stderr_logfile_backups=1
...
@@ -1158,7 +1197,7 @@ stderr_logfile_backups=1
# Try to process it anyway, it may need to be deleted.
# Try to process it anyway, it may need to be deleted.
software_path
=
None
software_path
=
None
computer_partition_state
=
computer_partition
.
get
State
(
)
computer_partition_state
=
computer_partition
.
get
(
"state"
)
periodicity
=
self
.
maximum_periodicity
periodicity
=
self
.
maximum_periodicity
if
software_path
:
if
software_path
:
periodicity_path
=
os
.
path
.
join
(
software_path
,
'periodicity'
)
periodicity_path
=
os
.
path
.
join
(
software_path
,
'periodicity'
)
...
@@ -1266,7 +1305,7 @@ stderr_logfile_backups=1
...
@@ -1266,7 +1305,7 @@ stderr_logfile_backups=1
local_partition
.
_updateCertificate
()
local_partition
.
_updateCertificate
()
# XXX this line breaks 37 tests
# XXX this line breaks 37 tests
# self.logger.info(' Instance type: %s' % computer_partition.get
Type(
))
# self.logger.info(' Instance type: %s' % computer_partition.get
("software_type"
))
self
.
logger
.
info
(
' Instance status: %s'
%
computer_partition_state
)
self
.
logger
.
info
(
' Instance status: %s'
%
computer_partition_state
)
if
os
.
path
.
exists
(
error_output_file
):
if
os
.
path
.
exists
(
error_output_file
):
...
@@ -1288,7 +1327,14 @@ stderr_logfile_backups=1
...
@@ -1288,7 +1327,14 @@ stderr_logfile_backups=1
partition_ip_list
)
partition_ip_list
)
if
not
self
.
force_stop
:
if
not
self
.
force_stop
:
self
.
_checkPromiseList
(
local_partition
)
self
.
_checkPromiseList
(
local_partition
)
computer_partition
.
started
()
if
not
self
.
api_backward_compatibility
:
self
.
slap
.
jio_api_connector
({
"portal_type"
:
"Software Instance"
,
"reference"
:
computer_partition
.
get
(
"reference"
),
"reported_state"
:
"started"
})
else
:
computer_partition
[
"slap_partition"
].
started
()
self
.
_endInstallationTransaction
(
computer_partition
)
self
.
_endInstallationTransaction
(
computer_partition
)
elif
computer_partition_state
==
COMPUTER_PARTITION_STOPPED_STATE
:
elif
computer_partition_state
==
COMPUTER_PARTITION_STOPPED_STATE
:
try
:
try
:
...
@@ -1302,9 +1348,16 @@ stderr_logfile_backups=1
...
@@ -1302,9 +1348,16 @@ stderr_logfile_backups=1
# Instance has to be stopped even if buildout/reporting is wrong.
# Instance has to be stopped even if buildout/reporting is wrong.
local_partition
.
stop
()
local_partition
.
stop
()
try
:
try
:
computer_partition
.
stopped
()
if
not
self
.
api_backward_compatibility
:
self
.
slap
.
jio_api_connector
({
"portal_type"
:
"Software Instance"
,
"reference"
:
computer_partition
.
get
(
"reference"
),
"reported_state"
:
"stopped"
})
else
:
computer_partition
[
"slap_partition"
].
stopped
()
except
(
SystemExit
,
KeyboardInterrupt
):
except
(
SystemExit
,
KeyboardInterrupt
):
computer_partition
.
error
(
traceback
.
format_exc
(),
logger
=
self
.
logger
)
self
.
sendPartitionError
(
computer_partition
,
traceback
.
format_exc
(),
logger
=
self
.
logger
)
raise
raise
except
Exception
:
except
Exception
:
pass
pass
...
@@ -1316,16 +1369,23 @@ stderr_logfile_backups=1
...
@@ -1316,16 +1369,23 @@ stderr_logfile_backups=1
partition_ip_list
,
partition_ip_list
,
drop_entries
=
True
)
drop_entries
=
True
)
try
:
try
:
computer_partition
.
stopped
()
if
not
self
.
api_backward_compatibility
:
self
.
slap
.
jio_api_connector
({
"portal_type"
:
"Software Instance"
,
"reference"
:
computer_partition
.
get
(
"reference"
),
"reported_state"
:
"stopped"
})
else
:
computer_partition
[
"slap_partition"
].
stopped
()
except
(
SystemExit
,
KeyboardInterrupt
):
except
(
SystemExit
,
KeyboardInterrupt
):
computer_partition
.
error
(
traceback
.
format_exc
(),
logger
=
self
.
logger
)
self
.
sendPartitionError
(
computer_partition
,
traceback
.
format_exc
(),
logger
=
self
.
logger
)
raise
raise
except
Exception
:
except
Exception
:
pass
pass
else
:
else
:
error_string
=
"Computer Partition %r has unsupported state: %s"
%
\
error_string
=
"Computer Partition %r has unsupported state: %s"
%
\
(
computer_partition_id
,
computer_partition_state
)
(
computer_partition_id
,
computer_partition_state
)
computer_partition
.
error
(
error_string
,
logger
=
self
.
logger
)
self
.
sendPartitionError
(
computer_partition
,
error_string
,
logger
=
self
.
logger
)
raise
NotImplementedError
(
error_string
)
raise
NotImplementedError
(
error_string
)
except
Exception
as
e
:
except
Exception
as
e
:
if
not
isinstance
(
e
,
PromiseError
):
if
not
isinstance
(
e
,
PromiseError
):
...
@@ -1361,7 +1421,7 @@ stderr_logfile_backups=1
...
@@ -1361,7 +1421,7 @@ stderr_logfile_backups=1
for
computer_partition
in
computer_partition_list
:
for
computer_partition
in
computer_partition_list
:
try
:
try
:
computer_partition_path
=
os
.
path
.
join
(
self
.
instance_root
,
computer_partition_path
=
os
.
path
.
join
(
self
.
instance_root
,
computer_partition
.
get
Id
(
))
computer_partition
.
get
(
"compute_partition_id"
))
if
not
os
.
path
.
exists
(
computer_partition_path
):
if
not
os
.
path
.
exists
(
computer_partition_path
):
raise
NotFoundError
(
'Partition directory %s does not exist.'
%
raise
NotFoundError
(
'Partition directory %s does not exist.'
%
computer_partition_path
)
computer_partition_path
)
...
@@ -1370,11 +1430,8 @@ stderr_logfile_backups=1
...
@@ -1370,11 +1430,8 @@ stderr_logfile_backups=1
# partition, and check if it has some Software information.
# partition, and check if it has some Software information.
# XXX-Cedric: Temporary AND ugly solution to check if an instance
# XXX-Cedric: Temporary AND ugly solution to check if an instance
# is in the partition. Dangerous because not 100% sure it is empty
# is in the partition. Dangerous because not 100% sure it is empty
computer_partition_state
=
computer_partition
.
getState
()
computer_partition_state
=
computer_partition
.
get
(
"state"
)
try
:
software_url
=
computer_partition
.
get
(
"software_release_uri"
)
software_url
=
computer_partition
.
getSoftwareRelease
().
getURI
()
except
(
NotFoundError
,
TypeError
,
NameError
):
software_url
=
None
if
computer_partition_state
==
COMPUTER_PARTITION_DESTROYED_STATE
and
\
if
computer_partition_state
==
COMPUTER_PARTITION_DESTROYED_STATE
and
\
not
software_url
:
not
software_url
:
# Exclude files which may come from concurrent processing
# Exclude files which may come from concurrent processing
...
@@ -1392,7 +1449,7 @@ stderr_logfile_backups=1
...
@@ -1392,7 +1449,7 @@ stderr_logfile_backups=1
# Ignore .slapos-resource file dumped by slapformat.
# Ignore .slapos-resource file dumped by slapformat.
if
os
.
listdir
(
computer_partition_path
)
not
in
empty_partition_listdir
:
if
os
.
listdir
(
computer_partition_path
)
not
in
empty_partition_listdir
:
self
.
logger
.
warning
(
"Free partition %s contains file(s) in %s."
%
(
self
.
logger
.
warning
(
"Free partition %s contains file(s) in %s."
%
(
computer_partition
.
get
Id
(
),
computer_partition_path
))
computer_partition
.
get
(
"compute_partition_id"
),
computer_partition_path
))
continue
continue
# Everything seems fine
# Everything seems fine
...
@@ -1402,7 +1459,7 @@ stderr_logfile_backups=1
...
@@ -1402,7 +1459,7 @@ stderr_logfile_backups=1
# Send log before exiting
# Send log before exiting
except
(
SystemExit
,
KeyboardInterrupt
):
except
(
SystemExit
,
KeyboardInterrupt
):
computer_partition
.
error
(
traceback
.
format_exc
(),
logger
=
self
.
logger
)
self
.
sendPartitionError
(
computer_partition
,
traceback
.
format_exc
(),
logger
=
self
.
logger
)
raise
raise
except
Exception
as
exc
:
except
Exception
as
exc
:
...
@@ -1411,7 +1468,7 @@ stderr_logfile_backups=1
...
@@ -1411,7 +1468,7 @@ stderr_logfile_backups=1
# For everything else: log it, send it, continue.
# For everything else: log it, send it, continue.
self
.
logger
.
exception
(
''
)
self
.
logger
.
exception
(
''
)
try
:
try
:
computer_partition
.
error
(
exc
,
logger
=
self
.
logger
)
self
.
sendPartitionError
(
computer_partition
,
exc
,
logger
=
self
.
logger
)
except
(
SystemExit
,
KeyboardInterrupt
):
except
(
SystemExit
,
KeyboardInterrupt
):
raise
raise
except
Exception
:
except
Exception
:
...
@@ -1433,12 +1490,14 @@ stderr_logfile_backups=1
...
@@ -1433,12 +1490,14 @@ stderr_logfile_backups=1
# Boolean to know if every promises correctly passed
# Boolean to know if every promises correctly passed
clean_run_promise
=
True
clean_run_promise
=
True
check_required_only_partitions
([
cp
.
getId
()
for
cp
in
self
.
getComputerPartitionList
()],
computer_partition_list
=
self
.
getComputerPartitionList
()
check_required_only_partitions
([
cp
.
get
(
"computer_partition_id"
,
""
)
for
cp
in
computer_partition_list
],
self
.
computer_partition_filter_list
)
self
.
computer_partition_filter_list
)
# Filter all dummy / empty partitions
# Filter all dummy / empty partitions
computer_partition_list
=
self
.
FilterComputerPartitionList
(
computer_partition_list
=
self
.
FilterComputerPartitionList
(
self
.
getComputerPartitionList
()
)
computer_partition_list
)
process_error_partition_list
=
[]
process_error_partition_list
=
[]
promise_error_partition_list
=
[]
promise_error_partition_list
=
[]
...
@@ -1447,20 +1506,25 @@ stderr_logfile_backups=1
...
@@ -1447,20 +1506,25 @@ stderr_logfile_backups=1
# Nothing should raise outside of the current loop iteration, so that
# Nothing should raise outside of the current loop iteration, so that
# even if something is terribly wrong while processing an instance, it
# even if something is terribly wrong while processing an instance, it
# won't prevent processing other ones.
# won't prevent processing other ones.
if
not
self
.
api_backward_compatibility
:
computer_partition
=
self
.
slap
.
jio_connector
.
get
({
"portal_type"
:
"Software Instance"
,
"reference"
:
computer_partition
[
"reference"
]
})
try
:
try
:
# Process the partition itself
# Process the partition itself
self
.
processComputerPartition
(
computer_partition
)
self
.
processComputerPartition
(
computer_partition
)
# Send log before exiting
# Send log before exiting
except
(
SystemExit
,
KeyboardInterrupt
):
except
(
SystemExit
,
KeyboardInterrupt
):
computer_partition
.
error
(
traceback
.
format_exc
(),
logger
=
self
.
logger
)
self
.
sendPartitionError
(
computer_partition
,
traceback
.
format_exc
(),
logger
=
self
.
logger
)
raise
raise
except
PromiseError
as
exc
:
except
PromiseError
as
exc
:
clean_run_promise
=
False
clean_run_promise
=
False
try
:
try
:
self
.
logger
.
error
(
exc
)
self
.
logger
.
error
(
exc
)
computer_partition
.
error
(
exc
,
logger
=
self
.
logger
)
self
.
sendPartitionError
(
computer_partition
,
exc
,
logger
=
self
.
logger
)
promise_error_partition_list
.
append
((
computer_partition
,
exc
))
promise_error_partition_list
.
append
((
computer_partition
,
exc
))
except
(
SystemExit
,
KeyboardInterrupt
):
except
(
SystemExit
,
KeyboardInterrupt
):
raise
raise
...
@@ -1474,7 +1538,7 @@ stderr_logfile_backups=1
...
@@ -1474,7 +1538,7 @@ stderr_logfile_backups=1
# For everything else: log it, send it, continue.
# For everything else: log it, send it, continue.
self
.
logger
.
exception
(
''
)
self
.
logger
.
exception
(
''
)
try
:
try
:
computer_partition
.
error
(
exc
,
logger
=
self
.
logger
)
self
.
sendPartitionError
(
computer_partition
,
exc
,
logger
=
self
.
logger
)
process_error_partition_list
.
append
((
computer_partition
,
exc
))
process_error_partition_list
.
append
((
computer_partition
,
exc
))
except
(
SystemExit
,
KeyboardInterrupt
):
except
(
SystemExit
,
KeyboardInterrupt
):
raise
raise
...
@@ -1484,9 +1548,8 @@ stderr_logfile_backups=1
...
@@ -1484,9 +1548,8 @@ stderr_logfile_backups=1
def
getPartitionType
(
part
):
def
getPartitionType
(
part
):
"""returns the partition type, if known at that point.
"""returns the partition type, if known at that point.
"""
"""
try
:
software_type
=
partition
.
get
(
"software_type"
,
None
)
return
part
.
getType
()
if
software_type
is
None
:
except
slapos
.
slap
.
ResourceNotReady
:
return
'(not ready)'
return
'(not ready)'
self
.
logger
.
info
(
'Finished computer partitions.'
)
self
.
logger
.
info
(
'Finished computer partitions.'
)
...
@@ -1494,11 +1557,11 @@ stderr_logfile_backups=1
...
@@ -1494,11 +1557,11 @@ stderr_logfile_backups=1
if
process_error_partition_list
:
if
process_error_partition_list
:
self
.
logger
.
info
(
'Error while processing the following partitions:'
)
self
.
logger
.
info
(
'Error while processing the following partitions:'
)
for
partition
,
exc
in
process_error_partition_list
:
for
partition
,
exc
in
process_error_partition_list
:
self
.
logger
.
info
(
' %s[%s]: %s'
,
partition
.
get
Id
(
),
getPartitionType
(
partition
),
exc
)
self
.
logger
.
info
(
' %s[%s]: %s'
,
partition
.
get
(
"compute_partition_id"
),
getPartitionType
(
partition
),
exc
)
if
promise_error_partition_list
:
if
promise_error_partition_list
:
self
.
logger
.
info
(
'Error with promises for the following partitions:'
)
self
.
logger
.
info
(
'Error with promises for the following partitions:'
)
for
partition
,
exc
in
promise_error_partition_list
:
for
partition
,
exc
in
promise_error_partition_list
:
self
.
logger
.
info
(
' %s[%s]: %s'
,
partition
.
get
Id
(
),
getPartitionType
(
partition
),
exc
)
self
.
logger
.
info
(
' %s[%s]: %s'
,
partition
.
get
(
"compute_partition_id"
),
getPartitionType
(
partition
),
exc
)
# Return success value
# Return success value
if
not
clean_run
:
if
not
clean_run
:
...
@@ -1514,15 +1577,23 @@ stderr_logfile_backups=1
...
@@ -1514,15 +1577,23 @@ stderr_logfile_backups=1
self
.
logger
.
info
(
'Processing promises...'
)
self
.
logger
.
info
(
'Processing promises...'
)
# Return success value
# Return success value
clean_run_promise
=
True
clean_run_promise
=
True
check_required_only_partitions
([
cp
.
getId
()
for
cp
in
self
.
getComputerPartitionList
()],
computer_partition_list
=
self
.
getComputerPartitionList
()
check_required_only_partitions
([
cp
.
get
(
"computer_partition_id"
,
""
)
for
cp
in
computer_partition_list
],
self
.
computer_partition_filter_list
)
self
.
computer_partition_filter_list
)
# Filter all dummy / empty partitions
# Filter all dummy / empty partitions
computer_partition_list
=
self
.
FilterComputerPartitionList
(
computer_partition_list
=
self
.
FilterComputerPartitionList
(
self
.
getComputerPartitionList
()
)
computer_partition_list
)
promise_error_partition_list
=
[]
promise_error_partition_list
=
[]
for
computer_partition
in
computer_partition_list
:
for
computer_partition
in
computer_partition_list
:
if
not
self
.
api_backward_compatibility
:
computer_partition
=
self
.
slap
.
jio_connector
.
get
({
"portal_type"
:
"Software Instance"
,
"reference"
:
computer_partition
[
"reference"
]
})
try
:
try
:
# Process the partition itself
# Process the partition itself
self
.
processPromise
(
computer_partition
)
self
.
processPromise
(
computer_partition
)
...
@@ -1538,15 +1609,14 @@ stderr_logfile_backups=1
...
@@ -1538,15 +1609,14 @@ stderr_logfile_backups=1
def
getPartitionType
(
part
):
def
getPartitionType
(
part
):
"""returns the partition type, if known at that point.
"""returns the partition type, if known at that point.
"""
"""
try
:
software_type
=
partition
.
get
(
"software_type"
,
None
)
return
part
.
getType
()
if
software_type
is
None
:
except
slapos
.
slap
.
ResourceNotReady
:
return
'(not ready)'
return
'(not ready)'
if
promise_error_partition_list
:
if
promise_error_partition_list
:
self
.
logger
.
info
(
'Finished computer partitions.'
)
self
.
logger
.
info
(
'Finished computer partitions.'
)
for
partition
,
exc
in
promise_error_partition_list
:
for
partition
,
exc
in
promise_error_partition_list
:
self
.
logger
.
info
(
' %s[%s]: %s'
,
partition
.
get
Id
(
),
getPartitionType
(
partition
),
exc
)
self
.
logger
.
info
(
' %s[%s]: %s'
,
partition
.
get
(
"compute_partition_id"
),
getPartitionType
(
partition
),
exc
)
# Return success value
# Return success value
if
not
clean_run_promise
:
if
not
clean_run_promise
:
...
@@ -1673,11 +1743,11 @@ stderr_logfile_backups=1
...
@@ -1673,11 +1743,11 @@ stderr_logfile_backups=1
for
computer_partition
in
computer_partition_list
:
for
computer_partition
in
computer_partition_list
:
try
:
try
:
computer_partition_id
=
computer_partition
.
get
Id
(
)
computer_partition_id
=
computer_partition
.
get
(
"compute_partition_id"
)
# We want to execute all the script in the report folder
# We want to execute all the script in the report folder
instance_path
=
os
.
path
.
join
(
self
.
instance_root
,
instance_path
=
os
.
path
.
join
(
self
.
instance_root
,
computer_partition
.
get
Id
(
))
computer_partition
.
get
(
"compute_partition_id"
))
report_path
=
os
.
path
.
join
(
instance_path
,
'etc'
,
'report'
)
report_path
=
os
.
path
.
join
(
instance_path
,
'etc'
,
'report'
)
if
os
.
path
.
isdir
(
report_path
):
if
os
.
path
.
isdir
(
report_path
):
script_list_to_run
=
os
.
listdir
(
report_path
)
script_list_to_run
=
os
.
listdir
(
report_path
)
...
@@ -1722,18 +1792,18 @@ stderr_logfile_backups=1
...
@@ -1722,18 +1792,18 @@ stderr_logfile_backups=1
failed_script_list
.
append
(
"Script %r failed."
%
script
)
failed_script_list
.
append
(
"Script %r failed."
%
script
)
self
.
logger
.
warning
(
'Failed to run %r'
%
invocation_list
)
self
.
logger
.
warning
(
'Failed to run %r'
%
invocation_list
)
if
len
(
failed_script_list
):
if
len
(
failed_script_list
):
computer_partition
.
error
(
'
\
n
'
.
join
(
failed_script_list
),
logger
=
self
.
logger
)
self
.
sendPartitionError
(
computer_partition
,
'
\
n
'
.
join
(
failed_script_list
),
logger
=
self
.
logger
)
# Whatever happens, don't stop processing other instances
# Whatever happens, don't stop processing other instances
except
Exception
:
except
Exception
:
self
.
logger
.
exception
(
'Cannot run usage script(s) for %r:'
%
self
.
logger
.
exception
(
'Cannot run usage script(s) for %r:'
%
computer_partition
.
get
Id
(
))
computer_partition
.
get
(
"compute_partition_id"
))
# Now we loop through the different computer partitions to report
# Now we loop through the different computer partitions to report
report_usage_issue_cp_list
=
[]
report_usage_issue_cp_list
=
[]
for
computer_partition
in
computer_partition_list
:
for
computer_partition
in
computer_partition_list
:
try
:
try
:
filename_delete_list
=
[]
filename_delete_list
=
[]
computer_partition_id
=
computer_partition
.
get
Id
(
)
computer_partition_id
=
computer_partition
.
get
(
"compute_partition_id"
)
instance_path
=
os
.
path
.
join
(
self
.
instance_root
,
computer_partition_id
)
instance_path
=
os
.
path
.
join
(
self
.
instance_root
,
computer_partition_id
)
dir_report_list
=
[
os
.
path
.
join
(
instance_path
,
'var'
,
'xml_report'
),
dir_report_list
=
[
os
.
path
.
join
(
instance_path
,
'var'
,
'xml_report'
),
os
.
path
.
join
(
self
.
instance_root
,
'var'
,
'xml_report'
,
os
.
path
.
join
(
self
.
instance_root
,
'var'
,
'xml_report'
,
...
@@ -1778,7 +1848,7 @@ stderr_logfile_backups=1
...
@@ -1778,7 +1848,7 @@ stderr_logfile_backups=1
# Whatever happens, don't stop processing other instances
# Whatever happens, don't stop processing other instances
except
Exception
:
except
Exception
:
self
.
logger
.
exception
(
'Cannot run usage script(s) for %r:'
%
self
.
logger
.
exception
(
'Cannot run usage script(s) for %r:'
%
computer_partition
.
get
Id
(
))
computer_partition
.
get
(
"compute_partition_id"
))
for
computer_partition_usage
in
computer_partition_usage_list
:
for
computer_partition_usage
in
computer_partition_usage_list
:
self
.
logger
.
info
(
'computer_partition_usage_list: %s - %s'
%
self
.
logger
.
info
(
'computer_partition_usage_list: %s - %s'
%
...
@@ -1830,28 +1900,29 @@ stderr_logfile_backups=1
...
@@ -1830,28 +1900,29 @@ stderr_logfile_backups=1
raise
ValueError
(
'XML file generated by asXML is not valid !'
)
raise
ValueError
(
'XML file generated by asXML is not valid !'
)
except
Exception
:
except
Exception
:
issue
=
"Cannot report usage for %r: %s"
%
(
issue
=
"Cannot report usage for %r: %s"
%
(
computer_partition
.
get
Id
(
),
computer_partition
.
get
(
"compute_partition_id"
),
traceback
.
format_exc
())
traceback
.
format_exc
())
self
.
logger
.
info
(
issue
)
self
.
logger
.
info
(
issue
)
computer_partition
.
error
(
issue
,
logger
=
self
.
logger
)
self
.
sendPartitionError
(
computer_partition
,
issue
,
logger
=
self
.
logger
)
report_usage_issue_cp_list
.
append
(
computer_partition_id
)
report_usage_issue_cp_list
.
append
(
computer_partition_id
)
for
computer_partition
in
computer_partition_list
:
for
computer_partition
in
computer_partition_list
:
if
computer_partition
.
get
State
(
)
==
COMPUTER_PARTITION_DESTROYED_STATE
:
if
computer_partition
.
get
(
"state"
)
==
COMPUTER_PARTITION_DESTROYED_STATE
:
destroyed
=
False
destroyed
=
False
try
:
try
:
computer_partition_id
=
computer_partition
.
getId
()
computer_partition_id
=
computer_partition
.
get
(
"compute_partition_id"
)
software_url
=
computer_partition
.
get
(
"software_release_uri"
)
try
:
try
:
software_url
=
computer_partition
.
getSoftwareRelease
().
getURI
()
software_path
=
os
.
path
.
join
(
self
.
software_root
,
md5digest
(
software_url
))
software_path
=
os
.
path
.
join
(
self
.
software_root
,
md5digest
(
software_url
))
except
(
NotFoundError
,
TypeError
):
except
TypeError
:
software_url
=
None
# Problem with instance: SR URI not set.
# Try to process it anyway, it may need to be deleted.
software_path
=
None
software_path
=
None
local_partition
=
Partition
(
local_partition
=
Partition
(
software_path
=
software_path
,
software_path
=
software_path
,
instance_path
=
os
.
path
.
join
(
self
.
instance_root
,
instance_path
=
os
.
path
.
join
(
self
.
instance_root
,
computer_partition
.
get
Id
(
)),
computer_partition
.
get
(
"compute_partition_id"
)),
supervisord_partition_configuration_path
=
os
.
path
.
join
(
supervisord_partition_configuration_path
=
os
.
path
.
join
(
_getSupervisordConfigurationDirectory
(
self
.
instance_root
),
'%s.conf'
%
_getSupervisordConfigurationDirectory
(
self
.
instance_root
),
'%s.conf'
%
computer_partition_id
),
computer_partition_id
),
...
@@ -1871,9 +1942,16 @@ stderr_logfile_backups=1
...
@@ -1871,9 +1942,16 @@ stderr_logfile_backups=1
local_partition
.
stop
()
local_partition
.
stop
()
local_partition
.
_updateCertificate
()
local_partition
.
_updateCertificate
()
try
:
try
:
computer_partition
.
stopped
()
if
not
self
.
api_backward_compatibility
:
self
.
slap
.
jio_api_connector
({
"portal_type"
:
"Software Instance"
,
"reference"
:
computer_partition
.
get
(
"reference"
),
"reported_state"
:
"started"
})
else
:
computer_partition
[
"slap_partition"
].
stopped
()
except
(
SystemExit
,
KeyboardInterrupt
):
except
(
SystemExit
,
KeyboardInterrupt
):
computer_partition
.
error
(
traceback
.
format_exc
(),
logger
=
self
.
logger
)
self
.
sendPartitionError
(
computer_partition
,
traceback
.
format_exc
(),
logger
=
self
.
logger
)
raise
raise
except
Exception
:
except
Exception
:
pass
pass
...
@@ -1881,9 +1959,9 @@ stderr_logfile_backups=1
...
@@ -1881,9 +1959,9 @@ stderr_logfile_backups=1
for
manager
in
self
.
_manager_list
:
for
manager
in
self
.
_manager_list
:
manager
.
report
(
local_partition
)
manager
.
report
(
local_partition
)
if
computer_partition
.
get
Id
(
)
in
report_usage_issue_cp_list
:
if
computer_partition
.
get
(
"compute_partition_id"
)
in
report_usage_issue_cp_list
:
self
.
logger
.
info
(
'Ignoring destruction of %r, as no report usage was sent'
%
self
.
logger
.
info
(
'Ignoring destruction of %r, as no report usage was sent'
%
computer_partition
.
get
Id
(
))
computer_partition
.
get
(
"compute_partition_id"
))
continue
continue
if
self
.
_checkWaitProcessList
(
local_partition
,
if
self
.
_checkWaitProcessList
(
local_partition
,
state_list
=
[
'RUNNING'
,
'STARTING'
]):
state_list
=
[
'RUNNING'
,
'STARTING'
]):
...
@@ -1892,24 +1970,31 @@ stderr_logfile_backups=1
...
@@ -1892,24 +1970,31 @@ stderr_logfile_backups=1
continue
continue
destroyed
=
local_partition
.
destroy
()
destroyed
=
local_partition
.
destroy
()
except
(
SystemExit
,
KeyboardInterrupt
):
except
(
SystemExit
,
KeyboardInterrupt
):
computer_partition
.
error
(
traceback
.
format_exc
(),
logger
=
self
.
logger
)
self
.
sendPartitionError
(
computer_partition
,
traceback
.
format_exc
(),
logger
=
self
.
logger
)
raise
raise
except
Exception
:
except
Exception
:
clean_run
=
False
clean_run
=
False
self
.
logger
.
exception
(
''
)
self
.
logger
.
exception
(
''
)
exc
=
traceback
.
format_exc
()
exc
=
traceback
.
format_exc
()
computer_partition
.
error
(
exc
,
logger
=
self
.
logger
)
self
.
sendPartitionError
(
computer_partition
,
exc
,
logger
=
self
.
logger
)
try
:
try
:
if
destroyed
:
if
destroyed
:
computer_partition
.
destroyed
()
if
not
self
.
api_backward_compatibility
:
self
.
slap
.
jio_api_connector
({
"portal_type"
:
"Software Instance"
,
"reference"
:
computer_partition
.
get
(
"reference"
),
"reported_state"
:
"destroyed"
})
else
:
computer_partition
[
"slap_partition"
].
destroyed
()
except
NotFoundError
:
except
NotFoundError
:
self
.
logger
.
debug
(
'Ignored slap error while trying to inform about '
self
.
logger
.
debug
(
'Ignored slap error while trying to inform about '
'destroying not fully configured Computer Partition %r'
%
'destroying not fully configured Computer Partition %r'
%
computer_partition
.
get
Id
(
))
computer_partition
.
get
(
"compute_partition_id"
))
except
ServerError
as
server_error
:
except
ServerError
as
server_error
:
self
.
logger
.
debug
(
'Ignored server error while trying to inform about '
self
.
logger
.
debug
(
'Ignored server error while trying to inform about '
'destroying Computer Partition %r. Error is:
\
n
%r'
%
'destroying Computer Partition %r. Error is:
\
n
%r'
%
(
computer_partition
.
get
Id
(
),
server_error
.
args
[
0
]))
(
computer_partition
.
get
(
"compute_partition_id"
),
server_error
.
args
[
0
]))
self
.
logger
.
info
(
'Finished usage reports.'
)
self
.
logger
.
info
(
'Finished usage reports.'
)
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment