Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
S
slapos.toolbox
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Łukasz Nowak
slapos.toolbox
Commits
cfa06942
Commit
cfa06942
authored
Dec 11, 2019
by
Łukasz Nowak
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Feature/promise check surykatka json
/reviewed-on
!72
parent
7ae224cb
Changes
2
Show whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
543 additions
and
0 deletions
+543
-0
slapos/promise/plugin/check_surykatka_json.py
slapos/promise/plugin/check_surykatka_json.py
+127
-0
slapos/test/promise/plugin/test_check_surykatka_json.py
slapos/test/promise/plugin/test_check_surykatka_json.py
+416
-0
No files found.
slapos/promise/plugin/check_surykatka_json.py
0 → 100644
View file @
cfa06942
from
zope.interface
import
implementer
from
slapos.grid.promise
import
interface
from
slapos.grid.promise.generic
import
GenericPromise
import
datetime
import
email.utils
import
json
import
os
import
time
@
implementer
(
interface
.
IPromise
)
class
RunPromise
(
GenericPromise
):
def
__init__
(
self
,
config
):
super
(
RunPromise
,
self
).
__init__
(
config
)
# Set frequency compatible to default surykatka interval - 2 minutes
self
.
setPeriodicity
(
float
(
self
.
getConfig
(
'frequency'
,
2
)))
def
senseBotStatus
(
self
):
key
=
'bot_status'
def
logError
(
msg
,
*
args
):
self
.
logger
.
error
(
key
+
': '
+
msg
,
*
args
)
if
key
not
in
self
.
surykatka_json
:
logError
(
"%r not in %r"
,
key
,
self
.
json_file
)
return
bot_status_list
=
self
.
surykatka_json
[
key
]
if
len
(
bot_status_list
)
==
0
:
logError
(
"%r empty in %r"
,
key
,
self
.
json_file
)
return
bot_status
=
bot_status_list
[
0
]
if
bot_status
.
get
(
'text'
)
!=
'loop'
:
logError
(
"No type loop detected in %r"
,
self
.
json_file
)
return
timetuple
=
email
.
utils
.
parsedate
(
bot_status
[
'date'
])
last_bot_datetime
=
datetime
.
datetime
.
fromtimestamp
(
time
.
mktime
(
timetuple
))
delta
=
self
.
utcnow
-
last_bot_datetime
# sanity check
if
delta
<
datetime
.
timedelta
(
minutes
=
0
):
logError
(
'Last bot datetime %s is in future, UTC now %s'
,
last_bot_datetime
,
self
.
utcnow
)
return
if
delta
>
datetime
.
timedelta
(
minutes
=
15
):
logError
(
'Last bot datetime %s is more than 15 minutes old, UTC now %s'
,
last_bot_datetime
,
self
.
utcnow
)
return
self
.
logger
.
info
(
'%s: Last bot status from %s ok, UTC now is %s'
,
key
,
last_bot_datetime
,
self
.
utcnow
)
def
senseHttpQuery
(
self
):
key
=
'http_query'
def
logError
(
msg
,
*
args
):
self
.
logger
.
error
(
key
+
': '
+
msg
,
*
args
)
if
key
not
in
self
.
surykatka_json
:
logError
(
"%r not in %r"
,
key
,
self
.
json_file
)
return
url
=
self
.
getConfig
(
'url'
)
status_code
=
self
.
getConfig
(
'status-code'
)
ip_list
=
self
.
getConfig
(
'ip-list'
,
''
).
split
()
entry_list
=
[
q
for
q
in
self
.
surykatka_json
[
key
]
if
q
[
'url'
]
==
url
]
if
len
(
entry_list
)
==
0
:
logError
(
'No data for %r'
,
url
)
return
error_list
=
[]
for
entry
in
entry_list
:
if
str
(
entry
[
'status_code'
])
!=
str
(
status_code
):
error_list
.
append
(
'IP %s got status code %s instead of %s'
%
(
entry
[
'ip'
],
entry
[
'status_code'
],
status_code
))
db_ip_list
=
[
q
[
'ip'
]
for
q
in
entry_list
]
if
len
(
ip_list
):
if
set
(
ip_list
)
!=
set
(
db_ip_list
):
error_list
.
append
(
'expected IPs %s differes from got %s'
%
(
' '
.
join
(
ip_list
),
' '
.
join
(
db_ip_list
)))
if
len
(
error_list
):
logError
(
'Problem with %s: '
%
(
url
,)
+
', '
.
join
(
error_list
))
return
if
len
(
ip_list
)
>
0
:
self
.
logger
.
info
(
'%s: %s replied correctly with status code %s on ip list %s'
,
key
,
url
,
status_code
,
' '
.
join
(
ip_list
))
else
:
self
.
logger
.
info
(
'%s: %s replied correctly with status code %s'
,
key
,
url
,
status_code
)
def
sense
(
self
):
"""
Check if frontend URL is available
"""
test_utcnow
=
self
.
getConfig
(
'test-utcnow'
)
if
test_utcnow
:
self
.
utcnow
=
datetime
.
datetime
.
fromtimestamp
(
time
.
mktime
(
email
.
utils
.
parsedate
(
test_utcnow
)))
else
:
self
.
utcnow
=
datetime
.
datetime
.
utcnow
()
self
.
json_file
=
self
.
getConfig
(
'json-file'
,
''
)
if
not
os
.
path
.
exists
(
self
.
json_file
):
self
.
logger
.
error
(
'File %r does not exists'
,
self
.
json_file
)
return
with
open
(
self
.
json_file
)
as
fh
:
try
:
self
.
surykatka_json
=
json
.
load
(
fh
)
except
Exception
:
self
.
logger
.
error
(
"Problem loading JSON from %r"
,
self
.
json_file
)
return
report
=
self
.
getConfig
(
'report'
)
if
report
==
'bot_status'
:
return
self
.
senseBotStatus
()
elif
report
==
'http_query'
:
return
self
.
senseHttpQuery
()
else
:
self
.
logger
.
error
(
"Report %r is not supported"
,
report
)
return
def
anomaly
(
self
):
return
self
.
_test
(
result_count
=
3
,
failure_amount
=
3
)
slapos/test/promise/plugin/test_check_surykatka_json.py
0 → 100644
View file @
cfa06942
from
slapos.grid.promise
import
PromiseError
from
slapos.test.promise.plugin
import
TestPromisePluginMixin
import
os
import
shutil
import
tempfile
class
CheckSurykatkaJSONMixin
(
TestPromisePluginMixin
):
promise_name
=
'check-surykatka-json.py'
def
setUp
(
self
):
self
.
working_directory
=
tempfile
.
mkdtemp
()
self
.
json_file
=
os
.
path
.
join
(
self
.
working_directory
,
'surykatka.json'
)
self
.
addCleanup
(
shutil
.
rmtree
,
self
.
working_directory
)
TestPromisePluginMixin
.
setUp
(
self
)
def
writeSurykatkaPromise
(
self
,
d
=
None
):
if
d
is
None
:
d
=
{}
content_list
=
[
"from slapos.promise.plugin.check_surykatka_json import RunPromise"
]
content_list
.
append
(
'extra_config_dict = {'
)
for
k
,
v
in
d
.
items
():
content_list
.
append
(
" '%s': '%s',"
%
(
k
,
v
))
content_list
.
append
(
'}'
)
self
.
writePromise
(
self
.
promise_name
,
'
\
n
'
.
join
(
content_list
))
def
writeSurykatkaJson
(
self
,
content
):
with
open
(
self
.
json_file
,
'w'
)
as
fh
:
fh
.
write
(
content
)
def
assertFailedMessage
(
self
,
result
,
message
):
self
.
assertEqual
(
result
[
'result'
][
'failed'
],
True
)
self
.
assertEqual
(
result
[
'result'
][
'message'
],
message
)
def
assertPassedMessage
(
self
,
result
,
message
):
self
.
assertEqual
(
result
[
'result'
][
'failed'
],
False
)
self
.
assertEqual
(
result
[
'result'
][
'message'
],
message
)
class
TestCheckSurykatkaJSON
(
CheckSurykatkaJSONMixin
):
def
test_no_config
(
self
):
self
.
writeSurykatkaPromise
()
self
.
configureLauncher
()
with
self
.
assertRaises
(
PromiseError
):
self
.
launcher
.
run
()
self
.
assertFailedMessage
(
self
.
getPromiseResult
(
self
.
promise_name
),
"File '' does not exists"
)
def
test_not_existing_file
(
self
):
self
.
writeSurykatkaPromise
({
'json-file'
:
self
.
json_file
})
self
.
configureLauncher
()
with
self
.
assertRaises
(
PromiseError
):
self
.
launcher
.
run
()
self
.
assertFailedMessage
(
self
.
getPromiseResult
(
self
.
promise_name
),
"File '%s' does not exists"
%
(
self
.
json_file
,))
def
test_empty_file
(
self
):
self
.
writeSurykatkaPromise
({
'json-file'
:
self
.
json_file
})
self
.
writeSurykatkaJson
(
''
)
self
.
configureLauncher
()
with
self
.
assertRaises
(
PromiseError
):
self
.
launcher
.
run
()
self
.
assertFailedMessage
(
self
.
getPromiseResult
(
self
.
promise_name
),
"Problem loading JSON from '%s'"
%
(
self
.
json_file
,))
class
TestCheckSurykatkaJSONUnknownReport
(
CheckSurykatkaJSONMixin
):
def
test
(
self
):
self
.
writeSurykatkaPromise
(
{
'report'
:
'NOT_EXISTING_ENTRY'
,
'json-file'
:
self
.
json_file
,
}
)
self
.
writeSurykatkaJson
(
"""{
}
"""
)
self
.
configureLauncher
()
with
self
.
assertRaises
(
PromiseError
):
self
.
launcher
.
run
()
self
.
assertFailedMessage
(
self
.
getPromiseResult
(
self
.
promise_name
),
"Report 'NOT_EXISTING_ENTRY' is not supported"
)
class
TestCheckSurykatkaJSONBotStatus
(
CheckSurykatkaJSONMixin
):
def
test
(
self
):
self
.
writeSurykatkaPromise
(
{
'report'
:
'bot_status'
,
'json-file'
:
self
.
json_file
,
'test-utcnow'
:
'Wed, 13 Dec 2222 09:11:12 -0000'
}
)
self
.
writeSurykatkaJson
(
"""{
"bot_status": [
{
"date": "Wed, 13 Dec 2222 09:10:11 -0000",
"text": "loop"
}
]
}
"""
)
self
.
configureLauncher
()
self
.
launcher
.
run
()
self
.
assertPassedMessage
(
self
.
getPromiseResult
(
self
.
promise_name
),
"bot_status: Last bot status from 2222-12-13 09:10:11 ok, "
"UTC now is 2222-12-13 09:11:12"
)
def
test_bot_status_future
(
self
):
self
.
writeSurykatkaPromise
(
{
'report'
:
'bot_status'
,
'json-file'
:
self
.
json_file
,
'test-utcnow'
:
'Wed, 13 Dec 2222 09:11:12 -0000'
}
)
self
.
writeSurykatkaJson
(
"""{
"bot_status": [
{
"date": "Wed, 13 Dec 2223 09:10:11 -0000",
"text": "loop"
}
]
}
"""
)
self
.
configureLauncher
()
with
self
.
assertRaises
(
PromiseError
):
self
.
launcher
.
run
()
self
.
assertFailedMessage
(
self
.
getPromiseResult
(
self
.
promise_name
),
"bot_status: Last bot datetime 2223-12-13 09:10:11 is in "
"future, UTC now 2222-12-13 09:11:12"
)
def
test_bot_status_old
(
self
):
self
.
writeSurykatkaPromise
(
{
'report'
:
'bot_status'
,
'json-file'
:
self
.
json_file
,
'test-utcnow'
:
'Wed, 13 Dec 2223 09:26:12 -0000'
}
)
self
.
writeSurykatkaJson
(
"""{
"bot_status": [
{
"date": "Wed, 13 Dec 2223 09:10:11 -0000",
"text": "loop"
}
]
}
"""
)
self
.
configureLauncher
()
with
self
.
assertRaises
(
PromiseError
):
self
.
launcher
.
run
()
self
.
assertFailedMessage
(
self
.
getPromiseResult
(
self
.
promise_name
),
"bot_status: Last bot datetime 2223-12-13 09:10:11 is "
"more than 15 minutes old, UTC now 2223-12-13 09:26:12"
)
def
test_not_bot_status
(
self
):
self
.
writeSurykatkaPromise
(
{
'report'
:
'bot_status'
,
'json-file'
:
self
.
json_file
,
}
)
self
.
writeSurykatkaJson
(
"""{
}
"""
)
self
.
configureLauncher
()
with
self
.
assertRaises
(
PromiseError
):
self
.
launcher
.
run
()
self
.
assertFailedMessage
(
self
.
getPromiseResult
(
self
.
promise_name
),
"bot_status: 'bot_status' not in '%s'"
%
(
self
.
json_file
,))
def
test_empty_bot_status
(
self
):
self
.
writeSurykatkaPromise
(
{
'report'
:
'bot_status'
,
'json-file'
:
self
.
json_file
,
}
)
self
.
writeSurykatkaJson
(
"""{
"bot_status": []
}
"""
)
self
.
configureLauncher
()
with
self
.
assertRaises
(
PromiseError
):
self
.
launcher
.
run
()
self
.
assertFailedMessage
(
self
.
getPromiseResult
(
self
.
promise_name
),
"bot_status: 'bot_status' empty in '%s'"
%
(
self
.
json_file
,))
class
TestCheckSurykatkaJSONHttpQuery
(
CheckSurykatkaJSONMixin
):
def
test
(
self
):
self
.
writeSurykatkaPromise
(
{
'report'
:
'http_query'
,
'json-file'
:
self
.
json_file
,
'url'
:
'https://www.erp5.com/'
,
'status-code'
:
'302'
,
'ip-list'
:
'127.0.0.1 127.0.0.2'
}
)
self
.
writeSurykatkaJson
(
"""{
"http_query": [
{
"date": "Wed, 11 Dec 2019 09:35:28 -0000",
"ip": "127.0.0.1",
"status_code": 302,
"url": "https://www.erp5.com/"
},
{
"date": "Wed, 11 Dec 2019 09:35:28 -0000",
"ip": "127.0.0.2",
"status_code": 302,
"url": "https://www.erp5.com/"
},
{
"date": "Wed, 11 Dec 2019 09:35:28 -0000",
"ip": "176.31.129.213",
"status_code": 200,
"url": "https://www.erp5.org/"
}
]
}
"""
)
self
.
configureLauncher
()
self
.
launcher
.
run
()
self
.
assertPassedMessage
(
self
.
getPromiseResult
(
self
.
promise_name
),
"http_query: https://www.erp5.com/ replied correctly with "
"status code 302 on ip list 127.0.0.1 127.0.0.2"
)
def
test_no_ip_list
(
self
):
self
.
writeSurykatkaPromise
(
{
'report'
:
'http_query'
,
'json-file'
:
self
.
json_file
,
'url'
:
'https://www.erp5.com/'
,
'status-code'
:
'302'
,
}
)
self
.
writeSurykatkaJson
(
"""{
"http_query": [
{
"date": "Wed, 11 Dec 2019 09:35:28 -0000",
"ip": "127.0.0.1",
"status_code": 302,
"url": "https://www.erp5.com/"
},
{
"date": "Wed, 11 Dec 2019 09:35:28 -0000",
"ip": "127.0.0.2",
"status_code": 302,
"url": "https://www.erp5.com/"
},
{
"date": "Wed, 11 Dec 2019 09:35:28 -0000",
"ip": "176.31.129.213",
"status_code": 200,
"url": "https://www.erp5.org/"
}
]
}
"""
)
self
.
configureLauncher
()
self
.
launcher
.
run
()
self
.
assertPassedMessage
(
self
.
getPromiseResult
(
self
.
promise_name
),
"http_query: https://www.erp5.com/ replied correctly with "
"status code 302"
)
def
test_bad_code
(
self
):
self
.
writeSurykatkaPromise
(
{
'report'
:
'http_query'
,
'json-file'
:
self
.
json_file
,
'url'
:
'https://www.erp5.com/'
,
'status-code'
:
'301'
,
}
)
self
.
writeSurykatkaJson
(
"""{
"http_query": [
{
"date": "Wed, 11 Dec 2019 09:35:28 -0000",
"ip": "127.0.0.1",
"status_code": 302,
"url": "https://www.erp5.com/"
},
{
"date": "Wed, 11 Dec 2019 09:35:28 -0000",
"ip": "127.0.0.2",
"status_code": 301,
"url": "https://www.erp5.com/"
},
{
"date": "Wed, 11 Dec 2019 09:35:28 -0000",
"ip": "176.31.129.213",
"status_code": 200,
"url": "https://www.erp5.org/"
}
]
}
"""
)
self
.
configureLauncher
()
with
self
.
assertRaises
(
PromiseError
):
self
.
launcher
.
run
()
self
.
assertFailedMessage
(
self
.
getPromiseResult
(
self
.
promise_name
),
"http_query: Problem with https://www.erp5.com/: "
"IP 127.0.0.1 got status code 302 instead of 301"
)
def
test_bad_ip
(
self
):
self
.
writeSurykatkaPromise
(
{
'report'
:
'http_query'
,
'json-file'
:
self
.
json_file
,
'url'
:
'https://www.erp5.com/'
,
'status-code'
:
'301'
,
'ip-list'
:
'127.0.0.1 127.0.0.2'
}
)
self
.
writeSurykatkaJson
(
"""{
"http_query": [
{
"date": "Wed, 11 Dec 2019 09:35:28 -0000",
"ip": "127.0.0.1",
"status_code": 301,
"url": "https://www.erp5.com/"
},
{
"date": "Wed, 11 Dec 2019 09:35:28 -0000",
"ip": "127.0.0.4",
"status_code": 301,
"url": "https://www.erp5.com/"
},
{
"date": "Wed, 11 Dec 2019 09:35:28 -0000",
"ip": "176.31.129.213",
"status_code": 200,
"url": "https://www.erp5.org/"
}
]
}
"""
)
self
.
configureLauncher
()
with
self
.
assertRaises
(
PromiseError
):
self
.
launcher
.
run
()
self
.
assertFailedMessage
(
self
.
getPromiseResult
(
self
.
promise_name
),
"http_query: Problem with https://www.erp5.com/: "
"expected IPs 127.0.0.1 127.0.0.2 differes from got 127.0.0.1 127.0.0.4"
)
def
test_bad_ip_status_code
(
self
):
self
.
writeSurykatkaPromise
(
{
'report'
:
'http_query'
,
'json-file'
:
self
.
json_file
,
'url'
:
'https://www.erp5.com/'
,
'status-code'
:
'301'
,
'ip-list'
:
'127.0.0.1 127.0.0.2'
}
)
self
.
writeSurykatkaJson
(
"""{
"http_query": [
{
"date": "Wed, 11 Dec 2019 09:35:28 -0000",
"ip": "127.0.0.1",
"status_code": 302,
"url": "https://www.erp5.com/"
},
{
"date": "Wed, 11 Dec 2019 09:35:28 -0000",
"ip": "127.0.0.4",
"status_code": 301,
"url": "https://www.erp5.com/"
},
{
"date": "Wed, 11 Dec 2019 09:35:28 -0000",
"ip": "176.31.129.213",
"status_code": 200,
"url": "https://www.erp5.org/"
}
]
}
"""
)
self
.
configureLauncher
()
with
self
.
assertRaises
(
PromiseError
):
self
.
launcher
.
run
()
self
.
assertFailedMessage
(
self
.
getPromiseResult
(
self
.
promise_name
),
"http_query: Problem with https://www.erp5.com/: "
"IP 127.0.0.1 got status code 302 instead of 301, "
"expected IPs 127.0.0.1 127.0.0.2 differes from got "
"127.0.0.1 127.0.0.4"
)
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment