Skip to content

S
slapos.toolbox
Commit 2af4ca37 authored by Hardik Juneja's avatar Hardik Juneja Committed by Rafael Monnerat
Browse files
Options

Add ptdigest and apachedex result check promises with tests 

/reviewed-on nexedi/slapos.toolbox!16
parent e47f066c
Show whitespace changes
Inline Side-by-side
Showing with 361 additions and 0 deletions
setup.py
View file @ 2af4ca37
...@@ -73,6 +73,8 @@ setup(name=name, ...@@ -73,6 +73,8 @@ setup(name=name,
'check-web-page-http-cache-hit = slapos.promise.check_web_page_http_cache_hit:main', 'check-web-page-http-cache-hit = slapos.promise.check_web_page_http_cache_hit:main',
'check-feed-as-promise = slapos.checkfeedaspromise:main', 'check-feed-as-promise = slapos.checkfeedaspromise:main',
'check-error-on-apache-log = slapos.promise.check_error_on_apache_log:main', 'check-error-on-apache-log = slapos.promise.check_error_on_apache_log:main',
'check-apachedex-result = slapos.promise.check_apachedex_result:main',
'check-slow-queries-digest-result = slapos.promise.check_slow_queries_digest_result:main',
'check-re6st-optimal-status = slapos.promise.check_re6st_optimal_status:main', 'check-re6st-optimal-status = slapos.promise.check_re6st_optimal_status:main',
'clouddestroy = slapos.cloudmgr.destroy:main', 'clouddestroy = slapos.cloudmgr.destroy:main',
'cloudgetprivatekey = slapos.cloudmgr.getprivatekey:main', 'cloudgetprivatekey = slapos.cloudmgr.getprivatekey:main',
......
slapos/promise/check_apachedex_result/__init__.py 0 → 100644
View file @ 2af4ca37
#!/usr/bin/env python
"""
Check if a apachedex result matches the desired threshold or raises an error.
"""
import json
import os
import re
import sys
import time
import datetime
import argparse
def checkApachedexResult(apachedex_file, apachedex_report_status_file, desired_threshold):
if not os.path.isfile(apachedex_file):
open(apachedex_file, 'a').close()
with open(apachedex_file, 'r') as content_file:
content = content_file.read()
if len(content) == 0:
# File is empty
# Check the creation date of the file
# and if the date is greater than 30 hour throw an error
date_created = os.path.getmtime(apachedex_file)
current_date = time.mktime(datetime.datetime.now().timetuple())
if current_date - date_created > 108000:
with open(apachedex_report_status_file) as f:
json_content = f.read()
# Print the message from the monitor report
if len(json_content) > 0:
message = json.loads(json_content)["message"]
return message + "\nFile modification date is greater than 30 hours"
return "File modification date is greater than 30 hour"
else:
#TODO: this is old regex for bash, improve it
regex = r"Overall<\/h2>.*\n<th>apdex<\/th><th>.*?\n<\/tr><tr>\n<td [^<]*>(.*?)%<\/td>"
m = re.findall(regex, content)
if len(m) > 0:
result=int(m[0])
if result > desired_threshold:
return "Thanks for keeping it all clean, result is " + str(result)
else:
return "Threshold is lower than expected: Expected was " + \
str(desired_threshold) +" and current result is " + str(result)
return "No result found in the apdex file or the file is corrupted"
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--apachedex_file", required=True)
parser.add_argument("--status_file", required=True)
parser.add_argument("--threshold", required=True)
args = parser.parse_args()
if args.apachedex_file:
args.apachedex_file = args.apachedex_file + "/ApacheDex-" + datetime.date.today().strftime('%Y-%m-%d') + ".html"
result = checkApachedexResult(args.apachedex_file, args.status_file, args.threshold)
print result
if result != "OK":
sys.exit(1)
slapos/promise/check_slow_queries_digest_result/__init__.py 0 → 100644
View file @ 2af4ca37
#!/usr/bin/env python
"""
Check if a mariadb result matches the desired threshold or raises an error.
"""
import json
import os
import re
import sys
import time
import datetime
import argparse
def checkMariadbDigestResult(mariadbdex_file, mariadbdex_report_status_file,
max_query_threshold, slowest_query_threshold):
if not os.path.isfile(mariadbdex_file):
open(mariadbdex_file, 'a').close()
with open(mariadbdex_file, 'r') as content_file:
content = content_file.read()
if len(content) == 0:
# File is empty
# Check the creation date of the file
# and if the date is greater than 30 hour throw an error
date_created = os.path.getmtime(mariadbdex_file)
current_date = time.mktime(datetime.datetime.now().timetuple())
if current_date - date_created > 108000:
with open(mariadbdex_report_status_file) as f:
json_content = f.read()
# Print the message from the monitor report
if len(json_content) > 0:
message = json.loads(json_content)["message"]
return message + "\nFile modification date is greater than 30 hours"
return "File modification date is greater than 30 hour"
else:
regex = r"Overall: (.*) total,[\S\s]*# Exec time( *([\d]+)m?s?){4}"
m = re.findall(regex, content)
if len(m) > 0:
total_queries_exec=m[0][0].strip()
slowest_query_time=int(m[0][2].strip())
has_k=total_queries_exec[-1:]
if has_k == "k":
pre=total_queries_exec[:-1]
total_queries_exec=float(pre)*1000
else:
total_queries_exec=int(total_queries_exec)
if total_queries_exec < max_query_threshold and slowest_query_time < slowest_query_threshold:
return "Thanks for keeping it all clean, total queries are : " + str(total_queries_exec) + \
" slowest query time is : " + str(slowest_query_time)
else:
return "Threshold is lower than expected: \nExpected total queries : " + \
str(max_query_threshold) +" and current is: " + str(total_queries_exec) + "\n"+ \
"Expected slowest query : " + str(slowest_query_threshold) + " and current is: " + str(slowest_query_time)
return "No result found in the apdex file or the file is corrupted"
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--ptdigest_file", required=True)
parser.add_argument("--status_file", required=True)
parser.add_argument("--max_queries_threshold", required=True)
parser.add_argument("--slowest_query_threshold", required=True)
args = parser.parse_args()
result = checkMariadbDigestResult(args.ptdigest_file, args.status_file, args.max_queries_threshold, args.slowest_query_threshold)
print result
if result != "OK":
sys.exit(1)
slapos/test/promise/data/apachedex.html 0 → 100644
View file @ 2af4ca37
<body>
<h1>Overall</h1><h2>Parameters</h2><table class="stats"><tr><th class="text">apdex threshold</th><td>1.00s</td></tr><tr><th class="text">period</th><td>halfday (auto)</td></tr><tr><th class="text">timezone</th><td>(input's)</td></tr></table><h2>Hits per halfday</h2><table class="stats"><tr><th>date</th><th>hits</th></tr></table><h1 id="0" title="'/erp5(/|$|/?)'">/erp5(/|$|/?)</h1><h2>Stats per module</h2><table class="stats stats_erp5"><tr><th rowspan="2" colspan="3">module</th><th colspan="4" class="overall_right">overall</th>
<th colspan="4">2017/08/08 00</th>
<th colspan="4">2017/08/08 12</th>
</tr><tr>
<th>apdex</th><th>hits</th><th>avg (s)</th><th class="overall_right">max (s)</th>
<th>apdex</th><th>hits</th><th>avg (s)</th><th>max (s)</th>
<th>apdex</th><th>hits</th><th>avg (s)</th><th>max (s)</th>
</tr>
<tr class="group_top group_bottom"><th colspan="2">site search</th>
<td class="text group_right hidden_graph">
</td>
<td style="" class="no_hit group_left">100%</td><td class="no_hit">0</td><td class=" no_hit">0.00</td><td class=" no_hit group_right overall_right">0.00</td>
<td style="" class="no_hit group_left">100%</td><td class="no_hit">0</td><td class=" no_hit">0.00</td><td class=" no_hit group_right ">0.00</td>
<td style="" class="no_hit group_left">100%</td><td class="no_hit">0</td><td class=" no_hit">0.00</td><td class=" no_hit group_right ">0.00</td>
<tr class="group_top group_bottom"><th colspan="2">other</th>
<td class="text group_right hidden_graph">
<span class="action" onclick="toggleGraph(this)">+</span><div class="positioner"><div class="container"><div class="title">other</div><div class="action close" onclick="hideGraph(this)">close</div>
<h2>apdex</h2><div class="graph" style="width:600px;height:300px" data-points="[[[1502175600000, 100.0], [1502177400000, 100.0], [1502179200000, 100.0], [1502181000000, 100.0], [1502182800000, 100.0], [1502184600000, 100.0], [1502186400000, 100.0], [1502188200000, 100.0], [1502190000000, 100.0], [1502191800000, 100.0], [1502193600000, 100.0], [1502195400000, 100.0], [1502197200000, 0.0], [1502199000000, 100.0], [1502200800000, 100.0], [1502202600000, 100.0], [1502204400000, 100.0], [1502206200000, 100.0], [1502208000000, 100.0], [1502209800000, 100.0], [1502211600000, 100.0], [1502213400000, 100.0], [1502215200000, 100.0], [1502217000000, 100.0]]]" data-options="{&quot;grid&quot;: {&quot;hoverable&quot;: true}, &quot;lines&quot;: {&quot;show&quot;: true}, &quot;xaxis&quot;: {&quot;timeformat&quot;: &quot;%Y/&lt;br/&gt;%m/%d&lt;br/&gt; %H:%M&quot;, &quot;minTickSize&quot;: [1.15, &quot;hour&quot;], &quot;mode&quot;: &quot;time&quot;}, &quot;yaxis&quot;: {&quot;max&quot;: 100, &quot;axisLabel&quot;: &quot;apdex (%)&quot;, &quot;labelWidth&quot;: 18, &quot;transform&quot;: null, &quot;min&quot;: null}}"></div><div class="tooltip"><span class="x"></span><br/><span class="y"></span></div><h2>Hits (per 30 minutes)</h2><div class="graph" style="width:600px;height:300px" data-points="[{&quot;color&quot;: &quot;red&quot;, &quot;data&quot;: [[1502175600000, 0], [1502177400000, 0], [1502179200000, 0], [1502181000000, 0], [1502182800000, 0], [1502184600000, 0], [1502186400000, 0], [1502188200000, 0], [1502190000000, 0], [1502191800000, 0], [1502193600000, 0], [1502195400000, 0], [1502197200000, 0], [1502199000000, 0], [1502200800000, 0], [1502202600000, 0], [1502204400000, 0], [1502206200000, 0], [1502208000000, 0], [1502209800000, 0], [1502211600000, 0], [1502213400000, 0], [1502215200000, 0], [1502217000000, 0]], &quot;label&quot;: &quot;Errors&quot;}, {&quot;data&quot;: [[1502175600000, 5], [1502177400000, 0], [1502179200000, 12], [1502181000000, 3], [1502182800000, 5], [1502184600000, 1], [1502186400000, 23], [1502188200000, 19], [1502190000000, 1], [1502191800000, 0], [1502193600000, 0], [1502195400000, 6], [1502197200000, 1], [1502199000000, 4], [1502200800000, 6], [1502202600000, 6], [1502204400000, 16], [1502206200000, 0], [1502208000000, 42], [1502209800000, 11], [1502211600000, 14], [1502213400000, 10], [1502215200000, 6], [1502217000000, 12]], &quot;label&quot;: &quot;Hits&quot;}]" data-options="{&quot;grid&quot;: {&quot;hoverable&quot;: true}, &quot;lines&quot;: {&quot;show&quot;: true}, &quot;legend&quot;: {&quot;backgroundOpacity&quot;: 0.25}, &quot;xaxis&quot;: {&quot;timeformat&quot;: &quot;%Y/&lt;br/&gt;%m/%d&lt;br/&gt; %H:%M&quot;, &quot;minTickSize&quot;: [1.15, &quot;hour&quot;], &quot;mode&quot;: &quot;time&quot;}, &quot;yaxis&quot;: {&quot;labelWidth&quot;: 18, &quot;min&quot;: null, &quot;max&quot;: null, &quot;axisLabel&quot;: &quot;Hits&quot;, &quot;tickDecimals&quot;: 0, &quot;transform&quot;: null}}"></div><div class="tooltip"><span class="x"></span><br/><span class="y"></span></div>
</div></div>
</td>
<td style="color: #000; background-color: #eee" class=" group_left">100%</td><td class="">203</td><td class=" ">0.05</td><td class=" group_right overall_right">0.19</td>
<td style="color: #000; background-color: #fff" class=" group_left">100%</td><td class="">69</td><td class=" ">0.05</td><td class=" group_right ">0.19</td>
<td style="color: #000; background-color: #eee" class=" group_left">99%</td><td class="">134</td><td class=" ">0.05</td><td class=" group_right ">0.12</td>
</tr></table><h2>Per-level overall</h2><table class="stats"><tr><th>level</th>
<th>apdex</th><th>hits</th><th>avg (s)</th><th>max (s)</th>
</tr><tr><th>other</th>
<td style="color: #000; background-color: #eee" class=" group_left">100%</td><td class="">203</td><td class=" ">0.05</td><td class=" group_right ">0.19</td>
</tr><tr><th>site search</th>
<td style="" class="no_hit group_left">100%</td><td class="no_hit">0</td><td class=" no_hit">0.00</td><td class=" no_hit group_right ">0.00</td>
</tr><tr><th>module</th>
<td style="" class="no_hit group_left">100%</td><td class="no_hit">0</td><td class=" no_hit">0.00</td><td class=" no_hit group_right ">0.00</td>
</tr><tr><th>document</th>
<td style="" class="no_hit group_left">100%</td><td class="no_hit">0</td><td class=" no_hit">0.00</td><td class=" no_hit group_right ">0.00</td>
</tr></table>
<h2>Overall</h2><table class="stats"><tr>
<th>apdex</th><th>hits</th><th>avg (s)</th><th>max (s)</th>
</tr><tr>
<td style="" class="no_hit group_left">80%</td><td class="no_hit">0</td><td class=" no_hit">0.00</td><td class=" no_hit group_right ">0.00</td>
</tr></table><h2>Hottest pages</h2><table class="stats"><tr>
<th>apdex</th><th>hits</th><th>avg (s)</th><th>max (s)</th>
<th>url</th></tr>
</table>
<h2>Hits per status code</h2><table class="stats"><tr><th>status</th><th>overall</th>
</tr>
</table></body>
slapos/test/promise/data/ptdigest.html 0 → 100644
View file @ 2af4ca37
# 2.1s user time, 10ms system time, 41.20M rss, 93.19M vsz
# Current date: Thu Aug 24 15:13:57 2017
# Hostname: maincloud-bhost-host-abc
# Files: /var/log/mariadb_slowquery.log
# Overall: 3.42k total, 16 unique, 0.06 QPS, 0.41x concurrency ___________
# Time range: 2017-05-23 00:00:36 to 15:39:06
# Attribute total min max avg 95% stddev median
# ============ ======= ======= ======= ======= ======= ======= =======
# Exec time 23286s 1s 237s 7s 23s 8s 4s
# Lock time 938ms 19us 107ms 274us 152us 4ms 73us
# Rows sent 191.65M 0 82.44M 57.40k 49.17 1.60M 0.99
# Rows examine 16.03G 0 82.44M 4.80M 7.29M 2.57M 4.93M
# Rows affecte 0 0 0 0 0 0 0
# Query size 10.65M 45 851.11k 3.19k 1.86k 42.91k 511.45
# Profile
# Rank Query ID Response time Calls R/Call V/M Item
# ==== ================== ================ ===== ======= ===== ===========
# 1 0xxxxxxxxxsccca666 14644.8405 62.9% 893 16.3996 2.02 SELECT catalo1
# 2 0xxxxxxxxxrcccb666 4216.8862 18.1% 780 5.4063 0.66 SELECT catelog2
# 3 0xxxxxxxxxqcccc666 3054.0369 13.1% 1387 2.2019 0.25 SELECT catalog3
# 4 0xxxxxxxxxpcccd666 828.5682 3.6% 20 41.4284 74.06 SELECT catelog5
# 5 0xxxxxxxxxoccce666 255.1623 1.1% 195 1.3085 0.05 SELECT catalog6
# 6 0xxxxxxxxxncccf666 108.4913 0.5% 35 3.0998 0.85 SELECT catlog7
# 7 0xxxxxxxxxmcccg666 68.8060 0.3% 56 1.2287 0.03 SELECT catalog8
# 9 0xxxxxxxxxlccch666 36.2955 0.2% 32 1.1342 0.01 SELECT catalog9
# MISC 0xMISC 73.0793 0.3% 21 3.4800 0.0 <8 ITEMS>
# Query 1: 0.02 QPS, 0.26x concurrency, ID 0xxxxxxxxxxcccc666 at byte 11808921
# Scores: V/M = 2.02
# Time range: 2017-05-23 00:01:01 to 15:38:13
# Attribute pct total min max avg 95% stddev median
# ============ === ======= ======= ======= ======= ======= ======= =======
# Count 26 893
# Exec time 62 14645s 9s 34s 16s 27s 6s 15s
# Lock time 11 109ms 72us 199us 121us 176us 31us 119us
# Rows sent 0 24.58k 0 50 28.18 49.17 23.27 46.83
# Rows examine 40 6.50G 7.43M 7.55M 7.46M 7.29M 0 7.29M
# Rows affecte 0 0 0 0 0 0 0 0
# Query size 15 1.68M 1.92k 1.92k 1.92k 1.92k 0 1.92k
# String:
# Databases erp
# Hosts 10.0.222.230
# Users user
slapos/test/promise/test_check_apachedex_result.py 0 → 100644
View file @ 2af4ca37
##############################################################################
#
# Copyright (c) 2017 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import unittest
import os
import time
import tempfile
from slapos.test.promise import data
from slapos.promise.check_apachedex_result import checkApachedexResult
class TestCheckApacheDigestResult(unittest.TestCase):
def setUp(self):
_, self.empty_apachedex_file = tempfile.mkstemp()
_, self.old_apachedex_file = tempfile.mkstemp()
_, self.status_file = tempfile.mkstemp()
self.base_path = "/".join(data.__file__.split("/")[:-1])
self.status = "ok"
def test_pass(self):
self.assertEquals("Thanks for keeping it all clean, result is 80",
checkApachedexResult(self.base_path + "/apachedex.html", self.status_file, 60))
def test_empty_file(self):
self.assertEquals("No result found in the apdex file or the file is corrupted",
checkApachedexResult(self.empty_apachedex_file, self.status_file, 60))
def test_fail(self):
self.assertEquals("Threshold is lower than expected: Expected was 90 and current result is 80",
checkApachedexResult(self.base_path + "/apachedex.html", self.status_file, 90))
def test_old_file(self):
os.utime(self.old_apachedex_file, (time.time() - 202800, time.time() - 202800))
self.assertEquals("File modification date is greater than 30 hour",
checkApachedexResult(self.old_apachedex_file, self.status_file, 60))
if __name__ == '__main__':
unittest.main()
slapos/test/promise/test_check_slow_queries_digest_result.py 0 → 100644
View file @ 2af4ca37
##############################################################################
#
# Copyright (c) 2017 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import unittest
import os
import time
import tempfile
from slapos.test.promise import data
from slapos.promise.check_slow_queries_digest_result import checkMariadbDigestResult
class TestCheckSlowQueriesDigestResult(unittest.TestCase):
def setUp(self):
_, self.empty_ptdigest_file = tempfile.mkstemp()
_, self.old_ptdigest_file = tempfile.mkstemp()
_, self.status_file = tempfile.mkstemp()
self.base_path = "/".join(data.__file__.split("/")[:-1])
def test_pass(self):
self.assertEquals("Thanks for keeping it all clean, total queries are : 3420.0 slowest query time is : 34",
checkMariadbDigestResult(self.base_path + "/ptdigest.html", self.status_file, 5000, 100))
def test_empty_file(self):
self.assertEquals("No result found in the apdex file or the file is corrupted",
checkMariadbDigestResult(self.empty_ptdigest_file, self.status_file, 60, 100))
def test_fail(self):
self.assertEquals("Threshold is lower than expected: \nExpected total queries : 90 and current is: 3420.0\nExpected slowest query : 100 and current is: 34",
checkMariadbDigestResult(self.base_path + "/ptdigest.html", self.status_file, 90, 100))
def test_old_file(self):
os.utime(self.old_ptdigest_file, (time.time() - 202800, time.time() - 202800))
self.assertEquals("File modification date is greater than 30 hour",
checkMariadbDigestResult(self.old_ptdigest_file, self.status_file, 60, 100))
if __name__ == '__main__':
unittest.main()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7