Commit 24a1e7d4 authored by Łukasz Nowak's avatar Łukasz Nowak

http: Allow to configure elapsed time intervals

parent 71776c9b
......@@ -84,6 +84,8 @@ class WebBot:
self.config = createConfiguration(**self.config_kw)
timeout = int(self.config["TIMEOUT"])
elapsed_fast = float(self.config["ELAPSED_FAST"])
elapsed_moderate = float(self.config["ELAPSED_moderate"])
# logPlatform(self._db, __version__, status_id)
# Calculate the resolver list
......@@ -142,7 +144,7 @@ class WebBot:
for url in url_dict:
for ip in url_dict[url]:
checkHttpStatus(
self._db, status_id, url, ip, __version__, timeout
self._db, status_id, url, ip, __version__, timeout, elapsed_fast, elapsed_moderate
)
# XXX Check location header and check new url recursively
# XXX Parse HTML, fetch found link, css, js, image
......
......@@ -56,6 +56,10 @@ def createConfiguration(
config[CONFIG_SECTION]["FORMAT"] = "json"
if "TIMEOUT" not in config[CONFIG_SECTION]:
config[CONFIG_SECTION]["TIMEOUT"] = "1"
if "ELAPSED_FAST" not in config[CONFIG_SECTION]:
config[CONFIG_SECTION]["ELAPSED_FAST"] = "0.2"
if "ELAPSED_MODERATE" not in config[CONFIG_SECTION]:
config[CONFIG_SECTION]["ELAPSED_MODERATE"] = "0.5"
if "RELOAD" not in config[CONFIG_SECTION]:
config[CONFIG_SECTION]["RELOAD"] = str(False)
......
......@@ -25,6 +25,8 @@ from peewee import fn
PREFERRED_TYPE = "text/html"
TIMEOUT = 2
ELAPSED_FAST = 0.2
ELAPSED_MODERATE = 0.5
def getUrlHostname(url):
......@@ -105,20 +107,20 @@ def reportHttp(db, ip=None, url=None):
return query
def calculateSpeedRange(total_seconds):
def calculateSpeedRange(total_seconds, fast, moderate):
# Prevent updating the DB by defining acceptable speed range
if total_seconds == 0:
# error cases
return "BAD"
elif total_seconds < 0.2:
elif total_seconds < fast:
return "FAST"
elif total_seconds < 0.5:
elif total_seconds < moderate:
return "MODERATE"
else:
return "SLOW"
def logHttpStatus(db, ip, url, code, total_seconds, status_id):
def logHttpStatus(db, ip, url, code, total_seconds, fast, moderate, status_id):
with db._db.atomic():
try:
......@@ -131,8 +133,8 @@ def logHttpStatus(db, ip, url, code, total_seconds, status_id):
(previous_entry is None)
or (previous_entry.status_code != code)
or (
calculateSpeedRange(previous_entry.total_seconds)
!= calculateSpeedRange(total_seconds)
calculateSpeedRange(previous_entry.total_seconds, fast, moderate)
!= calculateSpeedRange(total_seconds, fast, moderate)
)
):
previous_entry = db.HttpCodeChange.create(
......@@ -145,7 +147,7 @@ def logHttpStatus(db, ip, url, code, total_seconds, status_id):
return previous_entry.status_id
def checkHttpStatus(db, status_id, url, ip, bot_version, timeout=TIMEOUT):
def checkHttpStatus(db, status_id, url, ip, bot_version, timeout=TIMEOUT, elapsed_fast=ELAPSED_FAST, elapsed_moderate=ELAPSED_MODERATE):
parsed_url = urlparse(url)
hostname = parsed_url.hostname
request_kw = {"timeout": timeout}
......@@ -175,5 +177,7 @@ def checkHttpStatus(db, status_id, url, ip, bot_version, timeout=TIMEOUT):
url,
response.status_code,
response.elapsed.total_seconds(),
elapsed_fast,
elapsed_moderate,
status_id,
)
......@@ -185,9 +185,11 @@ class SurykatkaHttpTestCase(unittest.TestCase):
url = "http://example.org"
status_code = 200
total_seconds = 0.1
fast = 0.2
moderate = 0.5
status_id = logStatus(self.db, "foo")
result = logHttpStatus(
self.db, ip, url, status_code, total_seconds, status_id
self.db, ip, url, status_code, total_seconds, fast, moderate, status_id
)
assert self.db.HttpCodeChange.select().count() == 1
assert self.db.HttpCodeChange.get().ip == ip
......@@ -202,13 +204,15 @@ class SurykatkaHttpTestCase(unittest.TestCase):
url = "http://example.org"
status_code = 200
total_seconds = 0.1
fast = 0.2
moderate = 0.5
status_id = logStatus(self.db, "foo")
result = logHttpStatus(
self.db, ip, url, status_code, total_seconds, status_id
self.db, ip, url, status_code, total_seconds, fast, moderate, status_id
)
try:
logHttpStatus(
self.db, ip, url, status_code + 1, total_seconds + 1, status_id
self.db, ip, url, status_code + 1, total_seconds + 1, fast, moderate, status_id
)
except peewee.IntegrityError:
assert self.db.HttpCodeChange.select().count() == 1
......@@ -221,13 +225,15 @@ class SurykatkaHttpTestCase(unittest.TestCase):
url = "http://example.org"
status_code = 200
total_seconds = 0.1
fast = 0.2
moderate = 0.5
status_id = logStatus(self.db, "foo")
result = logHttpStatus(
self.db, ip, url, status_code, total_seconds, status_id
self.db, ip, url, status_code, total_seconds, fast, moderate, status_id
)
status_id_2 = logStatus(self.db, "foo")
result_2 = logHttpStatus(
self.db, ip, url, status_code, total_seconds, status_id_2
self.db, ip, url, status_code, total_seconds, fast, moderate, status_id_2
)
assert result_2 == result
assert self.db.HttpCodeChange.select().count() == 1
......@@ -242,14 +248,16 @@ class SurykatkaHttpTestCase(unittest.TestCase):
url = "http://example.org"
status_code = 200
total_seconds = 0.1
fast = 0.2
moderate = 0.5
status_code_2 = status_code + 1
status_id = logStatus(self.db, "foo")
result = logHttpStatus(
self.db, ip, url, status_code, total_seconds, status_id
self.db, ip, url, status_code, total_seconds, fast, moderate, status_id
)
status_id_2 = logStatus(self.db, "foo")
result_2 = logHttpStatus(
self.db, ip, url, status_code_2, total_seconds, status_id_2
self.db, ip, url, status_code_2, total_seconds, fast, moderate, status_id_2
)
assert result_2 != result
assert self.db.HttpCodeChange.select().count() == 2
......@@ -302,6 +310,63 @@ class SurykatkaHttpTestCase(unittest.TestCase):
== total_seconds
)
def test_logHttpStatus_configurableRanges(self):
ip = "127.0.0.1"
url = "http://example.org"
status_code = 200
total_seconds_error = 0
total_seconds_fast = 4
total_seconds_moderate = 5
total_seconds_slow = 30
fast = 5
moderate = 30
status_id = logStatus(self.db, "foo")
result = logHttpStatus(
self.db, ip, url, status_code, total_seconds_error, fast, moderate, status_id
)
status_id_2 = logStatus(self.db, "foo")
result_2 = logHttpStatus(
self.db, ip, url, status_code, total_seconds_fast, fast, moderate, status_id_2
)
status_id_3 = logStatus(self.db, "foo")
result_3 = logHttpStatus(
self.db, ip, url, status_code, total_seconds_moderate, fast, moderate, status_id_3
)
status_id_4 = logStatus(self.db, "foo")
result_4 = logHttpStatus(
self.db, ip, url, status_code, total_seconds_slow, fast, moderate, status_id_4
)
assert status_id == result
assert status_id_2 == result_2
assert status_id_3 == result_3
assert status_id_4 == result_4
assert self.db.HttpCodeChange.select().count() == 4
assert (
self.db.HttpCodeChange.get(
self.db.HttpCodeChange.status == status_id
).total_seconds
== total_seconds_error
)
assert (
self.db.HttpCodeChange.get(
self.db.HttpCodeChange.status == status_id_2
).total_seconds
== total_seconds_fast
)
assert (
self.db.HttpCodeChange.get(
self.db.HttpCodeChange.status == status_id_3
).total_seconds
== total_seconds_moderate
)
assert (
self.db.HttpCodeChange.get(
self.db.HttpCodeChange.status == status_id_4
).total_seconds
== total_seconds_slow
)
def test_logHttpStatus_insertWhenDifferentTotalSeconds(self):
ip = "127.0.0.1"
url = "http://example.org"
......@@ -310,21 +375,23 @@ class SurykatkaHttpTestCase(unittest.TestCase):
total_seconds_fast = 0.001
total_seconds_moderate = 0.2
total_seconds_slow = 0.5
fast = 0.2
moderate = 0.5
status_id = logStatus(self.db, "foo")
result = logHttpStatus(
self.db, ip, url, status_code, total_seconds_error, status_id
self.db, ip, url, status_code, total_seconds_error, fast, moderate, status_id
)
status_id_2 = logStatus(self.db, "foo")
result_2 = logHttpStatus(
self.db, ip, url, status_code, total_seconds_fast, status_id_2
self.db, ip, url, status_code, total_seconds_fast, fast, moderate, status_id_2
)
status_id_3 = logStatus(self.db, "foo")
result_3 = logHttpStatus(
self.db, ip, url, status_code, total_seconds_moderate, status_id_3
self.db, ip, url, status_code, total_seconds_moderate, fast, moderate, status_id_3
)
status_id_4 = logStatus(self.db, "foo")
result_4 = logHttpStatus(
self.db, ip, url, status_code, total_seconds_slow, status_id_4
self.db, ip, url, status_code, total_seconds_slow, fast, moderate, status_id_4
)
assert status_id == result
assert status_id_2 == result_2
......@@ -363,13 +430,15 @@ class SurykatkaHttpTestCase(unittest.TestCase):
status_code = 200
total_seconds = 0.001
total_seconds_2 = 0.199
fast = 0.2
moderate = 0.5
status_id = logStatus(self.db, "foo")
result = logHttpStatus(
self.db, ip, url, status_code, total_seconds, status_id
self.db, ip, url, status_code, total_seconds, fast, moderate, status_id
)
status_id_2 = logStatus(self.db, "foo")
result_2 = logHttpStatus(
self.db, ip, url, status_code, total_seconds_2, status_id_2
self.db, ip, url, status_code, total_seconds_2, fast, moderate, status_id_2
)
assert result_2 == result
assert self.db.HttpCodeChange.select().count() == 1
......@@ -385,13 +454,15 @@ class SurykatkaHttpTestCase(unittest.TestCase):
status_code = 200
total_seconds = 0.2
total_seconds_2 = 0.499
fast = 0.2
moderate = 0.5
status_id = logStatus(self.db, "foo")
result = logHttpStatus(
self.db, ip, url, status_code, total_seconds, status_id
self.db, ip, url, status_code, total_seconds, fast, moderate, status_id
)
status_id_2 = logStatus(self.db, "foo")
result_2 = logHttpStatus(
self.db, ip, url, status_code, total_seconds_2, status_id_2
self.db, ip, url, status_code, total_seconds_2, fast, moderate, status_id_2
)
assert result_2 == result
assert self.db.HttpCodeChange.select().count() == 1
......@@ -407,13 +478,15 @@ class SurykatkaHttpTestCase(unittest.TestCase):
status_code = 200
total_seconds = 0.5
total_seconds_2 = 99
fast = 0.2
moderate = 0.5
status_id = logStatus(self.db, "foo")
result = logHttpStatus(
self.db, ip, url, status_code, total_seconds, status_id
self.db, ip, url, status_code, total_seconds, fast, moderate, status_id
)
status_id_2 = logStatus(self.db, "foo")
result_2 = logHttpStatus(
self.db, ip, url, status_code, total_seconds_2, status_id_2
self.db, ip, url, status_code, total_seconds_2, fast, moderate, status_id_2
)
assert result_2 == result
assert self.db.HttpCodeChange.select().count() == 1
......@@ -430,16 +503,18 @@ class SurykatkaHttpTestCase(unittest.TestCase):
url_2 = url + "2"
total_seconds = 0.1
status_code = 200
fast = 0.2
moderate = 0.5
status_id = logStatus(self.db, "foo")
logHttpStatus(self.db, ip, url, status_code, total_seconds, status_id)
logHttpStatus(self.db, ip, url, status_code, total_seconds, fast, moderate, status_id)
logHttpStatus(
self.db, ip_2, url, status_code, total_seconds, status_id
self.db, ip_2, url, status_code, total_seconds, fast, moderate, status_id
)
logHttpStatus(
self.db, ip, url_2, status_code, total_seconds, status_id
self.db, ip, url_2, status_code, total_seconds, fast, moderate, status_id
)
logHttpStatus(
self.db, ip_2, url_2, status_code, total_seconds, status_id
self.db, ip_2, url_2, status_code, total_seconds, fast, moderate, status_id
)
assert self.db.HttpCodeChange.select().count() == 4
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment