Commit 24581bc6 authored by Romain Courteaud's avatar Romain Courteaud

WIP: store the http response headers

parent 65fd3852
......@@ -298,6 +298,7 @@ class WebBot:
result_dict["http_query"].append(
{
"status_code": network_change["status_code"],
"http_header_dict": network_change["http_header_dict"],
"total_seconds": network_change["total_seconds"],
"url": network_change["url"],
"ip": network_change["ip"],
......
......@@ -18,7 +18,7 @@
# See https://www.nexedi.com/licensing for rationale and options.
import peewee
from playhouse.sqlite_ext import SqliteExtDatabase
from playhouse.sqlite_ext import SqliteExtDatabase, JSONField
import datetime
from playhouse.migrate import migrate, SqliteMigrator
......@@ -115,6 +115,7 @@ class LogDB:
ip = peewee.TextField()
url = peewee.TextField()
status_code = peewee.IntegerField()
http_header_dict = JSONField(default=dict)
total_seconds = peewee.FloatField(default=0)
class Meta:
......@@ -131,7 +132,7 @@ class LogDB:
def createTables(self):
# http://www.sqlite.org/pragma.html#pragma_user_version
db_version = self._db.pragma("user_version")
expected_version = 3
expected_version = 4
if db_version != expected_version:
with self._db.transaction():
......@@ -152,10 +153,12 @@ class LogDB:
# version 1 without SSL support
self._db.create_tables([self.SslChange])
migrator = SqliteMigrator(self._db)
migration_list = []
if (0 < db_version) and (db_version <= 2):
# version 2 without the http total_seconds column
migrator = SqliteMigrator(self._db)
migrate(
migration_list.append(
migrator.add_column(
"HttpCodeChange",
"total_seconds",
......@@ -163,6 +166,19 @@ class LogDB:
)
)
if (0 < db_version) and (db_version <= 3):
# version 3 without the http header column
migration_list.append(
migrator.add_column(
"HttpCodeChange",
"http_header_dict",
self.HttpCodeChange.http_header_dict,
)
)
if migration_list:
migrate(*migration_list)
if db_version >= expected_version:
raise ValueError("Can not downgrade SQLite DB")
......
......@@ -120,7 +120,17 @@ def calculateSpeedRange(total_seconds, fast, moderate):
return "SLOW"
def logHttpStatus(db, ip, url, code, total_seconds, fast, moderate, status_id):
def logHttpStatus(
db,
ip,
url,
code,
http_header_dict,
total_seconds,
fast,
moderate,
status_id,
):
with db._db.atomic():
try:
......@@ -132,6 +142,7 @@ def logHttpStatus(db, ip, url, code, total_seconds, fast, moderate, status_id):
if (
(previous_entry is None)
or (previous_entry.status_code != code)
or (previous_entry.http_header_dict != http_header_dict)
or (
calculateSpeedRange(
previous_entry.total_seconds, fast, moderate
......@@ -144,6 +155,7 @@ def logHttpStatus(db, ip, url, code, total_seconds, fast, moderate, status_id):
ip=ip,
url=url,
status_code=code,
http_header_dict=http_header_dict,
total_seconds=total_seconds,
)
return previous_entry.status_id
......@@ -182,11 +194,51 @@ def checkHttpStatus(
response = request(
ip_url, headers={"Host": hostname}, version=bot_version, **request_kw
)
# Blacklisted, because of non stability
# 'Date', 'Age', 'Expires'
header_list = [
# Redirect
"Location",
# HTTP Range
"Accept-Ranges",
# HTTP Cache
"Etag",
"Last-Modified",
"Vary",
"Cache-Control",
"Set-Cookie",
"WWW-Authenticate"
# gzip
"Content-Type",
"Content-Encoding",
"Content-Disposition"
# Security
"Content-Security-Policy",
"Referrer-Policy",
"Strict-Transport-Policy",
"Feature-Policy",
"X-Frame-Options",
"X-Content-Type-Options"
# CORS
"Access-Control-Allow-Origin",
"Access-Control-Allow-Methods",
"Access-Control-Allow-Credentials",
"Access-Control-Allow-Headers",
"Access-Control-Expose-Headers",
]
header_dict = {}
for header_key in header_list:
header_value = response.headers.get(header_key, None)
if header_value is not None:
header_dict[header_key] = header_value
logHttpStatus(
db,
ip,
url,
response.status_code,
header_dict,
response.elapsed.total_seconds(),
elapsed_fast,
elapsed_moderate,
......
......@@ -115,6 +115,7 @@ class SurykatkaBotTestCase(unittest.TestCase):
"surykatka.http.request"
) as mock_request:
mock_request.return_value.headers = {"Etag": "foobar"}
mock_get_default_resolver.return_value = resolver
mock_query.return_value = [MockAnswer("1.2.3.4")]
mock_create_default_context.return_value.wrap_socket.return_value.getpeercert.side_effect = [
......@@ -178,6 +179,7 @@ class SurykatkaBotTestCase(unittest.TestCase):
"surykatka.http.request"
) as mock_request:
mock_request.return_value.headers = {"Etag": "foobar"}
mock_query.return_value = [MockAnswer("1.2.3.4")]
mock_create_default_context.return_value.wrap_socket.return_value.getpeercert.side_effect = [
b"",
......@@ -244,6 +246,7 @@ class SurykatkaBotTestCase(unittest.TestCase):
"surykatka.http.request"
) as mock_request:
mock_request.return_value.headers = {"Etag": "foobar"}
mock_query.return_value = [MockAnswer("1.2.3.4")]
mock_create_default_context.return_value.wrap_socket.return_value.getpeercert.side_effect = [
b"",
......@@ -312,6 +315,7 @@ class SurykatkaBotTestCase(unittest.TestCase):
"surykatka.http.request"
) as mock_request:
mock_request.return_value.headers = {"Etag": "foobar"}
mock_query.return_value = [
MockAnswer("1.2.3.4"),
MockAnswer("1.2.3.5"),
......@@ -392,6 +396,7 @@ class SurykatkaBotTestCase(unittest.TestCase):
"surykatka.http.request"
) as mock_request:
mock_request.return_value.headers = {"Etag": "foobar"}
mock_query.return_value = [MockAnswer("1.2.3.4")]
mock_create_default_context.return_value.wrap_socket.return_value.getpeercert.side_effect = [
b"",
......@@ -460,6 +465,7 @@ class SurykatkaBotTestCase(unittest.TestCase):
"surykatka.http.request"
) as mock_request:
mock_request.return_value.headers = {"Etag": "foobar"}
mock_query.return_value = [MockAnswer("1.2.3.4")]
mock_create_default_context.return_value.wrap_socket.return_value.getpeercert.side_effect = [
b"",
......@@ -515,6 +521,7 @@ class SurykatkaBotTestCase(unittest.TestCase):
"surykatka.http.request"
) as mock_request:
mock_request.return_value.headers = {"Etag": "foobar"}
mock_query.side_effect = [[MockAnswer("1.2.3.4")], []]
bot.iterateLoop()
......
......@@ -48,8 +48,10 @@ def validate_schema(model):
for column in intersect:
field = model._meta.columns[column]
db_field = db_model._meta.columns[column]
if (field.field_type != db_field.field_type) and (
not (field.field_type == "BIGINT")
if (
(field.field_type != db_field.field_type)
and (not (field.field_type == "BIGINT"))
and (not field.field_type == "JSON")
):
to_change.append((field, db_field))
......@@ -64,7 +66,7 @@ class SurykatkaDBTestCase(unittest.TestCase):
def test_createTable(self):
assert self.db._db.pragma("user_version") == 0
self.db.createTables()
assert self.db._db.pragma("user_version") == 3
assert self.db._db.pragma("user_version") == 4
def test_downgrade(self):
assert self.db._db.pragma("user_version") == 0
......@@ -97,10 +99,15 @@ class SurykatkaDBTestCase(unittest.TestCase):
"HttpCodeChange", "total_seconds"
),
)
migrate(
SqliteMigrator(self.db._db).drop_column(
"HttpCodeChange", "http_header_dict"
),
)
self.db._db.pragma("user_version", 1)
self.db.createTables()
assert self.db._db.pragma("user_version") == 3
assert self.db._db.pragma("user_version") == 4
assert validate_schema(self.db.SslChange).valid, validate_schema(
self.db.SslChange
)
......@@ -126,10 +133,44 @@ class SurykatkaDBTestCase(unittest.TestCase):
"HttpCodeChange", "total_seconds"
),
)
migrate(
SqliteMigrator(self.db._db).drop_column(
"HttpCodeChange", "http_header_dict"
),
)
self.db._db.pragma("user_version", 2)
self.db.createTables()
assert self.db._db.pragma("user_version") == 3
assert self.db._db.pragma("user_version") == 4
assert validate_schema(self.db.HttpCodeChange).valid, validate_schema(
self.db.HttpCodeChange
)
def test_migrationFromVersion3(self):
assert self.db._db.pragma("user_version") == 0
# Recreate version 3
with self.db._db.transaction():
self.db._db.create_tables(
[
self.db.Status,
self.db.ConfigurationChange,
self.db.HttpCodeChange,
self.db.NetworkChange,
self.db.PlatformChange,
self.db.DnsChange,
self.db.SslChange,
]
)
migrate(
SqliteMigrator(self.db._db).drop_column(
"HttpCodeChange", "http_header_dict"
),
)
self.db._db.pragma("user_version", 3)
self.db.createTables()
assert self.db._db.pragma("user_version") == 4
assert validate_schema(self.db.HttpCodeChange).valid, validate_schema(
self.db.HttpCodeChange
)
......
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment