WIP: store the http response headers

src/surykatka/bot.py

@@ -298,6 +298,7 @@ class WebBot:
                 result_dict["http_query"].append(
                     {
                         "status_code": network_change["status_code"],
+                        "http_header_dict": network_change["http_header_dict"],
                         "total_seconds": network_change["total_seconds"],
                         "url": network_change["url"],
                         "ip": network_change["ip"],

src/surykatka/db.py

@@ -18,7 +18,7 @@
 # See https://www.nexedi.com/licensing for rationale and options.
 
 import peewee
-from playhouse.sqlite_ext import SqliteExtDatabase
+from playhouse.sqlite_ext import SqliteExtDatabase, JSONField
 import datetime
 from playhouse.migrate import migrate, SqliteMigrator
 
@@ -115,6 +115,7 @@ class LogDB:
             ip = peewee.TextField()
             url = peewee.TextField()
             status_code = peewee.IntegerField()
+            http_header_dict = JSONField(default=dict)
             total_seconds = peewee.FloatField(default=0)
 
             class Meta:
@@ -131,7 +132,7 @@ class LogDB:
     def createTables(self):
         # http://www.sqlite.org/pragma.html#pragma_user_version
         db_version = self._db.pragma("user_version")
-        expected_version = 3
+        expected_version = 4
         if db_version != expected_version:
             with self._db.transaction():
 
@@ -152,10 +153,12 @@ class LogDB:
                     # version 1 without SSL support
                     self._db.create_tables([self.SslChange])
 
+                migrator = SqliteMigrator(self._db)
+                migration_list = []
+
                 if (0 < db_version) and (db_version <= 2):
                     # version 2 without the http total_seconds column
-                    migrator = SqliteMigrator(self._db)
-                    migrate(
+                    migration_list.append(
                         migrator.add_column(
                             "HttpCodeChange",
                             "total_seconds",
@@ -163,6 +166,19 @@ class LogDB:
                         )
                     )
 
+                if (0 < db_version) and (db_version <= 3):
+                    # version 3 without the http header column
+                    migration_list.append(
+                        migrator.add_column(
+                            "HttpCodeChange",
+                            "http_header_dict",
+                            self.HttpCodeChange.http_header_dict,
+                        )
+                    )
+
+                if migration_list:
+                    migrate(*migration_list)
+
                 if db_version >= expected_version:
                     raise ValueError("Can not downgrade SQLite DB")
 

src/surykatka/http.py

@@ -120,7 +120,17 @@ def calculateSpeedRange(total_seconds, fast, moderate):
         return "SLOW"
 
 
-def logHttpStatus(db, ip, url, code, total_seconds, fast, moderate, status_id):
+def logHttpStatus(
+    db,
+    ip,
+    url,
+    code,
+    http_header_dict,
+    total_seconds,
+    fast,
+    moderate,
+    status_id,
+):
 
     with db._db.atomic():
         try:
@@ -132,6 +142,7 @@ def logHttpStatus(db, ip, url, code, total_seconds, fast, moderate, status_id):
         if (
             (previous_entry is None)
             or (previous_entry.status_code != code)
+            or (previous_entry.http_header_dict != http_header_dict)
             or (
                 calculateSpeedRange(
                     previous_entry.total_seconds, fast, moderate
@@ -144,6 +155,7 @@ def logHttpStatus(db, ip, url, code, total_seconds, fast, moderate, status_id):
                 ip=ip,
                 url=url,
                 status_code=code,
+                http_header_dict=http_header_dict,
                 total_seconds=total_seconds,
             )
         return previous_entry.status_id
@@ -182,11 +194,51 @@ def checkHttpStatus(
     response = request(
         ip_url, headers={"Host": hostname}, version=bot_version, **request_kw
     )
+
+    # Blacklisted, because of non stability
+    # 'Date', 'Age', 'Expires'
+    header_list = [
+        # Redirect
+        "Location",
+        # HTTP Range
+        "Accept-Ranges",
+        # HTTP Cache
+        "Etag",
+        "Last-Modified",
+        "Vary",
+        "Cache-Control",
+        "Set-Cookie",
+        "WWW-Authenticate"
+        # gzip
+        "Content-Type",
+        "Content-Encoding",
+        "Content-Disposition"
+        # Security
+        "Content-Security-Policy",
+        "Referrer-Policy",
+        "Strict-Transport-Policy",
+        "Feature-Policy",
+        "X-Frame-Options",
+        "X-Content-Type-Options"
+        # CORS
+        "Access-Control-Allow-Origin",
+        "Access-Control-Allow-Methods",
+        "Access-Control-Allow-Credentials",
+        "Access-Control-Allow-Headers",
+        "Access-Control-Expose-Headers",
+    ]
+    header_dict = {}
+    for header_key in header_list:
+        header_value = response.headers.get(header_key, None)
+        if header_value is not None:
+            header_dict[header_key] = header_value
+
     logHttpStatus(
         db,
         ip,
         url,
         response.status_code,
+        header_dict,
         response.elapsed.total_seconds(),
         elapsed_fast,
         elapsed_moderate,

tests/test_bot.py

@@ -115,6 +115,7 @@ class SurykatkaBotTestCase(unittest.TestCase):
             "surykatka.http.request"
         ) as mock_request:
 
+            mock_request.return_value.headers = {"Etag": "foobar"}
             mock_get_default_resolver.return_value = resolver
             mock_query.return_value = [MockAnswer("1.2.3.4")]
             mock_create_default_context.return_value.wrap_socket.return_value.getpeercert.side_effect = [
@@ -178,6 +179,7 @@ class SurykatkaBotTestCase(unittest.TestCase):
             "surykatka.http.request"
         ) as mock_request:
 
+            mock_request.return_value.headers = {"Etag": "foobar"}
             mock_query.return_value = [MockAnswer("1.2.3.4")]
             mock_create_default_context.return_value.wrap_socket.return_value.getpeercert.side_effect = [
                 b"",
@@ -244,6 +246,7 @@ class SurykatkaBotTestCase(unittest.TestCase):
             "surykatka.http.request"
         ) as mock_request:
 
+            mock_request.return_value.headers = {"Etag": "foobar"}
             mock_query.return_value = [MockAnswer("1.2.3.4")]
             mock_create_default_context.return_value.wrap_socket.return_value.getpeercert.side_effect = [
                 b"",
@@ -312,6 +315,7 @@ class SurykatkaBotTestCase(unittest.TestCase):
             "surykatka.http.request"
         ) as mock_request:
 
+            mock_request.return_value.headers = {"Etag": "foobar"}
             mock_query.return_value = [
                 MockAnswer("1.2.3.4"),
                 MockAnswer("1.2.3.5"),
@@ -392,6 +396,7 @@ class SurykatkaBotTestCase(unittest.TestCase):
             "surykatka.http.request"
         ) as mock_request:
 
+            mock_request.return_value.headers = {"Etag": "foobar"}
             mock_query.return_value = [MockAnswer("1.2.3.4")]
             mock_create_default_context.return_value.wrap_socket.return_value.getpeercert.side_effect = [
                 b"",
@@ -460,6 +465,7 @@ class SurykatkaBotTestCase(unittest.TestCase):
             "surykatka.http.request"
         ) as mock_request:
 
+            mock_request.return_value.headers = {"Etag": "foobar"}
             mock_query.return_value = [MockAnswer("1.2.3.4")]
             mock_create_default_context.return_value.wrap_socket.return_value.getpeercert.side_effect = [
                 b"",
@@ -515,6 +521,7 @@ class SurykatkaBotTestCase(unittest.TestCase):
             "surykatka.http.request"
         ) as mock_request:
 
+            mock_request.return_value.headers = {"Etag": "foobar"}
             mock_query.side_effect = [[MockAnswer("1.2.3.4")], []]
 
             bot.iterateLoop()

tests/test_db.py

@@ -48,8 +48,10 @@ def validate_schema(model):
     for column in intersect:
         field = model._meta.columns[column]
         db_field = db_model._meta.columns[column]
-        if (field.field_type != db_field.field_type) and (
-            not (field.field_type == "BIGINT")
+        if (
+            (field.field_type != db_field.field_type)
+            and (not (field.field_type == "BIGINT"))
+            and (not field.field_type == "JSON")
         ):
             to_change.append((field, db_field))
 
@@ -64,7 +66,7 @@ class SurykatkaDBTestCase(unittest.TestCase):
     def test_createTable(self):
         assert self.db._db.pragma("user_version") == 0
         self.db.createTables()
-        assert self.db._db.pragma("user_version") == 3
+        assert self.db._db.pragma("user_version") == 4
 
     def test_downgrade(self):
         assert self.db._db.pragma("user_version") == 0
@@ -97,10 +99,15 @@ class SurykatkaDBTestCase(unittest.TestCase):
                     "HttpCodeChange", "total_seconds"
                 ),
             )
+            migrate(
+                SqliteMigrator(self.db._db).drop_column(
+                    "HttpCodeChange", "http_header_dict"
+                ),
+            )
             self.db._db.pragma("user_version", 1)
 
         self.db.createTables()
-        assert self.db._db.pragma("user_version") == 3
+        assert self.db._db.pragma("user_version") == 4
         assert validate_schema(self.db.SslChange).valid, validate_schema(
             self.db.SslChange
         )
@@ -126,10 +133,44 @@ class SurykatkaDBTestCase(unittest.TestCase):
                     "HttpCodeChange", "total_seconds"
                 ),
             )
+            migrate(
+                SqliteMigrator(self.db._db).drop_column(
+                    "HttpCodeChange", "http_header_dict"
+                ),
+            )
             self.db._db.pragma("user_version", 2)
 
         self.db.createTables()
-        assert self.db._db.pragma("user_version") == 3
+        assert self.db._db.pragma("user_version") == 4
+        assert validate_schema(self.db.HttpCodeChange).valid, validate_schema(
+            self.db.HttpCodeChange
+        )
+
+    def test_migrationFromVersion3(self):
+        assert self.db._db.pragma("user_version") == 0
+        # Recreate version 3
+        with self.db._db.transaction():
+
+            self.db._db.create_tables(
+                [
+                    self.db.Status,
+                    self.db.ConfigurationChange,
+                    self.db.HttpCodeChange,
+                    self.db.NetworkChange,
+                    self.db.PlatformChange,
+                    self.db.DnsChange,
+                    self.db.SslChange,
+                ]
+            )
+            migrate(
+                SqliteMigrator(self.db._db).drop_column(
+                    "HttpCodeChange", "http_header_dict"
+                ),
+            )
+            self.db._db.pragma("user_version", 3)
+
+        self.db.createTables()
+        assert self.db._db.pragma("user_version") == 4
         assert validate_schema(self.db.HttpCodeChange).valid, validate_schema(
             self.db.HttpCodeChange
         )