WIP: store the http response headers

src/surykatka/bot.py

@@ -298,6 +298,7 @@ class WebBot:
                 result_dict["http_query"].append(
                     {
                         "status_code": network_change["status_code"],
+                        "http_header_dict": network_change["http_header_dict"],
                         "total_seconds": network_change["total_seconds"],
                         "url": network_change["url"],
                         "ip": network_change["ip"],

src/surykatka/db.py

@@ -18,7 +18,7 @@
 # See https://www.nexedi.com/licensing for rationale and options.
 
 import peewee
-from playhouse.sqlite_ext import SqliteExtDatabase
+from playhouse.sqlite_ext import SqliteExtDatabase, JSONField
 import datetime
 from playhouse.migrate import migrate, SqliteMigrator
 
@@ -115,6 +115,7 @@ class LogDB:
             ip = peewee.TextField()
             url = peewee.TextField()
             status_code = peewee.IntegerField()
+            http_header_dict = JSONField(default=dict)
             total_seconds = peewee.FloatField(default=0)
 
             class Meta:
@@ -131,7 +132,7 @@ class LogDB:
     def createTables(self):
         # http://www.sqlite.org/pragma.html#pragma_user_version
         db_version = self._db.pragma("user_version")
-        expected_version = 3
+        expected_version = 4
         if db_version != expected_version:
             with self._db.transaction():
 
@@ -152,10 +153,12 @@ class LogDB:
                     # version 1 without SSL support
                     self._db.create_tables([self.SslChange])
 
+                migrator = SqliteMigrator(self._db)
+                migration_list = []
+
                 if (0 < db_version) and (db_version <= 2):
                     # version 2 without the http total_seconds column
-                    migrator = SqliteMigrator(self._db)
-                    migrate(
+                    migration_list.append(
                         migrator.add_column(
                             "HttpCodeChange",
                             "total_seconds",
@@ -163,6 +166,19 @@ class LogDB:
                         )
                     )
 
+                if (0 < db_version) and (db_version <= 3):
+                    # version 3 without the http header column
+                    migration_list.append(
+                        migrator.add_column(
+                            "HttpCodeChange",
+                            "http_header_dict",
+                            self.HttpCodeChange.http_header_dict,
+                        )
+                    )
+
+                if migration_list:
+                    migrate(*migration_list)
+
                 if db_version >= expected_version:
                     raise ValueError("Can not downgrade SQLite DB")
 

src/surykatka/http.py

@@ -120,7 +120,17 @@ def calculateSpeedRange(total_seconds, fast, moderate):
         return "SLOW"
 
 
-def logHttpStatus(db, ip, url, code, total_seconds, fast, moderate, status_id):
+def logHttpStatus(
+    db,
+    ip,
+    url,
+    code,
+    http_header_dict,
+    total_seconds,
+    fast,
+    moderate,
+    status_id,
+):
 
     with db._db.atomic():
         try:
@@ -132,6 +142,7 @@ def logHttpStatus(db, ip, url, code, total_seconds, fast, moderate, status_id):
         if (
             (previous_entry is None)
             or (previous_entry.status_code != code)
+            or (previous_entry.http_header_dict != http_header_dict)
             or (
                 calculateSpeedRange(
                     previous_entry.total_seconds, fast, moderate
@@ -144,6 +155,7 @@ def logHttpStatus(db, ip, url, code, total_seconds, fast, moderate, status_id):
                 ip=ip,
                 url=url,
                 status_code=code,
+                http_header_dict=http_header_dict,
                 total_seconds=total_seconds,
             )
         return previous_entry.status_id
@@ -182,11 +194,51 @@ def checkHttpStatus(
     response = request(
         ip_url, headers={"Host": hostname}, version=bot_version, **request_kw
     )
+
+    # Blacklisted, because of non stability
+    # 'Date', 'Age', 'Expires'
+    header_list = [
+        # Redirect
+        "Location",
+        # HTTP Range
+        "Accept-Ranges",
+        # HTTP Cache
+        "Etag",
+        "Last-Modified",
+        "Vary",
+        "Cache-Control",
+        "Set-Cookie",
+        "WWW-Authenticate"
+        # gzip
+        "Content-Type",
+        "Content-Encoding",
+        "Content-Disposition"
+        # Security
+        "Content-Security-Policy",
+        "Referrer-Policy",
+        "Strict-Transport-Policy",
+        "Feature-Policy",
+        "X-Frame-Options",
+        "X-Content-Type-Options"
+        # CORS
+        "Access-Control-Allow-Origin",
+        "Access-Control-Allow-Methods",
+        "Access-Control-Allow-Credentials",
+        "Access-Control-Allow-Headers",
+        "Access-Control-Expose-Headers",
+    ]
+    header_dict = {}
+    for header_key in header_list:
+        header_value = response.headers.get(header_key, None)
+        if header_value is not None:
+            header_dict[header_key] = header_value
+
     logHttpStatus(
         db,
         ip,
         url,
         response.status_code,
+        header_dict,
         response.elapsed.total_seconds(),
         elapsed_fast,
         elapsed_moderate,

tests/test_bot.py

@@ -115,6 +115,7 @@ class SurykatkaBotTestCase(unittest.TestCase):
             "surykatka.http.request"
         ) as mock_request:
 
+            mock_request.return_value.headers = {"Etag": "foobar"}
             mock_get_default_resolver.return_value = resolver
             mock_query.return_value = [MockAnswer("1.2.3.4")]
             mock_create_default_context.return_value.wrap_socket.return_value.getpeercert.side_effect = [
@@ -178,6 +179,7 @@ class SurykatkaBotTestCase(unittest.TestCase):
             "surykatka.http.request"
         ) as mock_request:
 
+            mock_request.return_value.headers = {"Etag": "foobar"}
             mock_query.return_value = [MockAnswer("1.2.3.4")]
             mock_create_default_context.return_value.wrap_socket.return_value.getpeercert.side_effect = [
                 b"",
@@ -244,6 +246,7 @@ class SurykatkaBotTestCase(unittest.TestCase):
             "surykatka.http.request"
         ) as mock_request:
 
+            mock_request.return_value.headers = {"Etag": "foobar"}
             mock_query.return_value = [MockAnswer("1.2.3.4")]
             mock_create_default_context.return_value.wrap_socket.return_value.getpeercert.side_effect = [
                 b"",
@@ -312,6 +315,7 @@ class SurykatkaBotTestCase(unittest.TestCase):
             "surykatka.http.request"
         ) as mock_request:
 
+            mock_request.return_value.headers = {"Etag": "foobar"}
             mock_query.return_value = [
                 MockAnswer("1.2.3.4"),
                 MockAnswer("1.2.3.5"),
@@ -392,6 +396,7 @@ class SurykatkaBotTestCase(unittest.TestCase):
             "surykatka.http.request"
         ) as mock_request:
 
+            mock_request.return_value.headers = {"Etag": "foobar"}
             mock_query.return_value = [MockAnswer("1.2.3.4")]
             mock_create_default_context.return_value.wrap_socket.return_value.getpeercert.side_effect = [
                 b"",
@@ -460,6 +465,7 @@ class SurykatkaBotTestCase(unittest.TestCase):
             "surykatka.http.request"
         ) as mock_request:
 
+            mock_request.return_value.headers = {"Etag": "foobar"}
             mock_query.return_value = [MockAnswer("1.2.3.4")]
             mock_create_default_context.return_value.wrap_socket.return_value.getpeercert.side_effect = [
                 b"",
@@ -515,6 +521,7 @@ class SurykatkaBotTestCase(unittest.TestCase):
             "surykatka.http.request"
         ) as mock_request:
 
+            mock_request.return_value.headers = {"Etag": "foobar"}
             mock_query.side_effect = [[MockAnswer("1.2.3.4")], []]
 
             bot.iterateLoop()

tests/test_db.py

@@ -48,8 +48,10 @@ def validate_schema(model):
     for column in intersect:
         field = model._meta.columns[column]
         db_field = db_model._meta.columns[column]
-        if (field.field_type != db_field.field_type) and (
-            not (field.field_type == "BIGINT")
+        if (
+            (field.field_type != db_field.field_type)
+            and (not (field.field_type == "BIGINT"))
+            and (not field.field_type == "JSON")
         ):
             to_change.append((field, db_field))
 
@@ -64,7 +66,7 @@ class SurykatkaDBTestCase(unittest.TestCase):
     def test_createTable(self):
         assert self.db._db.pragma("user_version") == 0
         self.db.createTables()
-        assert self.db._db.pragma("user_version") == 3
+        assert self.db._db.pragma("user_version") == 4
 
     def test_downgrade(self):
         assert self.db._db.pragma("user_version") == 0
@@ -97,10 +99,15 @@ class SurykatkaDBTestCase(unittest.TestCase):
                     "HttpCodeChange", "total_seconds"
                 ),
             )
+            migrate(
+                SqliteMigrator(self.db._db).drop_column(
+                    "HttpCodeChange", "http_header_dict"
+                ),
+            )
             self.db._db.pragma("user_version", 1)
 
         self.db.createTables()
-        assert self.db._db.pragma("user_version") == 3
+        assert self.db._db.pragma("user_version") == 4
         assert validate_schema(self.db.SslChange).valid, validate_schema(
             self.db.SslChange
         )
@@ -126,10 +133,44 @@ class SurykatkaDBTestCase(unittest.TestCase):
                     "HttpCodeChange", "total_seconds"
                 ),
             )
+            migrate(
+                SqliteMigrator(self.db._db).drop_column(
+                    "HttpCodeChange", "http_header_dict"
+                ),
+            )
             self.db._db.pragma("user_version", 2)
 
         self.db.createTables()
-        assert self.db._db.pragma("user_version") == 3
+        assert self.db._db.pragma("user_version") == 4
+        assert validate_schema(self.db.HttpCodeChange).valid, validate_schema(
+            self.db.HttpCodeChange
+        )
+
+    def test_migrationFromVersion3(self):
+        assert self.db._db.pragma("user_version") == 0
+        # Recreate version 3
+        with self.db._db.transaction():
+
+            self.db._db.create_tables(
+                [
+                    self.db.Status,
+                    self.db.ConfigurationChange,
+                    self.db.HttpCodeChange,
+                    self.db.NetworkChange,
+                    self.db.PlatformChange,
+                    self.db.DnsChange,
+                    self.db.SslChange,
+                ]
+            )
+            migrate(
+                SqliteMigrator(self.db._db).drop_column(
+                    "HttpCodeChange", "http_header_dict"
+                ),
+            )
+            self.db._db.pragma("user_version", 3)
+
+        self.db.createTables()
+        assert self.db._db.pragma("user_version") == 4
         assert validate_schema(self.db.HttpCodeChange).valid, validate_schema(
             self.db.HttpCodeChange
         )

tests/test_http.py

@@ -184,6 +184,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
         ip = "127.0.0.1"
         url = "http://example.org"
         status_code = 200
+        http_header_dict = {"a": "b"}
         total_seconds = 0.1
         fast = 0.2
         moderate = 0.5
@@ -193,6 +194,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
             ip,
             url,
             status_code,
+            http_header_dict,
             total_seconds,
             fast,
             moderate,
@@ -202,6 +204,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
         assert self.db.HttpCodeChange.get().ip == ip
         assert self.db.HttpCodeChange.get().url == url
         assert self.db.HttpCodeChange.get().status_code == status_code
+        assert self.db.HttpCodeChange.get().http_header_dict == {"a": "b"}
         assert self.db.HttpCodeChange.get().status_id == status_id
         assert self.db.HttpCodeChange.get().total_seconds == total_seconds
         assert result == status_id
@@ -210,6 +213,8 @@ class SurykatkaHttpTestCase(unittest.TestCase):
         ip = "127.0.0.1"
         url = "http://example.org"
         status_code = 200
+        http_header_dict = {"a": "b"}
+        http_header_dict_2 = {"a2": "b2"}
         total_seconds = 0.1
         fast = 0.2
         moderate = 0.5
@@ -219,6 +224,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
             ip,
             url,
             status_code,
+            http_header_dict,
             total_seconds,
             fast,
             moderate,
@@ -230,6 +236,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
                 ip,
                 url,
                 status_code + 1,
+                http_header_dict_2,
                 total_seconds + 1,
                 fast,
                 moderate,
@@ -245,6 +252,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
         ip = "127.0.0.1"
         url = "http://example.org"
         status_code = 200
+        http_header_dict = {"a": "b"}
         total_seconds = 0.1
         fast = 0.2
         moderate = 0.5
@@ -254,6 +262,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
             ip,
             url,
             status_code,
+            http_header_dict,
             total_seconds,
             fast,
             moderate,
@@ -265,6 +274,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
             ip,
             url,
             status_code,
+            http_header_dict,
             total_seconds,
             fast,
             moderate,
@@ -275,6 +285,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
         assert self.db.HttpCodeChange.get().ip == ip
         assert self.db.HttpCodeChange.get().url == url
         assert self.db.HttpCodeChange.get().status_code == status_code
+        assert self.db.HttpCodeChange.get().http_header_dict == {"a": "b"}
         assert self.db.HttpCodeChange.get().total_seconds == total_seconds
         assert self.db.HttpCodeChange.get().status_id == status_id
 
@@ -282,6 +293,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
         ip = "127.0.0.1"
         url = "http://example.org"
         status_code = 200
+        http_header_dict = {"a": "b"}
         total_seconds = 0.1
         fast = 0.2
         moderate = 0.5
@@ -292,6 +304,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
             ip,
             url,
             status_code,
+            http_header_dict,
             total_seconds,
             fast,
             moderate,
@@ -303,6 +316,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
             ip,
             url,
             status_code_2,
+            http_header_dict,
             total_seconds,
             fast,
             moderate,
@@ -328,6 +342,12 @@ class SurykatkaHttpTestCase(unittest.TestCase):
             ).status_code
             == status_code
         )
+        assert (
+            self.db.HttpCodeChange.get(
+                self.db.HttpCodeChange.status == status_id
+            ).http_header_dict
+            == http_header_dict
+        )
         assert (
             self.db.HttpCodeChange.get(
                 self.db.HttpCodeChange.status == status_id
@@ -352,6 +372,108 @@ class SurykatkaHttpTestCase(unittest.TestCase):
             ).status_code
             == status_code_2
         )
+        assert (
+            self.db.HttpCodeChange.get(
+                self.db.HttpCodeChange.status == status_id_2
+            ).http_header_dict
+            == http_header_dict
+        )
+        assert (
+            self.db.HttpCodeChange.get(
+                self.db.HttpCodeChange.status == status_id_2
+            ).total_seconds
+            == total_seconds
+        )
+
+    def test_logHttpStatus_insertWhenDifferentHttpHeader(self):
+        ip = "127.0.0.1"
+        url = "http://example.org"
+        status_code = 200
+        http_header_dict = {"a": "b"}
+        total_seconds = 0.1
+        fast = 0.2
+        moderate = 0.5
+        http_header_dict_2 = {"a2": "b2"}
+        status_id = logStatus(self.db, "foo")
+        result = logHttpStatus(
+            self.db,
+            ip,
+            url,
+            status_code,
+            http_header_dict,
+            total_seconds,
+            fast,
+            moderate,
+            status_id,
+        )
+        status_id_2 = logStatus(self.db, "foo")
+        result_2 = logHttpStatus(
+            self.db,
+            ip,
+            url,
+            status_code,
+            http_header_dict_2,
+            total_seconds,
+            fast,
+            moderate,
+            status_id_2,
+        )
+        assert result_2 != result
+        assert self.db.HttpCodeChange.select().count() == 2
+        assert (
+            self.db.HttpCodeChange.get(
+                self.db.HttpCodeChange.status == status_id
+            ).ip
+            == ip
+        )
+        assert (
+            self.db.HttpCodeChange.get(
+                self.db.HttpCodeChange.status == status_id
+            ).url
+            == url
+        )
+        assert (
+            self.db.HttpCodeChange.get(
+                self.db.HttpCodeChange.status == status_id
+            ).status_code
+            == status_code
+        )
+        assert (
+            self.db.HttpCodeChange.get(
+                self.db.HttpCodeChange.status == status_id
+            ).http_header_dict
+            == http_header_dict
+        )
+        assert (
+            self.db.HttpCodeChange.get(
+                self.db.HttpCodeChange.status == status_id
+            ).total_seconds
+            == total_seconds
+        )
+        assert (
+            self.db.HttpCodeChange.get(
+                self.db.HttpCodeChange.status == status_id_2
+            ).ip
+            == ip
+        )
+        assert (
+            self.db.HttpCodeChange.get(
+                self.db.HttpCodeChange.status == status_id_2
+            ).url
+            == url
+        )
+        assert (
+            self.db.HttpCodeChange.get(
+                self.db.HttpCodeChange.status == status_id_2
+            ).status_code
+            == status_code
+        )
+        assert (
+            self.db.HttpCodeChange.get(
+                self.db.HttpCodeChange.status == status_id_2
+            ).http_header_dict
+            == http_header_dict_2
+        )
         assert (
             self.db.HttpCodeChange.get(
                 self.db.HttpCodeChange.status == status_id_2
@@ -363,6 +485,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
         ip = "127.0.0.1"
         url = "http://example.org"
         status_code = 200
+        http_header_dict = {"a": "b"}
         total_seconds_error = 0
         total_seconds_fast = 4
         total_seconds_moderate = 5
@@ -375,6 +498,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
             ip,
             url,
             status_code,
+            http_header_dict,
             total_seconds_error,
             fast,
             moderate,
@@ -386,6 +510,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
             ip,
             url,
             status_code,
+            http_header_dict,
             total_seconds_fast,
             fast,
             moderate,
@@ -397,6 +522,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
             ip,
             url,
             status_code,
+            http_header_dict,
             total_seconds_moderate,
             fast,
             moderate,
@@ -408,6 +534,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
             ip,
             url,
             status_code,
+            http_header_dict,
             total_seconds_slow,
             fast,
             moderate,
@@ -448,6 +575,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
         ip = "127.0.0.1"
         url = "http://example.org"
         status_code = 200
+        http_header_dict = {"a": "b"}
         total_seconds_error = 0
         total_seconds_fast = 0.001
         total_seconds_moderate = 0.2
@@ -460,6 +588,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
             ip,
             url,
             status_code,
+            http_header_dict,
             total_seconds_error,
             fast,
             moderate,
@@ -471,6 +600,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
             ip,
             url,
             status_code,
+            http_header_dict,
             total_seconds_fast,
             fast,
             moderate,
@@ -482,6 +612,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
             ip,
             url,
             status_code,
+            http_header_dict,
             total_seconds_moderate,
             fast,
             moderate,
@@ -493,6 +624,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
             ip,
             url,
             status_code,
+            http_header_dict,
             total_seconds_slow,
             fast,
             moderate,
@@ -533,6 +665,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
         ip = "127.0.0.1"
         url = "http://example.org"
         status_code = 200
+        http_header_dict = {"a": "b"}
         total_seconds = 0.001
         total_seconds_2 = 0.199
         fast = 0.2
@@ -543,6 +676,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
             ip,
             url,
             status_code,
+            http_header_dict,
             total_seconds,
             fast,
             moderate,
@@ -554,6 +688,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
             ip,
             url,
             status_code,
+            http_header_dict,
             total_seconds_2,
             fast,
             moderate,
@@ -564,6 +699,9 @@ class SurykatkaHttpTestCase(unittest.TestCase):
         assert self.db.HttpCodeChange.get().ip == ip
         assert self.db.HttpCodeChange.get().url == url
         assert self.db.HttpCodeChange.get().status_code == status_code
+        assert (
+            self.db.HttpCodeChange.get().http_header_dict == http_header_dict
+        )
         assert self.db.HttpCodeChange.get().total_seconds == total_seconds
         assert self.db.HttpCodeChange.get().status_id == status_id
 
@@ -571,6 +709,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
         ip = "127.0.0.1"
         url = "http://example.org"
         status_code = 200
+        http_header_dict = {"a": "b"}
         total_seconds = 0.2
         total_seconds_2 = 0.499
         fast = 0.2
@@ -581,6 +720,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
             ip,
             url,
             status_code,
+            http_header_dict,
             total_seconds,
             fast,
             moderate,
@@ -592,6 +732,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
             ip,
             url,
             status_code,
+            http_header_dict,
             total_seconds_2,
             fast,
             moderate,
@@ -602,6 +743,9 @@ class SurykatkaHttpTestCase(unittest.TestCase):
         assert self.db.HttpCodeChange.get().ip == ip
         assert self.db.HttpCodeChange.get().url == url
         assert self.db.HttpCodeChange.get().status_code == status_code
+        assert (
+            self.db.HttpCodeChange.get().http_header_dict == http_header_dict
+        )
         assert self.db.HttpCodeChange.get().total_seconds == total_seconds
         assert self.db.HttpCodeChange.get().status_id == status_id
 
@@ -609,6 +753,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
         ip = "127.0.0.1"
         url = "http://example.org"
         status_code = 200
+        http_header_dict = {"a": "b"}
         total_seconds = 0.5
         total_seconds_2 = 99
         fast = 0.2
@@ -619,6 +764,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
             ip,
             url,
             status_code,
+            http_header_dict,
             total_seconds,
             fast,
             moderate,
@@ -630,6 +776,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
             ip,
             url,
             status_code,
+            http_header_dict,
             total_seconds_2,
             fast,
             moderate,
@@ -640,6 +787,9 @@ class SurykatkaHttpTestCase(unittest.TestCase):
         assert self.db.HttpCodeChange.get().ip == ip
         assert self.db.HttpCodeChange.get().url == url
         assert self.db.HttpCodeChange.get().status_code == status_code
+        assert (
+            self.db.HttpCodeChange.get().http_header_dict == http_header_dict
+        )
         assert self.db.HttpCodeChange.get().total_seconds == total_seconds
         assert self.db.HttpCodeChange.get().status_id == status_id
 
@@ -650,6 +800,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
         url_2 = url + "2"
         total_seconds = 0.1
         status_code = 200
+        http_header_dict = {"a": "b"}
         fast = 0.2
         moderate = 0.5
         status_id = logStatus(self.db, "foo")
@@ -658,6 +809,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
             ip,
             url,
             status_code,
+            http_header_dict,
             total_seconds,
             fast,
             moderate,
@@ -668,6 +820,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
             ip_2,
             url,
             status_code,
+            http_header_dict,
             total_seconds,
             fast,
             moderate,
@@ -678,6 +831,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
             ip,
             url_2,
             status_code,
+            http_header_dict,
             total_seconds,
             fast,
             moderate,
@@ -688,6 +842,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
             ip_2,
             url_2,
             status_code,
+            http_header_dict,
             total_seconds,
             fast,
             moderate,
@@ -704,7 +859,10 @@ class SurykatkaHttpTestCase(unittest.TestCase):
         url = "http://example.org/foo?bar=1"
         bot_version = 1
         httpretty.register_uri(
-            httpretty.GET, "http://127.0.0.1/foo?bar=1", status=418
+            httpretty.GET,
+            "http://127.0.0.1/foo?bar=1",
+            status=418,
+            adding_headers={"Etag": "bar"},
         )
 
         status_id = logStatus(self.db, "foo")
@@ -726,6 +884,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
         assert self.db.HttpCodeChange.get().ip == ip
         assert self.db.HttpCodeChange.get().url == url
         assert self.db.HttpCodeChange.get().status_code == 418
+        assert self.db.HttpCodeChange.get().http_header_dict == {"Etag": "bar"}
         assert self.db.HttpCodeChange.get().status_id == status_id
 
     def test_checkHttpStatus_https(self):
@@ -735,6 +894,8 @@ class SurykatkaHttpTestCase(unittest.TestCase):
 
         status_id = logStatus(self.db, "foo")
         with mock.patch("surykatka.http.request") as mock_request:
+            mock_request.return_value.headers = {"Etag": "foobar"}
+
             checkHttpStatus(self.db, status_id, url, ip, bot_version)
 
             assert mock_request.call_count == 1
@@ -756,6 +917,9 @@ class SurykatkaHttpTestCase(unittest.TestCase):
         assert self.db.HttpCodeChange.get().url == url
         # XXX No idea how to mock SSL
         assert self.db.HttpCodeChange.get().status_code == 1
+        assert self.db.HttpCodeChange.get().http_header_dict == {
+            "Etag": "foobar"
+        }
         assert self.db.HttpCodeChange.get().status_id == status_id
 
     def test_checkHttpStatus_relativeUrl(self):
@@ -771,6 +935,87 @@ class SurykatkaHttpTestCase(unittest.TestCase):
         else:
             raise NotImplementedError("Expected NotImplementedError")
 
+    def __generateHeaderDict(self, header_list):
+        result_dict = {}
+        for header in header_list:
+            result_dict[header] = header + " bar"
+        return result_dict
+
+    @httpretty.activate
+    def test_checkHttpStatus_whitelistHttpHeader(self):
+        ip = "127.0.0.1"
+        url = "http://example.org/foo?bar=1"
+        bot_version = 1
+        whitelist_header_list = [
+            # Redirect
+            "Location",
+            # HTTP Range
+            "Accept-Ranges",
+            # HTTP Cache
+            "Etag",
+            "Last-Modified",
+            "Vary",
+            "Cache-Control",
+            "Set-Cookie",
+            "WWW-Authenticate"
+            # gzip
+            "Content-Type",
+            "Content-Encoding",
+            "Content-Disposition"
+            # Security
+            "Content-Security-Policy",
+            "Referrer-Policy",
+            "Strict-Transport-Policy",
+            "Feature-Policy",
+            "X-Frame-Options",
+            "X-Content-Type-Options"
+            # CORS
+            "Access-Control-Allow-Origin",
+            "Access-Control-Allow-Methods",
+            "Access-Control-Allow-Credentials",
+            "Access-Control-Allow-Headers",
+            "Access-Control-Expose-Headers",
+        ]
+        blacklist_header_list = [
+            "Age",
+            "Date",
+            "Expires",
+            "Foo",
+        ]
+        header_dict = self.__generateHeaderDict(whitelist_header_list)
+        header_dict.update(self.__generateHeaderDict(blacklist_header_list))
+
+        httpretty.register_uri(
+            httpretty.GET,
+            "http://127.0.0.1/foo?bar=1",
+            status=418,
+            adding_headers=header_dict,
+        )
+
+        status_id = logStatus(self.db, "foo")
+        checkHttpStatus(self.db, status_id, url, ip, bot_version)
+
+        last_request = httpretty.last_request()
+        assert len(last_request.headers) == 5, last_request.headers.keys()
+        assert last_request.headers["Accept"] == "text/html;q=0.9,*/*;q=0.8"
+        assert last_request.headers["Accept-Encoding"] == "gzip, deflate"
+        assert last_request.headers["Connection"] == "keep-alive"
+        assert last_request.headers["Host"] == "example.org"
+        assert (
+            last_request.headers["User-Agent"]
+            == "SURYKATKA/1 (+https://lab.nexedi.com/nexedi/surykatka)"
+        )
+        assert len(last_request.body) == 0
+
+        assert self.db.HttpCodeChange.select().count() == 1
+        assert self.db.HttpCodeChange.get().ip == ip
+        assert self.db.HttpCodeChange.get().url == url
+        assert self.db.HttpCodeChange.get().status_code == 418
+        assert self.db.HttpCodeChange.get().http_header_dict == self.__generateHeaderDict(
+            whitelist_header_list
+        )
+        assert self.db.HttpCodeChange.get().status_id == status_id
+
 
 def suite():
     suite = unittest.TestSuite()