store key only for Etag, Last-Modified and Set-Cookie whose value is not stable.

src/surykatka/http.py

@@ -203,11 +203,8 @@ def checkHttpStatus(
         # HTTP Range
         "Accept-Ranges",
         # HTTP Cache
-        "Etag",
-        "Last-Modified",
         "Vary",
         "Cache-Control",
-        "Set-Cookie",
         "WWW-Authenticate"
         # gzip
         "Content-Type",
@@ -233,6 +230,18 @@ def checkHttpStatus(
         if header_value is not None:
             header_dict[header_key] = header_value
 
+    # Store key only, because of non stability
+    # 'Etag', 'Last-Modified', 'Set-Cookie',
+    key_only_header_list = [
+        "Etag",
+        "Last-Modified",
+        "Set-Cookie",
+    ]
+    for header_key in key_only_header_list:
+        header_value = response.headers.get(header_key, None)
+        if header_value is not None:
+            header_dict[header_key] = True
+
     logHttpStatus(
         db,
         ip,

tests/test_http.py

@@ -862,7 +862,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
             httpretty.GET,
             "http://127.0.0.1/foo?bar=1",
             status=418,
-            adding_headers={"Etag": "bar"},
+            adding_headers={"Etag": "bar", "Cache-Control": "public"},
         )
 
         status_id = logStatus(self.db, "foo")
@@ -884,7 +884,10 @@ class SurykatkaHttpTestCase(unittest.TestCase):
         assert self.db.HttpCodeChange.get().ip == ip
         assert self.db.HttpCodeChange.get().url == url
         assert self.db.HttpCodeChange.get().status_code == 418
-        assert self.db.HttpCodeChange.get().http_header_dict == {"Etag": "bar"}
+        assert self.db.HttpCodeChange.get().http_header_dict == {
+            "Cache-Control": "public",
+            "Etag": True,
+        }
         assert self.db.HttpCodeChange.get().status_id == status_id
 
     def test_checkHttpStatus_https(self):
@@ -894,7 +897,10 @@ class SurykatkaHttpTestCase(unittest.TestCase):
 
         status_id = logStatus(self.db, "foo")
         with mock.patch("surykatka.http.request") as mock_request:
-            mock_request.return_value.headers = {"Etag": "foobar"}
+            mock_request.return_value.headers = {
+                "Etag": "foobar",
+                "Cache-Control": "public",
+            }
 
             checkHttpStatus(self.db, status_id, url, ip, bot_version)
 
@@ -918,7 +924,8 @@ class SurykatkaHttpTestCase(unittest.TestCase):
         # XXX No idea how to mock SSL
         assert self.db.HttpCodeChange.get().status_code == 1
         assert self.db.HttpCodeChange.get().http_header_dict == {
-            "Etag": "foobar"
+            "Cache-Control": "public",
+            "Etag": True,
         }
         assert self.db.HttpCodeChange.get().status_id == status_id
 
@@ -952,11 +959,8 @@ class SurykatkaHttpTestCase(unittest.TestCase):
             # HTTP Range
             "Accept-Ranges",
             # HTTP Cache
-            "Etag",
-            "Last-Modified",
             "Vary",
             "Cache-Control",
-            "Set-Cookie",
             "WWW-Authenticate"
             # gzip
             "Content-Type",