Report http

urlchecker_http.py

 import requests
 from urllib.parse import urlparse, urlunsplit
 from forcediphttpsadapter.adapters import ForcedIPHTTPSAdapter
+from peewee import fn
 
 
 PREFERRED_TYPE = "text/html"
@@ -62,19 +63,58 @@ def request(url, headers=None, session=requests, version=0):
     return response
 
 
+def reportHttp(db, ip=None, url=None):
+    # http://charlesleifer.com/blog/techniques-for-querying-lists-of-objects-and-determining-the-top-related-item/
+    HttpCodeChangeAlias = db.HttpCodeChange.alias()
+
+    subquery = (
+        HttpCodeChangeAlias.select(
+            HttpCodeChangeAlias.ip,
+            HttpCodeChangeAlias.url,
+            fn.MAX(HttpCodeChangeAlias.status_id).alias("max_status_id"),
+        )
+        .group_by(HttpCodeChangeAlias.ip, HttpCodeChangeAlias.url)
+        .alias("http_change_max_subquery")
+    )
+
+    StatusAlias = db.Status.alias()
+
+    query = (
+        db.HttpCodeChange.select(db.HttpCodeChange, StatusAlias)
+        .join(StatusAlias)
+        .switch(db.HttpCodeChange)
+        .join(
+            subquery,
+            on=(
+                (db.HttpCodeChange.status_id == subquery.c.max_status_id)
+                & (db.HttpCodeChange.ip == subquery.c.ip)
+                & (db.HttpCodeChange.url == subquery.c.url)
+            ),
+        )
+        .order_by(db.HttpCodeChange.url.asc(), db.HttpCodeChange.ip.asc())
+    )
+
+    if ip is not None:
+        if type(ip) == list:
+            query = query.where(db.HttpCodeChange.ip << ip)
+        else:
+            query = query.where(db.HttpCodeChange.ip == ip)
+
+    if url is not None:
+        if type(url) == list:
+            query = query.where(db.HttpCodeChange.url << url)
+        else:
+            query = query.where(db.HttpCodeChange.url == url)
+
+    return query
+
+
 def logHttpStatus(db, ip, url, code, status_id):
 
     with db._db.atomic():
         try:
             # Check previous parameter value
-            previous_entry = (
-                db.HttpCodeChange.select()
-                .where(
-                    db.HttpCodeChange.ip == ip, db.HttpCodeChange.url == url
-                )
-                .order_by(db.HttpCodeChange.status.desc())
-                .get()
-            )
+            previous_entry = reportHttp(db, ip=ip, url=url).get()
         except db.HttpCodeChange.DoesNotExist:
             previous_entry = None
 

urlchecker_network.py

@@ -52,7 +52,10 @@ def reportNetwork(db, ip=None, transport=None, port=None):
         else:
             query = query.where(db.NetworkChange.transport == transport)
     if port is not None:
-        query = query.where(db.NetworkChange.port == port)
+        if type(port) == list:
+            query = query.where(db.NetworkChange.port << port)
+        else:
+            query = query.where(db.NetworkChange.port == port)
     if ip is not None:
         if type(ip) == list:
             query = query.where(db.NetworkChange.ip << ip)