Commit 0b516822 authored by Łukasz Nowak's avatar Łukasz Nowak

bot: Correctly group IPs per URLs and DOMAINs

parent 72bcbb48
...@@ -249,31 +249,31 @@ class WebBot: ...@@ -249,31 +249,31 @@ class WebBot:
url_dict[url].append(network_change["ip"]) url_dict[url].append(network_change["ip"])
# Report the SSL status # Report the SSL status
query = reportSslCertificate(
self._db,
ip=[x for x in server_ip_dict.keys()],
port=443,
hostname=domain_list,
)
result_dict["ssl_certificate"] = [] result_dict["ssl_certificate"] = []
for ssl_certificate in query.dicts().iterator(): for ip_, domain_list_ in server_ip_dict.items():
result_dict["ssl_certificate"].append( query = reportSslCertificate(
{ self._db, ip=ip_, port=443, hostname=domain_list_,
"hostname": ssl_certificate["hostname"],
"ip": ssl_certificate["ip"],
"port": ssl_certificate["port"],
"sha1_fingerprint": ssl_certificate["sha1_fingerprint"],
"subject": ssl_certificate["subject"],
"issuer": ssl_certificate["issuer"],
"not_before": rfc822(ssl_certificate["not_before"])
if (ssl_certificate["not_before"] is not None)
else None,
"not_after": rfc822(ssl_certificate["not_after"])
if (ssl_certificate["not_after"] is not None)
else None,
"date": rfc822(ssl_certificate["status"]),
}
) )
for ssl_certificate in query.dicts().iterator():
result_dict["ssl_certificate"].append(
{
"hostname": ssl_certificate["hostname"],
"ip": ssl_certificate["ip"],
"port": ssl_certificate["port"],
"sha1_fingerprint": ssl_certificate[
"sha1_fingerprint"
],
"subject": ssl_certificate["subject"],
"issuer": ssl_certificate["issuer"],
"not_before": rfc822(ssl_certificate["not_before"])
if (ssl_certificate["not_before"] is not None)
else None,
"not_after": rfc822(ssl_certificate["not_after"])
if (ssl_certificate["not_after"] is not None)
else None,
"date": rfc822(ssl_certificate["status"]),
}
)
# XXX put back orignal url list # XXX put back orignal url list
for url in self.calculateUrlList(): for url in self.calculateUrlList():
...@@ -282,23 +282,28 @@ class WebBot: ...@@ -282,23 +282,28 @@ class WebBot:
if root_url in url_dict: if root_url in url_dict:
url_dict[url] = url_dict[root_url] url_dict[url] = url_dict[root_url]
# map IP to URLs for less queries during fetching results
ip_to_url_dict = {}
for url, ip_list in url_dict.items():
for ip in ip_list:
ip_to_url_dict.setdefault(ip, [])
if url not in ip_to_url_dict[ip]:
ip_to_url_dict[ip].append(url)
# Get the list of HTTP servers to check # Get the list of HTTP servers to check
query = reportHttp(
self._db,
ip=[x for x in server_ip_dict.keys()],
url=[x for x in url_dict.keys()],
)
result_dict["http_query"] = [] result_dict["http_query"] = []
for network_change in query.dicts().iterator(): for ip, url_list in ip_to_url_dict.items():
result_dict["http_query"].append( query = reportHttp(self._db, ip=ip, url=url_list)
{ for network_change in query.dicts().iterator():
"status_code": network_change["status_code"], result_dict["http_query"].append(
"total_seconds": network_change["total_seconds"], {
"url": network_change["url"], "status_code": network_change["status_code"],
"ip": network_change["ip"], "total_seconds": network_change["total_seconds"],
"date": rfc822(network_change["status"]), "url": network_change["url"],
} "ip": network_change["ip"],
) "date": rfc822(network_change["status"]),
}
)
return result_dict return result_dict
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment