Add SNI support

CONTRIBUTE.md

 # URL checker contribution
 
 ## Install development environnment
+
 virtualenv venv
 . venv/bin/activate
 pip install black pyflakes
+pip install --upgrade --editable .
+
 
 ## Check the code
+
 pyflakes *py
 black -t py37 -l 79 *py
+
+## SNI Support
+
+https://stackoverflow.com/questions/18578439/using-requests-with-tls-doesnt-give-sni-support/18579484
\ No newline at end of file

setup.py

@@ -8,11 +8,11 @@ setup(
     include_package_data=False,
     zip_safe=True,
     install_requires=[
-        "setuptools==40.5.0",
-        "requests==2.20.0",
-        "requests_toolbelt==0.8.0",
-        "peewee==2.10.1",
-        "click==7.0",
+        "setuptools>40.5.0",
+        "requests>2.20.0",
+        "forcediphttpsadapter",
+        "peewee>2.10.1",
+        "click>=7.0",
     ],
     entry_points={
         "console_scripts": ["urlchecker=urlchecker_cli:runUrlChecker "]

urlchecker_http.py

-from urllib.parse import urlparse
+from urllib.parse import urlparse, urlunsplit
 import requests
 import socket
 import sys
 import traceback
 import time
+from forcediphttpsadapter.adapters import ForcedIPHTTPSAdapter
 
 
-__version__ = "0.0.1"
+__version__ = "0.0.2"
 PREFERRED_TYPE = "text/html"
 TIMEOUT = 2
 
@@ -31,6 +32,8 @@ class WebBot:
         stream=False,
         timeout=TIMEOUT,
         allow_redirects=False,
+        verify=True,
+        session=requests,
         **kwargs,
     ):
 
@@ -45,11 +48,13 @@ class WebBot:
         kwargs["stream"] = stream
         kwargs["timeout"] = timeout
         kwargs["allow_redirects"] = allow_redirects
-        args = [url]
+        kwargs["verify"] = verify
+        args = [method, url]
 
         kwargs["headers"] = headers
+
         try:
-            response = requests.get(*args, **kwargs)
+            response = session.request(*args, **kwargs)
         except requests.exceptions.SSLError:
             # XXX Enter into unknown host
             response = requests.models.Response()
@@ -72,8 +77,9 @@ class WebBot:
         return response
 
     def check(self, url):
-        print(url)
         parsed_url = urlparse(url)
+        response = self.request("GET", url)
+        print(url, response.status_code)
 
         # Get the list of available IPv4 frontend CDN
         hostname = parsed_url.hostname
@@ -81,12 +87,20 @@ class WebBot:
         ip_list = [x[4][0] for x in dns_info_list]
 
         for ip in ip_list:
-            parsed_url = parsed_url._replace(netloc=ip)
-            ip_url = parsed_url.geturl()
+            session = requests.Session()
+            # SNI Support
+            if parsed_url.scheme == "https":
+                # Provide SNI support
+                base_url = urlunsplit(
+                    (parsed_url.scheme, parsed_url.netloc, "", "", "")
+                )
+                session.mount(base_url, ForcedIPHTTPSAdapter(dest_ip=ip))
 
-            response = self.request("GET", ip_url, headers={"Host": hostname})
+            response = self.request(
+                "GET", url, headers={"Host": hostname}, session=session
+            )
 
-            print(ip, response.status_code)
+            print(ip, hostname, response.status_code)
 
     def run(self, url):
         print(time.strftime("%Y-%m-%d %H:%M:%S"))