Reread configuration file between every loop

urlchecker_bot.py

@@ -33,6 +33,7 @@ def rfc822(date):
 
 class WebBot:
     def __init__(self, **kw):
+        self.config_kw = kw
         self.config = createConfiguration(**kw)
 
     def closeDB(self):
@@ -61,6 +62,10 @@ class WebBot:
 
     def iterateLoop(self):
         status_id = logStatus(self._db, "loop")
+
+        if self.config["RELOAD"] == "True":
+            self.config = createConfiguration(**self.config_kw)
+
         timeout = int(self.config["TIMEOUT"])
         # logPlatform(self._db, __version__, status_id)
 

urlchecker_cli.py

@@ -22,6 +22,12 @@ from urlchecker_bot import create_bot
 @click.option(
     "--configuration", "-f", help="The path of the configuration file."
 )
+@click.option(
+    "--reload/--no-reload",
+    default=False,
+    help="Reload the configuration file between each crawl.",
+    show_default=True,
+)
 @click.option(
     "--output",
     "-o",
@@ -31,7 +37,15 @@ from urlchecker_bot import create_bot
     show_default=True,
 )
 def runUrlChecker(
-    run, sqlite, nameserver, url, domain, timeout, configuration, output
+    run,
+    sqlite,
+    nameserver,
+    url,
+    domain,
+    timeout,
+    configuration,
+    reload,
+    output,
 ):
 
     # click.echo("Running url checker bot")
@@ -47,6 +61,8 @@ def runUrlChecker(
         mapping["SQLITE"] = sqlite
     if nameserver:
         mapping["NAMESERVER"] = nameserver
+    if reload:
+        mapping["RELOAD"] = str(reload)
     mapping["FORMAT"] = output
     bot = create_bot(cfgfile=configuration, mapping=mapping)
     return bot.run(run)

urlchecker_configuration.py

@@ -32,6 +32,8 @@ def createConfiguration(
         config[CONFIG_SECTION]["FORMAT"] = "json"
     if "TIMEOUT" not in config[CONFIG_SECTION]:
         config[CONFIG_SECTION]["TIMEOUT"] = "1"
+    if "RELOAD" not in config[CONFIG_SECTION]:
+        config[CONFIG_SECTION]["RELOAD"] = str(False)
 
     if config[CONFIG_SECTION]["SQLITE"] == ":memory:":
         # Do not loop when using temporary DB