updater: Make stateful decision

If at least once certificate has been downloaded from KeDiFa it shall never
use again the fall-back, as otherwise it would result with a problem, that
next unsuccessful download from KeDiFa would result replacement with
fall-back.

In order to do so state file is introduced keeping list of overridden
certificates. As now there is critical path regarding fetching certificates,
the lock is created to avoid concurrent updates.

kedifa/cli.py

@@ -142,6 +142,13 @@ def updater(*args):
          'certificate, and DESTINATION is the output file.'
   )
 
+  parser.add_argument(
+    'state',
+    type=str,
+    help='Path to JSON state file for fallback recognition, on which locks '
+         'will happen.'
+  )
+
   parser.add_argument(
     '--identity',
     type=argparse.FileType('r'),
@@ -185,7 +192,7 @@ def updater(*args):
   parsed = parser.parse_args(args)
 
   u = Updater(
-    parsed.sleep, parsed.mapping.name, parsed.master_certificate,
+    parsed.sleep, parsed.mapping.name, parsed.state, parsed.master_certificate,
     parsed.on_update, parsed.identity.name, parsed.server_ca_certificate.name,
     parsed.once
   )

kedifa/test.py

@@ -22,6 +22,7 @@ import contextlib
 import datetime
 import httplib
 import ipaddress
+import json
 import mock
 import multiprocessing
 import os
@@ -308,11 +309,14 @@ class KedifaIntegrationTest(KedifaMixinCaucase, unittest.TestCase):
     mapping = tempfile.NamedTemporaryFile(dir=self.testdir, delete=False)
     mapping.write("%s %s" % (url, destination))
     mapping.close()
+    state = tempfile.NamedTemporaryFile(dir=self.testdir, delete=False)
+    state.close()
     updater(
       '--once',
       '--server-ca-certificate', self.ca_crt_pem,
       '--identity', certificate,
       mapping.name,
+      state.name
     )
 
   def updater_get(self, url, certificate):
@@ -1089,123 +1093,242 @@ class KedifaIntegrationTest(KedifaMixinCaucase, unittest.TestCase):
     )
 
 
-class KedifaUpdaterMappingTest(KedifaMixin, unittest.TestCase):
+class KedifaUpdaterMixin(KedifaMixin):
+  def setUp(self):
+    super(KedifaUpdaterMixin, self).setUp()
+
+    state = tempfile.NamedTemporaryFile(dir=self.testdir, delete=False)
+    state.close()
+    self.state = state.name
+
   def setupMapping(self, mapping_content=''):
     mapping = tempfile.NamedTemporaryFile(dir=self.testdir, delete=False)
     mapping.write(mapping_content)
     mapping.close()
     self.mapping = mapping.name
 
+
+class KedifaUpdaterMappingTest(KedifaUpdaterMixin, unittest.TestCase):
   def test_updateMapping_empty(self):
     self.setupMapping()
-    u = Updater(1, self.mapping, None, None, None, None, True)
+    u = Updater(1, self.mapping, self.state, None, None, None, None, True)
     u.updateMapping()
     self.assertEqual(u.mapping, {})
 
   def test_updateMapping_normal(self):
     self.setupMapping('url file')
-    u = Updater(1, self.mapping, None, None, None, None, True)
+    u = Updater(1, self.mapping, self.state, None, None, None, None, True)
     u.updateMapping()
-    self.assertEqual(u.mapping, {'file': 'url'})
+    self.assertEqual(u.mapping, {'file': ('url', None)})
 
   def test_updateMapping_morewhite(self):
     self.setupMapping('url \t file')
-    u = Updater(1, self.mapping, None, None, None, None, True)
+    u = Updater(1, self.mapping, self.state, None, None, None, None, True)
     u.updateMapping()
-    self.assertEqual(u.mapping, {'file': 'url'})
+    self.assertEqual(u.mapping, {'file': ('url', None)})
 
   def test_updateMapping_one_empty(self):
     self.setupMapping('url file\n     \n')
-    u = Updater(1, self.mapping, None, None, None, None, True)
+    u = Updater(1, self.mapping, self.state, None, None, None, None, True)
     u.updateMapping()
-    self.assertEqual(u.mapping, {'file': 'url'})
+    self.assertEqual(u.mapping, {'file': ('url', None)})
 
   def test_updateMapping_one_not_enough(self):
     self.setupMapping('url file\nbuzz\n')
-    u = Updater(1, self.mapping, None, None, None, None, True)
+    u = Updater(1, self.mapping, self.state, None, None, None, None, True)
     u.updateMapping()
-    self.assertEqual(u.mapping, {'file': 'url'})
+    self.assertEqual(u.mapping, {'file': ('url', None)})
 
-  def test_updateMapping_one_too_much(self):
-    self.setupMapping('url file\nbuzz oink aff\n')
-    u = Updater(1, self.mapping, None, None, None, None, True)
+  def test_updateMapping_with_fallback(self):
+    self.setupMapping('url file\nbuzz oink fallback\n')
+    u = Updater(1, self.mapping, self.state, None, None, None, None, True)
     u.updateMapping()
-    self.assertEqual(u.mapping, {'file': 'url'})
+    self.assertEqual(
+      u.mapping, {'file': ('url', None), 'oink': ('buzz', 'fallback')})
 
   def test_updateMapping_one_comment(self):
     self.setupMapping('url file\n#buzz uff\n')
-    u = Updater(1, self.mapping, None, None, None, None, True)
+    u = Updater(1, self.mapping, self.state, None, None, None, None, True)
     u.updateMapping()
-    self.assertEqual(u.mapping, {'file': 'url'})
+    self.assertEqual(u.mapping, {'file': ('url', None)})
 
 
-class KedifaUpdaterUpdateCertificateTest(KedifaMixin, unittest.TestCase):
-  def setupMapping(self, mapping_content=''):
-    mapping = tempfile.NamedTemporaryFile(dir=self.testdir, delete=False)
-    mapping.write(mapping_content)
-    mapping.close()
-    self.mapping = mapping.name
-
-  def _update(self, certificate, fetch, master_content):
+class KedifaUpdaterUpdateCertificateTest(
+  KedifaUpdaterMixin, unittest.TestCase):
+  def setUp(self):
+    super(KedifaUpdaterUpdateCertificateTest, self).setUp()
     certificate_file = tempfile.NamedTemporaryFile(
       dir=self.testdir, delete=False)
-    certificate_file.write(certificate)
     certificate_file.close()
-    self.setupMapping('http://example.com %s' % (certificate_file.name,))
+    self.certificate_file_name = certificate_file.name
+
+  def _update(self, certificate, fetch, master_content, fallback=None):
+    with open(self.certificate_file_name, 'w') as fh:
+      fh.write(certificate)
+    fallback_file = None
+    if fallback:
+      fallback_file = tempfile.NamedTemporaryFile(
+        dir=self.testdir, delete=False)
+      fallback_file.write(fallback)
+      fallback_file.close()
+    mapping = 'http://example.com %s' % (self.certificate_file_name,)
+    if fallback_file:
+      mapping = '%s %s' % (mapping, fallback_file.name)
+    self.setupMapping(mapping)
     u = Updater(
-      1, self.mapping, '/master/certificate/file', None, None, None, True)
+      1, self.mapping, self.state, '/master/certificate/file', None, None,
+      None, True)
     u.updateMapping()
+    u.readState()
     with mock.patch.object(
       Updater, 'fetchCertificate', return_value=fetch):
-      result = u.updateCertificate(certificate_file.name, master_content)
-    return open(certificate_file.name, 'r').read(), result
+      result = u.updateCertificate(self.certificate_file_name, master_content)
+    u.writeState()
+    return open(self.certificate_file_name, 'r').read(), result
+
+  def assertState(self, state):
+    with open(self.state, 'r') as fh:
+      json_state = json.load(fh)
+    self.assertEqual(
+      json_state,
+      state
+    )
 
-  def test_nocert_nofetch_nomaster(self):
+  def test_nocert_nofetch_nomaster_nofallback(self):
     certificate, update = self._update(
       certificate='', fetch='', master_content=None)
     self.assertEqual('', certificate)
     self.assertFalse(update)
+    self.assertState({})
 
-  def test_cert_nofetch_nomaster(self):
+  def test_cert_nofetch_nomaster_nofallback(self):
     certificate, update = self._update(
       certificate='old content', fetch='', master_content=None)
     self.assertEqual('old content', certificate)
     self.assertFalse(update)
+    self.assertState({})
 
-  def test_nocert_fetch_nomaster(self):
+  def test_nocert_fetch_nomaster_nofallback(self):
     certificate, update = self._update(
       certificate='', fetch='content', master_content=None)
     self.assertEqual('content', certificate)
     self.assertTrue(update)
+    self.assertState({self.certificate_file_name: True})
 
-  def test_cert_fetch_nomaster(self):
+  def test_cert_fetch_nomaster_nofallback(self):
     certificate, update = self._update(
       certificate='old content', fetch='content', master_content=None)
     self.assertEqual('content', certificate)
     self.assertTrue(update)
+    self.assertState({self.certificate_file_name: True})
 
-  def test_nocert_nofetch_master(self):
+  def test_nocert_nofetch_master_nofallback(self):
     certificate, update = self._update(
       certificate='', fetch='', master_content='master')
     self.assertEqual('master', certificate)
     self.assertTrue(update)
+    self.assertState({})
 
-  def test_cert_nofetch_master(self):
-    # This is important feature. Master certifcate does not override existing
-    # certificate, so it can use provided outside of KeDiFa
+  def test_cert_nofetch_master_nofallback(self):
     certificate, update = self._update(
       certificate='old content', fetch='', master_content='master')
     self.assertEqual('old content', certificate)
     self.assertFalse(update)
+    self.assertState({})
 
-  def test_nocert_fetch_master(self):
+  def test_nocert_fetch_master_nofallback(self):
     certificate, update = self._update(
       certificate='', fetch='content', master_content='master')
     self.assertEqual('content', certificate)
     self.assertTrue(update)
+    self.assertState({self.certificate_file_name: True})
 
-  def test_cert_fetch_master(self):
+  def test_cert_fetch_master_nofallback(self):
     certificate, update = self._update(
       certificate='old content', fetch='content', master_content='master')
     self.assertEqual('content', certificate)
     self.assertTrue(update)
+    self.assertState({self.certificate_file_name: True})
+
+  def test_nocert_nofetch_nomaster_fallback(self):
+    certificate, update = self._update(
+      certificate='', fetch='', master_content=None, fallback='fallback')
+    self.assertEqual('fallback', certificate)
+    self.assertTrue(update)
+    self.assertState({})
+
+  def test_cert_nofetch_nomaster_fallback(self):
+    certificate, update = self._update(
+      certificate='old content', fetch='', master_content=None,
+      fallback='fallback')
+    self.assertEqual('fallback', certificate)
+    self.assertTrue(update)
+    self.assertState({})
+
+  def test_cert_nofetch_nomaster_fallback_overridden(self):
+    with open(self.state, 'w') as fh:
+      json.dump({self.certificate_file_name: True}, fh)
+    certificate, update = self._update(
+      certificate='old content', fetch='', master_content=None,
+      fallback='fallback')
+    self.assertEqual('old content', certificate)
+    self.assertFalse(update)
+    self.assertState({self.certificate_file_name: True})
+
+  def test_nocert_fetch_nomaster_fallback(self):
+    certificate, update = self._update(
+      certificate='', fetch='content', master_content=None,
+      fallback='fallback')
+    self.assertEqual('content', certificate)
+    self.assertTrue(update)
+    self.assertState({self.certificate_file_name: True})
+
+  def test_cert_fetch_nomaster_fallback(self):
+    certificate, update = self._update(
+      certificate='old content', fetch='content', master_content=None,
+      fallback='fallback')
+    self.assertEqual('content', certificate)
+    self.assertTrue(update)
+    self.assertState({self.certificate_file_name: True})
+
+  def test_nocert_nofetch_master_fallback(self):
+    certificate, update = self._update(
+      certificate='', fetch='', master_content='master',
+      fallback='fallback')
+    self.assertEqual('fallback', certificate)
+    self.assertTrue(update)
+    self.assertState({})
+
+  def test_cert_nofetch_master_fallback(self):
+    certificate, update = self._update(
+      certificate='old content', fetch='', master_content='master',
+      fallback='fallback')
+    self.assertEqual('fallback', certificate)
+    self.assertTrue(update)
+    self.assertState({})
+
+  def test_cert_nofetch_master_fallback_overridden(self):
+    with open(self.state, 'w') as fh:
+      json.dump({self.certificate_file_name: True}, fh)
+    certificate, update = self._update(
+      certificate='old content', fetch='', master_content='master',
+      fallback='fallback')
+    self.assertEqual('old content', certificate)
+    self.assertFalse(update)
+    self.assertState({self.certificate_file_name: True})
+
+  def test_nocert_fetch_master_fallback(self):
+    certificate, update = self._update(
+      certificate='', fetch='content', master_content='master',
+      fallback='fallback')
+    self.assertEqual('content', certificate)
+    self.assertTrue(update)
+    self.assertState({self.certificate_file_name: True})
+
+  def test_cert_fetch_master_fallback(self):
+    certificate, update = self._update(
+      certificate='old content', fetch='content', master_content='master',
+      fallback='fallback')
+    self.assertEqual('content', certificate)
+    self.assertTrue(update)
+    self.assertState({self.certificate_file_name: True})

kedifa/updater.py

 import httplib
+import json
 import os
 import requests
+import sys
 import time
+import zc.lockfile
 
 
 class Updater(object):
-  def __init__(self, sleep, mapping_file, master_certificate_file, on_update,
-               identity_file, server_ca_certificate_file, once):
+  def __init__(self, sleep, mapping_file, state_file, master_certificate_file,
+               on_update, identity_file, server_ca_certificate_file, once):
     self.sleep = sleep
     self.mapping_file = mapping_file
+    self.state_file = state_file
+    self.state_lock_file = '%s.lock' % (state_file, )
     self.master_certificate_file = master_certificate_file
     self.on_update = on_update
     self.identity_file = identity_file
@@ -25,11 +30,15 @@ class Updater(object):
         if not line:
           continue
         line_content = line.split()
-        if len(line_content) != 2:
+        if len(line_content) == 2:
+          url, certificate = line_content
+          fallback = None
+        elif len(line_content) == 3:
+          url, certificate, fallback = line_content
+        else:
           print 'Line %r is incorrect' % (line,)
           continue
-        url, certificate = line_content
-        self.mapping[certificate] = url
+        self.mapping[certificate] = (url, fallback)
 
   def fetchCertificate(self, url, certificate_file):
     certificate = ''
@@ -50,9 +59,17 @@ class Updater(object):
     return certificate
 
   def updateCertificate(self, certificate_file, master_content=None):
-    url = self.mapping[certificate_file]
+    url, fallback_file = self.mapping[certificate_file]
     certificate = self.fetchCertificate(url, certificate_file)
 
+    fallback_overridden = self.state_dict.get(certificate_file, False)
+    fallback = ''
+    if fallback_file:
+      try:
+        with open(fallback_file, 'r') as fh:
+          fallback = fh.read() or None
+      except IOError:
+        pass
     current = ''
     try:
       with open(certificate_file, 'r') as fh:
@@ -61,11 +78,16 @@ class Updater(object):
       current = ''
 
     if not(certificate):
-      if not current and master_content is not None:
+      if fallback and not fallback_overridden:
+        certificate = fallback
+      elif not current and master_content is not None:
         url = self.master_certificate_file
         certificate = master_content
       else:
         return False
+    else:
+      self.state_dict[certificate_file] = True
+
     if current != certificate:
       with open(certificate_file, 'w') as fh:
         fh.write(certificate)
@@ -79,33 +101,63 @@ class Updater(object):
       status = os.system(self.on_update)
       print 'Called %r with status %i' % (self.on_update, status)
 
-  def loop(self):
-    while True:
-      self.updateMapping()
-      updated = False
-
-      if self.master_certificate_file in self.mapping:
-        updated = self.updateCertificate(self.master_certificate_file)
-        self.mapping.pop(self.master_certificate_file)
-
-      master_content = None
-      if self.master_certificate_file is not None:
+  def readState(self):
+    self.state_dict = {}
+    try:
+      with open(self.state_file, 'r') as fh:
         try:
-          with open(self.master_certificate_file, 'r') as fh:
-            master_content = fh.read() or None
-            if master_content:
-              print 'Using master certificate from %r' % (
-                self.master_certificate_file,)
-        except IOError:
+          self.state_dict = json.load(fh)
+        except ValueError:
           pass
+    except IOError:
+      pass
 
-      for certificate_file in self.mapping.keys():
-        if self.updateCertificate(certificate_file, master_content):
-          updated = True
+  def writeState(self):
+    with open(self.state_file, 'w') as fh:
+      json.dump(self.state_dict, fh, indent=2)
 
-      if updated:
-        self.callOnUpdate()
+  def action(self):
+    self.readState()
+    self.updateMapping()
+    updated = False
 
+    if self.master_certificate_file in self.mapping:
+      updated = self.updateCertificate(self.master_certificate_file)
+      self.mapping.pop(self.master_certificate_file)
+
+    master_content = None
+    if self.master_certificate_file is not None:
+      try:
+        with open(self.master_certificate_file, 'r') as fh:
+          master_content = fh.read() or None
+          if master_content:
+            print 'Using master certificate from %r' % (
+              self.master_certificate_file,)
+      except IOError:
+        pass
+
+    for certificate_file in self.mapping.keys():
+      if self.updateCertificate(certificate_file, master_content):
+        updated = True
+
+    if updated:
+      self.callOnUpdate()
+    self.writeState()
+
+  def loop(self):
+    while True:
+      try:
+        lock = zc.lockfile.LockFile(self.state_lock_file)
+      except zc.lockfile.LockError as e:
+        print e,
+        if self.once:
+          print '...exiting.'
+          sys.exit(1)
+        else:
+          print "...will try again later."
+      else:
+        self.action()
+        lock.close()
       if self.once:
         break
       print 'Sleeping for %is' % (self.sleep,)

setup.py

@@ -49,6 +49,7 @@ setup(
   install_requires=[
     'cryptography',  # for working with certificates
     'requests',  # for getter and updater
+    'zc.lockfile',  # for stateful updater
     'urllib3 >= 1.18',  # https://github.com/urllib3/urllib3/issues/258
     'caucase',  # provides utils for certificate management;
                 # version requirement caucase >= 0.9.3 is dropped, as it