WIP wsgi: Produce http response caching headers.

caucase/ca.py

@@ -193,6 +193,13 @@ class CertificateAuthority(object):
     self._loadCAKeyPairList()
     self._renewCAIfNeeded()
 
+  @property
+  def crt_life_time(self):
+    """
+    Read-only access to crt_life_time ctor parameter, as a timedelta.
+    """
+    return self._crt_life_time
+
   @property
   def digest_list(self):
     """
@@ -229,8 +236,13 @@ class CertificateAuthority(object):
       previous_crt_pem = crt_pem
       previous_key = key
     self._ca_key_pairs_list = ca_key_pair_list
-    self._ca_certificate_chain = tuple(
-      ca_certificate_chain
+    self._ca_certificate_chain_and_expiration_date = (
+      tuple(ca_certificate_chain),
+      (
+        None
+        if previous_crt is None else # Only True during __init__
+        previous_crt.not_valid_after
+      ),
     )
 
   def getCertificateSigningRequest(self, csr_id):
@@ -621,6 +633,14 @@ class CertificateAuthority(object):
     """
     return utils.dump_certificate(self._getCurrentCAKeypair()['crt'])
 
+  def getCACertificateAndExpirationDate(self):
+    """
+    Return current CA certificate, PEM-encoded, and its expiration date
+    (datetime).
+    """
+    certificate = self._getCurrentCAKeypair()['crt']
+    return utils.dump_certificate(certificate), certificate.not_valid_after
+
   def getCACertificateList(self):
     """
     Return the current list of CA certificates as X509 obbjects.
@@ -630,7 +650,8 @@ class CertificateAuthority(object):
 
   def getValidCACertificateChain(self):
     """
-    Return the CA certificate chain based on oldest CA certificate.
+    Return the CA certificate chain based on oldest CA certificate, and
+    expiration date of the last (most recent) CA certificate in the chain.
 
     Each item in the chain is a wrapped dict with the following keys:
     old (str)
@@ -655,7 +676,7 @@ class CertificateAuthority(object):
     purposes.
     """
     self._renewCAIfNeeded()
-    return self._ca_certificate_chain
+    return self._ca_certificate_chain_and_expiration_date
 
   def revoke(self, crt_pem):
     """

caucase/test.py

@@ -1644,6 +1644,7 @@ class CaucaseTest(unittest.TestCase):
       Mock CAU.
       """
       digest_list = ['sha256']
+      crt_life_time = datetime.timedelta(90, 0)
 
       @staticmethod
       def getCACertificateList():
@@ -1653,11 +1654,14 @@ class CaucaseTest(unittest.TestCase):
         return cau_list
 
       @staticmethod
-      def getCACertificate():
+      def getCACertificateAndExpirationDate():
         """
         Return a dummy string as CA certificate
         """
-        return b'notreallyPEM'
+        return (
+          b'notreallyPEM',
+          datetime.datetime.utcnow() + datetime.timedelta(130, 0),
+        )
 
       @staticmethod
       def getCertificateRevocationListDict():
@@ -2022,10 +2026,6 @@ class CaucaseTest(unittest.TestCase):
       header_dict['Access-Control-Allow-Origin'],
       cross_origin,
     )
-    self.assertEqual(
-      header_dict['Vary'],
-      'Origin',
-    )
     self.assertItemsEqual(
       [
         x.strip()

caucase/wsgi.py

@@ -24,6 +24,7 @@ Caucase - Certificate Authority for Users, Certificate Authority for SErvices
 from __future__ import absolute_import
 from binascii import unhexlify
 from Cookie import SimpleCookie, CookieError
+from functools import partial
 import httplib
 import json
 import os
@@ -260,6 +261,142 @@ class CORSTokenManager(object):
         pass
     return default
 
+# Order matters: higher is cachable in more places, lower is cacheable in
+# fewer places.
+CACHE_SCOPE_NO_STORE = 0
+CACHE_SCOPE_PRIVATE = 1
+CACHE_SCOPE_PUBLIC = 2
+CACHE_SCOPE_CAPTION_DICT = {
+  # Note: according to Mozilla Developer Network, no-cache, while requiring
+  # validation, allows caching. no-store actually forbids caching.
+  CACHE_SCOPE_NO_STORE: 'no-store',
+  CACHE_SCOPE_PRIVATE: 'private',
+  CACHE_SCOPE_PUBLIC: 'public',
+}
+
+# rfc7231#section-4.3
+UNCACHEABLE_METHOD_SET = {
+  'PUT',
+  'DELETE',
+  'CONNECT', # Should not see these
+  'OPTIONS',
+  'TRACE', # Should not see these
+}
+class AutoCacheEnviron(object):
+  """
+  Wraps a WSGI environ dict to keeping track of which keys were accessed,
+  and builds Vary and Cache-Control headers from them.
+  """
+  # These are explicitly excluded from Vary header, as they are either
+  # always considered by caches:
+  #    rfc7231#section-7.1.4
+  #    The "Vary" header field in a response describes what parts of a
+  #    request message, aside from the method, Host header field, and
+  #    request target, might influence the origin server's process for
+  #    selecting and representing this response.
+  # or just not part of the request.
+  __no_vary = {
+    'SERVER_NAME',       # part of Host
+    'GATEWAY_INTERFACE', # local
+    'SERVER_PORT',       # part of Host
+    #'SERVER_PROTOCOL',
+    'REQUEST_METHOD',    # referenced in RFC
+    'SCRIPT_NAME',       # part of target
+    'PATH_INFO',         # part of target
+    'QUERY_STRING',      # part of target
+    'HTTP_HOST',         # referenced in RFC
+    'wsgi.errors',       # local
+    'wsgi.version',      # local
+    'wsgi.run_once',     # local
+    # Rationale: Host header (explicitly mentionned in the spec) covers
+    # the port (implicit or explicit). As http does not support mixed-
+    # http and https traffic on a single Host value, this means the scheme
+    # is bijective to Host header, and varying on one means varying on the
+    # other. Host being already implicitly varied on, the scheme can be
+    # skipped.
+    'wsgi.url_scheme',   # part of Host
+    'wsgi.multithread',  # local
+    'wsgi.multiprocess', # local
+    'HTTPS',             # part of Host
+  }
+  # These do not have an HTTP_ prefix but are still acceptable Vary items.
+  __vary = {
+    'CONTENT_TYPE',
+    'CONTENT_LENGTH',
+  }
+  # From this point on, the settings are subjective and not based on hard RFCs.
+
+  # Vary items which also trigger a "Cache-Control: private" (unless already
+  # at a more restricting caching setting) as they can contain so many
+  # variations as to flood shared caches without improving hit-rates.
+  # Note: wsgi.input is not a header, so not a valid Vary, so it immediately
+  # becomes a "Vary: *", which is considered supervaried.
+  __supervaried = {
+    '*',
+    'ACCEPT',
+    'CACHE',
+    'USER_AGENT',
+    'REFERER',
+    'CONTENT_TYPE',
+    'CONTENT_LENGTH',
+  }
+  # Similar to __supervaried, but do not trigger Cache-Control change if
+  # request header value is in the given set.
+  __supervaried_unless = {
+    'ORIGIN': (None, ''),
+  }
+  __has_star_vary = False
+
+  def __init__(self, environ, vary_set, cache_scope_list):
+    self.__environ = environ
+    self.__vary_set = vary_set
+    self.__cache_scope_list = cache_scope_list
+
+  def __accessed(self, key, value):
+    if key in self.__no_vary:
+      return
+    if key.startswith('HTTP_'):
+      self.__varyOn(key[5:], value)
+    elif key in self.__vary:
+      self.__varyOn(key, value)
+    elif not self.__has_star_vary:
+      self.__has_star_vary = True
+      self.__vary_set.clear()
+      self.__varyOn('*', object)
+
+  def __varyOn(self, key, value):
+    if key in self.__supervaried or (
+      key in self.__supervaried_unless and
+      value not in self.__supervaried_unless[key]
+    ):
+      self.__cache_scope_list[0] = min(
+        CACHE_SCOPE_PRIVATE,
+        self.__cache_scope_list[0],
+      )
+    if not self.__has_star_vary:
+      self.__vary_set.add(key)
+
+  def get(self, key, default=None):
+    """
+    Returns environ.get(key, default) and mark key as accessed.
+    """
+    result = self.__environ.get(key, default)
+    self.__accessed(key, result)
+    return result
+
+  def __getitem__(self, key):
+    """
+    Returns environ[key] and mark key as accessed.
+    """
+    try:
+      result = self.__environ[key]
+    except KeyError:
+      result = None
+      raise
+    finally:
+      self.__accessed(key, result)
+    return result
+
 class Application(object):
   """
   WSGI application class
@@ -338,6 +475,10 @@ class Application(object):
     # - status: HTTP status code & reason
     # - header_list: HTTP reponse header list (see wsgi specs)
     # - iterator: HTTP response body generator (see wsgi specs)
+    # "cache-scope": one of the CACHE_SCOPE_* constants. Mandatory on cachable
+    #   request methods, forbidden on non-cachable request methods.
+    # "cache-control": string of extra values to put in response Cache-Control
+    #   header. Forbidden on non-cachable request methods.
     # "cors": CORS policy (default: ask)
     # "descriptor": list of descriptor dicts.
     # "context_is_routing": whether context should be set to routing dict for
@@ -358,6 +499,10 @@ class Application(object):
         'method': {
           'GET': {
             'do': self.getCertificateRevocationList,
+            'cache-scope': CACHE_SCOPE_PUBLIC,
+            # Note: using a short cache to not delay revocation propagation
+            # too much.
+            'cache-control': 'must-revalidate, max-age=60',
             'subpath': SUBPATH_OPTIONAL,
             'descriptor': [
               {
@@ -383,6 +528,11 @@ class Application(object):
         'method': {
           'GET': {
             'do': self.getCSR,
+            # Note: becomes CACHE_SCOPE_PRIVATE when authentication is checked.
+            'cache-scope': CACHE_SCOPE_PUBLIC,
+            # Note: using a very short cache as users will typically only check
+            # this URL just before doing actions which will change its content.
+            'cache-control': 'max-age=15',
             'subpath': SUBPATH_OPTIONAL,
             'descriptor': [{
               'name': 'getPendingCertificateRequestList',
@@ -419,6 +569,8 @@ class Application(object):
             'method': {
               'GET': {
                 'do': self.getCACertificate,
+                # Note: Max-Age generated during rendering.
+                'cache-scope': CACHE_SCOPE_PUBLIC,
                 'descriptor': [{
                   'name': 'getCACertificate',
                   'title': 'Retrieve current CA certificate.',
@@ -430,6 +582,8 @@ class Application(object):
             'method': {
               'GET': {
                 'do': self.getCACertificateChain,
+                # Note: Max-Age generated during rendering.
+                'cache-scope': CACHE_SCOPE_PUBLIC,
                 'descriptor': [{
                   'name': 'getCACertificateChain',
                   'title': 'Retrieve current CA certificate trust chain.',
@@ -463,6 +617,8 @@ class Application(object):
         'method': {
           'GET': {
             'do': self.getCertificate,
+            # Note: Max-Age generated during rendering.
+            'cache-scope': CACHE_SCOPE_PUBLIC,
             'subpath': SUBPATH_REQUIRED,
             'descriptor': [{
               'name': 'getCertificate',
@@ -488,6 +644,9 @@ class Application(object):
     getHALMethodDict = lambda name, title: {
       'GET': {
         'do': self.getHAL,
+        # Note: content may only change with software upgrades
+        'cache-scope': CACHE_SCOPE_PUBLIC,
+        'cache-control': 'max-age=3600',
         'context_is_routing': True,
         'cors': CORS_POLICY_ALWAYS_ALLOW,
         'descriptor': [{
@@ -501,6 +660,9 @@ class Application(object):
         'GET': {
           # XXX: Use full-recursion getHAL instead ?
           'do': self.getTopHAL,
+          # Note: content may only change with software upgrades
+          'cache-scope': CACHE_SCOPE_PUBLIC,
+          'cache-control': 'max-age=3600',
           'context_is_routing': True,
           'cors': CORS_POLICY_ALWAYS_ALLOW,
         },
@@ -510,9 +672,13 @@ class Application(object):
           'method': {
             'GET': {
               'do': self.getCORSForm,
+              # Note: content may only change with software upgrades
+              'cache-scope': CACHE_SCOPE_PUBLIC,
+              'cache-control': 'max-age=3600',
             },
             'POST': {
               'do': self.postCORSForm,
+              'cache-scope': CACHE_SCOPE_NO_STORE,
               'cors': CORS_POLICY_ALWAYS_DENY,
             },
           },
@@ -534,9 +700,14 @@ class Application(object):
     """
     WSGI entry point
     """
+    cache_control = None
     cors_header_list = []
+    # Mutables for AutoCacheEnviron to act on
+    vary_set = set()
+    cache_scope_list = [None]
     try: # Convert ApplicationError subclasses into error responses
       try: # Convert exceptions into ApplicationError subclass exceptions
+        request_method = environ['REQUEST_METHOD']
         path_item_list = [
           x
           for x in environ.get('PATH_INFO', '').split('/')
@@ -553,33 +724,55 @@ class Application(object):
           del path_item_list[0]
         # If this raises, it means the routing dict is inconsistent.
         method_dict = path_entry_dict['method']
-        request_method = environ['REQUEST_METHOD']
         try:
           action_dict = method_dict[request_method]
         except KeyError:
-          if request_method == 'OPTIONS':
-            status = STATUS_NO_CONTENT
-            header_list = []
-            result = []
-            self._checkCORSAccess(
-              environ=environ,
-              # Pre-flight is always allowed.
-              policy=CORS_POLICY_ALWAYS_ALLOW,
-              header_list=cors_header_list,
-              preflight=True,
-            )
-            if cors_header_list:
-              # CORS headers added, add more
-              self._optionAddCORSHeaders(method_dict, cors_header_list)
-          else:
+          if request_method != 'OPTIONS':
             raise BadMethod(method_dict.keys() + ['OPTIONS'])
+          action_dict = {
+            'do': partial(
+              self._optionCheckCORSAccess,
+              method_dict=method_dict,
+            ),
+            'subpath': (
+              SUBPATH_FORBIDDEN
+              if all(
+                x.get('subpath', SUBPATH_FORBIDDEN) is SUBPATH_FORBIDDEN
+                for x in method_dict.itervalues()
+              ) else
+              SUBPATH_OPTIONAL
+            ),
+          }
+          need_precheck_cors = False
         else:
+          need_precheck_cors = True
         subpath = action_dict.get('subpath', SUBPATH_FORBIDDEN)
         if (
           subpath is SUBPATH_FORBIDDEN and path_item_list or
           subpath is SUBPATH_REQUIRED and not path_item_list
         ):
           raise NotFound
+        # If method specified as uncachable, skip Vary generation.
+        # This double-negation is so unknown methods are treated as possibly
+        # cacheable (better safe than cached).
+        if request_method in UNCACHEABLE_METHOD_SET:
+          assert 'cache-scope' not in action_dict, action_dict
+          assert 'cache-control' not in action_dict, action_dict
+        else:
+          cache_scope_list[0] = cache_scope = action_dict['cache-scope']
+          # If action's cache scope is already no-store, skip Vary
+          # generation and custom cache-control retrieval.
+          if cache_scope > CACHE_SCOPE_NO_STORE:
+            environ = AutoCacheEnviron(
+              environ=environ,
+              vary_set=vary_set,
+              cache_scope_list=cache_scope_list,
+            )
+            cache_control = action_dict.get('cache-control')
+          else:
+            assert 'cache-control' not in action_dict
+       # Skip pre-action CORS checking if action *is* CORS checking.
+        if need_precheck_cors:
           self._checkCORSAccess(
             environ=environ,
             policy=action_dict.get('cors'),
@@ -619,9 +812,23 @@ class Application(object):
       header_list = e.response_headers
       result = [utils.toBytes(str(x)) for x in e.args]
     # Note: header_list and cors_header_list are expected to contain
-    # distinct header sets. This may not always stay true for "Vary".
+    # distinct header sets.
     header_list.extend(cors_header_list)
     header_list.append(('Date', utils.timestamp2IMFfixdate(time.time())))
+    cache_scope, = cache_scope_list
+    if cache_scope is not None:
+      final_cache_control = CACHE_SCOPE_CAPTION_DICT[cache_scope]
+      # other options are ignored in case of no-store
+      if cache_scope > CACHE_SCOPE_NO_STORE and cache_control is not None:
+        final_cache_control += ', ' + cache_control
+      header_list.append(('Cache-Control', final_cache_control))
+    if vary_set:
+      # All request headers we care about use "-" and not "_",
+      # so unconditionally undo the CGI mangling.
+      header_list.append((
+        'Vary',
+        ','.join(x.replace('_', '-') for x in vary_set),
+      ))
     start_response(status, header_list)
     return result
 
@@ -666,8 +873,10 @@ class Application(object):
     Verify user authentication.
 
     Raises SSLUnauthorized if authentication does not pass checks.
-    On success, appends a "Cache-Control" header.
+    On success, a "Cache-Control" header is added by automatic caching header
+    logic, as a non-header environment value is accessed.
     """
+    _ = header_list # Silence pylint
     try:
       ca_list = self._cau.getCACertificateList()
       utils.load_certificate(
@@ -680,7 +889,6 @@ class Application(object):
       )
     except (exceptions.CertificateVerificationError, ValueError):
       raise SSLUnauthorized
-    header_list.append(('Cache-Control', 'private'))
 
   def _readJSON(self, environ):
     """
@@ -719,8 +927,28 @@ class Application(object):
     # the validity period of their entry - a year by default).
     return cookie
 
-  @staticmethod
-  def _optionAddCORSHeaders(method_dict, header_list):
+  def _optionCheckCORSAccess(
+    self,
+    method_dict,
+    context,
+    environ,
+    subpath=None,
+  ):
+    """
+    Used as a stand-in action for OPTION method.
+    """
+    _ = context # Silence pylint
+    _ = subpath # Silence pylint
+    header_list = []
+    self._checkCORSAccess(
+      environ=environ,
+      # Pre-flight is always allowed.
+      policy=CORS_POLICY_ALWAYS_ALLOW,
+      header_list=header_list,
+      preflight=True,
+    )
+    if header_list:
+      # CORS headers added, add more
       header_list.append((
         'Access-Control-Allow-Methods',
         ', '.join(
@@ -738,6 +966,11 @@ class Application(object):
         # - forbidden names (handled by user agent, not controlled by script)
         'Content-Type, User-Agent',
       ))
+    return (
+      STATUS_NO_CONTENT,
+      header_list,
+      [],
+    )
 
   def _checkCORSAccess(
     self,
@@ -823,7 +1056,6 @@ class Application(object):
           # - forbidden names (handled by user agent, not controlled by script)
           'Location, WWW-Authenticate',
         ))
-      header_list.append(('Vary', 'Origin'))
     else:
       raise Forbidden
 
@@ -1102,9 +1334,18 @@ class Application(object):
     Handle GET /{context}/crt/ca.crt.pem urls.
     """
     _ = environ # Silence pylint
+    certificate, expiration_date = context.getCACertificateAndExpirationDate()
     return self._returnFile(
-      context.getCACertificate(),
+      certificate,
       'application/x-x509-ca-cert',
+      header_list=[
+        (
+          'Expires',
+          utils.timestamp2IMFfixdate(
+            utils.datetime2timestamp(expiration_date - context.crt_life_time),
+          ),
+        ),
+      ],
     )
 
   def getCACertificateChain(self, context, environ):
@@ -1112,9 +1353,18 @@ class Application(object):
     Handle GET /{context}/crt/ca.crt.json urls.
     """
     _ = environ # Silence pylint
+    certificate_chain, expiration_date = context.getValidCACertificateChain()
     return self._returnFile(
-      json.dumps(context.getValidCACertificateChain()).encode('utf-8'),
+      json.dumps(certificate_chain).encode('utf-8'),
       'application/json',
+      header_list=[
+        (
+          'Expires',
+          utils.timestamp2IMFfixdate(
+            utils.datetime2timestamp(expiration_date - context.crt_life_time),
+          ),
+        ),
+      ],
     )
 
   def getCertificate(self, context, environ, subpath):
@@ -1122,9 +1372,28 @@ class Application(object):
     Handle GET /{context}/crt/{crt_id} urls.
     """
     _ = environ # Silence pylint
+    certificate_pem = context.getCertificate(self._getCSRID(subpath))
+    ca_list = context.getCACertificateList()
     return self._returnFile(
-      context.getCertificate(self._getCSRID(subpath)),
+      certificate_pem,
       'application/pkix-cert',
+      header_list=[
+        (
+          'Expires',
+          utils.timestamp2IMFfixdate(
+            utils.datetime2timestamp(
+              utils.load_certificate(
+                certificate_pem,
+                ca_list,
+                utils.load_crl(
+                  context.getCertificateRevocationList(),
+                  ca_list,
+                ),
+              ).not_valid_after,
+            ),
+          ),
+        ),
+      ],
     )
 
   def revokeCertificate(self, context, environ):