apache-frontend: Slave with cache now have dedicated virtualhost, add...

apache-frontend: Slave with cache now have dedicated virtualhost, add checkProxyCertificate, Strip Via header option

software/apache-frontend/common.cfg

@@ -65,7 +65,7 @@ mode = 0644
 [template-apache-frontend]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-apache-frontend.cfg
-md5sum = cd5a385c44d56b4d13392eba4e938969
+md5sum = f65456f704a32c43822b1efefc7ae4b7
 output = ${buildout:directory}/template-apache-frontend.cfg
 mode = 0644
 
@@ -78,7 +78,7 @@ mode = 0644
 [template-slave-list]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/templates/apache-custom-slave-list.cfg.in
-md5sum = 1fe76dde85c488e94baf8510775ebcaf
+md5sum = 588cbae0ac6fdb65ae97d1c90c8d53cf
 mode = 640
 
 [template-slave-configuration]
@@ -102,7 +102,7 @@ mode = 640
 [template-apache-cached-configuration]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/templates/apache_cached.conf.in
-md5sum = 116271eafe80309a99203fd8a11a4558
+md5sum = a1c744e48b465a63c2d6f0f384466013
 mode = 640
 
 [template-rewrite-cached]
@@ -133,7 +133,13 @@ mode = 640
 [template-default-slave-virtualhost]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/templates/default-virtualhost.conf.in
-md5sum = 5dbfd59f9316b8a629f9f098a1cc1c72
+md5sum = aed0077ee82aaa7fbd2b7e84ce5fbd69
+mode = 640
+
+[template-cached-slave-virtualhost]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/templates/cached-virtualhost.conf.in
+md5sum = b1fd5f2b94f026ccca5ff47167015f23
 mode = 640
 
 [template-log-access]

software/apache-frontend/instance-apache-frontend.cfg

@@ -159,6 +159,8 @@ filename = custom-personal-instance-slave-list.cfg
 extensions = jinja2.ext.do
 extra-context =
     key apache_configuration_directory apache-directory:slave-configuration
+    key apache_cached_configuration_directory apache-directory:slave-with-cache-configuration
+    key cached_port apache-configuration:cache-through-port
     key http_port instance-parameter:configuration.plain_http_port
     key https_port instance-parameter:configuration.port
     key public_ipv4 instance-parameter:configuration.public-ipv4
@@ -172,7 +174,7 @@ extra-context =
     raw empty_template ${template-empty:target}
     raw template_custom_slave_configuration ${template-slave-configuration:target}
     raw template_default_slave_configuration ${template-default-slave-virtualhost:target}
-    raw template_rewrite_cached ${template-rewrite-cached:target}
+    raw template_cached_slave_configuration ${template-cached-slave-virtualhost:target}
     raw software_type single-custom-personal
     section logrotate_dict logrotate
     section frontend_configuration frontend-configuration
@@ -297,6 +299,7 @@ extra-context =
     key access_log apache-configuration:cache-access-log
     key error_log apache-configuration:cache-error-log
     key pid_file apache-configuration:cache-pid-file
+    key slave_with_cache_configuration_directory apache-directory:slave-with-cache-configuration
     key apachecachedmap_path apache-configuration:cached-rewrite-file
 
 [apache-cached]
@@ -317,6 +320,7 @@ link-binary =
 recipe = slapos.cookbook:mkdirectory
 document-root = $${directory:srv}/htdocs
 slave-configuration = $${directory:etc}/apache-slave-conf.d/
+slave-with-cache-configuration = $${directory:etc}/apache-slave-with-cache-conf.d/
 cache = $${directory:var}/cache
 mod-ssl = $${:cache}/httpd_mod_ssl
 vh-ssl = $${:slave-configuration}/ssl

software/apache-frontend/instance-slave-apache-input-schema.json

@@ -72,6 +72,21 @@
       "enum": ["false", "true"]
     },
 
+    "ssl-proxy-verify": {
+      "title": "Verify Backend Certificates",
+      "description": "If set to true, Backend Certificates are checked",
+      "type": "string",
+      "default": "false",
+      "enum": ["false", "true"]
+    },
+
+    "ssl_proxy_ca_crt": {
+      "title": "SSL Backend Authority's Certificate",
+      "description": "SSL Certificate Authority of the backen (to be used with ssl-proxy-verify)",
+      "type": "string",
+      "default": ""
+    },
+
     "enable_cache": {
       "title": "Enable Cache",
       "description": "If set to true, the cache is used",
@@ -88,6 +103,14 @@
       "enum": ["false", "true"]
     },
 
+    "disable-via-header": {
+      "title": "Disable 'Via' headers from cache",
+      "description": "If set to true, via headers will be disabled",
+      "type": "string",
+      "default": "false",
+      "enum": ["false", "true"]
+    },
+
     "prefer-gzip-encoding-to-backend": {
       "title": "Prefer gzip Encoding for Backend",
       "description": "If set to true, if a request is made with accept encoding 'gzip', only that one will be transferred to the backend",

software/apache-frontend/templates/apache-custom-slave-list.cfg.in

@@ -12,7 +12,6 @@
 {% endif -%}
 [jinja2-template-base]
 recipe = slapos.recipe.template:jinja2
-rendered = {{ apache_configuration_directory }}/${:filename}
 extra-context =
 context =
     key eggs_directory buildout:eggs-directory
@@ -31,6 +30,7 @@ context =
 {%   set slave_section_title = 'dynamic-template-slave-instance-%s' % slave_reference -%}
 {%   set slave_parameter_dict = generic_instance_parameter_dict.copy() -%}
 {%   set slave_publish_dict = {} -%}
+{%   set slave_configuration_section_name = 'slave-instance-%s-configuration' % slave_reference %}
 {%   do part_list.append(slave_section_title) -%}
 
 ############################
@@ -98,7 +98,7 @@ command = {{frontend_configuration.get('apache-directory')}}/bin/htpasswd -cb ${
 #### Set Slave Certificates if needed
 
 # Set ssl certificates for each slave
-{%   for cert_name in ('ssl_key', 'ssl_crt', 'ssl_ca_crt', 'ssl_csr')-%}
+{%   for cert_name in ('ssl_key', 'ssl_crt', 'ssl_ca_crt', 'ssl_csr', 'ssl_proxy_ca_crt')-%}
 {%     if cert_name in slave_instance -%}
 {%       set cert_title = '%s-%s' % (slave_reference, cert_name.replace('ssl_', '')) -%}
 {%       set cert_file = '/'.join([custom_ssl_directory, cert_title.replace('-','.')]) -%}
@@ -136,20 +136,24 @@ extra-context =
     raw http_port {{ http_port }}
 {{ '\n' }}
 
+# The slave use cache
+{%     if 'enable_cache' in slave_instance and 'url' in slave_instance and 'domain' in slave_instance -%}
+{%       do slave_instance.__setitem__('custom_domain', slapparameter_dict.get('domain')) -%}
+{%       do slave_instance.__setitem__('backend_url', slave_instance.get('url')) -%}
+{%       do cached_server_dict.__setitem__(slave_reference, slave_configuration_section_name) -%}
+{%     endif -%}
 
 # Set apache configuration value for slave
-[{{ ('slave-instance-%s-configuration' % slave_reference) }}]
-{%   set apache_custom_http = ((slave_instance.get('apache_custom_http', '')) % slave_parameter_dict) -%}
-{%   set apache_custom_https = ((slave_instance.get('apache_custom_https', '')) % slave_parameter_dict) -%}
+[{{ slave_configuration_section_name }}]
+{%   set apache_custom_http = ((slave_instance.pop('apache_custom_http', '')) % slave_parameter_dict) -%}
+{%   set apache_custom_https = ((slave_instance.pop('apache_custom_https', '')) % slave_parameter_dict) -%}
 apache_custom_http = {{ dumps(apache_custom_http) }}
 apache_custom_https = {{ dumps(apache_custom_https) }}
+{%     for key, value in slave_instance.iteritems() -%}
+{{ key }} = {{ dumps(value) }}
+{%     endfor %}
 {{ '\n' }}
 
-# The slave use cache
-{%     if 'enable_cache' in slave_instance and 'url' in slave_instance and 'domain' in slave_instance -%}
-{%       do cached_server_dict.__setitem__(slave_instance.get('domain'), slave_instance.get('url')) -%}
-{%     endif -%}
-
 # Publish information
 {%     do slave_publish_dict.update(**{'slave-reference':slave_instance.get('slave_reference'), 'public-ipv4':public_ipv4, 'log-access': slave_log_access_url}) %}
 
@@ -165,13 +169,14 @@ apache_custom_https = {{ dumps(apache_custom_https) }}
 # Next line is forbidden and people who copy it will be hanged short
 {%     set enable_cache = (('' ~ slave_instance.get('enable_cache', '')).lower() in TRUE_VALUES and slave_instance.get('type', '') != 'redirect') -%}
 {%     if enable_cache -%}
-{%       do cached_server_dict.__setitem__(slave_instance.get('custom_domain'), slave_instance.get('url')) -%}
+{%       do slave_instance.__setitem__('backend_url', slave_instance.get('url')) -%}
 {%       do slave_instance.__setitem__('url', cache_access) -%}
+{%       do cached_server_dict.__setitem__(slave_reference, slave_configuration_section_name) -%}
 {%     endif -%}
 {%     do part_list.append(slave_section_title) -%}
 
 
-[{{ ('slave-instance-%s-configuration' % slave_reference) }}]
+[{{ slave_configuration_section_name }}]
 {%     for key, value in slave_instance.iteritems() -%}
 {{ key }} = {{ dumps(value) }}
 {%     endfor %}
@@ -181,10 +186,10 @@ apache_custom_https = {{ dumps(apache_custom_https) }}
 < = jinja2-template-base
 template = {{ template_default_slave_configuration }}
 filename = {{ '%s.conf' % slave_reference }}
-
+rendered = {{ apache_configuration_directory }}/${:filename}
 extensions = jinja2.ext.do
 extra-context =
-    section slave_parameter {{ 'slave-instance-%s-configuration' % slave_reference }}
+    section slave_parameter {{ slave_configuration_section_name }}
     raw https_port {{ https_port }}
     raw http_port {{ http_port }}
 {{ '\n' }}
@@ -193,6 +198,25 @@ extra-context =
 
 {%   endif -%}
 
+############################
+### Prepare virtualhost for slaves using cache
+
+{% for slave_reference, slave_configuration_section_name in cached_server_dict.iteritems() %}
+{%   set cached_slave_configuration_section_title = '%s-cached-virtualhost' % slave_reference %}
+{%   do part_list.append(cached_slave_configuration_section_title) -%}
+[{{ cached_slave_configuration_section_title }}]
+< = jinja2-template-base
+template = {{ template_cached_slave_configuration }}
+filename = {{ '%s.conf' % slave_reference }}
+rendered = {{ apache_cached_configuration_directory }}/${:filename}
+extensions = jinja2.ext.do
+extra-context =
+    section slave_parameter {{ slave_configuration_section_name }}
+    raw cached_port {{ cached_port }}
+{{ '\n' }}
+{% endfor %}
+
+
 ############################
 #### Publish Slave Information
 
@@ -239,18 +263,6 @@ slave-instance-information-list = {{ json_module.dumps(slave_instance_informatio
 {% endif -%}
 monitor_url = {{ monitor_url }}
 
-{% do part_list.append('cached-rewrite-rules') -%}
-[cached-rewrite-rules]
-< = jinja2-template-base
-template = {{ template_rewrite_cached }}
-rendered = {{ rewrite_cached_configuration }}
-extra-context =
-    import json_module json
-    key server_dict rewrite-rules:rules
-
-[rewrite-rules]
-rules = {{ dumps(cached_server_dict) }}
-
 [buildout]
 parts +=
 {% for part in part_list -%}

software/apache-frontend/templates/apache_cached.conf.in

@@ -116,18 +116,10 @@ SSLHonorCipherOrder on
 SSLProxyCheckPeerCN off
 SSLProxyCheckPeerExpire off
 
-# Only accept generic (i.e not Zope) backends on http
-<VirtualHost *:{{ cached_port }}>
-  SSLProxyEngine on
-  # Rewrite part
-  ProxyPreserveHost On
-  ProxyTimeout 600
-  RewriteEngine On
-
-  RewriteMap apachemapcached txt:{{ apachecachedmap_path }}
-  RewriteCond ${apachemapcached:%{SERVER_NAME}} >""
-  RewriteRule ^/(.*)$ ${apachemapcached:%{SERVER_NAME}}/$1 [L,P]
-
-  # If nothing exist : put a nice error
-  ErrorDocument 404 /notfound.html
-</VirtualHost>
+NameVirtualHost *:{{ cached_port }}
+
+include {{ slave_with_cache_configuration_directory }}/*.conf
+
+ErrorDocument 404 /notfound.html
+RewriteRule (.*) /notfound.html [R=404,L]
+

software/apache-frontend/templates/cached-virtualhost.conf.in

+{% set TRUE_VALUES = ['y', 'yes', '1', 'true'] -%}
+# Only accept generic (i.e not Zope) backends on http
+<VirtualHost *:{{ cached_port }}>
+  ServerName {{ slave_parameter.get('custom_domain') }}
+  SSLProxyEngine on
+
+{% set ssl_proxy_verify = ('' ~ slave_parameter.get('ssl-proxy-verify', '')).lower() in TRUE_VALUES -%}
+{% if ssl_proxy_verify -%}
+{%   if 'ssl_proxy_ca_crt' in slave_parameter -%}
+  SSLProxyCACertificateFile {{ slave_parameter.get('path_to_ssl_proxy_ca_crt', '') }}
+{%   endif %}
+  SSLProxyVerify require
+  #SSLProxyCheckPeerCN on
+  SSLProxyCheckPeerExpire on
+{% endif %}
+  # Rewrite part
+  ProxyPreserveHost On
+  ProxyTimeout 600
+  RewriteEngine On
+
+  RewriteRule ^/(.*)$ {{ slave_parameter.get('backend_url', '') }}/$1 [L,P]
+
+</VirtualHost>

software/apache-frontend/templates/default-virtualhost.conf.in

 {% set TRUE_VALUES = ['y', 'yes', '1', 'true'] -%}
 {% set disable_no_cache_header = ('' ~ slave_parameter.get('disable-no-cache-request', '')).lower() in TRUE_VALUES -%}
+{% set disable_via_header = ('' ~ slave_parameter.get('disable-via-header', '')).lower() in TRUE_VALUES -%}
 {%- set prefer_gzip = ('' ~ slave_parameter.get('prefer-gzip-encoding-to-backend', '')).lower() in TRUE_VALUES -%}
  
 <VirtualHost *:{{ https_port }}>
@@ -15,6 +16,15 @@
 
   SSLEngine on
   SSLProxyEngine on
+{% set ssl_proxy_verify = ('' ~ slave_parameter.get('ssl-proxy-verify', '')).lower() in TRUE_VALUES -%}
+{% if ssl_proxy_verify -%}
+{%   if 'ssl_proxy_ca_crt' in slave_parameter -%}
+  SSLProxyCACertificateFile {{ slave_parameter.get('path_to_ssl_proxy_ca_crt', '') }}
+{%   endif %}
+  SSLProxyVerify require
+  #SSLProxyCheckPeerCN on
+  SSLProxyCheckPeerExpire on
+{% endif %}
   SSLProtocol all -SSLv2 -SSLv3
   SSLCipherSuite ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:HIGH:!aNULL:!MD5
   SSLHonorCipherOrder on
@@ -40,6 +50,9 @@
   # Rewrite part
   ProxyPreserveHost On
   ProxyTimeout 600
+{% if disable_via_header %}
+  Header unset Via
+{% endif -%}
   RewriteEngine On
 
 {% if disable_no_cache_header %}
@@ -88,9 +101,21 @@
 {% endif %}
 
   SSLProxyEngine on
+{% set ssl_proxy_verify = ('' ~ slave_parameter.get('ssl-proxy-verify', '')).lower() in TRUE_VALUES -%}
+{% if ssl_proxy_verify -%}
+{%   if 'ssl_proxy_ca_crt' in slave_parameter -%}
+  SSLProxyCACertificateFile {{ slave_parameter.get('path_to_ssl_proxy_ca_crt', '') }}
+{%   endif %}
+  SSLProxyVerify require
+  #SSLProxyCheckPeerCN on
+  SSLProxyCheckPeerExpire on
+{% endif %}
   # Rewrite part
   ProxyPreserveHost On
   ProxyTimeout 600
+{% if disable_via_header %}
+  Header unset Via
+{% endif -%}
   RewriteEngine On
 
   # One Slave two logs