Merge branch 'apache'

component/apache/buildout.cfg

@@ -45,6 +45,7 @@ configure-options = --prefix=${buildout:parts-directory}/${:_buildout_section_na
                     --enable-cgid
                     --enable-charset-lite
                     --enable-disk-cache
+                    --enable-mem-cache
                     --enable-echo
                     --enable-exception-hook
                     --enable-mods-shared=all
@@ -131,6 +132,7 @@ configure-options = --disable-static
                     --enable-cgid
                     --enable-charset-lite
                     --enable-disk-cache
+                    --enable-mem-cache
                     --enable-echo
                     --enable-exception-hook
                     --enable-mods-shared=all

slapos/recipe/apache_frontend/__init__.py

@@ -50,6 +50,8 @@ class Recipe(BaseSlapRecipe):
 
     # Define optional arguments
     frontend_port_number = self.parameter_dict.get("port", 4443)
+    frontend_plain_http_port_number = self.parameter_dict.get(
+        "plain_http_port", 8080)
     base_varnish_port = 26009
     slave_instance_list = self.parameter_dict.get("slave_instance_list", [])
 
@@ -111,7 +113,7 @@ class Recipe(BaseSlapRecipe):
         # rule structure.
         # So we will have one RewriteMap for normal websites, and one
         # RewriteMap for Zope Virtual Host Monster websites.
-        if slave_instance.get("zope", "").upper() in ('1', 'TRUE'):
+        if slave_instance.get("type", "").lower() in ('zope'):
           rewrite_rule_zope_list.append(rewrite_rule)
         else:
           rewrite_rule_list.append(rewrite_rule)
@@ -143,6 +145,7 @@ class Recipe(BaseSlapRecipe):
         ip_list=["[%s]" % self.getGlobalIPv6Address(),
                  self.getLocalIPv4Address()],
         port=frontend_port_number,
+        plain_http_port=frontend_plain_http_port_number,
         name=frontend_domain_name,
         rewrite_rule_list=rewrite_rule_list,
         rewrite_rule_zope_list=rewrite_rule_zope_list,
@@ -451,7 +454,8 @@ class Recipe(BaseSlapRecipe):
     self.path_list.append(wrapper)
     return stunnel_conf
 
-  def installFrontendApache(self, ip_list, port, key, certificate, name,
+  def installFrontendApache(self, ip_list, key, certificate, name,
+                            port, plain_http_port=8080, 
                             rewrite_rule_list=[], rewrite_rule_zope_list=[],
                             access_control_string=None):
     # Create htdocs, populate it with default 404 document
@@ -463,6 +467,34 @@ class Recipe(BaseSlapRecipe):
     notfound_file_content = open(notfound_template_file_location, 'r').read()
     self._writeFile(notfound_file_location, notfound_file_content)
 
+    # Create mod_ssl cache directory
+    cache_directory_location = os.path.join(self.var_directory, 'cache')
+    mod_ssl_cache_location = os.path.join(cache_directory_location,
+        'httpd_mod_ssl')
+    self._createDirectory(cache_directory_location)
+    self._createDirectory(mod_ssl_cache_location)
+
+    # Create "custom" apache configuration file if it does not exist.
+    # Note : This file won't be erased or changed when slapgrid is ran.
+    # It can be freely customized by node admin.
+    custom_apache_configuration_directory = os.path.join(
+        self.data_root_directory, 'apache-conf.d')
+    self._createDirectory(custom_apache_configuration_directory)
+    custom_apache_configuration_file_location = os.path.join(
+        custom_apache_configuration_directory, 'apache_frontend.custom.conf')
+    f = open(custom_apache_configuration_file_location, 'a')
+    f.close()
+
+    # Create backup of custom apache configuration
+    backup_path = self.createBackupDirectory('custom_apache_conf_backup')
+    backup_cron = os.path.join(self.cron_d, 'custom_apache_conf_backup')
+    open(backup_cron, 'w').write(
+        '''0 0 * * * %(rdiff_backup)s %(source)s %(destination)s'''%dict(
+          rdiff_backup=self.options['rdiff_backup_binary'],
+          source=custom_apache_configuration_directory,
+          destination=backup_path))
+    self.path_list.append(backup_cron)
+
     # Create configuration file and rewritemaps
     apachemap_name = "apachemap.txt"
     apachemapzope_name = "apachemapzope.txt"
@@ -472,9 +504,17 @@ class Recipe(BaseSlapRecipe):
     apache_conf = self._getApacheConfigurationDict(name, ip_list, port)
     apache_conf['ssl_snippet'] = self.substituteTemplate(
         self.getTemplateFilename('apache.ssl-snippet.conf.in'),
-        dict(login_certificate=certificate, login_key=key))
+        dict(login_certificate=certificate,
+            login_key=key,
+            httpd_mod_ssl_cache_directory=mod_ssl_cache_location,
+        )
+    )
 
-    apache_conf["listen"] = "\n".join(["Listen %s:%s" % (ip, port) for ip in ip_list])
+    apache_conf["listen"] = "\n".join([
+        "Listen %s:%s" % (ip, port)
+        for port in (plain_http_port, port)
+        for ip in ip_list
+    ])
 
     path = self.substituteTemplate(
         self.getTemplateFilename('apache.conf.path-protected.in'),
@@ -485,7 +525,9 @@ class Recipe(BaseSlapRecipe):
       apachemap_path=os.path.join(self.etc_directory, apachemap_name),
       apachemapzope_path=os.path.join(self.etc_directory, apachemapzope_name),
       apache_domain=name,
-      port=port,
+      https_port=port,
+      plain_http_port=plain_http_port,
+      custom_apache_conf=custom_apache_configuration_file_location,
     ))
 
     apache_conf_string = self.substituteTemplate(

slapos/recipe/apache_frontend/template/apache.conf.in

@@ -19,9 +19,6 @@ RequestHeader unset REMOTE_USER
 
 ServerTokens Prod
 
-# SSL Configuration
-%(ssl_snippet)s
-
 # Log configuration
 ErrorLog "%(error_log)s"
 LogLevel warn
@@ -31,27 +28,6 @@ CustomLog "%(access_log)s" common
 
 %(path_enable)s
 
-# Rewrite part
-ProxyVia On
-ProxyTimeout 600
-RewriteEngine On
-
-# Define the two rewritemaps : one for zope, one generic
-RewriteMap apachemapzope txt:%(apachemapzope_path)s
-RewriteMap apachemapgeneric txt:%(apachemap_path)s
-
-# First, we check if we have a zope backend server
-# If so, let's use Virtual Host Daemon rewrite
-RewriteCond ${apachemapzope:%%{SERVER_NAME}} >""
-RewriteRule ^/(.*)$ ${apachemapzope:%%{SERVER_NAME}}/VirtualHostBase/https/%%{SERVER_NAME}:%%{SERVER_PORT}/VirtualHostRoot/$1 [L,P]
-
-# If we have generic backend server, let's rewrite without virtual host daemon
-RewriteCond ${apachemapgeneric:%%{SERVER_NAME}} >""
-RewriteRule ^/(.*)$ ${apachemapgeneric:%%{SERVER_NAME}}/$1 [L,P]
-
-# If nothing exist : put a nice error
-ErrorDocument 404 /notfound.html
-
 # List of modules
 #LoadModule unixd_module modules/mod_unixd.so
 #LoadModule access_compat_module modules/mod_access_compat.so
@@ -70,14 +46,14 @@ LoadModule negotiation_module modules/mod_negotiation.so
 LoadModule rewrite_module modules/mod_rewrite.so
 LoadModule headers_module modules/mod_headers.so
 LoadModule cache_module modules/mod_cache.so
+LoadModule mem_cache_module modules/mod_mem_cache.so
 LoadModule antiloris_module modules/mod_antiloris.so
 
-CacheDefaultExpire 3600
-
 # The following directives modify normal HTTP response behavior to
 # handle known problems with browser implementations.
 BrowserMatch "Mozilla/2" nokeepalive
-BrowserMatch "MSIE 4\.0b2;" nokeepalive downgrade-1.0 force-response-1.0
+BrowserMatch ".*MSIE.*" nokeepalive ssl-unclean-shutdown \
+                        downgrade-1.0 force-response-1.0
 BrowserMatch "RealPlayer 4\.0" force-response-1.0
 BrowserMatch "Java/1\.0" force-response-1.0
 BrowserMatch "JDK/1\.0" force-response-1.0
@@ -93,3 +69,59 @@ BrowserMatch "^WebDAVFS/1.[0123]" redirect-carefully
 BrowserMatch "^gnome-vfs" redirect-carefully
 BrowserMatch "^XML Spy" redirect-carefully
 BrowserMatch "^Dreamweaver-WebDAV-SCM1" redirect-carefully
+
+# Cache directives
+CacheEnable mem /
+CacheDefaultExpire 3600
+MCacheSize 8192
+MCacheMaxObjectCount 1000
+MCacheMaxObjectSize 8192
+MCacheRemovalAlgorithm LRU
+
+# Deflate
+AddOutputFilterByType DEFLATE text/html text/plain text/xml text/css text/javascript application/x-javascript application/javascript
+BrowserMatch ^Mozilla/4 gzip-only-text/html
+BrowserMatch ^Mozilla/4\.0[678] no-gzip
+BrowserMatch \bMSIE !no-gzip !gzip-only-text/html 
+# Make sure proxies don't deliver the wrong content
+Header append Vary User-Agent
+
+# SSL Configuration
+%(ssl_snippet)s
+
+# Dummy virtualhost redirecting to https. Note: will work only if https listens
+# on standard port (443)
+<VirtualHost *:%(plain_http_port)s>
+  RewriteEngine On
+  # Not using HTTPS? Ask that guy over there.
+  RewriteRule ^/(.*)$ https://%%{SERVER_NAME}%%{REQUEST_URI}
+</VirtualHost>
+
+<VirtualHost *:%(https_port)s>
+  SSLEngine on
+  SSLProxyEngine on
+  # Rewrite part
+  ProxyVia On
+  ProxyTimeout 600
+  RewriteEngine On
+
+  # Define the two rewritemaps : one for zope, one generic
+  RewriteMap apachemapzope txt:%(apachemapzope_path)s
+  RewriteMap apachemapgeneric txt:%(apachemap_path)s
+
+  # First, we check if we have a zope backend server
+  # If so, let's use Virtual Host Daemon rewrite
+  RewriteCond ${apachemapzope:%%{SERVER_NAME}} >""
+  RewriteRule ^/(.*)$ ${apachemapzope:%%{SERVER_NAME}}/VirtualHostBase/https/%%{SERVER_NAME}:%%{SERVER_PORT}/VirtualHostRoot/$1 [L,P]
+
+  # If we have generic backend server, let's rewrite without virtual host daemon
+  RewriteCond ${apachemapgeneric:%%{SERVER_NAME}} >""
+  RewriteRule ^/(.*)$ ${apachemapgeneric:%%{SERVER_NAME}}/$1 [L,P]
+
+  # If nothing exist : put a nice error
+  ErrorDocument 404 /notfound.html
+</VirtualHost>
+
+# Include configuration file not operated by slapos. This file won't be erased
+# or changed when slapgrid is ran. It can be freely customized by node admin.
+Include %(custom_apache_conf)s

slapos/recipe/apache_frontend/template/apache.ssl-snippet.conf.in

-SSLEngine on
-SSLProxyEngine on
 SSLCertificateFile %(login_certificate)s
 SSLCertificateKeyFile %(login_key)s
 SSLRandomSeed startup builtin
 SSLRandomSeed connect builtin
+SSLSessionCache shmcb:/%(httpd_mod_ssl_cache_directory)s/ssl_scache(512000)
+SSLSessionCacheTimeout  300
+SSLRandomSeed startup /dev/urandom 256
+SSLRandomSeed connect builtin
+SSLProtocol -ALL +SSLv3 +TLSv1
+SSLHonorCipherOrder On
+SSLCipherSuite RC4-SHA:HIGH:!ADH
+<FilesMatch "\.(cgi|shtml|phtml|php)$">
+      SSLOptions +StdEnvVars
+</FilesMatch>
+# Accept proxy to sites using self-signed SSL certificates
 SSLProxyCheckPeerCN off
-SSLProxyCheckPeerExpire off
+SSLProxyCheckPeerExpire off
\ No newline at end of file

software/apache-frontend/README.apache_frontend.txt

@@ -13,28 +13,28 @@ How to use
 ==========
 
 First, you will need to request a "master" instance of Apache Frontend with
-"domain" parameter, like : 
-<?xml version='1.0' encoding='utf-8'?>
-<instance>
- <parameter id="domain">moulefrite.com</parameter>
- <parameter id="port">443</parameter>
-</instance>
+"domain" parameter, like::
+  <?xml version='1.0' encoding='utf-8'?>
+  <instance>
+   <parameter id="domain">moulefrite.org</parameter>
+   <parameter id="port">443</parameter>
+  </instance>
 
 Then, it is possible to request many slave instances
 (currently only from slapconsole, UI doesn't work yet)
-of Apache Frontend, like : 
-instance = request(
-       software_release=apache_frontend,
-       partition_reference='frontend2',
-       shared=True,
-       partition_parameter_kw={"url":"https://[1:2:3:4]:1234/someresource"}
-     )
+of Apache Frontend, like::
+  instance = request(
+    software_release=apache_frontend,
+    partition_reference='frontend2',
+    shared=True,
+    partition_parameter_kw={"url":"https://[1:2:3:4]:1234/someresource"}
+  )
 Those slave instances will be redirected to the "master" instance,
 and you will see on the "master" instance the associated RewriteRules of
 all slave instances.
 
-Finally, the slave instance will be accessible from :
-https://someidentifier.moulefrite.com.
+Finally, the slave instance will be accessible from:
+https://someidentifier.moulefrite.org.
 
 Instance Parameters
 ===================
@@ -44,16 +44,21 @@ Master Instance Parameters
 
 domain
 ~~~~~~
-name of the domain to be used (example: mydomain.com). Subdomains of this domain will be used for the slave instances (example: instance12345.mydomain.com). It is then recommended to add a wildcard in DNS for the subdomains of the chosen domain like::
+name of the domain to be used (example: mydomain.com). Subdomains of this
+domain will be used for the slave instances (example:
+instance12345.mydomain.com). It is then recommended to add a wildcard in DNS
+for the subdomains of the chosen domain like::
   *.mydomain.com. IN A 123.123.123.123
 Using the IP given by the Master Instance.
-
 "domain" is a mandatory Parameter.
 
 port
 ~~~~
-Port used by Apache. Optional parameter, defaults to 443.
+Port used by Apache. Optional parameter, defaults to 4443.
 
+plain_http_port
+Port used by apache to serve plain http (only used to redirect to https).
+Optional parameter, defaults to 8080.
 
 Slave Instance Parameters
 -------------------------
@@ -62,24 +67,53 @@ url
 ~~~
 url of backend to use.
 "url" is a mandatory parameter.
-Example : http://mybackend.com/myresource
+Example: http://mybackend.com/myresource
 
 cache
 ~~~~~
 Specify if slave instance should use a varnish / stunnel to connect to backend.
-Possible values : "true", "false".
+Possible values: "true", "false".
 "cache" is an optional parameter. Defaults to "false". 
-Example : true
+Example: true
 
-zope
+type
 ~~~~
-Specify if slave instance will redirect to a zope backend. If specified, Apache RewriteRule will use Zope's Virtual Host Daemon.
-Possible values : "true", "false".
-"zope" is an optional parameter. Defaults to "false". 
-Example : true
+Specify if slave instance will redirect to a zope backend. If specified, Apache
+RewriteRule will use Zope's Virtual Host Daemon.
+Possible values: "zope", "default".
+"type" is an optional parameter. Defaults to "default". 
+Example: zope
 
 custom_domain
 ~~~~~~~~~~~~~
 Domain name to use as frontend. The frontend will be accessible from this domain.
-"custom_domain" is an optional parameter. Defaults to [instancereference].[masterdomain]. 
-Example : www.mycustomdomain.com
+"custom_domain" is an optional parameter. Defaults to
+[instancereference].[masterdomain]. 
+Example: www.mycustomdomain.com
+
+
+Advanced example
+================
+
+Request slave frontend instance using a Zope backend, with Varnish activated,
+listening to a custom domain::
+  instance = request(
+    software_release=apache_frontend,
+    partition_reference='frontend2',
+    shared=True,
+    partition_parameter_kw={
+        "url":"https://[1:2:3:4]:1234/someresource",
+        "cache":"true",
+        "type":"zope",
+        "custom_domain":"mycustomdomain.com",
+    }
+  )
+
+Notes
+=====
+
+It is not possible with slapos to listen to port <= 1024, because process are
+not run as root. It is a good idea then to go on the node where the instance is
+and set some iptables rules like (if using default ports)::
+  iptables -t nat -A PREROUTING -p tcp -d {public ip} --dport 443 -j DNAT --to-destination {listening ip}:4443
+  iptables -t nat -A PREROUTING -p tcp -d {public_ip} --dport 80 -j DNAT --to-destination {listening ip}:8080