Merge branch 'varnish' into erp5

setup.py

@@ -95,6 +95,7 @@ setup(name=name,
           'tidstorage = slapos.recipe.tidstorage:Recipe',
           'erp5.update = slapos.recipe.erp5_update:Recipe',
           'erp5.test = slapos.recipe.erp5_test:Recipe',
+          'generic.varnish = slapos.recipe.generic_varnish:Recipe',
         ],
         'slapos.recipe.nosqltestbed.plugin': [
           'kumo = slapos.recipe.nosqltestbed.kumo:KumoTestBed',

slapos/recipe/generic_varnish/__init__.py

+##############################################################################
+#
+# Copyright (c) 2012 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+import os
+import re
+from slapos.recipe.librecipe import GenericSlapRecipe
+
+class Recipe(GenericSlapRecipe):
+  """
+    Instantiate varnish daemon
+
+    TODO:
+      - use varnish3.x and replace .vcl for it
+  """
+  def _install(self):
+    ip = self.options['ip']
+    backend_url = self.parameter_dict['tidstorage-url']
+    backend_ip, backend_port = self._getBackendServer(backend_url)
+    varnishd_manager_port = int(self.options['manager-port'])
+    varnishd_server_port = int(self.options['server-port'])
+    path_list = []
+    config = dict(
+      varnishd_binary=self.options['varnishd-binary'],
+      varnish_ip=ip,
+      varnishlog_binary=self.options['varnishlog-binary'],
+      varnishd_manager_port=varnishd_manager_port,
+      varnishd_server_port=varnishd_server_port,
+      varnishd_pid_file=self.options['pid-file'],
+      varnish_instance_name=self.options['varnish-instance-name'],
+      varnish_data=self.options['varnish-data'],
+      shell_path=self.options['shell-path'],
+      vcl_file=self.options['vcl-file'],
+      backend_ip = backend_ip,
+      backend_port = backend_port,
+      backend_server = "[%s]" % backend_ip,
+    )
+
+    path_list.append(self.createExecutable(self.options['varnishd-wrapper'],
+      self.substituteTemplate(self.getTemplateFilename('varnishd.in'),
+        config)))
+    path_list.append(self.createExecutable(self.options['varnishlog-wrapper'],
+      self.substituteTemplate(self.getTemplateFilename('varnishlog.in'),
+        config)))
+    path_list.append(self.createFile(self.options['vcl-file'],
+      self.substituteTemplate(self.getTemplateFilename('default.vcl.in'),
+        config)))
+    return path_list
+
+  def _getBackendServer(self, url):
+    r = re.compile('\/\/\[(.*)\]:(\d*)')
+    result = r.search(url)
+    ip = result.groups()[0]
+    port = result.groups()[1]
+    return (ip, port)

slapos/recipe/generic_varnish/template/default.vcl.in

+#This is a basic VCL configuration file for varnish.  See the vcl(7)
+#man page for details on VCL syntax and semantics.
+#
+#Default backend definition.  Set this to point to your content
+#server.
+#
+backend default {
+  .host = "%(backend_ip)s";
+  .port = "%(backend_port)s";
+  .probe = {
+           .timeout = 30s;
+           .interval = 5s;
+           .window = 4;
+           .threshold = 3;
+           .request =
+             "OPTIONS /erp5/getId HTTP/1.1"
+             "Host: %(backend_server)s:%(backend_port)s"
+             "Accept-Encoding: identity"
+             "Connection: close"
+             "User-Agent: Varnish";
+  }
+}
+#
+#Below is a commented-out copy of the default VCL logic.  If you
+#redefine any of these subroutines, the built-in logic will be
+#appended to your code.
+#
+
+sub vcl_recv {
+  if (req.request != "GET" &&
+    req.request != "HEAD" &&
+    req.request != "PUT" &&
+    req.request != "POST" &&
+    req.request != "TRACE" &&
+    req.request != "OPTIONS" &&
+    req.request != "PURGE" &&
+    req.request != "DELETE") {
+      /* Non-RFC2616 or CONNECT which is weird. */
+      pipe;
+  }
+  if (req.request != "GET" && req.request != "HEAD" && req.request != "PURGE") {
+      /* We only deal with GET and HEAD by default */
+      pass;
+  }
+  remove req.http.Cookie;
+  remove req.http.Set-Cookie;
+  if (req.http.Accept-Encoding) {
+    if (req.http.Accept-Encoding ~ "gzip") {
+      set req.http.Accept-Encoding = "gzip";
+    } elsif (req.http.Accept-Encoding ~ "deflate") {
+      set req.http.Accept-Encoding = "deflate";
+    } else {
+      # unkown algorithm
+      remove req.http.Accept-Encoding;
+    }
+  }
+  # Force deflate
+  remove req.http.Accept-Encoding;
+  # We do not care about Accept-Language, this is url controlled
+  remove req.http.Accept-Language;
+  #if (req.request == "PURGE") {
+  #  if (!client.ip ~ purge) {
+  #    error 405 "Not allowed.";
+  #  }
+  #  purge_url(req.url);
+  #  error 200 "HASHPURGED";
+  #  unset req.http.x;
+  #}
+  set req.grace = 30d;
+  lookup;
+}
+
+sub vcl_hash {
+    set req.hash += req.url;
+    hash;
+}
+
+sub vcl_hit {
+   #if (req.request == "PURGE" && client.ip ~ purge) {
+   #  set obj.ttl = 0s;
+   #  error 200 "Purged.";
+   #}
+
+   #if (client.ip ~ purge){
+   #  # Force refresh from localhost
+   #  set obj.ttl = 0s;
+   #  return (restart);
+   #}
+   # According Vary Header do not return those headers
+   remove req.http.Accept-Language;
+   remove req.http.Accept-Encoding;
+   remove req.http.Cookie;
+   deliver;
+}
+
+sub vcl_miss {
+    fetch;
+}
+
+sub vcl_fetch {
+    /* Never send request to backend even if client ask refreshed content */
+    if (obj.cacheable) {
+       /* Setup grace period for 30days for all cacheable contents */
+      #set req.grace = 30d;
+      set obj.grace = 30d;
+      }
+   deliver;
+   }
+
+
+sub vcl_deliver {
+   if (obj.hits > 0) {
+     set resp.http.X-Cache = obj.hits;
+   } else {
+     set resp.http.X-Cache = "MISS";
+   }
+   #if (obj.hash) {
+   #  set resp.http.X-Hash = obj.hash;
+   #} else {
+   #  set resp.http.X-Hash = "No hash";
+   #}
+
+   deliver;
+}

slapos/recipe/generic_varnish/template/varnishd.in

+#!%(shell_path)s
+
+DAEMON_OPTS="-F \
+             -a %(varnish_ip)s:%(varnishd_server_port)s \
+             -T %(varnish_ip)s:%(varnishd_manager_port)s \
+             -n %(varnish_instance_name)s \
+             -f %(vcl_file)s \
+             -s file,%(varnish_data)s/varnish_storage.bin,1G"
+
+PIDFILE=%(varnishd_pid_file)s
+# exporting PATH here so that we will pass the PATH variable to the subprocess
+export PATH
+output=$(/bin/tempfile -s.varnish)
+exec %(varnishd_binary)s -P ${PIDFILE} ${DAEMON_OPTS} > ${output} 2>&1
+

slapos/recipe/generic_varnish/template/varnishlog.in

+#!%(shell_path)s
+
+DAEMON_OPTS="-a %(varnish_ip)s:%(varnishd_server_port)s \
+             -n %(varnish_instance_name)s"
+
+# exporting PATH here so that it will pass the PATH variable to the subprocess
+export PATH
+%(varnishlog_binary)s ${DAEMON_OPTS}

slapos/recipe/generic_varnish/template/varnishlogd.in

+#!%(shell_path)s
+
+DAEMON_OPTS="-F \
+             -a %(varnish_ip)s:%(varnishd_server_port)s \
+             -T %(varnish_ip)s:%(varnishd_manager_port)s \
+             -n %(varnish_instance_name)s \
+             -f %(vcl_file)s \
+             -s file,%(varnish_data)s/varnish_storage.bin,1G"
+
+PIDFILE=%(varnishd_pid_file)s
+# exporting PATH here so that it will pass the PATH variable to the subprocess
+export PATH
+
+# If unset, or set to "0" or "no", exit
+if [ -z "${VARNISHLOG_ENABLED}" ] || \
+   [ "${VARNISHLOG_ENABLED}" = "0" ] || \
+   [ "${VARNISHLOG_ENABLED}" = "no" ]; then
+  exit 0;
+fi
+
+output=$(/bin/tempfile -s.varnish)
+exec %(varnishlog_binary)s ${DAEMON_OPTS} > ${output} 2>&1

software/erp5/instance-erp5-production.cfg

@@ -74,6 +74,14 @@ config-kumofs-url = $${request-kumofs:connection-url}
 software-type = tidstorage
 sla-computer_guid = $${slap-parameter:tidstorage-computer-guid}
 
+[request-varnish]
+<=request-common
+name = Varnish
+config = tidstorage-url
+config-tidstorage-url = $${request-tidstorage:connection-url-login}
+software-type = varnish
+sla-computer_guid = $${slap-parameter:varnish-computer-guid}
+
 [slap-parameter]
 # Default value if no computer_guid is specified for each type
 sphinx-computer-guid = $${slap-connection:computer-id}
@@ -82,6 +90,7 @@ cloudooo-computer-guid = $${slap-connection:computer-id}
 memcached-computer-guid = $${slap-connection:computer-id}
 kumofs-computer-guid = $${slap-connection:computer-id}
 tidstorage-computer-guid = $${slap-connection:computer-id}
+varnish-computer-guid = $${slap-connection:computer-id}
 cloudooo-json =
 
 # rest of parts are candidates for some generic stuff

software/erp5/instance-varnish.cfg

+[buildout]
+parts =
+  publish-varnish-connection-information
+  varnish-instance
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+[publish-varnish-connection-information]
+recipe = slapos.cookbook:publishurl
+url = http://$${varnish-instance:ip}:$${varnish-instance:server-port}/
+
+[varnish-instance]
+recipe = slapos.cookbook:generic.varnish
+
+# Network options
+ip = $${slap-network-information:local-ipv4}
+server-port = 6001
+manager-port = 6002
+
+# Paths: Running wrappers
+varnishd-wrapper = $${basedirectory:services}/varnishd
+varnishlog-wrapper = $${rootdirectory:bin}/varnishlog
+
+# Binary information
+varnishd-binary = ${varnish:location}/sbin/varnishd
+varnishlog-binary = ${varnish:location}/bin/varnishlog
+shell-path = ${dash:location}/bin/dash
+# Configuration by VCL
+vcl-file = $${rootdirectory:etc}/default.vcl
+pid-file = $${basedirectory:run}/varnishd.pid
+varnish-data = $${directory:varnish-data}
+# this will pass at -n option
+varnish-instance-name = $${directory:varnish-instance}
+
+[basedirectory]
+recipe = slapos.cookbook:mkdirectory
+services = $${rootdirectory:etc}/run
+run = $${rootdirectory:var}/run
+backup = $${rootdirectory:srv}/backup
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+varnish-data = $${rootdirectory:srv}/varnish
+varnish-instance = $${directory:varnish-data}/instance
+
+[rootdirectory]
+recipe = slapos.cookbook:mkdirectory
+etc = $${buildout:directory}/etc
+var = $${buildout:directory}/var
+srv = $${buildout:directory}/srv
+bin = $${buildout:directory}/bin
+

software/erp5/instance.cfg

@@ -33,6 +33,7 @@ zope = ${template-zope:output}
 mariadb = ${template-mariadb:output}
 sphinx = ${template-sphinx:output}
 tidstorage = $${dynamic-template-tidstorage:output}
+varnish = ${template-varnish:output}
 
 [slap-connection]
 # part to migrate to new - separated words

software/erp5/software.cfg

@@ -87,7 +87,7 @@ mode = 0644
 [template]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg
-md5sum = 961d404f5726fce7c4d8b34d7e120077
+md5sum = f3443d071d8d7330b3453583e096fef0
 output = ${buildout:directory}/template.cfg
 mode = 0644
 
@@ -108,7 +108,7 @@ mode = 0644
 [template-erp5-production]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-erp5-production.cfg
-md5sum = ae697a5c82f80388c674217d5e6f61a0
+md5sum = 6bf9242195b6fe4e5fb5d080cefef255
 output = ${buildout:directory}/template-erp5-production.cfg
 mode = 0644
 
@@ -140,6 +140,13 @@ md5sum = 396053ffe2904ad9933a42fe2f8cfcde
 output = ${buildout:directory}/template-snippet-backend.cfg
 mode = 0644
 
+[template-varnish]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-varnish.cfg
+md5sum = f708845296a1e9ea1c7fa699ace67a30
+output = ${buildout:directory}/template-varnish.cfg
+mode = 0644
+
 [validator]
 # Default json schema for instance parameters.
 recipe = slapos.recipe.template