erp5 stack: improve http monitor cfg files

- Use latest slapos.toolbox (0.43.0)
- Remove the use of slapos.recipe.template
- Remove monitor-templates/monitor-check-cache-hit.in
- Rename template-check-cache-hit to template-check-web-page-http-cache-hit

stack/erp5/buildout.cfg

@@ -153,19 +153,9 @@ context =
     ${:extra-context}
 
 [template-http-monitor]
-recipe = slapos.recipe.template
-filename = instance-http-monitor.cfg.in
-url = ${:_profile_base_location_}/${:filename}
-md5sum = 7fb20468f1825a82951a222df9c8b045
-output = ${buildout:directory}/template-http-monitor.cfg.in
-
-[template-monitor-check-cache-hit]
 < = download-base
-url = ${:_profile_base_location_}/monitor-templates/monitor-check-cache-hit.in
-download-only = true
-md5sum = 8e9ea2bec2fff4bd98c074cb69cf8592
-filename = monitor-check-cache-hit.in
-mode = 0644
+filename = instance-http-monitor.cfg.in
+md5sum = 16d1fc3cf26c3066ae612ded0ec64cd2
 
 [template-mariadb]
 < = download-base
@@ -228,7 +218,7 @@ md5sum = e40e2e39f4941c6372f4357e8589a5cf
 # XXX: "template.cfg" is hardcoded in instanciation recipe
 filename = template.cfg
 template = ${:_profile_base_location_}/instance.cfg.in
-md5sum = c49460630f8315adf7af03d2500c6ea7
+md5sum = 904a91e42c9db29be86978d972d05838
 extra-context =
     key mariadb_link_binary template-mariadb:link-binary
     key zope_link_binary template-zope:link-binary
@@ -278,7 +268,7 @@ extra-context =
     key template_create_erp5_site_real template-create-erp5-site-real:target
     key template_erp5 template-erp5:target
     key template_haproxy_cfg template-haproxy-cfg:target
-    key template_http_monitor template-http-monitor:output
+    key template_http_monitor template-http-monitor:target
     key template_kumofs template-kumofs:target
     key template_logrotate_base template-logrotate-base:rendered
     key template_mariadb template-mariadb:target
@@ -604,9 +594,10 @@ recipe = zc.recipe.egg
 eggs =
   ${lxml-python:egg}
   slapos.toolbox
-  slapos.toolbox[zodbpack]
+  slapos.toolbox[check_web_page_http_cache_hit,zodbpack]
 
 scripts =
+  check-web-page-http-cache-hit
   is-local-tcp-port-opened
   killpidfromfile
   onetimedownload
@@ -713,7 +704,7 @@ qrcode = 5.0.1
 restkit = 4.2.2
 rtjp-eventlet = 0.3.2
 slapos.recipe.template = 2.5
-slapos.toolbox = 0.41.0
+slapos.toolbox = 0.43.0
 smmap = 0.8.2
 socketpool = 0.5.3
 spyne = 2.11.0

stack/erp5/instance-http-monitor.cfg.in

@@ -17,35 +17,34 @@ parts =
   cgi-httpd-graceful-wrapper
   monitor-promise
   monitor-instance-log-access
-  monitor-check-cache-hit
+  monitor-check-web-page-http-cache-hit
   publish-http-monitor
 
-extends = ${monitor-template:output}
+extends = {{ parameter_dict["template-monitor"] }}
 
 eggs-directory = {{ eggs_directory }}
 develop-eggs-directory = {{ develop_eggs_directory }}
 
-[monitor-check-cache-hit]
-recipe = slapos.recipe.template:jinja2
-template = ${template-monitor-check-cache-hit:location}/${template-monitor-check-cache-hit:filename}
-rendered = $${monitor-directory:monitor-custom-scripts}/check-cache-hit.py
-mode = 700
-context =
-  raw config_cfg $${buildout:directory}/$${public:filename}
-  raw python_executable ${buildout:bin-directory}/python2.7
+[monitor-check-web-page-http-cache-hit]
+recipe = slapos.cookbook:wrapper
+command-line = "{{ parameter_dict['bin-directory'] }}/check-web-page-http-cache-hit" -K "${public:filename}"
+wrapper-path = ${monitor-directory:monitor-custom-scripts}/check-web-page-http-cache-hit
 
 [public]
 recipe = slapos.cookbook:zero-knowledge.write
 filename = knowledge0.cfg
-json_url_list = ["http://www.erp5.com/"]
-json_resolve_list = ["www.erp5.com:80:5.135.149.226"]
+status-history-length = 5
+# Each one of the following 2 list keys should be defined in only one line
+# ex: resolve-list = www.example.com:80:127.0.0.1 www.perdu.com:80:10.0.0.1
+url-list =
+resolve-list =
 
 [zero-parameters]
 recipe = slapos.cookbook:zero-knowledge.read
-filename = $${public:filename}
+filename = ${public:filename}
 
 [publish-http-monitor]
 recipe = slapos.cookbook:publish.serialised
-monitor-url = $${monitor-parameters:url}
+monitor-url = ${monitor-parameters:url}
 
 {% endif %}

stack/erp5/instance.cfg.in

@@ -217,11 +217,16 @@ extra-context =
 # Must match the key id in [switch-softwaretype] which uses this section.
     raw software_type create-erp5-site
 
+[dynamic-template-http-monitor-parameters]
+template-monitor = {{ template_monitor }}
+bin-directory = {{ bin_directory }}
+
 [dynamic-template-http-monitor]
 < = jinja2-template-base
 template = {{ template_http_monitor }}
 filename = instance-http-monitor.cfg
 extra-context =
+    section parameter_dict dynamic-template-http-monitor-parameters
     raw software_type http-monitor
 
 [switch-softwaretype]

stack/erp5/monitor-templates/monitor-check-cache-hit.in

-#!{{ python_executable }}
-# -*- coding: utf-8 -*-
-
-# dependency: python2-pycurl
-
-import sys
-import pycurl
-import ConfigParser
-import os
-import json
-
-from mimetools import Message
-from cStringIO import StringIO
-from HTMLParser import HTMLParser
-
-####################
-# Init logging tools
-def debug(*args):
-  # sys.stderr.write("DEBUG: " + "\n     : ".join(" ".join((str(arg) for arg in args)).split("\n")) + "\n")
-  pass
-def log(*args):
-  sys.stderr.write(" ".join((str(arg) for arg in args)) + "\n")
-  pass
-def info(*args):
-  sys.stderr.write("INFO : " + "\n     : ".join(" ".join((str(arg) for arg in args)).split("\n")) + "\n")
-  pass
-last_warn_log = None
-def warn(*args):
-  global last_warn_log
-  last_warn_log = "WARN : " + "\n     : ".join(" ".join((str(arg) for arg in args)).split("\n")) + "\n"
-  sys.stderr.write(last_warn_log)
-last_error_log = None
-def error(*args):
-  global last_error_log
-  last_error_log = "ERROR: " + "\n     : ".join(" ".join((str(arg) for arg in args)).split("\n")) + "\n"
-  sys.stderr.write(last_error_log)
-
-#############################
-# Checking configuration file
-config_file = "{{ config_cfg }}"
-if not os.path.exists(config_file):
-  error("Your software does *NOT* embed 0-knowledge. This interface is useless in this case")
-  exit(1)
-
-##################################
-# Getting configuration parameters
-config_parser = ConfigParser.ConfigParser()
-config_parser.read(config_file)
-
-if not config_parser.has_section('public'):
-  error("Your software does not use 0-knowledge settings.")
-  exit(1)
-
-url_list = json.loads(config_parser.get("public", "json_url_list"))
-resolve_list = json.loads(config_parser.get("public", "json_resolve_list"))
-
-########################
-# Prepare & run requests
-class MyHTMLParser(HTMLParser):
-  def __init__(self, base_url):
-    HTMLParser.__init__(self)
-    self.base_url = base_url
-  def handle_starttag(self, tag, attrs):
-    def handle_url(url):
-      if not url.startswith("http"):
-        url = self.base_url + url
-      do_request(url)
-    # handle images and js scripts
-    if tag == 'img' or tag == 'script':
-      debug(tag, attrs)
-      for attr in attrs:
-        if attr[0] == 'src':
-          handle_url(attr[1])
-    # handle css and favicon
-    if tag == 'link' and \
-       any(attr[0] == "rel" and attr[1] in ("stylesheet", "shortcut icon") for attr in attrs):
-      debug(tag, attrs)
-      for attr in attrs:
-        if attr[0] == "href":
-          handle_url(attr[1])
-
-headers = {
-  "User-Agent": "test",
-  "Accept-Encoding": "gzip, deflate",
-  "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
-  "Accept-Language": "en-US,en;q=0.5",
- #   "Connection": "keep-alive"
-}
-
-parsed_url_dict = {}
-
-info("Start checking for cache hits")
-info("Parameters:\nurl_list:", url_list, "\nresolve_list:", resolve_list)
-
-def do_request(url):
-  if parsed_url_dict.get("url") is not None: return
-  parsed_url_dict[url] = True
-  log("Checking cache hit for", url)
-  c = pycurl.Curl()
-  response_headers = StringIO()
-  output = StringIO()
-  c.setopt(c.URL, url)
-  c.setopt(c.RESOLVE, resolve_list)
-  c.setopt(c.WRITEFUNCTION, output.write)
-  c.setopt(c.HEADERFUNCTION, response_headers.write)
-  c.perform() # perform a request before testing if the cache is hit
-  response_headers.truncate(0)
-  output.truncate(0)
-  c.perform()
-
-  response_headers.seek(0)
-  status = response_headers.readline().split(" ")[1]
-
-  m = Message(response_headers)
-
-  # see http://labs.omniti.com/people/mark/ats_sa/slides.html#slide-18
-  if any("[cHs" in header for header in m.getheaders('via')) or \
-     any("HIT" in header for header in m.getheaders("x-cache")):
-    debug("Cache hit found in 'Via' or 'X-Cache' headers")
-  else:
-    error("No cache hit found in 'Via' or 'X-Cache' headers\n" + response_headers.getvalue().rstrip())
-
-  if m.getheader('content-type', '').startswith('text/html'):
-    MyHTMLParser(url).feed(output.getvalue())
-
-  response_headers.close()
-  output.close()
-
-for url in url_list:
-  do_request(url)
-
-if last_error_log is not None:
-  sys.exit(1)
-info("OK");