Add monitor2 to kvm software release

software/kvm/common.cfg

@@ -16,7 +16,7 @@ extends =
   ../../stack/slapos.cfg
   ../../stack/nodejs.cfg
   ../../stack/resilient/buildout.cfg
-  ../../stack/monitor/buildout.cfg
+  ../../stack/monitor2/buildout.cfg
 
 # stacks are listed from most generic to most specific,
 # to avoid versioning issues
@@ -90,7 +90,7 @@ command =
 [template]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg.in
-md5sum = ac94fdcf8e3db4bdb2dff4478426595d
+md5sum = c597309c00b657db92f8c43e733b1763
 output = ${buildout:directory}/template.cfg
 mode = 0644
 
@@ -98,7 +98,7 @@ mode = 0644
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/instance-kvm.cfg.jinja2
 mode = 644
-md5sum = e72f42d880877a841e87908566c28610
+md5sum = 38ebf7e22e674b9cb579592755b32e6b
 download-only = true
 on-update = true
 
@@ -106,7 +106,7 @@ on-update = true
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/instance-kvm-cluster.cfg.jinja2.in
 mode = 644
-md5sum = 6e81c08669e164b852bd8d062c620de2
+md5sum = 606e5475a461f5e6d93b3414cbc8ccc2
 download-only = true
 on-update = true
 
@@ -114,7 +114,7 @@ on-update = true
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/instance-kvm-resilient.cfg.jinja2
 mode = 644
-md5sum = 7564bfbb74e6557e1041e9d6d1bc5d14
+md5sum = 6c55729e6048ad9936c7bfd0671bf07b
 download-only = true
 on-update = true
 
@@ -127,11 +127,12 @@ download-only = true
 on-update = true
 
 [template-kvm-import]
-recipe = slapos.recipe.template
+recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/instance-kvm-import.cfg.in
-md5sum = 6835c9309ff4bf4a0efd1850e6c66b24
-output = ${buildout:directory}/template-kvm-import.cfg
+md5sum = 3177381b65b4b95ba29190a6ac03b771
 mode = 0644
+download-only = true
+on-update = true
 
 [template-kvm-import-script]
 recipe = hexagonit.recipe.download
@@ -145,7 +146,7 @@ mode = 0755
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/instance-kvm-export.cfg.jinja2
 mode = 644
-md5sum = c9f13c1f481ed08c75089aef1d3c6981
+md5sum = ff281bf8a8905632b32254622db105b7
 download-only = true
 on-update = true
 

software/kvm/instance-kvm-cluster.cfg.jinja2.in

@@ -8,6 +8,7 @@
 {% set slave_frontend_iguid = slave_frontend_dict.get('instance-guid', '') -%}
 {% set kvm_instance_dict = {} -%}
 {% set kvm_hostname_list = [] -%}
+{% set monitor_url_list = [] -%}
 
 [request-common]
 recipe = slapos.cookbook:request
@@ -71,6 +72,9 @@ config-httpd-port = {{ dumps(kvm_parameter_dict.get('httpd-port', 8081)) }}
 config-data-to-vm = {{ dumps(kvm_parameter_dict.get('data-to-vm', '')) }}
 {% endif -%}
 config-enable-monitor = {{ dumps(kvm_parameter_dict.get('enable-monitor', True)) }}
+config-monitor-cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', '') }}
+config-monitor-username = ${monitor-htpasswd:username}
+config-monitor-password = ${monitor-htpasswd:passwd}
 
 # Enable simple http server on ipv6 so all VMs will access it
 config-document-host = ${apache-conf:ip}
@@ -86,8 +90,9 @@ sla-fw_rejected_sources = {{ rejected_source_list | join(' ') }}
 sla-fw_restricted_access = {{ dumps(slapparameter_dict.get('fw-restricted-access', 'off')) }}
 
 return = 
-  backend-url
   url
+  backend-url
+  monitor-base-url
 {% if str(use_nat).lower() == 'true' -%}
 {%   for port in nat_rules_list -%}
 {{ '  ' }}nat-rule-url-{{ port }}
@@ -99,6 +104,9 @@ return =
 {% do publish_dict.__setitem__('lan-' ~ instance_name, '${' ~ section ~ ':connection-tap-ipv4}') -%}
 {% do kvm_hostname_list.append(instance_name ~ ' ' ~ '${' ~ section ~ ':connection-tap-ipv4}') -%}
 {% endif -%}
+{% if str(kvm_parameter_dict.get('enable-monitor', 'True')).lower() == 'true' -%}
+{%  do monitor_url_list.append('${' ~ section ~ ':connection-monitor-base-url}') -%}
+{% endif -%}
 {% do publish_dict.__setitem__(instance_name ~ '-backend-url', '${' ~ section ~ ':connection-backend-url}') -%}
 {% do publish_dict.__setitem__(instance_name ~ '-url', '${' ~ section ~ ':connection-url}') -%}
 {% do kvm_instance_dict.__setitem__(instance_name, (use_nat, nat_rules_list)) -%}
@@ -206,14 +214,41 @@ mode = {{ mode }}
 {{ writefile('cluster-data-content', '${directory:webroot}/${hash-code:passwd}/data', slapparameter_dict.get('cluster-data', ''), '700') }}
 {% endif -%}
 
+[monitor-htpasswd]
+recipe = slapos.cookbook:generate.password
+storage-path = ${directory:etc}/.monitor_user
+bytes = 8
+username = admin
+
+[monitor-instance-parameter]
+monitor-httpd-port = 8060
+monitor-title = KVM Cluster Main Instance
+cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', '') }}
+username = ${monitor-htpasswd:username}
+password = ${monitor-htpasswd:passwd}
+
+[monitor-conf-parameters]
+monitor-url-list +=
+{% for url in monitor_url_list -%}
+{{ '  ' ~ url }}
+{% endfor %}
+private-path-list += 
+  ${directory:webroot}/
+
 [publish]
 recipe = slapos.cookbook:publish
 {% for name, value in publish_dict.items() -%}
 {{   name }} = {{ value }}
 {% endfor %}
+{% set monitor_interface_url = slapparameter_dict.get('monitor-interface-url', '') -%}
+{% if monitor_interface_url -%}
+monitor-setup-url = {{ monitor_interface_url }}/#page=settings_configurator&url=${publish:monitor-url}
+{%   endif -%}
+
 [buildout]
 extends = 
   {{ template_httpd_cfg }}
+  {{ template_monitor }}
 
 parts = 
   httpd
@@ -221,6 +256,18 @@ parts =
   httpd-promise
   publish
   directory-doc
+
+  monitor-base
+  cron-entry-logrotate
+  certificate-authority
+  monitor-conf
+  start-monitor
+  ca-httpd
+  monitor-httpd-promise
+  monitor-httpd-promise-conf
+  monitor-status2rss-cron-entry
+# End monitor
+
 # Complete parts with sections
   {{ part_list | join('\n  ') }}
  

software/kvm/instance-kvm-export.cfg.jinja2

+{% set monitor = True  -%}
+
+{% if slapparameter_dict.get('enable-monitor', 'True').lower() == 'false' -%}
+{%   set monitor = False  -%}
+{% endif -%}
+
 [buildout]
 extends =
   {{ kvm_template }}
@@ -14,6 +20,18 @@ parts +=
   novnc-promise
   cron
   frontend-promise
+{% if monitor -%}
+# monitor parts
+  monitor-base
+  cron-entry-logrotate
+  certificate-authority
+  monitor-conf
+  start-monitor
+  ca-httpd
+  monitor-httpd-promise
+  monitor-httpd-promise-conf
+  monitor-status2rss-cron-entry
+{% endif %}
 
 # Create the exporter executable, which is a simple shell script
 [exporter]

software/kvm/instance-kvm-import.cfg.in

 [buildout]
+
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+offline = true
+
 # Here, we don't need KVM to run to import data, so we don't
 # even extend the kvm instance profile.
-extends = ${pbsready-import:output}
+extends = 
+  {{ pbsready_import_template }}
+{% if slapparameter_dict.get('enable-monitor', 'True').lower() == 'true' -%}
+{{ '  ' ~ template_monitor }}
 
-eggs-directory = ${buildout:eggs-directory}
-develop-eggs-directory = ${buildout:develop-eggs-directory}
-offline = true
+[resilient-publish-connection-parameter]
+monitor-base-url = ${publish:monitor-base-url}
+monitor-url = ${publish:monitor-url}
+monitor-user = ${publish:monitor-user}
+monitor-password = ${publish:monitor-password}
+
+[monitor-instance-parameter]
+monitor-httpd-port = 8276
+monitor-title = {{ slapparameter_dict.get('name', 'Kvm Resilient clone') }}
+cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', '') }}
+{%   if slapparameter_dict.get('monitor-username', '') -%}
+username = {{ slapparameter_dict['monitor-username'] }}
+{%   endif -%}
+{%   if slapparameter_dict.get('monitor-password', '') -%}
+password = {{ slapparameter_dict['monitor-password'] }}
+{%   endif -%}
+instance-configuration =
+  raw takeover-url ${resilient-publish-connection-parameter:takeover-url}
+  raw takeover-password ${resilient-publish-connection-parameter:takeover-password}
+{% endif -%}
 
 [directory]
 recipe = slapos.cookbook:mkdirectory
-etc = $${buildout:directory}/etc
-bin = $${buildout:directory}/bin
-srv = $${buildout:directory}/srv
-var = $${buildout:directory}/var
-log = $${:var}/log
-scripts = $${:etc}/run
-services = $${:etc}/service
-promises = $${:etc}/promise
-novnc-conf = $${:etc}/novnc
-run = $${:var}/run
-ca-dir = $${:srv}/ssl
-cron-entries = $${:etc}/cron.d
-crontabs = $${:etc}/crontabs
-cronstamps = $${:etc}/cronstamps
+etc = ${buildout:directory}/etc
+bin = ${buildout:directory}/bin
+srv = ${buildout:directory}/srv
+var = ${buildout:directory}/var
+log = ${:var}/log
+scripts = ${:etc}/run
+services = ${:etc}/service
+promises = ${:etc}/promise
+novnc-conf = ${:etc}/novnc
+run = ${:var}/run
+ca-dir = ${:srv}/ssl
+cron-entries = ${:etc}/cron.d
+crontabs = ${:etc}/crontabs
+cronstamps = ${:etc}/cronstamps
 
 [importer]
 recipe = slapos.recipe.template:jinja2
-template = ${template-kvm-import-script:location}/${template-kvm-import-script:filename}
-rendered = $${directory:bin}/$${slap-parameter:namebase}-importer
+template = {{ template_kvm_import }}
+rendered = ${directory:bin}/${slap-parameter:namebase}-importer
 mode = 0700
 # Resilient stack wants a "wrapper" parameter
-wrapper = $${:rendered}
+wrapper = ${:rendered}
 context =
     section directory directory
-    raw zcat_binary ${gzip:location}/bin/zcat
-    raw gzip_binary ${gzip:location}/bin/gzip
+    raw zcat_binary {{ zcat_binary }}
+    raw gzip_binary {{ gzip_binary }}
 
-backup-disk-path = $${directory:backup}/virtual.qcow2
+backup-disk-path = ${directory:backup}/virtual.qcow2

software/kvm/instance-kvm-resilient.cfg.jinja2

@@ -4,6 +4,14 @@
 {% import 'replicated' as replicated with context %}
 
 {% set backup_amount = slapparameter_dict.pop('resilient-clone-number', "1")|int + 1 -%}
+{% set monitor_dict = {} -%}
+
+{% if slapparameter_dict.get('enable-monitor', 'True').lower() == 'true' -%}
+{% set monitor_return = ['monitor-base-url', 'monitor-url', 'monitor-user', 'monitor-password'] -%}
+{% set monitor_parameter = {'monitor-cors-domains': slapparameter_dict.pop('monitor-cors-domains', "")} -%}
+{% set monitor_dict = {'parameter': monitor_parameter, 'return': monitor_return} -%}
+{% endif -%}
+{% set monitor_interface_url = slapparameter_dict.pop('monitor-interface-url', '') -%}
 
 [buildout]
 eggs-directory = {{ eggs_directory }}
@@ -17,7 +25,21 @@ parts +=
   kvm-frontend-url-promise
   kvm-backend-url-promise
 
-{{ replicated.replicate("kvm", backup_amount, "kvm-export", "kvm-import", slapparameter_dict=slapparameter_dict) }}
+{% if slapparameter_dict.get('enable-monitor', 'True').lower() == 'true' -%}
+extends = {{ template_monitor }}
+
+
+[monitor-htpasswd]
+recipe = slapos.cookbook:generate.password
+storage-path = ${directory:etc}/.monitor_user
+bytes = 8
+username = admin
+
+{%  do monitor_parameter.__setitem__('monitor-username', slapparameter_dict.get('monitor-username', 'admin'))%}
+{%  do monitor_parameter.__setitem__('monitor-password', slapparameter_dict.get('monitor-password', '${monitor-htpasswd:passwd}'))%}
+{% endif -%}
+
+{{ replicated.replicate("kvm", backup_amount, "kvm-export", "kvm-import", slapparameter_dict=slapparameter_dict, monitor_parameter_dict=monitor_dict) }}
 
 [directory]
 recipe = slapos.cookbook:mkdirectory
@@ -29,16 +51,28 @@ promises = ${:etc}/promise
 # Note: += doesn't work.
 return =
 # Resilient related parameters
-  url ssh-public-key ssh-url notification-id ip
+  url ssh-public-key ssh-url notification-id ip {{ monitor_return | join(' ') }}
 # KVM related parameters
 # XXX: return ALL parameters (like nat rules), through jinja
   backend-url url ip
 
+# XXX Monitoring Main Instane
+[monitor-instance-parameter]
+monitor-httpd-port = 8160
+cors-domains = {{ monitor_parameter.get('monitor-cors-domains', '') }}
+
 [publish-connection-information]
 recipe = slapos.cookbook:publish
 backend-url = ${request-kvm:connection-backend-url}
 url = ${request-kvm:connection-url}
 ipv6 = ${request-kvm:connection-ip}
+monitor-base-url = ${publish:monitor-base-url}
+monitor-url = ${publish:monitor-url}
+monitor-user = ${publish:monitor-user}
+monitor-password = ${publish:monitor-password}
+{% if monitor_interface_url -%}
+monitor-setup-url = {{ monitor_interface_url }}/#page=settings_configurator&url=${publish:monitor-url}
+{% endif -%}
 
 [kvm-frontend-url-promise]
 # Check that url parameter is complete

software/kvm/instance-kvm.cfg.jinja2

@@ -316,40 +316,18 @@ port = ${httpd:port}
 {% endif %}
 
 {% if monitor -%}
-[monitor-access-log]
-< = monitor-directory-access
-source = ${directory:log}
 
-[monitor-access-public]
-< = monitor-directory-access
-source = ${directory:public}
-
-[monitor-parameters]
-port = 8026
-
-{% if instance_type == 'cluster' -%}
-# XXX - Set frontend software type to 'custom-personal' by default for cluster instance
-{% set frontend_software_type = 'custom-personal' -%}
+[monitor-instance-parameter]
+monitor-httpd-port = 8026
+monitor-title = {{ slapparameter_dict.get('name', 'KVM Standalone') }}
+cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', '') }}
+{%   if slapparameter_dict.get('monitor-username', '') -%}
+username = {{ slapparameter_dict['monitor-username'] }}
+{%   endif -%}
+{%   if slapparameter_dict.get('monitor-password', '') -%}
+password = {{ slapparameter_dict['monitor-password'] }}
 {%   endif -%}
 
-[request-monitor-frontend]
-<= slap-connection
-recipe = slapos.cookbook:requestoptional
-name = Monitor {{ slapparameter_dict.get('name', '') }} Frontend
-# XXX We have hardcoded SR URL here.
-software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
-slave = true
-config-url = ${monitor-parameters:url}
-software-type = {{ slapparameter_dict.get('monitor-frontend-software-type', frontend_software_type) }}
-return = site_url domain
-
-[monitor-frontend-promise]
-recipe = slapos.cookbook:check_url_available
-path = ${directory:promises}/monitor_frontend
-url = ${publish-connection-information:monitor_url}
-dash_path = {{ dash_executable_location }}
-curl_path = {{ curl_executable_location }}
-check-secure = 1
 {% endif -%}
 
 [publish-connection-information]
@@ -387,8 +365,14 @@ tap-ipv4 = ${slap-network-information:tap-ipv4}
 {%   endif %}
 {% endif %}
 {% if monitor -%}
-monitor_url = ${request-monitor-frontend:connection-site_url}
-monitor_v6_url = ${monitor-parameters:url}
+monitor-base-url = ${publish:monitor-base-url}
+monitor-url = ${publish:monitor-url}
+monitor-user = ${publish:monitor-user}
+monitor-password = ${publish:monitor-password}
+{%   set monitor_interface_url = slapparameter_dict.get('monitor-interface-url', '') -%}
+{%   if monitor_interface_url -%}
+monitor-setup-url = {{ monitor_interface_url }}/#page=settings_configurator&url=${publish:monitor-url}
+{%   endif -%}
 {% endif -%}
 
 {% if use_tap == 'true' and tap_network_dict.has_key('ipv4') -%}
@@ -617,26 +601,18 @@ parts =
 #  kvm-monitor
   cron
   cron-entry-logrotate
-#  cron-entry-monitor
   frontend-promise
 {% if monitor -%}
 # monitor parts
-  cron-entry-monitor
-  cron-entry-rss
-  deploy-index
-  deploy-status-history-cgi
-  deploy-status-cgi
-#  deploy-logfile-cgi
-#  deploy-resource-consumption-monitoring-cgi
-  setup-static-files
-  public-symlink
-  cgi-httpd-wrapper
-  cgi-httpd-graceful-wrapper
-  monitor-promise
-  monitor-instance-log-access
-  monitor-access-log
-  monitor-access-public
-#  monitor-frontend-promise
+  monitor-base
+  cron-entry-logrotate
+  certificate-authority
+  monitor-conf
+  start-monitor
+  ca-httpd
+  monitor-httpd-promise
+  monitor-httpd-promise-conf
+  monitor-status2rss-cron-entry
 {% endif -%}
 # Complete parts with sections
   {{ part_list | join('\n  ') }}

software/kvm/instance.cfg.in

@@ -14,7 +14,7 @@ nbd = ${template-nbd:output}
 frontend = ${template-frontend:output}
 
 kvm-resilient = $${dynamic-template-kvm-resilient:rendered}
-kvm-import = ${template-kvm-import:output}
+kvm-import = $${dynamic-template-kvm-import:rendered}
 kvm-export = $${dynamic-template-kvm-export:rendered}
 
 # Used for the test of resiliency. The system wants a "test" software_type.
@@ -69,7 +69,7 @@ extra-context =
   raw logrotate_cfg ${template-logrotate-base:rendered}
   raw template_content ${template-content:location}/${template-content:filename}
   raw template_httpd_cfg ${template-httpd:rendered}
-  raw template_monitor ${monitor-template:output}
+  raw template_monitor ${monitor2-template:rendered}
 
 [dynamic-template-kvm]
 recipe = slapos.recipe.template:jinja2
@@ -100,7 +100,7 @@ context =
     raw template_content ${template-content:location}/${template-content:filename}
     raw template_kvm_controller_run ${template-kvm-controller:location}/${template-kvm-controller:filename}
     raw template_kvm_run ${template-kvm-run:location}/${template-kvm-run:filename}
-    raw template_monitor ${monitor-template:output}
+    raw template_monitor ${monitor2-template:rendered}
     raw websockify_executable_location ${buildout:directory}/bin/websockify
 template-parts-destination = ${template-parts:destination}
 template-replicated-destination = ${template-replicated:destination}
@@ -118,6 +118,7 @@ context =
     key eggs_directory buildout:eggs-directory
     key slapparameter_dict slap-configuration:configuration
     raw curl_executable_location ${curl:location}/bin/curl
+    raw template_monitor ${monitor2-template:rendered}
 template-parts-destination = ${template-parts:destination}
 template-replicated-destination = ${template-replicated:destination}
 import-list = file parts :template-parts-destination
@@ -136,6 +137,23 @@ context =
     raw template_kvm_export ${template-kvm-export-script:location}/${template-kvm-export-script:filename}
     raw pbsready_export_template ${pbsready-export:output}
     raw gzip_binary ${gzip:location}/bin/gzip
+    key slapparameter_dict slap-configuration:configuration
+mode = 0644
+
+[dynamic-template-kvm-import]
+recipe = slapos.recipe.template:jinja2
+template = ${template-kvm-import:location}/instance-kvm-import.cfg.in
+rendered = $${buildout:directory}/template-kvm-import.cfg
+extensions = jinja2.ext.do
+context =
+    key develop_eggs_directory buildout:develop-eggs-directory
+    key eggs_directory buildout:eggs-directory
+    raw template_kvm_import ${template-kvm-import-script:location}/${template-kvm-import-script:filename}
+    raw pbsready_import_template ${pbsready-import:output}
+    raw template_monitor ${monitor2-template:rendered}
+    key slapparameter_dict slap-configuration:configuration
+    raw zcat_binary ${gzip:location}/bin/zcat
+    raw gzip_binary ${gzip:location}/bin/gzip
 mode = 0644
 
 [dynamic-template-kvm-resilient-test]