do not create two wrappers for the same executable if hash change

Prevent creating 2 wrapper for the same service if hash changed. Here, one service is exited because port is used by the firt to service to start:

    slappart6:runner-sshd-4248650e36a9a26a6481df1baffd9f58-on-watch                RUNNING   pid 27835, uptime 0:03:45
    slappart6:runner-sshd-b3b68f4278ceb84691ec27521ea229eb-on-watch                EXITED    Mar 06 04:52 PM

To achieve that, update slapos.cookbook and use hash-existing-files option of wrapper recipe

hash-existing-files list all the files used for hash that are not
handled by buildout. For those files, the hash is calculated as soon as
the __init__ function so that if there is a change in those files,
buildout will remove the existing wrapper (it will uninstall the
section) and replace it with the new wrapper.

/reviewed-on nexedi/slapos!525

software/caddy-frontend/buildout.hash.cfg

@@ -22,15 +22,15 @@ md5sum = c801b7f9f11f0965677c22e6bbe9281b
 
 [template-apache-frontend]
 filename = instance-apache-frontend.cfg.in
-md5sum = 36eaee40086aef59b4ea11c52e6063e3
+md5sum = 7f2f7b40e3639c50cf7dd929f4425f20
 
 [template-caddy-replicate]
 filename = instance-apache-replicate.cfg.in
-md5sum = f874519f5c000d0c479dde34e1cbb268
+md5sum = bd73ad5ef84fe6bf0f479ee45da3139d
 
 [template-slave-list]
 filename = templates/apache-custom-slave-list.cfg.in
-md5sum = d2afd6ef52754b14ba9bf881b9f685c6
+md5sum = 5913a176c025cdadee6782f8ad285dc4
 
 [template-slave-configuration]
 filename = templates/custom-virtualhost.conf.in
@@ -118,4 +118,4 @@ md5sum = 38792c2dceae38ab411592ec36fff6a8
 
 [template-kedifa]
 filename = instance-kedifa.cfg.in
-md5sum = ff547807d8b703a124a43eab211666c2
+md5sum = bb1eb172e65d8ffe5b161eae191947e1

software/caddy-frontend/instance-apache-frontend.cfg.in

@@ -345,9 +345,8 @@ wrapper-path = ${directory:bin}/caddy-wrapper
 recipe = slapos.cookbook:wrapper
 command-line = ${caddy-wrapper:wrapper-path} -pidfile ${caddy-configuration:pid-file}
 wrapper-path = ${directory:service}/frontend_caddy
-hash-files =
-  ${buildout:directory}/software_release/buildout.cfg
-  ${caddy-wrapper:wrapper-path}
+hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
+hash-files = ${caddy-wrapper:wrapper-path}
 
 [not-found-html]
 recipe = slapos.cookbook:symbolic.link
@@ -454,7 +453,7 @@ recipe = slapos.cookbook:wrapper
 command-line = {{ parameter_dict['trafficserver'] }}/bin/traffic_cop
 wrapper-path = ${trafficserver-variable:wrapper-path}
 environment = TS_ROOT=${buildout:directory}
-hash-files = ${buildout:directory}/software_release/buildout.cfg
+hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
 
 [trafficserver-reload]
 recipe = slapos.cookbook:wrapper

software/caddy-frontend/instance-apache-replicate.cfg.in

@@ -533,8 +533,9 @@ command-line = {{ parameter_dict['caddy'] }}
   -root ${:directory}
 
 wrapper-path = ${directory:service}/rejected-slave-publish
-hash-files =
+hash-existing-files =
   ${buildout:directory}/software_release/buildout.cfg
+hash-files =
   ${rejected-slave-template:rendered}
   ${rejected-slave-certificate:certificate}
 

software/caddy-frontend/instance-kedifa.cfg.in

@@ -21,7 +21,7 @@ parts =
   promise-expose-csr_id-ip-port
 
 [caucased]
-hash-files = ${buildout:directory}/software_release/buildout.cfg
+hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
 
 {%  set caucase_host = '[' ~ instance_parameter['ipv6-random'] ~ ']' %}
 {%  set caucase_netloc = caucase_host ~ ':' ~ instance_parameter['configuration.caucase_port'] -%}
@@ -175,7 +175,7 @@ command-line = {{ parameter_dict['caddy'] }}
   -root ${directory:csr_id}
 
 wrapper-path = ${directory:service}/expose-csr_id
-hash-files = ${buildout:directory}/software_release/buildout.cfg
+hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
 
 [get-csr_id-certificate]
 recipe = collective.recipe.shelloutput
@@ -249,7 +249,7 @@ command-line = {{ parameter_dict['kedifa'] }}
   --logfile ${kedifa-config:logfile}
 
 wrapper-path = ${directory:service}/kedifa
-hash-files = ${buildout:directory}/software_release/buildout.cfg
+hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
 
 # Publish KeDiFa configuration for upload and download for each slave
 {%- set slave_kedifa_information = {} -%}

software/caddy-frontend/templates/apache-custom-slave-list.cfg.in

@@ -395,7 +395,7 @@ ipv4 = ${slap-network-information:local-ipv4}
 ipv6 = ${slap-network-information:global-ipv6}
 wrapper-path = {{ service_directory}}/6tunnel-${:ipv6-port}
 command-line = {{ sixtunnel_executable }} -6 -4 -d -l ${:ipv6} ${:ipv6-port} ${:ipv4} ${:ipv4-port}
-hash-files = ${buildout:directory}/software_release/buildout.cfg
+hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
 
 [tunnel-6to4-base-http_port]
 <= tunnel-6to4-base
@@ -464,7 +464,7 @@ command-line = {{ kedifa_updater }}
   {{ kedifa_updater_state_file }}
 
 wrapper-path = {{ service_directory }}/kedifa-updater
-hash-files = ${buildout:directory}/software_release/buildout.cfg
+hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
 
 [kedifa-updater-run]
 recipe = plone.recipe.command
@@ -579,7 +579,7 @@ command-line = {{ caddy_executable }}
   -root {{ directory_csr_id }}
 
 wrapper-path = {{ service_directory }}/expose-csr_id
-hash-files = ${buildout:directory}/software_release/buildout.cfg
+hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
 
 [get-csr_id-certificate]
 recipe = collective.recipe.shelloutput

software/caddy-frontend/test/test.py

@@ -379,7 +379,7 @@ class TestDataMixin(object):
       partition_id = caddy_wrapper_path.split('/')[-3]
       hash_value_dict[
         'caddy-%s' % (partition_id)] = generateHashFromFiles(
-        hash_file_list + [caddy_wrapper_path]
+        [caddy_wrapper_path] + hash_file_list
       )
     for rejected_slave_publish_path in glob.glob(os.path.join(
       self.instance_path, '*', 'etc', 'Caddyfile-rejected-slave')):
@@ -389,7 +389,7 @@ class TestDataMixin(object):
       hash_value_dict[
         'rejected-slave-publish'
       ] = generateHashFromFiles(
-        hash_file_list + [rejected_slave_publish_path, rejected_slave_pem_path]
+        [rejected_slave_publish_path, rejected_slave_pem_path] + hash_file_list
       )
 
     runtime_data = self.getTrimmedProcessInfo()

software/kvm/buildout.hash.cfg

@@ -19,7 +19,7 @@ md5sum = 028b6a6456d744c11b1bb2c51ecd51b2
 
 [template-kvm]
 filename = instance-kvm.cfg.jinja2
-md5sum = 12a99227828e834d97ee0c68246c467c
+md5sum = 325326fa7266757dfed028b11aae58a0
 
 [template-kvm-cluster]
 filename = instance-kvm-cluster.cfg.jinja2.in

software/kvm/instance-kvm.cfg.jinja2

@@ -181,7 +181,7 @@ ipv4 = ${slap-network-information:local-ipv4}
 ipv6 = ${slap-network-information:global-ipv6}
 wrapper-path = ${directory:services}/6tunnel-${:ipv6-port}
 command-line = {{ sixtunnel_executable_location }} -6 -4 -d -l ${:ipv6} ${:ipv6-port} ${:ipv4} ${:ipv4-port}
-hash-files = ${buildout:directory}/software_release/buildout.cfg
+hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
 
 {% if use_nat == 'true' and nat_rule_list -%}
 {%   for port in nat_rule_list.split(' ') -%}
@@ -206,7 +206,7 @@ socket-path = ${kvm-controller-parameter-dict:socket-path}
 wrapper-path = ${directory:services}/kvm
 command-line = ${kvm-run:rendered}
 kvm-controller = ${kvm-controller-wrapper:wrapper-path}
-hash-files = ${buildout:directory}/software_release/buildout.cfg
+hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
 
 
 [kvm-controller-wrapper]
@@ -286,7 +286,7 @@ wrapped-path = ${novnc-instance:path}
 recipe = slapos.cookbook:wrapper
 command-line = ${websockify-sighandler:wrapper-path}
 wrapper-path = ${directory:services}/websockify
-hash-files = ${buildout:directory}/software_release/buildout.cfg
+hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
 wait-for-files =
   ${ca-novnc:key-file}
   ${ca-novnc:cert-file}
@@ -306,7 +306,7 @@ ca-crl = ${cadirectory:crl}
 recipe = slapos.cookbook:wrapper
 command-line = ${certificate-authority:wrapper}
 wrapper-path = ${directory:services}/certificate_authority
-hash-files = ${buildout:directory}/software_release/buildout.cfg
+hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
 
 [cadirectory]
 recipe = slapos.cookbook:mkdirectory
@@ -348,7 +348,7 @@ binary = ${directory:bin}/crond_raw
 recipe = slapos.cookbook:wrapper
 command-line = ${cron:binary}
 wrapper-path = ${directory:services}/crond
-hash-files = ${buildout:directory}/software_release/buildout.cfg
+hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
 
 [cron-simplelogger]
 recipe = slapos.cookbook:simplelogger
@@ -397,7 +397,7 @@ use-hash-url = false
 recipe = slapos.cookbook:wrapper
 command-line = ${httpd:wrapper}
 wrapper-path = ${directory:services}/http-server
-hash-files = ${buildout:directory}/software_release/buildout.cfg
+hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
 
 [httpd-promise]
 recipe = slapos.cookbook:check_port_listening
@@ -623,7 +623,7 @@ runner-path = ${directory:bin}/6tunnel-cluster
 recipe = slapos.cookbook:wrapper
 command-line = ${tunnel-cluster-url:runner-path}
 wrapper-path = ${directory:services}/6tunnel-cluster
-hash-files = ${buildout:directory}/software_release/buildout.cfg
+hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
 
 [ansible-vm-promise]
 recipe = slapos.recipe.template:jinja2

software/plantuml/buildout.hash.cfg

@@ -15,7 +15,7 @@
 
 [instance]
 filename = instance.cfg.in
-md5sum = 8915151103355dd59da31979a14e59fd
+md5sum = 8ed7b3f59d94869a328c72fa9bbfd7d5
 
 [tomcat-server-xml]
 filename = server.xml.in

software/plantuml/instance.cfg.in

@@ -63,8 +63,9 @@ environment =
 # related links:
 #   https://docs.oracle.com/javase/8/docs/technotes/guides/intl/fontconfig.html
 #   https://bugs.openjdk.java.net/browse/JDK-7175487
-hash-files =
+hash-existing-files =
   $${buildout:directory}/software_release/buildout.cfg
+hash-files =
   $${tomcat-server-xml:output}
 
 ip =  $${tomcat-server-xml:ip}

software/plantuml/test/test.py

@@ -158,8 +158,8 @@ class ServicesTestCase(PlantUMLTestCase):
 
   def test_hashes(self):
     hash_files = [
-      'software_release/buildout.cfg',
-      'var/tomcat/conf/server.xml'
+      'var/tomcat/conf/server.xml',
+      'software_release/buildout.cfg'
     ]
     expected_process_names = [
       'tomcat-instance-{hash}-on-watch',

software/powerdns/instance-powerdns.cfg

@@ -86,7 +86,7 @@ extra-context =
 recipe = slapos.cookbook:wrapper
 command-line = ${powerdns:location}/sbin/pdns_server --config-dir=$${pdns-directory:configuration}
 wrapper-path = $${directory:service}/pdns
-hash-files = $${buildout:directory}/software_release/buildout.cfg
+hash-existing-files = $${buildout:directory}/software_release/buildout.cfg
 
 [pdns-reload]
 recipe = slapos.cookbook:wrapper

software/powerdns/software.cfg

@@ -25,7 +25,7 @@ mode = 0644
 [template-powerdns]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-powerdns.cfg
-md5sum = b2301de110ba95be46121dc18d7ca544
+md5sum = 5077ba344b641fa9703f9872a974d3d7
 output = ${buildout:directory}/template-powerdns.cfg
 mode = 0644
 

software/re6stnet/instance-re6stnet.cfg.in

@@ -92,7 +92,7 @@ cert-file = ${certificate-authority:ca-certs}/apache.crt
 recipe = slapos.cookbook:wrapper
 command-line = ${apache-ca:wrapper}
 wrapper-path = ${directory:services}/httpd
-hash-files = ${buildout:directory}/software_release/buildout.cfg
+hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
 {% endif %}
 
 [apache-httpd-graceful]

software/re6stnet/software.cfg

@@ -86,7 +86,7 @@ extra-context =
 [template-re6stnet]
 < = download-base
 filename = instance-re6stnet.cfg.in
-md5sum = 09683c66c16361b77867d8f2e1140a35
+md5sum = 73c8fad6db7c711b0822891aba574564
 
 [template-apache-conf]
 < = download-base

software/slaprunner/buildout.hash.cfg

@@ -18,7 +18,7 @@ md5sum = c44a7481bb85e3258128afe3fcf23f44
 
 [template-runner]
 filename = instance-runner.cfg
-md5sum = 48bff0b5f082f22d44966b1151f07133
+md5sum = 58a6f21021279ae52c91813f61cdbdd7
 
 [template-runner-import-script]
 filename = template/runner-import.sh.jinja2

software/slaprunner/instance-runner.cfg

@@ -212,7 +212,7 @@ recipe = slapos.cookbook:wrapper
 # XXX hardcoded locations
 command-line = $${directory:bin}/slapos node supervisord --cfg $${directory:etc}/slapos.cfg -n
 wrapper-path = $${directory:services}/slaprunner-supervisord
-hash-files = $${buildout:directory}/software_release/buildout.cfg
+hash-existing-files = $${buildout:directory}/software_release/buildout.cfg
 
 
 [test-runner]
@@ -303,7 +303,7 @@ keygen-binary = ${openssh:location}/bin/ssh-keygen
 recipe = slapos.cookbook:wrapper
 command-line = $${runner-sshkeys-authority:wrapper}
 wrapper-path = $${directory:services}/runner-sshkeys-authority
-hash-files = $${buildout:directory}/software_release/buildout.cfg
+hash-existing-files = $${buildout:directory}/software_release/buildout.cfg
 
 [runner-sshkeys-sshd]
 <= runner-sshkeys-authority
@@ -319,7 +319,7 @@ wrapper = $${directory:bin}/runner-sshd
 recipe = slapos.cookbook:wrapper
 command-line = $${runner-sshkeys-sshd:wrapper}
 wrapper-path = $${directory:services}/runner-sshd
-hash-files = $${buildout:directory}/software_release/buildout.cfg
+hash-existing-files = $${buildout:directory}/software_release/buildout.cfg
 
 [runner-sshd-add-authorized-key]
 recipe = slapos.cookbook:dropbear.add_authorized_key
@@ -431,7 +431,7 @@ access-url = https://[$${httpd-parameters:global_ip}]:$${httpd-parameters:global
 wait-for-files =
   $${ca-nginx:cert-file}
   $${ca-nginx:key-file}
-hash-files = $${buildout:directory}/software_release/buildout.cfg
+hash-existing-files = $${buildout:directory}/software_release/buildout.cfg
 
 [logrotate-entry-apache-httpd]
 <= logrotate-entry-base
@@ -482,7 +482,7 @@ wrapper-path = $${gunicorn:bin_launcher}
 environment = PATH=$${shell-environment:path}
   RUNNER_CONFIG=$${slaprunner:slapos.cfg}
   LANG=en_GB.UTF-8
-hash-files = $${buildout:directory}/software_release/buildout.cfg
+hash-existing-files = $${buildout:directory}/software_release/buildout.cfg
 
 [gunicorn-graceful]
 recipe = slapos.cookbook:wrapper
@@ -526,13 +526,13 @@ name = example.com
 recipe = slapos.cookbook:wrapper
 command-line = $${directory:bin}/nginx-frontend
 wrapper-path = $${directory:services}/nginx-frontend
-hash-files = $${buildout:directory}/software_release/buildout.cfg
+hash-existing-files = $${buildout:directory}/software_release/buildout.cfg
 
 [certificate-authority-service]
 recipe = slapos.cookbook:wrapper
 command-line = $${directory:bin}/certificate_authority
 wrapper-path = $${directory:services}/certificate_authority
-hash-files = $${buildout:directory}/software_release/buildout.cfg
+hash-existing-files = $${buildout:directory}/software_release/buildout.cfg
 
 #--------------------
 #--
@@ -706,7 +706,7 @@ template = inline:
 recipe = slapos.cookbook:wrapper
 command-line = $${directory:bin}/shellinaboxd
 wrapper-path = $${directory:services}/shellinaboxd
-hash-files = $${buildout:directory}/software_release/buildout.cfg
+hash-existing-files = $${buildout:directory}/software_release/buildout.cfg
 
 [shell-environment]
 shell = ${bash:location}/bin/bash
@@ -840,7 +840,7 @@ context =
 recipe = slapos.cookbook:wrapper
 command-line = $${buildout:directory}/bin/supervisord -c $${supervisord-conf:rendered} --nodaemon
 wrapper-path = $${directory:services}/supervisord
-hash-files = $${buildout:directory}/software_release/buildout.cfg
+hash-existing-files = $${buildout:directory}/software_release/buildout.cfg
 
 [logrotate-entry-supervisord]
 <= logrotate-entry-base

software/turnserver/instance-turnserver.cfg.jinja2.in

@@ -112,7 +112,7 @@ recipe = slapos.cookbook:wrapper
 command-line = {{ parameter_dict['turnserver-location'] }}/bin/turnserver
                -c ${turnserver-config:output}
 wrapper-path = ${directory:services}/turnserver
-hash-files = ${buildout:directory}/software_release/buildout.cfg
+hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
 
 [promise-check-turnserver-port]
 <= monitor-promise-base

software/turnserver/software.cfg

@@ -29,7 +29,7 @@ md5sum = d027a2dccaf15ae6e7d3a28cc02d70c3
 [template-turnserver]
 <= download-base
 filename = instance-turnserver.cfg.jinja2.in
-md5sum = 02bddf180519f7649d3b1f997a496eed
+md5sum = f275df4900a9db1d1a72b67b12ee8afe
 
 [versions]
 slapos.recipe.template = 4.3

stack/lamp/buildout.cfg

@@ -120,7 +120,7 @@ context =
 <= template-download-base
 filename = instance-apache-php.cfg.in
 output = ${buildout:parts-directory}/${:_buildout_section_name_}/${:filename}
-md5sum = 44796786448fd0319dde14923fbd798b
+md5sum = 9d3eb7fcc44ac25bef8ee9df6eac8974
 
 [instance-lamp]
 <= template-download-base

stack/lamp/instance-apache-php.cfg.in

@@ -74,7 +74,7 @@ ca-crl = ${ca-directory:crl}
 recipe = slapos.cookbook:wrapper
 command-line = ${certificate-authority:wrapper}
 wrapper-path = ${directory:services}/certificate_authority
-hash-files = ${buildout:directory}/software_release/buildout.cfg
+hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
 
 [ca-apache-php]
 <= certificate-authority
@@ -88,7 +88,7 @@ wrapper = ${directory:bin}/ca-apache-php
 recipe = slapos.cookbook:wrapper
 command-line = ${ca-apache-php:wrapper}
 wrapper-path = ${directory:services}/apache-php
-hash-files = ${buildout:directory}/software_release/buildout.cfg
+hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
 depends =
   ${copy-application:recipe}
   ${apache-graceful:recipe}

stack/logrotate/buildout.hash.cfg

@@ -22,4 +22,4 @@ md5sum = 117e46af6d9d31c09eeb86089d11407e
 
 [template-logrotate-base]
 filename = instance-logrotate-base.cfg.in
-md5sum = 338be3ee006c08b26abcc023f398247e
+md5sum = c2a06234d923c8e589cc0ddb1d6e0880

stack/logrotate/instance-logrotate-base.cfg.in

@@ -20,7 +20,7 @@ service = ${cron-service:wrapper-path}
 recipe = slapos.cookbook:wrapper
 command-line = ${cron:binary}
 wrapper-path = ${logrotate-directory:services}/crond
-hash-files = ${buildout:directory}/software_release/buildout.cfg
+hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
 
 [cron-simplelogger]
 recipe = slapos.cookbook:simplelogger

stack/monitor/buildout.hash.cfg

@@ -14,4 +14,4 @@
 # not need these here).
 [monitor2-template]
 filename = instance-monitor.cfg.jinja2.in
-md5sum = c0856969ea372d49ab60ba07b3bf2a51
+md5sum = d3d5a44db790f2bdf4fcbe6ebffe261f

stack/monitor/instance-monitor.cfg.jinja2.in

@@ -61,7 +61,7 @@ ca-crl = ${ca-directory:crl}
 recipe = slapos.cookbook:wrapper
 command-line = ${certificate-authority:wrapper}
 wrapper-path = ${directory:services}/certificate_authority
-hash-files = ${buildout:directory}/software_release/buildout.cfg
+hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
 
 [ca-monitor-httpd]
 <= certificate-authority
@@ -75,7 +75,7 @@ wrapper = ${directory:bin}/ca-monitor-httpd
 recipe = slapos.cookbook:wrapper
 command-line = ${ca-monitor-httpd:wrapper}
 wrapper-path = ${directory:services}/monitor-httpd
-hash-files = ${buildout:directory}/software_release/buildout.cfg
+hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
 
 [monitor-conf-parameters]
 title = ${monitor-instance-parameter:monitor-title}

stack/slapos.cfg

@@ -136,7 +136,7 @@ pyparsing = 2.2.0
 pytz = 2016.10
 requests = 2.13.0
 six = 1.12.0
-slapos.cookbook = 1.0.121
+slapos.cookbook = 1.0.123
 slapos.core = 1.4.28
 slapos.extension.strip = 0.4
 slapos.extension.shared = 1.0