helloworld: Service restructuring + Ruby version

For upcoming GitLab SR we need Ruby support.

We have minimal Ruby support in the form of Ruby component and
`rubygemsrecipe` recipe to install gems. However a lot of top-level
software/services in Ruby world is not released as gem and have to be
installed / worked with via Bundler[1], and for this way we do not have
an example.

Let's add such example in the form of extending helloworld SR to also
say hello to the web in both Python and Ruby via simple Ruby program
which is deployed via Bundler.

For this to happen, we need to make some preparatory changes first:

- move helloweb program(s) to own repository[2].
- modernize instance code and convert it to jinja2 to allow control structures.
- prepare instance infrastructure to support several helloweb program kinds.

and only then show how to do Ruby stuff via Bundler.

To me the effect of `software/helloworld` refactoring is good even without
Ruby part, just to improve reference material about showing how to do.

/generally-reviewed-by @jerome, @vpelletier  (on !23)
/cc @kazuhiko, @rafael, @alain.takoudjou, @cedric.leninivin

[1] http://bundler.io
[2] http://lab.nexedi.com/nexedi/helloweb

component/helloweb/buildout.cfg

+# helloweb - programs to say hello to the Web in various languages
+[buildout]
+extends =
+  ../git/buildout.cfg
+  ../ruby/buildout.cfg
+
+parts =
+  helloweb-python
+  helloweb-ruby
+
+
+# repository with examples
+[helloweb-repository]
+recipe  = slapos.recipe.build:gitclone
+git-executable = ${git:location}/bin/git
+repository = https://lab.nexedi.com/nexedi/helloweb.git
+revision = 7f5e49a541ae7b51da034dcdf20b91b64f4fb0e9
+location = ${buildout:parts-directory}/helloweb
+
+
+# -*- python -*-
+[helloweb-egg]
+recipe  = zc.recipe.egg:develop
+egg     = helloweb
+setup   = ${helloweb-repository:location}/python/
+
+[helloweb-python]
+recipe  = zc.recipe.egg:scripts
+eggs    = ${helloweb-egg:egg}
+scripts = helloweb=helloweb-python
+
+
+# -*- ruby -*-
+
+# if ruby program is represented as already-released gem, we can install it
+# with `gem install ...` (via rubygemsrecipe).
+#
+# Alternatively if we need to
+# install the program from source-checkout, the Ruby way is to use bundler,
+# install program dependencies via it, and run the program itself via it.
+#
+# Since for helloweb.rb we have source checkout - we go the second - bundler way.
+
+# bundler, that we'll use to install gems and run binaries (via `bundle exec ...`)
+[bundler]
+# rubygemsrecipe with fixed url and this way pinned rubygems version
+recipe  = rubygemsrecipe
+url     = https://rubygems.org/rubygems/rubygems-2.4.8.zip
+ruby-location = ${ruby2.1:location}
+ruby-executable = ${:ruby-location}/bin/ruby
+gems    = bundler==1.10.6
+
+# bin installed here
+bundle  = ${buildout:bin-directory}/bundle
+
+# install together with path to ruby enabled
+# ( reason: rubygemsrecipe hardcodes PATH inside generated bin/* and it is
+#   impossible to adjust it later )
+#
+# bundle exec <smth>                ; <smth> starts with `#!/usr/bin/env ruby` as rubygems
+environment =
+  PATH    = ${:ruby-location}/bin:%(PATH)s
+
+
+[helloweb-ruby-bundle]
+recipe  = slapos.recipe.cmmi
+path    = ${helloweb-repository:location}/ruby/
+
+configure-command = :
+make-binary =
+make-targets= cd ${:path} && ${bundler:bundle} install
+
+
+[helloweb-ruby]
+recipe  = slapos.cookbook:wrapper
+wrapper-path = ${buildout:bin-directory}/${:_buildout_section_name_}
+environment  =
+    BUNDLE_GEMFILE = ${helloweb-ruby-bundle:path}/Gemfile
+command-line =
+    ${bundler:bundle} exec sh -c 'helloweb.rb "$@"' ${:_buildout_section_name_}

software/helloworld/hello-web.in

-#!{{ python_executable }}
-"""Simple web-server that says "Hello World" for every path
-
-hello-web [--logfile <logfile>] <bind-ip> <bind-port> ...
-"""
-
-import sys
-import time
-import argparse
-from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
-from socket import AF_INET6
-
-
-class WebHello(BaseHTTPRequestHandler):
-
-    def do_GET(self):
-        self.send_response(200) # ok
-        self.send_header("Content-type", "text/plain")
-        self.end_headers()
-
-        print >>self.wfile, \
-            "Hello %s at `%s`  ; %s" % (
-                ' '.join(self.server.webhello_argv) or 'world',
-                self.path, time.asctime())
-
-
-class HTTPServerV6(HTTPServer):
-    address_family = AF_INET6
-
-def main():
-    parser = argparse.ArgumentParser()
-    parser.add_argument('--logfile', dest='logfile')
-    parser.add_argument('bind_ip')
-    parser.add_argument('bind_port', type=int)
-    parser.add_argument('argv_extra', metavar='...', nargs=argparse.REMAINDER)
-
-    args = parser.parse_args()
-
-    # HTTPServer logs to sys.stderr - override it if we have --logfile
-    if args.logfile:
-        f = open(args.logfile, 'a', buffering=1)
-        sys.stderr = f
-
-    print >>sys.stderr, '* %s Hello-Web starting at %s' % (
-        time.asctime(), (args.bind_ip, args.bind_port))
-
-    # TODO autodetect ipv6/ipv4
-    httpd = HTTPServerV6( (args.bind_ip, args.bind_port), WebHello)
-    httpd.webhello_argv = args.argv_extra
-    httpd.serve_forever()
-
-if __name__ == '__main__':
-    main()

software/helloworld/instance.cfg.in

@@ -6,15 +6,13 @@
 [buildout]
 parts =
   directory
-  hello-world
-  hello-world-promise
   publish-connection-parameter
 
 # Define egg directories to be the one from Software Release
 # (/opt/slapgrid/...)
 # Always the same.
-eggs-directory = ${buildout:eggs-directory}
-develop-eggs-directory = ${buildout:develop-eggs-directory}
+eggs-directory = {{ buildout['eggs-directory'] }}
+develop-eggs-directory = {{ buildout['develop-eggs-directory'] }}
 offline = true
 
 
@@ -23,15 +21,15 @@ offline = true
 # We use the slapconfiguration recipe with a few parameters (partition id,
 # computer id, certificate, etc).
 # It will then authenticate to SlapOS Master and fetch the instance parameters.
-# The parameters are accessible from $${instance-parameter:configuration.name-of-parameter}
+# The parameters are accessible from ${instance-parameter:configuration.name-of-parameter}
 # Always the same. Just copy/paste.
 # See docstring of slapos.cookbook:slapconfiguration for more information.
 recipe = slapos.cookbook:slapconfiguration
-computer = $${slap_connection:computer_id}
-partition = $${slap_connection:partition_id}
-url = $${slap_connection:server_url}
-key = $${slap_connection:key_file}
-cert = $${slap_connection:cert_file}
+computer = ${slap_connection:computer_id}
+partition = ${slap_connection:partition_id}
+url = ${slap_connection:server_url}
+key = ${slap_connection:key_file}
+cert = ${slap_connection:cert_file}
 
 # Define default parameter(s) that will be used later, in case user didn't
 # specify it.
@@ -46,22 +44,22 @@ configuration.name = John Doe
 # Create all needed directories, depending on your needs
 [directory]
 recipe = slapos.cookbook:mkdirectory
-home = $${buildout:directory}
-etc = $${:home}/etc
-var = $${:home}/var
+home = ${buildout:directory}
+etc = ${:home}/etc
+var = ${:home}/var
 # Executables put here will be started but not monitored (for startup scripts)
-script = $${:etc}/run/
+script = ${:etc}/run/
 # Executables put here will be started and monitored (for daemons)
-service = $${:etc}/service
+service = ${:etc}/service
 # Executables put here will be launched after buildout has completed to see
 # if instance is running
-promise = $${:etc}/promise/
-# Path of the log directory used by our service (see [hello-world])
-log = $${:var}/log
+promise = ${:etc}/promise/
+# Path of the log directory used by our service (see [helloweb])
+log = ${:var}/log
 
 
 # Create a simple web server that says "hello <configuration.name>" to the web.
-[hello-world]
+[helloweb]
 # helloworld service is listening on:
 # - global IPv6 address, and
 # - fixed port
@@ -69,32 +67,58 @@ log = $${:var}/log
 # NOTE because every computer partition is allocated its own global IPv6
 # address, it is ok to fix the port - different hello-world instances will have
 # different IPv6 addresses and they all will be accessible at the same time.
-ipv6 = $${instance-parameter:ipv6-random}
-port = 7777
+ipv6 = ${instance-parameter:ipv6-random}
 # full URL - for convenience
-url = http://[$${:ipv6}]:$${:port}
+url = http://[${:ipv6}]:${:port}
 
 # the service will log here
-logfile = $${directory:log}/hello-world.log
+logfile = ${directory:log}/helloweb-${:kind}.log
 
 # Actual script that starts the service:
 # This recipe will try to "exec" the command-line after separating parameters.
 recipe = slapos.cookbook:wrapper
 command-line =
-    ${hello-web-bin:rendered} --logfile $${hello-world:logfile}
-        $${:ipv6} $${:port} $${instance-parameter:configuration.name}
+    {{ buildout['bin-directory'] }}/helloweb-${:kind} --logfile ${:logfile}
+        ${:ipv6} ${:port} ${instance-parameter:configuration.name}
 # Put this shell script in the "etc/service" directory. Each executable of this
 # repository will be started and monitored by supervisord. If a service
 # exits/crashes, it will trigger a "bang" and cause a re-run of the instance.
-wrapper-path = $${directory:service}/hello-world
+wrapper-path = ${directory:service}/helloweb-${:kind}
 
 
-# promise, that checks that hello-world service is alive
-[hello-world-promise]
+# promise, that checks that helloweb service is alive
+[helloweb-promise]
 recipe = slapos.cookbook:check_port_listening
-path = $${directory:promise}/hello-world
-hostname= $${hello-world:ipv6}
-port = $${hello-world:port}
+path = ${directory:promise}/helloweb-${:kind}
+
+{# macro to instantiate service of `kind` to listen on `port` #}
+{% set service_list = [] %}
+{% macro hellowebsrv(kind, port) %}
+{% do service_list.append(kind) %}
+[helloweb-{{ kind }}]
+<= helloweb
+kind    = {{ kind }}
+port    = {{ port }}
+
+[helloweb-{{ kind }}-promise]
+<= helloweb-promise
+kind    = {{ kind }}
+hostname= ${helloweb-{{ kind }}:ipv6}
+port    = {{ port }}
+{% endmacro %}
+
+# services instantiation
+{{ hellowebsrv('python', 7777) }}
+{{ hellowebsrv('ruby',   7778) }}
+
+
+# register all services/promises to buildout parts
+[buildout]
+parts +=
+{%- for kind in service_list %}
+  helloweb-{{ kind }}
+  helloweb-{{ kind }}-promise
+{%- endfor %}
 
 
 # Publish all the parameters needed for the user to connect to the instance.
@@ -102,5 +126,7 @@ port = $${hello-world:port}
 # Here we'll just echo back the entered name as instance parameter
 [publish-connection-parameter]
 recipe = slapos.cookbook:publish
-name = Hello $${instance-parameter:configuration.name}!
-url = $${hello-world:url}
+name = Hello ${instance-parameter:configuration.name}!
+{%- for kind in service_list %}
+url.{{ kind }} = ${helloweb-{{ kind }}:url}
+{%- endfor %}

software/helloworld/software.cfg

@@ -6,10 +6,10 @@ extends =
   ../../stack/slapos.cfg
 # Extend here component profiles, like openssl, apache, mariadb, curl...
 # Or/and extend a stack (lamp, tomcat) that does most of the work for you
-# In this example we don't need anything more than python which is provided by
-# above stack/slapos.cfg
+# In this example we extend from helloweb component.
 # ../../component/component1/buildout.cfg
 # ../../component/component2/buildout.cfg
+  ../../component/helloweb/buildout.cfg
 
 parts =
 # Call installation of slapos.cookbook egg defined in stack/slapos.cfg (needed
@@ -19,35 +19,21 @@ parts =
 # instance
   instance-profile
 
-# "build" python program (install + correct shebang for our python)
-  hello-web-bin
+# build helloweb programs
+  helloweb-python
+  helloweb-ruby
 
 
 # Download instance.cfg.in (buildout profile used to deployment of instance),
 # replace all ${foo:bar} parameters by real values, and change $${foo:bar} to
 # ${foo:bar}
 [instance-profile]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance.cfg.in
-output = ${buildout:directory}/instance.cfg
+recipe = slapos.recipe.template:jinja2
+template = ${:_profile_base_location_}/instance.cfg.in
+rendered = ${buildout:directory}/instance.cfg
 # MD5 checksum can be skipped for development (easier to develop), but must be filled for production
-md5sum = 968bea0fc81dc604a874c53648b7d13f
+md5sum = be7f930921376b3c8eb51c975e24721f
 mode = 0644
-
-
-# install hello-web with correct python_executable
-[hello-web-bin]
-recipe = slapos.recipe.template:jinja2
-filename = hello-web
-md5sum = da4a93ff679d40c6682859476dcf4ce0
-template = ${:_profile_base_location_}/${:filename}.in
-rendered = ${buildout:bin-directory}/${:filename}
-mode = 0755
-# XXX python_executable should be ${${buildout:python}:executable}
-#     but buildout cannot support such indirection.
-#
-# in real-cases, python software is usually installed with zc.recipe.egg
-# which cares about correctly specifiing python interpreter for
-# entry-points automatically.
+extensions = jinja2.ext.do
 context =
-    raw python_executable ${buildout:executable}
+  section buildout  buildout