Merge remote-tracking branch 'kirrlab/y/go' into x/gitlab

component/golang/buildout.cfg

+# Go language - https://golang.org/
+[buildout]
+parts = golang
+
+[golang]
+<= golang15
+
+
+[golang-common]
+recipe = slapos.recipe.cmmi
+configure-command = :
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+make-binary =
+make-targets= cd src && ./all.bash && cp -al .. ${:location}
+environment =
+  GOROOT_FINAL=${:location}
+  ${:environment-extra}
+
+
+[golang14]
+<= golang-common
+url = https://storage.googleapis.com/golang/go1.4.3.src.tar.gz
+md5sum = dfb604511115dd402a77a553a5923a04
+environment-extra =
+
+
+[golang15]
+<= golang-common
+url = https://storage.googleapis.com/golang/go1.5.1.src.tar.gz
+md5sum = 4adfbdfca523cc1c229be8a321f3602f
+
+# go1.5 needs go1.4 too bootstrap
+environment-extra =
+  GOROOT_BOOTSTRAP=${golang14:location}

component/helloweb/buildout.cfg

+# helloweb - programs to say hello to the Web in various languages
+[buildout]
+extends =
+  ../git/buildout.cfg
+  ../ruby/buildout.cfg
+  ../golang/buildout.cfg
+
+parts =
+  helloweb-python
+  helloweb-ruby
+  helloweb-go
+
+
+# repository with examples
+[helloweb-repository]
+recipe  = slapos.recipe.build:gitclone
+git-executable = ${git:location}/bin/git
+repository = https://lab.nexedi.com/kirr/helloweb.git
+revision = 0487fa7bc88e83ce8b97d784ab11354aa8fac603
+location = ${buildout:parts-directory}/helloweb
+
+
+# -*- python -*-
+[helloweb-egg]
+recipe  = zc.recipe.egg:develop
+egg     = helloweb
+setup   = ${helloweb-repository:location}/python/
+
+[helloweb-python]
+recipe  = zc.recipe.egg:scripts
+eggs    = ${helloweb-egg:egg}
+scripts = helloweb=helloweb-python
+
+
+# -*- ruby -*-
+
+# if ruby program is represented as already-released gem, we can install it
+# with `gem install ...` (via rubygemsrecipe).
+#
+# Alternatively if we need to
+# install the program from source-checkout, the Ruby way is to use bundler,
+# install program dependencies via it, and run the program itself via it.
+#
+# Since for helloweb.rb we have source checkout - we go the second - bundler way.
+
+# bundler, that we'll use to install gems and run binaries (via `bundle exec ...`)
+[bundler]
+# rubygemsrecipe with fixed url and this way pinned rubygems version
+recipe  = rubygemsrecipe
+url     = https://rubygems.org/rubygems/rubygems-2.4.8.zip
+ruby-location = ${ruby2.1:location}
+ruby-executable = ${:ruby-location}/bin/ruby
+gems    = bundler==1.10.6
+
+# bin installed here
+bundle  = ${buildout:bin-directory}/bundle
+
+# install together with path to ruby enabled
+# ( reason: rubygemsrecipe hardcodes PATH inside generated bin/* and it is
+#   impossible to adjust it later )
+#
+# bundle exec <smth>                ; <smth> starts with `#!/usr/bin/env ruby` as rubygems
+environment =
+  PATH    = ${:ruby-location}/bin:%(PATH)s
+
+
+[helloweb-ruby-bundle]
+recipe  = slapos.recipe.cmmi
+path    = ${helloweb-repository:location}/ruby/
+
+configure-command = :
+make-binary =
+make-targets= cd ${:path} && ${bundler:bundle} install
+
+
+[helloweb-ruby]
+recipe  = slapos.cookbook:wrapper
+wrapper-path = ${buildout:bin-directory}/${:_buildout_section_name_}
+environment  =
+    BUNDLE_GEMFILE = ${helloweb-ruby-bundle:path}/Gemfile
+command-line =
+    ${bundler:bundle} exec sh -c 'helloweb.rb "$@"' ${:_buildout_section_name_}
+
+
+# -*- go -*-
+[helloweb-go]
+recipe  = slapos.recipe.cmmi
+path    = ${helloweb-repository:location}/go/
+
+go      = ${golang15:location}/bin/go
+configure-command = :
+make-binary =
+make-targets= cd ${:path} &&
+     ${:go} build
+        -o ${buildout:bin-directory}/${:_buildout_section_name_}
+        helloweb.go

software/helloworld/hello-web.in

-#!{{ python_executable }}
-"""Simple web-server that says "Hello World" for every path
-
-hello-web [--logfile <logfile>] <bind-ip> <bind-port> ...
-"""
-
-import sys
-import time
-import argparse
-from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
-from socket import AF_INET6
-
-
-class WebHello(BaseHTTPRequestHandler):
-
-    def do_GET(self):
-        self.send_response(200) # ok
-        self.send_header("Content-type", "text/plain")
-        self.end_headers()
-
-        print >>self.wfile, \
-            "Hello %s at `%s`  ; %s" % (
-                ' '.join(self.server.webhello_argv) or 'world',
-                self.path, time.asctime())
-
-
-class HTTPServerV6(HTTPServer):
-    address_family = AF_INET6
-
-def main():
-    parser = argparse.ArgumentParser()
-    parser.add_argument('--logfile', dest='logfile')
-    parser.add_argument('bind_ip')
-    parser.add_argument('bind_port', type=int)
-    parser.add_argument('argv_extra', metavar='...', nargs=argparse.REMAINDER)
-
-    args = parser.parse_args()
-
-    # HTTPServer logs to sys.stderr - override it if we have --logfile
-    if args.logfile:
-        f = open(args.logfile, 'a', buffering=1)
-        sys.stderr = f
-
-    print >>sys.stderr, '* %s Hello-Web starting at %s' % (
-        time.asctime(), (args.bind_ip, args.bind_port))
-
-    # TODO autodetect ipv6/ipv4
-    httpd = HTTPServerV6( (args.bind_ip, args.bind_port), WebHello)
-    httpd.webhello_argv = args.argv_extra
-    httpd.serve_forever()
-
-if __name__ == '__main__':
-    main()

software/helloworld/instance.cfg.in

@@ -6,15 +6,13 @@
 [buildout]
 parts =
   directory
-  hello-world
-  hello-world-promise
   publish-connection-parameter
 
 # Define egg directories to be the one from Software Release
 # (/opt/slapgrid/...)
 # Always the same.
-eggs-directory = ${buildout:eggs-directory}
-develop-eggs-directory = ${buildout:develop-eggs-directory}
+eggs-directory = {{ buildout['eggs-directory'] }}
+develop-eggs-directory = {{ buildout['develop-eggs-directory'] }}
 offline = true
 
 
@@ -23,15 +21,15 @@ offline = true
 # We use the slapconfiguration recipe with a few parameters (partition id,
 # computer id, certificate, etc).
 # It will then authenticate to SlapOS Master and fetch the instance parameters.
-# The parameters are accessible from $${instance-parameter:configuration.name-of-parameter}
+# The parameters are accessible from ${instance-parameter:configuration.name-of-parameter}
 # Always the same. Just copy/paste.
 # See docstring of slapos.cookbook:slapconfiguration for more information.
 recipe = slapos.cookbook:slapconfiguration
-computer = $${slap_connection:computer_id}
-partition = $${slap_connection:partition_id}
-url = $${slap_connection:server_url}
-key = $${slap_connection:key_file}
-cert = $${slap_connection:cert_file}
+computer = ${slap_connection:computer_id}
+partition = ${slap_connection:partition_id}
+url = ${slap_connection:server_url}
+key = ${slap_connection:key_file}
+cert = ${slap_connection:cert_file}
 
 # Define default parameter(s) that will be used later, in case user didn't
 # specify it.
@@ -46,22 +44,22 @@ configuration.name = John Doe
 # Create all needed directories, depending on your needs
 [directory]
 recipe = slapos.cookbook:mkdirectory
-home = $${buildout:directory}
-etc = $${:home}/etc
-var = $${:home}/var
+home = ${buildout:directory}
+etc = ${:home}/etc
+var = ${:home}/var
 # Executables put here will be started but not monitored (for startup scripts)
-script = $${:etc}/run/
+script = ${:etc}/run/
 # Executables put here will be started and monitored (for daemons)
-service = $${:etc}/service
+service = ${:etc}/service
 # Executables put here will be launched after buildout has completed to see
 # if instance is running
-promise = $${:etc}/promise/
-# Path of the log directory used by our service (see [hello-world])
-log = $${:var}/log
+promise = ${:etc}/promise/
+# Path of the log directory used by our service (see [helloweb])
+log = ${:var}/log
 
 
 # Create a simple web server that says "hello <configuration.name>" to the web.
-[hello-world]
+[helloweb]
 # helloworld service is listening on:
 # - global IPv6 address, and
 # - fixed port
@@ -69,32 +67,59 @@ log = $${:var}/log
 # NOTE because every computer partition is allocated its own global IPv6
 # address, it is ok to fix the port - different hello-world instances will have
 # different IPv6 addresses and they all will be accessible at the same time.
-ipv6 = $${instance-parameter:ipv6-random}
-port = 7777
+ipv6 = ${instance-parameter:ipv6-random}
 # full URL - for convenience
-url = http://[$${:ipv6}]:$${:port}
+url = http://[${:ipv6}]:${:port}
 
 # the service will log here
-logfile = $${directory:log}/hello-world.log
+logfile = ${directory:log}/helloweb-${:kind}.log
 
 # Actual script that starts the service:
 # This recipe will try to "exec" the command-line after separating parameters.
 recipe = slapos.cookbook:wrapper
 command-line =
-    ${hello-web-bin:rendered} --logfile $${hello-world:logfile}
-        $${:ipv6} $${:port} $${instance-parameter:configuration.name}
+    {{ buildout['bin-directory'] }}/helloweb-${:kind} --logfile ${:logfile}
+        ${:ipv6} ${:port} ${instance-parameter:configuration.name}
 # Put this shell script in the "etc/service" directory. Each executable of this
 # repository will be started and monitored by supervisord. If a service
 # exits/crashes, it will trigger a "bang" and cause a re-run of the instance.
-wrapper-path = $${directory:service}/hello-world
+wrapper-path = ${directory:service}/helloweb-${:kind}
 
 
-# promise, that checks that hello-world service is alive
-[hello-world-promise]
+# promise, that checks that helloweb service is alive
+[helloweb-promise]
 recipe = slapos.cookbook:check_port_listening
-path = $${directory:promise}/hello-world
-hostname= $${hello-world:ipv6}
-port = $${hello-world:port}
+path = ${directory:promise}/helloweb-${:kind}
+
+{# macro to instantiate service of `kind` to listen on `port` #}
+{% set service_list = [] %}
+{% macro hellowebsrv(kind, port) %}
+{% do service_list.append(kind) %}
+[helloweb-{{ kind }}]
+<= helloweb
+kind    = {{ kind }}
+port    = {{ port }}
+
+[helloweb-{{ kind }}-promise]
+<= helloweb-promise
+kind    = {{ kind }}
+hostname= ${helloweb-{{ kind }}:ipv6}
+port    = {{ port }}
+{% endmacro %}
+
+# services instantiation
+{{ hellowebsrv('python', 7777) }}
+{{ hellowebsrv('ruby',   7778) }}
+{{ hellowebsrv('go',     7779) }}
+
+
+# register all services/promises to buildout parts
+[buildout]
+parts +=
+{%- for kind in service_list %}
+  helloweb-{{ kind }}
+  helloweb-{{ kind }}-promise
+{%- endfor %}
 
 
 # Publish all the parameters needed for the user to connect to the instance.
@@ -102,5 +127,7 @@ port = $${hello-world:port}
 # Here we'll just echo back the entered name as instance parameter
 [publish-connection-parameter]
 recipe = slapos.cookbook:publish
-name = Hello $${instance-parameter:configuration.name}!
-url = $${hello-world:url}
+name = Hello ${instance-parameter:configuration.name}!
+{%- for kind in service_list %}
+url.{{ kind }} = ${helloweb-{{ kind }}:url}
+{%- endfor %}

software/helloworld/software.cfg

@@ -6,10 +6,10 @@ extends =
   ../../stack/slapos.cfg
 # Extend here component profiles, like openssl, apache, mariadb, curl...
 # Or/and extend a stack (lamp, tomcat) that does most of the work for you
-# In this example we don't need anything more than python which is provided by
-# above stack/slapos.cfg
+# In this example we extend from helloweb component.
 # ../../component/component1/buildout.cfg
 # ../../component/component2/buildout.cfg
+  ../../component/helloweb/buildout.cfg
 
 parts =
 # Call installation of slapos.cookbook egg defined in stack/slapos.cfg (needed
@@ -19,35 +19,22 @@ parts =
 # instance
   instance-profile
 
-# "build" python program (install + correct shebang for our python)
-  hello-web-bin
+# build helloweb programs
+  helloweb-python
+  helloweb-ruby
+  helloweb-go
 
 
 # Download instance.cfg.in (buildout profile used to deployment of instance),
 # replace all ${foo:bar} parameters by real values, and change $${foo:bar} to
 # ${foo:bar}
 [instance-profile]
-recipe = slapos.recipe.template
-url = ${:_profile_base_location_}/instance.cfg.in
-output = ${buildout:directory}/instance.cfg
+recipe = slapos.recipe.template:jinja2
+template = ${:_profile_base_location_}/instance.cfg.in
+rendered = ${buildout:directory}/instance.cfg
 # MD5 checksum can be skipped for development (easier to develop), but must be filled for production
-md5sum = 968bea0fc81dc604a874c53648b7d13f
+md5sum = 6567f8dedb5cdd93542dc29e96edb547
 mode = 0644
-
-
-# install hello-web with correct python_executable
-[hello-web-bin]
-recipe = slapos.recipe.template:jinja2
-filename = hello-web
-md5sum = da4a93ff679d40c6682859476dcf4ce0
-template = ${:_profile_base_location_}/${:filename}.in
-rendered = ${buildout:bin-directory}/${:filename}
-mode = 0755
-# XXX python_executable should be ${${buildout:python}:executable}
-#     but buildout cannot support such indirection.
-#
-# in real-cases, python software is usually installed with zc.recipe.egg
-# which cares about correctly specifiing python interpreter for
-# entry-points automatically.
+extensions = jinja2.ext.do
 context =
-    raw python_executable ${buildout:executable}
+  section buildout  buildout

software/slaprunner/common.cfg

@@ -156,7 +156,7 @@ mode = 0644
 [template-supervisord]
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/template/${:filename}
-md5sum = 069e593e50204b227bdb08d29d7292fd
+md5sum = d294d0dafd265048399de6da8c96345f
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
 filename = supervisord.conf.in
 download-only = true

software/slaprunner/template/supervisord.conf.in

@@ -26,7 +26,9 @@ autorestart = {{ supervisord['autorestart'] }}
 stdout_logfile = {{ supervisord['no_logfile'] }}
 stderr_logfile = {{ supervisord['no_logfile'] }}
 directory = {{ supervisord['directory'] }}
-environment = PATH="{{- supervisord['path'] -}}",MAKEFLAGS="-j{{- '%d' % builtin.max(1, (multiprocessing.cpu_count() / builtin.int(slapparameter_dict.get('cpu-usage-ratio', 4)))) -}}"
+{# how many parallel build jobs to spawn when compiling software -#}
+{% set njobs = builtin.max(1, (multiprocessing.cpu_count() // builtin.int(slapparameter_dict.get('cpu-usage-ratio', 4)))) -%}
+environment = PATH="{{- supervisord['path'] -}}",MAKEFLAGS="-j{{ njobs }}",NPY_NUM_BUILD_JOBS="{{ njobs }}",BUNDLE_JOBS="{{ njobs }}"
 
 [program:{{- supervisord['slapgrid-cp'] -}}]
 command = {{ supervisord['slapgrid-cp-command'] }}