golang: Infrastructure to build Go workspaces / projects

In Go world development workflow is organized around so-called workspace
where multiple packages can be installed / worked on etc. The following
page describes Go workspaces:

https://golang.org/doc/code.html

A new [gowork] section and infrastructure around it is introduced.
Quoting code:

    # gowork is a top-level section representing workspace
    #
    # users should add `install` field to [gowork] to describe packages they want to
    # be installed (+ automatically their dependencies are installed too). e.g.
    #
    #   [gowork]
    #   install =
    #       lab.nexedi.com/kirr/neo/go/...  \
    #       github.com/pkg/profile          \
    #       golang.org/x/perf/cmd/benchstat

The way it all works inside is:

- gowork organizes to create go.work/ directory in buildout root
- inside it creates env.sh which when sources in shell adjusts all paths
  so that appropriate go compiler is in path, GOPATH is also set
  appropriately, etc - in other words everything needs for using/working
  on this workspace is setup in the environment.
- in actual user software profile the list of Go projects which needs to
  be installed by SlapOS has to be listed. The [gowork] machinery takes
  care about git-cloning these projects and `go install`s them after.
- by SlapOS design builds have to be reproducible and so every component
  state/version has to be fixed.

  A convenient helper (gowork-snapshot) to get snapshot of git go
  packages installed with their exact revisions is added. This should be
  used to automatically (re-)generate list of go git repositories &
  their pinning for a workspace.

For a new Go project that needs to be slaposified the workflow should be:

- first get a project into working state via any convenient way (`go get`, etc.)
- generate list of go packages & their pinning with gowork-snapshot
- extend golang/buildout.cfg and in project software profile add

	[gowork]
  	install = your-top-level-packages

  as it was outlined above.

P.S.

[golang] is removed because

1. it is not used anywhere in slapos.git tree, and
2. the canonical way to work on go projects from now on will be to use
   [gowork]. There latest go version is preconfigured, but users can
   override it to some particular other version in their projects as they
   need.

/reviewed-on	 nexedi/slapos!242
/also-needed-for nexedi/slapos!243

component/golang/buildout.cfg

@@ -2,12 +2,11 @@
 [buildout]
 extends =
   ../findutils/buildout.cfg
+  ../git/buildout.cfg
 
-parts = golang
-
-[golang]
-<= golang19
+parts = gowork
 
+# ---- Go builds itself ----
 
 [golang-common]
 recipe = slapos.recipe.cmmi
@@ -46,3 +45,73 @@ md5sum = 27bce1ffb05f4f6bd90d90081e5d4169
 # go1.9 needs go1.4 to bootstrap
 environment-extra =
   GOROOT_BOOTSTRAP=${golang14:location}
+
+
+# ---- infrastructure to build Go workspaces / projects ----
+
+# gowork is a top-level section representing workspace
+#
+# users should add `install` field to [gowork] to describe packages they want to
+# be installed (+ automatically their dependencies are installed too). e.g.
+#
+#   [gowork]
+#   install =
+#       lab.nexedi.com/kirr/neo/go/...  \
+#       github.com/pkg/profile          \
+#       golang.org/x/perf/cmd/benchstat
+[gowork]
+directory = ${buildout:directory}/go.work
+src	= ${:directory}/src
+bin	= ${:directory}/bin
+depends = ${gowork.goinstall:recipe}
+
+# go version used for the workspace (possible to override in applications)
+golang  = ${golang19:location}
+
+# everything is done by dependent parts
+recipe  = plone.recipe.command
+command = :
+
+# env.sh for compiling and running go programs
+[gowork]
+env.sh  = ${gowork-env.sh:output}
+
+[gowork-env.sh]
+recipe	= slapos.recipe.template
+url     = ${:_profile_base_location_}/goenv.sh.in
+output	= ${gowork:directory}/env.sh
+depends = ${gowork.mkdir:recipe}
+md5sum	= a9a265135931b3da53f4392870748264
+
+[gowork.mkdir]
+# NOTE do not use slapos.cookbook:mkdirectory here - if anything in software (not instance)
+# uses slapos.cookbook:* in recipe - slapos.cookbook will get compiled against system
+# libxml/libxslt or fail to bootstrap at all if those are not present.
+recipe  = plone.recipe.command
+command = mkdir -p ${gowork:directory}
+update-command = ${:command}
+stop-on-error = true
+
+# install go packages
+# clients should put package list to install to gowork:install ("..." requests installing everything)
+[gowork.goinstall]
+recipe  = plone.recipe.command
+command = bash -c ". ${gowork:env.sh} && go install -v ${gowork:install}"
+update-command = ${:command}
+stop-on-error = true
+
+
+[git-repository]
+recipe  = slapos.recipe.build:gitclone
+git-executable = ${git:location}/bin/git
+
+# a go package should:
+# 1) <= go-git-package
+# 2) provide go.importpath
+# 3) provide repository (which is not the same as importpath in general case)
+#
+# the list of go packages for a go workspace state can be automatically
+# generated with the help of go-pkg-snapshot tool.
+[go-git-package]
+<= git-repository
+location = ${gowork:src}/${:go.importpath}

component/golang/goenv.sh.in

+# env.sh for a Go workspace
+# Usage: env.sh [/path/to/env.sh]
+
+# ---- 8< ---- (buildout substitution here)
+# PATH so that go & friends work out of the box
+export PATH=${gowork:golang}/bin:${git:location}/bin:${buildout:bin-directory}:$PATH
+X=${gowork:directory}
+# ---- 8< ----
+
+export GOPATH=$X:$GOPATH
+export PATH=$X/bin:$PATH
+export PS1="(`basename $X`) $PS1"
+
+# strip trailing : from $GOPATH
+GOPATH=$${GOPATH%:}

component/golang/gowork-snapshot

+#!/bin/bash
+# gowork-snapshot - find out installed go packages and produce buildout code to install them pinned
+
+# FIXME currently works only with cwd=gowork/src
+
+echo "# Code generated by gowork-snapshot; DO NOT EDIT."
+
+# list installed go git repositories.
+#
+# this gives something like:
+# github.com/cznic/strutil  https://github.com/cznic/strutil    529a34b1c1
+# golang.org/x/net          https://go.googlesource.com/net     1087133bc4
+# ...
+gogit_list() {
+    find . -name .git | sort | \
+    while read repo; do
+        importpath=${repo#./}
+        importpath=${importpath%/.git}
+        echo -ne "${importpath}\t"
+        echo -ne "`git_upstream_url $repo`\t"
+        git -C $repo describe --long --always --abbrev=10
+    done
+}
+
+# git_upstream_url <repo>   - show current branch upstream URL
+git_upstream_url() {
+    repo=$1
+    head="`git -C $repo symbolic-ref --short HEAD`"             # current branch - e.g. "t"
+    remote="`git -C $repo config --get branch.$head.remote`"    # upstream name, e.g. "kirr"
+    url="`git -C $repo config --get remote.$remote.url`"        # upstream URL
+    echo "$url"
+}
+
+# buildout_safe <name>  - canonicalize name to be allowed to be used in buildout section name
+# e.g. aaa/bbb -> aaa_bbb
+#
+# XXX can't use e.g. "go!lab.nexedi.com/kirr/neo" because buildout disallows
+# "!" or "/" in section name references.
+buildout_safe() {
+    # see _simple regex in slapos.buildout
+    echo -n "$1" | tr --complement -- "-a-zA-Z0-9 ._" _
+}
+
+echo
+echo "# list of go git repositories to fetch"
+echo "[gowork.goinstall]"
+echo "depends_gitfetch  ="
+
+gogit_list | \
+while read pkg _ _; do
+        echo "    \${go_`buildout_safe $pkg`:recipe}"
+done
+
+echo
+gogit_list | \
+while read pkg url rev; do
+    echo
+    echo "[go_`buildout_safe $pkg`]"
+    echo "<= go-git-package"
+    echo "go.importpath = $pkg"
+    echo "repository    = $url"
+    echo "revision  = $rev"
+done