tracing: Part 2 - gotrace utility

As it was said in the previous patch here goes gotrace utility to
process `//trace:event ...` and other tracing related directives.

Related excerpt from the documentation:

---- 8< ----
Gotrace

The way //trace:event and //trace:import works is via additional code being
generated for them. Whenever a package uses any //trace: directive,
it has to organize to run `gotrace gen` on its sources for them to work,
usually with the help of //go:generate. For example:

	package hello

	//go:generate gotrace gen .

	//trace:event ...

Besides `gotrace gen` gotrace has other subcommands also related to tracing,
for example `gotrace list` lists trace events a package provides.
---- 8< ----

Gotrace works by parsing and typechecking go sources via go/parse &
go/types and then for special comments generating corresponding
additional code that is supported by tracing runtime.

tracing/cmd/gotrace/.gitignore

+/gotrace

tracing/cmd/gotrace/gotrace_test.go

+// Copyright (C) 2017  Nexedi SA and Contributors.
+//                     Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"go/build"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"syscall"
+	"testing"
+
+	"lab.nexedi.com/kirr/go123/exc"
+)
+
+func xglob(t *testing.T, pattern string) []string {
+	t.Helper()
+	matchv, err := filepath.Glob(pattern)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return matchv
+}
+
+
+type TreePrepareMode int
+const (
+	TreePrepareGolden TreePrepareMode = iota // prepare golden tree - how `gotrace gen` result should look like
+	TreePrepareWork                          // prepare work tree - inital state for `gotrace gen` to run
+)
+
+// prepareTestTree copies files from src to dst recursively processing *.ok and *.rm depending on mode.
+//
+// dst should not initially exist
+func prepareTestTree(src, dst string, mode TreePrepareMode) error {
+	err := os.MkdirAll(dst, 0777)
+	if err != nil {
+		return err
+	}
+
+	return filepath.Walk(src, func(srcpath string, info os.FileInfo, err error) error {
+		if srcpath == src /* skip root */ || err != nil {
+			return err
+		}
+
+		dstpath := dst + strings.TrimPrefix(srcpath, src)
+		if info.IsDir() {
+			err := os.Mkdir(dstpath, 0777)
+			return err
+		}
+
+		// NOTE since files are walked in lexical order <f>.ok or
+		// <f>.rm is always guaranteed to go after <f>.
+
+		var isOk, isRm bool
+		if strings.HasSuffix(srcpath, ".ok") {
+			isOk = true
+			dstpath = strings.TrimSuffix(dstpath, ".ok")
+		}
+		if strings.HasSuffix(srcpath, ".rm") {
+			isRm = true
+			dstpath = strings.TrimSuffix(dstpath, ".rm")
+		}
+
+		data, err := ioutil.ReadFile(srcpath)
+		if err != nil {
+			return err
+		}
+
+		switch mode {
+		case TreePrepareGolden:
+			// ok files are written as is
+
+			// no removed files
+			if isRm {
+				return nil
+			}
+
+		case TreePrepareWork:
+			// no ok files initially
+			if isOk {
+				return nil
+			}
+			// files to remove - prepopulate with magic
+			if isRm {
+				data = []byte(magic)
+			}
+		}
+
+		err = ioutil.WriteFile(dstpath, data, info.Mode())
+		return err
+	})
+}
+
+func xprepareTree(src, dst string, mode TreePrepareMode) {
+	err := prepareTestTree(src, dst, mode)
+	exc.Raiseif(err)
+}
+
+// diffR compares two directories recursively
+func diffR(patha, pathb string) (diff string, err error) {
+	cmd := exec.Command("diff", "-urN", patha, pathb)
+	out, err := cmd.Output()
+	if e, ok := err.(*exec.ExitError); ok {
+		if e.Sys().(syscall.WaitStatus).ExitStatus() == 1 {
+			err = nil // diff signals with 1 just a difference - problem exit code is 2
+		} else {
+			err = fmt.Errorf("diff %s %s:\n%s", patha, pathb, e.Stderr)
+		}
+	}
+
+	return string(out), err
+}
+
+func TestGoTrace(t *testing.T) {
+	tmp, err := ioutil.TempDir("", "t-gotrace")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmp)
+
+	good := tmp + "/good"
+	work := tmp + "/work"
+
+	xprepareTree("testdata", good, TreePrepareGolden)
+	xprepareTree("testdata", work, TreePrepareWork)
+
+	// test build context with GOPATH set to work tree
+	var tBuildCtx = &build.Context{
+		GOARCH:     "amd64",
+		GOOS:       "linux",
+		GOROOT:     runtime.GOROOT(),
+		GOPATH:     work,
+		CgoEnabled: true,
+		Compiler:   runtime.Compiler,
+	}
+
+	// XXX autodetect (go list ?)
+	testv := []string{"a/pkg1", "b/pkg2", "c/pkg3", "d/pkg4"}
+
+	for _, tpkg := range testv {
+		// verify `gotrace gen`
+		err = tracegen(tpkg, tBuildCtx, "" /* = local imorts disabled */)
+		if err != nil {
+			t.Errorf("%v: %v", tpkg, err)
+		}
+
+		diff, err := diffR(good+"/src/"+tpkg, work+"/src/"+tpkg)
+		if err != nil {
+			t.Fatalf("%v: %v", tpkg, err)
+		}
+
+		if diff != "" {
+			t.Errorf("%v: gold & work differ:\n%s", tpkg, diff)
+		}
+
+		// verify `gotrace list`
+		var tlistBuf bytes.Buffer
+		err = tracelist(&tlistBuf, tpkg, tBuildCtx, "" /* = local imports disabled */)
+		if err != nil {
+			t.Fatalf("%v: %v", tpkg, err)
+		}
+
+		tlistOk, err := ioutil.ReadFile(work + "/src/" + tpkg + "/tracelist.txt")
+		if err != nil {
+			t.Fatalf("%v: %v", tpkg, err)
+		}
+
+		tlist := tlistBuf.Bytes()
+		if !bytes.Equal(tlist, tlistOk) {
+			t.Errorf("%v: tracelist differ:\nhave:\n%s\nwant:\n%s", tpkg, tlist, tlistOk)
+		}
+	}
+}

tracing/cmd/gotrace/testdata/src/a/pkg1/pkg1.go

+package pkg1
+
+import (
+	"net/url"
+
+	// extra import which is used in package but should not be used in tracing code
+	"fmt"
+)
+
+// probe receives no args
+//trace:event traceNewTPre()
+
+// probe receives type this package defines
+//trace:event traceNewT(t *T)
+
+type T struct {}
+
+func NewT() *T {
+	traceNewTPre()
+	t := &T{}
+	traceNewT(t)
+	return t
+}
+
+// probe receives type from another package
+//trace:event traceURLParsed(u *url.URL)
+
+func ParseURL(ustr string) (*url.URL, error) {
+	u, err := url.Parse(ustr)
+	if err != nil {
+		return nil, fmt.Errorf("oh my bad: %v", err)
+	}
+
+	traceURLParsed(u)
+	return u, nil
+}
+
+// probe receives builtin type
+//trace:event traceDoSomething(topic string)
+
+func DoSomething(topic string) {
+	traceDoSomething(topic)
+}
+
+
+// XXX do we need vvv ?
+// package-local non-exported tracepoint
+//type t struct {}
+////trace:event tracedoSomethingLocal(arg *t)

tracing/cmd/gotrace/testdata/src/a/pkg1/pkg1c.go

+package pkg1
+
+/*
+#include <stdio.h>
+
+void hello() {
+	printf("Hello World (from C)\n");
+}
+*/
+import "C"
+
+// FIXME vvv does not currently work because go/loader gives us already
+// preprocessed result without original comments.
+//
+// trace event defined in a cgo file
+//trace:event traceHello()
+
+func Hello() {
+	//traceHello()
+	C.hello()
+}

tracing/cmd/gotrace/testdata/src/a/pkg1/tracelist.txt

+a/pkg1:traceDoSomething
+a/pkg1:traceNewT
+a/pkg1:traceNewTPre
+a/pkg1:traceURLParsed

tracing/cmd/gotrace/testdata/src/a/pkg1/ztrace.go

+// Code generated by lab.nexedi.com/kirr/go123/tracing/cmd/gotrace; DO NOT EDIT.
+
+Bad bad bad - I'm invalid go file.

tracing/cmd/gotrace/testdata/src/a/pkg1/ztrace.go.ok

+// Code generated by lab.nexedi.com/kirr/go123/tracing/cmd/gotrace; DO NOT EDIT.
+
+package pkg1
+// code generated for tracepoints
+
+import (
+	"lab.nexedi.com/kirr/go123/tracing"
+	"unsafe"
+
+	"net/url"
+)
+
+// traceevent: traceDoSomething(topic string)
+
+type _t_traceDoSomething struct {
+	tracing.Probe
+	probefunc     func(topic string)
+}
+
+var _traceDoSomething *_t_traceDoSomething
+
+func traceDoSomething(topic string) {
+	if _traceDoSomething != nil {
+		_traceDoSomething_run(topic)
+	}
+}
+
+func _traceDoSomething_run(topic string) {
+	for p := _traceDoSomething; p != nil; p = (*_t_traceDoSomething)(unsafe.Pointer(p.Next())) {
+		p.probefunc(topic)
+	}
+}
+
+func traceDoSomething_Attach(pg *tracing.ProbeGroup, probe func(topic string)) *tracing.Probe {
+	p := _t_traceDoSomething{probefunc: probe}
+	tracing.AttachProbe(pg, (**tracing.Probe)(unsafe.Pointer(&_traceDoSomething)), &p.Probe)
+	return &p.Probe
+}
+
+// traceevent: traceNewT(t *T)
+
+type _t_traceNewT struct {
+	tracing.Probe
+	probefunc     func(t *T)
+}
+
+var _traceNewT *_t_traceNewT
+
+func traceNewT(t *T) {
+	if _traceNewT != nil {
+		_traceNewT_run(t)
+	}
+}
+
+func _traceNewT_run(t *T) {
+	for p := _traceNewT; p != nil; p = (*_t_traceNewT)(unsafe.Pointer(p.Next())) {
+		p.probefunc(t)
+	}
+}
+
+func traceNewT_Attach(pg *tracing.ProbeGroup, probe func(t *T)) *tracing.Probe {
+	p := _t_traceNewT{probefunc: probe}
+	tracing.AttachProbe(pg, (**tracing.Probe)(unsafe.Pointer(&_traceNewT)), &p.Probe)
+	return &p.Probe
+}
+
+// traceevent: traceNewTPre()
+
+type _t_traceNewTPre struct {
+	tracing.Probe
+	probefunc     func()
+}
+
+var _traceNewTPre *_t_traceNewTPre
+
+func traceNewTPre() {
+	if _traceNewTPre != nil {
+		_traceNewTPre_run()
+	}
+}
+
+func _traceNewTPre_run() {
+	for p := _traceNewTPre; p != nil; p = (*_t_traceNewTPre)(unsafe.Pointer(p.Next())) {
+		p.probefunc()
+	}
+}
+
+func traceNewTPre_Attach(pg *tracing.ProbeGroup, probe func()) *tracing.Probe {
+	p := _t_traceNewTPre{probefunc: probe}
+	tracing.AttachProbe(pg, (**tracing.Probe)(unsafe.Pointer(&_traceNewTPre)), &p.Probe)
+	return &p.Probe
+}
+
+// traceevent: traceURLParsed(u *url.URL)
+
+type _t_traceURLParsed struct {
+	tracing.Probe
+	probefunc     func(u *url.URL)
+}
+
+var _traceURLParsed *_t_traceURLParsed
+
+func traceURLParsed(u *url.URL) {
+	if _traceURLParsed != nil {
+		_traceURLParsed_run(u)
+	}
+}
+
+func _traceURLParsed_run(u *url.URL) {
+	for p := _traceURLParsed; p != nil; p = (*_t_traceURLParsed)(unsafe.Pointer(p.Next())) {
+		p.probefunc(u)
+	}
+}
+
+func traceURLParsed_Attach(pg *tracing.ProbeGroup, probe func(u *url.URL)) *tracing.Probe {
+	p := _t_traceURLParsed{probefunc: probe}
+	tracing.AttachProbe(pg, (**tracing.Probe)(unsafe.Pointer(&_traceURLParsed)), &p.Probe)
+	return &p.Probe
+}
+
+// trace export signature
+func _trace_exporthash_965fa599dc3a61119faba1eacf8493973c5d87ad() {}

tracing/cmd/gotrace/testdata/src/b/pkg2/pkg2.go

+package pkg2
+
+// trace-import another package
+// NOTE "a/pkg1" is not regularly imported
+//trace:import "a/pkg1"
+
+// additional tracepoint which pkg2 defines
+//trace:event traceDoSomething(i, j int, q string)
+
+func DoSomething(i, j int, q string) {
+	traceDoSomething(i, j, q)
+}

tracing/cmd/gotrace/testdata/src/b/pkg2/tracelist.txt

+b/pkg2:traceDoSomething

tracing/cmd/gotrace/testdata/src/b/pkg2/ztrace.go.ok

+// Code generated by lab.nexedi.com/kirr/go123/tracing/cmd/gotrace; DO NOT EDIT.
+
+package pkg2
+// code generated for tracepoints
+
+import (
+	"lab.nexedi.com/kirr/go123/tracing"
+	"unsafe"
+
+	"a/pkg1"
+	"net/url"
+)
+
+// traceevent: traceDoSomething(i, j int, q string)
+
+type _t_traceDoSomething struct {
+	tracing.Probe
+	probefunc     func(i, j int, q string)
+}
+
+var _traceDoSomething *_t_traceDoSomething
+
+func traceDoSomething(i, j int, q string) {
+	if _traceDoSomething != nil {
+		_traceDoSomething_run(i, j, q)
+	}
+}
+
+func _traceDoSomething_run(i, j int, q string) {
+	for p := _traceDoSomething; p != nil; p = (*_t_traceDoSomething)(unsafe.Pointer(p.Next())) {
+		p.probefunc(i, j, q)
+	}
+}
+
+func traceDoSomething_Attach(pg *tracing.ProbeGroup, probe func(i, j int, q string)) *tracing.Probe {
+	p := _t_traceDoSomething{probefunc: probe}
+	tracing.AttachProbe(pg, (**tracing.Probe)(unsafe.Pointer(&_traceDoSomething)), &p.Probe)
+	return &p.Probe
+}
+
+// trace export signature
+func _trace_exporthash_80ddfc2f6c72bdf357dedbb2f0bbec85e93106fc() {}
+
+// traceimport: "a/pkg1"
+
+// rerun "gotrace gen" if you see link failure ↓↓↓
+//go:linkname pkg1_trace_exporthash a/pkg1._trace_exporthash_965fa599dc3a61119faba1eacf8493973c5d87ad
+func pkg1_trace_exporthash()
+func init() { pkg1_trace_exporthash() }
+
+
+//go:linkname pkg1_traceDoSomething_Attach a/pkg1.traceDoSomething_Attach
+func pkg1_traceDoSomething_Attach(*tracing.ProbeGroup, func(topic string)) *tracing.Probe
+
+//go:linkname pkg1_traceNewT_Attach a/pkg1.traceNewT_Attach
+func pkg1_traceNewT_Attach(*tracing.ProbeGroup, func(t *pkg1.T)) *tracing.Probe
+
+//go:linkname pkg1_traceNewTPre_Attach a/pkg1.traceNewTPre_Attach
+func pkg1_traceNewTPre_Attach(*tracing.ProbeGroup, func()) *tracing.Probe
+
+//go:linkname pkg1_traceURLParsed_Attach a/pkg1.traceURLParsed_Attach
+func pkg1_traceURLParsed_Attach(*tracing.ProbeGroup, func(u *url.URL)) *tracing.Probe

tracing/cmd/gotrace/testdata/src/b/pkg2/ztrace.s.ok

+// Code generated by lab.nexedi.com/kirr/go123/tracing/cmd/gotrace; DO NOT EDIT.
+// empty .s so `go build` does not use -complete for go:linkname to work

tracing/cmd/gotrace/testdata/src/c/pkg3/pkg3.go

+package pkg3
+
+func zzz() int {
+	return 1
+}

tracing/cmd/gotrace/testdata/src/c/pkg3/pkg3_external_test.go

+package pkg3_test
+
+import "testing"
+
+// trace import that should be added only to external tests
+//trace:import "b/pkg2"
+
+func TestZzzExternal(t *testing.T) {
+	t.Fatal("...")
+}

tracing/cmd/gotrace/testdata/src/c/pkg3/pkg3_test.go

+package pkg3
+
+import "testing"
+
+// trace import that should be added only to tests, and under specified package name
+//trace:import aaa1 "a/pkg1"
+
+func TestZzz(t *testing.T) {
+	if zzz() != 2 {
+		t.Fatal("zzz wrong")
+	}
+}

tracing/cmd/gotrace/testdata/src/c/pkg3/ztrace_test.go.ok

+// Code generated by lab.nexedi.com/kirr/go123/tracing/cmd/gotrace; DO NOT EDIT.
+
+package pkg3
+// code generated for tracepoints
+
+import (
+	"lab.nexedi.com/kirr/go123/tracing"
+	_ "unsafe"
+
+	aaa1 "a/pkg1"
+	"net/url"
+)
+
+// traceimport: aaa1 "a/pkg1"
+
+// rerun "gotrace gen" if you see link failure ↓↓↓
+//go:linkname aaa1_trace_exporthash a/pkg1._trace_exporthash_965fa599dc3a61119faba1eacf8493973c5d87ad
+func aaa1_trace_exporthash()
+func init() { aaa1_trace_exporthash() }
+
+
+//go:linkname aaa1_traceDoSomething_Attach a/pkg1.traceDoSomething_Attach
+func aaa1_traceDoSomething_Attach(*tracing.ProbeGroup, func(topic string)) *tracing.Probe
+
+//go:linkname aaa1_traceNewT_Attach a/pkg1.traceNewT_Attach
+func aaa1_traceNewT_Attach(*tracing.ProbeGroup, func(t *aaa1.T)) *tracing.Probe
+
+//go:linkname aaa1_traceNewTPre_Attach a/pkg1.traceNewTPre_Attach
+func aaa1_traceNewTPre_Attach(*tracing.ProbeGroup, func()) *tracing.Probe
+
+//go:linkname aaa1_traceURLParsed_Attach a/pkg1.traceURLParsed_Attach
+func aaa1_traceURLParsed_Attach(*tracing.ProbeGroup, func(u *url.URL)) *tracing.Probe

tracing/cmd/gotrace/testdata/src/c/pkg3/ztrace_test.s.ok

+// Code generated by lab.nexedi.com/kirr/go123/tracing/cmd/gotrace; DO NOT EDIT.
+// empty .s so `go build` does not use -complete for go:linkname to work

tracing/cmd/gotrace/testdata/src/c/pkg3/ztrace_x_test.go.ok

+// Code generated by lab.nexedi.com/kirr/go123/tracing/cmd/gotrace; DO NOT EDIT.
+
+package pkg3_test
+// code generated for tracepoints
+
+import (
+	"lab.nexedi.com/kirr/go123/tracing"
+	_ "unsafe"
+)
+
+// traceimport: "b/pkg2"
+
+// rerun "gotrace gen" if you see link failure ↓↓↓
+//go:linkname pkg2_trace_exporthash b/pkg2._trace_exporthash_80ddfc2f6c72bdf357dedbb2f0bbec85e93106fc
+func pkg2_trace_exporthash()
+func init() { pkg2_trace_exporthash() }
+
+
+//go:linkname pkg2_traceDoSomething_Attach b/pkg2.traceDoSomething_Attach
+func pkg2_traceDoSomething_Attach(*tracing.ProbeGroup, func(i, j int, q string)) *tracing.Probe

tracing/cmd/gotrace/testdata/src/c/pkg3/ztrace_x_test.s.ok

+// Code generated by lab.nexedi.com/kirr/go123/tracing/cmd/gotrace; DO NOT EDIT.
+// empty .s so `go build` does not use -complete for go:linkname to work

tracing/cmd/gotrace/testdata/src/d/pkg4/pkg4.go

+package pkg4
+
+// this package does not use tracepoints at all

tracing/cmd/gotrace/util.go

+// Copyright (C) 2017  Nexedi SA and Contributors.
+//                     Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+package main
+// misc utilities
+
+import (
+	"bytes"
+	"fmt"
+	"sort"
+)
+
+// Buffer is bytes.Buffer + syntatic sugar
+type Buffer struct {
+	bytes.Buffer
+}
+
+func (b *Buffer) emit(format string, argv ...interface{}) {
+	fmt.Fprintf(b, format+"\n", argv...)
+}
+
+
+// StrSet is set<string>
+type StrSet map[string]struct{}
+
+func (s StrSet) Add(itemv ...string) {
+	for _, item := range itemv {
+		s[item] = struct{}{}
+	}
+}
+
+func (s StrSet) Delete(item string) {
+	delete(s, item)
+}
+
+func (s StrSet) Has(item string) bool {
+	_, has := s[item]
+	return has
+}
+
+// Itemv returns ordered slice of set items
+func (s StrSet) Itemv() []string {
+	itemv := make([]string, 0, len(s))
+	for item := range s {
+		itemv = append(itemv, item)
+	}
+	sort.Strings(itemv)
+	return itemv
+}