cmd/dist: new command

dist is short for distribution.  This is the new Go distribution tool.

The plan is to replace the Makefiles with what amounts to
'go tool dist bootstrap', although it cannot be invoked like
that since it is in charge of getting us to the point where we
can build the go command.

It will also add additional commands to replace bash scripts
like test/run (go tool dist testrun), eventually eliminating our
dependence on not just bash but all the Unix tools and all
of cygwin.

This is strong enough to build (cc *.c) and run (a.out bootstrap)
to build not just the C libraries and tools but also the basic
Go packages up to the bootstrap form of the go command
(go_bootstrap).  I've run it successfully on both Linux and Windows.
This means that once we've switched to this tool in the build,
we can delete the buildscripts.

This tool is not nearly as nice as the go tool.  There are many
special cases that turn into simple if statements or tables in
the code.  Please forgive that.  C does not enjoy the benefits
that we designed into Go.

I was planning to wait to do this until after Go 1, but the
Windows builders are both broken due to a bug in either
make or bash or both involving the parsing of quoted command
arguments.  Make thinks it is invoking

        quietgcc -fno-common -I"c:/go/include" -ggdb -O2 -c foo.c

but bash (quietgcc is a bash script) thinks it is being invoked as

        quietgcc -fno-common '-Ic:/go/include -ggdb' -O2 -c foo.c

which obviously does not have the desired effect.  Rather than fight
these clumsy ports, I accelerated the schedule for the new tool.
We should be completely off cygwin (using just the mingw gcc port,
which is much more standalone) before Go 1.

It is big for a single CL, and for that I apologize.  I can cut it into
separate CLs along file boundaries if people would prefer that.

R=golang-dev, adg, gri, bradfitz, alex.brainman, dsymonds, iant, ality, hcwfrichter
CC=golang-dev
https://golang.org/cl/5620045

src/cmd/dist/README

+This program, dist, is the bootstrapping tool for the Go distribution.
+It takes care of building the C programs (like the Go compiler) and
+the initial bootstrap copy of the go tool.  It also serves as a catch-all
+to replace odd jobs previously done with shell scripts.
+
+Dist is itself written in very simple C.  All interaction with C libraries,
+even standard C libraries, is confined to a single system-specific file
+(plan9.c, unix.c, windows.c), to aid portability.  Functionality needed
+by other files should be exposed via the portability layer.  Functions
+in the portability layer begin with an x prefix when they would otherwise
+use the same name as or be confused for an existing function.
+For example, xprintf is the portable printf.
+
+By far the most common data types in dist are strings and arrays of
+strings.  Instead of using char* and char**, though, dist uses two named
+data structures, Buf and Vec, which own all the data they point at.
+The Buf operations are functions beginning with b; the Vec operations
+are functions beginning with v.  The basic form of any function declaring
+Bufs or Vecs on the stack should be
+
+	void
+	myfunc(void)
+	{
+		Buf b1, b2;
+		Vec v1;
+		
+		binit(&b1);
+		binit(&b2);
+		vinit(&v1);
+		
+		... main code ...
+		bprintf(&b1, "hello, world");
+		vadd(&v1, bstr(&b1));  // v1 takes a copy of its argument
+		bprintf(&b1, "another string");
+		vadd(&v1, bstr(&b1));  // v1 now has two strings
+		
+		bfree(&b1);
+		bfree(&b2);
+		vfree(&v1);
+	}
+	
+The binit/vinit calls prepare a buffer or vector for use, initializing the 
+data structures, and the bfree/vfree calls free any memory they are still
+holding onto.  Use of this idiom gives us lexically scoped allocations.
+

src/cmd/dist/a.h

+// Copyright 2012 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+typedef int bool;
+
+// The Time unit is unspecified; we just need to
+// be able to compare whether t1 is older than t2 with t1 < t2.
+typedef long long Time;
+
+#define nil ((void*)0)
+#define nelem(x) (sizeof(x)/sizeof((x)[0]))
+#define USED(x) ((void)(x))
+
+// A Buf is a byte buffer, like Go's []byte.
+typedef struct Buf Buf;
+struct Buf
+{
+	char *p;
+	int len;
+	int cap;
+};
+
+// A Vec is a string vector, like Go's []string.
+typedef struct Vec Vec;
+struct Vec
+{
+	char **p;
+	int len;
+	int cap;
+};
+
+// Modes for run.
+enum {
+	CheckExit = 1,
+};
+
+// buf.c
+bool	bequal(Buf *s, Buf *t);
+void	bfree(Buf *b);
+void	bgrow(Buf *b, int n);
+void	binit(Buf *b);
+char*	bprintf(Buf *b, char *fmt, ...);
+void	breset(Buf *b);
+char*	bstr(Buf *b);
+char*	btake(Buf *b);
+void	bwrite(Buf *b, void *v, int n);
+void	bwriteb(Buf *dst, Buf *src);
+void	bwritestr(Buf *b, char *p);
+void	bswap(Buf *b, Buf *b1);
+void	vadd(Vec *v, char *p);
+void	vcopy(Vec *dst, char **src, int n);
+void	vfree(Vec *v);
+void	vgrow(Vec *v, int n);
+void	vinit(Vec *v);
+void	vreset(Vec *v);
+void	vuniq(Vec *v);
+void	splitlines(Vec*, char*);
+void	splitfields(Vec*, char*);
+
+// build.c
+extern char *default_goroot;
+extern char *goarch;
+extern char *gobin;
+extern char *gohostarch;
+extern char *gohostos;
+extern char *goos;
+extern char *goroot;
+extern char *workdir;
+extern char *slash;
+
+void	init(void);
+void	cmdbootstrap(int, char**);
+void	cmdenv(int, char**);
+void	cmdinstall(int, char**);
+
+// buildgc.c
+void	gcopnames(char*, char*);
+void	mkenam(char*, char*);
+
+// main.c
+void	xmain(int argc, char **argv);
+
+// portability layer (plan9.c, unix.c, windows.c)
+bool	contains(char *p, char *sep);
+void	fatal(char *msg, ...);
+bool	hasprefix(char *p, char *prefix);
+bool	hassuffix(char *p, char *suffix);
+bool	isabs(char*);
+bool	isdir(char *p);
+bool	isfile(char *p);
+char*	lastelem(char*);
+Time	mtime(char*);
+void	readfile(Buf*, char*);
+void	run(Buf *b, char *dir, int mode, char *cmd, ...);
+void	runv(Buf *b, char *dir, int mode, Vec *argv);
+bool	streq(char*, char*);
+void	writefile(Buf*, char*);
+void	xatexit(void (*f)(void));
+void	xexit(int);
+void	xfree(void*);
+void	xgetenv(Buf *b, char *name);
+void	xgetwd(Buf *b);
+void*	xmalloc(int n);
+void*	xmalloc(int);
+int	xmemcmp(void*, void*, int);
+void	xmemmove(void*, void*, int);
+void	xmkdir(char *p);
+void	xmkdirall(char*);
+Time	xmtime(char *p);
+void	xprintf(char*, ...);
+void	xqsort(void*, int, int, int(*)(const void*, const void*));
+void	xreaddir(Vec *dst, char *dir);
+void*	xrealloc(void*, int);
+void	xrealwd(Buf *b, char *path);
+void	xremove(char *p);
+void	xremoveall(char *p);
+void	xsetenv(char*, char*);
+int	xstrcmp(char*, char*);
+char*	xstrdup(char *p);
+int	xstreq(char*, char*);
+int	xstrlen(char*);
+char*	xstrrchr(char*, int);
+char*	xstrstr(char*, char*);
+char*	xworkdir(void);

src/cmd/dist/buf.c

+// Copyright 2012 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Byte buffers and string vectors.
+
+#include "a.h"
+
+// binit prepares an uninitialized buffer for use.
+void
+binit(Buf *b)
+{
+	b->p = nil;
+	b->len = 0;
+	b->cap = 0;
+}
+
+// breset truncates the buffer back to zero length.
+void
+breset(Buf *b)
+{
+	b->len = 0;
+}
+
+// bfree frees the storage associated with a buffer.
+void
+bfree(Buf *b)
+{
+	xfree(b->p);
+	binit(b);
+}
+
+// bgrow ensures that the buffer has at least n more bytes
+// between its len and cap.
+void
+bgrow(Buf *b, int n)
+{
+	int want;
+	
+	want = b->len+n;
+	if(want > b->cap) {
+		b->cap = 2*want;
+		if(b->cap < 64)
+			b->cap = 64;
+		b->p = xrealloc(b->p, b->cap);
+	}
+}
+
+// bwrite appends the n bytes at v to the buffer.
+void
+bwrite(Buf *b, void *v, int n)
+{
+	bgrow(b, n);
+	xmemmove(b->p+b->len, v, n);
+	b->len += n;
+}
+
+// bwritestr appends the string p to the buffer.
+void
+bwritestr(Buf *b, char *p)
+{
+	bwrite(b, p, xstrlen(p));
+}
+
+// bstr returns a pointer to a NUL-terminated string of the
+// buffer contents.  The pointer points into the buffer.
+char*
+bstr(Buf *b)
+{
+	bgrow(b, 1);
+	b->p[b->len] = '\0';
+	return b->p;
+}
+
+// btake takes ownership of the string form of the buffer.
+// After this call, the buffer has zero length and does not
+// refer to the memory that btake returned.
+char*
+btake(Buf *b)
+{
+	char *p;
+	
+	p = bstr(b);
+	binit(b);
+	return p;
+}
+
+// bwriteb appends the src buffer to the dst buffer.
+void
+bwriteb(Buf *dst, Buf *src)
+{
+	bwrite(dst, src->p, src->len);
+}
+
+// bequal reports whether the buffers have the same content.
+bool
+bequal(Buf *s, Buf *t)
+{
+	return s->len == t->len && xmemcmp(s->p, t->p, s->len) == 0;
+}
+
+// The invariant with the vectors is that v->p[0:v->len] is allocated
+// strings that are owned by the vector.  The data beyond v->len may
+// be garbage.
+
+// vinit prepares an uninitialized vector for use.
+void
+vinit(Vec *v)
+{
+	v->p = nil;
+	v->len = 0;
+	v->cap = 0;
+}
+
+// vreset truncates the vector back to zero length.
+void
+vreset(Vec *v)
+{
+	int i;
+	
+	for(i=0; i<v->len; i++) {
+		xfree(v->p[i]);
+		v->p[i] = nil;
+	}
+	v->len = 0;
+}
+
+// vfree frees the storage associated with the vector.
+void
+vfree(Vec *v)
+{
+	vreset(v);
+	xfree(v->p);
+	vinit(v);
+}
+
+
+// vgrow ensures that the vector has room for at least 
+// n more entries between len and cap.
+void
+vgrow(Vec *v, int n)
+{
+	int want;
+	
+	want = v->len+n;
+	if(want > v->cap) {
+		v->cap = 2*want;
+		if(v->cap < 64)
+			v->cap = 64;
+		v->p = xrealloc(v->p, v->cap*sizeof v->p[0]);
+	}
+}
+
+// vcopy copies the srclen strings at src into the vector.
+void
+vcopy(Vec *dst, char **src, int srclen)
+{
+	int i;
+	
+	// use vadd, to make copies of strings
+	for(i=0; i<srclen; i++)
+		vadd(dst, src[i]);
+}
+
+// vadd adds a copy of the string p to the vector.
+void
+vadd(Vec *v, char *p)
+{
+	vgrow(v, 1);
+	if(p != nil)
+		p = xstrdup(p);
+	v->p[v->len++] = p;
+}
+
+// vaddn adds a string consisting of the n bytes at p to the vector.
+void
+vaddn(Vec *v, char *p, int n)
+{
+	char *q;
+
+	vgrow(v, 1);
+	q = xmalloc(n+1);
+	xmemmove(q, p, n);
+	q[n] = '\0';
+	v->p[v->len++] = q;
+}
+
+static int
+strpcmp(const void *a, const void *b)
+{
+	return xstrcmp(*(char**)a, *(char**)b);
+}
+
+// vuniq sorts the vector and then discards duplicates,
+// in the manner of sort | uniq.
+void
+vuniq(Vec *v)
+{
+	int i, n;
+
+	xqsort(v->p, v->len, sizeof(v->p[0]), strpcmp);
+	n = 0;
+	for(i=0; i<v->len; i++) {
+		if(i>0 && streq(v->p[i], v->p[i-1]))
+			xfree(v->p[i]);
+		else
+			v->p[n++] = v->p[i];
+	}
+	v->len = n;
+}
+
+// splitlines replaces the vector v with the result of splitting
+// the input p after each \n.
+void
+splitlines(Vec *v, char *p)
+{
+	int i, c;
+	char *start;
+	
+	vreset(v);
+	start = p;
+	for(i=0; p[i]; i++) {
+		if(p[i] == '\n') {
+			vaddn(v, start, (p+i+1)-start);
+			start = p+i+1;
+		}
+	}
+	if(*start != '\0')
+		vadd(v, start);
+}
+
+// splitfields replaces the vector v with the result of splitting
+// the input p into non-empty fields containing no spaces.
+void
+splitfields(Vec *v, char *p)
+{
+	char *start;
+
+	vreset(v);
+	for(;;) {
+		while(*p == ' ' || *p == '\t' || *p == '\r' || *p == '\n')
+			p++;
+		if(*p == '\0')
+			break;
+		start = p;
+		while(*p != ' ' && *p != '\t' && *p != '\r' && *p != '\n' && *p != '\0')
+			p++;
+		vaddn(v, start, p-start);
+	}
+}

src/cmd/dist/buildgc.c

+// Copyright 2012 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+#include "a.h"
+#include <stdio.h>
+
+/*
+ * Helpers for building cmd/gc.
+ */
+
+// gcopnames creates opnames.h from go.h.
+// It finds the OXXX enum, pulls out all the constants
+// from OXXX to OEND, and writes a table mapping
+// op to string.
+void
+gcopnames(char *dir, char *file)
+{
+	char *p, *q;
+	int i, j, end;
+	Buf in, b, out;
+	Vec lines, fields;
+	
+	binit(&in);
+	binit(&b);
+	binit(&out);
+	vinit(&lines);
+	vinit(&fields);
+	
+	bwritestr(&out, bprintf(&b, "// auto generated by go tool dist\n"));
+	bwritestr(&out, bprintf(&b, "static char *opnames[] = {\n"));
+
+	readfile(&in, bprintf(&b, "%s/go.h", dir));
+	splitlines(&lines, bstr(&in));
+	i = 0;
+	while(i<lines.len && !contains(lines.p[i], "OXXX"))
+		i++;
+	end = 0;
+	for(; i<lines.len && !end; i++) {
+		p = xstrstr(lines.p[i], "//");
+		if(p != nil)
+			*p = '\0';
+		end = contains(lines.p[i], "OEND");
+		splitfields(&fields, lines.p[i]);
+		for(j=0; j<fields.len; j++) {
+			q = fields.p[j];
+			if(*q == 'O')
+				q++;
+			p = q+xstrlen(q)-1;
+			if(*p == ',')
+				*p = '\0';
+			bwritestr(&out, bprintf(&b, "	[O%s] = \"%s\",\n", q, q));
+		}
+	}
+	
+	bwritestr(&out, bprintf(&b, "};\n"));
+
+	writefile(&out, file);
+
+	bfree(&in);
+	bfree(&b);
+	bfree(&out);
+	vfree(&lines);
+	vfree(&fields);
+}
+
+// mkenam reads [568].out.h and writes enam.c
+// The format is much the same as the Go opcodes above.
+void
+mkenam(char *dir, char *file)
+{
+	int i, ch;
+	Buf in, b, out;
+	Vec lines;
+	char *p;
+
+	binit(&b);
+	binit(&in);
+	binit(&out);
+	vinit(&lines);
+
+	ch = dir[xstrlen(dir)-2];
+	bprintf(&b, "%s/../%cl/%c.out.h", dir, ch, ch);
+	readfile(&in, bstr(&b));
+	splitlines(&lines, bstr(&in));
+	bwritestr(&out, "char*	anames[] = {\n");
+	for(i=0; i<lines.len; i++) {
+		if(hasprefix(lines.p[i], "\tA")) {
+			p = xstrstr(lines.p[i], ",");
+			if(p)
+				*p = '\0';
+			p = xstrstr(lines.p[i], "\n");
+			if(p)
+				*p = '\0';
+			p = lines.p[i] + 2;
+			bwritestr(&out, bprintf(&b, "\t\"%s\",\n", p));
+		}
+	}
+	bwritestr(&out, "};\n");
+	writefile(&out, file);
+
+	bfree(&b);
+	bfree(&in);
+	bfree(&out);
+	vfree(&lines);
+}

src/cmd/dist/main.c

+// Copyright 2012 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+#include "a.h"
+
+// cmdtab records the available commands.
+static struct {
+	char *name;
+	void (*f)(int, char**);
+} cmdtab[] = {
+	{"bootstrap", cmdbootstrap},
+	{"env", cmdenv},
+	{"install", cmdinstall},
+};
+
+// The OS-specific main calls into the portable code here.
+void
+xmain(int argc, char **argv)
+{
+	int i;
+
+	if(argc <= 1) {
+		xprintf("go tool dist commands:\n");
+		for(i=0; i<nelem(cmdtab); i++)
+			xprintf("\t%s\n", cmdtab[i].name);
+		xexit(1);
+	}
+	
+	for(i=0; i<nelem(cmdtab); i++) {
+		if(streq(cmdtab[i].name, argv[1])) {
+			cmdtab[i].f(argc-1, argv+1);
+			return;
+		}
+	}
+
+	fatal("unknown command %s", argv[1]);
+}