cmd/ld: use native-endian symbol values in symbol table

The Plan 9 symbol table format defines big-endian symbol values
for portability, but we want to be able to generate an ELF object file
and let the host linker link it, as part of the solution to issue 4069.
The symbol table itself, since it is loaded into memory at run time,
must be filled in by the final host linker, using relocation directives
to set the symbol values. On a little-endian machine, the linker will
only fill in little-endian values during relocation, so we are forced
to use little-endian symbol values.

To preserve most of the original portability of the symbol table
format, we make the table itself say whether it uses big- or
little-endian values. If the table begins with the magic sequence
        fe ff ff ff 00 00
then the actual table begins after those six bytes and contains
little-endian symbol values. Otherwise, the table is in the original
format and contains big-endian symbol values. The magic sequence
looks like an "end of table" entry (the fifth byte is zero), so legacy
readers will see a little-endian table as an empty table.

All the gc architectures are little-endian today, so the practical
effect of this CL is to make all the generated tables little-endian,
but if a big-endian system comes along, ld will not generate
the magic sequence, and the various readers will fall back to the
original big-endian interpretation.

R=ken2
CC=golang-dev
https://golang.org/cl/7066043

doc/go1.1.html

@@ -66,6 +66,15 @@ to adjust frame pointer offsets.
 The implementation now includes a built-in <a href="/doc/articles/race_detector.html">data race detector</a>.
 </p>
 
+<h3 id="symtab">Symbol table</h3>
+
+<p>
+In the gc toolchain, the symbol table format has been extended to allow
+little-endian encoding of symbol values, and the extension is used in
+binaries generated by the Go 1.1 version of the gc linker.
+To the Go 1.0 toolchain and libraries, these new symbol tables appear empty.
+</p>
+
 <h2 id="library">Changes to the standard library</h2>
 
 <h3 id="debug/elf">debug/elf</h3>

src/cmd/ld/data.c

@@ -156,8 +156,8 @@ relocsym(Sym *s)
 	for(r=s->r; r<s->r+s->nr; r++) {
 		off = r->off;
 		siz = r->siz;
-		if(off < 0 || off+(siz&~Rbig) > s->np) {
-			diag("%s: invalid relocation %d+%d not in [%d,%d)", s->name, off, siz&~Rbig, 0, s->np);
+		if(off < 0 || off+siz > s->np) {
+			diag("%s: invalid relocation %d+%d not in [%d,%d)", s->name, off, siz, 0, s->np);
 			continue;
 		}
 		if(r->sym != S && (r->sym->type & SMASK == 0 || r->sym->type & SMASK == SXREF)) {
@@ -198,20 +198,6 @@ relocsym(Sym *s)
 		default:
 			cursym = s;
 			diag("bad reloc size %#ux for %s", siz, r->sym->name);
-		case 4 + Rbig:
-			fl = o;
-			s->p[off] = fl>>24;
-			s->p[off+1] = fl>>16;
-			s->p[off+2] = fl>>8;
-			s->p[off+3] = fl;
-			break;
-		case 4 + Rlittle:
-			fl = o;
-			s->p[off] = fl;
-			s->p[off+1] = fl>>8;
-			s->p[off+2] = fl>>16;
-			s->p[off+3] = fl>>24;
-			break;
 		case 4:
 			fl = o;
 			cast = (uchar*)&fl;

src/cmd/ld/lib.h

@@ -246,12 +246,6 @@ struct Endian
 
 extern Endian be, le;
 
-// relocation size bits
-enum {
-	Rbig = 128,
-	Rlittle = 64,
-};
-
 /* set by call to mywhatsys() */
 extern	char*	goroot;
 extern	char*	goarch;

src/cmd/ld/symtab.c

@@ -209,6 +209,22 @@ slputb(int32 v)
 	symt->size += 4;
 }
 
+static void
+slputl(int32 v)
+{
+	uchar *p;
+
+	symgrow(symt, symt->size+4);
+	p = symt->p + symt->size;
+	*p++ = v;
+	*p++ = v>>8;
+	*p++ = v>>16;
+	*p = v>>24;
+	symt->size += 4;
+}
+
+static void (*slput)(int32);
+
 void
 wputl(ushort w)
 {
@@ -269,15 +285,24 @@ putsymb(Sym *s, char *name, int t, vlong v, vlong size, int ver, Sym *typ)
 //		l = 8;
 	if(s != nil) {
 		rel = addrel(symt);
-		rel->siz = l + Rbig;
+		rel->siz = l;
 		rel->sym = s;
 		rel->type = D_ADDR;
 		rel->off = symt->size;
 		v = 0;
 	}
-	if(l == 8)
+
+	if(l == 8) {
+		if(slput == slputl) {
+			slputl(v);
+			slputl(v>>32);
+		} else {
 			slputb(v>>32);
 			slputb(v);
+		}
+	} else
+		slput(v);
+
 	if(ver)
 		t += 'a' - 'A';
 	scput(t+0x80);			/* 0x80 is variable length */
@@ -306,8 +331,8 @@ putsymb(Sym *s, char *name, int t, vlong v, vlong size, int ver, Sym *typ)
 		rel->off = symt->size;
 	}
 	if(l == 8)
-		slputb(0);
-	slputb(0);
+		slput(0);
+	slput(0);
 
 	if(debug['n']) {
 		if(t == 'z' || t == 'Z') {
@@ -396,5 +421,25 @@ symtab(void)
 
 	if(debug['s'])
 		return;
+
+	switch(thechar) {
+	default:
+		diag("unknown architecture %c", thechar);
+		errorexit();
+	case '5':
+	case '6':
+	case '8':
+		// magic entry to denote little-endian symbol table
+		slputl(0xfffffffe);
+		scput(0);
+		scput(0);
+		slput = slputl;
+		break;
+	case 'v':
+		// big-endian (in case one comes along)
+		slput = slputb;
+		break;
+	}
+
 	genasmsym(putsymb);
 }

src/libmach/sym.c

@@ -113,12 +113,18 @@ syminit(int fd, Fhdr *fp)
 	vlong vl;
 	Biobuf b;
 	int svalsz;
+	uvlong (*swav)(uvlong);
+	uint32 (*swal)(uint32);
+	uchar buf[6];
 
 	if(fp->symsz == 0)
 		return 0;
 	if(fp->type == FNONE)
 		return 0;
 
+	swav = beswav;
+	swal = beswal;
+
 	cleansyms();
 	textseg(fp->txtaddr, fp);
 		/* minimum symbol record size = 4+1+2 bytes */
@@ -129,6 +135,15 @@ syminit(int fd, Fhdr *fp)
 	}
 	Binit(&b, fd, OREAD);
 	Bseek(&b, fp->symoff, 0);
+	memset(buf, 0, sizeof buf);
+	Bread(&b, buf, sizeof buf);
+	if(memcmp(buf, "\xfe\xff\xff\xff\x00\x00", 6) == 0) {
+		swav = leswav;
+		swal = leswal;
+	} else {
+		Bseek(&b, fp->symoff, 0);
+	}
+
 	nsym = 0;
 	size = 0;
 	for(p = symbols; size < fp->symsz; p++, nsym++) {
@@ -136,13 +151,13 @@ syminit(int fd, Fhdr *fp)
 			svalsz = 8;
 			if(Bread(&b, &vl, 8) != 8)
 				return symerrmsg(8, "symbol");
-			p->value = beswav(vl);
+			p->value = swav(vl);
 		}
 		else{
 			svalsz = 4;
 			if(Bread(&b, &l, 4) != 4)
 				return symerrmsg(4, "symbol");
-			p->value = (u32int)beswal(l);
+			p->value = (u32int)swal(l);
 		}
 		if(Bread(&b, &p->type, sizeof(p->type)) != sizeof(p->type))
 			return symerrmsg(sizeof(p->value), "symbol");
@@ -155,12 +170,12 @@ syminit(int fd, Fhdr *fp)
 		if(svalsz == 8){
 			if(Bread(&b, &vl, 8) != 8)
 				return symerrmsg(8, "symbol");
-			p->gotype = beswav(vl);
+			p->gotype = swav(vl);
 		}
 		else{
 			if(Bread(&b, &l, 4) != 4)
 				return symerrmsg(4, "symbol");
-			p->gotype = (u32int)beswal(l);
+			p->gotype = (u32int)swal(l);
 		}
 		size += svalsz;
 

src/pkg/debug/gosym/pclntab_test.go

@@ -129,7 +129,7 @@ func TestLineFromAline(t *testing.T) {
 		if !ok {
 			t.Errorf("file %s starts on line %d", path, line)
 		} else if line != ll+1 {
-			t.Errorf("expected next line of file %s to be %d, got %d", path, ll+1, line)
+			t.Fatalf("expected next line of file %s to be %d, got %d", path, ll+1, line)
 		}
 		lastline[path] = line
 	}

src/pkg/debug/gosym/symtab.go

@@ -13,6 +13,7 @@ package gosym
 // and the Go format is the runtime source, specifically ../../runtime/symtab.c.
 
 import (
+	"bytes"
 	"encoding/binary"
 	"fmt"
 	"strconv"
@@ -104,11 +105,18 @@ type sym struct {
 	name   []byte
 }
 
+var littleEndianSymtab = []byte{0xFE, 0xFF, 0xFF, 0xFF, 0x00, 0x00}
+
 func walksymtab(data []byte, fn func(sym) error) error {
+	var order binary.ByteOrder = binary.BigEndian
+	if bytes.HasPrefix(data, littleEndianSymtab) {
+		data = data[6:]
+		order = binary.LittleEndian
+	}
 	var s sym
 	p := data
 	for len(p) >= 6 {
-		s.value = binary.BigEndian.Uint32(p[0:4])
+		s.value = order.Uint32(p[0:4])
 		typ := p[4]
 		if typ&0x80 == 0 {
 			return &DecodingError{len(data) - len(p) + 4, "bad symbol type", typ}
@@ -139,7 +147,7 @@ func walksymtab(data []byte, fn func(sym) error) error {
 		}
 		s.name = p[0:i]
 		i += nnul
-		s.gotype = binary.BigEndian.Uint32(p[i : i+4])
+		s.gotype = order.Uint32(p[i : i+4])
 		p = p[i+4:]
 		fn(s)
 	}

src/pkg/runtime/symtab.c

@@ -40,13 +40,26 @@ walksymtab(void (*fn)(Sym*))
 {
 	byte *p, *ep, *q;
 	Sym s;
+	int32 bigend;
 
 	p = symtab;
 	ep = esymtab;
+
+	// Default is big-endian value encoding.
+	// If table begins fe ff ff ff 00 00, little-endian.
+	bigend = 1;
+	if(symtab[0] == 0xfe && symtab[1] == 0xff && symtab[2] == 0xff && symtab[3] == 0xff && symtab[4] == 0x00 && symtab[5] == 0x00) {
+		p += 6;
+		bigend = 0;
+	}
 	while(p < ep) {
 		if(p + 7 > ep)
 			break;
+
+		if(bigend)
 			s.value = ((uint32)p[0]<<24) | ((uint32)p[1]<<16) | ((uint32)p[2]<<8) | ((uint32)p[3]);
+		else
+			s.value = ((uint32)p[3]<<24) | ((uint32)p[2]<<16) | ((uint32)p[1]<<8) | ((uint32)p[0]);
 
 		if(!(p[4]&0x80))
 			break;