cmd/ld: support for linking with host linker

A step toward a fix for issue 4069.

To allow linking with arbitrary host object files, add a linker mode
that can generate a host object file instead of an executable.
Then the host linker can be invoked to generate the final executable.

This CL adds a new -hostobj flag that instructs the linker to write
a host object file instead of an executable.

That is, this works:

        go tool 6g x.go
        go tool 6l -hostobj -o x.o x.6
        ld -e _rt0_amd64_linux x.o
        ./a.out

as does:

        go tool 8g x.go
        go tool 8l -hostld ignored -o x.o x.8
        ld -m elf_i386 -e _rt0_386_linux x.o
        ./a.out

Because 5l was never updated to use the standard relocation scheme,
it will take more work to get this working on ARM.

This is a checkpoint of the basic functionality. It does not work
with cgo yet, and cgo is the main reason for the change.
The command-line interface will likely change too.
The gc linker has other information that needs to be returned to
the caller for use when invoking the host linker besides the single
object file.

R=iant, iant
CC=golang-dev
https://golang.org/cl/7060044

src/cmd/5l/asm.c

@@ -239,6 +239,35 @@ adddynrel(Sym *s, Reloc *r)
 	diag("unsupported relocation for dynamic symbol %s (type=%d stype=%d)", targ->name, r->type, targ->type);
 }
 
+int
+elfreloc1(Reloc *r, vlong off, int32 elfsym, vlong add)
+{
+	USED(add);	// written to obj file by ../ld/data.c's reloc
+
+	LPUT(off);
+
+	switch(r->type) {
+	default:
+		return -1;
+
+	case D_ADDR:
+		if(r->siz == 4)
+			LPUT(R_ARM_ABS32 | elfsym<<8);
+		else
+			return -1;
+		break;
+
+	case D_PCREL:
+		if(r->siz == 4)
+			LPUT(R_ARM_REL32 | elfsym<<8);
+		else
+			return -1;
+		break;
+	}
+
+	return 0;
+}
+
 void
 elfsetupplt(void)
 {
@@ -573,6 +602,9 @@ asmb(void)
 				if(debug['v'])
 					Bprint(&bso, "%5.2f dwarf\n", cputime());
 				dwarfemitdebugsections();
+				
+				if(isobj)
+					elfemitreloc();
 			}
 			break;
 		case Hplan9x32:
@@ -809,6 +841,7 @@ if(debug['G']) print("%ux: %s: arm %d\n", (uint32)(p->pc), p->from.sym->name, p-
 
 	case 5:		/* bra s */
 		v = -8;
+		// TODO: Use addrel.
 		if(p->cond != P)
 			v = (p->cond->pc - pc) - 8;
 		o1 = opbra(p->as, p->scond);
@@ -1481,6 +1514,7 @@ if(debug['G']) print("%ux: %s: arm %d\n", (uint32)(p->pc), p->from.sym->name, p-
 		// It's not supposed to be reached, ever, but if it is, we'd
 		// like to be able to tell how we got there.  Assemble as
 		//	BL $0
+		// TODO: Use addrel.
 		v = (0 - pc) - 8;
 		o1 = opbra(ABL, C_SCOND_NONE);
 		o1 |= (v >> 2) & 0xffffff;

src/cmd/6l/asm.c

@@ -284,6 +284,37 @@ adddynrel(Sym *s, Reloc *r)
 	diag("unsupported relocation for dynamic symbol %s (type=%d stype=%d)", targ->name, r->type, targ->type);
 }
 
+int
+elfreloc1(Reloc *r, vlong off, int32 elfsym, vlong add)
+{
+	VPUT(off);
+
+	switch(r->type) {
+	default:
+		return -1;
+
+	case D_ADDR:
+		if(r->siz == 4)
+			VPUT(R_X86_64_32 | (uint64)elfsym<<32);
+		else if(r->siz == 8)
+			VPUT(R_X86_64_64 | (uint64)elfsym<<32);
+		else
+			return -1;
+		break;
+
+	case D_PCREL:
+		if(r->siz == 4)
+			VPUT(R_X86_64_PC32 | (uint64)elfsym<<32);
+		else
+			return -1;
+		add -= r->siz;
+		break;
+	}
+
+	VPUT(add);
+	return 0;
+}
+
 int
 archreloc(Reloc *r, Sym *s, vlong *val)
 {
@@ -674,6 +705,9 @@ asmb(void)
 				       Bprint(&bso, "%5.2f dwarf\n", cputime());
 
 				dwarfemitdebugsections();
+				
+				if(isobj)
+					elfemitreloc();
 			}
 			break;
 		case Hplan9x64:

src/cmd/6l/obj.c

@@ -108,6 +108,7 @@ main(int argc, char *argv[])
 	flagcount("d", "disable dynamic executable", &debug['d']);
 	flagcount("f", "ignore version mismatch", &debug['f']);
 	flagcount("g", "disable go package data checks", &debug['g']);
+	flagcount("hostobj", "generate host object file", &isobj);
 	flagstr("k", "sym: set field tracking symbol", &tracksym);
 	flagcount("n", "dump symbol table", &debug['n']);
 	flagstr("o", "outfile: set output file", &outfile);
@@ -130,6 +131,15 @@ main(int argc, char *argv[])
 	if(HEADTYPE == -1)
 		HEADTYPE = headtype(goos);
 
+	if(isobj) {
+		switch(HEADTYPE) {
+		default:
+			sysfatal("cannot use -hostobj with -H %s", headstr(HEADTYPE));
+		case Hlinux:
+			break;
+		}
+	}
+
 	if(outfile == nil) {
 		if(HEADTYPE == Hwindows)
 			outfile = "6.out.exe";

src/cmd/8l/asm.c

@@ -266,6 +266,35 @@ adddynrel(Sym *s, Reloc *r)
 	diag("unsupported relocation for dynamic symbol %s (type=%d stype=%d)", targ->name, r->type, targ->type);
 }
 
+int
+elfreloc1(Reloc *r, vlong off, int32 elfsym, vlong add)
+{
+	USED(add);	// written to obj file by ../ld/data.c's reloc
+
+	LPUT(off);
+
+	switch(r->type) {
+	default:
+		return -1;
+
+	case D_ADDR:
+		if(r->siz == 4)
+			LPUT(R_386_32 | elfsym<<8);
+		else
+			return -1;
+		break;
+
+	case D_PCREL:
+		if(r->siz == 4)
+			LPUT(R_386_PC32 | elfsym<<8);
+		else
+			return -1;
+		break;
+	}
+
+	return 0;
+}
+
 void
 elfsetupplt(void)
 {
@@ -633,6 +662,9 @@ asmb(void)
 				if(debug['v'])
 					Bprint(&bso, "%5.2f dwarf\n", cputime());
 				dwarfemitdebugsections();
+				
+				if(isobj)
+					elfemitreloc();
 			}
 			break;
 		case Hplan9x32:

src/cmd/8l/obj.c

@@ -114,6 +114,7 @@ main(int argc, char *argv[])
 	flagcount("d", "disable dynamic executable", &debug['d']);
 	flagcount("f", "ignore version mismatch", &debug['f']);
 	flagcount("g", "disable go package data checks", &debug['g']);
+	flagcount("hostobj", "generate host object file", &hostobj);
 	flagstr("k", "sym: set field tracking symbol", &tracksym);
 	flagstr("o", "outfile: set output file", &outfile);
 	flagcount("p", "insert profiling code", &debug['p']);
@@ -135,6 +136,15 @@ main(int argc, char *argv[])
 	if(HEADTYPE == -1)
 		HEADTYPE = headtype(goos);
 
+	if(isobj) {
+		switch(HEADTYPE) {
+		default:
+			sysfatal("cannot use -hostobj with -H %s", headstr(HEADTYPE));
+		case Hlinux:
+			break;
+		}
+	}
+
 	if(outfile == nil) {
 		if(HEADTYPE == Hwindows)
 			outfile = "8.out.exe";

src/cmd/ld/data.c

@@ -146,6 +146,7 @@ void
 relocsym(Sym *s)
 {
 	Reloc *r;
+	Sym *rs;
 	Prog p;
 	int32 i, off, siz, fl;
 	vlong o;
@@ -176,18 +177,35 @@ relocsym(Sym *s)
 		switch(r->type) {
 		default:
 			o = 0;
-			if(archreloc(r, s, &o) < 0)
+			if(isobj || archreloc(r, s, &o) < 0)
 				diag("unknown reloc %d", r->type);
 			break;
 		case D_ADDR:
 			o = symaddr(r->sym) + r->add;
+			if(isobj && r->sym->type != SCONST) {
+				if(thechar == '6')
+					o = 0;
+				else {
+					// set up addend for eventual relocation via outer symbol
+					rs = r->sym;
+					while(rs->outer != nil)
+						rs = rs->outer;
+					o -= symaddr(rs);
+				}
+			}
 			break;
 		case D_PCREL:
-			// r->sym can be null when CALL $(constant) is transformed from absoulte PC to relative PC call.
+			// r->sym can be null when CALL $(constant) is transformed from absolute PC to relative PC call.
 			o = 0;
 			if(r->sym)
 				o += symaddr(r->sym);
 			o += r->add - (s->value + r->off + r->siz);
+			if(isobj && r->sym->type != SCONST) {
+				if(thechar == '6')
+					o = 0;
+				else
+					o = r->add - r->siz;
+			}
 			break;
 		case D_SIZE:
 			o = r->sym->size + r->add;

src/cmd/ld/elf.c

@@ -758,6 +758,7 @@ elfshbits(Section *sect)
 		sh->flags |= SHF_EXECINSTR;
 	if(sect->rwx & 2)
 		sh->flags |= SHF_WRITE;
+	if(!isobj)
 		sh->addr = sect->vaddr;
 	sh->addralign = PtrSize;
 	sh->size = sect->len;
@@ -766,6 +767,117 @@ elfshbits(Section *sect)
 	return sh;
 }
 
+ElfShdr*
+elfshreloc(Section *sect)
+{
+	int typ;
+	ElfShdr *sh;
+	char *prefix;
+	char buf[100];
+	
+	// If main section is SHT_NOBITS, nothing to relocate.
+	// Also nothing to relocate in .shstrtab.
+	if(sect->vaddr >= sect->seg->vaddr + sect->seg->filelen)
+		return nil;
+	if(strcmp(sect->name, ".shstrtab") == 0)
+		return nil;
+
+	if(thechar == '6') {
+		prefix = ".rela";
+		typ = SHT_RELA;
+	} else {
+		prefix = ".rel";
+		typ = SHT_REL;
+	}
+
+	snprint(buf, sizeof buf, "%s%s", prefix, sect->name);
+	sh = elfshname(buf);
+	sh->type = typ;
+	sh->entsize = PtrSize*(2+(typ==SHT_RELA));
+	sh->link = elfshname(".symtab")->shnum;
+	sh->info = sect->elfsect->shnum;
+	sh->off = sect->reloff;
+	sh->size = sect->rellen;
+	sh->addralign = PtrSize;
+	return sh;
+}
+
+void
+elfrelocsect(Section *sect, Sym *first)
+{
+	Sym *sym, *rs;
+	int32 eaddr;
+	Reloc *r;
+	int64 add;
+
+	// If main section is SHT_NOBITS, nothing to relocate.
+	// Also nothing to relocate in .shstrtab.
+	if(sect->vaddr >= sect->seg->vaddr + sect->seg->filelen)
+		return;
+	if(strcmp(sect->name, ".shstrtab") == 0)
+		return;
+
+	sect->reloff = cpos();
+	for(sym = first; sym != nil; sym = sym->next) {
+		if(!sym->reachable)
+			continue;
+		if(sym->value >= sect->vaddr)
+			break;
+	}
+	
+	eaddr = sect->vaddr + sect->len;
+	for(; sym != nil; sym = sym->next) {
+		if(!sym->reachable)
+			continue;
+		if(sym->value >= eaddr)
+			break;
+		cursym = sym;
+		
+		for(r = sym->r; r < sym->r+sym->nr; r++) {
+			// Ignore relocations handled by reloc already.
+			switch(r->type) {
+			case D_SIZE:
+				continue;
+			case D_ADDR:
+			case D_PCREL:
+				if(r->sym->type == SCONST)
+					continue;
+				break;
+			}
+
+			add = r->add;
+			rs = r->sym;
+			while(rs->outer != nil) {
+				add += rs->value - rs->outer->value;
+				rs = rs->outer;
+			}
+				
+			if(rs->elfsym == 0)
+				diag("reloc %d to non-elf symbol %s (rs=%s) %d", r->type, r->sym->name, rs->name, rs->type);
+
+			if(elfreloc1(r, sym->value - sect->vaddr + r->off, rs->elfsym, add) < 0)
+				diag("unsupported obj reloc %d/%d to %s", r->type, r->siz, r->sym->name);
+		}
+	}
+		
+	sect->rellen = cpos() - sect->reloff;
+}	
+	
+void
+elfemitreloc(void)
+{
+	Section *sect;
+
+	while(cpos()&7)
+		cput(0);
+
+	elfrelocsect(segtext.sect, textp);
+	for(sect=segtext.sect->next; sect!=nil; sect=sect->next)
+		elfrelocsect(sect, datap);	
+	for(sect=segdata.sect; sect!=nil; sect=sect->next)
+		elfrelocsect(sect, datap);	
+}
+
 void
 doelf(void)
 {
@@ -801,6 +913,33 @@ doelf(void)
 	addstring(shstrtab, ".gosymtab");
 	addstring(shstrtab, ".gopclntab");
 	
+	if(isobj) {
+		debug['s'] = 0;
+		debug['d'] = 1;
+
+		if(thechar == '6') {
+			addstring(shstrtab, ".rela.text");
+			addstring(shstrtab, ".rela.rodata");
+			addstring(shstrtab, ".rela.typelink");
+			addstring(shstrtab, ".rela.gcdata");
+			addstring(shstrtab, ".rela.gcbss");
+			addstring(shstrtab, ".rela.gosymtab");
+			addstring(shstrtab, ".rela.gopclntab");
+			addstring(shstrtab, ".rela.noptrdata");
+			addstring(shstrtab, ".rela.data");
+		} else {
+			addstring(shstrtab, ".rel.text");
+			addstring(shstrtab, ".rel.rodata");
+			addstring(shstrtab, ".rel.typelink");
+			addstring(shstrtab, ".rel.gcdata");
+			addstring(shstrtab, ".rel.gcbss");
+			addstring(shstrtab, ".rel.gosymtab");
+			addstring(shstrtab, ".rel.gopclntab");
+			addstring(shstrtab, ".rel.noptrdata");
+			addstring(shstrtab, ".rel.data");
+		}
+	}
+
 	if(!debug['s']) {
 		addstring(shstrtab, ".symtab");
 		addstring(shstrtab, ".strtab");
@@ -1005,6 +1144,14 @@ asmbelf(vlong symo)
 	startva = INITTEXT - HEADR;
 	resoff = ELFRESERVE;
 	
+	pph = nil;
+	if(isobj) {
+		/* skip program headers */
+		eh->phoff = 0;
+		eh->phentsize = 0;
+		goto elfobj;
+	}
+
 	/* program header info */
 	pph = newElfPhdr();
 	pph->type = PT_PHDR;
@@ -1238,17 +1385,29 @@ asmbelf(vlong symo)
 	ph->flags = 0x2a00; // mprotect, randexec, emutramp disabled
 	ph->align = PtrSize;
 
+elfobj:
 	sh = elfshname(".shstrtab");
 	sh->type = SHT_STRTAB;
 	sh->addralign = 1;
 	shsym(sh, lookup(".shstrtab", 0));
 	eh->shstrndx = sh->shnum;
 
+	// put these sections early in the list
+	elfshname(".symtab");
+	elfshname(".strtab");
+
 	for(sect=segtext.sect; sect!=nil; sect=sect->next)
 		elfshbits(sect);
 	for(sect=segdata.sect; sect!=nil; sect=sect->next)
 		elfshbits(sect);
 
+	if(isobj) {
+		for(sect=segtext.sect; sect!=nil; sect=sect->next)
+			elfshreloc(sect);
+		for(sect=segdata.sect; sect!=nil; sect=sect->next)
+			elfshreloc(sect);
+	}
+		
 	if(!debug['s']) {
 		sh = elfshname(".symtab");
 		sh->type = SHT_SYMTAB;
@@ -1265,6 +1424,8 @@ asmbelf(vlong symo)
 		sh->size = elfstrsize;
 		sh->addralign = 1;
 
+		// TODO(rsc): Enable for isobj too, once we know it works.
+		if(!isobj)
 			dwarfaddelfheaders();
 	}
 
@@ -1288,14 +1449,20 @@ asmbelf(vlong symo)
 
 	if(flag_shared)
 		eh->type = ET_DYN;
+	else if(isobj)
+		eh->type = ET_REL;
 	else
 		eh->type = ET_EXEC;
 
-	eh->version = EV_CURRENT;
+	if(!isobj)
 		eh->entry = entryvalue();
 
+	eh->version = EV_CURRENT;
+
+	if(pph != nil) {
 		pph->filesz = eh->phnum * eh->phentsize;
 		pph->memsz = pph->filesz;
+	}
 
 	cseek(0);
 	a = 0;
@@ -1304,12 +1471,14 @@ asmbelf(vlong symo)
 	a += elfwriteshdrs();
 	if(!debug['d'])
 		a += elfwriteinterp();
+	if(!isobj) {
 		if(HEADTYPE == Hnetbsd)
 			a += elfwritenetbsdsig();
 		if(HEADTYPE == Hopenbsd)
 			a += elfwriteopenbsdsig();
 		if(buildinfolen > 0)
 			a += elfwritebuildinfo();
+	}
 	if(a > ELFRESERVE)	
 		diag("ELFRESERVE too small: %d > %d", a, ELFRESERVE);
 }

src/cmd/ld/elf.h

@@ -1005,6 +1005,7 @@ extern char linuxdynld[];
 extern char freebsddynld[];
 extern char netbsddynld[];
 extern char openbsddynld[];
+int	elfreloc1(Reloc*, vlong off, int32 elfsym, vlong add);
 
 EXTERN	int	elfstrsize;
 EXTERN	char*	elfstrdat;

src/cmd/ld/lib.c

@@ -1400,6 +1400,19 @@ headtype(char *name)
 	return -1;  // not reached
 }
 
+char*
+headstr(int v)
+{
+	static char buf[20];
+	int i;
+
+	for(i=0; headers[i].name; i++)
+		if(v == headers[i].val)
+			return headers[i].name;
+	snprint(buf, sizeof buf, "%d", v);
+	return buf;
+}
+
 void
 undef(void)
 {

src/cmd/ld/lib.h

@@ -109,6 +109,8 @@ struct Section
 	Section	*next;	// in segment list
 	Segment	*seg;
 	struct Elf64_Shdr *elfsect;
+	uvlong	reloff;
+	uvlong	rellen;
 };
 
 extern	char	symname[];
@@ -137,6 +139,7 @@ EXTERN	char*	thestring;
 EXTERN	int	ndynexp;
 EXTERN	int	havedynamic;
 EXTERN	int	iscgo;
+EXTERN	int	isobj;
 EXTERN	int	elfglobalsymndx;
 EXTERN	int	flag_race;
 EXTERN	int flag_shared;
@@ -302,6 +305,7 @@ EXTERN	char*	headstring;
 extern	Header	headers[];
 
 int	headtype(char*);
+char*	headstr(int);
 void	setheadtype(char*);
 
 int	Yconv(Fmt*);

src/cmd/ld/symtab.c

@@ -126,6 +126,8 @@ putelfsym(Sym *x, char *s, int t, vlong addr, vlong size, int ver, Sym *go)
 		return;
 
 	off = putelfstr(s);
+	if(isobj)
+		addr -= xo->sect->vaddr;
 	putelfsyment(off, addr, size, (bind<<4)|(type&0xf), xo->sect->elfsect->shnum, (x->type & SHIDDEN) ? 2 : 0);
 	x->elfsym = numelfsym++;
 }