[dev.link] cmd/link, cmd/internal/goobj2: adopt new DWARF compilation unit...

[dev.link] cmd/link, cmd/internal/goobj2: adopt new DWARF compilation unit logic with new object file

The dev.link branch was not sync'd with the new DWARF compilation
unit logic change on the master branch, and the new object file
format didn't support this.

This CL adds the new DWARF CU and file table support to the new
object file format. In the old object file, the DWARF file table
is a separate section. For now, we do the same with the new
object file, keeping it as a separate block.

While here, also refactor the loader code so it is easier for the
loader to carry per-object informations.

Change-Id: I4c317941fc0a5831acbc11ce8c2a8b7421471372
Reviewed-on: https://go-review.googlesource.com/c/go/+/198198Reviewed-by: Austin Clements <austin@google.com>

src/cmd/internal/goobj2/objfile.go

@@ -31,6 +31,8 @@ import (
 //
 //    PkgIndex [...]stringOff // TODO: add fingerprints
 //
+//    DwarfFiles [...]stringOff // XXX as a separate block for now
+//
 //    SymbolDefs [...]struct {
 //       Name stringOff
 //       ABI  uint16
@@ -126,6 +128,7 @@ const (
 // Blocks
 const (
 	BlkPkgIdx = iota
+	BlkDwarfFile
 	BlkSymdef
 	BlkNonpkgdef
 	BlkNonpkgref
@@ -471,6 +474,24 @@ func (r *Reader) Pkglist() []string {
 	return s
 }
 
+func (r *Reader) NPkg() int {
+	return int(r.h.Offsets[BlkPkgIdx+1]-r.h.Offsets[BlkPkgIdx]) / 4
+}
+
+func (r *Reader) Pkg(i int) string {
+	off := r.h.Offsets[BlkPkgIdx] + uint32(i)*4
+	return r.StringRef(off)
+}
+
+func (r *Reader) NDwarfFile() int {
+	return int(r.h.Offsets[BlkDwarfFile+1]-r.h.Offsets[BlkDwarfFile]) / 4
+}
+
+func (r *Reader) DwarfFile(i int) string {
+	off := r.h.Offsets[BlkDwarfFile] + uint32(i)*4
+	return r.StringRef(off)
+}
+
 func (r *Reader) NSym() int {
 	symsiz := (&Sym{}).Size()
 	return int(r.h.Offsets[BlkSymdef+1]-r.h.Offsets[BlkSymdef]) / symsiz
@@ -531,6 +552,11 @@ func (r *Reader) DataSize(i int) int {
 	return int(r.DataOff(i+1) - r.DataOff(i))
 }
 
+// Data returns the i-th symbol's data.
+func (r *Reader) Data(i int) []byte {
+	return r.BytesAt(r.DataOff(i), r.DataSize(i))
+}
+
 // AuxDataBase returns the base offset of the aux data block.
 func (r *Reader) PcdataBase() uint32 {
 	return r.h.Offsets[BlkPcdata]

src/cmd/internal/obj/objfile2.go

@@ -42,6 +42,12 @@ func WriteObjFile2(ctxt *Link, b *bio.Writer, pkgpath string) {
 		w.StringRef(pkg)
 	}
 
+	// DWARF file table
+	h.Offsets[goobj2.BlkDwarfFile] = w.Offset()
+	for _, f := range ctxt.PosTable.DebugLinesFileTable() {
+		w.StringRef(f)
+	}
+
 	// Symbol definitions
 	h.Offsets[goobj2.BlkSymdef] = w.Offset()
 	for _, s := range ctxt.defs {
@@ -198,6 +204,9 @@ func (w *writer) StringTable() {
 			w.AddString(f)
 		}
 	})
+	for _, f := range w.ctxt.PosTable.DebugLinesFileTable() {
+		w.AddString(f)
+	}
 }
 
 func (w *writer) Sym(s *LSym) {

src/cmd/link/internal/ld/lib.go

@@ -403,11 +403,7 @@ func (ctxt *Link) loadlib() {
 
 	if *flagNewobj {
 		// Add references of externally defined symbols.
-		for _, lib := range ctxt.Library {
-			for _, r := range lib.Readers {
-				objfile.LoadRefs(ctxt.loader, r.Reader, lib, ctxt.Arch, ctxt.Syms, r.Version)
-			}
-		}
+		objfile.LoadRefs(ctxt.loader, ctxt.Arch, ctxt.Syms)
 
 		// Load cgo directives.
 		for _, p := range ctxt.cgodata {
@@ -550,11 +546,7 @@ func (ctxt *Link) loadlib() {
 
 	// For now, load relocations for dead-code elimination.
 	if *flagNewobj {
-		for _, lib := range ctxt.Library {
-			for _, r := range lib.Readers {
-				objfile.LoadReloc(ctxt.loader, r.Reader, lib, r.Version, ctxt.LibraryByPkg)
-			}
-		}
+		objfile.LoadReloc(ctxt.loader)
 	}
 }
 
@@ -2545,11 +2537,7 @@ func dfs(lib *sym.Library, mark map[*sym.Library]markKind, order *[]*sym.Library
 
 func (ctxt *Link) loadlibfull() {
 	// Load full symbol contents, resolve indexed references.
-	for _, lib := range ctxt.Library {
-		for _, r := range lib.Readers {
-			objfile.LoadFull(ctxt.loader, r.Reader, lib, r.Version, ctxt.LibraryByPkg)
-		}
-	}
+	objfile.LoadFull(ctxt.loader)
 
 	// For now, add all symbols to ctxt.Syms.
 	for _, s := range ctxt.loader.Syms {
@@ -2557,6 +2545,9 @@ func (ctxt *Link) loadlibfull() {
 			ctxt.Syms.Add(s)
 		}
 	}
+
+	// Drop the reference.
+	ctxt.loader = nil
 }
 
 func (ctxt *Link) dumpsyms() {

src/cmd/link/internal/objfile/objfile2.go

@@ -5,7 +5,9 @@
 package objfile
 
 import (
+	"bytes"
 	"cmd/internal/bio"
+	"cmd/internal/dwarf"
 	"cmd/internal/goobj2"
 	"cmd/internal/obj"
 	"cmd/internal/objabi"
@@ -21,8 +23,18 @@ import (
 
 var _ = fmt.Print
 
+// oReader is a wrapper type of obj.Reader, along with some
+// extra information.
+// TODO: rename to objReader once the old one is gone?
+type oReader struct {
+	*goobj2.Reader
+	unit      *sym.CompilationUnit
+	version   int // version of static symbol
+	pkgprefix string
+}
+
 type objIdx struct {
-	r *goobj2.Reader
+	r *oReader
 	i int // start index
 }
 
@@ -35,34 +47,40 @@ type nameVer struct {
 //
 // TODO: describe local-global index mapping.
 type Loader struct {
-	start map[*goobj2.Reader]int // map from object file to its start index
+	start map[*oReader]int // map from object file to its start index
 	objs  []objIdx         // sorted by start index (i.e. objIdx.i)
 	max   int              // current max index
 
 	symsByName map[nameVer]int // map symbol name to index
 
+	objByPkg map[string]*oReader // map package path to its Go object reader
+
 	Syms []*sym.Symbol // indexed symbols. XXX we still make sym.Symbol for now.
 }
 
 func NewLoader() *Loader {
 	return &Loader{
-		start:      make(map[*goobj2.Reader]int),
+		start:      make(map[*oReader]int),
 		objs:       []objIdx{{nil, 0}},
 		symsByName: make(map[nameVer]int),
+		objByPkg:   make(map[string]*oReader),
 		Syms:       []*sym.Symbol{nil},
 	}
 }
 
 // Return the start index in the global index space for a given object file.
-func (l *Loader) StartIndex(r *goobj2.Reader) int {
+func (l *Loader) StartIndex(r *oReader) int {
 	return l.start[r]
 }
 
 // Add object file r, return the start index.
-func (l *Loader) AddObj(r *goobj2.Reader) int {
+func (l *Loader) AddObj(pkg string, r *oReader) int {
 	if _, ok := l.start[r]; ok {
 		panic("already added")
 	}
+	if _, ok := l.objByPkg[pkg]; !ok {
+		l.objByPkg[pkg] = r
+	}
 	n := r.NSym() + r.NNonpkgdef()
 	i := l.max + 1
 	l.start[r] = i
@@ -98,12 +116,12 @@ func (l *Loader) AddExtSym(name string, ver int) int {
 }
 
 // Convert a local index to a global index.
-func (l *Loader) ToGlobal(r *goobj2.Reader, i int) int {
+func (l *Loader) ToGlobal(r *oReader, i int) int {
 	return l.StartIndex(r) + i
 }
 
-// Convert a global index to a global index. Is it useful?
-func (l *Loader) ToLocal(i int) (*goobj2.Reader, int) {
+// Convert a global index to a local index.
+func (l *Loader) ToLocal(i int) (*oReader, int) {
 	k := sort.Search(i, func(k int) bool {
 		return l.objs[k].i >= i
 	})
@@ -113,6 +131,35 @@ func (l *Loader) ToLocal(i int) (*goobj2.Reader, int) {
 	return l.objs[k].r, i - l.objs[k].i
 }
 
+// Resolve a local symbol reference. Return global index.
+func (l *Loader) Resolve(r *oReader, s goobj2.SymRef) int {
+	var rr *oReader
+	switch p := s.PkgIdx; p {
+	case goobj2.PkgIdxInvalid:
+		if s.SymIdx != 0 {
+			panic("bad sym ref")
+		}
+		return 0
+	case goobj2.PkgIdxNone:
+		// Resolve by name
+		i := int(s.SymIdx) + r.NSym()
+		osym := goobj2.Sym{}
+		osym.Read(r.Reader, r.SymOff(i))
+		name := strings.Replace(osym.Name, "\"\".", r.pkgprefix, -1)
+		v := abiToVer(osym.ABI, r.version)
+		nv := nameVer{name, v}
+		return l.symsByName[nv]
+	case goobj2.PkgIdxBuiltin:
+		panic("PkgIdxBuiltin not used")
+	case goobj2.PkgIdxSelf:
+		rr = r
+	default:
+		pkg := r.Pkg(int(p))
+		rr = l.objByPkg[pkg]
+	}
+	return l.ToGlobal(rr, int(s.SymIdx))
+}
+
 // Look up a symbol by name, return global index, or 0 if not found.
 // This is more like Syms.ROLookup than Lookup -- it doesn't create
 // new symbol.
@@ -133,17 +180,24 @@ func LoadNew(l *Loader, arch *sys.Arch, syms *sym.Symbols, f *bio.Reader, lib *s
 		panic("cannot read object file")
 	}
 	localSymVersion := syms.IncVersion()
-	lib.Readers = append(lib.Readers, struct {
-		Reader  *goobj2.Reader
-		Version int
-	}{r, localSymVersion})
-
 	pkgprefix := objabi.PathToPrefix(lib.Pkg) + "."
+	or := &oReader{r, unit, localSymVersion, pkgprefix}
 
 	// Autolib
-	lib.ImportStrings = append(lib.ImportStrings, r.Pkglist()[1:]...)
+	npkg := r.NPkg()
+	lib.ImportStrings = append(lib.ImportStrings, make([]string, npkg-1)...)[:len(lib.ImportStrings)]
+	for i := 1; i < npkg; i++ {
+		lib.ImportStrings = append(lib.ImportStrings, r.Pkg(i))
+	}
 
-	istart := l.AddObj(r)
+	// DWARF file table
+	nfile := r.NDwarfFile()
+	unit.DWARFFileTable = make([]string, nfile)
+	for i := range unit.DWARFFileTable {
+		unit.DWARFFileTable[i] = r.DwarfFile(i)
+	}
+
+	istart := l.AddObj(lib.Pkg, or)
 
 	ndef := r.NSym()
 	nnonpkgdef := r.NNonpkgdef()
@@ -172,14 +226,21 @@ func LoadNew(l *Loader, arch *sys.Arch, syms *sym.Symbols, f *bio.Reader, lib *s
 
 // Make sure referenced symbols are added. Most of them should already be added.
 // This should only be needed for referenced external symbols.
-func LoadRefs(l *Loader, r *goobj2.Reader, lib *sym.Library, arch *sys.Arch, syms *sym.Symbols, localSymVersion int) {
+func LoadRefs(l *Loader, arch *sys.Arch, syms *sym.Symbols) {
+	for _, o := range l.objs[1:] {
+		loadObjRefs(l, o.r, arch, syms)
+	}
+}
+
+func loadObjRefs(l *Loader, r *oReader, arch *sys.Arch, syms *sym.Symbols) {
+	lib := r.unit.Lib
 	pkgprefix := objabi.PathToPrefix(lib.Pkg) + "."
 	ndef := r.NSym() + r.NNonpkgdef()
 	for i, n := 0, r.NNonpkgref(); i < n; i++ {
 		osym := goobj2.Sym{}
-		osym.Read(r, r.SymOff(ndef+i))
+		osym.Read(r.Reader, r.SymOff(ndef+i))
 		name := strings.Replace(osym.Name, "\"\".", pkgprefix, -1)
-		v := abiToVer(osym.ABI, localSymVersion)
+		v := abiToVer(osym.ABI, r.version)
 		if ii := l.AddExtSym(name, v); ii != 0 {
 			s := syms.Newsym(name, v)
 			preprocess(arch, s) // TODO: put this at a better place
@@ -231,40 +292,19 @@ func preprocess(arch *sys.Arch, s *sym.Symbol) {
 // Load relocations for building the dependency graph in deadcode pass.
 // For now, we load symbol types, relocations, gotype, and the contents
 // of type symbols, which are needed in deadcode.
-func LoadReloc(l *Loader, r *goobj2.Reader, lib *sym.Library, localSymVersion int, libByPkg map[string]*sym.Library) {
-	// PkgIdx
-	pkglist := r.Pkglist()
+func LoadReloc(l *Loader) {
+	for _, o := range l.objs[1:] {
+		loadObjReloc(l, o.r)
+	}
+}
 
+func loadObjReloc(l *Loader, r *oReader) {
+	lib := r.unit.Lib
 	pkgprefix := objabi.PathToPrefix(lib.Pkg) + "."
 	istart := l.StartIndex(r)
 
 	resolveSymRef := func(s goobj2.SymRef) *sym.Symbol {
-		var rr *goobj2.Reader
-		switch p := s.PkgIdx; p {
-		case goobj2.PkgIdxInvalid:
-			if s.SymIdx != 0 {
-				panic("bad sym ref")
-			}
-			return nil
-		case goobj2.PkgIdxNone:
-			// Resolve by name
-			i := int(s.SymIdx) + r.NSym()
-			osym := goobj2.Sym{}
-			osym.Read(r, r.SymOff(i))
-			name := strings.Replace(osym.Name, "\"\".", pkgprefix, -1)
-			v := abiToVer(osym.ABI, localSymVersion)
-			nv := nameVer{name, v}
-			i = l.symsByName[nv]
-			return l.Syms[i]
-		case goobj2.PkgIdxBuiltin:
-			panic("PkgIdxBuiltin is not used")
-		case goobj2.PkgIdxSelf:
-			rr = r
-		default:
-			pkg := pkglist[p]
-			rr = libByPkg[pkg].Readers[0].Reader // typically Readers[0] is go object (others are asm)
-		}
-		i := l.ToGlobal(rr, int(s.SymIdx))
+		i := l.Resolve(r, s)
 		return l.Syms[i]
 	}
 
@@ -275,7 +315,7 @@ func LoadReloc(l *Loader, r *goobj2.Reader, lib *sym.Library, localSymVersion in
 		}
 
 		osym := goobj2.Sym{}
-		osym.Read(r, r.SymOff(i))
+		osym.Read(r.Reader, r.SymOff(i))
 		name := strings.Replace(osym.Name, "\"\".", pkgprefix, -1)
 		if s.Name != name { // Sanity check. We can remove it in the final version.
 			fmt.Println("name mismatch:", lib, i, s.Name, name)
@@ -298,13 +338,14 @@ func LoadReloc(l *Loader, r *goobj2.Reader, lib *sym.Library, localSymVersion in
 			t = s.Type
 		}
 		s.Type = t
+		s.Unit = r.unit
 
 		// Reloc
 		nreloc := r.NReloc(i)
 		s.R = make([]sym.Reloc, nreloc)
 		for j := range s.R {
 			rel := goobj2.Reloc{}
-			rel.Read(r, r.RelocOff(i, j))
+			rel.Read(r.Reader, r.RelocOff(i, j))
 			s.R[j] = sym.Reloc{
 				Off:  rel.Off,
 				Siz:  rel.Siz,
@@ -316,7 +357,7 @@ func LoadReloc(l *Loader, r *goobj2.Reader, lib *sym.Library, localSymVersion in
 
 		// XXX deadcode needs symbol data for type symbols. Read it now.
 		if strings.HasPrefix(name, "type.") {
-			s.P = r.BytesAt(r.DataOff(i), r.DataSize(i))
+			s.P = r.Data(i)
 			s.Attr.Set(sym.AttrReadOnly, r.ReadOnly())
 			s.Size = int64(osym.Siz)
 		}
@@ -325,7 +366,7 @@ func LoadReloc(l *Loader, r *goobj2.Reader, lib *sym.Library, localSymVersion in
 		naux := r.NAux(i)
 		for j := 0; j < naux; j++ {
 			a := goobj2.Aux{}
-			a.Read(r, r.AuxOff(i, j))
+			a.Read(r.Reader, r.AuxOff(i, j))
 			switch a.Type {
 			case goobj2.AuxGotype:
 				typ := resolveSymRef(a.Sym)
@@ -363,38 +404,19 @@ func LoadReloc(l *Loader, r *goobj2.Reader, lib *sym.Library, localSymVersion in
 // TODO: For now, some contents are already load in LoadReloc. Maybe
 // we should combine LoadReloc back into this, once we rewrite deadcode
 // pass to use index directly.
-func LoadFull(l *Loader, r *goobj2.Reader, lib *sym.Library, localSymVersion int, libByPkg map[string]*sym.Library) {
-	// PkgIdx
-	pkglist := r.Pkglist()
+func LoadFull(l *Loader) {
+	for _, o := range l.objs[1:] {
+		loadObjFull(l, o.r)
+	}
+}
 
+func loadObjFull(l *Loader, r *oReader) {
+	lib := r.unit.Lib
 	pkgprefix := objabi.PathToPrefix(lib.Pkg) + "."
 	istart := l.StartIndex(r)
 
 	resolveSymRef := func(s goobj2.SymRef) *sym.Symbol {
-		var rr *goobj2.Reader
-		switch p := s.PkgIdx; p {
-		case goobj2.PkgIdxInvalid:
-			if s.SymIdx != 0 {
-				panic("bad sym ref")
-			}
-			return nil
-		case goobj2.PkgIdxNone:
-			// Resolve by name
-			i := int(s.SymIdx) + r.NSym()
-			osym := goobj2.Sym{}
-			osym.Read(r, r.SymOff(i))
-			name := strings.Replace(osym.Name, "\"\".", pkgprefix, -1)
-			v := abiToVer(osym.ABI, localSymVersion)
-			nv := nameVer{name, v}
-			i = l.symsByName[nv]
-			return l.Syms[i]
-		case goobj2.PkgIdxSelf:
-			rr = r
-		default:
-			pkg := pkglist[p]
-			rr = libByPkg[pkg].Readers[0].Reader // typically Readers[0] is go object (others are asm)
-		}
-		i := l.ToGlobal(rr, int(s.SymIdx))
+		i := l.Resolve(r, s)
 		return l.Syms[i]
 	}
 
@@ -411,7 +433,7 @@ func LoadFull(l *Loader, r *goobj2.Reader, lib *sym.Library, localSymVersion int
 		}
 
 		osym := goobj2.Sym{}
-		osym.Read(r, r.SymOff(i))
+		osym.Read(r.Reader, r.SymOff(i))
 		name := strings.Replace(osym.Name, "\"\".", pkgprefix, -1)
 		if s.Name != name { // Sanity check. We can remove it in the final version.
 			fmt.Println("name mismatch:", lib, i, s.Name, name)
@@ -421,11 +443,10 @@ func LoadFull(l *Loader, r *goobj2.Reader, lib *sym.Library, localSymVersion int
 		dupok := osym.Flag&goobj2.SymFlagDupok != 0
 		local := osym.Flag&goobj2.SymFlagLocal != 0
 		makeTypelink := osym.Flag&goobj2.SymFlagTypelink != 0
-		datasize := r.DataSize(i)
 		size := osym.Siz
 
 		// Symbol data
-		s.P = r.BytesAt(r.DataOff(i), datasize)
+		s.P = r.Data(i)
 		s.Attr.Set(sym.AttrReadOnly, r.ReadOnly())
 
 		// Aux symbol info
@@ -433,7 +454,7 @@ func LoadFull(l *Loader, r *goobj2.Reader, lib *sym.Library, localSymVersion int
 		naux := r.NAux(i)
 		for j := 0; j < naux; j++ {
 			a := goobj2.Aux{}
-			a.Read(r, r.AuxOff(i, j))
+			a.Read(r.Reader, r.AuxOff(i, j))
 			switch a.Type {
 			case goobj2.AuxGotype, goobj2.AuxFuncdata:
 				// already loaded
@@ -457,6 +478,14 @@ func LoadFull(l *Loader, r *goobj2.Reader, lib *sym.Library, localSymVersion int
 		s.Attr.Set(sym.AttrLocal, local)
 		s.Attr.Set(sym.AttrMakeTypelink, makeTypelink)
 
+		if s.Type == sym.SDWARFINFO {
+			// For DWARF symbols, replace `"".` to actual package prefix
+			// in the symbol content.
+			// TODO: maybe we should do this in the compiler and get rid
+			// of this.
+			patchDWARFName(s, r)
+		}
+
 		if s.Type != sym.STEXT {
 			continue
 		}
@@ -465,7 +494,7 @@ func LoadFull(l *Loader, r *goobj2.Reader, lib *sym.Library, localSymVersion int
 		if isym == -1 {
 			continue
 		}
-		b := r.BytesAt(r.DataOff(isym), r.DataSize(isym))
+		b := r.Data(isym)
 		info := goobj2.FuncInfo{}
 		info.Read(b)
 
@@ -508,3 +537,32 @@ func LoadFull(l *Loader, r *goobj2.Reader, lib *sym.Library, localSymVersion int
 		}
 	}
 }
+
+func patchDWARFName(s *sym.Symbol, r *oReader) {
+	// This is kind of ugly. Really the package name should not
+	// even be included here.
+	if s.Size < 1 || s.P[0] != dwarf.DW_ABRV_FUNCTION {
+		return
+	}
+	e := bytes.IndexByte(s.P, 0)
+	if e == -1 {
+		return
+	}
+	p := bytes.Index(s.P[:e], emptyPkg)
+	if p == -1 {
+		return
+	}
+	pkgprefix := []byte(r.pkgprefix)
+	patched := bytes.Replace(s.P[:e], emptyPkg, pkgprefix, -1)
+
+	s.P = append(patched, s.P[e:]...)
+	s.Attr.Set(sym.AttrReadOnly, false)
+	delta := int64(len(s.P)) - s.Size
+	s.Size = int64(len(s.P))
+	for i := range s.R {
+		r := &s.R[i]
+		if r.Off > int32(e) {
+			r.Off += int32(delta)
+		}
+	}
+}

src/cmd/link/internal/sym/library.go

@@ -4,8 +4,6 @@
 
 package sym
 
-import "cmd/internal/goobj2"
-
 type Library struct {
 	Objref        string
 	Srcref        string
@@ -20,11 +18,6 @@ type Library struct {
 	Main          bool
 	Safe          bool
 	Units         []*CompilationUnit
-
-	Readers []struct { // TODO: probably move this to Loader
-		Reader  *goobj2.Reader
-		Version int
-	}
 }
 
 func (l Library) String() string {