[dev.link] cmd/link: load full symbol contents after deadcode pass

If the new object file format is used, now we load full symbol
contents after the deadcode pass, for reachable symbols only.
We still load some informations early, like relocations and the
contents of type symbols, which are used in the deadcode pass.
If we rewrite deadcode to use index directly, we could delay more
of the loading (to sym.Symbol), and perhaps delay the creation of
sym.Symbol.

TODO: internal linking with host objects doesn't work yet.

Change-Id: I7d4880e8f150e8709ffac277e62191623440e4cf
Reviewed-on: https://go-review.googlesource.com/c/go/+/197258
Run-TryBot: Cherry Zhang <cherryyz@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Than McIntosh <thanm@google.com>

src/cmd/link/internal/ld/deadcode.go

@@ -60,8 +60,8 @@ func deadcode(ctxt *Link) {
 	d.init()
 	d.flood()
 
-	callSym := ctxt.Syms.ROLookup("reflect.Value.Call", sym.SymVerABIInternal)
-	methSym := ctxt.Syms.ROLookup("reflect.Value.Method", sym.SymVerABIInternal)
+	callSym := ctxt.Lookup("reflect.Value.Call", sym.SymVerABIInternal)
+	methSym := ctxt.Lookup("reflect.Value.Method", sym.SymVerABIInternal)
 	reflectSeen := false
 
 	if ctxt.DynlinkingGo() {
@@ -283,7 +283,7 @@ func (d *deadcodepass) init() {
 
 				// We don't keep the go.plugin.exports symbol,
 				// but we do keep the symbols it refers to.
-				exports := d.ctxt.Syms.ROLookup("go.plugin.exports", 0)
+				exports := d.ctxt.Lookup("go.plugin.exports", 0)
 				if exports != nil {
 					for i := range exports.R {
 						d.mark(exports.R[i].Sym, nil)
@@ -298,9 +298,9 @@ func (d *deadcodepass) init() {
 
 	for _, name := range names {
 		// Mark symbol as an data/ABI0 symbol.
-		d.mark(d.ctxt.Syms.ROLookup(name, 0), nil)
+		d.mark(d.ctxt.Lookup(name, 0), nil)
 		// Also mark any Go functions (internal ABI).
-		d.mark(d.ctxt.Syms.ROLookup(name, sym.SymVerABIInternal), nil)
+		d.mark(d.ctxt.Lookup(name, sym.SymVerABIInternal), nil)
 	}
 }
 

src/cmd/link/internal/ld/go.go

@@ -169,7 +169,7 @@ func loadcgo(ctxt *Link, file string, pkg string, p string) {
 			if i := strings.Index(remote, "#"); i >= 0 {
 				remote, q = remote[:i], remote[i+1:]
 			}
-			s := ctxt.Syms.Lookup(local, 0)
+			s := ctxt.LookupOrCreate(local, 0)
 			if s.Type == 0 || s.Type == sym.SXREF || s.Type == sym.SHOSTOBJ {
 				s.SetDynimplib(lib)
 				s.SetExtname(remote)
@@ -188,7 +188,7 @@ func loadcgo(ctxt *Link, file string, pkg string, p string) {
 			}
 			local := f[1]
 
-			s := ctxt.Syms.Lookup(local, 0)
+			s := ctxt.LookupOrCreate(local, 0)
 			s.Type = sym.SHOSTOBJ
 			s.Size = 0
 			continue
@@ -209,7 +209,7 @@ func loadcgo(ctxt *Link, file string, pkg string, p string) {
 			// functions. Link.loadlib will resolve any
 			// ABI aliases we find here (since we may not
 			// yet know it's an alias).
-			s := ctxt.Syms.Lookup(local, 0)
+			s := ctxt.LookupOrCreate(local, 0)
 
 			switch ctxt.BuildMode {
 			case BuildModeCShared, BuildModeCArchive, BuildModePlugin:

src/cmd/link/internal/ld/lib.go

@@ -401,21 +401,24 @@ func (ctxt *Link) loadlib() {
 		}
 	}
 
-	// XXX do it here for now
 	if *flagNewobj {
-		ctxt.loadlibfull()
-	}
-
-	for _, lib := range ctxt.Library {
-		if lib.Shlib != "" {
-			if ctxt.Debugvlog > 1 {
-				ctxt.Logf("%5.2f autolib: %s (from %s)\n", Cputime(), lib.Shlib, lib.Objref)
+		// Add references of externally defined symbols.
+		for _, lib := range ctxt.Library {
+			for _, r := range lib.Readers {
+				objfile.LoadRefs(ctxt.loader, r.Reader, lib, ctxt.Arch, ctxt.Syms, r.Version)
 			}
-			ldshlibsyms(ctxt, lib.Shlib)
+		}
+
+		// Load cgo directives.
+		for _, p := range ctxt.cgodata {
+			loadcgo(ctxt, p[0], p[1], p[2])
 		}
 	}
 
-	iscgo = ctxt.Syms.ROLookup("x_cgo_init", 0) != nil
+	iscgo = ctxt.Lookup("x_cgo_init", 0) != nil
+
+	// Record whether we can use plugins.
+	ctxt.canUsePlugins = (ctxt.Lookup("plugin.Open", sym.SymVerABIInternal) != nil)
 
 	// We now have enough information to determine the link mode.
 	determineLinkMode(ctxt)
@@ -448,8 +451,34 @@ func (ctxt *Link) loadlib() {
 		dynexp[i] = t
 	}
 
+	for _, lib := range ctxt.Library {
+		if lib.Shlib != "" {
+			if ctxt.Debugvlog > 1 {
+				ctxt.Logf("%5.2f autolib: %s (from %s)\n", Cputime(), lib.Shlib, lib.Objref)
+			}
+			ldshlibsyms(ctxt, lib.Shlib)
+		}
+	}
+
+	if ctxt.LinkMode == LinkExternal && !iscgo && ctxt.LibraryByPkg["runtime/cgo"] == nil && !(objabi.GOOS == "darwin" && (ctxt.Arch.Family == sys.AMD64 || ctxt.Arch.Family == sys.I386)) {
+		// This indicates a user requested -linkmode=external.
+		// The startup code uses an import of runtime/cgo to decide
+		// whether to initialize the TLS.  So give it one. This could
+		// be handled differently but it's an unusual case.
+		if lib := loadinternal(ctxt, "runtime/cgo"); lib != nil {
+			if lib.Shlib != "" {
+				ldshlibsyms(ctxt, lib.Shlib)
+			} else {
+				if ctxt.BuildMode == BuildModeShared || ctxt.linkShared {
+					Exitf("cannot implicitly include runtime/cgo in a shared library")
+				}
+				loadobjfile(ctxt, lib)
+			}
+		}
+	}
+
 	// In internal link mode, read the host object files.
-	if ctxt.LinkMode == LinkInternal {
+	if ctxt.LinkMode == LinkInternal && len(hostobj) != 0 {
 		// Drop all the cgo_import_static declarations.
 		// Turns out we won't be needing them.
 		for _, s := range ctxt.Syms.Allsym {
@@ -510,34 +539,23 @@ func (ctxt *Link) loadlib() {
 				*/
 			}
 		}
-	} else {
+	} else if ctxt.LinkMode == LinkExternal {
 		hostlinksetup(ctxt)
 	}
 
 	// We've loaded all the code now.
 	ctxt.Loaded = true
 
-	// Record whether we can use plugins.
-	ctxt.canUsePlugins = (ctxt.Syms.ROLookup("plugin.Open", sym.SymVerABIInternal) != nil)
+	importcycles()
 
-	if ctxt.LinkMode == LinkExternal && !iscgo && ctxt.LibraryByPkg["runtime/cgo"] == nil && !(objabi.GOOS == "darwin" && (ctxt.Arch.Family == sys.AMD64 || ctxt.Arch.Family == sys.I386)) {
-		// This indicates a user requested -linkmode=external.
-		// The startup code uses an import of runtime/cgo to decide
-		// whether to initialize the TLS.  So give it one. This could
-		// be handled differently but it's an unusual case.
-		if lib := loadinternal(ctxt, "runtime/cgo"); lib != nil {
-			if lib.Shlib != "" {
-				ldshlibsyms(ctxt, lib.Shlib)
-			} else {
-				if ctxt.BuildMode == BuildModeShared || ctxt.linkShared {
-					Exitf("cannot implicitly include runtime/cgo in a shared library")
-				}
-				loadobjfile(ctxt, lib)
+	// For now, load relocations for dead-code elimination.
+	if *flagNewobj {
+		for _, lib := range ctxt.Library {
+			for _, r := range lib.Readers {
+				objfile.LoadReloc(ctxt.loader, r.Reader, lib, r.Version, ctxt.LibraryByPkg)
 			}
 		}
 	}
-
-	importcycles()
 }
 
 // Set up flags and special symbols depending on the platform build mode.
@@ -1907,7 +1925,7 @@ func ldshlibsyms(ctxt *Link, shlib string) {
 			ver = sym.SymVerABIInternal
 		}
 
-		lsym := ctxt.Syms.Lookup(elfsym.Name, ver)
+		lsym := ctxt.LookupOrCreate(elfsym.Name, ver)
 		// Because loadlib above loads all .a files before loading any shared
 		// libraries, any non-dynimport symbols we find that duplicate symbols
 		// already loaded should be ignored (the symbols from the .a files
@@ -2526,17 +2544,10 @@ func dfs(lib *sym.Library, mark map[*sym.Library]markKind, order *[]*sym.Library
 }
 
 func (ctxt *Link) loadlibfull() {
-	// Add references of externally defined symbols.
-	for _, lib := range ctxt.Library {
-		for _, r := range lib.Readers {
-			objfile.LoadRefs(ctxt.loader, r.Reader, lib, ctxt.Arch, ctxt.Syms, r.Version)
-		}
-	}
-
 	// Load full symbol contents, resolve indexed references.
 	for _, lib := range ctxt.Library {
 		for _, r := range lib.Readers {
-			objfile.LoadFull(ctxt.loader, r.Reader, lib, ctxt.Syms, r.Version, ctxt.LibraryByPkg)
+			objfile.LoadFull(ctxt.loader, r.Reader, lib, r.Version, ctxt.LibraryByPkg)
 		}
 	}
 
@@ -2546,11 +2557,6 @@ func (ctxt *Link) loadlibfull() {
 			ctxt.Syms.Add(s)
 		}
 	}
-
-	// Now load cgo directives.
-	for _, p := range ctxt.cgodata {
-		loadcgo(ctxt, p[0], p[1], p[2])
-	}
 }
 
 func (ctxt *Link) dumpsyms() {

src/cmd/link/internal/ld/link.go

@@ -176,3 +176,32 @@ func addImports(ctxt *Link, l *sym.Library, pn string) {
 	}
 	l.ImportStrings = nil
 }
+
+// convenient helper during the transition period.
+func (ctxt *Link) Lookup(name string, ver int) *sym.Symbol {
+	if *flagNewobj {
+		i := ctxt.loader.Lookup(name, ver)
+		if i == 0 {
+			return nil
+		}
+		return ctxt.loader.Syms[i]
+	} else {
+		return ctxt.Syms.ROLookup(name, ver)
+	}
+}
+
+// convenient helper during the transition period.
+func (ctxt *Link) LookupOrCreate(name string, ver int) *sym.Symbol {
+	if *flagNewobj {
+		i := ctxt.loader.Lookup(name, ver)
+		if i != 0 {
+			return ctxt.loader.Syms[i]
+		}
+		ctxt.loader.AddExtSym(name, ver)
+		s := ctxt.Syms.Newsym(name, ver)
+		ctxt.loader.Syms = append(ctxt.loader.Syms, s)
+		return s
+	} else {
+		return ctxt.Syms.Lookup(name, ver)
+	}
+}

src/cmd/link/internal/ld/main.go

@@ -209,6 +209,9 @@ func Main(arch *sys.Arch, theArch Arch) {
 	ctxt.loadlib()
 
 	deadcode(ctxt)
+	if *flagNewobj {
+		ctxt.loadlibfull() // XXX do it here for now
+	}
 	ctxt.linksetup()
 	ctxt.dostrdata()
 

src/cmd/link/internal/objfile/objfile2.go

@@ -225,7 +225,10 @@ func preprocess(arch *sys.Arch, s *sym.Symbol) {
 	}
 }
 
-func LoadFull(l *Loader, r *goobj2.Reader, lib *sym.Library, syms *sym.Symbols, localSymVersion int, libByPkg map[string]*sym.Library) {
+// Load relocations for building the dependency graph in deadcode pass.
+// For now, we load symbol types, relocations, gotype, and the contents
+// of type symbols, which are needed in deadcode.
+func LoadReloc(l *Loader, r *goobj2.Reader, lib *sym.Library, localSymVersion int, libByPkg map[string]*sym.Library) {
 	// PkgIdx
 	pkglist := r.Pkglist()
 
@@ -262,7 +265,6 @@ func LoadFull(l *Loader, r *goobj2.Reader, lib *sym.Library, syms *sym.Symbols,
 		return l.Syms[i]
 	}
 
-	pcdataBase := r.PcdataBase()
 	for i, n := 0, r.NSym()+r.NNonpkgdef(); i < n; i++ {
 		s := l.Syms[istart+i]
 		if s == nil || s.Name == "" {
@@ -272,28 +274,30 @@ func LoadFull(l *Loader, r *goobj2.Reader, lib *sym.Library, syms *sym.Symbols,
 		osym := goobj2.Sym{}
 		osym.Read(r, r.SymOff(i))
 		name := strings.Replace(osym.Name, "\"\".", pkgprefix, -1)
-		if s.Name != name {
+		if s.Name != name { // Sanity check. We can remove it in the final version.
 			fmt.Println("name mismatch:", lib, i, s.Name, name)
 			panic("name mismatch")
 		}
 
-		dupok := osym.Flag&goobj2.SymFlagDupok != 0
-		local := osym.Flag&goobj2.SymFlagLocal != 0
-		makeTypelink := osym.Flag&goobj2.SymFlagTypelink != 0
-		nreloc := r.NReloc(i)
-		datasize := r.DataSize(i)
-		size := osym.Siz
-
-		t := sym.AbiSymKindToSymKind[objabi.SymKind(osym.Type)]
 		if s.Type != 0 && s.Type != sym.SXREF {
 			fmt.Println("symbol already processed:", lib, i, s)
 			panic("symbol already processed")
 		}
 
-		// Symbol data
-		s.P = r.BytesAt(r.DataOff(i), datasize)
+		t := sym.AbiSymKindToSymKind[objabi.SymKind(osym.Type)]
+		if t == sym.SXREF {
+			log.Fatalf("bad sxref")
+		}
+		if t == 0 {
+			log.Fatalf("missing type for %s in %s", s.Name, lib)
+		}
+		if t == sym.SBSS && (s.Type == sym.SRODATA || s.Type == sym.SNOPTRBSS) {
+			t = s.Type
+		}
+		s.Type = t
 
 		// Reloc
+		nreloc := r.NReloc(i)
 		s.R = make([]sym.Reloc, nreloc)
 		for j := range s.R {
 			rel := goobj2.Reloc{}
@@ -307,9 +311,13 @@ func LoadFull(l *Loader, r *goobj2.Reader, lib *sym.Library, syms *sym.Symbols,
 			}
 		}
 
-		// Aux symbol info
-		isym := -1
-		funcdata := make([]goobj2.SymRef, 0, 4)
+		// XXX deadcode needs symbol data for type symbols. Read it now.
+		if strings.HasPrefix(name, "type.") {
+			s.P = r.BytesAt(r.DataOff(i), r.DataSize(i))
+			s.Size = int64(osym.Siz)
+		}
+
+		// Aux symbol
 		naux := r.NAux(i)
 		for j := 0; j < naux; j++ {
 			a := goobj2.Aux{}
@@ -320,13 +328,115 @@ func LoadFull(l *Loader, r *goobj2.Reader, lib *sym.Library, syms *sym.Symbols,
 				if typ != nil {
 					s.Gotype = typ
 				}
+			case goobj2.AuxFuncdata:
+				pc := s.FuncInfo
+				if pc == nil {
+					pc = &sym.FuncInfo{Funcdata: make([]*sym.Symbol, 0, 4)}
+					s.FuncInfo = pc
+				}
+				pc.Funcdata = append(pc.Funcdata, resolveSymRef(a.Sym))
+			}
+		}
+
+		if s.Type == sym.STEXT {
+			dupok := osym.Flag&goobj2.SymFlagDupok != 0
+			if !dupok {
+				if s.Attr.OnList() {
+					log.Fatalf("symbol %s listed multiple times", s.Name)
+				}
+				s.Attr |= sym.AttrOnList
+				lib.Textp = append(lib.Textp, s)
+			} else {
+				// there may ba a dup in another package
+				// put into a temp list and add to text later
+				lib.DupTextSyms = append(lib.DupTextSyms, s)
+			}
+		}
+	}
+}
+
+// Load full contents.
+// TODO: For now, some contents are already load in LoadReloc. Maybe
+// we should combine LoadReloc back into this, once we rewrite deadcode
+// pass to use index directly.
+func LoadFull(l *Loader, r *goobj2.Reader, lib *sym.Library, localSymVersion int, libByPkg map[string]*sym.Library) {
+	// PkgIdx
+	pkglist := r.Pkglist()
+
+	pkgprefix := objabi.PathToPrefix(lib.Pkg) + "."
+	istart := l.StartIndex(r)
+
+	resolveSymRef := func(s goobj2.SymRef) *sym.Symbol {
+		var rr *goobj2.Reader
+		switch p := s.PkgIdx; p {
+		case goobj2.PkgIdxInvalid:
+			if s.SymIdx != 0 {
+				panic("bad sym ref")
+			}
+			return nil
+		case goobj2.PkgIdxNone:
+			// Resolve by name
+			i := int(s.SymIdx) + r.NSym()
+			osym := goobj2.Sym{}
+			osym.Read(r, r.SymOff(i))
+			name := strings.Replace(osym.Name, "\"\".", pkgprefix, -1)
+			v := abiToVer(osym.ABI, localSymVersion)
+			nv := nameVer{name, v}
+			i = l.symsByName[nv]
+			return l.Syms[i]
+		case goobj2.PkgIdxSelf:
+			rr = r
+		default:
+			pkg := pkglist[p]
+			rr = libByPkg[pkg].Readers[0].Reader // typically Readers[0] is go object (others are asm)
+		}
+		i := l.ToGlobal(rr, int(s.SymIdx))
+		return l.Syms[i]
+	}
+
+	pcdataBase := r.PcdataBase()
+	for i, n := 0, r.NSym()+r.NNonpkgdef(); i < n; i++ {
+		s := l.Syms[istart+i]
+		if s == nil || s.Name == "" {
+			continue
+		}
+		if !s.Attr.Reachable() && (s.Type < sym.SDWARFSECT || s.Type > sym.SDWARFLINES) {
+			// No need to load unreachable symbols.
+			// XXX DWARF symbols may be used but are not marked reachable.
+			continue
+		}
+
+		osym := goobj2.Sym{}
+		osym.Read(r, r.SymOff(i))
+		name := strings.Replace(osym.Name, "\"\".", pkgprefix, -1)
+		if s.Name != name { // Sanity check. We can remove it in the final version.
+			fmt.Println("name mismatch:", lib, i, s.Name, name)
+			panic("name mismatch")
+		}
+
+		dupok := osym.Flag&goobj2.SymFlagDupok != 0
+		local := osym.Flag&goobj2.SymFlagLocal != 0
+		makeTypelink := osym.Flag&goobj2.SymFlagTypelink != 0
+		datasize := r.DataSize(i)
+		size := osym.Siz
+
+		// Symbol data
+		s.P = r.BytesAt(r.DataOff(i), datasize)
+
+		// Aux symbol info
+		isym := -1
+		naux := r.NAux(i)
+		for j := 0; j < naux; j++ {
+			a := goobj2.Aux{}
+			a.Read(r, r.AuxOff(i, j))
+			switch a.Type {
+			case goobj2.AuxGotype, goobj2.AuxFuncdata:
+				// already loaded
 			case goobj2.AuxFuncInfo:
 				if a.Sym.PkgIdx != goobj2.PkgIdxSelf {
 					panic("funcinfo symbol not defined in current package")
 				}
 				isym = int(a.Sym.SymIdx)
-			case goobj2.AuxFuncdata:
-				funcdata = append(funcdata, a.Sym)
 			default:
 				panic("unknown aux type")
 			}
@@ -336,16 +446,6 @@ func LoadFull(l *Loader, r *goobj2.Reader, lib *sym.Library, syms *sym.Symbols,
 		if dupok {
 			s.Attr |= sym.AttrDuplicateOK
 		}
-		if t == sym.SXREF {
-			log.Fatalf("bad sxref")
-		}
-		if t == 0 {
-			log.Fatalf("missing type for %s in %s", s.Name, lib)
-		}
-		if t == sym.SBSS && (s.Type == sym.SRODATA || s.Type == sym.SNOPTRBSS) {
-			t = s.Type
-		}
-		s.Type = t
 		if s.Size < int64(size) {
 			s.Size = int64(size)
 		}
@@ -355,17 +455,6 @@ func LoadFull(l *Loader, r *goobj2.Reader, lib *sym.Library, syms *sym.Symbols,
 		if s.Type != sym.STEXT {
 			continue
 		}
-		if !dupok {
-			if s.Attr.OnList() {
-				log.Fatalf("symbol %s listed multiple times", s.Name)
-			}
-			s.Attr |= sym.AttrOnList
-			lib.Textp = append(lib.Textp, s)
-		} else {
-			// there may ba a dup in another package
-			// put into a temp list and add to text later
-			lib.DupTextSyms = append(lib.DupTextSyms, s)
-		}
 
 		// FuncInfo
 		if isym == -1 {
@@ -389,15 +478,16 @@ func LoadFull(l *Loader, r *goobj2.Reader, lib *sym.Library, syms *sym.Symbols,
 		}
 
 		info.Pcdata = append(info.Pcdata, info.PcdataEnd) // for the ease of knowing where it ends
-		pc := &sym.FuncInfo{
-			Args:        int32(info.Args),
-			Locals:      int32(info.Locals),
-			Pcdata:      make([]sym.Pcdata, len(info.Pcdata)-1), // -1 as we appended one above
-			Funcdata:    make([]*sym.Symbol, len(info.Funcdataoff)),
-			Funcdataoff: make([]int64, len(info.Funcdataoff)),
-			File:        make([]*sym.Symbol, len(info.File)),
-		}
-		s.FuncInfo = pc
+		pc := s.FuncInfo
+		if pc == nil {
+			pc = &sym.FuncInfo{}
+			s.FuncInfo = pc
+		}
+		pc.Args = int32(info.Args)
+		pc.Locals = int32(info.Locals)
+		pc.Pcdata = make([]sym.Pcdata, len(info.Pcdata)-1) // -1 as we appended one above
+		pc.Funcdataoff = make([]int64, len(info.Funcdataoff))
+		pc.File = make([]*sym.Symbol, len(info.File))
 		pc.Pcsp.P = r.BytesAt(pcdataBase+info.Pcsp, int(info.Pcfile-info.Pcsp))
 		pc.Pcfile.P = r.BytesAt(pcdataBase+info.Pcfile, int(info.Pcline-info.Pcfile))
 		pc.Pcline.P = r.BytesAt(pcdataBase+info.Pcline, int(info.Pcinline-info.Pcline))
@@ -405,8 +495,7 @@ func LoadFull(l *Loader, r *goobj2.Reader, lib *sym.Library, syms *sym.Symbols,
 		for k := range pc.Pcdata {
 			pc.Pcdata[k].P = r.BytesAt(pcdataBase+info.Pcdata[k], int(info.Pcdata[k+1]-info.Pcdata[k]))
 		}
-		for k := range pc.Funcdata {
-			pc.Funcdata[k] = resolveSymRef(funcdata[k])
+		for k := range pc.Funcdataoff {
 			pc.Funcdataoff[k] = int64(info.Funcdataoff[k])
 		}
 		for k := range pc.File {