[dev.link] cmd/link: create sym.Symbols after deadcode in newobj mode

With the new object files, now we can run the deadcode pass on
indices instead of Symbol structs, so we can delay creating
Symbols after the deadcode pass. Then we only need to create
reachable symbols.

Not create Symbols in LoadNew and LoadRefs, and recombine
LoadReloc into LoadFull.

Split loadcgo into two parts: the first finds root symbols, the
second create Symbols and sets attributes. The first runs before
the deadcode pass, while the second runs after.

TODO: currently there are still symbols that are not marked
reachable but still used. This includes DWARF symbols, file
symbols, and type symbols that are referenced by DWARF symbols.
We still need to create them (conservatively).

Change-Id: I695779c9312be9d49ab1683957ac3e72e1f65a1e
Reviewed-on: https://go-review.googlesource.com/c/go/+/199643
Run-TryBot: Cherry Zhang <cherryyz@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Than McIntosh <thanm@google.com>

src/cmd/link/internal/ld/deadcode.go

@@ -65,8 +65,8 @@ func deadcode(ctxt *Link) {
 	d.init()
 	d.flood()
 
-	callSym := ctxt.Lookup("reflect.Value.Call", sym.SymVerABIInternal)
-	methSym := ctxt.Lookup("reflect.Value.Method", sym.SymVerABIInternal)
+	callSym := ctxt.Syms.ROLookup("reflect.Value.Call", sym.SymVerABIInternal)
+	methSym := ctxt.Syms.ROLookup("reflect.Value.Method", sym.SymVerABIInternal)
 	reflectSeen := false
 
 	if ctxt.DynlinkingGo() {
@@ -292,7 +292,7 @@ func (d *deadcodepass) init() {
 
 				// We don't keep the go.plugin.exports symbol,
 				// but we do keep the symbols it refers to.
-				exports := d.ctxt.Lookup("go.plugin.exports", 0)
+				exports := d.ctxt.Syms.ROLookup("go.plugin.exports", 0)
 				if exports != nil {
 					for i := range exports.R {
 						d.mark(exports.R[i].Sym, nil)
@@ -307,9 +307,9 @@ func (d *deadcodepass) init() {
 
 	for _, name := range names {
 		// Mark symbol as an data/ABI0 symbol.
-		d.mark(d.ctxt.Lookup(name, 0), nil)
+		d.mark(d.ctxt.Syms.ROLookup(name, 0), nil)
 		// Also mark any Go functions (internal ABI).
-		d.mark(d.ctxt.Lookup(name, sym.SymVerABIInternal), nil)
+		d.mark(d.ctxt.Syms.ROLookup(name, sym.SymVerABIInternal), nil)
 	}
 }
 

src/cmd/link/internal/ld/deadcode2.go

@@ -76,8 +76,12 @@ func (d *deadcodePass2) init() {
 			}
 		}
 	}
-	for _, s := range dynexp {
-		d.mark(d.loader.Lookup(s.Name, int(s.Version)))
+	dynexpMap := d.ctxt.cgo_export_dynamic
+	if d.ctxt.LinkMode == LinkExternal {
+		dynexpMap = d.ctxt.cgo_export_static
+	}
+	for exp := range dynexpMap {
+		names = append(names, exp)
 	}
 
 	for _, name := range names {
@@ -215,16 +219,6 @@ func deadcode2(ctxt *Link) {
 			}
 		}
 	}
-
-	// Set reachable attr for now.
-	for i := 1; i < n; i++ {
-		if loader.Reachable.Has(objfile.Sym(i)) {
-			s := loader.Syms[i]
-			if s != nil && s.Name != "" {
-				s.Attr.Set(sym.AttrReachable, true)
-			}
-		}
-	}
 }
 
 // methodref2 holds the relocations from a receiver type symbol to its

src/cmd/link/internal/ld/go.go

@@ -110,13 +110,7 @@ func ldpkg(ctxt *Link, f *bio.Reader, lib *sym.Library, length int64, filename s
 			return
 		}
 		p1 += p0
-
-		if *flagNewobj {
-			// loadcgo creates sym.Symbol. Delay this until all the symbols are added.
-			ctxt.cgodata = append(ctxt.cgodata, [3]string{filename, objabi.PathToPrefix(lib.Pkg), data[p0:p1]})
-		} else {
-			loadcgo(ctxt, filename, objabi.PathToPrefix(lib.Pkg), data[p0:p1])
-		}
+		loadcgo(ctxt, filename, objabi.PathToPrefix(lib.Pkg), data[p0:p1])
 	}
 }
 
@@ -128,6 +122,39 @@ func loadcgo(ctxt *Link, file string, pkg string, p string) {
 		return
 	}
 
+	// Find cgo_export symbols. They are roots in the deadcode pass.
+	for _, f := range directives {
+		switch f[0] {
+		case "cgo_export_static", "cgo_export_dynamic":
+			if len(f) < 2 || len(f) > 3 {
+				continue
+			}
+			local := f[1]
+			switch ctxt.BuildMode {
+			case BuildModeCShared, BuildModeCArchive, BuildModePlugin:
+				if local == "main" {
+					continue
+				}
+			}
+			local = expandpkg(local, pkg)
+			if f[0] == "cgo_export_static" {
+				ctxt.cgo_export_static[local] = true
+			} else {
+				ctxt.cgo_export_dynamic[local] = true
+			}
+		}
+	}
+
+	if *flagNewobj {
+		// Record the directives. We'll process them later after Symbols are created.
+		ctxt.cgodata = append(ctxt.cgodata, cgodata{file, pkg, directives})
+	} else {
+		setCgoAttr(ctxt, file, pkg, directives)
+	}
+}
+
+// Set symbol attributes or flags based on cgo directives.
+func setCgoAttr(ctxt *Link, file string, pkg string, directives [][]string) {
 	for _, f := range directives {
 		switch f[0] {
 		case "cgo_import_dynamic":
@@ -169,7 +196,7 @@ func loadcgo(ctxt *Link, file string, pkg string, p string) {
 			if i := strings.Index(remote, "#"); i >= 0 {
 				remote, q = remote[:i], remote[i+1:]
 			}
-			s := ctxt.LookupOrCreate(local, 0)
+			s := ctxt.Syms.Lookup(local, 0)
 			if s.Type == 0 || s.Type == sym.SXREF || s.Type == sym.SHOSTOBJ {
 				s.SetDynimplib(lib)
 				s.SetExtname(remote)
@@ -188,7 +215,7 @@ func loadcgo(ctxt *Link, file string, pkg string, p string) {
 			}
 			local := f[1]
 
-			s := ctxt.LookupOrCreate(local, 0)
+			s := ctxt.Syms.Lookup(local, 0)
 			s.Type = sym.SHOSTOBJ
 			s.Size = 0
 			continue
@@ -209,7 +236,7 @@ func loadcgo(ctxt *Link, file string, pkg string, p string) {
 			// functions. Link.loadlib will resolve any
 			// ABI aliases we find here (since we may not
 			// yet know it's an alias).
-			s := ctxt.LookupOrCreate(local, 0)
+			s := ctxt.Syms.Lookup(local, 0)
 
 			switch ctxt.BuildMode {
 			case BuildModeCShared, BuildModeCArchive, BuildModePlugin:
@@ -228,7 +255,6 @@ func loadcgo(ctxt *Link, file string, pkg string, p string) {
 
 			if !s.Attr.CgoExport() {
 				s.SetExtname(remote)
-				dynexp = append(dynexp, s)
 			} else if s.Extname() != remote {
 				fmt.Fprintf(os.Stderr, "%s: conflicting cgo_export directives: %s as %s and %s\n", os.Args[0], s.Name, s.Extname(), remote)
 				nerrors++

src/cmd/link/internal/ld/lib.go

@@ -379,6 +379,9 @@ func (ctxt *Link) loadlib() {
 		ctxt.loader = objfile.NewLoader()
 	}
 
+	ctxt.cgo_export_static = make(map[string]bool)
+	ctxt.cgo_export_dynamic = make(map[string]bool)
+
 	loadinternal(ctxt, "runtime")
 	if ctxt.Arch.Family == sys.ARM {
 		loadinternal(ctxt, "math")
@@ -402,60 +405,16 @@ func (ctxt *Link) loadlib() {
 	}
 
 	if *flagNewobj {
-		// Add references of externally defined symbols.
-		objfile.LoadRefs(ctxt.loader, ctxt.Arch, ctxt.Syms)
-
-		// Load cgo directives.
-		for _, p := range ctxt.cgodata {
-			loadcgo(ctxt, p[0], p[1], p[2])
-		}
+		iscgo = ctxt.loader.Lookup("x_cgo_init", 0) != 0
+		ctxt.canUsePlugins = ctxt.loader.Lookup("plugin.Open", sym.SymVerABIInternal) != 0
+	} else {
+		iscgo = ctxt.Syms.ROLookup("x_cgo_init", 0) != nil
+		ctxt.canUsePlugins = ctxt.Syms.ROLookup("plugin.Open", sym.SymVerABIInternal) != nil
 	}
 
-	iscgo = ctxt.Lookup("x_cgo_init", 0) != nil
-
-	// Record whether we can use plugins.
-	ctxt.canUsePlugins = (ctxt.Lookup("plugin.Open", sym.SymVerABIInternal) != nil)
-
 	// We now have enough information to determine the link mode.
 	determineLinkMode(ctxt)
 
-	// Now that we know the link mode, trim the dynexp list.
-	x := sym.AttrCgoExportDynamic
-
-	if ctxt.LinkMode == LinkExternal {
-		x = sym.AttrCgoExportStatic
-	}
-	w := 0
-	for i := range dynexp {
-		if dynexp[i].Attr&x != 0 {
-			dynexp[w] = dynexp[i]
-			w++
-		}
-	}
-	dynexp = dynexp[:w]
-
-	// Resolve ABI aliases in the list of cgo-exported functions.
-	// This is necessary because we load the ABI0 symbol for all
-	// cgo exports.
-	for i, s := range dynexp {
-		if s.Type != sym.SABIALIAS {
-			continue
-		}
-		t := resolveABIAlias(s)
-		t.Attr |= s.Attr
-		t.SetExtname(s.Extname())
-		dynexp[i] = t
-	}
-
-	for _, lib := range ctxt.Library {
-		if lib.Shlib != "" {
-			if ctxt.Debugvlog > 1 {
-				ctxt.Logf("%5.2f autolib: %s (from %s)\n", Cputime(), lib.Shlib, lib.Objref)
-			}
-			ldshlibsyms(ctxt, lib.Shlib)
-		}
-	}
-
 	if ctxt.LinkMode == LinkExternal && !iscgo && ctxt.LibraryByPkg["runtime/cgo"] == nil && !(objabi.GOOS == "darwin" && (ctxt.Arch.Family == sys.AMD64 || ctxt.Arch.Family == sys.I386)) {
 		// This indicates a user requested -linkmode=external.
 		// The startup code uses an import of runtime/cgo to decide
@@ -473,6 +432,25 @@ func (ctxt *Link) loadlib() {
 		}
 	}
 
+	if *flagNewobj {
+		// Add references of externally defined symbols.
+		objfile.LoadRefs(ctxt.loader, ctxt.Arch, ctxt.Syms)
+	}
+
+	// Now that we know the link mode, set the dynexp list.
+	if !*flagNewobj { // set this later in newobj mode
+		setupdynexp(ctxt)
+	}
+
+	for _, lib := range ctxt.Library {
+		if lib.Shlib != "" {
+			if ctxt.Debugvlog > 1 {
+				ctxt.Logf("%5.2f autolib: %s (from %s)\n", Cputime(), lib.Shlib, lib.Objref)
+			}
+			ldshlibsyms(ctxt, lib.Shlib)
+		}
+	}
+
 	// In internal link mode, read the host object files.
 	if ctxt.LinkMode == LinkInternal && len(hostobj) != 0 {
 		// Drop all the cgo_import_static declarations.
@@ -545,6 +523,35 @@ func (ctxt *Link) loadlib() {
 	importcycles()
 }
 
+// Set up dynexp list.
+func setupdynexp(ctxt *Link) {
+	dynexpMap := ctxt.cgo_export_dynamic
+	if ctxt.LinkMode == LinkExternal {
+		dynexpMap = ctxt.cgo_export_static
+	}
+	dynexp = make([]*sym.Symbol, 0, len(dynexpMap))
+	for exp := range dynexpMap {
+		s := ctxt.Syms.Lookup(exp, 0)
+		dynexp = append(dynexp, s)
+	}
+
+	// Resolve ABI aliases in the list of cgo-exported functions.
+	// This is necessary because we load the ABI0 symbol for all
+	// cgo exports.
+	for i, s := range dynexp {
+		if s.Type != sym.SABIALIAS {
+			continue
+		}
+		t := resolveABIAlias(s)
+		t.Attr |= s.Attr
+		t.SetExtname(s.Extname())
+		dynexp[i] = t
+	}
+
+	ctxt.cgo_export_static = nil
+	ctxt.cgo_export_dynamic = nil
+}
+
 // Set up flags and special symbols depending on the platform build mode.
 func (ctxt *Link) linksetup() {
 	switch ctxt.BuildMode {
@@ -1923,7 +1930,7 @@ func ldshlibsyms(ctxt *Link, shlib string) {
 			ver = sym.SymVerABIInternal
 		}
 
-		lsym := ctxt.LookupOrCreate(elfsym.Name, ver)
+		lsym := ctxt.Syms.Lookup(elfsym.Name, ver)
 		// Because loadlib above loads all .a files before loading any shared
 		// libraries, any non-dynimport symbols we find that duplicate symbols
 		// already loaded should be ignored (the symbols from the .a files
@@ -2543,8 +2550,7 @@ func dfs(lib *sym.Library, mark map[*sym.Library]markKind, order *[]*sym.Library
 
 func (ctxt *Link) loadlibfull() {
 	// Load full symbol contents, resolve indexed references.
-	objfile.LoadReloc(ctxt.loader)
-	objfile.LoadFull(ctxt.loader)
+	objfile.LoadFull(ctxt.loader, ctxt.Arch, ctxt.Syms)
 
 	// For now, add all symbols to ctxt.Syms.
 	for _, s := range ctxt.loader.Syms {
@@ -2553,8 +2559,16 @@ func (ctxt *Link) loadlibfull() {
 		}
 	}
 
+	// Load cgo directives.
+	for _, d := range ctxt.cgodata {
+		setCgoAttr(ctxt, d.file, d.pkg, d.directives)
+	}
+
+	setupdynexp(ctxt)
+
 	// Drop the reference.
 	ctxt.loader = nil
+	ctxt.cgodata = nil
 
 	addToTextp(ctxt)
 }

src/cmd/link/internal/ld/link.go

@@ -99,7 +99,16 @@ type Link struct {
 	relocbuf []byte // temporary buffer for applying relocations
 
 	loader  *objfile.Loader
-	cgodata [][3]string // cgo directives to load, three strings are args for loadcgo
+	cgodata []cgodata // cgo directives to load, three strings are args for loadcgo
+
+	cgo_export_static  map[string]bool
+	cgo_export_dynamic map[string]bool
+}
+
+type cgodata struct {
+	file       string
+	pkg        string
+	directives [][]string
 }
 
 type unresolvedSymKey struct {
@@ -176,32 +185,3 @@ func addImports(ctxt *Link, l *sym.Library, pn string) {
 	}
 	l.ImportStrings = nil
 }
-
-// convenient helper during the transition period.
-func (ctxt *Link) Lookup(name string, ver int) *sym.Symbol {
-	if *flagNewobj {
-		i := ctxt.loader.Lookup(name, ver)
-		if i == 0 {
-			return nil
-		}
-		return ctxt.loader.Syms[i]
-	} else {
-		return ctxt.Syms.ROLookup(name, ver)
-	}
-}
-
-// convenient helper during the transition period.
-func (ctxt *Link) LookupOrCreate(name string, ver int) *sym.Symbol {
-	if *flagNewobj {
-		i := ctxt.loader.Lookup(name, ver)
-		if i != 0 {
-			return ctxt.loader.Syms[i]
-		}
-		ctxt.loader.AddExtSym(name, ver)
-		s := ctxt.Syms.Newsym(name, ver)
-		ctxt.loader.Syms = append(ctxt.loader.Syms, s)
-		return s
-	} else {
-		return ctxt.Syms.Lookup(name, ver)
-	}
-}