[dev.link] cmd/link: support new object file format

Parse new object file format in the linker. At least we can link
a hello-world program.

Add a basic "loader", which handles symbol references in the
object file.
- mapping between local and global indices
- resolve by-name references
  (TODO: the overwrite logic isn't implemented yet)

Currently we still create sym.Symbol rather early, and, after all
the object files are loaded and indexed references are resolved,
add all symbols to sym.Symbols.

The code here is probably not going in the final version. This
is basically only for debugging purposes -- to make sure the
writer and the reader work as expected.

Change-Id: I895aeea68326fabdb7e5aa1371b8cac7211a09dd
Reviewed-on: https://go-review.googlesource.com/c/go/+/196032
Run-TryBot: Cherry Zhang <cherryyz@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Than McIntosh <thanm@google.com>

src/cmd/link/internal/ld/go.go

@@ -111,7 +111,12 @@ func ldpkg(ctxt *Link, f *bio.Reader, lib *sym.Library, length int64, filename s
 		}
 		p1 += p0
 
-		loadcgo(ctxt, filename, objabi.PathToPrefix(lib.Pkg), data[p0:p1])
+		if *flagNewobj {
+			// loadcgo creates sym.Symbol. Delay this until all the symbols are added.
+			ctxt.cgodata = append(ctxt.cgodata, [3]string{filename, objabi.PathToPrefix(lib.Pkg), data[p0:p1]})
+		} else {
+			loadcgo(ctxt, filename, objabi.PathToPrefix(lib.Pkg), data[p0:p1])
+		}
 	}
 }
 

src/cmd/link/internal/ld/lib.go

@@ -375,15 +375,8 @@ func (ctxt *Link) findLibPath(libname string) string {
 }
 
 func (ctxt *Link) loadlib() {
-	switch ctxt.BuildMode {
-	case BuildModeCShared, BuildModePlugin:
-		s := ctxt.Syms.Lookup("runtime.islibrary", 0)
-		s.Attr |= sym.AttrDuplicateOK
-		s.AddUint8(1)
-	case BuildModeCArchive:
-		s := ctxt.Syms.Lookup("runtime.isarchive", 0)
-		s.Attr |= sym.AttrDuplicateOK
-		s.AddUint8(1)
+	if *flagNewobj {
+		ctxt.loader = objfile.NewLoader()
 	}
 
 	loadinternal(ctxt, "runtime")
@@ -408,6 +401,11 @@ func (ctxt *Link) loadlib() {
 		}
 	}
 
+	// XXX do it here for now
+	if *flagNewobj {
+		ctxt.loadlibfull()
+	}
+
 	for _, lib := range ctxt.Library {
 		if lib.Shlib != "" {
 			if ctxt.Debugvlog > 1 {
@@ -417,6 +415,19 @@ func (ctxt *Link) loadlib() {
 		}
 	}
 
+	switch ctxt.BuildMode {
+	case BuildModeCShared, BuildModePlugin:
+		s := ctxt.Syms.Lookup("runtime.islibrary", 0)
+		s.Type = sym.SNOPTRDATA
+		s.Attr |= sym.AttrDuplicateOK
+		s.AddUint8(1)
+	case BuildModeCArchive:
+		s := ctxt.Syms.Lookup("runtime.isarchive", 0)
+		s.Type = sym.SNOPTRDATA
+		s.Attr |= sym.AttrDuplicateOK
+		s.AddUint8(1)
+	}
+
 	iscgo = ctxt.Syms.ROLookup("x_cgo_init", 0) != nil
 
 	// We now have enough information to determine the link mode.
@@ -843,7 +854,7 @@ func loadobjfile(ctxt *Link, lib *sym.Library) {
 	if err != nil {
 		Exitf("cannot open file %s: %v", lib.File, err)
 	}
-	defer f.Close()
+	//defer f.Close()
 	defer func() {
 		if pkg == "main" && !lib.Main {
 			Exitf("%s: not package main", lib.File)
@@ -1773,7 +1784,12 @@ func ldobj(ctxt *Link, f *bio.Reader, lib *sym.Library, length int64, pn string,
 	default:
 		log.Fatalf("invalid -strictdups flag value %d", *FlagStrictDups)
 	}
-	c := objfile.Load(ctxt.Arch, ctxt.Syms, f, lib, unit, eof-f.Offset(), pn, flags)
+	var c int
+	if *flagNewobj {
+		objfile.LoadNew(ctxt.loader, ctxt.Arch, ctxt.Syms, f, lib, unit, eof-f.Offset(), pn, flags)
+	} else {
+		c = objfile.Load(ctxt.Arch, ctxt.Syms, f, lib, unit, eof-f.Offset(), pn, flags)
+	}
 	strictDupMsgCount += c
 	addImports(ctxt, lib, pn)
 	return nil
@@ -2545,3 +2561,40 @@ func dfs(lib *sym.Library, mark map[*sym.Library]markKind, order *[]*sym.Library
 	mark[lib] = visited
 	*order = append(*order, lib)
 }
+
+func (ctxt *Link) loadlibfull() {
+	// Add references of externally defined symbols.
+	for _, lib := range ctxt.Library {
+		for _, r := range lib.Readers {
+			objfile.LoadRefs(ctxt.loader, r.Reader, lib, ctxt.Arch, ctxt.Syms, r.Version)
+		}
+	}
+
+	// Load full symbol contents, resolve indexed references.
+	for _, lib := range ctxt.Library {
+		for _, r := range lib.Readers {
+			objfile.LoadFull(ctxt.loader, r.Reader, lib, ctxt.Syms, r.Version, ctxt.LibraryByPkg)
+		}
+	}
+
+	// For now, add all symbols to ctxt.Syms.
+	for _, s := range ctxt.loader.Syms {
+		if s != nil && s.Name != "" {
+			ctxt.Syms.Add(s)
+		}
+	}
+
+	// Now load cgo directives.
+	for _, p := range ctxt.cgodata {
+		loadcgo(ctxt, p[0], p[1], p[2])
+	}
+}
+
+func (ctxt *Link) dumpsyms() {
+	for _, s := range ctxt.Syms.Allsym {
+		fmt.Printf("%s %s %p\n", s, s.Type, s)
+		for i := range s.R {
+			fmt.Println("\t", s.R[i].Type, s.R[i].Sym)
+		}
+	}
+}

src/cmd/link/internal/ld/link.go

@@ -35,6 +35,7 @@ import (
 	"cmd/internal/obj"
 	"cmd/internal/objabi"
 	"cmd/internal/sys"
+	"cmd/link/internal/objfile"
 	"cmd/link/internal/sym"
 	"debug/elf"
 	"fmt"
@@ -96,6 +97,9 @@ type Link struct {
 	runtimeCU *sym.CompilationUnit   // One of the runtime CUs, the last one seen.
 
 	relocbuf []byte // temporary buffer for applying relocations
+
+	loader  *objfile.Loader
+	cgodata [][3]string // cgo directives to load, three strings are args for loadcgo
 }
 
 type unresolvedSymKey struct {

src/cmd/link/internal/ld/main.go

@@ -86,6 +86,7 @@ var (
 	flagInterpreter = flag.String("I", "", "use `linker` as ELF dynamic linker")
 	FlagDebugTramp  = flag.Int("debugtramp", 0, "debug trampolines")
 	FlagStrictDups  = flag.Int("strictdups", 0, "sanity check duplicate symbol contents during object file reading (1=warn 2=err).")
+	flagNewobj      = flag.Bool("newobj", false, "use new object file format")
 
 	FlagRound       = flag.Int("R", -1, "set address rounding `quantum`")
 	FlagTextAddr    = flag.Int64("T", -1, "set text segment `address`")
@@ -209,6 +210,7 @@ func Main(arch *sys.Arch, theArch Arch) {
 
 	ctxt.dostrdata()
 	deadcode(ctxt)
+	//ctxt.dumpsyms() // XXX
 	dwarfGenerateDebugInfo(ctxt)
 	if objabi.Fieldtrack_enabled != 0 {
 		fieldtrack(ctxt)

src/cmd/link/internal/sym/library.go

@@ -4,6 +4,8 @@
 
 package sym
 
+import "cmd/internal/goobj2"
+
 type Library struct {
 	Objref        string
 	Srcref        string
@@ -18,6 +20,11 @@ type Library struct {
 	Main          bool
 	Safe          bool
 	Units         []*CompilationUnit
+
+	Readers []struct { // TODO: probably move this to Loader
+		Reader  *goobj2.Reader
+		Version int
+	}
 }
 
 func (l Library) String() string {

src/cmd/link/internal/sym/symbols.go

@@ -86,6 +86,17 @@ func (syms *Symbols) ROLookup(name string, v int) *Symbol {
 	return syms.hash[v][name]
 }
 
+// Add an existing symbol to the symbol table.
+func (syms *Symbols) Add(s *Symbol) {
+	name := s.Name
+	v := int(s.Version)
+	m := syms.hash[v]
+	if _, ok := m[name]; ok {
+		panic(name + " already added")
+	}
+	m[name] = s
+}
+
 // Allocate a new version (i.e. symbol namespace).
 func (syms *Symbols) IncVersion() int {
 	syms.hash = append(syms.hash, make(map[string]*Symbol))