cmd/compile: export inlined function bodies

Completed implementation for exporting inlined functions
using the new binary export format. This change passes
(export GO_GCFLAGS=-newexport; make all.bash) but for
gc's builtin_test.go which we need to adjust before enabling
this code by default.

For a high-level description of the export format see the
comment at the top of bexport.go.

Major changes:

1) The export format for the platform independent export data
   changed: When we export inlined function bodies, additional
   objects (other functions, types, etc.) that are referred to
   by the function bodies will need to be exported. While this
   doesn't affect the platform-independent portion directly, it
   adds more objects to the exportlist while we are exporting.
   Instead of trying to sort the objects into groups, just export
   objects as they appear in the export list. This is slightly
   less compact (one extra byte per object), but it is simpler
   and much more flexible.

2) The export format contains now three sections: 1) The plat-
   form independent objects, 2) the objects pulled in for export
   via inlined function bodies, and 3) the inlined function bodies.

3) Completed the exporting and importing code for inlined function
   bodies. The format is completely compiler-specific and easily
   changeable w/o affecting other tools. There is still quite a
   bit of room for denser encoding. This can happen at any time
   in the future.

This change contains also the adjustments for go/internal/gcimporter,
necessary because of the export format change 1) mentioned above.

For #13241.

Change-Id: I86bca0bd984b12ccf13d0d30892e6e25f6d04ed5
Reviewed-on: https://go-review.googlesource.com/21172
Run-TryBot: Robert Griesemer <gri@golang.org>
Reviewed-by: Matthew Dempsky <mdempsky@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>

src/cmd/compile/internal/gc/dcl.go

@@ -79,18 +79,17 @@ func markdcl() {
 	block = blockgen
 }
 
-func dumpdcl(st string) {
+// keep around for debugging
+func dumpdclstack() {
 	i := 0
 	for d := dclstack; d != nil; d = d.Link {
-		i++
-		fmt.Printf("    %.2d %p", i, d)
-		if d.Name == "" {
-			fmt.Printf("\n")
-			continue
+		fmt.Printf("%6d  %p", i, d)
+		if d.Name != "" {
+			fmt.Printf("  '%s'  %v\n", d.Name, Pkglookup(d.Name, d.Pkg))
+		} else {
+			fmt.Printf("  ---\n")
 		}
-
-		fmt.Printf(" '%s'", d.Name)
-		fmt.Printf(" %v\n", Pkglookup(d.Name, d.Pkg))
+		i++
 	}
 }
 
@@ -250,11 +249,8 @@ func variter(vl []*Node, t *Node, el []*Node) []*Node {
 				Yyerror("missing expression in var declaration")
 				break
 			}
-
 			e = el[0]
 			el = el[1:]
-		} else {
-			e = nil
 		}
 
 		v.Op = ONAME
@@ -527,7 +523,7 @@ func ifacedcl(n *Node) {
 func funchdr(n *Node) {
 	// change the declaration context from extern to auto
 	if Funcdepth == 0 && dclcontext != PEXTERN {
-		Fatalf("funchdr: dclcontext")
+		Fatalf("funchdr: dclcontext = %d", dclcontext)
 	}
 
 	if importpkg == nil && n.Func.Nname != nil {
@@ -678,7 +674,7 @@ func funcargs2(t *Type) {
 func funcbody(n *Node) {
 	// change the declaration context from auto to extern
 	if dclcontext != PAUTO {
-		Fatalf("funcbody: dclcontext")
+		Fatalf("funcbody: unexpected dclcontext %d", dclcontext)
 	}
 	popdcl()
 	Funcdepth--

src/cmd/compile/internal/gc/export.go

@@ -380,9 +380,8 @@ func dumpexport() {
 	if forceNewExport || newexport != 0 {
 		// binary export
 		// The linker also looks for the $$ marker - use char after $$ to distinguish format.
-		exportf("\n$$B\n")        // indicate binary format
-		const verifyExport = true // enable to check format changes
-		if verifyExport {
+		exportf("\n$$B\n") // indicate binary format
+		if debugFormat {
 			// save a copy of the export data
 			var copy bytes.Buffer
 			bcopy := obj.Binitw(&copy)
@@ -430,9 +429,8 @@ func dumpexport() {
 		}
 
 		// exportlist grows during iteration - cannot use range
-		for len(exportlist) > 0 {
-			n := exportlist[0]
-			exportlist = exportlist[1:]
+		for i := 0; i < len(exportlist); i++ {
+			n := exportlist[i]
 			lineno = n.Lineno
 			dumpsym(n.Sym)
 		}

src/cmd/compile/internal/gc/fmt.go

@@ -801,7 +801,7 @@ func stmtfmt(n *Node) string {
 		}
 
 	// Don't export "v = <N>" initializing statements, hope they're always
-	// preceded by the DCL which will be re-parsed and typecheck to reproduce
+	// preceded by the DCL which will be re-parsed and typechecked to reproduce
 	// the "v = <N>" again.
 	case OAS, OASWB:
 		if fmtmode == FExp && n.Right == nil {
@@ -1146,9 +1146,7 @@ func exprfmt(n *Node, prec int) string {
 		if n.Left != nil {
 			return fmt.Sprintf("[]%v", n.Left)
 		}
-		var f string
-		f += fmt.Sprintf("[]%v", n.Right)
-		return f // happens before typecheck
+		return fmt.Sprintf("[]%v", n.Right) // happens before typecheck
 
 	case OTMAP:
 		return fmt.Sprintf("map[%v]%v", n.Left, n.Right)

src/cmd/compile/internal/gc/inl.go

@@ -66,7 +66,7 @@ func typecheckinl(fn *Node) {
 		return // typecheckinl on local function
 	}
 
-	if Debug['m'] > 2 {
+	if Debug['m'] > 2 || Debug_export != 0 {
 		fmt.Printf("typecheck import [%v] %v { %v }\n", fn.Sym, Nconv(fn, FmtLong), Hconv(fn.Func.Inl, FmtSharp))
 	}
 

src/cmd/compile/internal/gc/main.go

@@ -815,6 +815,9 @@ func importfile(f *Val, indent []byte) {
 
 	case 'B':
 		// new export format
+		if Debug_export != 0 {
+			fmt.Printf("importing %s (%s)\n", path_, file)
+		}
 		imp.ReadByte() // skip \n after $$B
 		Import(imp)
 

src/cmd/compile/internal/gc/parser.go

@@ -1588,7 +1588,7 @@ func (p *parser) onew_name() *Node {
 func (p *parser) sym() *Sym {
 	switch p.tok {
 	case LNAME:
-		s := p.sym_
+		s := p.sym_ // from localpkg
 		p.next()
 		// during imports, unqualified non-exported identifiers are from builtinpkg
 		if importpkg != nil && !exportname(s.Name) {

src/cmd/compile/internal/gc/pgen.go

@@ -490,6 +490,12 @@ func compile(fn *Node) {
 	}
 }
 
+type symByName []*Sym
+
+func (a symByName) Len() int           { return len(a) }
+func (a symByName) Less(i, j int) bool { return a[i].Name < a[j].Name }
+func (a symByName) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+
 // genlegacy compiles Curfn using the legacy non-SSA code generator.
 func genlegacy(ptxt *obj.Prog, gcargs, gclocals *Sym) {
 	Genlist(Curfn.Func.Enter)

src/go/internal/gcimporter/bimport.go

@@ -15,6 +15,18 @@ import (
 	"unicode/utf8"
 )
 
+type importer struct {
+	imports  map[string]*types.Package
+	data     []byte
+	buf      []byte   // for reading strings
+	bufarray [64]byte // initial underlying array for buf, large enough to avoid allocation when compiling std lib
+	pkgList  []*types.Package
+	typList  []types.Type
+
+	debugFormat bool
+	read        int // bytes read
+}
+
 // BImportData imports a package from the serialized package data
 // and returns the number of bytes consumed and a reference to the package.
 // If data is obviously malformed, an error is returned but in
@@ -39,7 +51,7 @@ func BImportData(imports map[string]*types.Package, data []byte, path string) (i
 	// --- generic export data ---
 
 	if v := p.string(); v != "v0" {
-		return p.read, nil, fmt.Errorf("unknown version: %s", v)
+		return p.read, nil, fmt.Errorf("unknown export data version: %s", v)
 	}
 
 	// populate typList with predeclared "known" types
@@ -69,37 +81,20 @@ func BImportData(imports map[string]*types.Package, data []byte, path string) (i
 	// read compiler-specific flags
 	p.string() // discard
 
-	// read consts
-	for i := p.int(); i > 0; i-- {
-		name := p.string()
-		typ := p.typ(nil)
-		val := p.value()
-		p.declare(types.NewConst(token.NoPos, pkg, name, typ, val))
-	}
-
-	// read vars
-	for i := p.int(); i > 0; i-- {
-		name := p.string()
-		typ := p.typ(nil)
-		p.declare(types.NewVar(token.NoPos, pkg, name, typ))
-	}
-
-	// read funcs
-	for i := p.int(); i > 0; i-- {
-		name := p.string()
-		params, isddd := p.paramList()
-		result, _ := p.paramList()
-		sig := types.NewSignature(nil, params, result, isddd)
-		p.int() // read and discard index of inlined function body
-		p.declare(types.NewFunc(token.NoPos, pkg, name, sig))
+	// read objects of phase 1 only (see cmd/compiler/internal/gc/bexport.go)
+	objcount := 0
+	for {
+		tag := p.tagOrIndex()
+		if tag == endTag {
+			break
+		}
+		p.obj(tag)
+		objcount++
 	}
 
-	// read types
-	for i := p.int(); i > 0; i-- {
-		// name is parsed as part of named type and the
-		// type object is added to scope via respective
-		// named type
-		_ = p.typ(nil).(*types.Named)
+	// self-verification
+	if count := p.int(); count != objcount {
+		panic(fmt.Sprintf("importer: got %d objects; want %d", objcount, count))
 	}
 
 	// ignore compiler-specific import data
@@ -122,25 +117,6 @@ func BImportData(imports map[string]*types.Package, data []byte, path string) (i
 	return p.read, pkg, nil
 }
 
-type importer struct {
-	imports  map[string]*types.Package
-	data     []byte
-	buf      []byte   // for reading strings
-	bufarray [64]byte // initial underlying array for buf, large enough to avoid allocation when compiling std lib
-	pkgList  []*types.Package
-	typList  []types.Type
-
-	debugFormat bool
-	read        int // bytes read
-}
-
-func (p *importer) declare(obj types.Object) {
-	if alt := p.pkgList[0].Scope().Insert(obj); alt != nil {
-		// This can only happen if we import a package a second time.
-		panic(fmt.Sprintf("%s already declared", alt.Name()))
-	}
-}
-
 func (p *importer) pkg() *types.Package {
 	// if the package was seen before, i is its index (>= 0)
 	i := p.tagOrIndex()
@@ -178,6 +154,55 @@ func (p *importer) pkg() *types.Package {
 	return pkg
 }
 
+func (p *importer) declare(obj types.Object) {
+	pkg := obj.Pkg()
+	if alt := pkg.Scope().Insert(obj); alt != nil {
+		// This could only trigger if we import a (non-type) object a second time.
+		// This should never happen because 1) we only import a package once; and
+		// b) we ignore compiler-specific export data which may contain functions
+		// whose inlined function bodies refer to other functions that were already
+		// imported.
+		// (See also the comment in cmd/compile/internal/gc/bimport.go importer.obj,
+		// switch case importing functions).
+		panic(fmt.Sprintf("%s already declared", alt.Name()))
+	}
+}
+
+func (p *importer) obj(tag int) {
+	switch tag {
+	case constTag:
+		pkg, name := p.qualifiedName()
+		typ := p.typ(nil)
+		val := p.value()
+		p.declare(types.NewConst(token.NoPos, pkg, name, typ, val))
+
+	case typeTag:
+		_ = p.typ(nil)
+
+	case varTag:
+		pkg, name := p.qualifiedName()
+		typ := p.typ(nil)
+		p.declare(types.NewVar(token.NoPos, pkg, name, typ))
+
+	case funcTag:
+		pkg, name := p.qualifiedName()
+		params, isddd := p.paramList()
+		result, _ := p.paramList()
+		sig := types.NewSignature(nil, params, result, isddd)
+		p.int() // read and discard index of inlined function body
+		p.declare(types.NewFunc(token.NoPos, pkg, name, sig))
+
+	default:
+		panic("unexpected object tag")
+	}
+}
+
+func (p *importer) qualifiedName() (pkg *types.Package, name string) {
+	name = p.string()
+	pkg = p.pkg()
+	return
+}
+
 func (p *importer) record(t types.Type) {
 	p.typList = append(p.typList, t)
 }
@@ -239,11 +264,17 @@ func (p *importer) typ(parent *types.Package) types.Type {
 
 		// read associated methods
 		for i := p.int(); i > 0; i-- {
+			// TODO(gri) replace this with something closer to fieldName
 			name := p.string()
+			if !exported(name) {
+				p.pkg()
+			}
+
 			recv, _ := p.paramList() // TODO(gri) do we need a full param list for the receiver?
 			params, isddd := p.paramList()
 			result, _ := p.paramList()
 			p.int() // read and discard index of inlined function body
+
 			sig := types.NewSignature(recv.At(0), params, result, isddd)
 			t0.AddMethod(types.NewFunc(token.NoPos, parent, name, sig))
 		}
@@ -432,18 +463,20 @@ func (p *importer) param(named bool) (*types.Var, bool) {
 		t = types.NewSlice(td.elem)
 	}
 
+	var pkg *types.Package
 	var name string
 	if named {
 		name = p.string()
 		if name == "" {
 			panic("expected named parameter")
 		}
+		pkg = p.pkg()
 	}
 
 	// read and discard compiler-specific info
 	p.string()
 
-	return types.NewVar(token.NoPos, nil, name, t), isddd
+	return types.NewVar(token.NoPos, pkg, name, t), isddd
 }
 
 func exported(name string) bool {
@@ -617,8 +650,13 @@ func (p *importer) byte() byte {
 
 // Tags. Must be < 0.
 const (
-	// Packages
+	// Objects
 	packageTag = -(iota + 1)
+	constTag
+	typeTag
+	varTag
+	funcTag
+	endTag
 
 	// Types
 	namedTag
@@ -640,6 +678,7 @@ const (
 	fractionTag // not used by gc
 	complexTag
 	stringTag
+	unknownTag // not used by gc (only appears in packages with errors)
 )
 
 var predeclared = []types.Type{