reflect: add ArrayOf, ChanOf, MapOf, SliceOf

In order to add these, we need to be able to find references
to such types that already exist in the binary. To do that, introduce
a new linker section holding a list of the types corresponding to
arrays, chans, maps, and slices.

To offset the storage cost of this list, and to simplify the code,
remove the interface{} header from the representation of a
runtime type. It was used in early versions of the code but was
made obsolete by the kind field: a switch on kind is more efficient
than a type switch.

In the godoc binary, removing the interface{} header cuts two
words from each of about 10,000 types. Adding back the list of pointers
to array, chan, map, and slice types reintroduces one word for
each of about 500 types. On a 64-bit machine, then, this CL *removes*
a net 156 kB of read-only data from the binary.

This CL does not include the needed support for precise garbage
collection. I have created issue 4375 to track that.

This CL also does not set the 'algorithm' - specifically the equality
and copy functions - for a new array correctly, so I have unexported
ArrayOf for now. That is also part of issue 4375.

Fixes #2339.

R=r, remyoudompheng, mirtchovski, iant
CC=golang-dev
https://golang.org/cl/6572043

src/cmd/gc/go.h

@@ -852,7 +852,8 @@ EXTERN	Pkg*	itabpkg;	// fake pkg for itab cache
 EXTERN	Pkg*	runtimepkg;	// package runtime
 EXTERN	Pkg*	racepkg;	// package runtime/race
 EXTERN	Pkg*	stringpkg;	// fake package for C strings
-EXTERN	Pkg*	typepkg;	// fake package for runtime type info
+EXTERN	Pkg*	typepkg;	// fake package for runtime type info (headers)
+EXTERN	Pkg*	typelinkpkg;	// fake package for runtime type info (data)
 EXTERN	Pkg*	weaktypepkg;	// weak references to runtime type info
 EXTERN	Pkg*	unsafepkg;	// package unsafe
 EXTERN	Pkg*	trackpkg;	// fake package for field tracking

src/cmd/gc/lex.c

@@ -224,6 +224,10 @@ main(int argc, char *argv[])
 	weaktypepkg = mkpkg(strlit("go.weak.type"));
 	weaktypepkg->name = "go.weak.type";
 	weaktypepkg->prefix = "go.weak.type";  // not go%2eweak%2etype
+	
+	typelinkpkg = mkpkg(strlit("go.typelink"));
+	typelinkpkg->name = "go.typelink";
+	typelinkpkg->prefix = "go.typelink"; // not go%2etypelink
 
 	trackpkg = mkpkg(strlit("go.track"));
 	trackpkg->name = "go.track";

src/cmd/gc/reflect.c

@@ -467,20 +467,6 @@ kinds[] =
 	[TUNSAFEPTR]	= KindUnsafePointer,
 };
 
-static Sym*
-typestruct(Type *t)
-{
-	// We use a weak reference to the reflect type
-	// to avoid requiring package reflect in every binary.
-	// If package reflect is available, the interface{} holding
-	// a runtime type will contain a *reflect.commonType.
-	// Otherwise it will use a nil type word but still be usable
-	// by package runtime (because we always use the memory
-	// after the interface value, not the interface value itself).
-	USED(t);
-	return pkglookup("*reflect.commonType", weaktypepkg);
-}
-
 int
 haspointers(Type *t)
 {
@@ -535,6 +521,9 @@ dcommontype(Sym *s, int ot, Type *t)
 	Sym *sptr, *algsym;
 	static Sym *algarray;
 	char *p;
+	
+	if(ot != 0)
+		fatal("dcommontype %d", ot);
 
 	sizeofAlg = 4*widthptr;
 	if(algarray == nil)
@@ -550,13 +539,6 @@ dcommontype(Sym *s, int ot, Type *t)
 	else
 		sptr = weaktypesym(ptrto(t));
 
-	// empty interface pointing at this type.
-	// all the references that we emit are *interface{};
-	// they point here.
-	ot = rnd(ot, widthptr);
-	ot = dsymptr(s, ot, typestruct(t), 0);
-	ot = dsymptr(s, ot, s, 2*widthptr);
-
 	// ../../pkg/reflect/type.go:/^type.commonType
 	// actual type structure
 	//	type commonType struct {
@@ -636,6 +618,27 @@ tracksym(Type *t)
 	return s;
 }
 
+Sym*
+typelinksym(Type *t)
+{
+	char *p;
+	Sym *s;
+
+	// %-uT is what the generated Type's string field says.
+	// It uses (ambiguous) package names instead of import paths.
+	// %-T is the complete, unambiguous type name.
+	// We want the types to end up sorted by string field,
+	// so use that first in the name, and then add :%-T to
+	// disambiguate. The names are a little long but they are
+	// discarded by the linker and do not end up in the symbol
+	// table of the final binary.
+	p = smprint("%-uT/%-T", t, t);
+	s = pkglookup(p, typelinkpkg);
+	//print("typelinksym: %s -> %+S\n", p, s);
+	free(p);
+	return s;
+}
+
 Sym*
 typesymprefix(char *prefix, Type *t)
 {
@@ -697,7 +700,7 @@ static Sym*
 dtypesym(Type *t)
 {
 	int ot, xt, n, isddd, dupok;
-	Sym *s, *s1, *s2;
+	Sym *s, *s1, *s2, *slink;
 	Sig *a, *m;
 	Type *t1, *tbase, *t2;
 
@@ -893,6 +896,23 @@ ok:
 	}
 	ot = dextratype(s, ot, t, xt);
 	ggloblsym(s, ot, dupok, 1);
+
+	// generate typelink.foo pointing at s = type.foo.
+	// The linker will leave a table of all the typelinks for
+	// types in the binary, so reflect can find them.
+	// We only need the link for unnamed composites that
+	// we want be able to find.
+	if(t->sym == S) {
+		switch(t->etype) {
+		case TARRAY:
+		case TCHAN:
+		case TMAP:
+			slink = typelinksym(t);
+			dsymptr(slink, 0, s, 0);
+			ggloblsym(slink, widthptr, dupok, 1);
+		}
+	}
+
 	return s;
 }
 

src/cmd/ld/data.c

@@ -1079,7 +1079,7 @@ dodata(void)
 	sect->vaddr = 0;
 	datsize = 0;
 	s = datap;
-	for(; s != nil && s->type < SGCDATA; s = s->next) {
+	for(; s != nil && s->type < STYPELINK; s = s->next) {
 		if(s->align != 0)
 			datsize = rnd(datsize, s->align);
 		s->type = SRODATA;
@@ -1089,6 +1089,17 @@ dodata(void)
 	sect->len = datsize - sect->vaddr;
 	datsize = rnd(datsize, PtrSize);
 
+	/* type */
+	sect = addsection(&segtext, ".typelink", 04);
+	sect->vaddr = datsize;
+	for(; s != nil && s->type == STYPELINK; s = s->next) {
+		s->type = SRODATA;
+		s->value = datsize;
+		datsize += s->size;
+	}
+	sect->len = datsize - sect->vaddr;
+	datsize = rnd(datsize, PtrSize);
+	
 	/* gcdata */
 	sect = addsection(&segtext, ".gcdata", 04);
 	sect->vaddr = datsize;
@@ -1194,7 +1205,7 @@ void
 address(void)
 {
 	Section *s, *text, *data, *rodata, *symtab, *pclntab, *noptr, *bss, *noptrbss;
-	Section *gcdata, *gcbss;
+	Section *gcdata, *gcbss, *typelink;
 	Sym *sym, *sub;
 	uvlong va;
 
@@ -1241,7 +1252,8 @@ address(void)
 
 	text = segtext.sect;
 	rodata = text->next;
-	gcdata = rodata->next;
+	typelink = rodata->next;
+	gcdata = typelink->next;
 	gcbss = gcdata->next;
 	symtab = gcbss->next;
 	pclntab = symtab->next;
@@ -1260,6 +1272,8 @@ address(void)
 	xdefine("etext", STEXT, text->vaddr + text->len);
 	xdefine("rodata", SRODATA, rodata->vaddr);
 	xdefine("erodata", SRODATA, rodata->vaddr + rodata->len);
+	xdefine("typelink", SRODATA, typelink->vaddr);
+	xdefine("etypelink", SRODATA, typelink->vaddr + typelink->len);
 	xdefine("gcdata", SGCDATA, gcdata->vaddr);
 	xdefine("egcdata", SGCDATA, gcdata->vaddr + gcdata->len);
 	xdefine("gcbss", SGCBSS, gcbss->vaddr);

src/cmd/ld/decodesym.c

@@ -71,21 +71,21 @@ decode_inuxi(uchar* p, int sz)
 uint8
 decodetype_kind(Sym *s)
 {
-	return s->p[3*PtrSize + 7] & ~KindNoPointers;	//  0x13 / 0x1f
+	return s->p[1*PtrSize + 7] & ~KindNoPointers;	//  0x13 / 0x1f
 }
 
 // Type.commonType.size
 vlong
 decodetype_size(Sym *s)
 {
-	return decode_inuxi(s->p + 2*PtrSize, PtrSize);	 // 0x8 / 0x10
+	return decode_inuxi(s->p, PtrSize);	 // 0x8 / 0x10
 }
 
 // Type.commonType.gc
 Sym*
 decodetype_gc(Sym *s)
 {
-	return decode_reloc_sym(s, 3*PtrSize + 8 + 1*PtrSize);
+	return decode_reloc_sym(s, 1*PtrSize + 8 + 1*PtrSize);
 }
 
 // Type.ArrayType.elem and Type.SliceType.Elem

src/cmd/ld/dwarf.c

@@ -2082,7 +2082,7 @@ dwarfemitdebugsections(void)
 	newattr(die, DW_AT_byte_size, DW_CLS_CONSTANT, PtrSize, 0);
 
 	// Needed by the prettyprinter code for interface inspection.
-	defgotype(lookup_or_diag("type.runtime.commonType"));
+	defgotype(lookup_or_diag("type.runtime.rtype"));
 	defgotype(lookup_or_diag("type.runtime.interfaceType"));
 	defgotype(lookup_or_diag("type.runtime.itab"));
 

src/cmd/ld/go.c

@@ -740,6 +740,12 @@ deadcode(void)
 
 	markflood();
 	
+	// keep each beginning with 'typelink.' if the symbol it points at is being kept.
+	for(s = allsym; s != S; s = s->allsym) {
+		if(strncmp(s->name, "go.typelink.", 12) == 0)
+			s->reachable = s->nr==1 && s->r[0].sym->reachable;
+	}
+
 	// remove dead text but keep file information (z symbols).
 	last = nil;
 	z = nil;
@@ -771,7 +777,7 @@ deadcode(void)
 			s->reachable = 1;
 			s->hide = 1;
 		}
-
+	
 	// record field tracking references
 	fmtstrinit(&fmt);
 	for(s = allsym; s != S; s = s->allsym) {

src/cmd/ld/lib.h

@@ -39,6 +39,7 @@ enum
 	SSTRING,
 	SGOSTRING,
 	SRODATA,
+	STYPELINK,
 	SGCDATA,
 	SGCBSS,
 	SSYMTAB,

src/cmd/ld/symtab.c

@@ -337,6 +337,8 @@ symtab(void)
 	// data.c:/^address will provide the actual values.
 	xdefine("text", STEXT, 0);
 	xdefine("etext", STEXT, 0);
+	xdefine("typelink", SRODATA, 0);
+	xdefine("etypelink", SRODATA, 0);
 	xdefine("rodata", SRODATA, 0);
 	xdefine("erodata", SRODATA, 0);
 	xdefine("gcdata", SGCDATA, 0);
@@ -382,6 +384,10 @@ symtab(void)
 			s->type = STYPE;
 			s->hide = 1;
 		}
+		if(strncmp(s->name, "go.typelink.", 12) == 0) {
+			s->type = STYPELINK;
+			s->hide = 1;
+		}
 		if(strncmp(s->name, "go.string.", 10) == 0) {
 			s->type = SGOSTRING;
 			s->hide = 1;

src/pkg/reflect/all_test.go

@@ -2703,6 +2703,88 @@ func TestOverflow(t *testing.T) {
 	}
 }
 
+func checkSameType(t *testing.T, x, y interface{}) {
+	if TypeOf(x) != TypeOf(y) {
+		t.Errorf("did not find preexisting type for %s (vs %s)", TypeOf(x), TypeOf(y))
+	}
+}
+
+func TestArrayOf(t *testing.T) {
+	// check construction and use of type not in binary
+	type T int
+	at := ArrayOf(10, TypeOf(T(1)))
+	v := New(at).Elem()
+	for i := 0; i < v.Len(); i++ {
+		v.Index(i).Set(ValueOf(T(i)))
+	}
+	s := fmt.Sprint(v.Interface())
+	want := "[0 1 2 3 4 5 6 7 8 9]"
+	if s != want {
+		t.Errorf("constructed array = %s, want %s", s, want)
+	}
+
+	// check that type already in binary is found
+	checkSameType(t, Zero(ArrayOf(5, TypeOf(T(1)))).Interface(), [5]T{})
+}
+
+func TestSliceOf(t *testing.T) {
+	// check construction and use of type not in binary
+	type T int
+	st := SliceOf(TypeOf(T(1)))
+	v := MakeSlice(st, 10, 10)
+	for i := 0; i < v.Len(); i++ {
+		v.Index(i).Set(ValueOf(T(i)))
+	}
+	s := fmt.Sprint(v.Interface())
+	want := "[0 1 2 3 4 5 6 7 8 9]"
+	if s != want {
+		t.Errorf("constructed slice = %s, want %s", s, want)
+	}
+
+	// check that type already in binary is found
+	type T1 int
+	checkSameType(t, Zero(SliceOf(TypeOf(T1(1)))).Interface(), []T1{})
+}
+
+func TestChanOf(t *testing.T) {
+	// check construction and use of type not in binary
+	type T string
+	ct := ChanOf(BothDir, TypeOf(T("")))
+	v := MakeChan(ct, 2)
+	v.Send(ValueOf(T("hello")))
+	v.Send(ValueOf(T("world")))
+
+	sv1, _ := v.Recv()
+	sv2, _ := v.Recv()
+	s1 := sv1.String()
+	s2 := sv2.String()
+	if s1 != "hello" || s2 != "world" {
+		t.Errorf("constructed chan: have %q, %q, want %q, %q", s1, s2, "hello", "world")
+	}
+
+	// check that type already in binary is found
+	type T1 int
+	checkSameType(t, Zero(ChanOf(BothDir, TypeOf(T1(1)))).Interface(), (chan T1)(nil))
+}
+
+func TestMapOf(t *testing.T) {
+	// check construction and use of type not in binary
+	type K string
+	type V float64
+
+	v := MakeMap(MapOf(TypeOf(K("")), TypeOf(V(0))))
+	v.SetMapIndex(ValueOf(K("a")), ValueOf(V(1)))
+
+	s := fmt.Sprint(v.Interface())
+	want := "map[a:1]"
+	if s != want {
+		t.Errorf("constructed map = %s, want %s", s, want)
+	}
+
+	// check that type already in binary is found
+	checkSameType(t, Zero(MapOf(TypeOf(V(0)), TypeOf(K("")))).Interface(), map[V]K(nil))
+}
+
 type B1 struct {
 	X int
 	Y int

src/pkg/reflect/export_test.go

@@ -14,3 +14,5 @@ func MakeRO(v Value) Value {
 func IsRO(v Value) bool {
 	return v.flag&flagRO != 0
 }
+
+var ArrayOf = arrayOf

src/pkg/reflect/makefunc.go

@@ -17,7 +17,7 @@ type makeFuncImpl struct {
 	// References visible to the garbage collector.
 	// The code array below contains the same references
 	// embedded in the machine code.
-	typ *commonType
+	typ *rtype
 	fn  func([]Value) []Value
 
 	// code is the actual machine code invoked for the closure.

src/pkg/runtime/iface.c

@@ -686,17 +686,10 @@ reflect·unsafe_Typeof(Eface e, Eface ret)
 }
 
 void
-reflect·unsafe_New(Eface typ, void *ret)
+reflect·unsafe_New(Type *t, void *ret)
 {
-	Type *t;
 	uint32 flag;
 
-	// Reflect library has reinterpreted typ
-	// as its own kind of type structure.
-	// We know that the pointer to the original
-	// type structure sits before the data pointer.
-	t = (Type*)((Eface*)typ.data-1);
-
 	flag = t->kind&KindNoPointers ? FlagNoPointers : 0;
 	ret = runtime·mallocgc(t->size, flag, 1, 1);
 
@@ -711,16 +704,9 @@ reflect·unsafe_New(Eface typ, void *ret)
 }
 
 void
-reflect·unsafe_NewArray(Eface typ, intgo n, void *ret)
+reflect·unsafe_NewArray(Type *t, intgo n, void *ret)
 {
 	uint64 size;
-	Type *t;
-
-	// Reflect library has reinterpreted typ
-	// as its own kind of type structure.
-	// We know that the pointer to the original
-	// type structure sits before the data pointer.
-	t = (Type*)((Eface*)typ.data-1);
 
 	size = n*t->size;
 	if(size == 0)
@@ -740,3 +726,14 @@ reflect·unsafe_NewArray(Eface typ, intgo n, void *ret)
 
 	FLUSH(&ret);
 }
+
+void
+reflect·typelinks(Slice ret)
+{
+	extern Type *typelink[], *etypelink[];
+	static int32 first = 1;
+	ret.array = (byte*)typelink;
+	ret.len = etypelink - typelink;
+	ret.cap = ret.len;
+	FLUSH(&ret);
+}

src/pkg/runtime/runtime-gdb.py

@@ -149,8 +149,8 @@ goobjfile.pretty_printers.extend([makematcher(k) for k in vars().values() if has
 
 #
 #  For reference, this is what we're trying to do:
-#  eface: p *(*(struct 'runtime.commonType'*)'main.e'->type_->data)->string
-#  iface: p *(*(struct 'runtime.commonType'*)'main.s'->tab->Type->data)->string
+#  eface: p *(*(struct 'runtime.rtype'*)'main.e'->type_->data)->string
+#  iface: p *(*(struct 'runtime.rtype'*)'main.s'->tab->Type->data)->string
 #
 # interface types can't be recognized by their name, instead we check
 # if they have the expected fields.  Unfortunately the mapping of
@@ -186,8 +186,7 @@ def lookup_type(name):
 	except:
 		pass
 
-_rctp_type = gdb.lookup_type("struct runtime.commonType").pointer()
-_rtp_type = gdb.lookup_type("struct runtime._type").pointer()
+_rctp_type = gdb.lookup_type("struct runtime.rtype").pointer()
 
 def iface_commontype(obj):
 	if is_iface(obj):
@@ -196,18 +195,13 @@ def iface_commontype(obj):
 		go_type_ptr = obj['_type']
 	else:
 		return
-
-	# sanity check: reflection type description ends in a loop.
-	tt = go_type_ptr['_type'].cast(_rtp_type).dereference()['_type']
-	if tt != tt.cast(_rtp_type).dereference()['_type']:
-		return
 	
-	return go_type_ptr['ptr'].cast(_rctp_type).dereference()
+	return go_type_ptr.cast(_rctp_type).dereference()
 	
 
 def iface_dtype(obj):
 	"Decode type of the data field of an eface or iface struct."
-	# known issue: dtype_name decoded from runtime.commonType is "nested.Foo"
+	# known issue: dtype_name decoded from runtime.rtype is "nested.Foo"
 	# but the dwarf table lists it as "full/path/to/nested.Foo"
 
 	dynamic_go_type = iface_commontype(obj)

src/pkg/runtime/type.go

@@ -14,7 +14,7 @@ package runtime
 
 import "unsafe"
 
-type commonType struct {
+type rtype struct {
 	size       uintptr
 	hash       uint32
 	_          uint8
@@ -25,14 +25,14 @@ type commonType struct {
 	gc         unsafe.Pointer
 	string     *string
 	*uncommonType
-	ptrToThis *interface{}
+	ptrToThis *rtype
 }
 
 type _method struct {
 	name    *string
 	pkgPath *string
-	mtyp    *interface{}
-	typ     *interface{}
+	mtyp    *rtype
+	typ     *rtype
 	ifn     unsafe.Pointer
 	tfn     unsafe.Pointer
 }
@@ -46,10 +46,10 @@ type uncommonType struct {
 type _imethod struct {
 	name    *string
 	pkgPath *string
-	typ     *interface{}
+	typ     *rtype
 }
 
 type interfaceType struct {
-	commonType
+	rtype
 	methods []_imethod
 }

src/pkg/runtime/type.h

@@ -5,13 +5,10 @@
 /*
  * Runtime type representation; master is type.go
  *
- * The *Types here correspond 1-1 to type.go's *Type's, but are
- * prefixed with an extra header of 2 pointers, corresponding to the
- * interface{} structure, which itself is called type Type again on
- * the Go side.
+ * The Type*s here correspond 1-1 to type.go's *rtype.
  */
 
-typedef struct CommonType CommonType;
+typedef struct Type Type;
 typedef struct UncommonType UncommonType;
 typedef struct InterfaceType InterfaceType;
 typedef struct Method Method;
@@ -21,7 +18,7 @@ typedef struct FuncType FuncType;
 typedef struct PtrType PtrType;
 
 // Needs to be in sync with typekind.h/CommonSize
-struct CommonType
+struct Type
 {
 	uintptr size;
 	uint32 hash;
@@ -54,13 +51,6 @@ struct UncommonType
 	Method m[];
 };
 
-struct Type
-{
-	void *type;	// interface{} value
-	void *ptr;
-	CommonType;
-};
-
 struct IMethod
 {
 	String *name;

src/pkg/runtime/typekind.h

@@ -35,7 +35,7 @@ enum {
 
 	KindNoPointers = 1<<7,
 
-	// size of Type interface header + CommonType structure.
-	CommonSize = 2*PtrSize + 6*PtrSize + 8,
+	// size of Type structure.
+	CommonSize = 6*PtrSize + 8,
 };