cmd/compile: move spills to loop exits when easy.

For call-free inner loops.

Revised statistics:
  85 inner loop spills sunk
 341 inner loop spills remaining
1162 inner loop spills that were candidates for sinking
     ended up completely register allocated
 119 inner loop spills could have been sunk were used in
     "shuffling" at the bottom of the loop.
   1 inner loop spill not sunk because the register assigned
     changed between def and exit,

 Understanding how to make an inner loop definition not be
 a candidate for from-memory shuffling (to force the shuffle
 code to choose some other value) should pick up some of the
 119 other spills disqualified for this reason.

 Modified the stats printing based on feedback from Austin.

Change-Id: If3fb9b5d5a028f42ccc36c4e3d9e0da39db5ca60
Reviewed-on: https://go-review.googlesource.com/21037Reviewed-by: Keith Randall <khr@golang.org>
Run-TryBot: David Chase <drchase@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>

src/cmd/compile/internal/ssa/likelyadjust.go

@@ -11,11 +11,24 @@ import (
 type loop struct {
 	header *Block // The header node of this (reducible) loop
 	outer  *loop  // loop containing this loop
-	// Next two fields not currently used, but cheap to maintain,
-	// and aid in computation of inner-ness and list of blocks.
-	nBlocks      int32 // Number of blocks in this loop but not within inner loops
-	isInner      bool  // True if never discovered to contain a loop
-	containsCall bool  // if any block in this loop or any loop it contains is a BlockCall or BlockDefer
+
+	// By default, children exits, and depth are not initialized.
+	children []*loop  // loops nested directly within this loop. Initialized by assembleChildren().
+	exits    []*Block // exits records blocks reached by exits from this loop. Initialized by findExits().
+
+	// Loops aren't that common, so rather than force regalloc to keep
+	// a map or slice for its data, just put it here.
+	spills  []*Value
+	scratch int32
+
+	// Next three fields used by regalloc and/or
+	// aid in computation of inner-ness and list of blocks.
+	nBlocks int32 // Number of blocks in this loop but not within inner loops
+	depth   int16 // Nesting depth of the loop; 1 is outermost. Initialized by calculateDepths().
+	isInner bool  // True if never discovered to contain a loop
+
+	// register allocation uses this.
+	containsCall bool // if any block in this loop or any loop it contains is a BlockCall or BlockDefer
 }
 
 // outerinner records that outer contains inner
@@ -48,6 +61,9 @@ type loopnest struct {
 	po    []*Block
 	sdom  sparseTree
 	loops []*loop
+
+	// Record which of the lazily initialized fields have actually been initialized.
+	initializedChildren, initializedDepth, initializedExits bool
 }
 
 func min8(a, b int8) int8 {
@@ -295,6 +311,35 @@ func loopnestfor(f *Func) *loopnest {
 			innermost.nBlocks++
 		}
 	}
+
+	ln := &loopnest{f: f, b2l: b2l, po: po, sdom: sdom, loops: loops}
+
+	// Curious about the loopiness? "-d=ssa/likelyadjust/stats"
+	if f.pass.stats > 0 && len(loops) > 0 {
+		ln.assembleChildren()
+		ln.calculateDepths()
+		ln.findExits()
+
+		// Note stats for non-innermost loops are slightly flawed because
+		// they don't account for inner loop exits that span multiple levels.
+
+		for _, l := range loops {
+			x := len(l.exits)
+			cf := 0
+			if !l.containsCall {
+				cf = 1
+			}
+			inner := 0
+			if l.isInner {
+				inner++
+			}
+
+			f.logStat("loopstats:",
+				l.depth, "depth", x, "exits",
+				inner, "is_inner", cf, "is_callfree", l.nBlocks, "n_blocks")
+		}
+	}
+
 	if f.pass.debug > 1 && len(loops) > 0 {
 		fmt.Printf("Loops in %s:\n", f.Name)
 		for _, l := range loops {
@@ -314,5 +359,90 @@ func loopnestfor(f *Func) *loopnest {
 		}
 		fmt.Print("\n")
 	}
-	return &loopnest{f, b2l, po, sdom, loops}
+	return ln
+}
+
+// assembleChildren initializes the children field of each
+// loop in the nest.  Loop A is a child of loop B if A is
+// directly nested within B (based on the reducible-loops
+// detection above)
+func (ln *loopnest) assembleChildren() {
+	if ln.initializedChildren {
+		return
+	}
+	for _, l := range ln.loops {
+		if l.outer != nil {
+			l.outer.children = append(l.outer.children, l)
+		}
+	}
+	ln.initializedChildren = true
+}
+
+// calculateDepths uses the children field of loops
+// to determine the nesting depth (outer=1) of each
+// loop.  This is helpful for finding exit edges.
+func (ln *loopnest) calculateDepths() {
+	if ln.initializedDepth {
+		return
+	}
+	ln.assembleChildren()
+	for _, l := range ln.loops {
+		if l.outer == nil {
+			l.setDepth(1)
+		}
+	}
+	ln.initializedDepth = true
+}
+
+// findExits uses loop depth information to find the
+// exits from a loop.
+func (ln *loopnest) findExits() {
+	if ln.initializedExits {
+		return
+	}
+	ln.calculateDepths()
+	b2l := ln.b2l
+	for _, b := range ln.po {
+		l := b2l[b.ID]
+		if l != nil && len(b.Succs) == 2 {
+			sl := b2l[b.Succs[0].ID]
+			if recordIfExit(l, sl, b.Succs[0]) {
+				continue
+			}
+			sl = b2l[b.Succs[1].ID]
+			if recordIfExit(l, sl, b.Succs[1]) {
+				continue
+			}
+		}
+	}
+	ln.initializedExits = true
+}
+
+// recordIfExit checks sl (the loop containing b) to see if it
+// is outside of loop l, and if so, records b as an exit block
+// from l and returns true.
+func recordIfExit(l, sl *loop, b *Block) bool {
+	if sl != l {
+		if sl == nil || sl.depth <= l.depth {
+			l.exits = append(l.exits, b)
+			return true
+		}
+		// sl is not nil, and is deeper than l
+		// it's possible for this to be a goto into an irreducible loop made from gotos.
+		for sl.depth > l.depth {
+			sl = sl.outer
+		}
+		if sl != l {
+			l.exits = append(l.exits, b)
+			return true
+		}
+	}
+	return false
+}
+
+func (l *loop) setDepth(d int16) {
+	l.depth = d
+	for _, c := range l.children {
+		c.setDepth(d + 1)
+	}
 }

src/cmd/compile/internal/ssa/sparsemap.go

@@ -32,6 +32,8 @@ func (s *sparseMap) contains(k ID) bool {
 	return i < len(s.dense) && s.dense[i].key == k
 }
 
+// get returns the value for key k, or -1 if k does
+// not appear in the map.
 func (s *sparseMap) get(k ID) int32 {
 	i := s.sparse[k]
 	if i < len(s.dense) && s.dense[i].key == k {
@@ -50,6 +52,20 @@ func (s *sparseMap) set(k ID, v int32) {
 	s.sparse[k] = len(s.dense) - 1
 }
 
+// setBit sets the v'th bit of k's value, where 0 <= v < 32
+func (s *sparseMap) setBit(k ID, v uint) {
+	if v >= 32 {
+		panic("bit index too large.")
+	}
+	i := s.sparse[k]
+	if i < len(s.dense) && s.dense[i].key == k {
+		s.dense[i].val |= 1 << v
+		return
+	}
+	s.dense = append(s.dense, sparseEntry{k, 1 << v})
+	s.sparse[k] = len(s.dense) - 1
+}
+
 func (s *sparseMap) remove(k ID) {
 	i := s.sparse[k]
 	if i < len(s.dense) && s.dense[i].key == k {