Skip to content

G
go
Commit ab48574b authored by David Chase's avatar David Chase
Browse files
Options

cmd/compile: use Block.Likely to put likely branch first 

When a neither of a conditional block's successors follows,
the block must end with a conditional branch followed by a
an unconditional branch.  If the (conditional) branch is
"unlikely", invert it and swap successors to make it
likely instead.

This doesn't matter to most benchmarks on amd64, but in one
instance on amd64 it caused a 30% improvement, and it is
otherwise harmless.  The problematic loop is

		for i := 0; i < w; i++ {
			if pw[i] != 0 {
				return true
			}
		}

compiled under GOEXPERIMENT=preemptibleloops
This the very worst-case benchmark for that experiment.

Also in this CL is a commoning up of heavily-repeated
boilerplate, which made it much easier to see that the
changes were applied correctly.  In the future this should
allow un-exporting of SSAGenState.Branches once the
boilerplate-replacement is done everywhere.

Change-Id: I0e5ded6eeb3ab1e3e0138e12d54c7e056bd99335
Reviewed-on: https://go-review.googlesource.com/104977
Run-TryBot: David Chase <drchase@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: default avatarCherry Zhang <cherryyz@google.com>
parent 2b239969
Hide whitespace changes
Inline Side-by-side
Showing with 102 additions and 134 deletions
src/cmd/compile/internal/amd64/ssa.go
View file @ ab48574b
...@@ -1201,23 +1201,19 @@ func ssaGenBlock(s *gc.SSAGenState, b, next *ssa.Block) { ...@@ -1201,23 +1201,19 @@ func ssaGenBlock(s *gc.SSAGenState, b, next *ssa.Block) {
ssa.BlockAMD64ULT, ssa.BlockAMD64UGT, ssa.BlockAMD64ULT, ssa.BlockAMD64UGT,
ssa.BlockAMD64ULE, ssa.BlockAMD64UGE: ssa.BlockAMD64ULE, ssa.BlockAMD64UGE:
jmp := blockJump[b.Kind] jmp := blockJump[b.Kind]
var p *obj.Prog
switch next { switch next {
case b.Succs[0].Block(): case b.Succs[0].Block():
p = s.Prog(jmp.invasm) s.Br(jmp.invasm, b.Succs[1].Block())
p.To.Type = obj.TYPE_BRANCH
s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[1].Block()})
case b.Succs[1].Block(): case b.Succs[1].Block():
p = s.Prog(jmp.asm) s.Br(jmp.asm, b.Succs[0].Block())
p.To.Type = obj.TYPE_BRANCH
s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()})
default: default:
p = s.Prog(jmp.asm) if b.Likely != ssa.BranchUnlikely {
p.To.Type = obj.TYPE_BRANCH s.Br(jmp.asm, b.Succs[0].Block())
s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()}) s.Br(obj.AJMP, b.Succs[1].Block())
q := s.Prog(obj.AJMP) } else {
q.To.Type = obj.TYPE_BRANCH s.Br(jmp.invasm, b.Succs[1].Block())
s.Branches = append(s.Branches, gc.Branch{P: q, B: b.Succs[1].Block()}) s.Br(obj.AJMP, b.Succs[0].Block())
}
} }
default: default:
......
src/cmd/compile/internal/arm/ssa.go
View file @ ab48574b
...@@ -884,23 +884,19 @@ func ssaGenBlock(s *gc.SSAGenState, b, next *ssa.Block) { ...@@ -884,23 +884,19 @@ func ssaGenBlock(s *gc.SSAGenState, b, next *ssa.Block) {
ssa.BlockARMULT, ssa.BlockARMUGT, ssa.BlockARMULT, ssa.BlockARMUGT,
ssa.BlockARMULE, ssa.BlockARMUGE: ssa.BlockARMULE, ssa.BlockARMUGE:
jmp := blockJump[b.Kind] jmp := blockJump[b.Kind]
var p *obj.Prog
switch next { switch next {
case b.Succs[0].Block(): case b.Succs[0].Block():
p = s.Prog(jmp.invasm) s.Br(jmp.invasm, b.Succs[1].Block())
p.To.Type = obj.TYPE_BRANCH
s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[1].Block()})
case b.Succs[1].Block(): case b.Succs[1].Block():
p = s.Prog(jmp.asm) s.Br(jmp.asm, b.Succs[0].Block())
p.To.Type = obj.TYPE_BRANCH
s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()})
default: default:
p = s.Prog(jmp.asm) if b.Likely != ssa.BranchUnlikely {
p.To.Type = obj.TYPE_BRANCH s.Br(jmp.asm, b.Succs[0].Block())
s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()}) s.Br(obj.AJMP, b.Succs[1].Block())
q := s.Prog(obj.AJMP) } else {
q.To.Type = obj.TYPE_BRANCH s.Br(jmp.invasm, b.Succs[1].Block())
s.Branches = append(s.Branches, gc.Branch{P: q, B: b.Succs[1].Block()}) s.Br(obj.AJMP, b.Succs[0].Block())
}
} }
default: default:
......
src/cmd/compile/internal/arm64/ssa.go
View file @ ab48574b
...@@ -874,20 +874,17 @@ func ssaGenBlock(s *gc.SSAGenState, b, next *ssa.Block) { ...@@ -874,20 +874,17 @@ func ssaGenBlock(s *gc.SSAGenState, b, next *ssa.Block) {
var p *obj.Prog var p *obj.Prog
switch next { switch next {
case b.Succs[0].Block(): case b.Succs[0].Block():
p = s.Prog(jmp.invasm) p = s.Br(jmp.invasm, b.Succs[1].Block())
p.To.Type = obj.TYPE_BRANCH
s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[1].Block()})
case b.Succs[1].Block(): case b.Succs[1].Block():
p = s.Prog(jmp.asm) p = s.Br(jmp.asm, b.Succs[0].Block())
p.To.Type = obj.TYPE_BRANCH
s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()})
default: default:
p = s.Prog(jmp.asm) if b.Likely != ssa.BranchUnlikely {
p.To.Type = obj.TYPE_BRANCH p = s.Br(jmp.asm, b.Succs[0].Block())
s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()}) s.Br(obj.AJMP, b.Succs[1].Block())
q := s.Prog(obj.AJMP) } else {
q.To.Type = obj.TYPE_BRANCH p = s.Br(jmp.invasm, b.Succs[1].Block())
s.Branches = append(s.Branches, gc.Branch{P: q, B: b.Succs[1].Block()}) s.Br(obj.AJMP, b.Succs[0].Block())
}
} }
if !b.Control.Type.IsFlags() { if !b.Control.Type.IsFlags() {
p.From.Type = obj.TYPE_REG p.From.Type = obj.TYPE_REG
...@@ -898,30 +895,21 @@ func ssaGenBlock(s *gc.SSAGenState, b, next *ssa.Block) { ...@@ -898,30 +895,21 @@ func ssaGenBlock(s *gc.SSAGenState, b, next *ssa.Block) {
var p *obj.Prog var p *obj.Prog
switch next { switch next {
case b.Succs[0].Block(): case b.Succs[0].Block():
p = s.Prog(jmp.invasm) p = s.Br(jmp.invasm, b.Succs[1].Block())
p.To.Type = obj.TYPE_BRANCH
p.From.Offset = b.Aux.(int64)
p.From.Type = obj.TYPE_CONST
p.Reg = b.Control.Reg()
s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[1].Block()})
case b.Succs[1].Block(): case b.Succs[1].Block():
p = s.Prog(jmp.asm) p = s.Br(jmp.asm, b.Succs[0].Block())
p.To.Type = obj.TYPE_BRANCH
p.From.Offset = b.Aux.(int64)
p.From.Type = obj.TYPE_CONST
p.Reg = b.Control.Reg()
s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()})
default: default:
p = s.Prog(jmp.asm) if b.Likely != ssa.BranchUnlikely {
p.To.Type = obj.TYPE_BRANCH p = s.Br(jmp.asm, b.Succs[0].Block())
p.From.Offset = b.Aux.(int64) s.Br(obj.AJMP, b.Succs[1].Block())
p.From.Type = obj.TYPE_CONST } else {
p.Reg = b.Control.Reg() p = s.Br(jmp.invasm, b.Succs[1].Block())
s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()}) s.Br(obj.AJMP, b.Succs[0].Block())
q := s.Prog(obj.AJMP) }
q.To.Type = obj.TYPE_BRANCH
s.Branches = append(s.Branches, gc.Branch{P: q, B: b.Succs[1].Block()})
} }
p.From.Offset = b.Aux.(int64)
p.From.Type = obj.TYPE_CONST
p.Reg = b.Control.Reg()
default: default:
b.Fatalf("branch not implemented: %s. Control: %s", b.LongString(), b.Control.LongString()) b.Fatalf("branch not implemented: %s. Control: %s", b.LongString(), b.Control.LongString())
......
src/cmd/compile/internal/gc/ssa.go
View file @ ab48574b
...@@ -4684,6 +4684,16 @@ func (s *SSAGenState) SetPos(pos src.XPos) { ...@@ -4684,6 +4684,16 @@ func (s *SSAGenState) SetPos(pos src.XPos) {
s.pp.pos = pos s.pp.pos = pos
} }
// Br emits a single branch instruction and returns the instruction.
// Not all architectures need the returned instruction, but otherwise
// the boilerplate is common to all.
func (s *SSAGenState) Br(op obj.As, target *ssa.Block) *obj.Prog {
p := s.Prog(op)
p.To.Type = obj.TYPE_BRANCH
s.Branches = append(s.Branches, Branch{P: p, B: target})
return p
}
// DebugFriendlySetPos sets the position subject to heuristics // DebugFriendlySetPos sets the position subject to heuristics
// that reduce "jumpy" line number churn when debugging. // that reduce "jumpy" line number churn when debugging.
// Spill/fill/copy instructions from the register allocator, // Spill/fill/copy instructions from the register allocator,
......
src/cmd/compile/internal/mips/ssa.go
View file @ ab48574b
...@@ -828,20 +828,17 @@ func ssaGenBlock(s *gc.SSAGenState, b, next *ssa.Block) { ...@@ -828,20 +828,17 @@ func ssaGenBlock(s *gc.SSAGenState, b, next *ssa.Block) {
var p *obj.Prog var p *obj.Prog
switch next { switch next {
case b.Succs[0].Block(): case b.Succs[0].Block():
p = s.Prog(jmp.invasm) p = s.Br(jmp.invasm, b.Succs[1].Block())
p.To.Type = obj.TYPE_BRANCH
s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[1].Block()})
case b.Succs[1].Block(): case b.Succs[1].Block():
p = s.Prog(jmp.asm) p = s.Br(jmp.asm, b.Succs[0].Block())
p.To.Type = obj.TYPE_BRANCH
s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()})
default: default:
p = s.Prog(jmp.asm) if b.Likely != ssa.BranchUnlikely {
p.To.Type = obj.TYPE_BRANCH p = s.Br(jmp.asm, b.Succs[0].Block())
s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()}) s.Br(obj.AJMP, b.Succs[1].Block())
q := s.Prog(obj.AJMP) } else {
q.To.Type = obj.TYPE_BRANCH p = s.Br(jmp.invasm, b.Succs[1].Block())
s.Branches = append(s.Branches, gc.Branch{P: q, B: b.Succs[1].Block()}) s.Br(obj.AJMP, b.Succs[0].Block())
}
} }
if !b.Control.Type.IsFlags() { if !b.Control.Type.IsFlags() {
p.From.Type = obj.TYPE_REG p.From.Type = obj.TYPE_REG
......
src/cmd/compile/internal/mips64/ssa.go
View file @ ab48574b
...@@ -799,20 +799,17 @@ func ssaGenBlock(s *gc.SSAGenState, b, next *ssa.Block) { ...@@ -799,20 +799,17 @@ func ssaGenBlock(s *gc.SSAGenState, b, next *ssa.Block) {
var p *obj.Prog var p *obj.Prog
switch next { switch next {
case b.Succs[0].Block(): case b.Succs[0].Block():
p = s.Prog(jmp.invasm) p = s.Br(jmp.invasm, b.Succs[1].Block())
p.To.Type = obj.TYPE_BRANCH
s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[1].Block()})
case b.Succs[1].Block(): case b.Succs[1].Block():
p = s.Prog(jmp.asm) p = s.Br(jmp.asm, b.Succs[0].Block())
p.To.Type = obj.TYPE_BRANCH
s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()})
default: default:
p = s.Prog(jmp.asm) if b.Likely != ssa.BranchUnlikely {
p.To.Type = obj.TYPE_BRANCH p = s.Br(jmp.asm, b.Succs[0].Block())
s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()}) s.Br(obj.AJMP, b.Succs[1].Block())
q := s.Prog(obj.AJMP) } else {
q.To.Type = obj.TYPE_BRANCH p = s.Br(jmp.invasm, b.Succs[1].Block())
s.Branches = append(s.Branches, gc.Branch{P: q, B: b.Succs[1].Block()}) s.Br(obj.AJMP, b.Succs[0].Block())
}
} }
if !b.Control.Type.IsFlags() { if !b.Control.Type.IsFlags() {
p.From.Type = obj.TYPE_REG p.From.Type = obj.TYPE_REG
......
src/cmd/compile/internal/ppc64/ssa.go
View file @ ab48574b
...@@ -1176,41 +1176,34 @@ func ssaGenBlock(s *gc.SSAGenState, b, next *ssa.Block) { ...@@ -1176,41 +1176,34 @@ func ssaGenBlock(s *gc.SSAGenState, b, next *ssa.Block) {
ssa.BlockPPC64FLT, ssa.BlockPPC64FGE, ssa.BlockPPC64FLT, ssa.BlockPPC64FGE,
ssa.BlockPPC64FLE, ssa.BlockPPC64FGT: ssa.BlockPPC64FLE, ssa.BlockPPC64FGT:
jmp := blockJump[b.Kind] jmp := blockJump[b.Kind]
var p *obj.Prog
switch next { switch next {
case b.Succs[0].Block(): case b.Succs[0].Block():
p = s.Prog(jmp.invasm) s.Br(jmp.invasm, b.Succs[1].Block())
p.To.Type = obj.TYPE_BRANCH
s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[1].Block()})
if jmp.invasmun { if jmp.invasmun {
// TODO: The second branch is probably predict-not-taken since it is for FP unordered // TODO: The second branch is probably predict-not-taken since it is for FP unordered
q := s.Prog(ppc64.ABVS) s.Br(ppc64.ABVS, b.Succs[1].Block())
q.To.Type = obj.TYPE_BRANCH
s.Branches = append(s.Branches, gc.Branch{P: q, B: b.Succs[1].Block()})
} }
case b.Succs[1].Block(): case b.Succs[1].Block():
p = s.Prog(jmp.asm) s.Br(jmp.asm, b.Succs[0].Block())
p.To.Type = obj.TYPE_BRANCH
s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()})
if jmp.asmeq { if jmp.asmeq {
q := s.Prog(ppc64.ABEQ) s.Br(ppc64.ABEQ, b.Succs[0].Block())
q.To.Type = obj.TYPE_BRANCH
s.Branches = append(s.Branches, gc.Branch{P: q, B: b.Succs[0].Block()})
} }
default: default:
p = s.Prog(jmp.asm) if b.Likely != ssa.BranchUnlikely {
p.To.Type = obj.TYPE_BRANCH s.Br(jmp.asm, b.Succs[0].Block())
s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()}) if jmp.asmeq {
if jmp.asmeq { s.Br(ppc64.ABEQ, b.Succs[0].Block())
q := s.Prog(ppc64.ABEQ) }
q.To.Type = obj.TYPE_BRANCH s.Br(obj.AJMP, b.Succs[1].Block())
s.Branches = append(s.Branches, gc.Branch{P: q, B: b.Succs[0].Block()}) } else {
s.Br(jmp.invasm, b.Succs[1].Block())
if jmp.invasmun {
// TODO: The second branch is probably predict-not-taken since it is for FP unordered
s.Br(ppc64.ABVS, b.Succs[1].Block())
}
s.Br(obj.AJMP, b.Succs[0].Block())
} }
q := s.Prog(obj.AJMP)
q.To.Type = obj.TYPE_BRANCH
s.Branches = append(s.Branches, gc.Branch{P: q, B: b.Succs[1].Block()})
} }
default: default:
b.Fatalf("branch not implemented: %s. Control: %s", b.LongString(), b.Control.LongString()) b.Fatalf("branch not implemented: %s. Control: %s", b.LongString(), b.Control.LongString())
} }
......
src/cmd/compile/internal/s390x/ssa.go
View file @ ab48574b
...@@ -821,23 +821,19 @@ func ssaGenBlock(s *gc.SSAGenState, b, next *ssa.Block) { ...@@ -821,23 +821,19 @@ func ssaGenBlock(s *gc.SSAGenState, b, next *ssa.Block) {
ssa.BlockS390XLE, ssa.BlockS390XGT, ssa.BlockS390XLE, ssa.BlockS390XGT,
ssa.BlockS390XGEF, ssa.BlockS390XGTF: ssa.BlockS390XGEF, ssa.BlockS390XGTF:
jmp := blockJump[b.Kind] jmp := blockJump[b.Kind]
var p *obj.Prog
switch next { switch next {
case b.Succs[0].Block(): case b.Succs[0].Block():
p = s.Prog(jmp.invasm) s.Br(jmp.invasm, b.Succs[1].Block())
p.To.Type = obj.TYPE_BRANCH
s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[1].Block()})
case b.Succs[1].Block(): case b.Succs[1].Block():
p = s.Prog(jmp.asm) s.Br(jmp.asm, b.Succs[0].Block())
p.To.Type = obj.TYPE_BRANCH
s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()})
default: default:
p = s.Prog(jmp.asm) if b.Likely != ssa.BranchUnlikely {
p.To.Type = obj.TYPE_BRANCH s.Br(jmp.asm, b.Succs[0].Block())
s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()}) s.Br(s390x.ABR, b.Succs[1].Block())
q := s.Prog(s390x.ABR) } else {
q.To.Type = obj.TYPE_BRANCH s.Br(jmp.invasm, b.Succs[1].Block())
s.Branches = append(s.Branches, gc.Branch{P: q, B: b.Succs[1].Block()}) s.Br(s390x.ABR, b.Succs[0].Block())
}
} }
default: default:
b.Fatalf("branch not implemented: %s. Control: %s", b.LongString(), b.Control.LongString()) b.Fatalf("branch not implemented: %s. Control: %s", b.LongString(), b.Control.LongString())
......
src/cmd/compile/internal/x86/ssa.go
View file @ ab48574b
...@@ -863,25 +863,20 @@ func ssaGenBlock(s *gc.SSAGenState, b, next *ssa.Block) { ...@@ -863,25 +863,20 @@ func ssaGenBlock(s *gc.SSAGenState, b, next *ssa.Block) {
ssa.Block386ULT, ssa.Block386UGT, ssa.Block386ULT, ssa.Block386UGT,
ssa.Block386ULE, ssa.Block386UGE: ssa.Block386ULE, ssa.Block386UGE:
jmp := blockJump[b.Kind] jmp := blockJump[b.Kind]
var p *obj.Prog
switch next { switch next {
case b.Succs[0].Block(): case b.Succs[0].Block():
p = s.Prog(jmp.invasm) s.Br(jmp.invasm, b.Succs[1].Block())
p.To.Type = obj.TYPE_BRANCH
s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[1].Block()})
case b.Succs[1].Block(): case b.Succs[1].Block():
p = s.Prog(jmp.asm) s.Br(jmp.asm, b.Succs[0].Block())
p.To.Type = obj.TYPE_BRANCH
s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()})
default: default:
p = s.Prog(jmp.asm) if b.Likely != ssa.BranchUnlikely {
p.To.Type = obj.TYPE_BRANCH s.Br(jmp.asm, b.Succs[0].Block())
s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()}) s.Br(obj.AJMP, b.Succs[1].Block())
q := s.Prog(obj.AJMP) } else {
q.To.Type = obj.TYPE_BRANCH s.Br(jmp.invasm, b.Succs[1].Block())
s.Branches = append(s.Branches, gc.Branch{P: q, B: b.Succs[1].Block()}) s.Br(obj.AJMP, b.Succs[0].Block())
}
} }
default: default:
b.Fatalf("branch not implemented: %s. Control: %s", b.LongString(), b.Control.LongString()) b.Fatalf("branch not implemented: %s. Control: %s", b.LongString(), b.Control.LongString())
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7