cmd/compile/internal/ssa: emit csel on arm64

Introduce a new SSA pass to generate CondSelect intstrutions,
and add CondSelect lowering rules for arm64.

In order to make the CSEL instruction easier to optimize,
and to simplify the introduction of CSNEG, CSINC, and CSINV
in the future, modify the CSEL instruction to accept a condition
code in the aux field.

Notably, this change makes the go1 Gzip benchmark
more than 10% faster.

Benchmarks on a Cavium ThunderX:

name                      old time/op    new time/op    delta
BinaryTree17-96              15.9s ± 6%     16.0s ± 4%     ~     (p=0.968 n=10+9)
Fannkuch11-96                7.17s ± 0%     7.00s ± 0%   -2.43%  (p=0.000 n=8+9)
FmtFprintfEmpty-96           208ns ± 1%     207ns ± 0%     ~     (p=0.152 n=10+8)
FmtFprintfString-96          379ns ± 0%     375ns ± 0%   -0.95%  (p=0.000 n=10+9)
FmtFprintfInt-96             385ns ± 0%     383ns ± 0%   -0.52%  (p=0.000 n=9+10)
FmtFprintfIntInt-96          591ns ± 0%     586ns ± 0%   -0.85%  (p=0.006 n=7+9)
FmtFprintfPrefixedInt-96     656ns ± 0%     667ns ± 0%   +1.71%  (p=0.000 n=10+10)
FmtFprintfFloat-96           967ns ± 0%     984ns ± 0%   +1.78%  (p=0.000 n=10+10)
FmtManyArgs-96              2.35µs ± 0%    2.25µs ± 0%   -4.63%  (p=0.000 n=9+8)
GobDecode-96                31.0ms ± 0%    30.8ms ± 0%   -0.36%  (p=0.006 n=9+9)
GobEncode-96                24.4ms ± 0%    24.5ms ± 0%   +0.30%  (p=0.000 n=9+9)
Gzip-96                      1.60s ± 0%     1.43s ± 0%  -10.58%  (p=0.000 n=9+10)
Gunzip-96                    167ms ± 0%     169ms ± 0%   +0.83%  (p=0.000 n=8+9)
HTTPClientServer-96          311µs ± 1%     308µs ± 0%   -0.75%  (p=0.000 n=10+10)
JSONEncode-96               65.0ms ± 0%    64.8ms ± 0%   -0.25%  (p=0.000 n=9+8)
JSONDecode-96                262ms ± 1%     261ms ± 1%     ~     (p=0.579 n=10+10)
Mandelbrot200-96            18.0ms ± 0%    18.1ms ± 0%   +0.17%  (p=0.000 n=8+10)
GoParse-96                  14.0ms ± 0%    14.1ms ± 1%   +0.42%  (p=0.003 n=9+10)
RegexpMatchEasy0_32-96       644ns ± 2%     645ns ± 2%     ~     (p=0.836 n=10+10)
RegexpMatchEasy0_1K-96      3.70µs ± 0%    3.49µs ± 0%   -5.58%  (p=0.000 n=10+10)
RegexpMatchEasy1_32-96       662ns ± 2%     657ns ± 2%     ~     (p=0.137 n=10+10)
RegexpMatchEasy1_1K-96      4.47µs ± 0%    4.31µs ± 0%   -3.48%  (p=0.000 n=10+10)
RegexpMatchMedium_32-96      844ns ± 2%     849ns ± 1%     ~     (p=0.208 n=10+10)
RegexpMatchMedium_1K-96      179µs ± 0%     182µs ± 0%   +1.20%  (p=0.000 n=10+10)
RegexpMatchHard_32-96       10.0µs ± 0%    10.1µs ± 0%   +0.48%  (p=0.000 n=10+9)
RegexpMatchHard_1K-96        297µs ± 0%     297µs ± 0%   -0.14%  (p=0.000 n=10+10)
Revcomp-96                   3.08s ± 0%     3.13s ± 0%   +1.56%  (p=0.000 n=9+9)
Template-96                  276ms ± 2%     275ms ± 1%     ~     (p=0.393 n=10+10)
TimeParse-96                1.37µs ± 0%    1.36µs ± 0%   -0.53%  (p=0.000 n=10+7)
TimeFormat-96               1.40µs ± 0%    1.42µs ± 0%   +0.97%  (p=0.000 n=10+10)
[Geo mean]                   264µs          262µs        -0.77%

Change-Id: Ie54eee4b3092af53e6da3baa6d1755098f57f3a2
Reviewed-on: https://go-review.googlesource.com/55670
Run-TryBot: Philip Hofer <phofer@umich.edu>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Cherry Zhang <cherryyz@google.com>
Reviewed-by: Keith Randall <khr@golang.org>

src/cmd/compile/internal/arm64/ssa.go

@@ -581,15 +581,14 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) {
 		p.From.Reg = (v.Args[0].Reg()-arm64.REG_F0)&31 + arm64.REG_ARNG + ((arm64.ARNG_8B & 15) << 5)
 		p.To.Type = obj.TYPE_REG
 		p.To.Reg = v.Reg() - arm64.REG_F0 + arm64.REG_V0
-	case ssa.OpARM64CSELULT,
-		ssa.OpARM64CSELULT0:
+	case ssa.OpARM64CSEL, ssa.OpARM64CSEL0:
 		r1 := int16(arm64.REGZERO)
-		if v.Op == ssa.OpARM64CSELULT {
+		if v.Op != ssa.OpARM64CSEL0 {
 			r1 = v.Args[1].Reg()
 		}
 		p := s.Prog(v.Op.Asm())
 		p.From.Type = obj.TYPE_REG // assembler encodes conditional bits in Reg
-		p.From.Reg = arm64.COND_LO
+		p.From.Reg = condBits[v.Aux.(ssa.Op)]
 		p.Reg = v.Args[0].Reg()
 		p.SetFrom3(obj.Addr{Type: obj.TYPE_REG, Reg: r1})
 		p.To.Type = obj.TYPE_REG

src/cmd/compile/internal/gc/asm_test.go

@@ -2890,6 +2890,19 @@ var linuxARM64Tests = []*asmTest{
 		`,
 		pos: []string{"FRINTZD"},
 	},
+	{
+		// make sure that CSEL is emitted for conditional moves
+		fn: `
+		func f37(c int) int {
+		     x := c + 4
+		     if c < 0 {
+		     	x = 182
+		     }
+		     return x
+		}
+		`,
+		pos: []string{"\tCSEL\t"},
+	},
 }
 
 var linuxMIPSTests = []*asmTest{

src/cmd/compile/internal/gc/swt.go

@@ -568,7 +568,7 @@ Outer:
 		if !ok {
 			// First entry for this hash.
 			nn = append(nn, c.node)
-			seen[c.hash] = nn[len(nn)-1 : len(nn) : len(nn)]
+			seen[c.hash] = nn[len(nn)-1 : len(nn):len(nn)]
 			continue
 		}
 		for _, n := range prev {

src/cmd/compile/internal/ssa/branchelim.go

+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package ssa
+
+// branchelim tries to elminiate branches by
+// generating CondSelect instructions.
+//
+// Search for basic blocks that look like
+//
+// bb0            bb0
+//  | \          /   \
+//  | bb1  or  bb1   bb2    <- trivial if/else blocks
+//  | /          \   /
+// bb2            bb3
+//
+// where the intermediate blocks are mostly empty (with no side-effects);
+// rewrite Phis in the postdominator as CondSelects.
+func branchelim(f *Func) {
+	// FIXME: add support for lowering CondSelects on more architectures
+	if f.Config.arch != "arm64" {
+		return
+	}
+
+	change := true
+	for change {
+		change = false
+		for _, b := range f.Blocks {
+			change = elimIf(f, b) || elimIfElse(f, b) || change
+		}
+	}
+}
+
+func canCondSelect(v *Value) bool {
+	// For now, stick to simple scalars that fit in registers
+	sz := v.Type.Size()
+	return sz <= v.Block.Func.Config.RegSize && (v.Type.IsInteger() || v.Type.IsPtrShaped())
+}
+
+func elimIf(f *Func, dom *Block) bool {
+	// See if dom is an If with one arm that
+	// is trivial and succeeded by the other
+	// successor of dom.
+	if dom.Kind != BlockIf || dom.Likely != BranchUnknown {
+		return false
+	}
+	var simple, post *Block
+	for i := range dom.Succs {
+		bb, other := dom.Succs[i].Block(), dom.Succs[i^1].Block()
+		if isLeafPlain(bb) && bb.Succs[0].Block() == other {
+			simple = bb
+			post = other
+			break
+		}
+	}
+	if simple == nil || len(post.Preds) != 2 || post == dom {
+		return false
+	}
+
+	// We've found our diamond CFG of blocks.
+	// Now decide if fusing 'simple' into dom+post
+	// looks profitable.
+
+	// Check that there are Phis, and that all of them
+	// can be safely rewritten to CondSelect.
+	hasphis := false
+	for _, v := range post.Values {
+		if v.Op == OpPhi {
+			hasphis = true
+			if !canCondSelect(v) {
+				return false
+			}
+		}
+	}
+	if !hasphis {
+		return false
+	}
+
+	// Pick some upper bound for the number of instructions
+	// we'd be willing to execute just to generate a dead
+	// argument to CondSelect. In the worst case, this is
+	// the number of useless instructions executed.
+	const maxfuseinsts = 2
+
+	if len(simple.Values) > maxfuseinsts || !allTrivial(simple) {
+		return false
+	}
+
+	// Replace Phi instructions in b with CondSelect instructions
+	swap := (post.Preds[0].Block() == dom) != (dom.Succs[0].Block() == post)
+	for _, v := range post.Values {
+		if v.Op != OpPhi {
+			continue
+		}
+		v.Op = OpCondSelect
+		if swap {
+			v.Args[0], v.Args[1] = v.Args[1], v.Args[0]
+		}
+		v.AddArg(dom.Control)
+	}
+
+	// Put all of the instructions into 'dom'
+	// and update the CFG appropriately.
+	dom.Kind = post.Kind
+	dom.SetControl(post.Control)
+	dom.Aux = post.Aux
+	dom.Succs = append(dom.Succs[:0], post.Succs...)
+	for i := range dom.Succs {
+		e := dom.Succs[i]
+		e.b.Preds[e.i].b = dom
+	}
+
+	for i := range simple.Values {
+		simple.Values[i].Block = dom
+	}
+	for i := range post.Values {
+		post.Values[i].Block = dom
+	}
+	dom.Values = append(dom.Values, simple.Values...)
+	dom.Values = append(dom.Values, post.Values...)
+
+	// Trash 'post' and 'simple'
+	clobberBlock(post)
+	clobberBlock(simple)
+
+	f.invalidateCFG()
+	return true
+}
+
+// is this a BlockPlain with one predecessor?
+func isLeafPlain(b *Block) bool {
+	return b.Kind == BlockPlain && len(b.Preds) == 1
+}
+
+func clobberBlock(b *Block) {
+	b.Values = nil
+	b.Preds = nil
+	b.Succs = nil
+	b.Aux = nil
+	b.SetControl(nil)
+	b.Kind = BlockInvalid
+}
+
+func elimIfElse(f *Func, b *Block) bool {
+	// See if 'b' ends in an if/else: it should
+	// have two successors, both of which are BlockPlain
+	// and succeeded by the same block.
+	if b.Kind != BlockIf || b.Likely != BranchUnknown {
+		return false
+	}
+	yes, no := b.Succs[0].Block(), b.Succs[1].Block()
+	if !isLeafPlain(yes) || len(yes.Values) > 1 || !allTrivial(yes) {
+		return false
+	}
+	if !isLeafPlain(no) || len(no.Values) > 1 || !allTrivial(no) {
+		return false
+	}
+	if b.Succs[0].Block().Succs[0].Block() != b.Succs[1].Block().Succs[0].Block() {
+		return false
+	}
+	// block that postdominates the if/else
+	post := b.Succs[0].Block().Succs[0].Block()
+	if len(post.Preds) != 2 || post == b {
+		return false
+	}
+	hasphis := false
+	for _, v := range post.Values {
+		if v.Op == OpPhi {
+			hasphis = true
+			if !canCondSelect(v) {
+				return false
+			}
+		}
+	}
+	if !hasphis {
+		return false
+	}
+
+	// now we're committed: rewrite each Phi as a CondSelect
+	swap := post.Preds[0].Block() != b.Succs[0].Block()
+	for _, v := range post.Values {
+		if v.Op != OpPhi {
+			continue
+		}
+		v.Op = OpCondSelect
+		if swap {
+			v.Args[0], v.Args[1] = v.Args[1], v.Args[0]
+		}
+		v.AddArg(b.Control)
+	}
+
+	// Move the contents of all of these
+	// blocks into 'b' and update CFG edges accordingly
+	b.Kind = post.Kind
+	b.SetControl(post.Control)
+	b.Aux = post.Aux
+	b.Succs = append(b.Succs[:0], post.Succs...)
+	for i := range b.Succs {
+		e := b.Succs[i]
+		e.b.Preds[e.i].b = b
+	}
+	for i := range post.Values {
+		post.Values[i].Block = b
+	}
+	for i := range yes.Values {
+		yes.Values[i].Block = b
+	}
+	for i := range no.Values {
+		no.Values[i].Block = b
+	}
+	b.Values = append(b.Values, yes.Values...)
+	b.Values = append(b.Values, no.Values...)
+	b.Values = append(b.Values, post.Values...)
+
+	// trash post, yes, and no
+	clobberBlock(yes)
+	clobberBlock(no)
+	clobberBlock(post)
+
+	f.invalidateCFG()
+	return true
+}
+
+func allTrivial(b *Block) bool {
+	// don't fuse memory ops, Phi ops, divides (can panic),
+	// or anything else with side-effects
+	for _, v := range b.Values {
+		if v.Op == OpPhi || isDivMod(v.Op) || v.Type.IsMemory() ||
+			v.MemoryArg() != nil || opcodeTable[v.Op].hasSideEffects {
+			return false
+		}
+	}
+	return true
+}
+
+func isDivMod(op Op) bool {
+	switch op {
+	case OpDiv8, OpDiv8u, OpDiv16, OpDiv16u,
+		OpDiv32, OpDiv32u, OpDiv64, OpDiv64u, OpDiv128u,
+		OpDiv32F, OpDiv64F,
+		OpMod8, OpMod8u, OpMod16, OpMod16u,
+		OpMod32, OpMod32u, OpMod64, OpMod64u:
+		return true
+	default:
+		return false
+	}
+}

src/cmd/compile/internal/ssa/branchelim_test.go

+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package ssa
+
+import (
+	"cmd/compile/internal/types"
+	"testing"
+)
+
+// Test that a trivial 'if' is eliminated
+func TestBranchElimIf(t *testing.T) {
+	c := testConfig(t)
+	c.config.arch = "arm64" // FIXME
+	boolType := types.New(types.TBOOL)
+	intType := types.New(types.TINT32)
+	fun := c.Fun("entry",
+		Bloc("entry",
+			Valu("start", OpInitMem, types.TypeMem, 0, nil),
+			Valu("sb", OpSB, types.TypeInvalid, 0, nil),
+			Valu("const1", OpConst32, intType, 1, nil),
+			Valu("const2", OpConst32, intType, 2, nil),
+			Valu("addr", OpAddr, boolType.PtrTo(), 0, nil, "sb"),
+			Valu("cond", OpLoad, boolType, 0, nil, "addr", "start"),
+			If("cond", "b2", "b3")),
+		Bloc("b2",
+			Goto("b3")),
+		Bloc("b3",
+			Valu("phi", OpPhi, intType, 0, nil, "const1", "const2"),
+			Valu("retstore", OpStore, types.TypeMem, 0, nil, "phi", "sb", "start"),
+			Exit("retstore")))
+
+	CheckFunc(fun.f)
+	branchelim(fun.f)
+	CheckFunc(fun.f)
+	Deadcode(fun.f)
+	CheckFunc(fun.f)
+
+	if len(fun.f.Blocks) != 1 {
+		t.Errorf("expected 1 block after branchelim and deadcode; found %d", len(fun.f.Blocks))
+	}
+	if fun.values["phi"].Op != OpCondSelect {
+		t.Errorf("expected phi op to be CondSelect; found op %s", fun.values["phi"].Op)
+	}
+	if fun.values["phi"].Args[2] != fun.values["cond"] {
+		t.Errorf("expected CondSelect condition to be %s; found %s", fun.values["cond"], fun.values["phi"].Args[2])
+	}
+	if fun.blocks["entry"].Kind != BlockExit {
+		t.Errorf("expected entry to be BlockExit; found kind %s", fun.blocks["entry"].Kind.String())
+	}
+}
+
+// Test that a trivial if/else is eliminated
+func TestBranchElimIfElse(t *testing.T) {
+	c := testConfig(t)
+	c.config.arch = "arm64" // FIXME
+	boolType := types.New(types.TBOOL)
+	intType := types.New(types.TINT32)
+	fun := c.Fun("entry",
+		Bloc("entry",
+			Valu("start", OpInitMem, types.TypeMem, 0, nil),
+			Valu("sb", OpSB, types.TypeInvalid, 0, nil),
+			Valu("const1", OpConst32, intType, 1, nil),
+			Valu("const2", OpConst32, intType, 2, nil),
+			Valu("addr", OpAddr, boolType.PtrTo(), 0, nil, "sb"),
+			Valu("cond", OpLoad, boolType, 0, nil, "addr", "start"),
+			If("cond", "b2", "b3")),
+		Bloc("b2",
+			Goto("b4")),
+		Bloc("b3",
+			Goto("b4")),
+		Bloc("b4",
+			Valu("phi", OpPhi, intType, 0, nil, "const1", "const2"),
+			Valu("retstore", OpStore, types.TypeMem, 0, nil, "phi", "sb", "start"),
+			Exit("retstore")))
+
+	CheckFunc(fun.f)
+	branchelim(fun.f)
+	CheckFunc(fun.f)
+	Deadcode(fun.f)
+	CheckFunc(fun.f)
+
+	if len(fun.f.Blocks) != 1 {
+		t.Errorf("expected 1 block after branchelim; found %d", len(fun.f.Blocks))
+	}
+	if fun.values["phi"].Op != OpCondSelect {
+		t.Errorf("expected phi op to be CondSelect; found op %s", fun.values["phi"].Op)
+	}
+	if fun.values["phi"].Args[2] != fun.values["cond"] {
+		t.Errorf("expected CondSelect condition to be %s; found %s", fun.values["cond"], fun.values["phi"].Args[2])
+	}
+	if fun.blocks["entry"].Kind != BlockExit {
+		t.Errorf("expected entry to be BlockExit; found kind %s", fun.blocks["entry"].Kind.String())
+	}
+}
+
+// Test that an if/else CFG that loops back
+// into itself does *not* get eliminated.
+func TestNoBranchElimLoop(t *testing.T) {
+	c := testConfig(t)
+	c.config.arch = "arm64" // FIXME
+	boolType := types.New(types.TBOOL)
+	intType := types.New(types.TINT32)
+
+	// The control flow here is totally bogus,
+	// but a dead cycle seems like the only plausible
+	// way to arrive at a diamond CFG that is also a loop.
+	fun := c.Fun("entry",
+		Bloc("entry",
+			Valu("start", OpInitMem, types.TypeMem, 0, nil),
+			Valu("sb", OpSB, types.TypeInvalid, 0, nil),
+			Valu("const2", OpConst32, intType, 2, nil),
+			Valu("const3", OpConst32, intType, 3, nil),
+			Goto("b5")),
+		Bloc("b2",
+			Valu("addr", OpAddr, boolType.PtrTo(), 0, nil, "sb"),
+			Valu("cond", OpLoad, boolType, 0, nil, "addr", "start"),
+			Valu("phi", OpPhi, intType, 0, nil, "const2", "const3"),
+			If("cond", "b3", "b4")),
+		Bloc("b3",
+			Goto("b2")),
+		Bloc("b4",
+			Goto("b2")),
+		Bloc("b5",
+			Exit("start")))
+
+	CheckFunc(fun.f)
+	branchelim(fun.f)
+	CheckFunc(fun.f)
+
+	if len(fun.f.Blocks) != 5 {
+		t.Errorf("expected 5 block after branchelim; found %d", len(fun.f.Blocks))
+	}
+	if fun.values["phi"].Op != OpPhi {
+		t.Errorf("expected phi op to be CondSelect; found op %s", fun.values["phi"].Op)
+	}
+}

src/cmd/compile/internal/ssa/check.go

@@ -160,6 +160,11 @@ func checkFunc(f *Func) {
 				}
 				canHaveAuxInt = true
 				canHaveAux = true
+			case auxCCop:
+				if _, ok := v.Aux.(Op); !ok {
+					f.Fatalf("bad type %T for CCop in %v", v.Aux, v)
+				}
+				canHaveAux = true
 			default:
 				f.Fatalf("unknown aux type for %s", v.Op)
 			}

src/cmd/compile/internal/ssa/compile.go

@@ -348,6 +348,7 @@ var passes = [...]pass{
 	{name: "late opt", fn: opt, required: true}, // TODO: split required rules and optimizing rules
 	{name: "generic deadcode", fn: deadcode},
 	{name: "check bce", fn: checkbce},
+	{name: "branchelim", fn: branchelim},
 	{name: "fuse", fn: fuse},
 	{name: "dse", fn: dse},
 	{name: "writebarrier", fn: writebarrier, required: true}, // expand write barrier ops

src/cmd/compile/internal/ssa/deadcode.go

@@ -27,6 +27,9 @@ func ReachableBlocks(f *Func) []bool {
 		}
 		for _, e := range s {
 			c := e.b
+			if int(c.ID) >= len(reachable) {
+				f.Fatalf("block %s >= f.NumBlocks()=%d?", c, len(reachable))
+			}
 			if !reachable[c.ID] {
 				reachable[c.ID] = true
 				p = append(p, c) // push

src/cmd/compile/internal/ssa/gen/ARM64Ops.go

@@ -329,9 +329,10 @@ func init() {
 		{name: "FRINTPD", argLength: 1, reg: fp11, asm: "FRINTPD"},
 		{name: "FRINTZD", argLength: 1, reg: fp11, asm: "FRINTZD"},
 
-		// conditional instructions
-		{name: "CSELULT", argLength: 3, reg: gp2flags1, asm: "CSEL"},  // returns arg0 if flags indicates unsigned LT, arg1 otherwise, arg2=flags
-		{name: "CSELULT0", argLength: 2, reg: gp1flags1, asm: "CSEL"}, // returns arg0 if flags indicates unsigned LT, 0 otherwise, arg1=flags
+		// conditional instructions; auxint is
+		// one of the arm64 comparison pseudo-ops (LessThan, LessThanU, etc.)
+		{name: "CSEL", argLength: 3, reg: gp2flags1, asm: "CSEL", aux: "CCop"},  // aux(flags) ? arg0 : arg1
+		{name: "CSEL0", argLength: 2, reg: gp1flags1, asm: "CSEL", aux: "CCop"}, // aux(flags) ? arg0 : 0
 
 		// function calls
 		{name: "CALLstatic", argLength: 1, reg: regInfo{clobbers: callerSave}, aux: "SymOff", clobberFlags: true, call: true, symEffect: "None"},                           // call static function aux.(*obj.LSym).  arg0=mem, auxint=argsize, returns mem

src/cmd/compile/internal/ssa/gen/genericOps.go

@@ -217,6 +217,11 @@ var genericOps = []opData{
 	{name: "Geq32F", argLength: 2, typ: "Bool"},
 	{name: "Geq64F", argLength: 2, typ: "Bool"},
 
+	// the type of a CondSelect is the same as the type of its first
+	// two arguments, which should be register-width scalars; the third
+	// argument should be a boolean
+	{name: "CondSelect", argLength: 3}, // arg2 ? arg0 : arg1
+
 	// boolean ops
 	{name: "AndB", argLength: 2, commutative: true, typ: "Bool"}, // arg0 && arg1 (not shortcircuited)
 	{name: "OrB", argLength: 2, commutative: true, typ: "Bool"},  // arg0 || arg1 (not shortcircuited)

src/cmd/compile/internal/ssa/gen/rulegen.go

@@ -727,7 +727,7 @@ func parseValue(val string, arch arch, loc string) (op opData, oparch string, ty
 	}
 	if aux != "" {
 		switch op.aux {
-		case "String", "Sym", "SymOff", "SymValAndOff", "SymInt32", "Typ", "TypSize":
+		case "String", "Sym", "SymOff", "SymValAndOff", "SymInt32", "Typ", "TypSize", "CCop":
 		default:
 			log.Fatalf("%s: op %s %s can't have aux", loc, op.name, op.aux)
 		}

src/cmd/compile/internal/ssa/op.go

@@ -72,6 +72,7 @@ const (
 	auxSymValAndOff         // aux is a symbol, auxInt is a ValAndOff
 	auxTyp                  // aux is a type
 	auxTypSize              // aux is a type, auxInt is a size, must have Aux.(Type).Size() == AuxInt
+	auxCCop                 // aux is a ssa.Op that represents a flags-to-bool conversion (e.g. LessThan)
 
 	auxSymInt32 // aux is a symbol, auxInt is a 32-bit integer
 )

src/cmd/compile/internal/ssa/opGen.go

@@ -1097,8 +1097,8 @@ const (
 	OpARM64FRINTMD
 	OpARM64FRINTPD
 	OpARM64FRINTZD
-	OpARM64CSELULT
-	OpARM64CSELULT0
+	OpARM64CSEL
+	OpARM64CSEL0
 	OpARM64CALLstatic
 	OpARM64CALLclosure
 	OpARM64CALLinter
@@ -1888,6 +1888,7 @@ const (
 	OpGeq64U
 	OpGeq32F
 	OpGeq64F
+	OpCondSelect
 	OpAndB
 	OpOrB
 	OpEqB
@@ -14028,9 +14029,10 @@ var opcodeTable = [...]opInfo{
 		},
 	},
 	{
-		name:   "CSELULT",
-		argLen: 3,
-		asm:    arm64.ACSEL,
+		name:    "CSEL",
+		auxType: auxCCop,
+		argLen:  3,
+		asm:     arm64.ACSEL,
 		reg: regInfo{
 			inputs: []inputInfo{
 				{0, 670826495}, // R0 R1 R2 R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R13 R14 R15 R16 R17 R19 R20 R21 R22 R23 R24 R25 R26 R30
@@ -14042,9 +14044,10 @@ var opcodeTable = [...]opInfo{
 		},
 	},
 	{
-		name:   "CSELULT0",
-		argLen: 2,
-		asm:    arm64.ACSEL,
+		name:    "CSEL0",
+		auxType: auxCCop,
+		argLen:  2,
+		asm:     arm64.ACSEL,
 		reg: regInfo{
 			inputs: []inputInfo{
 				{0, 805044223}, // R0 R1 R2 R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R13 R14 R15 R16 R17 R19 R20 R21 R22 R23 R24 R25 R26 g R30
@@ -23465,6 +23468,11 @@ var opcodeTable = [...]opInfo{
 		argLen:  2,
 		generic: true,
 	},
+	{
+		name:    "CondSelect",
+		argLen:  3,
+		generic: true,
+	},
 	{
 		name:        "AndB",
 		argLen:      2,

src/cmd/compile/internal/ssa/rewrite.go

@@ -534,6 +534,131 @@ func warnRule(cond bool, v *Value, s string) bool {
 	return true
 }
 
+// for a pseudo-op like (LessThan x), extract x
+func flagArg(v *Value) *Value {
+	if len(v.Args) != 1 || !v.Args[0].Type.IsFlags() {
+		return nil
+	}
+	return v.Args[0]
+}
+
+// arm64Negate finds the complement to an ARM64 condition code,
+// for example Equal -> NotEqual or LessThan -> GreaterEqual
+//
+// TODO: add floating-point conditions
+func arm64Negate(op Op) Op {
+	switch op {
+	case OpARM64LessThan:
+		return OpARM64GreaterEqual
+	case OpARM64LessThanU:
+		return OpARM64GreaterEqualU
+	case OpARM64GreaterThan:
+		return OpARM64LessEqual
+	case OpARM64GreaterThanU:
+		return OpARM64LessEqualU
+	case OpARM64LessEqual:
+		return OpARM64GreaterThan
+	case OpARM64LessEqualU:
+		return OpARM64GreaterThanU
+	case OpARM64GreaterEqual:
+		return OpARM64LessThan
+	case OpARM64GreaterEqualU:
+		return OpARM64LessThanU
+	case OpARM64Equal:
+		return OpARM64NotEqual
+	case OpARM64NotEqual:
+		return OpARM64Equal
+	default:
+		panic("unreachable")
+	}
+}
+
+// arm64Invert evaluates (InvertFlags op), which
+// is the same as altering the condition codes such
+// that the same result would be produced if the arguments
+// to the flag-generating instruction were reversed, e.g.
+// (InvertFlags (CMP x y)) -> (CMP y x)
+//
+// TODO: add floating-point conditions
+func arm64Invert(op Op) Op {
+	switch op {
+	case OpARM64LessThan:
+		return OpARM64GreaterThan
+	case OpARM64LessThanU:
+		return OpARM64GreaterThanU
+	case OpARM64GreaterThan:
+		return OpARM64LessThan
+	case OpARM64GreaterThanU:
+		return OpARM64LessThanU
+	case OpARM64LessEqual:
+		return OpARM64GreaterEqual
+	case OpARM64LessEqualU:
+		return OpARM64GreaterEqualU
+	case OpARM64GreaterEqual:
+		return OpARM64LessEqual
+	case OpARM64GreaterEqualU:
+		return OpARM64LessEqualU
+	case OpARM64Equal, OpARM64NotEqual:
+		return op
+	default:
+		panic("unreachable")
+	}
+}
+
+// evaluate an ARM64 op against a flags value
+// that is potentially constant; return 1 for true,
+// -1 for false, and 0 for not constant.
+func ccARM64Eval(cc interface{}, flags *Value) int {
+	op := cc.(Op)
+	fop := flags.Op
+	switch fop {
+	case OpARM64InvertFlags:
+		return -ccARM64Eval(op, flags.Args[0])
+	case OpARM64FlagEQ:
+		switch op {
+		case OpARM64Equal, OpARM64GreaterEqual, OpARM64LessEqual,
+			OpARM64GreaterEqualU, OpARM64LessEqualU:
+			return 1
+		default:
+			return -1
+		}
+	case OpARM64FlagLT_ULT:
+		switch op {
+		case OpARM64LessThan, OpARM64LessThanU,
+			OpARM64LessEqual, OpARM64LessEqualU:
+			return 1
+		default:
+			return -1
+		}
+	case OpARM64FlagLT_UGT:
+		switch op {
+		case OpARM64LessThan, OpARM64GreaterThanU,
+			OpARM64LessEqual, OpARM64GreaterEqualU:
+			return 1
+		default:
+			return -1
+		}
+	case OpARM64FlagGT_ULT:
+		switch op {
+		case OpARM64GreaterThan, OpARM64LessThanU,
+			OpARM64GreaterEqual, OpARM64LessEqualU:
+			return 1
+		default:
+			return -1
+		}
+	case OpARM64FlagGT_UGT:
+		switch op {
+		case OpARM64GreaterThan, OpARM64GreaterThanU,
+			OpARM64GreaterEqual, OpARM64GreaterEqualU:
+			return 1
+		default:
+			return -1
+		}
+	default:
+		return 0
+	}
+}
+
 // logRule logs the use of the rule s. This will only be enabled if
 // rewrite rules were generated with the -log option, see gen/rulegen.go.
 func logRule(s string) {

src/cmd/compile/internal/ssa/value.go

@@ -166,6 +166,8 @@ func (v *Value) auxString() string {
 			s = fmt.Sprintf(" {%v}", v.Aux)
 		}
 		return s + fmt.Sprintf(" [%s]", v.AuxValAndOff())
+	case auxCCop:
+		return fmt.Sprintf(" {%s}", v.Aux.(Op))
 	}
 	return ""
 }