Commit c8f38b33 authored by Cherry Zhang's avatar Cherry Zhang

cmd/compile: use type information in Aux for Store size

Remove size AuxInt in Store, and alignment in Move/Zero. We still
pass size AuxInt to Move/Zero, as it is used for partial Move/Zero
lowering (e.g. cmd/compile/internal/ssa/gen/386.rules:288).
SizeAndAlign is gone.

Passes "toolstash -cmp" on std.

Change-Id: I1ca34652b65dd30de886940e789fcf41d521475d
Reviewed-on: https://go-review.googlesource.com/38150
Run-TryBot: Cherry Zhang <cherryyz@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: default avatarKeith Randall <khr@golang.org>
parent d75925d6
...@@ -640,7 +640,6 @@ var knownFormats = map[string]string{ ...@@ -640,7 +640,6 @@ var knownFormats = map[string]string{
"cmd/compile/internal/ssa.Location %v": "", "cmd/compile/internal/ssa.Location %v": "",
"cmd/compile/internal/ssa.Op %s": "", "cmd/compile/internal/ssa.Op %s": "",
"cmd/compile/internal/ssa.Op %v": "", "cmd/compile/internal/ssa.Op %v": "",
"cmd/compile/internal/ssa.SizeAndAlign %s": "",
"cmd/compile/internal/ssa.Type %s": "", "cmd/compile/internal/ssa.Type %s": "",
"cmd/compile/internal/ssa.Type %v": "", "cmd/compile/internal/ssa.Type %v": "",
"cmd/compile/internal/ssa.ValAndOff %s": "", "cmd/compile/internal/ssa.ValAndOff %s": "",
......
This diff is collapsed.
...@@ -145,11 +145,9 @@ func checkFunc(f *Func) { ...@@ -145,11 +145,9 @@ func checkFunc(f *Func) {
if !isExactFloat32(v) { if !isExactFloat32(v) {
f.Fatalf("value %v has an AuxInt value that is not an exact float32", v) f.Fatalf("value %v has an AuxInt value that is not an exact float32", v)
} }
case auxSizeAndAlign: case auxString, auxSym, auxTyp:
canHaveAuxInt = true
case auxString, auxSym:
canHaveAux = true canHaveAux = true
case auxSymOff, auxSymValAndOff, auxSymSizeAndAlign: case auxSymOff, auxSymValAndOff, auxTypSize:
canHaveAuxInt = true canHaveAuxInt = true
canHaveAux = true canHaveAux = true
case auxSymInt32: case auxSymInt32:
......
...@@ -38,7 +38,7 @@ func TestCSEAuxPartitionBug(t *testing.T) { ...@@ -38,7 +38,7 @@ func TestCSEAuxPartitionBug(t *testing.T) {
Valu("r3", OpAdd64, TypeInt64, 0, nil, "arg1", "arg2"), Valu("r3", OpAdd64, TypeInt64, 0, nil, "arg1", "arg2"),
Valu("r5", OpAdd64, TypeInt64, 0, nil, "r2", "r3"), Valu("r5", OpAdd64, TypeInt64, 0, nil, "r2", "r3"),
Valu("r10", OpAdd64, TypeInt64, 0, nil, "r6", "r9"), Valu("r10", OpAdd64, TypeInt64, 0, nil, "r6", "r9"),
Valu("rstore", OpStore, TypeMem, 8, nil, "raddr", "r10", "raddrdef"), Valu("rstore", OpStore, TypeMem, 0, TypeInt64, "raddr", "r10", "raddrdef"),
Goto("exit")), Goto("exit")),
Bloc("exit", Bloc("exit",
Exit("rstore"))) Exit("rstore")))
...@@ -104,7 +104,7 @@ func TestZCSE(t *testing.T) { ...@@ -104,7 +104,7 @@ func TestZCSE(t *testing.T) {
Valu("r3", OpAdd64, TypeInt64, 0, nil, "r1", "r2"), Valu("r3", OpAdd64, TypeInt64, 0, nil, "r1", "r2"),
Valu("raddr", OpAddr, TypeInt64Ptr, 0, nil, "sp"), Valu("raddr", OpAddr, TypeInt64Ptr, 0, nil, "sp"),
Valu("raddrdef", OpVarDef, TypeMem, 0, nil, "start"), Valu("raddrdef", OpVarDef, TypeMem, 0, nil, "start"),
Valu("rstore", OpStore, TypeMem, 8, nil, "raddr", "r3", "raddrdef"), Valu("rstore", OpStore, TypeMem, 0, TypeInt64, "raddr", "r3", "raddrdef"),
Goto("exit")), Goto("exit")),
Bloc("exit", Bloc("exit",
Exit("rstore"))) Exit("rstore")))
......
...@@ -88,9 +88,9 @@ func dse(f *Func) { ...@@ -88,9 +88,9 @@ func dse(f *Func) {
if v.Op == OpStore || v.Op == OpZero { if v.Op == OpStore || v.Op == OpZero {
var sz int64 var sz int64
if v.Op == OpStore { if v.Op == OpStore {
sz = v.AuxInt sz = v.Aux.(Type).Size()
} else { // OpZero } else { // OpZero
sz = SizeAndAlign(v.AuxInt).Size() sz = v.AuxInt
} }
if shadowedSize := int64(shadowed.get(v.Args[0].ID)); shadowedSize != -1 && shadowedSize >= sz { if shadowedSize := int64(shadowed.get(v.Args[0].ID)); shadowedSize != -1 && shadowedSize >= sz {
// Modify store into a copy // Modify store into a copy
......
...@@ -18,11 +18,11 @@ func TestDeadStore(t *testing.T) { ...@@ -18,11 +18,11 @@ func TestDeadStore(t *testing.T) {
Valu("addr1", OpAddr, ptrType, 0, nil, "sb"), Valu("addr1", OpAddr, ptrType, 0, nil, "sb"),
Valu("addr2", OpAddr, ptrType, 0, nil, "sb"), Valu("addr2", OpAddr, ptrType, 0, nil, "sb"),
Valu("addr3", OpAddr, ptrType, 0, nil, "sb"), Valu("addr3", OpAddr, ptrType, 0, nil, "sb"),
Valu("zero1", OpZero, TypeMem, 1, nil, "addr3", "start"), Valu("zero1", OpZero, TypeMem, 1, TypeBool, "addr3", "start"),
Valu("store1", OpStore, TypeMem, 1, nil, "addr1", "v", "zero1"), Valu("store1", OpStore, TypeMem, 0, TypeBool, "addr1", "v", "zero1"),
Valu("store2", OpStore, TypeMem, 1, nil, "addr2", "v", "store1"), Valu("store2", OpStore, TypeMem, 0, TypeBool, "addr2", "v", "store1"),
Valu("store3", OpStore, TypeMem, 1, nil, "addr1", "v", "store2"), Valu("store3", OpStore, TypeMem, 0, TypeBool, "addr1", "v", "store2"),
Valu("store4", OpStore, TypeMem, 1, nil, "addr3", "v", "store3"), Valu("store4", OpStore, TypeMem, 0, TypeBool, "addr3", "v", "store3"),
Goto("exit")), Goto("exit")),
Bloc("exit", Bloc("exit",
Exit("store3"))) Exit("store3")))
...@@ -54,7 +54,7 @@ func TestDeadStorePhi(t *testing.T) { ...@@ -54,7 +54,7 @@ func TestDeadStorePhi(t *testing.T) {
Goto("loop")), Goto("loop")),
Bloc("loop", Bloc("loop",
Valu("phi", OpPhi, TypeMem, 0, nil, "start", "store"), Valu("phi", OpPhi, TypeMem, 0, nil, "start", "store"),
Valu("store", OpStore, TypeMem, 1, nil, "addr", "v", "phi"), Valu("store", OpStore, TypeMem, 0, TypeBool, "addr", "v", "phi"),
If("v", "loop", "exit")), If("v", "loop", "exit")),
Bloc("exit", Bloc("exit",
Exit("store"))) Exit("store")))
...@@ -79,8 +79,8 @@ func TestDeadStoreTypes(t *testing.T) { ...@@ -79,8 +79,8 @@ func TestDeadStoreTypes(t *testing.T) {
Valu("v", OpConstBool, TypeBool, 1, nil), Valu("v", OpConstBool, TypeBool, 1, nil),
Valu("addr1", OpAddr, t1, 0, nil, "sb"), Valu("addr1", OpAddr, t1, 0, nil, "sb"),
Valu("addr2", OpAddr, t2, 0, nil, "sb"), Valu("addr2", OpAddr, t2, 0, nil, "sb"),
Valu("store1", OpStore, TypeMem, 1, nil, "addr1", "v", "start"), Valu("store1", OpStore, TypeMem, 0, TypeBool, "addr1", "v", "start"),
Valu("store2", OpStore, TypeMem, 1, nil, "addr2", "v", "store1"), Valu("store2", OpStore, TypeMem, 0, TypeBool, "addr2", "v", "store1"),
Goto("exit")), Goto("exit")),
Bloc("exit", Bloc("exit",
Exit("store2"))) Exit("store2")))
...@@ -108,8 +108,8 @@ func TestDeadStoreUnsafe(t *testing.T) { ...@@ -108,8 +108,8 @@ func TestDeadStoreUnsafe(t *testing.T) {
Valu("sb", OpSB, TypeInvalid, 0, nil), Valu("sb", OpSB, TypeInvalid, 0, nil),
Valu("v", OpConstBool, TypeBool, 1, nil), Valu("v", OpConstBool, TypeBool, 1, nil),
Valu("addr1", OpAddr, ptrType, 0, nil, "sb"), Valu("addr1", OpAddr, ptrType, 0, nil, "sb"),
Valu("store1", OpStore, TypeMem, 8, nil, "addr1", "v", "start"), // store 8 bytes Valu("store1", OpStore, TypeMem, 0, TypeInt64, "addr1", "v", "start"), // store 8 bytes
Valu("store2", OpStore, TypeMem, 1, nil, "addr1", "v", "store1"), // store 1 byte Valu("store2", OpStore, TypeMem, 0, TypeBool, "addr1", "v", "store1"), // store 1 byte
Goto("exit")), Goto("exit")),
Bloc("exit", Bloc("exit",
Exit("store2"))) Exit("store2")))
......
...@@ -352,6 +352,21 @@ func (b *Block) NewValue3I(pos src.XPos, op Op, t Type, auxint int64, arg0, arg1 ...@@ -352,6 +352,21 @@ func (b *Block) NewValue3I(pos src.XPos, op Op, t Type, auxint int64, arg0, arg1
return v return v
} }
// NewValue3A returns a new value in the block with three argument and an aux value.
func (b *Block) NewValue3A(pos src.XPos, op Op, t Type, aux interface{}, arg0, arg1, arg2 *Value) *Value {
v := b.Func.newValue(op, t, b, pos)
v.AuxInt = 0
v.Aux = aux
v.Args = v.argstorage[:3]
v.argstorage[0] = arg0
v.argstorage[1] = arg1
v.argstorage[2] = arg2
arg0.Uses++
arg1.Uses++
arg2.Uses++
return v
}
// NewValue4 returns a new value in the block with four arguments and zero aux values. // NewValue4 returns a new value in the block with four arguments and zero aux values.
func (b *Block) NewValue4(pos src.XPos, op Op, t Type, arg0, arg1, arg2, arg3 *Value) *Value { func (b *Block) NewValue4(pos src.XPos, op Op, t Type, arg0, arg1, arg2, arg3 *Value) *Value {
v := b.Func.newValue(op, t, b, pos) v := b.Func.newValue(op, t, b, pos)
......
...@@ -256,47 +256,47 @@ ...@@ -256,47 +256,47 @@
// Lowering stores // Lowering stores
// These more-specific FP versions of Store pattern should come first. // These more-specific FP versions of Store pattern should come first.
(Store [8] ptr val mem) && is64BitFloat(val.Type) -> (MOVSDstore ptr val mem) (Store {t} ptr val mem) && t.(Type).Size() == 8 && is64BitFloat(val.Type) -> (MOVSDstore ptr val mem)
(Store [4] ptr val mem) && is32BitFloat(val.Type) -> (MOVSSstore ptr val mem) (Store {t} ptr val mem) && t.(Type).Size() == 4 && is32BitFloat(val.Type) -> (MOVSSstore ptr val mem)
(Store [4] ptr val mem) -> (MOVLstore ptr val mem) (Store {t} ptr val mem) && t.(Type).Size() == 4 -> (MOVLstore ptr val mem)
(Store [2] ptr val mem) -> (MOVWstore ptr val mem) (Store {t} ptr val mem) && t.(Type).Size() == 2 -> (MOVWstore ptr val mem)
(Store [1] ptr val mem) -> (MOVBstore ptr val mem) (Store {t} ptr val mem) && t.(Type).Size() == 1 -> (MOVBstore ptr val mem)
// Lowering moves // Lowering moves
(Move [s] _ _ mem) && SizeAndAlign(s).Size() == 0 -> mem (Move [0] _ _ mem) -> mem
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 1 -> (MOVBstore dst (MOVBload src mem) mem) (Move [1] dst src mem) -> (MOVBstore dst (MOVBload src mem) mem)
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 2 -> (MOVWstore dst (MOVWload src mem) mem) (Move [2] dst src mem) -> (MOVWstore dst (MOVWload src mem) mem)
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 4 -> (MOVLstore dst (MOVLload src mem) mem) (Move [4] dst src mem) -> (MOVLstore dst (MOVLload src mem) mem)
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 3 -> (Move [3] dst src mem) ->
(MOVBstore [2] dst (MOVBload [2] src mem) (MOVBstore [2] dst (MOVBload [2] src mem)
(MOVWstore dst (MOVWload src mem) mem)) (MOVWstore dst (MOVWload src mem) mem))
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 5 -> (Move [5] dst src mem) ->
(MOVBstore [4] dst (MOVBload [4] src mem) (MOVBstore [4] dst (MOVBload [4] src mem)
(MOVLstore dst (MOVLload src mem) mem)) (MOVLstore dst (MOVLload src mem) mem))
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 6 -> (Move [6] dst src mem) ->
(MOVWstore [4] dst (MOVWload [4] src mem) (MOVWstore [4] dst (MOVWload [4] src mem)
(MOVLstore dst (MOVLload src mem) mem)) (MOVLstore dst (MOVLload src mem) mem))
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 7 -> (Move [7] dst src mem) ->
(MOVLstore [3] dst (MOVLload [3] src mem) (MOVLstore [3] dst (MOVLload [3] src mem)
(MOVLstore dst (MOVLload src mem) mem)) (MOVLstore dst (MOVLload src mem) mem))
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 8 -> (Move [8] dst src mem) ->
(MOVLstore [4] dst (MOVLload [4] src mem) (MOVLstore [4] dst (MOVLload [4] src mem)
(MOVLstore dst (MOVLload src mem) mem)) (MOVLstore dst (MOVLload src mem) mem))
// Adjust moves to be a multiple of 4 bytes. // Adjust moves to be a multiple of 4 bytes.
(Move [s] dst src mem) (Move [s] dst src mem)
&& SizeAndAlign(s).Size() > 8 && SizeAndAlign(s).Size()%4 != 0 -> && s > 8 && s%4 != 0 ->
(Move [SizeAndAlign(s).Size()-SizeAndAlign(s).Size()%4] (Move [s-s%4]
(ADDLconst <dst.Type> dst [SizeAndAlign(s).Size()%4]) (ADDLconst <dst.Type> dst [s%4])
(ADDLconst <src.Type> src [SizeAndAlign(s).Size()%4]) (ADDLconst <src.Type> src [s%4])
(MOVLstore dst (MOVLload src mem) mem)) (MOVLstore dst (MOVLload src mem) mem))
// Medium copying uses a duff device. // Medium copying uses a duff device.
(Move [s] dst src mem) (Move [s] dst src mem)
&& SizeAndAlign(s).Size() > 8 && SizeAndAlign(s).Size() <= 4*128 && SizeAndAlign(s).Size()%4 == 0 && s > 8 && s <= 4*128 && s%4 == 0
&& !config.noDuffDevice -> && !config.noDuffDevice ->
(DUFFCOPY [10*(128-SizeAndAlign(s).Size()/4)] dst src mem) (DUFFCOPY [10*(128-s/4)] dst src mem)
// 10 and 128 are magic constants. 10 is the number of bytes to encode: // 10 and 128 are magic constants. 10 is the number of bytes to encode:
// MOVL (SI), CX // MOVL (SI), CX
// ADDL $4, SI // ADDL $4, SI
...@@ -305,42 +305,42 @@ ...@@ -305,42 +305,42 @@
// and 128 is the number of such blocks. See src/runtime/duff_386.s:duffcopy. // and 128 is the number of such blocks. See src/runtime/duff_386.s:duffcopy.
// Large copying uses REP MOVSL. // Large copying uses REP MOVSL.
(Move [s] dst src mem) && (SizeAndAlign(s).Size() > 4*128 || config.noDuffDevice) && SizeAndAlign(s).Size()%4 == 0 -> (Move [s] dst src mem) && (s > 4*128 || config.noDuffDevice) && s%4 == 0 ->
(REPMOVSL dst src (MOVLconst [SizeAndAlign(s).Size()/4]) mem) (REPMOVSL dst src (MOVLconst [s/4]) mem)
// Lowering Zero instructions // Lowering Zero instructions
(Zero [s] _ mem) && SizeAndAlign(s).Size() == 0 -> mem (Zero [0] _ mem) -> mem
(Zero [s] destptr mem) && SizeAndAlign(s).Size() == 1 -> (MOVBstoreconst [0] destptr mem) (Zero [1] destptr mem) -> (MOVBstoreconst [0] destptr mem)
(Zero [s] destptr mem) && SizeAndAlign(s).Size() == 2 -> (MOVWstoreconst [0] destptr mem) (Zero [2] destptr mem) -> (MOVWstoreconst [0] destptr mem)
(Zero [s] destptr mem) && SizeAndAlign(s).Size() == 4 -> (MOVLstoreconst [0] destptr mem) (Zero [4] destptr mem) -> (MOVLstoreconst [0] destptr mem)
(Zero [s] destptr mem) && SizeAndAlign(s).Size() == 3 -> (Zero [3] destptr mem) ->
(MOVBstoreconst [makeValAndOff(0,2)] destptr (MOVBstoreconst [makeValAndOff(0,2)] destptr
(MOVWstoreconst [0] destptr mem)) (MOVWstoreconst [0] destptr mem))
(Zero [s] destptr mem) && SizeAndAlign(s).Size() == 5 -> (Zero [5] destptr mem) ->
(MOVBstoreconst [makeValAndOff(0,4)] destptr (MOVBstoreconst [makeValAndOff(0,4)] destptr
(MOVLstoreconst [0] destptr mem)) (MOVLstoreconst [0] destptr mem))
(Zero [s] destptr mem) && SizeAndAlign(s).Size() == 6 -> (Zero [6] destptr mem) ->
(MOVWstoreconst [makeValAndOff(0,4)] destptr (MOVWstoreconst [makeValAndOff(0,4)] destptr
(MOVLstoreconst [0] destptr mem)) (MOVLstoreconst [0] destptr mem))
(Zero [s] destptr mem) && SizeAndAlign(s).Size() == 7 -> (Zero [7] destptr mem) ->
(MOVLstoreconst [makeValAndOff(0,3)] destptr (MOVLstoreconst [makeValAndOff(0,3)] destptr
(MOVLstoreconst [0] destptr mem)) (MOVLstoreconst [0] destptr mem))
// Strip off any fractional word zeroing. // Strip off any fractional word zeroing.
(Zero [s] destptr mem) && SizeAndAlign(s).Size()%4 != 0 && SizeAndAlign(s).Size() > 4 -> (Zero [s] destptr mem) && s%4 != 0 && s > 4 ->
(Zero [SizeAndAlign(s).Size()-SizeAndAlign(s).Size()%4] (ADDLconst destptr [SizeAndAlign(s).Size()%4]) (Zero [s-s%4] (ADDLconst destptr [s%4])
(MOVLstoreconst [0] destptr mem)) (MOVLstoreconst [0] destptr mem))
// Zero small numbers of words directly. // Zero small numbers of words directly.
(Zero [s] destptr mem) && SizeAndAlign(s).Size() == 8 -> (Zero [8] destptr mem) ->
(MOVLstoreconst [makeValAndOff(0,4)] destptr (MOVLstoreconst [makeValAndOff(0,4)] destptr
(MOVLstoreconst [0] destptr mem)) (MOVLstoreconst [0] destptr mem))
(Zero [s] destptr mem) && SizeAndAlign(s).Size() == 12 -> (Zero [12] destptr mem) ->
(MOVLstoreconst [makeValAndOff(0,8)] destptr (MOVLstoreconst [makeValAndOff(0,8)] destptr
(MOVLstoreconst [makeValAndOff(0,4)] destptr (MOVLstoreconst [makeValAndOff(0,4)] destptr
(MOVLstoreconst [0] destptr mem))) (MOVLstoreconst [0] destptr mem)))
(Zero [s] destptr mem) && SizeAndAlign(s).Size() == 16 -> (Zero [16] destptr mem) ->
(MOVLstoreconst [makeValAndOff(0,12)] destptr (MOVLstoreconst [makeValAndOff(0,12)] destptr
(MOVLstoreconst [makeValAndOff(0,8)] destptr (MOVLstoreconst [makeValAndOff(0,8)] destptr
(MOVLstoreconst [makeValAndOff(0,4)] destptr (MOVLstoreconst [makeValAndOff(0,4)] destptr
...@@ -348,20 +348,18 @@ ...@@ -348,20 +348,18 @@
// Medium zeroing uses a duff device. // Medium zeroing uses a duff device.
(Zero [s] destptr mem) (Zero [s] destptr mem)
&& SizeAndAlign(s).Size() > 16 && s > 16 && s <= 4*128 && s%4 == 0
&& SizeAndAlign(s).Size() <= 4*128
&& SizeAndAlign(s).Size()%4 == 0
&& !config.noDuffDevice -> && !config.noDuffDevice ->
(DUFFZERO [1*(128-SizeAndAlign(s).Size()/4)] destptr (MOVLconst [0]) mem) (DUFFZERO [1*(128-s/4)] destptr (MOVLconst [0]) mem)
// 1 and 128 are magic constants. 1 is the number of bytes to encode STOSL. // 1 and 128 are magic constants. 1 is the number of bytes to encode STOSL.
// 128 is the number of STOSL instructions in duffzero. // 128 is the number of STOSL instructions in duffzero.
// See src/runtime/duff_386.s:duffzero. // See src/runtime/duff_386.s:duffzero.
// Large zeroing uses REP STOSQ. // Large zeroing uses REP STOSQ.
(Zero [s] destptr mem) (Zero [s] destptr mem)
&& (SizeAndAlign(s).Size() > 4*128 || (config.noDuffDevice && SizeAndAlign(s).Size() > 16)) && (s > 4*128 || (config.noDuffDevice && s > 16))
&& SizeAndAlign(s).Size()%4 == 0 -> && s%4 == 0 ->
(REPSTOSL destptr (MOVLconst [SizeAndAlign(s).Size()/4]) (MOVLconst [0]) mem) (REPSTOSL destptr (MOVLconst [s/4]) (MOVLconst [0]) mem)
// Lowering constants // Lowering constants
(Const8 [val]) -> (MOVLconst [val]) (Const8 [val]) -> (MOVLconst [val])
......
...@@ -297,56 +297,56 @@ ...@@ -297,56 +297,56 @@
// Lowering stores // Lowering stores
// These more-specific FP versions of Store pattern should come first. // These more-specific FP versions of Store pattern should come first.
(Store [8] ptr val mem) && is64BitFloat(val.Type) -> (MOVSDstore ptr val mem) (Store {t} ptr val mem) && t.(Type).Size() == 8 && is64BitFloat(val.Type) -> (MOVSDstore ptr val mem)
(Store [4] ptr val mem) && is32BitFloat(val.Type) -> (MOVSSstore ptr val mem) (Store {t} ptr val mem) && t.(Type).Size() == 4 && is32BitFloat(val.Type) -> (MOVSSstore ptr val mem)
(Store [8] ptr val mem) -> (MOVQstore ptr val mem) (Store {t} ptr val mem) && t.(Type).Size() == 8 -> (MOVQstore ptr val mem)
(Store [4] ptr val mem) -> (MOVLstore ptr val mem) (Store {t} ptr val mem) && t.(Type).Size() == 4 -> (MOVLstore ptr val mem)
(Store [2] ptr val mem) -> (MOVWstore ptr val mem) (Store {t} ptr val mem) && t.(Type).Size() == 2 -> (MOVWstore ptr val mem)
(Store [1] ptr val mem) -> (MOVBstore ptr val mem) (Store {t} ptr val mem) && t.(Type).Size() == 1 -> (MOVBstore ptr val mem)
// Lowering moves // Lowering moves
(Move [s] _ _ mem) && SizeAndAlign(s).Size() == 0 -> mem (Move [0] _ _ mem) -> mem
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 1 -> (MOVBstore dst (MOVBload src mem) mem) (Move [1] dst src mem) -> (MOVBstore dst (MOVBload src mem) mem)
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 2 -> (MOVWstore dst (MOVWload src mem) mem) (Move [2] dst src mem) -> (MOVWstore dst (MOVWload src mem) mem)
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 4 -> (MOVLstore dst (MOVLload src mem) mem) (Move [4] dst src mem) -> (MOVLstore dst (MOVLload src mem) mem)
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 8 -> (MOVQstore dst (MOVQload src mem) mem) (Move [8] dst src mem) -> (MOVQstore dst (MOVQload src mem) mem)
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 16 -> (MOVOstore dst (MOVOload src mem) mem) (Move [16] dst src mem) -> (MOVOstore dst (MOVOload src mem) mem)
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 3 -> (Move [3] dst src mem) ->
(MOVBstore [2] dst (MOVBload [2] src mem) (MOVBstore [2] dst (MOVBload [2] src mem)
(MOVWstore dst (MOVWload src mem) mem)) (MOVWstore dst (MOVWload src mem) mem))
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 5 -> (Move [5] dst src mem) ->
(MOVBstore [4] dst (MOVBload [4] src mem) (MOVBstore [4] dst (MOVBload [4] src mem)
(MOVLstore dst (MOVLload src mem) mem)) (MOVLstore dst (MOVLload src mem) mem))
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 6 -> (Move [6] dst src mem) ->
(MOVWstore [4] dst (MOVWload [4] src mem) (MOVWstore [4] dst (MOVWload [4] src mem)
(MOVLstore dst (MOVLload src mem) mem)) (MOVLstore dst (MOVLload src mem) mem))
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 7 -> (Move [7] dst src mem) ->
(MOVLstore [3] dst (MOVLload [3] src mem) (MOVLstore [3] dst (MOVLload [3] src mem)
(MOVLstore dst (MOVLload src mem) mem)) (MOVLstore dst (MOVLload src mem) mem))
(Move [s] dst src mem) && SizeAndAlign(s).Size() > 8 && SizeAndAlign(s).Size() < 16 -> (Move [s] dst src mem) && s > 8 && s < 16 ->
(MOVQstore [SizeAndAlign(s).Size()-8] dst (MOVQload [SizeAndAlign(s).Size()-8] src mem) (MOVQstore [s-8] dst (MOVQload [s-8] src mem)
(MOVQstore dst (MOVQload src mem) mem)) (MOVQstore dst (MOVQload src mem) mem))
// Adjust moves to be a multiple of 16 bytes. // Adjust moves to be a multiple of 16 bytes.
(Move [s] dst src mem) (Move [s] dst src mem)
&& SizeAndAlign(s).Size() > 16 && SizeAndAlign(s).Size()%16 != 0 && SizeAndAlign(s).Size()%16 <= 8 -> && s > 16 && s%16 != 0 && s%16 <= 8 ->
(Move [SizeAndAlign(s).Size()-SizeAndAlign(s).Size()%16] (Move [s-s%16]
(OffPtr <dst.Type> dst [SizeAndAlign(s).Size()%16]) (OffPtr <dst.Type> dst [s%16])
(OffPtr <src.Type> src [SizeAndAlign(s).Size()%16]) (OffPtr <src.Type> src [s%16])
(MOVQstore dst (MOVQload src mem) mem)) (MOVQstore dst (MOVQload src mem) mem))
(Move [s] dst src mem) (Move [s] dst src mem)
&& SizeAndAlign(s).Size() > 16 && SizeAndAlign(s).Size()%16 != 0 && SizeAndAlign(s).Size()%16 > 8 -> && s > 16 && s%16 != 0 && s%16 > 8 ->
(Move [SizeAndAlign(s).Size()-SizeAndAlign(s).Size()%16] (Move [s-s%16]
(OffPtr <dst.Type> dst [SizeAndAlign(s).Size()%16]) (OffPtr <dst.Type> dst [s%16])
(OffPtr <src.Type> src [SizeAndAlign(s).Size()%16]) (OffPtr <src.Type> src [s%16])
(MOVOstore dst (MOVOload src mem) mem)) (MOVOstore dst (MOVOload src mem) mem))
// Medium copying uses a duff device. // Medium copying uses a duff device.
(Move [s] dst src mem) (Move [s] dst src mem)
&& SizeAndAlign(s).Size() >= 32 && SizeAndAlign(s).Size() <= 16*64 && SizeAndAlign(s).Size()%16 == 0 && s >= 32 && s <= 16*64 && s%16 == 0
&& !config.noDuffDevice -> && !config.noDuffDevice ->
(DUFFCOPY [14*(64-SizeAndAlign(s).Size()/16)] dst src mem) (DUFFCOPY [14*(64-s/16)] dst src mem)
// 14 and 64 are magic constants. 14 is the number of bytes to encode: // 14 and 64 are magic constants. 14 is the number of bytes to encode:
// MOVUPS (SI), X0 // MOVUPS (SI), X0
// ADDQ $16, SI // ADDQ $16, SI
...@@ -355,43 +355,43 @@ ...@@ -355,43 +355,43 @@
// and 64 is the number of such blocks. See src/runtime/duff_amd64.s:duffcopy. // and 64 is the number of such blocks. See src/runtime/duff_amd64.s:duffcopy.
// Large copying uses REP MOVSQ. // Large copying uses REP MOVSQ.
(Move [s] dst src mem) && (SizeAndAlign(s).Size() > 16*64 || config.noDuffDevice) && SizeAndAlign(s).Size()%8 == 0 -> (Move [s] dst src mem) && (s > 16*64 || config.noDuffDevice) && s%8 == 0 ->
(REPMOVSQ dst src (MOVQconst [SizeAndAlign(s).Size()/8]) mem) (REPMOVSQ dst src (MOVQconst [s/8]) mem)
// Lowering Zero instructions // Lowering Zero instructions
(Zero [s] _ mem) && SizeAndAlign(s).Size() == 0 -> mem (Zero [0] _ mem) -> mem
(Zero [s] destptr mem) && SizeAndAlign(s).Size() == 1 -> (MOVBstoreconst [0] destptr mem) (Zero [1] destptr mem) -> (MOVBstoreconst [0] destptr mem)
(Zero [s] destptr mem) && SizeAndAlign(s).Size() == 2 -> (MOVWstoreconst [0] destptr mem) (Zero [2] destptr mem) -> (MOVWstoreconst [0] destptr mem)
(Zero [s] destptr mem) && SizeAndAlign(s).Size() == 4 -> (MOVLstoreconst [0] destptr mem) (Zero [4] destptr mem) -> (MOVLstoreconst [0] destptr mem)
(Zero [s] destptr mem) && SizeAndAlign(s).Size() == 8 -> (MOVQstoreconst [0] destptr mem) (Zero [8] destptr mem) -> (MOVQstoreconst [0] destptr mem)
(Zero [s] destptr mem) && SizeAndAlign(s).Size() == 3 -> (Zero [3] destptr mem) ->
(MOVBstoreconst [makeValAndOff(0,2)] destptr (MOVBstoreconst [makeValAndOff(0,2)] destptr
(MOVWstoreconst [0] destptr mem)) (MOVWstoreconst [0] destptr mem))
(Zero [s] destptr mem) && SizeAndAlign(s).Size() == 5 -> (Zero [5] destptr mem) ->
(MOVBstoreconst [makeValAndOff(0,4)] destptr (MOVBstoreconst [makeValAndOff(0,4)] destptr
(MOVLstoreconst [0] destptr mem)) (MOVLstoreconst [0] destptr mem))
(Zero [s] destptr mem) && SizeAndAlign(s).Size() == 6 -> (Zero [6] destptr mem) ->
(MOVWstoreconst [makeValAndOff(0,4)] destptr (MOVWstoreconst [makeValAndOff(0,4)] destptr
(MOVLstoreconst [0] destptr mem)) (MOVLstoreconst [0] destptr mem))
(Zero [s] destptr mem) && SizeAndAlign(s).Size() == 7 -> (Zero [7] destptr mem) ->
(MOVLstoreconst [makeValAndOff(0,3)] destptr (MOVLstoreconst [makeValAndOff(0,3)] destptr
(MOVLstoreconst [0] destptr mem)) (MOVLstoreconst [0] destptr mem))
// Strip off any fractional word zeroing. // Strip off any fractional word zeroing.
(Zero [s] destptr mem) && SizeAndAlign(s).Size()%8 != 0 && SizeAndAlign(s).Size() > 8 -> (Zero [s] destptr mem) && s%8 != 0 && s > 8 ->
(Zero [SizeAndAlign(s).Size()-SizeAndAlign(s).Size()%8] (OffPtr <destptr.Type> destptr [SizeAndAlign(s).Size()%8]) (Zero [s-s%8] (OffPtr <destptr.Type> destptr [s%8])
(MOVQstoreconst [0] destptr mem)) (MOVQstoreconst [0] destptr mem))
// Zero small numbers of words directly. // Zero small numbers of words directly.
(Zero [s] destptr mem) && SizeAndAlign(s).Size() == 16 -> (Zero [16] destptr mem) ->
(MOVQstoreconst [makeValAndOff(0,8)] destptr (MOVQstoreconst [makeValAndOff(0,8)] destptr
(MOVQstoreconst [0] destptr mem)) (MOVQstoreconst [0] destptr mem))
(Zero [s] destptr mem) && SizeAndAlign(s).Size() == 24 -> (Zero [24] destptr mem) ->
(MOVQstoreconst [makeValAndOff(0,16)] destptr (MOVQstoreconst [makeValAndOff(0,16)] destptr
(MOVQstoreconst [makeValAndOff(0,8)] destptr (MOVQstoreconst [makeValAndOff(0,8)] destptr
(MOVQstoreconst [0] destptr mem))) (MOVQstoreconst [0] destptr mem)))
(Zero [s] destptr mem) && SizeAndAlign(s).Size() == 32 -> (Zero [32] destptr mem) ->
(MOVQstoreconst [makeValAndOff(0,24)] destptr (MOVQstoreconst [makeValAndOff(0,24)] destptr
(MOVQstoreconst [makeValAndOff(0,16)] destptr (MOVQstoreconst [makeValAndOff(0,16)] destptr
(MOVQstoreconst [makeValAndOff(0,8)] destptr (MOVQstoreconst [makeValAndOff(0,8)] destptr
...@@ -399,18 +399,18 @@ ...@@ -399,18 +399,18 @@
// Medium zeroing uses a duff device. // Medium zeroing uses a duff device.
(Zero [s] destptr mem) (Zero [s] destptr mem)
&& SizeAndAlign(s).Size() <= 1024 && SizeAndAlign(s).Size()%8 == 0 && SizeAndAlign(s).Size()%16 != 0 && s <= 1024 && s%8 == 0 && s%16 != 0
&& !config.noDuffDevice -> && !config.noDuffDevice ->
(Zero [SizeAndAlign(s).Size()-8] (OffPtr <destptr.Type> [8] destptr) (MOVQstore destptr (MOVQconst [0]) mem)) (Zero [s-8] (OffPtr <destptr.Type> [8] destptr) (MOVQstore destptr (MOVQconst [0]) mem))
(Zero [s] destptr mem) (Zero [s] destptr mem)
&& SizeAndAlign(s).Size() <= 1024 && SizeAndAlign(s).Size()%16 == 0 && !config.noDuffDevice -> && s <= 1024 && s%16 == 0 && !config.noDuffDevice ->
(DUFFZERO [SizeAndAlign(s).Size()] destptr (MOVOconst [0]) mem) (DUFFZERO [s] destptr (MOVOconst [0]) mem)
// Large zeroing uses REP STOSQ. // Large zeroing uses REP STOSQ.
(Zero [s] destptr mem) (Zero [s] destptr mem)
&& (SizeAndAlign(s).Size() > 1024 || (config.noDuffDevice && SizeAndAlign(s).Size() > 32)) && (s > 1024 || (config.noDuffDevice && s > 32))
&& SizeAndAlign(s).Size()%8 == 0 -> && s%8 == 0 ->
(REPSTOSQ destptr (MOVQconst [SizeAndAlign(s).Size()/8]) (MOVQconst [0]) mem) (REPSTOSQ destptr (MOVQconst [s/8]) (MOVQconst [0]) mem)
// Lowering constants // Lowering constants
(Const8 [val]) -> (MOVLconst [val]) (Const8 [val]) -> (MOVLconst [val])
......
...@@ -290,90 +290,90 @@ ...@@ -290,90 +290,90 @@
(Load <t> ptr mem) && is64BitFloat(t) -> (MOVDload ptr mem) (Load <t> ptr mem) && is64BitFloat(t) -> (MOVDload ptr mem)
// stores // stores
(Store [1] ptr val mem) -> (MOVBstore ptr val mem) (Store {t} ptr val mem) && t.(Type).Size() == 1 -> (MOVBstore ptr val mem)
(Store [2] ptr val mem) -> (MOVHstore ptr val mem) (Store {t} ptr val mem) && t.(Type).Size() == 2 -> (MOVHstore ptr val mem)
(Store [4] ptr val mem) && !is32BitFloat(val.Type) -> (MOVWstore ptr val mem) (Store {t} ptr val mem) && t.(Type).Size() == 4 && !is32BitFloat(val.Type) -> (MOVWstore ptr val mem)
(Store [4] ptr val mem) && is32BitFloat(val.Type) -> (MOVFstore ptr val mem) (Store {t} ptr val mem) && t.(Type).Size() == 4 && is32BitFloat(val.Type) -> (MOVFstore ptr val mem)
(Store [8] ptr val mem) && is64BitFloat(val.Type) -> (MOVDstore ptr val mem) (Store {t} ptr val mem) && t.(Type).Size() == 8 && is64BitFloat(val.Type) -> (MOVDstore ptr val mem)
// zero instructions // zero instructions
(Zero [s] _ mem) && SizeAndAlign(s).Size() == 0 -> mem (Zero [0] _ mem) -> mem
(Zero [s] ptr mem) && SizeAndAlign(s).Size() == 1 -> (MOVBstore ptr (MOVWconst [0]) mem) (Zero [1] ptr mem) -> (MOVBstore ptr (MOVWconst [0]) mem)
(Zero [s] ptr mem) && SizeAndAlign(s).Size() == 2 && SizeAndAlign(s).Align()%2 == 0 -> (Zero [2] {t} ptr mem) && t.(Type).Alignment()%2 == 0 ->
(MOVHstore ptr (MOVWconst [0]) mem) (MOVHstore ptr (MOVWconst [0]) mem)
(Zero [s] ptr mem) && SizeAndAlign(s).Size() == 2 -> (Zero [2] ptr mem) ->
(MOVBstore [1] ptr (MOVWconst [0]) (MOVBstore [1] ptr (MOVWconst [0])
(MOVBstore [0] ptr (MOVWconst [0]) mem)) (MOVBstore [0] ptr (MOVWconst [0]) mem))
(Zero [s] ptr mem) && SizeAndAlign(s).Size() == 4 && SizeAndAlign(s).Align()%4 == 0 -> (Zero [4] {t} ptr mem) && t.(Type).Alignment()%4 == 0 ->
(MOVWstore ptr (MOVWconst [0]) mem) (MOVWstore ptr (MOVWconst [0]) mem)
(Zero [s] ptr mem) && SizeAndAlign(s).Size() == 4 && SizeAndAlign(s).Align()%2 == 0 -> (Zero [4] {t} ptr mem) && t.(Type).Alignment()%2 == 0 ->
(MOVHstore [2] ptr (MOVWconst [0]) (MOVHstore [2] ptr (MOVWconst [0])
(MOVHstore [0] ptr (MOVWconst [0]) mem)) (MOVHstore [0] ptr (MOVWconst [0]) mem))
(Zero [s] ptr mem) && SizeAndAlign(s).Size() == 4 -> (Zero [4] ptr mem) ->
(MOVBstore [3] ptr (MOVWconst [0]) (MOVBstore [3] ptr (MOVWconst [0])
(MOVBstore [2] ptr (MOVWconst [0]) (MOVBstore [2] ptr (MOVWconst [0])
(MOVBstore [1] ptr (MOVWconst [0]) (MOVBstore [1] ptr (MOVWconst [0])
(MOVBstore [0] ptr (MOVWconst [0]) mem)))) (MOVBstore [0] ptr (MOVWconst [0]) mem))))
(Zero [s] ptr mem) && SizeAndAlign(s).Size() == 3 -> (Zero [3] ptr mem) ->
(MOVBstore [2] ptr (MOVWconst [0]) (MOVBstore [2] ptr (MOVWconst [0])
(MOVBstore [1] ptr (MOVWconst [0]) (MOVBstore [1] ptr (MOVWconst [0])
(MOVBstore [0] ptr (MOVWconst [0]) mem))) (MOVBstore [0] ptr (MOVWconst [0]) mem)))
// Medium zeroing uses a duff device // Medium zeroing uses a duff device
// 4 and 128 are magic constants, see runtime/mkduff.go // 4 and 128 are magic constants, see runtime/mkduff.go
(Zero [s] ptr mem) (Zero [s] {t} ptr mem)
&& SizeAndAlign(s).Size()%4 == 0 && SizeAndAlign(s).Size() > 4 && SizeAndAlign(s).Size() <= 512 && s%4 == 0 && s > 4 && s <= 512
&& SizeAndAlign(s).Align()%4 == 0 && !config.noDuffDevice -> && t.(Type).Alignment()%4 == 0 && !config.noDuffDevice ->
(DUFFZERO [4 * (128 - int64(SizeAndAlign(s).Size()/4))] ptr (MOVWconst [0]) mem) (DUFFZERO [4 * (128 - int64(s/4))] ptr (MOVWconst [0]) mem)
// Large zeroing uses a loop // Large zeroing uses a loop
(Zero [s] ptr mem) (Zero [s] {t} ptr mem)
&& (SizeAndAlign(s).Size() > 512 || config.noDuffDevice) || SizeAndAlign(s).Align()%4 != 0 -> && (s > 512 || config.noDuffDevice) || t.(Type).Alignment()%4 != 0 ->
(LoweredZero [SizeAndAlign(s).Align()] (LoweredZero [t.(Type).Alignment()]
ptr ptr
(ADDconst <ptr.Type> ptr [SizeAndAlign(s).Size()-moveSize(SizeAndAlign(s).Align(), config)]) (ADDconst <ptr.Type> ptr [s-moveSize(t.(Type).Alignment(), config)])
(MOVWconst [0]) (MOVWconst [0])
mem) mem)
// moves // moves
(Move [s] _ _ mem) && SizeAndAlign(s).Size() == 0 -> mem (Move [0] _ _ mem) -> mem
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 1 -> (MOVBstore dst (MOVBUload src mem) mem) (Move [1] dst src mem) -> (MOVBstore dst (MOVBUload src mem) mem)
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 2 && SizeAndAlign(s).Align()%2 == 0 -> (Move [2] {t} dst src mem) && t.(Type).Alignment()%2 == 0 ->
(MOVHstore dst (MOVHUload src mem) mem) (MOVHstore dst (MOVHUload src mem) mem)
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 2 -> (Move [2] dst src mem) ->
(MOVBstore [1] dst (MOVBUload [1] src mem) (MOVBstore [1] dst (MOVBUload [1] src mem)
(MOVBstore dst (MOVBUload src mem) mem)) (MOVBstore dst (MOVBUload src mem) mem))
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 4 && SizeAndAlign(s).Align()%4 == 0 -> (Move [4] {t} dst src mem) && t.(Type).Alignment()%4 == 0 ->
(MOVWstore dst (MOVWload src mem) mem) (MOVWstore dst (MOVWload src mem) mem)
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 4 && SizeAndAlign(s).Align()%2 == 0 -> (Move [4] {t} dst src mem) && t.(Type).Alignment()%2 == 0 ->
(MOVHstore [2] dst (MOVHUload [2] src mem) (MOVHstore [2] dst (MOVHUload [2] src mem)
(MOVHstore dst (MOVHUload src mem) mem)) (MOVHstore dst (MOVHUload src mem) mem))
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 4 -> (Move [4] dst src mem) ->
(MOVBstore [3] dst (MOVBUload [3] src mem) (MOVBstore [3] dst (MOVBUload [3] src mem)
(MOVBstore [2] dst (MOVBUload [2] src mem) (MOVBstore [2] dst (MOVBUload [2] src mem)
(MOVBstore [1] dst (MOVBUload [1] src mem) (MOVBstore [1] dst (MOVBUload [1] src mem)
(MOVBstore dst (MOVBUload src mem) mem)))) (MOVBstore dst (MOVBUload src mem) mem))))
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 3 -> (Move [3] dst src mem) ->
(MOVBstore [2] dst (MOVBUload [2] src mem) (MOVBstore [2] dst (MOVBUload [2] src mem)
(MOVBstore [1] dst (MOVBUload [1] src mem) (MOVBstore [1] dst (MOVBUload [1] src mem)
(MOVBstore dst (MOVBUload src mem) mem))) (MOVBstore dst (MOVBUload src mem) mem)))
// Medium move uses a duff device // Medium move uses a duff device
// 8 and 128 are magic constants, see runtime/mkduff.go // 8 and 128 are magic constants, see runtime/mkduff.go
(Move [s] dst src mem) (Move [s] {t} dst src mem)
&& SizeAndAlign(s).Size()%4 == 0 && SizeAndAlign(s).Size() > 4 && SizeAndAlign(s).Size() <= 512 && s%4 == 0 && s > 4 && s <= 512
&& SizeAndAlign(s).Align()%4 == 0 && !config.noDuffDevice -> && t.(Type).Alignment()%4 == 0 && !config.noDuffDevice ->
(DUFFCOPY [8 * (128 - int64(SizeAndAlign(s).Size()/4))] dst src mem) (DUFFCOPY [8 * (128 - int64(s/4))] dst src mem)
// Large move uses a loop // Large move uses a loop
(Move [s] dst src mem) (Move [s] {t} dst src mem)
&& (SizeAndAlign(s).Size() > 512 || config.noDuffDevice) || SizeAndAlign(s).Align()%4 != 0 -> && (s > 512 || config.noDuffDevice) || t.(Type).Alignment()%4 != 0 ->
(LoweredMove [SizeAndAlign(s).Align()] (LoweredMove [t.(Type).Alignment()]
dst dst
src src
(ADDconst <src.Type> src [SizeAndAlign(s).Size()-moveSize(SizeAndAlign(s).Align(), config)]) (ADDconst <src.Type> src [s-moveSize(t.(Type).Alignment(), config)])
mem) mem)
// calls // calls
......
...@@ -331,117 +331,117 @@ ...@@ -331,117 +331,117 @@
(Load <t> ptr mem) && is64BitFloat(t) -> (FMOVDload ptr mem) (Load <t> ptr mem) && is64BitFloat(t) -> (FMOVDload ptr mem)
// stores // stores
(Store [1] ptr val mem) -> (MOVBstore ptr val mem) (Store {t} ptr val mem) && t.(Type).Size() == 1 -> (MOVBstore ptr val mem)
(Store [2] ptr val mem) -> (MOVHstore ptr val mem) (Store {t} ptr val mem) && t.(Type).Size() == 2 -> (MOVHstore ptr val mem)
(Store [4] ptr val mem) && !is32BitFloat(val.Type) -> (MOVWstore ptr val mem) (Store {t} ptr val mem) && t.(Type).Size() == 4 && !is32BitFloat(val.Type) -> (MOVWstore ptr val mem)
(Store [8] ptr val mem) && !is64BitFloat(val.Type) -> (MOVDstore ptr val mem) (Store {t} ptr val mem) && t.(Type).Size() == 8 && !is64BitFloat(val.Type) -> (MOVDstore ptr val mem)
(Store [4] ptr val mem) && is32BitFloat(val.Type) -> (FMOVSstore ptr val mem) (Store {t} ptr val mem) && t.(Type).Size() == 4 && is32BitFloat(val.Type) -> (FMOVSstore ptr val mem)
(Store [8] ptr val mem) && is64BitFloat(val.Type) -> (FMOVDstore ptr val mem) (Store {t} ptr val mem) && t.(Type).Size() == 8 && is64BitFloat(val.Type) -> (FMOVDstore ptr val mem)
// zeroing // zeroing
(Zero [s] _ mem) && SizeAndAlign(s).Size() == 0 -> mem (Zero [0] _ mem) -> mem
(Zero [s] ptr mem) && SizeAndAlign(s).Size() == 1 -> (MOVBstore ptr (MOVDconst [0]) mem) (Zero [1] ptr mem) -> (MOVBstore ptr (MOVDconst [0]) mem)
(Zero [s] ptr mem) && SizeAndAlign(s).Size() == 2 -> (MOVHstore ptr (MOVDconst [0]) mem) (Zero [2] ptr mem) -> (MOVHstore ptr (MOVDconst [0]) mem)
(Zero [s] ptr mem) && SizeAndAlign(s).Size() == 4 -> (MOVWstore ptr (MOVDconst [0]) mem) (Zero [4] ptr mem) -> (MOVWstore ptr (MOVDconst [0]) mem)
(Zero [s] ptr mem) && SizeAndAlign(s).Size() == 8 -> (MOVDstore ptr (MOVDconst [0]) mem) (Zero [8] ptr mem) -> (MOVDstore ptr (MOVDconst [0]) mem)
(Zero [s] ptr mem) && SizeAndAlign(s).Size() == 3 -> (Zero [3] ptr mem) ->
(MOVBstore [2] ptr (MOVDconst [0]) (MOVBstore [2] ptr (MOVDconst [0])
(MOVHstore ptr (MOVDconst [0]) mem)) (MOVHstore ptr (MOVDconst [0]) mem))
(Zero [s] ptr mem) && SizeAndAlign(s).Size() == 5 -> (Zero [5] ptr mem) ->
(MOVBstore [4] ptr (MOVDconst [0]) (MOVBstore [4] ptr (MOVDconst [0])
(MOVWstore ptr (MOVDconst [0]) mem)) (MOVWstore ptr (MOVDconst [0]) mem))
(Zero [s] ptr mem) && SizeAndAlign(s).Size() == 6 -> (Zero [6] ptr mem) ->
(MOVHstore [4] ptr (MOVDconst [0]) (MOVHstore [4] ptr (MOVDconst [0])
(MOVWstore ptr (MOVDconst [0]) mem)) (MOVWstore ptr (MOVDconst [0]) mem))
(Zero [s] ptr mem) && SizeAndAlign(s).Size() == 7 -> (Zero [7] ptr mem) ->
(MOVBstore [6] ptr (MOVDconst [0]) (MOVBstore [6] ptr (MOVDconst [0])
(MOVHstore [4] ptr (MOVDconst [0]) (MOVHstore [4] ptr (MOVDconst [0])
(MOVWstore ptr (MOVDconst [0]) mem))) (MOVWstore ptr (MOVDconst [0]) mem)))
(Zero [s] ptr mem) && SizeAndAlign(s).Size() == 12 -> (Zero [12] ptr mem) ->
(MOVWstore [8] ptr (MOVDconst [0]) (MOVWstore [8] ptr (MOVDconst [0])
(MOVDstore ptr (MOVDconst [0]) mem)) (MOVDstore ptr (MOVDconst [0]) mem))
(Zero [s] ptr mem) && SizeAndAlign(s).Size() == 16 -> (Zero [16] ptr mem) ->
(MOVDstore [8] ptr (MOVDconst [0]) (MOVDstore [8] ptr (MOVDconst [0])
(MOVDstore ptr (MOVDconst [0]) mem)) (MOVDstore ptr (MOVDconst [0]) mem))
(Zero [s] ptr mem) && SizeAndAlign(s).Size() == 24 -> (Zero [24] ptr mem) ->
(MOVDstore [16] ptr (MOVDconst [0]) (MOVDstore [16] ptr (MOVDconst [0])
(MOVDstore [8] ptr (MOVDconst [0]) (MOVDstore [8] ptr (MOVDconst [0])
(MOVDstore ptr (MOVDconst [0]) mem))) (MOVDstore ptr (MOVDconst [0]) mem)))
// strip off fractional word zeroing // strip off fractional word zeroing
(Zero [s] ptr mem) && SizeAndAlign(s).Size()%8 != 0 && SizeAndAlign(s).Size() > 8 -> (Zero [s] ptr mem) && s%8 != 0 && s > 8 ->
(Zero [MakeSizeAndAlign(SizeAndAlign(s).Size()%8, 1).Int64()] (Zero [s%8]
(OffPtr <ptr.Type> ptr [SizeAndAlign(s).Size()-SizeAndAlign(s).Size()%8]) (OffPtr <ptr.Type> ptr [s-s%8])
(Zero [MakeSizeAndAlign(SizeAndAlign(s).Size()-SizeAndAlign(s).Size()%8, 1).Int64()] ptr mem)) (Zero [s-s%8] ptr mem))
// medium zeroing uses a duff device // medium zeroing uses a duff device
// 4, 8, and 128 are magic constants, see runtime/mkduff.go // 4, 8, and 128 are magic constants, see runtime/mkduff.go
(Zero [s] ptr mem) (Zero [s] ptr mem)
&& SizeAndAlign(s).Size()%8 == 0 && SizeAndAlign(s).Size() > 24 && SizeAndAlign(s).Size() <= 8*128 && s%8 == 0 && s > 24 && s <= 8*128
&& !config.noDuffDevice -> && !config.noDuffDevice ->
(DUFFZERO [4 * (128 - int64(SizeAndAlign(s).Size()/8))] ptr mem) (DUFFZERO [4 * (128 - int64(s/8))] ptr mem)
// large zeroing uses a loop // large zeroing uses a loop
(Zero [s] ptr mem) (Zero [s] ptr mem)
&& SizeAndAlign(s).Size()%8 == 0 && (SizeAndAlign(s).Size() > 8*128 || config.noDuffDevice) -> && s%8 == 0 && (s > 8*128 || config.noDuffDevice) ->
(LoweredZero (LoweredZero
ptr ptr
(ADDconst <ptr.Type> [SizeAndAlign(s).Size()-moveSize(SizeAndAlign(s).Align(), config)] ptr) (ADDconst <ptr.Type> [s-8] ptr)
mem) mem)
// moves // moves
(Move [s] _ _ mem) && SizeAndAlign(s).Size() == 0 -> mem (Move [0] _ _ mem) -> mem
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 1 -> (MOVBstore dst (MOVBUload src mem) mem) (Move [1] dst src mem) -> (MOVBstore dst (MOVBUload src mem) mem)
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 2 -> (MOVHstore dst (MOVHUload src mem) mem) (Move [2] dst src mem) -> (MOVHstore dst (MOVHUload src mem) mem)
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 4 -> (MOVWstore dst (MOVWUload src mem) mem) (Move [4] dst src mem) -> (MOVWstore dst (MOVWUload src mem) mem)
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 8 -> (MOVDstore dst (MOVDload src mem) mem) (Move [8] dst src mem) -> (MOVDstore dst (MOVDload src mem) mem)
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 3 -> (Move [3] dst src mem) ->
(MOVBstore [2] dst (MOVBUload [2] src mem) (MOVBstore [2] dst (MOVBUload [2] src mem)
(MOVHstore dst (MOVHUload src mem) mem)) (MOVHstore dst (MOVHUload src mem) mem))
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 5 -> (Move [5] dst src mem) ->
(MOVBstore [4] dst (MOVBUload [4] src mem) (MOVBstore [4] dst (MOVBUload [4] src mem)
(MOVWstore dst (MOVWUload src mem) mem)) (MOVWstore dst (MOVWUload src mem) mem))
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 6 -> (Move [6] dst src mem) ->
(MOVHstore [4] dst (MOVHUload [4] src mem) (MOVHstore [4] dst (MOVHUload [4] src mem)
(MOVWstore dst (MOVWUload src mem) mem)) (MOVWstore dst (MOVWUload src mem) mem))
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 7 -> (Move [7] dst src mem) ->
(MOVBstore [6] dst (MOVBUload [6] src mem) (MOVBstore [6] dst (MOVBUload [6] src mem)
(MOVHstore [4] dst (MOVHUload [4] src mem) (MOVHstore [4] dst (MOVHUload [4] src mem)
(MOVWstore dst (MOVWUload src mem) mem))) (MOVWstore dst (MOVWUload src mem) mem)))
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 12 -> (Move [12] dst src mem) ->
(MOVWstore [8] dst (MOVWUload [8] src mem) (MOVWstore [8] dst (MOVWUload [8] src mem)
(MOVDstore dst (MOVDload src mem) mem)) (MOVDstore dst (MOVDload src mem) mem))
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 16 -> (Move [16] dst src mem) ->
(MOVDstore [8] dst (MOVDload [8] src mem) (MOVDstore [8] dst (MOVDload [8] src mem)
(MOVDstore dst (MOVDload src mem) mem)) (MOVDstore dst (MOVDload src mem) mem))
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 24 -> (Move [24] dst src mem) ->
(MOVDstore [16] dst (MOVDload [16] src mem) (MOVDstore [16] dst (MOVDload [16] src mem)
(MOVDstore [8] dst (MOVDload [8] src mem) (MOVDstore [8] dst (MOVDload [8] src mem)
(MOVDstore dst (MOVDload src mem) mem))) (MOVDstore dst (MOVDload src mem) mem)))
// strip off fractional word move // strip off fractional word move
(Move [s] dst src mem) && SizeAndAlign(s).Size()%8 != 0 && SizeAndAlign(s).Size() > 8 -> (Move [s] dst src mem) && s%8 != 0 && s > 8 ->
(Move [MakeSizeAndAlign(SizeAndAlign(s).Size()%8, 1).Int64()] (Move [s%8]
(OffPtr <dst.Type> dst [SizeAndAlign(s).Size()-SizeAndAlign(s).Size()%8]) (OffPtr <dst.Type> dst [s-s%8])
(OffPtr <src.Type> src [SizeAndAlign(s).Size()-SizeAndAlign(s).Size()%8]) (OffPtr <src.Type> src [s-s%8])
(Move [MakeSizeAndAlign(SizeAndAlign(s).Size()-SizeAndAlign(s).Size()%8, 1).Int64()] dst src mem)) (Move [s-s%8] dst src mem))
// medium move uses a duff device // medium move uses a duff device
// 8 and 128 are magic constants, see runtime/mkduff.go // 8 and 128 are magic constants, see runtime/mkduff.go
(Move [s] dst src mem) (Move [s] dst src mem)
&& SizeAndAlign(s).Size()%8 == 0 && SizeAndAlign(s).Size() > 24 && SizeAndAlign(s).Size() <= 8*128 && s%8 == 0 && s > 24 && s <= 8*128
&& !config.noDuffDevice -> && !config.noDuffDevice ->
(DUFFCOPY [8 * (128 - int64(SizeAndAlign(s).Size()/8))] dst src mem) (DUFFCOPY [8 * (128 - int64(s/8))] dst src mem)
// large move uses a loop // large move uses a loop
(Move [s] dst src mem) (Move [s] dst src mem)
&& SizeAndAlign(s).Size() > 24 && SizeAndAlign(s).Size()%8 == 0 -> && s > 24 && s%8 == 0 ->
(LoweredMove (LoweredMove
dst dst
src src
(ADDconst <src.Type> src [SizeAndAlign(s).Size()-moveSize(SizeAndAlign(s).Align(), config)]) (ADDconst <src.Type> src [s-8])
mem) mem)
// calls // calls
......
...@@ -264,99 +264,98 @@ ...@@ -264,99 +264,98 @@
(Load <t> ptr mem) && is64BitFloat(t) -> (MOVDload ptr mem) (Load <t> ptr mem) && is64BitFloat(t) -> (MOVDload ptr mem)
// stores // stores
(Store [1] ptr val mem) -> (MOVBstore ptr val mem) (Store {t} ptr val mem) && t.(Type).Size() == 1 -> (MOVBstore ptr val mem)
(Store [2] ptr val mem) -> (MOVHstore ptr val mem) (Store {t} ptr val mem) && t.(Type).Size() == 2 -> (MOVHstore ptr val mem)
(Store [4] ptr val mem) && !is32BitFloat(val.Type) -> (MOVWstore ptr val mem) (Store {t} ptr val mem) && t.(Type).Size() == 4 && !is32BitFloat(val.Type) -> (MOVWstore ptr val mem)
(Store [8] ptr val mem) && !is64BitFloat(val.Type) -> (MOVWstore ptr val mem) (Store {t} ptr val mem) && t.(Type).Size() == 4 && is32BitFloat(val.Type) -> (MOVFstore ptr val mem)
(Store [4] ptr val mem) && is32BitFloat(val.Type) -> (MOVFstore ptr val mem) (Store {t} ptr val mem) && t.(Type).Size() == 8 && is64BitFloat(val.Type) -> (MOVDstore ptr val mem)
(Store [8] ptr val mem) && is64BitFloat(val.Type) -> (MOVDstore ptr val mem)
// zero instructions // zero instructions
(Zero [s] _ mem) && SizeAndAlign(s).Size() == 0 -> mem (Zero [0] _ mem) -> mem
(Zero [s] ptr mem) && SizeAndAlign(s).Size() == 1 -> (MOVBstore ptr (MOVWconst [0]) mem) (Zero [1] ptr mem) -> (MOVBstore ptr (MOVWconst [0]) mem)
(Zero [s] ptr mem) && SizeAndAlign(s).Size() == 2 && SizeAndAlign(s).Align()%2 == 0 -> (Zero [2] {t} ptr mem) && t.(Type).Alignment()%2 == 0 ->
(MOVHstore ptr (MOVWconst [0]) mem) (MOVHstore ptr (MOVWconst [0]) mem)
(Zero [s] ptr mem) && SizeAndAlign(s).Size() == 2 -> (Zero [2] ptr mem) ->
(MOVBstore [1] ptr (MOVWconst [0]) (MOVBstore [1] ptr (MOVWconst [0])
(MOVBstore [0] ptr (MOVWconst [0]) mem)) (MOVBstore [0] ptr (MOVWconst [0]) mem))
(Zero [s] ptr mem) && SizeAndAlign(s).Size() == 4 && SizeAndAlign(s).Align()%4 == 0 -> (Zero [4] {t} ptr mem) && t.(Type).Alignment()%4 == 0 ->
(MOVWstore ptr (MOVWconst [0]) mem) (MOVWstore ptr (MOVWconst [0]) mem)
(Zero [s] ptr mem) && SizeAndAlign(s).Size() == 4 && SizeAndAlign(s).Align()%2 == 0 -> (Zero [4] {t} ptr mem) && t.(Type).Alignment()%2 == 0 ->
(MOVHstore [2] ptr (MOVWconst [0]) (MOVHstore [2] ptr (MOVWconst [0])
(MOVHstore [0] ptr (MOVWconst [0]) mem)) (MOVHstore [0] ptr (MOVWconst [0]) mem))
(Zero [s] ptr mem) && SizeAndAlign(s).Size() == 4 -> (Zero [4] ptr mem) ->
(MOVBstore [3] ptr (MOVWconst [0]) (MOVBstore [3] ptr (MOVWconst [0])
(MOVBstore [2] ptr (MOVWconst [0]) (MOVBstore [2] ptr (MOVWconst [0])
(MOVBstore [1] ptr (MOVWconst [0]) (MOVBstore [1] ptr (MOVWconst [0])
(MOVBstore [0] ptr (MOVWconst [0]) mem)))) (MOVBstore [0] ptr (MOVWconst [0]) mem))))
(Zero [s] ptr mem) && SizeAndAlign(s).Size() == 3 -> (Zero [3] ptr mem) ->
(MOVBstore [2] ptr (MOVWconst [0]) (MOVBstore [2] ptr (MOVWconst [0])
(MOVBstore [1] ptr (MOVWconst [0]) (MOVBstore [1] ptr (MOVWconst [0])
(MOVBstore [0] ptr (MOVWconst [0]) mem))) (MOVBstore [0] ptr (MOVWconst [0]) mem)))
(Zero [s] ptr mem) && SizeAndAlign(s).Size() == 6 && SizeAndAlign(s).Align()%2 == 0 -> (Zero [6] {t} ptr mem) && t.(Type).Alignment()%2 == 0 ->
(MOVHstore [4] ptr (MOVWconst [0]) (MOVHstore [4] ptr (MOVWconst [0])
(MOVHstore [2] ptr (MOVWconst [0]) (MOVHstore [2] ptr (MOVWconst [0])
(MOVHstore [0] ptr (MOVWconst [0]) mem))) (MOVHstore [0] ptr (MOVWconst [0]) mem)))
(Zero [s] ptr mem) && SizeAndAlign(s).Size() == 8 && SizeAndAlign(s).Align()%4 == 0 -> (Zero [8] {t} ptr mem) && t.(Type).Alignment()%4 == 0 ->
(MOVWstore [4] ptr (MOVWconst [0]) (MOVWstore [4] ptr (MOVWconst [0])
(MOVWstore [0] ptr (MOVWconst [0]) mem)) (MOVWstore [0] ptr (MOVWconst [0]) mem))
(Zero [s] ptr mem) && SizeAndAlign(s).Size() == 12 && SizeAndAlign(s).Align()%4 == 0 -> (Zero [12] {t} ptr mem) && t.(Type).Alignment()%4 == 0 ->
(MOVWstore [8] ptr (MOVWconst [0]) (MOVWstore [8] ptr (MOVWconst [0])
(MOVWstore [4] ptr (MOVWconst [0]) (MOVWstore [4] ptr (MOVWconst [0])
(MOVWstore [0] ptr (MOVWconst [0]) mem))) (MOVWstore [0] ptr (MOVWconst [0]) mem)))
(Zero [s] ptr mem) && SizeAndAlign(s).Size() == 16 && SizeAndAlign(s).Align()%4 == 0 -> (Zero [16] {t} ptr mem) && t.(Type).Alignment()%4 == 0 ->
(MOVWstore [12] ptr (MOVWconst [0]) (MOVWstore [12] ptr (MOVWconst [0])
(MOVWstore [8] ptr (MOVWconst [0]) (MOVWstore [8] ptr (MOVWconst [0])
(MOVWstore [4] ptr (MOVWconst [0]) (MOVWstore [4] ptr (MOVWconst [0])
(MOVWstore [0] ptr (MOVWconst [0]) mem)))) (MOVWstore [0] ptr (MOVWconst [0]) mem))))
// large or unaligned zeroing uses a loop // large or unaligned zeroing uses a loop
(Zero [s] ptr mem) (Zero [s] {t} ptr mem)
&& (SizeAndAlign(s).Size() > 16 || SizeAndAlign(s).Align()%4 != 0) -> && (s > 16 || s%4 != 0) ->
(LoweredZero [SizeAndAlign(s).Align()] (LoweredZero [t.(Type).Alignment()]
ptr ptr
(ADDconst <ptr.Type> ptr [SizeAndAlign(s).Size()-moveSize(SizeAndAlign(s).Align(), config)]) (ADDconst <ptr.Type> ptr [s-moveSize(t.(Type).Alignment(), config)])
mem) mem)
// moves // moves
(Move [s] _ _ mem) && SizeAndAlign(s).Size() == 0 -> mem (Move [0] _ _ mem) -> mem
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 1 -> (MOVBstore dst (MOVBUload src mem) mem) (Move [1] dst src mem) -> (MOVBstore dst (MOVBUload src mem) mem)
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 2 && SizeAndAlign(s).Align()%2 == 0 -> (Move [2] {t} dst src mem) && t.(Type).Alignment()%2 == 0 ->
(MOVHstore dst (MOVHUload src mem) mem) (MOVHstore dst (MOVHUload src mem) mem)
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 2 -> (Move [2] dst src mem) ->
(MOVBstore [1] dst (MOVBUload [1] src mem) (MOVBstore [1] dst (MOVBUload [1] src mem)
(MOVBstore dst (MOVBUload src mem) mem)) (MOVBstore dst (MOVBUload src mem) mem))
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 4 && SizeAndAlign(s).Align()%4 == 0 -> (Move [4] {t} dst src mem) && t.(Type).Alignment()%4 == 0 ->
(MOVWstore dst (MOVWload src mem) mem) (MOVWstore dst (MOVWload src mem) mem)
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 4 && SizeAndAlign(s).Align()%2 == 0 -> (Move [4] {t} dst src mem) && t.(Type).Alignment()%2 == 0 ->
(MOVHstore [2] dst (MOVHUload [2] src mem) (MOVHstore [2] dst (MOVHUload [2] src mem)
(MOVHstore dst (MOVHUload src mem) mem)) (MOVHstore dst (MOVHUload src mem) mem))
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 4 -> (Move [4] dst src mem) ->
(MOVBstore [3] dst (MOVBUload [3] src mem) (MOVBstore [3] dst (MOVBUload [3] src mem)
(MOVBstore [2] dst (MOVBUload [2] src mem) (MOVBstore [2] dst (MOVBUload [2] src mem)
(MOVBstore [1] dst (MOVBUload [1] src mem) (MOVBstore [1] dst (MOVBUload [1] src mem)
(MOVBstore dst (MOVBUload src mem) mem)))) (MOVBstore dst (MOVBUload src mem) mem))))
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 3 -> (Move [3] dst src mem) ->
(MOVBstore [2] dst (MOVBUload [2] src mem) (MOVBstore [2] dst (MOVBUload [2] src mem)
(MOVBstore [1] dst (MOVBUload [1] src mem) (MOVBstore [1] dst (MOVBUload [1] src mem)
(MOVBstore dst (MOVBUload src mem) mem))) (MOVBstore dst (MOVBUload src mem) mem)))
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 8 && SizeAndAlign(s).Align()%4 == 0 -> (Move [8] {t} dst src mem) && t.(Type).Alignment()%4 == 0 ->
(MOVWstore [4] dst (MOVWload [4] src mem) (MOVWstore [4] dst (MOVWload [4] src mem)
(MOVWstore dst (MOVWload src mem) mem)) (MOVWstore dst (MOVWload src mem) mem))
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 8 && SizeAndAlign(s).Align()%2 == 0 -> (Move [8] {t} dst src mem) && t.(Type).Alignment()%2 == 0 ->
(MOVHstore [6] dst (MOVHload [6] src mem) (MOVHstore [6] dst (MOVHload [6] src mem)
(MOVHstore [4] dst (MOVHload [4] src mem) (MOVHstore [4] dst (MOVHload [4] src mem)
(MOVHstore [2] dst (MOVHload [2] src mem) (MOVHstore [2] dst (MOVHload [2] src mem)
(MOVHstore dst (MOVHload src mem) mem)))) (MOVHstore dst (MOVHload src mem) mem))))
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 6 && SizeAndAlign(s).Align()%2 == 0 -> (Move [6] {t} dst src mem) && t.(Type).Alignment()%2 == 0 ->
(MOVHstore [4] dst (MOVHload [4] src mem) (MOVHstore [4] dst (MOVHload [4] src mem)
(MOVHstore [2] dst (MOVHload [2] src mem) (MOVHstore [2] dst (MOVHload [2] src mem)
(MOVHstore dst (MOVHload src mem) mem))) (MOVHstore dst (MOVHload src mem) mem)))
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 12 && SizeAndAlign(s).Align()%4 == 0 -> (Move [12] {t} dst src mem) && t.(Type).Alignment()%4 == 0 ->
(MOVWstore [8] dst (MOVWload [8] src mem) (MOVWstore [8] dst (MOVWload [8] src mem)
(MOVWstore [4] dst (MOVWload [4] src mem) (MOVWstore [4] dst (MOVWload [4] src mem)
(MOVWstore dst (MOVWload src mem) mem))) (MOVWstore dst (MOVWload src mem) mem)))
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 16 && SizeAndAlign(s).Align()%4 == 0 -> (Move [16] {t} dst src mem) && t.(Type).Alignment()%4 == 0 ->
(MOVWstore [12] dst (MOVWload [12] src mem) (MOVWstore [12] dst (MOVWload [12] src mem)
(MOVWstore [8] dst (MOVWload [8] src mem) (MOVWstore [8] dst (MOVWload [8] src mem)
(MOVWstore [4] dst (MOVWload [4] src mem) (MOVWstore [4] dst (MOVWload [4] src mem)
...@@ -364,12 +363,12 @@ ...@@ -364,12 +363,12 @@
// large or unaligned move uses a loop // large or unaligned move uses a loop
(Move [s] dst src mem) (Move [s] {t} dst src mem)
&& (SizeAndAlign(s).Size() > 16 || SizeAndAlign(s).Align()%4 != 0) -> && (s > 16 || t.(Type).Alignment()%4 != 0) ->
(LoweredMove [SizeAndAlign(s).Align()] (LoweredMove [t.(Type).Alignment()]
dst dst
src src
(ADDconst <src.Type> src [SizeAndAlign(s).Size()-moveSize(SizeAndAlign(s).Align(), config)]) (ADDconst <src.Type> src [s-moveSize(t.(Type).Alignment(), config)])
mem) mem)
// calls // calls
......
...@@ -285,133 +285,133 @@ ...@@ -285,133 +285,133 @@
(Load <t> ptr mem) && is64BitFloat(t) -> (MOVDload ptr mem) (Load <t> ptr mem) && is64BitFloat(t) -> (MOVDload ptr mem)
// stores // stores
(Store [1] ptr val mem) -> (MOVBstore ptr val mem) (Store {t} ptr val mem) && t.(Type).Size() == 1 -> (MOVBstore ptr val mem)
(Store [2] ptr val mem) -> (MOVHstore ptr val mem) (Store {t} ptr val mem) && t.(Type).Size() == 2 -> (MOVHstore ptr val mem)
(Store [4] ptr val mem) && !is32BitFloat(val.Type) -> (MOVWstore ptr val mem) (Store {t} ptr val mem) && t.(Type).Size() == 4 && !is32BitFloat(val.Type) -> (MOVWstore ptr val mem)
(Store [8] ptr val mem) && !is64BitFloat(val.Type) -> (MOVVstore ptr val mem) (Store {t} ptr val mem) && t.(Type).Size() == 8 && !is64BitFloat(val.Type) -> (MOVVstore ptr val mem)
(Store [4] ptr val mem) && is32BitFloat(val.Type) -> (MOVFstore ptr val mem) (Store {t} ptr val mem) && t.(Type).Size() == 4 && is32BitFloat(val.Type) -> (MOVFstore ptr val mem)
(Store [8] ptr val mem) && is64BitFloat(val.Type) -> (MOVDstore ptr val mem) (Store {t} ptr val mem) && t.(Type).Size() == 8 && is64BitFloat(val.Type) -> (MOVDstore ptr val mem)
// zeroing // zeroing
(Zero [s] _ mem) && SizeAndAlign(s).Size() == 0 -> mem (Zero [0] _ mem) -> mem
(Zero [s] ptr mem) && SizeAndAlign(s).Size() == 1 -> (MOVBstore ptr (MOVVconst [0]) mem) (Zero [1] ptr mem) -> (MOVBstore ptr (MOVVconst [0]) mem)
(Zero [s] ptr mem) && SizeAndAlign(s).Size() == 2 && SizeAndAlign(s).Align()%2 == 0 -> (Zero [2] {t} ptr mem) && t.(Type).Alignment()%2 == 0 ->
(MOVHstore ptr (MOVVconst [0]) mem) (MOVHstore ptr (MOVVconst [0]) mem)
(Zero [s] ptr mem) && SizeAndAlign(s).Size() == 2 -> (Zero [2] ptr mem) ->
(MOVBstore [1] ptr (MOVVconst [0]) (MOVBstore [1] ptr (MOVVconst [0])
(MOVBstore [0] ptr (MOVVconst [0]) mem)) (MOVBstore [0] ptr (MOVVconst [0]) mem))
(Zero [s] ptr mem) && SizeAndAlign(s).Size() == 4 && SizeAndAlign(s).Align()%4 == 0 -> (Zero [4] {t} ptr mem) && t.(Type).Alignment()%4 == 0 ->
(MOVWstore ptr (MOVVconst [0]) mem) (MOVWstore ptr (MOVVconst [0]) mem)
(Zero [s] ptr mem) && SizeAndAlign(s).Size() == 4 && SizeAndAlign(s).Align()%2 == 0 -> (Zero [4] {t} ptr mem) && t.(Type).Alignment()%2 == 0 ->
(MOVHstore [2] ptr (MOVVconst [0]) (MOVHstore [2] ptr (MOVVconst [0])
(MOVHstore [0] ptr (MOVVconst [0]) mem)) (MOVHstore [0] ptr (MOVVconst [0]) mem))
(Zero [s] ptr mem) && SizeAndAlign(s).Size() == 4 -> (Zero [4] ptr mem) ->
(MOVBstore [3] ptr (MOVVconst [0]) (MOVBstore [3] ptr (MOVVconst [0])
(MOVBstore [2] ptr (MOVVconst [0]) (MOVBstore [2] ptr (MOVVconst [0])
(MOVBstore [1] ptr (MOVVconst [0]) (MOVBstore [1] ptr (MOVVconst [0])
(MOVBstore [0] ptr (MOVVconst [0]) mem)))) (MOVBstore [0] ptr (MOVVconst [0]) mem))))
(Zero [s] ptr mem) && SizeAndAlign(s).Size() == 8 && SizeAndAlign(s).Align()%8 == 0 -> (Zero [8] {t} ptr mem) && t.(Type).Alignment()%8 == 0 ->
(MOVVstore ptr (MOVVconst [0]) mem) (MOVVstore ptr (MOVVconst [0]) mem)
(Zero [s] ptr mem) && SizeAndAlign(s).Size() == 8 && SizeAndAlign(s).Align()%4 == 0 -> (Zero [8] {t} ptr mem) && t.(Type).Alignment()%4 == 0 ->
(MOVWstore [4] ptr (MOVVconst [0]) (MOVWstore [4] ptr (MOVVconst [0])
(MOVWstore [0] ptr (MOVVconst [0]) mem)) (MOVWstore [0] ptr (MOVVconst [0]) mem))
(Zero [s] ptr mem) && SizeAndAlign(s).Size() == 4 -> (Zero [8] {t} ptr mem) && t.(Type).Alignment()%2 == 0 ->
(MOVHstore [6] ptr (MOVVconst [0]) (MOVHstore [6] ptr (MOVVconst [0])
(MOVHstore [4] ptr (MOVVconst [0]) (MOVHstore [4] ptr (MOVVconst [0])
(MOVHstore [2] ptr (MOVVconst [0]) (MOVHstore [2] ptr (MOVVconst [0])
(MOVHstore [0] ptr (MOVVconst [0]) mem)))) (MOVHstore [0] ptr (MOVVconst [0]) mem))))
(Zero [s] ptr mem) && SizeAndAlign(s).Size() == 3 -> (Zero [3] ptr mem) ->
(MOVBstore [2] ptr (MOVVconst [0]) (MOVBstore [2] ptr (MOVVconst [0])
(MOVBstore [1] ptr (MOVVconst [0]) (MOVBstore [1] ptr (MOVVconst [0])
(MOVBstore [0] ptr (MOVVconst [0]) mem))) (MOVBstore [0] ptr (MOVVconst [0]) mem)))
(Zero [s] ptr mem) && SizeAndAlign(s).Size() == 6 && SizeAndAlign(s).Align()%2 == 0 -> (Zero [6] {t} ptr mem) && t.(Type).Alignment()%2 == 0 ->
(MOVHstore [4] ptr (MOVVconst [0]) (MOVHstore [4] ptr (MOVVconst [0])
(MOVHstore [2] ptr (MOVVconst [0]) (MOVHstore [2] ptr (MOVVconst [0])
(MOVHstore [0] ptr (MOVVconst [0]) mem))) (MOVHstore [0] ptr (MOVVconst [0]) mem)))
(Zero [s] ptr mem) && SizeAndAlign(s).Size() == 12 && SizeAndAlign(s).Align()%4 == 0 -> (Zero [12] {t} ptr mem) && t.(Type).Alignment()%4 == 0 ->
(MOVWstore [8] ptr (MOVVconst [0]) (MOVWstore [8] ptr (MOVVconst [0])
(MOVWstore [4] ptr (MOVVconst [0]) (MOVWstore [4] ptr (MOVVconst [0])
(MOVWstore [0] ptr (MOVVconst [0]) mem))) (MOVWstore [0] ptr (MOVVconst [0]) mem)))
(Zero [s] ptr mem) && SizeAndAlign(s).Size() == 16 && SizeAndAlign(s).Align()%8 == 0 -> (Zero [16] {t} ptr mem) && t.(Type).Alignment()%8 == 0 ->
(MOVVstore [8] ptr (MOVVconst [0]) (MOVVstore [8] ptr (MOVVconst [0])
(MOVVstore [0] ptr (MOVVconst [0]) mem)) (MOVVstore [0] ptr (MOVVconst [0]) mem))
(Zero [s] ptr mem) && SizeAndAlign(s).Size() == 24 && SizeAndAlign(s).Align()%8 == 0 -> (Zero [24] {t} ptr mem) && t.(Type).Alignment()%8 == 0 ->
(MOVVstore [16] ptr (MOVVconst [0]) (MOVVstore [16] ptr (MOVVconst [0])
(MOVVstore [8] ptr (MOVVconst [0]) (MOVVstore [8] ptr (MOVVconst [0])
(MOVVstore [0] ptr (MOVVconst [0]) mem))) (MOVVstore [0] ptr (MOVVconst [0]) mem)))
// medium zeroing uses a duff device // medium zeroing uses a duff device
// 8, and 128 are magic constants, see runtime/mkduff.go // 8, and 128 are magic constants, see runtime/mkduff.go
(Zero [s] ptr mem) (Zero [s] {t} ptr mem)
&& SizeAndAlign(s).Size()%8 == 0 && SizeAndAlign(s).Size() > 24 && SizeAndAlign(s).Size() <= 8*128 && s%8 == 0 && s > 24 && s <= 8*128
&& SizeAndAlign(s).Align()%8 == 0 && !config.noDuffDevice -> && t.(Type).Alignment()%8 == 0 && !config.noDuffDevice ->
(DUFFZERO [8 * (128 - int64(SizeAndAlign(s).Size()/8))] ptr mem) (DUFFZERO [8 * (128 - int64(s/8))] ptr mem)
// large or unaligned zeroing uses a loop // large or unaligned zeroing uses a loop
(Zero [s] ptr mem) (Zero [s] {t} ptr mem)
&& (SizeAndAlign(s).Size() > 8*128 || config.noDuffDevice) || SizeAndAlign(s).Align()%8 != 0 -> && (s > 8*128 || config.noDuffDevice) || t.(Type).Alignment()%8 != 0 ->
(LoweredZero [SizeAndAlign(s).Align()] (LoweredZero [t.(Type).Alignment()]
ptr ptr
(ADDVconst <ptr.Type> ptr [SizeAndAlign(s).Size()-moveSize(SizeAndAlign(s).Align(), config)]) (ADDVconst <ptr.Type> ptr [s-moveSize(t.(Type).Alignment(), config)])
mem) mem)
// moves // moves
(Move [s] _ _ mem) && SizeAndAlign(s).Size() == 0 -> mem (Move [0] _ _ mem) -> mem
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 1 -> (MOVBstore dst (MOVBload src mem) mem) (Move [1] dst src mem) -> (MOVBstore dst (MOVBload src mem) mem)
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 2 && SizeAndAlign(s).Align()%2 == 0 -> (Move [2] {t} dst src mem) && t.(Type).Alignment()%2 == 0 ->
(MOVHstore dst (MOVHload src mem) mem) (MOVHstore dst (MOVHload src mem) mem)
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 2 -> (Move [2] dst src mem) ->
(MOVBstore [1] dst (MOVBload [1] src mem) (MOVBstore [1] dst (MOVBload [1] src mem)
(MOVBstore dst (MOVBload src mem) mem)) (MOVBstore dst (MOVBload src mem) mem))
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 4 && SizeAndAlign(s).Align()%4 == 0 -> (Move [4] {t} dst src mem) && t.(Type).Alignment()%4 == 0 ->
(MOVWstore dst (MOVWload src mem) mem) (MOVWstore dst (MOVWload src mem) mem)
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 4 && SizeAndAlign(s).Align()%2 == 0 -> (Move [4] {t} dst src mem) && t.(Type).Alignment()%2 == 0 ->
(MOVHstore [2] dst (MOVHload [2] src mem) (MOVHstore [2] dst (MOVHload [2] src mem)
(MOVHstore dst (MOVHload src mem) mem)) (MOVHstore dst (MOVHload src mem) mem))
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 4 -> (Move [4] dst src mem) ->
(MOVBstore [3] dst (MOVBload [3] src mem) (MOVBstore [3] dst (MOVBload [3] src mem)
(MOVBstore [2] dst (MOVBload [2] src mem) (MOVBstore [2] dst (MOVBload [2] src mem)
(MOVBstore [1] dst (MOVBload [1] src mem) (MOVBstore [1] dst (MOVBload [1] src mem)
(MOVBstore dst (MOVBload src mem) mem)))) (MOVBstore dst (MOVBload src mem) mem))))
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 8 && SizeAndAlign(s).Align()%8 == 0 -> (Move [8] {t} dst src mem) && t.(Type).Alignment()%8 == 0 ->
(MOVVstore dst (MOVVload src mem) mem) (MOVVstore dst (MOVVload src mem) mem)
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 8 && SizeAndAlign(s).Align()%4 == 0 -> (Move [8] {t} dst src mem) && t.(Type).Alignment()%4 == 0 ->
(MOVWstore [4] dst (MOVWload [4] src mem) (MOVWstore [4] dst (MOVWload [4] src mem)
(MOVWstore dst (MOVWload src mem) mem)) (MOVWstore dst (MOVWload src mem) mem))
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 8 && SizeAndAlign(s).Align()%2 == 0 -> (Move [8] {t} dst src mem) && t.(Type).Alignment()%2 == 0 ->
(MOVHstore [6] dst (MOVHload [6] src mem) (MOVHstore [6] dst (MOVHload [6] src mem)
(MOVHstore [4] dst (MOVHload [4] src mem) (MOVHstore [4] dst (MOVHload [4] src mem)
(MOVHstore [2] dst (MOVHload [2] src mem) (MOVHstore [2] dst (MOVHload [2] src mem)
(MOVHstore dst (MOVHload src mem) mem)))) (MOVHstore dst (MOVHload src mem) mem))))
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 3 -> (Move [3] dst src mem) ->
(MOVBstore [2] dst (MOVBload [2] src mem) (MOVBstore [2] dst (MOVBload [2] src mem)
(MOVBstore [1] dst (MOVBload [1] src mem) (MOVBstore [1] dst (MOVBload [1] src mem)
(MOVBstore dst (MOVBload src mem) mem))) (MOVBstore dst (MOVBload src mem) mem)))
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 6 && SizeAndAlign(s).Align()%2 == 0 -> (Move [6] {t} dst src mem) && t.(Type).Alignment()%2 == 0 ->
(MOVHstore [4] dst (MOVHload [4] src mem) (MOVHstore [4] dst (MOVHload [4] src mem)
(MOVHstore [2] dst (MOVHload [2] src mem) (MOVHstore [2] dst (MOVHload [2] src mem)
(MOVHstore dst (MOVHload src mem) mem))) (MOVHstore dst (MOVHload src mem) mem)))
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 12 && SizeAndAlign(s).Align()%4 == 0 -> (Move [12] {t} dst src mem) && t.(Type).Alignment()%4 == 0 ->
(MOVWstore [8] dst (MOVWload [8] src mem) (MOVWstore [8] dst (MOVWload [8] src mem)
(MOVWstore [4] dst (MOVWload [4] src mem) (MOVWstore [4] dst (MOVWload [4] src mem)
(MOVWstore dst (MOVWload src mem) mem))) (MOVWstore dst (MOVWload src mem) mem)))
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 16 && SizeAndAlign(s).Align()%8 == 0 -> (Move [16] {t} dst src mem) && t.(Type).Alignment()%8 == 0 ->
(MOVVstore [8] dst (MOVVload [8] src mem) (MOVVstore [8] dst (MOVVload [8] src mem)
(MOVVstore dst (MOVVload src mem) mem)) (MOVVstore dst (MOVVload src mem) mem))
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 24 && SizeAndAlign(s).Align()%8 == 0 -> (Move [24] {t} dst src mem) && t.(Type).Alignment()%8 == 0 ->
(MOVVstore [16] dst (MOVVload [16] src mem) (MOVVstore [16] dst (MOVVload [16] src mem)
(MOVVstore [8] dst (MOVVload [8] src mem) (MOVVstore [8] dst (MOVVload [8] src mem)
(MOVVstore dst (MOVVload src mem) mem))) (MOVVstore dst (MOVVload src mem) mem)))
// large or unaligned move uses a loop // large or unaligned move uses a loop
(Move [s] dst src mem) (Move [s] {t} dst src mem)
&& SizeAndAlign(s).Size() > 24 || SizeAndAlign(s).Align()%8 != 0 -> && s > 24 || t.(Type).Alignment()%8 != 0 ->
(LoweredMove [SizeAndAlign(s).Align()] (LoweredMove [t.(Type).Alignment()]
dst dst
src src
(ADDVconst <src.Type> src [SizeAndAlign(s).Size()-moveSize(SizeAndAlign(s).Align(), config)]) (ADDVconst <src.Type> src [s-moveSize(t.(Type).Alignment(), config)])
mem) mem)
// calls // calls
......
...@@ -477,111 +477,111 @@ ...@@ -477,111 +477,111 @@
(Load <t> ptr mem) && is32BitFloat(t) -> (FMOVSload ptr mem) (Load <t> ptr mem) && is32BitFloat(t) -> (FMOVSload ptr mem)
(Load <t> ptr mem) && is64BitFloat(t) -> (FMOVDload ptr mem) (Load <t> ptr mem) && is64BitFloat(t) -> (FMOVDload ptr mem)
(Store [8] ptr val mem) && is64BitFloat(val.Type) -> (FMOVDstore ptr val mem) (Store {t} ptr val mem) && t.(Type).Size() == 8 && is64BitFloat(val.Type) -> (FMOVDstore ptr val mem)
(Store [8] ptr val mem) && is32BitFloat(val.Type) -> (FMOVDstore ptr val mem) // glitch from (Cvt32Fto64F x) -> x -- type is wrong (Store {t} ptr val mem) && t.(Type).Size() == 8 && is32BitFloat(val.Type) -> (FMOVDstore ptr val mem) // glitch from (Cvt32Fto64F x) -> x -- type is wrong
(Store [4] ptr val mem) && is32BitFloat(val.Type) -> (FMOVSstore ptr val mem) (Store {t} ptr val mem) && t.(Type).Size() == 4 && is32BitFloat(val.Type) -> (FMOVSstore ptr val mem)
(Store [8] ptr val mem) && (is64BitInt(val.Type) || isPtr(val.Type)) -> (MOVDstore ptr val mem) (Store {t} ptr val mem) && t.(Type).Size() == 8 && (is64BitInt(val.Type) || isPtr(val.Type)) -> (MOVDstore ptr val mem)
(Store [4] ptr val mem) && is32BitInt(val.Type) -> (MOVWstore ptr val mem) (Store {t} ptr val mem) && t.(Type).Size() == 4 && is32BitInt(val.Type) -> (MOVWstore ptr val mem)
(Store [2] ptr val mem) -> (MOVHstore ptr val mem) (Store {t} ptr val mem) && t.(Type).Size() == 2 -> (MOVHstore ptr val mem)
(Store [1] ptr val mem) -> (MOVBstore ptr val mem) (Store {t} ptr val mem) && t.(Type).Size() == 1 -> (MOVBstore ptr val mem)
(Zero [s] _ mem) && SizeAndAlign(s).Size() == 0 -> mem (Zero [0] _ mem) -> mem
(Zero [s] destptr mem) && SizeAndAlign(s).Size() == 1 -> (MOVBstorezero destptr mem) (Zero [1] destptr mem) -> (MOVBstorezero destptr mem)
(Zero [s] destptr mem) && SizeAndAlign(s).Size() == 2 && SizeAndAlign(s).Align()%2 == 0 -> (Zero [2] {t} destptr mem) && t.(Type).Alignment()%2 == 0 ->
(MOVHstorezero destptr mem) (MOVHstorezero destptr mem)
(Zero [s] destptr mem) && SizeAndAlign(s).Size() == 2 -> (Zero [2] destptr mem) ->
(MOVBstorezero [1] destptr (MOVBstorezero [1] destptr
(MOVBstorezero [0] destptr mem)) (MOVBstorezero [0] destptr mem))
(Zero [s] destptr mem) && SizeAndAlign(s).Size() == 4 && SizeAndAlign(s).Align()%4 == 0 -> (Zero [4] {t} destptr mem) && t.(Type).Alignment()%4 == 0 ->
(MOVWstorezero destptr mem) (MOVWstorezero destptr mem)
(Zero [s] destptr mem) && SizeAndAlign(s).Size() == 4 && SizeAndAlign(s).Align()%2 == 0 -> (Zero [4] {t} destptr mem) && t.(Type).Alignment()%2 == 0 ->
(MOVHstorezero [2] destptr (MOVHstorezero [2] destptr
(MOVHstorezero [0] destptr mem)) (MOVHstorezero [0] destptr mem))
(Zero [s] destptr mem) && SizeAndAlign(s).Size() == 4 -> (Zero [4] destptr mem) ->
(MOVBstorezero [3] destptr (MOVBstorezero [3] destptr
(MOVBstorezero [2] destptr (MOVBstorezero [2] destptr
(MOVBstorezero [1] destptr (MOVBstorezero [1] destptr
(MOVBstorezero [0] destptr mem)))) (MOVBstorezero [0] destptr mem))))
(Zero [s] destptr mem) && SizeAndAlign(s).Size() == 8 && SizeAndAlign(s).Align()%8 == 0 -> (Zero [8] {t} destptr mem) && t.(Type).Alignment()%8 == 0 ->
(MOVDstorezero [0] destptr mem) (MOVDstorezero [0] destptr mem)
(Zero [s] destptr mem) && SizeAndAlign(s).Size() == 8 && SizeAndAlign(s).Align()%4 == 0 -> (Zero [8] {t} destptr mem) && t.(Type).Alignment()%4 == 0 ->
(MOVWstorezero [4] destptr (MOVWstorezero [4] destptr
(MOVWstorezero [0] destptr mem)) (MOVWstorezero [0] destptr mem))
(Zero [s] destptr mem) && SizeAndAlign(s).Size() == 8 && SizeAndAlign(s).Align()%2 == 0 -> (Zero [8] {t} destptr mem) && t.(Type).Alignment()%2 == 0 ->
(MOVHstorezero [6] destptr (MOVHstorezero [6] destptr
(MOVHstorezero [4] destptr (MOVHstorezero [4] destptr
(MOVHstorezero [2] destptr (MOVHstorezero [2] destptr
(MOVHstorezero [0] destptr mem)))) (MOVHstorezero [0] destptr mem))))
(Zero [s] destptr mem) && SizeAndAlign(s).Size() == 3 -> (Zero [3] destptr mem) ->
(MOVBstorezero [2] destptr (MOVBstorezero [2] destptr
(MOVBstorezero [1] destptr (MOVBstorezero [1] destptr
(MOVBstorezero [0] destptr mem))) (MOVBstorezero [0] destptr mem)))
// Zero small numbers of words directly. // Zero small numbers of words directly.
(Zero [s] destptr mem) && SizeAndAlign(s).Size() == 16 && SizeAndAlign(s).Align()%8 == 0 -> (Zero [16] {t} destptr mem) && t.(Type).Alignment()%8 == 0 ->
(MOVDstorezero [8] destptr (MOVDstorezero [8] destptr
(MOVDstorezero [0] destptr mem)) (MOVDstorezero [0] destptr mem))
(Zero [s] destptr mem) && SizeAndAlign(s).Size() == 24 && SizeAndAlign(s).Align()%8 == 0 -> (Zero [24] {t} destptr mem) && t.(Type).Alignment()%8 == 0 ->
(MOVDstorezero [16] destptr (MOVDstorezero [16] destptr
(MOVDstorezero [8] destptr (MOVDstorezero [8] destptr
(MOVDstorezero [0] destptr mem))) (MOVDstorezero [0] destptr mem)))
(Zero [s] destptr mem) && SizeAndAlign(s).Size() == 32 && SizeAndAlign(s).Align()%8 == 0 -> (Zero [32] {t} destptr mem) && t.(Type).Alignment()%8 == 0 ->
(MOVDstorezero [24] destptr (MOVDstorezero [24] destptr
(MOVDstorezero [16] destptr (MOVDstorezero [16] destptr
(MOVDstorezero [8] destptr (MOVDstorezero [8] destptr
(MOVDstorezero [0] destptr mem)))) (MOVDstorezero [0] destptr mem))))
// Large zeroing uses a loop // Large zeroing uses a loop
(Zero [s] ptr mem) (Zero [s] {t} ptr mem)
&& (SizeAndAlign(s).Size() > 512 || config.noDuffDevice) || SizeAndAlign(s).Align()%8 != 0 -> && (s > 512 || config.noDuffDevice) || t.(Type).Alignment()%8 != 0 ->
(LoweredZero [SizeAndAlign(s).Align()] (LoweredZero [t.(Type).Alignment()]
ptr ptr
(ADDconst <ptr.Type> ptr [SizeAndAlign(s).Size()-moveSize(SizeAndAlign(s).Align(), config)]) (ADDconst <ptr.Type> ptr [s-moveSize(t.(Type).Alignment(), config)])
mem) mem)
// moves // moves
(Move [s] _ _ mem) && SizeAndAlign(s).Size() == 0 -> mem (Move [0] _ _ mem) -> mem
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 1 -> (MOVBstore dst (MOVBZload src mem) mem) (Move [1] dst src mem) -> (MOVBstore dst (MOVBZload src mem) mem)
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 2 && SizeAndAlign(s).Align()%2 == 0 -> (Move [2] {t} dst src mem) && t.(Type).Alignment()%2 == 0 ->
(MOVHstore dst (MOVHZload src mem) mem) (MOVHstore dst (MOVHZload src mem) mem)
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 2 -> (Move [2] dst src mem) ->
(MOVBstore [1] dst (MOVBZload [1] src mem) (MOVBstore [1] dst (MOVBZload [1] src mem)
(MOVBstore dst (MOVBZload src mem) mem)) (MOVBstore dst (MOVBZload src mem) mem))
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 4 && SizeAndAlign(s).Align()%4 == 0 -> (Move [4] {t} dst src mem) && t.(Type).Alignment()%4 == 0 ->
(MOVWstore dst (MOVWload src mem) mem) (MOVWstore dst (MOVWload src mem) mem)
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 4 && SizeAndAlign(s).Align()%2 == 0 -> (Move [4] {t} dst src mem) && t.(Type).Alignment()%2 == 0 ->
(MOVHstore [2] dst (MOVHZload [2] src mem) (MOVHstore [2] dst (MOVHZload [2] src mem)
(MOVHstore dst (MOVHZload src mem) mem)) (MOVHstore dst (MOVHZload src mem) mem))
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 4 -> (Move [4] dst src mem) ->
(MOVBstore [3] dst (MOVBZload [3] src mem) (MOVBstore [3] dst (MOVBZload [3] src mem)
(MOVBstore [2] dst (MOVBZload [2] src mem) (MOVBstore [2] dst (MOVBZload [2] src mem)
(MOVBstore [1] dst (MOVBZload [1] src mem) (MOVBstore [1] dst (MOVBZload [1] src mem)
(MOVBstore dst (MOVBZload src mem) mem)))) (MOVBstore dst (MOVBZload src mem) mem))))
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 8 && SizeAndAlign(s).Align()%8 == 0 -> (Move [8] {t} dst src mem) && t.(Type).Alignment()%8 == 0 ->
(MOVDstore dst (MOVDload src mem) mem) (MOVDstore dst (MOVDload src mem) mem)
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 8 && SizeAndAlign(s).Align()%4 == 0 -> (Move [8] {t} dst src mem) && t.(Type).Alignment()%4 == 0 ->
(MOVWstore [4] dst (MOVWZload [4] src mem) (MOVWstore [4] dst (MOVWZload [4] src mem)
(MOVWstore dst (MOVWZload src mem) mem)) (MOVWstore dst (MOVWZload src mem) mem))
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 8 && SizeAndAlign(s).Align()%2 == 0-> (Move [8] {t} dst src mem) && t.(Type).Alignment()%2 == 0->
(MOVHstore [6] dst (MOVHZload [6] src mem) (MOVHstore [6] dst (MOVHZload [6] src mem)
(MOVHstore [4] dst (MOVHZload [4] src mem) (MOVHstore [4] dst (MOVHZload [4] src mem)
(MOVHstore [2] dst (MOVHZload [2] src mem) (MOVHstore [2] dst (MOVHZload [2] src mem)
(MOVHstore dst (MOVHZload src mem) mem)))) (MOVHstore dst (MOVHZload src mem) mem))))
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 3 -> (Move [3] dst src mem) ->
(MOVBstore [2] dst (MOVBZload [2] src mem) (MOVBstore [2] dst (MOVBZload [2] src mem)
(MOVBstore [1] dst (MOVBZload [1] src mem) (MOVBstore [1] dst (MOVBZload [1] src mem)
(MOVBstore dst (MOVBZload src mem) mem))) (MOVBstore dst (MOVBZload src mem) mem)))
// Large move uses a loop // Large move uses a loop
(Move [s] dst src mem) (Move [s] {t} dst src mem)
&& (SizeAndAlign(s).Size() > 512 || config.noDuffDevice) || SizeAndAlign(s).Align()%8 != 0 -> && (s > 512 || config.noDuffDevice) || t.(Type).Alignment()%8 != 0 ->
(LoweredMove [SizeAndAlign(s).Align()] (LoweredMove [t.(Type).Alignment()]
dst dst
src src
(ADDconst <src.Type> src [SizeAndAlign(s).Size()-moveSize(SizeAndAlign(s).Align(), config)]) (ADDconst <src.Type> src [s-moveSize(t.(Type).Alignment(), config)])
mem) mem)
// Calls // Calls
......
...@@ -320,82 +320,82 @@ ...@@ -320,82 +320,82 @@
// Lowering stores // Lowering stores
// These more-specific FP versions of Store pattern should come first. // These more-specific FP versions of Store pattern should come first.
(Store [8] ptr val mem) && is64BitFloat(val.Type) -> (FMOVDstore ptr val mem) (Store {t} ptr val mem) && t.(Type).Size() == 8 && is64BitFloat(val.Type) -> (FMOVDstore ptr val mem)
(Store [4] ptr val mem) && is32BitFloat(val.Type) -> (FMOVSstore ptr val mem) (Store {t} ptr val mem) && t.(Type).Size() == 4 && is32BitFloat(val.Type) -> (FMOVSstore ptr val mem)
(Store [8] ptr val mem) -> (MOVDstore ptr val mem) (Store {t} ptr val mem) && t.(Type).Size() == 8 -> (MOVDstore ptr val mem)
(Store [4] ptr val mem) -> (MOVWstore ptr val mem) (Store {t} ptr val mem) && t.(Type).Size() == 4 -> (MOVWstore ptr val mem)
(Store [2] ptr val mem) -> (MOVHstore ptr val mem) (Store {t} ptr val mem) && t.(Type).Size() == 2 -> (MOVHstore ptr val mem)
(Store [1] ptr val mem) -> (MOVBstore ptr val mem) (Store {t} ptr val mem) && t.(Type).Size() == 1 -> (MOVBstore ptr val mem)
// Lowering moves // Lowering moves
// Load and store for small copies. // Load and store for small copies.
(Move [s] _ _ mem) && SizeAndAlign(s).Size() == 0 -> mem (Move [0] _ _ mem) -> mem
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 1 -> (MOVBstore dst (MOVBZload src mem) mem) (Move [1] dst src mem) -> (MOVBstore dst (MOVBZload src mem) mem)
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 2 -> (MOVHstore dst (MOVHZload src mem) mem) (Move [2] dst src mem) -> (MOVHstore dst (MOVHZload src mem) mem)
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 4 -> (MOVWstore dst (MOVWZload src mem) mem) (Move [4] dst src mem) -> (MOVWstore dst (MOVWZload src mem) mem)
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 8 -> (MOVDstore dst (MOVDload src mem) mem) (Move [8] dst src mem) -> (MOVDstore dst (MOVDload src mem) mem)
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 16 -> (Move [16] dst src mem) ->
(MOVDstore [8] dst (MOVDload [8] src mem) (MOVDstore [8] dst (MOVDload [8] src mem)
(MOVDstore dst (MOVDload src mem) mem)) (MOVDstore dst (MOVDload src mem) mem))
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 24 -> (Move [24] dst src mem) ->
(MOVDstore [16] dst (MOVDload [16] src mem) (MOVDstore [16] dst (MOVDload [16] src mem)
(MOVDstore [8] dst (MOVDload [8] src mem) (MOVDstore [8] dst (MOVDload [8] src mem)
(MOVDstore dst (MOVDload src mem) mem))) (MOVDstore dst (MOVDload src mem) mem)))
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 3 -> (Move [3] dst src mem) ->
(MOVBstore [2] dst (MOVBZload [2] src mem) (MOVBstore [2] dst (MOVBZload [2] src mem)
(MOVHstore dst (MOVHZload src mem) mem)) (MOVHstore dst (MOVHZload src mem) mem))
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 5 -> (Move [5] dst src mem) ->
(MOVBstore [4] dst (MOVBZload [4] src mem) (MOVBstore [4] dst (MOVBZload [4] src mem)
(MOVWstore dst (MOVWZload src mem) mem)) (MOVWstore dst (MOVWZload src mem) mem))
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 6 -> (Move [6] dst src mem) ->
(MOVHstore [4] dst (MOVHZload [4] src mem) (MOVHstore [4] dst (MOVHZload [4] src mem)
(MOVWstore dst (MOVWZload src mem) mem)) (MOVWstore dst (MOVWZload src mem) mem))
(Move [s] dst src mem) && SizeAndAlign(s).Size() == 7 -> (Move [7] dst src mem) ->
(MOVBstore [6] dst (MOVBZload [6] src mem) (MOVBstore [6] dst (MOVBZload [6] src mem)
(MOVHstore [4] dst (MOVHZload [4] src mem) (MOVHstore [4] dst (MOVHZload [4] src mem)
(MOVWstore dst (MOVWZload src mem) mem))) (MOVWstore dst (MOVWZload src mem) mem)))
// MVC for other moves. Use up to 4 instructions (sizes up to 1024 bytes). // MVC for other moves. Use up to 4 instructions (sizes up to 1024 bytes).
(Move [s] dst src mem) && SizeAndAlign(s).Size() > 0 && SizeAndAlign(s).Size() <= 256 -> (Move [s] dst src mem) && s > 0 && s <= 256 ->
(MVC [makeValAndOff(SizeAndAlign(s).Size(), 0)] dst src mem) (MVC [makeValAndOff(s, 0)] dst src mem)
(Move [s] dst src mem) && SizeAndAlign(s).Size() > 256 && SizeAndAlign(s).Size() <= 512 -> (Move [s] dst src mem) && s > 256 && s <= 512 ->
(MVC [makeValAndOff(SizeAndAlign(s).Size()-256, 256)] dst src (MVC [makeValAndOff(256, 0)] dst src mem)) (MVC [makeValAndOff(s-256, 256)] dst src (MVC [makeValAndOff(256, 0)] dst src mem))
(Move [s] dst src mem) && SizeAndAlign(s).Size() > 512 && SizeAndAlign(s).Size() <= 768 -> (Move [s] dst src mem) && s > 512 && s <= 768 ->
(MVC [makeValAndOff(SizeAndAlign(s).Size()-512, 512)] dst src (MVC [makeValAndOff(256, 256)] dst src (MVC [makeValAndOff(256, 0)] dst src mem))) (MVC [makeValAndOff(s-512, 512)] dst src (MVC [makeValAndOff(256, 256)] dst src (MVC [makeValAndOff(256, 0)] dst src mem)))
(Move [s] dst src mem) && SizeAndAlign(s).Size() > 768 && SizeAndAlign(s).Size() <= 1024 -> (Move [s] dst src mem) && s > 768 && s <= 1024 ->
(MVC [makeValAndOff(SizeAndAlign(s).Size()-768, 768)] dst src (MVC [makeValAndOff(256, 512)] dst src (MVC [makeValAndOff(256, 256)] dst src (MVC [makeValAndOff(256, 0)] dst src mem)))) (MVC [makeValAndOff(s-768, 768)] dst src (MVC [makeValAndOff(256, 512)] dst src (MVC [makeValAndOff(256, 256)] dst src (MVC [makeValAndOff(256, 0)] dst src mem))))
// Move more than 1024 bytes using a loop. // Move more than 1024 bytes using a loop.
(Move [s] dst src mem) && SizeAndAlign(s).Size() > 1024 -> (Move [s] dst src mem) && s > 1024 ->
(LoweredMove [SizeAndAlign(s).Size()%256] dst src (ADDconst <src.Type> src [(SizeAndAlign(s).Size()/256)*256]) mem) (LoweredMove [s%256] dst src (ADDconst <src.Type> src [(s/256)*256]) mem)
// Lowering Zero instructions // Lowering Zero instructions
(Zero [s] _ mem) && SizeAndAlign(s).Size() == 0 -> mem (Zero [0] _ mem) -> mem
(Zero [s] destptr mem) && SizeAndAlign(s).Size() == 1 -> (MOVBstoreconst [0] destptr mem) (Zero [1] destptr mem) -> (MOVBstoreconst [0] destptr mem)
(Zero [s] destptr mem) && SizeAndAlign(s).Size() == 2 -> (MOVHstoreconst [0] destptr mem) (Zero [2] destptr mem) -> (MOVHstoreconst [0] destptr mem)
(Zero [s] destptr mem) && SizeAndAlign(s).Size() == 4 -> (MOVWstoreconst [0] destptr mem) (Zero [4] destptr mem) -> (MOVWstoreconst [0] destptr mem)
(Zero [s] destptr mem) && SizeAndAlign(s).Size() == 8 -> (MOVDstoreconst [0] destptr mem) (Zero [8] destptr mem) -> (MOVDstoreconst [0] destptr mem)
(Zero [s] destptr mem) && SizeAndAlign(s).Size() == 3 -> (Zero [3] destptr mem) ->
(MOVBstoreconst [makeValAndOff(0,2)] destptr (MOVBstoreconst [makeValAndOff(0,2)] destptr
(MOVHstoreconst [0] destptr mem)) (MOVHstoreconst [0] destptr mem))
(Zero [s] destptr mem) && SizeAndAlign(s).Size() == 5 -> (Zero [5] destptr mem) ->
(MOVBstoreconst [makeValAndOff(0,4)] destptr (MOVBstoreconst [makeValAndOff(0,4)] destptr
(MOVWstoreconst [0] destptr mem)) (MOVWstoreconst [0] destptr mem))
(Zero [s] destptr mem) && SizeAndAlign(s).Size() == 6 -> (Zero [6] destptr mem) ->
(MOVHstoreconst [makeValAndOff(0,4)] destptr (MOVHstoreconst [makeValAndOff(0,4)] destptr
(MOVWstoreconst [0] destptr mem)) (MOVWstoreconst [0] destptr mem))
(Zero [s] destptr mem) && SizeAndAlign(s).Size() == 7 -> (Zero [7] destptr mem) ->
(MOVWstoreconst [makeValAndOff(0,3)] destptr (MOVWstoreconst [makeValAndOff(0,3)] destptr
(MOVWstoreconst [0] destptr mem)) (MOVWstoreconst [0] destptr mem))
(Zero [s] destptr mem) && SizeAndAlign(s).Size() > 0 && SizeAndAlign(s).Size() <= 1024 -> (Zero [s] destptr mem) && s > 0 && s <= 1024 ->
(CLEAR [makeValAndOff(SizeAndAlign(s).Size(), 0)] destptr mem) (CLEAR [makeValAndOff(s, 0)] destptr mem)
// Move more than 1024 bytes using a loop. // Move more than 1024 bytes using a loop.
(Zero [s] destptr mem) && SizeAndAlign(s).Size() > 1024 -> (Zero [s] destptr mem) && s > 1024 ->
(LoweredZero [SizeAndAlign(s).Size()%256] destptr (ADDconst <destptr.Type> destptr [(SizeAndAlign(s).Size()/256)*256]) mem) (LoweredZero [s%256] destptr (ADDconst <destptr.Type> destptr [(s/256)*256]) mem)
// Lowering constants // Lowering constants
(Const8 [val]) -> (MOVDconst [val]) (Const8 [val]) -> (MOVDconst [val])
......
...@@ -18,11 +18,11 @@ ...@@ -18,11 +18,11 @@
(OffPtr <config.fe.TypeFloat32().PtrTo()> [4] ptr) (OffPtr <config.fe.TypeFloat32().PtrTo()> [4] ptr)
mem) mem)
) )
(Store [8] dst (ComplexMake real imag) mem) -> (Store {t} dst (ComplexMake real imag) mem) && t.(Type).Size() == 8 ->
(Store [4] {config.fe.TypeFloat32()} (Store {config.fe.TypeFloat32()}
(OffPtr <config.fe.TypeFloat32().PtrTo()> [4] dst) (OffPtr <config.fe.TypeFloat32().PtrTo()> [4] dst)
imag imag
(Store [4] {config.fe.TypeFloat32()} dst real mem)) (Store {config.fe.TypeFloat32()} dst real mem))
(Load <t> ptr mem) && t.IsComplex() && t.Size() == 16 -> (Load <t> ptr mem) && t.IsComplex() && t.Size() == 16 ->
(ComplexMake (ComplexMake
(Load <config.fe.TypeFloat64()> ptr mem) (Load <config.fe.TypeFloat64()> ptr mem)
...@@ -30,11 +30,11 @@ ...@@ -30,11 +30,11 @@
(OffPtr <config.fe.TypeFloat64().PtrTo()> [8] ptr) (OffPtr <config.fe.TypeFloat64().PtrTo()> [8] ptr)
mem) mem)
) )
(Store [16] dst (ComplexMake real imag) mem) -> (Store {t} dst (ComplexMake real imag) mem) && t.(Type).Size() == 16 ->
(Store [8] {config.fe.TypeFloat64()} (Store {config.fe.TypeFloat64()}
(OffPtr <config.fe.TypeFloat64().PtrTo()> [8] dst) (OffPtr <config.fe.TypeFloat64().PtrTo()> [8] dst)
imag imag
(Store [8] {config.fe.TypeFloat64()} dst real mem)) (Store {config.fe.TypeFloat64()} dst real mem))
// string ops // string ops
(StringPtr (StringMake ptr _)) -> ptr (StringPtr (StringMake ptr _)) -> ptr
...@@ -46,11 +46,11 @@ ...@@ -46,11 +46,11 @@
(Load <config.fe.TypeInt()> (Load <config.fe.TypeInt()>
(OffPtr <config.fe.TypeInt().PtrTo()> [config.PtrSize] ptr) (OffPtr <config.fe.TypeInt().PtrTo()> [config.PtrSize] ptr)
mem)) mem))
(Store [2*config.PtrSize] dst (StringMake ptr len) mem) -> (Store dst (StringMake ptr len) mem) ->
(Store [config.PtrSize] {config.fe.TypeInt()} (Store {config.fe.TypeInt()}
(OffPtr <config.fe.TypeInt().PtrTo()> [config.PtrSize] dst) (OffPtr <config.fe.TypeInt().PtrTo()> [config.PtrSize] dst)
len len
(Store [config.PtrSize] {config.fe.TypeBytePtr()} dst ptr mem)) (Store {config.fe.TypeBytePtr()} dst ptr mem))
// slice ops // slice ops
(SlicePtr (SliceMake ptr _ _ )) -> ptr (SlicePtr (SliceMake ptr _ _ )) -> ptr
...@@ -66,14 +66,14 @@ ...@@ -66,14 +66,14 @@
(Load <config.fe.TypeInt()> (Load <config.fe.TypeInt()>
(OffPtr <config.fe.TypeInt().PtrTo()> [2*config.PtrSize] ptr) (OffPtr <config.fe.TypeInt().PtrTo()> [2*config.PtrSize] ptr)
mem)) mem))
(Store [3*config.PtrSize] dst (SliceMake ptr len cap) mem) -> (Store dst (SliceMake ptr len cap) mem) ->
(Store [config.PtrSize] {config.fe.TypeInt()} (Store {config.fe.TypeInt()}
(OffPtr <config.fe.TypeInt().PtrTo()> [2*config.PtrSize] dst) (OffPtr <config.fe.TypeInt().PtrTo()> [2*config.PtrSize] dst)
cap cap
(Store [config.PtrSize] {config.fe.TypeInt()} (Store {config.fe.TypeInt()}
(OffPtr <config.fe.TypeInt().PtrTo()> [config.PtrSize] dst) (OffPtr <config.fe.TypeInt().PtrTo()> [config.PtrSize] dst)
len len
(Store [config.PtrSize] {config.fe.TypeBytePtr()} dst ptr mem))) (Store {config.fe.TypeBytePtr()} dst ptr mem)))
// interface ops // interface ops
(ITab (IMake itab _)) -> itab (ITab (IMake itab _)) -> itab
...@@ -85,8 +85,8 @@ ...@@ -85,8 +85,8 @@
(Load <config.fe.TypeBytePtr()> (Load <config.fe.TypeBytePtr()>
(OffPtr <config.fe.TypeBytePtr().PtrTo()> [config.PtrSize] ptr) (OffPtr <config.fe.TypeBytePtr().PtrTo()> [config.PtrSize] ptr)
mem)) mem))
(Store [2*config.PtrSize] dst (IMake itab data) mem) -> (Store dst (IMake itab data) mem) ->
(Store [config.PtrSize] {config.fe.TypeBytePtr()} (Store {config.fe.TypeBytePtr()}
(OffPtr <config.fe.TypeBytePtr().PtrTo()> [config.PtrSize] dst) (OffPtr <config.fe.TypeBytePtr().PtrTo()> [config.PtrSize] dst)
data data
(Store [config.PtrSize] {config.fe.TypeUintptr()} dst itab mem)) (Store {config.fe.TypeUintptr()} dst itab mem))
...@@ -30,17 +30,17 @@ ...@@ -30,17 +30,17 @@
(Load <config.fe.TypeUInt32()> ptr mem) (Load <config.fe.TypeUInt32()> ptr mem)
(Load <config.fe.TypeUInt32()> (OffPtr <config.fe.TypeUInt32().PtrTo()> [4] ptr) mem)) (Load <config.fe.TypeUInt32()> (OffPtr <config.fe.TypeUInt32().PtrTo()> [4] ptr) mem))
(Store [8] dst (Int64Make hi lo) mem) && !config.BigEndian -> (Store {t} dst (Int64Make hi lo) mem) && t.(Type).Size() == 8 && !config.BigEndian ->
(Store [4] {hi.Type} (Store {hi.Type}
(OffPtr <hi.Type.PtrTo()> [4] dst) (OffPtr <hi.Type.PtrTo()> [4] dst)
hi hi
(Store [4] {lo.Type} dst lo mem)) (Store {lo.Type} dst lo mem))
(Store [8] dst (Int64Make hi lo) mem) && config.BigEndian -> (Store {t} dst (Int64Make hi lo) mem) && t.(Type).Size() == 8 && config.BigEndian ->
(Store [4] {lo.Type} (Store {lo.Type}
(OffPtr <lo.Type.PtrTo()> [4] dst) (OffPtr <lo.Type.PtrTo()> [4] dst)
lo lo
(Store [4] {hi.Type} dst hi mem)) (Store {hi.Type} dst hi mem))
(Arg {n} [off]) && is64BitInt(v.Type) && !config.BigEndian && v.Type.IsSigned() -> (Arg {n} [off]) && is64BitInt(v.Type) && !config.BigEndian && v.Type.IsSigned() ->
(Int64Make (Int64Make
......
...@@ -746,7 +746,7 @@ ...@@ -746,7 +746,7 @@
(NeqSlice x y) -> (NeqPtr (SlicePtr x) (SlicePtr y)) (NeqSlice x y) -> (NeqPtr (SlicePtr x) (SlicePtr y))
// Load of store of same address, with compatibly typed value and same size // Load of store of same address, with compatibly typed value and same size
(Load <t1> p1 (Store [w] p2 x _)) && isSamePtr(p1,p2) && t1.Compare(x.Type)==CMPeq && w == t1.Size() -> x (Load <t1> p1 (Store {t2} p2 x _)) && isSamePtr(p1,p2) && t1.Compare(x.Type)==CMPeq && t1.Size() == t2.(Type).Size() -> x
// Collapse OffPtr // Collapse OffPtr
(OffPtr (OffPtr p [b]) [a]) -> (OffPtr p [a+b]) (OffPtr (OffPtr p [b]) [a]) -> (OffPtr p [a+b])
...@@ -795,35 +795,35 @@ ...@@ -795,35 +795,35 @@
(Store _ (StructMake0) mem) -> mem (Store _ (StructMake0) mem) -> mem
(Store dst (StructMake1 <t> f0) mem) -> (Store dst (StructMake1 <t> f0) mem) ->
(Store [t.FieldType(0).Size()] {t.FieldType(0)} (OffPtr <t.FieldType(0).PtrTo()> [0] dst) f0 mem) (Store {t.FieldType(0)} (OffPtr <t.FieldType(0).PtrTo()> [0] dst) f0 mem)
(Store dst (StructMake2 <t> f0 f1) mem) -> (Store dst (StructMake2 <t> f0 f1) mem) ->
(Store [t.FieldType(1).Size()] {t.FieldType(1)} (Store {t.FieldType(1)}
(OffPtr <t.FieldType(1).PtrTo()> [t.FieldOff(1)] dst) (OffPtr <t.FieldType(1).PtrTo()> [t.FieldOff(1)] dst)
f1 f1
(Store [t.FieldType(0).Size()] {t.FieldType(0)} (Store {t.FieldType(0)}
(OffPtr <t.FieldType(0).PtrTo()> [0] dst) (OffPtr <t.FieldType(0).PtrTo()> [0] dst)
f0 mem)) f0 mem))
(Store dst (StructMake3 <t> f0 f1 f2) mem) -> (Store dst (StructMake3 <t> f0 f1 f2) mem) ->
(Store [t.FieldType(2).Size()] {t.FieldType(2)} (Store {t.FieldType(2)}
(OffPtr <t.FieldType(2).PtrTo()> [t.FieldOff(2)] dst) (OffPtr <t.FieldType(2).PtrTo()> [t.FieldOff(2)] dst)
f2 f2
(Store [t.FieldType(1).Size()] {t.FieldType(1)} (Store {t.FieldType(1)}
(OffPtr <t.FieldType(1).PtrTo()> [t.FieldOff(1)] dst) (OffPtr <t.FieldType(1).PtrTo()> [t.FieldOff(1)] dst)
f1 f1
(Store [t.FieldType(0).Size()] {t.FieldType(0)} (Store {t.FieldType(0)}
(OffPtr <t.FieldType(0).PtrTo()> [0] dst) (OffPtr <t.FieldType(0).PtrTo()> [0] dst)
f0 mem))) f0 mem)))
(Store dst (StructMake4 <t> f0 f1 f2 f3) mem) -> (Store dst (StructMake4 <t> f0 f1 f2 f3) mem) ->
(Store [t.FieldType(3).Size()] {t.FieldType(3)} (Store {t.FieldType(3)}
(OffPtr <t.FieldType(3).PtrTo()> [t.FieldOff(3)] dst) (OffPtr <t.FieldType(3).PtrTo()> [t.FieldOff(3)] dst)
f3 f3
(Store [t.FieldType(2).Size()] {t.FieldType(2)} (Store {t.FieldType(2)}
(OffPtr <t.FieldType(2).PtrTo()> [t.FieldOff(2)] dst) (OffPtr <t.FieldType(2).PtrTo()> [t.FieldOff(2)] dst)
f2 f2
(Store [t.FieldType(1).Size()] {t.FieldType(1)} (Store {t.FieldType(1)}
(OffPtr <t.FieldType(1).PtrTo()> [t.FieldOff(1)] dst) (OffPtr <t.FieldType(1).PtrTo()> [t.FieldOff(1)] dst)
f1 f1
(Store [t.FieldType(0).Size()] {t.FieldType(0)} (Store {t.FieldType(0)}
(OffPtr <t.FieldType(0).PtrTo()> [0] dst) (OffPtr <t.FieldType(0).PtrTo()> [0] dst)
f0 mem)))) f0 mem))))
...@@ -832,10 +832,10 @@ ...@@ -832,10 +832,10 @@
(StructSelect [0] x:(IData _)) -> x (StructSelect [0] x:(IData _)) -> x
// un-SSAable values use mem->mem copies // un-SSAable values use mem->mem copies
(Store [size] dst (Load <t> src mem) mem) && !config.fe.CanSSA(t) -> (Store {t} dst (Load src mem) mem) && !config.fe.CanSSA(t.(Type)) ->
(Move [MakeSizeAndAlign(size, t.Alignment()).Int64()] {t} dst src mem) (Move {t} [t.(Type).Size()] dst src mem)
(Store [size] dst (Load <t> src mem) (VarDef {x} mem)) && !config.fe.CanSSA(t) -> (Store {t} dst (Load src mem) (VarDef {x} mem)) && !config.fe.CanSSA(t.(Type)) ->
(Move [MakeSizeAndAlign(size, t.Alignment()).Int64()] {t} dst src (VarDef {x} mem)) (Move {t} [t.(Type).Size()] dst src (VarDef {x} mem))
// array ops // array ops
(ArraySelect (ArrayMake1 x)) -> x (ArraySelect (ArrayMake1 x)) -> x
...@@ -847,7 +847,7 @@ ...@@ -847,7 +847,7 @@
(ArrayMake1 (Load <t.ElemType()> ptr mem)) (ArrayMake1 (Load <t.ElemType()> ptr mem))
(Store _ (ArrayMake0) mem) -> mem (Store _ (ArrayMake0) mem) -> mem
(Store [size] dst (ArrayMake1 e) mem) -> (Store [size] {e.Type} dst e mem) (Store dst (ArrayMake1 e) mem) -> (Store {e.Type} dst e mem)
(ArraySelect [0] (Load ptr mem)) -> (Load ptr mem) (ArraySelect [0] (Load ptr mem)) -> (Load ptr mem)
......
...@@ -287,15 +287,15 @@ var genericOps = []opData{ ...@@ -287,15 +287,15 @@ var genericOps = []opData{
// Memory operations // Memory operations
{name: "Load", argLength: 2}, // Load from arg0. arg1=memory {name: "Load", argLength: 2}, // Load from arg0. arg1=memory
{name: "Store", argLength: 3, typ: "Mem", aux: "SymOff", symEffect: "None"}, // Store arg1 to arg0. arg2=memory, auxint=size, aux=type. Returns memory. {name: "Store", argLength: 3, typ: "Mem", aux: "Typ"}, // Store arg1 to arg0. arg2=memory, aux=type. Returns memory.
{name: "Move", argLength: 3, typ: "Mem", aux: "SymSizeAndAlign", symEffect: "None"}, // arg0=destptr, arg1=srcptr, arg2=mem, auxint=size+alignment, aux=type. Returns memory. {name: "Move", argLength: 3, typ: "Mem", aux: "TypSize"}, // arg0=destptr, arg1=srcptr, arg2=mem, auxint=size, aux=type. Returns memory.
{name: "Zero", argLength: 2, typ: "Mem", aux: "SymSizeAndAlign", symEffect: "None"}, // arg0=destptr, arg1=mem, auxint=size+alignment, aux=type. Returns memory. {name: "Zero", argLength: 2, typ: "Mem", aux: "TypSize"}, // arg0=destptr, arg1=mem, auxint=size, aux=type. Returns memory.
// Memory operations with write barriers. // Memory operations with write barriers.
// Expand to runtime calls. Write barrier will be removed if write on stack. // Expand to runtime calls. Write barrier will be removed if write on stack.
{name: "StoreWB", argLength: 3, typ: "Mem", aux: "SymOff", symEffect: "None"}, // Store arg1 to arg0. arg2=memory, auxint=size, aux=type. Returns memory. {name: "StoreWB", argLength: 3, typ: "Mem", aux: "Typ"}, // Store arg1 to arg0. arg2=memory, aux=type. Returns memory.
{name: "MoveWB", argLength: 3, typ: "Mem", aux: "SymSizeAndAlign", symEffect: "None"}, // arg0=destptr, arg1=srcptr, arg2=mem, auxint=size+alignment, aux=type. Returns memory. {name: "MoveWB", argLength: 3, typ: "Mem", aux: "TypSize"}, // arg0=destptr, arg1=srcptr, arg2=mem, auxint=size, aux=type. Returns memory.
{name: "ZeroWB", argLength: 2, typ: "Mem", aux: "SymSizeAndAlign", symEffect: "None"}, // arg0=destptr, arg1=mem, auxint=size+alignment, aux=type. Returns memory. {name: "ZeroWB", argLength: 2, typ: "Mem", aux: "TypSize"}, // arg0=destptr, arg1=mem, auxint=size, aux=type. Returns memory.
// Function calls. Arguments to the call have already been written to the stack. // Function calls. Arguments to the call have already been written to the stack.
// Return values appear on the stack. The method receiver, if any, is treated // Return values appear on the stack. The method receiver, if any, is treated
......
...@@ -657,14 +657,14 @@ func parseValue(val string, arch arch, loc string) (op opData, oparch string, ty ...@@ -657,14 +657,14 @@ func parseValue(val string, arch arch, loc string) (op opData, oparch string, ty
// Sanity check aux, auxint. // Sanity check aux, auxint.
if auxint != "" { if auxint != "" {
switch op.aux { switch op.aux {
case "Bool", "Int8", "Int16", "Int32", "Int64", "Int128", "Float32", "Float64", "SymOff", "SymValAndOff", "SymInt32", "SizeAndAlign", "SymSizeAndAlign": case "Bool", "Int8", "Int16", "Int32", "Int64", "Int128", "Float32", "Float64", "SymOff", "SymValAndOff", "SymInt32", "TypSize":
default: default:
log.Fatalf("%s: op %s %s can't have auxint", loc, op.name, op.aux) log.Fatalf("%s: op %s %s can't have auxint", loc, op.name, op.aux)
} }
} }
if aux != "" { if aux != "" {
switch op.aux { switch op.aux {
case "String", "Sym", "SymOff", "SymValAndOff", "SymInt32", "SymSizeAndAlign": case "String", "Sym", "SymOff", "SymValAndOff", "SymInt32", "Typ", "TypSize":
default: default:
log.Fatalf("%s: op %s %s can't have aux", loc, op.name, op.aux) log.Fatalf("%s: op %s %s can't have aux", loc, op.name, op.aux)
} }
......
...@@ -66,7 +66,7 @@ func TestLoopConditionS390X(t *testing.T) { ...@@ -66,7 +66,7 @@ func TestLoopConditionS390X(t *testing.T) {
Goto("b1")), Goto("b1")),
Bloc("b3", Bloc("b3",
Valu("retdef", OpVarDef, TypeMem, 0, nil, "mem"), Valu("retdef", OpVarDef, TypeMem, 0, nil, "mem"),
Valu("store", OpStore, TypeMem, 8, TypeInt64, "ret", "phisum", "retdef"), Valu("store", OpStore, TypeMem, 0, TypeInt64, "ret", "phisum", "retdef"),
Exit("store"))) Exit("store")))
CheckFunc(fun.f) CheckFunc(fun.f)
Compile(fun.f) Compile(fun.f)
......
...@@ -405,7 +405,7 @@ func TestNilcheckBug(t *testing.T) { ...@@ -405,7 +405,7 @@ func TestNilcheckBug(t *testing.T) {
If("bool2", "extra", "exit")), If("bool2", "extra", "exit")),
Bloc("extra", Bloc("extra",
// prevent fuse from eliminating this block // prevent fuse from eliminating this block
Valu("store", OpStore, TypeMem, 8, nil, "ptr1", "nilptr", "mem"), Valu("store", OpStore, TypeMem, 0, ptrType, "ptr1", "nilptr", "mem"),
Goto("exit")), Goto("exit")),
Bloc("exit", Bloc("exit",
Valu("phi", OpPhi, TypeMem, 0, nil, "mem", "store"), Valu("phi", OpPhi, TypeMem, 0, nil, "mem", "store"),
......
...@@ -66,12 +66,12 @@ const ( ...@@ -66,12 +66,12 @@ const (
auxInt128 // auxInt represents a 128-bit integer. Always 0. auxInt128 // auxInt represents a 128-bit integer. Always 0.
auxFloat32 // auxInt is a float32 (encoded with math.Float64bits) auxFloat32 // auxInt is a float32 (encoded with math.Float64bits)
auxFloat64 // auxInt is a float64 (encoded with math.Float64bits) auxFloat64 // auxInt is a float64 (encoded with math.Float64bits)
auxSizeAndAlign // auxInt is a SizeAndAlign
auxString // aux is a string auxString // aux is a string
auxSym // aux is a symbol auxSym // aux is a symbol
auxSymOff // aux is a symbol, auxInt is an offset auxSymOff // aux is a symbol, auxInt is an offset
auxSymValAndOff // aux is a symbol, auxInt is a ValAndOff auxSymValAndOff // aux is a symbol, auxInt is a ValAndOff
auxSymSizeAndAlign // aux is a symbol, auxInt is a SizeAndAlign auxTyp // aux is a type
auxTypSize // aux is a type, auxInt is a size, must have Aux.(Type).Size() == AuxInt
auxSymInt32 // aux is a symbol, auxInt is a 32-bit integer auxSymInt32 // aux is a symbol, auxInt is a 32-bit integer
) )
...@@ -154,31 +154,3 @@ func (x ValAndOff) add(off int64) int64 { ...@@ -154,31 +154,3 @@ func (x ValAndOff) add(off int64) int64 {
} }
return makeValAndOff(x.Val(), x.Off()+off) return makeValAndOff(x.Val(), x.Off()+off)
} }
// SizeAndAlign holds both the size and the alignment of a type,
// used in Zero and Move ops.
// The high 8 bits hold the alignment.
// The low 56 bits hold the size.
type SizeAndAlign int64
func (x SizeAndAlign) Size() int64 {
return int64(x) & (1<<56 - 1)
}
func (x SizeAndAlign) Align() int64 {
return int64(uint64(x) >> 56)
}
func (x SizeAndAlign) Int64() int64 {
return int64(x)
}
func (x SizeAndAlign) String() string {
return fmt.Sprintf("size=%d,align=%d", x.Size(), x.Align())
}
func MakeSizeAndAlign(size, align int64) SizeAndAlign {
if size&^(1<<56-1) != 0 {
panic("size too big in SizeAndAlign")
}
if align >= 1<<8 {
panic("alignment too big in SizeAndAlign")
}
return SizeAndAlign(size | align<<56)
}
...@@ -21558,44 +21558,38 @@ var opcodeTable = [...]opInfo{ ...@@ -21558,44 +21558,38 @@ var opcodeTable = [...]opInfo{
}, },
{ {
name: "Store", name: "Store",
auxType: auxSymOff, auxType: auxTyp,
argLen: 3, argLen: 3,
symEffect: SymNone,
generic: true, generic: true,
}, },
{ {
name: "Move", name: "Move",
auxType: auxSymSizeAndAlign, auxType: auxTypSize,
argLen: 3, argLen: 3,
symEffect: SymNone,
generic: true, generic: true,
}, },
{ {
name: "Zero", name: "Zero",
auxType: auxSymSizeAndAlign, auxType: auxTypSize,
argLen: 2, argLen: 2,
symEffect: SymNone,
generic: true, generic: true,
}, },
{ {
name: "StoreWB", name: "StoreWB",
auxType: auxSymOff, auxType: auxTyp,
argLen: 3, argLen: 3,
symEffect: SymNone,
generic: true, generic: true,
}, },
{ {
name: "MoveWB", name: "MoveWB",
auxType: auxSymSizeAndAlign, auxType: auxTypSize,
argLen: 3, argLen: 3,
symEffect: SymNone,
generic: true, generic: true,
}, },
{ {
name: "ZeroWB", name: "ZeroWB",
auxType: auxSymSizeAndAlign, auxType: auxTypSize,
argLen: 2, argLen: 2,
symEffect: SymNone,
generic: true, generic: true,
}, },
{ {
......
...@@ -77,15 +77,15 @@ func genFunction(size int) []bloc { ...@@ -77,15 +77,15 @@ func genFunction(size int) []bloc {
Valu(valn("addr", i, 1), OpAddr, ptrType, 0, nil, "sb"), Valu(valn("addr", i, 1), OpAddr, ptrType, 0, nil, "sb"),
Valu(valn("addr", i, 2), OpAddr, ptrType, 0, nil, "sb"), Valu(valn("addr", i, 2), OpAddr, ptrType, 0, nil, "sb"),
Valu(valn("addr", i, 3), OpAddr, ptrType, 0, nil, "sb"), Valu(valn("addr", i, 3), OpAddr, ptrType, 0, nil, "sb"),
Valu(valn("zero", i, 1), OpZero, TypeMem, 8, nil, valn("addr", i, 3), Valu(valn("zero", i, 1), OpZero, TypeMem, 8, elemType, valn("addr", i, 3),
valn("store", i-1, 4)), valn("store", i-1, 4)),
Valu(valn("store", i, 1), OpStore, TypeMem, 0, nil, valn("addr", i, 1), Valu(valn("store", i, 1), OpStore, TypeMem, 0, elemType, valn("addr", i, 1),
valn("v", i, 0), valn("zero", i, 1)), valn("v", i, 0), valn("zero", i, 1)),
Valu(valn("store", i, 2), OpStore, TypeMem, 0, nil, valn("addr", i, 2), Valu(valn("store", i, 2), OpStore, TypeMem, 0, elemType, valn("addr", i, 2),
valn("v", i, 0), valn("store", i, 1)), valn("v", i, 0), valn("store", i, 1)),
Valu(valn("store", i, 3), OpStore, TypeMem, 0, nil, valn("addr", i, 1), Valu(valn("store", i, 3), OpStore, TypeMem, 0, elemType, valn("addr", i, 1),
valn("v", i, 0), valn("store", i, 2)), valn("v", i, 0), valn("store", i, 2)),
Valu(valn("store", i, 4), OpStore, TypeMem, 0, nil, valn("addr", i, 3), Valu(valn("store", i, 4), OpStore, TypeMem, 0, elemType, valn("addr", i, 3),
valn("v", i, 0), valn("store", i, 3)), valn("v", i, 0), valn("store", i, 3)),
Goto(blockn(i+1)))) Goto(blockn(i+1))))
} }
......
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
...@@ -301,13 +301,11 @@ func rewriteValuedec_OpSlicePtr(v *Value, config *Config) bool { ...@@ -301,13 +301,11 @@ func rewriteValuedec_OpSlicePtr(v *Value, config *Config) bool {
func rewriteValuedec_OpStore(v *Value, config *Config) bool { func rewriteValuedec_OpStore(v *Value, config *Config) bool {
b := v.Block b := v.Block
_ = b _ = b
// match: (Store [8] dst (ComplexMake real imag) mem) // match: (Store {t} dst (ComplexMake real imag) mem)
// cond: // cond: t.(Type).Size() == 8
// result: (Store [4] {config.fe.TypeFloat32()} (OffPtr <config.fe.TypeFloat32().PtrTo()> [4] dst) imag (Store [4] {config.fe.TypeFloat32()} dst real mem)) // result: (Store {config.fe.TypeFloat32()} (OffPtr <config.fe.TypeFloat32().PtrTo()> [4] dst) imag (Store {config.fe.TypeFloat32()} dst real mem))
for { for {
if v.AuxInt != 8 { t := v.Aux
break
}
dst := v.Args[0] dst := v.Args[0]
v_1 := v.Args[1] v_1 := v.Args[1]
if v_1.Op != OpComplexMake { if v_1.Op != OpComplexMake {
...@@ -316,8 +314,10 @@ func rewriteValuedec_OpStore(v *Value, config *Config) bool { ...@@ -316,8 +314,10 @@ func rewriteValuedec_OpStore(v *Value, config *Config) bool {
real := v_1.Args[0] real := v_1.Args[0]
imag := v_1.Args[1] imag := v_1.Args[1]
mem := v.Args[2] mem := v.Args[2]
if !(t.(Type).Size() == 8) {
break
}
v.reset(OpStore) v.reset(OpStore)
v.AuxInt = 4
v.Aux = config.fe.TypeFloat32() v.Aux = config.fe.TypeFloat32()
v0 := b.NewValue0(v.Pos, OpOffPtr, config.fe.TypeFloat32().PtrTo()) v0 := b.NewValue0(v.Pos, OpOffPtr, config.fe.TypeFloat32().PtrTo())
v0.AuxInt = 4 v0.AuxInt = 4
...@@ -325,7 +325,6 @@ func rewriteValuedec_OpStore(v *Value, config *Config) bool { ...@@ -325,7 +325,6 @@ func rewriteValuedec_OpStore(v *Value, config *Config) bool {
v.AddArg(v0) v.AddArg(v0)
v.AddArg(imag) v.AddArg(imag)
v1 := b.NewValue0(v.Pos, OpStore, TypeMem) v1 := b.NewValue0(v.Pos, OpStore, TypeMem)
v1.AuxInt = 4
v1.Aux = config.fe.TypeFloat32() v1.Aux = config.fe.TypeFloat32()
v1.AddArg(dst) v1.AddArg(dst)
v1.AddArg(real) v1.AddArg(real)
...@@ -333,13 +332,11 @@ func rewriteValuedec_OpStore(v *Value, config *Config) bool { ...@@ -333,13 +332,11 @@ func rewriteValuedec_OpStore(v *Value, config *Config) bool {
v.AddArg(v1) v.AddArg(v1)
return true return true
} }
// match: (Store [16] dst (ComplexMake real imag) mem) // match: (Store {t} dst (ComplexMake real imag) mem)
// cond: // cond: t.(Type).Size() == 16
// result: (Store [8] {config.fe.TypeFloat64()} (OffPtr <config.fe.TypeFloat64().PtrTo()> [8] dst) imag (Store [8] {config.fe.TypeFloat64()} dst real mem)) // result: (Store {config.fe.TypeFloat64()} (OffPtr <config.fe.TypeFloat64().PtrTo()> [8] dst) imag (Store {config.fe.TypeFloat64()} dst real mem))
for { for {
if v.AuxInt != 16 { t := v.Aux
break
}
dst := v.Args[0] dst := v.Args[0]
v_1 := v.Args[1] v_1 := v.Args[1]
if v_1.Op != OpComplexMake { if v_1.Op != OpComplexMake {
...@@ -348,8 +345,10 @@ func rewriteValuedec_OpStore(v *Value, config *Config) bool { ...@@ -348,8 +345,10 @@ func rewriteValuedec_OpStore(v *Value, config *Config) bool {
real := v_1.Args[0] real := v_1.Args[0]
imag := v_1.Args[1] imag := v_1.Args[1]
mem := v.Args[2] mem := v.Args[2]
if !(t.(Type).Size() == 16) {
break
}
v.reset(OpStore) v.reset(OpStore)
v.AuxInt = 8
v.Aux = config.fe.TypeFloat64() v.Aux = config.fe.TypeFloat64()
v0 := b.NewValue0(v.Pos, OpOffPtr, config.fe.TypeFloat64().PtrTo()) v0 := b.NewValue0(v.Pos, OpOffPtr, config.fe.TypeFloat64().PtrTo())
v0.AuxInt = 8 v0.AuxInt = 8
...@@ -357,7 +356,6 @@ func rewriteValuedec_OpStore(v *Value, config *Config) bool { ...@@ -357,7 +356,6 @@ func rewriteValuedec_OpStore(v *Value, config *Config) bool {
v.AddArg(v0) v.AddArg(v0)
v.AddArg(imag) v.AddArg(imag)
v1 := b.NewValue0(v.Pos, OpStore, TypeMem) v1 := b.NewValue0(v.Pos, OpStore, TypeMem)
v1.AuxInt = 8
v1.Aux = config.fe.TypeFloat64() v1.Aux = config.fe.TypeFloat64()
v1.AddArg(dst) v1.AddArg(dst)
v1.AddArg(real) v1.AddArg(real)
...@@ -365,13 +363,10 @@ func rewriteValuedec_OpStore(v *Value, config *Config) bool { ...@@ -365,13 +363,10 @@ func rewriteValuedec_OpStore(v *Value, config *Config) bool {
v.AddArg(v1) v.AddArg(v1)
return true return true
} }
// match: (Store [2*config.PtrSize] dst (StringMake ptr len) mem) // match: (Store dst (StringMake ptr len) mem)
// cond: // cond:
// result: (Store [config.PtrSize] {config.fe.TypeInt()} (OffPtr <config.fe.TypeInt().PtrTo()> [config.PtrSize] dst) len (Store [config.PtrSize] {config.fe.TypeBytePtr()} dst ptr mem)) // result: (Store {config.fe.TypeInt()} (OffPtr <config.fe.TypeInt().PtrTo()> [config.PtrSize] dst) len (Store {config.fe.TypeBytePtr()} dst ptr mem))
for { for {
if v.AuxInt != 2*config.PtrSize {
break
}
dst := v.Args[0] dst := v.Args[0]
v_1 := v.Args[1] v_1 := v.Args[1]
if v_1.Op != OpStringMake { if v_1.Op != OpStringMake {
...@@ -381,7 +376,6 @@ func rewriteValuedec_OpStore(v *Value, config *Config) bool { ...@@ -381,7 +376,6 @@ func rewriteValuedec_OpStore(v *Value, config *Config) bool {
len := v_1.Args[1] len := v_1.Args[1]
mem := v.Args[2] mem := v.Args[2]
v.reset(OpStore) v.reset(OpStore)
v.AuxInt = config.PtrSize
v.Aux = config.fe.TypeInt() v.Aux = config.fe.TypeInt()
v0 := b.NewValue0(v.Pos, OpOffPtr, config.fe.TypeInt().PtrTo()) v0 := b.NewValue0(v.Pos, OpOffPtr, config.fe.TypeInt().PtrTo())
v0.AuxInt = config.PtrSize v0.AuxInt = config.PtrSize
...@@ -389,7 +383,6 @@ func rewriteValuedec_OpStore(v *Value, config *Config) bool { ...@@ -389,7 +383,6 @@ func rewriteValuedec_OpStore(v *Value, config *Config) bool {
v.AddArg(v0) v.AddArg(v0)
v.AddArg(len) v.AddArg(len)
v1 := b.NewValue0(v.Pos, OpStore, TypeMem) v1 := b.NewValue0(v.Pos, OpStore, TypeMem)
v1.AuxInt = config.PtrSize
v1.Aux = config.fe.TypeBytePtr() v1.Aux = config.fe.TypeBytePtr()
v1.AddArg(dst) v1.AddArg(dst)
v1.AddArg(ptr) v1.AddArg(ptr)
...@@ -397,13 +390,10 @@ func rewriteValuedec_OpStore(v *Value, config *Config) bool { ...@@ -397,13 +390,10 @@ func rewriteValuedec_OpStore(v *Value, config *Config) bool {
v.AddArg(v1) v.AddArg(v1)
return true return true
} }
// match: (Store [3*config.PtrSize] dst (SliceMake ptr len cap) mem) // match: (Store dst (SliceMake ptr len cap) mem)
// cond: // cond:
// result: (Store [config.PtrSize] {config.fe.TypeInt()} (OffPtr <config.fe.TypeInt().PtrTo()> [2*config.PtrSize] dst) cap (Store [config.PtrSize] {config.fe.TypeInt()} (OffPtr <config.fe.TypeInt().PtrTo()> [config.PtrSize] dst) len (Store [config.PtrSize] {config.fe.TypeBytePtr()} dst ptr mem))) // result: (Store {config.fe.TypeInt()} (OffPtr <config.fe.TypeInt().PtrTo()> [2*config.PtrSize] dst) cap (Store {config.fe.TypeInt()} (OffPtr <config.fe.TypeInt().PtrTo()> [config.PtrSize] dst) len (Store {config.fe.TypeBytePtr()} dst ptr mem)))
for { for {
if v.AuxInt != 3*config.PtrSize {
break
}
dst := v.Args[0] dst := v.Args[0]
v_1 := v.Args[1] v_1 := v.Args[1]
if v_1.Op != OpSliceMake { if v_1.Op != OpSliceMake {
...@@ -414,7 +404,6 @@ func rewriteValuedec_OpStore(v *Value, config *Config) bool { ...@@ -414,7 +404,6 @@ func rewriteValuedec_OpStore(v *Value, config *Config) bool {
cap := v_1.Args[2] cap := v_1.Args[2]
mem := v.Args[2] mem := v.Args[2]
v.reset(OpStore) v.reset(OpStore)
v.AuxInt = config.PtrSize
v.Aux = config.fe.TypeInt() v.Aux = config.fe.TypeInt()
v0 := b.NewValue0(v.Pos, OpOffPtr, config.fe.TypeInt().PtrTo()) v0 := b.NewValue0(v.Pos, OpOffPtr, config.fe.TypeInt().PtrTo())
v0.AuxInt = 2 * config.PtrSize v0.AuxInt = 2 * config.PtrSize
...@@ -422,7 +411,6 @@ func rewriteValuedec_OpStore(v *Value, config *Config) bool { ...@@ -422,7 +411,6 @@ func rewriteValuedec_OpStore(v *Value, config *Config) bool {
v.AddArg(v0) v.AddArg(v0)
v.AddArg(cap) v.AddArg(cap)
v1 := b.NewValue0(v.Pos, OpStore, TypeMem) v1 := b.NewValue0(v.Pos, OpStore, TypeMem)
v1.AuxInt = config.PtrSize
v1.Aux = config.fe.TypeInt() v1.Aux = config.fe.TypeInt()
v2 := b.NewValue0(v.Pos, OpOffPtr, config.fe.TypeInt().PtrTo()) v2 := b.NewValue0(v.Pos, OpOffPtr, config.fe.TypeInt().PtrTo())
v2.AuxInt = config.PtrSize v2.AuxInt = config.PtrSize
...@@ -430,7 +418,6 @@ func rewriteValuedec_OpStore(v *Value, config *Config) bool { ...@@ -430,7 +418,6 @@ func rewriteValuedec_OpStore(v *Value, config *Config) bool {
v1.AddArg(v2) v1.AddArg(v2)
v1.AddArg(len) v1.AddArg(len)
v3 := b.NewValue0(v.Pos, OpStore, TypeMem) v3 := b.NewValue0(v.Pos, OpStore, TypeMem)
v3.AuxInt = config.PtrSize
v3.Aux = config.fe.TypeBytePtr() v3.Aux = config.fe.TypeBytePtr()
v3.AddArg(dst) v3.AddArg(dst)
v3.AddArg(ptr) v3.AddArg(ptr)
...@@ -439,13 +426,10 @@ func rewriteValuedec_OpStore(v *Value, config *Config) bool { ...@@ -439,13 +426,10 @@ func rewriteValuedec_OpStore(v *Value, config *Config) bool {
v.AddArg(v1) v.AddArg(v1)
return true return true
} }
// match: (Store [2*config.PtrSize] dst (IMake itab data) mem) // match: (Store dst (IMake itab data) mem)
// cond: // cond:
// result: (Store [config.PtrSize] {config.fe.TypeBytePtr()} (OffPtr <config.fe.TypeBytePtr().PtrTo()> [config.PtrSize] dst) data (Store [config.PtrSize] {config.fe.TypeUintptr()} dst itab mem)) // result: (Store {config.fe.TypeBytePtr()} (OffPtr <config.fe.TypeBytePtr().PtrTo()> [config.PtrSize] dst) data (Store {config.fe.TypeUintptr()} dst itab mem))
for { for {
if v.AuxInt != 2*config.PtrSize {
break
}
dst := v.Args[0] dst := v.Args[0]
v_1 := v.Args[1] v_1 := v.Args[1]
if v_1.Op != OpIMake { if v_1.Op != OpIMake {
...@@ -455,7 +439,6 @@ func rewriteValuedec_OpStore(v *Value, config *Config) bool { ...@@ -455,7 +439,6 @@ func rewriteValuedec_OpStore(v *Value, config *Config) bool {
data := v_1.Args[1] data := v_1.Args[1]
mem := v.Args[2] mem := v.Args[2]
v.reset(OpStore) v.reset(OpStore)
v.AuxInt = config.PtrSize
v.Aux = config.fe.TypeBytePtr() v.Aux = config.fe.TypeBytePtr()
v0 := b.NewValue0(v.Pos, OpOffPtr, config.fe.TypeBytePtr().PtrTo()) v0 := b.NewValue0(v.Pos, OpOffPtr, config.fe.TypeBytePtr().PtrTo())
v0.AuxInt = config.PtrSize v0.AuxInt = config.PtrSize
...@@ -463,7 +446,6 @@ func rewriteValuedec_OpStore(v *Value, config *Config) bool { ...@@ -463,7 +446,6 @@ func rewriteValuedec_OpStore(v *Value, config *Config) bool {
v.AddArg(v0) v.AddArg(v0)
v.AddArg(data) v.AddArg(data)
v1 := b.NewValue0(v.Pos, OpStore, TypeMem) v1 := b.NewValue0(v.Pos, OpStore, TypeMem)
v1.AuxInt = config.PtrSize
v1.Aux = config.fe.TypeUintptr() v1.Aux = config.fe.TypeUintptr()
v1.AddArg(dst) v1.AddArg(dst)
v1.AddArg(itab) v1.AddArg(itab)
......
...@@ -2396,13 +2396,11 @@ func rewriteValuedec64_OpSignExt8to64(v *Value, config *Config) bool { ...@@ -2396,13 +2396,11 @@ func rewriteValuedec64_OpSignExt8to64(v *Value, config *Config) bool {
func rewriteValuedec64_OpStore(v *Value, config *Config) bool { func rewriteValuedec64_OpStore(v *Value, config *Config) bool {
b := v.Block b := v.Block
_ = b _ = b
// match: (Store [8] dst (Int64Make hi lo) mem) // match: (Store {t} dst (Int64Make hi lo) mem)
// cond: !config.BigEndian // cond: t.(Type).Size() == 8 && !config.BigEndian
// result: (Store [4] {hi.Type} (OffPtr <hi.Type.PtrTo()> [4] dst) hi (Store [4] {lo.Type} dst lo mem)) // result: (Store {hi.Type} (OffPtr <hi.Type.PtrTo()> [4] dst) hi (Store {lo.Type} dst lo mem))
for { for {
if v.AuxInt != 8 { t := v.Aux
break
}
dst := v.Args[0] dst := v.Args[0]
v_1 := v.Args[1] v_1 := v.Args[1]
if v_1.Op != OpInt64Make { if v_1.Op != OpInt64Make {
...@@ -2411,11 +2409,10 @@ func rewriteValuedec64_OpStore(v *Value, config *Config) bool { ...@@ -2411,11 +2409,10 @@ func rewriteValuedec64_OpStore(v *Value, config *Config) bool {
hi := v_1.Args[0] hi := v_1.Args[0]
lo := v_1.Args[1] lo := v_1.Args[1]
mem := v.Args[2] mem := v.Args[2]
if !(!config.BigEndian) { if !(t.(Type).Size() == 8 && !config.BigEndian) {
break break
} }
v.reset(OpStore) v.reset(OpStore)
v.AuxInt = 4
v.Aux = hi.Type v.Aux = hi.Type
v0 := b.NewValue0(v.Pos, OpOffPtr, hi.Type.PtrTo()) v0 := b.NewValue0(v.Pos, OpOffPtr, hi.Type.PtrTo())
v0.AuxInt = 4 v0.AuxInt = 4
...@@ -2423,7 +2420,6 @@ func rewriteValuedec64_OpStore(v *Value, config *Config) bool { ...@@ -2423,7 +2420,6 @@ func rewriteValuedec64_OpStore(v *Value, config *Config) bool {
v.AddArg(v0) v.AddArg(v0)
v.AddArg(hi) v.AddArg(hi)
v1 := b.NewValue0(v.Pos, OpStore, TypeMem) v1 := b.NewValue0(v.Pos, OpStore, TypeMem)
v1.AuxInt = 4
v1.Aux = lo.Type v1.Aux = lo.Type
v1.AddArg(dst) v1.AddArg(dst)
v1.AddArg(lo) v1.AddArg(lo)
...@@ -2431,13 +2427,11 @@ func rewriteValuedec64_OpStore(v *Value, config *Config) bool { ...@@ -2431,13 +2427,11 @@ func rewriteValuedec64_OpStore(v *Value, config *Config) bool {
v.AddArg(v1) v.AddArg(v1)
return true return true
} }
// match: (Store [8] dst (Int64Make hi lo) mem) // match: (Store {t} dst (Int64Make hi lo) mem)
// cond: config.BigEndian // cond: t.(Type).Size() == 8 && config.BigEndian
// result: (Store [4] {lo.Type} (OffPtr <lo.Type.PtrTo()> [4] dst) lo (Store [4] {hi.Type} dst hi mem)) // result: (Store {lo.Type} (OffPtr <lo.Type.PtrTo()> [4] dst) lo (Store {hi.Type} dst hi mem))
for { for {
if v.AuxInt != 8 { t := v.Aux
break
}
dst := v.Args[0] dst := v.Args[0]
v_1 := v.Args[1] v_1 := v.Args[1]
if v_1.Op != OpInt64Make { if v_1.Op != OpInt64Make {
...@@ -2446,11 +2440,10 @@ func rewriteValuedec64_OpStore(v *Value, config *Config) bool { ...@@ -2446,11 +2440,10 @@ func rewriteValuedec64_OpStore(v *Value, config *Config) bool {
hi := v_1.Args[0] hi := v_1.Args[0]
lo := v_1.Args[1] lo := v_1.Args[1]
mem := v.Args[2] mem := v.Args[2]
if !(config.BigEndian) { if !(t.(Type).Size() == 8 && config.BigEndian) {
break break
} }
v.reset(OpStore) v.reset(OpStore)
v.AuxInt = 4
v.Aux = lo.Type v.Aux = lo.Type
v0 := b.NewValue0(v.Pos, OpOffPtr, lo.Type.PtrTo()) v0 := b.NewValue0(v.Pos, OpOffPtr, lo.Type.PtrTo())
v0.AuxInt = 4 v0.AuxInt = 4
...@@ -2458,7 +2451,6 @@ func rewriteValuedec64_OpStore(v *Value, config *Config) bool { ...@@ -2458,7 +2451,6 @@ func rewriteValuedec64_OpStore(v *Value, config *Config) bool {
v.AddArg(v0) v.AddArg(v0)
v.AddArg(lo) v.AddArg(lo)
v1 := b.NewValue0(v.Pos, OpStore, TypeMem) v1 := b.NewValue0(v.Pos, OpStore, TypeMem)
v1.AuxInt = 4
v1.Aux = hi.Type v1.Aux = hi.Type
v1.AddArg(dst) v1.AddArg(dst)
v1.AddArg(hi) v1.AddArg(hi)
......
...@@ -14,9 +14,9 @@ func TestSchedule(t *testing.T) { ...@@ -14,9 +14,9 @@ func TestSchedule(t *testing.T) {
Valu("mem0", OpInitMem, TypeMem, 0, nil), Valu("mem0", OpInitMem, TypeMem, 0, nil),
Valu("ptr", OpConst64, TypeInt64, 0xABCD, nil), Valu("ptr", OpConst64, TypeInt64, 0xABCD, nil),
Valu("v", OpConst64, TypeInt64, 12, nil), Valu("v", OpConst64, TypeInt64, 12, nil),
Valu("mem1", OpStore, TypeMem, 8, nil, "ptr", "v", "mem0"), Valu("mem1", OpStore, TypeMem, 0, TypeInt64, "ptr", "v", "mem0"),
Valu("mem2", OpStore, TypeMem, 8, nil, "ptr", "v", "mem1"), Valu("mem2", OpStore, TypeMem, 0, TypeInt64, "ptr", "v", "mem1"),
Valu("mem3", OpStore, TypeInt64, 8, nil, "ptr", "sum", "mem2"), Valu("mem3", OpStore, TypeMem, 0, TypeInt64, "ptr", "sum", "mem2"),
Valu("l1", OpLoad, TypeInt64, 0, nil, "ptr", "mem1"), Valu("l1", OpLoad, TypeInt64, 0, nil, "ptr", "mem1"),
Valu("l2", OpLoad, TypeInt64, 0, nil, "ptr", "mem2"), Valu("l2", OpLoad, TypeInt64, 0, nil, "ptr", "mem2"),
Valu("sum", OpAdd64, TypeInt64, 0, nil, "l1", "l2"), Valu("sum", OpAdd64, TypeInt64, 0, nil, "l1", "l2"),
...@@ -66,8 +66,8 @@ func TestStoreOrder(t *testing.T) { ...@@ -66,8 +66,8 @@ func TestStoreOrder(t *testing.T) {
Valu("a", OpAdd64, TypeInt64, 0, nil, "b", "c"), // v2 Valu("a", OpAdd64, TypeInt64, 0, nil, "b", "c"), // v2
Valu("b", OpLoad, TypeInt64, 0, nil, "ptr", "mem1"), // v3 Valu("b", OpLoad, TypeInt64, 0, nil, "ptr", "mem1"), // v3
Valu("c", OpNeg64, TypeInt64, 0, nil, "b"), // v4 Valu("c", OpNeg64, TypeInt64, 0, nil, "b"), // v4
Valu("mem1", OpStore, TypeMem, 8, nil, "ptr", "v", "mem0"), // v5 Valu("mem1", OpStore, TypeMem, 0, TypeInt64, "ptr", "v", "mem0"), // v5
Valu("mem2", OpStore, TypeMem, 0, nil, "ptr", "a", "mem1"), Valu("mem2", OpStore, TypeMem, 0, TypeInt64, "ptr", "a", "mem1"),
Valu("ptr", OpConst64, TypeInt64, 0xABCD, nil), Valu("ptr", OpConst64, TypeInt64, 0xABCD, nil),
Valu("v", OpConst64, TypeInt64, 12, nil), Valu("v", OpConst64, TypeInt64, 12, nil),
Goto("exit")), Goto("exit")),
......
...@@ -41,7 +41,7 @@ func makeConstShiftFunc(c *Config, amount int64, op Op, typ Type) fun { ...@@ -41,7 +41,7 @@ func makeConstShiftFunc(c *Config, amount int64, op Op, typ Type) fun {
Valu("load", OpLoad, typ, 0, nil, "argptr", "mem"), Valu("load", OpLoad, typ, 0, nil, "argptr", "mem"),
Valu("c", OpConst64, TypeUInt64, amount, nil), Valu("c", OpConst64, TypeUInt64, amount, nil),
Valu("shift", op, typ, 0, nil, "load", "c"), Valu("shift", op, typ, 0, nil, "load", "c"),
Valu("store", OpStore, TypeMem, 8, TypeUInt64, "resptr", "shift", "mem"), Valu("store", OpStore, TypeMem, 0, TypeUInt64, "resptr", "shift", "mem"),
Exit("store"))) Exit("store")))
Compile(fun.f) Compile(fun.f)
return fun return fun
...@@ -101,7 +101,7 @@ func makeShiftExtensionFunc(c *Config, amount int64, lshift, rshift Op, typ Type ...@@ -101,7 +101,7 @@ func makeShiftExtensionFunc(c *Config, amount int64, lshift, rshift Op, typ Type
Valu("c", OpConst64, TypeUInt64, amount, nil), Valu("c", OpConst64, TypeUInt64, amount, nil),
Valu("lshift", lshift, typ, 0, nil, "load", "c"), Valu("lshift", lshift, typ, 0, nil, "load", "c"),
Valu("rshift", rshift, typ, 0, nil, "lshift", "c"), Valu("rshift", rshift, typ, 0, nil, "lshift", "c"),
Valu("store", OpStore, TypeMem, 8, TypeUInt64, "resptr", "rshift", "mem"), Valu("store", OpStore, TypeMem, 0, TypeUInt64, "resptr", "rshift", "mem"),
Exit("store"))) Exit("store")))
Compile(fun.f) Compile(fun.f)
return fun return fun
......
...@@ -128,17 +128,15 @@ func (v *Value) auxString() string { ...@@ -128,17 +128,15 @@ func (v *Value) auxString() string {
return fmt.Sprintf(" [%d]", v.AuxInt32()) return fmt.Sprintf(" [%d]", v.AuxInt32())
case auxInt64, auxInt128: case auxInt64, auxInt128:
return fmt.Sprintf(" [%d]", v.AuxInt) return fmt.Sprintf(" [%d]", v.AuxInt)
case auxSizeAndAlign:
return fmt.Sprintf(" [%s]", SizeAndAlign(v.AuxInt))
case auxFloat32, auxFloat64: case auxFloat32, auxFloat64:
return fmt.Sprintf(" [%g]", v.AuxFloat()) return fmt.Sprintf(" [%g]", v.AuxFloat())
case auxString: case auxString:
return fmt.Sprintf(" {%q}", v.Aux) return fmt.Sprintf(" {%q}", v.Aux)
case auxSym: case auxSym, auxTyp:
if v.Aux != nil { if v.Aux != nil {
return fmt.Sprintf(" {%v}", v.Aux) return fmt.Sprintf(" {%v}", v.Aux)
} }
case auxSymOff, auxSymInt32: case auxSymOff, auxSymInt32, auxTypSize:
s := "" s := ""
if v.Aux != nil { if v.Aux != nil {
s = fmt.Sprintf(" {%v}", v.Aux) s = fmt.Sprintf(" {%v}", v.Aux)
...@@ -153,12 +151,6 @@ func (v *Value) auxString() string { ...@@ -153,12 +151,6 @@ func (v *Value) auxString() string {
s = fmt.Sprintf(" {%v}", v.Aux) s = fmt.Sprintf(" {%v}", v.Aux)
} }
return s + fmt.Sprintf(" [%s]", v.AuxValAndOff()) return s + fmt.Sprintf(" [%s]", v.AuxValAndOff())
case auxSymSizeAndAlign:
s := ""
if v.Aux != nil {
s = fmt.Sprintf(" {%v}", v.Aux)
}
return s + fmt.Sprintf(" [%s]", SizeAndAlign(v.AuxInt))
} }
return "" return ""
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment