cmd/compile: use vsx loads and stores for LoweredMove, LoweredZero on ppc64x

This improves the code generated for LoweredMove and LoweredZero by
using LXVD2X and STXVD2X to move 16 bytes at a time. These instructions
are now used if the size to be moved or zeroed is >= 64. These same
instructions have already been used in the asm implementations for
memmove and memclr.

Some examples where this shows an improvement on power8:

MakeSlice/Byte                                  27.3ns ± 1%     25.2ns ± 0%    -7.69%
MakeSlice/Int16                                 40.2ns ± 0%     35.2ns ± 0%   -12.39%
MakeSlice/Int                                   94.9ns ± 1%     77.9ns ± 0%   -17.92%
MakeSlice/Ptr                                    129ns ± 1%      103ns ± 0%   -20.16%
MakeSlice/Struct/24                              176ns ± 1%      131ns ± 0%   -25.67%
MakeSlice/Struct/32                              200ns ± 1%      142ns ± 0%   -29.09%
MakeSlice/Struct/40                              220ns ± 2%      156ns ± 0%   -28.82%
GrowSlice/Byte                                  81.4ns ± 0%     73.4ns ± 0%    -9.88%
GrowSlice/Int16                                  118ns ± 1%       98ns ± 0%   -17.03%
GrowSlice/Int                                    178ns ± 1%      134ns ± 1%   -24.65%
GrowSlice/Ptr                                    249ns ± 4%      212ns ± 0%   -14.94%
GrowSlice/Struct/24                              294ns ± 5%      215ns ± 0%   -27.08%
GrowSlice/Struct/32                              315ns ± 1%      248ns ± 0%   -21.49%
GrowSlice/Struct/40                              382ns ± 4%      289ns ± 1%   -24.38%
ExtendSlice/IntSlice                             109ns ± 1%       90ns ± 1%   -17.51%
ExtendSlice/PointerSlice                         142ns ± 2%      118ns ± 0%   -16.75%
ExtendSlice/NoGrow                              6.02ns ± 0%     5.88ns ± 0%    -2.33%
Append                                          27.2ns ± 0%     27.6ns ± 0%    +1.38%
AppendGrowByte                                  4.20ms ± 3%     2.60ms ± 0%   -38.18%
AppendGrowString                                 134ms ± 3%      102ms ± 2%   -23.62%
AppendSlice/1Bytes                              5.65ns ± 0%     5.67ns ± 0%    +0.35%
AppendSlice/4Bytes                              6.40ns ± 0%     6.55ns ± 0%    +2.34%
AppendSlice/7Bytes                              8.74ns ± 0%     8.84ns ± 0%    +1.14%
AppendSlice/8Bytes                              5.68ns ± 0%     5.70ns ± 0%    +0.40%
AppendSlice/15Bytes                             9.31ns ± 0%     9.39ns ± 0%    +0.86%
AppendSlice/16Bytes                             14.0ns ± 0%      5.8ns ± 0%   -58.32%
AppendSlice/32Bytes                             5.72ns ± 0%     5.68ns ± 0%    -0.66%
AppendSliceLarge/1024Bytes                       918ns ± 8%      615ns ± 1%   -33.00%
AppendSliceLarge/4096Bytes                      3.25µs ± 1%     1.92µs ± 1%   -40.84%
AppendSliceLarge/16384Bytes                     8.70µs ± 2%     4.69µs ± 0%   -46.08%
AppendSliceLarge/65536Bytes                     18.1µs ± 3%      7.9µs ± 0%   -56.30%
AppendSliceLarge/262144Bytes                    69.8µs ± 2%     25.9µs ± 0%   -62.91%
AppendSliceLarge/1048576Bytes                    258µs ± 1%       93µs ± 0%   -63.96%

Change-Id: I21625dbe231a2029ddb9f7d73f5a6417b35c1e49
Reviewed-on: https://go-review.googlesource.com/c/go/+/199639
Run-TryBot: Lynn Boger <laboger@linux.vnet.ibm.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Cherry Zhang <cherryyz@google.com>

src/cmd/compile/internal/ppc64/ssa.go

@@ -855,13 +855,13 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) {
 		// for sizes >= 64 generate a loop as follows:
 
 		// set up loop counter in CTR, used by BC
+		//       XXLXOR VS32,VS32,VS32
 		//	 MOVD len/32,REG_TMP
 		//	 MOVD REG_TMP,CTR
+		//       MOVD $16,REG_TMP
 		//	 loop:
-		//	 MOVD R0,(R3)
-		//	 MOVD R0,8(R3)
-		//	 MOVD R0,16(R3)
-		//	 MOVD R0,24(R3)
+		//	 STXVD2X VS32,(R0)(R3)
+		//	 STXVD2X VS32,(R31)(R3)
 		//	 ADD  $32,R3
 		//	 BC   16, 0, loop
 		//
@@ -895,8 +895,16 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) {
 		// only generate a loop if there is more
 		// than 1 iteration.
 		if ctr > 1 {
+			// Set up VS32 (V0) to hold 0s
+			p := s.Prog(ppc64.AXXLXOR)
+			p.From.Type = obj.TYPE_REG
+			p.From.Reg = ppc64.REG_VS32
+			p.To.Type = obj.TYPE_REG
+			p.To.Reg = ppc64.REG_VS32
+			p.Reg = ppc64.REG_VS32
+
 			// Set up CTR loop counter
-			p := s.Prog(ppc64.AMOVD)
+			p = s.Prog(ppc64.AMOVD)
 			p.From.Type = obj.TYPE_CONST
 			p.From.Offset = ctr
 			p.To.Type = obj.TYPE_REG
@@ -908,23 +916,35 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) {
 			p.To.Type = obj.TYPE_REG
 			p.To.Reg = ppc64.REG_CTR
 
-			// generate 4 MOVDs
+			// Set up R31 to hold index value 16
+			p = s.Prog(ppc64.AMOVD)
+			p.From.Type = obj.TYPE_CONST
+			p.From.Offset = 16
+			p.To.Type = obj.TYPE_REG
+			p.To.Reg = ppc64.REGTMP
+
+			// generate 2 STXVD2Xs to store 16 bytes
 			// when this is a loop then the top must be saved
 			var top *obj.Prog
-			for offset := int64(0); offset < 32; offset += 8 {
-				// This is the top of loop
-				p := s.Prog(ppc64.AMOVD)
-				p.From.Type = obj.TYPE_REG
-				p.From.Reg = ppc64.REG_R0
-				p.To.Type = obj.TYPE_MEM
-				p.To.Reg = v.Args[0].Reg()
-				p.To.Offset = offset
-				// Save the top of loop
-				if top == nil {
-					top = p
-				}
+			// This is the top of loop
+			p = s.Prog(ppc64.ASTXVD2X)
+			p.From.Type = obj.TYPE_REG
+			p.From.Reg = ppc64.REG_VS32
+			p.To.Type = obj.TYPE_MEM
+			p.To.Reg = v.Args[0].Reg()
+			p.To.Index = ppc64.REGZERO
+			// Save the top of loop
+			if top == nil {
+				top = p
 			}
 
+			p = s.Prog(ppc64.ASTXVD2X)
+			p.From.Type = obj.TYPE_REG
+			p.From.Reg = ppc64.REG_VS32
+			p.To.Type = obj.TYPE_MEM
+			p.To.Reg = v.Args[0].Reg()
+			p.To.Index = ppc64.REGTMP
+
 			// Increment address for the
 			// 4 doublewords just zeroed.
 			p = s.Prog(ppc64.AADD)
@@ -994,30 +1014,27 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) {
 		// When moving >= 64 bytes a loop is used
 		//	MOVD len/32,REG_TMP
 		//	MOVD REG_TMP,CTR
+		//	MOVD $16,REG_TMP
 		// top:
-		//	MOVD (R4),R7
-		//	MOVD 8(R4),R8
-		//	MOVD 16(R4),R9
-		//	MOVD 24(R4),R10
-		//	ADD  R4,$32
-		//	MOVD R7,(R3)
-		//	MOVD R8,8(R3)
-		//	MOVD R9,16(R3)
-		//	MOVD R10,24(R3)
-		//	ADD  R3,$32
+		//	LXVD2X (R0)(R4),VS32
+		//	LXVD2X (R31)(R4),VS33
+		//	ADD $32,R4
+		//	STXVD2X VS32,(R0)(R3)
+		//	STXVD2X VS33,(R31)(R4)
+		//	ADD $32,R3
 		//	BC 16,0,top
 		// Bytes not moved by this loop are moved
 		// with a combination of the following instructions,
 		// starting with the largest sizes and generating as
 		// many as needed, using the appropriate offset value.
-		//	MOVD  n(R4),R7
-		//	MOVD  R7,n(R3)
-		//	MOVW  n1(R4),R7
-		//	MOVW  R7,n1(R3)
-		//	MOVH  n2(R4),R7
-		//	MOVH  R7,n2(R3)
-		//	MOVB  n3(R4),R7
-		//	MOVB  R7,n3(R3)
+		//	MOVD  n(R4),R14
+		//	MOVD  R14,n(R3)
+		//	MOVW  n1(R4),R14
+		//	MOVW  R14,n1(R3)
+		//	MOVH  n2(R4),R14
+		//	MOVH  R14,n2(R3)
+		//	MOVB  n3(R4),R14
+		//	MOVB  R14,n3(R3)
 
 		// Each loop iteration moves 32 bytes
 		ctr := v.AuxInt / 32
@@ -1030,7 +1047,6 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) {
 
 		// The set of registers used here, must match the clobbered reg list
 		// in PPC64Ops.go.
-		useregs := []int16{ppc64.REG_R7, ppc64.REG_R8, ppc64.REG_R9, ppc64.REG_R10}
 		offset := int64(0)
 
 		// top of the loop
@@ -1050,22 +1066,35 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) {
 			p.To.Type = obj.TYPE_REG
 			p.To.Reg = ppc64.REG_CTR
 
-			// Generate all the MOVDs for loads
-			// based off the same register, increasing
-			// the offset by 8 for each instruction
-			for _, rg := range useregs {
-				p := s.Prog(ppc64.AMOVD)
-				p.From.Type = obj.TYPE_MEM
-				p.From.Reg = src_reg
-				p.From.Offset = offset
-				p.To.Type = obj.TYPE_REG
-				p.To.Reg = rg
-				if top == nil {
-					top = p
-				}
-				offset += 8
+			// Use REGTMP as index reg
+			p = s.Prog(ppc64.AMOVD)
+			p.From.Type = obj.TYPE_CONST
+			p.From.Offset = 16
+			p.To.Type = obj.TYPE_REG
+			p.To.Reg = ppc64.REGTMP
+
+			// Generate 16 byte loads and stores.
+			// Use temp register for index (16)
+			// on the second one.
+			p = s.Prog(ppc64.ALXVD2X)
+			p.From.Type = obj.TYPE_MEM
+			p.From.Reg = src_reg
+			p.From.Index = ppc64.REGZERO
+			p.To.Type = obj.TYPE_REG
+			p.To.Reg = ppc64.REG_VS32
+
+			if top == nil {
+				top = p
 			}
-			// increment the src_reg for next iteration
+
+			p = s.Prog(ppc64.ALXVD2X)
+			p.From.Type = obj.TYPE_MEM
+			p.From.Reg = src_reg
+			p.From.Index = ppc64.REGTMP
+			p.To.Type = obj.TYPE_REG
+			p.To.Reg = ppc64.REG_VS33
+
+			// increment the src reg for next iteration
 			p = s.Prog(ppc64.AADD)
 			p.Reg = src_reg
 			p.From.Type = obj.TYPE_CONST
@@ -1073,20 +1102,22 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) {
 			p.To.Type = obj.TYPE_REG
 			p.To.Reg = src_reg
 
-			// generate the MOVDs for stores, based
-			// off the same register, using the same
-			// offsets as in the loads.
-			offset = int64(0)
-			for _, rg := range useregs {
-				p := s.Prog(ppc64.AMOVD)
-				p.From.Type = obj.TYPE_REG
-				p.From.Reg = rg
-				p.To.Type = obj.TYPE_MEM
-				p.To.Reg = dst_reg
-				p.To.Offset = offset
-				offset += 8
-			}
-			// increment the dst_reg for next iteration
+			// generate 16 byte stores
+			p = s.Prog(ppc64.ASTXVD2X)
+			p.From.Type = obj.TYPE_REG
+			p.From.Reg = ppc64.REG_VS32
+			p.To.Type = obj.TYPE_MEM
+			p.To.Reg = dst_reg
+			p.To.Index = ppc64.REGZERO
+
+			p = s.Prog(ppc64.ASTXVD2X)
+			p.From.Type = obj.TYPE_REG
+			p.From.Reg = ppc64.REG_VS33
+			p.To.Type = obj.TYPE_MEM
+			p.To.Reg = dst_reg
+			p.To.Index = ppc64.REGTMP
+
+			// increment the dst reg for next iteration
 			p = s.Prog(ppc64.AADD)
 			p.Reg = dst_reg
 			p.From.Type = obj.TYPE_CONST
@@ -1114,6 +1145,57 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) {
 			rem += 32
 		}
 
+		if rem >= 16 {
+			// Generate 16 byte loads and stores.
+			// Use temp register for index (value 16)
+			// on the second one.
+			p := s.Prog(ppc64.ALXVD2X)
+			p.From.Type = obj.TYPE_MEM
+			p.From.Reg = src_reg
+			p.From.Index = ppc64.REGZERO
+			p.To.Type = obj.TYPE_REG
+			p.To.Reg = ppc64.REG_VS32
+
+			p = s.Prog(ppc64.ASTXVD2X)
+			p.From.Type = obj.TYPE_REG
+			p.From.Reg = ppc64.REG_VS32
+			p.To.Type = obj.TYPE_MEM
+			p.To.Reg = dst_reg
+			p.To.Index = ppc64.REGZERO
+
+			offset = 16
+			rem -= 16
+
+			if rem >= 16 {
+				// Use REGTMP as index reg
+				p = s.Prog(ppc64.AMOVD)
+				p.From.Type = obj.TYPE_CONST
+				p.From.Offset = 16
+				p.To.Type = obj.TYPE_REG
+				p.To.Reg = ppc64.REGTMP
+
+				// Generate 16 byte loads and stores.
+				// Use temp register for index (16)
+				// on the second one.
+				p = s.Prog(ppc64.ALXVD2X)
+				p.From.Type = obj.TYPE_MEM
+				p.From.Reg = src_reg
+				p.From.Index = ppc64.REGTMP
+				p.To.Type = obj.TYPE_REG
+				p.To.Reg = ppc64.REG_VS32
+
+				p = s.Prog(ppc64.ASTXVD2X)
+				p.From.Type = obj.TYPE_REG
+				p.From.Reg = ppc64.REG_VS32
+				p.To.Type = obj.TYPE_MEM
+				p.To.Reg = dst_reg
+				p.To.Index = ppc64.REGTMP
+
+				offset = 32
+				rem -= 16
+			}
+		}
+
 		// Generate all the remaining load and store pairs, starting with
 		// as many 8 byte moves as possible, then 4, 2, 1.
 		for rem > 0 {
@@ -1129,7 +1211,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) {
 			// Load
 			p := s.Prog(op)
 			p.To.Type = obj.TYPE_REG
-			p.To.Reg = ppc64.REG_R7
+			p.To.Reg = ppc64.REG_R14
 			p.From.Type = obj.TYPE_MEM
 			p.From.Reg = src_reg
 			p.From.Offset = offset
@@ -1137,7 +1219,7 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) {
 			// Store
 			p = s.Prog(op)
 			p.From.Type = obj.TYPE_REG
-			p.From.Reg = ppc64.REG_R7
+			p.From.Reg = ppc64.REG_R14
 			p.To.Type = obj.TYPE_MEM
 			p.To.Reg = dst_reg
 			p.To.Offset = offset

src/cmd/compile/internal/ssa/gen/PPC64Ops.go

@@ -416,13 +416,13 @@ func init() {
 		// a loop is generated when there is more than one iteration
 		// needed to clear 4 doublewords
 		//
+		//	XXLXOR	VS32,VS32,VS32
 		// 	MOVD	$len/32,R31
 		//	MOVD	R31,CTR
+		//	MOVD	$16,R31
 		//	loop:
-		//	MOVD	R0,(R3)
-		//	MOVD	R0,8(R3)
-		//	MOVD	R0,16(R3)
-		//	MOVD	R0,24(R3)
+		//	STXVD2X VS32,(R0)(R3)
+		//	STXVD2X	VS32,(R31),R3)
 		//	ADD	R3,32
 		//	BC	loop
 
@@ -448,33 +448,38 @@ func init() {
 			typ:            "Mem",
 			faultOnNilArg0: true,
 		},
+		// R31 is temp register
 		// Loop code:
-		//	MOVD len/32,REG_TMP  only for loop
-		//	MOVD REG_TMP,CTR     only for loop
+		//	MOVD len/32,R31		set up loop ctr
+		//	MOVD R31,CTR
+		//	MOVD $16,R31		index register
 		// loop:
-		//	MOVD (R4),R7
-		//	MOVD 8(R4),R8
-		//	MOVD 16(R4),R9
-		//	MOVD 24(R4),R10
-		//	ADD  R4,$32          only with loop
-		//	MOVD R7,(R3)
-		//	MOVD R8,8(R3)
-		//	MOVD R9,16(R3)
-		//	MOVD R10,24(R3)
-		//	ADD  R3,$32          only with loop
-		//	BC 16,0,loop         only with loop
+		//	LXVD2X (R0)(R4),VS32
+		//	LXVD2X (R31)(R4),VS33
+		//	ADD  R4,$32          increment src
+		//	STXVD2X VS32,(R0)(R3)
+		//	STXVD2X VS33,(R31)(R3)
+		//	ADD  R3,$32          increment dst
+		//	BC 16,0,loop         branch ctr
+		// For this purpose, VS32 and VS33 are treated as
+		// scratch registers. Since regalloc does not
+		// track vector registers, even if it could be marked
+		// as clobbered it would have no effect.
+		// TODO: If vector registers are managed by regalloc
+		// mark these as clobbered.
+		//
 		// Bytes not moved by this loop are moved
 		// with a combination of the following instructions,
 		// starting with the largest sizes and generating as
 		// many as needed, using the appropriate offset value.
-		//	MOVD  n(R4),R7
-		//	MOVD  R7,n(R3)
-		//	MOVW  n1(R4),R7
-		//	MOVW  R7,n1(R3)
-		//	MOVH  n2(R4),R7
-		//	MOVH  R7,n2(R3)
-		//	MOVB  n3(R4),R7
-		//	MOVB  R7,n3(R3)
+		//	MOVD  n(R4),R14
+		//	MOVD  R14,n(R3)
+		//	MOVW  n1(R4),R14
+		//	MOVW  R14,n1(R3)
+		//	MOVH  n2(R4),R14
+		//	MOVH  R14,n2(R3)
+		//	MOVB  n3(R4),R14
+		//	MOVB  R14,n3(R3)
 
 		{
 			name:      "LoweredMove",
@@ -482,7 +487,7 @@ func init() {
 			argLength: 3,
 			reg: regInfo{
 				inputs:   []regMask{buildReg("R3"), buildReg("R4")},
-				clobbers: buildReg("R3 R4 R7 R8 R9 R10"),
+				clobbers: buildReg("R3 R4 R14"),
 			},
 			clobberFlags:   true,
 			typ:            "Mem",

src/cmd/compile/internal/ssa/opGen.go

@@ -24486,7 +24486,7 @@ var opcodeTable = [...]opInfo{
 				{0, 8},  // R3
 				{1, 16}, // R4
 			},
-			clobbers: 1944, // R3 R4 R7 R8 R9 R10
+			clobbers: 16408, // R3 R4 R14
 		},
 	},
 	{