cmd/internal/obj/x86: disallow PC/FP/SB scaled index

Reject to compile I386/AMD64 asm code that contains
(Register)(PseudoReg*scale) forms of memory operands.
Example of such program: "CALL (AX)(PC*2)".
PseudoReg is one of the PC, FP, SB (but not SP).

When pseudo-register is used in register indirect as
scaled index base, x86 backend will panic because
its register file misses SB/FP/PC registers.

Fixes #12657.

Change-Id: I30fca797b537cbc86ab47583ae96c6a0c59acaa1
Reviewed-on: https://go-review.googlesource.com/107835
Run-TryBot: Iskander Sharipov <iskander.sharipov@intel.com>
Reviewed-by: Cherry Zhang <cherryyz@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>

src/cmd/asm/internal/asm/testdata/amd64error.s

@@ -45,4 +45,8 @@ TEXT errors(SB),$0
 	ADDQ 433954697820(AX), AX       // ERROR "offset too large"
 	ADDL 433954697820(AX), AX       // ERROR "offset too large"
 	ADDW 433954697820(AX), AX       // ERROR "offset too large"
+	// Pseudo-registers should not be used as scaled index.
+	CALL (AX)(PC*1)                 // ERROR "invalid instruction"
+	CALL (AX)(SB*1)                 // ERROR "invalid instruction"
+	CALL (AX)(FP*1)                 // ERROR "invalid instruction"
 	RET

src/cmd/internal/obj/x86/asm6.go

@@ -2615,8 +2615,10 @@ func oclass(ctxt *obj.Link, p *obj.Prog, a *obj.Addr) int {
 		return Yxxx
 
 	case obj.TYPE_MEM:
-		if a.Index == REG_SP {
-			// Can't use SP as the index register
+		// Pseudo registers have negative index, but SP is
+		// not pseudo on x86, hence REG_SP check is not redundant.
+		if a.Index == REG_SP || a.Index < 0 {
+			// Can't use FP/SB/PC/SP as the index register.
 			return Yxxx
 		}
 		if a.Index >= REG_X0 && a.Index <= REG_X15 {