cmd/6g, cmd/8g: move panicindex calls out of line

The old code generated for a bounds check was
                CMP
                JLT ok
                CALL panicindex
        ok:
                ...

The new code is (once the linker finishes with it):
                CMP
                JGE panic
                ...
        panic:
                CALL panicindex

which moves the calls out of line, putting more useful
code in each cache line.  This matters especially in tight
loops, such as in Fannkuch.  The benefit is more modest
elsewhere, but real.

From test/bench/go1, amd64:

benchmark                old ns/op    new ns/op    delta
BenchmarkBinaryTree17   6096092000   6088808000   -0.12%
BenchmarkFannkuch11     6151404000   4020463000  -34.64%
BenchmarkGobDecode        28990050     28894630   -0.33%
BenchmarkGobEncode        12406310     12136730   -2.17%
BenchmarkGzip               179923       179903   -0.01%
BenchmarkGunzip              11219        11130   -0.79%
BenchmarkJSONEncode       86429350     86515900   +0.10%
BenchmarkJSONDecode      334593800    315728400   -5.64%
BenchmarkRevcomp25M     1219763000   1180767000   -3.20%
BenchmarkTemplate        492947600    483646800   -1.89%

And 386:

benchmark                old ns/op    new ns/op    delta
BenchmarkBinaryTree17   6354902000   6243000000   -1.76%
BenchmarkFannkuch11     8043769000   7326965000   -8.91%
BenchmarkGobDecode        19010800     18941230   -0.37%
BenchmarkGobEncode        14077500     13792460   -2.02%
BenchmarkGzip               194087       193619   -0.24%
BenchmarkGunzip              12495        12457   -0.30%
BenchmarkJSONEncode      125636400    125451400   -0.15%
BenchmarkJSONDecode      696648600    685032800   -1.67%
BenchmarkRevcomp25M     2058088000   2052545000   -0.27%
BenchmarkTemplate        602140000    589876800   -2.04%

To implement this, two new instruction forms:

        JLT target      // same as always
        JLT $0, target  // branch expected not taken
        JLT $1, target  // branch expected taken

The linker could also emit the prediction prefixes, but it
does not: expected taken branches are reversed so that the
expected case is not taken (as in example above), and
the default expectaton for such a jump is not taken
already.

R=golang-dev, gri, r, dave
CC=golang-dev
https://golang.org/cl/6248049

src/cmd/6a/a.y

@@ -176,6 +176,11 @@ nonrel:
 		$$.from = nullgen;
 		$$.to = $1;
 	}
+|	imm ',' rel
+	{
+		$$.from = $1;
+		$$.to = $3;
+	}
 
 spec1:	/* DATA */
 	nam '/' con ',' imm

src/cmd/6g/cgen.c

@@ -575,6 +575,7 @@ agen(Node *n, Node *res)
 					nodconst(&n2, types[TUINT32], v);
 					gins(optoas(OCMP, types[TUINT32]), &n1, &n2);
 					p1 = gbranch(optoas(OGT, types[TUINT32]), T);
+					expecttaken(p1, 1);
 					ginscall(panicindex, 0);
 					patch(p1, pc);
 				}
@@ -625,6 +626,7 @@ agen(Node *n, Node *res)
 			}
 			gins(optoas(OCMP, t), &n2, &n1);
 			p1 = gbranch(optoas(OLT, t), T);
+			expecttaken(p1, 1);
 			if(n5.op != OXXX)
 				regfree(&n5);
 			ginscall(panicindex, 0);

src/cmd/6g/gg.h

@@ -104,6 +104,7 @@ int	componentgen(Node*, Node*);
 void	clearp(Prog*);
 void	proglist(void);
 Prog*	gbranch(int, Type*);
+void	expecttaken(Prog*, int);
 Prog*	prog(int);
 void	gaddoffset(Node*);
 void	gconv(int, int);

src/cmd/6g/ggen.c

@@ -508,6 +508,7 @@ dodiv(int op, Node *nl, Node *nr, Node *res)
 		nodconst(&n4, t, -1);
 		gins(optoas(OCMP, t), &n3, &n4);
 		p1 = gbranch(optoas(ONE, t), T);
+		expecttaken(p1, 1);
 		nodconst(&n4, t, -1LL<<(t->width*8-1));
 		if(t->width == 8) {
 			n5 = n4;
@@ -516,6 +517,7 @@ dodiv(int op, Node *nl, Node *nr, Node *res)
 		}
 		gins(optoas(OCMP, t), &ax, &n4);
 		p2 = gbranch(optoas(ONE, t), T);
+		expecttaken(p2, 1);
 		if(op == ODIV)
 			gmove(&n4, res);
 		if(t->width == 8)
@@ -943,6 +945,7 @@ cgen_shift(int op, int bounded, Node *nl, Node *nr, Node *res)
 		nodconst(&n3, tcount, nl->type->width*8);
 		gins(optoas(OCMP, tcount), &n1, &n3);
 		p1 = gbranch(optoas(OLT, tcount), T);
+		expecttaken(p1, 1);
 		if(op == ORSH && issigned[nl->type->etype]) {
 			nodconst(&n3, types[TUINT32], nl->type->width*8-1);
 			gins(a, &n3, &n2);
@@ -1158,12 +1161,14 @@ cmpandthrow(Node *nl, Node *nr)
 		regfree(&n1);
 	if(throwpc == nil) {
 		p1 = gbranch(optoas(op, t), T);
+		expecttaken(p1, 1);
 		throwpc = pc;
 		ginscall(panicslice, 0);
 		patch(p1, pc);
 	} else {
 		op = brcom(op);
 		p1 = gbranch(optoas(op, t), T);
+		expecttaken(p1, 0);
 		patch(p1, throwpc);
 	}
 }

src/cmd/6g/gsubr.c

@@ -117,6 +117,16 @@ gbranch(int as, Type *t)
 	return p;
 }
 
+/*
+ * mark branch as expected taken or not.
+ */
+void
+expecttaken(Prog *p, int taken)
+{
+	p->from.type = D_CONST;
+	p->from.offset = taken;
+}
+
 /*
  * patch previous branch to jump to to.
  */
@@ -2120,6 +2130,7 @@ oindex:
 		}
 		gins(optoas(OCMP, t), reg1, &n2);
 		p1 = gbranch(optoas(OLT, t), T);
+		expecttaken(p1, 1);
 		if(n4.op != OXXX)
 			regfree(&n4);
 		ginscall(panicindex, 0);
@@ -2229,6 +2240,7 @@ oindex_const_sudo:
 		p1 = gins(optoas(OCMP, types[TUINT32]), N, &n2);
 		p1->from = *a;
 		p1 = gbranch(optoas(OGT, types[TUINT32]), T);
+		expecttaken(p1, 1);
 		ginscall(panicindex, 0);
 		patch(p1, pc);
 		a->offset -= Array_nel;

src/cmd/6l/optab.c

@@ -314,7 +314,9 @@ uchar	yscond[] =
 };
 uchar	yjcond[] =
 {
-	Ynone,	Ybr,	Zbr,	1,
+	Ynone,	Ybr,	Zbr,	0,
+	Yi0,	Ybr,	Zbr,	0,
+	Yi1,	Ybr,	Zbr,	1,
 	0
 };
 uchar	yloop[] =

src/cmd/6l/pass.c

@@ -192,12 +192,25 @@ loop:
 		 * recurse to follow one path.
 		 * continue loop on the other.
 		 */
-		q = brchain(p->link);
-		if(q != P && q->mark)
-		if(a != ALOOP) {
-			p->as = relinv(a);
-			p->link = p->pcond;
-			p->pcond = q;
+		if(p->from.type == D_CONST) {
+			if(p->from.offset == 1) {
+				/*
+				 * expect conditional jump to be taken.
+				 * rewrite so that's the fall-through case.
+				 */
+				p->as = relinv(a);
+				q = p->link;
+				p->link = p->pcond;
+				p->pcond = q;
+			}
+		} else {			
+			q = brchain(p->link);
+			if(q != P && q->mark)
+			if(a != ALOOP) {
+				p->as = relinv(a);
+				p->link = p->pcond;
+				p->pcond = q;
+			}
 		}
 		xfol(p->link, last);
 		q = brchain(p->pcond);
@@ -405,7 +418,7 @@ dostkoff(void)
 
 	for(cursym = textp; cursym != nil; cursym = cursym->next) {
 		if(cursym->text == nil || cursym->text->link == nil)
-			continue;
+			continue;				
 
 		p = cursym->text;
 		parsetextconst(p->to.offset);
@@ -413,6 +426,14 @@ dostkoff(void)
 		if(autoffset < 0)
 			autoffset = 0;
 
+		if(autoffset < StackSmall && !(p->from.scale & NOSPLIT)) {
+			for(q = p; q != P; q = q->link)
+				if(q->as == ACALL)
+					goto noleaf;
+			p->from.scale |= NOSPLIT;
+		noleaf:;
+		}
+
 		q = P;
 		if((p->from.scale & NOSPLIT) && autoffset >= StackSmall)
 			diag("nosplit func likely to overflow stack");

src/cmd/8a/a.y

@@ -177,6 +177,11 @@ nonrel:
 		$$.from = nullgen;
 		$$.to = $1;
 	}
+|	imm ',' rel
+	{
+		$$.from = $1;
+		$$.to = $3;
+	}
 
 spec1:	/* DATA */
 	nam '/' con ',' imm

src/cmd/8a/y.tab.h

-
-/* A Bison parser, made by GNU Bison 2.4.1.  */
+/* A Bison parser, made by GNU Bison 2.3.  */
 
 /* Skeleton interface for Bison's Yacc-like parsers in C
-   
-      Copyright (C) 1984, 1989, 1990, 2000, 2001, 2002, 2003, 2004, 2005, 2006
+
+   Copyright (C) 1984, 1989, 1990, 2000, 2001, 2002, 2003, 2004, 2005, 2006
    Free Software Foundation, Inc.
-   
-   This program is free software: you can redistribute it and/or modify
+
+   This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
-   the Free Software Foundation, either version 3 of the License, or
-   (at your option) any later version.
-   
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.
-   
+
    You should have received a copy of the GNU General Public License
-   along with this program.  If not, see <http://www.gnu.org/licenses/>.  */
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor,
+   Boston, MA 02110-1301, USA.  */
 
 /* As a special exception, you may create a larger work that contains
    part or all of the Bison parser skeleton and distribute that work
@@ -28,11 +29,10 @@
    special exception, which will cause the skeleton and the resulting
    Bison output files to be licensed under the GNU General Public
    License without this special exception.
-   
+
    This special exception was added by the Free Software Foundation in
    version 2.2 of Bison.  */
 
-
 /* Tokens.  */
 #ifndef YYTOKENTYPE
 # define YYTOKENTYPE
@@ -104,11 +104,8 @@
 
 #if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED
 typedef union YYSTYPE
-{
-
-/* Line 1676 of yacc.c  */
 #line 37 "a.y"
-
+{
 	Sym	*sym;
 	int32	lval;
 	struct {
@@ -119,17 +116,14 @@ typedef union YYSTYPE
 	char	sval[8];
 	Gen	gen;
 	Gen2	gen2;
-
-
-
-/* Line 1676 of yacc.c  */
-#line 127 "y.tab.h"
-} YYSTYPE;
-# define YYSTYPE_IS_TRIVIAL 1
+}
+/* Line 1529 of yacc.c.  */
+#line 122 "y.tab.h"
+	YYSTYPE;
 # define yystype YYSTYPE /* obsolescent; will be withdrawn */
 # define YYSTYPE_IS_DECLARED 1
+# define YYSTYPE_IS_TRIVIAL 1
 #endif
 
 extern YYSTYPE yylval;
 
-

src/cmd/8g/cgen.c

@@ -596,6 +596,7 @@ agen(Node *n, Node *res)
 					nodconst(&n2, types[TUINT32], v);
 					gins(optoas(OCMP, types[TUINT32]), &n1, &n2);
 					p1 = gbranch(optoas(OGT, types[TUINT32]), T);
+					expecttaken(p1, 1);
 					ginscall(panicindex, 0);
 					patch(p1, pc);
 				}
@@ -633,6 +634,7 @@ agen(Node *n, Node *res)
 				nodconst(&n1, types[TUINT32], nl->type->bound);
 			gins(optoas(OCMP, types[TUINT32]), &n2, &n1);
 			p1 = gbranch(optoas(OLT, types[TUINT32]), T);
+			expecttaken(p1, 1);
 			if(p2)
 				patch(p2, pc);
 			ginscall(panicindex, 0);

src/cmd/8g/gg.h

@@ -122,6 +122,7 @@ void	cgen64(Node*, Node*);
 void	clearp(Prog*);
 void	proglist(void);
 Prog*	gbranch(int, Type*);
+void	expecttaken(Prog*, int);
 Prog*	prog(int);
 void	gaddoffset(Node*);
 void	gconv(int, int);

src/cmd/8g/ggen.c

@@ -896,12 +896,14 @@ cmpandthrow(Node *nl, Node *nr)
 		regfree(&n1);
 	if(throwpc == nil) {
 		p1 = gbranch(optoas(op, t), T);
+		expecttaken(p1, 1);
 		throwpc = pc;
 		ginscall(panicslice, 0);
 		patch(p1, pc);
 	} else {
 		op = brcom(op);
 		p1 = gbranch(optoas(op, t), T);
+		expecttaken(p1, 0);
 		patch(p1, throwpc);
 	}
 }

src/cmd/8g/gsubr.c

@@ -118,6 +118,13 @@ gbranch(int as, Type *t)
 	return p;
 }
 
+void
+expecttaken(Prog *p, int taken)
+{
+	p->from.type = D_CONST;
+	p->from.offset = taken;
+}
+
 /*
  * patch previous branch to jump to to.
  */

src/cmd/8l/optab.c

@@ -254,7 +254,9 @@ uchar	yscond[] =
 };
 uchar	yjcond[] =
 {
-	Ynone,	Ybr,	Zbr,	1,
+	Ynone,	Ybr,	Zbr,	0,
+	Yi0,	Ybr,	Zbr,	0,
+	Yi1,	Ybr,	Zbr,	1,
 	0
 };
 uchar	yloop[] =

src/cmd/8l/pass.c

@@ -184,12 +184,25 @@ loop:
 		 * recurse to follow one path.
 		 * continue loop on the other.
 		 */
-		q = brchain(p->link);
-		if(q != P && q->mark)
-		if(a != ALOOP) {
-			p->as = relinv(a);
-			p->link = p->pcond;
-			p->pcond = q;
+		if(p->from.type == D_CONST) {
+			if(p->from.offset == 1) {
+				/*
+				 * expect conditional jump to be taken.
+				 * rewrite so that's the fall-through case.
+				 */
+				p->as = relinv(a);
+				q = p->link;
+				p->link = p->pcond;
+				p->pcond = q;
+			}
+		} else {
+			q = brchain(p->link);
+			if(q != P && q->mark)
+			if(a != ALOOP) {
+				p->as = relinv(a);
+				p->link = p->pcond;
+				p->pcond = q;
+			}
 		}
 		xfol(p->link, last);
 		q = brchain(p->pcond);