regexp: add one-pass optimization from RE2

This produces about a 2.3x speedup for patterns
that can be handled this way.

LGTM=rsc
R=rsc
CC=golang-codereviews
https://golang.org/cl/13345046

src/pkg/regexp/all_test.go

@@ -578,3 +578,58 @@ func BenchmarkAnchoredLongMatch(b *testing.B) {
 		re.Match(x)
 	}
 }
+
+func BenchmarkOnePassShortA(b *testing.B) {
+	b.StopTimer()
+	x := []byte("abcddddddeeeededd")
+	re := MustCompile("^.bc(d|e)*$")
+	b.StartTimer()
+	for i := 0; i < b.N; i++ {
+		re.Match(x)
+	}
+}
+func BenchmarkNotOnePassShortA(b *testing.B) {
+	b.StopTimer()
+	x := []byte("abcddddddeeeededd")
+	re := MustCompile(".bc(d|e)*$")
+	b.StartTimer()
+	for i := 0; i < b.N; i++ {
+		re.Match(x)
+	}
+}
+func BenchmarkOnePassShortB(b *testing.B) {
+	b.StopTimer()
+	x := []byte("abcddddddeeeededd")
+	re := MustCompile("^.bc(?:d|e)*$")
+	b.StartTimer()
+	for i := 0; i < b.N; i++ {
+		re.Match(x)
+	}
+}
+func BenchmarkNotOnePassShortB(b *testing.B) {
+	b.StopTimer()
+	x := []byte("abcddddddeeeededd")
+	re := MustCompile(".bc(?:d|e)*$")
+	b.StartTimer()
+	for i := 0; i < b.N; i++ {
+		re.Match(x)
+	}
+}
+func BenchmarkOnePassLongPrefix(b *testing.B) {
+	b.StopTimer()
+	x := []byte("abcdefghijklmnopqrstuvwxyz")
+	re := MustCompile("^abcdefghijklmnopqrstuvwxyz.*$")
+	b.StartTimer()
+	for i := 0; i < b.N; i++ {
+		re.Match(x)
+	}
+}
+func BenchmarkOnePassLongNotPrefix(b *testing.B) {
+	b.StopTimer()
+	x := []byte("abcdefghijklmnopqrstuvwxyz")
+	re := MustCompile("^.bcdefghijklmnopqrstuvwxyz.*$")
+	b.StartTimer()
+	for i := 0; i < b.N; i++ {
+		re.Match(x)
+	}
+}

src/pkg/regexp/exec.go

@@ -37,6 +37,7 @@ type thread struct {
 type machine struct {
 	re       *Regexp      // corresponding Regexp
 	p        *syntax.Prog // compiled program
+	op       *syntax.Prog // compiled onepass program, or syntax.NotOnePass
 	q0, q1   queue        // two queues for runq, nextq
 	pool     []*thread    // pool of available threads
 	matched  bool         // whether a match was found
@@ -66,8 +67,8 @@ func (m *machine) newInputReader(r io.RuneReader) input {
 }
 
 // progMachine returns a new machine running the prog p.
-func progMachine(p *syntax.Prog) *machine {
-	m := &machine{p: p}
+func progMachine(p, op *syntax.Prog) *machine {
+	m := &machine{p: p, op: op}
 	n := len(m.p.Inst)
 	m.q0 = queue{make([]uint32, n), make([]entry, 0, n)}
 	m.q1 = queue{make([]uint32, n), make([]entry, 0, n)}
@@ -312,6 +313,105 @@ func (m *machine) add(q *queue, pc uint32, pos int, cap []int, cond syntax.Empty
 	return t
 }
 
+// onepass runs the machine over the input starting at pos.
+// It reports whether a match was found.
+// If so, m.matchcap holds the submatch information.
+func (m *machine) onepass(i input, pos int) bool {
+	startCond := m.re.cond
+	if startCond == ^syntax.EmptyOp(0) { // impossible
+		return false
+	}
+	m.matched = false
+	for i := range m.matchcap {
+		m.matchcap[i] = -1
+	}
+	r, r1 := endOfText, endOfText
+	width, width1 := 0, 0
+	r, width = i.step(pos)
+	if r != endOfText {
+		r1, width1 = i.step(pos + width)
+	}
+	var flag syntax.EmptyOp
+	if pos == 0 {
+		flag = syntax.EmptyOpContext(-1, r)
+	} else {
+		flag = i.context(pos)
+	}
+	pc := m.op.Start
+	inst := m.op.Inst[pc]
+	// If there is a simple literal prefix, skip over it.
+	if pos == 0 && syntax.EmptyOp(inst.Arg)&^flag == 0 &&
+		len(m.re.prefix) > 0 && i.canCheckPrefix() {
+		// Match requires literal prefix; fast search for it.
+		if i.hasPrefix(m.re) {
+			pos += len(m.re.prefix)
+			r, width = i.step(pos)
+			r1, width1 = i.step(pos + width)
+			flag = i.context(pos)
+			pc = int(m.re.prefixEnd)
+		} else {
+			return m.matched
+		}
+	}
+	for {
+		inst = m.op.Inst[pc]
+		pc = int(inst.Out)
+		switch inst.Op {
+		default:
+			panic("bad inst")
+		case syntax.InstMatch:
+			m.matched = true
+			if len(m.matchcap) > 0 {
+				m.matchcap[0] = 0
+				m.matchcap[1] = pos
+			}
+			return m.matched
+		case syntax.InstRune:
+			if !inst.MatchRune(r) {
+				return m.matched
+			}
+		case syntax.InstRune1:
+			if r != inst.Rune[0] {
+				return m.matched
+			}
+		case syntax.InstRuneAny:
+			// Nothing
+		case syntax.InstRuneAnyNotNL:
+			if r == '\n' {
+				return m.matched
+			}
+		// peek at the input rune to see which branch of the Alt to take
+		case syntax.InstAlt, syntax.InstAltMatch:
+			pc = int(inst.OnePassNext(r))
+			continue
+		case syntax.InstFail:
+			return m.matched
+		case syntax.InstNop:
+			continue
+		case syntax.InstEmptyWidth:
+			if syntax.EmptyOp(inst.Arg)&^flag != 0 {
+				return m.matched
+			}
+			continue
+		case syntax.InstCapture:
+			if int(inst.Arg) < len(m.matchcap) {
+				m.matchcap[inst.Arg] = pos
+			}
+			continue
+		}
+		if width == 0 {
+			break
+		}
+		flag = syntax.EmptyOpContext(r, r1)
+		pos += width
+		r, width = r1, width1
+		if r != endOfText {
+			r1, width1 = i.step(pos + width)
+		}
+	}
+	return m.matched
+}
+
 // empty is a non-nil 0-element slice,
 // so doExecute can avoid an allocation
 // when 0 captures are requested from a successful match.
@@ -329,16 +429,23 @@ func (re *Regexp) doExecute(r io.RuneReader, b []byte, s string, pos int, ncap i
 	} else {
 		i = m.newInputString(s)
 	}
+	if m.op != syntax.NotOnePass {
+		if !m.onepass(i, pos) {
+			re.put(m)
+			return nil
+		}
+	} else {
 		m.init(ncap)
 		if !m.match(i, pos) {
 			re.put(m)
 			return nil
 		}
+	}
 	if ncap == 0 {
 		re.put(m)
 		return empty // empty but not nil
 	}
-	cap := make([]int, ncap)
+	cap := make([]int, len(m.matchcap))
 	copy(cap, m.matchcap)
 	re.put(m)
 	return cap

src/pkg/regexp/regexp.go

@@ -75,10 +75,12 @@ type Regexp struct {
 	// read-only after Compile
 	expr           string         // as passed to Compile
 	prog           *syntax.Prog   // compiled program
+	onepass        *syntax.Prog   // onpass program or nil
 	prefix         string         // required prefix in unanchored matches
 	prefixBytes    []byte         // prefix, as a []byte
 	prefixComplete bool           // prefix is the entire regexp
 	prefixRune     rune           // first rune in prefix
+	prefixEnd      uint32         // pc for last rune in prefix
 	cond           syntax.EmptyOp // empty-width conditions required at start of match
 	numSubexp      int
 	subexpNames    []string
@@ -155,12 +157,17 @@ func compile(expr string, mode syntax.Flags, longest bool) (*Regexp, error) {
 	regexp := &Regexp{
 		expr:        expr,
 		prog:        prog,
+		onepass:     prog.CompileOnePass(),
 		numSubexp:   maxCap,
 		subexpNames: capNames,
 		cond:        prog.StartCond(),
 		longest:     longest,
 	}
+	if regexp.onepass == syntax.NotOnePass {
 		regexp.prefix, regexp.prefixComplete = prog.Prefix()
+	} else {
+		regexp.prefix, regexp.prefixComplete, regexp.prefixEnd = prog.OnePassPrefix()
+	}
 	if regexp.prefix != "" {
 		// TODO(rsc): Remove this allocation by adding
 		// IndexString to package bytes.
@@ -182,7 +189,7 @@ func (re *Regexp) get() *machine {
 		return z
 	}
 	re.mu.Unlock()
-	z := progMachine(re.prog)
+	z := progMachine(re.prog, re.onepass)
 	z.re = re
 	return z
 }

src/pkg/regexp/syntax/prog_test.go

@@ -5,6 +5,7 @@
 package syntax
 
 import (
+	"reflect"
 	"testing"
 )
 
@@ -114,3 +115,200 @@ func BenchmarkEmptyOpContext(b *testing.B) {
 		EmptyOpContext(r1, -1)
 	}
 }
+
+var runeMergeTests = []struct {
+	left, right, merged []rune
+	next                []uint32
+	leftPC, rightPC     uint32
+}{
+	{
+		// empty rhs
+		[]rune{69, 69},
+		[]rune{},
+		[]rune{69, 69},
+		[]uint32{1},
+		1, 2,
+	},
+	{
+		// identical runes, identical targets
+		[]rune{69, 69},
+		[]rune{69, 69},
+		[]rune{},
+		[]uint32{mergeFailed},
+		1, 1,
+	},
+	{
+		// identical runes, different targets
+		[]rune{69, 69},
+		[]rune{69, 69},
+		[]rune{},
+		[]uint32{mergeFailed},
+		1, 2,
+	},
+	{
+		// append right-first
+		[]rune{69, 69},
+		[]rune{71, 71},
+		[]rune{69, 69, 71, 71},
+		[]uint32{1, 2},
+		1, 2,
+	},
+	{
+		// append, left-first
+		[]rune{71, 71},
+		[]rune{69, 69},
+		[]rune{69, 69, 71, 71},
+		[]uint32{2, 1},
+		1, 2,
+	},
+	{
+		// successful interleave
+		[]rune{60, 60, 71, 71, 101, 101},
+		[]rune{69, 69, 88, 88},
+		[]rune{60, 60, 69, 69, 71, 71, 88, 88, 101, 101},
+		[]uint32{1, 2, 1, 2, 1},
+		1, 2,
+	},
+	{
+		// left surrounds right
+		[]rune{69, 74},
+		[]rune{71, 71},
+		[]rune{},
+		[]uint32{mergeFailed},
+		1, 2,
+	},
+	{
+		// right surrounds left
+		[]rune{69, 74},
+		[]rune{68, 75},
+		[]rune{},
+		[]uint32{mergeFailed},
+		1, 2,
+	},
+	{
+		// overlap at interval begin
+		[]rune{69, 74},
+		[]rune{74, 75},
+		[]rune{},
+		[]uint32{mergeFailed},
+		1, 2,
+	},
+	{
+		// overlap ar interval end
+		[]rune{69, 74},
+		[]rune{65, 69},
+		[]rune{},
+		[]uint32{mergeFailed},
+		1, 2,
+	},
+	{
+		// overlap from above
+		[]rune{69, 74},
+		[]rune{71, 74},
+		[]rune{},
+		[]uint32{mergeFailed},
+		1, 2,
+	},
+	{
+		// overlap from below
+		[]rune{69, 74},
+		[]rune{65, 71},
+		[]rune{},
+		[]uint32{mergeFailed},
+		1, 2,
+	},
+	{
+		// out of order []rune
+		[]rune{69, 74, 60, 65},
+		[]rune{66, 67},
+		[]rune{},
+		[]uint32{mergeFailed},
+		1, 2,
+	},
+}
+
+func TestMergeRuneSet(t *testing.T) {
+	for ix, test := range runeMergeTests {
+		merged, next := mergeRuneSets(&test.left, &test.right, test.leftPC, test.rightPC)
+		if !reflect.DeepEqual(merged, test.merged) {
+			t.Errorf("mergeRuneSet :%d (%v, %v) merged\n have\n%v\nwant\n%v", ix, test.left, test.right, merged, test.merged)
+		}
+		if !reflect.DeepEqual(next, test.next) {
+			t.Errorf("mergeRuneSet :%d(%v, %v) next\n have\n%v\nwant\n%v", ix, test.left, test.right, next, test.next)
+		}
+	}
+}
+
+const noStr = `!`
+
+var onePass = &Prog{}
+
+var onePassTests = []struct {
+	re      string
+	onePass *Prog
+	prog    string
+}{
+	{`^(?:a|(?:a*))$`, NotOnePass, noStr},
+	{`^(?:(a)|(?:a*))$`, NotOnePass, noStr},
+	{`^(?:(?:(?:.(?:$))?))$`, onePass, `a`},
+	{`^abcd$`, onePass, `abcd`},
+	{`^abcd$`, onePass, `abcde`},
+	{`^(?:(?:a{0,})*?)$`, onePass, `a`},
+	{`^(?:(?:a+)*)$`, onePass, ``},
+	{`^(?:(?:a|(?:aa)))$`, onePass, ``},
+	{`^(?:[^\s\S])$`, onePass, ``},
+	{`^(?:(?:a{3,4}){0,})$`, NotOnePass, `aaaaaa`},
+	{`^(?:(?:a+)*)$`, onePass, `a`},
+	{`^(?:(?:(?:a*)+))$`, onePass, noStr},
+	{`^(?:(?:a+)*)$`, onePass, ``},
+	{`^[a-c]+$`, onePass, `abc`},
+	{`^[a-c]*$`, onePass, `abcdabc`},
+	{`^(?:a*)$`, onePass, `aaaaaaa`},
+	{`^(?:(?:aa)|a)$`, onePass, `a`},
+	{`^[a-c]*`, NotOnePass, `abcdabc`},
+	{`^[a-c]*$`, onePass, `abc`},
+	{`^...$`, onePass, ``},
+	{`^(?:a|(?:aa))$`, onePass, `a`},
+	{`^[a-c]*`, NotOnePass, `abcabc`},
+	{`^a((b))c$`, onePass, noStr},
+	{`^a.[l-nA-Cg-j]?e$`, onePass, noStr},
+	{`^a((b))$`, onePass, noStr},
+	{`^a(?:(b)|(c))c$`, onePass, noStr},
+	{`^a(?:(b*)|(c))c$`, NotOnePass, noStr},
+	{`^a(?:b|c)$`, onePass, noStr},
+	{`^a(?:b?|c)$`, onePass, noStr},
+	{`^a(?:b?|c?)$`, NotOnePass, noStr},
+	{`^a(?:b?|c+)$`, onePass, noStr},
+	{`^a(?:b+|(bc))d$`, NotOnePass, noStr},
+	{`^a(?:bc)+$`, onePass, noStr},
+	{`^a(?:[bcd])+$`, onePass, noStr},
+	{`^a((?:[bcd])+)$`, onePass, noStr},
+	{`^a(:?b|c)*d$`, onePass, `abbbccbbcbbd"`},
+	{`^.bc(d|e)*$`, onePass, `abcddddddeeeededd`},
+	{`^(?:(?:aa)|.)$`, NotOnePass, `a`},
+	{`^(?:(?:a{1,2}){1,2})$`, NotOnePass, `aaaa`},
+}
+
+func TestCompileOnePass(t *testing.T) {
+	var (
+		p   *Prog
+		re  *Regexp
+		err error
+	)
+	for _, test := range onePassTests {
+		if re, err = Parse(test.re, Perl); err != nil {
+			t.Errorf("Parse(%q) got err:%s, want success", test.re, err)
+			continue
+		}
+		// needs to be done before compile...
+		re = re.Simplify()
+		if p, err = Compile(re); err != nil {
+			t.Errorf("Compile(%q) got err:%s, want success", test.re, err)
+			continue
+		}
+		onePass = p.CompileOnePass()
+		if (onePass == NotOnePass) != (test.onePass == NotOnePass) {
+			t.Errorf("CompileOnePass(%q) got %v, expected %v", test.re, onePass, test.onePass)
+		}
+	}
+}