Skip to content

G
go
Commit 7e1a3e9f authored by Russ Cox's avatar Russ Cox
Browse files
Options

exp/regexp/syntax: incremental concat, alternate 

Also reuse of *Regexp nodes.

I believe this is the end of the parser.
The only non-execution code that remains is
the code to expand x{3,5} into simpler operations.

R=sam.thorogood, r
CC=golang-dev
https://golang.org/cl/4629078
parent 3379414b
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 365 additions and 175 deletions
src/pkg/exp/regexp/syntax/parse.go
View file @ 7e1a3e9f
This diff is collapsed.
src/pkg/exp/regexp/syntax/parse_test.go
View file @ 7e1a3e9f
......@@ -16,148 +16,146 @@ var parseTests = []struct {
Dump string
}{
// Base cases
{"a", "lit{a}"},
{"a.", "cat{lit{a}dot{}}"},
{"a.b", "cat{lit{a}dot{}lit{b}}"},
// { "ab", "str{ab}" },
{"ab", "cat{lit{a}lit{b}}"},
{"a.b.c", "cat{lit{a}dot{}lit{b}dot{}lit{c}}"},
// { "abc", "str{abc}" },
{"abc", "cat{lit{a}lit{b}lit{c}}"},
{"a|^", "alt{lit{a}bol{}}"},
// { "a|b", "cc{0x61-0x62}" },
{"a|b", "alt{lit{a}lit{b}}"},
{"(a)", "cap{lit{a}}"},
{"(a)|b", "alt{cap{lit{a}}lit{b}}"},
{"a*", "star{lit{a}}"},
{"a+", "plus{lit{a}}"},
{"a?", "que{lit{a}}"},
{"a{2}", "rep{2,2 lit{a}}"},
{"a{2,3}", "rep{2,3 lit{a}}"},
{"a{2,}", "rep{2,-1 lit{a}}"},
{"a*?", "nstar{lit{a}}"},
{"a+?", "nplus{lit{a}}"},
{"a??", "nque{lit{a}}"},
{"a{2}?", "nrep{2,2 lit{a}}"},
{"a{2,3}?", "nrep{2,3 lit{a}}"},
{"a{2,}?", "nrep{2,-1 lit{a}}"},
{"", "emp{}"},
// { "|", "emp{}" }, // alt{emp{}emp{}} but got factored
{"|", "alt{emp{}emp{}}"},
{"|x|", "alt{emp{}lit{x}emp{}}"},
{".", "dot{}"},
{"^", "bol{}"},
{"$", "eol{}"},
{"\\|", "lit{|}"},
{"\\(", "lit{(}"},
{"\\)", "lit{)}"},
{"\\*", "lit{*}"},
{"\\+", "lit{+}"},
{"\\?", "lit{?}"},
{"{", "lit{{}"},
{"}", "lit{}}"},
{"\\.", "lit{.}"},
{"\\^", "lit{^}"},
{"\\$", "lit{$}"},
{"\\\\", "lit{\\}"},
{"[ace]", "cc{0x61 0x63 0x65}"},
{"[abc]", "cc{0x61-0x63}"},
{"[a-z]", "cc{0x61-0x7a}"},
// { "[a]", "lit{a}" },
{"[a]", "cc{0x61}"},
{"\\-", "lit{-}"},
{"-", "lit{-}"},
{"\\_", "lit{_}"},
{`a`, `lit{a}`},
{`a.`, `cat{lit{a}dot{}}`},
{`a.b`, `cat{lit{a}dot{}lit{b}}`},
{`ab`, `str{ab}`},
{`a.b.c`, `cat{lit{a}dot{}lit{b}dot{}lit{c}}`},
{`abc`, `str{abc}`},
{`a|^`, `alt{lit{a}bol{}}`},
{`a|b`, `cc{0x61-0x62}`},
{`(a)`, `cap{lit{a}}`},
{`(a)|b`, `alt{cap{lit{a}}lit{b}}`},
{`a*`, `star{lit{a}}`},
{`a+`, `plus{lit{a}}`},
{`a?`, `que{lit{a}}`},
{`a{2}`, `rep{2,2 lit{a}}`},
{`a{2,3}`, `rep{2,3 lit{a}}`},
{`a{2,}`, `rep{2,-1 lit{a}}`},
{`a*?`, `nstar{lit{a}}`},
{`a+?`, `nplus{lit{a}}`},
{`a??`, `nque{lit{a}}`},
{`a{2}?`, `nrep{2,2 lit{a}}`},
{`a{2,3}?`, `nrep{2,3 lit{a}}`},
{`a{2,}?`, `nrep{2,-1 lit{a}}`},
{``, `emp{}`},
// { `|`, `emp{}` }, // alt{emp{}emp{}} but got factored
{`|`, `alt{emp{}emp{}}`},
{`|x|`, `alt{emp{}lit{x}emp{}}`},
{`.`, `dot{}`},
{`^`, `bol{}`},
{`$`, `eol{}`},
{`\|`, `lit{|}`},
{`\(`, `lit{(}`},
{`\)`, `lit{)}`},
{`\*`, `lit{*}`},
{`\+`, `lit{+}`},
{`\?`, `lit{?}`},
{`{`, `lit{{}`},
{`}`, `lit{}}`},
{`\.`, `lit{.}`},
{`\^`, `lit{^}`},
{`\$`, `lit{$}`},
{`\\`, `lit{\}`},
{`[ace]`, `cc{0x61 0x63 0x65}`},
{`[abc]`, `cc{0x61-0x63}`},
{`[a-z]`, `cc{0x61-0x7a}`},
{`[a]`, `lit{a}`},
{`\-`, `lit{-}`},
{`-`, `lit{-}`},
{`\_`, `lit{_}`},
// Posix and Perl extensions
{"[[:lower:]]", "cc{0x61-0x7a}"},
{"[a-z]", "cc{0x61-0x7a}"},
{"[^[:lower:]]", "cc{0x0-0x60 0x7b-0x10ffff}"},
{"[[:^lower:]]", "cc{0x0-0x60 0x7b-0x10ffff}"},
{"(?i)[[:lower:]]", "cc{0x41-0x5a 0x61-0x7a 0x17f 0x212a}"},
{"(?i)[a-z]", "cc{0x41-0x5a 0x61-0x7a 0x17f 0x212a}"},
{"(?i)[^[:lower:]]", "cc{0x0-0x40 0x5b-0x60 0x7b-0x17e 0x180-0x2129 0x212b-0x10ffff}"},
{"(?i)[[:^lower:]]", "cc{0x0-0x40 0x5b-0x60 0x7b-0x17e 0x180-0x2129 0x212b-0x10ffff}"},
{"\\d", "cc{0x30-0x39}"},
{"\\D", "cc{0x0-0x2f 0x3a-0x10ffff}"},
{"\\s", "cc{0x9-0xa 0xc-0xd 0x20}"},
{"\\S", "cc{0x0-0x8 0xb 0xe-0x1f 0x21-0x10ffff}"},
{"\\w", "cc{0x30-0x39 0x41-0x5a 0x5f 0x61-0x7a}"},
{"\\W", "cc{0x0-0x2f 0x3a-0x40 0x5b-0x5e 0x60 0x7b-0x10ffff}"},
{"(?i)\\w", "cc{0x30-0x39 0x41-0x5a 0x5f 0x61-0x7a 0x17f 0x212a}"},
{"(?i)\\W", "cc{0x0-0x2f 0x3a-0x40 0x5b-0x5e 0x60 0x7b-0x17e 0x180-0x2129 0x212b-0x10ffff}"},
{"[^\\\\]", "cc{0x0-0x5b 0x5d-0x10ffff}"},
// { "\\C", "byte{}" },
{`[[:lower:]]`, `cc{0x61-0x7a}`},
{`[a-z]`, `cc{0x61-0x7a}`},
{`[^[:lower:]]`, `cc{0x0-0x60 0x7b-0x10ffff}`},
{`[[:^lower:]]`, `cc{0x0-0x60 0x7b-0x10ffff}`},
{`(?i)[[:lower:]]`, `cc{0x41-0x5a 0x61-0x7a 0x17f 0x212a}`},
{`(?i)[a-z]`, `cc{0x41-0x5a 0x61-0x7a 0x17f 0x212a}`},
{`(?i)[^[:lower:]]`, `cc{0x0-0x40 0x5b-0x60 0x7b-0x17e 0x180-0x2129 0x212b-0x10ffff}`},
{`(?i)[[:^lower:]]`, `cc{0x0-0x40 0x5b-0x60 0x7b-0x17e 0x180-0x2129 0x212b-0x10ffff}`},
{`\d`, `cc{0x30-0x39}`},
{`\D`, `cc{0x0-0x2f 0x3a-0x10ffff}`},
{`\s`, `cc{0x9-0xa 0xc-0xd 0x20}`},
{`\S`, `cc{0x0-0x8 0xb 0xe-0x1f 0x21-0x10ffff}`},
{`\w`, `cc{0x30-0x39 0x41-0x5a 0x5f 0x61-0x7a}`},
{`\W`, `cc{0x0-0x2f 0x3a-0x40 0x5b-0x5e 0x60 0x7b-0x10ffff}`},
{`(?i)\w`, `cc{0x30-0x39 0x41-0x5a 0x5f 0x61-0x7a 0x17f 0x212a}`},
{`(?i)\W`, `cc{0x0-0x2f 0x3a-0x40 0x5b-0x5e 0x60 0x7b-0x17e 0x180-0x2129 0x212b-0x10ffff}`},
{`[^\\]`, `cc{0x0-0x5b 0x5d-0x10ffff}`},
// { `\C`, `byte{}` }, // probably never
// Unicode, negatives, and a double negative.
{"\\p{Braille}", "cc{0x2800-0x28ff}"},
{"\\P{Braille}", "cc{0x0-0x27ff 0x2900-0x10ffff}"},
{"\\p{^Braille}", "cc{0x0-0x27ff 0x2900-0x10ffff}"},
{"\\P{^Braille}", "cc{0x2800-0x28ff}"},
{"\\pZ", "cc{0x20 0xa0 0x1680 0x180e 0x2000-0x200a 0x2028-0x2029 0x202f 0x205f 0x3000}"},
{"[\\p{Braille}]", "cc{0x2800-0x28ff}"},
{"[\\P{Braille}]", "cc{0x0-0x27ff 0x2900-0x10ffff}"},
{"[\\p{^Braille}]", "cc{0x0-0x27ff 0x2900-0x10ffff}"},
{"[\\P{^Braille}]", "cc{0x2800-0x28ff}"},
{"[\\pZ]", "cc{0x20 0xa0 0x1680 0x180e 0x2000-0x200a 0x2028-0x2029 0x202f 0x205f 0x3000}"},
{"\\p{Lu}", mkCharClass(unicode.IsUpper)},
{"[\\p{Lu}]", mkCharClass(unicode.IsUpper)},
{"(?i)[\\p{Lu}]", mkCharClass(isUpperFold)},
{`\p{Braille}`, `cc{0x2800-0x28ff}`},
{`\P{Braille}`, `cc{0x0-0x27ff 0x2900-0x10ffff}`},
{`\p{^Braille}`, `cc{0x0-0x27ff 0x2900-0x10ffff}`},
{`\P{^Braille}`, `cc{0x2800-0x28ff}`},
{`\pZ`, `cc{0x20 0xa0 0x1680 0x180e 0x2000-0x200a 0x2028-0x2029 0x202f 0x205f 0x3000}`},
{`[\p{Braille}]`, `cc{0x2800-0x28ff}`},
{`[\P{Braille}]`, `cc{0x0-0x27ff 0x2900-0x10ffff}`},
{`[\p{^Braille}]`, `cc{0x0-0x27ff 0x2900-0x10ffff}`},
{`[\P{^Braille}]`, `cc{0x2800-0x28ff}`},
{`[\pZ]`, `cc{0x20 0xa0 0x1680 0x180e 0x2000-0x200a 0x2028-0x2029 0x202f 0x205f 0x3000}`},
{`\p{Lu}`, mkCharClass(unicode.IsUpper)},
{`[\p{Lu}]`, mkCharClass(unicode.IsUpper)},
{`(?i)[\p{Lu}]`, mkCharClass(isUpperFold)},
// Hex, octal.
{"[\\012-\\234]\\141", "cat{cc{0xa-0x9c}lit{a}}"},
{"[\\x{41}-\\x7a]\\x61", "cat{cc{0x41-0x7a}lit{a}}"},
{`[\012-\234]\141`, `cat{cc{0xa-0x9c}lit{a}}`},
{`[\x{41}-\x7a]\x61`, `cat{cc{0x41-0x7a}lit{a}}`},
// More interesting regular expressions.
// { "a{,2}", "str{a{,2}}" },
// { "\\.\\^\\$\\\\", "str{.^$\\}" },
{"[a-zABC]", "cc{0x41-0x43 0x61-0x7a}"},
{"[^a]", "cc{0x0-0x60 0x62-0x10ffff}"},
{"[\xce\xb1-\xce\xb5\xe2\x98\xba]", "cc{0x3b1-0x3b5 0x263a}"}, // utf-8
{"a*{", "cat{star{lit{a}}lit{{}}"},
{`a{,2}`, `str{a{,2}}`},
{`\.\^\$\\`, `str{.^$\}`},
{`[a-zABC]`, `cc{0x41-0x43 0x61-0x7a}`},
{`[^a]`, `cc{0x0-0x60 0x62-0x10ffff}`},
{`[α-ε☺]`, `cc{0x3b1-0x3b5 0x263a}`}, // utf-8
{`a*{`, `cat{star{lit{a}}lit{{}}`},
// Test precedences
// { "(?:ab)*", "star{str{ab}}" },
// { "(ab)*", "star{cap{str{ab}}}" },
// { "ab|cd", "alt{str{ab}str{cd}}" },
// { "a(b|c)d", "cat{lit{a}cap{cc{0x62-0x63}}lit{d}}" },
{"(?:ab)*", "star{cat{lit{a}lit{b}}}"},
{"(ab)*", "star{cap{cat{lit{a}lit{b}}}}"},
{"ab|cd", "alt{cat{lit{a}lit{b}}cat{lit{c}lit{d}}}"},
{"a(b|c)d", "cat{lit{a}cap{alt{lit{b}lit{c}}}lit{d}}"},
{`(?:ab)*`, `star{str{ab}}`},
{`(ab)*`, `star{cap{str{ab}}}`},
{`ab|cd`, `alt{str{ab}str{cd}}`},
{`a(b|c)d`, `cat{lit{a}cap{cc{0x62-0x63}}lit{d}}`},
// Test flattening.
{"(?:a)", "lit{a}"},
// { "(?:ab)(?:cd)", "str{abcd}" },
// { "(?:a|b)|(?:c|d)", "cc{0x61-0x64}" },
// { "a|.", "dot{}" },
// { ".|a", "dot{}" },
{`(?:a)`, `lit{a}`},
{`(?:ab)(?:cd)`, `str{abcd}`},
{`(?:a+b+)(?:c+d+)`, `cat{plus{lit{a}}plus{lit{b}}plus{lit{c}}plus{lit{d}}}`},
{`(?:a+|b+)|(?:c+|d+)`, `alt{plus{lit{a}}plus{lit{b}}plus{lit{c}}plus{lit{d}}}`},
{`(?:a|b)|(?:c|d)`, `cc{0x61-0x64}`},
{`a|.`, `dot{}`},
{`.|a`, `dot{}`},
{`(?:[abc]|A|Z|hello|world)`, `alt{cc{0x41 0x5a 0x61-0x63}str{hello}str{world}}`},
{`(?:[abc]|A|Z)`, `cc{0x41 0x5a 0x61-0x63}`},
// Test Perl quoted literals
{"\\Q+|*?{[\\E", "str{+|*?{[}"},
{"\\Q+\\E+", "plus{lit{+}}"},
{"\\Q\\\\E", "lit{\\}"},
{"\\Q\\\\\\E", "str{\\\\}"},
{`\Q+|*?{[\E`, `str{+|*?{[}`},
{`\Q+\E+`, `plus{lit{+}}`},
{`\Q\\E`, `lit{\}`},
{`\Q\\\E`, `str{\\}`},
// Test Perl \A and \z
{"(?m)^", "bol{}"},
{"(?m)$", "eol{}"},
{"(?-m)^", "bot{}"},
{"(?-m)$", "eot{}"},
{"(?m)\\A", "bot{}"},
{"(?m)\\z", "eot{\\z}"},
{"(?-m)\\A", "bot{}"},
{"(?-m)\\z", "eot{\\z}"},
{`(?m)^`, `bol{}`},
{`(?m)$`, `eol{}`},
{`(?-m)^`, `bot{}`},
{`(?-m)$`, `eot{}`},
{`(?m)\A`, `bot{}`},
{`(?m)\z`, `eot{\z}`},
{`(?-m)\A`, `bot{}`},
{`(?-m)\z`, `eot{\z}`},
// Test named captures
{"(?P<name>a)", "cap{name:lit{a}}"},
{`(?P<name>a)`, `cap{name:lit{a}}`},
// Case-folded literals
// { "[Aa]", "litfold{a}" },
{`[Aa]`, `litfold{A}`},
{`[\x{100}\x{101}]`, `litfold{Ā}`},
{`[Δδ]`, `litfold{Δ}`},
// Strings
// { "abcde", "str{abcde}" },
// { "[Aa][Bb]cd", "cat{strfold{ab}str{cd}}" },
{`abcde`, `str{abcde}`},
{`[Aa][Bb]cd`, `cat{strfold{AB}str{cd}}`},
}
const testFlags = MatchNL | PerlX | UnicodeGroups
......@@ -230,8 +228,9 @@ func dumpRegexp(b *bytes.Buffer, re *Regexp) {
}
if re.Flags&FoldCase != 0 {
for _, r := range re.Rune {
if unicode.ToUpper(r) != r {
if unicode.SimpleFold(r) != r {
b.WriteString("fold")
break
}
}
}
......
src/pkg/exp/regexp/syntax/regexp.go
View file @ 7e1a3e9f
......@@ -33,6 +33,8 @@ type Regexp struct {
type Op uint8
// Operators are listed in precedence order, tightest binding to weakest.
// Character class operators are listed simplest to most complex
// (OpLiteral, OpCharClass, OpAnyCharNotNL, OpAnyChar).
const (
OpNoMatch Op = 1 + iota // matches no strings
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7