Skip to content

G
go
Commit 1a4681ed authored by Russ Cox's avatar Russ Cox
Browse files
Options

exp/regexp/syntax: more escapes, character classes 

Still TODO: parsing optimizations

make_perl_groups.pl is copied with minimal modifications
(just to generate Go syntax instead of C++) from RE2.
Google Inc is "The RE2 Author" of that file and is one of
the Go Authors, so copyright changed to the Go Authors instead.

R=sam.thorogood, r, fvbommel, robert.hencke
CC=golang-dev
https://golang.org/cl/4612041
parent 340251e4
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 1006 additions and 104 deletions
src/pkg/exp/regexp/syntax/Makefile
View file @ 1a4681ed
...@@ -7,6 +7,7 @@ include ../../../../Make.inc ...@@ -7,6 +7,7 @@ include ../../../../Make.inc
TARG=exp/regexp/syntax TARG=exp/regexp/syntax
GOFILES=\ GOFILES=\
parse.go\ parse.go\
perl_groups.go\
regexp.go\ regexp.go\
include ../../../../Make.pkg include ../../../../Make.pkg
src/pkg/exp/regexp/syntax/make_perl_groups.pl 0 → 100755
View file @ 1a4681ed
#!/usr/bin/perl
# Copyright 2008 The Go Authors. All rights reserved.
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file.
# Modified version of RE2's make_perl_groups.pl.
# Generate table entries giving character ranges
# for POSIX/Perl character classes. Rather than
# figure out what the definition is, it is easier to ask
# Perl about each letter from 0-128 and write down
# its answer.
@posixclasses = (
"[:alnum:]",
"[:alpha:]",
"[:ascii:]",
"[:blank:]",
"[:cntrl:]",
"[:digit:]",
"[:graph:]",
"[:lower:]",
"[:print:]",
"[:punct:]",
"[:space:]",
"[:upper:]",
"[:word:]",
"[:xdigit:]",
);
@perlclasses = (
"\\d",
"\\s",
"\\w",
);
sub ComputeClass($) {
my @ranges;
my ($class) = @_;
my $regexp = "[$class]";
my $start = -1;
for (my $i=0; $i<=129; $i++) {
if ($i == 129) { $i = 256; }
if ($i <= 128 && chr($i) =~ $regexp) {
if ($start < 0) {
$start = $i;
}
} else {
if ($start >= 0) {
push @ranges, [$start, $i-1];
}
$start = -1;
}
}
return @ranges;
}
sub PrintClass($$@) {
my ($cname, $name, @ranges) = @_;
print "var code$cname = []int{ /* $name */\n";
for (my $i=0; $i<@ranges; $i++) {
my @a = @{$ranges[$i]};
printf "\t0x%x, 0x%x,\n", $a[0], $a[1];
}
print "}\n\n";
my $n = @ranges;
$negname = $name;
if ($negname =~ /:/) {
$negname =~ s/:/:^/;
} else {
$negname =~ y/a-z/A-Z/;
}
return "\t`$name`: {+1, code$cname},\n" .
"\t`$negname`: {-1, code$cname},\n";
}
my $gen = 0;
sub PrintClasses($@) {
my ($cname, @classes) = @_;
my @entries;
foreach my $cl (@classes) {
my @ranges = ComputeClass($cl);
push @entries, PrintClass(++$gen, $cl, @ranges);
}
print "var ${cname}Group = map[string]charGroup{\n";
foreach my $e (@entries) {
print $e;
}
print "}\n";
my $count = @entries;
}
print <<EOF;
// GENERATED BY make_perl_groups.pl; DO NOT EDIT.
// make_perl_groups.pl >perl_groups.go
package syntax
EOF
PrintClasses("perl", @perlclasses);
PrintClasses("posix", @posixclasses);
src/pkg/exp/regexp/syntax/parse.go
View file @ 1a4681ed
This diff is collapsed.
src/pkg/exp/regexp/syntax/parse_test.go
View file @ 1a4681ed
...@@ -32,68 +32,74 @@ var parseTests = []struct { ...@@ -32,68 +32,74 @@ var parseTests = []struct {
{"a*", "star{lit{a}}"}, {"a*", "star{lit{a}}"},
{"a+", "plus{lit{a}}"}, {"a+", "plus{lit{a}}"},
{"a?", "que{lit{a}}"}, {"a?", "que{lit{a}}"},
// { "a{2}", "rep{2,2 lit{a}}" }, {"a{2}", "rep{2,2 lit{a}}"},
// { "a{2,3}", "rep{2,3 lit{a}}" }, {"a{2,3}", "rep{2,3 lit{a}}"},
// { "a{2,}", "rep{2,-1 lit{a}}" }, {"a{2,}", "rep{2,-1 lit{a}}"},
// { "a*?", "nstar{lit{a}}" }, {"a*?", "nstar{lit{a}}"},
// { "a+?", "nplus{lit{a}}" }, {"a+?", "nplus{lit{a}}"},
// { "a??", "nque{lit{a}}" }, {"a??", "nque{lit{a}}"},
// { "a{2}?", "nrep{2,2 lit{a}}" }, {"a{2}?", "nrep{2,2 lit{a}}"},
// { "a{2,3}?", "nrep{2,3 lit{a}}" }, {"a{2,3}?", "nrep{2,3 lit{a}}"},
// { "a{2,}?", "nrep{2,-1 lit{a}}" }, {"a{2,}?", "nrep{2,-1 lit{a}}"},
{"", "emp{}"}, {"", "emp{}"},
// { "|", "emp{}" }, // alt{emp{}emp{}} but got factored // { "|", "emp{}" }, // alt{emp{}emp{}} but got factored
// { "|", "alt{emp{}emp{}}" }, {"|", "alt{emp{}emp{}}"},
{"|x|", "alt{emp{}lit{x}emp{}}"}, {"|x|", "alt{emp{}lit{x}emp{}}"},
{".", "dot{}"}, {".", "dot{}"},
{"^", "bol{}"}, {"^", "bol{}"},
{"$", "eol{}"}, {"$", "eol{}"},
// { "\\|", "lit{|}" }, {"\\|", "lit{|}"},
// { "\\(", "lit{(}" }, {"\\(", "lit{(}"},
// { "\\)", "lit{)}" }, {"\\)", "lit{)}"},
// { "\\*", "lit{*}" }, {"\\*", "lit{*}"},
// { "\\+", "lit{+}" }, {"\\+", "lit{+}"},
// { "\\?", "lit{?}" }, {"\\?", "lit{?}"},
// { "{", "lit{{}" }, {"{", "lit{{}"},
{"}", "lit{}}"}, {"}", "lit{}}"},
// { "\\.", "lit{.}" }, {"\\.", "lit{.}"},
// { "\\^", "lit{^}" }, {"\\^", "lit{^}"},
// { "\\$", "lit{$}" }, {"\\$", "lit{$}"},
// { "\\\\", "lit{\\}" }, {"\\\\", "lit{\\}"},
{"[ace]", "cc{0x61 0x63 0x65}"}, {"[ace]", "cc{0x61 0x63 0x65}"},
{"[abc]", "cc{0x61-0x63}"}, {"[abc]", "cc{0x61-0x63}"},
{"[a-z]", "cc{0x61-0x7a}"}, {"[a-z]", "cc{0x61-0x7a}"},
// { "[a]", "lit{a}" }, // { "[a]", "lit{a}" },
{"[a]", "cc{0x61}"}, {"[a]", "cc{0x61}"},
// { "\\-", "lit{-}" }, {"\\-", "lit{-}"},
{"-", "lit{-}"}, {"-", "lit{-}"},
// { "\\_", "lit{_}" }, {"\\_", "lit{_}"},
// Posix and Perl extensions // Posix and Perl extensions
// { "[[:lower:]]", "cc{0x61-0x7a}" }, {"[[:lower:]]", "cc{0x61-0x7a}"},
// { "[a-z]", "cc{0x61-0x7a}" }, {"[a-z]", "cc{0x61-0x7a}"},
// { "[^[:lower:]]", "cc{0x0-0x60 0x7b-0x10ffff}" }, {"[^[:lower:]]", "cc{0x0-0x60 0x7b-0x10ffff}"},
// { "[[:^lower:]]", "cc{0x0-0x60 0x7b-0x10ffff}" }, {"[[:^lower:]]", "cc{0x0-0x60 0x7b-0x10ffff}"},
// { "(?i)[[:lower:]]", "cc{0x41-0x5a 0x61-0x7a 0x17f 0x212a}" }, // { "(?i)[[:lower:]]", "cc{0x41-0x5a 0x61-0x7a 0x17f 0x212a}" },
// { "(?i)[a-z]", "cc{0x41-0x5a 0x61-0x7a 0x17f 0x212a}" }, // { "(?i)[a-z]", "cc{0x41-0x5a 0x61-0x7a 0x17f 0x212a}" },
// { "(?i)[^[:lower:]]", "cc{0x0-0x40 0x5b-0x60 0x7b-0x17e 0x180-0x2129 0x212b-0x10ffff}" }, // { "(?i)[^[:lower:]]", "cc{0x0-0x40 0x5b-0x60 0x7b-0x17e 0x180-0x2129 0x212b-0x10ffff}" },
// { "(?i)[[:^lower:]]", "cc{0x0-0x40 0x5b-0x60 0x7b-0x17e 0x180-0x2129 0x212b-0x10ffff}" }, // { "(?i)[[:^lower:]]", "cc{0x0-0x40 0x5b-0x60 0x7b-0x17e 0x180-0x2129 0x212b-0x10ffff}" },
// { "\\d", "cc{0x30-0x39}" }, {"\\d", "cc{0x30-0x39}"},
// { "\\D", "cc{0x0-0x2f 0x3a-0x10ffff}" }, {"\\D", "cc{0x0-0x2f 0x3a-0x10ffff}"},
// { "\\s", "cc{0x9-0xa 0xc-0xd 0x20}" }, {"\\s", "cc{0x9-0xa 0xc-0xd 0x20}"},
// { "\\S", "cc{0x0-0x8 0xb 0xe-0x1f 0x21-0x10ffff}" }, {"\\S", "cc{0x0-0x8 0xb 0xe-0x1f 0x21-0x10ffff}"},
// { "\\w", "cc{0x30-0x39 0x41-0x5a 0x5f 0x61-0x7a}" }, {"\\w", "cc{0x30-0x39 0x41-0x5a 0x5f 0x61-0x7a}"},
// { "\\W", "cc{0x0-0x2f 0x3a-0x40 0x5b-0x5e 0x60 0x7b-0x10ffff}" }, {"\\W", "cc{0x0-0x2f 0x3a-0x40 0x5b-0x5e 0x60 0x7b-0x10ffff}"},
// { "(?i)\\w", "cc{0x30-0x39 0x41-0x5a 0x5f 0x61-0x7a 0x17f 0x212a}" }, // { "(?i)\\w", "cc{0x30-0x39 0x41-0x5a 0x5f 0x61-0x7a 0x17f 0x212a}" },
// { "(?i)\\W", "cc{0x0-0x2f 0x3a-0x40 0x5b-0x5e 0x60 0x7b-0x17e 0x180-0x2129 0x212b-0x10ffff}" }, // { "(?i)\\W", "cc{0x0-0x2f 0x3a-0x40 0x5b-0x5e 0x60 0x7b-0x17e 0x180-0x2129 0x212b-0x10ffff}" },
// { "[^\\\\]", "cc{0x0-0x5b 0x5d-0x10ffff}" }, {"[^\\\\]", "cc{0x0-0x5b 0x5d-0x10ffff}"},
// { "\\C", "byte{}" }, // { "\\C", "byte{}" },
// Unicode, negatives, and a double negative. // Unicode, negatives, and a double negative.
// { "\\p{Braille}", "cc{0x2800-0x28ff}" }, {"\\p{Braille}", "cc{0x2800-0x28ff}"},
// { "\\P{Braille}", "cc{0x0-0x27ff 0x2900-0x10ffff}" }, {"\\P{Braille}", "cc{0x0-0x27ff 0x2900-0x10ffff}"},
// { "\\p{^Braille}", "cc{0x0-0x27ff 0x2900-0x10ffff}" }, {"\\p{^Braille}", "cc{0x0-0x27ff 0x2900-0x10ffff}"},
// { "\\P{^Braille}", "cc{0x2800-0x28ff}" }, {"\\P{^Braille}", "cc{0x2800-0x28ff}"},
{"\\pZ", "cc{0x20 0xa0 0x1680 0x180e 0x2000-0x200a 0x2028-0x2029 0x202f 0x205f 0x3000}"},
{"[\\p{Braille}]", "cc{0x2800-0x28ff}"},
{"[\\P{Braille}]", "cc{0x0-0x27ff 0x2900-0x10ffff}"},
{"[\\p{^Braille}]", "cc{0x0-0x27ff 0x2900-0x10ffff}"},
{"[\\P{^Braille}]", "cc{0x2800-0x28ff}"},
{"[\\pZ]", "cc{0x20 0xa0 0x1680 0x180e 0x2000-0x200a 0x2028-0x2029 0x202f 0x205f 0x3000}"},
// More interesting regular expressions. // More interesting regular expressions.
// { "a{,2}", "str{a{,2}}" }, // { "a{,2}", "str{a{,2}}" },
...@@ -101,7 +107,7 @@ var parseTests = []struct { ...@@ -101,7 +107,7 @@ var parseTests = []struct {
{"[a-zABC]", "cc{0x41-0x43 0x61-0x7a}"}, {"[a-zABC]", "cc{0x41-0x43 0x61-0x7a}"},
{"[^a]", "cc{0x0-0x60 0x62-0x10ffff}"}, {"[^a]", "cc{0x0-0x60 0x62-0x10ffff}"},
{"[\xce\xb1-\xce\xb5\xe2\x98\xba]", "cc{0x3b1-0x3b5 0x263a}"}, // utf-8 {"[\xce\xb1-\xce\xb5\xe2\x98\xba]", "cc{0x3b1-0x3b5 0x263a}"}, // utf-8
// { "a*{", "cat{star{lit{a}}lit{{}}" }, {"a*{", "cat{star{lit{a}}lit{{}}"},
// Test precedences // Test precedences
// { "(?:ab)*", "star{str{ab}}" }, // { "(?:ab)*", "star{str{ab}}" },
...@@ -114,30 +120,30 @@ var parseTests = []struct { ...@@ -114,30 +120,30 @@ var parseTests = []struct {
{"a(b|c)d", "cat{lit{a}cap{alt{lit{b}lit{c}}}lit{d}}"}, {"a(b|c)d", "cat{lit{a}cap{alt{lit{b}lit{c}}}lit{d}}"},
// Test flattening. // Test flattening.
// { "(?:a)", "lit{a}" }, {"(?:a)", "lit{a}"},
// { "(?:ab)(?:cd)", "str{abcd}" }, // { "(?:ab)(?:cd)", "str{abcd}" },
// { "(?:a|b)|(?:c|d)", "cc{0x61-0x64}" }, // { "(?:a|b)|(?:c|d)", "cc{0x61-0x64}" },
// { "a|.", "dot{}" }, // { "a|.", "dot{}" },
// { ".|a", "dot{}" }, // { ".|a", "dot{}" },
// Test Perl quoted literals // Test Perl quoted literals
// { "\\Q+|*?{[\\E", "str{+|*?{[}" }, {"\\Q+|*?{[\\E", "str{+|*?{[}"},
// { "\\Q+\\E+", "plus{lit{+}}" }, {"\\Q+\\E+", "plus{lit{+}}"},
// { "\\Q\\\\E", "lit{\\}" }, {"\\Q\\\\E", "lit{\\}"},
// { "\\Q\\\\\\E", "str{\\\\}" }, {"\\Q\\\\\\E", "str{\\\\}"},
// Test Perl \A and \z // Test Perl \A and \z
// { "(?m)^", "bol{}" }, {"(?m)^", "bol{}"},
// { "(?m)$", "eol{}" }, {"(?m)$", "eol{}"},
// { "(?-m)^", "bot{}" }, {"(?-m)^", "bot{}"},
// { "(?-m)$", "eot{}" }, {"(?-m)$", "eot{}"},
// { "(?m)\\A", "bot{}" }, {"(?m)\\A", "bot{}"},
// { "(?m)\\z", "eot{\\z}" }, {"(?m)\\z", "eot{\\z}"},
// { "(?-m)\\A", "bot{}" }, {"(?-m)\\A", "bot{}"},
// { "(?-m)\\z", "eot{\\z}" }, {"(?-m)\\z", "eot{\\z}"},
// Test named captures // Test named captures
// { "(?P<name>a)", "cap{name:lit{a}}" }, {"(?P<name>a)", "cap{name:lit{a}}"},
// Case-folded literals // Case-folded literals
// { "[Aa]", "litfold{a}" }, // { "[Aa]", "litfold{a}" },
......
src/pkg/exp/regexp/syntax/perl_groups.go 0 → 100644
View file @ 1a4681ed
// GENERATED BY make_perl_groups.pl; DO NOT EDIT.
// make_perl_groups.pl >perl_groups.go
package syntax
var code1 = []int{ /* \d */
0x30, 0x39,
}
var code2 = []int{ /* \s */
0x9, 0xa,
0xc, 0xd,
0x20, 0x20,
}
var code3 = []int{ /* \w */
0x30, 0x39,
0x41, 0x5a,
0x5f, 0x5f,
0x61, 0x7a,
}
var perlGroup = map[string]charGroup{
`\d`: {+1, code1},
`\D`: {-1, code1},
`\s`: {+1, code2},
`\S`: {-1, code2},
`\w`: {+1, code3},
`\W`: {-1, code3},
}
var code4 = []int{ /* [:alnum:] */
0x30, 0x39,
0x41, 0x5a,
0x61, 0x7a,
}
var code5 = []int{ /* [:alpha:] */
0x41, 0x5a,
0x61, 0x7a,
}
var code6 = []int{ /* [:ascii:] */
0x0, 0x7f,
}
var code7 = []int{ /* [:blank:] */
0x9, 0x9,
0x20, 0x20,
}
var code8 = []int{ /* [:cntrl:] */
0x0, 0x1f,
0x7f, 0x7f,
}
var code9 = []int{ /* [:digit:] */
0x30, 0x39,
}
var code10 = []int{ /* [:graph:] */
0x21, 0x7e,
}
var code11 = []int{ /* [:lower:] */
0x61, 0x7a,
}
var code12 = []int{ /* [:print:] */
0x20, 0x7e,
}
var code13 = []int{ /* [:punct:] */
0x21, 0x2f,
0x3a, 0x40,
0x5b, 0x60,
0x7b, 0x7e,
}
var code14 = []int{ /* [:space:] */
0x9, 0xd,
0x20, 0x20,
}
var code15 = []int{ /* [:upper:] */
0x41, 0x5a,
}
var code16 = []int{ /* [:word:] */
0x30, 0x39,
0x41, 0x5a,
0x5f, 0x5f,
0x61, 0x7a,
}
var code17 = []int{ /* [:xdigit:] */
0x30, 0x39,
0x41, 0x46,
0x61, 0x66,
}
var posixGroup = map[string]charGroup{
`[:alnum:]`: {+1, code4},
`[:^alnum:]`: {-1, code4},
`[:alpha:]`: {+1, code5},
`[:^alpha:]`: {-1, code5},
`[:ascii:]`: {+1, code6},
`[:^ascii:]`: {-1, code6},
`[:blank:]`: {+1, code7},
`[:^blank:]`: {-1, code7},
`[:cntrl:]`: {+1, code8},
`[:^cntrl:]`: {-1, code8},
`[:digit:]`: {+1, code9},
`[:^digit:]`: {-1, code9},
`[:graph:]`: {+1, code10},
`[:^graph:]`: {-1, code10},
`[:lower:]`: {+1, code11},
`[:^lower:]`: {-1, code11},
`[:print:]`: {+1, code12},
`[:^print:]`: {-1, code12},
`[:punct:]`: {+1, code13},
`[:^punct:]`: {-1, code13},
`[:space:]`: {+1, code14},
`[:^space:]`: {-1, code14},
`[:upper:]`: {+1, code15},
`[:^upper:]`: {-1, code15},
`[:word:]`: {+1, code16},
`[:^word:]`: {-1, code16},
`[:xdigit:]`: {+1, code17},
`[:^xdigit:]`: {-1, code17},
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7