Skip to content

G
go
Commit 746d6368 authored by Marcel van Lohuizen's avatar Marcel van Lohuizen
Browse files
Options

unicode: upgrade to Unicode 6.3.0 

This is a relatively minor change.

This does not result in changes to go.text/unicode/norm. The go.text
packages will therefore be relatively unaffected. It does make the
way for an upgrade to CLDR 24, though.

The tests of all.bash pass, as well as the tests in go.text after
this update.

LGTM=r
R=r
CC=golang-codereviews
https://golang.org/cl/65400044
parent 72f2b56a
Hide whitespace changes
Inline Side-by-side
Showing with 40 additions and 32 deletions
src/pkg/bufio/scan.go
View file @ 746d6368
......@@ -306,7 +306,7 @@ func isSpace(r rune) bool {
return true
}
switch r {
case '\u1680', '\u180e', '\u2028', '\u2029', '\u202f', '\u205f', '\u3000':
case '\u1680', '\u2028', '\u2029', '\u202f', '\u205f', '\u3000':
return true
}
return false
......
src/pkg/fmt/scan.go
View file @ 746d6368
......@@ -284,7 +284,6 @@ var space = [][2]uint16{
{0x0085, 0x0085},
{0x00a0, 0x00a0},
{0x1680, 0x1680},
{0x180e, 0x180e},
{0x2000, 0x200a},
{0x2028, 0x2029},
{0x202f, 0x202f},
......
src/pkg/regexp/syntax/parse_test.go
View file @ 746d6368
......@@ -100,12 +100,12 @@ var parseTests = []parseTest{
{`\P{Braille}`, `cc{0x0-0x27ff 0x2900-0x10ffff}`},
{`\p{^Braille}`, `cc{0x0-0x27ff 0x2900-0x10ffff}`},
{`\P{^Braille}`, `cc{0x2800-0x28ff}`},
{`\pZ`, `cc{0x20 0xa0 0x1680 0x180e 0x2000-0x200a 0x2028-0x2029 0x202f 0x205f 0x3000}`},
{`\pZ`, `cc{0x20 0xa0 0x1680 0x2000-0x200a 0x2028-0x2029 0x202f 0x205f 0x3000}`},
{`[\p{Braille}]`, `cc{0x2800-0x28ff}`},
{`[\P{Braille}]`, `cc{0x0-0x27ff 0x2900-0x10ffff}`},
{`[\p{^Braille}]`, `cc{0x0-0x27ff 0x2900-0x10ffff}`},
{`[\P{^Braille}]`, `cc{0x2800-0x28ff}`},
{`[\pZ]`, `cc{0x20 0xa0 0x1680 0x180e 0x2000-0x200a 0x2028-0x2029 0x202f 0x205f 0x3000}`},
{`[\pZ]`, `cc{0x20 0xa0 0x1680 0x2000-0x200a 0x2028-0x2029 0x202f 0x205f 0x3000}`},
{`\p{Lu}`, mkCharClass(unicode.IsUpper)},
{`[\p{Lu}]`, mkCharClass(unicode.IsUpper)},
{`(?i)[\p{Lu}]`, mkCharClass(isUpperFold)},
......
src/pkg/unicode/maketables.go
View file @ 746d6368
......@@ -40,7 +40,7 @@ func main() {
var dataURL = flag.String("data", "", "full URL for UnicodeData.txt; defaults to --url/UnicodeData.txt")
var casefoldingURL = flag.String("casefolding", "", "full URL for CaseFolding.txt; defaults to --url/CaseFolding.txt")
var url = flag.String("url",
"http://www.unicode.org/Public/6.2.0/ucd/",
"http://www.unicode.org/Public/6.3.0/ucd/",
"URL of Unicode database directory")
var tablelist = flag.String("tables",
"all",
......
src/pkg/unicode/script_test.go
View file @ 746d6368
......@@ -182,7 +182,7 @@ var inPropTest = []T{
{0x0EC4, "Logical_Order_Exception"},
{0x2FFFF, "Noncharacter_Code_Point"},
{0x065E, "Other_Alphabetic"},
{0x2069, "Other_Default_Ignorable_Code_Point"},
{0x2065, "Other_Default_Ignorable_Code_Point"},
{0x0BD7, "Other_Grapheme_Extend"},
{0x0387, "Other_ID_Continue"},
{0x212E, "Other_ID_Start"},
......
src/pkg/unicode/tables.go
View file @ 746d6368
......@@ -3,13 +3,13 @@
// license that can be found in the LICENSE file.
// Generated by running
// maketables --tables=all --data=http://www.unicode.org/Public/6.2.0/ucd/UnicodeData.txt --casefolding=http://www.unicode.org/Public/6.2.0/ucd/CaseFolding.txt
// maketables --tables=all --data=http://www.unicode.org/Public/6.3.0/ucd/UnicodeData.txt --casefolding=http://www.unicode.org/Public/6.3.0/ucd/CaseFolding.txt
// DO NOT EDIT
package unicode
// Version is the Unicode edition from which the tables are derived.
const Version = "6.2.0"
const Version = "6.3.0"
// Categories is the set of Unicode category tables.
var Categories = map[string]*RangeTable{
......@@ -57,11 +57,12 @@ var _C = &RangeTable{
{0x007f, 0x009f, 1},
{0x00ad, 0x0600, 1363},
{0x0601, 0x0604, 1},
{0x06dd, 0x070f, 50},
{0x061c, 0x06dd, 193},
{0x070f, 0x180e, 4351},
{0x200b, 0x200f, 1},
{0x202a, 0x202e, 1},
{0x2060, 0x2064, 1},
{0x206a, 0x206f, 1},
{0x2066, 0x206f, 1},
{0xd800, 0xf8ff, 1},
{0xfeff, 0xfff9, 250},
{0xfffa, 0xfffb, 1},
......@@ -89,11 +90,12 @@ var _Cf = &RangeTable{
R16: []Range16{
{0x00ad, 0x0600, 1363},
{0x0601, 0x0604, 1},
{0x06dd, 0x070f, 50},
{0x061c, 0x06dd, 193},
{0x070f, 0x180e, 4351},
{0x200b, 0x200f, 1},
{0x202a, 0x202e, 1},
{0x2060, 0x2064, 1},
{0x206a, 0x206f, 1},
{0x2066, 0x206f, 1},
{0xfeff, 0xfff9, 250},
{0xfffa, 0xfffb, 1},
},
......@@ -1549,7 +1551,7 @@ var _Mc = &RangeTable{
{0x1933, 0x1938, 1},
{0x19b0, 0x19c0, 1},
{0x19c8, 0x19c9, 1},
{0x1a19, 0x1a1b, 1},
{0x1a19, 0x1a1a, 1},
{0x1a55, 0x1a57, 2},
{0x1a61, 0x1a63, 2},
{0x1a64, 0x1a6d, 9},
......@@ -1721,8 +1723,8 @@ var _Mn = &RangeTable{
{0x1932, 0x1939, 7},
{0x193a, 0x193b, 1},
{0x1a17, 0x1a18, 1},
{0x1a56, 0x1a58, 2},
{0x1a59, 0x1a5e, 1},
{0x1a1b, 0x1a56, 59},
{0x1a58, 0x1a5e, 1},
{0x1a60, 0x1a62, 2},
{0x1a65, 0x1a6c, 1},
{0x1a73, 0x1a7c, 1},
......@@ -2090,6 +2092,7 @@ var _P = &RangeTable{
{0x2053, 0x205e, 1},
{0x207d, 0x207e, 1},
{0x208d, 0x208e, 1},
{0x2308, 0x230b, 1},
{0x2329, 0x232a, 1},
{0x2768, 0x2775, 1},
{0x27c5, 0x27c6, 1},
......@@ -2187,7 +2190,8 @@ var _Pe = &RangeTable{
{0x007d, 0x0f3b, 3774},
{0x0f3d, 0x169c, 1887},
{0x2046, 0x207e, 56},
{0x208e, 0x232a, 668},
{0x208e, 0x2309, 635},
{0x230b, 0x232a, 31},
{0x2769, 0x2775, 2},
{0x27c6, 0x27e7, 33},
{0x27e9, 0x27ef, 2},
......@@ -2364,7 +2368,8 @@ var _Ps = &RangeTable{
{0x0f3c, 0x169b, 1887},
{0x201a, 0x201e, 4},
{0x2045, 0x207d, 56},
{0x208d, 0x2329, 668},
{0x208d, 0x2308, 635},
{0x230a, 0x2329, 31},
{0x2768, 0x2774, 2},
{0x27c5, 0x27e6, 33},
{0x27e8, 0x27ee, 2},
......@@ -2454,7 +2459,8 @@ var _S = &RangeTable{
{0x2141, 0x2144, 1},
{0x214a, 0x214d, 1},
{0x214f, 0x2190, 65},
{0x2191, 0x2328, 1},
{0x2191, 0x2307, 1},
{0x230c, 0x2328, 1},
{0x232b, 0x23f3, 1},
{0x2400, 0x2426, 1},
{0x2440, 0x244a, 1},
......@@ -2634,7 +2640,6 @@ var _Sm = &RangeTable{
{0x21cf, 0x21d2, 3},
{0x21d4, 0x21f4, 32},
{0x21f5, 0x22ff, 1},
{0x2308, 0x230b, 1},
{0x2320, 0x2321, 1},
{0x237c, 0x239b, 31},
{0x239c, 0x23b3, 1},
......@@ -2822,8 +2827,8 @@ var _So = &RangeTable{
var _Z = &RangeTable{
R16: []Range16{
{0x0020, 0x00a0, 128},
{0x1680, 0x180e, 398},
{0x2000, 0x200a, 1},
{0x1680, 0x2000, 2432},
{0x2001, 0x200a, 1},
{0x2028, 0x2029, 1},
{0x202f, 0x205f, 48},
{0x3000, 0x3000, 1},
......@@ -2846,8 +2851,8 @@ var _Zp = &RangeTable{
var _Zs = &RangeTable{
R16: []Range16{
{0x0020, 0x00a0, 128},
{0x1680, 0x180e, 398},
{0x2000, 0x200a, 1},
{0x1680, 0x2000, 2432},
{0x2001, 0x200a, 1},
{0x202f, 0x205f, 48},
{0x3000, 0x3000, 1},
},
......@@ -2906,7 +2911,7 @@ var (
)
// Generated by running
// maketables --scripts=all --url=http://www.unicode.org/Public/6.2.0/ucd/
// maketables --scripts=all --url=http://www.unicode.org/Public/6.3.0/ucd/
// DO NOT EDIT
// Scripts is the set of Unicode script tables.
......@@ -3020,6 +3025,7 @@ var _Arabic = &RangeTable{
{0x0600, 0x0604, 1},
{0x0606, 0x060b, 1},
{0x060d, 0x061a, 1},
{0x061c, 0x061c, 1},
{0x061e, 0x061e, 1},
{0x0620, 0x063f, 1},
{0x0641, 0x064a, 1},
......@@ -3249,7 +3255,7 @@ var _Common = &RangeTable{
{0x1cf5, 0x1cf6, 1},
{0x2000, 0x200b, 1},
{0x200e, 0x2064, 1},
{0x206a, 0x2070, 1},
{0x2066, 0x2070, 1},
{0x2074, 0x207e, 1},
{0x2080, 0x208e, 1},
{0x20a0, 0x20ba, 1},
......@@ -3285,6 +3291,7 @@ var _Common = &RangeTable{
{0xa700, 0xa721, 1},
{0xa788, 0xa78a, 1},
{0xa830, 0xa839, 1},
{0xa9cf, 0xa9cf, 1},
{0xfd3e, 0xfd3f, 1},
{0xfdfd, 0xfdfd, 1},
{0xfe10, 0xfe19, 1},
......@@ -3714,7 +3721,7 @@ var _Inscriptional_Parthian = &RangeTable{
var _Javanese = &RangeTable{
R16: []Range16{
{0xa980, 0xa9cd, 1},
{0xa9cf, 0xa9d9, 1},
{0xa9d0, 0xa9d9, 1},
{0xa9de, 0xa9df, 1},
},
}
......@@ -4407,7 +4414,7 @@ var (
)
// Generated by running
// maketables --props=all --url=http://www.unicode.org/Public/6.2.0/ucd/
// maketables --props=all --url=http://www.unicode.org/Public/6.3.0/ucd/
// DO NOT EDIT
// Properties is the set of Unicode property tables.
......@@ -4457,8 +4464,10 @@ var _ASCII_Hex_Digit = &RangeTable{
var _Bidi_Control = &RangeTable{
R16: []Range16{
{0x061c, 0x061c, 1},
{0x200e, 0x200f, 1},
{0x202a, 0x202e, 1},
{0x2066, 0x2069, 1},
},
}
......@@ -4935,7 +4944,7 @@ var _Other_Default_Ignorable_Code_Point = &RangeTable{
{0x034f, 0x034f, 1},
{0x115f, 0x1160, 1},
{0x17b4, 0x17b5, 1},
{0x2065, 0x2069, 1},
{0x2065, 0x2065, 1},
{0x3164, 0x3164, 1},
{0xffa0, 0xffa0, 1},
{0xfff0, 0xfff8, 1},
......@@ -5057,6 +5066,7 @@ var _Other_Math = &RangeTable{
{0x21d5, 0x21db, 1},
{0x21dd, 0x21dd, 1},
{0x21e4, 0x21e5, 1},
{0x2308, 0x230b, 1},
{0x23b4, 0x23b5, 1},
{0x23b7, 0x23b7, 1},
{0x23d0, 0x23d0, 1},
......@@ -5444,7 +5454,6 @@ var _White_Space = &RangeTable{
{0x0085, 0x0085, 1},
{0x00a0, 0x00a0, 1},
{0x1680, 0x1680, 1},
{0x180e, 0x180e, 1},
{0x2000, 0x200a, 1},
{0x2028, 0x2029, 1},
{0x202f, 0x202f, 1},
......@@ -5491,7 +5500,7 @@ var (
)
// Generated by running
// maketables --data=http://www.unicode.org/Public/6.2.0/ucd/UnicodeData.txt --casefolding=http://www.unicode.org/Public/6.2.0/ucd/CaseFolding.txt
// maketables --data=http://www.unicode.org/Public/6.3.0/ucd/UnicodeData.txt --casefolding=http://www.unicode.org/Public/6.3.0/ucd/CaseFolding.txt
// DO NOT EDIT
// CaseRanges is the table describing case mappings for all letters with
......@@ -6376,7 +6385,7 @@ var foldMn = &RangeTable{
// If there is no entry for a script name, there are no such points.
var FoldScript = map[string]*RangeTable{}
// Range entries: 3462 16-bit, 832 32-bit, 4294 total.
// Range bytes: 20772 16-bit, 9984 32-bit, 30756 total.
// Range entries: 3471 16-bit, 832 32-bit, 4303 total.
// Range bytes: 20826 16-bit, 9984 32-bit, 30810 total.
// Fold orbit bytes: 63 pairs, 252 bytes
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7