unicode: add the first few property tests for printing.

The long-term goal is that %q will use IsPrint to decide
what to show natively vs. as hexadecimal.

R=rsc, r
CC=golang-dev
https://golang.org/cl/4526095

src/pkg/unicode/Makefile

@@ -8,6 +8,7 @@ TARG=unicode
 GOFILES=\
 	casetables.go\
 	digit.go\
+	graphic.go\
 	letter.go\
 	tables.go\
 

src/pkg/unicode/digit.go

@@ -6,7 +6,7 @@ package unicode
 
 // IsDigit reports whether the rune is a decimal digit.
 func IsDigit(rune int) bool {
-	if rune < 0x100 { // quick ASCII (Latin-1, really) check
+	if rune < Latin1Max {
 		return '0' <= rune && rune <= '9'
 	}
 	return Is(Digit, rune)

src/pkg/unicode/digit_test.go

@@ -118,7 +118,7 @@ func TestDigit(t *testing.T) {
 
 // Test that the special case in IsDigit agrees with the table
 func TestDigitOptimization(t *testing.T) {
-	for i := 0; i < 0x100; i++ {
+	for i := 0; i < Latin1Max; i++ {
 		if Is(Digit, i) != IsDigit(i) {
 			t.Errorf("IsDigit(U+%04X) disagrees with Is(Digit)", i)
 		}

src/pkg/unicode/graphic.go

+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package unicode
+
+// Bit masks for each code point under U+0100, for fast lookup.
+const (
+	pC  = 1 << iota // a control character.
+	pP              // a punctuation character.
+	pN              // a numeral.
+	pS              // a symbolic character.
+	pZ              // a spacing character.
+	pLu             // an upper-case letter.
+	pLl             // a lower-case letter.
+	pp              // a printable character according to Go's definition.
+	pg  = pp | pZ   // a graphical character according to the Unicode definition.
+)
+
+// GraphicRanges defines the set of graphic characters according to Unicode.
+var GraphicRanges = []*RangeTable{
+	L, M, N, P, S, Zs,
+}
+
+// PrintRanges defines the set of printable characters according to Go.
+// ASCII space, U+0020, is handled separately.
+var PrintRanges = []*RangeTable{
+	L, M, N, P, S,
+}
+
+// IsGraphic reports whether the rune is defined as a Graphic by Unicode.
+// Such characters include letters, marks, numbers, punctuation, symbols, and
+// spaces, from categories L, M, N, P, S, Zs.
+func IsGraphic(rune int) bool {
+	// We cast to uint32 to avoid the extra test for negative,
+	// and in the index we cast to uint8 to avoid the range check.
+	if uint32(rune) < Latin1Max {
+		return properties[uint8(rune)]&pg != 0
+	}
+	return IsOneOf(GraphicRanges, rune)
+}
+
+// IsPrint reports whether the rune is defined as printable by Go. Such
+// characters include letters, marks, numbers, punctuation, symbols, and the
+// ASCII space character, from categories L, M, N, P, S and the ASCII space
+// character.  This categorization is the same as IsGraphic except that the
+// only spacing character is ASCII space, U+0020.
+func IsPrint(rune int) bool {
+	if uint32(rune) < Latin1Max {
+		return properties[uint8(rune)]&pp != 0
+	}
+	return IsOneOf(PrintRanges, rune)
+}
+
+// IsOneOf reports whether the rune is a member of one of the ranges.
+// The rune is known to be above Latin-1.
+func IsOneOf(set []*RangeTable, rune int) bool {
+	for _, inside := range set {
+		if Is(inside, rune) {
+			return true
+		}
+	}
+	return false
+}
+
+// IsControl reports whether the rune is a control character.
+// The C (Other) Unicode category includes more code points
+// such as surrogates; use Is(C, rune) to test for them.
+func IsControl(rune int) bool {
+	if uint32(rune) < Latin1Max {
+		return properties[uint8(rune)]&pC != 0
+	}
+	// All control characters are < Latin1Max.
+	return false
+}
+
+// IsLetter reports whether the rune is a letter (category L).
+func IsLetter(rune int) bool {
+	if uint32(rune) < Latin1Max {
+		return properties[uint8(rune)]&(pLu|pLl) != 0
+	}
+	return Is(Letter, rune)
+}
+
+// IsMark reports whether the rune is a mark character (category M).
+func IsMark(rune int) bool {
+	// There are no mark characters in Latin-1.
+	return Is(Mark, rune)
+}
+
+// IsNumber reports whether the rune is a number (category N).
+func IsNumber(rune int) bool {
+	if uint32(rune) < Latin1Max {
+		return properties[uint8(rune)]&pN != 0
+	}
+	return Is(Number, rune)
+}
+
+// IsPunct reports whether the rune is a Unicode punctuation character
+// (category P).
+func IsPunct(rune int) bool {
+	if uint32(rune) < Latin1Max {
+		return properties[uint8(rune)]&pP != 0
+	}
+	return Is(Punct, rune)
+}
+
+// IsSpace reports whether the rune is a space character as defined
+// by Unicode's White Space property; in the Latin-1 space
+// this is
+//	'\t', '\n', '\v', '\f', '\r', ' ', U+0085 (NEL), U+00A0 (NBSP).
+// Other definitions of spacing characters are set by category
+// Z and property Pattern_White_Space.
+func IsSpace(rune int) bool {
+	// This property isn't the same as Z; special-case it.
+	if uint32(rune) < Latin1Max {
+		switch rune {
+		case '\t', '\n', '\v', '\f', '\r', ' ', 0x85, 0xA0:
+			return true
+		}
+		return false
+	}
+	return Is(White_Space, rune)
+}
+
+// IsSymbol reports whether the rune is a symbolic character.
+func IsSymbol(rune int) bool {
+	if uint32(rune) < Latin1Max {
+		return properties[uint8(rune)]&pS != 0
+	}
+	return Is(Symbol, rune)
+}

src/pkg/unicode/graphic_test.go

+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package unicode_test
+
+import (
+	"testing"
+	. "unicode"
+)
+
+// Independently check that the special "Is" functions work
+// in the Latin-1 range through the property table.
+
+func TestIsControlLatin1(t *testing.T) {
+	for i := 0; i < Latin1Max; i++ {
+		got := IsControl(i)
+		want := false
+		switch {
+		case 0x00 <= i && i <= 0x1F:
+			want = true
+		case 0x7F <= i && i <= 0x9F:
+			want = true
+		}
+		if got != want {
+			t.Errorf("%U incorrect: got %t; want %t", i, got, want)
+		}
+	}
+}
+
+func TestIsLetterLatin1(t *testing.T) {
+	for i := 0; i < Latin1Max; i++ {
+		got := IsLetter(i)
+		want := Is(Letter, i)
+		if got != want {
+			t.Errorf("%U incorrect: got %t; want %t", i, got, want)
+		}
+	}
+}
+
+func TestIsUpperLatin1(t *testing.T) {
+	for i := 0; i < Latin1Max; i++ {
+		got := IsUpper(i)
+		want := Is(Upper, i)
+		if got != want {
+			t.Errorf("%U incorrect: got %t; want %t", i, got, want)
+		}
+	}
+}
+
+func TestIsLowerLatin1(t *testing.T) {
+	for i := 0; i < Latin1Max; i++ {
+		got := IsLower(i)
+		want := Is(Lower, i)
+		if got != want {
+			t.Errorf("%U incorrect: got %t; want %t", i, got, want)
+		}
+	}
+}
+
+func TestNumberLatin1(t *testing.T) {
+	for i := 0; i < Latin1Max; i++ {
+		got := IsNumber(i)
+		want := Is(Number, i)
+		if got != want {
+			t.Errorf("%U incorrect: got %t; want %t", i, got, want)
+		}
+	}
+}
+
+func TestIsPrintLatin1(t *testing.T) {
+	for i := 0; i < Latin1Max; i++ {
+		got := IsPrint(i)
+		want := IsOneOf(PrintRanges, i)
+		if i == ' ' {
+			want = true
+		}
+		if got != want {
+			t.Errorf("%U incorrect: got %t; want %t", i, got, want)
+		}
+	}
+}
+
+func TestIsGraphicLatin1(t *testing.T) {
+	for i := 0; i < Latin1Max; i++ {
+		got := IsGraphic(i)
+		want := IsOneOf(GraphicRanges, i)
+		if got != want {
+			t.Errorf("%U incorrect: got %t; want %t", i, got, want)
+		}
+	}
+}
+
+func TestIsPunctLatin1(t *testing.T) {
+	for i := 0; i < Latin1Max; i++ {
+		got := IsPunct(i)
+		want := Is(Punct, i)
+		if got != want {
+			t.Errorf("%U incorrect: got %t; want %t", i, got, want)
+		}
+	}
+}
+
+func TestIsSpaceLatin1(t *testing.T) {
+	for i := 0; i < Latin1Max; i++ {
+		got := IsSpace(i)
+		want := Is(White_Space, i)
+		if got != want {
+			t.Errorf("%U incorrect: got %t; want %t", i, got, want)
+		}
+	}
+}
+
+func TestIsSymbolLatin1(t *testing.T) {
+	for i := 0; i < Latin1Max; i++ {
+		got := IsSymbol(i)
+		want := Is(Symbol, i)
+		if got != want {
+			t.Errorf("%U incorrect: got %t; want %t", i, got, want)
+		}
+	}
+}

src/pkg/unicode/letter.go

@@ -9,6 +9,8 @@ package unicode
 const (
 	MaxRune         = 0x10FFFF // Maximum valid Unicode code point.
 	ReplacementChar = 0xFFFD   // Represents invalid code points.
+	ASCIIMax        = 0x80     // (1 beyond) maximum ASCII value.
+	Latin1Max       = 0x100    // (1 beyond) maximum Latin-1 value.
 )
 
 // RangeTable defines a set of Unicode code points by listing the ranges of
@@ -121,7 +123,7 @@ func is32(ranges []Range32, rune uint32) bool {
 // Is tests whether rune is in the specified table of ranges.
 func Is(rangeTab *RangeTable, rune int) bool {
 	// common case: rune is ASCII or Latin-1.
-	if rune < 0x100 {
+	if uint32(rune) < Latin1Max {
 		// Only need to check R16, since R32 is always >= 1<<16.
 		r16 := uint16(rune)
 		for _, r := range rangeTab.R16 {
@@ -148,49 +150,30 @@ func Is(rangeTab *RangeTable, rune int) bool {
 
 // IsUpper reports whether the rune is an upper case letter.
 func IsUpper(rune int) bool {
-	if rune < 0x80 { // quick ASCII check
-		return 'A' <= rune && rune <= 'Z'
+	// See comment in IsGraphic.
+	if uint32(rune) < Latin1Max {
+		return properties[uint8(rune)]&pLu != 0
 	}
 	return Is(Upper, rune)
 }
 
 // IsLower reports whether the rune is a lower case letter.
 func IsLower(rune int) bool {
-	if rune < 0x80 { // quick ASCII check
-		return 'a' <= rune && rune <= 'z'
+	// See comment in IsGraphic.
+	if uint32(rune) < Latin1Max {
+		return properties[uint8(rune)]&pLl != 0
 	}
 	return Is(Lower, rune)
 }
 
 // IsTitle reports whether the rune is a title case letter.
 func IsTitle(rune int) bool {
-	if rune < 0x80 { // quick ASCII check
+	if rune < Latin1Max {
 		return false
 	}
 	return Is(Title, rune)
 }
 
-// IsLetter reports whether the rune is a letter.
-func IsLetter(rune int) bool {
-	if rune < 0x80 { // quick ASCII check
-		rune &^= 'a' - 'A'
-		return 'A' <= rune && rune <= 'Z'
-	}
-	return Is(Letter, rune)
-}
-
-// IsSpace reports whether the rune is a white space character.
-func IsSpace(rune int) bool {
-	if rune <= 0xFF { // quick Latin-1 check
-		switch rune {
-		case '\t', '\n', '\v', '\f', '\r', ' ', 0x85, 0xA0:
-			return true
-		}
-		return false
-	}
-	return Is(White_Space, rune)
-}
-
 // to maps the rune using the specified case mapping.
 func to(_case int, rune int, caseRange []CaseRange) int {
 	if _case < 0 || MaxCase <= _case {
@@ -235,7 +218,7 @@ func To(_case int, rune int) int {
 
 // ToUpper maps the rune to upper case.
 func ToUpper(rune int) int {
-	if rune < 0x80 { // quick ASCII check
+	if rune < ASCIIMax {
 		if 'a' <= rune && rune <= 'z' {
 			rune -= 'a' - 'A'
 		}
@@ -246,7 +229,7 @@ func ToUpper(rune int) int {
 
 // ToLower maps the rune to lower case.
 func ToLower(rune int) int {
-	if rune < 0x80 { // quick ASCII check
+	if rune < ASCIIMax {
 		if 'A' <= rune && rune <= 'Z' {
 			rune += 'a' - 'A'
 		}
@@ -257,7 +240,7 @@ func ToLower(rune int) int {
 
 // ToTitle maps the rune to title case.
 func ToTitle(rune int) int {
-	if rune < 0x80 { // quick ASCII check
+	if rune < ASCIIMax {
 		if 'a' <= rune && rune <= 'z' { // title case is upper case for ASCII
 			rune -= 'a' - 'A'
 		}

src/pkg/unicode/letter_test.go

@@ -323,7 +323,7 @@ func TestIsSpace(t *testing.T) {
 // Check that the optimizations for IsLetter etc. agree with the tables.
 // We only need to check the Latin-1 range.
 func TestLetterOptimizations(t *testing.T) {
-	for i := 0; i < 0x100; i++ {
+	for i := 0; i < Latin1Max; i++ {
 		if Is(Letter, i) != IsLetter(i) {
 			t.Errorf("IsLetter(U+%04X) disagrees with Is(Letter)", i)
 		}

src/pkg/unicode/maketables.go

@@ -28,6 +28,7 @@ func main() {
 	printScriptOrProperty(false)
 	printScriptOrProperty(true)
 	printCases()
+	printLatinProperties()
 	printSizes()
 }
 
@@ -54,7 +55,17 @@ var test = flag.Bool("test",
 var scriptRe = regexp.MustCompile(`^([0-9A-F]+)(\.\.[0-9A-F]+)? *; ([A-Za-z_]+)$`)
 var logger = log.New(os.Stderr, "", log.Lshortfile)
 
-var category = map[string]bool{"letter": true} // Nd Lu etc. letter is a special case
+var category = map[string]bool{
+	// Nd Lu etc.
+	// We use one-character names to identify merged categories
+	"L": true, // Lu Ll Lt Lm Lo
+	"P": true, // Pc Pd Ps Pe Pu Pf Po
+	"M": true, // Mn Mc Me
+	"N": true, // Nd Nl No
+	"S": true, // Sm Sc Sk So
+	"Z": true, // Zs Zl Zp
+	"C": true, // Cc Cf Cs Co Cn
+}
 
 // UnicodeData.txt has form:
 //	0037;DIGIT SEVEN;Nd;0;EN;;7;7;7;N;;;;;
@@ -247,12 +258,9 @@ func version() string {
 	return "Unknown"
 }
 
-func letterOp(code int) bool {
-	switch chars[code].category {
-	case "Lu", "Ll", "Lt", "Lm", "Lo":
-		return true
-	}
-	return false
+func categoryOp(code int, class uint8) bool {
+	category := chars[code].category
+	return len(category) > 0 && category[0] == class
 }
 
 func loadChars() {
@@ -348,8 +356,27 @@ func printCategories() {
 		// Cases deserving special comments
 		varDecl := ""
 		switch name {
-		case "letter":
-			varDecl = "\tLetter = letter;	// Letter is the set of Unicode letters.\n"
+		case "C":
+			varDecl = "\tOther = _C;	// Other/C is the set of Unicode control and special characters, category C.\n"
+			varDecl += "\tC = _C\n"
+		case "L":
+			varDecl = "\tLetter = _L;	// Letter/L is the set of Unicode letters, category L.\n"
+			varDecl += "\tL = _L\n"
+		case "M":
+			varDecl = "\tMark = _M;	// Mark/M is the set of Unicode mark characters, category  M.\n"
+			varDecl += "\tM = _M\n"
+		case "N":
+			varDecl = "\tNumber = _N;	// Number/N is the set of Unicode number characters, category N.\n"
+			varDecl += "\tN = _N\n"
+		case "P":
+			varDecl = "\tPunct = _P;	// Punct/P is the set of Unicode punctuation characters, category P.\n"
+			varDecl += "\tP = _P\n"
+		case "S":
+			varDecl = "\tSymbol = _S;	// Symbol/S is the set of Unicode symbol characters, category S.\n"
+			varDecl += "\tS = _S\n"
+		case "Z":
+			varDecl = "\tSpace = _Z;	// Space/Z is the set of Unicode space characters, category Z.\n"
+			varDecl += "\tZ = _Z\n"
 		case "Nd":
 			varDecl = "\tDigit = _Nd;	// Digit is the set of Unicode characters with the \"decimal digit\" property.\n"
 		case "Lu":
@@ -359,17 +386,18 @@ func printCategories() {
 		case "Lt":
 			varDecl = "\tTitle = _Lt;	// Title is the set of Unicode title case letters.\n"
 		}
-		if name != "letter" {
+		if len(name) > 1 {
 			varDecl += fmt.Sprintf(
 				"\t%s = _%s;	// %s is the set of Unicode characters in category %s.\n",
 				name, name, name, name)
 		}
 		decl[ndecl] = varDecl
 		ndecl++
-		if name == "letter" { // special case
+		if len(name) == 1 { // unified categories
+			decl := fmt.Sprintf("var _%s = &RangeTable{\n", name)
 			dumpRange(
-				"var letter = &RangeTable{\n",
-				letterOp)
+				decl,
+				func(code int) bool { return categoryOp(code, name[0]) })
 			continue
 		}
 		dumpRange(
@@ -446,7 +474,7 @@ func printRange(lo, hi, stride uint32, size int, count *int) (int, *int) {
 	if size == 16 && hi >= 1<<16 {
 		if lo < 1<<16 {
 			if lo+stride != hi {
-				log.Fatalf("unexpected straddle: %U %U %d", lo, hi, stride)
+				logger.Fatalf("unexpected straddle: %U %U %d", lo, hi, stride)
 			}
 			// No range contains U+FFFF as an instance, so split
 			// the range into two entries. That way we can maintain
@@ -472,11 +500,11 @@ func fullCategoryTest(list []string) {
 			logger.Fatal("unknown category", name)
 		}
 		r, ok := unicode.Categories[name]
-		if !ok {
-			logger.Fatal("unknown table", name)
+		if !ok && len(name) > 1 {
+			logger.Fatalf("unknown table %q", name)
 		}
-		if name == "letter" {
-			verifyRange(name, letterOp, r)
+		if len(name) == 1 {
+			verifyRange(name, func(code int) bool { return categoryOp(code, name[0]) }, r)
 		} else {
 			verifyRange(
 				name,
@@ -487,11 +515,16 @@ func fullCategoryTest(list []string) {
 }
 
 func verifyRange(name string, inCategory Op, table *unicode.RangeTable) {
+	count := 0
 	for i := range chars {
 		web := inCategory(i)
 		pkg := unicode.Is(table, i)
 		if web != pkg {
 			fmt.Fprintf(os.Stderr, "%s: %U: web=%t pkg=%t\n", name, i, web, pkg)
+			count++
+			if count > 10 {
+				break
+			}
 		}
 	}
 }
@@ -882,6 +915,42 @@ func fullCaseTest() {
 	}
 }
 
+func printLatinProperties() {
+	if *test {
+		return
+	}
+	fmt.Println("var properties = [Latin1Max]uint8{")
+	for code := 0; code < unicode.Latin1Max; code++ {
+		var property string
+		switch chars[code].category {
+		case "Cc", "": // NUL has no category.
+			property = "pC"
+		case "Cf": // soft hyphen, unique category, not printable.
+			property = "0"
+		case "Ll":
+			property = "pLl | pp"
+		case "Lu":
+			property = "pLu | pp"
+		case "Nd", "No":
+			property = "pN | pp"
+		case "Pc", "Pd", "Pe", "Pf", "Pi", "Po", "Ps":
+			property = "pP | pp"
+		case "Sc", "Sk", "Sm", "So":
+			property = "pS | pp"
+		case "Zs":
+			property = "pZ"
+		default:
+			logger.Fatalf("%U has unknown category %q", code, chars[code].category)
+		}
+		// Special case
+		if code == ' ' {
+			property = "pZ | pp"
+		}
+		fmt.Printf("\t0x%.2X: %s, // %q\n", code, property, code)
+	}
+	fmt.Println("}")
+}
+
 var range16Count = 0 // Number of entries in the 16-bit range tables.
 var range32Count = 0 // Number of entries in the 32-bit range tables.
 

src/pkg/unicode/script_test.go

@@ -149,7 +149,14 @@ var inCategoryTest = []T{
 	{0x2028, "Zl"},
 	{0x2029, "Zp"},
 	{0x202f, "Zs"},
-	{0x04aa, "letter"},
+	// Unifieds.
+	{0x04aa, "L"},
+	{0x0009, "C"},
+	{0x1712, "M"},
+	{0x0031, "N"},
+	{0x00bb, "P"},
+	{0x00a2, "S"},
+	{0x00a0, "Z"},
 }
 
 var inPropTest = []T{
@@ -197,13 +204,13 @@ func TestScripts(t *testing.T) {
 			t.Fatal(test.script, "not a known script")
 		}
 		if !Is(Scripts[test.script], test.rune) {
-			t.Errorf("IsScript(%#x, %s) = false, want true", test.rune, test.script)
+			t.Errorf("IsScript(%U, %s) = false, want true", test.rune, test.script)
 		}
 		notTested[test.script] = false, false
 	}
 	for _, test := range outTest {
 		if Is(Scripts[test.script], test.rune) {
-			t.Errorf("IsScript(%#x, %s) = true, want false", test.rune, test.script)
+			t.Errorf("IsScript(%U, %s) = true, want false", test.rune, test.script)
 		}
 	}
 	for k := range notTested {
@@ -221,7 +228,7 @@ func TestCategories(t *testing.T) {
 			t.Fatal(test.script, "not a known category")
 		}
 		if !Is(Categories[test.script], test.rune) {
-			t.Errorf("IsCategory(%#x, %s) = false, want true", test.rune, test.script)
+			t.Errorf("IsCategory(%U, %s) = false, want true", test.rune, test.script)
 		}
 		notTested[test.script] = false, false
 	}
@@ -240,7 +247,7 @@ func TestProperties(t *testing.T) {
 			t.Fatal(test.script, "not a known prop")
 		}
 		if !Is(Properties[test.script], test.rune) {
-			t.Errorf("IsCategory(%#x, %s) = false, want true", test.rune, test.script)
+			t.Errorf("IsCategory(%U, %s) = false, want true", test.rune, test.script)
 		}
 		notTested[test.script] = false, false
 	}