Skip to content

G
go
Commit 831db8d9 authored by Rob Pike's avatar Rob Pike
Browse files
Options

exp: delete all packages except norm 

They are moving to code.google.com/p/go.exp.
See also https://golang.org/cl/7463043

R=golang-dev, minux.ma
CC=golang-dev
https://golang.org/cl/7456047
parent 01a5b66d
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 0 additions and 76060 deletions
src/pkg/exp/README deleted 100644 → 0
View file @ 01a5b66d
This directory tree contains experimental packages and
unfinished code that is subject to change. It does not
have stable APIs, and is not present in stable releases.
src/pkg/exp/ebnf/ebnf.go deleted 100644 → 0
View file @ 01a5b66d
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package ebnf is a library for EBNF grammars. The input is text ([]byte)
// satisfying the following grammar (represented itself in EBNF):
//
// Production = name "=" [ Expression ] "." .
// Expression = Alternative { "|" Alternative } .
// Alternative = Term { Term } .
// Term = name | token [ "…" token ] | Group | Option | Repetition .
// Group = "(" Expression ")" .
// Option = "[" Expression "]" .
// Repetition = "{" Expression "}" .
//
// A name is a Go identifier, a token is a Go string, and comments
// and white space follow the same rules as for the Go language.
// Production names starting with an uppercase Unicode letter denote
// non-terminal productions (i.e., productions which allow white-space
// and comments between tokens); all other production names denote
// lexical productions.
//
package ebnf
import (
"errors"
"fmt"
"text/scanner"
"unicode"
"unicode/utf8"
)
// ----------------------------------------------------------------------------
// Error handling
type errorList []error
func (list errorList) Err() error {
if len(list) == 0 {
return nil
}
return list
}
func (list errorList) Error() string {
switch len(list) {
case 0:
return "no errors"
case 1:
return list[0].Error()
}
return fmt.Sprintf("%s (and %d more errors)", list[0], len(list)-1)
}
func newError(pos scanner.Position, msg string) error {
return errors.New(fmt.Sprintf("%s: %s", pos, msg))
}
// ----------------------------------------------------------------------------
// Internal representation
type (
// An Expression node represents a production expression.
Expression interface {
// Pos is the position of the first character of the syntactic construct
Pos() scanner.Position
}
// An Alternative node represents a non-empty list of alternative expressions.
Alternative []Expression // x | y | z
// A Sequence node represents a non-empty list of sequential expressions.
Sequence []Expression // x y z
// A Name node represents a production name.
Name struct {
StringPos scanner.Position
String string
}
// A Token node represents a literal.
Token struct {
StringPos scanner.Position
String string
}
// A List node represents a range of characters.
Range struct {
Begin, End *Token // begin ... end
}
// A Group node represents a grouped expression.
Group struct {
Lparen scanner.Position
Body Expression // (body)
}
// An Option node represents an optional expression.
Option struct {
Lbrack scanner.Position
Body Expression // [body]
}
// A Repetition node represents a repeated expression.
Repetition struct {
Lbrace scanner.Position
Body Expression // {body}
}
// A Production node represents an EBNF production.
Production struct {
Name *Name
Expr Expression
}
// A Bad node stands for pieces of source code that lead to a parse error.
Bad struct {
TokPos scanner.Position
Error string // parser error message
}
// A Grammar is a set of EBNF productions. The map
// is indexed by production name.
//
Grammar map[string]*Production
)
func (x Alternative) Pos() scanner.Position { return x[0].Pos() } // the parser always generates non-empty Alternative
func (x Sequence) Pos() scanner.Position { return x[0].Pos() } // the parser always generates non-empty Sequences
func (x *Name) Pos() scanner.Position { return x.StringPos }
func (x *Token) Pos() scanner.Position { return x.StringPos }
func (x *Range) Pos() scanner.Position { return x.Begin.Pos() }
func (x *Group) Pos() scanner.Position { return x.Lparen }
func (x *Option) Pos() scanner.Position { return x.Lbrack }
func (x *Repetition) Pos() scanner.Position { return x.Lbrace }
func (x *Production) Pos() scanner.Position { return x.Name.Pos() }
func (x *Bad) Pos() scanner.Position { return x.TokPos }
// ----------------------------------------------------------------------------
// Grammar verification
func isLexical(name string) bool {
ch, _ := utf8.DecodeRuneInString(name)
return !unicode.IsUpper(ch)
}
type verifier struct {
errors errorList
worklist []*Production
reached Grammar // set of productions reached from (and including) the root production
grammar Grammar
}
func (v *verifier) error(pos scanner.Position, msg string) {
v.errors = append(v.errors, newError(pos, msg))
}
func (v *verifier) push(prod *Production) {
name := prod.Name.String
if _, found := v.reached[name]; !found {
v.worklist = append(v.worklist, prod)
v.reached[name] = prod
}
}
func (v *verifier) verifyChar(x *Token) rune {
s := x.String
if utf8.RuneCountInString(s) != 1 {
v.error(x.Pos(), "single char expected, found "+s)
return 0
}
ch, _ := utf8.DecodeRuneInString(s)
return ch
}
func (v *verifier) verifyExpr(expr Expression, lexical bool) {
switch x := expr.(type) {
case nil:
// empty expression
case Alternative:
for _, e := range x {
v.verifyExpr(e, lexical)
}
case Sequence:
for _, e := range x {
v.verifyExpr(e, lexical)
}
case *Name:
// a production with this name must exist;
// add it to the worklist if not yet processed
if prod, found := v.grammar[x.String]; found {
v.push(prod)
} else {
v.error(x.Pos(), "missing production "+x.String)
}
// within a lexical production references
// to non-lexical productions are invalid
if lexical && !isLexical(x.String) {
v.error(x.Pos(), "reference to non-lexical production "+x.String)
}
case *Token:
// nothing to do for now
case *Range:
i := v.verifyChar(x.Begin)
j := v.verifyChar(x.End)
if i >= j {
v.error(x.Pos(), "decreasing character range")
}
case *Group:
v.verifyExpr(x.Body, lexical)
case *Option:
v.verifyExpr(x.Body, lexical)
case *Repetition:
v.verifyExpr(x.Body, lexical)
case *Bad:
v.error(x.Pos(), x.Error)
default:
panic(fmt.Sprintf("internal error: unexpected type %T", expr))
}
}
func (v *verifier) verify(grammar Grammar, start string) {
// find root production
root, found := grammar[start]
if !found {
var noPos scanner.Position
v.error(noPos, "no start production "+start)
return
}
// initialize verifier
v.worklist = v.worklist[0:0]
v.reached = make(Grammar)
v.grammar = grammar
// work through the worklist
v.push(root)
for {
n := len(v.worklist) - 1
if n < 0 {
break
}
prod := v.worklist[n]
v.worklist = v.worklist[0:n]
v.verifyExpr(prod.Expr, isLexical(prod.Name.String))
}
// check if all productions were reached
if len(v.reached) < len(v.grammar) {
for name, prod := range v.grammar {
if _, found := v.reached[name]; !found {
v.error(prod.Pos(), name+" is unreachable")
}
}
}
}
// Verify checks that:
// - all productions used are defined
// - all productions defined are used when beginning at start
// - lexical productions refer only to other lexical productions
//
// Position information is interpreted relative to the file set fset.
//
func Verify(grammar Grammar, start string) error {
var v verifier
v.verify(grammar, start)
return v.errors.Err()
}
src/pkg/exp/ebnf/ebnf_test.go deleted 100644 → 0
View file @ 01a5b66d
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package ebnf
import (
"bytes"
"testing"
)
var goodGrammars = []string{
`Program = .`,
`Program = foo .
foo = "foo" .`,
`Program = "a" | "b" "c" .`,
`Program = "a" … "z" .`,
`Program = Song .
Song = { Note } .
Note = Do | (Re | Mi | Fa | So | La) | Ti .
Do = "c" .
Re = "d" .
Mi = "e" .
Fa = "f" .
So = "g" .
La = "a" .
Ti = ti .
ti = "b" .`,
}
var badGrammars = []string{
`Program = | .`,
`Program = | b .`,
`Program = a … b .`,
`Program = "a" … .`,
`Program = … "b" .`,
`Program = () .`,
`Program = [] .`,
`Program = {} .`,
}
func checkGood(t *testing.T, src string) {
grammar, err := Parse("", bytes.NewBuffer([]byte(src)))
if err != nil {
t.Errorf("Parse(%s) failed: %v", src, err)
return
}
if err = Verify(grammar, "Program"); err != nil {
t.Errorf("Verify(%s) failed: %v", src, err)
}
}
func checkBad(t *testing.T, src string) {
_, err := Parse("", bytes.NewBuffer([]byte(src)))
if err == nil {
t.Errorf("Parse(%s) should have failed", src)
}
}
func TestGrammars(t *testing.T) {
for _, src := range goodGrammars {
checkGood(t, src)
}
for _, src := range badGrammars {
checkBad(t, src)
}
}
src/pkg/exp/ebnf/parser.go deleted 100644 → 0
View file @ 01a5b66d
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package ebnf
import (
"io"
"strconv"
"text/scanner"
)
type parser struct {
errors errorList
scanner scanner.Scanner
pos scanner.Position // token position
tok rune // one token look-ahead
lit string // token literal
}
func (p *parser) next() {
p.tok = p.scanner.Scan()
p.pos = p.scanner.Position
p.lit = p.scanner.TokenText()
}
func (p *parser) error(pos scanner.Position, msg string) {
p.errors = append(p.errors, newError(pos, msg))
}
func (p *parser) errorExpected(pos scanner.Position, msg string) {
msg = `expected "` + msg + `"`
if pos.Offset == p.pos.Offset {
// the error happened at the current position;
// make the error message more specific
msg += ", found " + scanner.TokenString(p.tok)
if p.tok < 0 {
msg += " " + p.lit
}
}
p.error(pos, msg)
}
func (p *parser) expect(tok rune) scanner.Position {
pos := p.pos
if p.tok != tok {
p.errorExpected(pos, scanner.TokenString(tok))
}
p.next() // make progress in any case
return pos
}
func (p *parser) parseIdentifier() *Name {
pos := p.pos
name := p.lit
p.expect(scanner.Ident)
return &Name{pos, name}
}
func (p *parser) parseToken() *Token {
pos := p.pos
value := ""
if p.tok == scanner.String {
value, _ = strconv.Unquote(p.lit)
// Unquote may fail with an error, but only if the scanner found
// an illegal string in the first place. In this case the error
// has already been reported.
p.next()
} else {
p.expect(scanner.String)
}
return &Token{pos, value}
}
// ParseTerm returns nil if no term was found.
func (p *parser) parseTerm() (x Expression) {
pos := p.pos
switch p.tok {
case scanner.Ident:
x = p.parseIdentifier()
case scanner.String:
tok := p.parseToken()
x = tok
const ellipsis = '…' // U+2026, the horizontal ellipsis character
if p.tok == ellipsis {
p.next()
x = &Range{tok, p.parseToken()}
}
case '(':
p.next()
x = &Group{pos, p.parseExpression()}
p.expect(')')
case '[':
p.next()
x = &Option{pos, p.parseExpression()}
p.expect(']')
case '{':
p.next()
x = &Repetition{pos, p.parseExpression()}
p.expect('}')
}
return x
}
func (p *parser) parseSequence() Expression {
var list Sequence
for x := p.parseTerm(); x != nil; x = p.parseTerm() {
list = append(list, x)
}
// no need for a sequence if list.Len() < 2
switch len(list) {
case 0:
p.errorExpected(p.pos, "term")
return &Bad{p.pos, "term expected"}
case 1:
return list[0]
}
return list
}
func (p *parser) parseExpression() Expression {
var list Alternative
for {
list = append(list, p.parseSequence())
if p.tok != '|' {
break
}
p.next()
}
// len(list) > 0
// no need for an Alternative node if list.Len() < 2
if len(list) == 1 {
return list[0]
}
return list
}
func (p *parser) parseProduction() *Production {
name := p.parseIdentifier()
p.expect('=')
var expr Expression
if p.tok != '.' {
expr = p.parseExpression()
}
p.expect('.')
return &Production{name, expr}
}
func (p *parser) parse(filename string, src io.Reader) Grammar {
p.scanner.Init(src)
p.scanner.Filename = filename
p.next() // initializes pos, tok, lit
grammar := make(Grammar)
for p.tok != scanner.EOF {
prod := p.parseProduction()
name := prod.Name.String
if _, found := grammar[name]; !found {
grammar[name] = prod
} else {
p.error(prod.Pos(), name+" declared already")
}
}
return grammar
}
// Parse parses a set of EBNF productions from source src.
// It returns a set of productions. Errors are reported
// for incorrect syntax and if a production is declared
// more than once; the filename is used only for error
// positions.
//
func Parse(filename string, src io.Reader) (Grammar, error) {
var p parser
grammar := p.parse(filename, src)
return grammar, p.errors.Err()
}
src/pkg/exp/ebnflint/doc.go deleted 100644 → 0
View file @ 01a5b66d
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
/*
Ebnflint verifies that EBNF productions are consistent and grammatically correct.
It reads them from an HTML document such as the Go specification.
Grammar productions are grouped in boxes demarcated by the HTML elements
<pre class="ebnf">
</pre>
Usage:
go tool ebnflint [--start production] [file]
The --start flag specifies the name of the start production for
the grammar; it defaults to "Start".
*/
package main
src/pkg/exp/ebnflint/ebnflint.go deleted 100644 → 0
View file @ 01a5b66d
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package main
import (
"bytes"
"exp/ebnf"
"flag"
"fmt"
"go/scanner"
"go/token"
"io"
"io/ioutil"
"os"
"path/filepath"
)
var fset = token.NewFileSet()
var start = flag.String("start", "Start", "name of start production")
func usage() {
fmt.Fprintf(os.Stderr, "usage: go tool ebnflint [flags] [filename]\n")
flag.PrintDefaults()
os.Exit(1)
}
// Markers around EBNF sections in .html files
var (
open = []byte(`<pre class="ebnf">`)
close = []byte(`</pre>`)
)
func report(err error) {
scanner.PrintError(os.Stderr, err)
os.Exit(1)
}
func extractEBNF(src []byte) []byte {
var buf bytes.Buffer
for {
// i = beginning of EBNF text
i := bytes.Index(src, open)
if i < 0 {
break // no EBNF found - we are done
}
i += len(open)
// write as many newlines as found in the excluded text
// to maintain correct line numbers in error messages
for _, ch := range src[0:i] {
if ch == '\n' {
buf.WriteByte('\n')
}
}
// j = end of EBNF text (or end of source)
j := bytes.Index(src[i:], close) // close marker
if j < 0 {
j = len(src) - i
}
j += i
// copy EBNF text
buf.Write(src[i:j])
// advance
src = src[j:]
}
return buf.Bytes()
}
func main() {
flag.Parse()
var (
name string
r io.Reader
)
switch flag.NArg() {
case 0:
name, r = "<stdin>", os.Stdin
case 1:
name = flag.Arg(0)
default:
usage()
}
if err := verify(name, *start, r); err != nil {
report(err)
}
}
func verify(name, start string, r io.Reader) error {
if r == nil {
f, err := os.Open(name)
if err != nil {
return err
}
defer f.Close()
r = f
}
src, err := ioutil.ReadAll(r)
if err != nil {
return err
}
if filepath.Ext(name) == ".html" || bytes.Index(src, open) >= 0 {
src = extractEBNF(src)
}
grammar, err := ebnf.Parse(name, bytes.NewBuffer(src))
if err != nil {
return err
}
return ebnf.Verify(grammar, start)
}
src/pkg/exp/ebnflint/ebnflint_test.go deleted 100644 → 0
View file @ 01a5b66d
// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package main
import (
"runtime"
"testing"
)
func TestSpec(t *testing.T) {
if err := verify(runtime.GOROOT()+"/doc/go_spec.html", "SourceFile", nil); err != nil {
t.Fatal(err)
}
}
src/pkg/exp/gotype/doc.go deleted 100644 → 0
View file @ 01a5b66d
// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
/*
The gotype command does syntactic and semantic analysis of Go files
and packages similar to the analysis performed by the front-end of
a Go compiler. Errors are reported if the analysis fails; otherwise
gotype is quiet (unless -v is set).
Without a list of paths, gotype processes the standard input, which must
be the source of a single package file.
Given a list of file names, each file must be a source file belonging to
the same package unless the package name is explicitly specified with the
-p flag.
Given a directory name, gotype collects all .go files in the directory
and processes them as if they were provided as an explicit list of file
names. Each directory is processed independently. Files starting with .
or not ending in .go are ignored.
Usage:
gotype [flags] [path ...]
The flags are:
-e
Print all (including spurious) errors.
-p pkgName
Process only those files in package pkgName.
-r
Recursively process subdirectories.
-v
Verbose mode.
Debugging flags:
-comments
Parse comments (ignored if -ast not set).
-ast
Print AST (disables concurrent parsing).
-trace
Print parse trace (disables concurrent parsing).
Examples
To check the files file.go, old.saved, and .ignored:
gotype file.go old.saved .ignored
To check all .go files belonging to package main in the current directory
and recursively in all subdirectories:
gotype -p main -r .
To verify the output of a pipe:
echo "package foo" | gotype
*/
package main
// BUG(gri): At the moment, only single-file scope analysis is performed.
src/pkg/exp/gotype/gotype.go deleted 100644 → 0
View file @ 01a5b66d
// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package main
import (
"flag"
"fmt"
"go/ast"
"go/parser"
"go/scanner"
"go/token"
"go/types"
"io/ioutil"
"os"
"path/filepath"
"strings"
)
var (
// main operation modes
pkgName = flag.String("p", "", "process only those files in package pkgName")
recursive = flag.Bool("r", false, "recursively process subdirectories")
verbose = flag.Bool("v", false, "verbose mode")
allErrors = flag.Bool("e", false, "report all errors (not just the first 10 on different lines)")
// debugging support
parseComments = flag.Bool("comments", false, "parse comments (ignored if -ast not set)")
printTrace = flag.Bool("trace", false, "print parse trace")
printAST = flag.Bool("ast", false, "print AST")
)
var errorCount int
func usage() {
fmt.Fprintf(os.Stderr, "usage: gotype [flags] [path ...]\n")
flag.PrintDefaults()
os.Exit(2)
}
func report(err error) {
scanner.PrintError(os.Stderr, err)
if list, ok := err.(scanner.ErrorList); ok {
errorCount += len(list)
return
}
errorCount++
}
// parse returns the AST for the Go source src.
// The filename is for error reporting only.
// The result is nil if there were errors or if
// the file does not belong to the -p package.
func parse(fset *token.FileSet, filename string, src []byte) *ast.File {
if *verbose {
fmt.Println(filename)
}
// ignore files with different package name
if *pkgName != "" {
file, err := parser.ParseFile(fset, filename, src, parser.PackageClauseOnly)
if err != nil {
report(err)
return nil
}
if file.Name.Name != *pkgName {
if *verbose {
fmt.Printf("\tignored (package %s)\n", file.Name.Name)
}
return nil
}
}
// parse entire file
mode := parser.DeclarationErrors
if *allErrors {
mode |= parser.AllErrors
}
if *parseComments && *printAST {
mode |= parser.ParseComments
}
if *printTrace {
mode |= parser.Trace
}
file, err := parser.ParseFile(fset, filename, src, mode)
if err != nil {
report(err)
return nil
}
if *printAST {
ast.Print(fset, file)
}
return file
}
func parseStdin(fset *token.FileSet) (files []*ast.File) {
src, err := ioutil.ReadAll(os.Stdin)
if err != nil {
report(err)
return
}
const filename = "<standard input>"
if file := parse(fset, filename, src); file != nil {
files = []*ast.File{file}
}
return
}
func parseFiles(fset *token.FileSet, filenames []string) (files []*ast.File) {
for _, filename := range filenames {
src, err := ioutil.ReadFile(filename)
if err != nil {
report(err)
continue
}
if file := parse(fset, filename, src); file != nil {
files = append(files, file)
}
}
return
}
func isGoFilename(filename string) bool {
// ignore non-Go files
return !strings.HasPrefix(filename, ".") && strings.HasSuffix(filename, ".go")
}
func processDirectory(dirname string) {
f, err := os.Open(dirname)
if err != nil {
report(err)
return
}
filenames, err := f.Readdirnames(-1)
f.Close()
if err != nil {
report(err)
// continue since filenames may not be empty
}
for i, filename := range filenames {
filenames[i] = filepath.Join(dirname, filename)
}
processFiles(filenames, false)
}
func processFiles(filenames []string, allFiles bool) {
i := 0
for _, filename := range filenames {
switch info, err := os.Stat(filename); {
case err != nil:
report(err)
case info.IsDir():
if allFiles || *recursive {
processDirectory(filename)
}
default:
if allFiles || isGoFilename(info.Name()) {
filenames[i] = filename
i++
}
}
}
fset := token.NewFileSet()
processPackage(fset, parseFiles(fset, filenames[0:i]))
}
func processPackage(fset *token.FileSet, files []*ast.File) {
type bailout struct{}
ctxt := types.Context{
Error: func(err error) {
if !*allErrors && errorCount >= 10 {
panic(bailout{})
}
report(err)
},
}
defer func() {
switch err := recover().(type) {
case nil, bailout:
default:
panic(err)
}
}()
ctxt.Check(fset, files)
}
func main() {
flag.Usage = usage
flag.Parse()
if flag.NArg() == 0 {
fset := token.NewFileSet()
processPackage(fset, parseStdin(fset))
} else {
processFiles(flag.Args(), true)
}
if errorCount > 0 {
os.Exit(2)
}
}
src/pkg/exp/gotype/gotype_test.go deleted 100644 → 0
View file @ 01a5b66d
// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package main
import (
"go/build"
"path/filepath"
"runtime"
"strings"
"testing"
)
func runTest(t *testing.T, path string) {
errorCount = 0
*recursive = false
*allErrors = true
if suffix := ".go"; strings.HasSuffix(path, suffix) {
// single file
path = filepath.Join(runtime.GOROOT(), "src/pkg", path)
path, file := filepath.Split(path)
*pkgName = file[:len(file)-len(suffix)]
processFiles([]string{path}, true)
} else {
// package directory
// TODO(gri) gotype should use the build package instead
ctxt := build.Default
ctxt.CgoEnabled = false
pkg, err := ctxt.Import(path, "", 0)
if err != nil {
t.Errorf("build.Import error for path = %s: %s", path, err)
return
}
// TODO(gri) there ought to be a more direct way using the build package...
files := make([]string, len(pkg.GoFiles))
for i, file := range pkg.GoFiles {
files[i] = filepath.Join(pkg.Dir, file)
}
*pkgName = pkg.Name
processFiles(files, true)
}
if errorCount > 0 {
t.Errorf("processing %s failed: %d errors", path, errorCount)
}
}
var tests = []string{
// individual files
"exp/gotype/testdata/test1.go",
// directories
// Note: Packages that don't typecheck yet are commented out.
// Unless there is a comment next to the commented out packages,
// the package doesn't typecheck due to errors in the shift
// expression checker.
"archive/tar",
"archive/zip",
"bufio",
"bytes",
"compress/bzip2",
"compress/flate",
"compress/gzip",
"compress/lzw",
"compress/zlib",
"container/heap",
"container/list",
"container/ring",
"crypto",
"crypto/aes",
"crypto/cipher",
"crypto/des",
"crypto/dsa",
"crypto/ecdsa",
"crypto/elliptic",
"crypto/hmac",
"crypto/md5",
"crypto/rand",
"crypto/rc4",
"crypto/rsa",
"crypto/sha1",
"crypto/sha256",
"crypto/sha512",
"crypto/subtle",
"crypto/tls",
"crypto/x509",
"crypto/x509/pkix",
"database/sql",
"database/sql/driver",
"debug/dwarf",
"debug/elf",
"debug/gosym",
"debug/macho",
"debug/pe",
"encoding/ascii85",
"encoding/asn1",
"encoding/base32",
"encoding/base64",
"encoding/binary",
"encoding/csv",
"encoding/gob",
"encoding/hex",
"encoding/json",
"encoding/pem",
"encoding/xml",
"errors",
"expvar",
"flag",
"fmt",
"exp/gotype",
"go/ast",
"go/build",
"go/doc",
"go/format",
"go/parser",
"go/printer",
"go/scanner",
"go/token",
"go/types",
"hash/adler32",
"hash/crc32",
"hash/crc64",
"hash/fnv",
"image",
"image/color",
"image/draw",
"image/gif",
"image/jpeg",
"image/png",
"index/suffixarray",
"io",
"io/ioutil",
"log",
"log/syslog",
"math",
"math/big",
"math/cmplx",
"math/rand",
"mime",
"mime/multipart",
"net",
"net/http",
"net/http/cgi",
"net/http/fcgi",
"net/http/httptest",
"net/http/httputil",
"net/http/pprof",
"net/mail",
"net/rpc",
"net/rpc/jsonrpc",
"net/smtp",
"net/textproto",
"net/url",
"path",
"path/filepath",
"reflect",
"regexp",
"regexp/syntax",
"runtime",
"runtime/cgo",
"runtime/debug",
"runtime/pprof",
"sort",
"strconv",
"strings",
"sync",
"sync/atomic",
"syscall",
"testing",
"testing/iotest",
"testing/quick",
"text/scanner",
"text/tabwriter",
"text/template",
"text/template/parse",
"time",
"unicode",
"unicode/utf16",
"unicode/utf8",
}
func Test(t *testing.T) {
for _, test := range tests {
runTest(t, test)
}
}
src/pkg/exp/gotype/testdata/test1.go deleted 100644 → 0
View file @ 01a5b66d
// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package test1
func _() {
// the scope of a local type declaration starts immediately after the type name
type T struct{ _ *T }
}
func _(x interface{}) {
// the variable defined by a TypeSwitchGuard is declared in each TypeCaseClause
switch t := x.(type) {
case int:
_ = t
case float32:
_ = t
default:
_ = t
}
// the variable defined by a TypeSwitchGuard must not conflict with other
// variables declared in the initial simple statement
switch t := 0; t := x.(type) {
}
}
src/pkg/exp/inotify/inotify_linux.go deleted 100644 → 0
View file @ 01a5b66d
// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
/*
Package inotify implements a wrapper for the Linux inotify system.
Example:
watcher, err := inotify.NewWatcher()
if err != nil {
log.Fatal(err)
}
err = watcher.Watch("/tmp")
if err != nil {
log.Fatal(err)
}
for {
select {
case ev := <-watcher.Event:
log.Println("event:", ev)
case err := <-watcher.Error:
log.Println("error:", err)
}
}
*/
package inotify
import (
"errors"
"fmt"
"os"
"strings"
"sync"
"syscall"
"unsafe"
)
type Event struct {
Mask uint32 // Mask of events
Cookie uint32 // Unique cookie associating related events (for rename(2))
Name string // File name (optional)
}
type watch struct {
wd uint32 // Watch descriptor (as returned by the inotify_add_watch() syscall)
flags uint32 // inotify flags of this watch (see inotify(7) for the list of valid flags)
}
type Watcher struct {
mu sync.Mutex
fd int // File descriptor (as returned by the inotify_init() syscall)
watches map[string]*watch // Map of inotify watches (key: path)
paths map[int]string // Map of watched paths (key: watch descriptor)
Error chan error // Errors are sent on this channel
Event chan *Event // Events are returned on this channel
done chan bool // Channel for sending a "quit message" to the reader goroutine
isClosed bool // Set to true when Close() is first called
}
// NewWatcher creates and returns a new inotify instance using inotify_init(2)
func NewWatcher() (*Watcher, error) {
fd, errno := syscall.InotifyInit()
if fd == -1 {
return nil, os.NewSyscallError("inotify_init", errno)
}
w := &Watcher{
fd: fd,
watches: make(map[string]*watch),
paths: make(map[int]string),
Event: make(chan *Event),
Error: make(chan error),
done: make(chan bool, 1),
}
go w.readEvents()
return w, nil
}
// Close closes an inotify watcher instance
// It sends a message to the reader goroutine to quit and removes all watches
// associated with the inotify instance
func (w *Watcher) Close() error {
if w.isClosed {
return nil
}
w.isClosed = true
// Send "quit" message to the reader goroutine
w.done <- true
for path := range w.watches {
w.RemoveWatch(path)
}
return nil
}
// AddWatch adds path to the watched file set.
// The flags are interpreted as described in inotify_add_watch(2).
func (w *Watcher) AddWatch(path string, flags uint32) error {
if w.isClosed {
return errors.New("inotify instance already closed")
}
watchEntry, found := w.watches[path]
if found {
watchEntry.flags |= flags
flags |= syscall.IN_MASK_ADD
}
w.mu.Lock() // synchronize with readEvents goroutine
wd, err := syscall.InotifyAddWatch(w.fd, path, flags)
if err != nil {
w.mu.Unlock()
return &os.PathError{
Op: "inotify_add_watch",
Path: path,
Err: err,
}
}
if !found {
w.watches[path] = &watch{wd: uint32(wd), flags: flags}
w.paths[wd] = path
}
w.mu.Unlock()
return nil
}
// Watch adds path to the watched file set, watching all events.
func (w *Watcher) Watch(path string) error {
return w.AddWatch(path, IN_ALL_EVENTS)
}
// RemoveWatch removes path from the watched file set.
func (w *Watcher) RemoveWatch(path string) error {
watch, ok := w.watches[path]
if !ok {
return errors.New(fmt.Sprintf("can't remove non-existent inotify watch for: %s", path))
}
success, errno := syscall.InotifyRmWatch(w.fd, watch.wd)
if success == -1 {
return os.NewSyscallError("inotify_rm_watch", errno)
}
delete(w.watches, path)
return nil
}
// readEvents reads from the inotify file descriptor, converts the
// received events into Event objects and sends them via the Event channel
func (w *Watcher) readEvents() {
var buf [syscall.SizeofInotifyEvent * 4096]byte
for {
n, err := syscall.Read(w.fd, buf[:])
// See if there is a message on the "done" channel
var done bool
select {
case done = <-w.done:
default:
}
// If EOF or a "done" message is received
if n == 0 || done {
// The syscall.Close can be slow. Close
// w.Event first.
close(w.Event)
err := syscall.Close(w.fd)
if err != nil {
w.Error <- os.NewSyscallError("close", err)
}
close(w.Error)
return
}
if n < 0 {
w.Error <- os.NewSyscallError("read", err)
continue
}
if n < syscall.SizeofInotifyEvent {
w.Error <- errors.New("inotify: short read in readEvents()")
continue
}
var offset uint32 = 0
// We don't know how many events we just read into the buffer
// While the offset points to at least one whole event...
for offset <= uint32(n-syscall.SizeofInotifyEvent) {
// Point "raw" to the event in the buffer
raw := (*syscall.InotifyEvent)(unsafe.Pointer(&buf[offset]))
event := new(Event)
event.Mask = uint32(raw.Mask)
event.Cookie = uint32(raw.Cookie)
nameLen := uint32(raw.Len)
// If the event happened to the watched directory or the watched file, the kernel
// doesn't append the filename to the event, but we would like to always fill the
// the "Name" field with a valid filename. We retrieve the path of the watch from
// the "paths" map.
w.mu.Lock()
event.Name = w.paths[int(raw.Wd)]
w.mu.Unlock()
if nameLen > 0 {
// Point "bytes" at the first byte of the filename
bytes := (*[syscall.PathMax]byte)(unsafe.Pointer(&buf[offset+syscall.SizeofInotifyEvent]))
// The filename is padded with NUL bytes. TrimRight() gets rid of those.
event.Name += "/" + strings.TrimRight(string(bytes[0:nameLen]), "\000")
}
// Send the event on the events channel
w.Event <- event
// Move to the next event in the buffer
offset += syscall.SizeofInotifyEvent + nameLen
}
}
}
// String formats the event e in the form
// "filename: 0xEventMask = IN_ACCESS|IN_ATTRIB_|..."
func (e *Event) String() string {
var events string = ""
m := e.Mask
for _, b := range eventBits {
if m&b.Value != 0 {
m &^= b.Value
events += "|" + b.Name
}
}
if m != 0 {
events += fmt.Sprintf("|%#x", m)
}
if len(events) > 0 {
events = " == " + events[1:]
}
return fmt.Sprintf("%q: %#x%s", e.Name, e.Mask, events)
}
const (
// Options for inotify_init() are not exported
// IN_CLOEXEC uint32 = syscall.IN_CLOEXEC
// IN_NONBLOCK uint32 = syscall.IN_NONBLOCK
// Options for AddWatch
IN_DONT_FOLLOW uint32 = syscall.IN_DONT_FOLLOW
IN_ONESHOT uint32 = syscall.IN_ONESHOT
IN_ONLYDIR uint32 = syscall.IN_ONLYDIR
// The "IN_MASK_ADD" option is not exported, as AddWatch
// adds it automatically, if there is already a watch for the given path
// IN_MASK_ADD uint32 = syscall.IN_MASK_ADD
// Events
IN_ACCESS uint32 = syscall.IN_ACCESS
IN_ALL_EVENTS uint32 = syscall.IN_ALL_EVENTS
IN_ATTRIB uint32 = syscall.IN_ATTRIB
IN_CLOSE uint32 = syscall.IN_CLOSE
IN_CLOSE_NOWRITE uint32 = syscall.IN_CLOSE_NOWRITE
IN_CLOSE_WRITE uint32 = syscall.IN_CLOSE_WRITE
IN_CREATE uint32 = syscall.IN_CREATE
IN_DELETE uint32 = syscall.IN_DELETE
IN_DELETE_SELF uint32 = syscall.IN_DELETE_SELF
IN_MODIFY uint32 = syscall.IN_MODIFY
IN_MOVE uint32 = syscall.IN_MOVE
IN_MOVED_FROM uint32 = syscall.IN_MOVED_FROM
IN_MOVED_TO uint32 = syscall.IN_MOVED_TO
IN_MOVE_SELF uint32 = syscall.IN_MOVE_SELF
IN_OPEN uint32 = syscall.IN_OPEN
// Special events
IN_ISDIR uint32 = syscall.IN_ISDIR
IN_IGNORED uint32 = syscall.IN_IGNORED
IN_Q_OVERFLOW uint32 = syscall.IN_Q_OVERFLOW
IN_UNMOUNT uint32 = syscall.IN_UNMOUNT
)
var eventBits = []struct {
Value uint32
Name string
}{
{IN_ACCESS, "IN_ACCESS"},
{IN_ATTRIB, "IN_ATTRIB"},
{IN_CLOSE, "IN_CLOSE"},
{IN_CLOSE_NOWRITE, "IN_CLOSE_NOWRITE"},
{IN_CLOSE_WRITE, "IN_CLOSE_WRITE"},
{IN_CREATE, "IN_CREATE"},
{IN_DELETE, "IN_DELETE"},
{IN_DELETE_SELF, "IN_DELETE_SELF"},
{IN_MODIFY, "IN_MODIFY"},
{IN_MOVE, "IN_MOVE"},
{IN_MOVED_FROM, "IN_MOVED_FROM"},
{IN_MOVED_TO, "IN_MOVED_TO"},
{IN_MOVE_SELF, "IN_MOVE_SELF"},
{IN_OPEN, "IN_OPEN"},
{IN_ISDIR, "IN_ISDIR"},
{IN_IGNORED, "IN_IGNORED"},
{IN_Q_OVERFLOW, "IN_Q_OVERFLOW"},
{IN_UNMOUNT, "IN_UNMOUNT"},
}
src/pkg/exp/inotify/inotify_linux_test.go deleted 100644 → 0
View file @ 01a5b66d
// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build linux
package inotify
import (
"io/ioutil"
"os"
"sync/atomic"
"testing"
"time"
)
func TestInotifyEvents(t *testing.T) {
// Create an inotify watcher instance and initialize it
watcher, err := NewWatcher()
if err != nil {
t.Fatalf("NewWatcher failed: %s", err)
}
dir, err := ioutil.TempDir("", "inotify")
if err != nil {
t.Fatalf("TempDir failed: %s", err)
}
defer os.RemoveAll(dir)
// Add a watch for "_test"
err = watcher.Watch(dir)
if err != nil {
t.Fatalf("Watch failed: %s", err)
}
// Receive errors on the error channel on a separate goroutine
go func() {
for err := range watcher.Error {
t.Fatalf("error received: %s", err)
}
}()
testFile := dir + "/TestInotifyEvents.testfile"
// Receive events on the event channel on a separate goroutine
eventstream := watcher.Event
var eventsReceived int32 = 0
done := make(chan bool)
go func() {
for event := range eventstream {
// Only count relevant events
if event.Name == testFile {
atomic.AddInt32(&eventsReceived, 1)
t.Logf("event received: %s", event)
} else {
t.Logf("unexpected event received: %s", event)
}
}
done <- true
}()
// Create a file
// This should add at least one event to the inotify event queue
_, err = os.OpenFile(testFile, os.O_WRONLY|os.O_CREATE, 0666)
if err != nil {
t.Fatalf("creating test file: %s", err)
}
// We expect this event to be received almost immediately, but let's wait 1 s to be sure
time.Sleep(1 * time.Second)
if atomic.AddInt32(&eventsReceived, 0) == 0 {
t.Fatal("inotify event hasn't been received after 1 second")
}
// Try closing the inotify instance
t.Log("calling Close()")
watcher.Close()
t.Log("waiting for the event channel to become closed...")
select {
case <-done:
t.Log("event channel closed")
case <-time.After(1 * time.Second):
t.Fatal("event stream was not closed after 1 second")
}
}
func TestInotifyClose(t *testing.T) {
watcher, _ := NewWatcher()
watcher.Close()
done := make(chan bool)
go func() {
watcher.Close()
done <- true
}()
select {
case <-done:
case <-time.After(50 * time.Millisecond):
t.Fatal("double Close() test failed: second Close() call didn't return")
}
err := watcher.Watch(os.TempDir())
if err == nil {
t.Fatal("expected error on Watch() after Close(), got nil")
}
}
src/pkg/exp/locale/collate/Makefile deleted 100644 → 0
View file @ 01a5b66d
# Copyright 2012 The Go Authors. All rights reserved.
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file.
CLEANFILES+=maketables
maketables: maketables.go
go build $^
tables: maketables
./maketables > tables.go
gofmt -w -s tables.go
# Build (but do not run) maketables during testing,
# just to make sure it still compiles.
testshort: maketables
src/pkg/exp/locale/collate/build/builder.go deleted 100644 → 0
View file @ 01a5b66d
This diff is collapsed.
src/pkg/exp/locale/collate/build/builder_test.go deleted 100644 → 0
View file @ 01a5b66d
// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package build
import "testing"
// cjk returns an implicit collation element for a CJK rune.
func cjk(r rune) []rawCE {
// A CJK character C is represented in the DUCET as
// [.AAAA.0020.0002.C][.BBBB.0000.0000.C]
// Where AAAA is the most significant 15 bits plus a base value.
// Any base value will work for the test, so we pick the common value of FB40.
const base = 0xFB40
return []rawCE{
{w: []int{base + int(r>>15), defaultSecondary, defaultTertiary, int(r)}},
{w: []int{int(r&0x7FFF) | 0x8000, 0, 0, int(r)}},
}
}
func pCE(p int) []rawCE {
return mkCE([]int{p, defaultSecondary, defaultTertiary, 0}, 0)
}
func pqCE(p, q int) []rawCE {
return mkCE([]int{p, defaultSecondary, defaultTertiary, q}, 0)
}
func ptCE(p, t int) []rawCE {
return mkCE([]int{p, defaultSecondary, t, 0}, 0)
}
func ptcCE(p, t int, ccc uint8) []rawCE {
return mkCE([]int{p, defaultSecondary, t, 0}, ccc)
}
func sCE(s int) []rawCE {
return mkCE([]int{0, s, defaultTertiary, 0}, 0)
}
func stCE(s, t int) []rawCE {
return mkCE([]int{0, s, t, 0}, 0)
}
func scCE(s int, ccc uint8) []rawCE {
return mkCE([]int{0, s, defaultTertiary, 0}, ccc)
}
func mkCE(w []int, ccc uint8) []rawCE {
return []rawCE{rawCE{w, ccc}}
}
// ducetElem is used to define test data that is used to generate a table.
type ducetElem struct {
str string
ces []rawCE
}
func newBuilder(t *testing.T, ducet []ducetElem) *Builder {
b := NewBuilder()
for _, e := range ducet {
ces := [][]int{}
for _, ce := range e.ces {
ces = append(ces, ce.w)
}
if err := b.Add([]rune(e.str), ces, nil); err != nil {
t.Errorf(err.Error())
}
}
b.t = &table{}
b.root.sort()
return b
}
type convertTest struct {
in, out []rawCE
err bool
}
var convLargeTests = []convertTest{
{pCE(0xFB39), pCE(0xFB39), false},
{cjk(0x2F9B2), pqCE(0x3F9B2, 0x2F9B2), false},
{pCE(0xFB40), pCE(0), true},
{append(pCE(0xFB40), pCE(0)[0]), pCE(0), true},
{pCE(0xFFFE), pCE(illegalOffset), false},
{pCE(0xFFFF), pCE(illegalOffset + 1), false},
}
func TestConvertLarge(t *testing.T) {
for i, tt := range convLargeTests {
e := new(entry)
for _, ce := range tt.in {
e.elems = append(e.elems, makeRawCE(ce.w, ce.ccc))
}
elems, err := convertLargeWeights(e.elems)
if tt.err {
if err == nil {
t.Errorf("%d: expected error; none found", i)
}
continue
} else if err != nil {
t.Errorf("%d: unexpected error: %v", i, err)
}
if !equalCEArrays(elems, tt.out) {
t.Errorf("%d: conversion was %x; want %x", i, elems, tt.out)
}
}
}
// Collation element table for simplify tests.
var simplifyTest = []ducetElem{
{"\u0300", sCE(30)}, // grave
{"\u030C", sCE(40)}, // caron
{"A", ptCE(100, 8)},
{"D", ptCE(104, 8)},
{"E", ptCE(105, 8)},
{"I", ptCE(110, 8)},
{"z", ptCE(130, 8)},
{"\u05F2", append(ptCE(200, 4), ptCE(200, 4)[0])},
{"\u05B7", sCE(80)},
{"\u00C0", append(ptCE(100, 8), sCE(30)...)}, // A with grave, can be removed
{"\u00C8", append(ptCE(105, 8), sCE(30)...)}, // E with grave
{"\uFB1F", append(ptCE(200, 4), ptCE(200, 4)[0], sCE(80)[0])}, // eliminated by NFD
{"\u00C8\u0302", ptCE(106, 8)}, // block previous from simplifying
{"\u01C5", append(ptCE(104, 9), ptCE(130, 4)[0], stCE(40, maxTertiary)[0])}, // eliminated by NFKD
// no removal: tertiary value of third element is not maxTertiary
{"\u2162", append(ptCE(110, 9), ptCE(110, 4)[0], ptCE(110, 8)[0])},
}
var genColTests = []ducetElem{
{"\uFA70", pqCE(0x1FA70, 0xFA70)},
{"A\u0300", append(ptCE(100, 8), sCE(30)...)},
{"A\u0300\uFA70", append(ptCE(100, 8), sCE(30)[0], pqCE(0x1FA70, 0xFA70)[0])},
{"A\u0300A\u0300", append(ptCE(100, 8), sCE(30)[0], ptCE(100, 8)[0], sCE(30)[0])},
}
func TestGenColElems(t *testing.T) {
b := newBuilder(t, simplifyTest[:5])
for i, tt := range genColTests {
res := b.root.genColElems(tt.str)
if !equalCEArrays(tt.ces, res) {
t.Errorf("%d: result %X; want %X", i, res, tt.ces)
}
}
}
type strArray []string
func (sa strArray) contains(s string) bool {
for _, e := range sa {
if e == s {
return true
}
}
return false
}
var simplifyRemoved = strArray{"\u00C0", "\uFB1F"}
var simplifyMarked = strArray{"\u01C5"}
func TestSimplify(t *testing.T) {
b := newBuilder(t, simplifyTest)
o := &b.root
simplify(o)
for i, tt := range simplifyTest {
if simplifyRemoved.contains(tt.str) {
continue
}
e := o.find(tt.str)
if e.str != tt.str || !equalCEArrays(e.elems, tt.ces) {
t.Errorf("%d: found element %s -> %X; want %s -> %X", i, e.str, e.elems, tt.str, tt.ces)
break
}
}
var i, k int
for e := o.front(); e != nil; e, _ = e.nextIndexed() {
gold := simplifyMarked.contains(e.str)
if gold {
k++
}
if gold != e.decompose {
t.Errorf("%d: %s has decompose %v; want %v", i, e.str, e.decompose, gold)
}
i++
}
if k != len(simplifyMarked) {
t.Errorf(" an entry that should be marked as decompose was deleted")
}
}
var expandTest = []ducetElem{
{"\u0300", append(scCE(29, 230), scCE(30, 230)...)},
{"\u00C0", append(ptCE(100, 8), scCE(30, 230)...)},
{"\u00C8", append(ptCE(105, 8), scCE(30, 230)...)},
{"\u00C9", append(ptCE(105, 8), scCE(30, 230)...)}, // identical expansion
{"\u05F2", append(ptCE(200, 4), ptCE(200, 4)[0], ptCE(200, 4)[0])},
{"\u01FF", append(ptCE(200, 4), ptcCE(201, 4, 0)[0], scCE(30, 230)[0])},
}
func TestExpand(t *testing.T) {
const (
totalExpansions = 5
totalElements = 2 + 2 + 2 + 3 + 3 + totalExpansions
)
b := newBuilder(t, expandTest)
o := &b.root
b.processExpansions(o)
e := o.front()
for _, tt := range expandTest {
exp := b.t.expandElem[e.expansionIndex:]
if int(exp[0]) != len(tt.ces) {
t.Errorf("%U: len(expansion)==%d; want %d", []rune(tt.str)[0], exp[0], len(tt.ces))
}
exp = exp[1:]
for j, w := range tt.ces {
if ce, _ := makeCE(w); exp[j] != ce {
t.Errorf("%U: element %d is %X; want %X", []rune(tt.str)[0], j, exp[j], ce)
}
}
e, _ = e.nextIndexed()
}
// Verify uniquing.
if len(b.t.expandElem) != totalElements {
t.Errorf("len(expandElem)==%d; want %d", len(b.t.expandElem), totalElements)
}
}
var contractTest = []ducetElem{
{"abc", pCE(102)},
{"abd", pCE(103)},
{"a", pCE(100)},
{"ab", pCE(101)},
{"ac", pCE(104)},
{"bcd", pCE(202)},
{"b", pCE(200)},
{"bc", pCE(201)},
{"bd", pCE(203)},
// shares suffixes with a*
{"Ab", pCE(301)},
{"A", pCE(300)},
{"Ac", pCE(304)},
{"Abc", pCE(302)},
{"Abd", pCE(303)},
// starter to be ignored
{"z", pCE(1000)},
}
func TestContract(t *testing.T) {
const (
totalElements = 5 + 5 + 4
)
b := newBuilder(t, contractTest)
o := &b.root
b.processContractions(o)
indexMap := make(map[int]bool)
handleMap := make(map[rune]*entry)
for e := o.front(); e != nil; e, _ = e.nextIndexed() {
if e.contractionHandle.n > 0 {
handleMap[e.runes[0]] = e
indexMap[e.contractionHandle.index] = true
}
}
// Verify uniquing.
if len(indexMap) != 2 {
t.Errorf("number of tries is %d; want %d", len(indexMap), 2)
}
for _, tt := range contractTest {
e, ok := handleMap[[]rune(tt.str)[0]]
if !ok {
continue
}
str := tt.str[1:]
offset, n := b.t.contractTries.lookup(e.contractionHandle, []byte(str))
if len(str) != n {
t.Errorf("%s: bytes consumed==%d; want %d", tt.str, n, len(str))
}
ce := b.t.contractElem[offset+e.contractionIndex]
if want, _ := makeCE(tt.ces[0]); want != ce {
t.Errorf("%s: element %X; want %X", tt.str, ce, want)
}
}
if len(b.t.contractElem) != totalElements {
t.Errorf("len(expandElem)==%d; want %d", len(b.t.contractElem), totalElements)
}
}
src/pkg/exp/locale/collate/build/colelem.go deleted 100644 → 0
View file @ 01a5b66d
// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package build
import (
"exp/locale/collate/colltab"
"fmt"
"unicode"
)
const (
defaultSecondary = 0x20
defaultTertiary = 0x2
maxTertiary = 0x1F
)
type rawCE struct {
w []int
ccc uint8
}
func makeRawCE(w []int, ccc uint8) rawCE {
ce := rawCE{w: make([]int, 4), ccc: ccc}
copy(ce.w, w)
return ce
}
// A collation element is represented as an uint32.
// In the typical case, a rune maps to a single collation element. If a rune
// can be the start of a contraction or expands into multiple collation elements,
// then the collation element that is associated with a rune will have a special
// form to represent such m to n mappings. Such special collation elements
// have a value >= 0x80000000.
const (
maxPrimaryBits = 21
maxSecondaryBits = 12
maxTertiaryBits = 8
)
func makeCE(ce rawCE) (uint32, error) {
v, e := colltab.MakeElem(ce.w[0], ce.w[1], ce.w[2], ce.ccc)
return uint32(v), e
}
// For contractions, collation elements are of the form
// 110bbbbb bbbbbbbb iiiiiiii iiiinnnn, where
// - n* is the size of the first node in the contraction trie.
// - i* is the index of the first node in the contraction trie.
// - b* is the offset into the contraction collation element table.
// See contract.go for details on the contraction trie.
const (
contractID = 0xC0000000
maxNBits = 4
maxTrieIndexBits = 12
maxContractOffsetBits = 13
)
func makeContractIndex(h ctHandle, offset int) (uint32, error) {
if h.n >= 1<<maxNBits {
return 0, fmt.Errorf("size of contraction trie node too large: %d >= %d", h.n, 1<<maxNBits)
}
if h.index >= 1<<maxTrieIndexBits {
return 0, fmt.Errorf("size of contraction trie offset too large: %d >= %d", h.index, 1<<maxTrieIndexBits)
}
if offset >= 1<<maxContractOffsetBits {
return 0, fmt.Errorf("contraction offset out of bounds: %x >= %x", offset, 1<<maxContractOffsetBits)
}
ce := uint32(contractID)
ce += uint32(offset << (maxNBits + maxTrieIndexBits))
ce += uint32(h.index << maxNBits)
ce += uint32(h.n)
return ce, nil
}
// For expansions, collation elements are of the form
// 11100000 00000000 bbbbbbbb bbbbbbbb,
// where b* is the index into the expansion sequence table.
const (
expandID = 0xE0000000
maxExpandIndexBits = 16
)
func makeExpandIndex(index int) (uint32, error) {
if index >= 1<<maxExpandIndexBits {
return 0, fmt.Errorf("expansion index out of bounds: %x >= %x", index, 1<<maxExpandIndexBits)
}
return expandID + uint32(index), nil
}
// Each list of collation elements corresponding to an expansion starts with
// a header indicating the length of the sequence.
func makeExpansionHeader(n int) (uint32, error) {
return uint32(n), nil
}
// Some runes can be expanded using NFKD decomposition. Instead of storing the full
// sequence of collation elements, we decompose the rune and lookup the collation
// elements for each rune in the decomposition and modify the tertiary weights.
// The collation element, in this case, is of the form
// 11110000 00000000 wwwwwwww vvvvvvvv, where
// - v* is the replacement tertiary weight for the first rune,
// - w* is the replacement tertiary weight for the second rune,
// Tertiary weights of subsequent runes should be replaced with maxTertiary.
// See http://www.unicode.org/reports/tr10/#Compatibility_Decompositions for more details.
const (
decompID = 0xF0000000
)
func makeDecompose(t1, t2 int) (uint32, error) {
if t1 >= 256 || t1 < 0 {
return 0, fmt.Errorf("first tertiary weight out of bounds: %d >= 256", t1)
}
if t2 >= 256 || t2 < 0 {
return 0, fmt.Errorf("second tertiary weight out of bounds: %d >= 256", t2)
}
return uint32(t2<<8+t1) + decompID, nil
}
const (
// These constants were taken from http://www.unicode.org/versions/Unicode6.0.0/ch12.pdf.
minUnified rune = 0x4E00
maxUnified = 0x9FFF
minCompatibility = 0xF900
maxCompatibility = 0xFAFF
minRare = 0x3400
maxRare = 0x4DBF
)
const (
commonUnifiedOffset = 0x10000
rareUnifiedOffset = 0x20000 // largest rune in common is U+FAFF
otherOffset = 0x50000 // largest rune in rare is U+2FA1D
illegalOffset = otherOffset + int(unicode.MaxRune)
maxPrimary = illegalOffset + 1
)
// implicitPrimary returns the primary weight for the a rune
// for which there is no entry for the rune in the collation table.
// We take a different approach from the one specified in
// http://unicode.org/reports/tr10/#Implicit_Weights,
// but preserve the resulting relative ordering of the runes.
func implicitPrimary(r rune) int {
if unicode.Is(unicode.Ideographic, r) {
if r >= minUnified && r <= maxUnified {
// The most common case for CJK.
return int(r) + commonUnifiedOffset
}
if r >= minCompatibility && r <= maxCompatibility {
// This will typically not hit. The DUCET explicitly specifies mappings
// for all characters that do not decompose.
return int(r) + commonUnifiedOffset
}
return int(r) + rareUnifiedOffset
}
return int(r) + otherOffset
}
// convertLargeWeights converts collation elements with large
// primaries (either double primaries or for illegal runes)
// to our own representation.
// A CJK character C is represented in the DUCET as
// [.FBxx.0020.0002.C][.BBBB.0000.0000.C]
// We will rewrite these characters to a single CE.
// We assume the CJK values start at 0x8000.
// See http://unicode.org/reports/tr10/#Implicit_Weights
func convertLargeWeights(elems []rawCE) (res []rawCE, err error) {
const (
cjkPrimaryStart = 0xFB40
rarePrimaryStart = 0xFB80
otherPrimaryStart = 0xFBC0
illegalPrimary = 0xFFFE
highBitsMask = 0x3F
lowBitsMask = 0x7FFF
lowBitsFlag = 0x8000
shiftBits = 15
)
for i := 0; i < len(elems); i++ {
ce := elems[i].w
p := ce[0]
if p < cjkPrimaryStart {
continue
}
if p > 0xFFFF {
return elems, fmt.Errorf("found primary weight %X; should be <= 0xFFFF", p)
}
if p >= illegalPrimary {
ce[0] = illegalOffset + p - illegalPrimary
} else {
if i+1 >= len(elems) {
return elems, fmt.Errorf("second part of double primary weight missing: %v", elems)
}
if elems[i+1].w[0]&lowBitsFlag == 0 {
return elems, fmt.Errorf("malformed second part of double primary weight: %v", elems)
}
np := ((p & highBitsMask) << shiftBits) + elems[i+1].w[0]&lowBitsMask
switch {
case p < rarePrimaryStart:
np += commonUnifiedOffset
case p < otherPrimaryStart:
np += rareUnifiedOffset
default:
p += otherOffset
}
ce[0] = np
for j := i + 1; j+1 < len(elems); j++ {
elems[j] = elems[j+1]
}
elems = elems[:len(elems)-1]
}
}
return elems, nil
}
// nextWeight computes the first possible collation weights following elems
// for the given level.
func nextWeight(level colltab.Level, elems []rawCE) []rawCE {
if level == colltab.Identity {
next := make([]rawCE, len(elems))
copy(next, elems)
return next
}
next := []rawCE{makeRawCE(elems[0].w, elems[0].ccc)}
next[0].w[level]++
if level < colltab.Secondary {
next[0].w[colltab.Secondary] = defaultSecondary
}
if level < colltab.Tertiary {
next[0].w[colltab.Tertiary] = defaultTertiary
}
// Filter entries that cannot influence ordering.
for _, ce := range elems[1:] {
skip := true
for i := colltab.Primary; i < level; i++ {
skip = skip && ce.w[i] == 0
}
if !skip {
next = append(next, ce)
}
}
return next
}
func nextVal(elems []rawCE, i int, level colltab.Level) (index, value int) {
for ; i < len(elems) && elems[i].w[level] == 0; i++ {
}
if i < len(elems) {
return i, elems[i].w[level]
}
return i, 0
}
// compareWeights returns -1 if a < b, 1 if a > b, or 0 otherwise.
// It also returns the collation level at which the difference is found.
func compareWeights(a, b []rawCE) (result int, level colltab.Level) {
for level := colltab.Primary; level < colltab.Identity; level++ {
var va, vb int
for ia, ib := 0, 0; ia < len(a) || ib < len(b); ia, ib = ia+1, ib+1 {
ia, va = nextVal(a, ia, level)
ib, vb = nextVal(b, ib, level)
if va != vb {
if va < vb {
return -1, level
} else {
return 1, level
}
}
}
}
return 0, colltab.Identity
}
func equalCE(a, b rawCE) bool {
for i := 0; i < 3; i++ {
if b.w[i] != a.w[i] {
return false
}
}
return true
}
func equalCEArrays(a, b []rawCE) bool {
if len(a) != len(b) {
return false
}
for i := range a {
if !equalCE(a[i], b[i]) {
return false
}
}
return true
}
src/pkg/exp/locale/collate/build/colelem_test.go deleted 100644 → 0
View file @ 01a5b66d
// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package build
import (
"exp/locale/collate/colltab"
"testing"
)
type ceTest struct {
f func(in []int) (uint32, error)
arg []int
val uint32
}
func normalCE(in []int) (ce uint32, err error) {
return makeCE(rawCE{w: in[:3], ccc: uint8(in[3])})
}
func expandCE(in []int) (ce uint32, err error) {
return makeExpandIndex(in[0])
}
func contractCE(in []int) (ce uint32, err error) {
return makeContractIndex(ctHandle{in[0], in[1]}, in[2])
}
func decompCE(in []int) (ce uint32, err error) {
return makeDecompose(in[0], in[1])
}
var ceTests = []ceTest{
{normalCE, []int{0, 0, 0, 0}, 0xA0000000},
{normalCE, []int{0, 0x28, 3, 0}, 0xA0002803},
{normalCE, []int{0, 0x28, 3, 0xFF}, 0xAFF02803},
{normalCE, []int{100, defaultSecondary, 3, 0}, 0x0000C883},
// non-ignorable primary with non-default secondary
{normalCE, []int{100, 0x28, defaultTertiary, 0}, 0x4000C828},
{normalCE, []int{100, defaultSecondary + 8, 3, 0}, 0x0000C983},
{normalCE, []int{100, 0, 3, 0}, 0xFFFF}, // non-ignorable primary with non-supported secondary
{normalCE, []int{100, 1, 3, 0}, 0xFFFF},
{normalCE, []int{1 << maxPrimaryBits, defaultSecondary, 0, 0}, 0xFFFF},
{normalCE, []int{0, 1 << maxSecondaryBits, 0, 0}, 0xFFFF},
{normalCE, []int{100, defaultSecondary, 1 << maxTertiaryBits, 0}, 0xFFFF},
{normalCE, []int{0x123, defaultSecondary, 8, 0xFF}, 0x88FF0123},
{normalCE, []int{0x123, defaultSecondary + 1, 8, 0xFF}, 0xFFFF},
{contractCE, []int{0, 0, 0}, 0xC0000000},
{contractCE, []int{1, 1, 1}, 0xC0010011},
{contractCE, []int{1, (1 << maxNBits) - 1, 1}, 0xC001001F},
{contractCE, []int{(1 << maxTrieIndexBits) - 1, 1, 1}, 0xC001FFF1},
{contractCE, []int{1, 1, (1 << maxContractOffsetBits) - 1}, 0xDFFF0011},
{contractCE, []int{1, (1 << maxNBits), 1}, 0xFFFF},
{contractCE, []int{(1 << maxTrieIndexBits), 1, 1}, 0xFFFF},
{contractCE, []int{1, (1 << maxContractOffsetBits), 1}, 0xFFFF},
{expandCE, []int{0}, 0xE0000000},
{expandCE, []int{5}, 0xE0000005},
{expandCE, []int{(1 << maxExpandIndexBits) - 1}, 0xE000FFFF},
{expandCE, []int{1 << maxExpandIndexBits}, 0xFFFF},
{decompCE, []int{0, 0}, 0xF0000000},
{decompCE, []int{1, 1}, 0xF0000101},
{decompCE, []int{0x1F, 0x1F}, 0xF0001F1F},
{decompCE, []int{256, 0x1F}, 0xFFFF},
{decompCE, []int{0x1F, 256}, 0xFFFF},
}
func TestColElem(t *testing.T) {
for i, tt := range ceTests {
in := make([]int, len(tt.arg))
copy(in, tt.arg)
ce, err := tt.f(in)
if tt.val == 0xFFFF {
if err == nil {
t.Errorf("%d: expected error for args %x", i, tt.arg)
}
continue
}
if err != nil {
t.Errorf("%d: unexpected error: %v", i, err.Error())
}
if ce != tt.val {
t.Errorf("%d: colElem=%X; want %X", i, ce, tt.val)
}
}
}
func mkRawCES(in [][]int) []rawCE {
out := []rawCE{}
for _, w := range in {
out = append(out, rawCE{w: w})
}
return out
}
type weightsTest struct {
a, b [][]int
level colltab.Level
result int
}
var nextWeightTests = []weightsTest{
{
a: [][]int{{100, 20, 5, 0}},
b: [][]int{{101, defaultSecondary, defaultTertiary, 0}},
level: colltab.Primary,
},
{
a: [][]int{{100, 20, 5, 0}},
b: [][]int{{100, 21, defaultTertiary, 0}},
level: colltab.Secondary,
},
{
a: [][]int{{100, 20, 5, 0}},
b: [][]int{{100, 20, 6, 0}},
level: colltab.Tertiary,
},
{
a: [][]int{{100, 20, 5, 0}},
b: [][]int{{100, 20, 5, 0}},
level: colltab.Identity,
},
}
var extra = [][]int{{200, 32, 8, 0}, {0, 32, 8, 0}, {0, 0, 8, 0}, {0, 0, 0, 0}}
func TestNextWeight(t *testing.T) {
for i, tt := range nextWeightTests {
test := func(l colltab.Level, tt weightsTest, a, gold [][]int) {
res := nextWeight(tt.level, mkRawCES(a))
if !equalCEArrays(mkRawCES(gold), res) {
t.Errorf("%d:%d: expected weights %d; found %d", i, l, gold, res)
}
}
test(-1, tt, tt.a, tt.b)
for l := colltab.Primary; l <= colltab.Tertiary; l++ {
if tt.level <= l {
test(l, tt, append(tt.a, extra[l]), tt.b)
} else {
test(l, tt, append(tt.a, extra[l]), append(tt.b, extra[l]))
}
}
}
}
var compareTests = []weightsTest{
{
[][]int{{100, 20, 5, 0}},
[][]int{{100, 20, 5, 0}},
colltab.Identity,
0,
},
{
[][]int{{100, 20, 5, 0}, extra[0]},
[][]int{{100, 20, 5, 1}},
colltab.Primary,
1,
},
{
[][]int{{100, 20, 5, 0}},
[][]int{{101, 20, 5, 0}},
colltab.Primary,
-1,
},
{
[][]int{{101, 20, 5, 0}},
[][]int{{100, 20, 5, 0}},
colltab.Primary,
1,
},
{
[][]int{{100, 0, 0, 0}, {0, 20, 5, 0}},
[][]int{{0, 20, 5, 0}, {100, 0, 0, 0}},
colltab.Identity,
0,
},
{
[][]int{{100, 20, 5, 0}},
[][]int{{100, 21, 5, 0}},
colltab.Secondary,
-1,
},
{
[][]int{{100, 20, 5, 0}},
[][]int{{100, 20, 2, 0}},
colltab.Tertiary,
1,
},
{
[][]int{{100, 20, 5, 1}},
[][]int{{100, 20, 5, 2}},
colltab.Quaternary,
-1,
},
}
func TestCompareWeights(t *testing.T) {
for i, tt := range compareTests {
test := func(tt weightsTest, a, b [][]int) {
res, level := compareWeights(mkRawCES(a), mkRawCES(b))
if res != tt.result {
t.Errorf("%d: expected comparisson result %d; found %d", i, tt.result, res)
}
if level != tt.level {
t.Errorf("%d: expected level %d; found %d", i, tt.level, level)
}
}
test(tt, tt.a, tt.b)
test(tt, append(tt.a, extra[0]), append(tt.b, extra[0]))
}
}
src/pkg/exp/locale/collate/build/contract.go deleted 100644 → 0
View file @ 01a5b66d
// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package build
import (
"fmt"
"io"
"reflect"
"sort"
"strings"
)
// This file contains code for detecting contractions and generating
// the necessary tables.
// Any Unicode Collation Algorithm (UCA) table entry that has more than
// one rune one the left-hand side is called a contraction.
// See http://www.unicode.org/reports/tr10/#Contractions for more details.
//
// We define the following terms:
// initial: a rune that appears as the first rune in a contraction.
// suffix: a sequence of runes succeeding the initial rune
// in a given contraction.
// non-initial: a rune that appears in a suffix.
//
// A rune may be both a initial and a non-initial and may be so in
// many contractions. An initial may typically also appear by itself.
// In case of ambiguities, the UCA requires we match the longest
// contraction.
//
// Many contraction rules share the same set of possible suffixes.
// We store sets of suffixes in a trie that associates an index with
// each suffix in the set. This index can be used to look up a
// collation element associated with the (starter rune, suffix) pair.
//
// The trie is defined on a UTF-8 byte sequence.
// The overall trie is represented as an array of ctEntries. Each node of the trie
// is represented as a subsequence of ctEntries, where each entry corresponds to
// a possible match of a next character in the search string. An entry
// also includes the length and offset to the next sequence of entries
// to check in case of a match.
const (
final = 0
noIndex = 0xFF
)
// ctEntry associates to a matching byte an offset and/or next sequence of
// bytes to check. A ctEntry c is called final if a match means that the
// longest suffix has been found. An entry c is final if c.n == 0.
// A single final entry can match a range of characters to an offset.
// A non-final entry always matches a single byte. Note that a non-final
// entry might still resemble a completed suffix.
// Examples:
// The suffix strings "ab" and "ac" can be represented as:
// []ctEntry{
// {'a', 1, 1, noIndex}, // 'a' by itself does not match, so i is 0xFF.
// {'b', 'c', 0, 1}, // "ab" -> 1, "ac" -> 2
// }
//
// The suffix strings "ab", "abc", "abd", and "abcd" can be represented as:
// []ctEntry{
// {'a', 1, 1, noIndex}, // 'a' must be followed by 'b'.
// {'b', 1, 2, 1}, // "ab" -> 1, may be followed by 'c' or 'd'.
// {'d', 'd', final, 3}, // "abd" -> 3
// {'c', 4, 1, 2}, // "abc" -> 2, may be followed by 'd'.
// {'d', 'd', final, 4}, // "abcd" -> 4
// }
// See genStateTests in contract_test.go for more examples.
type ctEntry struct {
l uint8 // non-final: byte value to match; final: lowest match in range.
h uint8 // non-final: relative index to next block; final: highest match in range.
n uint8 // non-final: length of next block; final: final
i uint8 // result offset. Will be noIndex if more bytes are needed to complete.
}
// contractTrieSet holds a set of contraction tries. The tries are stored
// consecutively in the entry field.
type contractTrieSet []struct{ l, h, n, i uint8 }
// ctHandle is used to identify a trie in the trie set, consisting in an offset
// in the array and the size of the first node.
type ctHandle struct {
index, n int
}
// appendTrie adds a new trie for the given suffixes to the trie set and returns
// a handle to it. The handle will be invalid on error.
func (ct *contractTrieSet) appendTrie(suffixes []string) (ctHandle, error) {
es := make([]stridx, len(suffixes))
for i, s := range suffixes {
es[i].str = s
}
sort.Sort(offsetSort(es))
for i := range es {
es[i].index = i + 1
}
sort.Sort(genidxSort(es))
i := len(*ct)
n, err := ct.genStates(es)
if err != nil {
*ct = (*ct)[:i]
return ctHandle{}, err
}
return ctHandle{i, n}, nil
}
// genStates generates ctEntries for a given suffix set and returns
// the number of entries for the first node.
func (ct *contractTrieSet) genStates(sis []stridx) (int, error) {
if len(sis) == 0 {
return 0, fmt.Errorf("genStates: list of suffices must be non-empty")
}
start := len(*ct)
// create entries for differing first bytes.
for _, si := range sis {
s := si.str
if len(s) == 0 {
continue
}
added := false
c := s[0]
if len(s) > 1 {
for j := len(*ct) - 1; j >= start; j-- {
if (*ct)[j].l == c {
added = true
break
}
}
if !added {
*ct = append(*ct, ctEntry{l: c, i: noIndex})
}
} else {
for j := len(*ct) - 1; j >= start; j-- {
// Update the offset for longer suffixes with the same byte.
if (*ct)[j].l == c {
(*ct)[j].i = uint8(si.index)
added = true
}
// Extend range of final ctEntry, if possible.
if (*ct)[j].h+1 == c {
(*ct)[j].h = c
added = true
}
}
if !added {
*ct = append(*ct, ctEntry{l: c, h: c, n: final, i: uint8(si.index)})
}
}
}
n := len(*ct) - start
// Append nodes for the remainder of the suffixes for each ctEntry.
sp := 0
for i, end := start, len(*ct); i < end; i++ {
fe := (*ct)[i]
if fe.h == 0 { // uninitialized non-final
ln := len(*ct) - start - n
if ln > 0xFF {
return 0, fmt.Errorf("genStates: relative block offset too large: %d > 255", ln)
}
fe.h = uint8(ln)
// Find first non-final strings with same byte as current entry.
for ; sis[sp].str[0] != fe.l; sp++ {
}
se := sp + 1
for ; se < len(sis) && len(sis[se].str) > 1 && sis[se].str[0] == fe.l; se++ {
}
sl := sis[sp:se]
sp = se
for i, si := range sl {
sl[i].str = si.str[1:]
}
nn, err := ct.genStates(sl)
if err != nil {
return 0, err
}
fe.n = uint8(nn)
(*ct)[i] = fe
}
}
sort.Sort(entrySort((*ct)[start : start+n]))
return n, nil
}
// There may be both a final and non-final entry for a byte if the byte
// is implied in a range of matches in the final entry.
// We need to ensure that the non-final entry comes first in that case.
type entrySort contractTrieSet
func (fe entrySort) Len() int { return len(fe) }
func (fe entrySort) Swap(i, j int) { fe[i], fe[j] = fe[j], fe[i] }
func (fe entrySort) Less(i, j int) bool {
return fe[i].l > fe[j].l
}
// stridx is used for sorting suffixes and their associated offsets.
type stridx struct {
str string
index int
}
// For computing the offsets, we first sort by size, and then by string.
// This ensures that strings that only differ in the last byte by 1
// are sorted consecutively in increasing order such that they can
// be packed as a range in a final ctEntry.
type offsetSort []stridx
func (si offsetSort) Len() int { return len(si) }
func (si offsetSort) Swap(i, j int) { si[i], si[j] = si[j], si[i] }
func (si offsetSort) Less(i, j int) bool {
if len(si[i].str) != len(si[j].str) {
return len(si[i].str) > len(si[j].str)
}
return si[i].str < si[j].str
}
// For indexing, we want to ensure that strings are sorted in string order, where
// for strings with the same prefix, we put longer strings before shorter ones.
type genidxSort []stridx
func (si genidxSort) Len() int { return len(si) }
func (si genidxSort) Swap(i, j int) { si[i], si[j] = si[j], si[i] }
func (si genidxSort) Less(i, j int) bool {
if strings.HasPrefix(si[j].str, si[i].str) {
return false
}
if strings.HasPrefix(si[i].str, si[j].str) {
return true
}
return si[i].str < si[j].str
}
// lookup matches the longest suffix in str and returns the associated offset
// and the number of bytes consumed.
func (ct *contractTrieSet) lookup(h ctHandle, str []byte) (index, ns int) {
states := (*ct)[h.index:]
p := 0
n := h.n
for i := 0; i < n && p < len(str); {
e := states[i]
c := str[p]
if c >= e.l {
if e.l == c {
p++
if e.i != noIndex {
index, ns = int(e.i), p
}
if e.n != final {
// set to new state
i, states, n = 0, states[int(e.h)+n:], int(e.n)
} else {
return
}
continue
} else if e.n == final && c <= e.h {
p++
return int(c-e.l) + int(e.i), p
}
}
i++
}
return
}
// print writes the contractTrieSet t as compilable Go code to w. It returns
// the total number of bytes written and the size of the resulting data structure in bytes.
func (t *contractTrieSet) print(w io.Writer, name string) (n, size int, err error) {
update3 := func(nn, sz int, e error) {
n += nn
if err == nil {
err = e
}
size += sz
}
update2 := func(nn int, e error) { update3(nn, 0, e) }
update3(t.printArray(w, name))
update2(fmt.Fprintf(w, "var %sContractTrieSet = ", name))
update3(t.printStruct(w, name))
update2(fmt.Fprintln(w))
return
}
func (ct contractTrieSet) printArray(w io.Writer, name string) (n, size int, err error) {
p := func(f string, a ...interface{}) {
nn, e := fmt.Fprintf(w, f, a...)
n += nn
if err == nil {
err = e
}
}
size = len(ct) * 4
p("// %sCTEntries: %d entries, %d bytes\n", name, len(ct), size)
p("var %sCTEntries = [%d]struct{l,h,n,i uint8}{\n", name, len(ct))
for _, fe := range ct {
p("\t{0x%X, 0x%X, %d, %d},\n", fe.l, fe.h, fe.n, fe.i)
}
p("}\n")
return
}
func (ct contractTrieSet) printStruct(w io.Writer, name string) (n, size int, err error) {
n, err = fmt.Fprintf(w, "contractTrieSet( %sCTEntries[:] )", name)
size = int(reflect.TypeOf(ct).Size())
return
}
src/pkg/exp/locale/collate/build/contract_test.go deleted 100644 → 0
View file @ 01a5b66d
// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package build
import (
"bytes"
"sort"
"testing"
)
var largetosmall = []stridx{
{"a", 5},
{"ab", 4},
{"abc", 3},
{"abcd", 2},
{"abcde", 1},
{"abcdef", 0},
}
var offsetSortTests = [][]stridx{
{
{"bcde", 1},
{"bc", 5},
{"ab", 4},
{"bcd", 3},
{"abcd", 0},
{"abc", 2},
},
largetosmall,
}
func TestOffsetSort(t *testing.T) {
for i, st := range offsetSortTests {
sort.Sort(offsetSort(st))
for j, si := range st {
if j != si.index {
t.Errorf("%d: failed: %v", i, st)
}
}
}
for i, tt := range genStateTests {
// ensure input is well-formed
sort.Sort(offsetSort(tt.in))
for j, si := range tt.in {
if si.index != j+1 {
t.Errorf("%dth sort failed: %v", i, tt.in)
}
}
}
}
var genidxtest1 = []stridx{
{"bcde", 3},
{"bc", 6},
{"ab", 2},
{"bcd", 5},
{"abcd", 0},
{"abc", 1},
{"bcdf", 4},
}
var genidxSortTests = [][]stridx{
genidxtest1,
largetosmall,
}
func TestGenIdxSort(t *testing.T) {
for i, st := range genidxSortTests {
sort.Sort(genidxSort(st))
for j, si := range st {
if j != si.index {
t.Errorf("%dth sort failed %v", i, st)
break
}
}
}
}
var entrySortTests = []contractTrieSet{
{
{10, 0, 1, 3},
{99, 0, 1, 0},
{20, 50, 0, 2},
{30, 0, 1, 1},
},
}
func TestEntrySort(t *testing.T) {
for i, et := range entrySortTests {
sort.Sort(entrySort(et))
for j, fe := range et {
if j != int(fe.i) {
t.Errorf("%dth sort failed %v", i, et)
break
}
}
}
}
type GenStateTest struct {
in []stridx
firstBlockLen int
out contractTrieSet
}
var genStateTests = []GenStateTest{
{[]stridx{
{"abc", 1},
},
1,
contractTrieSet{
{'a', 0, 1, noIndex},
{'b', 0, 1, noIndex},
{'c', 'c', final, 1},
},
},
{[]stridx{
{"abc", 1},
{"abd", 2},
{"abe", 3},
},
1,
contractTrieSet{
{'a', 0, 1, noIndex},
{'b', 0, 1, noIndex},
{'c', 'e', final, 1},
},
},
{[]stridx{
{"abc", 1},
{"ab", 2},
{"a", 3},
},
1,
contractTrieSet{
{'a', 0, 1, 3},
{'b', 0, 1, 2},
{'c', 'c', final, 1},
},
},
{[]stridx{
{"abc", 1},
{"abd", 2},
{"ab", 3},
{"ac", 4},
{"a", 5},
{"b", 6},
},
2,
contractTrieSet{
{'b', 'b', final, 6},
{'a', 0, 2, 5},
{'c', 'c', final, 4},
{'b', 0, 1, 3},
{'c', 'd', final, 1},
},
},
{[]stridx{
{"bcde", 2},
{"bc", 7},
{"ab", 6},
{"bcd", 5},
{"abcd", 1},
{"abc", 4},
{"bcdf", 3},
},
2,
contractTrieSet{
{'b', 3, 1, noIndex},
{'a', 0, 1, noIndex},
{'b', 0, 1, 6},
{'c', 0, 1, 4},
{'d', 'd', final, 1},
{'c', 0, 1, 7},
{'d', 0, 1, 5},
{'e', 'f', final, 2},
},
},
}
func TestGenStates(t *testing.T) {
for i, tt := range genStateTests {
si := []stridx{}
for _, e := range tt.in {
si = append(si, e)
}
// ensure input is well-formed
sort.Sort(genidxSort(si))
ct := contractTrieSet{}
n, _ := ct.genStates(si)
if nn := tt.firstBlockLen; nn != n {
t.Errorf("%d: block len %v; want %v", i, n, nn)
}
if lv, lw := len(ct), len(tt.out); lv != lw {
t.Errorf("%d: len %v; want %v", i, lv, lw)
continue
}
for j, fe := range tt.out {
const msg = "%d:%d: value %s=%v; want %v"
if fe.l != ct[j].l {
t.Errorf(msg, i, j, "l", ct[j].l, fe.l)
}
if fe.h != ct[j].h {
t.Errorf(msg, i, j, "h", ct[j].h, fe.h)
}
if fe.n != ct[j].n {
t.Errorf(msg, i, j, "n", ct[j].n, fe.n)
}
if fe.i != ct[j].i {
t.Errorf(msg, i, j, "i", ct[j].i, fe.i)
}
}
}
}
func TestLookupContraction(t *testing.T) {
for i, tt := range genStateTests {
input := []string{}
for _, e := range tt.in {
input = append(input, e.str)
}
cts := contractTrieSet{}
h, _ := cts.appendTrie(input)
for j, si := range tt.in {
str := si.str
for _, s := range []string{str, str + "X"} {
msg := "%d:%d: %s(%s) %v; want %v"
idx, sn := cts.lookup(h, []byte(s))
if idx != si.index {
t.Errorf(msg, i, j, "index", s, idx, si.index)
}
if sn != len(str) {
t.Errorf(msg, i, j, "sn", s, sn, len(str))
}
}
}
}
}
func TestPrintContractionTrieSet(t *testing.T) {
testdata := contractTrieSet(genStateTests[4].out)
buf := &bytes.Buffer{}
testdata.print(buf, "test")
if contractTrieOutput != buf.String() {
t.Errorf("output differs; found\n%s", buf.String())
println(string(buf.Bytes()))
}
}
const contractTrieOutput = `// testCTEntries: 8 entries, 32 bytes
var testCTEntries = [8]struct{l,h,n,i uint8}{
{0x62, 0x3, 1, 255},
{0x61, 0x0, 1, 255},
{0x62, 0x0, 1, 6},
{0x63, 0x0, 1, 4},
{0x64, 0x64, 0, 1},
{0x63, 0x0, 1, 7},
{0x64, 0x0, 1, 5},
{0x65, 0x66, 0, 2},
}
var testContractTrieSet = contractTrieSet( testCTEntries[:] )
`
src/pkg/exp/locale/collate/build/order.go deleted 100644 → 0
View file @ 01a5b66d
This diff is collapsed.
src/pkg/exp/locale/collate/build/order_test.go deleted 100644 → 0
View file @ 01a5b66d
// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package build
import (
"exp/locale/collate/colltab"
"strconv"
"testing"
)
type entryTest struct {
f func(in []int) (uint32, error)
arg []int
val uint32
}
// makeList returns a list of entries of length n+2, with n normal
// entries plus a leading and trailing anchor.
func makeList(n int) []*entry {
es := make([]*entry, n+2)
weights := []rawCE{{w: []int{100, 20, 5, 0}}}
for i := range es {
runes := []rune{rune(i)}
es[i] = &entry{
runes: runes,
elems: weights,
}
weights = nextWeight(colltab.Primary, weights)
}
for i := 1; i < len(es); i++ {
es[i-1].next = es[i]
es[i].prev = es[i-1]
_, es[i-1].level = compareWeights(es[i-1].elems, es[i].elems)
}
es[0].exclude = true
es[0].logical = firstAnchor
es[len(es)-1].exclude = true
es[len(es)-1].logical = lastAnchor
return es
}
func TestNextIndexed(t *testing.T) {
const n = 5
es := makeList(n)
for i := int64(0); i < 1<<n; i++ {
mask := strconv.FormatInt(i+(1<<n), 2)
for i, c := range mask {
es[i].exclude = c == '1'
}
e := es[0]
for i, c := range mask {
if c == '0' {
e, _ = e.nextIndexed()
if e != es[i] {
t.Errorf("%d: expected entry %d; found %d", i, es[i].elems, e.elems)
}
}
}
if e, _ = e.nextIndexed(); e != nil {
t.Errorf("%d: expected nil entry; found %d", i, e.elems)
}
}
}
func TestRemove(t *testing.T) {
const n = 5
for i := int64(0); i < 1<<n; i++ {
es := makeList(n)
mask := strconv.FormatInt(i+(1<<n), 2)
for i, c := range mask {
if c == '0' {
es[i].remove()
}
}
e := es[0]
for i, c := range mask {
if c == '1' {
if e != es[i] {
t.Errorf("%d: expected entry %d; found %d", i, es[i].elems, e.elems)
}
e, _ = e.nextIndexed()
}
}
if e != nil {
t.Errorf("%d: expected nil entry; found %d", i, e.elems)
}
}
}
// nextPerm generates the next permutation of the array. The starting
// permutation is assumed to be a list of integers sorted in increasing order.
// It returns false if there are no more permuations left.
func nextPerm(a []int) bool {
i := len(a) - 2
for ; i >= 0; i-- {
if a[i] < a[i+1] {
break
}
}
if i < 0 {
return false
}
for j := len(a) - 1; j >= i; j-- {
if a[j] > a[i] {
a[i], a[j] = a[j], a[i]
break
}
}
for j := i + 1; j < (len(a)+i+1)/2; j++ {
a[j], a[len(a)+i-j] = a[len(a)+i-j], a[j]
}
return true
}
func TestInsertAfter(t *testing.T) {
const n = 5
orig := makeList(n)
perm := make([]int, n)
for i := range perm {
perm[i] = i + 1
}
for ok := true; ok; ok = nextPerm(perm) {
es := makeList(n)
last := es[0]
for _, i := range perm {
last.insertAfter(es[i])
last = es[i]
}
for _, e := range es {
e.elems = es[0].elems
}
e := es[0]
for _, i := range perm {
e, _ = e.nextIndexed()
if e.runes[0] != orig[i].runes[0] {
t.Errorf("%d:%d: expected entry %X; found %X", perm, i, orig[i].runes, e.runes)
break
}
}
}
}
func TestInsertBefore(t *testing.T) {
const n = 5
orig := makeList(n)
perm := make([]int, n)
for i := range perm {
perm[i] = i + 1
}
for ok := true; ok; ok = nextPerm(perm) {
es := makeList(n)
last := es[len(es)-1]
for _, i := range perm {
last.insertBefore(es[i])
last = es[i]
}
for _, e := range es {
e.elems = es[0].elems
}
e := es[0]
for i := n - 1; i >= 0; i-- {
e, _ = e.nextIndexed()
if e.runes[0] != rune(perm[i]) {
t.Errorf("%d:%d: expected entry %X; found %X", perm, i, orig[i].runes, e.runes)
break
}
}
}
}
type entryLessTest struct {
a, b *entry
res bool
}
var (
w1 = []rawCE{{w: []int{100, 20, 5, 5}}}
w2 = []rawCE{{w: []int{101, 20, 5, 5}}}
)
var entryLessTests = []entryLessTest{
{&entry{str: "a", elems: w1},
&entry{str: "a", elems: w1},
false,
},
{&entry{str: "a", elems: w1},
&entry{str: "a", elems: w2},
true,
},
{&entry{str: "a", elems: w1},
&entry{str: "b", elems: w1},
true,
},
{&entry{str: "a", elems: w2},
&entry{str: "a", elems: w1},
false,
},
{&entry{str: "c", elems: w1},
&entry{str: "b", elems: w1},
false,
},
{&entry{str: "a", elems: w1, logical: firstAnchor},
&entry{str: "a", elems: w1},
true,
},
{&entry{str: "a", elems: w1},
&entry{str: "b", elems: w1, logical: firstAnchor},
false,
},
{&entry{str: "b", elems: w1},
&entry{str: "a", elems: w1, logical: lastAnchor},
true,
},
{&entry{str: "a", elems: w1, logical: lastAnchor},
&entry{str: "c", elems: w1},
false,
},
}
func TestEntryLess(t *testing.T) {
for i, tt := range entryLessTests {
if res := entryLess(tt.a, tt.b); res != tt.res {
t.Errorf("%d: was %v; want %v", i, res, tt.res)
}
}
}
src/pkg/exp/locale/collate/build/table.go deleted 100644 → 0
View file @ 01a5b66d
// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package build
import (
"fmt"
"io"
"reflect"
)
// table is an intermediate structure that roughly resembles the table in collate.
// It implements the non-exported interface collate.tableInitializer
type table struct {
index trie // main trie
root *trieHandle
// expansion info
expandElem []uint32
// contraction info
contractTries contractTrieSet
contractElem []uint32
maxContractLen int
variableTop uint32
}
func (t *table) TrieIndex() []uint16 {
return t.index.index
}
func (t *table) TrieValues() []uint32 {
return t.index.values
}
func (t *table) FirstBlockOffsets() (i, v uint16) {
return t.root.lookupStart, t.root.valueStart
}
func (t *table) ExpandElems() []uint32 {
return t.expandElem
}
func (t *table) ContractTries() []struct{ l, h, n, i uint8 } {
return t.contractTries
}
func (t *table) ContractElems() []uint32 {
return t.contractElem
}
func (t *table) MaxContractLen() int {
return t.maxContractLen
}
func (t *table) VariableTop() uint32 {
return t.variableTop
}
// print writes the table as Go compilable code to w. It prefixes the
// variable names with name. It returns the number of bytes written
// and the size of the resulting table.
func (t *table) fprint(w io.Writer, name string) (n, size int, err error) {
update := func(nn, sz int, e error) {
n += nn
if err == nil {
err = e
}
size += sz
}
// Write arrays needed for the structure.
update(printColElems(w, t.expandElem, name+"ExpandElem"))
update(printColElems(w, t.contractElem, name+"ContractElem"))
update(t.index.printArrays(w, name))
update(t.contractTries.printArray(w, name))
nn, e := fmt.Fprintf(w, "// Total size of %sTable is %d bytes\n", name, size)
update(nn, 0, e)
return
}
func (t *table) fprintIndex(w io.Writer, h *trieHandle) (n int, err error) {
p := func(f string, a ...interface{}) {
nn, e := fmt.Fprintf(w, f, a...)
n += nn
if err == nil {
err = e
}
}
p("tableIndex{\n")
p("\t\tlookupOffset: 0x%x,\n", h.lookupStart)
p("\t\tvaluesOffset: 0x%x,\n", h.valueStart)
p("\t}")
return
}
func printColElems(w io.Writer, a []uint32, name string) (n, sz int, err error) {
p := func(f string, a ...interface{}) {
nn, e := fmt.Fprintf(w, f, a...)
n += nn
if err == nil {
err = e
}
}
sz = len(a) * int(reflect.TypeOf(uint32(0)).Size())
p("// %s: %d entries, %d bytes\n", name, len(a), sz)
p("var %s = [%d]uint32 {", name, len(a))
for i, c := range a {
switch {
case i%64 == 0:
p("\n\t// Block %d, offset 0x%x\n", i/64, i)
case (i%64)%6 == 0:
p("\n\t")
}
p("0x%.8X, ", c)
}
p("\n}\n\n")
return
}
src/pkg/exp/locale/collate/build/trie.go deleted 100644 → 0
View file @ 01a5b66d
// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// The trie in this file is used to associate the first full character
// in a UTF-8 string to a collation element.
// All but the last byte in a UTF-8 byte sequence are
// used to look up offsets in the index table to be used for the next byte.
// The last byte is used to index into a table of collation elements.
// This file contains the code for the generation of the trie.
package build
import (
"fmt"
"hash/fnv"
"io"
"reflect"
)
const (
blockSize = 64
blockOffset = 2 // Subtract 2 blocks to compensate for the 0x80 added to continuation bytes.
)
type trieHandle struct {
lookupStart uint16 // offset in table for first byte
valueStart uint16 // offset in table for first byte
}
type trie struct {
index []uint16
values []uint32
}
// trieNode is the intermediate trie structure used for generating a trie.
type trieNode struct {
index []*trieNode
value []uint32
b byte
refValue uint16
refIndex uint16
}
func newNode() *trieNode {
return &trieNode{
index: make([]*trieNode, 64),
value: make([]uint32, 128), // root node size is 128 instead of 64
}
}
func (n *trieNode) isInternal() bool {
return n.value != nil
}
func (n *trieNode) insert(r rune, value uint32) {
const maskx = 0x3F // mask out two most-significant bits
str := string(r)
if len(str) == 1 {
n.value[str[0]] = value
return
}
for i := 0; i < len(str)-1; i++ {
b := str[i] & maskx
if n.index == nil {
n.index = make([]*trieNode, blockSize)
}
nn := n.index[b]
if nn == nil {
nn = &trieNode{}
nn.b = b
n.index[b] = nn
}
n = nn
}
if n.value == nil {
n.value = make([]uint32, blockSize)
}
b := str[len(str)-1] & maskx
n.value[b] = value
}
type trieBuilder struct {
t *trie
roots []*trieHandle
lookupBlocks []*trieNode
valueBlocks []*trieNode
lookupBlockIdx map[uint32]*trieNode
valueBlockIdx map[uint32]*trieNode
}
func newTrieBuilder() *trieBuilder {
index := &trieBuilder{}
index.lookupBlocks = make([]*trieNode, 0)
index.valueBlocks = make([]*trieNode, 0)
index.lookupBlockIdx = make(map[uint32]*trieNode)
index.valueBlockIdx = make(map[uint32]*trieNode)
// The third nil is the default null block. The other two blocks
// are used to guarantee an offset of at least 3 for each block.
index.lookupBlocks = append(index.lookupBlocks, nil, nil, nil)
index.t = &trie{}
return index
}
func (b *trieBuilder) computeOffsets(n *trieNode) *trieNode {
hasher := fnv.New32()
if n.index != nil {
for i, nn := range n.index {
var vi, vv uint16
if nn != nil {
nn = b.computeOffsets(nn)
n.index[i] = nn
vi = nn.refIndex
vv = nn.refValue
}
hasher.Write([]byte{byte(vi >> 8), byte(vi)})
hasher.Write([]byte{byte(vv >> 8), byte(vv)})
}
h := hasher.Sum32()
nn, ok := b.lookupBlockIdx[h]
if !ok {
n.refIndex = uint16(len(b.lookupBlocks)) - blockOffset
b.lookupBlocks = append(b.lookupBlocks, n)
b.lookupBlockIdx[h] = n
} else {
n = nn
}
} else {
for _, v := range n.value {
hasher.Write([]byte{byte(v >> 24), byte(v >> 16), byte(v >> 8), byte(v)})
}
h := hasher.Sum32()
nn, ok := b.valueBlockIdx[h]
if !ok {
n.refValue = uint16(len(b.valueBlocks)) - blockOffset
n.refIndex = n.refValue
b.valueBlocks = append(b.valueBlocks, n)
b.valueBlockIdx[h] = n
} else {
n = nn
}
}
return n
}
func (b *trieBuilder) addStartValueBlock(n *trieNode) uint16 {
hasher := fnv.New32()
for _, v := range n.value[:2*blockSize] {
hasher.Write([]byte{byte(v >> 24), byte(v >> 16), byte(v >> 8), byte(v)})
}
h := hasher.Sum32()
nn, ok := b.valueBlockIdx[h]
if !ok {
n.refValue = uint16(len(b.valueBlocks))
n.refIndex = n.refValue
b.valueBlocks = append(b.valueBlocks, n)
// Add a dummy block to accommodate the double block size.
b.valueBlocks = append(b.valueBlocks, nil)
b.valueBlockIdx[h] = n
} else {
n = nn
}
return n.refValue
}
func genValueBlock(t *trie, n *trieNode) {
if n != nil {
for _, v := range n.value {
t.values = append(t.values, v)
}
}
}
func genLookupBlock(t *trie, n *trieNode) {
for _, nn := range n.index {
v := uint16(0)
if nn != nil {
if n.index != nil {
v = nn.refIndex
} else {
v = nn.refValue
}
}
t.index = append(t.index, v)
}
}
func (b *trieBuilder) addTrie(n *trieNode) *trieHandle {
h := &trieHandle{}
b.roots = append(b.roots, h)
h.valueStart = b.addStartValueBlock(n)
if len(b.roots) == 1 {
// We insert a null block after the first start value block.
// This ensures that continuation bytes UTF-8 sequences of length
// greater than 2 will automatically hit a null block if there
// was an undefined entry.
b.valueBlocks = append(b.valueBlocks, nil)
}
n = b.computeOffsets(n)
// Offset by one extra block as the first byte starts at 0xC0 instead of 0x80.
h.lookupStart = n.refIndex - 1
return h
}
// generate generates and returns the trie for n.
func (b *trieBuilder) generate() (t *trie, err error) {
t = b.t
if len(b.valueBlocks) >= 1<<16 {
return nil, fmt.Errorf("maximum number of value blocks exceeded (%d > %d)", len(b.valueBlocks), 1<<16)
}
if len(b.lookupBlocks) >= 1<<16 {
return nil, fmt.Errorf("maximum number of lookup blocks exceeded (%d > %d)", len(b.lookupBlocks), 1<<16)
}
genValueBlock(t, b.valueBlocks[0])
genValueBlock(t, &trieNode{value: make([]uint32, 64)})
for i := 2; i < len(b.valueBlocks); i++ {
genValueBlock(t, b.valueBlocks[i])
}
n := &trieNode{index: make([]*trieNode, 64)}
genLookupBlock(t, n)
genLookupBlock(t, n)
genLookupBlock(t, n)
for i := 3; i < len(b.lookupBlocks); i++ {
genLookupBlock(t, b.lookupBlocks[i])
}
return b.t, nil
}
func (t *trie) printArrays(w io.Writer, name string) (n, size int, err error) {
p := func(f string, a ...interface{}) {
nn, e := fmt.Fprintf(w, f, a...)
n += nn
if err == nil {
err = e
}
}
nv := len(t.values)
p("// %sValues: %d entries, %d bytes\n", name, nv, nv*4)
p("// Block 2 is the null block.\n")
p("var %sValues = [%d]uint32 {", name, nv)
var printnewline bool
for i, v := range t.values {
if i%blockSize == 0 {
p("\n\t// Block %#x, offset %#x", i/blockSize, i)
}
if i%4 == 0 {
printnewline = true
}
if v != 0 {
if printnewline {
p("\n\t")
printnewline = false
}
p("%#04x:%#08x, ", i, v)
}
}
p("\n}\n\n")
ni := len(t.index)
p("// %sLookup: %d entries, %d bytes\n", name, ni, ni*2)
p("// Block 0 is the null block.\n")
p("var %sLookup = [%d]uint16 {", name, ni)
printnewline = false
for i, v := range t.index {
if i%blockSize == 0 {
p("\n\t// Block %#x, offset %#x", i/blockSize, i)
}
if i%8 == 0 {
printnewline = true
}
if v != 0 {
if printnewline {
p("\n\t")
printnewline = false
}
p("%#03x:%#02x, ", i, v)
}
}
p("\n}\n\n")
return n, nv*4 + ni*2, err
}
func (t *trie) printStruct(w io.Writer, handle *trieHandle, name string) (n, sz int, err error) {
const msg = "trie{ %sLookup[%d:], %sValues[%d:], %sLookup[:], %sValues[:]}"
n, err = fmt.Fprintf(w, msg, name, handle.lookupStart*blockSize, name, handle.valueStart*blockSize, name, name)
sz += int(reflect.TypeOf(trie{}).Size())
return
}
src/pkg/exp/locale/collate/build/trie_test.go deleted 100644 → 0
View file @ 01a5b66d
// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package build
import (
"bytes"
"fmt"
"testing"
)
// We take the smallest, largest and an arbitrary value for each
// of the UTF-8 sequence lengths.
var testRunes = []rune{
0x01, 0x0C, 0x7F, // 1-byte sequences
0x80, 0x100, 0x7FF, // 2-byte sequences
0x800, 0x999, 0xFFFF, // 3-byte sequences
0x10000, 0x10101, 0x10FFFF, // 4-byte sequences
0x200, 0x201, 0x202, 0x210, 0x215, // five entries in one sparse block
}
func makeTestTrie(t *testing.T) trie {
n := newNode()
for i, r := range testRunes {
n.insert(r, uint32(i))
}
idx := newTrieBuilder()
idx.addTrie(n)
tr, err := idx.generate()
if err != nil {
t.Errorf(err.Error())
}
return *tr
}
func TestGenerateTrie(t *testing.T) {
testdata := makeTestTrie(t)
buf := &bytes.Buffer{}
testdata.printArrays(buf, "test")
fmt.Fprintf(buf, "var testTrie = ")
testdata.printStruct(buf, &trieHandle{19, 0}, "test")
if output != buf.String() {
t.Error("output differs")
}
}
var output = `// testValues: 832 entries, 3328 bytes
// Block 2 is the null block.
var testValues = [832]uint32 {
// Block 0x0, offset 0x0
0x000c:0x00000001,
// Block 0x1, offset 0x40
0x007f:0x00000002,
// Block 0x2, offset 0x80
// Block 0x3, offset 0xc0
0x00c0:0x00000003,
// Block 0x4, offset 0x100
0x0100:0x00000004,
// Block 0x5, offset 0x140
0x0140:0x0000000c, 0x0141:0x0000000d, 0x0142:0x0000000e,
0x0150:0x0000000f,
0x0155:0x00000010,
// Block 0x6, offset 0x180
0x01bf:0x00000005,
// Block 0x7, offset 0x1c0
0x01c0:0x00000006,
// Block 0x8, offset 0x200
0x0219:0x00000007,
// Block 0x9, offset 0x240
0x027f:0x00000008,
// Block 0xa, offset 0x280
0x0280:0x00000009,
// Block 0xb, offset 0x2c0
0x02c1:0x0000000a,
// Block 0xc, offset 0x300
0x033f:0x0000000b,
}
// testLookup: 640 entries, 1280 bytes
// Block 0 is the null block.
var testLookup = [640]uint16 {
// Block 0x0, offset 0x0
// Block 0x1, offset 0x40
// Block 0x2, offset 0x80
// Block 0x3, offset 0xc0
0x0e0:0x05, 0x0e6:0x06,
// Block 0x4, offset 0x100
0x13f:0x07,
// Block 0x5, offset 0x140
0x140:0x08, 0x144:0x09,
// Block 0x6, offset 0x180
0x190:0x03,
// Block 0x7, offset 0x1c0
0x1ff:0x0a,
// Block 0x8, offset 0x200
0x20f:0x05,
// Block 0x9, offset 0x240
0x242:0x01, 0x244:0x02,
0x248:0x03,
0x25f:0x04,
0x260:0x01,
0x26f:0x02,
0x270:0x04, 0x274:0x06,
}
var testTrie = trie{ testLookup[1216:], testValues[0:], testLookup[:], testValues[:]}`
src/pkg/exp/locale/collate/collate.go deleted 100644 → 0
View file @ 01a5b66d
This diff is collapsed.
src/pkg/exp/locale/collate/collate_test.go deleted 100644 → 0
View file @ 01a5b66d
This diff is collapsed.
src/pkg/exp/locale/collate/colltab/colelem.go deleted 100644 → 0
View file @ 01a5b66d
This diff is collapsed.
src/pkg/exp/locale/collate/colltab/colelem_test.go deleted 100644 → 0
View file @ 01a5b66d
// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package colltab
import (
"testing"
"unicode"
)
type ceTest struct {
f func(inout []int) (Elem, ceType)
arg []int
}
func makeCE(weights []int) Elem {
ce, _ := MakeElem(weights[0], weights[1], weights[2], uint8(weights[3]))
return ce
}
func makeContractIndex(index, n, offset int) Elem {
const (
contractID = 0xC0000000
maxNBits = 4
maxTrieIndexBits = 12
maxContractOffsetBits = 13
)
ce := Elem(contractID)
ce += Elem(offset << (maxNBits + maxTrieIndexBits))
ce += Elem(index << maxNBits)
ce += Elem(n)
return ce
}
func makeExpandIndex(index int) Elem {
const expandID = 0xE0000000
return expandID + Elem(index)
}
func makeDecompose(t1, t2 int) Elem {
const decompID = 0xF0000000
return Elem(t2<<8+t1) + decompID
}
func normalCE(inout []int) (ce Elem, t ceType) {
ce = makeCE(inout)
inout[0] = ce.Primary()
inout[1] = ce.Secondary()
inout[2] = int(ce.Tertiary())
inout[3] = int(ce.CCC())
return ce, ceNormal
}
func expandCE(inout []int) (ce Elem, t ceType) {
ce = makeExpandIndex(inout[0])
inout[0] = splitExpandIndex(ce)
return ce, ceExpansionIndex
}
func contractCE(inout []int) (ce Elem, t ceType) {
ce = makeContractIndex(inout[0], inout[1], inout[2])
i, n, o := splitContractIndex(ce)
inout[0], inout[1], inout[2] = i, n, o
return ce, ceContractionIndex
}
func decompCE(inout []int) (ce Elem, t ceType) {
ce = makeDecompose(inout[0], inout[1])
t1, t2 := splitDecompose(ce)
inout[0], inout[1] = int(t1), int(t2)
return ce, ceDecompose
}
var ceTests = []ceTest{
{normalCE, []int{0, 0, 0, 0}},
{normalCE, []int{0, 30, 3, 0}},
{normalCE, []int{0, 30, 3, 0xFF}},
{normalCE, []int{100, defaultSecondary, defaultTertiary, 0}},
{normalCE, []int{100, defaultSecondary, defaultTertiary, 0xFF}},
{normalCE, []int{100, defaultSecondary, 3, 0}},
{normalCE, []int{0x123, defaultSecondary, 8, 0xFF}},
{contractCE, []int{0, 0, 0}},
{contractCE, []int{1, 1, 1}},
{contractCE, []int{1, (1 << maxNBits) - 1, 1}},
{contractCE, []int{(1 << maxTrieIndexBits) - 1, 1, 1}},
{contractCE, []int{1, 1, (1 << maxContractOffsetBits) - 1}},
{expandCE, []int{0}},
{expandCE, []int{5}},
{expandCE, []int{(1 << maxExpandIndexBits) - 1}},
{decompCE, []int{0, 0}},
{decompCE, []int{1, 1}},
{decompCE, []int{0x1F, 0x1F}},
}
func TestColElem(t *testing.T) {
for i, tt := range ceTests {
inout := make([]int, len(tt.arg))
copy(inout, tt.arg)
ce, typ := tt.f(inout)
if ce.ctype() != typ {
t.Errorf("%d: type is %d; want %d (ColElem: %X)", i, ce.ctype(), typ, ce)
}
for j, a := range tt.arg {
if inout[j] != a {
t.Errorf("%d: argument %d is %X; want %X (ColElem: %X)", i, j, inout[j], a, ce)
}
}
}
}
type implicitTest struct {
r rune
p int
}
var implicitTests = []implicitTest{
{0x33FF, 0x533FF},
{0x3400, 0x23400},
{0x4DC0, 0x54DC0},
{0x4DFF, 0x54DFF},
{0x4E00, 0x14E00},
{0x9FCB, 0x19FCB},
{0xA000, 0x5A000},
{0xF8FF, 0x5F8FF},
{0xF900, 0x1F900},
{0xFA23, 0x1FA23},
{0xFAD9, 0x1FAD9},
{0xFB00, 0x5FB00},
{0x20000, 0x40000},
{0x2B81C, 0x4B81C},
{unicode.MaxRune, 0x15FFFF}, // maximum primary value
}
func TestImplicit(t *testing.T) {
for _, tt := range implicitTests {
if p := implicitPrimary(tt.r); p != tt.p {
t.Errorf("%U: was %X; want %X", tt.r, p, tt.p)
}
}
}
func TestUpdateTertiary(t *testing.T) {
tests := []struct {
in, out Elem
t uint8
}{
{0x4000FE20, 0x0000FE8A, 0x0A},
{0x4000FE21, 0x0000FEAA, 0x0A},
{0x0000FE8B, 0x0000FE83, 0x03},
{0x82FF0188, 0x9BFF0188, 0x1B},
{0xAFF0CC02, 0xAFF0CC1B, 0x1B},
}
for i, tt := range tests {
if out := tt.in.updateTertiary(tt.t); out != tt.out {
t.Errorf("%d: was %X; want %X", i, out, tt.out)
}
}
}
src/pkg/exp/locale/collate/colltab/colltab.go deleted 100644 → 0
View file @ 01a5b66d
This diff is collapsed.
src/pkg/exp/locale/collate/colltab/contract.go deleted 100644 → 0
View file @ 01a5b66d
This diff is collapsed.
src/pkg/exp/locale/collate/colltab/contract_test.go deleted 100644 → 0
View file @ 01a5b66d
This diff is collapsed.
src/pkg/exp/locale/collate/colltab/export.go deleted 100644 → 0
View file @ 01a5b66d
This diff is collapsed.
src/pkg/exp/locale/collate/colltab/table.go deleted 100644 → 0
View file @ 01a5b66d
This diff is collapsed.
src/pkg/exp/locale/collate/colltab/trie.go deleted 100644 → 0
View file @ 01a5b66d
This diff is collapsed.
src/pkg/exp/locale/collate/colltab/trie_test.go deleted 100644 → 0
View file @ 01a5b66d
This diff is collapsed.
src/pkg/exp/locale/collate/export_test.go deleted 100644 → 0
View file @ 01a5b66d
This diff is collapsed.
src/pkg/exp/locale/collate/index.go deleted 100644 → 0
View file @ 01a5b66d
This diff is collapsed.
src/pkg/exp/locale/collate/maketables.go deleted 100644 → 0
View file @ 01a5b66d
This diff is collapsed.
src/pkg/exp/locale/collate/regtest.go deleted 100644 → 0
View file @ 01a5b66d
This diff is collapsed.
src/pkg/exp/locale/collate/sort.go deleted 100644 → 0
View file @ 01a5b66d
This diff is collapsed.
src/pkg/exp/locale/collate/sort_test.go deleted 100644 → 0
View file @ 01a5b66d
This diff is collapsed.
src/pkg/exp/locale/collate/table_test.go deleted 100644 → 0
View file @ 01a5b66d
This diff is collapsed.
src/pkg/exp/locale/collate/tables.go deleted 100644 → 0
View file @ 01a5b66d
This diff is collapsed.
src/pkg/exp/locale/collate/tools/colcmp/Makefile deleted 100644 → 0
View file @ 01a5b66d
# Copyright 2012 The Go Authors. All rights reserved.
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file.
chars:
go run ../maketables.go -tables=chars -package=main > chars.go
gofmt -w -s chars.go
src/pkg/exp/locale/collate/tools/colcmp/chars.go deleted 100644 → 0
View file @ 01a5b66d
This diff is collapsed.
src/pkg/exp/locale/collate/tools/colcmp/col.go deleted 100644 → 0
View file @ 01a5b66d
This diff is collapsed.
src/pkg/exp/locale/collate/tools/colcmp/colcmp.go deleted 100644 → 0
View file @ 01a5b66d
This diff is collapsed.
src/pkg/exp/locale/collate/tools/colcmp/darwin.go deleted 100644 → 0
View file @ 01a5b66d
This diff is collapsed.
src/pkg/exp/locale/collate/tools/colcmp/gen.go deleted 100644 → 0
View file @ 01a5b66d
This diff is collapsed.
src/pkg/exp/locale/collate/tools/colcmp/icu.go deleted 100644 → 0
View file @ 01a5b66d
This diff is collapsed.
src/pkg/exp/ssa/blockopt.go deleted 100644 → 0
View file @ 01a5b66d
This diff is collapsed.
src/pkg/exp/ssa/builder.go deleted 100644 → 0
View file @ 01a5b66d
This diff is collapsed.
src/pkg/exp/ssa/doc.go deleted 100644 → 0
View file @ 01a5b66d
This diff is collapsed.
src/pkg/exp/ssa/dom.go deleted 100644 → 0
View file @ 01a5b66d
This diff is collapsed.
src/pkg/exp/ssa/emit.go deleted 100644 → 0
View file @ 01a5b66d
This diff is collapsed.
src/pkg/exp/ssa/func.go deleted 100644 → 0
View file @ 01a5b66d
This diff is collapsed.
src/pkg/exp/ssa/importer.go deleted 100644 → 0
View file @ 01a5b66d
This diff is collapsed.
src/pkg/exp/ssa/interp/external.go deleted 100644 → 0
View file @ 01a5b66d
This diff is collapsed.
src/pkg/exp/ssa/interp/external_plan9.go deleted 100644 → 0
View file @ 01a5b66d
This diff is collapsed.
src/pkg/exp/ssa/interp/external_unix.go deleted 100644 → 0
View file @ 01a5b66d
This diff is collapsed.
src/pkg/exp/ssa/interp/external_windows.go deleted 100644 → 0
View file @ 01a5b66d
This diff is collapsed.
src/pkg/exp/ssa/interp/interp.go deleted 100644 → 0
View file @ 01a5b66d
This diff is collapsed.
src/pkg/exp/ssa/interp/interp_test.go deleted 100644 → 0
View file @ 01a5b66d
This diff is collapsed.
src/pkg/exp/ssa/interp/map.go deleted 100644 → 0
View file @ 01a5b66d
This diff is collapsed.
src/pkg/exp/ssa/interp/ops.go deleted 100644 → 0
View file @ 01a5b66d
This diff is collapsed.
src/pkg/exp/ssa/interp/reflect.go deleted 100644 → 0
View file @ 01a5b66d
This diff is collapsed.
src/pkg/exp/ssa/interp/testdata/coverage.go deleted 100644 → 0
View file @ 01a5b66d
This diff is collapsed.
src/pkg/exp/ssa/interp/value.go deleted 100644 → 0
View file @ 01a5b66d
This diff is collapsed.
src/pkg/exp/ssa/lift.go deleted 100644 → 0
View file @ 01a5b66d
This diff is collapsed.
src/pkg/exp/ssa/literal.go deleted 100644 → 0
View file @ 01a5b66d
This diff is collapsed.
src/pkg/exp/ssa/lvalue.go deleted 100644 → 0
View file @ 01a5b66d
This diff is collapsed.
src/pkg/exp/ssa/print.go deleted 100644 → 0
View file @ 01a5b66d
This diff is collapsed.
src/pkg/exp/ssa/promote.go deleted 100644 → 0
View file @ 01a5b66d
This diff is collapsed.
src/pkg/exp/ssa/sanity.go deleted 100644 → 0
View file @ 01a5b66d
This diff is collapsed.
src/pkg/exp/ssa/ssa.go deleted 100644 → 0
View file @ 01a5b66d
This diff is collapsed.
src/pkg/exp/ssa/ssadump.go deleted 100644 → 0
View file @ 01a5b66d
This diff is collapsed.
src/pkg/exp/ssa/util.go deleted 100644 → 0
View file @ 01a5b66d
This diff is collapsed.
src/pkg/exp/utf8string/string.go deleted 100644 → 0
View file @ 01a5b66d
This diff is collapsed.
src/pkg/exp/utf8string/string_test.go deleted 100644 → 0
View file @ 01a5b66d
This diff is collapsed.
src/pkg/exp/winfsnotify/winfsnotify.go deleted 100644 → 0
View file @ 01a5b66d
This diff is collapsed.
src/pkg/exp/winfsnotify/winfsnotify_test.go deleted 100644 → 0
View file @ 01a5b66d
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7