Commit 511484af authored by Russ Cox's avatar Russ Cox

cmd/go: add internal/note, internal/sumweb, internal/tlog from golang.org/x/exp/sumdb

Copied and updated import paths.
Eventually we will probably publish
these packages somewhere in golang.org/x
(as non-internal packages) and then we will
be able to vendor them properly.
For now, copy.

sumweb.globsMatchPath moved to str.GlobsMatchPath.

Change-Id: I4585e6dc5daa423e4ca9669195d41e58e7c8c275
Reviewed-on: https://go-review.googlesource.com/c/go/+/173950Reviewed-by: default avatarJay Conrod <jayconrod@google.com>
parent 3cafbeab
// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package note_test
import (
"fmt"
"io"
"os"
"cmd/go/internal/note"
)
func ExampleSign() {
skey := "PRIVATE+KEY+PeterNeumann+c74f20a3+AYEKFALVFGyNhPJEMzD1QIDr+Y7hfZx09iUvxdXHKDFz"
text := "If you think cryptography is the answer to your problem,\n" +
"then you don't know what your problem is.\n"
signer, err := note.NewSigner(skey)
if err != nil {
fmt.Println(err)
return
}
msg, err := note.Sign(&note.Note{Text: text}, signer)
if err != nil {
fmt.Println(err)
return
}
os.Stdout.Write(msg)
// Output:
// If you think cryptography is the answer to your problem,
// then you don't know what your problem is.
//
// — PeterNeumann x08go/ZJkuBS9UG/SffcvIAQxVBtiFupLLr8pAcElZInNIuGUgYN1FFYC2pZSNXgKvqfqdngotpRZb6KE6RyyBwJnAM=
}
func ExampleOpen() {
vkey := "PeterNeumann+c74f20a3+ARpc2QcUPDhMQegwxbzhKqiBfsVkmqq/LDE4izWy10TW"
msg := []byte("If you think cryptography is the answer to your problem,\n" +
"then you don't know what your problem is.\n" +
"\n" +
"— PeterNeumann x08go/ZJkuBS9UG/SffcvIAQxVBtiFupLLr8pAcElZInNIuGUgYN1FFYC2pZSNXgKvqfqdngotpRZb6KE6RyyBwJnAM=\n")
verifier, err := note.NewVerifier(vkey)
if err != nil {
fmt.Println(err)
return
}
verifiers := note.VerifierList(verifier)
n, err := note.Open(msg, verifiers)
if err != nil {
fmt.Println(err)
return
}
fmt.Printf("%s (%08x):\n%s", n.Sigs[0].Name, n.Sigs[0].Hash, n.Text)
// Output:
// PeterNeumann (c74f20a3):
// If you think cryptography is the answer to your problem,
// then you don't know what your problem is.
}
var rand = struct {
Reader io.Reader
}{
zeroReader{},
}
type zeroReader struct{}
func (zeroReader) Read(buf []byte) (int, error) {
for i := range buf {
buf[i] = 0
}
return len(buf), nil
}
func ExampleSign_add_signatures() {
vkey := "PeterNeumann+c74f20a3+ARpc2QcUPDhMQegwxbzhKqiBfsVkmqq/LDE4izWy10TW"
msg := []byte("If you think cryptography is the answer to your problem,\n" +
"then you don't know what your problem is.\n" +
"\n" +
"— PeterNeumann x08go/ZJkuBS9UG/SffcvIAQxVBtiFupLLr8pAcElZInNIuGUgYN1FFYC2pZSNXgKvqfqdngotpRZb6KE6RyyBwJnAM=\n")
verifier, err := note.NewVerifier(vkey)
if err != nil {
fmt.Println(err)
return
}
verifiers := note.VerifierList(verifier)
n, err := note.Open([]byte(msg), verifiers)
if err != nil {
fmt.Println(err)
return
}
skey, vkey, err := note.GenerateKey(rand.Reader, "EnochRoot")
if err != nil {
fmt.Println(err)
return
}
_ = vkey // give to verifiers
me, err := note.NewSigner(skey)
if err != nil {
fmt.Println(err)
return
}
msg, err = note.Sign(n, me)
if err != nil {
fmt.Println(err)
return
}
os.Stdout.Write(msg)
// Output:
// If you think cryptography is the answer to your problem,
// then you don't know what your problem is.
//
// — PeterNeumann x08go/ZJkuBS9UG/SffcvIAQxVBtiFupLLr8pAcElZInNIuGUgYN1FFYC2pZSNXgKvqfqdngotpRZb6KE6RyyBwJnAM=
// — EnochRoot rwz+eBzmZa0SO3NbfRGzPCpDckykFXSdeX+MNtCOXm2/5n2tiOHp+vAF1aGrQ5ovTG01oOTGwnWLox33WWd1RvMc+QQ=
}
This diff is collapsed.
This diff is collapsed.
......@@ -5,6 +5,7 @@
package str
import (
"path"
"path/filepath"
"strings"
)
......@@ -49,3 +50,47 @@ func HasFilePathPrefix(s, prefix string) bool {
return s[len(prefix)] == filepath.Separator && s[:len(prefix)] == prefix
}
}
// GlobsMatchPath reports whether any path prefix of target
// matches one of the glob patterns (as defined by path.Match)
// in the comma-separated globs list.
// It ignores any empty or malformed patterns in the list.
func GlobsMatchPath(globs, target string) bool {
for globs != "" {
// Extract next non-empty glob in comma-separated list.
var glob string
if i := strings.Index(globs, ","); i >= 0 {
glob, globs = globs[:i], globs[i+1:]
} else {
glob, globs = globs, ""
}
if glob == "" {
continue
}
// A glob with N+1 path elements (N slashes) needs to be matched
// against the first N+1 path elements of target,
// which end just before the N+1'th slash.
n := strings.Count(glob, "/")
prefix := target
// Walk target, counting slashes, truncating at the N+1'th slash.
for i := 0; i < len(target); i++ {
if target[i] == '/' {
if n == 0 {
prefix = target[:i]
break
}
n--
}
}
if n > 0 {
// Not enough prefix elements.
continue
}
matched, _ := path.Match(glob, prefix)
if matched {
return true
}
}
return false
}
// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Parallel cache.
// This file is copied from cmd/go/internal/par.
package sumweb
import (
"sync"
"sync/atomic"
)
// parCache runs an action once per key and caches the result.
type parCache struct {
m sync.Map
}
type cacheEntry struct {
done uint32
mu sync.Mutex
result interface{}
}
// Do calls the function f if and only if Do is being called for the first time with this key.
// No call to Do with a given key returns until the one call to f returns.
// Do returns the value returned by the one call to f.
func (c *parCache) Do(key interface{}, f func() interface{}) interface{} {
entryIface, ok := c.m.Load(key)
if !ok {
entryIface, _ = c.m.LoadOrStore(key, new(cacheEntry))
}
e := entryIface.(*cacheEntry)
if atomic.LoadUint32(&e.done) == 0 {
e.mu.Lock()
if atomic.LoadUint32(&e.done) == 0 {
e.result = f()
atomic.StoreUint32(&e.done, 1)
}
e.mu.Unlock()
}
return e.result
}
// Get returns the cached result associated with key.
// It returns nil if there is no such result.
// If the result for key is being computed, Get does not wait for the computation to finish.
func (c *parCache) Get(key interface{}) interface{} {
entryIface, ok := c.m.Load(key)
if !ok {
return nil
}
e := entryIface.(*cacheEntry)
if atomic.LoadUint32(&e.done) == 0 {
return nil
}
return e.result
}
This diff is collapsed.
This diff is collapsed.
// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// FS-safe encoding of module paths and versions.
// Copied from cmd/go/internal/module and unexported.
package sumweb
import (
"fmt"
"unicode/utf8"
)
// Safe encodings
//
// Module paths appear as substrings of file system paths
// (in the download cache) and of web server URLs in the proxy protocol.
// In general we cannot rely on file systems to be case-sensitive,
// nor can we rely on web servers, since they read from file systems.
// That is, we cannot rely on the file system to keep rsc.io/QUOTE
// and rsc.io/quote separate. Windows and macOS don't.
// Instead, we must never require two different casings of a file path.
// Because we want the download cache to match the proxy protocol,
// and because we want the proxy protocol to be possible to serve
// from a tree of static files (which might be stored on a case-insensitive
// file system), the proxy protocol must never require two different casings
// of a URL path either.
//
// One possibility would be to make the safe encoding be the lowercase
// hexadecimal encoding of the actual path bytes. This would avoid ever
// needing different casings of a file path, but it would be fairly illegible
// to most programmers when those paths appeared in the file system
// (including in file paths in compiler errors and stack traces)
// in web server logs, and so on. Instead, we want a safe encoding that
// leaves most paths unaltered.
//
// The safe encoding is this:
// replace every uppercase letter with an exclamation mark
// followed by the letter's lowercase equivalent.
//
// For example,
// github.com/Azure/azure-sdk-for-go -> github.com/!azure/azure-sdk-for-go.
// github.com/GoogleCloudPlatform/cloudsql-proxy -> github.com/!google!cloud!platform/cloudsql-proxy
// github.com/Sirupsen/logrus -> github.com/!sirupsen/logrus.
//
// Import paths that avoid upper-case letters are left unchanged.
// Note that because import paths are ASCII-only and avoid various
// problematic punctuation (like : < and >), the safe encoding is also ASCII-only
// and avoids the same problematic punctuation.
//
// Import paths have never allowed exclamation marks, so there is no
// need to define how to encode a literal !.
//
// Although paths are disallowed from using Unicode (see pathOK above),
// the eventual plan is to allow Unicode letters as well, to assume that
// file systems and URLs are Unicode-safe (storing UTF-8), and apply
// the !-for-uppercase convention. Note however that not all runes that
// are different but case-fold equivalent are an upper/lower pair.
// For example, U+004B ('K'), U+006B ('k'), and U+212A ('K' for Kelvin)
// are considered to case-fold to each other. When we do add Unicode
// letters, we must not assume that upper/lower are the only case-equivalent pairs.
// Perhaps the Kelvin symbol would be disallowed entirely, for example.
// Or perhaps it would encode as "!!k", or perhaps as "(212A)".
//
// Also, it would be nice to allow Unicode marks as well as letters,
// but marks include combining marks, and then we must deal not
// only with case folding but also normalization: both U+00E9 ('é')
// and U+0065 U+0301 ('e' followed by combining acute accent)
// look the same on the page and are treated by some file systems
// as the same path. If we do allow Unicode marks in paths, there
// must be some kind of normalization to allow only one canonical
// encoding of any character used in an import path.
// encodePath returns the safe encoding of the given module path.
// It fails if the module path is invalid.
func encodePath(path string) (encoding string, err error) {
return encodeString(path)
}
// encodeVersion returns the safe encoding of the given module version.
// Versions are allowed to be in non-semver form but must be valid file names
// and not contain exclamation marks.
func encodeVersion(v string) (encoding string, err error) {
return encodeString(v)
}
func encodeString(s string) (encoding string, err error) {
haveUpper := false
for _, r := range s {
if r == '!' || r >= utf8.RuneSelf {
// This should be disallowed by CheckPath, but diagnose anyway.
// The correctness of the encoding loop below depends on it.
return "", fmt.Errorf("internal error: inconsistency in EncodePath")
}
if 'A' <= r && r <= 'Z' {
haveUpper = true
}
}
if !haveUpper {
return s, nil
}
var buf []byte
for _, r := range s {
if 'A' <= r && r <= 'Z' {
buf = append(buf, '!', byte(r+'a'-'A'))
} else {
buf = append(buf, byte(r))
}
}
return string(buf), nil
}
// decodePath returns the module path of the given safe encoding.
// It fails if the encoding is invalid or encodes an invalid path.
func decodePath(encoding string) (path string, err error) {
path, ok := decodeString(encoding)
if !ok {
return "", fmt.Errorf("invalid module path encoding %q", encoding)
}
return path, nil
}
// decodeVersion returns the version string for the given safe encoding.
// It fails if the encoding is invalid or encodes an invalid version.
// Versions are allowed to be in non-semver form but must be valid file names
// and not contain exclamation marks.
func decodeVersion(encoding string) (v string, err error) {
v, ok := decodeString(encoding)
if !ok {
return "", fmt.Errorf("invalid version encoding %q", encoding)
}
return v, nil
}
func decodeString(encoding string) (string, bool) {
var buf []byte
bang := false
for _, r := range encoding {
if r >= utf8.RuneSelf {
return "", false
}
if bang {
bang = false
if r < 'a' || 'z' < r {
return "", false
}
buf = append(buf, byte(r+'A'-'a'))
continue
}
if r == '!' {
bang = true
continue
}
if 'A' <= r && r <= 'Z' {
return "", false
}
buf = append(buf, byte(r))
}
if bang {
return "", false
}
return string(buf), true
}
// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package sumweb
import "testing"
var encodeTests = []struct {
path string
enc string // empty means same as path
}{
{path: "ascii.com/abcdefghijklmnopqrstuvwxyz.-+/~_0123456789"},
{path: "github.com/GoogleCloudPlatform/omega", enc: "github.com/!google!cloud!platform/omega"},
}
func TestEncodePath(t *testing.T) {
// Check encodings.
for _, tt := range encodeTests {
enc, err := encodePath(tt.path)
if err != nil {
t.Errorf("encodePath(%q): unexpected error: %v", tt.path, err)
continue
}
want := tt.enc
if want == "" {
want = tt.path
}
if enc != want {
t.Errorf("encodePath(%q) = %q, want %q", tt.path, enc, want)
}
}
}
var badDecode = []string{
"github.com/GoogleCloudPlatform/omega",
"github.com/!google!cloud!platform!/omega",
"github.com/!0google!cloud!platform/omega",
"github.com/!_google!cloud!platform/omega",
"github.com/!!google!cloud!platform/omega",
}
func TestDecodePath(t *testing.T) {
// Check invalid decodings.
for _, bad := range badDecode {
_, err := decodePath(bad)
if err == nil {
t.Errorf("DecodePath(%q): succeeded, want error (invalid decoding)", bad)
}
}
// Check encodings.
for _, tt := range encodeTests {
enc := tt.enc
if enc == "" {
enc = tt.path
}
path, err := decodePath(enc)
if err != nil {
t.Errorf("decodePath(%q): unexpected error: %v", enc, err)
continue
}
if path != tt.path {
t.Errorf("decodePath(%q) = %q, want %q", enc, path, tt.path)
}
}
}
// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package sumweb implements the HTTP protocols for serving or accessing a go.sum database.
package sumweb
import (
"context"
"net/http"
"os"
"regexp"
"strings"
"cmd/go/internal/tlog"
)
// A Server provides the external operations
// (underlying database access and so on)
// needed to implement the HTTP server Handler.
type Server interface {
// NewContext returns the context to use for the request r.
NewContext(r *http.Request) (context.Context, error)
// Signed returns the signed hash of the latest tree.
Signed(ctx context.Context) ([]byte, error)
// ReadRecords returns the content for the n records id through id+n-1.
ReadRecords(ctx context.Context, id, n int64) ([][]byte, error)
// Lookup looks up a record by its associated key ("module@version"),
// returning the record ID.
Lookup(ctx context.Context, key string) (int64, error)
// ReadTileData reads the content of tile t.
// It is only invoked for hash tiles (t.L ≥ 0).
ReadTileData(ctx context.Context, t tlog.Tile) ([]byte, error)
}
// A Handler is the go.sum database server handler,
// which should be invoked to serve the paths listed in Paths.
// The calling code is responsible for initializing Server.
type Handler struct {
Server Server
}
// Paths are the URL paths for which Handler should be invoked.
//
// Typically a server will do:
//
// handler := &sumweb.Handler{Server: srv}
// for _, path := range sumweb.Paths {
// http.HandleFunc(path, handler)
// }
//
var Paths = []string{
"/lookup/",
"/latest",
"/tile/",
}
var modVerRE = regexp.MustCompile(`^[^@]+@v[0-9]+\.[0-9]+\.[0-9]+(-[^@]*)?$`)
func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
ctx, err := h.Server.NewContext(r)
if err != nil {
http.Error(w, err.Error(), 500)
return
}
switch {
default:
http.NotFound(w, r)
case strings.HasPrefix(r.URL.Path, "/lookup/"):
mod := strings.TrimPrefix(r.URL.Path, "/lookup/")
if !modVerRE.MatchString(mod) {
http.Error(w, "invalid module@version syntax", http.StatusBadRequest)
return
}
i := strings.Index(mod, "@")
encPath, encVers := mod[:i], mod[i+1:]
path, err := decodePath(encPath)
if err != nil {
reportError(w, r, err)
return
}
vers, err := decodeVersion(encVers)
if err != nil {
reportError(w, r, err)
return
}
id, err := h.Server.Lookup(ctx, path+"@"+vers)
if err != nil {
reportError(w, r, err)
return
}
records, err := h.Server.ReadRecords(ctx, id, 1)
if err != nil {
// This should never happen - the lookup says the record exists.
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
if len(records) != 1 {
http.Error(w, "invalid record count returned by ReadRecords", http.StatusInternalServerError)
return
}
msg, err := tlog.FormatRecord(id, records[0])
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
signed, err := h.Server.Signed(ctx)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "text/plain; charset=UTF-8")
w.Write(msg)
w.Write(signed)
case r.URL.Path == "/latest":
data, err := h.Server.Signed(ctx)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "text/plain; charset=UTF-8")
w.Write(data)
case strings.HasPrefix(r.URL.Path, "/tile/"):
t, err := tlog.ParseTilePath(r.URL.Path[1:])
if err != nil {
http.Error(w, "invalid tile syntax", http.StatusBadRequest)
return
}
if t.L == -1 {
// Record data.
start := t.N << uint(t.H)
records, err := h.Server.ReadRecords(ctx, start, int64(t.W))
if err != nil {
reportError(w, r, err)
return
}
if len(records) != t.W {
http.Error(w, "invalid record count returned by ReadRecords", http.StatusInternalServerError)
return
}
var data []byte
for i, text := range records {
msg, err := tlog.FormatRecord(start+int64(i), text)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
}
data = append(data, msg...)
}
w.Header().Set("Content-Type", "text/plain; charset=UTF-8")
w.Write(data)
return
}
data, err := h.Server.ReadTileData(ctx, t)
if err != nil {
reportError(w, r, err)
return
}
w.Header().Set("Content-Type", "application/octet-stream")
w.Write(data)
}
}
// reportError reports err to w.
// If it's a not-found, the reported error is 404.
// Otherwise it is an internal server error.
// The caller must only call reportError in contexts where
// a not-found err should be reported as 404.
func reportError(w http.ResponseWriter, r *http.Request, err error) {
if os.IsNotExist(err) {
http.Error(w, err.Error(), http.StatusNotFound)
return
}
http.Error(w, err.Error(), http.StatusInternalServerError)
}
// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package sumweb
import (
"context"
"fmt"
"net/http"
"strings"
"sync"
"cmd/go/internal/note"
"cmd/go/internal/tlog"
)
// NewTestServer constructs a new TestServer
// that will sign its tree with the given signer key
// (see cmd/go/internal/note)
// and fetch new records as needed by calling gosum.
func NewTestServer(signer string, gosum func(path, vers string) ([]byte, error)) *TestServer {
return &TestServer{signer: signer, gosum: gosum}
}
// A TestServer is an in-memory implementation of Server for testing.
type TestServer struct {
signer string
gosum func(path, vers string) ([]byte, error)
mu sync.Mutex
hashes testHashes
records [][]byte
lookup map[string]int64
}
// testHashes implements tlog.HashReader, reading from a slice.
type testHashes []tlog.Hash
func (h testHashes) ReadHashes(indexes []int64) ([]tlog.Hash, error) {
var list []tlog.Hash
for _, id := range indexes {
list = append(list, h[id])
}
return list, nil
}
func (s *TestServer) NewContext(r *http.Request) (context.Context, error) {
return nil, nil
}
func (s *TestServer) Signed(ctx context.Context) ([]byte, error) {
s.mu.Lock()
defer s.mu.Unlock()
size := int64(len(s.records))
h, err := tlog.TreeHash(size, s.hashes)
if err != nil {
return nil, err
}
text := tlog.FormatTree(tlog.Tree{N: size, Hash: h})
signer, err := note.NewSigner(s.signer)
if err != nil {
return nil, err
}
return note.Sign(&note.Note{Text: string(text)}, signer)
}
func (s *TestServer) ReadRecords(ctx context.Context, id, n int64) ([][]byte, error) {
s.mu.Lock()
defer s.mu.Unlock()
var list [][]byte
for i := int64(0); i < n; i++ {
if id+i >= int64(len(s.records)) {
return nil, fmt.Errorf("missing records")
}
list = append(list, s.records[id+i])
}
return list, nil
}
func (s *TestServer) Lookup(ctx context.Context, key string) (int64, error) {
s.mu.Lock()
id, ok := s.lookup[key]
s.mu.Unlock()
if ok {
return id, nil
}
// Look up module and compute go.sum lines.
i := strings.Index(key, "@")
if i < 0 {
return 0, fmt.Errorf("invalid lookup key %q", key)
}
path, vers := key[:i], key[i+1:]
data, err := s.gosum(path, vers)
if err != nil {
return 0, err
}
s.mu.Lock()
defer s.mu.Unlock()
// We ran the fetch without the lock.
// If another fetch happened and committed, use it instead.
id, ok = s.lookup[key]
if ok {
return id, nil
}
// Add record.
id = int64(len(s.records))
s.records = append(s.records, data)
if s.lookup == nil {
s.lookup = make(map[string]int64)
}
s.lookup[key] = id
hashes, err := tlog.StoredHashesForRecordHash(id, tlog.RecordHash([]byte(data)), s.hashes)
if err != nil {
panic(err)
}
s.hashes = append(s.hashes, hashes...)
return id, nil
}
func (s *TestServer) ReadTileData(ctx context.Context, t tlog.Tile) ([]byte, error) {
s.mu.Lock()
defer s.mu.Unlock()
return tlog.ReadTileData(t, s.hashes)
}
// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package tlog
import (
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"os"
"testing"
)
func TestCertificateTransparency(t *testing.T) {
// Test that we can verify actual Certificate Transparency proofs.
// (The other tests check that we can verify our own proofs;
// this is a test that the two are compatible.)
if testing.Short() {
t.Skip("skipping in -short mode")
}
var root ctTree
httpGET(t, "http://ct.googleapis.com/logs/argon2020/ct/v1/get-sth", &root)
var leaf ctEntries
httpGET(t, "http://ct.googleapis.com/logs/argon2020/ct/v1/get-entries?start=10000&end=10000", &leaf)
hash := RecordHash(leaf.Entries[0].Data)
var rp ctRecordProof
httpGET(t, "http://ct.googleapis.com/logs/argon2020/ct/v1/get-proof-by-hash?tree_size="+fmt.Sprint(root.Size)+"&hash="+url.QueryEscape(hash.String()), &rp)
err := CheckRecord(rp.Proof, root.Size, root.Hash, 10000, hash)
if err != nil {
t.Fatal(err)
}
var tp ctTreeProof
httpGET(t, "http://ct.googleapis.com/logs/argon2020/ct/v1/get-sth-consistency?first=3654490&second="+fmt.Sprint(root.Size), &tp)
oh, _ := ParseHash("AuIZ5V6sDUj1vn3Y1K85oOaQ7y+FJJKtyRTl1edIKBQ=")
err = CheckTree(tp.Proof, root.Size, root.Hash, 3654490, oh)
if err != nil {
t.Fatal(err)
}
}
type ctTree struct {
Size int64 `json:"tree_size"`
Hash Hash `json:"sha256_root_hash"`
}
type ctEntries struct {
Entries []*ctEntry
}
type ctEntry struct {
Data []byte `json:"leaf_input"`
}
type ctRecordProof struct {
Index int64 `json:"leaf_index"`
Proof RecordProof `json:"audit_path"`
}
type ctTreeProof struct {
Proof TreeProof `json:"consistency"`
}
func httpGET(t *testing.T, url string, targ interface{}) {
if testing.Verbose() {
println()
println(url)
}
resp, err := http.Get(url)
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
data, err := ioutil.ReadAll(resp.Body)
if err != nil {
t.Fatal(err)
}
if testing.Verbose() {
os.Stdout.Write(data)
}
err = json.Unmarshal(data, targ)
if err != nil {
println(url)
os.Stdout.Write(data)
t.Fatal(err)
}
}
// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package tlog
import (
"bytes"
"encoding/base64"
"errors"
"fmt"
"strconv"
"strings"
"unicode/utf8"
)
// A Tree is a tree description, to be signed by a go.sum database server.
type Tree struct {
N int64
Hash Hash
}
// FormatTree formats a tree description for inclusion in a note.
//
// The encoded form is three lines, each ending in a newline (U+000A):
//
// go.sum database tree
// N
// Hash
//
// where N is in decimal and Hash is in base64.
//
// A future backwards-compatible encoding may add additional lines,
// which the parser can ignore.
// A future backwards-incompatible encoding would use a different
// first line (for example, "go.sum database tree v2").
func FormatTree(tree Tree) []byte {
return []byte(fmt.Sprintf("go.sum database tree\n%d\n%s\n", tree.N, tree.Hash))
}
var errMalformedTree = errors.New("malformed tree note")
var treePrefix = []byte("go.sum database tree\n")
// ParseTree parses a tree root description.
func ParseTree(text []byte) (tree Tree, err error) {
// The message looks like:
//
// go.sum database tree
// 2
// nND/nri/U0xuHUrYSy0HtMeal2vzD9V4k/BO79C+QeI=
//
// For forwards compatibility, extra text lines after the encoding are ignored.
if !bytes.HasPrefix(text, treePrefix) || bytes.Count(text, []byte("\n")) < 3 || len(text) > 1e6 {
return Tree{}, errMalformedTree
}
lines := strings.SplitN(string(text), "\n", 4)
n, err := strconv.ParseInt(lines[1], 10, 64)
if err != nil || n < 0 || lines[1] != strconv.FormatInt(n, 10) {
return Tree{}, errMalformedTree
}
h, err := base64.StdEncoding.DecodeString(lines[2])
if err != nil || len(h) != HashSize {
return Tree{}, errMalformedTree
}
var hash Hash
copy(hash[:], h)
return Tree{n, hash}, nil
}
var errMalformedRecord = errors.New("malformed record data")
// FormatRecord formats a record for serving to a client
// in a lookup response or data tile.
//
// The encoded form is the record ID as a single number,
// then the text of the record, and then a terminating blank line.
// Record text must be valid UTF-8 and must not contain any ASCII control
// characters (those below U+0020) other than newline (U+000A).
// It must end in a terminating newline and not contain any blank lines.
func FormatRecord(id int64, text []byte) (msg []byte, err error) {
if !isValidRecordText(text) {
return nil, errMalformedRecord
}
msg = []byte(fmt.Sprintf("%d\n", id))
msg = append(msg, text...)
msg = append(msg, '\n')
return msg, nil
}
// isValidRecordText reports whether text is syntactically valid record text.
func isValidRecordText(text []byte) bool {
var last rune
for i := 0; i < len(text); {
r, size := utf8.DecodeRune(text[i:])
if r < 0x20 && r != '\n' || r == utf8.RuneError && size == 1 || last == '\n' && r == '\n' {
return false
}
i += size
last = r
}
if last != '\n' {
return false
}
return true
}
// ParseRecord parses a record description at the start of text,
// stopping immediately after the terminating blank line.
// It returns the record id, the record text, and the remainder of text.
func ParseRecord(msg []byte) (id int64, text, rest []byte, err error) {
// Leading record id.
i := bytes.IndexByte(msg, '\n')
if i < 0 {
return 0, nil, nil, errMalformedRecord
}
id, err = strconv.ParseInt(string(msg[:i]), 10, 64)
if err != nil {
return 0, nil, nil, errMalformedRecord
}
msg = msg[i+1:]
// Record text.
i = bytes.Index(msg, []byte("\n\n"))
if i < 0 {
return 0, nil, nil, errMalformedRecord
}
text, rest = msg[:i+1], msg[i+2:]
if !isValidRecordText(text) {
return 0, nil, nil, errMalformedRecord
}
return id, text, rest, nil
}
// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package tlog
import (
"strings"
"testing"
)
func TestFormatTree(t *testing.T) {
n := int64(123456789012)
h := RecordHash([]byte("hello world"))
golden := "go.sum database tree\n123456789012\nTszzRgjTG6xce+z2AG31kAXYKBgQVtCSCE40HmuwBb0=\n"
b := FormatTree(Tree{n, h})
if string(b) != golden {
t.Errorf("FormatTree(...) = %q, want %q", b, golden)
}
}
func TestParseTree(t *testing.T) {
in := "go.sum database tree\n123456789012\nTszzRgjTG6xce+z2AG31kAXYKBgQVtCSCE40HmuwBb0=\n"
goldH := RecordHash([]byte("hello world"))
goldN := int64(123456789012)
tree, err := ParseTree([]byte(in))
if tree.N != goldN || tree.Hash != goldH || err != nil {
t.Fatalf("ParseTree(...) = Tree{%d, %v}, %v, want Tree{%d, %v}, nil", tree.N, tree.Hash, err, goldN, goldH)
}
// Check invalid trees.
var badTrees = []string{
"not-" + in,
"go.sum database tree\n0xabcdef\nTszzRgjTG6xce+z2AG31kAXYKBgQVtCSCE40HmuwBb0=\n",
"go.sum database tree\n123456789012\nTszzRgjTG6xce+z2AG31kAXYKBgQVtCSCE40HmuwBTOOBIG=\n",
}
for _, bad := range badTrees {
_, err := ParseTree([]byte(bad))
if err == nil {
t.Fatalf("ParseTree(%q) succeeded, want failure", in)
}
}
// Check junk on end is ignored.
var goodTrees = []string{
in + "JOE",
in + "JOE\n",
in + strings.Repeat("JOE\n", 1000),
}
for _, good := range goodTrees {
_, err := ParseTree([]byte(good))
if tree.N != goldN || tree.Hash != goldH || err != nil {
t.Fatalf("ParseTree(...+%q) = Tree{%d, %v}, %v, want Tree{%d, %v}, nil", good[len(in):], tree.N, tree.Hash, err, goldN, goldH)
}
}
}
func TestFormatRecord(t *testing.T) {
id := int64(123456789012)
text := "hello, world\n"
golden := "123456789012\nhello, world\n\n"
msg, err := FormatRecord(id, []byte(text))
if err != nil {
t.Fatalf("FormatRecord: %v", err)
}
if string(msg) != golden {
t.Fatalf("FormatRecord(...) = %q, want %q", msg, golden)
}
var badTexts = []string{
"",
"hello\nworld",
"hello\n\nworld\n",
"hello\x01world\n",
}
for _, bad := range badTexts {
msg, err := FormatRecord(id, []byte(bad))
if err == nil {
t.Errorf("FormatRecord(id, %q) = %q, want error", bad, msg)
}
}
}
func TestParseRecord(t *testing.T) {
in := "123456789012\nhello, world\n\njunk on end\x01\xff"
goldID := int64(123456789012)
goldText := "hello, world\n"
goldRest := "junk on end\x01\xff"
id, text, rest, err := ParseRecord([]byte(in))
if id != goldID || string(text) != goldText || string(rest) != goldRest || err != nil {
t.Fatalf("ParseRecord(%q) = %d, %q, %q, %v, want %d, %q, %q, nil", in, id, text, rest, err, goldID, goldText, goldRest)
}
in = "123456789012\nhello, world\n\n"
id, text, rest, err = ParseRecord([]byte(in))
if id != goldID || string(text) != goldText || len(rest) != 0 || err != nil {
t.Fatalf("ParseRecord(%q) = %d, %q, %q, %v, want %d, %q, %q, nil", in, id, text, rest, err, goldID, goldText, "")
}
if rest == nil {
t.Fatalf("ParseRecord(%q): rest = []byte(nil), want []byte{}", in)
}
// Check invalid records.
var badRecords = []string{
"not-" + in,
"123\nhello\x01world\n\n",
"123\nhello\xffworld\n\n",
"123\nhello world\n",
"0x123\nhello world\n\n",
}
for _, bad := range badRecords {
id, text, rest, err := ParseRecord([]byte(bad))
if err == nil {
t.Fatalf("ParseRecord(%q) = %d, %q, %q, nil, want error", in, id, text, rest)
}
}
}
This diff is collapsed.
This diff is collapsed.
// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package tlog
import (
"bytes"
"fmt"
"testing"
)
type testHashStorage []Hash
func (t testHashStorage) ReadHash(level int, n int64) (Hash, error) {
return t[StoredHashIndex(level, n)], nil
}
func (t testHashStorage) ReadHashes(index []int64) ([]Hash, error) {
// It's not required by HashReader that indexes be in increasing order,
// but check that the functions we are testing only ever ask for
// indexes in increasing order.
for i := 1; i < len(index); i++ {
if index[i-1] >= index[i] {
panic("indexes out of order")
}
}
out := make([]Hash, len(index))
for i, x := range index {
out[i] = t[x]
}
return out, nil
}
type testTilesStorage struct {
unsaved int
m map[Tile][]byte
}
func (t testTilesStorage) Height() int {
return 2
}
func (t *testTilesStorage) SaveTiles(tiles []Tile, data [][]byte) {
t.unsaved -= len(tiles)
}
func (t *testTilesStorage) ReadTiles(tiles []Tile) ([][]byte, error) {
out := make([][]byte, len(tiles))
for i, tile := range tiles {
out[i] = t.m[tile]
}
t.unsaved += len(tiles)
return out, nil
}
func TestTree(t *testing.T) {
var trees []Hash
var leafhashes []Hash
var storage testHashStorage
tiles := make(map[Tile][]byte)
const testH = 2
for i := int64(0); i < 100; i++ {
data := []byte(fmt.Sprintf("leaf %d", i))
hashes, err := StoredHashes(i, data, storage)
if err != nil {
t.Fatal(err)
}
leafhashes = append(leafhashes, RecordHash(data))
oldStorage := len(storage)
storage = append(storage, hashes...)
if count := StoredHashCount(i + 1); count != int64(len(storage)) {
t.Errorf("StoredHashCount(%d) = %d, have %d StoredHashes", i+1, count, len(storage))
}
th, err := TreeHash(i+1, storage)
if err != nil {
t.Fatal(err)
}
for _, tile := range NewTiles(testH, i, i+1) {
data, err := ReadTileData(tile, storage)
if err != nil {
t.Fatal(err)
}
old := Tile{H: tile.H, L: tile.L, N: tile.N, W: tile.W - 1}
oldData := tiles[old]
if len(oldData) != len(data)-HashSize || !bytes.Equal(oldData, data[:len(oldData)]) {
t.Fatalf("tile %v not extending earlier tile %v", tile.Path(), old.Path())
}
tiles[tile] = data
}
for _, tile := range NewTiles(testH, 0, i+1) {
data, err := ReadTileData(tile, storage)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(tiles[tile], data) {
t.Fatalf("mismatch at %+v", tile)
}
}
for _, tile := range NewTiles(testH, i/2, i+1) {
data, err := ReadTileData(tile, storage)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(tiles[tile], data) {
t.Fatalf("mismatch at %+v", tile)
}
}
// Check that all the new hashes are readable from their tiles.
for j := oldStorage; j < len(storage); j++ {
tile := TileForIndex(testH, int64(j))
data, ok := tiles[tile]
if !ok {
t.Log(NewTiles(testH, 0, i+1))
t.Fatalf("TileForIndex(%d, %d) = %v, not yet stored (i=%d, stored %d)", testH, j, tile.Path(), i, len(storage))
continue
}
h, err := HashFromTile(tile, data, int64(j))
if err != nil {
t.Fatal(err)
}
if h != storage[j] {
t.Errorf("HashFromTile(%v, %d) = %v, want %v", tile.Path(), int64(j), h, storage[j])
}
}
trees = append(trees, th)
// Check that leaf proofs work, for all trees and leaves so far.
for j := int64(0); j <= i; j++ {
p, err := ProveRecord(i+1, j, storage)
if err != nil {
t.Fatalf("ProveRecord(%d, %d): %v", i+1, j, err)
}
if err := CheckRecord(p, i+1, th, j, leafhashes[j]); err != nil {
t.Fatalf("CheckRecord(%d, %d): %v", i+1, j, err)
}
for k := range p {
p[k][0] ^= 1
if err := CheckRecord(p, i+1, th, j, leafhashes[j]); err == nil {
t.Fatalf("CheckRecord(%d, %d) succeeded with corrupt proof hash #%d!", i+1, j, k)
}
p[k][0] ^= 1
}
}
// Check that leaf proofs work using TileReader.
// To prove a leaf that way, all you have to do is read and verify its hash.
storage := &testTilesStorage{m: tiles}
thr := TileHashReader(Tree{i + 1, th}, storage)
for j := int64(0); j <= i; j++ {
h, err := thr.ReadHashes([]int64{StoredHashIndex(0, j)})
if err != nil {
t.Fatalf("TileHashReader(%d).ReadHashes(%d): %v", i+1, j, err)
}
if h[0] != leafhashes[j] {
t.Fatalf("TileHashReader(%d).ReadHashes(%d) returned wrong hash", i+1, j)
}
// Even though reading the hash suffices,
// check we can generate the proof too.
p, err := ProveRecord(i+1, j, thr)
if err != nil {
t.Fatalf("ProveRecord(%d, %d, TileHashReader(%d)): %v", i+1, j, i+1, err)
}
if err := CheckRecord(p, i+1, th, j, leafhashes[j]); err != nil {
t.Fatalf("CheckRecord(%d, %d, TileHashReader(%d)): %v", i+1, j, i+1, err)
}
}
if storage.unsaved != 0 {
t.Fatalf("TileHashReader(%d) did not save %d tiles", i+1, storage.unsaved)
}
// Check that ReadHashes will give an error if the index is not in the tree.
if _, err := thr.ReadHashes([]int64{(i + 1) * 2}); err == nil {
t.Fatalf("TileHashReader(%d).ReadHashes(%d) for index not in tree <nil>, want err", i, i+1)
}
if storage.unsaved != 0 {
t.Fatalf("TileHashReader(%d) did not save %d tiles", i+1, storage.unsaved)
}
// Check that tree proofs work, for all trees so far, using TileReader.
// To prove a tree that way, all you have to do is compute and verify its hash.
for j := int64(0); j <= i; j++ {
h, err := TreeHash(j+1, thr)
if err != nil {
t.Fatalf("TreeHash(%d, TileHashReader(%d)): %v", j, i+1, err)
}
if h != trees[j] {
t.Fatalf("TreeHash(%d, TileHashReader(%d)) = %x, want %x (%v)", j, i+1, h[:], trees[j][:], trees[j])
}
// Even though computing the subtree hash suffices,
// check that we can generate the proof too.
p, err := ProveTree(i+1, j+1, thr)
if err != nil {
t.Fatalf("ProveTree(%d, %d): %v", i+1, j+1, err)
}
if err := CheckTree(p, i+1, th, j+1, trees[j]); err != nil {
t.Fatalf("CheckTree(%d, %d): %v [%v]", i+1, j+1, err, p)
}
for k := range p {
p[k][0] ^= 1
if err := CheckTree(p, i+1, th, j+1, trees[j]); err == nil {
t.Fatalf("CheckTree(%d, %d) succeeded with corrupt proof hash #%d!", i+1, j+1, k)
}
p[k][0] ^= 1
}
}
if storage.unsaved != 0 {
t.Fatalf("TileHashReader(%d) did not save %d tiles", i+1, storage.unsaved)
}
}
}
func TestSplitStoredHashIndex(t *testing.T) {
for l := 0; l < 10; l++ {
for n := int64(0); n < 100; n++ {
x := StoredHashIndex(l, n)
l1, n1 := SplitStoredHashIndex(x)
if l1 != l || n1 != n {
t.Fatalf("StoredHashIndex(%d, %d) = %d, but SplitStoredHashIndex(%d) = %d, %d", l, n, x, x, l1, n1)
}
}
}
}
// TODO(rsc): Test invalid paths too, like "tile/3/5/123/456/078".
var tilePaths = []struct {
path string
tile Tile
}{
{"tile/4/0/001", Tile{4, 0, 1, 16}},
{"tile/4/0/001.p/5", Tile{4, 0, 1, 5}},
{"tile/3/5/x123/x456/078", Tile{3, 5, 123456078, 8}},
{"tile/3/5/x123/x456/078.p/2", Tile{3, 5, 123456078, 2}},
{"tile/1/0/x003/x057/500", Tile{1, 0, 3057500, 2}},
{"tile/3/5/123/456/078", Tile{}},
{"tile/3/-1/123/456/078", Tile{}},
{"tile/1/data/x003/x057/500", Tile{1, -1, 3057500, 2}},
}
func TestTilePath(t *testing.T) {
for _, tt := range tilePaths {
if tt.tile.H > 0 {
p := tt.tile.Path()
if p != tt.path {
t.Errorf("%+v.Path() = %q, want %q", tt.tile, p, tt.path)
}
}
tile, err := ParseTilePath(tt.path)
if err != nil {
if tt.tile.H == 0 {
// Expected error.
continue
}
t.Errorf("ParseTilePath(%q): %v", tt.path, err)
} else if tile != tt.tile {
if tt.tile.H == 0 {
t.Errorf("ParseTilePath(%q): expected error, got %+v", tt.path, tt.tile)
continue
}
t.Errorf("ParseTilePath(%q) = %+v, want %+v", tt.path, tile, tt.tile)
}
}
}
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment