godoc: link identifiers to declarations

The changes are almost completely self-contained
in the new file linkify.go. The other changes are
minimal and should not disturb the currently
working godoc, in anticipation of Go 1.1.

To disable the feature in case of problems, set
-links=false.

Fixes #2063.

R=adg, r
CC=golang-dev
https://golang.org/cl/7883044

src/cmd/godoc/format.go

@@ -226,10 +226,10 @@ func lineSelection(text []byte) Selection {
 	}
 }
 
-// commentSelection returns the sequence of consecutive comments
-// in the Go src text as a Selection.
+// tokenSelection returns, as a selection, the sequence of
+// consecutive occurrences of token sel in the Go src text.
 //
-func commentSelection(src []byte) Selection {
+func tokenSelection(src []byte, sel token.Token) Selection {
 	var s scanner.Scanner
 	fset := token.NewFileSet()
 	file := fset.AddFile("", fset.Base(), len(src))
@@ -241,7 +241,7 @@ func commentSelection(src []byte) Selection {
 				break
 			}
 			offs := file.Offset(pos)
-			if tok == token.COMMENT {
+			if tok == sel {
 				seg = []int{offs, offs + len(lit)}
 				break
 			}
@@ -338,7 +338,7 @@ func selectionTag(w io.Writer, text []byte, selections int) {
 func FormatText(w io.Writer, text []byte, line int, goSource bool, pattern string, selection Selection) {
 	var comments, highlights Selection
 	if goSource {
-		comments = commentSelection(text)
+		comments = tokenSelection(text, token.COMMENT)
 	}
 	if pattern != "" {
 		highlights = regexpSelection(text, pattern)

src/cmd/godoc/godoc.go

@@ -66,6 +66,7 @@ var (
 	templateDir    = flag.String("templates", "", "directory containing alternate template files")
 	showPlayground = flag.Bool("play", false, "enable playground in web interface")
 	showExamples   = flag.Bool("ex", false, "show examples in command line mode")
+	declLinks      = flag.Bool("links", true, "link identifiers to their declarations")
 
 	// search index
 	indexEnabled = flag.Bool("index", false, "enable search index")
@@ -281,8 +282,21 @@ func nodeFunc(node interface{}, fset *token.FileSet) string {
 func node_htmlFunc(node interface{}, fset *token.FileSet) string {
 	var buf1 bytes.Buffer
 	writeNode(&buf1, fset, node)
+
 	var buf2 bytes.Buffer
-	FormatText(&buf2, buf1.Bytes(), -1, true, "", nil)
+	// BUG(gri):  When showing full source text (?m=src),
+	//            identifier links are incorrect.
+	// TODO(gri): Only linkify exported code snippets, not the
+	//            full source text: identifier resolution is
+	//            not sufficiently strong w/o type checking.
+	//            Need to check if info.PAst != nil - requires
+	//            to pass *PageInfo around instead of fset.
+	if n, _ := node.(ast.Node); n != nil && *declLinks {
+		LinkifyText(&buf2, buf1.Bytes(), n)
+	} else {
+		FormatText(&buf2, buf1.Bytes(), -1, true, "", nil)
+	}
+
 	return buf2.String()
 }
 
@@ -521,7 +535,7 @@ var fmap = template.FuncMap{
 	"filename": filenameFunc,
 	"repeat":   strings.Repeat,
 
-	// accss to FileInfos (directory listings)
+	// access to FileInfos (directory listings)
 	"fileInfoName": fileInfoNameFunc,
 	"fileInfoTime": fileInfoTimeFunc,
 
@@ -1020,6 +1034,23 @@ func collectExamples(pkg *ast.Package, testfiles map[string]*ast.File) []*doc.Ex
 	return examples
 }
 
+// poorMansImporter returns a (dummy) package object named
+// by the last path component of the provided package path
+// (as is the convention for packages). This is sufficient
+// to resolve package identifiers without doing an actual
+// import. It never returns an error.
+//
+func poorMansImporter(imports map[string]*ast.Object, path string) (*ast.Object, error) {
+	pkg := imports[path]
+	if pkg == nil {
+		// note that strings.LastIndex returns -1 if there is no "/"
+		pkg = ast.NewObj(ast.Pkg, path[strings.LastIndex(path, "/")+1:])
+		pkg.Data = ast.NewScope(nil) // required by ast.NewPackage for dot-import
+		imports[path] = pkg
+	}
+	return pkg, nil
+}
+
 // getPageInfo returns the PageInfo for a package directory abspath. If the
 // parameter genAST is set, an AST containing only the package exports is
 // computed (PageInfo.PAst), otherwise package documentation (PageInfo.Doc)
@@ -1071,7 +1102,9 @@ func (h *docServer) getPageInfo(abspath, relpath string, mode PageInfoMode) (inf
 			info.Err = err
 			return
 		}
-		pkg := &ast.Package{Name: pkgname, Files: files}
+
+		// ignore any errors - they are due to unresolved identifiers
+		pkg, _ := ast.NewPackage(fset, files, poorMansImporter, nil)
 
 		// extract package documentation
 		info.FSet = fset

src/cmd/godoc/linkify.go

+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This file implements LinkifyText which introduces
+// links for identifiers pointing to their declarations.
+// The approach does not cover all cases because godoc
+// doesn't have complete type information, but it's
+// reasonably good for browsing.
+
+package main
+
+import (
+	"fmt"
+	"go/ast"
+	"go/token"
+	"io"
+	"strconv"
+)
+
+// LinkifyText HTML-escapes source text and writes it to w.
+// Identifiers that are in a "use" position (i.e., that are
+// not being declared), are wrapped with HTML links pointing
+// to the respective declaration, if possible. Comments are
+// formatted the same way as with FormatText.
+//
+func LinkifyText(w io.Writer, text []byte, n ast.Node) {
+	links := links(n)
+
+	i := 0        // links index
+	open := false // status of html tag
+	linkWriter := func(w io.Writer, _ int, start bool) {
+		// end tag
+		if !start {
+			if open {
+				fmt.Fprintf(w, `</a>`)
+				open = false
+			}
+			return
+		}
+
+		// start tag
+		open = false
+		if i < len(links) {
+			switch info := links[i]; {
+			case info.path != "" && info.ident == nil:
+				// package path
+				fmt.Fprintf(w, `<a href="/pkg/%s/">`, info.path)
+				open = true
+			case info.path != "" && info.ident != nil:
+				// qualified identifier
+				fmt.Fprintf(w, `<a href="/pkg/%s/#%s">`, info.path, info.ident.Name)
+				open = true
+			case info.path == "" && info.ident != nil:
+				// locally declared identifier
+				fmt.Fprintf(w, `<a href="#%s">`, info.ident.Name)
+				open = true
+			}
+			i++
+		}
+	}
+
+	idents := tokenSelection(text, token.IDENT)
+	comments := tokenSelection(text, token.COMMENT)
+	FormatSelections(w, text, linkWriter, idents, selectionTag, comments)
+}
+
+// A link describes the (HTML) link information for an identifier.
+// The zero value of a link represents "no link".
+//
+type link struct {
+	path  string
+	ident *ast.Ident
+}
+
+// links returns the list of links for the identifiers used
+// by node in the same order as they appear in the source.
+//
+func links(node ast.Node) (list []link) {
+	defs := defs(node)
+
+	// NOTE: We are expecting ast.Inspect to call the
+	//       callback function in source text order.
+	ast.Inspect(node, func(node ast.Node) bool {
+		switch n := node.(type) {
+		case *ast.Ident:
+			info := link{}
+			if !defs[n] {
+				if n.Obj == nil && predeclared[n.Name] {
+					info.path = builtinPkgPath
+				}
+				info.ident = n
+			}
+			list = append(list, info)
+			return false
+		case *ast.SelectorExpr:
+			// Detect qualified identifiers of the form pkg.ident.
+			// If anything fails we return true and collect individual
+			// identifiers instead.
+			if x, _ := n.X.(*ast.Ident); x != nil {
+				// x must be a package for a qualified identifier
+				if obj := x.Obj; obj != nil && obj.Kind == ast.Pkg {
+					if spec, _ := obj.Decl.(*ast.ImportSpec); spec != nil {
+						// spec.Path.Value is the import path
+						if path, err := strconv.Unquote(spec.Path.Value); err == nil {
+							// Register two links, one for the package
+							// and one for the qualified identifier.
+							info := link{path: path}
+							list = append(list, info)
+							info.ident = n.Sel
+							list = append(list, info)
+							return false
+						}
+					}
+				}
+			}
+		}
+		return true
+	})
+
+	return
+}
+
+// defs returns the set of identifiers that are declared ("defined") by node.
+func defs(node ast.Node) map[*ast.Ident]bool {
+	m := make(map[*ast.Ident]bool)
+
+	ast.Inspect(node, func(node ast.Node) bool {
+		switch n := node.(type) {
+		case *ast.Field:
+			for _, n := range n.Names {
+				m[n] = true
+			}
+		case *ast.ImportSpec:
+			if name := n.Name; name != nil {
+				m[name] = true
+			}
+		case *ast.ValueSpec:
+			for _, n := range n.Names {
+				m[n] = true
+			}
+		case *ast.TypeSpec:
+			m[n.Name] = true
+		case *ast.FuncDecl:
+			m[n.Name] = true
+		case *ast.AssignStmt:
+			// Short variable declarations only show up if we apply
+			// this code to all source code (as opposed to exported
+			// declarations only).
+			if n.Tok == token.DEFINE {
+				// Some of the lhs variables may be re-declared,
+				// so technically they are not defs. We don't
+				// care for now.
+				for _, x := range n.Lhs {
+					// Each lhs expression should be an
+					// ident, but we are conservative and check.
+					if n, _ := x.(*ast.Ident); n != nil {
+						m[n] = true
+					}
+				}
+			}
+		}
+		return true
+	})
+
+	return m
+}
+
+// The predeclared map represents the set of all predeclared identifiers.
+var predeclared = map[string]bool{
+	"bool":       true,
+	"byte":       true,
+	"complex64":  true,
+	"complex128": true,
+	"error":      true,
+	"float32":    true,
+	"float64":    true,
+	"int":        true,
+	"int8":       true,
+	"int16":      true,
+	"int32":      true,
+	"int64":      true,
+	"rune":       true,
+	"string":     true,
+	"uint":       true,
+	"uint8":      true,
+	"uint16":     true,
+	"uint32":     true,
+	"uint64":     true,
+	"uintptr":    true,
+	"true":       true,
+	"false":      true,
+	"iota":       true,
+	"nil":        true,
+	"append":     true,
+	"cap":        true,
+	"close":      true,
+	"complex":    true,
+	"copy":       true,
+	"delete":     true,
+	"imag":       true,
+	"len":        true,
+	"make":       true,
+	"new":        true,
+	"panic":      true,
+	"print":      true,
+	"println":    true,
+	"real":       true,
+	"recover":    true,
+}