go/parser, go/ast: correctly take into account presence of } in block

Correctly track whether the closing } of a block (or a function body)
is present or not in the AST and report correct End() positions in
each case.

There are more cases like this but this CL addresses an immediate
issue and sets a precedent for how to fix similar cases if a need
arises.

Fixes #33649.

Change-Id: Id6662ddaac09f3c15f8003edc9275fe2b0c41c78
Reviewed-on: https://go-review.googlesource.com/c/go/+/202581
Run-TryBot: Robert Griesemer <gri@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Alan Donovan <adonovan@google.com>

src/go/ast/ast.go

@@ -634,7 +634,7 @@ type (
 	BlockStmt struct {
 		Lbrace token.Pos // position of "{"
 		List   []Stmt
-		Rbrace token.Pos // position of "}"
+		Rbrace token.Pos // position of "}", if any (may be absent due to syntax error)
 	}
 
 	// An IfStmt node represents an if statement.
@@ -757,7 +757,15 @@ func (s *BranchStmt) End() token.Pos {
 	}
 	return token.Pos(int(s.TokPos) + len(s.Tok.String()))
 }
-func (s *BlockStmt) End() token.Pos { return s.Rbrace + 1 }
+func (s *BlockStmt) End() token.Pos {
+	if s.Rbrace.IsValid() {
+		return s.Rbrace + 1
+	}
+	if n := len(s.List); n > 0 {
+		return s.List[n-1].End()
+	}
+	return s.Lbrace + 1
+}
 func (s *IfStmt) End() token.Pos {
 	if s.Else != nil {
 		return s.Else.End()

src/go/ast/issues_test.go

+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package ast_test
+
+import (
+	"go/ast"
+	"go/parser"
+	"go/token"
+	"testing"
+)
+
+func TestIssue33649(t *testing.T) {
+	for _, src := range []string{
+		`package p; func _()`,
+		`package p; func _() {`,
+		`package p; func _() { _ = 0`,
+		`package p; func _() { _ = 0 }`,
+	} {
+		fset := token.NewFileSet()
+		f, _ := parser.ParseFile(fset, "", src, parser.AllErrors)
+		if f == nil {
+			panic("invalid test setup: parser didn't return an AST")
+		}
+
+		// find corresponding token.File
+		var tf *token.File
+		fset.Iterate(func(f *token.File) bool {
+			tf = f
+			return true
+		})
+		tfEnd := tf.Base() + tf.Size()
+
+		fd := f.Decls[len(f.Decls)-1].(*ast.FuncDecl)
+		fdEnd := int(fd.End())
+
+		if fdEnd != tfEnd {
+			t.Errorf("%q: got fdEnd = %d; want %d (base = %d, size = %d)", src, fdEnd, tfEnd, tf.Base(), tf.Size())
+		}
+	}
+}

src/go/parser/parser.go

@@ -397,6 +397,18 @@ func (p *parser) expect(tok token.Token) token.Pos {
 	return pos
 }
 
+// expect2 is like expect, but it returns an invalid position
+// if the expected token is not found.
+func (p *parser) expect2(tok token.Token) (pos token.Pos) {
+	if p.tok == tok {
+		pos = p.pos
+	} else {
+		p.errorExpected(pos, "'"+tok.String()+"'")
+	}
+	p.next() // make progress
+	return
+}
+
 // expectClosing is like expect but provides a better error message
 // for the common case of a missing comma before a newline.
 //
@@ -1082,7 +1094,7 @@ func (p *parser) parseBody(scope *ast.Scope) *ast.BlockStmt {
 	list := p.parseStmtList()
 	p.closeLabelScope()
 	p.closeScope()
-	rbrace := p.expect(token.RBRACE)
+	rbrace := p.expect2(token.RBRACE)
 
 	return &ast.BlockStmt{Lbrace: lbrace, List: list, Rbrace: rbrace}
 }
@@ -1096,7 +1108,7 @@ func (p *parser) parseBlockStmt() *ast.BlockStmt {
 	p.openScope()
 	list := p.parseStmtList()
 	p.closeScope()
-	rbrace := p.expect(token.RBRACE)
+	rbrace := p.expect2(token.RBRACE)
 
 	return &ast.BlockStmt{Lbrace: lbrace, List: list, Rbrace: rbrace}
 }