go/scanner: strip carriage returns from commments

Also:
- cleaned up and simplified TestScan
- added tests for comments containing carriage returns

Fixes #3647.

R=rsc
CC=golang-dev
https://golang.org/cl/6225047

src/pkg/go/scanner/scanner.go

@@ -157,11 +157,15 @@ func (s *Scanner) interpretLineComment(text []byte) {
 func (s *Scanner) scanComment() string {
 	// initial '/' already consumed; s.ch == '/' || s.ch == '*'
 	offs := s.offset - 1 // position of initial '/'
+	hasCR := false
 
 	if s.ch == '/' {
 		//-style comment
 		s.next()
 		for s.ch != '\n' && s.ch >= 0 {
+			if s.ch == '\r' {
+				hasCR = true
+			}
 			s.next()
 		}
 		if offs == s.lineOffset {
@@ -175,6 +179,9 @@ func (s *Scanner) scanComment() string {
 	s.next()
 	for s.ch >= 0 {
 		ch := s.ch
+		if ch == '\r' {
+			hasCR = true
+		}
 		s.next()
 		if ch == '*' && s.ch == '/' {
 			s.next()
@@ -185,7 +192,12 @@ func (s *Scanner) scanComment() string {
 	s.error(offs, "comment not terminated")
 
 exit:
-	return string(s.src[offs:s.offset])
+	lit := s.src[offs:s.offset]
+	if hasCR {
+		lit = stripCR(lit)
+	}
+
+	return string(lit)
 }
 
 func (s *Scanner) findLineEnd() bool {
@@ -527,6 +539,8 @@ func (s *Scanner) switch4(tok0, tok1 token.Token, ch2 rune, tok2, tok3 token.Tok
 // token.IMAG, token.CHAR, token.STRING) or token.COMMENT, the literal string
 // has the corresponding value.
 //
+// If the returned token is a keyword, the literal string is the keyword.
+//
 // If the returned token is token.SEMICOLON, the corresponding
 // literal string is ";" if the semicolon was present in the source,
 // and "\n" if the semicolon was inserted because of a newline or

src/pkg/go/scanner/scanner_test.go

@@ -43,6 +43,8 @@ var tokens = [...]elt{
 	// Special tokens
 	{token.COMMENT, "/* a comment */", special},
 	{token.COMMENT, "// a comment \n", special},
+	{token.COMMENT, "/*\r*/", special},
+	{token.COMMENT, "//\r\n", special},
 
 	// Identifiers and basic type literals
 	{token.IDENT, "foobar", literal},
@@ -214,8 +216,6 @@ func checkPos(t *testing.T, lit string, p token.Pos, expected token.Position) {
 
 // Verify that calling Scan() provides the correct results.
 func TestScan(t *testing.T) {
-	// make source
-	src_linecount := newlineCount(string(source))
 	whitespace_linecount := newlineCount(whitespace)
 
 	// error handler
@@ -226,59 +226,81 @@ func TestScan(t *testing.T) {
 	// verify scan
 	var s Scanner
 	s.Init(fset.AddFile("", fset.Base(), len(source)), source, eh, ScanComments|dontInsertSemis)
-	index := 0
-	// epos is the expected position
+
+	// set up expected position
 	epos := token.Position{
 		Filename: "",
 		Offset:   0,
 		Line:     1,
 		Column:   1,
 	}
+
+	index := 0
 	for {
 		pos, tok, lit := s.Scan()
-		if lit == "" {
-			// no literal value for non-literal tokens
-			lit = tok.String()
+
+		// check position
+		if tok == token.EOF {
+			// correction for EOF
+			epos.Line = newlineCount(string(source))
+			epos.Column = 2
 		}
+		checkPos(t, lit, pos, epos)
+
+		// check token
 		e := elt{token.EOF, "", special}
 		if index < len(tokens) {
 			e = tokens[index]
+			index++
 		}
-		if tok == token.EOF {
-			lit = "<EOF>"
-			epos.Line = src_linecount
-			epos.Column = 2
-		}
-		checkPos(t, lit, pos, epos)
 		if tok != e.tok {
 			t.Errorf("bad token for %q: got %s, expected %s", lit, tok, e.tok)
 		}
-		if e.tok.IsLiteral() {
-			// no CRs in raw string literals
-			elit := e.lit
-			if elit[0] == '`' {
-				elit = string(stripCR([]byte(elit)))
-				epos.Offset += len(e.lit) - len(lit) // correct position
-			}
-			if lit != elit {
-				t.Errorf("bad literal for %q: got %q, expected %q", lit, lit, elit)
-			}
-		}
+
+		// check token class
 		if tokenclass(tok) != e.class {
 			t.Errorf("bad class for %q: got %d, expected %d", lit, tokenclass(tok), e.class)
 		}
-		epos.Offset += len(lit) + len(whitespace)
-		epos.Line += newlineCount(lit) + whitespace_linecount
-		if tok == token.COMMENT && lit[1] == '/' {
-			// correct for unaccounted '/n' in //-style comment
-			epos.Offset++
-			epos.Line++
+
+		// check literal
+		elit := ""
+		switch e.tok {
+		case token.COMMENT:
+			// no CRs in comments
+			elit = string(stripCR([]byte(e.lit)))
+			//-style comment literal doesn't contain newline
+			if elit[1] == '/' {
+				elit = elit[0 : len(elit)-1]
+			}
+		case token.IDENT:
+			elit = e.lit
+		case token.SEMICOLON:
+			elit = ";"
+		default:
+			if e.tok.IsLiteral() {
+				// no CRs in raw string literals
+				elit = e.lit
+				if elit[0] == '`' {
+					elit = string(stripCR([]byte(elit)))
+				}
+			} else if e.tok.IsKeyword() {
+				elit = e.lit
+			}
+		}
+		if lit != elit {
+			t.Errorf("bad literal for %q: got %q, expected %q", lit, lit, elit)
 		}
-		index++
+
 		if tok == token.EOF {
 			break
 		}
+
+		// update position
+		epos.Offset += len(e.lit) + len(whitespace)
+		epos.Line += newlineCount(e.lit) + whitespace_linecount
+
 	}
+
 	if s.ErrorCount != 0 {
 		t.Errorf("found %d errors", s.ErrorCount)
 	}