cmd/go/internal/module: define fs-safe module path encoding

Module paths, like import paths, are case-sensitive, for better or worse.
But not all file systems distinguish file paths with different cases.
If we are going to use module paths to construct file system paths,
we must apply an encoding that distinguishes case without relying
upon the file system to do it.

This CL defines that encoding, the "safe module path encoding".
Module paths today are ASCII-only with limited punctuation,
so the safe module path encoding is to convert the whole path
to lower case and insert an ! before every formerly upper-case letter:
github.com/Sirupsen/logrus is stored as github.com/!sirupsen/logrus.

Although this CL defines the encoding, it does not change the rest
of the go command to use the encoding. That will be done in
follow-up CLs.

Change-Id: I06e6188dcfcbbc1d88674f7c95e1cb45cb476238
Reviewed-on: https://go-review.googlesource.com/124378
Run-TryBot: Russ Cox <rsc@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Bryan C. Mills <bcmills@google.com>

src/cmd/go/internal/module/module_test.go

@@ -57,90 +57,109 @@ func TestCheck(t *testing.T) {
 var checkPathTests = []struct {
 	path     string
 	ok       bool
+	importOK bool
 }{
-	{"x.y/z", true},
-	{"x.y", true},
-
-	{"", false},
-	{"x.y/\xFFz", false},
-	{"/x.y/z", false},
-	{"x./z", false},
-	{".x/z", false},
-	{"-x/z", false},
-	{"x..y/z", false},
-	{"x.y/z/../../w", false},
-	{"x.y//z", false},
-	{"x.y/z//w", false},
-	{"x.y/z/", false},
-
-	{"x.y/z/v0", false},
-	{"x.y/z/v1", false},
-	{"x.y/z/v2", true},
-	{"x.y/z/v2.0", false},
-
-	{"!x.y/z", false},
-	{"_x.y/z", false},
-	{"x.y!/z", false},
-	{"x.y\"/z", false},
-	{"x.y#/z", false},
-	{"x.y$/z", false},
-	{"x.y%/z", false},
-	{"x.y&/z", false},
-	{"x.y'/z", false},
-	{"x.y(/z", false},
-	{"x.y)/z", false},
-	{"x.y*/z", false},
-	{"x.y+/z", false},
-	{"x.y,/z", false},
-	{"x.y-/z", true},
-	{"x.y./zt", false},
-	{"x.y:/z", false},
-	{"x.y;/z", false},
-	{"x.y</z", false},
-	{"x.y=/z", false},
-	{"x.y>/z", false},
-	{"x.y?/z", false},
-	{"x.y@/z", false},
-	{"x.y[/z", false},
-	{"x.y\\/z", false},
-	{"x.y]/z", false},
-	{"x.y^/z", false},
-	{"x.y_/z", false},
-	{"x.y`/z", false},
-	{"x.y{/z", false},
-	{"x.y}/z", false},
-	{"x.y~/z", false},
-	{"x.y/z!", false},
-	{"x.y/z\"", false},
-	{"x.y/z#", false},
-	{"x.y/z$", false},
-	{"x.y/z%", false},
-	{"x.y/z&", false},
-	{"x.y/z'", false},
-	{"x.y/z(", false},
-	{"x.y/z)", false},
-	{"x.y/z*", false},
-	{"x.y/z+", true},
-	{"x.y/z,", true},
-	{"x.y/z-", true},
-	{"x.y/z.t", true},
-	{"x.y/z/t", true},
-	{"x.y/z:", false},
-	{"x.y/z;", false},
-	{"x.y/z<", false},
-	{"x.y/z=", false},
-	{"x.y/z>", false},
-	{"x.y/z?", false},
-	{"x.y/z@", false},
-	{"x.y/z[", false},
-	{"x.y/z\\", false},
-	{"x.y/z]", false},
-	{"x.y/z^", false},
-	{"x.y/z_", true},
-	{"x.y/z`", false},
-	{"x.y/z{", false},
-	{"x.y/z}", false},
-	{"x.y/z~", true},
+	{"x.y/z", true, true},
+	{"x.y", true, true},
+
+	{"", false, false},
+	{"x.y/\xFFz", false, false},
+	{"/x.y/z", false, false},
+	{"x./z", false, false},
+	{".x/z", false, false},
+	{"-x/z", false, true},
+	{"x..y/z", false, false},
+	{"x.y/z/../../w", false, false},
+	{"x.y//z", false, false},
+	{"x.y/z//w", false, false},
+	{"x.y/z/", false, false},
+
+	{"x.y/z/v0", false, true},
+	{"x.y/z/v1", false, true},
+	{"x.y/z/v2", true, true},
+	{"x.y/z/v2.0", false, true},
+	{"X.y/z", false, true},
+
+	{"!x.y/z", false, false},
+	{"_x.y/z", false, true},
+	{"x.y!/z", false, false},
+	{"x.y\"/z", false, false},
+	{"x.y#/z", false, false},
+	{"x.y$/z", false, false},
+	{"x.y%/z", false, false},
+	{"x.y&/z", false, false},
+	{"x.y'/z", false, false},
+	{"x.y(/z", false, false},
+	{"x.y)/z", false, false},
+	{"x.y*/z", false, false},
+	{"x.y+/z", false, true},
+	{"x.y,/z", false, false},
+	{"x.y-/z", true, true},
+	{"x.y./zt", false, false},
+	{"x.y:/z", false, false},
+	{"x.y;/z", false, false},
+	{"x.y</z", false, false},
+	{"x.y=/z", false, false},
+	{"x.y>/z", false, false},
+	{"x.y?/z", false, false},
+	{"x.y@/z", false, false},
+	{"x.y[/z", false, false},
+	{"x.y\\/z", false, false},
+	{"x.y]/z", false, false},
+	{"x.y^/z", false, false},
+	{"x.y_/z", false, true},
+	{"x.y`/z", false, false},
+	{"x.y{/z", false, false},
+	{"x.y}/z", false, false},
+	{"x.y~/z", false, true},
+	{"x.y/z!", false, false},
+	{"x.y/z\"", false, false},
+	{"x.y/z#", false, false},
+	{"x.y/z$", false, false},
+	{"x.y/z%", false, false},
+	{"x.y/z&", false, false},
+	{"x.y/z'", false, false},
+	{"x.y/z(", false, false},
+	{"x.y/z)", false, false},
+	{"x.y/z*", false, false},
+	{"x.y/z+", true, true},
+	{"x.y/z,", false, false},
+	{"x.y/z-", true, true},
+	{"x.y/z.t", true, true},
+	{"x.y/z/t", true, true},
+	{"x.y/z:", false, false},
+	{"x.y/z;", false, false},
+	{"x.y/z<", false, false},
+	{"x.y/z=", false, false},
+	{"x.y/z>", false, false},
+	{"x.y/z?", false, false},
+	{"x.y/z@", false, false},
+	{"x.y/z[", false, false},
+	{"x.y/z\\", false, false},
+	{"x.y/z]", false, false},
+	{"x.y/z^", false, false},
+	{"x.y/z_", true, true},
+	{"x.y/z`", false, false},
+	{"x.y/z{", false, false},
+	{"x.y/z}", false, false},
+	{"x.y/z~", true, true},
+	{"x.y/x.foo", true, true},
+	{"x.y/aux.foo", false, false},
+	{"x.y/prn", false, false},
+	{"x.y/prn2", true, true},
+	{"x.y/com", true, true},
+	{"x.y/com1", false, false},
+	{"x.y/com1.txt", false, false},
+	{"x.y/calm1", true, true},
+	{"github.com/!123/logrus", false, false},
+
+	// TODO: CL 41822 allowed Unicode letters in old "go get"
+	// without due consideration of the implications, and only on github.com (!).
+	// For now, we disallow non-ASCII characters in module mode,
+	// in both module paths and general import paths,
+	// until we can get the implications right.
+	// When we do, we'll enable them everywhere, not just for GitHub.
+	{"github.com/user/unicode/испытание", false, false},
 }
 
 func TestCheckPath(t *testing.T) {
@@ -151,6 +170,13 @@ func TestCheckPath(t *testing.T) {
 		} else if !tt.ok && err == nil {
 			t.Errorf("CheckPath(%q) succeeded, wanted error", tt.path)
 		}
+
+		err = CheckImportPath(tt.path)
+		if tt.importOK && err != nil {
+			t.Errorf("CheckImportPath(%q) = %v, wanted nil error", tt.path, err)
+		} else if !tt.importOK && err == nil {
+			t.Errorf("CheckImportPath(%q) succeeded, wanted error", tt.path)
+		}
 	}
 }
 
@@ -182,3 +208,84 @@ func TestSplitPathVersion(t *testing.T) {
 		}
 	}
 }
+
+var encodeTests = []struct {
+	path string
+	enc  string // empty means same as path
+}{
+	{path: "ascii.com/abcdefghijklmnopqrstuvwxyz.-+/~_0123456789"},
+	{path: "github.com/GoogleCloudPlatform/omega", enc: "github.com/!google!cloud!platform/omega"},
+}
+
+func TestEncodePath(t *testing.T) {
+	// Check invalid paths.
+	for _, tt := range checkPathTests {
+		if !tt.ok {
+			_, err := EncodePath(tt.path)
+			if err == nil {
+				t.Errorf("EncodePath(%q): succeeded, want error (invalid path)", tt.path)
+			}
+		}
+	}
+
+	// Check encodings.
+	for _, tt := range encodeTests {
+		enc, err := EncodePath(tt.path)
+		if err != nil {
+			t.Errorf("EncodePath(%q): unexpected error: %v", tt.path, err)
+			continue
+		}
+		want := tt.enc
+		if want == "" {
+			want = tt.path
+		}
+		if enc != want {
+			t.Errorf("EncodePath(%q) = %q, want %q", tt.path, enc, want)
+		}
+	}
+}
+
+var badDecode = []string{
+	"github.com/GoogleCloudPlatform/omega",
+	"github.com/!google!cloud!platform!/omega",
+	"github.com/!0google!cloud!platform/omega",
+	"github.com/!_google!cloud!platform/omega",
+	"github.com/!!google!cloud!platform/omega",
+	"",
+}
+
+func TestDecodePath(t *testing.T) {
+	// Check invalid decodings.
+	for _, bad := range badDecode {
+		_, err := DecodePath(bad)
+		if err == nil {
+			t.Errorf("DecodePath(%q): succeeded, want error (invalid decoding)", bad)
+		}
+	}
+
+	// Check invalid paths (or maybe decodings).
+	for _, tt := range checkPathTests {
+		if !tt.ok {
+			path, err := DecodePath(tt.path)
+			if err == nil {
+				t.Errorf("DecodePath(%q) = %q, want error (invalid path)", tt.path, path)
+			}
+		}
+	}
+
+	// Check encodings.
+	for _, tt := range encodeTests {
+		enc := tt.enc
+		if enc == "" {
+			enc = tt.path
+		}
+		path, err := DecodePath(enc)
+		if err != nil {
+			t.Errorf("DecodePath(%q): unexpected error: %v", enc, err)
+			continue
+		}
+		if path != tt.path {
+			t.Errorf("DecodePath(%q) = %q, want %q", enc, path, tt.path)
+		}
+	}
+}