NXD blob/auth: Teach it to handle HTTP Basic Auth too

[ Not sent upstream.

  The patch was not sent upstream, because previous 2 raw blob patches
  were not accepted (see details there).

  OTOH it is very handy in SlapOS environment to use CI token auth for
  raw downloading, so just carry with us as NXD. ]

There are cases when using user:password for /raw/... access is handy:

- when using query for auth (private_token) is not convenient for some
  reason (e.g. client processing software does not handle queries well
  when generating URLs)

- when we do not want to organize many artificial users and use their
  tokens, but instead just use per-project automatically setup

    gitlab-ci-token : <ci-token>

  artificial user & "password" which are already handled by auth backend
  for `git fetch` requests.

Handling is easy: if main auth backend rejects access, and there is
user:password in original request, we retry asking auth backend the way
as `git fetch` would do.

Access is granted if any of two ways to ask auth backend succeeds. This
way both private tokens / cookies and HTTP auth are supported.

internal/api/auth.go

@@ -46,9 +46,10 @@ type AuthCacheEntry struct {
 
 // Entries are keyed by project + credentials
 type AuthCacheKey struct {
-	project string
-	query   string // e.g. with passing in private_token=...
-	header  string // request header url-encoded, e.g. PRIVATE-TOKEN=...
+	project  string
+	userinfo string // user[:password] or ""
+	query    string // e.g. with passing in private_token=...
+	header   string // request header url-encoded, e.g. PRIVATE-TOKEN=...
 }
 
 // Authorization reply cache
@@ -67,7 +68,13 @@ func NewAuthCache(a *API) *AuthCache {
 // Verify that download access is ok or not.
 // first we try to use the cache; if information is not there -> ask auth backend
 // download is ok if AuthReply.RepoPath != ""
-func (c *AuthCache) VerifyDownloadAccess(project string, query string, header http.Header) AuthReply {
+func (c *AuthCache) VerifyDownloadAccess(project string, userinfo *url.Userinfo, query string, header http.Header) AuthReply {
+	// In addition to userinfo:
+	u := ""
+	if userinfo != nil {
+		u = userinfo.String()
+	}
+
 	// Use only tokens from query/header and selected cookies to minimize cache and avoid
 	// creating redundant cache entries because of e.g. unrelated headers.
 	queryValues, _ := url.ParseQuery(query) // this is what URL.Query() does
@@ -99,7 +106,7 @@ func (c *AuthCache) VerifyDownloadAccess(project string, query string, header ht
 		h["Cookie"] = []string{hc}
 	}
 
-	key := AuthCacheKey{project, q.Encode(), h.Encode()}
+	key := AuthCacheKey{project, u, q.Encode(), h.Encode()}
 	return c.verifyDownloadAccess(key)
 }
 
@@ -198,6 +205,20 @@ func (c *AuthCache) refreshEntry(auth *AuthCacheEntry, key AuthCacheKey) {
 
 // Ask auth backend about cache key
 func (c *AuthCache) askAuthBackend(key AuthCacheKey) AuthReply {
+	// key.userinfo -> url.Userinfo
+	var user *url.Userinfo
+	if key.userinfo != "" {
+		u, err := url.Parse("http://" + key.userinfo + "@/")
+		// url prepared-to-parse userinfo must be valid
+		if err != nil {
+			panic(err)
+		}
+		if u.User == nil {
+			panic(fmt.Errorf("userinfo parse: `%s` -> empty", key.userinfo))
+		}
+		user = u.User
+	}
+
 	// key.header -> url.Values -> http.Header
 	hv, err := url.ParseQuery(key.header)
 	if err != nil {
@@ -210,7 +231,7 @@ func (c *AuthCache) askAuthBackend(key AuthCacheKey) AuthReply {
 		header[k] = v
 	}
 
-	return c.a.verifyDownloadAccess(key.project, key.query, header)
+	return c.a.verifyDownloadAccess(key.project, user, key.query, header)
 }
 
 // for detecting whether archive download is ok via senddata mechanism
@@ -237,11 +258,17 @@ func (aok *testDownloadOkViaSendArchive) Inject(w http.ResponseWriter, r *http.R
 //
 // Replies from authentication backend are cached for 30 seconds as each
 // request to Rails code is heavy and slow.
-func (a *API) VerifyDownloadAccess(project, query string, header http.Header) AuthReply {
-	return a.authCache.VerifyDownloadAccess(project, query, header)
+func (a *API) VerifyDownloadAccess(project string, user *url.Userinfo, query string, header http.Header) AuthReply {
+	return a.authCache.VerifyDownloadAccess(project, user, query, header)
+}
+
+// like Userinfo.Password(), "" if unset
+func xpassword(user *url.Userinfo) string {
+	password, _ := user.Password()
+	return password
 }
 
-func (a *API) verifyDownloadAccess(project, query string, header http.Header) AuthReply {
+func (a *API) verifyDownloadAccess(project string, user *url.Userinfo, query string, header http.Header) AuthReply {
 	authReply := AuthReply{
 		RawReply: httptest.NewRecorder(),
 	}
@@ -251,7 +278,6 @@ func (a *API) verifyDownloadAccess(project, query string, header http.Header) Au
 	//   side this supports only basic auth, not private token.
 	// - that's why we auth backend to authenticate as if it was request to
 	//   get repo archive and propagate request query and header.
-	// url := project + ".git/info/refs?service=git-upload-pack"
 	url := project + "/repository/archive.zip"
 	if query != "" {
 		url += "?" + query
@@ -261,6 +287,10 @@ func (a *API) verifyDownloadAccess(project, query string, header http.Header) Au
 		helper.Fail500(authReply.RawReply, fmt.Errorf("GET git-upload-pack: %v", err))
 		return authReply
 	}
+	if user != nil {
+		// just in case - Rails does not support HTTP Basic Auth for usual requests
+		reqDownloadAccess.SetBasicAuth(user.Username(), xpassword(user))
+	}
 	for k, v := range header {
 		reqDownloadAccess.Header[k] = v
 	}
@@ -278,5 +308,36 @@ func (a *API) verifyDownloadAccess(project, query string, header http.Header) Au
 	)
 	authProxy.ServeHTTP(authReply.RawReply, reqDownloadAccess)
 
+	// If not successful and userinfo is provided without query, retry
+	// authenticating as `git fetch` would do.
+	//
+	// The reason we want to do this second try is that HTTP auth is
+	// handled by upstream auth backend for git requests only, and we might
+	// want to use e.g. https://gitlab-ci-token:token@/.../raw/...
+	if authReply.RepoPath != "" || user == nil || query != "" {
+		return authReply
+	}
+
+	url = project + ".git/info/refs?service=git-upload-pack"
+	reqFetchAccess, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		helper.Fail500(authReply.RawReply, fmt.Errorf("GET git-upload-pack: %v", err))
+		return authReply
+	}
+	reqFetchAccess.SetBasicAuth(user.Username(), xpassword(user))
+	for k, v := range header {
+		reqFetchAccess.Header[k] = v
+	}
+
+	// reset RawReply - if failed we will return to client what this latter -
+	// - request to auth backend as `git fetch` - replies.
+	authReply.RawReply = httptest.NewRecorder()
+	a.PreAuthorizeHandler(
+		func(w http.ResponseWriter, req *http.Request, resp *Response) {
+			// if we ever get to this point - auth handler approved
+			// access and thus it is ok to download
+			authReply.Response = *resp
+		}, "").ServeHTTP(authReply.RawReply, reqFetchAccess)
+
 	return authReply
 }

internal/git/xblob.go

@@ -14,6 +14,7 @@ import (
 	"io"
 	"log"
 	"net/http"
+	"net/url"
 	"regexp"
 	"strings"
 )
@@ -39,8 +40,15 @@ func handleGetBlobRaw(a *api.API, w http.ResponseWriter, r *http.Request) {
 	project := u.Path[:rawLoc[0]]
 	refpath := u.Path[rawLoc[1]:]
 
+	// Prepare userinfo
+	var user *url.Userinfo
+	username, password, ok := r.BasicAuth()
+	if ok {
+		user = url.UserPassword(username, password)
+	}
+
 	// Query download access auth for this project
-	authReply := a.VerifyDownloadAccess(project, u.RawQuery, r.Header)
+	authReply := a.VerifyDownloadAccess(project, user, u.RawQuery, r.Header)
 	if authReply.RepoPath == "" {
 		// access denied - copy auth reply to client in full -
 		// there are HTTP code and other headers / body relevant for

main_test.go

@@ -662,11 +662,14 @@ func sha1s(data []byte) string {
 }
 
 // download an URL
-func download(t *testing.T, url string, h http.Header) (*http.Response, []byte) {
+func download(t *testing.T, url, username, password string, h http.Header) (*http.Response, []byte) {
 	req, err := http.NewRequest("GET", url, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
+	if !(username == "" && password == "") {
+		req.SetBasicAuth(username, password)
+	}
 	// copy header to request
 	for k, v := range h {
 		req.Header[k] = v
@@ -689,15 +692,17 @@ type DownloadContext struct {
 	t         *testing.T
 	urlPrefix string
 	Header    http.Header
+	username  string
+	password  string
 }
 
 func NewDownloadContext(t *testing.T, urlPrefix string) *DownloadContext {
 	h := make(http.Header)
-	return &DownloadContext{t, urlPrefix, h}
+	return &DownloadContext{t, urlPrefix, h, "", ""}
 }
 
 func (dl DownloadContext) download(path string) (*http.Response, []byte) {
-	return download(dl.t, dl.urlPrefix+path, dl.Header)
+	return download(dl.t, dl.urlPrefix+path, dl.username, dl.password, dl.Header)
 }
 
 // download `path` and expect content sha1 to be `expectSha1`
@@ -763,7 +768,14 @@ func TestPrivateBlobDownload(t *testing.T) {
 		token_ok2 := r.Header.Get("BBB-TOKEN") == "TOKEN-4BBB"
 		cookie, _ := r.Cookie("_gitlab_session")
 		cookie_ok3 := (cookie != nil && cookie.Value == "COOKIE-CCC")
-		if !(token_ok1 || token_ok2 || cookie_ok3) {
+		username, password, user_ok4 := r.BasicAuth()
+		if user_ok4 {
+			// user:password only accepted for `git fetch` requests
+			user_ok4 = (strings.HasSuffix(r.URL.Path, "/info/refs") &&
+				r.URL.RawQuery == "service=git-upload-pack" &&
+				username == "user-ddd" && password == "password-eee")
+		}
+		if !(token_ok1 || token_ok2 || cookie_ok3 || user_ok4) {
 			w.WriteHeader(403)
 			fmt.Fprintf(w, "Access denied")
 			return
@@ -804,6 +816,17 @@ func TestPrivateBlobDownload(t *testing.T) {
 	dl.Header.Set("Cookie", "alpha=1; _gitlab_session=COOKIE-CCC; beta=2")
 	dl.ExpectCode("/5f923865/README.md", 200)
 	dl.ExpectSha1("/5f923865/README.md", "5f7af35c185a9e5face2f4afb6d7c4f00328d04c")
+
+	dl.Header = make(http.Header) // clear
+	dl.ExpectCode("/5f923865/README.md", 403)
+	dl.username = "user-aaa"
+	dl.password = "password-bbb"
+	dl.ExpectCode("/5f923865/README.md", 403)
+	dl.username = "user-ddd"
+	dl.password = "password-eee"
+	dl.ExpectCode("/5f923865/README.md?qqq_token=1", 403)
+	dl.ExpectCode("/5f923865/README.md", 200)
+	dl.ExpectSha1("/5f923865/README.md", "5f7af35c185a9e5face2f4afb6d7c4f00328d04c")
 }
 
 // TestHealthChecksNoStaticHTML verifies that health endpoints pass errors through and don't return the static html error pages