Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
G
gitlab-workhorse
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
iv
gitlab-workhorse
Commits
e6c7d974
Commit
e6c7d974
authored
Nov 07, 2015
by
Marin Jankovski
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Handle lfs upload with middleware and callback.
parent
ec721eef
Changes
4
Hide whitespace changes
Inline
Side-by-side
Showing
4 changed files
with
95 additions
and
92 deletions
+95
-92
archive.go
archive.go
+2
-2
git-http.go
git-http.go
+1
-1
lfs.go
lfs.go
+88
-86
upstream.go
upstream.go
+4
-3
No files found.
archive.go
View file @
e6c7d974
...
@@ -5,6 +5,7 @@ In this file we handle 'git archive' downloads
...
@@ -5,6 +5,7 @@ In this file we handle 'git archive' downloads
package
main
package
main
import
(
import
(
"errors"
"fmt"
"fmt"
"io"
"io"
"io/ioutil"
"io/ioutil"
...
@@ -13,9 +14,8 @@ import (
...
@@ -13,9 +14,8 @@ import (
"os"
"os"
"os/exec"
"os/exec"
"path"
"path"
"time"
"path/filepath"
"path/filepath"
"
errors
"
"
time
"
)
)
func
handleGetArchive
(
w
http
.
ResponseWriter
,
r
*
gitRequest
)
{
func
handleGetArchive
(
w
http
.
ResponseWriter
,
r
*
gitRequest
)
{
...
...
git-http.go
View file @
e6c7d974
...
@@ -9,8 +9,8 @@ import (
...
@@ -9,8 +9,8 @@ import (
"fmt"
"fmt"
"io"
"io"
"net/http"
"net/http"
"strings"
"path/filepath"
"path/filepath"
"strings"
)
)
func
handleGetInfoRefs
(
w
http
.
ResponseWriter
,
r
*
gitRequest
)
{
func
handleGetInfoRefs
(
w
http
.
ResponseWriter
,
r
*
gitRequest
)
{
...
...
lfs.go
View file @
e6c7d974
...
@@ -5,11 +5,11 @@ In this file we handle git lfs objects downloads and uploads
...
@@ -5,11 +5,11 @@ In this file we handle git lfs objects downloads and uploads
package
main
package
main
import
(
import
(
"compress/gzip"
"crypto/sha256"
"crypto/sha256"
"encoding/hex"
"encoding/hex"
"errors"
"errors"
"io"
"io"
"io/ioutil"
"log"
"log"
"net/http"
"net/http"
"os"
"os"
...
@@ -23,100 +23,102 @@ var (
...
@@ -23,100 +23,102 @@ var (
errSizeMismatch
=
errors
.
New
(
"Content size does not match"
)
errSizeMismatch
=
errors
.
New
(
"Content size does not match"
)
)
)
func
handleStoreLfsObject
(
w
http
.
ResponseWriter
,
r
*
gitRequest
,
rpc
string
)
{
func
lfsAuthorizeHandler
(
handleFunc
serviceHandleFunc
)
serviceHandleFunc
{
var
body
io
.
ReadCloser
return
preAuthorizeHandler
(
func
(
w
http
.
ResponseWriter
,
r
*
gitRequest
)
{
urlPath
:=
r
.
URL
.
Path
if
r
.
StoreLFSPath
==
""
{
regExp
:=
regexp
.
MustCompile
(
`([0-9a-f]{64})/([0-9]+)`
)
fail500
(
w
,
"lfsAuthorizeHandler"
,
errors
.
New
(
"Don't know where to store object, no store path specified."
))
matches
:=
regExp
.
FindStringSubmatch
(
urlPath
)
return
}
if
matches
==
nil
{
handleFunc
(
w
,
r
)
log
.
Printf
(
"Found no object info in path: %s"
,
urlPath
)
},
""
)
return
}
}
oid
:=
matches
[
1
]
func
handleStoreLfsObject
(
handleFunc
serviceHandleFunc
)
serviceHandleFunc
{
size
:=
matches
[
2
]
return
func
(
w
http
.
ResponseWriter
,
r
*
gitRequest
)
{
log
.
Printf
(
"Found oid: %s and size: %s"
,
oid
,
size
)
urlPath
:=
r
.
URL
.
Path
storePath
:=
filepath
.
Join
(
r
.
StoreLFSPath
,
transformKey
(
oid
))
regExp
:=
regexp
.
MustCompile
(
`([0-9a-f]{64})/([0-9]+)`
)
matches
:=
regExp
.
FindStringSubmatch
(
urlPath
)
if
_
,
err
:=
os
.
Stat
(
storePath
);
os
.
IsNotExist
(
err
)
{
tmpPath
:=
filepath
.
Join
(
r
.
StoreLFSPath
,
"tmp"
,
oid
)
if
matches
==
nil
{
log
.
Printf
(
"Found no object info in path: %s"
,
urlPath
)
if
_
,
err
:=
os
.
Stat
(
tmpPath
);
os
.
IsNotExist
(
err
)
{
return
// TODO try removing gzip, possibly not needed
// The client request body may have been gzipped.
if
r
.
Header
.
Get
(
"Content-Encoding"
)
==
"gzip"
{
body
,
err
=
gzip
.
NewReader
(
r
.
Body
)
if
err
!=
nil
{
fail500
(
w
,
"Couldn't handle LFS upload request."
,
err
)
return
}
}
else
{
body
=
r
.
Body
}
defer
body
.
Close
()
// TODO maybe set dir permissions to 700
dir
:=
filepath
.
Dir
(
tmpPath
)
if
err
:=
os
.
MkdirAll
(
dir
,
0750
);
err
!=
nil
{
fail500
(
w
,
"Couldn't create directory for storing LFS objects."
,
err
)
return
}
// TODO use go library for creating TMP files
file
,
err
:=
os
.
OpenFile
(
tmpPath
,
os
.
O_CREATE
|
os
.
O_WRONLY
|
os
.
O_EXCL
,
0640
)
if
err
!=
nil
{
fail500
(
w
,
"Couldn't open tmp file for writing."
,
err
)
return
}
// defer os.Remove(tmpPath)
defer
file
.
Close
()
hash
:=
sha256
.
New
()
hw
:=
io
.
MultiWriter
(
hash
,
file
)
written
,
err
:=
io
.
Copy
(
hw
,
body
)
if
err
!=
nil
{
fail500
(
w
,
"Failed to save received LFS object."
,
err
)
return
}
file
.
Close
()
sizeInt
,
err
:=
strconv
.
ParseInt
(
size
,
10
,
64
)
if
err
!=
nil
{
fail500
(
w
,
"Couldn't read size: "
,
err
)
return
}
if
written
!=
sizeInt
{
fail500
(
w
,
"Inconsistent size: "
,
errSizeMismatch
)
return
}
shaStr
:=
hex
.
EncodeToString
(
hash
.
Sum
(
nil
))
if
shaStr
!=
oid
{
fail500
(
w
,
"Inconsistent size: "
,
errSizeMismatch
)
return
}
}
}
}
// if err := os.Rename(tmpPath, path); err != nil {
// fail500(w, "Failed to rename temporary LFS object.", err)
// return
// }
log
.
Printf
(
"Received the LFS object from client, oid: %s"
,
oid
)
oid
:=
matches
[
1
]
size
:=
matches
[
2
]
log
.
Printf
(
"Found oid: %s and size: %s"
,
oid
,
size
)
return
sha
:=
sha256
.
New
()
sha
.
Write
([]
byte
(
oid
))
tmp_hash
:=
hex
.
EncodeToString
(
sha
.
Sum
(
nil
))
tmpPath
:=
filepath
.
Join
(
r
.
StoreLFSPath
,
"tmp"
)
var
body
io
.
ReadCloser
body
=
r
.
Body
defer
body
.
Close
()
dir
:=
filepath
.
Dir
(
tmpPath
)
if
err
:=
os
.
MkdirAll
(
dir
,
0700
);
err
!=
nil
{
fail500
(
w
,
"Couldn't create directory for storing LFS objects."
,
err
)
return
}
file
,
err
:=
ioutil
.
TempFile
(
tmpPath
,
tmp_hash
)
if
err
!=
nil
{
fail500
(
w
,
"Couldn't open tmp file for writing."
,
err
)
return
}
defer
os
.
Remove
(
tmpPath
)
defer
file
.
Close
()
hash
:=
sha256
.
New
()
hw
:=
io
.
MultiWriter
(
hash
,
file
)
written
,
err
:=
io
.
Copy
(
hw
,
body
)
if
err
!=
nil
{
fail500
(
w
,
"Failed to save received LFS object."
,
err
)
return
}
file
.
Close
()
sizeInt
,
err
:=
strconv
.
ParseInt
(
size
,
10
,
64
)
if
err
!=
nil
{
fail500
(
w
,
"Couldn't read size: "
,
err
)
return
}
if
written
!=
sizeInt
{
fail500
(
w
,
"Inconsistent size: "
,
errSizeMismatch
)
return
}
shaStr
:=
hex
.
EncodeToString
(
hash
.
Sum
(
nil
))
if
shaStr
!=
oid
{
fail500
(
w
,
"Inconsistent size: "
,
errSizeMismatch
)
return
}
r
.
Header
.
Set
(
"X-GitLab-Lfs-Tmp"
,
filepath
.
Base
(
file
.
Name
()))
handleFunc
(
w
,
r
)
}
}
}
func
transformKey
(
key
string
)
string
{
func
lfsCallback
(
w
http
.
ResponseWriter
,
r
*
gitRequest
)
{
if
len
(
key
)
<
5
{
authReq
,
err
:=
r
.
u
.
newUpstreamRequest
(
r
.
Request
,
nil
,
"/authorize"
)
return
key
if
err
!=
nil
{
fail500
(
w
,
"newUpstreamRequestlfsCallback"
,
err
)
return
}
}
return
filepath
.
Join
(
key
[
0
:
2
],
key
[
2
:
4
],
key
[
4
:
len
(
key
)])
authResponse
,
err
:=
r
.
u
.
httpClient
.
Do
(
authReq
)
if
err
!=
nil
{
fail500
(
w
,
"doRequestlfsCallback"
,
err
)
return
}
defer
authResponse
.
Body
.
Close
()
return
}
}
upstream.go
View file @
e6c7d974
...
@@ -45,7 +45,8 @@ type authorizationResponse struct {
...
@@ -45,7 +45,8 @@ type authorizationResponse struct {
// in the GitLab Rails app and the 'time of use' in gitlab-workhorse.
// in the GitLab Rails app and the 'time of use' in gitlab-workhorse.
CommitId
string
CommitId
string
// TODO: say something about this
// StoreLFSPath is provided by the GitLab Rails application
// to mark where the tmp file should be placed
StoreLFSPath
string
StoreLFSPath
string
}
}
...
@@ -54,7 +55,7 @@ type authorizationResponse struct {
...
@@ -54,7 +55,7 @@ type authorizationResponse struct {
type
gitRequest
struct
{
type
gitRequest
struct
{
*
http
.
Request
*
http
.
Request
authorizationResponse
authorizationResponse
u
*
upstream
u
*
upstream
}
}
// Routing table
// Routing table
...
@@ -68,7 +69,7 @@ var gitServices = [...]gitService{
...
@@ -68,7 +69,7 @@ var gitServices = [...]gitService{
gitService
{
"GET"
,
regexp
.
MustCompile
(
`/repository/archive.tar.gz\z`
),
repoPreAuthorizeHandler
(
handleGetArchive
)},
gitService
{
"GET"
,
regexp
.
MustCompile
(
`/repository/archive.tar.gz\z`
),
repoPreAuthorizeHandler
(
handleGetArchive
)},
gitService
{
"GET"
,
regexp
.
MustCompile
(
`/repository/archive.tar.bz2\z`
),
repoPreAuthorizeHandler
(
handleGetArchive
)},
gitService
{
"GET"
,
regexp
.
MustCompile
(
`/repository/archive.tar.bz2\z`
),
repoPreAuthorizeHandler
(
handleGetArchive
)},
gitService
{
"GET"
,
regexp
.
MustCompile
(
`/uploads/`
),
handleSendFile
},
gitService
{
"GET"
,
regexp
.
MustCompile
(
`/uploads/`
),
handleSendFile
},
gitService
{
"PUT"
,
regexp
.
MustCompile
(
`/gitlab-lfs/objects/([0-9a-f]{64})/([0-9]+)\z`
),
repoPreAuthorizeHandler
(
handleStoreLfsObject
)},
gitService
{
"PUT"
,
regexp
.
MustCompile
(
`/gitlab-lfs/objects/([0-9a-f]{64})/([0-9]+)\z`
),
lfsAuthorizeHandler
(
handleStoreLfsObject
(
lfsCallback
)
)},
gitService
{
"GET"
,
regexp
.
MustCompile
(
`/gitlab-lfs/objects/([0-9a-f]{64})\z`
),
handleSendFile
},
gitService
{
"GET"
,
regexp
.
MustCompile
(
`/gitlab-lfs/objects/([0-9a-f]{64})\z`
),
handleSendFile
},
}
}
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment