Skip to content

G
git-backup
Commit 80a6e9b0 authored by Kirill Smelkov's avatar Kirill Smelkov
Browse files
Options

Cleanup git-backup lock and gitlab-backup tempporary folder in defer-style 

So that further `git-backup pull` run can proceed, and many gitlab-backup's left `$tmpd` don't eat up disk space.

Rework of kirr/git-backup!4.

/reviewed-on kirr/git-backup!5
parents 9791c04e c835bf38
Show whitespace changes
Inline Side-by-side
Showing with 168 additions and 158 deletions
contrib/gitlab-backup
View file @ 80a6e9b0
#!/bin/bash -e #!/bin/bash -e
# pull/restore gitlab data into/from git-backup # pull/restore gitlab data into/from git-backup
# Copyright (C) 2015-2016 Nexedi SA and Contributors. # Copyright (C) 2015-2020 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com> # Kirill Smelkov <kirr@nexedi.com>
# #
# This program is free software: you can Use, Study, Modify and Redistribute # This program is free software: you can Use, Study, Modify and Redistribute
...@@ -110,6 +110,8 @@ backup_pull() { ...@@ -110,6 +110,8 @@ backup_pull() {
# 1. dump all gitlab data except repositories & db # 1. dump all gitlab data except repositories & db
echo " * Dumping gitlab data (except repositories & db)" echo " * Dumping gitlab data (except repositories & db)"
tmpd=$(mktemp -d `pwd`/gitlab-backup.XXXXXX) tmpd=$(mktemp -d `pwd`/gitlab-backup.XXXXXX)
trap 'rm -rf "$tmpd"' EXIT
gitlab-rake gitlab:backup:create SKIP=repositories,db | tee "$tmpd/gitlab_backup_create.out" gitlab-rake gitlab:backup:create SKIP=repositories,db | tee "$tmpd/gitlab_backup_create.out"
backup_tar=`grep "^Creating backup archive: .* done" "$tmpd/gitlab_backup_create.out"` || \ backup_tar=`grep "^Creating backup archive: .* done" "$tmpd/gitlab_backup_create.out"` || \
die "E: Cannot detect backup tar" die "E: Cannot detect backup tar"
...@@ -218,7 +220,6 @@ backup_pull() { ...@@ -218,7 +220,6 @@ backup_pull() {
# mark backup_tar as pulled and cleanup # mark backup_tar as pulled and cleanup
mv "$backup_tar" "$backup_tar.pulled" mv "$backup_tar" "$backup_tar.pulled"
fi fi
rm -rf "$tmpd"
echo OK echo OK
} }
...@@ -235,6 +236,7 @@ backup_restore() { ...@@ -235,6 +236,7 @@ backup_restore() {
# 1. extract all gitlab data except repositories # 1. extract all gitlab data except repositories
echo " * Extracting gitlab data (except repositories)" echo " * Extracting gitlab data (except repositories)"
tmpd=$(mktemp -d `pwd`/gitlab-backup.XXXXXX) tmpd=$(mktemp -d `pwd`/gitlab-backup.XXXXXX)
trap 'rm -rf "$tmpd"' EXIT
$GIT_BACKUP restore $HEAD gitlab/misc:"$tmpd/gitlab_backup" $GIT_BACKUP restore $HEAD gitlab/misc:"$tmpd/gitlab_backup"
backup_info="$tmpd/gitlab_backup/backup_information.yml" backup_info="$tmpd/gitlab_backup/backup_information.yml"
...@@ -308,8 +310,6 @@ backup_restore() { ...@@ -308,8 +310,6 @@ backup_restore() {
test -e "$backup_tar" && die "E: $backup_tar already exists" test -e "$backup_tar" && die "E: $backup_tar already exists"
tar -C "$tmpd/gitlab_backup" -cf "$backup_tar" . tar -C "$tmpd/gitlab_backup" -cf "$backup_tar" .
rm -rf "$tmpd" # tmpd no longer needed
# 4. extract repositories into .../repositories.<timestamp> # 4. extract repositories into .../repositories.<timestamp>
echo " * Extracting repositories" echo " * Extracting repositories"
reposX="${GITLAB_REPOS_PATH}.${backup_created_at}" reposX="${GITLAB_REPOS_PATH}.${backup_created_at}"
......
git-backup.go
View file @ 80a6e9b0
// Copyright (C) 2015-2016 Nexedi SA and Contributors. // Copyright (C) 2015-2020 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com> // Kirill Smelkov <kirr@nexedi.com>
// //
// This program is free software: you can Use, Study, Modify and Redistribute // This program is free software: you can Use, Study, Modify and Redistribute
...@@ -67,17 +67,18 @@ NOTE the idea of pulling all refs together is similar to git-namespaces ...@@ -67,17 +67,18 @@ NOTE the idea of pulling all refs together is similar to git-namespaces
package main package main
import ( import (
"context"
"flag" "flag"
"fmt" "fmt"
"io/ioutil" "io/ioutil"
"os" "os"
"os/signal"
pathpkg "path" pathpkg "path"
"path/filepath" "path/filepath"
"runtime" "runtime"
"runtime/debug" "runtime/debug"
"sort" "sort"
"strings" "strings"
"sync"
"syscall" "syscall"
"time" "time"
...@@ -87,6 +88,7 @@ import ( ...@@ -87,6 +88,7 @@ import (
"lab.nexedi.com/kirr/go123/xerr" "lab.nexedi.com/kirr/go123/xerr"
"lab.nexedi.com/kirr/go123/xflag" "lab.nexedi.com/kirr/go123/xflag"
"lab.nexedi.com/kirr/go123/xstrings" "lab.nexedi.com/kirr/go123/xstrings"
"lab.nexedi.com/kirr/go123/xsync"
git "github.com/libgit2/git2go" git "github.com/libgit2/git2go"
) )
...@@ -184,7 +186,7 @@ func blob_to_file(g *git.Repository, blob_sha1 Sha1, mode uint32, path string) { ...@@ -184,7 +186,7 @@ func blob_to_file(g *git.Repository, blob_sha1 Sha1, mode uint32, path string) {
// another, so that on backup restore we only have to recreate original tag // another, so that on backup restore we only have to recreate original tag
// object and tagged object is kept there in repo thanks to it being reachable // object and tagged object is kept there in repo thanks to it being reachable
// through created commit. // through created commit.
func obj_represent_as_commit(g *git.Repository, sha1 Sha1, obj_type git.ObjectType) Sha1 { func obj_represent_as_commit(ctx context.Context, g *git.Repository, sha1 Sha1, obj_type git.ObjectType) Sha1 {
switch obj_type { switch obj_type {
case git.ObjectTag, git.ObjectTree, git.ObjectBlob: case git.ObjectTag, git.ObjectTree, git.ObjectBlob:
// ok // ok
...@@ -222,7 +224,7 @@ func obj_represent_as_commit(g *git.Repository, sha1 Sha1, obj_type git.ObjectTy ...@@ -222,7 +224,7 @@ func obj_represent_as_commit(g *git.Repository, sha1 Sha1, obj_type git.ObjectTy
// v .tree -> ø // v .tree -> ø
// Commit .parent -> Commit // Commit .parent -> Commit
if tagged_type == git.ObjectCommit { if tagged_type == git.ObjectCommit {
return zcommit_tree(mktree_empty(), []Sha1{tagged_sha1}, obj_encoded) return zcommit_tree(mktree_empty(ctx), []Sha1{tagged_sha1}, obj_encoded)
} }
// Tag ~> Commit* // Tag ~> Commit*
...@@ -238,7 +240,7 @@ func obj_represent_as_commit(g *git.Repository, sha1 Sha1, obj_type git.ObjectTy ...@@ -238,7 +240,7 @@ func obj_represent_as_commit(g *git.Repository, sha1 Sha1, obj_type git.ObjectTy
// v .tree -> Tree* "tagged" -> Blob // v .tree -> Tree* "tagged" -> Blob
// Blob .parent -> ø // Blob .parent -> ø
if tagged_type == git.ObjectBlob { if tagged_type == git.ObjectBlob {
tree_for_blob := xgitSha1("mktree", RunWith{stdin: fmt.Sprintf("100644 blob %s\ttagged\n", tagged_sha1)}) tree_for_blob := xgitSha1(ctx, "mktree", RunWith{stdin: fmt.Sprintf("100644 blob %s\ttagged\n", tagged_sha1)})
return zcommit_tree(tree_for_blob, []Sha1{}, obj_encoded) return zcommit_tree(tree_for_blob, []Sha1{}, obj_encoded)
} }
...@@ -247,8 +249,8 @@ func obj_represent_as_commit(g *git.Repository, sha1 Sha1, obj_type git.ObjectTy ...@@ -247,8 +249,8 @@ func obj_represent_as_commit(g *git.Repository, sha1 Sha1, obj_type git.ObjectTy
// v .tree -> ø // v .tree -> ø
// Tag₁ .parent -> Commit₁* // Tag₁ .parent -> Commit₁*
if tagged_type == git.ObjectTag { if tagged_type == git.ObjectTag {
commit1 := obj_represent_as_commit(g, tagged_sha1, tagged_type) commit1 := obj_represent_as_commit(ctx, g, tagged_sha1, tagged_type)
return zcommit_tree(mktree_empty(), []Sha1{commit1}, obj_encoded) return zcommit_tree(mktree_empty(ctx), []Sha1{commit1}, obj_encoded)
} }
exc.Raisef("%s (%q): unknown tagged type", sha1, tagged_type) exc.Raisef("%s (%q): unknown tagged type", sha1, tagged_type)
...@@ -334,7 +336,7 @@ type PullSpec struct { ...@@ -334,7 +336,7 @@ type PullSpec struct {
dir, prefix string dir, prefix string
} }
func cmd_pull(gb *git.Repository, argv []string) { func cmd_pull(ctx context.Context, gb *git.Repository, argv []string) {
flags := flag.FlagSet{Usage: cmd_pull_usage} flags := flag.FlagSet{Usage: cmd_pull_usage}
flags.Init("", flag.ExitOnError) flags.Init("", flag.ExitOnError)
flags.Parse(argv) flags.Parse(argv)
...@@ -357,7 +359,7 @@ func cmd_pull(gb *git.Repository, argv []string) { ...@@ -357,7 +359,7 @@ func cmd_pull(gb *git.Repository, argv []string) {
pullspecv = append(pullspecv, PullSpec{dir, prefix}) pullspecv = append(pullspecv, PullSpec{dir, prefix})
} }
cmd_pull_(gb, pullspecv) cmd_pull_(ctx, gb, pullspecv)
} }
// Ref is info about a reference pointing to sha1. // Ref is info about a reference pointing to sha1.
...@@ -366,25 +368,26 @@ type Ref struct { ...@@ -366,25 +368,26 @@ type Ref struct {
sha1 Sha1 sha1 Sha1
} }
func cmd_pull_(gb *git.Repository, pullspecv []PullSpec) { func cmd_pull_(ctx context.Context, gb *git.Repository, pullspecv []PullSpec) {
// while pulling, we'll keep refs from all pulled repositories under temp // while pulling, we'll keep refs from all pulled repositories under temp
// unique work refs namespace. // unique work refs namespace.
backup_time := time.Now().Format("20060102-1504") // %Y%m%d-%H%M backup_time := time.Now().Format("20060102-1504") // %Y%m%d-%H%M
backup_refs_work := fmt.Sprintf("refs/backup/%s/", backup_time) // refs/backup/20150820-2109/ backup_refs_work := fmt.Sprintf("refs/backup/%s/", backup_time) // refs/backup/20150820-2109/
backup_lock := "refs/backup.locked"
// make sure another `git-backup pull` is not running // prevent another `git-backup pull` from running simultaneously
xgit("update-ref", backup_lock, mktree_empty(), Sha1{}) backup_lock := "refs/backup.locked"
xgit(ctx, "update-ref", backup_lock, mktree_empty(ctx), Sha1{})
defer xgit(context.Background(), "update-ref", "-d", backup_lock)
// make sure there is root commit // make sure there is root commit
var HEAD Sha1 var HEAD Sha1
var err error var err error
gerr, __, _ := ggit("rev-parse", "--verify", "HEAD") gerr, __, _ := ggit(ctx, "rev-parse", "--verify", "HEAD")
if gerr != nil { if gerr != nil {
infof("# creating root commit") infof("# creating root commit")
// NOTE `git commit` does not work in bare repo - do commit by hand // NOTE `git commit` does not work in bare repo - do commit by hand
HEAD = xcommit_tree(gb, mktree_empty(), []Sha1{}, "Initialize git-backup repository") HEAD = xcommit_tree(gb, mktree_empty(ctx), []Sha1{}, "Initialize git-backup repository")
xgit("update-ref", "-m", "git-backup pull init", "HEAD", HEAD) xgit(ctx, "update-ref", "-m", "git-backup pull init", "HEAD", HEAD)
} else { } else {
HEAD, err = Sha1Parse(__) HEAD, err = Sha1Parse(__)
exc.Raiseif(err) exc.Raiseif(err)
...@@ -406,7 +409,7 @@ func cmd_pull_(gb *git.Repository, pullspecv []PullSpec) { ...@@ -406,7 +409,7 @@ func cmd_pull_(gb *git.Repository, pullspecv []PullSpec) {
// Of those there are ~ 1.9·10⁶ commit objects, i.e. ~10% of total. // Of those there are ~ 1.9·10⁶ commit objects, i.e. ~10% of total.
// Since 1 sha1 is 2·10¹ bytes, the space needed for keeping sha1 of all // Since 1 sha1 is 2·10¹ bytes, the space needed for keeping sha1 of all
// commits is ~ 4·10⁷B = ~40MB. It is thus ok to keep this index in RAM for now. // commits is ~ 4·10⁷B = ~40MB. It is thus ok to keep this index in RAM for now.
for _, __ := range xstrings.SplitLines(xgit("rev-list", HEAD), "\n") { for _, __ := range xstrings.SplitLines(xgit(ctx, "rev-list", HEAD), "\n") {
sha1, err := Sha1Parse(__) sha1, err := Sha1Parse(__)
exc.Raiseif(err) exc.Raiseif(err)
alreadyHave.Add(sha1) alreadyHave.Add(sha1)
...@@ -423,7 +426,7 @@ func cmd_pull_(gb *git.Repository, pullspecv []PullSpec) { ...@@ -423,7 +426,7 @@ func cmd_pull_(gb *git.Repository, pullspecv []PullSpec) {
htree, err := hcommit.Tree() htree, err := hcommit.Tree()
exc.Raiseif(err) exc.Raiseif(err)
if htree.EntryByName("backup.refs") != nil { if htree.EntryByName("backup.refs") != nil {
repotab, err := loadBackupRefs(fmt.Sprintf("%s:backup.refs", HEAD)) repotab, err := loadBackupRefs(ctx, fmt.Sprintf("%s:backup.refs", HEAD))
exc.Raiseif(err) exc.Raiseif(err)
for _, repo := range repotab { for _, repo := range repotab {
...@@ -447,7 +450,7 @@ func cmd_pull_(gb *git.Repository, pullspecv []PullSpec) { ...@@ -447,7 +450,7 @@ func cmd_pull_(gb *git.Repository, pullspecv []PullSpec) {
// make sure index is empty for prefix (so that we start from clean // make sure index is empty for prefix (so that we start from clean
// prefix namespace and this way won't leave stale removed things) // prefix namespace and this way won't leave stale removed things)
xgit("rm", "--cached", "-r", "--ignore-unmatch", "--", prefix) xgit(ctx, "rm", "--cached", "-r", "--ignore-unmatch", "--", prefix)
here := my.FuncName() here := my.FuncName()
err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) (errout error) { err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) (errout error) {
...@@ -461,6 +464,10 @@ func cmd_pull_(gb *git.Repository, pullspecv []PullSpec) { ...@@ -461,6 +464,10 @@ func cmd_pull_(gb *git.Repository, pullspecv []PullSpec) {
return err return err
} }
if ctx.Err() != nil {
return ctx.Err()
}
// propagate exceptions properly via filepath.Walk as errors with calling context // propagate exceptions properly via filepath.Walk as errors with calling context
// (filepath is not our code) // (filepath is not our code)
defer exc.Catch(func(e *exc.Error) { defer exc.Catch(func(e *exc.Error) {
...@@ -490,7 +497,7 @@ func cmd_pull_(gb *git.Repository, pullspecv []PullSpec) { ...@@ -490,7 +497,7 @@ func cmd_pull_(gb *git.Repository, pullspecv []PullSpec) {
// git repo - let's pull all refs from it to our backup refs namespace // git repo - let's pull all refs from it to our backup refs namespace
infof("# git %s\t<- %s", prefix, path) infof("# git %s\t<- %s", prefix, path)
refv, _, err := fetch(path, alreadyHave) refv, _, err := fetch(ctx, path, alreadyHave)
exc.Raiseif(err) exc.Raiseif(err)
// TODO don't store to git references all references from fetched repository: // TODO don't store to git references all references from fetched repository:
...@@ -533,7 +540,7 @@ func cmd_pull_(gb *git.Repository, pullspecv []PullSpec) { ...@@ -533,7 +540,7 @@ func cmd_pull_(gb *git.Repository, pullspecv []PullSpec) {
} }
// add to index files we converted to blobs // add to index files we converted to blobs
xgit("update-index", "--add", "--index-info", RunWith{stdin: strings.Join(blobbedv, "\n")}) xgit(ctx, "update-index", "--add", "--index-info", RunWith{stdin: strings.Join(blobbedv, "\n")})
// all refs from all found git repositories populated. // all refs from all found git repositories populated.
// now prepare manifest with ref -> sha1 and do a synthetic commit merging all that sha1 // now prepare manifest with ref -> sha1 and do a synthetic commit merging all that sha1
...@@ -551,7 +558,7 @@ func cmd_pull_(gb *git.Repository, pullspecv []PullSpec) { ...@@ -551,7 +558,7 @@ func cmd_pull_(gb *git.Repository, pullspecv []PullSpec) {
// //
// NOTE `git for-each-ref` sorts output by ref // NOTE `git for-each-ref` sorts output by ref
// -> backup_refs is sorted and stable between runs // -> backup_refs is sorted and stable between runs
backup_refs_dump := xgit("for-each-ref", backup_refs_work) backup_refs_dump := xgit(ctx, "for-each-ref", backup_refs_work)
backup_refs_list := []Ref{} // parsed dump backup_refs_list := []Ref{} // parsed dump
backup_refsv := []string{} // backup.refs content backup_refsv := []string{} // backup.refs content
backup_refs_parents := Sha1Set{} // sha1 for commit parents, obtained from refs backup_refs_parents := Sha1Set{} // sha1 for commit parents, obtained from refs
...@@ -577,7 +584,7 @@ func cmd_pull_(gb *git.Repository, pullspecv []PullSpec) { ...@@ -577,7 +584,7 @@ func cmd_pull_(gb *git.Repository, pullspecv []PullSpec) {
if !ok { if !ok {
exc.Raisef("%s: invalid git type in entry %q", backup_refs_work, __) exc.Raisef("%s: invalid git type in entry %q", backup_refs_work, __)
} }
sha1_ = obj_represent_as_commit(gb, sha1, obj_type) sha1_ = obj_represent_as_commit(ctx, gb, sha1, obj_type)
noncommit_seen[sha1] = sha1_ noncommit_seen[sha1] = sha1_
} }
...@@ -596,17 +603,17 @@ func cmd_pull_(gb *git.Repository, pullspecv []PullSpec) { ...@@ -596,17 +603,17 @@ func cmd_pull_(gb *git.Repository, pullspecv []PullSpec) {
sort.Sort(BySha1(backup_refs_parentv)) // so parents order is stable in between runs sort.Sort(BySha1(backup_refs_parentv)) // so parents order is stable in between runs
// backup_refs -> blob // backup_refs -> blob
backup_refs_sha1 := xgitSha1("hash-object", "-w", "--stdin", RunWith{stdin: backup_refs}) backup_refs_sha1 := xgitSha1(ctx, "hash-object", "-w", "--stdin", RunWith{stdin: backup_refs})
// add backup_refs blob to index // add backup_refs blob to index
xgit("update-index", "--add", "--cacheinfo", fmt.Sprintf("100644,%s,backup.refs", backup_refs_sha1)) xgit(ctx, "update-index", "--add", "--cacheinfo", fmt.Sprintf("100644,%s,backup.refs", backup_refs_sha1))
// index is ready - prepare tree and commit // index is ready - prepare tree and commit
backup_tree_sha1 := xgitSha1("write-tree") backup_tree_sha1 := xgitSha1(ctx, "write-tree")
commit_sha1 := xcommit_tree(gb, backup_tree_sha1, append([]Sha1{HEAD}, backup_refs_parentv...), commit_sha1 := xcommit_tree(gb, backup_tree_sha1, append([]Sha1{HEAD}, backup_refs_parentv...),
"Git-backup "+backup_time) "Git-backup "+backup_time)
xgit("update-ref", "-m", "git-backup pull", "HEAD", commit_sha1, HEAD) xgit(ctx, "update-ref", "-m", "git-backup pull", "HEAD", commit_sha1, HEAD)
// remove no-longer needed backup refs & verify they don't stay // remove no-longer needed backup refs & verify they don't stay
backup_refs_delete := "" backup_refs_delete := ""
...@@ -614,8 +621,8 @@ func cmd_pull_(gb *git.Repository, pullspecv []PullSpec) { ...@@ -614,8 +621,8 @@ func cmd_pull_(gb *git.Repository, pullspecv []PullSpec) {
backup_refs_delete += fmt.Sprintf("delete %s %s\n", ref.name, ref.sha1) backup_refs_delete += fmt.Sprintf("delete %s %s\n", ref.name, ref.sha1)
} }
xgit("update-ref", "--stdin", RunWith{stdin: backup_refs_delete}) xgit(ctx, "update-ref", "--stdin", RunWith{stdin: backup_refs_delete})
__ = xgit("for-each-ref", backup_refs_work) __ = xgit(ctx, "for-each-ref", backup_refs_work)
if __ != "" { if __ != "" {
exc.Raisef("Backup refs under %s not deleted properly", backup_refs_work) exc.Raisef("Backup refs under %s not deleted properly", backup_refs_work)
} }
...@@ -635,28 +642,25 @@ func cmd_pull_(gb *git.Repository, pullspecv []PullSpec) { ...@@ -635,28 +642,25 @@ func cmd_pull_(gb *git.Repository, pullspecv []PullSpec) {
// accumulate, the longer pull starts to be, so it becomes O(n^2). // accumulate, the longer pull starts to be, so it becomes O(n^2).
// //
// -> what to do is described nearby fetch/mkref call. // -> what to do is described nearby fetch/mkref call.
gitdir := xgit("rev-parse", "--git-dir") gitdir := xgit(ctx, "rev-parse", "--git-dir")
err = os.RemoveAll(gitdir + "/" + backup_refs_work) err = os.RemoveAll(gitdir + "/" + backup_refs_work)
exc.Raiseif(err) // NOTE err is nil if path does not exist exc.Raiseif(err) // NOTE err is nil if path does not exist
// if we have working copy - update it // if we have working copy - update it
bare := xgit("rev-parse", "--is-bare-repository") bare := xgit(ctx, "rev-parse", "--is-bare-repository")
if bare != "true" { if bare != "true" {
// `git checkout-index -af` -- does not delete deleted files // `git checkout-index -af` -- does not delete deleted files
// `git read-tree -v -u --reset HEAD~ HEAD` -- needs index matching // `git read-tree -v -u --reset HEAD~ HEAD` -- needs index matching
// original worktree to properly work, but we already have updated index // original worktree to properly work, but we already have updated index
// //
// so we get changes we committed as diff and apply to worktree // so we get changes we committed as diff and apply to worktree
diff := xgit("diff", "--binary", HEAD, "HEAD", RunWith{raw: true}) diff := xgit(ctx, "diff", "--binary", HEAD, "HEAD", RunWith{raw: true})
if diff != "" { if diff != "" {
diffstat := xgit("apply", "--stat", "--apply", "--binary", "--whitespace=nowarn", diffstat := xgit(ctx, "apply", "--stat", "--apply", "--binary", "--whitespace=nowarn",
RunWith{stdin: diff, raw: true}) RunWith{stdin: diff, raw: true})
infof("%s", diffstat) infof("%s", diffstat)
} }
} }
// we are done - unlock
xgit("update-ref", "-d", backup_lock)
} }
// fetch makes sure all objects from a repository are present in backup place. // fetch makes sure all objects from a repository are present in backup place.
...@@ -681,11 +685,11 @@ func cmd_pull_(gb *git.Repository, pullspecv []PullSpec) { ...@@ -681,11 +685,11 @@ func cmd_pull_(gb *git.Repository, pullspecv []PullSpec) {
// //
// Note: fetch does not create any local references - the references returned // Note: fetch does not create any local references - the references returned
// only describe state of references in fetched source repository. // only describe state of references in fetched source repository.
func fetch(repo string, alreadyHave Sha1Set) (refv, fetchedv []Ref, err error) { func fetch(ctx context.Context, repo string, alreadyHave Sha1Set) (refv, fetchedv []Ref, err error) {
defer xerr.Contextf(&err, "fetch %s", repo) defer xerr.Contextf(&err, "fetch %s", repo)
// first check which references are advertised // first check which references are advertised
refv, err = lsremote(repo) refv, err = lsremote(ctx, repo)
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
} }
...@@ -731,7 +735,7 @@ func fetch(repo string, alreadyHave Sha1Set) (refv, fetchedv []Ref, err error) { ...@@ -731,7 +735,7 @@ func fetch(repo string, alreadyHave Sha1Set) (refv, fetchedv []Ref, err error) {
} }
arg(RunWith{stderr: gitprogress()}) arg(RunWith{stderr: gitprogress()})
gerr, _, _ := ggit(argv...) gerr, _, _ := ggit(ctx, argv...)
if gerr != nil { if gerr != nil {
return nil, nil, gerr return nil, nil, gerr
} }
...@@ -755,7 +759,7 @@ func fetch(repo string, alreadyHave Sha1Set) (refv, fetchedv []Ref, err error) { ...@@ -755,7 +759,7 @@ func fetch(repo string, alreadyHave Sha1Set) (refv, fetchedv []Ref, err error) {
} }
arg(RunWith{stderr: gitprogress()}) arg(RunWith{stderr: gitprogress()})
gerr, _, _ = ggit(argv...) gerr, _, _ = ggit(ctx, argv...)
if gerr != nil { if gerr != nil {
return nil, nil, fmt.Errorf("remote did not send all neccessary objects") return nil, nil, fmt.Errorf("remote did not send all neccessary objects")
} }
...@@ -765,7 +769,7 @@ func fetch(repo string, alreadyHave Sha1Set) (refv, fetchedv []Ref, err error) { ...@@ -765,7 +769,7 @@ func fetch(repo string, alreadyHave Sha1Set) (refv, fetchedv []Ref, err error) {
} }
// lsremote lists all references advertised by repo. // lsremote lists all references advertised by repo.
func lsremote(repo string) (refv []Ref, err error) { func lsremote(ctx context.Context, repo string) (refv []Ref, err error) {
defer xerr.Contextf(&err, "lsremote %s", repo) defer xerr.Contextf(&err, "lsremote %s", repo)
// NOTE --refs instructs to omit peeled refs like // NOTE --refs instructs to omit peeled refs like
...@@ -778,7 +782,7 @@ func lsremote(repo string) (refv []Ref, err error) { ...@@ -778,7 +782,7 @@ func lsremote(repo string) (refv []Ref, err error) {
// https://public-inbox.org/git/20180610143231.7131-1-kirr@nexedi.com/ // https://public-inbox.org/git/20180610143231.7131-1-kirr@nexedi.com/
// //
// we don't need to pull them anyway. // we don't need to pull them anyway.
gerr, stdout, _ := ggit("ls-remote", "--refs", repo) gerr, stdout, _ := ggit(ctx, "ls-remote", "--refs", repo)
if gerr != nil { if gerr != nil {
return nil, gerr return nil, gerr
} }
...@@ -822,7 +826,7 @@ type RestoreSpec struct { ...@@ -822,7 +826,7 @@ type RestoreSpec struct {
prefix, dir string prefix, dir string
} }
func cmd_restore(gb *git.Repository, argv []string) { func cmd_restore(ctx context.Context, gb *git.Repository, argv []string) {
flags := flag.FlagSet{Usage: cmd_restore_usage} flags := flag.FlagSet{Usage: cmd_restore_usage}
flags.Init("", flag.ExitOnError) flags.Init("", flag.ExitOnError)
flags.Parse(argv) flags.Parse(argv)
...@@ -847,7 +851,7 @@ func cmd_restore(gb *git.Repository, argv []string) { ...@@ -847,7 +851,7 @@ func cmd_restore(gb *git.Repository, argv []string) {
restorespecv = append(restorespecv, RestoreSpec{prefix, dir}) restorespecv = append(restorespecv, RestoreSpec{prefix, dir})
} }
cmd_restore_(gb, HEAD, restorespecv) cmd_restore_(ctx, gb, HEAD, restorespecv)
} }
// kirr/wendelin.core.git/heads/master -> kirr/wendelin.core.git, heads/master // kirr/wendelin.core.git/heads/master -> kirr/wendelin.core.git, heads/master
...@@ -944,11 +948,11 @@ type PackExtractReq struct { ...@@ -944,11 +948,11 @@ type PackExtractReq struct {
prefix string prefix string
} }
func cmd_restore_(gb *git.Repository, HEAD_ string, restorespecv []RestoreSpec) { func cmd_restore_(ctx context.Context, gb *git.Repository, HEAD_ string, restorespecv []RestoreSpec) {
HEAD := xgitSha1("rev-parse", "--verify", HEAD_) HEAD := xgitSha1(ctx, "rev-parse", "--verify", HEAD_)
// read backup refs index // read backup refs index
repotab, err := loadBackupRefs(fmt.Sprintf("%s:backup.refs", HEAD)) repotab, err := loadBackupRefs(ctx, fmt.Sprintf("%s:backup.refs", HEAD))
exc.Raiseif(err) exc.Raiseif(err)
// flattened & sorted repotab // flattened & sorted repotab
...@@ -963,23 +967,18 @@ func cmd_restore_(gb *git.Repository, HEAD_ string, restorespecv []RestoreSpec) ...@@ -963,23 +967,18 @@ func cmd_restore_(gb *git.Repository, HEAD_ string, restorespecv []RestoreSpec)
repotab = nil repotab = nil
packxq := make(chan PackExtractReq, 2*njobs) // requests to extract packs packxq := make(chan PackExtractReq, 2*njobs) // requests to extract packs
errch := make(chan error) // errors from workers wg := xsync.NewWorkGroup(ctx)
stopch := make(chan struct{}) // broadcasts restore has to be cancelled
wg := sync.WaitGroup{}
// main worker: walk over specified prefixes restoring files and // main worker: walk over specified prefixes restoring files and
// scheduling pack extraction requests from *.git -> packxq // scheduling pack extraction requests from *.git -> packxq
wg.Add(1) wg.Go(func(ctx context.Context) (err error) {
go func() {
defer wg.Done()
defer close(packxq) defer close(packxq)
// raised err -> errch // raised err -> return
here := my.FuncName() here := my.FuncName()
defer exc.Catch(func(e *exc.Error) { defer exc.Catch(func(e *exc.Error) {
errch <- exc.Addcallingcontext(here, e) err = exc.Addcallingcontext(here, e)
}) })
runloop:
for _, __ := range restorespecv { for _, __ := range restorespecv {
prefix, dir := __.prefix, __.dir prefix, dir := __.prefix, __.dir
...@@ -988,7 +987,7 @@ func cmd_restore_(gb *git.Repository, HEAD_ string, restorespecv []RestoreSpec) ...@@ -988,7 +987,7 @@ func cmd_restore_(gb *git.Repository, HEAD_ string, restorespecv []RestoreSpec)
exc.Raiseif(err) exc.Raiseif(err)
// files // files
lstree := xgit("ls-tree", "--full-tree", "-r", "-z", "--", HEAD, prefix, RunWith{raw: true}) lstree := xgit(ctx, "ls-tree", "--full-tree", "-r", "-z", "--", HEAD, prefix, RunWith{raw: true})
repos_seen := StrSet{} // dirs of *.git seen while restoring files repos_seen := StrSet{} // dirs of *.git seen while restoring files
for _, __ := range xstrings.SplitLines(lstree, "\x00") { for _, __ := range xstrings.SplitLines(lstree, "\x00") {
mode, type_, sha1, filename, err := parse_lstree_entry(__) mode, type_, sha1, filename, err := parse_lstree_entry(__)
...@@ -1000,6 +999,8 @@ func cmd_restore_(gb *git.Repository, HEAD_ string, restorespecv []RestoreSpec) ...@@ -1000,6 +999,8 @@ func cmd_restore_(gb *git.Repository, HEAD_ string, restorespecv []RestoreSpec)
exc.Raisef("%s: invalid/unexpected ls-tree entry %q", HEAD, __) exc.Raisef("%s: invalid/unexpected ls-tree entry %q", HEAD, __)
} }
exc.Raiseif(ctx.Err())
filename = reprefix(prefix, dir, filename) filename = reprefix(prefix, dir, filename)
infof("# file %s\t-> %s", prefix, filename) infof("# file %s\t-> %s", prefix, filename)
blob_to_file(gb, sha1, mode, filename) blob_to_file(gb, sha1, mode, filename)
...@@ -1043,33 +1044,32 @@ func cmd_restore_(gb *git.Repository, HEAD_ string, restorespecv []RestoreSpec) ...@@ -1043,33 +1044,32 @@ func cmd_restore_(gb *git.Repository, HEAD_ string, restorespecv []RestoreSpec)
repopath: reprefix(prefix, dir, repo.repopath), repopath: reprefix(prefix, dir, repo.repopath),
prefix: prefix}: prefix: prefix}:
case <-stopch: case <-ctx.Done():
break runloop return ctx.Err()
} }
} }
} }
}()
return nil
})
// pack workers: packxq -> extract packs // pack workers: packxq -> extract packs
for i := 0; i < njobs; i++ { for i := 0; i < njobs; i++ {
wg.Add(1) wg.Go(func(ctx context.Context) (err error) {
go func() { // raised err -> return
defer wg.Done()
// raised err -> errch
here := my.FuncName() here := my.FuncName()
defer exc.Catch(func(e *exc.Error) { defer exc.Catch(func(e *exc.Error) {
errch <- exc.Addcallingcontext(here, e) err = exc.Addcallingcontext(here, e)
}) })
runloop:
for { for {
select { select {
case <-stopch: case <-ctx.Done():
break runloop return ctx.Err()
case p, ok := <-packxq: case p, ok := <-packxq:
if !ok { if !ok {
break runloop return nil
} }
infof("# git %s\t-> %s", p.prefix, p.repopath) infof("# git %s\t-> %s", p.prefix, p.repopath)
...@@ -1089,10 +1089,10 @@ func cmd_restore_(gb *git.Repository, HEAD_ string, restorespecv []RestoreSpec) ...@@ -1089,10 +1089,10 @@ func cmd_restore_(gb *git.Repository, HEAD_ string, restorespecv []RestoreSpec)
} }
pack_argv = append(pack_argv, p.repopath+"/objects/pack/pack") pack_argv = append(pack_argv, p.repopath+"/objects/pack/pack")
xgit2(pack_argv, RunWith{stdin: p.refs.Sha1HeadsStr(), stderr: gitprogress()}) xgit2(ctx, pack_argv, RunWith{stdin: p.refs.Sha1HeadsStr(), stderr: gitprogress()})
// verify that extracted repo refs match backup.refs index after extraction // verify that extracted repo refs match backup.refs index after extraction
x_ref_list := xgit("--git-dir="+p.repopath, x_ref_list := xgit(ctx, "--git-dir="+p.repopath,
"for-each-ref", "--format=%(objectname) %(refname)") "for-each-ref", "--format=%(objectname) %(refname)")
repo_refs := p.refs.Values() repo_refs := p.refs.Values()
sort.Sort(ByRefname(repo_refs)) sort.Sort(ByRefname(repo_refs))
...@@ -1114,7 +1114,7 @@ func cmd_restore_(gb *git.Repository, HEAD_ string, restorespecv []RestoreSpec) ...@@ -1114,7 +1114,7 @@ func cmd_restore_(gb *git.Repository, HEAD_ string, restorespecv []RestoreSpec)
// //
// Compared to fsck we do not re-compute sha1 sum of objects which // Compared to fsck we do not re-compute sha1 sum of objects which
// is significantly faster. // is significantly faster.
gerr, _, _ := ggit("--git-dir="+p.repopath, gerr, _, _ := ggit(ctx, "--git-dir="+p.repopath,
"rev-list", "--objects", "--stdin", "--quiet", RunWith{stdin: p.refs.Sha1HeadsStr()}) "rev-list", "--objects", "--stdin", "--quiet", RunWith{stdin: p.refs.Sha1HeadsStr()})
if gerr != nil { if gerr != nil {
fmt.Fprintln(os.Stderr, "E: Problem while checking connectivity of extracted repo:") fmt.Fprintln(os.Stderr, "E: Problem while checking connectivity of extracted repo:")
...@@ -1131,36 +1131,21 @@ func cmd_restore_(gb *git.Repository, HEAD_ string, restorespecv []RestoreSpec) ...@@ -1131,36 +1131,21 @@ func cmd_restore_(gb *git.Repository, HEAD_ string, restorespecv []RestoreSpec)
// RunWith{stdout: gitprogress(), stderr: gitprogress()}) // RunWith{stdout: gitprogress(), stderr: gitprogress()})
} }
} }
}() })
}
// wait for workers to finish & collect/reraise their errors
go func() {
wg.Wait()
close(errch)
}()
ev := xerr.Errorv{}
for e := range errch {
// tell everything to stop on first error
if len(ev) == 0 {
close(stopch)
}
ev = append(ev, e)
} }
if len(ev) != 0 { // wait for workers to finish & collect/reraise first error, if any
exc.Raise(ev) err = wg.Wait()
} exc.Raiseif(err)
} }
// loadBackupRefs loads 'backup.ref' content from a git object. // loadBackupRefs loads 'backup.ref' content from a git object.
// //
// an example of object is e.g. "HEAD:backup.ref". // an example of object is e.g. "HEAD:backup.ref".
func loadBackupRefs(object string) (repotab map[string]*BackupRepo, err error) { func loadBackupRefs(ctx context.Context, object string) (repotab map[string]*BackupRepo, err error) {
defer xerr.Contextf(&err, "load backup.refs %q", object) defer xerr.Contextf(&err, "load backup.refs %q", object)
gerr, backup_refs, _ := ggit("cat-file", "blob", object) gerr, backup_refs, _ := ggit(ctx, "cat-file", "blob", object)
if gerr != nil { if gerr != nil {
return nil, gerr return nil, gerr
} }
...@@ -1199,7 +1184,7 @@ func loadBackupRefs(object string) (repotab map[string]*BackupRepo, err error) { ...@@ -1199,7 +1184,7 @@ func loadBackupRefs(object string) (repotab map[string]*BackupRepo, err error) {
return repotab, nil return repotab, nil
} }
var commands = map[string]func(*git.Repository, []string){ var commands = map[string]func(context.Context, *git.Repository, []string){
"pull": cmd_pull, "pull": cmd_pull,
"restore": cmd_restore, "restore": cmd_restore,
} }
...@@ -1256,9 +1241,22 @@ func main() { ...@@ -1256,9 +1241,22 @@ func main() {
os.Exit(1) os.Exit(1)
}) })
// cancel what we'll do on SIGINT | SIGTERM
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
sigq := make(chan os.Signal, 1)
signal.Notify(sigq, os.Interrupt, syscall.SIGTERM)
go func() {
select {
case <-ctx.Done():
case <-sigq:
cancel()
}
}()
// backup repository // backup repository
gb, err := git.OpenRepository(".") gb, err := git.OpenRepository(".")
exc.Raiseif(err) exc.Raiseif(err)
cmd(gb, argv[1:]) cmd(ctx, gb, argv[1:])
} }
git-backup_test.go
View file @ 80a6e9b0
// Copyright (C) 2015-2016 Nexedi SA and Contributors. // Copyright (C) 2015-2020 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com> // Kirill Smelkov <kirr@nexedi.com>
// //
// This program is free software: you can Use, Study, Modify and Redistribute // This program is free software: you can Use, Study, Modify and Redistribute
...@@ -20,6 +20,7 @@ ...@@ -20,6 +20,7 @@
package main package main
import ( import (
"context"
"fmt" "fmt"
"io/ioutil" "io/ioutil"
"os" "os"
...@@ -68,9 +69,16 @@ func xgittype(s string) git.ObjectType { ...@@ -68,9 +69,16 @@ func xgittype(s string) git.ObjectType {
return type_ return type_
} }
// xnoref asserts that git reference ref does not exists.
func xnoref(ref string) {
xgit(context.Background(), "update-ref", "--stdin", RunWith{stdin: fmt.Sprintf("verify refs/%s %s\n", ref, Sha1{})})
}
// verify end-to-end pull-restore // verify end-to-end pull-restore
func TestPullRestore(t *testing.T) { func TestPullRestore(t *testing.T) {
ctx := context.Background()
// if something raises -> don't let testing panic - report it as proper error with context. // if something raises -> don't let testing panic - report it as proper error with context.
here := my.FuncName() here := my.FuncName()
defer exc.Catch(func(e *exc.Error) { defer exc.Catch(func(e *exc.Error) {
...@@ -109,7 +117,7 @@ func TestPullRestore(t *testing.T) { ...@@ -109,7 +117,7 @@ func TestPullRestore(t *testing.T) {
} }
// init backup repository // init backup repository
xgit("init", "--bare", "backup.git") xgit(ctx, "init", "--bare", "backup.git")
xchdir(t, "backup.git") xchdir(t, "backup.git")
gb, err := git.OpenRepository(".") gb, err := git.OpenRepository(".")
if err != nil { if err != nil {
...@@ -118,10 +126,10 @@ func TestPullRestore(t *testing.T) { ...@@ -118,10 +126,10 @@ func TestPullRestore(t *testing.T) {
// pull from testdata // pull from testdata
my0 := mydir + "/testdata/0" my0 := mydir + "/testdata/0"
cmd_pull(gb, []string{my0 + ":b0"}) // only empty repo in testdata/0 cmd_pull(ctx, gb, []string{my0 + ":b0"}) // only empty repo in testdata/0
my1 := mydir + "/testdata/1" my1 := mydir + "/testdata/1"
cmd_pull(gb, []string{my1 + ":b1"}) cmd_pull(ctx, gb, []string{my1 + ":b1"})
// verify tag/tree/blob encoding is 1) consistent and 2) always the same. // verify tag/tree/blob encoding is 1) consistent and 2) always the same.
// we need it be always the same so different git-backup versions can // we need it be always the same so different git-backup versions can
...@@ -153,8 +161,8 @@ func TestPullRestore(t *testing.T) { ...@@ -153,8 +161,8 @@ func TestPullRestore(t *testing.T) {
} }
// encoding original object should give sha1_ // encoding original object should give sha1_
obj_type := xgit("cat-file", "-t", nc.sha1) obj_type := xgit(ctx, "cat-file", "-t", nc.sha1)
sha1_ := obj_represent_as_commit(gb, nc.sha1, xgittype(obj_type)) sha1_ := obj_represent_as_commit(ctx, gb, nc.sha1, xgittype(obj_type))
if sha1_ != nc.sha1_ { if sha1_ != nc.sha1_ {
t.Fatalf("encode %s -> %s ; want %s", sha1, sha1_, nc.sha1_) t.Fatalf("encode %s -> %s ; want %s", sha1, sha1_, nc.sha1_)
} }
...@@ -176,10 +184,10 @@ func TestPullRestore(t *testing.T) { ...@@ -176,10 +184,10 @@ func TestPullRestore(t *testing.T) {
} }
// prune all non-reachable objects (e.g. tags just pulled - they were encoded as commits) // prune all non-reachable objects (e.g. tags just pulled - they were encoded as commits)
xgit("prune") xgit(ctx, "prune")
// verify backup repo is all ok // verify backup repo is all ok
xgit("fsck") xgit(ctx, "fsck")
// verify that just pulled tag objects are now gone after pruning - // verify that just pulled tag objects are now gone after pruning -
// - they become not directly git-present. The only possibility to // - they become not directly git-present. The only possibility to
...@@ -188,7 +196,7 @@ func TestPullRestore(t *testing.T) { ...@@ -188,7 +196,7 @@ func TestPullRestore(t *testing.T) {
if !nc.istag { if !nc.istag {
continue continue
} }
gerr, _, _ := ggit("cat-file", "-p", nc.sha1) gerr, _, _ := ggit(ctx, "cat-file", "-p", nc.sha1)
if gerr == nil { if gerr == nil {
t.Fatalf("tag %s still present in backup.git after git-prune", nc.sha1) t.Fatalf("tag %s still present in backup.git after git-prune", nc.sha1)
} }
...@@ -205,14 +213,14 @@ func TestPullRestore(t *testing.T) { ...@@ -205,14 +213,14 @@ func TestPullRestore(t *testing.T) {
afterPull() afterPull()
// pull again - it should be noop // pull again - it should be noop
h1 := xgitSha1("rev-parse", "HEAD") h1 := xgitSha1(ctx, "rev-parse", "HEAD")
cmd_pull(gb, []string{my1 + ":b1"}) cmd_pull(ctx, gb, []string{my1 + ":b1"})
afterPull() afterPull()
h2 := xgitSha1("rev-parse", "HEAD") h2 := xgitSha1(ctx, "rev-parse", "HEAD")
if h1 == h2 { if h1 == h2 {
t.Fatal("pull: second run did not ajusted HEAD") t.Fatal("pull: second run did not ajusted HEAD")
} }
δ12 := xgit("diff", h1, h2) δ12 := xgit(ctx, "diff", h1, h2)
if δ12 != "" { if δ12 != "" {
t.Fatalf("pull: second run was not noop: δ:\n%s", δ12) t.Fatalf("pull: second run was not noop: δ:\n%s", δ12)
} }
...@@ -220,10 +228,10 @@ func TestPullRestore(t *testing.T) { ...@@ -220,10 +228,10 @@ func TestPullRestore(t *testing.T) {
// restore backup // restore backup
work1 := workdir + "/1" work1 := workdir + "/1"
cmd_restore(gb, []string{"HEAD", "b1:" + work1}) cmd_restore(ctx, gb, []string{"HEAD", "b1:" + work1})
// verify files restored to the same as original // verify files restored to the same as original
gerr, diff, _ := ggit("diff", "--no-index", "--raw", "--exit-code", my1, work1) gerr, diff, _ := ggit(ctx, "diff", "--no-index", "--raw", "--exit-code", my1, work1)
// 0 - no diff, 1 - has diff, 2 - problem // 0 - no diff, 1 - has diff, 2 - problem
if gerr != nil && gerr.Sys().(syscall.WaitStatus).ExitStatus() > 1 { if gerr != nil && gerr.Sys().(syscall.WaitStatus).ExitStatus() > 1 {
t.Fatal(gerr) t.Fatal(gerr)
...@@ -262,12 +270,12 @@ func TestPullRestore(t *testing.T) { ...@@ -262,12 +270,12 @@ func TestPullRestore(t *testing.T) {
for _, repo := range R { for _, repo := range R {
// fsck just in case // fsck just in case
xgit("--git-dir="+repo.path, "fsck") xgit(ctx, "--git-dir="+repo.path, "fsck")
// NOTE for-each-ref sorts output by refname // NOTE for-each-ref sorts output by refname
repo.reflist = xgit("--git-dir="+repo.path, "for-each-ref") repo.reflist = xgit(ctx, "--git-dir="+repo.path, "for-each-ref")
// NOTE rev-list emits objects in reverse chronological order, // NOTE rev-list emits objects in reverse chronological order,
// starting from refs roots which are also ordered by refname // starting from refs roots which are also ordered by refname
repo.revlist = xgit("--git-dir="+repo.path, "rev-list", "--all", "--objects") repo.revlist = xgit(ctx, "--git-dir="+repo.path, "rev-list", "--all", "--objects")
} }
if R[0].reflist != R[1].reflist { if R[0].reflist != R[1].reflist {
...@@ -292,11 +300,11 @@ func TestPullRestore(t *testing.T) { ...@@ -292,11 +300,11 @@ func TestPullRestore(t *testing.T) {
defer exc.Catch(func(e *exc.Error) { defer exc.Catch(func(e *exc.Error) {
// it ok - pull should raise // it ok - pull should raise
// git-backup leaves backup repo locked on error // git-backup should not leave backup repo locked on error
xgit("update-ref", "-d", "refs/backup.locked") xnoref("backup.locked")
}) })
cmd_pull(gb, []string{my2 + ":b2"}) cmd_pull(ctx, gb, []string{my2 + ":b2"})
t.Fatal("pull corrupt.git: did not complain") t.Fatal("pull corrupt.git: did not complain")
}() }()
...@@ -318,8 +326,8 @@ func TestPullRestore(t *testing.T) { ...@@ -318,8 +326,8 @@ func TestPullRestore(t *testing.T) {
t.Fatalf("pull incomplete-send-pack.git/%s: complained, but error is wrong:\n%s\nerror: %s", kind, bad, estr) t.Fatalf("pull incomplete-send-pack.git/%s: complained, but error is wrong:\n%s\nerror: %s", kind, bad, estr)
} }
// git-backup leaves backup repo locked on error // git-backup should not leave backup repo locked on error
xgit("update-ref", "-d", "refs/backup.locked") xnoref("backup.locked")
}) })
// for incomplete-send-pack.git to indeed send incomplete pack, its git // for incomplete-send-pack.git to indeed send incomplete pack, its git
...@@ -336,7 +344,7 @@ func TestPullRestore(t *testing.T) { ...@@ -336,7 +344,7 @@ func TestPullRestore(t *testing.T) {
err = os.Setenv("HOME", my3+"/incomplete-send-pack.git/"+kind) err = os.Setenv("HOME", my3+"/incomplete-send-pack.git/"+kind)
exc.Raiseif(err) exc.Raiseif(err)
cmd_pull(gb, []string{my3 + ":b3"}) cmd_pull(ctx, gb, []string{my3 + ":b3"})
t.Fatalf("pull incomplete-send-pack.git/%s: did not complain", kind) t.Fatalf("pull incomplete-send-pack.git/%s: did not complain", kind)
} }
...@@ -353,7 +361,7 @@ func TestPullRestore(t *testing.T) { ...@@ -353,7 +361,7 @@ func TestPullRestore(t *testing.T) {
// pulling incomplete-send-pack.git without pack-objects hook must succeed: // pulling incomplete-send-pack.git without pack-objects hook must succeed:
// without $HOME tweaks full and complete pack is sent. // without $HOME tweaks full and complete pack is sent.
cmd_pull(gb, []string{my3 + ":b3"}) cmd_pull(ctx, gb, []string{my3 + ":b3"})
} }
func TestRepoRefSplit(t *testing.T) { func TestRepoRefSplit(t *testing.T) {
......
git.go
View file @ 80a6e9b0
// Copyright (C) 2015-2016 Nexedi SA and Contributors. // Copyright (C) 2015-2020 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com> // Kirill Smelkov <kirr@nexedi.com>
// //
// This program is free software: you can Use, Study, Modify and Redistribute // This program is free software: you can Use, Study, Modify and Redistribute
...@@ -22,6 +22,7 @@ package main ...@@ -22,6 +22,7 @@ package main
import ( import (
"bytes" "bytes"
"context"
"fmt" "fmt"
"os" "os"
"os/exec" "os/exec"
...@@ -48,18 +49,20 @@ type RunWith struct { ...@@ -48,18 +49,20 @@ type RunWith struct {
} }
// run `git *argv` -> error, stdout, stderr // run `git *argv` -> error, stdout, stderr
func _git(argv []string, ctx RunWith) (err error, stdout, stderr string) { func _git(ctx context.Context, argv []string, rctx RunWith) (err error, stdout, stderr string) {
debugf("git %s", strings.Join(argv, " ")) debugf("git %s", strings.Join(argv, " "))
cmd := exec.Command("git", argv...) // XXX exec.CommandContext does `kill -9` on ctx cancel
// XXX -> rework to `kill -TERM` so that spawned process can finish cleanly?
cmd := exec.CommandContext(ctx, "git", argv...)
stdoutBuf := bytes.Buffer{} stdoutBuf := bytes.Buffer{}
stderrBuf := bytes.Buffer{} stderrBuf := bytes.Buffer{}
if ctx.stdin != "" { if rctx.stdin != "" {
cmd.Stdin = strings.NewReader(ctx.stdin) cmd.Stdin = strings.NewReader(rctx.stdin)
} }
switch ctx.stdout { switch rctx.stdout {
case PIPE: case PIPE:
cmd.Stdout = &stdoutBuf cmd.Stdout = &stdoutBuf
case DontRedirect: case DontRedirect:
...@@ -68,7 +71,7 @@ func _git(argv []string, ctx RunWith) (err error, stdout, stderr string) { ...@@ -68,7 +71,7 @@ func _git(argv []string, ctx RunWith) (err error, stdout, stderr string) {
panic("git: stdout redirect mode invalid") panic("git: stdout redirect mode invalid")
} }
switch ctx.stderr { switch rctx.stderr {
case PIPE: case PIPE:
cmd.Stderr = &stderrBuf cmd.Stderr = &stderrBuf
case DontRedirect: case DontRedirect:
...@@ -77,9 +80,9 @@ func _git(argv []string, ctx RunWith) (err error, stdout, stderr string) { ...@@ -77,9 +80,9 @@ func _git(argv []string, ctx RunWith) (err error, stdout, stderr string) {
panic("git: stderr redirect mode invalid") panic("git: stderr redirect mode invalid")
} }
if ctx.env != nil { if rctx.env != nil {
env := []string{} env := []string{}
for k, v := range ctx.env { for k, v := range rctx.env {
env = append(env, k+"="+v) env = append(env, k+"="+v)
} }
cmd.Env = env cmd.Env = env
...@@ -89,7 +92,7 @@ func _git(argv []string, ctx RunWith) (err error, stdout, stderr string) { ...@@ -89,7 +92,7 @@ func _git(argv []string, ctx RunWith) (err error, stdout, stderr string) {
stdout = mem.String(stdoutBuf.Bytes()) stdout = mem.String(stdoutBuf.Bytes())
stderr = mem.String(stderrBuf.Bytes()) stderr = mem.String(stderrBuf.Bytes())
if !ctx.raw { if !rctx.raw {
// prettify stdout (e.g. so that 'sha1\n' becomes 'sha1' and can be used directly // prettify stdout (e.g. so that 'sha1\n' becomes 'sha1' and can be used directly
stdout = strings.TrimSpace(stdout) stdout = strings.TrimSpace(stdout)
stderr = strings.TrimSpace(stderr) stderr = strings.TrimSpace(stderr)
...@@ -138,9 +141,9 @@ func (e *GitErrContext) Error() string { ...@@ -138,9 +141,9 @@ func (e *GitErrContext) Error() string {
return msg return msg
} }
// argv -> []string, ctx (for passing argv + RunWith handy - see ggit() for details) // ctx, argv -> ctx, []string, rctx (for passing argv + RunWith handy - see ggit() for details)
func _gitargv(argv ...interface{}) (argvs []string, ctx RunWith) { func _gitargv(ctx context.Context, argv ...interface{}) (_ context.Context, argvs []string, rctx RunWith) {
ctx_seen := false rctx_seen := false
for _, arg := range argv { for _, arg := range argv {
switch arg := arg.(type) { switch arg := arg.(type) {
...@@ -149,47 +152,47 @@ func _gitargv(argv ...interface{}) (argvs []string, ctx RunWith) { ...@@ -149,47 +152,47 @@ func _gitargv(argv ...interface{}) (argvs []string, ctx RunWith) {
default: default:
argvs = append(argvs, fmt.Sprint(arg)) argvs = append(argvs, fmt.Sprint(arg))
case RunWith: case RunWith:
if ctx_seen { if rctx_seen {
panic("git: multiple RunWith contexts") panic("git: multiple RunWith contexts")
} }
ctx, ctx_seen = arg, true rctx, rctx_seen = arg, true
} }
} }
return argvs, ctx return ctx, argvs, rctx
} }
// run `git *argv` -> err, stdout, stderr // run `git *argv` -> err, stdout, stderr
// - arguments are automatically converted to strings // - arguments are automatically converted to strings
// - RunWith argument is passed as ctx // - RunWith argument is passed as rctx
// - error is returned only when git command could run and exits with error status // - error is returned only when git command could run and exits with error status
// - on other errors - exception is raised // - on other errors - exception is raised
// //
// NOTE err is concrete *GitError, not error // NOTE err is concrete *GitError, not error
func ggit(argv ...interface{}) (err *GitError, stdout, stderr string) { func ggit(ctx context.Context, argv ...interface{}) (err *GitError, stdout, stderr string) {
return ggit2(_gitargv(argv...)) return ggit2(_gitargv(ctx, argv...))
} }
func ggit2(argv []string, ctx RunWith) (err *GitError, stdout, stderr string) { func ggit2(ctx context.Context, argv []string, rctx RunWith) (err *GitError, stdout, stderr string) {
e, stdout, stderr := _git(argv, ctx) e, stdout, stderr := _git(ctx, argv, rctx)
eexec, _ := e.(*exec.ExitError) eexec, _ := e.(*exec.ExitError)
if e != nil && eexec == nil { if e != nil && eexec == nil {
exc.Raisef("git %s : %s", strings.Join(argv, " "), e) exc.Raisef("git %s : %s", strings.Join(argv, " "), e)
} }
if eexec != nil { if eexec != nil {
err = &GitError{GitErrContext{argv, ctx.stdin, stdout, stderr}, eexec} err = &GitError{GitErrContext{argv, rctx.stdin, stdout, stderr}, eexec}
} }
return err, stdout, stderr return err, stdout, stderr
} }
// run `git *argv` -> stdout // run `git *argv` -> stdout
// on error - raise exception // on error - raise exception
func xgit(argv ...interface{}) string { func xgit(ctx context.Context, argv ...interface{}) string {
return xgit2(_gitargv(argv...)) return xgit2(_gitargv(ctx, argv...))
} }
func xgit2(argv []string, ctx RunWith) string { func xgit2(ctx context.Context, argv []string, rctx RunWith) string {
gerr, stdout, _ := ggit2(argv, ctx) gerr, stdout, _ := ggit2(ctx, argv, rctx)
if gerr != nil { if gerr != nil {
exc.Raise(gerr) exc.Raise(gerr)
} }
...@@ -197,8 +200,8 @@ func xgit2(argv []string, ctx RunWith) string { ...@@ -197,8 +200,8 @@ func xgit2(argv []string, ctx RunWith) string {
} }
// like xgit(), but automatically parse stdout to Sha1 // like xgit(), but automatically parse stdout to Sha1
func xgitSha1(argv ...interface{}) Sha1 { func xgitSha1(ctx context.Context, argv ...interface{}) Sha1 {
return xgit2Sha1(_gitargv(argv...)) return xgit2Sha1(_gitargv(ctx, argv...))
} }
// error when git output is not valid sha1 // error when git output is not valid sha1
...@@ -212,14 +215,14 @@ func (e *GitSha1Error) Error() string { ...@@ -212,14 +215,14 @@ func (e *GitSha1Error) Error() string {
return msg return msg
} }
func xgit2Sha1(argv []string, ctx RunWith) Sha1 { func xgit2Sha1(ctx context.Context, argv []string, rctx RunWith) Sha1 {
gerr, stdout, stderr := ggit2(argv, ctx) gerr, stdout, stderr := ggit2(ctx, argv, rctx)
if gerr != nil { if gerr != nil {
exc.Raise(gerr) exc.Raise(gerr)
} }
sha1, err := Sha1Parse(stdout) sha1, err := Sha1Parse(stdout)
if err != nil { if err != nil {
exc.Raise(&GitSha1Error{GitErrContext{argv, ctx.stdin, stdout, stderr}}) exc.Raise(&GitSha1Error{GitErrContext{argv, rctx.stdin, stdout, stderr}})
} }
return sha1 return sha1
} }
gitobjects.go
View file @ 80a6e9b0
// Copyright (C) 2015-2016 Nexedi SA and Contributors. // Copyright (C) 2015-2020 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com> // Kirill Smelkov <kirr@nexedi.com>
// //
// This program is free software: you can Use, Study, Modify and Redistribute // This program is free software: you can Use, Study, Modify and Redistribute
...@@ -21,6 +21,7 @@ package main ...@@ -21,6 +21,7 @@ package main
// Git-backup | Git object: Blob Tree Commit Tag // Git-backup | Git object: Blob Tree Commit Tag
import ( import (
"context"
"errors" "errors"
"fmt" "fmt"
"os" "os"
...@@ -163,9 +164,9 @@ func (e *InvalidLstreeEntry) Error() string { ...@@ -163,9 +164,9 @@ func (e *InvalidLstreeEntry) Error() string {
// create empty git tree -> tree sha1 // create empty git tree -> tree sha1
var tree_empty Sha1 var tree_empty Sha1
func mktree_empty() Sha1 { func mktree_empty(ctx context.Context) Sha1 {
if tree_empty.IsNull() { if tree_empty.IsNull() {
tree_empty = xgitSha1("mktree", RunWith{stdin: ""}) tree_empty = xgitSha1(ctx, "mktree", RunWith{stdin: ""})
} }
return tree_empty return tree_empty
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7