Skip to content

G
git-backup
Commit 80a6e9b0 authored by Kirill Smelkov's avatar Kirill Smelkov
Browse files
Options

Cleanup git-backup lock and gitlab-backup tempporary folder in defer-style 

So that further `git-backup pull` run can proceed, and many gitlab-backup's left `$tmpd` don't eat up disk space.

Rework of kirr/git-backup!4.

/reviewed-on kirr/git-backup!5
parents 9791c04e c835bf38
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 168 additions and 158 deletions
contrib/gitlab-backup
View file @ 80a6e9b0
#!/bin/bash -e
# pull/restore gitlab data into/from git-backup
# Copyright (C) 2015-2016 Nexedi SA and Contributors.
# Copyright (C) 2015-2020 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -110,6 +110,8 @@ backup_pull() {
# 1. dump all gitlab data except repositories & db
echo " * Dumping gitlab data (except repositories & db)"
tmpd=$(mktemp -d `pwd`/gitlab-backup.XXXXXX)
trap 'rm -rf "$tmpd"' EXIT
gitlab-rake gitlab:backup:create SKIP=repositories,db | tee "$tmpd/gitlab_backup_create.out"
backup_tar=`grep "^Creating backup archive: .* done" "$tmpd/gitlab_backup_create.out"` || \
die "E: Cannot detect backup tar"
......@@ -218,7 +220,6 @@ backup_pull() {
# mark backup_tar as pulled and cleanup
mv "$backup_tar" "$backup_tar.pulled"
fi
rm -rf "$tmpd"
echo OK
}
......@@ -235,6 +236,7 @@ backup_restore() {
# 1. extract all gitlab data except repositories
echo " * Extracting gitlab data (except repositories)"
tmpd=$(mktemp -d `pwd`/gitlab-backup.XXXXXX)
trap 'rm -rf "$tmpd"' EXIT
$GIT_BACKUP restore $HEAD gitlab/misc:"$tmpd/gitlab_backup"
backup_info="$tmpd/gitlab_backup/backup_information.yml"
......@@ -308,8 +310,6 @@ backup_restore() {
test -e "$backup_tar" && die "E: $backup_tar already exists"
tar -C "$tmpd/gitlab_backup" -cf "$backup_tar" .
rm -rf "$tmpd" # tmpd no longer needed
# 4. extract repositories into .../repositories.<timestamp>
echo " * Extracting repositories"
reposX="${GITLAB_REPOS_PATH}.${backup_created_at}"
......
git-backup.go
View file @ 80a6e9b0
This diff is collapsed.
git-backup_test.go
View file @ 80a6e9b0
// Copyright (C) 2015-2016 Nexedi SA and Contributors.
// Copyright (C) 2015-2020 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -20,6 +20,7 @@
package main
import (
"context"
"fmt"
"io/ioutil"
"os"
......@@ -68,9 +69,16 @@ func xgittype(s string) git.ObjectType {
return type_
}
// xnoref asserts that git reference ref does not exists.
func xnoref(ref string) {
xgit(context.Background(), "update-ref", "--stdin", RunWith{stdin: fmt.Sprintf("verify refs/%s %s\n", ref, Sha1{})})
}
// verify end-to-end pull-restore
func TestPullRestore(t *testing.T) {
ctx := context.Background()
// if something raises -> don't let testing panic - report it as proper error with context.
here := my.FuncName()
defer exc.Catch(func(e *exc.Error) {
......@@ -109,7 +117,7 @@ func TestPullRestore(t *testing.T) {
}
// init backup repository
xgit("init", "--bare", "backup.git")
xgit(ctx, "init", "--bare", "backup.git")
xchdir(t, "backup.git")
gb, err := git.OpenRepository(".")
if err != nil {
......@@ -118,10 +126,10 @@ func TestPullRestore(t *testing.T) {
// pull from testdata
my0 := mydir + "/testdata/0"
cmd_pull(gb, []string{my0 + ":b0"}) // only empty repo in testdata/0
cmd_pull(ctx, gb, []string{my0 + ":b0"}) // only empty repo in testdata/0
my1 := mydir + "/testdata/1"
cmd_pull(gb, []string{my1 + ":b1"})
cmd_pull(ctx, gb, []string{my1 + ":b1"})
// verify tag/tree/blob encoding is 1) consistent and 2) always the same.
// we need it be always the same so different git-backup versions can
......@@ -153,8 +161,8 @@ func TestPullRestore(t *testing.T) {
}
// encoding original object should give sha1_
obj_type := xgit("cat-file", "-t", nc.sha1)
sha1_ := obj_represent_as_commit(gb, nc.sha1, xgittype(obj_type))
obj_type := xgit(ctx, "cat-file", "-t", nc.sha1)
sha1_ := obj_represent_as_commit(ctx, gb, nc.sha1, xgittype(obj_type))
if sha1_ != nc.sha1_ {
t.Fatalf("encode %s -> %s ; want %s", sha1, sha1_, nc.sha1_)
}
......@@ -176,10 +184,10 @@ func TestPullRestore(t *testing.T) {
}
// prune all non-reachable objects (e.g. tags just pulled - they were encoded as commits)
xgit("prune")
xgit(ctx, "prune")
// verify backup repo is all ok
xgit("fsck")
xgit(ctx, "fsck")
// verify that just pulled tag objects are now gone after pruning -
// - they become not directly git-present. The only possibility to
......@@ -188,7 +196,7 @@ func TestPullRestore(t *testing.T) {
if !nc.istag {
continue
}
gerr, _, _ := ggit("cat-file", "-p", nc.sha1)
gerr, _, _ := ggit(ctx, "cat-file", "-p", nc.sha1)
if gerr == nil {
t.Fatalf("tag %s still present in backup.git after git-prune", nc.sha1)
}
......@@ -205,14 +213,14 @@ func TestPullRestore(t *testing.T) {
afterPull()
// pull again - it should be noop
h1 := xgitSha1("rev-parse", "HEAD")
cmd_pull(gb, []string{my1 + ":b1"})
h1 := xgitSha1(ctx, "rev-parse", "HEAD")
cmd_pull(ctx, gb, []string{my1 + ":b1"})
afterPull()
h2 := xgitSha1("rev-parse", "HEAD")
h2 := xgitSha1(ctx, "rev-parse", "HEAD")
if h1 == h2 {
t.Fatal("pull: second run did not ajusted HEAD")
}
δ12 := xgit("diff", h1, h2)
δ12 := xgit(ctx, "diff", h1, h2)
if δ12 != "" {
t.Fatalf("pull: second run was not noop: δ:\n%s", δ12)
}
......@@ -220,10 +228,10 @@ func TestPullRestore(t *testing.T) {
// restore backup
work1 := workdir + "/1"
cmd_restore(gb, []string{"HEAD", "b1:" + work1})
cmd_restore(ctx, gb, []string{"HEAD", "b1:" + work1})
// verify files restored to the same as original
gerr, diff, _ := ggit("diff", "--no-index", "--raw", "--exit-code", my1, work1)
gerr, diff, _ := ggit(ctx, "diff", "--no-index", "--raw", "--exit-code", my1, work1)
// 0 - no diff, 1 - has diff, 2 - problem
if gerr != nil && gerr.Sys().(syscall.WaitStatus).ExitStatus() > 1 {
t.Fatal(gerr)
......@@ -262,12 +270,12 @@ func TestPullRestore(t *testing.T) {
for _, repo := range R {
// fsck just in case
xgit("--git-dir="+repo.path, "fsck")
xgit(ctx, "--git-dir="+repo.path, "fsck")
// NOTE for-each-ref sorts output by refname
repo.reflist = xgit("--git-dir="+repo.path, "for-each-ref")
repo.reflist = xgit(ctx, "--git-dir="+repo.path, "for-each-ref")
// NOTE rev-list emits objects in reverse chronological order,
// starting from refs roots which are also ordered by refname
repo.revlist = xgit("--git-dir="+repo.path, "rev-list", "--all", "--objects")
repo.revlist = xgit(ctx, "--git-dir="+repo.path, "rev-list", "--all", "--objects")
}
if R[0].reflist != R[1].reflist {
......@@ -292,11 +300,11 @@ func TestPullRestore(t *testing.T) {
defer exc.Catch(func(e *exc.Error) {
// it ok - pull should raise
// git-backup leaves backup repo locked on error
xgit("update-ref", "-d", "refs/backup.locked")
// git-backup should not leave backup repo locked on error
xnoref("backup.locked")
})
cmd_pull(gb, []string{my2 + ":b2"})
cmd_pull(ctx, gb, []string{my2 + ":b2"})
t.Fatal("pull corrupt.git: did not complain")
}()
......@@ -318,8 +326,8 @@ func TestPullRestore(t *testing.T) {
t.Fatalf("pull incomplete-send-pack.git/%s: complained, but error is wrong:\n%s\nerror: %s", kind, bad, estr)
}
// git-backup leaves backup repo locked on error
xgit("update-ref", "-d", "refs/backup.locked")
// git-backup should not leave backup repo locked on error
xnoref("backup.locked")
})
// for incomplete-send-pack.git to indeed send incomplete pack, its git
......@@ -336,7 +344,7 @@ func TestPullRestore(t *testing.T) {
err = os.Setenv("HOME", my3+"/incomplete-send-pack.git/"+kind)
exc.Raiseif(err)
cmd_pull(gb, []string{my3 + ":b3"})
cmd_pull(ctx, gb, []string{my3 + ":b3"})
t.Fatalf("pull incomplete-send-pack.git/%s: did not complain", kind)
}
......@@ -353,7 +361,7 @@ func TestPullRestore(t *testing.T) {
// pulling incomplete-send-pack.git without pack-objects hook must succeed:
// without $HOME tweaks full and complete pack is sent.
cmd_pull(gb, []string{my3 + ":b3"})
cmd_pull(ctx, gb, []string{my3 + ":b3"})
}
func TestRepoRefSplit(t *testing.T) {
......
git.go
View file @ 80a6e9b0
// Copyright (C) 2015-2016 Nexedi SA and Contributors.
// Copyright (C) 2015-2020 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -22,6 +22,7 @@ package main
import (
"bytes"
"context"
"fmt"
"os"
"os/exec"
......@@ -48,18 +49,20 @@ type RunWith struct {
}
// run `git *argv` -> error, stdout, stderr
func _git(argv []string, ctx RunWith) (err error, stdout, stderr string) {
func _git(ctx context.Context, argv []string, rctx RunWith) (err error, stdout, stderr string) {
debugf("git %s", strings.Join(argv, " "))
cmd := exec.Command("git", argv...)
// XXX exec.CommandContext does `kill -9` on ctx cancel
// XXX -> rework to `kill -TERM` so that spawned process can finish cleanly?
cmd := exec.CommandContext(ctx, "git", argv...)
stdoutBuf := bytes.Buffer{}
stderrBuf := bytes.Buffer{}
if ctx.stdin != "" {
cmd.Stdin = strings.NewReader(ctx.stdin)
if rctx.stdin != "" {
cmd.Stdin = strings.NewReader(rctx.stdin)
}
switch ctx.stdout {
switch rctx.stdout {
case PIPE:
cmd.Stdout = &stdoutBuf
case DontRedirect:
......@@ -68,7 +71,7 @@ func _git(argv []string, ctx RunWith) (err error, stdout, stderr string) {
panic("git: stdout redirect mode invalid")
}
switch ctx.stderr {
switch rctx.stderr {
case PIPE:
cmd.Stderr = &stderrBuf
case DontRedirect:
......@@ -77,9 +80,9 @@ func _git(argv []string, ctx RunWith) (err error, stdout, stderr string) {
panic("git: stderr redirect mode invalid")
}
if ctx.env != nil {
if rctx.env != nil {
env := []string{}
for k, v := range ctx.env {
for k, v := range rctx.env {
env = append(env, k+"="+v)
}
cmd.Env = env
......@@ -89,7 +92,7 @@ func _git(argv []string, ctx RunWith) (err error, stdout, stderr string) {
stdout = mem.String(stdoutBuf.Bytes())
stderr = mem.String(stderrBuf.Bytes())
if !ctx.raw {
if !rctx.raw {
// prettify stdout (e.g. so that 'sha1\n' becomes 'sha1' and can be used directly
stdout = strings.TrimSpace(stdout)
stderr = strings.TrimSpace(stderr)
......@@ -138,9 +141,9 @@ func (e *GitErrContext) Error() string {
return msg
}
// argv -> []string, ctx (for passing argv + RunWith handy - see ggit() for details)
func _gitargv(argv ...interface{}) (argvs []string, ctx RunWith) {
ctx_seen := false
// ctx, argv -> ctx, []string, rctx (for passing argv + RunWith handy - see ggit() for details)
func _gitargv(ctx context.Context, argv ...interface{}) (_ context.Context, argvs []string, rctx RunWith) {
rctx_seen := false
for _, arg := range argv {
switch arg := arg.(type) {
......@@ -149,47 +152,47 @@ func _gitargv(argv ...interface{}) (argvs []string, ctx RunWith) {
default:
argvs = append(argvs, fmt.Sprint(arg))
case RunWith:
if ctx_seen {
if rctx_seen {
panic("git: multiple RunWith contexts")
}
ctx, ctx_seen = arg, true
rctx, rctx_seen = arg, true
}
}
return argvs, ctx
return ctx, argvs, rctx
}
// run `git *argv` -> err, stdout, stderr
// - arguments are automatically converted to strings
// - RunWith argument is passed as ctx
// - RunWith argument is passed as rctx
// - error is returned only when git command could run and exits with error status
// - on other errors - exception is raised
//
// NOTE err is concrete *GitError, not error
func ggit(argv ...interface{}) (err *GitError, stdout, stderr string) {
return ggit2(_gitargv(argv...))
func ggit(ctx context.Context, argv ...interface{}) (err *GitError, stdout, stderr string) {
return ggit2(_gitargv(ctx, argv...))
}
func ggit2(argv []string, ctx RunWith) (err *GitError, stdout, stderr string) {
e, stdout, stderr := _git(argv, ctx)
func ggit2(ctx context.Context, argv []string, rctx RunWith) (err *GitError, stdout, stderr string) {
e, stdout, stderr := _git(ctx, argv, rctx)
eexec, _ := e.(*exec.ExitError)
if e != nil && eexec == nil {
exc.Raisef("git %s : %s", strings.Join(argv, " "), e)
}
if eexec != nil {
err = &GitError{GitErrContext{argv, ctx.stdin, stdout, stderr}, eexec}
err = &GitError{GitErrContext{argv, rctx.stdin, stdout, stderr}, eexec}
}
return err, stdout, stderr
}
// run `git *argv` -> stdout
// on error - raise exception
func xgit(argv ...interface{}) string {
return xgit2(_gitargv(argv...))
func xgit(ctx context.Context, argv ...interface{}) string {
return xgit2(_gitargv(ctx, argv...))
}
func xgit2(argv []string, ctx RunWith) string {
gerr, stdout, _ := ggit2(argv, ctx)
func xgit2(ctx context.Context, argv []string, rctx RunWith) string {
gerr, stdout, _ := ggit2(ctx, argv, rctx)
if gerr != nil {
exc.Raise(gerr)
}
......@@ -197,8 +200,8 @@ func xgit2(argv []string, ctx RunWith) string {
}
// like xgit(), but automatically parse stdout to Sha1
func xgitSha1(argv ...interface{}) Sha1 {
return xgit2Sha1(_gitargv(argv...))
func xgitSha1(ctx context.Context, argv ...interface{}) Sha1 {
return xgit2Sha1(_gitargv(ctx, argv...))
}
// error when git output is not valid sha1
......@@ -212,14 +215,14 @@ func (e *GitSha1Error) Error() string {
return msg
}
func xgit2Sha1(argv []string, ctx RunWith) Sha1 {
gerr, stdout, stderr := ggit2(argv, ctx)
func xgit2Sha1(ctx context.Context, argv []string, rctx RunWith) Sha1 {
gerr, stdout, stderr := ggit2(ctx, argv, rctx)
if gerr != nil {
exc.Raise(gerr)
}
sha1, err := Sha1Parse(stdout)
if err != nil {
exc.Raise(&GitSha1Error{GitErrContext{argv, ctx.stdin, stdout, stderr}})
exc.Raise(&GitSha1Error{GitErrContext{argv, rctx.stdin, stdout, stderr}})
}
return sha1
}
gitobjects.go
View file @ 80a6e9b0
// Copyright (C) 2015-2016 Nexedi SA and Contributors.
// Copyright (C) 2015-2020 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -21,6 +21,7 @@ package main
// Git-backup | Git object: Blob Tree Commit Tag
import (
"context"
"errors"
"fmt"
"os"
......@@ -163,9 +164,9 @@ func (e *InvalidLstreeEntry) Error() string {
// create empty git tree -> tree sha1
var tree_empty Sha1
func mktree_empty() Sha1 {
func mktree_empty(ctx context.Context) Sha1 {
if tree_empty.IsNull() {
tree_empty = xgitSha1("mktree", RunWith{stdin: ""})
tree_empty = xgitSha1(ctx, "mktree", RunWith{stdin: ""})
}
return tree_empty
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7