Skip to content

G
git-backup
Commit ff2f0b67 authored by Kirill Smelkov's avatar Kirill Smelkov
Browse files
Options

restore: Extract packs in multiple workers 

This way it allows us to leverage multiple CPUs on a system for pack
extractions, which are computation-heavy operations.

The way to do is more-or-less classical:

    - main worker prepares requests for pack extraction jobs

    - there are multiple pack-extraction workers, which read requests
      from jobs queue and perform them

    - at the end we wait for everything to stop, collect errors and
      optionally signalling the whole thing to cancel if we see an error
      coming. (it is only a signal and we still have to wait for
      everything to stop)

The default number of workers is N(CPU) on the system - because we spawn
separate `git pack-objects ...` for every request.

We also now explicitly limit N(CPU) each `git pack-objects ...` can use
to 1. This way control how many resources to use is in git-backup hand
and also git packs better this way (when only using 1 thread) because
when deltifying all objects are considered to each other, not only all
objects inside 1 thread's object poll, and even when pack.threads is not
1, first "objects counting" phase of pack is serial - wasting all but 1
core.

On lab.nexedi.com we already use pack.threads=1 by default in global
gitconfig, but the above change is for code to be universal.

Time to restore nexedi/ from lab.nexedi.com backup:

2CPU laptop:

    before (pack.threads=1)     10m11s
    before (pack.threads=NCPU)   9m13s
    after  -j1                  10m11s
    after                        6m17s

8CPU system (with other load present, noisy) :

    before (pack.threads=1)     ~5m
    after                       ~1m30s
parent 6c2abbbf
Hide whitespace changes
Inline Side-by-side
Showing with 202 additions and 100 deletions
error.go
View file @ ff2f0b67
...@@ -251,3 +251,18 @@ func erraddcallingcontext(topfunc string, e *Error) *Error { ...@@ -251,3 +251,18 @@ func erraddcallingcontext(topfunc string, e *Error) *Error {
return e return e
} }
// error merging multiple errors (e.g. after collecting them from several parallel workers)
type Errorv []error
func (ev Errorv) Error() string {
if len(ev) == 1 {
return ev[0].Error()
}
msg := fmt.Sprintf("%d errors:\n", len(ev))
for _, e := range ev {
msg += fmt.Sprintf("\t- %s\n", e)
}
return msg
}
git-backup.go
View file @ ff2f0b67
...@@ -67,9 +67,11 @@ import ( ...@@ -67,9 +67,11 @@ import (
"os" "os"
pathpkg "path" pathpkg "path"
"path/filepath" "path/filepath"
"runtime"
"runtime/debug" "runtime/debug"
"sort" "sort"
"strings" "strings"
"sync"
"syscall" "syscall"
"time" "time"
...@@ -106,6 +108,9 @@ func debugf(format string, a ...interface{}) { ...@@ -106,6 +108,9 @@ func debugf(format string, a ...interface{}) {
} }
} }
// how many max jobs to spawn
var njobs = runtime.NumCPU()
// -------- create/extract blob -------- // -------- create/extract blob --------
// file -> blob_sha1, mode // file -> blob_sha1, mode
...@@ -675,6 +680,15 @@ func (br ByRepoPath) Search(prefix string) int { ...@@ -675,6 +680,15 @@ func (br ByRepoPath) Search(prefix string) int {
}) })
} }
// request to extract a pack
type PackExtractReq struct {
refs RefMap // extract pack with objects from this heads
repopath string // into repository located here
// for info only: request was generated restoring from under this backup prefix
prefix string
}
func cmd_restore_(gb *git.Repository, HEAD_ string, restorespecv []RestoreSpec) { func cmd_restore_(gb *git.Repository, HEAD_ string, restorespecv []RestoreSpec) {
HEAD := xgitSha1("rev-parse", "--verify", HEAD_) HEAD := xgitSha1("rev-parse", "--verify", HEAD_)
...@@ -722,120 +736,191 @@ func cmd_restore_(gb *git.Repository, HEAD_ string, restorespecv []RestoreSpec) ...@@ -722,120 +736,191 @@ func cmd_restore_(gb *git.Repository, HEAD_ string, restorespecv []RestoreSpec)
// repotab no longer needed // repotab no longer needed
repotab = nil repotab = nil
// walk over specified prefixes restoring files and packs in *.git packxq := make(chan PackExtractReq, 2*njobs) // requests to extract packs
for _, __ := range restorespecv { errch := make(chan error) // errors from workers
prefix, dir := __.prefix, __.dir stopch := make(chan struct{}) // broadcasts restore has to be cancelled
wg := sync.WaitGroup{}
// main worker: walk over specified prefixes restoring files and
// scheduling pack extraction requests from *.git -> packxq
wg.Add(1)
go func() {
defer wg.Done()
defer close(packxq)
// raised err -> errch
here := myfuncname()
defer errcatch(func(e *Error) {
errch <- erraddcallingcontext(here, e)
})
// ensure dir did not exist before restore run runloop:
err := os.Mkdir(dir, 0777) for _, __ := range restorespecv {
raiseif(err) prefix, dir := __.prefix, __.dir
// files // ensure dir did not exist before restore run
lstree := xgit("ls-tree", "--full-tree", "-r", "-z", "--", HEAD, prefix, RunWith{raw: true}) err := os.Mkdir(dir, 0777)
repos_seen := StrSet{} // dirs of *.git seen while restoring files raiseif(err)
for _, __ := range strings.Split(lstree, "\x00") {
if __ == "" {
continue // last empty line after last \0
}
mode, type_, sha1, filename, err := parse_lstree_entry(__)
// NOTE
// - `ls-tree -r` shows only leaf objects
// - git-backup repository does not have submodules and the like
// -> type should be "blob" only
if err != nil || type_ != "blob" {
raisef("%s: invalid/unexpected ls-tree entry %q", HEAD, __)
}
filename = reprefix(prefix, dir, filename) // files
infof("# file %s\t-> %s", prefix, filename) lstree := xgit("ls-tree", "--full-tree", "-r", "-z", "--", HEAD, prefix, RunWith{raw: true})
blob_to_file(gb, sha1, mode, filename) repos_seen := StrSet{} // dirs of *.git seen while restoring files
for _, __ := range strings.Split(lstree, "\x00") {
// make sure git will recognize *.git as repo: if __ == "" {
// - it should have refs/{heads,tags}/ and objects/pack/ inside. continue // last empty line after last \0
// }
// NOTE doing it while restoring files, because a repo could be mode, type_, sha1, filename, err := parse_lstree_entry(__)
// empty - without refs at all, and thus next "git packs restore" // NOTE
// step will not be run for it. // - `ls-tree -r` shows only leaf objects
filedir := pathpkg.Dir(filename) // - git-backup repository does not have submodules and the like
if strings.HasSuffix(filedir, ".git") && !repos_seen.Contains(filedir) { // -> type should be "blob" only
infof("# repo %s\t-> %s", prefix, filedir) if err != nil || type_ != "blob" {
for _, __ := range []string{"refs/heads", "refs/tags", "objects/pack"} { raisef("%s: invalid/unexpected ls-tree entry %q", HEAD, __)
err := os.MkdirAll(filedir+"/"+__, 0777)
raiseif(err)
} }
repos_seen.Add(filedir)
}
}
// git packs filename = reprefix(prefix, dir, filename)
for i := ByRepoPath(repov).Search(prefix); i < len(repov); i++ { infof("# file %s\t-> %s", prefix, filename)
repo := repov[i] blob_to_file(gb, sha1, mode, filename)
if !strings.HasPrefix(repo.repopath, prefix) {
break // repov is sorted - end of repositories with prefix // make sure git will recognize *.git as repo:
// - it should have refs/{heads,tags}/ and objects/pack/ inside.
//
// NOTE doing it while restoring files, because a repo could be
// empty - without refs at all, and thus next "git packs restore"
// step will not be run for it.
filedir := pathpkg.Dir(filename)
if strings.HasSuffix(filedir, ".git") && !repos_seen.Contains(filedir) {
infof("# repo %s\t-> %s", prefix, filedir)
for _, __ := range []string{"refs/heads", "refs/tags", "objects/pack"} {
err := os.MkdirAll(filedir+"/"+__, 0777)
raiseif(err)
}
repos_seen.Add(filedir)
}
} }
repopath := reprefix(prefix, dir, repo.repopath) // git packs
infof("# git %s\t-> %s", prefix, repopath) for i := ByRepoPath(repov).Search(prefix); i < len(repov); i++ {
repo := repov[i]
if !strings.HasPrefix(repo.repopath, prefix) {
break // repov is sorted - end of repositories with prefix
}
// make sure tag/tree/blob objects represented as commits are
// present, before we generate pack for restored repo.
// ( such objects could be lost e.g. after backup repo repack as they
// are not reachable from backup repo HEAD )
for _, __ := range repo.refs {
if __.sha1 != __.sha1_ {
obj_recreate_from_commit(gb, __.sha1_)
}
}
select {
case packxq <- PackExtractReq{refs: repo.refs,
repopath: reprefix(prefix, dir, repo.repopath),
prefix: prefix}:
// make sure tag/tree/blob objects represented as commits are case <-stopch:
// present, before we generate pack for restored repo. break runloop
// ( such objects could be lost e.g. after backup repo repack as they
// are not reachable from backup repo HEAD )
for _, __ := range repo.refs {
if __.sha1 != __.sha1_ {
obj_recreate_from_commit(gb, __.sha1_)
} }
} }
}
}()
// pack workers: packxq -> extract packs
for i := 0; i < njobs; i++ {
wg.Add(1)
go func() {
defer wg.Done()
// raised err -> errch
here := myfuncname()
defer errcatch(func(e *Error) {
errch <- erraddcallingcontext(here, e)
})
// extract pack for that repo from big backup pack + decoded tags runloop:
pack_argv := []string{ for {
"pack-objects", select {
"--revs", // include all objects referencable from input sha1 list case <-stopch:
"--reuse-object", "--reuse-delta", "--delta-base-offset"} break runloop
if verbose <= 0 {
pack_argv = append(pack_argv, "-q") case p, ok := <-packxq:
} if !ok {
pack_argv = append(pack_argv, repopath+"/objects/pack/pack") break runloop
}
xgit2(pack_argv, RunWith{stdin: repo.refs.Sha1HeadsStr(), stderr: gitprogress()}) infof("# git %s\t-> %s", p.prefix, p.repopath)
// verify that extracted repo refs match backup.refs index after extraction // extract pack for that repo from big backup pack + decoded tags
x_ref_list := xgit("--git-dir=" + repopath, pack_argv := []string{
"for-each-ref", "--format=%(objectname) %(refname)") "-c", "pack.threads=1", // occupy only 1 CPU + it packs better
repo_refs := repo.refs.Values() "pack-objects",
sort.Sort(ByRefname(repo_refs)) "--revs", // include all objects referencable from input sha1 list
repo_ref_listv := make([]string, 0, len(repo_refs)) "--reuse-object", "--reuse-delta", "--delta-base-offset"}
for _, __ := range repo_refs { if verbose <= 0 {
repo_ref_listv = append(repo_ref_listv, fmt.Sprintf("%s refs/%s", __.sha1, __.refname)) pack_argv = append(pack_argv, "-q")
} }
repo_ref_list := strings.Join(repo_ref_listv, "\n") pack_argv = append(pack_argv, p.repopath+"/objects/pack/pack")
if x_ref_list != repo_ref_list {
raisef("E: extracted %s refs corrupt", repopath) xgit2(pack_argv, RunWith{stdin: p.refs.Sha1HeadsStr(), stderr: gitprogress()})
// verify that extracted repo refs match backup.refs index after extraction
x_ref_list := xgit("--git-dir=" + p.repopath,
"for-each-ref", "--format=%(objectname) %(refname)")
repo_refs := p.refs.Values()
sort.Sort(ByRefname(repo_refs))
repo_ref_listv := make([]string, 0, len(repo_refs))
for _, __ := range repo_refs {
repo_ref_listv = append(repo_ref_listv, fmt.Sprintf("%s refs/%s", __.sha1, __.refname))
}
repo_ref_list := strings.Join(repo_ref_listv, "\n")
if x_ref_list != repo_ref_list {
raisef("E: extracted %s refs corrupt", p.repopath)
}
// check connectivity in recreated repository.
//
// This way we verify that extracted pack indeed contains all
// objects for all refs in the repo.
//
// Compared to fsck we do not re-compute sha1 sum of objects which
// is significantly faster.
gerr, _, _ := ggit("--git-dir=" + p.repopath,
"rev-list", "--objects", "--stdin", "--quiet", RunWith{stdin: p.refs.Sha1HeadsStr()})
if gerr != nil {
fmt.Fprintln(os.Stderr, "E: Problem while checking connectivity of extracted repo:")
raise(gerr)
}
// XXX disabled because it is slow
// // NOTE progress goes to stderr, problems go to stdout
// xgit("--git-dir=" + p.repopath, "fsck",
// # only check that traversal from refs is ok: this unpacks
// # commits and trees and verifies blob objects are there,
// # but do _not_ unpack blobs =fast.
// "--connectivity-only",
// RunWith{stdout: gitprogress(), stderr: gitprogress()})
}
} }
}()
}
// check connectivity in recreated repository. // wait for workers to finish & collect/reraise their errors
// go func() {
// This way we verify that extracted pack indeed contains all wg.Wait()
// objects for all refs in the repo. close(errch)
// }()
// Compared to fsck we do not re-compute sha1 sum of objects which
// is significantly faster.
gerr, _, _ := ggit("--git-dir=" + repopath,
"rev-list", "--objects", "--stdin", "--quiet", RunWith{stdin: repo.refs.Sha1HeadsStr()})
if gerr != nil {
fmt.Fprintln(os.Stderr, "E: Problem while checking connectivity of extracted repo:")
raise(gerr)
}
// XXX disabled because it is slow ev := Errorv{}
// // NOTE progress goes to stderr, problems go to stdout for e := range errch {
// xgit("--git-dir=" + repopath, "fsck", // tell everything to stop on first error
// # only check that traversal from refs is ok: this unpacks if len(ev) == 0 {
// # commits and trees and verifies blob objects are there, close(stopch)
// # but do _not_ unpack blobs =fast.
// "--connectivity-only",
// RunWith{stdout: gitprogress(), stderr: gitprogress()})
} }
ev = append(ev, e)
}
if len(ev) != 0 {
raise(ev)
} }
} }
...@@ -856,7 +941,8 @@ func usage() { ...@@ -856,7 +941,8 @@ func usage() {
-h --help this help text. -h --help this help text.
-v increase verbosity. -v increase verbosity.
-q decrease verbosity. -q decrease verbosity.
`) -j N allow max N jobs to spawn; default=NPROC (%d on this system)
`, njobs)
} }
func main() { func main() {
...@@ -864,6 +950,7 @@ func main() { ...@@ -864,6 +950,7 @@ func main() {
quiet := 0 quiet := 0
flag.Var((*countFlag)(&verbose), "v", "verbosity level") flag.Var((*countFlag)(&verbose), "v", "verbosity level")
flag.Var((*countFlag)(&quiet), "q", "decrease verbosity") flag.Var((*countFlag)(&quiet), "q", "decrease verbosity")
flag.IntVar(&njobs, "j", njobs, "allow max N jobs to spawn")
flag.Parse() flag.Parse()
verbose -= quiet verbose -= quiet
argv := flag.Args() argv := flag.Args()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7