gitlab: changes after review on previous commit (to be squashed)

slapos/recipe/wrapper.py

@@ -37,6 +37,7 @@ class Recipe(GenericBaseRecipe):
         wait_files = self.options.get('wait-for-files')
         environment = self.options.get('environment')
         parameters_extra = self.options.get('parameters-extra')
+        pidfile = self.options.get('pidfile', None)
 
         if not wait_files and not environment:
           # Create a simple wrapper as shell script
@@ -45,6 +46,7 @@ class Recipe(GenericBaseRecipe):
              command=command_line[0],
              parameters=command_line[1:],
              parameters_extra=parameters_extra,
+             pidfile=pidfile,
           )]
 
         # More complex needs: create a Python script as wrapper
@@ -72,5 +74,7 @@ class Recipe(GenericBaseRecipe):
              name=wrapper_path,
              command=python_script,
              parameters=[],
-             parameters_extra=parameters_extra) ]
+             parameters_extra=parameters_extra,
+             pidfile=pidfile,
+        )]
 

software/gitlab/instance-gitlab.cfg.in

@@ -760,7 +760,9 @@ cron-entries    = ${cron-dir:cron.d}
 crontabs        = ${cron-dir:crontabs}
 cronstamps      = ${cron-dir:cronstamps}
 catcher         = ${cron-simplelogger:wrapper}
+
 dcrond-binary   = {{ dcron_bin }}
+
 depends =
     ${logrotate-entry-cron:recipe}
 
@@ -770,6 +772,7 @@ recipe  = slapos.cookbook:simplelogger
 wrapper = ${directory:bin}/${:_buildout_section_name_}
 log     = ${cron-dir:log}/cron.log
 
+
 # base entry for clients who registers to cron
 [cron-entry]
 recipe  = slapos.cookbook:cron.d
@@ -835,11 +838,12 @@ backup  = ...
 time    = daily
 command = ${logrotate:wrapper}
 
+
 [cron-entry-gitlab-backup]
 <= cron-entry
 # run backup script every four hours
 frequency = 0 */4 * * *
-command = {{ backup_script }}
+command = {{ gitlab_backup_pull }}
 
 # 6. on-reinstantiate actions
 

software/gitlab/instance.cfg.in

 # GitLab "switch-softwaretype" instance
 [buildout]
-parts =
-  switch-softwaretype
+parts = switch-softwaretype
 
 # std stuff for slapos instance
 eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
 offline = true
 
+
 [switch-softwaretype]
 recipe = slapos.cookbook:softwaretype
 default = $${instance-gitlab.cfg:rendered}
 # TODO -export, -import, -pull-backup
 
+
 [instance-gitlab.cfg]
 recipe  = slapos.recipe.template:jinja2
 mode    = 0644
@@ -49,8 +50,7 @@ context =
     raw ruby_location               ${bundler-4gitlab:ruby-location}
     raw watcher_sigkill             ${watcher-sigkill:rendered}
 
-# script
-    raw backup_script               $${backup_script.bash:rendered}
+    raw gitlab_backup_pull          $${gitlab-backup-pull:wrapper-path}
 
 # config files
     raw database_yml_in             ${database.yml.in:target}
@@ -67,13 +67,32 @@ context =
     raw smtp_settings_rb_in         ${smtp_settings.rb.in:target}
     raw unicorn_rb_in               ${unicorn.rb.in:target}
 
-[backup_script.bash]
+
+[gitlab-backup-directory]
+recipe            = slapos.cookbook:mkdirectory
+srv               = $${buildout:directory}/srv
+backup            = $${:srv}/backup
+backup-gitlab.git = $${:backup}/backup-gitlab.git
+var               = $${buildout:directory}/var
+pid               = $${:var}/pid
+
+
+[gitlab-backup-pull-unwrapped]
 recipe      = slapos.recipe.template:jinja2
 mode        = 0770
-template    = ${backup_script.bash.in:target}
-rendered    = $${buildout:directory}/backup_script.bash
+template    = ${gitlab-backup-pull.in:target}
+rendered    = $${buildout:directory}/bin/unwrapped-gitlab-backup-pull
 context     =
 # XXX remove /usr/bin below? this is currently useful for command `which`
     raw env_path      /usr/bin:${coreutils:location}/bin:${grep:location}/bin:${tar:location}/bin:${gzip:location}/bin:$${buildout:directory}/bin:${gopath:bin-directory}:${git:location}/bin:${xnice-repository:location}/bin
-    raw backup_repo   $${buildout:directory}/backup-gitlab.git
+    raw backup_repo   $${gitlab-backup-directory:backup-gitlab.git}
     raw bash_bin      ${bash:location}/bin/bash
+
+[gitlab-backup-pull]
+recipe        = slapos.cookbook:wrapper
+pidfile       = $${gitlab-backup-directory:pid}/gitlab-backup-pull.pid
+wrapper-path  = $${buildout:directory}/bin/gitlab-backup-pull
+# NOTE we perform db setup / migrations as part of unicorn startup.
+# Those operations require PG and Redis to be up and running already, that's
+# why we do it here. See gitlab-unicorn-startup for details.
+command-line  = $${gitlab-backup-pull-unwrapped:rendered}

software/gitlab/software.cfg

@@ -48,12 +48,9 @@ parts =
     eggs
 
     bash
-    coreutils
     curl
-    grep
-    gzip
-    tar
     watcher-sigkill
+    gzip
     dcron-output
     logrotate
 
@@ -71,6 +68,7 @@ recipe  = z3c.recipe.scripts
 interpreter = python2
 eggs    =
     docutils
+
 # rubygemsrecipe with fixed url and this way pinned rubygems version
 [rubygemsrecipe]
 recipe  = rubygemsrecipe
@@ -117,7 +115,7 @@ git-executable = ${git:location}/bin/git
 [gitlab-repository]
 <= git-repository
 #repository = https://gitlab.com/gitlab-org/gitlab-ce.git
-repository = https://lab.nexedi.com/iv/gitlab-ce.git
+repository = https://lab.nexedi.com/nexedi/gitlab-ce.git
 # 8.8.X + NXD patches:
 revision = v8.8.9-10-g967afbdc3a2d2bcc58ed31ca50d5293fa2a2e9ed
 location = ${buildout:parts-directory}/gitlab
@@ -140,6 +138,8 @@ repository = https://lab.nexedi.com/nexedi/gitlab-workhorse.git
 revision = v0.7.1-5-gd23a3247829fc3200e3dc784dcd57b5a0febac48
 location = ${buildout:parts-directory}/gitlab-workhorse
 
+
+
 # build needed-by-gitlab gems via bundler
 [gitlab/vendor/bundle]
 recipe  = slapos.recipe.cmmi
@@ -156,26 +156,29 @@ make-targets= cd ${:path} &&
 
 # directories and repositories required by gitlab-backup
 [gopath]
-directory = ${buildout:directory}/work
+directory = ${buildout:directory}/go.work
 src-directory = ${:directory}/src
 bin-directory = ${:directory}/bin
 
-[git2go-repository]
+[go-git-repository]
 <= git-repository
-git2go-path = github.com/libgit2/git2go
-repository = https://${:git2go-path}.git
+repository = https://${:go.importpath}.git
+location = ${gopath:src-directory}/${:go.importpath}
+
+[git2go-repository]
+<= go-git-repository
+go.importpath = github.com/libgit2/git2go
 # branch 'next' is required by git-backup
 revision = next-g53594d7581617dbae7bb5960b4ac5f0ff513c184
-location = ${gopath:src-directory}/${:git2go-path}
 
 [git-backup-repository]
-<= git-repository
-git-backup-path = lab.nexedi.com/kirr/git-backup
-repository = https://${:git-backup-path}.git
+<= go-git-repository
+go.importpath = lab.nexedi.com/kirr/git-backup
 revision = 3ba6cf73ba224c40f67f1fb87c855b915eb91f58
-location = ${gopath:src-directory}/${:git-backup-path}
+
 
 [git-backup]
+# install git2go, git-backup, gitlab-backup in <gopath>/bin
 recipe  = slapos.recipe.cmmi
 path    = ${git-backup-repository:location}
 configure-command = :
@@ -183,11 +186,9 @@ make-binary =
 make-targets= cd ${git2go-repository:location}
               && git submodule update --init
               && make install
-              && cd ${git-backup-repository:location}
               && install -d ${gopath:bin-directory}
-              && go build -o ${gopath:bin-directory}/git-backup .
-              && cp ${git-backup-repository:location}/contrib/gitlab-backup ${gopath:bin-directory}
-              && chmod +x ${gopath:bin-directory}/gitlab-backup
+              && go install ${git-backup-repository:go.importpath}
+              && cp -a ${git-backup-repository:location}/contrib/gitlab-backup ${gopath:bin-directory}
 environment =
   PATH=${cmake:location}/bin:${pkgconfig:location}/bin:${git:location}/bin:${golang16:location}/bin:${buildout:bin-directory}:%(PATH)s
   GOPATH=${gopath:directory}
@@ -225,6 +226,7 @@ make-targets= ${:_buildout_section_name_}
 environment =
   PATH=${golang16:location}/bin:%(PATH)s
 
+
 ###############################
 #   Trampoline for instance   #
 ###############################
@@ -241,7 +243,7 @@ eggs    =
 recipe  = slapos.recipe.template
 url     = ${:_profile_base_location_}/instance.cfg.in
 output  = ${buildout:directory}/instance.cfg
-md5sum  = 11d8a2628dd170169a334877968e0bcc
+md5sum  = 89cb23916ab1f573188a6987a8630b9a
 
 [watcher-sigkill]
 recipe  = slapos.recipe.template:jinja2
@@ -269,10 +271,6 @@ destination = ${buildout:directory}/${:_buildout_section_name_}
 url     = ${:_profile_base_location_}/template/${:_buildout_section_name_}
 
 
-[backup_script.bash.in]
-<= download-template
-md5sum  = 272b05a7694b950cf3efa64ea14f8bbb
-
 [database.yml.in]
 <= download-template
 md5sum  = 61d1d04b9347b3168a1ad7676e4681ef
@@ -281,6 +279,10 @@ md5sum  = 61d1d04b9347b3168a1ad7676e4681ef
 <= download-template
 md5sum  = eb1230fee50067924ba89f4dc6e82fa9
 
+[gitlab-backup-pull.in]
+<= download-template
+md5sum  = af2d7a5abc21c2ed457a6ea05ef055e2
+
 [gitlab-parameters.cfg]
 <= download-file
 md5sum  = a74670934ec0190cc212d1f3468c11ed
@@ -299,7 +301,7 @@ md5sum  = 176939a6428a7aca4767a36421b0af2b
 
 [instance-gitlab.cfg.in]
 <= download-file
-md5sum  = bf3b277dcf32adb08fbb1deb18ab3cfd
+md5sum  = cbe19d8aef2ebb05cee7c986f3c06108
 
 [macrolib.cfg.in]
 <= download-file

software/gitlab/template/backup_script.bash.in

-#! {{ bash_bin }}
-
-export PATH="{{ env_path }}"
-mkdir -p "{{ backup_repo }}"
-cd "{{ backup_repo }}"
-
-# calling_backup exists if backup script is currently executed
-if [ -d calling_backup ] ; then
-    exit 1
-fi
-
-# create the lock to avoid concurrent calls
-mkdir calling_backup
-if ! git rev-parse --is-inside-git-dir ; then
-  git init --bare
-fi
-
-xnice gitlab-backup pull
-
-# done, remove lock
-rmdir calling_backup

software/gitlab/template/gitlab-backup-pull.in

+#!{{ bash_bin }}
+
+export PATH="{{ env_path }}"
+
+# avoid concurrent calls (by using a pid file as lock), thanks to the
+# generic recipe:
+# https://lab.nexedi.com/nexedi/slapos/blob/fb8912e8/slapos/recipe/librecipe/generic.py#L154
+# XXX: this is not a 100% reliable protection against race conditions
+
+# create the backup repo if needed (in case it was deleted, for example)
+mkdir -p "{{ backup_repo }}"
+cd "{{ backup_repo }}"
+
+# verify we are in a git repository
+if ! git rev-parse --is-inside-git-dir ; then
+  git init --bare
+fi
+
+xnice gitlab-backup pull
\ No newline at end of file

[Jérome Perrin]

👍

[iv]

requires commit 6f436b16d8

[iv]

@kirr: what about these corrections? see also commit e3c50cf85b

[iv]

note: I still need to add a line on the commit message to explain about grep, tar and coreutils components now required.

[Kirill Smelkov]

follow-up: nexedi/slapos!124 (comment 21482)