Skip to content

slapos
slapos
Commit 81bbbcea authored by iv's avatar iv
6 Browse files
Options

gitlab: changes after review on previous commit (to be squashed)

parent 09252627
Hide whitespace changes
Inline Side-by-side
Showing with 82 additions and 55 deletions
slapos/recipe/wrapper.py
View file @ 81bbbcea
...@@ -37,6 +37,7 @@ class Recipe(GenericBaseRecipe): ...@@ -37,6 +37,7 @@ class Recipe(GenericBaseRecipe):
wait_files = self.options.get('wait-for-files') wait_files = self.options.get('wait-for-files')
environment = self.options.get('environment') environment = self.options.get('environment')
parameters_extra = self.options.get('parameters-extra') parameters_extra = self.options.get('parameters-extra')
pidfile = self.options.get('pidfile', None)
if not wait_files and not environment: if not wait_files and not environment:
# Create a simple wrapper as shell script # Create a simple wrapper as shell script
...@@ -45,6 +46,7 @@ class Recipe(GenericBaseRecipe): ...@@ -45,6 +46,7 @@ class Recipe(GenericBaseRecipe):
command=command_line[0], command=command_line[0],
parameters=command_line[1:], parameters=command_line[1:],
parameters_extra=parameters_extra, parameters_extra=parameters_extra,
pidfile=pidfile,
)] )]
# More complex needs: create a Python script as wrapper # More complex needs: create a Python script as wrapper
...@@ -72,5 +74,7 @@ class Recipe(GenericBaseRecipe): ...@@ -72,5 +74,7 @@ class Recipe(GenericBaseRecipe):
name=wrapper_path, name=wrapper_path,
command=python_script, command=python_script,
parameters=[], parameters=[],
parameters_extra=parameters_extra) ] parameters_extra=parameters_extra,
pidfile=pidfile,
)]
software/gitlab/instance-gitlab.cfg.in
View file @ 81bbbcea
...@@ -760,7 +760,9 @@ cron-entries = ${cron-dir:cron.d} ...@@ -760,7 +760,9 @@ cron-entries = ${cron-dir:cron.d}
crontabs = ${cron-dir:crontabs} crontabs = ${cron-dir:crontabs}
cronstamps = ${cron-dir:cronstamps} cronstamps = ${cron-dir:cronstamps}
catcher = ${cron-simplelogger:wrapper} catcher = ${cron-simplelogger:wrapper}
dcrond-binary = {{ dcron_bin }} dcrond-binary = {{ dcron_bin }}
depends = depends =
${logrotate-entry-cron:recipe} ${logrotate-entry-cron:recipe}
...@@ -770,6 +772,7 @@ recipe = slapos.cookbook:simplelogger ...@@ -770,6 +772,7 @@ recipe = slapos.cookbook:simplelogger
wrapper = ${directory:bin}/${:_buildout_section_name_} wrapper = ${directory:bin}/${:_buildout_section_name_}
log = ${cron-dir:log}/cron.log log = ${cron-dir:log}/cron.log
# base entry for clients who registers to cron # base entry for clients who registers to cron
[cron-entry] [cron-entry]
recipe = slapos.cookbook:cron.d recipe = slapos.cookbook:cron.d
...@@ -835,11 +838,12 @@ backup = ... ...@@ -835,11 +838,12 @@ backup = ...
time = daily time = daily
command = ${logrotate:wrapper} command = ${logrotate:wrapper}
[cron-entry-gitlab-backup] [cron-entry-gitlab-backup]
<= cron-entry <= cron-entry
# run backup script every four hours # run backup script every four hours
frequency = 0 */4 * * * frequency = 0 */4 * * *
command = {{ backup_script }} command = {{ gitlab_backup_pull }}
# 6. on-reinstantiate actions # 6. on-reinstantiate actions
......
software/gitlab/instance.cfg.in
View file @ 81bbbcea
# GitLab "switch-softwaretype" instance # GitLab "switch-softwaretype" instance
[buildout] [buildout]
parts = parts = switch-softwaretype
switch-softwaretype
# std stuff for slapos instance # std stuff for slapos instance
eggs-directory = ${buildout:eggs-directory} eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory} develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true offline = true
[switch-softwaretype] [switch-softwaretype]
recipe = slapos.cookbook:softwaretype recipe = slapos.cookbook:softwaretype
default = $${instance-gitlab.cfg:rendered} default = $${instance-gitlab.cfg:rendered}
# TODO -export, -import, -pull-backup # TODO -export, -import, -pull-backup
[instance-gitlab.cfg] [instance-gitlab.cfg]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
mode = 0644 mode = 0644
...@@ -49,8 +50,7 @@ context = ...@@ -49,8 +50,7 @@ context =
raw ruby_location ${bundler-4gitlab:ruby-location} raw ruby_location ${bundler-4gitlab:ruby-location}
raw watcher_sigkill ${watcher-sigkill:rendered} raw watcher_sigkill ${watcher-sigkill:rendered}
# script raw gitlab_backup_pull $${gitlab-backup-pull:wrapper-path}
raw backup_script $${backup_script.bash:rendered}
# config files # config files
raw database_yml_in ${database.yml.in:target} raw database_yml_in ${database.yml.in:target}
...@@ -67,13 +67,32 @@ context = ...@@ -67,13 +67,32 @@ context =
raw smtp_settings_rb_in ${smtp_settings.rb.in:target} raw smtp_settings_rb_in ${smtp_settings.rb.in:target}
raw unicorn_rb_in ${unicorn.rb.in:target} raw unicorn_rb_in ${unicorn.rb.in:target}
[backup_script.bash]
[gitlab-backup-directory]
recipe = slapos.cookbook:mkdirectory
srv = $${buildout:directory}/srv
backup = $${:srv}/backup
backup-gitlab.git = $${:backup}/backup-gitlab.git
var = $${buildout:directory}/var
pid = $${:var}/pid
[gitlab-backup-pull-unwrapped]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
mode = 0770 mode = 0770
template = ${backup_script.bash.in:target} template = ${gitlab-backup-pull.in:target}
rendered = $${buildout:directory}/backup_script.bash rendered = $${buildout:directory}/bin/unwrapped-gitlab-backup-pull
context = context =
# XXX remove /usr/bin below? this is currently useful for command `which` # XXX remove /usr/bin below? this is currently useful for command `which`
raw env_path /usr/bin:${coreutils:location}/bin:${grep:location}/bin:${tar:location}/bin:${gzip:location}/bin:$${buildout:directory}/bin:${gopath:bin-directory}:${git:location}/bin:${xnice-repository:location}/bin raw env_path /usr/bin:${coreutils:location}/bin:${grep:location}/bin:${tar:location}/bin:${gzip:location}/bin:$${buildout:directory}/bin:${gopath:bin-directory}:${git:location}/bin:${xnice-repository:location}/bin
raw backup_repo $${buildout:directory}/backup-gitlab.git raw backup_repo $${gitlab-backup-directory:backup-gitlab.git}
Please register or sign in to reply
raw bash_bin ${bash:location}/bin/bash raw bash_bin ${bash:location}/bin/bash
[gitlab-backup-pull]
recipe = slapos.cookbook:wrapper
pidfile = $${gitlab-backup-directory:pid}/gitlab-backup-pull.pid
wrapper-path = $${buildout:directory}/bin/gitlab-backup-pull
# NOTE we perform db setup / migrations as part of unicorn startup.
# Those operations require PG and Redis to be up and running already, that's
# why we do it here. See gitlab-unicorn-startup for details.
command-line = $${gitlab-backup-pull-unwrapped:rendered}
software/gitlab/software.cfg
View file @ 81bbbcea
...@@ -48,12 +48,9 @@ parts = ...@@ -48,12 +48,9 @@ parts =
eggs eggs
bash bash
coreutils
curl curl
grep
gzip
tar
watcher-sigkill watcher-sigkill
gzip
dcron-output dcron-output
logrotate logrotate
...@@ -71,6 +68,7 @@ recipe = z3c.recipe.scripts ...@@ -71,6 +68,7 @@ recipe = z3c.recipe.scripts
interpreter = python2 interpreter = python2
eggs = eggs =
docutils docutils
# rubygemsrecipe with fixed url and this way pinned rubygems version # rubygemsrecipe with fixed url and this way pinned rubygems version
[rubygemsrecipe] [rubygemsrecipe]
recipe = rubygemsrecipe recipe = rubygemsrecipe
...@@ -117,7 +115,7 @@ git-executable = ${git:location}/bin/git ...@@ -117,7 +115,7 @@ git-executable = ${git:location}/bin/git
[gitlab-repository] [gitlab-repository]
<= git-repository <= git-repository
#repository = https://gitlab.com/gitlab-org/gitlab-ce.git #repository = https://gitlab.com/gitlab-org/gitlab-ce.git
repository = https://lab.nexedi.com/iv/gitlab-ce.git repository = https://lab.nexedi.com/nexedi/gitlab-ce.git
# 8.8.X + NXD patches: # 8.8.X + NXD patches:
revision = v8.8.9-10-g967afbdc3a2d2bcc58ed31ca50d5293fa2a2e9ed revision = v8.8.9-10-g967afbdc3a2d2bcc58ed31ca50d5293fa2a2e9ed
location = ${buildout:parts-directory}/gitlab location = ${buildout:parts-directory}/gitlab
...@@ -140,6 +138,8 @@ repository = https://lab.nexedi.com/nexedi/gitlab-workhorse.git ...@@ -140,6 +138,8 @@ repository = https://lab.nexedi.com/nexedi/gitlab-workhorse.git
revision = v0.7.1-5-gd23a3247829fc3200e3dc784dcd57b5a0febac48 revision = v0.7.1-5-gd23a3247829fc3200e3dc784dcd57b5a0febac48
location = ${buildout:parts-directory}/gitlab-workhorse location = ${buildout:parts-directory}/gitlab-workhorse
# build needed-by-gitlab gems via bundler # build needed-by-gitlab gems via bundler
[gitlab/vendor/bundle] [gitlab/vendor/bundle]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
...@@ -156,26 +156,29 @@ make-targets= cd ${:path} && ...@@ -156,26 +156,29 @@ make-targets= cd ${:path} &&
# directories and repositories required by gitlab-backup # directories and repositories required by gitlab-backup
[gopath] [gopath]
directory = ${buildout:directory}/work directory = ${buildout:directory}/go.work
src-directory = ${:directory}/src src-directory = ${:directory}/src
bin-directory = ${:directory}/bin bin-directory = ${:directory}/bin
[git2go-repository] [go-git-repository]
<= git-repository <= git-repository
git2go-path = github.com/libgit2/git2go repository = https://${:go.importpath}.git
repository = https://${:git2go-path}.git location = ${gopath:src-directory}/${:go.importpath}
[git2go-repository]
<= go-git-repository
go.importpath = github.com/libgit2/git2go
# branch 'next' is required by git-backup # branch 'next' is required by git-backup
revision = next-g53594d7581617dbae7bb5960b4ac5f0ff513c184 revision = next-g53594d7581617dbae7bb5960b4ac5f0ff513c184
location = ${gopath:src-directory}/${:git2go-path}
[git-backup-repository] [git-backup-repository]
<= git-repository <= go-git-repository
git-backup-path = lab.nexedi.com/kirr/git-backup go.importpath = lab.nexedi.com/kirr/git-backup
repository = https://${:git-backup-path}.git
revision = 3ba6cf73ba224c40f67f1fb87c855b915eb91f58 revision = 3ba6cf73ba224c40f67f1fb87c855b915eb91f58
location = ${gopath:src-directory}/${:git-backup-path}
[git-backup] [git-backup]
# install git2go, git-backup, gitlab-backup in <gopath>/bin
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
path = ${git-backup-repository:location} path = ${git-backup-repository:location}
configure-command = : configure-command = :
...@@ -183,11 +186,9 @@ make-binary = ...@@ -183,11 +186,9 @@ make-binary =
make-targets= cd ${git2go-repository:location} make-targets= cd ${git2go-repository:location}
&& git submodule update --init && git submodule update --init
&& make install && make install
&& cd ${git-backup-repository:location}
&& install -d ${gopath:bin-directory} && install -d ${gopath:bin-directory}
&& go build -o ${gopath:bin-directory}/git-backup . && go install ${git-backup-repository:go.importpath}
&& cp ${git-backup-repository:location}/contrib/gitlab-backup ${gopath:bin-directory} && cp -a ${git-backup-repository:location}/contrib/gitlab-backup ${gopath:bin-directory}
&& chmod +x ${gopath:bin-directory}/gitlab-backup
environment = environment =
PATH=${cmake:location}/bin:${pkgconfig:location}/bin:${git:location}/bin:${golang16:location}/bin:${buildout:bin-directory}:%(PATH)s PATH=${cmake:location}/bin:${pkgconfig:location}/bin:${git:location}/bin:${golang16:location}/bin:${buildout:bin-directory}:%(PATH)s
GOPATH=${gopath:directory} GOPATH=${gopath:directory}
...@@ -225,6 +226,7 @@ make-targets= ${:_buildout_section_name_} ...@@ -225,6 +226,7 @@ make-targets= ${:_buildout_section_name_}
environment = environment =
PATH=${golang16:location}/bin:%(PATH)s PATH=${golang16:location}/bin:%(PATH)s
############################### ###############################
# Trampoline for instance # # Trampoline for instance #
############################### ###############################
...@@ -241,7 +243,7 @@ eggs = ...@@ -241,7 +243,7 @@ eggs =
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg.in url = ${:_profile_base_location_}/instance.cfg.in
output = ${buildout:directory}/instance.cfg output = ${buildout:directory}/instance.cfg
md5sum = 11d8a2628dd170169a334877968e0bcc md5sum = 89cb23916ab1f573188a6987a8630b9a
[watcher-sigkill] [watcher-sigkill]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
...@@ -269,10 +271,6 @@ destination = ${buildout:directory}/${:_buildout_section_name_} ...@@ -269,10 +271,6 @@ destination = ${buildout:directory}/${:_buildout_section_name_}
url = ${:_profile_base_location_}/template/${:_buildout_section_name_} url = ${:_profile_base_location_}/template/${:_buildout_section_name_}
[backup_script.bash.in]
<= download-template
md5sum = 272b05a7694b950cf3efa64ea14f8bbb
[database.yml.in] [database.yml.in]
<= download-template <= download-template
md5sum = 61d1d04b9347b3168a1ad7676e4681ef md5sum = 61d1d04b9347b3168a1ad7676e4681ef
...@@ -281,6 +279,10 @@ md5sum = 61d1d04b9347b3168a1ad7676e4681ef ...@@ -281,6 +279,10 @@ md5sum = 61d1d04b9347b3168a1ad7676e4681ef
<= download-template <= download-template
md5sum = eb1230fee50067924ba89f4dc6e82fa9 md5sum = eb1230fee50067924ba89f4dc6e82fa9
[gitlab-backup-pull.in]
<= download-template
md5sum = af2d7a5abc21c2ed457a6ea05ef055e2
[gitlab-parameters.cfg] [gitlab-parameters.cfg]
<= download-file <= download-file
md5sum = a74670934ec0190cc212d1f3468c11ed md5sum = a74670934ec0190cc212d1f3468c11ed
...@@ -299,7 +301,7 @@ md5sum = 176939a6428a7aca4767a36421b0af2b ...@@ -299,7 +301,7 @@ md5sum = 176939a6428a7aca4767a36421b0af2b
[instance-gitlab.cfg.in] [instance-gitlab.cfg.in]
<= download-file <= download-file
md5sum = bf3b277dcf32adb08fbb1deb18ab3cfd md5sum = cbe19d8aef2ebb05cee7c986f3c06108
[macrolib.cfg.in] [macrolib.cfg.in]
<= download-file <= download-file
......
software/gitlab/template/backup_script.bash.in deleted 100644 → 0
View file @ 09252627
#! {{ bash_bin }}
export PATH="{{ env_path }}"
mkdir -p "{{ backup_repo }}"
cd "{{ backup_repo }}"
# calling_backup exists if backup script is currently executed
if [ -d calling_backup ] ; then
exit 1
fi
# create the lock to avoid concurrent calls
mkdir calling_backup
if ! git rev-parse --is-inside-git-dir ; then
git init --bare
fi
xnice gitlab-backup pull
# done, remove lock
rmdir calling_backup
software/gitlab/template/gitlab-backup-pull.in 0 → 100644
View file @ 81bbbcea
#!{{ bash_bin }}
export PATH="{{ env_path }}"
# avoid concurrent calls (by using a pid file as lock), thanks to the
# generic recipe:
# https://lab.nexedi.com/nexedi/slapos/blob/fb8912e8/slapos/recipe/librecipe/generic.py#L154
# XXX: this is not a 100% reliable protection against race conditions
# create the backup repo if needed (in case it was deleted, for example)
mkdir -p "{{ backup_repo }}"
cd "{{ backup_repo }}"
# verify we are in a git repository
if ! git rev-parse --is-inside-git-dir ; then
git init --bare
fi
xnice gitlab-backup pull
\ No newline at end of file
  • iv @iv ·
    Maintainer

    requires commit 6f436b16d8

  • iv @iv

    mentioned in commit 09252627

     ·

    mentioned in commit 09252627

    Toggle commit list
  • iv @iv ·
    Maintainer

    @kirr: what about these corrections? see also commit e3c50cf85b

  • iv @iv ·
    Maintainer

    note: I still need to add a line on the commit message to explain about grep, tar and coreutils components now required.

  • Kirill Smelkov @kirr ·

    follow-up: nexedi/slapos!124 (comment 21482)

Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7