software/gitlab: upgrade to version 13.12.15

software/gitlab/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [instance.cfg]
 filename = instance.cfg.in
-md5sum = d1ca30a1b910b6b775f4f95bd91123a6
+md5sum = 956ae53af22b551fbb087415e835868b
 
 [watcher]
 _update_hash_filename_ = watcher.in
@@ -30,35 +30,35 @@ md5sum = 61d1d04b9347b3168a1ad7676e4681ef
 
 [gitconfig.in]
 _update_hash_filename_ = template/gitconfig.in
-md5sum = eb1230fee50067924ba89f4dc6e82fa9
+md5sum = c559a24ab6281268b608ed3bccb8e4ce
 
 [gitlab-parameters.cfg]
 _update_hash_filename_ = gitlab-parameters.cfg
-md5sum = cfda6d959bb90bf0b9c947383f45ce0a
+md5sum = 95b18789111ed239146d243e39ffefbe
 
 [gitlab-shell-config.yml.in]
 _update_hash_filename_ = template/gitlab-shell-config.yml.in
-md5sum = 69e8ed76b06233d11932a5c0ef16f03b
+md5sum = 70d394305f4e1482a5c1a673b0762c6a
 
-[gitlab-unicorn-startup.in]
-_update_hash_filename_ = gitlab-unicorn-startup.in
-md5sum = 705825e6d8c6b37699f1321805d09de3
+[gitlab-puma-startup.in]
+_update_hash_filename_ = gitlab-puma-startup.in
+md5sum = 838209b9246fa86d6a21fef910f17e25
 
 [gitlab.yml.in]
 _update_hash_filename_ = template/gitlab.yml.in
-md5sum = 673c393e6728a8d82e6b9a44886785a8
+md5sum = 6f8df1467a6168bde7d2f5f42abb1012
 
 [gitaly-config.toml.in]
 _update_hash_filename_ = template/gitaly-config.toml.in
-md5sum = 58e3d5bbda32583d00cd8f44ec0525b0
+md5sum = d769ea27820e932c596c35bbbf3f2902
 
 [instance-gitlab.cfg.in]
 _update_hash_filename_ = instance-gitlab.cfg.in
-md5sum = b913c4a1f199a87ad71da6d102adffa4
+md5sum = 6d8d20ded84622339d49c60b0e61380c
 
 [instance-gitlab-export.cfg.in]
 _update_hash_filename_ = instance-gitlab-export.cfg.in
-md5sum = b8dea5ca4c6f9fc1ca54eb0265e1fdee
+md5sum = c8231583d04bf0d3fe2d26230b94d78d
 
 [macrolib.cfg.in]
 _update_hash_filename_ = macrolib.cfg.in
@@ -72,22 +72,18 @@ md5sum = 4980c1571a4dd7753aaa60d065270849
 _update_hash_filename_ = template/nginx.conf.in
 md5sum = 8c904510eb39dc212204f68f2b81b068
 
-[rack_attack.rb.in]
-_update_hash_filename_ = template/rack_attack.rb.in
-md5sum = 7d0e6dc6b826f6df6b20d8574a29e2f8
-
 [resque.yml.in]
 _update_hash_filename_ = template/resque.yml.in
 md5sum = 7c89a730889e3224548d9abe51a2d719
 
 [smtp_settings.rb.in]
 _update_hash_filename_ = template/smtp_settings.rb.in
-md5sum = 4e1ced687a86e4cfff2dde91237e3942
+md5sum = b1becd9ec4c2eeefe573af4bb53c9751
 
 [template-gitlab-resiliency-restore.sh.in]
 _update_hash_filename_ = template/template-gitlab-resiliency-restore.sh.in
-md5sum = 87f16b4f4a2370acada46b2751ef3366
+md5sum = 8ce31a27e814e750dfd38c92a278fb9e
 
-[unicorn.rb.in]
-_update_hash_filename_ = template/unicorn.rb.in
-md5sum = b4758129a8d0c47b2c3adb10fefb8275
+[puma.rb.in]
+_update_hash_filename_ = template/puma.rb.in
+md5sum = 707c0c713af41518d21724c1be8efe22

software/gitlab/gitlab-parameters.cfg

@@ -15,8 +15,11 @@ configuration.external_url              = https://lab.example.com
 configuration.db_pool                   = 10
 
 # rack-attack
-configuration.rate_limit_requests_per_period    = 10
-configuration.rate_limit_period                 = 60
+configuration.rack_attack_enable        = true
+configuration.rack_attack_max_retry     = 10
+configuration.rack_attack_find_time     = 60
+configuration.rack_attack_ban_time      = 3600
+configuration.rack_attack_ip_whitelist  = 127.0.0.1
 
 configuration.time_zone                 = UTC
 
@@ -64,8 +67,10 @@ configuration.sidekiq_memory_killer_max_rss = 1000000
 
 
 # unicorn
-configuration.unicorn_worker_timeout    = 60
-configuration.unicorn_worker_processes  = 2
+configuration.puma_worker_timeout      = 60
+configuration.puma_worker_processes    = 2
+configuration.puma_min_threads         = 1
+configuration.puma_max_threads         = 16
 
 # unicorn advanced
 configuration.unicorn_backlog_socket    = 1024

software/gitlab/gitlab-unicorn-startup.in → software/gitlab/gitlab-puma-startup.in

@@ -38,6 +38,8 @@ echo "I: PostgreSQL ready." 1>&2
 # make sure pg_trgm extension is enabled for gitlab db
 psql -c 'CREATE EXTENSION IF NOT EXISTS pg_trgm;'   || die "pg_trgm setup failed"
 
+psql -c 'CREATE EXTENSION IF NOT EXISTS btree_gist;'  || die "btree_gist setup failed"
+
 if echo "$pgtables" | grep -q '^Did not find any relations' ; then
     $RAKE gitlab:setup RAILS_ENV=production force=yes || die "initial db setup failed"
 fi
@@ -70,8 +72,7 @@ $RAKE cache:clear   || die "cache:clear failed"
 force=yes $RAKE gitlab:shell:setup   || die "gitlab:shell:setup failed"
 
 
-# 3. finally exec to unicorn
-exec {{ gitlab_unicorn }}   \
-    -E production   \
-    -c {{ unicorn_rb.output }}    \
-    {{ gitlab_work.location }}/config.ru
+# 3. finally exec to puma
+exec {{ gitlab_puma }}   \
+    -e production \
+    -C {{ puma_rb.output }}

software/gitlab/gowork.cfg

@@ -6,6 +6,7 @@ depends_gitfetch  =
     ${go_github.com_pkg_errors:recipe}
     ${go_lab.nexedi.com_kirr_git-backup:recipe}
     ${go_lab.nexedi.com_kirr_go123:recipe}
+    ${go_golang.org_x_crypto:recipe}
 
 
 [go_github.com_libgit2_git2go]
@@ -13,7 +14,7 @@ depends_gitfetch  =
 go.importpath = github.com/libgit2/git2go
 repository    = https://github.com/libgit2/git2go.git
 # branch 'next' is required by git-backup
-revision = next-g5d0a4c752a74258a5f42e40fccd2908ac4e336b8
+revision = cbca5b82b8c22c08c183a1f44cad4b8b51ba6f25
 
 [go_github.com_pkg_errors]
 <= go-git-package
@@ -25,10 +26,17 @@ revision      = v0.8.0-12-g816c908556
 <= go-git-package
 go.importpath = lab.nexedi.com/kirr/git-backup
 repository    = https://lab.nexedi.com/kirr/git-backup.git
-revision      = da754af24da351291c99caa421a103db09e7a4c4
+revision      = 3327aedfbe903b210366105b8a4f9f85a313a0a3
 
 [go_lab.nexedi.com_kirr_go123]
 <= go-git-package
 go.importpath = lab.nexedi.com/kirr/go123
 repository    = https://lab.nexedi.com/kirr/go123.git
-revision      = 95433de34f
+revision      = 8299741f
+
+
+[go_golang.org_x_crypto]
+<= go-git-package
+go.importpath = golang.org/x/crypto
+repository    = https://go.googlesource.com/crypto.git
+revision      = 75b288015ac94e66e3d6715fb68a9b41bf046ec2

software/gitlab/instance-gitlab-export.cfg.in

@@ -54,7 +54,8 @@ input = inline: gitlab-shell-work*
   srv/backup/logrotate/**
   etc/service/postgres-start
   srv/redis/**
-  srv/unicorn/unicorn.socket
+  srv/puma/puma.socket
+  .cache
 output = ${directory:srv}/exporter.exclude
 
 [gitlab-resiliency-restore-script]
@@ -70,13 +71,14 @@ context =
   raw     git_location            {{ git_location }}
   raw     bin_directory           ${directory:bin}
   raw     etc_directory           ${directory:etc}
-  raw     run_directory           ${directory:run}
+  raw     var_directory           ${directory:var}
   raw     postgress_script        ${service-postgresql:services}/postgres-start
   raw     redis_script            ${service-redis:wrapper}
-  raw     unicorn_script          ${service-unicorn:wrapper-path}
+  raw     puma_script             ${service-puma:wrapper-path}
   raw     sidekiq_script          ${service-sidekiq:wrapper-path}
   raw     gitlab_backup_dir       ${gitlab-backup-directory:backup-gitlab.git}
   raw     redis_pid_file          ${service-redis:pid-file}
   raw     postgres_pid_file       ${service-postgresql:pgdata-directory}/postmaster.pid
+  raw     puma_pid_file           ${puma:pid}/puma.pid
   raw     gitlab_work_location    ${gitlab-work:location}
   raw     promise_lab_location    ${directory:promise.slow}

software/gitlab/instance-gitlab.cfg.in

@@ -12,7 +12,7 @@ parts =
 
 #   gitlab-<prog>
 # ? mailroom
-{% set gitlab_progv = 'rails rake unicorn sidekiq unicorn-startup' .split() %}
+{% set gitlab_progv = 'rails rake puma sidekiq puma-startup' .split() %}
 {% for prog in gitlab_progv %}
     gitlab-{{ prog }}
 {% endfor %}
@@ -23,7 +23,7 @@ parts =
     gitlab-shell-work
 
     service-gitlab-workhorse
-    service-unicorn
+    service-puma
     service-sidekiq
 
     service-nginx
@@ -51,29 +51,29 @@ offline = true
 
 [worker-processes]
 recipe = slapos.recipe.build
-unicorn-worker-processes = {{ instance_parameter_dict['configuration.unicorn_worker_processes'] }}
+puma-worker-processes = {{ instance_parameter_dict['configuration.puma_worker_processes'] }}
 init =
   import multiprocessing
-  worker_count = int(options['unicorn-worker-processes'])
+  worker_count = int(options['puma-worker-processes'])
   if worker_count == 0:
     # automatically load all available CPUs
     worker_count = multiprocessing.cpu_count() + 1
   worker_count = 2 if worker_count < 2 else worker_count
-  options['unicorn-worker-processes'] = worker_count
+  options['puma-worker-processes'] = worker_count
   options['nginx-worker-processes'] = worker_count -1
 
 [instance-parameter]
 {#- There are dangerous keys like recipe, etc #}
 {#- XXX: Some other approach would be useful #}
 {%- set DROP_KEY_LIST = ['recipe', '__buildout_signature__', 'computer', 'partition', 'url', 'key', 'cert',
-                         'configuration.unicorn_worker_processes', 'configuration.nginx_worker_processes'] %}
+                         'configuration.puma_worker_processes', 'configuration.nginx_worker_processes'] %}
 {%- for key, value in instance_parameter_dict.items() -%}
 {%-   if key not in DROP_KEY_LIST %}
 {{ key }} = {{ value }}
 {%-   endif -%}
 {%- endfor %}
 # settings for worker processes:
-configuration.unicorn_worker_processes = ${worker-processes:unicorn-worker-processes}
+configuration.puma_worker_processes = ${worker-processes:puma-worker-processes}
 configuration.nginx_worker_processes   = ${worker-processes:nginx-worker-processes}
 
 
@@ -186,16 +186,18 @@ mode    = 0700
 [gitaly-dir]
 recipe  = slapos.cookbook:mkdirectory
 gitaly  = ${directory:var}/gitaly
-sockets = ${:gitaly}/sockets
-internal = ${directory:var}/int
+sockets = ${:gitaly}/s
+internal = ${:sockets}/int
 log     = ${directory:log}/gitaly
 
 [gitaly]
-socket  = ${directory:var}/gitaly.socket
-log     = ${gitaly-dir:log}
+socket  = ${gitaly-dir:sockets}/gitaly.socket
+logdir     = ${gitaly-dir:log}
 location = {{ gitaly_location }}
 pid     = ${directory:run}/gitaly.pid
 internal_socket = ${gitaly-dir:internal}
+basedir = ${gitaly-dir:gitaly}
+num_workers = 2
 
 [gitaly-socket-listening-promise]
 <= monitor-promise-base
@@ -249,7 +251,7 @@ context-extra =
     section gitlab                  gitlab
     section gitlab_shell            gitlab-shell
     section gitlab_shell_work       gitlab-shell-work
-    section unicorn                 unicorn
+    section puma                    puma
     section service_redis           service-redis
     raw     redis_binprefix         {{ redis_binprefix }}
 
@@ -261,6 +263,7 @@ context-extra =
     section gitlab                  gitlab
     section gitlab_shell            gitlab-shell
     section gitlab_shell_work       gitlab-shell-work
+    section gitlab_workhorse        gitlab-workhorse
     section gitaly                  gitaly
 
 [nginx.conf]
@@ -288,12 +291,10 @@ context-extra =
     import  urllib                  urllib
     section gitlab                  gitlab
     section gitlab_shell_work       gitlab-shell-work
+    section gitlab_shell            gitlab-shell
+    section gitlab_workhorse        gitlab-workhorse
     section gitaly                  gitaly
 
-[rack_attack.rb]
-<= gitlab-etc-template
-url = {{ rack_attack_rb_in }}
-
 [resque.yml]
 <= gitlab-etc-template
 url = {{ resque_yml_in }}
@@ -306,11 +307,11 @@ url = {{ smtp_settings_rb_in }}
 # contains smtp password
 mode    = 0600
 
-[unicorn.rb]
+[puma.rb]
 <= gitlab-etc-template
-url = {{ unicorn_rb_in }}
+url = {{ puma_rb_in }}
 context-extra =
-    section unicorn                 unicorn
+    section puma                    puma
     section directory               directory
     section gitlab_work             gitlab-work
 
@@ -340,20 +341,20 @@ prog    = {{ prog }}
 {% endfor %}
 
 
-[gitlab-unicorn-startup]
+[gitlab-puma-startup]
 recipe  = slapos.recipe.template:jinja2
 mode    = 0755
-url = {{ gitlab_unicorn_startup_in }}
+url = {{ gitlab_puma_startup_in }}
 output= ${directory:bin}/${:_buildout_section_name_}
 context =
     raw     bash_bin                {{ bash_bin }}
     raw     gitlab_rake             ${gitlab-rake:wrapper-path}
-    raw     gitlab_unicorn          ${gitlab-unicorn:wrapper-path}
+    raw     gitlab_puma            ${gitlab-puma:wrapper-path}
     raw     psql_bin                {{ postgresql_location }}/bin/psql
     section pgsql                   service-postgresql
     raw     log_dir                 ${gitlab:log}
     raw     var_dir                 ${directory:var}
-    section unicorn_rb              unicorn.rb
+    section puma_rb                 puma.rb
     section gitlab_work             gitlab-work
 
 
@@ -421,14 +422,13 @@ tune-command =
     ln -sf ${gitlab-workhorse:secret} .gitlab_workhorse_secret
 # config/
     cd config  &&
-    ln -sf ${unicorn.rb:output} unicorn.rb  &&
+    ln -sf ${puma.rb:output} puma.rb  &&
     ln -sf ${gitlab.yml:output} gitlab.yml  &&
     ln -sf ${database.yml:output} database.yml  &&
     ln -sf ${resque.yml:output} resque.yml  &&
     ln -sf ${secrets:secrets}/gitlab_secrets.yml secrets.yml  &&
 # config/initializers/
     cd initializers  &&
-    ln -sf ${rack_attack.rb:output} rack_attack.rb  &&
     ln -sf ${smtp_settings.rb:output} smtp_settings.rb  &&
 # public/
     cd ../../public  &&
@@ -573,11 +573,12 @@ wrapper-path    = ${directory:service}/gitlab-workhorse
 command-line    = {{ gitlab_workhorse }}
     -listenNetwork unix
     -listenAddr ${gitlab-workhorse:socket}
-    -authSocket ${unicorn:socket}
+    -authSocket ${puma:socket}
     -documentRoot ${gitlab-work:location}/public
     -secretPath ${gitlab-workhorse:secret}
     -logFile ${gitlab-workhorse:log}
-    -repoPath ${gitlab-repo-dir:repositories}
+# repoPath is for patched gitlab-workhorse
+#    -repoPath ${gitlab-repo-dir:repositories}
 # NOTE for profiling
 #   -pprofListenAddr ...
 
@@ -606,41 +607,47 @@ config-command     = {{ curl_bin }} --unix-socket ${gitlab-workhorse:socket}  ht
 
 
 ######################
-#   unicorn worker   #
+#   puma worker   #
 ######################
-[unicorn-dir]
+[puma-dir]
 recipe  = slapos.cookbook:mkdirectory
-srv     = ${directory:srv}/unicorn
-log     = ${directory:log}/unicorn
+srv     = ${directory:srv}/puma
+log     = ${directory:log}/puma
+pid     = ${directory:srv}/pids
 
-[unicorn]
-srv     = ${unicorn-dir:srv}
-log     = ${unicorn-dir:log}
-socket  = ${directory:srv}/unicorn.socket
+[puma]
+srv     = ${puma-dir:srv}
+log     = ${puma-dir:log}
+socket  = ${puma-dir:srv}/puma.socket
+pid     = ${puma-dir:pid}
 
-[service-unicorn]
+[service-puma]
 recipe  = slapos.cookbook:wrapper
-wrapper-path    = ${directory:service}/unicorn
-# NOTE we perform db setup / migrations as part of unicorn startup.
+wrapper-path    = ${directory:service}/puma
+# NOTE we perform db setup / migrations as part of puma startup.
 # Those operations require PG and Redis to be up and running already, that's
-# why we do it here. See gitlab-unicorn-startup for details.
-command-line    = ${gitlab-unicorn-startup:output}
+# why we do it here. See gitlab-puma-startup for details.
+command-line    = ${gitlab-puma-startup:output}
+{% if instance_parameter_dict.get('configuration.root-password') -%}
+environment =
+  GITLAB_ROOT_PASSWORD={{ instance_parameter_dict['configuration.root-password'] }}
+{% endif %}
 
 depend  =
-    ${promise-unicorn:recipe}
+    ${promise-puma:recipe}
     ${promise-gitlab-app:recipe}
     ${promise-gitlab-shell:recipe}
 
-    ${logrotate-entry-unicorn:recipe}
-# gitlab is a service "run" under unicorn
+    ${logrotate-entry-puma:recipe}
+# gitlab is a service "run" under puma
 # gitlab-shell is called by gitlab
 # -> associate their logs rotation to here
     ${logrotate-entry-gitlab:recipe}
 
 
-[promise-unicorn]
+[promise-puma]
 <= promise-byurl
-config-command = {{ curl_bin }} --unix-socket ${unicorn:socket}  http://localhost/
+config-command = {{ curl_bin }} --unix-socket ${puma:socket}  http://localhost/
 
 [promise-rakebase]
 recipe  = slapos.cookbook:wrapper
@@ -662,10 +669,10 @@ command-line    = ${:rake} gitlab:gitlab_shell:check
 # rake gitlab:repo:check        (fsck all repos)
 
 
-[logrotate-entry-unicorn]
+[logrotate-entry-puma]
 <= logrotate-entry-base
-log     = ${unicorn:log}/*.log
-name    = unicorn
+log     = ${puma:log}/*.log
+name    = puma
 copytruncate = true
 
 [logrotate-entry-gitlab]
@@ -682,8 +689,8 @@ copytruncate = true
 
 [logrotate-entry-gitlab-workhorse]
 <= logrotate-entry-base
-log     = ${gitlab-workhorse-dir:log}//*.log
-name    = gitlab-shell
+log     = ${gitlab-workhorse-dir:log}/*.log
+name    = gitlab-workhorse
 copytruncate = true
 
 #######################################
@@ -828,18 +835,17 @@ cron-entries = ${cron:cron-entries}
 [service-gitaly]
 recipe  = slapos.cookbook:wrapper
 wrapper-path    = ${directory:service}/gitaly
-#command-line    = ${gitlab-work:location}/bin/daemon_with_pidfile ${gitaly:pid}
 command-line    = {{ gitaly_location }}/gitaly ${gitaly-config.toml:output}
 
 environment =
-    PATH={{ bundler_1_17_3_dir }}:{{ ruby_location }}/bin:/bin:/usr/bin
+    PATH={{ buildout_bin_directory }}:{{ ruby_location }}/bin:/bin:/usr/bin
 
 
 # 6. on-reinstantiate actions
 
 # NOTE here we only recompile assets. Other on-reinstantiate actions, which
-# require pg and redis running, are performed as part of unicorn service -
-# right before its startup (see gitlab-unicorn-startup).
+# require pg and redis running, are performed as part of puma service -
+# right before its startup (see gitlab-puma-startup).
 [on-reinstantiate]
 recipe  = plone.recipe.command
 stop-on-error   = true

software/gitlab/instance.cfg.in

@@ -53,6 +53,7 @@ context =
     section instance_parameter_dict slap-configuration
 
 # program binaries
+    raw buildout_bin_directory      ${buildout:bin-directory}
     raw bash_bin                    ${bash:location}/bin/bash
     raw bzip2_location              ${bzip2:location}
     raw bundler_4gitlab             ${bundler-4gitlab:bundle}
@@ -64,7 +65,7 @@ context =
     raw git_location                ${git:location}
     raw gitaly_location             ${gitaly-repository:location}
     raw gitlab_export               ${gitlab-export:output}
-    raw gitlab_workhorse            ${gowork:bin}/gitlab-workhorse
+    raw gitlab_workhorse            ${gitlab-workhorse:binary}
     raw gopath_bin                  ${gowork:bin}
     raw gunzip_bin                  ${gzip:location}/bin/gunzip
     raw grep_location               ${grep:location}
@@ -75,8 +76,8 @@ context =
     raw nginx_mime_types            ${nginx-output:mime}
     raw node_bin_location           ${nodejs:location}/bin/
     raw openssl_bin                 ${openssl-output:openssl}
-    raw postgresql_location         ${postgresql10:location}
-    raw redis_binprefix             ${redis28:location}/bin
+    raw postgresql_location         ${postgresql:location}
+    raw redis_binprefix             ${redis:location}/bin
     raw ruby_location               ${bundler-4gitlab:ruby-location}
     raw tar_location                ${tar:location}
     raw watcher                     ${watcher:output}
@@ -88,17 +89,16 @@ context =
     raw gitconfig_in                ${gitconfig.in:target}
     raw monitor_template            ${monitor2-template:output}
     raw gitlab_shell_config_yml_in  ${gitlab-shell-config.yml.in:target}
-    raw gitlab_unicorn_startup_in   ${gitlab-unicorn-startup.in:target}
+    raw gitlab_puma_startup_in      ${gitlab-puma-startup.in:target}
     raw gitlab_yml_in               ${gitlab.yml.in:target}
     raw gitaly_config_toml_in       ${gitaly-config.toml.in:target}
     raw macrolib_cfg_in             ${macrolib.cfg.in:target}
     raw nginx_conf_in               ${nginx.conf.in:target}
     raw nginx_gitlab_http_conf_in   ${nginx-gitlab-http.conf.in:target}
-    raw rack_attack_rb_in           ${rack_attack.rb.in:target}
     raw resque_yml_in               ${resque.yml.in:target}
     raw smtp_settings_rb_in         ${smtp_settings.rb.in:target}
     raw gitlab_restore_sh_in        ${template-gitlab-resiliency-restore.sh.in:target}
-    raw unicorn_rb_in               ${unicorn.rb.in:target}
+    raw puma_rb_in                  ${puma.rb.in:target}
 
     $${:context-extra}
 context-extra =

software/gitlab/software.cfg

@@ -5,8 +5,10 @@ extends =
     ../../stack/slapos.cfg
     ../../stack/nodejs.cfg
     ../../stack/monitor/buildout.cfg
+    ../../component/libgit2/buildout.cfg
     ../../component/ruby/buildout.cfg
     ../../component/golang/buildout.cfg
+    ../../component/perl-Image-ExifTool/buildout.cfg
     ../../component/postgresql/buildout.cfg
     ../../component/redis/buildout.cfg
     ../../component/cmake/buildout.cfg
@@ -28,19 +30,19 @@ extends =
     ../../component/gzip/buildout.cfg
     ../../component/dcron/buildout.cfg
     ../../component/logrotate/buildout.cfg
+    ../../component/libgpgme/buildout.cfg
+    ../../component/patchelf/buildout.cfg
 
 parts =
-    golang1.13
+    golang1.15
     git
-    postgresql10
-    redis28
+    postgresql
     cmake
     icu
     pkgconfig
     nginx-output
 
     gowork
-    gitlab-workhorse
     gitaly-build
     gitlab-shell/vendor
     gitlab/vendor/bundle
@@ -65,16 +67,16 @@ parts =
 revision = 571d6514f7290e8faa9439c4b86aa2f6c87df261
 
 [nodejs]
-<= nodejs-12.18.3
+<= nodejs-14.16.0
 [yarn]
 <= yarn-1.16.0
+[openssl]
+<= openssl-1.1
 
-# Gitlab backup (git-backup) is failing (segfault) with recent git version > 2.30.9
-# We will use git 2.30.9 version for production upgrade
-# TODO: fix the issue with git and use latest version
-[git]
-url = https://mirrors.edge.kernel.org/pub/software/scm/git/git-2.30.9.tar.xz
-md5sum = c1d42936036cc44a448738329c821569
+[libgit2]
+# This version is for rugged 1.1.0 needed by gitlab and gitaly 13.12.15,
+# see: https://github.com/libgit2/rugged/tree/v1.1.0/vendor
+version = 1.1.0
 
 ############################
 #   Software compilation   #
@@ -122,7 +124,7 @@ url     = https://rubygems.org/rubygems/rubygems-3.1.2.zip
 # - run gitlab services / jobs  (via `bundle exec ...`)
 [bundler-4gitlab]
 <= rubygemsrecipe
-ruby-location = ${ruby2.6:location}
+ruby-location = ${ruby:location}
 ruby-executable = ${:ruby-location}/bin/ruby
 gems    =
   bundler==1.17.3
@@ -148,7 +150,7 @@ bundle1.17.3 = ${buildout:parts-directory}/${:_buildout_section_name_}/lib/ruby/
 # gitlab (via github-markup) wants to convert rst -> html via running: python (with docutils egg)
 environment =
 
-  PATH    = ${python-4gitlab:bin}:${yarn:location}/bin:${:ruby-location}/bin:${cmake:location}/bin:${pkgconfig:location}/bin:${nodejs:location}/bin:${postgresql10:location}/bin:${redis28:location}/bin:${git:location}/bin:${buildout:bin-directory}:%(PATH)s
+  PATH    = ${python-4gitlab:bin}:${yarn:location}/bin:${:ruby-location}/bin:${cmake:location}/bin:${pkgconfig:location}/bin:${nodejs:location}/bin:${postgresql:location}/bin:${redis:location}/bin:${git:location}/bin:${libgpgme:location}/bin:${buildout:bin-directory}:%(PATH)s
 
 
 # gitlab, gitlab-shell & gitlab-workhorse checked out as git repositories
@@ -160,26 +162,21 @@ git-executable = ${git:location}/bin/git
 [gitlab-repository]
 <= git-repository
 repository = https://lab.nexedi.com/nexedi/gitlab-ce.git
-revision = v12.10.14-12-g7ce27b49193
+revision = v13.12.15-10-gf4b76cb11f
 location = ${buildout:parts-directory}/gitlab
 
 [gitlab-shell-repository]
 <= git-repository
 repository = https://gitlab.com/gitlab-org/gitlab-shell.git
-revision = v12.2.0
+revision = v13.18.1
 location = ${buildout:parts-directory}/gitlab-shell
 
 [gitaly-repository]
 <= git-repository
 repository = https://gitlab.com/gitlab-org/gitaly.git
-revision = v12.10.14
+revision = v13.12.15
 location = ${buildout:parts-directory}/gitaly
 
-[gitlab-workhorse-repository]
-<= git-repository
-repository = https://lab.nexedi.com/nexedi/gitlab-workhorse.git
-revision = v8.30.3-19-g919c9b532c
-
 # build needed-by-gitlab gems via bundler
 [gitlab/vendor/bundle]
 recipe  = slapos.recipe.cmmi
@@ -188,19 +185,30 @@ bundle  = ${bundler-4gitlab:bundle}
 
 configure-command = cd ${:path} &&
     ${:bundle} config --local build.charlock_holmes --with-icu-dir=${icu:location}  &&
-    ${:bundle} config --local build.pg --with-pg-config=${postgresql10:location}/bin/pg_config &&
+    ${:bundle} config --local build.pg --with-pg-config=${postgresql:location}/bin/pg_config &&
     ${:bundle} config --local build.re2 --with-re2-dir=${re2:location} &&
     ${:bundle} config --local build.nokogiri --with-zlib-dir=${zlib:location} --with-cflags=-I${xz-utils:location}/include --with-ldflags="-L${xz-utils:location}/lib -Wl,-rpath=${xz-utils:location}/lib"
+    ${:bundle} config --local build.rugged --use-system-libraries --with-git2-dir=${libgit2:location}
+    ${:bundle} config --local build.openssl --with-openssl-dir=${openssl:location}
+    ${:bundle} config --local build.puma --with-openssl-dir=${openssl:location}
+    ${:bundle} config --local build.gpgme --use-system-libraries
     ${:bundle} config set without 'development test mysql aws kerberos'
     ${:bundle} config set deployment 'true'
 
 make-binary =
 make-targets= cd ${:path} && ${:bundle} install
 environment =
-  PKG_CONFIG_PATH=${openssl-1.0:location}/lib/pkgconfig:${re2:location}/lib/pkgconfig:${icu:location}/lib/pkgconfig:${xz-utils:location}/lib/pkgconfig
+  PKG_CONFIG_PATH=${openssl:location}/lib/pkgconfig:${re2:location}/lib/pkgconfig:${icu:location}/lib/pkgconfig:${xz-utils:location}/lib/pkgconfig:${libgit2:location}/lib/pkgconfig
   PATH=${pkgconfig:location}/bin:%(PATH)s
   CFLAGS=-I${xz-utils:location}/include
 
+gpme-rpath = ${ruby:location}/lib:${gdbm:location}/lib:${libffi:location}/lib:${libyaml:location}/lib:${ncurses:location}/lib:${readline:location}/lib:${zlib:location}/lib:${libgpgme:location}/lib:${libgpg-error:location}/lib:${libassuan:location}/lib
+post-install =
+# bundle is not setting libgpgme lib and dependencies in gpgme_n.so rpath
+# it result to  `libgpgme.so.11 => not found`
+  find ${bundler-4gitlab:location}/lib/ruby/gems/ -type f -name gpgme_n.so -exec  \
+    ${patchelf:location}/bin/patchelf --set-rpath ${:gpme-rpath} {} \;
+
 ################## Google re2
 [re2]
 recipe = slapos.recipe.cmmi
@@ -208,7 +216,6 @@ url = https://github.com/google/re2/archive/2019-12-01.tar.gz
 md5sum = 527eab0c75d6a1a0044c6eefd816b2fb
 configure-command = :
 
-
 [gitlab_npm]
 recipe  = slapos.recipe.cmmi
 path    = ${gitlab-repository:location}
@@ -229,37 +236,39 @@ configure-command = :
 make-binary =
 make-targets= cd ${go_github.com_libgit2_git2go:location}
               && git submodule update --init
-              && sed -i 's/.*--build.*/cmake --build . --target install/' script/build-libgit2-static.sh
-              && make install
+              && make install-static
 environment =
-  PKG_CONFIG_PATH=${openssl-1.0:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig
-  PATH=${cmake:location}/bin:${pkgconfig:location}/bin:${git:location}/bin:${golang1.13:location}/bin:${buildout:bin-directory}:%(PATH)s
+  PKG_CONFIG_PATH=${openssl:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig
+  PATH=${cmake:location}/bin:${pkgconfig:location}/bin:${git:location}/bin:${golang1.15:location}/bin:${buildout:bin-directory}:%(PATH)s
   GOPATH=${gowork:directory}
 
 [gowork.goinstall]
-git2go = ${go_github.com_libgit2_git2go_prepare:path}/vendor/libgit2/install
+git2go = ${go_github.com_libgit2_git2go_prepare:path}/static-build/install
 command = bash -c ". ${gowork:env.sh} && CGO_CFLAGS=-I${:git2go}/include CGO_LDFLAGS='-L${:git2go}/lib -lgit2' go install ${gowork:buildflags} -v $(echo -n '${gowork:install}' |tr '\n' ' ') && go test -v lab.nexedi.com/kirr/git-backup"
 
 [gowork]
-golang  = ${golang1.13:location}
-# gitlab.com/gitlab-org/gitlab-workhorse
-# gitlab.com/gitlab-org/gitlab-workhorse/cmd/gitlab-zip-cat
-# gitlab.com/gitlab-org/gitlab-workhorse/cmd/gitlab-zip-metadata
+golang  = ${golang1.15:location}
 install =
   lab.nexedi.com/kirr/git-backup
 cpkgpath =
-    ${openssl-1.0:location}/lib/pkgconfig
+    ${openssl:location}/lib/pkgconfig
     ${zlib:location}/lib/pkgconfig
-    ${go_github.com_libgit2_git2go_prepare:path}/vendor/libgit2/install/lib/pkgconfig
+    ${go_github.com_libgit2_git2go_prepare:path}/static-build/install/lib/pkgconfig
 buildflags = --tags "static"
 
 [gitlab-workhorse]
 recipe = slapos.recipe.cmmi
-path = ${gitlab-workhorse-repository:location}
+path = ${gitlab-repository:location}/workhorse
 configure-command = :
 make-binary =
 make-targets =
   . ${gowork:env.sh} && make test && make install PREFIX=${gowork:directory}
+binary = ${gowork:bin}/${:_buildout_section_name_}
+# adding ExifTool in env so make test can work.
+# ExifTool is not used for now in gitlab-workhorse service for security reason.
+# see: https://lab.nexedi.com/nexedi/slapos/-/merge_requests/1558?diff_id=54289&start_sha=725054eeef9dcd8018abfbf6147cbbbfccfbba5c#note_208105
+environment =
+  PATH=${perl-Image-ExifTool:location}/bin/:%(PATH)s
 
 [gitlab-backup]
 recipe = plone.recipe.command
@@ -275,6 +284,7 @@ bundle  = ${bundler-4gitlab:bundle}
 
 configure-command = cd ${:path}/ruby &&
     ${:bundle} config --local build.charlock_holmes --with-icu-dir=${icu:location}
+    ${:bundle} config --local build.rugged --use-system-libraries --with-git2-dir=${libgit2:location}
 make-binary =
 make-targets =
   . ${gowork:env.sh} &&
@@ -284,8 +294,9 @@ post-install =
   # solve the problem error="not executable: ruby/git-hooks/pre-receive"
   chmod 755 ${:path}/ruby/git-hooks/gitlab-shell-hook
 environment =
-  PKG_CONFIG_PATH=${openssl-1.0:location}/lib/pkgconfig:${icu:location}/lib/pkgconfig
-  PATH=${pkgconfig:location}/bin:${ruby2.6:location}/bin:%(PATH)s
+  PKG_CONFIG_PATH=${openssl:location}/lib/pkgconfig:${icu:location}/lib/pkgconfig:${libgit2:location}/lib/pkgconfig
+  PATH=${cmake:location}/bin:${pkgconfig:location}/bin:${ruby:location}/bin:%(PATH)s
+  OPENSSL_ROOT_DIR=${openssl:location}
 
 [xnice-repository]
 # to get kirr's misc repo containing xnice script for executing processes
@@ -310,7 +321,7 @@ make-targets= cd ${:path} &&
     . ${gowork:env.sh} && make build &&
     ${:bundle} install --deployment  --without development test
 environment =
-  PATH=${ruby2.6:location}/bin:%(PATH)s
+  PATH=${ruby:location}/bin:%(PATH)s
 
 ###############################
 #   Trampoline for instance   #
@@ -368,7 +379,7 @@ destination = ${buildout:directory}/${:_buildout_section_name_}
 [gitlab-shell-config.yml.in]
 <= download-file
 
-[gitlab-unicorn-startup.in]
+[gitlab-puma-startup.in]
 <= download-file
 
 [gitlab.yml.in]
@@ -392,9 +403,6 @@ destination = ${buildout:directory}/${:_buildout_section_name_}
 [nginx.conf.in]
 <= download-file
 
-[rack_attack.rb.in]
-<= download-file
-
 [resque.yml.in]
 <= download-file
 
@@ -404,16 +412,12 @@ destination = ${buildout:directory}/${:_buildout_section_name_}
 [template-gitlab-resiliency-restore.sh.in]
 <= download-file
 
-[unicorn.rb.in]
+[puma.rb.in]
 <= download-file
 
-[gitlab-demo-backup.git]
-recipe = slapos.recipe.build:download-unpacked
-url = https://lab.nexedi.com/alain.takoudjou/labdemo.backup/repository/archive.tar.gz?ref=master
-md5sum = d40e5e211dc9a4e5ada9c0250377c639
-
 [versions]
 docutils = 0.16
 cns.recipe.symlink = 0.2.3
 plone.recipe.command = 1.1
 z3c.recipe.scripts = 1.0.1
+beautifulsoup4 = 4.12.3

software/gitlab/template/gitaly-config.toml.in

@@ -7,7 +7,12 @@ socket_path = "{{ gitaly.socket }}"
 # The directory where Gitaly's executables are stored
 bin_dir = "{{ gitaly.location }}"
 
-# # Optional: listen on a TCP socket. This is insecure (no authentication)
+# # Optional. The directory where Gitaly can create all files required to
+# # properly operate at runtime. If not set, Gitaly will create a directory in
+# # the global temporary directory. This directory must exist.
+runtime_dir = "{{ gitaly.basedir }}"
+
+# # Optional if socket_path is set. TCP address for Gitaly to listen on. This is insecure (unencrypted connection).
 # listen_addr = "localhost:9999"
 # tls_listen_addr = "localhost:8888
 
@@ -33,6 +38,10 @@ internal_socket_dir = "{{ gitaly.internal_socket }}"
 bin_path = "{{ git }}"
 # catfile_cache_size = 100
 
+# [[git.config]]
+# key = fetch.fsckObjects
+# value = true
+
 [[storage]]
 name = "default"
 path = "{{ gitlab.repositories }}"
@@ -47,9 +56,9 @@ path = "{{ gitlab.repositories }}"
 # You can optionally configure Gitaly to output JSON-formatted log messages to stdout
 [logging]
 # The directory where Gitaly stores extra log files
-dir = "{{ gitaly.log }}"
-# format = "json"
-# format = "json"
+dir = "{{ gitaly.logdir }}"
+# format = "text"
+format = "json"
 # # Optional: Set log level to only log entries with that severity or above
 # # One of, in order: debug, info, warn, errror, fatal, panic
 # # Defaults to "info"
@@ -79,7 +88,7 @@ dir = "{{ gitaly.location }}/ruby"
 # restart_delay = "5m"
 #
 # # Number of gitaly-ruby worker processes
-# num_workers = 2
+num_workers = {{ gitaly.num_workers }}
 #
 # # Search path for system gitconfig file (e.g. /etc, /opt/gitlab/embedded/etc)
 # # NOTE: This only affects RPCs that use Rugged.
@@ -89,7 +98,47 @@ dir = "{{ gitaly.location }}/ruby"
 # The directory where gitlab-shell is installed
 dir = "{{ gitlab_shell_work.location }}"
 
+[hooks]
+custom_hooks_dir = "{{ gitlab_shell_work.location }}/hooks/"
+
+[gitlab]
+secret_file = "{{ gitlab_shell.secret }}"
+url = "http+unix://{{ urllib.parse.unquote_plus(gitlab_workhorse.socket) }}"
+# Only needed if a UNIX socket is used in `url` and GitLab is configured to
+# use a relative path (e.g. /gitlab).
+# relative_url_root = '/'
+
+[gitlab.http-settings]
+# read_timeout = 300
+# user = someone
+# password = somepass
+# ca_file = /etc/ssl/cert.pem
+# ca_path = /etc/pki/tls/certs
+# self_signed_cert = false
+
 # # You can adjust the concurrency of each RPC endpoint
 # [[concurrency]]
 # rpc = "/gitaly.RepositoryService/GarbageCollect"
 # max_per_repo = 1
+
+# Daily maintenance designates time slots to run daily to optimize and maintain
+# enabled storages.
+# [daily_maintenance]
+# start_hour = 23
+# start_minute = 30
+# duration = "45m"
+# storages = ["default"]
+# disabled = false
+
+# [cgroups]
+# count = 10
+# mountpoint = "/sys/fs/cgroup"
+# hierarchy_root = "gitaly"
+
+# [cgroups.memory]
+# enabled = true
+# limit = 1048576
+
+# [cgroups.cpu]
+# enabled = true
+# shares = 512

software/gitlab/template/gitconfig.in

@@ -12,15 +12,24 @@
 [pack]
         threads = 1
 
+# Enable packfile bitmaps
+[repack]
+        writeBitmaps = true
+
 # don't allow corrupt/broken objects to go in
+# Enable push (advertisePushOptions) options
 [receive]
         fsckObjects = true
-
+        advertisePushOptions = true
 
 [user]
         name = {{ cfg('email_display_name') }}
         email = {{ cfg('email_from') }}
+
+# Enable fsyncObjectFiles to reduce risk of repository corruption if the server crashes
 [core]
         autocrlf = input
+        fsyncObjectFiles = true
 [gc]
         auto = 0
+

software/gitlab/template/gitlab-shell-config.yml.in

@@ -7,13 +7,15 @@
 # GitLab user. git by default
 user: {{ backend_info.user }}
 
-# Url to gitlab instance. Used for api calls. Should end with a slash.
-gitlab_url: "http+unix://{{ urllib.parse.quote_plus(unicorn.socket) }}/"
+# URL to GitLab instance, used for API calls. Default: http://localhost:8080.
+# For relative URL support read http://doc.gitlab.com/ce/install/relative_url.html
+gitlab_url: "http+unix://{{ urllib.parse.quote_plus(puma.socket) }}/"
 
 http_settings:
 {# we don't need any
   <%= @http_settings.to_json if @http_settings %>
 #}
+#  read_timeout: 300
 #  user: someone
 #  password: somepass
 #  ca_file: /etc/ssl/cert.pem
@@ -34,35 +36,17 @@ auth_file: "{{ gitlab.var }}/sshkeys-notused"
 # Default is .gitlab_shell_secret in the root directory.
 secret_file: "{{ gitlab_shell.secret }}"
 
-# Parent directory for global custom hook directories (pre-receive.d, update.d, post-receive.d)
-# Default is hooks in the gitlab-shell directory.
-custom_hooks_dir: "{{ gitlab_shell_work.location }}/hooks/"
-
-# Redis settings used for pushing commit notices to gitlab
-redis:
-  bin: {{ redis_binprefix }}/redis-cli
-  host: {# <%= @redis_host %> #}
-  port: {# <%= @redis_port %> #}
-  socket: {{ service_redis.unixsocket }}
-  database: {# <%= @redis_database %> #}
-  namespace: resque:gitlab
-
 # Log file.
 # Default is gitlab-shell.log in the root directory.
 log_file: "{{ gitlab_shell.log }}/gitlab-shell.log"
 
 # Log level. INFO by default
-log_level:
+log_level: INFO
+
+# Log format. 'text' by default
+log_format: text
 
 # Audit usernames.
 # Set to true to see real usernames in the logs instead of key ids, which is easier to follow, but
 # incurs an extra API call on every gitlab-shell command.
-audit_usernames:
-
-# Enable git-annex support
-# git-annex allows managing files with git, without checking the file contents into git
-# See https://git-annex.branchable.com/ for documentation
-# If enabled, git-annex needs to be installed on the server where gitlab-shell is setup
-# For Debian and Ubuntu systems this can be done with: sudo apt-get install git-annex
-# For CentOS: sudo yum install epel-release && sudo yum install git-annex
-git_annex_enabled:
+audit_usernames: false

software/gitlab/template/gitlab.yml.in

 {{ autogenerated }}
-# see:
-# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/gitlab.yml.example
-# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/gitlab.yml.erb
-# (last updated for omnibus-gitlab 8.8.9+ce.0-g25376053)
 
 {% from 'macrolib.cfg.in' import cfg, cfg_https, external_url  with context %}
 
+# # # # # # # # # # # # # # # # # #
+# GitLab application config file  #
+# # # # # # # # # # # # # # # # # #
+#
+###########################  NOTE  #####################################
+# This file should not receive new settings. All configuration options #
+# * are being moved to ApplicationSetting model!                       #
+# If a setting requires an application restart say so in that screen.  #
+# If you change this file in a merge request, please also create       #
+# a MR on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests. #
+# For more details see https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/doc/settings/gitlab.yml.md #
+########################################################################
+#
+#
+# How to use:
+# 1. Copy file as gitlab.yml
+# 2. Update gitlab -> host with your fully qualified domain name
+# 3. Update gitlab -> email_from
+# 4. If you installed Git from source, change git -> bin_path to /usr/local/bin/git
+#    IMPORTANT: If Git was installed in a different location use that instead.
+#    You can check with `which git`. If a wrong path of Git is specified, it will
+#     result in various issues such as failures of GitLab CI builds.
+# 5. Review this configuration file for other settings you may want to adjust
+
+
+
 production: &base
   #
   # 1. GitLab app settings
@@ -18,6 +40,9 @@ production: &base
     host: {{ external_url.hostname }}
     port: {{ external_url.port or default_port[external_url.scheme] }}
     https: {{ cfg_https }}
+    # The maximum time unicorn/puma can spend on the request. This needs to be smaller than the worker timeout.
+    # Default is 95% of the worker timeout
+    max_request_duration_seconds: 57
 
     {# ssh is disabled completely in slapos version
     # Uncommment this line below if your ssh host is different from HTTP/HTTPS one
@@ -55,6 +80,8 @@ production: &base
         worker_src: "'self' blob:"
         report_uri:
 
+    allowed_hosts: []
+
     # Trusted Proxies
     # Customize if you have GitLab behind a reverse proxy which is running on a different machine.
     # Add the IP address for your reverse proxy to the list, otherwise users will appear signed in from that address.
@@ -122,6 +149,15 @@ production: &base
     repository_downloads_path: <%= @gitlab_repository_downloads_path %>
     #}
 
+    ## Impersonation settings
+    impersonation_enabled: true
+
+    ## Disable jQuery and CSS animations
+    # disable_animations: true
+
+    ## Application settings cache expiry in seconds (default: 60)
+    # application_settings_cache_seconds: 60
+
   {# we do not support reply by email
   ## Reply by email
   # Allow users to comment on issues and merge requests by replying to notification emails.
@@ -414,7 +450,9 @@ production: &base
 
   # Gitaly settings
   gitaly:
-    # Default Gitaly authentication token. Can be overriden per storage. Can
+    # Path to the directory containing Gitaly client executables.
+    client_path: {{ gitaly.location }}
+    # Default Gitaly authentication token. Can be overridden per storage. Can
     # be left blank when Gitaly is running locally on a Unix socket, which
     # is the normal way to deploy Gitaly.
     token:
@@ -463,7 +501,6 @@ production: &base
     authorized_keys_file: {{ gitlab.var }}/sshkeys-notused
 
     repos_path: {{ gitlab.repositories }}
-    hooks_path: {{ gitlab_shell_work.location }}/hooks/
     secret_file: {{ gitlab_shell.secret }}
 
     # Git over HTTP
@@ -483,17 +520,16 @@ production: &base
     # gitlab-shell needs to be set to true
     git_annex_enabled: <%= @git_annex_enabled %>
 
+  workhorse:
+    # File that contains the secret key for verifying access for gitlab-workhorse.
+    secret_file: {{ gitlab_workhorse.secret }}
+
+
   ## Git settings
   # CAUTION!
   # Use the default values unless you really know what you are doing
   git:
     bin_path: {{ git }}
-    # The next value is the maximum memory size grit can use
-    # Given in number of bytes per git object (e.g. a commit)
-    # This value can be increased if you have very large commits
-    max_size: {{ cfg('git_max_size') }}
-    # Git timeout to read a commit, in seconds
-    timeout: {{ cfg('git_timeout') }}
 
   #
   # 5. Extra customization
@@ -515,11 +551,6 @@ production: &base
     <% end %>
     #}
 
-    {# we are ok (for now) with default rack-attack git settings
-    rack_attack:
-      git_basic_auth: <%= @rack_attack_git_basic_auth.to_json if @rack_attack_git_basic_auth %>
-    #}
-
 
     ## Site ICP License
     # XXX unquote needed only for slapos.core earlier than
@@ -530,6 +561,22 @@ production: &base
     {# ICP: '{{ cfg("icp_license") }}' #}
     {% endif %}
 
+  rack_attack:
+    git_basic_auth:
+      # Rack Attack IP banning enabled
+      enabled: {{ cfg("rack_attack_enable") }}
+      #
+      # Whitelist requests from 127.0.0.1 for web proxies (NGINX/Apache) with incorrect headers
+      ip_whitelist: [{{ cfg("rack_attack_ip_whitelist")}}]
+      #
+      # Limit the number of Git HTTP authentication attempts per IP
+      maxretry: {{ cfg("rack_attack_max_retry") }}
+      #
+      # Reset the auth attempt counter per IP after 60 seconds
+      findtime: {{ cfg("rack_attack_find_time") }}
+      #
+      # Ban an IP for one hour (3600s) after too many auth attempts
+      bantime: {{ cfg("rack_attack_ban_time") }}
 
 development:
   <<: *base

software/gitlab/template/puma.rb.in

+
+# see: https://gitlab.com/gitlab-org/omnibus-gitlab/-/blob/master/files/gitlab-cookbooks/gitlab/templates/default/puma.rb.erb
+
+{% from 'macrolib.cfg.in' import cfg  with context %}
+# frozen_string_literal: true
+
+# Load "path" as a rackup file.
+#
+# The default is "config.ru".
+#
+rackup 'config.ru'
+pidfile '{{ puma.pid }}/puma.pid'
+state_path '{{ puma.pid }}/puma.state'
+
+stdout_redirect '{{ puma.log }}/puma.stdout.log',
+  '{{ puma.log }}/puma.stderr.log',
+  true
+
+# Configure "min" to be the minimum number of threads to use to answer
+# requests and "max" the maximum.
+#
+# The default is "0, 16".
+#
+threads {{ cfg("puma_min_threads") }}, {{ cfg("puma_max_threads") }}
+
+# By default, workers accept all requests and queue them to pass to handlers.
+# When false, workers accept the number of simultaneous requests configured.
+#
+# Queueing requests generally improves performance, but can cause deadlocks if
+# the app is waiting on a request to itself. See https://github.com/puma/puma/issues/612
+#
+# When set to false this may require a reverse proxy to handle slow clients and
+# queue requests before they reach puma. This is due to disabling HTTP keepalive
+queue_requests false
+
+# Bind the server to "url". "tcp://", "unix://" and "ssl://" are the only
+# accepted protocols.
+bind 'unix://{{ puma.socket }}'
+
+directory '{{ gitlab_work.location }}'
+workers {{ cfg("puma_worker_processes") }}
+
+require_relative "{{ gitlab_work.location }}/lib/gitlab/cluster/lifecycle_events"
+require_relative "{{ gitlab_work.location }}/lib/gitlab/cluster/puma_worker_killer_initializer"
+
+on_restart do
+  # Signal application hooks that we're about to restart
+  Gitlab::Cluster::LifecycleEvents.do_before_master_restart
+end
+
+options = { workers: {{ cfg("puma_worker_processes") }} }
+
+before_fork do
+  # Signal to the puma killer
+  Gitlab::Cluster::PumaWorkerKillerInitializer.start options unless ENV['DISABLE_PUMA_WORKER_KILLER']
+
+  # Signal application hooks that we're about to fork
+  Gitlab::Cluster::LifecycleEvents.do_before_fork
+end
+
+Gitlab::Cluster::LifecycleEvents.set_puma_options options
+on_worker_boot do
+  # Signal application hooks of worker start
+  Gitlab::Cluster::LifecycleEvents.do_worker_start
+end
+
+# Preload the application before starting the workers; this conflicts with
+# phased restart feature. (off by default)
+preload_app!
+
+tag 'gitlab-puma-worker'
+
+# Verifies that all workers have checked in to the master process within
+# the given timeout. If not the worker process will be restarted. Default
+# value is 60 seconds.
+#
+worker_timeout {{ cfg("puma_worker_timeout") }}
+
+# https://github.com/puma/puma/blob/master/5.0-Upgrade.md#lower-latency-better-throughput
+wait_for_less_busy_worker ENV.fetch('PUMA_WAIT_FOR_LESS_BUSY_WORKER', 0.001).to_f
+
+# https://github.com/puma/puma/blob/master/5.0-Upgrade.md#nakayoshi_fork
+nakayoshi_fork unless ENV['DISABLE_PUMA_NAKAYOSHI_FORK'] == 'true'
+
+# Use json formatter
+require_relative "{{ gitlab_work.location }}/lib/gitlab/puma_logging/json_formatter"
+
+json_formatter = Gitlab::PumaLogging::JSONFormatter.new
+log_formatter do |str|
+  json_formatter.call(str)
+end

software/gitlab/template/rack_attack.rb.in

-{{ autogenerated }}
-# see:
-# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/initializers/rack_attack.rb.example
-# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/rack_attack.rb.erb
-# (last updated for omnibus-gitlab 8.8.9+ce.0-g25376053)
-
-{% from 'macrolib.cfg.in' import cfg  with context %}
-
-# 1. Rename this file to rack_attack.rb
-# 2. Review the paths_to_be_protected and add any other path you need protecting
-#
-
-paths_to_be_protected = [
-  "#{Rails.application.config.relative_url_root}/users/password",
-  "#{Rails.application.config.relative_url_root}/users/sign_in",
-  "#{Rails.application.config.relative_url_root}/api/#{API::API.version}/session.json",
-  "#{Rails.application.config.relative_url_root}/api/#{API::API.version}/session",
-  "#{Rails.application.config.relative_url_root}/users",
-  "#{Rails.application.config.relative_url_root}/users/confirmation",
-  "#{Rails.application.config.relative_url_root}/unsubscribes/"
-
-]
-
-# Create one big regular expression that matches strings starting with any of
-# the paths_to_be_protected.
-paths_regex = Regexp.union(paths_to_be_protected.map { |path| /\A#{Regexp.escape(path)}/ })
-rack_attack_enabled = Gitlab.config.rack_attack.git_basic_auth['enabled']
-
-unless Rails.env.test? || !rack_attack_enabled
-  Rack::Attack.throttle('protected paths', limit: {{ cfg('rate_limit_requests_per_period') }}, period: {{ cfg('rate_limit_period') }}.seconds) do |req|
-    if req.post? && req.path =~ paths_regex
-      req.ip
-    end
-  end
-end

software/gitlab/template/smtp_settings.rb.in

@@ -21,8 +21,6 @@ if Rails.env.production?
     enable_starttls_auto:   {{ cfg('smtp_enable_starttls_auto') }},
     # ssl:
     openssl_verify_mode:    '{{ cfg("smtp_openssl_verify_mode") }}'
-    # ca_path:
-    # ca_file:
   }
 end
 {% else %}

software/gitlab/template/template-gitlab-resiliency-restore.sh.in

@@ -21,15 +21,15 @@ redis_pid_file="{{ redis_pid_file }}"
 postgres_pid_file="{{ postgres_pid_file }}"
 
 bin_location="{{ bin_directory }}"
-run_location="{{ run_directory }}"
 git_location="{{ git_location }}"
 go_work_bin="{{ go_work_bin }}"
 etc_location="{{ etc_directory }}"
 gitlab_work="{{ gitlab_work_location }}"
 promise_check="{{ promise_lab_location }}"
-unicorn_script="{{ unicorn_script }}"
+puma_script="{{ puma_script }}"
+puma_pid_file="{{ puma_pid_file }}"
 sidekiq_script="{{ sidekiq_script }}"
-var_location="{{ run_directory }}/.."
+var_location="{{ var_directory }}"
 
 # export GIT_EXEC_PATH=$git_location/libexec/git-core/
 
@@ -56,7 +56,7 @@ kill_process () {
 
 check_process $postgres_pid_file "Postgres"
 check_process $redis_pid_file "Redis"
-check_process $run_location/unicorn.pid "Unicorn"
+check_process $puma_pid_file "Puma"
 
 if [ -f "$postgres_pid_file" ]; then
   rm $postgres_pid_file
@@ -90,14 +90,14 @@ echo "Checking gitlab promises..."
 echo "[info] Not all promises are checked!"
 $promise_check/gitlab-app
 
-echo "Starting Unicorn to check gitlab-shell promise..."
-$unicorn_script &
-unicorn_pid=$!
-trap "kill $postgres_pid $redis_pid $unicorn_pid" EXIT TERM INT
+echo "Starting Puma to check gitlab-shell promise..."
+$puma_script &
+puma_pid=$!
+trap "kill $postgres_pid $redis_pid $puma_pid" EXIT TERM INT
 sleep 60
-if [ -s "$run_location/unicorn.pid" ]; then
-  unicorn_ppid=$(head -n 1 $run_location/unicorn.pid) > /dev/null 2>&1
-  trap "kill $postgres_pid $redis_pid $unicorn_ppid" EXIT TERM INT
+if [ -s "$puma_pid_file" ]; then
+  puma_pid=$(head -n 1 $puma_pid_file) > /dev/null 2>&1
+  trap "kill $postgres_pid $redis_pid $puma_pid" EXIT TERM INT
 fi
 $promise_check/gitlab-shell
 
@@ -109,7 +109,7 @@ $promise_check/gitlab-shell
 
 kill_process $postgres_pid
 kill_process $redis_pid
-kill_process $unicorn_pid
+kill_process $puma_pid
 
 RESTORE_EXIT_CODE=$?
 

software/gitlab/template/unicorn.rb.in

-{{ autogenerated }}
-# see:
-# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/unicorn.rb.example
-# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/unicorn.rb.example.development
-# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/unicorn.rb.erb
-# (last updated for omnibus-gitlab 8.7.9+ce.1-0-gf589ad7)
-
-{% from 'macrolib.cfg.in' import cfg  with context %}
-
-# What ports/sockets to listen on, and what options for them.
-# we listen only on unix socket
-listen "{{ unicorn.socket }}", :backlog => {{ cfg('unicorn_backlog_socket') }}
-#listen "127.0.0.1:8888", :tcp_nopush => true
-
-working_directory '{{ gitlab_work.location }}'
-
-# What the timeout for killing busy workers is, in seconds
-timeout {{ cfg('unicorn_worker_timeout') }}
-
-# combine Ruby 2.0.0dev or REE with "preload_app true" for memory savings
-# http://rubyenterpriseedition.com/faq.html#adapt_apps_for_cow
-preload_app true
-
-
-# Enable this flag to have unicorn test client connections by writing the
-# beginning of the HTTP headers before calling the application.  This
-# prevents calling the application for connections that have disconnected
-# while queued.  This is only guaranteed to detect clients on the same
-# host unicorn runs on, and unlikely to detect disconnects even on a
-# fast LAN.
-check_client_connection false
-
-require_relative '{{ gitlab_work.location }}/lib/gitlab/cluster/lifecycle_events'
-
-before_exec do |server|
-  # Signal application hooks that we're about to restart
-  Gitlab::Cluster::LifecycleEvents.do_before_master_restart
-end
-
-# How many worker processes
-worker_processes {{ cfg('unicorn_worker_processes') }}
-
-# about before_fork / after_fork - see:
-#   https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/definitions/unicorn_service.rb
-#   http://bogomips.org/unicorn.git/tree/examples/unicorn.conf.rb?id=3312aca8#n75
-
-# What to do before we fork a worker
-before_fork do |server, worker|
-  # Signal application hooks that we're about to fork
-  Gitlab::Cluster::LifecycleEvents.do_before_fork
-
-  # The following is only recommended for memory/DB-constrained
-  # installations.  It is not needed if your system can house
-  # twice as many worker_processes as you have configured.
-  #
-  # This allows a new master process to incrementally
-  # phase out the old master process with SIGTTOU to avoid a
-  # thundering herd (especially in the "preload_app false" case)
-  # when doing a transparent upgrade.  The last worker spawned
-  # will then kill off the old master process with a SIGQUIT.
-  old_pid = "#{server.config[:pid]}.oldbin"
-  if old_pid != server.pid
-    begin
-      sig = (worker.nr + 1) >= server.worker_processes ? :QUIT : :TTOU
-      Process.kill(sig, File.read(old_pid).to_i)
-    rescue Errno::ENOENT, Errno::ESRCH
-    end
-  end
-  #
-  # Throttle the master from forking too quickly by sleeping.  Due
-  # to the implementation of standard Unix signal handlers, this
-  # helps (but does not completely) prevent identical, repeated signals
-  # from being lost when the receiving process is busy.
-  # sleep 1
-end
-
-
-# What to do after we fork a worker
-after_fork do |server, worker|
-  # Signal application hooks of worker start
-  Gitlab::Cluster::LifecycleEvents.do_worker_start
-
-  # per-process listener ports for debugging/admin/migrations
-  # addr = "127.0.0.1:#{9293 + worker.nr}"
-  # server.listen(addr, :tries => -1, :delay => 5, :tcp_nopush => true)
-
-end
-
-
-# Where to drop a pidfile
-pid '{{ directory.run }}/unicorn.pid'
-
-# Where stderr gets logged
-stderr_path '{{ unicorn.log }}/unicorn_stderr.log'
-
-# Where stdout gets logged
-stdout_path '{{ unicorn.log }}/unicorn_stdout.log'
-
-{# we do not support Relative url
-<%- if @relative_url %>
-# Relative url from where GitLab is served
-ENV['RAILS_RELATIVE_URL_ROOT'] = "<%= @relative_url %>"
-
-<%- end %>
-#}
-
-# Min memory size (RSS) per worker
-ENV['GITLAB_UNICORN_MEMORY_MIN'] = ({{ cfg('unicorn_worker_memory_limit_min') }}).to_s
-
-# Max memory size (RSS) per worker
-ENV['GITLAB_UNICORN_MEMORY_MAX'] = ({{ cfg('unicorn_worker_memory_limit_max') }}).to_s

software/gitlab/test/setup.py

@@ -46,6 +46,7 @@ setup(
         'erp5.util',
         'supervisor',
         'requests',
+        'beautifulsoup4'
     ],
     zip_safe=True,
     test_suite='test',

software/gitlab/test/test.py

@@ -26,10 +26,10 @@
 ##############################################################################
 
 import os
-import logging
-from urllib.parse import urlparse
-
 import requests
+import functools
+import bs4
+from urllib.parse import urljoin
 
 from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
 
@@ -46,6 +46,10 @@ class TestGitlab(SlapOSInstanceTestCase):
   def getInstanceSoftwareType(cls):
     return 'gitlab'
 
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {'root-password': 'admin1234'}
+
   def setUp(self):
     self.backend_url = self.computer_partition.getConnectionParameterDict(
     )['backend_url']
@@ -54,3 +58,33 @@ class TestGitlab(SlapOSInstanceTestCase):
     resp = requests.get(self.backend_url, verify=False)
     self.assertTrue(
       resp.status_code in [requests.codes.ok, requests.codes.found])
+
+  def test_rack_attack_sign_in_rate_limiting(self):
+    session = requests.session()
+
+    # Load the login page to get a CSRF token.
+    response = session.get(urljoin(self.backend_url, 'users/sign_in'), verify=False)
+    self.assertEqual(response.status_code, 200)
+
+    # Extract the CSRF token and param.
+    bsoup = bs4.BeautifulSoup(response.text, 'html.parser')
+    csrf_param = bsoup.find('meta', dict(name='csrf-param'))['content']
+    csrf_token = bsoup.find('meta', dict(name='csrf-token'))['content']
+
+    request_data = {
+                    'user[login]': 'test',
+                    'user[password]': 'random',
+                    csrf_param: csrf_token}
+
+    sign_in = functools.partial(
+       session.post,
+       response.url,
+       data=request_data,
+       verify=False)
+
+    for _ in range(10):
+      sign_in(headers={'X-Forwarded-For': '1.2.3.4'})
+    # after 10 authentication failures, this client is rate limited
+    self.assertEqual(sign_in(headers={'X-Forwarded-For': '1.2.3.4'}).status_code, 429)
+    # but other clients are not
+    self.assertNotEqual(sign_in(headers={'X-Forwarded-For': '5.6.7.8'}).status_code, 429)