Update Release Candidate

component/coreutils/buildout.cfg

@@ -19,6 +19,13 @@ environment =
   PATH=${perl:location}/bin:${xz-utils:location}/bin:%(PATH)s
   LDFLAGS=-Wl,--as-needed -L${gmp:location}/lib -Wl,-rpath=${gmp:location}/lib
 
+# Latest version of command split in coreutils is not working in gitlab backup
+# For more details, see: https://lab.nexedi.com/nexedi/slapos/merge_requests/1503/diffs#note_197515
+[coreutils-9.1]
+<= coreutils
+url = https://ftp.gnu.org/gnu/coreutils/coreutils-9.1.tar.xz
+md5sum = 8b1ca4e018a7dce9bb937faec6618671
+
 [coreutils-output]
 # Shared binary location to ease migration
 recipe = plone.recipe.command

component/golang/buildout.cfg

@@ -74,13 +74,12 @@ patches =
 [golang14:platform.machine() == 'aarch64']
 setarch = setarch arm
 
-
-[golang1.12]
+[golang1.13]
 <= golang-common-pre-1.19
-url = https://golang.org/dl/go1.12.17.src.tar.gz
-md5sum = 6b607fc795391dc609ffd79ebf41f080
+url = https://go.dev/dl/go1.13.15.src.tar.gz
+md5sum = 4f4af14d88352a62761a9dcedf863ac0
 
-# go1.12 needs go1.4 to bootstrap
+# go1.13 needs go1.4 to bootstrap
 environment-extra =
   GOROOT_BOOTSTRAP=${golang14:location}
 

component/nodejs/buildout.cfg

@@ -72,14 +72,6 @@ md5sum = 28bf6a4d98b238403fa58a0805f4a979
 PATH = ${pkgconfig:location}/bin:${python2.7:location}/bin:%(PATH)s
 configure-command = ./configure
 
-[nodejs-8.12.0]
-<= nodejs-base
-version = v8.12.0
-md5sum = 5690333b77964edf81945fc724f6ea85
-openssl-location = ${openssl-1.0:location}
-PATH = ${pkgconfig:location}/bin:${python2.7:location}/bin:%(PATH)s
-configure-command = ./configure
-
 [nodejs-base]
 # Server-side Javascript.
 version =

component/ruby/buildout.cfg

@@ -25,10 +25,10 @@ environment =
   PKG_CONFIG_PATH=${libyaml:location}/lib/
 
 
-[ruby2.3]
+[ruby2.6]
 <= ruby-common
-url = http://ftp.ruby-lang.org/pub/ruby/2.3/ruby-2.3.8.tar.xz
-md5sum = 927e1857f3dd5a1bdec26892dbae2a05
+url = http://ftp.ruby-lang.org/pub/ruby/2.6/ruby-2.6.5.tar.xz
+md5sum = b8a4e2bdbb76485c3d6690e57be67750
 
 [ruby]
-<= ruby2.3
+<= ruby2.6

software/gitlab/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [instance.cfg]
 filename = instance.cfg.in
-md5sum = 7fa9436be9a31bf4ee172951df2d9df4
+md5sum = ea1d4fb7b2330ae9d94df07f74b934b4
 
 [watcher]
 _update_hash_filename_ = watcher.in
@@ -38,35 +38,35 @@ md5sum = c2e23c0f7baa1633df0436ca4e728424
 
 [gitlab-shell-config.yml.in]
 _update_hash_filename_ = template/gitlab-shell-config.yml.in
-md5sum = 52d18b521b8cd16352fc88b1e1d79d53
+md5sum = 69e8ed76b06233d11932a5c0ef16f03b
 
 [gitlab-unicorn-startup.in]
 _update_hash_filename_ = gitlab-unicorn-startup.in
-md5sum = b0c3d465a8aaad9d2274934dcf208645
+md5sum = 705825e6d8c6b37699f1321805d09de3
 
 [gitlab.yml.in]
 _update_hash_filename_ = template/gitlab.yml.in
-md5sum = f4cc0bc898b8d59010d61473e2adc53b
+md5sum = 673c393e6728a8d82e6b9a44886785a8
 
 [gitaly-config.toml.in]
 _update_hash_filename_ = template/gitaly-config.toml.in
-md5sum = 0f1ec4077dab586cc003ae13f689eda2
+md5sum = 58e3d5bbda32583d00cd8f44ec0525b0
 
 [instance-gitlab.cfg.in]
 _update_hash_filename_ = instance-gitlab.cfg.in
-md5sum = 0445e54ee7ce1f65ec79801e128c80d4
+md5sum = 8e5b0ddb1b79679b4162f302aa438b62
 
 [instance-gitlab-export.cfg.in]
 _update_hash_filename_ = instance-gitlab-export.cfg.in
-md5sum = 9ed8220bb3ad71ff7e8638354127412c
+md5sum = b8dea5ca4c6f9fc1ca54eb0265e1fdee
 
 [macrolib.cfg.in]
 _update_hash_filename_ = macrolib.cfg.in
-md5sum = a56a44e96f65f5ed20211bb6a54279f4
+md5sum = 70612697434bf4fbe838fdf4fd867ed8
 
 [nginx-gitlab-http.conf.in]
 _update_hash_filename_ = template/nginx-gitlab-http.conf.in
-md5sum = cd7471a8c5d6f6bc848c62ce62dca966
+md5sum = 4980c1571a4dd7753aaa60d065270849
 
 [nginx.conf.in]
 _update_hash_filename_ = template/nginx.conf.in
@@ -86,8 +86,8 @@ md5sum = 4e1ced687a86e4cfff2dde91237e3942
 
 [template-gitlab-resiliency-restore.sh.in]
 _update_hash_filename_ = template/template-gitlab-resiliency-restore.sh.in
-md5sum = 16b9f52f00d55feab7e31a88029ad351
+md5sum = 87f16b4f4a2370acada46b2751ef3366
 
 [unicorn.rb.in]
 _update_hash_filename_ = template/unicorn.rb.in
-md5sum = 67728235a2c4c9425c80f0c856749885
+md5sum = b4758129a8d0c47b2c3adb10fefb8275

software/gitlab/gitlab-unicorn-startup.in

@@ -39,15 +39,10 @@ echo "I: PostgreSQL ready." 1>&2
 psql -c 'CREATE EXTENSION IF NOT EXISTS pg_trgm;'   || die "pg_trgm setup failed"
 
 if echo "$pgtables" | grep -q '^Did not find any relations' ; then
-    $RAKE db:schema:load db:seed_fu  || die "initial db setup failed"
+    $RAKE gitlab:setup RAILS_ENV=production force=yes || die "initial db setup failed"
 fi
 
 
-# re-build ssh keys
-# (we do not use them - just for cleannes)
-force=yes $RAKE gitlab:shell:setup   || die "gitlab:shell:setup failed"
-
-
 # 2. what to do when instance is upgraded
 # see
 #   https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/support/deploy/deploy.sh
@@ -64,10 +59,15 @@ $RAKE db:migrate >$migrate_log 2>&1  || die "db:migrate failed"
 # logs of actual migration run.
 test -s $migrate_log || rm $migrate_log
 
+touch {{ var_dir }}/gitlab_db_ok
 
 # clear cache
 $RAKE cache:clear   || die "cache:clear failed"
 
+# re-build ssh keys
+# (we do not use them - just for cleannes)
+# run before migration to avoir error on missing tables in db
+force=yes $RAKE gitlab:shell:setup   || die "gitlab:shell:setup failed"
 
 
 # 3. finally exec to unicorn

software/gitlab/gowork.cfg

@@ -25,10 +25,10 @@ revision      = v0.8.0-12-g816c908556
 <= go-git-package
 go.importpath = lab.nexedi.com/kirr/git-backup
 repository    = https://lab.nexedi.com/kirr/git-backup.git
-revision      = 3f6c4deec8834bdcd2c28c7c5eeacd8211e759b5
+revision      = da754af24da351291c99caa421a103db09e7a4c4
 
 [go_lab.nexedi.com_kirr_go123]
 <= go-git-package
 go.importpath = lab.nexedi.com/kirr/go123
 repository    = https://lab.nexedi.com/kirr/go123.git
-revision      = 56bf8f815a
+revision      = 95433de34f

software/gitlab/instance-gitlab-export.cfg.in

@@ -50,6 +50,8 @@ input = inline: gitlab-shell-work*
   var/repositories/**
   srv/postgresql/**
   srv/postgresql
+  srv/backup/logrotate
+  srv/backup/logrotate/**
   etc/service/postgres-start
   srv/redis/**
   srv/unicorn/unicorn.socket

software/gitlab/instance-gitlab.cfg.in

@@ -53,7 +53,7 @@ offline = true
 {#- There are dangerous keys like recipe, etc #}
 {#- XXX: Some other approach would be useful #}
 {%- set DROP_KEY_LIST = ['recipe', '__buildout_signature__', 'computer', 'partition', 'url', 'key', 'cert'] %}
-{%- for key, value in instance_parameter_dict.iteritems() -%}
+{%- for key, value in instance_parameter_dict.items() -%}
 {%-   if key not in DROP_KEY_LIST %}
 {{ key }} = {{ value }}
 {%-   endif -%}
@@ -198,7 +198,7 @@ context =
     raw     autogenerated       # This file was autogenerated. (DO NOT EDIT - changes will be lost)
     section instance_parameter  instance-parameter
     section backend_info        backend-info
-    import  urlparse            urlparse
+    import  urlparse            urllib.parse
     raw     git                 {{ git }}
     ${:context-extra}
 context-extra =
@@ -336,6 +336,7 @@ context =
     raw     psql_bin                {{ postgresql_location }}/bin/psql
     section pgsql                   service-postgresql
     raw     log_dir                 ${gitlab:log}
+    raw     var_dir                 ${directory:var}
     section unicorn_rb              unicorn.rb
     section gitlab_work             gitlab-work
 
@@ -427,6 +428,8 @@ tune-command =
 software = {{ gitlab_shell_repository_location }}
 
 tune-command =
+    if [ -d "bin" ]; then rm -rf bin; fi &&
+    ln -sf ${:software}/bin bin &&
     ln -sf ${gitlab-shell-config.yml:output}   config.yml  &&
     true
 
@@ -531,6 +534,7 @@ config-command = ${service-redis:promise-wrapper}
 <= logrotate-entry-base
 log     = ${redis:log}/*.log
 name = redis
+copytruncate = true
 
 
 ########################
@@ -557,6 +561,7 @@ command-line    = {{ gitlab_workhorse }}
     -documentRoot ${gitlab-work:location}/public
     -secretPath ${gitlab-workhorse:secret}
     -logFile ${gitlab-workhorse:log}
+    -repoPath ${gitlab-repo-dir:repositories}
 # NOTE for profiling
 #   -pprofListenAddr ...
 
@@ -645,21 +650,25 @@ command-line    = ${:rake} gitlab:gitlab_shell:check
 <= logrotate-entry-base
 log     = ${unicorn:log}/*.log
 name    = unicorn
+copytruncate = true
 
 [logrotate-entry-gitlab]
 <= logrotate-entry-base
 log     = ${gitlab:log}/*.log
 name    = gitlab
+copytruncate = true
 
 [logrotate-entry-gitlab-shell]
 <= logrotate-entry-base
 log     = ${gitlab-shell:log}/*.log
 name    = gitlab-shell
+copytruncate = true
 
 [logrotate-entry-gitlab-workhorse]
 <= logrotate-entry-base
 log     = ${gitlab-workhorse-dir:log}//*.log
 name    = gitlab-shell
+copytruncate = true
 
 #######################################
 #   sidekiq background jobs manager   #
@@ -709,6 +718,7 @@ command-line    = ${:rake} gitlab:sidekiq:check
 <= logrotate-entry-base
 log     = ${sidekiq:log}/*.log
 name    = sidekiq
+copytruncate = true
 
 
 ######################
@@ -781,6 +791,7 @@ promise = check_url_available
 <= logrotate-entry-base
 log     = ${nginx:log}/*.log
 name    = nginx
+post = kill -USR1 $(cat ${directory:run}/nginx.pid)
 
 # base entry for clients who registers to cron
 [cron-entry]
@@ -826,8 +837,7 @@ command =
     ${:rake} gitlab:assets:clean  &&
     ${:rake} gettext:compile RAILS_ENV=production &&
     cd ${gitlab-work:location} &&
-    PATH={{ node_bin_location }}:$PATH {{ yarn_location }}/bin/yarn add ajv@^4.11.2 &&
-    PATH={{ node_bin_location }}:$PATH {{ yarn_location }}/bin/yarn install --production --pure-lockfile &&
+    PATH={{ node_bin_location }}:{{ yarn_location }}/bin:$PATH yarn install --prefer-offline --production --pure-lockfile &&
     ${:rake} gitlab:assets:compile NODE_ENV=production NODE_OPTIONS="--max_old_space_size=4096" &&
     true
 

software/gitlab/instance.cfg.in

@@ -72,7 +72,7 @@ context =
     raw bzip2_location              ${bzip2:location}
     raw bundler_4gitlab             ${bundler-4gitlab:bundle}
     raw bundler_1_17_3_dir          ${bundler-4gitlab:bundle1.17.3}
-    raw coreutils_location          ${coreutils:location}
+    raw coreutils_location          ${coreutils-9.1:location}
     raw curl_bin                    ${curl:location}/bin/curl
     raw dcron_bin                   ${dcron-output:crond}
     raw git                         ${git:location}/bin/git
@@ -88,7 +88,7 @@ context =
     raw logrotate_bin               ${logrotate:location}/usr/sbin/logrotate
     raw nginx_bin                   ${nginx-output:nginx}
     raw nginx_mime_types            ${nginx-output:mime}
-    raw node_bin_location           ${nodejs-8.12.0:location}/bin/
+    raw node_bin_location           ${nodejs:location}/bin/
     raw openssl_bin                 ${openssl-output:openssl}
     raw postgresql_location         ${postgresql10:location}
     raw redis_binprefix             ${redis28:location}/bin

software/gitlab/macrolib.cfg.in

@@ -7,7 +7,6 @@
    NOTE macros can return only strings - that's why '' is used for false #}
 {% macro cfg_bool(name) %}{{ 'true' if (cfg(name).lower() in ('true', 'yes')) else '' }}{% endmacro %}
 
-
 {# deduce whether to use https from external url
    ( here - becasue we cannot use jinja2 logic in instance-gitlab.cfg.in to
      process instance parameters ) #}

software/gitlab/software.cfg

@@ -30,8 +30,7 @@ extends =
     ../../component/logrotate/buildout.cfg
 
 parts =
-    ruby2.3
-    golang1.12
+    golang1.13
     git
     postgresql10
     redis28
@@ -43,11 +42,9 @@ parts =
     gowork
     gitlab-workhorse
     gitaly-build
-    python-4gitlab
     gitlab-shell/vendor
     gitlab/vendor/bundle
     gitlab_npm
-    github-markup-patch
     gitlab-backup
 
 #   for instance
@@ -68,23 +65,53 @@ parts =
 revision = 571d6514f7290e8faa9439c4b86aa2f6c87df261
 
 [nodejs]
-<= nodejs-8.12.0
+<= nodejs-12.18.3
 [yarn]
-<= yarn-1.3.2
-[python]
-part = python2.7
+<= yarn-1.16.0
+
+# Gitlab backup (git-backup) is failing (segfault) with recent git version > 2.30.9
+# We will use git 2.30.9 version for production upgrade
+# TODO: fix the issue with git and use latest version
+[git]
+url = https://mirrors.edge.kernel.org/pub/software/scm/git/git-2.30.9.tar.xz
+md5sum = c1d42936036cc44a448738329c821569
 
 ############################
 #   Software compilation   #
 ############################
 
 # python with eggs, that will be used in gitlab
+# gitlab-markup call the command `python3 /path/to/commands/rest2html` which
+# require docutils
+# https://gitlab.com/gitlab-org/gitlab-markup/-/blob/master/lib/github/markups.rb
+[docutils-download]
+recipe = slapos.recipe.build:download
+shared = true
+url = https://files.pythonhosted.org/packages/2f/e0/3d435b34abd2d62e8206171892f174b180cd37b09d57b924ca5c2ef2219d/${:filename}
+filename = docutils-0.16.tar.gz
+md5sum = 44952782107930ddfcd37ae48eee0857
+
 [python-4gitlab]
-recipe  = zc.recipe.egg
-interpreter = python2
-eggs    =
-    docutils
+recipe = slapos.recipe.build
+docutils = ${docutils-download:target}
+init =
+  # add the python executable in the options dict so that
+  # buildout signature changes if python executable changes
+  import os, sys
+  options['bin'] = python = os.path.join(location, 'bin')
 
+install =
+  import os, sys
+  python = self.buildout['python3']['executable']
+  call([python, '-m', 'venv', '--clear', location])
+  pip = os.path.join(location, 'bin', 'pip')
+  call([pip, 'install', '--no-index', options['docutils']])
+  call([pip, 'uninstall', '-y', 'pip', 'setuptools'])
+  # selftest
+  python = os.path.join(location, 'bin', 'python')
+  call([python, '-c', 'import docutils'])
+
+# Need ruby 2.6.5
 # rubygemsrecipe with fixed url and this way pinned rubygems version
 [rubygemsrecipe]
 recipe  = rubygemsrecipe
@@ -95,7 +122,7 @@ url     = https://rubygems.org/rubygems/rubygems-3.1.2.zip
 # - run gitlab services / jobs  (via `bundle exec ...`)
 [bundler-4gitlab]
 <= rubygemsrecipe
-ruby-location = ${ruby2.3:location}
+ruby-location = ${ruby2.6:location}
 ruby-executable = ${:ruby-location}/bin/ruby
 gems    =
   bundler==1.17.3
@@ -103,7 +130,7 @@ gems    =
 # bin installed here
 bundle  = ${buildout:bin-directory}/bundle
 # Gitaly need bundler 1.17.3 which is not the default version at the end
-bundle1.17.3 = ${buildout:parts-directory}/${:_buildout_section_name_}/lib/ruby/gems/1.8/gems/bundler-1.17.3/exe/
+bundle1.17.3 = ${buildout:parts-directory}/${:_buildout_section_name_}/lib/ruby/gems/gems/bundler-1.17.3/exe/
 
 # install together with dependencies of gitlab, which we cannot specify using
 #   --with-... gem option
@@ -122,7 +149,7 @@ bundle1.17.3 = ${buildout:parts-directory}/${:_buildout_section_name_}/lib/ruby/
 # (python-4gitlab puts interpreter into ${buildout:bin-directory})
 environment =
 
-  PATH    = ${yarn:location}/bin:${:ruby-location}/bin:${cmake:location}/bin:${pkgconfig:location}/bin:${nodejs:location}/bin:${postgresql10:location}/bin:${redis28:location}/bin:${git:location}/bin:${buildout:bin-directory}:%(PATH)s
+  PATH    = ${python-4gitlab:bin}:${yarn:location}/bin:${:ruby-location}/bin:${cmake:location}/bin:${pkgconfig:location}/bin:${nodejs:location}/bin:${postgresql10:location}/bin:${redis28:location}/bin:${git:location}/bin:${buildout:bin-directory}:%(PATH)s
 
 
 # gitlab, gitlab-shell & gitlab-workhorse checked out as git repositories
@@ -134,44 +161,25 @@ git-executable = ${git:location}/bin/git
 [gitlab-repository]
 <= git-repository
 repository = https://lab.nexedi.com/nexedi/gitlab-ce.git
-# 9.5.10 + NXD patches:
-revision = v9.5.10-13-g2b98fc27fd2
+revision = v12.10.14-8-gd7e78e9013
 location = ${buildout:parts-directory}/gitlab
 
 [gitlab-shell-repository]
 <= git-repository
-#repository = https://lab.nexedi.com/nexedi/gitlab-shell.git
 repository = https://gitlab.com/gitlab-org/gitlab-shell.git
-# gitlab 9.5.10 wants gitlab-shell 5.6.1
-revision = v5.6.1-10-g1e587d3b7f
+revision = v12.2.0
 location = ${buildout:parts-directory}/gitlab-shell
 
 [gitaly-repository]
 <= git-repository
 repository = https://gitlab.com/gitlab-org/gitaly.git
-# for version v0.35.0 (gitlab 9.5.10)
-revision = v0.35.0-0-gf99a57b19a
+revision = v12.10.14
 location = ${buildout:parts-directory}/gitaly
 
 [gitlab-workhorse-repository]
 <= git-repository
 repository = https://lab.nexedi.com/nexedi/gitlab-workhorse.git
-revision = v3.0.0-8-g74793ad3cc
-
-# Patch github markup to not call "python2 -S /path/to/rest2html" but only "python2 /path/to/rest2html"
-# NOTE github-markup invokes it as `python2`, that's why we are naming it this way
-# https://github.com/github/markup/blob/5393ae93/lib/github/markups.rb#L36
-[github-markup-patch]
-recipe = plone.recipe.command
-command =
-  files=$(ls ${gitlab-repository:location}/vendor/bundle/ruby/*/gems/git*-markup-*/lib/github/markups.rb) || true
-  if [ ! -z "$files" ]; then
-    for file in $files; do
-      sed -i 's#python2 -S#python2#' $file
-    done
-  fi
-update-command = ${:command}
-stop-on-error = True
+revision = v8.30.3-19-g919c9b532c
 
 # build needed-by-gitlab gems via bundler
 [gitlab/vendor/bundle]
@@ -184,12 +192,13 @@ configure-command = cd ${:path} &&
     ${:bundle} config --local build.pg --with-pg-config=${postgresql10:location}/bin/pg_config &&
     ${:bundle} config --local build.re2 --with-re2-dir=${re2:location} &&
     ${:bundle} config --local build.nokogiri --with-zlib-dir=${zlib:location} --with-cflags=-I${xz-utils:location}/include --with-ldflags="-L${xz-utils:location}/lib -Wl,-rpath=${xz-utils:location}/lib"
+    ${:bundle} config set without 'development test mysql aws kerberos'
+    ${:bundle} config set deployment 'true'
 
 make-binary =
-make-targets= cd ${:path} &&
-    ${:bundle} install --deployment  --without development test mysql aws kerberos ed25519
+make-targets= cd ${:path} && ${:bundle} install
 environment =
-  PKG_CONFIG_PATH=${openssl-1.0:location}/lib/pkgconfig:${re2:location}/lib/pkgconfig:${xz-utils:location}/lib/pkgconfig
+  PKG_CONFIG_PATH=${openssl-1.0:location}/lib/pkgconfig:${re2:location}/lib/pkgconfig:${icu:location}/lib/pkgconfig:${xz-utils:location}/lib/pkgconfig
   PATH=${pkgconfig:location}/bin:%(PATH)s
   CFLAGS=-I${xz-utils:location}/include
 
@@ -225,7 +234,7 @@ make-targets= cd ${go_github.com_libgit2_git2go:location}
               && make install
 environment =
   PKG_CONFIG_PATH=${openssl-1.0:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig
-  PATH=${cmake:location}/bin:${pkgconfig:location}/bin:${git:location}/bin:${golang1.12:location}/bin:${buildout:bin-directory}:%(PATH)s
+  PATH=${cmake:location}/bin:${pkgconfig:location}/bin:${git:location}/bin:${golang1.13:location}/bin:${buildout:bin-directory}:%(PATH)s
   GOPATH=${gowork:directory}
 
 [gowork.goinstall]
@@ -233,7 +242,7 @@ git2go = ${go_github.com_libgit2_git2go_prepare:path}/vendor/libgit2/install
 command = bash -c ". ${gowork:env.sh} && CGO_CFLAGS=-I${:git2go}/include CGO_LDFLAGS='-L${:git2go}/lib -lgit2' go install ${gowork:buildflags} -v $(echo -n '${gowork:install}' |tr '\n' ' ')"
 
 [gowork]
-golang  = ${golang1.12:location}
+golang  = ${golang1.13:location}
 # gitlab.com/gitlab-org/gitlab-workhorse
 # gitlab.com/gitlab-org/gitlab-workhorse/cmd/gitlab-zip-cat
 # gitlab.com/gitlab-org/gitlab-workhorse/cmd/gitlab-zip-metadata
@@ -248,11 +257,10 @@ buildflags = --tags "static"
 [gitlab-workhorse]
 recipe = slapos.recipe.cmmi
 path = ${gitlab-workhorse-repository:location}
-md5sum = 2988c944d58c4a08880498c4981cc7b7
 configure-command = :
 make-binary =
 make-targets =
-  . ${gowork:env.sh} && make install PREFIX=${gowork:directory}
+ . ${gowork:env.sh} && make test && make install PREFIX=${gowork:directory}
 
 [gitlab-backup]
 recipe = plone.recipe.command
@@ -272,10 +280,12 @@ make-targets =
   . ${gowork:env.sh} &&
   unset GOBIN &&
   make
+post-install =
+  # solve the problem error="not executable: ruby/git-hooks/pre-receive"
+  chmod 755 ${:path}/ruby/git-hooks/gitlab-shell-hook
 environment =
   PKG_CONFIG_PATH=${openssl-1.0:location}/lib/pkgconfig:${icu:location}/lib/pkgconfig
-  PATH=${pkgconfig:location}/bin:${ruby2.3:location}/bin:%(PATH)s
-
+  PATH=${pkgconfig:location}/bin:${ruby2.6:location}/bin:%(PATH)s
 
 [xnice-repository]
 # to get kirr's misc repo containing xnice script for executing processes
@@ -296,8 +306,11 @@ bundle  = ${bundler-4gitlab:bundle}
 configure-command = true
 make-binary =
 make-targets= cd ${:path} &&
+# Compile go binary
+    . ${gowork:env.sh} && make build &&
     ${:bundle} install --deployment  --without development test
-
+environment =
+  PATH=${ruby2.6:location}/bin:%(PATH)s
 
 ###############################
 #   Trampoline for instance   #
@@ -400,7 +413,7 @@ url = https://lab.nexedi.com/alain.takoudjou/labdemo.backup/repository/archive.t
 md5sum = d40e5e211dc9a4e5ada9c0250377c639
 
 [versions]
+docutils = 0.16
 cns.recipe.symlink = 0.2.3
-docutils = 0.12
 plone.recipe.command = 1.1
 z3c.recipe.scripts = 1.0.1

software/gitlab/template/gitaly-config.toml.in

@@ -14,10 +14,24 @@ bin_dir = "{{ gitaly.location }}"
 # # Optional: export metrics via Prometheus
 # prometheus_listen_addr = "localhost:9236"
 
+# # Optional: configure where the Gitaly creates the sockets for internal connections. If unset, Gitaly will create a randomly
+# # named temp directory each time it boots.
+# # Non Gitaly clients should never connect to these sockets.
+internal_socket_dir = "{{ gitaly.internal_socket }}"
+
+# # Optional: authenticate Gitaly requests using a shared secret
+# [auth]
+# token = 'abc123secret'
+# transitioning = false # Set `transitioning` to true to temporarily allow unauthenticated while rolling out authentication.
+
+# [tls]
+# certificate_path = '/home/git/cert.cert'
+# key_path = '/home/git/key.pem'
 
 # # Git settings
 [git]
 bin_path = "{{ git }}"
+# catfile_cache_size = 100
 
 [[storage]]
 name = "default"
@@ -30,11 +44,21 @@ path = "{{ gitlab.repositories }}"
 # path = "/mnt/other_storage/repositories"
 #
 
-# # You can optionally configure Gitaly to output JSON-formatted log messages to stdout
-# [logging]
+# You can optionally configure Gitaly to output JSON-formatted log messages to stdout
+[logging]
+# The directory where Gitaly stores extra log files
+dir = "{{ gitaly.log }}"
+# format = "json"
 # format = "json"
-# # Additionally exceptions can be reported to Sentry
-# sentry_dsn = "https://<key>:<secret>@sentry.io/<project>
+# # Optional: Set log level to only log entries with that severity or above
+# # One of, in order: debug, info, warn, errror, fatal, panic
+# # Defaults to "info"
+level = "warn"
+#
+# # Additionally exceptions from the Go server can be reported to Sentry
+# sentry_dsn = "https://<key>:<secret>@sentry.io/<project>"
+# # Exceptions from gitaly-ruby can also be reported to Sentry
+# ruby_sentry_dsn = "https://<key>:<secret>@sentry.io/<project>"
 
 
 # # You can optionally configure Gitaly to record histogram latencies on GRPC method calls
@@ -45,7 +69,27 @@ path = "{{ gitlab.repositories }}"
 # The directory where gitaly-ruby is installed
 dir = "{{ gitaly.location }}/ruby"
 
+# # Gitaly-ruby resident set size (RSS) that triggers a memory restart (bytes)
+# max_rss = 200000000
+#
+# # Grace period before a gitaly-ruby process is forcibly terminated after exceeding max_rss (seconds)
+# graceful_restart_timeout = "10m"
+#
+# # Time that gitaly-ruby memory must remain high before a restart (seconds)
+# restart_delay = "5m"
+#
+# # Number of gitaly-ruby worker processes
+# num_workers = 2
+#
+# # Search path for system gitconfig file (e.g. /etc, /opt/gitlab/embedded/etc)
+# # NOTE: This only affects RPCs that use Rugged.
+# rugged_git_config_search_path = "/etc"
 
 [gitlab-shell]
 # The directory where gitlab-shell is installed
 dir = "{{ gitlab_shell_work.location }}"
+
+# # You can adjust the concurrency of each RPC endpoint
+# [[concurrency]]
+# rpc = "/gitaly.RepositoryService/GarbageCollect"
+# max_per_repo = 1

software/gitlab/template/gitlab-shell-config.yml.in

@@ -8,7 +8,7 @@
 user: {{ backend_info.user }}
 
 # Url to gitlab instance. Used for api calls. Should end with a slash.
-gitlab_url: "http+unix://{{ urllib.quote_plus(unicorn.socket) }}/"
+gitlab_url: "http+unix://{{ urllib.parse.quote_plus(unicorn.socket) }}/"
 
 http_settings:
 {# we don't need any
@@ -24,7 +24,7 @@ http_settings:
 # Give the canonicalized absolute pathname,
 # REPOS_PATH MUST NOT CONTAIN ANY SYMLINK!!!
 # Check twice that none of the components is a symlink, including "/home".
-# repos_path: "{{ gitlab.repositories }}"
+repos_path: "{{ gitlab.repositories }}"
 
 # File used as authorized_keys for gitlab user
 # NOTE not used in slapos version (all access via https only)

software/gitlab/template/gitlab.yml.in

@@ -171,6 +171,16 @@ production: &base
     storage_path: <%= @lfs_storage_path %>
   #}
 
+  ## Uploads
+  uploads:
+    # The location where uploads objects are stored (default: public/).
+    storage_path: "{{ gitlab.var }}"
+    # The location where uploads objects are stored (default: public/).
+    # storage_path: public/
+    # base_dir: uploads/-/system
+    object_store:
+      enabled: false
+      remote_directory: uploads # Bucket name
 
   {# we do not support container registry
   ## Container Registry
@@ -516,7 +526,7 @@ production: &base
     #   https://lab.nexedi.com/nexedi/slapos.core/commit/347d33d6
     # for now we have a lot of old slapos.core deployed...
     {% if cfg('icp_license') != '' -%}
-    ICP: {{ urllib.unquote_plus( str(cfg('icp_license')) ).decode('utf-8') }}
+    ICP: {{ urllib.parse.unquote_plus( str(cfg('icp_license')) ) }}
     {# ICP: '{{ cfg("icp_license") }}' #}
     {% endif %}
 

software/gitlab/template/nginx-gitlab-http.conf.in

@@ -74,7 +74,6 @@ server {
   {% if cfg_https %}
   ## Strong SSL Security
   ## https://raymii.org/s/tutorials/Strong_SSL_Security_On_nginx.html & https://cipherli.st/
-  ssl on;
   ssl_certificate {{ nginx.cert_file }};
   ssl_certificate_key {{ nginx.key_file }};
   {# we don't need - most root CA will be included by default
@@ -113,7 +112,7 @@ server {
 
   ## HSTS Config
   ## https://www.nginx.com/blog/http-strict-transport-security-hsts-and-nginx/
-  {% if cfg("nginx_hsts_max_age") > 0 -%}
+  {% if int(cfg("nginx_hsts_max_age")) > 0 -%}
   {%   if '{{ cfg("nginx_hsts_include_subdomains") }}' == 'true' -%}
   add_header Strict-Transport-Security "max-age={{ cfg('nginx_hsts_max_age') }}; includeSubDomains"
   {%   else -%}

software/gitlab/template/template-gitlab-resiliency-restore.sh.in

@@ -29,6 +29,7 @@ gitlab_work="{{ gitlab_work_location }}"
 promise_check="{{ promise_lab_location }}"
 unicorn_script="{{ unicorn_script }}"
 sidekiq_script="{{ sidekiq_script }}"
+var_location="{{ run_directory }}/.."
 
 # export GIT_EXEC_PATH=$git_location/libexec/git-core/
 
@@ -61,6 +62,12 @@ if [ -f "$postgres_pid_file" ]; then
   rm $postgres_pid_file
 fi
 
+# cleanup /var/backup and old repositories folders,
+# restoration will created them at every run
+echo "Cleanup gitlab backup and old repositories folders..."
+rm -rf $var_location/backup/*
+rm -rf $var_location/repositories*
+
 echo "Starting Postgres..."
 $postgres_executable &
 postgres_pid=$!

software/gitlab/template/unicorn.rb.in

@@ -20,8 +20,6 @@ timeout {{ cfg('unicorn_worker_timeout') }}
 # combine Ruby 2.0.0dev or REE with "preload_app true" for memory savings
 # http://rubyenterpriseedition.com/faq.html#adapt_apps_for_cow
 preload_app true
-GC.respond_to?(:copy_on_write_friendly=) and
-  GC.copy_on_write_friendly = true
 
 
 # Enable this flag to have unicorn test client connections by writing the
@@ -32,6 +30,13 @@ GC.respond_to?(:copy_on_write_friendly=) and
 # fast LAN.
 check_client_connection false
 
+require_relative '{{ gitlab_work.location }}/lib/gitlab/cluster/lifecycle_events'
+
+before_exec do |server|
+  # Signal application hooks that we're about to restart
+  Gitlab::Cluster::LifecycleEvents.do_before_master_restart
+end
+
 # How many worker processes
 worker_processes {{ cfg('unicorn_worker_processes') }}
 
@@ -41,11 +46,8 @@ worker_processes {{ cfg('unicorn_worker_processes') }}
 
 # What to do before we fork a worker
 before_fork do |server, worker|
-  # XXX why gitlab does not enable this?
-  # # the following is highly recomended for Rails + "preload_app true"
-  # # as there's no need for the master process to hold a connection
-  # defined?(ActiveRecord::Base) and
-  #   ActiveRecord::Base.connection.disconnect!
+  # Signal application hooks that we're about to fork
+  Gitlab::Cluster::LifecycleEvents.do_before_fork
 
   # The following is only recommended for memory/DB-constrained
   # installations.  It is not needed if your system can house
@@ -75,25 +77,13 @@ end
 
 # What to do after we fork a worker
 after_fork do |server, worker|
+  # Signal application hooks of worker start
+  Gitlab::Cluster::LifecycleEvents.do_worker_start
+
   # per-process listener ports for debugging/admin/migrations
   # addr = "127.0.0.1:#{9293 + worker.nr}"
   # server.listen(addr, :tries => -1, :delay => 5, :tcp_nopush => true)
 
-  # XXX why gitlab does not enable this?
-  # # the following is *required* for Rails + "preload_app true",
-  # defined?(ActiveRecord::Base) and
-  #   ActiveRecord::Base.establish_connection
-  
-  # reset prometheus client, this will cause any opened metrics files to be closed
-  #defined?(::Prometheus::Client.reinitialize_on_pid_change) &&
-  #  Prometheus::Client.reinitialize_on_pid_change
-
-  # if preload_app is true, then you may also want to check and
-  # restart any other shared sockets/descriptors such as Memcached,
-  # and Redis.  TokyoCabinet file handles are safe to reuse
-  # between any number of forked children (assuming your kernel
-  # correctly implements pread()/pwrite() system calls)
-
 end
 
 

software/jstestnode/buildout.hash.cfg

@@ -15,16 +15,16 @@
 
 [instance]
 filename = instance.cfg.in
-md5sum = 84380fe6c268301a1e1f501e53943f58
+md5sum = ad2797e1b83b6b3221f831950075a057
 
 [template-nginx-service]
 filename = template-nginx-service.sh.in
-md5sum = 458870b70c33a1621b68961ae2372ad5
+md5sum = d718fb950862769e57100986cfabb180
 
 [template-nginx-configuration]
 filename = template-nginx.cfg.in
-md5sum = 98faa5ad8cfb23a11d97a459078a1d05
+md5sum = f15c5d9b8c2cf39cb6b2070d8d9d3a92
 
 [template-runTestSuite]
 filename = runTestSuite.in
-md5sum = 5db53d622bd68fb07e078ddc4403a240
+md5sum = 98b7d79eb6af1c4120e3848e9e6fca61

software/jstestnode/instance.cfg.in

@@ -10,7 +10,7 @@ offline = true
 
 [publish]
 recipe = slapos.cookbook:publish.serialised
-nginx = http://[$${nginx-configuration:ip}]:$${nginx-configuration:port}/
+nginx = https://[$${nginx-configuration:ip}]:$${nginx-configuration:port}/
 
 [directory]
 recipe = slapos.cookbook:mkdirectory
@@ -97,13 +97,13 @@ virtual-depends =
 recipe = slapos.recipe.template
 url = ${template-nginx-configuration:output}
 output = $${directory:etc}/nginx.cfg
-access_log = $${directory:log}/nginx-access.log
-error_log = $${directory:log}/nginx-error.log
+access-log = $${directory:log}/nginx-access.log
+error-log = $${directory:log}/nginx-error.log
 ip = $${instance-parameter:ipv6-random}
 port = 9443
-ssl_key = $${directory:ssl}/nginx.key
-ssl_csr = $${directory:ssl}/nginx.csr
-ssl_crt = $${directory:ssl}/nginx.crt
+ssl-csr = $${directory:ssl}/nginx.csr
+ssl-key = $${directory:ssl}/nginx.key
+ssl-crt = $${directory:ssl}/nginx.crt
 
 [nginx-listen-promise]
 recipe = slapos.cookbook:check_port_listening

software/jstestnode/runTestSuite.in

@@ -22,7 +22,7 @@ os.environ['XORG_LOCK_DIR'] = '$${xvfb-instance:lock-dir}'
 os.environ['DISPLAY'] = '$${xvfb-instance:display}'
 os.environ['FONTCONFIG_FILE'] = '$${fontconfig-conf:output}'
 
-BASE_URL = 'http://[$${nginx-configuration:ip}]:$${nginx-configuration:port}/'
+BASE_URL = 'https://[$${nginx-configuration:ip}]:$${nginx-configuration:port}/'
 ETC_DIRECTORY = '$${directory:etc}'
 
 def main():
@@ -91,6 +91,7 @@ def main():
       if target == 'firefox':
         firefox_capabilities = webdriver.common.desired_capabilities.DesiredCapabilities.FIREFOX
         firefox_capabilities['marionette'] = True
+        firefox_capabilities['acceptInsecureCerts'] = True
         browser = webdriver.Firefox(
             capabilities=firefox_capabilities,
             firefox_binary='${firefox-wrapper:location}',

software/jstestnode/template-nginx-service.sh.in

@@ -2,16 +2,16 @@
 # BEWARE: This file is operated by slapos node
 # BEWARE: It will be overwritten automatically
 
-if [ ! -e $${nginx-configuration:ssl_crt} ]
+if [ ! -e $${nginx-configuration:ssl-crt} ]
 then
-  ${openssl-output:openssl} genrsa -out $${nginx-configuration:ssl_key} 2048
+  ${openssl-output:openssl} genrsa -out $${nginx-configuration:ssl-key} 2048
   ${openssl-output:openssl} req -new \
     -subj "/C=AA/ST=Denial/L=Nowhere/O=Dis/CN=$${nginx-configuration:ip}" \
-    -key $${nginx-configuration:ssl_key} -out $${nginx-configuration:ssl_csr}
+    -key $${nginx-configuration:ssl-key} -out $${nginx-configuration:ssl-csr}
   ${openssl-output:openssl} x509 -req -days 365 \
-    -in $${nginx-configuration:ssl_csr} \
-    -signkey $${nginx-configuration:ssl_key} \
-    -out $${nginx-configuration:ssl_crt}
+    -in $${nginx-configuration:ssl-csr} \
+    -signkey $${nginx-configuration:ssl-key} \
+    -out $${nginx-configuration:ssl-crt}
 fi
 
 exec ${nginx-output:nginx} \

software/jstestnode/template-nginx.cfg.in

@@ -8,7 +8,7 @@ events {
   # multi_accept on;
 }
 
-error_log $${nginx-configuration:error_log};
+error_log $${nginx-configuration:error-log};
 
 http {
 
@@ -33,8 +33,8 @@ http {
   # Logging Settings
   ##
 
-  access_log $${nginx-configuration:access_log};
-  error_log $${nginx-configuration:error_log};
+  access_log $${nginx-configuration:access-log};
+  error_log $${nginx-configuration:error-log};
 
   ##
   # Gzip Settings
@@ -51,11 +51,9 @@ http {
   gzip_types text/html text/plain text/css application/json application/javascript application/x-javascript text/xml application/xml application/xml+rss text/javascript;
 
 server {
-  listen [$${nginx-configuration:ip}]:$${nginx-configuration:port};
-
-  # ssl                 on;
-  # ssl_certificate $${nginx-configuration:ssl_crt};
-  # ssl_certificate_key $${nginx-configuration:ssl_key};
+  listen [$${nginx-configuration:ip}]:$${nginx-configuration:port} ssl;
+  ssl_certificate $${nginx-configuration:ssl-crt};
+  ssl_certificate_key $${nginx-configuration:ssl-key};
 
   fastcgi_temp_path  $${directory:varnginx} 1 2;
   uwsgi_temp_path  $${directory:varnginx} 1 2;

software/jstestnode/test/test.py

@@ -52,14 +52,14 @@ class TestJSTestNode(InstanceTestCase):
 
     self.assertEqual(
       {
-        'nginx': 'http://[%s]:9443/' % (self.computer_partition_ipv6_address, )
+        'nginx': 'https://[%s]:9443/' % (self.computer_partition_ipv6_address, )
       },
       connection_dict
     )
 
     # jio tests
     result = requests.get(
-      '%sjio/test/tests.html' % (connection_dict['nginx'], ), allow_redirects=False)
+      '%sjio/test/tests.html' % (connection_dict['nginx'], ), verify=False, allow_redirects=False)
     self.assertEqual(
       [requests.codes.ok, False],
       [result.status_code, result.is_redirect]
@@ -67,7 +67,7 @@ class TestJSTestNode(InstanceTestCase):
 
     # rjs tests
     result = requests.get(
-      '%srenderjs/test/' % (connection_dict['nginx'], ), allow_redirects=False)
+      '%srenderjs/test/' % (connection_dict['nginx'], ), verify=False, allow_redirects=False)
     self.assertEqual(
       [requests.codes.ok, False],
       [result.status_code, result.is_redirect]
@@ -75,7 +75,7 @@ class TestJSTestNode(InstanceTestCase):
 
     # rsvp tests
     result = requests.get(
-      '%srsvp/test/index.html' % (connection_dict['nginx'], ), allow_redirects=False)
+      '%srsvp/test/index.html' % (connection_dict['nginx'], ), verify=False, allow_redirects=False)
     self.assertEqual(
       [requests.codes.ok, False],
       [result.status_code, result.is_redirect]
@@ -83,7 +83,7 @@ class TestJSTestNode(InstanceTestCase):
 
     # Default access
     result = requests.get(
-      'http://[%s]:9443' % (self.computer_partition_ipv6_address, ), allow_redirects=False)
+      'https://[%s]:9443' % (self.computer_partition_ipv6_address, ), verify=False, allow_redirects=False)
     self.assertEqual(
       [requests.codes.forbidden, False],
       [result.status_code, result.is_redirect]

software/osie-coupler/software.cfg

@@ -44,7 +44,7 @@ environment +=
 recipe  = slapos.recipe.build:gitclone
 git-executable = ${git:location}/bin/git
 repository = https://lab.nexedi.com/nexedi/osie.git
-revision = 1e91e159d63d81462369c576e03935129aeb7ecb
+revision = a40573897e1ee9de7b3536daa58c6904384c10f9
 
 [compile-coupler]
 recipe = slapos.recipe.cmmi