gitlab-sr: upgrade to version 11.11.8

software/gitlab/buildout.hash.cfg

@@ -38,15 +38,15 @@ md5sum = c2e23c0f7baa1633df0436ca4e728424
 
 [gitlab-shell-config.yml.in]
 _update_hash_filename_ = template/gitlab-shell-config.yml.in
-md5sum = 52d18b521b8cd16352fc88b1e1d79d53
+md5sum = 9920d10442604f5d8a736825f20ebd2a
 
 [gitlab-unicorn-startup.in]
 _update_hash_filename_ = gitlab-unicorn-startup.in
-md5sum = aff91edaf9786c213db8ea703ab3571e
+md5sum = 240957eac87e0b074589acd27d284e04
 
 [gitlab.yml.in]
 _update_hash_filename_ = template/gitlab.yml.in
-md5sum = f4cc0bc898b8d59010d61473e2adc53b
+md5sum = bedbfc03eaf3560a93cfde1f497e99f6
 
 [gitaly-config.toml.in]
 _update_hash_filename_ = template/gitaly-config.toml.in
@@ -54,7 +54,7 @@ md5sum = 056d7ed09e1bf20d022d3ef6b9363e00
 
 [instance-gitlab.cfg.in]
 _update_hash_filename_ = instance-gitlab.cfg.in
-md5sum = 6effe68f7ef240780d1027dd834a562c
+md5sum = 52abdcb629c513cfa1c31d47ac082c16
 
 [instance-gitlab-export.cfg.in]
 _update_hash_filename_ = instance-gitlab-export.cfg.in
@@ -90,8 +90,8 @@ md5sum = e2144b03f7247636143c65dc81550d75
 
 [template-gitlab-resiliency-restore.sh.in]
 _update_hash_filename_ = template/template-gitlab-resiliency-restore.sh.in
-md5sum = 590fcadf26085fdd17487175bc0a469d
+md5sum = f85c5bf252b8756558f40c8b110731cf
 
 [unicorn.rb.in]
 _update_hash_filename_ = template/unicorn.rb.in
-md5sum = 67728235a2c4c9425c80f0c856749885
+md5sum = 046cee1a8a024e55f8e64078b1fd5cb2

software/gitlab/gitlab-unicorn-startup.in

@@ -43,11 +43,6 @@ if echo "$pgtables" | grep -q '^Did not find any relations' ; then
 fi
 
 
-# re-build ssh keys
-# (we do not use them - just for cleannes)
-force=yes $RAKE gitlab:shell:setup   || die "gitlab:shell:setup failed"
-
-
 # 2. what to do when instance is upgraded
 # see
 #   https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/support/deploy/deploy.sh
@@ -68,6 +63,10 @@ test -s $migrate_log || rm $migrate_log
 # clear cache
 $RAKE cache:clear   || die "cache:clear failed"
 
+# re-build ssh keys
+# (we do not use them - just for cleannes)
+# run before migration to avoir error on missing tables in db
+force=yes $RAKE gitlab:shell:setup   || die "gitlab:shell:setup failed"
 
 
 # 3. finally exec to unicorn

software/gitlab/gowork.cfg

@@ -25,7 +25,7 @@ revision      = v0.8.0-12-g816c908556
 <= go-git-package
 go.importpath = lab.nexedi.com/kirr/git-backup
 repository    = https://lab.nexedi.com/kirr/git-backup.git
-revision      = 3f6c4deec8834bdcd2c28c7c5eeacd8211e759b5
+revision      = 0985ff32d6b04c9d3c6124c0a46c239790151b81
 
 [go_lab.nexedi.com_kirr_go123]
 <= go-git-package

software/gitlab/instance-gitlab.cfg.in

@@ -432,6 +432,7 @@ tune-command =
 software = {{ gitlab_shell_repository_location }}
 
 tune-command =
+    cp ${:software}/bin/* bin &&
     ln -sf ${gitlab-shell-config.yml:rendered}   config.yml  &&
     true
 
@@ -537,6 +538,7 @@ config-command = ${service-redis:promise_wrapper}
 <= logrotate-entry-base
 log     = ${redis:log}/*.log
 name = redis
+truncate = true
 
 
 ########################
@@ -563,6 +565,7 @@ command-line    = {{ gitlab_workhorse }}
     -documentRoot ${gitlab-work:location}/public
     -secretPath ${gitlab-workhorse:secret}
     -logFile ${gitlab-workhorse:log}
+    -repoPath ${gitlab-repo-dir:repositories}
 # NOTE for profiling
 #   -pprofListenAddr ...
 
@@ -650,22 +653,26 @@ command-line    = ${:rake} gitlab:gitlab_shell:check
 [logrotate-entry-unicorn]
 <= logrotate-entry-base
 log     = ${unicorn:log}/*.log
-name    = unicorn 
+name    = unicorn
+truncate = true
 
 [logrotate-entry-gitlab]
 <= logrotate-entry-base
 log     = ${gitlab:log}/*.log
 name    = gitlab
+truncate = true
 
 [logrotate-entry-gitlab-shell]
 <= logrotate-entry-base
 log     = ${gitlab-shell:log}/*.log
 name    = gitlab-shell
+truncate = true
 
 [logrotate-entry-gitlab-workhorse]
 <= logrotate-entry-base
 log     = ${gitlab-workhorse-dir:log}//*.log
-name    = gitlab-shell
+name    = gitlab-workhorse
+truncate = true
 
 #######################################
 #   sidekiq background jobs manager   #
@@ -715,6 +722,7 @@ command-line    = ${:rake} gitlab:sidekiq:check
 <= logrotate-entry-base
 log     = ${sidekiq:log}/*.log
 name    = sidekiq
+truncate = true
 
 
 ######################
@@ -787,6 +795,7 @@ module = check_url_available
 <= logrotate-entry-base
 log     = ${nginx:log}/*.log
 name    = nginx
+post = kill -USR1 $(cat ${directory:run}/nginx.pid)
 
 # base entry for clients who registers to cron
 [cron-entry]
@@ -832,7 +841,6 @@ command =
     ${:rake} gitlab:assets:clean  &&
     ${:rake} gettext:compile RAILS_ENV=production &&
     cd ${gitlab-work:location} &&
-    PATH={{ node_bin_location }}:$PATH {{ yarn_location }}/bin/yarn add ajv@^4.11.2 &&
     PATH={{ node_bin_location }}:$PATH {{ yarn_location }}/bin/yarn install --production --pure-lockfile &&
     ${:rake} gitlab:assets:compile NODE_ENV=production NODE_OPTIONS="--max_old_space_size=4096" &&
     true

software/gitlab/software.cfg

@@ -15,7 +15,6 @@ extends =
     ../../component/openssl/buildout.cfg
     ../../component/nginx/buildout.cfg
     ../../component/zlib/buildout.cfg
-    ../../component/icu/buildout.cfg
     gowork.cfg
 
 #   for instance
@@ -30,7 +29,7 @@ extends =
     ../../component/logrotate/buildout.cfg
 
 parts =
-    ruby2.3
+    ruby2.6
     golang1.12
     git
     postgresql10
@@ -43,11 +42,9 @@ parts =
     gowork
     gitlab-workhorse
     gitaly-build
-    python-4gitlab
     gitlab-shell/vendor
     gitlab/vendor/bundle
     gitlab_npm
-    github-markup-patch
     gitlab-backup
 
 #   for instance
@@ -71,19 +68,36 @@ revision = 571d6514f7290e8faa9439c4b86aa2f6c87df261
 [yarn]
 # need this version of Yarn
 recipe = slapos.recipe.build:download-unpacked
-url = https://github.com/yarnpkg/yarn/releases/download/v1.3.2/yarn-v1.3.2.tar.gz
-md5sum = db82fa09c996e9318f2f1d2ab99228f9
+url = https://github.com/yarnpkg/yarn/releases/download/v1.10.0/yarn-v1.10.0.tar.gz
+md5sum = c8ad5d2a2f02ba666e92189dae951575
 
 ############################
 #   Software compilation   #
 ############################
 
 # python with eggs, that will be used in gitlab
+# gitlab-markup call the command `python3 /path/to/commands/rest2html` which
+# require docutils
+# https://gitlab.com/gitlab-org/gitlab-markup/-/blob/master/lib/github/markups.rb
 [python-4gitlab]
-recipe  = zc.recipe.egg
-interpreter = python2
-eggs    =
-    docutils
+recipe = plone.recipe.command
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+bin = ${:location}/bin
+eggs =
+  docutils
+python-path = ${:location}/lib/python${python3:version}/site-packages
+command =
+  ${python3:location}/bin/pip3 install ${:eggs} --prefix=${:location}
+  mkdir -p ${:bin}
+  cat <<EOF > ${:location}/bin/python3
+    #!/bin/sh
+    export PATH=${python3:location}/bin:\$PATH
+    export PYTHONPATH=${:python-path}
+
+    exec python3 "\$@"
+  EOF
+  chmod 700 ${:location}/bin/python3
+stop-on-error = true
 
 # rubygemsrecipe with fixed url and this way pinned rubygems version
 [rubygemsrecipe]
@@ -95,7 +109,7 @@ url     = https://rubygems.org/rubygems/rubygems-3.1.2.zip
 # - run gitlab services / jobs  (via `bundle exec ...`)
 [bundler-4gitlab]
 <= rubygemsrecipe
-ruby-location = ${ruby2.3:location}
+ruby-location = ${ruby2.6:location}
 ruby-executable = ${:ruby-location}/bin/ruby
 gems    =
   bundler==1.17.3
@@ -118,11 +132,11 @@ bundle1.17.3 = ${buildout:parts-directory}/${:_buildout_section_name_}/lib/ruby/
 # gitlab wants git to be really on path ( it uses git from abspath defined in
 #   gitlab.yml, but there are not all cases like this, e.g. in
 #   https://gitlab.com/gitlab-org/gitlab_git/blob/2f0d3c1a/lib/gitlab_git/repository.rb#L259 )
-# gitlab (via github-markup) wants to convert rst -> html via running: python2 (with docutils egg)
-# (python-4gitlab puts interpreter into ${buildout:bin-directory})
+# gitlab (via github-markup) wants to convert rst -> html via running: python3 (with docutils egg)
+# ======>  (python-4gitlab puts interpreter into ${buildout:bin-directory})
 environment =
 
-  PATH    = ${yarn:location}/bin:${:ruby-location}/bin:${cmake:location}/bin:${pkgconfig:location}/bin:${nodejs-8.12.0:location}/bin:${postgresql10:location}/bin:${redis28:location}/bin:${git:location}/bin:${buildout:bin-directory}:%(PATH)s
+  PATH    = ${python-4gitlab:bin}:${yarn:location}/bin:${:ruby-location}/bin:${cmake:location}/bin:${pkgconfig:location}/bin:${nodejs-8.12.0:location}/bin:${postgresql10:location}/bin:${redis28:location}/bin:${git:location}/bin:${buildout:bin-directory}:%(PATH)s
 
 
 # gitlab, gitlab-shell & gitlab-workhorse checked out as git repositories
@@ -134,44 +148,29 @@ git-executable = ${git:location}/bin/git
 [gitlab-repository]
 <= git-repository
 repository = https://lab.nexedi.com/nexedi/gitlab-ce.git
-# 9.5.10 + NXD patches:
-revision = v9.5.10-8-gc290e22a08cb
+# 11.11.8 + NXD patches:
+revision = v11.11.8-7-g193b94c3e2
 location = ${buildout:parts-directory}/gitlab
 
 [gitlab-shell-repository]
 <= git-repository
-#repository = https://lab.nexedi.com/nexedi/gitlab-shell.git
 repository = https://gitlab.com/gitlab-org/gitlab-shell.git
-# gitlab 9.5.10 wants gitlab-shell 5.6.1
-revision = v5.6.1-10-g1e587d3b7f
+# gitlab 11.11.8 wants gitlab-shell 9.1.0
+revision = v9.1.0-0-g0e3052be56
 location = ${buildout:parts-directory}/gitlab-shell
 
 [gitaly-repository]
 <= git-repository
 repository = https://gitlab.com/gitlab-org/gitaly.git
-# for version v0.35.0 (gitlab 9.5.10)
-revision = v0.35.0-0-gf99a57b19a
+# gitaly version v1.42.7 (gitlab 11.11.8)
+revision = v1.42.7-0-ge47c816beb
 location = ${buildout:parts-directory}/gitaly
 
 [gitlab-workhorse-repository]
 <= git-repository
 repository = https://lab.nexedi.com/nexedi/gitlab-workhorse.git
-revision = v3.0.0-8-g74793ad3cc
-
-# Patch github markup to not call "python2 -S /path/to/rest2html" but only "python2 /path/to/rest2html"
-# NOTE github-markup invokes it as `python2`, that's why we are naming it this way
-# https://github.com/github/markup/blob/5393ae93/lib/github/markups.rb#L36
-[github-markup-patch]
-recipe = plone.recipe.command
-command =
-  files=$(ls ${gitlab-repository:location}/vendor/bundle/ruby/*/gems/git*-markup-*/lib/github/markups.rb) || true
-  if [ ! -z "$files" ]; then
-    for file in $files; do
-      sed -i 's#python2 -S#python2#' $file
-    done
-  fi
-update-command = ${:command}
-stop-on-error = True
+revision = v8.7.0-11-gfc590d3581
+#revision = a2c07d3068717158ad710304a51062d7de24f37d
 
 # build needed-by-gitlab gems via bundler
 [gitlab/vendor/bundle]
@@ -183,12 +182,13 @@ configure-command = cd ${:path} &&
     ${:bundle} config --local build.charlock_holmes --with-icu-dir=${icu:location}  &&
     ${:bundle} config --local build.pg --with-pg-config=${postgresql10:location}/bin/pg_config &&
     ${:bundle} config --local build.re2 --with-re2-dir=${re2:location}
+    ${:bundle} config set without 'development test mysql aws kerberos'
+    ${:bundle} config set deployment 'true'
 
 make-binary =
-make-targets= cd ${:path} &&
-    ${:bundle} install --deployment  --without development test mysql aws kerberos
+make-targets= cd ${:path} && ${:bundle} install
 environment =
-  PKG_CONFIG_PATH=${openssl-1.0:location}/lib/pkgconfig:${re2:location}/lib/pkgconfig
+  PKG_CONFIG_PATH=${openssl-1.0:location}/lib/pkgconfig:${re2:location}/lib/pkgconfig:${icu:location}/lib/pkgconfig
   PATH=${pkgconfig:location}/bin:%(PATH)s
 
 ################## Google re2
@@ -270,7 +270,7 @@ make-targets =
   . ${gowork:env.sh} && make
 environment =
   PKG_CONFIG_PATH=${openssl-1.0:location}/lib/pkgconfig:${icu:location}/lib/pkgconfig
-  PATH=${pkgconfig:location}/bin:${ruby2.3:location}/bin:%(PATH)s
+  PATH=${pkgconfig:location}/bin:${ruby2.6:location}/bin:%(PATH)s
 
 
 [xnice-repository]
@@ -292,7 +292,11 @@ bundle  = ${bundler-4gitlab:bundle}
 configure-command = true
 make-binary =
 make-targets= cd ${:path} &&
+# Compile go binary
+    . ${gowork:env.sh} && bin/compile &&
     ${:bundle} install --deployment  --without development test
+environment =
+  PATH=${ruby2.6:location}/bin:%(PATH)s
 
 
 ###############################

software/gitlab/template/gitlab-shell-config.yml.in

@@ -24,7 +24,7 @@ http_settings:
 # Give the canonicalized absolute pathname,
 # REPOS_PATH MUST NOT CONTAIN ANY SYMLINK!!!
 # Check twice that none of the components is a symlink, including "/home".
-# repos_path: "{{ gitlab.repositories }}"
+repos_path: "{{ gitlab.repositories }}"
 
 # File used as authorized_keys for gitlab user
 # NOTE not used in slapos version (all access via https only)

software/gitlab/template/gitlab.yml.in

@@ -171,6 +171,16 @@ production: &base
     storage_path: <%= @lfs_storage_path %>
   #}
 
+  ## Uploads
+  uploads:
+    # The location where uploads objects are stored (default: public/).
+    storage_path: "{{ gitlab.var }}"
+    # The location where uploads objects are stored (default: public/).
+    # storage_path: public/
+    # base_dir: uploads/-/system
+    object_store:
+      enabled: false
+      remote_directory: uploads # Bucket name
 
   {# we do not support container registry
   ## Container Registry

software/gitlab/template/template-gitlab-resiliency-restore.sh.in

@@ -29,6 +29,7 @@ gitlab_work="{{ gitlab_work_location }}"
 promise_check="{{ promise_lab_location }}"
 unicorn_script="{{ unicorn_script }}"
 sidekiq_script="{{ sidekiq_script }}"
+var_location="{{ run_directory }}/.."
 
 # export GIT_EXEC_PATH=$git_location/libexec/git-core/
 
@@ -61,6 +62,12 @@ if [ -f "$postgres_pid_file" ]; then
   rm $postgres_pid_file
 fi
 
+# cleanup /var/backup and old repositories folders,
+# restoration will created them at every run
+echo "Cleanup gitlab backup and old repositories folders..."
+rm -rf $var_location/backup/*
+rm -rf $var_location/repositories*
+
 echo "Starting Postgres..."
 $postgres_executable &
 postgres_pid=$!

software/gitlab/template/unicorn.rb.in

@@ -20,8 +20,6 @@ timeout {{ cfg('unicorn_worker_timeout') }}
 # combine Ruby 2.0.0dev or REE with "preload_app true" for memory savings
 # http://rubyenterpriseedition.com/faq.html#adapt_apps_for_cow
 preload_app true
-GC.respond_to?(:copy_on_write_friendly=) and
-  GC.copy_on_write_friendly = true
 
 
 # Enable this flag to have unicorn test client connections by writing the
@@ -32,6 +30,13 @@ GC.respond_to?(:copy_on_write_friendly=) and
 # fast LAN.
 check_client_connection false
 
+require_relative '{{ gitlab_work.location }}/lib/gitlab/cluster/lifecycle_events'
+
+before_exec do |server|
+  # Signal application hooks that we're about to restart
+  Gitlab::Cluster::LifecycleEvents.do_before_master_restart
+end
+
 # How many worker processes
 worker_processes {{ cfg('unicorn_worker_processes') }}
 
@@ -41,11 +46,8 @@ worker_processes {{ cfg('unicorn_worker_processes') }}
 
 # What to do before we fork a worker
 before_fork do |server, worker|
-  # XXX why gitlab does not enable this?
-  # # the following is highly recomended for Rails + "preload_app true"
-  # # as there's no need for the master process to hold a connection
-  # defined?(ActiveRecord::Base) and
-  #   ActiveRecord::Base.connection.disconnect!
+  # Signal application hooks that we're about to fork
+  Gitlab::Cluster::LifecycleEvents.do_before_fork
 
   # The following is only recommended for memory/DB-constrained
   # installations.  It is not needed if your system can house
@@ -75,28 +77,16 @@ end
 
 # What to do after we fork a worker
 after_fork do |server, worker|
+  # Signal application hooks of worker start
+  Gitlab::Cluster::LifecycleEvents.do_worker_start
+
   # per-process listener ports for debugging/admin/migrations
   # addr = "127.0.0.1:#{9293 + worker.nr}"
   # server.listen(addr, :tries => -1, :delay => 5, :tcp_nopush => true)
-
-  # XXX why gitlab does not enable this?
-  # # the following is *required* for Rails + "preload_app true",
-  # defined?(ActiveRecord::Base) and
-  #   ActiveRecord::Base.establish_connection
-  
-  # reset prometheus client, this will cause any opened metrics files to be closed
-  #defined?(::Prometheus::Client.reinitialize_on_pid_change) &&
-  #  Prometheus::Client.reinitialize_on_pid_change
-
-  # if preload_app is true, then you may also want to check and
-  # restart any other shared sockets/descriptors such as Memcached,
-  # and Redis.  TokyoCabinet file handles are safe to reuse
-  # between any number of forked children (assuming your kernel
-  # correctly implements pread()/pwrite() system calls)
-
 end
 
 
+
 # Where to drop a pidfile
 pid '{{ directory.run }}/unicorn.pid'