gitlab: Hook gitlab- and gitlab-shell- configuration files into the system

- Download them on SR build and pass info to instance
- Instance prepares to process them as jinja2 templates
- Instance hooks the files into configuration location as appropriate

Every file so far is renamed *.erb -> *.in and a header added showing
that this file is autogenerated with links about what was the base
gitlab and/or omnibus version and omnibus reference revision this
template was last updated for.

So far all result configuration files are invalid - because ERB syntax
is there. We will convert the configuration files to proper jinja2
syntax and to using slapos parameters incrementally in the upcoming
patches.

NOTE (again): md5 sums are not yet fixed - we will fix them in the end
    of gitlab patches series after applying all tweaking changes.

/cc @kazuhiko, @jerome

software/gitlab/instance-gitlab.cfg.in

@@ -118,7 +118,58 @@ mode    = 0700
 
 
 
-# 2. bin/
+# 2. configuration files
+[etc-template]
+recipe  = slapos.recipe.template:jinja2
+extensions = jinja2.ext.do
+mode    = 0640
+context =
+    raw     autogenerated       # This file was autogenerated. (DO NOT EDIT - changes will be lost)
+    section instance_parameter  instance-parameter
+    ${:context-extra}
+context-extra =
+
+[gitlab-etc-template]
+<= etc-template
+rendered= ${gitlab:etc}/${:_buildout_section_name_}
+
+
+[config.ru]
+<= gitlab-etc-template
+template = {{ config_ru_in }}
+
+[database.yml]
+<= gitlab-etc-template
+template= {{ database_yml_in }}
+
+[gitlab-shell-config.yml]
+<= etc-template
+template= {{ gitlab_shell_config_yml_in }}
+rendered= ${gitlab-shell:etc}/config.yml
+
+[gitlab.yml]
+<= gitlab-etc-template
+template= {{ gitlab_yml_in }}
+
+[rack_attack.rb]
+<= gitlab-etc-template
+template = {{ rack_attack_rb_in }}
+
+[resque.yml]
+<= gitlab-etc-template
+template= {{ resque_yml_in }}
+
+[smtp_settings.rb]
+<= gitlab-etc-template
+template= {{ smtp_settings_rb_in }}
+
+[unicorn.rb]
+<= gitlab-etc-template
+template = {{ unicorn_rb_in }}
+
+
+
+# 3. bin/
 #   gitlab-<prog>
 [gitlab-bin]
 recipe  = slapos.cookbook:wrapper
@@ -140,7 +191,7 @@ prog    = {{ prog }}
 {% endfor %}
 
 
-# 3. gitlab- & gitlab-shell- work directories
+# 4. gitlab- & gitlab-shell- work directories
 #
 # Gitlab/Rails operation is tightened that config/ lives inside code, which goes
 # against having ability to create several instances configured differently
@@ -188,17 +239,27 @@ update-command =
 <= work-base
 software = {{ gitlab_repository_location }}
 tune-command =
-# secret* tmp/ log/
+# secret* config.ru tmp/ log/
     rm -f .secret  &&
+    rm -f config.ru  &&
     rm -rf log tmp  &&
     ln -sf ${secrets:secrets}/gitlab_rails_secret .secret  &&
+    ln -sf ${config.ru:rendered} config.ru  &&
     ln -sf ${gitlab:log} log  &&
     ln -sf ${gitlab:tmp} tmp  &&
 # config/
     cd config  &&
+    ln -sf ${unicorn.rb:rendered} unicorn.rb  &&
+    ln -sf ${gitlab.yml:rendered} gitlab.yml  &&
+    ln -sf ${database.yml:rendered} database.yml  &&
+    ln -sf ${resque.yml:rendered} resque.yml  &&
     ln -sf ${secrets:secrets}/gitlab_secrets.yml secrets.yml  &&
+# config/initializers/
+    cd initializers  &&
+    ln -sf ${rack_attack.rb:rendered} rack_attack.rb  &&
+    ln -sf ${smtp_settings.rb:rendered} smtp_settings.rb  &&
 # public/
-    cd ../public  &&
+    cd ../../public  &&
     rm -rf uploads assets  &&
     ln -sf ${gitlab:uploads} uploads  &&
     ln -sf ${gitlab:assets} assets  &&
@@ -211,12 +272,12 @@ tune-command =
 software = {{ gitlab_shell_repository_location }}
 
 tune-command =
+    ln -sf ${gitlab-shell-config.yml:rendered}   config.yml  &&
     true
 
 
 
-
-# 4. services
+# 5. services
 
 # [promise-<something>] to generate promise wrapper <something>
 [promise-wrapper]

software/gitlab/instance.cfg.in

@@ -34,3 +34,13 @@ context =
     raw logrotate_bin               ${logrotate:location}/usr/sbin/logrotate
     raw postgresql_location         ${postgresql92:location}
     raw redis_binprefix             ${redis28:location}/bin
+
+# config files
+    raw config_ru_in                ${config.ru.in:target}
+    raw database_yml_in             ${database.yml.in:target}
+    raw gitlab_shell_config_yml_in  ${gitlab-shell-config.yml.in:target}
+    raw gitlab_yml_in               ${gitlab.yml.in:target}
+    raw rack_attack_rb_in           ${rack_attack.rb.in:target}
+    raw resque_yml_in               ${resque.yml.in:target}
+    raw smtp_settings_rb_in         ${smtp_settings.rb.in:target}
+    raw unicorn_rb_in               ${unicorn.rb.in:target}

software/gitlab/software.cfg

@@ -178,11 +178,48 @@ recipe  = slapos.recipe.build:download
 url     = ${:_profile_base_location_}/${:_buildout_section_name_}
 destination = ${buildout:directory}/${:_buildout_section_name_}
 
+# like download-file, but download from template/<filename>
+[download-template]
+<= download-file
+url     = ${:_profile_base_location_}/template/${:_buildout_section_name_}
+
+
+[config.ru.in]
+<= download-template
+# md5sum = TODO
+
+[database.yml.in]
+<= download-template
+# md5sum = TODO
+
+[gitlab-shell-config.yml.in]
+<= download-template
+# md5sum = TODO
+
+[gitlab.yml.in]
+<= download-template
+# md5sum = TODO
 
 [instance-gitlab.cfg.in]
 <= download-file
 # md5sum = TODO
 
+[rack_attack.rb.in]
+<= download-template
+# md5sum = TODO
+
+[resque.yml.in]
+<= download-template
+# md5sum = TODO
+
+[smtp_settings.rb.in]
+<= download-template
+# md5sum = TODO
+
+[unicorn.rb.in]
+<= download-template
+# md5sum = TODO
+
 
 [versions]
 plone.recipe.command = 1.1

software/gitlab/template/gitlab-rails-config.ru.erb → software/gitlab/template/config.ru.in

+{{ autogenerated }}
+# see:
+# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config.ru
+# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/gitlab-rails-config.ru.erb
+# (last updated for omnibus-gitlab 8.2.3+ce.0-0-g8eda093)
+
 # This file is used by Rack-based servers to start the application.
 
 if defined?(Unicorn)

software/gitlab/template/database.yml.erb → software/gitlab/template/database.yml.in

-# This file is managed by gitlab-ctl. Manual changes will be
-# erased! To change the contents below, edit /etc/gitlab/gitlab.rb
-# and run `sudo gitlab-ctl reconfigure`.
+{{ autogenerated }}
+# see:
+# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/database.yml.postgresql
+# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/database.yml.erb
+# (last updated for 8.2.3+ce.0-0-g8eda093)
 
 production:
   adapter: <%= @db_adapter %>

software/gitlab/template/gitlab-shell-config.yml.erb → software/gitlab/template/gitlab-shell-config.yml.in

-# This file is managed by gitlab-ctl. Manual changes will be
-# erased! To change the contents below, edit /etc/gitlab/gitlab.rb
-# and run `sudo gitlab-ctl reconfigure`.
+{{ autogenerated }}
+# see:
+# https://gitlab.com/gitlab-org/gitlab-shell/blob/master/config.yml.example
+# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/gitlab-shell-config.yml.erb
+# (last updated for omnibus-gitlab 8.2.3+ce.0-0-g8eda093)
 
 # GitLab user. git by default
 user: <%= @user %>

software/gitlab/template/gitlab.yml.erb → software/gitlab/template/gitlab.yml.in

-# This file is managed by gitlab-ctl. Manual changes will be
-# erased! To change the contents below, edit /etc/gitlab/gitlab.rb
-# and run `sudo gitlab-ctl reconfigure`.
+{{ autogenerated }}
+# see:
+# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/gitlab.yml.example
+# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/gitlab.yml.erb
+# (last updated for omnibus-gitlab 8.2.3+ce.0-0-g8eda093)
 
 production: &base
   #

software/gitlab/template/rack_attack.rb.erb → software/gitlab/template/rack_attack.rb.in

-# This file is managed by gitlab-ctl. Manual changes will be
-# erased! To change the contents below, edit /etc/gitlab/gitlab.rb
-# and run `sudo gitlab-ctl reconfigure`.
+{{ autogenerated }}
+# see:
+# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/initializers/rack_attack.rb.example
+# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/rack_attack.rb.erb
+# (last updated for omnibus-gitlab 8.2.3+ce.0-0-g8eda093)
 
 # 1. Rename this file to rack_attack.rb
 # 2. Review the paths_to_be_protected and add any other path you need protecting

software/gitlab/template/resque.yml.erb

-production: <%= @redis_url %>

software/gitlab/template/resque.yml.in

+{{ autogenerated }}
+# see:
+# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/resque.yml.example
+# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/resque.yml.erb
+# (last udpdated for omnibus-gitlab 8.2.3+ce.0-0-g8eda093)
+
+production: <%= @redis_url %>

software/gitlab/template/smtp_settings.rb.erb → software/gitlab/template/smtp_settings.rb.in

-# This file is managed by gitlab-ctl. Manual changes will be
-# erased! To change the contents below, edit /etc/gitlab/gitlab.rb
-# and run `sudo gitlab-ctl reconfigure`.
+{{ autogenerated }}
+# see:
+# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/initializers/smtp_settings.rb.sample
+# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/smtp_settings.rb.erb
+# (last updated for omnibus-gitlab 8.2.3+ce.0-0-g8eda093)
 
 <% rails_app = @app == 'gitlab' ? 'Gitlab' : 'GitlabCi' %>
 <% container = @app == 'gitlab' ? 'gitlab-rails' : 'gitlab-ci' %>

software/gitlab/template/unicorn.rb.erb → software/gitlab/template/unicorn.rb.in

-# This file is managed by gitlab-ctl. Manual changes will be
-# erased! To change the contents below, edit /etc/gitlab/gitlab.rb
-# and run `sudo gitlab-ctl reconfigure`.
+{{ autogenerated }}
+# see:
+# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/unicorn.rb.example
+# https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/unicorn.rb.example.development
+# https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/unicorn.rb.erb
+# (last updated for omnibus-gitlab 8.2.3+ce.0-0-g8eda093)
 
 
 # What ports/sockets to listen on, and what options for them.