gitlab/nginx: Slapos'ify config and turn nginx into a service

Go through nginx configuration templates and convert them to jinja2 with
slapos parameters (reminder: names and default values are imported from
omnibus-gitlab 8.2.3+ce.0-0-g8eda093), except commenting out features we
do not want to support (yet ?).

As nginx is a reverse-proxy, i.e. it integrates all internal services
and works as frontend to them, our gitlab service is now ready to listen
and talk to the world over (standard to slapos services backend) IPv6.

Nginx also acts as SSL termination point - for it to work by default we
setup self-signed certificate for the backend, which can be manually
changed to proper certificate if needed. Backend certificate is used
if gitlab is configured to work in HTTPS mode (and frontend certificate
is another story).

NOTE ssl certificate is generated with just `openssl req ...` - yes, there
    is slapos.cookbook:certificate_authority.request but it requires
    to start whole service and has up to 60 seconds latency to generate
    certificate. And we only need to run 1 command to do that...

The features disabled are:

    - http -> https redirection

      not needed for us at nginx level - the frontend can do the
      redirection and also gitlab speaks HSTS on https port so when we access
      https port via http protocol, it gets redirected to https.

    - kerberos
    - ssl_dhparam
    - providing custom nginx configuration via instance parameter

/cc @kazuhiko, @jerome

software/gitlab/gitlab-parameters.cfg

@@ -74,3 +74,32 @@ configuration.unicorn_backlog_socket    = 1024
 
 configuration.unicorn_worker_memory_limit_min   = 200*(1024**2)
 configuration.unicorn_worker_memory_limit_max   = 250*(1024**2)
+
+
+# nginx
+configuration.nginx_client_max_body_size    = 250m
+
+# NOTE: we don't really need old ciphers - usually we talk directly to frontend only
+configuration.nginx_ssl_ciphers             = ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-DES-CBC3-SHA:AES256-GCM-SHA384:AES128-GCM-SHA256:AES256-SHA256:AES128-SHA256:AES256-SHA:AES128-SHA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!MD5:!PSK:!RC4
+configuration.nginx_ssl_prefer_server_ciphers = on
+configuration.nginx_ssl_protocols           = TLSv1 TLSv1.1 TLSv1.2
+# the following is gitlab-omnibus default but not nginx's default
+configuration.nginx_ssl_session_cache       = builtin:1000  shared:SSL:10m
+configuration.nginx_ssl_session_timeout     = 5m
+
+configuration.nginx_proxy_read_timeout      = 300
+configuration.nginx_proxy_connect_timeout   = 300
+
+# nginx advanced
+configuration.nginx_worker_processes    = 4
+configuration.nginx_worker_connections  = 10240
+configuration.nginx_log_format          = $remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"
+configuration.nginx_sendfile            = on
+configuration.nginx_tcp_nopush          = on
+configuration.nginx_tcp_nodelay         = on
+configuration.nginx_gzip                = on
+configuration.nginx_gzip_http_version   = 1.0
+configuration.nginx_gzip_comp_level     = 2
+configuration.nginx_gzip_proxied        = any
+configuration.nginx_gzip_types          = text/plain text/css application/x-javascript text/xml application/xml application/xml+rss text/javascript application/json
+configuration.nginx_keepalive_timeout   = 65

software/gitlab/instance-gitlab.cfg.in

@@ -24,6 +24,7 @@ parts =
     service-unicorn
     service-sidekiq
 
+    service-nginx
     service-postgresql
     service-redis
 
@@ -57,11 +58,21 @@ cert    = ${slap-connection:cert-file}
 
 # automatically load all available CPUs
 configuration.unicorn_worker_processes  = {{ multiprocessing.cpu_count() + 1 }}
+configuration.nginx_worker_processes    = {{ multiprocessing.cpu_count() }}
 
 
 
 # for convenience
+[external-url]
+recipe  = slapos.cookbook:urlparse
+url     = ${instance-parameter:configuration.external_url}
+
 [backend-info]
+host    = ${instance-parameter:ipv6-random}
+port    = 7777
+# whether to use http or https - determined by external url
+url     = ${external-url:scheme}://[${:host}]:${:port}
+
 # current slapuserX
 user    = {{ pwd.getpwuid(os.getuid())[0] }}
 
@@ -207,10 +218,19 @@ context-extra =
 [nginx.conf]
 <= nginx-etc-template
 template= {{ nginx_conf_in }}
+context-extra =
+    section directory               directory
+    raw     nginx_mime_types        {{ nginx_mime_types }}
+    raw     nginx_gitlab_http_conf  ${nginx-gitlab-http.conf:rendered}
 
 [nginx-gitlab-http.conf]
 <= nginx-etc-template
 template= {{ nginx_gitlab_http_conf_in }}
+context-extra =
+    section nginx                   nginx
+    section gitlab_work             gitlab-work
+    section gitlab_workhorse        gitlab-workhorse
+    section unicorn                 unicorn
 
 [rack_attack.rb]
 <= gitlab-etc-template
@@ -637,6 +657,74 @@ command-line    = ${:rake} gitlab:sidekiq:check
 log     = ${sidekiq:log}/*.log
 
 
+######################
+#   Nginx frontend   #
+######################
+
+# srv/nginx/ prefix  +  etc/ log/ ...
+[nginx-dir]
+recipe  = slapos.cookbook:mkdirectory
+srv     = ${directory:srv}/nginx
+etc     = ${directory:etc}/nginx
+log     = ${directory:log}/nginx
+
+[nginx-ssl-dir]
+recipe  = slapos.cookbook:mkdirectory
+ssl     = ${nginx-dir:etc}/ssl
+# contains https key
+mode    = 0700
+
+# self-signed certificate for https
+[nginx-generate-certificate]
+# NOTE there is slapos.cookbook:certificate_authority.request but it requires
+# to start whole service and has up to 60 seconds latency to generate
+# certificate. We only need to run 1 command to do it...
+recipe  = plone.recipe.command
+stop-on-error   = true
+cert_file   = ${nginx-ssl-dir:ssl}/gitlab_backend.crt
+key_file    = ${nginx-ssl-dir:ssl}/gitlab_backend.key
+
+command =
+    test -e ${:key_file} || \
+        {{ openssl_bin }} req -newkey rsa -batch -new -x509 -days 3650 -nodes   \
+        -keyout ${:key_file} -out ${:cert_file}
+update-command = ${:command}
+
+
+[nginx]
+srv     = ${nginx-dir:srv}
+etc     = ${nginx-dir:etc}
+log     = ${nginx-dir:log}
+ssl     = ${nginx-ssl-dir:ssl}
+
+cert_file   = ${nginx-generate-certificate:cert_file}
+key_file    = ${nginx-generate-certificate:key_file}
+
+
+[nginx-symlinks]
+# (nginx wants <prefix>/logs to be there from start - else it issues alarm to the log)
+recipe  = cns.recipe.symlink
+symlink = ${nginx:log}  = ${nginx:srv}/logs
+
+[service-nginx]
+recipe  = slapos.cookbook:wrapper
+wrapper-path    = ${directory:service}/nginx
+command-line    = {{ nginx_bin }} -p ${nginx:srv} -c ${nginx.conf:rendered}
+depend  =
+    ${nginx-symlinks:recipe}
+    ${promise-nginx:recipe}
+    ${logrotate-entry-nginx:recipe}
+
+
+[promise-nginx]
+<= promise-byurl
+url     = ${backend-info:url}/static.css
+
+[logrotate-entry-nginx]
+<= logrotate-entry
+log     = ${nginx:log}/*.log
+
+
 
 #############
 #   cron    #

software/gitlab/instance.cfg.in

@@ -40,6 +40,9 @@ context =
     raw gunzip_bin                  ${gzip:location}/bin/gunzip
     raw gzip_bin                    ${gzip:location}/bin/gzip
     raw logrotate_bin               ${logrotate:location}/usr/sbin/logrotate
+    raw nginx_bin                   ${nginx-output:nginx}
+    raw nginx_mime_types            ${nginx-output:mime}
+    raw openssl_bin                 ${openssl-output:openssl}
     raw postgresql_location         ${postgresql92:location}
     raw redis_binprefix             ${redis28:location}/bin
     raw ruby_location               ${bundler-4gitlab:ruby-location}

software/gitlab/macrolib.cfg.in

@@ -13,3 +13,6 @@
      process instance parameters ) #}
 {% set external_url = urlparse.urlparse(cfg('external_url')) %}
 {% set cfg_https = (true if external_url.scheme == 'https' else false) %}
+
+{# for convenience #}
+{% set fqdn = external_url.hostname %}

software/gitlab/software.cfg

@@ -165,6 +165,7 @@ environment =
 recipe  = zc.recipe.egg
 eggs    =
     plone.recipe.command
+    cns.recipe.symlink
 
 
 [instance.cfg]
@@ -261,6 +262,7 @@ url     = ${:_profile_base_location_}/template/${:_buildout_section_name_}
 
 
 [versions]
+cns.recipe.symlink = 0.2.3
 plone.recipe.command = 1.1
 rubygemsrecipe  = 0.2.2
 slapos.recipe.template = 2.9

software/gitlab/template/nginx-gitlab-http.conf.in

@@ -3,6 +3,8 @@
 # https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/nginx-gitlab-http.conf.erb
 # (last updated for omnibus-gitlab 8.2.3+ce.0-0-g8eda093)
 
+{% from 'macrolib.cfg.in' import cfg, cfg_bool, cfg_https, fqdn  with context %}
+
 ## GitLab
 ## Modified from https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/support/nginx/gitlab-ssl & https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/support/nginx/gitlab
 ##
@@ -32,13 +34,16 @@
 ###################################
 
 upstream gitlab {
-  server unix:<%= @socket %> fail_timeout=0;
+  server unix:{{ unicorn.socket }} fail_timeout=0;
 }
 
 upstream gitlab-workhorse {
-  server unix:<%= node['gitlab']['gitlab-workhorse']['listen_addr'] %>;
+  server unix:{{ gitlab_workhorse.socket }};
 }
 
+{# not needed for us - the frontend can do the redirection and also
+   gitlab/nginx speaks HSTS on https port so when we access https port via http
+   protocol, it gets redirected to https
 <% if @https && @redirect_http_to_https %>
 ## Redirects all HTTP traffic to the HTTPS host
 server {
@@ -52,49 +57,56 @@ server {
   error_log   <%= @log_directory %>/gitlab_error.log;
 }
 <% end %>
+#}
 
 server {
-<% @listen_addresses.each do |listen_address| %>
-  listen <%= listen_address %>:<%= @listen_port %><% if @https %> ssl spdy<% end %>;
+  listen [{{ backend_info.host }}]:{{ backend_info.port }}{% if cfg_https %} ssl spdy{% endif %};
 
+  {# we don't use: kerbeeros
   <% if @kerberos_enabled && @kerberos_use_dedicated_port %>
   listen <%= listen_address %>:<%= @kerberos_port %><% if @kerberos_https %> ssl<% end %>;
   <% end %>
+  #}
 
-<% end %>
-  server_name <%= @fqdn %>;
+  server_name {{ fqdn }};
   server_tokens off; ## Don't show the nginx version number, a security best practice
-  root /opt/gitlab/embedded/service/gitlab-rails/public;
+  root {{ gitlab_work.location }}/public;
 
   ## Increase this if you want to upload large attachments
   ## Or if you want to accept large git objects over http
-  client_max_body_size <%= @client_max_body_size %>;
+  client_max_body_size {{ cfg('nginx_client_max_body_size') }};
 
-  <% if @https %>
+  {% if cfg_https %}
   ## Strong SSL Security
   ## https://raymii.org/s/tutorials/Strong_SSL_Security_On_nginx.html & https://cipherli.st/
   ssl on;
-  ssl_certificate <%= @ssl_certificate %>;
-  ssl_certificate_key <%= @ssl_certificate_key %>;
+  ssl_certificate {{ nginx.cert_file }};
+  ssl_certificate_key {{ nginx.key_file }};
+  {# we don't need - most root CA will be included by default
   <% if @ssl_client_certificate %>
   ssl_client_certificate <%= @ssl_client_certificate%>;
  	<% end %>
+  #}
 
   # GitLab needs backwards compatible ciphers to retain compatibility with Java IDEs
-  ssl_ciphers '<%= @ssl_ciphers %>';
-  ssl_protocols  <%= @ssl_protocols %>;
-  ssl_prefer_server_ciphers <%= @ssl_prefer_server_ciphers %>;
-  ssl_session_cache  <%= @ssl_session_cache %>;
-  ssl_session_timeout  <%= @ssl_session_timeout %>;
-
+  # NOTE(slapos) ^^^ is not relevant for us - we are behind frontend and clients
+  #     directly connects to frontend
+  ssl_ciphers '{{ cfg("nginx_ssl_ciphers") }}';
+  ssl_protocols  {{ cfg('nginx_ssl_protocols') }};
+  ssl_prefer_server_ciphers {{ cfg('nginx_ssl_prefer_server_ciphers') }};
+  ssl_session_cache  {{ cfg('nginx_ssl_session_cache') }};
+  ssl_session_timeout  {{ cfg('nginx_ssl_session_timeout') }};
+
+  {# we do not use: ssl_dhparam
   <% if @ssl_dhparam %>
   ssl_dhparam <%= @ssl_dhparam %>;
   <% end %>
-  <% end %>
+  #}
+  {% endif %}
 
   ## Individual nginx logs for this GitLab vhost
-  access_log  <%= @log_directory %>/gitlab_access.log gitlab_access;
-  error_log   <%= @log_directory %>/gitlab_error.log;
+  access_log  {{ nginx.log }}/gitlab_access.log gitlab_access;
+  error_log   {{ nginx.log }}/gitlab_error.log;
 
   location / {
     ## Serve static files from defined root folder.
@@ -105,21 +117,21 @@ server {
   location /uploads/ {
     ## If you use HTTPS make sure you disable gzip compression
     ## to be safe against BREACH attack.
-    <%= 'gzip off;' if @https %>
+    {{ 'gzip off;' if cfg_https else ''}}
 
     ## https://github.com/gitlabhq/gitlabhq/issues/694
     ## Some requests take more than 30 seconds.
-    proxy_read_timeout      <%= @proxy_read_timeout %>;
-    proxy_connect_timeout   <%= @proxy_connect_timeout %>;
+    proxy_read_timeout      {{ cfg('nginx_proxy_read_timeout') }};
+    proxy_connect_timeout   {{ cfg('nginx_proxy_connect_timeout') }};
     proxy_redirect          off;
 
     proxy_set_header    Host                $http_host;
     proxy_set_header    X-Real-IP           $remote_addr;
-    <% if @https %>
+    {% if cfg_https %}
     proxy_set_header    X-Forwarded-Ssl     on;
-    <% end %>
+    {% endif %}
     proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
-    proxy_set_header    X-Forwarded-Proto   <%= @https ? "https" : "http" %>;
+    proxy_set_header    X-Forwarded-Proto   {{ "https" if cfg_https else "http" }};
     proxy_set_header    X-Frame-Options     SAMEORIGIN;
 
     proxy_pass http://gitlab;
@@ -130,21 +142,21 @@ server {
   location @gitlab {
     ## If you use HTTPS make sure you disable gzip compression
     ## to be safe against BREACH attack.
-    <%= 'gzip off;' if @https %>
+    {{ 'gzip off;' if cfg_https else ''}}
 
     ## https://github.com/gitlabhq/gitlabhq/issues/694
     ## Some requests take more than 30 seconds.
-    proxy_read_timeout      <%= @proxy_read_timeout %>;
-    proxy_connect_timeout   <%= @proxy_connect_timeout %>;
+    proxy_read_timeout      {{ cfg('nginx_proxy_read_timeout') }};
+    proxy_connect_timeout   {{ cfg('nginx_proxy_connect_timeout') }};
     proxy_redirect          off;
 
     proxy_set_header    Host                $http_host;
     proxy_set_header    X-Real-IP           $remote_addr;
-    <% if @https %>
+    {% if cfg_https %}
     proxy_set_header    X-Forwarded-Ssl     on;
-    <% end %>
+    {% endif %}
     proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
-    proxy_set_header    X-Forwarded-Proto   <%= @https ? "https" : "http" %>;
+    proxy_set_header    X-Forwarded-Proto   {{ "https" if cfg_https else "http" }};
     proxy_set_header    X-Frame-Options     SAMEORIGIN;
 
     proxy_pass http://gitlab;
@@ -198,21 +210,21 @@ server {
     client_max_body_size 0;
     ## If you use HTTPS make sure you disable gzip compression
     ## to be safe against BREACH attack.
-    <%= 'gzip off;' if @https %>
+    {{ 'gzip off;' if cfg_https else ''}}
 
     ## https://github.com/gitlabhq/gitlabhq/issues/694
     ## Some requests take more than 30 seconds.
-    proxy_read_timeout      <%= @proxy_read_timeout %>;
-    proxy_connect_timeout   <%= @proxy_connect_timeout %>;
+    proxy_read_timeout      {{ cfg('nginx_proxy_read_timeout') }};
+    proxy_connect_timeout   {{ cfg('nginx_proxy_connect_timeout') }};
     proxy_redirect          off;
 
     proxy_set_header    Host                $http_host;
     proxy_set_header    X-Real-IP           $remote_addr;
-    <% if @https %>
+    {% if cfg_https %}
     proxy_set_header    X-Forwarded-Ssl     on;
-    <% end %>
+    {% endif %}
     proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
-    proxy_set_header    X-Forwarded-Proto   <%= @https ? "https" : "http" %>;
+    proxy_set_header    X-Forwarded-Proto   {{ "https" if cfg_https else "http" }};
 
     proxy_pass http://gitlab-workhorse;
   }
@@ -223,7 +235,7 @@ server {
   ## See config/application.rb under "Relative url support" for the list of
   ## other files that need to be changed for relative url support
   location ~ ^/(assets)/ {
-    root /opt/gitlab/embedded/service/gitlab-rails/public;
+    root {{ gitlab_work.location }}/public;
     gzip_static on; # to serve pre-gzipped version
     expires max;
     add_header Cache-Control public;
@@ -232,5 +244,7 @@ server {
 
   error_page 502 /502.html;
 
+  {# we don't support custom nginx configs
   <%= @custom_gitlab_server_config %>
+  #}
 }

software/gitlab/template/nginx.conf.in

@@ -4,47 +4,47 @@
 # https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/nginx.conf.erb
 # (last updated for omnibus-gitlab 8.2.3+ce.0-0-g8eda093)
 
-user <%= node['gitlab']['web-server']['username'] %> <%= node['gitlab']['web-server']['group']%>;
-worker_processes <%= @worker_processes %>;
+{% from 'macrolib.cfg.in' import cfg  with context %}
+
+# user directive makes sense only when running initially as root
+# (and nginx will complain if not and directive given)
+# user {{ backend_info.user }};
+
+worker_processes {{ cfg('nginx_worker_processes') }};
 error_log stderr;
-pid nginx.pid;
+pid {{ directory.run }}/nginx.pid;
 
 daemon off;
 
 events {
-  worker_connections <%= @worker_connections %>;
+  worker_connections {{ cfg('nginx_worker_connections') }};
 }
 
 http {
-  log_format gitlab_access '<%= @gitlab_access_log_format %>';
+  log_format gitlab_access '{{ cfg("nginx_log_format") }}';
+  {# we do not use: ci, mattermost
   log_format gitlab_ci_access '<%= @gitlab_ci_access_log_format %>';
   log_format gitlab_mattermost_access '<%= @gitlab_mattermost_access_log_format %>';
+  #}
 
-  sendfile <%= @sendfile %>;
-  tcp_nopush <%= @tcp_nopush %>;
-  tcp_nodelay <%= @tcp_nodelay %>;
-
-  keepalive_timeout <%= @keepalive_timeout %>;
-
-  gzip <%= @gzip %>;
-  gzip_http_version <%= @gzip_http_version %>;
-  gzip_comp_level <%= @gzip_comp_level %>;
-  gzip_proxied <%= @gzip_proxied %>;
-  gzip_types <%= @gzip_types.join(' ') %>;
+  sendfile    {{ cfg('nginx_sendfile') }};
+  tcp_nopush  {{ cfg('nginx_tcp_nopush') }};
+  tcp_nodelay {{ cfg('nginx_tcp_nodelay') }};
 
-  include /opt/gitlab/embedded/conf/mime.types;
+  keepalive_timeout {{ cfg('nginx_keepalive_timeout') }};
 
-  <% if @gitlab_http_config %>
-  include <%= @gitlab_http_config %>;
-  <% end %>
+  gzip              {{ cfg('nginx_gzip') }};
+  gzip_http_version {{ cfg('nginx_gzip_http_version') }};
+  gzip_comp_level   {{ cfg('nginx_gzip_comp_level') }};
+  gzip_proxied      {{ cfg('nginx_gzip_proxied') }};
+  gzip_types        {{ cfg('nginx_gzip_types') }};
 
-  <% if @gitlab_ci_http_config %>
-  include <%= @gitlab_ci_http_config %>;
-  <% end %>
+  include {{ nginx_mime_types }};
 
-  <% if @gitlab_mattermost_http_config %>
-  include <%= @gitlab_mattermost_http_config %>;
-  <% end %>
+  include {{ nginx_gitlab_http_conf }};
 
-  <%= @custom_nginx_config %>
+  {# we don't need: ci, mattermost
+  include <%= @gitlab_ci_http_config %>
+  include <%= @gitlab_mattermost_http_config %>
+  #}
 }

[Vincent Pelletier]

Dangerous use of raw, use key or section.

[Vincent Pelletier]

Dangerous use of raw, use key.

[Kirill Smelkov]

/cc @alain.takoudjou @rafael