software/gitlab: use authenticate-to-backend for client IP

Introduces a new instance parameter, frontend-caucase-url-list which is
a space separated list of IP addresses (this software still uses xml
serialisation and does not have a parameter schema yet).

software/gitlab/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [instance.cfg]
 filename = instance.cfg.in
-md5sum = 3ffdd78aeb77ab581c51ce419176dd37
+md5sum = 3607ea995293975a736be136f0cdf675
 
 [watcher]
 _update_hash_filename_ = watcher.in
@@ -34,7 +34,7 @@ md5sum = c559a24ab6281268b608ed3bccb8e4ce
 
 [gitlab-parameters.cfg]
 _update_hash_filename_ = gitlab-parameters.cfg
-md5sum = 95b18789111ed239146d243e39ffefbe
+md5sum = 311f3b06ba5026b6aa958a86c58b815c
 
 [gitlab-shell-config.yml.in]
 _update_hash_filename_ = template/gitlab-shell-config.yml.in
@@ -54,7 +54,7 @@ md5sum = d769ea27820e932c596c35bbbf3f2902
 
 [instance-gitlab.cfg.in]
 _update_hash_filename_ = instance-gitlab.cfg.in
-md5sum = 6d8d20ded84622339d49c60b0e61380c
+md5sum = e17bfe96bb9a0f4666d90c877a663e88
 
 [instance-gitlab-export.cfg.in]
 _update_hash_filename_ = instance-gitlab-export.cfg.in
@@ -66,7 +66,7 @@ md5sum = 70612697434bf4fbe838fdf4fd867ed8
 
 [nginx-gitlab-http.conf.in]
 _update_hash_filename_ = template/nginx-gitlab-http.conf.in
-md5sum = 4980c1571a4dd7753aaa60d065270849
+md5sum = 093f8b1472294fd97213272f3fe1411a
 
 [nginx.conf.in]
 _update_hash_filename_ = template/nginx.conf.in

software/gitlab/gitlab-parameters.cfg

@@ -96,7 +96,7 @@ configuration.nginx_proxy_connect_timeout   = 300
 # nginx advanced
 configuration.nginx_worker_processes    = 4
 configuration.nginx_worker_connections  = 10240
-configuration.nginx_log_format          = $remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"
+configuration.nginx_log_format          = $trusted_remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"
 configuration.nginx_sendfile            = on
 configuration.nginx_tcp_nopush          = on
 configuration.nginx_tcp_nodelay         = on
@@ -115,7 +115,11 @@ configuration.nginx_gzip_enabled        = true
 # configuring trusted proxies
 # GitLab is behind a reverse proxy, so we don't want the IP address of the proxy
 # to show up as the client address (because rack attack blacklists the lab
-# frontend)
+# frontend). frontend-caucase-url-list offers a more manageable approach than listing
+# all frontends IPs.
 configuration.nginx_real_ip_trusted_addresses   =
 configuration.nginx_real_ip_header              = X-Forwarded-For
 configuration.nginx_real_ip_recursive           = off
+# space separated URLs of caucase service providing CA to validate frontends client
+# certificate and trust the frontend if they provide a valid certificate.
+configuration.frontend-caucase-url-list         =

software/gitlab/instance-gitlab.cfg.in

@@ -748,6 +748,63 @@ copytruncate = true
 #   Nginx frontend   #
 ######################
 
+[frontend-caucase-ssl]
+ca =
+crl =
+
+{% import "caucase" as caucase with context %}
+{% set frontend_caucase_url_list = instance_parameter_dict.get('configuration.frontend-caucase-url-list', '').split() -%}
+
+{% set frontend_caucase_url_hash_list = [] -%}
+{% set frontend_caucase_updater_section_list = [] -%}
+{% for frontend_caucase_url in frontend_caucase_url_list -%}
+{%   set hash = hashlib.md5(frontend_caucase_url.encode()).hexdigest() -%}
+{%   do frontend_caucase_url_hash_list.append(hash) -%}
+{%   set data_dir = '${nginx-ssl-dir:ssl}/%s' % hash -%}
+{{   caucase.updater(
+       prefix='frontend-caucase-updater-%s' % hash,
+       buildout_bin_directory=buildout_bin_directory,
+       updater_path='${directory:service}/frontend-caucase-updater-%s' % hash,
+       url=frontend_caucase_url,
+       data_dir=data_dir,
+       ca_path='%s/ca.crt' % data_dir,
+       crl_path='%s/crl.pem' % data_dir,
+       on_renew='${frontend-caucase-updater-housekeeper:output}',
+       max_sleep=1,
+       openssl=openssl_bin,
+     )}}
+{%   do frontend_caucase_updater_section_list.append('frontend-caucase-updater-%s' % hash) -%}
+{% endfor -%}
+
+{% if frontend_caucase_url_hash_list %}
+{%   do frontend_caucase_updater_section_list.append('frontend-caucase-updater-housekeeper') -%}
+[frontend-caucase-ssl]
+ca = ${nginx-ssl-dir:ssl}/frontend_ca.crt
+crl = ${nginx-ssl-dir:ssl}/frontend_crl.pem
+
+[frontend-caucase-updater-housekeeper]
+recipe = slapos.recipe.template
+output = ${directory:bin}/frontend-caucase-updater-housekeeper
+mode = 700
+inline =
+  #!/bin/sh -e
+  # assemble all frontends CA and CRL in one file
+  CA=${frontend-caucase-ssl:ca}
+  :> $CA.tmp
+  CRL=${frontend-caucase-ssl:crl}
+  :> $CRL.tmp
+{% for hash in frontend_caucase_url_hash_list %}
+{%   set data_dir = '${nginx-ssl-dir:ssl}/%s' % hash %}
+  echo "# {{ data_dir }}/ca.crt" >> $CA.tmp
+  cat "{{ data_dir }}/ca.crt" >> $CA.tmp
+  echo "# {{ data_dir }}/crl.pem" >> $CRL.tmp
+  cat "{{ data_dir }}/crl.pem" >> $CRL.tmp
+{% endfor %}
+  mv $CA.tmp $CA
+  mv $CRL.tmp $CRL
+  kill -HUP $(cat ${directory:run}/nginx.pid)
+{% endif %}
+
 # srv/nginx/ prefix  +  etc/ log/ ...
 [nginx-dir]
 recipe  = slapos.cookbook:mkdirectory
@@ -787,6 +844,9 @@ ssl     = ${nginx-ssl-dir:ssl}
 cert_file   = ${nginx-generate-certificate:cert_file}
 key_file    = ${nginx-generate-certificate:key_file}
 
+client_ca_file = ${frontend-caucase-ssl:ca}
+client_crl_file = ${frontend-caucase-ssl:crl}
+
 
 [nginx-symlinks]
 # (nginx wants <prefix>/logs to be there from start - else it issues alarm to the log)
@@ -801,6 +861,9 @@ depend  =
     ${nginx-symlinks:recipe}
     ${promise-nginx:recipe}
     ${logrotate-entry-nginx:recipe}
+{% for section in frontend_caucase_updater_section_list %}
+    {{ '${' ~  section ~ ':recipe}' }}
+{% endfor %}
 
 
 [promise-nginx]

software/gitlab/instance.cfg.in

@@ -41,8 +41,12 @@ configuration.icp_license  =
 recipe  = slapos.recipe.template:jinja2
 mode    = 0644
 output= $${buildout:directory}/$${:_buildout_section_name_}
+extensions = jinja2.ext.do
+import-list =
+    rawfile caucase ${caucase-jinja2-library:target}
 context =
     import os os
+    import hashlib hashlib
     import pwd pwd
 
     key bin_directory           buildout:bin-directory

software/gitlab/software.cfg

@@ -3,6 +3,7 @@
 extends =
     buildout.hash.cfg
     ../../stack/slapos.cfg
+    ../../stack/caucase/buildout.cfg
     ../../stack/nodejs.cfg
     ../../stack/monitor/buildout.cfg
     ../../component/libgit2/buildout.cfg
@@ -54,6 +55,7 @@ parts =
 
     slapos-cookbook
     eggs
+    caucase-eggs
 
     bash
     curl

software/gitlab/template/nginx-gitlab-http.conf.in

@@ -76,11 +76,12 @@ server {
   ## https://raymii.org/s/tutorials/Strong_SSL_Security_On_nginx.html & https://cipherli.st/
   ssl_certificate {{ nginx.cert_file }};
   ssl_certificate_key {{ nginx.key_file }};
-  {# we don't need - most root CA will be included by default
-  <% if @ssl_client_certificate %>
-  ssl_client_certificate <%= @ssl_client_certificate%>;
- 	<% end %>
-  #}
+
+  {%   if nginx.client_ca_file %}
+  ssl_client_certificate {{ nginx.client_ca_file }};
+  ssl_crl {{ nginx.client_crl_file }};
+  ssl_verify_client optional_no_ca;
+  {%   endif %}
 
   # GitLab needs backwards compatible ciphers to retain compatibility with Java IDEs
   # NOTE(slapos) ^^^ is not relevant for us - we are behind frontend and clients
@@ -110,6 +111,18 @@ server {
   set_real_ip_from {{ trusted_address }};
   {% endfor %}
 
+  ## SlapOS: For Real IP, instead of trusting the frontends through their IP addresses,
+  ## we expect the frontends to use a client certificate and we trust frontends only if
+  ## we can validate that certificate.
+  set $trusted_remote_addr $remote_addr;
+  {% if nginx.client_ca_file %}
+  set_real_ip_from 0.0.0.0/0;
+  set_real_ip_from ::/0;
+  if ($ssl_client_verify != SUCCESS) {
+    set $trusted_remote_addr $realip_remote_addr;
+  }
+  {% endif %}
+
   ## HSTS Config
   ## https://www.nginx.com/blog/http-strict-transport-security-hsts-and-nginx/
   {% if int(cfg("nginx_hsts_max_age")) > 0 -%}
@@ -163,7 +176,7 @@ server {
     {% if cfg_https %}
     proxy_set_header    X-Forwarded-Ssl     on;
     {% endif %}
-    proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
+    proxy_set_header    X-Forwarded-For     $trusted_remote_addr;
     proxy_set_header    X-Forwarded-Proto   {{ "https" if cfg_https else "http" }};
 
     proxy_pass http://gitlab-workhorse;
@@ -188,7 +201,7 @@ server {
     {% if cfg_https %}
     proxy_set_header    X-Forwarded-Ssl     on;
     {% endif %}
-    proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
+    proxy_set_header    X-Forwarded-For     $trusted_remote_addr;
     proxy_set_header    X-Forwarded-Proto   {{ "https" if cfg_https else "http" }};
 
     proxy_pass  http://gitlab-workhorse;

software/gitlab/test/test.py

@@ -26,12 +26,16 @@
 ##############################################################################
 
 import os
-import requests
 import functools
+import urllib.parse
+import time
+from typing import Optional, Tuple
+
 import bs4
-from urllib.parse import urljoin
+import requests
 
 from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
+from slapos.testing.caucase import CaucaseCertificate, CaucaseService
 
 
 setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
@@ -49,7 +53,13 @@ class TestGitlab(SlapOSInstanceTestCase):
 
   @classmethod
   def getInstanceParameterDict(cls):
-    return {"root-password": "admin1234"}
+    frontend_caucase = cls.getManagedResource("frontend_caucase", CaucaseService)
+    certificate = cls.getManagedResource("client_certificate", CaucaseCertificate)
+    certificate.request("shared frontend", frontend_caucase)
+    return {
+      "root-password": "admin1234",
+      "frontend-caucase-url-list": frontend_caucase.url,
+    }
 
   def setUp(self):
     self.backend_url = self.computer_partition.getConnectionParameterDict()[
@@ -61,10 +71,16 @@ class TestGitlab(SlapOSInstanceTestCase):
     self.assertTrue(resp.status_code in [requests.codes.ok, requests.codes.found])
 
   def test_rack_attack_sign_in_rate_limiting(self):
-    session = requests.session()
+    client_certificate = self.getManagedResource(
+      "client_certificate", CaucaseCertificate
+    )
+    session = requests.Session()
+    session.cert = (client_certificate.cert_file, client_certificate.key_file)
 
     # Load the login page to get a CSRF token.
-    response = session.get(urljoin(self.backend_url, "users/sign_in"), verify=False)
+    response = session.get(
+      urllib.parse.urljoin(self.backend_url, "users/sign_in"), verify=False
+    )
     self.assertEqual(response.status_code, 200)
 
     # Extract the CSRF token and param.
@@ -83,10 +99,66 @@ class TestGitlab(SlapOSInstanceTestCase):
     )
 
     for _ in range(10):
-      sign_in(headers={"X-Forwarded-For": "1.2.3.4"})
+      sign_in(headers={"X-Forwarded-For": "1.2.3.4"}).raise_for_status()
     # after 10 authentication failures, this client is rate limited
     self.assertEqual(sign_in(headers={"X-Forwarded-For": "1.2.3.4"}).status_code, 429)
     # but other clients are not
     self.assertNotEqual(
       sign_in(headers={"X-Forwarded-For": "5.6.7.8"}).status_code, 429
     )
+
+  def _get_client_ip_address_from_nginx_log(
+    self, cert: Optional[Tuple[str, str]]
+  ) -> str:
+    requests.get(
+      urllib.parse.urljoin(
+        self.backend_url,
+        f"/users/sign_in?request_id={self.id()}",
+      ),
+      verify=False,
+      cert=cert,
+      headers={"X-Forwarded-For": "1.2.3.4"},
+    ).raise_for_status()
+    nginx_log_file = (
+      self.computer_partition_root_path / "var" / "log" / "nginx" / "gitlab_access.log"
+    )
+    for _ in range(100):
+      last_log_line = nginx_log_file.read_text().splitlines()[-1]
+      if self.id() in last_log_line:
+        return last_log_line.split("-")[0].strip()
+      time.sleep(1)
+    raise RuntimeError(f"Could not find {self.id()} in {last_log_line=}")
+
+  def test_client_ip_in_nginx_log_with_certificate(self):
+    client_certificate = self.getManagedResource(
+      "client_certificate", CaucaseCertificate
+    )
+    self.assertEqual(
+      self._get_client_ip_address_from_nginx_log(
+        cert=(client_certificate.cert_file, client_certificate.key_file)
+      ),
+      "1.2.3.4",
+    )
+
+  def test_client_ip_in_nginx_log_without_certificate(self):
+    self.assertNotEqual(
+      self._get_client_ip_address_from_nginx_log(cert=None),
+      "1.2.3.4",
+    )
+
+  def test_client_ip_in_nginx_log_with_not_verified_certificate(self):
+    another_unrelated_caucase = self.getManagedResource(
+      "another_unrelated_caucase", CaucaseService
+    )
+    unknown_client_certificate = self.getManagedResource(
+      "unknown_client_certificate", CaucaseCertificate
+    )
+    unknown_client_certificate.request(
+      "unknown client certificate", another_unrelated_caucase
+    )
+    self.assertNotEqual(
+      self._get_client_ip_address_from_nginx_log(
+        cert=(unknown_client_certificate.cert_file, unknown_client_certificate.key_file)
+      ),
+      "1.2.3.4",
+    )