Merge branch 'alberts-review-app-dedicated-namespace' into 'master'

Deploy review app into dedicated namespace

See merge request gitlab-org/gitlab!63005

.gitlab/ci/review.gitlab-ci.yml

@@ -66,7 +66,7 @@ review-deploy:
     - *base-before_script
   script:
     - check_kube_domain
-    - ensure_namespace
+    - "ensure_namespace ${KUBE_NAMESPACE}"
     - install_external_dns
     - download_chart
     - date

.gitlab/ci/rules.gitlab-ci.yml

@@ -102,6 +102,7 @@
   - ".gitlab/ci/build-images.gitlab-ci.yml"
   - ".gitlab/ci/review.gitlab-ci.yml"
   - "scripts/review_apps/base-config.yaml"
+  - "scripts/review_apps/review-apps.sh"
   - "scripts/trigger-build"
 
 .ci-qa-patterns: &ci-qa-patterns

doc/development/testing_guide/review_apps.md

@@ -81,6 +81,8 @@ subgraph "CNG-mirror pipeline"
    - Since we're using [the official GitLab Helm chart](https://gitlab.com/gitlab-org/charts/gitlab/), this means
      you get a dedicated environment for your branch that's very close to what
      it would look in production.
+   - Each review app is deployed to its own Kubernetes namespace. The namespace is based on the Review App slug that is
+     unique to each branch.
 1. Once the [`review-deploy`](https://gitlab.com/gitlab-org/gitlab/-/jobs/467724810) job succeeds, you should be able to
    use your Review App thanks to the direct link to it from the MR widget. To log
    into the Review App, see "Log into my Review App?" below.
@@ -203,7 +205,7 @@ the GitLab handbook information for the [shared 1Password account](https://about
 1. Click on the `KUBECTL` dropdown, then `Exec` -> `task-runner`.
 1. Replace `-c task-runner -- ls` with `-it -- gitlab-rails console` from the
    default command or
-   - Run `kubectl exec --namespace review-apps review-qa-raise-e-12chm0-task-runner-d5455cc8-2lsvz -it -- gitlab-rails console` and
+   - Run `kubectl exec --namespace review-qa-raise-e-12chm0 review-qa-raise-e-12chm0-task-runner-d5455cc8-2lsvz -it -- gitlab-rails console` and
      - Replace `review-qa-raise-e-12chm0-task-runner-d5455cc8-2lsvz`
        with your Pod's name.
 
@@ -221,7 +223,7 @@ the GitLab handbook information for the [shared 1Password account](https://about
 ## Diagnosing unhealthy Review App releases
 
 If [Review App Stability](https://app.periscopedata.com/app/gitlab/496118/Engineering-Productivity-Sandbox?widget=6690556&udv=785399)
-dips this may be a signal that the `review-apps-ce/ee` cluster is unhealthy.
+dips this may be a signal that the `review-apps` cluster is unhealthy.
 Leading indicators may be health check failures leading to restarts or majority failure for Review App deployments.
 
 The [Review Apps Overview dashboard](https://console.cloud.google.com/monitoring/classic/dashboards/6798952013815386466?project=gitlab-review-apps&timeDomain=1d)

scripts/review_apps/base-config.yaml

@@ -5,9 +5,9 @@ global:
   ingress:
     annotations:
       external-dns.alpha.kubernetes.io/ttl: 10
+      cert-manager.io/cluster-issuer: review-apps-route53-dns01-wildcard-cluster-issuer
+      kubernetes.io/tls-acme: true
     configureCertmanager: false
-    tls:
-      secretName: review-apps-tls
   initialRootPassword:
     secret: shared-gitlab-initial-root-password
 certmanager:

scripts/review_apps/review-apps.sh

@@ -40,7 +40,7 @@ function previous_deploy_failed() {
 }
 
 function delete_release() {
-  local namespace="${KUBE_NAMESPACE}"
+  local namespace="${CI_ENVIRONMENT_SLUG}"
   local release="${CI_ENVIRONMENT_SLUG}"
 
   if [ -z "${release}" ]; then
@@ -48,39 +48,11 @@ function delete_release() {
     return
   fi
 
-  # Check if helm release exists before attempting to delete
-  # There may be situation where k8s resources exist, but helm release does not,
-  # for example, following a failed helm install.
-  # In such cases, we still want to continue to clean up k8s resources.
-  if deploy_exists "${namespace}" "${release}"; then
-    helm_delete_release "${namespace}" "${release}"
-  fi
-  kubectl_cleanup_release "${namespace}" "${release}"
-}
-
-function helm_delete_release() {
-  local namespace="${1}"
-  local release="${2}"
-
-  echoinfo "Deleting Helm release '${release}'..." true
-
-  helm uninstall --namespace "${namespace}" "${release}"
-}
-
-function kubectl_cleanup_release() {
-  local namespace="${1}"
-  local release="${2}"
-
-  echoinfo "Deleting all K8s resources matching '${release}'..." true
-  kubectl --namespace "${namespace}" get ingress,svc,pdb,hpa,deploy,statefulset,job,pod,secret,configmap,pvc,clusterrole,clusterrolebinding,role,rolebinding,sa,crd 2>&1 \
-    | grep "${release}" \
-    | awk '{print $1}' \
-    | xargs kubectl --namespace "${namespace}" delete --ignore-not-found \
-    || true
+  delete_k8s_release_namespace
 }
 
 function delete_failed_release() {
-  local namespace="${KUBE_NAMESPACE}"
+  local namespace="${CI_ENVIRONMENT_SLUG}"
   local release="${CI_ENVIRONMENT_SLUG}"
 
   if [ -z "${release}" ]; then
@@ -93,7 +65,7 @@ function delete_failed_release() {
   else
     # Cleanup and previous installs, as FAILED and PENDING_UPGRADE will cause errors with `upgrade`
     if previous_deploy_failed "${namespace}" "${release}" ; then
-      echoinfo "Review App deployment in bad state, cleaning up ${release}"
+      echoinfo "Review App deployment in bad state, cleaning up namespace ${release}"
       delete_release
     else
       echoinfo "Review App deployment in good state"
@@ -101,8 +73,14 @@ function delete_failed_release() {
   fi
 }
 
+function delete_k8s_release_namespace() {
+  local namespace="${CI_ENVIRONMENT_SLUG}"
+
+  kubectl delete namespace "${namespace}" --wait
+}
+
 function get_pod() {
-  local namespace="${KUBE_NAMESPACE}"
+  local namespace="${CI_ENVIRONMENT_SLUG}"
   local release="${CI_ENVIRONMENT_SLUG}"
   local app_name="${1}"
   local status="${2-Running}"
@@ -133,7 +111,7 @@ function get_pod() {
 }
 
 function run_task() {
-  local namespace="${KUBE_NAMESPACE}"
+  local namespace="${CI_ENVIRONMENT_SLUG}"
   local ruby_cmd="${1}"
   local task_runner_pod=$(get_pod "task-runner")
 
@@ -177,7 +155,7 @@ function check_kube_domain() {
 }
 
 function ensure_namespace() {
-  local namespace="${KUBE_NAMESPACE}"
+  local namespace="${1}"
 
   echoinfo "Ensuring the ${namespace} namespace exists..." true
 
@@ -245,7 +223,7 @@ function install_certmanager() {
 }
 
 function create_application_secret() {
-  local namespace="${KUBE_NAMESPACE}"
+  local namespace="${CI_ENVIRONMENT_SLUG}"
   local release="${CI_ENVIRONMENT_SLUG}"
   local initial_root_password_shared_secret
   local gitlab_license_shared_secret
@@ -306,7 +284,7 @@ function parse_gitaly_image_tag() {
 }
 
 function deploy() {
-  local namespace="${KUBE_NAMESPACE}"
+  local namespace="${CI_ENVIRONMENT_SLUG}"
   local release="${CI_ENVIRONMENT_SLUG}"
   local base_config_file_ref="${CI_DEFAULT_BRANCH}"
   if [[ "$(base_config_changed)" == "true" ]]; then base_config_file_ref="${CI_COMMIT_SHA}"; fi
@@ -324,11 +302,14 @@ function deploy() {
   gitlab_shell_image_repository="${IMAGE_REPOSITORY}/gitlab-shell"
   gitlab_workhorse_image_repository="${IMAGE_REPOSITORY}/gitlab-workhorse-ee"
 
+  ensure_namespace "${namespace}"
+
   create_application_secret
 
 HELM_CMD=$(cat << EOF
   helm upgrade \
     --namespace="${namespace}" \
+    --create-namespace \
     --install \
     --wait \
     --timeout "${HELM_INSTALL_TIMEOUT:-20m}" \
@@ -339,6 +320,9 @@ HELM_CMD=$(cat << EOF
     --set releaseOverride="${release}" \
     --set global.hosts.hostSuffix="${HOST_SUFFIX}" \
     --set global.hosts.domain="${REVIEW_APPS_DOMAIN}" \
+    --set gitlab.webservice.ingress.tls.secretName="${release}-gitlab-tls" \
+    --set registry.ingress.tls.secretName="${release}-registry-tls" \
+    --set minio.ingress.tls.secretName="${release}-minio-tls" \
     --set gitlab.migrations.image.repository="${gitlab_migrations_image_repository}" \
     --set gitlab.migrations.image.tag="${CI_COMMIT_REF_SLUG}" \
     --set gitlab.gitaly.image.repository="${gitlab_gitaly_image_repository}" \
@@ -382,7 +366,7 @@ EOF
 }
 
 function display_deployment_debug() {
-  local namespace="${KUBE_NAMESPACE}"
+  local namespace="${CI_ENVIRONMENT_SLUG}"
   local release="${CI_ENVIRONMENT_SLUG}"
 
   # Get all pods for this release