Merge branch '211579-gcp-firewall-rule-and-forwarding-rule-quota-gitlab-review-apps' into 'master'

Resolve "GCP firewall rule and forwarding rule quota -  gitlab-review-apps"

See merge request gitlab-org/gitlab!27872

.gitlab/ci/review.gitlab-ci.yml

@@ -42,6 +42,23 @@ review-cleanup:
   script:
     - ruby -rrubygems scripts/review_apps/automated_cleanup.rb
 
+review-gcp-cleanup:
+  extends:
+    - .review:rules:review-gcp-cleanup
+  stage: prepare
+  image: gcr.io/google.com/cloudsdktool/cloud-sdk:latest
+  allow_failure: true
+  environment:
+    name: review/auto-gcp-cleanup
+    action: stop
+  before_script:
+    - gcloud auth activate-service-account --key-file=$REVIEW_APPS_GCP_CREDENTIALS
+    - gcloud config set project $REVIEW_APPS_GCP_PROJECT
+    - apt-get install -y jq
+    - source scripts/review_apps/gcp_cleanup.sh
+  script:
+    - gcp_cleanup
+
 review-build-cng:
   extends:
     - .default-retry

.gitlab/ci/rules.gitlab-ci.yml

@@ -496,6 +496,13 @@
     - <<: *if-dot-com-gitlab-org-schedule
       when: on_success
 
+.review:rules:review-gcp-cleanup:
+  rules:
+    - <<: *if-dot-com-gitlab-org-merge-request
+      when: manual
+    - <<: *if-dot-com-gitlab-org-schedule
+      when: on_success
+
 .review:rules:danger:
   rules:
     - if: '$DANGER_GITLAB_API_TOKEN && $CI_MERGE_REQUEST_IID'

doc/development/testing_guide/review_apps.md

@@ -113,6 +113,10 @@ pipelines (and is manual in merge request) stops stale Review Apps after 5 days,
 deletes their environment after 6 days, and cleans up any dangling Helm releases
 and Kubernetes resources after 7 days.
 
+The `review-gcp-cleanup` job that automatically runs in scheduled pipelines
+(and is manual in merge request) removes any dangling GCP network resources
+that were not removed along with the Kubernetes resources.
+
 ## QA runs
 
 On every [pipeline][gitlab-pipeline] in the `qa` stage (which comes after the

scripts/review_apps/gcp_cleanup.sh

+#!/bin/bash
+
+source scripts/utils.sh
+
+# These scripts require the following environment variables:
+# - REVIEW_APPS_GCP_REGION - e.g `us-central1`
+# - KUBE_NAMESPACE - e.g `review-apps-ee`
+
+function delete_firewall_rules() {
+  if [[ ${#@} -eq 0 ]]; then
+    echoinfo "No firewall rules to be deleted" true
+    return
+  fi
+
+  echoinfo "Deleting firewall rules:" true
+  echo "${@}"
+
+  if [[ ${DRY_RUN} = 1 ]]; then
+    echo "[DRY RUN] gcloud compute firewall-rules delete -q" "${@}"
+  else
+    gcloud compute firewall-rules delete -q "${@}"
+  fi
+}
+
+function delete_forwarding_rules() {
+  if [[ ${#@} -eq 0 ]]; then
+    echoinfo "No forwarding rules to be deleted" true
+    return
+  fi
+
+  echoinfo "Deleting forwarding rules:" true
+  echo "${@}"
+
+  if [[ ${DRY_RUN} = 1 ]]; then
+    echo "[DRY RUN] gcloud compute forwarding-rules delete -q" "${@}" "--region ${REVIEW_APPS_GCP_REGION}"
+  else
+    gcloud compute forwarding-rules delete -q "${@}" --region "${REVIEW_APPS_GCP_REGION}"
+  fi
+}
+
+function delete_target_pools() {
+  if [[ ${#@} -eq 0 ]]; then
+    echoinfo "No target pools to be deleted" true
+    return
+  fi
+
+  echoinfo "Deleting target pools:" true
+  echo "${@}"
+
+  if [[ ${DRY_RUN} = 1 ]]; then
+    echo "[DRY RUN] gcloud compute target-pools delete -q" "${@}" "--region ${REVIEW_APPS_GCP_REGION}"
+  else
+    gcloud compute target-pools delete -q "${@}" --region "${REVIEW_APPS_GCP_REGION}"
+  fi
+}
+
+function delete_http_health_checks() {
+  if [[ ${#@} -eq 0 ]]; then
+    echoinfo "No http health checks to be deleted" true
+    return
+  fi
+
+  echoinfo "Deleting http health checks:" true
+  echo "${@}"
+
+  if [[ ${DRY_RUN} = 1 ]]; then
+    echo "[DRY RUN] gcloud compute http-health-checks delete -q" "${@}"
+  else
+    gcloud compute http-health-checks delete -q "${@}"
+  fi
+}
+
+function get_related_firewall_rules() {
+  local forwarding_rule=${1}
+
+  gcloud compute firewall-rules list --filter "name~${forwarding_rule}" --format "value(name)"
+}
+
+function get_service_name_in_forwarding_rule() {
+  local forwarding_rule=${1}
+
+  gcloud compute forwarding-rules describe "${forwarding_rule}" --region "${REVIEW_APPS_GCP_REGION}" --format "value(description)" | jq -r '.["kubernetes.io/service-name"]'
+}
+
+function forwarding_rule_k8s_service_exists() {
+  local namespace="${KUBE_NAMESPACE}"
+  local namespaced_service_name=$(get_service_name_in_forwarding_rule "$forwarding_rule")
+
+  if [[ ! $namespaced_service_name =~ ^"${namespace}" ]]; then
+    return 0 # this prevents `review-apps-ee` pipeline from deleting `review-apps-ce` resources and vice versa
+  fi
+
+  local service_name=$(echo "${namespaced_service_name}" | sed -e "s/${namespace}\///g")
+
+  kubectl get svc "${service_name}" -n "${namespace}" >/dev/null 2>&1
+  local status=$?
+
+  return $status
+}
+
+function gcp_cleanup() {
+  if [[ ! $(command -v kubectl) ]]; then
+    echoerr "kubectl executable not found"
+    return 1
+  fi
+
+  if [[ -z "${REVIEW_APPS_GCP_REGION}" ]]; then
+    echoerr "REVIEW_APPS_GCP_REGION is not set."
+    return 1
+  fi
+
+  if [[ -z "${KUBE_NAMESPACE}" ]]; then
+    echoerr "KUBE_NAMESPACE is not set."
+    return 1
+  fi
+
+  if [[ -n "${DRY_RUN}" ]]; then
+    echoinfo "Running in DRY_RUN"
+  fi
+
+  local target_pools_to_delete=()
+  local firewall_rules_to_delete=()
+  local forwarding_rules_to_delete=()
+  local http_health_checks_to_delete=()
+
+  for forwarding_rule in $(gcloud compute forwarding-rules list --filter="region:(${REVIEW_APPS_GCP_REGION})" --format "value(name)"); do
+    echoinfo "Inspecting forwarding rule ${forwarding_rule}" true
+
+    # We perform clean up when there is no more kubernetes service that require the resources.
+    # To identify the kubernetes service using the resources,
+    # we find the service name indicated in the forwarding rule description, e.g:
+    #
+    # $ gcloud compute forwarding-rules describe aff68b997da1211e984a042010af0019
+    # # ...
+    # description: '{"kubernetes.io/service-name":"review-apps-ee/review-winh-eslin-809vqz-nginx-ingress-controller"}'
+    # # ...
+    if forwarding_rule_k8s_service_exists "${forwarding_rule}"; then
+      echoinfo "Skip clean up for ${forwarding_rule}"
+    else
+      echoinfo "Queuing forwarding rule, firewall rule, target pool and health check for ${forwarding_rule} to be cleaned up"
+
+      firewall_rules_to_delete+=($(get_related_firewall_rules "${forwarding_rule}"))
+      forwarding_rules_to_delete+=(${forwarding_rule})
+      target_pools_to_delete+=(${forwarding_rule})
+      http_health_checks_to_delete+=(${forwarding_rule})
+    fi
+  done
+
+  delete_firewall_rules "${firewall_rules_to_delete[@]}"
+  delete_forwarding_rules "${forwarding_rules_to_delete[@]}"
+  delete_target_pools "${target_pools_to_delete[@]}"
+  delete_http_health_checks "${http_health_checks_to_delete[@]}"
+}