Commit 0d276f9d authored by Rémy Coutable's avatar Rémy Coutable

Merge branch 'caalberts-rebuild-review-app-cluster' into 'master'

Re-enable review app using a new cluster

Closes #222593

See merge request gitlab-org/gitlab!34996
parents 281e996f 247f1e3c
......@@ -16,25 +16,24 @@ review-cleanup:
- ruby -rrubygems scripts/review_apps/automated_cleanup.rb
- gcp_cleanup
# Temporarily disabling review apps
#review-build-cng:
# extends:
# - .default-retry
# - .review:rules:review-build-cng
# image: ruby:2.6-alpine
# stage: review-prepare
# before_script:
# - source scripts/utils.sh
# - install_api_client_dependencies_with_apk
# - install_gitlab_gem
# needs:
# - job: compile-production-assets
# artifacts: false
# script:
# - BUILD_TRIGGER_TOKEN=$REVIEW_APPS_BUILD_TRIGGER_TOKEN ./scripts/trigger-build cng
# # When the job is manual, review-deploy is also manual and we don't want people
# # to have to manually start the jobs in sequence, so we do it for them.
# - '[ -z $CI_JOB_MANUAL ] || play_job "review-deploy"'
review-build-cng:
extends:
- .default-retry
- .review:rules:review-build-cng
image: ruby:2.6-alpine
stage: review-prepare
before_script:
- source scripts/utils.sh
- install_api_client_dependencies_with_apk
- install_gitlab_gem
needs:
- job: compile-production-assets
artifacts: false
script:
- BUILD_TRIGGER_TOKEN=$REVIEW_APPS_BUILD_TRIGGER_TOKEN ./scripts/trigger-build cng
# When the job is manual, review-deploy is also manual and we don't want people
# to have to manually start the jobs in sequence, so we do it for them.
- '[ -z $CI_JOB_MANUAL ] || play_job "review-deploy"'
.review-workflow-base:
extends:
......@@ -42,6 +41,7 @@ review-cleanup:
image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-helm3-kubectl1.14
variables:
HOST_SUFFIX: "${CI_ENVIRONMENT_SLUG}"
REVIEW_APPS_DOMAIN: "temp.gitlab-review.app" # FIXME: using temporary domain
DOMAIN: "-${CI_ENVIRONMENT_SLUG}.${REVIEW_APPS_DOMAIN}"
GITLAB_HELM_CHART_REF: "master"
environment:
......@@ -50,37 +50,37 @@ review-cleanup:
on_stop: review-stop
auto_stop_in: 48 hours
# Temporarily disabling review apps
#review-deploy:
# extends:
# - .review-workflow-base
# - .review:rules:mr-and-schedule-auto-if-frontend-manual-otherwise
# stage: review
# dependencies: []
# resource_group: "review/${CI_COMMIT_REF_NAME}"
# before_script:
# - export GITLAB_SHELL_VERSION=$(<GITLAB_SHELL_VERSION)
# - export GITALY_VERSION=$(<GITALY_SERVER_VERSION)
# - export GITLAB_WORKHORSE_VERSION=$(<GITLAB_WORKHORSE_VERSION)
# - echo "${CI_ENVIRONMENT_URL}" > environment_url.txt
# - source ./scripts/utils.sh
# - install_api_client_dependencies_with_apk
# - source scripts/review_apps/review-apps.sh
# script:
# - check_kube_domain
# - ensure_namespace
# - install_external_dns
# - download_chart
# - date
# - deploy || (display_deployment_debug && exit 1)
# # When the job is manual, review-qa-smoke is also manual and we don't want people
# # to have to manually start the jobs in sequence, so we do it for them.
# - '[ -z $CI_JOB_MANUAL ] || play_job "review-qa-smoke"'
# - '[ -z $CI_JOB_MANUAL ] || play_job "review-performance"'
# artifacts:
# paths: [environment_url.txt]
# expire_in: 2 days
# when: always
review-deploy:
extends:
- .review-workflow-base
- .review:rules:mr-and-schedule-auto-if-frontend-manual-otherwise
stage: review
dependencies: []
resource_group: "review/${CI_COMMIT_REF_NAME}"
before_script:
- export GITLAB_SHELL_VERSION=$(<GITLAB_SHELL_VERSION)
- export GITALY_VERSION=$(<GITALY_SERVER_VERSION)
- export GITLAB_WORKHORSE_VERSION=$(<GITLAB_WORKHORSE_VERSION)
- echo "${CI_ENVIRONMENT_URL}" > environment_url.txt
- source ./scripts/utils.sh
- install_api_client_dependencies_with_apk
- source scripts/review_apps/review-apps.sh
script:
- check_kube_domain
- ensure_namespace
- install_external_dns
- download_chart
- date
- deploy || (display_deployment_debug && exit 1)
- disable_sign_ups
# When the job is manual, review-qa-smoke is also manual and we don't want people
# to have to manually start the jobs in sequence, so we do it for them.
- '[ -z $CI_JOB_MANUAL ] || play_job "review-qa-smoke"'
- '[ -z $CI_JOB_MANUAL ] || play_job "review-performance"'
artifacts:
paths: [environment_url.txt]
expire_in: 2 days
when: always
.review-stop-base:
extends: .review-workflow-base
......@@ -113,110 +113,110 @@ review-stop:
script:
- delete_release
# Temporarily disabling review apps
#.review-qa-base:
# extends:
# - .default-retry
# - .use-docker-in-docker
# image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-qa-alpine-ruby-2.6
# stage: qa
# # This is needed so that manual jobs with needs don't block the pipeline.
# # See https://gitlab.com/gitlab-org/gitlab/-/issues/199979.
# dependencies: ["review-deploy"]
# variables:
# QA_ARTIFACTS_DIR: "${CI_PROJECT_DIR}/qa"
# QA_CAN_TEST_GIT_PROTOCOL_V2: "false"
# QA_DEBUG: "true"
# GITLAB_USERNAME: "root"
# GITLAB_PASSWORD: "${REVIEW_APPS_ROOT_PASSWORD}"
# GITLAB_ADMIN_USERNAME: "root"
# GITLAB_ADMIN_PASSWORD: "${REVIEW_APPS_ROOT_PASSWORD}"
# GITHUB_ACCESS_TOKEN: "${REVIEW_APPS_QA_GITHUB_ACCESS_TOKEN}"
# EE_LICENSE: "${REVIEW_APPS_EE_LICENSE}"
# before_script:
# - export QA_IMAGE="${CI_REGISTRY}/${CI_PROJECT_PATH}/gitlab-ee-qa:${CI_COMMIT_REF_SLUG}"
# - export CI_ENVIRONMENT_URL="$(cat environment_url.txt)"
# - echo "${CI_ENVIRONMENT_URL}"
# - echo "${QA_IMAGE}"
# - source scripts/utils.sh
# - install_api_client_dependencies_with_apk
# - gem install gitlab-qa --no-document ${GITLAB_QA_VERSION:+ --version ${GITLAB_QA_VERSION}}
# artifacts:
# paths:
# - ./qa/gitlab-qa-run-*
# expire_in: 7 days
# when: always
#
#review-qa-smoke:
# extends:
# - .review-qa-base
# - .review:rules:review-qa-smoke
# script:
# - gitlab-qa Test::Instance::Smoke "${QA_IMAGE}" "${CI_ENVIRONMENT_URL}"
#
#review-qa-all:
# extends:
# - .review-qa-base
# - .review:rules:mr-only-manual
# parallel: 5
# script:
# - export KNAPSACK_REPORT_PATH=knapsack/master_report.json
# - export KNAPSACK_TEST_FILE_PATTERN=qa/specs/features/**/*_spec.rb
# - gitlab-qa Test::Instance::Any "${QA_IMAGE}" "${CI_ENVIRONMENT_URL}" -- --format RspecJunitFormatter --out tmp/rspec-${CI_JOB_ID}.xml --format html --out tmp/rspec.htm --color --format documentation
#
#review-performance:
# extends:
# - .default-retry
# - .review:rules:mr-and-schedule-auto-if-frontend-manual-otherwise
# image:
# name: sitespeedio/sitespeed.io:6.3.1
# entrypoint: [""]
# stage: qa
# # This is needed so that manual jobs with needs don't block the pipeline.
# # See https://gitlab.com/gitlab-org/gitlab/-/issues/199979.
# dependencies: ["review-deploy"]
# before_script:
# - export CI_ENVIRONMENT_URL="$(cat environment_url.txt)"
# - echo "${CI_ENVIRONMENT_URL}"
# - mkdir -p gitlab-exporter
# - wget -O ./gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/master/index.js
# - mkdir -p sitespeed-results
# script:
# - /start.sh --plugins.add ./gitlab-exporter --outputFolder sitespeed-results "${CI_ENVIRONMENT_URL}"
# after_script:
# - mv sitespeed-results/data/performance.json performance.json
# artifacts:
# paths:
# - sitespeed-results/
# reports:
# performance: performance.json
# expire_in: 31d
#
#parallel-spec-reports:
# extends:
# - .review:rules:mr-only-manual
# image: ruby:2.6-alpine
# stage: post-qa
# dependencies: ["review-qa-all"]
# variables:
# NEW_PARALLEL_SPECS_REPORT: qa/report-new.html
# BASE_ARTIFACT_URL: "${CI_PROJECT_URL}/-/jobs/${CI_JOB_ID}/artifacts/file/qa/"
# script:
# - apk add --update build-base libxml2-dev libxslt-dev && rm -rf /var/cache/apk/*
# - gem install nokogiri --no-document
# - cd qa/gitlab-qa-run-*/gitlab-*
# - ARTIFACT_DIRS=$(pwd |rev| awk -F / '{print $1,$2}' | rev | sed s_\ _/_)
# - cd -
# - '[[ -f $NEW_PARALLEL_SPECS_REPORT ]] || echo "{}" > ${NEW_PARALLEL_SPECS_REPORT}'
# - scripts/merge-html-reports ${NEW_PARALLEL_SPECS_REPORT} ${BASE_ARTIFACT_URL}${ARTIFACT_DIRS} qa/gitlab-qa-run-*/**/rspec.htm
# artifacts:
# when: always
# paths:
# - qa/report-new.html
# - qa/gitlab-qa-run-*
# reports:
# junit: qa/gitlab-qa-run-*/**/rspec-*.xml
# expire_in: 31d
.review-qa-base:
extends:
- .default-retry
- .use-docker-in-docker
image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-qa-alpine-ruby-2.6
stage: qa
# This is needed so that manual jobs with needs don't block the pipeline.
# See https://gitlab.com/gitlab-org/gitlab/-/issues/199979.
dependencies: ["review-deploy"]
variables:
QA_ARTIFACTS_DIR: "${CI_PROJECT_DIR}/qa"
QA_CAN_TEST_GIT_PROTOCOL_V2: "false"
QA_DEBUG: "true"
GITLAB_USERNAME: "root"
GITLAB_PASSWORD: "${REVIEW_APPS_ROOT_PASSWORD}"
GITLAB_ADMIN_USERNAME: "root"
GITLAB_ADMIN_PASSWORD: "${REVIEW_APPS_ROOT_PASSWORD}"
GITHUB_ACCESS_TOKEN: "${REVIEW_APPS_QA_GITHUB_ACCESS_TOKEN}"
EE_LICENSE: "${REVIEW_APPS_EE_LICENSE}"
SIGNUP_DISABLED: "true"
before_script:
- export QA_IMAGE="${CI_REGISTRY}/${CI_PROJECT_PATH}/gitlab-ee-qa:${CI_COMMIT_REF_SLUG}"
- export CI_ENVIRONMENT_URL="$(cat environment_url.txt)"
- echo "${CI_ENVIRONMENT_URL}"
- echo "${QA_IMAGE}"
- source scripts/utils.sh
- install_api_client_dependencies_with_apk
- gem install gitlab-qa --no-document ${GITLAB_QA_VERSION:+ --version ${GITLAB_QA_VERSION}}
artifacts:
paths:
- ./qa/gitlab-qa-run-*
expire_in: 7 days
when: always
review-qa-smoke:
extends:
- .review-qa-base
- .review:rules:review-qa-smoke
script:
- gitlab-qa Test::Instance::Smoke "${QA_IMAGE}" "${CI_ENVIRONMENT_URL}"
review-qa-all:
extends:
- .review-qa-base
- .review:rules:mr-only-manual
parallel: 5
script:
- export KNAPSACK_REPORT_PATH=knapsack/master_report.json
- export KNAPSACK_TEST_FILE_PATTERN=qa/specs/features/**/*_spec.rb
- gitlab-qa Test::Instance::Any "${QA_IMAGE}" "${CI_ENVIRONMENT_URL}" -- --format RspecJunitFormatter --out tmp/rspec-${CI_JOB_ID}.xml --format html --out tmp/rspec.htm --color --format documentation
review-performance:
extends:
- .default-retry
- .review:rules:mr-and-schedule-auto-if-frontend-manual-otherwise
image:
name: sitespeedio/sitespeed.io:6.3.1
entrypoint: [""]
stage: qa
# This is needed so that manual jobs with needs don't block the pipeline.
# See https://gitlab.com/gitlab-org/gitlab/-/issues/199979.
dependencies: ["review-deploy"]
before_script:
- export CI_ENVIRONMENT_URL="$(cat environment_url.txt)"
- echo "${CI_ENVIRONMENT_URL}"
- mkdir -p gitlab-exporter
- wget -O ./gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/master/index.js
- mkdir -p sitespeed-results
script:
- /start.sh --plugins.add ./gitlab-exporter --outputFolder sitespeed-results "${CI_ENVIRONMENT_URL}"
after_script:
- mv sitespeed-results/data/performance.json performance.json
artifacts:
paths:
- sitespeed-results/
reports:
performance: performance.json
expire_in: 31d
parallel-spec-reports:
extends:
- .review:rules:mr-only-manual
image: ruby:2.6-alpine
stage: post-qa
dependencies: ["review-qa-all"]
variables:
NEW_PARALLEL_SPECS_REPORT: qa/report-new.html
BASE_ARTIFACT_URL: "${CI_PROJECT_URL}/-/jobs/${CI_JOB_ID}/artifacts/file/qa/"
script:
- apk add --update build-base libxml2-dev libxslt-dev && rm -rf /var/cache/apk/*
- gem install nokogiri --no-document
- cd qa/gitlab-qa-run-*/gitlab-*
- ARTIFACT_DIRS=$(pwd |rev| awk -F / '{print $1,$2}' | rev | sed s_\ _/_)
- cd -
- '[[ -f $NEW_PARALLEL_SPECS_REPORT ]] || echo "{}" > ${NEW_PARALLEL_SPECS_REPORT}'
- scripts/merge-html-reports ${NEW_PARALLEL_SPECS_REPORT} ${BASE_ARTIFACT_URL}${ARTIFACT_DIRS} qa/gitlab-qa-run-*/**/rspec.htm
artifacts:
when: always
paths:
- qa/report-new.html
- qa/gitlab-qa-run-*
reports:
junit: qa/gitlab-qa-run-*/**/rspec-*.xml
expire_in: 31d
danger-review:
extends:
......
......@@ -30,7 +30,7 @@ subgraph "2. gitlab `review-prepare` stage"
end
subgraph "3. gitlab `review` stage"
C["review-deploy<br><br>Helm deploys the Review App using the Cloud<br/>Native images built by the CNG-mirror pipeline.<br><br>Cloud Native images are deployed to the `review-apps-ce` or `review-apps-ee`<br>Kubernetes (GKE) cluster, in the GCP `gitlab-review-apps` project."]
C["review-deploy<br><br>Helm deploys the Review App using the Cloud<br/>Native images built by the CNG-mirror pipeline.<br><br>Cloud Native images are deployed to the `review-apps`<br>Kubernetes (GKE) cluster, in the GCP `gitlab-review-apps` project."]
end
subgraph "4. gitlab `qa` stage"
......@@ -62,7 +62,7 @@ subgraph "CNG-mirror pipeline"
job, which runs only for tags, and triggers itself a [`CNG`](https://gitlab.com/gitlab-org/build/CNG) pipeline.
1. Once the `test` stage is done, the [`review-deploy`](https://gitlab.com/gitlab-org/gitlab/-/jobs/467724810) job
deploys the Review App using [the official GitLab Helm chart](https://gitlab.com/gitlab-org/charts/gitlab/) to
the [`review-apps-ce`](https://console.cloud.google.com/kubernetes/clusters/details/us-central1-a/review-apps-ce?project=gitlab-review-apps) / [`review-apps-ee`](https://console.cloud.google.com/kubernetes/clusters/details/us-central1-b/review-apps-ee?project=gitlab-review-apps)
the [`review-apps`](https://console.cloud.google.com/kubernetes/clusters/details/us-central1-b/review-apps?project=gitlab-review-apps)
Kubernetes cluster on GCP.
- The actual scripts used to deploy the Review App can be found at
[`scripts/review_apps/review-apps.sh`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/scripts/review_apps/review-apps.sh).
......@@ -136,11 +136,10 @@ browser performance testing using a
### Node pools
The `review-apps-ee` and `review-apps-ce` clusters are currently set up with
The `review-apps` cluster is currently set up with
the following node pools:
- `review-apps-ee` of pre-emptible `e2-highcpu-16` (16 vCPU, 16 GB memory) nodes with autoscaling
- `review-apps-ce` of pre-emptible `n1-standard-8` (8 vCPU, 16 GB memory) nodes with autoscaling
- `e2-highcpu-16` (16 vCPU, 16 GB memory) pre-emptible nodes with autoscaling
### Helm
......@@ -189,9 +188,7 @@ secure note named `gitlab-{ce,ee} Review App's root password`.
1. Click on the `KUBECTL` dropdown, then `Exec` -> `task-runner`.
1. Replace `-c task-runner -- ls` with `-it -- gitlab-rails console` from the
default command or
- Run `kubectl exec --namespace review-apps-ce review-qa-raise-e-12chm0-task-runner-d5455cc8-2lsvz -it -- gitlab-rails console` and
- Replace `review-apps-ce` with `review-apps-ee` if the Review App
is running EE, and
- Run `kubectl exec --namespace review-apps review-qa-raise-e-12chm0-task-runner-d5455cc8-2lsvz -it -- gitlab-rails console` and
- Replace `review-qa-raise-e-12chm0-task-runner-d5455cc8-2lsvz`
with your Pod's name.
......
......@@ -40,7 +40,7 @@ class AutomatedCleanup
end
def review_apps_namespace
self.class.ee? ? 'review-apps-ee' : 'review-apps-ce'
'review-apps'
end
def helm
......
......@@ -7,7 +7,7 @@ global:
external-dns.alpha.kubernetes.io/ttl: 10
configureCertmanager: false
tls:
secretName: tls-cert
secretName: review-apps-tls
initialRootPassword:
secret: shared-gitlab-initial-root-password
certmanager:
......@@ -61,11 +61,11 @@ gitlab:
task-runner:
resources:
requests:
cpu: 50m
memory: 350M
cpu: 300m
memory: 800M
limits:
cpu: 100m
memory: 700M
cpu: 450m
memory: 1200M
webservice:
resources:
requests:
......
......@@ -11,7 +11,7 @@ function setup_gcp_dependencies() {
# These scripts require the following environment variables:
# - REVIEW_APPS_GCP_REGION - e.g `us-central1`
# - KUBE_NAMESPACE - e.g `review-apps-ee`
# - KUBE_NAMESPACE - e.g `review-apps`
function delete_firewall_rules() {
if [[ ${#@} -eq 0 ]]; then
......
......@@ -66,7 +66,7 @@ function kubectl_cleanup_release() {
local release="${2}"
echoinfo "Deleting all K8s resources matching '${release}'..." true
kubectl --namespace "${namespace}" get ingress,svc,pdb,hpa,deploy,statefulset,job,pod,secret,configmap,pvc,secret,clusterrole,clusterrolebinding,role,rolebinding,sa,crd 2>&1 \
kubectl --namespace "${namespace}" get ingress,svc,pdb,hpa,deploy,statefulset,job,pod,secret,configmap,pvc,clusterrole,clusterrolebinding,role,rolebinding,sa,crd 2>&1 \
| grep "${release}" \
| awk '{print $1}' \
| xargs kubectl --namespace "${namespace}" delete \
......@@ -126,6 +126,38 @@ function get_pod() {
echo "${pod_name}"
}
function run_task() {
local namespace="${KUBE_NAMESPACE}"
local ruby_cmd="${1}"
local task_runner_pod=$(get_pod "task-runner")
kubectl exec -it --namespace "${namespace}" "${task_runner_pod}" -- gitlab-rails runner "${ruby_cmd}"
}
function disable_sign_ups() {
if [ -z ${REVIEW_APPS_ROOT_TOKEN+x} ]; then
echoerr "In order to protect Review Apps, REVIEW_APPS_ROOT_TOKEN variable must be set"
false
else
true
fi
# Create the root token
local ruby_cmd="token = User.find_by_username('root').personal_access_tokens.create(scopes: [:api], name: 'Token to disable sign-ups'); token.set_token('${REVIEW_APPS_ROOT_TOKEN}'); begin; token.save!; rescue(ActiveRecord::RecordNotUnique); end"
run_task "${ruby_cmd}"
# Disable sign-ups
curl --silent --show-error --request PUT --header "PRIVATE-TOKEN: ${REVIEW_APPS_ROOT_TOKEN}" "${CI_ENVIRONMENT_URL}/api/v4/application/settings?signup_enabled=false"
local signup_enabled=$(curl --silent --show-error --request GET --header "PRIVATE-TOKEN: ${REVIEW_APPS_ROOT_TOKEN}" "${CI_ENVIRONMENT_URL}/api/v4/application/settings" | jq ".signup_enabled")
if [[ "${signup_enabled}" == "false" ]]; then
echoinfo "Sign-ups have been disabled successfully."
else
echoerr "Sign-ups should be disabled but are still enabled!"
false
fi
}
function check_kube_domain() {
echoinfo "Checking that Kube domain exists..." true
......@@ -181,6 +213,32 @@ function install_external_dns() {
fi
}
# This script is used to install cert-manager in the cluster
# The installation steps are documented in
# https://gitlab.com/gitlab-org/quality/team-tasks/snippets/1990286
function install_certmanager() {
local namespace="${KUBE_NAMESPACE}"
local release="cert-manager-review-app-helm3"
echoinfo "Installing cert-manager..." true
if ! deploy_exists "${namespace}" "${release}" || previous_deploy_failed "${namespace}" "${release}" ; then
kubectl apply \
-f https://raw.githubusercontent.com/jetstack/cert-manager/release-0.10/deploy/manifests/00-crds.yaml
echoinfo "Installing cert-manager Helm chart"
helm repo add jetstack https://charts.jetstack.io
helm repo update
helm install "${release}" jetstack/cert-manager \
--namespace "${namespace}" \
--version v0.15.1 \
--set installCRDS=true
else
echoinfo "The cert-manager Helm chart is already successfully deployed."
fi
}
function create_application_secret() {
local namespace="${KUBE_NAMESPACE}"
local release="${CI_ENVIRONMENT_SLUG}"
......
......@@ -3,7 +3,7 @@
require 'fast_spec_helper'
RSpec.describe Quality::Helm3Client do
let(:namespace) { 'review-apps-ee' }
let(:namespace) { 'review-apps' }
let(:release_name) { 'my-release' }
let(:raw_helm_list_page1) do
<<~OUTPUT
......
......@@ -3,7 +3,7 @@
require 'fast_spec_helper'
RSpec.describe Quality::KubernetesClient do
let(:namespace) { 'review-apps-ee' }
let(:namespace) { 'review-apps' }
let(:release_name) { 'my-release' }
let(:pod_for_release) { "pod-my-release-abcd" }
let(:raw_resource_names_str) { "NAME\nfoo\n#{pod_for_release}\nbar" }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment