Merge branch 'ee-49570-add-review_qa-job' into 'master'

[EE] Add an automatic job to run QA smoke tests against Review Apps

Closes gitlab-ce#49570

See merge request gitlab-org/gitlab-ee!8499

.gitlab-ci.yml

@@ -576,6 +576,50 @@ setup-test-env:
       - config/secrets.yml
       - vendor/gitaly-ruby
 
+# GitLab Review apps
+.review-base: &review-base
+  <<: *dedicated-no-docs-no-db-pull-cache-job
+  image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base
+  stage: test
+  cache: {}
+  dependencies: []
+  environment: &review-environment
+    name: review/${CI_COMMIT_REF_NAME}
+    url: https://gitlab-${CI_ENVIRONMENT_SLUG}.${REVIEW_APPS_DOMAIN}
+  only:
+    refs:
+      - branches@gitlab-org/gitlab-ce
+      - branches@gitlab-org/gitlab-ee
+    kubernetes: active
+  except:
+    refs:
+      - master
+      - /(^docs[\/-].*|.*-docs$)/
+  before_script: []
+
+.review-docker: &review-docker
+  <<: *review-base
+  image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-qa-alpine
+  services:
+    - docker:stable-dind
+  tags:
+    - gitlab-org
+    - docker
+  variables: &review-docker-variables
+    GIT_DEPTH: "1"
+    DOCKER_DRIVER: overlay2
+    DOCKER_HOST: tcp://docker:2375
+    LATEST_QA_IMAGE: "gitlab/${CI_PROJECT_NAME}-qa:nightly"
+    QA_IMAGE: "${CI_REGISTRY}/${CI_PROJECT_PATH}/gitlab/${CI_PROJECT_NAME}-qa:${CI_COMMIT_REF_SLUG}"
+
+build-qa-image:
+  <<: *review-docker
+  stage: prepare
+  script:
+    - time docker build --cache-from ${LATEST_QA_IMAGE} --tag ${QA_IMAGE} ./qa/
+    - echo "${CI_JOB_TOKEN}" | docker login --username gitlab-ci-token --password-stdin ${CI_REGISTRY}
+    - time docker push ${QA_IMAGE}
+
 danger-review:
   <<: *pull-cache
   image: registry.gitlab.com/gitlab-org/gitlab-build-images:danger
@@ -1054,86 +1098,97 @@ no_ee_check:
     - //@gitlab-org/gitlab-ce
 
 # GitLab Review apps
-review:
-  image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base
-  stage: test
-  allow_failure: true
-  before_script:
-    - gem install gitlab --no-document
+review-deploy:
+  <<: *review-base
   variables:
     GIT_DEPTH: "1"
-    HOST_SUFFIX: "$CI_ENVIRONMENT_SLUG"
-    DOMAIN: "-$CI_ENVIRONMENT_SLUG.$REVIEW_APPS_DOMAIN"
+    HOST_SUFFIX: "${CI_ENVIRONMENT_SLUG}"
+    DOMAIN: "-${CI_ENVIRONMENT_SLUG}.${REVIEW_APPS_DOMAIN}"
     GITLAB_HELM_CHART_REF: "master"
+    API_TOKEN: "${GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN}"
+  environment:
+    <<: *review-environment
+    on_stop: review-stop
+  before_script:
+    - apk update && apk add jq
+    - gem install gitlab --no-document
   script:
     - export GITLAB_SHELL_VERSION=$(<GITLAB_SHELL_VERSION)
     - export GITALY_VERSION=$(<GITALY_SERVER_VERSION)
     - export GITLAB_WORKHORSE_VERSION=$(<GITLAB_WORKHORSE_VERSION)
     - source ./scripts/review_apps/review-apps.sh
+    - wait_for_job_to_be_done "gitlab:assets:compile"
     - BUILD_TRIGGER_TOKEN=$REVIEW_APPS_BUILD_TRIGGER_TOKEN ./scripts/trigger-build cng
     - check_kube_domain
     - download_gitlab_chart
     - ensure_namespace
     - install_tiller
     - install_external_dns
-    - deploy
+    - time deploy
     - add_license
-  environment:
-    name: review/$CI_COMMIT_REF_NAME
-    url: https://gitlab-$CI_ENVIRONMENT_SLUG.$REVIEW_APPS_DOMAIN
-    on_stop: stop_review
-  only:
-    refs:
-      - branches@gitlab-org/gitlab-ce
-      - branches@gitlab-org/gitlab-ee
-    kubernetes: active
-  except:
-    refs:
-      - master
-      - /(^docs[\/-].*|.*-docs$)/
 
-stop_review:
+.review-qa-base: &review-qa-base
+  <<: *review-docker
+  variables:
+    <<: *review-docker-variables
+    API_TOKEN: "${GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN}"
+    QA_ARTIFACTS_DIR: "${CI_PROJECT_DIR}/qa"
+    QA_CAN_TEST_GIT_PROTOCOL_V2: "false"
+    GITLAB_USERNAME: "root"
+    GITLAB_PASSWORD: "${REVIEW_APPS_ROOT_PASSWORD}"
+    GITLAB_ADMIN_USERNAME: "root"
+    GITLAB_ADMIN_PASSWORD: "${REVIEW_APPS_ROOT_PASSWORD}"
+    GITHUB_ACCESS_TOKEN: "${REVIEW_APPS_QA_GITHUB_ACCESS_TOKEN}"
+    EE_LICENSE: "${REVIEW_APPS_EE_LICENSE}"
+  artifacts:
+    paths:
+      - ./qa/gitlab-qa-run-*
+    expire_in: 7 days
+    when: always
+  before_script:
+    - echo "${QA_IMAGE}"
+    - echo "${CI_ENVIRONMENT_URL}"
+    - apk update && apk add curl jq
+    - source ./scripts/review_apps/review-apps.sh
+    - gem install gitlab-qa --no-document ${GITLAB_QA_VERSION:+ --version ${GITLAB_QA_VERSION}}
+    - wait_for_job_to_be_done "review-deploy"
+
+review-qa-smoke:
+  <<: *review-qa-base
+  script:
+    - gitlab-qa Test::Instance::Smoke "${QA_IMAGE}" "${CI_ENVIRONMENT_URL}"
+
+review-qa-all:
+  <<: *review-qa-base
+  script:
+    - gitlab-qa Test::Instance::Any "${QA_IMAGE}" "${CI_ENVIRONMENT_URL}"
+  when: manual
+
+review-stop:
+  <<: *review-base
   <<: *single-script-job
   image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base
-  stage: test
   allow_failure: true
-  cache: {}
-  dependencies: []
   variables:
+    <<: *single-script-job-variables
     SCRIPT_NAME: "review_apps/review-apps.sh"
+  when: manual
+  environment:
+    <<: *review-environment
+    action: stop
   script:
     - source $(basename "${SCRIPT_NAME}")
     - delete
     - cleanup
-  when: manual
-  environment:
-    name: review/$CI_COMMIT_REF_NAME
-    action: stop
-  only:
-    refs:
-      - branches@gitlab-org/gitlab-ce
-      - branches@gitlab-org/gitlab-ee
-    kubernetes: active
-  except:
-    - master
-    - /(^docs[\/-].*|.*-docs$)/
 
-schedule:review_apps_cleanup:
-  <<: *dedicated-no-docs-pull-cache-job
-  image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base
+schedule:review-cleanup:
+  <<: *review-base
   stage: build
   allow_failure: true
-  cache: {}
-  dependencies: []
-  before_script:
-    - gem install gitlab --no-document
   variables:
     GIT_DEPTH: "1"
-  script:
-    - ruby -rrubygems scripts/review_apps/automated_cleanup.rb
   environment:
     name: review/auto-cleanup
-    action: stop
   only:
     refs:
       - schedules@gitlab-org/gitlab-ce
@@ -1142,3 +1197,7 @@ schedule:review_apps_cleanup:
   except:
     - tags
     - /(^docs[\/-].*|.*-docs$)/
+  before_script:
+    - gem install gitlab --no-document
+  script:
+    - ruby -rrubygems scripts/review_apps/automated_cleanup.rb

doc/development/testing_guide/review_apps.md

@@ -7,31 +7,36 @@ Review Apps are automatically deployed by each pipeline, both in
 ## How does it work?
 
 1. On every [pipeline][gitlab-pipeline] during the `test` stage, the
-  [`review` job][review-job] is automatically started.
-1. The `review` job [triggers a pipeline][cng-pipeline] in the
-  [`CNG-mirror`][cng-mirror] project.
-    - We use the `CNG-mirror` project so that the `CNG`, (**C**loud **N**ative
-      **G**itLab), project's registry is not overloaded with a lot of transient
-      Docker images.
-1. The `CNG-mirror` pipeline creates the Docker images of each component (e.g.
-  `gitlab-rails-ee`, `gitlab-shell`, `gitaly` etc.) based on the commit from the
-  [GitLab pipeline][gitlab-pipeline] and store them in its
-  [registry][cng-mirror-registry].
-1. Once all images are built, the Review App is deployed using
-  [the official GitLab Helm chart][helm-chart] to the
-  [`review-apps-ee` Kubernetes cluster on GCP][review-apps-ee]
-    - The actual scripts used to deploy the Review App can be found at
-      [`scripts/review_apps/review-apps.sh`][review-apps.sh]
-    - These scripts are basically
-      [our official Auto DevOps scripts][Auto-DevOps.gitlab-ci.yml] where the
-      default CNG images are overridden with the images built and stored in the
-      [`CNG-mirror` project's registry][cng-mirror-registry].
-    - Since we're using [the official GitLab Helm chart][helm-chart], this means
-      you get a dedicated environment for your branch that's very close to what it
-      would look in production.
-1. Once the `review` job succeeds, you should be able to use your Review App
-  thanks to the direct link to it from the MR widget. The default username is
-  `root` and its password can be found in the 1Password secure note named
+  [`review-deploy`][review-deploy-job] job is automatically started.
+1. The `review-deploy` job:
+    1. Waits for the `gitlab:assets:compile` job to finish since the
+      [`CNG-mirror`][cng-mirror] pipeline triggerred in the following step
+      depends on it.
+    1. [Triggers a pipeline][cng-pipeline] in the [`CNG-mirror`][cng-mirror]
+      project.
+        - We use the `CNG-mirror` project so that the `CNG`, (**C**loud
+          **N**ative **G**itLab), project's registry is not overloaded with a
+          lot of transient Docker images.
+        - The `CNG-mirror` pipeline creates the Docker images of each component
+          (e.g. `gitlab-rails-ee`, `gitlab-shell`, `gitaly` etc.) based on the
+          commit from the [GitLab pipeline][gitlab-pipeline] and store them in
+          its [registry][cng-mirror-registry].
+    1. Once all images are built by [`CNG-mirror`][cng-mirror], the Review App
+      is deployed using [the official GitLab Helm chart][helm-chart] to the
+      [`review-apps-ce`][review-apps-ce] / [`review-apps-ee`][review-apps-ee]
+      Kubernetes cluster on GCP.
+        - The actual scripts used to deploy the Review App can be found at
+          [`scripts/review_apps/review-apps.sh`][review-apps.sh].
+        - These scripts are basically
+          [our official Auto DevOps scripts][Auto-DevOps.gitlab-ci.yml] where the
+          default CNG images are overridden with the images built and stored in the
+          [`CNG-mirror` project's registry][cng-mirror-registry].
+        - Since we're using [the official GitLab Helm chart][helm-chart], this means
+          you get a dedicated environment for your branch that's very close to what
+          it would look in production.
+1. Once the `review-deploy` job succeeds, you should be able to use your Review
+  App thanks to the direct link to it from the MR widget. The default username
+  is `root` and its password can be found in the 1Password secure note named
   **gitlab-{ce,ee} Review App's root password** (note that there's currently
   [a bug where the default password seems to be overridden][password-bug]).
 
@@ -39,16 +44,23 @@ Review Apps are automatically deployed by each pipeline, both in
 
 - The Kubernetes cluster is connected to the `gitlab-{ce,ee}` projects using
   [GitLab's Kubernetes integration][gitlab-k8s-integration]. This basically
-  allows to have a link to the Review App directly from the merge request widget.
-- The manual `stop_review` in the `test` stage can be used to stop a Review App
-  manually, and is also started by GitLab once a branch is deleted.
-- Review Apps are cleaned up regularly using a pipeline schedule that runs
-  the [`scripts/review_apps/automated_cleanup.rb`][automated_cleanup.rb] script.
+  allows to have a link to the Review App directly from the merge request
+  widget.
 - If the Review App deployment fails, you can simply retry it (there's no need
-  to run the `stop_review` job first).
-- If you're unable to log in using the `root` username and password, you may
-  encounter [this bug][password-bug]. Stop the Review App via the `stop_review`
-  manual job and then retry the `review` job to redeploy the Review App.
+  to run the [`review-stop`][gitlab-ci-yml] job first).
+- The manual [`review-stop`][gitlab-ci-yml] in the `test` stage can be used to
+  stop a Review App manually, and is also started by GitLab once a branch is
+  deleted.
+- Review Apps are cleaned up regularly using a pipeline schedule that runs
+  the [`schedule:review-cleanup`][gitlab-ci-yml] job.
+
+## QA runs
+
+On every [pipeline][gitlab-pipeline] during the `test` stage, the
+`review-qa-smoke` job is automatically started: it runs the smoke QA suite.
+You can also manually start the `review-qa-all`: it runs the full QA suite.
+
+Note that both jobs first wait for the `review-deploy` job to be finished.
 
 ## Frequently Asked Questions
 
@@ -74,15 +86,17 @@ find a way to limit it to only us.**
   > This isn't enabled for forks.
 
 [gitlab-pipeline]: https://gitlab.com/gitlab-org/gitlab-ce/pipelines/35850709
-[review-job]: https://gitlab.com/gitlab-org/gitlab-ce/-/jobs/118076368
+[review-deploy-job]: https://gitlab.com/gitlab-org/gitlab-ce/-/jobs/118076368
 [cng-mirror]: https://gitlab.com/gitlab-org/build/CNG-mirror
 [cng-pipeline]: https://gitlab.com/gitlab-org/build/CNG-mirror/pipelines/35883435
 [cng-mirror-registry]: https://gitlab.com/gitlab-org/build/CNG-mirror/container_registry
 [helm-chart]: https://gitlab.com/charts/gitlab/
+[review-apps-ce]: https://console.cloud.google.com/kubernetes/clusters/details/us-central1-a/review-apps-ce?project=gitlab-review-apps
 [review-apps-ee]: https://console.cloud.google.com/kubernetes/clusters/details/us-central1-b/review-apps-ee?project=gitlab-review-apps
 [review-apps.sh]: https://gitlab.com/gitlab-org/gitlab-ee/blob/master/scripts/review_apps/review-apps.sh
 [automated_cleanup.rb]: https://gitlab.com/gitlab-org/gitlab-ee/blob/master/scripts/review_apps/automated_cleanup.rb
 [Auto-DevOps.gitlab-ci.yml]: https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml
+[gitlab-ci-yml]: https://gitlab.com/gitlab-org/gitlab-ce/blob/master/.gitlab-ci.yml
 [gitlab-k8s-integration]: https://docs.gitlab.com/ee/user/project/clusters/index.html
 [password-bug]: https://gitlab.com/gitlab-org/gitlab-ce/issues/53621
 

scripts/review_apps/review-apps.sh

 [[ "$TRACE" ]] && set -x
 export TILLER_NAMESPACE="$KUBE_NAMESPACE"
 
+function echoerr() { printf "\033[0;31m%s\n\033[0m" "$*" >&2; }
+
 function check_kube_domain() {
   if [ -z ${REVIEW_APPS_DOMAIN+x} ]; then
     echo "In order to deploy or use Review Apps, REVIEW_APPS_DOMAIN variable must be set"
@@ -88,19 +90,16 @@ function deploy() {
   replicas="1"
   service_enabled="false"
   postgres_enabled="$POSTGRES_ENABLED"
-  gitlab_migrations_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-rails-ce"
-  gitlab_sidekiq_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-sidekiq-ce"
-  gitlab_unicorn_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-unicorn-ce"
-  gitlab_gitaly_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitaly"
-  gitlab_shell_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-shell"
-  gitlab_workhorse_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-workhorse-ce"
-
-  if [[ "$CI_PROJECT_NAME" == "gitlab-ee" ]]; then
-    gitlab_migrations_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-rails-ee"
-    gitlab_sidekiq_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-sidekiq-ee"
-    gitlab_unicorn_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-unicorn-ee"
-    gitlab_workhorse_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-workhorse-ee"
-  fi
+
+  IMAGE_REPOSITORY="registry.gitlab.com/gitlab-org/build/cng-mirror"
+  IMAGE_VERSION="${CI_PROJECT_NAME#gitlab-}"
+  gitlab_migrations_image_repository="${IMAGE_REPOSITORY}/gitlab-rails-${IMAGE_VERSION}"
+  gitlab_sidekiq_image_repository="${IMAGE_REPOSITORY}/gitlab-sidekiq-${IMAGE_VERSION}"
+  gitlab_unicorn_image_repository="${IMAGE_REPOSITORY}/gitlab-unicorn-${IMAGE_VERSION}"
+  gitlab_task_runner_image_repository="${IMAGE_REPOSITORY}/gitlab-task-runner-${IMAGE_VERSION}"
+  gitlab_gitaly_image_repository="${IMAGE_REPOSITORY}/gitaly"
+  gitlab_shell_image_repository="${IMAGE_REPOSITORY}/gitlab-shell"
+  gitlab_workhorse_image_repository="${IMAGE_REPOSITORY}/gitlab-workhorse-${IMAGE_VERSION}"
 
   # canary uses stable db
   [[ "$track" == "canary" ]] && postgres_enabled="false"
@@ -155,6 +154,8 @@ HELM_CMD=$(cat << EOF
     --set gitlab.sidekiq.image.tag="$CI_COMMIT_REF_NAME" \
     --set gitlab.unicorn.image.repository="$gitlab_unicorn_image_repository" \
     --set gitlab.unicorn.image.tag="$CI_COMMIT_REF_NAME" \
+    --set gitlab.task-runner.image.repository="$gitlab_task_runner_image_repository" \
+    --set gitlab.task-runner.image.tag="$CI_COMMIT_REF_NAME" \
     --set gitlab.gitaly.image.repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitaly" \
     --set gitlab.gitaly.image.tag="v$GITALY_VERSION" \
     --set gitlab.gitlab-shell.image.repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-shell" \
@@ -230,18 +231,29 @@ function install_external_dns() {
   fi
 }
 
-function add_license() {
-  if [ -z "${REVIEW_APPS_EE_LICENSE}" ]; then echo "License not found" && return; fi
+function get_pod() {
+  local app_name="${1}"
+  local status="${2-Running}"
+  get_pod_cmd="kubectl get pods -n ${KUBE_NAMESPACE} --field-selector=status.phase=${status} -lapp=${app_name},release=${CI_ENVIRONMENT_SLUG} --no-headers -o=custom-columns=NAME:.metadata.name"
+  echoerr "Running '${get_pod_cmd}'"
 
-  while [ -z "$(kubectl get pods -n ${KUBE_NAMESPACE} --field-selector=status.phase=Running -lapp=task-runner,release=${CI_ENVIRONMENT_SLUG} --no-headers -o=custom-columns=NAME:.metadata.name)" ]; do
-    echo "Waiting till task-runner pod is ready";
+  while true; do
+    local pod_name="$(eval $get_pod_cmd)"
+    [[ "${pod_name}" == "" ]] || break
+
+    echoerr "Waiting till '${app_name}' pod is ready";
     sleep 5;
   done
 
-  task_runner_pod=$(kubectl get pods -n ${KUBE_NAMESPACE} --field-selector=status.phase=Running -lapp=task-runner,release=${CI_ENVIRONMENT_SLUG} --no-headers -o=custom-columns=NAME:.metadata.name)
+  echoerr "The pod name is '${pod_name}'."
+  echo "${pod_name}"
+}
+
+function add_license() {
+  if [ -z "${REVIEW_APPS_EE_LICENSE}" ]; then echo "License not found" && return; fi
 
+  task_runner_pod=$(get_pod "task-runner");
   if [ -z "${task_runner_pod}" ]; then echo "Task runner pod not found" && return; fi
-  echo "Task runner pod is ${task_runner_pod}"
 
   echo "${REVIEW_APPS_EE_LICENSE}" > /tmp/license.gitlab
   kubectl -n "$KUBE_NAMESPACE" cp /tmp/license.gitlab ${task_runner_pod}:/tmp/license.gitlab
@@ -265,3 +277,74 @@ function add_license() {
     puts "License added";
     '
 }
+
+function get_job_id() {
+  local job_name="${1}"
+  local query_string="${2:+&${2}}"
+
+  local max_page=3
+  local page=1
+
+  while true; do
+    local url="https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/pipelines/${CI_PIPELINE_ID}/jobs?per_page=100&page=${page}${query_string}"
+    echoerr "GET ${url}"
+
+    local job_id=$(curl --silent --show-error --header "PRIVATE-TOKEN: ${API_TOKEN}" "${url}" | jq ".[] | select(.name == \"${job_name}\") | .id")
+    [[ "${job_id}" == "" && "${page}" -lt "$max_page" ]] || break
+
+    ((page++))
+  done
+
+  if [[ "${job_id}" == "" ]]; then
+    echoerr "The '${job_name}' job ID couldn't be retrieved!"
+  else
+    echoerr "The '${job_name}' job ID is ${job_id}"
+    echo "${job_id}"
+  fi
+}
+
+function play_job() {
+  local job_name="${1}"
+  local job_id=$(get_job_id "${job_name}" "scope=manual");
+  if [ -z "${job_id}" ]; then return; fi
+
+  local url="https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/jobs/${job_id}/play"
+  echoerr "POST ${url}"
+
+  local job_url=$(curl --silent --show-error --request POST --header "PRIVATE-TOKEN: ${API_TOKEN}" "${url}" | jq ".web_url")
+  echo "Manual job '${job_name}' started at: ${job_url}"
+}
+
+function wait_for_job_to_be_done() {
+  local job_name="${1}"
+  local query_string="${2}"
+  local job_id=$(get_job_id "${job_name}" "${query_string}");
+  if [ -z "${job_id}" ]; then return; fi
+
+  echoerr "Waiting for the '${job_name}' job to finish..."
+
+  local url="https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/jobs/${job_id}"
+  echo "GET ${url}"
+
+  # In case the job hasn't finished yet. Keep trying until the job times out.
+  local interval=30
+  local elapsed=0
+  while true; do
+    local job_status=$(curl --silent --show-error --header "PRIVATE-TOKEN: ${API_TOKEN}" "${url}" | jq ".status" | sed -e s/\"//g)
+    [[ "${job_status}" == "pending" || "${job_status}" == "running" ]] || break
+
+    printf "."
+    ((elapsed+=$interval))
+    sleep ${interval}
+  done
+
+  echoerr "Waited '${job_name}' for ${elapsed} seconds."
+
+  if [[ "${job_status}" == "failed" ]]; then
+    echo "The '${job_name}' failed."
+  elif [[ "${job_status}" == "manual" ]]; then
+    echo "The '${job_name}' is manual."
+  else
+    echo "The '${job_name}' passed."
+  fi
+}