Notify slack when `scheduled:package-and-qa` fails

With `allow_failure: true` on a prerequisite job,
when the job fails it would be considered a success.
This results in the `schedule:package-and-qa` job being skipped.

To work around this, we always run
`schedule:package-and-qa:notify-failure`,
and look for a failed `schedule:package-and-qa` in the pipeline.

.gitlab/ci/notifications.gitlab-ci.yml

 .notify:
-  image: alpine
+  image: ruby:2.6-alpine
   stage: notification
   dependencies: []
   cache: {}
   before_script:
     - apk update && apk add git curl bash
+    - source scripts/utils.sh
+    - source scripts/notifications.sh
+    - install_gitlab_gem
   variables:
-    COMMIT_NOTES_URL: "https://$CI_SERVER_HOST/$CI_PROJECT_PATH/commit/$CI_COMMIT_SHA#notes-list"
+    COMMIT_NOTES_URL: "https://${CI_SERVER_HOST}/${CI_PROJECT_PATH}/commit/${CI_COMMIT_SHA}#notes-list"
 
 schedule:package-and-qa:notify-failure:
   extends:
     - .only:variables_refs-canonical-dot-com-schedules
     - .notify
   script:
-    - 'scripts/notify-slack qa-master ":skull_and_crossbones: Scheduled QA against master failed! :skull_and_crossbones: See $CI_PIPELINE_URL. For downstream pipelines, see $COMMIT_NOTES_URL" ci_failing'
+    - 'export NOTIFICATION_MESSAGE=":skull_and_crossbones: Scheduled QA against master failed! :skull_and_crossbones: See ${CI_PIPELINE_URL}. For downstream pipelines, see ${COMMIT_NOTES_URL}"'
+    - 'notify_on_job_failure schedule:package-and-qa qa-master "${NOTIFICATION_MESSAGE}" ci_failing'
   needs: ["schedule:package-and-qa"]
-  when: on_failure
+  allow_failure: true
+  when: always

scripts/get-job-id

+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'gitlab'
+require 'optparse'
+
+#
+# Configure credentials to be used with gitlab gem
+#
+Gitlab.configure do |config|
+  config.endpoint = 'https://gitlab.com/api/v4'
+  config.private_token = ENV['GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN']
+end
+
+options = {}
+OptionParser.new do |opts|
+  opts.on("-s", "--scope=SCOPE", "Find job with matching scope") do |scope|
+    options[:scope] = scope
+  end
+end.parse!
+
+class PipelineJobFinder
+  def initialize(project_id, pipeline_id, job_name, options)
+    @project_id = project_id
+    @pipeline_id = pipeline_id
+    @job_name = job_name
+    @options = options
+  end
+
+  def execute
+    Gitlab.pipeline_jobs(@project_id, @pipeline_id, @options).auto_paginate do |job|
+      break job if job.name == @job_name
+    end
+  end
+end
+
+project_id, pipeline_id, job_name = ARGV
+
+job = PipelineJobFinder.new(project_id, pipeline_id, job_name, options).execute
+
+return if job.nil?
+
+puts job.id

scripts/notifications.sh

+# Sends Slack notification MSG to CI_SLACK_WEBHOOK_URL (which needs to be set).
+# ICON_EMOJI needs to be set to an icon emoji name (without the `:` around it).
+function notify_slack() {
+    CHANNEL=$1
+    MSG=$2
+    ICON_EMOJI=$3
+
+    if [ -z "$CHANNEL" ] || [ -z "$CI_SLACK_WEBHOOK_URL" ] || [ -z "$MSG" ] || [ -z "$ICON_EMOJI" ]; then
+        echo "Missing argument(s) - Use: $0 channel message icon_emoji"
+        echo "and set CI_SLACK_WEBHOOK_URL environment variable."
+    else
+        curl -X POST --data-urlencode 'payload={"channel": "#'"${CHANNEL}"'", "username": "GitLab QA Bot", "text": "'"${MSG}"'", "icon_emoji": "'":${ICON_EMOJI}:"'"}' "${CI_SLACK_WEBHOOK_URL}"
+    fi
+}
+
+function notify_on_job_failure() {
+    JOB_NAME=$1
+    CHANNEL=$2
+    MSG=$3
+    ICON_EMOJI=$4
+
+    local job_id
+    job_id=$(scripts/get-job-id "$CI_PROJECT_ID" "$CI_PIPELINE_ID" "$JOB_NAME" -s failed)
+    if [ -n "${job_id}" ]; then
+        notify_slack "${CHANNEL}" "${MSG}" "${ICON_EMOJI}"
+    fi
+}

scripts/notify-slack

-#!/bin/bash
-# Sends Slack notification MSG to CI_SLACK_WEBHOOK_URL (which needs to be set).
-# ICON_EMOJI needs to be set to an icon emoji name (without the `:` around it).
-
-CHANNEL=$1
-MSG=$2
-ICON_EMOJI=$3
-
-if [ -z "$CHANNEL" ] || [ -z "$CI_SLACK_WEBHOOK_URL" ] || [ -z "$MSG" ] || [ -z "$ICON_EMOJI" ]; then
-    echo "Missing argument(s) - Use: $0 channel message icon_emoji"
-    echo "and set CI_SLACK_WEBHOOK_URL environment variable."
-else
-    curl -X POST --data-urlencode 'payload={"channel": "#'"$CHANNEL"'", "username": "GitLab QA Bot", "text": "'"$MSG"'", "icon_emoji": "'":$ICON_EMOJI:"'"}' "$CI_SLACK_WEBHOOK_URL"
-fi