Merge branch 'use-true-triggered-pipelines-for-downstream-cng-pipelines' into 'master'

ci: Use the trigger keyword for CNG downstream pipelines

See merge request gitlab-org/gitlab!71185

.gitlab/ci/cng.gitlab-ci.yml

+cloud-native-image-env:
+  extends:
+    - .default-retry
+    - .cng:rules
+  image: ${GITLAB_DEPENDENCY_PROXY}ruby:2.7-alpine3.13
+  stage: post-test
+  before_script:
+    - source ./scripts/utils.sh
+    - install_gitlab_gem
+  script:
+    - 'ruby -r./scripts/trigger-build.rb -e "puts Trigger.variables_for_env_file(Trigger::CNG.new.variables)" > build.env'
+    - cat build.env
+  artifacts:
+    reports:
+      dotenv: build.env
+    paths:
+      - build.env
+    expire_in: 7 days
+    when: always
+
 cloud-native-image:
   extends: .cng:rules
-  image: ${GITLAB_DEPENDENCY_PROXY}ruby:2.7-alpine
-  dependencies: []
   stage: post-test
+  needs: ["cloud-native-image-env"]
+  inherit:
+    variables: false
   variables:
-    GIT_DEPTH: "1"
-  script:
-    - install_gitlab_gem
-    - ./scripts/trigger-build cng
+    TOP_UPSTREAM_SOURCE_PROJECT: "${TOP_UPSTREAM_SOURCE_PROJECT}"
+    TOP_UPSTREAM_SOURCE_REF: "${TOP_UPSTREAM_SOURCE_REF}"
+    TOP_UPSTREAM_SOURCE_JOB: "${TOP_UPSTREAM_SOURCE_JOB}"
+    TOP_UPSTREAM_SOURCE_SHA: "${TOP_UPSTREAM_SOURCE_SHA}"
+    TOP_UPSTREAM_MERGE_REQUEST_PROJECT_ID: "${TOP_UPSTREAM_MERGE_REQUEST_PROJECT_ID}"
+    TOP_UPSTREAM_MERGE_REQUEST_IID: "${TOP_UPSTREAM_MERGE_REQUEST_IID}"
+    GITLAB_REF_SLUG: "${GITLAB_REF_SLUG}"
+    # CNG pipeline specific variables
+    GITLAB_VERSION: "${GITLAB_VERSION}"
+    GITLAB_TAG: "${GITLAB_TAG}"
+    GITLAB_ASSETS_TAG: "${GITLAB_ASSETS_TAG}"
+    FORCE_RAILS_IMAGE_BUILDS: "${FORCE_RAILS_IMAGE_BUILDS}"
+    CE_PIPELINE: "${CE_PIPELINE}"  # Based on https://docs.gitlab.com/ee/ci/jobs/job_control.html#check-if-a-variable-exists, `if: '$CE_PIPELINE'` will evaluate to `false` when this variable is empty
+    EE_PIPELINE: "${EE_PIPELINE}"  # Based on https://docs.gitlab.com/ee/ci/jobs/job_control.html#check-if-a-variable-exists, `if: '$EE_PIPELINE'` will evaluate to `false` when this variable is empty
+    GITLAB_SHELL_VERSION: "${GITLAB_SHELL_VERSION}"
+    GITLAB_ELASTICSEARCH_INDEXER_VERSION: "${GITLAB_ELASTICSEARCH_INDEXER_VERSION}"
+    GITLAB_KAS_VERSION: "${GITLAB_KAS_VERSION}"
+    GITLAB_WORKHORSE_VERSION: "${GITLAB_WORKHORSE_VERSION}"
+    GITLAB_PAGES_VERSION: "${GITLAB_PAGES_VERSION}"
+    GITALY_SERVER_VERSION: "${GITALY_SERVER_VERSION}"
+  trigger:
+    project: gitlab-org/build/CNG
+    branch: $TRIGGER_BRANCH
+    strategy: depend

.gitlab/ci/docs.gitlab-ci.yml

@@ -28,7 +28,7 @@
 review-docs-deploy:
   extends: .review-docs
   script:
-    - ./scripts/trigger-build docs deploy
+    - ./scripts/trigger-build.rb docs deploy
 
 # Cleanup remote environment of gitlab-docs
 review-docs-cleanup:
@@ -37,7 +37,7 @@ review-docs-cleanup:
     name: review-docs/mr-${CI_MERGE_REQUEST_IID}
     action: stop
   script:
-    - ./scripts/trigger-build docs cleanup
+    - ./scripts/trigger-build.rb docs cleanup
 
 docs-lint markdown:
   extends:

.gitlab/ci/qa.gitlab-ci.yml

@@ -73,7 +73,7 @@ update-qa-cache:
     - echo $exit_code
     - |
       if [ $exit_code -eq 0 ]; then
-        ./scripts/trigger-build omnibus
+        ./scripts/trigger-build.rb omnibus
       elif [ $exit_code -eq 1 ]; then
         exit 1
       else
@@ -108,7 +108,7 @@ update-qa-cache:
       if [[ $feature_flags ]]; then
         export GITLAB_QA_OPTIONS="--set-feature-flags $feature_flags"
         echo $GITLAB_QA_OPTIONS
-        ./scripts/trigger-build omnibus
+        ./scripts/trigger-build.rb omnibus
       else
         echo "No changed feature flag found to test. The tests are skipped if the flag was removed."
       fi

.gitlab/ci/rails.gitlab-ci.yml

@@ -438,7 +438,7 @@ db:gitlabcom-database-testing:
   script:
     - source scripts/utils.sh
     - install_gitlab_gem
-    - ./scripts/trigger-build gitlab-com-database-testing
+    - ./scripts/trigger-build.rb gitlab-com-database-testing
 
 gitlab:setup:
   extends: .db-job-base

.gitlab/ci/review-apps/main.gitlab-ci.yml

@@ -16,20 +16,58 @@ include:
   - source ./scripts/review_apps/review-apps.sh
   - install_api_client_dependencies_with_apk
 
-review-build-cng:
+review-build-cng-env:
   extends:
     - .default-retry
     - .review:rules:review-build-cng
   image: ${GITLAB_DEPENDENCY_PROXY}ruby:2.7-alpine3.13
   stage: prepare
-  variables:
-    CNG_PROJECT_ACCESS_TOKEN: "${CNG_MIRROR_PROJECT_ACCESS_TOKEN}"  # "Multi-pipeline (from 'gitlab-org/gitlab' 'review-build-cng' job)" at https://gitlab.com/gitlab-org/build/CNG-mirror/-/settings/access_tokens
-    CNG_PROJECT_PATH: "gitlab-org/build/CNG-mirror"
+  needs: []
   before_script:
     - source ./scripts/utils.sh
     - install_gitlab_gem
   script:
-    - ./scripts/trigger-build cng
+    - 'ruby -r./scripts/trigger-build.rb -e "puts Trigger.variables_for_env_file(Trigger::CNG.new.variables)" > build.env'
+    - cat build.env
+  artifacts:
+    reports:
+      dotenv: build.env
+    paths:
+      - build.env
+    expire_in: 7 days
+    when: always
+
+review-build-cng:
+  extends: .review:rules:review-build-cng
+  stage: prepare
+  needs: ["review-build-cng-env"]
+  inherit:
+    variables: false
+  variables:
+    TOP_UPSTREAM_SOURCE_PROJECT: "${TOP_UPSTREAM_SOURCE_PROJECT}"
+    TOP_UPSTREAM_SOURCE_REF: "${TOP_UPSTREAM_SOURCE_REF}"
+    TOP_UPSTREAM_SOURCE_JOB: "${TOP_UPSTREAM_SOURCE_JOB}"
+    TOP_UPSTREAM_SOURCE_SHA: "${TOP_UPSTREAM_SOURCE_SHA}"
+    TOP_UPSTREAM_MERGE_REQUEST_PROJECT_ID: "${TOP_UPSTREAM_MERGE_REQUEST_PROJECT_ID}"
+    TOP_UPSTREAM_MERGE_REQUEST_IID: "${TOP_UPSTREAM_MERGE_REQUEST_IID}"
+    GITLAB_REF_SLUG: "${GITLAB_REF_SLUG}"
+    # CNG pipeline specific variables
+    GITLAB_VERSION: "${GITLAB_VERSION}"
+    GITLAB_TAG: "${GITLAB_TAG}"
+    GITLAB_ASSETS_TAG: "${GITLAB_ASSETS_TAG}"
+    FORCE_RAILS_IMAGE_BUILDS: "${FORCE_RAILS_IMAGE_BUILDS}"
+    CE_PIPELINE: "${CE_PIPELINE}"  # Based on https://docs.gitlab.com/ee/ci/jobs/job_control.html#check-if-a-variable-exists, `if: '$CE_PIPELINE'` will evaluate to `false` when this variable is empty
+    EE_PIPELINE: "${EE_PIPELINE}"  # Based on https://docs.gitlab.com/ee/ci/jobs/job_control.html#check-if-a-variable-exists, `if: '$EE_PIPELINE'` will evaluate to `false` when this variable is empty
+    GITLAB_SHELL_VERSION: "${GITLAB_SHELL_VERSION}"
+    GITLAB_ELASTICSEARCH_INDEXER_VERSION: "${GITLAB_ELASTICSEARCH_INDEXER_VERSION}"
+    GITLAB_KAS_VERSION: "${GITLAB_KAS_VERSION}"
+    GITLAB_WORKHORSE_VERSION: "${GITLAB_WORKHORSE_VERSION}"
+    GITLAB_PAGES_VERSION: "${GITLAB_PAGES_VERSION}"
+    GITALY_SERVER_VERSION: "${GITALY_SERVER_VERSION}"
+  trigger:
+    project: gitlab-org/build/CNG-mirror
+    branch: $TRIGGER_BRANCH
+    strategy: depend
 
 .review-workflow-base:
   extends:

.gitlab/ci/rules.gitlab-ci.yml

@@ -141,7 +141,7 @@
   - ".gitlab/ci/review-apps/**/*"
   - "scripts/review_apps/base-config.yaml"
   - "scripts/review_apps/review-apps.sh"
-  - "scripts/trigger-build"
+  - "scripts/trigger-build.rb"
   - "{,ee/,jh/}{bin,config}/**/*.rb"
 
 .ci-qa-patterns: &ci-qa-patterns

doc/development/documentation/review_apps.md

@@ -41,7 +41,7 @@ the GitLab team to run the job.
 If you want to know the in-depth details, here's what's really happening:
 
 1. You manually run the `review-docs-deploy` job in a merge request.
-1. The job runs the [`scripts/trigger-build`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/scripts/trigger-build)
+1. The job runs the [`scripts/trigger-build.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/scripts/trigger-build.rb)
    script with the `docs deploy` flag, which triggers the "Triggered from `gitlab-org/gitlab` 'review-docs-deploy' job"
    pipeline trigger in the `gitlab-org/gitlab-docs` project for the `$DOCS_BRANCH` (defaults to `main`).
 1. The preview URL is shown both at the job output and in the merge request

scripts/trigger-build → scripts/trigger-build.rb

@@ -21,6 +21,12 @@ module Trigger
     variable_value
   end
 
+  def self.variables_for_env_file(variables)
+    variables.map do |key, value|
+      %Q(#{key}=#{value})
+    end.join("\n")
+  end
+
   class Base
     # Can be overridden
     def self.access_token
@@ -57,6 +63,21 @@ module Trigger
       end
     end
 
+    def variables
+      simple_forwarded_variables.merge(base_variables, extra_variables, version_file_variables)
+    end
+
+    def simple_forwarded_variables
+      {
+        'TRIGGER_SOURCE' => ENV['CI_JOB_URL'],
+        'TOP_UPSTREAM_SOURCE_PROJECT' => ENV['CI_PROJECT_PATH'],
+        'TOP_UPSTREAM_SOURCE_REF' => ENV['CI_COMMIT_REF_NAME'],
+        'TOP_UPSTREAM_SOURCE_JOB' => ENV['CI_JOB_URL'],
+        'TOP_UPSTREAM_MERGE_REQUEST_PROJECT_ID' => ENV['CI_MERGE_REQUEST_PROJECT_ID'],
+        'TOP_UPSTREAM_MERGE_REQUEST_IID' => ENV['CI_MERGE_REQUEST_IID']
+      }
+    end
+
     private
 
     # Override to trigger and work with pipeline on different GitLab instance
@@ -95,23 +116,13 @@ module Trigger
       ENV[version_file]&.strip || File.read(version_file).strip
     end
 
-    def variables
-      base_variables.merge(extra_variables).merge(version_file_variables)
-    end
-
     def base_variables
       # Use CI_MERGE_REQUEST_SOURCE_BRANCH_SHA for omnibus checkouts due to pipeline for merged results,
       # and fallback to CI_COMMIT_SHA for the `detached` pipelines.
       {
         'GITLAB_REF_SLUG' => ENV['CI_COMMIT_TAG'] ? ENV['CI_COMMIT_REF_NAME'] : ENV['CI_COMMIT_REF_SLUG'],
         'TRIGGERED_USER' => ENV['TRIGGERED_USER'] || ENV['GITLAB_USER_NAME'],
-        'TRIGGER_SOURCE' => ENV['CI_JOB_URL'],
-        'TOP_UPSTREAM_SOURCE_PROJECT' => ENV['CI_PROJECT_PATH'],
-        'TOP_UPSTREAM_SOURCE_JOB' => ENV['CI_JOB_URL'],
-        'TOP_UPSTREAM_SOURCE_SHA' => Trigger.non_empty_variable_value('CI_MERGE_REQUEST_SOURCE_BRANCH_SHA') || ENV['CI_COMMIT_SHA'],
-        'TOP_UPSTREAM_SOURCE_REF' => ENV['CI_COMMIT_REF_NAME'],
-        'TOP_UPSTREAM_MERGE_REQUEST_PROJECT_ID' => ENV['CI_MERGE_REQUEST_PROJECT_ID'],
-        'TOP_UPSTREAM_MERGE_REQUEST_IID' => ENV['CI_MERGE_REQUEST_IID']
+        'TOP_UPSTREAM_SOURCE_SHA' => Trigger.non_empty_variable_value('CI_MERGE_REQUEST_SOURCE_BRANCH_SHA') || ENV['CI_COMMIT_SHA']
       }
     end
 
@@ -163,17 +174,16 @@ module Trigger
   end
 
   class CNG < Base
-    def self.access_token
-      # Default to "Multi-pipeline (from 'gitlab-org/gitlab' 'cloud-native-image' job)" at https://gitlab.com/gitlab-org/build/CNG/-/settings/access_tokens
-      ENV['CNG_PROJECT_ACCESS_TOKEN'] || super
+    def variables
+      # Delete variables that aren't useful when using native triggers.
+      super.tap do |hash|
+        hash.delete('TRIGGER_SOURCE')
+        hash.delete('TRIGGERED_USER')
+      end
     end
 
     private
 
-    def downstream_project_path
-      ENV.fetch('CNG_PROJECT_PATH', 'gitlab-org/build/CNG')
-    end
-
     def ref
       return ENV['CI_COMMIT_REF_NAME'] if ENV['CI_COMMIT_REF_NAME'] =~ /^[\d-]+-stable(-ee)?$/
 
@@ -181,17 +191,17 @@ module Trigger
     end
 
     def extra_variables
-      edition = Trigger.ee? ? 'EE' : 'CE'
       # Use CI_MERGE_REQUEST_SOURCE_BRANCH_SHA (MR HEAD commit) so that the image is in sync with the assets and QA images.
       source_sha = Trigger.non_empty_variable_value('CI_MERGE_REQUEST_SOURCE_BRANCH_SHA') || ENV['CI_COMMIT_SHA']
 
       {
-        "ee" => Trigger.ee? ? "true" : "false",
+        "TRIGGER_BRANCH" => ref,
         "GITLAB_VERSION" => source_sha,
-        "GITLAB_TAG" => ENV['CI_COMMIT_TAG'],
+        "GITLAB_TAG" => ENV['CI_COMMIT_TAG'], # Always set a value, even an empty string, so that the downstream pipeline can correctly check it.
         "GITLAB_ASSETS_TAG" => ENV['CI_COMMIT_TAG'] ? ENV['CI_COMMIT_REF_NAME'] : source_sha,
         "FORCE_RAILS_IMAGE_BUILDS" => 'true',
-        "#{edition}_PIPELINE" => 'true'
+        "CE_PIPELINE" => Trigger.ee? ? nil : "true", # Always set a value, even an empty string, so that the downstream pipeline can correctly check it.
+        "EE_PIPELINE" => Trigger.ee? ? "true" : nil # Always set a value, even an empty string, so that the downstream pipeline can correctly check it.
       }
     end
 
@@ -445,28 +455,30 @@ module Trigger
   Job = Class.new(Pipeline)
 end
 
-case ARGV[0]
-when 'omnibus'
-  Trigger::Omnibus.new.invoke!(post_comment: true, downstream_job_name: 'Trigger:qa-test').wait!
-when 'cng'
-  Trigger::CNG.new.invoke!.wait!
-when 'gitlab-com-database-testing'
-  Trigger::DatabaseTesting.new.invoke!
-when 'docs'
-  docs_trigger = Trigger::Docs.new
-
-  case ARGV[1]
-  when 'deploy'
-    docs_trigger.deploy!
-  when 'cleanup'
-    docs_trigger.cleanup!
+if $0 == __FILE__
+  case ARGV[0]
+  when 'omnibus'
+    Trigger::Omnibus.new.invoke!(post_comment: true, downstream_job_name: 'Trigger:qa-test').wait!
+  when 'cng'
+    Trigger::CNG.new.invoke!.wait!
+  when 'gitlab-com-database-testing'
+    Trigger::DatabaseTesting.new.invoke!
+  when 'docs'
+    docs_trigger = Trigger::Docs.new
+
+    case ARGV[1]
+    when 'deploy'
+      docs_trigger.deploy!
+    when 'cleanup'
+      docs_trigger.cleanup!
+    else
+      puts 'usage: trigger-build docs <deploy|cleanup>'
+      exit 1
+    end
   else
-    puts 'usage: trigger-build docs <deploy|cleanup>'
-    exit 1
+    puts "Please provide a valid option:
+    omnibus - Triggers a pipeline that builds the omnibus-gitlab package
+    cng - Triggers a pipeline that builds images used by the GitLab helm chart
+    gitlab-com-database-testing - Triggers a pipeline that tests database changes on GitLab.com data"
   end
-else
-  puts "Please provide a valid option:
-  omnibus - Triggers a pipeline that builds the omnibus-gitlab package
-  cng - Triggers a pipeline that builds images used by the GitLab helm chart
-  gitlab-com-database-testing - Triggers a pipeline that tests database changes on GitLab.com data"
 end