Merge branch 'dont_run_auto_devops' into 'master'

Add global rules to match herokuish detect for Auto-DevOps

See merge request gitlab-org/gitlab!20267

changelogs/unreleased/dont_run_auto_devops.yml

+---
+title: Don't run Auto DevOps when no dockerfile or matching buildpack exists
+merge_request: 20267
+author:
+type: changed

ee/spec/requests/api/triggers_spec.rb

@@ -4,7 +4,7 @@ require 'spec_helper'
 
 describe API::Triggers do
   let(:user) { create(:user) }
-  let(:project) { create(:project, :repository, creator: user) }
+  let(:project) { create(:project, :repository, :auto_devops, creator: user) }
 
   describe 'POST /projects/:project_id/trigger/pipeline' do
     context 'when triggering a pipeline from a job token' do

ee/spec/services/merge_trains/create_pipeline_service_spec.rb

@@ -3,7 +3,7 @@
 require 'spec_helper'
 
 describe MergeTrains::CreatePipelineService do
-  let(:project) { create(:project, :repository) }
+  let(:project) { create(:project, :repository, :auto_devops) }
   set(:maintainer) { create(:user) }
   let(:service) { described_class.new(project, maintainer) }
   let(:previous_ref) { 'refs/heads/master' }

lib/gitlab/ci/pipeline/chain/config/content/auto_devops.rb

@@ -11,7 +11,7 @@ module Gitlab
                 strong_memoize(:content) do
                   next unless project&.auto_devops_enabled?
 
-                  template = Gitlab::Template::GitlabCiYmlTemplate.find('Auto-DevOps')
+                  template = Gitlab::Template::GitlabCiYmlTemplate.find(template_name)
                   YAML.dump('include' => [{ 'template' => template.full_name }])
                 end
               end
@@ -19,6 +19,22 @@ module Gitlab
               def source
                 :auto_devops_source
               end
+
+              private
+
+              def template_name
+                if beta_enabled?
+                  'Beta/Auto-DevOps'
+                else
+                  'Auto-DevOps'
+                end
+              end
+
+              def beta_enabled?
+                Feature.enabled?(:auto_devops_beta, project, default_enabled: true) &&
+                  # workflow:rules are required by `Beta/Auto-DevOps.gitlab-ci.yml`
+                  Feature.enabled?(:workflow_rules, project, default_enabled: true)
+              end
             end
           end
         end

lib/gitlab/ci/pipeline/chain/config/content/legacy_auto_devops.rb

@@ -11,7 +11,7 @@ module Gitlab
                 strong_memoize(:content) do
                   next unless project&.auto_devops_enabled?
 
-                  template = Gitlab::Template::GitlabCiYmlTemplate.find('Auto-DevOps')
+                  template = Gitlab::Template::GitlabCiYmlTemplate.find(template_name)
                   template.content
                 end
               end
@@ -19,6 +19,22 @@ module Gitlab
               def source
                 :auto_devops_source
               end
+
+              private
+
+              def template_name
+                if beta_enabled?
+                  'Beta/Auto-DevOps'
+                else
+                  'Auto-DevOps'
+                end
+              end
+
+              def beta_enabled?
+                Feature.enabled?(:auto_devops_beta, project, default_enabled: true) &&
+                  # workflow:rules are required by `Beta/Auto-DevOps.gitlab-ci.yml`
+                  Feature.enabled?(:workflow_rules, project, default_enabled: true)
+              end
             end
           end
         end

lib/gitlab/ci/templates/Beta/Auto-DevOps.gitlab-ci.yml

+# Auto DevOps - BETA do not use
+# This CI/CD configuration provides a standard pipeline for
+# * building a Docker image (using a buildpack if necessary),
+# * storing the image in the container registry,
+# * running tests from a buildpack,
+# * running code quality analysis,
+# * creating a review app for each topic branch,
+# * and continuous deployment to production
+#
+# Test jobs may be disabled by setting environment variables:
+# * test: TEST_DISABLED
+# * code_quality: CODE_QUALITY_DISABLED
+# * license_management: LICENSE_MANAGEMENT_DISABLED
+# * performance: PERFORMANCE_DISABLED
+# * sast: SAST_DISABLED
+# * dependency_scanning: DEPENDENCY_SCANNING_DISABLED
+# * container_scanning: CONTAINER_SCANNING_DISABLED
+# * dast: DAST_DISABLED
+# * review: REVIEW_DISABLED
+# * stop_review: REVIEW_DISABLED
+#
+# In order to deploy, you must have a Kubernetes cluster configured either
+# via a project integration, or via group/project variables.
+# KUBE_INGRESS_BASE_DOMAIN must also be set on the cluster settings,
+# as a variable at the group or project level, or manually added below.
+#
+# Continuous deployment to production is enabled by default.
+# If you want to deploy to staging first, set STAGING_ENABLED environment variable.
+# If you want to enable incremental rollout, either manual or time based,
+# set INCREMENTAL_ROLLOUT_MODE environment variable to "manual" or "timed".
+# If you want to use canary deployments, set CANARY_ENABLED environment variable.
+#
+# If Auto DevOps fails to detect the proper buildpack, or if you want to
+# specify a custom buildpack, set a project variable `BUILDPACK_URL` to the
+# repository URL of the buildpack.
+# e.g. BUILDPACK_URL=https://github.com/heroku/heroku-buildpack-ruby.git#v142
+# If you need multiple buildpacks, add a file to your project called
+# `.buildpacks` that contains the URLs, one on each line, in order.
+# Note: Auto CI does not work with multiple buildpacks yet
+
+image: alpine:latest
+
+variables:
+  # KUBE_INGRESS_BASE_DOMAIN is the application deployment domain and should be set as a variable at the group or project level.
+  # KUBE_INGRESS_BASE_DOMAIN: domain.example.com
+
+  POSTGRES_USER: user
+  POSTGRES_PASSWORD: testing-password
+  POSTGRES_ENABLED: "true"
+  POSTGRES_DB: $CI_ENVIRONMENT_SLUG
+  POSTGRES_VERSION: 9.6.2
+
+  DOCKER_DRIVER: overlay2
+
+  ROLLOUT_RESOURCE_TYPE: deployment
+
+  DOCKER_TLS_CERTDIR: ""  # https://gitlab.com/gitlab-org/gitlab-runner/issues/4501
+
+stages:
+  - build
+  - test
+  - deploy  # dummy stage to follow the template guidelines
+  - review
+  - dast
+  - staging
+  - canary
+  - production
+  - incremental rollout 10%
+  - incremental rollout 25%
+  - incremental rollout 50%
+  - incremental rollout 100%
+  - performance
+  - cleanup
+
+workflow:
+  rules:
+    - if: '$BUILDPACK_URL || $AUTO_DEVOPS_EXPLICITLY_ENABLED == "1"'
+
+    - exists:
+        - Dockerfile
+
+    # https://github.com/heroku/heroku-buildpack-clojure
+    - exists:
+        - project.clj
+
+    # https://github.com/heroku/heroku-buildpack-go
+    - exists:
+        - go.mod
+        - Gopkg.mod
+        - Godeps/Godeps.json
+        - vendor/vendor.json
+        - glide.yaml
+        - src/**/*.go
+
+    # https://github.com/heroku/heroku-buildpack-gradle
+    - exists:
+        - gradlew
+        - build.gradle
+        - settings.gradle
+
+    # https://github.com/heroku/heroku-buildpack-java
+    - exists:
+        - pom.xml
+        - pom.atom
+        - pom.clj
+        - pom.groovy
+        - pom.rb
+        - pom.scala
+        - pom.yaml
+        - pom.yml
+
+    # https://github.com/heroku/heroku-buildpack-multi
+    - exists:
+        - .buildpacks
+
+    # https://github.com/heroku/heroku-buildpack-nodejs
+    - exists:
+        - package.json
+
+    # https://github.com/heroku/heroku-buildpack-php
+    - exists:
+        - composer.json
+        - index.php
+
+    # https://github.com/heroku/heroku-buildpack-play
+    # TODO: detect script excludes some scala files
+    - exists:
+        - '**/conf/application.conf'
+
+    # https://github.com/heroku/heroku-buildpack-python
+    # TODO: detect script checks that all of these exist, not any
+    - exists:
+        - requirements.txt
+        - setup.py
+        - Pipfile
+
+    # https://github.com/heroku/heroku-buildpack-ruby
+    - exists:
+        - Gemfile
+
+    # https://github.com/heroku/heroku-buildpack-scala
+    - exists:
+        - '*.sbt'
+        - project/*.scala
+        - .sbt/*.scala
+        - project/build.properties
+
+    # https://github.com/dokku/buildpack-nginx
+    - exists:
+        - .static
+
+include:
+  - template: Jobs/Build.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml
+  - template: Jobs/Test.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Jobs/Test.gitlab-ci.yml
+  - template: Jobs/Code-Quality.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml
+  - template: Jobs/Deploy.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml
+  - template: Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml
+  - template: Jobs/Browser-Performance-Testing.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml
+  - template: Security/DAST.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml
+  - template: Security/Container-Scanning.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml
+  - template: Security/Dependency-Scanning.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml
+  - template: Security/License-Management.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/License-Management.gitlab-ci.yml
+  - template: Security/SAST.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml

spec/lib/gitlab/ci/pipeline/chain/config/content_spec.rb

@@ -40,7 +40,7 @@ describe Gitlab::Ci::Pipeline::Chain::Config::Content do
           subject.perform!
 
           expect(pipeline.config_source).to eq 'auto_devops_source'
-          template = Gitlab::Template::GitlabCiYmlTemplate.find('Auto-DevOps')
+          template = Gitlab::Template::GitlabCiYmlTemplate.find('Beta/Auto-DevOps')
           expect(command.config_content).to eq(template.content)
         end
       end
@@ -52,7 +52,7 @@ describe Gitlab::Ci::Pipeline::Chain::Config::Content do
           subject.perform!
 
           expect(pipeline.config_source).to eq 'auto_devops_source'
-          template = Gitlab::Template::GitlabCiYmlTemplate.find('Auto-DevOps')
+          template = Gitlab::Template::GitlabCiYmlTemplate.find('Beta/Auto-DevOps')
           expect(command.config_content).to eq(template.content)
         end
       end
@@ -82,12 +82,32 @@ describe Gitlab::Ci::Pipeline::Chain::Config::Content do
           expect(project).to receive(:auto_devops_enabled?).and_return(true)
         end
 
-        it 'returns the content of AutoDevops template' do
-          subject.perform!
+        context 'when beta is enabled' do
+          before do
+            stub_feature_flags(auto_devops_beta: true)
+          end
 
-          expect(pipeline.config_source).to eq 'auto_devops_source'
-          template = Gitlab::Template::GitlabCiYmlTemplate.find('Auto-DevOps')
-          expect(command.config_content).to eq(template.content)
+          it 'returns the content of AutoDevops template' do
+            subject.perform!
+
+            expect(pipeline.config_source).to eq 'auto_devops_source'
+            template = Gitlab::Template::GitlabCiYmlTemplate.find('Beta/Auto-DevOps')
+            expect(command.config_content).to eq(template.content)
+          end
+        end
+
+        context 'when beta is disabled' do
+          before do
+            stub_feature_flags(auto_devops_beta: false)
+          end
+
+          it 'returns the content of AutoDevops template' do
+            subject.perform!
+
+            expect(pipeline.config_source).to eq 'auto_devops_source'
+            template = Gitlab::Template::GitlabCiYmlTemplate.find('Auto-DevOps')
+            expect(command.config_content).to eq(template.content)
+          end
         end
       end
 
@@ -190,15 +210,38 @@ describe Gitlab::Ci::Pipeline::Chain::Config::Content do
         expect(project).to receive(:auto_devops_enabled?).and_return(true)
       end
 
-      it 'builds root config including the auto-devops template' do
-        subject.perform!
+      context 'when beta is enabled' do
+        before do
+          stub_feature_flags(auto_devops_beta: true)
+        end
 
-        expect(pipeline.config_source).to eq 'auto_devops_source'
-        expect(command.config_content).to eq(<<~EOY)
-          ---
-          include:
-          - template: Auto-DevOps.gitlab-ci.yml
-        EOY
+        it 'builds root config including the auto-devops template' do
+          subject.perform!
+
+          expect(pipeline.config_source).to eq 'auto_devops_source'
+          expect(command.config_content).to eq(<<~EOY)
+            ---
+            include:
+            - template: Beta/Auto-DevOps.gitlab-ci.yml
+          EOY
+        end
+      end
+
+      context 'when beta is disabled' do
+        before do
+          stub_feature_flags(auto_devops_beta: false)
+        end
+
+        it 'builds root config including the auto-devops template' do
+          subject.perform!
+
+          expect(pipeline.config_source).to eq 'auto_devops_source'
+          expect(command.config_content).to eq(<<~EOY)
+            ---
+            include:
+            - template: Auto-DevOps.gitlab-ci.yml
+          EOY
+        end
       end
     end
 

spec/lib/gitlab/ci/templates/auto_devops_gitlab_ci_yaml_spec.rb

@@ -9,7 +9,7 @@ describe 'Auto-DevOps.gitlab-ci.yml' do
     let(:user) { create(:admin) }
     let(:default_branch) { 'master' }
     let(:pipeline_branch) { default_branch }
-    let(:project) { create(:project, :custom_repo, files: { 'README.md' => '' }) }
+    let(:project) { create(:project, :auto_devops, :custom_repo, files: { 'README.md' => '' }) }
     let(:service) { Ci::CreatePipelineService.new(project, user, ref: pipeline_branch ) }
     let(:pipeline) { service.execute!(:push) }
     let(:build_names) { pipeline.builds.pluck(:name) }
@@ -107,4 +107,52 @@ describe 'Auto-DevOps.gitlab-ci.yml' do
       end
     end
   end
+
+  describe 'build-pack detection' do
+    using RSpec::Parameterized::TableSyntax
+
+    where(:case_name, :files, :variables, :include_build_names, :not_include_build_names) do
+      'No match'        | { 'README.md' => '' }                   | {}                                          | %w()           | %w(build test)
+      'Buildpack'       | { 'README.md' => '' }                   | { 'BUILDPACK_URL' => 'http://example.com' } | %w(build test) | %w()
+      'Explicit set'    | { 'README.md' => '' }                   | { 'AUTO_DEVOPS_EXPLICITLY_ENABLED' => '1' } | %w(build test) | %w()
+      'Explicit unset'  | { 'README.md' => '' }                   | { 'AUTO_DEVOPS_EXPLICITLY_ENABLED' => '0' } | %w()           | %w(build test)
+      'Dockerfile'      | { 'Dockerfile' => '' }                  | {}                                          | %w(build test) | %w()
+      'Clojure'         | { 'project.clj' => '' }                 | {}                                          | %w(build test) | %w()
+      'Go modules'      | { 'go.mod' => '' }                      | {}                                          | %w(build test) | %w()
+      'Go gb'           | { 'src/gitlab.com/gopackage.go' => '' } | {}                                          | %w(build test) | %w()
+      'Gradle'          | { 'gradlew' => '' }                     | {}                                          | %w(build test) | %w()
+      'Java'            | { 'pom.xml' => '' }                     | {}                                          | %w(build test) | %w()
+      'Multi-buildpack' | { '.buildpacks' => '' }                 | {}                                          | %w(build test) | %w()
+      'NodeJS'          | { 'package.json' => '' }                | {}                                          | %w(build test) | %w()
+      'PHP'             | { 'composer.json' => '' }               | {}                                          | %w(build test) | %w()
+      'Play'            | { 'conf/application.conf' => '' }       | {}                                          | %w(build test) | %w()
+      'Python'          | { 'Pipfile' => '' }                     | {}                                          | %w(build test) | %w()
+      'Ruby'            | { 'Gemfile' => '' }                     | {}                                          | %w(build test) | %w()
+      'Scala'           | { 'build.sbt' => '' }                   | {}                                          | %w(build test) | %w()
+      'Static'          | { '.static' => '' }                     | {}                                          | %w(build test) | %w()
+    end
+
+    with_them do
+      subject(:template) { Gitlab::Template::GitlabCiYmlTemplate.find('Beta/Auto-DevOps') }
+
+      let(:user) { create(:admin) }
+      let(:project) { create(:project, :custom_repo, files: files) }
+      let(:service) { Ci::CreatePipelineService.new(project, user, ref: 'master' ) }
+      let(:pipeline) { service.execute(:push) }
+      let(:build_names) { pipeline.builds.pluck(:name) }
+
+      before do
+        stub_ci_pipeline_yaml_file(template.content)
+        allow_any_instance_of(Ci::BuildScheduleWorker).to receive(:perform).and_return(true)
+        variables.each do |(key, value)|
+          create(:ci_variable, project: project, key: key, value: value)
+        end
+      end
+
+      it 'creates a pipeline with the expected jobs' do
+        expect(build_names).to include(*include_build_names)
+        expect(build_names).not_to include(*not_include_build_names)
+      end
+    end
+  end
 end

spec/services/ci/create_pipeline_service_spec.rb

@@ -493,6 +493,7 @@ describe Ci::CreatePipelineService do
         before do
           stub_ci_pipeline_yaml_file(nil)
           allow_any_instance_of(Project).to receive(:auto_devops_enabled?).and_return(true)
+          create(:project_auto_devops, project: project)
         end
 
         it 'pull it from Auto-DevOps' do

spec/services/external_pull_requests/create_pipeline_service_spec.rb

@@ -4,7 +4,7 @@ require 'spec_helper'
 
 describe ExternalPullRequests::CreatePipelineService do
   describe '#execute' do
-    set(:project) { create(:project, :repository) }
+    set(:project) { create(:project, :auto_devops, :repository) }
     set(:user) { create(:user) }
     let(:pull_request) { create(:external_pull_request, project: project) }