Merge branch '32815--Add-Custom-CI-Config-Path' into 'master'

Resolve "Project option to allow customizing CI/CD config path"

Closes #32815 and #33130

See merge request !12509

app/controllers/projects/pipelines_settings_controller.rb

@@ -23,7 +23,7 @@ class Projects::PipelinesSettingsController < Projects::ApplicationController
   def update_params
     params.require(:project).permit(
       :runners_token, :builds_enabled, :build_allow_git_fetch, :build_timeout_in_minutes, :build_coverage_regex,
-      :public_builds, :auto_cancel_pending_pipelines
+      :public_builds, :auto_cancel_pending_pipelines, :ci_config_path
     )
   end
 end

app/models/ci/pipeline.rb

@@ -326,10 +326,24 @@ module Ci
       end
     end
 
+    def ci_yaml_file_path
+      if project.ci_config_path.blank?
+        '.gitlab-ci.yml'
+      else
+        project.ci_config_path
+      end
+    end
+
     def ci_yaml_file
       return @ci_yaml_file if defined?(@ci_yaml_file)
 
-      @ci_yaml_file = project.repository.gitlab_ci_yml_for(sha) rescue nil
+      @ci_yaml_file = begin
+        project.repository.gitlab_ci_yml_for(sha, ci_yaml_file_path)
+      rescue Rugged::ReferenceError, GRPC::NotFound, GRPC::Internal
+        self.yaml_errors =
+          "Failed to load CI/CD config file at #{ci_yaml_file_path}"
+        nil
+      end
     end
 
     def has_yaml_errors?
@@ -377,7 +391,8 @@ module Ci
 
     def predefined_variables
       [
-        { key: 'CI_PIPELINE_ID', value: id.to_s, public: true }
+        { key: 'CI_PIPELINE_ID', value: id.to_s, public: true },
+        { key: 'CI_CONFIG_PATH', value: ci_yaml_file_path, public: true }
       ]
     end
 

app/models/project.rb

@@ -186,6 +186,11 @@ class Project < ActiveRecord::Base
   # Validations
   validates :creator, presence: true, on: :create
   validates :description, length: { maximum: 2000 }, allow_blank: true
+  validates :ci_config_path,
+    format: { without: /\.{2}/,
+              message: 'cannot include directory traversal.' },
+    length: { maximum: 255 },
+    allow_blank: true
   validates :name,
     presence: true,
     length: { maximum: 255 },
@@ -521,6 +526,11 @@ class Project < ActiveRecord::Base
     import_data&.destroy
   end
 
+  def ci_config_path=(value)
+    # Strip all leading slashes so that //foo -> foo
+    super(value&.sub(%r{\A/+}, '')&.delete("\0"))
+  end
+
   def import_url=(value)
     return super(value) unless Gitlab::UrlSanitizer.valid?(value)
 
@@ -1015,7 +1025,8 @@ class Project < ActiveRecord::Base
       namespace: namespace.name,
       visibility_level: visibility_level,
       path_with_namespace: path_with_namespace,
-      default_branch: default_branch
+      default_branch: default_branch,
+      ci_config_path: ci_config_path
     }
 
     # Backward compatibility

app/models/repository.rb

@@ -931,7 +931,7 @@ class Repository
 
   def is_ancestor?(ancestor_id, descendant_id)
     return false if ancestor_id.nil? || descendant_id.nil?
-    
+
     Gitlab::GitalyClient.migrate(:is_ancestor) do |is_enabled|
       if is_enabled
         raw_repository.is_ancestor?(ancestor_id, descendant_id)
@@ -1078,8 +1078,8 @@ class Repository
     blob_data_at(sha, '.gitlab/route-map.yml')
   end
 
-  def gitlab_ci_yml_for(sha)
-    blob_data_at(sha, '.gitlab-ci.yml')
+  def gitlab_ci_yml_for(sha, path = '.gitlab-ci.yml')
+    blob_data_at(sha, path)
   end
 
   private

app/services/ci/create_pipeline_service.rb

@@ -33,7 +33,7 @@ module Ci
 
       unless pipeline.config_processor
         unless pipeline.ci_yaml_file
-          return error('Missing .gitlab-ci.yml file')
+          return error("Missing #{pipeline.ci_yaml_file_path} file")
         end
         return error(pipeline.yaml_errors, save: save_on_errors)
       end

app/views/projects/pipelines_settings/_show.html.haml

@@ -45,6 +45,14 @@
             Per job in minutes. If a job passes this threshold, it will be marked as failed
             = link_to icon('question-circle'), help_page_path('user/project/pipelines/settings', anchor: 'timeout'), target: '_blank'
 
+        %hr
+        .form-group
+          = f.label :ci_config_path, 'Custom CI config path', class: 'label-light'
+          = f.text_field :ci_config_path, class: 'form-control', placeholder: '.gitlab-ci.yml'
+          %p.help-block
+            The path to CI config file. Defaults to <code>.gitlab-ci.yml</code>
+            = link_to icon('question-circle'), help_page_path('user/project/pipelines/settings', anchor: 'custom-ci-config-path'), target: '_blank'
+
         %hr
         .form-group
           .checkbox

changelogs/unreleased/32815--Add-Custom-CI-Config-Path.yml

+---
+title: Allow customize CI config path
+merge_request: 12509
+author: Keith Pope

db/migrate/20160804142904_add_ci_config_file_to_project.rb

+class AddCiConfigFileToProject < ActiveRecord::Migration
+  DOWNTIME = false
+
+  def change
+    add_column :projects, :ci_config_path, :string
+  end
+
+  def down
+    remove_column :projects, :ci_config_path
+  end
+end

db/schema.rb

@@ -1086,6 +1086,7 @@ ActiveRecord::Schema.define(version: 20170703102400) do
     t.string "import_jid"
     t.integer "cached_markdown_version"
     t.datetime "last_repository_updated_at"
+    t.string "ci_config_path"
   end
 
   add_index "projects", ["ci_id"], name: "index_projects_on_ci_id", using: :btree

doc/api/projects.md

@@ -336,7 +336,7 @@ Parameters:
 | `snippets_enabled` | boolean | no | Enable snippets for this project |
 | `container_registry_enabled` | boolean | no | Enable container registry for this project |
 | `shared_runners_enabled` | boolean | no | Enable shared runners for this project |
-| `visibility` | String | no | See [project visibility level](#project-visibility-level) |
+| `visibility` | string | no | See [project visibility level](#project-visibility-level) |
 | `import_url` | string | no | URL to import repository from |
 | `public_jobs` | boolean | no | If `true`, jobs can be viewed by non-project-members |
 | `only_allow_merge_if_pipeline_succeeds` | boolean | no | Set whether merge requests can only be merged with successful jobs |
@@ -346,6 +346,7 @@ Parameters:
 | `tag_list`    | array   | no       | The list of tags for a project; put array of tags, that should be finally assigned to a project |
 | `avatar`    | mixed   | no      | Image file for avatar of the project                |
 | `printing_merge_request_link_enabled` | boolean | no | Show link to create/view merge request when pushing from the command line |
+| `ci_config_path` | string | no | The path to CI config file |
 
 ### Create project for user
 
@@ -382,6 +383,7 @@ Parameters:
 | `tag_list`    | array   | no       | The list of tags for a project; put array of tags, that should be finally assigned to a project |
 | `avatar`    | mixed   | no      | Image file for avatar of the project                |
 | `printing_merge_request_link_enabled` | boolean | no | Show link to create/view merge request when pushing from the command line |
+| `ci_config_path` | string | no | The path to CI config file |
 
 ### Edit project
 
@@ -416,6 +418,7 @@ Parameters:
 | `request_access_enabled` | boolean | no | Allow users to request member access |
 | `tag_list`    | array   | no       | The list of tags for a project; put array of tags, that should be finally assigned to a project |
 | `avatar`    | mixed   | no      | Image file for avatar of the project                |
+| `ci_config_path` | string | no | The path to CI config file |
 
 ### Fork project
 

doc/ci/variables/README.md

@@ -40,6 +40,7 @@ future GitLab releases.**
 | **CI_COMMIT_REF_SLUG**          | 9.0    | all    | `$CI_COMMIT_REF_NAME` lowercased, shortened to 63 bytes, and with everything except `0-9` and `a-z` replaced with `-`. No leading / trailing `-`. Use in URLs, host names and domain names. |
 | **CI_COMMIT_SHA**               | 9.0    | all    | The commit revision for which project is built |
 | **CI_COMMIT_TAG**               | 9.0    | 0.5    | The commit tag name. Present only when building tags. |
+| **CI_CONFIG_PATH**              | 9.4    | 0.5    | The path to CI config file. Defaults to `.gitlab-ci.yml` |
 | **CI_DEBUG_TRACE**              | all    | 1.7    | Whether [debug tracing](#debug-tracing) is enabled |
 | **CI_ENVIRONMENT_NAME**         | 8.15   | all    | The name of the environment for this job |
 | **CI_ENVIRONMENT_SLUG**         | 8.15   | all    | A simplified version of the environment name, suitable for inclusion in DNS, URLs, Kubernetes labels, etc. |

doc/user/project/pipelines/settings.md

@@ -27,6 +27,22 @@ The default value is 60 minutes. Decrease the time limit if you want to impose
 a hard limit on your jobs' running time or increase it otherwise. In any case,
 if the job surpasses the threshold, it is marked as failed.
 
+## Custom CI config path
+
+>  - [Introduced][ce-12509] in GitLab 9.4.
+
+By default we look for the `.gitlab-ci.yml` file in the project's root
+directory. If you require a different location **within** the repository,
+you can set a custom filepath that will be used to lookup the config file,
+this filepath should be **relative** to the root.
+
+Here are some valid examples:
+
+> * .gitlab-ci.yml
+> * .my-custom-file.yml
+> * my/path/.gitlab-ci.yml
+> * my/path/.my-custom-file.yml
+
 ## Test coverage parsing
 
 If you use test coverage in your code, GitLab can capture its output in the
@@ -59,8 +75,8 @@ pipelines** checkbox and save the changes.
 
 > [Introduced][ce-9362] in GitLab 9.1.
 
-If you want to auto-cancel all pending non-HEAD pipelines on branch, when 
-new pipeline will be created (after your git push or manually from UI), 
+If you want to auto-cancel all pending non-HEAD pipelines on branch, when
+new pipeline will be created (after your git push or manually from UI),
 check **Auto-cancel pending pipelines** checkbox and save the changes.
 
 ## Badges
@@ -115,3 +131,4 @@ into your `README.md`:
 [var]: ../../../ci/yaml/README.md#git-strategy
 [coverage report]: #test-coverage-parsing
 [ce-9362]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/9362
+[ce-12509]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/12509

lib/api/entities.rb

@@ -112,6 +112,7 @@ module API
       expose :open_issues_count, if: lambda { |project, options| project.feature_available?(:issues, options[:current_user]) && project.default_issues_tracker? }
       expose :runners_token, if: lambda { |_project, options| options[:user_can_admin_project] }
       expose :public_builds, as: :public_jobs
+      expose :ci_config_path
       expose :shared_with_groups do |project, options|
         SharedGroup.represent(project.project_group_links.all, options)
       end

lib/api/projects.rb

@@ -10,6 +10,7 @@ module API
     helpers do
       params :optional_params_ce do
         optional :description, type: String, desc: 'The description of the project'
+        optional :ci_config_path, type: String, desc: 'The path to CI config file. Defaults to `.gitlab-ci.yml`'
         optional :issues_enabled, type: Boolean, desc: 'Flag indication if the issue tracker is enabled'
         optional :merge_requests_enabled, type: Boolean, desc: 'Flag indication if merge requests are enabled'
         optional :wiki_enabled, type: Boolean, desc: 'Flag indication if the wiki is enabled'

spec/lib/gitlab/import_export/safe_model_attributes.yml

@@ -383,6 +383,7 @@ Project:
 - printing_merge_request_link_enabled
 - build_allow_git_fetch
 - last_repository_updated_at
+- ci_config_path
 Author:
 - name
 ProjectFeature:

spec/models/ci/build_spec.rb

@@ -1183,6 +1183,7 @@ describe Ci::Build, :models do
         { key: 'CI_PROJECT_NAMESPACE', value: project.namespace.full_path, public: true },
         { key: 'CI_PROJECT_URL', value: project.web_url, public: true },
         { key: 'CI_PIPELINE_ID', value: pipeline.id.to_s, public: true },
+        { key: 'CI_CONFIG_PATH', value: pipeline.ci_yaml_file_path, public: true },
         { key: 'CI_REGISTRY_USER', value: 'gitlab-ci-token', public: true },
         { key: 'CI_REGISTRY_PASSWORD', value: build.token, public: false },
         { key: 'CI_REPOSITORY_URL', value: build.repo_url, public: false }
@@ -1473,6 +1474,16 @@ describe Ci::Build, :models do
       it { is_expected.to include(deployment_variable) }
     end
 
+    context 'when project has custom CI config path' do
+      let(:ci_config_path) { { key: 'CI_CONFIG_PATH', value: 'custom', public: true } }
+
+      before do
+        project.update(ci_config_path: 'custom')
+      end
+
+      it { is_expected.to include(ci_config_path) }
+    end
+
     context 'returns variables in valid order' do
       let(:build_pre_var) { { key: 'build', value: 'value' } }
       let(:project_pre_var) { { key: 'project', value: 'value' } }

spec/models/ci/pipeline_spec.rb

@@ -748,6 +748,39 @@ describe Ci::Pipeline, models: true do
     end
   end
 
+  describe '#ci_yaml_file_path' do
+    subject { pipeline.ci_yaml_file_path }
+
+    it 'returns the path from project' do
+      allow(pipeline.project).to receive(:ci_config_path) { 'custom/path' }
+
+      is_expected.to eq('custom/path')
+    end
+
+    it 'returns default when custom path is nil' do
+      allow(pipeline.project).to receive(:ci_config_path) { nil }
+
+      is_expected.to eq('.gitlab-ci.yml')
+    end
+
+    it 'returns default when custom path is empty' do
+      allow(pipeline.project).to receive(:ci_config_path) { '' }
+
+      is_expected.to eq('.gitlab-ci.yml')
+    end
+  end
+
+  describe '#ci_yaml_file' do
+    it 'reports error if the file is not found' do
+      allow(pipeline.project).to receive(:ci_config_path) { 'custom' }
+
+      pipeline.ci_yaml_file
+
+      expect(pipeline.yaml_errors)
+        .to eq('Failed to load CI/CD config file at custom')
+    end
+  end
+
   describe '#detailed_status' do
     subject { pipeline.detailed_status(user) }
 

spec/models/project_spec.rb

@@ -143,6 +143,10 @@ describe Project, models: true do
 
     it { is_expected.to validate_length_of(:description).is_at_most(2000) }
 
+    it { is_expected.to validate_length_of(:ci_config_path).is_at_most(255) }
+    it { is_expected.to allow_value('').for(:ci_config_path) }
+    it { is_expected.not_to allow_value('test/../foo').for(:ci_config_path) }
+
     it { is_expected.to validate_presence_of(:creator) }
 
     it { is_expected.to validate_presence_of(:namespace) }
@@ -1504,6 +1508,28 @@ describe Project, models: true do
     end
   end
 
+  describe '#ci_config_path=' do
+    let(:project) { create(:empty_project) }
+
+    it 'sets nil' do
+      project.update!(ci_config_path: nil)
+
+      expect(project.ci_config_path).to be_nil
+    end
+
+    it 'sets a string' do
+      project.update!(ci_config_path: 'foo/.gitlab_ci.yml')
+
+      expect(project.ci_config_path).to eq('foo/.gitlab_ci.yml')
+    end
+
+    it 'sets a string but removes all leading slashes and null characters' do
+      project.update!(ci_config_path: "///f\0oo/\0/.gitlab_ci.yml")
+
+      expect(project.ci_config_path).to eq('foo//.gitlab_ci.yml')
+    end
+  end
+
   describe 'Project import job' do
     let(:project) { create(:empty_project, import_url: generate(:url)) }
 

spec/requests/api/projects_spec.rb

@@ -347,7 +347,8 @@ describe API::Projects do
         wiki_enabled: false,
         only_allow_merge_if_pipeline_succeeds: false,
         request_access_enabled: true,
-        only_allow_merge_if_all_discussions_are_resolved: false
+        only_allow_merge_if_all_discussions_are_resolved: false,
+        ci_config_path: 'a/custom/path'
       })
 
       post api('/projects', user), project
@@ -653,6 +654,7 @@ describe API::Projects do
         expect(json_response['star_count']).to be_present
         expect(json_response['forks_count']).to be_present
         expect(json_response['public_jobs']).to be_present
+        expect(json_response['ci_config_path']).to be_nil
         expect(json_response['shared_with_groups']).to be_an Array
         expect(json_response['shared_with_groups'].length).to eq(1)
         expect(json_response['shared_with_groups'][0]['group_id']).to eq(group.id)

spec/support/cycle_analytics_helpers.rb

@@ -74,7 +74,9 @@ module CycleAnalyticsHelpers
 
   def dummy_pipeline
     @dummy_pipeline ||=
-      Ci::Pipeline.new(sha: project.repository.commit('master').sha)
+      Ci::Pipeline.new(
+        sha: project.repository.commit('master').sha,
+        project: project)
   end
 
   def new_dummy_job(environment)