Merge branch '5306-more-custom-templates' into 'master'

Expand instance-level templates to cover .gitlab-ci.yml, .gitignore and Dockerfile

Closes #5306

See merge request gitlab-org/gitlab-ee!7000

app/finders/template_finder.rb

+class TemplateFinder
+  prepend ::EE::TemplateFinder
+
+  VENDORED_TEMPLATES = {
+    dockerfiles: ::Gitlab::Template::DockerfileTemplate,
+    gitignores: ::Gitlab::Template::GitignoreTemplate,
+    gitlab_ci_ymls: ::Gitlab::Template::GitlabCiYmlTemplate
+  }.freeze
+
+  class << self
+    def build(type, params = {})
+      if type == :licenses
+        LicenseTemplateFinder.new(params)
+      else
+        new(type, params)
+      end
+    end
+  end
+
+  attr_reader :type, :params
+
+  attr_reader :vendored_templates
+  private :vendored_templates
+
+  def initialize(type, params = {})
+    @type = type
+    @params = params
+
+    @vendored_templates = VENDORED_TEMPLATES.fetch(type)
+  end
+
+  def execute
+    if params[:name]
+      vendored_templates.find(params[:name])
+    else
+      vendored_templates.all
+    end
+  end
+end

app/helpers/blob_helper.rb

@@ -158,32 +158,35 @@ module BlobHelper
   end
 
   def licenses_for_select
-    return @licenses_for_select if defined?(@licenses_for_select)
-
-    grouped_licenses = LicenseTemplateFinder.new.execute.group_by(&:category)
-    categories = grouped_licenses.keys
-
-    @licenses_for_select = categories.each_with_object({}) do |category, hash|
-      hash[category] = grouped_licenses[category].map do |license|
-        { name: license.name, id: license.id }
-      end
-    end
+    @licenses_for_select ||= template_dropdown_names(TemplateFinder.build(:licenses).execute)
   end
 
   def ref_project
     @ref_project ||= @target_project || @project
   end
 
+  def template_dropdown_names(items)
+    grouped = items.group_by(&:category)
+    categories = grouped.keys
+
+    categories.each_with_object({}) do |category, hash|
+      hash[category] = grouped[category].map do |item|
+        { name: item.name, id: item.id }
+      end
+    end
+  end
+  private :template_dropdown_names
+
   def gitignore_names
-    @gitignore_names ||= Gitlab::Template::GitignoreTemplate.dropdown_names
+    @gitignore_names ||= template_dropdown_names(TemplateFinder.build(:gitignores).execute)
   end
 
   def gitlab_ci_ymls
-    @gitlab_ci_ymls ||= Gitlab::Template::GitlabCiYmlTemplate.dropdown_names(params[:context])
+    @gitlab_ci_ymls ||= template_dropdown_names(TemplateFinder.build(:gitlab_ci_ymls).execute)
   end
 
   def dockerfile_names
-    @dockerfile_names ||= Gitlab::Template::DockerfileTemplate.dropdown_names
+    @dockerfile_names ||= template_dropdown_names(TemplateFinder.build(:dockerfiles).execute)
   end
 
   def blob_editor_paths

doc/user/admin_area/settings/instance_template_repository.md

@@ -26,12 +26,28 @@ Templates must be added to a specific subdirectory in the repository,
 corresponding to the kind of template. They must also have the correct extension
 for the template type.
 
-Currently, only custom license templates are supported. This must go in the
-`LICENSE/` subdirectory, and must have `.txt` file extensions. So, the hierarchy
+Currently, the following types of custom template are supported:
+
+* `Dockerfile`: `Dockerfile` directory, `.dockerfile` extension
+* `.gitignore`: `gitignore` directory, `.gitignore` extension
+* `.gitlab-ci.yml`: `gitlab-ci` directory, `.yml` extension
+* `LICENSE`: `LICENSE` directory, `.txt` extension
+
+Each template must go in its respective subdirectory and have the correct
+extension. So, the hierarchy
 should look like this:
 
 ```text
 |-- README.md
+|-- Dockerfile
+    |-- custom_dockerfile.dockerfile
+    |-- another_dockerfile.dockerfile
+|-- gitignore
+    |-- custom_gitignore.gitignore
+    |-- another_gitignore.gitignore
+|-- gitlab-ci
+    |-- custom_gitlab-ci.yml
+    |-- another_gitlab-ci.yml
 |-- LICENSE
     |-- custom_license.txt
     |-- another_license.txt

ee/app/finders/ee/license_template_finder.rb

@@ -17,7 +17,7 @@ module EE
         )
       end
 
-      super + extra
+      extra.push(*super)
     end
 
     private
@@ -29,7 +29,7 @@ module EE
     end
 
     def custom_licenses
-      ::Gitlab::Template::LicenseTemplate.all(template_project)
+      ::Gitlab::Template::CustomLicenseTemplate.all(template_project)
     end
 
     def template_project

ee/app/finders/ee/template_finder.rb

+module EE
+  module TemplateFinder
+    include ::Gitlab::Utils::StrongMemoize
+    extend ::Gitlab::Utils::Override
+
+    CUSTOM_TEMPLATES = {
+      dockerfiles: ::Gitlab::Template::CustomDockerfileTemplate,
+      gitignores: ::Gitlab::Template::CustomGitignoreTemplate,
+      gitlab_ci_ymls: ::Gitlab::Template::CustomGitlabCiYmlTemplate
+    }.freeze
+
+    attr_reader :custom_templates
+    private :custom_templates
+
+    def initialize(type, *args, &blk)
+      super
+
+      @custom_templates = CUSTOM_TEMPLATES.fetch(type)
+    end
+
+    override :execute
+    def execute
+      return super unless custom_templates?
+
+      if params[:name]
+        find_custom_template || super
+      else
+        find_custom_templates + super
+      end
+    end
+
+    private
+
+    def find_custom_template
+      custom_templates.find(params[:name], template_project)
+    rescue ::Gitlab::Template::Finders::RepoTemplateFinder::FileNotFoundError
+      nil
+    end
+
+    def find_custom_templates
+      custom_templates.all(template_project)
+    end
+
+    def custom_templates?
+      ::License.feature_available?(:custom_file_templates) && template_project.present?
+    end
+
+    def template_project
+      strong_memoize(:template_project) { ::Gitlab::CurrentSettings.file_template_project }
+    end
+  end
+end

ee/changelogs/unreleased/5306-more-custom-templates.yml

+---
+title: Introduce custom instance-level templates for Dockerfile, .gitignore, and .gitlab-ci.yml files
+merge_request: 7000
+author:
+type: added

ee/lib/gitlab/template/custom_dockerfile_template.rb

+module Gitlab
+  module Template
+    class CustomDockerfileTemplate < CustomTemplate
+      class << self
+        def extension
+          '.dockerfile'
+        end
+
+        def base_dir
+          'Dockerfile/'
+        end
+      end
+    end
+  end
+end

ee/lib/gitlab/template/custom_gitignore_template.rb

+module Gitlab
+  module Template
+    class CustomGitignoreTemplate < CustomTemplate
+      class << self
+        def extension
+          '.gitignore'
+        end
+
+        def base_dir
+          'gitignore/'
+        end
+      end
+    end
+  end
+end

ee/lib/gitlab/template/custom_gitlab_ci_yml_template.rb

+module Gitlab
+  module Template
+    class CustomGitlabCiYmlTemplate < CustomTemplate
+      class << self
+        def extension
+          '.yml'
+        end
+
+        def base_dir
+          'gitlab-ci/'
+        end
+      end
+    end
+  end
+end

ee/lib/gitlab/template/custom_license_template.rb

+module Gitlab
+  module Template
+    class CustomLicenseTemplate < CustomTemplate
+      class << self
+        def extension
+          '.txt'
+        end
+
+        def base_dir
+          'LICENSE/'
+        end
+      end
+    end
+  end
+end

ee/lib/gitlab/template/license_template.rb → ee/lib/gitlab/template/custom_template.rb

 module Gitlab
   module Template
-    class LicenseTemplate < BaseTemplate
+    class CustomTemplate < BaseTemplate
       class << self
-        def extension
-          '.txt'
-        end
-
-        def base_dir
-          'LICENSE/'
+        def categories
+          { 'Custom' => '' }
         end
 
         def finder(project)

ee/spec/finders/license_template_finder_spec.rb

@@ -11,7 +11,7 @@ describe LicenseTemplateFinder do
 
     before do
       stub_ee_application_setting(file_template_project: project)
-      allow(Gitlab::Template::LicenseTemplate)
+      allow(Gitlab::Template::CustomLicenseTemplate)
         .to receive(:all)
         .with(project)
         .and_return([OpenStruct.new(name: "custom template")])

ee/spec/finders/template_finder_spec.rb

+require 'spec_helper'
+
+describe TemplateFinder do
+  using RSpec::Parameterized::TableSyntax
+
+  files = {
+    'Dockerfile/custom_dockerfile.dockerfile' => 'Custom Dockerfile',
+    'gitignore/custom_gitignore.gitignore'    => 'Custom .gitignore',
+    'gitlab-ci/custom_gitlab_ci.yml'          => 'Custom gitlab-ci.yml'
+  }
+
+  set(:project) { create(:project, :custom_repo, files: files) }
+
+  describe '#execute' do
+    before do
+      stub_licensed_features(custom_file_templates: true)
+      stub_ee_application_setting(file_template_project: project)
+    end
+
+    where(:type, :custom_name, :vendored_name) do
+      :dockerfiles    | 'custom_dockerfile' | 'Binary'
+      :gitignores     | 'custom_gitignore'  | 'Actionscript'
+      :gitlab_ci_ymls | 'custom_gitlab_ci'  | 'Android'
+    end
+
+    with_them do
+      subject(:result) { described_class.new(type, params).execute }
+
+      context 'specifying name' do
+        let(:params) { { name: custom_name } }
+
+        it { is_expected.to have_attributes(name: custom_name) }
+
+        context 'feature is disabled' do
+          before do
+            stub_licensed_features(custom_file_templates: false)
+          end
+
+          it { is_expected.to be_nil }
+        end
+      end
+
+      context 'not specifying name' do
+        let(:params) { {} }
+
+        it { is_expected.to include(have_attributes(name: custom_name)) }
+        it { is_expected.to include(have_attributes(name: vendored_name)) }
+
+        context 'feature is disabled' do
+          before do
+            stub_licensed_features(custom_file_templates: false)
+          end
+
+          it { is_expected.not_to include(have_attributes(name: custom_name)) }
+          it { is_expected.to include(have_attributes(name: vendored_name)) }
+        end
+      end
+    end
+  end
+end

ee/spec/helpers/ee/blob_helper_spec.rb

@@ -17,7 +17,7 @@ describe BlobHelper do
       stub_licensed_features(custom_file_templates: true)
       stub_ee_application_setting(file_template_project: project)
 
-      expect(Gitlab::Template::LicenseTemplate)
+      expect(Gitlab::Template::CustomLicenseTemplate)
         .to receive(:all)
         .with(project)
         .and_return([OpenStruct.new(name: "name")])

ee/spec/lib/gitlab/template/custom_templates_spec.rb

+require 'spec_helper'
+
+describe "Custom file template classes" do
+  files = {
+    'Dockerfile/foo.dockerfile' => 'CustomDockerfileTemplate Foo',
+    'Dockerfile/bar.dockerfile' => 'CustomDockerfileTemplate Bar',
+    'Dockerfile/bad.xyz'        => 'CustomDockerfileTemplate Bad',
+
+    'gitignore/foo.gitignore' => 'CustomGitignoreTemplate Foo',
+    'gitignore/bar.gitignore' => 'CustomGitignoreTemplate Bar',
+    'gitignore/bad.xyz'       => 'CustomGitignoreTemplate Bad',
+
+    'gitlab-ci/foo.yml' => 'CustomGitlabCiYmlTemplate Foo',
+    'gitlab-ci/bar.yml' => 'CustomGitlabCiYmlTemplate Bar',
+    'gitlab-ci/bad.xyz' => 'CustomGitlabCiYmlTemplate Bad',
+
+    'LICENSE/foo.txt' => 'CustomLicenseTemplate Foo',
+    'LICENSE/bar.txt' => 'CustomLicenseTemplate Bar',
+    'LICENSE/bad.xyz' => 'CustomLicenseTemplate Bad',
+
+    'Dockerfile/category/baz.txt' => 'CustomDockerfileTemplate category baz',
+    'gitignore/category/baz.txt'  => 'CustomGitignoreTemplate category baz',
+    'gitlab-ci/category/baz.yml'  => 'CustomGitlabCiYmlTemplate category baz',
+    'LICENSE/category/baz.txt'    => 'CustomLicenseTemplate category baz'
+  }
+
+  let(:project) { create(:project, :custom_repo, files: files) }
+
+  [
+    ::Gitlab::Template::CustomDockerfileTemplate,
+    ::Gitlab::Template::CustomGitignoreTemplate,
+    ::Gitlab::Template::CustomGitlabCiYmlTemplate,
+    ::Gitlab::Template::CustomLicenseTemplate
+  ].each do |template_class|
+    describe template_class do
+      let(:name) { template_class.name.demodulize }
+
+      describe '.all' do
+        it 'returns all valid templates' do
+          found = described_class.all(project)
+
+          aggregate_failures do
+            expect(found.map(&:name)).to contain_exactly('foo', 'bar')
+            expect(found.map(&:category).uniq).to contain_exactly('Custom')
+          end
+        end
+      end
+
+      describe '.find' do
+        let(:not_found_error) { ::Gitlab::Template::Finders::RepoTemplateFinder::FileNotFoundError }
+
+        it 'finds a valid template' do
+          found = described_class.find('foo', project)
+
+          expect(found.name).to eq('foo')
+          expect(found.content).to eq("#{name} Foo")
+        end
+
+        it 'sets the category correctly' do
+          pending("#{template_class}.find does not set category correctly")
+          found = described_class.find('foo', project)
+
+          expect(found.category).to eq('Custom')
+        end
+
+        it 'does not find a template with the wrong extension' do
+          expect { described_class.find('bad', project) }.to raise_error(not_found_error)
+        end
+
+        it 'does not find a template in a subdirectory' do
+          expect { described_class.find('baz', project) }.to raise_error(not_found_error)
+        end
+      end
+    end
+  end
+end

ee/spec/requests/api/templates_spec.rb

+require 'spec_helper'
+
+describe API::Templates do
+  files = {
+    'Dockerfile/custom.dockerfile' => 'Custom dockerfiles',
+    'gitignore/custom.gitignore'   => 'Custom gitignores',
+    'gitlab-ci/custom.yml'         => 'Custom gitlab_ci_ymls',
+    'LICENSE/custom.txt'           => 'Custom licenses'
+  }
+
+  set(:project) { create(:project, :custom_repo, files: files) }
+
+  before do
+    stub_ee_application_setting(file_template_project: project)
+  end
+
+  [
+    :dockerfiles,
+    :gitignores,
+    :gitlab_ci_ymls,
+    :licenses
+  ].each do |type|
+    describe "GET /templates/#{type}" do
+      it 'includes the custom template in the response' do
+        stub_licensed_features(custom_file_templates: true)
+        get api("/templates/#{type}")
+
+        expect(response).to have_gitlab_http_status(200)
+        expect(json_response).to satisfy_one { |template| template['name'] == 'custom' }
+      end
+
+      it 'excludes the custom template when the feature is disabled' do
+        stub_licensed_features(custom_file_templates: false)
+        get api("/templates/#{type}")
+
+        expect(response).to have_gitlab_http_status(200)
+        expect(json_response).to satisfy_none { |template| template['name'] == 'custom' }
+      end
+    end
+
+    describe "GET /templates/#{type}/custom" do
+      it 'returns the custom template' do
+        stub_licensed_features(custom_file_templates: true)
+        get api("/templates/#{type}/custom")
+
+        expect(response).to have_gitlab_http_status(200)
+        expect(json_response['name']).to eq('custom')
+        expect(json_response['content']).to eq("Custom #{type}")
+      end
+
+      it 'returns 404 when the feature is disabled' do
+        stub_licensed_features(custom_file_templates: false)
+        get api("/templates/#{type}/custom")
+
+        expect(response).to have_gitlab_http_status(404)
+      end
+    end
+  end
+end

lib/api/templates.rb

@@ -4,15 +4,12 @@ module API
 
     GLOBAL_TEMPLATE_TYPES = {
       gitignores: {
-        klass: Gitlab::Template::GitignoreTemplate,
         gitlab_version: 8.8
       },
       gitlab_ci_ymls: {
-        klass: Gitlab::Template::GitlabCiYmlTemplate,
         gitlab_version: 8.9
       },
       dockerfiles: {
-        klass: Gitlab::Template::DockerfileTemplate,
         gitlab_version: 8.15
       }
     }.freeze
@@ -36,7 +33,7 @@ module API
       popular = declared(params)[:popular]
       popular = to_boolean(popular) if popular.present?
 
-      templates = LicenseTemplateFinder.new(popular: popular).execute
+      templates = TemplateFinder.build(:licenses, popular: popular).execute
 
       present paginate(::Kaminari.paginate_array(templates)), with: ::API::Entities::License
     end
@@ -49,7 +46,7 @@ module API
       requires :name, type: String, desc: 'The name of the template'
     end
     get "templates/licenses/:name", requirements: { name: /[\w\.-]+/ } do
-      templates = LicenseTemplateFinder.new.execute
+      templates = TemplateFinder.build(:licenses).execute
       template = templates.find { |template| template.key == params[:name] }
 
       not_found!('License') unless template.present?
@@ -63,7 +60,6 @@ module API
     end
 
     GLOBAL_TEMPLATE_TYPES.each do |template_type, properties|
-      klass = properties[:klass]
       gitlab_version = properties[:gitlab_version]
 
       desc 'Get the list of the available template' do
@@ -74,7 +70,7 @@ module API
         use :pagination
       end
       get "templates/#{template_type}" do
-        templates = ::Kaminari.paginate_array(klass.all)
+        templates = ::Kaminari.paginate_array(TemplateFinder.new(template_type).execute)
         present paginate(templates), with: Entities::TemplatesList
       end
 
@@ -86,7 +82,8 @@ module API
         requires :name, type: String, desc: 'The name of the template'
       end
       get "templates/#{template_type}/:name" do
-        new_template = klass.find(declared(params)[:name])
+        finder = TemplateFinder.build(template_type, name: declared(params)[:name])
+        new_template = finder.execute
 
         render_response(template_type, new_template)
       end

lib/gitlab/template/base_template.rb

 module Gitlab
   module Template
     class BaseTemplate
-      def initialize(path, project = nil)
+      attr_reader :category
+
+      def initialize(path, project = nil, category: nil)
         @path = path
+        @category = category
         @finder = self.class.finder(project)
       end
 
       def name
         File.basename(@path, self.class.extension)
       end
+      alias_method :id, :name
 
       def content
         @finder.read(@path)
@@ -62,7 +66,7 @@ module Gitlab
           directory = category_directory(category)
           files = finder(project).list_files_for(directory)
 
-          files.map { |f| new(f, project) }.sort
+          files.map { |f| new(f, project, category: category) }.sort
         end
 
         def category_directory(category)

spec/factories/projects.rb

@@ -168,6 +168,33 @@ FactoryBot.define do
       end
     end
 
+    # Build a custom repository by specifying a hash of `filename => content` in
+    # the transient `files` attribute. Each file will be created in its own
+    # commit, operating against the master branch. So, the following call:
+    #
+    #     create(:project, :custom_repo, files: { 'foo/a.txt' => 'foo', 'b.txt' => bar' })
+    #
+    # will create a repository containing two files, and two commits, in master
+    trait :custom_repo do
+      transient do
+        files {}
+      end
+
+      after :create do |project, evaluator|
+        raise "Failed to create repository!" unless project.create_repository
+
+        evaluator.files.each do |filename, content|
+          project.repository.create_file(
+            project.creator,
+            filename,
+            content,
+            message: "Automatically created file #{filename}",
+            branch_name: 'master'
+          )
+        end
+      end
+    end
+
     # Test repository - https://gitlab.com/gitlab-org/gitlab-test
     trait :repository do
       test_repo

spec/finders/template_finder_spec.rb

+require 'spec_helper'
+
+describe TemplateFinder do
+  using RSpec::Parameterized::TableSyntax
+
+  describe '#build' do
+    where(:type, :expected_class) do
+      :dockerfiles    | described_class
+      :gitignores     | described_class
+      :gitlab_ci_ymls | described_class
+      :licenses       | ::LicenseTemplateFinder
+    end
+
+    with_them do
+      subject { described_class.build(type) }
+
+      it { is_expected.to be_a(expected_class) }
+    end
+  end
+
+  describe '#execute' do
+    where(:type, :vendored_name) do
+      :dockerfiles    | 'Binary'
+      :gitignores     | 'Actionscript'
+      :gitlab_ci_ymls | 'Android'
+    end
+
+    with_them do
+      it 'returns all vendored templates when no name is specified' do
+        result = described_class.new(type).execute
+
+        expect(result).to include(have_attributes(name: vendored_name))
+      end
+
+      it 'returns only the specified vendored template when a name is specified' do
+        result = described_class.new(type, name: vendored_name).execute
+
+        expect(result).to have_attributes(name: vendored_name)
+      end
+
+      it 'returns nil when an unknown name is specified' do
+        result = described_class.new(type, name: 'unknown').execute
+
+        expect(result).to be_nil
+      end
+    end
+  end
+end