Build barebones for ExternalFiles libraries

CE mirror of 30ca00f17034f654403ec7cb5dc370d1e5db8152

lib/gitlab/ci/config/entry/global.rb

@@ -33,11 +33,15 @@ module Gitlab
           entry :cache, Entry::Cache,
             description: 'Configure caching between build jobs.'
 
+          entry :includes, Entry::Includes,
+            description: 'External GitlLab Ci files'
+
           helpers :before_script, :image, :services, :after_script,
-                  :variables, :stages, :types, :cache, :jobs
+                  :variables, :stages, :types, :cache, :jobs, :includes
 
           def compose!(_deps = nil)
             super(self) do
+              append_external_files!
               compose_jobs!
               compose_deprecated_entries!
             end
@@ -45,6 +49,10 @@ module Gitlab
 
           private
 
+          def append_external_files!
+            return if includes_value.nil?
+          end
+
           def compose_jobs!
             factory = Entry::Factory.new(Entry::Jobs)
               .value(@config.except(*self.class.nodes.keys))

lib/gitlab/ci/config/entry/includes.rb

+module Gitlab
+  module Ci
+    class Config
+      module Entry
+        ##
+        # Entry that represents a Docker image.
+        #
+        class Includes < Node
+          include Validatable
+
+          validations do
+            validates :config, array_or_string: true, external_file: true, allow_nil: true
+          end
+
+          def value
+            Array(@config)
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/config/entry/validators.rb

@@ -60,6 +60,38 @@ module Gitlab
             end
           end
 
+          class ArrayOrStringValidator < ActiveModel::EachValidator
+            def validate_each(record, attribute, value)
+              unless value.is_a?(Array) || value.is_a?(String)
+                record.errors.add(attribute, 'should be an array or a string')
+              end
+            end
+          end
+
+          class ExternalFileValidator < ActiveModel::EachValidator
+            def validate_each(record, attribute, value)
+              if value.is_a?(Array)
+                value.each do |path|
+                  validate_external_file(path, record, attribute)
+                end
+              else
+                validate_external_file(value, record, attribute)
+              end
+            end
+
+            private
+
+            def validate_external_file(value, record, attribute)
+              unless valid_url?(value)
+                record.errors.add(attribute, 'should be a valid local or remote file')
+              end
+            end
+
+            def valid_url?(value)
+              Gitlab::UrlSanitizer.valid?(value) || File.exists?("#{Rails.root}/#{value}")
+            end
+          end
+
           class KeyValidator < ActiveModel::EachValidator
             include LegacyValidationHelpers
 

lib/gitlab/ci/external_files/external_file.rb

+require 'open-uri'
+
+module Gitlab
+  module Ci
+    module ExternalFiles
+      class ExternalFile
+
+        def initialize(value)
+          @value = value
+        end
+
+        def content
+          if remote_url?
+            open(value).read
+          else
+            File.read(base_path)
+          end
+        end
+
+        def valid?
+          remote_url? || File.exists?(base_path) 
+        end
+
+        private
+
+        attr_reader :value
+
+        def base_path
+          "#{Rails.root}/#{value}"
+        end
+
+        def remote_url?
+          ::Gitlab::UrlSanitizer.valid?(value)
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/external_files/mapper.rb

+module Gitlab
+  module Ci
+    module ExternalFiles
+      class Mapper
+
+        def self.fetch_paths(values)
+          paths = values.fetch(:includes, [])
+          normalize_paths(paths)
+        end
+
+        private
+
+        def self.normalize_paths(paths)
+          if paths.is_a?(String)
+            [build_external_file(paths)]
+          else
+            paths.map { |path| build_external_file(path) }
+          end
+        end
+
+        def self.build_external_file(path)
+          ::Gitlab::Ci::ExternalFiles::ExternalFile.new(path)
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/external_files/processor.rb

+module Gitlab
+  module Ci
+    module ExternalFiles
+      class Processor
+        ExternalFileError = Class.new(StandardError)
+
+        def initialize(values)
+          @values = values
+          @external_files = ::Gitlab::Ci::ExternalFiles::Mapper.fetch_paths(values)
+        end
+
+        def perform
+          return values if external_files.empty?
+
+          external_files.each do |external_file|
+            validate_external_file(external_file)
+            append_external_content(external_file)
+          end
+
+          remove_include_keyword
+        end
+
+        private
+
+        attr_reader :values, :external_files
+
+        def validate_external_file(external_file)
+          unless external_file.valid? 
+            raise ExternalFileError, 'External files should be a valid local or remote file'
+          end
+        end
+
+        def append_external_content(external_file)
+          external_values = ::Gitlab::Ci::Config::Loader.new(external_file.content).load!
+          @values.merge!(external_values)
+        end
+
+        def remove_include_keyword
+          values.delete(:includes)
+          values
+        end
+
+      end
+    end
+  end
+end

spec/fixtures/gitlab/ci/external_files/.gitlab-ci-template-1.yml

+before_script:
+  - apt-get update -qq && apt-get install -y -qq sqlite3 libsqlite3-dev nodejs
+  - ruby -v
+  - which ruby
+  - gem install bundler --no-ri --no-rdoc
+  - bundle install --jobs $(nproc)  "${FLAGS[@]}"
+
+rspec:
+  script:
+    - bundle exec rspec

spec/fixtures/gitlab/ci/external_files/.gitlab-ci-template-2.yml

+variables:
+  # AUTO_DEVOPS_DOMAIN is the application deployment domain and should be set as a variable at the group or project level.
+
+  AUTO_DEVOPS_DOMAIN: domain.example.com 
+  POSTGRES_USER: user
+  POSTGRES_PASSWORD: testing-password
+  POSTGRES_ENABLED: "true"
+  POSTGRES_DB: $CI_ENVIRONMENT_SLUG

spec/lib/gitlab/ci/config/entry/global_spec.rb

@@ -13,7 +13,7 @@ describe Gitlab::Ci::Config::Entry::Global do
         expect(described_class.nodes.keys)
           .to match_array(%i[before_script image services
                              after_script variables stages
-                             types cache])
+                             types cache includes])
       end
     end
   end
@@ -42,7 +42,7 @@ describe Gitlab::Ci::Config::Entry::Global do
         end
 
         it 'creates node object for each entry' do
-          expect(global.descendants.count).to eq 8
+          expect(global.descendants.count).to eq 9
         end
 
         it 'creates node object using valid class' do
@@ -189,7 +189,7 @@ describe Gitlab::Ci::Config::Entry::Global do
 
       describe '#nodes' do
         it 'instantizes all nodes' do
-          expect(global.descendants.count).to eq 8
+          expect(global.descendants.count).to eq 9
         end
 
         it 'contains unspecified nodes' do

spec/lib/gitlab/ci/config/entry/includes_spec.rb

+require 'rails_helper'
+
+describe Gitlab::Ci::Config::Entry::Includes do
+  let(:entry) { described_class.new(config) }
+
+  shared_examples 'valid external file' do
+    it 'should be valid' do
+      expect(entry).to be_valid
+    end
+
+    it 'should not return any error' do
+      expect(entry.errors).to be_empty
+    end
+  end
+
+  shared_examples 'invalid external file' do
+    it 'should not be valid' do
+      expect(entry).not_to be_valid
+    end
+
+    it 'should return an error' do
+      expect(entry.errors.first).to match(/should be a valid local or remote file/)
+    end
+  end
+
+  describe "#valid?" do
+    context 'with no external file given' do
+      let(:config) { nil }
+
+      it_behaves_like 'valid external file'
+    end
+
+    context 'with multiple external files' do
+      let(:config) { %w(https://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-1.yml https://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-2.yml) }
+
+      it_behaves_like 'valid external file'
+    end
+
+    context 'with just one external file' do
+      let(:config) { 'https://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-1.yml' } 
+
+      it_behaves_like 'valid external file'
+    end
+
+    context 'when they contain valid URLs' do
+      let(:config) { 'https://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-1.yml' } 
+
+      it_behaves_like 'valid external file'
+    end
+
+    context 'when they contain valid relative URLs' do
+      let(:config) { '/vendor/gitlab-ci-yml/Auto-DevOps.gitlab-ci.yml' } 
+
+      it_behaves_like 'valid external file'
+    end
+
+    context 'when they not contain valid URLs' do
+      let(:config) { 'not-valid://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-1.yml' }
+
+      it_behaves_like 'invalid external file'
+    end
+
+    context 'when they not contain valid relative URLs' do
+      let(:config) { '/vendor/gitlab-ci-yml/non-existent-file.yml' }
+
+      it_behaves_like 'invalid external file' 
+    end
+  end
+
+  describe "#value" do
+    context 'with multiple external files' do
+      let(:config) { %w(https://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-1.yml https://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-2.yml) }
+      it 'should return an array' do
+        expect(entry.value).to be_an(Array)
+        expect(entry.value.count).to eq(2)
+      end
+    end
+
+    context 'with just one external file' do
+      let(:config) { 'https://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-1.yml' }
+
+      it 'should return an array' do
+        expect(entry.value).to be_an(Array)
+        expect(entry.value.count).to eq(1)
+      end
+    end
+
+    context 'with no external file given' do
+      let(:config) { nil }
+
+      it 'should return an empty array' do
+        expect(entry.value).to be_an(Array)
+        expect(entry.value).to be_empty
+      end
+    end
+  end
+end

spec/lib/gitlab/ci/external_files/external_file_spec.rb

+require 'rails_helper'
+
+describe Gitlab::Ci::ExternalFiles::ExternalFile do
+  let(:external_file) { described_class.new(value) }
+
+  describe "#valid?" do
+    context 'when is a valid remote url' do
+      let(:value) { 'https://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-1.yml' }
+
+      it 'should return true' do
+        expect(external_file.valid?).to be_truthy
+      end
+    end
+
+    context 'when is not a valid remote url' do
+       let(:value) { 'not-valid://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-1.yml' }
+
+      it 'should return false' do
+        expect(external_file.valid?).to be_falsy
+      end
+    end
+
+    context 'when is a valid local path' do
+      let(:value) { '/vendor/gitlab-ci-yml/existent-file.yml' }
+
+      it 'should return true' do
+        allow(File).to receive(:exists?).and_return(true)
+        expect(external_file.valid?).to be_truthy
+      end
+    end
+
+    context 'when is not a valid local path' do
+      let(:value) { '/vendor/gitlab-ci-yml/non-existent-file.yml' }
+
+      it 'should return false' do
+        expect(external_file.valid?).to be_falsy
+      end
+    end
+  end
+
+  describe "#content" do
+    let(:external_file_content) {
+      <<-HEREDOC
+        before_script:
+          - apt-get update -qq && apt-get install -y -qq sqlite3 libsqlite3-dev nodejs
+          - ruby -v
+          - which ruby
+          - gem install bundler --no-ri --no-rdoc
+          - bundle install --jobs $(nproc)  "${FLAGS[@]}"
+      HEREDOC
+    }
+
+    context 'with a local file' do
+      let(:value) { '/vendor/gitlab-ci-yml/non-existent-file.yml' }
+
+      before do
+        allow(File).to receive(:exists?).and_return(true)
+        allow(File).to receive(:read).and_return(external_file_content)
+      end
+
+      it 'should return the content of the file' do
+        expect(external_file.content).to eq(external_file_content)
+      end
+    end
+
+    context 'with a valid remote file' do
+      let(:value) { 'https://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-1.yml' }
+
+      before do
+        allow_any_instance_of(Kernel).to receive_message_chain(:open, :read).and_return(external_file_content)
+      end
+
+      it 'should return the content of the file' do
+        expect(external_file.content).to eq(external_file_content)
+      end
+    end
+  end
+end

spec/lib/gitlab/ci/external_files/mapper_spec.rb

+require 'rails_helper'
+
+describe Gitlab::Ci::ExternalFiles::Mapper do
+  describe '.fetch_paths' do
+    context 'when includes is defined as string' do
+      let(:values) { { includes: '/vendor/gitlab-ci-yml/non-existent-file.yml', image: 'ruby:2.2'} }
+
+      it 'returns an array' do
+        expect(described_class.fetch_paths(values)).to be_an(Array)
+      end
+
+      it 'returns ExternalFile instances' do
+        expect(described_class.fetch_paths(values).first).to be_an_instance_of(::Gitlab::Ci::ExternalFiles::ExternalFile)
+      end
+    end
+
+    context 'when includes is defined as an array' do
+      let(:values) { { includes: ['https://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-1.yml', '/vendor/gitlab-ci-yml/template.yml'], image: 'ruby:2.2'} }
+      it 'returns an array' do
+        expect(described_class.fetch_paths(values)).to be_an(Array)
+      end
+
+      it 'returns ExternalFile instances' do
+        paths = described_class.fetch_paths(values)
+        paths.each do |path|
+          expect(path).to be_an_instance_of(::Gitlab::Ci::ExternalFiles::ExternalFile)
+        end
+      end
+    end
+
+    context 'when includes is not defined' do
+      let(:values) { { image: 'ruby:2.2'} }
+
+      it 'returns an empty array' do
+        expect(described_class.fetch_paths(values)).to be_empty
+      end
+    end
+  end
+end

spec/lib/gitlab/ci/external_files/processor_spec.rb

+require 'rails_helper'
+
+describe Gitlab::Ci::ExternalFiles::Processor do
+  let(:processor) { described_class.new(values) }
+
+  describe "#perform" do
+    context 'when no external files defined' do
+      let(:values) { { image: 'ruby:2.2' } }
+
+      it 'should return the same values' do
+        expect(processor.perform).to eq(values)
+      end
+    end
+
+    context 'when an invalid local file is defined' do
+      let(:values) { { includes: '/vendor/gitlab-ci-yml/non-existent-file.yml', image: 'ruby:2.2'} }
+
+      it 'should raise an error' do
+        expect { processor.perform }.to raise_error(described_class::ExternalFileError)
+      end
+    end
+
+    context 'when an invalid remote file is defined' do
+      let(:values) { { includes: 'not-valid://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-1.yml', image: 'ruby:2.2'} }
+
+      it 'should raise an error' do
+        expect { processor.perform }.to raise_error(described_class::ExternalFileError)
+      end
+    end
+
+    context 'with a valid remote external file is defined' do
+      let(:values) { { includes: 'https://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-1.yml', image: 'ruby:2.2' } }
+      let(:external_file_content) {
+        <<-HEREDOC
+        before_script:
+          - apt-get update -qq && apt-get install -y -qq sqlite3 libsqlite3-dev nodejs
+          - ruby -v
+          - which ruby
+          - gem install bundler --no-ri --no-rdoc
+          - bundle install --jobs $(nproc)  "${FLAGS[@]}"
+
+        rspec:
+          script:
+            - bundle exec rspec
+
+        rubocop:
+          script:
+            - bundle exec rubocop
+        HEREDOC
+      }
+
+      before do
+        allow_any_instance_of(Kernel).to receive_message_chain(:open, :read).and_return(external_file_content)
+      end
+
+      it 'should append the file to the values' do
+        output = processor.perform
+        expect(output.keys).to match_array([:image, :before_script, :rspec, :rubocop]) 
+      end
+
+      it "should remove the 'includes' keyword" do
+        expect(processor.perform[:includes]).to be_nil
+      end
+    end
+
+    context 'with a valid local external file is defined' do
+      let(:values) { { includes: '/vendor/gitlab-ci-yml/template.yml' , image: 'ruby:2.2'} }
+      let(:external_file_content) {
+        <<-HEREDOC
+        before_script:
+          - apt-get update -qq && apt-get install -y -qq sqlite3 libsqlite3-dev nodejs
+          - ruby -v
+          - which ruby
+          - gem install bundler --no-ri --no-rdoc
+          - bundle install --jobs $(nproc)  "${FLAGS[@]}"
+        HEREDOC
+      }
+
+      before do
+        allow(File).to receive(:exists?).and_return(true)
+        allow(File).to receive(:read).and_return(external_file_content)
+      end
+
+      it 'should append the file to the values' do
+        output = processor.perform
+        expect(output.keys).to match_array([:image, :before_script]) 
+      end
+
+      it "should remove the 'includes' keyword" do
+        expect(processor.perform[:includes]).to be_nil
+      end
+    end
+
+    context 'with multiple external files are defined' do
+      let(:external_files) { 
+        [
+          "/spec/ee/fixtures/gitlab/ci/external_files/.gitlab-ci-template-1.yml",
+          "/spec/ee/fixtures/gitlab/ci/external_files/.gitlab-ci-template-2.yml",
+          'https://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-1.yml'
+        ]
+      }
+      let(:values) { { includes: external_files, image: 'ruby:2.2'} }
+
+      let(:remote_file_content) {
+        <<-HEREDOC
+        stages:
+          - build
+          - review
+          - cleanup
+        HEREDOC
+      }
+
+      before do
+        allow_any_instance_of(Kernel).to receive_message_chain(:open, :read).and_return(remote_file_content)
+      end
+
+      it 'should append the files to the values' do
+        expect(processor.perform.keys).to match_array([:image, :variables, :stages, :before_script, :rspec])
+      end
+
+      it "should remove the 'includes' keyword" do
+        expect(processor.perform[:includes]).to be_nil
+      end
+    end
+
+    context 'when external files are defined but not valid' do
+      let(:values) { { includes: '/vendor/gitlab-ci-yml/template.yml', image: 'ruby:2.2'} }
+
+      let(:external_file_content) { 'invalid content file ////' }
+
+      before do
+        allow(File).to receive(:exists?).and_return(true)
+        allow(File).to receive(:read).and_return(external_file_content)
+      end
+
+      it 'should raise an error' do
+        expect { processor.perform }.to raise_error(Gitlab::Ci::Config::Loader::FormatError)
+      end
+    end
+  end
+end