Merge branch '38242-persisting-pipeline-config_content' into 'master'

Implement saving config content for pipelines

See merge request gitlab-org/gitlab!21827

app/models/ci/pipeline.rb

@@ -62,6 +62,8 @@ module Ci
     has_one :triggered_by_pipeline, through: :source_pipeline, source: :source_pipeline
     has_one :source_job, through: :source_pipeline, source: :source_job
 
+    has_one :pipeline_config, class_name: 'Ci::PipelineConfig', inverse_of: :pipeline
+
     accepts_nested_attributes_for :variables, reject_if: :persisted?
 
     delegate :id, to: :project, prefix: true

app/models/ci/pipeline_config.rb

+# frozen_string_literal: true
+
+module Ci
+  class PipelineConfig < ApplicationRecord
+    extend Gitlab::Ci::Model
+
+    self.table_name = 'ci_pipelines_config'
+    self.primary_key = :pipeline_id
+
+    belongs_to :pipeline, class_name: "Ci::Pipeline", inverse_of: :pipeline_config
+    validates :pipeline, presence: true
+    validates :content, presence: true
+  end
+end

changelogs/unreleased/38242-persisting-pipeline-config_content.yml

+---
+title: Implement saving config content for pipelines in a new table 'ci_pipelines_config'
+merge_request: 21827
+author:
+type: other

db/migrate/20191213143656_create_ci_pipelines_config.rb

+# frozen_string_literal: true
+
+class CreateCiPipelinesConfig < ActiveRecord::Migration[5.2]
+  DOWNTIME = false
+
+  def change
+    create_table :ci_pipelines_config, id: false do |t|
+      t.references :pipeline,
+                   primary_key: true,
+                   foreign_key: { to_table: :ci_pipelines, on_delete: :cascade }
+      t.text :content, null: false
+    end
+  end
+end

db/schema.rb

@@ -876,6 +876,11 @@ ActiveRecord::Schema.define(version: 2019_12_18_225624) do
     t.index ["user_id"], name: "index_ci_pipelines_on_user_id"
   end
 
+  create_table "ci_pipelines_config", primary_key: "pipeline_id", force: :cascade do |t|
+    t.text "content", null: false
+    t.index ["pipeline_id"], name: "index_ci_pipelines_config_on_pipeline_id"
+  end
+
   create_table "ci_resource_groups", force: :cascade do |t|
     t.datetime_with_timezone "created_at", null: false
     t.datetime_with_timezone "updated_at", null: false
@@ -4438,6 +4443,7 @@ ActiveRecord::Schema.define(version: 2019_12_18_225624) do
   add_foreign_key "ci_pipelines", "external_pull_requests", name: "fk_190998ef09", on_delete: :nullify
   add_foreign_key "ci_pipelines", "merge_requests", name: "fk_a23be95014", on_delete: :cascade
   add_foreign_key "ci_pipelines", "projects", name: "fk_86635dbd80", on_delete: :cascade
+  add_foreign_key "ci_pipelines_config", "ci_pipelines", column: "pipeline_id", on_delete: :cascade
   add_foreign_key "ci_resource_groups", "projects", name: "fk_774722d144", on_delete: :cascade
   add_foreign_key "ci_resources", "ci_builds", column: "build_id", name: "fk_e169a8e3d5", on_delete: :nullify
   add_foreign_key "ci_resources", "ci_resource_groups", column: "resource_group_id", on_delete: :cascade

lib/gitlab/ci/pipeline/chain/config/content.rb

@@ -24,8 +24,7 @@ module Gitlab
 
             def perform!
               if config = find_config
-                # TODO: we should persist config_content
-                # @pipeline.config_content = config.content
+                @pipeline.build_pipeline_config(content: config.content) if ci_root_config_content_enabled?
                 @command.config_content = config.content
                 @pipeline.config_source = config.source
               else
@@ -49,11 +48,11 @@ module Gitlab
             end
 
             def sources
-              if Feature.enabled?(:ci_root_config_content, @command.project, default_enabled: true)
-                SOURCES
-              else
-                LEGACY_SOURCES
-              end
+              ci_root_config_content_enabled? ? SOURCES : LEGACY_SOURCES
+            end
+
+            def ci_root_config_content_enabled?
+              Feature.enabled?(:ci_root_config_content, @command.project, default_enabled: true)
             end
           end
         end

spec/lib/gitlab/ci/pipeline/chain/config/content_spec.rb

@@ -29,6 +29,7 @@ describe Gitlab::Ci::Pipeline::Chain::Config::Content do
           subject.perform!
 
           expect(pipeline.config_source).to eq 'repository_source'
+          expect(pipeline.pipeline_config).to be_nil
           expect(command.config_content).to eq('the-content')
         end
       end
@@ -40,6 +41,7 @@ describe Gitlab::Ci::Pipeline::Chain::Config::Content do
           subject.perform!
 
           expect(pipeline.config_source).to eq 'auto_devops_source'
+          expect(pipeline.pipeline_config).to be_nil
           template = Gitlab::Template::GitlabCiYmlTemplate.find('Beta/Auto-DevOps')
           expect(command.config_content).to eq(template.content)
         end
@@ -52,6 +54,7 @@ describe Gitlab::Ci::Pipeline::Chain::Config::Content do
           subject.perform!
 
           expect(pipeline.config_source).to eq 'auto_devops_source'
+          expect(pipeline.pipeline_config).to be_nil
           template = Gitlab::Template::GitlabCiYmlTemplate.find('Beta/Auto-DevOps')
           expect(command.config_content).to eq(template.content)
         end
@@ -71,6 +74,7 @@ describe Gitlab::Ci::Pipeline::Chain::Config::Content do
           subject.perform!
 
           expect(pipeline.config_source).to eq 'repository_source'
+          expect(pipeline.pipeline_config).to be_nil
           expect(command.config_content).to eq('the-content')
         end
       end
@@ -91,6 +95,7 @@ describe Gitlab::Ci::Pipeline::Chain::Config::Content do
             subject.perform!
 
             expect(pipeline.config_source).to eq 'auto_devops_source'
+            expect(pipeline.pipeline_config).to be_nil
             template = Gitlab::Template::GitlabCiYmlTemplate.find('Beta/Auto-DevOps')
             expect(command.config_content).to eq(template.content)
           end
@@ -105,6 +110,7 @@ describe Gitlab::Ci::Pipeline::Chain::Config::Content do
             subject.perform!
 
             expect(pipeline.config_source).to eq 'auto_devops_source'
+            expect(pipeline.pipeline_config).to be_nil
             template = Gitlab::Template::GitlabCiYmlTemplate.find('Auto-DevOps')
             expect(command.config_content).to eq(template.content)
           end
@@ -122,6 +128,7 @@ describe Gitlab::Ci::Pipeline::Chain::Config::Content do
           subject.perform!
 
           expect(pipeline.config_source).to eq('unknown_source')
+          expect(pipeline.pipeline_config).to be_nil
           expect(command.config_content).to be_nil
           expect(pipeline.errors.full_messages).to include('Missing CI config file')
         end
@@ -130,6 +137,13 @@ describe Gitlab::Ci::Pipeline::Chain::Config::Content do
 
     context 'when config is defined in a custom path in the repository' do
       let(:ci_config_path) { 'path/to/config.yml' }
+      let(:config_content_result) do
+        <<~EOY
+          ---
+          include:
+          - local: #{ci_config_path}
+        EOY
+      end
 
       before do
         expect(project.repository)
@@ -142,47 +156,59 @@ describe Gitlab::Ci::Pipeline::Chain::Config::Content do
         subject.perform!
 
         expect(pipeline.config_source).to eq 'repository_source'
-        expect(command.config_content).to eq(<<~EOY)
-          ---
-          include:
-          - local: #{ci_config_path}
-        EOY
+        expect(pipeline.pipeline_config.content).to eq(config_content_result)
+        expect(command.config_content).to eq(config_content_result)
       end
     end
 
     context 'when config is defined remotely' do
       let(:ci_config_path) { 'http://example.com/path/to/ci/config.yml' }
+      let(:config_content_result) do
+        <<~EOY
+          ---
+          include:
+          - remote: #{ci_config_path}
+        EOY
+      end
 
       it 'builds root config including the remote config' do
         subject.perform!
 
         expect(pipeline.config_source).to eq 'remote_source'
-        expect(command.config_content).to eq(<<~EOY)
-          ---
-          include:
-          - remote: #{ci_config_path}
-        EOY
+        expect(pipeline.pipeline_config.content).to eq(config_content_result)
+        expect(command.config_content).to eq(config_content_result)
       end
     end
 
     context 'when config is defined in a separate repository' do
       let(:ci_config_path) { 'path/to/.gitlab-ci.yml@another-group/another-repo' }
-
-      it 'builds root config including the path to another repository' do
-        subject.perform!
-
-        expect(pipeline.config_source).to eq 'external_project_source'
-        expect(command.config_content).to eq(<<~EOY)
+      let(:config_content_result) do
+        <<~EOY
           ---
           include:
           - project: another-group/another-repo
             file: path/to/.gitlab-ci.yml
         EOY
       end
+
+      it 'builds root config including the path to another repository' do
+        subject.perform!
+
+        expect(pipeline.config_source).to eq 'external_project_source'
+        expect(pipeline.pipeline_config.content).to eq(config_content_result)
+        expect(command.config_content).to eq(config_content_result)
+      end
     end
 
     context 'when config is defined in the default .gitlab-ci.yml' do
       let(:ci_config_path) { nil }
+      let(:config_content_result) do
+        <<~EOY
+          ---
+          include:
+          - local: ".gitlab-ci.yml"
+        EOY
+      end
 
       before do
         expect(project.repository)
@@ -195,16 +221,20 @@ describe Gitlab::Ci::Pipeline::Chain::Config::Content do
         subject.perform!
 
         expect(pipeline.config_source).to eq 'repository_source'
-        expect(command.config_content).to eq(<<~EOY)
-          ---
-          include:
-          - local: ".gitlab-ci.yml"
-        EOY
+        expect(pipeline.pipeline_config.content).to eq(config_content_result)
+        expect(command.config_content).to eq(config_content_result)
       end
     end
 
     context 'when config is the Auto-Devops template' do
       let(:ci_config_path) { nil }
+      let(:config_content_result) do
+        <<~EOY
+          ---
+          include:
+          - template: Beta/Auto-DevOps.gitlab-ci.yml
+        EOY
+      end
 
       before do
         expect(project).to receive(:auto_devops_enabled?).and_return(true)
@@ -219,11 +249,8 @@ describe Gitlab::Ci::Pipeline::Chain::Config::Content do
           subject.perform!
 
           expect(pipeline.config_source).to eq 'auto_devops_source'
-          expect(command.config_content).to eq(<<~EOY)
-            ---
-            include:
-            - template: Beta/Auto-DevOps.gitlab-ci.yml
-          EOY
+          expect(pipeline.pipeline_config.content).to eq(config_content_result)
+          expect(command.config_content).to eq(config_content_result)
         end
       end
 
@@ -232,16 +259,21 @@ describe Gitlab::Ci::Pipeline::Chain::Config::Content do
           stub_feature_flags(auto_devops_beta: false)
         end
 
-        it 'builds root config including the auto-devops template' do
-          subject.perform!
-
-          expect(pipeline.config_source).to eq 'auto_devops_source'
-          expect(command.config_content).to eq(<<~EOY)
+        let(:config_content_result) do
+          <<~EOY
             ---
             include:
             - template: Auto-DevOps.gitlab-ci.yml
           EOY
         end
+
+        it 'builds root config including the auto-devops template' do
+          subject.perform!
+
+          expect(pipeline.config_source).to eq 'auto_devops_source'
+          expect(pipeline.pipeline_config.content).to eq(config_content_result)
+          expect(command.config_content).to eq(config_content_result)
+        end
       end
     end
 
@@ -256,6 +288,7 @@ describe Gitlab::Ci::Pipeline::Chain::Config::Content do
         subject.perform!
 
         expect(pipeline.config_source).to eq('unknown_source')
+        expect(pipeline.pipeline_config).to be_nil
         expect(command.config_content).to be_nil
         expect(pipeline.errors.full_messages).to include('Missing CI config file')
       end

spec/lib/gitlab/import_export/all_models.yml

@@ -190,6 +190,7 @@ ci_pipelines:
 - job_artifacts
 - vulnerabilities_occurrence_pipelines
 - vulnerability_findings
+- pipeline_config
 pipeline_variables:
 - pipeline
 stages:

spec/models/ci/pipeline_config_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Ci::PipelineConfig, type: :model do
+  it { is_expected.to belong_to(:pipeline) }
+
+  it { is_expected.to validate_presence_of(:pipeline) }
+  it { is_expected.to validate_presence_of(:content) }
+end

spec/models/ci/pipeline_spec.rb

@@ -35,6 +35,7 @@ describe Ci::Pipeline, :mailer do
   it { is_expected.to have_one(:source_pipeline) }
   it { is_expected.to have_one(:triggered_by_pipeline) }
   it { is_expected.to have_one(:source_job) }
+  it { is_expected.to have_one(:pipeline_config) }
 
   it { is_expected.to validate_presence_of(:sha) }
   it { is_expected.to validate_presence_of(:status) }