Commit 5c26f471 authored by Kamil Trzciński's avatar Kamil Trzciński

Merge branch 'feature/gb/persist-pipeline-stages' into 'master'

Persist stages in the database

Closes #26481

See merge request !11790
parents acd0b691 4edde47e
......@@ -99,7 +99,7 @@ class Projects::PipelinesController < Projects::ApplicationController
end
def stage
@stage = pipeline.stage(params[:stage])
@stage = pipeline.legacy_stage(params[:stage])
return not_found unless @stage
respond_to do |format|
......
module Ci
# Currently this is artificial object, constructed dynamically
# We should migrate this object to actual database record in the future
class LegacyStage
include StaticModel
attr_reader :pipeline, :name
delegate :project, to: :pipeline
def initialize(pipeline, name:, status: nil, warnings: nil)
@pipeline = pipeline
@name = name
@status = status
@warnings = warnings
end
def groups
@groups ||= statuses.ordered.latest
.sort_by(&:sortable_name).group_by(&:group_name)
.map do |group_name, grouped_statuses|
Ci::Group.new(self, name: group_name, jobs: grouped_statuses)
end
end
def to_param
name
end
def statuses_count
@statuses_count ||= statuses.count
end
def status
@status ||= statuses.latest.status
end
def detailed_status(current_user)
Gitlab::Ci::Status::Stage::Factory
.new(self, current_user)
.fabricate!
end
def statuses
@statuses ||= pipeline.statuses.where(stage: name)
end
def builds
@builds ||= pipeline.builds.where(stage: name)
end
def success?
status.to_s == 'success'
end
def has_warnings?
if @warnings.is_a?(Integer)
@warnings > 0
else
statuses.latest.failed_but_allowed.any?
end
end
end
end
......@@ -11,9 +11,7 @@ module Ci
belongs_to :auto_canceled_by, class_name: 'Ci::Pipeline'
belongs_to :pipeline_schedule, class_name: 'Ci::PipelineSchedule'
has_many :auto_canceled_pipelines, class_name: 'Ci::Pipeline', foreign_key: 'auto_canceled_by_id'
has_many :auto_canceled_jobs, class_name: 'CommitStatus', foreign_key: 'auto_canceled_by_id'
has_many :stages
has_many :statuses, class_name: 'CommitStatus', foreign_key: :commit_id
has_many :builds, foreign_key: :commit_id
has_many :trigger_requests, dependent: :destroy, foreign_key: :commit_id
......@@ -28,6 +26,9 @@ module Ci
has_many :manual_actions, -> { latest.manual_actions }, foreign_key: :commit_id, class_name: 'Ci::Build'
has_many :artifacts, -> { latest.with_artifacts_not_expired }, foreign_key: :commit_id, class_name: 'Ci::Build'
has_many :auto_canceled_pipelines, class_name: 'Ci::Pipeline', foreign_key: 'auto_canceled_by_id'
has_many :auto_canceled_jobs, class_name: 'CommitStatus', foreign_key: 'auto_canceled_by_id'
delegate :id, to: :project, prefix: true
validates :source, exclusion: { in: %w(unknown), unless: :importing? }, on: :create
......@@ -162,21 +163,21 @@ module Ci
where.not(duration: nil).sum(:duration)
end
def stage(name)
stage = Ci::Stage.new(self, name: name)
stage unless stage.statuses_count.zero?
end
def stages_count
statuses.select(:stage).distinct.count
end
def stages_name
def stages_names
statuses.order(:stage_idx).distinct.
pluck(:stage, :stage_idx).map(&:first)
end
def stages
def legacy_stage(name)
stage = Ci::LegacyStage.new(self, name: name)
stage unless stage.statuses_count.zero?
end
def legacy_stages
# TODO, this needs refactoring, see gitlab-ce#26481.
stages_query = statuses
......@@ -191,7 +192,7 @@ module Ci
.pluck('sg.stage', status_sql, "(#{warnings_sql})")
stages_with_statuses.map do |stage|
Ci::Stage.new(self, Hash[%i[name status warnings].zip(stage)])
Ci::LegacyStage.new(self, Hash[%i[name status warnings].zip(stage)])
end
end
......@@ -291,12 +292,14 @@ module Ci
end
end
def config_builds_attributes
def stage_seeds
return [] unless config_processor
config_processor.
builds_for_ref(ref, tag?, trigger_requests.first).
sort_by { |build| build[:stage_idx] }
@stage_seeds ||= config_processor.stage_seeds(self)
end
def has_stage_seeds?
stage_seeds.any?
end
def has_warnings?
......@@ -304,7 +307,7 @@ module Ci
end
def config_processor
return nil unless ci_yaml_file
return unless ci_yaml_file
return @config_processor if defined?(@config_processor)
@config_processor ||= begin
......
module Ci
# Currently this is artificial object, constructed dynamically
# We should migrate this object to actual database record in the future
class Stage
include StaticModel
class Stage < ActiveRecord::Base
extend Ci::Model
attr_reader :pipeline, :name
belongs_to :project
belongs_to :pipeline
delegate :project, to: :pipeline
def initialize(pipeline, name:, status: nil, warnings: nil)
@pipeline = pipeline
@name = name
@status = status
@warnings = warnings
end
def groups
@groups ||= statuses.ordered.latest
.sort_by(&:sortable_name).group_by(&:group_name)
.map do |group_name, grouped_statuses|
Ci::Group.new(self, name: group_name, jobs: grouped_statuses)
end
end
def to_param
name
end
def statuses_count
@statuses_count ||= statuses.count
end
def status
@status ||= statuses.latest.status
end
def detailed_status(current_user)
Gitlab::Ci::Status::Stage::Factory
.new(self, current_user)
.fabricate!
end
def statuses
@statuses ||= pipeline.statuses.where(stage: name)
end
def builds
@builds ||= pipeline.builds.where(stage: name)
end
def success?
status.to_s == 'success'
end
def has_warnings?
if @warnings.is_a?(Integer)
@warnings > 0
else
statuses.latest.failed_but_allowed.any?
end
end
has_many :statuses, class_name: 'CommitStatus', foreign_key: :commit_id
has_many :builds, foreign_key: :commit_id
end
end
......@@ -5,10 +5,10 @@ class CommitStatus < ActiveRecord::Base
self.table_name = 'ci_builds'
belongs_to :user
belongs_to :project
belongs_to :pipeline, class_name: 'Ci::Pipeline', foreign_key: :commit_id
belongs_to :auto_canceled_by, class_name: 'Ci::Pipeline'
belongs_to :user
delegate :commit, to: :pipeline
delegate :sha, :short_sha, to: :pipeline
......
class PipelineDetailsEntity < PipelineEntity
expose :details do
expose :stages, using: StageEntity
expose :legacy_stages, as: :stages, using: StageEntity
expose :artifacts, using: BuildArtifactEntity
expose :manual_actions, using: BuildActionEntity
end
......
module Ci
class CreatePipelineBuildsService < BaseService
attr_reader :pipeline
def execute(pipeline)
@pipeline = pipeline
new_builds.map do |build_attributes|
create_build(build_attributes)
end
end
delegate :project, to: :pipeline
private
def create_build(build_attributes)
build_attributes = build_attributes.merge(
pipeline: pipeline,
project: project,
ref: pipeline.ref,
tag: pipeline.tag,
user: current_user,
trigger_request: trigger_request
)
build = pipeline.builds.create(build_attributes)
# Create the environment before the build starts. This sets its slug and
# makes it available as an environment variable
project.environments.find_or_create_by(name: build.expanded_environment_name) if
build.has_environment?
build
end
def new_builds
@new_builds ||= pipeline.config_builds_attributes.
reject { |build| existing_build_names.include?(build[:name]) }
end
def existing_build_names
@existing_build_names ||= pipeline.builds.pluck(:name)
end
def trigger_request
return @trigger_request if defined?(@trigger_request)
@trigger_request ||= pipeline.trigger_requests.first
end
end
end
......@@ -43,14 +43,14 @@ module Ci
return pipeline
end
unless pipeline.config_builds_attributes.present?
return error('No builds for this pipeline.')
unless pipeline.has_stage_seeds?
return error('No stages / jobs for this pipeline.')
end
Ci::Pipeline.transaction do
update_merge_requests_head_pipeline if pipeline.save
Ci::CreatePipelineBuildsService
Ci::CreatePipelineStagesService
.new(project, current_user)
.execute(pipeline)
end
......
module Ci
class CreatePipelineStagesService < BaseService
def execute(pipeline)
pipeline.stage_seeds.each do |seed|
seed.user = current_user
seed.create! do |build|
##
# Create the environment before the build starts. This sets its slug and
# makes it available as an environment variable
#
if build.has_environment?
environment_name = build.expanded_environment_name
project.environments.find_or_create_by(name: environment_name)
end
end
end
end
end
end
module Ci
class RetryBuildService < ::BaseService
CLONE_ACCESSORS = %i[pipeline project ref tag options commands name
allow_failure stage stage_idx trigger_request
allow_failure stage_id stage stage_idx trigger_request
yaml_variables when environment coverage_regex
description tag_list].freeze
......
......@@ -72,8 +72,8 @@
Pipeline
= link_to "##{last_pipeline.id}", namespace_project_pipeline_path(@project.namespace, @project, last_pipeline.id)
= ci_label_for_status(last_pipeline.status)
- if last_pipeline.stages.any?
with #{"stage".pluralize(last_pipeline.stages.count)}
- if last_pipeline.stages_count.nonzero?
with #{"stage".pluralize(last_pipeline.stages_count)}
.mr-widget-pipeline-graph
= render 'shared/mini_pipeline_graph', pipeline: last_pipeline, klass: 'js-commit-pipeline-graph'
in
......
......@@ -111,7 +111,7 @@
%span.stage-selection More
= icon('chevron-down')
%ul.dropdown-menu
- @build.pipeline.stages.each do |stage|
- @build.pipeline.legacy_stages.each do |stage|
%li
%a.stage-item= stage.name
......
......@@ -42,7 +42,7 @@
%th
%th Coverage
%th
= render partial: "projects/stage/stage", collection: pipeline.stages, as: :stage
= render partial: "projects/stage/stage", collection: pipeline.legacy_stages, as: :stage
- if failed_builds.present?
#js-tab-failures.build-failures.tab-pane
- failed_builds.each_with_index do |build, index|
......
.stage-cell
- pipeline.stages.each do |stage|
- pipeline.legacy_stages.each do |stage|
- if stage.status
- detailed_status = stage.detailed_status(current_user)
- icon_status = "#{detailed_status.icon}_borderless"
......
---
title: Persist pipeline stages in the database
merge_request: 11790
author:
......@@ -50,10 +50,23 @@ module Ci
end
end
def stage_seeds(pipeline)
trigger_request = pipeline.trigger_requests.first
seeds = @stages.uniq.map do |stage|
builds = builds_for_stage_and_ref(
stage, pipeline.ref, pipeline.tag?, trigger_request)
Gitlab::Ci::Stage::Seed.new(pipeline, stage, builds) if builds.any?
end
seeds.compact
end
def build_attributes(name)
job = @jobs[name.to_sym] || {}
{
stage_idx: @stages.index(job[:stage]),
{ stage_idx: @stages.index(job[:stage]),
stage: job[:stage],
commands: job[:commands],
tag_list: job[:tags] || [],
......@@ -71,8 +84,7 @@ module Ci
dependencies: job[:dependencies],
after_script: job[:after_script],
environment: job[:environment]
}.compact
}
}.compact }
end
def self.validation_message(content)
......
module Gitlab
module Ci
module Stage
class Seed
attr_reader :pipeline
delegate :project, to: :pipeline
def initialize(pipeline, stage, jobs)
@pipeline = pipeline
@stage = { name: stage }
@jobs = jobs.to_a.dup
end
def user=(current_user)
@jobs.map! do |attributes|
attributes.merge(user: current_user)
end
end
def stage
@stage.merge(project: project)
end
def builds
trigger = pipeline.trigger_requests.first
@jobs.map do |attributes|
attributes.merge(project: project,
ref: pipeline.ref,
tag: pipeline.tag,
trigger_request: trigger)
end
end
def create!
pipeline.stages.create!(stage).tap do |stage|
builds_attributes = builds.map do |attributes|
attributes.merge(stage_id: stage.id)
end
pipeline.builds.create!(builds_attributes).each do |build|
yield build if block_given?
end
end
end
end
end
end
end
......@@ -22,7 +22,7 @@ module Gitlab
sha: pipeline.sha,
before_sha: pipeline.before_sha,
status: pipeline.status,
stages: pipeline.stages_name,
stages: pipeline.stages_names,
created_at: pipeline.created_at,
finished_at: pipeline.finished_at,
duration: pipeline.duration
......
......@@ -38,6 +38,7 @@ project_tree:
- notes:
- :author
- :events
- :stages
- :statuses
- :triggers
- :pipeline_schedules
......
......@@ -3,6 +3,7 @@ module Gitlab
class RelationFactory
OVERRIDES = { snippets: :project_snippets,
pipelines: 'Ci::Pipeline',
stages: 'Ci::Stage',
statuses: 'commit_status',
triggers: 'Ci::Trigger',
pipeline_schedules: 'Ci::PipelineSchedule',
......
FactoryGirl.define do
factory :ci_stage, class: Ci::Stage do
factory :ci_stage, class: Ci::LegacyStage do
skip_create
transient do
......@@ -10,7 +10,9 @@ FactoryGirl.define do
end
initialize_with do
Ci::Stage.new(pipeline, name: name, status: status, warnings: warnings)
Ci::LegacyStage.new(pipeline, name: name,
status: status,
warnings: warnings)
end
end
end
require 'spec_helper'
module Ci
describe GitlabCiYamlProcessor, lib: true do
describe GitlabCiYamlProcessor, :lib do
subject { described_class.new(config, path) }
let(:path) { 'path' }
describe 'our current .gitlab-ci.yml' do
......@@ -82,6 +83,48 @@ module Ci
end
end
describe '#stage_seeds' do
context 'when no refs policy is specified' do
let(:config) do
YAML.dump(production: { stage: 'deploy', script: 'cap prod' },
rspec: { stage: 'test', script: 'rspec' },
spinach: { stage: 'test', script: 'spinach' })
end
let(:pipeline) { create(:ci_empty_pipeline) }
it 'correctly fabricates a stage seeds object' do
seeds = subject.stage_seeds(pipeline)
expect(seeds.size).to eq 2
expect(seeds.first.stage[:name]).to eq 'test'
expect(seeds.second.stage[:name]).to eq 'deploy'
expect(seeds.first.builds.dig(0, :name)).to eq 'rspec'
expect(seeds.first.builds.dig(1, :name)).to eq 'spinach'
expect(seeds.second.builds.dig(0, :name)).to eq 'production'
end
end
context 'when refs policy is specified' do
let(:config) do
YAML.dump(production: { stage: 'deploy', script: 'cap prod', only: ['master'] },
spinach: { stage: 'test', script: 'spinach', only: ['tags'] })
end
let(:pipeline) do
create(:ci_empty_pipeline, ref: 'feature', tag: true)
end
it 'returns stage seeds only assigned to master to master' do
seeds = subject.stage_seeds(pipeline)
expect(seeds.size).to eq 1
expect(seeds.first.stage[:name]).to eq 'test'
expect(seeds.first.builds.dig(0, :name)).to eq 'spinach'
end
end
end
describe "#builds_for_ref" do
let(:type) { 'test' }
......
require 'spec_helper'
describe Gitlab::Ci::Stage::Seed do
let(:pipeline) { create(:ci_empty_pipeline) }
let(:builds) do
[{ name: 'rspec' }, { name: 'spinach' }]
end
subject do
described_class.new(pipeline, 'test', builds)
end
describe '#stage' do
it 'returns hash attributes of a stage' do
expect(subject.stage).to be_a Hash
expect(subject.stage).to include(:name, :project)
end
end
describe '#builds' do
it 'returns hash attributes of all builds' do
expect(subject.builds.size).to eq 2
expect(subject.builds).to all(include(ref: 'master'))
expect(subject.builds).to all(include(tag: false))
expect(subject.builds).to all(include(project: pipeline.project))
expect(subject.builds)
.to all(include(trigger_request: pipeline.trigger_requests.first))
end
end
describe '#user=' do
let(:user) { build(:user) }
it 'assignes relevant pipeline attributes' do
subject.user = user
expect(subject.builds).to all(include(user: user))
end
end
describe '#create!' do
it 'creates all stages and builds' do
subject.create!
expect(pipeline.reload.stages.count).to eq 1
expect(pipeline.reload.builds.count).to eq 2
expect(pipeline.builds).to all(satisfy { |job| job.stage_id.present? })
expect(pipeline.builds).to all(satisfy { |job| job.pipeline.present? })
expect(pipeline.builds).to all(satisfy { |job| job.project.present? })
expect(pipeline.stages)
.to all(satisfy { |stage| stage.pipeline.present? })
expect(pipeline.stages)
.to all(satisfy { |stage| stage.project.present? })
end
end
end
......@@ -91,6 +91,7 @@ merge_request_diff:
pipelines:
- project
- user
- stages
- statuses
- builds
- trigger_requests
......@@ -104,9 +105,15 @@ pipelines:
- artifacts
- pipeline_schedule
- merge_requests
stages:
- project
- pipeline
- statuses
- builds
statuses:
- project
- pipeline
- stage
- user
- auto_canceled_by
variables:
......
......@@ -175,6 +175,7 @@ MergeRequestDiff:
Ci::Pipeline:
- id
- project_id
- source
- ref
- sha
- before_sha
......@@ -192,7 +193,13 @@ Ci::Pipeline:
- lock_version
- auto_canceled_by_id
- pipeline_schedule_id
- source
Ci::Stage:
- id
- name
- project_id
- pipeline_id
- created_at
- updated_at
CommitStatus:
- id
- project_id
......
require 'spec_helper'
describe Ci::Stage, models: true do
describe Ci::LegacyStage, :models do
let(:stage) { build(:ci_stage) }
let(:pipeline) { stage.pipeline }
let(:stage_name) { stage.name }
......
......@@ -224,8 +224,19 @@ describe Ci::Pipeline, models: true do
status: 'success')
end
describe '#stages' do
subject { pipeline.stages }
describe '#stage_seeds' do
let(:pipeline) do
create(:ci_pipeline, config: { rspec: { script: 'rake' } })
end
it 'returns preseeded stage seeds object' do
expect(pipeline.stage_seeds).to all(be_a Gitlab::Ci::Stage::Seed)
expect(pipeline.stage_seeds.count).to eq 1
end
end
describe '#legacy_stages' do
subject { pipeline.legacy_stages }
context 'stages list' do
it 'returns ordered list of stages' do
......@@ -274,7 +285,7 @@ describe Ci::Pipeline, models: true do
end
it 'populates stage with correct number of warnings' do
deploy_stage = pipeline.stages.third
deploy_stage = pipeline.legacy_stages.third
expect(deploy_stage).not_to receive(:statuses)
expect(deploy_stage).to have_warnings
......@@ -288,22 +299,22 @@ describe Ci::Pipeline, models: true do
end
end
describe '#stages_name' do
describe '#stages_names' do
it 'returns a valid names of stages' do
expect(pipeline.stages_name).to eq(%w(build test deploy))
expect(pipeline.stages_names).to eq(%w(build test deploy))
end
end
end
describe '#stage' do
subject { pipeline.stage('test') }
describe '#legacy_stage' do
subject { pipeline.legacy_stage('test') }
context 'with status in stage' do
before do
create(:commit_status, pipeline: pipeline, stage: 'test')
end
it { expect(subject).to be_a Ci::Stage }
it { expect(subject).to be_a Ci::LegacyStage }
it { expect(subject.name).to eq 'test' }
it { expect(subject.statuses).not_to be_empty }
end
......@@ -524,6 +535,20 @@ describe Ci::Pipeline, models: true do
end
end
describe '#has_stage_seeds?' do
context 'when pipeline has stage seeds' do
subject { build(:ci_pipeline_with_one_job) }
it { is_expected.to have_stage_seeds }
end
context 'when pipeline does not have stage seeds' do
subject { create(:ci_pipeline_without_jobs) }
it { is_expected.not_to have_stage_seeds }
end
end
describe '#has_warnings?' do
subject { pipeline.has_warnings? }
......
require 'spec_helper'
describe Ci::CreatePipelineService, services: true do
describe Ci::CreatePipelineService, :services do
let(:project) { create(:project, :repository) }
let(:user) { create(:admin) }
......@@ -30,6 +30,7 @@ describe Ci::CreatePipelineService, services: true do
it 'creates a pipeline' do
expect(pipeline).to be_kind_of(Ci::Pipeline)
expect(pipeline).to be_valid
expect(pipeline).to be_persisted
expect(pipeline).to be_push
expect(pipeline).to eq(project.pipelines.last)
expect(pipeline).to have_attributes(user: user)
......
......@@ -18,20 +18,31 @@ describe Ci::RetryBuildService, :services do
updated_at started_at finished_at queued_at erased_by
erased_at auto_canceled_by].freeze
# TODO, move stage_id accessor to CLONE_ACCESSOR in a follow-up MR.
IGNORE_ACCESSORS =
%i[type lock_version target_url base_tags
commit_id deployments erased_by_id last_deployment project_id
runner_id tag_taggings taggings tags trigger_request_id
user_id auto_canceled_by_id retried stage_id].freeze
user_id auto_canceled_by_id retried].freeze
shared_examples 'build duplication' do
let(:stage) do
# TODO, we still do not have factory for new stages, we will need to
# switch existing factory to persist stages, instead of using LegacyStage
#
Ci::Stage.create!(project: project, pipeline: pipeline, name: 'test')
end
let(:build) do
create(:ci_build, :failed, :artifacts_expired, :erased,
:queued, :coverage, :tags, :allowed_to_fail, :on_tag,
:teardown_environment, :triggered, :trace,
description: 'some build', pipeline: pipeline,
auto_canceled_by: create(:ci_empty_pipeline))
:triggered, :trace, :teardown_environment,
description: 'my-job', stage: 'test', pipeline: pipeline,
auto_canceled_by: create(:ci_empty_pipeline)) do |build|
##
# TODO, workaround for FactoryGirl limitation when having both
# stage (text) and stage_id (integer) columns in the table.
build.stage_id = stage.id
end
end
describe 'clone accessors' do
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment