Commit d03aff88 authored by Shinya Maeda's avatar Shinya Maeda

Merge branch 'make-yaml-processor-return-results-2' into 'master'

Refactor YamlProcessor to always return a result object

See merge request gitlab-org/gitlab!40013
parents 87d894d3 ea79ef49
......@@ -89,8 +89,6 @@ module EE
def self.sast_ci_configuration(project)
::Security::CiConfiguration::SastParserService.new(project).configuration
rescue ::Gitlab::Ci::YamlProcessor::ValidationError => ex
raise ::GraphQL::ExecutionError, ex.message
end
end
end
......
......@@ -66,7 +66,10 @@ module Security
def build_sast_attributes(content)
options = { project: @project, user: current_user, sha: @project.repository.commit.sha }
sast_attributes = Gitlab::Ci::YamlProcessor.new(content, options).build_attributes(:sast)
yaml_result = Gitlab::Ci::YamlProcessor.new(content, options).execute
return {} unless yaml_result.valid?
sast_attributes = yaml_result.build_attributes(:sast)
extract_required_attributes(sast_attributes)
end
......
......@@ -25,7 +25,6 @@ RSpec.describe GitlabSchema.types['Project'] do
describe 'sast_ci_configuration' do
include_context 'read ci configuration for sast enabled project'
let(:error_message) { "This is an error for YamlProcessor." }
let_it_be(:query) do
%(
......@@ -110,14 +109,6 @@ RSpec.describe GitlabSchema.types['Project'] do
expect(analyzer['label']).to eq('Brakeman')
expect(analyzer['enabled']).to eq(true)
end
it 'returns an error if there is an exception in YamlProcessor' do
allow_next_instance_of(::Security::CiConfiguration::SastParserService) do |service|
allow(service).to receive(:configuration).and_raise(::Gitlab::Ci::YamlProcessor::ValidationError.new(error_message))
end
expect(subject["errors"].first["message"]).to eql(error_message)
end
end
describe 'security_scanners' do
......
......@@ -5,7 +5,7 @@ require 'spec_helper'
RSpec.describe "CI YML Templates" do
using RSpec::Parameterized::TableSyntax
subject { Gitlab::Ci::YamlProcessor.new(content) }
subject { Gitlab::Ci::YamlProcessor.new(content).execute }
where(:template_name) do
Gitlab::Template::GitlabCiYmlTemplate.all.map(&:full_name)
......@@ -24,7 +24,7 @@ RSpec.describe "CI YML Templates" do
end
it 'is valid' do
expect { subject }.not_to raise_error
expect(subject).to be_valid
end
it 'require default stages to be included' do
......
......@@ -11,7 +11,7 @@ RSpec.describe Gitlab::Ci::YamlProcessor do
}
end
subject { described_class.new(YAML.dump(config)) }
subject { described_class.new(YAML.dump(config)).execute }
context 'needs upstream pipeline' do
let(:needs) { { pipeline: 'some/project' } }
......@@ -174,8 +174,7 @@ RSpec.describe Gitlab::Ci::YamlProcessor do
end
it 'returns errors' do
expect { subject }
.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError,
expect(subject.errors).to include(
'jobs:bridge config should contain either a trigger or a needs:pipeline')
end
end
......@@ -198,8 +197,7 @@ RSpec.describe Gitlab::Ci::YamlProcessor do
end
it 'returns errors' do
expect { subject }
.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError,
expect(subject.errors).to contain_exactly(
'jobs:test:needs:need ref should be a string')
end
end
......@@ -216,10 +214,10 @@ RSpec.describe Gitlab::Ci::YamlProcessor do
let(:config) { { deploy_to_production: { stage: 'deploy', script: ['echo'], secrets: secrets } } }
subject(:processor) { described_class.new(YAML.dump(config)) }
subject(:result) { described_class.new(YAML.dump(config)).execute }
it "returns secrets info" do
secrets = processor.stage_builds_attributes('deploy').first.fetch(:secrets)
secrets = result.stage_builds_attributes('deploy').first.fetch(:secrets)
expect(secrets).to eq({
DATABASE_PASSWORD: {
......
......@@ -62,6 +62,10 @@ module Gitlab
root.jobs_value
end
def normalized_jobs
@normalized_jobs ||= Ci::Config::Normalizer.new(jobs).normalize_jobs
end
private
def expand_config(config)
......
......@@ -11,6 +11,7 @@ module Gitlab
end
def normalize_jobs
return {} unless @jobs_config
return @jobs_config if parallelized_jobs.empty?
expand_parallelize_jobs do |job_name, config|
......
......@@ -45,14 +45,15 @@ module Gitlab
end
def static_validation(content)
result = Gitlab::Ci::YamlProcessor.new_with_validation_errors(
result = Gitlab::Ci::YamlProcessor.new(
content,
project: @project,
user: @current_user,
sha: @project.repository.commit.sha)
sha: @project.repository.commit.sha
).execute
Result.new(
jobs: static_validation_convert_to_jobs(result.config&.stages, result.config&.builds),
jobs: static_validation_convert_to_jobs(result),
errors: result.errors,
warnings: result.warnings
)
......@@ -76,12 +77,12 @@ module Gitlab
end
end
def static_validation_convert_to_jobs(stages, all_jobs)
def static_validation_convert_to_jobs(result)
jobs = []
return jobs unless stages || all_jobs
return jobs unless result.valid?
stages.each do |stage_name|
all_jobs.each do |job|
result.stages.each do |stage_name|
result.builds.each do |job|
next unless job[:stage] == stage_name
jobs << {
......
......@@ -11,20 +11,23 @@ module Gitlab
def perform!
raise ArgumentError, 'missing config content' unless @command.config_content
@command.config_processor = ::Gitlab::Ci::YamlProcessor.new(
result = ::Gitlab::Ci::YamlProcessor.new(
@command.config_content, {
project: project,
sha: @pipeline.sha,
user: current_user,
parent_pipeline: parent_pipeline
}
)
).execute
add_warnings_to_pipeline(result.warnings)
add_warnings_to_pipeline(@command.config_processor.warnings)
rescue Gitlab::Ci::YamlProcessor::ValidationError => ex
add_warnings_to_pipeline(ex.warnings)
if result.valid?
@command.config_processor = result
else
error(result.errors.first, config_error: true)
end
error(ex.message, config_error: true)
rescue => ex
Gitlab::ErrorTracking.track_exception(ex,
project_id: project.id,
......
# frozen_string_literal: true
# This is the CI Linter component that runs the syntax validations
# while parsing the YAML config into a data structure that is
# then presented to the caller as result object.
# After syntax validations (done by Ci::Config), this component also
# runs logical validation on the built data structure.
module Gitlab
module Ci
class YamlProcessor
# ValidationError is treated like a result object in the form of an exception.
# We can return any warnings, raised during the config validation, along with
# the error object until we support multiple messages to be returned.
class ValidationError < StandardError
attr_reader :warnings
def initialize(message, warnings: [])
@warnings = warnings
super(message)
end
end
include Gitlab::Config::Entry::LegacyValidationHelpers
ValidationError = Class.new(StandardError)
attr_reader :stages, :jobs
class Result
attr_reader :config, :errors, :warnings
def self.validation_message(content, opts = {})
result = new(content, opts).execute
def initialize(config: nil, errors: [], warnings: [])
@config = config
@errors = errors
@warnings = warnings
result.errors.first
end
def valid?
config.present? && errors.empty?
def initialize(config_content, opts = {})
@config_content = config_content
@opts = opts
end
def execute
if @config_content.blank?
return Result.new(errors: ['Please provide content of .gitlab-ci.yml'])
end
def initialize(config, opts = {})
@ci_config = Gitlab::Ci::Config.new(config, **opts)
@config = @ci_config.to_hash
@ci_config = Gitlab::Ci::Config.new(@config_content, **@opts)
unless @ci_config.valid?
error!(@ci_config.errors.first)
return Result.new(ci_config: @ci_config, errors: @ci_config.errors, warnings: @ci_config.warnings)
end
initial_parsing
rescue Gitlab::Ci::Config::ConfigError => e
error!(e.message)
end
def self.new_with_validation_errors(content, opts = {})
return Result.new(errors: ['Please provide content of .gitlab-ci.yml']) if content.blank?
run_logical_validations!
config = Gitlab::Ci::Config.new(content, **opts)
return Result.new(errors: config.errors, warnings: config.warnings) unless config.valid?
config = Gitlab::Ci::YamlProcessor.new(content, opts)
Result.new(config: config, warnings: config.warnings)
rescue ValidationError => e
Result.new(errors: [e.message], warnings: e.warnings)
Result.new(ci_config: @ci_config, warnings: @ci_config&.warnings)
rescue Gitlab::Ci::Config::ConfigError => e
Result.new(errors: [e.message])
end
def warnings
@ci_config&.warnings || []
end
def builds
@jobs.map do |name, _|
build_attributes(name)
end
end
def build_attributes(name)
job = @jobs.fetch(name.to_sym, {})
{ stage_idx: @stages.index(job[:stage]),
stage: job[:stage],
tag_list: job[:tags],
name: job[:name].to_s,
allow_failure: job[:ignore],
when: job[:when] || 'on_success',
environment: job[:environment_name],
coverage_regex: job[:coverage],
yaml_variables: transform_to_yaml_variables(job[:variables]),
needs_attributes: job.dig(:needs, :job),
interruptible: job[:interruptible],
only: job[:only],
except: job[:except],
rules: job[:rules],
cache: job[:cache],
resource_group_key: job[:resource_group],
scheduling_type: job[:scheduling_type],
secrets: job[:secrets],
options: {
image: job[:image],
services: job[:services],
artifacts: job[:artifacts],
dependencies: job[:dependencies],
cross_dependencies: job.dig(:needs, :cross_dependency),
job_timeout: job[:timeout],
before_script: job[:before_script],
script: job[:script],
after_script: job[:after_script],
environment: job[:environment],
retry: job[:retry],
parallel: job[:parallel],
instance: job[:instance],
start_in: job[:start_in],
trigger: job[:trigger],
bridge_needs: job.dig(:needs, :bridge)&.first,
release: release(job)
}.compact }.compact
end
def release(job)
job[:release]
end
def stage_builds_attributes(stage)
@jobs.values
.select { |job| job[:stage] == stage }
.map { |job| build_attributes(job[:name]) }
end
Result.new(ci_config: @ci_config, errors: [e.message], warnings: @ci_config&.warnings)
def stages_attributes
@stages.uniq.map do |stage|
seeds = stage_builds_attributes(stage)
{ name: stage, index: @stages.index(stage), builds: seeds }
end
end
def workflow_attributes
{
rules: @config.dig(:workflow, :rules),
yaml_variables: transform_to_yaml_variables(@variables)
}
end
def self.validation_message(content, opts = {})
return 'Please provide content of .gitlab-ci.yml' if content.blank?
begin
Gitlab::Ci::YamlProcessor.new(content, opts)
nil
rescue ValidationError => e
e.message
end
Result.new(ci_config: @ci_config, errors: [e.message], warnings: @ci_config&.warnings)
end
private
def initial_parsing
##
# Global config
#
@variables = @ci_config.variables
def run_logical_validations!
@stages = @ci_config.stages
##
# Jobs
#
@jobs = Ci::Config::Normalizer.new(@ci_config.jobs).normalize_jobs
@jobs = @ci_config.normalized_jobs
@jobs.each do |name, job|
# logical validation for job
validate_job!(name, job)
end
end
def validate_job!(name, job)
validate_job_stage!(name, job)
validate_job_dependencies!(name, job)
validate_job_needs!(name, job)
validate_dynamic_child_pipeline_dependencies!(name, job)
validate_job_environment!(name, job)
end
end
def transform_to_yaml_variables(variables)
variables.to_h.map do |key, value|
{ key: key.to_s, value: value, public: true }
end
end
def validate_job_stage!(name, job)
return unless job[:stage]
......@@ -188,10 +70,6 @@ module Gitlab
end
end
def error!(message)
raise ValidationError.new(message, warnings: warnings)
end
def validate_job_dependencies!(name, job)
return unless job[:dependencies]
......@@ -267,6 +145,10 @@ module Gitlab
error!("#{name} job: on_stop job #{on_stop} needs to have action stop defined")
end
end
def error!(message)
raise ValidationError.new(message)
end
end
end
end
# frozen_string_literal: true
# A data object that wraps `Ci::Config` and any messages
# (errors, warnings) generated by the YamlProcessor.
module Gitlab
module Ci
class YamlProcessor
class Result
attr_reader :errors, :warnings
def initialize(ci_config: nil, errors: [], warnings: [])
@ci_config = ci_config
@errors = errors || []
@warnings = warnings || []
end
def valid?
errors.empty?
end
def stages_attributes
stages.uniq.map do |stage|
seeds = stage_builds_attributes(stage)
{ name: stage, index: stages.index(stage), builds: seeds }
end
end
def builds
jobs.map do |name, _|
build_attributes(name)
end
end
def stage_builds_attributes(stage)
jobs.values
.select { |job| job[:stage] == stage }
.map { |job| build_attributes(job[:name]) }
end
def workflow_attributes
{
rules: hash_config.dig(:workflow, :rules),
yaml_variables: transform_to_yaml_variables(variables)
}
end
def jobs
@jobs ||= @ci_config.normalized_jobs
end
def stages
@stages ||= @ci_config.stages
end
def build_attributes(name)
job = jobs.fetch(name.to_sym, {})
{ stage_idx: stages.index(job[:stage]),
stage: job[:stage],
tag_list: job[:tags],
name: job[:name].to_s,
allow_failure: job[:ignore],
when: job[:when] || 'on_success',
environment: job[:environment_name],
coverage_regex: job[:coverage],
yaml_variables: transform_to_yaml_variables(job[:variables]),
needs_attributes: job.dig(:needs, :job),
interruptible: job[:interruptible],
only: job[:only],
except: job[:except],
rules: job[:rules],
cache: job[:cache],
resource_group_key: job[:resource_group],
scheduling_type: job[:scheduling_type],
secrets: job[:secrets],
options: {
image: job[:image],
services: job[:services],
artifacts: job[:artifacts],
dependencies: job[:dependencies],
cross_dependencies: job.dig(:needs, :cross_dependency),
job_timeout: job[:timeout],
before_script: job[:before_script],
script: job[:script],
after_script: job[:after_script],
environment: job[:environment],
retry: job[:retry],
parallel: job[:parallel],
instance: job[:instance],
start_in: job[:start_in],
trigger: job[:trigger],
bridge_needs: job.dig(:needs, :bridge)&.first,
release: release(job)
}.compact }.compact
end
private
def variables
@variables ||= @ci_config.variables
end
def hash_config
@hash_config ||= @ci_config.to_hash
end
def release(job)
job[:release]
end
def transform_to_yaml_variables(variables)
variables.to_h.map do |key, value|
{ key: key.to_s, value: value, public: true }
end
end
end
end
end
end
......@@ -98,7 +98,7 @@ RSpec.describe Projects::Ci::LintsController do
it_behaves_like 'returns a successful validation'
it 'runs validations through YamlProcessor' do
expect(Gitlab::Ci::YamlProcessor).to receive(:new_with_validation_errors).and_call_original
expect(Gitlab::Ci::YamlProcessor).to receive(:new).and_call_original
subject
end
......@@ -126,7 +126,7 @@ RSpec.describe Projects::Ci::LintsController do
it_behaves_like 'returns a successful validation'
it 'runs validations through YamlProcessor' do
expect(Gitlab::Ci::YamlProcessor).to receive(:new_with_validation_errors).and_call_original
expect(Gitlab::Ci::YamlProcessor).to receive(:new).and_call_original
subject
end
......
......@@ -264,5 +264,11 @@ RSpec.describe Gitlab::Ci::Config::Normalizer do
is_expected.to match(config)
end
end
context 'when jobs config is nil' do
let(:config) { nil }
it { is_expected.to eq({}) }
end
end
end
......@@ -312,7 +312,7 @@ RSpec.describe Gitlab::Ci::Config do
HEREDOC
end
it 'raises error YamlProcessor validationError' do
it 'raises ConfigError' do
expect { config }.to raise_error(
described_class::ConfigError,
"Included file `invalid` does not have YAML extension!"
......@@ -329,7 +329,7 @@ RSpec.describe Gitlab::Ci::Config do
HEREDOC
end
it 'raises error YamlProcessor validationError' do
it 'raises ConfigError' do
expect { config }.to raise_error(
described_class::ConfigError,
'Include `{"remote":"http://url","local":"/local/file.yml"}` needs to match exactly one accessor!'
......
......@@ -157,7 +157,7 @@ RSpec.describe Gitlab::Ci::Lint do
it 'uses YamlProcessor' do
expect(Gitlab::Ci::YamlProcessor)
.to receive(:new_with_validation_errors)
.to receive(:new)
.and_call_original
subject
......
......@@ -31,20 +31,20 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Validate::External do
CI_YAML
end
let(:yaml_processor) do
let(:yaml_processor_result) do
::Gitlab::Ci::YamlProcessor.new(
ci_yaml, {
project: project,
sha: pipeline.sha,
user: user
}
)
).execute
end
let(:save_incompleted) { true }
let(:command) do
Gitlab::Ci::Pipeline::Chain::Command.new(
project: project, current_user: user, config_processor: yaml_processor, save_incompleted: save_incompleted
project: project, current_user: user, config_processor: yaml_processor_result, save_incompleted: save_incompleted
)
end
......
......@@ -3,7 +3,7 @@
require 'spec_helper'
RSpec.describe 'CI YML Templates' do
subject { Gitlab::Ci::YamlProcessor.new(content) }
subject { Gitlab::Ci::YamlProcessor.new(content).execute }
let(:all_templates) { Gitlab::Template::GitlabCiYmlTemplate.all.map(&:full_name) }
......@@ -33,7 +33,7 @@ RSpec.describe 'CI YML Templates' do
end
it 'is valid' do
expect { subject }.not_to raise_error
expect(subject).to be_valid
end
it 'require default stages to be included' do
......
This diff is collapsed.
......@@ -133,12 +133,6 @@ RSpec.describe Gitlab::ImportExport::FastHashSerializer do
expect(builds_count).to eq(1)
end
it 'has no when YML attributes but only the DB column' do
expect_any_instance_of(Gitlab::Ci::YamlProcessor).not_to receive(:build_attributes)
subject
end
it 'has pipeline commits' do
expect(subject['ci_pipelines']).not_to be_empty
end
......
......@@ -381,12 +381,6 @@ RSpec.describe Gitlab::ImportExport::Project::TreeSaver do
expect(project_tree_saver.save).to be true
end
it 'has no when YML attributes but only the DB column' do
expect_any_instance_of(Gitlab::Ci::YamlProcessor).not_to receive(:build_attributes)
project_tree_saver.save
end
end
end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment