Commit e8cadaa6 authored by Douglas Barbosa Alexandre's avatar Douglas Barbosa Alexandre

Merge branch 'mattkasa/handle-terraform-report' into 'master'

Add terraform_reports endpoint to MergeRequestsController

See merge request gitlab-org/gitlab!29088
parents 981cefbf 69099a4e
......@@ -14,7 +14,7 @@ class Projects::MergeRequestsController < Projects::MergeRequests::ApplicationCo
skip_before_action :merge_request, only: [:index, :bulk_update]
before_action :whitelist_query_limiting, only: [:assign_related_issues, :update]
before_action :authorize_update_issuable!, only: [:close, :edit, :update, :remove_wip, :sort]
before_action :authorize_read_actual_head_pipeline!, only: [:test_reports, :exposed_artifacts, :coverage_reports]
before_action :authorize_read_actual_head_pipeline!, only: [:test_reports, :exposed_artifacts, :coverage_reports, :terraform_reports]
before_action :set_issuables_index, only: [:index]
before_action :authenticate_user!, only: [:assign_related_issues]
before_action :check_user_can_push_to_source_branch!, only: [:rebase]
......@@ -143,6 +143,10 @@ class Projects::MergeRequestsController < Projects::MergeRequests::ApplicationCo
end
end
def terraform_reports
reports_response(@merge_request.find_terraform_reports)
end
def exposed_artifacts
if @merge_request.has_exposed_artifacts?
reports_response(@merge_request.find_exposed_artifacts)
......
......@@ -878,6 +878,14 @@ module Ci
coverage_report
end
def collect_terraform_reports!(terraform_reports)
each_report(::Ci::JobArtifact::TERRAFORM_REPORT_FILE_TYPES) do |file_type, blob, report_artifact|
::Gitlab::Ci::Parsers.fabricate!(file_type).parse!(blob, terraform_reports, artifact: report_artifact)
end
terraform_reports
end
def report_artifacts
job_artifacts.with_reports
end
......
......@@ -13,6 +13,7 @@ module Ci
TEST_REPORT_FILE_TYPES = %w[junit].freeze
COVERAGE_REPORT_FILE_TYPES = %w[cobertura].freeze
NON_ERASABLE_FILE_TYPES = %w[trace].freeze
TERRAFORM_REPORT_FILE_TYPES = %w[terraform].freeze
DEFAULT_FILE_NAMES = {
archive: nil,
metadata: nil,
......@@ -102,6 +103,10 @@ module Ci
with_file_types(COVERAGE_REPORT_FILE_TYPES)
end
scope :terraform_reports, -> do
with_file_types(TERRAFORM_REPORT_FILE_TYPES)
end
scope :erasable, -> do
types = self.file_types.reject { |file_type| NON_ERASABLE_FILE_TYPES.include?(file_type) }.values
......
......@@ -817,6 +817,14 @@ module Ci
end
end
def terraform_reports
::Gitlab::Ci::Reports::TerraformReports.new.tap do |terraform_reports|
builds.latest.with_reports(::Ci::JobArtifact.terraform_reports).each do |build|
build.collect_terraform_reports!(terraform_reports)
end
end
end
def has_exposed_artifacts?
complete? && builds.latest.with_exposed_artifacts.exists?
end
......
......@@ -1325,6 +1325,10 @@ class MergeRequest < ApplicationRecord
actual_head_pipeline&.has_reports?(Ci::JobArtifact.coverage_reports)
end
def has_terraform_reports?
actual_head_pipeline&.has_reports?(Ci::JobArtifact.terraform_reports)
end
# TODO: this method and compare_test_reports use the same
# result type, which is handled by the controller's #reports_response.
# we should minimize mistakes by isolating the common parts.
......@@ -1337,6 +1341,14 @@ class MergeRequest < ApplicationRecord
compare_reports(Ci::GenerateCoverageReportsService)
end
def find_terraform_reports
unless has_terraform_reports?
return { status: :error, status_reason: 'This merge request does not have terraform reports' }
end
compare_reports(Ci::GenerateTerraformReportsService)
end
def has_exposed_artifacts?
return false unless Feature.enabled?(:ci_expose_arbitrary_artifacts_in_mr, default_enabled: true)
......
......@@ -71,6 +71,12 @@ class MergeRequestPollWidgetEntity < Grape::Entity
end
end
expose :terraform_reports_path do |merge_request|
if merge_request.has_terraform_reports?
terraform_reports_project_merge_request_path(merge_request.project, merge_request, format: :json)
end
end
expose :exposed_artifacts_path do |merge_request|
if merge_request.has_exposed_artifacts?
exposed_artifacts_project_merge_request_path(merge_request.project, merge_request, format: :json)
......
# frozen_string_literal: true
module Ci
# TODO: a couple of points with this approach:
# + reuses existing architecture and reactive caching
# - it's not a report comparison and some comparing features must be turned off.
# see CompareReportsBaseService for more notes.
# issue: https://gitlab.com/gitlab-org/gitlab/issues/34224
class GenerateTerraformReportsService < CompareReportsBaseService
def execute(base_pipeline, head_pipeline)
{
status: :parsed,
key: key(base_pipeline, head_pipeline),
data: head_pipeline.terraform_reports.plans
}
rescue => e
Gitlab::ErrorTracking.track_exception(e, project_id: project.id)
{
status: :error,
key: key(base_pipeline, head_pipeline),
status_reason: _('An error occurred while fetching terraform reports.')
}
end
def latest?(base_pipeline, head_pipeline, data)
data&.fetch(:key, nil) == key(base_pipeline, head_pipeline)
end
end
end
......@@ -15,6 +15,7 @@ resources :merge_requests, concerns: :awardable, except: [:new, :create, :show],
get :test_reports
get :exposed_artifacts
get :coverage_reports
get :terraform_reports
scope constraints: ->(req) { req.format == :json }, as: :json do
get :commits
......
......@@ -10,7 +10,8 @@ module Gitlab
def self.parsers
{
junit: ::Gitlab::Ci::Parsers::Test::Junit,
cobertura: ::Gitlab::Ci::Parsers::Coverage::Cobertura
cobertura: ::Gitlab::Ci::Parsers::Coverage::Cobertura,
terraform: ::Gitlab::Ci::Parsers::Terraform::Tfplan
}
end
......
# frozen_string_literal: true
module Gitlab
module Ci
module Parsers
module Terraform
class Tfplan
TfplanParserError = Class.new(Gitlab::Ci::Parsers::ParserError)
def parse!(json_data, terraform_reports, artifact:)
tfplan = JSON.parse(json_data).tap do |parsed_data|
parsed_data['job_path'] = Gitlab::Routing.url_helpers.project_job_path(
artifact.job.project, artifact.job
)
end
raise TfplanParserError, 'Tfplan missing required key' unless valid_supported_keys?(tfplan)
terraform_reports.add_plan(artifact.filename, tfplan)
rescue JSON::ParserError
raise TfplanParserError, 'JSON parsing failed'
rescue
raise TfplanParserError, 'Tfplan parsing failed'
end
private
def valid_supported_keys?(tfplan)
tfplan.keys == %w[create update delete job_path]
end
end
end
end
end
end
# frozen_string_literal: true
module Gitlab
module Ci
module Reports
class TerraformReports
attr_reader :plans
def initialize
@plans = {}
end
def pick(keys)
terraform_plans = plans.select do |key|
keys.include?(key)
end
{ plans: terraform_plans }
end
def add_plan(name, plan)
plans[name] = plan
end
end
end
end
end
......@@ -1959,6 +1959,9 @@ msgstr ""
msgid "An error occurred while fetching sidebar data"
msgstr ""
msgid "An error occurred while fetching terraform reports."
msgstr ""
msgid "An error occurred while fetching the Service Desk address."
msgstr ""
......
......@@ -1114,6 +1114,150 @@ describe Projects::MergeRequestsController do
end
end
describe 'GET terraform_reports' do
let(:merge_request) do
create(:merge_request,
:with_merge_request_pipeline,
target_project: project,
source_project: project)
end
let(:pipeline) do
create(:ci_pipeline,
:success,
:with_terraform_reports,
project: merge_request.source_project,
ref: merge_request.source_branch,
sha: merge_request.diff_head_sha)
end
before do
allow_any_instance_of(MergeRequest)
.to receive(:find_terraform_reports)
.and_return(report)
allow_any_instance_of(MergeRequest)
.to receive(:actual_head_pipeline)
.and_return(pipeline)
end
subject do
get :terraform_reports, params: {
namespace_id: project.namespace.to_param,
project_id: project,
id: merge_request.iid
},
format: :json
end
describe 'permissions on a public project with private CI/CD' do
let(:project) { create :project, :repository, :public, :builds_private }
let(:report) { { status: :parsed, data: [] } }
context 'while signed out' do
before do
sign_out(user)
end
it 'responds with a 404' do
subject
expect(response).to have_gitlab_http_status(:not_found)
expect(response.body).to be_blank
end
end
context 'while signed in as an unrelated user' do
before do
sign_in(create(:user))
end
it 'responds with a 404' do
subject
expect(response).to have_gitlab_http_status(:not_found)
expect(response.body).to be_blank
end
end
end
context 'when pipeline has jobs with terraform reports' do
before do
allow_next_instance_of(MergeRequest) do |merge_request|
allow(merge_request).to receive(:has_terraform_reports?).and_return(true)
end
end
context 'when processing terraform reports is in progress' do
let(:report) { { status: :parsing } }
it 'sends polling interval' do
expect(Gitlab::PollingInterval).to receive(:set_header)
subject
end
it 'returns 204 HTTP status' do
subject
expect(response).to have_gitlab_http_status(:no_content)
end
end
context 'when processing terraform reports is completed' do
let(:report) { { status: :parsed, data: pipeline.terraform_reports.plans } }
it 'returns terraform reports' do
subject
expect(response).to have_gitlab_http_status(:ok)
expect(json_response).to match(
a_hash_including(
'tfplan.json' => hash_including(
'create' => 0,
'delete' => 0,
'update' => 1
)
)
)
end
end
context 'when user created corrupted terraform reports' do
let(:report) { { status: :error, status_reason: 'Failed to parse terraform reports' } }
it 'does not send polling interval' do
expect(Gitlab::PollingInterval).not_to receive(:set_header)
subject
end
it 'returns 400 HTTP status' do
subject
expect(response).to have_gitlab_http_status(:bad_request)
expect(json_response).to eq({ 'status_reason' => 'Failed to parse terraform reports' })
end
end
end
context 'when pipeline does not have jobs with terraform reports' do
before do
allow_next_instance_of(MergeRequest) do |merge_request|
allow(merge_request).to receive(:has_terraform_reports?).and_return(false)
end
end
let(:report) { { status: :error } }
it 'returns error' do
subject
expect(response).to have_gitlab_http_status(:bad_request)
end
end
end
describe 'GET test_reports' do
let(:merge_request) do
create(:merge_request,
......
......@@ -320,6 +320,12 @@ FactoryBot.define do
end
end
trait :terraform_reports do
after(:build) do |build|
build.job_artifacts << create(:ci_job_artifact, :terraform, job: build)
end
end
trait :expired do
artifacts_expire_at { 1.minute.ago }
end
......
......@@ -149,6 +149,26 @@ FactoryBot.define do
end
end
trait :terraform do
file_type { :terraform }
file_format { :raw }
after(:build) do |artifact, evaluator|
artifact.file = fixture_file_upload(
Rails.root.join('spec/fixtures/terraform/tfplan.json'), 'application/json')
end
end
trait :terraform_with_corrupted_data do
file_type { :terraform }
file_format { :raw }
after(:build) do |artifact, evaluator|
artifact.file = fixture_file_upload(
Rails.root.join('spec/fixtures/terraform/tfplan_with_corrupted_data.json'), 'application/json')
end
end
trait :coverage_gocov_xml do
file_type { :cobertura }
file_format { :gzip }
......
......@@ -83,6 +83,14 @@ FactoryBot.define do
end
end
trait :with_terraform_reports do
status { :success }
after(:build) do |pipeline, evaluator|
pipeline.builds << build(:ci_build, :terraform_reports, pipeline: pipeline, project: pipeline.project)
end
end
trait :with_exposed_artifacts do
status { :success }
......
......@@ -133,6 +133,18 @@ FactoryBot.define do
end
end
trait :with_terraform_reports do
after(:build) do |merge_request|
merge_request.head_pipeline = build(
:ci_pipeline,
:success,
:with_terraform_reports,
project: merge_request.source_project,
ref: merge_request.source_branch,
sha: merge_request.diff_head_sha)
end
end
trait :with_exposed_artifacts do
after(:build) do |merge_request|
merge_request.head_pipeline = build(
......
{"create": 0, "update": 1, "delete": 0}
# frozen_string_literal: true
require 'spec_helper'
describe Gitlab::Ci::Parsers::Terraform::Tfplan do
describe '#parse!' do
let_it_be(:artifact) { create(:ci_job_artifact, :terraform) }
let(:reports) { Gitlab::Ci::Reports::TerraformReports.new }
context 'when data is tfplan.json' do
context 'when there is no data' do
it 'raises an error' do
plan = '{}'
expect { subject.parse!(plan, reports, artifact: artifact) }.to raise_error(
described_class::TfplanParserError
)
end
end
context 'when there is data' do
it 'parses JSON and returns a report' do
plan = '{ "create": 0, "update": 1, "delete": 0 }'
expect { subject.parse!(plan, reports, artifact: artifact) }.not_to raise_error
expect(reports.plans).to match(
a_hash_including(
'tfplan.json' => a_hash_including(
'create' => 0,
'update' => 1,
'delete' => 0
)
)
)
end
end
end
context 'when data is not tfplan.json' do
it 'raises an error' do
plan = { 'create' => 0, 'update' => 1, 'delete' => 0 }.to_s
expect { subject.parse!(plan, reports, artifact: artifact) }.to raise_error(
described_class::TfplanParserError
)
end
end
end
end
......@@ -22,6 +22,14 @@ describe Gitlab::Ci::Parsers do
end
end
context 'when file_type is terraform' do
let(:file_type) { 'terraform' }
it 'fabricates the class' do
is_expected.to be_a(described_class::Terraform::Tfplan)
end
end
context 'when file_type does not exist' do
let(:file_type) { 'undefined' }
......
# frozen_string_literal: true
require 'spec_helper'
describe Gitlab::Ci::Reports::TerraformReports do
it 'initializes plans with and empty hash' do
expect(subject.plans).to eq({})
end
describe '#add_plan' do
context 'when providing two unique plans' do
it 'returns two plans' do
subject.add_plan('a/tfplan.json', { 'create' => 0, 'update' => 1, 'delete' => 0 })
subject.add_plan('b/tfplan.json', { 'create' => 0, 'update' => 1, 'delete' => 0 })
expect(subject.plans).to eq({
'a/tfplan.json' => { 'create' => 0, 'update' => 1, 'delete' => 0 },
'b/tfplan.json' => { 'create' => 0, 'update' => 1, 'delete' => 0 }
})
end
end
context 'when providing the same plan twice' do
it 'returns the last added plan' do
subject.add_plan('tfplan.json', { 'create' => 0, 'update' => 0, 'delete' => 0 })
subject.add_plan('tfplan.json', { 'create' => 0, 'update' => 1, 'delete' => 0 })
expect(subject.plans).to eq({
'tfplan.json' => { 'create' => 0, 'update' => 1, 'delete' => 0 }
})
end
end
end
end
......@@ -3866,6 +3866,48 @@ describe Ci::Build do
end
end
describe '#collect_terraform_reports!' do
let(:terraform_reports) { Gitlab::Ci::Reports::TerraformReports.new }
it 'returns an empty hash' do
expect(build.collect_terraform_reports!(terraform_reports).plans).to eq({})
end
context 'when build has a terraform report' do
context 'when there is a valid tfplan.json' do
before do
create(:ci_job_artifact, :terraform, job: build, project: build.project)
end
it 'parses blobs and add the results to the terraform report' do
expect { build.collect_terraform_reports!(terraform_reports) }.not_to raise_error
expect(terraform_reports.plans).to match(
a_hash_including(
'tfplan.json' => a_hash_including(
'create' => 0,
'update' => 1,
'delete' => 0
)
)
)
end
end
context 'when there is an invalid tfplan.json' do
before do
create(:ci_job_artifact, :terraform_with_corrupted_data, job: build, project: build.project)
end
it 'raises an error' do
expect { build.collect_terraform_reports!(terraform_reports) }.to raise_error(
Gitlab::Ci::Parsers::Terraform::Tfplan::TfplanParserError
)
end
end
end
end
describe '#report_artifacts' do
subject { build.report_artifacts }
......
......@@ -86,6 +86,22 @@ describe Ci::JobArtifact do
end
end
describe '.terraform_reports' do
context 'when there is a terraform report' do
it 'return the job artifact' do
artifact = create(:ci_job_artifact, :terraform)
expect(described_class.terraform_reports).to eq([artifact])
end
end
context 'when there are no terraform reports' do
it 'return the an empty array' do
expect(described_class.terraform_reports).to eq([])
end
end
end
describe '.erasable' do
subject { described_class.erasable }
......
......@@ -364,6 +364,16 @@ describe Ci::Pipeline, :mailer do
end
end
context 'when pipeline has a terraform report' do
it 'selects the pipeline' do
pipeline_with_report = create(:ci_pipeline, :with_terraform_reports)
expect(described_class.with_reports(Ci::JobArtifact.terraform_reports)).to eq(
[pipeline_with_report]
)
end
end
context 'when pipeline does not have metrics reports' do
subject { described_class.with_reports(Ci::JobArtifact.test_reports) }
......
......@@ -1628,6 +1628,26 @@ describe MergeRequest do
end
end
describe '#has_terraform_reports?' do
let_it_be(:project) { create(:project, :repository) }
context 'when head pipeline has terraform reports' do
it 'returns true' do
merge_request = create(:merge_request, :with_terraform_reports, source_project: project)
expect(merge_request.has_terraform_reports?).to be_truthy
end
end
context 'when head pipeline does not have terraform reports' do
it 'returns false' do
merge_request = create(:merge_request, source_project: project)
expect(merge_request.has_terraform_reports?).to be_falsey
end
end
end
describe '#calculate_reactive_cache' do
let(:project) { create(:project, :repository) }
let(:merge_request) { create(:merge_request, source_project: project) }
......
......@@ -71,6 +71,28 @@ describe MergeRequestPollWidgetEntity do
end
end
describe 'terraform_reports_path' do
context 'when merge request has terraform reports' do
before do
allow(resource).to receive(:has_terraform_reports?).and_return(true)
end
it 'set the path to poll data' do
expect(subject[:terraform_reports_path]).to be_present
end
end
context 'when merge request has no terraform reports' do
before do
allow(resource).to receive(:has_terraform_reports?).and_return(false)
end
it 'set the path to poll data' do
expect(subject[:terraform_reports_path]).to be_nil
end
end
end
describe 'exposed_artifacts_path' do
context 'when merge request has exposed artifacts' do
before do
......
# frozen_string_literal: true
require 'spec_helper'
describe Ci::GenerateTerraformReportsService do
let_it_be(:project) { create(:project, :repository) }
describe '#execute' do
let_it_be(:merge_request) { create(:merge_request, :with_terraform_reports, source_project: project) }
subject { described_class.new(project, nil, id: merge_request.id) }
context 'when head pipeline has terraform reports' do
it 'returns status and data' do
result = subject.execute(nil, merge_request.head_pipeline)
expect(result).to match(
status: :parsed,
data: match(
a_hash_including('tfplan.json' => a_hash_including('create' => 0, 'update' => 1, 'delete' => 0))
),
key: an_instance_of(Array)
)
end
end
context 'when head pipeline has corrupted terraform reports' do
it 'returns status and error message' do
build = create(:ci_build, pipeline: merge_request.head_pipeline, project: project)
create(:ci_job_artifact, :terraform_with_corrupted_data, job: build, project: project)
result = subject.execute(nil, merge_request.head_pipeline)
expect(result).to match(
status: :error,
status_reason: 'An error occurred while fetching terraform reports.',
key: an_instance_of(Array)
)
end
end
end
describe '#latest?' do
let_it_be(:head_pipeline) { create(:ci_pipeline, :with_test_reports, project: project) }
subject { described_class.new(project) }
it 'returns true when cache key is latest' do
cache_key = subject.send(:key, nil, head_pipeline)
result = subject.latest?(nil, head_pipeline, key: cache_key)
expect(result).to eq(true)
end
it 'returns false when cache key is outdated' do
cache_key = subject.send(:key, nil, head_pipeline)
head_pipeline.update_column(:updated_at, 10.minutes.ago)
result = subject.latest?(nil, head_pipeline, key: cache_key)
expect(result).to eq(false)
end
it 'returns false when cache key is nil' do
result = subject.latest?(nil, head_pipeline, key: nil)
expect(result).to eq(false)
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment