Commit d386a553 authored by Kamil Trzciński's avatar Kamil Trzciński

Merge branch 'revert-9ec3af0d' into 'master'

Revert "Merge branch 'ci-resource-group-model' into 'master'"

See merge request gitlab-org/gitlab!21833
parents ecf2d069 44cdba07
......@@ -23,7 +23,6 @@ module Ci
belongs_to :runner
belongs_to :trigger_request
belongs_to :erased_by, class_name: 'User'
belongs_to :resource_group, class_name: 'Ci::ResourceGroup', inverse_of: :builds
RUNNER_FEATURES = {
upload_multiple_artifacts: -> (build) { build.publishes_artifacts_reports? },
......@@ -35,7 +34,6 @@ module Ci
}.freeze
has_one :deployment, as: :deployable, class_name: 'Deployment'
has_one :resource, class_name: 'Ci::Resource', inverse_of: :build
has_many :trace_sections, class_name: 'Ci::BuildTraceSection'
has_many :trace_chunks, class_name: 'Ci::BuildTraceChunk', foreign_key: :build_id
......@@ -443,15 +441,6 @@ module Ci
environment.present?
end
def requires_resource?
Feature.enabled?(:ci_resource_group, project) &&
self.resource_group_id.present? && resource.nil?
end
def retains_resource?
self.resource_group_id.present? && resource.present?
end
def starts_environment?
has_environment? && self.environment_action == 'start'
end
......
# frozen_string_literal: true
module Ci
class Resource < ApplicationRecord
extend Gitlab::Ci::Model
belongs_to :resource_group, class_name: 'Ci::ResourceGroup', inverse_of: :resources
belongs_to :build, class_name: 'Ci::Build', inverse_of: :resource
scope :free, -> { where(build: nil) }
scope :retained_by, -> (build) { where(build: build) }
end
end
# frozen_string_literal: true
module Ci
class ResourceGroup < ApplicationRecord
extend Gitlab::Ci::Model
belongs_to :project, inverse_of: :resource_groups
has_many :resources, class_name: 'Ci::Resource', inverse_of: :resource_group
has_many :builds, class_name: 'Ci::Build', inverse_of: :resource_group
validates :key,
length: { maximum: 255 },
format: { with: Gitlab::Regex.environment_name_regex,
message: Gitlab::Regex.environment_name_regex_message }
before_create :ensure_resource
def retain_resource_for(build)
resources.free.limit(1).update_all(build_id: build.id) > 0
end
def release_resource_from(build)
resources.retained_by(build).update_all(build_id: nil) > 0
end
private
def ensure_resource
# Currently we only support one resource per group, which means
# maximum one build can be set to the resource group, thus builds
# belong to the same resource group are executed once at time.
self.resources.build if self.resources.empty?
end
end
end
......@@ -285,7 +285,6 @@ class Project < ApplicationRecord
has_many :pipeline_schedules, class_name: 'Ci::PipelineSchedule'
has_many :project_deploy_tokens
has_many :deploy_tokens, through: :project_deploy_tokens
has_many :resource_groups, class_name: 'Ci::ResourceGroup', inverse_of: :project
has_one :auto_devops, class_name: 'ProjectAutoDevops', inverse_of: :project, autosave: true
has_many :custom_attributes, class_name: 'ProjectCustomAttribute'
......
......@@ -5,7 +5,7 @@ module Ci
CLONE_ACCESSORS = %i[pipeline project ref tag options name
allow_failure stage stage_id stage_idx trigger_request
yaml_variables when environment coverage_regex
description tag_list protected needs resource_group].freeze
description tag_list protected needs].freeze
def execute(build)
reprocess!(build).tap do |new_build|
......
---
title: Add Ci Resource Group models
merge_request: 20950
author:
type: other
# frozen_string_literal: true
class AddCiResourceGroups < ActiveRecord::Migration[5.2]
DOWNTIME = false
def change
create_table :ci_resource_groups do |t|
t.timestamps_with_timezone
t.references :project, null: false, index: false, foreign_key: { on_delete: :cascade }
t.string :key, null: false, limit: 255
t.index %i[project_id key], unique: true
end
create_table :ci_resources do |t|
t.timestamps_with_timezone
t.references :resource_group, null: false, index: false, foreign_key: { to_table: :ci_resource_groups, on_delete: :cascade }
t.references :build, null: true, index: true, foreign_key: { to_table: :ci_builds, on_delete: :nullify }
t.index %i[resource_group_id build_id], unique: true
end
add_column :ci_builds, :resource_group_id, :bigint
add_column :ci_builds, :waiting_for_resource_at, :datetime_with_timezone
end
end
# frozen_string_literal: true
class AddIndexToResourceGroupId < ActiveRecord::Migration[5.2]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
INDEX_NAME = 'index_for_resource_group'.freeze
disable_ddl_transaction!
def up
add_concurrent_index :ci_builds, %i[resource_group_id id], where: 'resource_group_id IS NOT NULL', name: INDEX_NAME
add_concurrent_foreign_key :ci_builds, :ci_resource_groups, column: :resource_group_id, on_delete: :nullify
end
def down
remove_foreign_key_if_exists :ci_builds, column: :resource_group_id
remove_concurrent_index_by_name :ci_builds, INDEX_NAME
end
end
......@@ -684,8 +684,6 @@ ActiveRecord::Schema.define(version: 2019_12_08_071112) do
t.datetime_with_timezone "scheduled_at"
t.string "token_encrypted"
t.integer "upstream_pipeline_id"
t.bigint "resource_group_id"
t.datetime_with_timezone "waiting_for_resource_at"
t.index ["artifacts_expire_at"], name: "index_ci_builds_on_artifacts_expire_at", where: "(artifacts_file <> ''::text)"
t.index ["auto_canceled_by_id"], name: "index_ci_builds_on_auto_canceled_by_id"
t.index ["commit_id", "artifacts_expire_at", "id"], name: "index_ci_builds_on_commit_id_and_artifacts_expireatandidpartial", where: "(((type)::text = 'Ci::Build'::text) AND ((retried = false) OR (retried IS NULL)) AND ((name)::text = ANY (ARRAY[('sast'::character varying)::text, ('dependency_scanning'::character varying)::text, ('sast:container'::character varying)::text, ('container_scanning'::character varying)::text, ('dast'::character varying)::text])))"
......@@ -699,7 +697,6 @@ ActiveRecord::Schema.define(version: 2019_12_08_071112) do
t.index ["project_id"], name: "index_ci_builds_on_project_id_for_successfull_pages_deploy", where: "(((type)::text = 'GenericCommitStatus'::text) AND ((stage)::text = 'deploy'::text) AND ((name)::text = 'pages:deploy'::text) AND ((status)::text = 'success'::text))"
t.index ["protected"], name: "index_ci_builds_on_protected"
t.index ["queued_at"], name: "index_ci_builds_on_queued_at"
t.index ["resource_group_id", "id"], name: "index_for_resource_group", where: "(resource_group_id IS NOT NULL)"
t.index ["runner_id"], name: "index_ci_builds_on_runner_id"
t.index ["scheduled_at"], name: "partial_index_ci_builds_on_scheduled_at_with_scheduled_jobs", where: "((scheduled_at IS NOT NULL) AND ((type)::text = 'Ci::Build'::text) AND ((status)::text = 'scheduled'::text))"
t.index ["stage_id", "stage_idx"], name: "tmp_build_stage_position_index", where: "(stage_idx IS NOT NULL)"
......@@ -874,23 +871,6 @@ ActiveRecord::Schema.define(version: 2019_12_08_071112) do
t.index ["user_id"], name: "index_ci_pipelines_on_user_id"
end
create_table "ci_resource_groups", force: :cascade do |t|
t.datetime_with_timezone "created_at", null: false
t.datetime_with_timezone "updated_at", null: false
t.bigint "project_id", null: false
t.string "key", limit: 255, null: false
t.index ["project_id", "key"], name: "index_ci_resource_groups_on_project_id_and_key", unique: true
end
create_table "ci_resources", force: :cascade do |t|
t.datetime_with_timezone "created_at", null: false
t.datetime_with_timezone "updated_at", null: false
t.bigint "resource_group_id", null: false
t.bigint "build_id"
t.index ["build_id"], name: "index_ci_resources_on_build_id"
t.index ["resource_group_id", "build_id"], name: "index_ci_resources_on_resource_group_id_and_build_id", unique: true
end
create_table "ci_runner_namespaces", id: :serial, force: :cascade do |t|
t.integer "runner_id"
t.integer "namespace_id"
......@@ -4409,7 +4389,6 @@ ActiveRecord::Schema.define(version: 2019_12_08_071112) do
add_foreign_key "ci_builds", "ci_pipelines", column: "auto_canceled_by_id", name: "fk_a2141b1522", on_delete: :nullify
add_foreign_key "ci_builds", "ci_pipelines", column: "commit_id", name: "fk_d3130c9a7f", on_delete: :cascade
add_foreign_key "ci_builds", "ci_pipelines", column: "upstream_pipeline_id", name: "fk_87f4cefcda", on_delete: :cascade
add_foreign_key "ci_builds", "ci_resource_groups", column: "resource_group_id", name: "fk_6661f4f0e8", on_delete: :nullify
add_foreign_key "ci_builds", "ci_stages", column: "stage_id", name: "fk_3a9eaa254d", on_delete: :cascade
add_foreign_key "ci_builds", "projects", name: "fk_befce0568a", on_delete: :cascade
add_foreign_key "ci_builds_metadata", "ci_builds", column: "build_id", on_delete: :cascade
......@@ -4430,9 +4409,6 @@ ActiveRecord::Schema.define(version: 2019_12_08_071112) do
add_foreign_key "ci_pipelines", "external_pull_requests", name: "fk_190998ef09", on_delete: :nullify
add_foreign_key "ci_pipelines", "merge_requests", name: "fk_a23be95014", on_delete: :cascade
add_foreign_key "ci_pipelines", "projects", name: "fk_86635dbd80", on_delete: :cascade
add_foreign_key "ci_resource_groups", "projects", on_delete: :cascade
add_foreign_key "ci_resources", "ci_builds", column: "build_id", on_delete: :nullify
add_foreign_key "ci_resources", "ci_resource_groups", column: "resource_group_id", on_delete: :cascade
add_foreign_key "ci_runner_namespaces", "ci_runners", column: "runner_id", on_delete: :cascade
add_foreign_key "ci_runner_namespaces", "namespaces", on_delete: :cascade
add_foreign_key "ci_runner_projects", "projects", name: "fk_4478a6f1e4", on_delete: :cascade
......
......@@ -16,8 +16,7 @@ module Gitlab
ALLOWED_KEYS = %i[tags script only except rules type image services
allow_failure type stage when start_in artifacts cache
dependencies before_script needs after_script variables
environment coverage retry parallel extends interruptible timeout
resource_group].freeze
environment coverage retry parallel extends interruptible timeout].freeze
REQUIRED_BY_NEEDS = %i[stage].freeze
......@@ -49,7 +48,6 @@ module Gitlab
validates :dependencies, array_of_strings: true
validates :extends, array_of_strings_or_string: true
validates :rules, array_of_hashes: true
validates :resource_group, type: String
end
validates :start_in, duration: { limit: '1 week' }, if: :delayed?
......@@ -158,7 +156,7 @@ module Gitlab
attributes :script, :tags, :allow_failure, :when, :dependencies,
:needs, :retry, :parallel, :extends, :start_in, :rules,
:interruptible, :timeout, :resource_group
:interruptible, :timeout
def self.matching?(name, config)
!name.to_s.start_with?('.') &&
......@@ -245,8 +243,7 @@ module Gitlab
artifacts: artifacts_value,
after_script: after_script_value,
ignore: ignored?,
needs: needs_defined? ? needs_value : nil,
resource_group: resource_group }
needs: needs_defined? ? needs_value : nil }
end
end
end
......
......@@ -18,7 +18,6 @@ module Gitlab
@seed_attributes = attributes
@previous_stages = previous_stages
@needs_attributes = dig(:needs_attributes)
@resource_group_key = attributes.delete(:resource_group_key)
@using_rules = attributes.key?(:rules)
@using_only = attributes.key?(:only)
......@@ -79,7 +78,6 @@ module Gitlab
else
::Ci::Build.new(attributes).tap do |job|
job.deployment = Seed::Deployment.new(job).to_resource
job.resource_group = Seed::Build::ResourceGroup.new(job, @resource_group_key).to_resource
end
end
end
......
# frozen_string_literal: true
module Gitlab
module Ci
module Pipeline
module Seed
class Build
class ResourceGroup < Seed::Base
include Gitlab::Utils::StrongMemoize
attr_reader :build, :resource_group_key
def initialize(build, resource_group_key)
@build = build
@resource_group_key = resource_group_key
end
def to_resource
return unless Feature.enabled?(:ci_resource_group, build.project)
return unless resource_group_key.present?
resource_group = build.project.resource_groups
.safe_find_or_create_by(key: expanded_resource_group_key)
resource_group if resource_group.persisted?
end
private
def expanded_resource_group_key
strong_memoize(:expanded_resource_group_key) do
ExpandVariables.expand(resource_group_key, -> { build.simple_variables })
end
end
end
end
end
end
end
end
......@@ -64,7 +64,6 @@ module Gitlab
except: job[:except],
rules: job[:rules],
cache: job[:cache],
resource_group_key: job[:resource_group],
options: {
image: job[:image],
services: job[:services],
......
......@@ -203,8 +203,6 @@ excluded_attributes:
- :artifacts_metadata_store
- :artifacts_size
- :commands
- :resource_group_id
- :waiting_for_resource_at
push_event_payload:
- :event_id
project_badges:
......
......@@ -207,14 +207,6 @@ FactoryBot.define do
trigger_request factory: :ci_trigger_request
end
trait :resource_group do
waiting_for_resource_at { 5.minutes.ago }
after(:build) do |build, evaluator|
build.resource_group = create(:ci_resource_group, project: build.project)
end
end
after(:build) do |build, evaluator|
build.project ||= build.pipeline.project
end
......
# frozen_string_literal: true
FactoryBot.define do
factory :ci_resource, class: Ci::Resource do
resource_group factory: :ci_resource_group
trait(:retained) do
build factory: :ci_build
end
end
end
# frozen_string_literal: true
FactoryBot.define do
factory :ci_resource_group, class: Ci::ResourceGroup do
project
sequence(:key) { |n| "IOS_#{n}" }
end
end
# frozen_string_literal: true
require 'spec_helper'
describe Gitlab::Ci::Pipeline::Seed::Build::ResourceGroup do
let_it_be(:project) { create(:project) }
let(:job) { build(:ci_build, project: project) }
let(:seed) { described_class.new(job, resource_group_key) }
describe '#to_resource' do
subject { seed.to_resource }
context 'when resource group key is specified' do
let(:resource_group_key) { 'iOS' }
it 'returns a resource group object' do
is_expected.to be_a(Ci::ResourceGroup)
expect(subject.key).to eq('iOS')
end
context 'when environment has an invalid URL' do
let(:resource_group_key) { ':::' }
it 'returns nothing' do
is_expected.to be_nil
end
end
context 'when there is a resource group already' do
let!(:resource_group) { create(:ci_resource_group, project: project, key: 'iOS') }
it 'does not create a new resource group' do
expect { subject }.not_to change { Ci::ResourceGroup.count }
end
end
end
context 'when resource group key is nil' do
let(:resource_group_key) { nil }
it 'returns nothing' do
is_expected.to be_nil
end
end
end
end
......@@ -231,15 +231,6 @@ describe Gitlab::Ci::Pipeline::Seed::Build do
end
end
end
context 'when job belongs to a resource group' do
let(:attributes) { { name: 'rspec', ref: 'master', resource_group_key: 'iOS' } }
it 'returns a job with resource group' do
expect(subject.resource_group).not_to be_nil
expect(subject.resource_group.key).to eq('iOS')
end
end
end
context 'when job is a bridge' do
......
......@@ -241,21 +241,6 @@ module Gitlab
end
end
end
describe 'resource group' do
context 'when resource group is defined' do
let(:config) do
YAML.dump(rspec: {
script: 'test',
resource_group: 'iOS'
})
end
it 'has the attributes' do
expect(subject[:resource_group_key]).to eq 'iOS'
end
end
end
end
describe '#stages_attributes' do
......
......@@ -444,7 +444,6 @@ project:
- service_desk_setting
- import_failures
- container_expiration_policy
- resource_groups
award_emoji:
- awardable
- user
......
......@@ -1275,68 +1275,6 @@ describe Ci::Build do
end
end
describe '#requires_resource?' do
subject { build.requires_resource? }
context 'when build needs a resource from a resource group' do
let(:resource_group) { create(:ci_resource_group, project: project) }
let(:build) { create(:ci_build, resource_group: resource_group, project: project) }
context 'when build has not retained a resource' do
it { is_expected.to eq(true) }
end
context 'when build has retained a resource' do
before do
resource_group.retain_resource_for(build)
end
it { is_expected.to eq(false) }
context 'when ci_resource_group feature flag is disabled' do
before do
stub_feature_flags(ci_resource_group: false)
end
it { is_expected.to eq(false) }
end
end
end
context 'when build does not need a resource from a resource group' do
let(:build) { create(:ci_build, project: project) }
it { is_expected.to eq(false) }
end
end
describe '#retains_resource?' do
subject { build.retains_resource? }
context 'when build needs a resource from a resource group' do
let(:resource_group) { create(:ci_resource_group, project: project) }
let(:build) { create(:ci_build, resource_group: resource_group, project: project) }
context 'when build has retained a resource' do
before do
resource_group.retain_resource_for(build)
end
it { is_expected.to eq(true) }
end
context 'when build has not retained a resource' do
it { is_expected.to eq(false) }
end
end
context 'when build does not need a resource from a resource group' do
let(:build) { create(:ci_build, project: project) }
it { is_expected.to eq(false) }
end
end
describe '#stops_environment?' do
subject { build.stops_environment? }
......
# frozen_string_literal: true
require 'spec_helper'
describe Ci::ResourceGroup do
describe 'validation' do
it 'valids when key includes allowed character' do
resource_group = build(:ci_resource_group, key: 'test')
expect(resource_group).to be_valid
end
it 'invalids when key includes invalid character' do
resource_group = build(:ci_resource_group, key: ':::')
expect(resource_group).not_to be_valid
end
end
describe '#ensure_resource' do
it 'creates one resource when resource group is created' do
resource_group = create(:ci_resource_group)
expect(resource_group.resources.count).to eq(1)
expect(resource_group.resources.all?(&:persisted?)).to eq(true)
end
end
describe '#retain_resource_for' do
subject { resource_group.retain_resource_for(build) }
let(:build) { create(:ci_build) }
let(:resource_group) { create(:ci_resource_group) }
it 'retains resource for the build' do
expect(resource_group.resources.first.build).to be_nil
is_expected.to eq(true)
expect(resource_group.resources.first.build).to eq(build)
end
context 'when there are no free resources' do
before do
resource_group.retain_resource_for(create(:ci_build))
end
it 'fails to retain resource' do
is_expected.to eq(false)
end
end
context 'when the build has already retained a resource' do
let!(:another_resource) { create(:ci_resource, resource_group: resource_group, build: build) }
it 'fails to retain resource' do
expect { subject }.to raise_error(ActiveRecord::RecordNotUnique)
end
end
end
describe '#release_resource_from' do
subject { resource_group.release_resource_from(build) }
let(:build) { create(:ci_build) }
let(:resource_group) { create(:ci_resource_group) }
context 'when the build has already retained a resource' do
before do
resource_group.retain_resource_for(build)
end
it 'releases resource from the build' do
expect(resource_group.resources.first.build).to eq(build)
is_expected.to eq(true)
expect(resource_group.resources.first.build).to be_nil
end
end
context 'when the build has already released a resource' do
it 'fails to release resource' do
is_expected.to eq(false)
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
describe Ci::Resource do
describe '.free' do
subject { described_class.free }
let(:resource_group) { create(:ci_resource_group) }
let!(:free_resource) { resource_group.resources.take }
let!(:retained_resource) { create(:ci_resource, :retained, resource_group: resource_group) }
it 'returns free resources' do
is_expected.to eq([free_resource])
end
end
describe '.retained_by' do
subject { described_class.retained_by(build) }
let(:build) { create(:ci_build) }
let!(:resource) { create(:ci_resource, build: build) }
it 'returns retained resources' do
is_expected.to eq([resource])
end
end
end
......@@ -914,44 +914,6 @@ describe Ci::CreatePipelineService do
end
end
context 'with resource group' do
context 'when resource group is defined' do
before do
config = YAML.dump(
test: { stage: 'test', script: 'ls', resource_group: resource_group_key }
)
stub_ci_pipeline_yaml_file(config)
end
let(:resource_group_key) { 'iOS' }
it 'persists the association correctly' do
result = execute_service
deploy_job = result.builds.find_by_name!(:test)
resource_group = project.resource_groups.find_by_key!(resource_group_key)
expect(result).to be_persisted
expect(deploy_job.resource_group.key).to eq(resource_group_key)
expect(project.resource_groups.count).to eq(1)
expect(resource_group.builds.count).to eq(1)
expect(resource_group.resources.count).to eq(1)
expect(resource_group.resources.first.build).to eq(nil)
end
context 'when resourc group key includes predefined variables' do
let(:resource_group_key) { '$CI_COMMIT_REF_NAME-$CI_JOB_NAME' }
it 'interpolates the variables into the key correctly' do
result = execute_service
expect(result).to be_persisted
expect(project.resource_groups.exists?(key: 'master-test')).to eq(true)
end
end
end
end
context 'with timeout' do
context 'when builds with custom timeouts are configured' do
before do
......
......@@ -31,7 +31,7 @@ describe Ci::RetryBuildService do
job_artifacts_container_scanning job_artifacts_dast
job_artifacts_license_management job_artifacts_performance
job_artifacts_codequality job_artifacts_metrics scheduled_at
job_variables waiting_for_resource_at].freeze
job_variables].freeze
IGNORE_ACCESSORS =
%i[type lock_version target_url base_tags trace_sections
......@@ -40,14 +40,14 @@ describe Ci::RetryBuildService do
user_id auto_canceled_by_id retried failure_reason
sourced_pipelines artifacts_file_store artifacts_metadata_store
metadata runner_session trace_chunks upstream_pipeline_id
artifacts_file artifacts_metadata artifacts_size commands resource resource_group_id].freeze
artifacts_file artifacts_metadata artifacts_size commands].freeze
shared_examples 'build duplication' do
let(:another_pipeline) { create(:ci_empty_pipeline, project: project) }
let(:build) do
create(:ci_build, :failed, :expired, :erased, :queued, :coverage, :tags,
:allowed_to_fail, :on_tag, :triggered, :teardown_environment, :resource_group,
:allowed_to_fail, :on_tag, :triggered, :teardown_environment,
description: 'my-job', stage: 'test', stage_id: stage.id,
pipeline: pipeline, auto_canceled_by: another_pipeline,
scheduled_at: 10.seconds.since)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment