Commit c6016ac8 authored by Sanad Liaquat's avatar Sanad Liaquat

Merge branch 'qa-ml-quarantine-context' into 'master'

[QA] Skip contexts in quarantine

See merge request gitlab-org/gitlab-ce!24942
parents c3c43be3 89314202
...@@ -19,26 +19,16 @@ RSpec.configure do |config| ...@@ -19,26 +19,16 @@ RSpec.configure do |config|
end end
end end
config.before do |example| config.before(:context) do
QA::Runtime::Logger.debug("Starting test: #{example.full_description}") if QA::Runtime::Env.debug? if self.class.metadata.keys.include?(:quarantine)
skip_or_run_quarantined_tests(self.class.metadata.keys, config.inclusion_filter.rules.keys)
# If quarantine is tagged, skip tests that have other metadata unless
# they're also tagged. This lets us run quarantined tests in a particular
# category without running tests in other categories.
# E.g., if a test is tagged 'smoke' and 'quarantine', and another is tagged
# 'ldap' and 'quarantine', if we wanted to run just quarantined smoke tests
# using `--tag quarantine --tag smoke`, without this check we'd end up
# running that ldap test as well.
if config.inclusion_filter[:quarantine]
skip("Running tests tagged with all of #{config.inclusion_filter.rules.keys}") unless quarantine_and_optional_other_tag?(example, config)
end end
end end
config.before(:each, :quarantine) do |example| config.before do |example|
# Skip tests in quarantine unless we explicitly focus on them QA::Runtime::Logger.debug("Starting test: #{example.full_description}") if QA::Runtime::Env.debug?
# We could use an exclusion filter, but this way the test report will list
# the quarantined tests when they're not run so that we're aware of them skip_or_run_quarantined_tests(example.metadata.keys, config.inclusion_filter.rules.keys)
skip('In quarantine') unless config.inclusion_filter[:quarantine]
end end
config.expect_with :rspec do |expectations| config.expect_with :rspec do |expectations|
...@@ -57,18 +47,41 @@ RSpec.configure do |config| ...@@ -57,18 +47,41 @@ RSpec.configure do |config|
Kernel.srand config.seed Kernel.srand config.seed
end end
# Skip tests in quarantine unless we explicitly focus on them.
# Skip the entire context if a context is tagged. This avoids running before
# blocks unnecessarily.
# If quarantine is focussed, skip tests/contexts that have other metadata
# unless they're also focussed. This lets us run quarantined tests in a
# particular category without running tests in other categories.
# E.g., if a test is tagged 'smoke' and 'quarantine', and another is tagged
# 'ldap' and 'quarantine', if we wanted to run just quarantined smoke tests
# using `--tag quarantine --tag smoke`, without this check we'd end up
# running that ldap test as well.
# We could use an exclusion filter, but this way the test report will list
# the quarantined tests when they're not run so that we're aware of them
def skip_or_run_quarantined_tests(metadata_keys, filter_keys)
included_filters = filters_other_than_quarantine(filter_keys)
if filter_keys.include?(:quarantine)
skip("Only running tests tagged with :quarantine and any of #{included_filters}") unless quarantine_and_optional_other_tag?(metadata_keys, included_filters)
else
skip('In quarantine') if metadata_keys.include?(:quarantine)
end
end
def filters_other_than_quarantine(filter_keys)
filter_keys.reject { |key| key == :quarantine }
end
# Checks if a test has the 'quarantine' tag and other tags in the inclusion filter. # Checks if a test has the 'quarantine' tag and other tags in the inclusion filter.
# #
# Returns true if # Returns true if
# - the example metadata includes the quarantine tag # - the metadata includes the quarantine tag
# - and the metadata and inclusion filter both have any other tag # - and the metadata and inclusion filter both have any other tag
# - or no other tags are in the inclusion filter # - or no other tags are in the inclusion filter
def quarantine_and_optional_other_tag?(example, config) def quarantine_and_optional_other_tag?(metadata_keys, included_filters)
return false unless example.metadata.keys.include? :quarantine return false unless metadata_keys.include? :quarantine
return true if included_filters.empty?
filters_other_than_quarantine = config.inclusion_filter.rules.keys.reject { |key| key == :quarantine }
return true if filters_other_than_quarantine.empty?
filters_other_than_quarantine.any? { |key| example.metadata.keys.include? key } included_filters.any? { |key| metadata_keys.include? key }
end end
...@@ -10,49 +10,82 @@ describe 'rspec config tests' do ...@@ -10,49 +10,82 @@ describe 'rspec config tests' do
end end
end end
context 'default' do
it_behaves_like 'passing tests'
end
context 'foo', :foo do context 'foo', :foo do
it_behaves_like 'passing tests' it_behaves_like 'passing tests'
end end
context 'default' do context 'quarantine', :quarantine do
it_behaves_like 'passing tests'
end
context 'bar quarantine', :bar, :quarantine do
it_behaves_like 'passing tests' it_behaves_like 'passing tests'
end end
end end
end end
context 'default config' do context 'with no tags focussed' do
it 'tests are skipped if in quarantine' do before do
group.run group.run
end
foo_context = group.children.find { |c| c.description == "foo" } context 'in a context tagged :foo' do
foo_examples = foo_context.descendant_filtered_examples it 'skips tests in quarantine' do
expect(foo_examples.count).to eq(2) context = group.children.find { |c| c.description == "foo" }
examples = context.descendant_filtered_examples
expect(examples.count).to eq(2)
ex = foo_examples.find { |e| e.description == "not in quarantine" } ex = examples.find { |e| e.description == "not in quarantine" }
expect(ex.execution_result.status).to eq(:passed) expect(ex.execution_result.status).to eq(:passed)
ex = foo_examples.find { |e| e.description == "in quarantine" } ex = examples.find { |e| e.description == "in quarantine" }
expect(ex.execution_result.status).to eq(:pending) expect(ex.execution_result.status).to eq(:pending)
expect(ex.execution_result.pending_message).to eq('In quarantine') expect(ex.execution_result.pending_message).to eq('In quarantine')
end
end
default_context = group.children.find { |c| c.description == "default" } context 'in an untagged context' do
default_examples = default_context.descendant_filtered_examples it 'skips tests in quarantine' do
expect(default_examples.count).to eq(2) context = group.children.find { |c| c.description == "default" }
examples = context.descendant_filtered_examples
expect(examples.count).to eq(2)
ex = default_examples.find { |e| e.description == "not in quarantine" } ex = examples.find { |e| e.description == "not in quarantine" }
expect(ex.execution_result.status).to eq(:passed) expect(ex.execution_result.status).to eq(:passed)
ex = default_examples.find { |e| e.description == "in quarantine" } ex = examples.find { |e| e.description == "in quarantine" }
expect(ex.execution_result.status).to eq(:pending) expect(ex.execution_result.status).to eq(:pending)
expect(ex.execution_result.pending_message).to eq('In quarantine') expect(ex.execution_result.pending_message).to eq('In quarantine')
end end
end end
context "with 'quarantine' tagged" do context 'in a context tagged :quarantine' do
it 'skips all tests' do
context = group.children.find { |c| c.description == "quarantine" }
examples = context.descendant_filtered_examples
expect(examples.count).to eq(2)
ex = examples.find { |e| e.description == "not in quarantine" }
expect(ex.execution_result.status).to eq(:pending)
ex = examples.find { |e| e.description == "in quarantine" }
expect(ex.execution_result.status).to eq(:pending)
expect(ex.execution_result.pending_message).to eq('In quarantine')
end
end
end
context 'with :quarantine focussed' do
before do before do
RSpec.configure do |config| RSpec.configure do |config|
config.inclusion_filter = :quarantine config.inclusion_filter = :quarantine
end end
group.run
end end
after do after do
RSpec.configure do |config| RSpec.configure do |config|
...@@ -60,26 +93,44 @@ describe 'rspec config tests' do ...@@ -60,26 +93,44 @@ describe 'rspec config tests' do
end end
end end
it "only quarantined tests are run" do context 'in an untagged context' do
group.run it 'only runs quarantined tests' do
context = group.children.find { |c| c.description == "default" }
examples = context.descendant_filtered_examples
expect(examples.count).to be(1)
ex = examples.find { |e| e.description == "in quarantine" }
expect(ex.execution_result.status).to eq(:passed)
end
end
foo_context = group.children.find { |c| c.description == "foo" } context 'in a context tagged :foo' do
foo_examples = foo_context.descendant_filtered_examples it 'only runs quarantined tests' do
expect(foo_examples.count).to be(1) context = group.children.find { |c| c.description == "foo" }
examples = context.descendant_filtered_examples
expect(examples.count).to be(1)
ex = foo_examples.find { |e| e.description == "in quarantine" } ex = examples.find { |e| e.description == "in quarantine" }
expect(ex.execution_result.status).to eq(:passed) expect(ex.execution_result.status).to eq(:passed)
end
end
context 'in a context tagged :quarantine' do
it 'runs all tests' do
context = group.children.find { |c| c.description == "quarantine" }
examples = context.descendant_filtered_examples
expect(examples.count).to be(2)
default_context = group.children.find { |c| c.description == "default" } ex = examples.find { |e| e.description == "in quarantine" }
default_examples = default_context.descendant_filtered_examples expect(ex.execution_result.status).to eq(:passed)
expect(default_examples.count).to be(1)
ex = default_examples.find { |e| e.description == "in quarantine" } ex = examples.find { |e| e.description == "not in quarantine" }
expect(ex.execution_result.status).to eq(:passed) expect(ex.execution_result.status).to eq(:passed)
end end
end end
end
context "with 'foo' tagged" do context 'with a non-quarantine tag (:foo) focussed' do
before do before do
RSpec.configure do |config| RSpec.configure do |config|
config.inclusion_filter = :foo config.inclusion_filter = :foo
...@@ -93,69 +144,43 @@ describe 'rspec config tests' do ...@@ -93,69 +144,43 @@ describe 'rspec config tests' do
end end
end end
it "tests are not run if not tagged 'foo'" do context 'in an untagged context' do
default_context = group.children.find { |c| c.description == "default" } it 'runs no tests' do
expect(default_context.descendant_filtered_examples.count).to eq(0) context = group.children.find { |c| c.description == "default" }
expect(context.descendant_filtered_examples.count).to eq(0)
end
end end
it "tests are skipped if in quarantine" do context 'in a context tagged :foo' do
foo_context = group.children.find { |c| c.description == "foo" } it 'skips quarantined tests' do
foo_examples = foo_context.descendant_filtered_examples context = group.children.find { |c| c.description == "foo" }
expect(foo_examples.count).to eq(2) examples = context.descendant_filtered_examples
expect(examples.count).to be(2)
ex = foo_examples.find { |e| e.description == "not in quarantine" } ex = examples.find { |e| e.description == "not in quarantine" }
expect(ex.execution_result.status).to eq(:passed) expect(ex.execution_result.status).to eq(:passed)
ex = foo_examples.find { |e| e.description == "in quarantine" } ex = examples.find { |e| e.description == "in quarantine" }
expect(ex.execution_result.status).to eq(:pending) expect(ex.execution_result.status).to eq(:pending)
expect(ex.execution_result.pending_message).to eq('In quarantine') expect(ex.execution_result.pending_message).to eq('In quarantine')
end end
end end
context "with 'quarantine' and 'foo' tagged" do context 'in a context tagged :quarantine' do
before do it 'runs no tests' do
RSpec.configure do |config| context = group.children.find { |c| c.description == "quarantine" }
config.inclusion_filter = { quarantine: true, foo: true } expect(context.descendant_filtered_examples.count).to eq(0)
end end
end end
after do
RSpec.configure do |config|
config.inclusion_filter.clear
end
end
it 'of tests tagged foo, only tests in quarantine run' do
group.run
foo_context = group.children.find { |c| c.description == "foo" }
foo_examples = foo_context.descendant_filtered_examples
expect(foo_examples.count).to eq(2)
ex = foo_examples.find { |e| e.description == "not in quarantine" }
expect(ex.execution_result.status).to eq(:pending)
expect(ex.execution_result.pending_message).to eq('Running tests tagged with all of [:quarantine, :foo]')
ex = foo_examples.find { |e| e.description == "in quarantine" }
expect(ex.execution_result.status).to eq(:passed)
end end
it 'if tests are not tagged they are skipped, even if they are in quarantine' do context 'with :quarantine and a non-quarantine tag (:foo) focussed' do
group.run
default_context = group.children.find { |c| c.description == "default" }
default_examples = default_context.descendant_filtered_examples
expect(default_examples.count).to eq(1)
ex = default_examples.find { |e| e.description == "in quarantine" }
expect(ex.execution_result.status).to eq(:pending)
expect(ex.execution_result.pending_message).to eq('Running tests tagged with all of [:quarantine, :foo]')
end
end
context "with 'foo' and 'bar' tagged" do
before do before do
RSpec.configure do |config| RSpec.configure do |config|
config.inclusion_filter = { bar: true, foo: true } config.inclusion_filter = { quarantine: true, foo: true }
end end
group.run
end end
after do after do
RSpec.configure do |config| RSpec.configure do |config|
...@@ -163,67 +188,67 @@ describe 'rspec config tests' do ...@@ -163,67 +188,67 @@ describe 'rspec config tests' do
end end
end end
it "runs tests tagged either 'foo' or 'bar'" do context 'in an untagged context' do
group = RSpec.describe do it 'ignores untagged tests and skips tests even if in quarantine' do
example 'foo', :foo do context = group.children.find { |c| c.description == "default" }
end examples = context.descendant_filtered_examples
example 'bar', :bar do expect(examples.count).to eq(1)
end
example 'foo and bar', :foo, :bar do ex = examples.find { |e| e.description == "in quarantine" }
expect(ex.execution_result.status).to eq(:pending)
end end
end end
group.run context 'in a context tagged :foo' do
expect(group.examples.count).to eq(3) it 'only runs quarantined tests' do
context = group.children.find { |c| c.description == "foo" }
examples = context.descendant_filtered_examples
expect(examples.count).to be(2)
ex = group.examples.find { |e| e.description == "foo" } ex = examples.find { |e| e.description == "in quarantine" }
expect(ex.execution_result.status).to eq(:passed) expect(ex.execution_result.status).to eq(:passed)
ex = group.examples.find { |e| e.description == "bar" } ex = examples.find { |e| e.description == "not in quarantine" }
expect(ex.execution_result.status).to eq(:passed) expect(ex.execution_result.status).to eq(:pending)
ex = group.examples.find { |e| e.description == "foo and bar" }
expect(ex.execution_result.status).to eq(:passed)
end
it "skips quarantined tests tagged 'foo' and/or 'bar'" do
group = RSpec.describe do
example 'foo in quarantine', :foo, :quarantine do
end
example 'foo and bar in quarantine', :foo, :bar, :quarantine do
end end
end end
group.run context 'in a context tagged :quarantine' do
expect(group.examples.count).to eq(2) it 'skips all tests' do
context = group.children.find { |c| c.description == "quarantine" }
examples = context.descendant_filtered_examples
expect(examples.count).to be(2)
ex = group.examples.find { |e| e.description == "foo in quarantine" } ex = examples.find { |e| e.description == "in quarantine" }
expect(ex.execution_result.status).to eq(:pending) expect(ex.execution_result.status).to eq(:pending)
expect(ex.execution_result.pending_message).to eq('In quarantine')
ex = group.examples.find { |e| e.description == "foo and bar in quarantine" } ex = examples.find { |e| e.description == "not in quarantine" }
expect(ex.execution_result.status).to eq(:pending) expect(ex.execution_result.status).to eq(:pending)
expect(ex.execution_result.pending_message).to eq('In quarantine')
end
it "ignores quarantined tests not tagged either 'foo' or 'bar'" do
group = RSpec.describe do
example 'in quarantine', :quarantine do
end end
end end
group.run context 'in a context tagged :bar and :quarantine' do
it 'skips all tests' do
context = group.children.find { |c| c.description == "quarantine" }
examples = context.descendant_filtered_examples
expect(examples.count).to be(2)
ex = examples.find { |e| e.description == "in quarantine" }
expect(ex.execution_result.status).to eq(:pending)
ex = group.examples.find { |e| e.description == "in quarantine" } ex = examples.find { |e| e.description == "not in quarantine" }
expect(ex.execution_result.status).to be_nil expect(ex.execution_result.status).to eq(:pending)
end
end end
end end
context "with 'foo' and 'bar' and 'quarantined' tagged" do context 'with :quarantine and multiple non-quarantine tags focussed' do
before do before do
RSpec.configure do |config| RSpec.configure do |config|
config.inclusion_filter = { bar: true, foo: true, quarantine: true } config.inclusion_filter = { bar: true, foo: true, quarantine: true }
end end
group.run
end end
after do after do
RSpec.configure do |config| RSpec.configure do |config|
...@@ -231,34 +256,49 @@ describe 'rspec config tests' do ...@@ -231,34 +256,49 @@ describe 'rspec config tests' do
end end
end end
it "runs tests tagged 'quarantine' and 'foo' or 'bar'" do context 'in a context tagged :foo' do
group = RSpec.describe do it 'only runs quarantined tests' do
example 'foo', :foo do context = group.children.find { |c| c.description == "foo" }
end examples = context.descendant_filtered_examples
example 'bar and quarantine', :bar, :quarantine do expect(examples.count).to be(2)
end
example 'foo and bar', :foo, :bar do ex = examples.find { |e| e.description == "in quarantine" }
end expect(ex.execution_result.status).to eq(:passed)
example 'foo, bar, and quarantine', :foo, :bar, :quarantine do
ex = examples.find { |e| e.description == "not in quarantine" }
expect(ex.execution_result.status).to eq(:pending)
expect(ex.execution_result.pending_message).to eq('Only running tests tagged with :quarantine and any of [:bar, :foo]')
end end
end end
group.run context 'in a context tagged :quarantine' do
expect(group.examples.count).to eq(4) it 'skips all tests' do
context = group.children.find { |c| c.description == "quarantine" }
examples = context.descendant_filtered_examples
expect(examples.count).to be(2)
ex = group.examples.find { |e| e.description == "foo" } ex = examples.find { |e| e.description == "in quarantine" }
expect(ex.execution_result.status).to eq(:pending) expect(ex.execution_result.status).to eq(:pending)
expect(ex.execution_result.pending_message).to eq('Running tests tagged with all of [:bar, :foo, :quarantine]') expect(ex.execution_result.pending_message).to eq('Only running tests tagged with :quarantine and any of [:bar, :foo]')
ex = group.examples.find { |e| e.description == "bar and quarantine" }
expect(ex.execution_result.status).to eq(:passed)
ex = group.examples.find { |e| e.description == "foo and bar" } ex = examples.find { |e| e.description == "not in quarantine" }
expect(ex.execution_result.status).to eq(:pending) expect(ex.execution_result.status).to eq(:pending)
expect(ex.execution_result.pending_message).to eq('Running tests tagged with all of [:bar, :foo, :quarantine]') expect(ex.execution_result.pending_message).to eq('Only running tests tagged with :quarantine and any of [:bar, :foo]')
end
end
context 'in a context tagged :bar and :quarantine' do
it 'runs all tests' do
context = group.children.find { |c| c.description == "bar quarantine" }
examples = context.descendant_filtered_examples
expect(examples.count).to be(2)
ex = group.examples.find { |e| e.description == "foo, bar, and quarantine" } ex = examples.find { |e| e.description == "in quarantine" }
expect(ex.execution_result.status).to eq(:passed) expect(ex.execution_result.status).to eq(:passed)
ex = examples.find { |e| e.description == "not in quarantine" }
expect(ex.execution_result.status).to eq(:passed)
end
end end
end end
end end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment