Commit 12147d80 authored by Mark Lapierre's avatar Mark Lapierre

Merge branch 'ml-allow-qa-tests-to-toggle-feature-flags-per-project-group-user' into 'master'

Allow scoped feature flag toggling in QA tests

See merge request gitlab-org/gitlab!42923
parents 13f67f91 feaf0837
# Testing with feature flags
To run a specific test with a feature flag enabled you can use the `QA::Runtime::Feature` class to enable and disable feature flags ([via the API](../../../api/features.md)).
To run a specific test with a feature flag enabled you can use the `QA::Runtime::Feature` class to
enable and disable feature flags ([via the API](../../../api/features.md)).
Note that administrator authorization is required to change feature flags. `QA::Runtime::Feature` will automatically authenticate as an administrator as long as you provide an appropriate access token via `GITLAB_QA_ADMIN_ACCESS_TOKEN` (recommended), or provide `GITLAB_ADMIN_USERNAME` and `GITLAB_ADMIN_PASSWORD`.
Note that administrator authorization is required to change feature flags. `QA::Runtime::Feature`
will automatically authenticate as an administrator as long as you provide an appropriate access
token via `GITLAB_QA_ADMIN_ACCESS_TOKEN` (recommended), or provide `GITLAB_ADMIN_USERNAME`
and `GITLAB_ADMIN_PASSWORD`.
Please be sure to include the tag `:requires_admin` so that the test can be skipped in environments where admin access is not available.
Please be sure to include the tag `:requires_admin` so that the test can be skipped in environments
where admin access is not available.
CAUTION: **Caution:**
You are strongly advised to [enable feature flags only for a group, project, user](../../feature_flags/development.md#feature-actors),
or [feature group](../../feature_flags/development.md#feature-groups). This makes it possible to
test a feature in a shared environment without affecting other users.
For example, the code below would enable a feature flag named `:feature_flag_name` for the project
created by the test:
```ruby
RSpec.describe "with feature flag enabled", :requires_admin do
let(:project) { Resource::Project.fabricate_via_api! }
before do
Runtime::Feature.enable('feature_flag_name')
Runtime::Feature.enable(:feature_flag_name, project: project)
end
it "feature flag test" do
# Execute a test with a feature flag enabled
# Execute the test with the feature flag enabled.
# It will only affect the project created in this test.
end
after do
Runtime::Feature.disable('feature_flag_name')
Runtime::Feature.disable(:feature_flag_name, project: project)
end
end
```
Note that the `enable` and `disable` methods first set the flag and then check that the updated
value is returned by the API.
Similarly, you can enable a feature for a group, user, or feature group:
```ruby
group = Resource::Group.fabricate_via_api!
Runtime::Feature.enable(:feature_flag_name, group: group)
user = Resource::User.fabricate_via_api!
Runtime::Feature.enable(:feature_flag_name, user: user)
feature_group = "a_feature_group"
Runtime::Feature.enable(:feature_flag_name, feature_group: feature_group)
```
If no scope is provided, the feature flag will be set instance-wide:
```ruby
# This will affect all users!
Runtime::Feature.enable(:feature_flag_name)
```
## Running a scenario with a feature flag enabled
It's also possible to run an entire scenario with a feature flag enabled, without having to edit existing tests or write new ones.
It's also possible to run an entire scenario with a feature flag enabled, without having to edit
existing tests or write new ones.
Please see the [QA README](https://gitlab.com/gitlab-org/gitlab/tree/master/qa#running-tests-with-a-feature-flag-enabled) for details.
Please see the [QA README](https://gitlab.com/gitlab-org/gitlab/tree/master/qa#running-tests-with-a-feature-flag-enabled)
for details.
# frozen_string_literal: true
require 'active_support/core_ext/object/blank'
module QA
module Runtime
module Feature
extend self
extend Support::Api
class Feature
class << self
# Documentation: https://docs.gitlab.com/ee/api/features.html
SetFeatureError = Class.new(RuntimeError)
AuthorizationError = Class.new(RuntimeError)
include Support::Api
def enable(key)
QA::Runtime::Logger.info("Enabling feature: #{key}")
set_feature(key, true)
end
SetFeatureError = Class.new(RuntimeError)
AuthorizationError = Class.new(RuntimeError)
UnknownScopeError = Class.new(RuntimeError)
def disable(key)
QA::Runtime::Logger.info("Disabling feature: #{key}")
set_feature(key, false)
end
def remove(key)
request = Runtime::API::Request.new(api_client, "/features/#{key}")
response = delete(request.url)
unless response.code == QA::Support::Api::HTTP_STATUS_NO_CONTENT
raise SetFeatureError, "Deleting feature flag #{key} failed with `#{response}`."
def remove(key)
request = Runtime::API::Request.new(api_client, "/features/#{key}")
response = delete(request.url)
unless response.code == QA::Support::Api::HTTP_STATUS_NO_CONTENT
raise SetFeatureError, "Deleting feature flag #{key} failed with `#{response}`."
end
end
end
def enable_and_verify(key)
set_and_verify(key, enable: true)
end
def disable_and_verify(key)
set_and_verify(key, enable: false)
end
def enable(key, **scopes)
set_and_verify(key, enable: true, **scopes)
end
def enabled?(key)
feature = JSON.parse(get_features).find { |flag| flag["name"] == key }
feature && feature["state"] == "on"
end
def disable(key, **scopes)
set_and_verify(key, enable: false, **scopes)
end
def get_features
request = Runtime::API::Request.new(api_client, "/features")
response = get(request.url)
response.body
end
def enabled?(key, **scopes)
feature = JSON.parse(get_features).find { |flag| flag['name'] == key.to_s }
feature && feature['state'] == 'on' || feature['state'] == 'conditional' && scopes.present? && enabled_scope?(feature['gates'], scopes)
end
private
private
def api_client
@api_client ||= begin
if Runtime::Env.admin_personal_access_token
Runtime::API::Client.new(:gitlab, personal_access_token: Runtime::Env.admin_personal_access_token)
else
user = Resource::User.fabricate_via_api! do |user|
user.username = Runtime::User.admin_username
user.password = Runtime::User.admin_password
end
def api_client
@api_client ||= Runtime::API::Client.as_admin
rescue Runtime::API::Client::AuthorizationError => e
raise AuthorizationError, "Administrator access is required to enable/disable feature flags. #{e.message}"
end
unless user.admin?
raise AuthorizationError, "Administrator access is required to enable/disable feature flags. User '#{user.username}' is not an administrator."
def enabled_scope?(gates, scopes)
scopes.each do |key, value|
case key
when :project, :group, :user
actors = gates.filter { |i| i['key'] == 'actors' }.first['value']
break actors.include?("#{key.to_s.capitalize}:#{value.id}")
when :feature_group
groups = gates.filter { |i| i['key'] == 'groups' }.first['value']
break groups.include?(value)
else
raise UnknownScopeError, "Unknown scope: #{key}"
end
Runtime::API::Client.new(:gitlab, user: user)
end
end
end
# Change a feature flag and verify that the change was successful
# Arguments:
# key: The feature flag to set (as a string)
# enable: `true` to enable the flag, `false` to disable it
def set_and_verify(key, enable:)
Support::Retrier.retry_on_exception(sleep_interval: 2) do
enable ? enable(key) : disable(key)
def get_features
request = Runtime::API::Request.new(api_client, '/features')
response = get(request.url)
response.body
end
is_enabled = nil
# Change a feature flag and verify that the change was successful
# Arguments:
# key: The feature flag to set (as a string)
# enable: `true` to enable the flag, `false` to disable it
# scopes: Any scope (user, project, group) to restrict the change to
def set_and_verify(key, enable:, **scopes)
msg = "#{enable ? 'En' : 'Dis'}abling feature: #{key}"
msg += " for scope \"#{scopes_to_s(scopes)}\"" if scopes.present?
QA::Runtime::Logger.info(msg)
QA::Support::Waiter.wait_until(sleep_interval: 1) do
is_enabled = enabled?(key)
is_enabled == enable
end
Support::Retrier.retry_on_exception(sleep_interval: 2) do
set_feature(key, enable, scopes)
is_enabled = nil
QA::Support::Waiter.wait_until(sleep_interval: 1) do
is_enabled = enabled?(key, scopes)
is_enabled == enable || !enable && scopes.present?
end
if is_enabled == enable
QA::Runtime::Logger.info("Successfully #{enable ? 'en' : 'dis'}abled and verified feature flag: #{key}")
else
raise SetFeatureError, "#{key} was not #{enable ? 'en' : 'dis'}abled!" if enable
raise SetFeatureError, "#{key} was not #{enable ? 'enabled' : 'disabled'}!" unless is_enabled == enable
QA::Runtime::Logger.warn("Feature flag scope was removed but the flag is still enabled globally.")
end
end
end
QA::Runtime::Logger.info("Successfully #{enable ? 'enabled' : 'disabled'} and verified feature flag: #{key}")
def set_feature(key, value, **scopes)
scopes[:project] = scopes[:project].full_path if scopes.key?(:project)
scopes[:group] = scopes[:group].full_path if scopes.key?(:group)
scopes[:user] = scopes[:user].username if scopes.key?(:user)
request = Runtime::API::Request.new(api_client, "/features/#{key}")
response = post(request.url, scopes.merge({ value: value }))
unless response.code == QA::Support::Api::HTTP_STATUS_CREATED
raise SetFeatureError, "Setting feature flag #{key} to #{value} failed with `#{response}`."
end
end
end
def set_feature(key, value)
request = Runtime::API::Request.new(api_client, "/features/#{key}")
response = post(request.url, { value: value })
unless response.code == QA::Support::Api::HTTP_STATUS_CREATED
raise SetFeatureError, "Setting feature flag #{key} to #{value} failed with `#{response}`."
def scopes_to_s(**scopes)
key = scopes.each_key.first
s = "#{key}: "
case key
when :project, :group
s += scopes[key].full_path
when :user
s += scopes[key].username
when :feature_group
s += scopes[key]
else
raise UnknownScopeError, "Unknown scope: #{key}"
end
s
end
end
end
......
......@@ -17,12 +17,12 @@ module QA
end
before do
Runtime::Feature.enable_and_verify('gitaly_distributed_reads')
Runtime::Feature.enable(:gitaly_distributed_reads)
praefect_manager.wait_for_replication(project.id)
end
after do
Runtime::Feature.disable_and_verify('gitaly_distributed_reads')
Runtime::Feature.disable(:gitaly_distributed_reads)
end
it 'reads from each node', testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/issues/979' do
......
......@@ -4,7 +4,7 @@ module QA
RSpec.describe 'Create' do
describe 'Push mirror a repository over HTTP' do
it 'configures and syncs LFS objects for a (push) mirrored repository', :requires_admin, testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/issues/414' do
Runtime::Feature.enable_and_verify('push_mirror_syncs_lfs')
Runtime::Feature.enable(:push_mirror_syncs_lfs)
Runtime::Browser.visit(:gitlab, Page::Main::Login)
Page::Main::Login.perform(&:sign_in_using_credentials)
......
......@@ -4,11 +4,11 @@ module QA
RSpec.describe 'Create', :requires_admin do
describe 'Multiple file snippet' do
before do
Runtime::Feature.enable_and_verify('snippet_multiple_files')
Runtime::Feature.enable('snippet_multiple_files')
end
after do
Runtime::Feature.disable_and_verify('snippet_multiple_files')
Runtime::Feature.disable('snippet_multiple_files')
end
it 'creates a personal snippet with multiple files', testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/issues/842' do
......
......@@ -51,9 +51,7 @@ module QA
after do
page.visit Runtime::Scenario.gitlab_address
%w[group_administration_nav_item].each do |flag|
Runtime::Feature.remove(flag)
end
Runtime::Feature.remove(:group_administration_nav_item)
@group.remove_via_api!
......@@ -64,9 +62,7 @@ module QA
end
def setup_and_enable_enforce_sso
%w[group_administration_nav_item].each do |flag|
Runtime::Feature.enable_and_verify(flag)
end
Runtime::Feature.enable(:group_administration_nav_item)
page.visit Runtime::Scenario.gitlab_address
Page::Main::Login.perform(&:sign_in_using_credentials) unless Page::Main::Menu.perform(&:signed_in?)
......
......@@ -85,7 +85,7 @@ module QA
after(:all) do
page.visit Runtime::Scenario.gitlab_address
%w[group_managed_accounts sign_up_on_sso group_scim group_administration_nav_item].each do |flag|
[:group_managed_accounts, :sign_up_on_sso, :group_scim, :group_administration_nav_item].each do |flag|
Runtime::Feature.remove(flag)
end
......@@ -119,8 +119,8 @@ module QA
end
def setup_and_enable_group_managed_accounts
%w[group_managed_accounts sign_up_on_sso group_scim group_administration_nav_item].each do |flag|
Runtime::Feature.enable_and_verify(flag)
[:group_managed_accounts, :sign_up_on_sso, :group_scim, :group_administration_nav_item].each do |flag|
Runtime::Feature.enable(flag)
end
Support::Retrier.retry_on_exception do
......
......@@ -10,7 +10,7 @@ module QA
sandbox_group.path = "saml_sso_group_#{SecureRandom.hex(8)}"
end
Runtime::Feature.enable_and_verify('group_administration_nav_item')
Runtime::Feature.enable(:group_administration_nav_item)
@saml_idp_service = Flow::Saml.run_saml_idp_service(@group.path)
end
......@@ -96,7 +96,7 @@ module QA
after(:all) do
@group.remove_via_api!
Runtime::Feature.remove('group_administration_nav_item')
Runtime::Feature.remove(:group_administration_nav_item)
page.visit Runtime::Scenario.gitlab_address
Page::Main::Menu.perform(&:sign_out_if_signed_in)
......
......@@ -4,84 +4,212 @@ RSpec.describe QA::Runtime::Feature do
let(:api_client) { double('QA::Runtime::API::Client') }
let(:request) { Struct.new(:url).new('http://api') }
let(:response_post) { Struct.new(:code).new(201) }
let(:response_get) { Struct.new(:code, :body).new(200, '[{ "name": "a-flag", "state": "on" }]') }
before do
allow(described_class).to receive(:api_client).and_return(api_client)
end
describe '.enable' do
it 'enables a feature flag' do
expect(QA::Runtime::API::Request)
.to receive(:new)
.with(api_client, "/features/a-flag")
.and_return(request)
expect(described_class)
.to receive(:post)
.with(request.url, { value: true })
.and_return(response_post)
subject.enable('a-flag')
end
where(:feature_flag) do
['a_flag', :a_flag]
end
describe '.enable_and_verify' do
it 'enables a feature flag' do
allow(described_class).to receive(:get).and_return(response_get)
with_them do
shared_examples 'enables a feature flag' do
it 'enables a feature flag for a scope' do
allow(described_class).to receive(:get)
.and_return(Struct.new(:code, :body).new(200, '[{ "name": "a_flag", "state": "on" }]'))
expect(QA::Runtime::API::Request).to receive(:new)
.with(api_client, "/features/a-flag").and_return(request)
expect(described_class).to receive(:post)
.with(request.url, { value: true }).and_return(response_post)
expect(QA::Runtime::API::Request).to receive(:new)
.with(api_client, "/features").and_return(request)
expect(QA::Runtime::API::Request).to receive(:new)
.with(api_client, "/features/a_flag").and_return(request)
expect(described_class).to receive(:post)
.with(request.url, { value: true, scope => actor_name }).and_return(response_post)
expect(QA::Runtime::API::Request).to receive(:new)
.with(api_client, "/features").and_return(request)
expect(QA::Runtime::Logger).to receive(:info).with("Enabling feature: a_flag for scope \"#{scope}: #{actor_name}\"")
expect(QA::Runtime::Logger).to receive(:info).with("Successfully enabled and verified feature flag: a_flag")
subject.enable_and_verify('a-flag')
described_class.enable(feature_flag, scope => actor)
end
end
end
describe '.disable' do
it 'disables a feature flag' do
expect(QA::Runtime::API::Request)
.to receive(:new)
.with(api_client, "/features/a-flag")
.and_return(request)
expect(described_class)
.to receive(:post)
.with(request.url, { value: false })
.and_return(response_post)
subject.disable('a-flag')
shared_examples 'disables a feature flag' do
it 'disables a feature flag for a scope' do
allow(described_class).to receive(:get)
.and_return(Struct.new(:code, :body).new(200, '[{ "name": "a_flag", "state": "off" }]'))
expect(QA::Runtime::API::Request).to receive(:new)
.with(api_client, "/features/a_flag").and_return(request)
expect(described_class).to receive(:post)
.with(request.url, { value: false, scope => actor_name }).and_return(response_post)
expect(QA::Runtime::API::Request).to receive(:new)
.with(api_client, "/features").and_return(request)
expect(QA::Runtime::Logger).to receive(:info).with("Disabling feature: a_flag for scope \"#{scope}: #{actor_name}\"")
expect(QA::Runtime::Logger).to receive(:info).with("Successfully disabled and verified feature flag: a_flag")
described_class.disable(feature_flag, scope => actor )
end
end
end
describe '.disable_and_verify' do
it 'disables a feature flag' do
allow(described_class).to receive(:get)
.and_return(Struct.new(:code, :body).new(200, '[{ "name": "a-flag", "state": "off" }]'))
shared_examples 'checks a feature flag' do
context 'when the flag is enabled for a scope' do
it 'returns the feature flag state' do
expect(QA::Runtime::API::Request)
.to receive(:new)
.with(api_client, "/features")
.and_return(request)
expect(described_class)
.to receive(:get)
.and_return(Struct.new(:code, :body).new(200, %Q([{ "name": "a_flag", "state": "conditional", "gates": #{gates} }])))
expect(described_class.enabled?(feature_flag, scope => actor)).to be_truthy
end
end
end
expect(QA::Runtime::API::Request).to receive(:new)
.with(api_client, "/features/a-flag").and_return(request)
expect(described_class).to receive(:post)
.with(request.url, { value: false }).and_return(response_post)
expect(QA::Runtime::API::Request).to receive(:new)
.with(api_client, "/features").and_return(request)
describe '.enable' do
it 'enables a feature flag' do
allow(described_class).to receive(:get)
.and_return(Struct.new(:code, :body).new(200, '[{ "name": "a_flag", "state": "on" }]'))
subject.disable_and_verify('a-flag')
expect(QA::Runtime::API::Request).to receive(:new)
.with(api_client, "/features/a_flag").and_return(request)
expect(described_class).to receive(:post)
.with(request.url, { value: true }).and_return(response_post)
expect(QA::Runtime::API::Request).to receive(:new)
.with(api_client, "/features").and_return(request)
described_class.enable(feature_flag)
end
context 'when a project scope is provided' do
it_behaves_like 'enables a feature flag' do
let(:scope) { :project }
let(:actor_name) { 'group-name/project-name' }
let(:actor) { Struct.new(:full_path).new(actor_name) }
end
end
context 'when a group scope is provided' do
it_behaves_like 'enables a feature flag' do
let(:scope) { :group }
let(:actor_name) { 'group-name' }
let(:actor) { Struct.new(:full_path).new(actor_name) }
end
end
context 'when a user scope is provided' do
it_behaves_like 'enables a feature flag' do
let(:scope) { :user }
let(:actor_name) { 'user-name' }
let(:actor) { Struct.new(:username).new(actor_name) }
end
end
context 'when a feature group scope is provided' do
it_behaves_like 'enables a feature flag' do
let(:scope) { :feature_group }
let(:actor_name) { 'foo' }
let(:actor) { "foo" }
end
end
end
end
describe '.enabled?' do
it 'returns a feature flag state' do
expect(QA::Runtime::API::Request)
.to receive(:new)
.with(api_client, "/features")
.and_return(request)
expect(described_class)
.to receive(:get)
.and_return(response_get)
expect(subject.enabled?('a-flag')).to be_truthy
describe '.disable' do
it 'disables a feature flag' do
allow(described_class).to receive(:get)
.and_return(Struct.new(:code, :body).new(200, '[{ "name": "a_flag", "state": "off" }]'))
expect(QA::Runtime::API::Request).to receive(:new)
.with(api_client, "/features/a_flag").and_return(request)
expect(described_class).to receive(:post)
.with(request.url, { value: false }).and_return(response_post)
expect(QA::Runtime::API::Request).to receive(:new)
.with(api_client, "/features").and_return(request)
described_class.disable(feature_flag)
end
context 'when a project scope is provided' do
it_behaves_like 'disables a feature flag' do
let(:scope) { :project }
let(:actor_name) { 'group-name/project-name' }
let(:actor) { Struct.new(:full_path).new(actor_name) }
end
end
context 'when a group scope is provided' do
it_behaves_like 'disables a feature flag' do
let(:scope) { :group }
let(:actor_name) { 'group-name' }
let(:actor) { Struct.new(:full_path).new(actor_name) }
end
end
context 'when a user scope is provided' do
it_behaves_like 'disables a feature flag' do
let(:scope) { :user }
let(:actor_name) { 'user-name' }
let(:actor) { Struct.new(:username).new(actor_name) }
end
end
context 'when a feature group scope is provided' do
it_behaves_like 'disables a feature flag' do
let(:scope) { :feature_group }
let(:actor_name) { 'foo' }
let(:actor) { "foo" }
end
end
end
describe '.enabled?' do
it 'returns a feature flag state' do
expect(QA::Runtime::API::Request)
.to receive(:new)
.with(api_client, "/features")
.and_return(request)
expect(described_class)
.to receive(:get)
.and_return(Struct.new(:code, :body).new(200, '[{ "name": "a_flag", "state": "on" }]'))
expect(described_class.enabled?(feature_flag)).to be_truthy
end
context 'when a project scope is provided' do
it_behaves_like 'checks a feature flag' do
let(:scope) { :project }
let(:actor_name) { 'group-name/project-name' }
let(:actor) { Struct.new(:full_path, :id).new(actor_name, 270) }
let(:gates) { %q([{"key": "actors", "value": ["Project:270"]}]) }
end
end
context 'when a group scope is provided' do
it_behaves_like 'checks a feature flag' do
let(:scope) { :group }
let(:actor_name) { 'group-name' }
let(:actor) { Struct.new(:full_path, :id).new(actor_name, 33) }
let(:gates) { %q([{"key": "actors", "value": ["Group:33"]}]) }
end
end
context 'when a user scope is provided' do
it_behaves_like 'checks a feature flag' do
let(:scope) { :user }
let(:actor_name) { 'user-name' }
let(:actor) { Struct.new(:full_path, :id).new(actor_name, 13) }
let(:gates) { %q([{"key": "actors", "value": ["User:13"]}]) }
end
end
context 'when a feature group scope is provided' do
it_behaves_like 'checks a feature flag' do
let(:scope) { :feature_group }
let(:actor_name) { 'foo' }
let(:actor) { "foo" }
let(:gates) { %q([{"key": "groups", "value": ["foo"]}]) }
end
end
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment