Commit 49163249 authored by Sean McGivern's avatar Sean McGivern

Merge branch 'mk/add-package-file-registry-to-graphql' into 'master'

Geo: Add PackageFileRegistry to GraphQL

Closes #199879

See merge request gitlab-org/gitlab!26719
parents 2e767e28 2445a121
......@@ -3450,6 +3450,36 @@ type GeoNode {
"""
name: String
"""
Package file registries of the GeoNode. Available only when feature flag `geo_self_service_framework` is enabled
"""
packageFileRegistries(
"""
Returns the elements in the list that come after the specified cursor.
"""
after: String
"""
Returns the elements in the list that come before the specified cursor.
"""
before: String
"""
Returns the first _n_ elements from the list.
"""
first: Int
"""
Filters registries by their ID
"""
ids: [ID!]
"""
Returns the last _n_ elements from the list.
"""
last: Int
): PackageFileRegistryConnection
"""
Indicates whether this Geo node is the primary
"""
......@@ -6329,6 +6359,86 @@ interface Noteable {
): NoteConnection!
}
"""
Represents the sync and verification state of a package file
"""
type PackageFileRegistry {
"""
Timestamp when the PackageFileRegistry was created
"""
createdAt: Time
"""
ID of the PackageFileRegistry
"""
id: ID!
"""
Error message during sync of the PackageFileRegistry
"""
lastSyncFailure: String
"""
Timestamp of the most recent successful sync of the PackageFileRegistry
"""
lastSyncedAt: Time
"""
ID of the PackageFile
"""
packageFileId: ID!
"""
Timestamp after which the PackageFileRegistry should be resynced
"""
retryAt: Time
"""
Number of consecutive failed sync attempts of the PackageFileRegistry
"""
retryCount: Int
"""
Sync state of the PackageFileRegistry
"""
state: RegistryState
}
"""
The connection type for PackageFileRegistry.
"""
type PackageFileRegistryConnection {
"""
A list of edges.
"""
edges: [PackageFileRegistryEdge]
"""
A list of nodes.
"""
nodes: [PackageFileRegistry]
"""
Information to aid in pagination.
"""
pageInfo: PageInfo!
}
"""
An edge in a connection.
"""
type PackageFileRegistryEdge {
"""
A cursor for use in pagination.
"""
cursor: String!
"""
The item at the end of the edge.
"""
node: PackageFileRegistry
}
"""
Information about pagination in a connection.
"""
......@@ -7805,6 +7915,31 @@ type Query {
): VulnerabilityConnection
}
"""
State of a Geo registry.
"""
enum RegistryState {
"""
Registry that failed to sync
"""
FAILED
"""
Registry waiting to be synced
"""
PENDING
"""
Registry currently syncing
"""
STARTED
"""
Registry that is synced
"""
SYNCED
}
"""
Autogenerated input type of RemoveAwardEmoji
"""
......
......@@ -962,6 +962,21 @@ Represents a milestone.
| `readNote` | Boolean! | Indicates the user can perform `read_note` on this resource |
| `resolveNote` | Boolean! | Indicates the user can perform `resolve_note` on this resource |
## PackageFileRegistry
Represents the sync and verification state of a package file
| Name | Type | Description |
| --- | ---- | ---------- |
| `createdAt` | Time | Timestamp when the PackageFileRegistry was created |
| `id` | ID! | ID of the PackageFileRegistry |
| `lastSyncFailure` | String | Error message during sync of the PackageFileRegistry |
| `lastSyncedAt` | Time | Timestamp of the most recent successful sync of the PackageFileRegistry |
| `packageFileId` | ID! | ID of the PackageFile |
| `retryAt` | Time | Timestamp after which the PackageFileRegistry should be resynced |
| `retryCount` | Int | Number of consecutive failed sync attempts of the PackageFileRegistry |
| `state` | RegistryState | Sync state of the PackageFileRegistry |
## PageInfo
Information about pagination in a connection.
......
......@@ -161,49 +161,7 @@ state.
For example, to add support for files referenced by a `Widget` model with a
`widgets` table, you would perform the following steps:
1. Add verification state fields to the `widgets` table so the Geo primary can
track verification state:
```ruby
# frozen_string_literal: true
class AddVerificationStateToWidgets < ActiveRecord::Migration[6.0]
DOWNTIME = false
def change
add_column :widgets, :verification_retry_at, :datetime_with_timezone
add_column :widgets, :verified_at, :datetime_with_timezone
add_column :widgets, :verification_checksum, :string
add_column :widgets, :verification_failure, :string
add_column :widgets, :verification_retry_count, :integer
end
end
```
1. Add a partial index on `verification_failure` and `verification_checksum` to ensure
re-verification can be performed efficiently:
```ruby
# frozen_string_literal: true
class AddVerificationFailureIndexToWidgets < ActiveRecord::Migration[6.0]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
def up
add_concurrent_index :widgets, :verification_failure, where: "(verification_failure IS NOT NULL)", name: "widgets_verification_failure_partial"
add_concurrent_index :widgets, :verification_checksum, where: "(verification_checksum IS NOT NULL)", name: "widgets_verification_checksum_partial"
end
def down
remove_concurrent_index :widgets, :verification_failure
remove_concurrent_index :widgets, :verification_checksum
end
end
```
#### Replication
1. Include `Gitlab::Geo::ReplicableModel` in the `Widget` class, and specify
the Replicator class `with_replicator Geo::WidgetReplicator`.
......@@ -350,11 +308,53 @@ For example, to add support for files referenced by a `Widget` model with a
end
```
Widget files should now be replicated and verified by Geo!
Widgets should now be replicated by Geo!
#### Verification
1. Add verification state fields to the `widgets` table so the Geo primary can
track verification state:
```ruby
# frozen_string_literal: true
class AddVerificationStateToWidgets < ActiveRecord::Migration[6.0]
DOWNTIME = false
### Verification statistics with Blob Replicator Strategy
def change
add_column :widgets, :verification_retry_at, :datetime_with_timezone
add_column :widgets, :verified_at, :datetime_with_timezone
add_column :widgets, :verification_checksum, :string
add_column :widgets, :verification_failure, :string
add_column :widgets, :verification_retry_count, :integer
end
end
```
GitLab Geo stores statistic data in the `geo_node_statuses` table.
1. Add a partial index on `verification_failure` and `verification_checksum` to ensure
re-verification can be performed efficiently:
```ruby
# frozen_string_literal: true
class AddVerificationFailureIndexToWidgets < ActiveRecord::Migration[6.0]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
def up
add_concurrent_index :widgets, :verification_failure, where: "(verification_failure IS NOT NULL)", name: "widgets_verification_failure_partial"
add_concurrent_index :widgets, :verification_checksum, where: "(verification_checksum IS NOT NULL)", name: "widgets_verification_checksum_partial"
end
def down
remove_concurrent_index :widgets, :verification_failure
remove_concurrent_index :widgets, :verification_checksum
end
end
```
1. Add fields `widget_count`, `widget_checksummed_count`, and `widget_checksum_failed_count`
to `GeoNodeStatus#RESOURCE_STATUS_FIELDS` array in `ee/app/models/geo_node_status.rb`.
......@@ -378,3 +378,134 @@ GitLab Geo stores statistic data in the `geo_node_statuses` table.
1. Update `Sidekiq metrics` table in `doc/administration/monitoring/prometheus/gitlab_metrics.md` with new fields.
1. Update `GET /geo_nodes/status` example response in `doc/api/geo_nodes.md` with new fields.
1. Update `ee/spec/models/geo_node_status_spec.rb` and `ee/spec/factories/geo_node_statuses.rb` with new fields.
To do: Add verification on secondaries.
Widgets should now be verified by Geo!
#### GraphQL API
1. Add a new field to `GeoNodeType` in
`ee/app/graphql/types/geo/geo_node_type.rb`:
```ruby
field :widget_registries, ::Types::Geo::WidgetRegistryType.connection_type,
null: true,
resolver: ::Resolvers::Geo::WidgetRegistriesResolver,
description: 'Find widget registries on this Geo node',
feature_flag: :geo_self_service_framework
```
1. Add the new `widget_registries` field name to the `expected_fields` array in
`ee/spec/graphql/types/geo/geo_node_type_spec.rb`.
1. Create `ee/app/graphql/resolvers/geo/widget_registries_resolver.rb`:
```ruby
# frozen_string_literal: true
module Resolvers
module Geo
class WidgetRegistriesResolver < BaseResolver
include RegistriesResolver
end
end
end
```
1. Create `ee/spec/graphql/resolvers/geo/widget_registries_resolver_spec.rb`:
```ruby
# frozen_string_literal: true
require 'spec_helper'
describe Resolvers::Geo::WidgetRegistriesResolver do
it_behaves_like 'a Geo registries resolver', :widget_registry
end
```
1. Create `ee/app/finders/geo/widget_registry_finder.rb`:
```ruby
# frozen_string_literal: true
module Geo
class WidgetRegistryFinder
include FrameworkRegistryFinder
end
end
```
1. Create `ee/spec/finders/geo/widget_registry_finder_spec.rb`:
```ruby
# frozen_string_literal: true
require 'spec_helper'
describe Geo::WidgetRegistryFinder do
it_behaves_like 'a framework registry finder', :widget_registry
end
```
1. Create `ee/app/graphql/types/geo/package_file_registry_type.rb`:
```ruby
# frozen_string_literal: true
module Types
module Geo
# rubocop:disable Graphql/AuthorizeTypes because it is included
class WidgetRegistryType < BaseObject
include ::Types::Geo::RegistryType
graphql_name 'WidgetRegistry'
description 'Represents the sync and verification state of a widget'
field :widget_id, GraphQL::ID_TYPE, null: false, description: 'ID of the Widget'
end
end
end
```
1. Create `ee/spec/graphql/types/geo/widget_registry_type_spec.rb`:
```ruby
# frozen_string_literal: true
require 'spec_helper'
describe GitlabSchema.types['WidgetRegistry'] do
it_behaves_like 'a Geo registry type'
it 'has the expected fields (other than those included in RegistryType)' do
expected_fields = %i[widget_id]
expect(described_class).to have_graphql_fields(*expected_fields).at_least
end
end
```
1. Add integration tests for providing Widget registry data to the frontend via
the GraphQL API, by duplicating and modifying the following shared examples
in `ee/spec/requests/api/graphql/geo/registries_spec.rb`:
```ruby
it_behaves_like 'gets registries for', {
field_name: 'widgetRegistries',
registry_class_name: 'WidgetRegistry',
registry_factory: :widget_registry,
registry_foreign_key_field_name: 'widgetId'
}
```
Individual widget synchronization and verification data should now be available
via the GraphQL API!
#### Admin UI
To do.
Widget sync and verification data (aggregate and individual) should now be
available in the Admin UI!
# frozen_string_literal: true
module Resolvers
module Geo
class PackageFileRegistriesResolver < BaseResolver
include RegistriesResolver
end
end
end
# frozen_string_literal: true
module Resolvers
module Geo
module RegistriesResolver
extend ActiveSupport::Concern
included do
def self.replicator_class
Gitlab::Geo::Replicator.for_class_name(self.name)
end
delegate :registry_class, :registry_finder_class, to: :replicator_class
type replicator_class.graphql_registry_type, null: true
argument :ids,
[GraphQL::ID_TYPE],
required: false,
description: 'Filters registries by their ID'
def resolve(ids: nil)
return registry_class.none unless geo_node_is_current?
registry_finder_class.new(
context[:current_user],
ids: registry_ids(ids)
).execute
end
private
def replicator_class
self.class.replicator_class
end
def registry_ids(ids)
ids&.map { |id| GlobalID.parse(id)&.model_id }&.compact
end
# We can't query other nodes' tracking databases
def geo_node_is_current?
geo_node&.current?
end
def geo_node
object
end
end
end
end
end
......@@ -22,6 +22,11 @@ module Types
field :selective_sync_shards, type: [GraphQL::STRING_TYPE], null: true, description: 'The repository storages whose projects should be synced, if `selective_sync_type` == `shards`'
field :selective_sync_namespaces, ::Types::NamespaceType.connection_type, null: true, method: :namespaces, description: 'The namespaces that should be synced, if `selective_sync_type` == `namespaces`'
field :minimum_reverification_interval, GraphQL::INT_TYPE, null: true, description: 'The interval (in days) in which the repository verification is valid. Once expired, it will be reverified'
field :package_file_registries, ::Types::Geo::PackageFileRegistryType.connection_type,
null: true,
resolver: ::Resolvers::Geo::PackageFileRegistriesResolver,
description: 'Package file registries of the GeoNode',
feature_flag: :geo_self_service_framework
end
end
end
# frozen_string_literal: true
module Types
module Geo
# rubocop:disable Graphql/AuthorizeTypes because it is included
class PackageFileRegistryType < BaseObject
include ::Types::Geo::RegistryType
graphql_name 'PackageFileRegistry'
description 'Represents the sync and verification state of a package file'
field :package_file_id, GraphQL::ID_TYPE, null: false, description: 'ID of the PackageFile'
end
end
end
# frozen_string_literal: true
module Types
module Geo
class RegistryStateEnum < BaseEnum
graphql_name 'RegistryState'
description 'State of a Geo registry.'
value 'PENDING', value: :pending, description: 'Registry waiting to be synced'
value 'STARTED', value: :started, description: 'Registry currently syncing'
value 'SYNCED', value: :synced, description: 'Registry that is synced'
value 'FAILED', value: :failed, description: 'Registry that failed to sync'
end
end
end
# frozen_string_literal: true
module Types
module Geo
module RegistryType
extend ActiveSupport::Concern
included do
authorize :read_geo_registry
field :id, GraphQL::ID_TYPE, null: false, description: "ID of the #{graphql_name}"
field :state, Types::Geo::RegistryStateEnum, null: true, method: :state_name, description: "Sync state of the #{graphql_name}"
field :retry_count, GraphQL::INT_TYPE, null: true, description: "Number of consecutive failed sync attempts of the #{graphql_name}"
field :last_sync_failure, GraphQL::STRING_TYPE, null: true, description: "Error message during sync of the #{graphql_name}"
field :retry_at, Types::TimeType, null: true, description: "Timestamp after which the #{graphql_name} should be resynced"
field :last_synced_at, Types::TimeType, null: true, description: "Timestamp of the most recent successful sync of the #{graphql_name}"
field :created_at, Types::TimeType, null: true, description: "Timestamp when the #{graphql_name} was created"
end
end
end
end
......@@ -5,6 +5,8 @@ class Geo::BaseRegistry < Geo::TrackingBase
self.abstract_class = true
include GlobalID::Identification
def self.pluck_model_ids_in_range(range)
where(self::MODEL_FOREIGN_KEY => range).pluck(self::MODEL_FOREIGN_KEY)
end
......
......@@ -3,6 +3,10 @@
class Geo::PackageFileRegistry < Geo::BaseRegistry
include ::Delay
def self.declarative_policy_class
'Geo::RegistryPolicy'
end
STATE_VALUES = {
pending: 0,
started: 1,
......
# frozen_string_literal: true
module Geo
class RegistryPolicy < ::BasePolicy
condition(:can_read_all_geo, scope: :user) { can?(:read_all_geo, :global) }
rule { can_read_all_geo }.enable :read_geo_registry
end
end
......@@ -69,6 +69,10 @@ module Gitlab
const_get("::Geo::#{replicable_name.camelize}RegistryFinder", false)
end
def self.graphql_registry_type
const_get("::Types::Geo::#{replicable_name.camelize}RegistryType", false)
end
# Given a `replicable_name`, return the corresponding replicator
#
# @param [String] replicable_name the replicable slug
......
# frozen_string_literal: true
require 'spec_helper'
describe Resolvers::Geo::PackageFileRegistriesResolver do
it_behaves_like 'a Geo registries resolver', :package_file_registry
end
......@@ -11,7 +11,7 @@ describe GitlabSchema.types['GeoNode'] do
repos_max_capacity verification_max_capacity
container_repositories_max_capacity sync_object_storage
selective_sync_type selective_sync_shards selective_sync_namespaces
minimum_reverification_interval
minimum_reverification_interval package_file_registries
]
expect(described_class).to have_graphql_fields(*expected_fields)
......
# frozen_string_literal: true
require 'spec_helper'
describe GitlabSchema.types['PackageFileRegistry'] do
it_behaves_like 'a Geo registry type'
it 'has the expected fields (other than those included in RegistryType)' do
expected_fields = %i[package_file_id]
expect(described_class).to have_graphql_fields(*expected_fields).at_least
end
end
# frozen_string_literal: true
require 'spec_helper'
describe GitlabSchema.types['RegistryState'] do
it { expect(described_class.graphql_name).to eq('RegistryState') }
it 'exposes the correct registry states' do
expect(described_class.values.keys).to include(*%w[PENDING STARTED SYNCED FAILED])
end
end
# frozen_string_literal: true
require 'spec_helper'
describe Geo::RegistryPolicy do
let!(:registry) { create(:package_file_registry) }
subject(:policy) { described_class.new(current_user, registry) }
context 'when the user is an admin' do
let(:current_user) { create(:user, :admin) }
it 'allows read_geo_registry for any registry' do
expect(policy).to be_allowed(:read_geo_registry)
end
end
context 'when the user is not an admin' do
let(:current_user) { create(:user) }
it 'disallows read_geo_registry for any registry' do
expect(policy).to be_disallowed(:read_geo_registry)
end
end
end
# frozen_string_literal: true
require 'spec_helper'
describe 'Gets registries' do
it_behaves_like 'gets registries for', {
field_name: 'packageFileRegistries',
registry_class_name: 'PackageFileRegistry',
registry_factory: :package_file_registry,
registry_foreign_key_field_name: 'packageFileId'
}
end
# frozen_string_literal: true
RSpec.shared_examples_for 'a Geo registries resolver' do |registry_factory_name|
include GraphqlHelpers
include EE::GeoHelpers
describe '#resolve' do
let_it_be(:secondary) { create(:geo_node) }
let_it_be(:registry1) { create(registry_factory_name) }
let_it_be(:registry2) { create(registry_factory_name) }
let_it_be(:registry3) { create(registry_factory_name) }
let(:registries) { [registry1, registry2, registry3] }
let(:gql_context) { { current_user: current_user } }
context 'when the parent object is the current node' do
before do
stub_current_geo_node(secondary)
end
context 'when the user has permission to view Geo data' do
let_it_be(:current_user) { create(:admin) }
context 'when the ids argument is null' do
it 'returns registries, in order' do
expect(resolve_registries.to_a).to eq(registries)
end
end
context 'when the ids argument is present' do
it 'returns the requested registries, in order' do
requested_ids = [registry3.to_global_id, registry1.to_global_id]
args = { ids: requested_ids }
expected = [registry1, registry3]
expect(resolve_registries(args).to_a).to eq(expected)
end
end
end
context 'when the user does not have permission to view Geo data' do
let_it_be(:current_user) { create(:user) }
it 'returns nothing' do
expect(resolve_registries).to be_empty
end
end
end
context 'when the parent object is not the current node' do
context 'when the user has permission to view Geo data' do
let_it_be(:current_user) { create(:admin) }
it "returns nothing, because we can't query other nodes' tracking databases" do
result = resolve(described_class, obj: create(:geo_node), args: {}, ctx: gql_context)
expect(result).to be_empty
end
end
end
end
def resolve_registries(args = {})
resolve(described_class, obj: secondary, args: args, ctx: gql_context)
end
end
# frozen_string_literal: true
RSpec.shared_examples_for 'a Geo registry type' do |registry_factory_name|
it { expect(described_class).to require_graphql_authorizations(:read_geo_registry) }
it 'has the expected fields' do
expected_fields = %i[
id state retry_count last_sync_failure retry_at last_synced_at created_at
]
expect(described_class).to have_graphql_fields(*expected_fields).at_least
end
end
# frozen_string_literal: true
RSpec.shared_examples 'gets registries for' do |args|
let(:field_name) { args[:field_name] }
let(:registry_class_name) { args[:registry_class_name] }
let(:registry_factory) { args[:registry_factory] }
let(:registry_foreign_key_field_name) { args[:registry_foreign_key_field_name] }
let(:registry_foreign_key) { registry_foreign_key_field_name.underscore }
let(:field_name_sym) { field_name.underscore.to_sym }
include GraphqlHelpers
include EE::GeoHelpers
let_it_be(:secondary) { create(:geo_node) }
let!(:registry1) { create(registry_factory) }
let!(:registry2) { create(registry_factory) }
let(:query) do
<<~QUERY
{
geoNode {
#{field_name} {
nodes {
#{all_graphql_fields_for(registry_class_name)}
}
}
}
}
QUERY
end
let(:current_user) { create(:user, :admin) }
before do
stub_current_geo_node(secondary)
stub_current_node_name(secondary.name)
end
it_behaves_like 'a working graphql query' do
before do
post_graphql(query, current_user: current_user)
end
end
it 'returns registries' do
expected = [registry1, registry2].map do |registry|
registry_to_graphql_data_hash(registry)
end
post_graphql(query, current_user: current_user)
actual = graphql_data_at(:geo_node, field_name_sym, :nodes)
expect(actual).to eq(expected)
end
context 'when paginating' do
let!(:expected_registry1) { create(registry_factory) }
let!(:expected_registry2) { create(registry_factory) }
def query(registries_params)
<<~QUERY
{
geoNode {
#{field_name}(#{registries_params}) {
edges {
node {
id
}
cursor
}
pageInfo {
endCursor
hasNextPage
}
}
}
}
QUERY
end
it 'supports cursor-based pagination' do
post_graphql(query('first: 2'), current_user: current_user)
edges = graphql_data_at(:geo_node, field_name_sym, :edges)
page_info = graphql_data_at(:geo_node, field_name_sym, :page_info)
has_next_page = graphql_data_at(:geo_node, field_name_sym, :page_info, :has_next_page)
expect(edges.size).to eq(2)
expect(page_info).to be_present
expect(has_next_page).to eq(true)
end
it 'returns the correct page of registries' do
# Get first page
post_graphql(query('first: 2'), current_user: current_user)
end_cursor = graphql_data_at(:geo_node, field_name_sym, :page_info, :end_cursor)
# Get second page
post_graphql(query("first: 2, after: \"#{end_cursor}\""), current_user: current_user)
response_data = JSON.parse(response.body).dig('data', 'geoNode', GraphqlHelpers.fieldnamerize(field_name), 'edges')
first_result = response_data.first['node']
second_result = response_data.second['node']
expect(first_result).to eq('id' => expected_registry1.to_global_id.to_s)
expect(second_result).to eq('id' => expected_registry2.to_global_id.to_s)
end
end
context 'when the geo_self_service_framework feature is disabled' do
before do
stub_feature_flags(geo_self_service_framework: false)
end
it 'errors when requesting registries' do
post_graphql(query, current_user: current_user)
expect_graphql_errors_to_include(/Field '#{field_name}' doesn't exist on type 'GeoNode'/)
end
end
def registry_to_graphql_data_hash(registry)
{
'id' => registry.to_global_id.to_s,
registry_foreign_key_field_name => registry.send(registry_foreign_key).to_s,
'state' => registry.state_name.to_s.upcase,
'retryCount' => registry.retry_count,
'lastSyncFailure' => registry.last_sync_failure,
'retryAt' => registry.retry_at,
'lastSyncedAt' => registry.last_synced_at,
'createdAt' => registry.created_at.iso8601
}
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment