Commit 88456ec2 authored by GitLab Bot's avatar GitLab Bot

Automatic merge of gitlab-org/gitlab master

parents 41a9a6d4 ae8fd6ad
......@@ -27,10 +27,6 @@ class PagesDeployment < ApplicationRecord
mount_file_store_uploader ::Pages::DeploymentUploader
def log_geo_deleted_event
# this is to be adressed in https://gitlab.com/groups/gitlab-org/-/epics/589
end
def migrated?
file.filename == MIGRATED_FILE_NAME
end
......@@ -41,3 +37,5 @@ class PagesDeployment < ApplicationRecord
self.size = file.size
end
end
PagesDeployment.prepend_mod
......@@ -25,6 +25,7 @@ ActiveSupport::Inflector.inflections do |inflect|
lfs_object_registry
merge_request_diff_registry
package_file_registry
pages_deployment_registry
pipeline_artifact_registry
project_auto_devops
project_registry
......
......@@ -252,6 +252,16 @@ configuration option in `gitlab.yml`. These metrics are served from the
| `geo_group_wiki_repositories_synced` | Gauge | 13.10 | Number of syncable group wikis synced on secondary | `url` |
| `geo_group_wiki_repositories_failed` | Gauge | 13.10 | Number of syncable group wikis failed on secondary | `url` |
| `geo_group_wiki_repositories_registry` | Gauge | 13.10 | Number of syncable group wikis in the registry | `url` |
| `geo_pages_deployments` | Gauge | 14.3 | Number of pages deployments on primary | `url` |
| `geo_pages_deployments_checksum_total` | Gauge | 14.3 | Number of pages deployments tried to checksum on primary | `url` |
| `geo_pages_deployments_checksummed` | Gauge | 14.3 | Number of pages deployments successfully checksummed on primary | `url` |
| `geo_pages_deployments_checksum_failed` | Gauge | 14.3 | Number of pages deployments failed to calculate the checksum on primary | `url` |
| `geo_pages_deployments_synced` | Gauge | 14.3 | Number of syncable pages deployments synced on secondary | `url` |
| `geo_pages_deployments_failed` | Gauge | 14.3 | Number of syncable pages deployments failed to sync on secondary | `url` |
| `geo_pages_deployments_registry` | Gauge | 14.3 | Number of pages deployments in the registry | `url` |
| `geo_pages_deployments_verification_total` | Gauge | 14.3 | Number of pages deployments verifications tried on secondary | `url` |
| `geo_pages_deployments_verified` | Gauge | 14.3 | Number of pages deployments verified on secondary | `url` |
| `geo_pages_deployments_verification_failed` | Gauge | 14.3 | Number of pages deployments verifications failed on secondary | `url` |
| `limited_capacity_worker_running_jobs` | Gauge | 13.5 | Number of running jobs | `worker` |
| `limited_capacity_worker_max_running_jobs` | Gauge | 13.5 | Maximum number of running jobs | `worker` |
| `limited_capacity_worker_remaining_work_count` | Gauge | 13.5 | Number of jobs waiting to be enqueued | `worker` |
......
......@@ -393,6 +393,18 @@ Example response:
"package_files_verification_failed_count": null,
"package_files_synced_in_percentage": "0.00%",
"package_files_verified_in_percentage": "0.00%",
"pages_deployments_count": 5,
"pages_deployments_checksum_total_count": 5,
"pages_deployments_checksummed_count": 5,
"pages_deployments_checksum_failed_count": 0,
"pages_deployments_synced_count": null,
"pages_deployments_failed_count": null,
"pages_deployments_registry_count": null,
"pages_deployments_verification_total_count": null,
"pages_deployments_verified_count": null,
"pages_deployments_verification_failed_count": null,
"pages_deployments_synced_in_percentage": "0.00%",
"pages_deployments_verified_in_percentage": "0.00%",
"terraform_state_versions_count": 5,
"terraform_state_versions_checksum_total_count": 5,
"terraform_state_versions_checksummed_count": 5,
......
......@@ -6450,6 +6450,29 @@ The edge type for [`PackageTag`](#packagetag).
| <a id="packagetagedgecursor"></a>`cursor` | [`String!`](#string) | A cursor for use in pagination. |
| <a id="packagetagedgenode"></a>`node` | [`PackageTag`](#packagetag) | The item at the end of the edge. |
#### `PagesDeploymentRegistryConnection`
The connection type for [`PagesDeploymentRegistry`](#pagesdeploymentregistry).
##### Fields
| Name | Type | Description |
| ---- | ---- | ----------- |
| <a id="pagesdeploymentregistryconnectionedges"></a>`edges` | [`[PagesDeploymentRegistryEdge]`](#pagesdeploymentregistryedge) | A list of edges. |
| <a id="pagesdeploymentregistryconnectionnodes"></a>`nodes` | [`[PagesDeploymentRegistry]`](#pagesdeploymentregistry) | A list of nodes. |
| <a id="pagesdeploymentregistryconnectionpageinfo"></a>`pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. |
#### `PagesDeploymentRegistryEdge`
The edge type for [`PagesDeploymentRegistry`](#pagesdeploymentregistry).
##### Fields
| Name | Type | Description |
| ---- | ---- | ----------- |
| <a id="pagesdeploymentregistryedgecursor"></a>`cursor` | [`String!`](#string) | A cursor for use in pagination. |
| <a id="pagesdeploymentregistryedgenode"></a>`node` | [`PagesDeploymentRegistry`](#pagesdeploymentregistry) | The item at the end of the edge. |
#### `PathLockConnection`
The connection type for [`PathLock`](#pathlock).
......@@ -9606,6 +9629,22 @@ four standard [pagination arguments](#connection-pagination-arguments):
| ---- | ---- | ----------- |
| <a id="geonodepackagefileregistriesids"></a>`ids` | [`[ID!]`](#id) | Filters registries by their ID. |
##### `GeoNode.pagesDeploymentRegistries`
Find Pages Deployment registries on this Geo node Available only when feature flag `geo_pages_deployment_replication` is enabled. This flag is disabled by default, because the feature is experimental and is subject to change without notice.
Returns [`PagesDeploymentRegistryConnection`](#pagesdeploymentregistryconnection).
This field returns a [connection](#connections). It accepts the
four standard [pagination arguments](#connection-pagination-arguments):
`before: String`, `after: String`, `first: Int`, `last: Int`.
###### Arguments
| Name | Type | Description |
| ---- | ---- | ----------- |
| <a id="geonodepagesdeploymentregistriesids"></a>`ids` | [`[ID!]`](#id) | Filters registries by their ID. |
##### `GeoNode.pipelineArtifactRegistries`
Find pipeline artifact registries on this Geo node.
......@@ -11791,6 +11830,23 @@ Information about pagination in a connection.
| <a id="pageinfohaspreviouspage"></a>`hasPreviousPage` | [`Boolean!`](#boolean) | When paginating backwards, are there more items?. |
| <a id="pageinfostartcursor"></a>`startCursor` | [`String`](#string) | When paginating backwards, the cursor to continue. |
### `PagesDeploymentRegistry`
Represents the Geo replication and verification state of a pages_deployment.
#### Fields
| Name | Type | Description |
| ---- | ---- | ----------- |
| <a id="pagesdeploymentregistrycreatedat"></a>`createdAt` | [`Time`](#time) | Timestamp when the PagesDeploymentRegistry was created. |
| <a id="pagesdeploymentregistryid"></a>`id` | [`ID!`](#id) | ID of the PagesDeploymentRegistry. |
| <a id="pagesdeploymentregistrylastsyncfailure"></a>`lastSyncFailure` | [`String`](#string) | Error message during sync of the PagesDeploymentRegistry. |
| <a id="pagesdeploymentregistrylastsyncedat"></a>`lastSyncedAt` | [`Time`](#time) | Timestamp of the most recent successful sync of the PagesDeploymentRegistry. |
| <a id="pagesdeploymentregistrypagesdeploymentid"></a>`pagesDeploymentId` | [`ID!`](#id) | ID of the Pages Deployment. |
| <a id="pagesdeploymentregistryretryat"></a>`retryAt` | [`Time`](#time) | Timestamp after which the PagesDeploymentRegistry should be resynced. |
| <a id="pagesdeploymentregistryretrycount"></a>`retryCount` | [`Int`](#int) | Number of consecutive failed sync attempts of the PagesDeploymentRegistry. |
| <a id="pagesdeploymentregistrystate"></a>`state` | [`RegistryState`](#registrystate) | Sync state of the PagesDeploymentRegistry. |
### `PathLock`
Represents a file or directory in the project repository that has been locked.
......
......@@ -168,7 +168,7 @@ You must have the [Owner role](../../user/permissions.md#group-members-permissio
| Attribute | Description |
| ------------ | ----------- |
| Type | Displays the runner type: `group` or `specific`, together with the optional states `locked` and `paused` |
| Type | Displays the runner type: `group` or `specific`, and the optional state `paused` |
| Runner token | Token used to identify the runner, and that the runner uses to communicate with the GitLab instance |
| Description | Description given to the runner when it was created |
| Version | GitLab Runner version |
......@@ -242,10 +242,10 @@ You can configure a specific runner so it is "locked" and cannot be enabled for
This setting can be enabled when you first [register a runner](https://docs.gitlab.com/runner/register/),
but can also be changed later.
To lock or unlock a runner:
To lock or unlock a specific runner:
1. Go to the project's **Settings > CI/CD** and expand the **Runners** section.
1. Find the runner you want to lock or unlock. Make sure it's enabled.
1. Find the specific runner you want to lock or unlock. Make sure it's enabled. You cannot lock shared or group runners.
1. Click the pencil button.
1. Check the **Lock to current projects** option.
1. Click **Save changes**.
# frozen_string_literal: true
module Geo
class PagesDeploymentRegistryFinder
include FrameworkRegistryFinder
end
end
# frozen_string_literal: true
module Resolvers
module Geo
class PagesDeploymentRegistriesResolver < BaseResolver
type ::Types::Geo::GeoNodeType.connection_type, null: true
include RegistriesResolver
end
end
end
......@@ -50,6 +50,11 @@ module Types
null: true,
resolver: ::Resolvers::Geo::PipelineArtifactRegistriesResolver,
description: 'Find pipeline artifact registries on this Geo node.'
field :pages_deployment_registries, ::Types::Geo::PagesDeploymentRegistryType.connection_type,
null: true,
resolver: ::Resolvers::Geo::PagesDeploymentRegistriesResolver,
description: 'Find Pages Deployment registries on this Geo node',
feature_flag: :geo_pages_deployment_replication
end
end
end
# frozen_string_literal: true
module Types
module Geo
# rubocop:disable Graphql/AuthorizeTypes because it is included
class PagesDeploymentRegistryType < BaseObject
include ::Types::Geo::RegistryType
graphql_name 'PagesDeploymentRegistry'
description 'Represents the Geo replication and verification state of a pages_deployment'
field :pages_deployment_id, GraphQL::Types::ID, null: false, description: 'ID of the Pages Deployment.'
end
end
end
# frozen_string_literal: true
module EE
module PagesDeployment
extend ActiveSupport::Concern
prepended do
include ::Gitlab::Geo::ReplicableModel
with_replicator Geo::PagesDeploymentReplicator
end
class_methods do
def replicables_for_current_secondary(primary_key_in)
node = ::Gitlab::Geo.current_node
primary_key_in(primary_key_in)
.merge(selective_sync_scope(node))
.merge(object_storage_scope(node))
end
private
def object_storage_scope(node)
return all if node.sync_object_storage?
with_files_stored_locally
end
def selective_sync_scope(node)
return all unless node.selective_sync?
project_id_in(node.projects)
end
end
def log_geo_deleted_event
# Keep empty for now. Should be addressed in future
# by https://gitlab.com/gitlab-org/gitlab/-/issues/232917
end
end
end
# frozen_string_literal: true
class Geo::PagesDeploymentRegistry < Geo::BaseRegistry
include ::Geo::ReplicableRegistry
MODEL_CLASS = ::PagesDeployment
MODEL_FOREIGN_KEY = :pages_deployment_id
belongs_to :pages_deployment, class_name: 'PagesDeployment'
end
# frozen_string_literal: true
module Geo
class PagesDeploymentReplicator < Gitlab::Geo::Replicator
include ::Geo::BlobReplicatorStrategy
extend ::Gitlab::Utils::Override
def self.model
::PagesDeployment
end
def carrierwave_uploader
model_record.file
end
# The feature flag follows the format `geo_#{replicable_name}_replication`,
# so here it would be `geo_pages_deployment_replication`
def self.replication_enabled_by_default?
false
end
override :verification_feature_flag_enabled?
def self.verification_feature_flag_enabled?
false
end
end
end
......@@ -30,7 +30,8 @@ module Geo
Geo::TerraformStateVersionRegistry,
Geo::UploadRegistry,
Geo::SnippetRepositoryRegistry,
Geo::GroupWikiRepositoryRegistry
Geo::GroupWikiRepositoryRegistry,
Geo::PagesDeploymentRegistry
].freeze
BATCH_SIZE = 10000
......
---
name: geo_pages_deployment_replication
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/68662
rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/337676
milestone: '14.3'
type: development
group: group::geo
default_enabled: false
# frozen_string_literal: true
class CreatePagesDeploymentRegistry < ActiveRecord::Migration[6.1]
include Gitlab::Database::MigrationHelpers
def change
create_table :pages_deployment_registry, id: :bigserial, force: :cascade do |t|
t.bigint :pages_deployment_id, null: false
t.datetime_with_timezone :created_at, null: false
t.datetime_with_timezone :last_synced_at
t.datetime_with_timezone :retry_at
t.integer :state, default: 0, null: false, limit: 2
t.integer :retry_count, default: 0, limit: 2, null: false
t.string :last_sync_failure, limit: 255 # rubocop:disable Migration/PreventStrings see https://gitlab.com/gitlab-org/gitlab/-/issues/323806
t.index :pages_deployment_id, name: :index_pages_deployment_registry_on_pages_deployment_id, unique: true
t.index :retry_at
t.index :state
end
end
end
......@@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema.define(version: 2021_06_24_160455) do
ActiveRecord::Schema.define(version: 2021_08_20_152707) do
# These are extensions that must be enabled in order to support this database
enable_extension "plpgsql"
......@@ -160,6 +160,19 @@ ActiveRecord::Schema.define(version: 2021_06_24_160455) do
t.index ["verified_at"], name: "package_file_registry_pending_verification", order: "NULLS FIRST", where: "((state = 2) AND (verification_state = 0))"
end
create_table "pages_deployment_registry", force: :cascade do |t|
t.bigint "pages_deployment_id", null: false
t.datetime_with_timezone "created_at", null: false
t.datetime_with_timezone "last_synced_at"
t.datetime_with_timezone "retry_at"
t.integer "state", limit: 2, default: 0, null: false
t.integer "retry_count", limit: 2, default: 0, null: false
t.string "last_sync_failure", limit: 255
t.index ["pages_deployment_id"], name: "index_pages_deployment_registry_on_pages_deployment_id", unique: true
t.index ["retry_at"], name: "index_pages_deployment_registry_on_retry_at"
t.index ["state"], name: "index_pages_deployment_registry_on_state"
end
create_table "pipeline_artifact_registry", force: :cascade do |t|
t.bigint "pipeline_artifact_id", null: false
t.datetime_with_timezone "created_at", null: false
......
......@@ -26,7 +26,8 @@ module Gitlab
::Geo::TerraformStateVersionReplicator,
::Geo::SnippetRepositoryReplicator,
::Geo::GroupWikiRepositoryReplicator,
::Geo::PipelineArtifactReplicator
::Geo::PipelineArtifactReplicator,
::Geo::PagesDeploymentReplicator
].freeze
def self.current_node
......
# frozen_string_literal: true
FactoryBot.define do
factory :geo_pages_deployment_registry, class: 'Geo::PagesDeploymentRegistry' do
pages_deployment # This association should have data, like a file or repository
state { Geo::PagesDeploymentRegistry.state_value(:pending) }
trait :synced do
state { Geo::PagesDeploymentRegistry.state_value(:synced) }
last_synced_at { 5.days.ago }
end
trait :failed do
state { Geo::PagesDeploymentRegistry.state_value(:failed) }
last_synced_at { 1.day.ago }
retry_count { 2 }
last_sync_failure { 'Random error' }
end
trait :started do
state { Geo::PagesDeploymentRegistry.state_value(:started) }
last_synced_at { 1.day.ago }
retry_count { 0 }
end
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Geo::PagesDeploymentRegistryFinder do
it_behaves_like 'a framework registry finder', :geo_pages_deployment_registry
end
......@@ -78,6 +78,18 @@
"package_files_verification_total_count",
"package_files_verified_count",
"package_files_verified_in_percentage",
"pages_deployments_count",
"pages_deployments_checksum_failed_count",
"pages_deployments_checksum_total_count",
"pages_deployments_checksummed_count",
"pages_deployments_registry_count",
"pages_deployments_failed_count",
"pages_deployments_synced_count",
"pages_deployments_synced_in_percentage",
"pages_deployments_verification_failed_count",
"pages_deployments_verification_total_count",
"pages_deployments_verified_count",
"pages_deployments_verified_in_percentage",
"terraform_state_versions_count",
"terraform_state_versions_checksum_failed_count",
"terraform_state_versions_checksum_total_count",
......@@ -244,6 +256,18 @@
"package_files_verification_total_count": { "type": ["integer", "null"] },
"package_files_verified_count": { "type": ["integer", "null"] },
"package_files_verified_in_percentage": { "type": "string" },
"pages_deployments_count": { "type": ["integer", "null"] },
"pages_deployments_checksummed_count": { "type": ["integer", "null"] },
"pages_deployments_checksum_failed_count": { "type": ["integer", "null"] },
"pages_deployments_checksum_total_count": { "type": ["integer", "null"] },
"pages_deployments_registry_count": { "type": ["integer", "null"] },
"pages_deployments_failed_count": { "type": ["integer", "null"] },
"pages_deployments_synced_count": { "type": ["integer", "null"] },
"pages_deployments_synced_in_percentage": { "type": "string" },
"pages_deployments_verification_failed_count": { "type": ["integer", "null"] },
"pages_deployments_verification_total_count": { "type": ["integer", "null"] },
"pages_deployments_verified_count": { "type": ["integer", "null"] },
"pages_deployments_verified_in_percentage": { "type": "string" },
"terraform_state_versions_count": { "type": ["integer", "null"] },
"terraform_state_versions_checksummed_count": { "type": ["integer", "null"] },
"terraform_state_versions_checksum_failed_count": { "type": ["integer", "null"] },
......
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Resolvers::Geo::PagesDeploymentRegistriesResolver do
it_behaves_like 'a Geo registries resolver', :geo_pages_deployment_registry
end
......@@ -16,6 +16,7 @@ RSpec.describe GitlabSchema.types['GeoNode'] do
terraform_state_version_registries group_wiki_repository_registries
lfs_object_registries
pipeline_artifact_registries
pages_deployment_registries
]
expect(described_class).to have_graphql_fields(*expected_fields)
......
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe GitlabSchema.types['PagesDeploymentRegistry'] do
it_behaves_like 'a Geo registry type'
it 'has the expected fields (other than those included in RegistryType)' do
expected_fields = %i[pages_deployment_id]
expect(described_class).to have_graphql_fields(*expected_fields).at_least
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Geo::PagesDeploymentRegistry, :geo, type: :model do
let_it_be(:registry) { create(:geo_pages_deployment_registry) }
specify 'factory is valid' do
expect(registry).to be_valid
end
include_examples 'a Geo framework registry'
end
......@@ -1129,6 +1129,7 @@ RSpec.describe GeoNodeStatus, :geo do
Geo::TerraformStateVersionReplicator | :terraform_state_version | :geo_terraform_state_version_registry
Geo::SnippetRepositoryReplicator | :snippet_repository | :geo_snippet_repository_registry
Geo::GroupWikiRepositoryReplicator | :group_wiki_repository | :geo_group_wiki_repository_registry
Geo::PagesDeploymentReplicator | :pages_deployment | :geo_pages_deployment_registry
end
with_them do
......
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Geo::PagesDeploymentReplicator do
let(:model_record) { build(:pages_deployment) }
include_examples 'a blob replicator'
end
......@@ -44,4 +44,11 @@ RSpec.describe 'Gets registries' do
registry_factory: :geo_pipeline_artifact_registry,
registry_foreign_key_field_name: 'pipelineArtifactId'
}
it_behaves_like 'gets registries for', {
field_name: 'pagesDeploymentRegistries',
registry_class_name: 'PagesDeploymentRegistry',
registry_factory: :geo_pages_deployment_registry,
registry_foreign_key_field_name: 'pagesDeploymentId'
}
end
......@@ -87,6 +87,7 @@ RSpec.describe Geo::Secondary::RegistryConsistencyWorker, :geo do
terraform_state_version = create(:terraform_state_version)
pipeline_artifact = create(:ci_pipeline_artifact)
upload = create(:upload)
pages_deployment = create(:pages_deployment)
expect(Geo::ContainerRepositoryRegistry.where(container_repository_id: container_repository.id).count).to eq(0)
expect(Geo::DesignRegistry.where(project_id: project.id).count).to eq(0)
......@@ -98,6 +99,7 @@ RSpec.describe Geo::Secondary::RegistryConsistencyWorker, :geo do
expect(Geo::ProjectRegistry.where(project_id: project.id).count).to eq(0)
expect(Geo::TerraformStateVersionRegistry.where(terraform_state_version_id: terraform_state_version.id).count).to eq(0)
expect(Geo::UploadRegistry.where(file_id: upload.id).count).to eq(0)
expect(Geo::PagesDeploymentRegistry.where(pages_deployment: pages_deployment.id).count).to eq(0)
subject.perform
......@@ -111,6 +113,7 @@ RSpec.describe Geo::Secondary::RegistryConsistencyWorker, :geo do
expect(Geo::ProjectRegistry.where(project_id: project.id).count).to eq(1)
expect(Geo::TerraformStateVersionRegistry.where(terraform_state_version_id: terraform_state_version.id).count).to eq(1)
expect(Geo::UploadRegistry.where(file_id: upload.id).count).to eq(1)
expect(Geo::PagesDeploymentRegistry.where(pages_deployment: pages_deployment.id).count).to eq(1)
end
context 'when the current Geo node is disabled or primary' do
......
......@@ -129,13 +129,14 @@ RSpec.describe 'gitlab:db namespace rake task', :silence_stdout do
let(:output) { StringIO.new }
before do
structure_files = %w[db/structure.sql db/ci_structure.sql]
structure_files = %w[structure.sql ci_structure.sql]
allow(File).to receive(:open).and_call_original
structure_files.each do |structure_file|
structure_files.each do |structure_file_name|
structure_file = File.join(ActiveRecord::Tasks::DatabaseTasks.db_dir, structure_file_name)
stub_file_read(structure_file, content: input)
allow(File).to receive(:open).with(Rails.root.join(structure_file).to_s, any_args).and_yield(output)
allow(File).to receive(:open).with(structure_file.to_s, any_args).and_yield(output)
end
end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment