Commit 56d96ad7 authored by GitLab Bot's avatar GitLab Bot

Add latest changes from gitlab-org/gitlab@master

parent 8078bd18
......@@ -67,7 +67,7 @@ docs lint:
# Check the internal anchor links
- bundle exec nanoc check internal_anchors
graphql-docs-verify:
graphql-reference-verify:
extends:
- .only-ee
- .default-tags
......@@ -82,3 +82,4 @@ graphql-docs-verify:
needs: ["setup-test-env"]
script:
- bundle exec rake gitlab:graphql:check_docs
- bundle exec rake gitlab:graphql:check_schema
......@@ -3,10 +3,12 @@
class Projects::ReleasesController < Projects::ApplicationController
# Authorize
before_action :require_non_empty_project, except: [:index]
before_action :release, only: %i[edit update]
before_action :authorize_read_release!
before_action do
push_frontend_feature_flag(:release_edit_page, project)
end
before_action :authorize_update_release!, only: %i[edit update]
def index
respond_to do |format|
......@@ -22,4 +24,25 @@ class Projects::ReleasesController < Projects::ApplicationController
def releases
ReleasesFinder.new(@project, current_user).execute
end
def edit
respond_to do |format|
format.html { render 'edit' }
end
end
private
def authorize_update_release!
access_denied! unless Feature.enabled?(:release_edit_page, project)
access_denied! unless can?(current_user, :update_release, release)
end
def release
@release ||= project.releases.find_by_tag!(sanitized_tag_name)
end
def sanitized_tag_name
CGI.unescape(params[:tag])
end
end
......@@ -6,9 +6,11 @@ class ReleasesFinder
@current_user = current_user
end
def execute
def execute(preload: true)
return Release.none unless Ability.allowed?(@current_user, :read_release, @project)
@project.releases.sorted
releases = @project.releases
releases = releases.preloaded if preload
releases.sorted
end
end
......@@ -3,6 +3,7 @@
module Clusters
module Providers
class Aws < ApplicationRecord
include Gitlab::Utils::StrongMemoize
include Clusters::Concerns::ProviderStatus
self.table_name = 'cluster_providers_aws'
......@@ -42,6 +43,18 @@ module Clusters
session_token: nil
)
end
def api_client
strong_memoize(:api_client) do
::Aws::CloudFormation::Client.new(credentials: credentials, region: region)
end
end
def credentials
strong_memoize(:credentials) do
::Aws::Credentials.new(access_key_id, secret_access_key, session_token)
end
end
end
end
end
......@@ -27,13 +27,17 @@ class Release < ApplicationRecord
validates_associated :milestone_releases, message: -> (_, obj) { obj[:value].map(&:errors).map(&:full_messages).join(",") }
scope :sorted, -> { order(released_at: :desc) }
scope :with_project_and_namespace, -> { includes(project: :namespace) }
scope :preloaded, -> { includes(project: :namespace) }
delegate :repository, to: :project
after_commit :create_evidence!, on: :create
after_commit :notify_new_release, on: :create
def to_param
CGI.escape(tag)
end
def commit
strong_memoize(:commit) do
repository.commit(actual_sha)
......
......@@ -31,6 +31,12 @@ class ReleasePresenter < Gitlab::View::Presenter::Delegated
project_issues_url(project, params_for_issues_and_mrs)
end
def edit_url
return unless release_edit_page_available?
edit_project_release_url(project, release)
end
private
def can_download_code?
......@@ -44,4 +50,8 @@ class ReleasePresenter < Gitlab::View::Presenter::Delegated
def release_mr_issue_urls_available?
::Feature.enabled?(:release_mr_issue_urls, project)
end
def release_edit_page_available?
::Feature.enabled?(:release_edit_page, project)
end
end
# frozen_string_literal: true
module Clusters
module Aws
class FetchCredentialsService
attr_reader :provider
MissingRoleError = Class.new(StandardError)
def initialize(provider)
@provider = provider
end
def execute
raise MissingRoleError.new('AWS provisioning role not configured') unless provision_role.present?
::Aws::AssumeRoleCredentials.new(
client: client,
role_arn: provision_role.role_arn,
role_session_name: session_name,
external_id: provision_role.role_external_id
).credentials
end
private
def provision_role
provider.created_by_user.aws_role
end
def client
::Aws::STS::Client.new(credentials: gitlab_credentials, region: provider.region)
end
def gitlab_credentials
::Aws::Credentials.new(access_key_id, secret_access_key)
end
##
# This setting is not yet configurable or documented as these
# services are not currently used. This will be addressed in
# https://gitlab.com/gitlab-org/gitlab/merge_requests/18307
def access_key_id
Gitlab.config.kubernetes.provisioners.aws.access_key_id
end
##
# This setting is not yet configurable or documented as these
# services are not currently used. This will be addressed in
# https://gitlab.com/gitlab-org/gitlab/merge_requests/18307
def secret_access_key
Gitlab.config.kubernetes.provisioners.aws.secret_access_key
end
def session_name
"gitlab-eks-cluster-#{provider.cluster_id}-user-#{provider.created_by_user_id}"
end
end
end
end
# frozen_string_literal: true
module Clusters
module Aws
class FinalizeCreationService
include Gitlab::Utils::StrongMemoize
attr_reader :provider
delegate :cluster, to: :provider
def execute(provider)
@provider = provider
configure_provider
create_gitlab_service_account!
configure_platform_kubernetes
configure_node_authentication!
cluster.save!
rescue ::Aws::CloudFormation::Errors::ServiceError => e
log_service_error(e.class.name, provider.id, e.message)
provider.make_errored!(s_('ClusterIntegration|Failed to fetch CloudFormation stack: %{message}') % { message: e.message })
rescue Kubeclient::HttpError => e
log_service_error(e.class.name, provider.id, e.message)
provider.make_errored!(s_('ClusterIntegration|Failed to run Kubeclient: %{message}') % { message: e.message })
rescue ActiveRecord::RecordInvalid => e
log_service_error(e.class.name, provider.id, e.message)
provider.make_errored!(s_('ClusterIntegration|Failed to configure EKS provider: %{message}') % { message: e.message })
end
private
def create_gitlab_service_account!
Clusters::Kubernetes::CreateOrUpdateServiceAccountService.gitlab_creator(
kube_client,
rbac: true
).execute
end
def configure_provider
provider.status_event = :make_created
end
def configure_platform_kubernetes
cluster.build_platform_kubernetes(
api_url: cluster_endpoint,
ca_cert: cluster_certificate,
token: request_kubernetes_token)
end
def request_kubernetes_token
Clusters::Kubernetes::FetchKubernetesTokenService.new(
kube_client,
Clusters::Kubernetes::GITLAB_ADMIN_TOKEN_NAME,
Clusters::Kubernetes::GITLAB_SERVICE_ACCOUNT_NAMESPACE
).execute
end
def kube_client
@kube_client ||= build_kube_client!(
cluster_endpoint,
cluster_certificate
)
end
def build_kube_client!(api_url, ca_pem)
raise "Incomplete settings" unless api_url
Gitlab::Kubernetes::KubeClient.new(
api_url,
auth_options: kubeclient_auth_options,
ssl_options: kubeclient_ssl_options(ca_pem),
http_proxy_uri: ENV['http_proxy']
)
end
def kubeclient_auth_options
{ bearer_token: Kubeclient::AmazonEksCredentials.token(provider.credentials, cluster.name) }
end
def kubeclient_ssl_options(ca_pem)
opts = { verify_ssl: OpenSSL::SSL::VERIFY_PEER }
if ca_pem.present?
opts[:cert_store] = OpenSSL::X509::Store.new
opts[:cert_store].add_cert(OpenSSL::X509::Certificate.new(ca_pem))
end
opts
end
def cluster_stack
@cluster_stack ||= provider.api_client.describe_stacks(stack_name: provider.cluster.name).stacks.first
end
def stack_output_value(key)
cluster_stack.outputs.detect { |output| output.output_key == key }.output_value
end
def node_instance_role_arn
stack_output_value('NodeInstanceRole')
end
def cluster_endpoint
strong_memoize(:cluster_endpoint) do
stack_output_value('ClusterEndpoint')
end
end
def cluster_certificate
strong_memoize(:cluster_certificate) do
Base64.decode64(stack_output_value('ClusterCertificate'))
end
end
def configure_node_authentication!
kube_client.create_config_map(node_authentication_config)
end
def node_authentication_config
Gitlab::Kubernetes::ConfigMaps::AwsNodeAuth.new(node_instance_role_arn).generate
end
def logger
@logger ||= Gitlab::Kubernetes::Logger.build
end
def log_service_error(exception, provider_id, message)
logger.error(
exception: exception.class.name,
service: self.class.name,
provider_id: provider_id,
message: message
)
end
end
end
end
# frozen_string_literal: true
module Clusters
module Aws
class ProvisionService
attr_reader :provider
def execute(provider)
@provider = provider
configure_provider_credentials
provision_cluster
if provider.make_creating
WaitForClusterCreationWorker.perform_in(
Clusters::Aws::VerifyProvisionStatusService::INITIAL_INTERVAL,
provider.cluster_id
)
else
provider.make_errored!("Failed to update provider record; #{provider.errors.full_messages}")
end
rescue Clusters::Aws::FetchCredentialsService::MissingRoleError
provider.make_errored!('Amazon role is not configured')
rescue ::Aws::Errors::MissingCredentialsError, Settingslogic::MissingSetting
provider.make_errored!('Amazon credentials are not configured')
rescue ::Aws::STS::Errors::ServiceError => e
provider.make_errored!("Amazon authentication failed; #{e.message}")
rescue ::Aws::CloudFormation::Errors::ServiceError => e
provider.make_errored!("Amazon CloudFormation request failed; #{e.message}")
end
private
def credentials
@credentials ||= Clusters::Aws::FetchCredentialsService.new(provider).execute
end
def configure_provider_credentials
provider.update!(
access_key_id: credentials.access_key_id,
secret_access_key: credentials.secret_access_key,
session_token: credentials.session_token
)
end
def provision_cluster
provider.api_client.create_stack(
stack_name: provider.cluster.name,
template_body: stack_template,
parameters: parameters,
capabilities: ["CAPABILITY_IAM"]
)
end
def parameters
[
parameter('ClusterName', provider.cluster.name),
parameter('ClusterRole', provider.role_arn),
parameter('ClusterControlPlaneSecurityGroup', provider.security_group_id),
parameter('VpcId', provider.vpc_id),
parameter('Subnets', provider.subnet_ids.join(',')),
parameter('NodeAutoScalingGroupDesiredCapacity', provider.num_nodes.to_s),
parameter('NodeInstanceType', provider.instance_type),
parameter('KeyName', provider.key_name)
]
end
def parameter(key, value)
{ parameter_key: key, parameter_value: value }
end
def stack_template
File.read(Rails.root.join('vendor', 'aws', 'cloudformation', 'eks_cluster.yaml'))
end
end
end
end
# frozen_string_literal: true
module Clusters
module Aws
class VerifyProvisionStatusService
attr_reader :provider
INITIAL_INTERVAL = 5.minutes
POLL_INTERVAL = 1.minute
TIMEOUT = 30.minutes
def execute(provider)
@provider = provider
case cluster_stack.stack_status
when 'CREATE_IN_PROGRESS'
continue_creation
when 'CREATE_COMPLETE'
finalize_creation
else
provider.make_errored!("Unexpected status; #{cluster_stack.stack_status}")
end
rescue ::Aws::CloudFormation::Errors::ServiceError => e
provider.make_errored!("Amazon CloudFormation request failed; #{e.message}")
end
private
def cluster_stack
@cluster_stack ||= provider.api_client.describe_stacks(stack_name: provider.cluster.name).stacks.first
end
def continue_creation
if timeout_threshold.future?
WaitForClusterCreationWorker.perform_in(POLL_INTERVAL, provider.cluster_id)
else
provider.make_errored!(_('Kubernetes cluster creation time exceeds timeout; %{timeout}') % { timeout: TIMEOUT })
end
end
def timeout_threshold
cluster_stack.creation_time + TIMEOUT
end
def finalize_creation
Clusters::Aws::FinalizeCreationService.new.execute(provider)
end
end
end
end
......@@ -9,7 +9,11 @@ class ClusterProvisionWorker
def perform(cluster_id)
Clusters::Cluster.find_by_id(cluster_id).try do |cluster|
cluster.provider.try do |provider|
Clusters::Gcp::ProvisionService.new.execute(provider) if cluster.gcp?
if cluster.gcp?
Clusters::Gcp::ProvisionService.new.execute(provider)
elsif cluster.aws?
Clusters::Aws::ProvisionService.new.execute(provider)
end
end
end
end
......
......@@ -7,7 +7,7 @@ class NewReleaseWorker
feature_category :release_orchestration
def perform(release_id)
release = Release.with_project_and_namespace.find_by_id(release_id)
release = Release.preloaded.find_by_id(release_id)
return unless release
NotificationService.new.send_new_release_notifications(release)
......
......@@ -9,7 +9,11 @@ class WaitForClusterCreationWorker
def perform(cluster_id)
Clusters::Cluster.find_by_id(cluster_id).try do |cluster|
cluster.provider.try do |provider|
Clusters::Gcp::VerifyProvisionStatusService.new.execute(provider) if cluster.gcp?
if cluster.gcp?
Clusters::Gcp::VerifyProvisionStatusService.new.execute(provider)
elsif cluster.aws?
Clusters::Aws::VerifyProvisionStatusService.new.execute(provider)
end
end
end
end
......
......@@ -179,7 +179,7 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do
end
end
resources :releases, only: [:index]
resources :releases, only: [:index, :edit], param: :tag, constraints: { tag: %r{[^/]+} }
resources :starrers, only: [:index]
resources :forks, only: [:index, :new, :create]
resources :group_links, only: [:index, :create, :update, :destroy], constraints: { id: /\d+/ }
......
......@@ -53,6 +53,11 @@ GitLab's GraphQL reference [is available](reference/index.md).
It is automatically generated from GitLab's GraphQL schema and embedded in a Markdown file.
Machine-readable versions are also available:
- [JSON format](reference/gitlab_schema.json)
- [IDL format](reference/gitlab_schema.graphql)
## GraphiQL
The API can be explored by using the GraphiQL IDE, it is available on your
......
This diff is collapsed.
This diff is collapsed.
......@@ -551,7 +551,7 @@ it 'returns a successful response' do
end
```
## Documentation
## Documentation and Schema
For information on generating GraphQL documentation, see
[Rake tasks related to GraphQL](rake_tasks.md#update-graphql-documentation).
For information on generating GraphQL documentation and schema files, see
[Rake tasks related to GraphQL](rake_tasks.md#update-graphql-documentation-and-schema-definitions).
......@@ -74,6 +74,50 @@ We have some Webmock stubs in
[`KubernetesHelpers`](https://gitlab.com/gitlab-org/gitlab/blob/master/spec/support/helpers/kubernetes_helpers.rb)
which can help with mocking out calls to Kubernetes API in your tests.
### Amazon EKS integration
This section outlines the process for allowing a GitLab instance to create EKS clusters.
The following prerequisites are required:
A `Customer` AWS account. This is the account in which the
EKS cluster will be created. The following resources must be present:
- A provisioning role that has permissions to create the cluster
and associated resources. It must list the `GitLab` AWS account
as a trusted entity.
- A VPC, management role, security group, and subnets for use by the cluster.
A `GitLab` AWS account. This is the account which performs
the provisioning actions. The following resources must be present:
- A service account with permissions to assume the provisioning
role in the `Customer` account above.
- Credentials for this service account configured in GitLab via
the `kubernetes` section of `gitlab.yml`
The process for creating a cluster is as follows:
1. Using the :provision_role_external_id, GitLab assumes the role provided
by :provision_role_arn and stores a set of temporary credentials on the
provider record. By default these credentials are valid for one hour.
1. A CloudFormation stack is created, based on the
[`AWS CloudFormation EKS template`](https://gitlab.com/gitlab-org/gitlab/blob/master/vendor/aws/cloudformation/eks_cluster.yaml).
This triggers creation of all resources required for an EKS cluster.
1. GitLab polls the status of the stack until all resources are ready,
which takes somewhere between 10 and 15 minutes in most cases.
1. When the stack is ready, GitLab stores the cluster details and generates
another set of temporary credentials, this time to allow connecting to
the cluster via Kubeclient. These credentials are valid for one minute.
1. GitLab configures the worker nodes so that they are able to authenticate
to the cluster, and creates a service account for itself for future operations.
1. Credentials that are no longer required are removed. This deletes the following
attributes:
- `access_key_id`
- `secret_access_key`
- `session_token`
## Security
### SSRF
......
......@@ -221,7 +221,7 @@ bundle exec rake db:obsolete_ignored_columns
Feel free to remove their definitions from their `ignored_columns` definitions.
## Update GraphQL Documentation
## Update GraphQL Documentation and Schema definitions
To generate GraphQL documentation based on the GitLab schema, run:
......@@ -243,3 +243,13 @@ The actual renderer is at `Gitlab::Graphql::Docs::Renderer`.
`@parsed_schema` is an instance variable that the `graphql-docs` gem expects to have available.
`Gitlab::Graphql::Docs::Helper` defines the `object` method we currently use. This is also where you
should implement any new methods for new types you'd like to display.
### Update machine-readable schema files
To generate GraphQL schema files based on the GitLab schema, run:
```shell
bundle exec rake gitlab:graphql:schema:dump
```
This uses graphql-ruby's built-in rake tasks to generate files in both [IDL](https://www.prisma.io/blog/graphql-sdl-schema-definition-language-6755bcb9ce51) and JSON formats.
......@@ -1324,6 +1324,7 @@ module API
expose :_links do
expose :merge_requests_url, expose_nil: false
expose :issues_url, expose_nil: false
expose :edit_url, expose_nil: false
end
private
......
# frozen_string_literal: true
module Gitlab
module Kubernetes
module ConfigMaps
class AwsNodeAuth
attr_reader :node_role
def initialize(node_role)
@node_role = node_role
end
def generate
Kubeclient::Resource.new(
metadata: metadata,
data: data
)
end
private
def metadata
{
'name' => 'aws-auth',
'namespace' => 'kube-system'
}
end
def data
{ 'mapRoles' => instance_role_config(node_role) }
end
def instance_role_config(role)
[{
'rolearn' => role,
'username' => 'system:node:{{EC2PrivateDNSName}}',
'groups' => [
'system:bootstrappers',
'system:nodes'
]
}].to_yaml
end
end
end
end
end
......@@ -2,10 +2,24 @@
return if Rails.env.production?
require 'graphql/rake_task'
namespace :gitlab do
OUTPUT_DIR = Rails.root.join("doc/api/graphql/reference")
TEMPLATES_DIR = 'lib/gitlab/graphql/docs/templates/'
# Defines tasks for dumping the GraphQL schema:
# - gitlab:graphql:schema:dump
# - gitlab:graphql:schema:idl
# - gitlab:graphql:schema:json
GraphQL::RakeTask.new(
schema_name: 'GitlabSchema',
dependencies: [:environment],
directory: OUTPUT_DIR,
idl_outfile: "gitlab_schema.graphql",
json_outfile: "gitlab_schema.json"
)
namespace :graphql do
desc 'GitLab | Generate GraphQL docs'
task compile_docs: :environment do
......@@ -25,11 +39,20 @@ namespace :gitlab do
if doc == renderer.contents
puts "GraphQL documentation is up to date"
else
puts '#' * 10
puts '#'
puts '# GraphQL documentation is outdated! Please update it by running `bundle exec rake gitlab:graphql:compile_docs`.'
puts '#'
puts '#' * 10
format_output('GraphQL documentation is outdated! Please update it by running `bundle exec rake gitlab:graphql:compile_docs`.')
abort
end
end
desc 'GitLab | Check if GraphQL schemas are up to date'
task check_schema: :environment do
idl_doc = File.read(Rails.root.join(OUTPUT_DIR, 'gitlab_schema.graphql'))
json_doc = File.read(Rails.root.join(OUTPUT_DIR, 'gitlab_schema.json'))
if idl_doc == GitlabSchema.to_definition && json_doc == GitlabSchema.to_json
puts "GraphQL schema is up to date"
else
format_output('GraphQL schema is outdated! Please update it by running `bundle exec rake gitlab:graphql:schema:dump`.')
abort
end
end
......@@ -42,3 +65,12 @@ def render_options
template: Rails.root.join(TEMPLATES_DIR, 'default.md.haml')
}
end
def format_output(str)
heading = '#' * 10
puts heading
puts '#'
puts "# #{str}"
puts '#'
puts heading
end
......@@ -3585,9 +3585,15 @@ msgstr ""
msgid "ClusterIntegration|Every new Google Cloud Platform (GCP) account receives $300 in credit upon %{sign_up_link}. In partnership with Google, GitLab is able to offer an additional $200 for both new and existing GCP accounts to get started with GitLab's Google Kubernetes Engine Integration."
msgstr ""
msgid "ClusterIntegration|Failed to configure EKS provider: %{message}"
msgstr ""
msgid "ClusterIntegration|Failed to configure Google Kubernetes Engine Cluster: %{message}"
msgstr ""
msgid "ClusterIntegration|Failed to fetch CloudFormation stack: %{message}"
msgstr ""
msgid "ClusterIntegration|Failed to request to Google Cloud Platform: %{message}"
msgstr ""
......
......@@ -8,8 +8,7 @@ describe ReleasesFinder do
let(:repository) { project.repository }
let(:v1_0_0) { create(:release, project: project, tag: 'v1.0.0') }
let(:v1_1_0) { create(:release, project: project, tag: 'v1.1.0') }
subject { described_class.new(project, user)}
let(:finder) { described_class.new(project, user) }
before do
v1_0_0.update_attribute(:released_at, 2.days.ago)
......@@ -17,11 +16,13 @@ describe ReleasesFinder do
end
describe '#execute' do
subject { finder.execute(**args) }
let(:args) { {} }
context 'when the user is not part of the project' do
it 'returns no releases' do
releases = subject.execute
expect(releases).to be_empty
is_expected.to be_empty
end
end
......@@ -31,11 +32,25 @@ describe ReleasesFinder do
end
it 'sorts by release date' do
releases = subject.execute
is_expected.to be_present
expect(subject.size).to eq(2)
expect(subject).to eq([v1_1_0, v1_0_0])
end
it 'preloads associations' do
expect(Release).to receive(:preloaded).once.and_call_original
subject
end
context 'when preload is false' do
let(:args) { { preload: false } }
it 'does not preload associations' do
expect(Release).not_to receive(:preloaded)
expect(releases).to be_present
expect(releases.size).to eq(2)
expect(releases).to eq([v1_1_0, v1_0_0])
subject
end
end
end
end
......
......@@ -38,10 +38,11 @@
"additionalProperties": false
},
"_links": {
"required": ["merge_requests_url", "issues_url"],
"required": ["merge_requests_url", "issues_url", "edit_url"],
"properties": {
"merge_requests_url": { "type": "string" },
"issues_url": { "type": "string" }
"issues_url": { "type": "string" },
"edit_url": { "type": "string"}
}
}
},
......
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment