Add latest changes from gitlab-org/gitlab@master

app/assets/javascripts/lib/utils/common_utils.js

@@ -177,6 +177,15 @@ export const urlParamsToArray = (path = '') =>
 
 export const getUrlParamsArray = () => urlParamsToArray(window.location.search);
 
+/**
+ * Accepts encoding string which includes query params being
+ * sent to URL.
+ *
+ * @param {string} path Query param string
+ *
+ * @returns {object} Query params object containing key-value pairs
+ *                   with both key and values decoded into plain string.
+ */
 export const urlParamsToObject = (path = '') =>
   splitPath(path).reduce((dataParam, filterParam) => {
     if (filterParam === '') {
@@ -185,6 +194,7 @@ export const urlParamsToObject = (path = '') =>
 
     const data = dataParam;
     let [key, value] = filterParam.split('=');
+    key = /%\w+/g.test(key) ? decodeURIComponent(key) : key;
     const isArray = key.includes('[]');
     key = key.replace('[]', '');
     value = decodeURIComponent(value.replace(/\+/g, ' '));

app/models/clusters/cluster.rb

@@ -24,6 +24,7 @@ module Clusters
     KUBE_INGRESS_BASE_DOMAIN = 'KUBE_INGRESS_BASE_DOMAIN'
 
     belongs_to :user
+    belongs_to :management_project, class_name: '::Project', optional: true
 
     has_many :cluster_projects, class_name: 'Clusters::Project'
     has_many :projects, through: :cluster_projects, class_name: '::Project'
@@ -63,6 +64,7 @@ module Clusters
     validate :restrict_modification, on: :update
     validate :no_groups, unless: :group_type?
     validate :no_projects, unless: :project_type?
+    validate :unique_management_project_environment_scope
 
     after_save :clear_reactive_cache!
 
@@ -200,6 +202,18 @@ module Clusters
 
     private
 
+    def unique_management_project_environment_scope
+      return unless management_project
+
+      duplicate_management_clusters = management_project.management_clusters
+        .where(environment_scope: environment_scope)
+        .where.not(id: id)
+
+      if duplicate_management_clusters.any?
+        errors.add(:environment_scope, "cannot add duplicated environment scope")
+      end
+    end
+
     def instance_domain
       @instance_domain ||= Gitlab::CurrentSettings.auto_devops_domain
     end

app/models/clusters/clusters_hierarchy.rb

@@ -4,8 +4,9 @@ module Clusters
   class ClustersHierarchy
     DEPTH_COLUMN = :depth
 
-    def initialize(clusterable)
+    def initialize(clusterable, include_management_project: true)
       @clusterable = clusterable
+      @include_management_project = include_management_project
     end
 
     # Returns clusters in order from deepest to highest group
@@ -24,7 +25,7 @@ module Clusters
 
     private
 
-    attr_reader :clusterable
+    attr_reader :clusterable, :include_management_project
 
     def recursive_cte
       cte = Gitlab::SQL::RecursiveCTE.new(:clusters_cte)
@@ -38,12 +39,25 @@ module Clusters
                      raise ArgumentError, "unknown type for #{clusterable}"
                    end
 
+      if clusterable.is_a?(::Project) && include_management_project
+        cte << management_clusters_query
+      end
+
       cte << base_query
       cte << parent_query(cte)
 
       cte
     end
 
+    # Management clusters should be first in the hierarchy so we use 0 for the
+    # depth column.
+    #
+    # group_parent_id is un-used but we still need to match the same number of
+    # columns as other queries in the CTE.
+    def management_clusters_query
+      clusterable.management_clusters.select([clusters_star, 'NULL AS group_parent_id', "0 AS #{DEPTH_COLUMN}"])
+    end
+
     def group_clusters_base_query
       group_parent_id_alias = alias_as_column(groups[:parent_id], 'group_parent_id')
       join_sources = ::Group.left_joins(:clusters).arel.join_sources

app/models/clusters/platforms/kubernetes.rb

@@ -73,7 +73,7 @@ module Clusters
               .append(key: 'KUBE_CA_PEM_FILE', value: ca_pem, file: true)
           end
 
-          if !cluster.managed?
+          if !cluster.managed? || cluster.management_project == project
             namespace = Gitlab::Kubernetes::DefaultNamespace.new(cluster, project: project).from_environment_name(environment_name)
 
             variables

app/models/concerns/deployment_platform.rb

@@ -11,6 +11,10 @@ module DeploymentPlatform
 
   private
 
+  def cluster_management_project_enabled?
+    Feature.enabled?(:cluster_management_project, default_enabled: true)
+  end
+
   def find_deployment_platform(environment)
     find_platform_kubernetes_with_cte(environment) ||
       find_instance_cluster_platform_kubernetes(environment: environment)
@@ -18,7 +22,7 @@ module DeploymentPlatform
 
   # EE would override this and utilize environment argument
   def find_platform_kubernetes_with_cte(_environment)
-    Clusters::ClustersHierarchy.new(self).base_and_ancestors
+    Clusters::ClustersHierarchy.new(self, include_management_project: cluster_management_project_enabled?).base_and_ancestors
       .enabled.default_environment
       .first&.platform_kubernetes
   end

app/models/project.rb

@@ -68,7 +68,7 @@ class Project < ApplicationRecord
     :snippets_access_level, :builds_access_level, :repository_access_level,
     to: :project_feature, allow_nil: true
 
-  delegate :base_dir, :disk_path, :ensure_storage_path_exists, to: :storage
+  delegate :base_dir, :disk_path, to: :storage
 
   delegate :scheduled?, :started?, :in_progress?,
     :failed?, :finished?,
@@ -122,8 +122,6 @@ class Project < ApplicationRecord
   # Storage specific hooks
   after_initialize :use_hashed_storage
   after_create :check_repository_absence!
-  after_create :ensure_storage_path_exists
-  after_save :ensure_storage_path_exists, if: :saved_change_to_namespace_id?
 
   acts_as_ordered_taggable
 
@@ -247,6 +245,7 @@ class Project < ApplicationRecord
   has_one :cluster_project, class_name: 'Clusters::Project'
   has_many :clusters, through: :cluster_project, class_name: 'Clusters::Cluster'
   has_many :kubernetes_namespaces, class_name: 'Clusters::KubernetesNamespace'
+  has_many :management_clusters, class_name: 'Clusters::Cluster', foreign_key: :management_project_id, inverse_of: :management_project
 
   has_many :prometheus_metrics
 

app/models/storage/hashed_project.rb

@@ -27,14 +27,6 @@ module Storage
       "#{base_dir}/#{disk_hash}" if disk_hash
     end
 
-    # TODO: remove this method entirely after 12.4 https://gitlab.com/gitlab-org/gitlab/issues/33244
-    # we no longer need ensure_storage_path_exists to call add_namespace since both creating and moving
-    # repositories will be preceded by a mkdir -p in gitaly to ensure the parent of the destination directory
-    # exists.
-    def ensure_storage_path_exists
-      true
-    end
-
     def rename_repo(old_full_path: nil, new_full_path: nil)
       true
     end

app/models/storage/legacy_project.rb

@@ -23,16 +23,6 @@ module Storage
       project.full_path
     end
 
-    # TODO: remove this method entirely after 12.4 https://gitlab.com/gitlab-org/gitlab/issues/33244
-    # we no longer need ensure_storage_path_exists to call add_namespace since both creating and moving
-    # repositories will be preceded by a mkdir -p in gitaly to ensure the parent of the destination directory
-    # exists.
-    def ensure_storage_path_exists
-      return unless namespace
-
-      true
-    end
-
     def rename_repo(old_full_path: nil, new_full_path: nil)
       old_full_path ||= project.full_path_before_last_save
       new_full_path ||= project.build_full_path

app/services/projects/hashed_storage/migrate_repository_service.rb

@@ -8,7 +8,6 @@ module Projects
 
         @old_storage_version = project.storage_version
         project.storage_version = ::Project::HASHED_STORAGE_FEATURES[:repository]
-        project.ensure_storage_path_exists
 
         @new_disk_path = project.disk_path
 

app/services/projects/hashed_storage/rollback_repository_service.rb

@@ -8,7 +8,6 @@ module Projects
 
         @old_storage_version = project.storage_version
         project.storage_version = nil
-        project.ensure_storage_path_exists
 
         @new_disk_path = project.disk_path
 

app/views/search/_form.html.haml

@@ -18,4 +18,3 @@
       = render 'filter'
     .d-flex-center.flex-column.flex-lg-row
       = button_tag _("Search"), class: "btn btn-success btn-search form-control mt-lg-0 ml-lg-1 align-self-end"
-      = render_if_exists 'search/form_elasticsearch'

app/views/search/show.html.haml

@@ -2,9 +2,10 @@
 - page_title @search_term
 - @hide_breadcrumbs = true
 
-.page-title-holder.d-flex.align-items-center
+.page-title-holder.d-sm-flex.align-items-sm-center
   %h1.page-title<
     = _('Search')
+  = render_if_exists 'search/form_elasticsearch', attrs: { class: 'ml-sm-auto' }
 
 .prepend-top-default
   = render 'search/form'

changelogs/unreleased/cluster_management_projects.yml

+---
+title: Adds management project for a cluster
+merge_request: 17866
+author:
+type: changed

db/migrate/20190930063627_add_management_project_id_to_clusters.rb

+# frozen_string_literal: true
+
+class AddManagementProjectIdToClusters < ActiveRecord::Migration[5.2]
+  DOWNTIME = false
+
+  def change
+    add_column :clusters, :management_project_id, :integer
+  end
+end

db/migrate/20191001040549_add_management_project_id_index_fk_to_clusters.rb

+# frozen_string_literal: true
+
+class AddManagementProjectIdIndexFkToClusters < ActiveRecord::Migration[5.2]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_foreign_key :clusters, :projects, column: :management_project_id, on_delete: :nullify
+    add_concurrent_index :clusters, :management_project_id, where: 'management_project_id IS NOT NULL'
+  end
+
+  def down
+    remove_concurrent_index :clusters, :management_project_id
+    remove_foreign_key_if_exists :clusters, column: :management_project_id
+  end
+end

db/schema.rb

@@ -991,7 +991,9 @@ ActiveRecord::Schema.define(version: 2019_10_04_134055) do
     t.string "domain"
     t.boolean "managed", default: true, null: false
     t.boolean "namespace_per_environment", default: true, null: false
+    t.integer "management_project_id"
     t.index ["enabled"], name: "index_clusters_on_enabled"
+    t.index ["management_project_id"], name: "index_clusters_on_management_project_id", where: "(management_project_id IS NOT NULL)"
     t.index ["user_id"], name: "index_clusters_on_user_id"
   end
 
@@ -3983,6 +3985,7 @@ ActiveRecord::Schema.define(version: 2019_10_04_134055) do
   add_foreign_key "cluster_projects", "clusters", on_delete: :cascade
   add_foreign_key "cluster_projects", "projects", on_delete: :cascade
   add_foreign_key "cluster_providers_gcp", "clusters", on_delete: :cascade
+  add_foreign_key "clusters", "projects", column: "management_project_id", name: "fk_f05c5e5a42", on_delete: :nullify
   add_foreign_key "clusters", "users", on_delete: :nullify
   add_foreign_key "clusters_applications_cert_managers", "clusters", on_delete: :cascade
   add_foreign_key "clusters_applications_helm", "clusters", on_delete: :cascade

doc/user/clusters/management_project.md

+# Cluster management project (alpha)
+
+CAUTION: **Warning:**
+This is an _alpha_ feature, and it is subject to change at any time without
+prior notice.
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab/merge_requests/17866) in GitLab 12.4
+
+A project can be designated as the management project for a cluster.
+A management project can be used to run deployment jobs with
+Kubernetes
+[`cluster-admin`](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles)
+privileges.
+
+This can be useful for:
+
+- Creating pipelines to install cluster-wide applications into your cluster.
+- Any jobs that require `cluster-admin` privileges.
+
+## Permissions
+
+Only the management project will receive `cluster-admin` privileges. All
+other projects will continue to receive [namespace scoped `edit` level privileges](../project/clusters/index.md#rbac-cluster-resources).
+
+## Usage
+
+### Selecting a cluster management project
+
+This will be implemented as part of [this
+issue](https://gitlab.com/gitlab-org/gitlab/issues/32810).
+
+### Configuring your pipeline
+
+After designating a project as the management project for the cluster,
+write a [`.gitlab-ci,yml`](../../ci/yaml/README.md) in that project. For example:
+
+```yaml
+configure cluster:
+  stage: deploy
+  script: kubectl get namespaces
+  environment:
+    name: production
+```
+
+### Setting the environment scope **(PREMIUM)**
+
+[Environment
+scopes](../project/clusters/index.md#setting-the-environment-scope-premium)
+are usable when associating multiple clusters to the same management
+project.
+
+Each scope can only be used by a single cluster for a management project.
+
+For example, let's say the following Kubernetes clusters are associated
+to a management project:
+
+| Cluster     | Environment scope |
+| ----------- | ----------------- |
+| Development | `*`               |
+| Staging     | `staging`         |
+| Production  | `production`      |
+
+The the following environments set in
+[`.gitlab-ci.yml`](../../ci/yaml/README.md) will deploy to the
+Development, Staging, and Production cluster respectively.
+
+```yaml
+stages:
+- deploy
+
+configure development cluster:
+  stage: deploy
+  script: kubectl get namespaces
+  environment:
+    name: development
+
+configure staging cluster:
+  stage: deploy
+  script: kubectl get namespaces
+  environment:
+    name: staging
+
+configure production cluster:
+  stage: deploy
+  script: kubectl get namespaces
+  environment:
+    name: production
+```
+
+## Disabling this feature
+
+This feature is enabled by default. To disable this feature, disable the
+feature flag `:cluster_management_project`.
+
+To check if the feature flag is enabled on your GitLab instance,
+please ask an administrator to execute the following in a Rails console:
+
+```ruby
+Feature.enabled?(:cluster_management_project)     # Check if it's enabled or not.
+Feature.disable(:cluster_management_project)      # Disable the feature flag.
+```

doc/user/discussions/index.md

@@ -25,7 +25,7 @@ creates another standard comment. Replying to a threaded comment creates a reply
 [Markdown] and [quick actions], just as if you replied from the web.
 
 NOTE: **Note:**
-There is a limit of 5,000 comments for every object.
+There is a limit of 5,000 comments for every object, for example: issue, epic, and merge request.
 
 ## Resolvable comments and threads
 

lib/backup/repository.rb

@@ -80,7 +80,6 @@ module Backup
       Project.find_each(batch_size: 1000) do |project|
         progress.print " * #{project.full_path} ... "
         path_to_project_bundle = path_to_bundle(project)
-        project.ensure_storage_path_exists
 
         restore_repo_success = nil
         if File.exist?(path_to_project_bundle)

spec/factories/clusters/clusters.rb

@@ -30,6 +30,10 @@ FactoryBot.define do
       end
     end
 
+    trait :management_project do
+      management_project factory: :project
+    end
+
     trait :namespace_per_environment_disabled do
       namespace_per_environment { false }
     end

spec/lib/gitlab/import_export/all_models.yml

@@ -256,6 +256,7 @@ project:
 - cycle_analytics_stages
 - group
 - namespace
+- management_clusters
 - boards
 - last_event
 - services

spec/models/clusters/cluster_spec.rb

@@ -11,6 +11,7 @@ describe Clusters::Cluster, :use_clean_rails_memory_store_caching do
   subject { build(:cluster) }
 
   it { is_expected.to belong_to(:user) }
+  it { is_expected.to belong_to(:management_project).class_name('::Project') }
   it { is_expected.to have_many(:cluster_projects) }
   it { is_expected.to have_many(:projects) }
   it { is_expected.to have_many(:cluster_groups) }
@@ -289,6 +290,20 @@ describe Clusters::Cluster, :use_clean_rails_memory_store_caching do
         it { is_expected.to be_valid }
       end
     end
+
+    describe 'unique scope for management_project' do
+      let(:project) { create(:project) }
+      let!(:cluster_with_management_project) { create(:cluster, management_project: project) }
+
+      context 'duplicate scopes for the same management project' do
+        let(:cluster) { build(:cluster, management_project: project) }
+
+        it 'adds an error on environment_scope' do
+          expect(cluster).not_to be_valid
+          expect(cluster.errors[:environment_scope].first).to eq('cannot add duplicated environment scope')
+        end
+      end
+    end
   end
 
   describe '.ancestor_clusters_for_clusterable' do

spec/models/clusters/clusters_hierarchy_spec.rb

@@ -4,8 +4,8 @@ require 'spec_helper'
 
 describe Clusters::ClustersHierarchy do
   describe '#base_and_ancestors' do
-    def base_and_ancestors(clusterable)
-      described_class.new(clusterable).base_and_ancestors
+    def base_and_ancestors(clusterable, include_management_project: true)
+      described_class.new(clusterable, include_management_project: include_management_project).base_and_ancestors
     end
 
     context 'project in nested group with clusters at every level' do
@@ -44,14 +44,44 @@ describe Clusters::ClustersHierarchy do
       end
     end
 
+    context 'cluster has management project' do
+      let!(:project_cluster) { create(:cluster, :project, projects: [project]) }
+      let!(:group_cluster) { create(:cluster, :group, groups: [group], management_project: management_project) }
+
+      let(:group) { create(:group) }
+      let(:project) { create(:project, group: group) }
+      let(:management_project) { create(:project) }
+
+      it 'returns clusters for management_project' do
+        expect(base_and_ancestors(management_project)).to eq([group_cluster])
+      end
+
+      it 'returns nothing if include_management_project is false' do
+        expect(base_and_ancestors(management_project, include_management_project: false)).to be_empty
+      end
+
+      it 'returns clusters for project' do
+        expect(base_and_ancestors(project)).to eq([project_cluster, group_cluster])
+      end
+
+      it 'returns clusters for group' do
+        expect(base_and_ancestors(group)).to eq([group_cluster])
+      end
+    end
+
     context 'project in nested group with clusters at some levels' do
-      let!(:child) { create(:cluster, :group, groups: [child_group]) }
+      let!(:child) { create(:cluster, :group, groups: [child_group], management_project: management_project) }
       let!(:ancestor) { create(:cluster, :group, groups: [ancestor_group]) }
 
       let(:ancestor_group) { create(:group) }
       let(:parent_group) { create(:group, parent: ancestor_group) }
       let(:child_group) { create(:group, parent: parent_group) }
       let(:project) { create(:project, group: child_group) }
+      let(:management_project) { create(:project) }
+
+      it 'returns clusters for management_project' do
+        expect(base_and_ancestors(management_project)).to eq([child])
+      end
 
       it 'returns clusters for project' do
         expect(base_and_ancestors(project)).to eq([child, ancestor])

spec/models/clusters/platforms/kubernetes_spec.rb

@@ -241,6 +241,23 @@ describe Clusters::Platforms::Kubernetes do
       it { is_expected.to include(key: 'KUBE_CA_PEM_FILE', value: ca_pem, public: true, file: true) }
     end
 
+    context 'cluster is managed by project' do
+      before do
+        allow(Gitlab::Kubernetes::DefaultNamespace).to receive(:new)
+          .with(cluster, project: project).and_return(double(from_environment_name: namespace))
+
+        allow(platform).to receive(:kubeconfig).with(namespace).and_return('kubeconfig')
+      end
+
+      let(:cluster) { create(:cluster, :group, platform_kubernetes: platform, management_project: project) }
+      let(:namespace) { 'kubernetes-namespace' }
+      let(:kubeconfig) { 'kubeconfig' }
+
+      it { is_expected.to include(key: 'KUBE_TOKEN', value: platform.token, public: false, masked: true) }
+      it { is_expected.to include(key: 'KUBE_NAMESPACE', value: namespace) }
+      it { is_expected.to include(key: 'KUBECONFIG', value: kubeconfig, public: false, file: true) }
+    end
+
     context 'kubernetes namespace exists' do
       let(:variable) { Hash(key: :fake_key, value: 'fake_value') }
       let(:namespace_variables) { Gitlab::Ci::Variables::Collection.new([variable]) }

spec/models/concerns/deployment_platform_spec.rb

@@ -12,6 +12,26 @@ describe DeploymentPlatform do
       it { is_expected.to be_nil }
     end
 
+    context 'when project is the cluster\'s management project ' do
+      let!(:cluster_with_management_project) { create(:cluster, :provided_by_user, management_project: project) }
+
+      context 'cluster_management_project feature is enabled' do
+        it 'returns the cluster with management project' do
+          is_expected.to eq(cluster_with_management_project.platform_kubernetes)
+        end
+      end
+
+      context 'cluster_management_project feature is disabled' do
+        before do
+          stub_feature_flags(cluster_management_project: false)
+        end
+
+        it 'returns nothing' do
+          is_expected.to be_nil
+        end
+      end
+    end
+
     context 'when project has configured kubernetes from CI/CD > Clusters' do
       let!(:cluster) { create(:cluster, :provided_by_gcp, projects: [project]) }
       let(:platform_kubernetes) { cluster.platform_kubernetes }
@@ -45,6 +65,35 @@ describe DeploymentPlatform do
         is_expected.to eq(group_cluster.platform_kubernetes)
       end
 
+      context 'when project is the cluster\'s management project ' do
+        let!(:cluster_with_management_project) { create(:cluster, :provided_by_user, management_project: project) }
+
+        context 'cluster_management_project feature is enabled' do
+          it 'returns the cluster with management project' do
+            is_expected.to eq(cluster_with_management_project.platform_kubernetes)
+          end
+        end
+
+        context 'cluster_management_project feature is disabled' do
+          before do
+            stub_feature_flags(cluster_management_project: false)
+          end
+
+          it 'returns the group cluster' do
+            is_expected.to eq(group_cluster.platform_kubernetes)
+          end
+        end
+      end
+
+      context 'when project is not the cluster\'s management project' do
+        let(:another_project) { create(:project, group: group) }
+        let!(:cluster_with_management_project) { create(:cluster, :provided_by_user, management_project: another_project) }
+
+        it 'returns the group cluster' do
+          is_expected.to eq(group_cluster.platform_kubernetes)
+        end
+      end
+
       context 'when child group has configured kubernetes cluster' do
         let(:child_group1) { create(:group, parent: group) }
         let!(:child_group1_cluster) { create(:cluster_for_group, groups: [child_group1]) }

spec/models/project_spec.rb

@@ -92,6 +92,7 @@ describe Project do
     it { is_expected.to have_many(:pipeline_schedules) }
     it { is_expected.to have_many(:members_and_requesters) }
     it { is_expected.to have_many(:clusters) }
+    it { is_expected.to have_many(:management_clusters).class_name('Clusters::Cluster') }
     it { is_expected.to have_many(:kubernetes_namespaces) }
     it { is_expected.to have_many(:custom_attributes).class_name('ProjectCustomAttribute') }
     it { is_expected.to have_many(:project_badges).class_name('ProjectBadge') }

spec/services/projects/hashed_storage/migrate_repository_service_spec.rb

@@ -91,7 +91,6 @@ describe Projects::HashedStorage::MigrateRepositoryService do
 
       context 'when rollback fails' do
         before do
-          hashed_storage.ensure_storage_path_exists
           gitlab_shell.mv_repository(project.repository_storage, old_disk_path, new_disk_path)
         end
 

spec/services/projects/hashed_storage/rollback_repository_service_spec.rb

@@ -91,7 +91,6 @@ describe Projects::HashedStorage::RollbackRepositoryService, :clean_gitlab_redis
 
       context 'when rollback fails' do
         before do
-          legacy_storage.ensure_storage_path_exists
           gitlab_shell.mv_repository(project.repository_storage, old_disk_path, new_disk_path)
         end
 

spec/workers/namespaceless_project_destroy_worker_spec.rb

@@ -9,7 +9,6 @@ describe NamespacelessProjectDestroyWorker do
 
   before do
     # Stub after_save callbacks that will fail when Project has no namespace
-    allow_any_instance_of(Project).to receive(:ensure_storage_path_exists).and_return(nil)
     allow_any_instance_of(Project).to receive(:update_project_statistics).and_return(nil)
   end