Add latest changes from gitlab-org/gitlab@master

app/controllers/dashboard/projects_controller.rb

@@ -66,7 +66,7 @@ class Dashboard::ProjectsController < Dashboard::ApplicationController
     @total_user_projects_count = ProjectsFinder.new(params: { non_public: true }, current_user: current_user).execute
     @total_starred_projects_count = ProjectsFinder.new(params: { starred: true }, current_user: current_user).execute
 
-    finder_params[:use_cte] = Feature.enabled?(:use_cte_for_projects_finder, default_enabled: true)
+    finder_params[:use_cte] = true if use_cte_for_finder?
 
     projects = ProjectsFinder
                 .new(params: finder_params, current_user: current_user)
@@ -79,6 +79,11 @@ class Dashboard::ProjectsController < Dashboard::ApplicationController
   end
   # rubocop: enable CodeReuse/ActiveRecord
 
+  def use_cte_for_finder?
+    # The starred action loads public projects, which causes the CTE to be less efficient
+    action_name == 'index' && Feature.enabled?(:use_cte_for_projects_finder, default_enabled: true)
+  end
+
   def load_events
     projects = load_projects(params.merge(non_public: true))
 

app/workers/all_queues.yml

@@ -160,7 +160,7 @@
   :resource_boundary: :cpu
   :weight: 1
 - :name: cronjob:remove_unreferenced_lfs_objects
-  :feature_category: :source_code_management
+  :feature_category: :git_lfs
   :has_external_dependencies: 
   :latency_sensitive: 
   :resource_boundary: :unknown

app/workers/remove_unreferenced_lfs_objects_worker.rb

@@ -4,7 +4,7 @@ class RemoveUnreferencedLfsObjectsWorker
   include ApplicationWorker
   include CronjobQueue # rubocop:disable Scalability/CronWorkerContext
 
-  feature_category :source_code_management
+  feature_category :git_lfs
 
   def perform
     LfsObject.destroy_unreferenced

changelogs/unreleased/25970-mirror-pull-for-archived-projects.yml

+---
+title: Disable pull mirror importing for archived projects
+merge_request: 24029
+author:
+type: fixed

changelogs/unreleased/38096-splitmr-add-resource-milestone-events-table-pd.yml

+---
+title: Add migration to create resource milestone events table
+merge_request: 23965
+author:
+type: added

changelogs/unreleased/dast-deploy-bump-0-9-1.yml

+---
+title: Bump DAST deploy auto-deploy-image to 0.9.1
+merge_request: 24232
+author:
+type: other

changelogs/unreleased/fix_graphiql_relative_url.yml

+---
+title: Fix GraphiQL when GitLab is installed under a relative URL
+merge_request: 23143
+author: Mathieu Parent
+type: fixed

changelogs/unreleased/nicolasdular-change-broadcast-index.yml

+---
+title: Change broadcast message index
+merge_request: 23986
+author:
+type: performance

changelogs/unreleased/reschedule-issue-tracker-migration.yml

+---
+title: Migrate issue tracker data to data field tables
+merge_request: 24076
+author:
+type: other

config/feature_categories.yml

@@ -21,6 +21,7 @@
 - cloud_native_installation
 - cluster_cost_optimization
 - cluster_monitoring
+- code_analytics
 - code_quality
 - code_review
 - collection
@@ -31,8 +32,8 @@
 - container_scanning
 - continuous_delivery
 - continuous_integration
-- data_loss_prevention
 - ddos_protection
+- dependency_firewall
 - dependency_proxy
 - dependency_scanning
 - design_management
@@ -42,8 +43,11 @@
 - epics
 - error_tracking
 - feature_flags
+- frontend_foundation
 - fuzzing
+- gdk
 - geo_replication
+- git_lfs
 - gitaly
 - gitlab_handbook
 - gitter
@@ -52,11 +56,11 @@
 - incident_management
 - incremental_rollout
 - infrastructure_as_code
-- integration_testing
 - integrations
 - interactive_application_security_testing
 - internationalization
 - issue_tracking
+- jupyter_notebooks
 - kanban_boards
 - kubernetes_management
 - language_specific
@@ -65,6 +69,7 @@
 - load_testing
 - logging
 - malware_scanning
+- merge_trains
 - metrics
 - omnibus_package
 - package_registry
@@ -93,6 +98,7 @@
 - subgroups
 - synthetic_monitoring
 - system_testing
+- teams
 - templates
 - threat_detection
 - time_tracking
@@ -107,3 +113,4 @@
 - web_ide
 - web_performance
 - wiki
+- workspaces

config/initializers/1_settings.rb

@@ -192,6 +192,7 @@ Settings.gitlab['username_changing_enabled'] = true if Settings.gitlab['username
 Settings.gitlab['issue_closing_pattern'] = '\b((?:[Cc]los(?:e[sd]?|ing)|\b[Ff]ix(?:e[sd]|ing)?|\b[Rr]esolv(?:e[sd]?|ing)|\b[Ii]mplement(?:s|ed|ing)?)(:?) +(?:(?:issues? +)?%{issue_ref}(?:(?: *,? +and +| *,? *)?)|([A-Z][A-Z0-9_]+-\d+))+)' if Settings.gitlab['issue_closing_pattern'].nil?
 Settings.gitlab['default_projects_features'] ||= {}
 Settings.gitlab['webhook_timeout'] ||= 10
+Settings.gitlab['graphql_timeout'] ||= 30
 Settings.gitlab['max_attachment_size'] ||= 10
 Settings.gitlab['session_expire_delay'] ||= 10080
 Settings.gitlab['unauthenticated_session_expire_delay'] ||= 2.hours.to_i

config/initializers/graphql.rb

@@ -7,7 +7,7 @@ GraphQL::Schema::Object.accepts_definition(:authorize)
 GraphQL::Schema::Field.accepts_definition(:authorize)
 
 Gitlab::Application.config.after_initialize do
-  GitlabSchema.middleware << GraphQL::Schema::TimeoutMiddleware.new(max_seconds: ENV.fetch('GITLAB_RAILS_GRAPHQL_TIMEOUT', 30).to_i) do |timeout_error, query|
+  GitlabSchema.middleware << GraphQL::Schema::TimeoutMiddleware.new(max_seconds: Gitlab.config.gitlab.graphql_timeout) do |timeout_error, query|
     Gitlab::GraphqlLogger.error(message: timeout_error.to_s, query: query.query_string, query_variables: query.provided_variables)
   end
 end

config/routes/api.rb

 post '/api/graphql', to: 'graphql#execute'
-mount GraphiQL::Rails::Engine, at: '/-/graphql-explorer', graphql_path: '/api/graphql'
+mount GraphiQL::Rails::Engine, at: '/-/graphql-explorer', graphql_path: Gitlab::Utils.append_path(Gitlab.config.gitlab.relative_url_root, '/api/graphql')
 
 ::API::API.logger Rails.logger # rubocop:disable Gitlab/RailsLogger
 mount ::API::API => '/'

db/migrate/20200129133716_add_resource_milestone_events_table.rb

+# frozen_string_literal: true
+
+class AddResourceMilestoneEventsTable < ActiveRecord::Migration[5.2]
+  DOWNTIME = false
+
+  def change
+    create_table :resource_milestone_events, id: :bigserial do |t|
+      t.references :user, null: false, foreign_key: { on_delete: :nullify },
+                   index: { name: 'index_resource_milestone_events_on_user_id' }
+      t.references :issue, null: true, foreign_key: { on_delete: :cascade },
+                   index: { name: 'index_resource_milestone_events_on_issue_id' }
+      t.references :merge_request, null: true, foreign_key: { on_delete: :cascade },
+                   index: { name: 'index_resource_milestone_events_on_merge_request_id' }
+      t.references :milestone, foreign_key: { on_delete: :cascade },
+                   index: { name: 'index_resource_milestone_events_on_milestone_id' }
+
+      t.integer :action, limit: 2, null: false
+      t.integer :state, limit: 2, null: false
+      t.integer :cached_markdown_version
+      t.text :reference
+      t.text :reference_html
+      t.datetime_with_timezone :created_at, null: false
+    end
+  end
+end

db/migrate/20200204131054_change_broadcast_message_index.rb

+# frozen_string_literal: true
+
+class ChangeBroadcastMessageIndex < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :broadcast_messages, %i(ends_at broadcast_type id), name: 'index_broadcast_message_on_ends_at_and_broadcast_type_and_id'
+    remove_concurrent_index_by_name :broadcast_messages, :index_broadcast_messages_on_starts_at_and_ends_at_and_id
+  end
+
+  def down
+    add_concurrent_index :broadcast_messages, %i(starts_at ends_at id), name: 'index_broadcast_messages_on_starts_at_and_ends_at_and_id'
+    remove_concurrent_index_by_name :broadcast_messages, :index_broadcast_message_on_ends_at_and_broadcast_type_and_id
+  end
+end

db/post_migrate/20200130145430_reschedule_migrate_issue_trackers_data.rb

+# frozen_string_literal: true
+
+class RescheduleMigrateIssueTrackersData < ActiveRecord::Migration[5.1]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  INTERVAL = 3.minutes.to_i
+  BATCH_SIZE = 5_000
+  MIGRATION = 'MigrateIssueTrackersSensitiveData'
+
+  disable_ddl_transaction!
+
+  class Service < ActiveRecord::Base
+    self.table_name = 'services'
+    self.inheritance_column = :_type_disabled
+
+    include ::EachBatch
+  end
+
+  def up
+    relation = Service.where(category: 'issue_tracker').where("properties IS NOT NULL AND properties != '{}' AND properties != ''")
+    queue_background_migration_jobs_by_range_at_intervals(relation,
+                                                          MIGRATION,
+                                                          INTERVAL,
+                                                          batch_size: BATCH_SIZE)
+  end
+
+  def down
+    remove_issue_tracker_data_sql = "DELETE FROM issue_tracker_data WHERE \
+        (length(encrypted_issues_url) > 0 AND encrypted_issues_url_iv IS NULL) \
+        OR (length(encrypted_new_issue_url) > 0 AND encrypted_new_issue_url_iv IS NULL) \
+        OR (length(encrypted_project_url) > 0 AND encrypted_project_url_iv IS NULL)"
+
+    execute(remove_issue_tracker_data_sql)
+
+    remove_jira_tracker_data_sql = "DELETE FROM jira_tracker_data WHERE \
+        (length(encrypted_api_url) > 0 AND encrypted_api_url_iv IS NULL) \
+        OR (length(encrypted_url) > 0 AND encrypted_url_iv IS NULL) \
+        OR (length(encrypted_username) > 0 AND encrypted_username_iv IS NULL) \
+        OR (length(encrypted_password) > 0 AND encrypted_password_iv IS NULL)"
+
+    execute(remove_jira_tracker_data_sql)
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2020_02_03_025821) do
+ActiveRecord::Schema.define(version: 2020_02_04_131054) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "pg_trgm"
@@ -564,7 +564,7 @@ ActiveRecord::Schema.define(version: 2020_02_03_025821) do
     t.integer "cached_markdown_version"
     t.string "target_path", limit: 255
     t.integer "broadcast_type", limit: 2, default: 1, null: false
-    t.index ["starts_at", "ends_at", "id"], name: "index_broadcast_messages_on_starts_at_and_ends_at_and_id"
+    t.index ["ends_at", "broadcast_type", "id"], name: "index_broadcast_message_on_ends_at_and_broadcast_type_and_id"
   end
 
   create_table "chat_names", id: :serial, force: :cascade do |t|
@@ -3680,6 +3680,23 @@ ActiveRecord::Schema.define(version: 2020_02_03_025821) do
     t.index ["user_id"], name: "index_resource_label_events_on_user_id"
   end
 
+  create_table "resource_milestone_events", force: :cascade do |t|
+    t.bigint "user_id", null: false
+    t.bigint "issue_id"
+    t.bigint "merge_request_id"
+    t.bigint "milestone_id"
+    t.integer "action", limit: 2, null: false
+    t.integer "state", limit: 2, null: false
+    t.integer "cached_markdown_version"
+    t.text "reference"
+    t.text "reference_html"
+    t.datetime_with_timezone "created_at", null: false
+    t.index ["issue_id"], name: "index_resource_milestone_events_on_issue_id"
+    t.index ["merge_request_id"], name: "index_resource_milestone_events_on_merge_request_id"
+    t.index ["milestone_id"], name: "index_resource_milestone_events_on_milestone_id"
+    t.index ["user_id"], name: "index_resource_milestone_events_on_user_id"
+  end
+
   create_table "resource_weight_events", force: :cascade do |t|
     t.bigint "user_id", null: false
     t.bigint "issue_id", null: false
@@ -4842,6 +4859,10 @@ ActiveRecord::Schema.define(version: 2020_02_03_025821) do
   add_foreign_key "resource_label_events", "labels", on_delete: :nullify
   add_foreign_key "resource_label_events", "merge_requests", on_delete: :cascade
   add_foreign_key "resource_label_events", "users", on_delete: :nullify
+  add_foreign_key "resource_milestone_events", "issues", on_delete: :cascade
+  add_foreign_key "resource_milestone_events", "merge_requests", on_delete: :cascade
+  add_foreign_key "resource_milestone_events", "milestones", on_delete: :cascade
+  add_foreign_key "resource_milestone_events", "users", on_delete: :nullify
   add_foreign_key "resource_weight_events", "issues", on_delete: :cascade
   add_foreign_key "resource_weight_events", "users", on_delete: :nullify
   add_foreign_key "reviews", "merge_requests", on_delete: :cascade

doc/.linting/vale/styles/gitlab/SentenceSpacing.yml

+---
+# `extends` indicates the Vale extension point being used.
+# Full list of styles: https://errata-ai.github.io/vale/styles/
+extends: existence
+
+# Existence rules can display the matched strings in the user message.
+message: "'%s' should have one space between sentences."
+
+# Should a result be flagged as a suggestion, warning, or error?
+# Results that fall below the MinAlertLevel set in 
+# https://gitlab.com/gitlab-org/gitlab/blob/master/.vale.ini won't be shown.
+level: suggestion
+
+# Should a match be case-insensitive or case-sensitive?
+# Acceptable values are 'true' or 'false'
+# This value is irrelevant when testing non-alphabetical characters
+#ignorecase: true
+
+# Should this rule be limited to a specific scope? If yes, uncomment the line.
+# Possible scopes: https://errata-ai.github.io/vale/formats/#available-scopes
+# scope: heading
+
+# Should this rule ignore normal word boundaries, such as \b ?
+# Acceptable values are 'true' or 'false'
+nonword: true
+
+# What is the source for this rule?
+link: https://docs.gitlab.com/ee/development/documentation/styleguide.html#punctuation
+
+tokens:
+  - '[a-z][.?!][A-Z]'
+  - '[.?!] {2,}[A-Z]'

doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md

@@ -126,6 +126,18 @@ Benchmark.bm do |x|
 end
 ```
 
+## Feature flags
+
+### Show all feature flags that are enabled
+
+```ruby
+# Regular output
+Feature.all
+
+# Nice output
+Feature.all.map {|f| [f.name, f.state]}
+```
+
 ## Command Line
 
 ### Check the GitLab version fast

doc/administration/troubleshooting/ssl.md

@@ -46,6 +46,21 @@ After configuring a GitLab instance with an internal CA certificate, you might n
 
 If you have the problems listed above, add your certificate to `/etc/gitlab/trusted-certs` and run `sudo gitlab-ctl reconfigure`.
 
+## Using GitLab Runner with a GitLab instance configured with internal CA certificate or self-signed certificate
+
+Besides getting the errors mentioned in
+[Using an internal CA certificate with GitLab](ssl.md#using-an-internal-ca-certificate-with-gitlab),
+your CI pipelines may stuck stuck in `Pending` status. In the runner logs you may see the below error:
+
+```shell
+Dec  6 02:43:17 runner-host01 gitlab-runner[15131]: #033[0;33mWARNING: Checking for jobs... failed
+#033[0;m  #033[0;33mrunner#033[0;m=Bfkz1fyb #033[0;33mstatus#033[0;m=couldn't execute POST against
+https://gitlab.domain.tld/api/v4/jobs/request: Post https://gitlab.domain.tld/api/v4/jobs/request:
+x509: certificate signed by unknown authority
+```
+
+If you face similar problem, add your certificate to `/etc/gitlab-runner/certs` and restart the runner via `gitlab-runner restart`.
+
 ## Mirroring a remote GitLab repository that uses a self-signed SSL certificate
 
 **Scenario:** When configuring a local GitLab instance to [mirror a repository](../../user/project/repository/repository_mirroring.md) from a remote GitLab instance that uses a self-signed certificate, you may see the `SSL certificate problem: self signed certificate` error in the UI.
@@ -79,6 +94,18 @@ To fix this problem:
 - Add the self-signed certificate from the remote GitLab instance to the `/etc/gitlab/trusted-certs` directory on the local GitLab instance and run `sudo gitlab-ctl reconfigure` as per the instructions for [installing custom public certificates](https://docs.gitlab.com/omnibus/settings/ssl.html#install-custom-public-certificates).
 - If your local GitLab instance was installed using the Helm Charts, you can [add your self-signed certificate to your GitLab instance](https://docs.gitlab.com/runner/install/kubernetes.html#providing-a-custom-certificate-for-accessing-gitlab).
 
+You may also get another error when trying to mirror a repository from a remote GitLab instance that uses a self-signed certificate:
+
+```shell
+2:Fetching remote upstream failed: fatal: unable to access 'https://gitlab.domain.tld/root/test-repo/':
+SSL: unable to obtain common name from peer certificate
+```
+
+In this case, the problem can be related to the certificate itself:
+
+- Double check that your self-signed certificate is not missing a common name. If it is then regenerate a valid certificate
+- add it to `/etc/gitlab/trusted-certs` and run `sudo gitlab-ctl reconfigure`
+
 ## Unable to perform Git operations due to an internal or self-signed certificate
 
 If your GitLab instance is using a self-signed certificate, or the certificate is signed by an internal certificate authority (CA), you might run into the following errors when attempting to perform Git operations:

lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml

 .dast-auto-deploy:
-  image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.8.3"
+  image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.9.1"
 
 dast_environment_deploy:
   extends: .dast-auto-deploy

spec/features/graphiql_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe 'GraphiQL' do
+  context 'without relative_url_root' do
+    before do
+      visit '/-/graphql-explorer'
+    end
+
+    it 'has the correct graphQLEndpoint' do
+      expect(page.body).to include('var graphQLEndpoint = "/api/graphql";')
+    end
+  end
+
+  context 'with relative_url_root' do
+    before do
+      stub_config_setting(relative_url_root: '/gitlab/root')
+      Rails.application.reload_routes!
+
+      visit '/-/graphql-explorer'
+    end
+
+    after do
+      Rails.application.reload_routes!
+    end
+
+    it 'has the correct graphQLEndpoint' do
+      expect(page.body).to include('var graphQLEndpoint = "/gitlab/root/api/graphql";')
+    end
+  end
+end

spec/lib/gitlab/background_migration/migrate_issue_trackers_sensitive_data_spec.rb

 # frozen_string_literal: true
 require 'spec_helper'
 
-describe Gitlab::BackgroundMigration::MigrateIssueTrackersSensitiveData, :migration, schema: 20190924152703 do
+describe Gitlab::BackgroundMigration::MigrateIssueTrackersSensitiveData, :migration, schema: 20200130145430 do
   let(:services) { table(:services) }
 
   # we need to define the classes due to encryption

spec/migrations/20200130145430_reschedule_migrate_issue_trackers_data_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require Rails.root.join('db', 'post_migrate', '20200130145430_reschedule_migrate_issue_trackers_data.rb')
+
+describe RescheduleMigrateIssueTrackersData, :migration, :sidekiq do
+  let(:services) { table(:services) }
+  let(:migration_class) { Gitlab::BackgroundMigration::MigrateIssueTrackersSensitiveData }
+  let(:migration_name)  { migration_class.to_s.demodulize }
+
+  let(:properties) do
+    {
+      'url' => 'http://example.com'
+    }
+  end
+  let!(:jira_service) do
+    services.create(id: 10, type: 'JiraService', properties: properties, category: 'issue_tracker')
+  end
+  let!(:jira_service_nil) do
+    services.create(id: 11, type: 'JiraService', properties: nil, category: 'issue_tracker')
+  end
+  let!(:bugzilla_service) do
+    services.create(id: 12, type: 'BugzillaService', properties: properties, category: 'issue_tracker')
+  end
+  let!(:youtrack_service) do
+    services.create(id: 13, type: 'YoutrackService', properties: properties, category: 'issue_tracker')
+  end
+  let!(:youtrack_service_empty) do
+    services.create(id: 14, type: 'YoutrackService', properties: '', category: 'issue_tracker')
+  end
+  let!(:gitlab_service) do
+    services.create(id: 15, type: 'GitlabIssueTrackerService', properties: properties, category: 'issue_tracker')
+  end
+  let!(:gitlab_service_empty) do
+    services.create(id: 16, type: 'GitlabIssueTrackerService', properties: {}, category: 'issue_tracker')
+  end
+  let!(:other_service) do
+    services.create(id: 17, type: 'OtherService', properties: properties, category: 'other_category')
+  end
+
+  before do
+    stub_const("#{described_class}::BATCH_SIZE", 2)
+  end
+
+  describe "#up" do
+    it 'schedules background migrations at correct time' do
+      Sidekiq::Testing.fake! do
+        Timecop.freeze do
+          migrate!
+
+          expect(migration_name).to be_scheduled_delayed_migration(3.minutes, jira_service.id, bugzilla_service.id)
+          expect(migration_name).to be_scheduled_delayed_migration(6.minutes, youtrack_service.id, gitlab_service.id)
+          expect(BackgroundMigrationWorker.jobs.size).to eq(2)
+        end
+      end
+    end
+  end
+
+  describe "#down" do
+    let(:issue_tracker_data) { table(:issue_tracker_data) }
+    let(:jira_tracker_data) { table(:jira_tracker_data) }
+
+    let!(:valid_issue_tracker_data) do
+      issue_tracker_data.create(
+        service_id: bugzilla_service.id,
+        encrypted_issues_url: 'http://url.com',
+        encrypted_issues_url_iv: 'somevalue'
+      )
+    end
+    let!(:invalid_issue_tracker_data) do
+      issue_tracker_data.create(
+        service_id: bugzilla_service.id,
+        encrypted_issues_url: 'http:url.com',
+        encrypted_issues_url_iv: nil
+      )
+    end
+    let!(:valid_jira_tracker_data) do
+      jira_tracker_data.create(
+        service_id: bugzilla_service.id,
+        encrypted_url: 'http://url.com',
+        encrypted_url_iv: 'somevalue'
+      )
+    end
+    let!(:invalid_jira_tracker_data) do
+      jira_tracker_data.create(
+        service_id: bugzilla_service.id,
+        encrypted_url: 'http://url.com',
+        encrypted_url_iv: nil
+      )
+    end
+
+    it 'removes the invalid jira tracker data' do
+      expect { described_class.new.down }.to change { jira_tracker_data.count }.from(2).to(1)
+
+      expect(jira_tracker_data.all).to eq([valid_jira_tracker_data])
+    end
+
+    it 'removes the invalid issue tracker data' do
+      expect { described_class.new.down }.to change { issue_tracker_data.count }.from(2).to(1)
+
+      expect(issue_tracker_data.all).to eq([valid_issue_tracker_data])
+    end
+  end
+end