Add latest changes from gitlab-org/gitlab@master

app/models/issue.rb

@@ -66,6 +66,8 @@ class Issue < ApplicationRecord
   scope :public_only, -> { where(confidential: false) }
   scope :confidential_only, -> { where(confidential: true) }
 
+  scope :counts_by_state, -> { reorder(nil).group(:state).count }
+
   after_commit :expire_etag_cache
   after_save :ensure_metrics, unless: :imported?
 

db/migrate/20191016133352_create_ci_subscriptions_projects.rb

+# frozen_string_literal: true
+
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class CreateCiSubscriptionsProjects < ActiveRecord::Migration[5.2]
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  def change
+    create_table :ci_subscriptions_projects do |t|
+      t.references :downstream_project, null: false, index: false, foreign_key: { to_table: :projects, on_delete: :cascade }
+      t.references :upstream_project, null: false, foreign_key: { to_table: :projects, on_delete: :cascade }
+    end
+
+    add_index :ci_subscriptions_projects, [:downstream_project_id, :upstream_project_id],
+      unique: true, name: 'index_ci_subscriptions_projects_unique_subscription'
+  end
+end

db/schema.rb

@@ -924,6 +924,13 @@ ActiveRecord::Schema.define(version: 2019_11_12_221821) do
     t.index ["project_id"], name: "index_ci_stages_on_project_id"
   end
 
+  create_table "ci_subscriptions_projects", force: :cascade do |t|
+    t.bigint "downstream_project_id", null: false
+    t.bigint "upstream_project_id", null: false
+    t.index ["downstream_project_id", "upstream_project_id"], name: "index_ci_subscriptions_projects_unique_subscription", unique: true
+    t.index ["upstream_project_id"], name: "index_ci_subscriptions_projects_on_upstream_project_id"
+  end
+
   create_table "ci_trigger_requests", id: :serial, force: :cascade do |t|
     t.integer "trigger_id", null: false
     t.text "variables"
@@ -4190,6 +4197,8 @@ ActiveRecord::Schema.define(version: 2019_11_12_221821) do
   add_foreign_key "ci_sources_pipelines", "projects", name: "fk_1e53c97c0a", on_delete: :cascade
   add_foreign_key "ci_stages", "ci_pipelines", column: "pipeline_id", name: "fk_fb57e6cc56", on_delete: :cascade
   add_foreign_key "ci_stages", "projects", name: "fk_2360681d1d", on_delete: :cascade
+  add_foreign_key "ci_subscriptions_projects", "projects", column: "downstream_project_id", on_delete: :cascade
+  add_foreign_key "ci_subscriptions_projects", "projects", column: "upstream_project_id", on_delete: :cascade
   add_foreign_key "ci_trigger_requests", "ci_triggers", column: "trigger_id", name: "fk_b8ec8b7245", on_delete: :cascade
   add_foreign_key "ci_triggers", "projects", name: "fk_e3e63f966e", on_delete: :cascade
   add_foreign_key "ci_triggers", "users", column: "owner_id", name: "fk_e8e10d1964", on_delete: :cascade

doc/api/graphql/reference/gitlab_schema.graphql

@@ -1124,6 +1124,11 @@ type Epic implements Noteable {
   ): EpicConnection
   closedAt: Time
   createdAt: Time
+
+  """
+  Number of open and closed descendant epics and issues
+  """
+  descendantCounts: EpicDescendantCount
   description: String
 
   """
@@ -1304,6 +1309,28 @@ type EpicConnection {
   pageInfo: PageInfo!
 }
 
+type EpicDescendantCount {
+  """
+  Number of closed sub-epics
+  """
+  closedEpics: Int
+
+  """
+  Number of closed epic issues
+  """
+  closedIssues: Int
+
+  """
+  Number of opened sub-epics
+  """
+  openedEpics: Int
+
+  """
+  Number of opened epic issues
+  """
+  openedIssues: Int
+}
+
 """
 An edge in a connection.
 """

doc/api/graphql/reference/gitlab_schema.json

@@ -3518,6 +3518,20 @@
               "isDeprecated": false,
               "deprecationReason": null
             },
+            {
+              "name": "descendantCounts",
+              "description": "Number of open and closed descendant epics and issues",
+              "args": [
+
+              ],
+              "type": {
+                "kind": "OBJECT",
+                "name": "EpicDescendantCount",
+                "ofType": null
+              },
+              "isDeprecated": false,
+              "deprecationReason": null
+            },
             {
               "name": "description",
               "description": null,
@@ -9619,6 +9633,75 @@
           "enumValues": null,
           "possibleTypes": null
         },
+        {
+          "kind": "OBJECT",
+          "name": "EpicDescendantCount",
+          "description": null,
+          "fields": [
+            {
+              "name": "closedEpics",
+              "description": "Number of closed sub-epics",
+              "args": [
+
+              ],
+              "type": {
+                "kind": "SCALAR",
+                "name": "Int",
+                "ofType": null
+              },
+              "isDeprecated": false,
+              "deprecationReason": null
+            },
+            {
+              "name": "closedIssues",
+              "description": "Number of closed epic issues",
+              "args": [
+
+              ],
+              "type": {
+                "kind": "SCALAR",
+                "name": "Int",
+                "ofType": null
+              },
+              "isDeprecated": false,
+              "deprecationReason": null
+            },
+            {
+              "name": "openedEpics",
+              "description": "Number of opened sub-epics",
+              "args": [
+
+              ],
+              "type": {
+                "kind": "SCALAR",
+                "name": "Int",
+                "ofType": null
+              },
+              "isDeprecated": false,
+              "deprecationReason": null
+            },
+            {
+              "name": "openedIssues",
+              "description": "Number of opened epic issues",
+              "args": [
+
+              ],
+              "type": {
+                "kind": "SCALAR",
+                "name": "Int",
+                "ofType": null
+              },
+              "isDeprecated": false,
+              "deprecationReason": null
+            }
+          ],
+          "inputFields": null,
+          "interfaces": [
+
+          ],
+          "enumValues": null,
+          "possibleTypes": null
+        },
         {
           "kind": "OBJECT",
           "name": "ProjectStatistics",

doc/api/graphql/reference/index.md

@@ -220,6 +220,16 @@ The API can be explored interactively using the [GraphiQL IDE](../index.md#graph
 | `relationPath` | String |  |
 | `reference` | String! |  |
 | `subscribed` | Boolean! | Boolean flag for whether the currently logged in user is subscribed to this epic |
+| `descendantCounts` | EpicDescendantCount | Number of open and closed descendant epics and issues |
+
+### EpicDescendantCount
+
+| Name  | Type  | Description |
+| ---   |  ---- | ----------  |
+| `openedEpics` | Int | Number of opened sub-epics |
+| `closedEpics` | Int | Number of closed sub-epics |
+| `openedIssues` | Int | Number of opened epic issues |
+| `closedIssues` | Int | Number of closed epic issues |
 
 ### EpicIssue
 

doc/development/packages.md

@@ -4,12 +4,13 @@ This document will guide you through adding another [package management system](
 
 See already supported package types in [Packages documentation](../administration/packages/index.md)
 
-Since GitLab packages' UI is pretty generic, it is possible to add new
+Since GitLab packages' UI is pretty generic, it is possible to add basic new
 package system support by solely backend changes. This guide is superficial and does
 not cover the way the code should be written. However, you can find a good example
 by looking at existing merge requests with Maven and NPM support:
 
 - [NPM registry support](https://gitlab.com/gitlab-org/gitlab/merge_requests/8673).
+- [Conan repository](https://gitlab.com/gitlab-org/gitlab/issues/8248).
 - [Maven repository](https://gitlab.com/gitlab-org/gitlab/merge_requests/6607).
 - [Instance level endpoint for Maven repository](https://gitlab.com/gitlab-org/gitlab/merge_requests/8757)
 
@@ -34,7 +35,7 @@ endpoints like:
 - GET package file content.
 - PUT upload package.
 
-Since the packages belong to a project, it's expected to have project-level endpoint
+Since the packages belong to a project, it's expected to have project-level endpoint (remote)
 for uploading and downloading them. For example:
 
 ```
@@ -44,9 +45,48 @@ PUT https://gitlab.com/api/v4/projects/<your_project_id>/packages/npm/
 
 Group-level and instance-level endpoints are good to have but are optional.
 
-NOTE: **Note:**
-To avoid name conflict for instance-level endpoints we use
-[the package naming convention](../user/packages/npm_registry/index.md#package-naming-convention)
+## Naming conventions
+
+To avoid name conflict for instance-level endpoints you will need to define a package naming convention
+that gives a way to identify the project that the package belongs to. This generally involves using the project
+id or full project path in the package name. See
+[Conan's naming convention](../user/packages/conan_repository/index.md#package-recipe-naming-convention) as an example.
+
+For group and project-level endpoints, naming can be less constrained, and it will be up to the group and project
+members to be certain that there is no conflict between two package names, however the system should prevent
+a user from reusing an existing name within a given scope.
+
+Otherwise, naming should follow the package manager's naming conventions and include a validation in the `package.md`
+model for that package type.
+
+## File uploads
+
+File uploads should be handled by GitLab workhorse using object accelerated uploads. What this means is that
+the workhorse proxy that checks all incoming requests to GitLab will intercept the upload request,
+upload the file, and forward a request to the main GitLab codebase only containing the metadata
+and file location rather than the file itself. An overview of this process can be found in the
+[development documentation](uploads.md#workhorse-object-storage-acceleration).
+
+In terms of code, this means a route will need to be added to the
+[gitlab-workhorse project](https://gitlab.com/gitlab-org/gitlab-workhorse) for each level of remote being added
+(instance, group, project). [This merge request](https://gitlab.com/gitlab-org/gitlab-workhorse/merge_requests/412/diffs)
+demonstrates adding an instance-level endpoint for Conan to workhorse. You can also see the Maven project level endpoint
+implemented in the same file.
+
+Once the route has been added, you will need to add an additional `/authorize` version of the upload endpoint to your API file.
+[Here is an example](https://gitlab.com/gitlab-org/gitlab/blob/398fef1ca26ae2b2c3dc89750f6b20455a1e5507/ee/lib/api/maven_packages.rb#L164)
+of the additional endpoint added for Maven. The `/authorize` endpoint verifies and authorizes the request from workhorse,
+then the normal upload endpoint is implemented below, consuming the metadata that workhorse provides in order to
+create the package record. Workhorse provides a variety of file metadata such as type, size, and different checksum formats.
+
+For testing purposes, you may want to [enable object storage](https://gitlab.com/gitlab-org/gitlab-development-kit/blob/master/doc/howto/object_storage.md)
+in your local development environment.
+
+## Services and finders
+
+Logic for performing tasks such as creating package or package file records or finding packages should not live
+within the API file, but should live in services and finders. Existing services and finders should be used or
+extended when possible to keep the common package logic grouped as much as possible.
 
 ## Configuration
 
@@ -56,7 +96,7 @@ to add anything there.
 
 Packages can be configured to use object storage, therefore your code must support it.
 
-## Database
+## Database and handling metadata
 
 The current database model allows you to store a name and a version for each package.
 Every time you upload a new package, you can either create a new record of `Package`
@@ -65,4 +105,58 @@ information like the file `name`, `side`, `sha1`, etc.
 
 If there is specific data necessary to be stored for only one package system support,
 consider creating a separate metadata model. See `packages_maven_metadata` table
-and `Packages::MavenMetadatum` model as example for package specific data.
+and `Packages::MavenMetadatum` model as an example for package specific data, and `packages_conan_file_metadata` table
+and `Packages::ConanFileMetadatum` model as an example for package file specific data.
+
+If there is package specific behavior for a given package manager, add those methods to the metadata models and
+delegate from the package model.
+
+Note that the existing package UI only displays information within the `packages_packages` and `packages_package_files`
+tables. If the data stored in the metadata tables need to be displayed, a ~frontend change will be required.
+
+## Authorization
+
+There are project and group level permissions for `read_package`, `create_package`, and `destroy_package`. Each
+endpoint should
+[authorize the requesting user](https://gitlab.com/gitlab-org/gitlab/blob/398fef1ca26ae2b2c3dc89750f6b20455a1e5507/ee/lib/api/conan_packages.rb#L84)
+against the project or group before continuing.
+
+## Keep iterations small
+
+When implementing a new package manager, it is easy to end up creating one large merge request containing all of the
+necessary endpoints and services necessary to support basic usage. If this is the case, consider putting the
+API endpoints behind a [feature flag](feature_flags/development.md) and
+submitting each endpoint or behavior (download, upload, etc) in different merge requests to shorten the review
+process.
+
+### Potential MRs for any given package system
+
+#### MVC MRs
+
+These changes represent all that is needed to deliver a minimally usable package management system.
+
+1. Empty file structure (api file, base service for this package)
+1. Authentication system for 'logging in' to the package manager
+1. Identify metadata and create applicable tables
+1. Workhorse route for [object storage accelerated uploads](uploads.md#workhorse-object-storage-acceleration)
+1. Endpoints required for upload/publish
+1. Endpoints required for install/download
+1. Endpoints required for remove/delete
+
+#### Possible post-MVC MRs
+
+These updates are not essential to be able to publish and consume packages, but may be desired as the system is
+released for general use.
+
+1. Endpoints required for search
+1. Front end updates to display additional package information and metadata
+1. Limits on file sizes
+1. Tracking for metrics
+
+## Exceptions
+
+This documentation is just guidelines on how to implement a package manager to match the existing structure and logic
+already present within GitLab. While the structure is intended to be extendable and flexible enough to allow for
+any given package manager, if there is good reason to stray due to the constraints or needs of a given package
+manager, then it should be raised and discussed within the implementation issue or merge request to work towards
+the most efficient outcome.

doc/development/understanding_explain_plans.md

@@ -705,6 +705,43 @@ For more information about the available options, run:
 /chatops run explain --help
 ```
 
+### `#database-lab`
+
+Another tool GitLab employees can use is a chatbot powered by [Joe](https://gitlab.com/postgres-ai/joe), available in the [`#database-lab`](https://gitlab.slack.com/archives/CLJMDRD8C) channel on Slack.
+Unlike chatops, it gives you a way to execute DDL statements (like creating indexes and tables) and get query plan not only for `SELECT` but also `UPDATE` and `DELETE`.
+
+For example, in order to test new index you can do the following:
+
+Create the index:
+
+```
+exec CREATE INDEX index_projects_marked_for_deletion ON projects (marked_for_deletion_at) WHERE marked_for_deletion_at IS NOT NULL
+```
+
+Analyze the table to update its statistics:
+
+```
+exec ANALYZE projects
+```
+
+Get the query plan:
+
+```
+explain SELECT * FROM projects WHERE marked_for_deletion_at < CURRENT_DATE
+```
+
+Once done you can rollback your changes:
+
+```
+reset
+```
+
+For more information about the available options, run:
+
+```
+help
+```
+
 ## Further reading
 
 A more extensive guide on understanding query plans can be found in

doc/user/project/operations/error_tracking.md

@@ -20,6 +20,7 @@ You will need at least Maintainer [permissions](../../permissions.md) to enable
 GitLab provides an easy way to connect Sentry to your project:
 
 1. Sign up to Sentry.io or [deploy your own](#deploying-sentry) Sentry instance.
+1. [Create](https://docs.sentry.io/guides/integrate-frontend/create-new-project/) a new Sentry project. For each GitLab project that you want to integrate, we recommend that you create a new Sentry project.
 1. [Find or generate](https://docs.sentry.io/api/auth/) a Sentry auth token for your Sentry project.
    Make sure to give the token at least the following scopes: `event:read` and `project:read`.
 1. Navigate to your project’s **Settings > Operations**.

locale/gitlab.pot

@@ -1417,6 +1417,9 @@ msgstr ""
 msgid "All projects"
 msgstr ""
 
+msgid "All security scans are enabled because %{linkStart}Auto DevOps%{linkEnd} is enabled on this project"
+msgstr ""
+
 msgid "All users"
 msgstr ""
 
@@ -4422,6 +4425,9 @@ msgstr ""
 msgid "Configure Prometheus"
 msgstr ""
 
+msgid "Configure Security %{wordBreakOpportunity}and Compliance"
+msgstr ""
+
 msgid "Configure Tracing"
 msgstr ""
 
@@ -15081,9 +15087,30 @@ msgstr ""
 msgid "Security Reports|While it's rare to have no vulnerabilities for your group, it can happen. In any event, we ask that you please double check your settings to make sure you've set up your dashboard correctly."
 msgstr ""
 
+msgid "Security configuration help link"
+msgstr ""
+
 msgid "Security dashboard"
 msgstr ""
 
+msgid "SecurityConfiguration|Configured"
+msgstr ""
+
+msgid "SecurityConfiguration|Feature"
+msgstr ""
+
+msgid "SecurityConfiguration|Feature documentation"
+msgstr ""
+
+msgid "SecurityConfiguration|Not yet configured"
+msgstr ""
+
+msgid "SecurityConfiguration|Secure features"
+msgstr ""
+
+msgid "SecurityConfiguration|Status"
+msgstr ""
+
 msgid "SecurityDashboard| The security dashboard displays the latest security report. Use it to find and fix vulnerabilities."
 msgstr ""
 
@@ -16376,6 +16403,15 @@ msgstr ""
 msgid "Subscription"
 msgstr ""
 
+msgid "Subscription deletion failed."
+msgstr ""
+
+msgid "Subscription successfully created."
+msgstr ""
+
+msgid "Subscription successfully deleted."
+msgstr ""
+
 msgid "SubscriptionTable|Billing"
 msgstr ""
 
@@ -16873,6 +16909,9 @@ msgstr ""
 msgid "The collection of events added to the data gathered for that stage."
 msgstr ""
 
+msgid "The configuration status of the table below only applies to the default branch and is based on the %{linkStart}latest pipeline%{linkEnd}. Once you've configured a scan for the default branch, any subsequent feature branch you create will include the scan."
+msgstr ""
+
 msgid "The connection will time out after %{timeout}. For repositories that take longer, use a clone/push combination."
 msgstr ""
 
@@ -17527,6 +17566,9 @@ msgstr ""
 msgid "This project does not have billing enabled. To create a cluster, <a href=%{linkToBilling} target=\"_blank\" rel=\"noopener noreferrer\">enable billing <i class=\"fa fa-external-link\" aria-hidden=\"true\"></i></a> and try again."
 msgstr ""
 
+msgid "This project path either does not exist or is private."
+msgstr ""
+
 msgid "This repository"
 msgstr ""
 

spec/lib/gitlab/import_export/all_models.yml

@@ -426,6 +426,10 @@ project:
 - grafana_integration
 - remove_source_branch_after_merge
 - deleting_user
+- upstream_projects
+- downstream_projects
+- upstream_project_subscriptions
+- downstream_project_subscriptions
 award_emoji:
 - awardable
 - user