Automatic merge of gitlab-org/gitlab-ce master

app/models/environment_status.rb

@@ -40,7 +40,7 @@ class EnvironmentStatus
   end
 
   def changes
-    return [] if project.route_map_for(sha).nil?
+    return [] unless has_route_map?
 
     changed_files.map { |file| build_change(file) }.compact
   end
@@ -50,6 +50,10 @@ class EnvironmentStatus
       .merge_request_diff_files.where(deleted_file: false)
   end
 
+  def has_route_map?
+    project.route_map_for(sha).present?
+  end
+
   private
 
   PAGE_EXTENSIONS = /\A\.(s?html?|php|asp|cgi|pl)\z/i.freeze

app/models/project.rb

@@ -1914,9 +1914,8 @@ class Project < ApplicationRecord
     @route_maps_by_commit ||= Hash.new do |h, sha|
       h[sha] = begin
         data = repository.route_map_for(sha)
-        next unless data
 
-        Gitlab::RouteMap.new(data)
+        Gitlab::RouteMap.new(data) if data
       rescue Gitlab::RouteMap::FormatError
         nil
       end

changelogs/unreleased/backstage-gb-improve-performance-environment-statuses-endpoint.yml

+---
+title: Improve performance of fetching environments statuses
+merge_request: 30560
+author:
+type: performance

doc/administration/high_availability/database.md

@@ -478,88 +478,7 @@ Check the [Troubleshooting section](#troubleshooting) before proceeding.
 
 #### Configuring the Pgbouncer node
 
-1. Make sure you collect [`CONSUL_SERVER_NODES`](#consul-information), [`CONSUL_PASSWORD_HASH`](#consul-information), and [`PGBOUNCER_PASSWORD_HASH`](#pgbouncer-information) before executing the next step.
-
-1. Edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section:
-
-    ```ruby
-    # Disable all components except Pgbouncer and Consul agent
-    roles ['pgbouncer_role']
-
-    # Configure Pgbouncer
-    pgbouncer['admin_users'] = %w(pgbouncer gitlab-consul)
-
-    # Configure Consul agent
-    consul['watchers'] = %w(postgresql)
-
-    # START user configuration
-    # Please set the real values as explained in Required Information section
-    # Replace CONSUL_PASSWORD_HASH with with a generated md5 value
-    # Replace PGBOUNCER_PASSWORD_HASH with with a generated md5 value
-    pgbouncer['users'] = {
-      'gitlab-consul': {
-        password: 'CONSUL_PASSWORD_HASH'
-      },
-      'pgbouncer': {
-        password: 'PGBOUNCER_PASSWORD_HASH'
-      }
-    }
-    # Replace placeholders:
-    #
-    # Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z
-    # with the addresses gathered for CONSUL_SERVER_NODES
-    consul['configuration'] = {
-      retry_join: %w(Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z)
-    }
-    #
-    # END user configuration
-    ```
-
-    > `pgbouncer_role` was introduced with GitLab 10.3
-
-1. [Reconfigure GitLab] for the changes to take effect.
-
-1. Create a `.pgpass` file so Consul is able to
-   reload pgbouncer. Enter the `PGBOUNCER_PASSWORD` twice when asked:
-
-    ```sh
-    gitlab-ctl write-pgpass --host 127.0.0.1 --database pgbouncer --user pgbouncer --hostuser gitlab-consul
-    ```
-
-##### PGBouncer Checkpoint
-
-1. Ensure the node is talking to the current master:
-
-    ```sh
-    gitlab-ctl pgb-console # You will be prompted for PGBOUNCER_PASSWORD
-    ```
-
-    If there is an error `psql: ERROR:  Auth failed` after typing in the
-    password, ensure you previously generated the MD5 password hashes with the correct
-    format. The correct format is to concatenate the password and the username:
-    `PASSWORDUSERNAME`. For example, `Sup3rS3cr3tpgbouncer` would be the text
-    needed to generate an MD5 password hash for the `pgbouncer` user.
-
-1. Once the console prompt is available, run the following queries:
-
-    ```sh
-    show databases ; show clients ;
-    ```
-
-    The output should be similar to the following:
-
-    ```
-            name         |  host       | port |      database       | force_user | pool_size | reserve_pool | pool_mode | max_connections | current_connections
-    ---------------------+-------------+------+---------------------+------------+-----------+--------------+-----------+-----------------+---------------------
-     gitlabhq_production | MASTER_HOST | 5432 | gitlabhq_production |            |        20 |            0 |           |               0 |                   0
-     pgbouncer           |             | 6432 | pgbouncer           | pgbouncer  |         2 |            0 | statement |               0 |                   0
-    (2 rows)
-
-     type |   user    |      database       |  state  |   addr         | port  | local_addr | local_port |    connect_time     |    request_time     |    ptr    | link | remote_pid | tls
-    ------+-----------+---------------------+---------+----------------+-------+------------+------------+---------------------+---------------------+-----------+------+------------+-----
-     C    | pgbouncer | pgbouncer           | active  | 127.0.0.1      | 56846 | 127.0.0.1  |       6432 | 2017-08-21 18:09:59 | 2017-08-21 18:10:48 | 0x22b3880 |      |          0 |
-    (2 rows)
-    ```
+See our [documentation for Pgbouncer](pgbouncer.md) for information on running Pgbouncer as part of an HA setup.
 
 #### Configuring the Application nodes
 

doc/administration/high_availability/pgbouncer.md

@@ -14,7 +14,88 @@ It is recommended to run pgbouncer alongside the `gitlab-rails` service, or on i
 
 ### Running Pgbouncer as part of an HA GitLab installation
 
-See our [HA documentation for PostgreSQL](database.md) for information on running pgbouncer as part of a HA setup
+1. Make sure you collect [`CONSUL_SERVER_NODES`](database.md#consul-information), [`CONSUL_PASSWORD_HASH`](database.md#consul-information), and [`PGBOUNCER_PASSWORD_HASH`](database.md#pgbouncer-information) before executing the next step.
+
+1. Edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section:
+
+    ```ruby
+    # Disable all components except Pgbouncer and Consul agent
+    roles ['pgbouncer_role']
+
+    # Configure Pgbouncer
+    pgbouncer['admin_users'] = %w(pgbouncer gitlab-consul)
+
+    # Configure Consul agent
+    consul['watchers'] = %w(postgresql)
+
+    # START user configuration
+    # Please set the real values as explained in Required Information section
+    # Replace CONSUL_PASSWORD_HASH with with a generated md5 value
+    # Replace PGBOUNCER_PASSWORD_HASH with with a generated md5 value
+    pgbouncer['users'] = {
+      'gitlab-consul': {
+        password: 'CONSUL_PASSWORD_HASH'
+      },
+      'pgbouncer': {
+        password: 'PGBOUNCER_PASSWORD_HASH'
+      }
+    }
+    # Replace placeholders:
+    #
+    # Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z
+    # with the addresses gathered for CONSUL_SERVER_NODES
+    consul['configuration'] = {
+      retry_join: %w(Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z)
+    }
+    #
+    # END user configuration
+    ```
+
+    > `pgbouncer_role` was introduced with GitLab 10.3
+
+1.  Run `gitlab-ctl reconfigure`
+
+1. Create a `.pgpass` file so Consul is able to
+   reload pgbouncer. Enter the `PGBOUNCER_PASSWORD` twice when asked:
+
+    ```sh
+    gitlab-ctl write-pgpass --host 127.0.0.1 --database pgbouncer --user pgbouncer --hostuser gitlab-consul
+    ```
+
+#### PGBouncer Checkpoint
+
+1. Ensure the node is talking to the current master:
+
+    ```sh
+    gitlab-ctl pgb-console # You will be prompted for PGBOUNCER_PASSWORD
+    ```
+
+    If there is an error `psql: ERROR:  Auth failed` after typing in the
+    password, ensure you previously generated the MD5 password hashes with the correct
+    format. The correct format is to concatenate the password and the username:
+    `PASSWORDUSERNAME`. For example, `Sup3rS3cr3tpgbouncer` would be the text
+    needed to generate an MD5 password hash for the `pgbouncer` user.
+
+1. Once the console prompt is available, run the following queries:
+
+    ```sh
+    show databases ; show clients ;
+    ```
+
+    The output should be similar to the following:
+
+    ```
+            name         |  host       | port |      database       | force_user | pool_size | reserve_pool | pool_mode | max_connections | current_connections
+    ---------------------+-------------+------+---------------------+------------+-----------+--------------+-----------+-----------------+---------------------
+     gitlabhq_production | MASTER_HOST | 5432 | gitlabhq_production |            |        20 |            0 |           |               0 |                   0
+     pgbouncer           |             | 6432 | pgbouncer           | pgbouncer  |         2 |            0 | statement |               0 |                   0
+    (2 rows)
+
+     type |   user    |      database       |  state  |   addr         | port  | local_addr | local_port |    connect_time     |    request_time     |    ptr    | link | remote_pid | tls
+    ------+-----------+---------------------+---------+----------------+-------+------------+------------+---------------------+---------------------+-----------+------+------------+-----
+     C    | pgbouncer | pgbouncer           | active  | 127.0.0.1      | 56846 | 127.0.0.1  |       6432 | 2017-08-21 18:09:59 | 2017-08-21 18:10:48 | 0x22b3880 |      |          0 |
+    (2 rows)
+    ```
 
 ### Running Pgbouncer as part of a non-HA GitLab installation
 

doc/administration/integration/plantuml.md

@@ -57,9 +57,9 @@ you can change these defaults by editing the `/etc/tomcat7/server.xml` file.
 You need to enable PlantUML integration from Settings under Admin Area. To do
 that, login with an Admin account and do following:
 
- - in GitLab go to **Admin Area**->**Settings**->**Integrations**->**PlantUML**
- - check **Enable PlantUML** checkbox
- - set the PlantUML instance as **PlantUML URL**
+- in GitLab go to **Admin Area**->**Settings**->**Integrations**->**PlantUML**
+- check **Enable PlantUML** checkbox
+- set the PlantUML instance as **PlantUML URL**
 
 ## Creating Diagrams
 
@@ -74,6 +74,7 @@ our AsciiDoc snippets, wikis and repos using delimited blocks:
   Alice -> Bob : Go Away
   ```</pre>
 
+
 - **AsciiDoc**
 
    ```
@@ -111,11 +112,11 @@ diagram delimiters `@startuml`/`@enduml` as these are replaced by the AsciiDoc `
 
 Some parameters can be added to the AsciiDoc block definition:
 
- - *format*: Can be either `png` or `svg`. Note that `svg` is not supported by
+- *format*: Can be either `png` or `svg`. Note that `svg` is not supported by
   all browsers so use with care. The default is `png`.
- - *id*: A CSS id added to the diagram HTML tag.
- - *width*: Width attribute added to the img tag.
- - *height*: Height attribute added to the img tag.
+- *id*: A CSS id added to the diagram HTML tag.
+- *width*: Width attribute added to the img tag.
+- *height*: Height attribute added to the img tag.
 
 Markdown does not support any parameters and will always use PNG format.
 

doc/administration/operations/filesystem_benchmarking.md

@@ -81,16 +81,19 @@ executed, and then read the same 1,000 files.
    ```sh
    mkdir test; cd test
    ```
+
 1. Run the command:
 
    ```sh
    time for i in {0..1000}; do echo 'test' > "test${i}.txt"; done
    ```
+
 1. To benchmark read performance, run the command:
 
    ```sh
    time for i in {0..1000}; do cat "test${i}.txt" > /dev/null; done
    ```
+
 1. Remove the test files:
 
   ```sh

doc/administration/pages/index.md

@@ -395,6 +395,7 @@ Follow the steps below to configure GitLab Pages in a separate server.
    node_exporter['enable'] = false
    gitlab_rails['auto_migrate'] = false
    ```
+
 1. Run `sudo gitlab-ctl reconfigure`.
 1. On `app1` apply the following changes to `/etc/gitlab/gitlab.rb`:
 

doc/api/README.md

@@ -154,9 +154,10 @@ the `/Users` endpoint. The base URL is: `/api/scim/v2/groups/:group_path/Users/`
 
 ## Road to GraphQL
 
-Going forward, we will start on moving to
-[GraphQL](graphql/index.md) and deprecate the use of
-controller-specific endpoints. GraphQL has a number of benefits:
+[GraphQL](graphql/index.md) is available in GitLab, which will
+allow deprecation of controller-specific endpoints.
+
+GraphQL has a number of benefits:
 
 1. We avoid having to maintain two different APIs.
 1. Callers of the API can request only what they need.

doc/api/graphql/index.md

 # GraphQL API
 
-> [Introduced][ce-19008] in GitLab 11.0.
+> - [Introduced][ce-19008] in GitLab 11.0 (enabled by feature flag `graphql`).
+> - [Always enabled](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/30444)
+  in GitLab 12.1.
 
 [GraphQL](https://graphql.org/) is a query language for APIs that
 allows clients to request exactly the data they need, making it
@@ -29,8 +31,6 @@ There are no plans to deprecate the REST API. To reduce the technical burden of
 supporting two APIs in parallel, they should share implementations as much as
 possible.
 
-As of the 12.1 release, GraphQL is always enabled.
-
 ## Available queries
 
 A first iteration of a GraphQL API includes the following queries

spec/controllers/projects/merge_requests_controller_spec.rb

@@ -878,7 +878,7 @@ describe Projects::MergeRequestsController do
         expect(control_count).to be <= 137
       end
 
-      it 'has no N+1 issues for environments', :request_store, retry: 0 do
+      it 'has no N+1 SQL issues for environments', :request_store, retry: 0 do
         # First run to insert test data from lets, which does take up some 30 queries
         get_ci_environments_status
 
@@ -893,6 +893,54 @@ describe Projects::MergeRequestsController do
         leeway = 11
         expect { get_ci_environments_status }.not_to exceed_all_query_limit(control_count + leeway)
       end
+    end
+
+    context 'when a merge request has multiple environments with deployments' do
+      let(:sha) { merge_request.diff_head_sha }
+      let(:ref) { merge_request.source_branch }
+
+      let!(:build) { create(:ci_build, pipeline: pipeline) }
+      let!(:pipeline) { create(:ci_pipeline, sha: sha, project: project) }
+      let!(:environment) { create(:environment, name: 'env_a', project: project) }
+      let!(:another_environment) { create(:environment, name: 'env_b', project: project) }
+
+      before do
+        merge_request.update_head_pipeline
+
+        create(:deployment, :succeed, environment: environment, sha: sha, ref: ref, deployable: build)
+        create(:deployment, :succeed, environment: another_environment, sha: sha, ref: ref, deployable: build)
+      end
+
+      it 'exposes multiple environment statuses' do
+        get_ci_environments_status
+
+        expect(json_response.count).to eq 2
+      end
+
+      context 'when route map is not present in the project' do
+        it 'does not have N+1 Gitaly requests for environments', :request_store do
+          expect(merge_request).to be_present
+
+          expect { get_ci_environments_status }
+            .to change { Gitlab::GitalyClient.get_request_count }.by_at_most(1)
+        end
+      end
+
+      context 'when there is route map present in a project' do
+        before do
+          allow_any_instance_of(EnvironmentStatus)
+            .to receive(:has_route_map?)
+            .and_return(true)
+        end
+
+        it 'does not have N+1 Gitaly requests for diff files', :request_store do
+          expect(merge_request.merge_request_diff.merge_request_diff_files).to be_many
+
+          expect { get_ci_environments_status }
+            .to change { Gitlab::GitalyClient.get_request_count }.by_at_most(1)
+        end
+      end
+    end
 
     def get_ci_environments_status(extra_params = {})
       params = {
@@ -905,7 +953,6 @@ describe Projects::MergeRequestsController do
       get :ci_environments_status, params: params.merge(extra_params)
     end
   end
-  end
 
   describe 'GET pipeline_status.json' do
     context 'when head_pipeline exists' do