Automatic merge of gitlab-org/gitlab master

GITALY_SERVER_VERSION

-c9f804dd7049915ab276d8897d0e9f938cf8b362
+1ce9d77e4ed41e124d840b23689392fda6945b3c

app/graphql/types/ci/runner_type.rb

@@ -54,10 +54,12 @@ module Types
       # rubocop: disable CodeReuse/ActiveRecord
       def project_count
         BatchLoader::GraphQL.for(runner.id).batch(key: :runner_project_count) do |ids, loader, args|
-          counts = ::Ci::RunnerProject.select(:runner_id, 'COUNT(*) as count')
-            .where(runner_id: ids)
-            .group(:runner_id)
-            .index_by(&:runner_id)
+          counts = ::Ci::Runner.project_type
+            .select(:id, 'COUNT(ci_runner_projects.id) as count')
+            .left_outer_joins(:runner_projects)
+            .where(id: ids)
+            .group(:id)
+            .index_by(&:id)
 
           ids.each do |id|
             loader.call(id, counts[id]&.count)

app/models/ci/build_metadata.rb

@@ -39,6 +39,7 @@ module Ci
     }
 
     ignore_column :build_id_convert_to_bigint, remove_with: '14.2', remove_after: '2021-08-22'
+    ignore_columns :id_convert_to_bigint, remove_with: '14.3', remove_after: '2021-09-22'
 
     def update_timeout_state
       timeout = timeout_with_highest_precedence

db/migrate/20210624112821_add_devops_adoption_coverage_fuzzing.rb

+# frozen_string_literal: true
+
+class AddDevopsAdoptionCoverageFuzzing < ActiveRecord::Migration[6.1]
+  def change
+    add_column :analytics_devops_adoption_snapshots, :coverage_fuzzing_enabled_count, :integer
+  end
+end

db/migrate/20210705142714_initialize_conversion_of_ci_builds_metadata_id_to_bigint.rb

+# frozen_string_literal: true
+
+class InitializeConversionOfCiBuildsMetadataIdToBigint < ActiveRecord::Migration[6.1]
+  include Gitlab::Database::MigrationHelpers
+
+  TABLE = :ci_builds_metadata
+  COLUMN = :id
+
+  def up
+    initialize_conversion_of_integer_to_bigint(TABLE, COLUMN)
+  end
+
+  def down
+    revert_initialize_conversion_of_integer_to_bigint(TABLE, COLUMN)
+  end
+end

db/post_migrate/20210705143150_backfill_ci_builds_metadata_id_for_bigint_conversion.rb

+# frozen_string_literal: true
+
+class BackfillCiBuildsMetadataIdForBigintConversion < ActiveRecord::Migration[6.1]
+  include Gitlab::Database::MigrationHelpers
+
+  TABLE = :ci_builds_metadata
+  COLUMN = :id
+
+  def up
+    backfill_conversion_of_integer_to_bigint TABLE, COLUMN, batch_size: 15000, sub_batch_size: 100
+  end
+
+  def down
+    revert_backfill_conversion_of_integer_to_bigint TABLE, COLUMN
+  end
+end

db/schema_migrations/20210624112821

+2fca5d3203b7bb766274fc333e9a2a267340142856bd23efaabb1cb2c1cb7cb7
\ No newline at end of file

db/schema_migrations/20210705142714

+b47570ac9018a50206f239ffdafce2e672e6888a11a51e01f6d59d62a6a929af
\ No newline at end of file

db/schema_migrations/20210705143150

+9240ebbc69525a5bc1732f5e0ee2903f2b40b8693f24fcb911e72b5ba943357e
\ No newline at end of file

db/structure.sql

@@ -87,6 +87,15 @@ BEGIN
 END;
 $$;
 
+CREATE FUNCTION trigger_542d6c2ad72e() RETURNS trigger
+    LANGUAGE plpgsql
+    AS $$
+BEGIN
+  NEW."id_convert_to_bigint" := NEW."id";
+  RETURN NEW;
+END;
+$$;
+
 CREATE FUNCTION trigger_69523443cc10() RETURNS trigger
     LANGUAGE plpgsql
     AS $$
@@ -9129,6 +9138,7 @@ CREATE TABLE analytics_devops_adoption_snapshots (
     sast_enabled_count integer,
     dast_enabled_count integer,
     dependency_scanning_enabled_count integer,
+    coverage_fuzzing_enabled_count integer,
     CONSTRAINT check_3f472de131 CHECK ((namespace_id IS NOT NULL))
 );
 
@@ -10579,7 +10589,8 @@ CREATE TABLE ci_builds_metadata (
     environment_auto_stop_in character varying(255),
     expanded_environment_name character varying(255),
     secrets jsonb DEFAULT '{}'::jsonb NOT NULL,
-    build_id_convert_to_bigint bigint DEFAULT 0 NOT NULL
+    build_id_convert_to_bigint bigint DEFAULT 0 NOT NULL,
+    id_convert_to_bigint bigint DEFAULT 0 NOT NULL
 );
 
 CREATE SEQUENCE ci_builds_metadata_id_seq
@@ -25548,6 +25559,8 @@ CREATE TRIGGER trigger_490d204c00b3 BEFORE INSERT OR UPDATE ON ci_stages FOR EAC
 
 CREATE TRIGGER trigger_51ab7cef8934 BEFORE INSERT OR UPDATE ON ci_builds_runner_session FOR EACH ROW EXECUTE FUNCTION trigger_51ab7cef8934();
 
+CREATE TRIGGER trigger_542d6c2ad72e BEFORE INSERT OR UPDATE ON ci_builds_metadata FOR EACH ROW EXECUTE FUNCTION trigger_542d6c2ad72e();
+
 CREATE TRIGGER trigger_69523443cc10 BEFORE INSERT OR UPDATE ON events FOR EACH ROW EXECUTE FUNCTION trigger_69523443cc10();
 
 CREATE TRIGGER trigger_77f5e1d20482 BEFORE INSERT OR UPDATE ON deployments FOR EACH ROW EXECUTE FUNCTION trigger_77f5e1d20482();

doc/api/graphql/reference/index.md

@@ -8506,6 +8506,7 @@ Snapshot.
 | Name | Type | Description |
 | ---- | ---- | ----------- |
 | <a id="devopsadoptionsnapshotcodeownersusedcount"></a>`codeOwnersUsedCount` | [`Int`](#int) | Total number of projects with existing CODEOWNERS file. |
+| <a id="devopsadoptionsnapshotcoveragefuzzingenabledcount"></a>`coverageFuzzingEnabledCount` | [`Int`](#int) | Total number of projects with enabled coverage fuzzing. |
 | <a id="devopsadoptionsnapshotdastenabledcount"></a>`dastEnabledCount` | [`Int`](#int) | Total number of projects with enabled DAST. |
 | <a id="devopsadoptionsnapshotdependencyscanningenabledcount"></a>`dependencyScanningEnabledCount` | [`Int`](#int) | Total number of projects with enabled dependency scanning. |
 | <a id="devopsadoptionsnapshotdeploysucceeded"></a>`deploySucceeded` | [`Boolean!`](#boolean) | At least one deployment succeeded. |

doc/development/database_review.md

@@ -43,7 +43,7 @@ If your merge request description does not include these items, the review will
 
 If new migrations are introduced, in the MR **you are required to provide**:
 
-- The output of both migrating and rolling back for all migrations
+- The output of both migrating (`db:migrate`) and rolling back (`db:rollback`) for all migrations.
 
 If new queries have been introduced or existing queries have been updated, **you are required to provide**:
 
@@ -104,7 +104,7 @@ the following preparations into account.
 `db/schema_migrations` were added or removed.
 - Make migrations reversible by using the `change` method or include a `down` method when using `up`.
   - Include either a rollback procedure or describe how to rollback changes.
-- Add the output of both migrating and rolling back for all migrations into the MR description.
+- Add the output of both migrating (`db:migrate`) and rolling back (`db:rollback`) for all migrations into the MR description.
   - Ensure the down method reverts the changes in `db/structure.sql`.
   - Update the migration output whenever you modify the migrations during the review process.
 - Add tests for the migration in `spec/migrations` if necessary. See [Testing Rails migrations at GitLab](testing_guide/testing_migrations_guide.md) for more details.

doc/development/migration_style_guide.md

@@ -19,8 +19,8 @@ Migrations are **not** allowed to require GitLab installations to be taken
 offline ever. Migrations always must be written in such a way to avoid
 downtime. In the past we had a process for defining migrations that allowed for
 downtime by setting a `DOWNTIME` constant. You may see this when looking at
-older migrations. This process was in place for 4 years without every being
-used and as such we've learnt we can always figure out how to write a migration
+older migrations. This process was in place for 4 years without ever being
+used and as such we've learned we can always figure out how to write a migration
 differently to avoid downtime.
 
 When writing your migrations, also consider that databases might have stale data
@@ -35,10 +35,21 @@ For GitLab.com, please take into consideration that regular migrations (under `d
 are run before [Canary is deployed](https://gitlab.com/gitlab-com/gl-infra/readiness/-/tree/master/library/canary/#configuration-and-deployment),
 and [post-deployment migrations](post_deployment_migrations.md) (`db/post_migrate`) are run after the deployment to production has finished.
 
+## Create database migrations
+
+To create a migration you can use the following Rails generator:
+
+```shell
+bundle exec rails g migration migration_name_here
+```
+
+This generates the migration file in `db/migrate`.
+
 ## Schema Changes
 
 Changes to the schema should be committed to `db/structure.sql`. This
-file is automatically generated by Rails, so you normally should not
+file is automatically generated by Rails when you run
+`bundle exec rails db:migrate`, so you normally should not
 edit this file by hand. If your migration is adding a column to a
 table, that column is added at the bottom. Please do not reorder
 columns manually for existing tables as this causes confusion to

doc/update/index.md

@@ -194,7 +194,7 @@ Find where your version sits in the upgrade path below, and upgrade GitLab
 accordingly, while also consulting the
 [version-specific upgrade instructions](#version-specific-upgrading-instructions):
 
-`8.11.Z` -> `8.12.0` -> `8.17.7` -> `9.5.10` -> `10.8.7` -> `11.11.8` -> `12.0.12` -> `12.1.17` -> `12.10.14` -> `13.0.14` -> `13.1.11` -> [latest `13.12.Z`](https://about.gitlab.com/releases/categories/releases/) -> [latest `14.0.Z`](https://about.gitlab.com/releases/categories/releases/)
+`8.11.Z` -> [`8.12.0`](#upgrades-from-versions-earlier-than-812) -> `8.17.7` -> `9.5.10` -> `10.8.7` -> [`11.11.8`](#1200) -> `12.0.12` -> [`12.1.17`](#1210) -> `12.10.14` -> `13.0.14` -> [`13.1.11`](#1310) -> [latest `13.12.Z`](https://about.gitlab.com/releases/categories/releases/) -> [latest `14.0.Z`](https://about.gitlab.com/releases/categories/releases/) -> [latest `14.Y.Z`](https://about.gitlab.com/releases/categories/releases/)
 
 The following table, while not exhaustive, shows some examples of the supported
 upgrade paths.
@@ -213,7 +213,7 @@ upgrade paths.
 Upgrading the *major* version requires more attention.
 Backward-incompatible changes and migrations are reserved for major versions.
 We cannot guarantee that upgrading between major versions will be seamless.
-It is suggested to upgrade to the latest available *minor* version within
+It is required to upgrade to the latest available *minor* version within
 your major version before proceeding to the next major version.
 Doing this addresses any backward-incompatible changes or deprecations
 to help ensure a successful upgrade to the next major release.
@@ -373,11 +373,9 @@ and [Helm Chart deployments](https://docs.gitlab.com/charts/). They come with ap
 
 - In GitLab 13.3 some [pipeline processing methods were deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/218536)
   and this code was completely removed in GitLab 14.0. If you plan to upgrade from
-  **GitLab 13.2 or older** directly to 14.0, you should not have any pipelines running
-  when you upgrade. The pipelines might report the wrong status when the upgrade completes.
-  You should shut down GitLab and wait for all pipelines on runners to complete, then upgrade
-  GitLab to 14.0. Alternatively, you can first upgrade GitLab to a version between 13.3 and
-  13.12, then upgrade to 14.0.
+  **GitLab 13.2 or older** directly to 14.0 ([unsupported](#upgrading-to-a-new-major-version)), you should not have any pipelines running
+  when you upgrade or the pipelines might report the wrong status when the upgrade completes.
+  You should instead follow a [supported upgrade path](#upgrade-paths).
 - The support of PostgreSQL 11 [has been dropped](../install/requirements.md#database). Make sure to [update your database](https://docs.gitlab.com/omnibus/settings/database.html#upgrade-packaged-postgresql-server) to version 12 before updating to GitLab 14.0.
 
 ### 13.11.0
@@ -477,7 +475,7 @@ So, if you are using multiple Rails servers and specifically upgrading from 13.0
 all servers must first be upgraded to 13.1.Z before upgrading to 13.2.0 or later:
 
 1. Ensure all GitLab web nodes are running GitLab 13.1.Z.
-1. Optionally, enable the `global_csrf_token` feature flag to enable new
+1. Enable the `global_csrf_token` feature flag to enable new
    method of CSRF token generation:
 
    ```ruby

doc/user/project/integrations/overview.md

@@ -114,6 +114,6 @@ plugins. This allows the community to keep the plugins up to date so that they
 always work in newer GitLab versions.
 
 For an overview of what integrations are available, please see the
-[project_services source directory](https://gitlab.com/gitlab-org/gitlab/-/tree/master/app/models/project_services).
+[integrations source directory](https://gitlab.com/gitlab-org/gitlab/-/tree/master/app/models/integrations).
 
 Contributions are welcome!

ee/app/graphql/types/analytics/devops_adoption/snapshot_type.rb

@@ -30,6 +30,8 @@ module Types
               description: 'Total number of projects with enabled DAST.'
         field :dependency_scanning_enabled_count, GraphQL::INT_TYPE, null: true,
               description: 'Total number of projects with enabled dependency scanning.'
+        field :coverage_fuzzing_enabled_count, GraphQL::INT_TYPE, null: true,
+              description: 'Total number of projects with enabled coverage fuzzing.'
         field :total_projects_count, GraphQL::INT_TYPE, null: true,
               description: 'Total number of projects.'
         field :recorded_at, Types::TimeType, null: false,

ee/app/models/analytics/devops_adoption/snapshot.rb

@@ -18,6 +18,7 @@ class Analytics::DevopsAdoption::Snapshot < ApplicationRecord
     :sast_enabled_count,
     :dast_enabled_count,
     :dependency_scanning_enabled_count,
+    :coverage_fuzzing_enabled_count,
     :total_projects_count
   ].freeze
 

ee/app/services/app_sec/dast/site_profiles/update_service.rb

@@ -22,7 +22,7 @@ module AppSec
           return ServiceResponse.error(message: _('Cannot modify %{profile_name} referenced in security policy') % { profile_name: dast_site_profile.name }) if referenced_in_security_policy?
 
           ActiveRecord::Base.transaction do
-            auditor = Audit::UpdateService.new(project, current_user, {
+            auditor = AppSec::Dast::SiteProfiles::Audit::UpdateService.new(project, current_user, {
               dast_site_profile: dast_site_profile,
               new_params: params.dup,
               old_params: dast_site_profile.attributes.symbolize_keys.merge(

ee/lib/analytics/devops_adoption/snapshot_calculator.rb

@@ -113,6 +113,10 @@ module Analytics
         projects_count_with_artifact(Ci::JobArtifact.dependency_list_reports)
       end
 
+      def coverage_fuzzing_enabled_count
+        projects_count_with_artifact(Ci::JobArtifact.coverage_fuzzing_reports)
+      end
+
       # rubocop: disable CodeReuse/ActiveRecord
       def projects_count_with_artifact(artifacts_scope)
         subquery = artifacts_scope.created_in_time_range(from: range_start, to: range_end)

ee/spec/lib/analytics/devops_adoption/snapshot_calculator_spec.rb

@@ -197,6 +197,12 @@ RSpec.describe Analytics::DevopsAdoption::SnapshotCalculator do
     include_examples 'calculates artifact type count', :dependency_scanning
   end
 
+  describe 'coverage_fuzzing_enabled_count' do
+    subject { data[:coverage_fuzzing_enabled_count] }
+
+    include_examples 'calculates artifact type count', :coverage_fuzzing
+  end
+
   context 'when snapshot already exists' do
     subject(:data) { described_class.new(enabled_namespace: enabled_namespace, range_end: range_end, snapshot: snapshot).calculate }
 

lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb

@@ -133,11 +133,7 @@ module Gitlab
         end
 
         def idempotency_string
-          # TODO: dump the argument's JSON using `Sidekiq.dump_json` instead
-          # this should be done in the next release so all jobs are written
-          # with their idempotency key.
-          # see https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1090
-          "#{worker_class_name}:#{arguments.join('-')}"
+          "#{worker_class_name}:#{Sidekiq.dump_json(arguments)}"
         end
       end
     end

spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job_spec.rb

@@ -13,7 +13,7 @@ RSpec.describe Gitlab::SidekiqMiddleware::DuplicateJobs::DuplicateJob, :clean_gi
   let(:queue) { 'authorized_projects' }
 
   let(:idempotency_key) do
-    hash = Digest::SHA256.hexdigest("#{job['class']}:#{job['args'].join('-')}")
+    hash = Digest::SHA256.hexdigest("#{job['class']}:#{Sidekiq.dump_json(job['args'])}")
     "#{Gitlab::Redis::Queues::SIDEKIQ_NAMESPACE}:duplicate:#{queue}:#{hash}"
   end
 

spec/requests/api/graphql/ci/runner_spec.rb

@@ -5,25 +5,25 @@ require 'spec_helper'
 RSpec.describe 'Query.runner(id)' do
   include GraphqlHelpers
 
-  let_it_be(:user) { create_default(:user, :admin) }
+  let_it_be(:user) { create(:user, :admin) }
 
-  let_it_be(:active_runner) do
+  let_it_be(:active_instance_runner) do
     create(:ci_runner, :instance, description: 'Runner 1', contacted_at: 2.hours.ago,
            active: true, version: 'adfe156', revision: 'a', locked: true, ip_address: '127.0.0.1', maximum_timeout: 600,
            access_level: 0, tag_list: %w[tag1 tag2], run_untagged: true)
   end
 
-  let_it_be(:inactive_runner) do
+  let_it_be(:inactive_instance_runner) do
     create(:ci_runner, :instance, description: 'Runner 2', contacted_at: 1.day.ago, active: false,
            version: 'adfe157', revision: 'b', ip_address: '10.10.10.10', access_level: 1, run_untagged: true)
   end
 
   def get_runner(id)
     case id
-    when :active_runner
-      active_runner
-    when :inactive_runner
-      inactive_runner
+    when :active_instance_runner
+      active_instance_runner
+    when :inactive_instance_runner
+      inactive_instance_runner
     end
   end
 
@@ -86,7 +86,7 @@ RSpec.describe 'Query.runner(id)' do
   end
 
   describe 'for active runner' do
-    it_behaves_like 'runner details fetch', :active_runner
+    it_behaves_like 'runner details fetch', :active_instance_runner
 
     context 'when tagList is not requested' do
       let(:query) do
@@ -95,7 +95,7 @@ RSpec.describe 'Query.runner(id)' do
 
       let(:query_path) do
         [
-          [:runner, { id: active_runner.to_global_id.to_s }]
+          [:runner, { id: active_instance_runner.to_global_id.to_s }]
         ]
       end
 
@@ -110,29 +110,30 @@ RSpec.describe 'Query.runner(id)' do
   end
 
   describe 'for inactive runner' do
-    it_behaves_like 'runner details fetch', :inactive_runner
+    it_behaves_like 'runner details fetch', :inactive_instance_runner
   end
 
   describe 'for multiple runners' do
     let_it_be(:project1) { create(:project, :test_repo) }
     let_it_be(:project2) { create(:project, :test_repo) }
-    let_it_be(:project_runner) do
-      create(:ci_runner, :project, projects: [project1, project2], description: 'Runner 1', contacted_at: 2.hours.ago,
-             active: true, version: 'adfe156', revision: 'a', locked: true, ip_address: '127.0.0.1', maximum_timeout: 600,
-             access_level: 0, run_untagged: true)
-    end
+    let_it_be(:project_runner1) { create(:ci_runner, :project, projects: [project1, project2], description: 'Runner 1') }
+    let_it_be(:project_runner2) { create(:ci_runner, :project, projects: [], description: 'Runner 2') }
 
-    let!(:job) { create(:ci_build, runner: project_runner) }
+    let!(:job) { create(:ci_build, runner: project_runner1) }
 
     context 'requesting project and job counts' do
       let(:query) do
         %(
           query {
-            projectRunner: runner(id: "#{project_runner.to_global_id}") {
+            projectRunner1: runner(id: "#{project_runner1.to_global_id}") {
+              projectCount
+              jobCount
+            }
+            projectRunner2: runner(id: "#{project_runner2.to_global_id}") {
               projectCount
               jobCount
             }
-            activeRunner: runner(id: "#{active_runner.to_global_id}") {
+            activeInstanceRunner: runner(id: "#{active_instance_runner.to_global_id}") {
               projectCount
               jobCount
             }
@@ -141,17 +142,23 @@ RSpec.describe 'Query.runner(id)' do
       end
 
       before do
+        project_runner2.projects.clear
+
         post_graphql(query, current_user: user)
       end
 
       it 'retrieves expected fields' do
-        runner1_data = graphql_data_at(:project_runner)
-        runner2_data = graphql_data_at(:active_runner)
+        runner1_data = graphql_data_at(:project_runner1)
+        runner2_data = graphql_data_at(:project_runner2)
+        runner3_data = graphql_data_at(:active_instance_runner)
 
         expect(runner1_data).to match a_hash_including(
           'jobCount' => 1,
           'projectCount' => 2)
         expect(runner2_data).to match a_hash_including(
+          'jobCount' => 0,
+          'projectCount' => 0)
+        expect(runner3_data).to match a_hash_including(
           'jobCount' => 0,
           'projectCount' => nil)
       end
@@ -159,15 +166,15 @@ RSpec.describe 'Query.runner(id)' do
   end
 
   describe 'by regular user' do
-    let(:user) { create_default(:user) }
+    let(:user) { create(:user) }
 
-    it_behaves_like 'retrieval by unauthorized user', :active_runner
+    it_behaves_like 'retrieval by unauthorized user', :active_instance_runner
   end
 
   describe 'by unauthenticated user' do
     let(:user) { nil }
 
-    it_behaves_like 'retrieval by unauthorized user', :active_runner
+    it_behaves_like 'retrieval by unauthorized user', :active_instance_runner
   end
 
   describe 'Query limits' do
@@ -185,7 +192,7 @@ RSpec.describe 'Query.runner(id)' do
     let(:single_query) do
       <<~QUERY
         {
-          active: #{runner_query(active_runner)}
+          active: #{runner_query(active_instance_runner)}
         }
       QUERY
     end
@@ -193,8 +200,8 @@ RSpec.describe 'Query.runner(id)' do
     let(:double_query) do
       <<~QUERY
         {
-          active: #{runner_query(active_runner)}
-          inactive: #{runner_query(inactive_runner)}
+          active: #{runner_query(active_instance_runner)}
+          inactive: #{runner_query(inactive_instance_runner)}
         }
       QUERY
     end