Merge branch '325640-query-limiting-report-script' into 'master'

Create a script to help reconcile and clean up all Gitlab::QueryLimiting.disable references with corresponding issues

See merge request gitlab-org/gitlab!57209

app/controllers/admin/application_settings_controller.rb

@@ -191,7 +191,7 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController
   end
 
   def disable_query_limiting
-    Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab-foss/issues/63107')
+    Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/29418')
   end
 
   def application_setting_params

app/controllers/import/gitlab_projects_controller.rb

@@ -43,7 +43,7 @@ class Import::GitlabProjectsController < Import::BaseController
   end
 
   def disable_query_limiting
-    Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab-foss/issues/42437')
+    Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20823')
   end
 
   def uploader_class

app/controllers/import/manifest_controller.rb

@@ -116,6 +116,6 @@ class Import::ManifestController < Import::BaseController
   end
 
   def disable_query_limiting
-    Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab-foss/issues/48939')
+    Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/23147')
   end
 end

app/controllers/projects/forks_controller.rb

@@ -119,7 +119,7 @@ class Projects::ForksController < Projects::ApplicationController
   end
 
   def disable_query_limiting
-    Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab-foss/issues/42335')
+    Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20783')
   end
 
   def load_namespaces_with_associations

app/controllers/projects/issues_controller.rb

@@ -356,10 +356,10 @@ class Projects::IssuesController < Projects::ApplicationController
   def disable_query_limiting
     # Also see the following issues:
     #
-    # 1. https://gitlab.com/gitlab-org/gitlab-foss/issues/42423
-    # 2. https://gitlab.com/gitlab-org/gitlab-foss/issues/42424
-    # 3. https://gitlab.com/gitlab-org/gitlab-foss/issues/42426
-    Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab-foss/issues/42422')
+    # 1. https://gitlab.com/gitlab-org/gitlab/-/issues/20815
+    # 2. https://gitlab.com/gitlab-org/gitlab/-/issues/20816
+    # 3. https://gitlab.com/gitlab-org/gitlab/-/issues/21068
+    Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20814')
   end
 
   private

app/controllers/projects/merge_requests/creations_controller.rb

@@ -134,7 +134,7 @@ class Projects::MergeRequests::CreationsController < Projects::MergeRequests::Ap
   # rubocop: enable CodeReuse/ActiveRecord
 
   def disable_query_limiting
-    Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab-foss/issues/42384')
+    Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20801')
   end
 
   def incr_count_webide_merge_request

app/controllers/projects/merge_requests_controller.rb

@@ -470,8 +470,8 @@ class Projects::MergeRequestsController < Projects::MergeRequests::ApplicationCo
   end
 
   def disable_query_limiting
-    # Also see https://gitlab.com/gitlab-org/gitlab-foss/issues/42441
-    Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab-foss/issues/42438')
+    # Also see https://gitlab.com/gitlab-org/gitlab/-/issues/20827
+    Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20824')
   end
 
   def reports_response(report_comparison, pipeline = nil)

app/controllers/projects/network_controller.rb

@@ -43,6 +43,6 @@ class Projects::NetworkController < Projects::ApplicationController
   end
 
   def disable_query_limiting
-    Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab-foss/issues/42333')
+    Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20782')
   end
 end

app/controllers/projects/notes_controller.rb

@@ -88,6 +88,6 @@ class Projects::NotesController < Projects::ApplicationController
   end
 
   def disable_query_limiting
-    Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab-foss/issues/42383')
+    Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20800')
   end
 end

app/controllers/projects/pipelines_controller.rb

@@ -272,8 +272,8 @@ class Projects::PipelinesController < Projects::ApplicationController
   end
 
   def disable_query_limiting
-    # Also see https://gitlab.com/gitlab-org/gitlab-foss/issues/42343
-    Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab-foss/issues/42339')
+    # Also see https://gitlab.com/gitlab-org/gitlab/-/issues/20785
+    Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20784')
   end
 
   def authorize_update_pipeline!

app/controllers/registrations_controller.rb

@@ -163,7 +163,7 @@ class RegistrationsController < Devise::RegistrationsController
   end
 
   def disable_query_limiting
-    Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab-foss/issues/42380')
+    Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20798')
   end
 
   def load_recaptcha

app/graphql/mutations/issues/move.rb

@@ -11,7 +11,7 @@ module Mutations
                description: 'The project to move the issue to.'
 
       def resolve(project_path:, iid:, target_project_path:)
-        Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/267762')
+        Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20816')
 
         issue = authorized_find!(project_path: project_path, iid: iid)
         source_project = issue.project

app/graphql/mutations/merge_requests/accept.rb

@@ -42,7 +42,7 @@ module Mutations
                description: 'Squash commits on the source branch before merge.'
 
       def resolve(project_path:, iid:, **args)
-        Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab-foss/issues/42317')
+        Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/4796')
 
         merge_request = authorized_find!(project_path: project_path, iid: iid)
         project = merge_request.target_project

app/services/post_receive_service.rb

@@ -48,7 +48,7 @@ class PostReceiveService
   end
 
   def process_mr_push_options(push_options, changes)
-    Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab-foss/issues/61359')
+    Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/28494')
     return unless repository
 
     unless repository.repo_type.project?

lib/api/ci/pipelines.rb

@@ -70,7 +70,7 @@ module API
           optional :variables, Array, desc: 'Array of variables available in the pipeline'
         end
         post ':id/pipeline' do
-          Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab-foss/issues/42124')
+          Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20711')
 
           authorize! :create_pipeline, user_project
 

lib/api/groups.rb

@@ -112,7 +112,7 @@ module API
       end
 
       def delete_group(group)
-        Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab-foss/issues/46285')
+        Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/22226')
         destroy_conditionally!(group) do |group|
           ::Groups::DestroyService.new(group, current_user).async_execute
         end

lib/api/issues.rb

@@ -242,7 +242,7 @@ module API
         use :issue_params
       end
       post ':id/issues' do
-        Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab-foss/issues/42320')
+        Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20773')
 
         check_rate_limit! :issues_create, [current_user]
 
@@ -288,7 +288,7 @@ module API
       end
       # rubocop: disable CodeReuse/ActiveRecord
       put ':id/issues/:issue_iid' do
-        Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab-foss/issues/42322')
+        Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20775')
 
         issue = user_project.issues.find_by!(iid: params.delete(:issue_iid))
         authorize! :update_issue, issue
@@ -346,7 +346,7 @@ module API
       end
       # rubocop: disable CodeReuse/ActiveRecord
       post ':id/issues/:issue_iid/move' do
-        Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab-foss/issues/42323')
+        Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20776')
 
         issue = user_project.issues.find_by(iid: params[:issue_iid])
         not_found!('Issue') unless issue

lib/api/merge_requests.rb

@@ -207,7 +207,7 @@ module API
         use :optional_params
       end
       post ":id/merge_requests" do
-        Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab-foss/issues/42316')
+        Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20770')
 
         authorize! :create_merge_request_from, user_project
 
@@ -416,7 +416,7 @@ module API
         at_least_one_of(*::API::MergeRequests.update_params_at_least_one_of)
       end
       put ':id/merge_requests/:merge_request_iid' do
-        Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab-foss/issues/42318')
+        Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20772')
 
         merge_request = find_merge_request_with_access(params.delete(:merge_request_iid), :update_merge_request)
 
@@ -445,7 +445,7 @@ module API
         optional :squash, type: Grape::API::Boolean, desc: 'When true, the commits will be squashed into a single commit on merge'
       end
       put ':id/merge_requests/:merge_request_iid/merge' do
-        Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab-foss/issues/42317')
+        Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/4796')
 
         merge_request = find_project_merge_request(params[:merge_request_iid])
 

lib/api/project_import.rb

@@ -67,7 +67,7 @@ module API
 
         check_rate_limit! :project_import, [current_user, :project_import]
 
-        Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab-foss/issues/42437')
+        Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20823')
 
         validate_file!
 

lib/api/projects.rb

@@ -310,7 +310,7 @@ module API
         optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The visibility of the fork'
       end
       post ':id/fork', feature_category: :source_code_management do
-        Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab-foss/issues/42284')
+        Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20759')
 
         not_found! unless can?(current_user, :fork_project, user_project)
 

lib/api/triggers.rb

@@ -21,7 +21,7 @@ module API
         optional :variables, type: Hash, desc: 'The list of variables to be injected into build'
       end
       post ":id/(ref/:ref/)trigger/pipeline", requirements: { ref: /.+/ } do
-        Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab-foss/issues/42283')
+        Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20758')
 
         forbidden! if gitlab_pipeline_hook_request?
 

scripts/perf/query_limiting_report.rb

+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+####
+# Prints a report which helps reconcile occurrences of the `QueryLimiting.disable(ISSUE_LINK)`
+# allowlist block against the corresponding open issues.
+#
+# If everything is consistent, the script should ideally not report any issues or code lines,
+# other than possibly remaining "calls with no issue iid" which use variables/etc.
+#
+# - See https://gitlab.com/gitlab-org/gitlab/-/issues/325640
+# - See https://gitlab.com/groups/gitlab-org/-/epics/5670
+
+require 'rubygems'
+require 'gitlab'
+require 'optparse'
+
+class QueryLimitingReport
+  GITLAB_PROJECT_ID = 278964 # gitlab-org/gitlab project
+  ISSUES_SEARCH_LABEL = 'querylimiting-disable'
+  CODE_LINES_SEARCH_STRING = 'QueryLimiting.disable'
+  PAGINATION_LIMIT = 500
+
+  DEFAULT_OPTIONS = {
+    api_token: ENV['API_TOKEN']
+  }.freeze
+
+  def initialize(options)
+    @options = options
+
+    Gitlab.configure do |config|
+      config.endpoint = 'https://gitlab.com/api/v4'
+      config.private_token = options.fetch(:api_token)
+    end
+  end
+
+  def execute
+    # PLAN:
+    # Read all issues matching criteria and extract array of issue iids
+    # Find all code references and extract issue iids
+    # Print list of all issues without code references
+    # Print list of all code references issue iids that don't have search label
+    # Print list of all code references with no issue iids (i.e. dynamic or variable argument)
+
+    total_issues = find_issues_by_label(ISSUES_SEARCH_LABEL)
+    issues = total_issues.select { |issue| issue[:state] == 'opened' }
+    code_lines = find_code_lines
+
+    code_lines_grouped = code_lines.group_by { |code_line| code_line[:has_issue_iid] }
+    code_lines_without_issue_iid = code_lines_grouped[false]
+    code_lines_with_issue_iid = code_lines_grouped[true]
+
+    all_issue_iids_in_code_lines = code_lines_with_issue_iid.map { |line| line[:issue_iid] }
+
+    issues_without_code_references = issues.reject do |issue|
+      all_issue_iids_in_code_lines.include?(issue[:iid])
+    end
+
+    all_issue_iids = issues.map { |issue| issue[:iid] }
+    code_lines_with_missing_issues = code_lines_with_issue_iid.reject do |code_line|
+      all_issue_iids.include?(code_line[:issue_iid])
+    end
+
+    puts "\n\n\nREPORT:"
+
+    puts "\n\nFound #{total_issues.length} total issues with '#{ISSUES_SEARCH_LABEL}' search label, #{issues.length} are still opened..."
+    puts "\n\nFound #{code_lines.length} total occurrences of '#{CODE_LINES_SEARCH_STRING}' in code..."
+
+    puts "\n" + '-' * 80
+
+    puts "\n\nIssues without any '#{CODE_LINES_SEARCH_STRING}' code references (#{issues_without_code_references.length} total):"
+    pp issues_without_code_references
+
+    puts "\n" + '-' * 80
+
+    puts "\n\n'#{CODE_LINES_SEARCH_STRING}' calls with references to an issue which doesn't have '#{ISSUES_SEARCH_LABEL}' search label (#{code_lines_with_missing_issues.length} total):"
+    pp code_lines_with_missing_issues
+
+    puts "\n" + '-' * 80
+
+    puts "\n\n'#{CODE_LINES_SEARCH_STRING}' calls with no issue iid (#{code_lines_without_issue_iid&.length || 0} total):"
+    pp code_lines_without_issue_iid
+  end
+
+  private
+
+  attr_reader :options
+
+  def find_issues_by_label(label)
+    issues = []
+
+    puts("Finding issues by label #{label}...")
+    paginated_issues = Gitlab.issues(GITLAB_PROJECT_ID, 'labels' => label)
+    paginated_issues.paginate_with_limit(PAGINATION_LIMIT) do |item|
+      item_hash = item.to_hash
+
+      issue_iid = item_hash.fetch('iid')
+      issue = {
+        iid: issue_iid,
+        state: item_hash.fetch('state'),
+        title: item_hash.fetch('title'),
+        issue_url: "https://gitlab.com/gitlab-org/gitlab/issues/#{issue_iid}"
+      }
+
+      issues << issue
+    end
+
+    issues
+  end
+
+  def find_code_lines
+    code_lines = []
+
+    puts("Finding code lines...")
+    paginated_blobs = Gitlab.search_in_project(GITLAB_PROJECT_ID, 'blobs', CODE_LINES_SEARCH_STRING)
+    paginated_blobs.paginate_with_limit(PAGINATION_LIMIT) do |item|
+      item_hash = item.to_hash
+
+      filename = item_hash.fetch('filename')
+      next if filename !~ /\.rb\Z/
+
+      file_contents = Gitlab.file_contents(GITLAB_PROJECT_ID, filename)
+      file_lines = file_contents.split("\n")
+
+      file_lines.each_index do |index|
+        line = file_lines[index]
+        if line =~ /#{CODE_LINES_SEARCH_STRING}/
+          issue_iid = line.slice(%r{issues/(\d+)\D}, 1)
+          line_number = index + 1
+          code_line = {
+            file_location: "#{filename}:#{line_number}",
+            filename: filename,
+            line_number: line_number,
+            line: line,
+            issue_iid: issue_iid.to_i,
+            has_issue_iid: !issue_iid.nil?
+          }
+          code_lines << code_line
+        end
+      end
+    end
+
+    code_lines.sort_by! { |line| "#{line[:filename]}-#{line[:line_number].to_s.rjust(4, '0')}" }
+    code_lines.map do |line|
+      line.delete(:filename)
+      line.delete(:line_number)
+      line
+    end
+  end
+end
+
+if $0 == __FILE__
+  options = QueryLimitingReport::DEFAULT_OPTIONS.dup
+
+  OptionParser.new do |opts|
+    opts.on("-t", "--api-token API_TOKEN", String, "A value API token with the `read_api` scope. Can be set as an env variable 'API_TOKEN'.") do |value|
+      options[:api_token] = value
+    end
+
+    opts.on("-h", "--help", "Prints this help") do
+      puts opts
+      exit
+    end
+  end.parse!
+
+  QueryLimitingReport.new(options).execute
+end