Add latest changes from gitlab-org/gitlab@master

app/graphql/resolvers/base_resolver.rb

@@ -58,5 +58,9 @@ module Resolvers
     def single?
       false
     end
+
+    def current_user
+      context[:current_user]
+    end
   end
 end

app/graphql/types/query_type.rb

@@ -40,3 +40,5 @@ module Types
           resolver: Resolvers::EchoResolver
   end
 end
+
+Types::QueryType.prepend_if_ee('EE::Types::QueryType')

doc/api/graphql/reference/gitlab_schema.graphql

@@ -654,6 +654,16 @@ type Design implements DesignFields & Noteable {
     """
     before: String
 
+    """
+    The Global ID of the most recent acceptable version
+    """
+    earlierOrEqualToId: ID
+
+    """
+    The SHA256 of the most recent acceptable version
+    """
+    earlierOrEqualToSha: String
+
     """
     Returns the first _n_ elements from the list.
     """
@@ -666,10 +676,130 @@ type Design implements DesignFields & Noteable {
   ): DesignVersionConnection!
 }
 
+"""
+A design pinned to a specific version. The image field reflects the design as of the associated version.
+"""
+type DesignAtVersion implements DesignFields {
+  """
+  The underlying design.
+  """
+  design: Design!
+
+  """
+  The diff refs for this design
+  """
+  diffRefs: DiffRefs!
+
+  """
+  How this design was changed in the current version
+  """
+  event: DesignVersionEvent!
+
+  """
+  The filename of the design
+  """
+  filename: String!
+
+  """
+  The full path to the design file
+  """
+  fullPath: String!
+
+  """
+  The ID of this design
+  """
+  id: ID!
+
+  """
+  The URL of the image
+  """
+  image: String!
+
+  """
+  The issue the design belongs to
+  """
+  issue: Issue!
+
+  """
+  The total count of user-created notes for this design
+  """
+  notesCount: Int!
+
+  """
+  The project the design belongs to
+  """
+  project: Project!
+
+  """
+  The version this design-at-versions is pinned to
+  """
+  version: DesignVersion!
+}
+
+"""
+The connection type for DesignAtVersion.
+"""
+type DesignAtVersionConnection {
+  """
+  A list of edges.
+  """
+  edges: [DesignAtVersionEdge]
+
+  """
+  A list of nodes.
+  """
+  nodes: [DesignAtVersion]
+
+  """
+  Information to aid in pagination.
+  """
+  pageInfo: PageInfo!
+}
+
+"""
+An edge in a connection.
+"""
+type DesignAtVersionEdge {
+  """
+  A cursor for use in pagination.
+  """
+  cursor: String!
+
+  """
+  The item at the end of the edge.
+  """
+  node: DesignAtVersion
+}
+
 """
 A collection of designs.
 """
 type DesignCollection {
+  """
+  Find a specific design
+  """
+  design(
+    """
+    Find a design by its filename
+    """
+    filename: String
+
+    """
+    Find a design by its ID
+    """
+    id: ID
+  ): Design
+
+  """
+  Find a design as of a version
+  """
+  designAtVersion(
+    """
+    The Global ID of the design at this version
+    """
+    id: ID!
+  ): DesignAtVersion
+
   """
   All designs for the design collection
   """
@@ -721,6 +851,21 @@ type DesignCollection {
   """
   project: Project!
 
+  """
+  A specific version
+  """
+  version(
+    """
+    The Global ID of the version
+    """
+    id: ID
+
+    """
+    The SHA256 of a specific version
+    """
+    sha: String
+  ): DesignVersion
+
   """
   All versions related to all designs, ordered newest first
   """
@@ -735,6 +880,16 @@ type DesignCollection {
     """
     before: String
 
+    """
+    The Global ID of the most recent acceptable version
+    """
+    earlierOrEqualToId: ID
+
+    """
+    The SHA256 of the most recent acceptable version
+    """
+    earlierOrEqualToSha: String
+
     """
     Returns the first _n_ elements from the list.
     """
@@ -829,6 +984,28 @@ interface DesignFields {
   project: Project!
 }
 
+type DesignManagement {
+  """
+  Find a design as of a version
+  """
+  designAtVersion(
+    """
+    The Global ID of the design at this version
+    """
+    id: ID!
+  ): DesignAtVersion
+
+  """
+  Find a version
+  """
+  version(
+    """
+    The Global ID of the version
+    """
+    id: ID!
+  ): DesignVersion
+}
+
 """
 Autogenerated input type of DesignManagementDelete
 """
@@ -924,7 +1101,30 @@ type DesignManagementUploadPayload {
   skippedDesigns: [Design!]!
 }
 
+"""
+A specific version in which designs were added, modified or deleted
+"""
 type DesignVersion {
+  """
+  A particular design as of this version, provided it is visible at this version
+  """
+  designAtVersion(
+    """
+    The ID of a specific design
+    """
+    designId: ID
+
+    """
+    The filename of a specific design
+    """
+    filename: String
+
+    """
+    The ID of the DesignAtVersion
+    """
+    id: ID
+  ): DesignAtVersion!
+
   """
   All designs that were changed in the version
   """
@@ -950,6 +1150,41 @@ type DesignVersion {
     last: Int
   ): DesignConnection!
 
+  """
+  All designs that are visible at this version, as of this version
+  """
+  designsAtVersion(
+    """
+    Returns the elements in the list that come after the specified cursor.
+    """
+    after: String
+
+    """
+    Returns the elements in the list that come before the specified cursor.
+    """
+    before: String
+
+    """
+    Filters designs by their filename
+    """
+    filenames: [String!]
+
+    """
+    Returns the first _n_ elements from the list.
+    """
+    first: Int
+
+    """
+    Filters designs by their ID
+    """
+    ids: [ID!]
+
+    """
+    Returns the last _n_ elements from the list.
+    """
+    last: Int
+  ): DesignAtVersionConnection!
+
   """
   ID of the design version
   """
@@ -5604,6 +5839,11 @@ type Query {
   """
   currentUser: User
 
+  """
+  Fields related to design management
+  """
+  designManagement: DesignManagement!
+
   """
   Text to echo back
   """

doc/api/graphql/reference/index.md

@@ -130,6 +130,24 @@ A single design
 | `event` | DesignVersionEvent! | How this design was changed in the current version |
 | `notesCount` | Int! | The total count of user-created notes for this design |
 
+## DesignAtVersion
+
+A design pinned to a specific version. The image field reflects the design as of the associated version.
+
+| Name  | Type  | Description |
+| ---   |  ---- | ----------  |
+| `id` | ID! | The ID of this design |
+| `project` | Project! | The project the design belongs to |
+| `issue` | Issue! | The issue the design belongs to |
+| `filename` | String! | The filename of the design |
+| `fullPath` | String! | The full path to the design file |
+| `image` | String! | The URL of the image |
+| `diffRefs` | DiffRefs! | The diff refs for this design |
+| `event` | DesignVersionEvent! | How this design was changed in the current version |
+| `notesCount` | Int! | The total count of user-created notes for this design |
+| `version` | DesignVersion! | The version this design-at-versions is pinned to |
+| `design` | Design! | The underlying design. |
+
 ## DesignCollection
 
 A collection of designs.
@@ -138,6 +156,16 @@ A collection of designs.
 | ---   |  ---- | ----------  |
 | `project` | Project! | Project associated with the design collection |
 | `issue` | Issue! | Issue associated with the design collection |
+| `version` | DesignVersion | A specific version |
+| `designAtVersion` | DesignAtVersion | Find a design as of a version |
+| `design` | Design | Find a specific design |
+
+## DesignManagement
+
+| Name  | Type  | Description |
+| ---   |  ---- | ----------  |
+| `version` | DesignVersion | Find a version |
+| `designAtVersion` | DesignAtVersion | Find a design as of a version |
 
 ## DesignManagementDeletePayload
 
@@ -162,10 +190,13 @@ Autogenerated return type of DesignManagementUpload
 
 ## DesignVersion
 
+A specific version in which designs were added, modified or deleted
+
 | Name  | Type  | Description |
 | ---   |  ---- | ----------  |
 | `id` | ID! | ID of the design version |
 | `sha` | ID! | SHA of the design version |
+| `designAtVersion` | DesignAtVersion! | A particular design as of this version, provided it is visible at this version |
 
 ## DestroyNotePayload
 

lib/gitlab/database/migration_helpers.rb

@@ -280,6 +280,46 @@ module Gitlab
         end
       end
 
+      # Executes the block with a retry mechanism that alters the +lock_timeout+ and +sleep_time+ between attempts.
+      # The timings can be controlled via the +timing_configuration+ parameter.
+      # If the lock was not acquired within the retry period, a last attempt is made without using +lock_timeout+.
+      #
+      # ==== Examples
+      #   # Invoking without parameters
+      #   with_lock_retries do
+      #     drop_table :my_table
+      #   end
+      #
+      #   # Invoking with custom +timing_configuration+
+      #   t = [
+      #     [1.second, 1.second],
+      #     [2.seconds, 2.seconds]
+      #   ]
+      #
+      #   with_lock_retries(timing_configuration: t) do
+      #     drop_table :my_table # this will be retried twice
+      #   end
+      #
+      #   # Disabling the retries using an environment variable
+      #   > export DISABLE_LOCK_RETRIES=true
+      #
+      #   with_lock_retries do
+      #     drop_table :my_table # one invocation, it will not retry at all
+      #   end
+      #
+      # ==== Parameters
+      # * +timing_configuration+ - [[ActiveSupport::Duration, ActiveSupport::Duration], ...] lock timeout for the block, sleep time before the next iteration, defaults to `Gitlab::Database::WithLockRetries::DEFAULT_TIMING_CONFIGURATION`
+      # * +logger+ - [Gitlab::JsonLogger]
+      # * +env+ - [Hash] custom environment hash, see the example with `DISABLE_LOCK_RETRIES`
+      def with_lock_retries(**args, &block)
+        merged_args = {
+          klass: self.class,
+          logger: Gitlab::BackgroundMigration::Logger
+        }.merge(args)
+
+        Gitlab::Database::WithLockRetries.new(merged_args).run(&block)
+      end
+
       def true_value
         Database.true_value
       end

lib/gitlab/database/with_lock_retries.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    class WithLockRetries
+      NULL_LOGGER = Gitlab::JsonLogger.new('/dev/null')
+
+      # Each element of the array represents a retry iteration.
+      # - DEFAULT_TIMING_CONFIGURATION.size provides the iteration count.
+      # - First element: DB lock_timeout
+      # - Second element: Sleep time after unsuccessful lock attempt (LockWaitTimeout error raised)
+      # - Worst case, this configuration would retry for about 40 minutes.
+      DEFAULT_TIMING_CONFIGURATION = [
+        [0.1.seconds, 0.05.seconds], # short timings, lock_timeout: 100ms, sleep after LockWaitTimeout: 50ms
+        [0.1.seconds, 0.05.seconds],
+        [0.2.seconds, 0.05.seconds],
+        [0.3.seconds, 0.10.seconds],
+        [0.4.seconds, 0.15.seconds],
+        [0.5.seconds, 2.seconds],
+        [0.5.seconds, 2.seconds],
+        [0.5.seconds, 2.seconds],
+        [0.5.seconds, 2.seconds],
+        [1.second, 5.seconds], # probably high traffic, increase timings
+        [1.second, 1.minute],
+        [0.1.seconds, 0.05.seconds],
+        [0.1.seconds, 0.05.seconds],
+        [0.2.seconds, 0.05.seconds],
+        [0.3.seconds, 0.10.seconds],
+        [0.4.seconds, 0.15.seconds],
+        [0.5.seconds, 2.seconds],
+        [0.5.seconds, 2.seconds],
+        [0.5.seconds, 2.seconds],
+        [3.seconds, 3.minutes], # probably high traffic or long locks, increase timings
+        [0.1.seconds, 0.05.seconds],
+        [0.1.seconds, 0.05.seconds],
+        [0.5.seconds, 2.seconds],
+        [0.5.seconds, 2.seconds],
+        [5.seconds, 2.minutes],
+        [0.5.seconds, 0.5.seconds],
+        [0.5.seconds, 0.5.seconds],
+        [7.seconds, 5.minutes],
+        [0.5.seconds, 0.5.seconds],
+        [0.5.seconds, 0.5.seconds],
+        [7.seconds, 5.minutes],
+        [0.5.seconds, 0.5.seconds],
+        [0.5.seconds, 0.5.seconds],
+        [7.seconds, 5.minutes],
+        [0.1.seconds, 0.05.seconds],
+        [0.1.seconds, 0.05.seconds],
+        [0.5.seconds, 2.seconds],
+        [10.seconds, 10.minutes],
+        [0.1.seconds, 0.05.seconds],
+        [0.5.seconds, 2.seconds],
+        [10.seconds, 10.minutes]
+      ].freeze
+
+      def initialize(logger: NULL_LOGGER, timing_configuration: DEFAULT_TIMING_CONFIGURATION, klass: nil, env: ENV)
+        @logger = logger
+        @klass = klass
+        @timing_configuration = timing_configuration
+        @env = env
+        @current_iteration = 1
+        @log_params = { method: 'with_lock_retries', class: klass.to_s }
+      end
+
+      def run(&block)
+        raise 'no block given' unless block_given?
+
+        @block = block
+
+        if lock_retries_disabled?
+          log(message: 'DISABLE_LOCK_RETRIES environment variable is true, executing the block without retry')
+
+          return run_block
+        end
+
+        begin
+          run_block_with_transaction
+        rescue ActiveRecord::LockWaitTimeout
+          if retry_with_lock_timeout?
+            wait_until_next_retry
+
+            retry
+          else
+            run_block_without_lock_timeout
+          end
+        end
+      end
+
+      private
+
+      attr_reader :logger, :env, :block, :current_iteration, :log_params, :timing_configuration
+
+      def run_block
+        block.call
+      end
+
+      def run_block_with_transaction
+        ActiveRecord::Base.transaction(requires_new: true) do
+          execute("SET LOCAL lock_timeout TO '#{current_lock_timeout_in_ms}ms'")
+
+          log(message: 'Lock timeout is set', current_iteration: current_iteration, lock_timeout_in_ms: current_lock_timeout_in_ms)
+
+          run_block
+
+          log(message: 'Migration finished', current_iteration: current_iteration, lock_timeout_in_ms: current_lock_timeout_in_ms)
+        end
+      end
+
+      def retry_with_lock_timeout?
+        current_iteration != retry_count
+      end
+
+      def wait_until_next_retry
+        log(message: 'ActiveRecord::LockWaitTimeout error, retrying after sleep', current_iteration: current_iteration, sleep_time_in_seconds: current_sleep_time_in_seconds)
+
+        sleep(current_sleep_time_in_seconds)
+
+        @current_iteration += 1
+      end
+
+      def run_block_without_lock_timeout
+        log(message: "Couldn't acquire lock to perform the migration", current_iteration: current_iteration)
+        log(message: "Executing the migration without lock timeout", current_iteration: current_iteration)
+
+        execute("SET LOCAL lock_timeout TO '0'")
+
+        run_block
+
+        log(message: 'Migration finished', current_iteration: current_iteration)
+      end
+
+      def lock_retries_disabled?
+        Gitlab::Utils.to_boolean(env['DISABLE_LOCK_RETRIES'])
+      end
+
+      def log(params)
+        logger.info(log_params.merge(params))
+      end
+
+      def execute(statement)
+        ActiveRecord::Base.connection.execute(statement)
+      end
+
+      def retry_count
+        timing_configuration.size
+      end
+
+      def current_lock_timeout_in_ms
+        timing_configuration[current_iteration - 1][0].in_milliseconds
+      end
+
+      def current_sleep_time_in_seconds
+        timing_configuration[current_iteration - 1][1].to_i
+      end
+    end
+  end
+end

spec/graphql/types/query_type_spec.rb

@@ -7,15 +7,10 @@ describe GitlabSchema.types['Query'] do
     expect(described_class.graphql_name).to eq('Query')
   end
 
-  it do
-    is_expected.to have_graphql_fields(:project,
-                                       :namespace,
-                                       :group,
-                                       :echo,
-                                       :metadata,
-                                       :current_user,
-                                       :snippets
-                                      ).at_least
+  it 'has the expected fields' do
+    expected_fields = %i[project namespace group echo metadata current_user snippets]
+
+    expect(described_class).to have_graphql_fields(*expected_fields).at_least
   end
 
   describe 'namespace field' do

spec/lib/gitlab/database/migration_helpers_spec.rb

@@ -1518,4 +1518,17 @@ describe Gitlab::Database::MigrationHelpers do
       model.create_or_update_plan_limit('project_hooks', 'free', 10)
     end
   end
+
+  describe '#with_lock_retries' do
+    let(:buffer) { StringIO.new }
+    let(:in_memory_logger) { Gitlab::JsonLogger.new(buffer) }
+    let(:env) { { 'DISABLE_LOCK_RETRIES' => 'true' } }
+
+    it 'sets the migration class name in the logs' do
+      model.with_lock_retries(env: env, logger: in_memory_logger) { }
+
+      buffer.rewind
+      expect(buffer.read).to include("\"class\":\"#{model.class}\"")
+    end
+  end
 end

spec/lib/gitlab/database/with_lock_retries_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::Database::WithLockRetries do
+  let(:env) { {} }
+  let(:logger) { Gitlab::Database::WithLockRetries::NULL_LOGGER }
+  let(:subject) { described_class.new(env: env, logger: logger, timing_configuration: timing_configuration) }
+
+  let(:timing_configuration) do
+    [
+      [1.second, 1.second],
+      [1.second, 1.second],
+      [1.second, 1.second],
+      [1.second, 1.second],
+      [1.second, 1.second]
+    ]
+  end
+
+  describe '#run' do
+    it 'requires block' do
+      expect { subject.run }.to raise_error(StandardError, 'no block given')
+    end
+
+    context 'when DISABLE_LOCK_RETRIES is set' do
+      let(:env) { { 'DISABLE_LOCK_RETRIES' => 'true' } }
+
+      it 'executes the passed block without retrying' do
+        object = double
+
+        expect(object).to receive(:method).once
+
+        subject.run { object.method }
+      end
+    end
+
+    context 'when lock retry is enabled' do
+      class ActiveRecordSecond < ActiveRecord::Base
+      end
+
+      let(:lock_fiber) do
+        Fiber.new do
+          # Initiating a second DB connection for the lock
+          conn = ActiveRecordSecond.establish_connection(Rails.configuration.database_configuration[Rails.env]).connection
+          conn.transaction do
+            conn.execute("LOCK TABLE #{Project.table_name} in exclusive mode")
+
+            Fiber.yield
+          end
+          ActiveRecordSecond.remove_connection # force disconnect
+        end
+      end
+
+      before do
+        lock_fiber.resume # start the transaction and lock the table
+      end
+
+      context 'lock_fiber' do
+        it 'acquires lock successfully' do
+          check_exclusive_lock_query = """
+            SELECT 1
+            FROM pg_locks l
+            JOIN pg_class t ON l.relation = t.oid
+            WHERE t.relkind = 'r' AND l.mode = 'ExclusiveLock' AND t.relname = '#{Project.table_name}'
+          """
+
+          expect(ActiveRecord::Base.connection.execute(check_exclusive_lock_query).to_a).to be_present
+        end
+      end
+
+      shared_examples 'retriable exclusive lock on `projects`' do
+        it 'succeeds executing the given block' do
+          lock_attempts = 0
+          lock_acquired = false
+
+          expect_any_instance_of(Gitlab::Database::WithLockRetries).to receive(:sleep).exactly(retry_count - 1).times # we don't sleep in the last iteration
+
+          allow_any_instance_of(Gitlab::Database::WithLockRetries).to receive(:run_block_with_transaction).and_wrap_original do |method|
+            lock_fiber.resume if lock_attempts == retry_count
+
+            method.call
+          end
+
+          subject.run do
+            lock_attempts += 1
+
+            if lock_attempts == retry_count # we reached the last retry iteration, if we kill the thread, the last try (no lock_timeout) will succeed)
+              lock_fiber.resume
+            end
+
+            ActiveRecord::Base.transaction do
+              ActiveRecord::Base.connection.execute("LOCK TABLE #{Project.table_name} in exclusive mode")
+              lock_acquired = true
+            end
+          end
+
+          expect(lock_attempts).to eq(retry_count)
+          expect(lock_acquired).to eq(true)
+        end
+      end
+
+      context 'after 3 iterations' do
+        let(:retry_count) { 4 }
+
+        it_behaves_like 'retriable exclusive lock on `projects`'
+      end
+
+      context 'after the retries, without setting lock_timeout' do
+        let(:retry_count) { timing_configuration.size }
+
+        it_behaves_like 'retriable exclusive lock on `projects`'
+      end
+
+      context 'when statement timeout is reached' do
+        it 'raises QueryCanceled error' do
+          lock_acquired = false
+          ActiveRecord::Base.connection.execute("SET LOCAL statement_timeout='100ms'")
+
+          expect do
+            subject.run do
+              ActiveRecord::Base.connection.execute("SELECT 1 FROM pg_sleep(0.11)") # 110ms
+              lock_acquired = true
+            end
+          end.to raise_error(ActiveRecord::QueryCanceled)
+
+          expect(lock_acquired).to eq(false)
+        end
+      end
+    end
+  end
+end

spec/requests/api/graphql/project/merge_request_spec.rb

@@ -14,7 +14,7 @@ describe 'getting merge request information nested in a project' do
     graphql_query_for(
       'project',
       { 'fullPath' => project.full_path },
-      query_graphql_field('mergeRequest', iid: merge_request.iid)
+      query_graphql_field('mergeRequest', iid: merge_request.iid.to_s)
     )
   end
 

spec/requests/api/graphql/tasks/task_completion_status_spec.rb

@@ -25,7 +25,7 @@ describe 'getting task completion status information' do
     graphql_query_for(
       'project',
         { 'fullPath' => project.full_path },
-        query_graphql_field(type, { iid: iid }, fields)
+        query_graphql_field(type, { iid: iid.to_s }, fields)
     )
   end
 

spec/support/helpers/graphql_helpers.rb

@@ -16,6 +16,20 @@ module GraphqlHelpers
     resolver_class.new(object: obj, context: ctx).resolve(args)
   end
 
+  # Eagerly run a loader's named resolver
+  # (syncs any lazy values returned by resolve)
+  def eager_resolve(resolver_class, **opts)
+    sync(resolve(resolver_class, **opts))
+  end
+
+  def sync(value)
+    if GitlabSchema.lazy?(value)
+      GitlabSchema.sync_lazy(value)
+    else
+      value
+    end
+  end
+
   # Runs a block inside a BatchLoader::Executor wrapper
   def batch(max_queries: nil, &blk)
     wrapper = proc do
@@ -39,7 +53,7 @@ module GraphqlHelpers
   def batch_sync(max_queries: nil, &blk)
     wrapper = proc do
       lazy_vals = yield
-      lazy_vals.is_a?(Array) ? lazy_vals.map(&:sync) : lazy_vals&.sync
+      lazy_vals.is_a?(Array) ? lazy_vals.map { |val| sync(val) } : sync(lazy_vals)
     end
 
     batch(max_queries: max_queries, &wrapper)
@@ -164,16 +178,26 @@ module GraphqlHelpers
 
   def attributes_to_graphql(attributes)
     attributes.map do |name, value|
-      value_str = if value.is_a?(Array)
-                    '["' + value.join('","') + '"]'
-                  else
-                    "\"#{value}\""
-                  end
+      value_str = as_graphql_literal(value)
 
       "#{GraphqlHelpers.fieldnamerize(name.to_s)}: #{value_str}"
     end.join(", ")
   end
 
+  # Fairly dumb Ruby => GraphQL rendering function. Only suitable for testing.
+  # Missing support for Enums (feel free to add if you need it).
+  def as_graphql_literal(value)
+    case value
+    when Array then "[#{value.map { |v| as_graphql_literal(v) }.join(',')}]"
+    when Integer, Float then value.to_s
+    when String then "\"#{value.gsub(/"/, '\\"')}\""
+    when nil then 'null'
+    when true then 'true'
+    when false then 'false'
+    else raise ArgumentError, "Cannot represent #{value} as GraphQL literal"
+    end
+  end
+
   def post_multiplex(queries, current_user: nil, headers: {})
     post api('/', current_user, version: 'graphql'), params: { _json: queries }, headers: headers
   end
@@ -216,6 +240,11 @@ module GraphqlHelpers
     json_response['data'] || (raise NoData, graphql_errors)
   end
 
+  def graphql_data_at(*path)
+    keys = path.map { |segment| GraphqlHelpers.fieldnamerize(segment) }
+    graphql_data.dig(*keys)
+  end
+
   def graphql_errors
     case json_response
     when Hash # regular query