Commit 3fa2eb4e authored by Andreas Brandl's avatar Andreas Brandl

Refactor, extract class and improve comments.

parent 0360b092
# Include atomic internal id generation scheme for a model
#
# This allows to atomically generate internal ids that are
# unique within a given scope.
#
# For example, let's generate internal ids for Issue per Project:
# ```
# class Issue < ActiveRecord::Base
# has_internal_id :iid, scope: :project, init: ->(s) { s.project.issues.maximum(:iid) }
# end
# ```
#
# This generates unique internal ids per project for newly created issues.
# The generated internal id is saved in the `iid` attribute of `Issue`.
#
# This concern uses InternalId records to facilitate atomicity.
# In the absence of a record for the given scope, one will be created automatically.
# In this situation, the `init` block is called to calculate the initial value.
# In the example above, we calculate the maximum `iid` of all issues
# within the given project.
#
# Note that a model may have more than one internal id associated with possibly
# different scopes.
module AtomicInternalId module AtomicInternalId
extend ActiveSupport::Concern extend ActiveSupport::Concern
included do included do
class << self class << self
def has_internal_id(on, scope:, usage: nil, init: nil) def has_internal_id(on, scope:, usage: nil, init: nil) # rubocop:disable Naming/PredicateName
before_validation(on: :create) do before_validation(on: :create) do
if self.public_send(on).blank? # rubocop:disable GitlabSecurity/PublicSend if self.public_send(on).blank? # rubocop:disable GitlabSecurity/PublicSend
scope_attrs = [scope].flatten.compact.each_with_object({}) do |e, h|
h[e] = self.public_send(e) # rubocop:disable GitlabSecurity/PublicSend
end
usage = (usage || self.class.name.tableize).to_sym usage = (usage || self.class.name.tableize).to_sym
new_iid = InternalId.generate_next(self, scope, usage, init) new_iid = InternalId.generate_next(self, scope_attrs, usage, init)
self.public_send("#{on}=", new_iid) # rubocop:disable GitlabSecurity/PublicSend self.public_send("#{on}=", new_iid) # rubocop:disable GitlabSecurity/PublicSend
end end
end end
......
# An InternalId is a strictly monotone sequence of integers # An InternalId is a strictly monotone sequence of integers
# for a given project and usage (e.g. issues). # generated for a given scope and usage.
# #
# For possible usages, see InternalId#usage enum. # For example, issues use their project to scope internal ids:
# In that sense, scope is "project" and usage is "issues".
# Generated internal ids for an issue are unique per project.
#
# See InternalId#usage enum for available usages.
#
# In order to leverage InternalId for other usages, the idea is to
# * Add `usage` value to enum
# * (Optionally) add columns to `internal_ids` if needed for scope.
class InternalId < ActiveRecord::Base class InternalId < ActiveRecord::Base
belongs_to :project belongs_to :project
enum usage: { issues: 0 } enum usage: { issues: 0 }
validates :usage, presence: true validates :usage, presence: true
validates :project_id, presence: true
# Increments #last_value and saves the record # Increments #last_value and saves the record
# #
# The operation locks the record and gathers # The operation locks the record and gathers a `ROW SHARE` lock (in PostgreSQL).
# a `ROW SHARE` lock (in PostgreSQL). As such, # As such, the increment is atomic and safe to be called concurrently.
# the increment is atomic and safe to be called
# concurrently.
def increment_and_save! def increment_and_save!
lock! lock!
self.last_value = (last_value || 0) + 1 self.last_value = (last_value || 0) + 1
...@@ -24,59 +29,79 @@ class InternalId < ActiveRecord::Base ...@@ -24,59 +29,79 @@ class InternalId < ActiveRecord::Base
end end
class << self class << self
# Generate next internal id for a given project and usage. def generate_next(subject, scope, usage, init)
InternalIdGenerator.new(subject, scope, usage, init).generate
end
end
class InternalIdGenerator
# Generate next internal id for a given scope and usage.
# #
# For currently supported usages, see #usage enum. # For currently supported usages, see #usage enum.
# #
# The method implements a locking scheme that has the following properties: # The method implements a locking scheme that has the following properties:
# 1) Generated sequence of internal ids is unique per (project, usage) # 1) Generated sequence of internal ids is unique per (scope and usage)
# 2) The method is thread-safe and may be used in concurrent threads/processes. # 2) The method is thread-safe and may be used in concurrent threads/processes.
# 3) The generated sequence is gapless. # 3) The generated sequence is gapless.
# 4) In the absence of a record in the internal_ids table, one will be created # 4) In the absence of a record in the internal_ids table, one will be created
# and last_value will be calculated on the fly. # and last_value will be calculated on the fly.
def generate_next(subject, scope, usage, init) #
scope = [scope].flatten.compact # subject: The instance we're generating an internal id for. Gets passed to init if called.
raise 'scope is not well-defined, need at least one column for scope (given: 0)' if scope.empty? # scope: Attributes that define the scope for id generation.
raise "usage #{usage} is unknown. Supported values are InternalId.usages = #{InternalId.usages.keys.to_s}" unless InternalId.usages.include?(usage.to_sym) # usage: Symbol to define the usage of the internal id, see InternalId.usages
# init: Block that gets called to initialize InternalId record if not yet present (optional)
attr_reader :subject, :scope, :init, :scope_attrs, :usage
def initialize(subject, scope, usage, init)
@subject = subject
@scope = scope
@init = init || ->(s) { 0 }
@usage = usage
init ||= ->(s) { 0 } raise 'scope is not well-defined, need at least one column for scope (given: 0)' if scope.empty?
scope_attrs = scope.inject({}) do |h, e| unless InternalId.usages.keys.include?(usage.to_s)
h[e] = subject.public_send(e) raise "Usage '#{usage}' is unknown. Supported values are #{InternalId.usages.keys} from InternalId.usages"
h
end end
end
transaction do # Generates next internal id and returns it
def generate
subject.transaction do
# Create a record in internal_ids if one does not yet exist # Create a record in internal_ids if one does not yet exist
id = (lookup(scope_attrs, usage) || create_record(scope_attrs, usage, init, subject)) # and increment it's last value
#
# This will lock the InternalId record with ROW SHARE # Note this will acquire a ROW SHARE lock on the InternalId record
# and increment #last_value (lookup || create_record).increment_and_save!
id.increment_and_save!
end end
end end
private private
# Retrieve InternalId record for (project, usage) combination, if it exists # Retrieve InternalId record for (project, usage) combination, if it exists
def lookup(scope_attrs, usage) def lookup
InternalId.find_by(usage: usages[usage.to_s], **scope_attrs) InternalId.find_by(**scope, usage: usage_value)
end
def usage_value
@usage_value ||= InternalId.usages[usage.to_s]
end end
# Create InternalId record for (project, usage) combination, if it doesn't exist # Create InternalId record for (scope, usage) combination, if it doesn't exist
# #
# We blindly insert without any synchronization. If another process # We blindly insert without synchronization. If another process
# was faster in doing this, we'll realize once we hit the unique key constraint # was faster in doing this, we'll realize once we hit the unique key constraint
# violation. We can safely roll-back the nested transaction and perform # violation. We can safely roll-back the nested transaction and perform
# a lookup instead to retrieve the record. # a lookup instead to retrieve the record.
def create_record(scope_attrs, usage, init, subject) def create_record
begin subject.transaction(requires_new: true) do
transaction(requires_new: true) do InternalId.create!(
create!(usage: usages[usage.to_s], **scope_attrs, last_value: init.call(subject) || 0) **scope,
end usage: usage_value,
rescue ActiveRecord::RecordNotUnique last_value: init.call(subject) || 0
lookup(scope_attrs, usage) )
end end
rescue ActiveRecord::RecordNotUnique
lookup
end end
end end
end end
...@@ -24,7 +24,7 @@ class Issue < ActiveRecord::Base ...@@ -24,7 +24,7 @@ class Issue < ActiveRecord::Base
belongs_to :project belongs_to :project
belongs_to :moved_to, class_name: 'Issue' belongs_to :moved_to, class_name: 'Issue'
has_internal_id :iid, scope: :project, init: ->(s) { s.project.issues.maximum(:iid) } has_internal_id :iid, scope: :project, init: ->(s) { s&.project&.issues&.maximum(:iid) }
has_many :events, as: :target, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_many :events, as: :target, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
......
...@@ -34,7 +34,7 @@ describe Issuable do ...@@ -34,7 +34,7 @@ describe Issuable do
subject { build(:issue) } subject { build(:issue) }
before do before do
allow(subject).to receive(:set_iid).and_return(false) allow(InternalId).to receive(:generate_next).and_return(nil)
end end
it { is_expected.to validate_presence_of(:project) } it { is_expected.to validate_presence_of(:project) }
......
...@@ -4,12 +4,11 @@ describe InternalId do ...@@ -4,12 +4,11 @@ describe InternalId do
let(:project) { create(:project) } let(:project) { create(:project) }
let(:usage) { :issues } let(:usage) { :issues }
let(:issue) { build(:issue, project: project) } let(:issue) { build(:issue, project: project) }
let(:scope) { :project } let(:scope) { { project: project } }
let(:init) { ->(s) { project.issues.size } } let(:init) { ->(s) { s.project.issues.size } }
context 'validations' do context 'validations' do
it { is_expected.to validate_presence_of(:usage) } it { is_expected.to validate_presence_of(:usage) }
it { is_expected.to validate_presence_of(:project_id) }
end end
describe '.generate_next' do describe '.generate_next' do
...@@ -31,8 +30,8 @@ describe InternalId do ...@@ -31,8 +30,8 @@ describe InternalId do
context 'with existing issues' do context 'with existing issues' do
before do before do
rand(10).times { create(:issue, project: project) } rand(1..10).times { create(:issue, project: project) }
InternalId.delete_all described_class.delete_all
end end
it 'calculates last_value values automatically' do it 'calculates last_value values automatically' do
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment