Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
G
gitlab-ce
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
1
Merge Requests
1
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
nexedi
gitlab-ce
Commits
c2cd7a8b
Commit
c2cd7a8b
authored
Sep 15, 2021
by
Dmitry Gruzd
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Revert "Merge branch '340478-fix-migration-worker' into 'master'"
This reverts merge request !69958
parent
da560e59
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
120 additions
and
136 deletions
+120
-136
ee/app/workers/elastic/migration_worker.rb
ee/app/workers/elastic/migration_worker.rb
+2
-2
ee/spec/workers/elastic/migration_worker_spec.rb
ee/spec/workers/elastic/migration_worker_spec.rb
+118
-134
No files found.
ee/app/workers/elastic/migration_worker.rb
View file @
c2cd7a8b
...
@@ -87,9 +87,9 @@ module Elastic
...
@@ -87,9 +87,9 @@ module Elastic
end
end
def
current_migration
def
current_migration
uncompleted_migrations
=
Elastic
::
MigrationRecord
.
load_versions
(
completed:
fals
e
)
completed_migrations
=
Elastic
::
MigrationRecord
.
load_versions
(
completed:
tru
e
)
Elastic
::
DataMigrationService
.
migrations
.
find
{
|
migration
|
un
completed_migrations
.
include?
(
migration
.
version
)
}
Elastic
::
DataMigrationService
.
migrations
.
find
{
|
migration
|
!
completed_migrations
.
include?
(
migration
.
version
)
}
end
end
def
pause_indexing!
(
migration
)
def
pause_indexing!
(
migration
)
...
...
ee/spec/workers/elastic/migration_worker_spec.rb
View file @
c2cd7a8b
...
@@ -22,184 +22,168 @@ RSpec.describe Elastic::MigrationWorker, :elastic do
...
@@ -22,184 +22,168 @@ RSpec.describe Elastic::MigrationWorker, :elastic do
before
do
before
do
stub_ee_application_setting
(
elasticsearch_indexing:
true
)
stub_ee_application_setting
(
elasticsearch_indexing:
true
)
allow
(
subject
).
to
receive
(
:current_migration
).
and_return
(
migration
)
end
end
context
'an unexecuted migration present'
do
it
'creates an index if it does not exist'
do
Gitlab
::
Elastic
::
Helper
.
default
.
delete_migrations_index
expect
{
subject
.
perform
}.
to
change
{
Gitlab
::
Elastic
::
Helper
.
default
.
migrations_index_exists?
}.
from
(
false
).
to
(
true
)
end
context
'no unexecuted migrations'
do
before
do
before
do
allow
(
subject
).
to
receive
(
:current_migration
).
and_return
(
migration
)
allow
(
subject
).
to
receive
(
:current_migration
).
and_return
(
nil
)
end
end
it
'
creates an index if it does not exist
'
do
it
'
skips execution
'
do
Gitlab
::
Elastic
::
Helper
.
default
.
delete_migrations_index
expect
(
subject
).
not_to
receive
(
:execute_migration
)
expect
{
subject
.
perform
}.
to
change
{
Gitlab
::
Elastic
::
Helper
.
default
.
migrations_index_exists?
}.
from
(
false
).
to
(
true
)
expect
(
subject
.
perform
).
to
be_falsey
end
end
end
context
'migration is halted'
do
context
'migration is halted'
do
using
RSpec
::
Parameterized
::
TableSyntax
using
RSpec
::
Parameterized
::
TableSyntax
where
(
:pause_indexing
,
:halted_indexing_unpaused
,
:unpause
)
do
where
(
:pause_indexing
,
:halted_indexing_unpaused
,
:unpause
)
do
false
|
false
|
false
false
|
false
|
false
false
|
true
|
false
false
|
true
|
false
true
|
false
|
true
true
|
false
|
true
true
|
true
|
false
true
|
true
|
false
end
with_them
do
before
do
allow
(
Gitlab
::
CurrentSettings
).
to
receive
(
:elasticsearch_pause_indexing?
).
and_return
(
true
)
allow
(
migration
).
to
receive
(
:pause_indexing?
).
and_return
(
true
)
migration
.
save_state!
(
halted:
true
,
pause_indexing:
pause_indexing
,
halted_indexing_unpaused:
halted_indexing_unpaused
)
end
end
with_them
do
it
'unpauses indexing'
do
before
do
if
unpause
allow
(
Gitlab
::
CurrentSettings
).
to
receive
(
:elasticsearch_pause_indexing?
).
and_return
(
tru
e
)
expect
(
Gitlab
::
CurrentSettings
).
to
receive
(
:update!
).
with
(
elasticsearch_pause_indexing:
fals
e
)
allow
(
migration
).
to
receive
(
:pause_indexing?
).
and_return
(
true
)
else
migration
.
save_state!
(
halted:
true
,
pause_indexing:
pause_indexing
,
halted_indexing_unpaused:
halted_indexing_unpaused
)
expect
(
Gitlab
::
CurrentSettings
).
not_to
receive
(
:update!
)
end
end
it
'unpauses indexing'
do
expect
(
migration
).
not_to
receive
(
:migrate
)
if
unpause
expect
(
Gitlab
::
CurrentSettings
).
to
receive
(
:update!
).
with
(
elasticsearch_pause_indexing:
false
)
else
expect
(
Gitlab
::
CurrentSettings
).
not_to
receive
(
:update!
)
end
expect
(
migration
).
not_to
receive
(
:migrate
)
subject
.
perform
subject
.
perform
end
end
end
end
end
end
context
'migration process'
do
context
'migration process'
do
before
do
before
do
allow
(
migration
).
to
receive
(
:started?
).
and_return
(
started
)
allow
(
migration
).
to
receive
(
:started?
).
and_return
(
started
)
allow
(
migration
).
to
receive
(
:completed?
).
and_return
(
completed
)
allow
(
migration
).
to
receive
(
:completed?
).
and_return
(
completed
)
allow
(
migration
).
to
receive
(
:batched?
).
and_return
(
batched
)
allow
(
migration
).
to
receive
(
:batched?
).
and_return
(
batched
)
end
end
using
RSpec
::
Parameterized
::
TableSyntax
# completed is evaluated after migrate method is executed
where
(
:started
,
:completed
,
:execute_migration
,
:batched
)
do
false
|
false
|
true
|
false
false
|
true
|
true
|
false
false
|
false
|
true
|
true
false
|
true
|
true
|
true
true
|
false
|
false
|
false
true
|
true
|
false
|
false
true
|
false
|
true
|
true
true
|
true
|
true
|
true
end
with_them
do
it
'calls migration only when needed'
,
:aggregate_failures
do
if
execute_migration
expect
(
migration
).
to
receive
(
:migrate
).
once
else
expect
(
migration
).
not_to
receive
(
:migrate
)
end
expect
(
migration
).
to
receive
(
:save!
).
with
(
completed:
completed
)
using
RSpec
::
Parameterized
::
TableSyntax
expect
(
Elastic
::
DataMigrationService
).
to
receive
(
:drop_migration_has_finished_cache!
).
with
(
migration
)
# completed is evaluated after migrate method is executed
where
(
:started
,
:completed
,
:execute_migration
,
:batched
)
do
false
|
false
|
true
|
false
false
|
true
|
true
|
false
false
|
false
|
true
|
true
false
|
true
|
true
|
true
true
|
false
|
false
|
false
true
|
true
|
false
|
false
true
|
false
|
true
|
true
true
|
true
|
true
|
true
end
subject
.
perform
with_them
do
it
'calls migration only when needed'
,
:aggregate_failures
do
if
execute_migration
expect
(
migration
).
to
receive
(
:migrate
).
once
else
expect
(
migration
).
not_to
receive
(
:migrate
)
end
end
it
'handles batched migrations'
do
expect
(
migration
).
to
receive
(
:save!
).
with
(
completed:
completed
)
if
batched
&&
!
completed
expect
(
Elastic
::
DataMigrationService
).
to
receive
(
:drop_migration_has_finished_cache!
).
with
(
migration
)
# default throttle_delay is 5.minutes
expect
(
Elastic
::
MigrationWorker
).
to
receive
(
:perform_in
)
.
with
(
5
.
minutes
)
else
expect
(
Elastic
::
MigrationWorker
).
not_to
receive
(
:perform_in
)
end
subject
.
perform
subject
.
perform
end
end
end
context
'indexing pause'
do
it
'handles batched migrations'
do
before
do
if
batched
&&
!
completed
allow
(
migration
).
to
receive
(
:pause_indexing?
).
and_return
(
true
)
# default throttle_delay is 5.minutes
expect
(
Elastic
::
MigrationWorker
).
to
receive
(
:perform_in
)
.
with
(
5
.
minutes
)
else
expect
(
Elastic
::
MigrationWorker
).
not_to
receive
(
:perform_in
)
end
end
let
(
:batched
)
{
true
}
subject
.
perform
end
where
(
:started
,
:completed
,
:expected
)
do
end
false
|
false
|
false
true
|
false
|
false
true
|
true
|
true
end
with_them
do
it
'pauses and unpauses indexing'
do
expect
(
Gitlab
::
CurrentSettings
).
to
receive
(
:update!
).
with
(
elasticsearch_pause_indexing:
true
)
expect
(
Gitlab
::
CurrentSettings
).
to
receive
(
:update!
).
with
(
elasticsearch_pause_indexing:
false
)
if
expected
subject
.
perform
context
'indexing pause'
do
end
before
do
end
allow
(
migration
).
to
receive
(
:pause_indexing?
).
and_return
(
true
)
end
end
context
'checks space required'
do
let
(
:batched
)
{
true
}
let
(
:helper
)
{
Gitlab
::
Elastic
::
Helper
.
new
}
let
(
:started
)
{
false
}
let
(
:completed
)
{
false
}
let
(
:batched
)
{
false
}
before
do
where
(
:started
,
:completed
,
:expected
)
do
allow
(
Gitlab
::
Elastic
::
Helper
).
to
receive
(
:default
).
and_return
(
helper
)
false
|
false
|
false
allow
(
migration
).
to
receive
(
:space_requirements?
).
and_return
(
true
)
true
|
false
|
false
allow
(
migration
).
to
receive
(
:space_required_bytes
).
and_return
(
10
)
true
|
true
|
true
end
end
it
'halts the migration if there is not enough space'
do
with_them
do
allow
(
helper
).
to
receive
(
:cluster_free_size_bytes
).
and_return
(
5
)
it
'pauses and unpauses indexing'
do
expect
(
migration
).
to
receive
(
:halt!
)
expect
(
Gitlab
::
CurrentSettings
).
to
receive
(
:update!
).
with
(
elasticsearch_pause_indexing:
true
)
expect
(
migration
).
not_to
receive
(
:migrate
)
expect
(
Gitlab
::
CurrentSettings
).
to
receive
(
:update!
).
with
(
elasticsearch_pause_indexing:
false
)
if
expected
subject
.
perform
subject
.
perform
end
end
end
end
it
'runs the migration if there is enough space'
do
context
'checks space required'
do
allow
(
helper
).
to
receive
(
:cluster_free_size_bytes
).
and_return
(
20
)
let
(
:helper
)
{
Gitlab
::
Elastic
::
Helper
.
new
}
expect
(
migration
).
not_to
receive
(
:halt!
)
let
(
:started
)
{
false
}
expect
(
migration
).
to
receive
(
:migrate
).
once
let
(
:completed
)
{
false
}
let
(
:batched
)
{
false
}
subject
.
perform
end
context
'when migration is already started'
do
before
do
let
(
:started
)
{
true
}
allow
(
Gitlab
::
Elastic
::
Helper
).
to
receive
(
:default
).
and_return
(
helper
)
allow
(
migration
).
to
receive
(
:space_requirements?
).
and_return
(
true
)
allow
(
migration
).
to
receive
(
:space_required_bytes
).
and_return
(
10
)
end
it
'does not check space requirements'
do
it
'halts the migration if there is not enough space'
do
expect
(
helper
).
not_to
receive
(
:cluster_free_size_bytes
)
allow
(
helper
).
to
receive
(
:cluster_free_size_bytes
).
and_return
(
5
)
expect
(
migration
).
not_to
receive
(
:space_required_bytes
)
expect
(
migration
).
to
receive
(
:halt!
)
expect
(
migration
).
not_to
receive
(
:migrate
)
subject
.
perform
subject
.
perform
end
end
end
end
end
end
context
'no unexecuted migrations'
do
it
'runs the migration if there is enough space'
do
before
do
allow
(
helper
).
to
receive
(
:cluster_free_size_bytes
).
and_return
(
20
)
allow
(
subject
).
to
receive
(
:current_migration
).
and_return
(
nil
)
expect
(
migration
).
not_to
receive
(
:halt!
)
end
expect
(
migration
).
to
receive
(
:migrate
).
once
it
'skips execution'
do
expect
(
subject
).
not_to
receive
(
:execute_migration
)
expect
(
subject
.
perform
).
to
be_falsey
subject
.
perform
end
end
end
context
'load_versions returns empty array'
do
context
'when migration is already started'
do
before
do
let
(
:started
)
{
true
}
allow
(
Elastic
::
MigrationRecord
).
to
receive
(
:load_versions
).
and_return
([])
end
it
'skips execution'
do
it
'does not check space requirements'
do
expect
(
subject
).
not_to
receive
(
:execute_migration
)
expect
(
helper
).
not_to
receive
(
:cluster_free_size_bytes
)
expect
(
migration
).
not_to
receive
(
:space_required_bytes
)
expect
(
subject
.
perform
).
to
be_falsey
subject
.
perform
end
end
end
end
end
end
end
end
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment