Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
G
gitlab-ce
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
1
Merge Requests
1
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
nexedi
gitlab-ce
Commits
c38c23d1
Commit
c38c23d1
authored
Jun 26, 2018
by
Toon Claes
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Take Gitlab::ShardHealthCache out the Geo namespace
parent
7bba90cd
Changes
17
Hide whitespace changes
Inline
Side-by-side
Showing
17 changed files
with
63 additions
and
66 deletions
+63
-66
app/workers/concerns/each_shard_worker.rb
app/workers/concerns/each_shard_worker.rb
+2
-0
ee/app/workers/geo/project_sync_worker.rb
ee/app/workers/geo/project_sync_worker.rb
+1
-1
ee/app/workers/geo/repository_shard_sync_worker.rb
ee/app/workers/geo/repository_shard_sync_worker.rb
+2
-2
ee/app/workers/geo/repository_verification/primary/shard_worker.rb
...rkers/geo/repository_verification/primary/shard_worker.rb
+1
-1
ee/app/workers/geo/repository_verification/secondary/shard_worker.rb
...ers/geo/repository_verification/secondary/shard_worker.rb
+1
-1
ee/app/workers/geo/scheduler/per_shard_scheduler_worker.rb
ee/app/workers/geo/scheduler/per_shard_scheduler_worker.rb
+0
-2
ee/lib/gitlab/geo/log_cursor/events/base_event.rb
ee/lib/gitlab/geo/log_cursor/events/base_event.rb
+1
-1
ee/lib/gitlab/geo/shard_health_cache.rb
ee/lib/gitlab/geo/shard_health_cache.rb
+0
-43
ee/spec/lib/gitlab/geo/log_cursor/daemon_spec.rb
ee/spec/lib/gitlab/geo/log_cursor/daemon_spec.rb
+1
-1
ee/spec/lib/gitlab/geo/log_cursor/events/repository_created_event_spec.rb
...ab/geo/log_cursor/events/repository_created_event_spec.rb
+1
-1
ee/spec/lib/gitlab/geo/log_cursor/events/repository_updated_event_spec.rb
...ab/geo/log_cursor/events/repository_updated_event_spec.rb
+2
-2
ee/spec/workers/geo/project_sync_worker_spec.rb
ee/spec/workers/geo/project_sync_worker_spec.rb
+2
-2
ee/spec/workers/geo/repository_shard_sync_worker_spec.rb
ee/spec/workers/geo/repository_shard_sync_worker_spec.rb
+3
-4
ee/spec/workers/geo/repository_verification/primary/shard_worker_spec.rb
.../geo/repository_verification/primary/shard_worker_spec.rb
+2
-2
ee/spec/workers/geo/repository_verification/secondary/shard_worker_spec.rb
...eo/repository_verification/secondary/shard_worker_spec.rb
+2
-2
lib/gitlab/shard_health_cache.rb
lib/gitlab/shard_health_cache.rb
+41
-0
spec/lib/gitlab/shard_health_cache_spec.rb
spec/lib/gitlab/shard_health_cache_spec.rb
+1
-1
No files found.
app/workers/concerns/each_shard_worker.rb
View file @
c38c23d1
...
...
@@ -7,6 +7,8 @@ module EachShardWorker
].
freeze
def
each_shard
Gitlab
::
ShardHealthCache
.
update
(
eligible_shard_names
)
eligible_shard_names
.
each
do
|
shard_name
|
yield
shard_name
end
...
...
ee/app/workers/geo/project_sync_worker.rb
View file @
c38c23d1
...
...
@@ -22,7 +22,7 @@ module Geo
end
shard_name
=
project
.
repository_storage
unless
Gitlab
::
Geo
::
ShardHealthCache
.
healthy_shard?
(
shard_name
)
unless
Gitlab
::
ShardHealthCache
.
healthy_shard?
(
shard_name
)
log_error
(
"Project shard '
#{
shard_name
}
' is unhealthy, skipping syncing"
,
project_id:
project_id
)
return
end
...
...
ee/app/workers/geo/repository_shard_sync_worker.rb
View file @
c38c23d1
...
...
@@ -7,7 +7,7 @@ module Geo
def
perform
(
shard_name
)
@shard_name
=
shard_name
return
unless
Gitlab
::
Geo
::
ShardHealthCache
.
healthy_shard?
(
shard_name
)
return
unless
Gitlab
::
ShardHealthCache
.
healthy_shard?
(
shard_name
)
super
()
end
...
...
@@ -24,7 +24,7 @@ module Geo
end
def
max_capacity
healthy_count
=
Gitlab
::
Geo
::
ShardHealthCache
.
healthy_shard_count
healthy_count
=
Gitlab
::
ShardHealthCache
.
healthy_shard_count
# If we don't have a count, that means that for some reason
# RepositorySyncWorker stopped running/updating the cache. We might
...
...
ee/app/workers/geo/repository_verification/primary/shard_worker.rb
View file @
c38c23d1
...
...
@@ -9,7 +9,7 @@ module Geo
def
perform
(
shard_name
)
@shard_name
=
shard_name
return
unless
Gitlab
::
Geo
::
ShardHealthCache
.
healthy_shard?
(
shard_name
)
return
unless
Gitlab
::
ShardHealthCache
.
healthy_shard?
(
shard_name
)
super
()
end
...
...
ee/app/workers/geo/repository_verification/secondary/shard_worker.rb
View file @
c38c23d1
...
...
@@ -9,7 +9,7 @@ module Geo
def
perform
(
shard_name
)
@shard_name
=
shard_name
return
unless
Gitlab
::
Geo
::
ShardHealthCache
.
healthy_shard?
(
shard_name
)
return
unless
Gitlab
::
ShardHealthCache
.
healthy_shard?
(
shard_name
)
super
()
end
...
...
ee/app/workers/geo/scheduler/per_shard_scheduler_worker.rb
View file @
c38c23d1
...
...
@@ -7,8 +7,6 @@ module Geo
include
::
EachShardWorker
def
perform
Gitlab
::
Geo
::
ShardHealthCache
.
update
(
eligible_shard_names
)
each_shard
{
|
shard_name
|
schedule_job
(
shard_name
)
}
end
...
...
ee/lib/gitlab/geo/log_cursor/events/base_event.rb
View file @
c38c23d1
...
...
@@ -26,7 +26,7 @@ module Gitlab
def
healthy_shard_for?
(
event
)
return
true
unless
event
.
respond_to?
(
:project
)
Gitlab
::
Geo
::
ShardHealthCache
.
healthy_shard?
(
event
.
project
.
repository_storage
)
Gitlab
::
ShardHealthCache
.
healthy_shard?
(
event
.
project
.
repository_storage
)
end
def
enqueue_job_if_shard_healthy
(
event
)
...
...
ee/lib/gitlab/geo/shard_health_cache.rb
deleted
100644 → 0
View file @
7bba90cd
module
Gitlab
module
Geo
class
ShardHealthCache
HEALTHY_SHARDS_KEY
=
'gitlab-geo-healthy-shards'
.
freeze
HEALTHY_SHARDS_TIMEOUT
=
300
# Clears the Redis set storing the list of healthy shards
def
self
.
clear
Gitlab
::
Redis
::
Cache
.
with
{
|
redis
|
redis
.
del
(
HEALTHY_SHARDS_KEY
)
}
end
# Updates the list of healthy shards using a Redis set
#
# shards - An array of shard names to store
def
self
.
update
(
shards
)
Gitlab
::
Redis
::
Cache
.
with
do
|
redis
|
redis
.
multi
do
|
m
|
m
.
del
(
HEALTHY_SHARDS_KEY
)
shards
.
each
{
|
shard_name
|
m
.
sadd
(
HEALTHY_SHARDS_KEY
,
shard_name
)
}
m
.
expire
(
HEALTHY_SHARDS_KEY
,
HEALTHY_SHARDS_TIMEOUT
)
end
end
end
# Returns an array of strings of healthy shards
def
self
.
cached_healthy_shards
Gitlab
::
Redis
::
Cache
.
with
{
|
redis
|
redis
.
smembers
(
HEALTHY_SHARDS_KEY
)
}
end
# Checks whether the given shard name is in the list of healthy shards.
#
# shard_name - The string to check
def
self
.
healthy_shard?
(
shard_name
)
Gitlab
::
Redis
::
Cache
.
with
{
|
redis
|
redis
.
sismember
(
HEALTHY_SHARDS_KEY
,
shard_name
)
}
end
# Returns the number of healthy shards in the Redis set
def
self
.
healthy_shard_count
Gitlab
::
Redis
::
Cache
.
with
{
|
redis
|
redis
.
scard
(
HEALTHY_SHARDS_KEY
)
}
end
end
end
end
ee/spec/lib/gitlab/geo/log_cursor/daemon_spec.rb
View file @
c38c23d1
...
...
@@ -90,7 +90,7 @@ describe Gitlab::Geo::LogCursor::Daemon, :postgresql, :clean_gitlab_redis_shared
let!
(
:registry
)
{
create
(
:geo_project_registry
,
:synced
,
project:
project
)
}
before
do
allow
(
Gitlab
::
Geo
::
ShardHealthCache
).
to
receive
(
:healthy_shard?
).
with
(
'default'
).
and_return
(
true
)
allow
(
Gitlab
::
ShardHealthCache
).
to
receive
(
:healthy_shard?
).
with
(
'default'
).
and_return
(
true
)
end
it
'replays events for projects that belong to selected namespaces to replicate'
do
...
...
ee/spec/lib/gitlab/geo/log_cursor/events/repository_created_event_spec.rb
View file @
c38c23d1
...
...
@@ -37,7 +37,7 @@ describe Gitlab::Geo::LogCursor::Events::RepositoryCreatedEvent, :postgresql, :c
describe
'#process'
do
before
do
allow
(
Gitlab
::
Geo
::
ShardHealthCache
).
to
receive
(
:healthy_shard?
).
with
(
'default'
).
and_return
(
healthy
)
allow
(
Gitlab
::
ShardHealthCache
).
to
receive
(
:healthy_shard?
).
with
(
'default'
).
and_return
(
healthy
)
end
context
'when the associated shard is healthy'
do
...
...
ee/spec/lib/gitlab/geo/log_cursor/events/repository_updated_event_spec.rb
View file @
c38c23d1
...
...
@@ -18,7 +18,7 @@ describe Gitlab::Geo::LogCursor::Events::RepositoryUpdatedEvent, :postgresql, :c
before
do
stub_current_geo_node
(
secondary
)
allow
(
Gitlab
::
Geo
::
ShardHealthCache
).
to
receive
(
:healthy_shard?
).
with
(
'broken'
).
and_return
(
false
)
allow
(
Gitlab
::
ShardHealthCache
).
to
receive
(
:healthy_shard?
).
with
(
'broken'
).
and_return
(
false
)
end
RSpec
.
shared_examples
'RepositoryUpdatedEvent'
do
...
...
@@ -81,7 +81,7 @@ describe Gitlab::Geo::LogCursor::Events::RepositoryUpdatedEvent, :postgresql, :c
let
(
:now
)
{
Time
.
now
}
before
do
allow
(
Gitlab
::
Geo
::
ShardHealthCache
).
to
receive
(
:healthy_shard?
).
with
(
'default'
).
and_return
(
healthy
)
allow
(
Gitlab
::
ShardHealthCache
).
to
receive
(
:healthy_shard?
).
with
(
'default'
).
and_return
(
healthy
)
end
context
'when the associated shard is healthy'
do
...
...
ee/spec/workers/geo/project_sync_worker_spec.rb
View file @
c38c23d1
...
...
@@ -8,10 +8,10 @@ RSpec.describe Geo::ProjectSyncWorker do
let
(
:wiki_sync_service
)
{
spy
}
before
do
allow
(
Gitlab
::
Geo
::
ShardHealthCache
).
to
receive
(
:healthy_shard?
)
allow
(
Gitlab
::
ShardHealthCache
).
to
receive
(
:healthy_shard?
)
.
with
(
project
.
repository_storage
).
once
.
and_return
(
true
)
allow
(
Gitlab
::
Geo
::
ShardHealthCache
).
to
receive
(
:healthy_shard?
)
allow
(
Gitlab
::
ShardHealthCache
).
to
receive
(
:healthy_shard?
)
.
with
(
project_with_broken_storage
.
repository_storage
).
once
.
and_return
(
false
)
allow
(
Geo
::
RepositorySyncService
).
to
receive
(
:new
)
...
...
ee/spec/workers/geo/repository_shard_sync_worker_spec.rb
View file @
c38c23d1
...
...
@@ -28,7 +28,7 @@ describe Geo::RepositoryShardSyncWorker, :geo, :delete, :clean_gitlab_redis_cach
allow_any_instance_of
(
Gitlab
::
ExclusiveLease
).
to
receive
(
:try_obtain
)
{
true
}
allow_any_instance_of
(
Gitlab
::
ExclusiveLease
).
to
receive
(
:renew
)
{
true
}
Gitlab
::
Geo
::
ShardHealthCache
.
update
([
shard_name
])
Gitlab
::
ShardHealthCache
.
update
([
shard_name
])
end
it
'performs Geo::ProjectSyncWorker for each project'
do
...
...
@@ -47,8 +47,7 @@ describe Geo::RepositoryShardSyncWorker, :geo, :delete, :clean_gitlab_redis_cach
end
it
'does not perform Geo::ProjectSyncWorker when shard becomes unhealthy'
do
Gitlab
::
Geo
::
ShardHealthCache
.
update
([])
Gitlab
::
ShardHealthCache
.
update
([])
expect
(
Geo
::
ProjectSyncWorker
).
not_to
receive
(
:perform_async
)
subject
.
perform
(
shard_name
)
...
...
@@ -108,7 +107,7 @@ describe Geo::RepositoryShardSyncWorker, :geo, :delete, :clean_gitlab_redis_cach
it
'uses two loops to schedule jobs'
do
expect
(
subject
).
to
receive
(
:schedule_jobs
).
twice
.
and_call_original
Gitlab
::
Geo
::
ShardHealthCache
.
update
([
shard_name
,
'shard2'
,
'shard3'
,
'shard4'
,
'shard5'
])
Gitlab
::
ShardHealthCache
.
update
([
shard_name
,
'shard2'
,
'shard3'
,
'shard4'
,
'shard5'
])
secondary
.
update!
(
repos_max_capacity:
5
)
subject
.
perform
(
shard_name
)
...
...
ee/spec/workers/geo/repository_verification/primary/shard_worker_spec.rb
View file @
c38c23d1
...
...
@@ -16,7 +16,7 @@ describe Geo::RepositoryVerification::Primary::ShardWorker, :postgresql, :clean_
allow_any_instance_of
(
Gitlab
::
ExclusiveLease
).
to
receive
(
:try_obtain
)
{
true
}
allow_any_instance_of
(
Gitlab
::
ExclusiveLease
).
to
receive
(
:renew
)
{
true
}
Gitlab
::
Geo
::
ShardHealthCache
.
update
([
shard_name
])
Gitlab
::
ShardHealthCache
.
update
([
shard_name
])
end
it
'performs Geo::RepositoryVerification::Primary::SingleWorker for each project'
do
...
...
@@ -66,7 +66,7 @@ describe Geo::RepositoryVerification::Primary::ShardWorker, :postgresql, :clean_
it
'does not perform Geo::RepositoryVerification::Primary::SingleWorker when shard becomes unhealthy'
do
create
(
:project
)
Gitlab
::
Geo
::
ShardHealthCache
.
update
([])
Gitlab
::
ShardHealthCache
.
update
([])
expect
(
primary_singleworker
).
not_to
receive
(
:perform_async
)
...
...
ee/spec/workers/geo/repository_verification/secondary/shard_worker_spec.rb
View file @
c38c23d1
...
...
@@ -19,7 +19,7 @@ describe Geo::RepositoryVerification::Secondary::ShardWorker, :postgresql, :clea
allow_any_instance_of
(
Gitlab
::
ExclusiveLease
).
to
receive
(
:try_obtain
)
{
true
}
allow_any_instance_of
(
Gitlab
::
ExclusiveLease
).
to
receive
(
:renew
)
{
true
}
Gitlab
::
Geo
::
ShardHealthCache
.
update
([
shard_name
])
Gitlab
::
ShardHealthCache
.
update
([
shard_name
])
end
it
'schedules job for each project'
do
...
...
@@ -116,7 +116,7 @@ describe Geo::RepositoryVerification::Secondary::ShardWorker, :postgresql, :clea
it
'does not schedule jobs when shard becomes unhealthy'
do
create
(
:repository_state
,
project:
project
)
Gitlab
::
Geo
::
ShardHealthCache
.
update
([])
Gitlab
::
ShardHealthCache
.
update
([])
expect
(
secondary_singleworker
).
not_to
receive
(
:perform_async
)
...
...
lib/gitlab/shard_health_cache.rb
0 → 100644
View file @
c38c23d1
module
Gitlab
class
ShardHealthCache
HEALTHY_SHARDS_KEY
=
'gitlab-healthy-shards'
.
freeze
HEALTHY_SHARDS_TIMEOUT
=
300
# Clears the Redis set storing the list of healthy shards
def
self
.
clear
Gitlab
::
Redis
::
Cache
.
with
{
|
redis
|
redis
.
del
(
HEALTHY_SHARDS_KEY
)
}
end
# Updates the list of healthy shards using a Redis set
#
# shards - An array of shard names to store
def
self
.
update
(
shards
)
Gitlab
::
Redis
::
Cache
.
with
do
|
redis
|
redis
.
multi
do
|
m
|
m
.
del
(
HEALTHY_SHARDS_KEY
)
shards
.
each
{
|
shard_name
|
m
.
sadd
(
HEALTHY_SHARDS_KEY
,
shard_name
)
}
m
.
expire
(
HEALTHY_SHARDS_KEY
,
HEALTHY_SHARDS_TIMEOUT
)
end
end
end
# Returns an array of strings of healthy shards
def
self
.
cached_healthy_shards
Gitlab
::
Redis
::
Cache
.
with
{
|
redis
|
redis
.
smembers
(
HEALTHY_SHARDS_KEY
)
}
end
# Checks whether the given shard name is in the list of healthy shards.
#
# shard_name - The string to check
def
self
.
healthy_shard?
(
shard_name
)
Gitlab
::
Redis
::
Cache
.
with
{
|
redis
|
redis
.
sismember
(
HEALTHY_SHARDS_KEY
,
shard_name
)
}
end
# Returns the number of healthy shards in the Redis set
def
self
.
healthy_shard_count
Gitlab
::
Redis
::
Cache
.
with
{
|
redis
|
redis
.
scard
(
HEALTHY_SHARDS_KEY
)
}
end
end
end
ee/spec/lib/gitlab/geo
/shard_health_cache_spec.rb
→
spec/lib/gitlab
/shard_health_cache_spec.rb
View file @
c38c23d1
require
'spec_helper'
describe
Gitlab
::
Geo
::
ShardHealthCache
,
:clean_gitlab_redis_cache
do
describe
Gitlab
::
ShardHealthCache
,
:clean_gitlab_redis_cache
do
let
(
:shards
)
{
%w(foo bar)
}
before
do
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment