Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
G
gitlab-ce
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
1
Merge Requests
1
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
nexedi
gitlab-ce
Commits
b7444374
Commit
b7444374
authored
May 04, 2018
by
Michael Kozono
Committed by
Nick Thomas
May 04, 2018
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Geo: Refactor FileDownloadDispatchWorker JobFinders and FileRegistryFinders
parent
79f13b9d
Changes
16
Show whitespace changes
Inline
Side-by-side
Showing
16 changed files
with
728 additions
and
758 deletions
+728
-758
ee/app/finders/geo/attachment_registry_finder.rb
ee/app/finders/geo/attachment_registry_finder.rb
+88
-84
ee/app/finders/geo/file_registry_finder.rb
ee/app/finders/geo/file_registry_finder.rb
+73
-3
ee/app/finders/geo/job_artifact_registry_finder.rb
ee/app/finders/geo/job_artifact_registry_finder.rb
+77
-82
ee/app/finders/geo/lfs_object_registry_finder.rb
ee/app/finders/geo/lfs_object_registry_finder.rb
+78
-74
ee/app/models/geo_node_status.rb
ee/app/models/geo_node_status.rb
+15
-15
ee/app/workers/geo/file_download_dispatch_worker.rb
ee/app/workers/geo/file_download_dispatch_worker.rb
+12
-11
ee/app/workers/geo/file_download_dispatch_worker/attachment_job_finder.rb
...eo/file_download_dispatch_worker/attachment_job_finder.rb
+10
-15
ee/app/workers/geo/file_download_dispatch_worker/job_artifact_job_finder.rb
.../file_download_dispatch_worker/job_artifact_job_finder.rb
+5
-19
ee/app/workers/geo/file_download_dispatch_worker/job_finder.rb
...p/workers/geo/file_download_dispatch_worker/job_finder.rb
+36
-13
ee/app/workers/geo/file_download_dispatch_worker/lfs_object_job_finder.rb
...eo/file_download_dispatch_worker/lfs_object_job_finder.rb
+5
-19
ee/app/workers/geo/migrated_local_files_clean_up_worker.rb
ee/app/workers/geo/migrated_local_files_clean_up_worker.rb
+3
-3
ee/spec/finders/geo/attachment_registry_finder_spec.rb
ee/spec/finders/geo/attachment_registry_finder_spec.rb
+132
-236
ee/spec/finders/geo/job_artifact_registry_finder_spec.rb
ee/spec/finders/geo/job_artifact_registry_finder_spec.rb
+78
-98
ee/spec/finders/geo/lfs_object_registry_finder_spec.rb
ee/spec/finders/geo/lfs_object_registry_finder_spec.rb
+66
-86
ee/spec/finders/geo/project_registry_finder_spec.rb
ee/spec/finders/geo/project_registry_finder_spec.rb
+4
-0
ee/spec/support/shared_examples/finders/file_registry_finders.rb
.../support/shared_examples/finders/file_registry_finders.rb
+46
-0
No files found.
ee/app/finders/geo/attachment_registry_finder.rb
View file @
b7444374
module
Geo
class
AttachmentRegistryFinder
<
FileRegistryFinder
def
attachments
if
selective_sync?
Upload
.
where
(
group_uploads
.
or
(
project_uploads
).
or
(
other_uploads
))
else
Upload
.
all
end
def
syncable
all
.
geo_syncable
end
def
syncable_attachments
attachments
.
geo_syncable
def
count_syncable
syncable
.
count
end
def
count_syncable_attachments
syncable_attachments
.
count
end
def
count_synced_attachments
def
count_synced
if
aggregate_pushdown_supported?
find_synced
_attachments
.
count
find_synced
.
count
else
legacy_find_synced
_attachments
.
count
legacy_find_synced
.
count
end
end
def
count_failed
_attachments
def
count_failed
if
aggregate_pushdown_supported?
find_failed
_attachments
.
count
find_failed
.
count
else
legacy_find_failed
_attachments
.
count
legacy_find_failed
.
count
end
end
def
count_synced_missing_on_primary
_attachments
def
count_synced_missing_on_primary
if
aggregate_pushdown_supported?
&&
!
use_legacy_queries?
fdw_find_synced_missing_on_primary
_attachments
.
count
fdw_find_synced_missing_on_primary
.
count
else
legacy_find_synced_missing_on_primary
_attachments
.
count
legacy_find_synced_missing_on_primary
.
count
end
end
def
count_registry
_attachments
def
count_registry
Geo
::
FileRegistry
.
attachments
.
count
end
def
find_synced_attachments
if
use_legacy_queries?
legacy_find_synced_attachments
else
fdw_find_synced_attachments
end
end
def
find_failed_attachments
if
use_legacy_queries?
legacy_find_failed_attachments
else
fdw_find_failed_attachments
end
end
# Find limited amount of non replicated attachments.
#
# You can pass a list with `except_file_ids:` so you can exclude items you
...
...
@@ -70,51 +46,51 @@ module Geo
#
# @param [Integer] batch_size used to limit the results returned
# @param [Array<Integer>] except_file_ids ids that will be ignored from the query
def
find_unsynced
_attachments
(
batch_size
:,
except_file_ids:
[])
def
find_unsynced
(
batch_size
:,
except_file_ids:
[])
relation
=
if
use_legacy_queries?
legacy_find_unsynced
_attachments
(
except_file_ids:
except_file_ids
)
legacy_find_unsynced
(
except_file_ids:
except_file_ids
)
else
fdw_find_unsynced
_attachments
(
except_file_ids:
except_file_ids
)
fdw_find_unsynced
(
except_file_ids:
except_file_ids
)
end
relation
.
limit
(
batch_size
)
end
def
find_migrated_local
_attachments
(
batch_size
:,
except_file_ids:
[])
def
find_migrated_local
(
batch_size
:,
except_file_ids:
[])
relation
=
if
use_legacy_queries?
legacy_find_migrated_local
_attachments
(
except_file_ids:
except_file_ids
)
legacy_find_migrated_local
(
except_file_ids:
except_file_ids
)
else
fdw_find_migrated_local
_attachments
(
except_file_ids:
except_file_ids
)
fdw_find_migrated_local
(
except_file_ids:
except_file_ids
)
end
relation
.
limit
(
batch_size
)
end
def
find_retryable_failed_
attachments_
registries
(
batch_size
:,
except_file_ids:
[])
find_failed_
attachments_
registries
def
find_retryable_failed_registries
(
batch_size
:,
except_file_ids:
[])
find_failed_registries
.
retry_due
.
where
.
not
(
file_id:
except_file_ids
)
.
limit
(
batch_size
)
end
def
find_retryable_synced_missing_on_primary_
attachments_
registries
(
batch_size
:,
except_file_ids:
[])
find_synced_missing_on_primary_
attachments_
registries
def
find_retryable_synced_missing_on_primary_registries
(
batch_size
:,
except_file_ids:
[])
find_synced_missing_on_primary_registries
.
retry_due
.
where
.
not
(
file_id:
except_file_ids
)
.
limit
(
batch_size
)
end
def
find_failed_attachments_registries
Geo
::
FileRegistry
.
attachments
.
failed
end
private
def
find_synced_missing_on_primary_attachments_registries
Geo
::
FileRegistry
.
attachments
.
synced
.
missing_on_primary
def
all
if
selective_sync?
Upload
.
where
(
group_uploads
.
or
(
project_uploads
).
or
(
other_uploads
))
else
Upload
.
all
end
end
private
def
group_uploads
namespace_ids
=
...
...
@@ -147,40 +123,68 @@ module Geo
Upload
.
arel_table
end
def
find_synced
if
use_legacy_queries?
legacy_find_synced
else
fdw_find_synced
end
end
def
find_failed
if
use_legacy_queries?
legacy_find_failed
else
fdw_find_failed
end
end
def
find_synced_registries
Geo
::
FileRegistry
.
attachments
.
synced
end
def
find_failed_registries
Geo
::
FileRegistry
.
attachments
.
failed
end
def
find_synced_missing_on_primary_registries
find_synced_registries
.
missing_on_primary
end
#
# FDW accessors
#
def
fdw_find_synced
_attachments
fdw_find_syncable
_attachments
.
merge
(
Geo
::
FileRegistry
.
synced
)
def
fdw_find_synced
fdw_find_syncable
.
merge
(
Geo
::
FileRegistry
.
synced
)
end
def
fdw_find_failed
_attachments
fdw_find_syncable
_attachments
.
merge
(
Geo
::
FileRegistry
.
failed
)
def
fdw_find_failed
fdw_find_syncable
.
merge
(
Geo
::
FileRegistry
.
failed
)
end
def
fdw_find_syncable
_attachments
fdw_a
ttachments
.
joins
(
"INNER JOIN file_registry ON file_registry.file_id =
#{
fdw_attachments
_table
}
.id"
)
def
fdw_find_syncable
fdw_a
ll
.
joins
(
"INNER JOIN file_registry ON file_registry.file_id =
#{
fdw
_table
}
.id"
)
.
geo_syncable
.
merge
(
Geo
::
FileRegistry
.
attachments
)
end
def
fdw_find_unsynced
_attachments
(
except_file_ids
:)
def
fdw_find_unsynced
(
except_file_ids
:)
upload_types
=
Geo
::
FileService
::
DEFAULT_OBJECT_TYPES
.
map
{
|
val
|
"'
#{
val
}
'"
}.
join
(
','
)
fdw_a
ttachments
.
joins
(
"LEFT OUTER JOIN file_registry
ON file_registry.file_id =
#{
fdw_
attachments_
table
}
.id
fdw_a
ll
.
joins
(
"LEFT OUTER JOIN file_registry
ON file_registry.file_id =
#{
fdw_table
}
.id
AND file_registry.file_type IN (
#{
upload_types
}
)"
)
.
geo_syncable
.
where
(
file_registry:
{
id:
nil
})
.
where
.
not
(
id:
except_file_ids
)
end
def
fdw_find_synced_missing_on_primary
_attachments
fdw_find_synced
_attachments
.
merge
(
Geo
::
FileRegistry
.
missing_on_primary
)
def
fdw_find_synced_missing_on_primary
fdw_find_synced
.
merge
(
Geo
::
FileRegistry
.
missing_on_primary
)
end
def
fdw_a
ttachments
def
fdw_a
ll
if
selective_sync?
Geo
::
Fdw
::
Upload
.
where
(
group_uploads
.
or
(
project_uploads
).
or
(
other_uploads
))
else
...
...
@@ -188,12 +192,12 @@ module Geo
end
end
def
fdw_
attachments_
table
def
fdw_table
Geo
::
Fdw
::
Upload
.
table_name
end
def
fdw_find_migrated_local
_attachments
(
except_file_ids
:)
fdw_a
ttachments
.
joins
(
"INNER JOIN file_registry ON file_registry.file_id =
#{
fdw_attachments
_table
}
.id"
)
def
fdw_find_migrated_local
(
except_file_ids
:)
fdw_a
ll
.
joins
(
"INNER JOIN file_registry ON file_registry.file_id =
#{
fdw
_table
}
.id"
)
.
with_files_stored_remotely
.
merge
(
Geo
::
FileRegistry
.
attachments
)
.
where
.
not
(
id:
except_file_ids
)
...
...
@@ -203,46 +207,46 @@ module Geo
# Legacy accessors (non FDW)
#
def
legacy_find_synced
_attachments
def
legacy_find_synced
legacy_inner_join_registry_ids
(
syncable
_attachments
,
Geo
::
FileRegistry
.
attachments
.
synced
.
pluck
(
:file_id
),
syncable
,
find_synced_registries
.
pluck
(
:file_id
),
Upload
)
end
def
legacy_find_failed
_attachments
def
legacy_find_failed
legacy_inner_join_registry_ids
(
syncable
_attachments
,
find_failed_
attachments_
registries
.
pluck
(
:file_id
),
syncable
,
find_failed_registries
.
pluck
(
:file_id
),
Upload
)
end
def
legacy_find_unsynced
_attachments
(
except_file_ids
:)
registry_file_ids
=
legacy_pluck_registry_file_ids
(
file_types:
Geo
::
FileService
::
DEFAULT_OBJECT_TYPES
)
|
except_file_ids
def
legacy_find_unsynced
(
except_file_ids
:)
registry_file_ids
=
Geo
::
FileRegistry
.
attachments
.
pluck
(
:file_id
)
|
except_file_ids
legacy_left_outer_join_registry_ids
(
syncable
_attachments
,
syncable
,
registry_file_ids
,
Upload
)
end
def
legacy_find_migrated_local
_attachments
(
except_file_ids
:)
def
legacy_find_migrated_local
(
except_file_ids
:)
registry_file_ids
=
Geo
::
FileRegistry
.
attachments
.
pluck
(
:file_id
)
-
except_file_ids
legacy_inner_join_registry_ids
(
a
ttachments
.
with_files_stored_remotely
,
a
ll
.
with_files_stored_remotely
,
registry_file_ids
,
Upload
)
end
def
legacy_find_synced_missing_on_primary
_attachments
def
legacy_find_synced_missing_on_primary
legacy_inner_join_registry_ids
(
syncable
_attachments
,
Geo
::
FileRegistry
.
attachments
.
synced
.
missing_on_primary
.
pluck
(
:file_id
),
syncable
,
find_synced_missing_on_primary_registries
.
pluck
(
:file_id
),
Upload
)
end
...
...
ee/app/finders/geo/file_registry_finder.rb
View file @
b7444374
module
Geo
class
FileRegistryFinder
<
RegistryFinder
protected
# @abstract Subclass is expected to implement the declared methods
def
legacy_pluck_registry_file_ids
(
file_types
:)
Geo
::
FileRegistry
.
where
(
file_type:
file_types
).
pluck
(
:file_id
)
# @!method syncable
# Return an ActiveRecord::Relation of tracked resource records, filtered
# by selective sync, with files stored locally
def
syncable
raise
NotImplementedError
end
# @!method count_syncable
# Return a count of tracked resource records, filtered by selective
# sync, with files stored locally
def
count_syncable
raise
NotImplementedError
end
# @!method count_synced
# Return a count of tracked resource records, filtered by selective
# sync, with files stored locally, and are synced
def
count_synced
raise
NotImplementedError
end
# @!method count_failed
# Return a count of tracked resource records, filtered by selective
# sync, with files stored locally, and are failed
def
count_failed
raise
NotImplementedError
end
# @!method count_synced_missing_on_primary
# Return a count of tracked resource records, filtered by selective
# sync, with files stored locally, and are synced and missing on the
# primary
def
count_synced_missing_on_primary
raise
NotImplementedError
end
# @!method count_registry
# Return a count of the registry records for the tracked file_type(s)
def
count_registry
raise
NotImplementedError
end
# @!method find_unsynced
# Return an ActiveRecord::Relation of not-yet-tracked resource records,
# filtered by selective sync, with files stored locally, excluding
# specified IDs, limited to batch_size
def
find_unsynced
raise
NotImplementedError
end
# @!method find_migrated_local
# Return an ActiveRecord::Relation of tracked resource records, filtered
# by selective sync, with files stored remotely, excluding
# specified IDs, limited to batch_size
def
find_migrated_local
raise
NotImplementedError
end
# @!method find_retryable_failed_registries
# Return an ActiveRecord::Relation of registry records marked as failed,
# which are ready to be retried, excluding specified IDs, limited to
# batch_size
def
find_retryable_failed_registries
raise
NotImplementedError
end
# @!method find_retryable_synced_missing_on_primary_registries
# Return an ActiveRecord::Relation of registry records marked as synced
# and missing on the primary, which are ready to be retried, excluding
# specified IDs, limited to batch_size
def
find_retryable_synced_missing_on_primary_registries
raise
NotImplementedError
end
end
end
ee/app/finders/geo/job_artifact_registry_finder.rb
View file @
b7444374
module
Geo
class
JobArtifactRegistryFinder
<
RegistryFinder
def
count_syncable
_job_artifacts
syncable
_job_artifacts
.
count
def
count_syncable
syncable
.
count
end
def
count_synced
_job_artifacts
def
count_synced
if
aggregate_pushdown_supported?
find_synced
_job_artifacts
.
count
find_synced
.
count
else
legacy_find_synced
_job_artifacts
.
count
legacy_find_synced
.
count
end
end
def
count_failed
_job_artifacts
def
count_failed
if
aggregate_pushdown_supported?
find_failed
_job_artifacts
.
count
find_failed
.
count
else
legacy_find_failed
_job_artifacts
.
count
legacy_find_failed
.
count
end
end
def
count_synced_missing_on_primary
_job_artifacts
def
count_synced_missing_on_primary
if
aggregate_pushdown_supported?
find_synced_missing_on_primary
_job_artifacts
.
count
find_synced_missing_on_primary
.
count
else
legacy_find_synced_missing_on_primary
_job_artifacts
.
count
legacy_find_synced_missing_on_primary
.
count
end
end
def
count_registry
_job_artifacts
def
count_registry
Geo
::
JobArtifactRegistry
.
count
end
...
...
@@ -42,117 +42,117 @@ module Geo
#
# @param [Integer] batch_size used to limit the results returned
# @param [Array<Integer>] except_artifact_ids ids that will be ignored from the query
def
find_unsynced
_job_artifacts
(
batch_size
:,
except_artifact_ids:
[])
def
find_unsynced
(
batch_size
:,
except_artifact_ids:
[])
relation
=
if
use_legacy_queries?
legacy_find_unsynced
_job_artifacts
(
except_artifact_ids:
except_artifact_ids
)
legacy_find_unsynced
(
except_artifact_ids:
except_artifact_ids
)
else
fdw_find_unsynced
_job_artifacts
(
except_artifact_ids:
except_artifact_ids
)
fdw_find_unsynced
(
except_artifact_ids:
except_artifact_ids
)
end
relation
.
limit
(
batch_size
)
end
def
find_migrated_local
_job_artifacts
(
batch_size
:,
except_artifact_ids:
[])
def
find_migrated_local
(
batch_size
:,
except_artifact_ids:
[])
relation
=
if
use_legacy_queries?
legacy_find_migrated_local
_job_artifacts
(
except_artifact_ids:
except_artifact_ids
)
legacy_find_migrated_local
(
except_artifact_ids:
except_artifact_ids
)
else
fdw_find_migrated_local
_job_artifacts
(
except_artifact_ids:
except_artifact_ids
)
fdw_find_migrated_local
(
except_artifact_ids:
except_artifact_ids
)
end
relation
.
limit
(
batch_size
)
end
def
job_artifacts
if
selective_sync?
Ci
::
JobArtifact
.
joins
(
:project
).
where
(
projects:
{
id:
current_node
.
projects
})
else
Ci
::
JobArtifact
.
all
end
end
def
syncable_job_artifacts
job_artifacts
.
geo_syncable
def
syncable
all
.
geo_syncable
end
def
find_retryable_failed_
job_artifacts_
registries
(
batch_size
:,
except_artifact_ids:
[])
find_failed_
job_artifacts_
registries
def
find_retryable_failed_registries
(
batch_size
:,
except_artifact_ids:
[])
find_failed_registries
.
retry_due
.
where
.
not
(
artifact_id:
except_artifact_ids
)
.
limit
(
batch_size
)
end
def
find_retryable_synced_missing_on_primary_
job_artifacts_
registries
(
batch_size
:,
except_artifact_ids:
[])
find_synced_missing_on_primary_
job_artifacts_
registries
def
find_retryable_synced_missing_on_primary_registries
(
batch_size
:,
except_artifact_ids:
[])
find_synced_missing_on_primary_registries
.
retry_due
.
where
.
not
(
artifact_id:
except_artifact_ids
)
.
limit
(
batch_size
)
end
def
find_synced_job_artifacts_registries
Geo
::
JobArtifactRegistry
.
synced
end
private
def
find_synced_missing_on_primary_job_artifacts_registries
Geo
::
JobArtifactRegistry
.
synced
.
missing_on_primary
def
all
if
selective_sync?
Ci
::
JobArtifact
.
joins
(
:project
).
where
(
projects:
{
id:
current_node
.
projects
})
else
Ci
::
JobArtifact
.
all
end
def
find_failed_job_artifacts_registries
Geo
::
JobArtifactRegistry
.
failed
end
private
def
find_synced_job_artifacts
def
find_synced
if
use_legacy_queries?
legacy_find_synced
_job_artifacts
legacy_find_synced
else
fdw_find
_job_artifacts
.
merge
(
find_synced_job_artifacts
_registries
)
fdw_find
.
merge
(
find_synced
_registries
)
end
end
def
find_synced_missing_on_primary
_job_artifacts
def
find_synced_missing_on_primary
if
use_legacy_queries?
legacy_find_synced_missing_on_primary
_job_artifacts
legacy_find_synced_missing_on_primary
else
fdw_find
_job_artifacts
.
merge
(
find_synced_missing_on_primary_job_artifacts
_registries
)
fdw_find
.
merge
(
find_synced_missing_on_primary
_registries
)
end
end
def
find_failed
_job_artifacts
def
find_failed
if
use_legacy_queries?
legacy_find_failed
_job_artifacts
legacy_find_failed
else
fdw_find_job_artifacts
.
merge
(
find_failed_job_artifacts_registries
)
fdw_find
.
merge
(
find_failed_registries
)
end
end
def
find_synced_registries
Geo
::
JobArtifactRegistry
.
synced
end
def
find_synced_missing_on_primary_registries
find_synced_registries
.
missing_on_primary
end
def
find_failed_registries
Geo
::
JobArtifactRegistry
.
failed
end
#
# FDW accessors
#
def
fdw_find
_job_artifacts
fdw_
job_artifacts
.
joins
(
"INNER JOIN job_artifact_registry ON job_artifact_registry.artifact_id =
#{
fdw_job_artifacts
_table
}
.id"
)
def
fdw_find
fdw_
all
.
joins
(
"INNER JOIN job_artifact_registry ON job_artifact_registry.artifact_id =
#{
fdw
_table
}
.id"
)
.
geo_syncable
end
def
fdw_find_unsynced
_job_artifacts
(
except_artifact_ids
:)
fdw_
job_artifacts
.
joins
(
"LEFT OUTER JOIN job_artifact_registry
ON job_artifact_registry.artifact_id =
#{
fdw_
job_artifacts_
table
}
.id"
)
def
fdw_find_unsynced
(
except_artifact_ids
:)
fdw_
all
.
joins
(
"LEFT OUTER JOIN job_artifact_registry
ON job_artifact_registry.artifact_id =
#{
fdw_table
}
.id"
)
.
geo_syncable
.
where
(
job_artifact_registry:
{
artifact_id:
nil
})
.
where
.
not
(
id:
except_artifact_ids
)
end
def
fdw_find_migrated_local
_job_artifacts
(
except_artifact_ids
:)
fdw_
job_artifacts
.
joins
(
"INNER JOIN job_artifact_registry ON job_artifact_registry.artifact_id =
#{
fdw_job_artifacts
_table
}
.id"
)
def
fdw_find_migrated_local
(
except_artifact_ids
:)
fdw_
all
.
joins
(
"INNER JOIN job_artifact_registry ON job_artifact_registry.artifact_id =
#{
fdw
_table
}
.id"
)
.
with_files_stored_remotely
.
where
.
not
(
id:
except_artifact_ids
)
.
merge
(
Geo
::
JobArtifactRegistry
.
all
)
end
def
fdw_
job_artifacts
def
fdw_
all
if
selective_sync?
Geo
::
Fdw
::
Ci
::
JobArtifact
.
joins
(
:project
).
where
(
projects:
{
id:
current_node
.
projects
})
else
...
...
@@ -160,7 +160,7 @@ module Geo
end
end
def
fdw_
job_artifacts_
table
def
fdw_table
Geo
::
Fdw
::
Ci
::
JobArtifact
.
table_name
end
...
...
@@ -168,51 +168,46 @@ module Geo
# Legacy accessors (non FDW)
#
def
legacy_find_synced
_job_artifacts
def
legacy_find_synced
legacy_inner_join_registry_ids
(
syncable
_job_artifacts
,
find_synced_
job_artifacts_
registries
.
pluck
(
:artifact_id
),
syncable
,
find_synced_registries
.
pluck
(
:artifact_id
),
Ci
::
JobArtifact
)
end
def
legacy_find_failed
_job_artifacts
def
legacy_find_failed
legacy_inner_join_registry_ids
(
syncable
_job_artifacts
,
find_failed_
job_artifacts_
registries
.
pluck
(
:artifact_id
),
syncable
,
find_failed_registries
.
pluck
(
:artifact_id
),
Ci
::
JobArtifact
)
end
def
legacy_find_unsynced
_job_artifacts
(
except_artifact_ids
:)
registry_artifact_ids
=
legacy_pluck_artifact_ids
(
include_registry_ids:
except_artifact_ids
)
def
legacy_find_unsynced
(
except_artifact_ids
:)
registry_artifact_ids
=
Geo
::
JobArtifactRegistry
.
pluck
(
:artifact_id
)
|
except_artifact_ids
legacy_left_outer_join_registry_ids
(
syncable
_job_artifacts
,
syncable
,
registry_artifact_ids
,
Ci
::
JobArtifact
)
end
def
legacy_pluck_artifact_ids
(
include_registry_ids
:)
ids
=
Geo
::
JobArtifactRegistry
.
pluck
(
:artifact_id
)
(
ids
+
include_registry_ids
).
uniq
end
def
legacy_find_migrated_local_job_artifacts
(
except_artifact_ids
:)
registry_file_ids
=
Geo
::
JobArtifactRegistry
.
pluck
(
:artifact_id
)
-
except_artifact_ids
def
legacy_find_migrated_local
(
except_artifact_ids
:)
registry_artifact_ids
=
Geo
::
JobArtifactRegistry
.
pluck
(
:artifact_id
)
-
except_artifact_ids
legacy_inner_join_registry_ids
(
job_artifacts
.
with_files_stored_remotely
,
registry_
file
_ids
,
all
.
with_files_stored_remotely
,
registry_
artifact
_ids
,
Ci
::
JobArtifact
)
end
def
legacy_find_synced_missing_on_primary
_job_artifacts
def
legacy_find_synced_missing_on_primary
legacy_inner_join_registry_ids
(
syncable
_job_artifacts
,
find_synced_missing_on_primary_
job_artifacts_
registries
.
pluck
(
:artifact_id
),
syncable
,
find_synced_missing_on_primary_registries
.
pluck
(
:artifact_id
),
Ci
::
JobArtifact
)
end
...
...
ee/app/finders/geo/lfs_object_registry_finder.rb
View file @
b7444374
module
Geo
class
LfsObjectRegistryFinder
<
FileRegistryFinder
def
count_syncable
_lfs_objects
syncable
_lfs_objects
.
count
def
count_syncable
syncable
.
count
end
def
count_synced
_lfs_objects
def
count_synced
if
aggregate_pushdown_supported?
find_synced
_lfs_objects
.
count
find_synced
.
count
else
legacy_find_synced
_lfs_objects
.
count
legacy_find_synced
.
count
end
end
def
count_failed
_lfs_objects
def
count_failed
if
aggregate_pushdown_supported?
find_failed
_lfs_objects
.
count
find_failed
.
count
else
legacy_find_failed
_lfs_objects
.
count
legacy_find_failed
.
count
end
end
def
count_synced_missing_on_primary
_lfs_objects
def
count_synced_missing_on_primary
if
aggregate_pushdown_supported?
&&
!
use_legacy_queries?
fdw_find_synced_missing_on_primary
_lfs_objects
.
count
fdw_find_synced_missing_on_primary
.
count
else
legacy_find_synced_missing_on_primary
_lfs_objects
.
count
legacy_find_synced_missing_on_primary
.
count
end
end
def
count_registry
_lfs_objects
def
count_registry
Geo
::
FileRegistry
.
lfs_objects
.
count
end
...
...
@@ -42,119 +42,123 @@ module Geo
#
# @param [Integer] batch_size used to limit the results returned
# @param [Array<Integer>] except_file_ids ids that will be ignored from the query
def
find_unsynced
_lfs_objects
(
batch_size
:,
except_file_ids:
[])
def
find_unsynced
(
batch_size
:,
except_file_ids:
[])
relation
=
if
use_legacy_queries?
legacy_find_unsynced
_lfs_objects
(
except_file_ids:
except_file_ids
)
legacy_find_unsynced
(
except_file_ids:
except_file_ids
)
else
fdw_find_unsynced
_lfs_objects
(
except_file_ids:
except_file_ids
)
fdw_find_unsynced
(
except_file_ids:
except_file_ids
)
end
relation
.
limit
(
batch_size
)
end
def
find_migrated_local
_lfs_objects
(
batch_size
:,
except_file_ids:
[])
def
find_migrated_local
(
batch_size
:,
except_file_ids:
[])
relation
=
if
use_legacy_queries?
legacy_find_migrated_local
_lfs_objects
(
except_file_ids:
except_file_ids
)
legacy_find_migrated_local
(
except_file_ids:
except_file_ids
)
else
fdw_find_migrated_local
_lfs_objects
(
except_file_ids:
except_file_ids
)
fdw_find_migrated_local
(
except_file_ids:
except_file_ids
)
end
relation
.
limit
(
batch_size
)
end
def
lfs_objects
if
selective_sync?
LfsObject
.
joins
(
:projects
).
where
(
projects:
{
id:
current_node
.
projects
})
else
LfsObject
.
all
end
end
def
syncable_lfs_objects
lfs_objects
.
geo_syncable
def
syncable
all
.
geo_syncable
end
def
find_retryable_failed_
lfs_objects_
registries
(
batch_size
:,
except_file_ids:
[])
find_failed_
lfs_objects_
registries
def
find_retryable_failed_registries
(
batch_size
:,
except_file_ids:
[])
find_failed_registries
.
retry_due
.
where
.
not
(
file_id:
except_file_ids
)
.
limit
(
batch_size
)
end
def
find_retryable_synced_missing_on_primary_
lfs_objects_
registries
(
batch_size
:,
except_file_ids:
[])
find_synced_missing_on_primary_
lfs_objects_
registries
def
find_retryable_synced_missing_on_primary_registries
(
batch_size
:,
except_file_ids:
[])
find_synced_missing_on_primary_registries
.
retry_due
.
where
.
not
(
file_id:
except_file_ids
)
.
limit
(
batch_size
)
end
def
find_failed_lfs_objects_registries
Geo
::
FileRegistry
.
lfs_objects
.
failed
end
private
def
find_synced_missing_on_primary_lfs_objects_registries
Geo
::
FileRegistry
.
lfs_objects
.
synced
.
missing_on_primary
def
all
if
selective_sync?
LfsObject
.
joins
(
:projects
).
where
(
projects:
{
id:
current_node
.
projects
})
else
LfsObject
.
all
end
end
private
def
find_synced_lfs_objects
def
find_synced
if
use_legacy_queries?
legacy_find_synced
_lfs_objects
legacy_find_synced
else
fdw_find_synced
_lfs_objects
fdw_find_synced
end
end
def
find_failed
_lfs_objects
def
find_failed
if
use_legacy_queries?
legacy_find_failed
_lfs_objects
legacy_find_failed
else
fdw_find_failed_lfs_objects
fdw_find_failed
end
end
def
find_synced_registries
Geo
::
FileRegistry
.
lfs_objects
.
synced
end
def
find_failed_registries
Geo
::
FileRegistry
.
lfs_objects
.
failed
end
def
find_synced_missing_on_primary_registries
find_synced_registries
.
missing_on_primary
end
#
# FDW accessors
#
def
fdw_find
_lfs_objects
fdw_
lfs_objects
.
joins
(
"INNER JOIN file_registry ON file_registry.file_id =
#{
fdw_lfs_objects
_table
}
.id"
)
def
fdw_find
fdw_
all
.
joins
(
"INNER JOIN file_registry ON file_registry.file_id =
#{
fdw
_table
}
.id"
)
.
geo_syncable
.
merge
(
Geo
::
FileRegistry
.
lfs_objects
)
end
def
fdw_find_unsynced
_lfs_objects
(
except_file_ids
:)
fdw_
lfs_objects
.
joins
(
"LEFT OUTER JOIN file_registry
ON file_registry.file_id =
#{
fdw_
lfs_objects_
table
}
.id
def
fdw_find_unsynced
(
except_file_ids
:)
fdw_
all
.
joins
(
"LEFT OUTER JOIN file_registry
ON file_registry.file_id =
#{
fdw_table
}
.id
AND file_registry.file_type = 'lfs'"
)
.
geo_syncable
.
where
(
file_registry:
{
id:
nil
})
.
where
.
not
(
id:
except_file_ids
)
end
def
fdw_find_migrated_local
_lfs_objects
(
except_file_ids
:)
fdw_
lfs_objects
.
joins
(
"INNER JOIN file_registry ON file_registry.file_id =
#{
fdw_lfs_objects
_table
}
.id"
)
def
fdw_find_migrated_local
(
except_file_ids
:)
fdw_
all
.
joins
(
"INNER JOIN file_registry ON file_registry.file_id =
#{
fdw
_table
}
.id"
)
.
with_files_stored_remotely
.
where
.
not
(
id:
except_file_ids
)
.
merge
(
Geo
::
FileRegistry
.
lfs_objects
)
end
def
fdw_find_synced
_lfs_objects
fdw_find
_lfs_objects
.
merge
(
Geo
::
FileRegistry
.
synced
)
def
fdw_find_synced
fdw_find
.
merge
(
Geo
::
FileRegistry
.
synced
)
end
def
fdw_find_synced_missing_on_primary
_lfs_objects
fdw_find
_lfs_objects
.
merge
(
Geo
::
FileRegistry
.
synced
.
missing_on_primary
)
def
fdw_find_synced_missing_on_primary
fdw_find
.
merge
(
Geo
::
FileRegistry
.
synced
.
missing_on_primary
)
end
def
fdw_find_failed
_lfs_objects
fdw_find
_lfs_objects
.
merge
(
Geo
::
FileRegistry
.
failed
)
def
fdw_find_failed
fdw_find
.
merge
(
Geo
::
FileRegistry
.
failed
)
end
def
fdw_
lfs_objects
def
fdw_
all
if
selective_sync?
Geo
::
Fdw
::
LfsObject
.
joins
(
:project
).
where
(
projects:
{
id:
current_node
.
projects
})
else
...
...
@@ -162,7 +166,7 @@ module Geo
end
end
def
fdw_
lfs_objects_
table
def
fdw_table
Geo
::
Fdw
::
LfsObject
.
table_name
end
...
...
@@ -170,46 +174,46 @@ module Geo
# Legacy accessors (non FDW)
#
def
legacy_find_synced
_lfs_objects
def
legacy_find_synced
legacy_inner_join_registry_ids
(
syncable
_lfs_objects
,
Geo
::
FileRegistry
.
lfs_objects
.
synced
.
pluck
(
:file_id
),
syncable
,
find_synced_registries
.
pluck
(
:file_id
),
LfsObject
)
end
def
legacy_find_failed
_lfs_objects
def
legacy_find_failed
legacy_inner_join_registry_ids
(
syncable
_lfs_objects
,
find_failed_
lfs_objects_
registries
.
pluck
(
:file_id
),
syncable
,
find_failed_registries
.
pluck
(
:file_id
),
LfsObject
)
end
def
legacy_find_unsynced
_lfs_objects
(
except_file_ids
:)
registry_file_ids
=
legacy_pluck_registry_file_ids
(
file_types: :lfs
)
|
except_file_ids
def
legacy_find_unsynced
(
except_file_ids
:)
registry_file_ids
=
Geo
::
FileRegistry
.
lfs_objects
.
pluck
(
:file_id
)
|
except_file_ids
legacy_left_outer_join_registry_ids
(
syncable
_lfs_objects
,
syncable
,
registry_file_ids
,
LfsObject
)
end
def
legacy_find_migrated_local
_lfs_objects
(
except_file_ids
:)
def
legacy_find_migrated_local
(
except_file_ids
:)
registry_file_ids
=
Geo
::
FileRegistry
.
lfs_objects
.
pluck
(
:file_id
)
-
except_file_ids
legacy_inner_join_registry_ids
(
lfs_objects
.
with_files_stored_remotely
,
all
.
with_files_stored_remotely
,
registry_file_ids
,
LfsObject
)
end
def
legacy_find_synced_missing_on_primary
_lfs_objects
def
legacy_find_synced_missing_on_primary
legacy_inner_join_registry_ids
(
syncable
_lfs_objects
,
Geo
::
FileRegistry
.
lfs_objects
.
synced
.
missing_on_primary
.
pluck
(
:file_id
),
syncable
,
find_synced_missing_on_primary_registries
.
pluck
(
:file_id
),
LfsObject
)
end
...
...
ee/app/models/geo_node_status.rb
View file @
b7444374
...
...
@@ -150,9 +150,9 @@ class GeoNodeStatus < ActiveRecord::Base
self
.
last_event_date
=
latest_event
&
.
created_at
self
.
repositories_count
=
projects_finder
.
count_repositories
self
.
wikis_count
=
projects_finder
.
count_wikis
self
.
lfs_objects_count
=
lfs_objects_finder
.
count_syncable
_lfs_objects
self
.
job_artifacts_count
=
job_artifacts_finder
.
count_syncable
_job_artifacts
self
.
attachments_count
=
attachments_finder
.
count_syncable
_attachments
self
.
lfs_objects_count
=
lfs_objects_finder
.
count_syncable
self
.
job_artifacts_count
=
job_artifacts_finder
.
count_syncable
self
.
attachments_count
=
attachments_finder
.
count_syncable
self
.
last_successful_status_check_at
=
Time
.
now
self
.
storage_shards
=
StorageShard
.
all
...
...
@@ -202,18 +202,18 @@ class GeoNodeStatus < ActiveRecord::Base
self
.
repositories_failed_count
=
projects_finder
.
count_failed_repositories
self
.
wikis_synced_count
=
projects_finder
.
count_synced_wikis
self
.
wikis_failed_count
=
projects_finder
.
count_failed_wikis
self
.
lfs_objects_synced_count
=
lfs_objects_finder
.
count_synced
_lfs_objects
self
.
lfs_objects_failed_count
=
lfs_objects_finder
.
count_failed
_lfs_objects
self
.
lfs_objects_registry_count
=
lfs_objects_finder
.
count_registry
_lfs_objects
self
.
lfs_objects_synced_missing_on_primary_count
=
lfs_objects_finder
.
count_synced_missing_on_primary
_lfs_objects
self
.
job_artifacts_synced_count
=
job_artifacts_finder
.
count_synced
_job_artifacts
self
.
job_artifacts_failed_count
=
job_artifacts_finder
.
count_failed
_job_artifacts
self
.
job_artifacts_registry_count
=
job_artifacts_finder
.
count_registry
_job_artifacts
self
.
job_artifacts_synced_missing_on_primary_count
=
job_artifacts_finder
.
count_synced_missing_on_primary
_job_artifacts
self
.
attachments_synced_count
=
attachments_finder
.
count_synced
_attachments
self
.
attachments_failed_count
=
attachments_finder
.
count_failed
_attachments
self
.
attachments_registry_count
=
attachments_finder
.
count_registry
_attachments
self
.
attachments_synced_missing_on_primary_count
=
attachments_finder
.
count_synced_missing_on_primary
_attachments
self
.
lfs_objects_synced_count
=
lfs_objects_finder
.
count_synced
self
.
lfs_objects_failed_count
=
lfs_objects_finder
.
count_failed
self
.
lfs_objects_registry_count
=
lfs_objects_finder
.
count_registry
self
.
lfs_objects_synced_missing_on_primary_count
=
lfs_objects_finder
.
count_synced_missing_on_primary
self
.
job_artifacts_synced_count
=
job_artifacts_finder
.
count_synced
self
.
job_artifacts_failed_count
=
job_artifacts_finder
.
count_failed
self
.
job_artifacts_registry_count
=
job_artifacts_finder
.
count_registry
self
.
job_artifacts_synced_missing_on_primary_count
=
job_artifacts_finder
.
count_synced_missing_on_primary
self
.
attachments_synced_count
=
attachments_finder
.
count_synced
self
.
attachments_failed_count
=
attachments_finder
.
count_failed
self
.
attachments_registry_count
=
attachments_finder
.
count_registry
self
.
attachments_synced_missing_on_primary_count
=
attachments_finder
.
count_synced_missing_on_primary
load_verification_data
end
...
...
ee/app/workers/geo/file_download_dispatch_worker.rb
View file @
b7444374
...
...
@@ -28,24 +28,25 @@ module Geo
end
end
# Get a batch of unsynced resources, taking equal parts from each resource.
#
# @return [Array] job arguments of unsynced resources
def
find_unsynced_jobs
(
batch_size
:)
find_jobs
(
sync_statuses:
[
:unsynced
],
batch_size:
batch_size
)
jobs
=
job_finders
.
reduce
([])
do
|
jobs
,
job_finder
|
jobs
<<
job_finder
.
find_unsynced_jobs
(
batch_size:
batch_size
)
end
# @return [Array] job arguments of low priority resources
def
find_low_priority_jobs
(
batch_size
:)
find_jobs
(
sync_statuses:
[
:failed
,
:synced_missing_on_primary
],
batch_size:
batch_size
)
take_batch
(
*
jobs
,
batch_size:
batch_size
)
end
# Get a batch of resources taking equal parts from each resource.
# Get a batch of failed and synced-but-missing-on-primary resources, taking
# equal parts from each resource.
#
# @return [Array] job arguments of
a batch of
resources
def
find_
jobs
(
sync_statuses
:,
batch_size
:)
# @return [Array] job arguments of
low priority
resources
def
find_
low_priority_jobs
(
batch_size
:)
jobs
=
job_finders
.
reduce
([])
do
|
jobs
,
job_finder
|
sync_statuses
.
reduce
(
jobs
)
do
|
jobs
,
sync_status
|
jobs
<<
job_finder
.
find_jobs
(
sync_status:
sync_status
,
batch_size:
batch_size
)
end
jobs
<<
job_finder
.
find_failed_jobs
(
batch_size:
batch_size
)
jobs
<<
job_finder
.
find_synced_missing_on_primary_jobs
(
batch_size:
batch_size
)
end
take_batch
(
*
jobs
,
batch_size:
batch_size
)
...
...
ee/app/workers/geo/file_download_dispatch_worker/attachment_job_finder.rb
View file @
b7444374
module
Geo
class
FileDownloadDispatchWorker
class
AttachmentJobFinder
<
JobFinder
def
resource_type
:attachment
end
EXCEPT_RESOURCE_IDS_KEY
=
:except_file_ids
def
except_resource_ids_key
:except_file_ids
def
registry_finder
@registry_finder
||=
Geo
::
AttachmentRegistryFinder
.
new
(
current_node:
Gitlab
::
Geo
.
current_node
)
end
private
# Why do we need a different `file_type` for each Uploader? Why not just use 'upload'?
def
find_unsynced_jobs
(
batch_size
:)
registry_finder
.
find_unsynced_attachments
(
batch_size:
batch_size
,
except_file_ids:
scheduled_file_ids
)
.
pluck
(
:id
,
:uploader
)
def
convert_resource_relation_to_job_args
(
relation
)
relation
.
pluck
(
:id
,
:uploader
)
.
map
{
|
id
,
uploader
|
[
uploader
.
sub
(
/Uploader\z/
,
''
).
underscore
,
id
]
}
end
def
find_failed_jobs
(
batch_size
:)
find_failed_registries
(
batch_size:
batch_size
).
pluck
(
:file_type
,
:file_id
)
end
def
find_synced_missing_on_primary_jobs
(
batch_size
:)
find_synced_missing_on_primary_registries
(
batch_size:
batch_size
).
pluck
(
:file_type
,
:file_id
)
def
convert_registry_relation_to_job_args
(
relation
)
relation
.
pluck
(
:file_type
,
:file_id
)
end
end
end
...
...
ee/app/workers/geo/file_download_dispatch_worker/job_artifact_job_finder.rb
View file @
b7444374
module
Geo
class
FileDownloadDispatchWorker
class
JobArtifactJobFinder
<
JobFinder
def
resource_type
:job_artifact
end
def
resource_id_prefix
:artifact
end
def
find_unsynced_jobs
(
batch_size
:)
registry_finder
.
find_unsynced_job_artifacts
(
batch_size:
batch_size
,
except_artifact_ids:
scheduled_file_ids
)
.
pluck
(
:id
)
.
map
{
|
id
|
[
'job_artifact'
,
id
]
}
end
def
find_failed_jobs
(
batch_size
:)
find_failed_registries
(
batch_size:
batch_size
).
pluck
(
:artifact_id
).
map
{
|
id
|
[
'job_artifact'
,
id
]
}
end
RESOURCE_ID_KEY
=
:artifact_id
EXCEPT_RESOURCE_IDS_KEY
=
:except_artifact_ids
FILE_SERVICE_OBJECT_TYPE
=
:job_artifact
def
find_synced_missing_on_primary_jobs
(
batch_size
:)
find_synced_missing_on_primary_registries
(
batch_size:
batch_size
).
pluck
(
:artifact_id
).
map
{
|
id
|
[
'job_artifact'
,
id
]
}
def
registry_finder
@registry_finder
||=
Geo
::
JobArtifactRegistryFinder
.
new
(
current_node:
Gitlab
::
Geo
.
current_node
)
end
end
end
...
...
ee/app/workers/geo/file_download_dispatch_worker/job_finder.rb
View file @
b7444374
module
Geo
class
FileDownloadDispatchWorker
# This class is meant to be inherited, and is responsible for generating
# batches of job arguments for FileDownloadWorker.
#
# The subclass should define
#
# * registry_finder
# * EXCEPT_RESOURCE_IDS_KEY
# * RESOURCE_ID_KEY
# * FILE_SERVICE_OBJECT_TYPE
#
class
JobFinder
include
Gitlab
::
Utils
::
StrongMemoize
attr_reader
:
registry_finder
,
:
scheduled_file_ids
attr_reader
:scheduled_file_ids
def
initialize
(
scheduled_file_ids
)
current_node
=
Gitlab
::
Geo
.
current_node
@registry_finder
=
registry_finder_class
.
new
(
current_node:
current_node
)
@scheduled_file_ids
=
scheduled_file_ids
end
def
registry_finder_class
"Geo::
#{
resource_type
.
to_s
.
classify
}
RegistryFinder"
.
constantize
def
find_unsynced_jobs
(
batch_size
:)
convert_resource_relation_to_job_args
(
registry_finder
.
find_unsynced
(
find_batch_params
(
batch_size
))
)
end
def
except_resource_ids_key
:"except_
#{
resource_id_prefix
}
_ids"
def
find_failed_jobs
(
batch_size
:)
convert_registry_relation_to_job_args
(
registry_finder
.
find_retryable_failed_registries
(
find_batch_params
(
batch_size
))
)
end
def
find_jobs
(
sync_status
:,
batch_size
:)
self
.
public_send
(
:"find_
#{
sync_status
}
_jobs"
,
batch_size:
batch_size
)
# rubocop:disable GitlabSecurity/PublicSend
def
find_synced_missing_on_primary_jobs
(
batch_size
:)
convert_registry_relation_to_job_args
(
registry_finder
.
find_retryable_synced_missing_on_primary_registries
(
find_batch_params
(
batch_size
))
)
end
def
find_failed_registries
(
batch_size
:)
registry_finder
.
public_send
(
:"find_retryable_failed_
#{
resource_type
}
s_registries"
,
batch_size:
batch_size
,
except_resource_ids_key
=>
scheduled_file_ids
)
# rubocop:disable GitlabSecurity/PublicSend
private
def
find_batch_params
(
batch_size
)
{
:batch_size
=>
batch_size
,
self
.
class
::
EXCEPT_RESOURCE_IDS_KEY
=>
scheduled_file_ids
}
end
def
convert_resource_relation_to_job_args
(
relation
)
relation
.
pluck
(
:id
).
map
{
|
id
|
[
self
.
class
::
FILE_SERVICE_OBJECT_TYPE
.
to_s
,
id
]
}
end
def
find_synced_missing_on_primary_registries
(
batch_size
:
)
re
gistry_finder
.
public_send
(
:"find_retryable_synced_missing_on_primary_
#{
resource_type
}
s_registries"
,
batch_size:
batch_size
,
except_resource_ids_key
=>
scheduled_file_ids
)
# rubocop:disable GitlabSecurity/PublicSend
def
convert_registry_relation_to_job_args
(
relation
)
re
lation
.
pluck
(
self
.
class
::
RESOURCE_ID_KEY
).
map
{
|
id
|
[
self
.
class
::
FILE_SERVICE_OBJECT_TYPE
.
to_s
,
id
]
}
end
end
end
...
...
ee/app/workers/geo/file_download_dispatch_worker/lfs_object_job_finder.rb
View file @
b7444374
module
Geo
class
FileDownloadDispatchWorker
class
LfsObjectJobFinder
<
JobFinder
def
resource_type
:lfs_object
end
def
except_resource_ids_key
:except_file_ids
end
def
find_unsynced_jobs
(
batch_size
:)
registry_finder
.
find_unsynced_lfs_objects
(
batch_size:
batch_size
,
except_file_ids:
scheduled_file_ids
)
.
pluck
(
:id
)
.
map
{
|
id
|
[
'lfs'
,
id
]
}
end
def
find_failed_jobs
(
batch_size
:)
find_failed_registries
(
batch_size:
batch_size
).
pluck
(
:file_id
).
map
{
|
id
|
[
'lfs'
,
id
]
}
end
RESOURCE_ID_KEY
=
:file_id
EXCEPT_RESOURCE_IDS_KEY
=
:except_file_ids
FILE_SERVICE_OBJECT_TYPE
=
:lfs
def
find_synced_missing_on_primary_jobs
(
batch_size
:)
find_synced_missing_on_primary_registries
(
batch_size:
batch_size
).
pluck
(
:file_id
).
map
{
|
id
|
[
'lfs'
,
id
]
}
def
registry_finder
@registry_finder
||=
Geo
::
LfsObjectRegistryFinder
.
new
(
current_node:
Gitlab
::
Geo
.
current_node
)
end
end
end
...
...
ee/app/workers/geo/migrated_local_files_clean_up_worker.rb
View file @
b7444374
...
...
@@ -45,7 +45,7 @@ module Geo
def
find_migrated_local_lfs_objects_ids
(
batch_size
:)
return
[]
unless
lfs_objects_object_store_enabled?
lfs_objects_finder
.
find_migrated_local
_lfs_objects
(
batch_size:
batch_size
,
except_file_ids:
scheduled_file_ids
(
:lfs
))
lfs_objects_finder
.
find_migrated_local
(
batch_size:
batch_size
,
except_file_ids:
scheduled_file_ids
(
:lfs
))
.
pluck
(
:id
)
.
map
{
|
id
|
[
'lfs'
,
id
]
}
end
...
...
@@ -53,7 +53,7 @@ module Geo
def
find_migrated_local_attachments_ids
(
batch_size
:)
return
[]
unless
attachments_object_store_enabled?
attachments_finder
.
find_migrated_local
_attachments
(
batch_size:
batch_size
,
except_file_ids:
scheduled_file_ids
(
Geo
::
FileService
::
DEFAULT_OBJECT_TYPES
))
attachments_finder
.
find_migrated_local
(
batch_size:
batch_size
,
except_file_ids:
scheduled_file_ids
(
Geo
::
FileService
::
DEFAULT_OBJECT_TYPES
))
.
pluck
(
:uploader
,
:id
)
.
map
{
|
uploader
,
id
|
[
uploader
.
sub
(
/Uploader\z/
,
''
).
underscore
,
id
]
}
end
...
...
@@ -61,7 +61,7 @@ module Geo
def
find_migrated_local_job_artifacts_ids
(
batch_size
:)
return
[]
unless
job_artifacts_object_store_enabled?
job_artifacts_finder
.
find_migrated_local
_job_artifacts
(
batch_size:
batch_size
,
except_artifact_ids:
scheduled_file_ids
(
:job_artifact
))
job_artifacts_finder
.
find_migrated_local
(
batch_size:
batch_size
,
except_artifact_ids:
scheduled_file_ids
(
:job_artifact
))
.
pluck
(
:id
)
.
map
{
|
id
|
[
'job_artifact'
,
id
]
}
end
...
...
ee/spec/finders/geo/attachment_registry_finder_spec.rb
View file @
b7444374
...
...
@@ -3,6 +3,10 @@ require 'spec_helper'
describe
Geo
::
AttachmentRegistryFinder
,
:geo
do
include
::
EE
::
GeoHelpers
# Using let() instead of set() because set() does not work properly
# when using the :delete DatabaseCleaner strategy, which is required for FDW
# tests because a foreign table can't see changes inside a transaction of a
# different connection.
let
(
:secondary
)
{
create
(
:geo_node
)
}
let
(
:synced_group
)
{
create
(
:group
)
}
...
...
@@ -30,7 +34,7 @@ describe Geo::AttachmentRegistryFinder, :geo do
end
shared_examples
'counts all the things'
do
describe
'#count_syncable
_attachments
'
do
describe
'#count_syncable'
do
before
do
upload_1
upload_2
...
...
@@ -39,13 +43,13 @@ describe Geo::AttachmentRegistryFinder, :geo do
end
it
'counts attachments'
do
expect
(
subject
.
count_syncable
_attachments
).
to
eq
4
expect
(
subject
.
count_syncable
).
to
eq
4
end
it
'ignores remote attachments'
do
upload_1
.
update!
(
store:
ObjectStorage
::
Store
::
REMOTE
)
expect
(
subject
.
count_syncable
_attachments
).
to
eq
3
expect
(
subject
.
count_syncable
).
to
eq
3
end
context
'with selective sync'
do
...
...
@@ -54,32 +58,32 @@ describe Geo::AttachmentRegistryFinder, :geo do
end
it
'counts attachments'
do
expect
(
subject
.
count_syncable
_attachments
).
to
eq
2
expect
(
subject
.
count_syncable
).
to
eq
2
end
it
'ignores remote attachments'
do
upload_1
.
update!
(
store:
ObjectStorage
::
Store
::
REMOTE
)
expect
(
subject
.
count_syncable
_attachments
).
to
eq
1
expect
(
subject
.
count_syncable
).
to
eq
1
end
end
end
describe
'#count_synced
_attachments
'
do
it
'delegates to #legacy_find_synced
_attachments
'
do
describe
'#count_synced'
do
it
'delegates to #legacy_find_synced'
do
allow
(
subject
).
to
receive
(
:aggregate_pushdown_supported?
).
and_return
(
false
)
expect
(
subject
).
to
receive
(
:legacy_find_synced
_attachments
).
and_call_original
expect
(
subject
).
to
receive
(
:legacy_find_synced
).
and_call_original
subject
.
count_synced
_attachments
subject
.
count_synced
end
it
'delegates to #find_synced
_attachments
for PostgreSQL 10'
do
it
'delegates to #find_synced for PostgreSQL 10'
do
allow
(
subject
).
to
receive
(
:aggregate_pushdown_supported?
).
and_return
(
true
)
expect
(
subject
).
to
receive
(
:find_synced
_attachments
).
and_call_original
expect
(
subject
).
to
receive
(
:find_synced
).
and_call_original
subject
.
count_synced
_attachments
subject
.
count_synced
end
it
'counts attachments that have been synced'
do
...
...
@@ -87,7 +91,7 @@ describe Geo::AttachmentRegistryFinder, :geo do
create
(
:geo_file_registry
,
:attachment
,
file_id:
upload_2
.
id
)
create
(
:geo_file_registry
,
:attachment
,
file_id:
upload_3
.
id
)
expect
(
subject
.
count_synced
_attachments
).
to
eq
2
expect
(
subject
.
count_synced
).
to
eq
2
end
it
'ignores remote attachments'
do
...
...
@@ -95,18 +99,18 @@ describe Geo::AttachmentRegistryFinder, :geo do
create
(
:geo_file_registry
,
:attachment
,
file_id:
upload_2
.
id
)
create
(
:geo_file_registry
,
:attachment
,
file_id:
upload_3
.
id
)
expect
(
subject
.
count_synced
_attachments
).
to
eq
2
expect
(
subject
.
count_synced
).
to
eq
2
end
context
'with selective sync'
do
context
'with selective sync
by namespace
'
do
before
do
secondary
.
update!
(
selective_sync_type:
'namespaces'
,
namespaces:
[
synced_group
])
end
it
'delegates to #legacy_find_synced
_attachments
'
do
expect
(
subject
).
to
receive
(
:legacy_find_synced
_attachments
).
and_call_original
it
'delegates to #legacy_find_synced'
do
expect
(
subject
).
to
receive
(
:legacy_find_synced
).
and_call_original
subject
.
count_synced
_attachments
subject
.
count_synced
end
it
'counts attachments that has been synced'
do
...
...
@@ -114,7 +118,7 @@ describe Geo::AttachmentRegistryFinder, :geo do
create
(
:geo_file_registry
,
:attachment
,
file_id:
upload_2
.
id
)
create
(
:geo_file_registry
,
:attachment
,
file_id:
upload_3
.
id
)
expect
(
subject
.
count_synced
_attachments
).
to
eq
1
expect
(
subject
.
count_synced
).
to
eq
1
end
it
'ignores remote attachments'
do
...
...
@@ -122,26 +126,54 @@ describe Geo::AttachmentRegistryFinder, :geo do
create
(
:geo_file_registry
,
:attachment
,
file_id:
upload_2
.
id
)
create
(
:geo_file_registry
,
:attachment
,
file_id:
upload_3
.
id
)
expect
(
subject
.
count_synced_attachments
).
to
eq
1
expect
(
subject
.
count_synced
).
to
eq
1
end
end
context
'with selective sync by shard'
do
before
do
secondary
.
update!
(
selective_sync_type:
'shards'
,
selective_sync_shards:
[
'default'
])
end
it
'delegates to #legacy_find_synced'
do
expect
(
subject
).
to
receive
(
:legacy_find_synced
).
and_call_original
subject
.
count_synced
end
it
'counts attachments that has been synced'
do
create
(
:geo_file_registry
,
:attachment
,
file_id:
upload_1
.
id
,
success:
false
)
create
(
:geo_file_registry
,
:attachment
,
file_id:
upload_2
.
id
)
create
(
:geo_file_registry
,
:attachment
,
file_id:
upload_3
.
id
)
expect
(
subject
.
count_synced
).
to
eq
1
end
describe
'#count_failed_attachments'
do
it
'delegates to #legacy_find_failed_attachments'
do
it
'ignores remote attachments'
do
create
(
:geo_file_registry
,
:attachment
,
file_id:
upload_remote_1
.
id
)
create
(
:geo_file_registry
,
:attachment
,
file_id:
upload_2
.
id
)
create
(
:geo_file_registry
,
:attachment
,
file_id:
upload_3
.
id
)
expect
(
subject
.
count_synced
).
to
eq
1
end
end
end
describe
'#count_failed'
do
it
'delegates to #legacy_find_failed'
do
allow
(
subject
).
to
receive
(
:aggregate_pushdown_supported?
).
and_return
(
false
)
expect
(
subject
).
to
receive
(
:legacy_find_failed
_attachments
).
and_call_original
expect
(
subject
).
to
receive
(
:legacy_find_failed
).
and_call_original
subject
.
count_failed
_attachments
subject
.
count_failed
end
it
'delegates to #find_failed
_attachments
'
do
it
'delegates to #find_failed'
do
allow
(
subject
).
to
receive
(
:aggregate_pushdown_supported?
).
and_return
(
true
)
expect
(
subject
).
to
receive
(
:find_failed
_attachments
).
and_call_original
expect
(
subject
).
to
receive
(
:find_failed
).
and_call_original
subject
.
count_failed
_attachments
subject
.
count_failed
end
it
'counts attachments that sync has failed'
do
...
...
@@ -149,7 +181,7 @@ describe Geo::AttachmentRegistryFinder, :geo do
create
(
:geo_file_registry
,
:attachment
,
file_id:
upload_2
.
id
)
create
(
:geo_file_registry
,
:attachment
,
file_id:
upload_3
.
id
,
success:
false
)
expect
(
subject
.
count_failed
_attachments
).
to
eq
2
expect
(
subject
.
count_failed
).
to
eq
2
end
it
'ignores remote attachments'
do
...
...
@@ -157,31 +189,31 @@ describe Geo::AttachmentRegistryFinder, :geo do
create
(
:geo_file_registry
,
:attachment
,
file_id:
upload_2
.
id
,
success:
false
)
create
(
:geo_file_registry
,
:attachment
,
file_id:
upload_3
.
id
,
success:
false
)
expect
(
subject
.
count_failed
_attachments
).
to
eq
2
expect
(
subject
.
count_failed
).
to
eq
2
end
context
'with selective sync'
do
context
'with selective sync
by namespace
'
do
before
do
secondary
.
update!
(
selective_sync_type:
'namespaces'
,
namespaces:
[
synced_group
])
end
it
'delegates to #legacy_find_failed
_attachments
'
do
expect
(
subject
).
to
receive
(
:legacy_find_failed
_attachments
).
and_call_original
it
'delegates to #legacy_find_failed'
do
expect
(
subject
).
to
receive
(
:legacy_find_failed
).
and_call_original
subject
.
count_failed
_attachments
subject
.
count_failed
end
it
'counts attachments that sync has failed'
do
create
(
:geo_file_registry
,
:attachment
,
file_id:
upload_1
.
id
,
success:
false
)
create
(
:geo_file_registry
,
:attachment
,
file_id:
upload_3
.
id
)
expect
(
subject
.
count_failed
_attachments
).
to
eq
1
expect
(
subject
.
count_failed
).
to
eq
1
end
it
'does not count attachments of unsynced projects'
do
create
(
:geo_file_registry
,
:attachment
,
file_id:
upload_2
.
id
,
success:
false
)
expect
(
subject
.
count_failed
_attachments
).
to
eq
0
expect
(
subject
.
count_failed
).
to
eq
0
end
it
'ignores remote attachments'
do
...
...
@@ -190,51 +222,85 @@ describe Geo::AttachmentRegistryFinder, :geo do
create
(
:geo_file_registry
,
:attachment
,
file_id:
upload_3
.
id
,
success:
false
)
upload_1
.
update!
(
store:
ObjectStorage
::
Store
::
REMOTE
)
expect
(
subject
.
count_failed
_attachments
).
to
eq
1
expect
(
subject
.
count_failed
).
to
eq
1
end
end
context
'with selective sync by shard'
do
before
do
secondary
.
update!
(
selective_sync_type:
'shards'
,
selective_sync_shards:
[
'default'
])
end
describe
'#count_synced_missing_on_primary_attachments'
do
it
'delegates to #legacy_find_synced_missing_on_primary_attachments'
do
it
'delegates to #legacy_find_failed'
do
expect
(
subject
).
to
receive
(
:legacy_find_failed
).
and_call_original
subject
.
count_failed
end
it
'counts attachments that sync has failed'
do
create
(
:geo_file_registry
,
:attachment
,
file_id:
upload_1
.
id
,
success:
false
)
create
(
:geo_file_registry
,
:attachment
,
file_id:
upload_3
.
id
)
expect
(
subject
.
count_failed
).
to
eq
1
end
it
'does not count attachments of unsynced projects'
do
create
(
:geo_file_registry
,
:attachment
,
file_id:
upload_2
.
id
,
success:
false
)
expect
(
subject
.
count_failed
).
to
eq
0
end
it
'ignores remote attachments'
do
create
(
:geo_file_registry
,
:attachment
,
file_id:
upload_1
.
id
,
success:
false
)
create
(
:geo_file_registry
,
:attachment
,
file_id:
upload_2
.
id
,
success:
false
)
create
(
:geo_file_registry
,
:attachment
,
file_id:
upload_3
.
id
,
success:
false
)
upload_1
.
update!
(
store:
ObjectStorage
::
Store
::
REMOTE
)
expect
(
subject
.
count_failed
).
to
eq
1
end
end
end
describe
'#count_synced_missing_on_primary'
do
it
'delegates to #legacy_find_synced_missing_on_primary'
do
allow
(
subject
).
to
receive
(
:aggregate_pushdown_supported?
).
and_return
(
false
)
expect
(
subject
).
to
receive
(
:legacy_find_synced_missing_on_primary
_attachments
).
and_call_original
expect
(
subject
).
to
receive
(
:legacy_find_synced_missing_on_primary
).
and_call_original
subject
.
count_synced_missing_on_primary
_attachments
subject
.
count_synced_missing_on_primary
end
it
'delegates to #fdw_find_synced_missing_on_primary
_attachments
for PostgreSQL 10'
do
it
'delegates to #fdw_find_synced_missing_on_primary for PostgreSQL 10'
do
allow
(
subject
).
to
receive
(
:aggregate_pushdown_supported?
).
and_return
(
true
)
allow
(
subject
).
to
receive
(
:use_legacy_queries?
).
and_return
(
false
)
expect
(
subject
).
to
receive
(
:fdw_find_synced_missing_on_primary
_attachments
).
and_return
(
double
(
count:
1
))
expect
(
subject
).
to
receive
(
:fdw_find_synced_missing_on_primary
).
and_return
(
double
(
count:
1
))
subject
.
count_synced_missing_on_primary
_attachments
subject
.
count_synced_missing_on_primary
end
it
'counts attachments that have been synced and are missing on the primary'
do
create
(
:geo_file_registry
,
:attachment
,
file_id:
upload_1
.
id
,
missing_on_primary:
true
)
expect
(
subject
.
count_synced_missing_on_primary
_attachments
).
to
eq
1
expect
(
subject
.
count_synced_missing_on_primary
).
to
eq
1
end
it
'excludes attachments that are not missing on the primary'
do
create
(
:geo_file_registry
,
:attachment
,
file_id:
upload_1
.
id
)
expect
(
subject
.
count_synced_missing_on_primary
_attachments
).
to
eq
0
expect
(
subject
.
count_synced_missing_on_primary
).
to
eq
0
end
it
'excludes attachments that are not synced'
do
create
(
:geo_file_registry
,
:attachment
,
file_id:
upload_1
.
id
,
success:
false
,
missing_on_primary:
true
)
expect
(
subject
.
count_synced_missing_on_primary
_attachments
).
to
eq
0
expect
(
subject
.
count_synced_missing_on_primary
).
to
eq
0
end
it
'ignores remote attachments'
do
create
(
:geo_file_registry
,
:attachment
,
file_id:
upload_remote_1
.
id
,
missing_on_primary:
true
)
expect
(
subject
.
count_synced_missing_on_primary
_attachments
).
to
eq
0
expect
(
subject
.
count_synced_missing_on_primary
).
to
eq
0
end
context
'with selective sync'
do
...
...
@@ -242,10 +308,10 @@ describe Geo::AttachmentRegistryFinder, :geo do
secondary
.
update!
(
selective_sync_type:
'namespaces'
,
namespaces:
[
synced_group
])
end
it
'delegates to #legacy_find_synced_missing_on_primary
_attachments
'
do
expect
(
subject
).
to
receive
(
:legacy_find_synced_missing_on_primary
_attachments
).
and_call_original
it
'delegates to #legacy_find_synced_missing_on_primary'
do
expect
(
subject
).
to
receive
(
:legacy_find_synced_missing_on_primary
).
and_call_original
subject
.
count_synced_missing_on_primary
_attachments
subject
.
count_synced_missing_on_primary
end
it
'counts attachments that has been synced'
do
...
...
@@ -253,164 +319,36 @@ describe Geo::AttachmentRegistryFinder, :geo do
create
(
:geo_file_registry
,
:attachment
,
file_id:
upload_2
.
id
,
missing_on_primary:
true
)
create
(
:geo_file_registry
,
:attachment
,
file_id:
upload_3
.
id
,
missing_on_primary:
true
)
expect
(
subject
.
count_synced_missing_on_primary
_attachments
).
to
eq
2
expect
(
subject
.
count_synced_missing_on_primary
).
to
eq
2
end
it
'ignores remote attachments'
do
create
(
:geo_file_registry
,
:attachment
,
file_id:
upload_remote_1
.
id
,
missing_on_primary:
true
)
expect
(
subject
.
count_synced_missing_on_primary
_attachments
).
to
eq
0
expect
(
subject
.
count_synced_missing_on_primary
).
to
eq
0
end
end
end
end
shared_examples
'finds all the things'
do
describe
'#find_
synced_attachments
'
do
describe
'#find_
unsynced
'
do
it
'delegates to the correct method'
do
expect
(
subject
).
to
receive
(
"
#{
method_prefix
}
_find_
synced_attachments
"
.
to_sym
).
and_call_original
expect
(
subject
).
to
receive
(
"
#{
method_prefix
}
_find_
unsynced
"
.
to_sym
).
and_call_original
subject
.
find_synced_attachments
end
it
'returns synced avatars, attachment, personal snippets and files'
do
create
(
:geo_file_registry
,
:avatar
,
file_id:
upload_1
.
id
)
create
(
:geo_file_registry
,
:avatar
,
file_id:
upload_2
.
id
)
create
(
:geo_file_registry
,
:avatar
,
file_id:
upload_3
.
id
,
success:
false
)
create
(
:geo_file_registry
,
:avatar
,
file_id:
upload_6
.
id
)
create
(
:geo_file_registry
,
:avatar
,
file_id:
upload_7
.
id
)
create
(
:geo_file_registry
,
:lfs
,
file_id:
lfs_object
.
id
)
synced_attachments
=
subject
.
find_synced_attachments
expect
(
synced_attachments
).
to
match_ids
(
upload_1
,
upload_2
,
upload_6
,
upload_7
)
end
it
'only finds syncable attachments'
do
create
(
:geo_file_registry
,
:avatar
,
file_id:
upload_1
.
id
)
create
(
:geo_file_registry
,
:avatar
,
file_id:
upload_2
.
id
)
upload_1
.
update!
(
store:
ObjectStorage
::
Store
::
REMOTE
)
synced_attachments
=
subject
.
find_synced_attachments
expect
(
synced_attachments
).
to
match_ids
(
upload_2
)
end
context
'with selective sync by namespace'
do
it
'returns synced avatars, attachment, personal snippets and files'
do
create
(
:geo_file_registry
,
:avatar
,
file_id:
upload_1
.
id
)
create
(
:geo_file_registry
,
:avatar
,
file_id:
upload_2
.
id
)
create
(
:geo_file_registry
,
:avatar
,
file_id:
upload_3
.
id
)
create
(
:geo_file_registry
,
:avatar
,
file_id:
upload_4
.
id
)
create
(
:geo_file_registry
,
:avatar
,
file_id:
upload_5
.
id
,
success:
false
)
create
(
:geo_file_registry
,
:avatar
,
file_id:
upload_6
.
id
)
create
(
:geo_file_registry
,
:avatar
,
file_id:
upload_7
.
id
)
create
(
:geo_file_registry
,
:lfs
,
file_id:
lfs_object
.
id
)
secondary
.
update!
(
selective_sync_type:
'namespaces'
,
namespaces:
[
synced_group
])
synced_attachments
=
subject
.
find_synced_attachments
expect
(
synced_attachments
).
to
match_ids
(
upload_1
,
upload_3
,
upload_6
,
upload_7
)
end
end
context
'with selective sync by shard'
do
it
'returns synced avatars, attachment, personal snippets and files'
do
create
(
:geo_file_registry
,
:avatar
,
file_id:
upload_1
.
id
)
create
(
:geo_file_registry
,
:avatar
,
file_id:
upload_2
.
id
)
create
(
:geo_file_registry
,
:avatar
,
file_id:
upload_3
.
id
)
create
(
:geo_file_registry
,
:avatar
,
file_id:
upload_4
.
id
)
create
(
:geo_file_registry
,
:avatar
,
file_id:
upload_5
.
id
,
success:
false
)
create
(
:geo_file_registry
,
:avatar
,
file_id:
upload_6
.
id
)
create
(
:geo_file_registry
,
:avatar
,
file_id:
upload_7
.
id
)
create
(
:geo_file_registry
,
:lfs
,
file_id:
lfs_object
.
id
)
secondary
.
update!
(
selective_sync_type:
'shards'
,
selective_sync_shards:
[
'default'
])
synced_attachments
=
subject
.
find_synced_attachments
expect
(
synced_attachments
).
to
match_ids
(
upload_1
,
upload_3
,
upload_6
)
end
end
end
describe
'#find_failed_attachments'
do
it
'delegates to the correct method'
do
expect
(
subject
).
to
receive
(
"
#{
method_prefix
}
_find_failed_attachments"
.
to_sym
).
and_call_original
subject
.
find_failed_attachments
end
it
'returns failed avatars, attachment, personal snippets and files'
do
create
(
:geo_file_registry
,
:avatar
,
file_id:
upload_1
.
id
)
create
(
:geo_file_registry
,
:avatar
,
file_id:
upload_2
.
id
)
create
(
:geo_file_registry
,
:avatar
,
file_id:
upload_3
.
id
,
success:
false
)
create
(
:geo_file_registry
,
:avatar
,
file_id:
upload_6
.
id
,
success:
false
)
create
(
:geo_file_registry
,
:avatar
,
file_id:
upload_7
.
id
,
success:
false
)
create
(
:geo_file_registry
,
:lfs
,
file_id:
lfs_object
.
id
,
success:
false
)
failed_attachments
=
subject
.
find_failed_attachments
expect
(
failed_attachments
).
to
match_ids
(
upload_3
,
upload_6
,
upload_7
)
end
context
'with selective sync by namespace'
do
it
'returns failed avatars, attachment, personal snippets and files'
do
create
(
:geo_file_registry
,
:avatar
,
file_id:
upload_1
.
id
,
success:
false
)
create
(
:geo_file_registry
,
:avatar
,
file_id:
upload_2
.
id
)
create
(
:geo_file_registry
,
:avatar
,
file_id:
upload_3
.
id
,
success:
false
)
create
(
:geo_file_registry
,
:avatar
,
file_id:
upload_4
.
id
,
success:
false
)
create
(
:geo_file_registry
,
:avatar
,
file_id:
upload_5
.
id
)
create
(
:geo_file_registry
,
:avatar
,
file_id:
upload_6
.
id
,
success:
false
)
create
(
:geo_file_registry
,
:avatar
,
file_id:
upload_7
.
id
,
success:
false
)
create
(
:geo_file_registry
,
:lfs
,
file_id:
lfs_object
.
id
,
success:
false
)
secondary
.
update!
(
selective_sync_type:
'namespaces'
,
namespaces:
[
synced_group
])
failed_attachments
=
subject
.
find_failed_attachments
expect
(
failed_attachments
).
to
match_ids
(
upload_1
,
upload_3
,
upload_6
,
upload_7
)
end
end
context
'with selective sync by shard'
do
it
'returns failed avatars, attachment, personal snippets and files'
do
create
(
:geo_file_registry
,
:avatar
,
file_id:
upload_1
.
id
,
success:
false
)
create
(
:geo_file_registry
,
:avatar
,
file_id:
upload_2
.
id
)
create
(
:geo_file_registry
,
:avatar
,
file_id:
upload_3
.
id
,
success:
false
)
create
(
:geo_file_registry
,
:avatar
,
file_id:
upload_4
.
id
,
success:
false
)
create
(
:geo_file_registry
,
:avatar
,
file_id:
upload_5
.
id
)
create
(
:geo_file_registry
,
:avatar
,
file_id:
upload_6
.
id
,
success:
false
)
create
(
:geo_file_registry
,
:avatar
,
file_id:
upload_7
.
id
,
success:
false
)
create
(
:geo_file_registry
,
:lfs
,
file_id:
lfs_object
.
id
,
success:
false
)
secondary
.
update!
(
selective_sync_type:
'shards'
,
selective_sync_shards:
[
'default'
])
failed_attachments
=
subject
.
find_failed_attachments
expect
(
failed_attachments
).
to
match_ids
(
upload_1
,
upload_3
,
upload_6
)
end
end
end
describe
'#find_unsynced_attachments'
do
it
'delegates to the correct method'
do
expect
(
subject
).
to
receive
(
"
#{
method_prefix
}
_find_unsynced_attachments"
.
to_sym
).
and_call_original
subject
.
find_unsynced_attachments
(
batch_size:
10
)
subject
.
find_unsynced
(
batch_size:
10
)
end
it
'returns uploads without an entry on the tracking database'
do
create
(
:geo_file_registry
,
:avatar
,
file_id:
upload_1
.
id
,
success:
true
)
uploads
=
subject
.
find_unsynced
_attachments
(
batch_size:
10
)
uploads
=
subject
.
find_unsynced
(
batch_size:
10
)
expect
(
uploads
).
to
match_ids
(
upload_2
,
upload_3
,
upload_4
)
end
it
'excludes uploads in the except_file_ids option'
do
uploads
=
subject
.
find_unsynced
_attachments
(
batch_size:
10
,
except_file_ids:
[
upload_2
.
id
])
uploads
=
subject
.
find_unsynced
(
batch_size:
10
,
except_file_ids:
[
upload_2
.
id
])
expect
(
uploads
).
to
match_ids
(
upload_1
,
upload_3
,
upload_4
)
end
...
...
@@ -418,24 +356,24 @@ describe Geo::AttachmentRegistryFinder, :geo do
it
'excludes remote uploads'
do
upload_1
.
update!
(
store:
ObjectStorage
::
Store
::
REMOTE
)
uploads
=
subject
.
find_unsynced
_attachments
(
batch_size:
10
)
uploads
=
subject
.
find_unsynced
(
batch_size:
10
)
expect
(
uploads
).
to
match_ids
(
upload_2
,
upload_3
,
upload_4
)
end
end
describe
'#find_migrated_local
_attachments
'
do
describe
'#find_migrated_local'
do
it
'delegates to the correct method'
do
expect
(
subject
).
to
receive
(
"
#{
method_prefix
}
_find_migrated_local
_attachments
"
.
to_sym
).
and_call_original
expect
(
subject
).
to
receive
(
"
#{
method_prefix
}
_find_migrated_local"
.
to_sym
).
and_call_original
subject
.
find_migrated_local
_attachments
(
batch_size:
100
)
subject
.
find_migrated_local
(
batch_size:
100
)
end
it
'returns uploads stored remotely and successfully synced locally'
do
upload
=
create
(
:upload
,
:object_storage
,
model:
synced_group
)
create
(
:geo_file_registry
,
:avatar
,
file_id:
upload
.
id
)
uploads
=
subject
.
find_migrated_local
_attachments
(
batch_size:
100
)
uploads
=
subject
.
find_migrated_local
(
batch_size:
100
)
expect
(
uploads
).
to
match_ids
(
upload
)
end
...
...
@@ -443,7 +381,7 @@ describe Geo::AttachmentRegistryFinder, :geo do
it
'excludes uploads stored remotely, but not synced yet'
do
create
(
:upload
,
:object_storage
,
model:
synced_group
)
uploads
=
subject
.
find_migrated_local
_attachments
(
batch_size:
100
)
uploads
=
subject
.
find_migrated_local
(
batch_size:
100
)
expect
(
uploads
).
to
be_empty
end
...
...
@@ -451,7 +389,7 @@ describe Geo::AttachmentRegistryFinder, :geo do
it
'excludes synced uploads that are stored locally'
do
create
(
:geo_file_registry
,
:avatar
,
file_id:
upload_5
.
id
)
uploads
=
subject
.
find_migrated_local
_attachments
(
batch_size:
100
)
uploads
=
subject
.
find_migrated_local
(
batch_size:
100
)
expect
(
uploads
).
to
be_empty
end
...
...
@@ -462,54 +400,12 @@ describe Geo::AttachmentRegistryFinder, :geo do
create
(
:geo_file_registry
,
:avatar
,
file_id:
upload_a
.
id
,
success:
true
)
create
(
:geo_file_registry
,
:avatar
,
file_id:
upload_b
.
id
,
success:
true
)
uploads
=
subject
.
find_migrated_local
_attachments
(
batch_size:
10
,
except_file_ids:
[
upload_a
.
id
])
uploads
=
subject
.
find_migrated_local
(
batch_size:
10
,
except_file_ids:
[
upload_a
.
id
])
expect
(
uploads
).
to
match_ids
(
upload_b
)
end
end
end
# Disable transactions via :delete method because a foreign table
# can't see changes inside a transaction of a different connection.
context
'FDW'
,
:delete
do
before
do
skip
(
'FDW is not configured'
)
if
Gitlab
::
Database
.
postgresql?
&&
!
Gitlab
::
Geo
::
Fdw
.
enabled?
end
include_examples
'counts all the things'
include_examples
'finds all the things'
do
let
(
:method_prefix
)
{
'fdw'
}
end
context
'with selective sync'
do
before
do
secondary
.
update!
(
selective_sync_type:
'namespaces'
,
namespaces:
[
synced_group
])
end
it
'#find_synced_attachments falls back to legacy queries'
do
expect
(
subject
).
to
receive
(
:legacy_find_synced_attachments
)
subject
.
find_synced_attachments
end
it
'#find_failed_attachments falls back to legacy queries'
do
expect
(
subject
).
to
receive
(
:legacy_find_failed_attachments
)
subject
.
find_failed_attachments
end
end
end
context
'Legacy'
do
before
do
allow
(
Gitlab
::
Geo
::
Fdw
).
to
receive
(
:enabled?
).
and_return
(
false
)
end
include_examples
'counts all the things'
include_examples
'finds all the things'
do
let
(
:method_prefix
)
{
'legacy'
}
end
end
it_behaves_like
'a file registry finder'
end
ee/spec/finders/geo/job_artifact_registry_finder_spec.rb
View file @
b7444374
...
...
@@ -3,6 +3,10 @@ require 'spec_helper'
describe
Geo
::
JobArtifactRegistryFinder
,
:geo
do
include
::
EE
::
GeoHelpers
# Using let() instead of set() because set() does not work properly
# when using the :delete DatabaseCleaner strategy, which is required for FDW
# tests because a foreign table can't see changes inside a transaction of a
# different connection.
let
(
:secondary
)
{
create
(
:geo_node
)
}
let
(
:synced_group
)
{
create
(
:group
)
}
let
(
:synced_project
)
{
create
(
:project
,
group:
synced_group
)
}
...
...
@@ -23,7 +27,7 @@ describe Geo::JobArtifactRegistryFinder, :geo do
end
shared_examples
'counts all the things'
do
describe
'#count_syncable
_job_artifacts
'
do
describe
'#count_syncable'
do
before
do
job_artifact_1
job_artifact_2
...
...
@@ -32,19 +36,19 @@ describe Geo::JobArtifactRegistryFinder, :geo do
end
it
'counts job artifacts'
do
expect
(
subject
.
count_syncable
_job_artifacts
).
to
eq
4
expect
(
subject
.
count_syncable
).
to
eq
4
end
it
'ignores remote job artifacts'
do
job_artifact_1
.
update_column
(
:file_store
,
ObjectStorage
::
Store
::
REMOTE
)
expect
(
subject
.
count_syncable
_job_artifacts
).
to
eq
3
expect
(
subject
.
count_syncable
).
to
eq
3
end
it
'ignores expired job artifacts'
do
job_artifact_1
.
update_column
(
:expire_at
,
Date
.
yesterday
)
expect
(
subject
.
count_syncable
_job_artifacts
).
to
eq
3
expect
(
subject
.
count_syncable
).
to
eq
3
end
context
'with selective sync'
do
...
...
@@ -53,38 +57,38 @@ describe Geo::JobArtifactRegistryFinder, :geo do
end
it
'counts job artifacts'
do
expect
(
subject
.
count_syncable
_job_artifacts
).
to
eq
2
expect
(
subject
.
count_syncable
).
to
eq
2
end
it
'ignores remote job artifacts'
do
job_artifact_1
.
update_column
(
:file_store
,
ObjectStorage
::
Store
::
REMOTE
)
expect
(
subject
.
count_syncable
_job_artifacts
).
to
eq
1
expect
(
subject
.
count_syncable
).
to
eq
1
end
it
'ignores expired job artifacts'
do
job_artifact_1
.
update_column
(
:expire_at
,
Date
.
yesterday
)
expect
(
subject
.
count_syncable
_job_artifacts
).
to
eq
1
expect
(
subject
.
count_syncable
).
to
eq
1
end
end
end
describe
'#count_synced
_job_artifacts
'
do
it
'delegates to #legacy_find_synced
_job_artifacts
'
do
describe
'#count_synced'
do
it
'delegates to #legacy_find_synced'
do
allow
(
subject
).
to
receive
(
:aggregate_pushdown_supported?
).
and_return
(
false
)
expect
(
subject
).
to
receive
(
:legacy_find_synced
_job_artifacts
).
and_call_original
expect
(
subject
).
to
receive
(
:legacy_find_synced
).
and_call_original
subject
.
count_synced
_job_artifacts
subject
.
count_synced
end
it
'delegates to #find_synced
_job_artifacts
for PostgreSQL 10'
do
it
'delegates to #find_synced for PostgreSQL 10'
do
allow
(
subject
).
to
receive
(
:aggregate_pushdown_supported?
).
and_return
(
true
)
expect
(
subject
).
to
receive
(
:find_synced
_job_artifacts
).
and_call_original
expect
(
subject
).
to
receive
(
:find_synced
).
and_call_original
subject
.
count_synced
_job_artifacts
subject
.
count_synced
end
it
'counts job artifacts that have been synced'
do
...
...
@@ -92,7 +96,7 @@ describe Geo::JobArtifactRegistryFinder, :geo do
create
(
:geo_job_artifact_registry
,
artifact_id:
job_artifact_2
.
id
)
create
(
:geo_job_artifact_registry
,
artifact_id:
job_artifact_3
.
id
)
expect
(
subject
.
count_synced
_job_artifacts
).
to
eq
2
expect
(
subject
.
count_synced
).
to
eq
2
end
it
'ignores remote job artifacts'
do
...
...
@@ -100,7 +104,7 @@ describe Geo::JobArtifactRegistryFinder, :geo do
create
(
:geo_job_artifact_registry
,
artifact_id:
job_artifact_2
.
id
)
create
(
:geo_job_artifact_registry
,
artifact_id:
job_artifact_3
.
id
)
expect
(
subject
.
count_synced
_job_artifacts
).
to
eq
2
expect
(
subject
.
count_synced
).
to
eq
2
end
it
'ignores expired job artifacts'
do
...
...
@@ -109,7 +113,7 @@ describe Geo::JobArtifactRegistryFinder, :geo do
create
(
:geo_job_artifact_registry
,
artifact_id:
job_artifact_2
.
id
)
create
(
:geo_job_artifact_registry
,
artifact_id:
job_artifact_3
.
id
)
expect
(
subject
.
count_synced
_job_artifacts
).
to
eq
2
expect
(
subject
.
count_synced
).
to
eq
2
end
context
'with selective sync'
do
...
...
@@ -117,10 +121,10 @@ describe Geo::JobArtifactRegistryFinder, :geo do
secondary
.
update!
(
selective_sync_type:
'namespaces'
,
namespaces:
[
synced_group
])
end
it
'delegates to #legacy_find_synced
_job_artifacts
'
do
expect
(
subject
).
to
receive
(
:legacy_find_synced
_job_artifacts
).
and_call_original
it
'delegates to #legacy_find_synced'
do
expect
(
subject
).
to
receive
(
:legacy_find_synced
).
and_call_original
subject
.
count_synced
_job_artifacts
subject
.
count_synced
end
it
'counts job artifacts that has been synced'
do
...
...
@@ -128,7 +132,7 @@ describe Geo::JobArtifactRegistryFinder, :geo do
create
(
:geo_job_artifact_registry
,
artifact_id:
job_artifact_2
.
id
)
create
(
:geo_job_artifact_registry
,
artifact_id:
job_artifact_3
.
id
)
expect
(
subject
.
count_synced
_job_artifacts
).
to
eq
1
expect
(
subject
.
count_synced
).
to
eq
1
end
it
'ignores remote job artifacts'
do
...
...
@@ -136,7 +140,7 @@ describe Geo::JobArtifactRegistryFinder, :geo do
create
(
:geo_job_artifact_registry
,
artifact_id:
job_artifact_2
.
id
)
create
(
:geo_job_artifact_registry
,
artifact_id:
job_artifact_3
.
id
)
expect
(
subject
.
count_synced
_job_artifacts
).
to
eq
1
expect
(
subject
.
count_synced
).
to
eq
1
end
it
'ignores expired job artifacts'
do
...
...
@@ -145,26 +149,26 @@ describe Geo::JobArtifactRegistryFinder, :geo do
create
(
:geo_job_artifact_registry
,
artifact_id:
job_artifact_2
.
id
)
create
(
:geo_job_artifact_registry
,
artifact_id:
job_artifact_3
.
id
)
expect
(
subject
.
count_synced
_job_artifacts
).
to
eq
1
expect
(
subject
.
count_synced
).
to
eq
1
end
end
end
describe
'#count_failed
_job_artifacts
'
do
it
'delegates to #legacy_find_failed
_job_artifacts
'
do
describe
'#count_failed'
do
it
'delegates to #legacy_find_failed'
do
allow
(
subject
).
to
receive
(
:aggregate_pushdown_supported?
).
and_return
(
false
)
expect
(
subject
).
to
receive
(
:legacy_find_failed
_job_artifacts
).
and_call_original
expect
(
subject
).
to
receive
(
:legacy_find_failed
).
and_call_original
subject
.
count_failed
_job_artifacts
subject
.
count_failed
end
it
'delegates to #find_failed
_job_artifacts
'
do
it
'delegates to #find_failed'
do
allow
(
subject
).
to
receive
(
:aggregate_pushdown_supported?
).
and_return
(
true
)
expect
(
subject
).
to
receive
(
:find_failed
_job_artifacts
).
and_call_original
expect
(
subject
).
to
receive
(
:find_failed
).
and_call_original
subject
.
count_failed
_job_artifacts
subject
.
count_failed
end
it
'counts job artifacts that sync has failed'
do
...
...
@@ -172,7 +176,7 @@ describe Geo::JobArtifactRegistryFinder, :geo do
create
(
:geo_job_artifact_registry
,
artifact_id:
job_artifact_2
.
id
)
create
(
:geo_job_artifact_registry
,
artifact_id:
job_artifact_3
.
id
,
success:
false
)
expect
(
subject
.
count_failed
_job_artifacts
).
to
eq
2
expect
(
subject
.
count_failed
).
to
eq
2
end
it
'ignores remote job artifacts'
do
...
...
@@ -180,7 +184,7 @@ describe Geo::JobArtifactRegistryFinder, :geo do
create
(
:geo_job_artifact_registry
,
artifact_id:
job_artifact_2
.
id
,
success:
false
)
create
(
:geo_job_artifact_registry
,
artifact_id:
job_artifact_3
.
id
,
success:
false
)
expect
(
subject
.
count_failed
_job_artifacts
).
to
eq
2
expect
(
subject
.
count_failed
).
to
eq
2
end
it
'ignores expired job artifacts'
do
...
...
@@ -189,7 +193,7 @@ describe Geo::JobArtifactRegistryFinder, :geo do
create
(
:geo_job_artifact_registry
,
artifact_id:
job_artifact_2
.
id
,
success:
false
)
create
(
:geo_job_artifact_registry
,
artifact_id:
job_artifact_3
.
id
,
success:
false
)
expect
(
subject
.
count_failed
_job_artifacts
).
to
eq
2
expect
(
subject
.
count_failed
).
to
eq
2
end
context
'with selective sync'
do
...
...
@@ -197,23 +201,23 @@ describe Geo::JobArtifactRegistryFinder, :geo do
secondary
.
update!
(
selective_sync_type:
'namespaces'
,
namespaces:
[
synced_group
])
end
it
'delegates to #legacy_find_failed
_job_artifacts
'
do
expect
(
subject
).
to
receive
(
:legacy_find_failed
_job_artifacts
).
and_call_original
it
'delegates to #legacy_find_failed'
do
expect
(
subject
).
to
receive
(
:legacy_find_failed
).
and_call_original
subject
.
count_failed
_job_artifacts
subject
.
count_failed
end
it
'counts job artifacts that sync has failed'
do
create
(
:geo_job_artifact_registry
,
artifact_id:
job_artifact_1
.
id
,
success:
false
)
create
(
:geo_job_artifact_registry
,
artifact_id:
job_artifact_3
.
id
)
expect
(
subject
.
count_failed
_job_artifacts
).
to
eq
1
expect
(
subject
.
count_failed
).
to
eq
1
end
it
'does not count job artifacts of unsynced projects'
do
create
(
:geo_job_artifact_registry
,
artifact_id:
job_artifact_2
.
id
,
success:
false
)
expect
(
subject
.
count_failed
_job_artifacts
).
to
eq
0
expect
(
subject
.
count_failed
).
to
eq
0
end
it
'ignores remote job artifacts'
do
...
...
@@ -222,7 +226,7 @@ describe Geo::JobArtifactRegistryFinder, :geo do
create
(
:geo_job_artifact_registry
,
artifact_id:
job_artifact_2
.
id
,
success:
false
)
create
(
:geo_job_artifact_registry
,
artifact_id:
job_artifact_3
.
id
,
success:
false
)
expect
(
subject
.
count_failed
_job_artifacts
).
to
eq
1
expect
(
subject
.
count_failed
).
to
eq
1
end
it
'ignores expired job artifacts'
do
...
...
@@ -231,57 +235,57 @@ describe Geo::JobArtifactRegistryFinder, :geo do
create
(
:geo_job_artifact_registry
,
artifact_id:
job_artifact_2
.
id
,
success:
false
)
create
(
:geo_job_artifact_registry
,
artifact_id:
job_artifact_3
.
id
,
success:
false
)
expect
(
subject
.
count_failed
_job_artifacts
).
to
eq
1
expect
(
subject
.
count_failed
).
to
eq
1
end
end
end
describe
'#count_synced_missing_on_primary
_job_artifacts
'
do
it
'delegates to #legacy_find_synced_missing_on_primary
_job_artifacts
'
do
describe
'#count_synced_missing_on_primary'
do
it
'delegates to #legacy_find_synced_missing_on_primary'
do
allow
(
subject
).
to
receive
(
:aggregate_pushdown_supported?
).
and_return
(
false
)
expect
(
subject
).
to
receive
(
:legacy_find_synced_missing_on_primary
_job_artifacts
).
and_call_original
expect
(
subject
).
to
receive
(
:legacy_find_synced_missing_on_primary
).
and_call_original
subject
.
count_synced_missing_on_primary
_job_artifacts
subject
.
count_synced_missing_on_primary
end
it
'delegates to #find_synced_missing_on_primary
_job_artifacts
for PostgreSQL 10'
do
it
'delegates to #find_synced_missing_on_primary for PostgreSQL 10'
do
allow
(
subject
).
to
receive
(
:aggregate_pushdown_supported?
).
and_return
(
true
)
expect
(
subject
).
to
receive
(
:find_synced_missing_on_primary
_job_artifacts
).
and_call_original
expect
(
subject
).
to
receive
(
:find_synced_missing_on_primary
).
and_call_original
subject
.
count_synced_missing_on_primary
_job_artifacts
subject
.
count_synced_missing_on_primary
end
it
'counts job artifacts that have been synced and are missing on the primary'
do
create
(
:geo_job_artifact_registry
,
artifact_id:
job_artifact_1
.
id
,
missing_on_primary:
true
)
expect
(
subject
.
count_synced_missing_on_primary
_job_artifacts
).
to
eq
1
expect
(
subject
.
count_synced_missing_on_primary
).
to
eq
1
end
it
'excludes job artifacts that are not missing on the primary'
do
create
(
:geo_job_artifact_registry
,
artifact_id:
job_artifact_1
.
id
)
expect
(
subject
.
count_synced_missing_on_primary
_job_artifacts
).
to
eq
0
expect
(
subject
.
count_synced_missing_on_primary
).
to
eq
0
end
it
'excludes job artifacts that are not synced'
do
create
(
:geo_job_artifact_registry
,
artifact_id:
job_artifact_1
.
id
,
success:
false
,
missing_on_primary:
true
)
expect
(
subject
.
count_synced_missing_on_primary
_job_artifacts
).
to
eq
0
expect
(
subject
.
count_synced_missing_on_primary
).
to
eq
0
end
it
'ignores remote job artifacts'
do
create
(
:geo_job_artifact_registry
,
artifact_id:
job_artifact_remote_1
.
id
,
missing_on_primary:
true
)
expect
(
subject
.
count_synced_missing_on_primary
_job_artifacts
).
to
eq
0
expect
(
subject
.
count_synced_missing_on_primary
).
to
eq
0
end
it
'ignores expired job artifacts'
do
job_artifact_1
.
update_column
(
:expire_at
,
Date
.
yesterday
)
create
(
:geo_job_artifact_registry
,
artifact_id:
job_artifact_1
.
id
,
missing_on_primary:
true
)
expect
(
subject
.
count_synced_missing_on_primary
_job_artifacts
).
to
eq
0
expect
(
subject
.
count_synced_missing_on_primary
).
to
eq
0
end
context
'with selective sync'
do
...
...
@@ -289,10 +293,10 @@ describe Geo::JobArtifactRegistryFinder, :geo do
secondary
.
update!
(
selective_sync_type:
'namespaces'
,
namespaces:
[
synced_group
])
end
it
'delegates to #legacy_find_synced_missing_on_primary
_job_artifacts
'
do
expect
(
subject
).
to
receive
(
:legacy_find_synced_missing_on_primary
_job_artifacts
).
and_call_original
it
'delegates to #legacy_find_synced_missing_on_primary'
do
expect
(
subject
).
to
receive
(
:legacy_find_synced_missing_on_primary
).
and_call_original
subject
.
count_synced_missing_on_primary
_job_artifacts
subject
.
count_synced_missing_on_primary
end
it
'counts job artifacts that has been synced'
do
...
...
@@ -300,38 +304,38 @@ describe Geo::JobArtifactRegistryFinder, :geo do
create
(
:geo_job_artifact_registry
,
artifact_id:
job_artifact_2
.
id
,
missing_on_primary:
true
)
create
(
:geo_job_artifact_registry
,
artifact_id:
job_artifact_3
.
id
,
missing_on_primary:
true
)
expect
(
subject
.
count_synced_missing_on_primary
_job_artifacts
).
to
eq
2
expect
(
subject
.
count_synced_missing_on_primary
).
to
eq
2
end
it
'ignores remote job artifacts'
do
create
(
:geo_job_artifact_registry
,
artifact_id:
job_artifact_remote_1
.
id
,
missing_on_primary:
true
)
expect
(
subject
.
count_synced_missing_on_primary
_job_artifacts
).
to
eq
0
expect
(
subject
.
count_synced_missing_on_primary
).
to
eq
0
end
it
'ignores expired job artifacts'
do
job_artifact_1
.
update_column
(
:expire_at
,
Date
.
yesterday
)
create
(
:geo_job_artifact_registry
,
artifact_id:
job_artifact_1
.
id
,
missing_on_primary:
true
)
expect
(
subject
.
count_synced_missing_on_primary
_job_artifacts
).
to
eq
0
expect
(
subject
.
count_synced_missing_on_primary
).
to
eq
0
end
end
end
end
shared_examples
'finds all the things'
do
describe
'#find_unsynced
_job_artifacts
'
do
describe
'#find_unsynced'
do
it
'delegates to the correct method'
do
expect
(
subject
).
to
receive
(
"
#{
method_prefix
}
_find_unsynced
_job_artifacts
"
.
to_sym
).
and_call_original
expect
(
subject
).
to
receive
(
"
#{
method_prefix
}
_find_unsynced"
.
to_sym
).
and_call_original
subject
.
find_unsynced
_job_artifacts
(
batch_size:
10
)
subject
.
find_unsynced
(
batch_size:
10
)
end
it
'returns job artifacts without an entry on the tracking database'
do
create
(
:geo_job_artifact_registry
,
artifact_id:
job_artifact_1
.
id
,
success:
true
)
create
(
:geo_job_artifact_registry
,
artifact_id:
job_artifact_3
.
id
,
success:
false
)
job_artifacts
=
subject
.
find_unsynced
_job_artifacts
(
batch_size:
10
)
job_artifacts
=
subject
.
find_unsynced
(
batch_size:
10
)
expect
(
job_artifacts
).
to
match_ids
(
job_artifact_2
,
job_artifact_4
)
end
...
...
@@ -340,7 +344,7 @@ describe Geo::JobArtifactRegistryFinder, :geo do
create
(
:geo_job_artifact_registry
,
artifact_id:
job_artifact_1
.
id
,
success:
true
)
create
(
:geo_job_artifact_registry
,
artifact_id:
job_artifact_3
.
id
,
success:
false
)
job_artifacts
=
subject
.
find_unsynced
_job_artifacts
(
batch_size:
10
,
except_artifact_ids:
[
job_artifact_2
.
id
])
job_artifacts
=
subject
.
find_unsynced
(
batch_size:
10
,
except_artifact_ids:
[
job_artifact_2
.
id
])
expect
(
job_artifacts
).
to
match_ids
(
job_artifact_4
)
end
...
...
@@ -348,7 +352,7 @@ describe Geo::JobArtifactRegistryFinder, :geo do
it
'ignores remote job artifacts'
do
job_artifact_2
.
update_column
(
:file_store
,
ObjectStorage
::
Store
::
REMOTE
)
job_artifacts
=
subject
.
find_unsynced
_job_artifacts
(
batch_size:
10
)
job_artifacts
=
subject
.
find_unsynced
(
batch_size:
10
)
expect
(
job_artifacts
).
to
match_ids
(
job_artifact_4
)
end
...
...
@@ -356,24 +360,24 @@ describe Geo::JobArtifactRegistryFinder, :geo do
it
'ignores expired job artifacts'
do
job_artifact_2
.
update_column
(
:expire_at
,
Date
.
yesterday
)
job_artifacts
=
subject
.
find_unsynced
_job_artifacts
(
batch_size:
10
)
job_artifacts
=
subject
.
find_unsynced
(
batch_size:
10
)
expect
(
job_artifacts
).
to
match_ids
(
job_artifact_4
)
end
end
describe
'#find_migrated_local
_job_artifacts
'
do
describe
'#find_migrated_local'
do
it
'delegates to the correct method'
do
expect
(
subject
).
to
receive
(
"
#{
method_prefix
}
_find_migrated_local
_job_artifacts
"
.
to_sym
).
and_call_original
expect
(
subject
).
to
receive
(
"
#{
method_prefix
}
_find_migrated_local"
.
to_sym
).
and_call_original
subject
.
find_migrated_local
_job_artifacts
(
batch_size:
10
)
subject
.
find_migrated_local
(
batch_size:
10
)
end
it
'returns job artifacts remotely and successfully synced locally'
do
job_artifact
=
create
(
:ci_job_artifact
,
:remote_store
,
project:
synced_project
)
create
(
:geo_job_artifact_registry
,
artifact_id:
job_artifact
.
id
)
job_artifacts
=
subject
.
find_migrated_local
_job_artifacts
(
batch_size:
10
)
job_artifacts
=
subject
.
find_migrated_local
(
batch_size:
10
)
expect
(
job_artifacts
).
to
match_ids
(
job_artifact
)
end
...
...
@@ -381,7 +385,7 @@ describe Geo::JobArtifactRegistryFinder, :geo do
it
'excludes job artifacts stored remotely, but not synced yet'
do
create
(
:ci_job_artifact
,
:remote_store
,
project:
synced_project
)
job_artifacts
=
subject
.
find_migrated_local
_job_artifacts
(
batch_size:
10
)
job_artifacts
=
subject
.
find_migrated_local
(
batch_size:
10
)
expect
(
job_artifacts
).
to
be_empty
end
...
...
@@ -389,7 +393,7 @@ describe Geo::JobArtifactRegistryFinder, :geo do
it
'excludes synced job artifacts that are stored locally'
do
create
(
:geo_job_artifact_registry
,
artifact_id:
job_artifact_1
.
id
)
job_artifacts
=
subject
.
find_migrated_local
_job_artifacts
(
batch_size:
10
)
job_artifacts
=
subject
.
find_migrated_local
(
batch_size:
10
)
expect
(
job_artifacts
).
to
be_empty
end
...
...
@@ -398,7 +402,7 @@ describe Geo::JobArtifactRegistryFinder, :geo do
create
(
:geo_job_artifact_registry
,
artifact_id:
job_artifact_remote_1
.
id
)
create
(
:geo_job_artifact_registry
,
artifact_id:
job_artifact_remote_2
.
id
)
job_artifacts
=
subject
.
find_migrated_local
_job_artifacts
(
batch_size:
10
,
except_artifact_ids:
[
job_artifact_remote_1
.
id
])
job_artifacts
=
subject
.
find_migrated_local
(
batch_size:
10
,
except_artifact_ids:
[
job_artifact_remote_1
.
id
])
expect
(
job_artifacts
).
to
match_ids
(
job_artifact_remote_2
)
end
...
...
@@ -407,36 +411,12 @@ describe Geo::JobArtifactRegistryFinder, :geo do
job_artifact
=
create
(
:ci_job_artifact
,
:remote_store
,
project:
synced_project
,
expire_at:
Date
.
yesterday
)
create
(
:geo_job_artifact_registry
,
artifact_id:
job_artifact
.
id
)
job_artifacts
=
subject
.
find_migrated_local
_job_artifacts
(
batch_size:
10
)
job_artifacts
=
subject
.
find_migrated_local
(
batch_size:
10
)
expect
(
job_artifacts
).
to
match_ids
(
job_artifact
)
end
end
end
# Disable transactions via :delete method because a foreign table
# can't see changes inside a transaction of a different connection.
context
'FDW'
,
:delete
do
before
do
skip
(
'FDW is not configured'
)
if
Gitlab
::
Database
.
postgresql?
&&
!
Gitlab
::
Geo
::
Fdw
.
enabled?
end
include_examples
'counts all the things'
include_examples
'finds all the things'
do
let
(
:method_prefix
)
{
'fdw'
}
end
end
context
'Legacy'
do
before
do
allow
(
Gitlab
::
Geo
::
Fdw
).
to
receive
(
:enabled?
).
and_return
(
false
)
end
include_examples
'counts all the things'
include_examples
'finds all the things'
do
let
(
:method_prefix
)
{
'legacy'
}
end
end
it_behaves_like
'a file registry finder'
end
ee/spec/finders/geo/lfs_object_registry_finder_spec.rb
View file @
b7444374
...
...
@@ -3,6 +3,10 @@ require 'spec_helper'
describe
Geo
::
LfsObjectRegistryFinder
,
:geo
do
include
::
EE
::
GeoHelpers
# Using let() instead of set() because set() does not work properly
# when using the :delete DatabaseCleaner strategy, which is required for FDW
# tests because a foreign table can't see changes inside a transaction of a
# different connection.
let
(
:secondary
)
{
create
(
:geo_node
)
}
let
(
:synced_group
)
{
create
(
:group
)
}
let
(
:synced_project
)
{
create
(
:project
,
group:
synced_group
)
}
...
...
@@ -23,7 +27,7 @@ describe Geo::LfsObjectRegistryFinder, :geo do
end
shared_examples
'counts all the things'
do
describe
'#count_syncable
_lfs_objects
'
do
describe
'#count_syncable'
do
before
do
lfs_object_1
lfs_object_2
...
...
@@ -32,13 +36,13 @@ describe Geo::LfsObjectRegistryFinder, :geo do
end
it
'counts LFS objects'
do
expect
(
subject
.
count_syncable
_lfs_objects
).
to
eq
4
expect
(
subject
.
count_syncable
).
to
eq
4
end
it
'ignores remote LFS objects'
do
lfs_object_1
.
update_column
(
:file_store
,
ObjectStorage
::
Store
::
REMOTE
)
expect
(
subject
.
count_syncable
_lfs_objects
).
to
eq
3
expect
(
subject
.
count_syncable
).
to
eq
3
end
context
'with selective sync'
do
...
...
@@ -53,33 +57,33 @@ describe Geo::LfsObjectRegistryFinder, :geo do
end
it
'counts LFS objects'
do
expect
(
subject
.
count_syncable
_lfs_objects
).
to
eq
2
expect
(
subject
.
count_syncable
).
to
eq
2
end
it
'ignores remote LFS objects'
do
lfs_object_1
.
update_column
(
:file_store
,
ObjectStorage
::
Store
::
REMOTE
)
expect
(
subject
.
count_syncable
_lfs_objects
).
to
eq
1
expect
(
subject
.
count_syncable
).
to
eq
1
end
end
end
describe
'#count_synced
_lfs_objects
'
do
it
'delegates to #legacy_find_synced
_lfs_objects
'
do
describe
'#count_synced'
do
it
'delegates to #legacy_find_synced'
do
allow
(
subject
).
to
receive
(
:aggregate_pushdown_supported?
).
and_return
(
false
)
expect
(
subject
).
to
receive
(
:legacy_find_synced
_lfs_objects
).
and_call_original
expect
(
subject
).
to
receive
(
:legacy_find_synced
).
and_call_original
subject
.
count_synced
_lfs_objects
subject
.
count_synced
end
it
'delegates to #fdw_find_synced
_lfs_objects
for PostgreSQL 10'
do
it
'delegates to #fdw_find_synced for PostgreSQL 10'
do
allow
(
subject
).
to
receive
(
:aggregate_pushdown_supported?
).
and_return
(
true
)
allow
(
subject
).
to
receive
(
:use_legacy_queries?
).
and_return
(
false
)
expect
(
subject
).
to
receive
(
:fdw_find_synced
_lfs_objects
).
and_return
(
double
(
count:
1
))
expect
(
subject
).
to
receive
(
:fdw_find_synced
).
and_return
(
double
(
count:
1
))
subject
.
count_synced
_lfs_objects
subject
.
count_synced
end
it
'counts LFS objects that has been synced'
do
...
...
@@ -87,7 +91,7 @@ describe Geo::LfsObjectRegistryFinder, :geo do
create
(
:geo_file_registry
,
:lfs
,
file_id:
lfs_object_2
.
id
)
create
(
:geo_file_registry
,
:lfs
,
file_id:
lfs_object_3
.
id
)
expect
(
subject
.
count_synced
_lfs_objects
).
to
eq
2
expect
(
subject
.
count_synced
).
to
eq
2
end
it
'ignores remote LFS objects'
do
...
...
@@ -95,7 +99,7 @@ describe Geo::LfsObjectRegistryFinder, :geo do
create
(
:geo_file_registry
,
:lfs
,
file_id:
lfs_object_2
.
id
)
create
(
:geo_file_registry
,
:lfs
,
file_id:
lfs_object_3
.
id
)
expect
(
subject
.
count_synced
_lfs_objects
).
to
eq
2
expect
(
subject
.
count_synced
).
to
eq
2
end
context
'with selective sync'
do
...
...
@@ -109,10 +113,10 @@ describe Geo::LfsObjectRegistryFinder, :geo do
secondary
.
update!
(
selective_sync_type:
'namespaces'
,
namespaces:
[
synced_group
])
end
it
'delegates to #legacy_find_synced
_lfs_objects
'
do
expect
(
subject
).
to
receive
(
:legacy_find_synced
_lfs_objects
).
and_call_original
it
'delegates to #legacy_find_synced'
do
expect
(
subject
).
to
receive
(
:legacy_find_synced
).
and_call_original
subject
.
count_synced
_lfs_objects
subject
.
count_synced
end
it
'counts LFS objects that has been synced'
do
...
...
@@ -120,7 +124,7 @@ describe Geo::LfsObjectRegistryFinder, :geo do
create
(
:geo_file_registry
,
:lfs
,
file_id:
lfs_object_2
.
id
)
create
(
:geo_file_registry
,
:lfs
,
file_id:
lfs_object_3
.
id
)
expect
(
subject
.
count_synced
_lfs_objects
).
to
eq
1
expect
(
subject
.
count_synced
).
to
eq
1
end
it
'ignores remote LFS objects'
do
...
...
@@ -129,26 +133,26 @@ describe Geo::LfsObjectRegistryFinder, :geo do
create
(
:geo_file_registry
,
:lfs
,
file_id:
lfs_object_3
.
id
)
lfs_object_1
.
update_column
(
:file_store
,
ObjectStorage
::
Store
::
REMOTE
)
expect
(
subject
.
count_synced
_lfs_objects
).
to
eq
1
expect
(
subject
.
count_synced
).
to
eq
1
end
end
end
describe
'#count_failed
_lfs_objects
'
do
it
'delegates to #legacy_find_failed
_lfs_objects
'
do
describe
'#count_failed'
do
it
'delegates to #legacy_find_failed'
do
allow
(
subject
).
to
receive
(
:aggregate_pushdown_supported?
).
and_return
(
false
)
expect
(
subject
).
to
receive
(
:legacy_find_failed
_lfs_objects
).
and_call_original
expect
(
subject
).
to
receive
(
:legacy_find_failed
).
and_call_original
subject
.
count_failed
_lfs_objects
subject
.
count_failed
end
it
'delegates to #find_failed
_lfs_objects
'
do
it
'delegates to #find_failed'
do
allow
(
subject
).
to
receive
(
:aggregate_pushdown_supported?
).
and_return
(
true
)
expect
(
subject
).
to
receive
(
:find_failed
_lfs_objects
).
and_call_original
expect
(
subject
).
to
receive
(
:find_failed
).
and_call_original
subject
.
count_failed
_lfs_objects
subject
.
count_failed
end
it
'counts LFS objects that sync has failed'
do
...
...
@@ -156,7 +160,7 @@ describe Geo::LfsObjectRegistryFinder, :geo do
create
(
:geo_file_registry
,
:lfs
,
file_id:
lfs_object_2
.
id
)
create
(
:geo_file_registry
,
:lfs
,
file_id:
lfs_object_3
.
id
,
success:
false
)
expect
(
subject
.
count_failed
_lfs_objects
).
to
eq
2
expect
(
subject
.
count_failed
).
to
eq
2
end
it
'ignores remote LFS objects'
do
...
...
@@ -164,7 +168,7 @@ describe Geo::LfsObjectRegistryFinder, :geo do
create
(
:geo_file_registry
,
:lfs
,
file_id:
lfs_object_2
.
id
,
success:
false
)
create
(
:geo_file_registry
,
:lfs
,
file_id:
lfs_object_3
.
id
,
success:
false
)
expect
(
subject
.
count_failed
_lfs_objects
).
to
eq
2
expect
(
subject
.
count_failed
).
to
eq
2
end
context
'with selective sync'
do
...
...
@@ -178,10 +182,10 @@ describe Geo::LfsObjectRegistryFinder, :geo do
secondary
.
update!
(
selective_sync_type:
'namespaces'
,
namespaces:
[
synced_group
])
end
it
'delegates to #legacy_find_failed
_lfs_objects
'
do
expect
(
subject
).
to
receive
(
:legacy_find_failed
_lfs_objects
).
and_call_original
it
'delegates to #legacy_find_failed'
do
expect
(
subject
).
to
receive
(
:legacy_find_failed
).
and_call_original
subject
.
count_failed
_lfs_objects
subject
.
count_failed
end
it
'counts LFS objects that sync has failed'
do
...
...
@@ -189,7 +193,7 @@ describe Geo::LfsObjectRegistryFinder, :geo do
create
(
:geo_file_registry
,
:lfs
,
file_id:
lfs_object_2
.
id
)
create
(
:geo_file_registry
,
:lfs
,
file_id:
lfs_object_3
.
id
,
success:
false
)
expect
(
subject
.
count_failed
_lfs_objects
).
to
eq
1
expect
(
subject
.
count_failed
).
to
eq
1
end
it
'ignores remote LFS objects'
do
...
...
@@ -198,51 +202,51 @@ describe Geo::LfsObjectRegistryFinder, :geo do
create
(
:geo_file_registry
,
:lfs
,
file_id:
lfs_object_3
.
id
,
success:
false
)
lfs_object_1
.
update_column
(
:file_store
,
ObjectStorage
::
Store
::
REMOTE
)
expect
(
subject
.
count_failed
_lfs_objects
).
to
eq
1
expect
(
subject
.
count_failed
).
to
eq
1
end
end
end
describe
'#count_synced_missing_on_primary
_lfs_objects
'
do
it
'delegates to #legacy_find_synced_missing_on_primary
_lfs_objects
'
do
describe
'#count_synced_missing_on_primary'
do
it
'delegates to #legacy_find_synced_missing_on_primary'
do
allow
(
subject
).
to
receive
(
:aggregate_pushdown_supported?
).
and_return
(
false
)
expect
(
subject
).
to
receive
(
:legacy_find_synced_missing_on_primary
_lfs_objects
).
and_call_original
expect
(
subject
).
to
receive
(
:legacy_find_synced_missing_on_primary
).
and_call_original
subject
.
count_synced_missing_on_primary
_lfs_objects
subject
.
count_synced_missing_on_primary
end
it
'delegates to #fdw_find_synced_missing_on_primary
_lfs_objects
for PostgreSQL 10'
do
it
'delegates to #fdw_find_synced_missing_on_primary for PostgreSQL 10'
do
allow
(
subject
).
to
receive
(
:aggregate_pushdown_supported?
).
and_return
(
true
)
allow
(
subject
).
to
receive
(
:use_legacy_queries?
).
and_return
(
false
)
expect
(
subject
).
to
receive
(
:fdw_find_synced_missing_on_primary
_lfs_objects
).
and_return
(
double
(
count:
1
))
expect
(
subject
).
to
receive
(
:fdw_find_synced_missing_on_primary
).
and_return
(
double
(
count:
1
))
subject
.
count_synced_missing_on_primary
_lfs_objects
subject
.
count_synced_missing_on_primary
end
it
'counts LFS objects that have been synced and are missing on the primary'
do
create
(
:geo_file_registry
,
:lfs
,
file_id:
lfs_object_1
.
id
,
missing_on_primary:
true
)
expect
(
subject
.
count_synced_missing_on_primary
_lfs_objects
).
to
eq
1
expect
(
subject
.
count_synced_missing_on_primary
).
to
eq
1
end
it
'excludes LFS objects that are not missing on the primary'
do
create
(
:geo_file_registry
,
:lfs
,
file_id:
lfs_object_1
.
id
)
expect
(
subject
.
count_synced_missing_on_primary
_lfs_objects
).
to
eq
0
expect
(
subject
.
count_synced_missing_on_primary
).
to
eq
0
end
it
'excludes LFS objects that are not synced'
do
create
(
:geo_file_registry
,
:lfs
,
file_id:
lfs_object_1
.
id
,
success:
false
,
missing_on_primary:
true
)
expect
(
subject
.
count_synced_missing_on_primary
_lfs_objects
).
to
eq
0
expect
(
subject
.
count_synced_missing_on_primary
).
to
eq
0
end
it
'ignores remote LFS objects'
do
create
(
:geo_file_registry
,
:lfs
,
file_id:
lfs_object_remote_1
.
id
,
missing_on_primary:
true
)
expect
(
subject
.
count_synced_missing_on_primary
_lfs_objects
).
to
eq
0
expect
(
subject
.
count_synced_missing_on_primary
).
to
eq
0
end
context
'with selective sync'
do
...
...
@@ -256,10 +260,10 @@ describe Geo::LfsObjectRegistryFinder, :geo do
secondary
.
update!
(
selective_sync_type:
'namespaces'
,
namespaces:
[
synced_group
])
end
it
'delegates to #legacy_find_synced_missing_on_primary
_lfs_objects
'
do
expect
(
subject
).
to
receive
(
:legacy_find_synced_missing_on_primary
_lfs_objects
).
and_call_original
it
'delegates to #legacy_find_synced_missing_on_primary'
do
expect
(
subject
).
to
receive
(
:legacy_find_synced_missing_on_primary
).
and_call_original
subject
.
count_synced_missing_on_primary
_lfs_objects
subject
.
count_synced_missing_on_primary
end
it
'counts LFS objects that has been synced'
do
...
...
@@ -267,31 +271,31 @@ describe Geo::LfsObjectRegistryFinder, :geo do
create
(
:geo_file_registry
,
:lfs
,
file_id:
lfs_object_2
.
id
,
missing_on_primary:
true
)
create
(
:geo_file_registry
,
:lfs
,
file_id:
lfs_object_3
.
id
,
missing_on_primary:
true
)
expect
(
subject
.
count_synced_missing_on_primary
_lfs_objects
).
to
eq
2
expect
(
subject
.
count_synced_missing_on_primary
).
to
eq
2
end
it
'ignores remote LFS objects'
do
create
(
:geo_file_registry
,
:lfs
,
file_id:
lfs_object_remote_1
.
id
,
missing_on_primary:
true
)
expect
(
subject
.
count_synced_missing_on_primary
_lfs_objects
).
to
eq
0
expect
(
subject
.
count_synced_missing_on_primary
).
to
eq
0
end
end
end
end
shared_examples
'finds all the things'
do
describe
'#find_unsynced
_lfs_objects
'
do
describe
'#find_unsynced'
do
it
'delegates to the correct method'
do
expect
(
subject
).
to
receive
(
"
#{
method_prefix
}
_find_unsynced
_lfs_objects
"
.
to_sym
).
and_call_original
expect
(
subject
).
to
receive
(
"
#{
method_prefix
}
_find_unsynced"
.
to_sym
).
and_call_original
subject
.
find_unsynced
_lfs_objects
(
batch_size:
10
)
subject
.
find_unsynced
(
batch_size:
10
)
end
it
'returns LFS objects without an entry on the tracking database'
do
create
(
:geo_file_registry
,
:lfs
,
file_id:
lfs_object_1
.
id
,
success:
true
)
create
(
:geo_file_registry
,
:lfs
,
file_id:
lfs_object_3
.
id
,
success:
false
)
lfs_objects
=
subject
.
find_unsynced
_lfs_objects
(
batch_size:
10
)
lfs_objects
=
subject
.
find_unsynced
(
batch_size:
10
)
expect
(
lfs_objects
).
to
match_ids
(
lfs_object_2
,
lfs_object_4
)
end
...
...
@@ -300,23 +304,23 @@ describe Geo::LfsObjectRegistryFinder, :geo do
create
(
:geo_file_registry
,
:lfs
,
file_id:
lfs_object_1
.
id
,
success:
true
)
create
(
:geo_file_registry
,
:lfs
,
file_id:
lfs_object_3
.
id
,
success:
false
)
lfs_objects
=
subject
.
find_unsynced
_lfs_objects
(
batch_size:
10
,
except_file_ids:
[
lfs_object_2
.
id
])
lfs_objects
=
subject
.
find_unsynced
(
batch_size:
10
,
except_file_ids:
[
lfs_object_2
.
id
])
expect
(
lfs_objects
).
to
match_ids
(
lfs_object_4
)
end
end
describe
'#find_migrated_local
_lfs_objects
'
do
describe
'#find_migrated_local'
do
it
'delegates to the correct method'
do
expect
(
subject
).
to
receive
(
"
#{
method_prefix
}
_find_migrated_local
_lfs_objects
"
.
to_sym
).
and_call_original
expect
(
subject
).
to
receive
(
"
#{
method_prefix
}
_find_migrated_local"
.
to_sym
).
and_call_original
subject
.
find_migrated_local
_lfs_objects
(
batch_size:
10
)
subject
.
find_migrated_local
(
batch_size:
10
)
end
it
'returns LFS objects remotely and successfully synced locally'
do
create
(
:geo_file_registry
,
:lfs
,
file_id:
lfs_object_remote_1
.
id
)
lfs_objects
=
subject
.
find_migrated_local
_lfs_objects
(
batch_size:
10
)
lfs_objects
=
subject
.
find_migrated_local
(
batch_size:
10
)
expect
(
lfs_objects
).
to
match_ids
(
lfs_object_remote_1
)
end
...
...
@@ -324,7 +328,7 @@ describe Geo::LfsObjectRegistryFinder, :geo do
it
'excludes LFS objects stored remotely, but not synced yet'
do
create
(
:lfs_object
,
:object_storage
)
lfs_objects
=
subject
.
find_migrated_local
_lfs_objects
(
batch_size:
10
)
lfs_objects
=
subject
.
find_migrated_local
(
batch_size:
10
)
expect
(
lfs_objects
).
to
be_empty
end
...
...
@@ -332,7 +336,7 @@ describe Geo::LfsObjectRegistryFinder, :geo do
it
'excludes synced LFS objects that are stored locally'
do
create
(
:geo_file_registry
,
:avatar
,
file_id:
lfs_object_1
.
id
)
lfs_objects
=
subject
.
find_migrated_local
_lfs_objects
(
batch_size:
10
)
lfs_objects
=
subject
.
find_migrated_local
(
batch_size:
10
)
expect
(
lfs_objects
).
to
be_empty
end
...
...
@@ -341,36 +345,12 @@ describe Geo::LfsObjectRegistryFinder, :geo do
create
(
:geo_file_registry
,
:lfs
,
file_id:
lfs_object_remote_1
.
id
)
create
(
:geo_file_registry
,
:lfs
,
file_id:
lfs_object_remote_2
.
id
)
lfs_objects
=
subject
.
find_migrated_local
_lfs_objects
(
batch_size:
10
,
except_file_ids:
[
lfs_object_remote_1
.
id
])
lfs_objects
=
subject
.
find_migrated_local
(
batch_size:
10
,
except_file_ids:
[
lfs_object_remote_1
.
id
])
expect
(
lfs_objects
).
to
match_ids
(
lfs_object_remote_2
)
end
end
end
# Disable transactions via :delete method because a foreign table
# can't see changes inside a transaction of a different connection.
context
'FDW'
,
:delete
do
before
do
skip
(
'FDW is not configured'
)
if
Gitlab
::
Database
.
postgresql?
&&
!
Gitlab
::
Geo
::
Fdw
.
enabled?
end
include_examples
'counts all the things'
include_examples
'finds all the things'
do
let
(
:method_prefix
)
{
'fdw'
}
end
end
context
'Legacy'
do
before
do
allow
(
Gitlab
::
Geo
::
Fdw
).
to
receive
(
:enabled?
).
and_return
(
false
)
end
include_examples
'counts all the things'
include_examples
'finds all the things'
do
let
(
:method_prefix
)
{
'legacy'
}
end
end
it_behaves_like
'a file registry finder'
end
ee/spec/finders/geo/project_registry_finder_spec.rb
View file @
b7444374
...
...
@@ -3,6 +3,10 @@ require 'spec_helper'
describe
Geo
::
ProjectRegistryFinder
,
:geo
do
include
::
EE
::
GeoHelpers
# Using let() instead of set() because set() does not work properly
# when using the :delete DatabaseCleaner strategy, which is required for FDW
# tests because a foreign table can't see changes inside a transaction of a
# different connection.
let
(
:secondary
)
{
create
(
:geo_node
)
}
let
(
:synced_group
)
{
create
(
:group
)
}
let!
(
:project_not_synced
)
{
create
(
:project
)
}
...
...
ee/spec/support/shared_examples/finders/file_registry_finders.rb
0 → 100644
View file @
b7444374
shared_examples_for
'a file registry finder'
do
it
'responds to file registry finder methods'
do
file_registry_finder_methods
=
%i{
syncable
count_syncable
count_synced
count_failed
count_synced_missing_on_primary
count_registry
find_unsynced
find_migrated_local
find_retryable_failed_registries
find_retryable_synced_missing_on_primary_registries
}
file_registry_finder_methods
.
each
do
|
method
|
expect
(
subject
).
to
respond_to
(
method
)
end
end
# Disable transactions via :delete method because a foreign table
# can't see changes inside a transaction of a different connection.
context
'FDW'
,
:delete
do
before
do
skip
(
'FDW is not configured'
)
if
Gitlab
::
Database
.
postgresql?
&&
!
Gitlab
::
Geo
::
Fdw
.
enabled?
end
include_examples
'counts all the things'
include_examples
'finds all the things'
do
let
(
:method_prefix
)
{
'fdw'
}
end
end
context
'Legacy'
do
before
do
allow
(
Gitlab
::
Geo
::
Fdw
).
to
receive
(
:enabled?
).
and_return
(
false
)
end
include_examples
'counts all the things'
include_examples
'finds all the things'
do
let
(
:method_prefix
)
{
'legacy'
}
end
end
end
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment