Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
G
gitlab-ce
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
1
Merge Requests
1
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
nexedi
gitlab-ce
Commits
4bcc3573
Commit
4bcc3573
authored
May 18, 2021
by
Quang-Minh Nguyen
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Address code review feedback
Issue
https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1054
parent
91eb12f7
Changes
5
Show whitespace changes
Inline
Side-by-side
Showing
5 changed files
with
32 additions
and
7 deletions
+32
-7
lib/gitlab/sidekiq_middleware/size_limiter/compressor.rb
lib/gitlab/sidekiq_middleware/size_limiter/compressor.rb
+1
-1
lib/gitlab/sidekiq_middleware/size_limiter/server.rb
lib/gitlab/sidekiq_middleware/size_limiter/server.rb
+3
-0
lib/gitlab/sidekiq_middleware/size_limiter/validator.rb
lib/gitlab/sidekiq_middleware/size_limiter/validator.rb
+4
-2
spec/lib/gitlab/sidekiq_middleware/size_limiter/compressor_spec.rb
...gitlab/sidekiq_middleware/size_limiter/compressor_spec.rb
+2
-2
spec/lib/gitlab/sidekiq_middleware/size_limiter/validator_spec.rb
.../gitlab/sidekiq_middleware/size_limiter/validator_spec.rb
+22
-2
No files found.
lib/gitlab/sidekiq_middleware/size_limiter/compressor.rb
View file @
4bcc3573
...
...
@@ -43,7 +43,7 @@ module Gitlab
exception
=
PayloadDecompressionConflictError
.
new
(
'Sidekiq argument list should include 1 argument.\
There should be a middleware interferes the job payload.\
That conflicts with the payload compressor'
)
::
Gitlab
::
ErrorTracking
.
track_and_raise_
for_dev_
exception
(
exception
)
::
Gitlab
::
ErrorTracking
.
track_and_raise_exception
(
exception
)
end
end
end
...
...
lib/gitlab/sidekiq_middleware/size_limiter/server.rb
View file @
4bcc3573
...
...
@@ -5,6 +5,9 @@ module Gitlab
module
SizeLimiter
class
Server
def
call
(
worker
,
job
,
queue
)
# This middleware should always decompress jobs regardless of the
# limiter mode or size limit. Otherwise, this could leave compressed
# payloads in queues that are then not able to be processed.
::
Gitlab
::
SidekiqMiddleware
::
SizeLimiter
::
Compressor
.
decompress
(
job
)
yield
...
...
lib/gitlab/sidekiq_middleware/size_limiter/validator.rb
View file @
4bcc3573
...
...
@@ -73,8 +73,9 @@ module Gitlab
def
set_compression_threshold
(
compression_threshold
)
@compression_threshold
=
(
compression_threshold
||
DEFAULT_COMPRESION_THRESHOLD_BYTES
).
to_i
if
@compression_threshold
<
0
if
@compression_threshold
<
=
0
::
Sidekiq
.
logger
.
warn
"Invalid Sidekiq size limiter compression threshold:
#{
@compression_threshold
}
"
@compression_threshold
=
DEFAULT_COMPRESION_THRESHOLD_BYTES
end
end
...
...
@@ -95,7 +96,8 @@ module Gitlab
end
def
compress_if_necessary
(
job_args
)
return
job_args
if
!
compress_mode?
||
job_args
.
bytesize
<
@compression_threshold
return
job_args
unless
compress_mode?
return
job_args
if
job_args
.
bytesize
<
@compression_threshold
::
Gitlab
::
SidekiqMiddleware
::
SizeLimiter
::
Compressor
.
compress
(
@job
,
job_args
)
end
...
...
spec/lib/gitlab/sidekiq_middleware/size_limiter/compressor_spec.rb
View file @
4bcc3573
...
...
@@ -119,7 +119,7 @@ RSpec.describe Gitlab::SidekiqMiddleware::SizeLimiter::Compressor do
context
'job payload is compressed with a different level'
do
let
(
:payload
)
do
base_payload
.
merge
(
'args'
=>
[
'eNqLVspIzcnJV9JRKs8vyklRigUAMq0FqQ=='
],
'args'
=>
[
Base64
.
strict_encode64
(
Zlib
::
Deflate
.
deflate
(
Sidekiq
.
dump_json
(
%w[hello world]
),
9
))
],
'compressed'
=>
true
)
end
...
...
@@ -141,7 +141,7 @@ RSpec.describe Gitlab::SidekiqMiddleware::SizeLimiter::Compressor do
end
it
'tracks the conflicting exception'
do
expect
(
::
Gitlab
::
ErrorTracking
).
to
receive
(
:track_and_raise_
for_dev_
exception
).
with
(
expect
(
::
Gitlab
::
ErrorTracking
).
to
receive
(
:track_and_raise_exception
).
with
(
be_a
(
::
Gitlab
::
SidekiqMiddleware
::
SizeLimiter
::
Compressor
::
PayloadDecompressionConflictError
)
)
...
...
spec/lib/gitlab/sidekiq_middleware/size_limiter/validator_spec.rb
View file @
4bcc3573
...
...
@@ -96,16 +96,36 @@ RSpec.describe Gitlab::SidekiqMiddleware::SizeLimiter::Validator do
expect
(
::
Sidekiq
.
logger
).
not_to
receive
(
:warn
)
described_class
.
new
(
TestSizeLimiterWorker
,
job_payload
,
compression_threshold:
300
)
described_class
.
new
(
TestSizeLimiterWorker
,
job_payload
,
compression_threshold:
0
)
described_class
.
new
(
TestSizeLimiterWorker
,
job_payload
,
compression_threshold:
1
)
end
end
context
'when the compression threshold is
invalid
'
do
context
'when the compression threshold is
negative
'
do
it
'logs a warning message'
do
expect
(
::
Sidekiq
.
logger
).
to
receive
(
:warn
).
with
(
'Invalid Sidekiq size limiter compression threshold: -1'
)
described_class
.
new
(
TestSizeLimiterWorker
,
job_payload
,
compression_threshold:
-
1
)
end
it
'falls back to the default'
do
validator
=
described_class
.
new
(
TestSizeLimiterWorker
,
job_payload
,
compression_threshold:
-
1
)
expect
(
validator
.
compression_threshold
).
to
be
(
100_000
)
end
end
context
'when the compression threshold is zero'
do
it
'logs a warning message'
do
expect
(
::
Sidekiq
.
logger
).
to
receive
(
:warn
).
with
(
'Invalid Sidekiq size limiter compression threshold: 0'
)
described_class
.
new
(
TestSizeLimiterWorker
,
job_payload
,
compression_threshold:
0
)
end
it
'falls back to the default'
do
validator
=
described_class
.
new
(
TestSizeLimiterWorker
,
job_payload
,
compression_threshold:
0
)
expect
(
validator
.
compression_threshold
).
to
be
(
100_000
)
end
end
context
'when the compression threshold is empty'
do
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment