Commit bc56a61f authored by Igor Drozdov's avatar Igor Drozdov Committed by Stan Hu

Resolve Yajl encoding incompatibility

When Yajl encoder was provided a string with ASCII-8BIT,
"incompatible character encodings: ASCII-8BIT and UTF-8" error
was raised.

Let's use StringIO in order to build the result out of chunks.

Changelog: fixed
MR: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/63379
parent 43239d5a
...@@ -242,7 +242,7 @@ module Gitlab ...@@ -242,7 +242,7 @@ module Gitlab
def self.encode(object, limit: 25.megabytes) def self.encode(object, limit: 25.megabytes)
return ::Gitlab::Json.dump(object) unless Feature.enabled?(:json_limited_encoder) return ::Gitlab::Json.dump(object) unless Feature.enabled?(:json_limited_encoder)
buffer = [] buffer = StringIO.new
buffer_size = 0 buffer_size = 0
::Yajl::Encoder.encode(object) do |data_chunk| ::Yajl::Encoder.encode(object) do |data_chunk|
...@@ -254,7 +254,7 @@ module Gitlab ...@@ -254,7 +254,7 @@ module Gitlab
buffer_size += chunk_size buffer_size += chunk_size
end end
buffer.join('') buffer.string
end end
end end
end end
......
...@@ -411,7 +411,7 @@ RSpec.describe Gitlab::Json do ...@@ -411,7 +411,7 @@ RSpec.describe Gitlab::Json do
end end
describe Gitlab::Json::LimitedEncoder do describe Gitlab::Json::LimitedEncoder do
subject { described_class.encode(obj, limit: 8.kilobytes) } subject { described_class.encode(obj, limit: 10.kilobytes) }
context 'when object size is acceptable' do context 'when object size is acceptable' do
let(:obj) { { test: true } } let(:obj) { { test: true } }
...@@ -431,6 +431,16 @@ RSpec.describe Gitlab::Json do ...@@ -431,6 +431,16 @@ RSpec.describe Gitlab::Json do
end end
end end
context 'when object contains ASCII-8BIT encoding' do
let(:obj) { [{ a: "\x8F" }] * 1000 }
it 'does not raise encoding error' do
expect { subject }.not_to raise_error
expect(subject).to be_a(String)
expect(subject.size).to eq(10001)
end
end
context 'when json_limited_encoder is disabled' do context 'when json_limited_encoder is disabled' do
let(:obj) { [{ test: true }] * 1000 } let(:obj) { [{ test: true }] * 1000 }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment