Add streaming serializer and legacy writer

lib/gitlab/import_export/json/legacy_writer.rb

+module Gitlab
+  module ImportExport
+    module JSON
+      class LegacyWriter
+        include Gitlab::ImportExport::CommandLineUtil
+
+        attr_reader :path
+
+        def initialize(path)
+          @path = path
+          @last_array = nil
+          @keys = Set.new
+
+          mkdir_p(File.dirname(@path))
+          file.write('{}')
+        end
+
+        def close
+          @file&.close
+          @file = nil
+        end
+
+        def set(hash)
+          hash.each do |key, value|
+            write(key, value)
+          end
+        end
+
+        def write(key, value)
+          raise RuntimeError, "key '#{key}' already written" if @keys.include?(key)
+
+          # rewind by one byte, to overwrite '}'
+          file.pos = file.size-1
+
+          file.write(',') if @keys.any?
+          file.write(key.to_json)
+          file.write(':')
+          file.write(value.to_json)
+          file.write('}')
+
+          @keys.add(key)
+          @last_array = nil
+          @last_array_count = nil
+        end
+
+        def append(key, value)
+          unless @last_array == key
+            write(key, [])
+
+            @last_array = key
+            @last_array_count = 0
+          end
+
+          # rewind by two bytes, to overwrite ']}'
+          file.pos = file.size-2
+
+          file.write(',') if @last_array_count > 0
+          file.write(value.to_json)
+          file.write(']}')
+          @last_array_count += 1
+        end
+
+        private
+
+        def file
+          @file ||= File.open(@path, "wb")
+        end
+      end
+    end
+  end
+end

lib/gitlab/import_export/json/streaming_serializer.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module ImportExport
+    module JSON
+      class StreamingSerializer
+        include Gitlab::ImportExport::CommandLineUtil
+
+        attr_reader :overrides
+        attr_reader :additional_relations
+
+        BATCH_SIZE = 100
+
+        class Raw < String
+          def to_json(*_args)
+            to_s
+          end
+        end
+
+        def initialize(exportable, relations_schema, json_writer)
+          @exportable = exportable
+          @relations_schema = relations_schema
+          @overrides = {}
+          @json_writer = json_writer
+          @additional_relations = {}
+        end
+
+        def execute
+          serialize_root
+
+          includes.each do |relation_definition|
+            serialize_relation(relation_definition)
+          end
+        end
+
+        private
+
+        attr_reader :json_writer, :relations_schema, :exportable
+
+        def serialize_root
+          attributes = exportable.as_json(
+            relations_schema.merge(include: nil, preloads: nil))
+
+          data = attributes.merge(overrides)
+
+          json_writer.set(data)
+        end
+
+        def serialize_relation(definition)
+          raise ArgumentError, 'definition needs to be Hash' unless definition.is_a?(Hash)
+          raise ArgumentError, 'definition needs to have exactly one Hash element' unless definition.one?
+
+          key = definition.first.first
+          options = definition.first.second
+
+          record = exportable.public_send(key) # rubocop: disable GitlabSecurity/PublicSend
+
+          if record.is_a?(ActiveRecord::Relation)
+            serialize_many_relations(key, record, options)
+          else
+            serialize_single_relation(key, record, options)
+          end
+        end
+
+        def serialize_many_relations(key, record, options)
+          key_preloads = preloads&.dig(key)
+
+          record.in_batches(of: BATCH_SIZE) do |batch| # rubocop:disable Cop/InBatches
+            batch = batch.preload(key_preloads) if key_preloads
+
+            batch.each do |item|
+              item = Raw.new(item.to_json(options))
+
+              json_writer.append(key, item)
+            end
+          end
+
+          additional_relations[key].to_a.each do |item|
+            item = Raw.new(item.to_json(options))
+
+            json_writer.append(key, item)
+          end
+        end
+
+        def serialize_single_relation(key, record, options)
+          json = Raw.new(record.to_json(options))
+
+          json_writer.write(key, json)
+        end
+
+        def includes
+          relations_schema[:include]
+        end
+
+        def preloads
+          relations_schema[:preload]
+        end
+      end
+    end
+  end
+end