Introduces Group export feature with ndjson

To improve performance, and the memory footprint of the group export
feature, the ability to export a group-tree in the ndjson format is
being introduced.
This feature is behind a feature flag (:group_import_export_ndjson) for
now.

app/services/groups/import_export/export_service.rb

@@ -56,7 +56,7 @@ module Groups
       end
 
       def tree_exporter
-        Gitlab::ImportExport::Group::LegacyTreeSaver.new(
+        tree_exporter_class.new(
           group: @group,
           current_user: @current_user,
           shared: @shared,
@@ -64,6 +64,14 @@ module Groups
         )
       end
 
+      def tree_exporter_class
+        if ::Feature.enabled?(:group_import_export_ndjson, @group&.parent)
+          Gitlab::ImportExport::Group::TreeSaver
+        else
+          Gitlab::ImportExport::Group::LegacyTreeSaver
+        end
+      end
+
       def file_saver
         Gitlab::ImportExport::Saver.new(exportable: @group, shared: @shared)
       end

ee/spec/lib/ee/gitlab/import_export/group/tree_saver_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::ImportExport::Group::TreeSaver do
+  describe 'saves the group tree into a json object' do
+    let_it_be(:user) { create(:user) }
+    let_it_be(:group) { create(:group) }
+    let_it_be(:label) { create(:group_label) }
+    let_it_be(:parent_epic) { create(:epic, group: group) }
+    let_it_be(:epic) { create(:epic, group: group, parent: parent_epic) }
+    let_it_be(:epic_event) { create(:event, :created, target: epic, group: group, author: user) }
+    let_it_be(:epic_push_event) { create(:event, :pushed, target: epic, group: group, author: user) }
+    let_it_be(:milestone) { create(:milestone, group: group) }
+    let_it_be(:board) { create(:board, group: group, assignee: user, labels: [label]) }
+    let_it_be(:note) { create(:note, noteable: epic) }
+    let_it_be(:note_event) { create(:event, :created, target: note, author: user) }
+    let_it_be(:epic_emoji) { create(:award_emoji, awardable: epic) }
+    let_it_be(:epic_note_emoji) { create(:award_emoji, awardable: note) }
+
+    let(:shared) { Gitlab::ImportExport::Shared.new(group) }
+    let(:export_path) { "#{Dir.tmpdir}/group_tree_saver_spec_ee" }
+
+    subject(:group_tree_saver) { described_class.new(group: group, current_user: user, shared: shared) }
+
+    before_all do
+      group.add_maintainer(user)
+    end
+
+    after do
+      FileUtils.rm_rf(export_path)
+    end
+
+    it 'saves successfully' do
+      expect_successful_save(group_tree_saver)
+    end
+
+    context 'epics relation' do
+      let(:epic_json) do
+        read_association(group, 'epics').find do |attrs|
+          attrs['id'] == epic.id
+        end
+      end
+
+      it 'saves top level epics' do
+        expect_successful_save(group_tree_saver)
+
+        expect(read_association(group, "epics").size).to eq(2)
+      end
+
+      it 'saves parent of epic' do
+        expect_successful_save(group_tree_saver)
+
+        parent = epic_json['parent']
+
+        expect(parent).not_to be_empty
+        expect(parent['id']).to eq(parent_epic.id)
+      end
+
+      it 'saves epic notes' do
+        expect_successful_save(group_tree_saver)
+
+        notes = epic_json['notes']
+
+        expect(notes).not_to be_empty
+        expect(notes.first['note']).to eq(note.note)
+        expect(notes.first['noteable_id']).to eq(epic.id)
+      end
+
+      it 'saves epic events' do
+        expect_successful_save(group_tree_saver)
+
+        events = epic_json['events']
+        expect(events).not_to be_empty
+
+        event_actions = events.map { |event| event['action'] }
+        expect(event_actions).to contain_exactly(epic_event.action, epic_push_event.action)
+      end
+
+      it "saves epic's note events" do
+        expect_successful_save(group_tree_saver)
+
+        notes = epic_json['notes']
+        expect(notes.first['events'].first['action']).to eq(note_event.action)
+      end
+
+      it "saves epic's award emojis" do
+        expect_successful_save(group_tree_saver)
+
+        award_emoji = epic_json['award_emoji'].first
+        expect(award_emoji['name']).to eq(epic_emoji.name)
+      end
+
+      it "saves epic's note award emojis" do
+        expect_successful_save(group_tree_saver)
+
+        award_emoji = epic_json['notes'].first['award_emoji'].first
+        expect(award_emoji['name']).to eq(epic_note_emoji.name)
+      end
+    end
+
+    context 'boards relation' do
+      before do
+        stub_licensed_features(board_assignee_lists: true, board_milestone_lists: true)
+
+        create(:list, board: board, user: user, list_type: List.list_types[:assignee], position: 0)
+        create(:list, board: board, milestone: milestone, list_type: List.list_types[:milestone], position: 1)
+
+        expect_successful_save(group_tree_saver)
+      end
+
+      it 'saves top level boards' do
+        expect(read_association(group, 'boards').size).to eq(1)
+      end
+
+      it 'saves board assignee' do
+        expect(read_association(group, 'boards').first['board_assignee']['assignee_id']).to eq(user.id)
+      end
+
+      it 'saves board labels' do
+        labels = read_association(group, 'boards').first['labels']
+
+        expect(labels).not_to be_empty
+        expect(labels.first['title']).to eq(label.title)
+      end
+
+      it 'saves board lists' do
+        lists = read_association(group, 'boards').first['lists']
+
+        expect(lists).not_to be_empty
+
+        milestone_list = lists.find { |list| list['list_type'] == 'milestone' }
+        assignee_list = lists.find { |list| list['list_type'] == 'assignee' }
+
+        expect(milestone_list['milestone_id']).to eq(milestone.id)
+        expect(assignee_list['user_id']).to eq(user.id)
+      end
+    end
+
+    it 'saves the milestone data when there are boards with predefined milestones' do
+      milestone = Milestone::Upcoming
+      board_with_milestone = create(:board, group: group, milestone_id: milestone.id)
+
+      expect_successful_save(group_tree_saver)
+
+      board_data = read_association(group, 'boards').find { |board| board['id'] == board_with_milestone.id }
+
+      expect(board_data).to include(
+        'milestone_id' => milestone.id,
+        'milestone'    => {
+          'id'    => milestone.id,
+          'name'  => milestone.name,
+          'title' => milestone.title
+        }
+      )
+    end
+
+    it 'saves the milestone data when there are boards with persisted milestones' do
+      milestone = create(:milestone)
+      board_with_milestone = create(:board, group: group, milestone_id: milestone.id)
+
+      expect_successful_save(group_tree_saver)
+
+      board_data = read_association(group, 'boards').find { |board| board['id'] == board_with_milestone.id }
+
+      expect(board_data).to include(
+        'milestone_id' => milestone.id,
+        'milestone'    => a_hash_including(
+          'id'    => milestone.id,
+          'title' => milestone.title
+        )
+      )
+    end
+  end
+
+  def exported_path_for(file)
+    File.join(group_tree_saver.full_path, 'groups', file)
+  end
+
+  def read_association(group, association)
+    path = exported_path_for(File.join("#{group.id}", "#{association}.ndjson"))
+
+    File.foreach(path).map {|line| JSON.parse(line) }
+  end
+
+  def expect_successful_save(group_tree_saver)
+    expect(group_tree_saver.save).to be true
+    expect(group_tree_saver.shared.errors).to be_empty
+  end
+end

lib/gitlab/import_export.rb

@@ -91,6 +91,10 @@ module Gitlab
     def legacy_group_config_file
       Rails.root.join('lib/gitlab/import_export/group/legacy_import_export.yml')
     end
+
+    def group_config_file
+      Rails.root.join('lib/gitlab/import_export/group/import_export.yml')
+    end
   end
 end
 

lib/gitlab/import_export/group/import_export.yml

+# Model relationships to be included in the group import/export
+#
+# This list _must_ only contain relationships that are available to both FOSS and
+# Enterprise editions. EE specific relationships must be defined in the `ee` section further
+# down below.
+tree:
+  group:
+    - :milestones
+    - :badges
+    - labels:
+      - :priorities
+    - boards:
+      - lists:
+        - label:
+          - :priorities
+        - :board
+    - members:
+      - :user
+
+included_attributes:
+  user:
+    - :id
+    - :email
+    - :username
+  author:
+    - :name
+
+excluded_attributes:
+  group:
+    - :owner_id
+    - :created_at
+    - :updated_at
+    - :runners_token
+    - :runners_token_encrypted
+    - :saml_discovery_token
+    - :visibility_level
+    - :trial_ends_on
+    - :shared_runners_minute_limit
+    - :extra_shared_runners_minutes_limit
+  epics:
+    - :state_id
+
+methods:
+  labels:
+    - :type
+  label:
+    - :type
+  badges:
+    - :type
+  notes:
+    - :type
+  events:
+    - :action
+  lists:
+    - :list_type
+  epics:
+    - :state
+
+preloads:
+
+# EE specific relationships and settings to include. All of this will be merged
+# into the previous structures if EE is used.
+ee:
+  tree:
+    group:
+      - epics:
+        - :parent
+        - :award_emoji
+        - events:
+          - :push_event_payload
+        - notes:
+          - :author
+          - :award_emoji
+          - events:
+            - :push_event_payload
+      - boards:
+        - :board_assignee
+        - :milestone
+        - labels:
+          - :priorities
+        - lists:
+          - milestone:
+            - events:
+              - :push_event_payload

lib/gitlab/import_export/group/tree_saver.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module ImportExport
+    module Group
+      class TreeSaver
+        attr_reader :full_path, :shared
+
+        def initialize(group:, current_user:, shared:, params: {})
+          @params = params
+          @current_user = current_user
+          @shared = shared
+          @group = group
+          @full_path = File.join(@shared.export_path, 'tree')
+        end
+
+        def save
+          all_groups = Enumerator.new do |group_ids|
+            groups.each do |group|
+              serialize(group)
+              group_ids << group.id
+            end
+          end
+
+          json_writer.write_relation_array('groups', '_all', all_groups)
+
+          true
+        rescue => e
+          @shared.error(e)
+          false
+        ensure
+          json_writer&.close
+        end
+
+        private
+
+        def groups
+          @groups ||= Gitlab::ObjectHierarchy
+            .new(::Group.where(id: @group.id))
+            .base_and_descendants(with_depth: true)
+            .order_by(:depth)
+        end
+
+        def serialize(group)
+          ImportExport::JSON::StreamingSerializer.new(
+            group,
+            group_tree,
+            json_writer,
+            exportable_path: "groups/#{group.id}"
+          ).execute
+        end
+
+        def group_tree
+          @group_tree ||= Gitlab::ImportExport::Reader.new(
+            shared: @shared,
+            config: group_config
+          ).group_tree
+        end
+
+        def group_config
+          Gitlab::ImportExport::Config.new(
+            config: Gitlab::ImportExport.group_config_file
+          ).to_h
+        end
+
+        def json_writer
+          @json_writer ||= ImportExport::JSON::NdjsonWriter.new(@full_path)
+        end
+      end
+    end
+  end
+end

spec/lib/gitlab/import_export/group/tree_saver_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::ImportExport::Group::TreeSaver do
+  describe 'saves the group tree into a json object' do
+    let_it_be(:user) { create(:user) }
+    let_it_be(:group) { setup_groups }
+
+    let(:shared) { Gitlab::ImportExport::Shared.new(group) }
+    let(:export_path) { "#{Dir.tmpdir}/group_tree_saver_spec" }
+
+    subject(:group_tree_saver) { described_class.new(group: group, current_user: user, shared: shared) }
+
+    before_all do
+      group.add_maintainer(user)
+    end
+
+    before do
+      allow_next_instance_of(Gitlab::ImportExport) do |import_export|
+        allow(import_export).to receive(:storage_path).and_return(export_path)
+      end
+    end
+
+    after do
+      FileUtils.rm_rf(export_path)
+    end
+
+    it 'saves the group successfully' do
+      expect(group_tree_saver.save).to be true
+    end
+
+    it 'fails to export a group' do
+      allow_next_instance_of(Gitlab::ImportExport::JSON::NdjsonWriter) do |ndjson_writer|
+        allow(ndjson_writer).to receive(:write_relation_array).and_raise(RuntimeError, 'exception')
+      end
+
+      expect(shared).to receive(:error).with(RuntimeError).and_call_original
+
+      expect(group_tree_saver.save).to be false
+    end
+
+    context 'exported files' do
+      before do
+        group_tree_saver.save
+      end
+
+      it 'has one group per line' do
+        groups_catalog =
+          File.readlines(exported_path_for('_all.ndjson'))
+          .map { |line| Integer(line) }
+
+        expect(groups_catalog.size).to eq(3)
+        expect(groups_catalog).to eq([
+          group.id,
+          group.descendants.first.id,
+          group.descendants.first.descendants.first.id
+        ])
+      end
+
+      it 'has a file per group' do
+        group.self_and_descendants.pluck(:id).each do |id|
+          group_attributes_file = exported_path_for("#{id}.json")
+
+          expect(File.exist?(group_attributes_file)).to be(true)
+        end
+      end
+
+      context 'group attributes file' do
+        let(:group_attributes_file) { exported_path_for("#{group.id}.json") }
+        let(:group_attributes) { ::JSON.parse(File.read(group_attributes_file)) }
+
+        it 'has a file for each group with its attributes' do
+          expect(group_attributes['description']).to eq(group.description)
+          expect(group_attributes['parent_id']).to eq(group.parent_id)
+        end
+
+        shared_examples 'excluded attributes' do
+          excluded_attributes = %w[
+            owner_id
+            created_at
+            updated_at
+            runners_token
+            runners_token_encrypted
+            saml_discovery_token
+          ]
+
+          excluded_attributes.each do |excluded_attribute|
+            it 'does not contain excluded attribute' do
+              expect(group_attributes).not_to include(excluded_attribute => group.public_send(excluded_attribute))
+            end
+          end
+        end
+
+        include_examples 'excluded attributes'
+      end
+
+      it 'has a file for each group association' do
+        group.self_and_descendants do |g|
+          %w[
+            badges
+            boards
+            epics
+            labels
+            members
+            milestones
+          ].each do |association|
+            path = exported_path_for("#{g.id}", "#{association}.ndjson")
+            expect(File.exist?(path)).to eq(true), "#{path} does not exist"
+          end
+        end
+      end
+    end
+  end
+
+  def exported_path_for(*file)
+    File.join(group_tree_saver.full_path, 'groups', *file)
+  end
+
+  def setup_groups
+    root = setup_group
+    subgroup = setup_group(parent: root)
+    setup_group(parent: subgroup)
+
+    root
+  end
+
+  def setup_group(parent: nil)
+    group = create(:group, description: 'description', parent: parent)
+    create(:milestone, group: group)
+    create(:group_badge, group: group)
+    group_label = create(:group_label, group: group)
+    board = create(:board, group: group, milestone_id: Milestone::Upcoming.id)
+    create(:list, board: board, label: group_label)
+    create(:group_badge, group: group)
+    create(:label_priority, label: group_label, priority: 1)
+
+    group
+  end
+end

spec/services/groups/import_export/export_service_spec.rb

@@ -11,7 +11,7 @@ describe Groups::ImportExport::ExportService do
       let(:export_service) { described_class.new(group: group, user: user) }
 
       it 'enqueues an export job' do
-        expect(GroupExportWorker).to receive(:perform_async).with(user.id, group.id, {})
+        allow(GroupExportWorker).to receive(:perform_async).with(user.id, group.id, {})
 
         export_service.async_execute
       end
@@ -49,7 +49,17 @@ describe Groups::ImportExport::ExportService do
       FileUtils.rm_rf(archive_path)
     end
 
-    it 'saves the models' do
+    it 'saves the models using ndjson tree saver' do
+      stub_feature_flags(group_import_export_ndjson: true)
+
+      expect(Gitlab::ImportExport::Group::TreeSaver).to receive(:new).and_call_original
+
+      service.execute
+    end
+
+    it 'saves the models using legacy tree saver' do
+      stub_feature_flags(group_import_export_ndjson: false)
+
       expect(Gitlab::ImportExport::Group::LegacyTreeSaver).to receive(:new).and_call_original
 
       service.execute