Skip to content

G
gitlab-ce
Commit 73ac1007 authored by George Koltsov's avatar George Koltsov
Browse files
Options

Update Bulk Import default page size to 500 

- Process 500 items intead of 100 at a time in order to reduce
  the amount of network requests made when running Bulk Import
parent 4816f7ef
Hide whitespace changes
Inline Side-by-side
Showing with 36 additions and 21 deletions
app/models/bulk_imports/tracker.rb
View file @ 73ac1007
......@@ -18,6 +18,8 @@ class BulkImports::Tracker < ApplicationRecord
validates :stage, presence: true
DEFAULT_PAGE_SIZE = 500
state_machine :status, initial: :created do
state :created, value: 0
state :started, value: 1
......
changelogs/unreleased/georgekoltsov-increase-bulk-import-default-page-size.yml 0 → 100644
View file @ 73ac1007
---
title: Update BulkImport default page size to 500 in order to process larger page
of data
merge_request: 57594
author:
type: other
ee/lib/ee/bulk_imports/groups/graphql/get_epic_award_emoji_query.rb
View file @ 73ac1007
......@@ -9,10 +9,10 @@ module EE
def to_s
<<-'GRAPHQL'
query($full_path: ID!, $epic_iid: ID!, $cursor: String) {
query($full_path: ID!, $epic_iid: ID!, $cursor: String, $per_page: Int) {
group(fullPath: $full_path) {
epic(iid: $epic_iid) {
award_emoji: awardEmoji(first: 100, after: $cursor) {
award_emoji: awardEmoji(first: $per_page, after: $cursor) {
page_info: pageInfo {
end_cursor: endCursor
has_next_page: hasNextPage
......@@ -36,7 +36,8 @@ module EE
{
full_path: context.entity.source_full_path,
cursor: context.tracker.next_page,
epic_iid: iid
epic_iid: iid,
per_page: ::BulkImports::Tracker::DEFAULT_PAGE_SIZE
}
end
......
ee/lib/ee/bulk_imports/groups/graphql/get_epic_events_query.rb
View file @ 73ac1007
......@@ -9,10 +9,10 @@ module EE
def to_s
<<-'GRAPHQL'
query($full_path: ID!, $epic_iid: ID!, $cursor: String) {
query($full_path: ID!, $epic_iid: ID!, $cursor: String, $per_page: Int) {
group(fullPath: $full_path) {
epic(iid: $epic_iid) {
events(first: 100, after: $cursor) {
events(first: $per_page, after: $cursor) {
page_info: pageInfo {
end_cursor: endCursor
has_next_page: hasNextPage
......@@ -38,7 +38,8 @@ module EE
{
full_path: context.entity.source_full_path,
cursor: context.tracker.next_page,
epic_iid: iid
epic_iid: iid,
per_page: ::BulkImports::Tracker::DEFAULT_PAGE_SIZE
}
end
......
ee/lib/ee/bulk_imports/groups/graphql/get_epics_query.rb
View file @ 73ac1007
......@@ -9,11 +9,11 @@ module EE
def to_s
<<-'GRAPHQL'
query($full_path: ID!, $cursor: String) {
query($full_path: ID!, $cursor: String, $per_page: Int) {
group(fullPath: $full_path) {
epics(
includeDescendantGroups: false,
first: 100,
first: $per_page,
after: $cursor
) {
page_info: pageInfo {
......@@ -61,7 +61,8 @@ module EE
def variables(context)
{
full_path: context.entity.source_full_path,
cursor: context.tracker.next_page
cursor: context.tracker.next_page,
per_page: ::BulkImports::Tracker::DEFAULT_PAGE_SIZE
}
end
......
ee/lib/ee/bulk_imports/groups/graphql/get_iterations_query.rb
View file @ 73ac1007
......@@ -9,9 +9,9 @@ module EE
def to_s
<<-'GRAPHQL'
query($full_path: ID!, $cursor: String) {
query($full_path: ID!, $cursor: String, $per_page: Int) {
group(fullPath: $full_path) {
iterations(first: 100, after: $cursor, includeAncestors: false) {
iterations(first: $per_page, after: $cursor, includeAncestors: false) {
page_info: pageInfo {
end_cursor: endCursor
has_next_page: hasNextPage
......@@ -35,7 +35,8 @@ module EE
def variables(context)
{
full_path: context.entity.source_full_path,
cursor: context.tracker.next_page
cursor: context.tracker.next_page,
per_page: ::BulkImports::Tracker::DEFAULT_PAGE_SIZE
}
end
......
lib/bulk_imports/groups/graphql/get_labels_query.rb
View file @ 73ac1007
......@@ -8,9 +8,9 @@ module BulkImports
def to_s
<<-'GRAPHQL'
query ($full_path: ID!, $cursor: String) {
query ($full_path: ID!, $cursor: String, $per_page: Int) {
group(fullPath: $full_path) {
labels(first: 100, after: $cursor, onlyGroupLabels: true) {
labels(first: $per_page, after: $cursor, onlyGroupLabels: true) {
page_info: pageInfo {
end_cursor: endCursor
has_next_page: hasNextPage
......@@ -31,7 +31,8 @@ module BulkImports
def variables(context)
{
full_path: context.entity.source_full_path,
cursor: context.tracker.next_page
cursor: context.tracker.next_page,
per_page: ::BulkImports::Tracker::DEFAULT_PAGE_SIZE
}
end
......
lib/bulk_imports/groups/graphql/get_members_query.rb
View file @ 73ac1007
......@@ -7,9 +7,9 @@ module BulkImports
extend self
def to_s
<<-'GRAPHQL'
query($full_path: ID!, $cursor: String) {
query($full_path: ID!, $cursor: String, $per_page: Int) {
group(fullPath: $full_path) {
group_members: groupMembers(relations: DIRECT, first: 100, after: $cursor) {
group_members: groupMembers(relations: DIRECT, first: $per_page, after: $cursor) {
page_info: pageInfo {
end_cursor: endCursor
has_next_page: hasNextPage
......@@ -34,7 +34,8 @@ module BulkImports
def variables(context)
{
full_path: context.entity.source_full_path,
cursor: context.tracker.next_page
cursor: context.tracker.next_page,
per_page: ::BulkImports::Tracker::DEFAULT_PAGE_SIZE
}
end
......
lib/bulk_imports/groups/graphql/get_milestones_query.rb
View file @ 73ac1007
......@@ -8,9 +8,9 @@ module BulkImports
def to_s
<<-'GRAPHQL'
query ($full_path: ID!, $cursor: String) {
query ($full_path: ID!, $cursor: String, $per_page: Int) {
group(fullPath: $full_path) {
milestones(first: 100, after: $cursor, includeDescendants: false) {
milestones(first: $per_page, after: $cursor, includeDescendants: false) {
page_info: pageInfo {
end_cursor: endCursor
has_next_page: hasNextPage
......@@ -33,7 +33,8 @@ module BulkImports
def variables(context)
{
full_path: context.entity.source_full_path,
cursor: context.tracker.next_page
cursor: context.tracker.next_page,
per_page: ::BulkImports::Tracker::DEFAULT_PAGE_SIZE
}
end
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7