Commit d4022ac5 authored by Mayra Cabrera's avatar Mayra Cabrera

Merge branch 'jcunha-add-relations-export-api-for-projects' into 'master'

Add relations export API for Projects

See merge request gitlab-org/gitlab!70330
parents e6e4124a 5e9ebbd1
...@@ -233,6 +233,7 @@ class Project < ApplicationRecord ...@@ -233,6 +233,7 @@ class Project < ApplicationRecord
has_one :import_state, autosave: true, class_name: 'ProjectImportState', inverse_of: :project has_one :import_state, autosave: true, class_name: 'ProjectImportState', inverse_of: :project
has_one :import_export_upload, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent has_one :import_export_upload, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
has_many :export_jobs, class_name: 'ProjectExportJob' has_many :export_jobs, class_name: 'ProjectExportJob'
has_many :bulk_import_exports, class_name: 'BulkImports::Export', inverse_of: :project
has_one :project_repository, inverse_of: :project has_one :project_repository, inverse_of: :project
has_one :tracing_setting, class_name: 'ProjectTracingSetting' has_one :tracing_setting, class_name: 'ProjectTracingSetting'
has_one :incident_management_setting, inverse_of: :project, class_name: 'IncidentManagement::ProjectIncidentManagementSetting' has_one :incident_management_setting, inverse_of: :project, class_name: 'IncidentManagement::ProjectIncidentManagementSetting'
......
...@@ -10,3 +10,4 @@ Grape::Validations.register_validator(:check_assignees_count, ::API::Validations ...@@ -10,3 +10,4 @@ Grape::Validations.register_validator(:check_assignees_count, ::API::Validations
Grape::Validations.register_validator(:untrusted_regexp, ::API::Validations::Validators::UntrustedRegexp) Grape::Validations.register_validator(:untrusted_regexp, ::API::Validations::Validators::UntrustedRegexp)
Grape::Validations.register_validator(:email_or_email_list, ::API::Validations::Validators::EmailOrEmailList) Grape::Validations.register_validator(:email_or_email_list, ::API::Validations::Validators::EmailOrEmailList)
Grape::Validations.register_validator(:iteration_id, ::API::Validations::Validators::IntegerOrCustomValue) Grape::Validations.register_validator(:iteration_id, ::API::Validations::Validators::IntegerOrCustomValue)
Grape::Validations.register_validator(:project_portable, ::API::Validations::Validators::ProjectPortable)
---
stage: Manage
group: Import
info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments
---
# Project Relations Export API **(FREE)**
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/70330) in GitLab 14.4 behind the `bulk_import` [feature flag](../administration/feature_flags.md), disabled by default.
FLAG:
On GitLab.com, this feature is available.
On self-managed GitLab, by default this feature is available. To hide the feature, ask an administrator to
[disable the `bulk_import` flag](../administration/feature_flags.md).
The feature is not ready for production use. It is still in experimental stage and might change in the future.
With the Project Relations Export API, you can partially export project structure. This API is
similar to [project export](project_import_export.md),
but it exports each top-level relation (for example, milestones/boards/labels) as a separate file
instead of one archive. The project relations export API is primarily used in
[group migration](../user/group/import/index.md#enable-or-disable-gitlab-group-migration)
to support group project import.
## Schedule new export
Start a new project relations export:
```plaintext
POST /projects/:id/export_relations
```
| Attribute | Type | Required | Description |
| --------- | -------------- | -------- | ---------------------------------------- |
| `id` | integer/string | yes | ID of the project owned by the authenticated user. |
```shell
curl --request POST --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/export_relations"
```
```json
{
"message": "202 Accepted"
}
```
## Export status
View the status of the relations export:
```plaintext
GET /projects/:id/export_relations/status
```
| Attribute | Type | Required | Description |
| --------- | -------------- | -------- | ---------------------------------------- |
| `id` | integer/string | yes | ID of the project owned by the authenticated user. |
```shell
curl --request GET --header "PRIVATE-TOKEN: <your_access_token>" \
"https://gitlab.example.com/api/v4/projects/1/export_relations/status"
```
The status can be one of the following:
- `0`: `started`
- `1`: `finished`
- `-1`: `failed`
- `0` - `started`
- `1` - `finished`
- `-1` - `failed`
```json
[
{
"relation": "project_badges",
"status": 1,
"error": null,
"updated_at": "2021-05-04T11:25:20.423Z"
},
{
"relation": "boards",
"status": 1,
"error": null,
"updated_at": "2021-05-04T11:25:20.085Z"
}
]
```
## Export download
Download the finished relations export:
```plaintext
GET /projects/:id/export_relations/download
```
| Attribute | Type | Required | Description |
| --------------- | -------------- | -------- | ---------------------------------------- |
| `id` | integer/string | yes | ID of the project owned by the authenticated user. |
| `relation` | string | yes | Name of the project top-level relation to download. |
```shell
curl --header "PRIVATE-TOKEN: <your_access_token>" --remote-header-name \
--remote-name "https://gitlab.example.com/api/v4/projects/1/export_relations/download?relation=labels"
```
```shell
ls labels.ndjson.gz
labels.ndjson.gz
```
...@@ -74,6 +74,52 @@ module API ...@@ -74,6 +74,52 @@ module API
accepted! accepted!
end end
resource do
before do
not_found! unless ::Feature.enabled?(:bulk_import, default_enabled: :yaml)
end
desc 'Start relations export' do
detail 'This feature was introduced in GitLab 14.4'
end
post ':id/export_relations' do
response = ::BulkImports::ExportService.new(portable: user_project, user: current_user).execute
if response.success?
accepted!
else
render_api_error!(message: 'Project relations export could not be started.')
end
end
desc 'Download relations export' do
detail 'This feature was introduced in GitLab 14.4'
end
params do
requires :relation,
type: String,
project_portable: true,
desc: 'Project relation name'
end
get ':id/export_relations/download' do
export = user_project.bulk_import_exports.find_by_relation(params[:relation])
file = export&.upload&.export_file
if file
present_carrierwave_file!(file)
else
render_api_error!('404 Not found', 404)
end
end
desc 'Relations export status' do
detail 'This feature was introduced in GitLab 14.4'
end
get ':id/export_relations/status' do
present user_project.bulk_import_exports, with: Entities::BulkImports::ExportStatus
end
end
end end
end end
end end
# frozen_string_literal: true
module API
module Validations
module Validators
class ProjectPortable < Grape::Validations::Base
def validate_param!(attr_name, params)
portable = params[attr_name]
portable_relations = ::BulkImports::FileTransfer.config_for(::Project.new).portable_relations
return if portable_relations.include?(portable)
raise Grape::Exceptions::Validation.new(
params: [@scope.full_name(attr_name)],
message: "is not portable"
)
end
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe API::Validations::Validators::ProjectPortable do
include ApiValidatorsHelpers
let(:portable) { 'labels' }
let(:not_portable) { 'project_members' }
subject do
described_class.new(['test'], {}, false, scope.new)
end
context 'valid portable' do
it 'does not raise a validation error' do
expect_no_validation_error('test' => portable)
end
end
context 'empty params' do
it 'raises a validation error' do
expect_validation_error('test' => nil)
expect_validation_error('test' => '')
end
end
context 'not portable' do
it 'raises a validation error' do
expect_validation_error('test' => not_portable) # Sha length > 40
end
end
end
...@@ -595,6 +595,7 @@ project: ...@@ -595,6 +595,7 @@ project:
- pending_builds - pending_builds
- security_scans - security_scans
- ci_feature_usages - ci_feature_usages
- bulk_import_exports
award_emoji: award_emoji:
- awardable - awardable
- user - user
......
...@@ -36,6 +36,7 @@ RSpec.describe Group do ...@@ -36,6 +36,7 @@ RSpec.describe Group do
it { is_expected.to have_many(:debian_distributions).class_name('Packages::Debian::GroupDistribution').dependent(:destroy) } it { is_expected.to have_many(:debian_distributions).class_name('Packages::Debian::GroupDistribution').dependent(:destroy) }
it { is_expected.to have_many(:daily_build_group_report_results).class_name('Ci::DailyBuildGroupReportResult') } it { is_expected.to have_many(:daily_build_group_report_results).class_name('Ci::DailyBuildGroupReportResult') }
it { is_expected.to have_many(:group_callouts).class_name('Users::GroupCallout').with_foreign_key(:group_id) } it { is_expected.to have_many(:group_callouts).class_name('Users::GroupCallout').with_foreign_key(:group_id) }
it { is_expected.to have_many(:bulk_import_exports).class_name('BulkImports::Export') }
describe '#members & #requesters' do describe '#members & #requesters' do
let(:requester) { create(:user) } let(:requester) { create(:user) }
......
...@@ -140,6 +140,7 @@ RSpec.describe Project, factory_default: :keep do ...@@ -140,6 +140,7 @@ RSpec.describe Project, factory_default: :keep do
it { is_expected.to have_many(:error_tracking_client_keys).class_name('ErrorTracking::ClientKey') } it { is_expected.to have_many(:error_tracking_client_keys).class_name('ErrorTracking::ClientKey') }
it { is_expected.to have_many(:pending_builds).class_name('Ci::PendingBuild') } it { is_expected.to have_many(:pending_builds).class_name('Ci::PendingBuild') }
it { is_expected.to have_many(:ci_feature_usages).class_name('Projects::CiFeatureUsage') } it { is_expected.to have_many(:ci_feature_usages).class_name('Projects::CiFeatureUsage') }
it { is_expected.to have_many(:bulk_import_exports).class_name('BulkImports::Export') }
# GitLab Pages # GitLab Pages
it { is_expected.to have_many(:pages_domains) } it { is_expected.to have_many(:pages_domains) }
......
...@@ -457,4 +457,143 @@ RSpec.describe API::ProjectExport, :clean_gitlab_redis_cache do ...@@ -457,4 +457,143 @@ RSpec.describe API::ProjectExport, :clean_gitlab_redis_cache do
end end
end end
end end
describe 'export relations' do
let(:relation) { 'labels' }
let(:download_path) { "/projects/#{project.id}/export_relations/download?relation=#{relation}" }
let(:path) { "/projects/#{project.id}/export_relations" }
let_it_be(:status_path) { "/projects/#{project.id}/export_relations/status" }
context 'when user is a maintainer' do
before do
project.add_maintainer(user)
end
describe 'POST /projects/:id/export_relations' do
it 'accepts the request' do
post api(path, user)
expect(response).to have_gitlab_http_status(:accepted)
end
context 'when response is not success' do
it 'returns api error' do
allow_next_instance_of(BulkImports::ExportService) do |service|
allow(service).to receive(:execute).and_return(ServiceResponse.error(message: 'error', http_status: :error))
end
post api(path, user)
expect(response).to have_gitlab_http_status(:error)
end
end
end
describe 'GET /projects/:id/export_relations/download' do
let_it_be(:export) { create(:bulk_import_export, project: project, relation: 'labels') }
let_it_be(:upload) { create(:bulk_import_export_upload, export: export) }
context 'when export file exists' do
it 'downloads exported project relation archive' do
upload.update!(export_file: fixture_file_upload('spec/fixtures/bulk_imports/gz/labels.ndjson.gz'))
get api(download_path, user)
expect(response).to have_gitlab_http_status(:ok)
expect(response.header['Content-Disposition']).to eq("attachment; filename=\"labels.ndjson.gz\"; filename*=UTF-8''labels.ndjson.gz")
end
end
context 'when relation is not portable' do
let(:relation) { ::BulkImports::FileTransfer::ProjectConfig.new(project).skipped_relations.first }
it_behaves_like '400 response' do
let(:request) { get api(download_path, user) }
end
end
context 'when export file does not exist' do
it 'returns 404' do
allow(upload).to receive(:export_file).and_return(nil)
get api(download_path, user)
expect(response).to have_gitlab_http_status(:not_found)
end
end
end
describe 'GET /projects/:id/export_relations/status' do
it 'returns a list of relation export statuses' do
create(:bulk_import_export, :started, project: project, relation: 'labels')
create(:bulk_import_export, :finished, project: project, relation: 'milestones')
create(:bulk_import_export, :failed, project: project, relation: 'project_badges')
get api(status_path, user)
expect(response).to have_gitlab_http_status(:ok)
expect(json_response.pluck('relation')).to contain_exactly('labels', 'milestones', 'project_badges')
expect(json_response.pluck('status')).to contain_exactly(-1, 0, 1)
end
end
context 'with bulk_import FF disabled' do
before do
stub_feature_flags(bulk_import: false)
end
describe 'POST /projects/:id/export_relations' do
it_behaves_like '404 response' do
let(:request) { post api(path, user) }
end
end
describe 'GET /projects/:id/export_relations/download' do
let_it_be(:export) { create(:bulk_import_export, project: project, relation: 'labels') }
let_it_be(:upload) { create(:bulk_import_export_upload, export: export) }
before do
upload.update!(export_file: fixture_file_upload('spec/fixtures/bulk_imports/gz/labels.ndjson.gz'))
end
it_behaves_like '404 response' do
let(:request) { post api(path, user) }
end
end
describe 'GET /projects/:id/export_relations/status' do
it_behaves_like '404 response' do
let(:request) { get api(status_path, user) }
end
end
end
end
context 'when user is a developer' do
let_it_be(:developer) { create(:user) }
before do
project.add_developer(developer)
end
describe 'POST /projects/:id/export_relations' do
it_behaves_like '403 response' do
let(:request) { post api(path, developer) }
end
end
describe 'GET /projects/:id/export_relations/download' do
it_behaves_like '403 response' do
let(:request) { get api(download_path, developer) }
end
end
describe 'GET /projects/:id/export_relations/status' do
it_behaves_like '403 response' do
let(:request) { get api(status_path, developer) }
end
end
end
end
end end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment