Commit 98537f1b authored by Bob Van Landuyt's avatar Bob Van Landuyt

Merge branch 'eb-merge-rake-task-for-artifacts-size-recalculation' into 'master'

Make refresh task fetch IDs from CSV URL

See merge request gitlab-org/gitlab!83194
parents 1d5bb484 87d61289
# frozen_string_literal: true
require 'httparty'
require 'csv'
namespace :ci do
namespace :build_artifacts do
desc "GitLab | CI | Fetch projects with incorrect artifact size on GitLab.com"
task :project_with_incorrect_artifact_size do
csv_url = ENV['SISENSE_PROJECT_IDS_WITH_INCORRECT_ARTIFACTS_URL']
# rubocop: disable Gitlab/HTTParty
body = HTTParty.get(csv_url)
# rubocop: enable Gitlab/HTTParty
table = CSV.parse(body.parsed_response, headers: true)
puts table['PROJECT_ID'].join(' ')
end
end
end
# frozen_string_literal: true # frozen_string_literal: true
require 'httparty'
require 'csv'
namespace :gitlab do namespace :gitlab do
desc "GitLab | Refresh build artifacts size project statistics for given project IDs" desc "GitLab | Refresh build artifacts size project statistics for given list of Project IDs from remote CSV"
BUILD_ARTIFACTS_SIZE_REFRESH_ENQUEUE_BATCH_SIZE = 500 BUILD_ARTIFACTS_SIZE_REFRESH_ENQUEUE_BATCH_SIZE = 500
task :refresh_project_statistics_build_artifacts_size, [:project_ids] => :environment do |_t, args| task :refresh_project_statistics_build_artifacts_size, [:csv_url] => :environment do |_t, args|
project_ids = [] csv_url = args.csv_url
project_ids = $stdin.read.split unless $stdin.tty?
project_ids = args.project_ids.to_s.split unless project_ids.any? # rubocop: disable Gitlab/HTTParty
body = HTTParty.get(csv_url)
# rubocop: enable Gitlab/HTTParty
table = CSV.parse(body.to_s, headers: true)
project_ids = table['PROJECT_ID']
puts "Loaded #{project_ids.size} project ids to import"
imported = 0
missing = 0
if project_ids.any? if project_ids.any?
project_ids.in_groups_of(BUILD_ARTIFACTS_SIZE_REFRESH_ENQUEUE_BATCH_SIZE) do |ids| project_ids.in_groups_of(BUILD_ARTIFACTS_SIZE_REFRESH_ENQUEUE_BATCH_SIZE, false) do |ids|
projects = Project.where(id: ids) projects = Project.where(id: ids)
Projects::BuildArtifactsSizeRefresh.enqueue_refresh(projects) Projects::BuildArtifactsSizeRefresh.enqueue_refresh(projects)
imported += projects.size
missing += ids.size - projects.size
puts "#{imported}/#{project_ids.size} (missing projects: #{missing})"
end end
puts 'Done.'.green puts 'Done.'.green
else else
puts 'Please provide a string of space-separated project IDs as the argument or through the STDIN'.red puts 'Project IDs must be listed in the CSV under the header PROJECT_ID'.red
end end
end end
end end
...@@ -11,37 +11,44 @@ RSpec.describe 'gitlab:refresh_project_statistics_build_artifacts_size rake task ...@@ -11,37 +11,44 @@ RSpec.describe 'gitlab:refresh_project_statistics_build_artifacts_size rake task
let_it_be(:project_3) { create(:project) } let_it_be(:project_3) { create(:project) }
let(:string_of_ids) { "#{project_1.id} #{project_2.id} #{project_3.id} 999999" } let(:string_of_ids) { "#{project_1.id} #{project_2.id} #{project_3.id} 999999" }
let(:csv_url) { 'https://www.example.com/foo.csv' }
let(:csv_body) do
<<~BODY
PROJECT_ID
#{project_1.id}
#{project_2.id}
#{project_3.id}
BODY
end
before do before do
Rake.application.rake_require('tasks/gitlab/refresh_project_statistics_build_artifacts_size') Rake.application.rake_require('tasks/gitlab/refresh_project_statistics_build_artifacts_size')
stub_const("BUILD_ARTIFACTS_SIZE_REFRESH_ENQUEUE_BATCH_SIZE", 2) stub_const("BUILD_ARTIFACTS_SIZE_REFRESH_ENQUEUE_BATCH_SIZE", 2)
end
context 'when given a list of space-separated IDs through STDIN' do stub_request(:get, csv_url).to_return(status: 200, body: csv_body)
before do
allow($stdin).to receive(:tty?).and_return(false)
allow($stdin).to receive(:read).and_return(string_of_ids)
end end
context 'when given a list of space-separated IDs through rake argument' do
it 'enqueues the projects for refresh' do it 'enqueues the projects for refresh' do
expect { run_rake_task(rake_task) }.to output(/Done/).to_stdout expect { run_rake_task(rake_task, csv_url) }.to output(/Done/).to_stdout
expect(Projects::BuildArtifactsSizeRefresh.all.map(&:project)).to match_array([project_1, project_2, project_3]) expect(Projects::BuildArtifactsSizeRefresh.all.map(&:project)).to match_array([project_1, project_2, project_3])
end end
end end
context 'when given a list of space-separated IDs through rake argument' do context 'when CSV has invalid header' do
it 'enqueues the projects for refresh' do let(:csv_body) do
expect { run_rake_task(rake_task, string_of_ids) }.to output(/Done/).to_stdout <<~BODY
projectid
expect(Projects::BuildArtifactsSizeRefresh.all.map(&:project)).to match_array([project_1, project_2, project_3]) #{project_1.id}
end #{project_2.id}
#{project_3.id}
BODY
end end
context 'when not given any IDs' do
it 'returns an error message' do it 'returns an error message' do
expect { run_rake_task(rake_task) }.to output(/Please provide a string of space-separated project IDs/).to_stdout expect { run_rake_task(rake_task, csv_url) }.to output(/Project IDs must be listed in the CSV under the header PROJECT_ID/).to_stdout
end end
end end
end end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment