Allow downloading of daily coverage

This is the backend work for the download feature.
The contoller exposes options for the frontend to use for rendering
the download button and later on the actual code coverag graph.

Also, adds a single batch csv builder into its own class.

This simplifies things and makes intent clearer when using a
separate class for single batch CSV builder.

app/controllers/projects/ci/daily_build_group_report_results_controller.rb

+# frozen_string_literal: true
+
+class Projects::Ci::DailyBuildGroupReportResultsController < Projects::ApplicationController
+  include Gitlab::Utils::StrongMemoize
+
+  MAX_ITEMS = 1000
+  REPORT_WINDOW = 90.days
+
+  before_action :validate_feature_flag!
+  before_action :authorize_download_code! # Share the same authorization rules as the graphs controller
+  before_action :validate_param_type!
+
+  def index
+    respond_to do |format|
+      format.csv { send_data(render_csv(results), type: 'text/csv; charset=utf-8') }
+    end
+  end
+
+  private
+
+  def validate_feature_flag!
+    render_404 unless Feature.enabled?(:ci_download_daily_code_coverage, project, default_enabled: true)
+  end
+
+  def validate_param_type!
+    respond_422 unless allowed_param_types.include?(param_type)
+  end
+
+  def render_csv(collection)
+    CsvBuilders::SingleBatch.new(
+      collection,
+      {
+        date: 'date',
+        group_name: 'group_name',
+        param_type => -> (record) { record.data[param_type] }
+      }
+    ).render
+  end
+
+  def results
+    Ci::DailyBuildGroupReportResultsFinder.new(finder_params).execute
+  end
+
+  def finder_params
+    {
+      current_user: current_user,
+      project: project,
+      ref_path: params.require(:ref_path),
+      start_date: start_date,
+      end_date: end_date,
+      limit: MAX_ITEMS
+    }
+  end
+
+  def start_date
+    strong_memoize(:start_date) do
+      start_date = Date.parse(params.require(:start_date))
+
+      # The start_date cannot be older than `end_date - 90 days`
+      [start_date, end_date - REPORT_WINDOW].max
+    end
+  end
+
+  def end_date
+    strong_memoize(:end_date) do
+      Date.parse(params.require(:end_date))
+    end
+  end
+
+  def allowed_param_types
+    Ci::DailyBuildGroupReportResult::PARAM_TYPES
+  end
+
+  def param_type
+    params.require(:param_type)
+  end
+end

app/controllers/projects/graphs_controller.rb

@@ -28,6 +28,7 @@ class Projects::GraphsController < Projects::ApplicationController
   def charts
     get_commits
     get_languages
+    get_daily_coverage_options
   end
 
   def ci
@@ -52,6 +53,27 @@ class Projects::GraphsController < Projects::ApplicationController
       end
   end
 
+  def get_daily_coverage_options
+    return unless Feature.enabled?(:ci_download_daily_code_coverage, default_enabled: true)
+
+    date_today = Date.current
+    report_window = Projects::Ci::DailyBuildGroupReportResultsController::REPORT_WINDOW
+
+    @daily_coverage_options = {
+      base_params: {
+        start_date: date_today - report_window,
+        end_date: date_today,
+        ref_path: @project.repository.expand_ref(@ref),
+        param_type: 'coverage'
+      },
+      download_path: namespace_project_ci_daily_build_group_report_results_path(
+        namespace_id: @project.namespace,
+        project_id: @project,
+        format: :csv
+      )
+    }
+  end
+
   def fetch_graph
     @commits = @project.repository.commits(@ref, limit: 6000, skip_merges: true)
     @log = []

app/finders/ci/daily_build_group_report_results_finder.rb

+# frozen_string_literal: true
+
+module Ci
+  class DailyBuildGroupReportResultsFinder
+    include Gitlab::Allowable
+
+    def initialize(current_user:, project:, ref_path:, start_date:, end_date:, limit: nil)
+      @current_user = current_user
+      @project = project
+      @ref_path = ref_path
+      @start_date = start_date
+      @end_date = end_date
+      @limit = limit
+    end
+
+    def execute
+      return none unless can?(current_user, :download_code, project)
+
+      Ci::DailyBuildGroupReportResult.recent_results(
+        {
+          project_id: project,
+          ref_path: ref_path,
+          date: start_date..end_date
+        },
+        limit: @limit
+      )
+    end
+
+    private
+
+    attr_reader :current_user, :project, :ref_path, :start_date, :end_date
+
+    def none
+      Ci::DailyBuildGroupReportResult.none
+    end
+  end
+end

app/models/ci/daily_build_group_report_result.rb

@@ -4,11 +4,17 @@ module Ci
   class DailyBuildGroupReportResult < ApplicationRecord
     extend Gitlab::Ci::Model
 
+    PARAM_TYPES = %w[coverage].freeze
+
     belongs_to :last_pipeline, class_name: 'Ci::Pipeline', foreign_key: :last_pipeline_id
     belongs_to :project
 
     def self.upsert_reports(data)
       upsert_all(data, unique_by: :index_daily_build_group_report_results_unique_columns) if data.any?
     end
+
+    def self.recent_results(attrs, limit: nil)
+      where(attrs).order(date: :desc, group_name: :asc).limit(limit)
+    end
   end
 end

app/services/ci/daily_build_group_report_result_service.rb

@@ -21,7 +21,9 @@ module Ci
       aggregate(pipeline.builds.with_coverage).map do |group_name, group|
         base_attrs.merge(
           group_name: group_name,
-          data: { coverage: average_coverage(group) }
+          data: {
+            'coverage' => average_coverage(group)
+          }
         )
       end
     end

app/views/projects/graphs/charts.html.haml

@@ -13,6 +13,10 @@
 
   #js-languages-chart{ data: { chart_data: @languages.to_json.html_safe } }
 
+- if defined?(@daily_coverage_options)
+  .repo-charts
+    #js-code-coverage-chart{ data: { daily_coverage_options: @daily_coverage_options.to_json.html_safe } }
+
 .repo-charts
   .sub-header-block.border-top
 

changelogs/unreleased/eb-download-daily-coverage-csv.yml

+---
+title: Allow users to download a CSV of the recent daily code coverage values per
+  job
+merge_request: 27094
+author:
+type: added

config/routes/project.rb

@@ -65,6 +65,7 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do
 
         namespace :ci do
           resource :lint, only: [:show, :create]
+          resources :daily_build_group_report_results, only: [:index], constraints: { format: 'csv' }
         end
 
         namespace :settings do

lib/csv_builder.rb

@@ -14,6 +14,9 @@
 #     CsvBuilder.new(@posts, columns).render
 #
 class CsvBuilder
+  DEFAULT_ORDER_BY = 'id'.freeze
+  DEFAULT_BATCH_SIZE = 1000
+
   attr_reader :rows_written
 
   #
@@ -68,6 +71,12 @@ class CsvBuilder
     }
   end
 
+  protected
+
+  def each(&block)
+    @collection.find_each(&block) # rubocop: disable CodeReuse/ActiveRecord
+  end
+
   private
 
   def headers
@@ -91,7 +100,7 @@ class CsvBuilder
   def write_csv(csv, until_condition:)
     csv << headers
 
-    @collection.find_each do |object|
+    each do |object|
       csv << row(object)
 
       @rows_written += 1

lib/csv_builders/single_batch.rb

+# frozen_string_literal: true
+
+module CsvBuilders
+  class SingleBatch < CsvBuilder
+    protected
+
+    def each(&block)
+      @collection.each(&block)
+    end
+  end
+end

spec/controllers/projects/ci/daily_build_group_report_results_controller_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Projects::Ci::DailyBuildGroupReportResultsController do
+  describe 'GET index' do
+    let(:project) { create(:project, :public, :repository) }
+    let(:ref_path) { 'refs/heads/master' }
+    let(:param_type) { 'coverage' }
+    let(:start_date) { '2019-12-10' }
+    let(:end_date) { '2020-03-09' }
+
+    def create_daily_coverage(group_name, coverage, date)
+      create(
+        :ci_daily_build_group_report_result,
+        project: project,
+        ref_path: ref_path,
+        group_name: group_name,
+        data: { 'coverage' => coverage },
+        date: date
+      )
+    end
+
+    def csv_response
+      CSV.parse(response.body)
+    end
+
+    before do
+      create_daily_coverage('rspec', 79.0, '2020-03-09')
+      create_daily_coverage('karma', 81.0, '2019-12-10')
+      create_daily_coverage('rspec', 67.0, '2019-12-09')
+      create_daily_coverage('karma', 71.0, '2019-12-09')
+
+      get :index, params: {
+        namespace_id: project.namespace,
+        project_id: project,
+        ref_path: ref_path,
+        param_type: param_type,
+        start_date: start_date,
+        end_date: end_date,
+        format: :csv
+      }
+    end
+
+    it 'serves the results in CSV' do
+      expect(response).to have_gitlab_http_status(:ok)
+      expect(response.headers['Content-Type']).to eq('text/csv; charset=utf-8')
+
+      expect(csv_response).to eq([
+        %w[date group_name coverage],
+        ['2020-03-09', 'rspec', '79.0'],
+        ['2019-12-10', 'karma', '81.0']
+      ])
+    end
+
+    context 'when given date range spans more than 90 days' do
+      let(:start_date) { '2019-12-09' }
+      let(:end_date) { '2020-03-09' }
+
+      it 'limits the result to 90 days from the given start_date' do
+        expect(response).to have_gitlab_http_status(:ok)
+        expect(response.headers['Content-Type']).to eq('text/csv; charset=utf-8')
+
+        expect(csv_response).to eq([
+          %w[date group_name coverage],
+          ['2020-03-09', 'rspec', '79.0'],
+          ['2019-12-10', 'karma', '81.0']
+        ])
+      end
+    end
+
+    context 'when given param_type is invalid' do
+      let(:param_type) { 'something_else' }
+
+      it 'responds with 422 error' do
+        expect(response).to have_gitlab_http_status(:unprocessable_entity)
+      end
+    end
+  end
+end

spec/controllers/projects/graphs_controller_spec.rb

@@ -41,6 +41,26 @@ describe Projects::GraphsController do
         expect(response).to have_gitlab_http_status(:ok)
         expect(response).to render_template(:charts)
       end
+
+      it 'sets the daily coverage options' do
+        Timecop.freeze do
+          get(:charts, params: { namespace_id: project.namespace.path, project_id: project.path, id: 'master' })
+
+          expect(assigns[:daily_coverage_options]).to eq(
+            base_params: {
+              start_date: Time.now.to_date - 90.days,
+              end_date: Time.now.to_date,
+              ref_path: project.repository.expand_ref('master'),
+              param_type: 'coverage'
+            },
+            download_path: namespace_project_ci_daily_build_group_report_results_path(
+              namespace_id: project.namespace,
+              project_id: project,
+              format: :csv
+            )
+          )
+        end
+      end
     end
 
     context 'when languages were previously detected' do

spec/factories/ci/daily_build_group_report_results.rb

@@ -8,7 +8,7 @@ FactoryBot.define do
     last_pipeline factory: :ci_pipeline
     group_name { 'rspec' }
     data do
-      { coverage: 77.0 }
+      { 'coverage' => 77.0 }
     end
   end
 end

spec/finders/ci/daily_build_group_report_results_finder_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Ci::DailyBuildGroupReportResultsFinder do
+  describe '#execute' do
+    let(:project) { create(:project, :private) }
+    let(:ref_path) { 'refs/heads/master' }
+    let(:limit) { nil }
+
+    def create_daily_coverage(group_name, coverage, date)
+      create(
+        :ci_daily_build_group_report_result,
+        project: project,
+        ref_path: ref_path,
+        group_name: group_name,
+        data: { 'coverage' => coverage },
+        date: date
+      )
+    end
+
+    let!(:rspec_coverage_1) { create_daily_coverage('rspec', 79.0, '2020-03-09') }
+    let!(:karma_coverage_1) { create_daily_coverage('karma', 89.0, '2020-03-09') }
+    let!(:rspec_coverage_2) { create_daily_coverage('rspec', 95.0, '2020-03-10') }
+    let!(:karma_coverage_2) { create_daily_coverage('karma', 92.0, '2020-03-10') }
+    let!(:rspec_coverage_3) { create_daily_coverage('rspec', 97.0, '2020-03-11') }
+    let!(:karma_coverage_3) { create_daily_coverage('karma', 99.0, '2020-03-11') }
+
+    subject do
+      described_class.new(
+        current_user: current_user,
+        project: project,
+        ref_path: ref_path,
+        start_date: '2020-03-09',
+        end_date: '2020-03-10',
+        limit: limit
+      ).execute
+    end
+
+    context 'when current user is allowed to download project code' do
+      let(:current_user) { project.owner }
+
+      it 'returns all matching results within the given date range' do
+        expect(subject).to match_array([
+          karma_coverage_2,
+          rspec_coverage_2,
+          karma_coverage_1,
+          rspec_coverage_1
+        ])
+      end
+
+      context 'and limit is specified' do
+        let(:limit) { 2 }
+
+        it 'returns limited number of matching results within the given date range' do
+          expect(subject).to match_array([
+            karma_coverage_2,
+            rspec_coverage_2
+          ])
+        end
+      end
+    end
+
+    context 'when current user is not allowed to download project code' do
+      let(:current_user) { create(:user) }
+
+      it 'returns an empty result' do
+        expect(subject).to be_empty
+      end
+    end
+  end
+end