Merge branch 'eb-download-daily-coverage-csv' into 'master'

Implement download feature for daily code coverage

See merge request gitlab-org/gitlab!27094

app/controllers/projects/ci/daily_build_group_report_results_controller.rb

+# frozen_string_literal: true
+
+class Projects::Ci::DailyBuildGroupReportResultsController < Projects::ApplicationController
+  include Gitlab::Utils::StrongMemoize
+
+  MAX_ITEMS = 1000
+  REPORT_WINDOW = 90.days
+
+  before_action :validate_feature_flag!
+  before_action :authorize_download_code! # Share the same authorization rules as the graphs controller
+  before_action :validate_param_type!
+
+  def index
+    respond_to do |format|
+      format.csv { send_data(render_csv(results), type: 'text/csv; charset=utf-8') }
+    end
+  end
+
+  private
+
+  def validate_feature_flag!
+    render_404 unless Feature.enabled?(:ci_download_daily_code_coverage, project, default_enabled: true)
+  end
+
+  def validate_param_type!
+    respond_422 unless allowed_param_types.include?(param_type)
+  end
+
+  def render_csv(collection)
+    CsvBuilders::SingleBatch.new(
+      collection,
+      {
+        date: 'date',
+        group_name: 'group_name',
+        param_type => -> (record) { record.data[param_type] }
+      }
+    ).render
+  end
+
+  def results
+    Ci::DailyBuildGroupReportResultsFinder.new(finder_params).execute
+  end
+
+  def finder_params
+    {
+      current_user: current_user,
+      project: project,
+      ref_path: params.require(:ref_path),
+      start_date: start_date,
+      end_date: end_date,
+      limit: MAX_ITEMS
+    }
+  end
+
+  def start_date
+    strong_memoize(:start_date) do
+      start_date = Date.parse(params.require(:start_date))
+
+      # The start_date cannot be older than `end_date - 90 days`
+      [start_date, end_date - REPORT_WINDOW].max
+    end
+  end
+
+  def end_date
+    strong_memoize(:end_date) do
+      Date.parse(params.require(:end_date))
+    end
+  end
+
+  def allowed_param_types
+    Ci::DailyBuildGroupReportResult::PARAM_TYPES
+  end
+
+  def param_type
+    params.require(:param_type)
+  end
+end

app/controllers/projects/graphs_controller.rb

@@ -28,6 +28,7 @@ class Projects::GraphsController < Projects::ApplicationController
   def charts
     get_commits
     get_languages
+    get_daily_coverage_options
   end
 
   def ci
@@ -52,6 +53,27 @@ class Projects::GraphsController < Projects::ApplicationController
       end
   end
 
+  def get_daily_coverage_options
+    return unless Feature.enabled?(:ci_download_daily_code_coverage, default_enabled: true)
+
+    date_today = Date.current
+    report_window = Projects::Ci::DailyBuildGroupReportResultsController::REPORT_WINDOW
+
+    @daily_coverage_options = {
+      base_params: {
+        start_date: date_today - report_window,
+        end_date: date_today,
+        ref_path: @project.repository.expand_ref(@ref),
+        param_type: 'coverage'
+      },
+      download_path: namespace_project_ci_daily_build_group_report_results_path(
+        namespace_id: @project.namespace,
+        project_id: @project,
+        format: :csv
+      )
+    }
+  end
+
   def fetch_graph
     @commits = @project.repository.commits(@ref, limit: 6000, skip_merges: true)
     @log = []

app/finders/ci/daily_build_group_report_results_finder.rb

+# frozen_string_literal: true
+
+module Ci
+  class DailyBuildGroupReportResultsFinder
+    include Gitlab::Allowable
+
+    def initialize(current_user:, project:, ref_path:, start_date:, end_date:, limit: nil)
+      @current_user = current_user
+      @project = project
+      @ref_path = ref_path
+      @start_date = start_date
+      @end_date = end_date
+      @limit = limit
+    end
+
+    def execute
+      return none unless can?(current_user, :download_code, project)
+
+      Ci::DailyBuildGroupReportResult.recent_results(
+        {
+          project_id: project,
+          ref_path: ref_path,
+          date: start_date..end_date
+        },
+        limit: @limit
+      )
+    end
+
+    private
+
+    attr_reader :current_user, :project, :ref_path, :start_date, :end_date
+
+    def none
+      Ci::DailyBuildGroupReportResult.none
+    end
+  end
+end

app/models/ci/daily_build_group_report_result.rb

@@ -4,11 +4,17 @@ module Ci
   class DailyBuildGroupReportResult < ApplicationRecord
     extend Gitlab::Ci::Model
 
+    PARAM_TYPES = %w[coverage].freeze
+
     belongs_to :last_pipeline, class_name: 'Ci::Pipeline', foreign_key: :last_pipeline_id
     belongs_to :project
 
     def self.upsert_reports(data)
       upsert_all(data, unique_by: :index_daily_build_group_report_results_unique_columns) if data.any?
     end
+
+    def self.recent_results(attrs, limit: nil)
+      where(attrs).order(date: :desc, group_name: :asc).limit(limit)
+    end
   end
 end

app/services/ci/daily_build_group_report_result_service.rb

@@ -21,7 +21,9 @@ module Ci
       aggregate(pipeline.builds.with_coverage).map do |group_name, group|
         base_attrs.merge(
           group_name: group_name,
-          data: { coverage: average_coverage(group) }
+          data: {
+            'coverage' => average_coverage(group)
+          }
         )
       end
     end

app/views/projects/graphs/charts.html.haml

@@ -13,6 +13,23 @@
 
   #js-languages-chart{ data: { chart_data: @languages.to_json.html_safe } }
 
+- if defined?(@daily_coverage_options)
+  .repo-charts.my-5
+    .sub-header-block.border-top
+    .d-flex.justify-content-between.align-items-center
+      %h4.sub-header.m-0
+        - start_date = capture do
+          #{@daily_coverage_options[:base_params][:start_date].strftime('%b %d')}
+        - end_date = capture do
+          #{@daily_coverage_options[:base_params][:end_date].strftime('%b %d')}
+        = (_("Code coverage statistics for master %{start_date} - %{end_date}") % {start_date: start_date, end_date: end_date})
+      - download_path = capture do
+        #{@daily_coverage_options[:download_path]}
+      %a.btn.btn-sm{ href: "#{download_path}?#{@daily_coverage_options[:base_params].to_query}" }
+        %small
+          = _("Download raw data (.csv)")
+    #js-code-coverage-chart{ data: { daily_coverage_options: @daily_coverage_options.to_json.html_safe } }
+
 .repo-charts
   .sub-header-block.border-top
 

changelogs/unreleased/eb-download-daily-coverage-csv.yml

+---
+title: Allow users to download a CSV of the recent daily code coverage values per
+  job
+merge_request: 27094
+author:
+type: added

config/routes/project.rb

@@ -65,6 +65,7 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do
 
         namespace :ci do
           resource :lint, only: [:show, :create]
+          resources :daily_build_group_report_results, only: [:index], constraints: { format: 'csv' }
         end
 
         namespace :settings do

doc/ci/pipelines/settings.md

@@ -130,6 +130,16 @@ in the jobs table.
 A few examples of known coverage tools for a variety of languages can be found
 in the pipelines settings page.
 
+### Download test coverage history
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/209121) in GitLab 12.10.
+
+If you want to see the evolution of your project code coverage over time,
+you can download a CSV file with this data. From your project:
+
+1. Go to **{chart}** **Project Analytics > Repository**.
+1. Click **Download raw data (.csv)**
+
 ### Removing color codes
 
 Some test coverage tools output with ANSI color codes that won't be

lib/csv_builder.rb

@@ -14,6 +14,9 @@
 #     CsvBuilder.new(@posts, columns).render
 #
 class CsvBuilder
+  DEFAULT_ORDER_BY = 'id'.freeze
+  DEFAULT_BATCH_SIZE = 1000
+
   attr_reader :rows_written
 
   #
@@ -68,6 +71,12 @@ class CsvBuilder
     }
   end
 
+  protected
+
+  def each(&block)
+    @collection.find_each(&block) # rubocop: disable CodeReuse/ActiveRecord
+  end
+
   private
 
   def headers
@@ -91,7 +100,7 @@ class CsvBuilder
   def write_csv(csv, until_condition:)
     csv << headers
 
-    @collection.find_each do |object|
+    each do |object|
       csv << row(object)
 
       @rows_written += 1

lib/csv_builders/single_batch.rb

+# frozen_string_literal: true
+
+module CsvBuilders
+  class SingleBatch < CsvBuilder
+    protected
+
+    def each(&block)
+      @collection.each(&block)
+    end
+  end
+end

locale/gitlab.pot

@@ -5327,6 +5327,9 @@ msgstr ""
 msgid "Code Review Analytics displays a table of open merge requests considered to be in code review. There are currently no merge requests in review for this project and/or filters."
 msgstr ""
 
+msgid "Code coverage statistics for master %{start_date} - %{end_date}"
+msgstr ""
+
 msgid "Code owner approval is required"
 msgstr ""
 
@@ -7673,6 +7676,9 @@ msgstr ""
 msgid "Download license"
 msgstr ""
 
+msgid "Download raw data (.csv)"
+msgstr ""
+
 msgid "Download source code"
 msgstr ""
 

spec/controllers/projects/ci/daily_build_group_report_results_controller_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Projects::Ci::DailyBuildGroupReportResultsController do
+  describe 'GET index' do
+    let(:project) { create(:project, :public, :repository) }
+    let(:ref_path) { 'refs/heads/master' }
+    let(:param_type) { 'coverage' }
+    let(:start_date) { '2019-12-10' }
+    let(:end_date) { '2020-03-09' }
+
+    def create_daily_coverage(group_name, coverage, date)
+      create(
+        :ci_daily_build_group_report_result,
+        project: project,
+        ref_path: ref_path,
+        group_name: group_name,
+        data: { 'coverage' => coverage },
+        date: date
+      )
+    end
+
+    def csv_response
+      CSV.parse(response.body)
+    end
+
+    before do
+      create_daily_coverage('rspec', 79.0, '2020-03-09')
+      create_daily_coverage('karma', 81.0, '2019-12-10')
+      create_daily_coverage('rspec', 67.0, '2019-12-09')
+      create_daily_coverage('karma', 71.0, '2019-12-09')
+
+      get :index, params: {
+        namespace_id: project.namespace,
+        project_id: project,
+        ref_path: ref_path,
+        param_type: param_type,
+        start_date: start_date,
+        end_date: end_date,
+        format: :csv
+      }
+    end
+
+    it 'serves the results in CSV' do
+      expect(response).to have_gitlab_http_status(:ok)
+      expect(response.headers['Content-Type']).to eq('text/csv; charset=utf-8')
+
+      expect(csv_response).to eq([
+        %w[date group_name coverage],
+        ['2020-03-09', 'rspec', '79.0'],
+        ['2019-12-10', 'karma', '81.0']
+      ])
+    end
+
+    context 'when given date range spans more than 90 days' do
+      let(:start_date) { '2019-12-09' }
+      let(:end_date) { '2020-03-09' }
+
+      it 'limits the result to 90 days from the given start_date' do
+        expect(response).to have_gitlab_http_status(:ok)
+        expect(response.headers['Content-Type']).to eq('text/csv; charset=utf-8')
+
+        expect(csv_response).to eq([
+          %w[date group_name coverage],
+          ['2020-03-09', 'rspec', '79.0'],
+          ['2019-12-10', 'karma', '81.0']
+        ])
+      end
+    end
+
+    context 'when given param_type is invalid' do
+      let(:param_type) { 'something_else' }
+
+      it 'responds with 422 error' do
+        expect(response).to have_gitlab_http_status(:unprocessable_entity)
+      end
+    end
+  end
+end

spec/controllers/projects/graphs_controller_spec.rb

@@ -41,6 +41,26 @@ describe Projects::GraphsController do
         expect(response).to have_gitlab_http_status(:ok)
         expect(response).to render_template(:charts)
       end
+
+      it 'sets the daily coverage options' do
+        Timecop.freeze do
+          get(:charts, params: { namespace_id: project.namespace.path, project_id: project.path, id: 'master' })
+
+          expect(assigns[:daily_coverage_options]).to eq(
+            base_params: {
+              start_date: Time.now.to_date - 90.days,
+              end_date: Time.now.to_date,
+              ref_path: project.repository.expand_ref('master'),
+              param_type: 'coverage'
+            },
+            download_path: namespace_project_ci_daily_build_group_report_results_path(
+              namespace_id: project.namespace,
+              project_id: project,
+              format: :csv
+            )
+          )
+        end
+      end
     end
 
     context 'when languages were previously detected' do

spec/factories/ci/daily_build_group_report_results.rb

@@ -8,7 +8,7 @@ FactoryBot.define do
     last_pipeline factory: :ci_pipeline
     group_name { 'rspec' }
     data do
-      { coverage: 77.0 }
+      { 'coverage' => 77.0 }
     end
   end
 end

spec/finders/ci/daily_build_group_report_results_finder_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Ci::DailyBuildGroupReportResultsFinder do
+  describe '#execute' do
+    let(:project) { create(:project, :private) }
+    let(:ref_path) { 'refs/heads/master' }
+    let(:limit) { nil }
+
+    def create_daily_coverage(group_name, coverage, date)
+      create(
+        :ci_daily_build_group_report_result,
+        project: project,
+        ref_path: ref_path,
+        group_name: group_name,
+        data: { 'coverage' => coverage },
+        date: date
+      )
+    end
+
+    let!(:rspec_coverage_1) { create_daily_coverage('rspec', 79.0, '2020-03-09') }
+    let!(:karma_coverage_1) { create_daily_coverage('karma', 89.0, '2020-03-09') }
+    let!(:rspec_coverage_2) { create_daily_coverage('rspec', 95.0, '2020-03-10') }
+    let!(:karma_coverage_2) { create_daily_coverage('karma', 92.0, '2020-03-10') }
+    let!(:rspec_coverage_3) { create_daily_coverage('rspec', 97.0, '2020-03-11') }
+    let!(:karma_coverage_3) { create_daily_coverage('karma', 99.0, '2020-03-11') }
+
+    subject do
+      described_class.new(
+        current_user: current_user,
+        project: project,
+        ref_path: ref_path,
+        start_date: '2020-03-09',
+        end_date: '2020-03-10',
+        limit: limit
+      ).execute
+    end
+
+    context 'when current user is allowed to download project code' do
+      let(:current_user) { project.owner }
+
+      it 'returns all matching results within the given date range' do
+        expect(subject).to match_array([
+          karma_coverage_2,
+          rspec_coverage_2,
+          karma_coverage_1,
+          rspec_coverage_1
+        ])
+      end
+
+      context 'and limit is specified' do
+        let(:limit) { 2 }
+
+        it 'returns limited number of matching results within the given date range' do
+          expect(subject).to match_array([
+            karma_coverage_2,
+            rspec_coverage_2
+          ])
+        end
+      end
+    end
+
+    context 'when current user is not allowed to download project code' do
+      let(:current_user) { create(:user) }
+
+      it 'returns an empty result' do
+        expect(subject).to be_empty
+      end
+    end
+  end
+end