Merge branch 'zj-job-view-goes-real-time' into 'master'

Initial implementation for real time job view

Closes #31397

See merge request !11651

app/controllers/projects/jobs_controller.rb

@@ -45,6 +45,17 @@ class Projects::JobsController < Projects::ApplicationController
     @builds = @project.pipelines.find_by_sha(@build.sha).builds.order('id DESC')
     @builds = @builds.where("id not in (?)", @build.id)
     @pipeline = @build.pipeline
+
+    respond_to do |format|
+      format.html
+      format.json do
+        Gitlab::PollingInterval.set_header(response, interval: 10_000)
+
+        render json: BuildSerializer
+          .new(project: @project, current_user: @current_user)
+          .represent(@build, {}, BuildDetailsEntity)
+      end
+    end
   end
 
   def trace

app/models/ci/build.rb

@@ -213,14 +213,19 @@ module Ci
     end
 
     def merge_request
-      merge_requests = MergeRequest.includes(:merge_request_diff)
-                                   .where(source_branch: ref,
-                                          source_project: pipeline.project)
-                                   .reorder(iid: :asc)
-
-      merge_requests.find do |merge_request|
-        merge_request.commits_sha.include?(pipeline.sha)
-      end
+      return @merge_request if defined?(@merge_request)
+
+      @merge_request ||=
+        begin
+          merge_requests = MergeRequest.includes(:merge_request_diff)
+            .where(source_branch: ref,
+                   source_project: pipeline.project)
+            .reorder(iid: :desc)
+
+          merge_requests.find do |merge_request|
+            merge_request.commits_sha.include?(pipeline.sha)
+          end
+        end
     end
 
     def repo_url
@@ -344,7 +349,7 @@ module Ci
     end
 
     def has_expiring_artifacts?
-      artifacts_expire_at.present?
+      artifacts_expire_at.present? && artifacts_expire_at > Time.now
     end
 
     def keep_artifacts!

app/models/commit_status.rb

@@ -127,6 +127,11 @@ class CommitStatus < ActiveRecord::Base
     false
   end
 
+  # To be overriden when inherrited from
+  def retryable?
+    false
+  end
+
   def stuck?
     false
   end

app/serializers/build_artifact_entity.rb

 class BuildArtifactEntity < Grape::Entity
   include RequestAwareEntity
 
-  expose :name do |build|
-    build.name
+  expose :name do |job|
+    job.name
   end
 
-  expose :path do |build|
+  expose :artifacts_expired?, as: :expired
+  expose :artifacts_expire_at, as: :expire_at
+
+  expose :path do |job|
     download_namespace_project_job_artifacts_path(
-      build.project.namespace,
-      build.project,
-      build)
+      project.namespace,
+      project,
+      job)
+  end
+
+  expose :keep_path, if: -> (*) { job.has_expiring_artifacts? } do |job|
+    keep_namespace_project_job_artifacts_path(
+      project.namespace,
+      project,
+      job)
+  end
+
+  expose :browse_path do |job|
+    browse_namespace_project_job_artifacts_path(
+      project.namespace,
+      project,
+      job)
+  end
+
+  private
+
+  alias_method :job, :object
+
+  def project
+    job.project
   end
 end

app/serializers/build_details_entity.rb

+class BuildDetailsEntity < BuildEntity
+  expose :coverage, :erased_at, :duration
+  expose :tag_list, as: :tags
+
+  expose :user, using: UserEntity
+
+  expose :erased_by, if: -> (*) { build.erased? }, using: UserEntity
+  expose :erase_path, if: -> (*) { build.erasable? && can?(current_user, :update_build, project) } do |build|
+    erase_namespace_project_job_path(project.namespace, project, build)
+  end
+
+  expose :artifacts, using: BuildArtifactEntity
+  expose :runner, using: RunnerEntity
+  expose :pipeline, using: PipelineEntity
+
+  expose :merge_request, if: -> (*) { can?(current_user, :read_merge_request, build.merge_request) } do
+    expose :iid do |build|
+      build.merge_request.iid
+    end
+
+    expose :path do |build|
+      namespace_project_merge_request_path(project.namespace, project, build.merge_request)
+    end
+  end
+
+  expose :new_issue_path, if: -> (*) { can?(request.current_user, :create_issue, project) && build.failed? } do |build|
+    new_namespace_project_issue_path(project.namespace, project, issue: build_failed_issue_options)
+  end
+
+  expose :raw_path do |build|
+    raw_namespace_project_build_path(project.namespace, project, build)
+  end
+
+  private
+
+  def build_failed_issue_options
+    {
+      title: "Build Failed ##{build.id}",
+      description: namespace_project_job_url(project.namespace, project, build)
+    }
+  end
+
+  def current_user
+    request.current_user
+  end
+
+  def project
+    build.project
+  end
+end

app/serializers/build_entity.rb

@@ -8,7 +8,7 @@ class BuildEntity < Grape::Entity
     path_to(:namespace_project_job, build)
   end
 
-  expose :retry_path do |build|
+  expose :retry_path, if: -> (*) { build&.retryable? } do |build|
     path_to(:retry_namespace_project_job, build)
   end
 

app/serializers/merge_request_entity.rb

@@ -29,7 +29,7 @@ class MergeRequestEntity < IssuableEntity
 
   expose :merge_commit_sha
   expose :merge_commit_message
-  expose :head_pipeline, with: PipelineEntity, as: :pipeline
+  expose :head_pipeline, with: PipelineDetailsEntity, as: :pipeline
 
   # Booleans
   expose :work_in_progress?, as: :work_in_progress

app/serializers/pipeline_details_entity.rb

+class PipelineDetailsEntity < PipelineEntity
+  expose :details do
+    expose :stages, using: StageEntity
+    expose :artifacts, using: BuildArtifactEntity
+    expose :manual_actions, using: BuildActionEntity
+  end
+end

app/serializers/pipeline_entity.rb

@@ -7,6 +7,8 @@ class PipelineEntity < Grape::Entity
   expose :coverage
   expose :source
 
+  expose :created_at, :updated_at
+
   expose :path do |pipeline|
     namespace_project_pipeline_path(
       pipeline.project.namespace,
@@ -14,15 +16,6 @@ class PipelineEntity < Grape::Entity
       pipeline)
   end
 
-  expose :details do
-    expose :detailed_status, as: :status, with: StatusEntity
-    expose :duration
-    expose :finished_at
-    expose :stages, using: StageEntity
-    expose :artifacts, using: BuildArtifactEntity
-    expose :manual_actions, using: BuildActionEntity
-  end
-
   expose :flags do
     expose :latest?, as: :latest
     expose :stuck?, as: :stuck
@@ -31,6 +24,12 @@ class PipelineEntity < Grape::Entity
     expose :can_cancel?, as: :cancelable
   end
 
+  expose :details do
+    expose :detailed_status, as: :status, with: StatusEntity
+    expose :duration
+    expose :finished_at
+  end
+
   expose :ref do
     expose :name do |pipeline|
       pipeline.ref
@@ -47,7 +46,6 @@ class PipelineEntity < Grape::Entity
   end
 
   expose :commit, using: CommitEntity
-  expose :yaml_errors, if: -> (pipeline, _) { pipeline.has_yaml_errors? }
 
   expose :retry_path, if: -> (*) { can_retry? }  do |pipeline|
     retry_namespace_project_pipeline_path(pipeline.project.namespace,
@@ -61,7 +59,7 @@ class PipelineEntity < Grape::Entity
                                            pipeline.id)
   end
 
-  expose :created_at, :updated_at
+  expose :yaml_errors, if: -> (pipeline, _) { pipeline.has_yaml_errors? }
 
   private
 

app/serializers/pipeline_serializer.rb

 class PipelineSerializer < BaseSerializer
   InvalidResourceError = Class.new(StandardError)
 
-  entity PipelineEntity
+  entity PipelineDetailsEntity
 
   def with_pagination(request, response)
     tap { @paginator = Gitlab::Serializer::Pagination.new(request, response) }

app/serializers/runner_entity.rb

+class RunnerEntity < Grape::Entity
+  include RequestAwareEntity
+
+  expose :id, :description
+
+  expose :edit_path,
+    if: -> (*) { can?(request.current_user, :admin_build, project) && runner.specific? } do |runner|
+    edit_namespace_project_runner_path(project.namespace, project, runner)
+  end
+
+  private
+
+  alias_method :runner, :object
+
+  def project
+    request.project
+  end
+end

changelogs/unreleased/zj-job-view-goes-real-time.yml

+---
+title: Job details page update real time
+merge_request: 11651
+author:

lib/gitlab/etag_caching/router.rb

@@ -9,8 +9,8 @@ module Gitlab
       #   - Ending in `noteable/issue/<id>/notes` for the `issue_notes` route
       #   - Ending in `issues/id`/realtime_changes` for the `issue_title` route
       USED_IN_ROUTES = %w[noteable issue notes issues realtime_changes
-                          commit pipelines merge_requests new
-                          environments].freeze
+                          commit pipelines merge_requests builds
+                          new environments].freeze
       RESERVED_WORDS = Gitlab::PathRegex::ILLEGAL_PROJECT_PATH_WORDS - USED_IN_ROUTES
       RESERVED_WORDS_REGEX = Regexp.union(*RESERVED_WORDS.map(&Regexp.method(:escape)))
 
@@ -43,6 +43,10 @@ module Gitlab
           %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/pipelines/\d+\.json\z),
           'project_pipeline'
         ),
+        Gitlab::EtagCaching::Router::Route.new(
+          %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/builds/\d+\.json\z),
+          'project_build'
+        ),
         Gitlab::EtagCaching::Router::Route.new(
           %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/environments\.json\z),
           'environments'

spec/controllers/projects/jobs_controller_spec.rb

@@ -101,26 +101,49 @@ describe Projects::JobsController do
   end
 
   describe 'GET show' do
-    context 'when build exists' do
-      let!(:build) { create(:ci_build, pipeline: pipeline) }
+    let!(:build) { create(:ci_build, :failed, pipeline: pipeline) }
 
-      before do
-        get_show(id: build.id)
+    context 'when requesting HTML' do
+      context 'when build exists' do
+        before do
+          get_show(id: build.id)
+        end
+
+        it 'has a build' do
+          expect(response).to have_http_status(:ok)
+          expect(assigns(:build).id).to eq(build.id)
+        end
       end
 
-      it 'has a build' do
-        expect(response).to have_http_status(:ok)
-        expect(assigns(:build).id).to eq(build.id)
+      context 'when build does not exist' do
+        before do
+          get_show(id: 1234)
+        end
+
+        it 'renders not_found' do
+          expect(response).to have_http_status(:not_found)
+        end
       end
     end
 
-    context 'when build does not exist' do
+    context 'when requesting JSON' do
+      let(:merge_request) { create(:merge_request, source_project: project) }
+
       before do
-        get_show(id: 1234)
+        project.add_developer(user)
+        sign_in(user)
+
+        allow_any_instance_of(Ci::Build).to receive(:merge_request).and_return(merge_request)
+
+        get_show(id: build.id, format: :json)
       end
 
-      it 'renders not_found' do
-        expect(response).to have_http_status(:not_found)
+      it 'exposes needed information' do
+        expect(response).to have_http_status(:ok)
+        expect(json_response['raw_path']).to match(/builds\/\d+\/raw\z/)
+        expect(json_response.dig('merge_request', 'path')).to match(/merge_requests\/\d+\z/)
+        expect(json_response['new_issue_path'])
+          .to include('/issues/new')
       end
     end
 

spec/lib/gitlab/etag_caching/router_spec.rb

@@ -67,6 +67,17 @@ describe Gitlab::EtagCaching::Router do
     expect(result.name).to eq 'merge_request_pipelines'
   end
 
+  it 'matches build endpoint' do
+    env = build_env(
+      '/my-group/my-project/builds/234.json'
+    )
+
+    result = described_class.match(env)
+
+    expect(result).to be_present
+    expect(result.name).to eq 'project_build'
+  end
+
   it 'does not match blob with confusing name' do
     env = build_env(
       '/my-group/my-project/blob/master/pipelines.json'

spec/serializers/build_artifact_entity_spec.rb

 require 'spec_helper'
 
 describe BuildArtifactEntity do
-  let(:job) { create(:ci_build, name: 'test:job') }
+  let(:job) { create(:ci_build, name: 'test:job', artifacts_expire_at: 1.hour.from_now) }
 
   let(:entity) do
     described_class.new(job, request: double)
@@ -14,9 +14,19 @@ describe BuildArtifactEntity do
       expect(subject[:name]).to eq 'test:job'
     end
 
-    it 'contains path to the artifacts' do
+    it 'exposes information about expiration of artifacts' do
+      expect(subject).to include(:expired, :expire_at)
+    end
+
+    it 'contains paths to the artifacts' do
       expect(subject[:path])
         .to include "jobs/#{job.id}/artifacts/download"
+
+      expect(subject[:keep_path])
+        .to include "jobs/#{job.id}/artifacts/keep"
+
+      expect(subject[:browse_path])
+        .to include "jobs/#{job.id}/artifacts/browse"
     end
   end
 end

spec/serializers/build_details_entity_spec.rb

+require 'spec_helper'
+
+describe BuildDetailsEntity do
+  set(:user) { create(:admin) }
+
+  it 'inherits from BuildEntity' do
+    expect(described_class).to be < BuildEntity
+  end
+
+  describe '#as_json' do
+    let(:project) { create(:project, :repository) }
+    let!(:build) { create(:ci_build, :failed, project: project) }
+    let(:request) { double('request') }
+    let(:entity) { described_class.new(build, request: request, current_user: user, project: project) }
+    subject { entity.as_json }
+
+    before do
+      allow(request).to receive(:current_user).and_return(user)
+    end
+
+    context 'when the user has access to issues and merge requests' do
+      let!(:merge_request) do
+        create(:merge_request, source_project: project, source_branch: build.ref)
+      end
+
+      before do
+        allow(build).to receive(:merge_request).and_return(merge_request)
+      end
+
+      it 'contains the needed key value pairs' do
+        expect(subject).to include(:coverage, :erased_at, :duration)
+        expect(subject).to include(:artifacts, :runner, :pipeline)
+        expect(subject).to include(:raw_path, :merge_request)
+        expect(subject).to include(:new_issue_path)
+      end
+
+      it 'exposes details of the merge request' do
+        expect(subject[:merge_request]).to include(:iid, :path)
+      end
+
+      context 'when the build has been erased' do
+        let!(:build) { create(:ci_build, :erasable, project: project) }
+
+        it 'exposes the user whom erased the build' do
+          expect(subject).to include(:erase_path)
+        end
+      end
+
+      context 'when the build has been erased' do
+        let!(:build) { create(:ci_build, erased_at: Time.now, project: project, erased_by: user) }
+
+        it 'exposes the user whom erased the build' do
+          expect(subject).to include(:erased_by)
+        end
+      end
+    end
+
+    context 'when the user can only read the build' do
+      let(:user) { create(:user) }
+
+      it "won't display the paths to issues and merge requests" do
+        expect(subject['new_issue_path']).to be_nil
+        expect(subject['merge_request_path']).to be_nil
+      end
+    end
+  end
+end

spec/serializers/build_entity_spec.rb

@@ -2,7 +2,7 @@ require 'spec_helper'
 
 describe BuildEntity do
   let(:user) { create(:user) }
-  let(:build) { create(:ci_build) }
+  let(:build) { create(:ci_build, :failed) }
   let(:project) { build.project }
   let(:request) { double('request') }
 
@@ -18,6 +18,7 @@ describe BuildEntity do
 
   it 'contains paths to build page and retry action' do
     expect(subject).to include(:build_path, :retry_path)
+    expect(subject[:retry_path]).not_to be_nil
   end
 
   it 'does not contain sensitive information' do

spec/serializers/merge_request_entity_spec.rb

@@ -26,7 +26,7 @@ describe MergeRequestEntity do
     pipeline = build_stubbed(:ci_pipeline)
     allow(resource).to receive(:head_pipeline).and_return(pipeline)
 
-    pipeline_payload = PipelineEntity
+    pipeline_payload = PipelineDetailsEntity
       .represent(pipeline, request: req)
       .as_json
 

spec/serializers/pipeline_details_entity_spec.rb

+require 'spec_helper'
+
+describe PipelineDetailsEntity do
+  set(:user) { create(:user) }
+  let(:request) { double('request') }
+
+  it 'inherrits from PipelineEntity' do
+    expect(described_class).to be < PipelineEntity
+  end
+
+  before do
+    allow(request).to receive(:current_user).and_return(user)
+  end
+
+  let(:entity) do
+    described_class.represent(pipeline, request: request)
+  end
+
+  describe '#as_json' do
+    subject { entity.as_json }
+
+    context 'when pipeline is empty' do
+      let(:pipeline) { create(:ci_empty_pipeline) }
+
+      it 'contains details' do
+        expect(subject).to include :details
+        expect(subject[:details])
+          .to include :duration, :finished_at
+        expect(subject[:details])
+          .to include :stages, :artifacts, :manual_actions
+        expect(subject[:details][:status]).to include :icon, :favicon, :text, :label
+      end
+
+      it 'contains flags' do
+        expect(subject).to include :flags
+        expect(subject[:flags])
+          .to include :latest, :stuck,
+                      :yaml_errors, :retryable, :cancelable
+      end
+    end
+
+    context 'when pipeline is retryable' do
+      let(:project) { create(:empty_project) }
+
+      let(:pipeline) do
+        create(:ci_pipeline, status: :success, project: project)
+      end
+
+      before do
+        create(:ci_build, :failed, pipeline: pipeline)
+      end
+
+      context 'user has ability to retry pipeline' do
+        before { project.team << [user, :developer] }
+
+        it 'retryable flag is true' do
+          expect(subject[:flags][:retryable]).to eq true
+        end
+      end
+
+      context 'user does not have ability to retry pipeline' do
+        it 'retryable flag is false' do
+          expect(subject[:flags][:retryable]).to eq false
+        end
+      end
+    end
+
+    context 'when pipeline is cancelable' do
+      let(:project) { create(:empty_project) }
+
+      let(:pipeline) do
+        create(:ci_pipeline, status: :running, project: project)
+      end
+
+      before do
+        create(:ci_build, :pending, pipeline: pipeline)
+      end
+
+      context 'user has ability to cancel pipeline' do
+        before { project.add_developer(user) }
+
+        it 'cancelable flag is true' do
+          expect(subject[:flags][:cancelable]).to eq true
+        end
+      end
+
+      context 'user does not have ability to cancel pipeline' do
+        it 'cancelable flag is false' do
+          expect(subject[:flags][:cancelable]).to eq false
+        end
+      end
+    end
+
+    context 'when pipeline has YAML errors' do
+      let(:pipeline) do
+        create(:ci_pipeline, config: { rspec: { invalid: :value } })
+      end
+
+      it 'contains information about error' do
+        expect(subject[:yaml_errors]).to be_present
+      end
+
+      it 'contains flag that indicates there are errors' do
+        expect(subject[:flags][:yaml_errors]).to be true
+      end
+    end
+
+    context 'when pipeline does not have YAML errors' do
+      let(:pipeline) { create(:ci_empty_pipeline) }
+
+      it 'does not contain field that normally holds an error' do
+        expect(subject).not_to have_key(:yaml_errors)
+      end
+
+      it 'contains flag that indicates there are no errors' do
+        expect(subject[:flags][:yaml_errors]).to be false
+      end
+    end
+  end
+end

spec/serializers/pipeline_entity_spec.rb

 require 'spec_helper'
 
 describe PipelineEntity do
-  let(:user) { create(:user) }
+  set(:user) { create(:user) }
   let(:request) { double('request') }
 
   before do
@@ -28,8 +28,6 @@ describe PipelineEntity do
         expect(subject).to include :details
         expect(subject[:details])
           .to include :duration, :finished_at
-        expect(subject[:details])
-          .to include :stages, :artifacts, :manual_actions
         expect(subject[:details][:status]).to include :icon, :favicon, :text, :label
       end
 
@@ -55,20 +53,12 @@ describe PipelineEntity do
       context 'user has ability to retry pipeline' do
         before { project.team << [user, :developer] }
 
-        it 'retryable flag is true' do
-          expect(subject[:flags][:retryable]).to eq true
-        end
-
         it 'contains retry path' do
           expect(subject[:retry_path]).to be_present
         end
       end
 
       context 'user does not have ability to retry pipeline' do
-        it 'retryable flag is false' do
-          expect(subject[:flags][:retryable]).to eq false
-        end
-
         it 'does not contain retry path' do
           expect(subject).not_to have_key(:retry_path)
         end
@@ -87,11 +77,7 @@ describe PipelineEntity do
       end
 
       context 'user has ability to cancel pipeline' do
-        before { project.team << [user, :developer] }
-
-        it 'cancelable flag is true' do
-          expect(subject[:flags][:cancelable]).to eq true
-        end
+        before { project.add_developer(user) }
 
         it 'contains cancel path' do
           expect(subject[:cancel_path]).to be_present
@@ -99,42 +85,12 @@ describe PipelineEntity do
       end
 
       context 'user does not have ability to cancel pipeline' do
-        it 'cancelable flag is false' do
-          expect(subject[:flags][:cancelable]).to eq false
-        end
-
         it 'does not contain cancel path' do
           expect(subject).not_to have_key(:cancel_path)
         end
       end
     end
 
-    context 'when pipeline has YAML errors' do
-      let(:pipeline) do
-        create(:ci_pipeline, config: { rspec: { invalid: :value } })
-      end
-
-      it 'contains flag that indicates there are errors' do
-        expect(subject[:flags][:yaml_errors]).to be true
-      end
-
-      it 'contains information about error' do
-        expect(subject[:yaml_errors]).to be_present
-      end
-    end
-
-    context 'when pipeline does not have YAML errors' do
-      let(:pipeline) { create(:ci_empty_pipeline) }
-
-      it 'contains flag that indicates there are no errors' do
-        expect(subject[:flags][:yaml_errors]).to be false
-      end
-
-      it 'does not contain field that normally holds an error' do
-        expect(subject).not_to have_key(:yaml_errors)
-      end
-    end
-
     context 'when pipeline ref is empty' do
       let(:pipeline) { create(:ci_empty_pipeline) }
 

spec/serializers/runner_entity_spec.rb

+require 'spec_helper'
+
+describe RunnerEntity do
+  let(:runner) { create(:ci_runner, :specific) }
+  let(:entity) { described_class.new(runner, request: request, current_user: user) }
+  let(:request) { double('request') }
+  let(:project) { create(:empty_project) }
+  let(:user) { create(:admin) }
+
+  before do
+    allow(request).to receive(:current_user).and_return(user)
+    allow(request).to receive(:project).and_return(project)
+  end
+
+  describe '#as_json' do
+    subject { entity.as_json }
+
+    it 'contains required fields' do
+      expect(subject).to include(:id, :description)
+      expect(subject).to include(:edit_path)
+    end
+  end
+end