Merge branch 'zj-real-time-pipelines' into 'master'

Real time pipeline show action

Closes #25226

See merge request !10777

app/controllers/projects/pipelines_controller.rb

@@ -8,6 +8,8 @@ class Projects::PipelinesController < Projects::ApplicationController
 
   wrap_parameters Ci::Pipeline
 
+  POLLING_INTERVAL = 10_000
+
   def index
     @scope = params[:scope]
     @pipelines = PipelinesFinder
@@ -31,7 +33,7 @@ class Projects::PipelinesController < Projects::ApplicationController
     respond_to do |format|
       format.html
       format.json do
-        Gitlab::PollingInterval.set_header(response, interval: 10_000)
+        Gitlab::PollingInterval.set_header(response, interval: POLLING_INTERVAL)
 
         render json: {
           pipelines: PipelineSerializer
@@ -57,15 +59,25 @@ class Projects::PipelinesController < Projects::ApplicationController
     @pipeline = Ci::CreatePipelineService
       .new(project, current_user, create_params)
       .execute(ignore_skip_ci: true, save_on_errors: false)
-    unless @pipeline.persisted?
-      render 'new'
-      return
-    end
 
+    if @pipeline.persisted?
       redirect_to namespace_project_pipeline_path(project.namespace, project, @pipeline)
+    else
+      render 'new'
+    end
   end
 
   def show
+    respond_to do |format|
+      format.html
+      format.json do
+        Gitlab::PollingInterval.set_header(response, interval: POLLING_INTERVAL)
+
+        render json: PipelineSerializer
+          .new(project: @project, user: @current_user)
+          .represent(@pipeline, grouped: true)
+      end
+    end
   end
 
   def builds

app/models/ci/group.rb

+module Ci
+  ##
+  # This domain model is a representation of a group of jobs that are related
+  # to each other, like `rspec 0 1`, `rspec 0 2`.
+  #
+  # It is not persisted in the database.
+  #
+  class Group
+    include StaticModel
+
+    attr_reader :stage, :name, :jobs
+
+    delegate :size, to: :jobs
+
+    def initialize(stage, name:, jobs:)
+      @stage = stage
+      @name = name
+      @jobs = jobs
+    end
+
+    def status
+      @status ||= commit_statuses.status
+    end
+
+    def detailed_status(current_user)
+      if jobs.one?
+        jobs.first.detailed_status(current_user)
+      else
+        Gitlab::Ci::Status::Group::Factory
+          .new(self, current_user).fabricate!
+      end
+    end
+
+    private
+
+    def commit_statuses
+      @commit_statuses ||= CommitStatus.where(id: jobs.map(&:id))
+    end
+  end
+end

app/models/ci/stage.rb

@@ -15,6 +15,14 @@ module Ci
       @warnings = warnings
     end
 
+    def groups
+      @groups ||= statuses.ordered.latest
+        .sort_by(&:sortable_name).group_by(&:group_name)
+        .map do |group_name, grouped_statuses|
+          Ci::Group.new(self, name: group_name, jobs: grouped_statuses)
+        end
+    end
+
     def to_param
       name
     end

app/serializers/job_group_entity.rb

+class JobGroupEntity < Grape::Entity
+  include RequestAwareEntity
+
+  expose :name
+  expose :size
+  expose :detailed_status, as: :status, with: StatusEntity
+  expose :jobs, with: BuildEntity
+
+  private
+
+  alias_method :group, :object
+
+  def detailed_status
+    group.detailed_status(request.user)
+  end
+end

app/serializers/stage_entity.rb

@@ -7,9 +7,11 @@ class StageEntity < Grape::Entity
     "#{stage.name}: #{detailed_status.label}"
   end
 
-  expose :detailed_status,
-    as: :status,
-    with: StatusEntity
+  expose :groups,
+    if: -> (_, opts) { opts[:grouped] },
+    with: JobGroupEntity
+
+  expose :detailed_status, as: :status, with: StatusEntity
 
   expose :path do |stage|
     namespace_project_pipeline_path(

app/serializers/status_entity.rb

@@ -12,4 +12,11 @@ class StatusEntity < Grape::Entity
 
     ActionController::Base.helpers.image_path(File.join(dir, "#{status.favicon}.ico"))
   end
+
+  expose :action, if: -> (status, _) { status.has_action? } do
+    expose :action_icon, as: :icon
+    expose :action_title, as: :title
+    expose :action_path, as: :path
+    expose :action_method, as: :method
+  end
 end

changelogs/unreleased/zj-real-time-pipelines.yml

+---
+title: Pipeline view updates in near real time
+merge_request: 10777
+author:

lib/gitlab/ci/status/group/common.rb

+module Gitlab
+  module Ci
+    module Status
+      module Group
+        module Common
+          def has_details?
+            false
+          end
+
+          def details_path
+            nil
+          end
+
+          def has_action?
+            false
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/status/group/factory.rb

+module Gitlab
+  module Ci
+    module Status
+      module Group
+        class Factory < Status::Factory
+          def self.common_helpers
+            Status::Group::Common
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/etag_caching/router.rb

@@ -36,7 +36,11 @@ module Gitlab
         Gitlab::EtagCaching::Router::Route.new(
           %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/pipelines\.json\z),
           'project_pipelines'
-        )
+        ),
+        Gitlab::EtagCaching::Router::Route.new(
+          %r(^(?!.*(#{RESERVED_WORDS})).*/pipelines/\d+\.json\z),
+          'project_pipeline'
+        ),
       ].freeze
 
       def self.match(env)

spec/controllers/projects/pipelines_controller_spec.rb

 require 'spec_helper'
 
 describe Projects::PipelinesController do
+  include ApiHelpers
+
   let(:user) { create(:user) }
   let(:project) { create(:empty_project, :public) }
 
@@ -24,6 +26,7 @@ describe Projects::PipelinesController do
 
     it 'returns JSON with serialized pipelines' do
       expect(response).to have_http_status(:ok)
+      expect(response).to match_response_schema('pipeline')
 
       expect(json_response).to include('pipelines')
       expect(json_response['pipelines'].count).to eq 4
@@ -34,6 +37,34 @@ describe Projects::PipelinesController do
     end
   end
 
+  describe 'GET show JSON' do
+    let!(:pipeline) { create(:ci_pipeline_with_one_job, project: project) }
+
+    it 'returns the pipeline' do
+      get_pipeline_json
+
+      expect(response).to have_http_status(:ok)
+      expect(json_response).not_to be_an(Array)
+      expect(json_response['id']).to be(pipeline.id)
+      expect(json_response['details']).to have_key 'stages'
+    end
+
+    context 'when the pipeline has multiple jobs' do
+      it 'does not perform N + 1 queries' do
+        control_count = ActiveRecord::QueryRecorder.new { get_pipeline_json }.count
+
+        create(:ci_build, pipeline: pipeline)
+
+        # The plus 2 is needed to group and sort
+        expect { get_pipeline_json }.not_to exceed_query_limit(control_count + 2)
+      end
+    end
+
+    def get_pipeline_json
+      get :show, namespace_id: project.namespace, project_id: project, id: pipeline, format: :json
+    end
+  end
+
   describe 'GET stages.json' do
     let(:pipeline) { create(:ci_pipeline, project: project) }
 

spec/fixtures/api/schemas/pipeline.json

+{
+    "$schema": "http://json-schema.org/draft-04/schema#",
+    "definitions": {},
+    "id": "http://example.com/example.json",
+    "properties": {
+        "commit": {
+            "id": "/properties/commit",
+            "properties": {
+                "author": {
+                    "id": "/properties/commit/properties/author",
+                    "type": "null"
+                },
+                "author_email": {
+                    "id": "/properties/commit/properties/author_email",
+                    "type": "string"
+                },
+                "author_gravatar_url": {
+                    "id": "/properties/commit/properties/author_gravatar_url",
+                    "type": "string"
+                },
+                "author_name": {
+                    "id": "/properties/commit/properties/author_name",
+                    "type": "string"
+                },
+                "authored_date": {
+                    "id": "/properties/commit/properties/authored_date",
+                    "type": "string"
+                },
+                "commit_path": {
+                    "id": "/properties/commit/properties/commit_path",
+                    "type": "string"
+                },
+                "commit_url": {
+                    "id": "/properties/commit/properties/commit_url",
+                    "type": "string"
+                },
+                "committed_date": {
+                    "id": "/properties/commit/properties/committed_date",
+                    "type": "string"
+                },
+                "committer_email": {
+                    "id": "/properties/commit/properties/committer_email",
+                    "type": "string"
+                },
+                "committer_name": {
+                    "id": "/properties/commit/properties/committer_name",
+                    "type": "string"
+                },
+                "created_at": {
+                    "id": "/properties/commit/properties/created_at",
+                    "type": "string"
+                },
+                "id": {
+                    "id": "/properties/commit/properties/id",
+                    "type": "string"
+                },
+                "message": {
+                    "id": "/properties/commit/properties/message",
+                    "type": "string"
+                },
+                "parent_ids": {
+                    "id": "/properties/commit/properties/parent_ids",
+                    "items": {
+                        "id": "/properties/commit/properties/parent_ids/items",
+                        "type": "string"
+                    },
+                    "type": "array"
+                },
+                "short_id": {
+                    "id": "/properties/commit/properties/short_id",
+                    "type": "string"
+                },
+                "title": {
+                    "id": "/properties/commit/properties/title",
+                    "type": "string"
+                }
+            },
+            "type": "object"
+        },
+        "created_at": {
+            "id": "/properties/created_at",
+            "type": "string"
+        },
+        "details": {
+            "id": "/properties/details",
+            "properties": {
+                "artifacts": {
+                    "id": "/properties/details/properties/artifacts",
+                    "items": {},
+                    "type": "array"
+                },
+                "duration": {
+                    "id": "/properties/details/properties/duration",
+                    "type": "integer"
+                },
+                "finished_at": {
+                    "id": "/properties/details/properties/finished_at",
+                    "type": "string"
+                },
+                "manual_actions": {
+                    "id": "/properties/details/properties/manual_actions",
+                    "items": {},
+                    "type": "array"
+                },
+                "stages": {
+                    "id": "/properties/details/properties/stages",
+                    "items": {
+                        "id": "/properties/details/properties/stages/items",
+                        "properties": {
+                            "dropdown_path": {
+                                "id": "/properties/details/properties/stages/items/properties/dropdown_path",
+                                "type": "string"
+                            },
+                            "groups": {
+                                "id": "/properties/details/properties/stages/items/properties/groups",
+                                "items": {
+                                    "id": "/properties/details/properties/stages/items/properties/groups/items",
+                                    "properties": {
+                                        "name": {
+                                            "id": "/properties/details/properties/stages/items/properties/groups/items/properties/name",
+                                            "type": "string"
+                                        },
+                                        "size": {
+                                            "id": "/properties/details/properties/stages/items/properties/groups/items/properties/size",
+                                            "type": "integer"
+                                        },
+                                        "status": {
+                                            "id": "/properties/details/properties/stages/items/properties/groups/items/properties/status",
+                                            "properties": {
+                                                "details_path": {
+                                                    "id": "/properties/details/properties/stages/items/properties/groups/items/properties/status/properties/details_path",
+                                                    "type": "null"
+                                                },
+                                                "favicon": {
+                                                    "id": "/properties/details/properties/stages/items/properties/groups/items/properties/status/properties/favicon",
+                                                    "type": "string"
+                                                },
+                                                "group": {
+                                                    "id": "/properties/details/properties/stages/items/properties/groups/items/properties/status/properties/group",
+                                                    "type": "string"
+                                                },
+                                                "has_details": {
+                                                    "id": "/properties/details/properties/stages/items/properties/groups/items/properties/status/properties/has_details",
+                                                    "type": "boolean"
+                                                },
+                                                "icon": {
+                                                    "id": "/properties/details/properties/stages/items/properties/groups/items/properties/status/properties/icon",
+                                                    "type": "string"
+                                                },
+                                                "label": {
+                                                    "id": "/properties/details/properties/stages/items/properties/groups/items/properties/status/properties/label",
+                                                    "type": "string"
+                                                },
+                                                "text": {
+                                                    "id": "/properties/details/properties/stages/items/properties/groups/items/properties/status/properties/text",
+                                                    "type": "string"
+                                                }
+                                            },
+                                            "type": "object"
+                                        }
+                                    },
+                                    "type": "object"
+                                },
+                                "type": "array"
+                            },
+                            "name": {
+                                "id": "/properties/details/properties/stages/items/properties/name",
+                                "type": "string"
+                            },
+                            "path": {
+                                "id": "/properties/details/properties/stages/items/properties/path",
+                                "type": "string"
+                            },
+                            "status": {
+                                "id": "/properties/details/properties/stages/items/properties/status",
+                                "properties": {
+                                    "details_path": {
+                                        "id": "/properties/details/properties/stages/items/properties/status/properties/details_path",
+                                        "type": "string"
+                                    },
+                                    "favicon": {
+                                        "id": "/properties/details/properties/stages/items/properties/status/properties/favicon",
+                                        "type": "string"
+                                    },
+                                    "group": {
+                                        "id": "/properties/details/properties/stages/items/properties/status/properties/group",
+                                        "type": "string"
+                                    },
+                                    "has_details": {
+                                        "id": "/properties/details/properties/stages/items/properties/status/properties/has_details",
+                                        "type": "boolean"
+                                    },
+                                    "icon": {
+                                        "id": "/properties/details/properties/stages/items/properties/status/properties/icon",
+                                        "type": "string"
+                                    },
+                                    "label": {
+                                        "id": "/properties/details/properties/stages/items/properties/status/properties/label",
+                                        "type": "string"
+                                    },
+                                    "text": {
+                                        "id": "/properties/details/properties/stages/items/properties/status/properties/text",
+                                        "type": "string"
+                                    }
+                                },
+                                "type": "object"
+                            },
+                            "title": {
+                                "id": "/properties/details/properties/stages/items/properties/title",
+                                "type": "string"
+                            }
+                        },
+                        "type": "object"
+                    },
+                    "type": "array"
+                },
+                "status": {
+                    "id": "/properties/details/properties/status",
+                    "properties": {
+                        "details_path": {
+                            "id": "/properties/details/properties/status/properties/details_path",
+                            "type": "string"
+                        },
+                        "favicon": {
+                            "id": "/properties/details/properties/status/properties/favicon",
+                            "type": "string"
+                        },
+                        "group": {
+                            "id": "/properties/details/properties/status/properties/group",
+                            "type": "string"
+                        },
+                        "has_details": {
+                            "id": "/properties/details/properties/status/properties/has_details",
+                            "type": "boolean"
+                        },
+                        "icon": {
+                            "id": "/properties/details/properties/status/properties/icon",
+                            "type": "string"
+                        },
+                        "label": {
+                            "id": "/properties/details/properties/status/properties/label",
+                            "type": "string"
+                        },
+                        "text": {
+                            "id": "/properties/details/properties/status/properties/text",
+                            "type": "string"
+                        }
+                    },
+                    "type": "object"
+                }
+            },
+            "type": "object"
+        },
+        "flags": {
+            "id": "/properties/flags",
+            "properties": {
+                "cancelable": {
+                    "id": "/properties/flags/properties/cancelable",
+                    "type": "boolean"
+                },
+                "latest": {
+                    "id": "/properties/flags/properties/latest",
+                    "type": "boolean"
+                },
+                "retryable": {
+                    "id": "/properties/flags/properties/retryable",
+                    "type": "boolean"
+                },
+                "stuck": {
+                    "id": "/properties/flags/properties/stuck",
+                    "type": "boolean"
+                },
+                "triggered": {
+                    "id": "/properties/flags/properties/triggered",
+                    "type": "boolean"
+                },
+                "yaml_errors": {
+                    "id": "/properties/flags/properties/yaml_errors",
+                    "type": "boolean"
+                }
+            },
+            "type": "object"
+        },
+        "id": {
+            "id": "/properties/id",
+            "type": "integer"
+        },
+        "path": {
+            "id": "/properties/path",
+            "type": "string"
+        },
+        "ref": {
+            "id": "/properties/ref",
+            "properties": {
+                "branch": {
+                    "id": "/properties/ref/properties/branch",
+                    "type": "boolean"
+                },
+                "name": {
+                    "id": "/properties/ref/properties/name",
+                    "type": "string"
+                },
+                "path": {
+                    "id": "/properties/ref/properties/path",
+                    "type": "string"
+                },
+                "tag": {
+                    "id": "/properties/ref/properties/tag",
+                    "type": "boolean"
+                }
+            },
+            "type": "object"
+        },
+        "retry_path": {
+            "id": "/properties/retry_path",
+            "type": "string"
+        },
+        "updated_at": {
+            "id": "/properties/updated_at",
+            "type": "string"
+        },
+        "user": {
+            "id": "/properties/user",
+            "properties": {
+                "avatar_url": {
+                    "id": "/properties/user/properties/avatar_url",
+                    "type": "string"
+                },
+                "id": {
+                    "id": "/properties/user/properties/id",
+                    "type": "integer"
+                },
+                "name": {
+                    "id": "/properties/user/properties/name",
+                    "type": "string"
+                },
+                "state": {
+                    "id": "/properties/user/properties/state",
+                    "type": "string"
+                },
+                "username": {
+                    "id": "/properties/user/properties/username",
+                    "type": "string"
+                },
+                "web_url": {
+                    "id": "/properties/user/properties/web_url",
+                    "type": "string"
+                }
+            },
+            "type": "object"
+        }
+    },
+    "type": "object"
+}

spec/lib/gitlab/ci/status/group/common_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Ci::Status::Group::Common do
+  subject do
+    Gitlab::Ci::Status::Core.new(double, double)
+      .extend(described_class)
+  end
+
+  it 'does not have action' do
+    expect(subject).not_to have_action
+  end
+
+  it 'has details' do
+    expect(subject).not_to have_details
+  end
+
+  it 'has no details_path' do
+    expect(subject.details_path).to be_falsy
+  end
+end

spec/lib/gitlab/ci/status/group/factory_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Ci::Status::Group::Factory do
+  it 'inherits from the core factory' do
+    expect(described_class)
+      .to be < Gitlab::Ci::Status::Factory
+  end
+
+  it 'exposes group helpers' do
+    expect(described_class.common_helpers)
+      .to eq Gitlab::Ci::Status::Group::Common
+  end
+end

spec/models/ci/group_spec.rb

+require 'spec_helper'
+
+describe Ci::Group, models: true do
+  subject do
+    described_class.new('test', name: 'rspec', jobs: jobs)
+  end
+
+  let!(:jobs) { build_list(:ci_build, 1, :success) }
+
+  it { is_expected.to include_module(StaticModel) }
+
+  it { is_expected.to respond_to(:stage) }
+  it { is_expected.to respond_to(:name) }
+  it { is_expected.to respond_to(:jobs) }
+  it { is_expected.to respond_to(:status) }
+
+  describe '#size' do
+    it 'returns the number of statuses in the group' do
+      expect(subject.size).to eq(1)
+    end
+  end
+
+  describe '#detailed_status' do
+    context 'when there is only one item in the group' do
+      it 'calls the status from the object itself' do
+        expect(jobs.first).to receive(:detailed_status)
+
+        expect(subject.detailed_status(double(:user)))
+      end
+    end
+
+    context 'when there are more than one commit status in the group' do
+      let(:jobs) do
+        [create(:ci_build, :failed),
+         create(:ci_build, :success)]
+      end
+
+      it 'fabricates a new detailed status object' do
+        expect(subject.detailed_status(double(:user)))
+          .to be_a(Gitlab::Ci::Status::Failed)
+      end
+    end
+  end
+end

spec/models/ci/stage_spec.rb

@@ -28,6 +28,35 @@ describe Ci::Stage, models: true do
     end
   end
 
+  describe '#groups' do
+    before do
+      create_job(:ci_build, name: 'rspec 0 2')
+      create_job(:ci_build, name: 'rspec 0 1')
+      create_job(:ci_build, name: 'spinach 0 1')
+      create_job(:commit_status, name: 'aaaaa')
+    end
+
+    it 'returns an array of three groups' do
+      expect(stage.groups).to be_a Array
+      expect(stage.groups).to all(be_a Ci::Group)
+      expect(stage.groups.size).to eq 3
+    end
+
+    it 'returns groups with correctly ordered statuses' do
+      expect(stage.groups.first.jobs.map(&:name))
+        .to eq ['aaaaa']
+      expect(stage.groups.second.jobs.map(&:name))
+        .to eq ['rspec 0 1', 'rspec 0 2']
+      expect(stage.groups.third.jobs.map(&:name))
+        .to eq ['spinach 0 1']
+    end
+
+    it 'returns groups with correct names' do
+      expect(stage.groups.map(&:name))
+        .to eq %w[aaaaa rspec spinach]
+    end
+  end
+
   describe '#statuses_count' do
     before do
       create_job(:ci_build)
@@ -223,7 +252,7 @@ describe Ci::Stage, models: true do
     end
   end
 
-  def create_job(type, status: 'success', stage: stage_name)
-    create(type, pipeline: pipeline, stage: stage, status: status)
+  def create_job(type, status: 'success', stage: stage_name, **opts)
+    create(type, pipeline: pipeline, stage: stage, status: status, **opts)
   end
 end

spec/serializers/stage_entity_spec.rb

@@ -47,5 +47,13 @@ describe StageEntity do
     it 'contains stage title' do
       expect(subject[:title]).to eq 'test: passed'
     end
+
+    context 'when the jobs should be grouped' do
+      let(:entity) { described_class.new(stage, request: request, grouped: true) }
+
+      it 'exposes the group key' do
+        expect(subject).to include :groups
+      end
+    end
   end
 end