pipeline.rb 11.5 KB
Newer Older
1
module Ci
2
  class Pipeline < ActiveRecord::Base
3
    extend Ci::Model
4
    include HasStatus
5
    include Importable
6
    include AfterCommitQueue
7
    include Presentable
Kamil Trzcinski's avatar
WIP  
Kamil Trzcinski committed
8

9
    belongs_to :project
10
    belongs_to :user
11
    belongs_to :auto_canceled_by, class_name: 'Ci::Pipeline'
12
    belongs_to :pipeline_schedule, class_name: 'Ci::PipelineSchedule'
13

14
    has_many :stages
15
    has_many :statuses, class_name: 'CommitStatus', foreign_key: :commit_id
16 17
    has_many :builds, foreign_key: :commit_id
    has_many :trigger_requests, dependent: :destroy, foreign_key: :commit_id
Felipe Artur's avatar
Felipe Artur committed
18 19 20

    # Merge requests for which the current pipeline is running against
    # the merge request's latest commit.
21
    has_many :merge_requests, foreign_key: "head_pipeline_id"
22

23 24 25
    has_many :pending_builds, -> { pending }, foreign_key: :commit_id, class_name: 'Ci::Build'
    has_many :retryable_builds, -> { latest.failed_or_canceled }, foreign_key: :commit_id, class_name: 'Ci::Build'
    has_many :cancelable_statuses, -> { cancelable }, foreign_key: :commit_id, class_name: 'CommitStatus'
26 27
    has_many :manual_actions, -> { latest.manual_actions.includes(:project) }, foreign_key: :commit_id, class_name: 'Ci::Build'
    has_many :artifacts, -> { latest.with_artifacts_not_expired.includes(:project) }, foreign_key: :commit_id, class_name: 'Ci::Build'
28

29 30 31
    has_many :auto_canceled_pipelines, class_name: 'Ci::Pipeline', foreign_key: 'auto_canceled_by_id'
    has_many :auto_canceled_jobs, class_name: 'CommitStatus', foreign_key: 'auto_canceled_by_id'

Douwe Maan's avatar
Douwe Maan committed
32 33
    delegate :id, to: :project, prefix: true

34
    validates :source, exclusion: { in: %w(unknown), unless: :importing? }, on: :create
Douwe Maan's avatar
Douwe Maan committed
35 36 37
    validates :sha, presence: { unless: :importing? }
    validates :ref, presence: { unless: :importing? }
    validates :status, presence: { unless: :importing? }
38
    validate :valid_commit_sha, unless: :importing?
39

40
    after_create :keep_around_commits, unless: :importing?
Kamil Trzcinski's avatar
Kamil Trzcinski committed
41

42 43 44 45 46 47 48 49 50
    enum source: {
      unknown: nil,
      push: 1,
      web: 2,
      trigger: 3,
      schedule: 4,
      api: 5,
      external: 6
    }
51

52
    state_machine :status, initial: :created do
53
      event :enqueue do
Kamil Trzcinski's avatar
Kamil Trzcinski committed
54
        transition created: :pending
55
        transition [:success, :failed, :canceled, :skipped] => :running
56 57 58
      end

      event :run do
Kamil Trzcinski's avatar
Kamil Trzcinski committed
59
        transition any - [:running] => :running
60 61
      end

62
      event :skip do
Kamil Trzcinski's avatar
Kamil Trzcinski committed
63
        transition any - [:skipped] => :skipped
64 65 66
      end

      event :drop do
Kamil Trzcinski's avatar
Kamil Trzcinski committed
67
        transition any - [:failed] => :failed
68 69
      end

70
      event :succeed do
Kamil Trzcinski's avatar
Kamil Trzcinski committed
71
        transition any - [:success] => :success
72 73 74
      end

      event :cancel do
Kamil Trzcinski's avatar
Kamil Trzcinski committed
75
        transition any - [:canceled] => :canceled
76 77
      end

78
      event :block do
79
        transition any - [:manual] => :manual
80 81
      end

82 83 84 85
      # IMPORTANT
      # Do not add any operations to this state_machine
      # Create a separate worker for each new operation

86
      before_transition [:created, :pending] => :running do |pipeline|
87
        pipeline.started_at = Time.now
88 89
      end

90
      before_transition any => [:success, :failed, :canceled] do |pipeline|
91
        pipeline.finished_at = Time.now
92 93 94
        pipeline.update_duration
      end

95 96 97 98
      before_transition any => [:manual] do |pipeline|
        pipeline.update_duration
      end

Lin Jen-Shin's avatar
Lin Jen-Shin committed
99
      before_transition canceled: any - [:canceled] do |pipeline|
Lin Jen-Shin's avatar
Lin Jen-Shin committed
100
        pipeline.auto_canceled_by = nil
101 102
      end

103
      after_transition [:created, :pending] => :running do |pipeline|
104
        pipeline.run_after_commit { PipelineMetricsWorker.perform_async(pipeline.id) }
105 106 107
      end

      after_transition any => [:success] do |pipeline|
108
        pipeline.run_after_commit { PipelineMetricsWorker.perform_async(pipeline.id) }
109 110
      end

111
      after_transition [:created, :pending, :running] => :success do |pipeline|
112
        pipeline.run_after_commit { PipelineSuccessWorker.perform_async(pipeline.id) }
113
      end
114 115

      after_transition do |pipeline, transition|
116 117 118
        next if transition.loopback?

        pipeline.run_after_commit do
119
          PipelineHooksWorker.perform_async(pipeline.id)
120
          ExpirePipelineCacheWorker.perform_async(pipeline.id)
121
        end
122
      end
123

124
      after_transition any => [:success, :failed] do |pipeline|
125
        pipeline.run_after_commit do
126
          PipelineNotificationWorker.perform_async(pipeline.id)
127
        end
128
      end
129 130
    end

131
    # ref can't be HEAD or SHA, can only be branch/tag name
132
    scope :latest, ->(ref = nil) do
133 134 135
      max_id = unscope(:select)
        .select("max(#{quoted_table_name}.id)")
        .group(:ref, :sha)
136

137 138 139 140 141
      if ref
        where(ref: ref, id: max_id.where(ref: ref))
      else
        where(id: max_id)
      end
142
    end
143
    scope :internal, -> { where(source: internal_sources) }
144

145 146 147 148
    def self.latest_status(ref = nil)
      latest(ref).status
    end

149
    def self.latest_successful_for(ref)
150
      success.latest(ref).order(id: :desc).first
151 152
    end

153 154 155 156
    def self.latest_successful_for_refs(refs)
      success.latest(refs).order(id: :desc).each_with_object({}) do |pipeline, hash|
        hash[pipeline.ref] ||= pipeline
      end
157 158 159 160
    end

    def self.truncate_sha(sha)
      sha[0...8]
161 162
    end

163
    def self.total_duration
Lin Jen-Shin's avatar
Lin Jen-Shin committed
164
      where.not(duration: nil).sum(:duration)
165 166
    end

167 168 169 170
    def self.internal_sources
      sources.reject { |source| source == "external" }.values
    end

171 172
    def stages_count
      statuses.select(:stage).distinct.count
Kamil Trzcinski's avatar
Kamil Trzcinski committed
173 174
    end

175
    def stages_names
176 177
      statuses.order(:stage_idx).distinct
        .pluck(:stage, :stage_idx).map(&:first)
178 179
    end

180
    def legacy_stage(name)
181
      stage = Ci::LegacyStage.new(self, name: name)
182 183 184 185
      stage unless stage.statuses_count.zero?
    end

    def legacy_stages
186 187
      # TODO, this needs refactoring, see gitlab-ce#26481.

188 189
      stages_query = statuses
        .group('stage').select(:stage).order('max(stage_idx)')
190

Kamil Trzcinski's avatar
Kamil Trzcinski committed
191 192
      status_sql = statuses.latest.where('stage=sg.stage').status_sql

193
      warnings_sql = statuses.latest.select('COUNT(*)')
194
        .where('stage=sg.stage').failed_but_allowed.to_sql
195

196 197
      stages_with_statuses = CommitStatus.from(stages_query, :sg)
        .pluck('sg.stage', status_sql, "(#{warnings_sql})")
Kamil Trzcinski's avatar
Kamil Trzcinski committed
198 199

      stages_with_statuses.map do |stage|
200
        Ci::LegacyStage.new(self, Hash[%i[name status warnings].zip(stage)])
Kamil Trzcinski's avatar
Kamil Trzcinski committed
201
      end
Kamil Trzcinski's avatar
WIP  
Kamil Trzcinski committed
202 203
    end

204
    def valid_commit_sha
205
      if self.sha == Gitlab::Git::BLANK_SHA
206 207 208 209 210
        self.errors.add(:sha, " cant be 00000000 (branch removal)")
      end
    end

    def git_author_name
211
      commit.try(:author_name)
212 213 214
    end

    def git_author_email
215
      commit.try(:author_email)
216 217 218
    end

    def git_commit_message
219
      commit.try(:message)
220 221
    end

222 223 224 225
    def git_commit_title
      commit.try(:title)
    end

226
    def short_sha
227
      Ci::Pipeline.truncate_sha(sha)
228 229
    end

230
    def commit
231
      @commit ||= project.commit(sha)
232 233 234 235
    rescue
      nil
    end

236 237 238 239
    def branch?
      !tag?
    end

240
    def stuck?
241
      pending_builds.any?(&:stuck?)
242 243
    end

244
    def retryable?
245
      retryable_builds.any?
246 247
    end

248
    def cancelable?
249
      cancelable_statuses.any?
250 251
    end

Lin Jen-Shin's avatar
Lin Jen-Shin committed
252 253
    def auto_canceled?
      canceled? && auto_canceled_by_id?
254 255
    end

Kamil Trzcinski's avatar
Kamil Trzcinski committed
256
    def cancel_running
257
      Gitlab::OptimisticLocking.retry_lock(cancelable_statuses) do |cancelable|
Lin Jen-Shin's avatar
Lin Jen-Shin committed
258 259 260
        cancelable.find_each do |job|
          yield(job) if block_given?
          job.cancel
261
        end
Lin Jen-Shin's avatar
Lin Jen-Shin committed
262
      end
Kamil Trzcinski's avatar
Kamil Trzcinski committed
263 264
    end

265 266 267 268 269
    def auto_cancel_running(pipeline)
      update(auto_canceled_by: pipeline)

      cancel_running do |job|
        job.auto_canceled_by = pipeline
270
      end
Kamil Trzcinski's avatar
Kamil Trzcinski committed
271 272
    end

273
    def retry_failed(current_user)
274 275
      Ci::RetryPipelineService.new(project, current_user)
        .execute(self)
Kamil Trzcinski's avatar
Kamil Trzcinski committed
276 277
    end

278
    def mark_as_processable_after_stage(stage_idx)
279
      builds.skipped.after_stage(stage_idx).find_each(&:process)
280 281
    end

Kamil Trzcinski's avatar
Kamil Trzcinski committed
282 283 284 285 286 287 288
    def latest?
      return false unless ref
      commit = project.commit(ref)
      return false unless commit
      commit.sha == sha
    end

289 290
    def retried
      @retried ||= (statuses.order(id: :desc) - statuses.latest)
291 292 293
    end

    def coverage
294
      coverage_array = statuses.latest.map(&:coverage).compact
295 296
      if coverage_array.size >= 1
        '%.2f' % (coverage_array.reduce(:+) / coverage_array.size)
297 298 299
      end
    end

300
    def stage_seeds
301 302
      return [] unless config_processor

303
      @stage_seeds ||= config_processor.stage_seeds(self)
304 305
    end

306 307
    def has_stage_seeds?
      stage_seeds.any?
308 309
    end

Connor Shea's avatar
Connor Shea committed
310
    def has_warnings?
311
      builds.latest.failed_but_allowed.any?
312 313
    end

314
    def config_processor
315
      return unless ci_yaml_file
316 317 318 319 320
      return @config_processor if defined?(@config_processor)

      @config_processor ||= begin
        Ci::GitlabCiYamlProcessor.new(ci_yaml_file, project.path_with_namespace)
      rescue Ci::GitlabCiYamlProcessor::ValidationError, Psych::SyntaxError => e
321
        self.yaml_errors = e.message
322 323
        nil
      rescue
324
        self.yaml_errors = 'Undefined error'
325 326
        nil
      end
327 328
    end

329
    def ci_yaml_file
330 331
      return @ci_yaml_file if defined?(@ci_yaml_file)

332 333
      @ci_yaml_file = begin
        project.repository.gitlab_ci_yml_for(sha, ci_yaml_file_path)
334
      rescue
335 336
        self.yaml_errors =
          "Failed to load CI/CD config file at #{ci_yaml_file_path}"
337
        nil
338
      end
339 340
    end

341 342 343 344
    def has_yaml_errors?
      yaml_errors.present?
    end

345
    def ci_yaml_file_path
346 347 348 349 350
      if project.ci_config_file.blank?
        '.gitlab-ci.yml'
      else
        project.ci_config_file
      end
351 352
    end

Kamil Trzcinski's avatar
Kamil Trzcinski committed
353 354 355 356
    def environments
      builds.where.not(environment: nil).success.pluck(:environment).uniq
    end

James Lopez's avatar
James Lopez committed
357 358 359 360 361 362 363 364 365 366 367 368 369
    # Manually set the notes for a Ci::Pipeline
    # There is no ActiveRecord relation between Ci::Pipeline and notes
    # as they are related to a commit sha. This method helps importing
    # them using the +Gitlab::ImportExport::RelationFactory+ class.
    def notes=(notes)
      notes.each do |note|
        note[:id] = nil
        note[:commit_id] = sha
        note[:noteable_id] = self['id']
        note.save!
      end
    end

370 371 372 373
    def notes
      Note.for_commit_id(sha)
    end

374 375 376
    def process!
      Ci::ProcessPipelineService.new(project, user).execute(self)
    end
377

378
    def update_status
379
      Gitlab::OptimisticLocking.retry_lock(self) do
Kamil Trzcinski's avatar
Kamil Trzcinski committed
380
        case latest_builds_status
381 382 383 384 385 386
        when 'pending' then enqueue
        when 'running' then run
        when 'success' then succeed
        when 'failed' then drop
        when 'canceled' then cancel
        when 'skipped' then skip
387
        when 'manual' then block
388
        end
389
      end
390 391
    end

392 393 394 395 396 397
    def predefined_variables
      [
        { key: 'CI_PIPELINE_ID', value: id.to_s, public: true }
      ]
    end

398 399 400 401 402 403 404
    def queued_duration
      return unless started_at

      seconds = (started_at - created_at).to_i
      seconds unless seconds.zero?
    end

405
    def update_duration
406 407
      return unless started_at

408
      self.duration = Gitlab::Ci::PipelineDuration.from_pipeline(self)
409 410 411
    end

    def execute_hooks
412 413 414
      data = pipeline_data
      project.execute_hooks(data, :pipeline_hooks)
      project.execute_services(data, :pipeline_hooks)
415 416
    end

417 418
    # All the merge requests for which the current pipeline runs/ran against
    def all_merge_requests
419
      @all_merge_requests ||= project.merge_requests.where(source_branch: ref)
420 421
    end

422
    def detailed_status(current_user)
423 424 425
      Gitlab::Ci::Status::Pipeline::Factory
        .new(self, current_user)
        .fabricate!
426 427
    end

428 429
    private

430
    def pipeline_data
431
      Gitlab::DataBuilder::Pipeline.build(self)
Kamil Trzcinski's avatar
Kamil Trzcinski committed
432
    end
433

434
    def latest_builds_status
435 436 437
      return 'failed' unless yaml_errors.blank?

      statuses.latest.status || 'skipped'
Kamil Trzcinski's avatar
Kamil Trzcinski committed
438
    end
439 440

    def keep_around_commits
441
      return unless project
442

443 444 445
      project.repository.keep_around(self.sha)
      project.repository.keep_around(self.before_sha)
    end
446 447
  end
end