Merge branch 'feature/gb/migrate-pipeline-stages' into 'master'

Migrate pipeline stages in the database

See merge request !11714

db/migrate/20170525132202_create_pipeline_stages.rb

+class CreatePipelineStages < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    create_table :ci_stages do |t|
+      t.integer :project_id
+      t.integer :pipeline_id
+      t.timestamps null: true
+      t.string :name
+    end
+
+    add_concurrent_foreign_key :ci_stages, :projects, column: :project_id, on_delete: :cascade
+    add_concurrent_foreign_key :ci_stages, :ci_pipelines, column: :pipeline_id, on_delete: :cascade
+    add_concurrent_index :ci_stages, :project_id
+    add_concurrent_index :ci_stages, :pipeline_id
+  end
+
+  def down
+    drop_table :ci_stages
+  end
+end

db/migrate/20170526185602_add_stage_id_to_ci_builds.rb

+class AddStageIdToCiBuilds < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_column :ci_builds, :stage_id, :integer
+
+    add_concurrent_foreign_key :ci_builds, :ci_stages, column: :stage_id, on_delete: :cascade
+    add_concurrent_index :ci_builds, :stage_id
+  end
+
+  def down
+    remove_foreign_key :ci_builds, column: :stage_id
+    remove_concurrent_index :ci_builds, :stage_id
+
+    remove_column :ci_builds, :stage_id, :integer
+  end
+end

db/post_migrate/20170526185842_migrate_pipeline_stages.rb

+class MigratePipelineStages < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    disable_statement_timeout
+
+    execute <<-SQL.strip_heredoc
+      INSERT INTO ci_stages (project_id, pipeline_id, name)
+        SELECT project_id, commit_id, stage FROM ci_builds
+          WHERE stage IS NOT NULL
+          AND stage_id IS NULL
+          AND EXISTS (SELECT 1 FROM projects WHERE projects.id = ci_builds.project_id)
+          AND EXISTS (SELECT 1 FROM ci_pipelines WHERE ci_pipelines.id = ci_builds.commit_id)
+          GROUP BY project_id, commit_id, stage
+          ORDER BY MAX(stage_idx)
+    SQL
+  end
+end

db/post_migrate/20170526185858_create_index_in_pipeline_stages.rb

+class CreateIndexInPipelineStages < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index(:ci_stages, [:pipeline_id, :name])
+  end
+
+  def down
+    remove_concurrent_index(:ci_stages, [:pipeline_id, :name])
+  end
+end

db/post_migrate/20170526185921_migrate_build_stage_reference.rb

+class MigrateBuildStageReference < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def up
+    disable_statement_timeout
+
+    stage_id = Arel.sql <<-SQL.strip_heredoc
+      (SELECT id FROM ci_stages
+         WHERE ci_stages.pipeline_id = ci_builds.commit_id
+           AND ci_stages.name = ci_builds.stage)
+    SQL
+
+    update_column_in_batches(:ci_builds, :stage_id, stage_id) do |table, query|
+      query.where(table[:stage_id].eq(nil))
+    end
+  end
+
+  def down
+    disable_statement_timeout
+
+    update_column_in_batches(:ci_builds, :stage_id, nil)
+  end
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20170525174156) do
+ActiveRecord::Schema.define(version: 20170526185921) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -234,6 +234,7 @@ ActiveRecord::Schema.define(version: 20170525174156) do
     t.string "coverage_regex"
     t.integer "auto_canceled_by_id"
     t.boolean "retried"
+    t.integer "stage_id"
   end
 
   add_index "ci_builds", ["auto_canceled_by_id"], name: "index_ci_builds_on_auto_canceled_by_id", using: :btree
@@ -243,6 +244,7 @@ ActiveRecord::Schema.define(version: 20170525174156) do
   add_index "ci_builds", ["commit_id", "type", "ref"], name: "index_ci_builds_on_commit_id_and_type_and_ref", using: :btree
   add_index "ci_builds", ["project_id"], name: "index_ci_builds_on_project_id", using: :btree
   add_index "ci_builds", ["runner_id"], name: "index_ci_builds_on_runner_id", using: :btree
+  add_index "ci_builds", ["stage_id"], name: "index_ci_builds_on_stage_id", using: :btree
   add_index "ci_builds", ["status", "type", "runner_id"], name: "index_ci_builds_on_status_and_type_and_runner_id", using: :btree
   add_index "ci_builds", ["status"], name: "index_ci_builds_on_status", using: :btree
   add_index "ci_builds", ["token"], name: "index_ci_builds_on_token", unique: true, using: :btree
@@ -327,6 +329,18 @@ ActiveRecord::Schema.define(version: 20170525174156) do
   add_index "ci_runners", ["locked"], name: "index_ci_runners_on_locked", using: :btree
   add_index "ci_runners", ["token"], name: "index_ci_runners_on_token", using: :btree
 
+  create_table "ci_stages", force: :cascade do |t|
+    t.integer "project_id"
+    t.integer "pipeline_id"
+    t.datetime "created_at"
+    t.datetime "updated_at"
+    t.string "name"
+  end
+
+  add_index "ci_stages", ["pipeline_id", "name"], name: "index_ci_stages_on_pipeline_id_and_name", using: :btree
+  add_index "ci_stages", ["pipeline_id"], name: "index_ci_stages_on_pipeline_id", using: :btree
+  add_index "ci_stages", ["project_id"], name: "index_ci_stages_on_project_id", using: :btree
+
   create_table "ci_trigger_requests", force: :cascade do |t|
     t.integer "trigger_id", null: false
     t.text "variables"
@@ -1484,10 +1498,13 @@ ActiveRecord::Schema.define(version: 20170525174156) do
   add_foreign_key "boards", "projects"
   add_foreign_key "chat_teams", "namespaces", on_delete: :cascade
   add_foreign_key "ci_builds", "ci_pipelines", column: "auto_canceled_by_id", name: "fk_a2141b1522", on_delete: :nullify
+  add_foreign_key "ci_builds", "ci_stages", column: "stage_id", name: "fk_3a9eaa254d", on_delete: :cascade
   add_foreign_key "ci_pipeline_schedules", "projects", name: "fk_8ead60fcc4", on_delete: :cascade
   add_foreign_key "ci_pipeline_schedules", "users", column: "owner_id", name: "fk_9ea99f58d2", on_delete: :nullify
   add_foreign_key "ci_pipelines", "ci_pipeline_schedules", column: "pipeline_schedule_id", name: "fk_3d34ab2e06", on_delete: :nullify
   add_foreign_key "ci_pipelines", "ci_pipelines", column: "auto_canceled_by_id", name: "fk_262d4c2d19", on_delete: :nullify
+  add_foreign_key "ci_stages", "ci_pipelines", column: "pipeline_id", name: "fk_fb57e6cc56", on_delete: :cascade
+  add_foreign_key "ci_stages", "projects", name: "fk_2360681d1d", on_delete: :cascade
   add_foreign_key "ci_trigger_requests", "ci_triggers", column: "trigger_id", name: "fk_b8ec8b7245", on_delete: :cascade
   add_foreign_key "ci_triggers", "users", column: "owner_id", name: "fk_e8e10d1964", on_delete: :cascade
   add_foreign_key "ci_variables", "projects", name: "fk_ada5eb64b3", on_delete: :cascade

spec/lib/gitlab/import_export/safe_model_attributes.yml

@@ -214,6 +214,7 @@ CommitStatus:
 - stage
 - trigger_request_id
 - stage_idx
+- stage_id
 - tag
 - ref
 - user_id

spec/migrations/migrate_build_stage_reference_spec.rb

+require 'spec_helper'
+require Rails.root.join('db', 'post_migrate', '20170526185921_migrate_build_stage_reference.rb')
+
+describe MigrateBuildStageReference, :migration do
+  ##
+  # Create test data - pipeline and CI/CD jobs.
+  #
+
+  let(:jobs) { table(:ci_builds) }
+  let(:stages) { table(:ci_stages) }
+  let(:pipelines) { table(:ci_pipelines) }
+  let(:projects) { table(:projects) }
+
+  before do
+    # Create projects
+    #
+    projects.create!(id: 123, name: 'gitlab1', path: 'gitlab1')
+    projects.create!(id: 456, name: 'gitlab2', path: 'gitlab2')
+
+    # Create CI/CD pipelines
+    #
+    pipelines.create!(id: 1, project_id: 123, ref: 'master', sha: 'adf43c3a')
+    pipelines.create!(id: 2, project_id: 456, ref: 'feature', sha: '21a3deb')
+
+    # Create CI/CD jobs
+    #
+    jobs.create!(id: 1, commit_id: 1, project_id: 123, stage_idx: 2, stage: 'build')
+    jobs.create!(id: 2, commit_id: 1, project_id: 123, stage_idx: 2, stage: 'build')
+    jobs.create!(id: 3, commit_id: 1, project_id: 123, stage_idx: 1, stage: 'test')
+    jobs.create!(id: 4, commit_id: 1, project_id: 123, stage_idx: 3, stage: 'deploy')
+    jobs.create!(id: 5, commit_id: 2, project_id: 456, stage_idx: 2, stage: 'test:2')
+    jobs.create!(id: 6, commit_id: 2, project_id: 456, stage_idx: 1, stage: 'test:1')
+    jobs.create!(id: 7, commit_id: 2, project_id: 456, stage_idx: 1, stage: 'test:1')
+    jobs.create!(id: 8, commit_id: 3, project_id: 789, stage_idx: 3, stage: 'deploy')
+
+    # Create CI/CD stages
+    #
+    stages.create(id: 101, pipeline_id: 1, project_id: 123, name: 'test')
+    stages.create(id: 102, pipeline_id: 1, project_id: 123, name: 'build')
+    stages.create(id: 103, pipeline_id: 1, project_id: 123, name: 'deploy')
+    stages.create(id: 104, pipeline_id: 2, project_id: 456, name: 'test:1')
+    stages.create(id: 105, pipeline_id: 2, project_id: 456, name: 'test:2')
+    stages.create(id: 106, pipeline_id: 2, project_id: 456, name: 'deploy')
+  end
+
+  it 'correctly migrate build stage references' do
+    expect(jobs.where(stage_id: nil).count).to eq 8
+
+    migrate!
+
+    expect(jobs.where(stage_id: nil).count).to eq 1
+
+    expect(jobs.find(1).stage_id).to eq 102
+    expect(jobs.find(2).stage_id).to eq 102
+    expect(jobs.find(3).stage_id).to eq 101
+    expect(jobs.find(4).stage_id).to eq 103
+    expect(jobs.find(5).stage_id).to eq 105
+    expect(jobs.find(6).stage_id).to eq 104
+    expect(jobs.find(7).stage_id).to eq 104
+    expect(jobs.find(8).stage_id).to eq nil
+  end
+end

spec/migrations/migrate_pipeline_stages_spec.rb

+require 'spec_helper'
+require Rails.root.join('db', 'post_migrate', '20170526185842_migrate_pipeline_stages.rb')
+
+describe MigratePipelineStages, :migration do
+  ##
+  # Create test data - pipeline and CI/CD jobs.
+  #
+
+  let(:jobs) { table(:ci_builds) }
+  let(:stages) { table(:ci_stages) }
+  let(:pipelines) { table(:ci_pipelines) }
+  let(:projects) { table(:projects) }
+
+  before do
+    # Create projects
+    #
+    projects.create!(id: 123, name: 'gitlab1', path: 'gitlab1')
+    projects.create!(id: 456, name: 'gitlab2', path: 'gitlab2')
+
+    # Create CI/CD pipelines
+    #
+    pipelines.create!(id: 1, project_id: 123, ref: 'master', sha: 'adf43c3a')
+    pipelines.create!(id: 2, project_id: 456, ref: 'feature', sha: '21a3deb')
+
+    # Create CI/CD jobs
+    #
+    jobs.create!(id: 1, commit_id: 1, project_id: 123, stage_idx: 2, stage: 'build')
+    jobs.create!(id: 2, commit_id: 1, project_id: 123, stage_idx: 2, stage: 'build')
+    jobs.create!(id: 3, commit_id: 1, project_id: 123, stage_idx: 1, stage: 'test')
+    jobs.create!(id: 4, commit_id: 1, project_id: 123, stage_idx: 1, stage: 'test')
+    jobs.create!(id: 5, commit_id: 1, project_id: 123, stage_idx: 3, stage: 'deploy')
+    jobs.create!(id: 6, commit_id: 2, project_id: 456, stage_idx: 3, stage: 'deploy')
+    jobs.create!(id: 7, commit_id: 2, project_id: 456, stage_idx: 2, stage: 'test:2')
+    jobs.create!(id: 8, commit_id: 2, project_id: 456, stage_idx: 1, stage: 'test:1')
+    jobs.create!(id: 9, commit_id: 2, project_id: 456, stage_idx: 1, stage: 'test:1')
+    jobs.create!(id: 10, commit_id: 2, project_id: 456, stage_idx: 2, stage: 'test:2')
+    jobs.create!(id: 11, commit_id: 3, project_id: 456, stage_idx: 3, stage: 'deploy')
+    jobs.create!(id: 12, commit_id: 2, project_id: 789, stage_idx: 3, stage: 'deploy')
+  end
+
+  it 'correctly migrates pipeline stages' do
+    expect(stages.count).to be_zero
+
+    migrate!
+
+    expect(stages.count).to eq 6
+    expect(stages.all.pluck(:name))
+      .to match_array %w[test build deploy test:1 test:2 deploy]
+    expect(stages.where(pipeline_id: 1).order(:id).pluck(:name))
+      .to eq %w[test build deploy]
+    expect(stages.where(pipeline_id: 2).order(:id).pluck(:name))
+      .to eq %w[test:1 test:2 deploy]
+    expect(stages.where(pipeline_id: 3).count).to be_zero
+    expect(stages.where(project_id: 789).count).to be_zero
+  end
+end

spec/services/ci/retry_build_service_spec.rb

@@ -18,11 +18,12 @@ describe Ci::RetryBuildService, :services do
        updated_at started_at finished_at queued_at erased_by
        erased_at auto_canceled_by].freeze
 
+  # TODO, move stage_id accessor to CLONE_ACCESSOR in a follow-up MR.
   IGNORE_ACCESSORS =
     %i[type lock_version target_url base_tags
        commit_id deployments erased_by_id last_deployment project_id
        runner_id tag_taggings taggings tags trigger_request_id
-       user_id auto_canceled_by_id retried].freeze
+       user_id auto_canceled_by_id retried stage_id].freeze
 
   shared_examples 'build duplication' do
     let(:build) do

spec/spec_helper.rb

@@ -56,6 +56,7 @@ RSpec.configure do |config|
   config.include StubGitlabCalls
   config.include StubGitlabData
   config.include ApiHelpers, :api
+  config.include MigrationsHelpers, :migration
 
   config.infer_spec_type_from_file_location!
 
@@ -97,6 +98,17 @@ RSpec.configure do |config|
     Sidekiq.redis(&:flushall)
   end
 
+  config.around(:example, :migration) do |example|
+    begin
+      ActiveRecord::Migrator
+        .migrate(migrations_paths, previous_migration.version)
+
+      example.run
+    ensure
+      ActiveRecord::Migrator.migrate(migrations_paths)
+    end
+  end
+
   config.around(:each, :nested_groups) do |example|
     example.run if Group.supports_nested_groups?
   end

spec/support/db_cleaner.rb

@@ -19,6 +19,10 @@ RSpec.configure do |config|
     DatabaseCleaner.strategy = :truncation
   end
 
+  config.before(:each, :migration) do
+    DatabaseCleaner.strategy = :truncation
+  end
+
   config.before(:each) do
     DatabaseCleaner.start
   end

spec/support/migrations_helpers.rb

+module MigrationsHelpers
+  def table(name)
+    Class.new(ActiveRecord::Base) { self.table_name = name }
+  end
+
+  def migrations_paths
+    ActiveRecord::Migrator.migrations_paths
+  end
+
+  def table_exists?(name)
+    ActiveRecord::Base.connection.table_exists?(name)
+  end
+
+  def migrations
+    ActiveRecord::Migrator.migrations(migrations_paths)
+  end
+
+  def previous_migration
+    migrations.each_cons(2) do |previous, migration|
+      break previous if migration.name == described_class.name
+    end
+  end
+
+  def migrate!
+    ActiveRecord::Migrator.up(migrations_paths) do |migration|
+      migration.name == described_class.name
+    end
+  end
+end