Merge remote-tracking branch 'upstream/master' into ce-to-ee-2018-05-04

# Conflicts:
#	app/controllers/import/base_controller.rb
#	app/models/project_import_state.rb
#	db/schema.rb
#	lib/gitlab/background_migration/populate_import_state.rb
#	lib/gitlab/background_migration/rollback_import_state_data.rb
#	lib/gitlab/legacy_github_import/importer.rb
#	spec/factories/import_state.rb
#	spec/factories/projects.rb
#	spec/lib/gitlab/background_migration/populate_import_state_spec.rb
#	spec/lib/gitlab/background_migration/rollback_import_state_data_spec.rb
#	spec/lib/gitlab/import_export/all_models.yml
#	spec/migrations/migrate_import_attributes_data_from_projects_to_project_mirror_data_spec.rb

[ci skip]

app/assets/stylesheets/pages/repo.scss

@@ -440,6 +440,7 @@
   padding-right: 3px;
 
   .projects-sidebar {
+    min-height: 0;
     display: flex;
     flex-direction: column;
     flex: 1;

app/controllers/import/base_controller.rb

@@ -7,9 +7,15 @@ class Import::BaseController < ApplicationController
 
   def find_jobs(import_type)
     current_user.created_projects
+<<<<<<< HEAD
       .includes(:import_state)
       .where(import_type: import_type)
       .to_json(only: [:id], methods: [:import_status])
+=======
+        .includes(:import_state)
+        .where(import_type: import_type)
+        .to_json(only: [:id], methods: [:import_status])
+>>>>>>> upstream/master
   end
 
   def find_or_create_namespace(names, owner)

app/models/project_import_state.rb

@@ -3,8 +3,11 @@ class ProjectImportState < ActiveRecord::Base
 
   self.table_name = "project_mirror_data"
 
+<<<<<<< HEAD
   prepend EE::ProjectImportState
 
+=======
+>>>>>>> upstream/master
   belongs_to :project, inverse_of: :import_state
 
   validates :project, presence: true

app/services/web_hook_service.rb

@@ -41,7 +41,7 @@ class WebHookService
       http_status: response.code,
       message: response.to_s
     }
-  rescue SocketError, OpenSSL::SSL::SSLError, Errno::ECONNRESET, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Net::OpenTimeout, Net::ReadTimeout => e
+  rescue SocketError, OpenSSL::SSL::SSLError, Errno::ECONNRESET, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Net::OpenTimeout, Net::ReadTimeout, Gitlab::HTTP::BlockedUrlError => e
     log_execution(
       trigger: hook_name,
       url: hook.url,

changelogs/unreleased/dm-webhook-catch-blocked-url-exception.yml

+---
+title: Ensure web hook 'blocked URL' errors are stored in web hook logs and properly
+  surfaced to the user
+merge_request:
+author:
+type: fixed

db/migrate/20180502122856_create_project_mirror_data.rb

+class CreateProjectMirrorData < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def up
+    return if table_exists?(:project_mirror_data)
+
+    create_table :project_mirror_data do |t|
+      t.references :project, index: true, foreign_key: { on_delete: :cascade }
+      t.string :status
+      t.string :jid
+      t.text :last_error
+    end
+  end
+
+  def down
+    drop_table(:project_mirror_data) if table_exists?(:project_mirror_data)
+  end
+end

db/migrate/20180503175054_add_indexes_to_project_mirror_data.rb

+class AddIndexesToProjectMirrorData < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :project_mirror_data, :jid
+    add_concurrent_index :project_mirror_data, :status
+  end
+
+  def down
+    remove_index :project_mirror_data, :jid if index_exists? :project_mirror_data, :jid
+    remove_index :project_mirror_data, :status if index_exists? :project_mirror_data, :status
+  end
+end

db/post_migrate/20180502134117_migrate_import_attributes_data_from_projects_to_project_mirror_data.rb

+class MigrateImportAttributesDataFromProjectsToProjectMirrorData < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  UP_MIGRATION = 'PopulateImportState'.freeze
+  DOWN_MIGRATION = 'RollbackImportStateData'.freeze
+
+  BATCH_SIZE = 1000
+  DELAY_INTERVAL = 5.minutes
+
+  disable_ddl_transaction!
+
+  class Project < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'projects'
+  end
+
+  class ProjectImportState < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'project_mirror_data'
+  end
+
+  def up
+    projects = Project.where.not(import_status: :none)
+
+    queue_background_migration_jobs_by_range_at_intervals(projects, UP_MIGRATION, DELAY_INTERVAL, batch_size: BATCH_SIZE)
+  end
+
+  def down
+    import_state = ProjectImportState.where.not(status: :none)
+
+    queue_background_migration_jobs_by_range_at_intervals(import_state, DOWN_MIGRATION, DELAY_INTERVAL, batch_size: BATCH_SIZE)
+  end
+
+end

db/schema.rb

@@ -11,7 +11,11 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
+<<<<<<< HEAD
 ActiveRecord::Schema.define(version: 20180503154922) do
+=======
+ActiveRecord::Schema.define(version: 20180503175054) do
+>>>>>>> upstream/master
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -1974,6 +1978,7 @@ ActiveRecord::Schema.define(version: 20180503154922) do
 
   create_table "project_mirror_data", force: :cascade do |t|
     t.integer "project_id"
+<<<<<<< HEAD
     t.integer "retry_count", default: 0, null: false
     t.datetime "last_update_started_at"
     t.datetime "last_update_scheduled_at"
@@ -1984,10 +1989,15 @@ ActiveRecord::Schema.define(version: 20180503154922) do
     t.string "jid"
     t.datetime_with_timezone "last_update_at"
     t.datetime_with_timezone "last_successful_update_at"
+=======
+    t.string "status"
+    t.string "jid"
+>>>>>>> upstream/master
     t.text "last_error"
   end
 
   add_index "project_mirror_data", ["jid"], name: "index_project_mirror_data_on_jid", using: :btree
+<<<<<<< HEAD
   add_index "project_mirror_data", ["last_successful_update_at"], name: "index_project_mirror_data_on_last_successful_update_at", using: :btree
   add_index "project_mirror_data", ["next_execution_timestamp", "retry_count"], name: "index_mirror_data_on_next_execution_and_retry_count", using: :btree
   add_index "project_mirror_data", ["project_id"], name: "index_project_mirror_data_on_project_id", unique: true, using: :btree
@@ -2006,6 +2016,11 @@ ActiveRecord::Schema.define(version: 20180503154922) do
   add_index "project_repository_states", ["project_id"], name: "index_project_repository_states_on_project_id", unique: true, using: :btree
   add_index "project_repository_states", ["repository_verification_checksum", "wiki_verification_checksum"], name: "idx_repository_states_on_checksums_partial", where: "((repository_verification_checksum IS NULL) OR (wiki_verification_checksum IS NULL))", using: :btree
 
+=======
+  add_index "project_mirror_data", ["project_id"], name: "index_project_mirror_data_on_project_id", using: :btree
+  add_index "project_mirror_data", ["status"], name: "index_project_mirror_data_on_status", using: :btree
+
+>>>>>>> upstream/master
   create_table "project_statistics", force: :cascade do |t|
     t.integer "project_id", null: false
     t.integer "namespace_id", null: false
@@ -2875,8 +2890,12 @@ ActiveRecord::Schema.define(version: 20180503154922) do
   add_foreign_key "project_features", "projects", name: "fk_18513d9b92", on_delete: :cascade
   add_foreign_key "project_group_links", "projects", name: "fk_daa8cee94c", on_delete: :cascade
   add_foreign_key "project_import_data", "projects", name: "fk_ffb9ee3a10", on_delete: :cascade
+<<<<<<< HEAD
   add_foreign_key "project_mirror_data", "projects", name: "fk_d1aad367d7", on_delete: :cascade
   add_foreign_key "project_repository_states", "projects", on_delete: :cascade
+=======
+  add_foreign_key "project_mirror_data", "projects", on_delete: :cascade
+>>>>>>> upstream/master
   add_foreign_key "project_statistics", "projects", on_delete: :cascade
   add_foreign_key "prometheus_metrics", "projects", on_delete: :cascade
   add_foreign_key "protected_branch_merge_access_levels", "namespaces", column: "group_id", name: "fk_98f3d044fe", on_delete: :cascade

lib/gitlab/background_migration/populate_import_state.rb

@@ -15,8 +15,13 @@ module Gitlab
         Rails.logger.info("#{self.class.name} - Moving import attributes data to project mirror data table: #{start_id} - #{end_id}")
 
         ActiveRecord::Base.connection.execute <<~SQL
+<<<<<<< HEAD
           INSERT INTO project_mirror_data (project_id, status, jid, last_update_at, last_successful_update_at, last_error)
           SELECT id, import_status, import_jid, mirror_last_update_at, mirror_last_successful_update_at, import_error
+=======
+          INSERT INTO project_mirror_data (project_id, status, jid, last_error)
+          SELECT id, import_status, import_jid, import_error
+>>>>>>> upstream/master
           FROM projects
           WHERE projects.import_status != 'none'
           AND projects.id BETWEEN #{start_id} AND #{end_id}

lib/gitlab/background_migration/rollback_import_state_data.rb

@@ -18,8 +18,11 @@ module Gitlab
             SET
               projects.import_status = project_mirror_data.status,
               projects.import_jid = project_mirror_data.jid,
+<<<<<<< HEAD
               projects.mirror_last_update_at = project_mirror_data.last_update_at,
               projects.mirror_last_successful_update_at = project_mirror_data.last_successful_update_at,
+=======
+>>>>>>> upstream/master
               projects.import_error = project_mirror_data.last_error
             WHERE project_mirror_data.project_id = projects.id
             AND project_mirror_data.id BETWEEN #{start_id} AND #{end_id}
@@ -30,8 +33,11 @@ module Gitlab
             SET
               import_status = project_mirror_data.status,
               import_jid = project_mirror_data.jid,
+<<<<<<< HEAD
               mirror_last_update_at = project_mirror_data.last_update_at,
               mirror_last_successful_update_at = project_mirror_data.last_successful_update_at,
+=======
+>>>>>>> upstream/master
               import_error = project_mirror_data.last_error
             FROM project_mirror_data
             WHERE project_mirror_data.project_id = projects.id

lib/gitlab/legacy_github_import/importer.rb

@@ -79,7 +79,10 @@ module Gitlab
         return unless errors.any?
 
         project.ensure_import_state
+<<<<<<< HEAD
 
+=======
+>>>>>>> upstream/master
         project.import_state&.update_column(:last_error, {
           message: 'The remote data could not be fully imported.',
           errors: errors

spec/factories/import_state.rb

@@ -11,6 +11,7 @@ FactoryBot.define do
       association :project, factory: [:project, :repository]
     end
 
+<<<<<<< HEAD
     trait :mirror do
       transient do
         mirror true
@@ -25,17 +26,23 @@ FactoryBot.define do
       end
     end
 
+=======
+>>>>>>> upstream/master
     trait :none do
       status :none
     end
 
     trait :scheduled do
       status :scheduled
+<<<<<<< HEAD
       last_update_scheduled_at { Time.now }
+=======
+>>>>>>> upstream/master
     end
 
     trait :started do
       status :started
+<<<<<<< HEAD
       last_update_started_at { Time.now }
     end
 
@@ -45,10 +52,17 @@ FactoryBot.define do
       status :finished
       last_update_at timestamp
       last_successful_update_at timestamp
+=======
+    end
+
+    trait :finished do
+      status :finished
+>>>>>>> upstream/master
     end
 
     trait :failed do
       status :failed
+<<<<<<< HEAD
       last_update_at { Time.now }
     end
 
@@ -56,6 +70,8 @@ FactoryBot.define do
       status :failed
       retry_count { Gitlab::Mirror::MAX_RETRY + 1 }
       last_update_at { Time.now - 1.minute }
+=======
+>>>>>>> upstream/master
     end
 
     after(:create) do |import_state, evaluator|

spec/factories/projects.rb

@@ -81,29 +81,44 @@ FactoryBot.define do
     trait :import_scheduled do
       transient do
         status :scheduled
+<<<<<<< HEAD
         last_update_scheduled_at Time.now
       end
 
       before(:create) do |project, evaluator|
         project.create_import_state(status: evaluator.status,
                                     last_update_scheduled_at: evaluator.last_update_scheduled_at)
+=======
+      end
+
+      before(:create) do |project, evaluator|
+        project.create_import_state(status: evaluator.status)
+>>>>>>> upstream/master
       end
     end
 
     trait :import_started do
       transient do
         status :started
+<<<<<<< HEAD
         last_update_started_at Time.now
       end
 
       before(:create) do |project, evaluator|
         project.create_import_state(status: evaluator.status,
                                     last_update_started_at: evaluator.last_update_started_at)
+=======
+      end
+
+      before(:create) do |project, evaluator|
+        project.create_import_state(status: evaluator.status)
+>>>>>>> upstream/master
       end
     end
 
     trait :import_finished do
       transient do
+<<<<<<< HEAD
         timestamp = Time.now
 
         status :finished
@@ -115,12 +130,20 @@ FactoryBot.define do
         project.create_import_state(status: evaluator.status,
                                     last_update_at: evaluator.last_update_at,
                                     last_successful_update_at: evaluator.last_successful_update_at)
+=======
+        status :finished
+      end
+
+      before(:create) do |project, evaluator|
+        project.create_import_state(status: evaluator.status)
+>>>>>>> upstream/master
       end
     end
 
     trait :import_failed do
       transient do
         status :failed
+<<<<<<< HEAD
         last_update_at { Time.now }
       end
 
@@ -154,6 +177,13 @@ FactoryBot.define do
       mirror true
       import_url { generate(:url) }
       mirror_user_id { creator_id }
+=======
+      end
+
+      before(:create) do |project, evaluator|
+        project.create_import_state(status: evaluator.status)
+      end
+>>>>>>> upstream/master
     end
 
     trait :archived do

spec/lib/gitlab/background_migration/populate_import_state_spec.rb

 require 'spec_helper'
 
+<<<<<<< HEAD
 describe Gitlab::BackgroundMigration::PopulateImportState, :migration, schema: 20180430144643 do
+=======
+describe Gitlab::BackgroundMigration::PopulateImportState, :migration, schema: 20180502134117 do
+>>>>>>> upstream/master
   let(:migration) { described_class.new }
   let(:namespaces) { table(:namespaces) }
   let(:projects) { table(:projects) }

spec/lib/gitlab/background_migration/rollback_import_state_data_spec.rb

 require 'spec_helper'
 
+<<<<<<< HEAD
 describe Gitlab::BackgroundMigration::RollbackImportStateData, :migration, schema: 20180430144643 do
+=======
+describe Gitlab::BackgroundMigration::RollbackImportStateData, :migration, schema: 20180502134117 do
+>>>>>>> upstream/master
   let(:migration) { described_class.new }
   let(:namespaces) { table(:namespaces) }
   let(:projects) { table(:projects) }

spec/lib/gitlab/import_export/all_models.yml

@@ -308,9 +308,12 @@ project:
 - container_repositories
 - uploads
 - import_state
+<<<<<<< HEAD
 - repository_state
 - source_pipelines
 - sourced_pipelines
+=======
+>>>>>>> upstream/master
 - members_and_requesters
 - build_trace_section_names
 - root_of_fork_network

spec/migrations/migrate_import_attributes_data_from_projects_to_project_mirror_data_spec.rb

 require 'spec_helper'
+<<<<<<< HEAD
 require Rails.root.join('db', 'post_migrate', '20180430144643_migrate_import_attributes_data_from_projects_to_project_mirror_data.rb')
+=======
+require Rails.root.join('db', 'post_migrate', '20180502134117_migrate_import_attributes_data_from_projects_to_project_mirror_data.rb')
+>>>>>>> upstream/master
 
 describe MigrateImportAttributesDataFromProjectsToProjectMirrorData, :sidekiq, :migration do
   let(:namespaces) { table(:namespaces) }

spec/services/web_hook_service_spec.rb

@@ -67,7 +67,7 @@ describe WebHookService do
     end
 
     it 'handles exceptions' do
-      exceptions = [SocketError, OpenSSL::SSL::SSLError, Errno::ECONNRESET, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Net::OpenTimeout, Net::ReadTimeout]
+      exceptions = [SocketError, OpenSSL::SSL::SSLError, Errno::ECONNRESET, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Net::OpenTimeout, Net::ReadTimeout, Gitlab::HTTP::BlockedUrlError]
       exceptions.each do |exception_class|
         exception = exception_class.new('Exception message')