Merge branch 'ce-to-ee-2018-02-13' into 'master'

CE upstream - 2018-02-13 15:25 UTC

See merge request gitlab-org/gitlab-ee!4512

app/controllers/projects/issues_controller.rb

@@ -126,8 +126,7 @@ class Projects::IssuesController < Projects::ApplicationController
   end
 
   def referenced_merge_requests
-    @merge_requests = @issue.referenced_merge_requests(current_user)
-    @closed_by_merge_requests = @issue.closed_by_merge_requests(current_user)
+    @merge_requests, @closed_by_merge_requests = ::Issues::FetchReferencedMergeRequestsService.new(project, current_user).execute(issue)
 
     respond_to do |format|
       format.json do

app/services/issues/fetch_referenced_merge_requests_service.rb

+module Issues
+  class FetchReferencedMergeRequestsService < Issues::BaseService
+    def execute(issue)
+      referenced_merge_requests = issue.referenced_merge_requests(current_user)
+      referenced_merge_requests = Gitlab::IssuableSorter.sort(project, referenced_merge_requests) { |i| i.iid.to_s }
+      closed_by_merge_requests = issue.closed_by_merge_requests(current_user)
+      closed_by_merge_requests = Gitlab::IssuableSorter.sort(project, closed_by_merge_requests) { |i| i.iid.to_s }
+
+      [referenced_merge_requests, closed_by_merge_requests]
+    end
+  end
+end

changelogs/unreleased/26466-natural-sort-mrs.yml

+---
+title: Group MRs on issue page by project and namespace.
+merge_request: 8494
+author: Jeff Stubler

changelogs/unreleased/mk-fix-no-untracked-upload-files-error.yml

+---
+title: Resolve PrepareUntrackedUploads PostgreSQL syntax error
+merge_request: 17019
+author:
+type: fixed

db/migrate/20180208183958_schedule_populate_untracked_uploads_if_needed.rb

+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class SchedulePopulateUntrackedUploadsIfNeeded < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  FOLLOW_UP_MIGRATION = 'PopulateUntrackedUploads'.freeze
+
+  class UntrackedFile < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'untracked_files_for_uploads'
+  end
+
+  def up
+    if table_exists?(:untracked_files_for_uploads)
+      process_or_remove_table
+    end
+  end
+
+  def down
+    # nothing
+  end
+
+  private
+
+  def process_or_remove_table
+    if UntrackedFile.all.empty?
+      drop_temp_table
+    else
+      schedule_populate_untracked_uploads_jobs
+    end
+  end
+
+  def drop_temp_table
+    drop_table(:untracked_files_for_uploads, if_exists: true)
+  end
+
+  def schedule_populate_untracked_uploads_jobs
+    say "Scheduling #{FOLLOW_UP_MIGRATION} background migration jobs since there are rows in untracked_files_for_uploads."
+
+    bulk_queue_background_migration_jobs_by_range(
+      UntrackedFile, FOLLOW_UP_MIGRATION)
+  end
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20180206200543) do
+ActiveRecord::Schema.define(version: 20180208183958) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"

lib/gitlab/background_migration/prepare_untracked_uploads.rb

@@ -43,7 +43,11 @@ module Gitlab
 
         store_untracked_file_paths
 
-        schedule_populate_untracked_uploads_jobs
+        if UntrackedFile.all.empty?
+          drop_temp_table
+        else
+          schedule_populate_untracked_uploads_jobs
+        end
       end
 
       private
@@ -92,7 +96,7 @@ module Gitlab
           end
         end
 
-        yield(paths)
+        yield(paths) if paths.any?
       end
 
       def build_find_command(search_dir)
@@ -165,6 +169,11 @@ module Gitlab
         bulk_queue_background_migration_jobs_by_range(
           UntrackedFile, FOLLOW_UP_MIGRATION)
       end
+
+      def drop_temp_table
+        UntrackedFile.connection.drop_table(:untracked_files_for_uploads,
+                                            if_exists: true)
+      end
     end
   end
 end

qa/qa/specs/features/project/pipelines_spec.rb

@@ -69,7 +69,7 @@ module QA
             tags:
               - qa
               - test
-            script: echo "CONTENTS" > my-artifacts/artifact.txt
+            script: mkdir my-artifacts; echo "CONTENTS" > my-artifacts/artifact.txt
             artifacts:
               paths:
                 - my-artifacts/
@@ -95,7 +95,7 @@ module QA
         expect(pipeline).to have_build('test-success', status: :success)
         expect(pipeline).to have_build('test-failure', status: :failed)
         expect(pipeline).to have_build('test-tags', status: :pending)
-        expect(pipeline).to have_build('test-artifacts', status: :failed)
+        expect(pipeline).to have_build('test-artifacts', status: :success)
       end
     end
   end

spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb

@@ -8,8 +8,6 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :sidekiq do
 
   before do
     DatabaseCleaner.clean
-
-    drop_temp_table_if_exists
   end
 
   after do
@@ -23,57 +21,7 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :sidekiq do
     end
   end
 
-  # E.g. The installation is in use at the time of migration, and someone has
-  # just uploaded a file
-  shared_examples 'does not add files in /uploads/tmp' do
-    let(:tmp_file) { Rails.root.join(described_class::ABSOLUTE_UPLOAD_DIR, 'tmp', 'some_file.jpg') }
-
-    before do
-      FileUtils.mkdir(File.dirname(tmp_file))
-      FileUtils.touch(tmp_file)
-    end
-
-    after do
-      FileUtils.rm(tmp_file)
-    end
-
-    it 'does not add files from /uploads/tmp' do
-      described_class.new.perform
-
-      expect(untracked_files_for_uploads.count).to eq(5)
-    end
-  end
-
-  it 'ensures the untracked_files_for_uploads table exists' do
-    expect do
-      described_class.new.perform
-    end.to change { ActiveRecord::Base.connection.table_exists?(:untracked_files_for_uploads) }.from(false).to(true)
-  end
-
-  it 'has a path field long enough for really long paths' do
-    described_class.new.perform
-
-    component = 'a' * 255
-
-    long_path = [
-      'uploads',
-      component, # project.full_path
-      component  # filename
-    ].flatten.join('/')
-
-    record = untracked_files_for_uploads.create!(path: long_path)
-    expect(record.reload.path.size).to eq(519)
-  end
-
-  context "test bulk insert with ON CONFLICT DO NOTHING or IGNORE" do
-    around do |example|
-      # If this is CI, we use Postgres 9.2 so this whole context should be
-      # skipped since we're unable to use ON CONFLICT DO NOTHING or IGNORE.
-      if described_class.new.send(:can_bulk_insert_and_ignore_duplicates?)
-        example.run
-      end
-    end
-
+  shared_examples 'prepares the untracked_files_for_uploads table' do
     context 'when files were uploaded before and after hashed storage was enabled' do
       let!(:appearance) { create_or_update_appearance(logo: uploaded_file, header_logo: uploaded_file) }
       let!(:user) { create(:user, :with_avatar) }
@@ -90,6 +38,21 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :sidekiq do
         UploadService.new(project2, uploaded_file, FileUploader).execute
       end
 
+      it 'has a path field long enough for really long paths' do
+        described_class.new.perform
+
+        component = 'a' * 255
+
+        long_path = [
+          'uploads',
+          component, # project.full_path
+          component  # filename
+        ].flatten.join('/')
+
+        record = untracked_files_for_uploads.create!(path: long_path)
+        expect(record.reload.path.size).to eq(519)
+      end
+
       it 'adds unhashed files to the untracked_files_for_uploads table' do
         described_class.new.perform
 
@@ -130,91 +93,66 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :sidekiq do
         end
       end
 
+      # E.g. The installation is in use at the time of migration, and someone has
+      # just uploaded a file
       context 'when there are files in /uploads/tmp' do
-        it_behaves_like 'does not add files in /uploads/tmp'
-      end
-    end
-  end
-
-  context 'test bulk insert without ON CONFLICT DO NOTHING or IGNORE' do
-    before do
-      # If this is CI, we use Postgres 9.2 so this stub has no effect.
-      #
-      # If this is being run on Postgres 9.5+ or MySQL, then this stub allows us
-      # to test the bulk insert functionality without ON CONFLICT DO NOTHING or
-      # IGNORE.
-      allow_any_instance_of(described_class).to receive(:postgresql_pre_9_5?).and_return(true)
-    end
-
-    context 'when files were uploaded before and after hashed storage was enabled' do
-      let!(:appearance) { create_or_update_appearance(logo: uploaded_file, header_logo: uploaded_file) }
-      let!(:user) { create(:user, :with_avatar) }
-      let!(:project1) { create(:project, :with_avatar, :legacy_storage) }
-      let(:project2) { create(:project) } # instantiate after enabling hashed_storage
-
-      before do
-        # Markdown upload before enabling hashed_storage
-        UploadService.new(project1, uploaded_file, FileUploader).execute
+        let(:tmp_file) { Rails.root.join(described_class::ABSOLUTE_UPLOAD_DIR, 'tmp', 'some_file.jpg') }
 
-        stub_application_setting(hashed_storage_enabled: true)
-
-        # Markdown upload after enabling hashed_storage
-        UploadService.new(project2, uploaded_file, FileUploader).execute
-      end
-
-      it 'adds unhashed files to the untracked_files_for_uploads table' do
-        described_class.new.perform
-
-        expect(untracked_files_for_uploads.count).to eq(5)
-      end
-
-      it 'adds files with paths relative to CarrierWave.root' do
-        described_class.new.perform
-        untracked_files_for_uploads.all.each do |file|
-          expect(file.path.start_with?('uploads/')).to be_truthy
+        before do
+          FileUtils.mkdir(File.dirname(tmp_file))
+          FileUtils.touch(tmp_file)
         end
-      end
-
-      it 'does not add hashed files to the untracked_files_for_uploads table' do
-        described_class.new.perform
 
-        hashed_file_path = project2.uploads.where(uploader: 'FileUploader').first.path
-        expect(untracked_files_for_uploads.where("path like '%#{hashed_file_path}%'").exists?).to be_falsey
-      end
+        after do
+          FileUtils.rm(tmp_file)
+        end
 
-      it 'correctly schedules the follow-up background migration jobs' do
-        described_class.new.perform
+        it 'does not add files from /uploads/tmp' do
+          described_class.new.perform
 
-        expect(described_class::FOLLOW_UP_MIGRATION).to be_scheduled_migration(1, 5)
-        expect(BackgroundMigrationWorker.jobs.size).to eq(1)
+          expect(untracked_files_for_uploads.count).to eq(5)
+        end
       end
 
-      # E.g. from a previous failed run of this background migration
-      context 'when there is existing data in untracked_files_for_uploads' do
-        before do
-          described_class.new.perform
-        end
+      context 'when the last batch size exactly matches the max batch size' do
+        it 'does not raise error' do
+          stub_const("#{described_class}::FIND_BATCH_SIZE", 5)
 
-        it 'does not error or produce duplicates of existing data' do
           expect do
             described_class.new.perform
-          end.not_to change { untracked_files_for_uploads.count }.from(5)
+          end.not_to raise_error
+
+          expect(untracked_files_for_uploads.count).to eq(5)
         end
       end
+    end
+  end
 
-      context 'when there are files in /uploads/tmp' do
-        it_behaves_like 'does not add files in /uploads/tmp'
-      end
+  # If running on Postgres 9.2 (like on CI), this whole context is skipped
+  # since we're unable to use ON CONFLICT DO NOTHING or IGNORE.
+  context "test bulk insert with ON CONFLICT DO NOTHING or IGNORE", if: described_class.new.send(:can_bulk_insert_and_ignore_duplicates?) do
+    it_behaves_like 'prepares the untracked_files_for_uploads table'
+  end
+
+  # If running on Postgres 9.2 (like on CI), the stubbed method has no effect.
+  #
+  # If running on Postgres 9.5+ or MySQL, then this context effectively tests
+  # the bulk insert functionality without ON CONFLICT DO NOTHING or IGNORE.
+  context 'test bulk insert without ON CONFLICT DO NOTHING or IGNORE' do
+    before do
+      allow_any_instance_of(described_class).to receive(:postgresql_pre_9_5?).and_return(true)
     end
+
+    it_behaves_like 'prepares the untracked_files_for_uploads table'
   end
 
   # Very new or lightly-used installations that are running this migration
   # may not have an upload directory because they have no uploads.
   context 'when no files were ever uploaded' do
-    it 'does not add to the untracked_files_for_uploads table (and does not raise error)' do
+    it 'deletes the `untracked_files_for_uploads` table (and does not raise error)' do
       described_class.new.perform
 
-      expect(untracked_files_for_uploads.count).to eq(0)
+      expect(untracked_files_for_uploads.connection.table_exists?(:untracked_files_for_uploads)).to be_falsey
     end
   end
 end

spec/requests/openid_connect_spec.rb

@@ -81,7 +81,7 @@ describe 'OpenID Connect requests' do
       it 'includes all user information and group memberships' do
         request_user_info
 
-        expect(json_response).to eq({
+        expect(json_response).to match(a_hash_including({
           'sub'            => hashed_subject,
           'name'           => 'Alice',
           'nickname'       => 'alice',
@@ -90,13 +90,17 @@ describe 'OpenID Connect requests' do
           'website'        => 'https://example.com',
           'profile'        => 'http://localhost/alice',
           'picture'        => "http://localhost/uploads/-/system/user/avatar/#{user.id}/dk.png",
-          'groups'         =>
-            if Group.supports_nested_groups?
-              ['group1', 'group2/group3', 'group2/group3/group4']
-            else
-              ['group1', 'group2/group3']
-            end
-        })
+          'groups'         => anything
+        }))
+
+        expected_groups =
+          if Group.supports_nested_groups?
+            ['group1', 'group2/group3', 'group2/group3/group4']
+          else
+            ['group1', 'group2/group3']
+          end
+
+        expect(json_response['groups']).to match(array_including(expected_groups))
       end
     end
 

spec/services/issues/fetch_referenced_merge_requests_service_spec.rb

+require 'spec_helper.rb'
+
+describe Issues::FetchReferencedMergeRequestsService do
+  let(:project) { create(:project) }
+  let(:issue) { create(:issue, project: project) }
+  let(:other_project) { create(:project) }
+
+  let(:mr) { create(:merge_request, source_project: project, target_project: project, id: 2)}
+  let(:other_mr) { create(:merge_request, source_project: other_project, target_project: other_project, id: 1)}
+
+  let(:user) { create(:user) }
+  let(:service) { described_class.new(project, user) }
+
+  context 'with mentioned merge requests' do
+    it 'returns a list of sorted merge requests' do
+      allow(issue).to receive(:referenced_merge_requests).with(user).and_return([other_mr, mr])
+
+      mrs, closed_by_mrs = service.execute(issue)
+
+      expect(mrs).to match_array([mr, other_mr])
+      expect(closed_by_mrs).to match_array([])
+    end
+  end
+
+  context 'with closed-by merge requests' do
+    it 'returns a list of sorted merge requests' do
+      allow(issue).to receive(:closed_by_merge_requests).with(user).and_return([other_mr, mr])
+
+      mrs, closed_by_mrs = service.execute(issue)
+
+      expect(mrs).to match_array([])
+      expect(closed_by_mrs).to match_array([mr, other_mr])
+    end
+  end
+end