Add latest changes from gitlab-org/gitlab@13-7-stable-ee

app/controllers/repositories/lfs_api_controller.rb

@@ -91,12 +91,17 @@ module Repositories
     def upload_actions(object)
       {
         upload: {
-          href: "#{project.http_url_to_repo}/gitlab-lfs/objects/#{object[:oid]}/#{object[:size]}",
+          href: "#{upload_http_url_to_repo}/gitlab-lfs/objects/#{object[:oid]}/#{object[:size]}",
           header: upload_headers
         }
       }
     end
 
+    # Overridden in EE
+    def upload_http_url_to_repo
+      project.http_url_to_repo
+    end
+
     def upload_headers
       headers = {
         Authorization: authorization_header,

changelogs/unreleased/ak-fix-project-transfer-2.yml

+---
+title: Fix project transfer corrupting shared runners state
+merge_request: 47316
+author:
+type: fixed

db/post_migrate/20201110161542_cleanup_transfered_projects_shared_runners.rb

+# frozen_string_literal: true
+
+class CleanupTransferedProjectsSharedRunners < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  BATCH_SIZE = 25_000
+  INTERVAL = 3.minutes
+  MIGRATION = 'ResetSharedRunnersForTransferredProjects'
+
+  disable_ddl_transaction!
+
+  class Namespace < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'namespaces'
+  end
+
+  def up
+    queue_background_migration_jobs_by_range_at_intervals(Namespace,
+                                                          MIGRATION,
+                                                          INTERVAL,
+                                                          batch_size: BATCH_SIZE)
+  end
+
+  def down
+    # This migration fixes an inconsistent database state resulting from https://gitlab.com/gitlab-org/gitlab/-/issues/271728
+    # and as such does not require a down migration
+  end
+end

db/schema_migrations/20201110161542

+ebc304867bab499da3f9fa69b32373a328948b5068b226df5166042d1c1c7604
\ No newline at end of file

doc/.vale/gitlab/spelling-exceptions.txt

@@ -404,6 +404,8 @@ reusability
 reverified
 reverifies
 reverify
+roadmap
+roadmaps
 rollout
 rollouts
 rsync

doc/development/rake_tasks.md

@@ -88,13 +88,14 @@ bundle exec rake 'gitlab:seed:development_metrics[your_project_id]'
 ### Automation
 
 If you're very sure that you want to **wipe the current database** and refill
-seeds, you could:
+seeds, you can set the `FORCE` environment variable to `yes`:
 
 ```shell
-echo 'yes' | bundle exec rake setup
+FORCE=yes bundle exec rake setup
 ```
 
-To save you from answering `yes` manually.
+This will skip the action confirmation/safety check, saving you from answering
+`yes` manually.
 
 ### Discard `stdout`
 

doc/user/group/roadmap/index.md

@@ -15,7 +15,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w
 > - In [GitLab 13.2](https://gitlab.com/gitlab-org/gitlab/-/issues/214375) and later, the Roadmap also shows milestones in projects in a group.
 > - In [GitLab 13.2](https://gitlab.com/gitlab-org/gitlab/-/issues/212494) and later, milestone bars can be collapsed and expanded.
 
-Epics and milestones within a group containing a start date or due date can be visualized in a form
+Epics and milestones in a group containing a start date or due date can be visualized in a form
 of a timeline (that is, a Gantt chart). The Roadmap page shows the epics and milestones in a
 group, one of its subgroups, or a project in one of the groups.
 
@@ -24,14 +24,31 @@ When you hover over an epic bar, a popover appears with the epic's title, start
 weight completed.
 
 You can expand epics that contain child epics to show their child epics in the roadmap.
-You can click the chevron (**{chevron-down}**) next to the epic title to expand and collapse the child epics.
+You can click the chevron (**{chevron-down}**) next to the epic title to expand and collapse the
+child epics.
 
-On top of the milestone bars, you can see their title. When you hover a milestone bar or title, a popover appears with its title, start date and due date.
-You can also click the chevron (**{chevron-down}**) next to the **Milestones** heading to toggle the list of the milestone bars.
+On top of the milestone bars, you can see their title.
+When you hover over a milestone bar or title, a popover appears with its title, start date, and due
+date. You can also click the chevron (**{chevron-down}**) next to the **Milestones** heading to
+toggle the list of the milestone bars.
 
 ![roadmap view](img/roadmap_view_v13_2.png)
 
-A dropdown menu allows you to show only open or closed epics. By default, all epics are shown.
+## Sort and filter the Roadmap
+
+> - Filtering roadmaps by milestone [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/218621) in GitLab 13.7.
+> - Filtering roadmaps by milestone is [deployed behind a feature flag](../../feature_flags.md), enabled by default.
+> - Filtering roadmaps by milestone is enabled on GitLab.com.
+> - Filtering roadmaps by milestone is recommended for production use.
+> - For GitLab self-managed instances, GitLab administrators can opt to [disable it](#enable-or-disable-filtering-roadmaps-by-milestone). **(PREMIUM ONLY)**
+
+WARNING:
+Filtering roadmaps by milestone might not be available to you. Check the **version history** note above for details.
+
+When you want to explore a roadmap, there are several ways to make it easier by sorting epics or
+filtering them by what's important for you.
+
+A dropdown menu lets you show only open or closed epics. By default, all epics are shown.
 
 ![epics state dropdown](img/epics_state_dropdown_v12_10.png)
 
@@ -45,8 +62,35 @@ You can sort epics in the Roadmap view by:
 Each option contains a button that toggles the sort order between **ascending** and **descending**.
 The sort option and order persist when browsing Epics, including the [epics list view](../epics/index.md).
 
+You can also filter epics in the Roadmap view by:
+
+- Author
+- Label
+- Milestone
+
+![roadmap date range in weeks](img/roadmap_filters_v13_7.png)
+
 Roadmaps can also be [visualized inside an epic](../epics/index.md#roadmap-in-epics).
 
+### Enable or disable filtering roadmaps by milestone **(PREMIUM ONLY)**
+
+Filtering roadmaps by milestone is under development but ready for production use.
+It is deployed behind a feature flag that is **enabled by default**.
+[GitLab administrators with access to the GitLab Rails console](../../../administration/feature_flags.md)
+can opt to disable it.
+
+To enable it:
+
+```ruby
+Feature.enable(:async_filtering)
+```
+
+To disable it:
+
+```ruby
+Feature.disable(:async_filtering)
+```
+
 ## Timeline duration
 
 > - Introduced in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.0.

doc/user/project/clusters/add_eks_clusters.md

@@ -41,14 +41,32 @@ For example, the following policy document allows assuming a role whose name sta
 }
 ```
 
+### Administration settings
+
 Generate an access key for the IAM user, and configure GitLab with the credentials:
 
 1. Navigate to **Admin Area > Settings > General** and expand the **Amazon EKS** section.
 1. Check **Enable Amazon EKS integration**.
-1. Enter the account ID and access key credentials into the respective
-   `Account ID`, `Access key ID` and `Secret access key` fields.
+1. Enter your **Account ID**.
+1. Depending on your configuration, enter your access key and ID:
+
+   - _GitLab 13.7 and later, and using an instance profile_: You may leave
+     **Access key ID** and **Secret access key** blank.
+     Read [Instance profiles](#instance-profiles) for more information.
+   - _All GitLab versions_: Enter your access key credentials into
+     **Access key ID** and **Secret access key**.
+
 1. Click **Save changes**.
 
+#### Instance profiles
+
+> Introduced in [GitLab 13.7](https://gitlab.com/gitlab-org/gitlab/-/issues/291015).
+
+You may leave `Access key ID` and `Secret access key` fields blank if
+you are using an instance profile
+[to pass an IAM role to an EC2 instance](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2_instance-profiles.html).
+Instance profiles dynamically retrieve temporary credentials from AWS when needed.
+
 ## New EKS cluster
 
 > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/22392) in GitLab 12.5.
@@ -175,17 +193,96 @@ You must add your AWS external ID to the
 [IAM Role in the AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-role.html#cli-configure-role-xaccount)
 to manage your cluster using `kubectl`.
 
+### Cluster creation flow
+
+The following sequence illustrates how GitLab works with AWS to create an EKS cluster:
+
+```mermaid
+sequenceDiagram
+    autonumber
+    participant G as GitLab
+    participant A as AWS
+    participant E as EKS cluster
+    alt static credentials
+      G->>G: Load AWS Access and secret key
+    end
+    alt IAM instance profile
+      G->>A: Fetch temporary credentials
+      A->>G: Temporary access credentials
+    end
+    G->>A: AssumeRole: EKS Provision Role
+    A->>A: Check account, external IDs
+    A->>A: Check permissions
+    A->>G: New access credentials
+    note over G: user selects EKS cluster options
+    note over G,A: Use Service Role credentials
+    G->>A: CreateStack (CloudFormation)
+    A->>G: Received
+    G->>G: Wait 5 minutes
+    loop Poll for cluster creation
+      G->>A: DescribeStacks
+      A->>G: CREATE_IN_PROGRESS
+    end
+    note over G,E: EKS Cluster Created
+    G->>A: DescribeStacks
+    A->>G: CREATE_COMPLETE
+    G->>E: kubectl create role (service account)
+    E->>G: OK
+```
+
+First, GitLab must obtain an initial set of credentials to communicate with the AWS API.
+These credentials can be retrieved in one of two ways:
+
+- Statically through the [Administration settings](#administration-settings).
+- Dynamically via an IAM instance profile ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/291015) in GitLab 13.7).
+
+After GitLab retrieves the AWS credentials, it makes an
+[AssumeRole](https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html)
+API call to obtain credentials for the Provision Role. AWS confirms
+the request has the correct account ID, external ID, and permissions.
+
+If the request is valid, AWS returns a new set of temporary credentials GitLab
+uses to load the **Create cluster** options page.
+
+On the **Create cluster** page, the user must select a **Service Role**, which is
+the IAM role that is actually used to create the cluster, and other options
+such as the Kubernetes cluster name, Kubernetes version, and region.
+After the user clicks the **Create Kubernetes cluster** button, GitLab
+submits a CloudFormation API request to create an EKS cluster with the given parameters
+from the user. GitLab waits 5 minutes before checking whether the cluster was created,
+and polls once a minute for up to 30 minutes.
+
+After GitLab receives a `CREATE_COMPLETE` message from AWS, GitLab talks
+to the EKS cluster to create a Kubernetes service account with `cluster-admin`
+privileges, and updates its internal database to reflect the newly-created
+Kubernetes cluster. From this point forward, GitLab uses this service account to
+interact with the cluster.
+
 ### Troubleshooting creating a new cluster
 
 The following errors are commonly encountered when creating a new cluster.
 
-#### Error: Request failed with status code 422
+#### Validation failed: Role ARN must be a valid Amazon Resource Name
+
+Check that the `Provision Role ARN` is correct. An example of a valid ARN:
+
+```plaintext
+arn:aws:iam::123456789012:role/gitlab-eks-provision'
+```
+
+#### Access denied: User `arn:aws:iam::x` is not authorized to perform: `sts:AssumeRole` on resource: `arn:aws:iam::y`
+
+This error occurs when the credentials defined in the
+[Administration settings](#administration-settings) cannot assume the role defined by the
+Provision Role ARN. Check that:
+
+1. The initial set of AWS credentials [has the AssumeRole policy](#additional-requirements-for-self-managed-instances).
+1. The Provision Role has access to create clusters in the given region.
+1. The account ID and
+   [external ID](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_create_for-user_externalid.html)
+   match the value defined in the **Trust relationships** tab in AWS:
 
-When submitting the initial authentication form, GitLab returns a status code 422
-error when it can't determine the role or region you've provided. Make sure you've
-correctly configured your role with the **Account ID** and **External ID**
-provided by GitLab. In GitLab, make sure to enter the correct **Role ARN**.
-Make sure you also have access to the chosen region.
+   ![AWS IAM Trust relationships](img/aws_iam_role_trust.png)
 
 #### Could not load Security Groups for this VPC
 

lib/gitlab/background_migration/reset_shared_runners_for_transferred_projects.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # Resets inconsistent state of shared_runners_enabled for projects that have been transferred
+    class ResetSharedRunnersForTransferredProjects
+      # Model specifically used for migration.
+      class Namespace < ActiveRecord::Base
+        include EachBatch
+
+        self.table_name = 'namespaces'
+      end
+
+      # Model specifically used for migration.
+      class Project < ActiveRecord::Base
+        self.table_name = 'projects'
+      end
+
+      def perform(start_id, stop_id)
+        Project.reset_column_information
+
+        Namespace.where(id: start_id..stop_id).each_batch(of: 1_000) do |relation|
+          ids = relation.where(shared_runners_enabled: false, allow_descendants_override_disabled_shared_runners: false).select(:id)
+
+          Project.where(namespace_id: ids).update_all(shared_runners_enabled: false)
+        end
+      end
+    end
+  end
+end

lib/gitlab/usage_data_counters/hll_redis_counter.rb

@@ -336,12 +336,10 @@ module Gitlab
         end
 
         def weekly_redis_keys(events:, start_date:, end_date:, context: '')
-          weeks = end_date.to_date.cweek - start_date.to_date.cweek
-          weeks = 1 if weeks == 0
-
-          (0..(weeks - 1)).map do |week_increment|
-            events.map { |event| redis_key(event, start_date + week_increment * 7.days, context) }
-          end.flatten
+          end_date = end_date.end_of_week - 1.week
+          (start_date.to_date..end_date.to_date).map do |date|
+            events.map { |event| redis_key(event, date, context) }
+          end.flatten.uniq
         end
       end
     end

spec/lib/gitlab/background_migration/reset_shared_runners_for_transferred_projects_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::ResetSharedRunnersForTransferredProjects, schema: 20201110161542 do
+  let(:namespaces) { table(:namespaces) }
+  let(:projects) { table(:projects) }
+
+  let(:namespace_1) { namespaces.create!(name: 'foo', path: 'foo', shared_runners_enabled: true, allow_descendants_override_disabled_shared_runners: false ) }
+  let(:namespace_2) { namespaces.create!(name: 'foo', path: 'foo', shared_runners_enabled: false, allow_descendants_override_disabled_shared_runners: false ) }
+  let(:namespace_3) { namespaces.create!(name: 'bar', path: 'bar', shared_runners_enabled: false, allow_descendants_override_disabled_shared_runners: true ) }
+  let(:project_1_1) { projects.create!(namespace_id: namespace_1.id, shared_runners_enabled: true) }
+  let(:project_1_2) { projects.create!(namespace_id: namespace_1.id, shared_runners_enabled: false) }
+  let(:project_2_1) { projects.create!(namespace_id: namespace_2.id, shared_runners_enabled: true) }
+  let(:project_2_2) { projects.create!(namespace_id: namespace_2.id, shared_runners_enabled: false) }
+  let(:project_3_1) { projects.create!(namespace_id: namespace_3.id, shared_runners_enabled: true) }
+  let(:project_3_2) { projects.create!(namespace_id: namespace_3.id, shared_runners_enabled: false) }
+
+  it 'corrects each project shared_runners_enabled column' do
+    expect do
+      described_class.new.perform(namespace_1.id, namespace_3.id)
+      project_1_1.reload
+      project_1_2.reload
+      project_2_1.reload
+      project_2_2.reload
+      project_3_1.reload
+      project_3_2.reload
+    end.to not_change(project_1_1, :shared_runners_enabled).from(true)
+    .and not_change(project_1_2, :shared_runners_enabled).from(false)
+    .and change(project_2_1, :shared_runners_enabled).from(true).to(false)
+    .and not_change(project_2_2, :shared_runners_enabled).from(false)
+    .and not_change(project_3_1, :shared_runners_enabled).from(true)
+    .and not_change(project_3_2, :shared_runners_enabled).from(false)
+  end
+end

spec/lib/gitlab/usage_data_counters/hll_redis_counter_spec.rb

@@ -283,6 +283,50 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s
       context 'when no slot is set' do
         it { expect(described_class.unique_events(event_names: [no_slot], start_date: 7.days.ago, end_date: Date.current)).to eq(1) }
       end
+
+      context 'when data crosses into new year' do
+        it 'does not raise error' do
+          expect { described_class.unique_events(event_names: [weekly_event], start_date: DateTime.parse('2020-12-26'), end_date: DateTime.parse('2021-02-01')) }
+            .not_to raise_error
+        end
+      end
+    end
+  end
+
+  describe '.weekly_redis_keys' do
+    using RSpec::Parameterized::TableSyntax
+
+    let(:weekly_event) { 'g_compliance_dashboard' }
+    let(:redis_event) { described_class.send(:event_for, weekly_event) }
+
+    subject(:weekly_redis_keys) { described_class.send(:weekly_redis_keys, events: [redis_event], start_date: DateTime.parse(start_date), end_date: DateTime.parse(end_date)) }
+
+    where(:start_date, :end_date, :keys) do
+      '2020-12-21' | '2020-12-21' | []
+      '2020-12-21' | '2020-12-20' | []
+      '2020-12-21' | '2020-11-21' | []
+      '2021-01-01' | '2020-12-28' | []
+      '2020-12-21' | '2020-12-28' | ['g_{compliance}_dashboard-2020-52']
+      '2020-12-21' | '2021-01-01' | ['g_{compliance}_dashboard-2020-52']
+      '2020-12-27' | '2021-01-01' | ['g_{compliance}_dashboard-2020-52']
+      '2020-12-26' | '2021-01-04' | ['g_{compliance}_dashboard-2020-52', 'g_{compliance}_dashboard-2020-53']
+      '2020-12-26' | '2021-01-11' | ['g_{compliance}_dashboard-2020-52', 'g_{compliance}_dashboard-2020-53', 'g_{compliance}_dashboard-2021-01']
+      '2020-12-26' | '2021-01-17' | ['g_{compliance}_dashboard-2020-52', 'g_{compliance}_dashboard-2020-53', 'g_{compliance}_dashboard-2021-01']
+      '2020-12-26' | '2021-01-18' | ['g_{compliance}_dashboard-2020-52', 'g_{compliance}_dashboard-2020-53', 'g_{compliance}_dashboard-2021-01', 'g_{compliance}_dashboard-2021-02']
+    end
+
+    with_them do
+      it "returns the correct keys" do
+        expect(subject).to match(keys)
+      end
+    end
+
+    it 'returns 1 key for last for week' do
+      expect(described_class.send(:weekly_redis_keys, events: [redis_event], start_date: 7.days.ago.to_date, end_date: Date.current).size).to eq 1
+    end
+
+    it 'returns 4 key for last for weeks' do
+      expect(described_class.send(:weekly_redis_keys, events: [redis_event], start_date: 4.weeks.ago.to_date, end_date: Date.current).size).to eq 4
     end
   end
 

spec/lib/gitlab/usage_data_spec.rb

@@ -54,6 +54,32 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do
 
       expect { subject }.to raise_error('Stopped calculating recorded_at')
     end
+
+    context 'when generating usage ping in critical weeks' do
+      it 'does not raise error when generated in last week of the year' do
+        travel_to(DateTime.parse('2020-12-29')) do
+          expect { subject }.not_to raise_error
+        end
+      end
+
+      it 'does not raise error when generated in first week of the year' do
+        travel_to(DateTime.parse('2021-01-01')) do
+          expect { subject }.not_to raise_error
+        end
+      end
+
+      it 'does not raise error when generated in second week of the year' do
+        travel_to(DateTime.parse('2021-01-07')) do
+          expect { subject }.not_to raise_error
+        end
+      end
+
+      it 'does not raise error when generated in 3rd week of the year' do
+        travel_to(DateTime.parse('2021-01-14')) do
+          expect { subject }.not_to raise_error
+        end
+      end
+    end
   end
 
   describe 'usage_activity_by_stage_package' do

spec/migrations/20201110161542_cleanup_transfered_projects_shared_runners_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require Rails.root.join('db', 'post_migrate', '20201110161542_cleanup_transfered_projects_shared_runners.rb')
+
+RSpec.describe CleanupTransferedProjectsSharedRunners, :sidekiq, schema: 20201110161542 do
+  let(:namespaces) { table(:namespaces) }
+  let(:migration) { described_class.new }
+  let(:batch_interval) { described_class::INTERVAL }
+
+  let!(:namespace_1) { namespaces.create!(name: 'foo', path: 'foo') }
+  let!(:namespace_2) { namespaces.create!(name: 'bar', path: 'bar') }
+  let!(:namespace_3) { namespaces.create!(name: 'baz', path: 'baz') }
+
+  describe '#up' do
+    before do
+      stub_const("#{described_class}::BATCH_SIZE", 2)
+    end
+
+    it 'schedules ResetSharedRunnersForTransferredProjects background jobs' do
+      Sidekiq::Testing.fake! do
+        freeze_time do
+          migration.up
+
+          expect(BackgroundMigrationWorker.jobs.size).to eq(2)
+          expect(described_class::MIGRATION).to be_scheduled_delayed_migration(batch_interval, namespace_1.id, namespace_2.id)
+          expect(described_class::MIGRATION).to be_scheduled_delayed_migration(batch_interval * 2, namespace_3.id, namespace_3.id)
+        end
+      end
+    end
+  end
+end

spec/support/shared_examples/lib/gitlab/usage_data_counters/incident_management_activity_shared_examples.rb

@@ -3,8 +3,8 @@
 RSpec.shared_examples 'an incident management tracked event' do |event|
   describe ".track_event", :clean_gitlab_redis_shared_state do
     let(:counter) { Gitlab::UsageDataCounters::HLLRedisCounter }
-    let(:start_time) { 1.minute.ago }
-    let(:end_time) { 1.minute.from_now }
+    let(:start_time) { 1.week.ago }
+    let(:end_time) { 1.week.from_now }
 
     it "tracks the event using redis" do
       # Allow other subsequent calls