Add latest changes from gitlab-org/gitlab@master

.gitlab-ci.yml

@@ -3,6 +3,7 @@ image: "registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.5-golang-1.1
 stages:
   - sync
   - prepare
+  - fixtures
   - test
   - post-test
   - review-prepare

.gitlab/CODEOWNERS

@@ -43,3 +43,6 @@ Dangerfile @gl-quality/eng-prod
 /danger/ @gl-quality/eng-prod
 /lib/gitlab/danger/ @gl-quality/eng-prod
 /scripts/ @gl-quality/eng-prod
+
+# Delivery owner files
+/.gitlab/ci/releases.gitlab-ci.yml @gitlab-org/delivery

.gitlab/ci/dev-fixtures.gitlab-ci.yml

@@ -7,7 +7,6 @@
     - .use-pg9
   stage: test
   needs: ["setup-test-env"]
-  dependencies: ["setup-test-env"]
   variables:
     FIXTURE_PATH: "db/fixtures/development"
     SEED_CYCLE_ANALYTICS: "true"

.gitlab/ci/docs.gitlab-ci.yml

@@ -46,7 +46,9 @@ docs lint:
     - .docs:rules:docs-lint
   image: "registry.gitlab.com/gitlab-org/gitlab-docs:docs-lint"
   stage: test
-  dependencies: []
+  needs:
+    - job: "retrieve-tests-metadata"
+      artifacts: false
   script:
     - scripts/lint-doc.sh
     # Lint Markdown

.gitlab/ci/frontend.gitlab-ci.yml

@@ -120,15 +120,54 @@ compile-assets pull-cache as-if-foss:
     policy: pull
     key: "assets-compile:v9:foss"
 
-.frontend-job-base:
+.frontend-fixtures-base:
   extends:
     - .default-tags
     - .default-retry
     - .default-cache
     - .default-before_script
     - .use-pg9
+  stage: fixtures
+  needs:
+    - job: "setup-test-env"
+      artifacts: true
+    - job: "compile-assets pull-cache"
+      artifacts: true
+  script:
+    - date
+    - scripts/gitaly-test-spawn
+    - date
+    - bundle exec rake frontend:fixtures
+  artifacts:
+    name: frontend-fixtures
+    expire_in: 31d
+    when: always
+    paths:
+      - node_modules
+      - public/assets
+      - tmp/tests/frontend/
+
+frontend-fixtures:
+  extends:
+    - .frontend-fixtures-base
+    - .frontend:rules:default-frontend-jobs
+
+frontend-fixtures-as-if-foss:
+  extends:
+    - .frontend-fixtures-base
+    - .frontend:rules:default-frontend-jobs-as-if-foss
+    - .as-if-foss
+
+.frontend-job-base:
+  extends:
+    - .default-tags
+    - .default-retry
+    - .default-cache
+    - .default-before_script
+  variables:
+    USE_BUNDLE_INSTALL: "false"
+    SETUP_DB: "false"
   stage: test
-  needs: ["setup-test-env", "compile-assets pull-cache"]
 
 .karma-base:
   extends: .frontend-job-base
@@ -138,14 +177,13 @@ compile-assets pull-cache as-if-foss:
   script:
     - export BABEL_ENV=coverage CHROME_LOG_FILE=chrome_debug.log
     - date
-    - scripts/gitaly-test-spawn
-    - date
-    - bundle exec rake karma
+    - yarn karma
 
 karma:
   extends:
     - .karma-base
     - .frontend:rules:default-frontend-jobs
+  needs: ["frontend-fixtures"]
   coverage: '/^Statements *: (\d+\.\d+%)/'
   artifacts:
     name: coverage-javascript
@@ -163,13 +201,11 @@ karma-as-if-foss:
     - .karma-base
     - .frontend:rules:default-frontend-jobs-as-if-foss
     - .as-if-foss
+  needs: ["frontend-fixtures-as-if-foss"]
 
 .jest-base:
   extends: .frontend-job-base
   script:
-    - scripts/gitaly-test-spawn
-    - date
-    - bundle exec rake frontend:fixtures
     - date
     - yarn jest --ci --coverage
   cache:
@@ -182,6 +218,7 @@ jest:
   extends:
     - .jest-base
     - .frontend:rules:default-frontend-jobs
+  needs: ["frontend-fixtures"]
   artifacts:
     name: coverage-frontend
     expire_in: 31d
@@ -198,6 +235,7 @@ jest-as-if-foss:
     - .jest-base
     - .frontend:rules:default-frontend-jobs-as-if-foss
     - .as-if-foss
+  needs: ["frontend-fixtures-as-if-foss"]
   cache:
     policy: pull
 

.gitlab/ci/qa.gitlab-ci.yml

@@ -3,7 +3,9 @@
     - .default-tags
     - .default-retry
   stage: test
-  dependencies: []
+  needs:
+    - job: "retrieve-tests-metadata"
+      artifacts: false
   cache:
     key: "qa-framework-jobs:v1"
     paths:

.gitlab/ci/reports.gitlab-ci.yml

@@ -12,6 +12,9 @@ code_quality:
     - .default-retry
     - .reports:rules:code_quality
   stage: test
+  needs:
+    - job: "retrieve-tests-metadata"
+      artifacts: false
   image: docker:stable
   allow_failure: true
   services:
@@ -39,7 +42,6 @@ code_quality:
     paths:
       - gl-code-quality-report.json  # GitLab-specific
     expire_in: 1 week  # GitLab-specific
-  dependencies: []
 
 # We need to duplicate this job's definition because it seems it's impossible to
 # override an included `only.refs`.
@@ -52,7 +54,9 @@ sast:
     - .reports:rules:sast
   stage: test
   allow_failure: true
-  dependencies: []  # GitLab-specific
+  needs:
+    - job: "retrieve-tests-metadata"
+      artifacts: false
   artifacts:
     paths:
       - gl-sast-report.json  # GitLab-specific
@@ -90,6 +94,9 @@ dependency_scanning:
     - .default-retry
     - .reports:rules:dependency_scanning
   stage: test
+  needs:
+    - job: "retrieve-tests-metadata"
+      artifacts: false
   image: docker:stable
   variables:
     DOCKER_DRIVER: overlay2
@@ -148,7 +155,6 @@ dependency_scanning:
     reports:
       dependency_scanning: gl-dependency-scanning-report.json
     expire_in: 1 week  # GitLab-specific
-  dependencies: []
 
 # We need to duplicate this job's definition because it seems it's impossible to
 # override an included `only.refs`.

.gitlab/ci/review.gitlab-ci.yml

@@ -248,7 +248,9 @@ danger-review:
     - .review:rules:danger
   image: registry.gitlab.com/gitlab-org/gitlab-build-images:danger
   stage: test
-  dependencies: []
+  needs:
+    - job: "retrieve-tests-metadata"
+      artifacts: false
   script:
     - git version
     - node --version

.gitlab/ci/rules.gitlab-ci.yml

@@ -371,11 +371,11 @@
 ##################
 .releases:rules:canonical-dot-com-gitlab-stable-branch-only:
   rules:
-    - if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAME == "gitlab-org/gitlab" && $CI_COMMIT_REF_NAME =~ /^[\d-]+-stable-ee$/'
+    - if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_PATH == "gitlab-org/gitlab" && $CI_COMMIT_REF_NAME =~ /^[\d-]+-stable-ee$/'
 
 .releases:rules:canonical-dot-com-security-gitlab-stable-branch-only:
   rules:
-    - if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAME == "gitlab-org/security/gitlab" && $CI_COMMIT_REF_NAME =~ /^[\d-]+-stable-ee$/'
+    - if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_PATH == "gitlab-org/security/gitlab" && $CI_COMMIT_REF_NAME =~ /^[\d-]+-stable-ee$/'
 
 #################
 # Reports rules #

Gemfile

@@ -355,7 +355,7 @@ group :development, :test do
 
   gem 'database_cleaner', '~> 1.7.0'
   gem 'factory_bot_rails', '~> 5.1.0'
-  gem 'rspec-rails', '~> 4.0.0.beta3'
+  gem 'rspec-rails', '~> 4.0.0.beta4'
 
   # Prevent occasions where minitest is not bundled in packaged versions of ruby (see #3826)
   gem 'minitest', '~> 5.11.0'

Gemfile.lock

@@ -894,36 +894,36 @@ GEM
       chunky_png
     rqrcode-rails3 (0.1.7)
       rqrcode (>= 0.4.2)
-    rspec (3.8.0)
-      rspec-core (~> 3.8.0)
-      rspec-expectations (~> 3.8.0)
-      rspec-mocks (~> 3.8.0)
-    rspec-core (3.8.2)
-      rspec-support (~> 3.8.0)
-    rspec-expectations (3.8.4)
+    rspec (3.9.0)
+      rspec-core (~> 3.9.0)
+      rspec-expectations (~> 3.9.0)
+      rspec-mocks (~> 3.9.0)
+    rspec-core (3.9.1)
+      rspec-support (~> 3.9.1)
+    rspec-expectations (3.9.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.8.0)
-    rspec-mocks (3.8.1)
+      rspec-support (~> 3.9.0)
+    rspec-mocks (3.9.1)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.8.0)
+      rspec-support (~> 3.9.0)
     rspec-parameterized (0.4.2)
       binding_ninja (>= 0.2.3)
       parser
       proc_to_ast
       rspec (>= 2.13, < 4)
       unparser
-    rspec-rails (4.0.0.beta3)
+    rspec-rails (4.0.0.beta4)
       actionpack (>= 4.2)
       activesupport (>= 4.2)
       railties (>= 4.2)
-      rspec-core (~> 3.8)
-      rspec-expectations (~> 3.8)
-      rspec-mocks (~> 3.8)
-      rspec-support (~> 3.8)
+      rspec-core (~> 3.9)
+      rspec-expectations (~> 3.9)
+      rspec-mocks (~> 3.9)
+      rspec-support (~> 3.9)
     rspec-retry (0.6.1)
       rspec-core (> 3.3)
     rspec-set (0.1.3)
-    rspec-support (3.8.2)
+    rspec-support (3.9.2)
     rspec_junit_formatter (0.4.1)
       rspec-core (>= 2, < 4, != 2.12.0)
     rspec_profiling (0.0.5)
@@ -1350,7 +1350,7 @@ DEPENDENCIES
   rouge (~> 3.16.0)
   rqrcode-rails3 (~> 0.1.7)
   rspec-parameterized
-  rspec-rails (~> 4.0.0.beta3)
+  rspec-rails (~> 4.0.0.beta4)
   rspec-retry (~> 0.6.1)
   rspec-set (~> 0.1.3)
   rspec_junit_formatter

db/migrate/20200212014653_rename_security_dashboard_feature_flag_to_instance_security_dashboard.rb

+# frozen_string_literal: true
+
+class RenameSecurityDashboardFeatureFlagToInstanceSecurityDashboard < ActiveRecord::Migration[6.0]
+  DOWNTIME = false
+
+  class FeatureGate < ApplicationRecord
+    self.table_name = 'feature_gates'
+  end
+
+  def up
+    security_dashboard_feature = FeatureGate.find_by(feature_key: :security_dashboard, key: :boolean)
+
+    if security_dashboard_feature.present?
+      FeatureGate.safe_find_or_create_by!(
+        feature_key: :instance_security_dashboard,
+        key: :boolean,
+        value: security_dashboard_feature.value
+      )
+    end
+  end
+
+  def down
+    FeatureGate.find_by(feature_key: :instance_security_dashboard, key: :boolean)&.delete
+  end
+end

db/post_migrate/20200214034836_remove_security_dashboard_feature_flag.rb

+# frozen_string_literal: true
+
+class RemoveSecurityDashboardFeatureFlag < ActiveRecord::Migration[6.0]
+  DOWNTIME = false
+
+  class FeatureGate < ApplicationRecord
+    self.table_name = 'feature_gates'
+  end
+
+  def up
+    FeatureGate.find_by(feature_key: :security_dashboard, key: :boolean)&.delete
+  end
+
+  def down
+    instance_security_dashboard_feature = FeatureGate.find_by(feature_key: :instance_security_dashboard, key: :boolean)
+
+    if instance_security_dashboard_feature.present?
+      FeatureGate.safe_find_or_create_by!(
+        feature_key: :security_dashboard,
+        key: instance_security_dashboard_feature.key,
+        value: instance_security_dashboard_feature.value
+      )
+    end
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2020_02_13_220211) do
+ActiveRecord::Schema.define(version: 2020_02_14_034836) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "pg_trgm"

doc/development/pipelines.md

@@ -136,17 +136,23 @@ graph RL;
   E[review-build-cng];
   F[build-qa-image];
   G[review-deploy];
-  I["karma, jest, webpack-dev-server, static-analysis"];
+  I["karma, jest"];
   I2["karma-as-if-foss, jest-as-if-foss<br/>(EE default refs only)"];
   J["compile-assets pull-push-cache<br/>(master only)"];
   J2["compile-assets pull-push-cache as-if-foss<br/>(EE master only)"];
   K[compile-assets pull-cache];
   K2["compile-assets pull-cache as-if-foss<br/>(EE default refs only)"];
+  U[frontend-fixtures];
+  U2["frontend-fixtures-as-if-foss<br/>(EE default refs only)"];
+  V["webpack-dev-server, static-analysis"];
   M[coverage];
   N["pages (master only)"];
   Q[package-and-qa];
   S["RSpec<br/>(e.g. rspec unit pg9)"]
   T[retrieve-tests-metadata];
+  QA["qa:internal, qa:selectors"];
+  QA2["qa:internal-as-if-foss, qa:selectors-as-if-foss<br/>(EE default refs only)"];
+  X["docs lint, code_quality, sast, dependency_scanning, danger-review"];
 
 subgraph "`prepare` stage"
     A
@@ -160,17 +166,26 @@ subgraph "`prepare` stage"
     T
     end
 
+subgraph "`fixture` stage"
+    U -.-> |needs and depends on| A;
+    U -.-> |needs and depends on| K;
+    U2 -.-> |needs and depends on| A;
+    U2 -.-> |needs and depends on| K2;
+    end
+
 subgraph "`test` stage"
     D -.-> |needs| A;
-    I -.-> |needs and depends on| A;
-    I -.-> |needs and depends on| K;
-    I2 -.-> |needs and depends on| A;
-    I2 -.-> |needs and depends on| K;
+    I -.-> |needs and depends on| U;
+    I2 -.-> |needs and depends on| U2;
     L -.-> |needs and depends on| A;
     S -.-> |needs and depends on| A;
     S -.-> |needs and depends on| K;
     S -.-> |needs and depends on| T;
     L["db:*, gitlab:setup, graphql-docs-verify, downtime_check"] -.-> |needs| A;
+    V -.-> |needs and depends on| K;
+    X -.-> |needs| T;
+    QA -.-> |needs| T;
+    QA2 -.-> |needs| T;
     end
 
 subgraph "`post-test` stage"

lib/gitlab/git/blob.rb

@@ -124,11 +124,12 @@ module Gitlab
           self.__send__("#{key}=", options[key.to_sym]) # rubocop:disable GitlabSecurity/PublicSend
         end
 
-        record_metric_blob_size
-
         # Retain the actual size before it is encoded
         @loaded_size = @data.bytesize if @data
         @loaded_all_data = @loaded_size == size
+
+        record_metric_blob_size
+        record_metric_truncated(truncated?)
       end
 
       def binary_in_repo?
@@ -210,6 +211,14 @@ module Gitlab
         self.class.gitlab_blob_size.observe({}, size)
       end
 
+      def record_metric_truncated(bool)
+        if bool
+          self.class.gitlab_blob_truncated_true.increment
+        else
+          self.class.gitlab_blob_truncated_false.increment
+        end
+      end
+
       def has_lfs_version_key?
         !empty? && text_in_repo? && data.start_with?("version https://git-lfs.github.com/spec")
       end

spec/frontend/notes/old_notes_spec.js

@@ -28,7 +28,10 @@ window.gl = window.gl || {};
 gl.utils = gl.utils || {};
 gl.utils.disableButtonIfEmptyField = () => {};
 
-describe('Old Notes (~/notes.js)', () => {
+// the following test is unreliable and failing in master 2-3 times a day
+// see https://gitlab.com/gitlab-org/gitlab/issues/206906#note_290602581
+// eslint-disable-next-line jest/no-disabled-tests
+describe.skip('Old Notes (~/notes.js)', () => {
   beforeEach(() => {
     jest.useFakeTimers();
     loadFixtures(fixture);

spec/lib/gitlab/git/blob_spec.rb

@@ -22,7 +22,23 @@ describe Gitlab::Git::Blob, :seed_helper do
     it 'records blob size' do
       expect(described_class).to receive(:gitlab_blob_size).and_call_original
 
-      Gitlab::Git::Blob.new(name: 'test', size: 1234)
+      Gitlab::Git::Blob.new(name: 'test', size: 4, data: 'abcd')
+    end
+
+    context 'when untruncated' do
+      it 'attempts to record gitlab_blob_truncated_false' do
+        expect(described_class).to receive(:gitlab_blob_truncated_false).and_call_original
+
+        Gitlab::Git::Blob.new(name: 'test', size: 4, data: 'abcd')
+      end
+    end
+
+    context 'when truncated' do
+      it 'attempts to record gitlab_blob_truncated_true' do
+        expect(described_class).to receive(:gitlab_blob_truncated_true).and_call_original
+
+        Gitlab::Git::Blob.new(name: 'test', size: 40, data: 'abcd')
+      end
     end
   end
 

spec/migrations/remove_security_dashboard_feature_flag_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+require Rails.root.join('db', 'post_migrate', '20200214034836_remove_security_dashboard_feature_flag.rb')
+
+describe RemoveSecurityDashboardFeatureFlag, :migration do
+  let(:feature_gates) { table(:feature_gates) }
+
+  subject(:migration) { described_class.new }
+
+  describe '#up' do
+    it 'deletes the security_dashboard feature gate' do
+      security_dashboard_feature = feature_gates.create!(feature_key: :security_dashboard, key: :boolean, value: 'false')
+      actors_security_dashboard_feature = feature_gates.create!(feature_key: :security_dashboard, key: :actors, value: 'Project:1')
+
+      migration.up
+
+      expect { security_dashboard_feature.reload }.to raise_error(ActiveRecord::RecordNotFound)
+      expect(actors_security_dashboard_feature.reload).to be_present
+    end
+  end
+
+  describe '#down' do
+    it 'copies the instance_security_dashboard feature gate to a security_dashboard gate' do
+      feature_gates.create!(feature_key: :instance_security_dashboard, key: :actors, value: 'Project:1')
+      feature_gates.create!(feature_key: :instance_security_dashboard, key: 'boolean', value: 'false')
+
+      migration.down
+
+      security_dashboard_feature = feature_gates.find_by(feature_key: :security_dashboard, key: :boolean)
+      expect(security_dashboard_feature.value).to eq('false')
+    end
+
+    context 'when there is no instance_security_dashboard gate' do
+      it 'does nothing' do
+        migration.down
+
+        security_dashboard_feature = feature_gates.find_by(feature_key: :security_dashboard, key: :boolean)
+        expect(security_dashboard_feature).to be_nil
+      end
+    end
+
+    context 'when there already is a security_dashboard gate' do
+      it 'does nothing' do
+        feature_gates.create!(feature_key: :security_dashboard, key: 'boolean', value: 'false')
+        feature_gates.create!(feature_key: :instance_security_dashboard, key: 'boolean', value: 'false')
+
+        expect { migration.down }.not_to raise_error
+      end
+    end
+  end
+end

spec/migrations/rename_security_dashboard_feature_flag_to_instance_security_dashboard_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+require Rails.root.join('db', 'migrate', '20200212014653_rename_security_dashboard_feature_flag_to_instance_security_dashboard.rb')
+
+describe RenameSecurityDashboardFeatureFlagToInstanceSecurityDashboard, :migration do
+  let(:feature_gates) { table(:feature_gates) }
+
+  subject(:migration) { described_class.new }
+
+  describe '#up' do
+    it 'copies the security_dashboard feature gate to a new instance_security_dashboard gate' do
+      feature_gates.create!(feature_key: :security_dashboard, key: :actors, value: 'Project:1')
+      feature_gates.create!(feature_key: :security_dashboard, key: :boolean, value: 'false')
+
+      migration.up
+
+      instance_security_dashboard_feature = feature_gates.find_by(feature_key: :instance_security_dashboard, key: :boolean)
+      expect(instance_security_dashboard_feature.value).to eq('false')
+    end
+
+    context 'when there is no security_dashboard gate' do
+      it 'does nothing' do
+        migration.up
+
+        instance_security_dashboard_feature = feature_gates.find_by(feature_key: :instance_security_dashboard, key: :boolean)
+        expect(instance_security_dashboard_feature).to be_nil
+      end
+    end
+
+    context 'when there is already an instance_security_dashboard gate' do
+      it 'does nothing' do
+        feature_gates.create!(feature_key: :security_dashboard, key: 'boolean', value: 'false')
+        feature_gates.create!(feature_key: :instance_security_dashboard, key: 'boolean', value: 'false')
+
+        expect { migration.up }.not_to raise_error
+      end
+    end
+  end
+
+  describe '#down' do
+    it 'removes the instance_security_dashboard gate' do
+      actors_instance_security_dashboard_feature = feature_gates.create!(feature_key: :instance_security_dashboard, key: :actors, value: 'Project:1')
+      instance_security_dashboard_feature = feature_gates.create!(feature_key: :instance_security_dashboard, key: :boolean, value: 'false')
+
+      migration.down
+
+      expect { instance_security_dashboard_feature.reload }.to raise_error(ActiveRecord::RecordNotFound)
+      expect(actors_instance_security_dashboard_feature.reload).to be_present
+    end
+  end
+end