Automatic merge of gitlab-org/gitlab master

.gitlab/ci/docs.gitlab-ci.yml

@@ -10,17 +10,18 @@
     # because some repos are private and CI_JOB_TOKEN cannot access files.
     # See https://gitlab.com/gitlab-org/gitlab/issues/191273
     GIT_DEPTH: 1
+    # By default, deploy the Review App using the `master` branch of the `gitlab-org/gitlab-docs` project
+    DOCS_BRANCH: master
   environment:
-    name: review-docs/$DOCS_GITLAB_REPO_SUFFIX-$CI_MERGE_REQUEST_IID
+    name: review-docs/mr-${CI_MERGE_REQUEST_IID}
     # DOCS_REVIEW_APPS_DOMAIN and DOCS_GITLAB_REPO_SUFFIX are CI variables
     # Discussion: https://gitlab.com/gitlab-org/gitlab-foss/merge_requests/14236/diffs#note_40140693
     auto_stop_in: 2 weeks
-    url: http://docs-preview-$DOCS_GITLAB_REPO_SUFFIX-$CI_MERGE_REQUEST_IID.$DOCS_REVIEW_APPS_DOMAIN/$DOCS_GITLAB_REPO_SUFFIX
+    url: http://${DOCS_BRANCH}-${DOCS_GITLAB_REPO_SUFFIX}-${CI_MERGE_REQUEST_IID}.${DOCS_REVIEW_APPS_DOMAIN}/${DOCS_GITLAB_REPO_SUFFIX}
     on_stop: review-docs-cleanup
   before_script:
-    - apk add --update openssl
-    - gem install httparty --no-document --version 0.17.3
-    - gem install gitlab --no-document --version 4.13.0
+    - source ./scripts/utils.sh
+    - install_gitlab_gem
 
 # Always trigger a docs build in gitlab-docs only on docs-only branches.
 # Useful to preview the docs changes live.
@@ -33,7 +34,7 @@ review-docs-deploy:
 review-docs-cleanup:
   extends: .review-docs
   environment:
-    name: review-docs/$DOCS_GITLAB_REPO_SUFFIX-$CI_MERGE_REQUEST_IID
+    name: review-docs/mr-${CI_MERGE_REQUEST_IID}
     action: stop
   script:
     - ./scripts/trigger-build docs cleanup

doc/api/graphql/reference/index.md

@@ -4709,7 +4709,7 @@ An edge in a connection.
 | `alertManagementIntegrations` | [`AlertManagementIntegrationConnection`](#alertmanagementintegrationconnection) | Integrations which can receive alerts for the project. |
 | `alertManagementPayloadFields` | [`[AlertManagementPayloadAlertField!]`](#alertmanagementpayloadalertfield) | Extract alert fields from payload for custom mapping. |
 | `allowMergeOnSkippedPipeline` | [`Boolean`](#boolean) | If `only_allow_merge_if_pipeline_succeeds` is true, indicates if merge requests of the project can also be merged with skipped jobs. |
-| `apiFuzzingCiConfiguration` | [`ApiFuzzingCiConfiguration`](#apifuzzingciconfiguration) | API fuzzing configuration for the project. Available only when feature flag `api_fuzzing_configuration_ui` is enabled. |
+| `apiFuzzingCiConfiguration` | [`ApiFuzzingCiConfiguration`](#apifuzzingciconfiguration) | API fuzzing configuration for the project. Null unless feature flag `api_fuzzing_configuration_ui` is enabled. |
 | `archived` | [`Boolean`](#boolean) | Indicates the archived status of the project. |
 | `autocloseReferencedIssues` | [`Boolean`](#boolean) | Indicates if issues referenced by merge requests and commits within the default branch are closed automatically. |
 | `avatarUrl` | [`String`](#string) | URL to avatar image file of the project. |

doc/development/documentation/index.md

@@ -387,11 +387,13 @@ To preview your changes to documentation locally, follow this
 The live preview is currently enabled for the following projects:
 
 - [`gitlab`](https://gitlab.com/gitlab-org/gitlab)
+- [`omnibus-gitlab`](https://gitlab.com/gitlab-org/omnibus-gitlab)
 - [`gitlab-runner`](https://gitlab.com/gitlab-org/gitlab-runner)
 
 If your merge request has docs changes, you can use the manual `review-docs-deploy` job
 to deploy the docs review app for your merge request.
-You need at least Maintainer permissions to be able to run it.
+As the job creates a pipeline in `gitlab-org/gitlab-docs`, you need at least Maintainer
+permission in the `gitlab-org/gitlab-docs` project in order to successfully run the job.
 
 ![Manual trigger a docs build](img/manual_build_docs.png)
 
@@ -399,11 +401,6 @@ You must push a branch to those repositories, as it doesn't work for forks.
 
 The `review-docs-deploy*` job:
 
-1. Creates a new branch in the [`gitlab-docs`](https://gitlab.com/gitlab-org/gitlab-docs)
-   project named after the scheme: `docs-preview-$DOCS_GITLAB_REPO_SUFFIX-$CI_MERGE_REQUEST_IID`,
-   where `DOCS_GITLAB_REPO_SUFFIX` is the suffix for each product, e.g, `ee` for
-   EE, `omnibus` for Omnibus GitLab, etc, and `CI_MERGE_REQUEST_IID` is the ID
-   of the respective merge request.
 1. Triggers a cross project pipeline and build the docs site with your changes.
 
 In case the review app URL returns 404, this means that either the site is not
@@ -412,11 +409,6 @@ minutes and it should appear online, otherwise you can check the status of the
 remote pipeline from the link in the merge request's job output.
 If the pipeline failed or got stuck, drop a line in the `#docs` chat channel.
 
-Make sure that you always delete the branch of the merge request you were
-working on. If you don't, the remote docs branch isn't removed either,
-and the server where the Review Apps are hosted can eventually run out of
-disk space.
-
 NOTE:
 Someone with no merge rights to the GitLab projects (think of forks from
 contributors) cannot run the manual job. In that case, you can

doc/development/usage_ping/dictionary.md

@@ -6416,30 +6416,6 @@ Status: `data_available`
 
 Tiers: `premium`, `ultimate`
 
-### `g_project_management_epic_created_monthly`
-
-Count of MAU creating epics
-
-[YAML definition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/config/metrics/counts_28d/20210305144719_g_product_planning_epic_created_monthly.yml)
-
-Group: `group::product planning`
-
-Status: `implemented`
-
-Tiers: `premium`, `ultimate`
-
-### `g_project_management_epic_created_weekly`
-
-Count of WAU creating epics
-
-[YAML definition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/config/metrics/counts_7d/20210305145820_g_product_planning_epic_created_weekly.yml)
-
-Group: `group::product planning`
-
-Status: `implemented`
-
-Tiers: `premium`, `ultimate`
-
 ### `geo_enabled`
 
 Is Geo enabled?
@@ -9812,6 +9788,30 @@ Status: `implemented`
 
 Tiers: `free`, `premium`, `ultimate`
 
+### `redis_hll_counters.epics_usage.g_project_management_epic_created_monthly`
+
+Count of MAU creating epics
+
+[YAML definition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/config/metrics/counts_28d/20210305144719_g_product_planning_epic_created_monthly.yml)
+
+Group: `group::product planning`
+
+Status: `implemented`
+
+Tiers: `premium`, `ultimate`
+
+### `redis_hll_counters.epics_usage.g_project_management_epic_created_weekly`
+
+Count of WAU creating epics
+
+[YAML definition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/config/metrics/counts_7d/20210305145820_g_product_planning_epic_created_weekly.yml)
+
+Group: `group::product planning`
+
+Status: `implemented`
+
+Tiers: `premium`, `ultimate`
+
 ### `redis_hll_counters.ide_edit.g_edit_by_sfe_monthly`
 
 Missing description

doc/development/usage_ping/metrics_dictionary.md

@@ -32,7 +32,7 @@ Each metric is defined in a separate YAML file consisting of a number of fields:
 | `product_stage`     | no       | The [stage](https://gitlab.com/gitlab-com/www-gitlab-com/blob/master/data/stages.yml) for the metric. |
 | `product_group`     | yes      | The [group](https://gitlab.com/gitlab-com/www-gitlab-com/blob/master/data/stages.yml) that owns the metric. |
 | `product_category`  | no       | The [product category](https://gitlab.com/gitlab-com/www-gitlab-com/blob/master/data/categories.yml) for the metric. |
-| `value_type`        | yes      | `string`; one of `string`, `number`, `boolean`, `object`.                                                     |
+| `value_type`        | yes      | `string`; one of [`string`, `number`, `boolean`, `object`](https://json-schema.org/understanding-json-schema/reference/type.html).                                                     |
 | `status`            | yes      | `string`; status of the metric, may be set to `data_available`, `planned`, `in_progress`, `implemented`, `not_used`, `deprecated` |
 | `time_frame`        | yes      | `string`; may be set to a value like `7d`, `28d`, `all`, `none`. |
 | `data_source`       | yes      | `string`; may be set to a value like `database`, `redis`, `redis_hll`, `prometheus`, `ruby`. |

ee/app/graphql/ee/types/project_type.rb

@@ -133,8 +133,8 @@ module EE
         field :api_fuzzing_ci_configuration,
               ::Types::AppSec::Fuzzing::Api::CiConfigurationType,
               null: true,
-              description: 'API fuzzing configuration for the project.',
-              feature_flag: :api_fuzzing_configuration_ui
+              description: 'API fuzzing configuration for the project. '\
+                           'Null unless feature flag `api_fuzzing_configuration_ui` is enabled.'
 
         field :push_rules,
               ::Types::PushRulesType,
@@ -144,7 +144,8 @@ module EE
       end
 
       def api_fuzzing_ci_configuration
-        return unless Ability.allowed?(current_user, :read_vulnerability, object)
+        return unless ::Feature.enabled?(:api_fuzzing_configuration_ui, object, default_enabled: :yaml) && \
+                      Ability.allowed?(current_user, :read_vulnerability, object)
 
         configuration = ::AppSec::Fuzzing::Api::CiConfiguration.new(project: object)
 

ee/app/models/ee/project.rb

@@ -197,10 +197,6 @@ module EE
 
       delegate :ci_minutes_quota, to: :shared_runners_limit_namespace
 
-      delegate :last_update_succeeded?, :last_update_failed?,
-        :ever_updated_successfully?, :hard_failed?,
-        to: :import_state, prefix: :mirror, allow_nil: true
-
       delegate :merge_pipelines_enabled, :merge_pipelines_enabled=, :merge_pipelines_enabled?, :merge_pipelines_were_disabled?, to: :ci_cd_settings
       delegate :merge_trains_enabled, :merge_trains_enabled=, :merge_trains_enabled?, to: :ci_cd_settings
 
@@ -256,6 +252,22 @@ module EE
       end
     end
 
+    def mirror_last_update_succeeded?
+      !!import_state&.last_update_succeeded?
+    end
+
+    def mirror_last_update_failed?
+      !!import_state&.last_update_failed?
+    end
+
+    def mirror_ever_updated_successfully?
+      !!import_state&.ever_updated_successfully?
+    end
+
+    def mirror_hard_failed?
+      !!import_state&.hard_failed?
+    end
+
     class_methods do
       extend ::Gitlab::Utils::Override
 

ee/changelogs/unreleased/299465-default-enable-api-fuzzing-configuration-ui-ff.yml

+---
+title: Add a form to generate API Fuzzing configuration YAML snippets
+merge_request: 56548
+author:
+type: added

ee/config/feature_flags/development/api_fuzzing_configuration_ui.yml

@@ -5,4 +5,4 @@ rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/299234
 milestone: '13.9'
 type: development
 group: group::fuzz testing
-default_enabled: false
+default_enabled: true

ee/config/metrics/counts_28d/20210305144719_g_product_planning_epic_created_monthly.yml

@@ -2,7 +2,7 @@
 # Name of this metric contains g_project_management prefix
 # because we are using the same slot from issue_tracking to
 # allow data aggregation.
-key_path: g_project_management_epic_created_monthly
+key_path: redis_hll_counters.epics_usage.g_project_management_epic_created_monthly
 description: Count of MAU creating epics
 product_section: dev
 product_stage: plan

ee/config/metrics/counts_7d/20210305145820_g_product_planning_epic_created_weekly.yml

@@ -2,7 +2,7 @@
 # Name of this metric contains g_project_management prefix
 # because we are using the same slot from issue_tracking to
 # allow data aggregation.
-key_path: g_project_management_epic_created_weekly
+key_path: redis_hll_counters.epics_usage.g_project_management_epic_created_weekly
 description: Count of WAU creating epics
 product_section: dev
 product_stage: plan

ee/spec/models/project_spec.rb

@@ -125,6 +125,46 @@ RSpec.describe Project do
       end
     end
 
+    context 'import_state dependant predicate method' do
+      shared_examples 'returns expected values' do
+        context 'when project lacks a import_state relation' do
+          it 'returns false' do
+            expect(project.send("mirror_#{method}")).to be_falsey
+          end
+        end
+
+        context 'when project has a import_state relation' do
+          before do
+            create(:import_state, project: project)
+          end
+
+          it 'accesses the value from the import_state' do
+            expect(project.import_state).to receive(method)
+
+            project.send("mirror_#{method}")
+          end
+        end
+      end
+
+      describe '#mirror_last_update_succeeded?' do
+        it_behaves_like 'returns expected values' do
+          let(:method) { "last_update_succeeded?" }
+        end
+      end
+
+      describe '#mirror_last_update_failed?' do
+        it_behaves_like 'returns expected values' do
+          let(:method) { "last_update_failed?" }
+        end
+      end
+
+      describe '#mirror_ever_updated_successfully?' do
+        it_behaves_like 'returns expected values' do
+          let(:method) { "ever_updated_successfully?" }
+        end
+      end
+    end
+
     describe 'approval_rules association' do
       let_it_be(:rule, reload: true) { create(:approval_project_rule) }
       let(:project) { rule.project }

ee/spec/requests/api/graphql/app_sec/fuzzing/api/ci_configuration_type_spec.rb

@@ -74,7 +74,7 @@ RSpec.describe 'Query.project(fullPath).apiFuzzingCiConfiguration' do
         stub_licensed_features(security_dashboard: false)
       end
 
-      it 'returns null' do
+      it 'returns nil' do
         post_graphql(query, current_user: user)
 
         expect(response).to have_gitlab_http_status(:ok)
@@ -90,13 +90,12 @@ RSpec.describe 'Query.project(fullPath).apiFuzzingCiConfiguration' do
       stub_feature_flags(api_fuzzing_configuration_ui: false)
     end
 
-    it 'errors' do
+    it 'returns nil' do
       post_graphql(query, current_user: user)
 
       expect(response).to have_gitlab_http_status(:ok)
-      expect(graphql_errors.first['message']).to eq(
-        "Field 'apiFuzzingCiConfiguration' doesn't exist on type 'Project'"
-      )
+      fuzzing_config = graphql_data.dig('project', 'apiFuzzingCiConfiguration')
+      expect(fuzzing_config).to be_nil
     end
   end
 end

lib/gitlab/usage_data_counters/known_events/common.yml

@@ -444,13 +444,3 @@
   redis_slot: pipeline_authoring
   aggregation: weekly
   feature_flag: usage_data_o_pipeline_authoring_unique_users_pushing_mr_ciconfigfile
-# Epic events
-#
-# We are using the same slot of issue events 'project_management' for
-# epic events to allow data aggregation.
-# More information in: https://gitlab.com/gitlab-org/gitlab/-/issues/322405
-- name: g_project_management_epic_created
-  category: epics_usage
-  redis_slot: project_management
-  aggregation: daily
-  feature_flag: track_epics_activity

lib/gitlab/usage_data_counters/known_events/epic_events.yml

+# Epic events
+#
+# We are using the same slot of issue events 'project_management' for
+# epic events to allow data aggregation.
+# More information in: https://gitlab.com/gitlab-org/gitlab/-/issues/322405
+- name: g_project_management_epic_created
+  category: epics_usage
+  redis_slot: project_management
+  aggregation: daily
+  feature_flag: track_epics_activity

scripts/trigger-build

@@ -214,15 +214,7 @@ module Trigger
     => If something doesn't work, drop a line in the #docs chat channel.
     MSG
 
-    # Create a remote branch in gitlab-docs and immediately cancel the pipeline
-    # to avoid race conditions, since a triggered pipeline will also run right
-    # after the branch creation. This only happens the very first time a branch
-    # is created and will be skipped in subsequent runs. Read more in
-    # https://gitlab.com/gitlab-org/gitlab-docs/issues/154.
-    #
     def deploy!
-      create_remote_branch!
-      cancel_latest_pipeline!
       invoke!.wait!
       display_success_message
     end
@@ -231,31 +223,47 @@ module Trigger
     # Remove a remote branch in gitlab-docs.
     #
     def cleanup!
-      gitlab_client(:downstream).delete_branch(downstream_project_path, ref)
-      puts "=> Remote branch '#{downstream_project_path}' deleted"
+      environment = gitlab_client(:downstream).environments(downstream_project_path, name: downstream_environment).first
+      return unless environment
+
+      environment = gitlab_client(:downstream).stop_environment(downstream_project_path, environment.id)
+      if environment.state == 'stopped'
+        puts "=> Downstream environment '#{downstream_environment}' stopped"
+      else
+        puts "=> Downstream environment '#{downstream_environment}' failed to stop."
+      end
     end
 
     private
 
+    def downstream_environment
+      "review/#{ref}#{review_slug}"
+    end
+
+    # We prepend the `-` here because we cannot use variable substitution in `environment.name`/`environment.url`
+    # Some projects (e.g. `omnibus-gitlab`) use this script for branch pipelines, so we fallback to using `CI_COMMIT_REF_SLUG` for those cases.
+    def review_slug
+      identifier = ENV['CI_MERGE_REQUEST_IID'] || ENV['CI_COMMIT_REF_SLUG']
+
+      "-#{project_slug}-#{identifier}"
+    end
+
     def downstream_project_path
       ENV['DOCS_PROJECT_PATH'] || 'gitlab-org/gitlab-docs'
     end
 
     def ref
-      if ENV['CI_MERGE_REQUEST_IID'].nil?
-        "docs-preview-#{slug}-#{ENV['CI_COMMIT_REF_SLUG']}"
-      else
-        "docs-preview-#{slug}-#{ENV['CI_MERGE_REQUEST_IID']}"
-      end
+      ENV['DOCS_BRANCH'] || 'master'
     end
 
     def extra_variables
       {
-        "BRANCH_#{slug.upcase}" => ENV['CI_COMMIT_REF_NAME']
+        "BRANCH_#{project_slug.upcase}" => ENV['CI_COMMIT_REF_NAME'],
+        "REVIEW_SLUG" => review_slug
       }
     end
 
-    def slug
+    def project_slug
       case ENV['CI_PROJECT_PATH']
       when 'gitlab-org/gitlab-foss'
         'ce'
@@ -270,37 +278,14 @@ module Trigger
       end
     end
 
+    # app_url is the URL of the `gitlab-docs` Review App URL defined in
+    # https://gitlab.com/gitlab-org/gitlab-docs/-/blob/b38038132cf82a24271bbb294dead7c2f529e275/.gitlab-ci.yml#L383
     def app_url
-      "http://#{ref}.#{ENV['DOCS_REVIEW_APPS_DOMAIN']}/#{slug}"
-    end
-
-    def create_remote_branch!
-      gitlab_client(:downstream).create_branch(downstream_project_path, ref, 'master')
-      puts "=> Remote branch '#{ref}' created"
-    rescue Gitlab::Error::BadRequest
-      puts "=> Remote branch '#{ref}' already exists!"
-    end
-
-    def cancel_latest_pipeline!
-      pipelines = nil
-
-      # Wait until the pipeline is started
-      loop do
-        sleep 1
-        puts "=> Waiting for pipeline to start..."
-        pipelines = gitlab_client(:downstream).pipelines(downstream_project_path, { ref: ref })
-        break if pipelines.any?
-      end
-
-      # Get the first pipeline ID which should be the only one for the branch
-      pipeline_id = pipelines.first.id
-
-      # Cancel the pipeline
-      gitlab_client(:downstream).cancel_pipeline(downstream_project_path, pipeline_id)
+      "http://#{ref}#{review_slug}.#{ENV['DOCS_REVIEW_APPS_DOMAIN']}/#{project_slug}"
     end
 
     def display_success_message
-      format(SUCCESS_MESSAGE, app_url: app_url)
+      puts format(SUCCESS_MESSAGE, app_url: app_url)
     end
   end
 

scripts/utils.sh

@@ -32,7 +32,7 @@ function install_api_client_dependencies_with_apt() {
 
 function install_gitlab_gem() {
   gem install httparty --no-document --version 0.18.1
-  gem install gitlab --no-document --version 4.14.1
+  gem install gitlab --no-document --version 4.17.0
 }
 
 function install_tff_gem() {