Automatic merge of gitlab-org/gitlab master

app/assets/javascripts/environments/components/environment_item.vue

 <script>
-import { GlDropdown, GlTooltipDirective, GlIcon, GlLink, GlSprintf } from '@gitlab/ui';
+import { GlDropdown, GlTooltipDirective, GlIcon, GlLink, GlSprintf, GlBadge } from '@gitlab/ui';
 import { isEmpty } from 'lodash';
 import { convertObjectPropsToCamelCase } from '~/lib/utils/common_utils';
 import { __, s__, sprintf } from '~/locale';
@@ -30,6 +30,7 @@ export default {
     CommitComponent,
     ExternalUrlComponent,
     GlDropdown,
+    GlBadge,
     GlIcon,
     GlLink,
     GlSprintf,
@@ -621,9 +622,9 @@ export default {
           <span v-if="model.size === 1">{{ model.name }}</span>
           <span v-else>{{ model.name_without_type }}</span>
         </a>
-        <span v-if="isProtected" class="badge badge-success">
-          {{ s__('Environments|protected') }}
-        </span>
+        <gl-badge v-if="isProtected" variant="success">{{
+          s__('Environments|protected')
+        }}</gl-badge>
       </span>
       <span
         v-else
@@ -639,7 +640,7 @@ export default {
 
         <span> {{ model.folderName }} </span>
 
-        <span class="badge badge-pill"> {{ model.size }} </span>
+        <gl-badge>{{ model.size }}</gl-badge>
       </span>
     </div>
 

app/views/admin/users/projects.html.haml

@@ -5,7 +5,7 @@
 
 - if @user.groups.any?
   .card
-    .card-header= _('Group projects')
+    .card-header= _('Groups')
     %ul.hover-list
       - @user.group_members.includes(:source).each do |group_member| # rubocop: disable CodeReuse/ActiveRecord
         - group = group_member.group

doc/policy/alpha-beta-support.md

@@ -4,6 +4,8 @@ group: Distribution
 info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments
 ---
 
+<!-- any changes made to this page should be reflected in https://about.gitlab.com/support/statement-of-support.html#alpha--beta-features and https://about.gitlab.com/handbook/product/gitlab-the-product/#alpha-beta-ga -->
+
 # Support for Alpha, Beta, and Generally Available Features **(PREMIUM)**
 
 Some GitLab features are released as [Alpha or Beta versions](https://about.gitlab.com/support/statement-of-support.html#alpha--beta-features) which are not fully supported. All other features are considered to be Generally Available (GA).
@@ -30,7 +32,7 @@ Characteristics of beta features:
 - Features and functions are not likely to change.
 - Data loss is not likely.
 
-Your Support Contract provides **best-efforts** support for Beta features, with the expectation that issues will require extra time and assistance from development to troubleshoot.
+Your Support Contract provides **commercially-reasonable effort** support for Beta features, with the expectation that issues will require extra time and assistance from development to troubleshoot.
 
 ## Generally Available (GA)
 

doc/raketasks/backup_restore.md

@@ -1305,9 +1305,11 @@ If you're using backup restore procedures, you may encounter the following
 warning messages:
 
 ```plaintext
-psql:/var/opt/gitlab/backups/db/database.sql:22: ERROR:  must be owner of extension plpgsql
-psql:/var/opt/gitlab/backups/db/database.sql:2931: WARNING:  no privileges could be revoked for "public" (two occurrences)
-psql:/var/opt/gitlab/backups/db/database.sql:2933: WARNING:  no privileges were granted for "public" (two occurrences)
+ERROR: must be owner of extension pg_trgm
+ERROR: must be owner of extension btree_gist
+ERROR: must be owner of extension plpgsql
+WARNING:  no privileges could be revoked for "public" (two occurrences)
+WARNING:  no privileges were granted for "public" (two occurrences)
 ```
 
 Be advised that the backup is successfully restored in spite of these warning

ee/app/assets/javascripts/security_dashboard/components/shared/filters/scanner_filter.vue

@@ -3,6 +3,7 @@ import { GlDropdownDivider, GlDropdownItem, GlTruncate } from '@gitlab/ui';
 import { union, without, get, set, keyBy } from 'lodash';
 import { DEFAULT_SCANNER, SCANNER_ID_PREFIX } from 'ee/security_dashboard/constants';
 import { createScannerOption } from 'ee/security_dashboard/helpers';
+import { REPORT_TYPE_CLUSTER_IMAGE_SCANNING } from '~/vue_shared/security_reports/constants';
 import FilterBody from './filter_body.vue';
 import FilterItem from './filter_item.vue';
 import SimpleFilter from './simple_filter.vue';
@@ -53,7 +54,11 @@ export default {
       const options = keyBy(this.filter.options, 'reportType');
       const groups = { GitLab: options };
 
-      this.scanners.forEach((scanner) => {
+      const scanners = this.scanners.filter(
+        (x) => x.report_type.toLowerCase() !== REPORT_TYPE_CLUSTER_IMAGE_SCANNING,
+      );
+
+      scanners.forEach((scanner) => {
         const vendor = scanner.vendor || DEFAULT_SCANNER; // Default to GitLab if there's no vendor.
         const reportType = scanner.report_type;
         const id = `${vendor}.${reportType}`;

ee/spec/frontend/security_dashboard/components/shared/filters/scanner_filter_spec.js

@@ -5,7 +5,7 @@ import VueRouter from 'vue-router';
 import FilterItem from 'ee/security_dashboard/components/shared/filters/filter_item.vue';
 import ScannerFilter from 'ee/security_dashboard/components/shared/filters/scanner_filter.vue';
 import { DEFAULT_SCANNER, SCANNER_ID_PREFIX } from 'ee/security_dashboard/constants';
-import { vendorScannerFilter } from 'ee/security_dashboard/helpers';
+import { vendorScannerFilterNoClusterImage } from 'ee/security_dashboard/helpers';
 
 const localVue = createLocalVue();
 localVue.use(VueRouter);
@@ -41,7 +41,7 @@ describe('Scanner Filter component', () => {
   let filter;
 
   const createWrapper = ({ scanners = customScanners } = {}) => {
-    filter = cloneDeep(vendorScannerFilter);
+    filter = cloneDeep(vendorScannerFilterNoClusterImage);
 
     wrapper = shallowMount(ScannerFilter, {
       localVue,
@@ -125,7 +125,7 @@ describe('Scanner Filter component', () => {
 
     it('emits filter-changed event with expected data for selected options', async () => {
       const ids = ['GitLab.SAST', 'Custom.SAST', 'GitLab.API_FUZZING', 'GitLab.COVERAGE_FUZZING'];
-      router.replace({ query: { [vendorScannerFilter.id]: ids } });
+      router.replace({ query: { [vendorScannerFilterNoClusterImage.id]: ids } });
       const selectedScanners = customScanners.filter((x) =>
         ids.includes(`${x.vendor}.${x.report_type}`),
       );

lib/gitlab/metrics/exporter/base_exporter.rb

@@ -39,13 +39,8 @@ module Gitlab
 
           @server = ::WEBrick::HTTPServer.new(
             Port: settings.port, BindAddress: settings.address,
-            Logger: logger, AccessLog: access_log)
-          server.mount_proc '/readiness' do |req, res|
-            render_probe(readiness_probe, req, res)
-          end
-          server.mount_proc '/liveness' do |req, res|
-            render_probe(liveness_probe, req, res)
-          end
+            Logger: logger, AccessLog: access_log
+          )
           server.mount '/', Rack::Handler::WEBrick, rack_app
 
           true
@@ -72,8 +67,14 @@ module Gitlab
         end
 
         def rack_app
+          readiness = readiness_probe
+          liveness = liveness_probe
+          pid = thread_name
+
           Rack::Builder.app do
             use Rack::Deflater
+            use Gitlab::Metrics::Exporter::MetricsMiddleware, pid
+            use Gitlab::Metrics::Exporter::HealthChecksMiddleware, readiness, liveness
             use ::Prometheus::Client::Rack::Exporter if ::Gitlab::Metrics.metrics_folder_present?
             run -> (env) { [404, {}, ['']] }
           end
@@ -86,14 +87,6 @@ module Gitlab
         def liveness_probe
           ::Gitlab::HealthChecks::Probes::Collection.new
         end
-
-        def render_probe(probe, req, res)
-          result = probe.execute
-
-          res.status = result.http_status
-          res.content_type = 'application/json; charset=utf-8'
-          res.body = result.json.to_json
-        end
       end
     end
   end

lib/gitlab/metrics/exporter/health_checks_middleware.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Metrics
+    module Exporter
+      class HealthChecksMiddleware
+        def initialize(app, readiness_probe, liveness_probe)
+          @app = app
+          @readiness_probe = readiness_probe
+          @liveness_probe = liveness_probe
+        end
+
+        def call(env)
+          case env['PATH_INFO']
+          when '/readiness' then render_probe(@readiness_probe)
+          when '/liveness' then render_probe(@liveness_probe)
+          else @app.call(env)
+          end
+        end
+
+        private
+
+        def render_probe(probe)
+          result = probe.execute
+
+          [
+            result.http_status,
+            { 'Content-Type' => 'application/json; charset=utf-8' },
+            [result.json.to_json]
+          ]
+        end
+      end
+    end
+  end
+end

lib/gitlab/metrics/exporter/metrics_middleware.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Metrics
+    module Exporter
+      class MetricsMiddleware
+        def initialize(app, pid)
+          @app = app
+          default_labels = {
+            pid: pid
+          }
+          @requests_total = Gitlab::Metrics.counter(
+            :exporter_http_requests_total, 'Total number of HTTP requests', default_labels
+          )
+          @request_durations = Gitlab::Metrics.histogram(
+            :exporter_http_request_duration_seconds,
+            'HTTP request duration histogram (seconds)',
+            default_labels,
+            [0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10]
+          )
+        end
+
+        def call(env)
+          start = Gitlab::Metrics::System.monotonic_time
+          @app.call(env).tap do |response|
+            duration = Gitlab::Metrics::System.monotonic_time - start
+
+            labels = {
+              method: env['REQUEST_METHOD'].downcase,
+              path:   env['PATH_INFO'].to_s,
+              code:   response.first.to_s
+            }
+
+            @requests_total.increment(labels)
+            @request_durations.observe(labels, duration)
+          end
+        end
+      end
+    end
+  end
+end

locale/gitlab.pot

@@ -16787,9 +16787,6 @@ msgstr ""
 msgid "Group project URLs are prefixed with the group namespace"
 msgstr ""
 
-msgid "Group projects"
-msgstr ""
-
 msgid "Group requires separate account"
 msgstr ""
 

metrics_server/dependencies.rb

@@ -24,6 +24,8 @@ require_relative '../lib/gitlab/metrics/samplers/base_sampler'
 require_relative '../lib/gitlab/metrics/samplers/ruby_sampler'
 require_relative '../lib/gitlab/metrics/exporter/base_exporter'
 require_relative '../lib/gitlab/metrics/exporter/sidekiq_exporter'
+require_relative '../lib/gitlab/metrics/exporter/metrics_middleware'
+require_relative '../lib/gitlab/metrics/exporter/health_checks_middleware'
 require_relative '../lib/gitlab/health_checks/probes/collection'
 require_relative '../lib/gitlab/health_checks/probes/status'
 require_relative '../lib/gitlab/process_management'

package.json

@@ -133,7 +133,7 @@
     "fast-mersenne-twister": "1.0.2",
     "file-loader": "^6.2.0",
     "fuzzaldrin-plus": "^0.6.0",
-    "graphql": "^15.4.0",
+    "graphql": "^15.7.2",
     "graphql-tag": "^2.11.0",
     "highlight.js": "^11.3.1",
     "immer": "^7.0.7",
@@ -209,7 +209,7 @@
     "@babel/plugin-transform-modules-commonjs": "^7.10.1",
     "@gitlab/eslint-plugin": "10.0.0",
     "@gitlab/stylelint-config": "2.6.0",
-    "@graphql-eslint/eslint-plugin": "2.3.0",
+    "@graphql-eslint/eslint-plugin": "3.0.0",
     "@testing-library/dom": "^7.16.2",
     "@vue/test-utils": "1.2.0",
     "acorn": "^6.3.0",

spec/commands/metrics_server/metrics_server_spec.rb

@@ -23,6 +23,8 @@ RSpec.describe 'bin/metrics-server', :aggregate_failures do
   end
 
   context 'with a running server' do
+    let(:metrics_dir) { Dir.mktmpdir }
+
     before do
       # We need to send a request to localhost
       WebMock.allow_net_connect!
@@ -33,7 +35,8 @@ RSpec.describe 'bin/metrics-server', :aggregate_failures do
       env = {
         'GITLAB_CONFIG' => config_file.path,
         'METRICS_SERVER_TARGET' => 'sidekiq',
-        'WIPE_METRICS_DIR' => '1'
+        'WIPE_METRICS_DIR' => '1',
+        'prometheus_multiproc_dir' => metrics_dir
       }
       @pid = Process.spawn(env, 'bin/metrics-server', pgroup: true)
     end
@@ -55,6 +58,7 @@ RSpec.describe 'bin/metrics-server', :aggregate_failures do
       # 'No such process' means the process died before
     ensure
       config_file.unlink
+      FileUtils.rm_rf(metrics_dir, secure: true)
     end
 
     it 'serves /metrics endpoint' do

spec/features/admin/users/users_spec.rb

@@ -462,9 +462,9 @@ RSpec.describe 'Admin::Users' do
       visit projects_admin_user_path(user)
     end
 
-    it 'lists group projects' do
+    it 'lists groups' do
       within(:css, '.gl-mb-3 + .card') do
-        expect(page).to have_content 'Group projects'
+        expect(page).to have_content 'Groups'
         expect(page).to have_link group.name, href: admin_group_path(group)
       end
     end

spec/lib/gitlab/metrics/exporter/base_exporter_spec.rb

@@ -111,6 +111,18 @@ RSpec.describe Gitlab::Metrics::Exporter::BaseExporter do
   describe 'request handling' do
     using RSpec::Parameterized::TableSyntax
 
+    let(:fake_collector) do
+      Class.new do
+        def initialize(app, ...)
+          @app = app
+        end
+
+        def call(env)
+          @app.call(env)
+        end
+      end
+    end
+
     where(:method_class, :path, :http_status) do
       Net::HTTP::Get | '/metrics' | 200
       Net::HTTP::Get | '/liveness' | 200
@@ -123,6 +135,8 @@ RSpec.describe Gitlab::Metrics::Exporter::BaseExporter do
       allow(settings).to receive(:port).and_return(0)
       allow(settings).to receive(:address).and_return('127.0.0.1')
 
+      stub_const('Gitlab::Metrics::Exporter::MetricsMiddleware', fake_collector)
+
       # We want to wrap original method
       # and run handling of requests
       # in separate thread
@@ -134,8 +148,6 @@ RSpec.describe Gitlab::Metrics::Exporter::BaseExporter do
           # is raised as we close listeners
         end
       end
-
-      exporter.start.join
     end
 
     after do
@@ -146,12 +158,25 @@ RSpec.describe Gitlab::Metrics::Exporter::BaseExporter do
       let(:config) { exporter.server.config }
       let(:request) { method_class.new(path) }
 
-      it 'responds with proper http_status' do
+      subject(:response) do
         http = Net::HTTP.new(config[:BindAddress], config[:Port])
-        response = http.request(request)
+        http.request(request)
+      end
+
+      it 'responds with proper http_status' do
+        exporter.start.join
 
         expect(response.code).to eq(http_status.to_s)
       end
+
+      it 'collects request metrics' do
+        expect_next_instance_of(fake_collector) do |instance|
+          expect(instance).to receive(:call).and_call_original
+        end
+
+        exporter.start.join
+        response
+      end
     end
   end
 

spec/lib/gitlab/metrics/exporter/health_checks_middleware_spec.rb

+# frozen_string_literal: true
+
+require 'fast_spec_helper'
+
+RSpec.describe Gitlab::Metrics::Exporter::HealthChecksMiddleware do
+  let(:app) { double(:app) }
+  let(:env) { { 'PATH_INFO' => path } }
+
+  let(:readiness_probe) { double(:readiness_probe) }
+  let(:liveness_probe) { double(:liveness_probe) }
+  let(:probe_result) { Gitlab::HealthChecks::Probes::Status.new(200, { status: 'ok' }) }
+
+  subject(:middleware) { described_class.new(app, readiness_probe, liveness_probe) }
+
+  describe '#call' do
+    context 'handling /readiness requests' do
+      let(:path) { '/readiness' }
+
+      it 'handles the request' do
+        expect(readiness_probe).to receive(:execute).and_return(probe_result)
+
+        response = middleware.call(env)
+
+        expect(response).to eq([200, { 'Content-Type' => 'application/json; charset=utf-8' }, ['{"status":"ok"}']])
+      end
+    end
+
+    context 'handling /liveness requests' do
+      let(:path) { '/liveness' }
+
+      it 'handles the request' do
+        expect(liveness_probe).to receive(:execute).and_return(probe_result)
+
+        response = middleware.call(env)
+
+        expect(response).to eq([200, { 'Content-Type' => 'application/json; charset=utf-8' }, ['{"status":"ok"}']])
+      end
+    end
+
+    context 'handling other requests' do
+      let(:path) { '/other_path' }
+
+      it 'forwards them to the next middleware' do
+        expect(app).to receive(:call).with(env).and_return([201, {}, []])
+
+        response = middleware.call(env)
+
+        expect(response).to eq([201, {}, []])
+      end
+    end
+  end
+end

spec/lib/gitlab/metrics/exporter/metrics_middleware_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Metrics::Exporter::MetricsMiddleware do
+  let(:app) { double(:app) }
+  let(:pid) { 'fake_exporter' }
+  let(:env) { { 'PATH_INFO' => '/path', 'REQUEST_METHOD' => 'GET' } }
+
+  subject(:middleware) { described_class.new(app, pid) }
+
+  def metric(name, method, path, status)
+    metric = ::Prometheus::Client.registry.get(name)
+    return unless metric
+
+    values = metric.values.transform_keys { |k| k.slice(:method, :path, :pid, :code) }
+    values[{ method: method, path: path, pid: pid, code: status.to_s }]&.get
+  end
+
+  before do
+    expect(app).to receive(:call).with(env).and_return([200, {}, []])
+  end
+
+  describe '#call', :prometheus do
+    it 'records a total requests metric' do
+      response = middleware.call(env)
+
+      expect(response).to eq([200, {}, []])
+      expect(metric(:exporter_http_requests_total, 'get', '/path', 200)).to eq(1.0)
+    end
+
+    it 'records a request duration histogram' do
+      response = middleware.call(env)
+
+      expect(response).to eq([200, {}, []])
+      expect(metric(:exporter_http_request_duration_seconds, 'get', '/path', 200)).to be_a(Hash)
+    end
+  end
+end

spec/lib/gitlab/metrics/exporter/web_exporter_spec.rb

@@ -24,14 +24,14 @@ RSpec.describe Gitlab::Metrics::Exporter::WebExporter do
     exporter.stop
   end
 
-  context 'when running server' do
+  context 'when running server', :prometheus do
     it 'readiness probe returns succesful status' do
       expect(readiness_probe.http_status).to eq(200)
       expect(readiness_probe.json).to include(status: 'ok')
       expect(readiness_probe.json).to include('web_exporter' => [{ 'status': 'ok' }])
     end
 
-    it 'initializes request metrics', :prometheus do
+    it 'initializes request metrics' do
       expect(Gitlab::Metrics::RailsSlis).to receive(:initialize_request_slis_if_needed!).and_call_original
 
       http = Net::HTTP.new(exporter.server.config[:BindAddress], exporter.server.config[:Port])
@@ -42,7 +42,7 @@ RSpec.describe Gitlab::Metrics::Exporter::WebExporter do
   end
 
   describe '#mark_as_not_running!' do
-    it 'readiness probe returns a failure status' do
+    it 'readiness probe returns a failure status', :prometheus do
       exporter.mark_as_not_running!
 
       expect(readiness_probe.http_status).to eq(503)