Commit cd631619 authored by GitLab Bot's avatar GitLab Bot

Add latest changes from gitlab-org/gitlab@master

parent f03a645e
......@@ -12,7 +12,7 @@
}
.log-line {
padding: 1px $gl-padding 1px $job-log-line-padding;
padding: 1px $gl-padding-8 1px $job-log-line-padding;
min-height: $gl-line-height-20;
}
......@@ -20,7 +20,7 @@
color: $gl-text-color-inverted;
padding: 0 $gl-padding-8;
min-width: $job-line-number-width;
margin-left: -$job-line-number-width;
margin-left: -$job-line-number-margin;
padding-right: 1em;
&:hover,
......
......@@ -606,9 +606,10 @@ $blame-blue: #254e77;
*/
$builds-trace-bg: #111;
$job-log-highlight-height: 18px;
$job-log-line-padding: 62px;
$job-line-number-width: 40px;
$job-arrow-margin: 50px;
$job-log-line-padding: 55px;
$job-line-number-width: 50px;
$job-line-number-margin: 43px;
$job-arrow-margin: 55px;
/*
* Commit Page
......
......@@ -128,6 +128,12 @@ module Ci
scope :with_stale_live_trace, -> { with_live_trace.finished_before(12.hours.ago) }
scope :finished_before, -> (date) { finished.where('finished_at < ?', date) }
scope :with_secure_reports_from_options, -> (job_type) { where('options like :job_type', job_type: "%:artifacts:%:reports:%:#{job_type}:%") }
scope :with_secure_reports_from_config_options, -> (job_types) do
joins(:metadata).where("ci_builds_metadata.config_options -> 'artifacts' -> 'reports' ?| array[:job_types]", job_types: job_types)
end
scope :matches_tag_ids, -> (tag_ids) do
matcher = ::ActsAsTaggableOn::Tagging
.where(taggable_type: CommitStatus.name)
......
---
title: Fix error where helper was incorrectly returning `true`
merge_request: 18231
author:
type: fixed
---
title: Update Workhorse to v8.14.0
merge_request: 18391
author:
type: other
......@@ -25,40 +25,43 @@ three Gitaly nodes, `Praefect-Git-1`, `Praefect-Git-2`, and `Praefect-Git-3`.
There should be no knowledge other than with Praefect about the existence of
the `Praefect-Git-X` nodes.
### Enable the daemon
### Setup
Praefect is expected to run on their own host, this means that no other service
other than the support services run on this machine.
In this setup guide, the Gitaly node will be added first, then Praefect, and
lastly we update the GitLab configuration.
Praefect is disabled by default, to enable praefect uncomment the following line
and set it to `true`: `# praefect['enable'] = false'`
#### Gitaly
```ruby
praefect['enable'] = true
```
By default praefect will listen on port `:2305`. It's recommended to enable
prometheus to expose metrics. Uncomment the line so it looks like:
```ruby
praefect['prometheus_listen_addr'] = "localhost:9652"
```
In their own machine, configure the Gitaly server as described in the
[gitaly documentation](index.md#3-gitaly-server-configuration).
Preafect needs at least one storage to store the Git data on. This node should
run Gitaly and should not be listed as storage for GitLab itself, that is, the
only way it receives traffic is through Praefect and it's not listed in the
`git_data_dirs` on any `gitlab.rb` in your GitLab cluster.
#### Praefect
To set the nodes as depicted in the diagram above, the configuration should look
like:
Next, Praefect has to be enabled on its own node. Disable all other services,
and add each Gitaly node that will be connected to Praefect. In the example below,
the Gitaly nodes are named `praefect-git-X`. Note that one node is designated as
primary, by setting the primary to `true`:
```ruby
# /etc/gitlab/gitlab.rb
# Avoid running unnecessary services on the Gitaly server
postgresql['enable'] = false
redis['enable'] = false
nginx['enable'] = false
prometheus['enable'] = false
unicorn['enable'] = false
sidekiq['enable'] = false
gitlab_workhorse['enable'] = false
gitaly['enable'] = false
praefect['enable'] = true
praefect['storage_nodes'] = [
{
'storage' => 'praefect-git-1',
'address' => 'tcp://praefect-git-1.internal',
'primary' => true
}
},
{
'storage' => 'praefect-git-2',
'address' => 'tcp://praefect-git-2.internal'
......@@ -70,10 +73,9 @@ praefect['storage_nodes'] = [
]
```
Save the file, and run `gitlab-ctl reconfigure`. To test if Praefect is running,
you could run `gitlab-ctl status` which should list praefect as being up.
Save the file and [reconfigure Praefect](../restart_gitlab.md#omnibus-gitlab-reconfigure).
### Enable Preafect as storage backend in GitLab
#### GitLab
When Praefect is running, it should be exposed as a storage to GitLab. This
is done through setting the `git_data_dirs`. Assuming the default storage
......
......@@ -12,8 +12,9 @@ content of a previous job. It can be particularly useful when you are
developing software that depends on other libraries which are fetched via the
internet during build time.
If caching is enabled, it's shared between pipelines and jobs by default,
starting from GitLab 9.0.
If caching is enabled, it's shared between pipelines and jobs at the project
level by default, starting from GitLab 9.0. Caches are not shared across
projects.
Make sure you read the [`cache` reference](../yaml/README.md#cache) to learn
how it is defined in `.gitlab-ci.yml`.
......
......@@ -220,6 +220,8 @@ requirements.
1. Working and clean code that is commented where needed.
1. [Unit, integration, and system tests](../testing_guide/index.md) that all pass
on the CI server.
1. Regressions and bugs are covered with tests that reduce the risk of the issue happening
again.
1. Performance/scalability implications have been considered, addressed, and tested.
1. [Documented](../documentation/index.md) in the `/doc` directory.
1. [Changelog entry added](../changelog.md), if necessary.
......
......@@ -47,6 +47,7 @@ POST /internal/allowed
| `protocol` | string | yes | SSH when called from GitLab-shell, HTTP or SSH when called from Gitaly |
| `action` | string | yes | Git command being run (`git-upload-pack`, `git-receive-pack`, `git-upload-archive`) |
| `changes` | string | yes | `<oldrev> <newrev> <refname>` when called from Gitaly, The magic string `_any` when called from GitLab Shell |
| `check_ip` | string | no | Ip adress from which call to GitLab Shell was made |
Example request:
......
......@@ -351,7 +351,7 @@ Add one or more whitelisted IP subnets using CIDR notation in comma separated fo
coming from a different IP address won't be able to access the restricted
content.
Restriction currently applies to UI and API access, Git actions via SSH are not restricted.
Restriction currently applies to UI, API access and Git actions via SSH.
To avoid accidental lock-out, admins and group owners are are able to access
the group regardless of the IP restriction.
......
......@@ -26,20 +26,11 @@ module API
def ee_post_receive_response_hook(response)
# Hook for EE to add messages
end
end
namespace 'internal' do
# Check if git command is allowed for project
#
# Params:
# key_id - ssh key id for Git over SSH
# user_id - user id for Git over HTTP or over SSH in keyless SSH CERT mode
# username - user name for Git over SSH in keyless SSH cert mode
# protocol - Git access protocol being used, e.g. HTTP or SSH
# project - project full_path (not path on disk)
# action - git action (git-upload-pack or git-receive-pack)
# changes - changes as "oldrev newrev ref", see Gitlab::ChangesList
post "/allowed" do
def check_allowed(params)
# This is a separate method so that EE can alter its behaviour more
# easily.
# Stores some Git-specific env thread-safely
env = parse_env
Gitlab::Git::HookEnv.set(gl_repository, env) if project
......@@ -53,11 +44,11 @@ module API
@project ||= access_checker.project
result
rescue Gitlab::GitAccess::UnauthorizedError => e
break response_with_status(code: 401, success: false, message: e.message)
return response_with_status(code: 401, success: false, message: e.message)
rescue Gitlab::GitAccess::TimeoutError => e
break response_with_status(code: 503, success: false, message: e.message)
return response_with_status(code: 503, success: false, message: e.message)
rescue Gitlab::GitAccess::NotFoundError => e
break response_with_status(code: 404, success: false, message: e.message)
return response_with_status(code: 404, success: false, message: e.message)
end
log_user_activity(actor.user)
......@@ -91,6 +82,26 @@ module API
response_with_status(code: 500, success: false, message: UNKNOWN_CHECK_RESULT_ERROR)
end
end
end
namespace 'internal' do
# Check if git command is allowed for project
#
# Params:
# key_id - ssh key id for Git over SSH
# user_id - user id for Git over HTTP or over SSH in keyless SSH CERT mode
# username - user name for Git over SSH in keyless SSH cert mode
# protocol - Git access protocol being used, e.g. HTTP or SSH
# project - project full_path (not path on disk)
# action - git action (git-upload-pack or git-receive-pack)
# changes - changes as "oldrev newrev ref", see Gitlab::ChangesList
# check_ip - optional, only in EE version, may limit access to
# group resources based on its IP restrictions
post "/allowed" do
# It was moved to a separate method so that EE can alter its behaviour more
# easily.
check_allowed(params)
end
# rubocop: disable CodeReuse/ActiveRecord
post "/lfs_authenticate" do
......
......@@ -39,8 +39,8 @@ module Gitlab
while enabled?
begin
restart_sidekiq unless rss_within_range?
sleep(CHECK_INTERVAL_SECONDS)
restart_sidekiq unless rss_within_range?
rescue => e
log_exception(e, __method__)
rescue Exception => e # rubocop:disable Lint/RescueException
......
......@@ -70,10 +70,11 @@ module Gitlab
end
def add_time_keys!(time, payload)
payload['duration'] = time[:duration].round(3)
payload['system_s'] = time[:stime].round(3)
payload['user_s'] = time[:utime].round(3)
payload['child_s'] = time[:ctime].round(3) if time[:ctime] > 0
payload['duration'] = time[:duration].round(6)
# ignore `cpu_s` if the platform does not support Process::CLOCK_THREAD_CPUTIME_ID (time[:cputime] == 0)
# supported OS version can be found at: https://www.rubydoc.info/stdlib/core/2.1.6/Process:clock_gettime
payload['cpu_s'] = time[:cputime].round(6) if time[:cputime] > 0
payload['completed_at'] = Time.now.utc
end
......@@ -98,30 +99,24 @@ module Gitlab
end
def elapsed_by_absolute_time(start)
(Time.now.utc - start).to_f.round(3)
(Time.now.utc - start).to_f.round(6)
end
def elapsed(t0)
t1 = get_time
{
duration: t1[:now] - t0[:now],
stime: t1[:times][:stime] - t0[:times][:stime],
utime: t1[:times][:utime] - t0[:times][:utime],
ctime: ctime(t1[:times]) - ctime(t0[:times])
cputime: t1[:thread_cputime] - t0[:thread_cputime]
}
end
def get_time
{
now: current_time,
times: Process.times
thread_cputime: defined?(Process::CLOCK_THREAD_CPUTIME_ID) ? Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID) : 0
}
end
def ctime(times)
times[:cstime] + times[:cutime]
end
def current_time
Gitlab::Metrics::System.monotonic_time
end
......@@ -129,7 +124,7 @@ module Gitlab
def format_time(timestamp)
return timestamp if timestamp.is_a?(String)
Time.at(timestamp).utc.iso8601(3)
Time.at(timestamp).utc.iso8601(6)
end
def limited_job_args(args)
......
......@@ -330,6 +330,38 @@ FactoryBot.define do
options { {} }
end
trait :dast do
options do
{
artifacts: { reports: { dast: 'gl-dast-report.json' } }
}
end
end
trait :sast do
options do
{
artifacts: { reports: { sast: 'gl-sast-report.json' } }
}
end
end
trait :dependency_scanning do
options do
{
artifacts: { reports: { dependency_scanning: 'gl-dependency-scanning-report.json' } }
}
end
end
trait :container_scanning do
options do
{
artifacts: { reports: { container_scanning: 'gl-container-scanning-report.json' } }
}
end
end
trait :non_playable do
status { 'created' }
self.when { 'manual' }
......
......@@ -40,6 +40,7 @@ describe Gitlab::SidekiqDaemon::MemoryKiller do
message: "Exception from start_working: My Exception")
expect(memory_killer).to receive(:rss_within_range?).twice.and_raise(StandardError, 'My Exception')
expect(memory_killer).to receive(:sleep).twice.with(Gitlab::SidekiqDaemon::MemoryKiller::CHECK_INTERVAL_SECONDS)
expect { subject }.not_to raise_exception
end
......@@ -53,6 +54,7 @@ describe Gitlab::SidekiqDaemon::MemoryKiller do
expect(memory_killer).to receive(:rss_within_range?).once.and_raise(Exception, 'My Exception')
expect(memory_killer).to receive(:sleep).with(Gitlab::SidekiqDaemon::MemoryKiller::CHECK_INTERVAL_SECONDS)
expect(Sidekiq.logger).to receive(:warn).once
.with(
class: described_class.to_s,
......
......@@ -23,13 +23,15 @@ describe Gitlab::SidekiqLogging::StructuredLogger do
end
let(:logger) { double }
let(:clock_thread_cputime_start) { 0.222222299 }
let(:clock_thread_cputime_end) { 1.333333799 }
let(:start_payload) do
job.merge(
'message' => 'TestWorker JID-da883554ee4fe414012f5f42: start',
'job_status' => 'start',
'pid' => Process.pid,
'created_at' => created_at.iso8601(3),
'enqueued_at' => created_at.iso8601(3),
'created_at' => created_at.iso8601(6),
'enqueued_at' => created_at.iso8601(6),
'scheduling_latency_s' => scheduling_latency_s
)
end
......@@ -38,9 +40,8 @@ describe Gitlab::SidekiqLogging::StructuredLogger do
'message' => 'TestWorker JID-da883554ee4fe414012f5f42: done: 0.0 sec',
'job_status' => 'done',
'duration' => 0.0,
"completed_at" => timestamp.iso8601(3),
"system_s" => 0.0,
"user_s" => 0.0
"completed_at" => timestamp.iso8601(6),
"cpu_s" => 1.111112
)
end
let(:exception_payload) do
......@@ -57,12 +58,7 @@ describe Gitlab::SidekiqLogging::StructuredLogger do
allow(subject).to receive(:current_time).and_return(timestamp.to_f)
allow(Process).to receive(:times).and_return(
stime: 0.0,
utime: 0.0,
cutime: 0.0,
cstime: 0.0
)
allow(Process).to receive(:clock_gettime).with(Process::CLOCK_THREAD_CPUTIME_ID).and_return(clock_thread_cputime_start, clock_thread_cputime_end)
end
subject { described_class.new }
......@@ -187,31 +183,22 @@ describe Gitlab::SidekiqLogging::StructuredLogger do
end
end
end
def ctime(times)
times[:cstime] + times[:cutime]
end
context 'with ctime value greater than 0' do
let(:times_start) { { stime: 0.04999, utime: 0.0483, cstime: 0.0188, cutime: 0.0188 } }
let(:times_end) { { stime: 0.0699, utime: 0.0699, cstime: 0.0399, cutime: 0.0399 } }
describe '#add_time_keys!' do
let(:time) { { duration: 0.1231234, cputime: 1.2342345 } }
let(:payload) { { 'class' => 'my-class', 'message' => 'my-message', 'job_status' => 'my-job-status' } }
let(:current_utc_time) { '2019-09-23 10:00:58 UTC' }
let(:payload_with_time_keys) { { 'class' => 'my-class', 'message' => 'my-message', 'job_status' => 'my-job-status', 'duration' => 0.123123, 'cpu_s' => 1.234235, 'completed_at' => current_utc_time } }
before do
end_payload['system_s'] = 0.02
end_payload['user_s'] = 0.022
end_payload['child_s'] = 0.042
subject { described_class.new }
allow(Process).to receive(:times).and_return(times_start, times_end)
end
it 'update payload correctly' do
expect(Time).to receive_message_chain(:now, :utc).and_return(current_utc_time)
it 'logs with ctime data and other cpu data' do
Timecop.freeze(timestamp) do
expect(logger).to receive(:info).with(start_payload.except('args')).ordered
expect(logger).to receive(:info).with(end_payload.except('args')).ordered
subject.send(:add_time_keys!, time, payload)
subject.call(job, 'test_queue') { }
end
end
expect(payload).to eq(payload_with_time_keys)
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment