Commit 7f500acb authored by Kamil Trzciński's avatar Kamil Trzciński

Merge branch '33772-readonly-gitlab-ci-cache' into 'master'

Introduce cache policies for CI jobs

Closes #33772

See merge request !12483
parents 6afe25ef 35f4a00f
---
title: Introduce cache policies for CI jobs
merge_request: 12483
author:
...@@ -306,6 +306,53 @@ cache: ...@@ -306,6 +306,53 @@ cache:
untracked: true untracked: true
``` ```
### cache:policy
> Introduced in GitLab 9.4.
The default behaviour of a caching job is to download the files at the start of
execution, and to re-upload them at the end. This allows any changes made by the
job to be persisted for future runs, and is known as the `pull-push` cache
policy.
If you know the job doesn't alter the cached files, you can skip the upload step
by setting `policy: pull` in the job specification. Typically, this would be
twinned with an ordinary cache job at an earlier stage to ensure the cache
is updated from time to time:
```yaml
stages:
- setup
- test
prepare:
stage: setup
cache:
key: gems
paths:
- vendor/bundle
script:
- bundle install --deployment
rspec:
stage: test
cache:
key: gems
paths:
- vendor/bundle
policy: pull
script:
- bundle exec rspec ...
```
This helps to speed up job execution and reduce load on the cache server,
especially when you have a large number of cache-using jobs executing in
parallel.
Additionally, if you have a job that unconditionally recreates the cache without
reference to its previous contents, you can use `policy: push` in that job to
skip the download step.
## Jobs ## Jobs
`.gitlab-ci.yml` allows you to specify an unlimited number of jobs. Each job `.gitlab-ci.yml` allows you to specify an unlimited number of jobs. Each job
......
...@@ -832,7 +832,7 @@ module API ...@@ -832,7 +832,7 @@ module API
end end
class Cache < Grape::Entity class Cache < Grape::Entity
expose :key, :untracked, :paths expose :key, :untracked, :paths, :policy
end end
class Credentials < Grape::Entity class Credentials < Grape::Entity
......
...@@ -7,11 +7,14 @@ module Gitlab ...@@ -7,11 +7,14 @@ module Gitlab
# #
class Cache < Node class Cache < Node
include Configurable include Configurable
include Attributable
ALLOWED_KEYS = %i[key untracked paths].freeze ALLOWED_KEYS = %i[key untracked paths policy].freeze
DEFAULT_POLICY = 'pull-push'.freeze
validations do validations do
validates :config, allowed_keys: ALLOWED_KEYS validates :config, allowed_keys: ALLOWED_KEYS
validates :policy, inclusion: { in: %w[pull-push push pull], message: 'should be pull-push, push, or pull' }, allow_blank: true
end end
entry :key, Entry::Key, entry :key, Entry::Key,
...@@ -25,8 +28,15 @@ module Gitlab ...@@ -25,8 +28,15 @@ module Gitlab
helpers :key helpers :key
attributes :policy
def value def value
super.merge(key: key_value) result = super
result[:key] = key_value
result[:policy] = policy || DEFAULT_POLICY
result
end end
end end
end end
......
...@@ -207,7 +207,8 @@ FactoryGirl.define do ...@@ -207,7 +207,8 @@ FactoryGirl.define do
cache: { cache: {
key: 'cache_key', key: 'cache_key',
untracked: false, untracked: false,
paths: ['vendor/*'] paths: ['vendor/*'],
policy: 'pull-push'
} }
} }
end end
......
...@@ -878,7 +878,8 @@ module Ci ...@@ -878,7 +878,8 @@ module Ci
expect(config_processor.builds_for_stage_and_ref("test", "master").first[:options][:cache]).to eq( expect(config_processor.builds_for_stage_and_ref("test", "master").first[:options][:cache]).to eq(
paths: ["logs/", "binaries/"], paths: ["logs/", "binaries/"],
untracked: true, untracked: true,
key: 'key' key: 'key',
policy: 'pull-push'
) )
end end
...@@ -896,7 +897,8 @@ module Ci ...@@ -896,7 +897,8 @@ module Ci
expect(config_processor.builds_for_stage_and_ref("test", "master").first[:options][:cache]).to eq( expect(config_processor.builds_for_stage_and_ref("test", "master").first[:options][:cache]).to eq(
paths: ["logs/", "binaries/"], paths: ["logs/", "binaries/"],
untracked: true, untracked: true,
key: 'key' key: 'key',
policy: 'pull-push'
) )
end end
...@@ -915,7 +917,8 @@ module Ci ...@@ -915,7 +917,8 @@ module Ci
expect(config_processor.builds_for_stage_and_ref("test", "master").first[:options][:cache]).to eq( expect(config_processor.builds_for_stage_and_ref("test", "master").first[:options][:cache]).to eq(
paths: ["test/"], paths: ["test/"],
untracked: false, untracked: false,
key: 'local' key: 'local',
policy: 'pull-push'
) )
end end
end end
......
require 'spec_helper' require 'spec_helper'
describe Gitlab::Ci::Config::Entry::Cache do describe Gitlab::Ci::Config::Entry::Cache do
let(:entry) { described_class.new(config) } subject(:entry) { described_class.new(config) }
describe 'validations' do describe 'validations' do
before do before do
...@@ -9,22 +9,44 @@ describe Gitlab::Ci::Config::Entry::Cache do ...@@ -9,22 +9,44 @@ describe Gitlab::Ci::Config::Entry::Cache do
end end
context 'when entry config value is correct' do context 'when entry config value is correct' do
let(:policy) { nil }
let(:config) do let(:config) do
{ key: 'some key', { key: 'some key',
untracked: true, untracked: true,
paths: ['some/path/'] } paths: ['some/path/'],
policy: policy }
end end
describe '#value' do describe '#value' do
it 'returns hash value' do it 'returns hash value' do
expect(entry.value).to eq config expect(entry.value).to eq(key: 'some key', untracked: true, paths: ['some/path/'], policy: 'pull-push')
end end
end end
describe '#valid?' do describe '#valid?' do
it 'is valid' do it { is_expected.to be_valid }
expect(entry).to be_valid end
end
context 'policy is pull-push' do
let(:policy) { 'pull-push' }
it { is_expected.to be_valid }
it { expect(entry.value).to include(policy: 'pull-push') }
end
context 'policy is push' do
let(:policy) { 'push' }
it { is_expected.to be_valid }
it { expect(entry.value).to include(policy: 'push') }
end
context 'policy is pull' do
let(:policy) { 'pull' }
it { is_expected.to be_valid }
it { expect(entry.value).to include(policy: 'pull') }
end end
context 'when key is missing' do context 'when key is missing' do
...@@ -44,12 +66,20 @@ describe Gitlab::Ci::Config::Entry::Cache do ...@@ -44,12 +66,20 @@ describe Gitlab::Ci::Config::Entry::Cache do
context 'when entry value is not correct' do context 'when entry value is not correct' do
describe '#errors' do describe '#errors' do
subject { entry.errors }
context 'when is not a hash' do context 'when is not a hash' do
let(:config) { 'ls' } let(:config) { 'ls' }
it 'reports errors with config value' do it 'reports errors with config value' do
expect(entry.errors) is_expected.to include 'cache config should be a hash'
.to include 'cache config should be a hash' end
end
context 'when policy is unknown' do
let(:config) { { policy: "unknown" } }
it 'reports error' do
is_expected.to include('cache policy should be pull-push, push, or pull')
end end
end end
...@@ -57,8 +87,7 @@ describe Gitlab::Ci::Config::Entry::Cache do ...@@ -57,8 +87,7 @@ describe Gitlab::Ci::Config::Entry::Cache do
let(:config) { { key: 1 } } let(:config) { { key: 1 } }
it 'reports error with descendants' do it 'reports error with descendants' do
expect(entry.errors) is_expected.to include 'key config should be a string or symbol'
.to include 'key config should be a string or symbol'
end end
end end
...@@ -66,8 +95,7 @@ describe Gitlab::Ci::Config::Entry::Cache do ...@@ -66,8 +95,7 @@ describe Gitlab::Ci::Config::Entry::Cache do
let(:config) { { invalid: true } } let(:config) { { invalid: true } }
it 'reports error with descendants' do it 'reports error with descendants' do
expect(entry.errors) is_expected.to include 'cache config contains unknown keys: invalid'
.to include 'cache config contains unknown keys: invalid'
end end
end end
end end
......
...@@ -143,7 +143,7 @@ describe Gitlab::Ci::Config::Entry::Global do ...@@ -143,7 +143,7 @@ describe Gitlab::Ci::Config::Entry::Global do
describe '#cache_value' do describe '#cache_value' do
it 'returns cache configuration' do it 'returns cache configuration' do
expect(global.cache_value) expect(global.cache_value)
.to eq(key: 'k', untracked: true, paths: ['public/']) .to eq(key: 'k', untracked: true, paths: ['public/'], policy: 'pull-push')
end end
end end
...@@ -157,7 +157,7 @@ describe Gitlab::Ci::Config::Entry::Global do ...@@ -157,7 +157,7 @@ describe Gitlab::Ci::Config::Entry::Global do
image: { name: 'ruby:2.2' }, image: { name: 'ruby:2.2' },
services: [{ name: 'postgres:9.1' }, { name: 'mysql:5.5' }], services: [{ name: 'postgres:9.1' }, { name: 'mysql:5.5' }],
stage: 'test', stage: 'test',
cache: { key: 'k', untracked: true, paths: ['public/'] }, cache: { key: 'k', untracked: true, paths: ['public/'], policy: 'pull-push' },
variables: { 'VAR' => 'value' }, variables: { 'VAR' => 'value' },
ignore: false, ignore: false,
after_script: ['make clean'] }, after_script: ['make clean'] },
...@@ -168,7 +168,7 @@ describe Gitlab::Ci::Config::Entry::Global do ...@@ -168,7 +168,7 @@ describe Gitlab::Ci::Config::Entry::Global do
image: { name: 'ruby:2.2' }, image: { name: 'ruby:2.2' },
services: [{ name: 'postgres:9.1' }, { name: 'mysql:5.5' }], services: [{ name: 'postgres:9.1' }, { name: 'mysql:5.5' }],
stage: 'test', stage: 'test',
cache: { key: 'k', untracked: true, paths: ['public/'] }, cache: { key: 'k', untracked: true, paths: ['public/'], policy: 'pull-push' },
variables: {}, variables: {},
ignore: false, ignore: false,
after_script: ['make clean'] } after_script: ['make clean'] }
...@@ -212,7 +212,7 @@ describe Gitlab::Ci::Config::Entry::Global do ...@@ -212,7 +212,7 @@ describe Gitlab::Ci::Config::Entry::Global do
describe '#cache_value' do describe '#cache_value' do
it 'returns correct cache definition' do it 'returns correct cache definition' do
expect(global.cache_value).to eq(key: 'a') expect(global.cache_value).to eq(key: 'a', policy: 'pull-push')
end end
end end
end end
......
...@@ -109,7 +109,7 @@ describe Gitlab::Ci::Config::Entry::Job do ...@@ -109,7 +109,7 @@ describe Gitlab::Ci::Config::Entry::Job do
it 'overrides global config' do it 'overrides global config' do
expect(entry[:image].value).to eq(name: 'some_image') expect(entry[:image].value).to eq(name: 'some_image')
expect(entry[:cache].value).to eq(key: 'test') expect(entry[:cache].value).to eq(key: 'test', policy: 'pull-push')
end end
end end
...@@ -123,7 +123,7 @@ describe Gitlab::Ci::Config::Entry::Job do ...@@ -123,7 +123,7 @@ describe Gitlab::Ci::Config::Entry::Job do
it 'uses config from global entry' do it 'uses config from global entry' do
expect(entry[:image].value).to eq 'specified' expect(entry[:image].value).to eq 'specified'
expect(entry[:cache].value).to eq(key: 'test') expect(entry[:cache].value).to eq(key: 'test', policy: 'pull-push')
end end
end end
end end
......
...@@ -351,7 +351,8 @@ describe API::Runner do ...@@ -351,7 +351,8 @@ describe API::Runner do
let(:expected_cache) do let(:expected_cache) do
[{ 'key' => 'cache_key', [{ 'key' => 'cache_key',
'untracked' => false, 'untracked' => false,
'paths' => ['vendor/*'] }] 'paths' => ['vendor/*'],
'policy' => 'pull-push' }]
end end
it 'picks a job' do it 'picks a job' do
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment