Skip to content

G
gitlab-ce
Commit da246582 authored by Grzegorz Bizon's avatar Grzegorz Bizon
Browse files
Options

Merge branch 'master' into 'backstage/gb/use-persisted-stages-to-improve-pipelines-table' 

# Conflicts:
#   db/schema.rb
parents 809a50fc 99218302
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 2159 additions and 387 deletions
CONTRIBUTING.md
View file @ da246582
......@@ -182,7 +182,7 @@ Assigning a team label makes sure issues get the attention of the appropriate
people.
The current team labels are ~Distribution, ~"CI/CD", ~Discussion, ~Documentation, ~Quality,
~Geo, ~Gitaly, ~Monitoring, ~Platform, ~Release, ~"Security Products" and ~"UX".
~Geo, ~Gitaly, ~Monitoring, ~Platform, ~Release, ~"Security Products", ~"Configuration", and ~"UX".
The descriptions on the [labels page][labels-page] explain what falls under the
responsibility of each team.
......
Gemfile.rails5.lock
View file @ da246582
......@@ -72,8 +72,6 @@ GEM
attr_encrypted (3.1.0)
encryptor (~> 3.0.0)
attr_required (1.0.1)
autoprefixer-rails (8.1.0.1)
execjs
awesome_print (1.2.0)
axiom-types (0.1.1)
descendants_tracker (~> 0.0.4)
......@@ -93,9 +91,6 @@ GEM
binding_of_caller (0.7.3)
debug_inspector (>= 0.0.1)
blankslate (2.1.2.4)
bootstrap-sass (3.3.7)
autoprefixer-rails (>= 5.2.1)
sass (>= 3.3.4)
bootstrap_form (2.7.0)
brakeman (4.2.1)
browser (2.5.3)
......@@ -175,7 +170,7 @@ GEM
diff-lcs (1.3)
diffy (3.1.0)
docile (1.1.5)
domain_name (0.5.20170404)
domain_name (0.5.20180417)
unf (>= 0.0.5, < 1.0.0)
doorkeeper (4.3.1)
railties (>= 4.2)
......@@ -185,9 +180,10 @@ GEM
dropzonejs-rails (0.7.4)
rails (> 3.1)
email_reply_trimmer (0.1.10)
email_spec (1.6.0)
email_spec (2.2.0)
htmlentities (~> 4.3.3)
launchy (~> 2.1)
mail (~> 2.2)
mail (~> 2.7)
encryptor (3.0.0)
equalizer (0.0.11)
erubis (2.7.0)
......@@ -288,7 +284,7 @@ GEM
gettext_i18n_rails (>= 0.7.1)
po_to_json (>= 1.0.0)
rails (>= 3.2.0)
gitaly-proto (0.99.0)
gitaly-proto (0.100.0)
google-protobuf (~> 3.1)
grpc (~> 1.10)
github-linguist (5.3.3)
......@@ -365,9 +361,9 @@ GEM
grape-entity (0.7.1)
activesupport (>= 4.0)
multi_json (>= 1.3.2)
grape-route-helpers (2.1.0)
grape-path-helpers (1.0.0)
activesupport
grape (>= 0.16.0)
grape (~> 1.0)
rake
grape_logging (1.7.0)
grape
......@@ -417,6 +413,7 @@ GEM
httpclient (2.8.3)
i18n (1.0.1)
concurrent-ruby (~> 1.0)
icalendar (2.4.1)
ice_nine (0.11.2)
influxdb (0.5.3)
ipaddress (0.8.3)
......@@ -450,9 +447,9 @@ GEM
kgio (2.11.2)
knapsack (1.16.0)
rake
kubeclient (3.0.0)
kubeclient (3.1.1)
http (~> 2.2.2)
recursive-open-struct (~> 1.0.4)
recursive-open-struct (~> 1.0, >= 1.0.4)
rest-client (~> 2.0)
launchy (2.4.3)
addressable (~> 2.3)
......@@ -521,15 +518,16 @@ GEM
multi_json (~> 1.3)
multi_xml (~> 0.5)
rack (>= 1.2, < 3)
octokit (4.8.0)
octokit (4.9.0)
sawyer (~> 0.8.0, >= 0.5.3)
omniauth (1.8.1)
hashie (>= 3.4.6, < 3.6.0)
rack (>= 1.6.2, < 3)
omniauth-auth0 (2.0.0)
omniauth-oauth2 (~> 1.4)
omniauth-authentiq (0.3.1)
omniauth-oauth2 (~> 1.3, >= 1.3.1)
omniauth-authentiq (0.3.3)
jwt (>= 1.5)
omniauth-oauth2 (>= 1.5)
omniauth-azure-oauth2 (0.0.9)
jwt (~> 1.0)
omniauth (~> 1.0)
......@@ -628,7 +626,7 @@ GEM
parser
unparser
procto (0.0.3)
prometheus-client-mmap (0.9.2)
prometheus-client-mmap (0.9.3)
pry (0.11.3)
coderay (~> 1.1.0)
method_source (~> 0.9.0)
......@@ -702,11 +700,11 @@ GEM
ffi
rbnacl-libsodium (1.0.16)
rbnacl (>= 3.0.1)
rdoc (4.3.0)
rdoc (6.0.4)
re2 (1.1.1)
recaptcha (3.4.0)
json
recursive-open-struct (1.0.5)
recursive-open-struct (1.1.0)
redcarpet (3.4.0)
redis (3.3.5)
redis-actionpack (5.0.2)
......@@ -716,8 +714,8 @@ GEM
redis-activesupport (5.0.4)
activesupport (>= 3, < 6)
redis-store (>= 1.3, < 2)
redis-namespace (1.5.3)
redis (~> 3.0, >= 3.0.4)
redis-namespace (1.6.0)
redis (>= 3.0.4)
redis-rack (2.0.4)
rack (>= 1.5, < 3)
redis-store (>= 1.2, < 2)
......@@ -836,7 +834,7 @@ GEM
activesupport (>= 3.1)
select2-rails (3.5.10)
thor (~> 0.14)
selenium-webdriver (3.11.0)
selenium-webdriver (3.12.0)
childprocess (~> 0.5)
rubyzip (~> 1.2)
sentry-raven (2.7.2)
......@@ -986,7 +984,7 @@ DEPENDENCIES
asciidoctor-plantuml (= 0.0.8)
asset_sync (~> 2.4)
attr_encrypted (~> 3.1.0)
awesome_print (~> 1.2.0)
awesome_print
babosa (~> 1.0.2)
base32 (~> 0.3.0)
batch-loader (~> 1.2.1)
......@@ -994,7 +992,6 @@ DEPENDENCIES
benchmark-ips (~> 2.3.0)
better_errors (~> 2.1.0)
binding_of_caller (~> 0.7.2)
bootstrap-sass (~> 3.3.0)
bootstrap_form (~> 2.7.0)
brakeman (~> 4.2)
browser (~> 2.2)
......@@ -1021,7 +1018,7 @@ DEPENDENCIES
doorkeeper-openid_connect (~> 1.3)
dropzonejs-rails (~> 0.7.1)
email_reply_trimmer (~> 0.1)
email_spec (~> 1.6.0)
email_spec (~> 2.2.0)
factory_bot_rails (~> 4.8.2)
faraday (~> 0.12)
fast_blank
......@@ -1045,7 +1042,7 @@ DEPENDENCIES
gettext (~> 3.2.2)
gettext_i18n_rails (~> 1.8.0)
gettext_i18n_rails_js (~> 1.3)
gitaly-proto (~> 0.99.0)
gitaly-proto (~> 0.100.0)
github-linguist (~> 5.3.3)
gitlab-flowdock-git-hook (~> 1.0.1)
gitlab-gollum-lib (~> 4.2)
......@@ -1059,7 +1056,7 @@ DEPENDENCIES
gpgme
grape (~> 1.0)
grape-entity (~> 0.7.1)
grape-route-helpers (~> 2.1.0)
grape-path-helpers (~> 1.0)
grape_logging (~> 1.7)
grpc (~> 1.11.0)
haml_lint (~> 0.26.0)
......@@ -1070,6 +1067,7 @@ DEPENDENCIES
html-pipeline (~> 2.7.1)
html2text
httparty (~> 0.13.3)
icalendar
influxdb (~> 0.2)
jira-ruby (~> 1.4)
jquery-atwho-rails (~> 1.3.2)
......@@ -1077,7 +1075,7 @@ DEPENDENCIES
jwt (~> 1.5.6)
kaminari (~> 1.0)
knapsack (~> 1.16)
kubeclient (~> 3.0)
kubeclient (~> 3.1.0)
letter_opener_web (~> 1.3.0)
license_finder (~> 3.1)
licensee (~> 8.9)
......@@ -1092,10 +1090,10 @@ DEPENDENCIES
net-ssh (~> 4.2.0)
nokogiri (~> 1.8.2)
oauth2 (~> 1.4)
octokit (~> 4.8)
octokit (~> 4.9)
omniauth (~> 1.8)
omniauth-auth0 (~> 2.0.0)
omniauth-authentiq (~> 0.3.1)
omniauth-authentiq (~> 0.3.3)
omniauth-azure-oauth2 (~> 0.0.9)
omniauth-cas3 (~> 1.1.4)
omniauth-facebook (~> 4.0.0)
......@@ -1118,7 +1116,7 @@ DEPENDENCIES
peek-sidekiq (~> 1.0.3)
pg (~> 0.18.2)
premailer-rails (~> 1.9.7)
prometheus-client-mmap (~> 0.9.2)
prometheus-client-mmap (~> 0.9.3)
pry-byebug (~> 3.4.1)
pry-rails (~> 0.3.4)
rack-attack (~> 4.4.1)
......@@ -1134,12 +1132,12 @@ DEPENDENCIES
rblineprof (~> 0.3.6)
rbnacl (~> 4.0)
rbnacl-libsodium
rdoc (~> 4.2)
rdoc (~> 6.0)
re2 (~> 1.1.1)
recaptcha (~> 3.0)
redcarpet (~> 3.4)
redis (~> 3.2)
redis-namespace (~> 1.5.2)
redis-namespace (~> 1.6.0)
redis-rails (~> 5.0.2)
request_store (~> 1.3)
responders (~> 2.0)
......@@ -1154,6 +1152,7 @@ DEPENDENCIES
rubocop-rspec (~> 1.22.1)
ruby-fogbugz (~> 0.2.1)
ruby-prof (~> 0.17.0)
ruby-progressbar
ruby_parser (~> 3.8)
rufus-scheduler (~> 3.4)
rugged (~> 0.27)
......@@ -1162,12 +1161,12 @@ DEPENDENCIES
scss_lint (~> 0.56.0)
seed-fu (~> 2.3.7)
select2-rails (~> 3.5.9)
selenium-webdriver (~> 3.5)
selenium-webdriver (~> 3.12)
sentry-raven (~> 2.7)
settingslogic (~> 2.0.9)
sham_rack (~> 1.3.6)
shoulda-matchers (~> 3.1.2)
sidekiq (~> 5.0)
sidekiq (~> 5.1)
sidekiq-cron (~> 0.6.0)
sidekiq-limit_fetch (~> 3.4)
simple_po_parser (~> 1.1.2)
......@@ -1199,4 +1198,4 @@ DEPENDENCIES
wikicloth (= 0.8.1)
BUNDLED WITH
1.16.1
1.16.2
app/assets/javascripts/ide/components/activity_bar.vue
View file @ da246582
<script>
import $ from 'jquery';
import { mapActions, mapGetters, mapState } from 'vuex';
import Icon from '~/vue_shared/components/icon.vue';
import tooltip from '~/vue_shared/directives/tooltip';
......@@ -20,6 +21,13 @@ export default {
},
methods: {
...mapActions(['updateActivityBarView']),
changedActivityView(e, view) {
e.currentTarget.blur();
this.updateActivityBarView(view);
$(e.currentTarget).tooltip('hide');
},
},
activityBarViews,
};
......@@ -54,7 +62,7 @@ export default {
:class="{
active: currentActivityView === $options.activityBarViews.edit
}"
@click.prevent="updateActivityBarView($options.activityBarViews.edit)"
@click.prevent="changedActivityView($event, $options.activityBarViews.edit)"
:title="s__('IDE|Edit')"
:aria-label="s__('IDE|Edit')"
>
......@@ -73,7 +81,7 @@ export default {
:class="{
active: currentActivityView === $options.activityBarViews.review
}"
@click.prevent="updateActivityBarView($options.activityBarViews.review)"
@click.prevent="changedActivityView($event, $options.activityBarViews.review)"
:title="s__('IDE|Review')"
:aria-label="s__('IDE|Review')"
>
......@@ -92,7 +100,7 @@ export default {
:class="{
active: currentActivityView === $options.activityBarViews.commit
}"
@click.prevent="updateActivityBarView($options.activityBarViews.commit)"
@click.prevent="changedActivityView($event, $options.activityBarViews.commit)"
:title="s__('IDE|Commit')"
:aria-label="s__('IDE|Commit')"
>
......
app/assets/javascripts/notes/constants.js
View file @ da246582
......@@ -14,6 +14,7 @@ export const EPIC_NOTEABLE_TYPE = 'epic';
export const MERGE_REQUEST_NOTEABLE_TYPE = 'merge_request';
export const UNRESOLVE_NOTE_METHOD_NAME = 'delete';
export const RESOLVE_NOTE_METHOD_NAME = 'post';
export const DESCRIPTION_TYPE = 'changed the description';
export const NOTEABLE_TYPE_MAPPING = {
Issue: ISSUE_NOTEABLE_TYPE,
......
app/assets/javascripts/notes/stores/collapse_utils.js 0 → 100644
View file @ da246582
import { n__, s__, sprintf } from '~/locale';
import { DESCRIPTION_TYPE } from '../constants';
/**
* Changes the description from a note, returns 'changed the description n number of times'
*/
export const changeDescriptionNote = (note, descriptionChangedTimes, timeDifferenceMinutes) => {
const descriptionNote = Object.assign({}, note);
descriptionNote.note_html = sprintf(
s__(`MergeRequest|
%{paragraphStart}changed the description %{descriptionChangedTimes} times %{timeDifferenceMinutes}%{paragraphEnd}`),
{
paragraphStart: '<p dir="auto">',
paragraphEnd: '</p>',
descriptionChangedTimes,
timeDifferenceMinutes: n__('within %d minute ', 'within %d minutes ', timeDifferenceMinutes),
},
false,
);
descriptionNote.times_updated = descriptionChangedTimes;
return descriptionNote;
};
/**
* Checks the time difference between two notes from their 'created_at' dates
* returns an integer
*/
export const getTimeDifferenceMinutes = (noteBeggining, noteEnd) => {
const descriptionNoteBegin = new Date(noteBeggining.created_at);
const descriptionNoteEnd = new Date(noteEnd.created_at);
const timeDifferenceMinutes = (descriptionNoteEnd - descriptionNoteBegin) / 1000 / 60;
return Math.ceil(timeDifferenceMinutes);
};
/**
* Checks if a note is a system note and if the content is description
*
* @param {Object} note
* @returns {Boolean}
*/
export const isDescriptionSystemNote = note => note.system && note.note === DESCRIPTION_TYPE;
/**
* Collapses the system notes of a description type, e.g. Changed the description, n minutes ago
* the notes will collapse as long as they happen no more than 10 minutes away from each away
* in between the notes can be anything, another type of system note
* (such as 'changed the weight') or a comment.
*
* @param {Array} notes
* @returns {Array}
*/
export const collapseSystemNotes = notes => {
let lastDescriptionSystemNote = null;
let lastDescriptionSystemNoteIndex = -1;
let descriptionChangedTimes = 1;
return notes.slice(0).reduce((acc, currentNote) => {
const note = currentNote.notes[0];
if (isDescriptionSystemNote(note)) {
// is it the first one?
if (!lastDescriptionSystemNote) {
lastDescriptionSystemNote = note;
lastDescriptionSystemNoteIndex = acc.length;
} else if (lastDescriptionSystemNote) {
const timeDifferenceMinutes = getTimeDifferenceMinutes(
lastDescriptionSystemNote,
note,
);
// are they less than 10 minutes appart?
if (timeDifferenceMinutes > 10) {
// reset counter
descriptionChangedTimes = 1;
// update the previous system note
lastDescriptionSystemNote = note;
lastDescriptionSystemNoteIndex = acc.length;
} else {
// increase counter
descriptionChangedTimes += 1;
// delete the previous one
acc.splice(lastDescriptionSystemNoteIndex, 1);
// replace the text of the current system note with the collapsed note.
currentNote.notes.splice(
0,
1,
changeDescriptionNote(note, descriptionChangedTimes, timeDifferenceMinutes),
);
// update the previous system note index
lastDescriptionSystemNoteIndex = acc.length;
}
}
}
acc.push(currentNote);
return acc;
}, []);
};
// for babel-rewire
export default {};
app/assets/javascripts/notes/stores/getters.js
View file @ da246582
import _ from 'underscore';
import { collapseSystemNotes } from './collapse_utils';
export const notes = state => collapseSystemNotes(state.notes);
export const notes = state => state.notes;
export const targetNoteHash = state => state.targetNoteHash;
export const getNotesData = state => state.notesData;
......
app/assets/stylesheets/bootstrap_migration.scss
View file @ da246582
......@@ -69,6 +69,11 @@ code {
background-color: inherit;
padding: unset;
}
.build-trace & {
background-color: inherit;
padding: inherit;
}
}
.code {
......
app/assets/stylesheets/pages/repo.scss
View file @ da246582
......@@ -183,7 +183,7 @@
svg {
position: relative;
top: -1px;
top: -2px;
}
.ide-file-changed-icon {
......@@ -458,6 +458,10 @@
width: auto;
margin-right: 0;
a {
height: 60px;
}
a:hover,
a:focus {
text-decoration: none;
......@@ -718,9 +722,17 @@
}
.ide-new-btn {
.btn {
padding-top: 3px;
padding-bottom: 3px;
}
.dropdown {
display: flex;
}
.dropdown-toggle svg {
margin-top: -2px;
margin-bottom: 2px;
top: 0;
}
.dropdown-menu {
......@@ -877,6 +889,7 @@
border-top: 1px solid transparent;
border-bottom: 1px solid transparent;
outline: 0;
cursor: pointer;
svg {
margin: 0 auto;
......
app/assets/stylesheets/print.scss
View file @ da246582
......@@ -22,9 +22,9 @@
header,
nav,
nav.main-nav,
nav.navbar-collapse,
nav.navbar-collapse.collapse,
.nav-sidebar,
.profiler-results,
.tree-ref-holder,
.tree-holder .breadcrumb,
......@@ -38,7 +38,8 @@ ul.notes-form,
.edit-link,
.note-action-button,
.right-sidebar,
.flash-container {
.flash-container,
#js-peek {
display: none !important;
}
......
app/controllers/projects/lfs_storage_controller.rb
View file @ da246582
......@@ -18,7 +18,7 @@ class Projects::LfsStorageController < Projects::GitHttpClientController
def upload_authorize
set_workhorse_internal_api_content_type
authorized = LfsObjectUploader.workhorse_authorize
authorized = LfsObjectUploader.workhorse_authorize(has_length: true)
authorized.merge!(LfsOid: oid, LfsSize: size)
render json: authorized
......
app/helpers/projects_helper.rb
View file @ da246582
......@@ -238,6 +238,14 @@ module ProjectsHelper
"git push --set-upstream #{repository_url}/$(git rev-parse --show-toplevel | xargs basename).git $(git rev-parse --abbrev-ref HEAD)"
end
def show_xcode_link?(project = @project)
browser.platform.mac? && project.repository.xcode_project?
end
def xcode_uri_to_repo(project = @project)
"xcode://clone?repo=#{CGI.escape(default_url_to_repo(project))}"
end
private
def get_project_nav_tabs(project, current_user)
......
app/models/ci/build.rb
View file @ da246582
......@@ -55,6 +55,11 @@ module Ci
where('(artifacts_file IS NOT NULL AND artifacts_file <> ?) OR EXISTS (?)',
'', Ci::JobArtifact.select(1).where('ci_builds.id = ci_job_artifacts.job_id').archive)
end
scope :without_archived_trace, ->() do
where('NOT EXISTS (?)', Ci::JobArtifact.select(1).where('ci_builds.id = ci_job_artifacts.job_id').trace)
end
scope :with_artifacts_stored_locally, -> { with_artifacts_archive.where(artifacts_file_store: [nil, LegacyArtifactUploader::Store::LOCAL]) }
scope :with_artifacts_not_expired, ->() { with_artifacts_archive.where('artifacts_expire_at IS NULL OR artifacts_expire_at > ?', Time.now) }
scope :with_expired_artifacts, ->() { with_artifacts_archive.where('artifacts_expire_at < ?', Time.now) }
......@@ -144,6 +149,7 @@ module Ci
after_transition any => [:success] do |build|
build.run_after_commit do
BuildSuccessWorker.perform_async(id)
PagesWorker.perform_async(:deploy, id) if build.pages_generator?
end
end
......@@ -183,6 +189,11 @@ module Ci
pipeline.manual_actions.where.not(name: name)
end
def pages_generator?
Gitlab.config.pages.enabled &&
self.name == 'pages'
end
def playable?
action? && (manual? || retryable?)
end
......@@ -402,8 +413,6 @@ module Ci
build_data = Gitlab::DataBuilder::Build.build(self)
project.execute_hooks(build_data.dup, :job_hooks)
project.execute_services(build_data.dup, :job_hooks)
PagesService.new(build_data).execute
project.running_or_pending_build_count(force: true)
end
def browsable_artifacts?
......
app/models/project.rb
View file @ da246582
......@@ -1656,12 +1656,6 @@ class Project < ActiveRecord::Base
import_state.update_column(:jid, nil)
end
def running_or_pending_build_count(force: false)
Rails.cache.fetch(['projects', id, 'running_or_pending_build_count'], force: force) do
builds.running_or_pending.count(:all)
end
end
# Lazy loading of the `pipeline_status` attribute
def pipeline_status
@pipeline_status ||= Gitlab::Cache::Ci::ProjectPipelineStatus.load_for_project(self)
......
app/services/pages_service.rb deleted 100644 → 0
View file @ 809a50fc
class PagesService
attr_reader :data
def initialize(data)
@data = data
end
def execute
return unless Settings.pages.enabled
return unless data[:build_name] == 'pages'
return unless data[:build_status] == 'success'
PagesWorker.perform_async(:deploy, data[:build_id])
end
end
app/services/projects/update_service.rb
View file @ da246582
......@@ -17,6 +17,8 @@ module Projects
ensure_wiki_exists if enabling_wiki?
yield if block_given?
if project.update_attributes(params.except(:default_branch))
if project.previous_changes.include?('path')
project.rename_repo
......@@ -36,7 +38,7 @@ module Projects
end
def run_auto_devops_pipeline?
return false if project.repository.gitlab_ci_yml || !project.auto_devops.previous_changes.include?('enabled')
return false if project.repository.gitlab_ci_yml || !project.auto_devops&.previous_changes&.include?('enabled')
project.auto_devops.enabled? || (project.auto_devops.enabled.nil? && Gitlab::CurrentSettings.auto_devops_enabled?)
end
......@@ -53,8 +55,8 @@ module Projects
def changing_default_branch?
new_branch = params[:default_branch]
project.repository.exists? &&
new_branch && new_branch != project.default_branch
new_branch && project.repository.exists? &&
new_branch != project.default_branch
end
def enabling_wiki?
......
app/uploaders/object_storage.rb
View file @ da246582
......@@ -10,8 +10,6 @@ module ObjectStorage
UnknownStoreError = Class.new(StandardError)
ObjectStorageUnavailable = Class.new(StandardError)
DIRECT_UPLOAD_TIMEOUT = 4.hours
DIRECT_UPLOAD_EXPIRE_OFFSET = 15.minutes
TMP_UPLOAD_PATH = 'tmp/uploads'.freeze
module Store
......@@ -157,9 +155,9 @@ module ObjectStorage
model_class.uploader_options.dig(mount_point, :mount_on) || mount_point
end
def workhorse_authorize
def workhorse_authorize(has_length:, maximum_size: nil)
{
RemoteObject: workhorse_remote_upload_options,
RemoteObject: workhorse_remote_upload_options(has_length: has_length, maximum_size: maximum_size),
TempPath: workhorse_local_upload_path
}.compact
end
......@@ -168,23 +166,16 @@ module ObjectStorage
File.join(self.root, TMP_UPLOAD_PATH)
end
def workhorse_remote_upload_options
def workhorse_remote_upload_options(has_length:, maximum_size: nil)
return unless self.object_store_enabled?
return unless self.direct_upload_enabled?
id = [CarrierWave.generate_cache_id, SecureRandom.hex].join('-')
upload_path = File.join(TMP_UPLOAD_PATH, id)
connection = ::Fog::Storage.new(self.object_store_credentials)
expire_at = Time.now + DIRECT_UPLOAD_TIMEOUT + DIRECT_UPLOAD_EXPIRE_OFFSET
options = { 'Content-Type' => 'application/octet-stream' }
direct_upload = ObjectStorage::DirectUpload.new(self.object_store_credentials, remote_store_path, upload_path,
has_length: has_length, maximum_size: maximum_size)
{
ID: id,
Timeout: DIRECT_UPLOAD_TIMEOUT,
GetURL: connection.get_object_url(remote_store_path, upload_path, expire_at),
DeleteURL: connection.delete_object_url(remote_store_path, upload_path, expire_at),
StoreURL: connection.put_object_url(remote_store_path, upload_path, expire_at, options)
}
direct_upload.to_hash.merge(ID: id)
end
end
......
app/views/projects/_home_panel.html.haml
View file @ da246582
......@@ -42,6 +42,10 @@
.project-clone-holder
= render "shared/clone_panel"
- if show_xcode_link?(@project)
.project-action-button.project-xcode.inline
= render "projects/buttons/xcode_link"
- if current_user
- if can?(current_user, :download_code, @project)
= render 'projects/buttons/download', project: @project, ref: @ref
......
app/views/projects/buttons/_xcode_link.html.haml 0 → 100644
View file @ da246582
%a.btn.btn-default{ href: xcode_uri_to_repo(@project) }
= _("Open in Xcode")
changelogs/unreleased/45820-add-xcode-link.yml 0 → 100644
View file @ da246582
---
title: Add Open in Xcode link for xcode repositories
merge_request:
author:
type: added
changelogs/unreleased/46452-nomethoderror-undefined-method-previous_changes-for-nil-nilclass.yml 0 → 100644
View file @ da246582
---
title: Check for nil AutoDevOps when saving project CI/CD settings.
merge_request: 19190
author:
type: fixed
changelogs/unreleased/add-background-migrations-for-not-archived-traces.yml 0 → 100644
View file @ da246582
---
title: Add background migrations for archiving legacy job traces
merge_request: 19194
author:
type: performance
changelogs/unreleased/gh-importer-transactions.yml 0 → 100644
View file @ da246582
---
title: Move PR IO operations out of a transaction
merge_request:
author:
type: performance
changelogs/unreleased/jivl-smarter-system-notes.yml 0 → 100644
View file @ da246582
---
title: Add support for smarter system notes
merge_request: 17164
author:
type: changed
changelogs/unreleased/optimise-pages-service-calling.yml 0 → 100644
View file @ da246582
---
title: Optimise PagesWorker usage
merge_request:
author:
type: performance
changelogs/unreleased/presigned-multipart-uploads.yml 0 → 100644
View file @ da246582
---
title: Support direct_upload with S3 Multipart uploads
merge_request:
author:
type: added
changelogs/unreleased/rails5-fix-46236.yml 0 → 100644
View file @ da246582
---
title: Support rails5 in postgres indexes function and fix some migrations
merge_request: 19400
author: Jasper Maes
type: fixed
changelogs/unreleased/remove-unused-query-in-hooks.yml 0 → 100644
View file @ da246582
---
title: Remove unused running_or_pending_build_count
merge_request:
author:
type: performance
changelogs/unreleased/sh-add-uncached-query-limiter.yml 0 → 100644
View file @ da246582
---
title: Remove N+1 query for author in issues API
merge_request:
author:
type: performance
changelogs/unreleased/sh-fix-pipeline-jobs-nplus-one.yml 0 → 100644
View file @ da246582
---
title: Eliminate N+1 queries for CI job artifacts in /api/prjoects/:id/pipelines/:pipeline_id/jobs
merge_request:
author:
type: performance
config/initializers/artifacts_direct_upload_support.rb deleted 100644 → 0
View file @ 809a50fc
artifacts_object_store = Gitlab.config.artifacts.object_store
if artifacts_object_store.enabled &&
artifacts_object_store.direct_upload &&
artifacts_object_store.connection&.provider.to_s != 'Google'
raise "Only 'Google' is supported as a object storage provider when 'direct_upload' of artifacts is used"
end
config/initializers/direct_upload_support.rb 0 → 100644
View file @ da246582
class DirectUploadsValidator
SUPPORTED_DIRECT_UPLOAD_PROVIDERS = %w(Google AWS).freeze
ValidationError = Class.new(StandardError)
def verify!(object_store)
return unless object_store.enabled
return unless object_store.direct_upload
return if SUPPORTED_DIRECT_UPLOAD_PROVIDERS.include?(object_store.connection&.provider.to_s)
raise ValidationError, "Only #{SUPPORTED_DIRECT_UPLOAD_PROVIDERS.join(',')} are supported as a object storage provider when 'direct_upload' is used"
end
end
DirectUploadsValidator.new.tap do |validator|
[Gitlab.config.artifacts, Gitlab.config.uploads, Gitlab.config.lfs].each do |uploader|
validator.verify!(uploader.object_store)
end
end
config/initializers/postgresql_opclasses_support.rb
View file @ da246582
......@@ -107,8 +107,15 @@ module ActiveRecord
result.map do |row|
index_name = row[0]
unique = row[1] == 't'
unique = if Gitlab.rails5?
row[1]
else
row[1] == 't'
end
indkey = row[2].split(" ")
if Gitlab.rails5?
indkey = indkey.map(&:to_i)
end
inddef = row[3]
oid = row[4]
......
db/migrate/20160226114608_add_trigram_indexes_for_searching.rb
View file @ da246582
......@@ -37,7 +37,12 @@ class AddTrigramIndexesForSearching < ActiveRecord::Migration
res = execute("SELECT true AS enabled FROM pg_available_extensions WHERE name = 'pg_trgm' AND installed_version IS NOT NULL;")
row = res.first
row && row['enabled'] == 't' ? true : false
check = if Gitlab.rails5?
true
else
't'
end
row && row['enabled'] == check ? true : false
end
def create_trigrams_extension
......
db/migrate/20170622135728_add_unique_constraint_to_ci_variables.rb
View file @ da246582
......@@ -2,12 +2,13 @@ class AddUniqueConstraintToCiVariables < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
INDEX_NAME = 'index_ci_variables_on_project_id_and_key_and_environment_scope'
disable_ddl_transaction!
def up
unless this_index_exists?
add_concurrent_index(:ci_variables, columns, name: index_name, unique: true)
add_concurrent_index(:ci_variables, columns, name: INDEX_NAME, unique: true)
end
end
......@@ -18,21 +19,17 @@ class AddUniqueConstraintToCiVariables < ActiveRecord::Migration
add_concurrent_index(:ci_variables, :project_id)
end
remove_concurrent_index(:ci_variables, columns, name: index_name)
remove_concurrent_index(:ci_variables, columns, name: INDEX_NAME)
end
end
private
def this_index_exists?
index_exists?(:ci_variables, columns, name: index_name)
index_exists?(:ci_variables, columns, name: INDEX_NAME)
end
def columns
@columns ||= [:project_id, :key, :environment_scope]
end
def index_name
'index_ci_variables_on_project_id_and_key_and_environment_scope'
end
end
db/migrate/20171106155656_turn_issues_due_date_index_to_partial_index.rb
View file @ da246582
......@@ -20,9 +20,7 @@ class TurnIssuesDueDateIndexToPartialIndex < ActiveRecord::Migration
name: NEW_INDEX_NAME
)
# We set the column name to nil as otherwise Rails will ignore the custom
# index name and remove the wrong index.
remove_concurrent_index(:issues, nil, name: OLD_INDEX_NAME)
remove_concurrent_index_by_name(:issues, OLD_INDEX_NAME)
end
def down
......@@ -32,6 +30,6 @@ class TurnIssuesDueDateIndexToPartialIndex < ActiveRecord::Migration
name: OLD_INDEX_NAME
)
remove_concurrent_index(:issues, nil, name: NEW_INDEX_NAME)
remove_concurrent_index_by_name(:issues, NEW_INDEX_NAME)
end
end
db/migrate/20180201110056_add_foreign_keys_to_todos.rb
View file @ da246582
......@@ -31,7 +31,7 @@ class AddForeignKeysToTodos < ActiveRecord::Migration
end
def down
remove_foreign_key :todos, :users
remove_foreign_key :todos, column: :user_id
remove_foreign_key :todos, column: :author_id
remove_foreign_key :todos, :notes
end
......
db/post_migrate/20180529152628_schedule_to_archive_legacy_traces.rb 0 → 100644
View file @ da246582
class ScheduleToArchiveLegacyTraces < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
BATCH_SIZE = 5000
BACKGROUND_MIGRATION_CLASS = 'ArchiveLegacyTraces'
disable_ddl_transaction!
class Build < ActiveRecord::Base
include EachBatch
self.table_name = 'ci_builds'
self.inheritance_column = :_type_disabled # Disable STI
scope :type_build, -> { where(type: 'Ci::Build') }
scope :finished, -> { where(status: [:success, :failed, :canceled]) }
scope :without_archived_trace, -> do
where('NOT EXISTS (SELECT 1 FROM ci_job_artifacts WHERE ci_builds.id = ci_job_artifacts.job_id AND ci_job_artifacts.file_type = 3)')
end
end
def up
queue_background_migration_jobs_by_range_at_intervals(
::ScheduleToArchiveLegacyTraces::Build.type_build.finished.without_archived_trace,
BACKGROUND_MIGRATION_CLASS,
5.minutes,
batch_size: BATCH_SIZE)
end
def down
# noop
end
end
doc/administration/job_artifacts.md
View file @ da246582
......@@ -94,6 +94,7 @@ _The artifacts are stored by default in
> Available in [GitLab Premium](https://about.gitlab.com/products/) and
[GitLab.com Silver](https://about.gitlab.com/gitlab-com/).
> Since version 10.6, available in [GitLab CE](https://about.gitlab.com/products/)
> Since version 11.0, we support direct_upload to S3.
If you don't want to use the local disk where GitLab is installed to store the
artifacts, you can use an object storage like AWS S3 instead.
......@@ -108,7 +109,7 @@ For source installations the following settings are nested under `artifacts:` an
|---------|-------------|---------|
| `enabled` | Enable/disable object storage | `false` |
| `remote_directory` | The bucket name where Artifacts will be stored| |
| `direct_upload` | Set to true to enable direct upload of Artifacts without the need of local shared storage. Option may be removed once we decide to support only single storage for all files. Currently only `Google` provider is supported | `false` |
| `direct_upload` | Set to true to enable direct upload of Artifacts without the need of local shared storage. Option may be removed once we decide to support only single storage for all files. | `false` |
| `background_upload` | Set to false to disable automatic upload. Option may be removed once upload is direct to S3 | `true` |
| `proxy_download` | Set to true to enable proxying all files served. Option allows to reduce egress traffic as this allows clients to download directly from remote storage instead of proxying all data | `false` |
| `connection` | Various connection options described below | |
......
doc/development/query_recorder.md
View file @ da246582
......@@ -22,6 +22,19 @@ As an example you might create 5 issues in between counts, which would cause the
> **Note:** In some cases the query count might change slightly between runs for unrelated reasons. In this case you might need to test `exceed_query_limit(control_count + acceptable_change)`, but this should be avoided if possible.
## Cached queries
By default, QueryRecorder will ignore cached queries in the count. However, it may be better to count
all queries to avoid introducing an N+1 query that may be masked by the statement cache. To do this,
pass the `skip_cached` variable to `QueryRecorder` and use the `exceed_all_query_limit` matcher:
it "avoids N+1 database queries" do
control_count = ActiveRecord::QueryRecorder.new(skip_cached: false) { visit_some_page }.count
create_list(:issue, 5)
expect { visit_some_page }.not_to exceed_all_query_limit(control_count)
end
```
## Finding the source of the query
It may be useful to identify the source of the queries by looking at the call backtrace.
......
doc/topics/autodevops/index.md
View file @ da246582
......@@ -497,10 +497,10 @@ also be customized, and you can easily use a [custom buildpack](#custom-buildpac
| `CANARY_ENABLED` | From GitLab 11.0, this variable can be used to define a [deploy policy for canary environments](#deploy-policy-for-canary-environments). |
| `INCREMENTAL_ROLLOUT_ENABLED`| From GitLab 10.8, this variable can be used to enable an [incremental rollout](#incremental-rollout-to-production) of your application for the production environment. |
| `TEST_DISABLED` | From GitLab 11.0, this variable can be used to disable the `test` job. If the variable is present, the job will not be created. |
| `CODEQUALITY_DISABLED` | From GitLab 11.0, this variable can be used to disable the `codequality` job. If the variable is present, the job will not be created. |
| `CODE_QUALITY_DISABLED` | From GitLab 11.0, this variable can be used to disable the `code_quality` job. If the variable is present, the job will not be created. |
| `SAST_DISABLED` | From GitLab 11.0, this variable can be used to disable the `sast` job. If the variable is present, the job will not be created. |
| `DEPENDENCY_SCANNING_DISABLED` | From GitLab 11.0, this variable can be used to disable the `dependency_scanning` job. If the variable is present, the job will not be created. |
| `CONTAINER_SCANNING_DISABLED` | From GitLab 11.0, this variable can be used to disable the `sast:container` job. If the variable is present, the job will not be created. |
| `CONTAINER_SCANNING_DISABLED` | From GitLab 11.0, this variable can be used to disable the `container_scanning` job. If the variable is present, the job will not be created. |
| `REVIEW_DISABLED` | From GitLab 11.0, this variable can be used to disable the `review` and the manual `review:stop` job. If the variable is present, these jobs will not be created. |
| `DAST_DISABLED` | From GitLab 11.0, this variable can be used to disable the `dast` job. If the variable is present, the job will not be created. |
| `PERFORMANCE_DISABLED` | From GitLab 11.0, this variable can be used to disable the `performance` job. If the variable is present, the job will not be created. |
......
lib/api/issues.rb
View file @ da246582
......@@ -16,7 +16,7 @@ module API
args[:scope] = args[:scope].underscore if args[:scope]
issues = IssuesFinder.new(current_user, args).execute
.preload(:assignees, :labels, :notes, :timelogs, :project)
.preload(:assignees, :labels, :notes, :timelogs, :project, :author)
issues.reorder(args[:order_by] => args[:sort])
end
......
lib/api/jobs.rb
View file @ da246582
......@@ -54,6 +54,7 @@ module API
pipeline = user_project.pipelines.find(params[:pipeline_id])
builds = pipeline.builds
builds = filter_builds(builds, params[:scope])
builds = builds.preload(:job_artifacts_archive)
present paginate(builds), with: Entities::Job
end
......
lib/api/runner.rb
View file @ da246582
......@@ -205,7 +205,7 @@ module API
status 200
content_type Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE
JobArtifactUploader.workhorse_authorize
JobArtifactUploader.workhorse_authorize(has_length: false, maximum_size: max_artifacts_size)
end
desc 'Upload artifacts for job' do
......
lib/gitlab/auth.rb
View file @ da246582
......@@ -240,7 +240,7 @@ module Gitlab
return unless login == 'gitlab-ci-token'
return unless password
build = ::Ci::Build.running.find_by_token(password)
build = find_build_by_token(password)
return unless build
return unless build.project.builds_enabled?
......@@ -301,6 +301,12 @@ module Gitlab
REGISTRY_SCOPES
end
private
def find_build_by_token(token)
::Ci::Build.running.find_by_token(token)
end
end
end
end
lib/gitlab/background_migration/archive_legacy_traces.rb 0 → 100644
View file @ da246582
# frozen_string_literal: true
# rubocop:disable Metrics/AbcSize
# rubocop:disable Style/Documentation
module Gitlab
module BackgroundMigration
class ArchiveLegacyTraces
def perform(start_id, stop_id)
# This background migration directly refers to ::Ci::Build model which is defined in application code.
# In general, migration code should be isolated as much as possible in order to be idempotent.
# However, `archive!` method is too complicated to be replicated by coping its subsequent code.
# So we chose a way to use ::Ci::Build directly and we don't change the `archive!` method until 11.1
::Ci::Build.finished.without_archived_trace
.where(id: start_id..stop_id).find_each do |build|
begin
build.trace.archive!
rescue => e
Rails.logger.error "Failed to archive live trace. id: #{build.id} message: #{e.message}"
end
end
end
end
end
end
lib/gitlab/git/repository.rb
View file @ da246582
......@@ -1397,6 +1397,11 @@ module Gitlab
def write_config(full_path:)
return unless full_path.present?
# This guard avoids Gitaly log/error spam
unless exists?
raise NoRepository, 'repository does not exist'
end
gitaly_migrate(:write_config) do |is_enabled|
if is_enabled
gitaly_repository_client.write_config(full_path: full_path)
......
lib/gitlab/github_import/importer/pull_request_importer.rb
View file @ da246582
......@@ -22,15 +22,22 @@ module Gitlab
end
def execute
if (mr_id = create_merge_request)
issuable_finder.cache_database_id(mr_id)
mr, already_exists = create_merge_request
if mr
insert_git_data(mr, already_exists)
issuable_finder.cache_database_id(mr.id)
end
end
# Creates the merge request and returns its ID.
#
# This method will return `nil` if the merge request could not be
# created.
# created, otherwise it will return an Array containing the following
# values:
#
# 1. A MergeRequest instance.
# 2. A boolean indicating if the MR already exists.
def create_merge_request
author_id, author_found = user_finder.author_id_for(pull_request)
......@@ -69,21 +76,42 @@ module Gitlab
merge_request_id = GithubImport
.insert_and_return_id(attributes, project.merge_requests)
merge_request = project.merge_requests.find(merge_request_id)
# These fields are set so we can create the correct merge request
# diffs.
merge_request.source_branch_sha = pull_request.source_branch_sha
merge_request.target_branch_sha = pull_request.target_branch_sha
merge_request.keep_around_commit
merge_request.merge_request_diffs.create
merge_request.id
[project.merge_requests.find(merge_request_id), false]
end
rescue ActiveRecord::InvalidForeignKey
# It's possible the project has been deleted since scheduling this
# job. In this case we'll just skip creating the merge request.
[]
rescue ActiveRecord::RecordNotUnique
# It's possible we previously created the MR, but failed when updating
# the Git data. In this case we'll just continue working on the
# existing row.
[project.merge_requests.find_by(iid: pull_request.iid), true]
end
def insert_git_data(merge_request, already_exists = false)
# These fields are set so we can create the correct merge request
# diffs.
merge_request.source_branch_sha = pull_request.source_branch_sha
merge_request.target_branch_sha = pull_request.target_branch_sha
merge_request.keep_around_commit
# MR diffs normally use an "after_save" hook to pull data from Git.
# All of this happens in the transaction started by calling
# create/save/etc. This in turn can lead to these transactions being
# held open for much longer than necessary. To work around this we
# first save the diff, then populate it.
diff =
if already_exists
merge_request.merge_request_diffs.take
else
merge_request.merge_request_diffs.build
end
diff.importing = true
diff.save
diff.save_git_content
end
end
end
......
lib/gitlab/utils/override.rb
View file @ da246582
......@@ -87,18 +87,28 @@ module Gitlab
end
def included(base = nil)
return super if base.nil? # Rails concern, ignoring it
super
queue_verification(base)
end
alias_method :prepended, :included
def extended(mod)
super
queue_verification(mod.singleton_class)
end
def queue_verification(base)
return unless ENV['STATIC_VERIFICATION']
if base.is_a?(Class) # We could check for Class in `override`
# This could be `nil` if `override` was never called
Override.extensions[self]&.add_class(base)
end
end
alias_method :prepended, :included
def self.extensions
@extensions ||= {}
end
......
lib/object_storage/direct_upload.rb 0 → 100644
View file @ da246582
module ObjectStorage
#
# The DirectUpload c;ass generates a set of presigned URLs
# that can be used to upload data to object storage from untrusted component: Workhorse, Runner?
#
# For Google it assumes that the platform supports variable Content-Length.
#
# For AWS it initiates Multipart Upload and presignes a set of part uploads.
# Class calculates the best part size to be able to upload up to asked maximum size.
# The number of generated parts will never go above 100,
# but we will always try to reduce amount of generated parts.
# The part size is rounded-up to 5MB.
#
class DirectUpload
include Gitlab::Utils::StrongMemoize
TIMEOUT = 4.hours
EXPIRE_OFFSET = 15.minutes
MAXIMUM_MULTIPART_PARTS = 100
MINIMUM_MULTIPART_SIZE = 5.megabytes
attr_reader :credentials, :bucket_name, :object_name
attr_reader :has_length, :maximum_size
def initialize(credentials, bucket_name, object_name, has_length:, maximum_size: nil)
unless has_length
raise ArgumentError, 'maximum_size has to be specified if length is unknown' unless maximum_size
end
@credentials = credentials
@bucket_name = bucket_name
@object_name = object_name
@has_length = has_length
@maximum_size = maximum_size
end
def to_hash
{
Timeout: TIMEOUT,
GetURL: get_url,
StoreURL: store_url,
DeleteURL: delete_url,
MultipartUpload: multipart_upload_hash
}.compact
end
def multipart_upload_hash
return unless requires_multipart_upload?
{
PartSize: rounded_multipart_part_size,
PartURLs: multipart_part_urls,
CompleteURL: multipart_complete_url,
AbortURL: multipart_abort_url
}
end
def provider
credentials[:provider].to_s
end
# Implements https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectGET.html
def get_url
connection.get_object_url(bucket_name, object_name, expire_at)
end
# Implements https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectDELETE.html
def delete_url
connection.delete_object_url(bucket_name, object_name, expire_at)
end
# Implements https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPUT.html
def store_url
connection.put_object_url(bucket_name, object_name, expire_at, upload_options)
end
def multipart_part_urls
Array.new(number_of_multipart_parts) do |part_index|
multipart_part_upload_url(part_index + 1)
end
end
# Implements https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadUploadPart.html
def multipart_part_upload_url(part_number)
connection.signed_url({
method: 'PUT',
bucket_name: bucket_name,
object_name: object_name,
query: { uploadId: upload_id, partNumber: part_number },
headers: upload_options
}, expire_at)
end
# Implements https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadComplete.html
def multipart_complete_url
connection.signed_url({
method: 'POST',
bucket_name: bucket_name,
object_name: object_name,
query: { uploadId: upload_id },
headers: { 'Content-Type' => 'application/xml' }
}, expire_at)
end
# Implements https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadAbort.html
def multipart_abort_url
connection.signed_url({
method: 'DELETE',
bucket_name: bucket_name,
object_name: object_name,
query: { uploadId: upload_id }
}, expire_at)
end
private
def rounded_multipart_part_size
# round multipart_part_size up to minimum_mulitpart_size
(multipart_part_size + MINIMUM_MULTIPART_SIZE - 1) / MINIMUM_MULTIPART_SIZE * MINIMUM_MULTIPART_SIZE
end
def multipart_part_size
maximum_size / number_of_multipart_parts
end
def number_of_multipart_parts
[
# round maximum_size up to minimum_mulitpart_size
(maximum_size + MINIMUM_MULTIPART_SIZE - 1) / MINIMUM_MULTIPART_SIZE,
MAXIMUM_MULTIPART_PARTS
].min
end
def aws?
provider == 'AWS'
end
def requires_multipart_upload?
aws? && !has_length
end
def upload_id
return unless requires_multipart_upload?
strong_memoize(:upload_id) do
new_upload = connection.initiate_multipart_upload(bucket_name, object_name)
new_upload.body["UploadId"]
end
end
def expire_at
strong_memoize(:expire_at) do
Time.now + TIMEOUT + EXPIRE_OFFSET
end
end
def upload_options
{ 'Content-Type' => 'application/octet-stream' }
end
def connection
@connection ||= ::Fog::Storage.new(credentials)
end
end
end
lib/tasks/gitlab/traces.rake
View file @ da246582
......@@ -8,9 +8,7 @@ namespace :gitlab do
logger = Logger.new(STDOUT)
logger.info('Archiving legacy traces')
Ci::Build.finished
.where('NOT EXISTS (?)',
Ci::JobArtifact.select(1).trace.where('ci_builds.id = ci_job_artifacts.job_id'))
Ci::Build.finished.without_archived_trace
.order(id: :asc)
.find_in_batches(batch_size: 1000) do |jobs|
job_ids = jobs.map { |job| [job.id] }
......
spec/features/projects/jobs/user_browses_job_spec.rb
View file @ da246582
......@@ -32,8 +32,6 @@ describe 'User browses a job', :js do
page.within('.erased') do
expect(page).to have_content('Job has been erased')
end
expect(build.project.running_or_pending_build_count).to eq(build.project.builds.running_or_pending.count(:all))
end
context 'with a failed job' do
......
spec/helpers/projects_helper_spec.rb
View file @ da246582
......@@ -435,4 +435,46 @@ describe ProjectsHelper do
expect(helper.send(:git_user_name)).to eq('John \"A\" Doe53')
end
end
describe 'show_xcode_link' do
let!(:project) { create(:project) }
let(:mac_ua) { 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36' }
let(:ios_ua) { 'Mozilla/5.0 (iPad; CPU OS 5_1_1 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9B206 Safari/7534.48.3' }
context 'when the repository is xcode compatible' do
before do
allow(project.repository).to receive(:xcode_project?).and_return(true)
end
it 'returns false if the visitor is not using macos' do
allow(helper).to receive(:browser).and_return(Browser.new(ios_ua))
expect(helper.show_xcode_link?(project)).to eq(false)
end
it 'returns true if the visitor is using macos' do
allow(helper).to receive(:browser).and_return(Browser.new(mac_ua))
expect(helper.show_xcode_link?(project)).to eq(true)
end
end
context 'when the repository is not xcode compatible' do
before do
allow(project.repository).to receive(:xcode_project?).and_return(false)
end
it 'returns false if the visitor is not using macos' do
allow(helper).to receive(:browser).and_return(Browser.new(ios_ua))
expect(helper.show_xcode_link?(project)).to eq(false)
end
it 'returns false if the visitor is using macos' do
allow(helper).to receive(:browser).and_return(Browser.new(mac_ua))
expect(helper.show_xcode_link?(project)).to eq(false)
end
end
end
end
spec/initializers/artifacts_direct_upload_support_spec.rb deleted 100644 → 0
View file @ 809a50fc
require 'spec_helper'
describe 'Artifacts direct upload support' do
subject do
load Rails.root.join('config/initializers/artifacts_direct_upload_support.rb')
end
let(:connection) do
{ provider: provider }
end
before do
stub_artifacts_setting(
object_store: {
enabled: enabled,
direct_upload: direct_upload,
connection: connection
})
end
context 'when object storage is enabled' do
let(:enabled) { true }
context 'when direct upload is enabled' do
let(:direct_upload) { true }
context 'when provider is Google' do
let(:provider) { 'Google' }
it 'succeeds' do
expect { subject }.not_to raise_error
end
end
context 'when connection is empty' do
let(:connection) { nil }
it 'raises an error' do
expect { subject }.to raise_error /object storage provider when 'direct_upload' of artifacts is used/
end
end
context 'when other provider is used' do
let(:provider) { 'AWS' }
it 'raises an error' do
expect { subject }.to raise_error /object storage provider when 'direct_upload' of artifacts is used/
end
end
end
context 'when direct upload is disabled' do
let(:direct_upload) { false }
let(:provider) { 'AWS' }
it 'succeeds' do
expect { subject }.not_to raise_error
end
end
end
context 'when object storage is disabled' do
let(:enabled) { false }
let(:direct_upload) { false }
let(:provider) { 'AWS' }
it 'succeeds' do
expect { subject }.not_to raise_error
end
end
end
spec/initializers/direct_upload_support_spec.rb 0 → 100644
View file @ da246582
require 'spec_helper'
describe 'Direct upload support' do
subject do
load Rails.root.join('config/initializers/direct_upload_support.rb')
end
where(:config_name) do
%w(lfs artifacts uploads)
end
with_them do
let(:connection) do
{ provider: provider }
end
let(:object_store) do
{
enabled: enabled,
direct_upload: direct_upload,
connection: connection
}
end
before do
allow(Gitlab.config).to receive_messages(to_settings(config_name => {
object_store: object_store
}))
end
context 'when object storage is enabled' do
let(:enabled) { true }
context 'when direct upload is enabled' do
let(:direct_upload) { true }
context 'when provider is AWS' do
let(:provider) { 'AWS' }
it 'succeeds' do
expect { subject }.not_to raise_error
end
end
context 'when provider is Google' do
let(:provider) { 'Google' }
it 'succeeds' do
expect { subject }.not_to raise_error
end
end
context 'when connection is empty' do
let(:connection) { nil }
it 'raises an error' do
expect { subject }.to raise_error /are supported as a object storage provider when 'direct_upload' is used/
end
end
context 'when other provider is used' do
let(:provider) { 'Rackspace' }
it 'raises an error' do
expect { subject }.to raise_error /are supported as a object storage provider when 'direct_upload' is used/
end
end
end
context 'when direct upload is disabled' do
let(:direct_upload) { false }
let(:provider) { 'AWS' }
it 'succeeds' do
expect { subject }.not_to raise_error
end
end
end
context 'when object storage is disabled' do
let(:enabled) { false }
let(:direct_upload) { false }
let(:provider) { 'Rackspace' }
it 'succeeds' do
expect { subject }.not_to raise_error
end
end
end
end
spec/javascripts/notes/mock_data.js
View file @ da246582
This diff is collapsed.
spec/javascripts/notes/stores/collapse_utils_spec.js 0 → 100644
View file @ da246582
import {
isDescriptionSystemNote,
changeDescriptionNote,
getTimeDifferenceMinutes,
collapseSystemNotes,
} from '~/notes/stores/collapse_utils';
import {
notesWithDescriptionChanges,
collapsedSystemNotes,
} from '../mock_data';
describe('Collapse utils', () => {
const mockSystemNote = {
note: 'changed the description',
note_html: '<p dir="auto">changed the description</p>',
system: true,
created_at: '2018-05-14T21:28:00.000Z',
};
it('checks if a system note is of a description type', () => {
expect(isDescriptionSystemNote(mockSystemNote)).toEqual(true);
});
it('returns false when a system note is not a description type', () => {
expect(isDescriptionSystemNote(Object.assign({}, mockSystemNote, { note: 'foo' }))).toEqual(false);
});
it('changes the description to contain the number of changed times', () => {
const changedNote = changeDescriptionNote(mockSystemNote, 3, 5);
expect(changedNote.times_updated).toEqual(3);
expect(changedNote.note_html.trim()).toContain('<p dir="auto">changed the description 3 times within 5 minutes </p>');
});
it('gets the time difference between two notes', () => {
const anotherSystemNote = {
created_at: '2018-05-14T21:33:00.000Z',
};
expect(getTimeDifferenceMinutes(mockSystemNote, anotherSystemNote)).toEqual(5);
});
it('collapses all description system notes made within 10 minutes or less from each other', () => {
expect(collapseSystemNotes(notesWithDescriptionChanges)).toEqual(collapsedSystemNotes);
});
});
spec/javascripts/notes/stores/getters_spec.js
View file @ da246582
import * as getters from '~/notes/stores/getters';
import { notesDataMock, userDataMock, noteableDataMock, individualNote } from '../mock_data';
import { notesDataMock, userDataMock, noteableDataMock, individualNote, collapseNotesMock } from '../mock_data';
describe('Getters Notes Store', () => {
let state;
beforeEach(() => {
state = {
notes: [individualNote],
......@@ -20,6 +21,22 @@ describe('Getters Notes Store', () => {
});
});
describe('Collapsed notes', () => {
const stateCollapsedNotes = {
notes: collapseNotesMock,
targetNoteHash: 'hash',
lastFetchedAt: 'timestamp',
notesData: notesDataMock,
userData: userDataMock,
noteableData: noteableDataMock,
};
it('should return a single system note when a description was updated multiple times', () => {
expect(getters.notes(stateCollapsedNotes).length).toEqual(1);
});
});
describe('targetNoteHash', () => {
it('should return `targetNoteHash`', () => {
expect(getters.targetNoteHash(state)).toEqual('hash');
......
spec/lib/gitlab/background_migration/archive_legacy_traces_spec.rb 0 → 100644
View file @ da246582
require 'spec_helper'
describe Gitlab::BackgroundMigration::ArchiveLegacyTraces, :migration, schema: 20180529152628 do
include TraceHelpers
let(:namespaces) { table(:namespaces) }
let(:projects) { table(:projects) }
let(:builds) { table(:ci_builds) }
let(:job_artifacts) { table(:ci_job_artifacts) }
before do
namespaces.create!(id: 123, name: 'gitlab1', path: 'gitlab1')
projects.create!(id: 123, name: 'gitlab1', path: 'gitlab1', namespace_id: 123)
@build = builds.create!(id: 1, project_id: 123, status: 'success', type: 'Ci::Build')
end
context 'when trace file exsits at the right place' do
before do
create_legacy_trace(@build, 'trace in file')
end
it 'correctly archive legacy traces' do
expect(job_artifacts.count).to eq(0)
expect(File.exist?(legacy_trace_path(@build))).to be_truthy
described_class.new.perform(1, 1)
expect(job_artifacts.count).to eq(1)
expect(File.exist?(legacy_trace_path(@build))).to be_falsy
expect(File.read(archived_trace_path(job_artifacts.first))).to eq('trace in file')
end
end
context 'when trace file does not exsits at the right place' do
it 'does not raise errors nor create job artifact' do
expect { described_class.new.perform(1, 1) }.not_to raise_error
expect(job_artifacts.count).to eq(0)
end
end
context 'when trace data exsits in database' do
before do
create_legacy_trace_in_db(@build, 'trace in db')
end
it 'correctly archive legacy traces' do
expect(job_artifacts.count).to eq(0)
expect(@build.read_attribute(:trace)).not_to be_empty
described_class.new.perform(1, 1)
@build.reload
expect(job_artifacts.count).to eq(1)
expect(@build.read_attribute(:trace)).to be_nil
expect(File.read(archived_trace_path(job_artifacts.first))).to eq('trace in db')
end
end
end
spec/lib/gitlab/git/repository_spec.rb
View file @ da246582
......@@ -2002,6 +2002,18 @@ describe Gitlab::Git::Repository, seed_helper: true do
expect(config).to include("fullpath = #{repository_path}")
end
end
context 'repository does not exist' do
it 'raises NoRepository and does not call Gitaly WriteConfig' do
repository = Gitlab::Git::Repository.new('default', 'does/not/exist.git', '')
expect(repository.gitaly_repository_client).not_to receive(:write_config)
expect do
repository.write_config(full_path: 'foo/bar.git')
end.to raise_error(Gitlab::Git::Repository::NoRepository)
end
end
end
context "when gitaly_write_config is enabled" do
......
spec/lib/gitlab/github_import/importer/issue_importer_spec.rb
View file @ da246582
......@@ -180,12 +180,12 @@ describe Gitlab::GithubImport::Importer::IssueImporter, :clean_gitlab_redis_cach
allow(importer.user_finder)
.to receive(:user_id_for)
.ordered.with(issue.assignees[0])
.with(issue.assignees[0])
.and_return(4)
allow(importer.user_finder)
.to receive(:user_id_for)
.ordered.with(issue.assignees[1])
.with(issue.assignees[1])
.and_return(5)
expect(Gitlab::Database)
......
spec/lib/gitlab/github_import/importer/pull_request_importer_spec.rb
View file @ da246582
......@@ -40,13 +40,19 @@ describe Gitlab::GithubImport::Importer::PullRequestImporter, :clean_gitlab_redi
describe '#execute' do
it 'imports the pull request' do
mr = double(:merge_request, id: 10)
expect(importer)
.to receive(:create_merge_request)
.and_return(10)
.and_return([mr, false])
expect(importer)
.to receive(:insert_git_data)
.with(mr, false)
expect_any_instance_of(Gitlab::GithubImport::IssuableFinder)
.to receive(:cache_database_id)
.with(10)
.with(mr.id)
importer.execute
end
......@@ -99,18 +105,11 @@ describe Gitlab::GithubImport::Importer::PullRequestImporter, :clean_gitlab_redi
importer.create_merge_request
end
it 'returns the ID of the created merge request' do
id = importer.create_merge_request
expect(id).to be_a_kind_of(Numeric)
end
it 'creates the merge request diffs' do
importer.create_merge_request
mr = project.merge_requests.take
it 'returns the created merge request' do
mr, exists = importer.create_merge_request
expect(mr.merge_request_diffs.exists?).to eq(true)
expect(mr).to be_instance_of(MergeRequest)
expect(exists).to eq(false)
end
end
......@@ -217,5 +216,65 @@ describe Gitlab::GithubImport::Importer::PullRequestImporter, :clean_gitlab_redi
expect { importer.create_merge_request }.not_to raise_error
end
end
context 'when the merge request already exists' do
before do
allow(importer.user_finder)
.to receive(:author_id_for)
.with(pull_request)
.and_return([user.id, true])
allow(importer.user_finder)
.to receive(:assignee_id_for)
.with(pull_request)
.and_return(user.id)
end
it 'returns the existing merge request' do
mr1, exists1 = importer.create_merge_request
mr2, exists2 = importer.create_merge_request
expect(mr2).to eq(mr1)
expect(exists1).to eq(false)
expect(exists2).to eq(true)
end
end
end
describe '#insert_git_data' do
before do
allow(importer.milestone_finder)
.to receive(:id_for)
.with(pull_request)
.and_return(milestone.id)
allow(importer.user_finder)
.to receive(:author_id_for)
.with(pull_request)
.and_return([user.id, true])
allow(importer.user_finder)
.to receive(:assignee_id_for)
.with(pull_request)
.and_return(user.id)
end
it 'creates the merge request diffs' do
mr, exists = importer.create_merge_request
importer.insert_git_data(mr, exists)
expect(mr.merge_request_diffs.exists?).to eq(true)
end
it 'creates the merge request diff commits' do
mr, exists = importer.create_merge_request
importer.insert_git_data(mr, exists)
diff = mr.merge_request_diffs.take
expect(diff.merge_request_diff_commits.exists?).to eq(true)
end
end
end
spec/lib/gitlab/utils/override_spec.rb
View file @ da246582
require 'spec_helper'
require 'fast_spec_helper'
describe Gitlab::Utils::Override do
let(:base) { Struct.new(:good) }
let(:base) do
Struct.new(:good) do
def self.good
0
end
end
end
let(:derived) { Class.new(base).tap { |m| m.extend described_class } }
let(:extension) { Module.new.tap { |m| m.extend described_class } }
......@@ -9,6 +15,14 @@ describe Gitlab::Utils::Override do
let(:prepending_class) { base.tap { |m| m.prepend extension } }
let(:including_class) { base.tap { |m| m.include extension } }
let(:prepending_class_methods) do
base.tap { |m| m.singleton_class.prepend extension }
end
let(:extending_class_methods) do
base.tap { |m| m.extend extension }
end
let(:klass) { subject }
def good(mod)
......@@ -36,7 +50,7 @@ describe Gitlab::Utils::Override do
shared_examples 'checking as intended' do
it 'checks ok for overriding method' do
good(subject)
result = klass.new(0).good
result = instance.good
expect(result).to eq(1)
described_class.verify!
......@@ -45,7 +59,25 @@ describe Gitlab::Utils::Override do
it 'raises NotImplementedError when it is not overriding anything' do
expect do
bad(subject)
klass.new(0).bad
instance.bad
described_class.verify!
end.to raise_error(NotImplementedError)
end
end
shared_examples 'checking as intended, nothing was overridden' do
it 'raises NotImplementedError because it is not overriding it' do
expect do
good(subject)
instance.good
described_class.verify!
end.to raise_error(NotImplementedError)
end
it 'raises NotImplementedError when it is not overriding anything' do
expect do
bad(subject)
instance.bad
described_class.verify!
end.to raise_error(NotImplementedError)
end
......@@ -54,7 +86,7 @@ describe Gitlab::Utils::Override do
shared_examples 'nothing happened' do
it 'does not complain when it is overriding something' do
good(subject)
result = klass.new(0).good
result = instance.good
expect(result).to eq(1)
described_class.verify!
......@@ -62,7 +94,7 @@ describe Gitlab::Utils::Override do
it 'does not complain when it is not overriding anything' do
bad(subject)
result = klass.new(0).bad
result = instance.bad
expect(result).to eq(true)
described_class.verify!
......@@ -75,83 +107,97 @@ describe Gitlab::Utils::Override do
end
describe '#override' do
context 'when STATIC_VERIFICATION is set' do
before do
stub_env('STATIC_VERIFICATION', 'true')
end
context 'when instance is klass.new(0)' do
let(:instance) { klass.new(0) }
context 'when subject is a class' do
subject { derived }
context 'when STATIC_VERIFICATION is set' do
before do
stub_env('STATIC_VERIFICATION', 'true')
end
it_behaves_like 'checking as intended'
end
context 'when subject is a class' do
subject { derived }
it_behaves_like 'checking as intended'
end
context 'when subject is a module, and class is prepending it' do
subject { extension }
let(:klass) { prepending_class }
it_behaves_like 'checking as intended'
end
context 'when subject is a module, and class is prepending it' do
subject { extension }
let(:klass) { prepending_class }
context 'when subject is a module, and class is including it' do
subject { extension }
let(:klass) { including_class }
it_behaves_like 'checking as intended'
it_behaves_like 'checking as intended, nothing was overridden'
end
end
context 'when subject is a module, and class is including it' do
subject { extension }
let(:klass) { including_class }
context 'when STATIC_VERIFICATION is not set' do
before do
stub_env('STATIC_VERIFICATION', nil)
end
it 'raises NotImplementedError because it is not overriding it' do
expect do
good(subject)
klass.new(0).good
described_class.verify!
end.to raise_error(NotImplementedError)
context 'when subject is a class' do
subject { derived }
it_behaves_like 'nothing happened'
end
it 'raises NotImplementedError when it is not overriding anything' do
expect do
bad(subject)
klass.new(0).bad
described_class.verify!
end.to raise_error(NotImplementedError)
context 'when subject is a module, and class is prepending it' do
subject { extension }
let(:klass) { prepending_class }
it_behaves_like 'nothing happened'
end
end
end
end
context 'when STATIC_VERIFICATION is not set' do
before do
stub_env('STATIC_VERIFICATION', nil)
end
context 'when subject is a module, and class is including it' do
subject { extension }
let(:klass) { including_class }
context 'when subject is a class' do
subject { derived }
it 'does not complain when it is overriding something' do
good(subject)
result = instance.good
it_behaves_like 'nothing happened'
end
expect(result).to eq(0)
described_class.verify!
end
context 'when subject is a module, and class is prepending it' do
subject { extension }
let(:klass) { prepending_class }
it 'does not complain when it is not overriding anything' do
bad(subject)
result = instance.bad
it_behaves_like 'nothing happened'
expect(result).to eq(true)
described_class.verify!
end
end
end
end
context 'when subject is a module, and class is including it' do
subject { extension }
let(:klass) { including_class }
context 'when instance is klass' do
let(:instance) { klass }
it 'does not complain when it is overriding something' do
good(subject)
result = klass.new(0).good
context 'when STATIC_VERIFICATION is set' do
before do
stub_env('STATIC_VERIFICATION', 'true')
end
expect(result).to eq(0)
described_class.verify!
end
context 'when subject is a module, and class is prepending it' do
subject { extension }
let(:klass) { prepending_class_methods }
it 'does not complain when it is not overriding anything' do
bad(subject)
result = klass.new(0).bad
it_behaves_like 'checking as intended'
end
expect(result).to eq(true)
described_class.verify!
context 'when subject is a module, and class is extending it' do
subject { extension }
let(:klass) { extending_class_methods }
it_behaves_like 'checking as intended, nothing was overridden'
end
end
end
end
......
spec/lib/object_storage/direct_upload_spec.rb 0 → 100644
View file @ da246582
require 'spec_helper'
describe ObjectStorage::DirectUpload do
let(:credentials) do
{
provider: 'AWS',
aws_access_key_id: 'AWS_ACCESS_KEY_ID',
aws_secret_access_key: 'AWS_SECRET_ACCESS_KEY'
}
end
let(:storage_url) { 'https://uploads.s3.amazonaws.com/' }
let(:bucket_name) { 'uploads' }
let(:object_name) { 'tmp/uploads/my-file' }
let(:maximum_size) { 1.gigabyte }
let(:direct_upload) { described_class.new(credentials, bucket_name, object_name, has_length: has_length, maximum_size: maximum_size) }
describe '#has_length' do
context 'is known' do
let(:has_length) { true }
let(:maximum_size) { nil }
it "maximum size is not required" do
expect { direct_upload }.not_to raise_error
end
end
context 'is unknown' do
let(:has_length) { false }
context 'and maximum size is specified' do
let(:maximum_size) { 1.gigabyte }
it "does not raise an error" do
expect { direct_upload }.not_to raise_error
end
end
context 'and maximum size is not specified' do
let(:maximum_size) { nil }
it "raises an error" do
expect { direct_upload }.to raise_error /maximum_size has to be specified if length is unknown/
end
end
end
end
describe '#to_hash' do
subject { direct_upload.to_hash }
shared_examples 'a valid upload' do
it "returns valid structure" do
expect(subject).to have_key(:Timeout)
expect(subject[:GetURL]).to start_with(storage_url)
expect(subject[:StoreURL]).to start_with(storage_url)
expect(subject[:DeleteURL]).to start_with(storage_url)
end
end
shared_examples 'a valid upload with multipart data' do
before do
stub_object_storage_multipart_init(storage_url, "myUpload")
end
it_behaves_like 'a valid upload'
it "returns valid structure" do
expect(subject).to have_key(:MultipartUpload)
expect(subject[:MultipartUpload]).to have_key(:PartSize)
expect(subject[:MultipartUpload][:PartURLs]).to all(start_with(storage_url))
expect(subject[:MultipartUpload][:PartURLs]).to all(include('uploadId=myUpload'))
expect(subject[:MultipartUpload][:CompleteURL]).to start_with(storage_url)
expect(subject[:MultipartUpload][:CompleteURL]).to include('uploadId=myUpload')
expect(subject[:MultipartUpload][:AbortURL]).to start_with(storage_url)
expect(subject[:MultipartUpload][:AbortURL]).to include('uploadId=myUpload')
end
end
shared_examples 'a valid upload without multipart data' do
it_behaves_like 'a valid upload'
it "returns valid structure" do
expect(subject).not_to have_key(:MultipartUpload)
end
end
context 'when AWS is used' do
context 'when length is known' do
let(:has_length) { true }
it_behaves_like 'a valid upload without multipart data'
end
context 'when length is unknown' do
let(:has_length) { false }
it_behaves_like 'a valid upload with multipart data' do
context 'when maximum upload size is 10MB' do
let(:maximum_size) { 10.megabyte }
it 'returns only 2 parts' do
expect(subject[:MultipartUpload][:PartURLs].length).to eq(2)
end
it 'part size is mimimum, 5MB' do
expect(subject[:MultipartUpload][:PartSize]).to eq(5.megabyte)
end
end
context 'when maximum upload size is 12MB' do
let(:maximum_size) { 12.megabyte }
it 'returns only 3 parts' do
expect(subject[:MultipartUpload][:PartURLs].length).to eq(3)
end
it 'part size is rounded-up to 5MB' do
expect(subject[:MultipartUpload][:PartSize]).to eq(5.megabyte)
end
end
context 'when maximum upload size is 49GB' do
let(:maximum_size) { 49.gigabyte }
it 'returns maximum, 100 parts' do
expect(subject[:MultipartUpload][:PartURLs].length).to eq(100)
end
it 'part size is rounded-up to 5MB' do
expect(subject[:MultipartUpload][:PartSize]).to eq(505.megabyte)
end
end
end
end
end
context 'when Google is used' do
let(:credentials) do
{
provider: 'Google',
google_storage_access_key_id: 'GOOGLE_ACCESS_KEY_ID',
google_storage_secret_access_key: 'GOOGLE_SECRET_ACCESS_KEY'
}
end
let(:storage_url) { 'https://storage.googleapis.com/uploads/' }
context 'when length is known' do
let(:has_length) { true }
it_behaves_like 'a valid upload without multipart data'
end
context 'when length is unknown' do
let(:has_length) { false }
it_behaves_like 'a valid upload without multipart data'
end
end
end
end
spec/migrations/schedule_to_archive_legacy_traces_spec.rb 0 → 100644
View file @ da246582
require 'spec_helper'
require Rails.root.join('db', 'post_migrate', '20180529152628_schedule_to_archive_legacy_traces')
describe ScheduleToArchiveLegacyTraces, :migration do
include TraceHelpers
let(:namespaces) { table(:namespaces) }
let(:projects) { table(:projects) }
let(:builds) { table(:ci_builds) }
let(:job_artifacts) { table(:ci_job_artifacts) }
before do
namespaces.create!(id: 123, name: 'gitlab1', path: 'gitlab1')
projects.create!(id: 123, name: 'gitlab1', path: 'gitlab1', namespace_id: 123)
@build_success = builds.create!(id: 1, project_id: 123, status: 'success', type: 'Ci::Build')
@build_failed = builds.create!(id: 2, project_id: 123, status: 'failed', type: 'Ci::Build')
@builds_canceled = builds.create!(id: 3, project_id: 123, status: 'canceled', type: 'Ci::Build')
@build_running = builds.create!(id: 4, project_id: 123, status: 'running', type: 'Ci::Build')
create_legacy_trace(@build_success, 'This job is done')
create_legacy_trace(@build_failed, 'This job is done')
create_legacy_trace(@builds_canceled, 'This job is done')
create_legacy_trace(@build_running, 'This job is not done yet')
end
it 'correctly archive legacy traces' do
expect(job_artifacts.count).to eq(0)
expect(File.exist?(legacy_trace_path(@build_success))).to be_truthy
expect(File.exist?(legacy_trace_path(@build_failed))).to be_truthy
expect(File.exist?(legacy_trace_path(@builds_canceled))).to be_truthy
expect(File.exist?(legacy_trace_path(@build_running))).to be_truthy
migrate!
expect(job_artifacts.count).to eq(3)
expect(File.exist?(legacy_trace_path(@build_success))).to be_falsy
expect(File.exist?(legacy_trace_path(@build_failed))).to be_falsy
expect(File.exist?(legacy_trace_path(@builds_canceled))).to be_falsy
expect(File.exist?(legacy_trace_path(@build_running))).to be_truthy
expect(File.exist?(archived_trace_path(job_artifacts.where(job_id: @build_success.id).first))).to be_truthy
expect(File.exist?(archived_trace_path(job_artifacts.where(job_id: @build_failed.id).first))).to be_truthy
expect(File.exist?(archived_trace_path(job_artifacts.where(job_id: @builds_canceled.id).first))).to be_truthy
expect(job_artifacts.where(job_id: @build_running.id)).not_to be_exist
end
end
spec/models/ci/build_spec.rb
View file @ da246582
......@@ -2506,4 +2506,76 @@ describe Ci::Build do
end
end
end
describe 'pages deployments' do
set(:build) { create(:ci_build, project: project, user: user) }
context 'when job is "pages"' do
before do
build.name = 'pages'
end
context 'when pages are enabled' do
before do
allow(Gitlab.config.pages).to receive_messages(enabled: true)
end
it 'is marked as pages generator' do
expect(build).to be_pages_generator
end
context 'job succeeds' do
it "calls pages worker" do
expect(PagesWorker).to receive(:perform_async).with(:deploy, build.id)
build.success!
end
end
context 'job fails' do
it "does not call pages worker" do
expect(PagesWorker).not_to receive(:perform_async)
build.drop!
end
end
end
context 'when pages are disabled' do
before do
allow(Gitlab.config.pages).to receive_messages(enabled: false)
end
it 'is not marked as pages generator' do
expect(build).not_to be_pages_generator
end
context 'job succeeds' do
it "does not call pages worker" do
expect(PagesWorker).not_to receive(:perform_async)
build.success!
end
end
end
end
context 'when job is not "pages"' do
before do
build.name = 'other-job'
end
it 'is not marked as pages generator' do
expect(build).not_to be_pages_generator
end
context 'job succeeds' do
it "does not call pages worker" do
expect(PagesWorker).not_to receive(:perform_async)
build.success
end
end
end
end
end
spec/requests/api/issues_spec.rb
View file @ da246582
......@@ -630,15 +630,17 @@ describe API::Issues do
end
it 'avoids N+1 queries' do
control_count = ActiveRecord::QueryRecorder.new do
get api("/projects/#{project.id}/issues", user)
control_count = ActiveRecord::QueryRecorder.new(skip_cached: false) do
get api("/projects/#{project.id}/issues", user)
end.count
create(:issue, author: user, project: project)
create_list(:issue, 3, project: project)
expect do
get api("/projects/#{project.id}/issues", user)
end.not_to exceed_query_limit(control_count)
end.not_to exceed_all_query_limit(control_count)
end
it 'returns 404 when project does not exist' do
......
spec/requests/api/jobs_spec.rb
View file @ da246582
......@@ -177,6 +177,18 @@ describe API::Jobs do
json_response.each { |job| expect(job['pipeline']['id']).to eq(pipeline.id) }
end
end
it 'avoids N+1 queries' do
control_count = ActiveRecord::QueryRecorder.new(skip_cached: false) do
get api("/projects/#{project.id}/pipelines/#{pipeline.id}/jobs", api_user), query
end.count
3.times { create(:ci_build, :artifacts, pipeline: pipeline) }
expect do
get api("/projects/#{project.id}/pipelines/#{pipeline.id}/jobs", api_user), query
end.not_to exceed_all_query_limit(control_count)
end
end
context 'unauthorized user' do
......
spec/requests/api/runner_spec.rb
View file @ da246582
......@@ -1101,6 +1101,7 @@ describe API::Runner, :clean_gitlab_redis_shared_state do
expect(json_response['RemoteObject']).to have_key('GetURL')
expect(json_response['RemoteObject']).to have_key('StoreURL')
expect(json_response['RemoteObject']).to have_key('DeleteURL')
expect(json_response['RemoteObject']).to have_key('MultipartUpload')
end
end
......
spec/requests/lfs_http_spec.rb
View file @ da246582
......@@ -1021,6 +1021,7 @@ describe 'Git LFS API and storage' do
expect(json_response['RemoteObject']).to have_key('GetURL')
expect(json_response['RemoteObject']).to have_key('StoreURL')
expect(json_response['RemoteObject']).to have_key('DeleteURL')
expect(json_response['RemoteObject']).not_to have_key('MultipartUpload')
expect(json_response['LfsOid']).to eq(sample_oid)
expect(json_response['LfsSize']).to eq(sample_size)
end
......
spec/services/pages_service_spec.rb deleted 100644 → 0
View file @ 809a50fc
require 'spec_helper'
describe PagesService do
let(:build) { create(:ci_build) }
let(:data) { Gitlab::DataBuilder::Build.build(build) }
let(:service) { described_class.new(data) }
before do
allow(Gitlab.config.pages).to receive(:enabled).and_return(true)
end
context 'execute asynchronously for pages job' do
before do
build.name = 'pages'
end
context 'on success' do
before do
build.success
end
it 'executes worker' do
expect(PagesWorker).to receive(:perform_async)
service.execute
end
end
%w(pending running failed canceled).each do |status|
context "on #{status}" do
before do
build.status = status
end
it 'does not execute worker' do
expect(PagesWorker).not_to receive(:perform_async)
service.execute
end
end
end
end
context 'for other jobs' do
before do
build.name = 'other job'
build.success
end
it 'does not execute worker' do
expect(PagesWorker).not_to receive(:perform_async)
service.execute
end
end
end
spec/services/projects/update_service_spec.rb
View file @ da246582
......@@ -275,6 +275,10 @@ describe Projects::UpdateService do
it { is_expected.to eq(false) }
end
context 'when auto devops is nil' do
it { is_expected.to eq(false) }
end
context 'when auto devops is explicitly enabled' do
before do
project.create_auto_devops!(enabled: true)
......
spec/support/helpers/query_recorder.rb
View file @ da246582
module ActiveRecord
class QueryRecorder
attr_reader :log, :cached
attr_reader :log, :skip_cached, :cached
def initialize(&block)
def initialize(skip_cached: true, &block)
@log = []
@cached = []
@skip_cached = skip_cached
ActiveSupport::Notifications.subscribed(method(:callback), 'sql.active_record', &block)
end
......@@ -16,7 +17,7 @@ module ActiveRecord
def callback(name, start, finish, message_id, values)
show_backtrace(values) if ENV['QUERY_RECORDER_DEBUG']
if values[:name]&.include?("CACHE")
if values[:name]&.include?("CACHE") && skip_cached
@cached << values[:sql]
elsif !values[:name]&.include?("SCHEMA")
@log << values[:sql]
......
spec/support/helpers/stub_object_storage.rb
View file @ da246582
......@@ -45,4 +45,16 @@ module StubObjectStorage
remote_directory: 'uploads',
**params)
end
def stub_object_storage_multipart_init(endpoint, upload_id = "upload_id")
stub_request(:post, %r{\A#{endpoint}tmp/uploads/[a-z0-9-]*\?uploads\z})
.to_return status: 200, body: <<-EOS.strip_heredoc
<?xml version="1.0" encoding="UTF-8"?>
<InitiateMultipartUploadResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Bucket>example-bucket</Bucket>
<Key>example-object</Key>
<UploadId>#{upload_id}</UploadId>
</InitiateMultipartUploadResult>
EOS
end
end
spec/support/matchers/exceed_query_limit.rb
View file @ da246582
RSpec::Matchers.define :exceed_query_limit do |expected|
supports_block_expectations
match do |block|
@subject_block = block
actual_count > expected_count + threshold
end
failure_message_when_negated do |actual|
threshold_message = threshold > 0 ? " (+#{@threshold})" : ''
counts = "#{expected_count}#{threshold_message}"
"Expected a maximum of #{counts} queries, got #{actual_count}:\n\n#{log_message}"
end
module ExceedQueryLimitHelpers
def with_threshold(threshold)
@threshold = threshold
self
......@@ -43,7 +30,7 @@ RSpec::Matchers.define :exceed_query_limit do |expected|
end
def recorder
@recorder ||= ActiveRecord::QueryRecorder.new(&@subject_block)
@recorder ||= ActiveRecord::QueryRecorder.new(skip_cached: skip_cached, &@subject_block)
end
def count_queries(queries)
......@@ -61,4 +48,52 @@ RSpec::Matchers.define :exceed_query_limit do |expected|
@recorder.log_message
end
end
def skip_cached
true
end
def verify_count(&block)
@subject_block = block
actual_count > expected_count + threshold
end
def failure_message
threshold_message = threshold > 0 ? " (+#{@threshold})" : ''
counts = "#{expected_count}#{threshold_message}"
"Expected a maximum of #{counts} queries, got #{actual_count}:\n\n#{log_message}"
end
end
RSpec::Matchers.define :exceed_all_query_limit do |expected|
supports_block_expectations
include ExceedQueryLimitHelpers
match do |block|
verify_count(&block)
end
failure_message_when_negated do |actual|
failure_message
end
def skip_cached
false
end
end
# Excludes cached methods from the query count
RSpec::Matchers.define :exceed_query_limit do |expected|
supports_block_expectations
include ExceedQueryLimitHelpers
match do |block|
verify_count(&block)
end
failure_message_when_negated do |actual|
failure_message
end
end
spec/support/trace/trace_helpers.rb 0 → 100644
View file @ da246582
module TraceHelpers
def create_legacy_trace(build, content)
File.open(legacy_trace_path(build), 'wb') { |stream| stream.write(content) }
end
def create_legacy_trace_in_db(build, content)
build.update_column(:trace, content)
end
def legacy_trace_path(build)
legacy_trace_dir = File.join(Settings.gitlab_ci.builds_path,
build.created_at.utc.strftime("%Y_%m"),
build.project_id.to_s)
FileUtils.mkdir_p(legacy_trace_dir)
File.join(legacy_trace_dir, "#{build.id}.log")
end
def archived_trace_path(job_artifact)
disk_hash = Digest::SHA2.hexdigest(job_artifact.project_id.to_s)
creation_date = job_artifact.created_at.utc.strftime('%Y_%m_%d')
File.join(Gitlab.config.artifacts.path, disk_hash[0..1], disk_hash[2..3], disk_hash,
creation_date, job_artifact.job_id.to_s, job_artifact.id.to_s, 'job.log')
end
end
spec/uploaders/object_storage_spec.rb
View file @ da246582
......@@ -355,7 +355,10 @@ describe ObjectStorage do
end
describe '.workhorse_authorize' do
subject { uploader_class.workhorse_authorize }
let(:has_length) { true }
let(:maximum_size) { nil }
subject { uploader_class.workhorse_authorize(has_length: has_length, maximum_size: maximum_size) }
before do
# ensure that we use regular Fog libraries
......@@ -371,10 +374,6 @@ describe ObjectStorage do
expect(subject[:TempPath]).to start_with(uploader_class.root)
expect(subject[:TempPath]).to include(described_class::TMP_UPLOAD_PATH)
end
it "does not return remote store" do
is_expected.not_to have_key('RemoteObject')
end
end
shared_examples 'uses remote storage' do
......@@ -383,7 +382,7 @@ describe ObjectStorage do
expect(subject[:RemoteObject]).to have_key(:ID)
expect(subject[:RemoteObject]).to include(Timeout: a_kind_of(Integer))
expect(subject[:RemoteObject][:Timeout]).to be(ObjectStorage::DIRECT_UPLOAD_TIMEOUT)
expect(subject[:RemoteObject][:Timeout]).to be(ObjectStorage::DirectUpload::TIMEOUT)
expect(subject[:RemoteObject]).to have_key(:GetURL)
expect(subject[:RemoteObject]).to have_key(:DeleteURL)
expect(subject[:RemoteObject]).to have_key(:StoreURL)
......@@ -391,9 +390,31 @@ describe ObjectStorage do
expect(subject[:RemoteObject][:DeleteURL]).to include(described_class::TMP_UPLOAD_PATH)
expect(subject[:RemoteObject][:StoreURL]).to include(described_class::TMP_UPLOAD_PATH)
end
end
it "does not return local store" do
is_expected.not_to have_key('TempPath')
shared_examples 'uses remote storage with multipart uploads' do
it_behaves_like 'uses remote storage' do
it "returns multipart upload" do
is_expected.to have_key(:RemoteObject)
expect(subject[:RemoteObject]).to have_key(:MultipartUpload)
expect(subject[:RemoteObject][:MultipartUpload]).to have_key(:PartSize)
expect(subject[:RemoteObject][:MultipartUpload]).to have_key(:PartURLs)
expect(subject[:RemoteObject][:MultipartUpload]).to have_key(:CompleteURL)
expect(subject[:RemoteObject][:MultipartUpload]).to have_key(:AbortURL)
expect(subject[:RemoteObject][:MultipartUpload][:PartURLs]).to all(include(described_class::TMP_UPLOAD_PATH))
expect(subject[:RemoteObject][:MultipartUpload][:CompleteURL]).to include(described_class::TMP_UPLOAD_PATH)
expect(subject[:RemoteObject][:MultipartUpload][:AbortURL]).to include(described_class::TMP_UPLOAD_PATH)
end
end
end
shared_examples 'uses remote storage without multipart uploads' do
it_behaves_like 'uses remote storage' do
it "does not return multipart upload" do
is_expected.to have_key(:RemoteObject)
expect(subject[:RemoteObject]).not_to have_key(:MultipartUpload)
end
end
end
......@@ -416,6 +437,8 @@ describe ObjectStorage do
end
context 'uses AWS' do
let(:storage_url) { "https://uploads.s3-eu-central-1.amazonaws.com/" }
before do
expect(uploader_class).to receive(:object_store_credentials) do
{ provider: "AWS",
......@@ -425,18 +448,40 @@ describe ObjectStorage do
end
end
it_behaves_like 'uses remote storage' do
let(:storage_url) { "https://uploads.s3-eu-central-1.amazonaws.com/" }
context 'for known length' do
it_behaves_like 'uses remote storage without multipart uploads' do
it 'returns links for S3' do
expect(subject[:RemoteObject][:GetURL]).to start_with(storage_url)
expect(subject[:RemoteObject][:DeleteURL]).to start_with(storage_url)
expect(subject[:RemoteObject][:StoreURL]).to start_with(storage_url)
end
end
end
context 'for unknown length' do
let(:has_length) { false }
let(:maximum_size) { 1.gigabyte }
it 'returns links for S3' do
expect(subject[:RemoteObject][:GetURL]).to start_with(storage_url)
expect(subject[:RemoteObject][:DeleteURL]).to start_with(storage_url)
expect(subject[:RemoteObject][:StoreURL]).to start_with(storage_url)
before do
stub_object_storage_multipart_init(storage_url)
end
it_behaves_like 'uses remote storage with multipart uploads' do
it 'returns links for S3' do
expect(subject[:RemoteObject][:GetURL]).to start_with(storage_url)
expect(subject[:RemoteObject][:DeleteURL]).to start_with(storage_url)
expect(subject[:RemoteObject][:StoreURL]).to start_with(storage_url)
expect(subject[:RemoteObject][:MultipartUpload][:PartURLs]).to all(start_with(storage_url))
expect(subject[:RemoteObject][:MultipartUpload][:CompleteURL]).to start_with(storage_url)
expect(subject[:RemoteObject][:MultipartUpload][:AbortURL]).to start_with(storage_url)
end
end
end
end
context 'uses Google' do
let(:storage_url) { "https://storage.googleapis.com/uploads/" }
before do
expect(uploader_class).to receive(:object_store_credentials) do
{ provider: "Google",
......@@ -445,36 +490,71 @@ describe ObjectStorage do
end
end
it_behaves_like 'uses remote storage' do
let(:storage_url) { "https://storage.googleapis.com/uploads/" }
context 'for known length' do
it_behaves_like 'uses remote storage without multipart uploads' do
it 'returns links for Google Cloud' do
expect(subject[:RemoteObject][:GetURL]).to start_with(storage_url)
expect(subject[:RemoteObject][:DeleteURL]).to start_with(storage_url)
expect(subject[:RemoteObject][:StoreURL]).to start_with(storage_url)
end
end
end
context 'for unknown length' do
let(:has_length) { false }
let(:maximum_size) { 1.gigabyte }
it 'returns links for Google Cloud' do
expect(subject[:RemoteObject][:GetURL]).to start_with(storage_url)
expect(subject[:RemoteObject][:DeleteURL]).to start_with(storage_url)
expect(subject[:RemoteObject][:StoreURL]).to start_with(storage_url)
it_behaves_like 'uses remote storage without multipart uploads' do
it 'returns links for Google Cloud' do
expect(subject[:RemoteObject][:GetURL]).to start_with(storage_url)
expect(subject[:RemoteObject][:DeleteURL]).to start_with(storage_url)
expect(subject[:RemoteObject][:StoreURL]).to start_with(storage_url)
end
end
end
end
context 'uses GDK/minio' do
let(:storage_url) { "http://minio:9000/uploads/" }
before do
expect(uploader_class).to receive(:object_store_credentials) do
{ provider: "AWS",
aws_access_key_id: "AWS_ACCESS_KEY_ID",
aws_secret_access_key: "AWS_SECRET_ACCESS_KEY",
endpoint: 'http://127.0.0.1:9000',
endpoint: 'http://minio:9000',
path_style: true,
region: "gdk" }
end
end
it_behaves_like 'uses remote storage' do
let(:storage_url) { "http://127.0.0.1:9000/uploads/" }
context 'for known length' do
it_behaves_like 'uses remote storage without multipart uploads' do
it 'returns links for S3' do
expect(subject[:RemoteObject][:GetURL]).to start_with(storage_url)
expect(subject[:RemoteObject][:DeleteURL]).to start_with(storage_url)
expect(subject[:RemoteObject][:StoreURL]).to start_with(storage_url)
end
end
end
context 'for unknown length' do
let(:has_length) { false }
let(:maximum_size) { 1.gigabyte }
it 'returns links for S3' do
expect(subject[:RemoteObject][:GetURL]).to start_with(storage_url)
expect(subject[:RemoteObject][:DeleteURL]).to start_with(storage_url)
expect(subject[:RemoteObject][:StoreURL]).to start_with(storage_url)
before do
stub_object_storage_multipart_init(storage_url)
end
it_behaves_like 'uses remote storage with multipart uploads' do
it 'returns links for S3' do
expect(subject[:RemoteObject][:GetURL]).to start_with(storage_url)
expect(subject[:RemoteObject][:DeleteURL]).to start_with(storage_url)
expect(subject[:RemoteObject][:StoreURL]).to start_with(storage_url)
expect(subject[:RemoteObject][:MultipartUpload][:PartURLs]).to all(start_with(storage_url))
expect(subject[:RemoteObject][:MultipartUpload][:CompleteURL]).to start_with(storage_url)
expect(subject[:RemoteObject][:MultipartUpload][:AbortURL]).to start_with(storage_url)
end
end
end
end
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7