Commit 0c833498 authored by Douwe Maan's avatar Douwe Maan

Merge branch 'master' into rename-reply-by-email

parents ee028d9d d2c90d74
Please view this file on the master branch, on stable branches it's out of date. Please view this file on the master branch, on stable branches it's out of date.
v 8.0.0 (unreleased) v 8.0.0 (unreleased)
- Fix Markdown links not showing up in dashboard activity feed (Stan Hu)
- Fix HTML link that was improperly escaped in new user e-mail (Stan Hu) - Fix HTML link that was improperly escaped in new user e-mail (Stan Hu)
- Fix broken sort in merge request API (Stan Hu) - Fix broken sort in merge request API (Stan Hu)
- Bump rouge to 1.10.1 to remove warning noise and fix other syntax highlighting bugs (Stan Hu) - Bump rouge to 1.10.1 to remove warning noise and fix other syntax highlighting bugs (Stan Hu)
...@@ -42,6 +43,7 @@ v 8.0.0 (unreleased) ...@@ -42,6 +43,7 @@ v 8.0.0 (unreleased)
- Retrieving oauth token with LDAP credentials - Retrieving oauth token with LDAP credentials
- Load Application settings from running database unless env var USE_DB=false - Load Application settings from running database unless env var USE_DB=false
- Added Drone CI integration (Kirill Zaitsev) - Added Drone CI integration (Kirill Zaitsev)
- Fail builds if no .gitlab-ci.yml is found
- Refactored service API and added automatically service docs generator (Kirill Zaitsev) - Refactored service API and added automatically service docs generator (Kirill Zaitsev)
- Added web_url key project hook_attrs (Kirill Zaitsev) - Added web_url key project hook_attrs (Kirill Zaitsev)
- Add ability to get user information by ID of an SSH key via the API - Add ability to get user information by ID of an SSH key via the API
......
...@@ -45,7 +45,7 @@ module GitlabMarkdownHelper ...@@ -45,7 +45,7 @@ module GitlabMarkdownHelper
end end
def markdown(text, context = {}) def markdown(text, context = {})
context.merge!( context.reverse_merge!(
current_user: current_user, current_user: current_user,
path: @path, path: @path,
project: @project, project: @project,
...@@ -59,7 +59,7 @@ module GitlabMarkdownHelper ...@@ -59,7 +59,7 @@ module GitlabMarkdownHelper
# TODO (rspeicher): Remove all usages of this helper and just call `markdown` # TODO (rspeicher): Remove all usages of this helper and just call `markdown`
# with a custom pipeline depending on the content being rendered # with a custom pipeline depending on the content being rendered
def gfm(text, options = {}) def gfm(text, options = {})
options.merge!( options.reverse_merge!(
current_user: current_user, current_user: current_user,
path: @path, path: @path,
project: @project, project: @project,
......
...@@ -12,7 +12,7 @@ module Emails ...@@ -12,7 +12,7 @@ module Emails
to: recipient(recipient_id), to: recipient(recipient_id),
subject: subject("#{@commit.title} (#{@commit.short_id})")) subject: subject("#{@commit.title} (#{@commit.short_id})"))
SentNotification.record(@commit, recipient_id, reply_key) SentNotification.record_note(@note, recipient_id, reply_key)
end end
def note_issue_email(recipient_id, note_id) def note_issue_email(recipient_id, note_id)
...@@ -27,7 +27,7 @@ module Emails ...@@ -27,7 +27,7 @@ module Emails
to: recipient(recipient_id), to: recipient(recipient_id),
subject: subject("#{@issue.title} (##{@issue.iid})")) subject: subject("#{@issue.title} (##{@issue.iid})"))
SentNotification.record(@issue, recipient_id, reply_key) SentNotification.record_note(@note, recipient_id, reply_key)
end end
def note_merge_request_email(recipient_id, note_id) def note_merge_request_email(recipient_id, note_id)
...@@ -43,7 +43,7 @@ module Emails ...@@ -43,7 +43,7 @@ module Emails
to: recipient(recipient_id), to: recipient(recipient_id),
subject: subject("#{@merge_request.title} (##{@merge_request.iid})")) subject: subject("#{@merge_request.title} (##{@merge_request.iid})"))
SentNotification.record(@merge_request, recipient_id, reply_key) SentNotification.record_note(@note, recipient_id, reply_key)
end end
end end
end end
...@@ -236,7 +236,7 @@ module Ci ...@@ -236,7 +236,7 @@ module Ci
end end
def config_processor def config_processor
@config_processor ||= Ci::GitlabCiYamlProcessor.new(push_data[:ci_yaml_file] || project.generated_yaml_config) @config_processor ||= Ci::GitlabCiYamlProcessor.new(push_data[:ci_yaml_file])
rescue Ci::GitlabCiYamlProcessor::ValidationError => e rescue Ci::GitlabCiYamlProcessor::ValidationError => e
save_yaml_error(e.message) save_yaml_error(e.message)
nil nil
......
...@@ -428,7 +428,7 @@ class Project < ActiveRecord::Base ...@@ -428,7 +428,7 @@ class Project < ActiveRecord::Base
end end
def gitlab_ci? def gitlab_ci?
gitlab_ci_service && gitlab_ci_service.active gitlab_ci_service && gitlab_ci_service.active && gitlab_ci_project.present?
end end
def ci_services def ci_services
......
...@@ -8,6 +8,7 @@ ...@@ -8,6 +8,7 @@
# noteable_type :string(255) # noteable_type :string(255)
# recipient_id :integer # recipient_id :integer
# commit_id :string(255) # commit_id :string(255)
# line_code :string(255)
# reply_key :string(255) not null # reply_key :string(255) not null
# #
...@@ -21,6 +22,7 @@ class SentNotification < ActiveRecord::Base ...@@ -21,6 +22,7 @@ class SentNotification < ActiveRecord::Base
validates :noteable_id, presence: true, unless: :for_commit? validates :noteable_id, presence: true, unless: :for_commit?
validates :commit_id, presence: true, if: :for_commit? validates :commit_id, presence: true, if: :for_commit?
validates :line_code, format: { with: /\A[a-z0-9]+_\d+_\d+\Z/ }, allow_blank: true
class << self class << self
def reply_key def reply_key
...@@ -33,7 +35,7 @@ class SentNotification < ActiveRecord::Base ...@@ -33,7 +35,7 @@ class SentNotification < ActiveRecord::Base
find_by(reply_key: reply_key) find_by(reply_key: reply_key)
end end
def record(noteable, recipient_id, reply_key) def record(noteable, recipient_id, reply_key, params = {})
return unless reply_key return unless reply_key
noteable_id = nil noteable_id = nil
...@@ -44,7 +46,7 @@ class SentNotification < ActiveRecord::Base ...@@ -44,7 +46,7 @@ class SentNotification < ActiveRecord::Base
noteable_id = noteable.id noteable_id = noteable.id
end end
create( params.reverse_merge!(
project: noteable.project, project: noteable.project,
noteable_type: noteable.class.name, noteable_type: noteable.class.name,
noteable_id: noteable_id, noteable_id: noteable_id,
...@@ -52,6 +54,14 @@ class SentNotification < ActiveRecord::Base ...@@ -52,6 +54,14 @@ class SentNotification < ActiveRecord::Base
recipient_id: recipient_id, recipient_id: recipient_id,
reply_key: reply_key reply_key: reply_key
) )
create(params)
end
def record_note(note, recipient_id, reply_key, params = {})
params[:line_code] = note.line_code
record(note.noteable, recipient_id, reply_key, params)
end end
end end
......
...@@ -131,7 +131,7 @@ ...@@ -131,7 +131,7 @@
.checkbox .checkbox
= f.label :ci_enabled do = f.label :ci_enabled do
= f.check_box :ci_enabled = f.check_box :ci_enabled
Enable Continuous Integration Disable to prevent CI usage until rake ci:migrate is run (8.0 only)
.form-actions .form-actions
= f.submit 'Save', class: 'btn btn-primary' = f.submit 'Save', class: 'btn btn-primary'
Continuous Integration has been disabled. Please ask your administrator to enable it. Continuous Integration has been disabled for time of the migration.
class AddLineCodeToSentNotification < ActiveRecord::Migration
def change
add_column :sent_notifications, :line_code, :string
end
end
...@@ -11,7 +11,7 @@ ...@@ -11,7 +11,7 @@
# #
# It's strongly recommended that you check this file into your version control system. # It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema.define(version: 20150920010715) do ActiveRecord::Schema.define(version: 20150920161119) do
# These are extensions that must be enabled in order to support this database # These are extensions that must be enabled in order to support this database
enable_extension "plpgsql" enable_extension "plpgsql"
...@@ -623,6 +623,7 @@ ActiveRecord::Schema.define(version: 20150920010715) do ...@@ -623,6 +623,7 @@ ActiveRecord::Schema.define(version: 20150920010715) do
t.integer "recipient_id" t.integer "recipient_id"
t.string "commit_id" t.string "commit_id"
t.string "reply_key", null: false t.string "reply_key", null: false
t.string "line_code"
end end
add_index "sent_notifications", ["reply_key"], name: "index_sent_notifications_on_reply_key", unique: true, using: :btree add_index "sent_notifications", ["reply_key"], name: "index_sent_notifications_on_reply_key", unique: true, using: :btree
......
...@@ -10,7 +10,7 @@ See the documentation below for details on how to configure these services. ...@@ -10,7 +10,7 @@ See the documentation below for details on how to configure these services.
- [SAML](saml.md) Configure GitLab as a SAML 2.0 Service Provider - [SAML](saml.md) Configure GitLab as a SAML 2.0 Service Provider
- [Slack](slack.md) Integrate with the Slack chat service - [Slack](slack.md) Integrate with the Slack chat service
- [OAuth2 provider](oauth_provider.md) OAuth2 application creation - [OAuth2 provider](oauth_provider.md) OAuth2 application creation
- [Gmail](gitlab_buttons_in_gmail.md) Adds GitLab actions to messages - [Gmail actions buttons](gmail_action_buttons_for_gitlab.md) Adds GitLab actions to messages
GitLab Enterprise Edition contains [advanced JIRA support](http://doc.gitlab.com/ee/integration/jira.html) and [advanced Jenkins support](http://doc.gitlab.com/ee/integration/jenkins.html). GitLab Enterprise Edition contains [advanced JIRA support](http://doc.gitlab.com/ee/integration/jira.html) and [advanced Jenkins support](http://doc.gitlab.com/ee/integration/jenkins.html).
......
# Gmail actions buttons for GitLab
GitLab supports [Google actions in email](https://developers.google.com/gmail/markup/actions/actions-overview).
If correctly setup, emails that require an action will be marked in Gmail.
![gmail_actions_button.png](gmail_actions_button.png)
To get this functioning, you need to be registered with Google.
[See how to register with Google in this document.](https://developers.google.com/gmail/markup/registering-with-google)
*This process has a lot of steps so make sure that you fulfill all requirements set by Google.*
*Your application will be rejected by Google if you fail to do so.*
Pay close attention to:
* Email account used by GitLab to send notification emails needs to have "Consistent history of sending a high volume of mail from your domain (order of hundred emails a day minimum to Gmail) for a few weeks at least".
* "A very very low rate of spam complaints from users."
* Emails must be authenticated via DKIM or SPF
* Before sending the final form("Gmail Schema Whitelist Request"), you must send a real email from your production server. This means that you will have to find a way to send this email from the email address you are registering. You can do this by, for example, forwarding the real email from the email address you are registering or going into the rails console on the GitLab server and triggering the email sending from there.
You can check how it looks going through all the steps laid out in the "Registering with Google" doc in [this GitLab.com issue](https://gitlab.com/gitlab-org/gitlab-ce/issues/1517).
module Ci
module Migrate
class Builds
attr_reader :app_builds_dir, :backup_builds_tarball, :backup_dir
def initialize
@app_builds_dir = Settings.gitlab_ci.builds_path
@backup_dir = Gitlab.config.backup.path
@backup_builds_tarball = File.join(backup_dir, 'builds/builds.tar.gz')
end
def restore
backup_existing_builds_dir
FileUtils.mkdir_p(app_builds_dir, mode: 0700)
unless system('tar', '-C', app_builds_dir, '-zxvf', backup_builds_tarball)
abort 'Restore failed'.red
end
end
def backup_existing_builds_dir
timestamped_builds_path = File.join(app_builds_dir, '..', "builds.#{Time.now.to_i}")
if File.exists?(app_builds_dir)
FileUtils.mv(app_builds_dir, File.expand_path(timestamped_builds_path))
end
end
end
end
end
...@@ -9,32 +9,32 @@ module Ci ...@@ -9,32 +9,32 @@ module Ci
@config = YAML.load_file(File.join(Rails.root, 'config', 'database.yml'))[Rails.env] @config = YAML.load_file(File.join(Rails.root, 'config', 'database.yml'))[Rails.env]
end end
def restore(ci_dump) def restore
puts 'Deleting all CI related data ... ' decompress_rd, decompress_wr = IO.pipe
truncate_ci_tables decompress_pid = spawn(*%W(gzip -cd), out: decompress_wr, in: db_file_name)
decompress_wr.close
puts 'Restoring CI data ... ' restore_pid = case config["adapter"]
case config["adapter"]
when /^mysql/ then when /^mysql/ then
print "Restoring MySQL database #{config['database']} ... " $progress.print "Restoring MySQL database #{config['database']} ... "
# Workaround warnings from MySQL 5.6 about passwords on cmd line # Workaround warnings from MySQL 5.6 about passwords on cmd line
ENV['MYSQL_PWD'] = config["password"].to_s if config["password"] ENV['MYSQL_PWD'] = config["password"].to_s if config["password"]
system('mysql', *mysql_args, config['database'], in: ci_dump) spawn('mysql', *mysql_args, config['database'], in: decompress_rd)
when "postgresql" then when "postgresql" then
puts "Restoring PostgreSQL database #{config['database']} ... " $progress.print "Restoring PostgreSQL database #{config['database']} ... "
pg_env pg_env
system('psql', config['database'], '-f', ci_dump) spawn('psql', config['database'], in: decompress_rd)
end end
decompress_rd.close
success = [decompress_pid, restore_pid].all? { |pid| Process.waitpid(pid); $?.success? }
abort 'Restore failed' unless success
end end
protected protected
def truncate_ci_tables def db_file_name
c = ActiveRecord::Base.connection File.join(Gitlab.config.backup.path, 'db', 'database.sql.gz')
c.tables.select { |t| t.start_with?('ci_') }.each do |table|
puts "Deleting data from #{table}..."
c.execute("DELETE FROM #{table}")
end
end end
def mysql_args def mysql_args
......
module Ci
module Migrate
class Manager
VERSION = '8.0.0.pre'
def cleanup
$progress.print "Deleting tmp directories ... "
backup_contents.each do |dir|
next unless File.exist?(File.join(Gitlab.config.backup.path, dir))
if FileUtils.rm_rf(File.join(Gitlab.config.backup.path, dir))
$progress.puts "done".green
else
puts "deleting tmp directory '#{dir}' failed".red
abort 'Backup failed'
end
end
end
def unpack
Dir.chdir(Gitlab.config.backup.path)
# check for existing backups in the backup dir
file_list = Dir.glob("*_gitlab_ci_backup.tar").each.map { |f| f.split(/_/).first.to_i }
puts "no backups found" if file_list.count == 0
if file_list.count > 1 && ENV["BACKUP"].nil?
puts "Found more than one backup, please specify which one you want to restore:"
puts "rake gitlab:backup:restore BACKUP=timestamp_of_backup"
exit 1
end
tar_file = ENV["BACKUP"].nil? ? File.join("#{file_list.first}_gitlab_ci_backup.tar") : File.join(ENV["BACKUP"] + "_gitlab_ci_backup.tar")
unless File.exists?(tar_file)
puts "The specified CI backup doesn't exist!"
exit 1
end
$progress.print "Unpacking backup ... "
unless Kernel.system(*%W(tar -xf #{tar_file}))
puts "unpacking backup failed".red
exit 1
else
$progress.puts "done".green
end
ENV["VERSION"] = "#{settings[:db_version]}" if settings[:db_version].to_i > 0
# restoring mismatching backups can lead to unexpected problems
if settings[:gitlab_version] != VERSION
puts "GitLab CI version mismatch:".red
puts " Your current GitLab CI version (#{VERSION}) differs from the GitLab CI (#{settings[:gitlab_version]}) version in the backup!".red
exit 1
end
end
private
def backup_contents
["db", "builds", "backup_information.yml"]
end
def settings
@settings ||= YAML.load_file("backup_information.yml")
end
end
end
end
...@@ -4,45 +4,38 @@ module Ci ...@@ -4,45 +4,38 @@ module Ci
module Migrate module Migrate
class Tags class Tags
def restore def restore
puts 'Migrating tags for Runners... ' puts 'Inserting tags...'
list_objects('Runner').each do |id| connection.select_all('SELECT ci_tags.name FROM ci_tags').each do |tag|
putc '.' begin
runner = Ci::Runner.find_by_id(id) connection.execute("INSERT INTO tags (name) VALUES(#{ActiveRecord::Base::sanitize(tag['name'])})")
if runner rescue ActiveRecord::RecordNotUnique
tags = list_tags('Runner', id)
runner.update_attributes(tag_list: tags)
end end
end end
puts ''
puts 'Migrating tags for Builds... ' ActiveRecord::Base.transaction do
list_objects('Build').each do |id| puts 'Deleting old taggings...'
putc '.' connection.execute "DELETE FROM taggings WHERE context = 'tags' AND taggable_type LIKE 'Ci::%'"
build = Ci::Build.find_by_id(id)
if build
tags = list_tags('Build', id)
build.update_attributes(tag_list: tags)
end
end
puts ''
end
protected puts 'Inserting taggings...'
connection.execute(
'INSERT INTO taggings (taggable_type, taggable_id, tag_id, context) ' +
"SELECT CONCAT('Ci::', ci_taggings.taggable_type), ci_taggings.taggable_id, tags.id, 'tags' FROM ci_taggings " +
'JOIN ci_tags ON ci_tags.id = ci_taggings.tag_id ' +
'JOIN tags ON tags.name = ci_tags.name '
)
def list_objects(type) puts 'Resetting counters... '
ids = ActiveRecord::Base.connection.select_all( connection.execute(
"select distinct taggable_id from ci_taggings where taggable_type = #{ActiveRecord::Base::sanitize(type)}" 'UPDATE tags SET ' +
'taggings_count = (SELECT COUNT(*) FROM taggings WHERE tags.id = taggings.tag_id)'
) )
ids.map { |id| id['taggable_id'] } end
end end
def list_tags(type, id) protected
tags = ActiveRecord::Base.connection.select_all(
'select ci_tags.name from ci_tags ' + def connection
'join ci_taggings on ci_tags.id = ci_taggings.tag_id ' + ActiveRecord::Base.connection
"where taggable_type = #{ActiveRecord::Base::sanitize(type)} and taggable_id = #{ActiveRecord::Base::sanitize(id)} and context = 'tags'"
)
tags.map { |tag| tag['name'] }
end end
end end
end end
......
namespace :ci do namespace :ci do
desc 'GitLab | Import and migrate CI database' desc 'GitLab | Import and migrate CI database'
task migrate: :environment do task migrate: :environment do
warn_user_is_not_gitlab
configure_cron_mode
unless ENV['force'] == 'yes' unless ENV['force'] == 'yes'
puts "This will truncate all CI tables and restore it from provided backup." puts 'This will remove all CI related data and restore it from the provided backup.'
puts "You will lose any previous CI data stored in the database."
ask_to_continue ask_to_continue
puts "" puts ''
end end
Rake::Task["ci:migrate:db"].invoke # disable CI for time of migration
Rake::Task["ci:migrate:autoincrements"].invoke enable_ci(false)
Rake::Task["ci:migrate:tags"].invoke
Rake::Task["ci:migrate:services"].invoke # unpack archives
migrate = Ci::Migrate::Manager.new
migrate.unpack
Rake::Task['ci:migrate:db'].invoke
Rake::Task['ci:migrate:builds'].invoke
Rake::Task['ci:migrate:tags'].invoke
Rake::Task['ci:migrate:services'].invoke
# enable CI for time of migration
enable_ci(true)
migrate.cleanup
end end
namespace :migrate do namespace :migrate do
desc 'GitLab | Import CI database' desc 'GitLab | Import CI database'
task db: :environment do task db: :environment do
if ENV["CI_DUMP"].nil? configure_cron_mode
puts "No CI SQL dump specified:" $progress.puts 'Restoring database ... '.blue
puts "rake gitlab:backup:restore CI_DUMP=ci_dump.sql" Ci::Migrate::Database.new.restore
exit 1 $progress.puts 'done'.green
end end
ci_dump = ENV["CI_DUMP"] desc 'GitLab | Import CI builds'
unless File.exists?(ci_dump) task builds: :environment do
puts "The specified sql dump doesn't exist!" configure_cron_mode
exit 1 $progress.puts 'Restoring builds ... '.blue
end Ci::Migrate::Builds.new.restore
$progress.puts 'done'.green
::Ci::Migrate::Database.new.restore(ci_dump)
end end
desc 'GitLab | Migrate CI tags' desc 'GitLab | Migrate CI tags'
task tags: :environment do task tags: :environment do
configure_cron_mode
$progress.puts 'Migrating tags ... '.blue
::Ci::Migrate::Tags.new.restore ::Ci::Migrate::Tags.new.restore
$progress.puts 'done'.green
end end
desc 'GitLab | Migrate CI auto-increments' desc 'GitLab | Migrate CI auto-increments'
...@@ -56,8 +72,16 @@ namespace :ci do ...@@ -56,8 +72,16 @@ namespace :ci do
desc 'GitLab | Migrate CI services' desc 'GitLab | Migrate CI services'
task services: :environment do task services: :environment do
$progress.puts 'Migrating services ... '.blue
c = ActiveRecord::Base.connection c = ActiveRecord::Base.connection
c.execute("UPDATE ci_services SET type=CONCAT('Ci::', type) WHERE type NOT LIKE 'Ci::%'") c.execute("UPDATE ci_services SET type=CONCAT('Ci::', type) WHERE type NOT LIKE 'Ci::%'")
$progress.puts 'done'.green
end end
end end
def enable_ci(enabled)
settings = ApplicationSetting.current || ApplicationSetting.create_from_defaults
settings.ci_enabled = enabled
settings.save!
end
end end
...@@ -38,6 +38,17 @@ describe GitlabMarkdownHelper do ...@@ -38,6 +38,17 @@ describe GitlabMarkdownHelper do
expect(markdown(actual)).to match(expected) expect(markdown(actual)).to match(expected)
end end
end end
describe "override default project" do
let(:actual) { issue.to_reference }
let(:second_project) { create(:project) }
let(:second_issue) { create(:issue, project: second_project) }
it 'should link to the issue' do
expected = namespace_project_issue_path(second_project.namespace, second_project, second_issue)
expect(markdown(actual, project: second_project)).to match(expected)
end
end
end end
describe '#link_to_gfm' do describe '#link_to_gfm' do
......
...@@ -50,6 +50,19 @@ module Ci ...@@ -50,6 +50,19 @@ module Ci
end end
end end
it 'fails commits without .gitlab-ci.yml' do
result = service.execute(project,
ref: 'refs/heads/0_1',
before: '00000000',
after: '31das312',
ci_yaml_file: config,
commits: [ { message: 'Message' } ]
)
expect(result).to be_persisted
expect(result.builds.any?).to be_falsey
expect(result.status).to eq('failed')
end
describe :ci_skip? do describe :ci_skip? do
it "skips builds creation if there is [ci skip] tag in commit message" do it "skips builds creation if there is [ci skip] tag in commit message" do
commits = [{ message: "some message[ci skip]" }] commits = [{ message: "some message[ci skip]" }]
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment