Merge remote-tracking branch 'origin/master' into environments-and-deployments

# Conflicts:
#	db/schema.rb

Gemfile

@@ -210,6 +210,9 @@ gem 'mousetrap-rails', '~> 1.4.6'
 # Detect and convert string character encoding
 gem 'charlock_holmes', '~> 0.7.3'
 
+# Parse duration
+gem 'chronic_duration', '~> 0.10.6'
+
 gem "sass-rails", '~> 5.0.0'
 gem "coffee-rails", '~> 4.1.0'
 gem "uglifier", '~> 2.7.2'

Gemfile.lock

@@ -124,6 +124,8 @@ GEM
       mime-types (>= 1.16)
     cause (0.1)
     charlock_holmes (0.7.3)
+    chronic_duration (0.10.6)
+      numerizer (~> 0.1.1)
     chunky_png (1.3.5)
     cliver (0.3.2)
     coderay (1.1.0)
@@ -414,6 +416,7 @@ GEM
     nokogiri (1.6.8)
       mini_portile2 (~> 2.1.0)
       pkg-config (~> 1.1.7)
+    numerizer (0.1.1)
     oauth (0.4.7)
     oauth2 (1.0.0)
       faraday (>= 0.8, < 0.10)
@@ -839,6 +842,7 @@ DEPENDENCIES
   capybara-screenshot (~> 1.0.0)
   carrierwave (~> 0.10.0)
   charlock_holmes (~> 0.7.3)
+  chronic_duration (~> 0.10.6)
   coffee-rails (~> 4.1.0)
   connection_pool (~> 2.0)
   coveralls (~> 0.8.2)

app/assets/javascripts/ci/build.coffee

@@ -17,6 +17,8 @@ class @CiBuild
       .off 'resize.build'
       .on 'resize.build', @hideSidebar
 
+    @updateArtifactRemoveDate()
+
     if $('#build-trace').length
       @getInitialBuildTrace()
       @initScrollButtonAffix()
@@ -103,3 +105,10 @@ class @CiBuild
       $('.js-build-sidebar')
         .removeClass 'right-sidebar-collapsed'
         .addClass 'right-sidebar-expanded'
+
+  updateArtifactRemoveDate: ->
+    $date = $('.js-artifacts-remove')
+
+    if $date.length
+      date = $date.text()
+      $date.text $.timefor(new Date(date), ' ')

app/controllers/projects/artifacts_controller.rb

 class Projects::ArtifactsController < Projects::ApplicationController
   layout 'project'
   before_action :authorize_read_build!
+  before_action :authorize_update_build!, only: [:keep]
+  before_action :validate_artifacts!
 
   def download
     unless artifacts_file.file_storage?
       return redirect_to artifacts_file.url
     end
 
-    unless artifacts_file.exists?
-      return render_404
-    end
-
     send_file artifacts_file.path, disposition: 'attachment'
   end
 
   def browse
-    return render_404 unless build.artifacts?
-
     directory = params[:path] ? "#{params[:path]}/" : ''
     @entry = build.artifacts_metadata_entry(directory)
 
@@ -34,8 +30,17 @@ class Projects::ArtifactsController < Projects::ApplicationController
     end
   end
 
+  def keep
+    build.keep_artifacts!
+    redirect_to namespace_project_build_path(project.namespace, project, build)
+  end
+
   private
 
+  def validate_artifacts!
+    render_404 unless build.artifacts?
+  end
+
   def build
     @build ||= project.builds.find_by!(id: params[:build_id])
   end

app/helpers/time_helper.rb

@@ -20,7 +20,6 @@ module TimeHelper
     end
   end
 
-
   def date_from_to(from, to)
     "#{from.to_s(:short)} - #{to.to_s(:short)}"
   end

app/models/ci/build.rb

@@ -11,6 +11,8 @@ module Ci
 
     scope :unstarted, ->() { where(runner_id: nil) }
     scope :ignore_failures, ->() { where(allow_failure: false) }
+    scope :with_artifacts, ->() { where.not(artifacts_file: nil) }
+    scope :with_expired_artifacts, ->() { with_artifacts.where('artifacts_expire_at < ?', Time.now) }
 
     mount_uploader :artifacts_file, ArtifactUploader
     mount_uploader :artifacts_metadata, ArtifactUploader
@@ -324,7 +326,7 @@ module Ci
     end
 
     def artifacts?
-      artifacts_file.exists?
+      !artifacts_expired? && artifacts_file.exists?
     end
 
     def artifacts_metadata?
@@ -335,11 +337,15 @@ module Ci
       Gitlab::Ci::Build::Artifacts::Metadata.new(artifacts_metadata.path, path, **options).to_entry
     end
 
+    def erase_artifacts!
+      remove_artifacts_file!
+      remove_artifacts_metadata!
+    end
+
     def erase(opts = {})
       return false unless erasable?
 
-      remove_artifacts_file!
-      remove_artifacts_metadata!
+      erase_artifacts!
       erase_trace!
       update_erased!(opts[:erased_by])
     end
@@ -352,6 +358,25 @@ module Ci
       !self.erased_at.nil?
     end
 
+    def artifacts_expired?
+      artifacts_expire_at && artifacts_expire_at < Time.now
+    end
+
+    def artifacts_expire_in
+      artifacts_expire_at - Time.now if artifacts_expire_at
+    end
+
+    def artifacts_expire_in=(value)
+      self.artifacts_expire_at =
+        if value
+          Time.now + ChronicDuration.parse(value)
+        end
+    end
+
+    def keep_artifacts!
+      self.update(artifacts_expire_at: nil)
+    end
+
     private
 
     def erase_trace!
@@ -359,7 +384,7 @@ module Ci
     end
 
     def update_erased!(user = nil)
-      self.update(erased_by: user, erased_at: Time.now)
+      self.update(erased_by: user, erased_at: Time.now, artifacts_expire_at: nil)
     end
 
     def yaml_variables

app/views/layouts/nav/_project.html.haml

 - if current_user
+  - access = user_max_access_in_project(current_user.id, @project)
+  - can_edit = can?(current_user, :admin_project, @project)
   .controls
-    - access = user_max_access_in_project(current_user.id, @project)
-    - can_edit = can?(current_user, :admin_project, @project)
     .dropdown.project-settings-dropdown
       %a.dropdown-new.btn.btn-default#project-settings-button{href: '#', 'data-toggle' => 'dropdown'}
         = icon('cog')
         = icon('caret-down')
       %ul.dropdown-menu.dropdown-menu-align-right
-        = render 'layouts/nav/project_settings'
-        %li.divider
-        - if can_edit
-          %li
-            = link_to edit_project_path(@project) do
-              Edit Project
-        - if access
-          %li
-            = link_to leave_namespace_project_project_members_path(@project.namespace, @project),
-              data: { confirm: leave_project_message(@project) }, method: :delete, title: 'Leave project' do
-              Leave Project
+        = render 'layouts/nav/project_settings', access: access, can_edit: can_edit
+        - if can_edit || access
+          %li.divider
+          - if can_edit
+            %li
+              = link_to edit_project_path(@project) do
+                Edit Project
+          - if access
+            %li
+              = link_to leave_namespace_project_project_members_path(@project.namespace, @project),
+                data: { confirm: leave_project_message(@project) }, method: :delete, title: 'Leave project' do
+                Leave Project
 
 %div{ class: nav_control_class }
   %ul.nav-links.scrolling-tabs

app/views/layouts/nav/_project_settings.html.haml

@@ -3,43 +3,43 @@
     = link_to namespace_project_project_members_path(@project.namespace, @project), title: 'Members', class: 'team-tab tab' do
       %span
         Members
-
-- if @project.allowed_to_share_with_group?
-  = nav_link(controller: :group_links) do
-    = link_to namespace_project_group_links_path(@project.namespace, @project), title: "Groups" do
-      %span
-        Groups
-= nav_link(controller: :deploy_keys) do
-  = link_to namespace_project_deploy_keys_path(@project.namespace, @project), title: 'Deploy Keys' do
-    %span
-      Deploy Keys
-= nav_link(controller: :hooks) do
-  = link_to namespace_project_hooks_path(@project.namespace, @project), title: 'Webhooks' do
-    %span
-      Webhooks
-= nav_link(controller: :services) do
-  = link_to namespace_project_services_path(@project.namespace, @project), title: 'Services' do
-    %span
-      Services
-= nav_link(controller: :protected_branches) do
-  = link_to namespace_project_protected_branches_path(@project.namespace, @project), title: 'Protected Branches' do
-    %span
-      Protected Branches
-
-- if @project.builds_enabled?
-  = nav_link(controller: :runners) do
-    = link_to namespace_project_runners_path(@project.namespace, @project), title: 'Runners' do
+- if access && can_edit
+  - if @project.allowed_to_share_with_group?
+    = nav_link(controller: :group_links) do
+      = link_to namespace_project_group_links_path(@project.namespace, @project), title: "Groups" do
+        %span
+          Groups
+  = nav_link(controller: :deploy_keys) do
+    = link_to namespace_project_deploy_keys_path(@project.namespace, @project), title: 'Deploy Keys' do
       %span
-        Runners
-  = nav_link(controller: :variables) do
-    = link_to namespace_project_variables_path(@project.namespace, @project), title: 'Variables' do
+        Deploy Keys
+  = nav_link(controller: :hooks) do
+    = link_to namespace_project_hooks_path(@project.namespace, @project), title: 'Webhooks' do
       %span
-        Variables
-  = nav_link(controller: :triggers) do
-    = link_to namespace_project_triggers_path(@project.namespace, @project), title: 'Triggers' do
+        Webhooks
+  = nav_link(controller: :services) do
+    = link_to namespace_project_services_path(@project.namespace, @project), title: 'Services' do
       %span
-        Triggers
-  = nav_link(controller: :badges) do
-    = link_to namespace_project_badges_path(@project.namespace, @project), title: 'Badges' do
+        Services
+  = nav_link(controller: :protected_branches) do
+    = link_to namespace_project_protected_branches_path(@project.namespace, @project), title: 'Protected Branches' do
       %span
-        Badges
+        Protected Branches
+
+  - if @project.builds_enabled?
+    = nav_link(controller: :runners) do
+      = link_to namespace_project_runners_path(@project.namespace, @project), title: 'Runners' do
+        %span
+          Runners
+    = nav_link(controller: :variables) do
+      = link_to namespace_project_variables_path(@project.namespace, @project), title: 'Variables' do
+        %span
+          Variables
+    = nav_link(controller: :triggers) do
+      = link_to namespace_project_triggers_path(@project.namespace, @project), title: 'Triggers' do
+        %span
+          Triggers
+    = nav_link(controller: :badges) do
+      = link_to namespace_project_badges_path(@project.namespace, @project), title: 'Badges' do
+        %span
+          Badges

app/views/projects/builds/_sidebar.html.haml

@@ -11,19 +11,33 @@
       %p.build-detail-row
         #{@build.coverage}%
 
-  - if can?(current_user, :read_build, @project) && @build.artifacts?
+  - if can?(current_user, :read_build, @project) && (@build.artifacts? || @build.artifacts_expired?)
     .block{ class: ("block-first" if !@build.coverage) }
       .title
         Build artifacts
-      .btn-group.btn-group-justified{ role: :group }
-        = link_to download_namespace_project_build_artifacts_path(@project.namespace, @project, @build), class: 'btn btn-sm btn-default' do
-          Download
+      - if @build.artifacts_expired?
+        %p.build-detail-row
+          The artifacts were removed
+          #{time_ago_with_tooltip(@build.artifacts_expire_at)}
+      - elsif @build.artifacts_expire_at
+        %p.build-detail-row
+          The artifacts will be removed in
+          %span.js-artifacts-remove= @build.artifacts_expire_at
 
-        - if @build.artifacts_metadata?
-          = link_to browse_namespace_project_build_artifacts_path(@project.namespace, @project, @build), class: 'btn btn-sm btn-default' do
-            Browse
+      - if @build.artifacts?
+        .btn-group.btn-group-justified{ role: :group }
+          - if @build.artifacts_expire_at
+            = link_to keep_namespace_project_build_artifacts_path(@project.namespace, @project, @build), class: 'btn btn-sm btn-default', method: :post do
+              Keep
 
-  .block{ class: ("block-first" if !@build.coverage && !(can?(current_user, :read_build, @project) && @build.artifacts?)) }
+          = link_to download_namespace_project_build_artifacts_path(@project.namespace, @project, @build), class: 'btn btn-sm btn-default' do
+            Download
+
+          - if @build.artifacts_metadata?
+            = link_to browse_namespace_project_build_artifacts_path(@project.namespace, @project, @build), class: 'btn btn-sm btn-default' do
+              Browse
+
+  .block{ class: ("block-first" if !@build.coverage && !(can?(current_user, :read_build, @project) && (@build.artifacts? || @build.artifacts_expired?))) }
     .title
       Build details
       - if @build.retryable?

app/workers/expire_build_artifacts_worker.rb

+class ExpireBuildArtifactsWorker
+  include Sidekiq::Worker
+
+  def perform
+    Rails.logger.info 'Cleaning old build artifacts'
+
+    builds = Ci::Build.with_expired_artifacts
+    builds.find_each(batch_size: 50).each do |build|
+      Rails.logger.debug "Removing artifacts build #{build.id}..."
+      build.erase_artifacts!
+    end
+  end
+end

config/gitlab.yml.example

@@ -164,6 +164,9 @@ production: &base
     # Flag stuck CI builds as failed
     stuck_ci_builds_worker:
       cron: "0 0 * * *"
+    # Remove expired build artifacts
+    expire_build_artifacts_worker:
+      cron: "50 * * * *"
     # Periodically run 'git fsck' on all repositories. If started more than
     # once per hour you will have concurrent 'git fsck' jobs.
     repository_check_worker:

config/initializers/1_settings.rb

@@ -279,6 +279,9 @@ Settings['cron_jobs'] ||= Settingslogic.new({})
 Settings.cron_jobs['stuck_ci_builds_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['stuck_ci_builds_worker']['cron'] ||= '0 0 * * *'
 Settings.cron_jobs['stuck_ci_builds_worker']['job_class'] = 'StuckCiBuildsWorker'
+Settings.cron_jobs['expire_build_artifacts_worker'] ||= Settingslogic.new({})
+Settings.cron_jobs['expire_build_artifacts_worker']['cron'] ||= '50 * * * *'
+Settings.cron_jobs['expire_build_artifacts_worker']['job_class'] = 'ExpireBuildArtifactsWorker'
 Settings.cron_jobs['repository_check_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['repository_check_worker']['cron'] ||= '20 * * * *'
 Settings.cron_jobs['repository_check_worker']['job_class'] = 'RepositoryCheck::BatchWorker'

config/initializers/chronic_duration.rb

+ChronicDuration.raise_exceptions = true

config/routes.rb

@@ -724,6 +724,7 @@ Rails.application.routes.draw do
             get :download
             get :browse, path: 'browse(/*path)', format: false
             get :file, path: 'file/*path', format: false
+            post :keep
           end
         end
 

db/migrate/20160416182152_convert_award_note_to_emoji_award.rb

 # rubocop:disable all
 class ConvertAwardNoteToEmojiAward < ActiveRecord::Migration
-  def change
-    def up
-      execute "INSERT INTO award_emoji (awardable_type, awardable_id, user_id, name, created_at, updated_at) (SELECT noteable_type, noteable_id, author_id, note, created_at, updated_at FROM notes WHERE is_award = true)"
+  disable_ddl_transaction!
+
+  def up
+    if Gitlab::Database.postgresql?
+      migrate_postgresql
+    else
+      migrate_mysql
+    end
+  end
+
+  def down
+    add_column :notes, :is_award, :boolean
 
+    # This migration does NOT move the awards on notes, if the table is dropped in another migration, these notes will be lost.
+    execute "INSERT INTO notes (noteable_type, noteable_id, author_id, note, created_at, updated_at, is_award) (SELECT awardable_type, awardable_id, user_id, name, created_at, updated_at, TRUE FROM award_emoji)"
+  end
+
+  def migrate_postgresql
+    connection.transaction do
+      execute 'LOCK notes IN EXCLUSIVE MODE'
+      execute "INSERT INTO award_emoji (awardable_type, awardable_id, user_id, name, created_at, updated_at) (SELECT noteable_type, noteable_id, author_id, note, created_at, updated_at FROM notes WHERE is_award = true)"
       execute "DELETE FROM notes WHERE is_award = true"
+      remove_column :notes, :is_award, :boolean
     end
   end
+
+  def migrate_mysql
+    execute 'LOCK TABLES notes WRITE, award_emoji WRITE;'
+    execute 'INSERT INTO award_emoji (awardable_type, awardable_id, user_id, name, created_at, updated_at) (SELECT noteable_type, noteable_id, author_id, note, created_at, updated_at FROM notes WHERE is_award = true);'
+    execute "DELETE FROM notes WHERE is_award = true"
+    remove_column :notes, :is_award, :boolean
+  ensure
+    execute 'UNLOCK TABLES'
+  end
 end

db/migrate/20160416190505_remove_note_is_award.rb

-# rubocop:disable all
-class RemoveNoteIsAward < ActiveRecord::Migration
-  def change
-    remove_column :notes, :is_award, :boolean
-  end
-end

db/migrate/20160518200441_add_artifacts_expire_date_to_ci_builds.rb

+class AddArtifactsExpireDateToCiBuilds < ActiveRecord::Migration
+  def change
+    add_column :ci_builds, :artifacts_expire_at, :timestamp
+  end
+end

db/schema.rb

@@ -162,6 +162,7 @@ ActiveRecord::Schema.define(version: 20160610301627) do
     t.integer  "erased_by_id"
     t.datetime "erased_at"
     t.string   "environment"
+    t.datetime "artifacts_expire_at"
   end
 
   add_index "ci_builds", ["commit_id", "stage_idx", "created_at"], name: "index_ci_builds_on_commit_id_and_stage_idx_and_created_at", using: :btree

doc/api/README.md

@@ -8,32 +8,39 @@ under [`/lib/api`](https://gitlab.com/gitlab-org/gitlab-ce/tree/master/lib/api).
 Documentation for various API resources can be found separately in the
 following locations:
 
-- [Users](users.md)
-- [Session](session.md)
-- [Projects](projects.md) including setting Webhooks
-- [Project Snippets](project_snippets.md)
-- [Services](services.md)
-- [Repositories](repositories.md)
-- [Repository Files](repository_files.md)
-- [Commits](commits.md)
-- [Tags](tags.md)
 - [Branches](branches.md)
-- [Merge Requests](merge_requests.md)
+- [Builds](builds.md)
+- [Build triggers](build_triggers.md)
+- [Build Variables](build_variables.md)
+- [Commits](commits.md)
+- [Deploy Keys](deploy_keys.md)
+- [Groups](groups.md)
 - [Issues](issues.md)
+- [Keys](keys.md)
 - [Labels](labels.md)
+- [Merge Requests](merge_requests.md)
 - [Milestones](milestones.md)
-- [Notes](notes.md) (comments)
-- [Deploy Keys](deploy_keys.md)
-- [System Hooks](system_hooks.md)
-- [Groups](groups.md)
+- [Open source license templates](licenses.md)
 - [Namespaces](namespaces.md)
-- [Settings](settings.md)
-- [Keys](keys.md)
-- [Builds](builds.md)
-- [Build triggers](build_triggers.md)
-- [Build Variables](build_variables.md)
+- [Notes](notes.md) (comments)
+- [Projects](projects.md) including setting Webhooks
+- [Project Snippets](project_snippets.md)
+- [Repositories](repositories.md)
+- [Repository Files](repository_files.md)
 - [Runners](runners.md)
-- [Open source license templates](licenses.md)
+- [Services](services.md)
+- [Session](session.md)
+- [Settings](settings.md)
+- [System Hooks](system_hooks.md)
+- [Tags](tags.md)
+- [Users](users.md)
+
+### Internal CI API
+
+The following documentation is for the [internal CI API](ci/README.md):
+
+- [Builds](ci/builds.md)
+- [Runners](ci/runners.md)
 
 ## Authentication
 

doc/api/ci/README.md

+# GitLab CI API
+
+## Purpose
+
+The main purpose of GitLab CI API is to provide the necessary data and context
+for GitLab CI Runners.
+
+All relevant information about the consumer API can be found in a
+[separate document](../../api/README.md).
+
+## API Prefix
+
+The current CI API prefix is `/ci/api/v1`.
+
+You need to prepend this prefix to all examples in this documentation, like:
+
+```bash
+GET /ci/api/v1/builds/:id/artifacts
+```
+
+## Resources
+
+- [Builds](builds.md)
+- [Runners](runners.md)

doc/api/ci/builds.md

+# Builds API
+
+API used by runners to receive and update builds.
+
+>**Note:**
+This API is intended to be used only by Runners as their own
+communication channel. For the consumer API see the
+[Builds API](../builds.md).
+
+## Authentication
+
+This API uses two types of authentication:
+
+1. Unique Runner's token which is the token assigned to the Runner after it
+   has been registered.
+
+2. Using the build authorization token.
+   This is project's CI token that can be found under the **Builds** section of
+   a project's settings. The build authorization token can be passed as a
+   parameter or a value of `BUILD-TOKEN` header.
+
+These two methods of authentication are interchangeable.
+
+## Builds
+
+### Runs oldest pending build by runner
+
+```
+POST /ci/api/v1/builds/register
+```
+
+| Attribute | Type    | Required | Description         |
+|-----------|---------|----------|---------------------|
+| `token`   | string  | yes      | Unique runner token |
+
+
+```
+curl -X POST "https://gitlab.example.com/ci/api/v1/builds/register" -F "token=t0k3n"
+```
+
+### Update details of an existing build
+
+```
+PUT /ci/api/v1/builds/:id
+```
+
+| Attribute | Type    | Required | Description          |
+|-----------|---------|----------|----------------------|
+| `id`      | integer | yes      | The ID of a project  |
+| `token`   | string  | yes      | Unique runner token  |
+| `state`   | string  | no       | The state of a build |
+| `trace`   | string  | no       | The trace of a build |
+
+```
+curl -X PUT "https://gitlab.example.com/ci/api/v1/builds/1234" -F "token=t0k3n" -F "state=running" -F "trace=Running git clone...\n"
+```
+
+### Incremental build trace update
+
+Using this method you need to send trace content as a request body. You also need to provide the `Content-Range` header
+with a range of sent trace part. Note that you need to send parts in the proper order, so the begining of the part
+must start just after the end of the previous part. If you provide the wrong part, then GitLab CI API will return `416
+Range Not Satisfiable` response with a header `Range: 0-X`, where `X` is the current trace length.
+
+For example, if you receive `Range: 0-11` in the response, then your next part must contain a `Content-Range: 11-...`
+header and a trace part covered by this range.
+
+For a valid update API will return `202` response with:
+* `Build-Status: {status}` header containing current status of the build,
+* `Range: 0-{length}` header with the current trace length.
+
+```
+PATCH /ci/api/v1/builds/:id/trace.txt
+```
+
+Parameters:
+
+| Attribute | Type    | Required | Description          |
+|-----------|---------|----------|----------------------|
+| `id`      | integer | yes      | The ID of a build    |
+
+Headers:
+
+| Attribute       | Type    | Required | Description                       |
+|-----------------|---------|----------|-----------------------------------|
+| `BUILD-TOKEN`   | string  | yes      | The build authorization token     |
+| `Content-Range` | string  | yes      | Bytes range of trace that is sent |
+
+```
+curl -X PATCH "https://gitlab.example.com/ci/api/v1/builds/1234/trace.txt" -H "BUILD-TOKEN=build_t0k3n" -H "Content-Range=0-21" -d "Running git clone...\n"
+```
+
+
+### Upload artifacts to build
+
+```
+POST /ci/api/v1/builds/:id/artifacts
+```
+
+| Attribute | Type    | Required | Description                   |
+|-----------|---------|----------|-------------------------------|
+| `id`      | integer | yes      | The ID of a build             |
+| `token`   | string  | yes      | The build authorization token |
+| `file`    | mixed   | yes      | Artifacts file                |
+
+```
+curl -X POST "https://gitlab.example.com/ci/api/v1/builds/1234/artifacts" -F "token=build_t0k3n" -F "file=@/path/to/file"
+```
+
+### Download the artifacts file from build
+
+```
+GET /ci/api/v1/builds/:id/artifacts
+```
+
+| Attribute | Type    | Required | Description                   |
+|-----------|---------|----------|-------------------------------|
+| `id`      | integer | yes      | The ID of a build             |
+| `token`   | string  | yes      | The build authorization token |
+
+```
+curl "https://gitlab.example.com/ci/api/v1/builds/1234/artifacts" -F "token=build_t0k3n"
+```
+
+### Remove the artifacts file from build
+
+```
+DELETE /ci/api/v1/builds/:id/artifacts
+```
+
+| Attribute | Type    | Required | Description                   |
+|-----------|---------|----------|-------------------------------|
+| ` id`     | integer | yes      | The ID of a build             |
+| `token`   | string  | yes      | The build authorization token |
+
+```
+curl -X DELETE "https://gitlab.example.com/ci/api/v1/builds/1234/artifacts" -F "token=build_t0k3n"
+```

doc/api/ci/runners.md

+# Runners API
+
+API used by Runners to register and delete themselves.
+
+>**Note:**
+This API is intended to be used only by Runners as their own
+communication channel. For the consumer API see the
+[new Runners API](../runners.md).
+
+## Authentication
+
+This API uses two types of authentication:
+
+1. Unique Runner's token, which is the token assigned to the Runner after it
+   has been registered.
+
+2. Using Runners' registration token.
+   This is a token that can be found in project's settings.
+   It can also be found in the **Admin > Runners** settings area.
+   There are two types of tokens you can pass: shared Runner registration
+   token or project specific registration token.
+
+## Register a new runner
+
+Used to make GitLab CI aware of available runners.
+
+```sh
+POST /ci/api/v1/runners/register
+```
+
+| Attribute | Type    | Required  | Description |
+| --------- | ------- | --------- | ----------- |
+| `token`   | string  | yes       | Runner's registration token |
+
+Example request:
+
+```sh
+curl -X POST "https://gitlab.example.com/ci/api/v1/runners/register" -F "token=t0k3n"
+```
+
+## Delete a Runner
+
+Used to remove a Runner.
+
+```sh
+DELETE /ci/api/v1/runners/delete
+```
+
+| Attribute | Type    | Required  | Description |
+| --------- | ------- | --------- | ----------- |
+| `token`   | string  | yes       | Runner's registration token |
+
+Example request:
+
+```sh
+curl -X DELETE "https://gitlab.example.com/ci/api/v1/runners/delete" -F "token=t0k3n"
+```

doc/ci/README.md

@@ -14,5 +14,5 @@
 - [Trigger builds through the API](triggers/README.md)
 - [Build artifacts](build_artifacts/README.md)
 - [User permissions](permissions/README.md)
-- [API](api/README.md)
+- [API](../../api/ci/README.md)
 - [CI services (linked docker containers)](services/README.md)

doc/ci/api/README.md

 # GitLab CI API
 
-## Purpose
-
-Main purpose of GitLab CI API is to provide necessary data and context for
-GitLab CI Runners.
-
-For consumer API take a look at this [documentation](../../api/README.md) where
-you will find all relevant information.
-
-## API Prefix
-
-Current CI API prefix is `/ci/api/v1`.
-
-You need to prepend this prefix to all examples in this documentation, like:
-
-    GET /ci/api/v1/builds/:id/artifacts
-
-## Resources
-
-- [Builds](builds.md)
-- [Runners](runners.md)
+This document was moved to a [new location](../../api/ci/README.md).

doc/ci/api/builds.md

 # Builds API
 
-API used by runners to receive and update builds.
-
-_**Note:** This API is intended to be used only by Runners as their own
-communication channel. For the consumer API see the
-[Builds API](../../api/builds.md)._
-
-## Authentication
-
-This API uses two types of authentication:
-
-1.   Unique runner's token
-
-     Token assigned to runner after it has been registered.
-
-2.   Using build authorization token
-
-     This is project's CI token that can be found in Continuous Integration
-     project settings.
-
-     Build authorization token can be passed as a parameter or a value of
-     `BUILD-TOKEN` header. This method are interchangeable.
-
-## Builds
-
-### Runs oldest pending build by runner
-
-```
-POST /ci/api/v1/builds/register
-```
-
-| Attribute | Type    | Required | Description         |
-|-----------|---------|----------|---------------------|
-| `token`   | string  | yes      | Unique runner token |
-
-
-```
-curl -X POST "https://gitlab.example.com/ci/api/v1/builds/register" -F "token=t0k3n"
-```
-
-### Update details of an existing build
-
-```
-PUT /ci/api/v1/builds/:id
-```
-
-| Attribute | Type    | Required | Description          |
-|-----------|---------|----------|----------------------|
-| `id`      | integer | yes      | The ID of a project  |
-| `token`   | string  | yes      | Unique runner token  |
-| `state`   | string  | no       | The state of a build |
-| `trace`   | string  | no       | The trace of a build |
-
-```
-curl -X PUT "https://gitlab.example.com/ci/api/v1/builds/1234" -F "token=t0k3n" -F "state=running" -F "trace=Running git clone...\n"
-```
-
-### Incremental build trace update
-
-Using this method you need to send trace content as a request body. You also need to provide the `Content-Range` header
-with a range of sent trace part. Note that you need to send parts in the proper order, so the begining of the part
-must start just after the end of the previous part. If you provide the wrong part, then GitLab CI API will return `416
-Range Not Satisfiable` response with a header `Range: 0-X`, where `X` is the current trace length.
-
-For example, if you receive `Range: 0-11` in the response, then your next part must contain a `Content-Range: 11-...`
-header and a trace part covered by this range.
-
-For a valid update API will return `202` response with:
-* `Build-Status: {status}` header containing current status of the build,
-* `Range: 0-{length}` header with the current trace length.
-
-```
-PATCH /ci/api/v1/builds/:id/trace.txt
-```
-
-Parameters:
-
-| Attribute | Type    | Required | Description          |
-|-----------|---------|----------|----------------------|
-| `id`      | integer | yes      | The ID of a build    |
-
-Headers:
-
-| Attribute       | Type    | Required | Description                       |
-|-----------------|---------|----------|-----------------------------------|
-| `BUILD-TOKEN`   | string  | yes      | The build authorization token     |
-| `Content-Range` | string  | yes      | Bytes range of trace that is sent |
-
-```
-curl -X PATCH "https://gitlab.example.com/ci/api/v1/builds/1234/trace.txt" -H "BUILD-TOKEN=build_t0k3n" -H "Content-Range=0-21" -d "Running git clone...\n"
-```
-
-
-### Upload artifacts to build
-
-```
-POST /ci/api/v1/builds/:id/artifacts
-```
-
-| Attribute | Type    | Required | Description                   |
-|-----------|---------|----------|-------------------------------|
-| `id`      | integer | yes      | The ID of a build             |
-| `token`   | string  | yes      | The build authorization token |
-| `file`    | mixed   | yes      | Artifacts file                |
-
-```
-curl -X POST "https://gitlab.example.com/ci/api/v1/builds/1234/artifacts" -F "token=build_t0k3n" -F "file=@/path/to/file"
-```
-
-### Download the artifacts file from build
-
-```
-GET /ci/api/v1/builds/:id/artifacts
-```
-
-| Attribute | Type    | Required | Description                   |
-|-----------|---------|----------|-------------------------------|
-| `id`      | integer | yes      | The ID of a build             |
-| `token`   | string  | yes      | The build authorization token |
-
-```
-curl "https://gitlab.example.com/ci/api/v1/builds/1234/artifacts" -F "token=build_t0k3n"
-```
-
-### Remove the artifacts file from build
-
-```
-DELETE /ci/api/v1/builds/:id/artifacts
-```
-
-| Attribute | Type    | Required | Description                   |
-|-----------|---------|----------|-------------------------------|
-| ` id`     | integer | yes      | The ID of a build             |
-| `token`   | string  | yes      | The build authorization token |
-
-```
-curl -X DELETE "https://gitlab.example.com/ci/api/v1/builds/1234/artifacts" -F "token=build_t0k3n"
-```
+This document was moved to a [new location](../../api/ci/builds.md).

doc/ci/api/runners.md

 # Runners API
 
-API used by runners to register and delete themselves.
-
-_**Note:** This API is intended to be used only by Runners as their own
-communication channel. For the consumer API see the
-[new Runners API](../../api/runners.md)._
-
-## Authentication
-
-This API uses two types of authentication:
-
-1.   Unique runner's token
-
-     Token assigned to runner after it has been registered.
-
-2.   Using runners' registration token
-
-     This is a token that can be found in project's settings.
-     It can be also found in Admin area » Runners settings.
-
-     There are two types of tokens you can pass - shared runner registration
-     token or project specific registration token.
-
-## Runners
-
-### Register a new runner
-
-Used to make GitLab CI aware of available runners.
-
-    POST /ci/api/v1/runners/register
-
-Parameters:
-
-  * `token` (required) - Registration token
-
-
-### Delete a runner
-
-Used to remove runner.
-
-    DELETE /ci/api/v1/runners/delete
-
-Parameters:
-
-  * `token` (required) - Unique runner token
+This document was moved to a [new location](../../api/ci/runners.md).

doc/ci/yaml/README.md

@@ -32,6 +32,7 @@ If you want a quick introduction to GitLab CI, follow our
     - [artifacts](#artifacts)
         - [artifacts:name](#artifacts-name)
         - [artifacts:when](#artifacts-when)
+        - [artifacts:expire_in](#artifacts-expire_in)
     - [dependencies](#dependencies)
     - [before_script and after_script](#before_script-and-after_script)
 - [Hidden jobs](#hidden-jobs)
@@ -705,6 +706,40 @@ job:
     when: on_failure
 ```
 
+#### artifacts:expire_in
+
+>**Note:**
+Introduced in GitLab 8.9 and GitLab Runner v1.3.0.
+
+`artifacts:expire_in` is used to remove uploaded artifacts after specified time.
+By default artifacts are stored on GitLab forver.
+`expire_in` allows to specify after what time the artifacts should be removed.
+The artifacts will expire counting from the moment when they are uploaded and stored on GitLab.
+
+After artifacts uploading you can use the **Keep** button on build page to keep the artifacts forever.
+
+Artifacts are removed every hour, but they are not accessible after expire date.
+
+The value of `expire_in` is a elapsed time. The example of parsable values:
+- '3 mins 4 sec'
+- '2 hrs 20 min'
+- '2h20min'
+- '6 mos 1 day'
+- '47 yrs 6 mos and 4d'
+- '3 weeks and 2 days'
+
+---
+
+**Example configurations**
+
+To expire artifacts after 1 week from the moment that they are uploaded:
+
+```yaml
+job:
+  artifacts:
+    expire_in: 1 week
+```
+
 ### dependencies
 
 >**Note:**

lib/api/builds.rb

@@ -166,6 +166,26 @@ module API
         present build, with: Entities::Build,
                        user_can_download_artifacts: can?(current_user, :download_build_artifacts, user_project)
       end
+
+      # Keep the artifacts to prevent them from being deleted
+      #
+      # Parameters:
+      #   id (required) - the id of a project
+      #   build_id (required) - The ID of a build
+      # Example Request:
+      #   POST /projects/:id/builds/:build_id/artifacts/keep
+      post ':id/builds/:build_id/artifacts/keep' do
+        authorize_update_builds!
+
+        build = get_build(params[:build_id])
+        return not_found!(build) unless build && build.artifacts?
+
+        build.keep_artifacts!
+
+        status 200
+        present build, with: Entities::Build,
+                       user_can_download_artifacts: can?(current_user, :read_build, user_project)
+      end
     end
 
     helpers do

lib/ci/api/builds.rb

@@ -114,6 +114,7 @@ module Ci
         #   id (required) - The ID of a build
         #   token (required) - The build authorization token
         #   file (required) - Artifacts file
+        #   expire_in (optional) - Specify when artifacts should expire (ex. 7d)
         # Parameters (accelerated by GitLab Workhorse):
         #   file.path - path to locally stored body (generated by Workhorse)
         #   file.name - real filename as send in Content-Disposition
@@ -145,6 +146,7 @@ module Ci
 
           build.artifacts_file = artifacts
           build.artifacts_metadata = metadata
+          build.artifacts_expire_in = params['expire_in']
 
           if build.save
             present(build, with: Entities::BuildDetails)

lib/ci/api/entities.rb

@@ -20,7 +20,7 @@ module Ci
         expose :name, :token, :stage
         expose :project_id
         expose :project_name
-        expose :artifacts_file, using: ArtifactFile, if: lambda { |build, opts| build.artifacts? }
+        expose :artifacts_file, using: ArtifactFile, if: ->(build, _) { build.artifacts? }
       end
 
       class BuildDetails < Build
@@ -29,6 +29,7 @@ module Ci
         expose :before_sha
         expose :allow_git_fetch
         expose :token
+        expose :artifacts_expire_at, if: ->(build, _) { build.artifacts? }
 
         expose :options do |model|
           model.options

lib/ci/gitlab_ci_yaml_processor.rb

@@ -12,7 +12,7 @@ module Ci
                         :dependencies, :before_script, :after_script, :variables,
                         :environment]
     ALLOWED_CACHE_KEYS = [:key, :untracked, :paths]
-    ALLOWED_ARTIFACTS_KEYS = [:name, :untracked, :paths, :when]
+    ALLOWED_ARTIFACTS_KEYS = [:name, :untracked, :paths, :when, :expire_in]
 
     attr_reader :after_script, :image, :services, :path, :cache
 
@@ -291,6 +291,10 @@ module Ci
       if job[:artifacts][:when] && !job[:artifacts][:when].in?(%w[on_success on_failure always])
         raise ValidationError, "#{name} job: artifacts:when parameter should be on_success, on_failure or always"
       end
+
+      if job[:artifacts][:expire_in] && !validate_duration(job[:artifacts][:expire_in])
+        raise ValidationError, "#{name} job: artifacts:expire_in parameter should be a duration"
+      end
     end
 
     def validate_job_dependencies!(name, job)

lib/gitlab/ci/config/node/validation_helpers.rb

@@ -5,6 +5,12 @@ module Gitlab
         module ValidationHelpers
           private
 
+          def validate_duration(value)
+            value.is_a?(String) && ChronicDuration.parse(value)
+          rescue ChronicDuration::DurationParseError
+            false
+          end
+
           def validate_array_of_strings(values)
             values.is_a?(Array) && values.all? { |value| validate_string(value) }
           end

spec/features/builds_spec.rb

@@ -97,6 +97,42 @@ describe "Builds" do
       end
     end
 
+    context 'Artifacts expire date' do
+      before do
+        @build.update_attributes(artifacts_file: artifacts_file, artifacts_expire_at: expire_at)
+        visit namespace_project_build_path(@project.namespace, @project, @build)
+      end
+
+      context 'no expire date defined' do
+        let(:expire_at) { nil }
+
+        it 'does not have the Keep button' do
+          expect(page).not_to have_content 'Keep'
+        end
+      end
+
+      context 'when expire date is defined' do
+        let(:expire_at) { Time.now + 7.days }
+
+        it 'keeps artifacts when Keep button is clicked' do
+          expect(page).to have_content 'The artifacts will be removed'
+          click_link 'Keep'
+
+          expect(page).not_to have_link 'Keep'
+          expect(page).not_to have_content 'The artifacts will be removed'
+        end
+      end
+
+      context 'when artifacts expired' do
+        let(:expire_at) { Time.now - 7.days }
+
+        it 'does not have the Keep button' do
+          expect(page).to have_content 'The artifacts were removed'
+          expect(page).not_to have_link 'Keep'
+        end
+      end
+    end
+
     context 'Build raw trace' do
       before do
         @build.run!

spec/lib/ci/gitlab_ci_yaml_processor_spec.rb

@@ -576,7 +576,12 @@ module Ci
                              services:      ["mysql"],
                              before_script: ["pwd"],
                              rspec:         {
-                               artifacts: { paths: ["logs/", "binaries/"], untracked: true, name: "custom_name" },
+                               artifacts: {
+                                 paths: ["logs/", "binaries/"],
+                                 untracked: true,
+                                 name: "custom_name",
+                                 expire_in: "7d"
+                               },
                                script: "rspec"
                              }
                            })
@@ -598,7 +603,8 @@ module Ci
             artifacts: {
               name: "custom_name",
               paths: ["logs/", "binaries/"],
-              untracked: true
+              untracked: true,
+              expire_in: "7d"
             }
           },
           when: "on_success",
@@ -1044,6 +1050,20 @@ EOT
         end.to raise_error(GitlabCiYamlProcessor::ValidationError, "rspec job: artifacts:when parameter should be on_success, on_failure or always")
       end
 
+      it "returns errors if job artifacts:expire_in is not an a string" do
+        config = YAML.dump({ types: ["build", "test"], rspec: { script: "test", artifacts: { expire_in: 1 } } })
+        expect do
+          GitlabCiYamlProcessor.new(config)
+        end.to raise_error(GitlabCiYamlProcessor::ValidationError, "rspec job: artifacts:expire_in parameter should be a duration")
+      end
+
+      it "returns errors if job artifacts:expire_in is not an a valid duration" do
+        config = YAML.dump({ types: ["build", "test"], rspec: { script: "test", artifacts: { expire_in: "7 elephants" } } })
+        expect do
+          GitlabCiYamlProcessor.new(config)
+        end.to raise_error(GitlabCiYamlProcessor::ValidationError, "rspec job: artifacts:expire_in parameter should be a duration")
+      end
+
       it "returns errors if job artifacts:untracked is not an array of strings" do
         config = YAML.dump({ types: ["build", "test"], rspec: { script: "test", artifacts: { untracked: "string" } } })
         expect do

spec/models/build_spec.rb

@@ -397,9 +397,34 @@ describe Ci::Build, models: true do
     context 'artifacts archive exists' do
       let(:build) { create(:ci_build, :artifacts) }
       it { is_expected.to be_truthy }
+
+      context 'is expired' do
+        before { build.update(artifacts_expire_at: Time.now - 7.days)  }
+        it { is_expected.to be_falsy }
+      end
+
+      context 'is not expired' do
+        before { build.update(artifacts_expire_at: Time.now + 7.days)  }
+        it { is_expected.to be_truthy }
+      end
     end
   end
 
+  describe '#artifacts_expired?' do
+    subject { build.artifacts_expired? }
+
+    context 'is expired' do
+      before { build.update(artifacts_expire_at: Time.now - 7.days)  }
+
+      it { is_expected.to be_truthy }
+    end
+
+    context 'is not expired' do
+      before { build.update(artifacts_expire_at: Time.now + 7.days)  }
+
+      it { is_expected.to be_falsey }
+    end
+  end
 
   describe '#artifacts_metadata?' do
     subject { build.artifacts_metadata? }
@@ -412,7 +437,6 @@ describe Ci::Build, models: true do
       it { is_expected.to be_truthy }
     end
   end
-
   describe '#repo_url' do
     let(:build) { create(:ci_build) }
     let(:project) { build.project }
@@ -427,6 +451,50 @@ describe Ci::Build, models: true do
     it { is_expected.to include(project.web_url[7..-1]) }
   end
 
+  describe '#artifacts_expire_in' do
+    subject { build.artifacts_expire_in }
+    it { is_expected.to be_nil }
+
+    context 'when artifacts_expire_at is specified' do
+      let(:expire_at) { Time.now + 7.days }
+
+      before { build.artifacts_expire_at = expire_at }
+
+      it { is_expected.to be_within(5).of(expire_at - Time.now) }
+    end
+  end
+
+  describe '#artifacts_expire_in=' do
+    subject { build.artifacts_expire_in }
+
+    it 'when assigning valid duration' do
+      build.artifacts_expire_in = '7 days'
+
+      is_expected.to be_within(10).of(7.days.to_i)
+    end
+
+    it 'when assigning invalid duration' do
+      expect { build.artifacts_expire_in = '7 elephants' }.to raise_error(ChronicDuration::DurationParseError)
+      is_expected.to be_nil
+    end
+
+    it 'when resseting value' do
+      build.artifacts_expire_in = nil
+
+      is_expected.to be_nil
+    end
+  end
+
+  describe '#keep_artifacts!' do
+    let(:build) { create(:ci_build, artifacts_expire_at: Time.now + 7.days) }
+
+    it 'to reset expire_at' do
+      build.keep_artifacts!
+
+      expect(build.artifacts_expire_at).to be_nil
+    end
+  end
+
   describe '#depends_on_builds' do
     let!(:build) { create(:ci_build, pipeline: pipeline, name: 'build', stage_idx: 0, stage: 'build') }
     let!(:rspec_test) { create(:ci_build, pipeline: pipeline, name: 'rspec', stage_idx: 1, stage: 'test') }

spec/requests/api/builds_spec.rb

@@ -241,4 +241,30 @@ describe API::API, api: true  do
       end
     end
   end
+
+  describe 'POST /projects/:id/builds/:build_id/artifacts/keep' do
+    before do
+      post api("/projects/#{project.id}/builds/#{build.id}/artifacts/keep", user)
+    end
+
+    context 'artifacts did not expire' do
+      let(:build) do
+        create(:ci_build, :trace, :artifacts, :success,
+               project: project, pipeline: pipeline, artifacts_expire_at: Time.now + 7.days)
+      end
+
+      it 'keeps artifacts' do
+        expect(response.status).to eq 200
+        expect(build.reload.artifacts_expire_at).to be_nil
+      end
+    end
+
+    context 'no artifacts' do
+      let(:build) { create(:ci_build, project: project, pipeline: pipeline) }
+
+      it 'responds with not found' do
+        expect(response.status).to eq 404
+      end
+    end
+  end
 end

spec/requests/ci/api/builds_spec.rb

@@ -364,6 +364,42 @@ describe Ci::API::API do
             end
           end
 
+          context 'with an expire date' do
+            let!(:artifacts) { file_upload }
+
+            let(:post_data) do
+              { 'file.path' => artifacts.path,
+                'file.name' => artifacts.original_filename,
+                'expire_in' => expire_in }
+            end
+
+            before do
+              post(post_url, post_data, headers_with_token)
+            end
+
+            context 'with an expire_in given' do
+              let(:expire_in) { '7 days' }
+
+              it 'updates when specified' do
+                build.reload
+                expect(response.status).to eq(201)
+                expect(json_response['artifacts_expire_at']).not_to be_empty
+                expect(build.artifacts_expire_at).to be_within(5.minutes).of(Time.now + 7.days)
+              end
+            end
+
+            context 'with no expire_in given' do
+              let(:expire_in) { nil }
+
+              it 'ignores if not specified' do
+                build.reload
+                expect(response.status).to eq(201)
+                expect(json_response['artifacts_expire_at']).to be_nil
+                expect(build.artifacts_expire_at).to be_nil
+              end
+            end
+          end
+
           context "artifacts file is too large" do
             it "should fail to post too large artifact" do
               stub_application_setting(max_artifacts_size: 0)

spec/workers/expire_build_artifacts_worker_spec.rb

+require 'spec_helper'
+
+describe ExpireBuildArtifactsWorker do
+  include RepoHelpers
+
+  let(:worker) { described_class.new }
+
+  describe '#perform' do
+    before { build }
+
+    subject! { worker.perform }
+
+    context 'with expired artifacts' do
+      let(:build) { create(:ci_build, :artifacts, artifacts_expire_at: Time.now - 7.days) }
+
+      it 'does expire' do
+        expect(build.reload.artifacts_expired?).to be_truthy
+      end
+
+      it 'does remove files' do
+        expect(build.reload.artifacts_file.exists?).to be_falsey
+      end
+    end
+
+    context 'with not yet expired artifacts' do
+      let(:build) { create(:ci_build, :artifacts, artifacts_expire_at: Time.now + 7.days) }
+
+      it 'does not expire' do
+        expect(build.reload.artifacts_expired?).to be_falsey
+      end
+
+      it 'does not remove files' do
+        expect(build.reload.artifacts_file.exists?).to be_truthy
+      end
+    end
+
+    context 'without expire date' do
+      let(:build) { create(:ci_build, :artifacts) }
+
+      it 'does not expire' do
+        expect(build.reload.artifacts_expired?).to be_falsey
+      end
+
+      it 'does not remove files' do
+        expect(build.reload.artifacts_file.exists?).to be_truthy
+      end
+    end
+
+    context 'for expired artifacts' do
+      let(:build) { create(:ci_build, artifacts_expire_at: Time.now - 7.days) }
+
+      it 'is still expired' do
+        expect(build.reload.artifacts_expired?).to be_truthy
+      end
+    end
+  end
+end