Fix migrate_artifacts.rake task (+1 squashed commit)

Squashed commits:
[cd77fee] Migrate artifacts to a new storage

- Store information in database what storage is being used by artifacts,
- Artifacts are now stored as `artifacts/project_id/commit_id/job_id`.

app/models/ci/build.rb

@@ -52,6 +52,11 @@ module Ci
     after_commit :update_project_statistics_after_save, on: [:create, :update]
     after_commit :update_project_statistics, on: :destroy
 
+    enum artifacts_storage: { 
+      artifacts_storage_undefined: nil,
+      artifacts_storage_upgraded: 1,
+    }
+
     class << self
       # This is needed for url_for to work,
       # as the controller is JobsController
@@ -286,18 +291,28 @@ module Ci
       !artifacts_expired? && artifacts_file.exists?
     end
 
+    def browsable_artifacts?
+      artifacts_metadata?
+    end
+
+    def downloadable_single_artifacts_file?
+      artifacts_metadata? && artifacts_file.local_file?
+    end
+
     def artifacts_metadata?
       artifacts? && artifacts_metadata.exists?
     end
 
     def artifacts_metadata_entry(path, **options)
+      artifacts_metadata.use_file do |metadata_path|
         metadata = Gitlab::Ci::Build::Artifacts::Metadata.new(
-        artifacts_metadata.path,
+          metadata_path,
           path,
           **options)
 
         metadata.to_entry
       end
+    end
 
     def erase_artifacts!
       remove_artifacts_file!

app/services/projects/update_pages_service.rb

@@ -96,10 +96,12 @@ module Projects
       # -n  never overwrite existing files
       # We add * to end of SITE_PATH, because we want to extract SITE_PATH and all subdirectories
       site_path = File.join(SITE_PATH, '*')
-      unless system(*%W(unzip -n #{artifacts} #{site_path} -d #{temp_path}))
+      build.artifacts_file.use_file do |artifacts_path|
+        unless system(*%W(unzip -n #{artifacts_path} #{site_path} -d #{temp_path}))
           raise 'pages failed to extract'
         end
       end
+    end
 
     def deploy_page!(archive_public_path)
       # Do atomic move of pages
@@ -152,10 +154,6 @@ module Projects
       build.ref
     end
 
-    def artifacts
-      build.artifacts_file.path
-    end
-
     def latest_sha
       project.commit(build.ref).try(:sha).to_s
     end

app/uploaders/artifact_uploader.rb

 class ArtifactUploader < GitlabUploader
-  storage :file
+  include ObjectStoreable
 
   attr_reader :job, :field
 
@@ -16,11 +16,23 @@ class ArtifactUploader < GitlabUploader
   end
 
   def store_dir
+    if file_storage?
       default_local_path
+    else
+      default_path
+    end
   end
 
   def cache_dir
-    File.join(self.class.local_artifacts_store, 'tmp/cache')
+    if file_cache_storage?
+      File.join(self.local_artifacts_store, 'tmp/cache')
+    else
+      'tmp/cache'
+    end
+  end
+
+  def migrate!
+    # TODO
   end
 
   private

app/uploaders/concerns/object_storeable.rb

+module ObjectStoreable
+  extend ActiveSupport::Concern
+
+  module ClassMethods
+    def use_object_store?
+      @storage_options.object_store.enabled
+    end
+
+    def storage_options(options)
+      @storage_options = options
+
+      class_eval do
+        storage use_object_store? ? :fog : :file
+      end
+    end
+  end
+
+  def fog_directory
+    return super unless use_object_store?
+
+    @storage_options.bucket
+  end
+
+  # Override the credentials
+  def fog_credentials
+    return super unless use_object_store?
+
+    {
+      provider:              @storage_options.provider,
+      aws_access_key_id:     @storage_options.access_key_id,
+      aws_secret_access_key: @storage_options.secret_access_key,
+      region:                @storage_options.region,
+      endpoint:              @storage_options.endpoint,
+      path_style:            true
+    }
+  end
+
+  def fog_public
+    false
+  end
+
+  def use_object_store?
+    @storage_options.object_store.enabled
+  end
+
+  def move_to_store
+    !use_object_store?
+  end
+
+  def move_to_cache
+    !use_object_store?
+  end
+
+  def use_file
+    if use_object_store?
+      return yield path
+    end
+
+    begin
+      cache_stored_file!
+      yield cache_path
+    ensure
+      cache_storage.delete_dir!(cache_path(nil))
+    end
+  end
+
+  def upload_authorize
+    self.cache_id = CarrierWave.generate_cache_id
+    self.original_filename = SecureRandom.hex
+
+    result = { TempPath: cache_path }
+
+    use_cache_object_storage do
+      expire_at = ::Fog::Time.now + fog_authenticated_url_expiration
+      result[:ObjectStore] = {
+        ObjectID: cache_name,
+        StoreURL: storage.connection.put_object_url(
+          fog_directory, cache_path, expire_at)
+      }
+    end
+
+    result
+  end
+
+  def cache!(new_file = nil)
+    unless retrive_uploaded_file!(new_file&.object_id, new_file.original_filename)
+      super
+    end
+  end
+
+  def cache_storage
+    if @use_storage_for_cache || cached? && remote_file?
+      storage
+    else
+      super
+    end
+  end
+
+  def retrive_uploaded_file!(identifier, filename)
+    return unless identifier
+    return unless filename
+    return unless use_object_store?
+
+    @use_storage_for_cache = true
+
+    retrieve_from_cache!(identifier)
+    @filename = filename
+  ensure
+    @use_storage_for_cache = false
+  end
+end

app/uploaders/gitlab_uploader.rb

@@ -38,6 +38,10 @@ class GitlabUploader < CarrierWave::Uploader::Base
     self.file.path.sub("#{root}/", '')
   end
 
+  def use_file
+    yield path
+  end
+
   def exists?
     file.try(:exists?)
   end

app/views/projects/artifacts/_tree_file.html.haml

-- path_to_file = file_namespace_project_job_artifacts_path(@project.namespace, @project, @build, path: file.path)
+- path_to_file = file_namespace_project_job_artifacts_path(@project.namespace, @project, @build, path: file.path) if @build.downloadable_single_artifacts_file?
 
 %tr.tree-item{ 'data-link' => path_to_file }
   - blob = file.blob
   %td.tree-item-file-name
     = tree_icon('file', blob.mode, blob.name)
-    = link_to path_to_file do
-      %span.str-truncated= blob.name
+    %span.str-truncated
+      - if path_to_file
+        = link_to file.name, path_to_file
+      - else
+        = file.name
   %td
     = number_to_human_size(blob.size, precision: 2)

app/views/projects/jobs/_sidebar.html.haml

@@ -36,7 +36,7 @@
             = link_to download_namespace_project_job_artifacts_path(@project.namespace, @project, @build), rel: 'nofollow', download: '', class: 'btn btn-sm btn-default' do
               Download
 
-            - if @build.artifacts_metadata?
+            - if @build.browsable_artifacts?
               = link_to browse_namespace_project_job_artifacts_path(@project.namespace, @project, @build), class: 'btn btn-sm btn-default' do
                 Browse
 

config/application.rb

@@ -28,7 +28,8 @@ module Gitlab
                                      #{config.root}/app/models/members
                                      #{config.root}/app/models/project_services
                                      #{config.root}/app/workers/concerns
-                                     #{config.root}/app/services/concerns))
+                                     #{config.root}/app/services/concerns
+                                     #{config.root}/app/uploaders/concerns))
 
     config.generators.templates.push("#{config.root}/generator_templates")
 

config/gitlab.yml.example

@@ -138,6 +138,13 @@ production: &base
     enabled: true
     # The location where build artifacts are stored (default: shared/artifacts).
     # path: shared/artifacts
+    object_store:
+      enabled: false
+      provider: AWS # Only AWS supported at the moment
+      access_key_id: VXKLW2P7WP83RM3OQAYU
+      secret_access_key: hEm7WuxW3Qct9tsxNqSw+iyP26fcCacz78vErkiI
+      bucket: docker
+      region: eu-central-1
 
   ## Git LFS
   lfs:

db/migrate/20170504203205_add_file_storage_to_ci_build.rb

+class AddArtifactsFileStorageToCiBuild < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    add_column :ci_builds, :artifacts_storage, :integer
+  end
+end

db/schema.rb

@@ -284,6 +284,7 @@ ActiveRecord::Schema.define(version: 20170602003304) do
     t.string "coverage_regex"
     t.integer "auto_canceled_by_id"
     t.boolean "retried"
+    t.integer "artifacts_storage"
   end
 
   add_index "ci_builds", ["auto_canceled_by_id"], name: "index_ci_builds_on_auto_canceled_by_id", using: :btree

doc/administration/job_artifacts.md

@@ -82,6 +82,17 @@ _The artifacts are stored by default in
 
 1. Save the file and [restart GitLab][] for the changes to take effect.
 
+---
+
+**Using Object Store**
+
+The previously mentioned methods use the local disk to store artifacts. However,
+there is the option to use object stores like AWS' S3. To do this, set the
+`object_store` flag to true in your `gitlab.rb`. This relies on valid AWS
+credentials to be configured already. Please note, that enabling this feature
+will have the effect that artifacts are _not_ browsable anymore through the web
+interface.
+
 ## Set the maximum file size of the artifacts
 
 Provided the artifacts are enabled, you can change the maximum file size of the

lib/api/helpers.rb

@@ -290,7 +290,7 @@ module API
 
     # file helpers
 
-    def uploaded_file(field, uploads_path)
+    def uploaded_file(uploader, field)
       if params[field]
         bad_request!("#{field} is not a file") unless params[field].respond_to?(:filename)
         return params[field]
@@ -301,14 +301,15 @@ module API
       # sanitize file paths
       # this requires all paths to exist
       required_attributes! %W(#{field}.path)
-      uploads_path = File.realpath(uploads_path)
+      artifacts_store = File.realpath(uploader.local_artifacts_store)
       file_path = File.realpath(params["#{field}.path"])
-      bad_request!('Bad file path') unless file_path.start_with?(uploads_path)
+      bad_request!('Bad file path') unless file_path.start_with?(artifacts_store)
 
       UploadedFile.new(
         file_path,
         params["#{field}.name"],
-        params["#{field}.type"] || 'application/octet-stream'
+        params["#{field}.type"] || 'application/octet-stream',
+        params["#{field}.object_id"]
       )
     end
 

lib/api/runner.rb

@@ -181,7 +181,7 @@ module API
 
         status 200
         content_type Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE
-        Gitlab::Workhorse.artifact_upload_ok
+        job.artifacts_file.upload_authorize
       end
 
       desc 'Upload artifacts for job' do
@@ -199,6 +199,7 @@ module API
         optional :file, type: File, desc: %q(Artifact's file)
         optional 'file.path', type: String, desc: %q(path to locally stored body (generated by Workhorse))
         optional 'file.name', type: String, desc: %q(real filename as send in Content-Disposition (generated by Workhorse))
+        optional 'file.object_id', type: String, desc: %q(object_id as send by authorize (generated by Workhorse))
         optional 'file.type', type: String, desc: %q(real content type as send in Content-Type (generated by Workhorse))
         optional 'metadata.path', type: String, desc: %q(path to locally stored body (generated by Workhorse))
         optional 'metadata.name', type: String, desc: %q(filename (generated by Workhorse))
@@ -210,13 +211,13 @@ module API
         job = authenticate_job!
         forbidden!('Job is not running!') unless job.running?
 
-        artifacts_upload_path = ArtifactUploader.artifacts_upload_path
-        artifacts = uploaded_file(:file, artifacts_upload_path)
-        metadata = uploaded_file(:metadata, artifacts_upload_path)
+        artifacts = uploaded_file(job.artifacts_file, :file)
+        metadata = uploaded_file(job.artifacts_metadata, :metadata)
 
         bad_request!('Missing artifacts file!') unless artifacts
         file_to_large! unless artifacts.size < max_artifacts_size
 
+        job.artifacts_storage_upgraded!
         job.artifacts_file = artifacts
         job.artifacts_metadata = metadata
         job.artifacts_expire_in = params['expire_in'] ||

lib/ci/api/builds.rb

@@ -124,7 +124,7 @@ module Ci
 
           status 200
           content_type Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE
-          Gitlab::Workhorse.artifact_upload_ok
+          build.artifacts_file.upload_authorize
         end
 
         # Upload artifacts to build - Runners only
@@ -153,13 +153,13 @@ module Ci
           build = authenticate_build!
           forbidden!('Build is not running!') unless build.running?
 
-          artifacts_upload_path = ArtifactUploader.artifacts_upload_path
-          artifacts = uploaded_file(:file, artifacts_upload_path)
-          metadata = uploaded_file(:metadata, artifacts_upload_path)
+          artifacts = uploaded_file(build.artifacts_file, :file)
+          metadata = uploaded_file(build.artifacts_metadata, :metadata)
 
           bad_request!('Missing artifacts file!') unless artifacts
           file_to_large! unless artifacts.size < max_artifacts_size
 
+          build.artifacts_storage_upgraded!
           build.artifacts_file = artifacts
           build.artifacts_metadata = metadata
           build.artifacts_expire_in =

lib/tasks/migrate/migrate_artifacts.rake

+desc "GitLab | Migrate files for artifacts to comply with new storage format"
+task migrate_artifacts: :environment do
+  puts 'Artifacts'.color(:yellow)
+  Ci::Build.joins(:project)
+    .with_artifacts
+    .where(artifacts_file_migrated: nil)
+    .find_each(batch_size: 100) do |issue|
+    begin
+      build.artifacts_file.migrate!
+      build.artifacts_metadata.migrate!
+      build.save! if build.changed?
+      print '.'
+    rescue
+      print 'F'
+    end
+  end
+end

lib/uploaded_file.rb

@@ -9,15 +9,19 @@ class UploadedFile
   # The tempfile
   attr_reader :tempfile
 
+  # The object_id for asynchronous uploads
+  attr_reader :object_id
+
   # The content type of the "uploaded" file
   attr_accessor :content_type
 
-  def initialize(path, filename, content_type = "text/plain")
+  def initialize(path, filename, content_type = "text/plain", object_id = nil)
     raise "#{path} file does not exist" unless ::File.exist?(path)
 
     @content_type = content_type
     @original_filename = filename || ::File.basename(path)
     @tempfile = File.new(path, 'rb')
+    @object_id = object_id
   end
 
   def path