Commit f03da18e authored by Grzegorz Bizon's avatar Grzegorz Bizon

Merge branch 'ci/view-build-artifacts' into 'master'

Add browser for build artifacts

Discussion in #3426, closes #3426.

See merge request !2123
parents f981da44 be764a3a
...@@ -50,6 +50,7 @@ v 8.4.0 (unreleased) ...@@ -50,6 +50,7 @@ v 8.4.0 (unreleased)
v 8.3.4 v 8.3.4
- Use gitlab-workhorse 0.5.4 (fixes API routing bug) - Use gitlab-workhorse 0.5.4 (fixes API routing bug)
- Add build artifacts browser
v 8.3.3 v 8.3.3
- Preserve CE behavior with JIRA integration by only calling API if URL is set - Preserve CE behavior with JIRA integration by only calling API if URL is set
......
class Projects::ArtifactsController < Projects::ApplicationController
layout 'project'
before_action :authorize_read_build_artifacts!
def download
unless artifacts_file.file_storage?
return redirect_to artifacts_file.url
end
unless artifacts_file.exists?
return not_found!
end
send_file artifacts_file.path, disposition: 'attachment'
end
def browse
return render_404 unless build.artifacts?
directory = params[:path] ? "#{params[:path]}/" : ''
@entry = build.artifacts_metadata_entry(directory)
return render_404 unless @entry.exists?
end
def file
entry = build.artifacts_metadata_entry(params[:path])
if entry.exists?
render json: { archive: build.artifacts_file.path,
entry: Base64.encode64(entry.path) }
else
render json: {}, status: 404
end
end
private
def build
@build ||= project.builds.unscoped.find_by!(id: params[:build_id])
end
def artifacts_file
@artifacts_file ||= build.artifacts_file
end
def authorize_read_build_artifacts!
unless can?(current_user, :read_build_artifacts, @project)
if current_user.nil?
return authenticate_user!
else
return render_404
end
end
end
end
...@@ -2,7 +2,6 @@ class Projects::BuildsController < Projects::ApplicationController ...@@ -2,7 +2,6 @@ class Projects::BuildsController < Projects::ApplicationController
before_action :build, except: [:index, :cancel_all] before_action :build, except: [:index, :cancel_all]
before_action :authorize_manage_builds!, except: [:index, :show, :status] before_action :authorize_manage_builds!, except: [:index, :show, :status]
before_action :authorize_download_build_artifacts!, only: [:download]
layout "project" layout "project"
...@@ -51,18 +50,6 @@ class Projects::BuildsController < Projects::ApplicationController ...@@ -51,18 +50,6 @@ class Projects::BuildsController < Projects::ApplicationController
redirect_to build_path(build) redirect_to build_path(build)
end end
def download
unless artifacts_file.file_storage?
return redirect_to artifacts_file.url
end
unless artifacts_file.exists?
return not_found!
end
send_file artifacts_file.path, disposition: 'attachment'
end
def status def status
render json: @build.to_json(only: [:status, :id, :sha, :coverage], methods: :sha) render json: @build.to_json(only: [:status, :id, :sha, :coverage], methods: :sha)
end end
...@@ -79,10 +66,6 @@ class Projects::BuildsController < Projects::ApplicationController ...@@ -79,10 +66,6 @@ class Projects::BuildsController < Projects::ApplicationController
@build ||= project.builds.unscoped.find_by!(id: params[:id]) @build ||= project.builds.unscoped.find_by!(id: params[:id])
end end
def artifacts_file
build.artifacts_file
end
def build_path(build) def build_path(build)
namespace_project_build_path(build.project.namespace, build.project, build) namespace_project_build_path(build.project.namespace, build.project, build)
end end
...@@ -92,14 +75,4 @@ class Projects::BuildsController < Projects::ApplicationController ...@@ -92,14 +75,4 @@ class Projects::BuildsController < Projects::ApplicationController
return page_404 return page_404
end end
end end
def authorize_download_build_artifacts!
unless can?(current_user, :download_build_artifacts, @project)
if current_user.nil?
return authenticate_user!
else
return render_404
end
end
end
end end
...@@ -175,7 +175,7 @@ class Ability ...@@ -175,7 +175,7 @@ class Ability
:create_merge_request, :create_merge_request,
:create_wiki, :create_wiki,
:manage_builds, :manage_builds,
:download_build_artifacts, :read_build_artifacts,
:push_code :push_code
] ]
end end
......
...@@ -30,10 +30,12 @@ ...@@ -30,10 +30,12 @@
# description :string(255) # description :string(255)
# artifacts_file :text # artifacts_file :text
# gl_project_id :integer # gl_project_id :integer
# artifacts_metadata :text
# #
module Ci module Ci
class Build < CommitStatus class Build < CommitStatus
include Gitlab::Application.routes.url_helpers
LAZY_ATTRIBUTES = ['trace'] LAZY_ATTRIBUTES = ['trace']
belongs_to :runner, class_name: 'Ci::Runner' belongs_to :runner, class_name: 'Ci::Runner'
...@@ -49,6 +51,7 @@ module Ci ...@@ -49,6 +51,7 @@ module Ci
scope :similar, ->(build) { where(ref: build.ref, tag: build.tag, trigger_request_id: build.trigger_request_id) } scope :similar, ->(build) { where(ref: build.ref, tag: build.tag, trigger_request_id: build.trigger_request_id) }
mount_uploader :artifacts_file, ArtifactUploader mount_uploader :artifacts_file, ArtifactUploader
mount_uploader :artifacts_metadata, ArtifactUploader
acts_as_taggable acts_as_taggable
...@@ -291,21 +294,18 @@ module Ci ...@@ -291,21 +294,18 @@ module Ci
end end
def target_url def target_url
Gitlab::Application.routes.url_helpers. namespace_project_build_url(project.namespace, project, self)
namespace_project_build_url(project.namespace, project, self)
end end
def cancel_url def cancel_url
if active? if active?
Gitlab::Application.routes.url_helpers. cancel_namespace_project_build_path(project.namespace, project, self)
cancel_namespace_project_build_path(project.namespace, project, self)
end end
end end
def retry_url def retry_url
if retryable? if retryable?
Gitlab::Application.routes.url_helpers. retry_namespace_project_build_path(project.namespace, project, self)
retry_namespace_project_build_path(project.namespace, project, self)
end end
end end
...@@ -321,20 +321,35 @@ module Ci ...@@ -321,20 +321,35 @@ module Ci
pending? && !any_runners_online? pending? && !any_runners_online?
end end
def download_url
if artifacts_file.exists?
Gitlab::Application.routes.url_helpers.
download_namespace_project_build_path(project.namespace, project, self)
end
end
def execute_hooks def execute_hooks
build_data = Gitlab::BuildDataBuilder.build(self) build_data = Gitlab::BuildDataBuilder.build(self)
project.execute_hooks(build_data.dup, :build_hooks) project.execute_hooks(build_data.dup, :build_hooks)
project.execute_services(build_data.dup, :build_hooks) project.execute_services(build_data.dup, :build_hooks)
end end
def artifacts?
artifacts_file.exists?
end
def artifacts_download_url
if artifacts?
download_namespace_project_build_artifacts_path(project.namespace, project, self)
end
end
def artifacts_browse_url
if artifacts_browser_supported?
browse_namespace_project_build_artifacts_path(project.namespace, project, self)
end
end
def artifacts_browser_supported?
artifacts? && artifacts_metadata.exists?
end
def artifacts_metadata_entry(path)
Gitlab::Ci::Build::Artifacts::Metadata.new(artifacts_metadata.path, path).to_entry
end
private private
......
...@@ -131,7 +131,11 @@ class CommitStatus < ActiveRecord::Base ...@@ -131,7 +131,11 @@ class CommitStatus < ActiveRecord::Base
false false
end end
def download_url def artifacts_download_url
nil
end
def artifacts_browse_url
nil nil
end end
end end
...@@ -60,8 +60,8 @@ ...@@ -60,8 +60,8 @@
%td %td
.pull-right .pull-right
- if current_user && can?(current_user, :download_build_artifacts, project) && build.download_url - if current_user && can?(current_user, :read_build_artifacts, project) && build.artifacts?
= link_to build.download_url, title: 'Download artifacts' do = link_to build.artifacts_download_url, title: 'Download artifacts' do
%i.fa.fa-download %i.fa.fa-download
- if current_user && can?(current_user, :manage_builds, build.project) - if current_user && can?(current_user, :manage_builds, build.project)
- if build.active? - if build.active?
......
%tr{ class: 'tree-item' }
%td.tree-item-file-name
= tree_icon('folder', '755', directory.name)
%span.str-truncated
= link_to directory.name, browse_namespace_project_build_artifacts_path(@project.namespace, @project, @build, path: directory.path)
%td
%td
%tr{ class: 'tree-item' }
%td.tree-item-file-name
= tree_icon('file', '664', file.name)
%span.str-truncated
= file.name
%td
= number_to_human_size(file.metadata[:size], precision: 2)
%td
= link_to file_namespace_project_build_artifacts_path(@project.namespace, @project, @build, path: file.path),
class: 'btn btn-xs btn-default artifact-download' do
= icon('download')
- page_title 'Artifacts', "#{@build.name} (##{@build.id})", 'Builds'
= render 'projects/builds/header_title'
#tree-holder.tree-holder
.gray-content-block.top-block.clearfix
.pull-right
= link_to download_namespace_project_build_artifacts_path(@project.namespace, @project, @build),
class: 'btn btn-default' do
= icon('download')
Download artifacts archive
%div.tree-content-holder
.table-holder
%table.table.tree-table.table-striped
%thead
%tr
%th Name
%th Size
%th Download
= render partial: 'tree_directory', collection: @entry.directories(parent: true), as: :directory
= render partial: 'tree_file', collection: @entry.files, as: :file
- if @entry.empty?
.center Empty
...@@ -89,9 +89,15 @@ ...@@ -89,9 +89,15 @@
Test coverage Test coverage
%h1 #{@build.coverage}% %h1 #{@build.coverage}%
- if current_user && can?(current_user, :download_build_artifacts, @project) && @build.download_url - if current_user && can?(current_user, :read_build_artifacts, @project) && @build.artifacts?
.build-widget.center
= link_to "Download artifacts", @build.download_url, class: 'btn btn-sm btn-primary' .build-widget.artifacts
%h4.title Build artifacts
.center
.btn-group{ role: :group }
= link_to "Download", @build.artifacts_download_url, class: 'btn btn-sm btn-primary'
- if @build.artifacts_browser_supported?
= link_to "Browse", @build.artifacts_browse_url, class: 'btn btn-sm btn-primary'
.build-widget .build-widget
%h4.title %h4.title
......
...@@ -66,8 +66,8 @@ ...@@ -66,8 +66,8 @@
%td %td
.pull-right .pull-right
- if current_user && can?(current_user, :download_build_artifacts, commit_status.project) && commit_status.download_url - if current_user && can?(current_user, :read_build_artifacts, commit_status.project) && commit_status.artifacts?
= link_to commit_status.download_url, title: 'Download artifacts' do = link_to commit_status.artifacts_download_url, title: 'Download artifacts' do
%i.fa.fa-download %i.fa.fa-download
- if current_user && can?(current_user, :manage_builds, commit_status.project) - if current_user && can?(current_user, :manage_builds, commit_status.project)
- if commit_status.active? - if commit_status.active?
......
...@@ -604,9 +604,14 @@ Rails.application.routes.draw do ...@@ -604,9 +604,14 @@ Rails.application.routes.draw do
member do member do
get :status get :status
post :cancel post :cancel
get :download
post :retry post :retry
end end
resource :artifacts, only: [] do
get :download
get :browse, path: 'browse(/*path)', format: false
get :file, path: 'file/*path', format: false
end
end end
resources :hooks, only: [:index, :create, :destroy], constraints: { id: /\d+/ } do resources :hooks, only: [:index, :create, :destroy], constraints: { id: /\d+/ } do
......
class Gitlab::Seeder::Builds
BUILD_STATUSES = %w(running pending success failed canceled)
def initialize(project)
@project = project
end
def seed!
ci_commits.each do |ci_commit|
build = Ci::Build.new(build_attributes_for(ci_commit))
artifacts_cache_file(artifacts_archive_path) do |file|
build.artifacts_file = file
end
artifacts_cache_file(artifacts_metadata_path) do |file|
build.artifacts_metadata = file
end
begin
build.save!
print '.'
rescue ActiveRecord::RecordInvalid
print 'F'
end
end
end
def ci_commits
commits = @project.repository.commits('master', nil, 5)
commits_sha = commits.map { |commit| commit.raw.id }
commits_sha.map do |sha|
@project.ensure_ci_commit(sha)
end
rescue
[]
end
def build_attributes_for(ci_commit)
{ name: 'test build', commands: "$ build command",
stage: 'test', stage_idx: 1, ref: 'master',
user_id: build_user, gl_project_id: @project.id,
status: build_status, commit_id: ci_commit.id,
created_at: Time.now, updated_at: Time.now }
end
def build_user
@project.team.users.sample
end
def build_status
BUILD_STATUSES.sample
end
def artifacts_archive_path
Rails.root + 'spec/fixtures/ci_build_artifacts.zip'
end
def artifacts_metadata_path
Rails.root + 'spec/fixtures/ci_build_artifacts_metadata.gz'
end
def artifacts_cache_file(file_path)
cache_path = file_path.to_s.gsub('ci_', "p#{@project.id}_")
FileUtils.copy(file_path, cache_path)
File.open(cache_path) do |file|
yield file
end
end
end
Gitlab::Seeder.quiet do
Project.all.sample(10).each do |project|
project_builds = Gitlab::Seeder::Builds.new(project)
project_builds.seed!
end
end
class AddArtifactsMetadataToCiBuild < ActiveRecord::Migration
def change
add_column :ci_builds, :artifacts_metadata, :text
end
end
...@@ -123,6 +123,7 @@ ActiveRecord::Schema.define(version: 20160113111034) do ...@@ -123,6 +123,7 @@ ActiveRecord::Schema.define(version: 20160113111034) do
t.string "description" t.string "description"
t.text "artifacts_file" t.text "artifacts_file"
t.integer "gl_project_id" t.integer "gl_project_id"
t.text "artifacts_metadata"
end end
add_index "ci_builds", ["commit_id", "stage_idx", "created_at"], name: "index_ci_builds_on_commit_id_and_stage_idx_and_created_at", using: :btree add_index "ci_builds", ["commit_id", "stage_idx", "created_at"], name: "index_ci_builds_on_commit_id_and_stage_idx_and_created_at", using: :btree
......
Feature: Project Builds
Background:
Given I sign in as a user
And I own a project
And CI is enabled
And I have recent build for my project
Scenario: I browse build summary page
When I visit recent build summary page
Then I see summary for build
And I see build trace
Scenario: I download build artifacts
Given recent build has artifacts available
When I visit recent build summary page
And I click artifacts download button
Then download of build artifacts archive starts
Scenario: I browse build artifacts
Given recent build has artifacts available
And recent build has artifacts metadata available
When I visit recent build summary page
And I click artifacts browse button
Then I should see content of artifacts archive
Scenario: I browse subdirectory of build artifacts
Given recent build has artifacts available
And recent build has artifacts metadata available
When I visit recent build summary page
And I click artifacts browse button
And I click link to subdirectory within build artifacts
Then I should see content of subdirectory within artifacts archive
Scenario: I browse directory with UTF-8 characters in name
Given recent build has artifacts available
And recent build has artifacts metadata available
And recent build artifacts contain directory with UTF-8 characters
When I visit recent build summary page
And I click artifacts browse button
And I navigate to directory with UTF-8 characters in name
Then I should see content of directory with UTF-8 characters in name
Scenario: I try to browse directory with invalid UTF-8 characters in name
Given recent build has artifacts available
And recent build has artifacts metadata available
And recent build artifacts contain directory with invalid UTF-8 characters
When I visit recent build summary page
And I click artifacts browse button
And I navigate to parent directory of directory with invalid name
Then I should not see directory with invalid name on the list
Scenario: I download a single file from build artifacts
Given recent build has artifacts available
And recent build has artifacts metadata available
When I visit recent build summary page
And I click artifacts browse button
And I click download button for a file within build artifacts
Then download of a file extracted from build artifacts should start
class Spinach::Features::ProjectBuilds < Spinach::FeatureSteps
include SharedAuthentication
include SharedProject
include SharedBuilds
include RepoHelpers
step 'I see summary for build' do
expect(page).to have_content "Build ##{@build.id}"
end
step 'I see build trace' do
expect(page).to have_css '#build-trace'
end
step 'I click artifacts download button' do
page.within('.artifacts') { click_link 'Download' }
end
step 'download of build artifacts archive starts' do
expect(page.response_headers['Content-Type']).to eq 'application/zip'
expect(page.response_headers['Content-Transfer-Encoding']).to eq 'binary'
end
step 'I click artifacts browse button' do
page.within('.artifacts') { click_link 'Browse' }
end
step 'I should see content of artifacts archive' do
page.within('.tree-table') do
expect(page).to have_no_content '..'
expect(page).to have_content 'other_artifacts_0.1.2'
expect(page).to have_content 'ci_artifacts.txt'
expect(page).to have_content 'rails_sample.jpg'
end
end
step 'I click link to subdirectory within build artifacts' do
page.within('.tree-table') { click_link 'other_artifacts_0.1.2' }
end
step 'I should see content of subdirectory within artifacts archive' do
page.within('.tree-table') do
expect(page).to have_content '..'
expect(page).to have_content 'another-subdirectory'
expect(page).to have_content 'doc_sample.txt'
end
end
step 'recent build artifacts contain directory with UTF-8 characters' do
# metadata fixture contains relevant directory
end
step 'I navigate to directory with UTF-8 characters in name' do
page.within('.tree-table') { click_link 'tests_encoding' }
page.within('.tree-table') { click_link 'utf8 test dir ✓' }
end
step 'I should see content of directory with UTF-8 characters in name' do
page.within('.tree-table') do
expect(page).to have_content '..'
expect(page).to have_content 'regular_file_2'
end
end
step 'recent build artifacts contain directory with invalid UTF-8 characters' do
# metadata fixture contains relevant directory
end
step 'I navigate to parent directory of directory with invalid name' do
page.within('.tree-table') { click_link 'tests_encoding' }
end
step 'I should not see directory with invalid name on the list' do
page.within('.tree-table') do
expect(page).to have_no_content('non-utf8-dir')
end
end
step 'I click download button for a file within build artifacts' do
page.within('.tree-table') { first('.artifact-download').click }
end
step 'download of a file extracted from build artifacts should start' do
# this will be accelerated by Workhorse
response_json = JSON.parse(page.body, symbolize_names: true)
expect(response_json[:archive]).to end_with('build_artifacts.zip')
expect(response_json[:entry]).to eq Base64.encode64('ci_artifacts.txt')
end
end
module SharedBuilds
include Spinach::DSL
step 'CI is enabled' do
@project.enable_ci
end
step 'I have recent build for my project' do
ci_commit = create :ci_commit, project: @project, sha: sample_commit.id
@build = create :ci_build, commit: ci_commit
end
step 'I visit recent build summary page' do
visit namespace_project_build_path(@project.namespace, @project, @build)
end
step 'recent build has artifacts available' do
artifacts = Rails.root + 'spec/fixtures/ci_build_artifacts.zip'
archive = fixture_file_upload(artifacts, 'application/zip')
@build.update_attributes(artifacts_file: archive)
end
step 'recent build has artifacts metadata available' do
metadata = Rails.root + 'spec/fixtures/ci_build_artifacts_metadata.gz'
gzip = fixture_file_upload(metadata, 'application/x-gzip')
@build.update_attributes(artifacts_metadata: gzip)
end
end
...@@ -289,12 +289,14 @@ module API ...@@ -289,12 +289,14 @@ module API
# file helpers # file helpers
def uploaded_file!(field, uploads_path) def uploaded_file(field, uploads_path)
if params[field] if params[field]
bad_request!("#{field} is not a file") unless params[field].respond_to?(:filename) bad_request!("#{field} is not a file") unless params[field].respond_to?(:filename)
return params[field] return params[field]
end end
return nil unless params["#{field}.path"] && params["#{field}.name"]
# sanitize file paths # sanitize file paths
# this requires all paths to exist # this requires all paths to exist
required_attributes! %W(#{field}.path) required_attributes! %W(#{field}.path)
......
...@@ -78,11 +78,13 @@ module Ci ...@@ -78,11 +78,13 @@ module Ci
# Parameters: # Parameters:
# id (required) - The ID of a build # id (required) - The ID of a build
# token (required) - The build authorization token # token (required) - The build authorization token
# file (required) - The uploaded file # file (required) - Artifacts file
# Parameters (accelerated by GitLab Workhorse): # Parameters (accelerated by GitLab Workhorse):
# file.path - path to locally stored body (generated by Workhorse) # file.path - path to locally stored body (generated by Workhorse)
# file.name - real filename as send in Content-Disposition # file.name - real filename as send in Content-Disposition
# file.type - real content type as send in Content-Type # file.type - real content type as send in Content-Type
# metadata.path - path to locally stored body (generated by Workhorse)
# metadata.name - filename (generated by Workhorse)
# Headers: # Headers:
# BUILD-TOKEN (required) - The build authorization token, the same as token # BUILD-TOKEN (required) - The build authorization token, the same as token
# Body: # Body:
...@@ -96,13 +98,20 @@ module Ci ...@@ -96,13 +98,20 @@ module Ci
build = Ci::Build.find_by_id(params[:id]) build = Ci::Build.find_by_id(params[:id])
not_found! unless build not_found! unless build
authenticate_build_token!(build) authenticate_build_token!(build)
forbidden!('build is not running') unless build.running? forbidden!('Build is not running!') unless build.running?
file = uploaded_file!(:file, ArtifactUploader.artifacts_upload_path) artifacts_upload_path = ArtifactUploader.artifacts_upload_path
file_to_large! unless file.size < max_artifacts_size artifacts = uploaded_file(:file, artifacts_upload_path)
metadata = uploaded_file(:metadata, artifacts_upload_path)
if build.update_attributes(artifacts_file: file) bad_request!('Missing artifacts file!') unless artifacts
present build, with: Entities::Build file_to_large! unless artifacts.size < max_artifacts_size
build.artifacts_file = artifacts
build.artifacts_metadata = metadata
if build.save
present(build, with: Entities::Build)
else else
render_validation_error!(build) render_validation_error!(build)
end end
...@@ -148,6 +157,7 @@ module Ci ...@@ -148,6 +157,7 @@ module Ci
not_found! unless build not_found! unless build
authenticate_build_token!(build) authenticate_build_token!(build)
build.remove_artifacts_file! build.remove_artifacts_file!
build.remove_artifacts_metadata!
end end
end end
end end
......
require 'zlib'
require 'json'
module Gitlab
module Ci
module Build
module Artifacts
class Metadata
class ParserError < StandardError; end
VERSION_PATTERN = /^[\w\s]+(\d+\.\d+\.\d+)/
INVALID_PATH_PATTERN = %r{(^\.?\.?/)|(/\.?\.?/)}
attr_reader :file, :path, :full_version
def initialize(file, path)
@file, @path = file, path
@full_version = read_version
end
def version
@full_version.match(VERSION_PATTERN)[1]
end
def errors
gzip do |gz|
read_string(gz) # version
errors = read_string(gz)
raise ParserError, 'Errors field not found!' unless errors
begin
JSON.parse(errors)
rescue JSON::ParserError
raise ParserError, 'Invalid errors field!'
end
end
end
def find_entries!
gzip do |gz|
2.times { read_string(gz) } # version and errors fields
match_entries(gz)
end
end
def to_entry
entries = find_entries!
Entry.new(@path, entries)
end
private
def match_entries(gz)
entries = {}
match_pattern = %r{^#{Regexp.escape(@path)}[^/]*/?$}
until gz.eof? do
begin
path = read_string(gz).force_encoding('UTF-8')
meta = read_string(gz).force_encoding('UTF-8')
next unless path.valid_encoding? && meta.valid_encoding?
next unless path =~ match_pattern
next if path =~ INVALID_PATH_PATTERN
entries[path] = JSON.parse(meta, symbolize_names: true)
rescue JSON::ParserError, Encoding::CompatibilityError
next
end
end
entries
end
def read_version
gzip do |gz|
version_string = read_string(gz)
unless version_string
raise ParserError, 'Artifacts metadata file empty!'
end
unless version_string =~ VERSION_PATTERN
raise ParserError, 'Invalid version!'
end
version_string.chomp
end
end
def read_uint32(gz)
binary = gz.read(4)
binary.unpack('L>')[0] if binary
end
def read_string(gz)
string_size = read_uint32(gz)
return nil unless string_size
gz.read(string_size)
end
def gzip(&block)
Zlib::GzipReader.open(@file, &block)
end
end
end
end
end
end
module Gitlab
module Ci::Build::Artifacts
class Metadata
##
# Class that represents an entry (path and metadata) to a file or
# directory in GitLab CI Build Artifacts binary file / archive
#
# This is IO-operations safe class, that does similar job to
# Ruby's Pathname but without the risk of accessing filesystem.
#
# This class is working only with UTF-8 encoded paths.
#
class Entry
attr_reader :path, :entries
attr_accessor :name
def initialize(path, entries)
@path = path.dup.force_encoding('UTF-8')
@entries = entries
if path.include?("\0")
raise ArgumentError, 'Path contains zero byte character!'
end
unless path.valid_encoding?
raise ArgumentError, 'Path contains non-UTF-8 byte sequence!'
end
end
def directory?
blank_node? || @path.end_with?('/')
end
def file?
!directory?
end
def has_parent?
nodes > 0
end
def parent
return nil unless has_parent?
self.class.new(@path.chomp(basename), @entries)
end
def basename
(directory? && !blank_node?) ? name + '/' : name
end
def name
@name || @path.split('/').last.to_s
end
def children
return [] unless directory?
return @children if @children
child_pattern = %r{^#{Regexp.escape(@path)}[^/]+/?$}
@children = select_entries { |path| path =~ child_pattern }
end
def directories(opts = {})
return [] unless directory?
dirs = children.select(&:directory?)
return dirs unless has_parent? && opts[:parent]
dotted_parent = parent
dotted_parent.name = '..'
dirs.prepend(dotted_parent)
end
def files
return [] unless directory?
children.select(&:file?)
end
def metadata
@entries[@path] || {}
end
def nodes
@path.count('/') + (file? ? 1 : 0)
end
def blank_node?
@path.empty? # "" is considered to be './'
end
def exists?
blank_node? || @entries.include?(@path)
end
def empty?
children.empty?
end
def to_s
@path
end
def ==(other)
@path == other.path && @entries == other.entries
end
def inspect
"#{self.class.name}: #{@path}"
end
private
def select_entries
selected = @entries.select { |path, _metadata| yield path }
selected.map { |path, _metadata| self.class.new(path, @entries) }
end
end
end
end
end
...@@ -80,7 +80,11 @@ describe "Builds" do ...@@ -80,7 +80,11 @@ describe "Builds" do
visit namespace_project_build_path(@project.namespace, @project, @build) visit namespace_project_build_path(@project.namespace, @project, @build)
end end
it { expect(page).to have_content 'Download artifacts' } it 'has button to download artifacts' do
page.within('.artifacts') do
expect(page).to have_content 'Download'
end
end
end end
end end
...@@ -111,7 +115,7 @@ describe "Builds" do ...@@ -111,7 +115,7 @@ describe "Builds" do
before do before do
@build.update_attributes(artifacts_file: artifacts_file) @build.update_attributes(artifacts_file: artifacts_file)
visit namespace_project_build_path(@project.namespace, @project, @build) visit namespace_project_build_path(@project.namespace, @project, @build)
click_link 'Download artifacts' page.within('.artifacts') { click_link 'Download' }
end end
it { expect(page.response_headers['Content-Type']).to eq(artifacts_file.content_type) } it { expect(page.response_headers['Content-Type']).to eq(artifacts_file.content_type) }
......
require 'spec_helper'
describe Gitlab::Ci::Build::Artifacts::Metadata::Entry do
let(:entries) do
{ 'path/' => {},
'path/dir_1/' => {},
'path/dir_1/file_1' => {},
'path/dir_1/file_b' => {},
'path/dir_1/subdir/' => {},
'path/dir_1/subdir/subfile' => {},
'path/second_dir' => {},
'path/second_dir/dir_3/file_2' => {},
'path/second_dir/dir_3/file_3'=> {},
'another_directory/'=> {},
'another_file' => {},
'/file/with/absolute_path' => {} }
end
def path(example)
entry(example.metadata[:path])
end
def entry(path)
described_class.new(path, entries)
end
describe '/file/with/absolute_path', path: '/file/with/absolute_path' do
subject { |example| path(example) }
it { is_expected.to be_file }
it { is_expected.to have_parent }
describe '#basename' do
subject { |example| path(example).basename }
it { is_expected.to eq 'absolute_path' }
end
end
describe 'path/dir_1/', path: 'path/dir_1/' do
subject { |example| path(example) }
it { is_expected.to have_parent }
it { is_expected.to be_directory }
describe '#basename' do
subject { |example| path(example).basename }
it { is_expected.to eq 'dir_1/' }
end
describe '#name' do
subject { |example| path(example).name }
it { is_expected.to eq 'dir_1' }
end
describe '#parent' do
subject { |example| path(example).parent }
it { is_expected.to eq entry('path/') }
end
describe '#children' do
subject { |example| path(example).children }
it { is_expected.to all(be_an_instance_of described_class) }
it do
is_expected.to contain_exactly entry('path/dir_1/file_1'),
entry('path/dir_1/file_b'),
entry('path/dir_1/subdir/')
end
end
describe '#files' do
subject { |example| path(example).files }
it { is_expected.to all(be_file) }
it { is_expected.to all(be_an_instance_of described_class) }
it do
is_expected.to contain_exactly entry('path/dir_1/file_1'),
entry('path/dir_1/file_b')
end
end
describe '#directories' do
context 'without options' do
subject { |example| path(example).directories }
it { is_expected.to all(be_directory) }
it { is_expected.to all(be_an_instance_of described_class) }
it { is_expected.to contain_exactly entry('path/dir_1/subdir/') }
end
context 'with option parent: true' do
subject { |example| path(example).directories(parent: true) }
it { is_expected.to all(be_directory) }
it { is_expected.to all(be_an_instance_of described_class) }
it do
is_expected.to contain_exactly entry('path/dir_1/subdir/'),
entry('path/')
end
end
describe '#nodes' do
subject { |example| path(example).nodes }
it { is_expected.to eq 2 }
end
describe '#exists?' do
subject { |example| path(example).exists? }
it { is_expected.to be true }
end
describe '#empty?' do
subject { |example| path(example).empty? }
it { is_expected.to be false }
end
end
end
describe 'empty path', path: '' do
subject { |example| path(example) }
it { is_expected.to_not have_parent }
describe '#children' do
subject { |example| path(example).children }
it { expect(subject.count).to eq 3 }
end
end
describe 'path/dir_1/subdir/subfile', path: 'path/dir_1/subdir/subfile' do
describe '#nodes' do
subject { |example| path(example).nodes }
it { is_expected.to eq 4 }
end
end
describe 'non-existent/', path: 'non-existent/' do
describe '#empty?' do
subject { |example| path(example).empty? }
it { is_expected.to be true }
end
describe '#exists?' do
subject { |example| path(example).exists? }
it { is_expected.to be false }
end
end
describe 'another_directory/', path: 'another_directory/' do
describe '#empty?' do
subject { |example| path(example).empty? }
it { is_expected.to be true }
end
end
describe '#metadata' do
let(:entries) do
{ 'path/' => { name: '/path/' },
'path/file1' => { name: '/path/file1' },
'path/file2' => { name: '/path/file2' } }
end
subject do
described_class.new('path/file1', entries).metadata[:name]
end
it { is_expected.to eq '/path/file1' }
end
end
require 'spec_helper'
describe Gitlab::Ci::Build::Artifacts::Metadata do
def metadata(path = '')
described_class.new(metadata_file_path, path)
end
let(:metadata_file_path) do
Rails.root + 'spec/fixtures/ci_build_artifacts_metadata.gz'
end
context 'metadata file exists' do
describe '#find_entries! empty string' do
subject { metadata('').find_entries! }
it 'matches correct paths' do
expect(subject.keys).to contain_exactly 'ci_artifacts.txt',
'other_artifacts_0.1.2/',
'rails_sample.jpg',
'tests_encoding/'
end
it 'matches metadata for every path' do
expect(subject.keys.count).to eq 4
end
it 'return Hashes for each metadata' do
expect(subject.values).to all(be_kind_of(Hash))
end
end
describe '#find_entries! other_artifacts_0.1.2/' do
subject { metadata('other_artifacts_0.1.2/').find_entries! }
it 'matches correct paths' do
expect(subject.keys).
to contain_exactly 'other_artifacts_0.1.2/',
'other_artifacts_0.1.2/doc_sample.txt',
'other_artifacts_0.1.2/another-subdirectory/'
end
end
describe '#find_entries! other_artifacts_0.1.2/another-subdirectory/' do
subject { metadata('other_artifacts_0.1.2/another-subdirectory/').find_entries! }
it 'matches correct paths' do
expect(subject.keys).
to contain_exactly 'other_artifacts_0.1.2/another-subdirectory/',
'other_artifacts_0.1.2/another-subdirectory/empty_directory/',
'other_artifacts_0.1.2/another-subdirectory/banana_sample.gif'
end
end
describe '#to_entry' do
subject { metadata('').to_entry }
it { is_expected.to be_an_instance_of(Gitlab::Ci::Build::Artifacts::Metadata::Entry) }
end
describe '#full_version' do
subject { metadata('').full_version }
it { is_expected.to eq 'GitLab Build Artifacts Metadata 0.0.1' }
end
describe '#version' do
subject { metadata('').version }
it { is_expected.to eq '0.0.1' }
end
describe '#errors' do
subject { metadata('').errors }
it { is_expected.to eq({}) }
end
end
context 'metadata file does not exist' do
let(:metadata_file_path) { '' }
describe '#find_entries!' do
it 'raises error' do
expect { metadata.find_entries! }.to raise_error(Errno::ENOENT)
end
end
end
end
# == Schema Information
#
# Table name: builds
#
# id :integer not null, primary key
# project_id :integer
# status :string(255)
# finished_at :datetime
# trace :text
# created_at :datetime
# updated_at :datetime
# started_at :datetime
# runner_id :integer
# commit_id :integer
# coverage :float
# commands :text
# job_id :integer
# name :string(255)
# deploy :boolean default(FALSE)
# options :text
# allow_failure :boolean default(FALSE), not null
# stage :string(255)
# trigger_request_id :integer
#
require 'spec_helper' require 'spec_helper'
describe Ci::Build, models: true do describe Ci::Build, models: true do
...@@ -368,21 +343,75 @@ describe Ci::Build, models: true do ...@@ -368,21 +343,75 @@ describe Ci::Build, models: true do
end end
end end
describe :download_url do describe :artifacts_download_url do
subject { build.download_url } subject { build.artifacts_download_url }
it "should be nil if artifact doesn't exist" do it "should be nil if artifact doesn't exist" do
build.update_attributes(artifacts_file: nil) build.update_attributes(artifacts_file: nil)
is_expected.to be_nil is_expected.to be_nil
end end
it 'should be nil if artifact exist' do it 'should not be nil if artifact exist' do
gif = fixture_file_upload(Rails.root + 'spec/fixtures/banana_sample.gif', 'image/gif') gif = fixture_file_upload(Rails.root + 'spec/fixtures/banana_sample.gif', 'image/gif')
build.update_attributes(artifacts_file: gif) build.update_attributes(artifacts_file: gif)
is_expected.to_not be_nil is_expected.to_not be_nil
end end
end end
describe :artifacts_browse_url do
subject { build.artifacts_browse_url }
it "should be nil if artifacts browser is unsupported" do
allow(build).to receive(:artifacts_browser_supported?).and_return(false)
is_expected.to be_nil
end
it 'should not be nil if artifacts browser is supported' do
allow(build).to receive(:artifacts_browser_supported?).and_return(true)
is_expected.to_not be_nil
end
end
describe :artifacts? do
subject { build.artifacts? }
context 'artifacts archive does not exist' do
before { build.update_attributes(artifacts_file: nil) }
it { is_expected.to be_falsy }
end
context 'artifacts archive exists' do
before do
gif = fixture_file_upload(Rails.root + 'spec/fixtures/banana_sample.gif', 'image/gif')
build.update_attributes(artifacts_file: gif)
end
it { is_expected.to be_truthy }
end
end
describe :artifacts_browser_supported? do
subject { build.artifacts_browser_supported? }
context 'artifacts metadata does not exist' do
it { is_expected.to be_falsy }
end
context 'artifacts archive is a zip file and metadata exists' do
before do
fixture_dir = Rails.root + 'spec/fixtures/'
archive = fixture_file_upload(fixture_dir + 'ci_build_artifacts.zip',
'application/zip')
metadata = fixture_file_upload(fixture_dir + 'ci_build_artifacts_metadata.gz',
'application/x-gzip')
build.update_attributes(artifacts_file: archive)
build.update_attributes(artifacts_metadata: metadata)
end
it { is_expected.to be_truthy }
end
end
describe :repo_url do describe :repo_url do
let(:build) { FactoryGirl.create :ci_build } let(:build) { FactoryGirl.create :ci_build }
let(:project) { build.project } let(:project) { build.project }
......
...@@ -210,6 +210,52 @@ describe Ci::API::API do ...@@ -210,6 +210,52 @@ describe Ci::API::API do
end end
end end
context 'should post artifacts file and metadata file' do
let!(:artifacts) { file_upload }
let!(:metadata) { file_upload2 }
let(:stored_artifacts_file) { build.reload.artifacts_file.file }
let(:stored_metadata_file) { build.reload.artifacts_metadata.file }
before do
build.run!
post(post_url, post_data, headers_with_token)
end
context 'post data accelerated by workhorse is correct' do
let(:post_data) do
{ 'file.path' => artifacts.path,
'file.name' => artifacts.original_filename,
'metadata.path' => metadata.path,
'metadata.name' => metadata.original_filename }
end
it 'responds with valid status' do
expect(response.status).to eq(201)
end
it 'stores artifacts and artifacts metadata' do
expect(stored_artifacts_file.original_filename).to eq(artifacts.original_filename)
expect(stored_metadata_file.original_filename).to eq(metadata.original_filename)
end
end
context 'no artifacts file in post data' do
let(:post_data) do
{ 'metadata' => metadata }
end
it 'is expected to respond with bad request' do
expect(response.status).to eq(400)
end
it 'does not store metadata' do
expect(stored_metadata_file).to be_nil
end
end
end
context "should fail to post too large artifact" do context "should fail to post too large artifact" do
before do before do
build.run! build.run!
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment