Commit f7b8ae3f authored by Micaël Bergeron's avatar Micaël Bergeron

apply feedback

parent 6466739e
......@@ -51,7 +51,6 @@ module Ci
scope :with_artifacts_not_expired, ->() { with_artifacts.where('artifacts_expire_at IS NULL OR artifacts_expire_at > ?', Time.now) }
scope :with_expired_artifacts, ->() { with_artifacts.where('artifacts_expire_at < ?', Time.now) }
scope :with_artifacts_stored_locally, ->() { with_artifacts.where(artifacts_file_store: [nil, LegacyArtifactUploader::Store::LOCAL]) }
scope :last_month, ->() { where('created_at > ?', Date.today - 1.month) }
scope :manual_actions, ->() { where(when: :manual, status: COMPLETED_STATUSES + [:manual]) }
scope :ref_protected, -> { where(protected: true) }
......
......@@ -71,9 +71,9 @@ module Projects
end
def extract_archive!(temp_path)
if artifacts_filename.ends_with?('.tar.gz') || artifacts_filename.ends_with?('.tgz')
if artifacts.ends_with?('.tar.gz') || artifacts.ends_with?('.tgz')
extract_tar_archive!(temp_path)
elsif artifacts_filename.ends_with?('.zip')
elsif artifacts.ends_with?('.zip')
extract_zip_archive!(temp_path)
else
raise FailedToExtractError, 'unsupported artifacts format'
......@@ -86,7 +86,7 @@ module Projects
%W(dd bs=#{BLOCK_SIZE} count=#{blocks}),
%W(tar -x -C #{temp_path} #{SITE_PATH}),
err: '/dev/null')
raise 'pages failed to extract' unless results.compact.all?(&:success?)
raise FailedToExtractError, 'pages failed to extract' unless results.compact.all?(&:success?)
end
end
......@@ -107,7 +107,7 @@ module Projects
site_path = File.join(SITE_PATH, '*')
build.artifacts_file.use_file do |artifacts_path|
unless system(*%W(unzip -n #{artifacts_path} #{site_path} -d #{temp_path}))
raise 'pages failed to extract'
raise FailedToExtractError, 'pages failed to extract'
end
end
end
......@@ -139,10 +139,6 @@ module Projects
1 + max_size / BLOCK_SIZE
end
def artifacts_filename
build.artifacts_file.filename
end
def max_size
max_pages_size = Gitlab::CurrentSettings.max_pages_size.megabytes
......@@ -171,6 +167,15 @@ module Projects
build.ref
end
def artifacts
build.artifacts_file.path
end
def delete_artifact!
build.reload # Reload stable object to prevent erase artifacts with old state
build.erase_artifacts! unless build.has_expiring_artifacts?
end
def latest_sha
project.commit(build.ref).try(:sha).to_s
end
......
......@@ -108,5 +108,3 @@
- update_user_activity
- upload_checksum
- web_hook
# Geo with Object storage
Geo can be used in combination with Object Storage (AWS S3, or
other compatible object storage).
## Configuration
At this time it is required that if object storage is enabled on the
primary, it must also be enabled on the secondary.
The secondary nodes can use the same storage bucket as the primary, or
they can use a replicated storage bucket. At this time GitLab does not
take care of content replication in object storage.
For LFS, follow the documentation to
[set up LFS object storage](../../../workflow/lfs/lfs_administration.md#setting-up-s3-compatible-object-storage).
For CI job artifacts, there is similar documentation to configure
[jobs artifact object storage](../../job_artifacts.md#using-object-storage)
Complete these steps on all nodes, primary **and** secondary.
## Replication
When using Amazon S3, you can use
[CRR](https://docs.aws.amazon.com/AmazonS3/latest/dev/crr.html) to
have automatic replication between the bucket used by the primary and
the bucket used by the secondary.
If you are using Google Cloud Storage, consider using
[Multi-Regional Storage](https://cloud.google.com/storage/docs/storage-classes#multi-regional).
Or you can use the [Storage Transfer Service](https://cloud.google.com/storage/transfer/),
although this only supports daily synchronization.
For manual synchronization, or scheduled by `cron`, please have a look at:
- [`s3cmd sync`](http://s3tools.org/s3cmd-sync)
- [`gsutil rsync`](https://cloud.google.com/storage/docs/gsutil/commands/rsync)
This document was moved to [another location](../administration/geo/replication/object_storage.md).
......@@ -47,7 +47,7 @@ In `config/gitlab.yml`:
## Setting up S3 compatible object storage
> **Note:** [Introduced][ee-2760] in [GitLab Premium][eep] 10.0.
> Available in [GitLab CE][ce] 10.6
> Available in [GitLab CE][ce] 10.7
It is possible to store LFS objects on remote object storage instead of on a local disk.
......
module UploadTaskHelpers
def batch_size
ENV.fetch('BATCH', 200).to_i
end
def calculate_checksum(absolute_path)
Digest::SHA256.file(absolute_path).hexdigest
end
def check_checksum(upload)
checksum = calculate_checksum(upload.absolute_path)
if checksum != upload.checksum
puts " * File checksum (#{checksum}) does not match the one in the database (#{upload.checksum})".color(:red)
end
end
def uploads_batches(&block)
Upload.all.in_batches(of: batch_size, start: ENV['ID_FROM'], finish: ENV['ID_TO']) do |relation| # rubocop: disable Cop/InBatches
yield relation
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment