Commit 4c622b71 authored by Robert Speicher's avatar Robert Speicher

Add Upload model and UploadChecksumWorker worker

parent 2a2b24d5
class Upload < ActiveRecord::Base
# Upper limit for foreground checksum processing
CHECKSUM_THRESHOLD = 100.megabytes
belongs_to :model, polymorphic: true
validates :size, presence: true
validates :path, presence: true
validates :model, presence: true
validates :uploader, presence: true
before_save :calculate_checksum, if: :foreground_checksum?
after_commit :schedule_checksum, unless: :foreground_checksum?
def self.remove_path(path)
where(path: path).destroy_all
end
def self.record(uploader)
remove_path(uploader.relative_path)
create(
size: uploader.file.size,
path: uploader.relative_path,
model: uploader.model,
uploader: uploader.class.to_s
)
end
def absolute_path
return path unless relative_path?
uploader_class.absolute_path(self)
end
def calculate_checksum
return unless exist?
self.checksum = Digest::SHA256.file(absolute_path).hexdigest
end
def exist?
File.exist?(absolute_path)
end
private
def foreground_checksum?
size <= CHECKSUM_THRESHOLD
end
def schedule_checksum
UploadChecksumWorker.perform_async(id)
end
def relative_path?
!path.start_with?('/')
end
def uploader_class
Object.const_get(uploader)
end
end
class UploadChecksumWorker
include Sidekiq::Worker
include DedicatedSidekiqQueue
def perform(upload_id)
upload = Upload.find(upload_id)
upload.calculate_checksum
upload.save!
rescue ActiveRecord::RecordNotFound
Rails.logger.error("UploadChecksumWorker: couldn't find upload #{upload_id}, skipping")
end
end
...@@ -29,6 +29,7 @@ ...@@ -29,6 +29,7 @@
- [email_receiver, 2] - [email_receiver, 2]
- [emails_on_push, 2] - [emails_on_push, 2]
- [mailers, 2] - [mailers, 2]
- [upload_checksum, 1]
- [use_key, 1] - [use_key, 1]
- [repository_fork, 1] - [repository_fork, 1]
- [repository_import, 1] - [repository_import, 1]
......
class CreateUploads < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
def change
create_table :uploads do |t|
t.integer :size, limit: 8, null: false
t.string :path, null: false
t.string :checksum, limit: 64
t.references :model, polymorphic: true
t.string :uploader, null: false
t.datetime :created_at, null: false
end
add_index :uploads, :path
add_index :uploads, :checksum
add_index :uploads, [:model_id, :model_type]
end
end
...@@ -1214,6 +1214,20 @@ ActiveRecord::Schema.define(version: 20170306170512) do ...@@ -1214,6 +1214,20 @@ ActiveRecord::Schema.define(version: 20170306170512) do
add_index "u2f_registrations", ["key_handle"], name: "index_u2f_registrations_on_key_handle", using: :btree add_index "u2f_registrations", ["key_handle"], name: "index_u2f_registrations_on_key_handle", using: :btree
add_index "u2f_registrations", ["user_id"], name: "index_u2f_registrations_on_user_id", using: :btree add_index "u2f_registrations", ["user_id"], name: "index_u2f_registrations_on_user_id", using: :btree
create_table "uploads", force: :cascade do |t|
t.integer "size", limit: 8, null: false
t.string "path", null: false
t.string "checksum", limit: 64
t.integer "model_id"
t.string "model_type"
t.string "uploader", null: false
t.datetime "created_at", null: false
end
add_index "uploads", ["checksum"], name: "index_uploads_on_checksum", using: :btree
add_index "uploads", ["model_id", "model_type"], name: "index_uploads_on_model_id_and_model_type", using: :btree
add_index "uploads", ["path"], name: "index_uploads_on_path", using: :btree
create_table "user_agent_details", force: :cascade do |t| create_table "user_agent_details", force: :cascade do |t|
t.string "user_agent", null: false t.string "user_agent", null: false
t.string "ip_address", null: false t.string "ip_address", null: false
......
require 'rails_helper'
describe Upload, type: :model do
describe 'assocations' do
it { is_expected.to belong_to(:model) }
end
describe 'validations' do
it { is_expected.to validate_presence_of(:size) }
it { is_expected.to validate_presence_of(:path) }
it { is_expected.to validate_presence_of(:model) }
it { is_expected.to validate_presence_of(:uploader) }
end
describe 'callbacks' do
context 'for a file above the checksum threshold' do
it 'schedules checksum calculation' do
stub_const('UploadChecksumWorker', spy)
upload = described_class.create(
path: __FILE__,
size: described_class::CHECKSUM_THRESHOLD + 1.kilobyte,
model: build_stubbed(:user),
uploader: double('ExampleUploader')
)
expect(UploadChecksumWorker)
.to have_received(:perform_async).with(upload.id)
end
end
context 'for a file at or below the checksum threshold' do
it 'calculates checksum immediately before save' do
upload = described_class.new(
path: __FILE__,
size: described_class::CHECKSUM_THRESHOLD,
model: build_stubbed(:user),
uploader: double('ExampleUploader')
)
expect { upload.save }
.to change { upload.checksum }.from(nil)
.to(a_string_matching(/\A\h{64}\z/))
end
end
end
describe '.remove_path' do
it 'removes all records at the given path' do
described_class.create!(
size: File.size(__FILE__),
path: __FILE__,
model: build_stubbed(:user),
uploader: 'AvatarUploader'
)
expect { described_class.remove_path(__FILE__) }.
to change { described_class.count }.from(1).to(0)
end
end
describe '.record' do
let(:fake_uploader) do
double(
file: double(size: 12_345),
relative_path: 'foo/bar.jpg',
model: build_stubbed(:user),
class: 'AvatarUploader'
)
end
it 'removes existing paths before creation' do
expect(described_class).to receive(:remove_path)
.with(fake_uploader.relative_path)
described_class.record(fake_uploader)
end
it 'creates a new record and assigns size, path, model, and uploader' do
upload = described_class.record(fake_uploader)
aggregate_failures do
expect(upload).to be_persisted
expect(upload.size).to eq fake_uploader.file.size
expect(upload.path).to eq fake_uploader.relative_path
expect(upload.model_id).to eq fake_uploader.model.id
expect(upload.model_type).to eq fake_uploader.model.class.to_s
expect(upload.uploader).to eq fake_uploader.class
end
end
end
describe '#absolute_path' do
it 'returns the path directly when already absolute' do
path = '/path/to/namespace/project/secret/file.jpg'
upload = described_class.new(path: path)
expect(upload).not_to receive(:uploader_class)
expect(upload.absolute_path).to eq path
end
it "delegates to the uploader's absolute_path method" do
uploader = spy('FakeUploader')
upload = described_class.new(path: 'secret/file.jpg')
expect(upload).to receive(:uploader_class).and_return(uploader)
upload.absolute_path
expect(uploader).to have_received(:absolute_path).with(upload)
end
end
describe '#calculate_checksum' do
it 'calculates the SHA256 sum' do
upload = described_class.new(
path: __FILE__,
size: described_class::CHECKSUM_THRESHOLD - 1.megabyte
)
expected = Digest::SHA256.file(__FILE__).hexdigest
expect { upload.calculate_checksum }
.to change { upload.checksum }.from(nil).to(expected)
end
it 'returns nil for a non-existant file' do
upload = described_class.new(
path: __FILE__,
size: described_class::CHECKSUM_THRESHOLD - 1.megabyte
)
expect(upload).to receive(:exist?).and_return(false)
expect(upload.calculate_checksum).to be_nil
end
end
describe '#exist?' do
it 'returns true when the file exists' do
upload = described_class.new(path: __FILE__)
expect(upload).to exist
end
it 'returns false when the file does not exist' do
upload = described_class.new(path: "#{__FILE__}-nope")
expect(upload).not_to exist
end
end
end
CarrierWave.root = 'tmp/tests/uploads' CarrierWave.root = File.expand_path('tmp/tests/uploads', Rails.root)
RSpec.configure do |config| RSpec.configure do |config|
config.after(:each) do config.after(:each) do
FileUtils.rm_rf('tmp/tests/uploads') FileUtils.rm_rf(CarrierWave.root)
end end
end end
require 'rails_helper'
describe UploadChecksumWorker do
describe '#perform' do
it 'rescues ActiveRecord::RecordNotFound' do
expect { described_class.new.perform(999_999) }.not_to raise_error
end
it 'calls calculate_checksum_without_delay and save!' do
upload = spy
expect(Upload).to receive(:find).with(999_999).and_return(upload)
described_class.new.perform(999_999)
expect(upload).to have_received(:calculate_checksum)
expect(upload).to have_received(:save!)
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment