Commit 99a03fd6 authored by Bob Van Landuyt's avatar Bob Van Landuyt

Move ReservedPathsMigration into V1 namespace

parent 12735eef
...@@ -2,33 +2,36 @@ ...@@ -2,33 +2,36 @@
# for more information on how to write migrations for GitLab. # for more information on how to write migrations for GitLab.
class RenameReservedDynamicPaths < ActiveRecord::Migration class RenameReservedDynamicPaths < ActiveRecord::Migration
include Gitlab::Database::RenameReservedPathsMigration include Gitlab::Database::RenameReservedPathsMigration::V1
DOWNTIME = false DOWNTIME = false
disable_ddl_transaction! disable_ddl_transaction!
DISALLOWED_ROOT_PATHS = %w[ DISALLOWED_ROOT_PATHS = %w[
-
abuse_reports
api api
autocomplete autocomplete
member
explore explore
uploads health_check
import import
notification_settings
abuse_reports
invites invites
koding
health_check
jwt jwt
koding
member
notification_settings
oauth oauth
sent_notifications sent_notifications
- uploads
users users
] ]
DISALLOWED_WILDCARD_PATHS = %w[info/lfs/objects gitlab-lfs/objects DISALLOWED_WILDCARD_PATHS = %w[
environments/folders] environments/folders
gitlab-lfs/objects
info/lfs/objects
]
def up def up
rename_root_paths(DISALLOWED_ROOT_PATHS) rename_root_paths(DISALLOWED_ROOT_PATHS)
......
module Gitlab
module Database
module RenameReservedPathsMigration
def self.included(kls)
kls.include(MigrationHelpers)
end
def rename_wildcard_paths(one_or_more_paths)
paths = Array(one_or_more_paths)
RenameNamespaces.new(paths, self).rename_namespaces(type: :wildcard)
RenameProjects.new(paths, self).rename_projects
end
def rename_root_paths(paths)
paths = Array(paths)
RenameNamespaces.new(paths, self).rename_namespaces(type: :top_level)
end
end
end
end
module Gitlab
module Database
module RenameReservedPathsMigration
module MigrationClasses
module Routable
def full_path
if route && route.path.present?
@full_path ||= route.path
else
update_route if persisted?
build_full_path
end
end
def build_full_path
if parent && path
parent.full_path + '/' + path
else
path
end
end
def update_route
prepare_route
route.save
end
def prepare_route
route || build_route(source: self)
route.path = build_full_path
route.name = build_full_name
@full_path = nil
@full_name = nil
end
def build_full_name
if parent && name
parent.human_name + ' / ' + name
else
name
end
end
def human_name
owner&.name
end
end
class User < ActiveRecord::Base
self.table_name = 'users'
end
class Namespace < ActiveRecord::Base
include MigrationClasses::Routable
self.table_name = 'namespaces'
belongs_to :parent,
class_name: "#{MigrationClasses.name}::Namespace"
has_one :route, as: :source
has_many :children,
class_name: "#{MigrationClasses.name}::Namespace",
foreign_key: :parent_id
belongs_to :owner,
class_name: "#{MigrationClasses.name}::User"
# Overridden to have the correct `source_type` for the `route` relation
def self.name
'Namespace'
end
end
class Route < ActiveRecord::Base
self.table_name = 'routes'
belongs_to :source, polymorphic: true
end
class Project < ActiveRecord::Base
include MigrationClasses::Routable
has_one :route, as: :source
self.table_name = 'projects'
def repository_storage_path
Gitlab.config.repositories.storages[repository_storage]['path']
end
# Overridden to have the correct `source_type` for the `route` relation
def self.name
'Project'
end
end
end
end
end
end
module Gitlab
module Database
module RenameReservedPathsMigration
class RenameBase
attr_reader :paths, :migration
delegate :update_column_in_batches,
:replace_sql,
to: :migration
def initialize(paths, migration)
@paths = paths
@migration = migration
end
def path_patterns
@path_patterns ||= paths.map { |path| "%#{path}" }
end
def rename_path_for_routable(routable)
old_path = routable.path
old_full_path = routable.full_path
# Only remove the last occurrence of the path name to get the parent namespace path
namespace_path = remove_last_occurrence(old_full_path, old_path)
new_path = rename_path(namespace_path, old_path)
new_full_path = join_routable_path(namespace_path, new_path)
# skips callbacks & validations
routable.class.where(id: routable).
update_all(path: new_path)
rename_routes(old_full_path, new_full_path)
[old_full_path, new_full_path]
end
def rename_routes(old_full_path, new_full_path)
replace_statement = replace_sql(Route.arel_table[:path],
old_full_path,
new_full_path)
update_column_in_batches(:routes, :path, replace_statement) do |table, query|
query.where(MigrationClasses::Route.arel_table[:path].matches("#{old_full_path}%"))
end
end
def rename_path(namespace_path, path_was)
counter = 0
path = "#{path_was}#{counter}"
while route_exists?(join_routable_path(namespace_path, path))
counter += 1
path = "#{path_was}#{counter}"
end
path
end
def remove_last_occurrence(string, pattern)
string.reverse.sub(pattern.reverse, "").reverse
end
def join_routable_path(namespace_path, top_level)
if namespace_path.present?
File.join(namespace_path, top_level)
else
top_level
end
end
def route_exists?(full_path)
MigrationClasses::Route.where(Route.arel_table[:path].matches(full_path)).any?
end
def move_pages(old_path, new_path)
move_folders(pages_dir, old_path, new_path)
end
def move_uploads(old_path, new_path)
return unless file_storage?
move_folders(uploads_dir, old_path, new_path)
end
def move_folders(directory, old_relative_path, new_relative_path)
old_path = File.join(directory, old_relative_path)
return unless File.directory?(old_path)
new_path = File.join(directory, new_relative_path)
FileUtils.mv(old_path, new_path)
end
def remove_cached_html_for_projects(project_ids)
update_column_in_batches(:projects, :description_html, nil) do |table, query|
query.where(table[:id].in(project_ids))
end
update_column_in_batches(:issues, :description_html, nil) do |table, query|
query.where(table[:project_id].in(project_ids))
end
update_column_in_batches(:merge_requests, :description_html, nil) do |table, query|
query.where(table[:target_project_id].in(project_ids))
end
update_column_in_batches(:notes, :note_html, nil) do |table, query|
query.where(table[:project_id].in(project_ids))
end
end
def file_storage?
CarrierWave::Uploader::Base.storage == CarrierWave::Storage::File
end
def uploads_dir
File.join(CarrierWave.root, "uploads")
end
def pages_dir
Settings.pages.path
end
end
end
end
end
module Gitlab
module Database
module RenameReservedPathsMigration
class RenameNamespaces < RenameBase
include Gitlab::ShellAdapter
def rename_namespaces(type:)
namespaces_for_paths(type: type).each do |namespace|
rename_namespace(namespace)
end
end
def namespaces_for_paths(type:)
namespaces = case type
when :wildcard
MigrationClasses::Namespace.where.not(parent_id: nil)
when :top_level
MigrationClasses::Namespace.where(parent_id: nil)
end
with_paths = MigrationClasses::Route.arel_table[:path].
matches_any(path_patterns)
namespaces.joins(:route).where(with_paths)
end
def rename_namespace(namespace)
old_full_path, new_full_path = rename_path_for_routable(namespace)
move_repositories(namespace, old_full_path, new_full_path)
move_uploads(old_full_path, new_full_path)
move_pages(old_full_path, new_full_path)
remove_cached_html_for_projects(projects_for_namespace(namespace).map(&:id))
end
def move_repositories(namespace, old_full_path, new_full_path)
repo_paths_for_namespace(namespace).each do |repository_storage_path|
# Ensure old directory exists before moving it
gitlab_shell.add_namespace(repository_storage_path, old_full_path)
unless gitlab_shell.mv_namespace(repository_storage_path, old_full_path, new_full_path)
message = "Exception moving path #{repository_storage_path} \
from #{old_full_path} to #{new_full_path}"
Rails.logger.error message
end
end
end
def repo_paths_for_namespace(namespace)
projects_for_namespace(namespace).distinct.select(:repository_storage).
map(&:repository_storage_path)
end
def projects_for_namespace(namespace)
namespace_ids = child_ids_for_parent(namespace, ids: [namespace.id])
namespace_or_children = MigrationClasses::Project.
arel_table[:namespace_id].
in(namespace_ids)
MigrationClasses::Project.where(namespace_or_children)
end
def child_ids_for_parent(namespace, ids: [])
namespace.children.each do |child|
ids << child.id
child_ids_for_parent(child, ids: ids) if child.children.any?
end
ids
end
end
end
end
end
module Gitlab
module Database
module RenameReservedPathsMigration
class RenameProjects < RenameBase
include Gitlab::ShellAdapter
def rename_projects
projects_for_paths.each do |project|
rename_project(project)
end
remove_cached_html_for_projects(projects_for_paths.map(&:id))
end
def rename_project(project)
old_full_path, new_full_path = rename_path_for_routable(project)
move_repository(project, old_full_path, new_full_path)
move_repository(project, "#{old_full_path}.wiki", "#{new_full_path}.wiki")
move_uploads(old_full_path, new_full_path)
move_pages(old_full_path, new_full_path)
end
def move_repository(project, old_path, new_path)
unless gitlab_shell.mv_repository(project.repository_storage_path,
old_path,
new_path)
Rails.logger.error "Error moving #{old_path} to #{new_path}"
end
end
def projects_for_paths
return @projects_for_paths if @projects_for_paths
with_paths = MigrationClasses::Route.arel_table[:path]
.matches_any(path_patterns)
@projects_for_paths = MigrationClasses::Project.joins(:route).where(with_paths)
end
end
end
end
end
module Gitlab
module Database
module RenameReservedPathsMigration
module V1
def self.included(kls)
kls.include(MigrationHelpers)
end
def rename_wildcard_paths(one_or_more_paths)
paths = Array(one_or_more_paths)
RenameNamespaces.new(paths, self).rename_namespaces(type: :wildcard)
RenameProjects.new(paths, self).rename_projects
end
def rename_root_paths(paths)
paths = Array(paths)
RenameNamespaces.new(paths, self).rename_namespaces(type: :top_level)
end
end
end
end
end
module Gitlab
module Database
module RenameReservedPathsMigration
module V1
module MigrationClasses
module Routable
def full_path
if route && route.path.present?
@full_path ||= route.path
else
update_route if persisted?
build_full_path
end
end
def build_full_path
if parent && path
parent.full_path + '/' + path
else
path
end
end
def update_route
prepare_route
route.save
end
def prepare_route
route || build_route(source: self)
route.path = build_full_path
route.name = build_full_name
@full_path = nil
@full_name = nil
end
def build_full_name
if parent && name
parent.human_name + ' / ' + name
else
name
end
end
def human_name
owner&.name
end
end
class User < ActiveRecord::Base
self.table_name = 'users'
end
class Namespace < ActiveRecord::Base
include MigrationClasses::Routable
self.table_name = 'namespaces'
belongs_to :parent,
class_name: "#{MigrationClasses.name}::Namespace"
has_one :route, as: :source
has_many :children,
class_name: "#{MigrationClasses.name}::Namespace",
foreign_key: :parent_id
belongs_to :owner,
class_name: "#{MigrationClasses.name}::User"
# Overridden to have the correct `source_type` for the `route` relation
def self.name
'Namespace'
end
end
class Route < ActiveRecord::Base
self.table_name = 'routes'
belongs_to :source, polymorphic: true
end
class Project < ActiveRecord::Base
include MigrationClasses::Routable
has_one :route, as: :source
self.table_name = 'projects'
def repository_storage_path
Gitlab.config.repositories.storages[repository_storage]['path']
end
# Overridden to have the correct `source_type` for the `route` relation
def self.name
'Project'
end
end
end
end
end
end
end
module Gitlab
module Database
module RenameReservedPathsMigration
module V1
class RenameBase
attr_reader :paths, :migration
delegate :update_column_in_batches,
:replace_sql,
to: :migration
def initialize(paths, migration)
@paths = paths
@migration = migration
end
def path_patterns
@path_patterns ||= paths.map { |path| "%#{path}" }
end
def rename_path_for_routable(routable)
old_path = routable.path
old_full_path = routable.full_path
# Only remove the last occurrence of the path name to get the parent namespace path
namespace_path = remove_last_occurrence(old_full_path, old_path)
new_path = rename_path(namespace_path, old_path)
new_full_path = join_routable_path(namespace_path, new_path)
# skips callbacks & validations
routable.class.where(id: routable).
update_all(path: new_path)
rename_routes(old_full_path, new_full_path)
[old_full_path, new_full_path]
end
def rename_routes(old_full_path, new_full_path)
replace_statement = replace_sql(Route.arel_table[:path],
old_full_path,
new_full_path)
update_column_in_batches(:routes, :path, replace_statement) do |table, query|
query.where(MigrationClasses::Route.arel_table[:path].matches("#{old_full_path}%"))
end
end
def rename_path(namespace_path, path_was)
counter = 0
path = "#{path_was}#{counter}"
while route_exists?(join_routable_path(namespace_path, path))
counter += 1
path = "#{path_was}#{counter}"
end
path
end
def remove_last_occurrence(string, pattern)
string.reverse.sub(pattern.reverse, "").reverse
end
def join_routable_path(namespace_path, top_level)
if namespace_path.present?
File.join(namespace_path, top_level)
else
top_level
end
end
def route_exists?(full_path)
MigrationClasses::Route.where(Route.arel_table[:path].matches(full_path)).any?
end
def move_pages(old_path, new_path)
move_folders(pages_dir, old_path, new_path)
end
def move_uploads(old_path, new_path)
return unless file_storage?
move_folders(uploads_dir, old_path, new_path)
end
def move_folders(directory, old_relative_path, new_relative_path)
old_path = File.join(directory, old_relative_path)
return unless File.directory?(old_path)
new_path = File.join(directory, new_relative_path)
FileUtils.mv(old_path, new_path)
end
def remove_cached_html_for_projects(project_ids)
update_column_in_batches(:projects, :description_html, nil) do |table, query|
query.where(table[:id].in(project_ids))
end
update_column_in_batches(:issues, :description_html, nil) do |table, query|
query.where(table[:project_id].in(project_ids))
end
update_column_in_batches(:merge_requests, :description_html, nil) do |table, query|
query.where(table[:target_project_id].in(project_ids))
end
update_column_in_batches(:notes, :note_html, nil) do |table, query|
query.where(table[:project_id].in(project_ids))
end
end
def file_storage?
CarrierWave::Uploader::Base.storage == CarrierWave::Storage::File
end
def uploads_dir
File.join(CarrierWave.root, "uploads")
end
def pages_dir
Settings.pages.path
end
end
end
end
end
end
module Gitlab
module Database
module RenameReservedPathsMigration
module V1
class RenameNamespaces < RenameBase
include Gitlab::ShellAdapter
def rename_namespaces(type:)
namespaces_for_paths(type: type).each do |namespace|
rename_namespace(namespace)
end
end
def namespaces_for_paths(type:)
namespaces = case type
when :wildcard
MigrationClasses::Namespace.where.not(parent_id: nil)
when :top_level
MigrationClasses::Namespace.where(parent_id: nil)
end
with_paths = MigrationClasses::Route.arel_table[:path].
matches_any(path_patterns)
namespaces.joins(:route).where(with_paths)
end
def rename_namespace(namespace)
old_full_path, new_full_path = rename_path_for_routable(namespace)
move_repositories(namespace, old_full_path, new_full_path)
move_uploads(old_full_path, new_full_path)
move_pages(old_full_path, new_full_path)
remove_cached_html_for_projects(projects_for_namespace(namespace).map(&:id))
end
def move_repositories(namespace, old_full_path, new_full_path)
repo_paths_for_namespace(namespace).each do |repository_storage_path|
# Ensure old directory exists before moving it
gitlab_shell.add_namespace(repository_storage_path, old_full_path)
unless gitlab_shell.mv_namespace(repository_storage_path, old_full_path, new_full_path)
message = "Exception moving path #{repository_storage_path} \
from #{old_full_path} to #{new_full_path}"
Rails.logger.error message
end
end
end
def repo_paths_for_namespace(namespace)
projects_for_namespace(namespace).distinct.select(:repository_storage).
map(&:repository_storage_path)
end
def projects_for_namespace(namespace)
namespace_ids = child_ids_for_parent(namespace, ids: [namespace.id])
namespace_or_children = MigrationClasses::Project.
arel_table[:namespace_id].
in(namespace_ids)
MigrationClasses::Project.where(namespace_or_children)
end
def child_ids_for_parent(namespace, ids: [])
namespace.children.each do |child|
ids << child.id
child_ids_for_parent(child, ids: ids) if child.children.any?
end
ids
end
end
end
end
end
end
module Gitlab
module Database
module RenameReservedPathsMigration
module V1
class RenameProjects < RenameBase
include Gitlab::ShellAdapter
def rename_projects
projects_for_paths.each do |project|
rename_project(project)
end
remove_cached_html_for_projects(projects_for_paths.map(&:id))
end
def rename_project(project)
old_full_path, new_full_path = rename_path_for_routable(project)
move_repository(project, old_full_path, new_full_path)
move_repository(project, "#{old_full_path}.wiki", "#{new_full_path}.wiki")
move_uploads(old_full_path, new_full_path)
move_pages(old_full_path, new_full_path)
end
def move_repository(project, old_path, new_path)
unless gitlab_shell.mv_repository(project.repository_storage_path,
old_path,
new_path)
Rails.logger.error "Error moving #{old_path} to #{new_path}"
end
end
def projects_for_paths
return @projects_for_paths if @projects_for_paths
with_paths = MigrationClasses::Route.arel_table[:path]
.matches_any(path_patterns)
@projects_for_paths = MigrationClasses::Project.joins(:route).where(with_paths)
end
end
end
end
end
end
require 'spec_helper' require 'spec_helper'
describe Gitlab::Database::RenameReservedPathsMigration::RenameBase do describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameBase do
let(:migration) { FakeRenameReservedPathMigration.new } let(:migration) { FakeRenameReservedPathMigrationV1.new }
let(:subject) { described_class.new(['the-path'], migration) } let(:subject) { described_class.new(['the-path'], migration) }
before do before do
...@@ -9,12 +9,12 @@ describe Gitlab::Database::RenameReservedPathsMigration::RenameBase do ...@@ -9,12 +9,12 @@ describe Gitlab::Database::RenameReservedPathsMigration::RenameBase do
end end
def migration_namespace(namespace) def migration_namespace(namespace)
Gitlab::Database::RenameReservedPathsMigration::MigrationClasses:: Gitlab::Database::RenameReservedPathsMigration::V1::MigrationClasses::
Namespace.find(namespace.id) Namespace.find(namespace.id)
end end
def migration_project(project) def migration_project(project)
Gitlab::Database::RenameReservedPathsMigration::MigrationClasses:: Gitlab::Database::RenameReservedPathsMigration::V1::MigrationClasses::
Project.find(project.id) Project.find(project.id)
end end
......
require 'spec_helper' require 'spec_helper'
describe Gitlab::Database::RenameReservedPathsMigration::RenameNamespaces do describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameNamespaces do
let(:migration) { FakeRenameReservedPathMigration.new } let(:migration) { FakeRenameReservedPathMigrationV1.new }
let(:subject) { described_class.new(['the-path'], migration) } let(:subject) { described_class.new(['the-path'], migration) }
before do before do
...@@ -9,7 +9,7 @@ describe Gitlab::Database::RenameReservedPathsMigration::RenameNamespaces do ...@@ -9,7 +9,7 @@ describe Gitlab::Database::RenameReservedPathsMigration::RenameNamespaces do
end end
def migration_namespace(namespace) def migration_namespace(namespace)
Gitlab::Database::RenameReservedPathsMigration::MigrationClasses:: Gitlab::Database::RenameReservedPathsMigration::V1::MigrationClasses::
Namespace.find(namespace.id) Namespace.find(namespace.id)
end end
......
require 'spec_helper' require 'spec_helper'
describe Gitlab::Database::RenameReservedPathsMigration::RenameProjects do describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameProjects do
let(:migration) { FakeRenameReservedPathMigration.new } let(:migration) { FakeRenameReservedPathMigrationV1.new }
let(:subject) { described_class.new(['the-path'], migration) } let(:subject) { described_class.new(['the-path'], migration) }
before do before do
......
require 'spec_helper' require 'spec_helper'
describe Gitlab::Database::RenameReservedPathsMigration do describe Gitlab::Database::RenameReservedPathsMigration::V1 do
let(:subject) { FakeRenameReservedPathMigration.new } let(:subject) { FakeRenameReservedPathMigrationV1.new }
before do before do
allow(subject).to receive(:say) allow(subject).to receive(:say)
...@@ -10,7 +10,7 @@ describe Gitlab::Database::RenameReservedPathsMigration do ...@@ -10,7 +10,7 @@ describe Gitlab::Database::RenameReservedPathsMigration do
describe '#rename_wildcard_paths' do describe '#rename_wildcard_paths' do
it 'should rename namespaces' do it 'should rename namespaces' do
rename_namespaces = double rename_namespaces = double
expect(Gitlab::Database::RenameReservedPathsMigration::RenameNamespaces). expect(described_class::RenameNamespaces).
to receive(:new).with(['first-path', 'second-path'], subject). to receive(:new).with(['first-path', 'second-path'], subject).
and_return(rename_namespaces) and_return(rename_namespaces)
expect(rename_namespaces).to receive(:rename_namespaces). expect(rename_namespaces).to receive(:rename_namespaces).
...@@ -21,7 +21,7 @@ describe Gitlab::Database::RenameReservedPathsMigration do ...@@ -21,7 +21,7 @@ describe Gitlab::Database::RenameReservedPathsMigration do
it 'should rename projects' do it 'should rename projects' do
rename_projects = double rename_projects = double
expect(Gitlab::Database::RenameReservedPathsMigration::RenameProjects). expect(described_class::RenameProjects).
to receive(:new).with(['the-path'], subject). to receive(:new).with(['the-path'], subject).
and_return(rename_projects) and_return(rename_projects)
...@@ -34,7 +34,7 @@ describe Gitlab::Database::RenameReservedPathsMigration do ...@@ -34,7 +34,7 @@ describe Gitlab::Database::RenameReservedPathsMigration do
describe '#rename_root_paths' do describe '#rename_root_paths' do
it 'should rename namespaces' do it 'should rename namespaces' do
rename_namespaces = double rename_namespaces = double
expect(Gitlab::Database::RenameReservedPathsMigration::RenameNamespaces). expect(described_class::RenameNamespaces).
to receive(:new).with(['the-path'], subject). to receive(:new).with(['the-path'], subject).
and_return(rename_namespaces) and_return(rename_namespaces)
expect(rename_namespaces).to receive(:rename_namespaces). expect(rename_namespaces).to receive(:rename_namespaces).
......
class FakeRenameReservedPathMigration < ActiveRecord::Migration class FakeRenameReservedPathMigrationV1 < ActiveRecord::Migration
include Gitlab::Database::RenameReservedPathsMigration include Gitlab::Database::RenameReservedPathsMigration::V1
end end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment