Merge branch...

Merge branch '282465-add-service-for-migrating-individual-project-with-pages-to-zip-deployment' into 'master'

Add service for zipping pages directory

See merge request gitlab-org/gitlab!49028

app/services/pages/zip_directory_service.rb

+# frozen_string_literal: true
+
+module Pages
+  class ZipDirectoryService
+    InvalidArchiveError = Class.new(RuntimeError)
+    InvalidEntryError = Class.new(RuntimeError)
+
+    PUBLIC_DIR = 'public'
+
+    def initialize(input_dir)
+      @input_dir = File.realpath(input_dir)
+      @output_file = File.join(@input_dir, "@migrated.zip") # '@' to avoid any name collision with groups or projects
+    end
+
+    def execute
+      FileUtils.rm_f(@output_file)
+
+      count = 0
+      ::Zip::File.open(@output_file, ::Zip::File::CREATE) do |zipfile|
+        write_entry(zipfile, PUBLIC_DIR)
+        count = zipfile.entries.count
+      end
+
+      [@output_file, count]
+    end
+
+    private
+
+    def write_entry(zipfile, zipfile_path)
+      disk_file_path = File.join(@input_dir, zipfile_path)
+
+      unless valid_path?(disk_file_path)
+        # archive without public directory is completelly unusable
+        raise InvalidArchiveError if zipfile_path == PUBLIC_DIR
+
+        # archive with invalid entry will just have this entry missing
+        raise InvalidEntryError
+      end
+
+      case File.lstat(disk_file_path).ftype
+      when 'directory'
+        recursively_zip_directory(zipfile, disk_file_path, zipfile_path)
+      when 'file', 'link'
+        zipfile.add(zipfile_path, disk_file_path)
+      else
+        raise InvalidEntryError
+      end
+    rescue InvalidEntryError => e
+      Gitlab::ErrorTracking.track_exception(e, input_dir: @input_dir, disk_file_path: disk_file_path)
+    end
+
+    def recursively_zip_directory(zipfile, disk_file_path, zipfile_path)
+      zipfile.mkdir(zipfile_path)
+
+      entries = Dir.entries(disk_file_path) - %w[. ..]
+      entries = entries.map { |entry| File.join(zipfile_path, entry) }
+
+      write_entries(zipfile, entries)
+    end
+
+    def write_entries(zipfile, entries)
+      entries.each do |zipfile_path|
+        write_entry(zipfile, zipfile_path)
+      end
+    end
+
+    # that should never happen, but we want to be safer
+    # in theory without this we would allow to use symlinks
+    # to pack any directory on disk
+    # it isn't possible because SafeZip doesn't extract such archives
+    def valid_path?(disk_file_path)
+      realpath = File.realpath(disk_file_path)
+
+      realpath == File.join(@input_dir, PUBLIC_DIR) ||
+        realpath.start_with?(File.join(@input_dir, PUBLIC_DIR + "/"))
+    # happens if target of symlink isn't there
+    rescue => e
+      Gitlab::ErrorTracking.track_exception(e, input_dir: @input_dir, disk_file_path: disk_file_path)
+
+      false
+    end
+  end
+end

spec/services/pages/zip_directory_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Pages::ZipDirectoryService do
+  around do |example|
+    Dir.mktmpdir do |dir|
+      @work_dir = dir
+      example.run
+    end
+  end
+
+  let(:result) do
+    described_class.new(@work_dir).execute
+  end
+
+  let(:archive) { result.first }
+  let(:entries_count) { result.second }
+
+  it 'raises error if there is no public directory' do
+    expect { archive }.to raise_error(described_class::InvalidArchiveError)
+  end
+
+  it 'raises error if public directory is a symlink' do
+    create_dir('target')
+    create_file('./target/index.html', 'hello')
+    create_link("public", "./target")
+
+    expect { archive }.to raise_error(described_class::InvalidArchiveError)
+  end
+
+  context 'when there is a public directory' do
+    before do
+      create_dir('public')
+    end
+
+    it 'creates the file next the public directory' do
+      expect(archive).to eq(File.join(@work_dir, "@migrated.zip"))
+    end
+
+    it 'includes public directory' do
+      with_zip_file do |zip_file|
+        entry = zip_file.get_entry("public/")
+        expect(entry.ftype).to eq(:directory)
+      end
+    end
+
+    it 'returns number of entries' do
+      create_file("public/index.html", "hello")
+      create_link("public/link.html", "./index.html")
+      expect(entries_count).to eq(3) # + 'public' directory
+    end
+
+    it 'removes the old file if it exists' do
+      # simulate the old run
+      described_class.new(@work_dir).execute
+
+      with_zip_file do |zip_file|
+        expect(zip_file.entries.count).to eq(1)
+      end
+    end
+
+    it 'ignores other top level files and directories' do
+      create_file("top_level.html", "hello")
+      create_dir("public2")
+
+      with_zip_file do |zip_file|
+        expect { zip_file.get_entry("top_level.html") }.to raise_error(Errno::ENOENT)
+        expect { zip_file.get_entry("public2/") }.to raise_error(Errno::ENOENT)
+      end
+    end
+
+    it 'includes index.html file' do
+      create_file("public/index.html", "Hello!")
+
+      with_zip_file do |zip_file|
+        entry = zip_file.get_entry("public/index.html")
+        expect(zip_file.read(entry)).to eq("Hello!")
+      end
+    end
+
+    it 'includes hidden file' do
+      create_file("public/.hidden.html", "Hello!")
+
+      with_zip_file do |zip_file|
+        entry = zip_file.get_entry("public/.hidden.html")
+        expect(zip_file.read(entry)).to eq("Hello!")
+      end
+    end
+
+    it 'includes nested directories and files' do
+      create_dir("public/nested")
+      create_dir("public/nested/nested2")
+      create_file("public/nested/nested2/nested.html", "Hello nested")
+
+      with_zip_file do |zip_file|
+        entry = zip_file.get_entry("public/nested")
+        expect(entry.ftype).to eq(:directory)
+
+        entry = zip_file.get_entry("public/nested/nested2")
+        expect(entry.ftype).to eq(:directory)
+
+        entry = zip_file.get_entry("public/nested/nested2/nested.html")
+        expect(zip_file.read(entry)).to eq("Hello nested")
+      end
+    end
+
+    it 'adds a valid symlink' do
+      create_file("public/target.html", "hello")
+      create_link("public/link.html", "./target.html")
+
+      with_zip_file do |zip_file|
+        entry = zip_file.get_entry("public/link.html")
+        expect(entry.ftype).to eq(:symlink)
+        expect(zip_file.read(entry)).to eq("./target.html")
+      end
+    end
+
+    it 'ignores the symlink pointing outside of public directory' do
+      create_file("target.html", "hello")
+      create_link("public/link.html", "../target.html")
+
+      with_zip_file do |zip_file|
+        expect { zip_file.get_entry("public/link.html") }.to raise_error(Errno::ENOENT)
+      end
+    end
+
+    it 'ignores the symlink if target is absent' do
+      create_link("public/link.html", "./target.html")
+
+      with_zip_file do |zip_file|
+        expect { zip_file.get_entry("public/link.html") }.to raise_error(Errno::ENOENT)
+      end
+    end
+
+    it 'ignores symlink if is absolute and points to outside of directory' do
+      target = File.join(@work_dir, "target")
+      FileUtils.touch(target)
+
+      create_link("public/link.html", target)
+
+      with_zip_file do |zip_file|
+        expect { zip_file.get_entry("public/link.html") }.to raise_error(Errno::ENOENT)
+      end
+    end
+
+    it "includes raw symlink if it's target is a valid directory" do
+      create_dir("public/target")
+      create_file("public/target/index.html", "hello")
+      create_link("public/link", "./target")
+
+      with_zip_file do |zip_file|
+        expect(zip_file.entries.count).to eq(4) # /public and 3 created above
+
+        entry = zip_file.get_entry("public/link")
+        expect(entry.ftype).to eq(:symlink)
+        expect(zip_file.read(entry)).to eq("./target")
+      end
+    end
+  end
+
+  context "validating fixtures pages archives" do
+    using RSpec::Parameterized::TableSyntax
+
+    where(:fixture_path) do
+      ["spec/fixtures/pages.zip", "spec/fixtures/pages_non_writeable.zip"]
+    end
+
+    with_them do
+      let(:full_fixture_path) { Rails.root.join(fixture_path) }
+
+      it 'a created archives contains exactly the same entries' do
+        SafeZip::Extract.new(full_fixture_path).extract(directories: ['public'], to: @work_dir)
+
+        with_zip_file do |created_archive|
+          Zip::File.open(full_fixture_path) do |original_archive|
+            original_archive.entries do |original_entry|
+              created_entry = created_archive.get_entry(original_entry.name)
+
+              expect(created_entry.name).to eq(original_entry.name)
+              expect(created_entry.ftype).to eq(original_entry.ftype)
+              expect(created_archive.read(created_entry)).to eq(original_archive.read(original_entry))
+            end
+          end
+        end
+      end
+    end
+  end
+
+  def create_file(name, content)
+    File.open(File.join(@work_dir, name), "w") do |f|
+      f.write(content)
+    end
+  end
+
+  def create_dir(dir)
+    Dir.mkdir(File.join(@work_dir, dir))
+  end
+
+  def create_link(new_name, target)
+    File.symlink(target, File.join(@work_dir, new_name))
+  end
+
+  def with_zip_file
+    Zip::File.open(archive) do |zip_file|
+      yield zip_file
+    end
+  end
+end