Lzma Cleanup & replacement with newer xzutils, Add pixz support (#5726)

* replace lzma with current xz-utils

* add po4a build dep to xzutils

* Add xzutils, pixz

* is_fake doesn't need source_url

* add pixz tpxz format support to crew

* Add xzutils to pixz deps

* Add pixz if not installed.

* change pixz logic

* pixz changes

* xzutils is a runtime dep for pixz

* grammar fix

* remove pixz warning

bin/crew

@@ -234,9 +234,13 @@ def generate_compatible
   Dir[CREW_PACKAGES_PATH + '*.rb'].each do |filename|
     pkgName = File.basename filename, '.rb'
     set_package pkgName, filename
+    puts "Checking #{pkgName} for compatibility.".orange if @opt_verbose
     if @pkg.compatibility.include? 'all' or @pkg.compatibility.include? ARCH
       #add to compatible packages
+      puts "Adding #{pkgName} to compatible packages.".lightgreen if @opt_verbose
       @device[:compatible_packages].push(name: @pkg.name)
+    else
+      puts "#{pkgName} is not a compatible package.".lightred if @opt_verbose
     end
   end
   File.open(CREW_CONFIG_PATH + 'device.json', 'w') do |file|
@@ -488,7 +492,7 @@ def update
   #update compatible packages
   puts 'Generating compatible packages...'
   generate_compatible
-
+  puts 'Generating compatible packages done.'.orange if @opt_verbose
   #check for outdated installed packages
   puts 'Checking for package updates...'
 
@@ -595,7 +599,7 @@ def download
 
     case File.basename(filename)
     # Sources that download with curl
-    when /\.zip$/i, /\.(tar(\.(gz|bz2|xz|lz))?|tgz|tbz|txz)$/i, /\.deb$/i, /\.AppImage$/i
+    when /\.zip$/i, /\.(tar(\.(gz|bz2|xz|lz))?|tgz|tbz|tpxz|txz)$/i, /\.deb$/i, /\.AppImage$/i
       # Recall file from cache if requested
       if CREW_CACHE_ENABLED
         puts "Looking for archive in cache".orange if @opt_verbose
@@ -725,6 +729,12 @@ def unpack (meta)
       Dir.chdir @extract_dir do
         system "../#{meta[:filename]} --appimage-extract", exception: true
       end
+    when /\.tpxz$/i
+      unless File.exist?("#{CREW_PREFIX}/bin/pixz")
+        abort 'Pixz is needed for this install. Please install it with \'crew install pixz\''.lightred
+      end
+      puts "Unpacking 'tpxz' archive using 'tar', this may take a while..."
+      system "tar -Ipixz -x#{@verbose}f #{meta[:filename]} -C #{@extract_dir}", exception: true
     end
     if meta[:source] == true
       # Check the number of directories in the archive
@@ -809,6 +819,12 @@ def prepare_package (destdir)
     # than install-info.
     # https://www.debian.org/doc/debian-policy/ch-docs.html#info-documents
     FileUtils.rm "#{CREW_DEST_PREFIX}/share/info/dir" if File.exist?("#{CREW_DEST_PREFIX}/share/info/dir")
+    
+    # Remove all perl module files which will conflict
+    if @pkg.name.include? 'perl_'
+      puts "Removing .packlist and perllocal.pod files to avoid conflicts with other perl packages.".orange
+      system "find #{CREW_DEST_DIR} -type f \\( -name '.packlist' -o -name perllocal.pod \\) -delete" 
+    end
 
     # compress manual files
     compress_doc "#{CREW_DEST_PREFIX}/man"
@@ -871,7 +887,7 @@ def strip_dir (dir)
 
       # Strip binaries but not compressed archives
       puts "Stripping binaries..."
-      extensions = [ 'bz2', 'gz', 'lha', 'lz', 'lzh', 'rar', 'tar', 'tbz', 'tgz', 'txz', 'xz', 'Z', 'zip' ]
+      extensions = [ 'bz2', 'gz', 'lha', 'lz', 'lzh', 'rar', 'tar', 'tbz', 'tgz', 'tpxz', 'txz', 'xz', 'Z', 'zip' ]
       inames = extensions.join(' ! -iname *\.')
       strip_find_files "find . -type f ! -iname *\.#{inames} -perm /111 -print | sed -e '/lib.*\.a$/d' -e '/lib.*\.so/d'"
     end
@@ -1122,9 +1138,18 @@ def build_package (pwd)
 end
 
 def archive_package (pwd)
-  pkg_name = "#{@pkg.name}-#{@pkg.version}-chromeos-#{@device[:architecture]}.tar.xz"
-  Dir.chdir CREW_DEST_DIR do
-    system "tar c#{@verbose}Jf #{pwd}/#{pkg_name} *"
+  unless ENV['CREW_USE_PIXZ'] == '1'
+    pkg_name = "#{@pkg.name}-#{@pkg.version}-chromeos-#{@device[:architecture]}.tar.xz"
+    Dir.chdir CREW_DEST_DIR do
+      system "tar c#{@verbose}Jf #{pwd}/#{pkg_name} *"
+    end
+  else
+    puts "Using pixz to compress archive."
+    pkg_name = "#{@pkg.name}-#{@pkg.version}-chromeos-#{@device[:architecture]}.tpxz"
+    Dir.chdir CREW_DEST_DIR do
+      # Use smaller blocks with "-f0.25" to make random access faster.
+      system "tar c#{@verbose} * | pixz -f0.25 -9 > #{pwd}/#{pkg_name}"
+    end
   end
   system "sha256sum #{pwd}/#{pkg_name} > #{pwd}/#{pkg_name}.sha256"
 end

lib/const.rb

 # Defines common constants used in different parts of crew
 
-CREW_VERSION = '1.9.2'
+CREW_VERSION = '1.9.3'
 
 ARCH_ACTUAL = `uname -m`.strip
 # This helps with virtualized builds on aarch64 machines

packages/balena_etcher.rb

@@ -19,7 +19,7 @@ class Balena_etcher < Package
   depends_on 'gtk2'
   depends_on 'freetype'
   depends_on 'cairo'
-  depends_on 'lzma'
+  depends_on 'xzutils'
   depends_on 'libnotify'
   depends_on 'nspr'
   depends_on 'libgconf'

packages/imagemagick6.rb

@@ -27,7 +27,7 @@ class Imagemagick6 < Package
   depends_on 'graphviz'
   depends_on 'jbigkit'
   depends_on 'jemalloc'
-  depends_on 'lzma'
+  depends_on 'xzutils'
   depends_on 'libheif'
   depends_on 'librsvg'
   depends_on 'libwebp'

packages/imagemagick7.rb

@@ -30,7 +30,7 @@ class Imagemagick7 < Package
   depends_on 'graphviz'
   depends_on 'jbigkit'
   depends_on 'jemalloc'
-  depends_on 'lzma'
+  depends_on 'xzutils'
   depends_on 'libheif'
   depends_on 'librsvg'
   depends_on 'libwebp'

packages/libsolv.rb

@@ -22,7 +22,7 @@ class Libsolv < Package
      x86_64: '1c9062a1b1cc87345941eedb01d7aacd9f942c7924d81db8c52d77cde19a32ed',
   })
 
-  depends_on 'lzma'
+  depends_on 'xzutils'
   depends_on 'swig'
   depends_on 'zstd'
 

packages/lzma.rb

@@ -3,31 +3,12 @@ require 'package'
 class Lzma < Package
   description 'LZMA Utils are legacy data compression software with high compression ratio.'
   homepage 'https://tukaani.org/lzma/'
-  version '4.32.7-1'
+  version '4.98'
   license 'public-domain'
   compatibility 'all'
-  source_url 'https://tukaani.org/lzma/lzma-4.32.7.tar.bz2'
-  source_sha256 '618e54513993b3a153fa1c150fccdf25788c72b36e84ab4db71911083531cf6a'
 
-  binary_url ({
-    aarch64: 'https://gitlab.com/api/v4/projects/26210301/packages/generic/lzma/4.32.7-1_armv7l/lzma-4.32.7-1-chromeos-armv7l.tar.xz',
-     armv7l: 'https://gitlab.com/api/v4/projects/26210301/packages/generic/lzma/4.32.7-1_armv7l/lzma-4.32.7-1-chromeos-armv7l.tar.xz',
-       i686: 'https://gitlab.com/api/v4/projects/26210301/packages/generic/lzma/4.32.7-1_i686/lzma-4.32.7-1-chromeos-i686.tar.xz',
-     x86_64: 'https://gitlab.com/api/v4/projects/26210301/packages/generic/lzma/4.32.7-1_x86_64/lzma-4.32.7-1-chromeos-x86_64.tar.xz',
-  })
-  binary_sha256 ({
-    aarch64: 'cb62ba6e36b24cf50d0490e3ed9c35c35a76ca394f98e6c572bf3e49ab48f205',
-     armv7l: 'cb62ba6e36b24cf50d0490e3ed9c35c35a76ca394f98e6c572bf3e49ab48f205',
-       i686: '73b723ede8d112ff824f7ab62bb5108f711d0a0cad6daf9472d2303b634e1516',
-     x86_64: 'c8a08a86689baf36796baa5c86c1507b10b0b5cdaca733737be6c59782b07b1a',
-  })
+  is_fake
 
-  def self.build
-    system "./configure --prefix=#{CREW_PREFIX} --libdir=#{CREW_LIB_PREFIX}"
-    system "make"
-  end
+  depends_on 'xzutils'
 
-  def self.install
-    system "make", "DESTDIR=#{CREW_DEST_DIR}", "install"
-  end
 end

packages/pixz.rb

+# Adapted from Arch Linux pixz PKGBUILD at:
+# https://github.com/archlinux/svntogit-community/raw/packages/pixz/trunk/PKGBUILD
+
+require 'package'
+
+class Pixz < Package
+  description 'Parallel, indexed xz compressor'
+  homepage 'https://github.com/vasi/pixz'
+  version '1.0.7-0829'
+  license 'BSD'
+  compatibility 'all'
+  source_url 'https://github.com/vasi/pixz.git'
+  git_hashtag '0829c7315c804a4e40abd63a9d624194dc1e4f0a'
+
+  binary_url({
+    aarch64: 'https://gitlab.com/api/v4/projects/26210301/packages/generic/pixz/1.0.7-0829_armv7l/pixz-1.0.7-0829-chromeos-armv7l.tar.xz',
+     armv7l: 'https://gitlab.com/api/v4/projects/26210301/packages/generic/pixz/1.0.7-0829_armv7l/pixz-1.0.7-0829-chromeos-armv7l.tar.xz',
+       i686: 'https://gitlab.com/api/v4/projects/26210301/packages/generic/pixz/1.0.7-0829_i686/pixz-1.0.7-0829-chromeos-i686.tar.xz',
+     x86_64: 'https://gitlab.com/api/v4/projects/26210301/packages/generic/pixz/1.0.7-0829_x86_64/pixz-1.0.7-0829-chromeos-x86_64.tar.xz'
+  })
+  binary_sha256({
+    aarch64: 'b3775ba946ecd95e87b68c8bd2d0c93c78027677eb4de15fc9df5d2e4d80f7c2',
+     armv7l: 'b3775ba946ecd95e87b68c8bd2d0c93c78027677eb4de15fc9df5d2e4d80f7c2',
+       i686: '8fff1c81a1b0f693a5e6fa592fba32410875e53e754c8b70e9bf5f14f8654627',
+     x86_64: '2112ae37128533e88bd492d0aae688028fe7ea8b8149e50c16866aa69f90034e'
+  })
+
+  depends_on 'libarchive'
+  depends_on 'asciidoc' => ':build'
+  depends_on 'xzutils'
+
+  def self.build
+    system '[ -x configure ] || NOCONFIGURE=1 ./autogen.sh'
+    system "#{CREW_ENV_OPTIONS} \
+      manpage=true \
+      ./configure #{CREW_OPTIONS}"
+    system 'make'
+  end
+
+  def self.install
+    system "make DESTDIR=#{CREW_DEST_DIR} install"
+  end
+end

packages/xorg_server.rb

@@ -39,7 +39,7 @@ class Xorg_server < Package
   depends_on 'font_util'
   depends_on 'libbsd'
   depends_on 'dbus'
-  depends_on 'lzma' => :build
+  depends_on 'xzutils' => :build
   depends_on 'xkbcomp'
   depends_on 'glproto'
   depends_on 'xcb_util_renderutil' => :build

packages/xwayland.rb

@@ -36,7 +36,7 @@ class Xwayland < Package
   depends_on 'libxkbfile'
   depends_on 'libxtrans'
   depends_on 'libxdmcp'
-  depends_on 'lzma' => :build
+  depends_on 'xzutils' => :build
   depends_on 'mesa'
   depends_on 'pixman'
   depends_on 'wayland'

packages/xzutils.rb

@@ -3,28 +3,33 @@ require 'package'
 class Xzutils < Package
   description 'XZ Utils is free general-purpose data compression software with a high compression ratio.'
   homepage 'http://tukaani.org/xz/'
-  version '5.2.5'
+  version '5.2.5-e7da'
   license 'public-domain, LGPL-2.1+ and GPL-2+'
   compatibility 'all'
-  source_url 'https://tukaani.org/xz/xz-5.2.5.tar.gz'
-  source_sha256 'f6f4910fd033078738bd82bfba4f49219d03b17eb0794eb91efbae419f4aba10'
+  source_url 'https://github.com/xz-mirror/xz.git'
+  git_hashtag 'e7da44d5151e21f153925781ad29334ae0786101'
 
-  binary_url ({
-    aarch64: 'https://gitlab.com/api/v4/projects/26210301/packages/generic/xzutils/5.2.5_armv7l/xzutils-5.2.5-chromeos-armv7l.tar.xz',
-     armv7l: 'https://gitlab.com/api/v4/projects/26210301/packages/generic/xzutils/5.2.5_armv7l/xzutils-5.2.5-chromeos-armv7l.tar.xz',
-       i686: 'https://gitlab.com/api/v4/projects/26210301/packages/generic/xzutils/5.2.5_i686/xzutils-5.2.5-chromeos-i686.tar.xz',
-     x86_64: 'https://gitlab.com/api/v4/projects/26210301/packages/generic/xzutils/5.2.5_x86_64/xzutils-5.2.5-chromeos-x86_64.tar.xz',
+  binary_url({
+    aarch64: 'https://gitlab.com/api/v4/projects/26210301/packages/generic/xzutils/5.2.5-e7da_armv7l/xzutils-5.2.5-e7da-chromeos-armv7l.tar.xz',
+     armv7l: 'https://gitlab.com/api/v4/projects/26210301/packages/generic/xzutils/5.2.5-e7da_armv7l/xzutils-5.2.5-e7da-chromeos-armv7l.tar.xz',
+       i686: 'https://gitlab.com/api/v4/projects/26210301/packages/generic/xzutils/5.2.5-e7da_i686/xzutils-5.2.5-e7da-chromeos-i686.tar.xz',
+     x86_64: 'https://gitlab.com/api/v4/projects/26210301/packages/generic/xzutils/5.2.5-e7da_x86_64/xzutils-5.2.5-e7da-chromeos-x86_64.tar.xz'
   })
-  binary_sha256 ({
-    aarch64: 'bbb38dfbc5f7845dde5d4e6c1aa46be40d4d94f5db26b0aa20f1603a14c6e61d',
-     armv7l: 'bbb38dfbc5f7845dde5d4e6c1aa46be40d4d94f5db26b0aa20f1603a14c6e61d',
-       i686: '34db46c8dc58db4c212ba83e36eba17c8852eab14964a6a1d7ab75aaf23be91c',
-     x86_64: '05f0e86514ed8fefa240e0fc2d40cb2b95414d628f670b19a230a3867e992a24',
+  binary_sha256({
+    aarch64: '3b87b8150061f2f894f9ee69fceb6ab0b010fa9bdf63f79280c683eae28d588f',
+     armv7l: '3b87b8150061f2f894f9ee69fceb6ab0b010fa9bdf63f79280c683eae28d588f',
+       i686: 'a7da9276492788d722a0c853a81d8a5aff164eb16f49882514a1870d3f1397c7',
+     x86_64: 'a3080d777b7f220b35e9c368925e3099796de08c292239c2c5ca117b9beeb859'
   })
 
   def self.build
-    system './configure', "--prefix=#{CREW_PREFIX}", "--libdir=#{CREW_LIB_PREFIX}",
-           '--disable-docs', '--enable-shared', '--disable-static', '--with-pic'
+    system '[ -x configure ] || NOCONFIGURE=1 ./autogen.sh --no-po4a'
+    system "#{CREW_ENV_OPTIONS} \
+      ./configure \
+      #{CREW_OPTIONS} \
+      --disable-docs \
+      --enable-shared \
+      --with-pic"
     system 'make'
   end