Merge pull request #1167 from jam7/update-tools

Add tools to create pre-built binary and upload

tools/Makefile

+CREW_PREFIX = /usr/local
+CREW_LIB_PATH = $(CREW_PREFIX)/lib/crew
+
+ALLTARGETS = $(sort $(basename $(notdir $(wildcard $(CREW_LIB_PATH)/packages/*))))
+MINTARGETS = compressdoc libsigsegv patch diffutils ncursesw ncurses \
+  readline pkgconfig make zlibpkg xzutils lzip m4 flex bison ed perl \
+  libxml2 gettext texinfo bc openssl libssh2 gdbm sqlite curl expat \
+  bz2 python27 git ruby less python3 libpipeline groff mandb fuse \
+  libffi glib sshfs unzip
+MIN2TARGETS = erlang filecmd go_bootstrap go gawk lz4 acl attr libarchive \
+  cmake autoconf autoconf_archive automake libtool emacs vim neovim \
+  nano libpng libxslt pcre php5 php7 screen readline7 tcl expect
+MIN3TARGETS = pixman cairo a2png gperf a2ps icu4c harfbuzz freetype \
+  fontconfig pango optipng netcat ncftp ncdu patchelf openssh patchutils \
+  protobuf cpio ctags scons libpcap tcpdump tcpstat lha unrar zsh \
+  libjpeg libtiff imagemagick imagemagick6
+
+TOOLCHAINS = binutils cloog gcc glibc219 glibc223 gmp isl linuxheaders mpc mpfr
+FAKES = glibc buildessential parallel wayland xorg_proto
+LICENSEPROHIBIT = jdk8
+# PRECOMPILED = elixir sejda_console platformsh
+# SCRIPTS = gifgen yarn nodebrew
+# MODIFYENV = nvm
+CRASH = get_iplayer
+BINTARGETS =  lldb $(TOOLCHAINS) $(FAKES) $(LICENSEPROHIBIT) $(PRECOMPILED) $(SCRIPTS) $(MODIFYENV) $(CRASH)
+
+#scripts
+GENPKGNAME = ./genpkgname.sh
+BUILD = ./build.sh
+
+all: $(ALLTARGETS)
+min: $(MINTARGETS)
+min2: $(MIN2TARGETS)
+min3: $(MIN3TARGETS)
+
+define make-target
+  $1: $(shell $(GENPKGNAME) $1)
+  $(shell $(GENPKGNAME) $1):
+	$(BUILD) $1
+#  all-targets:: $(shell $(GENPKGNAME) $1)
+endef
+
+define make-dummy-target
+  $1: $(shell $(GENPKGNAME) $1)
+  $(shell $(GENPKGNAME) $1): $(CREW_LIB_PATH)/packages/$1.rb
+#	@echo nothing to do for binary or fake package $1
+endef
+
+$(foreach pkg, $(filter-out $(BINTARGETS),$(ALLTARGETS)), $(eval $(call make-target,$(pkg))))
+
+$(foreach pkg, $(BINTARGETS), $(eval $(call make-dummy-target,$(pkg))))

tools/README.md

+### list of all tools
+
+Tools to build pre-build binary easy.
+
+ - Makefile
+ - build.sh
+ - genpkgname.sh
+
+Tools to upload and update URL in package files.
+
+ - chkfrog.sh
+ - upfrog.sh
+ - upload_github_release.sh
+
+Tools to check URL or SHA stuff from packages files.
+
+ - create_sha_list.sh
+ - create_url_list.sh
+
+Obsolete tools to create package.  Now, we use `crew build`.
+
+ - create_package.sh
+
+### Usage of Makefile
+
+Make working directory on your Chromebook and copy files there.
+Then, performs `make` there.
+
+```
+$ mkdir /usr/local/work
+$ cd /usr/local/work
+$ cp your-path-for-original-tool-directory/{Makefile,build.sh,genpkgname.sh} .
+$ make -k
+```
+
+### Usage of upfrog.sh
+
+First, compile jfrog CLI from `https://www.jfrog.com/getcli/`.  Then, set it up.
+After that, copy compiled pre-built binary to `release/$ARCH`.
+
+```
+$ ./tools/upfrog.sh pango-1.40.9
+[Info] Verifying repository chromebrew exists...
+[Info] Verifying package pango exists...
+[Info] Verifying version 1.40.9 exists...
+[Info] Creating version...
+[Info] Created version 1.40.9.
+[Info] Collecting files for upload:. Done.
+[Info] [Thread 2] Uploading artifact: release/x86_64/pango-1.40.9-chromeos-x86_64.tar.xz
+[Info] [Thread 0] Uploading artifact: release/armv7l/pango-1.40.9-chromeos-armv7l.tar.xz
+[Info] [Thread 1] Uploading artifact: release/i686/pango-1.40.9-chromeos-i686.tar.xz
+[Info] Uploaded 3 artifacts.
+[Info] Publishing version...
+[Info] Published version 1.40.9, details:
+{
+  "files": 3
+}
+```
+
+### Usage of chkfrog.sh
+
+Copy compiled pre-built binary's SHA256 files to `release/$ARCH`.
+
+```
+$ ./tools/chkfrog.sh pango
+pango 1.40.9
+https://dl.bintray.com/chromebrew/chromebrew/pango-1.40.9-chromeos-x86_64.tar.xz : c90a74dcba01ac1731aca5879b5b54dc9e1c49c3f61ec5f2861384db2bafa4eb
+https://dl.bintray.com/chromebrew/chromebrew/pango-1.40.9-chromeos-i686.tar.xz : c24290af6d40c9fe1b9797942941e5a181533574188678b8f1f6f6a9ea319ba4
+https://dl.bintray.com/chromebrew/chromebrew/pango-1.40.9-chromeos-armv7l.tar.xz : 60fb01a75558724abfbd879cef6877fca2ca17d802abeca7c487bb9d8b75a08c
+```

tools/build.sh

+#!/bin/bash
+case $1 in
+-k) keep="keep"; shift;;
+*)  keep="";;
+esac
+set -o pipefail
+# yes | crew install buildessential $keep
+arch=`uname -m`
+for i in "$@"; do
+    version=`grep '\<version ['"'"'"]' /usr/local/lib/crew/packages/$i.rb | head -1 | sed -e 's/^.*version *['"'"'"]//' -e 's/['"'"'"].*$//'`
+    yes | crew build $i $keep 2>&1 | tee "$i-$version-chromeos-$arch.log"
+    case .$? in
+    .141) ;;        # ignore SIGPIPE
+    *)    exit 1;;  # abort at errors
+    esac
+done

tools/chkfrog.sh

+#!/bin/bash
+
+# A script to check jfrog (bintray) for binaries existence and update package file
+#
+# Usage
+#
+#   chkfrog package...              Check packages
+#
+# Directory structure
+#
+#   Unfortunately, we need to have sha256 files locally to update URL correctly.
+#
+#   release/armv7l/*.tar.xz.sha256  SHA256 files for armv7l
+#   release/i686/*.tar.xz.sha256    SHA256 files for i686
+#   release/x86_64/*.tar.xz.sha256  SHA256 files for x86_64
+
+URL="https://dl.bintray.com/chromebrew/chromebrew"
+
+case $# in
+0) set `basename -s .rb packages/*`
+esac
+
+function update_url () {
+    if sed -e '/binary_url.*({/,/})/p; d' packages/$1.rb | grep -q $2:; then
+#       sed -e "/binary_url.*({/,/})/s,$2:.*,$2: '$3'," -i packages/$1.rb
+        sed -e "/$2:.*\(http\|https\|ftp\):/c\ \ \ \ $(printf %7s $2): '$3'," -i packages/$1.rb
+    else
+        sed -e "/binary_url.*({/a\ \ \ \ $(printf %7s $2): '$3'," -i packages/$1.rb
+    fi
+}
+function update_sha256 () {
+    if sed -e '/binary_sha256.*({/,/})/p; d' packages/$1.rb | grep -q $2:; then
+#       sed -e "/binary_sha256.*({/,/})/s,$2:.*,$2: '$3'," -i packages/$1.rb
+        sed -e "/$2:.*['\"][0-9a-f]*['\"]/c\ \ \ \ $(printf %7s $2): '$3'," -i packages/$1.rb
+    else
+        sed -e "/binary_sha256.*({/a\ \ \ \ $(printf %7s $2): '$3'," -i packages/$1.rb
+    fi
+}
+function get_sha256 () {
+    if [ -f release/$2/$1.sha256 ]; then
+        cut -d' ' -f1 release/$2/$1.sha256
+    else
+        echo not found
+    fi
+}
+function check_url () {
+    curl --output /dev/null --silent --header -L --fail $1
+}
+
+for name in "$@"; do 
+    name=`basename -s .rb $name`
+    pkg=`echo $name | sed -e 's/-.*//'`
+  
+    if ! grep -q binary_url packages/$pkg.rb; then
+        sed -e '/source_sha256/ a\
+\
+\ \ binary_url ({\
+\ \ })\
+\ \ binary_sha256 ({\
+\ \ })' -i packages/$pkg.rb
+    fi
+    version=`grep '\<version ['"'"'"]' packages/$pkg.rb | head -1 | sed -e 's/^.*version *['"'"'"]//' -e 's/['"'"'"].*$//'`
+    echo $pkg $version
+    for arch in x86_64 i686 armv7l; do
+        tarname="$pkg-$version-chromeos-$arch.tar"
+        if check_url "$URL/$tarname.xz"; then
+            url="$URL/$tarname.xz"
+            sha256=`get_sha256 $tarname.xz $arch`
+        elif check_url "$URL/$tarname.gz"; then
+            url="$URL/$tarname.gz"
+            sha256=`get_sha256 $tarname.gz $arch`
+        else
+            url=""
+        fi
+        echo $url : $sha256
+        case x$url in
+        x)  ;;
+        *)  
+            update_url $pkg $arch $url
+            update_sha256 $pkg $arch $sha256
+            case $arch in
+            armv7l)
+                update_url $pkg aarch64 $url
+                update_sha256 $pkg aarch64 $sha256
+                ;;
+            esac
+            ;;
+        esac
+    done
+done

tools/genpkgname.sh

+#!/bin/bash
+
+version=`grep '\<version ['"'"'"]' /usr/local/lib/crew/packages/$1.rb | head -1 | sed -e 's/^.*version *['"'"'"]//' -e 's/['"'"'"].*$//'`
+arch=`uname -m`
+echo $1-$version-chromeos-$arch.tar.xz

tools/upfrog.sh

+#!/bin/bash
+
+# Upload script to jfrog (bintray)
+#
+# Usage
+#
+#   upfrog package...              Upload all version of packages
+#   upfrog package-version...      Upload only a given version of packages
+#   upfrog -t package...           Performs only dry-run (not upload)
+#   upfrog -d package...           Performs only debug-run (not execute jfrog)
+#
+# Note: OSS free account has limitation of the number of REST API uses by a day.
+#       If you see 'Forbidden' errors, please try it again in a next day.
+#
+
+ORG=chromebrew
+REPO=chromebrew
+DRYRUN=
+DEBUG=
+
+# Define your environment
+#
+#   LOCALPATHS:      Multiple paths for your pre-compiled binaries
+#   REGEXPLOCALPATH: Single path using regexp to points all of your LOCALPATHS.
+#                    If your LOCALPATHS contains single directory, still "(|)"
+#                    is required by jfrog, so write something like
+#                    "(real-path|dummy-path)".
+
+# For the case of multiple paths
+LOCALPATHS="release/armv7l release/i686 release/x86_64"
+REGEXPLOCALPATH="release/(armv7l|i686|x86_64)"
+
+# For the case of single paths
+#LOCALPATHS="release/bin"
+#REGEXPLOCALPATH="release/(bin|dummy_zzzzz)"
+
+case $1 in
+-d) # debug
+  DEBUG=1
+  DRYRUN=--dry-run
+  shift;;
+-t) # test
+  DRYRUN=--dry-run
+  shift;;
+esac
+
+function check_pkg () {
+  jfrog bt ps $ORG/$REPO/$1 > /dev/null 2> /dev/null
+}
+
+function create_pkg () {
+  jfrog bt pc $ORG/$REPO/$1 --pub-dn=true --vcs-url=https://github.com/ > /dev/null 2> /dev/null
+}
+
+function check_pkg_ver () {
+  jfrog bt vs $ORG/$REPO/$1/$2 > /dev/null 2> /dev/null
+}
+
+function upload_pkg_ver () {
+  jfrog bt u $DRYRUN --override=true --regexp "$REGEXPLOCALPATH/$1-$2-chrome.*\.tar\..z$" "$ORG/$REPO/$1/$2" # > /dev/null 2> /dev/null
+}
+
+function publish_pkg_ver () {
+  jfrog bt vp "$ORG/$REPO/$1/$2" # > /dev/null 2> /dev/null
+}
+
+function error () {
+  >&2 echo $1
+  exit 1
+}
+
+for name in "$@"; do 
+  name=`basename -s .rb $name`
+  pkg=`echo $name | sed -e 's/-.*//'`
+  
+  # check file existence
+  files=
+  for path in $LOCALPATHS; do 
+    for file in $path/$name-*.tar.?z; do
+      [ -f $file ] && files="$file $files"
+    done
+  done
+  [ -s "$files" ] && continue
+
+  # check version numbers
+  versions=
+  while read version; do
+    versions="$versions $version"
+  done < <(for file in $files; do
+    echo $file | sed -e "s:^.*/$pkg-::" -e 's:-chrome.*$::'
+  done | sort -n | uniq)
+
+  if [ ! -s $DEBUG ]; then
+    echo "$name -> $pkg, $versions: $files"
+    continue
+  fi
+
+  # create package first
+  check_pkg $pkg || create_pkg $pkg || error "failed to create $pkg package"
+
+  # upload and publish
+  for v in $versions; do
+    v=`echo $v | sed -e s/,/./g`
+    # check_pkg_ver $pkg $v || error "failed to check version $pkg-$v"
+    upload_pkg_ver $pkg $v || error "failed to upload $pkg-$v"
+    publish_pkg_ver $pkg $v || error "failed to publish $pkg-$v"
+  done
+done
+