#!/bin/bash -e
# neotest: run tests and benchmarks against FileStorage, ZEO and various NEO/py and NEO/go clusters

# Copyright (C) 2017-2018  Nexedi SA and Contributors.
#                          Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.

set -o pipefail

# FIXME warn if/when thermal throttling activates

# ---- deploy NEO for tests/benchmarks at a node ----

die() {
	echo 2>&1 "$@"
	exit 1
}

# cmd_deploy [user@]<host>:<path>	- deploy NEO & needed software for tests there
# ssh-key or password for access should be available
cmd_deploy() {
	host=`echo $1 |sed -e 's/:[^:]*$//'`	# user@host
	path=${1:$((${#host} + 1))}		# path
	test -z "$host" -o -z "$path" && die "Usage: neotest deploy [user@]<host>:<path>"
	echo -e "\n*** deploying to $@ ..."
	scp $0 $host:neotest
	ssh $host ./neotest deploy-local "$path"
}

# cmd_deploy-local <path>		- deploy NEO & needed software for tests @path
cmd_deploy-local() {
	path=$1
	test -z "$path" && die "Usage: neotest deploy-local <path>"
	test -e $path/deployed && echo "# already deployed" && return
	mkdir -p $path
	cd $path

	# python part
	virtualenv venv

	# env.sh for deployment
	cat >env.sh << 'EOF'
X=${1:-${BASH_SOURCE[0]}}       # path to original env.sh is explicitly passed
X=$(cd `dirname $X` && pwd)     # when there is other env.sh wrapping us

export GOPATH=$X:$GOPATH
export PATH=$X/bin:$PATH
export PS1="(`basename $X`) $PS1"

# strip trailing : from $GOPATH
GOPATH=${GOPATH%:}

# python
. $X/venv/bin/activate

# lmbench
export PATH=$X/lmbench/lmbench3/bin/`cd $X/lmbench/lmbench3/src; ../scripts/os`:$PATH

# ioping
export PATH=$X/ioping:$PATH

# XXX for mysqld, ethtool
export PATH=$PATH:/sbin:/usr/sbin
EOF

	# NOTE lmbench before env.sh because env.sh uses `scripts/os` from lmbench
	git clone -o kirr -b x/kirr https://lab.nexedi.com/kirr/lmbench.git
	pushd lmbench/lmbench3/src
	make -j`nproc`
	go build -o ../bin/`../scripts/os`/lat_tcp_go lat_tcp.go
	popd

	. env.sh

	pip install git+https://lab.nexedi.com/nexedi/wendelin.core.git@master	# XXX does not show git in ver
	pip install zodbtools

	mkdir -p src/lab.nexedi.com/kirr
	pushd src/lab.nexedi.com/kirr
	test -d neo || git clone -o kirr -b t https://lab.nexedi.com/kirr/neo.git neo	# XXX t is temp
	cd neo
	pip install -e .[admin,client,ctl,master,storage-sqlite,storage-mysqldb]
	popd

	pip install cython	# for tcpenalty.py

	go get -v lab.nexedi.com/kirr/neo/go/...
	go get -v github.com/pkg/profile			# used by zhash.go
	go get -v golang.org/x/perf/cmd/benchstat		# to summarize/diff benchmark results
	#go get -v github.com/aclements/perflock/cmd/perflock	# handy to fix CPU frequency/etc

	git clone -o kirr -b x/hist https://lab.nexedi.com/kirr/ioping.git
	pushd ioping
	make -j`nproc`
	popd

	echo ok >deployed
	echo "# deployed ok"
}

# jump to deploy command early if we have to
case "$1" in
deploy|deploy-local)
	cmd="$1"
	shift
	cmd_$cmd "$@"
	exit
	;;
esac

# on <url> ...		- run ... on deployed url from inside dir of neotest
on() {
	#echo "on $@"
	host=`echo $1 |sed -e 's/:[^:]*$//'`	# user@host
	path=${1:$((${#host} + 1))}		# path
	test -z "$host" -o -z "$path" && die "on $1: invalid URL"
	shift
	ssh $host "bash -c \"test -e $path/deployed || { echo 1>&2 '$url not yet deployed'; exit 1; }
cd $path
. env.sh
#set -x
cd src/lab.nexedi.com/kirr/neo/go/neo/t
$@
\""
}

# ---- go/py unit tests ----
cmd_test-go() {
	go test lab.nexedi.com/kirr/neo/go/...
}

cmd_test-py() {
	# NOTE testing with only sqlite should be ok to check for client correctness
	NEO_TESTS_ADAPTER=SQLite python -m neo.scripts.runner -ufz
}

cmd_test-local() {
	cmd_test-go
	cmd_test-py
}

cmd_test() {
	url="$1"
	test -z "$url" && die "Usage neotest test [user@]<host>:<path>"
	on $url ./neotest test-local
}


# ---- net/fs setup + processes control/teardown ----

# init_net	- initialize networking
init_net() {
	# determine our external addresses IPv4 or IPv6
	# 2: wlan0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
	#     inet 192.168.102.52/24 brd 192.168.102.255 scope global dynamic wlan0
	#        valid_lft 82495sec preferred_lft 82495sec
	#
	# 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 state UNKNOWN qlen 1000
	#     inet6 2401:5180:0:37::1/64 scope global
	#        valid_lft forever preferred_lft forever
	# 2: wlan0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 state UP qlen 1000
	#     inet6 2401:5180:0:1d:429a:e612:c957:29/64 scope global noprefixroute dynamic
	#        valid_lft 86232sec preferred_lft 14232sec
	myaddr4v=(`ip -4 addr show scope global |grep inet |awk '{print $2}' |sed -e 's|/.*$||'`) || :
	myaddr6v=(`ip -6 addr show scope global |grep inet |awk '{print $2}' |sed -e 's|/.*$||'`) || :
	test "${#myaddr4v[@]}" -gt 0 || die "init_net: cannot determine my IPv4 network addresses"
	test "${#myaddr6v[@]}" -gt 0 || die "init_net: cannot determine my IPv6 network addresses"

	# prefer ipv4 for now
	myaddr="${myaddr4v[0]}"

	# port allocations ([] works for IPv4 too)
	Abind=[$myaddr]:5551	# NEO admin
	Mbind=[$myaddr]:5552	# NEO master
	Zbind=[$myaddr]:5553	# ZEO

}

# Sbind n	- return bind address for storage #n	(n=1...)
Sbind() {
	# for NEO storage. bind not strictly needed but this way we also make sure
	# no 2 storages are started at the same time.
	echo "[$myaddr]:$((5554 + $1 - 1))"
}

# init_fs	- do initial disk allocations
init_fs() {
	log=`pwd`/log;		mkdir -p $log
	var=`pwd`/var;		mkdir -p $var
}

dataset=	# name of current dataset
dataset_size=	# size parameter for current dataset
ds=		# top-level dir of current dataset
neocluster=	# NEO cluster name (different for different datasets to avoid confusion)

# switch_dataset <name> <size>	- switch benchmarking dataset to named with size
switch_dataset() {
	dataset=$1
	dataset_size=$2
	ds=`echo $dataset $dataset_size |tr ' ' '-'`	# wczblk1 8 -> wczblk1-8
	neocluster=pygotest-$ds
	echo -e "\ndataset:\t$ds"
	ds=$var/$ds

	fs1=$ds/fs1;		mkdir -p $fs1		# FileStorage (and so ZEO and NEO/go) data
}

Pneo=1		# N(partition) for NEO (= N(S) nodes to run)
# XXX switch_neo_P (like switch_dataset)


# neolite n	- NEO: sqlite for partition n
neolite() {
	echo $ds/neo.sqlite·P$Pneo-$1
}

# neosql n	- NEO: mariadb directory for partition n
neosql() {
	local d=$ds/neo.sql·P$Pneo-$1;	mkdir -p $d
	echo $d
}

# mycnf n	- NEO: mariadb config for partition n
mycnf() {
	echo `neosql $1`/mariadb.cnf
}

# mysock n	- NEO: mariadb socket for partition n
mysock() {
	echo $(realpath `neosql $1`)/my.sock
}

# mylog n	- NEO: mariadb log for partition n
mylog() {
	echo $log/mdb.log·P$Pneo-$1
}


# foreach_dataset <command>	- run command once for each dataset serially
foreach_dataset() {
	for d in "${datasetv[@]}" ; do
		switch_dataset $d
		"$@"
	done
}

# xneoctl ...	- control started NEO cluster
xneoctl() {
	neoctl -a $Abind "$@"
}

# xmysql ...	- control all started MariaDB servers
xmysql() {
	for i in `seq $Pneo`; do
		mysql --defaults-file=`mycnf $i` "$@"
	done
}

# if we are abnormally terminating
install_trap() {
	trap 'set +e
echo "E: abnormal termination - stopping..."
xneoctl set cluster stopping
sleep 1
xmysql -e "SHUTDOWN"
sleep 1
j="$(jobs -p)"
test -z "$j" && exit
echo "E: killing left jobs..."
jobs -l
kill $j' EXIT
}

# ---- start NEO/ZEO nodes ----

# neopy_log logname	- `--logfile ...` argument for logging from a NEO/py node
# empty if $X_NEOPY_LOG_SKIP set
neopy_log() {
	test -z "$X_NEOPY_LOG_SKIP" || return
	echo --logfile=$log/$1.log
}

# M{py,go} n ...	- spawn master with N(partition) = n
Mpy() {
	local n=$1
	shift

	# XXX neo/py meaning for --replicas is `n(real-replica) - 1`
	exec -a Mpy \
		neomaster --cluster=$neocluster --bind=$Mbind --masters=$Mbind	\
		--replicas 0 --partitions $n `neopy_log Mpy` "$@" &
}

Mgo() {
	exec -a Mgo \
		neo --log_dir=$log master -cluster=$neocluster -bind=$Mbind "$@" &
}

# Spy <n> ...	- spawn NEO/py storage handling partition #n
Spy() {
	local n=$1
	shift

	# --adapter=...
	# --database=...
	# --engine=...
	exec -a Spy \
		neostorage --cluster=$neocluster --bind=`Sbind $n` --masters=$Mbind `neopy_log Spy` "$@" &
}

# Sgo <n>	- spawn NEO/go storage
Sgo() {
	local n=$1
	shift
	test $n = 1 || die "Sgo: TODO ·P<n>"

	# -alsologtostderr
	exec -a Sgo \
		neo -log_dir=$log storage -cluster=$neocluster -bind=`Sbind $n` -masters=$Mbind "$@" &
}

# Apy ...	- spawn NEO/py admin
Apy() {
	exec -a Apy \
		neoadmin --cluster=$neocluster --bind=$Abind --masters=$Mbind `neopy_log Apy` "$@" &
}

# Zpy <data.fs> ...	- spawn ZEO
Zpy() {
	exec -a Zpy \
		runzeo --address $Zbind --filename "$@" 2>>$log/Zpy.log &
}


# ---- start NEO clusters ----

# spawn NEO/go cluster (Sgo+Mpy+Apy) working on data.fs
NEOgofs1() {
	Mpy 1 --autostart=1
	Sgo 1 fs1://$fs1/data.fs
	Apy
}

# spawn NEO/go cluster (Sgo+Mpy+Apy) working on sqlite db
NEOgolite() {
	test $Pneo == 1 || die "NEOgolite: Sgo not ready to handle ·P<n>"
	Mpy 1 --autostart=1
	Sgo 1 sqlite://`neolite 1`
	Apy
}

# spawn NEO/py cluster working on sqlite db
NEOpylite() {
	Mpy $Pneo --autostart=$Pneo
	for i in `seq $Pneo`; do
		Spy $i --adapter=SQLite --database=`neolite $i`
	done
	Apy
}

# spawn NEO/py cluster working on mariadb
NEOpysql() {
	for i in `seq $Pneo`; do
		MDB $i
	done
	sleep 1	# XXX fragile
	xmysql -e "CREATE DATABASE IF NOT EXISTS neo"

	Mpy $Pneo --autostart=$Pneo
	for i in `seq $Pneo`; do
		Spy $i --adapter=MySQL --engine=InnoDB --database=root@neo`mysock $i`
	done
	Apy
}


# MDB <n>	- setup/spawn mariadb for partition n
MDB() {
	local n=$1
	cat >`mycnf $n` <<EOF
[mysqld]
skip_networking
socket		= `mysock $n`
datadir		= `neosql $n`/data
log_error	= `mylog $n`

# the following comes from
# https://lab.nexedi.com/nexedi/slapos/blob/bd197876/software/neoppod/my.cnf.in#L18
# ---- 8< ----

# kirr: disabled
#plugin-load = ha_tokudb;ha_rocksdb

log_warnings = 1
disable-log-bin

## The following settings come from ERP5 configuration.

max_allowed_packet = 128M
query_cache_size = 32M
innodb_locks_unsafe_for_binlog = 1

# Some dangerous settings you may want to uncomment temporarily
# if you only want performance or less disk access.
#innodb_flush_log_at_trx_commit = 0
#innodb_flush_method = nosync
#innodb_doublewrite = 0
#sync_frm = 0

# Extra parameters.
log_slow_verbosity = explain,query_plan
# kirr: rocksb disabled
# rocksdb_block_cache_size = 10G
# rocksdb_max_background_compactions = 3
long_query_time = 1
innodb_file_per_table = 1

# Force utf8 usage
collation_server = utf8_unicode_ci
character_set_server = utf8
skip_character_set_client_handshake

[client]
socket = `mysock $n`
user = root
EOF

	# setup system tables on first run
	if ! test -e `neosql $n`/data ; then
		# XXX --cross-bootstrap only to avoid final large print notice
		# XXX but cross-bootstrap filters out current host name from installed tables - is it ok?
		mysql_bin=$(dirname `which mysql`)	# XXX under slapos mysql_install_db cannot find its base automatically
		mysql_install_db --basedir=${mysql_bin%/bin} --defaults-file=`mycnf $n` --cross-bootstrap
	fi

	mysqld --defaults-file=`mycnf $n` &
}

# ---- generate test data ----

# dataset 1: wendelin.core array with many small (4K) objects
wczblk1_gen_data() {
	url=$1
	size=$2
	WENDELIN_CORE_ZBLK_FMT=ZBlk1 demo-zbigarray --worksize=$size gen $url
}

# dataset 2: synthetic test database according to NEO PROD1 statistics
prod1_gen_data() {
	url=$1
	size=$2
	zgenprod1.py $url $size
}


# run benchmarks on this datasets:
datasetv=(
	"wczblk1 8"	# size: array size (MB)		XXX raise 32,64,512...
	"prod1   1024"	# size: # of transactions	XXX raise size
	# TODO + wczblk0
)

# generate data in data.fs
GENfs() {
	test -e $ds/generated.fs && return
	echo -e '\n*** generating fs1 data...'
	${dataset}_gen_data $fs1/data.fs $dataset_size
	sync
	touch $ds/generated.fs

	# remember correct hash to later check in benchmarks
	# crc32:1552c530   ; oid=0..2127  nread=8534126  t=0.033s (15.7μs / object)  x=zhash.py
	tzodb.py zhash --$zhashfunc $fs1/data.fs |awk '{print $1}' >$ds/zhash.ok
}

# generate data in sqlite
GENsqlite() {
	test -e $ds/generated.sqlite·P$Pneo && return
	echo -e '\n*** generating sqlite data...'
	NEOpylite
	# NOTE compression is disabled because when benchmarking server latency
	# we do not want the time client(s) take to decompress data to interfere.
	${dataset}_gen_data neo://$neocluster@$Mbind?compress=false $dataset_size
	xneoctl set cluster stopping
	wait	# XXX fragile - won't work if there are children spawned outside
	sync
	touch $ds/generated.sqlite·P$Pneo
}

# generate data in mariadb
GENsql() {
	test -e $ds/generated.sql·P$Pneo && return
	echo -e '\n*** generating sql data...'
	NEOpysql
	# NOTE compression is disabled - see ^^^ (sqlite) for rationale.
	${dataset}_gen_data neo://$neocluster@$Mbind?compress=false $dataset_size
	xneoctl set cluster stopping
	sleep 1	# XXX fragile
	xmysql -e "SHUTDOWN"
	wait	# XXX fragile
	sync
	touch $ds/generated.sql·P$Pneo
}

# generate all test databases
gen_data() {
	GENfs
	GENsqlite
	GENsql
	wait
	sync
}


# ---- information about system ----

# pyver <egg> (<showas>) - print version of egg
pyver() {
	local egg=$1
	local showas=$2
	test "$showas" == "" && showas=$egg
	local loc
	local pyver
	{
		read loc
		read pyver
	} < <(python -c "
import pkg_resources as p
try:
	e=p.require(\"$egg\")[0]
except p.DistributionNotFound:
	print(\"\nø\")
else:
	print(\"%s\n%s\" % (e.location, e.version))
	")
	local gitver=$(git -C $loc describe --long --dirty 2>/dev/null)
	local ver
	test "$gitver" != "" && ver="$gitver" || ver="$pyver"
	printf "sw/%-16s %s\n" "${showas}:" "$ver"
}

# proginfo <prog> ...	- run `prog ...` or print that prog is missing
proginfo() {
	prog=$1
	shift
	which $prog >/dev/null 2>&1 && $prog "$@" || printf "%-16s: ø\n" "$prog"
}

# fkghz file	- extract value from file (in KHz) and render it as GHz
fkghz() {
	python -c "print '%.2fGHz' % (`cat $1` / 1E6)"
}

# lspci1 <pcidev> <field>	- show <field> from lspci information about <pcidev>
lspci1() {
	lspci -vmm -s $1 |grep "^$2:\\s*" |sed -e "s/^$2:\\s*//"
}

# xhostname	- show short system host name
xhostname() {
	# prefer first part of FQDN for misconfigured systems like
	# fqdn=z6001.ivan.nexedi.com, hostname=z6001-COMP-2784
	fqdn=`hostname --fqdn 2>/dev/null || :`
	if test -n "$fqdn"; then
		echo "$fqdn" |sed -e 's/\./ /' |awk '{print $1}'
	else
		hostname
	fi
}

# show information about local system (os, hardware, versions, ...)
system_info() {
	echo -ne "date:\t"; date --rfc-2822
	echo -ne "xnode:\t`whoami`@`hostname --fqdn 2>/dev/null || hostname` ("
	echo -n "${myaddr6v[0]}"
	test "${#myaddr6v[@]}" -eq 1 || echo -n " (+ $((${#myaddr6v[@]} - 1))·ipv6)"
	echo -n " ${myaddr4v[0]}"
	test "${#myaddr4v[@]}" -eq 1 || echo -n " (+ $((${#myaddr4v[@]} - 1))·ipv4)"
	echo ")"
	echo -ne "uname:\t"; uname -a

	# cpu
	echo -ne "cpu:\t"; grep "^model name" /proc/cpuinfo |head -1 |sed -e 's/model name\s*: //'
	syscpu=/sys/devices/system/cpu
	sysidle=$syscpu/cpuidle

	cpuvabbrev() {	# cpuvabbrev cpu0 cpu1 cpu2 ... cpuN	-> cpu/[0-N]
		test $# -le 1 && echo "$@" && return

		min=""
		max=""
		while [ $# -ne 0 ]; do
			v=$1
			shift
			n=${v#cpu}

			test -z "$min" && min=$n && max=$n continue
			if (( $n != $max + 1 )); then
				die "cpuvabbrev: assert: nonconsecutive $max $n"
			fi
			max=$n
		done
		echo "cpu/[$min-$max]"
	}

	freqcpuv=()	# [] of cpu
	freqstr=""	# text about cpufreq for cpus in ^^^
	freqdump() {
		test "${#freqcpuv[@]}" = 0 && return
		echo "`cpuvabbrev ${freqcpuv[*]}`/freq: $freqstr"
		freqcpuv=()
		freqstr=""
	}

	idlecpuv=()	# ----//---- for cpuidle
	idlestr=""
	idledump() {
		test "${#idlecpuv[@]}" = 0 && return
		echo "`cpuvabbrev ${idlecpuv[*]}`/idle: $idlestr"
		idlecpuv=()
		idlestr=""
	}

	freqstable=y
	while read cpu; do
		f="$cpu/cpufreq"
		fmin=`fkghz $f/scaling_min_freq`
		fmax=`fkghz $f/scaling_max_freq`
		fs="`cat $f/scaling_driver`/`cat $f/scaling_governor` [$fmin - $fmax]"
		if [ "$fs" != "$freqstr" ]; then
			freqdump
			freqstr="$fs"
		fi
		freqcpuv+=(`basename $cpu`)
		test "$fmin" != "$fmax" && freqstable=n
	done \
	< <(ls -vd $syscpu/cpu[0-9]*)
	freqdump

	latmax=0
	while read cpu; do
		is="`cat $sysidle/current_driver`/`cat $sysidle/current_governor_ro`:"
		while read state; do
			is+=" "
			lat=`cat $state/latency`
			res=`cat $state/residency 2>/dev/null` || res="?"	# added in linux 3.15
			test "`cat $state/disable`" = "1" && is+="!" || latmax=$(($lat>$latmax?$lat:$latmax))
			is+="`cat $state/name`·${lat}/${res}"
		done \
		< <(ls -vd $cpu/cpuidle/state[0-9]*)

		is+=" # elat/tres µs"

		if [ "$is" != "$idlestr" ]; then
			idledump
			idlestr="$is"
		fi
		idlecpuv+=(`basename $cpu`)
	done \
	< <(ls -vd $syscpu/cpu[0-9]*)
	idledump

	test "$freqstable" = y || echo "WARNING: cpu: frequency not fixed - benchmark timings won't be stable"
	test "$latmax" -le 10  || {
		echo "WARNING: cpu: C-state exit-latency is max ${latmax}μs - benchmark timings won't be stable"
		echo "WARNING: cpu: (up to that might be adding to networked and IPC request-reply latency)"
	}


	# disk under .
	# TODO show all disks and just mark which is current
	mntpt=`stat -c '%m' .`				# mountpoint of current filesystem
	blkdev=`findmnt -n -o source -v -u $mntpt`	# mountpoint -> device
	blkdev=`realpath $blkdev`			# /dev/mapper/vg0-root	-> /dev/dm-0
	blkdev1=`basename $blkdev`			# /dev/sda  -> sda

	# showblk1 <device>
	showblk1() {
		blkdev=$1
		blkdev1=`basename $blkdev`			# /dev/sda  -> sda
		# XXX lsblk: tmpfs: not a block device
		printf "disk/%s: %s\n" "$blkdev1" "`lsblk -dn -o MODEL $blkdev`  rev `lsblk -dn -o REV,SIZE $blkdev`"
	}

	case "$blkdev1" in
	md*)
		# software raid
		slavev=`ls -x /sys/class/block/$blkdev1/slaves`
		printf "disk/%s:\t%s\n" "$blkdev1" "(`cat /sys/class/block/$blkdev1/md/level`) -> $slavev"
		# XXX dup wrt dm-*; move recursion to common place
		for s in $slavev; do
			s=`echo $s |sed -e 's/[0-9]*$//'`	# sda3 -> sda
			showblk1 /dev/$s
		done
		;;
	dm-*)
		# device mapper
		slavev=`ls -x /sys/class/block/$blkdev1/slaves`
		printf "disk/%s:\t%s\n" "$blkdev1" "(`cat /sys/class/block/$blkdev1/dm/name`) -> $slavev"
		# XXX dup wrt md*; move recursion to common place
		for s in $slavev; do
			s=`echo $s |sed -e 's/[0-9]*$//'`	# sda3 -> sda
			showblk1 /dev/$s
		done
		;;
	*)
		blkdev_main=`echo $blkdev |sed -e 's/[0-9]*$//'`	# /dev/sda3 -> /dev/sda
		showblk1 $blkdev_main
		;;
	esac

	# all NICs
	# XXX warn if ethtool is not there
	find /sys/class/net -type l -not -lname '*virtual*' |sort | \
	while read nic; do
		nicname=`basename $nic`		# /sys/class/net/eth0	-> eth0
		echo -n "nic/$nicname: "
		nicdev=`realpath $nic/device`	# /sys/class/net/eth0	-> /sys/devices/pci0000:00/0000:00:1f.6

		case "$nicdev" in
		*usb*)
			# /sys/devices/pci0000:00/0000:00:1d.0/usb2/2-1/2-1.6/2-1.6:1.6 -> .../usb2/2-1/2-1.6
			usbdev="$nicdev/.."
			usbdev=`cat $usbdev/busnum`:`cat $usbdev/devnum`	# ... -> 2:4
			product=`lsusb -s $usbdev |sed -e 's/^Bus\s[0-9]*\sDevice\s[0-9]*:\sID\s[0-9a-f]*:[0-9a-f]*\s//'`
			echo "$product (usb)"
			;;

		*pci*)
			pcidev=`basename $nicdev`	# /sys/devices/pci0000:00/0000:00:1f.6	-> 0000:00:1f.6
			#lspci -s $pcidev
			echo "`lspci1 $pcidev Vendor` `lspci1 $pcidev Device` rev `lspci1 $pcidev Rev`"
			;;

		*)
			echo "$nicdev (TODO)"
			;;
		esac

		nicwarnv=()

		# show relevant features
		featok=y
		feat=`ethtool -k $nicname 2>/dev/null` || featok=n
		if [ $featok != y ]; then
			echo "nic/$nicname/features: ?"
		else
			# feat1 name abbrev -> abbrev. value  (e.g. "tx" or "!tx")
			feat1() {
				# ntuple-filters: off [fixed]
				v=`echo "$feat" |grep "^$1:\\s*" |awk '{print $2}'`
				case $v in
				on)
					echo "$2"
					;;
				off)
					echo "!$2"
					;;
				*)
					echo "?($v)$2"
					;;
				esac
			}

			s="nic/$nicname/features:"
			# NOTE feature abbrevs are those used by `ethtool -K` to set them
			s+=" `feat1 rx-checksumming			rx`"
			s+=" `feat1 tx-checksumming			tx`"
			s+=" `feat1 scatter-gather			sg`"
			tso="`feat1 tcp-segmentation-offload            tso`"
			s+=" $tso"
			s+=" `feat1 udp-fragmentation-offload		ufo`"
			s+=" `feat1 generic-segmentation-offload	gso`"
			s+=" `feat1 generic-receive-offload		gro`"
			s+=" `feat1 large-receive-offload		lro`"
			s+=" `feat1 rx-vlan-offload			rxvlan`"
			s+=" `feat1 tx-vlan-offload			txvlan`"
			s+=" `feat1 ntuple-filters			ntuple`"
			s+=" `feat1 receive-hashing			rxhash`"

			# ^^^ are the common features - others are specific to kernel/device
			# XXX or list them all?
			s+=" ..."

			echo "$s"

			# warn if !tso (linux starts enabling autocorking for e.g. small second segment and probably
			# something else and lat_tcp latency grows stepwise from ~130μs to 500μs and more)
			test "$tso" == "tso" || nicwarnv+=("TSO not enabled - TCP latency with packets > MSS will be poor")
		fi


		# show rx/tx coalescing latency
		echo -n "nic/$nicname/coalesce:"
		coalok=y
		coal=`ethtool -c $nicname 2>/dev/null` || coalok=n
		if [ $coalok != y ]; then
			echo " rxc: ?,  txc: ?"
		else
			# coal1 name -> value
			coal1() {
				echo "$coal" |grep "^$1:\\s*" | sed -e "s/^$1:\\s*//"
			}
			rxt=`coal1 rx-usecs`
			rxf=`coal1 rx-frames`
			rxt_irq=`coal1 rx-usecs-irq`
			rxf_irq=`coal1 rx-frames-irq`

			txt=`coal1 tx-usecs`
			txf=`coal1 tx-frames`
			txt_irq=`coal1 tx-usecs-irq`
			txf_irq=`coal1 tx-frames-irq`

			echo -en  " rxc: ${rxt}μs/${rxf}f/${rxt_irq}μs-irq/${rxf_irq}f-irq,"
			echo -e  "  txc: ${txt}μs/${txf}f/${txt_irq}μs-irq/${txf_irq}f-irq"

			# XXX also add -low and -high ?

			# warn if rx latency is too high
			rxlat=$(($rxt>$rxt_irq?$rxt:$rxt_irq))
			test "$rxlat" -le 10 || nicwarnv+=("RX coalesce latency is max ${rxlat}μs - that will add to networked request-reply latency")
		fi

		# show main parameters + GRO flush time
		s="nic/$nicname/status:  "
		s+=" `cat $nic/operstate`"
		speed=`cat $nic/speed 2>/dev/null` || speed=?	# returns EINVAL for wifi
		s+=", speed=$speed"
		s+=", mtu=`cat $nic/mtu`"
		s+=", txqlen=`cat $nic/tx_queue_len`"
		if test -e $nic/gro_flush_timeout ; then
			tgroflush_ns=`cat $nic/gro_flush_timeout`
			s+=", gro_flush_timeout=`python -c "print '%.3f' % ($tgroflush_ns / 1E3)"`µs"
		else
			s+=", !gro_flush_timeout"
		fi
		echo "$s"

		# XXX warn if gro_flush_timeout=0 or unsupported ?


		# emit NIC warnings
		for warn in "${nicwarnv[@]}"; do
			echo "WARNING: nic/$nicname: $warn"
		done

	done

	printf "%-20s" "sw/python:";	proginfo python --version 2>&1	# https://bugs.python.org/issue18338
	printf "%-20s" "sw/go:";	proginfo go version
	printf "%-20s" "sw/sqlite:";	proginfo python -c \
		'import sqlite3 as s; print "sqlite %s (py mod %s)" % (s.sqlite_version, s.version)'
	printf "%-20s" "sw/mysqld:";	proginfo mysqld --version

	pyver neoppod neo
	pyver zodb
	pyver zeo
	pyver mysqlclient
	pyver wendelin.core
}


# ---- benchmarking ----

# cpustat ...	- run ... and print CPU C-states statistic
cpustat() {
	# XXX +cpufreq transition statistics (CPU_FREQ_STAT) ?

	syscpu=/sys/devices/system/cpu
	cpuv=( `ls -vd $syscpu/cpu[0-9]*` )
	# XXX we assume cpuidle states are the same for all cpus and get list of them from cpu0
	statev=( `ls -vd ${cpuv[0]}/cpuidle/state[0-9]* |xargs -n 1 basename` )

	# get current [state]usage. usage for a state is summed across all cpus
	statev_usage() {
		usagev=()
		for s in ${statev[*]}; do
			#echo >&2 $s
			susage=0
			for u in `cat $syscpu/cpu[0-9]*/cpuidle/$s/usage`; do
				#echo -e >&2 "\t$u"
				((susage+=$u))
			done
			usagev+=($susage)
		done
		echo ${usagev[*]}
	}

	ustartv=( `statev_usage` )
	#echo >&2 "--------"
	#sleep 1
	ret=0
	out="$("$@" 2>&1)" || ret=$?
	uendv=( `statev_usage` )

	stat="#"
	for ((i=0;i<${#statev[*]};i++)); do
		s=${statev[$i]}
		sname=`cat ${cpuv[0]}/cpuidle/$s/name`
		du=$((${uendv[$i]} - ${ustartv[$i]}))
		#stat+=" $sname(+$du)"
		stat+=" $sname·$du"
		#stat+=" $du·$sname"
	done

	if [ `echo "$out" | wc -l` -gt 1 ]; then
		# multiline out - add another line
		echo "$out"
		echo "$stat"
	else
		# 1-line out	- add stats at line tail
		echo -n "$out"
		echo -e "\t$stat"
	fi

	return $ret
}

Nrun=5			# repeat benchmarks N time
Nparv="1 2 3 4 8 12 16"	# run parallel zwrk benchmarks with so many clients
#Npar=16		# run so many parallel clients in parallel phase
Tzwrk1=1s		# time to run 1 zwrk iteration

#profile=
profile=cpustat

# nrun ...	- run ... $Nrun times serially
nrun() {
	for i in `seq $Nrun`; do
		$profile "$@"
	done
}

# nrunpar ...	- run $Npar ... instances in parallel and wait for completion
# XXX running processes in parallel is deprecated in favour of zwrk.
nrunpar() {
	$profile _nrunpar "$@"
}

_nrunpar() {
	local jobv
	for i in `seq $Npar`; do
		"$@" &
		jobv="$jobv $!"
	done
	wait $jobv
}

# bench_cpu	- microbenchmark CPU
bench_cpu() {
	echo -ne "node:\t"; xhostname
	echo     "cluster:"
	nrun sh -c "python -m test.pystone |tail -1 |sed -e \
		\"s|^This machine benchmarks at \([0-9.]\+\) pystones/second$|Benchmarkpystone 1 \1 pystone/s|\""

	sizev="4096"		# 1024 $((2*1024*1024))
	benchv="crc32 sha1"	# adler32
	for bench in $benchv; do
		for size in $sizev; do
			nrun tcpu.py $bench $size
			nrun tcpu_go $bench $size
		done
	done

	datav="wczdata prod1-avg"	# null-1K null-4K null-2M prod1-max
	for data in $datav; do
		nrun tcpu.py unzlib $data
		nrun tcpu_go unzlib $data
	done

	# TODO bench compress
}

# bench_disk	- benchmark direct (uncached) and cached random reads
bench_disk() {
	echo -ne "node:\t"; xhostname
	echo     "cluster:"
	# ioping2bench <topic>	- converts timings from ioping to std benchmark
	ioping2bench() {
		# min/avg/max/mdev = 102.2 us / 138.6 us / 403.3 us / 12.2 us
		sed -u -e \
		"s|^min/avg/max/mdev = \([0-9.]\+\) \([^ ]\+\) / \([0-9.]\+\) \([^ ]\+\) / \([0-9.]\+\)\+ \([^ ]\+\) / \([0-9.]\+\) \([^ ]\+\)\$|&\n\
Benchmark$1-min 1 \\1 \\2/op\n\
Benchmark$1-avg 1 \\3 \\4/op\
|"
	}

	sizev="4K"	# 2M (XXX it is not linear, but we save benchmarking time for now)
	benchtime=3s

	for size in $sizev; do
		echo -e "\n*** disk: random direct (no kernel cache) $size-read latency"
		nrun ioping -D -i 0ms -s $size -S 1024M -w $benchtime -q -k . |\
			ioping2bench "disk/randread/direct/$size"
	done


	# warmup so kernel puts the file into pagecache
	for i in `seq 3`; do
		cat ioping.tmp >/dev/null
	done

	for size in $sizev; do
		echo -e "\n*** disk: random cached $size-read latency"
		nrun ioping -C -i 0ms -s $size -S 1024M -w $benchtime -q -k . |\
			ioping2bench "disk/randread/pagecache/$size"
	done
}

# hostof <url>		- return hostname part of <url>
hostof() {
	url=$1
	python -c "import urlparse as p; u=p.urlparse(\"scheme://$url\"); print u.hostname"
}

# bench_net <url>	- benchmark network
bench_net() {
	url=$1
	peer=`hostof $url`
	shortpeer=`echo $peer |sed -e 's/\./ /' |awk '{print $1}'`	# name.domain -> name

	echo    "node:"
	echo -e "cluster:\t`xhostname`-$shortpeer"

	echo -e "\n*** link latency:"

	# ping2bench <topic>	- converts timings from ping to std benchmark format
	ping2bench() {
		# rtt min/avg/max/mdev = 0.028/0.031/0.064/0.007 ms, ipg/ewma 0.038/0.032 ms
		sed -u -e \
		"s|^rtt min/avg/max/mdev = \([0-9.]\+\)/\([0-9.]\+\)/\([0-9.]\+\)/\([0-9.]\+\) ms.*\$|&\n\
Benchmark$1-min 1 \\1 ms/op\n\
Benchmark$1-avg 1 \\2 ms/op\
|"
	}

	#   16 = minimum ping payload size at which it starts putting struct timeval into payload and print RTT
	# 1472 = 1500 (Ethernet MTU) - 20 (IPv4 header !options) - 8 (ICMPv4 header)
	# 1452 = 1500 (Ethernet MTU) - 40 (IPv6 header !options) - 8 (ICMPv6 header)
	# FIXME somehow IPv6 uses lower MTU than 1500 - recheck
	sizev="16 1452"	# max = min(IPv4, IPv6) so that it is always only 1 Ethernet frame on the wire
	for size in $sizev; do
		echo -e "\n# `xhostname` ⇄ $peer (ping ${size}B)"
		{ $profile sudo -n ping -i0 -w 3 -s $size -q $peer	|| \
			echo "# skipped -> enable ping in sudo for `whoami`@`xhostname`"; } | \
			ping2bench pingrtt-/${size}B

		echo -e "\n# $peer ⇄ `xhostname` (ping ${size}B)"
		# TODO profile remotely
		on $url "sudo -n ping -i0 -w3 -s ${size} -q \$(echo \${SSH_CONNECTION%% *}) || \
			echo \\\"# skipped -> enable ping in sudo for \`whoami\`@${peer}\\\"" | \
			ping2bench -pingrtt/${size}B
	done

	# TODO
	# echo 1 > /proc/sys/net/ipv4/tcp_low_latency
	# netstat -s
	# /sys/class/net/ethX/gro_flush_timeout
	# /proc/sys/net/ipv4/tcp_limit_output_bytes
	#  ( https://lwn.net/Articles/507065/	"The default value of this
	#    limit is 128KB; it could be set lower on systems where latency is the primary concern" )
	# ? tcp pacing
	# net.ipv4.tcp_autocorking (f54b3111 "tcp: auto corking")
	echo -e "\n*** TCP latency:"

	# lattcp2bench <topic>	- convert timings from lat_tcp to std benchmark format
	lattcp2bench() {
		# TCP latency using neo2: 52.3468 microseconds
		sed -u -e \
		"s|^TCP latency using .*: \([0-9.]\+\) microseconds.*\$|Benchmark$1 1 \\1 µs/op\t# &|"
	}

	#    1 = minimum TCP payload
	# 1460 = 1500 (Ethernet MTU) - 20 (IPv4 header !options) - 20 (TCP header !options)
	# 1440 = 1500 (Ethernet MTU) - 40 (IPv6 header !options) - 20 (TCP header !options)
	# FIXME somehow IPv6 uses lower MTU than 1500 - recheck
	sizev="1 1400 1500 4096" # 1400 = 1440 - ε (1 eth frame); 1500 = 1440 + ε (2 eth frames); 4096 - just big 4K (3 eth frames)
	for size in $sizev; do
		echo -e "\n# `xhostname` ⇄ $peer (lat_tcp.c ${size}B  -> lat_tcp.c -s)"
		# TODO profile remotely
		on $url "nohup lat_tcp -s </dev/null >/dev/null 2>/dev/null &"
		nrun lat_tcp -m $size $peer | lattcp2bench "tcprtt(c_c)-/${size}B"
		lat_tcp -S $peer

		echo -e "\n# `xhostname` ⇄ $peer (lat_tcp.c ${size}B  -> lat_tcp.go -s)"
		# TODO profile remotely
		on $url "nohup lat_tcp_go -s </dev/null >/dev/null 2>/dev/null &"
		nrun lat_tcp -m $size $peer | lattcp2bench "tcprtt(c_go)-/${size}B"
		lat_tcp -S $peer

		echo -e "\n# $peer ⇄ `xhostname` (lat_tcp.c ${size}B  -> lat_tcp.c -s)"
		lat_tcp -s
		# TODO profile remotely
		nrun on $url "lat_tcp -m $size \${SSH_CONNECTION%% *}" | lattcp2bench "-tcprtt(c_c)/${size}B"
		lat_tcp -S localhost

		echo -e "\n# $peer ⇄ `xhostname` (lat_tcp.c ${size}B  -> lat_tcp.go -s)"
		lat_tcp_go -s 2>/dev/null &
		# TODO profile remotely
		nrun on $url "lat_tcp -m $size \${SSH_CONNECTION%% *}" | lattcp2bench "-tcprtt(c_go)/${size}B"
		lat_tcp -S localhost
	done
}


# hash function to compute via zhash in tests/benchmarks
zhashfunc=crc32	# sha1, adler32, null, ...

# zbench <url> <topic> <zhashok>	- run ZODB client benchmarks against URL
zbench() {
	url=$1
	topic=$2
	zhashok=$3
#	nrun time demo-zbigarray read $url

	nrun tzodb.py zhash --check=$zhashok --bench=$topic-%s --$zhashfunc $url
# XXX running processes in parallel is deprecated in favour of zwrk.
#	echo -e "\n# ${Npar} clients in parallel"
#	nrunpar tzodb.py zhash --check=$zhashok --bench=$topic-%s·P$Npar --$zhashfunc $url
	echo
	zbench_go $url $topic $zhashok
}

# go-only part of zbench
zbench_go() {
	url=$1
	topic=$2-%s
	zhashok=$3
	test "$X_NEOGO_SHA1_SKIP" = y && topic="$topic(!sha1)" || :

	nrun tzodb_go -log_dir=$log zhash -check=$zhashok -bench=$topic -$zhashfunc $url
	nrun tzodb_go -log_dir=$log zhash -check=$zhashok -bench=$topic -$zhashfunc -useprefetch $url

# XXX running processes in parallel is deprecated in favour of zwrk.
#	echo -e "\n# ${Npar} clients in parallel"
#	nrunpar tzodb_go -log_dir=$log zhash -check=$zhashok -bench=$topic·P$Npar -$zhashfunc $url

	for i in ${Nparv}; do
		echo -e "\n# $i clients in parallel"
		nrun tzodb_go -test.benchtime $Tzwrk1 -log_dir=$log \
			zwrk -nclient $i -check=$zhashok -bench=$topic -$zhashfunc $url
	done
}


# command: benchmark when client and storage are on the same computer
cmd_bench-local() {
	echo -e ">>> bench-local"
	echo -e "\n*** neotest: node"
	system_info
	echo -e "\n*** cpu:\n"
	bench_cpu
	bench_disk
	cmd_zbench-local
}

# command: benchmark ZODB client/server over localhost
cmd_zbench-local() {
	install_trap

	foreach_dataset zbench_local

	# all ok
	trap - EXIT
}

# zodb part of cmd_bench-local
zbench_local() {
	echo "node:"
	echo -ne "cluster:\t"; xhostname

	gen_data
	zhashok=`cat $ds/zhash.ok`

	echo -e "\n*** FileStorage"
	zbench $fs1/data.fs fs1 $zhashok

	echo -e "\n*** ZEO"
	Zpy $fs1/data.fs
	Zpy_job=$!
	zbench zeo://$Zbind zeo/py/fs1 $zhashok
	kill $Zpy_job
	wait $Zpy_job

	# XXX save time - we show only neo/py(!log)/sqlite
	#echo -e "\n*** NEO/py sqlite"
	#NEOpylite
	#zbench neo://$neocluster@$Mbind neo/py/sqlite·P$Pneo $zhashok
	#xneoctl set cluster stopping
	#wait

	# XXX JM asked to also have NEO/py with logging disabled
	echo -e "\n*** NEO/py sqlite (logging disabled)"
	X_NEOPY_LOG_SKIP=y NEOpylite
	zbench neo://$neocluster@$Mbind "neo/py(!log)/sqlite·P$Pneo" $zhashok
	xneoctl set cluster stopping
	wait

	# XXX save time - we show only neo/py(!log)/sql
	#echo -e "\n*** NEO/py sql"
	#NEOpysql
	#zbench neo://$neocluster@$Mbind neo/py/sql·P$Pneo $zhashok
	#xneoctl set cluster stopping
	#xmysql -e "SHUTDOWN"
	#wait

	# XXX JM asked to also have NEO/py with logging disabled
	echo -e "\n*** NEO/py sql (logging disabled)"
	X_NEOPY_LOG_SKIP=y NEOpysql
	zbench neo://$neocluster@$Mbind "neo/py(!log)/sql·P$Pneo" $zhashok
	xneoctl set cluster stopping
	xmysql -e "SHUTDOWN"
	wait

	echo -e "\n*** NEO/go fs1"
	NEOgofs1
	zbench neo://$neocluster@$Mbind neo/go/fs1·P1 $zhashok
	xneoctl set cluster stopping
	wait

	echo -e "\n*** NEO/go fs1 (sha1 disabled on: storage, client)"
	X_NEOGO_SHA1_SKIP=y NEOgofs1
	X_NEOGO_SHA1_SKIP=y zbench_go neo://$neocluster@$Mbind "neo/go/fs1(!sha1)·P1" $zhashok
	xneoctl set cluster stopping
	wait

	echo -e "\n*** NEO/go sqlite"
	if [ $Pneo == 1 ]; then
		NEOgolite
		zbench neo://$neocluster@$Mbind neo/go/sqlite·P$Pneo $zhashok
		xneoctl set cluster stopping
		wait
	else
		echo "# skipping (NEO/go sqlite not yet ready to handle ·P<n>"
	fi

	echo -e "\n*** NEO/go sqlite (sha1 disabled on: client)"
	if [ $Pneo == 1 ]; then
		NEOgolite
		X_NEOGO_SHA1_SKIP=y zbench_go neo://$neocluster@$Mbind "neo/go/sqlite·P$Pneo" $zhashok
		xneoctl set cluster stopping
		wait
	else
		echo "# skipping (NEO/go sqlite not yet ready to handle ·P<n>"
	fi
}

# command: benchmark when server runs locally and client is on another node
cmd_bench-cluster() {
	url=$1
	test -z "$url" && die "Usage: neotest bench-cluster [user@]<host>:<path>"

	echo -e ">>> bench-cluster $url"
	echo -e "\n*** neotest: node: (server)"
	system_info
	echo -e "\n*** neotest: node: (client)"
	on $url ./neotest info-local

	echo -e "\n*** server cpu:"
	bench_cpu

	echo -e "\n*** client cpu:"
	on $url ./neotest bench-cpu

	echo -e "\n*** server disk:"
	bench_disk

	bench_net $url

	# zodb benchmarks
	echo
	cmd_zbench-cluster $url
}

# command: benchmark ZODB client/server over network
cmd_zbench-cluster() {
	url=$1
	test -z "$url" && die "Usage: neotest zbench-cluster [user@]<host>:<path>"

	install_trap

	foreach_dataset zbench_cluster $url

	# all ok
	trap - EXIT
}

# zbench_cluster <url>	- zodb part of cmd_bench-cluster
zbench_cluster() {
	url=$1
	peer=`hostof $url`

	echo "node:"
	echo -e "cluster:\t`xhostname`-$peer"

	gen_data
	zhashok=`cat $ds/zhash.ok`

	echo -e "\n*** ZEO"
	Zpy $fs1/data.fs
	Zpy_job=$!
	on $url ./neotest zbench-client zeo://$Zbind zeo/py/fs1 $zhashok
	kill $Zpy_job
	wait $Zpy_job

	# XXX save time - we show only neo/py(!log)/sqlite
	#echo -e "\n*** NEO/py sqlite"
	#NEOpylite
	#on $url ./neotest zbench-client neo://$neocluster@$Mbind neo/py/sqlite·P$Pneo $zhashok
	#xneoctl set cluster stopping
	#wait

	# XXX JM asked to also have NEO/py with logging disabled
	echo -e "\n*** NEO/py sqlite (logging disabled)"
	X_NEOPY_LOG_SKIP=y NEOpylite
	on $url ./neotest zbench-client neo://$neocluster@$Mbind "\\\"neo/py(!log)/sqlite\\\"·P$Pneo" $zhashok
	xneoctl set cluster stopping
	wait

	# XXX save time - we show only neo/py(!log)/sql
	#echo -e "\n*** NEO/py sql"
	#NEOpysql
	#on $url ./neotest zbench-client neo://$neocluster@$Mbind neo/py/sql·P$Pneo $zhashok
	#xneoctl set cluster stopping
	#xmysql -e "SHUTDOWN"
	#wait

	# XXX JM asked to also have NEO/py with logging disabled
	echo -e "\n*** NEO/py sql (logging disabled)"
	X_NEOPY_LOG_SKIP=y NEOpysql
	on $url ./neotest zbench-client neo://$neocluster@$Mbind "\\\"neo/py(!log)/sql\\\"·P$Pneo" $zhashok
	xneoctl set cluster stopping
	xmysql -e "SHUTDOWN"
	wait

	echo -e "\n*** NEO/go fs"
	NEOgofs1
	on $url ./neotest zbench-client neo://$neocluster@$Mbind neo/go/fs1·P1 $zhashok
	xneoctl set cluster stopping
	wait

	echo -e "\n*** NEO/go fs1 (sha1 disabled on: storage, client)"
	X_NEOGO_SHA1_SKIP=y NEOgofs1
	on $url X_NEOGO_SHA1_SKIP=y ./neotest zbench-client --goonly neo://$neocluster@$Mbind "\\\"neo/go/fs1(!sha1)\\\"·P1" $zhashok
	xneoctl set cluster stopping
	wait

	echo -e "\n*** NEO/go sqlite"
	if [ $Pneo == 1 ]; then
		NEOgolite
		on $url ./neotest zbench-client neo://$neocluster@$Mbind neo/go/sqlite·P$Pneo $zhashok
		xneoctl set cluster stopping
		wait
	else
		echo "# skipping (NEO/go sqlite not yet ready to handle ·P<n>"
	fi

	echo -e "\n*** NEO/go sqlite (sha1 disabled on: client)"
	if [ $Pneo == 1 ]; then
		NEOgolite
		on $url X_NEOGO_SHA1_SKIP=y ./neotest zbench-client --goonly neo://$neocluster@$Mbind "\\\"neo/go/sqlite\\\"·P$Pneo" $zhashok
		xneoctl set cluster stopping
		wait
	else
		echo "# skipping (NEO/go sqlite not yet ready to handle ·P<n>"
	fi
}

# command: benchmark client workload against separate server
cmd_zbench-client() {
	goonly=""
	case "$1" in
	--goonly)
		goonly=y
		shift
		;;
	esac

	url=$1
	topic=$2
	zhashok=$3
	test -z "$url" -o -z "$topic" -o -z "$zhashok" && die "Usage: neotest zbench-client <url> <topic> <zhashok>"

	test -z "$goonly" && zbench $url $topic $zhashok || zbench_go $url $topic $zhashok
}

# command: benchmark local disk
cmd_bench-disk() {
	bench_disk
}

# command: benchmark local cpu
cmd_bench-cpu() {
	bench_cpu
}

# command: benchmark network
cmd_bench-net() {
	url=$1
	test -z "$url" && die "Usage: neotest bench-net [user@]<host>:<path>"
	bench_net $url
}

# command: print information about local node
cmd_info-local() {
	init_net
	system_info
}

# command: print information about remote node
cmd_info() {
	url="$1"
	test -z "$url" && die "Usage neotest info [user@]<host>:<path>"
	on $url ./neotest info-local
}

# utility: cpustat on running arbitrary command
cmd_cpustat() {
	cpustat "$@"
}

# ---- main driver ----

usage() {
cat 1>&2 << EOF
Neotest is a tool to test and benchmark NEO.

Usage:

	neotest command [arguments]

The commands are:

	test		run all tests on a remote host
	test-local	run all tests locally

	test-go		run NEO/go unit tests	(part of test-local)
	test-py		run NEO/py unit tests	(part of test-local)


	bench-local	run all benchmarks when client and server are both on the same localhost
	bench-cluster	run all benchmarks when server is local and client is on another node

	bench-cpu	benchmark local cpu	(part of bench-{local,cluster})
	bench-disk	benchmark local disk	(part of bench-{local,cluster})
	bench-net	benchmark network	(part of bench-cluster)

	zbench-local	run ZODB benchmarks on localhost	(part of bench-local)
	zbench-cluster	run ZODB benchmarks via network		(part of bench-cluster)
	zbench-client	run ZODB client benchmarks against separate server (part of zbench-cluster)


	deploy		deploy NEO & needed software for tests to remote host
	deploy-local	deploy NEO & needed software for tests locally

	info		print information about a node
	info-local	print information about local deployment

Additional utility commands:

	cpustat		run a command and print CPU-related statistics
EOF
}

case "$1" in
# commands that require build
test-local	| \
test-go		| \
test-py		| \
bench-local	| \
bench-cluster	| \
zbench-local	| \
zbench-cluster	| \
zbench-client	| \
bench-cpu	| \
bench-disk	| \
bench-net)
	;;

info)
	shift
	cmd_info "$@"
	exit 0
	;;

info-local)
	shift
	cmd_info-local "$@"
	exit 0
	;;

cpustat)
	shift
	cmd_cpustat "$@"
	exit 0
	;;

-h)
	usage
	exit 0
	;;
*)
	usage
	exit 1
	;;
esac

# make sure tzodb*, tcpu* and zgenprod are on PATH (because we could be invoked from another dir)
X=$(cd `dirname $0` && pwd)
export PATH=$X:$PATH

# rebuild go bits
# neo/py, wendelin.core, ... - must be pip install'ed - `neotest deploy` cares about that
go install -v lab.nexedi.com/kirr/neo/go/...
go build -o $X/tzodb_go $X/tzodb.go
#go build -race -o $X/tzodb_go $X/tzodb.go
go build -o $X/tcpu_go $X/tcpu.go

# setup network & fs environment
init_net
init_fs

# run the command
cmd="$1"
shift
cmd_$cmd "$@"
