Skip to content

M
mariadb
Commit ea0ba57f authored by Rich Prohaska's avatar Rich Prohaska Committed by Yoni Fogel
Browse files
Options

#4084 try to speed up fractal tree tests on pointy refs[t:4084] 

git-svn-id: file:///svn/toku/tokudb@38775 c7de825b-a66e-492c-adef-691d508d4ae1
parent 6549c635
Hide whitespace changes
Inline Side-by-side
Showing with 156 additions and 555 deletions
scripts/build.check.bash
View file @ ea0ba57f
#!/bin/bash #!/bin/bash
# build.check.bash --revision=10500 # build.check.bash --revision=10500 --ftcc=icc
# build.check.bash --tokudb=tokudb.1489 --revision=10500 # build.check.bash --tokudb=tokudb.1489 --revision=10500 --ftcc=gcc44
# build.check.bash --branch=mysql.branches/1.1.3 --revision=10500 # build.check.bash --branch=mysql.branches/1.1.3 --revision=10500
# build.check.bash --windows=1 --revision=10500
function usage() { function usage() {
echo "build tokudb and run regressions" echo "build tokudb and run regressions"
echo "--windows=$dowindows (if yes/1 must be first option)" echo "--branch=$branch --tokudb=$tokudb --revision=$revision --bdb=$bdb"
echo "--branch=$branch" echo "--ftcc=$ftcc --VALGRIND=$VALGRIND --makejobs=$makejobs --parallel=$parallel"
echo "--tokudb=$tokudb" echo "--debugtests=$debugtests --releasetests=$releasetests"
echo "--revision=$revision" echo "--commit=$commit"
echo "--bdb=$bdb"
echo "--valgrind=$dovalgrind"
echo "--commit=$docommit"
echo "--j=$makejobs"
echo "--deleteafter=$deleteafter"
echo "--doclean=$doclean"
} }
function retry() { function retry() {
...@@ -78,27 +71,28 @@ function runcmd() { ...@@ -78,27 +71,28 @@ function runcmd() {
echo `mydate` $makedir $cmd echo `mydate` $makedir $cmd
pushd $makedir pushd $makedir
$cmd if [ $? = 0 ] ; then
exitcode=$? $cmd
local dir=$makedir exitcode=$?
if [[ $dir =~ "$HOME/svn.build/(.*)" ]] ; then local dir=$makedir
dir=${BASH_REMATCH[1]} if [[ $dir =~ "$HOME/svn.build/(.*)" ]] ; then
fi dir=${BASH_REMATCH[1]}
if [ $fail -eq 0 ] ; then fi
if [ $exitcode -eq 0 ] ; then if [ $fail -eq 0 ] ; then
result="PASS `mydate` $dir $cmd" if [ $exitcode -eq 0 ] ; then
let npass=npass+1 result="PASS `mydate` $dir $cmd"
else let npass=npass+1
result="FAIL `mydate` $dir $cmd" else
let nfail=nfail+1 result="FAIL `mydate` $dir $cmd"
fi let nfail=nfail+1
else fi
if [ $exitcode -eq 0 ] ; then else
result="XPASS `mydate` $dir $cmd" if [ $exitcode -eq 0 ] ; then
let nfail=nfail+1 result="XPASS `mydate` $dir $cmd"
else else
result="XFAIL `mydate` $dir $cmd" result="XFAIL `mydate` $dir $cmd"
fi fi
fi
fi fi
echo $result echo $result
echo $result >>$commit_msg echo $result >>$commit_msg
...@@ -106,6 +100,11 @@ function runcmd() { ...@@ -106,6 +100,11 @@ function runcmd() {
popd popd
} }
function my_mktemp() {
local prefix=$1
mktemp /tmp/$prefix.XXXXXXXXXX
}
# build a version of tokudb with a specific BDB target # build a version of tokudb with a specific BDB target
function build() { function build() {
...@@ -117,11 +116,7 @@ function build() { ...@@ -117,11 +116,7 @@ function build() {
else else
return 1 return 1
fi fi
if [ $dowindows -eq 0 ] ; then export BDBDIR=/usr/local/BerkeleyDB.$BDB
export BDBDIR=/usr/local/BerkeleyDB.$BDB
else
export BDBDIR=c:/cygwin/usr/local/BerkeleyDB.$BDB
fi
if [ ! -d $BDBDIR ] ; then return 2; fi if [ ! -d $BDBDIR ] ; then return 2; fi
tokudb_name=`make_tokudb_name $branch $tokudb` tokudb_name=`make_tokudb_name $branch $tokudb`
...@@ -134,7 +129,7 @@ function build() { ...@@ -134,7 +129,7 @@ function build() {
latestrev=`get_latest_svn_revision $checkout` latestrev=`get_latest_svn_revision $checkout`
if [ $latestrev -eq 0 ] ; then return 3; fi if [ $latestrev -eq 0 ] ; then return 3; fi
commit_msg=`mktemp` commit_msg=$(my_mktemp ft)
svnbase=~/svn.build svnbase=~/svn.build
if [ ! -d $svnbase ] ; then mkdir $svnbase ; fi if [ ! -d $svnbase ] ; then mkdir $svnbase ; fi
...@@ -155,19 +150,21 @@ function build() { ...@@ -155,19 +150,21 @@ function build() {
done done
popd popd
tracefile=$builddir/build+$productname-$CC-$BDB+$nodename+$system+$release+$arch tracefile=$builddir/ft+$productname+$ftcc-$GCCVERSION+bdb-$BDB+$nodename+$system+$release+$arch
if [ $debugtests != 0 ] ; then tracefile=$tracefile+debug; fi
if [ $releasetests != 0 ] ; then tracefile=$tracefile+release; fi
# get some config info # get some config info
uname -a >>$tracefile 2>&1 uname -a >$tracefile 2>&1
$CC -v >>$tracefile 2>&1 ulimit -a >>$tracefile 2>&1
$CXX -v >>$tracefile 2>&1 $ftcc -v >>$tracefile 2>&1
if [ -f /proc/version ] ; then cat /proc/version >>$tracefile 2>&1
cat /proc/version >>$tracefile cat /proc/cpuinfo >>$tracefile 2>&1
fi
env >>$tracefile 2>&1 env >>$tracefile 2>&1
# checkout the source dir # checkout the source dir
productbuilddir=$svnbase/$productname productbuilddir=$svnbase/$productname
# cleanup # cleanup
rm -rf $productbuilddir rm -rf $productbuilddir
mkdir -p $productbuilddir mkdir -p $productbuilddir
...@@ -178,83 +175,105 @@ function build() { ...@@ -178,83 +175,105 @@ function build() {
# checkout into $productbuilddir # checkout into $productbuilddir
runcmd 0 $productbuilddir retry svn checkout -q -r $revision $svnserver/$checkout . >>$tracefile 2>&1 runcmd 0 $productbuilddir retry svn checkout -q -r $revision $svnserver/$checkout . >>$tracefile 2>&1
# portability if [ $debugtests != 0 ] ; then
runcmd 0 $productbuilddir/$oschoice make local -k -s >>$tracefile 2>&1
runcmd 0 $productbuilddir/$oschoice/tests make -k -s >>$tracefile 2>&1 # make debug
runcmd 0 $productbuilddir/$oschoice/tests make check -k -j$makejobs -s SUMMARIZE=1 >>$tracefile 2>&1 eval runcmd 0 $productbuilddir make release DEBUG=1 CC=$ftcc HAVE_CILK=$have_cilk >>$tracefile 2>&1
eval runcmd 0 $productbuilddir/utils make -j$makejobs DEBUG=1 CC=$ftcc HAVE_CILK=$have_cilk >>$tracefile 2>&1
# newbrt eval runcmd 0 $productbuilddir/newbrt/tests make -j$makejobs DEBUG=1 CC=$ftcc HAVE_CILK=$have_cilk >>$tracefile 2>&1
runcmd 0 $productbuilddir/newbrt make local -k -s SUMMARIZE=1 >>$tracefile 2>&1 eval runcmd 0 $productbuilddir/src/tests make tests -j$makejobs -k -s SUMMARIZE=1 DEBUG=1 CC=$ftcc HAVE_CILK=$have_cilk >>$tracefile 2>&1
if [ $dovalgrind -ne 0 ] ; then
runcmd 0 $productbuilddir/newbrt make check -j$makejobs -k -s SUMMARIZE=1 >>$tracefile 2>&1 # debug tests
else eval runcmd 0 $productbuilddir/$system/tests make check -k -s SUMMARIZE=1 DEBUG=1 CC=$ftcc HAVE_CILK=$have_cilk >>$tracefile 2>&1
runcmd 0 $productbuilddir/newbrt make check -j$makejobs -k -s SUMMARIZE=1 VGRIND="" >>$tracefile 2>&1 eval runcmd 0 $productbuilddir/utils make check -k -j$makejobs -s SUMMARIZE=1 DEBUG=1 CC=$ftcc HAVE_CILK=$have_cilk >>$tracefile 2>&1
range_trace=$(my_mktemp range)
eval runcmd 0 $productbuilddir/src/range_tree/tests make check -k -j$makejobs -s SUMMARIZE=1 CC=$ftcc DEBUG=1 HAVE_CILK=$have_cilk VGRIND= >>$range_trace 2>&1 $BG
lock_trace=$(my_mktemp lock)
eval runcmd 0 $productbuilddir/src/lock_tree/tests make check -k -j$makejobs -s SUMMARIZE=1 CC=$ftcc DEBUG=1 HAVE_CILK=$have_cilk VGRIND= >>$lock_trace 2>&1 $BG
wait
cat $range_trace >>$tracefile; rm $range_trace
cat $lock_trace >>$tracefile; rm $lock_trace
eval runcmd 0 $productbuilddir/newbrt make check -j$n -k -s SUMMARIZE=1 DEBUG=1 CC=$ftcc HAVE_CILK=$have_cilk >>$tracefile 2>&1
eval runcmd 0 $productbuilddir/src/tests make check.tdb -j$n -k -s SUMMARIZE=1 DEBUG=1 CC=$ftcc HAVE_CILK=$have_cilk >>$tracefile 2>&1
# brtloader tests
newbrt_loader_trace=$(my_mktemp newbrt_loader)
eval runcmd 0 $productbuilddir/newbrt/tests make check_brtloader -k -j$makejobs -s SUMMARIZE=1 DEBUG=1 CC=$ftcc HAVE_CILK=$have_cilk >>$newbrt_loader_trace 2>&1 $BG
loader_trace=$(my_mktemp loader)
eval runcmd 0 $productbuilddir/src/tests make loader-tests -k -j$makejobs -s SUMMARIZE=1 DEBUG=1 CC=$ftcc HAVE_CILK=$have_cilk >>$loader_trace 2>&1 $BG
# stress tests
stress_trace=$(my_mktemp stress)
eval runcmd 0 $productbuilddir/src/tests make stress_tests.tdbrun -j$makejobs -k -s SUMMARIZE=1 DEBUG=1 CC=$ftcc HAVE_CILK=$have_cilk >>$stress_trace 2>&1 $BG
wait
cat $newbrt_loader_trace >>$tracefile; rm $newbrt_loader_trace
cat $loader_trace >>$tracefile; rm $loader_trace
cat $stress_trace >>$tracefile; rm $stress_trace
# benchmark tests
eval runcmd 0 $productbuilddir/db-benchmark-test make -k -j$makejobs DEBUG=1 CC=$ftcc HAVE_CILK=$have_cilk >>$tracefile 2>&1
eval runcmd 0 $productbuilddir/db-benchmark-test make check -k -j$makejobs -k -s SUMMARIZE=1 DEBUG=1 CC=$ftcc HAVE_CILK=$have_cilk >>$tracefile 2>&1
# upgrade tests
if [ $upgradetests != 0 ] ; then
runcmd 0 $productbuilddir/src/tests make upgrade-tests.tdbrun -k -s SUMMARIZE=1 DEBUG=1 CC=$ftcc HAVE_CILK=$have_cilk >>$tracefile 2>&1
fi
# drd tests
fi fi
# lock tree if [ $releasetests != 0 ] ; then
runcmd 0 $productbuilddir/src/range_tree make -k -s >>$tracefile 2>&1
runcmd 0 $productbuilddir/src/lock_tree make -k -s >>$tracefile 2>&1 # release build
runcmd 0 $productbuilddir/src/range_tree/tests make check -k -j$makejobs -s SUMMARIZE=1 >>$tracefile 2>&1 eval runcmd 0 $productbuilddir make clean >>$tracefile 2>&1
runcmd 0 $productbuilddir/src/lock_tree/tests make check -k -j$makejobs -s SUMMARIZE=1 >>$tracefile 2>&1 eval runcmd 0 $productbuilddir make release CC=$ftcc HAVE_CILK=$have_cilk >>$tracefile 2>&1
eval runcmd 0 $productbuilddir/utils make -j$makejobs CC=$ftcc HAVE_CILK=$have_cilk >>$tracefile 2>&1
# src
runcmd 0 $productbuilddir/src make local -k -s >>$tracefile 2>&1 # release tests
runcmd $dowindows $productbuilddir/src make check_globals >>$tracefile 2>&1 eval runcmd 0 $productbuilddir/$system/tests make check -k -s SUMMARIZE=1 CC=$ftcc HAVE_CILK=$have_cilk VGRIND= >>$tracefile 2>&1
runcmd 0 $productbuilddir/src/tests make -k -s -j$makejobs >>$tracefile 2>&1 eval runcmd 0 $productbuilddir/utils make check -k -j$makejobs -s SUMMARIZE=1 CC=$ftcc HAVE_CILK=$have_cilk VGRIND= >>$tracefile 2>&1
range_trace=$(my_mktemp range)
# utils eval runcmd 0 $productbuilddir/src/range_tree/tests make check -k -j$makejobs -s SUMMARIZE=1 CC=$ftcc HAVE_CILK=$have_cilk VGRIND= >>$range_trace 2>&1 $BG
runcmd 0 $productbuilddir/utils make -k -s >>$tracefile 2>&1 lock_trace=$(my_mktemp lock)
runcmd 0 $productbuilddir/utils make check -k -j$makejobs -s SUMMARIZE=1 >>$tracefile 2>&1 eval runcmd 0 $productbuilddir/src/lock_tree/tests make check -k -j$makejobs -s SUMMARIZE=1 CC=$ftcc HAVE_CILK=$have_cilk VGRIND= >>$lock_trace 2>&1 $BG
wait
# src/tests cat $range_trace >>$tracefile; rm $range_trace
runcmd 0 $productbuilddir/src/tests make check.bdb -j$makejobs -k -s SUMMARIZE=1 VGRIND="" BDB_SUPPRESSIONS="" >>$tracefile 2>&1 cat $lock_trace >>$tracefile; rm $lock_trace
if [ $dovalgrind -ne 0 ] ; then eval runcmd 0 $productbuilddir/newbrt/tests make check -j$makejobs -k -s SUMMARIZE=1 CC=$ftcc HAVE_CILK=$have_cilk VGRIND= >>$tracefile 2>&1
runcmd 0 $productbuilddir/src/tests make check.tdb -j$makejobs -k -s SUMMARIZE=1 >>$tracefile 2>&1 eval runcmd 0 $productbuilddir/newbrt/tests make check_brtloader -k -j$makejobs -s SUMMARIZE=1 CC=$ftcc HAVE_CILK=$have_cilk VGRIND= >>$tracefile 2>&1
else eval runcmd 0 $productbuilddir/src/tests make check.tdb -j$makejobs -k -s SUMMARIZE=1 CC=$ftcc HAVE_CILK=$have_cilk VGRIND= >>$tracefile 2>&1
runcmd 0 $productbuilddir/src/tests make check.tdb -j$makejobs -k -s SUMMARIZE=1 VGRIND="" >>$tracefile 2>&1 eval runcmd 0 $productbuilddir/src/tests make stress_tests.tdbrun -j$makejobs -k -s SUMMARIZE=1 CC=$ftcc HAVE_CILK=$have_cilk VGRIND= >>$tracefile 2>&1
fi fi
# benchmark tests # cilk tests
runcmd 0 $productbuilddir/db-benchmark-test make -k -j$makejobs >>$tracefile 2>&1 if [ $cilktests != 0 ] ; then
runcmd 0 $productbuilddir/db-benchmark-test make check -j$makejobs -k >>$tracefile 2>&1 runcmd 0 $productbuilddir make clean >>$tracefile 2>&1
runcmd 0 $productbuilddir make release CC=$ftcc DEBUG=1 >>$tracefile 2>&1
runcmd 0 $productbuilddir/newbrt/tests make cilkscreen_brtloader -k -s SUMMARIZE=1 CC=$ftcc DEBUG=1 >>$tracefile 2>&1
fi
# cxx # cxx
if [ $cxxtests != 0 ] ; then
runcmd $dowindows $productbuilddir/cxx make -k -s >>$tracefile 2>&1 runcmd $dowindows $productbuilddir/cxx make -k -s >>$tracefile 2>&1
runcmd $dowindows $productbuilddir/cxx make -k -s install >>$tracefile 2>&1 runcmd $dowindows $productbuilddir/cxx make -k -s install >>$tracefile 2>&1
runcmd $dowindows $productbuilddir/cxx/tests make -k -s check SUMMARIZE=1 >>$tracefile 2>&1 runcmd $dowindows $productbuilddir/cxx/tests make -k -s check SUMMARIZE=1 >>$tracefile 2>&1
runcmd $dowindows $productbuilddir/db-benchmark-test-cxx make -k -s >>$tracefile 2>&1 runcmd $dowindows $productbuilddir/db-benchmark-test-cxx make -k -s >>$tracefile 2>&1
runcmd $dowindows $productbuilddir/db-benchmark-test-cxx make -k -s check >>$tracefile 2>&1 runcmd $dowindows $productbuilddir/db-benchmark-test-cxx make -k -s check >>$tracefile 2>&1
fi
# Makefile for release/examples is NOT ported to windows. Expect it to fail. # Makefile for release/examples is NOT ported to windows. Expect it to fail.
if [ 0 = 1 ] ; then
runcmd $dowindows $productbuilddir/release make -k setup >>$tracefile 2>&1 runcmd $dowindows $productbuilddir/release make -k setup >>$tracefile 2>&1
runcmd $dowindows $productbuilddir/release/examples make -k check >>$tracefile 2>&1 runcmd $dowindows $productbuilddir/release/examples make -k check >>$tracefile 2>&1
# man
if [ $dowindows -eq 0 ] ; then
runcmd 0 $productbuilddir/man/texi make -k -s -j$makejobs >>$tracefile 2>&1
else
# Don't use XFAIL because on cygwin man/texi just hangs
runcmd 0 $productbuilddir/man/texi echo SKIPPED make -k -s -j$makejobs >>$tracefile 2>&1
fi fi
# debug build # man
runcmd 0 $productbuilddir make clean -s >>$tracefile 2>&1 if [ 0 = 1 ] ; then
runcmd 0 $productbuilddir make release -s DEBUG=1 >>$tracefile 2>&1 runcmd 0 $productbuilddir/man/texi make -k -s -j$makejobs >>$tracefile 2>&1
# run the brtloader tests with a debug build
runcmd 0 $productbuilddir/newbrt/tests make check_brtloader -k -j$makejobs DEBUG=1 -s SUMMARIZE=1 >>$tracefile 2>&1
runcmd 0 $productbuilddir/src/tests make loader-tests -k -j$makejobs DEBUG=1 -s SUMMARIZE=1 >>$tracefile 2>&1
if [ $dowindows -eq 0 ] ; then
# cilk debug build
runcmd 0 $productbuilddir make clean -s >>$tracefile 2>&1
runcmd 0 $productbuilddir make release -s DEBUG=1 BRTLOADER=cilk >>$tracefile 2>&1
# run the loader tests with cilkscreen
runcmd 0 $productbuilddir/newbrt/tests make cilkscreen_brtloader DEBUG=1 BRTLOADER=cilk -s SUMMARIZE=1 >>$tracefile 2>&1
fi fi
if [ $docoverage -eq 1 ] ; then if [ 0 = 1 -a $docoverage -eq 1 ] ; then
# run coverage # run coverage
runcmd 0 $productbuilddir make -s clean >>$tracefile 2>&1 runcmd 0 $productbuilddir make -s clean >>$tracefile 2>&1
runcmd 0 $productbuilddir make build-coverage >>$tracefile 2>&1 runcmd 0 $productbuilddir make build-coverage >>$tracefile 2>&1
...@@ -262,7 +281,7 @@ function build() { ...@@ -262,7 +281,7 @@ function build() {
# summarize the coverage data # summarize the coverage data
coveragefile=$builddir/coverage+$productname-$BDB+$nodename+$system+$release+$arch coveragefile=$builddir/coverage+$productname-$BDB+$nodename+$system+$release+$arch
rawcoverage=`mktemp` rawcoverage=$(my_mktemp ftcover)
for d in newbrt src src/range_tree src/lock_tree; do for d in newbrt src src/range_tree src/lock_tree; do
(cd $productbuilddir/$d; python ~/bin/gcovsumdir.py -b *.gcno >>$rawcoverage) (cd $productbuilddir/$d; python ~/bin/gcovsumdir.py -b *.gcno >>$rawcoverage)
if [ -d $productbuilddir/$d/tests ] ; then if [ -d $productbuilddir/$d/tests ] ; then
...@@ -274,14 +293,17 @@ function build() { ...@@ -274,14 +293,17 @@ function build() {
fi fi
# put the trace into svn # put the trace into svn
if [ $docommit -ne 0 ] ; then if [ $commit -ne 0 ] ; then
testresult="PASS=$npass" testresult="PASS=$npass"
if [ $nfail -ne 0 ] ; then testresult="FAIL=$nfail $testresult"; fi if [ $nfail != 0 ] ; then
testresult="FAIL=$nfail $testresult"
fi
local cf=`mktemp` local cf=$(my_mktemp ftresult)
echo "$testresult tokudb-build $productname-$BDB $system $release $arch $nodename" >$cf echo "$testresult tokudb-build $productname $CC $GCCVERSION $system $release $arch $nodename" >$cf
echo >>$cf; echo >>$cf echo >>$cf; echo >>$cf
cat $commit_msg >>$cf cat $commit_msg >>$cf
if [ $nfail != 0 ] ; then egrep " FAIL" $tracefile >>$cf; fi
svn add $tracefile $coveragefile svn add $tracefile $coveragefile
retry svn commit -F "$cf" $tracefile $coveragefile retry svn commit -F "$cf" $tracefile $coveragefile
...@@ -289,11 +311,7 @@ function build() { ...@@ -289,11 +311,7 @@ function build() {
else else
cat $commit_msg cat $commit_msg
fi fi
rm $commit_msg rm $commit_msg
if [ $deleteafter -eq 1 ] ; then
rm -rfv $productbuilddir > /dev/null #windows rm sometimes hangs on giant dirs without -v
fi
return 0 return 0
} }
...@@ -302,7 +320,7 @@ function build() { ...@@ -302,7 +320,7 @@ function build() {
exitcode=0 exitcode=0
svnserver=https://svn.tokutek.com/tokudb svnserver=https://svn.tokutek.com/tokudb
nodename=`uname -n` nodename=`uname -n`
system=`uname -s | sanitize` system=`uname -s | tr '[:upper:]' '[:lower:]' | sanitize`
release=`uname -r | sanitize` release=`uname -r | sanitize`
arch=`uname -m | sanitize` arch=`uname -m | sanitize`
date=`date +%Y%m%d` date=`date +%Y%m%d`
...@@ -311,51 +329,30 @@ tokudb="tokudb" ...@@ -311,51 +329,30 @@ tokudb="tokudb"
bdb="4.6" bdb="4.6"
makejobs=`get_ncpus` makejobs=`get_ncpus`
revision=0 revision=0
dovalgrind=1 VALGRIND=tokugrind
VALGRIND=valgrind commit=1
docommit=1
docoverage=0 docoverage=0
dowindows=0 ftcc=gcc
oschoice=linux have_cilk=0
deleteafter=0 have_poly=1
j=-1 debugtests=1
cc=gcc44 releasetests=1
cxx=g++44 upgradetests=0
cilktests=0
arg=$1; cxxtests=0
shopt -s compat31 #Necessary in some flavors of linux and windows parallel=0
if [ "$arg" = "--windows=yes" -o "$arg" = "--windows=1" ] ; then
shift
dowindows=1
export CC=icc
export CXX=icc
export CYGWIN=CYGWIN
oschoice=windows
fi
# import the environment
while [ $# -gt 0 ] ; do while [ $# -gt 0 ] ; do
arg=$1; shift arg=$1; shift
if [ "$arg" = "--valgrind=no" -o "$arg" = "--valgrind=0" ] ; then if [[ $arg =~ ^--(.*)=(.*) ]] ; then
dovalgrind=0
elif [ "$arg" = "--commit=no" -o "$arg" = "--commit=0" ] ; then
docommit=0
elif [ "$arg" = "--windows=no" -o "$arg" = "--windows=0" ] ; then
dowindows=0
elif [ "$arg" = "--windows=yes" -o "$arg" = "--windows=1" ] ; then
usage; exit 1
elif [[ $arg =~ ^--(.*)=(.*) ]] ; then
eval ${BASH_REMATCH[1]}=${BASH_REMATCH[2]} eval ${BASH_REMATCH[1]}=${BASH_REMATCH[2]}
else else
usage; exit 1 usage; exit 1
fi fi
done done
# setup default compiler # setup intel compiler env
if [ "$CC" = "" ] ; then export CC=$cc; fi if [ $ftcc = icc ] ; then
if [ "$CXX" = "" ] ; then export CXX=$cxx; fi
if [ $CC = icc ] ; then
d=/opt/intel/bin d=/opt/intel/bin
if [ -d $d ] ; then if [ -d $d ] ; then
export PATH=$d:$PATH export PATH=$d:$PATH
...@@ -367,17 +364,17 @@ if [ $CC = icc ] ; then ...@@ -367,17 +364,17 @@ if [ $CC = icc ] ; then
fi fi
fi fi
if [ $j -ne "-1" ] ; then makejobs=$j; fi
if [ $makejobs -eq 0 ] ; then usage; exit 1; fi
if [ $branch = "." ] ; then branch="toku"; fi if [ $branch = "." ] ; then branch="toku"; fi
if [ $revision -eq 0 ] ; then revision=`get_latest_svn_revision`; fi if [ $revision -eq 0 ] ; then revision=`get_latest_svn_revision`; fi
if [ $parallel -ne 0 ] ; then BG="&"; fi
# setup GCCVERSION # setup GCCVERSION
export GCCVERSION=`$CC --version|head -1|cut -f3 -d" "` export GCCVERSION=`$ftcc --version|head -1|cut -f3 -d" "`
export VALGRIND=$VALGRIND export VALGRIND=$VALGRIND
# setup VGRIND # limit execution time to 3 hours
if [ $dovalgrind = 0 ] ; then export VGRIND=""; fi let t=3*3600
ulimit -t $t
ulimit -c unlimited
build $bdb build $bdb
scripts/build.check.icc.bash
View file @ ea0ba57f
#!/bin/bash #!/usr/bin/env bash
build.check.bash --ftcc=icc $*
# build.check.bash --revision=10500
# build.check.bash --tokudb=tokudb.1489 --revision=10500
# build.check.bash --branch=mysql.branches/1.1.3 --revision=10500
# build.check.bash --windows=1 --revision=10500
function usage() {
echo "build tokudb and run regressions"
echo "--windows=$dowindows (if yes/1 must be first option)"
echo "--branch=$branch --tokudb=$tokudb --revision=$revision --bdb=$bdb"
echo "--cc=$cc --cxx=$cxx --valgrind=$dovalgrind --VALGRIND=$VALGRIND"
echo "--j=$makejobs"
echo "--debugtests=$debugtests --releasetests=$releasetests --deleteafter=$deleteafter --doclean=$doclean"
echo "--commit=$docommit"
}
function retry() {
local cmd
local retries
local exitcode
cmd=$*
let retries=0
while [ $retries -le 10 ] ; do
echo `date` $cmd
bash -c "$cmd"
exitcode=$?
echo `date` $cmd $exitcode $retries
let retries=retries+1
if [ $exitcode -eq 0 ] ; then break; fi
sleep 10
done
test $exitcode = 0
}
# delete some characters that cygwin and osx have trouble with
function sanitize() {
tr -d '[/:\\\\()]'
}
function mydate() {
date +"%Y%m%d %H:%M:%S"
}
function get_ncpus() {
grep bogomips /proc/cpuinfo | wc -l
}
function get_latest_svn_revision() {
local revision=0
local svntarget=$svnserver/$*
local latest=`retry svn info $svntarget`
if [[ $latest =~ "Last Changed Rev: ([0-9]+)" ]] ; then
revision=${BASH_REMATCH[1]}
fi
echo $revision
}
function make_tokudb_name() {
local tokudb_dir=$1
local tokudb=$2
if [ $tokudb_dir = "toku" ] ; then
echo $tokudb
else
echo `echo $tokudb_dir-$tokudb | tr / -`
fi
}
# run a command and trace its result
function runcmd() {
local fail=$1; shift
local makedir=$1; shift
local cmd=$*
echo `mydate` $makedir $cmd
pushd $makedir
$cmd
exitcode=$?
local dir=$makedir
if [[ $dir =~ "$HOME/svn.build/(.*)" ]] ; then
dir=${BASH_REMATCH[1]}
fi
if [ $fail -eq 0 ] ; then
if [ $exitcode -eq 0 ] ; then
result="PASS `mydate` $dir $cmd"
let npass=npass+1
else
result="FAIL `mydate` $dir $cmd"
let nfail=nfail+1
fi
else
if [ $exitcode -eq 0 ] ; then
result="XPASS `mydate` $dir $cmd"
else
result="XFAIL `mydate` $dir $cmd"
fi
fi
echo $result
echo $result >>$commit_msg
echo $result >>/tmp/tokubuild.trace
popd
}
# build a version of tokudb with a specific BDB target
function build() {
# setup build environment
export BDB=$1
if [[ $BDB =~ "(.*)\.(.*)" ]] ;then
export BDBMAJOR=${BASH_REMATCH[1]}
export BDBMINOR=${BASH_REMATCH[2]}
else
return 1
fi
if [ $dowindows -eq 0 ] ; then
export BDBDIR=/usr/local/BerkeleyDB.$BDB
else
export BDBDIR=c:/cygwin/usr/local/BerkeleyDB.$BDB
fi
if [ ! -d $BDBDIR ] ; then return 2; fi
tokudb_name=`make_tokudb_name $branch $tokudb`
export TOKUDB_NAME=$tokudb_name
export TOKUDB_REVISION=$revision
productname=$tokudb_name-$revision
checkout=$branch/$tokudb
latestrev=`get_latest_svn_revision $checkout`
if [ $latestrev -eq 0 ] ; then return 3; fi
commit_msg=`mktemp`
svnbase=~/svn.build
if [ ! -d $svnbase ] ; then mkdir $svnbase ; fi
# checkout the build dir
buildbase=$svnbase/tokudb.build
if [ ! -d $buildbase ] ; then
mkdir $buildbase
fi
# make the build directory, possibly on multiple machines simultaneously, there can be only one
builddir=$buildbase/$date
pushd $buildbase
while [ ! -d $date ] ; do
svn mkdir $svnserver/tokudb.build/$date -m ""
svn co -q $svnserver/tokudb.build/$date
if [ $? -ne 0 ] ; then rm -rf $date; fi
done
popd
tracefile=$builddir/build+$productname-$CC-$BDB+$nodename+$system+$release+$arch
# get some config info
uname -a >>$tracefile 2>&1
ulimit -a >>$tracefile 2>&1
$CC -v >>$tracefile 2>&1
$CXX -v >>$tracefile 2>&1
cat /proc/version >>$tracefile 2>&1
cat /proc/cpuinfo >>$tracefile 2>&1
env >>$tracefile 2>&1
# checkout the source dir
productbuilddir=$svnbase/$productname
# cleanup
rm -rf $productbuilddir
mkdir -p $productbuilddir
let nfail=0
let npass=0
# checkout into $productbuilddir
runcmd 0 $productbuilddir retry svn checkout -q -r $revision $svnserver/$checkout . >>$tracefile 2>&1
if [ $debugtests != 0 ] ; then
# portability
runcmd 0 $productbuilddir/$oschoice make local DEBUG=1 CC=icc HAVE_CILK=0 >>$tracefile 2>&1
runcmd 0 $productbuilddir/$oschoice/tests make check -k -s SUMMARIZE=1 DEBUG=1 CC=icc HAVE_CILK=0 >>$tracefile 2>&1
# newbrt
runcmd 0 $productbuilddir/newbrt make local -j$makejobs -k -s SUMMARIZE=1 DEBUG=1 CC=icc HAVE_CILK=0 >>$tracefile 2>&1
runcmd 0 $productbuilddir/newbrt make check -j$makejobs -k -s SUMMARIZE=1 DEBUG=1 CC=icc HAVE_CILK=0 >>$tracefile 2>&1
# src
runcmd 0 $productbuilddir/src make local -k -s -j$makejobs DEBUG=1 CC=icc HAVE_CILK=0 >>$tracefile 2>&1
# utils
runcmd 0 $productbuilddir/utils make -k -s -j$makejobs DEBUG=1 CC=icc HAVE_CILK=0 >>$tracefile 2>&1
runcmd 0 $productbuilddir/utils make check -k -j$makejobs -s SUMMARIZE=1 DEBUG=1 CC=icc HAVE_CILK=0 >>$tracefile 2>&1
# lock tree
runcmd 0 $productbuilddir/src/range_tree/tests make check -k -j$makejobs -s SUMMARIZE=1 DEBUG=1 CC=icc HAVE_CILK=0 >>$tracefile 2>&1
runcmd 0 $productbuilddir/src/lock_tree/tests make check -k -j$makejobs -s SUMMARIZE=1 DEBUG=1 CC=icc HAVE_CILK=0 >>$tracefile 2>&1
# src/tests
runcmd 0 $productbuilddir/src/tests make tests.bdb -j$makejobs -k -s SUMMARIZE=1 DEBUG=1 CC=icc HAVE_CILK=0 >>$tracefile 2>&1
runcmd 0 $productbuilddir/src/tests make tests.tdb -j$makejobs -k -s SUMMARIZE=1 DEBUG=1 CC=icc HAVE_CILK=0 >>$tracefile 2>&1
runcmd 0 $productbuilddir/src/tests make check.tdb -j$makejobs -k -s SUMMARIZE=1 DEBUG=1 CC=icc HAVE_CILK=0 >>$tracefile 2>&1
runcmd 0 $productbuilddir/src/tests make stress_tests.tdbrun -j$makejobs -k -s SUMMARIZE=1 DEBUG=1 CC=icc HAVE_CILK=0 >>$tracefile 2>&1
# benchmark tests
runcmd 0 $productbuilddir/db-benchmark-test make -k -j$makejobs DEBUG=1 CC=icc HAVE_CILK=0 >>$tracefile 2>&1
runcmd 0 $productbuilddir/db-benchmark-test make check -k -j$makejobs -k -s SUMMARIZE=1 DEBUG=1 CC=icc HAVE_CILK=0 >>$tracefile 2>&1
# run the brtloader tests with a debug build
runcmd 0 $productbuilddir/newbrt/tests make check_brtloader -k -j$makejobs -s SUMMARIZE=1 DEBUG=1 CC=icc HAVE_CILK=0 >>$tracefile 2>&1
runcmd 0 $productbuilddir/src/tests make loader-tests -k -j$makejobs -s SUMMARIZE=1 DEBUG=1 CC=icc HAVE_CILK=0 >>$tracefile 2>&1
# upgrade tests
if [ $upgradetests != 0 ] ; then
runcmd 0 $productbuilddir/src/tests make upgrade-tests.tdbrun -k -s SUMMARIZE=1 DEBUG=1 CC=icc HAVE_CILK=0 >>$tracefile 2>&1
fi
fi
if [ $releasetests != 0 ] ; then
# release build
runcmd 0 $productbuilddir make clean >>$tracefile 2>&1
runcmd 0 $productbuilddir make release CC=icc HAVE_CILK=0 >>$tracefile 2>&1
runcmd 0 $productbuilddir/utils make CC=icc HAVE_CILK=0 >>$tracefile 2>&1
# release tests
runcmd 0 $productbuilddir/newbrt/tests make check -j$makejobs -k -s SUMMARIZE=1 CC=icc HAVE_CILK=0 VGRIND= >>$tracefile 2>&1
runcmd 0 $productbuilddir/newbrt/tests make check_brtloader -k -j$makejobs -s SUMMARIZE=1 CC=icc HAVE_CILK=0 VGRIND= >>$tracefile 2>&1
runcmd 0 $productbuilddir/src/tests make tests.tdb -j$makejobs -k -s SUMMARIZE=1 CC=icc HAVE_CILK=0 >>$tracefile 2>&1
runcmd 0 $productbuilddir/src/tests make check.tdb -j$makejobs -k -s SUMMARIZE=1 CC=icc HAVE_CILK=0 VGRIND= >>$tracefile 2>&1
runcmd 0 $productbuilddir/src/tests make stress_tests.tdbrun -j$makejobs -k -s SUMMARIZE=1 CC=icc HAVE_CILK=0 VGRIND= >>$tracefile 2>&1
fi
# cilk tests
if [ $cilktests != 0 ] ; then
runcmd 0 $productbuilddir make clean >>$tracefile 2>&1
runcmd 0 $productbuilddir make release CC=icc DEBUG=1 >>$tracefile 2>&1
runcmd 0 $productbuilddir/newbrt/tests make cilkscreen_brtloader -k -s SUMMARIZE=1 CC=icc DEBUG=1 >>$tracefile 2>&1
fi
# cxx
if [ $cxxtests != 0 ] ; then
runcmd $dowindows $productbuilddir/cxx make -k -s >>$tracefile 2>&1
runcmd $dowindows $productbuilddir/cxx make -k -s install >>$tracefile 2>&1
runcmd $dowindows $productbuilddir/cxx/tests make -k -s check SUMMARIZE=1 >>$tracefile 2>&1
runcmd $dowindows $productbuilddir/db-benchmark-test-cxx make -k -s >>$tracefile 2>&1
runcmd $dowindows $productbuilddir/db-benchmark-test-cxx make -k -s check >>$tracefile 2>&1
fi
# Makefile for release/examples is NOT ported to windows. Expect it to fail.
if [ 0 = 1 ] ; then
runcmd $dowindows $productbuilddir/release make -k setup >>$tracefile 2>&1
runcmd $dowindows $productbuilddir/release/examples make -k check >>$tracefile 2>&1
fi
# man
if [ 0 = 1 ] ; then
runcmd 0 $productbuilddir/man/texi make -k -s -j$makejobs >>$tracefile 2>&1
fi
if [ 0 = 1 -a $docoverage -eq 1 ] ; then
# run coverage
runcmd 0 $productbuilddir make -s clean >>$tracefile 2>&1
runcmd 0 $productbuilddir make build-coverage >>$tracefile 2>&1
runcmd 0 $productbuilddir make check-coverage >>$tracefile 2>&1
# summarize the coverage data
coveragefile=$builddir/coverage+$productname-$BDB+$nodename+$system+$release+$arch
rawcoverage=`mktemp`
for d in newbrt src src/range_tree src/lock_tree; do
(cd $productbuilddir/$d; python ~/bin/gcovsumdir.py -b *.gcno >>$rawcoverage)
if [ -d $productbuilddir/$d/tests ] ; then
(cd $productbuilddir/$d/tests; python ~/bin/gcovsumdir.py -b *.gcno >>$rawcoverage)
fi
done
python ~/bin/gcovsumsum.py <$rawcoverage >$coveragefile
rm $rawcoverage
fi
# put the trace into svn
if [ $docommit -ne 0 ] ; then
testresult="PASS=$npass"
if [ $nfail != 0 ] ; then
testresult="FAIL=$nfail $testresult"
fi
local cf=`mktemp`
echo "$testresult tokudb-build $productname $CC $GCCVERSION $system $release $arch $nodename" >$cf
echo >>$cf; echo >>$cf
cat $commit_msg >>$cf
if [ $nfail != 0 ] ; then egrep " FAIL" $tracefile >>$cf; fi
svn add $tracefile $coveragefile
retry svn commit -F "$cf" $tracefile $coveragefile
rm $cf
else
cat $commit_msg
fi
rm $commit_msg
if [ $deleteafter -eq 1 ] ; then
rm -rfv $productbuilddir > /dev/null #windows rm sometimes hangs on giant dirs without -v
fi
return 0
}
# set defaults
exitcode=0
svnserver=https://svn.tokutek.com/tokudb
nodename=`uname -n`
system=`uname -s | sanitize`
release=`uname -r | sanitize`
arch=`uname -m | sanitize`
date=`date +%Y%m%d`
branch="."
tokudb="tokudb"
bdb="4.6"
makejobs=`get_ncpus`
revision=0
dovalgrind=1
VALGRIND=tokugrind
docommit=1
docoverage=0
dowindows=0
oschoice=linux
deleteafter=0
j=-1
cc=icc
cxx=icpc
debugtests=1
releasetests=1
upgradetests=0
cilktests=0
cxxtests=0
arg=$1;
shopt -s compat31 #Necessary in some flavors of linux and windows
if [ "$arg" = "--windows=yes" -o "$arg" = "--windows=1" ] ; then
shift
dowindows=1
export CC=icc
export CXX=icc
export CYGWIN=CYGWIN
oschoice=windows
fi
# import the environment
while [ $# -gt 0 ] ; do
arg=$1; shift
if [ "$arg" = "--valgrind=no" -o "$arg" = "--valgrind=0" ] ; then
dovalgrind=0
elif [ "$arg" = "--commit=no" -o "$arg" = "--commit=0" ] ; then
docommit=0
elif [ "$arg" = "--windows=no" -o "$arg" = "--windows=0" ] ; then
dowindows=0
elif [ "$arg" = "--windows=yes" -o "$arg" = "--windows=1" ] ; then
usage; exit 1
elif [[ $arg =~ ^--(.*)=(.*) ]] ; then
eval ${BASH_REMATCH[1]}=${BASH_REMATCH[2]}
else
usage; exit 1
fi
done
# setup default compiler
if [ "$CC" = "" ] ; then export CC=$cc; fi
if [ "$CXX" = "" ] ; then export CXX=$cxx; fi
if [ $CC = icc ] ; then
d=/opt/intel/bin
if [ -d $d ] ; then
export PATH=$d:$PATH
. $d/compilervars.sh intel64
fi
d=/opt/intel/cilkutil/bin
if [ -d $d ] ; then
export PATH=$d:$PATH
fi
fi
if [ $j -ne "-1" ] ; then makejobs=$j; fi
if [ $makejobs -eq 0 ] ; then usage; exit 1; fi
if [ $branch = "." ] ; then branch="toku"; fi
if [ $revision -eq 0 ] ; then revision=`get_latest_svn_revision`; fi
# setup GCCVERSION
export GCCVERSION=`$CC --version|head -1|cut -f3 -d" "`
export VALGRIND=$VALGRIND
# setup VGRIND
if [ $dovalgrind = 0 ] ; then export VGRIND=""; fi
# limit execution time to 3 hours
let t =3*3600
ulimit -t $t
ulimit -c unlimited
build $bdb
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7