#3752 script to run fractal tree tests with icc refs[t:3752]

git-svn-id: file:///svn/toku/tokudb@33394 c7de825b-a66e-492c-adef-691d508d4ae1

scripts/build.check.icc.bash

+#!/bin/bash
+
+# build.check.bash --revision=10500
+# build.check.bash --tokudb=tokudb.1489 --revision=10500
+# build.check.bash --branch=mysql.branches/1.1.3 --revision=10500
+# build.check.bash --windows=1 --revision=10500
+
+function usage() {
+    echo "build tokudb and run regressions"
+    echo "--windows=$dowindows (if yes/1 must be first option)"
+    echo "--branch=$branch"
+    echo "--tokudb=$tokudb"
+    echo "--revision=$revision"
+    echo "--bdb=$bdb"
+    echo "--valgrind=$dovalgrind"
+    echo "--commit=$docommit"
+    echo "--j=$makejobs"
+    echo "--deleteafter=$deleteafter"
+    echo "--doclean=$doclean"
+}
+
+function retry() {
+    local cmd
+    local retries
+    local exitcode
+    cmd=$*
+    let retries=0
+    while [ $retries -le 10 ] ; do
+	echo `date` $cmd
+	bash -c "$cmd"
+	exitcode=$?
+	echo `date` $cmd $exitcode $retries
+	let retries=retries+1
+	if [ $exitcode -eq 0 ] ; then break; fi
+	sleep 10
+    done
+    test $exitcode = 0
+}
+
+# delete some characters that cygwin and osx have trouble with
+function sanitize() {
+    tr -d '[/:\\\\()]'
+}
+
+function mydate() {
+    date +"%Y%m%d %H:%M:%S"
+}
+
+function get_ncpus() {
+    grep bogomips /proc/cpuinfo | wc -l
+}
+
+function get_latest_svn_revision() {
+    local revision=0
+    local svntarget=$svnserver/$*
+    local latest=`retry svn info $svntarget`
+    if [[ $latest =~ "Last Changed Rev: ([0-9]+)" ]] ; then
+        revision=${BASH_REMATCH[1]}
+    fi
+    echo $revision
+}
+
+function make_tokudb_name() {
+    local tokudb_dir=$1
+    local tokudb=$2
+    if [ $tokudb_dir = "toku" ] ; then
+	echo $tokudb
+    else
+	echo `echo $tokudb_dir-$tokudb | tr / -`
+    fi
+}
+
+# run a command and trace its result
+function runcmd() {
+    local fail=$1; shift
+    local makedir=$1; shift
+    local cmd=$*
+
+    echo `mydate` $makedir $cmd
+    pushd $makedir
+    $cmd
+    exitcode=$?
+    local dir=$makedir
+    if [[ $dir =~ "$HOME/svn.build/(.*)" ]] ; then
+	dir=${BASH_REMATCH[1]}
+    fi
+    if [ $fail -eq 0 ] ; then
+        if [ $exitcode -eq 0 ] ; then
+            result="PASS `mydate` $dir $cmd"
+        else
+            result="FAIL `mydate` $dir $cmd"
+            let nfail=nfail+1
+        fi
+    else
+        if [ $exitcode -eq 0 ] ; then
+            result="XPASS `mydate` $dir $cmd"
+            let nfail=nfail+1
+        else
+            result="XFAIL `mydate` $dir $cmd"
+        fi
+    fi
+    echo $result
+    echo $result >>$commit_msg
+    echo $result >>/tmp/tokubuild.trace
+    popd
+}
+
+# build a version of tokudb with a specific BDB target
+function build() {
+
+    # setup build environment
+    export BDB=$1
+    if [[ $BDB =~ "(.*)\.(.*)" ]] ;then
+        export BDBMAJOR=${BASH_REMATCH[1]}
+        export BDBMINOR=${BASH_REMATCH[2]}
+    else
+        return 1
+    fi
+    if [ $dowindows -eq 0 ] ; then
+        export BDBDIR=/usr/local/BerkeleyDB.$BDB
+    else
+        export BDBDIR=c:/cygwin/usr/local/BerkeleyDB.$BDB
+    fi
+    if [ ! -d $BDBDIR ] ; then return 2; fi
+
+    tokudb_name=`make_tokudb_name $branch $tokudb`
+    export TOKUDB_NAME=$tokudb_name
+    export TOKUDB_REVISION=$revision
+
+    productname=$tokudb_name-$revision
+    checkout=$branch/$tokudb
+
+    latestrev=`get_latest_svn_revision $checkout`
+    if [ $latestrev -eq 0 ] ; then return 3; fi
+
+    commit_msg=`mktemp`
+
+    svnbase=~/svn.build
+    if [ ! -d $svnbase ] ; then mkdir $svnbase ; fi
+
+    # checkout the build dir
+    buildbase=$svnbase/tokudb.build
+    if [ ! -d $buildbase ] ; then
+	mkdir $buildbase
+    fi
+
+    # make the build directory, possibly on multiple machines simultaneously, there can be only one
+    builddir=$buildbase/$date
+    pushd $buildbase
+    while [ ! -d $date ] ; do
+	svn mkdir $svnserver/tokudb.build/$date -m ""
+	svn co -q $svnserver/tokudb.build/$date
+	if [ $? -ne 0 ] ; then rm -rf $date; fi
+    done
+    popd
+
+    tracefile=$builddir/build+$productname-$CC-$BDB+$nodename+$system+$release+$arch
+
+    # get some config info
+    uname -a >>$tracefile 2>&1
+    $CC -v >>$tracefile 2>&1
+    $CXX -v >>$tracefile 2>&1
+    if [ -f /proc/version ] ; then
+	cat /proc/version >>$tracefile
+    fi
+    env >>$tracefile 2>&1
+
+    # checkout the source dir
+    productbuilddir=$svnbase/$productname
+    # cleanup
+    rm -rf $productbuilddir
+    mkdir -p $productbuilddir
+
+    let nfail=0
+
+    # checkout into $productbuilddir
+    runcmd 0 $productbuilddir retry svn checkout -q -r $revision $svnserver/$checkout . >>$tracefile 2>&1
+
+    # portability
+    runcmd 0 $productbuilddir/$oschoice       make local DEBUG=1 CC=icc HAVE_CILK=0 >>$tracefile 2>&1
+    runcmd 0 $productbuilddir/$oschoice/tests make check -k -s SUMMARIZE=1 DEBUG=1 CC=icc HAVE_CILK=0 >>$tracefile 2>&1
+
+    # newbrt
+    runcmd 0 $productbuilddir/newbrt make local -j$makejobs -k -s SUMMARIZE=1 DEBUG=1 CC=icc HAVE_CILK=0 >>$tracefile 2>&1
+    runcmd 0 $productbuilddir/newbrt make check -j$makejobs -k -s SUMMARIZE=1 DEBUG=1 CC=icc HAVE_CILK=0 >>$tracefile 2>&1
+
+    # src
+    runcmd 0 $productbuilddir/src make local -k -s -j$makejobs DEBUG=1 CC=icc HAVE_CILK=0  >>$tracefile 2>&1
+
+    # utils
+    runcmd 0 $productbuilddir/utils make -k -s -j$makejobs DEBUG=1 CC=icc HAVE_CILK=0 >>$tracefile 2>&1
+    runcmd 0 $productbuilddir/utils make check -k -j$makejobs -s SUMMARIZE=1 DEBUG=1 CC=icc HAVE_CILK=0 >>$tracefile 2>&1
+
+    # lock tree
+    runcmd 0 $productbuilddir/src/range_tree/tests make check -k -j$makejobs -s SUMMARIZE=1 DEBUG=1 CC=icc HAVE_CILK=0 >>$tracefile 2>&1
+    runcmd 0 $productbuilddir/src/lock_tree/tests  make check -k -j$makejobs -s SUMMARIZE=1 DEBUG=1 CC=icc HAVE_CILK=0 >>$tracefile 2>&1
+
+    # src/tests
+    runcmd 0 $productbuilddir/src/tests make tests.tdb -j$makejobs -k -s SUMMARIZE=1 DEBUG=1 CC=icc HAVE_CILK=0 >>$tracefile 2>&1
+    runcmd 0 $productbuilddir/src/tests make check.tdb -j$makejobs -k -s SUMMARIZE=1 DEBUG=1 CC=icc HAVE_CILK=0 >>$tracefile 2>&1
+
+    # benchmark tests
+    runcmd 0 $productbuilddir/db-benchmark-test make -k -j$makejobs DEBUG=1 CC=icc HAVE_CILK=0 >>$tracefile 2>&1
+    runcmd 0 $productbuilddir/db-benchmark-test make check -k -j$makejobs -k DEBUG=1 CC=icc HAVE_CILK=0 >>$tracefile 2>&1
+
+    # run the brtloader tests with a debug build
+    runcmd 0 $productbuilddir/newbrt/tests make check_brtloader -k -j$makejobs -s SUMMARIZE=1 DEBUG=1 CC=icc HAVE_CILK=0 >>$tracefile 2>&1
+    runcmd 0 $productbuilddir/src/tests    make loader-tests    -k -j$makejobs -s SUMMARIZE=1 DEBUG=1 CC=icc HAVE_CILK=0 >>$tracefile 2>&1
+
+    # release build 
+    runcmd 0 $productbuilddir make clean >>$tracefile 2>&1
+    runcmd 0 $productbuilddir make release CC=icc >>$tracefile 2>&1
+    runcmd 0 $productbuilddir/utils make CC=icc >>$tracefile 2>&1
+
+    # release tests
+    runcmd 0 $productbuilddir/newbrt/tests make check -j$makejobs -k -s SUMMARIZE=1 CC=icc VGRIND= >>$tracefile 2>&1
+    runcmd 0 $productbuilddir/newbrt/tests make check_brtloader -k -j$makejobs -s SUMMARIZE=1 CC=icc VGRIND= >>$tracefile 2>&1
+    runcmd 0 $productbuilddir/src/tests make tests.tdb -j$makejobs -k -s SUMMARIZE=1 CC=icc >>$tracefile 2>&1
+    runcmd 0 $productbuilddir/src/tests make check.tdb -j$makejobs -k -s SUMMARIZE=1 CC=icc VGRIND= >>$tracefile 2>&1
+
+    # cilk tests
+    if [ 0 = 1 ] ; then 
+    runcmd 0 $productbuilddir/newbrt/tests make cilkscreen_brtloader DEBUG=1 BRTLOADER=cilk -s SUMMARIZE=1 >>$tracefile 2>&1
+    fi
+
+    # cxx
+    if [ 0 = 1 ] ; then
+    runcmd $dowindows $productbuilddir/cxx make -k -s >>$tracefile 2>&1
+    runcmd $dowindows $productbuilddir/cxx make -k -s install >>$tracefile 2>&1
+    runcmd $dowindows $productbuilddir/cxx/tests make -k -s check SUMMARIZE=1 >>$tracefile 2>&1
+    runcmd $dowindows $productbuilddir/db-benchmark-test-cxx make -k -s >>$tracefile 2>&1
+    runcmd $dowindows $productbuilddir/db-benchmark-test-cxx make -k -s check >>$tracefile 2>&1
+    fi
+    
+    # Makefile for release/examples is NOT ported to windows.  Expect it to fail.
+    if [ 0 = 1 ] ; then 
+    runcmd $dowindows $productbuilddir/release make -k setup >>$tracefile 2>&1
+    runcmd $dowindows $productbuilddir/release/examples make -k check >>$tracefile 2>&1
+    fi
+
+    # man
+    if [ 0 = 1 ] ; then 
+    runcmd 0 $productbuilddir/man/texi make -k -s -j$makejobs >>$tracefile 2>&1
+    fi
+
+    if [ 0 = 1 -a $docoverage -eq 1 ] ; then
+        # run coverage
+        runcmd 0 $productbuilddir make -s clean >>$tracefile 2>&1
+        runcmd 0 $productbuilddir make build-coverage >>$tracefile 2>&1
+        runcmd 0 $productbuilddir make check-coverage >>$tracefile 2>&1
+
+        # summarize the coverage data
+        coveragefile=$builddir/coverage+$productname-$BDB+$nodename+$system+$release+$arch
+        rawcoverage=`mktemp`
+        for d in newbrt src src/range_tree src/lock_tree; do
+            (cd $productbuilddir/$d; python ~/bin/gcovsumdir.py -b *.gcno >>$rawcoverage)
+            if [ -d $productbuilddir/$d/tests ] ; then
+                (cd $productbuilddir/$d/tests; python ~/bin/gcovsumdir.py -b *.gcno >>$rawcoverage)
+            fi
+        done
+        python ~/bin/gcovsumsum.py <$rawcoverage >$coveragefile
+        rm $rawcoverage
+    fi
+
+    # put the trace into svn
+    if [ $docommit -ne 0 ] ; then
+	testresult="PASS"
+	if [ $nfail -ne 0 ] ; then testresult="FAIL"; fi
+
+	local cf=`mktemp`
+	echo "$testresult tokudb-build $productname $CC $GCCVERSION $system $release $arch $nodename" >$cf
+	echo >>$cf; echo >>$cf
+	cat $commit_msg >>$cf
+	
+	svn add $tracefile $coveragefile
+	retry svn commit -F "$cf" $tracefile $coveragefile
+	rm $cf
+    else
+	cat $commit_msg
+    fi
+    rm $commit_msg	
+
+    if [ $deleteafter -eq 1 ] ; then
+        rm -rfv $productbuilddir > /dev/null #windows rm sometimes hangs on giant dirs without -v
+    fi
+
+    return 0
+}
+
+# set defaults
+exitcode=0
+svnserver=https://svn.tokutek.com/tokudb
+nodename=`uname -n`
+system=`uname -s | sanitize`
+release=`uname -r | sanitize`
+arch=`uname -m | sanitize`
+date=`date +%Y%m%d`
+branch="."
+tokudb="tokudb"
+bdb="4.6"
+makejobs=`get_ncpus`
+revision=0
+dovalgrind=1
+VALGRIND=valgrind
+docommit=1
+docoverage=0
+dowindows=0
+oschoice=linux
+deleteafter=0
+j=-1
+
+arg=$1;
+shopt -s compat31 #Necessary in some flavors of linux and windows
+if [ "$arg" = "--windows=yes" -o "$arg" = "--windows=1" ] ; then
+    shift
+    dowindows=1
+    export CC=icc
+    export CXX=icc
+    export CYGWIN=CYGWIN
+    oschoice=windows
+fi
+
+# setup default compiler
+if [ "$CC" = "" ] ; then export CC=icc; fi
+if [ "$CXX" = "" ] ; then export CXX=icpc; fi
+
+# import the environment
+while [ $# -gt 0 ] ; do
+    arg=$1; shift
+    if [ "$arg" = "--valgrind=no" -o "$arg" = "--valgrind=0" ] ; then
+        dovalgrind=0
+    elif [ "$arg" = "--commit=no" -o "$arg" = "--commit=0" ] ; then
+	docommit=0
+    elif [ "$arg" = "--windows=no" -o "$arg" = "--windows=0" ] ; then
+        dowindows=0
+    elif [ "$arg" = "--windows=yes" -o "$arg" = "--windows=1" ] ; then
+        usage; exit 1
+    elif [ $arg = "--gcc44" ] ; then
+	export CC=gcc44; export CXX=g++44
+    elif [ $arg = "--icc" ] ; then
+	export CC=icc; export CXX=icpc
+    elif [[ $arg =~ ^--(.*)=(.*) ]] ; then
+	eval ${BASH_REMATCH[1]}=${BASH_REMATCH[2]}
+    else
+	usage; exit 1
+    fi
+done
+
+if [ $CC = icc ] ; then
+    d=/opt/intel/bin
+    if [ -d $d ] ; then
+	export PATH=$d:$PATH
+	. $d/compilervars.sh intel64
+    fi
+    d=/opt/intel/cilkutil/bin
+    if [ -d $d ] ; then
+	export PATH=$d:$PATH
+    fi
+fi
+
+if [ $j -ne "-1" ] ; then makejobs=$j; fi
+if [ $makejobs -eq 0 ] ; then usage; exit 1; fi
+
+if [ $branch = "." ] ; then branch="toku"; fi
+if [ $revision -eq 0 ] ; then revision=`get_latest_svn_revision`; fi
+
+# setup GCCVERSION
+export GCCVERSION=`$CC --version|head -1|cut -f3 -d" "`
+export VALGRIND=$VALGRIND
+
+# setup VGRIND
+if [ $dovalgrind = 0 ] ; then export VGRIND=""; fi
+
+build $bdb