move the fractal tree build scripts from tokudb.build to tokudb/scripts so we can branch properly

git-svn-id: file:///svn/toku/tokudb@20822 c7de825b-a66e-492c-adef-691d508d4ae1

scripts/build.benchmark.bash

+#!/bin/bash
+
+# build.check.bash --revision=10500
+# build.check.bash --tokudb=tokudb.1489 --revision=10500
+# build.check.bash --branch=mysql.branches/1.1.3 --revision=10500
+# build.check.bash --windows=1 --revision=10500
+
+function usage() {
+    echo "build tokudb and run regressions"
+    echo "--windows=$dowindows (if yes/1 must be first option)"
+    echo "--branch=$branch"
+    echo "--tokudb=$tokudb"
+    echo "--revision=$revision"
+    echo "--bdb=$bdb"
+    echo "--valgrind=$dovalgrind"
+    echo "--commit=$docommit"
+    echo "--j=$makejobs"
+    echo "--deleteafter=$deleteafter"
+    echo "--doclean=$doclean"
+}
+
+# delete some characters that cygwin and osx have trouble with
+function sanitize() {
+    tr -d [/:\\\\]
+}
+
+function mydate() {
+    date +"%Y%m%d %H:%M:%S"
+}
+
+function get_ncpus() {
+    grep bogomips /proc/cpuinfo | wc -l
+}
+
+function get_latest_svn_revision() {
+    local revision=0
+    local svntarget=$svnserver/$*
+    local latest=`svn info $svntarget | grep "Last Changed Rev:"`
+    if [[ $latest =~ "Last Changed Rev: (.*)" ]] ; then
+        revision=${BASH_REMATCH[1]}
+    fi
+    echo $revision
+}
+
+function make_tokudb_name() {
+    local tokudb_dir=$1
+    local tokudb=$2
+    if [ $tokudb_dir = "toku" ] ; then
+	echo $tokudb
+    else
+	echo `echo $tokudb_dir-$tokudb | tr / -`
+    fi
+}
+
+# run a command and trace its result
+function runcmd() {
+    local fail=$1; shift
+    local makedir=$1; shift
+    local cmd=$*
+
+    echo `mydate` $makedir $cmd
+    pushd $makedir
+    $cmd
+    exitcode=$?
+    local dir=$makedir
+    if [[ $dir =~ "$HOME/svn.build/(.*)" ]] ; then
+	dir=${BASH_REMATCH[1]}
+    fi
+    if [ $fail -eq 0 ] ; then
+        if [ $exitcode -eq 0 ] ; then
+            result="PASS `mydate` $dir $cmd"
+        else
+            result="FAIL `mydate` $dir $cmd"
+            let nfail=nfail+1
+        fi
+    else
+        if [ $exitcode -eq 0 ] ; then
+            result="XPASS `mydate` $dir $cmd"
+            let nfail=nfail+1
+        else
+            result="XFAIL `mydate` $dir $cmd"
+        fi
+    fi
+    echo $result
+    echo $result >>$commit_msg
+    echo $result >>/tmp/tokubuild.trace
+    popd
+}
+
+# build a version of tokudb with a specific BDB target
+function build() {
+
+    # setup build environment
+    export BDB=$1
+    if [[ $BDB =~ "(.*)\.(.*)" ]] ;then
+        export BDBMAJOR=${BASH_REMATCH[1]}
+        export BDBMINOR=${BASH_REMATCH[2]}
+    else
+        return 1
+    fi
+    if [ $dowindows -eq 0 ] ; then
+        export BDBDIR=/usr/local/BerkeleyDB.$BDB
+    else
+        export BDBDIR=c:/cygwin/usr/local/BerkeleyDB.$BDB
+    fi
+    if [ ! -d $BDBDIR ] ; then return 2; fi
+
+    tokudb_name=`make_tokudb_name $branch $tokudb`
+    export TOKUDB_NAME=$tokudb_name
+    export TOKUDB_REVISION=$revision
+
+    productname=$tokudb_name-$revision
+    checkout=$branch/$tokudb
+
+    latestrev=`get_latest_svn_revision $checkout`
+    if [ $latestrev -eq 0 ] ; then return 3; fi
+
+    commit_msg=`mktemp`
+
+    svnbase=~/svn.build
+    if [ ! -d $svnbase ] ; then mkdir $svnbase ; fi
+
+    # checkout the build dir
+    buildbase=$svnbase/tokudb.build
+    if [ ! -d $buildbase ] ; then
+        cd $svnbase
+        svn checkout -q $svnserver/tokudb.build
+        cd $buildbase
+    else
+        cd $buildbase
+        svn update -q
+	svn update -q $date # Could be a sparse directory
+    fi
+
+    # make the build directory, possibly on multiple machines simultaneously, there can be only one
+    builddir=$buildbase/$date
+    while [ ! -d $builddir ] ; do
+        mkdir -p $builddir
+        svn add $builddir
+        svn commit $builddir -m ""
+	if [ $? -eq 0 ] ; then break ; fi
+	rmdir $builddir
+	svn update -q
+	svn update -q $date # Could be a sparse directory
+    done
+
+    tracefile=$builddir/benchmark+$productname-$BDB+$nodename+$system+$release+$arch
+
+    # get some config info
+    uname -a >>"$tracefile" 2>&1
+    cc -v >>"$tracefile" 2>&1
+    if [ -f /proc/version ] ; then
+	cat /proc/version >>"$tracefile"
+    fi
+
+    # checkout the source dir
+    productbuilddir=$svnbase/$productname/$BDB
+    # cleanup
+    rm -rf $productbuilddir
+    mkdir -p $productbuilddir
+
+    let nfail=0
+
+    # checkout into $productbuilddir
+    runcmd 0 $productbuilddir svn checkout -q -r $revision $svnserver/$checkout . >>"$tracefile" 2>&1
+
+    # update the db.h file
+    runcmd 0 $productbuilddir/buildheader cp db.h_"$BDBMAJOR"_"$BDBMINOR" ../include/db.h >>"$tracefile" 2>&1
+
+    # build
+    runcmd 0 $productbuilddir/src make -k install >>"$tracefile" 2>&1
+    runcmd 0 $productbuilddir/db-benchmark-test make build.tdb -k >>"$tracefile" 2>&1
+
+    for i in no-txn txn abort child child_abort child_abortfirst txn1 abort1 child1 child-abort1 child_abortfirst1; do
+        # Run benchmarks
+        runcmd 0 $productbuilddir/db-benchmark-test make $i.benchmark.dir -k >>"$tracefile" 2>&1
+        runcmd 0 $productbuilddir/db-benchmark-test make $i.manybenchmark -k >>"$tracefile" 2>&1
+
+        # Flattening scan
+        runcmd 0 $productbuilddir/db-benchmark-test make $i.flatteningscan -k >>"$tracefile" 2>&1
+        runcmd 0 $productbuilddir/db-benchmark-test make $i.manyflatteningscan -k >>"$tracefile" 2>&1
+
+        runcmd 0 $productbuilddir/db-benchmark-test make $i.flattenedscan -k >>"$tracefile" 2>&1
+        runcmd 0 $productbuilddir/db-benchmark-test make $i.manyflattenedscan -k >>"$tracefile" 2>&1
+    done
+
+    # clean
+    if [ $doclean -eq 1 ] ; then
+	# remove the huge 4g test file
+	runcmd 0 $productbuilddir/db-benchmark-test make -k clean >>"$tracefile" 2>&1    
+    fi
+
+    # put the trace into svn
+    if [ $docommit -ne 0 ] ; then
+	testresult="PASS"
+	if [ $nfail -ne 0 ] ; then testresult="FAIL"; fi
+
+	local cf=`mktemp`
+	echo "$testresult tokudb benchmark-test $productname-$BDB $system $release $arch $nodename" >$cf
+	echo >>$cf; echo >>$cf
+	cat $commit_msg >>$cf
+	
+	svn add "$tracefile" "$coveragefile"
+	svn commit -F "$cf" "$tracefile" "$coveragefile"
+	rm $cf
+    else
+	cat "$commit_msg"
+    fi
+    rm "$commit_msg"	
+    if [ $deleteafter -eq 1 ] ; then
+        rm -rf $productbuilddir
+    fi
+}
+
+# set defaults
+exitcode=0
+svnserver=https://svn.tokutek.com/tokudb
+nodename=`uname -n`
+system=`uname -s | sanitize`
+release=`uname -r | sanitize`
+arch=`uname -m | sanitize`
+date=`date +%Y%m%d`
+branch="toku"
+tokudb="tokudb"
+bdb="4.6"
+makejobs=`get_ncpus`
+revision=0
+dovalgrind=1
+docommit=1
+docoverage=0
+dowindows=0
+oschoice=linux
+deleteafter=0
+doclean=1
+j=-1
+export BINSUF=
+
+arg=$1;
+shopt -s compat31 #Necessary in some flavors of linux and windows
+if [ "$arg" = "--windows=yes" -o "$arg" = "--windows=1" ] ; then
+    shift
+    dowindows=1
+    export CC=icc
+    export CXX=icc
+    export CYGWIN=CYGWIN
+    oschoice=windows
+    export BINSUF=.exe
+fi
+
+# import the environment
+while [ $# -gt 0 ] ; do
+    arg=$1; shift
+    if [ "$arg" = "--valgrind=no" -o "$arg" = "--valgrind=0" ] ; then
+        dovalgrind=0
+    elif [ "$arg" = "--commit=no" -o "$arg" = "--commit=0" ] ; then
+	docommit=0
+    elif [ "$arg" = "--windows=no" -o "$arg" = "--windows=0" ] ; then
+        dowindows=0
+    elif [ "$arg" = "--windows=yes" -o "$arg" = "--windows=1" ] ; then
+        usage; exit 1
+    elif [[ "$arg" =~ "^--(.*)=(.*)" ]] ; then
+	eval ${BASH_REMATCH[1]}=${BASH_REMATCH[2]}
+    else
+	usage; exit 1
+    fi
+done
+
+if [ $j -ne "-1" ] ; then
+    makejobs=$j
+fi
+if [ $makejobs -eq 0 ] ; then
+    usage
+    exit 1
+fi
+
+if [ $revision -eq 0 ] ; then
+    revision=`get_latest_svn_revision`
+fi
+
+build $bdb
+
+exit $exitcode

scripts/build.check.crontab

+# run daily at 2am
+0 2 * * * $HOME/bin/tokudb.build.check.bash

scripts/check.copyright.py

+#!/usr/bin/python
+#indent "Copyright (c) Tokutek Inc.  All rights reserved."
+
+# check files for tokutek copyright.  exit 0 if all files have the copyright
+# otherwise, exit 1
+
+import sys
+import os
+import re
+
+def checkcopyright(f, verbose):
+    cmd = "egrep -q \"Copyright.*Tokutek\" " + f
+    exitcode = os.system(cmd)
+    if exitcode != 0 and verbose:
+        print f
+    return exitcode
+
+def checkdir(d, verbose):
+    nocopyright = 0
+    for dirpath, dirnames, filenames in os.walk(d):
+        for f in filenames:
+            fullname = dirpath + "/" + f
+            # skip svn metadata
+            matches = re.match(".*\.svn.*", fullname)
+            if matches is None:
+                nocopyright += checkcopyright(fullname, verbose)
+    return nocopyright
+
+def usage():
+    print "copyright.check [--help] [--verbose] DIRS"
+    print " default DIRS is \".\""
+    return 1
+
+def main():
+    verbose = 0
+    nocopyright = 0
+    dirname = "."
+    try:
+        if len(sys.argv) <= 1:
+            nocopyright += checkdir(dirname, verbose)
+        else:
+            ndirs = 0
+            for arg in sys.argv[1:]:
+                if arg == "-h" or arg == "--help":
+                    return usage()
+                elif arg == "-v" or arg == "--verbose":
+                    verbose += 1
+                else:
+                    ndirs += 1
+                    dirname = arg
+                    nocopyright += checkdir(dirname, verbose)
+            if ndirs == 0:
+                nocopyright += checkdir(dirname, verbose)
+    except:
+        return 1
+    
+    if nocopyright > 0: exitcode = 1
+    else: exitcode = 0
+    return exitcode
+
+sys.exit(main())

scripts/coverage.bash

+#!/bin/bash 
+# measure code coverage of the tokudb tests
+
+tokudb_version=tokudb
+tokudb=$tokudb_version
+tokudb_checkout_dir=$tokudb
+
+while [ $# -gt 0 ] ; do
+    arg=$1; shift
+    if [[ $arg =~ "--tokudb=(.*)" ]] ; then
+	tokudb_version=${BASH_REMATCH[1]}
+	tokudb="tokudb-$tokudb_version"
+	tokudb_checkout_dir="tokudb.branches/$tokudb_version"
+    fi
+done
+
+echo $tokudb
+echo $tokudb_checkout_dir
+
+coveragedir=~/svn.coverage.$tokudb.`date +%Y%m%d`
+mkdir $coveragedir
+cd $coveragedir
+svn co -q https://svn.tokutek.com/tokudb/$tokudb_checkout_dir
+if [ $tokudb != "tokudb" ] ; then
+    mv $tokudb_version $tokudb
+fi
+cd $tokudb
+
+# build tokudb with coverage enable
+make -k build-coverage
+
+# run the tests
+make -k check-coverage
+(cd src/tests;make -k all.recover VGRIND="")
+
+# make -k measure-coverage
+rm $coveragedir/raw.test.coverage
+for d in newbrt src utils cxx src/range_tree src/lock_tree; do
+	(cd $d; python ~/bin/gcovsumdir.py -b *.c *.cpp >>$coveragedir/raw.test.coverage)
+done
+python ~/bin/gcovsumsum.py $coveragedir/raw.test.coverage >$coveragedir/test.coverage
+

scripts/gcovsumdir.py

+#! /usr/bin/env python
+# summarize the code coverage of a collection of named files
+
+import re
+import os
+import sys
+import math
+import glob
+
+verbose = 0
+dobranches = 0
+dofunctions = 0
+functions = {}
+lines = {}
+branches = {}
+taken = {}
+calls = {}
+
+def add(coverage, fname, m, n):
+    global verbose
+    addit = 0
+    if coverage.has_key(fname):
+        (oldm, oldn) = coverage[fname]
+        if verbose: print "old", oldm, oldn, m, n
+        if m > oldm:
+            addit = 1
+    else:
+        addit = 1
+    if addit:
+        if verbose: print "add", fname, m, n
+        coverage[fname] = (m, n)
+    return addit
+
+def add_match(coverage, fname, match):
+    percent = float(match.group(1))
+    n = int(match.group(2))
+    m = int(math.ceil((float(percent)/100)*n))
+    return add(coverage, fname, m, n)
+    
+def gcov(gcovargs, fnames):
+    global verbose
+    if verbose: print "gcov ", gcovargs, fnames
+    f = os.popen("gcov " + gcovargs + " " + fnames)
+    fname = ""
+    while 1:
+        b = f.readline()
+        if b == "": break
+        if verbose: print ">", b
+        match = re.match("File \'(.*)\'", b)
+        if match:
+            fname = match.group(1)
+            coverage = lines
+        else:
+            match = re.match("Function \'(.*)\'", b)
+            if match:
+                fname = match.group(1)
+                coverage = functions
+        if fname[0] == '/' or not os.path.exists(fname):
+            continue
+        match = re.match("Lines executed:(.*)% of (.*)", b)
+        if match:
+            if add_match(coverage, fname, match) and coverage == lines:
+                cpcmd = "cp " + fname + ".gcov" + " " + fname + ".gcov.best"
+                if verbose: print "system", cpcmd
+                os.system(cpcmd)
+            continue
+        match = re.match("Branches executed:(.*)% of (.*)", b)
+        if match:
+            add_match(branches, fname, match)
+            continue
+        match = re.match("Taken.*:(.*)% of (.*)", b)
+        if match:
+            add_match(taken, fname, match)
+            continue
+        match = re.match("Calls executed:(.*)% of (.*)", b)
+        if match:
+            add_match(calls, fname, match)
+            continue
+
+def usage():
+    print "gcovsummary.py [-h] [-v] [-b] [-f] FILENAMES"
+    return 1
+
+def percent(m, n):
+    return (float(m)/float(n))*100
+
+def main():
+    global verbose, dobranches, dofunctions
+    # coverage hashes filenames -> (lines_executes, total_lines) tuples
+    gcovargs = ""
+    threshold = 1
+
+    i = 1
+    while i < len(sys.argv):
+        arg = sys.argv[i]
+        if arg == "-h" or arg == "--help":
+            return usage()
+        elif arg == "-v" or arg == "--verbose":
+            verbose = 1
+        elif arg == "-b":
+            dobranches = 1
+            gcovargs = gcovargs + " " + arg
+        elif arg == "-f":
+            dofunctions = 1
+            gcovargs = gcovargs + " " + arg
+        elif arg == "--threshold":
+            if i+1 < len(sys.argv):
+                i += 1
+                threshold = float(sys.argv[i])
+        else:
+            gcov(gcovargs, arg)
+        i += 1
+
+    # print a coverage summary
+    if len(functions) > 0:
+        fnames = functions.keys()
+        fnames.sort()
+        for fname in fnames:
+            (m,n) = functions[fname]
+            if float(m)/float(n) <= threshold:
+                print "%s %d/%d %.2f%%" % (fname, m, n, percent(m, n))
+    else:
+        fnames = lines.keys()
+        fnames.sort()
+        for fname in fnames:
+            (m,n) = lines[fname]
+            if float(m)/float(n) > threshold:
+                continue
+            print "%s" % fname
+            print "\t%s %d/%d %.2f%%" % ("Lines", m, n, percent(m, n))
+            if branches.has_key(fname):
+                (m,n) = branches[fname]
+                print "\t%s %d/%d %.2f%%" % ("Branches", m, n, percent(m, n))
+            if taken.has_key(fname):
+                (m,n) = taken[fname]
+                print "\t%s %d/%d %.2f%%" % ("Taken", m, n, percent(m, n))
+            if calls.has_key(fname):
+                (m,n) = calls[fname]
+                print "\t%s %d/%d %.2f%%" % ("Calls", m, n, percent(m, n))
+
+    # rename the best gcov files
+    fnames = glob.glob("*.best")
+    for fname in fnames:
+        if verbose: print "test", fname
+        match = re.match("(.*)\.gcov\.best", fname)
+        if match != None:
+            mvcmd = "mv " + fname + " " + match.group(1) + ".gcov"
+            if verbose: print "system", mvcmd
+            os.system(mvcmd)
+    return 0
+
+sys.exit(main())

scripts/gcovsumsum.py

+#! /usr/bin/env python
+
+import sys
+import re
+import os.path
+
+verbose = 0
+
+def percent(a, b):
+    return (float(a)/float(b))*100
+
+def sumarize():
+    fname = None
+    cfiles = {}
+    while 1:
+        b = sys.stdin.readline()
+        if b == "":
+            break
+        b = b.rstrip('\n')
+        if verbose: print ">", b
+        match = re.match("^(\S*)$", b)
+        if match:
+            maybe = match.group(1)
+            # skip test files
+            match = re.match(".*test|keyrange-|bug|shortcut|manyfiles", maybe)
+            if match == None:
+                fname = os.path.basename(maybe)
+                # print fname
+            else:
+                fname = None
+        else:
+            match = re.match("\t(.*) (.*)\/(.*) (.*)%", b)
+            if match and fname:
+                # print b
+                type = match.group(1)
+                m = int(match.group(2))
+                n = int(match.group(3))
+                if cfiles.has_key(fname):
+                    h = cfiles[fname]
+                else:
+                    h = {}
+                    cfiles[fname] = h
+                if h.has_key(type):
+                    (om,on) = h[type]
+                    if percent(m,n) > percent(om,on):
+                        h[type] = (m,n)
+                else:
+                    h[type] = (m,n)
+
+    summary = {}
+    keys = cfiles.keys(); keys.sort()
+    for k in keys:
+        h = cfiles[k]
+        print k
+        ktypes = h.keys(); ktypes.sort()
+        for t in ktypes:
+            (m,n) = h[t]
+            print "\t%s %d/%d %.2f%%" % (t, m, n, percent(m,n))
+            if summary.has_key(t):
+                (om,on) = summary[t]; summary[t] = (om+m,on+n)
+            else:
+                summary[t] = (m,n)
+    print
+    printhash("Summary", summary)
+
+def printhash(s, h):
+    print s
+    ktypes = h.keys(); ktypes.sort()
+    for t in ktypes:
+        (m,n) = h[t]
+        print "\t%s %d/%d %.2f%%" % (t, m, n, percent(m,n))
+
+def main():
+    global verbose
+    for arg in sys.argv[1:]:
+        if arg == "-v" or arg == "--verbose":
+            verbose = 1
+    sumarize()
+
+sys.exit(main())