mirror of
https://github.com/MariaDB/server.git
synced 2025-02-02 03:51:50 +01:00
move the fractal tree build scripts from tokudb.build to tokudb/scripts so we can branch properly
git-svn-id: file:///svn/toku/tokudb@20822 c7de825b-a66e-492c-adef-691d508d4ae1
This commit is contained in:
Rich Prohaska
Yoni Fogel
parent
01755d03e1
commit
f5ead6763b
7 changed files with 985 additions and 0 deletions
282
scripts/build.benchmark.bash
Executable file
282
scripts/build.benchmark.bash
Executable file
|
|
@ -0,0 +1,282 @@
|
|||
#!/bin/bash
|
||||
|
||||
# build.check.bash --revision=10500
|
||||
# build.check.bash --tokudb=tokudb.1489 --revision=10500
|
||||
# build.check.bash --branch=mysql.branches/1.1.3 --revision=10500
|
||||
# build.check.bash --windows=1 --revision=10500
|
||||
|
||||
function usage() {
|
||||
echo "build tokudb and run regressions"
|
||||
echo "--windows=$dowindows (if yes/1 must be first option)"
|
||||
echo "--branch=$branch"
|
||||
echo "--tokudb=$tokudb"
|
||||
echo "--revision=$revision"
|
||||
echo "--bdb=$bdb"
|
||||
echo "--valgrind=$dovalgrind"
|
||||
echo "--commit=$docommit"
|
||||
echo "--j=$makejobs"
|
||||
echo "--deleteafter=$deleteafter"
|
||||
echo "--doclean=$doclean"
|
||||
}
|
||||
|
||||
# delete some characters that cygwin and osx have trouble with
|
||||
function sanitize() {
|
||||
tr -d [/:\\\\]
|
||||
}
|
||||
|
||||
function mydate() {
|
||||
date +"%Y%m%d %H:%M:%S"
|
||||
}
|
||||
|
||||
function get_ncpus() {
|
||||
grep bogomips /proc/cpuinfo | wc -l
|
||||
}
|
||||
|
||||
function get_latest_svn_revision() {
|
||||
local revision=0
|
||||
local svntarget=$svnserver/$*
|
||||
local latest=`svn info $svntarget | grep "Last Changed Rev:"`
|
||||
if [[ $latest =~ "Last Changed Rev: (.*)" ]] ; then
|
||||
revision=${BASH_REMATCH[1]}
|
||||
fi
|
||||
echo $revision
|
||||
}
|
||||
|
||||
function make_tokudb_name() {
|
||||
local tokudb_dir=$1
|
||||
local tokudb=$2
|
||||
if [ $tokudb_dir = "toku" ] ; then
|
||||
echo $tokudb
|
||||
else
|
||||
echo `echo $tokudb_dir-$tokudb | tr / -`
|
||||
fi
|
||||
}
|
||||
|
||||
# run a command and trace its result
|
||||
function runcmd() {
|
||||
local fail=$1; shift
|
||||
local makedir=$1; shift
|
||||
local cmd=$*
|
||||
|
||||
echo `mydate` $makedir $cmd
|
||||
pushd $makedir
|
||||
$cmd
|
||||
exitcode=$?
|
||||
local dir=$makedir
|
||||
if [[ $dir =~ "$HOME/svn.build/(.*)" ]] ; then
|
||||
dir=${BASH_REMATCH[1]}
|
||||
fi
|
||||
if [ $fail -eq 0 ] ; then
|
||||
if [ $exitcode -eq 0 ] ; then
|
||||
result="PASS `mydate` $dir $cmd"
|
||||
else
|
||||
result="FAIL `mydate` $dir $cmd"
|
||||
let nfail=nfail+1
|
||||
fi
|
||||
else
|
||||
if [ $exitcode -eq 0 ] ; then
|
||||
result="XPASS `mydate` $dir $cmd"
|
||||
let nfail=nfail+1
|
||||
else
|
||||
result="XFAIL `mydate` $dir $cmd"
|
||||
fi
|
||||
fi
|
||||
echo $result
|
||||
echo $result >>$commit_msg
|
||||
echo $result >>/tmp/tokubuild.trace
|
||||
popd
|
||||
}
|
||||
|
||||
# build a version of tokudb with a specific BDB target
|
||||
function build() {
|
||||
|
||||
# setup build environment
|
||||
export BDB=$1
|
||||
if [[ $BDB =~ "(.*)\.(.*)" ]] ;then
|
||||
export BDBMAJOR=${BASH_REMATCH[1]}
|
||||
export BDBMINOR=${BASH_REMATCH[2]}
|
||||
else
|
||||
return 1
|
||||
fi
|
||||
if [ $dowindows -eq 0 ] ; then
|
||||
export BDBDIR=/usr/local/BerkeleyDB.$BDB
|
||||
else
|
||||
export BDBDIR=c:/cygwin/usr/local/BerkeleyDB.$BDB
|
||||
fi
|
||||
if [ ! -d $BDBDIR ] ; then return 2; fi
|
||||
|
||||
tokudb_name=`make_tokudb_name $branch $tokudb`
|
||||
export TOKUDB_NAME=$tokudb_name
|
||||
export TOKUDB_REVISION=$revision
|
||||
|
||||
productname=$tokudb_name-$revision
|
||||
checkout=$branch/$tokudb
|
||||
|
||||
latestrev=`get_latest_svn_revision $checkout`
|
||||
if [ $latestrev -eq 0 ] ; then return 3; fi
|
||||
|
||||
commit_msg=`mktemp`
|
||||
|
||||
svnbase=~/svn.build
|
||||
if [ ! -d $svnbase ] ; then mkdir $svnbase ; fi
|
||||
|
||||
# checkout the build dir
|
||||
buildbase=$svnbase/tokudb.build
|
||||
if [ ! -d $buildbase ] ; then
|
||||
cd $svnbase
|
||||
svn checkout -q $svnserver/tokudb.build
|
||||
cd $buildbase
|
||||
else
|
||||
cd $buildbase
|
||||
svn update -q
|
||||
svn update -q $date # Could be a sparse directory
|
||||
fi
|
||||
|
||||
# make the build directory, possibly on multiple machines simultaneously, there can be only one
|
||||
builddir=$buildbase/$date
|
||||
while [ ! -d $builddir ] ; do
|
||||
mkdir -p $builddir
|
||||
svn add $builddir
|
||||
svn commit $builddir -m ""
|
||||
if [ $? -eq 0 ] ; then break ; fi
|
||||
rmdir $builddir
|
||||
svn update -q
|
||||
svn update -q $date # Could be a sparse directory
|
||||
done
|
||||
|
||||
tracefile=$builddir/benchmark+$productname-$BDB+$nodename+$system+$release+$arch
|
||||
|
||||
# get some config info
|
||||
uname -a >>"$tracefile" 2>&1
|
||||
cc -v >>"$tracefile" 2>&1
|
||||
if [ -f /proc/version ] ; then
|
||||
cat /proc/version >>"$tracefile"
|
||||
fi
|
||||
|
||||
# checkout the source dir
|
||||
productbuilddir=$svnbase/$productname/$BDB
|
||||
# cleanup
|
||||
rm -rf $productbuilddir
|
||||
mkdir -p $productbuilddir
|
||||
|
||||
let nfail=0
|
||||
|
||||
# checkout into $productbuilddir
|
||||
runcmd 0 $productbuilddir svn checkout -q -r $revision $svnserver/$checkout . >>"$tracefile" 2>&1
|
||||
|
||||
# update the db.h file
|
||||
runcmd 0 $productbuilddir/buildheader cp db.h_"$BDBMAJOR"_"$BDBMINOR" ../include/db.h >>"$tracefile" 2>&1
|
||||
|
||||
# build
|
||||
runcmd 0 $productbuilddir/src make -k install >>"$tracefile" 2>&1
|
||||
runcmd 0 $productbuilddir/db-benchmark-test make build.tdb -k >>"$tracefile" 2>&1
|
||||
|
||||
for i in no-txn txn abort child child_abort child_abortfirst txn1 abort1 child1 child-abort1 child_abortfirst1; do
|
||||
# Run benchmarks
|
||||
runcmd 0 $productbuilddir/db-benchmark-test make $i.benchmark.dir -k >>"$tracefile" 2>&1
|
||||
runcmd 0 $productbuilddir/db-benchmark-test make $i.manybenchmark -k >>"$tracefile" 2>&1
|
||||
|
||||
# Flattening scan
|
||||
runcmd 0 $productbuilddir/db-benchmark-test make $i.flatteningscan -k >>"$tracefile" 2>&1
|
||||
runcmd 0 $productbuilddir/db-benchmark-test make $i.manyflatteningscan -k >>"$tracefile" 2>&1
|
||||
|
||||
runcmd 0 $productbuilddir/db-benchmark-test make $i.flattenedscan -k >>"$tracefile" 2>&1
|
||||
runcmd 0 $productbuilddir/db-benchmark-test make $i.manyflattenedscan -k >>"$tracefile" 2>&1
|
||||
done
|
||||
|
||||
# clean
|
||||
if [ $doclean -eq 1 ] ; then
|
||||
# remove the huge 4g test file
|
||||
runcmd 0 $productbuilddir/db-benchmark-test make -k clean >>"$tracefile" 2>&1
|
||||
fi
|
||||
|
||||
# put the trace into svn
|
||||
if [ $docommit -ne 0 ] ; then
|
||||
testresult="PASS"
|
||||
if [ $nfail -ne 0 ] ; then testresult="FAIL"; fi
|
||||
|
||||
local cf=`mktemp`
|
||||
echo "$testresult tokudb benchmark-test $productname-$BDB $system $release $arch $nodename" >$cf
|
||||
echo >>$cf; echo >>$cf
|
||||
cat $commit_msg >>$cf
|
||||
|
||||
svn add "$tracefile" "$coveragefile"
|
||||
svn commit -F "$cf" "$tracefile" "$coveragefile"
|
||||
rm $cf
|
||||
else
|
||||
cat "$commit_msg"
|
||||
fi
|
||||
rm "$commit_msg"
|
||||
if [ $deleteafter -eq 1 ] ; then
|
||||
rm -rf $productbuilddir
|
||||
fi
|
||||
}
|
||||
|
||||
# set defaults
|
||||
exitcode=0
|
||||
svnserver=https://svn.tokutek.com/tokudb
|
||||
nodename=`uname -n`
|
||||
system=`uname -s | sanitize`
|
||||
release=`uname -r | sanitize`
|
||||
arch=`uname -m | sanitize`
|
||||
date=`date +%Y%m%d`
|
||||
branch="toku"
|
||||
tokudb="tokudb"
|
||||
bdb="4.6"
|
||||
makejobs=`get_ncpus`
|
||||
revision=0
|
||||
dovalgrind=1
|
||||
docommit=1
|
||||
docoverage=0
|
||||
dowindows=0
|
||||
oschoice=linux
|
||||
deleteafter=0
|
||||
doclean=1
|
||||
j=-1
|
||||
export BINSUF=
|
||||
|
||||
arg=$1;
|
||||
shopt -s compat31 #Necessary in some flavors of linux and windows
|
||||
if [ "$arg" = "--windows=yes" -o "$arg" = "--windows=1" ] ; then
|
||||
shift
|
||||
dowindows=1
|
||||
export CC=icc
|
||||
export CXX=icc
|
||||
export CYGWIN=CYGWIN
|
||||
oschoice=windows
|
||||
export BINSUF=.exe
|
||||
fi
|
||||
|
||||
# import the environment
|
||||
while [ $# -gt 0 ] ; do
|
||||
arg=$1; shift
|
||||
if [ "$arg" = "--valgrind=no" -o "$arg" = "--valgrind=0" ] ; then
|
||||
dovalgrind=0
|
||||
elif [ "$arg" = "--commit=no" -o "$arg" = "--commit=0" ] ; then
|
||||
docommit=0
|
||||
elif [ "$arg" = "--windows=no" -o "$arg" = "--windows=0" ] ; then
|
||||
dowindows=0
|
||||
elif [ "$arg" = "--windows=yes" -o "$arg" = "--windows=1" ] ; then
|
||||
usage; exit 1
|
||||
elif [[ "$arg" =~ "^--(.*)=(.*)" ]] ; then
|
||||
eval ${BASH_REMATCH[1]}=${BASH_REMATCH[2]}
|
||||
else
|
||||
usage; exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
if [ $j -ne "-1" ] ; then
|
||||
makejobs=$j
|
||||
fi
|
||||
if [ $makejobs -eq 0 ] ; then
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ $revision -eq 0 ] ; then
|
||||
revision=`get_latest_svn_revision`
|
||||
fi
|
||||
|
||||
build $bdb
|
||||
|
||||
exit $exitcode
|
||||
366
scripts/build.check.bash
Executable file
366
scripts/build.check.bash
Executable file
|
|
@ -0,0 +1,366 @@
|
|||
#!/bin/bash
|
||||
|
||||
# build.check.bash --revision=10500
|
||||
# build.check.bash --tokudb=tokudb.1489 --revision=10500
|
||||
# build.check.bash --branch=mysql.branches/1.1.3 --revision=10500
|
||||
# build.check.bash --windows=1 --revision=10500
|
||||
|
||||
function usage() {
|
||||
echo "build tokudb and run regressions"
|
||||
echo "--windows=$dowindows (if yes/1 must be first option)"
|
||||
echo "--branch=$branch"
|
||||
echo "--tokudb=$tokudb"
|
||||
echo "--revision=$revision"
|
||||
echo "--bdb=$bdb"
|
||||
echo "--valgrind=$dovalgrind"
|
||||
echo "--commit=$docommit"
|
||||
echo "--j=$makejobs"
|
||||
echo "--deleteafter=$deleteafter"
|
||||
echo "--doclean=$doclean"
|
||||
}
|
||||
|
||||
function retry() {
|
||||
local cmd
|
||||
local retries
|
||||
local exitcode
|
||||
cmd=$*
|
||||
let retries=0
|
||||
while [ $retries -le 10 ] ; do
|
||||
echo `date` $cmd
|
||||
bash -c "$cmd"
|
||||
exitcode=$?
|
||||
echo `date` $cmd $exitcode $retries
|
||||
let retries=retries+1
|
||||
if [ $exitcode -eq 0 ] ; then break; fi
|
||||
sleep 10
|
||||
done
|
||||
test $exitcode = 0
|
||||
}
|
||||
|
||||
# delete some characters that cygwin and osx have trouble with
|
||||
function sanitize() {
|
||||
tr -d '[/:\\\\()]'
|
||||
}
|
||||
|
||||
function mydate() {
|
||||
date +"%Y%m%d %H:%M:%S"
|
||||
}
|
||||
|
||||
function get_ncpus() {
|
||||
grep bogomips /proc/cpuinfo | wc -l
|
||||
}
|
||||
|
||||
function get_latest_svn_revision() {
|
||||
local revision=0
|
||||
local svntarget=$svnserver/$*
|
||||
local latest=`retry svn info $svntarget`
|
||||
if [[ $latest =~ "Last Changed Rev: ([0-9]+)" ]] ; then
|
||||
revision=${BASH_REMATCH[1]}
|
||||
fi
|
||||
echo $revision
|
||||
}
|
||||
|
||||
function make_tokudb_name() {
|
||||
local tokudb_dir=$1
|
||||
local tokudb=$2
|
||||
if [ $tokudb_dir = "toku" ] ; then
|
||||
echo $tokudb
|
||||
else
|
||||
echo `echo $tokudb_dir-$tokudb | tr / -`
|
||||
fi
|
||||
}
|
||||
|
||||
# run a command and trace its result
|
||||
function runcmd() {
|
||||
local fail=$1; shift
|
||||
local makedir=$1; shift
|
||||
local cmd=$*
|
||||
|
||||
echo `mydate` $makedir $cmd
|
||||
pushd $makedir
|
||||
$cmd
|
||||
exitcode=$?
|
||||
local dir=$makedir
|
||||
if [[ $dir =~ "$HOME/svn.build/(.*)" ]] ; then
|
||||
dir=${BASH_REMATCH[1]}
|
||||
fi
|
||||
if [ $fail -eq 0 ] ; then
|
||||
if [ $exitcode -eq 0 ] ; then
|
||||
result="PASS `mydate` $dir $cmd"
|
||||
else
|
||||
result="FAIL `mydate` $dir $cmd"
|
||||
let nfail=nfail+1
|
||||
fi
|
||||
else
|
||||
if [ $exitcode -eq 0 ] ; then
|
||||
result="XPASS `mydate` $dir $cmd"
|
||||
let nfail=nfail+1
|
||||
else
|
||||
result="XFAIL `mydate` $dir $cmd"
|
||||
fi
|
||||
fi
|
||||
echo $result
|
||||
echo $result >>$commit_msg
|
||||
echo $result >>/tmp/tokubuild.trace
|
||||
popd
|
||||
}
|
||||
|
||||
# build a version of tokudb with a specific BDB target
|
||||
function build() {
|
||||
|
||||
# setup build environment
|
||||
export BDB=$1
|
||||
if [[ $BDB =~ "(.*)\.(.*)" ]] ;then
|
||||
export BDBMAJOR=${BASH_REMATCH[1]}
|
||||
export BDBMINOR=${BASH_REMATCH[2]}
|
||||
else
|
||||
return 1
|
||||
fi
|
||||
if [ $dowindows -eq 0 ] ; then
|
||||
export BDBDIR=/usr/local/BerkeleyDB.$BDB
|
||||
else
|
||||
export BDBDIR=c:/cygwin/usr/local/BerkeleyDB.$BDB
|
||||
fi
|
||||
if [ ! -d $BDBDIR ] ; then return 2; fi
|
||||
|
||||
tokudb_name=`make_tokudb_name $branch $tokudb`
|
||||
export TOKUDB_NAME=$tokudb_name
|
||||
export TOKUDB_REVISION=$revision
|
||||
|
||||
productname=$tokudb_name-$revision
|
||||
checkout=$branch/$tokudb
|
||||
|
||||
latestrev=`get_latest_svn_revision $checkout`
|
||||
if [ $latestrev -eq 0 ] ; then return 3; fi
|
||||
|
||||
commit_msg=`mktemp`
|
||||
|
||||
svnbase=~/svn.build
|
||||
if [ ! -d $svnbase ] ; then mkdir $svnbase ; fi
|
||||
|
||||
# checkout the build dir
|
||||
buildbase=$svnbase/tokudb.build
|
||||
if [ ! -d $buildbase ] ; then
|
||||
cd $svnbase
|
||||
svn checkout -q $svnserver/tokudb.build
|
||||
cd $buildbase
|
||||
else
|
||||
cd $buildbase
|
||||
svn update -q
|
||||
svn update -q $date # Could be a sparse directory
|
||||
fi
|
||||
|
||||
# make the build directory, possibly on multiple machines simultaneously, there can be only one
|
||||
builddir=$buildbase/$date
|
||||
pushd $buildbase
|
||||
while [ ! -d $date ] ; do
|
||||
mkdir -p $date
|
||||
svn add $date
|
||||
svn commit $date -m ""
|
||||
if [ $? -eq 0 ] ; then break ; fi
|
||||
rm -rf $date
|
||||
svn remove $date
|
||||
svn update -q
|
||||
svn update -q $date # Could be a sparse directory
|
||||
done
|
||||
popd
|
||||
|
||||
tracefile=$builddir/build+$productname-$BDB+$nodename+$system+$release+$arch
|
||||
|
||||
# get some config info
|
||||
uname -a >>$tracefile 2>&1
|
||||
cc -v >>$tracefile 2>&1
|
||||
if [ -f /proc/version ] ; then
|
||||
cat /proc/version >>$tracefile
|
||||
fi
|
||||
|
||||
# checkout the source dir
|
||||
productbuilddir=$svnbase/$productname
|
||||
# cleanup
|
||||
rm -rf $productbuilddir
|
||||
mkdir -p $productbuilddir
|
||||
|
||||
let nfail=0
|
||||
|
||||
# checkout into $productbuilddir
|
||||
runcmd 0 $productbuilddir retry svn checkout -q -r $revision $svnserver/$checkout . >>$tracefile 2>&1
|
||||
|
||||
# portability
|
||||
runcmd 0 $productbuilddir/$oschoice make -k -j$makejobs >>$tracefile 2>&1
|
||||
runcmd 0 $productbuilddir/$oschoice/tests make -k -j$makejobs >>$tracefile 2>&1
|
||||
runcmd 0 $productbuilddir/$oschoice/tests make -k -j$makejobs check >>$tracefile 2>&1
|
||||
|
||||
# newbrt
|
||||
runcmd 0 $productbuilddir/newbrt make -k -j$makejobs checko2 >>$tracefile 2>&1
|
||||
runcmd 0 $productbuilddir/newbrt make -k -j$makejobs >>$tracefile 2>&1
|
||||
if [ $dovalgrind -ne 0 ] ; then
|
||||
runcmd 0 $productbuilddir/newbrt make -j$makejobs -k check -s SUMMARIZE=1 >>$tracefile 2>&1
|
||||
else
|
||||
runcmd 0 $productbuilddir/newbrt make -j$makejobs -k check -s SUMMARIZE=1 VGRIND="" >>$tracefile 2>&1
|
||||
fi
|
||||
|
||||
# lock tree
|
||||
runcmd 0 $productbuilddir/src/range_tree make -k -j$makejobs >>$tracefile 2>&1
|
||||
runcmd 0 $productbuilddir/src/lock_tree make -k -j$makejobs >>$tracefile 2>&1
|
||||
runcmd 0 $productbuilddir/src/range_tree/tests make -k -j$makejobs check >>$tracefile 2>&1
|
||||
runcmd 0 $productbuilddir/src/lock_tree/tests make -k -j$makejobs check >>$tracefile 2>&1
|
||||
|
||||
# src
|
||||
runcmd 0 $productbuilddir/src make -k -j$makejobs local >>$tracefile 2>&1
|
||||
runcmd $dowindows $productbuilddir/src make -k -j$makejobs check_globals >>$tracefile 2>&1
|
||||
runcmd 0 $productbuilddir/src make -k -j$makejobs install >>$tracefile 2>&1
|
||||
runcmd 0 $productbuilddir/src/tests make -k -j$makejobs >>$tracefile 2>&1
|
||||
|
||||
# utils
|
||||
runcmd 0 $productbuilddir/utils make -k -j$makejobs >>$tracefile 2>&1
|
||||
runcmd 0 $productbuilddir/utils make -k -j$makejobs check >>$tracefile 2>&1
|
||||
|
||||
# src/tests
|
||||
runcmd 0 $productbuilddir/src/tests make -j$makejobs -k check.bdb VGRIND="" BDB_SUPPRESSIONS="" SUMMARIZE=1 -s >>$tracefile 2>&1
|
||||
if [ $dovalgrind -ne 0 ] ; then
|
||||
runcmd 0 $productbuilddir/src/tests make -j$makejobs -k check.tdb SUMMARIZE=1 -s >>$tracefile 2>&1
|
||||
else
|
||||
runcmd 0 $productbuilddir/src/tests make -j$makejobs -k check.tdb VGRIND="" SUMMARIZE=1 -s>>$tracefile 2>&1
|
||||
fi
|
||||
|
||||
# benchmark tests
|
||||
runcmd 0 $productbuilddir/db-benchmark-test make -k -j$makejobs >>$tracefile 2>&1
|
||||
runcmd 0 $productbuilddir/db-benchmark-test make -j$makejobs -k check >>$tracefile 2>&1
|
||||
|
||||
# cxx
|
||||
runcmd $dowindows $productbuilddir/cxx make -k >>$tracefile 2>&1
|
||||
runcmd $dowindows $productbuilddir/cxx make -k install >>$tracefile 2>&1
|
||||
runcmd $dowindows $productbuilddir/cxx/tests make -k check >>$tracefile 2>&1
|
||||
runcmd $dowindows $productbuilddir/db-benchmark-test-cxx make -k >>$tracefile 2>&1
|
||||
runcmd $dowindows $productbuilddir/db-benchmark-test-cxx make -k check >>$tracefile 2>&1
|
||||
|
||||
# Makefile for release/examples is NOT ported to windows. Expect it to fail.
|
||||
runcmd $dowindows $productbuilddir/release make -k setup >>$tracefile 2>&1
|
||||
runcmd $dowindows $productbuilddir/release/examples make -k check >>$tracefile 2>&1
|
||||
|
||||
# man
|
||||
if [ $dowindows -eq 0 ] ; then
|
||||
runcmd 0 $productbuilddir/man/texi make -k -j$makejobs >>$tracefile 2>&1
|
||||
else
|
||||
# Don't use XFAIL because on cygwin man/texi just hangs
|
||||
runcmd 0 $productbuilddir/man/texi echo SKIPPED make -k -j$makejobs >>$tracefile 2>&1
|
||||
fi
|
||||
|
||||
# debug build
|
||||
runcmd 0 $productbuilddir make -s clean >>$tracefile 2>&1
|
||||
runcmd 0 $productbuilddir make -s -j$makejobs DEBUG=1 >>$tracefile 2>&1
|
||||
|
||||
# run the brtloader tests with a debug build
|
||||
runcmd 0 $productbuilddir/newbrt/tests make check_brtloader -k -j$makejobs DEBUG=1 -s SUMMARIZE=1 >>$tracefile 2>&1
|
||||
runcmd 0 $productbuilddir/src/tests make loader-tests -k -j$makejobs DEBUG=1 -s SUMMARIZE=1 >>$tracefile 2>&1
|
||||
|
||||
if [ $dowindows -eq 0 ] ; then
|
||||
# cilk debug build
|
||||
runcmd 0 $productbuilddir make -s clean >>$tracefile 2>&1
|
||||
runcmd 0 $productbuilddir/newbrt/tests make -s DEBUG=1 BRTLOADER=cilk >>$tracefile 2>&1
|
||||
|
||||
# run the loader tests with cilkscreen
|
||||
runcmd 0 $productbuilddir/newbrt/tests make cilkscreen_brtloader DEBUG=1 BRTLOADER=cilk -s SUMMARIZE=1 >>$tracefile 2>&1
|
||||
fi
|
||||
|
||||
if [ $docoverage -eq 1 ] ; then
|
||||
# run coverage
|
||||
runcmd 0 $productbuilddir make -s clean >>$tracefile 2>&1
|
||||
runcmd 0 $productbuilddir make build-coverage >>$tracefile 2>&1
|
||||
runcmd 0 $productbuilddir make check-coverage >>$tracefile 2>&1
|
||||
|
||||
# summarize the coverage data
|
||||
coveragefile=$builddir/coverage+$productname-$BDB+$nodename+$system+$release+$arch
|
||||
rawcoverage=`mktemp`
|
||||
for d in newbrt src src/range_tree src/lock_tree; do
|
||||
(cd $productbuilddir/$d; python ~/bin/gcovsumdir.py -b *.gcno >>$rawcoverage)
|
||||
if [ -d $productbuilddir/$d/tests ] ; then
|
||||
(cd $productbuilddir/$d/tests; python ~/bin/gcovsumdir.py -b *.gcno >>$rawcoverage)
|
||||
fi
|
||||
done
|
||||
python ~/bin/gcovsumsum.py <$rawcoverage >$coveragefile
|
||||
rm $rawcoverage
|
||||
fi
|
||||
|
||||
# put the trace into svn
|
||||
if [ $docommit -ne 0 ] ; then
|
||||
testresult="PASS"
|
||||
if [ $nfail -ne 0 ] ; then testresult="FAIL"; fi
|
||||
|
||||
local cf=`mktemp`
|
||||
echo "$testresult tokudb-build $productname-$BDB $system $release $arch $nodename" >$cf
|
||||
echo >>$cf; echo >>$cf
|
||||
cat $commit_msg >>$cf
|
||||
|
||||
svn add $tracefile $coveragefile
|
||||
retry svn commit -F "$cf" $tracefile $coveragefile
|
||||
rm $cf
|
||||
else
|
||||
cat $commit_msg
|
||||
fi
|
||||
rm $commit_msg
|
||||
|
||||
if [ $deleteafter -eq 1 ] ; then
|
||||
rm -rfv $productbuilddir > /dev/null #windows rm sometimes hangs on giant dirs without -v
|
||||
fi
|
||||
}
|
||||
|
||||
# set defaults
|
||||
exitcode=0
|
||||
svnserver=https://svn.tokutek.com/tokudb
|
||||
nodename=`uname -n`
|
||||
system=`uname -s | sanitize`
|
||||
release=`uname -r | sanitize`
|
||||
arch=`uname -m | sanitize`
|
||||
date=`date +%Y%m%d`
|
||||
branch="."
|
||||
tokudb="tokudb"
|
||||
bdb="4.6"
|
||||
makejobs=`get_ncpus`
|
||||
revision=0
|
||||
dovalgrind=1
|
||||
docommit=1
|
||||
docoverage=0
|
||||
dowindows=0
|
||||
oschoice=linux
|
||||
deleteafter=0
|
||||
j=-1
|
||||
|
||||
arg=$1;
|
||||
shopt -s compat31 #Necessary in some flavors of linux and windows
|
||||
if [ "$arg" = "--windows=yes" -o "$arg" = "--windows=1" ] ; then
|
||||
shift
|
||||
dowindows=1
|
||||
export CC=icc
|
||||
export CXX=icc
|
||||
export CYGWIN=CYGWIN
|
||||
oschoice=windows
|
||||
fi
|
||||
|
||||
# import the environment
|
||||
while [ $# -gt 0 ] ; do
|
||||
arg=$1; shift
|
||||
if [ "$arg" = "--valgrind=no" -o "$arg" = "--valgrind=0" ] ; then
|
||||
dovalgrind=0
|
||||
elif [ "$arg" = "--commit=no" -o "$arg" = "--commit=0" ] ; then
|
||||
docommit=0
|
||||
elif [ "$arg" = "--windows=no" -o "$arg" = "--windows=0" ] ; then
|
||||
dowindows=0
|
||||
elif [ "$arg" = "--windows=yes" -o "$arg" = "--windows=1" ] ; then
|
||||
usage; exit 1
|
||||
elif [[ "$arg" =~ "^--(.*)=(.*)" ]] ; then
|
||||
eval ${BASH_REMATCH[1]}=${BASH_REMATCH[2]}
|
||||
else
|
||||
usage; exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
if [ $j -ne "-1" ] ; then makejobs=$j; fi
|
||||
if [ $makejobs -eq 0 ] ; then usage; exit 1; fi
|
||||
|
||||
if [ $branch = "." ] ; then branch="toku"; fi
|
||||
if [ $revision -eq 0 ] ; then revision=`get_latest_svn_revision`; fi
|
||||
|
||||
build $bdb
|
||||
|
||||
exit $exitcode
|
||||
2
scripts/build.check.crontab
Normal file
2
scripts/build.check.crontab
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
# run daily at 2am
|
||||
0 2 * * * $HOME/bin/tokudb.build.check.bash
|
||||
61
scripts/check.copyright.py
Executable file
61
scripts/check.copyright.py
Executable file
|
|
@ -0,0 +1,61 @@
|
|||
#!/usr/bin/python
|
||||
#indent "Copyright (c) Tokutek Inc. All rights reserved."
|
||||
|
||||
# check files for tokutek copyright. exit 0 if all files have the copyright
|
||||
# otherwise, exit 1
|
||||
|
||||
import sys
|
||||
import os
|
||||
import re
|
||||
|
||||
def checkcopyright(f, verbose):
|
||||
cmd = "egrep -q \"Copyright.*Tokutek\" " + f
|
||||
exitcode = os.system(cmd)
|
||||
if exitcode != 0 and verbose:
|
||||
print f
|
||||
return exitcode
|
||||
|
||||
def checkdir(d, verbose):
|
||||
nocopyright = 0
|
||||
for dirpath, dirnames, filenames in os.walk(d):
|
||||
for f in filenames:
|
||||
fullname = dirpath + "/" + f
|
||||
# skip svn metadata
|
||||
matches = re.match(".*\.svn.*", fullname)
|
||||
if matches is None:
|
||||
nocopyright += checkcopyright(fullname, verbose)
|
||||
return nocopyright
|
||||
|
||||
def usage():
|
||||
print "copyright.check [--help] [--verbose] DIRS"
|
||||
print " default DIRS is \".\""
|
||||
return 1
|
||||
|
||||
def main():
|
||||
verbose = 0
|
||||
nocopyright = 0
|
||||
dirname = "."
|
||||
try:
|
||||
if len(sys.argv) <= 1:
|
||||
nocopyright += checkdir(dirname, verbose)
|
||||
else:
|
||||
ndirs = 0
|
||||
for arg in sys.argv[1:]:
|
||||
if arg == "-h" or arg == "--help":
|
||||
return usage()
|
||||
elif arg == "-v" or arg == "--verbose":
|
||||
verbose += 1
|
||||
else:
|
||||
ndirs += 1
|
||||
dirname = arg
|
||||
nocopyright += checkdir(dirname, verbose)
|
||||
if ndirs == 0:
|
||||
nocopyright += checkdir(dirname, verbose)
|
||||
except:
|
||||
return 1
|
||||
|
||||
if nocopyright > 0: exitcode = 1
|
||||
else: exitcode = 0
|
||||
return exitcode
|
||||
|
||||
sys.exit(main())
|
||||
42
scripts/coverage.bash
Normal file
42
scripts/coverage.bash
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
#!/bin/bash
|
||||
# measure code coverage of the tokudb tests
|
||||
|
||||
tokudb_version=tokudb
|
||||
tokudb=$tokudb_version
|
||||
tokudb_checkout_dir=$tokudb
|
||||
|
||||
while [ $# -gt 0 ] ; do
|
||||
arg=$1; shift
|
||||
if [[ $arg =~ "--tokudb=(.*)" ]] ; then
|
||||
tokudb_version=${BASH_REMATCH[1]}
|
||||
tokudb="tokudb-$tokudb_version"
|
||||
tokudb_checkout_dir="tokudb.branches/$tokudb_version"
|
||||
fi
|
||||
done
|
||||
|
||||
echo $tokudb
|
||||
echo $tokudb_checkout_dir
|
||||
|
||||
coveragedir=~/svn.coverage.$tokudb.`date +%Y%m%d`
|
||||
mkdir $coveragedir
|
||||
cd $coveragedir
|
||||
svn co -q https://svn.tokutek.com/tokudb/$tokudb_checkout_dir
|
||||
if [ $tokudb != "tokudb" ] ; then
|
||||
mv $tokudb_version $tokudb
|
||||
fi
|
||||
cd $tokudb
|
||||
|
||||
# build tokudb with coverage enable
|
||||
make -k build-coverage
|
||||
|
||||
# run the tests
|
||||
make -k check-coverage
|
||||
(cd src/tests;make -k all.recover VGRIND="")
|
||||
|
||||
# make -k measure-coverage
|
||||
rm $coveragedir/raw.test.coverage
|
||||
for d in newbrt src utils cxx src/range_tree src/lock_tree; do
|
||||
(cd $d; python ~/bin/gcovsumdir.py -b *.c *.cpp >>$coveragedir/raw.test.coverage)
|
||||
done
|
||||
python ~/bin/gcovsumsum.py $coveragedir/raw.test.coverage >$coveragedir/test.coverage
|
||||
|
||||
152
scripts/gcovsumdir.py
Normal file
152
scripts/gcovsumdir.py
Normal file
|
|
@ -0,0 +1,152 @@
|
|||
#! /usr/bin/env python
|
||||
# summarize the code coverage of a collection of named files
|
||||
|
||||
import re
|
||||
import os
|
||||
import sys
|
||||
import math
|
||||
import glob
|
||||
|
||||
verbose = 0
|
||||
dobranches = 0
|
||||
dofunctions = 0
|
||||
functions = {}
|
||||
lines = {}
|
||||
branches = {}
|
||||
taken = {}
|
||||
calls = {}
|
||||
|
||||
def add(coverage, fname, m, n):
|
||||
global verbose
|
||||
addit = 0
|
||||
if coverage.has_key(fname):
|
||||
(oldm, oldn) = coverage[fname]
|
||||
if verbose: print "old", oldm, oldn, m, n
|
||||
if m > oldm:
|
||||
addit = 1
|
||||
else:
|
||||
addit = 1
|
||||
if addit:
|
||||
if verbose: print "add", fname, m, n
|
||||
coverage[fname] = (m, n)
|
||||
return addit
|
||||
|
||||
def add_match(coverage, fname, match):
|
||||
percent = float(match.group(1))
|
||||
n = int(match.group(2))
|
||||
m = int(math.ceil((float(percent)/100)*n))
|
||||
return add(coverage, fname, m, n)
|
||||
|
||||
def gcov(gcovargs, fnames):
|
||||
global verbose
|
||||
if verbose: print "gcov ", gcovargs, fnames
|
||||
f = os.popen("gcov " + gcovargs + " " + fnames)
|
||||
fname = ""
|
||||
while 1:
|
||||
b = f.readline()
|
||||
if b == "": break
|
||||
if verbose: print ">", b
|
||||
match = re.match("File \'(.*)\'", b)
|
||||
if match:
|
||||
fname = match.group(1)
|
||||
coverage = lines
|
||||
else:
|
||||
match = re.match("Function \'(.*)\'", b)
|
||||
if match:
|
||||
fname = match.group(1)
|
||||
coverage = functions
|
||||
if fname[0] == '/' or not os.path.exists(fname):
|
||||
continue
|
||||
match = re.match("Lines executed:(.*)% of (.*)", b)
|
||||
if match:
|
||||
if add_match(coverage, fname, match) and coverage == lines:
|
||||
cpcmd = "cp " + fname + ".gcov" + " " + fname + ".gcov.best"
|
||||
if verbose: print "system", cpcmd
|
||||
os.system(cpcmd)
|
||||
continue
|
||||
match = re.match("Branches executed:(.*)% of (.*)", b)
|
||||
if match:
|
||||
add_match(branches, fname, match)
|
||||
continue
|
||||
match = re.match("Taken.*:(.*)% of (.*)", b)
|
||||
if match:
|
||||
add_match(taken, fname, match)
|
||||
continue
|
||||
match = re.match("Calls executed:(.*)% of (.*)", b)
|
||||
if match:
|
||||
add_match(calls, fname, match)
|
||||
continue
|
||||
|
||||
def usage():
|
||||
print "gcovsummary.py [-h] [-v] [-b] [-f] FILENAMES"
|
||||
return 1
|
||||
|
||||
def percent(m, n):
|
||||
return (float(m)/float(n))*100
|
||||
|
||||
def main():
|
||||
global verbose, dobranches, dofunctions
|
||||
# coverage hashes filenames -> (lines_executes, total_lines) tuples
|
||||
gcovargs = ""
|
||||
threshold = 1
|
||||
|
||||
i = 1
|
||||
while i < len(sys.argv):
|
||||
arg = sys.argv[i]
|
||||
if arg == "-h" or arg == "--help":
|
||||
return usage()
|
||||
elif arg == "-v" or arg == "--verbose":
|
||||
verbose = 1
|
||||
elif arg == "-b":
|
||||
dobranches = 1
|
||||
gcovargs = gcovargs + " " + arg
|
||||
elif arg == "-f":
|
||||
dofunctions = 1
|
||||
gcovargs = gcovargs + " " + arg
|
||||
elif arg == "--threshold":
|
||||
if i+1 < len(sys.argv):
|
||||
i += 1
|
||||
threshold = float(sys.argv[i])
|
||||
else:
|
||||
gcov(gcovargs, arg)
|
||||
i += 1
|
||||
|
||||
# print a coverage summary
|
||||
if len(functions) > 0:
|
||||
fnames = functions.keys()
|
||||
fnames.sort()
|
||||
for fname in fnames:
|
||||
(m,n) = functions[fname]
|
||||
if float(m)/float(n) <= threshold:
|
||||
print "%s %d/%d %.2f%%" % (fname, m, n, percent(m, n))
|
||||
else:
|
||||
fnames = lines.keys()
|
||||
fnames.sort()
|
||||
for fname in fnames:
|
||||
(m,n) = lines[fname]
|
||||
if float(m)/float(n) > threshold:
|
||||
continue
|
||||
print "%s" % fname
|
||||
print "\t%s %d/%d %.2f%%" % ("Lines", m, n, percent(m, n))
|
||||
if branches.has_key(fname):
|
||||
(m,n) = branches[fname]
|
||||
print "\t%s %d/%d %.2f%%" % ("Branches", m, n, percent(m, n))
|
||||
if taken.has_key(fname):
|
||||
(m,n) = taken[fname]
|
||||
print "\t%s %d/%d %.2f%%" % ("Taken", m, n, percent(m, n))
|
||||
if calls.has_key(fname):
|
||||
(m,n) = calls[fname]
|
||||
print "\t%s %d/%d %.2f%%" % ("Calls", m, n, percent(m, n))
|
||||
|
||||
# rename the best gcov files
|
||||
fnames = glob.glob("*.best")
|
||||
for fname in fnames:
|
||||
if verbose: print "test", fname
|
||||
match = re.match("(.*)\.gcov\.best", fname)
|
||||
if match != None:
|
||||
mvcmd = "mv " + fname + " " + match.group(1) + ".gcov"
|
||||
if verbose: print "system", mvcmd
|
||||
os.system(mvcmd)
|
||||
return 0
|
||||
|
||||
sys.exit(main())
|
||||
80
scripts/gcovsumsum.py
Normal file
80
scripts/gcovsumsum.py
Normal file
|
|
@ -0,0 +1,80 @@
|
|||
#! /usr/bin/env python
|
||||
|
||||
import sys
|
||||
import re
|
||||
import os.path
|
||||
|
||||
verbose = 0
|
||||
|
||||
def percent(a, b):
|
||||
return (float(a)/float(b))*100
|
||||
|
||||
def sumarize():
|
||||
fname = None
|
||||
cfiles = {}
|
||||
while 1:
|
||||
b = sys.stdin.readline()
|
||||
if b == "":
|
||||
break
|
||||
b = b.rstrip('\n')
|
||||
if verbose: print ">", b
|
||||
match = re.match("^(\S*)$", b)
|
||||
if match:
|
||||
maybe = match.group(1)
|
||||
# skip test files
|
||||
match = re.match(".*test|keyrange-|bug|shortcut|manyfiles", maybe)
|
||||
if match == None:
|
||||
fname = os.path.basename(maybe)
|
||||
# print fname
|
||||
else:
|
||||
fname = None
|
||||
else:
|
||||
match = re.match("\t(.*) (.*)\/(.*) (.*)%", b)
|
||||
if match and fname:
|
||||
# print b
|
||||
type = match.group(1)
|
||||
m = int(match.group(2))
|
||||
n = int(match.group(3))
|
||||
if cfiles.has_key(fname):
|
||||
h = cfiles[fname]
|
||||
else:
|
||||
h = {}
|
||||
cfiles[fname] = h
|
||||
if h.has_key(type):
|
||||
(om,on) = h[type]
|
||||
if percent(m,n) > percent(om,on):
|
||||
h[type] = (m,n)
|
||||
else:
|
||||
h[type] = (m,n)
|
||||
|
||||
summary = {}
|
||||
keys = cfiles.keys(); keys.sort()
|
||||
for k in keys:
|
||||
h = cfiles[k]
|
||||
print k
|
||||
ktypes = h.keys(); ktypes.sort()
|
||||
for t in ktypes:
|
||||
(m,n) = h[t]
|
||||
print "\t%s %d/%d %.2f%%" % (t, m, n, percent(m,n))
|
||||
if summary.has_key(t):
|
||||
(om,on) = summary[t]; summary[t] = (om+m,on+n)
|
||||
else:
|
||||
summary[t] = (m,n)
|
||||
print
|
||||
printhash("Summary", summary)
|
||||
|
||||
def printhash(s, h):
|
||||
print s
|
||||
ktypes = h.keys(); ktypes.sort()
|
||||
for t in ktypes:
|
||||
(m,n) = h[t]
|
||||
print "\t%s %d/%d %.2f%%" % (t, m, n, percent(m,n))
|
||||
|
||||
def main():
|
||||
global verbose
|
||||
for arg in sys.argv[1:]:
|
||||
if arg == "-v" or arg == "--verbose":
|
||||
verbose = 1
|
||||
sumarize()
|
||||
|
||||
sys.exit(main())
|
||||
Loading…
Add table
Reference in a new issue