rocksdb/tools/regression_test.sh
Yueh-Hsuan Chiang fca5aa6fcc Initial script for the new regression test
Summary:
This diff includes an initial script running a set of benchmarks for
regression test.  The script does the following things:

  checkout the specified rocksdb commit (or origin/master as default)
  make clean && DEBUG_LEVEL=0 make db_bench
  setup test directories
  run set of benchmarks and store results

Currently, the script will run couple benchmarks, store all the benchmark
output, extract micros per op and percentile information for each benchmark
and store them in a single SUMMARY.csv file.  The SUMMARY.csv will make the
follow-up regression detection easier.

In addition, the current script only takes env arguments to set important
attributes of db_bench.  Will follow-up with a patch that allows db_bench
to construct options from an options file.

Test Plan:
NUM_KEYS=100 ./tools/regression_test.sh

  Sample SUMMARY.csv file:

                                     commit id,                      benchmark,  ms-per-op,        p50,        p75,        p99,      p99.9,     p99.99
      7e23ddf575890510e7d2fc7a79b31a1bbf317917,                        fillseq,      15.28,      54.66,      77.14,    5000.00,   17900.00,   18483.00
      7e23ddf575890510e7d2fc7a79b31a1bbf317917,                      overwrite,      13.54,      57.69,      86.39,    3000.00,   15600.00,   17013.00
      7e23ddf575890510e7d2fc7a79b31a1bbf317917,                     readrandom,       1.04,       0.80,       1.67,     293.33,     395.00,     504.00
      7e23ddf575890510e7d2fc7a79b31a1bbf317917,               readwhilewriting,       2.75,       1.01,       1.87,     200.00,     460.00,     485.00
      7e23ddf575890510e7d2fc7a79b31a1bbf317917,                   deleterandom,       3.64,      48.12,      70.09,     200.00,     336.67,     347.00
      7e23ddf575890510e7d2fc7a79b31a1bbf317917,                     seekrandom,      24.31,     391.87,     513.69,     872.73,     990.00,    1048.00
      7e23ddf575890510e7d2fc7a79b31a1bbf317917,         seekrandomwhilewriting,      14.02,     185.14,     294.15,     700.00,    1440.00,    1527.00

Reviewers: sdong, IslamAbdelRahman, kradhakrishnan, yiwu, andrewkr, gunnarku

Reviewed By: gunnarku

Subscribers: gunnarku, MarkCallaghan, andrewkr, dhruba, leveldb

Differential Revision: https://reviews.facebook.net/D57597
2016-05-09 13:32:57 -07:00

177 lines
4.8 KiB
Bash
Executable File

#!/bin/bash
# REQUIRE: db_bench binary exists in the current directory
function main {
commit=${1:-"origin/master"}
test_root_dir=${2:-"/tmp/rocksdb/regression_test"}
init_arguments
checkout_rocksdb $commit
build_db_bench
setup_test_directory
# an additional dot indicates we share same env variables
run_db_bench "fillseq" 0
run_db_bench "overwrite"
run_db_bench "readrandom"
run_db_bench "readwhilewriting"
run_db_bench "deleterandom"
run_db_bench "seekrandom"
run_db_bench "seekrandomwhilewriting"
echo ""
echo "Benchmark completed! Results are available in $RESULT_PATH"
}
############################################################################
function init_arguments {
K=1024
M=$((1024 * K))
G=$((1024 * M))
current_time=$(date +"%F-%H:%M:%S")
RESULT_PATH=${4:-"/tmp/rocksdb/regression/results/$current_time"}
COMMIT_ID=`git log | head -n1 | cut -c 8-`
SUMMARY_FILE="$RESULT_PATH/SUMMARY.csv"
DB_PATH=${2:-"/tmp/rocksdb/regression/db/"}
WAL_PATH=${3:-"/tmp/rocksdb/regression/wal/"}
NUM_THREADS=${NUM_THREADS:-16}
NUM_KEYS=${NUM_KEYS:-$((1 * G))}
KEY_SIZE=${KEY_SIZE:-100}
VALUE_SIZE=${VALUE_SIZE:-900}
CACHE_SIZE=${CACHE_SIZE:-$((1 * G))}
STATISTICS=${STATISTICS:-0}
COMPRESSION_RATIO=${COMPRESSION_RATIO:-0.5}
HISTOGRAM=${HISTOGRAM:-1}
STATS_PER_INTERVAL=${STATS_PER_INTERVAL:-1}
STATS_INTERVAL_SECONDS=${STATS_INTERVAL_SECONDS:-60}
MAX_BACKGROUND_FLUSHES=${MAX_BACKGROUND_FLUSHES:-4}
MAX_BACKGROUND_COMPACTIONS=${MAX_BACKGROUND_COMPACTIONS:-16}
SEEK_NEXTS=${SEEK_NEXTS:-10}
SEED=${SEED:-$( date +%s )}
}
# $1 --- benchmark name
# $2 --- use_existing_db (optional)
function run_db_bench {
USE_EXISTING_DB=${2:-1}
echo ""
echo "======================================================================="
echo "Benchmark $1"
echo "======================================================================="
echo ""
db_bench_error=0
cmd="(./db_bench --benchmarks=$1 --db=$DB_PATH --wal_dir=$WAL_PATH \
--use_existing_db=$USE_EXISTING_DB \
--threads=$NUM_THREADS \
--num=$NUM_KEYS \
--key_size=$KEY_SIZE \
--value_size=$VALUE_SIZE \
--cache_size=$CACHE_SIZE \
--statistics=$STATISTICS \
--compression_ratio=$COMPRESSION_RATIO \
--histogram=$HISTOGRAM \
--seek_nexts=$SEEK_NEXTS \
--stats_per_interval=$STATS_PER_INTERVAL \
--stats_interval_seconds=$STATS_INTERVAL_SECONDS \
--max_background_flushes=$MAX_BACKGROUND_FLUSHES \
--max_background_compactions=$MAX_BACKGROUND_COMPACTIONS \
--seed=$SEED \
2>&1 || db_bench_error=1) | tee -a $RESULT_PATH/$1"
echo $cmd
eval $cmd
exit_on_error $db_bench_error
update_report "$1" "$RESULT_PATH/$1"
}
# $1 --- name of the benchmark
# $2 --- the filename of the output log of db_bench
function update_report {
main_result=`cat $2 | grep $1`
perc_statement=`cat $2 | grep Percentile`
# Obtain micros / op
main_pattern="$1"'[[:blank:]]+:[[:blank:]]+([0-9\.]+)[[:blank:]]+micros/op'
[[ $main_result =~ $main_pattern ]]
micros_op=${BASH_REMATCH[1]}
# Obtain percentile information
perc_pattern='Percentiles: P50: ([0-9\.]+) P75: ([0-9\.]+) P99: ([0-9\.]+) P99.9: ([0-9\.]+) P99.99: ([0-9\.]+)'
[[ $perc_statement =~ $perc_pattern ]]
perc[0]=${BASH_REMATCH[1]} # p50
perc[1]=${BASH_REMATCH[2]} # p75
perc[2]=${BASH_REMATCH[3]} # p99
perc[3]=${BASH_REMATCH[4]} # p99.9
perc[4]=${BASH_REMATCH[5]} # p99.99
printf "$COMMIT_ID, %30s, %10.2f, %10.2f, %10.2f, %10.2f, %10.2f, %10.2f\n" \
$1 $micros_op ${perc[0]} ${perc[1]} ${perc[2]} ${perc[3]} ${perc[4]} \
>> $SUMMARY_FILE
exit_on_error $?
}
function exit_on_error {
if [ $1 -ne 0 ]; then
echo ""
echo "ERROR: Benchmark did not complete successfully. " \
"Partial results are output to $RESULT_PATH"
echo "ERROR" >> $SUMMARY_FILE
exit $1
fi
}
function checkout_rocksdb {
echo "Checking out commit $1 ..."
git fetch --all
exit_on_error $?
git checkout $1
exit_on_error $?
}
function build_db_bench {
echo "Building db_bench ..."
make clean
exit_on_error $?
DEBUG_LEVEL=0 make db_bench -j32
exit_on_error $?
}
function setup_test_directory {
echo "Deleting old regression test directories and creating new ones"
rm -rf "$DB_PATH"
exit_on_error $?
rm -rf "$WAL_PATH"
exit_on_error $?
rm -rf "$RESULT_PATH"
exit_on_error $?
mkdir -p "$DB_PATH"
exit_on_error $?
mkdir -p "$WAL_PATH"
exit_on_error $?
mkdir -p "$RESULT_PATH"
exit_on_error $?
printf "%40s, %30s, %10s, %10s, %10s, %10s, %10s, %10s\n" \
"commit id" "benchmark" "ms-per-op" "p50" "p75" "p99" "p99.9" "p99.99" \
$micros_op ${perc[0]} ${perc[1]} ${perc[2]} ${perc[3]} ${perc[4]} \
>> $SUMMARY_FILE
}
############################################################################
main $@