Add option to run fillseq with WAL enabled in addition to WAL disabled

Summary: This set of changes is part of the work to introduce benchmark for universal style compaction in RocksDB. It's conceptually separate from the compaction work, so sending it out as a separate diff to get it out of the way.

Test Plan:
  - Run `./tools/run_flash_bench.sh`.
  - Look at the contents of `report.txt` and `report2.txt` to make sure that data is reported and attributed correctly.
  - During `db_bench` execution time make sure that the correct flags are passed to `--disable_wal` depending on the benchmark being executed.

Reviewers: MarkCallaghan

Reviewed By: MarkCallaghan

Subscribers: dhruba, leveldb

Differential Revision: https://reviews.facebook.net/D53865
This commit is contained in:
Gunnar Kudrjavets 2016-02-05 13:20:56 -08:00
parent 73a9b0f4ba
commit 8ed3438778
2 changed files with 45 additions and 12 deletions

View File

@ -109,6 +109,10 @@ function summarize_result {
test_name=$2
bench_name=$3
# Note that this function assumes that the benchmark executes long enough so
# that "Compaction Stats" is written to stdout at least once. If it won't
# happen then empty output from grep when searching for "Sum" will cause
# syntax errors.
uptime=$( grep ^Uptime\(secs $test_out | tail -1 | awk '{ printf "%.0f", $2 }' )
stall_time=$( grep "^Cumulative stall" $test_out | tail -1 | awk '{ print $3 }' )
stall_pct=$( grep "^Cumulative stall" $test_out| tail -1 | awk '{ print $5 }' )
@ -159,8 +163,22 @@ function run_bulkload {
}
function run_fillseq {
# This runs with a vector memtable and the WAL disabled to load faster. It is still crash safe and the
# client can discover where to restart a load after a crash. I think this is a good way to load.
# This runs with a vector memtable. WAL can be either disabled or enabled
# depending on the input parameter (1 for disabled, 0 for enabled). The main
# benefit behind disabling WAL is to make loading faster. It is still crash
# safe and the client can discover where to restart a load after a crash. I
# think this is a good way to load.
# Make sure that we'll have unique names for all the files so that data won't
# be overwritten.
if [ $1 == 1 ]; then
log_file_name=$output_dir/benchmark_fillseq.wal_disabled.v${value_size}.log
test_name=fillseq.wal_disabled.v${value_size}
else
log_file_name=$output_dir/benchmark_fillseq.wal_enabled.v${value_size}.log
test_name=fillseq.wal_enabled.v${value_size}
fi
echo "Loading $num_keys keys sequentially"
cmd="./db_bench --benchmarks=fillseq \
--use_existing_db=0 \
@ -169,12 +187,14 @@ function run_fillseq {
--min_level_to_compress=0 \
--threads=1 \
--memtablerep=vector \
--disable_wal=1 \
--disable_wal=$1 \
--seed=$( date +%s ) \
2>&1 | tee -a $output_dir/benchmark_fillseq.v${value_size}.log"
echo $cmd | tee $output_dir/benchmark_fillseq.v${value_size}.log
2>&1 | tee -a $log_file_name"
echo $cmd | tee $log_file_name
eval $cmd
summarize_result $output_dir/benchmark_fillseq.v${value_size}.log fillseq.v${value_size} fillseq
# The constant "fillseq" which we pass to db_bench is the benchmark name.
summarize_result $log_file_name $test_name fillseq
}
function run_change {
@ -310,8 +330,10 @@ for job in ${jobs[@]}; do
start=$(now)
if [ $job = bulkload ]; then
run_bulkload
elif [ $job = fillseq ]; then
run_fillseq
elif [ $job = fillseq_disable_wal ]; then
run_fillseq 1
elif [ $job = fillseq_enable_wal ]; then
run_fillseq 0
elif [ $job = overwrite ]; then
run_change overwrite
elif [ $job = updaterandom ]; then

View File

@ -137,10 +137,17 @@ if [[ $do_setup != 0 ]]; then
# Test 2a: sequential fill with large values to get peak ingest
# adjust NUM_KEYS given the use of larger values
env $ARGS BLOCK_SIZE=$((1 * M)) VALUE_SIZE=$((32 * K)) NUM_KEYS=$(( num_keys / 64 )) \
./tools/benchmark.sh fillseq
./tools/benchmark.sh fillseq_disable_wal
# Test 2b: sequential fill with the configured value size
env $ARGS ./tools/benchmark.sh fillseq
env $ARGS ./tools/benchmark.sh fillseq_disable_wal
# Test 2c: same as 2a, but with WAL being enabled.
env $ARGS BLOCK_SIZE=$((1 * M)) VALUE_SIZE=$((32 * K)) NUM_KEYS=$(( num_keys / 64 )) \
./tools/benchmark.sh fillseq_enable_wal
# Test 2d: same as 2b, but with WAL being enabled.
env $ARGS ./tools/benchmark.sh fillseq_enable_wal
# Test 3: single-threaded overwrite
env $ARGS NUM_THREADS=1 DB_BENCH_NO_SYNC=1 ./tools/benchmark.sh overwrite
@ -263,9 +270,13 @@ if [[ $skip_low_pri_tests != 1 ]]; then
grep bulkload $output_dir/report.txt >> $output_dir/report2.txt
fi
echo fillseq >> $output_dir/report2.txt
echo fillseq_wal_disabled >> $output_dir/report2.txt
head -1 $output_dir/report.txt >> $output_dir/report2.txt
grep fillseq $output_dir/report.txt >> $output_dir/report2.txt
grep fillseq.wal_disabled $output_dir/report.txt >> $output_dir/report2.txt
echo fillseq_wal_enabled >> $output_dir/report2.txt
head -1 $output_dir/report.txt >> $output_dir/report2.txt
grep fillseq.wal_enabled $output_dir/report.txt >> $output_dir/report2.txt
echo overwrite sync=0 >> $output_dir/report2.txt
head -1 $output_dir/report.txt >> $output_dir/report2.txt