Add option to run fillseq with WAL enabled in addition to WAL disabled
Summary: This set of changes is part of the work to introduce benchmark for universal style compaction in RocksDB. It's conceptually separate from the compaction work, so sending it out as a separate diff to get it out of the way. Test Plan: - Run `./tools/run_flash_bench.sh`. - Look at the contents of `report.txt` and `report2.txt` to make sure that data is reported and attributed correctly. - During `db_bench` execution time make sure that the correct flags are passed to `--disable_wal` depending on the benchmark being executed. Reviewers: MarkCallaghan Reviewed By: MarkCallaghan Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D53865
This commit is contained in:
Gunnar Kudrjavets
parent
73a9b0f4ba
commit
8ed3438778
@ -109,6 +109,10 @@ function summarize_result {
|
||||
test_name=$2
|
||||
bench_name=$3
|
||||
|
||||
# Note that this function assumes that the benchmark executes long enough so
|
||||
# that "Compaction Stats" is written to stdout at least once. If it won't
|
||||
# happen then empty output from grep when searching for "Sum" will cause
|
||||
# syntax errors.
|
||||
uptime=$( grep ^Uptime\(secs $test_out | tail -1 | awk '{ printf "%.0f", $2 }' )
|
||||
stall_time=$( grep "^Cumulative stall" $test_out | tail -1 | awk '{ print $3 }' )
|
||||
stall_pct=$( grep "^Cumulative stall" $test_out| tail -1 | awk '{ print $5 }' )
|
||||
@ -159,8 +163,22 @@ function run_bulkload {
|
||||
}
|
||||
|
||||
function run_fillseq {
|
||||
# This runs with a vector memtable and the WAL disabled to load faster. It is still crash safe and the
|
||||
# client can discover where to restart a load after a crash. I think this is a good way to load.
|
||||
# This runs with a vector memtable. WAL can be either disabled or enabled
|
||||
# depending on the input parameter (1 for disabled, 0 for enabled). The main
|
||||
# benefit behind disabling WAL is to make loading faster. It is still crash
|
||||
# safe and the client can discover where to restart a load after a crash. I
|
||||
# think this is a good way to load.
|
||||
|
||||
# Make sure that we'll have unique names for all the files so that data won't
|
||||
# be overwritten.
|
||||
if [ $1 == 1 ]; then
|
||||
log_file_name=$output_dir/benchmark_fillseq.wal_disabled.v${value_size}.log
|
||||
test_name=fillseq.wal_disabled.v${value_size}
|
||||
else
|
||||
log_file_name=$output_dir/benchmark_fillseq.wal_enabled.v${value_size}.log
|
||||
test_name=fillseq.wal_enabled.v${value_size}
|
||||
fi
|
||||
|
||||
echo "Loading $num_keys keys sequentially"
|
||||
cmd="./db_bench --benchmarks=fillseq \
|
||||
--use_existing_db=0 \
|
||||
@ -169,12 +187,14 @@ function run_fillseq {
|
||||
--min_level_to_compress=0 \
|
||||
--threads=1 \
|
||||
--memtablerep=vector \
|
||||
--disable_wal=1 \
|
||||
--disable_wal=$1 \
|
||||
--seed=$( date +%s ) \
|
||||
2>&1 | tee -a $output_dir/benchmark_fillseq.v${value_size}.log"
|
||||
echo $cmd | tee $output_dir/benchmark_fillseq.v${value_size}.log
|
||||
2>&1 | tee -a $log_file_name"
|
||||
echo $cmd | tee $log_file_name
|
||||
eval $cmd
|
||||
summarize_result $output_dir/benchmark_fillseq.v${value_size}.log fillseq.v${value_size} fillseq
|
||||
|
||||
# The constant "fillseq" which we pass to db_bench is the benchmark name.
|
||||
summarize_result $log_file_name $test_name fillseq
|
||||
}
|
||||
|
||||
function run_change {
|
||||
@ -310,8 +330,10 @@ for job in ${jobs[@]}; do
|
||||
start=$(now)
|
||||
if [ $job = bulkload ]; then
|
||||
run_bulkload
|
||||
elif [ $job = fillseq ]; then
|
||||
run_fillseq
|
||||
elif [ $job = fillseq_disable_wal ]; then
|
||||
run_fillseq 1
|
||||
elif [ $job = fillseq_enable_wal ]; then
|
||||
run_fillseq 0
|
||||
elif [ $job = overwrite ]; then
|
||||
run_change overwrite
|
||||
elif [ $job = updaterandom ]; then
|
||||
|
||||
@ -137,10 +137,17 @@ if [[ $do_setup != 0 ]]; then
|
||||
# Test 2a: sequential fill with large values to get peak ingest
|
||||
# adjust NUM_KEYS given the use of larger values
|
||||
env $ARGS BLOCK_SIZE=$((1 * M)) VALUE_SIZE=$((32 * K)) NUM_KEYS=$(( num_keys / 64 )) \
|
||||
./tools/benchmark.sh fillseq
|
||||
./tools/benchmark.sh fillseq_disable_wal
|
||||
|
||||
# Test 2b: sequential fill with the configured value size
|
||||
env $ARGS ./tools/benchmark.sh fillseq
|
||||
env $ARGS ./tools/benchmark.sh fillseq_disable_wal
|
||||
|
||||
# Test 2c: same as 2a, but with WAL being enabled.
|
||||
env $ARGS BLOCK_SIZE=$((1 * M)) VALUE_SIZE=$((32 * K)) NUM_KEYS=$(( num_keys / 64 )) \
|
||||
./tools/benchmark.sh fillseq_enable_wal
|
||||
|
||||
# Test 2d: same as 2b, but with WAL being enabled.
|
||||
env $ARGS ./tools/benchmark.sh fillseq_enable_wal
|
||||
|
||||
# Test 3: single-threaded overwrite
|
||||
env $ARGS NUM_THREADS=1 DB_BENCH_NO_SYNC=1 ./tools/benchmark.sh overwrite
|
||||
@ -263,9 +270,13 @@ if [[ $skip_low_pri_tests != 1 ]]; then
|
||||
grep bulkload $output_dir/report.txt >> $output_dir/report2.txt
|
||||
fi
|
||||
|
||||
echo fillseq >> $output_dir/report2.txt
|
||||
echo fillseq_wal_disabled >> $output_dir/report2.txt
|
||||
head -1 $output_dir/report.txt >> $output_dir/report2.txt
|
||||
grep fillseq $output_dir/report.txt >> $output_dir/report2.txt
|
||||
grep fillseq.wal_disabled $output_dir/report.txt >> $output_dir/report2.txt
|
||||
|
||||
echo fillseq_wal_enabled >> $output_dir/report2.txt
|
||||
head -1 $output_dir/report.txt >> $output_dir/report2.txt
|
||||
grep fillseq.wal_enabled $output_dir/report.txt >> $output_dir/report2.txt
|
||||
|
||||
echo overwrite sync=0 >> $output_dir/report2.txt
|
||||
head -1 $output_dir/report.txt >> $output_dir/report2.txt
|
||||
|
||||
Loading…
Reference in New Issue
Block a user