andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Configurable compression in db_bench

Browse Source 
Summary:
Made compression type and dictionary size configurable via environment
variables.

Depends on D52287.

Test Plan:
check these options are passed to the db.

  $ COMPRESSION_MAX_DICT_BYTES=65536 COMPRESSION_TYPE=LZ4 NUM_KEYS=10000000 DB_DIR=./tmp/ WAL_DIR=./tmp/ ./tools/benchmark.sh filluniquerandom
  ...
  $ grep Options.compression tmp/LOG
  2016/04/22-19:11:30.397829 7f5f263a2980          Options.compression: LZ4
  ...
  2016/04/22-19:11:30.397837 7f5f263a2980         Options.compression_opts.max_dict_bytes: 65536

Reviewers: IslamAbdelRahman, sdong

Reviewed By: sdong

Subscribers: andrewkr, dhruba, leveldb

Differential Revision: https://reviews.facebook.net/D57141
This commit is contained in:
Andrew Kryczka 2016-04-27 17:39:18 -07:00
parent 843d2e3137
commit 4032145adc
2 changed files with 9 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
tools/benchmark.sh
View File

@ -49,6 +49,8 @@ mb_written_per_sec=${MB_WRITE_PER_SEC:-0}
# Only for tests that do range scans # Only for tests that do range scans
num_nexts_per_seek=${NUM_NEXTS_PER_SEEK:-10} num_nexts_per_seek=${NUM_NEXTS_PER_SEEK:-10}
cache_size=${CACHE_SIZE:-$((1 * G))} cache_size=${CACHE_SIZE:-$((1 * G))}
compression_max_dict_bytes=${COMPRESSION_MAX_DICT_BYTES:-0}
compression_type=${COMPRESSION_TYPE:-snappy}
duration=${DURATION:-0} duration=${DURATION:-0}
num_keys=${NUM_KEYS:-$((1 * G))} num_keys=${NUM_KEYS:-$((1 * G))}
@ -68,9 +70,9 @@ const_params="
--block_size=$block_size \ --block_size=$block_size \
--cache_size=$cache_size \ --cache_size=$cache_size \
--cache_numshardbits=6 \ --cache_numshardbits=6 \
--compression_type=snappy \ --compression_max_dict_bytes=$compression_max_dict_bytes \
--min_level_to_compress=3 \
--compression_ratio=0.5 \ --compression_ratio=0.5 \
--compression_type=$compression_type \
--level_compaction_dynamic_level_bytes=true \ --level_compaction_dynamic_level_bytes=true \
--bytes_per_sync=$((8 * M)) \ --bytes_per_sync=$((8 * M)) \
--cache_index_and_filter_blocks=0 \ --cache_index_and_filter_blocks=0 \

5
tools/db_bench_tool.cc
View File

@ -562,6 +562,10 @@ DEFINE_int32(compression_level, -1,
"Compression level. For zlib this should be -1 for the " "Compression level. For zlib this should be -1 for the "
"default level, or between 0 and 9."); "default level, or between 0 and 9.");
DEFINE_int32(compression_max_dict_bytes, 0,
"Maximum size of dictionary used to prime the compression "
"library.");
static bool ValidateCompressionLevel(const char* flagname, int32_t value) { static bool ValidateCompressionLevel(const char* flagname, int32_t value) {
if (value < -1 || value > 9) { if (value < -1 || value > 9) {
fprintf(stderr, "Invalid value for --%s: %d, must be between -1 and 9\n", fprintf(stderr, "Invalid value for --%s: %d, must be between -1 and 9\n",
@ -2556,6 +2560,7 @@ class Benchmark {
FLAGS_level0_slowdown_writes_trigger; FLAGS_level0_slowdown_writes_trigger;
options.compression = FLAGS_compression_type_e; options.compression = FLAGS_compression_type_e;
options.compression_opts.level = FLAGS_compression_level; options.compression_opts.level = FLAGS_compression_level;
options.compression_opts.max_dict_bytes = FLAGS_compression_max_dict_bytes;
options.WAL_ttl_seconds = FLAGS_wal_ttl_seconds; options.WAL_ttl_seconds = FLAGS_wal_ttl_seconds;
options.WAL_size_limit_MB = FLAGS_wal_size_limit_MB; options.WAL_size_limit_MB = FLAGS_wal_size_limit_MB;
options.max_total_wal_size = FLAGS_max_total_wal_size; options.max_total_wal_size = FLAGS_max_total_wal_size;