Compare commits
16 Commits
Author | SHA1 | Date | |
---|---|---|---|
|
4d6ba85469 | ||
|
c1ba8bee23 | ||
|
f6acd9661d | ||
|
06b24a3eb3 | ||
|
1a79de2ae0 | ||
|
f1b6077755 | ||
|
8f52326bd5 | ||
|
70ebf2275b | ||
|
8fffeb27d5 | ||
|
8df2bc008e | ||
|
ec03b510a6 | ||
|
e5d21bfbf1 | ||
|
e189801f76 | ||
|
fd728c781c | ||
|
7761a5faa2 | ||
|
027f9f8cf6 |
@ -399,6 +399,7 @@ set(TESTS
|
||||
util/iostats_context_test.cc
|
||||
util/memenv_test.cc
|
||||
util/mock_env_test.cc
|
||||
util/options_settable_test.cc
|
||||
util/options_test.cc
|
||||
util/rate_limiter_test.cc
|
||||
util/slice_transform_test.cc
|
||||
|
@ -1,4 +1,5 @@
|
||||
## Unreleased
|
||||
# RocksDB default options change log
|
||||
## 4.7.0 (4/8/2016)
|
||||
* options.write_buffer_size changes from 4MB to 64MB
|
||||
* options.target_file_size_base changes from 2MB to 64MB
|
||||
* options.max_bytes_for_level_base changes from 10MB to 256MB
|
||||
|
@ -1,6 +1,7 @@
|
||||
# Rocksdb Change Log
|
||||
## Unreleased
|
||||
## 4.7.0 (4/8/2016)
|
||||
### Public API Change
|
||||
* rename options compaction_measure_io_stats to report_bg_io_stats and include flush too.
|
||||
* Change some default options. Now default options will optimize for server-workloads. Also enable slowdown and full stop triggers for pending compaction bytes. These changes may cause sub-optimal performance or significant increase of resource usage. To avoid these risks, users can open existing RocksDB with options extracted from RocksDB option files. See https://github.com/facebook/rocksdb/wiki/RocksDB-Options-File for how to use RocksDB option files. Or you can call Options.OldDefaults() to recover old defaults. DEFAULT_OPTIONS_HISTORY.md will track change history of default options.
|
||||
|
||||
## 4.6.0 (3/10/2016)
|
||||
|
10
Makefile
10
Makefile
@ -200,10 +200,6 @@ default: all
|
||||
WARNING_FLAGS = -W -Wextra -Wall -Wsign-compare -Wshadow \
|
||||
-Wno-unused-parameter
|
||||
|
||||
ifndef DISABLE_WARNING_AS_ERROR
|
||||
WARNING_FLAGS += -Werror
|
||||
endif
|
||||
|
||||
CFLAGS += $(WARNING_FLAGS) -I. -I./include $(PLATFORM_CCFLAGS) $(OPT)
|
||||
CXXFLAGS += $(WARNING_FLAGS) -I. -I./include $(PLATFORM_CXXFLAGS) $(OPT) -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers
|
||||
|
||||
@ -330,6 +326,7 @@ TESTS = \
|
||||
rate_limiter_test \
|
||||
delete_scheduler_test \
|
||||
options_test \
|
||||
options_settable_test \
|
||||
options_util_test \
|
||||
event_logger_test \
|
||||
cuckoo_table_builder_test \
|
||||
@ -637,7 +634,7 @@ ubsan_crash_test:
|
||||
$(MAKE) clean
|
||||
|
||||
valgrind_check: $(TESTS)
|
||||
for t in $(filter-out %skiplist_test,$(TESTS)); do \
|
||||
for t in $(filter-out %skiplist_test options_settable_test,$(TESTS)); do \
|
||||
$(VALGRIND_VER) $(VALGRIND_OPTS) ./$$t; \
|
||||
ret_code=$$?; \
|
||||
if [ $$ret_code -ne 0 ]; then \
|
||||
@ -1024,6 +1021,9 @@ compact_files_test: db/compact_files_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
options_test: util/options_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(AM_LINK)
|
||||
|
||||
options_settable_test: util/options_settable_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(AM_LINK)
|
||||
|
||||
options_util_test: utilities/options/options_util_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(AM_LINK)
|
||||
|
||||
|
@ -52,12 +52,7 @@ if [ -z "$ROCKSDB_NO_FBCODE" -a -d /mnt/gvfs/third-party ]; then
|
||||
FBCODE_BUILD="true"
|
||||
# If we're compiling with TSAN we need pic build
|
||||
PIC_BUILD=$COMPILE_WITH_TSAN
|
||||
if [ -z "$ROCKSDB_FBCODE_BUILD_WITH_481" ]; then
|
||||
source "$PWD/build_tools/fbcode_config.sh"
|
||||
else
|
||||
# we need this to build with MySQL. Don't use for other purposes.
|
||||
source "$PWD/build_tools/fbcode_config4.8.1.sh"
|
||||
fi
|
||||
source "$PWD/build_tools/fbcode_config.sh"
|
||||
fi
|
||||
|
||||
# Delete existing output, if it exists
|
||||
|
@ -1,16 +1,19 @@
|
||||
GCC_BASE=/mnt/vol/engshare/fbcode/third-party2/gcc/4.9.x/centos6-native/1317bc4/
|
||||
CLANG_BASE=/mnt/gvfs/third-party2/clang/fc904e50a9266b9d7b98cae1993afa0c5aae1440/3.7.1/centos6-native/9d9ecb9/
|
||||
LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/f97108c12512b3b0789ac4515d836bdb1eae1142/4.9.x/gcc-4.9-glibc-2.20/024dbc3
|
||||
GLIBC_BASE=/mnt/gvfs/third-party2/glibc/f5484f168c0e4d19823d41df052c5870c6e575a4/2.20/gcc-4.9-glibc-2.20/500e281
|
||||
SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/cbf6f1f209e5bd160bdc5d971744e039f36b1566/1.1.3/gcc-4.9-glibc-2.20/e9936bf
|
||||
ZLIB_BASE=/mnt/gvfs/third-party2/zlib/6d39cb54708049f527e713ad19f2aadb9d3667e8/1.2.8/gcc-4.9-glibc-2.20/e9936bf
|
||||
BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/2ddd45f0853bfc8bb1c27f0f447236a1a26c338a/1.0.6/gcc-4.9-glibc-2.20/e9936bf
|
||||
LZ4_BASE=/mnt/gvfs/third-party2/lz4/6858fac689e0f92e584224d91bdb0e39f6c8320d/r131/gcc-4.9-glibc-2.20/e9936bf
|
||||
ZSTD_BASE=/mnt/gvfs/third-party2/zstd/cb6c4880fcb4fee471574ba6af63a3882155a16a/0.5.1/gcc-4.9-glibc-2.20/e9936bf
|
||||
GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/c7275a4ceae0aca0929e56964a31dafc53c1ee96/2.1.1/gcc-4.8.1-glibc-2.17/c3f970a
|
||||
JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/d9acac8a9a2c9378eb696e22ffa8bd0e97d9206b/master/gcc-4.9-glibc-2.20/a6c5e1e
|
||||
NUMA_BASE=/mnt/gvfs/third-party2/numa/ae54a5ed22cdabb1c6446dce4e8ffae5b4446d73/2.0.8/gcc-4.9-glibc-2.20/e9936bf
|
||||
LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/41bfa3759ce52c071f5fd547ec9ecd2522929f0a/trunk/gcc-4.9-glibc-2.20/12266b1
|
||||
KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/1a48835975c66d30e47770ec419758ed3b9ba010/3.10.62-62_fbk17_03959_ge29cc63/gcc-4.9-glibc-2.20/da39a3e
|
||||
BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/71454c53fffcb716a0beb9a90047aff7fb5c984a/2.26/centos6-native/da39a3e
|
||||
VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/af85c56f424cd5edfc2c97588299b44ecdec96bb/3.10.0/gcc-4.9-glibc-2.20/e9936bf
|
||||
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
|
||||
GCC_BASE=/mnt/gvfs/third-party2/gcc/7331085db891a2ef4a88a48a751d834e8d68f4cb/7.x/centos7-native/b2ef2b6
|
||||
CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/963d9aeda70cc4779885b1277484fe7544a04e3e/9.0.0/platform007/9e92d53/
|
||||
LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/6ace84e956873d53638c738b6f65f3f469cca74c/7.x/platform007/5620abc
|
||||
GLIBC_BASE=/mnt/gvfs/third-party2/glibc/192b0f42d63dcf6210d6ceae387b49af049e6e0c/2.26/platform007/f259413
|
||||
SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/7f9bdaada18f59bc27ec2b0871eb8a6144343aef/1.1.3/platform007/ca4da3d
|
||||
ZLIB_BASE=/mnt/gvfs/third-party2/zlib/2d9f0b9a4274cc21f61272a9e89bdb859bce8f1f/1.2.8/platform007/ca4da3d
|
||||
BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/dc49a21c5fceec6456a7a28a94dcd16690af1337/1.0.6/platform007/ca4da3d
|
||||
LZ4_BASE=/mnt/gvfs/third-party2/lz4/0f607f8fc442ea7d6b876931b1898bb573d5e5da/1.9.1/platform007/ca4da3d
|
||||
ZSTD_BASE=/mnt/gvfs/third-party2/zstd/ca22bc441a4eb709e9e0b1f9fec9750fed7b31c5/1.4.x/platform007/15a3614
|
||||
GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/0b9929d2588991c65a57168bf88aff2db87c5d48/2.2.0/platform007/ca4da3d
|
||||
JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/c26f08f47ac35fc31da2633b7da92d6b863246eb/master/platform007/c26c002
|
||||
NUMA_BASE=/mnt/gvfs/third-party2/numa/3f3fb57a5ccc5fd21c66416c0b83e0aa76a05376/2.0.11/platform007/ca4da3d
|
||||
LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/40c73d874898b386a71847f1b99115d93822d11f/1.4/platform007/6f3e0a9
|
||||
TBB_BASE=/mnt/gvfs/third-party2/tbb/4ce8e8dba77cdbd81b75d6f0c32fd7a1b76a11ec/2018_U5/platform007/ca4da3d
|
||||
KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/fb251ecd2f5ae16f8671f7014c246e52a748fe0b/fb/platform007/da39a3e
|
||||
BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/ab9f09bba370e7066cafd4eb59752db93f2e8312/2.29.1/platform007/15a3614
|
||||
VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/d42d152a15636529b0861ec493927200ebebca8e/3.15.0/platform007/ca4da3d
|
||||
LUA_BASE=/mnt/gvfs/third-party2/lua/f0cd714433206d5139df61659eb7b28b1dea6683/5.3.4/platform007/5007832
|
||||
|
@ -1,4 +1,4 @@
|
||||
GCC_BASE=/mnt/vol/engshare/fbcode/third-party2/gcc/4.8.1/centos6-native/cc6c9dc/
|
||||
GCC_BASE=/mnt/gvfs/third-party2/gcc/ebc96bc2fb751b5a0300b8d91a95bdf24ac1d88b/4.8.1/centos6-native/cc6c9dc
|
||||
CLANG_BASE=/mnt/gvfs/third-party2/clang/fc904e50a9266b9d7b98cae1993afa0c5aae1440/3.7.1/centos6-native/9d9ecb9/
|
||||
LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/f97108c12512b3b0789ac4515d836bdb1eae1142/4.8.1/gcc-4.8.1-glibc-2.17/8aac7fc
|
||||
GLIBC_BASE=/mnt/gvfs/third-party2/glibc/f5484f168c0e4d19823d41df052c5870c6e575a4/2.17/gcc-4.8.1-glibc-2.17/99df8fc
|
||||
|
@ -13,12 +13,12 @@ source "$BASEDIR/dependencies.sh"
|
||||
CFLAGS=""
|
||||
|
||||
# libgcc
|
||||
LIBGCC_INCLUDE="$LIBGCC_BASE/include"
|
||||
LIBGCC_LIBS=" -L $LIBGCC_BASE/libs"
|
||||
LIBGCC_INCLUDE="$LIBGCC_BASE/include/c++/7.3.0"
|
||||
LIBGCC_LIBS=" -L $LIBGCC_BASE/lib"
|
||||
|
||||
# glibc
|
||||
GLIBC_INCLUDE="$GLIBC_BASE/include"
|
||||
GLIBC_LIBS=" -L $GLIB_BASE/lib"
|
||||
GLIBC_LIBS=" -L $GLIBC_BASE/lib"
|
||||
|
||||
# snappy
|
||||
SNAPPY_INCLUDE=" -I $SNAPPY_BASE/include/"
|
||||
@ -43,12 +43,16 @@ if test -z $PIC_BUILD; then
|
||||
LZ4_INCLUDE=" -I $LZ4_BASE/include/"
|
||||
LZ4_LIBS=" $LZ4_BASE/lib/liblz4.a"
|
||||
CFLAGS+=" -DLZ4"
|
||||
|
||||
ZSTD_INCLUDE=" -I $ZSTD_BASE/include/"
|
||||
ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd.a"
|
||||
CFLAGS+=" -DZSTD"
|
||||
fi
|
||||
|
||||
ZSTD_INCLUDE=" -I $ZSTD_BASE/include/"
|
||||
if test -z $PIC_BUILD; then
|
||||
ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd.a"
|
||||
else
|
||||
ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd_pic.a"
|
||||
fi
|
||||
CFLAGS+=" -DZSTD"
|
||||
|
||||
# location of gflags headers and libraries
|
||||
GFLAGS_INCLUDE=" -I $GFLAGS_BASE/include/"
|
||||
if test -z $PIC_BUILD; then
|
||||
@ -56,7 +60,7 @@ if test -z $PIC_BUILD; then
|
||||
else
|
||||
GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags_pic.a"
|
||||
fi
|
||||
CFLAGS+=" -DGFLAGS=google"
|
||||
CFLAGS+=" -DGFLAGS=gflags"
|
||||
|
||||
# location of jemalloc
|
||||
JEMALLOC_INCLUDE=" -I $JEMALLOC_BASE/include/"
|
||||
@ -72,13 +76,22 @@ if test -z $PIC_BUILD; then
|
||||
LIBUNWIND="$LIBUNWIND_BASE/lib/libunwind.a"
|
||||
fi
|
||||
|
||||
# location of TBB
|
||||
TBB_INCLUDE=" -isystem $TBB_BASE/include/"
|
||||
if test -z $PIC_BUILD; then
|
||||
TBB_LIBS="$TBB_BASE/lib/libtbb.a"
|
||||
else
|
||||
TBB_LIBS="$TBB_BASE/lib/libtbb_pic.a"
|
||||
fi
|
||||
CFLAGS+=" -DTBB"
|
||||
|
||||
# use Intel SSE support for checksum calculations
|
||||
export USE_SSE=1
|
||||
|
||||
BINUTILS="$BINUTILS_BASE/bin"
|
||||
AR="$BINUTILS/ar"
|
||||
|
||||
DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE"
|
||||
DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE $TBB_INCLUDE"
|
||||
|
||||
STDLIBS="-L $GCC_BASE/lib64"
|
||||
|
||||
@ -87,7 +100,7 @@ CLANG_LIB="$CLANG_BASE/lib"
|
||||
CLANG_SRC="$CLANG_BASE/../../src"
|
||||
|
||||
CLANG_ANALYZER="$CLANG_BIN/clang++"
|
||||
CLANG_SCAN_BUILD="$CLANG_SRC/clang/tools/scan-build/scan-build"
|
||||
CLANG_SCAN_BUILD="$CLANG_SRC/llvm/tools/clang/tools/scan-build/bin/scan-build"
|
||||
|
||||
if [ -z "$USE_CLANG" ]; then
|
||||
# gcc
|
||||
@ -95,39 +108,44 @@ if [ -z "$USE_CLANG" ]; then
|
||||
CXX="$GCC_BASE/bin/g++"
|
||||
|
||||
CFLAGS+=" -B$BINUTILS/gold"
|
||||
CFLAGS+=" -isystem $GLIBC_INCLUDE"
|
||||
CFLAGS+=" -isystem $LIBGCC_INCLUDE"
|
||||
CFLAGS+=" -isystem $GLIBC_INCLUDE"
|
||||
JEMALLOC=1
|
||||
else
|
||||
# clang
|
||||
CLANG_INCLUDE="$CLANG_LIB/clang/*/include"
|
||||
CLANG_INCLUDE="$CLANG_LIB/clang/stable/include"
|
||||
CC="$CLANG_BIN/clang"
|
||||
CXX="$CLANG_BIN/clang++"
|
||||
|
||||
KERNEL_HEADERS_INCLUDE="$KERNEL_HEADERS_BASE/include"
|
||||
|
||||
CFLAGS+=" -B$BINUTILS/gold -nostdinc -nostdlib"
|
||||
CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/4.9.x "
|
||||
CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/4.9.x/x86_64-facebook-linux "
|
||||
CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/7.x "
|
||||
CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/7.x/x86_64-facebook-linux "
|
||||
CFLAGS+=" -isystem $GLIBC_INCLUDE"
|
||||
CFLAGS+=" -isystem $LIBGCC_INCLUDE"
|
||||
CFLAGS+=" -isystem $CLANG_INCLUDE"
|
||||
CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE/linux "
|
||||
CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE "
|
||||
CFLAGS+=" -Wno-expansion-to-defined "
|
||||
CXXFLAGS="-nostdinc++"
|
||||
fi
|
||||
|
||||
CFLAGS+=" $DEPS_INCLUDE"
|
||||
CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE"
|
||||
CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT -DROCKSDB_SUPPORT_THREAD_LOCAL -DHAVE_SSE42"
|
||||
CXXFLAGS+=" $CFLAGS"
|
||||
|
||||
EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB"
|
||||
EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/gcc-4.9-glibc-2.20/lib/ld.so"
|
||||
EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB $TBB_LIBS"
|
||||
EXEC_LDFLAGS+=" -B$BINUTILS/gold"
|
||||
EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/platform007/lib/ld.so"
|
||||
EXEC_LDFLAGS+=" $LIBUNWIND"
|
||||
EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/gcc-4.9-glibc-2.20/lib"
|
||||
EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/platform007/lib"
|
||||
# required by libtbb
|
||||
EXEC_LDFLAGS+=" -ldl"
|
||||
|
||||
PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS $STDLIBS -lgcc -lstdc++"
|
||||
|
||||
EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS"
|
||||
EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $TBB_LIBS"
|
||||
|
||||
VALGRIND_VER="$VALGRIND_BASE/bin/"
|
||||
|
||||
|
@ -64,7 +64,7 @@ touch "$OUTPUT"
|
||||
echo "Writing dependencies to $OUTPUT"
|
||||
|
||||
# Compilers locations
|
||||
GCC_BASE=`ls -d1 $TP2_LATEST/gcc/4.9.x/centos6-native/*/ | head -n1`
|
||||
GCC_BASE=`readlink -f $TP2_LATEST/gcc/4.9.x/centos6-native/*/`
|
||||
CLANG_BASE=`ls -d1 /mnt/gvfs/third-party2/clang/fc904e50a9266b9d7b98cae1993afa0c5aae1440/3.7.1/centos6-native/*/ | head -n1`
|
||||
|
||||
log_variable GCC_BASE
|
||||
@ -101,7 +101,7 @@ touch "$OUTPUT"
|
||||
echo "Writing 4.8.1 dependencies to $OUTPUT"
|
||||
|
||||
# Compilers locations
|
||||
GCC_BASE=`ls -d1 $TP2_LATEST/gcc/4.8.1/centos6-native/*/ | head -n1`
|
||||
GCC_BASE=`readlink -f $TP2_LATEST/gcc/4.8.1/centos6-native/*/`
|
||||
CLANG_BASE=`ls -d1 /mnt/gvfs/third-party2/clang/fc904e50a9266b9d7b98cae1993afa0c5aae1440/3.7.1/centos6-native/*/ | head -n1`
|
||||
|
||||
log_variable GCC_BASE
|
||||
|
@ -660,7 +660,7 @@ TEST_P(CompactionJobStatsTest, CompactionJobStatsTest) {
|
||||
options.max_subcompactions = max_subcompactions_;
|
||||
options.bytes_per_sync = 512 * 1024;
|
||||
|
||||
options.compaction_measure_io_stats = true;
|
||||
options.report_bg_io_stats = true;
|
||||
for (int test = 0; test < 2; ++test) {
|
||||
DestroyAndReopen(options);
|
||||
CreateAndReopenWithCF({"pikachu"}, options);
|
||||
|
@ -1513,12 +1513,13 @@ Status DBImpl::FlushMemTableToOutputFile(
|
||||
std::vector<SequenceNumber> snapshot_seqs =
|
||||
snapshots_.GetAll(&earliest_write_conflict_snapshot);
|
||||
|
||||
FlushJob flush_job(
|
||||
dbname_, cfd, db_options_, mutable_cf_options, env_options_,
|
||||
versions_.get(), &mutex_, &shutting_down_, snapshot_seqs,
|
||||
earliest_write_conflict_snapshot, job_context, log_buffer,
|
||||
directories_.GetDbDir(), directories_.GetDataDir(0U),
|
||||
GetCompressionFlush(*cfd->ioptions()), stats_, &event_logger_);
|
||||
FlushJob flush_job(dbname_, cfd, db_options_, mutable_cf_options,
|
||||
env_options_, versions_.get(), &mutex_, &shutting_down_,
|
||||
snapshot_seqs, earliest_write_conflict_snapshot,
|
||||
job_context, log_buffer, directories_.GetDbDir(),
|
||||
directories_.GetDataDir(0U),
|
||||
GetCompressionFlush(*cfd->ioptions()), stats_,
|
||||
&event_logger_, mutable_cf_options.report_bg_io_stats);
|
||||
|
||||
FileMetaData file_meta;
|
||||
|
||||
@ -1867,7 +1868,7 @@ Status DBImpl::CompactFilesImpl(
|
||||
directories_.GetDataDir(c->output_path_id()), stats_, &mutex_, &bg_error_,
|
||||
snapshot_seqs, earliest_write_conflict_snapshot, table_cache_,
|
||||
&event_logger_, c->mutable_cf_options()->paranoid_file_checks,
|
||||
c->mutable_cf_options()->compaction_measure_io_stats, dbname_,
|
||||
c->mutable_cf_options()->report_bg_io_stats, dbname_,
|
||||
nullptr); // Here we pass a nullptr for CompactionJobStats because
|
||||
// CompactFiles does not trigger OnCompactionCompleted(),
|
||||
// which is the only place where CompactionJobStats is
|
||||
@ -3073,7 +3074,7 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
|
||||
&bg_error_, snapshot_seqs, earliest_write_conflict_snapshot,
|
||||
table_cache_, &event_logger_,
|
||||
c->mutable_cf_options()->paranoid_file_checks,
|
||||
c->mutable_cf_options()->compaction_measure_io_stats, dbname_,
|
||||
c->mutable_cf_options()->report_bg_io_stats, dbname_,
|
||||
&compaction_job_stats);
|
||||
compaction_job.Prepare();
|
||||
|
||||
|
@ -23,6 +23,68 @@ class DBTest2 : public DBTestBase {
|
||||
DBTest2() : DBTestBase("/db_test2") {}
|
||||
};
|
||||
|
||||
class PrefixFullBloomWithReverseComparator
|
||||
: public DBTestBase,
|
||||
public ::testing::WithParamInterface<bool> {
|
||||
public:
|
||||
PrefixFullBloomWithReverseComparator()
|
||||
: DBTestBase("/prefix_bloom_reverse") {}
|
||||
virtual void SetUp() override { if_cache_filter_ = GetParam(); }
|
||||
bool if_cache_filter_;
|
||||
};
|
||||
|
||||
TEST_P(PrefixFullBloomWithReverseComparator,
|
||||
PrefixFullBloomWithReverseComparator) {
|
||||
Options options = last_options_;
|
||||
options.comparator = ReverseBytewiseComparator();
|
||||
options.prefix_extractor.reset(NewCappedPrefixTransform(3));
|
||||
options.statistics = rocksdb::CreateDBStatistics();
|
||||
BlockBasedTableOptions bbto;
|
||||
if (if_cache_filter_) {
|
||||
bbto.no_block_cache = false;
|
||||
bbto.cache_index_and_filter_blocks = true;
|
||||
bbto.block_cache = NewLRUCache(1);
|
||||
}
|
||||
bbto.filter_policy.reset(NewBloomFilterPolicy(10, false));
|
||||
bbto.whole_key_filtering = false;
|
||||
options.table_factory.reset(NewBlockBasedTableFactory(bbto));
|
||||
DestroyAndReopen(options);
|
||||
|
||||
ASSERT_OK(dbfull()->Put(WriteOptions(), "bar123", "foo"));
|
||||
ASSERT_OK(dbfull()->Put(WriteOptions(), "bar234", "foo2"));
|
||||
ASSERT_OK(dbfull()->Put(WriteOptions(), "foo123", "foo3"));
|
||||
|
||||
dbfull()->Flush(FlushOptions());
|
||||
|
||||
if (bbto.block_cache) {
|
||||
bbto.block_cache->EraseUnRefEntries();
|
||||
}
|
||||
|
||||
unique_ptr<Iterator> iter(db_->NewIterator(ReadOptions()));
|
||||
iter->Seek("bar345");
|
||||
ASSERT_OK(iter->status());
|
||||
ASSERT_TRUE(iter->Valid());
|
||||
ASSERT_EQ("bar234", iter->key().ToString());
|
||||
ASSERT_EQ("foo2", iter->value().ToString());
|
||||
iter->Next();
|
||||
ASSERT_TRUE(iter->Valid());
|
||||
ASSERT_EQ("bar123", iter->key().ToString());
|
||||
ASSERT_EQ("foo", iter->value().ToString());
|
||||
|
||||
iter->Seek("foo234");
|
||||
ASSERT_OK(iter->status());
|
||||
ASSERT_TRUE(iter->Valid());
|
||||
ASSERT_EQ("foo123", iter->key().ToString());
|
||||
ASSERT_EQ("foo3", iter->value().ToString());
|
||||
|
||||
iter->Seek("bar");
|
||||
ASSERT_OK(iter->status());
|
||||
ASSERT_TRUE(!iter->Valid());
|
||||
}
|
||||
|
||||
INSTANTIATE_TEST_CASE_P(PrefixFullBloomWithReverseComparator,
|
||||
PrefixFullBloomWithReverseComparator, testing::Bool());
|
||||
|
||||
TEST_F(DBTest2, IteratorPropertyVersionNumber) {
|
||||
Put("", "");
|
||||
Iterator* iter1 = db_->NewIterator(ReadOptions());
|
||||
|
@ -303,6 +303,7 @@ Options DBTestBase::CurrentOptions(
|
||||
case kPerfOptions:
|
||||
options.soft_rate_limit = 2.0;
|
||||
options.delayed_write_rate = 8 * 1024 * 1024;
|
||||
options.report_bg_io_stats = true;
|
||||
// TODO(3.13) -- test more options
|
||||
break;
|
||||
case kDeletesFilterFirst:
|
||||
|
@ -66,7 +66,7 @@ FlushJob::FlushJob(const std::string& dbname, ColumnFamilyData* cfd,
|
||||
JobContext* job_context, LogBuffer* log_buffer,
|
||||
Directory* db_directory, Directory* output_file_directory,
|
||||
CompressionType output_compression, Statistics* stats,
|
||||
EventLogger* event_logger)
|
||||
EventLogger* event_logger, bool measure_io_stats)
|
||||
: dbname_(dbname),
|
||||
cfd_(cfd),
|
||||
db_options_(db_options),
|
||||
@ -83,7 +83,8 @@ FlushJob::FlushJob(const std::string& dbname, ColumnFamilyData* cfd,
|
||||
output_file_directory_(output_file_directory),
|
||||
output_compression_(output_compression),
|
||||
stats_(stats),
|
||||
event_logger_(event_logger) {
|
||||
event_logger_(event_logger),
|
||||
measure_io_stats_(measure_io_stats) {
|
||||
// Update the thread status to indicate flush.
|
||||
ReportStartedFlush();
|
||||
TEST_SYNC_POINT("FlushJob::FlushJob()");
|
||||
@ -121,6 +122,21 @@ void FlushJob::RecordFlushIOStats() {
|
||||
Status FlushJob::Run(FileMetaData* file_meta) {
|
||||
AutoThreadOperationStageUpdater stage_run(
|
||||
ThreadStatus::STAGE_FLUSH_RUN);
|
||||
// I/O measurement variables
|
||||
PerfLevel prev_perf_level = PerfLevel::kEnableTime;
|
||||
uint64_t prev_write_nanos = 0;
|
||||
uint64_t prev_fsync_nanos = 0;
|
||||
uint64_t prev_range_sync_nanos = 0;
|
||||
uint64_t prev_prepare_write_nanos = 0;
|
||||
if (measure_io_stats_) {
|
||||
prev_perf_level = GetPerfLevel();
|
||||
SetPerfLevel(PerfLevel::kEnableTime);
|
||||
prev_write_nanos = IOSTATS(write_nanos);
|
||||
prev_fsync_nanos = IOSTATS(fsync_nanos);
|
||||
prev_range_sync_nanos = IOSTATS(range_sync_nanos);
|
||||
prev_prepare_write_nanos = IOSTATS(prepare_write_nanos);
|
||||
}
|
||||
|
||||
// Save the contents of the earliest memtable as a new Table
|
||||
FileMetaData meta;
|
||||
autovector<MemTable*> mems;
|
||||
@ -180,6 +196,18 @@ Status FlushJob::Run(FileMetaData* file_meta) {
|
||||
}
|
||||
stream.EndArray();
|
||||
|
||||
if (measure_io_stats_) {
|
||||
if (prev_perf_level != PerfLevel::kEnableTime) {
|
||||
SetPerfLevel(prev_perf_level);
|
||||
}
|
||||
stream << "file_write_nanos" << (IOSTATS(write_nanos) - prev_write_nanos);
|
||||
stream << "file_range_sync_nanos"
|
||||
<< (IOSTATS(range_sync_nanos) - prev_range_sync_nanos);
|
||||
stream << "file_fsync_nanos" << (IOSTATS(fsync_nanos) - prev_fsync_nanos);
|
||||
stream << "file_prepare_write_nanos"
|
||||
<< (IOSTATS(prepare_write_nanos) - prev_prepare_write_nanos);
|
||||
}
|
||||
|
||||
return s;
|
||||
}
|
||||
|
||||
|
@ -62,7 +62,7 @@ class FlushJob {
|
||||
JobContext* job_context, LogBuffer* log_buffer,
|
||||
Directory* db_directory, Directory* output_file_directory,
|
||||
CompressionType output_compression, Statistics* stats,
|
||||
EventLogger* event_logger);
|
||||
EventLogger* event_logger, bool measure_io_stats);
|
||||
|
||||
~FlushJob();
|
||||
|
||||
@ -93,6 +93,7 @@ class FlushJob {
|
||||
Statistics* stats_;
|
||||
EventLogger* event_logger_;
|
||||
TableProperties table_properties_;
|
||||
bool measure_io_stats_;
|
||||
};
|
||||
|
||||
} // namespace rocksdb
|
||||
|
@ -93,7 +93,7 @@ TEST_F(FlushJobTest, Empty) {
|
||||
db_options_, *cfd->GetLatestMutableCFOptions(),
|
||||
env_options_, versions_.get(), &mutex_, &shutting_down_,
|
||||
{}, kMaxSequenceNumber, &job_context, nullptr, nullptr,
|
||||
nullptr, kNoCompression, nullptr, &event_logger);
|
||||
nullptr, kNoCompression, nullptr, &event_logger, false);
|
||||
ASSERT_OK(flush_job.Run());
|
||||
job_context.Clean();
|
||||
}
|
||||
@ -132,7 +132,7 @@ TEST_F(FlushJobTest, NonEmpty) {
|
||||
db_options_, *cfd->GetLatestMutableCFOptions(),
|
||||
env_options_, versions_.get(), &mutex_, &shutting_down_,
|
||||
{}, kMaxSequenceNumber, &job_context, nullptr, nullptr,
|
||||
nullptr, kNoCompression, nullptr, &event_logger);
|
||||
nullptr, kNoCompression, nullptr, &event_logger, true);
|
||||
FileMetaData fd;
|
||||
mutex_.Lock();
|
||||
ASSERT_OK(flush_job.Run(&fd));
|
||||
@ -192,11 +192,11 @@ TEST_F(FlushJobTest, Snapshots) {
|
||||
}
|
||||
|
||||
EventLogger event_logger(db_options_.info_log.get());
|
||||
FlushJob flush_job(dbname_, versions_->GetColumnFamilySet()->GetDefault(),
|
||||
db_options_, *cfd->GetLatestMutableCFOptions(),
|
||||
env_options_, versions_.get(), &mutex_, &shutting_down_,
|
||||
snapshots, kMaxSequenceNumber, &job_context, nullptr,
|
||||
nullptr, nullptr, kNoCompression, nullptr, &event_logger);
|
||||
FlushJob flush_job(
|
||||
dbname_, versions_->GetColumnFamilySet()->GetDefault(), db_options_,
|
||||
*cfd->GetLatestMutableCFOptions(), env_options_, versions_.get(), &mutex_,
|
||||
&shutting_down_, snapshots, kMaxSequenceNumber, &job_context, nullptr,
|
||||
nullptr, nullptr, kNoCompression, nullptr, &event_logger, true);
|
||||
mutex_.Lock();
|
||||
ASSERT_OK(flush_job.Run());
|
||||
mutex_.Unlock();
|
||||
|
@ -61,7 +61,7 @@ struct CompactionJobStats {
|
||||
uint64_t num_corrupt_keys;
|
||||
|
||||
// Following counters are only populated if
|
||||
// options.compaction_measure_io_stats = true;
|
||||
// options.report_bg_io_stats = true;
|
||||
|
||||
// Time spent on file's Append() call.
|
||||
uint64_t file_write_nanos;
|
||||
|
@ -790,9 +790,9 @@ struct ColumnFamilyOptions {
|
||||
// Default: false
|
||||
bool paranoid_file_checks;
|
||||
|
||||
// Measure IO stats in compactions, if true.
|
||||
// Measure IO stats in compactions and flushes, if true.
|
||||
// Default: false
|
||||
bool compaction_measure_io_stats;
|
||||
bool report_bg_io_stats;
|
||||
|
||||
// Create ColumnFamilyOptions with default values for all fields
|
||||
ColumnFamilyOptions();
|
||||
|
@ -5,8 +5,8 @@
|
||||
#pragma once
|
||||
|
||||
#define ROCKSDB_MAJOR 4
|
||||
#define ROCKSDB_MINOR 6
|
||||
#define ROCKSDB_PATCH 0
|
||||
#define ROCKSDB_MINOR 7
|
||||
#define ROCKSDB_PATCH 1
|
||||
|
||||
// Do not use these. We made the mistake of declaring macros starting with
|
||||
// double underscore. Now we have to live with our choice. We'll deprecate these
|
||||
|
@ -1224,45 +1224,45 @@ bool BlockBasedTable::PrefixMayMatch(const Slice& internal_key) {
|
||||
// First, try check with full filter
|
||||
auto filter_entry = GetFilter(true /* no io */);
|
||||
FilterBlockReader* filter = filter_entry.value;
|
||||
if (filter != nullptr && !filter->IsBlockBased()) {
|
||||
may_match = filter->PrefixMayMatch(prefix);
|
||||
}
|
||||
if (filter != nullptr) {
|
||||
if (!filter->IsBlockBased()) {
|
||||
may_match = filter->PrefixMayMatch(prefix);
|
||||
} else {
|
||||
// Then, try find it within each block
|
||||
unique_ptr<InternalIterator> iiter(NewIndexIterator(no_io_read_options));
|
||||
iiter->Seek(internal_prefix);
|
||||
|
||||
// Then, try find it within each block
|
||||
if (may_match) {
|
||||
unique_ptr<InternalIterator> iiter(NewIndexIterator(no_io_read_options));
|
||||
iiter->Seek(internal_prefix);
|
||||
|
||||
if (!iiter->Valid()) {
|
||||
// we're past end of file
|
||||
// if it's incomplete, it means that we avoided I/O
|
||||
// and we're not really sure that we're past the end
|
||||
// of the file
|
||||
may_match = iiter->status().IsIncomplete();
|
||||
} else if (ExtractUserKey(iiter->key()).starts_with(
|
||||
ExtractUserKey(internal_prefix))) {
|
||||
// we need to check for this subtle case because our only
|
||||
// guarantee is that "the key is a string >= last key in that data
|
||||
// block" according to the doc/table_format.txt spec.
|
||||
//
|
||||
// Suppose iiter->key() starts with the desired prefix; it is not
|
||||
// necessarily the case that the corresponding data block will
|
||||
// contain the prefix, since iiter->key() need not be in the
|
||||
// block. However, the next data block may contain the prefix, so
|
||||
// we return true to play it safe.
|
||||
may_match = true;
|
||||
} else if (filter != nullptr && filter->IsBlockBased()) {
|
||||
// iiter->key() does NOT start with the desired prefix. Because
|
||||
// Seek() finds the first key that is >= the seek target, this
|
||||
// means that iiter->key() > prefix. Thus, any data blocks coming
|
||||
// after the data block corresponding to iiter->key() cannot
|
||||
// possibly contain the key. Thus, the corresponding data block
|
||||
// is the only on could potentially contain the prefix.
|
||||
Slice handle_value = iiter->value();
|
||||
BlockHandle handle;
|
||||
s = handle.DecodeFrom(&handle_value);
|
||||
assert(s.ok());
|
||||
may_match = filter->PrefixMayMatch(prefix, handle.offset());
|
||||
if (!iiter->Valid()) {
|
||||
// we're past end of file
|
||||
// if it's incomplete, it means that we avoided I/O
|
||||
// and we're not really sure that we're past the end
|
||||
// of the file
|
||||
may_match = iiter->status().IsIncomplete();
|
||||
} else if (ExtractUserKey(iiter->key())
|
||||
.starts_with(ExtractUserKey(internal_prefix))) {
|
||||
// we need to check for this subtle case because our only
|
||||
// guarantee is that "the key is a string >= last key in that data
|
||||
// block" according to the doc/table_format.txt spec.
|
||||
//
|
||||
// Suppose iiter->key() starts with the desired prefix; it is not
|
||||
// necessarily the case that the corresponding data block will
|
||||
// contain the prefix, since iiter->key() need not be in the
|
||||
// block. However, the next data block may contain the prefix, so
|
||||
// we return true to play it safe.
|
||||
may_match = true;
|
||||
} else if (filter->IsBlockBased()) {
|
||||
// iiter->key() does NOT start with the desired prefix. Because
|
||||
// Seek() finds the first key that is >= the seek target, this
|
||||
// means that iiter->key() > prefix. Thus, any data blocks coming
|
||||
// after the data block corresponding to iiter->key() cannot
|
||||
// possibly contain the key. Thus, the corresponding data block
|
||||
// is the only on could potentially contain the prefix.
|
||||
Slice handle_value = iiter->value();
|
||||
BlockHandle handle;
|
||||
s = handle.DecodeFrom(&handle_value);
|
||||
assert(s.ok());
|
||||
may_match = filter->PrefixMayMatch(prefix, handle.offset());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -516,7 +516,7 @@ DEFINE_uint64(transaction_lock_timeout, 100,
|
||||
" milliseconds before failing a transaction waiting on a lock");
|
||||
#endif // ROCKSDB_LITE
|
||||
|
||||
DEFINE_bool(compaction_measure_io_stats, false,
|
||||
DEFINE_bool(report_bg_io_stats, false,
|
||||
"Measure times spents on I/Os while in compactions. ");
|
||||
|
||||
enum rocksdb::CompressionType StringToCompressionType(const char* ctype) {
|
||||
@ -2599,7 +2599,7 @@ class Benchmark {
|
||||
exit(1);
|
||||
}
|
||||
options.max_successive_merges = FLAGS_max_successive_merges;
|
||||
options.compaction_measure_io_stats = FLAGS_compaction_measure_io_stats;
|
||||
options.report_bg_io_stats = FLAGS_report_bg_io_stats;
|
||||
|
||||
// set universal style compaction configurations, if applicable
|
||||
if (FLAGS_universal_size_ratio != 0) {
|
||||
|
@ -47,7 +47,7 @@ struct MutableCFOptions {
|
||||
max_sequential_skip_in_iterations(
|
||||
options.max_sequential_skip_in_iterations),
|
||||
paranoid_file_checks(options.paranoid_file_checks),
|
||||
compaction_measure_io_stats(options.compaction_measure_io_stats)
|
||||
report_bg_io_stats(options.report_bg_io_stats)
|
||||
|
||||
{
|
||||
RefreshDerivedOptions(ioptions);
|
||||
@ -80,7 +80,7 @@ struct MutableCFOptions {
|
||||
max_subcompactions(1),
|
||||
max_sequential_skip_in_iterations(0),
|
||||
paranoid_file_checks(false),
|
||||
compaction_measure_io_stats(false) {}
|
||||
report_bg_io_stats(false) {}
|
||||
|
||||
// Must be called after any change to MutableCFOptions
|
||||
void RefreshDerivedOptions(const ImmutableCFOptions& ioptions);
|
||||
@ -135,7 +135,7 @@ struct MutableCFOptions {
|
||||
// Misc options
|
||||
uint64_t max_sequential_skip_in_iterations;
|
||||
bool paranoid_file_checks;
|
||||
bool compaction_measure_io_stats;
|
||||
bool report_bg_io_stats;
|
||||
|
||||
// Derived options
|
||||
// Per-level target file size.
|
||||
|
@ -129,7 +129,7 @@ ColumnFamilyOptions::ColumnFamilyOptions()
|
||||
min_partial_merge_operands(2),
|
||||
optimize_filters_for_hits(false),
|
||||
paranoid_file_checks(false),
|
||||
compaction_measure_io_stats(false) {
|
||||
report_bg_io_stats(false) {
|
||||
assert(memtable_factory.get() != nullptr);
|
||||
}
|
||||
|
||||
@ -198,7 +198,7 @@ ColumnFamilyOptions::ColumnFamilyOptions(const Options& options)
|
||||
min_partial_merge_operands(options.min_partial_merge_operands),
|
||||
optimize_filters_for_hits(options.optimize_filters_for_hits),
|
||||
paranoid_file_checks(options.paranoid_file_checks),
|
||||
compaction_measure_io_stats(options.compaction_measure_io_stats) {
|
||||
report_bg_io_stats(options.report_bg_io_stats) {
|
||||
assert(memtable_factory.get() != nullptr);
|
||||
if (max_bytes_for_level_multiplier_additional.size() <
|
||||
static_cast<unsigned int>(num_levels)) {
|
||||
@ -604,8 +604,8 @@ void ColumnFamilyOptions::Dump(Logger* log) const {
|
||||
optimize_filters_for_hits);
|
||||
Header(log, " Options.paranoid_file_checks: %d",
|
||||
paranoid_file_checks);
|
||||
Header(log, " Options.compaction_measure_io_stats: %d",
|
||||
compaction_measure_io_stats);
|
||||
Header(log, " Options.report_bg_io_stats: %d",
|
||||
report_bg_io_stats);
|
||||
} // ColumnFamilyOptions::Dump
|
||||
|
||||
void Options::Dump(Logger* log) const {
|
||||
|
@ -4,6 +4,7 @@
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
|
||||
#include <math.h>
|
||||
#include <cmath>
|
||||
#include <algorithm>
|
||||
#include "rocksdb/options.h"
|
||||
|
||||
|
@ -1452,8 +1452,7 @@ ColumnFamilyOptions BuildColumnFamilyOptions(
|
||||
cf_opts.max_sequential_skip_in_iterations =
|
||||
mutable_cf_options.max_sequential_skip_in_iterations;
|
||||
cf_opts.paranoid_file_checks = mutable_cf_options.paranoid_file_checks;
|
||||
cf_opts.compaction_measure_io_stats =
|
||||
mutable_cf_options.compaction_measure_io_stats;
|
||||
cf_opts.report_bg_io_stats = mutable_cf_options.report_bg_io_stats;
|
||||
|
||||
cf_opts.table_factory = options.table_factory;
|
||||
// TODO(yhchiang): find some way to handle the following derived options
|
||||
|
@ -329,9 +329,12 @@ static std::unordered_map<std::string, OptionTypeInfo> cf_options_type_info = {
|
||||
std::string* merged_value);
|
||||
std::vector<int> max_bytes_for_level_multiplier_additional;
|
||||
*/
|
||||
{"compaction_measure_io_stats",
|
||||
{offsetof(struct ColumnFamilyOptions, compaction_measure_io_stats),
|
||||
{"report_bg_io_stats",
|
||||
{offsetof(struct ColumnFamilyOptions, report_bg_io_stats),
|
||||
OptionType::kBoolean, OptionVerificationType::kNormal}},
|
||||
{"compaction_measure_io_stats",
|
||||
{offsetof(struct ColumnFamilyOptions, report_bg_io_stats),
|
||||
OptionType::kBoolean, OptionVerificationType::kDeprecated}},
|
||||
{"disable_auto_compactions",
|
||||
{offsetof(struct ColumnFamilyOptions, disable_auto_compactions),
|
||||
OptionType::kBoolean, OptionVerificationType::kNormal}},
|
||||
|
448
util/options_settable_test.cc
Normal file
448
util/options_settable_test.cc
Normal file
@ -0,0 +1,448 @@
|
||||
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#ifndef __STDC_FORMAT_MACROS
|
||||
#define __STDC_FORMAT_MACROS
|
||||
#endif
|
||||
|
||||
#include <inttypes.h>
|
||||
#include <cctype>
|
||||
#include <cstring>
|
||||
#include <unordered_map>
|
||||
|
||||
#include "rocksdb/cache.h"
|
||||
#include "rocksdb/convenience.h"
|
||||
#include "rocksdb/memtablerep.h"
|
||||
#include "rocksdb/utilities/leveldb_options.h"
|
||||
#include "util/options_helper.h"
|
||||
#include "util/options_parser.h"
|
||||
#include "util/options_sanity_check.h"
|
||||
#include "util/random.h"
|
||||
#include "util/stderr_logger.h"
|
||||
#include "util/testharness.h"
|
||||
#include "util/testutil.h"
|
||||
|
||||
#ifndef GFLAGS
|
||||
bool FLAGS_enable_print = false;
|
||||
#else
|
||||
#include <gflags/gflags.h>
|
||||
using GFLAGS::ParseCommandLineFlags;
|
||||
DEFINE_bool(enable_print, false, "Print options generated to console.");
|
||||
#endif // GFLAGS
|
||||
|
||||
namespace rocksdb {
|
||||
|
||||
// Verify options are settable from options strings.
|
||||
// We take the approach that depends on compiler behavior that copy constructor
|
||||
// won't touch implicit padding bytes, so that the test is fragile.
|
||||
// As a result, we only run the tests to verify new fields in options are
|
||||
// settable through string on limited platforms as it depends on behavior of
|
||||
// compilers.
|
||||
#ifndef ROCKSDB_LITE
|
||||
#ifdef OS_LINUX
|
||||
#ifndef __clang__
|
||||
|
||||
class OptionsSettableTest : public testing::Test {
|
||||
public:
|
||||
OptionsSettableTest() {}
|
||||
};
|
||||
|
||||
const char kSpecialChar = 'z';
|
||||
typedef std::vector<std::pair<int, size_t>> OffsetGap;
|
||||
|
||||
void FillWithSpecialChar(char* start_ptr, size_t total_size,
|
||||
const OffsetGap& blacklist) {
|
||||
size_t offset = 0;
|
||||
for (auto& pair : blacklist) {
|
||||
std::memset(start_ptr + offset, kSpecialChar, pair.first - offset);
|
||||
offset = pair.first + pair.second;
|
||||
}
|
||||
std::memset(start_ptr + offset, kSpecialChar, total_size - offset);
|
||||
}
|
||||
|
||||
int NumUnsetBytes(char* start_ptr, size_t total_size,
|
||||
const OffsetGap& blacklist) {
|
||||
int total_unset_bytes_base = 0;
|
||||
size_t offset = 0;
|
||||
for (auto& pair : blacklist) {
|
||||
for (char* ptr = start_ptr + offset; ptr < start_ptr + pair.first; ptr++) {
|
||||
if (*ptr == kSpecialChar) {
|
||||
total_unset_bytes_base++;
|
||||
}
|
||||
}
|
||||
offset = pair.first + pair.second;
|
||||
}
|
||||
for (char* ptr = start_ptr + offset; ptr < start_ptr + total_size; ptr++) {
|
||||
if (*ptr == kSpecialChar) {
|
||||
total_unset_bytes_base++;
|
||||
}
|
||||
}
|
||||
return total_unset_bytes_base;
|
||||
}
|
||||
|
||||
// If the test fails, likely a new option is added to BlockBasedTableOptions
|
||||
// but it cannot be set through GetBlockBasedTableOptionsFromString(), or the
|
||||
// test is not updated accordingly.
|
||||
// After adding an option, we need to make sure it is settable by
|
||||
// GetBlockBasedTableOptionsFromString() and add the option to the input string
|
||||
// passed to the GetBlockBasedTableOptionsFromString() in this test.
|
||||
// If it is a complicated type, you also need to add the field to
|
||||
// kBbtoBlacklist, and maybe add customized verification for it.
|
||||
TEST_F(OptionsSettableTest, BlockBasedTableOptionsAllFieldsSettable) {
|
||||
// Items in the form of <offset, size>. Need to be in ascending order
|
||||
// and not overlapping. Need to updated if new pointer-option is added.
|
||||
const OffsetGap kBbtoBlacklist = {
|
||||
{offsetof(struct BlockBasedTableOptions, flush_block_policy_factory),
|
||||
sizeof(std::shared_ptr<FlushBlockPolicyFactory>)},
|
||||
{offsetof(struct BlockBasedTableOptions, block_cache),
|
||||
sizeof(std::shared_ptr<Cache>)},
|
||||
{offsetof(struct BlockBasedTableOptions, block_cache_compressed),
|
||||
sizeof(std::shared_ptr<Cache>)},
|
||||
{offsetof(struct BlockBasedTableOptions, filter_policy),
|
||||
sizeof(std::shared_ptr<const FilterPolicy>)},
|
||||
};
|
||||
|
||||
// In this test, we catch a new option of BlockBasedTableOptions that is not
|
||||
// settable through GetBlockBasedTableOptionsFromString().
|
||||
// We count padding bytes of the option struct, and assert it to be the same
|
||||
// as unset bytes of an option struct initialized by
|
||||
// GetBlockBasedTableOptionsFromString().
|
||||
|
||||
char* bbto_ptr = new char[sizeof(BlockBasedTableOptions)];
|
||||
|
||||
// Count padding bytes by setting all bytes in the memory to a special char,
|
||||
// copy a well constructed struct to this memory and see how many special
|
||||
// bytes left.
|
||||
BlockBasedTableOptions* bbto = new (bbto_ptr) BlockBasedTableOptions();
|
||||
FillWithSpecialChar(bbto_ptr, sizeof(BlockBasedTableOptions), kBbtoBlacklist);
|
||||
// It based on the behavior of compiler that padding bytes are not changed
|
||||
// when copying the struct. It's prone to failure when compiler behavior
|
||||
// changes. We verify there is unset bytes to detect the case.
|
||||
*bbto = BlockBasedTableOptions();
|
||||
int unset_bytes_base =
|
||||
NumUnsetBytes(bbto_ptr, sizeof(BlockBasedTableOptions), kBbtoBlacklist);
|
||||
ASSERT_GT(unset_bytes_base, 0);
|
||||
bbto->~BlockBasedTableOptions();
|
||||
|
||||
// Construct the base option passed into
|
||||
// GetBlockBasedTableOptionsFromString().
|
||||
bbto = new (bbto_ptr) BlockBasedTableOptions();
|
||||
FillWithSpecialChar(bbto_ptr, sizeof(BlockBasedTableOptions), kBbtoBlacklist);
|
||||
// This option is not setable:
|
||||
bbto->use_delta_encoding = true;
|
||||
|
||||
char* new_bbto_ptr = new char[sizeof(BlockBasedTableOptions)];
|
||||
BlockBasedTableOptions* new_bbto =
|
||||
new (new_bbto_ptr) BlockBasedTableOptions();
|
||||
FillWithSpecialChar(new_bbto_ptr, sizeof(BlockBasedTableOptions),
|
||||
kBbtoBlacklist);
|
||||
|
||||
// Need to update the option string if a new option is added.
|
||||
ASSERT_OK(GetBlockBasedTableOptionsFromString(
|
||||
*bbto,
|
||||
"cache_index_and_filter_blocks=1;"
|
||||
"pin_l0_filter_and_index_blocks_in_cache=1;"
|
||||
"index_type=kHashSearch;"
|
||||
"checksum=kxxHash;hash_index_allow_collision=1;no_block_cache=1;"
|
||||
"block_cache=1M;block_cache_compressed=1k;block_size=1024;"
|
||||
"block_size_deviation=8;block_restart_interval=4; "
|
||||
"index_block_restart_interval=4;"
|
||||
"filter_policy=bloomfilter:4:true;whole_key_filtering=1;"
|
||||
"skip_table_builder_flush=1;format_version=1;"
|
||||
"hash_index_allow_collision=false;",
|
||||
new_bbto));
|
||||
|
||||
ASSERT_EQ(unset_bytes_base,
|
||||
NumUnsetBytes(new_bbto_ptr, sizeof(BlockBasedTableOptions),
|
||||
kBbtoBlacklist));
|
||||
|
||||
ASSERT_TRUE(new_bbto->block_cache.get() != nullptr);
|
||||
ASSERT_TRUE(new_bbto->block_cache_compressed.get() != nullptr);
|
||||
ASSERT_TRUE(new_bbto->filter_policy.get() != nullptr);
|
||||
|
||||
bbto->~BlockBasedTableOptions();
|
||||
new_bbto->~BlockBasedTableOptions();
|
||||
|
||||
delete[] bbto_ptr;
|
||||
delete[] new_bbto_ptr;
|
||||
}
|
||||
|
||||
// If the test fails, likely a new option is added to DBOptions
|
||||
// but it cannot be set through GetDBOptionsFromString(), or the test is not
|
||||
// updated accordingly.
|
||||
// After adding an option, we need to make sure it is settable by
|
||||
// GetDBOptionsFromString() and add the option to the input string passed to
|
||||
// DBOptionsFromString()in this test.
|
||||
// If it is a complicated type, you also need to add the field to
|
||||
// kDBOptionsBlacklist, and maybe add customized verification for it.
|
||||
TEST_F(OptionsSettableTest, DBOptionsAllFieldsSettable) {
|
||||
const OffsetGap kDBOptionsBlacklist = {
|
||||
{offsetof(struct DBOptions, env), sizeof(Env*)},
|
||||
{offsetof(struct DBOptions, rate_limiter),
|
||||
sizeof(std::shared_ptr<RateLimiter>)},
|
||||
{offsetof(struct DBOptions, sst_file_manager),
|
||||
sizeof(std::shared_ptr<SstFileManager>)},
|
||||
{offsetof(struct DBOptions, info_log), sizeof(std::shared_ptr<Logger>)},
|
||||
{offsetof(struct DBOptions, statistics),
|
||||
sizeof(std::shared_ptr<Statistics>)},
|
||||
{offsetof(struct DBOptions, db_paths), sizeof(std::vector<DbPath>)},
|
||||
{offsetof(struct DBOptions, db_log_dir), sizeof(std::string)},
|
||||
{offsetof(struct DBOptions, wal_dir), sizeof(std::string)},
|
||||
{offsetof(struct DBOptions, listeners),
|
||||
sizeof(std::vector<std::shared_ptr<EventListener>>)},
|
||||
{offsetof(struct DBOptions, row_cache), sizeof(std::shared_ptr<Cache>)},
|
||||
{offsetof(struct DBOptions, wal_filter), sizeof(const WalFilter*)},
|
||||
};
|
||||
|
||||
char* options_ptr = new char[sizeof(DBOptions)];
|
||||
|
||||
// Count padding bytes by setting all bytes in the memory to a special char,
|
||||
// copy a well constructed struct to this memory and see how many special
|
||||
// bytes left.
|
||||
DBOptions* options = new (options_ptr) DBOptions();
|
||||
FillWithSpecialChar(options_ptr, sizeof(DBOptions), kDBOptionsBlacklist);
|
||||
// It based on the behavior of compiler that padding bytes are not changed
|
||||
// when copying the struct. It's prone to failure when compiler behavior
|
||||
// changes. We verify there is unset bytes to detect the case.
|
||||
*options = DBOptions();
|
||||
int unset_bytes_base =
|
||||
NumUnsetBytes(options_ptr, sizeof(DBOptions), kDBOptionsBlacklist);
|
||||
ASSERT_GT(unset_bytes_base, 0);
|
||||
options->~DBOptions();
|
||||
|
||||
options = new (options_ptr) DBOptions();
|
||||
FillWithSpecialChar(options_ptr, sizeof(DBOptions), kDBOptionsBlacklist);
|
||||
|
||||
char* new_options_ptr = new char[sizeof(DBOptions)];
|
||||
DBOptions* new_options = new (new_options_ptr) DBOptions();
|
||||
FillWithSpecialChar(new_options_ptr, sizeof(DBOptions), kDBOptionsBlacklist);
|
||||
|
||||
// Need to update the option string if a new option is added.
|
||||
ASSERT_OK(
|
||||
GetDBOptionsFromString(*options,
|
||||
"wal_bytes_per_sync=4295048118;"
|
||||
"delete_obsolete_files_period_micros=4294967758;"
|
||||
"WAL_ttl_seconds=4295008036;"
|
||||
"WAL_size_limit_MB=4295036161;"
|
||||
"wal_dir=path/to/wal_dir;"
|
||||
"db_write_buffer_size=2587;"
|
||||
"max_subcompactions=64330;"
|
||||
"table_cache_numshardbits=28;"
|
||||
"max_open_files=72;"
|
||||
"max_file_opening_threads=35;"
|
||||
"base_background_compactions=3;"
|
||||
"max_background_compactions=33;"
|
||||
"use_fsync=true;"
|
||||
"use_adaptive_mutex=false;"
|
||||
"max_total_wal_size=4295005604;"
|
||||
"compaction_readahead_size=0;"
|
||||
"new_table_reader_for_compaction_inputs=false;"
|
||||
"keep_log_file_num=4890;"
|
||||
"skip_stats_update_on_db_open=false;"
|
||||
"max_manifest_file_size=4295009941;"
|
||||
"db_log_dir=path/to/db_log_dir;"
|
||||
"skip_log_error_on_recovery=true;"
|
||||
"writable_file_max_buffer_size=1048576;"
|
||||
"paranoid_checks=true;"
|
||||
"is_fd_close_on_exec=false;"
|
||||
"bytes_per_sync=4295013613;"
|
||||
"enable_thread_tracking=false;"
|
||||
"disable_data_sync=false;"
|
||||
"recycle_log_file_num=0;"
|
||||
"disableDataSync=false;"
|
||||
"create_missing_column_families=true;"
|
||||
"log_file_time_to_roll=3097;"
|
||||
"max_background_flushes=35;"
|
||||
"create_if_missing=false;"
|
||||
"error_if_exists=true;"
|
||||
"allow_os_buffer=false;"
|
||||
"delayed_write_rate=4294976214;"
|
||||
"manifest_preallocation_size=1222;"
|
||||
"allow_mmap_writes=false;"
|
||||
"stats_dump_period_sec=70127;"
|
||||
"allow_fallocate=true;"
|
||||
"allow_mmap_reads=false;"
|
||||
"max_log_file_size=4607;"
|
||||
"random_access_max_buffer_size=1048576;"
|
||||
"advise_random_on_open=true;"
|
||||
"fail_if_options_file_error=false;"
|
||||
"allow_concurrent_memtable_write=true;"
|
||||
"wal_recovery_mode=kPointInTimeRecovery;"
|
||||
"enable_write_thread_adaptive_yield=true;"
|
||||
"write_thread_slow_yield_usec=5;"
|
||||
"write_thread_max_yield_usec=1000;"
|
||||
"access_hint_on_compaction_start=NONE;"
|
||||
"info_log_level=DEBUG_LEVEL;",
|
||||
new_options));
|
||||
|
||||
ASSERT_EQ(unset_bytes_base, NumUnsetBytes(new_options_ptr, sizeof(DBOptions),
|
||||
kDBOptionsBlacklist));
|
||||
|
||||
options->~DBOptions();
|
||||
new_options->~DBOptions();
|
||||
|
||||
delete[] options_ptr;
|
||||
delete[] new_options_ptr;
|
||||
}
|
||||
|
||||
// If the test fails, likely a new option is added to ColumnFamilyOptions
|
||||
// but it cannot be set through GetColumnFamilyOptionsFromString(), or the
|
||||
// test is not updated accordingly.
|
||||
// After adding an option, we need to make sure it is settable by
|
||||
// GetColumnFamilyOptionsFromString() and add the option to the input
|
||||
// string passed to GetColumnFamilyOptionsFromString()in this test.
|
||||
// If it is a complicated type, you also need to add the field to
|
||||
// kColumnFamilyOptionsBlacklist, and maybe add customized verification
|
||||
// for it.
|
||||
TEST_F(OptionsSettableTest, ColumnFamilyOptionsAllFieldsSettable) {
|
||||
const OffsetGap kColumnFamilyOptionsBlacklist = {
|
||||
{offsetof(struct ColumnFamilyOptions, comparator), sizeof(Comparator*)},
|
||||
{offsetof(struct ColumnFamilyOptions, merge_operator),
|
||||
sizeof(std::shared_ptr<MergeOperator>)},
|
||||
{offsetof(struct ColumnFamilyOptions, compaction_filter),
|
||||
sizeof(const CompactionFilter*)},
|
||||
{offsetof(struct ColumnFamilyOptions, compaction_filter_factory),
|
||||
sizeof(std::shared_ptr<CompactionFilterFactory>)},
|
||||
{offsetof(struct ColumnFamilyOptions, compression_per_level),
|
||||
sizeof(std::vector<CompressionType>)},
|
||||
{offsetof(struct ColumnFamilyOptions, prefix_extractor),
|
||||
sizeof(std::shared_ptr<const SliceTransform>)},
|
||||
{offsetof(struct ColumnFamilyOptions,
|
||||
max_bytes_for_level_multiplier_additional),
|
||||
sizeof(std::vector<int>)},
|
||||
{offsetof(struct ColumnFamilyOptions, memtable_factory),
|
||||
sizeof(std::shared_ptr<MemTableRepFactory>)},
|
||||
{offsetof(struct ColumnFamilyOptions, table_factory),
|
||||
sizeof(std::shared_ptr<TableFactory>)},
|
||||
{offsetof(struct ColumnFamilyOptions,
|
||||
table_properties_collector_factories),
|
||||
sizeof(ColumnFamilyOptions::TablePropertiesCollectorFactories)},
|
||||
{offsetof(struct ColumnFamilyOptions, inplace_callback),
|
||||
sizeof(UpdateStatus(*)(char*, uint32_t*, Slice, std::string*))},
|
||||
};
|
||||
|
||||
char* options_ptr = new char[sizeof(ColumnFamilyOptions)];
|
||||
|
||||
// Count padding bytes by setting all bytes in the memory to a special char,
|
||||
// copy a well constructed struct to this memory and see how many special
|
||||
// bytes left.
|
||||
ColumnFamilyOptions* options = new (options_ptr) ColumnFamilyOptions();
|
||||
FillWithSpecialChar(options_ptr, sizeof(ColumnFamilyOptions),
|
||||
kColumnFamilyOptionsBlacklist);
|
||||
// It based on the behavior of compiler that padding bytes are not changed
|
||||
// when copying the struct. It's prone to failure when compiler behavior
|
||||
// changes. We verify there is unset bytes to detect the case.
|
||||
*options = ColumnFamilyOptions();
|
||||
|
||||
// Deprecatd option which is not initialized. Need to set it to avoid
|
||||
// Valgrind error
|
||||
options->max_mem_compaction_level = 0;
|
||||
|
||||
int unset_bytes_base = NumUnsetBytes(options_ptr, sizeof(ColumnFamilyOptions),
|
||||
kColumnFamilyOptionsBlacklist);
|
||||
ASSERT_GT(unset_bytes_base, 0);
|
||||
options->~ColumnFamilyOptions();
|
||||
|
||||
options = new (options_ptr) ColumnFamilyOptions();
|
||||
FillWithSpecialChar(options_ptr, sizeof(ColumnFamilyOptions),
|
||||
kColumnFamilyOptionsBlacklist);
|
||||
|
||||
// Following options are not settable through
|
||||
// GetColumnFamilyOptionsFromString():
|
||||
options->rate_limit_delay_max_milliseconds = 33;
|
||||
options->compaction_pri = CompactionPri::kOldestSmallestSeqFirst;
|
||||
options->compaction_options_universal = CompactionOptionsUniversal();
|
||||
options->compression_opts = CompressionOptions();
|
||||
options->hard_rate_limit = 0;
|
||||
options->soft_rate_limit = 0;
|
||||
options->compaction_options_fifo = CompactionOptionsFIFO();
|
||||
options->max_mem_compaction_level = 0;
|
||||
|
||||
char* new_options_ptr = new char[sizeof(ColumnFamilyOptions)];
|
||||
ColumnFamilyOptions* new_options =
|
||||
new (new_options_ptr) ColumnFamilyOptions();
|
||||
FillWithSpecialChar(new_options_ptr, sizeof(ColumnFamilyOptions),
|
||||
kColumnFamilyOptionsBlacklist);
|
||||
|
||||
// Need to update the option string if a new option is added.
|
||||
ASSERT_OK(GetColumnFamilyOptionsFromString(
|
||||
*options,
|
||||
"compaction_filter_factory=mpudlojcujCompactionFilterFactory;"
|
||||
"table_factory=PlainTable;"
|
||||
"prefix_extractor=rocksdb.CappedPrefix.13;"
|
||||
"comparator=leveldb.BytewiseComparator;"
|
||||
"compression_per_level=kBZip2Compression:kBZip2Compression:"
|
||||
"kBZip2Compression:kNoCompression:kZlibCompression:kBZip2Compression:"
|
||||
"kSnappyCompression;"
|
||||
"max_bytes_for_level_base=986;"
|
||||
"bloom_locality=8016;"
|
||||
"target_file_size_base=4294976376;"
|
||||
"memtable_prefix_bloom_huge_page_tlb_size=2557;"
|
||||
"max_successive_merges=5497;"
|
||||
"max_sequential_skip_in_iterations=4294971408;"
|
||||
"arena_block_size=1893;"
|
||||
"target_file_size_multiplier=35;"
|
||||
"source_compaction_factor=54;"
|
||||
"min_write_buffer_number_to_merge=9;"
|
||||
"max_write_buffer_number=84;"
|
||||
"write_buffer_size=1653;"
|
||||
"max_grandparent_overlap_factor=64;"
|
||||
"max_bytes_for_level_multiplier=60;"
|
||||
"memtable_factory=SkipListFactory;"
|
||||
"compression=kNoCompression;"
|
||||
"min_partial_merge_operands=7576;"
|
||||
"level0_stop_writes_trigger=33;"
|
||||
"num_levels=99;"
|
||||
"level0_slowdown_writes_trigger=22;"
|
||||
"level0_file_num_compaction_trigger=14;"
|
||||
"expanded_compaction_factor=34;"
|
||||
"compaction_filter=urxcqstuwnCompactionFilter;"
|
||||
"soft_rate_limit=530.615385;"
|
||||
"soft_pending_compaction_bytes_limit=0;"
|
||||
"max_write_buffer_number_to_maintain=84;"
|
||||
"verify_checksums_in_compaction=false;"
|
||||
"merge_operator=aabcxehazrMergeOperator;"
|
||||
"memtable_prefix_bloom_bits=4642;"
|
||||
"paranoid_file_checks=true;"
|
||||
"inplace_update_num_locks=7429;"
|
||||
"optimize_filters_for_hits=false;"
|
||||
"level_compaction_dynamic_level_bytes=false;"
|
||||
"inplace_update_support=false;"
|
||||
"compaction_style=kCompactionStyleFIFO;"
|
||||
"memtable_prefix_bloom_probes=2511;"
|
||||
"purge_redundant_kvs_while_flush=true;"
|
||||
"filter_deletes=false;"
|
||||
"hard_pending_compaction_bytes_limit=0;"
|
||||
"disable_auto_compactions=false;"
|
||||
"report_bg_io_stats=true;",
|
||||
new_options));
|
||||
|
||||
ASSERT_EQ(unset_bytes_base,
|
||||
NumUnsetBytes(new_options_ptr, sizeof(ColumnFamilyOptions),
|
||||
kColumnFamilyOptionsBlacklist));
|
||||
|
||||
options->~ColumnFamilyOptions();
|
||||
new_options->~ColumnFamilyOptions();
|
||||
|
||||
delete[] options_ptr;
|
||||
delete[] new_options_ptr;
|
||||
}
|
||||
#endif // !__clang__
|
||||
#endif // OS_LINUX
|
||||
#endif // !ROCKSDB_LITE
|
||||
|
||||
} // namespace rocksdb
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
::testing::InitGoogleTest(&argc, argv);
|
||||
#ifdef GFLAGS
|
||||
ParseCommandLineFlags(&argc, &argv, true);
|
||||
#endif // GFLAGS
|
||||
return RUN_ALL_TESTS();
|
||||
}
|
@ -127,7 +127,8 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) {
|
||||
{"filter_deletes", "0"},
|
||||
{"max_sequential_skip_in_iterations", "24"},
|
||||
{"inplace_update_support", "true"},
|
||||
{"compaction_measure_io_stats", "true"},
|
||||
{"report_bg_io_stats", "true"},
|
||||
{"compaction_measure_io_stats", "false"},
|
||||
{"inplace_update_num_locks", "25"},
|
||||
{"memtable_prefix_bloom_bits", "26"},
|
||||
{"memtable_prefix_bloom_probes", "27"},
|
||||
@ -1497,395 +1498,7 @@ TEST_F(OptionsParserTest, EscapeOptionString) {
|
||||
"Escape \\# and # comment together ."),
|
||||
"Escape \\# and");
|
||||
}
|
||||
|
||||
// Only run the tests to verify new fields in options are settable through
|
||||
// string on limited platforms as it depends on behavior of compilers.
|
||||
#ifdef OS_LINUX
|
||||
#ifndef __clang__
|
||||
const char kSpecialChar = 'z';
|
||||
typedef std::vector<std::pair<int, size_t>> OffsetGap;
|
||||
|
||||
void FillWithSpecialChar(char* start_ptr, size_t total_size,
|
||||
const OffsetGap& blacklist) {
|
||||
size_t offset = 0;
|
||||
for (auto& pair : blacklist) {
|
||||
std::memset(start_ptr + offset, kSpecialChar, pair.first - offset);
|
||||
offset = pair.first + pair.second;
|
||||
}
|
||||
std::memset(start_ptr + offset, kSpecialChar, total_size - offset);
|
||||
}
|
||||
|
||||
int NumUnsetBytes(char* start_ptr, size_t total_size,
|
||||
const OffsetGap& blacklist) {
|
||||
int total_unset_bytes_base = 0;
|
||||
size_t offset = 0;
|
||||
for (auto& pair : blacklist) {
|
||||
for (char* ptr = start_ptr + offset; ptr < start_ptr + pair.first; ptr++) {
|
||||
if (*ptr == kSpecialChar) {
|
||||
total_unset_bytes_base++;
|
||||
}
|
||||
offset = pair.first + pair.second;
|
||||
}
|
||||
}
|
||||
for (char* ptr = start_ptr + offset; ptr < start_ptr + total_size; ptr++) {
|
||||
if (*ptr == kSpecialChar) {
|
||||
total_unset_bytes_base++;
|
||||
}
|
||||
}
|
||||
return total_unset_bytes_base;
|
||||
}
|
||||
|
||||
// If the test fails, likely a new option is added to BlockBasedTableOptions
|
||||
// but it cannot be set through GetBlockBasedTableOptionsFromString(), or the
|
||||
// test is not updated accordingly.
|
||||
// After adding an option, we need to make sure it is settable by
|
||||
// GetBlockBasedTableOptionsFromString() and add the option to the input string
|
||||
// passed to the GetBlockBasedTableOptionsFromString() in this test.
|
||||
// If it is a complicated type, you also need to add the field to
|
||||
// kBbtoBlacklist, and maybe add customized verification for it.
|
||||
TEST_F(OptionsParserTest, BlockBasedTableOptionsAllFieldsSettable) {
|
||||
// Items in the form of <offset, size>. Need to be in ascending order
|
||||
// and not overlapping. Need to updated if new pointer-option is added.
|
||||
const OffsetGap kBbtoBlacklist = {
|
||||
{offsetof(struct BlockBasedTableOptions, flush_block_policy_factory),
|
||||
sizeof(std::shared_ptr<FlushBlockPolicyFactory>)},
|
||||
{offsetof(struct BlockBasedTableOptions, block_cache),
|
||||
sizeof(std::shared_ptr<Cache>)},
|
||||
{offsetof(struct BlockBasedTableOptions, block_cache_compressed),
|
||||
sizeof(std::shared_ptr<Cache>)},
|
||||
{offsetof(struct BlockBasedTableOptions, filter_policy),
|
||||
sizeof(std::shared_ptr<const FilterPolicy>)},
|
||||
};
|
||||
|
||||
// In this test, we catch a new option of BlockBasedTableOptions that is not
|
||||
// settable through GetBlockBasedTableOptionsFromString().
|
||||
// We count padding bytes of the option struct, and assert it to be the same
|
||||
// as unset bytes of an option struct initialized by
|
||||
// GetBlockBasedTableOptionsFromString().
|
||||
|
||||
char* bbto_ptr = new char[sizeof(BlockBasedTableOptions)];
|
||||
|
||||
// Count padding bytes by setting all bytes in the memory to a special char,
|
||||
// copy a well constructed struct to this memory and see how many special
|
||||
// bytes left.
|
||||
BlockBasedTableOptions* bbto = new (bbto_ptr) BlockBasedTableOptions();
|
||||
FillWithSpecialChar(bbto_ptr, sizeof(BlockBasedTableOptions), kBbtoBlacklist);
|
||||
// It based on the behavior of compiler that padding bytes are not changed
|
||||
// when copying the struct. It's prone to failure when compiler behavior
|
||||
// changes. We verify there is unset bytes to detect the case.
|
||||
*bbto = BlockBasedTableOptions();
|
||||
int unset_bytes_base =
|
||||
NumUnsetBytes(bbto_ptr, sizeof(BlockBasedTableOptions), kBbtoBlacklist);
|
||||
ASSERT_GT(unset_bytes_base, 0);
|
||||
bbto->~BlockBasedTableOptions();
|
||||
|
||||
// Construct the base option passed into
|
||||
// GetBlockBasedTableOptionsFromString().
|
||||
bbto = new (bbto_ptr) BlockBasedTableOptions();
|
||||
FillWithSpecialChar(bbto_ptr, sizeof(BlockBasedTableOptions), kBbtoBlacklist);
|
||||
// This option is not setable:
|
||||
bbto->use_delta_encoding = true;
|
||||
|
||||
char* new_bbto_ptr = new char[sizeof(BlockBasedTableOptions)];
|
||||
BlockBasedTableOptions* new_bbto =
|
||||
new (new_bbto_ptr) BlockBasedTableOptions();
|
||||
FillWithSpecialChar(new_bbto_ptr, sizeof(BlockBasedTableOptions),
|
||||
kBbtoBlacklist);
|
||||
|
||||
// Need to update the option string if a new option is added.
|
||||
ASSERT_OK(GetBlockBasedTableOptionsFromString(
|
||||
*bbto,
|
||||
"cache_index_and_filter_blocks=1;"
|
||||
"pin_l0_filter_and_index_blocks_in_cache=1;"
|
||||
"index_type=kHashSearch;"
|
||||
"checksum=kxxHash;hash_index_allow_collision=1;no_block_cache=1;"
|
||||
"block_cache=1M;block_cache_compressed=1k;block_size=1024;"
|
||||
"block_size_deviation=8;block_restart_interval=4; "
|
||||
"index_block_restart_interval=4;"
|
||||
"filter_policy=bloomfilter:4:true;whole_key_filtering=1;"
|
||||
"skip_table_builder_flush=1;format_version=1;"
|
||||
"hash_index_allow_collision=false;",
|
||||
new_bbto));
|
||||
|
||||
ASSERT_EQ(unset_bytes_base,
|
||||
NumUnsetBytes(new_bbto_ptr, sizeof(BlockBasedTableOptions),
|
||||
kBbtoBlacklist));
|
||||
|
||||
ASSERT_TRUE(new_bbto->block_cache.get() != nullptr);
|
||||
ASSERT_TRUE(new_bbto->block_cache_compressed.get() != nullptr);
|
||||
ASSERT_TRUE(new_bbto->filter_policy.get() != nullptr);
|
||||
|
||||
bbto->~BlockBasedTableOptions();
|
||||
new_bbto->~BlockBasedTableOptions();
|
||||
|
||||
delete[] bbto_ptr;
|
||||
delete[] new_bbto_ptr;
|
||||
}
|
||||
|
||||
// If the test fails, likely a new option is added to DBOptions
|
||||
// but it cannot be set through GetDBOptionsFromString(), or the test is not
|
||||
// updated accordingly.
|
||||
// After adding an option, we need to make sure it is settable by
|
||||
// GetDBOptionsFromString() and add the option to the input string passed to
|
||||
// DBOptionsFromString()in this test.
|
||||
// If it is a complicated type, you also need to add the field to
|
||||
// kDBOptionsBlacklist, and maybe add customized verification for it.
|
||||
TEST_F(OptionsParserTest, DBOptionsAllFieldsSettable) {
|
||||
const OffsetGap kDBOptionsBlacklist = {
|
||||
{offsetof(struct DBOptions, env), sizeof(Env*)},
|
||||
{offsetof(struct DBOptions, rate_limiter),
|
||||
sizeof(std::shared_ptr<RateLimiter>)},
|
||||
{offsetof(struct DBOptions, sst_file_manager),
|
||||
sizeof(std::shared_ptr<SstFileManager>)},
|
||||
{offsetof(struct DBOptions, info_log), sizeof(std::shared_ptr<Logger>)},
|
||||
{offsetof(struct DBOptions, statistics),
|
||||
sizeof(std::shared_ptr<Statistics>)},
|
||||
{offsetof(struct DBOptions, db_paths), sizeof(std::vector<DbPath>)},
|
||||
{offsetof(struct DBOptions, db_log_dir), sizeof(std::string)},
|
||||
{offsetof(struct DBOptions, wal_dir), sizeof(std::string)},
|
||||
{offsetof(struct DBOptions, listeners),
|
||||
sizeof(std::vector<std::shared_ptr<EventListener>>)},
|
||||
{offsetof(struct DBOptions, row_cache), sizeof(std::shared_ptr<Cache>)},
|
||||
{offsetof(struct DBOptions, wal_filter), sizeof(const WalFilter*)},
|
||||
};
|
||||
|
||||
char* options_ptr = new char[sizeof(DBOptions)];
|
||||
|
||||
// Count padding bytes by setting all bytes in the memory to a special char,
|
||||
// copy a well constructed struct to this memory and see how many special
|
||||
// bytes left.
|
||||
DBOptions* options = new (options_ptr) DBOptions();
|
||||
FillWithSpecialChar(options_ptr, sizeof(DBOptions), kDBOptionsBlacklist);
|
||||
// It based on the behavior of compiler that padding bytes are not changed
|
||||
// when copying the struct. It's prone to failure when compiler behavior
|
||||
// changes. We verify there is unset bytes to detect the case.
|
||||
*options = DBOptions();
|
||||
int unset_bytes_base =
|
||||
NumUnsetBytes(options_ptr, sizeof(DBOptions), kDBOptionsBlacklist);
|
||||
ASSERT_GT(unset_bytes_base, 0);
|
||||
options->~DBOptions();
|
||||
|
||||
options = new (options_ptr) DBOptions();
|
||||
FillWithSpecialChar(options_ptr, sizeof(DBOptions), kDBOptionsBlacklist);
|
||||
|
||||
char* new_options_ptr = new char[sizeof(DBOptions)];
|
||||
DBOptions* new_options = new (new_options_ptr) DBOptions();
|
||||
FillWithSpecialChar(new_options_ptr, sizeof(DBOptions), kDBOptionsBlacklist);
|
||||
|
||||
// Need to update the option string if a new option is added.
|
||||
ASSERT_OK(
|
||||
GetDBOptionsFromString(*options,
|
||||
"wal_bytes_per_sync=4295048118;"
|
||||
"delete_obsolete_files_period_micros=4294967758;"
|
||||
"WAL_ttl_seconds=4295008036;"
|
||||
"WAL_size_limit_MB=4295036161;"
|
||||
"wal_dir=path/to/wal_dir;"
|
||||
"db_write_buffer_size=2587;"
|
||||
"max_subcompactions=64330;"
|
||||
"table_cache_numshardbits=28;"
|
||||
"max_open_files=72;"
|
||||
"max_file_opening_threads=35;"
|
||||
"base_background_compactions=3;"
|
||||
"max_background_compactions=33;"
|
||||
"use_fsync=true;"
|
||||
"use_adaptive_mutex=false;"
|
||||
"max_total_wal_size=4295005604;"
|
||||
"compaction_readahead_size=0;"
|
||||
"new_table_reader_for_compaction_inputs=false;"
|
||||
"keep_log_file_num=4890;"
|
||||
"skip_stats_update_on_db_open=false;"
|
||||
"max_manifest_file_size=4295009941;"
|
||||
"db_log_dir=path/to/db_log_dir;"
|
||||
"skip_log_error_on_recovery=true;"
|
||||
"writable_file_max_buffer_size=1048576;"
|
||||
"paranoid_checks=true;"
|
||||
"is_fd_close_on_exec=false;"
|
||||
"bytes_per_sync=4295013613;"
|
||||
"enable_thread_tracking=false;"
|
||||
"disable_data_sync=false;"
|
||||
"recycle_log_file_num=0;"
|
||||
"disableDataSync=false;"
|
||||
"create_missing_column_families=true;"
|
||||
"log_file_time_to_roll=3097;"
|
||||
"max_background_flushes=35;"
|
||||
"create_if_missing=false;"
|
||||
"error_if_exists=true;"
|
||||
"allow_os_buffer=false;"
|
||||
"delayed_write_rate=4294976214;"
|
||||
"manifest_preallocation_size=1222;"
|
||||
"allow_mmap_writes=false;"
|
||||
"stats_dump_period_sec=70127;"
|
||||
"allow_fallocate=true;"
|
||||
"allow_mmap_reads=false;"
|
||||
"max_log_file_size=4607;"
|
||||
"random_access_max_buffer_size=1048576;"
|
||||
"advise_random_on_open=true;"
|
||||
"fail_if_options_file_error=false;"
|
||||
"allow_concurrent_memtable_write=true;"
|
||||
"wal_recovery_mode=kPointInTimeRecovery;"
|
||||
"enable_write_thread_adaptive_yield=true;"
|
||||
"write_thread_slow_yield_usec=5;"
|
||||
"write_thread_max_yield_usec=1000;"
|
||||
"access_hint_on_compaction_start=NONE;"
|
||||
"info_log_level=DEBUG_LEVEL;",
|
||||
new_options));
|
||||
|
||||
ASSERT_EQ(unset_bytes_base, NumUnsetBytes(new_options_ptr, sizeof(DBOptions),
|
||||
kDBOptionsBlacklist));
|
||||
|
||||
options->~DBOptions();
|
||||
new_options->~DBOptions();
|
||||
|
||||
delete[] options_ptr;
|
||||
delete[] new_options_ptr;
|
||||
}
|
||||
|
||||
// If the test fails, likely a new option is added to ColumnFamilyOptions
|
||||
// but it cannot be set through GetColumnFamilyOptionsFromString(), or the
|
||||
// test is not updated accordingly.
|
||||
// After adding an option, we need to make sure it is settable by
|
||||
// GetColumnFamilyOptionsFromString() and add the option to the input
|
||||
// string passed to GetColumnFamilyOptionsFromString()in this test.
|
||||
// If it is a complicated type, you also need to add the field to
|
||||
// kColumnFamilyOptionsBlacklist, and maybe add customized verification
|
||||
// for it.
|
||||
TEST_F(OptionsParserTest, ColumnFamilyOptionsAllFieldsSettable) {
|
||||
const OffsetGap kColumnFamilyOptionsBlacklist = {
|
||||
{offsetof(struct ColumnFamilyOptions, comparator), sizeof(Comparator*)},
|
||||
{offsetof(struct ColumnFamilyOptions, merge_operator),
|
||||
sizeof(std::shared_ptr<MergeOperator>)},
|
||||
{offsetof(struct ColumnFamilyOptions, compaction_filter),
|
||||
sizeof(const CompactionFilter*)},
|
||||
{offsetof(struct ColumnFamilyOptions, compaction_filter_factory),
|
||||
sizeof(std::shared_ptr<CompactionFilterFactory>)},
|
||||
{offsetof(struct ColumnFamilyOptions, compression_per_level),
|
||||
sizeof(std::vector<CompressionType>)},
|
||||
{offsetof(struct ColumnFamilyOptions, prefix_extractor),
|
||||
sizeof(std::shared_ptr<const SliceTransform>)},
|
||||
{offsetof(struct ColumnFamilyOptions,
|
||||
max_bytes_for_level_multiplier_additional),
|
||||
sizeof(std::vector<int>)},
|
||||
{offsetof(struct ColumnFamilyOptions, memtable_factory),
|
||||
sizeof(std::shared_ptr<MemTableRepFactory>)},
|
||||
{offsetof(struct ColumnFamilyOptions, table_factory),
|
||||
sizeof(std::shared_ptr<TableFactory>)},
|
||||
{offsetof(struct ColumnFamilyOptions,
|
||||
table_properties_collector_factories),
|
||||
sizeof(ColumnFamilyOptions::TablePropertiesCollectorFactories)},
|
||||
{offsetof(struct ColumnFamilyOptions, inplace_callback),
|
||||
sizeof(UpdateStatus (*)(char*, uint32_t*, Slice, std::string*))},
|
||||
};
|
||||
|
||||
char* options_ptr = new char[sizeof(ColumnFamilyOptions)];
|
||||
|
||||
// Count padding bytes by setting all bytes in the memory to a special char,
|
||||
// copy a well constructed struct to this memory and see how many special
|
||||
// bytes left.
|
||||
ColumnFamilyOptions* options = new (options_ptr) ColumnFamilyOptions();
|
||||
FillWithSpecialChar(options_ptr, sizeof(ColumnFamilyOptions),
|
||||
kColumnFamilyOptionsBlacklist);
|
||||
// It based on the behavior of compiler that padding bytes are not changed
|
||||
// when copying the struct. It's prone to failure when compiler behavior
|
||||
// changes. We verify there is unset bytes to detect the case.
|
||||
*options = ColumnFamilyOptions();
|
||||
|
||||
// Deprecatd option which is not initialized. Need to set it to avoid
|
||||
// Valgrind error
|
||||
options->max_mem_compaction_level = 0;
|
||||
|
||||
int unset_bytes_base = NumUnsetBytes(options_ptr, sizeof(ColumnFamilyOptions),
|
||||
kColumnFamilyOptionsBlacklist);
|
||||
ASSERT_GT(unset_bytes_base, 0);
|
||||
options->~ColumnFamilyOptions();
|
||||
|
||||
options = new (options_ptr) ColumnFamilyOptions();
|
||||
FillWithSpecialChar(options_ptr, sizeof(ColumnFamilyOptions),
|
||||
kColumnFamilyOptionsBlacklist);
|
||||
|
||||
// Following options are not settable through
|
||||
// GetColumnFamilyOptionsFromString():
|
||||
options->rate_limit_delay_max_milliseconds = 33;
|
||||
options->compaction_pri = CompactionPri::kOldestSmallestSeqFirst;
|
||||
options->compaction_options_universal = CompactionOptionsUniversal();
|
||||
options->compression_opts = CompressionOptions();
|
||||
options->hard_rate_limit = 0;
|
||||
options->soft_rate_limit = 0;
|
||||
options->compaction_options_fifo = CompactionOptionsFIFO();
|
||||
options->max_mem_compaction_level = 0;
|
||||
|
||||
char* new_options_ptr = new char[sizeof(ColumnFamilyOptions)];
|
||||
ColumnFamilyOptions* new_options =
|
||||
new (new_options_ptr) ColumnFamilyOptions();
|
||||
FillWithSpecialChar(new_options_ptr, sizeof(ColumnFamilyOptions),
|
||||
kColumnFamilyOptionsBlacklist);
|
||||
|
||||
// Need to update the option string if a new option is added.
|
||||
ASSERT_OK(GetColumnFamilyOptionsFromString(
|
||||
*options,
|
||||
"compaction_filter_factory=mpudlojcujCompactionFilterFactory;"
|
||||
"table_factory=PlainTable;"
|
||||
"prefix_extractor=rocksdb.CappedPrefix.13;"
|
||||
"comparator=leveldb.BytewiseComparator;"
|
||||
"compression_per_level=kBZip2Compression:kBZip2Compression:"
|
||||
"kBZip2Compression:kNoCompression:kZlibCompression:kBZip2Compression:"
|
||||
"kSnappyCompression;"
|
||||
"max_bytes_for_level_base=986;"
|
||||
"bloom_locality=8016;"
|
||||
"target_file_size_base=4294976376;"
|
||||
"memtable_prefix_bloom_huge_page_tlb_size=2557;"
|
||||
"max_successive_merges=5497;"
|
||||
"max_sequential_skip_in_iterations=4294971408;"
|
||||
"arena_block_size=1893;"
|
||||
"target_file_size_multiplier=35;"
|
||||
"source_compaction_factor=54;"
|
||||
"min_write_buffer_number_to_merge=9;"
|
||||
"max_write_buffer_number=84;"
|
||||
"write_buffer_size=1653;"
|
||||
"max_grandparent_overlap_factor=64;"
|
||||
"max_bytes_for_level_multiplier=60;"
|
||||
"memtable_factory=SkipListFactory;"
|
||||
"compression=kNoCompression;"
|
||||
"min_partial_merge_operands=7576;"
|
||||
"level0_stop_writes_trigger=33;"
|
||||
"num_levels=99;"
|
||||
"level0_slowdown_writes_trigger=22;"
|
||||
"level0_file_num_compaction_trigger=14;"
|
||||
"expanded_compaction_factor=34;"
|
||||
"compaction_filter=urxcqstuwnCompactionFilter;"
|
||||
"soft_rate_limit=530.615385;"
|
||||
"soft_pending_compaction_bytes_limit=0;"
|
||||
"max_write_buffer_number_to_maintain=84;"
|
||||
"verify_checksums_in_compaction=false;"
|
||||
"merge_operator=aabcxehazrMergeOperator;"
|
||||
"memtable_prefix_bloom_bits=4642;"
|
||||
"paranoid_file_checks=true;"
|
||||
"inplace_update_num_locks=7429;"
|
||||
"optimize_filters_for_hits=false;"
|
||||
"level_compaction_dynamic_level_bytes=false;"
|
||||
"inplace_update_support=false;"
|
||||
"compaction_style=kCompactionStyleFIFO;"
|
||||
"memtable_prefix_bloom_probes=2511;"
|
||||
"purge_redundant_kvs_while_flush=true;"
|
||||
"filter_deletes=false;"
|
||||
"hard_pending_compaction_bytes_limit=0;"
|
||||
"disable_auto_compactions=false;"
|
||||
"compaction_measure_io_stats=true;",
|
||||
new_options));
|
||||
|
||||
ASSERT_EQ(unset_bytes_base,
|
||||
NumUnsetBytes(new_options_ptr, sizeof(ColumnFamilyOptions),
|
||||
kColumnFamilyOptionsBlacklist));
|
||||
|
||||
options->~ColumnFamilyOptions();
|
||||
new_options->~ColumnFamilyOptions();
|
||||
|
||||
delete[] options_ptr;
|
||||
delete[] new_options_ptr;
|
||||
}
|
||||
#endif // !__clang__
|
||||
#endif // OS_LINUX
|
||||
#endif // !ROCKSDB_LITE
|
||||
|
||||
} // namespace rocksdb
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
|
@ -6,6 +6,7 @@
|
||||
|
||||
#include <assert.h>
|
||||
#include <condition_variable>
|
||||
#include <functional>
|
||||
#include <mutex>
|
||||
#include <string>
|
||||
#include <unordered_set>
|
||||
|
@ -294,7 +294,7 @@ void RandomInitCFOptions(ColumnFamilyOptions* cf_opt, Random* rnd) {
|
||||
cf_opt->compaction_style = (CompactionStyle)(rnd->Uniform(4));
|
||||
|
||||
// boolean options
|
||||
cf_opt->compaction_measure_io_stats = rnd->Uniform(2);
|
||||
cf_opt->report_bg_io_stats = rnd->Uniform(2);
|
||||
cf_opt->disable_auto_compactions = rnd->Uniform(2);
|
||||
cf_opt->filter_deletes = rnd->Uniform(2);
|
||||
cf_opt->inplace_update_support = rnd->Uniform(2);
|
||||
|
@ -10,6 +10,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <atomic>
|
||||
#include <functional>
|
||||
#include <memory>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
@ -119,9 +119,9 @@ class BackupEngineImpl : public BackupEngine {
|
||||
void DeleteChildren(const std::string& dir, uint32_t file_type_filter = 0);
|
||||
|
||||
// Extends the "result" map with pathname->size mappings for the contents of
|
||||
// "dir". Pathnames are prefixed with "dir".
|
||||
// "dir" in "env". Pathnames are prefixed with "dir".
|
||||
Status InsertPathnameToSizeBytes(
|
||||
const std::string& dir,
|
||||
const std::string& dir, Env* env,
|
||||
std::unordered_map<std::string, uint64_t>* result);
|
||||
|
||||
struct FileInfo {
|
||||
@ -594,12 +594,13 @@ Status BackupEngineImpl::Initialize() {
|
||||
for (const auto& rel_dir :
|
||||
{GetSharedFileRel(), GetSharedFileWithChecksumRel()}) {
|
||||
const auto abs_dir = GetAbsolutePath(rel_dir);
|
||||
InsertPathnameToSizeBytes(abs_dir, &abs_path_to_size);
|
||||
InsertPathnameToSizeBytes(abs_dir, backup_env_, &abs_path_to_size);
|
||||
}
|
||||
// load the backups if any
|
||||
for (auto& backup : backups_) {
|
||||
InsertPathnameToSizeBytes(
|
||||
GetAbsolutePath(GetPrivateFileRel(backup.first)), &abs_path_to_size);
|
||||
GetAbsolutePath(GetPrivateFileRel(backup.first)), backup_env_,
|
||||
&abs_path_to_size);
|
||||
Status s =
|
||||
backup.second->LoadFromFile(options_.backup_dir, abs_path_to_size);
|
||||
if (!s.ok()) {
|
||||
@ -718,7 +719,7 @@ Status BackupEngineImpl::CreateNewBackupWithMetadata(
|
||||
// Pre-fetch sizes for data files
|
||||
std::unordered_map<std::string, uint64_t> data_path_to_size;
|
||||
if (s.ok()) {
|
||||
s = InsertPathnameToSizeBytes(db->GetName(), &data_path_to_size);
|
||||
s = InsertPathnameToSizeBytes(db->GetName(), db_env_, &data_path_to_size);
|
||||
}
|
||||
|
||||
std::vector<BackupAfterCopyOrCreateWorkItem> backup_items_to_finish;
|
||||
@ -776,7 +777,7 @@ Status BackupEngineImpl::CreateNewBackupWithMetadata(
|
||||
std::unordered_map<std::string, uint64_t> wal_path_to_size;
|
||||
if (s.ok()) {
|
||||
if (db->GetOptions().wal_dir != "") {
|
||||
s = InsertPathnameToSizeBytes(db->GetOptions().wal_dir,
|
||||
s = InsertPathnameToSizeBytes(db->GetOptions().wal_dir, db_env_,
|
||||
&wal_path_to_size);
|
||||
} else {
|
||||
wal_path_to_size = std::move(data_path_to_size);
|
||||
@ -1131,7 +1132,7 @@ Status BackupEngineImpl::VerifyBackup(BackupID backup_id) {
|
||||
for (const auto& rel_dir : {GetPrivateFileRel(backup_id), GetSharedFileRel(),
|
||||
GetSharedFileWithChecksumRel()}) {
|
||||
const auto abs_dir = GetAbsolutePath(rel_dir);
|
||||
InsertPathnameToSizeBytes(abs_dir, &curr_abs_path_to_size);
|
||||
InsertPathnameToSizeBytes(abs_dir, backup_env_, &curr_abs_path_to_size);
|
||||
}
|
||||
|
||||
for (const auto& file_info : backup->GetFiles()) {
|
||||
@ -1445,10 +1446,11 @@ void BackupEngineImpl::DeleteChildren(const std::string& dir,
|
||||
}
|
||||
|
||||
Status BackupEngineImpl::InsertPathnameToSizeBytes(
|
||||
const std::string& dir, std::unordered_map<std::string, uint64_t>* result) {
|
||||
const std::string& dir, Env* env,
|
||||
std::unordered_map<std::string, uint64_t>* result) {
|
||||
assert(result != nullptr);
|
||||
std::vector<Env::FileAttributes> files_attrs;
|
||||
Status status = backup_env_->GetChildrenFileAttributes(dir, &files_attrs);
|
||||
Status status = env->GetChildrenFileAttributes(dir, &files_attrs);
|
||||
if (!status.ok()) {
|
||||
return status;
|
||||
}
|
||||
|
@ -756,7 +756,7 @@ TEST_F(BackupableDBTest, NoDoubleCopy) {
|
||||
dummy_db_->live_files_ = { "/00010.sst", "/00011.sst",
|
||||
"/CURRENT", "/MANIFEST-01" };
|
||||
dummy_db_->wal_files_ = {{"/00011.log", true}, {"/00012.log", false}};
|
||||
test_backup_env_->SetFilenamesForMockedAttrs(dummy_db_->live_files_);
|
||||
test_db_env_->SetFilenamesForMockedAttrs(dummy_db_->live_files_);
|
||||
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), false));
|
||||
std::vector<std::string> should_have_written = {
|
||||
"/shared/00010.sst.tmp", "/shared/00011.sst.tmp",
|
||||
@ -773,7 +773,7 @@ TEST_F(BackupableDBTest, NoDoubleCopy) {
|
||||
dummy_db_->live_files_ = { "/00010.sst", "/00015.sst",
|
||||
"/CURRENT", "/MANIFEST-01" };
|
||||
dummy_db_->wal_files_ = {{"/00011.log", true}, {"/00012.log", false}};
|
||||
test_backup_env_->SetFilenamesForMockedAttrs(dummy_db_->live_files_);
|
||||
test_db_env_->SetFilenamesForMockedAttrs(dummy_db_->live_files_);
|
||||
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), false));
|
||||
// should not open 00010.sst - it's already there
|
||||
should_have_written = {
|
||||
@ -824,7 +824,7 @@ TEST_F(BackupableDBTest, DifferentEnvs) {
|
||||
dummy_db_->live_files_ = { "/00010.sst", "/00011.sst",
|
||||
"/CURRENT", "/MANIFEST-01" };
|
||||
dummy_db_->wal_files_ = {{"/00011.log", true}, {"/00012.log", false}};
|
||||
test_backup_env_->SetFilenamesForMockedAttrs(dummy_db_->live_files_);
|
||||
test_db_env_->SetFilenamesForMockedAttrs(dummy_db_->live_files_);
|
||||
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), false));
|
||||
|
||||
CloseDBAndBackupEngine();
|
||||
@ -836,7 +836,7 @@ TEST_F(BackupableDBTest, DifferentEnvs) {
|
||||
CloseDBAndBackupEngine();
|
||||
DestroyDB(dbname_, Options());
|
||||
|
||||
test_backup_env_->SetFilenamesForMockedAttrs({});
|
||||
test_db_env_->SetFilenamesForMockedAttrs({});
|
||||
AssertBackupConsistency(0, 0, 100, 500);
|
||||
}
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user