Compare commits
2 Commits
main
...
siying-pat
Author | SHA1 | Date | |
---|---|---|---|
|
ae2168838e | ||
|
f78e7672d1 |
@ -2,6 +2,7 @@ version: 2.1
|
|||||||
|
|
||||||
orbs:
|
orbs:
|
||||||
win: circleci/windows@2.4.0
|
win: circleci/windows@2.4.0
|
||||||
|
slack: circleci/slack@3.4.2
|
||||||
|
|
||||||
aliases:
|
aliases:
|
||||||
- ¬ify-on-main-failure
|
- ¬ify-on-main-failure
|
||||||
@ -56,6 +57,7 @@ commands:
|
|||||||
|
|
||||||
post-steps:
|
post-steps:
|
||||||
steps:
|
steps:
|
||||||
|
- slack/status: *notify-on-main-failure
|
||||||
- store_test_results: # store test result if there's any
|
- store_test_results: # store test result if there's any
|
||||||
path: /tmp/test-results
|
path: /tmp/test-results
|
||||||
- store_artifacts: # store LOG for debugging if there's any
|
- store_artifacts: # store LOG for debugging if there's any
|
||||||
@ -180,7 +182,7 @@ jobs:
|
|||||||
- increase-max-open-files-on-macos
|
- increase-max-open-files-on-macos
|
||||||
- install-gflags-on-macos
|
- install-gflags-on-macos
|
||||||
- pre-steps-macos
|
- pre-steps-macos
|
||||||
- run: ulimit -S -n `ulimit -H -n` && OPT=-DCIRCLECI make V=1 J=32 -j32 all
|
- run: ulimit -S -n 1048576 && OPT=-DCIRCLECI make V=1 J=32 -j32 all
|
||||||
- post-steps
|
- post-steps
|
||||||
|
|
||||||
build-macos-cmake:
|
build-macos-cmake:
|
||||||
@ -199,7 +201,7 @@ jobs:
|
|||||||
- pre-steps-macos
|
- pre-steps-macos
|
||||||
- run:
|
- run:
|
||||||
name: "cmake generate project file"
|
name: "cmake generate project file"
|
||||||
command: ulimit -S -n `ulimit -H -n` && mkdir build && cd build && cmake -DWITH_GFLAGS=1 ..
|
command: ulimit -S -n 1048576 && mkdir build && cd build && cmake -DWITH_GFLAGS=1 ..
|
||||||
- run:
|
- run:
|
||||||
name: "Build tests"
|
name: "Build tests"
|
||||||
command: cd build && make V=1 -j32
|
command: cd build && make V=1 -j32
|
||||||
@ -208,14 +210,14 @@ jobs:
|
|||||||
steps:
|
steps:
|
||||||
- run:
|
- run:
|
||||||
name: "Run even tests"
|
name: "Run even tests"
|
||||||
command: ulimit -S -n `ulimit -H -n` && cd build && ctest -j32 -I 0,,2
|
command: ulimit -S -n 1048576 && cd build && ctest -j32 -I 0,,2
|
||||||
- when:
|
- when:
|
||||||
condition:
|
condition:
|
||||||
not: << parameters.run_even_tests >>
|
not: << parameters.run_even_tests >>
|
||||||
steps:
|
steps:
|
||||||
- run:
|
- run:
|
||||||
name: "Run odd tests"
|
name: "Run odd tests"
|
||||||
command: ulimit -S -n `ulimit -H -n` && cd build && ctest -j32 -I 1,,2
|
command: ulimit -S -n 1048576 && cd build && ctest -j32 -I 1,,2
|
||||||
- post-steps
|
- post-steps
|
||||||
|
|
||||||
build-linux:
|
build-linux:
|
||||||
@ -411,13 +413,11 @@ jobs:
|
|||||||
build-linux-unity-and-headers:
|
build-linux-unity-and-headers:
|
||||||
docker: # executor type
|
docker: # executor type
|
||||||
- image: gcc:latest
|
- image: gcc:latest
|
||||||
environment:
|
|
||||||
EXTRA_CXXFLAGS: -mno-avx512f # Warnings-as-error in avx512fintrin.h, would be used on newer hardware
|
|
||||||
resource_class: large
|
resource_class: large
|
||||||
steps:
|
steps:
|
||||||
- checkout # check out the code in the project directory
|
- checkout # check out the code in the project directory
|
||||||
- run: apt-get update -y && apt-get install -y libgflags-dev
|
- run: apt-get update -y && apt-get install -y libgflags-dev
|
||||||
- run: make V=1 -j8 unity_test
|
- run: TEST_TMPDIR=/dev/shm && make V=1 -j8 unity_test
|
||||||
- run: make V=1 -j8 -k check-headers # could be moved to a different build
|
- run: make V=1 -j8 -k check-headers # could be moved to a different build
|
||||||
- post-steps
|
- post-steps
|
||||||
|
|
||||||
@ -506,7 +506,7 @@ jobs:
|
|||||||
- pre-steps
|
- pre-steps
|
||||||
- install-gflags
|
- install-gflags
|
||||||
- install-compression-libs
|
- install-compression-libs
|
||||||
- run: ulimit -S -n `ulimit -H -n` && make V=1 -j8 CRASH_TEST_EXT_ARGS=--duration=960 blackbox_crash_test_with_atomic_flush
|
- run: make V=1 -j8 CRASH_TEST_EXT_ARGS=--duration=960 blackbox_crash_test_with_atomic_flush
|
||||||
- post-steps
|
- post-steps
|
||||||
|
|
||||||
build-windows:
|
build-windows:
|
||||||
@ -828,76 +828,108 @@ jobs:
|
|||||||
|
|
||||||
workflows:
|
workflows:
|
||||||
version: 2
|
version: 2
|
||||||
jobs-linux-run-tests:
|
build-linux:
|
||||||
jobs:
|
jobs:
|
||||||
- build-linux
|
- build-linux
|
||||||
|
build-linux-cmake:
|
||||||
|
jobs:
|
||||||
- build-linux-cmake-with-folly
|
- build-linux-cmake-with-folly
|
||||||
- build-linux-gcc-7-with-folly
|
|
||||||
- build-linux-cmake-with-benchmark
|
- build-linux-cmake-with-benchmark
|
||||||
|
build-linux-encrypted_env-no_compression:
|
||||||
|
jobs:
|
||||||
- build-linux-encrypted_env-no_compression
|
- build-linux-encrypted_env-no_compression
|
||||||
|
build-linux-shared_lib-alt_namespace-status_checked:
|
||||||
|
jobs:
|
||||||
|
- build-linux-shared_lib-alt_namespace-status_checked
|
||||||
|
build-linux-lite:
|
||||||
|
jobs:
|
||||||
- build-linux-lite
|
- build-linux-lite
|
||||||
jobs-linux-run-tests-san:
|
build-linux-release:
|
||||||
|
jobs:
|
||||||
|
- build-linux-release
|
||||||
|
build-linux-release-rtti:
|
||||||
|
jobs:
|
||||||
|
- build-linux-release-rtti
|
||||||
|
build-linux-lite-release:
|
||||||
|
jobs:
|
||||||
|
- build-linux-lite-release
|
||||||
|
build-linux-clang10-asan:
|
||||||
jobs:
|
jobs:
|
||||||
- build-linux-clang10-asan
|
- build-linux-clang10-asan
|
||||||
- build-linux-clang10-ubsan
|
build-linux-clang10-mini-tsan:
|
||||||
|
jobs:
|
||||||
- build-linux-clang10-mini-tsan:
|
- build-linux-clang10-mini-tsan:
|
||||||
start_test: ""
|
start_test: ""
|
||||||
end_test: "env_test"
|
end_test: "env_test"
|
||||||
- build-linux-clang10-mini-tsan:
|
- build-linux-clang10-mini-tsan:
|
||||||
start_test: "env_test"
|
start_test: "env_test"
|
||||||
end_test: ""
|
end_test: ""
|
||||||
- build-linux-shared_lib-alt_namespace-status_checked
|
build-linux-clang10-ubsan:
|
||||||
jobs-linux-no-test-run:
|
|
||||||
jobs:
|
jobs:
|
||||||
- build-linux-release
|
- build-linux-clang10-ubsan
|
||||||
- build-linux-release-rtti
|
build-linux-clang10-clang-analyze:
|
||||||
- build-linux-lite-release
|
|
||||||
- build-examples
|
|
||||||
- build-fuzzers
|
|
||||||
- build-linux-clang-no_test_run
|
|
||||||
- build-linux-clang-13-no_test_run
|
|
||||||
- build-linux-gcc-8-no_test_run
|
|
||||||
- build-linux-gcc-10-cxx20-no_test_run
|
|
||||||
- build-linux-gcc-11-no_test_run
|
|
||||||
- build-linux-arm-cmake-no_test_run
|
|
||||||
jobs-linux-other-checks:
|
|
||||||
jobs:
|
jobs:
|
||||||
- build-linux-clang10-clang-analyze
|
- build-linux-clang10-clang-analyze
|
||||||
|
build-linux-unity-and-headers:
|
||||||
|
jobs:
|
||||||
- build-linux-unity-and-headers
|
- build-linux-unity-and-headers
|
||||||
|
build-linux-mini-crashtest:
|
||||||
|
jobs:
|
||||||
- build-linux-mini-crashtest
|
- build-linux-mini-crashtest
|
||||||
jobs-windows:
|
build-windows-vs2019:
|
||||||
jobs:
|
jobs:
|
||||||
- build-windows:
|
- build-windows:
|
||||||
name: "build-windows-vs2019"
|
name: "build-windows-vs2019"
|
||||||
|
build-windows-vs2019-cxx20:
|
||||||
|
jobs:
|
||||||
- build-windows:
|
- build-windows:
|
||||||
name: "build-windows-vs2019-cxx20"
|
name: "build-windows-vs2019-cxx20"
|
||||||
extra_cmake_opt: -DCMAKE_CXX_STANDARD=20
|
extra_cmake_opt: -DCMAKE_CXX_STANDARD=20
|
||||||
|
build-windows-vs2017:
|
||||||
|
jobs:
|
||||||
- build-windows:
|
- build-windows:
|
||||||
name: "build-windows-vs2017"
|
name: "build-windows-vs2017"
|
||||||
vs_year: "2017"
|
vs_year: "2017"
|
||||||
cmake_generator: "Visual Studio 15 Win64"
|
cmake_generator: "Visual Studio 15 Win64"
|
||||||
- build-cmake-mingw
|
build-java:
|
||||||
jobs-java:
|
|
||||||
jobs:
|
jobs:
|
||||||
- build-linux-java
|
- build-linux-java
|
||||||
- build-linux-java-static
|
- build-linux-java-static
|
||||||
- build-macos-java
|
- build-macos-java
|
||||||
- build-macos-java-static
|
- build-macos-java-static
|
||||||
- build-macos-java-static-universal
|
- build-macos-java-static-universal
|
||||||
jobs-macos:
|
build-examples:
|
||||||
|
jobs:
|
||||||
|
- build-examples
|
||||||
|
build-linux-compilers-no_test_run:
|
||||||
|
jobs:
|
||||||
|
- build-linux-clang-no_test_run
|
||||||
|
- build-linux-clang-13-no_test_run
|
||||||
|
- build-linux-gcc-7-with-folly
|
||||||
|
- build-linux-gcc-8-no_test_run
|
||||||
|
- build-linux-gcc-10-cxx20-no_test_run
|
||||||
|
- build-linux-gcc-11-no_test_run
|
||||||
|
- build-linux-arm-cmake-no_test_run
|
||||||
|
build-macos:
|
||||||
jobs:
|
jobs:
|
||||||
- build-macos
|
- build-macos
|
||||||
- build-macos-cmake:
|
- build-macos-cmake:
|
||||||
run_even_tests: true
|
run_even_tests: true
|
||||||
- build-macos-cmake:
|
- build-macos-cmake:
|
||||||
run_even_tests: false
|
run_even_tests: false
|
||||||
jobs-linux-arm:
|
build-cmake-mingw:
|
||||||
|
jobs:
|
||||||
|
- build-cmake-mingw
|
||||||
|
build-linux-arm:
|
||||||
jobs:
|
jobs:
|
||||||
- build-linux-arm
|
- build-linux-arm
|
||||||
|
build-fuzzers:
|
||||||
|
jobs:
|
||||||
|
- build-fuzzers
|
||||||
nightly:
|
nightly:
|
||||||
triggers:
|
triggers:
|
||||||
- schedule:
|
- schedule:
|
||||||
cron: "0 9 * * *"
|
cron: "0 0 * * *"
|
||||||
filters:
|
filters:
|
||||||
branches:
|
branches:
|
||||||
only:
|
only:
|
||||||
|
@ -40,8 +40,6 @@ include(GoogleTest)
|
|||||||
get_rocksdb_version(rocksdb_VERSION)
|
get_rocksdb_version(rocksdb_VERSION)
|
||||||
project(rocksdb
|
project(rocksdb
|
||||||
VERSION ${rocksdb_VERSION}
|
VERSION ${rocksdb_VERSION}
|
||||||
DESCRIPTION "An embeddable persistent key-value store for fast storage"
|
|
||||||
HOMEPAGE_URL https://rocksdb.org/
|
|
||||||
LANGUAGES CXX C ASM)
|
LANGUAGES CXX C ASM)
|
||||||
|
|
||||||
if(POLICY CMP0042)
|
if(POLICY CMP0042)
|
||||||
@ -598,7 +596,6 @@ set(SOURCES
|
|||||||
cache/cache_reservation_manager.cc
|
cache/cache_reservation_manager.cc
|
||||||
cache/clock_cache.cc
|
cache/clock_cache.cc
|
||||||
cache/compressed_secondary_cache.cc
|
cache/compressed_secondary_cache.cc
|
||||||
cache/fast_lru_cache.cc
|
|
||||||
cache/lru_cache.cc
|
cache/lru_cache.cc
|
||||||
cache/sharded_cache.cc
|
cache/sharded_cache.cc
|
||||||
db/arena_wrapped_db_iter.cc
|
db/arena_wrapped_db_iter.cc
|
||||||
@ -1123,12 +1120,6 @@ if(NOT WIN32 OR ROCKSDB_INSTALL_ON_WINDOWS)
|
|||||||
COMPATIBILITY SameMajorVersion
|
COMPATIBILITY SameMajorVersion
|
||||||
)
|
)
|
||||||
|
|
||||||
configure_file(
|
|
||||||
${CMAKE_CURRENT_SOURCE_DIR}/${PROJECT_NAME}.pc.in
|
|
||||||
${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}.pc
|
|
||||||
@ONLY
|
|
||||||
)
|
|
||||||
|
|
||||||
install(DIRECTORY include/rocksdb COMPONENT devel DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}")
|
install(DIRECTORY include/rocksdb COMPONENT devel DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}")
|
||||||
|
|
||||||
install(DIRECTORY "${PROJECT_SOURCE_DIR}/cmake/modules" COMPONENT devel DESTINATION ${package_config_destination})
|
install(DIRECTORY "${PROJECT_SOURCE_DIR}/cmake/modules" COMPONENT devel DESTINATION ${package_config_destination})
|
||||||
@ -1167,13 +1158,6 @@ if(NOT WIN32 OR ROCKSDB_INSTALL_ON_WINDOWS)
|
|||||||
COMPONENT devel
|
COMPONENT devel
|
||||||
DESTINATION ${package_config_destination}
|
DESTINATION ${package_config_destination}
|
||||||
)
|
)
|
||||||
|
|
||||||
install(
|
|
||||||
FILES
|
|
||||||
${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}.pc
|
|
||||||
COMPONENT devel
|
|
||||||
DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig
|
|
||||||
)
|
|
||||||
endif()
|
endif()
|
||||||
|
|
||||||
option(WITH_ALL_TESTS "Build all test, rather than a small subset" ON)
|
option(WITH_ALL_TESTS "Build all test, rather than a small subset" ON)
|
||||||
|
@ -5,7 +5,6 @@
|
|||||||
* Fixed a bug where RocksDB could corrupt DBs with `avoid_flush_during_recovery == true` by removing valid WALs, leading to `Status::Corruption` with message like "SST file is ahead of WALs" when attempting to reopen.
|
* Fixed a bug where RocksDB could corrupt DBs with `avoid_flush_during_recovery == true` by removing valid WALs, leading to `Status::Corruption` with message like "SST file is ahead of WALs" when attempting to reopen.
|
||||||
* Fixed a bug in async_io path where incorrect length of data is read by FilePrefetchBuffer if data is consumed from two populated buffers and request for more data is sent.
|
* Fixed a bug in async_io path where incorrect length of data is read by FilePrefetchBuffer if data is consumed from two populated buffers and request for more data is sent.
|
||||||
* Fixed a CompactionFilter bug. Compaction filter used to use `Delete` to remove keys, even if the keys should be removed with `SingleDelete`. Mixing `Delete` and `SingleDelete` may cause undefined behavior.
|
* Fixed a CompactionFilter bug. Compaction filter used to use `Delete` to remove keys, even if the keys should be removed with `SingleDelete`. Mixing `Delete` and `SingleDelete` may cause undefined behavior.
|
||||||
* Fixed a bug which might cause process crash when I/O error happens when reading an index block in MultiGet().
|
|
||||||
|
|
||||||
### New Features
|
### New Features
|
||||||
* DB::GetLiveFilesStorageInfo is ready for production use.
|
* DB::GetLiveFilesStorageInfo is ready for production use.
|
||||||
@ -15,9 +14,6 @@
|
|||||||
* Add rollback_deletion_type_callback to TransactionDBOptions so that write-prepared transactions know whether to issue a Delete or SingleDelete to cancel a previous key written during prior prepare phase. The PR aims to prevent mixing SingleDeletes and Deletes for the same key that can lead to undefined behaviors for write-prepared transactions.
|
* Add rollback_deletion_type_callback to TransactionDBOptions so that write-prepared transactions know whether to issue a Delete or SingleDelete to cancel a previous key written during prior prepare phase. The PR aims to prevent mixing SingleDeletes and Deletes for the same key that can lead to undefined behaviors for write-prepared transactions.
|
||||||
* EXPERIMENTAL: Add new API AbortIO in file_system to abort the read requests submitted asynchronously.
|
* EXPERIMENTAL: Add new API AbortIO in file_system to abort the read requests submitted asynchronously.
|
||||||
* CompactionFilter::Decision has a new value: kRemoveWithSingleDelete. If CompactionFilter returns this decision, then CompactionIterator will use `SingleDelete` to mark a key as removed.
|
* CompactionFilter::Decision has a new value: kRemoveWithSingleDelete. If CompactionFilter returns this decision, then CompactionIterator will use `SingleDelete` to mark a key as removed.
|
||||||
* Renamed CompactionFilter::Decision::kRemoveWithSingleDelete to kPurge since the latter sounds more general and hides the implementation details of how compaction iterator handles keys.
|
|
||||||
* Added ability to specify functions for Prepare and Validate to OptionsTypeInfo. Added methods to OptionTypeInfo to set the functions via an API. These methods are intended for RocksDB plugin developers for configuration management.
|
|
||||||
* Added a new immutable db options, enforce_single_del_contracts. If set to false (default is true), compaction will NOT fail due to a single delete followed by a delete for the same key. The purpose of this temporay option is to help existing use cases migrate.
|
|
||||||
|
|
||||||
### Bug Fixes
|
### Bug Fixes
|
||||||
* RocksDB calls FileSystem::Poll API during FilePrefetchBuffer destruction which impacts performance as it waits for read requets completion which is not needed anymore. Calling FileSystem::AbortIO to abort those requests instead fixes that performance issue.
|
* RocksDB calls FileSystem::Poll API during FilePrefetchBuffer destruction which impacts performance as it waits for read requets completion which is not needed anymore. Calling FileSystem::AbortIO to abort those requests instead fixes that performance issue.
|
||||||
@ -25,7 +21,6 @@
|
|||||||
|
|
||||||
### Behavior changes
|
### Behavior changes
|
||||||
* Enforce the existing contract of SingleDelete so that SingleDelete cannot be mixed with Delete because it leads to undefined behavior. Fix a number of unit tests that violate the contract but happen to pass.
|
* Enforce the existing contract of SingleDelete so that SingleDelete cannot be mixed with Delete because it leads to undefined behavior. Fix a number of unit tests that violate the contract but happen to pass.
|
||||||
* ldb `--try_load_options` default to true if `--db` is specified and not creating a new DB, the user can still explicitly disable that by `--try_load_options=false` (or explicitly enable that by `--try_load_options`).
|
|
||||||
|
|
||||||
## 7.2.0 (04/15/2022)
|
## 7.2.0 (04/15/2022)
|
||||||
### Bug Fixes
|
### Bug Fixes
|
||||||
|
116
Makefile
116
Makefile
@ -8,7 +8,7 @@
|
|||||||
|
|
||||||
BASH_EXISTS := $(shell which bash)
|
BASH_EXISTS := $(shell which bash)
|
||||||
SHELL := $(shell which bash)
|
SHELL := $(shell which bash)
|
||||||
include common.mk
|
include python.mk
|
||||||
|
|
||||||
CLEAN_FILES = # deliberately empty, so we can append below.
|
CLEAN_FILES = # deliberately empty, so we can append below.
|
||||||
CFLAGS += ${EXTRA_CFLAGS}
|
CFLAGS += ${EXTRA_CFLAGS}
|
||||||
@ -795,10 +795,17 @@ $(SHARED4): $(LIB_OBJECTS)
|
|||||||
$(AM_V_CCLD) $(CXX) $(PLATFORM_SHARED_LDFLAGS)$(SHARED3) $(LIB_OBJECTS) $(LDFLAGS) -o $@
|
$(AM_V_CCLD) $(CXX) $(PLATFORM_SHARED_LDFLAGS)$(SHARED3) $(LIB_OBJECTS) $(LDFLAGS) -o $@
|
||||||
endif # PLATFORM_SHARED_EXT
|
endif # PLATFORM_SHARED_EXT
|
||||||
|
|
||||||
.PHONY: check clean coverage ldb_tests package dbg gen-pc build_size \
|
.PHONY: blackbox_crash_test check clean coverage crash_test ldb_tests package \
|
||||||
release tags tags0 valgrind_check format static_lib shared_lib all \
|
release tags tags0 valgrind_check whitebox_crash_test format static_lib shared_lib all \
|
||||||
rocksdbjavastatic rocksdbjava install install-static install-shared \
|
dbg rocksdbjavastatic rocksdbjava gen-pc install install-static install-shared uninstall \
|
||||||
uninstall analyze tools tools_lib check-headers checkout_folly
|
analyze tools tools_lib check-headers checkout_folly \
|
||||||
|
blackbox_crash_test_with_atomic_flush whitebox_crash_test_with_atomic_flush \
|
||||||
|
blackbox_crash_test_with_txn whitebox_crash_test_with_txn \
|
||||||
|
blackbox_crash_test_with_best_efforts_recovery \
|
||||||
|
blackbox_crash_test_with_ts whitebox_crash_test_with_ts \
|
||||||
|
blackbox_crash_test_with_multiops_wc_txn \
|
||||||
|
blackbox_crash_test_with_multiops_wp_txn
|
||||||
|
|
||||||
|
|
||||||
all: $(LIBRARY) $(BENCHMARKS) tools tools_lib test_libs $(TESTS)
|
all: $(LIBRARY) $(BENCHMARKS) tools tools_lib test_libs $(TESTS)
|
||||||
|
|
||||||
@ -835,6 +842,18 @@ coverage: clean
|
|||||||
# Delete intermediate files
|
# Delete intermediate files
|
||||||
$(FIND) . -type f \( -name "*.gcda" -o -name "*.gcno" \) -exec rm -f {} \;
|
$(FIND) . -type f \( -name "*.gcda" -o -name "*.gcno" \) -exec rm -f {} \;
|
||||||
|
|
||||||
|
ifneq (,$(filter check parallel_check,$(MAKECMDGOALS)),)
|
||||||
|
# Use /dev/shm if it has the sticky bit set (otherwise, /tmp),
|
||||||
|
# and create a randomly-named rocksdb.XXXX directory therein.
|
||||||
|
# We'll use that directory in the "make check" rules.
|
||||||
|
ifeq ($(TMPD),)
|
||||||
|
TMPDIR := $(shell echo $${TMPDIR:-/tmp})
|
||||||
|
TMPD := $(shell f=/dev/shm; test -k $$f || f=$(TMPDIR); \
|
||||||
|
perl -le 'use File::Temp "tempdir";' \
|
||||||
|
-e 'print tempdir("'$$f'/rocksdb.XXXX", CLEANUP => 0)')
|
||||||
|
endif
|
||||||
|
endif
|
||||||
|
|
||||||
# Run all tests in parallel, accumulating per-test logs in t/log-*.
|
# Run all tests in parallel, accumulating per-test logs in t/log-*.
|
||||||
#
|
#
|
||||||
# Each t/run-* file is a tiny generated bourne shell script that invokes one of
|
# Each t/run-* file is a tiny generated bourne shell script that invokes one of
|
||||||
@ -874,7 +893,7 @@ $(parallel_tests):
|
|||||||
TEST_SCRIPT=t/run-$$TEST_BINARY-$${TEST_NAME//\//-}; \
|
TEST_SCRIPT=t/run-$$TEST_BINARY-$${TEST_NAME//\//-}; \
|
||||||
printf '%s\n' \
|
printf '%s\n' \
|
||||||
'#!/bin/sh' \
|
'#!/bin/sh' \
|
||||||
"d=\$(TEST_TMPDIR)$$TEST_SCRIPT" \
|
"d=\$(TMPD)$$TEST_SCRIPT" \
|
||||||
'mkdir -p $$d' \
|
'mkdir -p $$d' \
|
||||||
"TEST_TMPDIR=\$$d $(DRIVER) ./$$TEST_BINARY --gtest_filter=$$TEST_NAME" \
|
"TEST_TMPDIR=\$$d $(DRIVER) ./$$TEST_BINARY --gtest_filter=$$TEST_NAME" \
|
||||||
> $$TEST_SCRIPT; \
|
> $$TEST_SCRIPT; \
|
||||||
@ -934,6 +953,7 @@ endif
|
|||||||
|
|
||||||
.PHONY: check_0
|
.PHONY: check_0
|
||||||
check_0:
|
check_0:
|
||||||
|
$(AM_V_GEN)export TEST_TMPDIR=$(TMPD); \
|
||||||
printf '%s\n' '' \
|
printf '%s\n' '' \
|
||||||
'To monitor subtest <duration,pass/fail,name>,' \
|
'To monitor subtest <duration,pass/fail,name>,' \
|
||||||
' run "make watch-log" in a separate window' ''; \
|
' run "make watch-log" in a separate window' ''; \
|
||||||
@ -944,8 +964,7 @@ check_0:
|
|||||||
| $(prioritize_long_running_tests) \
|
| $(prioritize_long_running_tests) \
|
||||||
| grep -E '$(tests-regexp)' \
|
| grep -E '$(tests-regexp)' \
|
||||||
| grep -E -v '$(EXCLUDE_TESTS_REGEX)' \
|
| grep -E -v '$(EXCLUDE_TESTS_REGEX)' \
|
||||||
| build_tools/gnu_parallel -j$(J) --plain --joblog=LOG --eta --gnu \
|
| build_tools/gnu_parallel -j$(J) --plain --joblog=LOG --eta --gnu '{} $(parallel_redir)' ; \
|
||||||
--tmpdir=$(TEST_TMPDIR) '{} $(parallel_redir)' ; \
|
|
||||||
parallel_retcode=$$? ; \
|
parallel_retcode=$$? ; \
|
||||||
awk '{ if ($$7 != 0 || $$8 != 0) { if ($$7 == "Exitval") { h = $$0; } else { if (!f) print h; print; f = 1 } } } END { if(f) exit 1; }' < LOG ; \
|
awk '{ if ($$7 != 0 || $$8 != 0) { if ($$7 == "Exitval") { h = $$0; } else { if (!f) print h; print; f = 1 } } } END { if(f) exit 1; }' < LOG ; \
|
||||||
awk_retcode=$$?; \
|
awk_retcode=$$?; \
|
||||||
@ -956,6 +975,7 @@ valgrind-exclude-regexp = InlineSkipTest.ConcurrentInsert|TransactionStressTest.
|
|||||||
.PHONY: valgrind_check_0
|
.PHONY: valgrind_check_0
|
||||||
valgrind_check_0: test_log_prefix := valgrind_
|
valgrind_check_0: test_log_prefix := valgrind_
|
||||||
valgrind_check_0:
|
valgrind_check_0:
|
||||||
|
$(AM_V_GEN)export TEST_TMPDIR=$(TMPD); \
|
||||||
printf '%s\n' '' \
|
printf '%s\n' '' \
|
||||||
'To monitor subtest <duration,pass/fail,name>,' \
|
'To monitor subtest <duration,pass/fail,name>,' \
|
||||||
' run "make watch-log" in a separate window' ''; \
|
' run "make watch-log" in a separate window' ''; \
|
||||||
@ -967,11 +987,10 @@ valgrind_check_0:
|
|||||||
| grep -E '$(tests-regexp)' \
|
| grep -E '$(tests-regexp)' \
|
||||||
| grep -E -v '$(valgrind-exclude-regexp)' \
|
| grep -E -v '$(valgrind-exclude-regexp)' \
|
||||||
| build_tools/gnu_parallel -j$(J) --plain --joblog=LOG --eta --gnu \
|
| build_tools/gnu_parallel -j$(J) --plain --joblog=LOG --eta --gnu \
|
||||||
--tmpdir=$(TEST_TMPDIR) \
|
|
||||||
'(if [[ "{}" == "./"* ]] ; then $(DRIVER) {}; else {}; fi) \
|
'(if [[ "{}" == "./"* ]] ; then $(DRIVER) {}; else {}; fi) \
|
||||||
$(parallel_redir)' \
|
$(parallel_redir)' \
|
||||||
|
|
||||||
CLEAN_FILES += t LOG $(TEST_TMPDIR)
|
CLEAN_FILES += t LOG $(TMPD)
|
||||||
|
|
||||||
# When running parallel "make check", you can monitor its progress
|
# When running parallel "make check", you can monitor its progress
|
||||||
# from another window.
|
# from another window.
|
||||||
@ -994,12 +1013,12 @@ check: all
|
|||||||
&& (build_tools/gnu_parallel --gnu --help 2>/dev/null) | \
|
&& (build_tools/gnu_parallel --gnu --help 2>/dev/null) | \
|
||||||
grep -q 'GNU Parallel'; \
|
grep -q 'GNU Parallel'; \
|
||||||
then \
|
then \
|
||||||
$(MAKE) T="$$t" check_0; \
|
$(MAKE) T="$$t" TMPD=$(TMPD) check_0; \
|
||||||
else \
|
else \
|
||||||
for t in $(TESTS); do \
|
for t in $(TESTS); do \
|
||||||
echo "===== Running $$t (`date`)"; ./$$t || exit 1; done; \
|
echo "===== Running $$t (`date`)"; ./$$t || exit 1; done; \
|
||||||
fi
|
fi
|
||||||
rm -rf $(TEST_TMPDIR)
|
rm -rf $(TMPD)
|
||||||
ifneq ($(PLATFORM), OS_AIX)
|
ifneq ($(PLATFORM), OS_AIX)
|
||||||
$(PYTHON) tools/check_all_python.py
|
$(PYTHON) tools/check_all_python.py
|
||||||
ifeq ($(filter -DROCKSDB_LITE,$(OPT)),)
|
ifeq ($(filter -DROCKSDB_LITE,$(OPT)),)
|
||||||
@ -1099,7 +1118,7 @@ valgrind_check: $(TESTS)
|
|||||||
&& (build_tools/gnu_parallel --gnu --help 2>/dev/null) | \
|
&& (build_tools/gnu_parallel --gnu --help 2>/dev/null) | \
|
||||||
grep -q 'GNU Parallel'; \
|
grep -q 'GNU Parallel'; \
|
||||||
then \
|
then \
|
||||||
$(MAKE) \
|
$(MAKE) TMPD=$(TMPD) \
|
||||||
DRIVER="$(VALGRIND_VER) $(VALGRIND_OPTS)" valgrind_check_0; \
|
DRIVER="$(VALGRIND_VER) $(VALGRIND_OPTS)" valgrind_check_0; \
|
||||||
else \
|
else \
|
||||||
for t in $(filter-out %skiplist_test options_settable_test,$(TESTS)); do \
|
for t in $(filter-out %skiplist_test options_settable_test,$(TESTS)); do \
|
||||||
@ -1120,6 +1139,27 @@ valgrind_check_some: $(ROCKSDBTESTS_SUBSET)
|
|||||||
fi; \
|
fi; \
|
||||||
done
|
done
|
||||||
|
|
||||||
|
ifneq ($(PAR_TEST),)
|
||||||
|
parloop:
|
||||||
|
ret_bad=0; \
|
||||||
|
for t in $(PAR_TEST); do \
|
||||||
|
echo "===== Running $$t in parallel $(NUM_PAR) (`date`)";\
|
||||||
|
if [ $(db_test) -eq 1 ]; then \
|
||||||
|
seq $(J) | v="$$t" build_tools/gnu_parallel --gnu --plain 's=$(TMPD)/rdb-{}; export TEST_TMPDIR=$$s;' \
|
||||||
|
'timeout 2m ./db_test --gtest_filter=$$v >> $$s/log-{} 2>1'; \
|
||||||
|
else\
|
||||||
|
seq $(J) | v="./$$t" build_tools/gnu_parallel --gnu --plain 's=$(TMPD)/rdb-{};' \
|
||||||
|
'export TEST_TMPDIR=$$s; timeout 10m $$v >> $$s/log-{} 2>1'; \
|
||||||
|
fi; \
|
||||||
|
ret_code=$$?; \
|
||||||
|
if [ $$ret_code -ne 0 ]; then \
|
||||||
|
ret_bad=$$ret_code; \
|
||||||
|
echo $$t exited with $$ret_code; \
|
||||||
|
fi; \
|
||||||
|
done; \
|
||||||
|
exit $$ret_bad;
|
||||||
|
endif
|
||||||
|
|
||||||
test_names = \
|
test_names = \
|
||||||
./db_test --gtest_list_tests \
|
./db_test --gtest_list_tests \
|
||||||
| perl -n \
|
| perl -n \
|
||||||
@ -1127,6 +1167,24 @@ test_names = \
|
|||||||
-e '/^(\s*)(\S+)/; !$$1 and do {$$p=$$2; break};' \
|
-e '/^(\s*)(\S+)/; !$$1 and do {$$p=$$2; break};' \
|
||||||
-e 'print qq! $$p$$2!'
|
-e 'print qq! $$p$$2!'
|
||||||
|
|
||||||
|
parallel_check: $(TESTS)
|
||||||
|
$(AM_V_GEN)if test "$(J)" > 1 \
|
||||||
|
&& (build_tools/gnu_parallel --gnu --help 2>/dev/null) | \
|
||||||
|
grep -q 'GNU Parallel'; \
|
||||||
|
then \
|
||||||
|
echo Running in parallel $(J); \
|
||||||
|
else \
|
||||||
|
echo "Need to have GNU Parallel and J > 1"; exit 1; \
|
||||||
|
fi; \
|
||||||
|
ret_bad=0; \
|
||||||
|
echo $(J);\
|
||||||
|
echo Test Dir: $(TMPD); \
|
||||||
|
seq $(J) | build_tools/gnu_parallel --gnu --plain 's=$(TMPD)/rdb-{}; rm -rf $$s; mkdir $$s'; \
|
||||||
|
$(MAKE) PAR_TEST="$(shell $(test_names))" TMPD=$(TMPD) \
|
||||||
|
J=$(J) db_test=1 parloop; \
|
||||||
|
$(MAKE) PAR_TEST="$(filter-out db_test, $(TESTS))" \
|
||||||
|
TMPD=$(TMPD) J=$(J) db_test=0 parloop;
|
||||||
|
|
||||||
analyze: clean
|
analyze: clean
|
||||||
USE_CLANG=1 $(MAKE) analyze_incremental
|
USE_CLANG=1 $(MAKE) analyze_incremental
|
||||||
|
|
||||||
@ -2346,38 +2404,6 @@ checkout_folly:
|
|||||||
@# NOTE: this hack is not needed if using FBCODE compiler config
|
@# NOTE: this hack is not needed if using FBCODE compiler config
|
||||||
perl -pi -e 's/^(#include <boost)/\/\/$$1/' third-party/folly/folly/functional/Invoke.h
|
perl -pi -e 's/^(#include <boost)/\/\/$$1/' third-party/folly/folly/functional/Invoke.h
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# Build size testing
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
REPORT_BUILD_STATISTIC?=echo STATISTIC:
|
|
||||||
|
|
||||||
build_size:
|
|
||||||
# === normal build, static ===
|
|
||||||
$(MAKE) clean
|
|
||||||
$(MAKE) static_lib
|
|
||||||
$(REPORT_BUILD_STATISTIC) rocksdb.build_size.static_lib $$(stat --printf="%s" librocksdb.a)
|
|
||||||
strip librocksdb.a
|
|
||||||
$(REPORT_BUILD_STATISTIC) rocksdb.build_size.static_lib_stripped $$(stat --printf="%s" librocksdb.a)
|
|
||||||
# === normal build, shared ===
|
|
||||||
$(MAKE) clean
|
|
||||||
$(MAKE) shared_lib
|
|
||||||
$(REPORT_BUILD_STATISTIC) rocksdb.build_size.shared_lib $$(stat --printf="%s" `readlink -f librocksdb.so`)
|
|
||||||
strip `readlink -f librocksdb.so`
|
|
||||||
$(REPORT_BUILD_STATISTIC) rocksdb.build_size.shared_lib_stripped $$(stat --printf="%s" `readlink -f librocksdb.so`)
|
|
||||||
# === lite build, static ===
|
|
||||||
$(MAKE) clean
|
|
||||||
$(MAKE) LITE=1 static_lib
|
|
||||||
$(REPORT_BUILD_STATISTIC) rocksdb.build_size.static_lib_lite $$(stat --printf="%s" librocksdb.a)
|
|
||||||
strip librocksdb.a
|
|
||||||
$(REPORT_BUILD_STATISTIC) rocksdb.build_size.static_lib_lite_stripped $$(stat --printf="%s" librocksdb.a)
|
|
||||||
# === lite build, shared ===
|
|
||||||
$(MAKE) clean
|
|
||||||
$(MAKE) LITE=1 shared_lib
|
|
||||||
$(REPORT_BUILD_STATISTIC) rocksdb.build_size.shared_lib_lite $$(stat --printf="%s" `readlink -f librocksdb.so`)
|
|
||||||
strip `readlink -f librocksdb.so`
|
|
||||||
$(REPORT_BUILD_STATISTIC) rocksdb.build_size.shared_lib_lite_stripped $$(stat --printf="%s" `readlink -f librocksdb.so`)
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Platform-specific compilation
|
# Platform-specific compilation
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
@ -4,4 +4,3 @@ This is the list of all known third-party plugins for RocksDB. If something is m
|
|||||||
* [HDFS](https://github.com/riversand963/rocksdb-hdfs-env): an Env used for interacting with HDFS. Migrated from main RocksDB repo
|
* [HDFS](https://github.com/riversand963/rocksdb-hdfs-env): an Env used for interacting with HDFS. Migrated from main RocksDB repo
|
||||||
* [ZenFS](https://github.com/westerndigitalcorporation/zenfs): a file system for zoned block devices
|
* [ZenFS](https://github.com/westerndigitalcorporation/zenfs): a file system for zoned block devices
|
||||||
* [RADOS](https://github.com/riversand963/rocksdb-rados-env): an Env used for interacting with RADOS. Migrated from RocksDB main repo.
|
* [RADOS](https://github.com/riversand963/rocksdb-rados-env): an Env used for interacting with RADOS. Migrated from RocksDB main repo.
|
||||||
* [PMEM](https://github.com/pmem/pmem-rocksdb-plugin): a collection of plugins to enable Persistent Memory on RocksDB.
|
|
||||||
|
2
TARGETS
2
TARGETS
@ -15,7 +15,6 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[
|
|||||||
"cache/cache_reservation_manager.cc",
|
"cache/cache_reservation_manager.cc",
|
||||||
"cache/clock_cache.cc",
|
"cache/clock_cache.cc",
|
||||||
"cache/compressed_secondary_cache.cc",
|
"cache/compressed_secondary_cache.cc",
|
||||||
"cache/fast_lru_cache.cc",
|
|
||||||
"cache/lru_cache.cc",
|
"cache/lru_cache.cc",
|
||||||
"cache/sharded_cache.cc",
|
"cache/sharded_cache.cc",
|
||||||
"db/arena_wrapped_db_iter.cc",
|
"db/arena_wrapped_db_iter.cc",
|
||||||
@ -336,7 +335,6 @@ cpp_library_wrapper(name="rocksdb_whole_archive_lib", srcs=[
|
|||||||
"cache/cache_reservation_manager.cc",
|
"cache/cache_reservation_manager.cc",
|
||||||
"cache/clock_cache.cc",
|
"cache/clock_cache.cc",
|
||||||
"cache/compressed_secondary_cache.cc",
|
"cache/compressed_secondary_cache.cc",
|
||||||
"cache/fast_lru_cache.cc",
|
|
||||||
"cache/lru_cache.cc",
|
"cache/lru_cache.cc",
|
||||||
"cache/sharded_cache.cc",
|
"cache/sharded_cache.cc",
|
||||||
"db/arena_wrapped_db_iter.cc",
|
"db/arena_wrapped_db_iter.cc",
|
||||||
|
@ -469,7 +469,7 @@ EOF
|
|||||||
|
|
||||||
if ! test $ROCKSDB_DISABLE_MEMKIND; then
|
if ! test $ROCKSDB_DISABLE_MEMKIND; then
|
||||||
# Test whether memkind library is installed
|
# Test whether memkind library is installed
|
||||||
$CXX $PLATFORM_CXXFLAGS $LDFLAGS -x c++ - -o test.o -lmemkind 2>/dev/null <<EOF
|
$CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS -lmemkind -x c++ - -o test.o 2>/dev/null <<EOF
|
||||||
#include <memkind.h>
|
#include <memkind.h>
|
||||||
int main() {
|
int main() {
|
||||||
memkind_malloc(MEMKIND_DAX_KMEM, 1024);
|
memkind_malloc(MEMKIND_DAX_KMEM, 1024);
|
||||||
|
@ -118,19 +118,10 @@ if [ -z "$USE_CLANG" ]; then
|
|||||||
CXX="$GCC_BASE/bin/g++"
|
CXX="$GCC_BASE/bin/g++"
|
||||||
AR="$GCC_BASE/bin/gcc-ar"
|
AR="$GCC_BASE/bin/gcc-ar"
|
||||||
|
|
||||||
CFLAGS+=" -B$BINUTILS -nostdinc -nostdlib"
|
|
||||||
CFLAGS+=" -I$GCC_BASE/include"
|
CFLAGS+=" -B$BINUTILS"
|
||||||
CFLAGS+=" -isystem $GCC_BASE/lib/gcc/x86_64-redhat-linux-gnu/11.2.1/include"
|
|
||||||
CFLAGS+=" -isystem $GCC_BASE/lib/gcc/x86_64-redhat-linux-gnu/11.2.1/install-tools/include"
|
|
||||||
CFLAGS+=" -isystem $GCC_BASE/lib/gcc/x86_64-redhat-linux-gnu/11.2.1/include-fixed/"
|
|
||||||
CFLAGS+=" -isystem $LIBGCC_INCLUDE"
|
CFLAGS+=" -isystem $LIBGCC_INCLUDE"
|
||||||
CFLAGS+=" -isystem $GLIBC_INCLUDE"
|
CFLAGS+=" -isystem $GLIBC_INCLUDE"
|
||||||
CFLAGS+=" -I$GLIBC_INCLUDE"
|
|
||||||
CFLAGS+=" -I$LIBGCC_BASE/include"
|
|
||||||
CFLAGS+=" -I$LIBGCC_BASE/include/c++/11.x/"
|
|
||||||
CFLAGS+=" -I$LIBGCC_BASE/include/c++/11.x/x86_64-facebook-linux/"
|
|
||||||
CFLAGS+=" -I$LIBGCC_BASE/include/c++/11.x/backward"
|
|
||||||
CFLAGS+=" -isystem $GLIBC_INCLUDE -I$GLIBC_INCLUDE"
|
|
||||||
JEMALLOC=1
|
JEMALLOC=1
|
||||||
else
|
else
|
||||||
# clang
|
# clang
|
||||||
|
46
cache/cache_test.cc
vendored
46
cache/cache_test.cc
vendored
@ -14,9 +14,7 @@
|
|||||||
#include <iostream>
|
#include <iostream>
|
||||||
#include <string>
|
#include <string>
|
||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
#include "cache/clock_cache.h"
|
#include "cache/clock_cache.h"
|
||||||
#include "cache/fast_lru_cache.h"
|
|
||||||
#include "cache/lru_cache.h"
|
#include "cache/lru_cache.h"
|
||||||
#include "test_util/testharness.h"
|
#include "test_util/testharness.h"
|
||||||
#include "util/coding.h"
|
#include "util/coding.h"
|
||||||
@ -41,7 +39,6 @@ static int DecodeValue(void* v) {
|
|||||||
|
|
||||||
const std::string kLRU = "lru";
|
const std::string kLRU = "lru";
|
||||||
const std::string kClock = "clock";
|
const std::string kClock = "clock";
|
||||||
const std::string kFast = "fast";
|
|
||||||
|
|
||||||
void dumbDeleter(const Slice& /*key*/, void* /*value*/) {}
|
void dumbDeleter(const Slice& /*key*/, void* /*value*/) {}
|
||||||
|
|
||||||
@ -86,9 +83,6 @@ class CacheTest : public testing::TestWithParam<std::string> {
|
|||||||
if (type == kClock) {
|
if (type == kClock) {
|
||||||
return NewClockCache(capacity);
|
return NewClockCache(capacity);
|
||||||
}
|
}
|
||||||
if (type == kFast) {
|
|
||||||
return NewFastLRUCache(capacity);
|
|
||||||
}
|
|
||||||
return nullptr;
|
return nullptr;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -109,10 +103,6 @@ class CacheTest : public testing::TestWithParam<std::string> {
|
|||||||
return NewClockCache(capacity, num_shard_bits, strict_capacity_limit,
|
return NewClockCache(capacity, num_shard_bits, strict_capacity_limit,
|
||||||
charge_policy);
|
charge_policy);
|
||||||
}
|
}
|
||||||
if (type == kFast) {
|
|
||||||
return NewFastLRUCache(capacity, num_shard_bits, strict_capacity_limit,
|
|
||||||
charge_policy);
|
|
||||||
}
|
|
||||||
return nullptr;
|
return nullptr;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -193,7 +183,7 @@ TEST_P(CacheTest, UsageTest) {
|
|||||||
|
|
||||||
// make sure the cache will be overloaded
|
// make sure the cache will be overloaded
|
||||||
for (uint64_t i = 1; i < kCapacity; ++i) {
|
for (uint64_t i = 1; i < kCapacity; ++i) {
|
||||||
auto key = std::to_string(i);
|
auto key = ToString(i);
|
||||||
ASSERT_OK(cache->Insert(key, reinterpret_cast<void*>(value), key.size() + 5,
|
ASSERT_OK(cache->Insert(key, reinterpret_cast<void*>(value), key.size() + 5,
|
||||||
dumbDeleter));
|
dumbDeleter));
|
||||||
ASSERT_OK(precise_cache->Insert(key, reinterpret_cast<void*>(value),
|
ASSERT_OK(precise_cache->Insert(key, reinterpret_cast<void*>(value),
|
||||||
@ -265,7 +255,7 @@ TEST_P(CacheTest, PinnedUsageTest) {
|
|||||||
|
|
||||||
// check that overloading the cache does not change the pinned usage
|
// check that overloading the cache does not change the pinned usage
|
||||||
for (uint64_t i = 1; i < 2 * kCapacity; ++i) {
|
for (uint64_t i = 1; i < 2 * kCapacity; ++i) {
|
||||||
auto key = std::to_string(i);
|
auto key = ToString(i);
|
||||||
ASSERT_OK(cache->Insert(key, reinterpret_cast<void*>(value), key.size() + 5,
|
ASSERT_OK(cache->Insert(key, reinterpret_cast<void*>(value), key.size() + 5,
|
||||||
dumbDeleter));
|
dumbDeleter));
|
||||||
ASSERT_OK(precise_cache->Insert(key, reinterpret_cast<void*>(value),
|
ASSERT_OK(precise_cache->Insert(key, reinterpret_cast<void*>(value),
|
||||||
@ -585,7 +575,7 @@ TEST_P(CacheTest, SetCapacity) {
|
|||||||
std::vector<Cache::Handle*> handles(10);
|
std::vector<Cache::Handle*> handles(10);
|
||||||
// Insert 5 entries, but not releasing.
|
// Insert 5 entries, but not releasing.
|
||||||
for (size_t i = 0; i < 5; i++) {
|
for (size_t i = 0; i < 5; i++) {
|
||||||
std::string key = std::to_string(i + 1);
|
std::string key = ToString(i+1);
|
||||||
Status s = cache->Insert(key, new Value(i + 1), 1, &deleter, &handles[i]);
|
Status s = cache->Insert(key, new Value(i + 1), 1, &deleter, &handles[i]);
|
||||||
ASSERT_TRUE(s.ok());
|
ASSERT_TRUE(s.ok());
|
||||||
}
|
}
|
||||||
@ -600,7 +590,7 @@ TEST_P(CacheTest, SetCapacity) {
|
|||||||
// then decrease capacity to 7, final capacity should be 7
|
// then decrease capacity to 7, final capacity should be 7
|
||||||
// and usage should be 7
|
// and usage should be 7
|
||||||
for (size_t i = 5; i < 10; i++) {
|
for (size_t i = 5; i < 10; i++) {
|
||||||
std::string key = std::to_string(i + 1);
|
std::string key = ToString(i+1);
|
||||||
Status s = cache->Insert(key, new Value(i + 1), 1, &deleter, &handles[i]);
|
Status s = cache->Insert(key, new Value(i + 1), 1, &deleter, &handles[i]);
|
||||||
ASSERT_TRUE(s.ok());
|
ASSERT_TRUE(s.ok());
|
||||||
}
|
}
|
||||||
@ -631,7 +621,7 @@ TEST_P(LRUCacheTest, SetStrictCapacityLimit) {
|
|||||||
std::vector<Cache::Handle*> handles(10);
|
std::vector<Cache::Handle*> handles(10);
|
||||||
Status s;
|
Status s;
|
||||||
for (size_t i = 0; i < 10; i++) {
|
for (size_t i = 0; i < 10; i++) {
|
||||||
std::string key = std::to_string(i + 1);
|
std::string key = ToString(i + 1);
|
||||||
s = cache->Insert(key, new Value(i + 1), 1, &deleter, &handles[i]);
|
s = cache->Insert(key, new Value(i + 1), 1, &deleter, &handles[i]);
|
||||||
ASSERT_OK(s);
|
ASSERT_OK(s);
|
||||||
ASSERT_NE(nullptr, handles[i]);
|
ASSERT_NE(nullptr, handles[i]);
|
||||||
@ -655,7 +645,7 @@ TEST_P(LRUCacheTest, SetStrictCapacityLimit) {
|
|||||||
// test3: init with flag being true.
|
// test3: init with flag being true.
|
||||||
std::shared_ptr<Cache> cache2 = NewCache(5, 0, true);
|
std::shared_ptr<Cache> cache2 = NewCache(5, 0, true);
|
||||||
for (size_t i = 0; i < 5; i++) {
|
for (size_t i = 0; i < 5; i++) {
|
||||||
std::string key = std::to_string(i + 1);
|
std::string key = ToString(i + 1);
|
||||||
s = cache2->Insert(key, new Value(i + 1), 1, &deleter, &handles[i]);
|
s = cache2->Insert(key, new Value(i + 1), 1, &deleter, &handles[i]);
|
||||||
ASSERT_OK(s);
|
ASSERT_OK(s);
|
||||||
ASSERT_NE(nullptr, handles[i]);
|
ASSERT_NE(nullptr, handles[i]);
|
||||||
@ -685,14 +675,14 @@ TEST_P(CacheTest, OverCapacity) {
|
|||||||
|
|
||||||
// Insert n+1 entries, but not releasing.
|
// Insert n+1 entries, but not releasing.
|
||||||
for (size_t i = 0; i < n + 1; i++) {
|
for (size_t i = 0; i < n + 1; i++) {
|
||||||
std::string key = std::to_string(i + 1);
|
std::string key = ToString(i+1);
|
||||||
Status s = cache->Insert(key, new Value(i + 1), 1, &deleter, &handles[i]);
|
Status s = cache->Insert(key, new Value(i + 1), 1, &deleter, &handles[i]);
|
||||||
ASSERT_TRUE(s.ok());
|
ASSERT_TRUE(s.ok());
|
||||||
}
|
}
|
||||||
|
|
||||||
// Guess what's in the cache now?
|
// Guess what's in the cache now?
|
||||||
for (size_t i = 0; i < n + 1; i++) {
|
for (size_t i = 0; i < n + 1; i++) {
|
||||||
std::string key = std::to_string(i + 1);
|
std::string key = ToString(i+1);
|
||||||
auto h = cache->Lookup(key);
|
auto h = cache->Lookup(key);
|
||||||
ASSERT_TRUE(h != nullptr);
|
ASSERT_TRUE(h != nullptr);
|
||||||
if (h) cache->Release(h);
|
if (h) cache->Release(h);
|
||||||
@ -713,7 +703,7 @@ TEST_P(CacheTest, OverCapacity) {
|
|||||||
// This is consistent with the LRU policy since the element 0
|
// This is consistent with the LRU policy since the element 0
|
||||||
// was released first
|
// was released first
|
||||||
for (size_t i = 0; i < n + 1; i++) {
|
for (size_t i = 0; i < n + 1; i++) {
|
||||||
std::string key = std::to_string(i + 1);
|
std::string key = ToString(i+1);
|
||||||
auto h = cache->Lookup(key);
|
auto h = cache->Lookup(key);
|
||||||
if (h) {
|
if (h) {
|
||||||
ASSERT_NE(i, 0U);
|
ASSERT_NE(i, 0U);
|
||||||
@ -754,9 +744,9 @@ TEST_P(CacheTest, ApplyToAllEntriesTest) {
|
|||||||
std::vector<std::string> callback_state;
|
std::vector<std::string> callback_state;
|
||||||
const auto callback = [&](const Slice& key, void* value, size_t charge,
|
const auto callback = [&](const Slice& key, void* value, size_t charge,
|
||||||
Cache::DeleterFn deleter) {
|
Cache::DeleterFn deleter) {
|
||||||
callback_state.push_back(std::to_string(DecodeKey(key)) + "," +
|
callback_state.push_back(ToString(DecodeKey(key)) + "," +
|
||||||
std::to_string(DecodeValue(value)) + "," +
|
ToString(DecodeValue(value)) + "," +
|
||||||
std::to_string(charge));
|
ToString(charge));
|
||||||
assert(deleter == &CacheTest::Deleter);
|
assert(deleter == &CacheTest::Deleter);
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -765,8 +755,8 @@ TEST_P(CacheTest, ApplyToAllEntriesTest) {
|
|||||||
|
|
||||||
for (int i = 0; i < 10; ++i) {
|
for (int i = 0; i < 10; ++i) {
|
||||||
Insert(i, i * 2, i + 1);
|
Insert(i, i * 2, i + 1);
|
||||||
inserted.push_back(std::to_string(i) + "," + std::to_string(i * 2) + "," +
|
inserted.push_back(ToString(i) + "," + ToString(i * 2) + "," +
|
||||||
std::to_string(i + 1));
|
ToString(i + 1));
|
||||||
}
|
}
|
||||||
cache_->ApplyToAllEntries(callback, /*opts*/ {});
|
cache_->ApplyToAllEntries(callback, /*opts*/ {});
|
||||||
|
|
||||||
@ -848,13 +838,11 @@ TEST_P(CacheTest, GetChargeAndDeleter) {
|
|||||||
std::shared_ptr<Cache> (*new_clock_cache_func)(
|
std::shared_ptr<Cache> (*new_clock_cache_func)(
|
||||||
size_t, int, bool, CacheMetadataChargePolicy) = NewClockCache;
|
size_t, int, bool, CacheMetadataChargePolicy) = NewClockCache;
|
||||||
INSTANTIATE_TEST_CASE_P(CacheTestInstance, CacheTest,
|
INSTANTIATE_TEST_CASE_P(CacheTestInstance, CacheTest,
|
||||||
testing::Values(kLRU, kClock, kFast));
|
testing::Values(kLRU, kClock));
|
||||||
#else
|
#else
|
||||||
INSTANTIATE_TEST_CASE_P(CacheTestInstance, CacheTest,
|
INSTANTIATE_TEST_CASE_P(CacheTestInstance, CacheTest, testing::Values(kLRU));
|
||||||
testing::Values(kLRU, kFast));
|
|
||||||
#endif // SUPPORT_CLOCK_CACHE
|
#endif // SUPPORT_CLOCK_CACHE
|
||||||
INSTANTIATE_TEST_CASE_P(CacheTestInstance, LRUCacheTest,
|
INSTANTIATE_TEST_CASE_P(CacheTestInstance, LRUCacheTest, testing::Values(kLRU));
|
||||||
testing::Values(kLRU, kFast));
|
|
||||||
|
|
||||||
} // namespace ROCKSDB_NAMESPACE
|
} // namespace ROCKSDB_NAMESPACE
|
||||||
|
|
||||||
|
511
cache/fast_lru_cache.cc
vendored
511
cache/fast_lru_cache.cc
vendored
@ -1,511 +0,0 @@
|
|||||||
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
|
||||||
// This source code is licensed under both the GPLv2 (found in the
|
|
||||||
// COPYING file in the root directory) and Apache 2.0 License
|
|
||||||
// (found in the LICENSE.Apache file in the root directory).
|
|
||||||
//
|
|
||||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
|
||||||
// Use of this source code is governed by a BSD-style license that can be
|
|
||||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
|
||||||
|
|
||||||
#include "cache/fast_lru_cache.h"
|
|
||||||
|
|
||||||
#include <cassert>
|
|
||||||
#include <cstdint>
|
|
||||||
#include <cstdio>
|
|
||||||
|
|
||||||
#include "monitoring/perf_context_imp.h"
|
|
||||||
#include "monitoring/statistics.h"
|
|
||||||
#include "port/lang.h"
|
|
||||||
#include "util/mutexlock.h"
|
|
||||||
|
|
||||||
namespace ROCKSDB_NAMESPACE {
|
|
||||||
|
|
||||||
namespace fast_lru_cache {
|
|
||||||
|
|
||||||
LRUHandleTable::LRUHandleTable(int max_upper_hash_bits)
|
|
||||||
: length_bits_(/* historical starting size*/ 4),
|
|
||||||
list_(new LRUHandle* [size_t{1} << length_bits_] {}),
|
|
||||||
elems_(0),
|
|
||||||
max_length_bits_(max_upper_hash_bits) {}
|
|
||||||
|
|
||||||
LRUHandleTable::~LRUHandleTable() {
|
|
||||||
ApplyToEntriesRange(
|
|
||||||
[](LRUHandle* h) {
|
|
||||||
if (!h->HasRefs()) {
|
|
||||||
h->Free();
|
|
||||||
}
|
|
||||||
},
|
|
||||||
0, uint32_t{1} << length_bits_);
|
|
||||||
}
|
|
||||||
|
|
||||||
LRUHandle* LRUHandleTable::Lookup(const Slice& key, uint32_t hash) {
|
|
||||||
return *FindPointer(key, hash);
|
|
||||||
}
|
|
||||||
|
|
||||||
LRUHandle* LRUHandleTable::Insert(LRUHandle* h) {
|
|
||||||
LRUHandle** ptr = FindPointer(h->key(), h->hash);
|
|
||||||
LRUHandle* old = *ptr;
|
|
||||||
h->next_hash = (old == nullptr ? nullptr : old->next_hash);
|
|
||||||
*ptr = h;
|
|
||||||
if (old == nullptr) {
|
|
||||||
++elems_;
|
|
||||||
if ((elems_ >> length_bits_) > 0) { // elems_ >= length
|
|
||||||
// Since each cache entry is fairly large, we aim for a small
|
|
||||||
// average linked list length (<= 1).
|
|
||||||
Resize();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return old;
|
|
||||||
}
|
|
||||||
|
|
||||||
LRUHandle* LRUHandleTable::Remove(const Slice& key, uint32_t hash) {
|
|
||||||
LRUHandle** ptr = FindPointer(key, hash);
|
|
||||||
LRUHandle* result = *ptr;
|
|
||||||
if (result != nullptr) {
|
|
||||||
*ptr = result->next_hash;
|
|
||||||
--elems_;
|
|
||||||
}
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
LRUHandle** LRUHandleTable::FindPointer(const Slice& key, uint32_t hash) {
|
|
||||||
LRUHandle** ptr = &list_[hash >> (32 - length_bits_)];
|
|
||||||
while (*ptr != nullptr && ((*ptr)->hash != hash || key != (*ptr)->key())) {
|
|
||||||
ptr = &(*ptr)->next_hash;
|
|
||||||
}
|
|
||||||
return ptr;
|
|
||||||
}
|
|
||||||
|
|
||||||
void LRUHandleTable::Resize() {
|
|
||||||
if (length_bits_ >= max_length_bits_) {
|
|
||||||
// Due to reaching limit of hash information, if we made the table bigger,
|
|
||||||
// we would allocate more addresses but only the same number would be used.
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if (length_bits_ >= 31) {
|
|
||||||
// Avoid undefined behavior shifting uint32_t by 32.
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
uint32_t old_length = uint32_t{1} << length_bits_;
|
|
||||||
int new_length_bits = length_bits_ + 1;
|
|
||||||
std::unique_ptr<LRUHandle* []> new_list {
|
|
||||||
new LRUHandle* [size_t{1} << new_length_bits] {}
|
|
||||||
};
|
|
||||||
uint32_t count = 0;
|
|
||||||
for (uint32_t i = 0; i < old_length; i++) {
|
|
||||||
LRUHandle* h = list_[i];
|
|
||||||
while (h != nullptr) {
|
|
||||||
LRUHandle* next = h->next_hash;
|
|
||||||
uint32_t hash = h->hash;
|
|
||||||
LRUHandle** ptr = &new_list[hash >> (32 - new_length_bits)];
|
|
||||||
h->next_hash = *ptr;
|
|
||||||
*ptr = h;
|
|
||||||
h = next;
|
|
||||||
count++;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
assert(elems_ == count);
|
|
||||||
list_ = std::move(new_list);
|
|
||||||
length_bits_ = new_length_bits;
|
|
||||||
}
|
|
||||||
|
|
||||||
LRUCacheShard::LRUCacheShard(size_t capacity, bool strict_capacity_limit,
|
|
||||||
CacheMetadataChargePolicy metadata_charge_policy,
|
|
||||||
int max_upper_hash_bits)
|
|
||||||
: capacity_(0),
|
|
||||||
strict_capacity_limit_(strict_capacity_limit),
|
|
||||||
table_(max_upper_hash_bits),
|
|
||||||
usage_(0),
|
|
||||||
lru_usage_(0) {
|
|
||||||
set_metadata_charge_policy(metadata_charge_policy);
|
|
||||||
// Make empty circular linked list.
|
|
||||||
lru_.next = &lru_;
|
|
||||||
lru_.prev = &lru_;
|
|
||||||
lru_low_pri_ = &lru_;
|
|
||||||
SetCapacity(capacity);
|
|
||||||
}
|
|
||||||
|
|
||||||
void LRUCacheShard::EraseUnRefEntries() {
|
|
||||||
autovector<LRUHandle*> last_reference_list;
|
|
||||||
{
|
|
||||||
MutexLock l(&mutex_);
|
|
||||||
while (lru_.next != &lru_) {
|
|
||||||
LRUHandle* old = lru_.next;
|
|
||||||
// LRU list contains only elements which can be evicted.
|
|
||||||
assert(old->InCache() && !old->HasRefs());
|
|
||||||
LRU_Remove(old);
|
|
||||||
table_.Remove(old->key(), old->hash);
|
|
||||||
old->SetInCache(false);
|
|
||||||
size_t total_charge = old->CalcTotalCharge(metadata_charge_policy_);
|
|
||||||
assert(usage_ >= total_charge);
|
|
||||||
usage_ -= total_charge;
|
|
||||||
last_reference_list.push_back(old);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Free the entries here outside of mutex for performance reasons.
|
|
||||||
for (auto entry : last_reference_list) {
|
|
||||||
entry->Free();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
void LRUCacheShard::ApplyToSomeEntries(
|
|
||||||
const std::function<void(const Slice& key, void* value, size_t charge,
|
|
||||||
DeleterFn deleter)>& callback,
|
|
||||||
uint32_t average_entries_per_lock, uint32_t* state) {
|
|
||||||
// The state is essentially going to be the starting hash, which works
|
|
||||||
// nicely even if we resize between calls because we use upper-most
|
|
||||||
// hash bits for table indexes.
|
|
||||||
MutexLock l(&mutex_);
|
|
||||||
uint32_t length_bits = table_.GetLengthBits();
|
|
||||||
uint32_t length = uint32_t{1} << length_bits;
|
|
||||||
|
|
||||||
assert(average_entries_per_lock > 0);
|
|
||||||
// Assuming we are called with same average_entries_per_lock repeatedly,
|
|
||||||
// this simplifies some logic (index_end will not overflow).
|
|
||||||
assert(average_entries_per_lock < length || *state == 0);
|
|
||||||
|
|
||||||
uint32_t index_begin = *state >> (32 - length_bits);
|
|
||||||
uint32_t index_end = index_begin + average_entries_per_lock;
|
|
||||||
if (index_end >= length) {
|
|
||||||
// Going to end
|
|
||||||
index_end = length;
|
|
||||||
*state = UINT32_MAX;
|
|
||||||
} else {
|
|
||||||
*state = index_end << (32 - length_bits);
|
|
||||||
}
|
|
||||||
|
|
||||||
table_.ApplyToEntriesRange(
|
|
||||||
[callback](LRUHandle* h) {
|
|
||||||
callback(h->key(), h->value, h->charge, h->deleter);
|
|
||||||
},
|
|
||||||
index_begin, index_end);
|
|
||||||
}
|
|
||||||
|
|
||||||
void LRUCacheShard::LRU_Remove(LRUHandle* e) {
|
|
||||||
assert(e->next != nullptr);
|
|
||||||
assert(e->prev != nullptr);
|
|
||||||
e->next->prev = e->prev;
|
|
||||||
e->prev->next = e->next;
|
|
||||||
e->prev = e->next = nullptr;
|
|
||||||
size_t total_charge = e->CalcTotalCharge(metadata_charge_policy_);
|
|
||||||
assert(lru_usage_ >= total_charge);
|
|
||||||
lru_usage_ -= total_charge;
|
|
||||||
}
|
|
||||||
|
|
||||||
void LRUCacheShard::LRU_Insert(LRUHandle* e) {
|
|
||||||
assert(e->next == nullptr);
|
|
||||||
assert(e->prev == nullptr);
|
|
||||||
size_t total_charge = e->CalcTotalCharge(metadata_charge_policy_);
|
|
||||||
// Inset "e" to head of LRU list.
|
|
||||||
e->next = &lru_;
|
|
||||||
e->prev = lru_.prev;
|
|
||||||
e->prev->next = e;
|
|
||||||
e->next->prev = e;
|
|
||||||
lru_usage_ += total_charge;
|
|
||||||
}
|
|
||||||
|
|
||||||
void LRUCacheShard::EvictFromLRU(size_t charge,
|
|
||||||
autovector<LRUHandle*>* deleted) {
|
|
||||||
while ((usage_ + charge) > capacity_ && lru_.next != &lru_) {
|
|
||||||
LRUHandle* old = lru_.next;
|
|
||||||
// LRU list contains only elements which can be evicted.
|
|
||||||
assert(old->InCache() && !old->HasRefs());
|
|
||||||
LRU_Remove(old);
|
|
||||||
table_.Remove(old->key(), old->hash);
|
|
||||||
old->SetInCache(false);
|
|
||||||
size_t old_total_charge = old->CalcTotalCharge(metadata_charge_policy_);
|
|
||||||
assert(usage_ >= old_total_charge);
|
|
||||||
usage_ -= old_total_charge;
|
|
||||||
deleted->push_back(old);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
void LRUCacheShard::SetCapacity(size_t capacity) {
|
|
||||||
autovector<LRUHandle*> last_reference_list;
|
|
||||||
{
|
|
||||||
MutexLock l(&mutex_);
|
|
||||||
capacity_ = capacity;
|
|
||||||
EvictFromLRU(0, &last_reference_list);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Free the entries here outside of mutex for performance reasons.
|
|
||||||
for (auto entry : last_reference_list) {
|
|
||||||
entry->Free();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
void LRUCacheShard::SetStrictCapacityLimit(bool strict_capacity_limit) {
|
|
||||||
MutexLock l(&mutex_);
|
|
||||||
strict_capacity_limit_ = strict_capacity_limit;
|
|
||||||
}
|
|
||||||
|
|
||||||
Status LRUCacheShard::InsertItem(LRUHandle* e, Cache::Handle** handle,
|
|
||||||
bool free_handle_on_fail) {
|
|
||||||
Status s = Status::OK();
|
|
||||||
autovector<LRUHandle*> last_reference_list;
|
|
||||||
size_t total_charge = e->CalcTotalCharge(metadata_charge_policy_);
|
|
||||||
|
|
||||||
{
|
|
||||||
MutexLock l(&mutex_);
|
|
||||||
|
|
||||||
// Free the space following strict LRU policy until enough space
|
|
||||||
// is freed or the lru list is empty.
|
|
||||||
EvictFromLRU(total_charge, &last_reference_list);
|
|
||||||
|
|
||||||
if ((usage_ + total_charge) > capacity_ &&
|
|
||||||
(strict_capacity_limit_ || handle == nullptr)) {
|
|
||||||
e->SetInCache(false);
|
|
||||||
if (handle == nullptr) {
|
|
||||||
// Don't insert the entry but still return ok, as if the entry inserted
|
|
||||||
// into cache and get evicted immediately.
|
|
||||||
last_reference_list.push_back(e);
|
|
||||||
} else {
|
|
||||||
if (free_handle_on_fail) {
|
|
||||||
delete[] reinterpret_cast<char*>(e);
|
|
||||||
*handle = nullptr;
|
|
||||||
}
|
|
||||||
s = Status::Incomplete("Insert failed due to LRU cache being full.");
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
// Insert into the cache. Note that the cache might get larger than its
|
|
||||||
// capacity if not enough space was freed up.
|
|
||||||
LRUHandle* old = table_.Insert(e);
|
|
||||||
usage_ += total_charge;
|
|
||||||
if (old != nullptr) {
|
|
||||||
s = Status::OkOverwritten();
|
|
||||||
assert(old->InCache());
|
|
||||||
old->SetInCache(false);
|
|
||||||
if (!old->HasRefs()) {
|
|
||||||
// old is on LRU because it's in cache and its reference count is 0.
|
|
||||||
LRU_Remove(old);
|
|
||||||
size_t old_total_charge =
|
|
||||||
old->CalcTotalCharge(metadata_charge_policy_);
|
|
||||||
assert(usage_ >= old_total_charge);
|
|
||||||
usage_ -= old_total_charge;
|
|
||||||
last_reference_list.push_back(old);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (handle == nullptr) {
|
|
||||||
LRU_Insert(e);
|
|
||||||
} else {
|
|
||||||
// If caller already holds a ref, no need to take one here.
|
|
||||||
if (!e->HasRefs()) {
|
|
||||||
e->Ref();
|
|
||||||
}
|
|
||||||
*handle = reinterpret_cast<Cache::Handle*>(e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Free the entries here outside of mutex for performance reasons.
|
|
||||||
for (auto entry : last_reference_list) {
|
|
||||||
entry->Free();
|
|
||||||
}
|
|
||||||
|
|
||||||
return s;
|
|
||||||
}
|
|
||||||
|
|
||||||
Cache::Handle* LRUCacheShard::Lookup(const Slice& key, uint32_t hash) {
|
|
||||||
LRUHandle* e = nullptr;
|
|
||||||
{
|
|
||||||
MutexLock l(&mutex_);
|
|
||||||
e = table_.Lookup(key, hash);
|
|
||||||
if (e != nullptr) {
|
|
||||||
assert(e->InCache());
|
|
||||||
if (!e->HasRefs()) {
|
|
||||||
// The entry is in LRU since it's in hash and has no external references
|
|
||||||
LRU_Remove(e);
|
|
||||||
}
|
|
||||||
e->Ref();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return reinterpret_cast<Cache::Handle*>(e);
|
|
||||||
}
|
|
||||||
|
|
||||||
bool LRUCacheShard::Ref(Cache::Handle* h) {
|
|
||||||
LRUHandle* e = reinterpret_cast<LRUHandle*>(h);
|
|
||||||
MutexLock l(&mutex_);
|
|
||||||
// To create another reference - entry must be already externally referenced.
|
|
||||||
assert(e->HasRefs());
|
|
||||||
e->Ref();
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
bool LRUCacheShard::Release(Cache::Handle* handle, bool erase_if_last_ref) {
|
|
||||||
if (handle == nullptr) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
LRUHandle* e = reinterpret_cast<LRUHandle*>(handle);
|
|
||||||
bool last_reference = false;
|
|
||||||
{
|
|
||||||
MutexLock l(&mutex_);
|
|
||||||
last_reference = e->Unref();
|
|
||||||
if (last_reference && e->InCache()) {
|
|
||||||
// The item is still in cache, and nobody else holds a reference to it.
|
|
||||||
if (usage_ > capacity_ || erase_if_last_ref) {
|
|
||||||
// The LRU list must be empty since the cache is full.
|
|
||||||
assert(lru_.next == &lru_ || erase_if_last_ref);
|
|
||||||
// Take this opportunity and remove the item.
|
|
||||||
table_.Remove(e->key(), e->hash);
|
|
||||||
e->SetInCache(false);
|
|
||||||
} else {
|
|
||||||
// Put the item back on the LRU list, and don't free it.
|
|
||||||
LRU_Insert(e);
|
|
||||||
last_reference = false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// If it was the last reference, then decrement the cache usage.
|
|
||||||
if (last_reference) {
|
|
||||||
size_t total_charge = e->CalcTotalCharge(metadata_charge_policy_);
|
|
||||||
assert(usage_ >= total_charge);
|
|
||||||
usage_ -= total_charge;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Free the entry here outside of mutex for performance reasons.
|
|
||||||
if (last_reference) {
|
|
||||||
e->Free();
|
|
||||||
}
|
|
||||||
return last_reference;
|
|
||||||
}
|
|
||||||
|
|
||||||
Status LRUCacheShard::Insert(const Slice& key, uint32_t hash, void* value,
|
|
||||||
size_t charge, Cache::DeleterFn deleter,
|
|
||||||
Cache::Handle** handle,
|
|
||||||
Cache::Priority /*priority*/) {
|
|
||||||
// Allocate the memory here outside of the mutex.
|
|
||||||
// If the cache is full, we'll have to release it.
|
|
||||||
// It shouldn't happen very often though.
|
|
||||||
LRUHandle* e = reinterpret_cast<LRUHandle*>(
|
|
||||||
new char[sizeof(LRUHandle) - 1 + key.size()]);
|
|
||||||
|
|
||||||
e->value = value;
|
|
||||||
e->flags = 0;
|
|
||||||
e->deleter = deleter;
|
|
||||||
e->charge = charge;
|
|
||||||
e->key_length = key.size();
|
|
||||||
e->hash = hash;
|
|
||||||
e->refs = 0;
|
|
||||||
e->next = e->prev = nullptr;
|
|
||||||
e->SetInCache(true);
|
|
||||||
memcpy(e->key_data, key.data(), key.size());
|
|
||||||
|
|
||||||
return InsertItem(e, handle, /* free_handle_on_fail */ true);
|
|
||||||
}
|
|
||||||
|
|
||||||
void LRUCacheShard::Erase(const Slice& key, uint32_t hash) {
|
|
||||||
LRUHandle* e;
|
|
||||||
bool last_reference = false;
|
|
||||||
{
|
|
||||||
MutexLock l(&mutex_);
|
|
||||||
e = table_.Remove(key, hash);
|
|
||||||
if (e != nullptr) {
|
|
||||||
assert(e->InCache());
|
|
||||||
e->SetInCache(false);
|
|
||||||
if (!e->HasRefs()) {
|
|
||||||
// The entry is in LRU since it's in hash and has no external references
|
|
||||||
LRU_Remove(e);
|
|
||||||
size_t total_charge = e->CalcTotalCharge(metadata_charge_policy_);
|
|
||||||
assert(usage_ >= total_charge);
|
|
||||||
usage_ -= total_charge;
|
|
||||||
last_reference = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Free the entry here outside of mutex for performance reasons.
|
|
||||||
// last_reference will only be true if e != nullptr.
|
|
||||||
if (last_reference) {
|
|
||||||
e->Free();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
size_t LRUCacheShard::GetUsage() const {
|
|
||||||
MutexLock l(&mutex_);
|
|
||||||
return usage_;
|
|
||||||
}
|
|
||||||
|
|
||||||
size_t LRUCacheShard::GetPinnedUsage() const {
|
|
||||||
MutexLock l(&mutex_);
|
|
||||||
assert(usage_ >= lru_usage_);
|
|
||||||
return usage_ - lru_usage_;
|
|
||||||
}
|
|
||||||
|
|
||||||
std::string LRUCacheShard::GetPrintableOptions() const { return std::string{}; }
|
|
||||||
|
|
||||||
LRUCache::LRUCache(size_t capacity, int num_shard_bits,
|
|
||||||
bool strict_capacity_limit,
|
|
||||||
CacheMetadataChargePolicy metadata_charge_policy)
|
|
||||||
: ShardedCache(capacity, num_shard_bits, strict_capacity_limit) {
|
|
||||||
num_shards_ = 1 << num_shard_bits;
|
|
||||||
shards_ = reinterpret_cast<LRUCacheShard*>(
|
|
||||||
port::cacheline_aligned_alloc(sizeof(LRUCacheShard) * num_shards_));
|
|
||||||
size_t per_shard = (capacity + (num_shards_ - 1)) / num_shards_;
|
|
||||||
for (int i = 0; i < num_shards_; i++) {
|
|
||||||
new (&shards_[i])
|
|
||||||
LRUCacheShard(per_shard, strict_capacity_limit, metadata_charge_policy,
|
|
||||||
/* max_upper_hash_bits */ 32 - num_shard_bits);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
LRUCache::~LRUCache() {
|
|
||||||
if (shards_ != nullptr) {
|
|
||||||
assert(num_shards_ > 0);
|
|
||||||
for (int i = 0; i < num_shards_; i++) {
|
|
||||||
shards_[i].~LRUCacheShard();
|
|
||||||
}
|
|
||||||
port::cacheline_aligned_free(shards_);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
CacheShard* LRUCache::GetShard(uint32_t shard) {
|
|
||||||
return reinterpret_cast<CacheShard*>(&shards_[shard]);
|
|
||||||
}
|
|
||||||
|
|
||||||
const CacheShard* LRUCache::GetShard(uint32_t shard) const {
|
|
||||||
return reinterpret_cast<CacheShard*>(&shards_[shard]);
|
|
||||||
}
|
|
||||||
|
|
||||||
void* LRUCache::Value(Handle* handle) {
|
|
||||||
return reinterpret_cast<const LRUHandle*>(handle)->value;
|
|
||||||
}
|
|
||||||
|
|
||||||
size_t LRUCache::GetCharge(Handle* handle) const {
|
|
||||||
return reinterpret_cast<const LRUHandle*>(handle)->charge;
|
|
||||||
}
|
|
||||||
|
|
||||||
Cache::DeleterFn LRUCache::GetDeleter(Handle* handle) const {
|
|
||||||
auto h = reinterpret_cast<const LRUHandle*>(handle);
|
|
||||||
return h->deleter;
|
|
||||||
}
|
|
||||||
|
|
||||||
uint32_t LRUCache::GetHash(Handle* handle) const {
|
|
||||||
return reinterpret_cast<const LRUHandle*>(handle)->hash;
|
|
||||||
}
|
|
||||||
|
|
||||||
void LRUCache::DisownData() {
|
|
||||||
// Leak data only if that won't generate an ASAN/valgrind warning.
|
|
||||||
if (!kMustFreeHeapAllocations) {
|
|
||||||
shards_ = nullptr;
|
|
||||||
num_shards_ = 0;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
} // namespace fast_lru_cache
|
|
||||||
|
|
||||||
std::shared_ptr<Cache> NewFastLRUCache(
|
|
||||||
size_t capacity, int num_shard_bits, bool strict_capacity_limit,
|
|
||||||
CacheMetadataChargePolicy metadata_charge_policy) {
|
|
||||||
if (num_shard_bits >= 20) {
|
|
||||||
return nullptr; // The cache cannot be sharded into too many fine pieces.
|
|
||||||
}
|
|
||||||
if (num_shard_bits < 0) {
|
|
||||||
num_shard_bits = GetDefaultCacheShardBits(capacity);
|
|
||||||
}
|
|
||||||
return std::make_shared<fast_lru_cache::LRUCache>(
|
|
||||||
capacity, num_shard_bits, strict_capacity_limit, metadata_charge_policy);
|
|
||||||
}
|
|
||||||
|
|
||||||
} // namespace ROCKSDB_NAMESPACE
|
|
299
cache/fast_lru_cache.h
vendored
299
cache/fast_lru_cache.h
vendored
@ -1,299 +0,0 @@
|
|||||||
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved
|
|
||||||
// This source code is licensed under both the GPLv2 (found in the
|
|
||||||
// COPYING file in the root directory) and Apache 2.0 License
|
|
||||||
// (found in the LICENSE.Apache file in the root directory).
|
|
||||||
//
|
|
||||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
|
||||||
// Use of this source code is governed by a BSD-style license that can be
|
|
||||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
|
||||||
#pragma once
|
|
||||||
|
|
||||||
#include <memory>
|
|
||||||
#include <string>
|
|
||||||
|
|
||||||
#include "cache/sharded_cache.h"
|
|
||||||
#include "port/lang.h"
|
|
||||||
#include "port/malloc.h"
|
|
||||||
#include "port/port.h"
|
|
||||||
#include "rocksdb/secondary_cache.h"
|
|
||||||
#include "util/autovector.h"
|
|
||||||
|
|
||||||
namespace ROCKSDB_NAMESPACE {
|
|
||||||
namespace fast_lru_cache {
|
|
||||||
|
|
||||||
// An experimental (under development!) alternative to LRUCache
|
|
||||||
|
|
||||||
struct LRUHandle {
|
|
||||||
void* value;
|
|
||||||
Cache::DeleterFn deleter;
|
|
||||||
LRUHandle* next_hash;
|
|
||||||
LRUHandle* next;
|
|
||||||
LRUHandle* prev;
|
|
||||||
size_t charge; // TODO(opt): Only allow uint32_t?
|
|
||||||
size_t key_length;
|
|
||||||
// The hash of key(). Used for fast sharding and comparisons.
|
|
||||||
uint32_t hash;
|
|
||||||
// The number of external refs to this entry. The cache itself is not counted.
|
|
||||||
uint32_t refs;
|
|
||||||
|
|
||||||
enum Flags : uint8_t {
|
|
||||||
// Whether this entry is referenced by the hash table.
|
|
||||||
IN_CACHE = (1 << 0),
|
|
||||||
};
|
|
||||||
uint8_t flags;
|
|
||||||
|
|
||||||
// Beginning of the key (MUST BE THE LAST FIELD IN THIS STRUCT!)
|
|
||||||
char key_data[1];
|
|
||||||
|
|
||||||
Slice key() const { return Slice(key_data, key_length); }
|
|
||||||
|
|
||||||
// Increase the reference count by 1.
|
|
||||||
void Ref() { refs++; }
|
|
||||||
|
|
||||||
// Just reduce the reference count by 1. Return true if it was last reference.
|
|
||||||
bool Unref() {
|
|
||||||
assert(refs > 0);
|
|
||||||
refs--;
|
|
||||||
return refs == 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Return true if there are external refs, false otherwise.
|
|
||||||
bool HasRefs() const { return refs > 0; }
|
|
||||||
|
|
||||||
bool InCache() const { return flags & IN_CACHE; }
|
|
||||||
|
|
||||||
void SetInCache(bool in_cache) {
|
|
||||||
if (in_cache) {
|
|
||||||
flags |= IN_CACHE;
|
|
||||||
} else {
|
|
||||||
flags &= ~IN_CACHE;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
void Free() {
|
|
||||||
assert(refs == 0);
|
|
||||||
if (deleter) {
|
|
||||||
(*deleter)(key(), value);
|
|
||||||
}
|
|
||||||
delete[] reinterpret_cast<char*>(this);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Calculate the memory usage by metadata.
|
|
||||||
inline size_t CalcTotalCharge(
|
|
||||||
CacheMetadataChargePolicy metadata_charge_policy) {
|
|
||||||
size_t meta_charge = 0;
|
|
||||||
if (metadata_charge_policy == kFullChargeCacheMetadata) {
|
|
||||||
#ifdef ROCKSDB_MALLOC_USABLE_SIZE
|
|
||||||
meta_charge += malloc_usable_size(static_cast<void*>(this));
|
|
||||||
#else
|
|
||||||
// This is the size that is used when a new handle is created.
|
|
||||||
meta_charge += sizeof(LRUHandle) - 1 + key_length;
|
|
||||||
#endif
|
|
||||||
}
|
|
||||||
return charge + meta_charge;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
// We provide our own simple hash table since it removes a whole bunch
|
|
||||||
// of porting hacks and is also faster than some of the built-in hash
|
|
||||||
// table implementations in some of the compiler/runtime combinations
|
|
||||||
// we have tested. E.g., readrandom speeds up by ~5% over the g++
|
|
||||||
// 4.4.3's builtin hashtable.
|
|
||||||
class LRUHandleTable {
|
|
||||||
public:
|
|
||||||
// If the table uses more hash bits than `max_upper_hash_bits`,
|
|
||||||
// it will eat into the bits used for sharding, which are constant
|
|
||||||
// for a given LRUHandleTable.
|
|
||||||
explicit LRUHandleTable(int max_upper_hash_bits);
|
|
||||||
~LRUHandleTable();
|
|
||||||
|
|
||||||
LRUHandle* Lookup(const Slice& key, uint32_t hash);
|
|
||||||
LRUHandle* Insert(LRUHandle* h);
|
|
||||||
LRUHandle* Remove(const Slice& key, uint32_t hash);
|
|
||||||
|
|
||||||
template <typename T>
|
|
||||||
void ApplyToEntriesRange(T func, uint32_t index_begin, uint32_t index_end) {
|
|
||||||
for (uint32_t i = index_begin; i < index_end; i++) {
|
|
||||||
LRUHandle* h = list_[i];
|
|
||||||
while (h != nullptr) {
|
|
||||||
auto n = h->next_hash;
|
|
||||||
assert(h->InCache());
|
|
||||||
func(h);
|
|
||||||
h = n;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
int GetLengthBits() const { return length_bits_; }
|
|
||||||
|
|
||||||
private:
|
|
||||||
// Return a pointer to slot that points to a cache entry that
|
|
||||||
// matches key/hash. If there is no such cache entry, return a
|
|
||||||
// pointer to the trailing slot in the corresponding linked list.
|
|
||||||
LRUHandle** FindPointer(const Slice& key, uint32_t hash);
|
|
||||||
|
|
||||||
void Resize();
|
|
||||||
|
|
||||||
// Number of hash bits (upper because lower bits used for sharding)
|
|
||||||
// used for table index. Length == 1 << length_bits_
|
|
||||||
int length_bits_;
|
|
||||||
|
|
||||||
// The table consists of an array of buckets where each bucket is
|
|
||||||
// a linked list of cache entries that hash into the bucket.
|
|
||||||
std::unique_ptr<LRUHandle*[]> list_;
|
|
||||||
|
|
||||||
// Number of elements currently in the table.
|
|
||||||
uint32_t elems_;
|
|
||||||
|
|
||||||
// Set from max_upper_hash_bits (see constructor).
|
|
||||||
const int max_length_bits_;
|
|
||||||
};
|
|
||||||
|
|
||||||
// A single shard of sharded cache.
|
|
||||||
class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShard {
|
|
||||||
public:
|
|
||||||
LRUCacheShard(size_t capacity, bool strict_capacity_limit,
|
|
||||||
CacheMetadataChargePolicy metadata_charge_policy,
|
|
||||||
int max_upper_hash_bits);
|
|
||||||
~LRUCacheShard() override = default;
|
|
||||||
|
|
||||||
// Separate from constructor so caller can easily make an array of LRUCache
|
|
||||||
// if current usage is more than new capacity, the function will attempt to
|
|
||||||
// free the needed space.
|
|
||||||
void SetCapacity(size_t capacity) override;
|
|
||||||
|
|
||||||
// Set the flag to reject insertion if cache if full.
|
|
||||||
void SetStrictCapacityLimit(bool strict_capacity_limit) override;
|
|
||||||
|
|
||||||
// Like Cache methods, but with an extra "hash" parameter.
|
|
||||||
Status Insert(const Slice& key, uint32_t hash, void* value, size_t charge,
|
|
||||||
Cache::DeleterFn deleter, Cache::Handle** handle,
|
|
||||||
Cache::Priority priority) override;
|
|
||||||
|
|
||||||
Status Insert(const Slice& key, uint32_t hash, void* value,
|
|
||||||
const Cache::CacheItemHelper* helper, size_t charge,
|
|
||||||
Cache::Handle** handle, Cache::Priority priority) override {
|
|
||||||
return Insert(key, hash, value, charge, helper->del_cb, handle, priority);
|
|
||||||
}
|
|
||||||
|
|
||||||
Cache::Handle* Lookup(const Slice& key, uint32_t hash,
|
|
||||||
const Cache::CacheItemHelper* /*helper*/,
|
|
||||||
const Cache::CreateCallback& /*create_cb*/,
|
|
||||||
Cache::Priority /*priority*/, bool /*wait*/,
|
|
||||||
Statistics* /*stats*/) override {
|
|
||||||
return Lookup(key, hash);
|
|
||||||
}
|
|
||||||
Cache::Handle* Lookup(const Slice& key, uint32_t hash) override;
|
|
||||||
|
|
||||||
bool Release(Cache::Handle* handle, bool /*useful*/,
|
|
||||||
bool erase_if_last_ref) override {
|
|
||||||
return Release(handle, erase_if_last_ref);
|
|
||||||
}
|
|
||||||
bool IsReady(Cache::Handle* /*handle*/) override { return true; }
|
|
||||||
void Wait(Cache::Handle* /*handle*/) override {}
|
|
||||||
|
|
||||||
bool Ref(Cache::Handle* handle) override;
|
|
||||||
bool Release(Cache::Handle* handle, bool erase_if_last_ref = false) override;
|
|
||||||
void Erase(const Slice& key, uint32_t hash) override;
|
|
||||||
|
|
||||||
size_t GetUsage() const override;
|
|
||||||
size_t GetPinnedUsage() const override;
|
|
||||||
|
|
||||||
void ApplyToSomeEntries(
|
|
||||||
const std::function<void(const Slice& key, void* value, size_t charge,
|
|
||||||
DeleterFn deleter)>& callback,
|
|
||||||
uint32_t average_entries_per_lock, uint32_t* state) override;
|
|
||||||
|
|
||||||
void EraseUnRefEntries() override;
|
|
||||||
|
|
||||||
std::string GetPrintableOptions() const override;
|
|
||||||
|
|
||||||
private:
|
|
||||||
friend class LRUCache;
|
|
||||||
// Insert an item into the hash table and, if handle is null, insert into
|
|
||||||
// the LRU list. Older items are evicted as necessary. If the cache is full
|
|
||||||
// and free_handle_on_fail is true, the item is deleted and handle is set to
|
|
||||||
// nullptr.
|
|
||||||
Status InsertItem(LRUHandle* item, Cache::Handle** handle,
|
|
||||||
bool free_handle_on_fail);
|
|
||||||
|
|
||||||
void LRU_Remove(LRUHandle* e);
|
|
||||||
void LRU_Insert(LRUHandle* e);
|
|
||||||
|
|
||||||
// Free some space following strict LRU policy until enough space
|
|
||||||
// to hold (usage_ + charge) is freed or the lru list is empty
|
|
||||||
// This function is not thread safe - it needs to be executed while
|
|
||||||
// holding the mutex_.
|
|
||||||
void EvictFromLRU(size_t charge, autovector<LRUHandle*>* deleted);
|
|
||||||
|
|
||||||
// Initialized before use.
|
|
||||||
size_t capacity_;
|
|
||||||
|
|
||||||
// Whether to reject insertion if cache reaches its full capacity.
|
|
||||||
bool strict_capacity_limit_;
|
|
||||||
|
|
||||||
// Dummy head of LRU list.
|
|
||||||
// lru.prev is newest entry, lru.next is oldest entry.
|
|
||||||
// LRU contains items which can be evicted, ie reference only by cache
|
|
||||||
LRUHandle lru_;
|
|
||||||
|
|
||||||
// Pointer to head of low-pri pool in LRU list.
|
|
||||||
LRUHandle* lru_low_pri_;
|
|
||||||
|
|
||||||
// ------------^^^^^^^^^^^^^-----------
|
|
||||||
// Not frequently modified data members
|
|
||||||
// ------------------------------------
|
|
||||||
//
|
|
||||||
// We separate data members that are updated frequently from the ones that
|
|
||||||
// are not frequently updated so that they don't share the same cache line
|
|
||||||
// which will lead into false cache sharing
|
|
||||||
//
|
|
||||||
// ------------------------------------
|
|
||||||
// Frequently modified data members
|
|
||||||
// ------------vvvvvvvvvvvvv-----------
|
|
||||||
LRUHandleTable table_;
|
|
||||||
|
|
||||||
// Memory size for entries residing in the cache.
|
|
||||||
size_t usage_;
|
|
||||||
|
|
||||||
// Memory size for entries residing only in the LRU list.
|
|
||||||
size_t lru_usage_;
|
|
||||||
|
|
||||||
// mutex_ protects the following state.
|
|
||||||
// We don't count mutex_ as the cache's internal state so semantically we
|
|
||||||
// don't mind mutex_ invoking the non-const actions.
|
|
||||||
mutable port::Mutex mutex_;
|
|
||||||
};
|
|
||||||
|
|
||||||
class LRUCache
|
|
||||||
#ifdef NDEBUG
|
|
||||||
final
|
|
||||||
#endif
|
|
||||||
: public ShardedCache {
|
|
||||||
public:
|
|
||||||
LRUCache(size_t capacity, int num_shard_bits, bool strict_capacity_limit,
|
|
||||||
CacheMetadataChargePolicy metadata_charge_policy =
|
|
||||||
kDontChargeCacheMetadata);
|
|
||||||
~LRUCache() override;
|
|
||||||
const char* Name() const override { return "LRUCache"; }
|
|
||||||
CacheShard* GetShard(uint32_t shard) override;
|
|
||||||
const CacheShard* GetShard(uint32_t shard) const override;
|
|
||||||
void* Value(Handle* handle) override;
|
|
||||||
size_t GetCharge(Handle* handle) const override;
|
|
||||||
uint32_t GetHash(Handle* handle) const override;
|
|
||||||
DeleterFn GetDeleter(Handle* handle) const override;
|
|
||||||
void DisownData() override;
|
|
||||||
|
|
||||||
private:
|
|
||||||
LRUCacheShard* shards_ = nullptr;
|
|
||||||
int num_shards_ = 0;
|
|
||||||
};
|
|
||||||
} // namespace fast_lru_cache
|
|
||||||
|
|
||||||
std::shared_ptr<Cache> NewFastLRUCache(
|
|
||||||
size_t capacity, int num_shard_bits = -1,
|
|
||||||
bool strict_capacity_limit = false,
|
|
||||||
CacheMetadataChargePolicy metadata_charge_policy =
|
|
||||||
kDefaultCacheMetadataChargePolicy);
|
|
||||||
|
|
||||||
} // namespace ROCKSDB_NAMESPACE
|
|
3
cache/lru_cache.cc
vendored
3
cache/lru_cache.cc
vendored
@ -19,7 +19,6 @@
|
|||||||
#include "util/mutexlock.h"
|
#include "util/mutexlock.h"
|
||||||
|
|
||||||
namespace ROCKSDB_NAMESPACE {
|
namespace ROCKSDB_NAMESPACE {
|
||||||
namespace lru_cache {
|
|
||||||
|
|
||||||
LRUHandleTable::LRUHandleTable(int max_upper_hash_bits)
|
LRUHandleTable::LRUHandleTable(int max_upper_hash_bits)
|
||||||
: length_bits_(/* historical starting size*/ 4),
|
: length_bits_(/* historical starting size*/ 4),
|
||||||
@ -760,8 +759,6 @@ void LRUCache::WaitAll(std::vector<Handle*>& handles) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
} // namespace lru_cache
|
|
||||||
|
|
||||||
std::shared_ptr<Cache> NewLRUCache(
|
std::shared_ptr<Cache> NewLRUCache(
|
||||||
size_t capacity, int num_shard_bits, bool strict_capacity_limit,
|
size_t capacity, int num_shard_bits, bool strict_capacity_limit,
|
||||||
double high_pri_pool_ratio,
|
double high_pri_pool_ratio,
|
||||||
|
7
cache/lru_cache.h
vendored
7
cache/lru_cache.h
vendored
@ -19,7 +19,6 @@
|
|||||||
#include "util/autovector.h"
|
#include "util/autovector.h"
|
||||||
|
|
||||||
namespace ROCKSDB_NAMESPACE {
|
namespace ROCKSDB_NAMESPACE {
|
||||||
namespace lru_cache {
|
|
||||||
|
|
||||||
// LRU cache implementation. This class is not thread-safe.
|
// LRU cache implementation. This class is not thread-safe.
|
||||||
|
|
||||||
@ -480,10 +479,4 @@ class LRUCache
|
|||||||
std::shared_ptr<SecondaryCache> secondary_cache_;
|
std::shared_ptr<SecondaryCache> secondary_cache_;
|
||||||
};
|
};
|
||||||
|
|
||||||
} // namespace lru_cache
|
|
||||||
|
|
||||||
using LRUCache = lru_cache::LRUCache;
|
|
||||||
using LRUHandle = lru_cache::LRUHandle;
|
|
||||||
using LRUCacheShard = lru_cache::LRUCacheShard;
|
|
||||||
|
|
||||||
} // namespace ROCKSDB_NAMESPACE
|
} // namespace ROCKSDB_NAMESPACE
|
||||||
|
30
common.mk
30
common.mk
@ -1,30 +0,0 @@
|
|||||||
ifndef PYTHON
|
|
||||||
|
|
||||||
# Default to python3. Some distros like CentOS 8 do not have `python`.
|
|
||||||
ifeq ($(origin PYTHON), undefined)
|
|
||||||
PYTHON := $(shell which python3 || which python || echo python3)
|
|
||||||
endif
|
|
||||||
export PYTHON
|
|
||||||
|
|
||||||
endif
|
|
||||||
|
|
||||||
# To setup tmp directory, first recognize some old variables for setting
|
|
||||||
# test tmp directory or base tmp directory. TEST_TMPDIR is usually read
|
|
||||||
# by RocksDB tools though Env/FileSystem::GetTestDirectory.
|
|
||||||
ifeq ($(TEST_TMPDIR),)
|
|
||||||
TEST_TMPDIR := $(TMPD)
|
|
||||||
endif
|
|
||||||
ifeq ($(TEST_TMPDIR),)
|
|
||||||
ifeq ($(BASE_TMPDIR),)
|
|
||||||
BASE_TMPDIR :=$(TMPDIR)
|
|
||||||
endif
|
|
||||||
ifeq ($(BASE_TMPDIR),)
|
|
||||||
BASE_TMPDIR :=/tmp
|
|
||||||
endif
|
|
||||||
# Use /dev/shm if it has the sticky bit set (otherwise, /tmp or other
|
|
||||||
# base dir), and create a randomly-named rocksdb.XXXX directory therein.
|
|
||||||
TEST_TMPDIR := $(shell f=/dev/shm; test -k $$f || f=$(BASE_TMPDIR); \
|
|
||||||
perl -le 'use File::Temp "tempdir";' \
|
|
||||||
-e 'print tempdir("'$$f'/rocksdb.XXXX", CLEANUP => 0)')
|
|
||||||
endif
|
|
||||||
export TEST_TMPDIR
|
|
@ -5,7 +5,7 @@
|
|||||||
# build DB_STRESS_CMD so it must exist prior.
|
# build DB_STRESS_CMD so it must exist prior.
|
||||||
DB_STRESS_CMD?=./db_stress
|
DB_STRESS_CMD?=./db_stress
|
||||||
|
|
||||||
include common.mk
|
include python.mk
|
||||||
|
|
||||||
CRASHTEST_MAKE=$(MAKE) -f crash_test.mk
|
CRASHTEST_MAKE=$(MAKE) -f crash_test.mk
|
||||||
CRASHTEST_PY=$(PYTHON) -u tools/db_crashtest.py --stress_cmd=$(DB_STRESS_CMD)
|
CRASHTEST_PY=$(PYTHON) -u tools/db_crashtest.py --stress_cmd=$(DB_STRESS_CMD)
|
||||||
@ -65,10 +65,10 @@ blackbox_crash_test_with_ts: $(DB_STRESS_CMD)
|
|||||||
$(CRASHTEST_PY) --enable_ts blackbox $(CRASH_TEST_EXT_ARGS)
|
$(CRASHTEST_PY) --enable_ts blackbox $(CRASH_TEST_EXT_ARGS)
|
||||||
|
|
||||||
blackbox_crash_test_with_multiops_wc_txn: $(DB_STRESS_CMD)
|
blackbox_crash_test_with_multiops_wc_txn: $(DB_STRESS_CMD)
|
||||||
$(CRASHTEST_PY) --test_multiops_txn --write_policy write_committed blackbox $(CRASH_TEST_EXT_ARGS)
|
$(PYTHON) -u tools/db_crashtest.py --test_multiops_txn --write_policy write_committed blackbox $(CRASH_TEST_EXT_ARGS)
|
||||||
|
|
||||||
blackbox_crash_test_with_multiops_wp_txn: $(DB_STRESS_CMD)
|
blackbox_crash_test_with_multiops_wp_txn: $(DB_STRESS_CMD)
|
||||||
$(CRASHTEST_PY) --test_multiops_txn --write_policy write_prepared blackbox $(CRASH_TEST_EXT_ARGS)
|
$(PYTHON) -u tools/db_crashtest.py --test_multiops_txn --write_policy write_prepared blackbox $(CRASH_TEST_EXT_ARGS)
|
||||||
|
|
||||||
ifeq ($(CRASH_TEST_KILL_ODD),)
|
ifeq ($(CRASH_TEST_KILL_ODD),)
|
||||||
CRASH_TEST_KILL_ODD=888887
|
CRASH_TEST_KILL_ODD=888887
|
||||||
|
@ -23,7 +23,7 @@ Status ArenaWrappedDBIter::GetProperty(std::string prop_name,
|
|||||||
if (prop_name == "rocksdb.iterator.super-version-number") {
|
if (prop_name == "rocksdb.iterator.super-version-number") {
|
||||||
// First try to pass the value returned from inner iterator.
|
// First try to pass the value returned from inner iterator.
|
||||||
if (!db_iter_->GetProperty(prop_name, prop).ok()) {
|
if (!db_iter_->GetProperty(prop_name, prop).ok()) {
|
||||||
*prop = std::to_string(sv_number_);
|
*prop = ToString(sv_number_);
|
||||||
}
|
}
|
||||||
return Status::OK();
|
return Status::OK();
|
||||||
}
|
}
|
||||||
|
@ -96,9 +96,9 @@ class BlobIndex {
|
|||||||
assert(slice.size() > 0);
|
assert(slice.size() > 0);
|
||||||
type_ = static_cast<Type>(*slice.data());
|
type_ = static_cast<Type>(*slice.data());
|
||||||
if (type_ >= Type::kUnknown) {
|
if (type_ >= Type::kUnknown) {
|
||||||
return Status::Corruption(kErrorMessage,
|
return Status::Corruption(
|
||||||
"Unknown blob index type: " +
|
kErrorMessage,
|
||||||
std::to_string(static_cast<char>(type_)));
|
"Unknown blob index type: " + ToString(static_cast<char>(type_)));
|
||||||
}
|
}
|
||||||
slice = Slice(slice.data() + 1, slice.size() - 1);
|
slice = Slice(slice.data() + 1, slice.size() - 1);
|
||||||
if (HasTTL()) {
|
if (HasTTL()) {
|
||||||
|
@ -153,11 +153,11 @@ TEST_F(DBBlobIndexTest, Write) {
|
|||||||
key_values.reserve(num_key_values);
|
key_values.reserve(num_key_values);
|
||||||
|
|
||||||
for (size_t i = 1; i <= num_key_values; ++i) {
|
for (size_t i = 1; i <= num_key_values; ++i) {
|
||||||
std::string key = "key" + std::to_string(i);
|
std::string key = "key" + ToString(i);
|
||||||
|
|
||||||
std::string blob_index;
|
std::string blob_index;
|
||||||
BlobIndex::EncodeInlinedTTL(&blob_index, /* expiration */ 9876543210,
|
BlobIndex::EncodeInlinedTTL(&blob_index, /* expiration */ 9876543210,
|
||||||
"blob" + std::to_string(i));
|
"blob" + ToString(i));
|
||||||
|
|
||||||
key_values.emplace_back(std::move(key), std::move(blob_index));
|
key_values.emplace_back(std::move(key), std::move(blob_index));
|
||||||
}
|
}
|
||||||
@ -230,7 +230,7 @@ TEST_F(DBBlobIndexTest, Updated) {
|
|||||||
DestroyAndReopen(GetTestOptions());
|
DestroyAndReopen(GetTestOptions());
|
||||||
WriteBatch batch;
|
WriteBatch batch;
|
||||||
for (int i = 0; i < 10; i++) {
|
for (int i = 0; i < 10; i++) {
|
||||||
ASSERT_OK(PutBlobIndex(&batch, "key" + std::to_string(i), blob_index));
|
ASSERT_OK(PutBlobIndex(&batch, "key" + ToString(i), blob_index));
|
||||||
}
|
}
|
||||||
ASSERT_OK(Write(&batch));
|
ASSERT_OK(Write(&batch));
|
||||||
// Avoid blob values from being purged.
|
// Avoid blob values from being purged.
|
||||||
@ -248,7 +248,7 @@ TEST_F(DBBlobIndexTest, Updated) {
|
|||||||
ASSERT_OK(dbfull()->DeleteRange(WriteOptions(), cfh(), "key6", "key9"));
|
ASSERT_OK(dbfull()->DeleteRange(WriteOptions(), cfh(), "key6", "key9"));
|
||||||
MoveDataTo(tier);
|
MoveDataTo(tier);
|
||||||
for (int i = 0; i < 10; i++) {
|
for (int i = 0; i < 10; i++) {
|
||||||
ASSERT_EQ(blob_index, GetBlobIndex("key" + std::to_string(i), snapshot));
|
ASSERT_EQ(blob_index, GetBlobIndex("key" + ToString(i), snapshot));
|
||||||
}
|
}
|
||||||
ASSERT_EQ("new_value", Get("key1"));
|
ASSERT_EQ("new_value", Get("key1"));
|
||||||
if (tier <= kImmutableMemtables) {
|
if (tier <= kImmutableMemtables) {
|
||||||
@ -260,7 +260,7 @@ TEST_F(DBBlobIndexTest, Updated) {
|
|||||||
ASSERT_EQ("NOT_FOUND", Get("key4"));
|
ASSERT_EQ("NOT_FOUND", Get("key4"));
|
||||||
ASSERT_EQ("a,b,c", GetImpl("key5"));
|
ASSERT_EQ("a,b,c", GetImpl("key5"));
|
||||||
for (int i = 6; i < 9; i++) {
|
for (int i = 6; i < 9; i++) {
|
||||||
ASSERT_EQ("NOT_FOUND", Get("key" + std::to_string(i)));
|
ASSERT_EQ("NOT_FOUND", Get("key" + ToString(i)));
|
||||||
}
|
}
|
||||||
ASSERT_EQ(blob_index, GetBlobIndex("key9"));
|
ASSERT_EQ(blob_index, GetBlobIndex("key9"));
|
||||||
dbfull()->ReleaseSnapshot(snapshot);
|
dbfull()->ReleaseSnapshot(snapshot);
|
||||||
@ -301,7 +301,7 @@ TEST_F(DBBlobIndexTest, Iterate) {
|
|||||||
};
|
};
|
||||||
|
|
||||||
auto get_value = [&](int index, int version) {
|
auto get_value = [&](int index, int version) {
|
||||||
return get_key(index) + "_value" + std::to_string(version);
|
return get_key(index) + "_value" + ToString(version);
|
||||||
};
|
};
|
||||||
|
|
||||||
auto check_iterator = [&](Iterator* iterator, Status::Code expected_status,
|
auto check_iterator = [&](Iterator* iterator, Status::Code expected_status,
|
||||||
@ -501,7 +501,7 @@ TEST_F(DBBlobIndexTest, IntegratedBlobIterate) {
|
|||||||
auto get_key = [](size_t index) { return ("key" + std::to_string(index)); };
|
auto get_key = [](size_t index) { return ("key" + std::to_string(index)); };
|
||||||
|
|
||||||
auto get_value = [&](size_t index, size_t version) {
|
auto get_value = [&](size_t index, size_t version) {
|
||||||
return get_key(index) + "_value" + std::to_string(version);
|
return get_key(index) + "_value" + ToString(version);
|
||||||
};
|
};
|
||||||
|
|
||||||
auto check_iterator = [&](Iterator* iterator, Status expected_status,
|
auto check_iterator = [&](Iterator* iterator, Status expected_status,
|
||||||
|
@ -115,7 +115,6 @@ Status BuildTable(
|
|||||||
assert(fs);
|
assert(fs);
|
||||||
|
|
||||||
TableProperties tp;
|
TableProperties tp;
|
||||||
bool table_file_created = false;
|
|
||||||
if (iter->Valid() || !range_del_agg->IsEmpty()) {
|
if (iter->Valid() || !range_del_agg->IsEmpty()) {
|
||||||
std::unique_ptr<CompactionFilter> compaction_filter;
|
std::unique_ptr<CompactionFilter> compaction_filter;
|
||||||
if (ioptions.compaction_filter_factory != nullptr &&
|
if (ioptions.compaction_filter_factory != nullptr &&
|
||||||
@ -159,8 +158,6 @@ Status BuildTable(
|
|||||||
file_checksum_func_name);
|
file_checksum_func_name);
|
||||||
return s;
|
return s;
|
||||||
}
|
}
|
||||||
|
|
||||||
table_file_created = true;
|
|
||||||
FileTypeSet tmp_set = ioptions.checksum_handoff_file_types;
|
FileTypeSet tmp_set = ioptions.checksum_handoff_file_types;
|
||||||
file->SetIOPriority(io_priority);
|
file->SetIOPriority(io_priority);
|
||||||
file->SetWriteLifeTimeHint(write_hint);
|
file->SetWriteLifeTimeHint(write_hint);
|
||||||
@ -196,7 +193,6 @@ Status BuildTable(
|
|||||||
ShouldReportDetailedTime(env, ioptions.stats),
|
ShouldReportDetailedTime(env, ioptions.stats),
|
||||||
true /* internal key corruption is not ok */, range_del_agg.get(),
|
true /* internal key corruption is not ok */, range_del_agg.get(),
|
||||||
blob_file_builder.get(), ioptions.allow_data_in_errors,
|
blob_file_builder.get(), ioptions.allow_data_in_errors,
|
||||||
ioptions.enforce_single_del_contracts,
|
|
||||||
/*compaction=*/nullptr, compaction_filter.get(),
|
/*compaction=*/nullptr, compaction_filter.get(),
|
||||||
/*shutting_down=*/nullptr,
|
/*shutting_down=*/nullptr,
|
||||||
/*manual_compaction_paused=*/nullptr,
|
/*manual_compaction_paused=*/nullptr,
|
||||||
@ -375,16 +371,14 @@ Status BuildTable(
|
|||||||
|
|
||||||
constexpr IODebugContext* dbg = nullptr;
|
constexpr IODebugContext* dbg = nullptr;
|
||||||
|
|
||||||
if (table_file_created) {
|
|
||||||
Status ignored = fs->DeleteFile(fname, IOOptions(), dbg);
|
Status ignored = fs->DeleteFile(fname, IOOptions(), dbg);
|
||||||
ignored.PermitUncheckedError();
|
ignored.PermitUncheckedError();
|
||||||
}
|
|
||||||
|
|
||||||
assert(blob_file_additions || blob_file_paths.empty());
|
assert(blob_file_additions || blob_file_paths.empty());
|
||||||
|
|
||||||
if (blob_file_additions) {
|
if (blob_file_additions) {
|
||||||
for (const std::string& blob_file_path : blob_file_paths) {
|
for (const std::string& blob_file_path : blob_file_paths) {
|
||||||
Status ignored = DeleteDBFile(&db_options, blob_file_path, dbname,
|
ignored = DeleteDBFile(&db_options, blob_file_path, dbname,
|
||||||
/*force_bg=*/false, /*force_fg=*/false);
|
/*force_bg=*/false, /*force_fg=*/false);
|
||||||
ignored.PermitUncheckedError();
|
ignored.PermitUncheckedError();
|
||||||
TEST_SYNC_POINT("BuildTable::AfterDeleteFile");
|
TEST_SYNC_POINT("BuildTable::AfterDeleteFile");
|
||||||
|
37
db/c.cc
37
db/c.cc
@ -1163,43 +1163,6 @@ void rocksdb_multi_get_cf(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void rocksdb_batched_multi_get_cf(rocksdb_t* db,
|
|
||||||
const rocksdb_readoptions_t* options,
|
|
||||||
rocksdb_column_family_handle_t* column_family,
|
|
||||||
size_t num_keys, const char* const* keys_list,
|
|
||||||
const size_t* keys_list_sizes,
|
|
||||||
rocksdb_pinnableslice_t** values, char** errs,
|
|
||||||
const bool sorted_input) {
|
|
||||||
Slice* key_slices = new Slice[num_keys];
|
|
||||||
PinnableSlice* value_slices = new PinnableSlice[num_keys];
|
|
||||||
Status* statuses = new Status[num_keys];
|
|
||||||
for (size_t i = 0; i < num_keys; ++i) {
|
|
||||||
key_slices[i] = Slice(keys_list[i], keys_list_sizes[i]);
|
|
||||||
}
|
|
||||||
|
|
||||||
db->rep->MultiGet(options->rep, column_family->rep, num_keys, key_slices,
|
|
||||||
value_slices, statuses, sorted_input);
|
|
||||||
|
|
||||||
for (size_t i = 0; i < num_keys; ++i) {
|
|
||||||
if (statuses[i].ok()) {
|
|
||||||
values[i] = new (rocksdb_pinnableslice_t);
|
|
||||||
values[i]->rep = std::move(value_slices[i]);
|
|
||||||
errs[i] = nullptr;
|
|
||||||
} else {
|
|
||||||
values[i] = nullptr;
|
|
||||||
if (!statuses[i].IsNotFound()) {
|
|
||||||
errs[i] = strdup(statuses[i].ToString().c_str());
|
|
||||||
} else {
|
|
||||||
errs[i] = nullptr;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
delete[] key_slices;
|
|
||||||
delete[] value_slices;
|
|
||||||
delete[] statuses;
|
|
||||||
}
|
|
||||||
|
|
||||||
unsigned char rocksdb_key_may_exist(rocksdb_t* db,
|
unsigned char rocksdb_key_may_exist(rocksdb_t* db,
|
||||||
const rocksdb_readoptions_t* options,
|
const rocksdb_readoptions_t* options,
|
||||||
const char* key, size_t key_len,
|
const char* key, size_t key_len,
|
||||||
|
27
db/c_test.c
27
db/c_test.c
@ -1260,18 +1260,15 @@ int main(int argc, char** argv) {
|
|||||||
rocksdb_writebatch_clear(wb);
|
rocksdb_writebatch_clear(wb);
|
||||||
rocksdb_writebatch_put_cf(wb, handles[1], "bar", 3, "b", 1);
|
rocksdb_writebatch_put_cf(wb, handles[1], "bar", 3, "b", 1);
|
||||||
rocksdb_writebatch_put_cf(wb, handles[1], "box", 3, "c", 1);
|
rocksdb_writebatch_put_cf(wb, handles[1], "box", 3, "c", 1);
|
||||||
rocksdb_writebatch_put_cf(wb, handles[1], "buff", 4, "rocksdb", 7);
|
|
||||||
rocksdb_writebatch_delete_cf(wb, handles[1], "bar", 3);
|
rocksdb_writebatch_delete_cf(wb, handles[1], "bar", 3);
|
||||||
rocksdb_write(db, woptions, wb, &err);
|
rocksdb_write(db, woptions, wb, &err);
|
||||||
CheckNoError(err);
|
CheckNoError(err);
|
||||||
CheckGetCF(db, roptions, handles[1], "baz", NULL);
|
CheckGetCF(db, roptions, handles[1], "baz", NULL);
|
||||||
CheckGetCF(db, roptions, handles[1], "bar", NULL);
|
CheckGetCF(db, roptions, handles[1], "bar", NULL);
|
||||||
CheckGetCF(db, roptions, handles[1], "box", "c");
|
CheckGetCF(db, roptions, handles[1], "box", "c");
|
||||||
CheckGetCF(db, roptions, handles[1], "buff", "rocksdb");
|
|
||||||
CheckPinGetCF(db, roptions, handles[1], "baz", NULL);
|
CheckPinGetCF(db, roptions, handles[1], "baz", NULL);
|
||||||
CheckPinGetCF(db, roptions, handles[1], "bar", NULL);
|
CheckPinGetCF(db, roptions, handles[1], "bar", NULL);
|
||||||
CheckPinGetCF(db, roptions, handles[1], "box", "c");
|
CheckPinGetCF(db, roptions, handles[1], "box", "c");
|
||||||
CheckPinGetCF(db, roptions, handles[1], "buff", "rocksdb");
|
|
||||||
rocksdb_writebatch_destroy(wb);
|
rocksdb_writebatch_destroy(wb);
|
||||||
|
|
||||||
rocksdb_flush_wal(db, 1, &err);
|
rocksdb_flush_wal(db, 1, &err);
|
||||||
@ -1302,26 +1299,6 @@ int main(int argc, char** argv) {
|
|||||||
Free(&vals[i]);
|
Free(&vals[i]);
|
||||||
}
|
}
|
||||||
|
|
||||||
{
|
|
||||||
const char* batched_keys[4] = {"box", "buff", "barfooxx", "box"};
|
|
||||||
const size_t batched_keys_sizes[4] = {3, 4, 8, 3};
|
|
||||||
const char* expected_value[4] = {"c", "rocksdb", NULL, "c"};
|
|
||||||
char* batched_errs[4];
|
|
||||||
|
|
||||||
rocksdb_pinnableslice_t* pvals[4];
|
|
||||||
rocksdb_batched_multi_get_cf(db, roptions, handles[1], 4, batched_keys,
|
|
||||||
batched_keys_sizes, pvals, batched_errs,
|
|
||||||
false);
|
|
||||||
const char* val;
|
|
||||||
size_t val_len;
|
|
||||||
for (i = 0; i < 4; ++i) {
|
|
||||||
val = rocksdb_pinnableslice_value(pvals[i], &val_len);
|
|
||||||
CheckNoError(batched_errs[i]);
|
|
||||||
CheckEqual(expected_value[i], val, val_len);
|
|
||||||
rocksdb_pinnableslice_destroy(pvals[i]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
{
|
{
|
||||||
unsigned char value_found = 0;
|
unsigned char value_found = 0;
|
||||||
|
|
||||||
@ -1353,7 +1330,7 @@ int main(int argc, char** argv) {
|
|||||||
for (i = 0; rocksdb_iter_valid(iter) != 0; rocksdb_iter_next(iter)) {
|
for (i = 0; rocksdb_iter_valid(iter) != 0; rocksdb_iter_next(iter)) {
|
||||||
i++;
|
i++;
|
||||||
}
|
}
|
||||||
CheckCondition(i == 4);
|
CheckCondition(i == 3);
|
||||||
rocksdb_iter_get_error(iter, &err);
|
rocksdb_iter_get_error(iter, &err);
|
||||||
CheckNoError(err);
|
CheckNoError(err);
|
||||||
rocksdb_iter_destroy(iter);
|
rocksdb_iter_destroy(iter);
|
||||||
@ -1377,7 +1354,7 @@ int main(int argc, char** argv) {
|
|||||||
for (i = 0; rocksdb_iter_valid(iter) != 0; rocksdb_iter_next(iter)) {
|
for (i = 0; rocksdb_iter_valid(iter) != 0; rocksdb_iter_next(iter)) {
|
||||||
i++;
|
i++;
|
||||||
}
|
}
|
||||||
CheckCondition(i == 4);
|
CheckCondition(i == 3);
|
||||||
rocksdb_iter_get_error(iter, &err);
|
rocksdb_iter_get_error(iter, &err);
|
||||||
CheckNoError(err);
|
CheckNoError(err);
|
||||||
rocksdb_iter_destroy(iter);
|
rocksdb_iter_destroy(iter);
|
||||||
|
@ -501,8 +501,7 @@ std::vector<std::string> ColumnFamilyData::GetDbPaths() const {
|
|||||||
return paths;
|
return paths;
|
||||||
}
|
}
|
||||||
|
|
||||||
const uint32_t ColumnFamilyData::kDummyColumnFamilyDataId =
|
const uint32_t ColumnFamilyData::kDummyColumnFamilyDataId = port::kMaxUint32;
|
||||||
std::numeric_limits<uint32_t>::max();
|
|
||||||
|
|
||||||
ColumnFamilyData::ColumnFamilyData(
|
ColumnFamilyData::ColumnFamilyData(
|
||||||
uint32_t id, const std::string& name, Version* _dummy_versions,
|
uint32_t id, const std::string& name, Version* _dummy_versions,
|
||||||
@ -827,8 +826,8 @@ int GetL0ThresholdSpeedupCompaction(int level0_file_num_compaction_trigger,
|
|||||||
// condition.
|
// condition.
|
||||||
// Or twice as compaction trigger, if it is smaller.
|
// Or twice as compaction trigger, if it is smaller.
|
||||||
int64_t res = std::min(twice_level0_trigger, one_fourth_trigger_slowdown);
|
int64_t res = std::min(twice_level0_trigger, one_fourth_trigger_slowdown);
|
||||||
if (res >= std::numeric_limits<int32_t>::max()) {
|
if (res >= port::kMaxInt32) {
|
||||||
return std::numeric_limits<int32_t>::max();
|
return port::kMaxInt32;
|
||||||
} else {
|
} else {
|
||||||
// res fits in int
|
// res fits in int
|
||||||
return static_cast<int>(res);
|
return static_cast<int>(res);
|
||||||
|
@ -383,7 +383,7 @@ class ColumnFamilyTestBase : public testing::Test {
|
|||||||
|
|
||||||
int NumTableFilesAtLevel(int level, int cf) {
|
int NumTableFilesAtLevel(int level, int cf) {
|
||||||
return GetProperty(cf,
|
return GetProperty(cf,
|
||||||
"rocksdb.num-files-at-level" + std::to_string(level));
|
"rocksdb.num-files-at-level" + ToString(level));
|
||||||
}
|
}
|
||||||
|
|
||||||
#ifndef ROCKSDB_LITE
|
#ifndef ROCKSDB_LITE
|
||||||
@ -783,7 +783,7 @@ TEST_P(ColumnFamilyTest, BulkAddDrop) {
|
|||||||
std::vector<std::string> cf_names;
|
std::vector<std::string> cf_names;
|
||||||
std::vector<ColumnFamilyHandle*> cf_handles;
|
std::vector<ColumnFamilyHandle*> cf_handles;
|
||||||
for (int i = 1; i <= kNumCF; i++) {
|
for (int i = 1; i <= kNumCF; i++) {
|
||||||
cf_names.push_back("cf1-" + std::to_string(i));
|
cf_names.push_back("cf1-" + ToString(i));
|
||||||
}
|
}
|
||||||
ASSERT_OK(db_->CreateColumnFamilies(cf_options, cf_names, &cf_handles));
|
ASSERT_OK(db_->CreateColumnFamilies(cf_options, cf_names, &cf_handles));
|
||||||
for (int i = 1; i <= kNumCF; i++) {
|
for (int i = 1; i <= kNumCF; i++) {
|
||||||
@ -796,8 +796,7 @@ TEST_P(ColumnFamilyTest, BulkAddDrop) {
|
|||||||
}
|
}
|
||||||
cf_handles.clear();
|
cf_handles.clear();
|
||||||
for (int i = 1; i <= kNumCF; i++) {
|
for (int i = 1; i <= kNumCF; i++) {
|
||||||
cf_descriptors.emplace_back("cf2-" + std::to_string(i),
|
cf_descriptors.emplace_back("cf2-" + ToString(i), ColumnFamilyOptions());
|
||||||
ColumnFamilyOptions());
|
|
||||||
}
|
}
|
||||||
ASSERT_OK(db_->CreateColumnFamilies(cf_descriptors, &cf_handles));
|
ASSERT_OK(db_->CreateColumnFamilies(cf_descriptors, &cf_handles));
|
||||||
for (int i = 1; i <= kNumCF; i++) {
|
for (int i = 1; i <= kNumCF; i++) {
|
||||||
@ -821,7 +820,7 @@ TEST_P(ColumnFamilyTest, DropTest) {
|
|||||||
Open({"default"});
|
Open({"default"});
|
||||||
CreateColumnFamiliesAndReopen({"pikachu"});
|
CreateColumnFamiliesAndReopen({"pikachu"});
|
||||||
for (int i = 0; i < 100; ++i) {
|
for (int i = 0; i < 100; ++i) {
|
||||||
ASSERT_OK(Put(1, std::to_string(i), "bar" + std::to_string(i)));
|
ASSERT_OK(Put(1, ToString(i), "bar" + ToString(i)));
|
||||||
}
|
}
|
||||||
ASSERT_OK(Flush(1));
|
ASSERT_OK(Flush(1));
|
||||||
|
|
||||||
@ -1345,7 +1344,7 @@ TEST_P(ColumnFamilyTest, DifferentCompactionStyles) {
|
|||||||
PutRandomData(1, 10, 12000);
|
PutRandomData(1, 10, 12000);
|
||||||
PutRandomData(1, 1, 10);
|
PutRandomData(1, 1, 10);
|
||||||
WaitForFlush(1);
|
WaitForFlush(1);
|
||||||
AssertFilesPerLevel(std::to_string(i + 1), 1);
|
AssertFilesPerLevel(ToString(i + 1), 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
// SETUP column family "two" -- level style with 4 levels
|
// SETUP column family "two" -- level style with 4 levels
|
||||||
@ -1353,7 +1352,7 @@ TEST_P(ColumnFamilyTest, DifferentCompactionStyles) {
|
|||||||
PutRandomData(2, 10, 12000);
|
PutRandomData(2, 10, 12000);
|
||||||
PutRandomData(2, 1, 10);
|
PutRandomData(2, 1, 10);
|
||||||
WaitForFlush(2);
|
WaitForFlush(2);
|
||||||
AssertFilesPerLevel(std::to_string(i + 1), 2);
|
AssertFilesPerLevel(ToString(i + 1), 2);
|
||||||
}
|
}
|
||||||
|
|
||||||
// TRIGGER compaction "one"
|
// TRIGGER compaction "one"
|
||||||
@ -1417,7 +1416,7 @@ TEST_P(ColumnFamilyTest, MultipleManualCompactions) {
|
|||||||
PutRandomData(1, 10, 12000, true);
|
PutRandomData(1, 10, 12000, true);
|
||||||
PutRandomData(1, 1, 10, true);
|
PutRandomData(1, 1, 10, true);
|
||||||
WaitForFlush(1);
|
WaitForFlush(1);
|
||||||
AssertFilesPerLevel(std::to_string(i + 1), 1);
|
AssertFilesPerLevel(ToString(i + 1), 1);
|
||||||
}
|
}
|
||||||
bool cf_1_1 = true;
|
bool cf_1_1 = true;
|
||||||
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency(
|
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency(
|
||||||
@ -1447,7 +1446,7 @@ TEST_P(ColumnFamilyTest, MultipleManualCompactions) {
|
|||||||
PutRandomData(2, 10, 12000);
|
PutRandomData(2, 10, 12000);
|
||||||
PutRandomData(2, 1, 10);
|
PutRandomData(2, 1, 10);
|
||||||
WaitForFlush(2);
|
WaitForFlush(2);
|
||||||
AssertFilesPerLevel(std::to_string(i + 1), 2);
|
AssertFilesPerLevel(ToString(i + 1), 2);
|
||||||
}
|
}
|
||||||
threads.emplace_back([&] {
|
threads.emplace_back([&] {
|
||||||
TEST_SYNC_POINT("ColumnFamilyTest::MultiManual:1");
|
TEST_SYNC_POINT("ColumnFamilyTest::MultiManual:1");
|
||||||
@ -1534,7 +1533,7 @@ TEST_P(ColumnFamilyTest, AutomaticAndManualCompactions) {
|
|||||||
PutRandomData(1, 10, 12000, true);
|
PutRandomData(1, 10, 12000, true);
|
||||||
PutRandomData(1, 1, 10, true);
|
PutRandomData(1, 1, 10, true);
|
||||||
WaitForFlush(1);
|
WaitForFlush(1);
|
||||||
AssertFilesPerLevel(std::to_string(i + 1), 1);
|
AssertFilesPerLevel(ToString(i + 1), 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST_SYNC_POINT("ColumnFamilyTest::AutoManual:1");
|
TEST_SYNC_POINT("ColumnFamilyTest::AutoManual:1");
|
||||||
@ -1544,7 +1543,7 @@ TEST_P(ColumnFamilyTest, AutomaticAndManualCompactions) {
|
|||||||
PutRandomData(2, 10, 12000);
|
PutRandomData(2, 10, 12000);
|
||||||
PutRandomData(2, 1, 10);
|
PutRandomData(2, 1, 10);
|
||||||
WaitForFlush(2);
|
WaitForFlush(2);
|
||||||
AssertFilesPerLevel(std::to_string(i + 1), 2);
|
AssertFilesPerLevel(ToString(i + 1), 2);
|
||||||
}
|
}
|
||||||
ROCKSDB_NAMESPACE::port::Thread threads([&] {
|
ROCKSDB_NAMESPACE::port::Thread threads([&] {
|
||||||
CompactRangeOptions compact_options;
|
CompactRangeOptions compact_options;
|
||||||
@ -1616,7 +1615,7 @@ TEST_P(ColumnFamilyTest, ManualAndAutomaticCompactions) {
|
|||||||
PutRandomData(1, 10, 12000, true);
|
PutRandomData(1, 10, 12000, true);
|
||||||
PutRandomData(1, 1, 10, true);
|
PutRandomData(1, 1, 10, true);
|
||||||
WaitForFlush(1);
|
WaitForFlush(1);
|
||||||
AssertFilesPerLevel(std::to_string(i + 1), 1);
|
AssertFilesPerLevel(ToString(i + 1), 1);
|
||||||
}
|
}
|
||||||
bool cf_1_1 = true;
|
bool cf_1_1 = true;
|
||||||
bool cf_1_2 = true;
|
bool cf_1_2 = true;
|
||||||
@ -1651,7 +1650,7 @@ TEST_P(ColumnFamilyTest, ManualAndAutomaticCompactions) {
|
|||||||
PutRandomData(2, 10, 12000);
|
PutRandomData(2, 10, 12000);
|
||||||
PutRandomData(2, 1, 10);
|
PutRandomData(2, 1, 10);
|
||||||
WaitForFlush(2);
|
WaitForFlush(2);
|
||||||
AssertFilesPerLevel(std::to_string(i + 1), 2);
|
AssertFilesPerLevel(ToString(i + 1), 2);
|
||||||
}
|
}
|
||||||
TEST_SYNC_POINT("ColumnFamilyTest::ManualAuto:5");
|
TEST_SYNC_POINT("ColumnFamilyTest::ManualAuto:5");
|
||||||
threads.join();
|
threads.join();
|
||||||
@ -1710,7 +1709,7 @@ TEST_P(ColumnFamilyTest, SameCFManualManualCompactions) {
|
|||||||
PutRandomData(1, 10, 12000, true);
|
PutRandomData(1, 10, 12000, true);
|
||||||
PutRandomData(1, 1, 10, true);
|
PutRandomData(1, 1, 10, true);
|
||||||
WaitForFlush(1);
|
WaitForFlush(1);
|
||||||
AssertFilesPerLevel(std::to_string(i + 1), 1);
|
AssertFilesPerLevel(ToString(i + 1), 1);
|
||||||
}
|
}
|
||||||
bool cf_1_1 = true;
|
bool cf_1_1 = true;
|
||||||
bool cf_1_2 = true;
|
bool cf_1_2 = true;
|
||||||
@ -1749,8 +1748,8 @@ TEST_P(ColumnFamilyTest, SameCFManualManualCompactions) {
|
|||||||
PutRandomData(1, 10, 12000, true);
|
PutRandomData(1, 10, 12000, true);
|
||||||
PutRandomData(1, 1, 10, true);
|
PutRandomData(1, 1, 10, true);
|
||||||
WaitForFlush(1);
|
WaitForFlush(1);
|
||||||
AssertFilesPerLevel(
|
AssertFilesPerLevel(ToString(one.level0_file_num_compaction_trigger + i),
|
||||||
std::to_string(one.level0_file_num_compaction_trigger + i), 1);
|
1);
|
||||||
}
|
}
|
||||||
|
|
||||||
ROCKSDB_NAMESPACE::port::Thread threads1([&] {
|
ROCKSDB_NAMESPACE::port::Thread threads1([&] {
|
||||||
@ -1812,7 +1811,7 @@ TEST_P(ColumnFamilyTest, SameCFManualAutomaticCompactions) {
|
|||||||
PutRandomData(1, 10, 12000, true);
|
PutRandomData(1, 10, 12000, true);
|
||||||
PutRandomData(1, 1, 10, true);
|
PutRandomData(1, 1, 10, true);
|
||||||
WaitForFlush(1);
|
WaitForFlush(1);
|
||||||
AssertFilesPerLevel(std::to_string(i + 1), 1);
|
AssertFilesPerLevel(ToString(i + 1), 1);
|
||||||
}
|
}
|
||||||
bool cf_1_1 = true;
|
bool cf_1_1 = true;
|
||||||
bool cf_1_2 = true;
|
bool cf_1_2 = true;
|
||||||
@ -1850,8 +1849,8 @@ TEST_P(ColumnFamilyTest, SameCFManualAutomaticCompactions) {
|
|||||||
PutRandomData(1, 10, 12000, true);
|
PutRandomData(1, 10, 12000, true);
|
||||||
PutRandomData(1, 1, 10, true);
|
PutRandomData(1, 1, 10, true);
|
||||||
WaitForFlush(1);
|
WaitForFlush(1);
|
||||||
AssertFilesPerLevel(
|
AssertFilesPerLevel(ToString(one.level0_file_num_compaction_trigger + i),
|
||||||
std::to_string(one.level0_file_num_compaction_trigger + i), 1);
|
1);
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST_SYNC_POINT("ColumnFamilyTest::ManualAuto:1");
|
TEST_SYNC_POINT("ColumnFamilyTest::ManualAuto:1");
|
||||||
@ -1905,7 +1904,7 @@ TEST_P(ColumnFamilyTest, SameCFManualAutomaticCompactionsLevel) {
|
|||||||
PutRandomData(1, 10, 12000, true);
|
PutRandomData(1, 10, 12000, true);
|
||||||
PutRandomData(1, 1, 10, true);
|
PutRandomData(1, 1, 10, true);
|
||||||
WaitForFlush(1);
|
WaitForFlush(1);
|
||||||
AssertFilesPerLevel(std::to_string(i + 1), 1);
|
AssertFilesPerLevel(ToString(i + 1), 1);
|
||||||
}
|
}
|
||||||
bool cf_1_1 = true;
|
bool cf_1_1 = true;
|
||||||
bool cf_1_2 = true;
|
bool cf_1_2 = true;
|
||||||
@ -1943,8 +1942,8 @@ TEST_P(ColumnFamilyTest, SameCFManualAutomaticCompactionsLevel) {
|
|||||||
PutRandomData(1, 10, 12000, true);
|
PutRandomData(1, 10, 12000, true);
|
||||||
PutRandomData(1, 1, 10, true);
|
PutRandomData(1, 1, 10, true);
|
||||||
WaitForFlush(1);
|
WaitForFlush(1);
|
||||||
AssertFilesPerLevel(
|
AssertFilesPerLevel(ToString(one.level0_file_num_compaction_trigger + i),
|
||||||
std::to_string(one.level0_file_num_compaction_trigger + i), 1);
|
1);
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST_SYNC_POINT("ColumnFamilyTest::ManualAuto:1");
|
TEST_SYNC_POINT("ColumnFamilyTest::ManualAuto:1");
|
||||||
@ -2025,7 +2024,7 @@ TEST_P(ColumnFamilyTest, SameCFAutomaticManualCompactions) {
|
|||||||
PutRandomData(1, 10, 12000, true);
|
PutRandomData(1, 10, 12000, true);
|
||||||
PutRandomData(1, 1, 10, true);
|
PutRandomData(1, 1, 10, true);
|
||||||
WaitForFlush(1);
|
WaitForFlush(1);
|
||||||
AssertFilesPerLevel(std::to_string(i + 1), 1);
|
AssertFilesPerLevel(ToString(i + 1), 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST_SYNC_POINT("ColumnFamilyTest::AutoManual:5");
|
TEST_SYNC_POINT("ColumnFamilyTest::AutoManual:5");
|
||||||
|
@ -91,8 +91,8 @@ TEST_F(CompactFilesTest, L0ConflictsFiles) {
|
|||||||
// create couple files
|
// create couple files
|
||||||
// Background compaction starts and waits in BackgroundCallCompaction:0
|
// Background compaction starts and waits in BackgroundCallCompaction:0
|
||||||
for (int i = 0; i < kLevel0Trigger * 4; ++i) {
|
for (int i = 0; i < kLevel0Trigger * 4; ++i) {
|
||||||
ASSERT_OK(db->Put(WriteOptions(), std::to_string(i), ""));
|
ASSERT_OK(db->Put(WriteOptions(), ToString(i), ""));
|
||||||
ASSERT_OK(db->Put(WriteOptions(), std::to_string(100 - i), ""));
|
ASSERT_OK(db->Put(WriteOptions(), ToString(100 - i), ""));
|
||||||
ASSERT_OK(db->Flush(FlushOptions()));
|
ASSERT_OK(db->Flush(FlushOptions()));
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -136,7 +136,7 @@ TEST_F(CompactFilesTest, MultipleLevel) {
|
|||||||
// create couple files in L0, L3, L4 and L5
|
// create couple files in L0, L3, L4 and L5
|
||||||
for (int i = 5; i > 2; --i) {
|
for (int i = 5; i > 2; --i) {
|
||||||
collector->ClearFlushedFiles();
|
collector->ClearFlushedFiles();
|
||||||
ASSERT_OK(db->Put(WriteOptions(), std::to_string(i), ""));
|
ASSERT_OK(db->Put(WriteOptions(), ToString(i), ""));
|
||||||
ASSERT_OK(db->Flush(FlushOptions()));
|
ASSERT_OK(db->Flush(FlushOptions()));
|
||||||
// Ensure background work is fully finished including listener callbacks
|
// Ensure background work is fully finished including listener callbacks
|
||||||
// before accessing listener state.
|
// before accessing listener state.
|
||||||
@ -145,11 +145,11 @@ TEST_F(CompactFilesTest, MultipleLevel) {
|
|||||||
ASSERT_OK(db->CompactFiles(CompactionOptions(), l0_files, i));
|
ASSERT_OK(db->CompactFiles(CompactionOptions(), l0_files, i));
|
||||||
|
|
||||||
std::string prop;
|
std::string prop;
|
||||||
ASSERT_TRUE(db->GetProperty(
|
ASSERT_TRUE(
|
||||||
"rocksdb.num-files-at-level" + std::to_string(i), &prop));
|
db->GetProperty("rocksdb.num-files-at-level" + ToString(i), &prop));
|
||||||
ASSERT_EQ("1", prop);
|
ASSERT_EQ("1", prop);
|
||||||
}
|
}
|
||||||
ASSERT_OK(db->Put(WriteOptions(), std::to_string(0), ""));
|
ASSERT_OK(db->Put(WriteOptions(), ToString(0), ""));
|
||||||
ASSERT_OK(db->Flush(FlushOptions()));
|
ASSERT_OK(db->Flush(FlushOptions()));
|
||||||
|
|
||||||
ColumnFamilyMetaData meta;
|
ColumnFamilyMetaData meta;
|
||||||
@ -218,7 +218,7 @@ TEST_F(CompactFilesTest, ObsoleteFiles) {
|
|||||||
|
|
||||||
// create couple files
|
// create couple files
|
||||||
for (int i = 1000; i < 2000; ++i) {
|
for (int i = 1000; i < 2000; ++i) {
|
||||||
ASSERT_OK(db->Put(WriteOptions(), std::to_string(i),
|
ASSERT_OK(db->Put(WriteOptions(), ToString(i),
|
||||||
std::string(kWriteBufferSize / 10, 'a' + (i % 26))));
|
std::string(kWriteBufferSize / 10, 'a' + (i % 26))));
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -257,14 +257,14 @@ TEST_F(CompactFilesTest, NotCutOutputOnLevel0) {
|
|||||||
|
|
||||||
// create couple files
|
// create couple files
|
||||||
for (int i = 0; i < 500; ++i) {
|
for (int i = 0; i < 500; ++i) {
|
||||||
ASSERT_OK(db->Put(WriteOptions(), std::to_string(i),
|
ASSERT_OK(db->Put(WriteOptions(), ToString(i),
|
||||||
std::string(1000, 'a' + (i % 26))));
|
std::string(1000, 'a' + (i % 26))));
|
||||||
}
|
}
|
||||||
ASSERT_OK(static_cast_with_check<DBImpl>(db)->TEST_WaitForFlushMemTable());
|
ASSERT_OK(static_cast_with_check<DBImpl>(db)->TEST_WaitForFlushMemTable());
|
||||||
auto l0_files_1 = collector->GetFlushedFiles();
|
auto l0_files_1 = collector->GetFlushedFiles();
|
||||||
collector->ClearFlushedFiles();
|
collector->ClearFlushedFiles();
|
||||||
for (int i = 0; i < 500; ++i) {
|
for (int i = 0; i < 500; ++i) {
|
||||||
ASSERT_OK(db->Put(WriteOptions(), std::to_string(i),
|
ASSERT_OK(db->Put(WriteOptions(), ToString(i),
|
||||||
std::string(1000, 'a' + (i % 26))));
|
std::string(1000, 'a' + (i % 26))));
|
||||||
}
|
}
|
||||||
ASSERT_OK(static_cast_with_check<DBImpl>(db)->TEST_WaitForFlushMemTable());
|
ASSERT_OK(static_cast_with_check<DBImpl>(db)->TEST_WaitForFlushMemTable());
|
||||||
@ -295,7 +295,7 @@ TEST_F(CompactFilesTest, CapturingPendingFiles) {
|
|||||||
|
|
||||||
// Create 5 files.
|
// Create 5 files.
|
||||||
for (int i = 0; i < 5; ++i) {
|
for (int i = 0; i < 5; ++i) {
|
||||||
ASSERT_OK(db->Put(WriteOptions(), "key" + std::to_string(i), "value"));
|
ASSERT_OK(db->Put(WriteOptions(), "key" + ToString(i), "value"));
|
||||||
ASSERT_OK(db->Flush(FlushOptions()));
|
ASSERT_OK(db->Flush(FlushOptions()));
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -465,7 +465,7 @@ TEST_F(CompactFilesTest, GetCompactionJobInfo) {
|
|||||||
|
|
||||||
// create couple files
|
// create couple files
|
||||||
for (int i = 0; i < 500; ++i) {
|
for (int i = 0; i < 500; ++i) {
|
||||||
ASSERT_OK(db->Put(WriteOptions(), std::to_string(i),
|
ASSERT_OK(db->Put(WriteOptions(), ToString(i),
|
||||||
std::string(1000, 'a' + (i % 26))));
|
std::string(1000, 'a' + (i % 26))));
|
||||||
}
|
}
|
||||||
ASSERT_OK(static_cast_with_check<DBImpl>(db)->TEST_WaitForFlushMemTable());
|
ASSERT_OK(static_cast_with_check<DBImpl>(db)->TEST_WaitForFlushMemTable());
|
||||||
|
@ -518,7 +518,7 @@ uint64_t Compaction::OutputFilePreallocationSize() const {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (max_output_file_size_ != std::numeric_limits<uint64_t>::max() &&
|
if (max_output_file_size_ != port::kMaxUint64 &&
|
||||||
(immutable_options_.compaction_style == kCompactionStyleLevel ||
|
(immutable_options_.compaction_style == kCompactionStyleLevel ||
|
||||||
output_level() > 0)) {
|
output_level() > 0)) {
|
||||||
preallocation_size = std::min(max_output_file_size_, preallocation_size);
|
preallocation_size = std::min(max_output_file_size_, preallocation_size);
|
||||||
@ -616,7 +616,7 @@ bool Compaction::DoesInputReferenceBlobFiles() const {
|
|||||||
|
|
||||||
uint64_t Compaction::MinInputFileOldestAncesterTime(
|
uint64_t Compaction::MinInputFileOldestAncesterTime(
|
||||||
const InternalKey* start, const InternalKey* end) const {
|
const InternalKey* start, const InternalKey* end) const {
|
||||||
uint64_t min_oldest_ancester_time = std::numeric_limits<uint64_t>::max();
|
uint64_t min_oldest_ancester_time = port::kMaxUint64;
|
||||||
const InternalKeyComparator& icmp =
|
const InternalKeyComparator& icmp =
|
||||||
column_family_data()->internal_comparator();
|
column_family_data()->internal_comparator();
|
||||||
for (const auto& level_files : inputs_) {
|
for (const auto& level_files : inputs_) {
|
||||||
|
@ -28,8 +28,7 @@ CompactionIterator::CompactionIterator(
|
|||||||
Env* env, bool report_detailed_time, bool expect_valid_internal_key,
|
Env* env, bool report_detailed_time, bool expect_valid_internal_key,
|
||||||
CompactionRangeDelAggregator* range_del_agg,
|
CompactionRangeDelAggregator* range_del_agg,
|
||||||
BlobFileBuilder* blob_file_builder, bool allow_data_in_errors,
|
BlobFileBuilder* blob_file_builder, bool allow_data_in_errors,
|
||||||
bool enforce_single_del_contracts, const Compaction* compaction,
|
const Compaction* compaction, const CompactionFilter* compaction_filter,
|
||||||
const CompactionFilter* compaction_filter,
|
|
||||||
const std::atomic<bool>* shutting_down,
|
const std::atomic<bool>* shutting_down,
|
||||||
const std::atomic<int>* manual_compaction_paused,
|
const std::atomic<int>* manual_compaction_paused,
|
||||||
const std::atomic<bool>* manual_compaction_canceled,
|
const std::atomic<bool>* manual_compaction_canceled,
|
||||||
@ -39,7 +38,7 @@ CompactionIterator::CompactionIterator(
|
|||||||
input, cmp, merge_helper, last_sequence, snapshots,
|
input, cmp, merge_helper, last_sequence, snapshots,
|
||||||
earliest_write_conflict_snapshot, job_snapshot, snapshot_checker, env,
|
earliest_write_conflict_snapshot, job_snapshot, snapshot_checker, env,
|
||||||
report_detailed_time, expect_valid_internal_key, range_del_agg,
|
report_detailed_time, expect_valid_internal_key, range_del_agg,
|
||||||
blob_file_builder, allow_data_in_errors, enforce_single_del_contracts,
|
blob_file_builder, allow_data_in_errors,
|
||||||
std::unique_ptr<CompactionProxy>(
|
std::unique_ptr<CompactionProxy>(
|
||||||
compaction ? new RealCompaction(compaction) : nullptr),
|
compaction ? new RealCompaction(compaction) : nullptr),
|
||||||
compaction_filter, shutting_down, manual_compaction_paused,
|
compaction_filter, shutting_down, manual_compaction_paused,
|
||||||
@ -53,7 +52,6 @@ CompactionIterator::CompactionIterator(
|
|||||||
Env* env, bool report_detailed_time, bool expect_valid_internal_key,
|
Env* env, bool report_detailed_time, bool expect_valid_internal_key,
|
||||||
CompactionRangeDelAggregator* range_del_agg,
|
CompactionRangeDelAggregator* range_del_agg,
|
||||||
BlobFileBuilder* blob_file_builder, bool allow_data_in_errors,
|
BlobFileBuilder* blob_file_builder, bool allow_data_in_errors,
|
||||||
bool enforce_single_del_contracts,
|
|
||||||
std::unique_ptr<CompactionProxy> compaction,
|
std::unique_ptr<CompactionProxy> compaction,
|
||||||
const CompactionFilter* compaction_filter,
|
const CompactionFilter* compaction_filter,
|
||||||
const std::atomic<bool>* shutting_down,
|
const std::atomic<bool>* shutting_down,
|
||||||
@ -82,7 +80,6 @@ CompactionIterator::CompactionIterator(
|
|||||||
manual_compaction_canceled_(manual_compaction_canceled),
|
manual_compaction_canceled_(manual_compaction_canceled),
|
||||||
info_log_(info_log),
|
info_log_(info_log),
|
||||||
allow_data_in_errors_(allow_data_in_errors),
|
allow_data_in_errors_(allow_data_in_errors),
|
||||||
enforce_single_del_contracts_(enforce_single_del_contracts),
|
|
||||||
timestamp_size_(cmp_ ? cmp_->timestamp_size() : 0),
|
timestamp_size_(cmp_ ? cmp_->timestamp_size() : 0),
|
||||||
full_history_ts_low_(full_history_ts_low),
|
full_history_ts_low_(full_history_ts_low),
|
||||||
current_user_key_sequence_(0),
|
current_user_key_sequence_(0),
|
||||||
@ -310,7 +307,7 @@ bool CompactionIterator::InvokeFilterIfNeeded(bool* need_skip,
|
|||||||
// no value associated with delete
|
// no value associated with delete
|
||||||
value_.clear();
|
value_.clear();
|
||||||
iter_stats_.num_record_drop_user++;
|
iter_stats_.num_record_drop_user++;
|
||||||
} else if (filter == CompactionFilter::Decision::kPurge) {
|
} else if (filter == CompactionFilter::Decision::kRemoveWithSingleDelete) {
|
||||||
// convert the current key to a single delete; key_ is pointing into
|
// convert the current key to a single delete; key_ is pointing into
|
||||||
// current_key_ at this point, so updating current_key_ updates key()
|
// current_key_ at this point, so updating current_key_ updates key()
|
||||||
ikey_.type = kTypeSingleDeletion;
|
ikey_.type = kTypeSingleDeletion;
|
||||||
@ -660,15 +657,10 @@ void CompactionIterator::NextFromInput() {
|
|||||||
"TransactionDBOptions::rollback_deletion_type_callback is "
|
"TransactionDBOptions::rollback_deletion_type_callback is "
|
||||||
"configured properly. Mixing SD and DEL can lead to "
|
"configured properly. Mixing SD and DEL can lead to "
|
||||||
"undefined behaviors";
|
"undefined behaviors";
|
||||||
++iter_stats_.num_record_drop_obsolete;
|
|
||||||
++iter_stats_.num_single_del_mismatch;
|
|
||||||
if (enforce_single_del_contracts_) {
|
|
||||||
ROCKS_LOG_ERROR(info_log_, "%s", oss.str().c_str());
|
ROCKS_LOG_ERROR(info_log_, "%s", oss.str().c_str());
|
||||||
valid_ = false;
|
valid_ = false;
|
||||||
status_ = Status::Corruption(oss.str());
|
status_ = Status::Corruption(oss.str());
|
||||||
return;
|
return;
|
||||||
}
|
|
||||||
ROCKS_LOG_WARN(info_log_, "%s", oss.str().c_str());
|
|
||||||
} else if (!is_timestamp_eligible_for_gc) {
|
} else if (!is_timestamp_eligible_for_gc) {
|
||||||
// We cannot drop the SingleDelete as timestamp is enabled, and
|
// We cannot drop the SingleDelete as timestamp is enabled, and
|
||||||
// timestamp of this key is greater than or equal to
|
// timestamp of this key is greater than or equal to
|
||||||
|
@ -176,7 +176,7 @@ class CompactionIterator {
|
|||||||
Env* env, bool report_detailed_time, bool expect_valid_internal_key,
|
Env* env, bool report_detailed_time, bool expect_valid_internal_key,
|
||||||
CompactionRangeDelAggregator* range_del_agg,
|
CompactionRangeDelAggregator* range_del_agg,
|
||||||
BlobFileBuilder* blob_file_builder, bool allow_data_in_errors,
|
BlobFileBuilder* blob_file_builder, bool allow_data_in_errors,
|
||||||
bool enforce_single_del_contracts, const Compaction* compaction = nullptr,
|
const Compaction* compaction = nullptr,
|
||||||
const CompactionFilter* compaction_filter = nullptr,
|
const CompactionFilter* compaction_filter = nullptr,
|
||||||
const std::atomic<bool>* shutting_down = nullptr,
|
const std::atomic<bool>* shutting_down = nullptr,
|
||||||
const std::atomic<int>* manual_compaction_paused = nullptr,
|
const std::atomic<int>* manual_compaction_paused = nullptr,
|
||||||
@ -193,7 +193,6 @@ class CompactionIterator {
|
|||||||
Env* env, bool report_detailed_time, bool expect_valid_internal_key,
|
Env* env, bool report_detailed_time, bool expect_valid_internal_key,
|
||||||
CompactionRangeDelAggregator* range_del_agg,
|
CompactionRangeDelAggregator* range_del_agg,
|
||||||
BlobFileBuilder* blob_file_builder, bool allow_data_in_errors,
|
BlobFileBuilder* blob_file_builder, bool allow_data_in_errors,
|
||||||
bool enforce_single_del_contracts,
|
|
||||||
std::unique_ptr<CompactionProxy> compaction,
|
std::unique_ptr<CompactionProxy> compaction,
|
||||||
const CompactionFilter* compaction_filter = nullptr,
|
const CompactionFilter* compaction_filter = nullptr,
|
||||||
const std::atomic<bool>* shutting_down = nullptr,
|
const std::atomic<bool>* shutting_down = nullptr,
|
||||||
@ -333,8 +332,6 @@ class CompactionIterator {
|
|||||||
|
|
||||||
bool allow_data_in_errors_;
|
bool allow_data_in_errors_;
|
||||||
|
|
||||||
const bool enforce_single_del_contracts_;
|
|
||||||
|
|
||||||
// Comes from comparator.
|
// Comes from comparator.
|
||||||
const size_t timestamp_size_;
|
const size_t timestamp_size_;
|
||||||
|
|
||||||
|
@ -279,8 +279,7 @@ class CompactionIteratorTest : public testing::TestWithParam<bool> {
|
|||||||
snapshot_checker_.get(), Env::Default(),
|
snapshot_checker_.get(), Env::Default(),
|
||||||
false /* report_detailed_time */, false, range_del_agg_.get(),
|
false /* report_detailed_time */, false, range_del_agg_.get(),
|
||||||
nullptr /* blob_file_builder */, true /*allow_data_in_errors*/,
|
nullptr /* blob_file_builder */, true /*allow_data_in_errors*/,
|
||||||
true /*enforce_single_del_contracts*/, std::move(compaction), filter,
|
std::move(compaction), filter, &shutting_down_,
|
||||||
&shutting_down_,
|
|
||||||
/*manual_compaction_paused=*/nullptr,
|
/*manual_compaction_paused=*/nullptr,
|
||||||
/*manual_compaction_canceled=*/nullptr, /*info_log=*/nullptr,
|
/*manual_compaction_canceled=*/nullptr, /*info_log=*/nullptr,
|
||||||
full_history_ts_low));
|
full_history_ts_low));
|
||||||
@ -314,7 +313,7 @@ class CompactionIteratorTest : public testing::TestWithParam<bool> {
|
|||||||
key_not_exists_beyond_output_level, full_history_ts_low);
|
key_not_exists_beyond_output_level, full_history_ts_low);
|
||||||
c_iter_->SeekToFirst();
|
c_iter_->SeekToFirst();
|
||||||
for (size_t i = 0; i < expected_keys.size(); i++) {
|
for (size_t i = 0; i < expected_keys.size(); i++) {
|
||||||
std::string info = "i = " + std::to_string(i);
|
std::string info = "i = " + ToString(i);
|
||||||
ASSERT_TRUE(c_iter_->Valid()) << info;
|
ASSERT_TRUE(c_iter_->Valid()) << info;
|
||||||
ASSERT_OK(c_iter_->status()) << info;
|
ASSERT_OK(c_iter_->status()) << info;
|
||||||
ASSERT_EQ(expected_keys[i], c_iter_->key().ToString()) << info;
|
ASSERT_EQ(expected_keys[i], c_iter_->key().ToString()) << info;
|
||||||
|
@ -1479,9 +1479,9 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
|
|||||||
snapshot_checker_, env_, ShouldReportDetailedTime(env_, stats_),
|
snapshot_checker_, env_, ShouldReportDetailedTime(env_, stats_),
|
||||||
/*expect_valid_internal_key=*/true, &range_del_agg,
|
/*expect_valid_internal_key=*/true, &range_del_agg,
|
||||||
blob_file_builder.get(), db_options_.allow_data_in_errors,
|
blob_file_builder.get(), db_options_.allow_data_in_errors,
|
||||||
db_options_.enforce_single_del_contracts, sub_compact->compaction,
|
sub_compact->compaction, compaction_filter, shutting_down_,
|
||||||
compaction_filter, shutting_down_, manual_compaction_paused_,
|
manual_compaction_paused_, manual_compaction_canceled_,
|
||||||
manual_compaction_canceled_, db_options_.info_log, full_history_ts_low));
|
db_options_.info_log, full_history_ts_low));
|
||||||
auto c_iter = sub_compact->c_iter.get();
|
auto c_iter = sub_compact->c_iter.get();
|
||||||
c_iter->SeekToFirst();
|
c_iter->SeekToFirst();
|
||||||
if (c_iter->Valid() && sub_compact->compaction->output_level() != 0) {
|
if (c_iter->Valid() && sub_compact->compaction->output_level() != 0) {
|
||||||
@ -1974,8 +1974,7 @@ Status CompactionJob::FinishCompactionOutputFile(
|
|||||||
refined_oldest_ancester_time =
|
refined_oldest_ancester_time =
|
||||||
sub_compact->compaction->MinInputFileOldestAncesterTime(
|
sub_compact->compaction->MinInputFileOldestAncesterTime(
|
||||||
&(meta->smallest), &(meta->largest));
|
&(meta->smallest), &(meta->largest));
|
||||||
if (refined_oldest_ancester_time !=
|
if (refined_oldest_ancester_time != port::kMaxUint64) {
|
||||||
std::numeric_limits<uint64_t>::max()) {
|
|
||||||
meta->oldest_ancester_time = refined_oldest_ancester_time;
|
meta->oldest_ancester_time = refined_oldest_ancester_time;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -2265,7 +2264,7 @@ Status CompactionJob::OpenCompactionOutputFile(
|
|||||||
sub_compact->compaction->MinInputFileOldestAncesterTime(
|
sub_compact->compaction->MinInputFileOldestAncesterTime(
|
||||||
(sub_compact->start != nullptr) ? &tmp_start : nullptr,
|
(sub_compact->start != nullptr) ? &tmp_start : nullptr,
|
||||||
(sub_compact->end != nullptr) ? &tmp_end : nullptr);
|
(sub_compact->end != nullptr) ? &tmp_end : nullptr);
|
||||||
if (oldest_ancester_time == std::numeric_limits<uint64_t>::max()) {
|
if (oldest_ancester_time == port::kMaxUint64) {
|
||||||
oldest_ancester_time = current_time;
|
oldest_ancester_time = current_time;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -2459,7 +2458,7 @@ void CompactionJob::LogCompaction() {
|
|||||||
<< "compaction_reason"
|
<< "compaction_reason"
|
||||||
<< GetCompactionReasonString(compaction->compaction_reason());
|
<< GetCompactionReasonString(compaction->compaction_reason());
|
||||||
for (size_t i = 0; i < compaction->num_input_levels(); ++i) {
|
for (size_t i = 0; i < compaction->num_input_levels(); ++i) {
|
||||||
stream << ("files_L" + std::to_string(compaction->level(i)));
|
stream << ("files_L" + ToString(compaction->level(i)));
|
||||||
stream.StartArray();
|
stream.StartArray();
|
||||||
for (auto f : *compaction->inputs(i)) {
|
for (auto f : *compaction->inputs(i)) {
|
||||||
stream << f->fd.GetNumber();
|
stream << f->fd.GetNumber();
|
||||||
@ -2952,7 +2951,6 @@ static std::unordered_map<std::string, OptionTypeInfo> cs_result_type_info = {
|
|||||||
const void* addr1, const void* addr2, std::string* mismatch) {
|
const void* addr1, const void* addr2, std::string* mismatch) {
|
||||||
const auto status1 = static_cast<const Status*>(addr1);
|
const auto status1 = static_cast<const Status*>(addr1);
|
||||||
const auto status2 = static_cast<const Status*>(addr2);
|
const auto status2 = static_cast<const Status*>(addr2);
|
||||||
|
|
||||||
StatusSerializationAdapter adatper1(*status1);
|
StatusSerializationAdapter adatper1(*status1);
|
||||||
StatusSerializationAdapter adapter2(*status2);
|
StatusSerializationAdapter adapter2(*status2);
|
||||||
return OptionTypeInfo::TypesAreEqual(opts, status_adapter_type_info,
|
return OptionTypeInfo::TypesAreEqual(opts, status_adapter_type_info,
|
||||||
@ -3010,7 +3008,7 @@ Status CompactionServiceInput::Read(const std::string& data_str,
|
|||||||
} else {
|
} else {
|
||||||
return Status::NotSupported(
|
return Status::NotSupported(
|
||||||
"Compaction Service Input data version not supported: " +
|
"Compaction Service Input data version not supported: " +
|
||||||
std::to_string(format_version));
|
ToString(format_version));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -3039,7 +3037,7 @@ Status CompactionServiceResult::Read(const std::string& data_str,
|
|||||||
} else {
|
} else {
|
||||||
return Status::NotSupported(
|
return Status::NotSupported(
|
||||||
"Compaction Service Result data version not supported: " +
|
"Compaction Service Result data version not supported: " +
|
||||||
std::to_string(format_version));
|
ToString(format_version));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -268,10 +268,10 @@ class CompactionJobStatsTest : public testing::Test,
|
|||||||
if (cf == 0) {
|
if (cf == 0) {
|
||||||
// default cfd
|
// default cfd
|
||||||
EXPECT_TRUE(db_->GetProperty(
|
EXPECT_TRUE(db_->GetProperty(
|
||||||
"rocksdb.num-files-at-level" + std::to_string(level), &property));
|
"rocksdb.num-files-at-level" + ToString(level), &property));
|
||||||
} else {
|
} else {
|
||||||
EXPECT_TRUE(db_->GetProperty(
|
EXPECT_TRUE(db_->GetProperty(
|
||||||
handles_[cf], "rocksdb.num-files-at-level" + std::to_string(level),
|
handles_[cf], "rocksdb.num-files-at-level" + ToString(level),
|
||||||
&property));
|
&property));
|
||||||
}
|
}
|
||||||
return atoi(property.c_str());
|
return atoi(property.c_str());
|
||||||
@ -672,7 +672,7 @@ TEST_P(CompactionJobStatsTest, CompactionJobStatsTest) {
|
|||||||
snprintf(buf, kBufSize, "%d", ++num_L0_files);
|
snprintf(buf, kBufSize, "%d", ++num_L0_files);
|
||||||
ASSERT_EQ(std::string(buf), FilesPerLevel(1));
|
ASSERT_EQ(std::string(buf), FilesPerLevel(1));
|
||||||
}
|
}
|
||||||
ASSERT_EQ(std::to_string(num_L0_files), FilesPerLevel(1));
|
ASSERT_EQ(ToString(num_L0_files), FilesPerLevel(1));
|
||||||
|
|
||||||
// 2nd Phase: perform L0 -> L1 compaction.
|
// 2nd Phase: perform L0 -> L1 compaction.
|
||||||
int L0_compaction_count = 6;
|
int L0_compaction_count = 6;
|
||||||
|
@ -236,8 +236,8 @@ class CompactionJobTestBase : public testing::Test {
|
|||||||
for (int i = 0; i < 2; ++i) {
|
for (int i = 0; i < 2; ++i) {
|
||||||
auto contents = mock::MakeMockFile();
|
auto contents = mock::MakeMockFile();
|
||||||
for (int k = 0; k < kKeysPerFile; ++k) {
|
for (int k = 0; k < kKeysPerFile; ++k) {
|
||||||
auto key = std::to_string(i * kMatchingKeys + k);
|
auto key = ToString(i * kMatchingKeys + k);
|
||||||
auto value = std::to_string(i * kKeysPerFile + k);
|
auto value = ToString(i * kKeysPerFile + k);
|
||||||
InternalKey internal_key(key, ++sequence_number, kTypeValue);
|
InternalKey internal_key(key, ++sequence_number, kTypeValue);
|
||||||
|
|
||||||
// This is how the key will look like once it's written in bottommost
|
// This is how the key will look like once it's written in bottommost
|
||||||
@ -1107,21 +1107,6 @@ TEST_F(CompactionJobTest, OldestBlobFileNumber) {
|
|||||||
/* expected_oldest_blob_file_number */ 19);
|
/* expected_oldest_blob_file_number */ 19);
|
||||||
}
|
}
|
||||||
|
|
||||||
TEST_F(CompactionJobTest, NoEnforceSingleDeleteContract) {
|
|
||||||
db_options_.enforce_single_del_contracts = false;
|
|
||||||
NewDB();
|
|
||||||
|
|
||||||
auto file =
|
|
||||||
mock::MakeMockFile({{KeyStr("a", 4U, kTypeSingleDeletion), ""},
|
|
||||||
{KeyStr("a", 3U, kTypeDeletion), "dontcare"}});
|
|
||||||
AddMockFile(file);
|
|
||||||
SetLastSequence(4U);
|
|
||||||
|
|
||||||
auto expected_results = mock::MakeMockFile();
|
|
||||||
auto files = cfd_->current()->storage_info()->LevelFiles(0);
|
|
||||||
RunCompaction({files}, expected_results);
|
|
||||||
}
|
|
||||||
|
|
||||||
TEST_F(CompactionJobTest, InputSerialization) {
|
TEST_F(CompactionJobTest, InputSerialization) {
|
||||||
// Setup a random CompactionServiceInput
|
// Setup a random CompactionServiceInput
|
||||||
CompactionServiceInput input;
|
CompactionServiceInput input;
|
||||||
|
@ -65,7 +65,7 @@ bool FindIntraL0Compaction(const std::vector<FileMetaData*>& level_files,
|
|||||||
size_t compact_bytes = static_cast<size_t>(level_files[start]->fd.file_size);
|
size_t compact_bytes = static_cast<size_t>(level_files[start]->fd.file_size);
|
||||||
uint64_t compensated_compact_bytes =
|
uint64_t compensated_compact_bytes =
|
||||||
level_files[start]->compensated_file_size;
|
level_files[start]->compensated_file_size;
|
||||||
size_t compact_bytes_per_del_file = std::numeric_limits<size_t>::max();
|
size_t compact_bytes_per_del_file = port::kMaxSizet;
|
||||||
// Compaction range will be [start, limit).
|
// Compaction range will be [start, limit).
|
||||||
size_t limit;
|
size_t limit;
|
||||||
// Pull in files until the amount of compaction work per deleted file begins
|
// Pull in files until the amount of compaction work per deleted file begins
|
||||||
@ -401,7 +401,7 @@ Status CompactionPicker::GetCompactionInputsFromFileNumbers(
|
|||||||
"Cannot find matched SST files for the following file numbers:");
|
"Cannot find matched SST files for the following file numbers:");
|
||||||
for (auto fn : *input_set) {
|
for (auto fn : *input_set) {
|
||||||
message += " ";
|
message += " ";
|
||||||
message += std::to_string(fn);
|
message += ToString(fn);
|
||||||
}
|
}
|
||||||
return Status::InvalidArgument(message);
|
return Status::InvalidArgument(message);
|
||||||
}
|
}
|
||||||
@ -717,7 +717,7 @@ Compaction* CompactionPicker::CompactRange(
|
|||||||
// files that are created during the current compaction.
|
// files that are created during the current compaction.
|
||||||
if (compact_range_options.bottommost_level_compaction ==
|
if (compact_range_options.bottommost_level_compaction ==
|
||||||
BottommostLevelCompaction::kForceOptimized &&
|
BottommostLevelCompaction::kForceOptimized &&
|
||||||
max_file_num_to_ignore != std::numeric_limits<uint64_t>::max()) {
|
max_file_num_to_ignore != port::kMaxUint64) {
|
||||||
assert(input_level == output_level);
|
assert(input_level == output_level);
|
||||||
// inputs_shrunk holds a continuous subset of input files which were all
|
// inputs_shrunk holds a continuous subset of input files which were all
|
||||||
// created before the current manual compaction
|
// created before the current manual compaction
|
||||||
@ -1004,14 +1004,14 @@ Status CompactionPicker::SanitizeCompactionInputFiles(
|
|||||||
return Status::InvalidArgument(
|
return Status::InvalidArgument(
|
||||||
"Output level for column family " + cf_meta.name +
|
"Output level for column family " + cf_meta.name +
|
||||||
" must between [0, " +
|
" must between [0, " +
|
||||||
std::to_string(cf_meta.levels[cf_meta.levels.size() - 1].level) + "].");
|
ToString(cf_meta.levels[cf_meta.levels.size() - 1].level) + "].");
|
||||||
}
|
}
|
||||||
|
|
||||||
if (output_level > MaxOutputLevel()) {
|
if (output_level > MaxOutputLevel()) {
|
||||||
return Status::InvalidArgument(
|
return Status::InvalidArgument(
|
||||||
"Exceed the maximum output level defined by "
|
"Exceed the maximum output level defined by "
|
||||||
"the current compaction algorithm --- " +
|
"the current compaction algorithm --- " +
|
||||||
std::to_string(MaxOutputLevel()));
|
ToString(MaxOutputLevel()));
|
||||||
}
|
}
|
||||||
|
|
||||||
if (output_level < 0) {
|
if (output_level < 0) {
|
||||||
@ -1061,8 +1061,8 @@ Status CompactionPicker::SanitizeCompactionInputFiles(
|
|||||||
return Status::InvalidArgument(
|
return Status::InvalidArgument(
|
||||||
"Cannot compact file to up level, input file: " +
|
"Cannot compact file to up level, input file: " +
|
||||||
MakeTableFileName("", file_num) + " level " +
|
MakeTableFileName("", file_num) + " level " +
|
||||||
std::to_string(input_file_level) + " > output level " +
|
ToString(input_file_level) + " > output level " +
|
||||||
std::to_string(output_level));
|
ToString(output_level));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -504,7 +504,7 @@ bool LevelCompactionBuilder::PickIntraL0Compaction() {
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
return FindIntraL0Compaction(level_files, kMinFilesForIntraL0Compaction,
|
return FindIntraL0Compaction(level_files, kMinFilesForIntraL0Compaction,
|
||||||
std::numeric_limits<uint64_t>::max(),
|
port::kMaxUint64,
|
||||||
mutable_cf_options_.max_compaction_bytes,
|
mutable_cf_options_.max_compaction_bytes,
|
||||||
&start_level_inputs_, earliest_mem_seqno_);
|
&start_level_inputs_, earliest_mem_seqno_);
|
||||||
}
|
}
|
||||||
|
@ -273,9 +273,9 @@ TEST_F(CompactionPickerTest, NeedsCompactionLevel) {
|
|||||||
// start a brand new version in each test.
|
// start a brand new version in each test.
|
||||||
NewVersionStorage(kLevels, kCompactionStyleLevel);
|
NewVersionStorage(kLevels, kCompactionStyleLevel);
|
||||||
for (int i = 0; i < file_count; ++i) {
|
for (int i = 0; i < file_count; ++i) {
|
||||||
Add(level, i, std::to_string((i + 100) * 1000).c_str(),
|
Add(level, i, ToString((i + 100) * 1000).c_str(),
|
||||||
std::to_string((i + 100) * 1000 + 999).c_str(), file_size, 0,
|
ToString((i + 100) * 1000 + 999).c_str(),
|
||||||
i * 100, i * 100 + 99);
|
file_size, 0, i * 100, i * 100 + 99);
|
||||||
}
|
}
|
||||||
UpdateVersionStorageInfo();
|
UpdateVersionStorageInfo();
|
||||||
ASSERT_EQ(vstorage_->CompactionScoreLevel(0), level);
|
ASSERT_EQ(vstorage_->CompactionScoreLevel(0), level);
|
||||||
@ -439,8 +439,8 @@ TEST_F(CompactionPickerTest, NeedsCompactionUniversal) {
|
|||||||
for (int i = 1;
|
for (int i = 1;
|
||||||
i <= mutable_cf_options_.level0_file_num_compaction_trigger * 2; ++i) {
|
i <= mutable_cf_options_.level0_file_num_compaction_trigger * 2; ++i) {
|
||||||
NewVersionStorage(1, kCompactionStyleUniversal);
|
NewVersionStorage(1, kCompactionStyleUniversal);
|
||||||
Add(0, i, std::to_string((i + 100) * 1000).c_str(),
|
Add(0, i, ToString((i + 100) * 1000).c_str(),
|
||||||
std::to_string((i + 100) * 1000 + 999).c_str(), 1000000, 0, i * 100,
|
ToString((i + 100) * 1000 + 999).c_str(), 1000000, 0, i * 100,
|
||||||
i * 100 + 99);
|
i * 100 + 99);
|
||||||
UpdateVersionStorageInfo();
|
UpdateVersionStorageInfo();
|
||||||
ASSERT_EQ(level_compaction_picker.NeedsCompaction(vstorage_.get()),
|
ASSERT_EQ(level_compaction_picker.NeedsCompaction(vstorage_.get()),
|
||||||
@ -852,17 +852,17 @@ TEST_F(CompactionPickerTest, UniversalIncrementalSpace4) {
|
|||||||
// L3: (1101, 1180) (1201, 1280) ... (7901, 7908)
|
// L3: (1101, 1180) (1201, 1280) ... (7901, 7908)
|
||||||
// L4: (1130, 1150) (1160, 1210) (1230, 1250) (1260 1310) ... (7960, 8010)
|
// L4: (1130, 1150) (1160, 1210) (1230, 1250) (1260 1310) ... (7960, 8010)
|
||||||
for (int i = 11; i < 79; i++) {
|
for (int i = 11; i < 79; i++) {
|
||||||
Add(3, 100 + i * 3, std::to_string(i * 100).c_str(),
|
Add(3, 100 + i * 3, ToString(i * 100).c_str(),
|
||||||
std::to_string(i * 100 + 80).c_str(), kFileSize, 0, 200, 251);
|
ToString(i * 100 + 80).c_str(), kFileSize, 0, 200, 251);
|
||||||
// Add a tie breaker
|
// Add a tie breaker
|
||||||
if (i == 66) {
|
if (i == 66) {
|
||||||
Add(3, 10000U, "6690", "6699", kFileSize, 0, 200, 251);
|
Add(3, 10000U, "6690", "6699", kFileSize, 0, 200, 251);
|
||||||
}
|
}
|
||||||
|
|
||||||
Add(4, 100 + i * 3 + 1, std::to_string(i * 100 + 30).c_str(),
|
Add(4, 100 + i * 3 + 1, ToString(i * 100 + 30).c_str(),
|
||||||
std::to_string(i * 100 + 50).c_str(), kFileSize, 0, 200, 251);
|
ToString(i * 100 + 50).c_str(), kFileSize, 0, 200, 251);
|
||||||
Add(4, 100 + i * 3 + 2, std::to_string(i * 100 + 60).c_str(),
|
Add(4, 100 + i * 3 + 2, ToString(i * 100 + 60).c_str(),
|
||||||
std::to_string(i * 100 + 110).c_str(), kFileSize, 0, 200, 251);
|
ToString(i * 100 + 110).c_str(), kFileSize, 0, 200, 251);
|
||||||
}
|
}
|
||||||
UpdateVersionStorageInfo();
|
UpdateVersionStorageInfo();
|
||||||
|
|
||||||
@ -899,14 +899,14 @@ TEST_F(CompactionPickerTest, UniversalIncrementalSpace5) {
|
|||||||
// L3: (1101, 1180) (1201, 1280) ... (7901, 7908)
|
// L3: (1101, 1180) (1201, 1280) ... (7901, 7908)
|
||||||
// L4: (1130, 1150) (1160, 1210) (1230, 1250) (1260 1310) ... (7960, 8010)
|
// L4: (1130, 1150) (1160, 1210) (1230, 1250) (1260 1310) ... (7960, 8010)
|
||||||
for (int i = 11; i < 70; i++) {
|
for (int i = 11; i < 70; i++) {
|
||||||
Add(3, 100 + i * 3, std::to_string(i * 100).c_str(),
|
Add(3, 100 + i * 3, ToString(i * 100).c_str(),
|
||||||
std::to_string(i * 100 + 80).c_str(),
|
ToString(i * 100 + 80).c_str(),
|
||||||
i % 10 == 9 ? kFileSize * 100 : kFileSize, 0, 200, 251);
|
i % 10 == 9 ? kFileSize * 100 : kFileSize, 0, 200, 251);
|
||||||
|
|
||||||
Add(4, 100 + i * 3 + 1, std::to_string(i * 100 + 30).c_str(),
|
Add(4, 100 + i * 3 + 1, ToString(i * 100 + 30).c_str(),
|
||||||
std::to_string(i * 100 + 50).c_str(), kFileSize, 0, 200, 251);
|
ToString(i * 100 + 50).c_str(), kFileSize, 0, 200, 251);
|
||||||
Add(4, 100 + i * 3 + 2, std::to_string(i * 100 + 60).c_str(),
|
Add(4, 100 + i * 3 + 2, ToString(i * 100 + 60).c_str(),
|
||||||
std::to_string(i * 100 + 110).c_str(), kFileSize, 0, 200, 251);
|
ToString(i * 100 + 110).c_str(), kFileSize, 0, 200, 251);
|
||||||
}
|
}
|
||||||
UpdateVersionStorageInfo();
|
UpdateVersionStorageInfo();
|
||||||
|
|
||||||
@ -941,8 +941,8 @@ TEST_F(CompactionPickerTest, NeedsCompactionFIFO) {
|
|||||||
// size of L0 files.
|
// size of L0 files.
|
||||||
for (int i = 1; i <= kFileCount; ++i) {
|
for (int i = 1; i <= kFileCount; ++i) {
|
||||||
NewVersionStorage(1, kCompactionStyleFIFO);
|
NewVersionStorage(1, kCompactionStyleFIFO);
|
||||||
Add(0, i, std::to_string((i + 100) * 1000).c_str(),
|
Add(0, i, ToString((i + 100) * 1000).c_str(),
|
||||||
std::to_string((i + 100) * 1000 + 999).c_str(), kFileSize, 0, i * 100,
|
ToString((i + 100) * 1000 + 999).c_str(), kFileSize, 0, i * 100,
|
||||||
i * 100 + 99);
|
i * 100 + 99);
|
||||||
UpdateVersionStorageInfo();
|
UpdateVersionStorageInfo();
|
||||||
ASSERT_EQ(fifo_compaction_picker.NeedsCompaction(vstorage_.get()),
|
ASSERT_EQ(fifo_compaction_picker.NeedsCompaction(vstorage_.get()),
|
||||||
@ -2653,8 +2653,8 @@ TEST_F(CompactionPickerTest, UniversalMarkedManualCompaction) {
|
|||||||
universal_compaction_picker.CompactRange(
|
universal_compaction_picker.CompactRange(
|
||||||
cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(),
|
cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(),
|
||||||
ColumnFamilyData::kCompactAllLevels, 6, CompactRangeOptions(),
|
ColumnFamilyData::kCompactAllLevels, 6, CompactRangeOptions(),
|
||||||
nullptr, nullptr, &manual_end, &manual_conflict,
|
nullptr, nullptr, &manual_end, &manual_conflict, port::kMaxUint64,
|
||||||
std::numeric_limits<uint64_t>::max(), ""));
|
""));
|
||||||
|
|
||||||
ASSERT_TRUE(compaction);
|
ASSERT_TRUE(compaction);
|
||||||
|
|
||||||
|
@ -1371,7 +1371,7 @@ Compaction* UniversalCompactionBuilder::PickPeriodicCompaction() {
|
|||||||
|
|
||||||
uint64_t UniversalCompactionBuilder::GetMaxOverlappingBytes() const {
|
uint64_t UniversalCompactionBuilder::GetMaxOverlappingBytes() const {
|
||||||
if (!mutable_cf_options_.compaction_options_universal.incremental) {
|
if (!mutable_cf_options_.compaction_options_universal.incremental) {
|
||||||
return std::numeric_limits<uint64_t>::max();
|
return port::kMaxUint64;
|
||||||
} else {
|
} else {
|
||||||
// Try to align cutting boundary with files at the next level if the
|
// Try to align cutting boundary with files at the next level if the
|
||||||
// file isn't end up with 1/2 of target size, or it would overlap
|
// file isn't end up with 1/2 of target size, or it would overlap
|
||||||
|
@ -82,7 +82,8 @@ class MyTestCompactionService : public CompactionService {
|
|||||||
options.canceled = &canceled_;
|
options.canceled = &canceled_;
|
||||||
|
|
||||||
Status s = DB::OpenAndCompact(
|
Status s = DB::OpenAndCompact(
|
||||||
options, db_path_, db_path_ + "/" + std::to_string(info.job_id),
|
options, db_path_,
|
||||||
|
db_path_ + "/" + ROCKSDB_NAMESPACE::ToString(info.job_id),
|
||||||
compaction_input, compaction_service_result, options_override);
|
compaction_input, compaction_service_result, options_override);
|
||||||
if (is_override_wait_result_) {
|
if (is_override_wait_result_) {
|
||||||
*compaction_service_result = override_wait_result_;
|
*compaction_service_result = override_wait_result_;
|
||||||
@ -176,7 +177,7 @@ class CompactionServiceTest : public DBTestBase {
|
|||||||
for (int i = 0; i < 20; i++) {
|
for (int i = 0; i < 20; i++) {
|
||||||
for (int j = 0; j < 10; j++) {
|
for (int j = 0; j < 10; j++) {
|
||||||
int key_id = i * 10 + j;
|
int key_id = i * 10 + j;
|
||||||
ASSERT_OK(Put(Key(key_id), "value" + std::to_string(key_id)));
|
ASSERT_OK(Put(Key(key_id), "value" + ToString(key_id)));
|
||||||
}
|
}
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
}
|
}
|
||||||
@ -186,7 +187,7 @@ class CompactionServiceTest : public DBTestBase {
|
|||||||
for (int i = 0; i < 10; i++) {
|
for (int i = 0; i < 10; i++) {
|
||||||
for (int j = 0; j < 10; j++) {
|
for (int j = 0; j < 10; j++) {
|
||||||
int key_id = i * 20 + j * 2;
|
int key_id = i * 20 + j * 2;
|
||||||
ASSERT_OK(Put(Key(key_id), "value_new" + std::to_string(key_id)));
|
ASSERT_OK(Put(Key(key_id), "value_new" + ToString(key_id)));
|
||||||
}
|
}
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
}
|
}
|
||||||
@ -198,9 +199,9 @@ class CompactionServiceTest : public DBTestBase {
|
|||||||
for (int i = 0; i < 200; i++) {
|
for (int i = 0; i < 200; i++) {
|
||||||
auto result = Get(Key(i));
|
auto result = Get(Key(i));
|
||||||
if (i % 2) {
|
if (i % 2) {
|
||||||
ASSERT_EQ(result, "value" + std::to_string(i));
|
ASSERT_EQ(result, "value" + ToString(i));
|
||||||
} else {
|
} else {
|
||||||
ASSERT_EQ(result, "value_new" + std::to_string(i));
|
ASSERT_EQ(result, "value_new" + ToString(i));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -223,7 +224,7 @@ TEST_F(CompactionServiceTest, BasicCompactions) {
|
|||||||
for (int i = 0; i < 20; i++) {
|
for (int i = 0; i < 20; i++) {
|
||||||
for (int j = 0; j < 10; j++) {
|
for (int j = 0; j < 10; j++) {
|
||||||
int key_id = i * 10 + j;
|
int key_id = i * 10 + j;
|
||||||
ASSERT_OK(Put(Key(key_id), "value" + std::to_string(key_id)));
|
ASSERT_OK(Put(Key(key_id), "value" + ToString(key_id)));
|
||||||
}
|
}
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
}
|
}
|
||||||
@ -231,7 +232,7 @@ TEST_F(CompactionServiceTest, BasicCompactions) {
|
|||||||
for (int i = 0; i < 10; i++) {
|
for (int i = 0; i < 10; i++) {
|
||||||
for (int j = 0; j < 10; j++) {
|
for (int j = 0; j < 10; j++) {
|
||||||
int key_id = i * 20 + j * 2;
|
int key_id = i * 20 + j * 2;
|
||||||
ASSERT_OK(Put(Key(key_id), "value_new" + std::to_string(key_id)));
|
ASSERT_OK(Put(Key(key_id), "value_new" + ToString(key_id)));
|
||||||
}
|
}
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
}
|
}
|
||||||
@ -241,9 +242,9 @@ TEST_F(CompactionServiceTest, BasicCompactions) {
|
|||||||
for (int i = 0; i < 200; i++) {
|
for (int i = 0; i < 200; i++) {
|
||||||
auto result = Get(Key(i));
|
auto result = Get(Key(i));
|
||||||
if (i % 2) {
|
if (i % 2) {
|
||||||
ASSERT_EQ(result, "value" + std::to_string(i));
|
ASSERT_EQ(result, "value" + ToString(i));
|
||||||
} else {
|
} else {
|
||||||
ASSERT_EQ(result, "value_new" + std::to_string(i));
|
ASSERT_EQ(result, "value_new" + ToString(i));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
auto my_cs = GetCompactionService();
|
auto my_cs = GetCompactionService();
|
||||||
@ -280,7 +281,7 @@ TEST_F(CompactionServiceTest, BasicCompactions) {
|
|||||||
for (int i = 0; i < 10; i++) {
|
for (int i = 0; i < 10; i++) {
|
||||||
for (int j = 0; j < 10; j++) {
|
for (int j = 0; j < 10; j++) {
|
||||||
int key_id = i * 20 + j * 2;
|
int key_id = i * 20 + j * 2;
|
||||||
s = Put(Key(key_id), "value_new" + std::to_string(key_id));
|
s = Put(Key(key_id), "value_new" + ToString(key_id));
|
||||||
if (s.IsAborted()) {
|
if (s.IsAborted()) {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
@ -467,7 +468,7 @@ TEST_F(CompactionServiceTest, CompactionFilter) {
|
|||||||
for (int i = 0; i < 20; i++) {
|
for (int i = 0; i < 20; i++) {
|
||||||
for (int j = 0; j < 10; j++) {
|
for (int j = 0; j < 10; j++) {
|
||||||
int key_id = i * 10 + j;
|
int key_id = i * 10 + j;
|
||||||
ASSERT_OK(Put(Key(key_id), "value" + std::to_string(key_id)));
|
ASSERT_OK(Put(Key(key_id), "value" + ToString(key_id)));
|
||||||
}
|
}
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
}
|
}
|
||||||
@ -475,7 +476,7 @@ TEST_F(CompactionServiceTest, CompactionFilter) {
|
|||||||
for (int i = 0; i < 10; i++) {
|
for (int i = 0; i < 10; i++) {
|
||||||
for (int j = 0; j < 10; j++) {
|
for (int j = 0; j < 10; j++) {
|
||||||
int key_id = i * 20 + j * 2;
|
int key_id = i * 20 + j * 2;
|
||||||
ASSERT_OK(Put(Key(key_id), "value_new" + std::to_string(key_id)));
|
ASSERT_OK(Put(Key(key_id), "value_new" + ToString(key_id)));
|
||||||
}
|
}
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
}
|
}
|
||||||
@ -489,9 +490,9 @@ TEST_F(CompactionServiceTest, CompactionFilter) {
|
|||||||
if (i > 5 && i <= 105) {
|
if (i > 5 && i <= 105) {
|
||||||
ASSERT_EQ(result, "NOT_FOUND");
|
ASSERT_EQ(result, "NOT_FOUND");
|
||||||
} else if (i % 2) {
|
} else if (i % 2) {
|
||||||
ASSERT_EQ(result, "value" + std::to_string(i));
|
ASSERT_EQ(result, "value" + ToString(i));
|
||||||
} else {
|
} else {
|
||||||
ASSERT_EQ(result, "value_new" + std::to_string(i));
|
ASSERT_EQ(result, "value_new" + ToString(i));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
auto my_cs = GetCompactionService();
|
auto my_cs = GetCompactionService();
|
||||||
@ -546,9 +547,9 @@ TEST_F(CompactionServiceTest, ConcurrentCompaction) {
|
|||||||
for (int i = 0; i < 200; i++) {
|
for (int i = 0; i < 200; i++) {
|
||||||
auto result = Get(Key(i));
|
auto result = Get(Key(i));
|
||||||
if (i % 2) {
|
if (i % 2) {
|
||||||
ASSERT_EQ(result, "value" + std::to_string(i));
|
ASSERT_EQ(result, "value" + ToString(i));
|
||||||
} else {
|
} else {
|
||||||
ASSERT_EQ(result, "value_new" + std::to_string(i));
|
ASSERT_EQ(result, "value_new" + ToString(i));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
auto my_cs = GetCompactionService();
|
auto my_cs = GetCompactionService();
|
||||||
@ -563,7 +564,7 @@ TEST_F(CompactionServiceTest, CompactionInfo) {
|
|||||||
for (int i = 0; i < 20; i++) {
|
for (int i = 0; i < 20; i++) {
|
||||||
for (int j = 0; j < 10; j++) {
|
for (int j = 0; j < 10; j++) {
|
||||||
int key_id = i * 10 + j;
|
int key_id = i * 10 + j;
|
||||||
ASSERT_OK(Put(Key(key_id), "value" + std::to_string(key_id)));
|
ASSERT_OK(Put(Key(key_id), "value" + ToString(key_id)));
|
||||||
}
|
}
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
}
|
}
|
||||||
@ -571,7 +572,7 @@ TEST_F(CompactionServiceTest, CompactionInfo) {
|
|||||||
for (int i = 0; i < 10; i++) {
|
for (int i = 0; i < 10; i++) {
|
||||||
for (int j = 0; j < 10; j++) {
|
for (int j = 0; j < 10; j++) {
|
||||||
int key_id = i * 20 + j * 2;
|
int key_id = i * 20 + j * 2;
|
||||||
ASSERT_OK(Put(Key(key_id), "value_new" + std::to_string(key_id)));
|
ASSERT_OK(Put(Key(key_id), "value_new" + ToString(key_id)));
|
||||||
}
|
}
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
}
|
}
|
||||||
@ -616,7 +617,7 @@ TEST_F(CompactionServiceTest, CompactionInfo) {
|
|||||||
for (int i = 0; i < 20; i++) {
|
for (int i = 0; i < 20; i++) {
|
||||||
for (int j = 0; j < 10; j++) {
|
for (int j = 0; j < 10; j++) {
|
||||||
int key_id = i * 10 + j;
|
int key_id = i * 10 + j;
|
||||||
ASSERT_OK(Put(Key(key_id), "value" + std::to_string(key_id)));
|
ASSERT_OK(Put(Key(key_id), "value" + ToString(key_id)));
|
||||||
}
|
}
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
}
|
}
|
||||||
@ -624,7 +625,7 @@ TEST_F(CompactionServiceTest, CompactionInfo) {
|
|||||||
for (int i = 0; i < 4; i++) {
|
for (int i = 0; i < 4; i++) {
|
||||||
for (int j = 0; j < 10; j++) {
|
for (int j = 0; j < 10; j++) {
|
||||||
int key_id = i * 20 + j * 2;
|
int key_id = i * 20 + j * 2;
|
||||||
ASSERT_OK(Put(Key(key_id), "value_new" + std::to_string(key_id)));
|
ASSERT_OK(Put(Key(key_id), "value_new" + ToString(key_id)));
|
||||||
}
|
}
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
}
|
}
|
||||||
@ -652,7 +653,7 @@ TEST_F(CompactionServiceTest, FallbackLocalAuto) {
|
|||||||
for (int i = 0; i < 20; i++) {
|
for (int i = 0; i < 20; i++) {
|
||||||
for (int j = 0; j < 10; j++) {
|
for (int j = 0; j < 10; j++) {
|
||||||
int key_id = i * 10 + j;
|
int key_id = i * 10 + j;
|
||||||
ASSERT_OK(Put(Key(key_id), "value" + std::to_string(key_id)));
|
ASSERT_OK(Put(Key(key_id), "value" + ToString(key_id)));
|
||||||
}
|
}
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
}
|
}
|
||||||
@ -660,7 +661,7 @@ TEST_F(CompactionServiceTest, FallbackLocalAuto) {
|
|||||||
for (int i = 0; i < 10; i++) {
|
for (int i = 0; i < 10; i++) {
|
||||||
for (int j = 0; j < 10; j++) {
|
for (int j = 0; j < 10; j++) {
|
||||||
int key_id = i * 20 + j * 2;
|
int key_id = i * 20 + j * 2;
|
||||||
ASSERT_OK(Put(Key(key_id), "value_new" + std::to_string(key_id)));
|
ASSERT_OK(Put(Key(key_id), "value_new" + ToString(key_id)));
|
||||||
}
|
}
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
}
|
}
|
||||||
@ -670,9 +671,9 @@ TEST_F(CompactionServiceTest, FallbackLocalAuto) {
|
|||||||
for (int i = 0; i < 200; i++) {
|
for (int i = 0; i < 200; i++) {
|
||||||
auto result = Get(Key(i));
|
auto result = Get(Key(i));
|
||||||
if (i % 2) {
|
if (i % 2) {
|
||||||
ASSERT_EQ(result, "value" + std::to_string(i));
|
ASSERT_EQ(result, "value" + ToString(i));
|
||||||
} else {
|
} else {
|
||||||
ASSERT_EQ(result, "value_new" + std::to_string(i));
|
ASSERT_EQ(result, "value_new" + ToString(i));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -795,7 +796,7 @@ TEST_F(CompactionServiceTest, RemoteEventListener) {
|
|||||||
for (int i = 0; i < 20; i++) {
|
for (int i = 0; i < 20; i++) {
|
||||||
for (int j = 0; j < 10; j++) {
|
for (int j = 0; j < 10; j++) {
|
||||||
int key_id = i * 10 + j;
|
int key_id = i * 10 + j;
|
||||||
ASSERT_OK(Put(Key(key_id), "value" + std::to_string(key_id)));
|
ASSERT_OK(Put(Key(key_id), "value" + ToString(key_id)));
|
||||||
}
|
}
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
}
|
}
|
||||||
@ -803,7 +804,7 @@ TEST_F(CompactionServiceTest, RemoteEventListener) {
|
|||||||
for (int i = 0; i < 10; i++) {
|
for (int i = 0; i < 10; i++) {
|
||||||
for (int j = 0; j < 10; j++) {
|
for (int j = 0; j < 10; j++) {
|
||||||
int key_id = i * 20 + j * 2;
|
int key_id = i * 20 + j * 2;
|
||||||
ASSERT_OK(Put(Key(key_id), "value_new" + std::to_string(key_id)));
|
ASSERT_OK(Put(Key(key_id), "value_new" + ToString(key_id)));
|
||||||
}
|
}
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
}
|
}
|
||||||
@ -820,9 +821,9 @@ TEST_F(CompactionServiceTest, RemoteEventListener) {
|
|||||||
for (int i = 0; i < 200; i++) {
|
for (int i = 0; i < 200; i++) {
|
||||||
auto result = Get(Key(i));
|
auto result = Get(Key(i));
|
||||||
if (i % 2) {
|
if (i % 2) {
|
||||||
ASSERT_EQ(result, "value" + std::to_string(i));
|
ASSERT_EQ(result, "value" + ToString(i));
|
||||||
} else {
|
} else {
|
||||||
ASSERT_EQ(result, "value_new" + std::to_string(i));
|
ASSERT_EQ(result, "value_new" + ToString(i));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -397,7 +397,7 @@ TEST_P(ComparatorDBTest, DoubleComparator) {
|
|||||||
for (uint32_t j = 0; j < divide_order; j++) {
|
for (uint32_t j = 0; j < divide_order; j++) {
|
||||||
to_divide *= 10.0;
|
to_divide *= 10.0;
|
||||||
}
|
}
|
||||||
source_strings.push_back(std::to_string(r / to_divide));
|
source_strings.push_back(ToString(r / to_divide));
|
||||||
}
|
}
|
||||||
|
|
||||||
DoRandomIteraratorTest(GetDB(), source_strings, &rnd, 200, 1000, 66);
|
DoRandomIteraratorTest(GetDB(), source_strings, &rnd, 200, 1000, 66);
|
||||||
|
@ -308,7 +308,7 @@ class CorruptionTest : public testing::Test {
|
|||||||
uint64_t new_size = old_size - bytes_to_truncate;
|
uint64_t new_size = old_size - bytes_to_truncate;
|
||||||
// If bytes_to_truncate == 0, it will do full truncation.
|
// If bytes_to_truncate == 0, it will do full truncation.
|
||||||
if (bytes_to_truncate == 0) {
|
if (bytes_to_truncate == 0) {
|
||||||
new_size = 0;
|
new_size = old_size;
|
||||||
}
|
}
|
||||||
ASSERT_OK(test::TruncateFile(env_, path, new_size));
|
ASSERT_OK(test::TruncateFile(env_, path, new_size));
|
||||||
}
|
}
|
||||||
@ -1049,23 +1049,16 @@ INSTANTIATE_TEST_CASE_P(CorruptionTest, CrashDuringRecoveryWithCorruptionTest,
|
|||||||
// "Column family inconsistency: SST file contains data beyond the point of
|
// "Column family inconsistency: SST file contains data beyond the point of
|
||||||
// corruption" error will be hit, causing recovery to fail.
|
// corruption" error will be hit, causing recovery to fail.
|
||||||
//
|
//
|
||||||
// After adding the fix, only after new WAL is synced, RocksDB persist a new
|
// After adding the fix, corrupted WALs whose numbers are larger than the
|
||||||
// MANIFEST with column families to ensure RocksDB is in consistent state.
|
// corrupted wal and smaller than the new WAL are moved to a separate folder.
|
||||||
|
// Only after new WAL is synced, RocksDB persist a new MANIFEST with column
|
||||||
|
// families to ensure RocksDB is in consistent state.
|
||||||
// RocksDB writes an empty WriteBatch as a sentinel to the new WAL which is
|
// RocksDB writes an empty WriteBatch as a sentinel to the new WAL which is
|
||||||
// synced immediately afterwards. The sequence number of the sentinel
|
// synced immediately afterwards. The sequence number of the sentinel
|
||||||
// WriteBatch will be the next sequence number immediately after the largest
|
// WriteBatch will be the next sequence number immediately after the largest
|
||||||
// sequence number recovered from previous WALs and MANIFEST because of which DB
|
// sequence number recovered from previous WALs and MANIFEST because of which DB
|
||||||
// will be in consistent state.
|
// will be in consistent state.
|
||||||
// If a future recovery starts from the new MANIFEST, then it means the new WAL
|
TEST_P(CrashDuringRecoveryWithCorruptionTest, CrashDuringRecovery) {
|
||||||
// is successfully synced. Due to the sentinel empty write batch at the
|
|
||||||
// beginning, kPointInTimeRecovery of WAL is guaranteed to go after this point.
|
|
||||||
// If future recovery starts from the old MANIFEST, it means the writing the new
|
|
||||||
// MANIFEST failed. It won't have the "SST ahead of WAL" error.
|
|
||||||
//
|
|
||||||
// The combination of corrupting a WAL and injecting an error during subsequent
|
|
||||||
// re-open exposes the bug of prematurely persisting a new MANIFEST with
|
|
||||||
// advanced ColumnFamilyData::log_number.
|
|
||||||
TEST_P(CrashDuringRecoveryWithCorruptionTest, DISABLED_CrashDuringRecovery) {
|
|
||||||
CloseDb();
|
CloseDb();
|
||||||
Options options;
|
Options options;
|
||||||
options.track_and_verify_wals_in_manifest =
|
options.track_and_verify_wals_in_manifest =
|
||||||
@ -1075,7 +1068,7 @@ TEST_P(CrashDuringRecoveryWithCorruptionTest, DISABLED_CrashDuringRecovery) {
|
|||||||
options.env = env_;
|
options.env = env_;
|
||||||
ASSERT_OK(DestroyDB(dbname_, options));
|
ASSERT_OK(DestroyDB(dbname_, options));
|
||||||
options.create_if_missing = true;
|
options.create_if_missing = true;
|
||||||
options.max_write_buffer_number = 8;
|
options.max_write_buffer_number = 3;
|
||||||
|
|
||||||
Reopen(&options);
|
Reopen(&options);
|
||||||
Status s;
|
Status s;
|
||||||
@ -1101,15 +1094,13 @@ TEST_P(CrashDuringRecoveryWithCorruptionTest, DISABLED_CrashDuringRecovery) {
|
|||||||
|
|
||||||
// Write one key to test_cf.
|
// Write one key to test_cf.
|
||||||
ASSERT_OK(db_->Put(WriteOptions(), handles[1], "old_key", "dontcare"));
|
ASSERT_OK(db_->Put(WriteOptions(), handles[1], "old_key", "dontcare"));
|
||||||
ASSERT_OK(db_->Flush(FlushOptions(), handles[1]));
|
|
||||||
|
|
||||||
// Write to default_cf and flush this cf several times to advance wal
|
// Write to default_cf and flush this cf several times to advance wal
|
||||||
// number. TEST_SwitchMemtable makes sure WALs are not synced and test can
|
// number.
|
||||||
// corrupt un-sync WAL.
|
|
||||||
for (int i = 0; i < 2; ++i) {
|
for (int i = 0; i < 2; ++i) {
|
||||||
ASSERT_OK(db_->Put(WriteOptions(), "key" + std::to_string(i), "value"));
|
ASSERT_OK(db_->Put(WriteOptions(), "key" + std::to_string(i), "value"));
|
||||||
ASSERT_OK(dbimpl->TEST_SwitchMemtable());
|
ASSERT_OK(dbimpl->TEST_SwitchMemtable());
|
||||||
}
|
}
|
||||||
|
ASSERT_OK(db_->Put(WriteOptions(), handles[1], "dontcare", "dontcare"));
|
||||||
|
|
||||||
for (auto* h : handles) {
|
for (auto* h : handles) {
|
||||||
delete h;
|
delete h;
|
||||||
@ -1118,13 +1109,12 @@ TEST_P(CrashDuringRecoveryWithCorruptionTest, DISABLED_CrashDuringRecovery) {
|
|||||||
CloseDb();
|
CloseDb();
|
||||||
}
|
}
|
||||||
|
|
||||||
// 2. Corrupt second last un-syned wal file to emulate power reset which
|
// 2. Corrupt second last wal file to emulate power reset which caused the DB
|
||||||
// caused the DB to lose the un-synced WAL.
|
// to lose the un-synced WAL.
|
||||||
{
|
{
|
||||||
std::vector<uint64_t> file_nums;
|
std::vector<uint64_t> file_nums;
|
||||||
GetSortedWalFiles(file_nums);
|
GetSortedWalFiles(file_nums);
|
||||||
size_t size = file_nums.size();
|
size_t size = file_nums.size();
|
||||||
assert(size >= 2);
|
|
||||||
uint64_t log_num = file_nums[size - 2];
|
uint64_t log_num = file_nums[size - 2];
|
||||||
CorruptFileWithTruncation(FileType::kWalFile, log_num,
|
CorruptFileWithTruncation(FileType::kWalFile, log_num,
|
||||||
/*bytes_to_truncate=*/8);
|
/*bytes_to_truncate=*/8);
|
||||||
@ -1141,31 +1131,25 @@ TEST_P(CrashDuringRecoveryWithCorruptionTest, DISABLED_CrashDuringRecovery) {
|
|||||||
// Case2: If avoid_flush_during_recovery = false, all column families have
|
// Case2: If avoid_flush_during_recovery = false, all column families have
|
||||||
// flushed their data from WAL to L0 during recovery, and none of them will
|
// flushed their data from WAL to L0 during recovery, and none of them will
|
||||||
// ever need to read the WALs again.
|
// ever need to read the WALs again.
|
||||||
|
|
||||||
// 4. Fault is injected to fail the recovery.
|
|
||||||
{
|
{
|
||||||
SyncPoint::GetInstance()->DisableProcessing();
|
options.avoid_flush_during_recovery = avoid_flush_during_recovery_;
|
||||||
SyncPoint::GetInstance()->ClearAllCallBacks();
|
|
||||||
SyncPoint::GetInstance()->SetCallBack(
|
|
||||||
"DBImpl::GetLogSizeAndMaybeTruncate:0", [&](void* arg) {
|
|
||||||
auto* tmp_s = reinterpret_cast<Status*>(arg);
|
|
||||||
assert(tmp_s);
|
|
||||||
*tmp_s = Status::IOError("Injected");
|
|
||||||
});
|
|
||||||
SyncPoint::GetInstance()->EnableProcessing();
|
|
||||||
|
|
||||||
handles.clear();
|
|
||||||
options.avoid_flush_during_recovery = true;
|
|
||||||
s = DB::Open(options, dbname_, cf_descs, &handles, &db_);
|
s = DB::Open(options, dbname_, cf_descs, &handles, &db_);
|
||||||
ASSERT_TRUE(s.IsIOError());
|
ASSERT_OK(s);
|
||||||
ASSERT_EQ("IO error: Injected", s.ToString());
|
|
||||||
for (auto* h : handles) {
|
for (auto* h : handles) {
|
||||||
delete h;
|
delete h;
|
||||||
}
|
}
|
||||||
|
handles.clear();
|
||||||
CloseDb();
|
CloseDb();
|
||||||
|
}
|
||||||
|
|
||||||
SyncPoint::GetInstance()->DisableProcessing();
|
// 4. Corrupt max_wal_num to emulate second power reset which caused the
|
||||||
SyncPoint::GetInstance()->ClearAllCallBacks();
|
// DB to again lose the un-synced WAL.
|
||||||
|
{
|
||||||
|
std::vector<uint64_t> file_nums;
|
||||||
|
GetSortedWalFiles(file_nums);
|
||||||
|
size_t size = file_nums.size();
|
||||||
|
uint64_t log_num = file_nums[size - 1];
|
||||||
|
CorruptFileWithTruncation(FileType::kWalFile, log_num);
|
||||||
}
|
}
|
||||||
|
|
||||||
// 5. After second crash reopen the db with second corruption. Default family
|
// 5. After second crash reopen the db with second corruption. Default family
|
||||||
@ -1203,24 +1187,16 @@ TEST_P(CrashDuringRecoveryWithCorruptionTest, DISABLED_CrashDuringRecovery) {
|
|||||||
// file contains data beyond the point of corruption" error will be hit, causing
|
// file contains data beyond the point of corruption" error will be hit, causing
|
||||||
// recovery to fail.
|
// recovery to fail.
|
||||||
//
|
//
|
||||||
// After adding the fix, only after new WAL is synced, RocksDB persist a new
|
// After adding the fix, corrupted WALs whose numbers are larger than the
|
||||||
// MANIFEST with column families to ensure RocksDB is in consistent state.
|
// corrupted wal and smaller than the new WAL are moved to a separate folder.
|
||||||
|
// Only after new WAL is synced, RocksDB persist a new MANIFEST with column
|
||||||
|
// families to ensure RocksDB is in consistent state.
|
||||||
// RocksDB writes an empty WriteBatch as a sentinel to the new WAL which is
|
// RocksDB writes an empty WriteBatch as a sentinel to the new WAL which is
|
||||||
// synced immediately afterwards. The sequence number of the sentinel
|
// synced immediately afterwards. The sequence number of the sentinel
|
||||||
// WriteBatch will be the next sequence number immediately after the largest
|
// WriteBatch will be the next sequence number immediately after the largest
|
||||||
// sequence number recovered from previous WALs and MANIFEST because of which DB
|
// sequence number recovered from previous WALs and MANIFEST because of which DB
|
||||||
// will be in consistent state.
|
// will be in consistent state.
|
||||||
// If a future recovery starts from the new MANIFEST, then it means the new WAL
|
TEST_P(CrashDuringRecoveryWithCorruptionTest, TxnDbCrashDuringRecovery) {
|
||||||
// is successfully synced. Due to the sentinel empty write batch at the
|
|
||||||
// beginning, kPointInTimeRecovery of WAL is guaranteed to go after this point.
|
|
||||||
// If future recovery starts from the old MANIFEST, it means the writing the new
|
|
||||||
// MANIFEST failed. It won't have the "SST ahead of WAL" error.
|
|
||||||
//
|
|
||||||
// The combination of corrupting a WAL and injecting an error during subsequent
|
|
||||||
// re-open exposes the bug of prematurely persisting a new MANIFEST with
|
|
||||||
// advanced ColumnFamilyData::log_number.
|
|
||||||
TEST_P(CrashDuringRecoveryWithCorruptionTest,
|
|
||||||
DISABLED_TxnDbCrashDuringRecovery) {
|
|
||||||
CloseDb();
|
CloseDb();
|
||||||
Options options;
|
Options options;
|
||||||
options.wal_recovery_mode = WALRecoveryMode::kPointInTimeRecovery;
|
options.wal_recovery_mode = WALRecoveryMode::kPointInTimeRecovery;
|
||||||
@ -1253,6 +1229,7 @@ TEST_P(CrashDuringRecoveryWithCorruptionTest,
|
|||||||
// advance wal number so that some column families have advanced log_number
|
// advance wal number so that some column families have advanced log_number
|
||||||
// while other don't.
|
// while other don't.
|
||||||
{
|
{
|
||||||
|
options.avoid_flush_during_recovery = avoid_flush_during_recovery_;
|
||||||
ASSERT_OK(TransactionDB::Open(options, txn_db_opts, dbname_, cf_descs,
|
ASSERT_OK(TransactionDB::Open(options, txn_db_opts, dbname_, cf_descs,
|
||||||
&handles, &txn_db));
|
&handles, &txn_db));
|
||||||
|
|
||||||
@ -1261,8 +1238,6 @@ TEST_P(CrashDuringRecoveryWithCorruptionTest,
|
|||||||
ASSERT_OK(txn->Put(handles[1], "foo", "value"));
|
ASSERT_OK(txn->Put(handles[1], "foo", "value"));
|
||||||
ASSERT_OK(txn->SetName("txn0"));
|
ASSERT_OK(txn->SetName("txn0"));
|
||||||
ASSERT_OK(txn->Prepare());
|
ASSERT_OK(txn->Prepare());
|
||||||
ASSERT_OK(txn_db->Flush(FlushOptions()));
|
|
||||||
|
|
||||||
delete txn;
|
delete txn;
|
||||||
txn = nullptr;
|
txn = nullptr;
|
||||||
|
|
||||||
@ -1282,7 +1257,6 @@ TEST_P(CrashDuringRecoveryWithCorruptionTest,
|
|||||||
|
|
||||||
delete txn;
|
delete txn;
|
||||||
txn = nullptr;
|
txn = nullptr;
|
||||||
|
|
||||||
for (auto* h : handles) {
|
for (auto* h : handles) {
|
||||||
delete h;
|
delete h;
|
||||||
}
|
}
|
||||||
@ -1296,7 +1270,6 @@ TEST_P(CrashDuringRecoveryWithCorruptionTest,
|
|||||||
std::vector<uint64_t> file_nums;
|
std::vector<uint64_t> file_nums;
|
||||||
GetSortedWalFiles(file_nums);
|
GetSortedWalFiles(file_nums);
|
||||||
size_t size = file_nums.size();
|
size_t size = file_nums.size();
|
||||||
assert(size >= 2);
|
|
||||||
uint64_t log_num = file_nums[size - 2];
|
uint64_t log_num = file_nums[size - 2];
|
||||||
CorruptFileWithTruncation(FileType::kWalFile, log_num,
|
CorruptFileWithTruncation(FileType::kWalFile, log_num,
|
||||||
/*bytes_to_truncate=*/8);
|
/*bytes_to_truncate=*/8);
|
||||||
@ -1306,38 +1279,24 @@ TEST_P(CrashDuringRecoveryWithCorruptionTest,
|
|||||||
// family has higher log number than corrupted wal number. There may be old
|
// family has higher log number than corrupted wal number. There may be old
|
||||||
// WAL files that it must not delete because they can contain data of
|
// WAL files that it must not delete because they can contain data of
|
||||||
// uncommitted transactions. As a result, min_log_number_to_keep won't change.
|
// uncommitted transactions. As a result, min_log_number_to_keep won't change.
|
||||||
|
|
||||||
{
|
{
|
||||||
SyncPoint::GetInstance()->DisableProcessing();
|
options.avoid_flush_during_recovery = avoid_flush_during_recovery_;
|
||||||
SyncPoint::GetInstance()->ClearAllCallBacks();
|
ASSERT_OK(TransactionDB::Open(options, txn_db_opts, dbname_, cf_descs,
|
||||||
SyncPoint::GetInstance()->SetCallBack(
|
&handles, &txn_db));
|
||||||
"DBImpl::Open::BeforeSyncWAL", [&](void* arg) {
|
|
||||||
auto* tmp_s = reinterpret_cast<Status*>(arg);
|
|
||||||
assert(tmp_s);
|
|
||||||
*tmp_s = Status::IOError("Injected");
|
|
||||||
});
|
|
||||||
SyncPoint::GetInstance()->EnableProcessing();
|
|
||||||
|
|
||||||
handles.clear();
|
|
||||||
s = TransactionDB::Open(options, txn_db_opts, dbname_, cf_descs, &handles,
|
|
||||||
&txn_db);
|
|
||||||
ASSERT_TRUE(s.IsIOError());
|
|
||||||
ASSERT_EQ("IO error: Injected", s.ToString());
|
|
||||||
for (auto* h : handles) {
|
for (auto* h : handles) {
|
||||||
delete h;
|
delete h;
|
||||||
}
|
}
|
||||||
CloseDb();
|
handles.clear();
|
||||||
|
delete txn_db;
|
||||||
SyncPoint::GetInstance()->DisableProcessing();
|
|
||||||
SyncPoint::GetInstance()->ClearAllCallBacks();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// 4. Corrupt max_wal_num.
|
// 4. Corrupt max_wal_num to emulate second power reset which caused the
|
||||||
|
// DB to again lose the un-synced WAL.
|
||||||
{
|
{
|
||||||
std::vector<uint64_t> file_nums;
|
std::vector<uint64_t> file_nums;
|
||||||
GetSortedWalFiles(file_nums);
|
GetSortedWalFiles(file_nums);
|
||||||
size_t size = file_nums.size();
|
size_t size = file_nums.size();
|
||||||
assert(size >= 2);
|
|
||||||
uint64_t log_num = file_nums[size - 1];
|
uint64_t log_num = file_nums[size - 1];
|
||||||
CorruptFileWithTruncation(FileType::kWalFile, log_num);
|
CorruptFileWithTruncation(FileType::kWalFile, log_num);
|
||||||
}
|
}
|
||||||
@ -1352,6 +1311,8 @@ TEST_P(CrashDuringRecoveryWithCorruptionTest,
|
|||||||
// recovered from previous WALs and MANIFEST, db will be in consistent state
|
// recovered from previous WALs and MANIFEST, db will be in consistent state
|
||||||
// and opens successfully.
|
// and opens successfully.
|
||||||
{
|
{
|
||||||
|
options.avoid_flush_during_recovery = false;
|
||||||
|
|
||||||
ASSERT_OK(TransactionDB::Open(options, txn_db_opts, dbname_, cf_descs,
|
ASSERT_OK(TransactionDB::Open(options, txn_db_opts, dbname_, cf_descs,
|
||||||
&handles, &txn_db));
|
&handles, &txn_db));
|
||||||
for (auto* h : handles) {
|
for (auto* h : handles) {
|
||||||
@ -1361,134 +1322,6 @@ TEST_P(CrashDuringRecoveryWithCorruptionTest,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// This test is similar to
|
|
||||||
// CrashDuringRecoveryWithCorruptionTest.CrashDuringRecovery except it calls
|
|
||||||
// flush and corrupts Last WAL. It calls flush to sync some of the WALs and
|
|
||||||
// remaining are unsyned one of which is then corrupted to simulate crash.
|
|
||||||
//
|
|
||||||
// In case of non-TransactionDB with avoid_flush_during_recovery = true, RocksDB
|
|
||||||
// won't flush the data from WAL to L0 for all column families if possible. As a
|
|
||||||
// result, not all column families can increase their log_numbers, and
|
|
||||||
// min_log_number_to_keep won't change.
|
|
||||||
// It may prematurely persist a new MANIFEST even before we can declare the DB
|
|
||||||
// is in consistent state after recovery (this is when the new WAL is synced)
|
|
||||||
// and advances log_numbers for some column families.
|
|
||||||
//
|
|
||||||
// If there is power failure before we sync the new WAL, we will end up in
|
|
||||||
// a situation in which after persisting the MANIFEST, RocksDB will see some
|
|
||||||
// column families' log_numbers larger than the corrupted wal, and
|
|
||||||
// "Column family inconsistency: SST file contains data beyond the point of
|
|
||||||
// corruption" error will be hit, causing recovery to fail.
|
|
||||||
//
|
|
||||||
// After adding the fix, only after new WAL is synced, RocksDB persist a new
|
|
||||||
// MANIFEST with column families to ensure RocksDB is in consistent state.
|
|
||||||
// RocksDB writes an empty WriteBatch as a sentinel to the new WAL which is
|
|
||||||
// synced immediately afterwards. The sequence number of the sentinel
|
|
||||||
// WriteBatch will be the next sequence number immediately after the largest
|
|
||||||
// sequence number recovered from previous WALs and MANIFEST because of which DB
|
|
||||||
// will be in consistent state.
|
|
||||||
// If a future recovery starts from the new MANIFEST, then it means the new WAL
|
|
||||||
// is successfully synced. Due to the sentinel empty write batch at the
|
|
||||||
// beginning, kPointInTimeRecovery of WAL is guaranteed to go after this point.
|
|
||||||
// If future recovery starts from the old MANIFEST, it means the writing the new
|
|
||||||
// MANIFEST failed. It won't have the "SST ahead of WAL" error.
|
|
||||||
|
|
||||||
// The combination of corrupting a WAL and injecting an error during subsequent
|
|
||||||
// re-open exposes the bug of prematurely persisting a new MANIFEST with
|
|
||||||
// advanced ColumnFamilyData::log_number.
|
|
||||||
TEST_P(CrashDuringRecoveryWithCorruptionTest,
|
|
||||||
DISABLED_CrashDuringRecoveryWithFlush) {
|
|
||||||
CloseDb();
|
|
||||||
Options options;
|
|
||||||
options.wal_recovery_mode = WALRecoveryMode::kPointInTimeRecovery;
|
|
||||||
options.avoid_flush_during_recovery = false;
|
|
||||||
options.env = env_;
|
|
||||||
options.create_if_missing = true;
|
|
||||||
|
|
||||||
ASSERT_OK(DestroyDB(dbname_, options));
|
|
||||||
Reopen(&options);
|
|
||||||
|
|
||||||
ColumnFamilyHandle* cfh = nullptr;
|
|
||||||
const std::string test_cf_name = "test_cf";
|
|
||||||
Status s = db_->CreateColumnFamily(options, test_cf_name, &cfh);
|
|
||||||
ASSERT_OK(s);
|
|
||||||
delete cfh;
|
|
||||||
|
|
||||||
CloseDb();
|
|
||||||
|
|
||||||
std::vector<ColumnFamilyDescriptor> cf_descs;
|
|
||||||
cf_descs.emplace_back(kDefaultColumnFamilyName, options);
|
|
||||||
cf_descs.emplace_back(test_cf_name, options);
|
|
||||||
std::vector<ColumnFamilyHandle*> handles;
|
|
||||||
|
|
||||||
{
|
|
||||||
ASSERT_OK(DB::Open(options, dbname_, cf_descs, &handles, &db_));
|
|
||||||
|
|
||||||
// Write one key to test_cf.
|
|
||||||
ASSERT_OK(db_->Put(WriteOptions(), handles[1], "old_key", "dontcare"));
|
|
||||||
|
|
||||||
// Write to default_cf and flush this cf several times to advance wal
|
|
||||||
// number.
|
|
||||||
for (int i = 0; i < 2; ++i) {
|
|
||||||
ASSERT_OK(db_->Put(WriteOptions(), "key" + std::to_string(i), "value"));
|
|
||||||
ASSERT_OK(db_->Flush(FlushOptions()));
|
|
||||||
}
|
|
||||||
|
|
||||||
ASSERT_OK(db_->Put(WriteOptions(), handles[1], "dontcare", "dontcare"));
|
|
||||||
for (auto* h : handles) {
|
|
||||||
delete h;
|
|
||||||
}
|
|
||||||
handles.clear();
|
|
||||||
CloseDb();
|
|
||||||
}
|
|
||||||
|
|
||||||
// Corrupt second last un-syned wal file to emulate power reset which
|
|
||||||
// caused the DB to lose the un-synced WAL.
|
|
||||||
{
|
|
||||||
std::vector<uint64_t> file_nums;
|
|
||||||
GetSortedWalFiles(file_nums);
|
|
||||||
size_t size = file_nums.size();
|
|
||||||
uint64_t log_num = file_nums[size - 1];
|
|
||||||
CorruptFileWithTruncation(FileType::kWalFile, log_num,
|
|
||||||
/*bytes_to_truncate=*/8);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Fault is injected to fail the recovery.
|
|
||||||
{
|
|
||||||
SyncPoint::GetInstance()->DisableProcessing();
|
|
||||||
SyncPoint::GetInstance()->ClearAllCallBacks();
|
|
||||||
SyncPoint::GetInstance()->SetCallBack(
|
|
||||||
"DBImpl::GetLogSizeAndMaybeTruncate:0", [&](void* arg) {
|
|
||||||
auto* tmp_s = reinterpret_cast<Status*>(arg);
|
|
||||||
assert(tmp_s);
|
|
||||||
*tmp_s = Status::IOError("Injected");
|
|
||||||
});
|
|
||||||
SyncPoint::GetInstance()->EnableProcessing();
|
|
||||||
|
|
||||||
handles.clear();
|
|
||||||
options.avoid_flush_during_recovery = true;
|
|
||||||
s = DB::Open(options, dbname_, cf_descs, &handles, &db_);
|
|
||||||
ASSERT_TRUE(s.IsIOError());
|
|
||||||
ASSERT_EQ("IO error: Injected", s.ToString());
|
|
||||||
for (auto* h : handles) {
|
|
||||||
delete h;
|
|
||||||
}
|
|
||||||
CloseDb();
|
|
||||||
|
|
||||||
SyncPoint::GetInstance()->DisableProcessing();
|
|
||||||
SyncPoint::GetInstance()->ClearAllCallBacks();
|
|
||||||
}
|
|
||||||
|
|
||||||
// Reopen db again
|
|
||||||
{
|
|
||||||
options.avoid_flush_during_recovery = avoid_flush_during_recovery_;
|
|
||||||
ASSERT_OK(DB::Open(options, dbname_, cf_descs, &handles, &db_));
|
|
||||||
for (auto* h : handles) {
|
|
||||||
delete h;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
} // namespace ROCKSDB_NAMESPACE
|
} // namespace ROCKSDB_NAMESPACE
|
||||||
|
|
||||||
int main(int argc, char** argv) {
|
int main(int argc, char** argv) {
|
||||||
|
@ -95,8 +95,8 @@ class CuckooTableDBTest : public testing::Test {
|
|||||||
|
|
||||||
int NumTableFilesAtLevel(int level) {
|
int NumTableFilesAtLevel(int level) {
|
||||||
std::string property;
|
std::string property;
|
||||||
EXPECT_TRUE(db_->GetProperty(
|
EXPECT_TRUE(db_->GetProperty("rocksdb.num-files-at-level" + ToString(level),
|
||||||
"rocksdb.num-files-at-level" + std::to_string(level), &property));
|
&property));
|
||||||
return atoi(property.c_str());
|
return atoi(property.c_str());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -3783,7 +3783,7 @@ TEST_P(DBBasicTestDeadline, PointLookupDeadline) {
|
|||||||
|
|
||||||
Random rnd(301);
|
Random rnd(301);
|
||||||
for (int i = 0; i < 400; ++i) {
|
for (int i = 0; i < 400; ++i) {
|
||||||
std::string key = "k" + std::to_string(i);
|
std::string key = "k" + ToString(i);
|
||||||
ASSERT_OK(Put(key, rnd.RandomString(100)));
|
ASSERT_OK(Put(key, rnd.RandomString(100)));
|
||||||
}
|
}
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
@ -3866,7 +3866,7 @@ TEST_P(DBBasicTestDeadline, IteratorDeadline) {
|
|||||||
|
|
||||||
Random rnd(301);
|
Random rnd(301);
|
||||||
for (int i = 0; i < 400; ++i) {
|
for (int i = 0; i < 400; ++i) {
|
||||||
std::string key = "k" + std::to_string(i);
|
std::string key = "k" + ToString(i);
|
||||||
ASSERT_OK(Put(key, rnd.RandomString(100)));
|
ASSERT_OK(Put(key, rnd.RandomString(100)));
|
||||||
}
|
}
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
|
@ -13,7 +13,6 @@
|
|||||||
|
|
||||||
#include "cache/cache_entry_roles.h"
|
#include "cache/cache_entry_roles.h"
|
||||||
#include "cache/cache_key.h"
|
#include "cache/cache_key.h"
|
||||||
#include "cache/fast_lru_cache.h"
|
|
||||||
#include "cache/lru_cache.h"
|
#include "cache/lru_cache.h"
|
||||||
#include "db/column_family.h"
|
#include "db/column_family.h"
|
||||||
#include "db/db_impl/db_impl.h"
|
#include "db/db_impl/db_impl.h"
|
||||||
@ -76,7 +75,7 @@ class DBBlockCacheTest : public DBTestBase {
|
|||||||
void InitTable(const Options& /*options*/) {
|
void InitTable(const Options& /*options*/) {
|
||||||
std::string value(kValueSize, 'a');
|
std::string value(kValueSize, 'a');
|
||||||
for (size_t i = 0; i < kNumBlocks; i++) {
|
for (size_t i = 0; i < kNumBlocks; i++) {
|
||||||
ASSERT_OK(Put(std::to_string(i), value.c_str()));
|
ASSERT_OK(Put(ToString(i), value.c_str()));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -205,7 +204,7 @@ TEST_F(DBBlockCacheTest, IteratorBlockCacheUsage) {
|
|||||||
|
|
||||||
ASSERT_EQ(0, cache->GetUsage());
|
ASSERT_EQ(0, cache->GetUsage());
|
||||||
iter = db_->NewIterator(read_options);
|
iter = db_->NewIterator(read_options);
|
||||||
iter->Seek(std::to_string(0));
|
iter->Seek(ToString(0));
|
||||||
ASSERT_LT(0, cache->GetUsage());
|
ASSERT_LT(0, cache->GetUsage());
|
||||||
delete iter;
|
delete iter;
|
||||||
iter = nullptr;
|
iter = nullptr;
|
||||||
@ -236,7 +235,7 @@ TEST_F(DBBlockCacheTest, TestWithoutCompressedBlockCache) {
|
|||||||
// Load blocks into cache.
|
// Load blocks into cache.
|
||||||
for (size_t i = 0; i + 1 < kNumBlocks; i++) {
|
for (size_t i = 0; i + 1 < kNumBlocks; i++) {
|
||||||
iter = db_->NewIterator(read_options);
|
iter = db_->NewIterator(read_options);
|
||||||
iter->Seek(std::to_string(i));
|
iter->Seek(ToString(i));
|
||||||
ASSERT_OK(iter->status());
|
ASSERT_OK(iter->status());
|
||||||
CheckCacheCounters(options, 1, 0, 1, 0);
|
CheckCacheCounters(options, 1, 0, 1, 0);
|
||||||
iterators[i].reset(iter);
|
iterators[i].reset(iter);
|
||||||
@ -249,7 +248,7 @@ TEST_F(DBBlockCacheTest, TestWithoutCompressedBlockCache) {
|
|||||||
// Test with strict capacity limit.
|
// Test with strict capacity limit.
|
||||||
cache->SetStrictCapacityLimit(true);
|
cache->SetStrictCapacityLimit(true);
|
||||||
iter = db_->NewIterator(read_options);
|
iter = db_->NewIterator(read_options);
|
||||||
iter->Seek(std::to_string(kNumBlocks - 1));
|
iter->Seek(ToString(kNumBlocks - 1));
|
||||||
ASSERT_TRUE(iter->status().IsIncomplete());
|
ASSERT_TRUE(iter->status().IsIncomplete());
|
||||||
CheckCacheCounters(options, 1, 0, 0, 1);
|
CheckCacheCounters(options, 1, 0, 0, 1);
|
||||||
delete iter;
|
delete iter;
|
||||||
@ -263,7 +262,7 @@ TEST_F(DBBlockCacheTest, TestWithoutCompressedBlockCache) {
|
|||||||
ASSERT_EQ(0, cache->GetPinnedUsage());
|
ASSERT_EQ(0, cache->GetPinnedUsage());
|
||||||
for (size_t i = 0; i + 1 < kNumBlocks; i++) {
|
for (size_t i = 0; i + 1 < kNumBlocks; i++) {
|
||||||
iter = db_->NewIterator(read_options);
|
iter = db_->NewIterator(read_options);
|
||||||
iter->Seek(std::to_string(i));
|
iter->Seek(ToString(i));
|
||||||
ASSERT_OK(iter->status());
|
ASSERT_OK(iter->status());
|
||||||
CheckCacheCounters(options, 0, 1, 0, 0);
|
CheckCacheCounters(options, 0, 1, 0, 0);
|
||||||
iterators[i].reset(iter);
|
iterators[i].reset(iter);
|
||||||
@ -289,7 +288,7 @@ TEST_F(DBBlockCacheTest, TestWithCompressedBlockCache) {
|
|||||||
|
|
||||||
std::string value(kValueSize, 'a');
|
std::string value(kValueSize, 'a');
|
||||||
for (size_t i = 0; i < kNumBlocks; i++) {
|
for (size_t i = 0; i < kNumBlocks; i++) {
|
||||||
ASSERT_OK(Put(std::to_string(i), value));
|
ASSERT_OK(Put(ToString(i), value));
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -313,7 +312,7 @@ TEST_F(DBBlockCacheTest, TestWithCompressedBlockCache) {
|
|||||||
|
|
||||||
// Load blocks into cache.
|
// Load blocks into cache.
|
||||||
for (size_t i = 0; i < kNumBlocks - 1; i++) {
|
for (size_t i = 0; i < kNumBlocks - 1; i++) {
|
||||||
ASSERT_EQ(value, Get(std::to_string(i)));
|
ASSERT_EQ(value, Get(ToString(i)));
|
||||||
CheckCacheCounters(options, 1, 0, 1, 0);
|
CheckCacheCounters(options, 1, 0, 1, 0);
|
||||||
CheckCompressedCacheCounters(options, 1, 0, 1, 0);
|
CheckCompressedCacheCounters(options, 1, 0, 1, 0);
|
||||||
}
|
}
|
||||||
@ -334,7 +333,7 @@ TEST_F(DBBlockCacheTest, TestWithCompressedBlockCache) {
|
|||||||
|
|
||||||
// Load last key block.
|
// Load last key block.
|
||||||
ASSERT_EQ("Result incomplete: Insert failed due to LRU cache being full.",
|
ASSERT_EQ("Result incomplete: Insert failed due to LRU cache being full.",
|
||||||
Get(std::to_string(kNumBlocks - 1)));
|
Get(ToString(kNumBlocks - 1)));
|
||||||
// Failure will also record the miss counter.
|
// Failure will also record the miss counter.
|
||||||
CheckCacheCounters(options, 1, 0, 0, 1);
|
CheckCacheCounters(options, 1, 0, 0, 1);
|
||||||
CheckCompressedCacheCounters(options, 1, 0, 1, 0);
|
CheckCompressedCacheCounters(options, 1, 0, 1, 0);
|
||||||
@ -343,7 +342,7 @@ TEST_F(DBBlockCacheTest, TestWithCompressedBlockCache) {
|
|||||||
// cache and load into block cache.
|
// cache and load into block cache.
|
||||||
cache->SetStrictCapacityLimit(false);
|
cache->SetStrictCapacityLimit(false);
|
||||||
// Load last key block.
|
// Load last key block.
|
||||||
ASSERT_EQ(value, Get(std::to_string(kNumBlocks - 1)));
|
ASSERT_EQ(value, Get(ToString(kNumBlocks - 1)));
|
||||||
CheckCacheCounters(options, 1, 0, 1, 0);
|
CheckCacheCounters(options, 1, 0, 1, 0);
|
||||||
CheckCompressedCacheCounters(options, 0, 1, 0, 0);
|
CheckCompressedCacheCounters(options, 0, 1, 0, 0);
|
||||||
}
|
}
|
||||||
@ -568,7 +567,7 @@ TEST_F(DBBlockCacheTest, FillCacheAndIterateDB) {
|
|||||||
Iterator* iter = nullptr;
|
Iterator* iter = nullptr;
|
||||||
|
|
||||||
iter = db_->NewIterator(read_options);
|
iter = db_->NewIterator(read_options);
|
||||||
iter->Seek(std::to_string(0));
|
iter->Seek(ToString(0));
|
||||||
while (iter->Valid()) {
|
while (iter->Valid()) {
|
||||||
iter->Next();
|
iter->Next();
|
||||||
}
|
}
|
||||||
@ -646,10 +645,10 @@ TEST_F(DBBlockCacheTest, WarmCacheWithDataBlocksDuringFlush) {
|
|||||||
|
|
||||||
std::string value(kValueSize, 'a');
|
std::string value(kValueSize, 'a');
|
||||||
for (size_t i = 1; i <= kNumBlocks; i++) {
|
for (size_t i = 1; i <= kNumBlocks; i++) {
|
||||||
ASSERT_OK(Put(std::to_string(i), value));
|
ASSERT_OK(Put(ToString(i), value));
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
ASSERT_EQ(i, options.statistics->getTickerCount(BLOCK_CACHE_DATA_ADD));
|
ASSERT_EQ(i, options.statistics->getTickerCount(BLOCK_CACHE_DATA_ADD));
|
||||||
ASSERT_EQ(value, Get(std::to_string(i)));
|
ASSERT_EQ(value, Get(ToString(i)));
|
||||||
ASSERT_EQ(0, options.statistics->getTickerCount(BLOCK_CACHE_DATA_MISS));
|
ASSERT_EQ(0, options.statistics->getTickerCount(BLOCK_CACHE_DATA_MISS));
|
||||||
ASSERT_EQ(i, options.statistics->getTickerCount(BLOCK_CACHE_DATA_HIT));
|
ASSERT_EQ(i, options.statistics->getTickerCount(BLOCK_CACHE_DATA_HIT));
|
||||||
}
|
}
|
||||||
@ -706,7 +705,7 @@ TEST_P(DBBlockCacheTest1, WarmCacheWithBlocksDuringFlush) {
|
|||||||
|
|
||||||
std::string value(kValueSize, 'a');
|
std::string value(kValueSize, 'a');
|
||||||
for (size_t i = 1; i <= kNumBlocks; i++) {
|
for (size_t i = 1; i <= kNumBlocks; i++) {
|
||||||
ASSERT_OK(Put(std::to_string(i), value));
|
ASSERT_OK(Put(ToString(i), value));
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
ASSERT_EQ(i, options.statistics->getTickerCount(BLOCK_CACHE_DATA_ADD));
|
ASSERT_EQ(i, options.statistics->getTickerCount(BLOCK_CACHE_DATA_ADD));
|
||||||
if (filter_type == 1) {
|
if (filter_type == 1) {
|
||||||
@ -718,7 +717,7 @@ TEST_P(DBBlockCacheTest1, WarmCacheWithBlocksDuringFlush) {
|
|||||||
ASSERT_EQ(i, options.statistics->getTickerCount(BLOCK_CACHE_INDEX_ADD));
|
ASSERT_EQ(i, options.statistics->getTickerCount(BLOCK_CACHE_INDEX_ADD));
|
||||||
ASSERT_EQ(i, options.statistics->getTickerCount(BLOCK_CACHE_FILTER_ADD));
|
ASSERT_EQ(i, options.statistics->getTickerCount(BLOCK_CACHE_FILTER_ADD));
|
||||||
}
|
}
|
||||||
ASSERT_EQ(value, Get(std::to_string(i)));
|
ASSERT_EQ(value, Get(ToString(i)));
|
||||||
|
|
||||||
ASSERT_EQ(0, options.statistics->getTickerCount(BLOCK_CACHE_DATA_MISS));
|
ASSERT_EQ(0, options.statistics->getTickerCount(BLOCK_CACHE_DATA_MISS));
|
||||||
ASSERT_EQ(i, options.statistics->getTickerCount(BLOCK_CACHE_DATA_HIT));
|
ASSERT_EQ(i, options.statistics->getTickerCount(BLOCK_CACHE_DATA_HIT));
|
||||||
@ -773,12 +772,12 @@ TEST_F(DBBlockCacheTest, DynamicallyWarmCacheDuringFlush) {
|
|||||||
std::string value(kValueSize, 'a');
|
std::string value(kValueSize, 'a');
|
||||||
|
|
||||||
for (size_t i = 1; i <= 5; i++) {
|
for (size_t i = 1; i <= 5; i++) {
|
||||||
ASSERT_OK(Put(std::to_string(i), value));
|
ASSERT_OK(Put(ToString(i), value));
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
ASSERT_EQ(1,
|
ASSERT_EQ(1,
|
||||||
options.statistics->getAndResetTickerCount(BLOCK_CACHE_DATA_ADD));
|
options.statistics->getAndResetTickerCount(BLOCK_CACHE_DATA_ADD));
|
||||||
|
|
||||||
ASSERT_EQ(value, Get(std::to_string(i)));
|
ASSERT_EQ(value, Get(ToString(i)));
|
||||||
ASSERT_EQ(0,
|
ASSERT_EQ(0,
|
||||||
options.statistics->getAndResetTickerCount(BLOCK_CACHE_DATA_ADD));
|
options.statistics->getAndResetTickerCount(BLOCK_CACHE_DATA_ADD));
|
||||||
ASSERT_EQ(
|
ASSERT_EQ(
|
||||||
@ -791,12 +790,12 @@ TEST_F(DBBlockCacheTest, DynamicallyWarmCacheDuringFlush) {
|
|||||||
{{"block_based_table_factory", "{prepopulate_block_cache=kDisable;}"}}));
|
{{"block_based_table_factory", "{prepopulate_block_cache=kDisable;}"}}));
|
||||||
|
|
||||||
for (size_t i = 6; i <= kNumBlocks; i++) {
|
for (size_t i = 6; i <= kNumBlocks; i++) {
|
||||||
ASSERT_OK(Put(std::to_string(i), value));
|
ASSERT_OK(Put(ToString(i), value));
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
ASSERT_EQ(0,
|
ASSERT_EQ(0,
|
||||||
options.statistics->getAndResetTickerCount(BLOCK_CACHE_DATA_ADD));
|
options.statistics->getAndResetTickerCount(BLOCK_CACHE_DATA_ADD));
|
||||||
|
|
||||||
ASSERT_EQ(value, Get(std::to_string(i)));
|
ASSERT_EQ(value, Get(ToString(i)));
|
||||||
ASSERT_EQ(1,
|
ASSERT_EQ(1,
|
||||||
options.statistics->getAndResetTickerCount(BLOCK_CACHE_DATA_ADD));
|
options.statistics->getAndResetTickerCount(BLOCK_CACHE_DATA_ADD));
|
||||||
ASSERT_EQ(
|
ASSERT_EQ(
|
||||||
@ -935,8 +934,7 @@ TEST_F(DBBlockCacheTest, AddRedundantStats) {
|
|||||||
int iterations_tested = 0;
|
int iterations_tested = 0;
|
||||||
for (std::shared_ptr<Cache> base_cache :
|
for (std::shared_ptr<Cache> base_cache :
|
||||||
{NewLRUCache(capacity, num_shard_bits),
|
{NewLRUCache(capacity, num_shard_bits),
|
||||||
NewClockCache(capacity, num_shard_bits),
|
NewClockCache(capacity, num_shard_bits)}) {
|
||||||
NewFastLRUCache(capacity, num_shard_bits)}) {
|
|
||||||
if (!base_cache) {
|
if (!base_cache) {
|
||||||
// Skip clock cache when not supported
|
// Skip clock cache when not supported
|
||||||
continue;
|
continue;
|
||||||
@ -1290,8 +1288,7 @@ TEST_F(DBBlockCacheTest, CacheEntryRoleStats) {
|
|||||||
int iterations_tested = 0;
|
int iterations_tested = 0;
|
||||||
for (bool partition : {false, true}) {
|
for (bool partition : {false, true}) {
|
||||||
for (std::shared_ptr<Cache> cache :
|
for (std::shared_ptr<Cache> cache :
|
||||||
{NewLRUCache(capacity), NewClockCache(capacity),
|
{NewLRUCache(capacity), NewClockCache(capacity)}) {
|
||||||
NewFastLRUCache(capacity)}) {
|
|
||||||
if (!cache) {
|
if (!cache) {
|
||||||
// Skip clock cache when not supported
|
// Skip clock cache when not supported
|
||||||
continue;
|
continue;
|
||||||
@ -1409,7 +1406,7 @@ TEST_F(DBBlockCacheTest, CacheEntryRoleStats) {
|
|||||||
|
|
||||||
for (size_t i = 0; i < kNumCacheEntryRoles; ++i) {
|
for (size_t i = 0; i < kNumCacheEntryRoles; ++i) {
|
||||||
auto role = static_cast<CacheEntryRole>(i);
|
auto role = static_cast<CacheEntryRole>(i);
|
||||||
EXPECT_EQ(std::to_string(expected[i]),
|
EXPECT_EQ(ToString(expected[i]),
|
||||||
values[BlockCacheEntryStatsMapKeys::EntryCount(role)]);
|
values[BlockCacheEntryStatsMapKeys::EntryCount(role)]);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1422,7 +1419,7 @@ TEST_F(DBBlockCacheTest, CacheEntryRoleStats) {
|
|||||||
// re-scanning stats, but not totally aggressive.
|
// re-scanning stats, but not totally aggressive.
|
||||||
// Within some time window, we will get cached entry stats
|
// Within some time window, we will get cached entry stats
|
||||||
env_->MockSleepForSeconds(1);
|
env_->MockSleepForSeconds(1);
|
||||||
EXPECT_EQ(std::to_string(prev_expected[static_cast<size_t>(
|
EXPECT_EQ(ToString(prev_expected[static_cast<size_t>(
|
||||||
CacheEntryRole::kWriteBuffer)]),
|
CacheEntryRole::kWriteBuffer)]),
|
||||||
values[BlockCacheEntryStatsMapKeys::EntryCount(
|
values[BlockCacheEntryStatsMapKeys::EntryCount(
|
||||||
CacheEntryRole::kWriteBuffer)]);
|
CacheEntryRole::kWriteBuffer)]);
|
||||||
@ -1432,7 +1429,7 @@ TEST_F(DBBlockCacheTest, CacheEntryRoleStats) {
|
|||||||
ASSERT_TRUE(db_->GetMapProperty(DB::Properties::kBlockCacheEntryStats,
|
ASSERT_TRUE(db_->GetMapProperty(DB::Properties::kBlockCacheEntryStats,
|
||||||
&values));
|
&values));
|
||||||
EXPECT_EQ(
|
EXPECT_EQ(
|
||||||
std::to_string(
|
ToString(
|
||||||
expected[static_cast<size_t>(CacheEntryRole::kWriteBuffer)]),
|
expected[static_cast<size_t>(CacheEntryRole::kWriteBuffer)]),
|
||||||
values[BlockCacheEntryStatsMapKeys::EntryCount(
|
values[BlockCacheEntryStatsMapKeys::EntryCount(
|
||||||
CacheEntryRole::kWriteBuffer)]);
|
CacheEntryRole::kWriteBuffer)]);
|
||||||
@ -1640,7 +1637,7 @@ TEST_P(DBBlockCacheKeyTest, StableCacheKeys) {
|
|||||||
SstFileWriter sst_file_writer(EnvOptions(), options);
|
SstFileWriter sst_file_writer(EnvOptions(), options);
|
||||||
std::vector<std::string> external;
|
std::vector<std::string> external;
|
||||||
for (int i = 0; i < 2; ++i) {
|
for (int i = 0; i < 2; ++i) {
|
||||||
std::string f = dbname_ + "/external" + std::to_string(i) + ".sst";
|
std::string f = dbname_ + "/external" + ToString(i) + ".sst";
|
||||||
external.push_back(f);
|
external.push_back(f);
|
||||||
ASSERT_OK(sst_file_writer.Open(f));
|
ASSERT_OK(sst_file_writer.Open(f));
|
||||||
ASSERT_OK(sst_file_writer.Put(Key(key_count), "abc"));
|
ASSERT_OK(sst_file_writer.Put(Key(key_count), "abc"));
|
||||||
@ -1724,7 +1721,7 @@ class CacheKeyTest : public testing::Test {
|
|||||||
// Like SemiStructuredUniqueIdGen::GenerateNext
|
// Like SemiStructuredUniqueIdGen::GenerateNext
|
||||||
tp_.db_session_id = EncodeSessionId(base_session_upper_,
|
tp_.db_session_id = EncodeSessionId(base_session_upper_,
|
||||||
base_session_lower_ ^ session_counter_);
|
base_session_lower_ ^ session_counter_);
|
||||||
tp_.db_id = std::to_string(db_id_);
|
tp_.db_id = ToString(db_id_);
|
||||||
tp_.orig_file_number = file_number_;
|
tp_.orig_file_number = file_number_;
|
||||||
bool is_stable;
|
bool is_stable;
|
||||||
std::string cur_session_id = ""; // ignored
|
std::string cur_session_id = ""; // ignored
|
||||||
|
@ -858,7 +858,7 @@ TEST_F(DBBloomFilterTest, BloomFilterCompatibility) {
|
|||||||
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
|
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
|
||||||
Reopen(options);
|
Reopen(options);
|
||||||
|
|
||||||
std::string prefix = std::to_string(i) + "_";
|
std::string prefix = ToString(i) + "_";
|
||||||
ASSERT_OK(Put(prefix + "A", "val"));
|
ASSERT_OK(Put(prefix + "A", "val"));
|
||||||
ASSERT_OK(Put(prefix + "Z", "val"));
|
ASSERT_OK(Put(prefix + "Z", "val"));
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
@ -873,7 +873,7 @@ TEST_F(DBBloomFilterTest, BloomFilterCompatibility) {
|
|||||||
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
|
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
|
||||||
Reopen(options);
|
Reopen(options);
|
||||||
for (size_t j = 0; j < kCompatibilityConfigs.size(); ++j) {
|
for (size_t j = 0; j < kCompatibilityConfigs.size(); ++j) {
|
||||||
std::string prefix = std::to_string(j) + "_";
|
std::string prefix = ToString(j) + "_";
|
||||||
ASSERT_EQ("val", Get(prefix + "A")); // Filter positive
|
ASSERT_EQ("val", Get(prefix + "A")); // Filter positive
|
||||||
ASSERT_EQ("val", Get(prefix + "Z")); // Filter positive
|
ASSERT_EQ("val", Get(prefix + "Z")); // Filter positive
|
||||||
// Filter negative, with high probability
|
// Filter negative, with high probability
|
||||||
@ -1713,11 +1713,11 @@ class TestingContextCustomFilterPolicy
|
|||||||
test_report_ +=
|
test_report_ +=
|
||||||
OptionsHelper::compaction_style_to_string[context.compaction_style];
|
OptionsHelper::compaction_style_to_string[context.compaction_style];
|
||||||
test_report_ += ",n=";
|
test_report_ += ",n=";
|
||||||
test_report_ += std::to_string(context.num_levels);
|
test_report_ += ROCKSDB_NAMESPACE::ToString(context.num_levels);
|
||||||
test_report_ += ",l=";
|
test_report_ += ",l=";
|
||||||
test_report_ += std::to_string(context.level_at_creation);
|
test_report_ += ROCKSDB_NAMESPACE::ToString(context.level_at_creation);
|
||||||
test_report_ += ",b=";
|
test_report_ += ",b=";
|
||||||
test_report_ += std::to_string(int{context.is_bottommost});
|
test_report_ += ROCKSDB_NAMESPACE::ToString(int{context.is_bottommost});
|
||||||
test_report_ += ",r=";
|
test_report_ += ",r=";
|
||||||
test_report_ += table_file_creation_reason_to_string[context.reason];
|
test_report_ += table_file_creation_reason_to_string[context.reason];
|
||||||
test_report_ += "\n";
|
test_report_ += "\n";
|
||||||
|
@ -454,7 +454,7 @@ TEST_F(DBTestCompactionFilter, CompactionFilterDeletesAll) {
|
|||||||
// put some data
|
// put some data
|
||||||
for (int table = 0; table < 4; ++table) {
|
for (int table = 0; table < 4; ++table) {
|
||||||
for (int i = 0; i < 10 + table; ++i) {
|
for (int i = 0; i < 10 + table; ++i) {
|
||||||
ASSERT_OK(Put(std::to_string(table * 100 + i), "val"));
|
ASSERT_OK(Put(ToString(table * 100 + i), "val"));
|
||||||
}
|
}
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
}
|
}
|
||||||
@ -755,7 +755,7 @@ TEST_F(DBTestCompactionFilter, CompactionFilterContextCfId) {
|
|||||||
#ifndef ROCKSDB_LITE
|
#ifndef ROCKSDB_LITE
|
||||||
// Compaction filters aplies to all records, regardless snapshots.
|
// Compaction filters aplies to all records, regardless snapshots.
|
||||||
TEST_F(DBTestCompactionFilter, CompactionFilterIgnoreSnapshot) {
|
TEST_F(DBTestCompactionFilter, CompactionFilterIgnoreSnapshot) {
|
||||||
std::string five = std::to_string(5);
|
std::string five = ToString(5);
|
||||||
Options options = CurrentOptions();
|
Options options = CurrentOptions();
|
||||||
options.compaction_filter_factory = std::make_shared<DeleteISFilterFactory>();
|
options.compaction_filter_factory = std::make_shared<DeleteISFilterFactory>();
|
||||||
options.disable_auto_compactions = true;
|
options.disable_auto_compactions = true;
|
||||||
@ -766,7 +766,7 @@ TEST_F(DBTestCompactionFilter, CompactionFilterIgnoreSnapshot) {
|
|||||||
const Snapshot* snapshot = nullptr;
|
const Snapshot* snapshot = nullptr;
|
||||||
for (int table = 0; table < 4; ++table) {
|
for (int table = 0; table < 4; ++table) {
|
||||||
for (int i = 0; i < 10; ++i) {
|
for (int i = 0; i < 10; ++i) {
|
||||||
ASSERT_OK(Put(std::to_string(table * 100 + i), "val"));
|
ASSERT_OK(Put(ToString(table * 100 + i), "val"));
|
||||||
}
|
}
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
|
|
||||||
@ -998,7 +998,7 @@ TEST_F(DBTestCompactionFilter, DropKeyWithSingleDelete) {
|
|||||||
std::string* /*new_value*/,
|
std::string* /*new_value*/,
|
||||||
std::string* /*skip_until*/) const override {
|
std::string* /*skip_until*/) const override {
|
||||||
if (key.starts_with("b")) {
|
if (key.starts_with("b")) {
|
||||||
return Decision::kPurge;
|
return Decision::kRemoveWithSingleDelete;
|
||||||
}
|
}
|
||||||
return Decision::kRemove;
|
return Decision::kRemove;
|
||||||
}
|
}
|
||||||
|
@ -2817,7 +2817,7 @@ TEST_P(DBCompactionTestWithParam, DISABLED_CompactFilesOnLevelCompaction) {
|
|||||||
|
|
||||||
Random rnd(301);
|
Random rnd(301);
|
||||||
for (int key = 64 * kEntriesPerBuffer; key >= 0; --key) {
|
for (int key = 64 * kEntriesPerBuffer; key >= 0; --key) {
|
||||||
ASSERT_OK(Put(1, std::to_string(key), rnd.RandomString(kTestValueSize)));
|
ASSERT_OK(Put(1, ToString(key), rnd.RandomString(kTestValueSize)));
|
||||||
}
|
}
|
||||||
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[1]));
|
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[1]));
|
||||||
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
||||||
@ -2849,7 +2849,7 @@ TEST_P(DBCompactionTestWithParam, DISABLED_CompactFilesOnLevelCompaction) {
|
|||||||
|
|
||||||
// make sure all key-values are still there.
|
// make sure all key-values are still there.
|
||||||
for (int key = 64 * kEntriesPerBuffer; key >= 0; --key) {
|
for (int key = 64 * kEntriesPerBuffer; key >= 0; --key) {
|
||||||
ASSERT_NE(Get(1, std::to_string(key)), "NOT_FOUND");
|
ASSERT_NE(Get(1, ToString(key)), "NOT_FOUND");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -4404,8 +4404,7 @@ TEST_F(DBCompactionTest, LevelPeriodicCompactionWithCompactionFilters) {
|
|||||||
for (CompactionFilterType comp_filter_type :
|
for (CompactionFilterType comp_filter_type :
|
||||||
{kUseCompactionFilter, kUseCompactionFilterFactory}) {
|
{kUseCompactionFilter, kUseCompactionFilterFactory}) {
|
||||||
// Assert that periodic compactions are not enabled.
|
// Assert that periodic compactions are not enabled.
|
||||||
ASSERT_EQ(std::numeric_limits<uint64_t>::max() - 1,
|
ASSERT_EQ(port::kMaxUint64 - 1, options.periodic_compaction_seconds);
|
||||||
options.periodic_compaction_seconds);
|
|
||||||
|
|
||||||
if (comp_filter_type == kUseCompactionFilter) {
|
if (comp_filter_type == kUseCompactionFilter) {
|
||||||
options.compaction_filter = &test_compaction_filter;
|
options.compaction_filter = &test_compaction_filter;
|
||||||
@ -4668,9 +4667,9 @@ TEST_F(DBCompactionTest, CompactRangeSkipFlushAfterDelay) {
|
|||||||
});
|
});
|
||||||
|
|
||||||
TEST_SYNC_POINT("DBCompactionTest::CompactRangeSkipFlushAfterDelay:PreFlush");
|
TEST_SYNC_POINT("DBCompactionTest::CompactRangeSkipFlushAfterDelay:PreFlush");
|
||||||
ASSERT_OK(Put(std::to_string(0), rnd.RandomString(1024)));
|
ASSERT_OK(Put(ToString(0), rnd.RandomString(1024)));
|
||||||
ASSERT_OK(dbfull()->Flush(flush_opts));
|
ASSERT_OK(dbfull()->Flush(flush_opts));
|
||||||
ASSERT_OK(Put(std::to_string(0), rnd.RandomString(1024)));
|
ASSERT_OK(Put(ToString(0), rnd.RandomString(1024)));
|
||||||
TEST_SYNC_POINT("DBCompactionTest::CompactRangeSkipFlushAfterDelay:PostFlush");
|
TEST_SYNC_POINT("DBCompactionTest::CompactRangeSkipFlushAfterDelay:PostFlush");
|
||||||
manual_compaction_thread.join();
|
manual_compaction_thread.join();
|
||||||
|
|
||||||
@ -4679,7 +4678,7 @@ TEST_F(DBCompactionTest, CompactRangeSkipFlushAfterDelay) {
|
|||||||
std::string num_keys_in_memtable;
|
std::string num_keys_in_memtable;
|
||||||
ASSERT_TRUE(db_->GetProperty(DB::Properties::kNumEntriesActiveMemTable,
|
ASSERT_TRUE(db_->GetProperty(DB::Properties::kNumEntriesActiveMemTable,
|
||||||
&num_keys_in_memtable));
|
&num_keys_in_memtable));
|
||||||
ASSERT_EQ(std::to_string(1), num_keys_in_memtable);
|
ASSERT_EQ(ToString(1), num_keys_in_memtable);
|
||||||
|
|
||||||
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
|
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
|
||||||
}
|
}
|
||||||
@ -4828,7 +4827,7 @@ TEST_F(DBCompactionTest, SubcompactionEvent) {
|
|||||||
for (int i = 0; i < 4; i++) {
|
for (int i = 0; i < 4; i++) {
|
||||||
for (int j = 0; j < 10; j++) {
|
for (int j = 0; j < 10; j++) {
|
||||||
int key_id = i * 10 + j;
|
int key_id = i * 10 + j;
|
||||||
ASSERT_OK(Put(Key(key_id), "value" + std::to_string(key_id)));
|
ASSERT_OK(Put(Key(key_id), "value" + ToString(key_id)));
|
||||||
}
|
}
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
}
|
}
|
||||||
@ -4838,7 +4837,7 @@ TEST_F(DBCompactionTest, SubcompactionEvent) {
|
|||||||
for (int i = 0; i < 2; i++) {
|
for (int i = 0; i < 2; i++) {
|
||||||
for (int j = 0; j < 10; j++) {
|
for (int j = 0; j < 10; j++) {
|
||||||
int key_id = i * 20 + j * 2;
|
int key_id = i * 20 + j * 2;
|
||||||
ASSERT_OK(Put(Key(key_id), "value" + std::to_string(key_id)));
|
ASSERT_OK(Put(Key(key_id), "value" + ToString(key_id)));
|
||||||
}
|
}
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
}
|
}
|
||||||
@ -5830,7 +5829,7 @@ TEST_P(DBCompactionTestWithBottommostParam, SequenceKeysManualCompaction) {
|
|||||||
}
|
}
|
||||||
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
|
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
|
||||||
|
|
||||||
ASSERT_EQ(std::to_string(kSstNum), FilesPerLevel(0));
|
ASSERT_EQ(ToString(kSstNum), FilesPerLevel(0));
|
||||||
|
|
||||||
auto cro = CompactRangeOptions();
|
auto cro = CompactRangeOptions();
|
||||||
cro.bottommost_level_compaction = bottommost_level_compaction_;
|
cro.bottommost_level_compaction = bottommost_level_compaction_;
|
||||||
@ -5843,7 +5842,7 @@ TEST_P(DBCompactionTestWithBottommostParam, SequenceKeysManualCompaction) {
|
|||||||
ASSERT_EQ("0,1", FilesPerLevel(0));
|
ASSERT_EQ("0,1", FilesPerLevel(0));
|
||||||
} else {
|
} else {
|
||||||
// Just trivial move from level 0 -> 1
|
// Just trivial move from level 0 -> 1
|
||||||
ASSERT_EQ("0," + std::to_string(kSstNum), FilesPerLevel(0));
|
ASSERT_EQ("0," + ToString(kSstNum), FilesPerLevel(0));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -7174,7 +7173,7 @@ TEST_F(DBCompactionTest, DisableManualCompactionThreadQueueFull) {
|
|||||||
ASSERT_OK(Put(Key(2), "value2"));
|
ASSERT_OK(Put(Key(2), "value2"));
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
}
|
}
|
||||||
ASSERT_EQ(std::to_string(kNumL0Files + (kNumL0Files / 2)), FilesPerLevel(0));
|
ASSERT_EQ(ToString(kNumL0Files + (kNumL0Files / 2)), FilesPerLevel(0));
|
||||||
|
|
||||||
db_->DisableManualCompaction();
|
db_->DisableManualCompaction();
|
||||||
|
|
||||||
@ -7231,7 +7230,7 @@ TEST_F(DBCompactionTest, DisableManualCompactionThreadQueueFullDBClose) {
|
|||||||
ASSERT_OK(Put(Key(2), "value2"));
|
ASSERT_OK(Put(Key(2), "value2"));
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
}
|
}
|
||||||
ASSERT_EQ(std::to_string(kNumL0Files + (kNumL0Files / 2)), FilesPerLevel(0));
|
ASSERT_EQ(ToString(kNumL0Files + (kNumL0Files / 2)), FilesPerLevel(0));
|
||||||
|
|
||||||
db_->DisableManualCompaction();
|
db_->DisableManualCompaction();
|
||||||
|
|
||||||
@ -7291,7 +7290,7 @@ TEST_F(DBCompactionTest, DBCloseWithManualCompaction) {
|
|||||||
ASSERT_OK(Put(Key(2), "value2"));
|
ASSERT_OK(Put(Key(2), "value2"));
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
}
|
}
|
||||||
ASSERT_EQ(std::to_string(kNumL0Files + (kNumL0Files / 2)), FilesPerLevel(0));
|
ASSERT_EQ(ToString(kNumL0Files + (kNumL0Files / 2)), FilesPerLevel(0));
|
||||||
|
|
||||||
// Close DB with manual compaction and auto triggered compaction in the queue.
|
// Close DB with manual compaction and auto triggered compaction in the queue.
|
||||||
auto s = db_->Close();
|
auto s = db_->Close();
|
||||||
|
@ -177,7 +177,7 @@ Status DBImpl::GetLiveFilesStorageInfo(
|
|||||||
VectorLogPtr live_wal_files;
|
VectorLogPtr live_wal_files;
|
||||||
bool flush_memtable = true;
|
bool flush_memtable = true;
|
||||||
if (!immutable_db_options_.allow_2pc) {
|
if (!immutable_db_options_.allow_2pc) {
|
||||||
if (opts.wal_size_for_flush == std::numeric_limits<uint64_t>::max()) {
|
if (opts.wal_size_for_flush == port::kMaxUint64) {
|
||||||
flush_memtable = false;
|
flush_memtable = false;
|
||||||
} else if (opts.wal_size_for_flush > 0) {
|
} else if (opts.wal_size_for_flush > 0) {
|
||||||
// If the outstanding log files are small, we skip the flush.
|
// If the outstanding log files are small, we skip the flush.
|
||||||
|
@ -2356,7 +2356,7 @@ TEST_P(DBAtomicFlushTest, PrecomputeMinLogNumberToKeepNon2PC) {
|
|||||||
ASSERT_OK(Flush(cf_ids));
|
ASSERT_OK(Flush(cf_ids));
|
||||||
uint64_t log_num_after_flush = dbfull()->TEST_GetCurrentLogNumber();
|
uint64_t log_num_after_flush = dbfull()->TEST_GetCurrentLogNumber();
|
||||||
|
|
||||||
uint64_t min_log_number_to_keep = std::numeric_limits<uint64_t>::max();
|
uint64_t min_log_number_to_keep = port::kMaxUint64;
|
||||||
autovector<ColumnFamilyData*> flushed_cfds;
|
autovector<ColumnFamilyData*> flushed_cfds;
|
||||||
autovector<autovector<VersionEdit*>> flush_edits;
|
autovector<autovector<VersionEdit*>> flush_edits;
|
||||||
for (size_t i = 0; i != num_cfs; ++i) {
|
for (size_t i = 0; i != num_cfs; ++i) {
|
||||||
|
@ -567,7 +567,7 @@ Status DBImpl::CloseHelper() {
|
|||||||
// flushing by first checking if there is a need for
|
// flushing by first checking if there is a need for
|
||||||
// flushing (but need to implement something
|
// flushing (but need to implement something
|
||||||
// else than imm()->IsFlushPending() because the output
|
// else than imm()->IsFlushPending() because the output
|
||||||
// memtables added to imm() don't trigger flushes).
|
// memtables added to imm() dont trigger flushes).
|
||||||
if (immutable_db_options_.experimental_mempurge_threshold > 0.0) {
|
if (immutable_db_options_.experimental_mempurge_threshold > 0.0) {
|
||||||
Status flush_ret;
|
Status flush_ret;
|
||||||
mutex_.Unlock();
|
mutex_.Unlock();
|
||||||
@ -849,8 +849,7 @@ void DBImpl::PersistStats() {
|
|||||||
if (stats_slice_.find(stat.first) != stats_slice_.end()) {
|
if (stats_slice_.find(stat.first) != stats_slice_.end()) {
|
||||||
uint64_t delta = stat.second - stats_slice_[stat.first];
|
uint64_t delta = stat.second - stats_slice_[stat.first];
|
||||||
s = batch.Put(persist_stats_cf_handle_,
|
s = batch.Put(persist_stats_cf_handle_,
|
||||||
Slice(key, std::min(100, length)),
|
Slice(key, std::min(100, length)), ToString(delta));
|
||||||
std::to_string(delta));
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -3356,7 +3355,7 @@ bool DBImpl::GetProperty(ColumnFamilyHandle* column_family,
|
|||||||
bool ret_value =
|
bool ret_value =
|
||||||
GetIntPropertyInternal(cfd, *property_info, false, &int_value);
|
GetIntPropertyInternal(cfd, *property_info, false, &int_value);
|
||||||
if (ret_value) {
|
if (ret_value) {
|
||||||
*value = std::to_string(int_value);
|
*value = ToString(int_value);
|
||||||
}
|
}
|
||||||
return ret_value;
|
return ret_value;
|
||||||
} else if (property_info->handle_string) {
|
} else if (property_info->handle_string) {
|
||||||
@ -3991,8 +3990,8 @@ Status DBImpl::CheckConsistency() {
|
|||||||
} else if (fsize != md.size) {
|
} else if (fsize != md.size) {
|
||||||
corruption_messages += "Sst file size mismatch: " + file_path +
|
corruption_messages += "Sst file size mismatch: " + file_path +
|
||||||
". Size recorded in manifest " +
|
". Size recorded in manifest " +
|
||||||
std::to_string(md.size) + ", actual size " +
|
ToString(md.size) + ", actual size " +
|
||||||
std::to_string(fsize) + "\n";
|
ToString(fsize) + "\n";
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -5339,7 +5338,7 @@ Status DBImpl::ReserveFileNumbersBeforeIngestion(
|
|||||||
|
|
||||||
Status DBImpl::GetCreationTimeOfOldestFile(uint64_t* creation_time) {
|
Status DBImpl::GetCreationTimeOfOldestFile(uint64_t* creation_time) {
|
||||||
if (mutable_db_options_.max_open_files == -1) {
|
if (mutable_db_options_.max_open_files == -1) {
|
||||||
uint64_t oldest_time = std::numeric_limits<uint64_t>::max();
|
uint64_t oldest_time = port::kMaxUint64;
|
||||||
for (auto cfd : *versions_->GetColumnFamilySet()) {
|
for (auto cfd : *versions_->GetColumnFamilySet()) {
|
||||||
if (!cfd->IsDropped()) {
|
if (!cfd->IsDropped()) {
|
||||||
uint64_t ctime;
|
uint64_t ctime;
|
||||||
|
@ -2299,7 +2299,7 @@ class DBImpl : public DB {
|
|||||||
|
|
||||||
static const int KEEP_LOG_FILE_NUM = 1000;
|
static const int KEEP_LOG_FILE_NUM = 1000;
|
||||||
// MSVC version 1800 still does not have constexpr for ::max()
|
// MSVC version 1800 still does not have constexpr for ::max()
|
||||||
static const uint64_t kNoTimeOut = std::numeric_limits<uint64_t>::max();
|
static const uint64_t kNoTimeOut = port::kMaxUint64;
|
||||||
|
|
||||||
std::string db_absolute_path_;
|
std::string db_absolute_path_;
|
||||||
|
|
||||||
|
@ -188,7 +188,7 @@ Status DBImpl::FlushMemTableToOutputFile(
|
|||||||
// a memtable without knowing such snapshot(s).
|
// a memtable without knowing such snapshot(s).
|
||||||
uint64_t max_memtable_id = needs_to_sync_closed_wals
|
uint64_t max_memtable_id = needs_to_sync_closed_wals
|
||||||
? cfd->imm()->GetLatestMemTableID()
|
? cfd->imm()->GetLatestMemTableID()
|
||||||
: std::numeric_limits<uint64_t>::max();
|
: port::kMaxUint64;
|
||||||
|
|
||||||
// If needs_to_sync_closed_wals is false, then the flush job will pick ALL
|
// If needs_to_sync_closed_wals is false, then the flush job will pick ALL
|
||||||
// existing memtables of the column family when PickMemTable() is called
|
// existing memtables of the column family when PickMemTable() is called
|
||||||
@ -1041,8 +1041,7 @@ Status DBImpl::CompactRangeInternal(const CompactRangeOptions& options,
|
|||||||
}
|
}
|
||||||
s = RunManualCompaction(cfd, ColumnFamilyData::kCompactAllLevels,
|
s = RunManualCompaction(cfd, ColumnFamilyData::kCompactAllLevels,
|
||||||
final_output_level, options, begin, end, exclusive,
|
final_output_level, options, begin, end, exclusive,
|
||||||
false, std::numeric_limits<uint64_t>::max(),
|
false, port::kMaxUint64, trim_ts);
|
||||||
trim_ts);
|
|
||||||
} else {
|
} else {
|
||||||
int first_overlapped_level = kInvalidLevel;
|
int first_overlapped_level = kInvalidLevel;
|
||||||
int max_overlapped_level = kInvalidLevel;
|
int max_overlapped_level = kInvalidLevel;
|
||||||
@ -1079,7 +1078,7 @@ Status DBImpl::CompactRangeInternal(const CompactRangeOptions& options,
|
|||||||
if (s.ok() && first_overlapped_level != kInvalidLevel) {
|
if (s.ok() && first_overlapped_level != kInvalidLevel) {
|
||||||
// max_file_num_to_ignore can be used to filter out newly created SST
|
// max_file_num_to_ignore can be used to filter out newly created SST
|
||||||
// files, useful for bottom level compaction in a manual compaction
|
// files, useful for bottom level compaction in a manual compaction
|
||||||
uint64_t max_file_num_to_ignore = std::numeric_limits<uint64_t>::max();
|
uint64_t max_file_num_to_ignore = port::kMaxUint64;
|
||||||
uint64_t next_file_number = versions_->current_next_file_number();
|
uint64_t next_file_number = versions_->current_next_file_number();
|
||||||
final_output_level = max_overlapped_level;
|
final_output_level = max_overlapped_level;
|
||||||
int output_level;
|
int output_level;
|
||||||
@ -1705,9 +1704,11 @@ Status DBImpl::Flush(const FlushOptions& flush_options,
|
|||||||
Status s;
|
Status s;
|
||||||
if (immutable_db_options_.atomic_flush) {
|
if (immutable_db_options_.atomic_flush) {
|
||||||
s = AtomicFlushMemTables({cfh->cfd()}, flush_options,
|
s = AtomicFlushMemTables({cfh->cfd()}, flush_options,
|
||||||
FlushReason::kManualFlush);
|
FlushReason::kManualFlush,
|
||||||
|
write_controller().IsStopped());
|
||||||
} else {
|
} else {
|
||||||
s = FlushMemTable(cfh->cfd(), flush_options, FlushReason::kManualFlush);
|
s = FlushMemTable(cfh->cfd(), flush_options, FlushReason::kManualFlush,
|
||||||
|
write_controller().IsStopped());
|
||||||
}
|
}
|
||||||
|
|
||||||
ROCKS_LOG_INFO(immutable_db_options_.info_log,
|
ROCKS_LOG_INFO(immutable_db_options_.info_log,
|
||||||
@ -2014,7 +2015,7 @@ Status DBImpl::FlushMemTable(ColumnFamilyData* cfd,
|
|||||||
// be created and scheduled, status::OK() will be returned.
|
// be created and scheduled, status::OK() will be returned.
|
||||||
s = SwitchMemtable(cfd, &context);
|
s = SwitchMemtable(cfd, &context);
|
||||||
}
|
}
|
||||||
const uint64_t flush_memtable_id = std::numeric_limits<uint64_t>::max();
|
const uint64_t flush_memtable_id = port::kMaxUint64;
|
||||||
if (s.ok()) {
|
if (s.ok()) {
|
||||||
if (cfd->imm()->NumNotFlushed() != 0 || !cfd->mem()->IsEmpty() ||
|
if (cfd->imm()->NumNotFlushed() != 0 || !cfd->mem()->IsEmpty() ||
|
||||||
!cached_recoverable_state_empty_.load()) {
|
!cached_recoverable_state_empty_.load()) {
|
||||||
|
@ -118,10 +118,9 @@ Status DBImpl::TEST_CompactRange(int level, const Slice* begin,
|
|||||||
cfd->ioptions()->compaction_style == kCompactionStyleFIFO)
|
cfd->ioptions()->compaction_style == kCompactionStyleFIFO)
|
||||||
? level
|
? level
|
||||||
: level + 1;
|
: level + 1;
|
||||||
return RunManualCompaction(
|
return RunManualCompaction(cfd, level, output_level, CompactRangeOptions(),
|
||||||
cfd, level, output_level, CompactRangeOptions(), begin, end, true,
|
begin, end, true, disallow_trivial_move,
|
||||||
disallow_trivial_move,
|
port::kMaxUint64 /*max_file_num_to_ignore*/,
|
||||||
std::numeric_limits<uint64_t>::max() /*max_file_num_to_ignore*/,
|
|
||||||
"" /*trim_ts*/);
|
"" /*trim_ts*/);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -761,7 +761,7 @@ uint64_t PrecomputeMinLogNumberToKeepNon2PC(
|
|||||||
assert(!cfds_to_flush.empty());
|
assert(!cfds_to_flush.empty());
|
||||||
assert(cfds_to_flush.size() == edit_lists.size());
|
assert(cfds_to_flush.size() == edit_lists.size());
|
||||||
|
|
||||||
uint64_t min_log_number_to_keep = std::numeric_limits<uint64_t>::max();
|
uint64_t min_log_number_to_keep = port::kMaxUint64;
|
||||||
for (const auto& edit_list : edit_lists) {
|
for (const auto& edit_list : edit_lists) {
|
||||||
uint64_t log = 0;
|
uint64_t log = 0;
|
||||||
for (const auto& e : edit_list) {
|
for (const auto& e : edit_list) {
|
||||||
@ -773,7 +773,7 @@ uint64_t PrecomputeMinLogNumberToKeepNon2PC(
|
|||||||
min_log_number_to_keep = std::min(min_log_number_to_keep, log);
|
min_log_number_to_keep = std::min(min_log_number_to_keep, log);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (min_log_number_to_keep == std::numeric_limits<uint64_t>::max()) {
|
if (min_log_number_to_keep == port::kMaxUint64) {
|
||||||
min_log_number_to_keep = cfds_to_flush[0]->GetLogNumber();
|
min_log_number_to_keep = cfds_to_flush[0]->GetLogNumber();
|
||||||
for (size_t i = 1; i < cfds_to_flush.size(); i++) {
|
for (size_t i = 1; i < cfds_to_flush.size(); i++) {
|
||||||
min_log_number_to_keep =
|
min_log_number_to_keep =
|
||||||
|
@ -760,11 +760,11 @@ Status DBImpl::PersistentStatsProcessFormatVersion() {
|
|||||||
WriteBatch batch;
|
WriteBatch batch;
|
||||||
if (s.ok()) {
|
if (s.ok()) {
|
||||||
s = batch.Put(persist_stats_cf_handle_, kFormatVersionKeyString,
|
s = batch.Put(persist_stats_cf_handle_, kFormatVersionKeyString,
|
||||||
std::to_string(kStatsCFCurrentFormatVersion));
|
ToString(kStatsCFCurrentFormatVersion));
|
||||||
}
|
}
|
||||||
if (s.ok()) {
|
if (s.ok()) {
|
||||||
s = batch.Put(persist_stats_cf_handle_, kCompatibleVersionKeyString,
|
s = batch.Put(persist_stats_cf_handle_, kCompatibleVersionKeyString,
|
||||||
std::to_string(kStatsCFCompatibleFormatVersion));
|
ToString(kStatsCFCompatibleFormatVersion));
|
||||||
}
|
}
|
||||||
if (s.ok()) {
|
if (s.ok()) {
|
||||||
WriteOptions wo;
|
WriteOptions wo;
|
||||||
@ -1326,7 +1326,6 @@ Status DBImpl::GetLogSizeAndMaybeTruncate(uint64_t wal_number, bool truncate,
|
|||||||
Status s;
|
Status s;
|
||||||
// This gets the appear size of the wals, not including preallocated space.
|
// This gets the appear size of the wals, not including preallocated space.
|
||||||
s = env_->GetFileSize(fname, &log.size);
|
s = env_->GetFileSize(fname, &log.size);
|
||||||
TEST_SYNC_POINT_CALLBACK("DBImpl::GetLogSizeAndMaybeTruncate:0", /*arg=*/&s);
|
|
||||||
if (s.ok() && truncate) {
|
if (s.ok() && truncate) {
|
||||||
std::unique_ptr<FSWritableFile> last_log;
|
std::unique_ptr<FSWritableFile> last_log;
|
||||||
Status truncate_status = fs_->ReopenWritableFile(
|
Status truncate_status = fs_->ReopenWritableFile(
|
||||||
@ -1826,7 +1825,6 @@ Status DBImpl::Open(const DBOptions& db_options, const std::string& dbname,
|
|||||||
if (s.ok()) {
|
if (s.ok()) {
|
||||||
// Need to fsync, otherwise it might get lost after a power reset.
|
// Need to fsync, otherwise it might get lost after a power reset.
|
||||||
s = impl->FlushWAL(false);
|
s = impl->FlushWAL(false);
|
||||||
TEST_SYNC_POINT_CALLBACK("DBImpl::Open::BeforeSyncWAL", /*arg=*/&s);
|
|
||||||
if (s.ok()) {
|
if (s.ok()) {
|
||||||
s = log_writer->file()->Sync(impl->immutable_db_options_.use_fsync);
|
s = log_writer->file()->Sync(impl->immutable_db_options_.use_fsync);
|
||||||
}
|
}
|
||||||
|
@ -247,15 +247,14 @@ Status DBImplSecondary::RecoverLogFiles(
|
|||||||
if (seq_of_batch <= seq) {
|
if (seq_of_batch <= seq) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
auto curr_log_num = std::numeric_limits<uint64_t>::max();
|
auto curr_log_num = port::kMaxUint64;
|
||||||
if (cfd_to_current_log_.count(cfd) > 0) {
|
if (cfd_to_current_log_.count(cfd) > 0) {
|
||||||
curr_log_num = cfd_to_current_log_[cfd];
|
curr_log_num = cfd_to_current_log_[cfd];
|
||||||
}
|
}
|
||||||
// If the active memtable contains records added by replaying an
|
// If the active memtable contains records added by replaying an
|
||||||
// earlier WAL, then we need to seal the memtable, add it to the
|
// earlier WAL, then we need to seal the memtable, add it to the
|
||||||
// immutable memtable list and create a new active memtable.
|
// immutable memtable list and create a new active memtable.
|
||||||
if (!cfd->mem()->IsEmpty() &&
|
if (!cfd->mem()->IsEmpty() && (curr_log_num == port::kMaxUint64 ||
|
||||||
(curr_log_num == std::numeric_limits<uint64_t>::max() ||
|
|
||||||
curr_log_num != log_number)) {
|
curr_log_num != log_number)) {
|
||||||
const MutableCFOptions mutable_cf_options =
|
const MutableCFOptions mutable_cf_options =
|
||||||
*cfd->GetLatestMutableCFOptions();
|
*cfd->GetLatestMutableCFOptions();
|
||||||
|
@ -35,12 +35,10 @@ void DumpDBFileSummary(const ImmutableDBOptions& options,
|
|||||||
Header(options.info_log, "DB SUMMARY\n");
|
Header(options.info_log, "DB SUMMARY\n");
|
||||||
Header(options.info_log, "DB Session ID: %s\n", session_id.c_str());
|
Header(options.info_log, "DB Session ID: %s\n", session_id.c_str());
|
||||||
|
|
||||||
Status s;
|
|
||||||
// Get files in dbname dir
|
// Get files in dbname dir
|
||||||
s = env->GetChildren(dbname, &files);
|
if (!env->GetChildren(dbname, &files).ok()) {
|
||||||
if (!s.ok()) {
|
Error(options.info_log,
|
||||||
Error(options.info_log, "Error when reading %s dir %s\n", dbname.c_str(),
|
"Error when reading %s dir\n", dbname.c_str());
|
||||||
s.ToString().c_str());
|
|
||||||
}
|
}
|
||||||
std::sort(files.begin(), files.end());
|
std::sort(files.begin(), files.end());
|
||||||
for (const std::string& file : files) {
|
for (const std::string& file : files) {
|
||||||
@ -55,27 +53,24 @@ void DumpDBFileSummary(const ImmutableDBOptions& options,
|
|||||||
Header(options.info_log, "IDENTITY file: %s\n", file.c_str());
|
Header(options.info_log, "IDENTITY file: %s\n", file.c_str());
|
||||||
break;
|
break;
|
||||||
case kDescriptorFile:
|
case kDescriptorFile:
|
||||||
s = env->GetFileSize(dbname + "/" + file, &file_size);
|
if (env->GetFileSize(dbname + "/" + file, &file_size).ok()) {
|
||||||
if (s.ok()) {
|
|
||||||
Header(options.info_log,
|
Header(options.info_log,
|
||||||
"MANIFEST file: %s size: %" PRIu64 " Bytes\n", file.c_str(),
|
"MANIFEST file: %s size: %" PRIu64 " Bytes\n", file.c_str(),
|
||||||
file_size);
|
file_size);
|
||||||
} else {
|
} else {
|
||||||
Error(options.info_log,
|
Error(options.info_log, "Error when reading MANIFEST file: %s/%s\n",
|
||||||
"Error when reading MANIFEST file: %s/%s %s\n", dbname.c_str(),
|
dbname.c_str(), file.c_str());
|
||||||
file.c_str(), s.ToString().c_str());
|
|
||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
case kWalFile:
|
case kWalFile:
|
||||||
s = env->GetFileSize(dbname + "/" + file, &file_size);
|
if (env->GetFileSize(dbname + "/" + file, &file_size).ok()) {
|
||||||
if (s.ok()) {
|
|
||||||
wal_info.append(file)
|
wal_info.append(file)
|
||||||
.append(" size: ")
|
.append(" size: ")
|
||||||
.append(std::to_string(file_size))
|
.append(std::to_string(file_size))
|
||||||
.append(" ; ");
|
.append(" ; ");
|
||||||
} else {
|
} else {
|
||||||
Error(options.info_log, "Error when reading LOG file: %s/%s %s\n",
|
Error(options.info_log, "Error when reading LOG file: %s/%s\n",
|
||||||
dbname.c_str(), file.c_str(), s.ToString().c_str());
|
dbname.c_str(), file.c_str());
|
||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
case kTableFile:
|
case kTableFile:
|
||||||
@ -91,10 +86,10 @@ void DumpDBFileSummary(const ImmutableDBOptions& options,
|
|||||||
// Get sst files in db_path dir
|
// Get sst files in db_path dir
|
||||||
for (auto& db_path : options.db_paths) {
|
for (auto& db_path : options.db_paths) {
|
||||||
if (dbname.compare(db_path.path) != 0) {
|
if (dbname.compare(db_path.path) != 0) {
|
||||||
s = env->GetChildren(db_path.path, &files);
|
if (!env->GetChildren(db_path.path, &files).ok()) {
|
||||||
if (!s.ok()) {
|
Error(options.info_log,
|
||||||
Error(options.info_log, "Error when reading %s dir %s\n",
|
"Error when reading %s dir\n",
|
||||||
db_path.path.c_str(), s.ToString().c_str());
|
db_path.path.c_str());
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
std::sort(files.begin(), files.end());
|
std::sort(files.begin(), files.end());
|
||||||
@ -116,25 +111,22 @@ void DumpDBFileSummary(const ImmutableDBOptions& options,
|
|||||||
// Get wal file in wal_dir
|
// Get wal file in wal_dir
|
||||||
const auto& wal_dir = options.GetWalDir(dbname);
|
const auto& wal_dir = options.GetWalDir(dbname);
|
||||||
if (!options.IsWalDirSameAsDBPath(dbname)) {
|
if (!options.IsWalDirSameAsDBPath(dbname)) {
|
||||||
s = env->GetChildren(wal_dir, &files);
|
if (!env->GetChildren(wal_dir, &files).ok()) {
|
||||||
if (!s.ok()) {
|
Error(options.info_log, "Error when reading %s dir\n", wal_dir.c_str());
|
||||||
Error(options.info_log, "Error when reading %s dir %s\n", wal_dir.c_str(),
|
|
||||||
s.ToString().c_str());
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
wal_info.clear();
|
wal_info.clear();
|
||||||
for (const std::string& file : files) {
|
for (const std::string& file : files) {
|
||||||
if (ParseFileName(file, &number, &type)) {
|
if (ParseFileName(file, &number, &type)) {
|
||||||
if (type == kWalFile) {
|
if (type == kWalFile) {
|
||||||
s = env->GetFileSize(wal_dir + "/" + file, &file_size);
|
if (env->GetFileSize(wal_dir + "/" + file, &file_size).ok()) {
|
||||||
if (s.ok()) {
|
|
||||||
wal_info.append(file)
|
wal_info.append(file)
|
||||||
.append(" size: ")
|
.append(" size: ")
|
||||||
.append(std::to_string(file_size))
|
.append(std::to_string(file_size))
|
||||||
.append(" ; ");
|
.append(" ; ");
|
||||||
} else {
|
} else {
|
||||||
Error(options.info_log, "Error when reading LOG file %s/%s %s\n",
|
Error(options.info_log, "Error when reading LOG file %s/%s\n",
|
||||||
wal_dir.c_str(), file.c_str(), s.ToString().c_str());
|
wal_dir.c_str(), file.c_str());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -414,7 +414,7 @@ TEST_F(DBIteratorStressTest, StressTest) {
|
|||||||
a /= 10;
|
a /= 10;
|
||||||
++len;
|
++len;
|
||||||
}
|
}
|
||||||
std::string s = std::to_string(rnd.Next() % static_cast<uint64_t>(max_key));
|
std::string s = ToString(rnd.Next() % static_cast<uint64_t>(max_key));
|
||||||
s.insert(0, len - (int)s.size(), '0');
|
s.insert(0, len - (int)s.size(), '0');
|
||||||
return s;
|
return s;
|
||||||
};
|
};
|
||||||
@ -444,13 +444,12 @@ TEST_F(DBIteratorStressTest, StressTest) {
|
|||||||
for (double mutation_probability : {0.01, 0.5}) {
|
for (double mutation_probability : {0.01, 0.5}) {
|
||||||
for (double target_hidden_fraction : {0.1, 0.5}) {
|
for (double target_hidden_fraction : {0.1, 0.5}) {
|
||||||
std::string trace_str =
|
std::string trace_str =
|
||||||
"entries: " + std::to_string(num_entries) +
|
"entries: " + ToString(num_entries) +
|
||||||
", key_space: " + std::to_string(key_space) +
|
", key_space: " + ToString(key_space) +
|
||||||
", error_probability: " + std::to_string(error_probability) +
|
", error_probability: " + ToString(error_probability) +
|
||||||
", mutation_probability: " +
|
", mutation_probability: " + ToString(mutation_probability) +
|
||||||
std::to_string(mutation_probability) +
|
|
||||||
", target_hidden_fraction: " +
|
", target_hidden_fraction: " +
|
||||||
std::to_string(target_hidden_fraction);
|
ToString(target_hidden_fraction);
|
||||||
SCOPED_TRACE(trace_str);
|
SCOPED_TRACE(trace_str);
|
||||||
if (trace) {
|
if (trace) {
|
||||||
std::cout << trace_str << std::endl;
|
std::cout << trace_str << std::endl;
|
||||||
@ -471,7 +470,7 @@ TEST_F(DBIteratorStressTest, StressTest) {
|
|||||||
types[rnd.Next() % (sizeof(types) / sizeof(types[0]))];
|
types[rnd.Next() % (sizeof(types) / sizeof(types[0]))];
|
||||||
}
|
}
|
||||||
e.sequence = i;
|
e.sequence = i;
|
||||||
e.value = "v" + std::to_string(i);
|
e.value = "v" + ToString(i);
|
||||||
ParsedInternalKey internal_key(e.key, e.sequence, e.type);
|
ParsedInternalKey internal_key(e.key, e.sequence, e.type);
|
||||||
AppendInternalKey(&e.ikey, internal_key);
|
AppendInternalKey(&e.ikey, internal_key);
|
||||||
|
|
||||||
|
@ -766,7 +766,7 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {
|
|||||||
internal_iter->AddMerge("b", "merge_1");
|
internal_iter->AddMerge("b", "merge_1");
|
||||||
internal_iter->AddMerge("a", "merge_2");
|
internal_iter->AddMerge("a", "merge_2");
|
||||||
for (size_t k = 0; k < 200; ++k) {
|
for (size_t k = 0; k < 200; ++k) {
|
||||||
internal_iter->AddPut("c", std::to_string(k));
|
internal_iter->AddPut("c", ToString(k));
|
||||||
}
|
}
|
||||||
internal_iter->Finish();
|
internal_iter->Finish();
|
||||||
|
|
||||||
@ -780,7 +780,7 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {
|
|||||||
ASSERT_TRUE(db_iter->Valid());
|
ASSERT_TRUE(db_iter->Valid());
|
||||||
|
|
||||||
ASSERT_EQ(db_iter->key().ToString(), "c");
|
ASSERT_EQ(db_iter->key().ToString(), "c");
|
||||||
ASSERT_EQ(db_iter->value().ToString(), std::to_string(i));
|
ASSERT_EQ(db_iter->value().ToString(), ToString(i));
|
||||||
db_iter->Prev();
|
db_iter->Prev();
|
||||||
ASSERT_TRUE(db_iter->Valid());
|
ASSERT_TRUE(db_iter->Valid());
|
||||||
|
|
||||||
@ -925,11 +925,11 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {
|
|||||||
internal_iter->AddMerge("b", "merge_1");
|
internal_iter->AddMerge("b", "merge_1");
|
||||||
internal_iter->AddMerge("a", "merge_2");
|
internal_iter->AddMerge("a", "merge_2");
|
||||||
for (size_t k = 0; k < 200; ++k) {
|
for (size_t k = 0; k < 200; ++k) {
|
||||||
internal_iter->AddPut("d", std::to_string(k));
|
internal_iter->AddPut("d", ToString(k));
|
||||||
}
|
}
|
||||||
|
|
||||||
for (size_t k = 0; k < 200; ++k) {
|
for (size_t k = 0; k < 200; ++k) {
|
||||||
internal_iter->AddPut("c", std::to_string(k));
|
internal_iter->AddPut("c", ToString(k));
|
||||||
}
|
}
|
||||||
internal_iter->Finish();
|
internal_iter->Finish();
|
||||||
|
|
||||||
@ -942,7 +942,7 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {
|
|||||||
ASSERT_TRUE(db_iter->Valid());
|
ASSERT_TRUE(db_iter->Valid());
|
||||||
|
|
||||||
ASSERT_EQ(db_iter->key().ToString(), "d");
|
ASSERT_EQ(db_iter->key().ToString(), "d");
|
||||||
ASSERT_EQ(db_iter->value().ToString(), std::to_string(i));
|
ASSERT_EQ(db_iter->value().ToString(), ToString(i));
|
||||||
db_iter->Prev();
|
db_iter->Prev();
|
||||||
ASSERT_TRUE(db_iter->Valid());
|
ASSERT_TRUE(db_iter->Valid());
|
||||||
|
|
||||||
@ -966,7 +966,7 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {
|
|||||||
internal_iter->AddMerge("b", "b");
|
internal_iter->AddMerge("b", "b");
|
||||||
internal_iter->AddMerge("a", "a");
|
internal_iter->AddMerge("a", "a");
|
||||||
for (size_t k = 0; k < 200; ++k) {
|
for (size_t k = 0; k < 200; ++k) {
|
||||||
internal_iter->AddMerge("c", std::to_string(k));
|
internal_iter->AddMerge("c", ToString(k));
|
||||||
}
|
}
|
||||||
internal_iter->Finish();
|
internal_iter->Finish();
|
||||||
|
|
||||||
@ -981,7 +981,7 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {
|
|||||||
ASSERT_EQ(db_iter->key().ToString(), "c");
|
ASSERT_EQ(db_iter->key().ToString(), "c");
|
||||||
std::string merge_result = "0";
|
std::string merge_result = "0";
|
||||||
for (size_t j = 1; j <= i; ++j) {
|
for (size_t j = 1; j <= i; ++j) {
|
||||||
merge_result += "," + std::to_string(j);
|
merge_result += "," + ToString(j);
|
||||||
}
|
}
|
||||||
ASSERT_EQ(db_iter->value().ToString(), merge_result);
|
ASSERT_EQ(db_iter->value().ToString(), merge_result);
|
||||||
|
|
||||||
@ -3156,7 +3156,7 @@ TEST_F(DBIteratorTest, ReverseToForwardWithDisappearingKeys) {
|
|||||||
internal_iter->AddPut("a", "A");
|
internal_iter->AddPut("a", "A");
|
||||||
internal_iter->AddPut("b", "B");
|
internal_iter->AddPut("b", "B");
|
||||||
for (int i = 0; i < 100; ++i) {
|
for (int i = 0; i < 100; ++i) {
|
||||||
internal_iter->AddPut("c" + std::to_string(i), "");
|
internal_iter->AddPut("c" + ToString(i), "");
|
||||||
}
|
}
|
||||||
internal_iter->Finish();
|
internal_iter->Finish();
|
||||||
|
|
||||||
|
@ -3160,7 +3160,7 @@ TEST_F(DBIteratorWithReadCallbackTest, ReadCallback) {
|
|||||||
uint64_t num_versions =
|
uint64_t num_versions =
|
||||||
CurrentOptions().max_sequential_skip_in_iterations + 10;
|
CurrentOptions().max_sequential_skip_in_iterations + 10;
|
||||||
for (uint64_t i = 0; i < num_versions; i++) {
|
for (uint64_t i = 0; i < num_versions; i++) {
|
||||||
ASSERT_OK(Put("bar", std::to_string(i)));
|
ASSERT_OK(Put("bar", ToString(i)));
|
||||||
}
|
}
|
||||||
SequenceNumber seq3 = db_->GetLatestSequenceNumber();
|
SequenceNumber seq3 = db_->GetLatestSequenceNumber();
|
||||||
TestReadCallback callback2(seq3);
|
TestReadCallback callback2(seq3);
|
||||||
@ -3189,7 +3189,7 @@ TEST_F(DBIteratorWithReadCallbackTest, ReadCallback) {
|
|||||||
ASSERT_TRUE(iter->Valid());
|
ASSERT_TRUE(iter->Valid());
|
||||||
ASSERT_OK(iter->status());
|
ASSERT_OK(iter->status());
|
||||||
ASSERT_EQ("bar", iter->key());
|
ASSERT_EQ("bar", iter->key());
|
||||||
ASSERT_EQ(std::to_string(num_versions - 1), iter->value());
|
ASSERT_EQ(ToString(num_versions - 1), iter->value());
|
||||||
|
|
||||||
delete iter;
|
delete iter;
|
||||||
}
|
}
|
||||||
|
@ -79,7 +79,7 @@ class DbKvChecksumTest
|
|||||||
|
|
||||||
void CorruptNextByteCallBack(void* arg) {
|
void CorruptNextByteCallBack(void* arg) {
|
||||||
Slice encoded = *static_cast<Slice*>(arg);
|
Slice encoded = *static_cast<Slice*>(arg);
|
||||||
if (entry_len_ == std::numeric_limits<size_t>::max()) {
|
if (entry_len_ == port::kMaxSizet) {
|
||||||
// We learn the entry size on the first attempt
|
// We learn the entry size on the first attempt
|
||||||
entry_len_ = encoded.size();
|
entry_len_ = encoded.size();
|
||||||
}
|
}
|
||||||
@ -96,7 +96,7 @@ class DbKvChecksumTest
|
|||||||
WriteBatchOpType op_type_;
|
WriteBatchOpType op_type_;
|
||||||
char corrupt_byte_addend_;
|
char corrupt_byte_addend_;
|
||||||
size_t corrupt_byte_offset_ = 0;
|
size_t corrupt_byte_offset_ = 0;
|
||||||
size_t entry_len_ = std::numeric_limits<size_t>::max();
|
size_t entry_len_ = port::kMaxSizet;
|
||||||
};
|
};
|
||||||
|
|
||||||
std::string GetTestNameSuffix(
|
std::string GetTestNameSuffix(
|
||||||
|
@ -187,7 +187,7 @@ TEST_F(DBTestXactLogIterator, TransactionLogIteratorCorruptedLog) {
|
|||||||
DestroyAndReopen(options);
|
DestroyAndReopen(options);
|
||||||
|
|
||||||
for (int i = 0; i < 1024; i++) {
|
for (int i = 0; i < 1024; i++) {
|
||||||
ASSERT_OK(Put("key" + std::to_string(i), DummyString(10)));
|
ASSERT_OK(Put("key" + ToString(i), DummyString(10)));
|
||||||
}
|
}
|
||||||
|
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
@ -263,20 +263,20 @@ TEST_F(DBTestXactLogIterator, TransactionLogIteratorBlobs) {
|
|||||||
struct Handler : public WriteBatch::Handler {
|
struct Handler : public WriteBatch::Handler {
|
||||||
std::string seen;
|
std::string seen;
|
||||||
Status PutCF(uint32_t cf, const Slice& key, const Slice& value) override {
|
Status PutCF(uint32_t cf, const Slice& key, const Slice& value) override {
|
||||||
seen += "Put(" + std::to_string(cf) + ", " + key.ToString() + ", " +
|
seen += "Put(" + ToString(cf) + ", " + key.ToString() + ", " +
|
||||||
std::to_string(value.size()) + ")";
|
ToString(value.size()) + ")";
|
||||||
return Status::OK();
|
return Status::OK();
|
||||||
}
|
}
|
||||||
Status MergeCF(uint32_t cf, const Slice& key, const Slice& value) override {
|
Status MergeCF(uint32_t cf, const Slice& key, const Slice& value) override {
|
||||||
seen += "Merge(" + std::to_string(cf) + ", " + key.ToString() + ", " +
|
seen += "Merge(" + ToString(cf) + ", " + key.ToString() + ", " +
|
||||||
std::to_string(value.size()) + ")";
|
ToString(value.size()) + ")";
|
||||||
return Status::OK();
|
return Status::OK();
|
||||||
}
|
}
|
||||||
void LogData(const Slice& blob) override {
|
void LogData(const Slice& blob) override {
|
||||||
seen += "LogData(" + blob.ToString() + ")";
|
seen += "LogData(" + blob.ToString() + ")";
|
||||||
}
|
}
|
||||||
Status DeleteCF(uint32_t cf, const Slice& key) override {
|
Status DeleteCF(uint32_t cf, const Slice& key) override {
|
||||||
seen += "Delete(" + std::to_string(cf) + ", " + key.ToString() + ")";
|
seen += "Delete(" + ToString(cf) + ", " + key.ToString() + ")";
|
||||||
return Status::OK();
|
return Status::OK();
|
||||||
}
|
}
|
||||||
} handler;
|
} handler;
|
||||||
|
@ -97,7 +97,7 @@ class MockMemTableRepFactory : public MemTableRepFactory {
|
|||||||
|
|
||||||
private:
|
private:
|
||||||
MockMemTableRep* mock_rep_;
|
MockMemTableRep* mock_rep_;
|
||||||
// workaround since there's no std::numeric_limits<uint32_t>::max() yet.
|
// workaround since there's no port::kMaxUint32 yet.
|
||||||
uint32_t last_column_family_id_ = static_cast<uint32_t>(-1);
|
uint32_t last_column_family_id_ = static_cast<uint32_t>(-1);
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -171,7 +171,7 @@ TEST_F(DBMemTableTest, DuplicateSeq) {
|
|||||||
if (!insert_dup) {
|
if (!insert_dup) {
|
||||||
seq++;
|
seq++;
|
||||||
}
|
}
|
||||||
Status s = mem->Add(seq, kTypeValue, "foo", "value" + std::to_string(seq),
|
Status s = mem->Add(seq, kTypeValue, "foo", "value" + ToString(seq),
|
||||||
nullptr /* kv_prot_info */);
|
nullptr /* kv_prot_info */);
|
||||||
if (insert_dup) {
|
if (insert_dup) {
|
||||||
ASSERT_TRUE(s.IsTryAgain());
|
ASSERT_TRUE(s.IsTryAgain());
|
||||||
|
@ -424,8 +424,8 @@ TEST_F(DBOptionsTest, WritableFileMaxBufferSize) {
|
|||||||
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
|
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
|
||||||
int i = 0;
|
int i = 0;
|
||||||
for (; i < 3; i++) {
|
for (; i < 3; i++) {
|
||||||
ASSERT_OK(Put("foo", std::to_string(i)));
|
ASSERT_OK(Put("foo", ToString(i)));
|
||||||
ASSERT_OK(Put("bar", std::to_string(i)));
|
ASSERT_OK(Put("bar", ToString(i)));
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
}
|
}
|
||||||
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
||||||
@ -442,8 +442,8 @@ TEST_F(DBOptionsTest, WritableFileMaxBufferSize) {
|
|||||||
dbfull()->GetDBOptions().writable_file_max_buffer_size);
|
dbfull()->GetDBOptions().writable_file_max_buffer_size);
|
||||||
i = 0;
|
i = 0;
|
||||||
for (; i < 3; i++) {
|
for (; i < 3; i++) {
|
||||||
ASSERT_OK(Put("foo", std::to_string(i)));
|
ASSERT_OK(Put("foo", ToString(i)));
|
||||||
ASSERT_OK(Put("bar", std::to_string(i)));
|
ASSERT_OK(Put("bar", ToString(i)));
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
}
|
}
|
||||||
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
||||||
@ -573,8 +573,8 @@ TEST_F(DBOptionsTest, SetOptionsMayTriggerCompaction) {
|
|||||||
Reopen(options);
|
Reopen(options);
|
||||||
for (int i = 0; i < 3; i++) {
|
for (int i = 0; i < 3; i++) {
|
||||||
// Need to insert two keys to avoid trivial move.
|
// Need to insert two keys to avoid trivial move.
|
||||||
ASSERT_OK(Put("foo", std::to_string(i)));
|
ASSERT_OK(Put("foo", ToString(i)));
|
||||||
ASSERT_OK(Put("bar", std::to_string(i)));
|
ASSERT_OK(Put("bar", ToString(i)));
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
}
|
}
|
||||||
ASSERT_EQ("3", FilesPerLevel());
|
ASSERT_EQ("3", FilesPerLevel());
|
||||||
@ -717,8 +717,8 @@ TEST_F(DBOptionsTest, SetStatsDumpPeriodSec) {
|
|||||||
|
|
||||||
for (int i = 0; i < 20; i++) {
|
for (int i = 0; i < 20; i++) {
|
||||||
unsigned int num = rand() % 5000 + 1;
|
unsigned int num = rand() % 5000 + 1;
|
||||||
ASSERT_OK(dbfull()->SetDBOptions(
|
ASSERT_OK(
|
||||||
{{"stats_dump_period_sec", std::to_string(num)}}));
|
dbfull()->SetDBOptions({{"stats_dump_period_sec", ToString(num)}}));
|
||||||
ASSERT_EQ(num, dbfull()->GetDBOptions().stats_dump_period_sec);
|
ASSERT_EQ(num, dbfull()->GetDBOptions().stats_dump_period_sec);
|
||||||
}
|
}
|
||||||
Close();
|
Close();
|
||||||
@ -909,7 +909,7 @@ TEST_F(DBOptionsTest, SetFIFOCompactionOptions) {
|
|||||||
for (int i = 0; i < 10; i++) {
|
for (int i = 0; i < 10; i++) {
|
||||||
// Generate and flush a file about 10KB.
|
// Generate and flush a file about 10KB.
|
||||||
for (int j = 0; j < 10; j++) {
|
for (int j = 0; j < 10; j++) {
|
||||||
ASSERT_OK(Put(std::to_string(i * 20 + j), rnd.RandomString(980)));
|
ASSERT_OK(Put(ToString(i * 20 + j), rnd.RandomString(980)));
|
||||||
}
|
}
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
}
|
}
|
||||||
@ -940,7 +940,7 @@ TEST_F(DBOptionsTest, SetFIFOCompactionOptions) {
|
|||||||
for (int i = 0; i < 10; i++) {
|
for (int i = 0; i < 10; i++) {
|
||||||
// Generate and flush a file about 10KB.
|
// Generate and flush a file about 10KB.
|
||||||
for (int j = 0; j < 10; j++) {
|
for (int j = 0; j < 10; j++) {
|
||||||
ASSERT_OK(Put(std::to_string(i * 20 + j), rnd.RandomString(980)));
|
ASSERT_OK(Put(ToString(i * 20 + j), rnd.RandomString(980)));
|
||||||
}
|
}
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
}
|
}
|
||||||
@ -972,7 +972,7 @@ TEST_F(DBOptionsTest, SetFIFOCompactionOptions) {
|
|||||||
for (int i = 0; i < 10; i++) {
|
for (int i = 0; i < 10; i++) {
|
||||||
// Generate and flush a file about 10KB.
|
// Generate and flush a file about 10KB.
|
||||||
for (int j = 0; j < 10; j++) {
|
for (int j = 0; j < 10; j++) {
|
||||||
ASSERT_OK(Put(std::to_string(i * 20 + j), rnd.RandomString(980)));
|
ASSERT_OK(Put(ToString(i * 20 + j), rnd.RandomString(980)));
|
||||||
}
|
}
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
}
|
}
|
||||||
@ -1036,7 +1036,7 @@ TEST_F(DBOptionsTest, FIFOTtlBackwardCompatible) {
|
|||||||
for (int i = 0; i < 10; i++) {
|
for (int i = 0; i < 10; i++) {
|
||||||
// Generate and flush a file about 10KB.
|
// Generate and flush a file about 10KB.
|
||||||
for (int j = 0; j < 10; j++) {
|
for (int j = 0; j < 10; j++) {
|
||||||
ASSERT_OK(Put(std::to_string(i * 20 + j), rnd.RandomString(980)));
|
ASSERT_OK(Put(ToString(i * 20 + j), rnd.RandomString(980)));
|
||||||
}
|
}
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
}
|
}
|
||||||
|
@ -593,8 +593,8 @@ TEST_F(DBPropertiesTest, AggregatedTablePropertiesAtLevel) {
|
|||||||
ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
|
ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
|
||||||
ResetTableProperties(&sum_tp);
|
ResetTableProperties(&sum_tp);
|
||||||
for (int level = 0; level < kMaxLevel; ++level) {
|
for (int level = 0; level < kMaxLevel; ++level) {
|
||||||
db_->GetProperty(DB::Properties::kAggregatedTablePropertiesAtLevel +
|
db_->GetProperty(
|
||||||
std::to_string(level),
|
DB::Properties::kAggregatedTablePropertiesAtLevel + ToString(level),
|
||||||
&level_tp_strings[level]);
|
&level_tp_strings[level]);
|
||||||
ParseTablePropertiesString(level_tp_strings[level], &level_tps[level]);
|
ParseTablePropertiesString(level_tp_strings[level], &level_tps[level]);
|
||||||
sum_tp.data_size += level_tps[level].data_size;
|
sum_tp.data_size += level_tps[level].data_size;
|
||||||
@ -1091,7 +1091,7 @@ TEST_F(DBPropertiesTest, EstimateCompressionRatio) {
|
|||||||
for (int j = 0; j < kNumEntriesPerFile; ++j) {
|
for (int j = 0; j < kNumEntriesPerFile; ++j) {
|
||||||
// Put common data ("key") at end to prevent delta encoding from
|
// Put common data ("key") at end to prevent delta encoding from
|
||||||
// compressing the key effectively
|
// compressing the key effectively
|
||||||
std::string key = std::to_string(i) + std::to_string(j) + "key";
|
std::string key = ToString(i) + ToString(j) + "key";
|
||||||
ASSERT_OK(dbfull()->Put(WriteOptions(), key, kVal));
|
ASSERT_OK(dbfull()->Put(WriteOptions(), key, kVal));
|
||||||
}
|
}
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
@ -1185,7 +1185,7 @@ class CountingDeleteTabPropCollector : public TablePropertiesCollector {
|
|||||||
|
|
||||||
Status Finish(UserCollectedProperties* properties) override {
|
Status Finish(UserCollectedProperties* properties) override {
|
||||||
*properties =
|
*properties =
|
||||||
UserCollectedProperties{{"num_delete", std::to_string(num_deletes_)}};
|
UserCollectedProperties{{"num_delete", ToString(num_deletes_)}};
|
||||||
return Status::OK();
|
return Status::OK();
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1215,7 +1215,7 @@ class BlockCountingTablePropertiesCollector : public TablePropertiesCollector {
|
|||||||
|
|
||||||
Status Finish(UserCollectedProperties* properties) override {
|
Status Finish(UserCollectedProperties* properties) override {
|
||||||
(*properties)[kNumSampledBlocksPropertyName] =
|
(*properties)[kNumSampledBlocksPropertyName] =
|
||||||
std::to_string(num_sampled_blocks_);
|
ToString(num_sampled_blocks_);
|
||||||
return Status::OK();
|
return Status::OK();
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1235,7 +1235,7 @@ class BlockCountingTablePropertiesCollector : public TablePropertiesCollector {
|
|||||||
|
|
||||||
UserCollectedProperties GetReadableProperties() const override {
|
UserCollectedProperties GetReadableProperties() const override {
|
||||||
return UserCollectedProperties{
|
return UserCollectedProperties{
|
||||||
{kNumSampledBlocksPropertyName, std::to_string(num_sampled_blocks_)},
|
{kNumSampledBlocksPropertyName, ToString(num_sampled_blocks_)},
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1272,8 +1272,7 @@ TEST_F(DBPropertiesTest, GetUserDefinedTableProperties) {
|
|||||||
// Create 4 tables
|
// Create 4 tables
|
||||||
for (int table = 0; table < 4; ++table) {
|
for (int table = 0; table < 4; ++table) {
|
||||||
for (int i = 0; i < 10 + table; ++i) {
|
for (int i = 0; i < 10 + table; ++i) {
|
||||||
ASSERT_OK(
|
ASSERT_OK(db_->Put(WriteOptions(), ToString(table * 100 + i), "val"));
|
||||||
db_->Put(WriteOptions(), std::to_string(table * 100 + i), "val"));
|
|
||||||
}
|
}
|
||||||
ASSERT_OK(db_->Flush(FlushOptions()));
|
ASSERT_OK(db_->Flush(FlushOptions()));
|
||||||
}
|
}
|
||||||
@ -1313,7 +1312,7 @@ TEST_F(DBPropertiesTest, UserDefinedTablePropertiesContext) {
|
|||||||
// Create 2 files
|
// Create 2 files
|
||||||
for (int table = 0; table < 2; ++table) {
|
for (int table = 0; table < 2; ++table) {
|
||||||
for (int i = 0; i < 10 + table; ++i) {
|
for (int i = 0; i < 10 + table; ++i) {
|
||||||
ASSERT_OK(Put(1, std::to_string(table * 100 + i), "val"));
|
ASSERT_OK(Put(1, ToString(table * 100 + i), "val"));
|
||||||
}
|
}
|
||||||
ASSERT_OK(Flush(1));
|
ASSERT_OK(Flush(1));
|
||||||
}
|
}
|
||||||
@ -1323,7 +1322,7 @@ TEST_F(DBPropertiesTest, UserDefinedTablePropertiesContext) {
|
|||||||
// Trigger automatic compactions.
|
// Trigger automatic compactions.
|
||||||
for (int table = 0; table < 3; ++table) {
|
for (int table = 0; table < 3; ++table) {
|
||||||
for (int i = 0; i < 10 + table; ++i) {
|
for (int i = 0; i < 10 + table; ++i) {
|
||||||
ASSERT_OK(Put(1, std::to_string(table * 100 + i), "val"));
|
ASSERT_OK(Put(1, ToString(table * 100 + i), "val"));
|
||||||
}
|
}
|
||||||
ASSERT_OK(Flush(1));
|
ASSERT_OK(Flush(1));
|
||||||
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
||||||
@ -1340,7 +1339,7 @@ TEST_F(DBPropertiesTest, UserDefinedTablePropertiesContext) {
|
|||||||
// Create 4 tables in default column family
|
// Create 4 tables in default column family
|
||||||
for (int table = 0; table < 2; ++table) {
|
for (int table = 0; table < 2; ++table) {
|
||||||
for (int i = 0; i < 10 + table; ++i) {
|
for (int i = 0; i < 10 + table; ++i) {
|
||||||
ASSERT_OK(Put(std::to_string(table * 100 + i), "val"));
|
ASSERT_OK(Put(ToString(table * 100 + i), "val"));
|
||||||
}
|
}
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
}
|
}
|
||||||
@ -1350,7 +1349,7 @@ TEST_F(DBPropertiesTest, UserDefinedTablePropertiesContext) {
|
|||||||
// Trigger automatic compactions.
|
// Trigger automatic compactions.
|
||||||
for (int table = 0; table < 3; ++table) {
|
for (int table = 0; table < 3; ++table) {
|
||||||
for (int i = 0; i < 10 + table; ++i) {
|
for (int i = 0; i < 10 + table; ++i) {
|
||||||
ASSERT_OK(Put(std::to_string(table * 100 + i), "val"));
|
ASSERT_OK(Put(ToString(table * 100 + i), "val"));
|
||||||
}
|
}
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
||||||
@ -1546,7 +1545,7 @@ TEST_F(DBPropertiesTest, BlockAddForCompressionSampling) {
|
|||||||
user_props.end());
|
user_props.end());
|
||||||
ASSERT_EQ(user_props.at(BlockCountingTablePropertiesCollector::
|
ASSERT_EQ(user_props.at(BlockCountingTablePropertiesCollector::
|
||||||
kNumSampledBlocksPropertyName),
|
kNumSampledBlocksPropertyName),
|
||||||
std::to_string(sample_for_compression ? 1 : 0));
|
ToString(sample_for_compression ? 1 : 0));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -1743,11 +1742,11 @@ TEST_F(DBPropertiesTest, SstFilesSize) {
|
|||||||
Reopen(options);
|
Reopen(options);
|
||||||
|
|
||||||
for (int i = 0; i < 10; i++) {
|
for (int i = 0; i < 10; i++) {
|
||||||
ASSERT_OK(Put("key" + std::to_string(i), std::string(1000, 'v')));
|
ASSERT_OK(Put("key" + ToString(i), std::string(1000, 'v')));
|
||||||
}
|
}
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
for (int i = 0; i < 5; i++) {
|
for (int i = 0; i < 5; i++) {
|
||||||
ASSERT_OK(Delete("key" + std::to_string(i)));
|
ASSERT_OK(Delete("key" + ToString(i)));
|
||||||
}
|
}
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
uint64_t sst_size;
|
uint64_t sst_size;
|
||||||
|
@ -190,10 +190,9 @@ TEST_F(DBRangeDelTest, MaxCompactionBytesCutsOutputFiles) {
|
|||||||
ASSERT_EQ(0, NumTableFilesAtLevel(0));
|
ASSERT_EQ(0, NumTableFilesAtLevel(0));
|
||||||
ASSERT_EQ(NumTableFilesAtLevel(2), 2);
|
ASSERT_EQ(NumTableFilesAtLevel(2), 2);
|
||||||
|
|
||||||
ASSERT_OK(
|
ASSERT_OK(db_->SetOptions(
|
||||||
db_->SetOptions(db_->DefaultColumnFamily(),
|
db_->DefaultColumnFamily(),
|
||||||
{{"target_file_size_base",
|
{{"target_file_size_base", ToString(100 * opts.max_compaction_bytes)}}));
|
||||||
std::to_string(100 * opts.max_compaction_bytes)}}));
|
|
||||||
|
|
||||||
// It spans the whole key-range, thus will be included in all output files
|
// It spans the whole key-range, thus will be included in all output files
|
||||||
ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(),
|
ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(),
|
||||||
@ -501,8 +500,7 @@ TEST_F(DBRangeDelTest, ValidUniversalSubcompactionBoundaries) {
|
|||||||
1 /* input_level */, 2 /* output_level */, CompactRangeOptions(),
|
1 /* input_level */, 2 /* output_level */, CompactRangeOptions(),
|
||||||
nullptr /* begin */, nullptr /* end */, true /* exclusive */,
|
nullptr /* begin */, nullptr /* end */, true /* exclusive */,
|
||||||
true /* disallow_trivial_move */,
|
true /* disallow_trivial_move */,
|
||||||
std::numeric_limits<uint64_t>::max() /* max_file_num_to_ignore */,
|
port::kMaxUint64 /* max_file_num_to_ignore */, "" /*trim_ts*/));
|
||||||
"" /*trim_ts*/));
|
|
||||||
}
|
}
|
||||||
#endif // ROCKSDB_LITE
|
#endif // ROCKSDB_LITE
|
||||||
|
|
||||||
|
@ -212,20 +212,20 @@ TEST_F(DBSecondaryTest, InternalCompactionMultiLevels) {
|
|||||||
const int kRangeL2 = 10;
|
const int kRangeL2 = 10;
|
||||||
const int kRangeL1 = 30;
|
const int kRangeL1 = 30;
|
||||||
for (int i = 0; i < 10; i++) {
|
for (int i = 0; i < 10; i++) {
|
||||||
ASSERT_OK(Put(Key(i * kRangeL2), "value" + std::to_string(i)));
|
ASSERT_OK(Put(Key(i * kRangeL2), "value" + ToString(i)));
|
||||||
ASSERT_OK(Put(Key((i + 1) * kRangeL2 - 1), "value" + std::to_string(i)));
|
ASSERT_OK(Put(Key((i + 1) * kRangeL2 - 1), "value" + ToString(i)));
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
}
|
}
|
||||||
MoveFilesToLevel(2);
|
MoveFilesToLevel(2);
|
||||||
for (int i = 0; i < 5; i++) {
|
for (int i = 0; i < 5; i++) {
|
||||||
ASSERT_OK(Put(Key(i * kRangeL1), "value" + std::to_string(i)));
|
ASSERT_OK(Put(Key(i * kRangeL1), "value" + ToString(i)));
|
||||||
ASSERT_OK(Put(Key((i + 1) * kRangeL1 - 1), "value" + std::to_string(i)));
|
ASSERT_OK(Put(Key((i + 1) * kRangeL1 - 1), "value" + ToString(i)));
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
}
|
}
|
||||||
MoveFilesToLevel(1);
|
MoveFilesToLevel(1);
|
||||||
for (int i = 0; i < 4; i++) {
|
for (int i = 0; i < 4; i++) {
|
||||||
ASSERT_OK(Put(Key(i * 30), "value" + std::to_string(i)));
|
ASSERT_OK(Put(Key(i * 30), "value" + ToString(i)));
|
||||||
ASSERT_OK(Put(Key(i * 30 + 50), "value" + std::to_string(i)));
|
ASSERT_OK(Put(Key(i * 30 + 50), "value" + ToString(i)));
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -280,58 +280,6 @@ TEST_F(DBSSTTest, DeleteObsoleteFilesPendingOutputs) {
|
|||||||
listener->VerifyMatchedCount(1);
|
listener->VerifyMatchedCount(1);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Test that producing an empty .sst file does not write it out to
|
|
||||||
// disk, and that the DeleteFile() env method is not called for
|
|
||||||
// removing the non-existing file later.
|
|
||||||
TEST_F(DBSSTTest, DeleteFileNotCalledForNotCreatedSSTFile) {
|
|
||||||
Options options = CurrentOptions();
|
|
||||||
options.env = env_;
|
|
||||||
|
|
||||||
OnFileDeletionListener* listener = new OnFileDeletionListener();
|
|
||||||
options.listeners.emplace_back(listener);
|
|
||||||
|
|
||||||
Reopen(options);
|
|
||||||
|
|
||||||
// Flush the empty database.
|
|
||||||
ASSERT_OK(Flush());
|
|
||||||
ASSERT_EQ("", FilesPerLevel(0));
|
|
||||||
|
|
||||||
// We expect no .sst files.
|
|
||||||
std::vector<LiveFileMetaData> metadata;
|
|
||||||
db_->GetLiveFilesMetaData(&metadata);
|
|
||||||
ASSERT_EQ(metadata.size(), 0U);
|
|
||||||
|
|
||||||
// We expect no file deletions.
|
|
||||||
listener->VerifyMatchedCount(0);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Test that producing a non-empty .sst file does write it out to
|
|
||||||
// disk, and that the DeleteFile() env method is not called for removing
|
|
||||||
// the file later.
|
|
||||||
TEST_F(DBSSTTest, DeleteFileNotCalledForCreatedSSTFile) {
|
|
||||||
Options options = CurrentOptions();
|
|
||||||
options.env = env_;
|
|
||||||
|
|
||||||
OnFileDeletionListener* listener = new OnFileDeletionListener();
|
|
||||||
options.listeners.emplace_back(listener);
|
|
||||||
|
|
||||||
Reopen(options);
|
|
||||||
|
|
||||||
ASSERT_OK(Put("pika", "choo"));
|
|
||||||
|
|
||||||
// Flush the non-empty database.
|
|
||||||
ASSERT_OK(Flush());
|
|
||||||
ASSERT_EQ("1", FilesPerLevel(0));
|
|
||||||
|
|
||||||
// We expect 1 .sst files.
|
|
||||||
std::vector<LiveFileMetaData> metadata;
|
|
||||||
db_->GetLiveFilesMetaData(&metadata);
|
|
||||||
ASSERT_EQ(metadata.size(), 1U);
|
|
||||||
|
|
||||||
// We expect no file deletions.
|
|
||||||
listener->VerifyMatchedCount(0);
|
|
||||||
}
|
|
||||||
|
|
||||||
TEST_F(DBSSTTest, DBWithSstFileManager) {
|
TEST_F(DBSSTTest, DBWithSstFileManager) {
|
||||||
std::shared_ptr<SstFileManager> sst_file_manager(NewSstFileManager(env_));
|
std::shared_ptr<SstFileManager> sst_file_manager(NewSstFileManager(env_));
|
||||||
auto sfm = static_cast<SstFileManagerImpl*>(sst_file_manager.get());
|
auto sfm = static_cast<SstFileManagerImpl*>(sst_file_manager.get());
|
||||||
@ -999,7 +947,7 @@ TEST_F(DBSSTTest, DeleteSchedulerMultipleDBPaths) {
|
|||||||
|
|
||||||
// Create 4 files in L0
|
// Create 4 files in L0
|
||||||
for (int i = 0; i < 4; i++) {
|
for (int i = 0; i < 4; i++) {
|
||||||
ASSERT_OK(Put("Key" + std::to_string(i), DummyString(1024, 'A'), wo));
|
ASSERT_OK(Put("Key" + ToString(i), DummyString(1024, 'A'), wo));
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
}
|
}
|
||||||
// We created 4 sst files in L0
|
// We created 4 sst files in L0
|
||||||
@ -1015,7 +963,7 @@ TEST_F(DBSSTTest, DeleteSchedulerMultipleDBPaths) {
|
|||||||
|
|
||||||
// Create 4 files in L0
|
// Create 4 files in L0
|
||||||
for (int i = 4; i < 8; i++) {
|
for (int i = 4; i < 8; i++) {
|
||||||
ASSERT_OK(Put("Key" + std::to_string(i), DummyString(1024, 'B'), wo));
|
ASSERT_OK(Put("Key" + ToString(i), DummyString(1024, 'B'), wo));
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
}
|
}
|
||||||
ASSERT_EQ("4,1", FilesPerLevel(0));
|
ASSERT_EQ("4,1", FilesPerLevel(0));
|
||||||
@ -1061,7 +1009,7 @@ TEST_F(DBSSTTest, DestroyDBWithRateLimitedDelete) {
|
|||||||
|
|
||||||
// Create 4 files in L0
|
// Create 4 files in L0
|
||||||
for (int i = 0; i < 4; i++) {
|
for (int i = 0; i < 4; i++) {
|
||||||
ASSERT_OK(Put("Key" + std::to_string(i), DummyString(1024, 'A')));
|
ASSERT_OK(Put("Key" + ToString(i), DummyString(1024, 'A')));
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
}
|
}
|
||||||
// We created 4 sst files in L0
|
// We created 4 sst files in L0
|
||||||
@ -1530,7 +1478,7 @@ TEST_F(DBSSTTest, GetTotalSstFilesSize) {
|
|||||||
// Generate 5 files in L0
|
// Generate 5 files in L0
|
||||||
for (int i = 0; i < 5; i++) {
|
for (int i = 0; i < 5; i++) {
|
||||||
for (int j = 0; j < 10; j++) {
|
for (int j = 0; j < 10; j++) {
|
||||||
std::string val = "val_file_" + std::to_string(i);
|
std::string val = "val_file_" + ToString(i);
|
||||||
ASSERT_OK(Put(Key(j), val));
|
ASSERT_OK(Put(Key(j), val));
|
||||||
}
|
}
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
|
@ -84,8 +84,7 @@ TEST_F(DBTablePropertiesTest, GetPropertiesOfAllTablesTest) {
|
|||||||
}
|
}
|
||||||
// Build file
|
// Build file
|
||||||
for (int i = 0; i < 10 + table; ++i) {
|
for (int i = 0; i < 10 + table; ++i) {
|
||||||
ASSERT_OK(
|
ASSERT_OK(db_->Put(WriteOptions(), ToString(table * 100 + i), "val"));
|
||||||
db_->Put(WriteOptions(), std::to_string(table * 100 + i), "val"));
|
|
||||||
}
|
}
|
||||||
ASSERT_OK(db_->Flush(FlushOptions()));
|
ASSERT_OK(db_->Flush(FlushOptions()));
|
||||||
}
|
}
|
||||||
@ -114,7 +113,7 @@ TEST_F(DBTablePropertiesTest, GetPropertiesOfAllTablesTest) {
|
|||||||
// fetch key from 1st and 2nd table, which will internally place that table to
|
// fetch key from 1st and 2nd table, which will internally place that table to
|
||||||
// the table cache.
|
// the table cache.
|
||||||
for (int i = 0; i < 2; ++i) {
|
for (int i = 0; i < 2; ++i) {
|
||||||
Get(std::to_string(i * 100 + 0));
|
Get(ToString(i * 100 + 0));
|
||||||
}
|
}
|
||||||
|
|
||||||
VerifyTableProperties(db_, 10 + 11 + 12 + 13);
|
VerifyTableProperties(db_, 10 + 11 + 12 + 13);
|
||||||
@ -123,7 +122,7 @@ TEST_F(DBTablePropertiesTest, GetPropertiesOfAllTablesTest) {
|
|||||||
Reopen(options);
|
Reopen(options);
|
||||||
// fetch key from all tables, which will place them in table cache.
|
// fetch key from all tables, which will place them in table cache.
|
||||||
for (int i = 0; i < 4; ++i) {
|
for (int i = 0; i < 4; ++i) {
|
||||||
Get(std::to_string(i * 100 + 0));
|
Get(ToString(i * 100 + 0));
|
||||||
}
|
}
|
||||||
VerifyTableProperties(db_, 10 + 11 + 12 + 13);
|
VerifyTableProperties(db_, 10 + 11 + 12 + 13);
|
||||||
|
|
||||||
@ -157,7 +156,7 @@ TEST_F(DBTablePropertiesTest, GetPropertiesOfAllTablesTest) {
|
|||||||
} else {
|
} else {
|
||||||
bool found_corruption = false;
|
bool found_corruption = false;
|
||||||
for (int i = 0; i < 4; ++i) {
|
for (int i = 0; i < 4; ++i) {
|
||||||
std::string result = Get(std::to_string(i * 100 + 0));
|
std::string result = Get(ToString(i * 100 + 0));
|
||||||
if (result.find_first_of("Corruption: block checksum mismatch") !=
|
if (result.find_first_of("Corruption: block checksum mismatch") !=
|
||||||
std::string::npos) {
|
std::string::npos) {
|
||||||
found_corruption = true;
|
found_corruption = true;
|
||||||
@ -188,7 +187,7 @@ TEST_F(DBTablePropertiesTest, InvalidIgnored) {
|
|||||||
|
|
||||||
// Build file
|
// Build file
|
||||||
for (int i = 0; i < 10; ++i) {
|
for (int i = 0; i < 10; ++i) {
|
||||||
ASSERT_OK(db_->Put(WriteOptions(), std::to_string(i), "val"));
|
ASSERT_OK(db_->Put(WriteOptions(), ToString(i), "val"));
|
||||||
}
|
}
|
||||||
ASSERT_OK(db_->Flush(FlushOptions()));
|
ASSERT_OK(db_->Flush(FlushOptions()));
|
||||||
|
|
||||||
|
127
db/db_test.cc
127
db/db_test.cc
@ -2734,7 +2734,7 @@ TEST_P(MultiThreadedDBTest, MultiThreaded) {
|
|||||||
Options options = CurrentOptions(options_override);
|
Options options = CurrentOptions(options_override);
|
||||||
std::vector<std::string> cfs;
|
std::vector<std::string> cfs;
|
||||||
for (int i = 1; i < kColumnFamilies; ++i) {
|
for (int i = 1; i < kColumnFamilies; ++i) {
|
||||||
cfs.push_back(std::to_string(i));
|
cfs.push_back(ToString(i));
|
||||||
}
|
}
|
||||||
Reopen(options);
|
Reopen(options);
|
||||||
CreateAndReopenWithCF(cfs, options);
|
CreateAndReopenWithCF(cfs, options);
|
||||||
@ -2786,7 +2786,7 @@ static void GCThreadBody(void* arg) {
|
|||||||
WriteOptions wo;
|
WriteOptions wo;
|
||||||
|
|
||||||
for (int i = 0; i < kGCNumKeys; ++i) {
|
for (int i = 0; i < kGCNumKeys; ++i) {
|
||||||
std::string kv(std::to_string(i + id * kGCNumKeys));
|
std::string kv(ToString(i + id * kGCNumKeys));
|
||||||
ASSERT_OK(db->Put(wo, kv, kv));
|
ASSERT_OK(db->Put(wo, kv, kv));
|
||||||
}
|
}
|
||||||
t->done = true;
|
t->done = true;
|
||||||
@ -2822,7 +2822,7 @@ TEST_F(DBTest, GroupCommitTest) {
|
|||||||
|
|
||||||
std::vector<std::string> expected_db;
|
std::vector<std::string> expected_db;
|
||||||
for (int i = 0; i < kGCNumThreads * kGCNumKeys; ++i) {
|
for (int i = 0; i < kGCNumThreads * kGCNumKeys; ++i) {
|
||||||
expected_db.push_back(std::to_string(i));
|
expected_db.push_back(ToString(i));
|
||||||
}
|
}
|
||||||
std::sort(expected_db.begin(), expected_db.end());
|
std::sort(expected_db.begin(), expected_db.end());
|
||||||
|
|
||||||
@ -3591,7 +3591,7 @@ TEST_P(DBTestWithParam, FIFOCompactionTest) {
|
|||||||
Random rnd(301);
|
Random rnd(301);
|
||||||
for (int i = 0; i < 6; ++i) {
|
for (int i = 0; i < 6; ++i) {
|
||||||
for (int j = 0; j < 110; ++j) {
|
for (int j = 0; j < 110; ++j) {
|
||||||
ASSERT_OK(Put(std::to_string(i * 100 + j), rnd.RandomString(980)));
|
ASSERT_OK(Put(ToString(i * 100 + j), rnd.RandomString(980)));
|
||||||
}
|
}
|
||||||
// flush should happen here
|
// flush should happen here
|
||||||
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
|
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
|
||||||
@ -3607,7 +3607,7 @@ TEST_P(DBTestWithParam, FIFOCompactionTest) {
|
|||||||
ASSERT_EQ(NumTableFilesAtLevel(0), 5);
|
ASSERT_EQ(NumTableFilesAtLevel(0), 5);
|
||||||
for (int i = 0; i < 50; ++i) {
|
for (int i = 0; i < 50; ++i) {
|
||||||
// these keys should be deleted in previous compaction
|
// these keys should be deleted in previous compaction
|
||||||
ASSERT_EQ("NOT_FOUND", Get(std::to_string(i)));
|
ASSERT_EQ("NOT_FOUND", Get(ToString(i)));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -3629,7 +3629,7 @@ TEST_F(DBTest, FIFOCompactionTestWithCompaction) {
|
|||||||
for (int i = 0; i < 60; i++) {
|
for (int i = 0; i < 60; i++) {
|
||||||
// Generate and flush a file about 20KB.
|
// Generate and flush a file about 20KB.
|
||||||
for (int j = 0; j < 20; j++) {
|
for (int j = 0; j < 20; j++) {
|
||||||
ASSERT_OK(Put(std::to_string(i * 20 + j), rnd.RandomString(980)));
|
ASSERT_OK(Put(ToString(i * 20 + j), rnd.RandomString(980)));
|
||||||
}
|
}
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
||||||
@ -3640,7 +3640,7 @@ TEST_F(DBTest, FIFOCompactionTestWithCompaction) {
|
|||||||
for (int i = 0; i < 60; i++) {
|
for (int i = 0; i < 60; i++) {
|
||||||
// Generate and flush a file about 20KB.
|
// Generate and flush a file about 20KB.
|
||||||
for (int j = 0; j < 20; j++) {
|
for (int j = 0; j < 20; j++) {
|
||||||
ASSERT_OK(Put(std::to_string(i * 20 + j + 2000), rnd.RandomString(980)));
|
ASSERT_OK(Put(ToString(i * 20 + j + 2000), rnd.RandomString(980)));
|
||||||
}
|
}
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
||||||
@ -3670,27 +3670,27 @@ TEST_F(DBTest, FIFOCompactionStyleWithCompactionAndDelete) {
|
|||||||
Random rnd(301);
|
Random rnd(301);
|
||||||
for (int i = 0; i < 3; i++) {
|
for (int i = 0; i < 3; i++) {
|
||||||
// Each file contains a different key which will be dropped later.
|
// Each file contains a different key which will be dropped later.
|
||||||
ASSERT_OK(Put("a" + std::to_string(i), rnd.RandomString(500)));
|
ASSERT_OK(Put("a" + ToString(i), rnd.RandomString(500)));
|
||||||
ASSERT_OK(Put("key" + std::to_string(i), ""));
|
ASSERT_OK(Put("key" + ToString(i), ""));
|
||||||
ASSERT_OK(Put("z" + std::to_string(i), rnd.RandomString(500)));
|
ASSERT_OK(Put("z" + ToString(i), rnd.RandomString(500)));
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
||||||
}
|
}
|
||||||
ASSERT_EQ(NumTableFilesAtLevel(0), 1);
|
ASSERT_EQ(NumTableFilesAtLevel(0), 1);
|
||||||
for (int i = 0; i < 3; i++) {
|
for (int i = 0; i < 3; i++) {
|
||||||
ASSERT_EQ("", Get("key" + std::to_string(i)));
|
ASSERT_EQ("", Get("key" + ToString(i)));
|
||||||
}
|
}
|
||||||
for (int i = 0; i < 3; i++) {
|
for (int i = 0; i < 3; i++) {
|
||||||
// Each file contains a different key which will be dropped later.
|
// Each file contains a different key which will be dropped later.
|
||||||
ASSERT_OK(Put("a" + std::to_string(i), rnd.RandomString(500)));
|
ASSERT_OK(Put("a" + ToString(i), rnd.RandomString(500)));
|
||||||
ASSERT_OK(Delete("key" + std::to_string(i)));
|
ASSERT_OK(Delete("key" + ToString(i)));
|
||||||
ASSERT_OK(Put("z" + std::to_string(i), rnd.RandomString(500)));
|
ASSERT_OK(Put("z" + ToString(i), rnd.RandomString(500)));
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
||||||
}
|
}
|
||||||
ASSERT_EQ(NumTableFilesAtLevel(0), 2);
|
ASSERT_EQ(NumTableFilesAtLevel(0), 2);
|
||||||
for (int i = 0; i < 3; i++) {
|
for (int i = 0; i < 3; i++) {
|
||||||
ASSERT_EQ("NOT_FOUND", Get("key" + std::to_string(i)));
|
ASSERT_EQ("NOT_FOUND", Get("key" + ToString(i)));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -3759,7 +3759,7 @@ TEST_F(DBTest, FIFOCompactionWithTTLTest) {
|
|||||||
for (int i = 0; i < 10; i++) {
|
for (int i = 0; i < 10; i++) {
|
||||||
// Generate and flush a file about 10KB.
|
// Generate and flush a file about 10KB.
|
||||||
for (int j = 0; j < 10; j++) {
|
for (int j = 0; j < 10; j++) {
|
||||||
ASSERT_OK(Put(std::to_string(i * 20 + j), rnd.RandomString(980)));
|
ASSERT_OK(Put(ToString(i * 20 + j), rnd.RandomString(980)));
|
||||||
}
|
}
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
||||||
@ -3791,7 +3791,7 @@ TEST_F(DBTest, FIFOCompactionWithTTLTest) {
|
|||||||
for (int i = 0; i < 10; i++) {
|
for (int i = 0; i < 10; i++) {
|
||||||
// Generate and flush a file about 10KB.
|
// Generate and flush a file about 10KB.
|
||||||
for (int j = 0; j < 10; j++) {
|
for (int j = 0; j < 10; j++) {
|
||||||
ASSERT_OK(Put(std::to_string(i * 20 + j), rnd.RandomString(980)));
|
ASSERT_OK(Put(ToString(i * 20 + j), rnd.RandomString(980)));
|
||||||
}
|
}
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
||||||
@ -3807,7 +3807,7 @@ TEST_F(DBTest, FIFOCompactionWithTTLTest) {
|
|||||||
// Create 1 more file to trigger TTL compaction. The old files are dropped.
|
// Create 1 more file to trigger TTL compaction. The old files are dropped.
|
||||||
for (int i = 0; i < 1; i++) {
|
for (int i = 0; i < 1; i++) {
|
||||||
for (int j = 0; j < 10; j++) {
|
for (int j = 0; j < 10; j++) {
|
||||||
ASSERT_OK(Put(std::to_string(i * 20 + j), rnd.RandomString(980)));
|
ASSERT_OK(Put(ToString(i * 20 + j), rnd.RandomString(980)));
|
||||||
}
|
}
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
}
|
}
|
||||||
@ -3833,7 +3833,7 @@ TEST_F(DBTest, FIFOCompactionWithTTLTest) {
|
|||||||
for (int i = 0; i < 3; i++) {
|
for (int i = 0; i < 3; i++) {
|
||||||
// Generate and flush a file about 10KB.
|
// Generate and flush a file about 10KB.
|
||||||
for (int j = 0; j < 10; j++) {
|
for (int j = 0; j < 10; j++) {
|
||||||
ASSERT_OK(Put(std::to_string(i * 20 + j), rnd.RandomString(980)));
|
ASSERT_OK(Put(ToString(i * 20 + j), rnd.RandomString(980)));
|
||||||
}
|
}
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
||||||
@ -3848,7 +3848,7 @@ TEST_F(DBTest, FIFOCompactionWithTTLTest) {
|
|||||||
|
|
||||||
for (int i = 0; i < 5; i++) {
|
for (int i = 0; i < 5; i++) {
|
||||||
for (int j = 0; j < 140; j++) {
|
for (int j = 0; j < 140; j++) {
|
||||||
ASSERT_OK(Put(std::to_string(i * 20 + j), rnd.RandomString(980)));
|
ASSERT_OK(Put(ToString(i * 20 + j), rnd.RandomString(980)));
|
||||||
}
|
}
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
||||||
@ -3871,7 +3871,7 @@ TEST_F(DBTest, FIFOCompactionWithTTLTest) {
|
|||||||
for (int i = 0; i < 10; i++) {
|
for (int i = 0; i < 10; i++) {
|
||||||
// Generate and flush a file about 10KB.
|
// Generate and flush a file about 10KB.
|
||||||
for (int j = 0; j < 10; j++) {
|
for (int j = 0; j < 10; j++) {
|
||||||
ASSERT_OK(Put(std::to_string(i * 20 + j), rnd.RandomString(980)));
|
ASSERT_OK(Put(ToString(i * 20 + j), rnd.RandomString(980)));
|
||||||
}
|
}
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
||||||
@ -3890,7 +3890,7 @@ TEST_F(DBTest, FIFOCompactionWithTTLTest) {
|
|||||||
// Create 10 more files. The old 5 files are dropped as their ttl expired.
|
// Create 10 more files. The old 5 files are dropped as their ttl expired.
|
||||||
for (int i = 0; i < 10; i++) {
|
for (int i = 0; i < 10; i++) {
|
||||||
for (int j = 0; j < 10; j++) {
|
for (int j = 0; j < 10; j++) {
|
||||||
ASSERT_OK(Put(std::to_string(i * 20 + j), rnd.RandomString(980)));
|
ASSERT_OK(Put(ToString(i * 20 + j), rnd.RandomString(980)));
|
||||||
}
|
}
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
||||||
@ -3915,7 +3915,7 @@ TEST_F(DBTest, FIFOCompactionWithTTLTest) {
|
|||||||
for (int i = 0; i < 60; i++) {
|
for (int i = 0; i < 60; i++) {
|
||||||
// Generate and flush a file about 20KB.
|
// Generate and flush a file about 20KB.
|
||||||
for (int j = 0; j < 20; j++) {
|
for (int j = 0; j < 20; j++) {
|
||||||
ASSERT_OK(Put(std::to_string(i * 20 + j), rnd.RandomString(980)));
|
ASSERT_OK(Put(ToString(i * 20 + j), rnd.RandomString(980)));
|
||||||
}
|
}
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
||||||
@ -3926,8 +3926,7 @@ TEST_F(DBTest, FIFOCompactionWithTTLTest) {
|
|||||||
for (int i = 0; i < 60; i++) {
|
for (int i = 0; i < 60; i++) {
|
||||||
// Generate and flush a file about 20KB.
|
// Generate and flush a file about 20KB.
|
||||||
for (int j = 0; j < 20; j++) {
|
for (int j = 0; j < 20; j++) {
|
||||||
ASSERT_OK(
|
ASSERT_OK(Put(ToString(i * 20 + j + 2000), rnd.RandomString(980)));
|
||||||
Put(std::to_string(i * 20 + j + 2000), rnd.RandomString(980)));
|
|
||||||
}
|
}
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
||||||
@ -4208,7 +4207,7 @@ TEST_F(DBTest, ConcurrentFlushWAL) {
|
|||||||
std::vector<port::Thread> threads;
|
std::vector<port::Thread> threads;
|
||||||
threads.emplace_back([&] {
|
threads.emplace_back([&] {
|
||||||
for (size_t i = 0; i < cnt; i++) {
|
for (size_t i = 0; i < cnt; i++) {
|
||||||
auto istr = std::to_string(i);
|
auto istr = ToString(i);
|
||||||
ASSERT_OK(db_->Put(wopt, db_->DefaultColumnFamily(), "a" + istr,
|
ASSERT_OK(db_->Put(wopt, db_->DefaultColumnFamily(), "a" + istr,
|
||||||
"b" + istr));
|
"b" + istr));
|
||||||
}
|
}
|
||||||
@ -4216,7 +4215,7 @@ TEST_F(DBTest, ConcurrentFlushWAL) {
|
|||||||
if (two_write_queues) {
|
if (two_write_queues) {
|
||||||
threads.emplace_back([&] {
|
threads.emplace_back([&] {
|
||||||
for (size_t i = cnt; i < 2 * cnt; i++) {
|
for (size_t i = cnt; i < 2 * cnt; i++) {
|
||||||
auto istr = std::to_string(i);
|
auto istr = ToString(i);
|
||||||
WriteBatch batch;
|
WriteBatch batch;
|
||||||
ASSERT_OK(batch.Put("a" + istr, "b" + istr));
|
ASSERT_OK(batch.Put("a" + istr, "b" + istr));
|
||||||
ASSERT_OK(
|
ASSERT_OK(
|
||||||
@ -4237,7 +4236,7 @@ TEST_F(DBTest, ConcurrentFlushWAL) {
|
|||||||
Reopen(options);
|
Reopen(options);
|
||||||
for (size_t i = 0; i < cnt; i++) {
|
for (size_t i = 0; i < cnt; i++) {
|
||||||
PinnableSlice pval;
|
PinnableSlice pval;
|
||||||
auto istr = std::to_string(i);
|
auto istr = ToString(i);
|
||||||
ASSERT_OK(
|
ASSERT_OK(
|
||||||
db_->Get(ropt, db_->DefaultColumnFamily(), "a" + istr, &pval));
|
db_->Get(ropt, db_->DefaultColumnFamily(), "a" + istr, &pval));
|
||||||
ASSERT_TRUE(pval == ("b" + istr));
|
ASSERT_TRUE(pval == ("b" + istr));
|
||||||
@ -4260,7 +4259,7 @@ TEST_F(DBTest, ManualFlushWalAndWriteRace) {
|
|||||||
|
|
||||||
port::Thread writeThread([&]() {
|
port::Thread writeThread([&]() {
|
||||||
for (int i = 0; i < 100; i++) {
|
for (int i = 0; i < 100; i++) {
|
||||||
auto istr = std::to_string(i);
|
auto istr = ToString(i);
|
||||||
ASSERT_OK(dbfull()->Put(wopts, "key_" + istr, "value_" + istr));
|
ASSERT_OK(dbfull()->Put(wopts, "key_" + istr, "value_" + istr));
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
@ -4608,7 +4607,7 @@ TEST_P(DBTestWithParam, ThreadStatusSingleCompaction) {
|
|||||||
// The Put Phase.
|
// The Put Phase.
|
||||||
for (int file = 0; file < kNumL0Files; ++file) {
|
for (int file = 0; file < kNumL0Files; ++file) {
|
||||||
for (int key = 0; key < kEntriesPerBuffer; ++key) {
|
for (int key = 0; key < kEntriesPerBuffer; ++key) {
|
||||||
ASSERT_OK(Put(std::to_string(key + file * kEntriesPerBuffer),
|
ASSERT_OK(Put(ToString(key + file * kEntriesPerBuffer),
|
||||||
rnd.RandomString(kTestValueSize)));
|
rnd.RandomString(kTestValueSize)));
|
||||||
}
|
}
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
@ -4759,7 +4758,7 @@ TEST_P(DBTestWithParam, PreShutdownMultipleCompaction) {
|
|||||||
int operation_count[ThreadStatus::NUM_OP_TYPES] = {0};
|
int operation_count[ThreadStatus::NUM_OP_TYPES] = {0};
|
||||||
for (int file = 0; file < 16 * kNumL0Files; ++file) {
|
for (int file = 0; file < 16 * kNumL0Files; ++file) {
|
||||||
for (int k = 0; k < kEntriesPerBuffer; ++k) {
|
for (int k = 0; k < kEntriesPerBuffer; ++k) {
|
||||||
ASSERT_OK(Put(std::to_string(key++), rnd.RandomString(kTestValueSize)));
|
ASSERT_OK(Put(ToString(key++), rnd.RandomString(kTestValueSize)));
|
||||||
}
|
}
|
||||||
|
|
||||||
ASSERT_OK(env_->GetThreadList(&thread_list));
|
ASSERT_OK(env_->GetThreadList(&thread_list));
|
||||||
@ -4846,7 +4845,7 @@ TEST_P(DBTestWithParam, PreShutdownCompactionMiddle) {
|
|||||||
int operation_count[ThreadStatus::NUM_OP_TYPES] = {0};
|
int operation_count[ThreadStatus::NUM_OP_TYPES] = {0};
|
||||||
for (int file = 0; file < 16 * kNumL0Files; ++file) {
|
for (int file = 0; file < 16 * kNumL0Files; ++file) {
|
||||||
for (int k = 0; k < kEntriesPerBuffer; ++k) {
|
for (int k = 0; k < kEntriesPerBuffer; ++k) {
|
||||||
ASSERT_OK(Put(std::to_string(key++), rnd.RandomString(kTestValueSize)));
|
ASSERT_OK(Put(ToString(key++), rnd.RandomString(kTestValueSize)));
|
||||||
}
|
}
|
||||||
|
|
||||||
ASSERT_OK(env_->GetThreadList(&thread_list));
|
ASSERT_OK(env_->GetThreadList(&thread_list));
|
||||||
@ -5157,9 +5156,8 @@ TEST_F(DBTest, DynamicCompactionOptions) {
|
|||||||
// Writing to 64KB L0 files should trigger a compaction. Since these
|
// Writing to 64KB L0 files should trigger a compaction. Since these
|
||||||
// 2 L0 files have the same key range, compaction merge them and should
|
// 2 L0 files have the same key range, compaction merge them and should
|
||||||
// result in 2 32KB L1 files.
|
// result in 2 32KB L1 files.
|
||||||
ASSERT_OK(
|
ASSERT_OK(dbfull()->SetOptions({{"level0_file_num_compaction_trigger", "2"},
|
||||||
dbfull()->SetOptions({{"level0_file_num_compaction_trigger", "2"},
|
{"target_file_size_base", ToString(k32KB)}}));
|
||||||
{"target_file_size_base", std::to_string(k32KB)}}));
|
|
||||||
|
|
||||||
gen_l0_kb(0, 64, 1);
|
gen_l0_kb(0, 64, 1);
|
||||||
ASSERT_EQ("1,1", FilesPerLevel());
|
ASSERT_EQ("1,1", FilesPerLevel());
|
||||||
@ -5178,8 +5176,8 @@ TEST_F(DBTest, DynamicCompactionOptions) {
|
|||||||
// Increase level base size to 256KB and write enough data that will
|
// Increase level base size to 256KB and write enough data that will
|
||||||
// fill L1 and L2. L1 size should be around 256KB while L2 size should be
|
// fill L1 and L2. L1 size should be around 256KB while L2 size should be
|
||||||
// around 256KB x 4.
|
// around 256KB x 4.
|
||||||
ASSERT_OK(dbfull()->SetOptions(
|
ASSERT_OK(
|
||||||
{{"max_bytes_for_level_base", std::to_string(k1MB)}}));
|
dbfull()->SetOptions({{"max_bytes_for_level_base", ToString(k1MB)}}));
|
||||||
|
|
||||||
// writing 96 x 64KB => 6 * 1024KB
|
// writing 96 x 64KB => 6 * 1024KB
|
||||||
// (L1 + L2) = (1 + 4) * 1024KB
|
// (L1 + L2) = (1 + 4) * 1024KB
|
||||||
@ -5198,9 +5196,9 @@ TEST_F(DBTest, DynamicCompactionOptions) {
|
|||||||
// max_bytes_for_level_base. Now, reduce both mulitplier and level base,
|
// max_bytes_for_level_base. Now, reduce both mulitplier and level base,
|
||||||
// After filling enough data that can fit in L1 - L3, we should see L1 size
|
// After filling enough data that can fit in L1 - L3, we should see L1 size
|
||||||
// reduces to 128KB from 256KB which was asserted previously. Same for L2.
|
// reduces to 128KB from 256KB which was asserted previously. Same for L2.
|
||||||
ASSERT_OK(dbfull()->SetOptions(
|
ASSERT_OK(
|
||||||
{{"max_bytes_for_level_multiplier", "2"},
|
dbfull()->SetOptions({{"max_bytes_for_level_multiplier", "2"},
|
||||||
{"max_bytes_for_level_base", std::to_string(k128KB)}}));
|
{"max_bytes_for_level_base", ToString(k128KB)}}));
|
||||||
|
|
||||||
// writing 20 x 64KB = 10 x 128KB
|
// writing 20 x 64KB = 10 x 128KB
|
||||||
// (L1 + L2 + L3) = (1 + 2 + 4) * 128KB
|
// (L1 + L2 + L3) = (1 + 2 + 4) * 128KB
|
||||||
@ -5856,7 +5854,7 @@ TEST_P(DBTestWithParam, FilterCompactionTimeTest) {
|
|||||||
// put some data
|
// put some data
|
||||||
for (int table = 0; table < 4; ++table) {
|
for (int table = 0; table < 4; ++table) {
|
||||||
for (int i = 0; i < 10 + table; ++i) {
|
for (int i = 0; i < 10 + table; ++i) {
|
||||||
ASSERT_OK(Put(std::to_string(table * 100 + i), "val"));
|
ASSERT_OK(Put(ToString(table * 100 + i), "val"));
|
||||||
++n;
|
++n;
|
||||||
}
|
}
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
@ -6240,7 +6238,7 @@ TEST_F(DBTest, LargeBatchWithColumnFamilies) {
|
|||||||
(write_size / 1024 / 1024), pass);
|
(write_size / 1024 / 1024), pass);
|
||||||
for (;;) {
|
for (;;) {
|
||||||
std::string data(3000, j++ % 127 + 20);
|
std::string data(3000, j++ % 127 + 20);
|
||||||
data += std::to_string(j);
|
data += ToString(j);
|
||||||
ASSERT_OK(batch.Put(handles_[0], Slice(data), Slice(data)));
|
ASSERT_OK(batch.Put(handles_[0], Slice(data), Slice(data)));
|
||||||
if (batch.GetDataSize() > write_size) {
|
if (batch.GetDataSize() > write_size) {
|
||||||
break;
|
break;
|
||||||
@ -7036,6 +7034,51 @@ TEST_F(DBTest, MemoryUsageWithMaxWriteBufferSizeToMaintain) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
TEST_F(DBTest, ManualFlushWithStoppedWritesTest) {
|
||||||
|
Options options = CurrentOptions();
|
||||||
|
|
||||||
|
// Setting a small write buffer size. This will block writes
|
||||||
|
// rather quickly until a flush is made.
|
||||||
|
constexpr unsigned int memtableSize = 1000;
|
||||||
|
options.db_write_buffer_size = memtableSize;
|
||||||
|
options.max_background_flushes = 1;
|
||||||
|
Reopen(options);
|
||||||
|
|
||||||
|
std::atomic<bool> done(false);
|
||||||
|
|
||||||
|
// Will suppress future flushes.
|
||||||
|
// This simulates a situation where the writes will be stopped for any reason.
|
||||||
|
ASSERT_OK(dbfull()->PauseBackgroundWork());
|
||||||
|
|
||||||
|
port::Thread backgroundWriter([&]() {
|
||||||
|
Random rnd(301);
|
||||||
|
// These writes won't finish.
|
||||||
|
ASSERT_OK(Put("k1", rnd.RandomString(memtableSize / 2)));
|
||||||
|
ASSERT_OK(Put("k2", rnd.RandomString(memtableSize / 2)));
|
||||||
|
ASSERT_OK(Put("k3", rnd.RandomString(memtableSize / 2)));
|
||||||
|
ASSERT_OK(Put("k4", rnd.RandomString(memtableSize / 2)));
|
||||||
|
done.store(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
env_->SleepForMicroseconds(1000000 / 2);
|
||||||
|
// make sure thread is stuck waiting for flush.
|
||||||
|
ASSERT_FALSE(done.load());
|
||||||
|
|
||||||
|
FlushOptions flushOptions;
|
||||||
|
flushOptions.wait = false;
|
||||||
|
flushOptions.allow_write_stall = true;
|
||||||
|
|
||||||
|
// This is the main goal of the test. This manual flush should not deadlock
|
||||||
|
// as we use wait=false, and even allow_write_stall=true for extra safety.
|
||||||
|
ASSERT_OK(dbfull()->Flush(flushOptions));
|
||||||
|
|
||||||
|
// This will re-allow background flushes.
|
||||||
|
ASSERT_OK(dbfull()->ContinueBackgroundWork());
|
||||||
|
|
||||||
|
backgroundWriter.join();
|
||||||
|
ASSERT_TRUE(done.load());
|
||||||
|
}
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
} // namespace ROCKSDB_NAMESPACE
|
} // namespace ROCKSDB_NAMESPACE
|
||||||
|
@ -38,9 +38,9 @@ class DBTest2 : public DBTestBase {
|
|||||||
#ifndef ROCKSDB_LITE
|
#ifndef ROCKSDB_LITE
|
||||||
uint64_t GetSstSizeHelper(Temperature temperature) {
|
uint64_t GetSstSizeHelper(Temperature temperature) {
|
||||||
std::string prop;
|
std::string prop;
|
||||||
EXPECT_TRUE(dbfull()->GetProperty(
|
EXPECT_TRUE(
|
||||||
DB::Properties::kLiveSstFilesSizeAtTemperature +
|
dbfull()->GetProperty(DB::Properties::kLiveSstFilesSizeAtTemperature +
|
||||||
std::to_string(static_cast<uint8_t>(temperature)),
|
ToString(static_cast<uint8_t>(temperature)),
|
||||||
&prop));
|
&prop));
|
||||||
return static_cast<uint64_t>(std::atoi(prop.c_str()));
|
return static_cast<uint64_t>(std::atoi(prop.c_str()));
|
||||||
}
|
}
|
||||||
@ -1694,8 +1694,8 @@ class CompactionCompressionListener : public EventListener {
|
|||||||
int bottommost_level = 0;
|
int bottommost_level = 0;
|
||||||
for (int level = 0; level < db->NumberLevels(); level++) {
|
for (int level = 0; level < db->NumberLevels(); level++) {
|
||||||
std::string files_at_level;
|
std::string files_at_level;
|
||||||
ASSERT_TRUE(
|
ASSERT_TRUE(db->GetProperty(
|
||||||
db->GetProperty("rocksdb.num-files-at-level" + std::to_string(level),
|
"rocksdb.num-files-at-level" + ROCKSDB_NAMESPACE::ToString(level),
|
||||||
&files_at_level));
|
&files_at_level));
|
||||||
if (files_at_level != "0") {
|
if (files_at_level != "0") {
|
||||||
bottommost_level = level;
|
bottommost_level = level;
|
||||||
@ -2492,14 +2492,14 @@ TEST_F(DBTest2, TestPerfContextIterCpuTime) {
|
|||||||
|
|
||||||
const size_t kNumEntries = 10;
|
const size_t kNumEntries = 10;
|
||||||
for (size_t i = 0; i < kNumEntries; ++i) {
|
for (size_t i = 0; i < kNumEntries; ++i) {
|
||||||
ASSERT_OK(Put("k" + std::to_string(i), "v" + std::to_string(i)));
|
ASSERT_OK(Put("k" + ToString(i), "v" + ToString(i)));
|
||||||
}
|
}
|
||||||
ASSERT_OK(Flush());
|
ASSERT_OK(Flush());
|
||||||
for (size_t i = 0; i < kNumEntries; ++i) {
|
for (size_t i = 0; i < kNumEntries; ++i) {
|
||||||
ASSERT_EQ("v" + std::to_string(i), Get("k" + std::to_string(i)));
|
ASSERT_EQ("v" + ToString(i), Get("k" + ToString(i)));
|
||||||
}
|
}
|
||||||
std::string last_key = "k" + std::to_string(kNumEntries - 1);
|
std::string last_key = "k" + ToString(kNumEntries - 1);
|
||||||
std::string last_value = "v" + std::to_string(kNumEntries - 1);
|
std::string last_value = "v" + ToString(kNumEntries - 1);
|
||||||
env_->now_cpu_count_.store(0);
|
env_->now_cpu_count_.store(0);
|
||||||
env_->SetMockSleep();
|
env_->SetMockSleep();
|
||||||
|
|
||||||
@ -5553,7 +5553,7 @@ TEST_F(DBTest2, MultiDBParallelOpenTest) {
|
|||||||
Options options = CurrentOptions();
|
Options options = CurrentOptions();
|
||||||
std::vector<std::string> dbnames;
|
std::vector<std::string> dbnames;
|
||||||
for (int i = 0; i < kNumDbs; ++i) {
|
for (int i = 0; i < kNumDbs; ++i) {
|
||||||
dbnames.emplace_back(test::PerThreadDBPath(env_, "db" + std::to_string(i)));
|
dbnames.emplace_back(test::PerThreadDBPath(env_, "db" + ToString(i)));
|
||||||
ASSERT_OK(DestroyDB(dbnames.back(), options));
|
ASSERT_OK(DestroyDB(dbnames.back(), options));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1086,11 +1086,11 @@ int DBTestBase::NumTableFilesAtLevel(int level, int cf) {
|
|||||||
std::string property;
|
std::string property;
|
||||||
if (cf == 0) {
|
if (cf == 0) {
|
||||||
// default cfd
|
// default cfd
|
||||||
EXPECT_TRUE(db_->GetProperty(
|
EXPECT_TRUE(db_->GetProperty("rocksdb.num-files-at-level" + ToString(level),
|
||||||
"rocksdb.num-files-at-level" + std::to_string(level), &property));
|
&property));
|
||||||
} else {
|
} else {
|
||||||
EXPECT_TRUE(db_->GetProperty(
|
EXPECT_TRUE(db_->GetProperty(handles_[cf],
|
||||||
handles_[cf], "rocksdb.num-files-at-level" + std::to_string(level),
|
"rocksdb.num-files-at-level" + ToString(level),
|
||||||
&property));
|
&property));
|
||||||
}
|
}
|
||||||
return atoi(property.c_str());
|
return atoi(property.c_str());
|
||||||
@ -1101,12 +1101,10 @@ double DBTestBase::CompressionRatioAtLevel(int level, int cf) {
|
|||||||
if (cf == 0) {
|
if (cf == 0) {
|
||||||
// default cfd
|
// default cfd
|
||||||
EXPECT_TRUE(db_->GetProperty(
|
EXPECT_TRUE(db_->GetProperty(
|
||||||
"rocksdb.compression-ratio-at-level" + std::to_string(level),
|
"rocksdb.compression-ratio-at-level" + ToString(level), &property));
|
||||||
&property));
|
|
||||||
} else {
|
} else {
|
||||||
EXPECT_TRUE(db_->GetProperty(
|
EXPECT_TRUE(db_->GetProperty(
|
||||||
handles_[cf],
|
handles_[cf], "rocksdb.compression-ratio-at-level" + ToString(level),
|
||||||
"rocksdb.compression-ratio-at-level" + std::to_string(level),
|
|
||||||
&property));
|
&property));
|
||||||
}
|
}
|
||||||
return std::stod(property);
|
return std::stod(property);
|
||||||
|
@ -549,7 +549,7 @@ TEST_P(DBTestUniversalCompaction, CompactFilesOnUniversalCompaction) {
|
|||||||
ASSERT_EQ(options.compaction_style, kCompactionStyleUniversal);
|
ASSERT_EQ(options.compaction_style, kCompactionStyleUniversal);
|
||||||
Random rnd(301);
|
Random rnd(301);
|
||||||
for (int key = 1024 * kEntriesPerBuffer; key >= 0; --key) {
|
for (int key = 1024 * kEntriesPerBuffer; key >= 0; --key) {
|
||||||
ASSERT_OK(Put(1, std::to_string(key), rnd.RandomString(kTestValueSize)));
|
ASSERT_OK(Put(1, ToString(key), rnd.RandomString(kTestValueSize)));
|
||||||
}
|
}
|
||||||
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[1]));
|
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[1]));
|
||||||
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
||||||
|
@ -1009,7 +1009,7 @@ TEST_F(DBWALTest, RecoveryWithLogDataForSomeCFs) {
|
|||||||
if (log_files.size() > 0) {
|
if (log_files.size() > 0) {
|
||||||
earliest_log_nums[i] = log_files[0]->LogNumber();
|
earliest_log_nums[i] = log_files[0]->LogNumber();
|
||||||
} else {
|
} else {
|
||||||
earliest_log_nums[i] = std::numeric_limits<uint64_t>::max();
|
earliest_log_nums[i] = port::kMaxUint64;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// Check at least the first WAL was cleaned up during the recovery.
|
// Check at least the first WAL was cleaned up during the recovery.
|
||||||
@ -1288,7 +1288,7 @@ class RecoveryTestHelper {
|
|||||||
|
|
||||||
WriteBatch batch;
|
WriteBatch batch;
|
||||||
for (int i = 0; i < kKeysPerWALFile; i++) {
|
for (int i = 0; i < kKeysPerWALFile; i++) {
|
||||||
std::string key = "key" + std::to_string((*count)++);
|
std::string key = "key" + ToString((*count)++);
|
||||||
std::string value = test->DummyString(kValueSize);
|
std::string value = test->DummyString(kValueSize);
|
||||||
ASSERT_NE(current_log_writer.get(), nullptr);
|
ASSERT_NE(current_log_writer.get(), nullptr);
|
||||||
uint64_t seq = versions->LastSequence() + 1;
|
uint64_t seq = versions->LastSequence() + 1;
|
||||||
@ -1319,7 +1319,7 @@ class RecoveryTestHelper {
|
|||||||
static size_t GetData(DBWALTestBase* test) {
|
static size_t GetData(DBWALTestBase* test) {
|
||||||
size_t count = 0;
|
size_t count = 0;
|
||||||
for (size_t i = 0; i < kWALFilesCount * kKeysPerWALFile; i++) {
|
for (size_t i = 0; i < kWALFilesCount * kKeysPerWALFile; i++) {
|
||||||
if (test->Get("key" + std::to_string(i)) != "NOT_FOUND") {
|
if (test->Get("key" + ToString(i)) != "NOT_FOUND") {
|
||||||
++count;
|
++count;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -1616,7 +1616,7 @@ TEST_P(DBWALTestWithParams, kPointInTimeRecovery) {
|
|||||||
if (!trunc || corrupt_offset != 0) {
|
if (!trunc || corrupt_offset != 0) {
|
||||||
bool expect_data = true;
|
bool expect_data = true;
|
||||||
for (size_t k = 0; k < maxkeys; ++k) {
|
for (size_t k = 0; k < maxkeys; ++k) {
|
||||||
bool found = Get("key" + std::to_string(k)) != "NOT_FOUND";
|
bool found = Get("key" + ToString(k)) != "NOT_FOUND";
|
||||||
if (expect_data && !found) {
|
if (expect_data && !found) {
|
||||||
expect_data = false;
|
expect_data = false;
|
||||||
}
|
}
|
||||||
@ -1752,7 +1752,7 @@ TEST_F(DBWALTest, RecoverWithoutFlush) {
|
|||||||
size_t count = RecoveryTestHelper::FillData(this, &options);
|
size_t count = RecoveryTestHelper::FillData(this, &options);
|
||||||
auto validateData = [this, count]() {
|
auto validateData = [this, count]() {
|
||||||
for (size_t i = 0; i < count; i++) {
|
for (size_t i = 0; i < count; i++) {
|
||||||
ASSERT_NE(Get("key" + std::to_string(i)), "NOT_FOUND");
|
ASSERT_NE(Get("key" + ToString(i)), "NOT_FOUND");
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
Reopen(options);
|
Reopen(options);
|
||||||
@ -1891,7 +1891,7 @@ TEST_P(DBWALTestWithParamsVaryingRecoveryMode,
|
|||||||
ASSERT_OK(TryReopen(options));
|
ASSERT_OK(TryReopen(options));
|
||||||
// Append some more data.
|
// Append some more data.
|
||||||
for (int k = 0; k < kAppendKeys; k++) {
|
for (int k = 0; k < kAppendKeys; k++) {
|
||||||
std::string key = "extra_key" + std::to_string(k);
|
std::string key = "extra_key" + ToString(k);
|
||||||
std::string value = DummyString(RecoveryTestHelper::kValueSize);
|
std::string value = DummyString(RecoveryTestHelper::kValueSize);
|
||||||
ASSERT_OK(Put(key, value));
|
ASSERT_OK(Put(key, value));
|
||||||
}
|
}
|
||||||
@ -1925,7 +1925,7 @@ TEST_F(DBWALTest, RestoreTotalLogSizeAfterRecoverWithoutFlush) {
|
|||||||
std::string value_300k(300 * kKB, 'v');
|
std::string value_300k(300 * kKB, 'v');
|
||||||
ASSERT_OK(Put(0, "foo", "v1"));
|
ASSERT_OK(Put(0, "foo", "v1"));
|
||||||
for (int i = 0; i < 9; i++) {
|
for (int i = 0; i < 9; i++) {
|
||||||
ASSERT_OK(Put(1, "key" + std::to_string(i), value_100k));
|
ASSERT_OK(Put(1, "key" + ToString(i), value_100k));
|
||||||
}
|
}
|
||||||
// Get log files before reopen.
|
// Get log files before reopen.
|
||||||
VectorLogPtr log_files_before;
|
VectorLogPtr log_files_before;
|
||||||
|
@ -1492,8 +1492,8 @@ TEST_F(DBBasicTestWithTimestamp, MultiGetRangeFiltering) {
|
|||||||
|
|
||||||
// random data
|
// random data
|
||||||
for (int i = 0; i < 3; i++) {
|
for (int i = 0; i < 3; i++) {
|
||||||
auto key = std::to_string(i * 10);
|
auto key = ToString(i * 10);
|
||||||
auto value = std::to_string(i * 10);
|
auto value = ToString(i * 10);
|
||||||
Slice key_slice = key;
|
Slice key_slice = key;
|
||||||
Slice value_slice = value;
|
Slice value_slice = value;
|
||||||
ASSERT_OK(db_->Put(write_opts, key_slice, ts, value_slice));
|
ASSERT_OK(db_->Put(write_opts, key_slice, ts, value_slice));
|
||||||
@ -1824,8 +1824,8 @@ class DataVisibilityTest : public DBBasicTestWithTimestampBase {
|
|||||||
DataVisibilityTest() : DBBasicTestWithTimestampBase("data_visibility_test") {
|
DataVisibilityTest() : DBBasicTestWithTimestampBase("data_visibility_test") {
|
||||||
// Initialize test data
|
// Initialize test data
|
||||||
for (int i = 0; i < kTestDataSize; i++) {
|
for (int i = 0; i < kTestDataSize; i++) {
|
||||||
test_data_[i].key = "key" + std::to_string(i);
|
test_data_[i].key = "key" + ToString(i);
|
||||||
test_data_[i].value = "value" + std::to_string(i);
|
test_data_[i].value = "value" + ToString(i);
|
||||||
test_data_[i].timestamp = Timestamp(i, 0);
|
test_data_[i].timestamp = Timestamp(i, 0);
|
||||||
test_data_[i].ts = i;
|
test_data_[i].ts = i;
|
||||||
test_data_[i].seq_num = kMaxSequenceNumber;
|
test_data_[i].seq_num = kMaxSequenceNumber;
|
||||||
|
@ -289,7 +289,7 @@ TEST_P(DBWriteTest, IOErrorOnWALWritePropagateToWriteThreadFollower) {
|
|||||||
threads.push_back(port::Thread(
|
threads.push_back(port::Thread(
|
||||||
[&](int index) {
|
[&](int index) {
|
||||||
// All threads should fail.
|
// All threads should fail.
|
||||||
auto res = Put("key" + std::to_string(index), "value");
|
auto res = Put("key" + ToString(index), "value");
|
||||||
if (options.manual_wal_flush) {
|
if (options.manual_wal_flush) {
|
||||||
ASSERT_TRUE(res.ok());
|
ASSERT_TRUE(res.ok());
|
||||||
// we should see fs error when we do the flush
|
// we should see fs error when we do the flush
|
||||||
@ -322,13 +322,13 @@ TEST_P(DBWriteTest, ManualWalFlushInEffect) {
|
|||||||
Options options = GetOptions();
|
Options options = GetOptions();
|
||||||
Reopen(options);
|
Reopen(options);
|
||||||
// try the 1st WAL created during open
|
// try the 1st WAL created during open
|
||||||
ASSERT_TRUE(Put("key" + std::to_string(0), "value").ok());
|
ASSERT_TRUE(Put("key" + ToString(0), "value").ok());
|
||||||
ASSERT_TRUE(options.manual_wal_flush != dbfull()->TEST_WALBufferIsEmpty());
|
ASSERT_TRUE(options.manual_wal_flush != dbfull()->TEST_WALBufferIsEmpty());
|
||||||
ASSERT_TRUE(dbfull()->FlushWAL(false).ok());
|
ASSERT_TRUE(dbfull()->FlushWAL(false).ok());
|
||||||
ASSERT_TRUE(dbfull()->TEST_WALBufferIsEmpty());
|
ASSERT_TRUE(dbfull()->TEST_WALBufferIsEmpty());
|
||||||
// try the 2nd wal created during SwitchWAL
|
// try the 2nd wal created during SwitchWAL
|
||||||
ASSERT_OK(dbfull()->TEST_SwitchWAL());
|
ASSERT_OK(dbfull()->TEST_SwitchWAL());
|
||||||
ASSERT_TRUE(Put("key" + std::to_string(0), "value").ok());
|
ASSERT_TRUE(Put("key" + ToString(0), "value").ok());
|
||||||
ASSERT_TRUE(options.manual_wal_flush != dbfull()->TEST_WALBufferIsEmpty());
|
ASSERT_TRUE(options.manual_wal_flush != dbfull()->TEST_WALBufferIsEmpty());
|
||||||
ASSERT_TRUE(dbfull()->FlushWAL(false).ok());
|
ASSERT_TRUE(dbfull()->FlushWAL(false).ok());
|
||||||
ASSERT_TRUE(dbfull()->TEST_WALBufferIsEmpty());
|
ASSERT_TRUE(dbfull()->TEST_WALBufferIsEmpty());
|
||||||
@ -344,7 +344,7 @@ TEST_P(DBWriteTest, IOErrorOnWALWriteTriggersReadOnlyMode) {
|
|||||||
// Forcibly fail WAL write for the first Put only. Subsequent Puts should
|
// Forcibly fail WAL write for the first Put only. Subsequent Puts should
|
||||||
// fail due to read-only mode
|
// fail due to read-only mode
|
||||||
mock_env->SetFilesystemActive(i != 0);
|
mock_env->SetFilesystemActive(i != 0);
|
||||||
auto res = Put("key" + std::to_string(i), "value");
|
auto res = Put("key" + ToString(i), "value");
|
||||||
// TSAN reports a false alarm for lock-order-inversion but Open and
|
// TSAN reports a false alarm for lock-order-inversion but Open and
|
||||||
// FlushWAL are not run concurrently. Disabling this until TSAN is
|
// FlushWAL are not run concurrently. Disabling this until TSAN is
|
||||||
// fixed.
|
// fixed.
|
||||||
@ -398,14 +398,14 @@ TEST_P(DBWriteTest, LockWalInEffect) {
|
|||||||
Options options = GetOptions();
|
Options options = GetOptions();
|
||||||
Reopen(options);
|
Reopen(options);
|
||||||
// try the 1st WAL created during open
|
// try the 1st WAL created during open
|
||||||
ASSERT_OK(Put("key" + std::to_string(0), "value"));
|
ASSERT_OK(Put("key" + ToString(0), "value"));
|
||||||
ASSERT_TRUE(options.manual_wal_flush != dbfull()->TEST_WALBufferIsEmpty());
|
ASSERT_TRUE(options.manual_wal_flush != dbfull()->TEST_WALBufferIsEmpty());
|
||||||
ASSERT_OK(dbfull()->LockWAL());
|
ASSERT_OK(dbfull()->LockWAL());
|
||||||
ASSERT_TRUE(dbfull()->TEST_WALBufferIsEmpty(false));
|
ASSERT_TRUE(dbfull()->TEST_WALBufferIsEmpty(false));
|
||||||
ASSERT_OK(dbfull()->UnlockWAL());
|
ASSERT_OK(dbfull()->UnlockWAL());
|
||||||
// try the 2nd wal created during SwitchWAL
|
// try the 2nd wal created during SwitchWAL
|
||||||
ASSERT_OK(dbfull()->TEST_SwitchWAL());
|
ASSERT_OK(dbfull()->TEST_SwitchWAL());
|
||||||
ASSERT_OK(Put("key" + std::to_string(0), "value"));
|
ASSERT_OK(Put("key" + ToString(0), "value"));
|
||||||
ASSERT_TRUE(options.manual_wal_flush != dbfull()->TEST_WALBufferIsEmpty());
|
ASSERT_TRUE(options.manual_wal_flush != dbfull()->TEST_WALBufferIsEmpty());
|
||||||
ASSERT_OK(dbfull()->LockWAL());
|
ASSERT_OK(dbfull()->LockWAL());
|
||||||
ASSERT_TRUE(dbfull()->TEST_WALBufferIsEmpty(false));
|
ASSERT_TRUE(dbfull()->TEST_WALBufferIsEmpty(false));
|
||||||
|
@ -90,8 +90,7 @@ inline bool IsExtendedValueType(ValueType t) {
|
|||||||
// can be packed together into 64-bits.
|
// can be packed together into 64-bits.
|
||||||
static const SequenceNumber kMaxSequenceNumber = ((0x1ull << 56) - 1);
|
static const SequenceNumber kMaxSequenceNumber = ((0x1ull << 56) - 1);
|
||||||
|
|
||||||
static const SequenceNumber kDisableGlobalSequenceNumber =
|
static const SequenceNumber kDisableGlobalSequenceNumber = port::kMaxUint64;
|
||||||
std::numeric_limits<uint64_t>::max();
|
|
||||||
|
|
||||||
constexpr uint64_t kNumInternalBytes = 8;
|
constexpr uint64_t kNumInternalBytes = 8;
|
||||||
|
|
||||||
|
@ -56,7 +56,7 @@ class DeleteFileTest : public DBTestBase {
|
|||||||
options.sync = false;
|
options.sync = false;
|
||||||
ReadOptions roptions;
|
ReadOptions roptions;
|
||||||
for (int i = startkey; i < (numkeys + startkey) ; i++) {
|
for (int i = startkey; i < (numkeys + startkey) ; i++) {
|
||||||
std::string temp = std::to_string(i);
|
std::string temp = ToString(i);
|
||||||
Slice key(temp);
|
Slice key(temp);
|
||||||
Slice value(temp);
|
Slice value(temp);
|
||||||
ASSERT_OK(db_->Put(options, key, value));
|
ASSERT_OK(db_->Put(options, key, value));
|
||||||
|
@ -1583,11 +1583,11 @@ TEST_F(DBErrorHandlingFSTest, MultiDBCompactionError) {
|
|||||||
std::string prop;
|
std::string prop;
|
||||||
ASSERT_EQ(listener[i]->WaitForRecovery(5000000), true);
|
ASSERT_EQ(listener[i]->WaitForRecovery(5000000), true);
|
||||||
ASSERT_OK(static_cast<DBImpl*>(db[i])->TEST_WaitForCompact(true));
|
ASSERT_OK(static_cast<DBImpl*>(db[i])->TEST_WaitForCompact(true));
|
||||||
EXPECT_TRUE(db[i]->GetProperty(
|
EXPECT_TRUE(
|
||||||
"rocksdb.num-files-at-level" + std::to_string(0), &prop));
|
db[i]->GetProperty("rocksdb.num-files-at-level" + ToString(0), &prop));
|
||||||
EXPECT_EQ(atoi(prop.c_str()), 0);
|
EXPECT_EQ(atoi(prop.c_str()), 0);
|
||||||
EXPECT_TRUE(db[i]->GetProperty(
|
EXPECT_TRUE(
|
||||||
"rocksdb.num-files-at-level" + std::to_string(1), &prop));
|
db[i]->GetProperty("rocksdb.num-files-at-level" + ToString(1), &prop));
|
||||||
EXPECT_EQ(atoi(prop.c_str()), 1);
|
EXPECT_EQ(atoi(prop.c_str()), 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1720,11 +1720,11 @@ TEST_F(DBErrorHandlingFSTest, MultiDBVariousErrors) {
|
|||||||
if (i == 1) {
|
if (i == 1) {
|
||||||
ASSERT_OK(static_cast<DBImpl*>(db[i])->TEST_WaitForCompact(true));
|
ASSERT_OK(static_cast<DBImpl*>(db[i])->TEST_WaitForCompact(true));
|
||||||
}
|
}
|
||||||
EXPECT_TRUE(db[i]->GetProperty(
|
EXPECT_TRUE(
|
||||||
"rocksdb.num-files-at-level" + std::to_string(0), &prop));
|
db[i]->GetProperty("rocksdb.num-files-at-level" + ToString(0), &prop));
|
||||||
EXPECT_EQ(atoi(prop.c_str()), 0);
|
EXPECT_EQ(atoi(prop.c_str()), 0);
|
||||||
EXPECT_TRUE(db[i]->GetProperty(
|
EXPECT_TRUE(
|
||||||
"rocksdb.num-files-at-level" + std::to_string(1), &prop));
|
db[i]->GetProperty("rocksdb.num-files-at-level" + ToString(1), &prop));
|
||||||
EXPECT_EQ(atoi(prop.c_str()), 1);
|
EXPECT_EQ(atoi(prop.c_str()), 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -91,7 +91,7 @@ class ExternalSSTFileBasicTest
|
|||||||
bool write_global_seqno, bool verify_checksums_before_ingest,
|
bool write_global_seqno, bool verify_checksums_before_ingest,
|
||||||
std::map<std::string, std::string>* true_data) {
|
std::map<std::string, std::string>* true_data) {
|
||||||
assert(value_types.size() == 1 || keys.size() == value_types.size());
|
assert(value_types.size() == 1 || keys.size() == value_types.size());
|
||||||
std::string file_path = sst_files_dir_ + std::to_string(file_id);
|
std::string file_path = sst_files_dir_ + ToString(file_id);
|
||||||
SstFileWriter sst_file_writer(EnvOptions(), options);
|
SstFileWriter sst_file_writer(EnvOptions(), options);
|
||||||
|
|
||||||
Status s = sst_file_writer.Open(file_path);
|
Status s = sst_file_writer.Open(file_path);
|
||||||
@ -123,7 +123,7 @@ class ExternalSSTFileBasicTest
|
|||||||
}
|
}
|
||||||
for (size_t i = 0; i < keys.size(); i++) {
|
for (size_t i = 0; i < keys.size(); i++) {
|
||||||
std::string key = Key(keys[i]);
|
std::string key = Key(keys[i]);
|
||||||
std::string value = Key(keys[i]) + std::to_string(file_id);
|
std::string value = Key(keys[i]) + ToString(file_id);
|
||||||
ValueType value_type =
|
ValueType value_type =
|
||||||
(value_types.size() == 1 ? value_types[0] : value_types[i]);
|
(value_types.size() == 1 ? value_types[0] : value_types[i]);
|
||||||
switch (value_type) {
|
switch (value_type) {
|
||||||
@ -190,9 +190,9 @@ class ExternalSSTFileBasicTest
|
|||||||
#ifndef ROCKSDB_LITE
|
#ifndef ROCKSDB_LITE
|
||||||
uint64_t GetSstSizeHelper(Temperature temperature) {
|
uint64_t GetSstSizeHelper(Temperature temperature) {
|
||||||
std::string prop;
|
std::string prop;
|
||||||
EXPECT_TRUE(dbfull()->GetProperty(
|
EXPECT_TRUE(
|
||||||
DB::Properties::kLiveSstFilesSizeAtTemperature +
|
dbfull()->GetProperty(DB::Properties::kLiveSstFilesSizeAtTemperature +
|
||||||
std::to_string(static_cast<uint8_t>(temperature)),
|
ToString(static_cast<uint8_t>(temperature)),
|
||||||
&prop));
|
&prop));
|
||||||
return static_cast<uint64_t>(std::atoi(prop.c_str()));
|
return static_cast<uint64_t>(std::atoi(prop.c_str()));
|
||||||
}
|
}
|
||||||
@ -1184,7 +1184,7 @@ TEST_F(ExternalSSTFileBasicTest, SyncFailure) {
|
|||||||
std::unique_ptr<SstFileWriter> sst_file_writer(
|
std::unique_ptr<SstFileWriter> sst_file_writer(
|
||||||
new SstFileWriter(EnvOptions(), sst_file_writer_options));
|
new SstFileWriter(EnvOptions(), sst_file_writer_options));
|
||||||
std::string file_name =
|
std::string file_name =
|
||||||
sst_files_dir_ + "sync_failure_test_" + std::to_string(i) + ".sst";
|
sst_files_dir_ + "sync_failure_test_" + ToString(i) + ".sst";
|
||||||
ASSERT_OK(sst_file_writer->Open(file_name));
|
ASSERT_OK(sst_file_writer->Open(file_name));
|
||||||
ASSERT_OK(sst_file_writer->Put("bar", "v2"));
|
ASSERT_OK(sst_file_writer->Put("bar", "v2"));
|
||||||
ASSERT_OK(sst_file_writer->Finish());
|
ASSERT_OK(sst_file_writer->Finish());
|
||||||
@ -1514,13 +1514,13 @@ TEST_P(ExternalSSTFileBasicTest, IngestFileWithFirstByteTampered) {
|
|||||||
EnvOptions env_options;
|
EnvOptions env_options;
|
||||||
do {
|
do {
|
||||||
Options options = CurrentOptions();
|
Options options = CurrentOptions();
|
||||||
std::string file_path = sst_files_dir_ + std::to_string(file_id++);
|
std::string file_path = sst_files_dir_ + ToString(file_id++);
|
||||||
SstFileWriter sst_file_writer(env_options, options);
|
SstFileWriter sst_file_writer(env_options, options);
|
||||||
Status s = sst_file_writer.Open(file_path);
|
Status s = sst_file_writer.Open(file_path);
|
||||||
ASSERT_OK(s);
|
ASSERT_OK(s);
|
||||||
for (int i = 0; i != 100; ++i) {
|
for (int i = 0; i != 100; ++i) {
|
||||||
std::string key = Key(i);
|
std::string key = Key(i);
|
||||||
std::string value = Key(i) + std::to_string(0);
|
std::string value = Key(i) + ToString(0);
|
||||||
ASSERT_OK(sst_file_writer.Put(key, value));
|
ASSERT_OK(sst_file_writer.Put(key, value));
|
||||||
}
|
}
|
||||||
ASSERT_OK(sst_file_writer.Finish());
|
ASSERT_OK(sst_file_writer.Finish());
|
||||||
@ -1585,14 +1585,14 @@ TEST_P(ExternalSSTFileBasicTest, IngestExternalFileWithCorruptedPropsBlock) {
|
|||||||
int file_id = 0;
|
int file_id = 0;
|
||||||
Random64 rand(time(nullptr));
|
Random64 rand(time(nullptr));
|
||||||
do {
|
do {
|
||||||
std::string file_path = sst_files_dir_ + std::to_string(file_id++);
|
std::string file_path = sst_files_dir_ + ToString(file_id++);
|
||||||
Options options = CurrentOptions();
|
Options options = CurrentOptions();
|
||||||
SstFileWriter sst_file_writer(EnvOptions(), options);
|
SstFileWriter sst_file_writer(EnvOptions(), options);
|
||||||
Status s = sst_file_writer.Open(file_path);
|
Status s = sst_file_writer.Open(file_path);
|
||||||
ASSERT_OK(s);
|
ASSERT_OK(s);
|
||||||
for (int i = 0; i != 100; ++i) {
|
for (int i = 0; i != 100; ++i) {
|
||||||
std::string key = Key(i);
|
std::string key = Key(i);
|
||||||
std::string value = Key(i) + std::to_string(0);
|
std::string value = Key(i) + ToString(0);
|
||||||
ASSERT_OK(sst_file_writer.Put(key, value));
|
ASSERT_OK(sst_file_writer.Put(key, value));
|
||||||
}
|
}
|
||||||
ASSERT_OK(sst_file_writer.Finish());
|
ASSERT_OK(sst_file_writer.Finish());
|
||||||
@ -1799,7 +1799,7 @@ TEST_F(ExternalSSTFileBasicTest, IngestWithTemperature) {
|
|||||||
TEST_F(ExternalSSTFileBasicTest, FailIfNotBottommostLevel) {
|
TEST_F(ExternalSSTFileBasicTest, FailIfNotBottommostLevel) {
|
||||||
Options options = GetDefaultOptions();
|
Options options = GetDefaultOptions();
|
||||||
|
|
||||||
std::string file_path = sst_files_dir_ + std::to_string(1);
|
std::string file_path = sst_files_dir_ + ToString(1);
|
||||||
SstFileWriter sfw(EnvOptions(), options);
|
SstFileWriter sfw(EnvOptions(), options);
|
||||||
|
|
||||||
ASSERT_OK(sfw.Open(file_path));
|
ASSERT_OK(sfw.Open(file_path));
|
||||||
|
@ -118,7 +118,7 @@ class ExternalSSTFileTest
|
|||||||
});
|
});
|
||||||
data.resize(uniq_iter - data.begin());
|
data.resize(uniq_iter - data.begin());
|
||||||
}
|
}
|
||||||
std::string file_path = sst_files_dir_ + std::to_string(file_id);
|
std::string file_path = sst_files_dir_ + ToString(file_id);
|
||||||
SstFileWriter sst_file_writer(EnvOptions(), options, cfh);
|
SstFileWriter sst_file_writer(EnvOptions(), options, cfh);
|
||||||
Status s = sst_file_writer.Open(file_path);
|
Status s = sst_file_writer.Open(file_path);
|
||||||
if (!s.ok()) {
|
if (!s.ok()) {
|
||||||
@ -172,7 +172,7 @@ class ExternalSSTFileTest
|
|||||||
});
|
});
|
||||||
data.resize(uniq_iter - data.begin());
|
data.resize(uniq_iter - data.begin());
|
||||||
}
|
}
|
||||||
std::string file_path = sst_files_dir_ + std::to_string(file_id);
|
std::string file_path = sst_files_dir_ + ToString(file_id);
|
||||||
SstFileWriter sst_file_writer(EnvOptions(), options, cfh);
|
SstFileWriter sst_file_writer(EnvOptions(), options, cfh);
|
||||||
|
|
||||||
Status s = sst_file_writer.Open(file_path);
|
Status s = sst_file_writer.Open(file_path);
|
||||||
@ -270,7 +270,7 @@ class ExternalSSTFileTest
|
|||||||
ColumnFamilyHandle* cfh = nullptr) {
|
ColumnFamilyHandle* cfh = nullptr) {
|
||||||
std::vector<std::pair<std::string, std::string>> file_data;
|
std::vector<std::pair<std::string, std::string>> file_data;
|
||||||
for (auto& k : keys) {
|
for (auto& k : keys) {
|
||||||
file_data.emplace_back(Key(k), Key(k) + std::to_string(file_id));
|
file_data.emplace_back(Key(k), Key(k) + ToString(file_id));
|
||||||
}
|
}
|
||||||
return GenerateAndAddExternalFile(options, file_data, file_id,
|
return GenerateAndAddExternalFile(options, file_data, file_id,
|
||||||
allow_global_seqno, write_global_seqno,
|
allow_global_seqno, write_global_seqno,
|
||||||
@ -966,7 +966,7 @@ TEST_F(ExternalSSTFileTest, MultiThreaded) {
|
|||||||
// Generate file names
|
// Generate file names
|
||||||
std::vector<std::string> file_names;
|
std::vector<std::string> file_names;
|
||||||
for (int i = 0; i < num_files; i++) {
|
for (int i = 0; i < num_files; i++) {
|
||||||
std::string file_name = "file_" + std::to_string(i) + ".sst";
|
std::string file_name = "file_" + ToString(i) + ".sst";
|
||||||
file_names.push_back(sst_files_dir_ + file_name);
|
file_names.push_back(sst_files_dir_ + file_name);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1116,7 +1116,7 @@ TEST_F(ExternalSSTFileTest, OverlappingRanges) {
|
|||||||
int range_end = key_ranges[i].second;
|
int range_end = key_ranges[i].second;
|
||||||
|
|
||||||
Status s;
|
Status s;
|
||||||
std::string range_val = "range_" + std::to_string(i);
|
std::string range_val = "range_" + ToString(i);
|
||||||
|
|
||||||
// For 20% of ranges we use DB::Put, for 80% we use DB::AddFile
|
// For 20% of ranges we use DB::Put, for 80% we use DB::AddFile
|
||||||
if (i && i % 5 == 0) {
|
if (i && i % 5 == 0) {
|
||||||
@ -1456,7 +1456,7 @@ TEST_F(ExternalSSTFileTest, CompactDuringAddFileRandom) {
|
|||||||
ASSERT_EQ(Get(Key(range_start)), Key(range_start)) << rid;
|
ASSERT_EQ(Get(Key(range_start)), Key(range_start)) << rid;
|
||||||
ASSERT_EQ(Get(Key(range_end)), Key(range_end)) << rid;
|
ASSERT_EQ(Get(Key(range_end)), Key(range_end)) << rid;
|
||||||
for (int k = range_start + 1; k < range_end; k++) {
|
for (int k = range_start + 1; k < range_end; k++) {
|
||||||
std::string v = Key(k) + std::to_string(rid);
|
std::string v = Key(k) + ToString(rid);
|
||||||
ASSERT_EQ(Get(Key(k)), v) << rid;
|
ASSERT_EQ(Get(Key(k)), v) << rid;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -2405,7 +2405,7 @@ TEST_P(ExternalSSTBlockChecksumTest, DISABLED_HugeBlockChecksum) {
|
|||||||
SstFileWriter sst_file_writer(EnvOptions(), options);
|
SstFileWriter sst_file_writer(EnvOptions(), options);
|
||||||
|
|
||||||
// 2^32 - 1, will lead to data block with more than 2^32 bytes
|
// 2^32 - 1, will lead to data block with more than 2^32 bytes
|
||||||
size_t huge_size = std::numeric_limits<uint32_t>::max();
|
size_t huge_size = port::kMaxUint32;
|
||||||
|
|
||||||
std::string f = sst_files_dir_ + "f.sst";
|
std::string f = sst_files_dir_ + "f.sst";
|
||||||
ASSERT_OK(sst_file_writer.Open(f));
|
ASSERT_OK(sst_file_writer.Open(f));
|
||||||
|
@ -58,7 +58,10 @@ class FileIndexer {
|
|||||||
void UpdateIndex(Arena* arena, const size_t num_levels,
|
void UpdateIndex(Arena* arena, const size_t num_levels,
|
||||||
std::vector<FileMetaData*>* const files);
|
std::vector<FileMetaData*>* const files);
|
||||||
|
|
||||||
enum { kLevelMaxIndex = std::numeric_limits<int32_t>::max() };
|
enum {
|
||||||
|
// MSVC version 1800 still does not have constexpr for ::max()
|
||||||
|
kLevelMaxIndex = ROCKSDB_NAMESPACE::port::kMaxInt32
|
||||||
|
};
|
||||||
|
|
||||||
private:
|
private:
|
||||||
size_t num_levels_;
|
size_t num_levels_;
|
||||||
|
@ -464,7 +464,6 @@ Status FlushJob::MemPurge() {
|
|||||||
env, ShouldReportDetailedTime(env, ioptions->stats),
|
env, ShouldReportDetailedTime(env, ioptions->stats),
|
||||||
true /* internal key corruption is not ok */, range_del_agg.get(),
|
true /* internal key corruption is not ok */, range_del_agg.get(),
|
||||||
nullptr, ioptions->allow_data_in_errors,
|
nullptr, ioptions->allow_data_in_errors,
|
||||||
ioptions->enforce_single_del_contracts,
|
|
||||||
/*compaction=*/nullptr, compaction_filter.get(),
|
/*compaction=*/nullptr, compaction_filter.get(),
|
||||||
/*shutting_down=*/nullptr,
|
/*shutting_down=*/nullptr,
|
||||||
/*manual_compaction_paused=*/nullptr,
|
/*manual_compaction_paused=*/nullptr,
|
||||||
@ -932,9 +931,9 @@ Status FlushJob::WriteLevel0Table() {
|
|||||||
assert(!s.ok() || io_s.ok());
|
assert(!s.ok() || io_s.ok());
|
||||||
io_s.PermitUncheckedError();
|
io_s.PermitUncheckedError();
|
||||||
if (num_input_entries != total_num_entries && s.ok()) {
|
if (num_input_entries != total_num_entries && s.ok()) {
|
||||||
std::string msg = "Expected " + std::to_string(total_num_entries) +
|
std::string msg = "Expected " + ToString(total_num_entries) +
|
||||||
" entries in memtables, but read " +
|
" entries in memtables, but read " +
|
||||||
std::to_string(num_input_entries);
|
ToString(num_input_entries);
|
||||||
ROCKS_LOG_WARN(db_options_.info_log, "[%s] [JOB %d] Level-0 flush %s",
|
ROCKS_LOG_WARN(db_options_.info_log, "[%s] [JOB %d] Level-0 flush %s",
|
||||||
cfd_->GetName().c_str(), job_context_->job_id,
|
cfd_->GetName().c_str(), job_context_->job_id,
|
||||||
msg.c_str());
|
msg.c_str());
|
||||||
|
@ -164,12 +164,12 @@ TEST_F(FlushJobTest, Empty) {
|
|||||||
SnapshotChecker* snapshot_checker = nullptr; // not relavant
|
SnapshotChecker* snapshot_checker = nullptr; // not relavant
|
||||||
FlushJob flush_job(
|
FlushJob flush_job(
|
||||||
dbname_, versions_->GetColumnFamilySet()->GetDefault(), db_options_,
|
dbname_, versions_->GetColumnFamilySet()->GetDefault(), db_options_,
|
||||||
*cfd->GetLatestMutableCFOptions(),
|
*cfd->GetLatestMutableCFOptions(), port::kMaxUint64 /* memtable_id */,
|
||||||
std::numeric_limits<uint64_t>::max() /* memtable_id */, env_options_,
|
env_options_, versions_.get(), &mutex_, &shutting_down_, {},
|
||||||
versions_.get(), &mutex_, &shutting_down_, {}, kMaxSequenceNumber,
|
kMaxSequenceNumber, snapshot_checker, &job_context, nullptr, nullptr,
|
||||||
snapshot_checker, &job_context, nullptr, nullptr, nullptr, kNoCompression,
|
nullptr, kNoCompression, nullptr, &event_logger, false,
|
||||||
nullptr, &event_logger, false, true /* sync_output_directory */,
|
true /* sync_output_directory */, true /* write_manifest */,
|
||||||
true /* write_manifest */, Env::Priority::USER, nullptr /*IOTracer*/);
|
Env::Priority::USER, nullptr /*IOTracer*/);
|
||||||
{
|
{
|
||||||
InstrumentedMutexLock l(&mutex_);
|
InstrumentedMutexLock l(&mutex_);
|
||||||
flush_job.PickMemTable();
|
flush_job.PickMemTable();
|
||||||
@ -191,7 +191,7 @@ TEST_F(FlushJobTest, NonEmpty) {
|
|||||||
// range-delete "9995" -> "9999" at seqno 10000
|
// range-delete "9995" -> "9999" at seqno 10000
|
||||||
// blob references with seqnos 10001..10006
|
// blob references with seqnos 10001..10006
|
||||||
for (int i = 1; i < 10000; ++i) {
|
for (int i = 1; i < 10000; ++i) {
|
||||||
std::string key(std::to_string((i + 1000) % 10000));
|
std::string key(ToString((i + 1000) % 10000));
|
||||||
std::string value("value" + key);
|
std::string value("value" + key);
|
||||||
ASSERT_OK(new_mem->Add(SequenceNumber(i), kTypeValue, key, value,
|
ASSERT_OK(new_mem->Add(SequenceNumber(i), kTypeValue, key, value,
|
||||||
nullptr /* kv_prot_info */));
|
nullptr /* kv_prot_info */));
|
||||||
@ -214,7 +214,7 @@ TEST_F(FlushJobTest, NonEmpty) {
|
|||||||
constexpr std::array<uint64_t, 6> blob_file_numbers{{
|
constexpr std::array<uint64_t, 6> blob_file_numbers{{
|
||||||
kInvalidBlobFileNumber, 5, 103, 17, 102, 101}};
|
kInvalidBlobFileNumber, 5, 103, 17, 102, 101}};
|
||||||
for (size_t i = 0; i < blob_file_numbers.size(); ++i) {
|
for (size_t i = 0; i < blob_file_numbers.size(); ++i) {
|
||||||
std::string key(std::to_string(i + 10001));
|
std::string key(ToString(i + 10001));
|
||||||
std::string blob_index;
|
std::string blob_index;
|
||||||
if (i == 0) {
|
if (i == 0) {
|
||||||
BlobIndex::EncodeInlinedTTL(&blob_index, /* expiration */ 1234567890ULL,
|
BlobIndex::EncodeInlinedTTL(&blob_index, /* expiration */ 1234567890ULL,
|
||||||
@ -248,12 +248,11 @@ TEST_F(FlushJobTest, NonEmpty) {
|
|||||||
SnapshotChecker* snapshot_checker = nullptr; // not relavant
|
SnapshotChecker* snapshot_checker = nullptr; // not relavant
|
||||||
FlushJob flush_job(
|
FlushJob flush_job(
|
||||||
dbname_, versions_->GetColumnFamilySet()->GetDefault(), db_options_,
|
dbname_, versions_->GetColumnFamilySet()->GetDefault(), db_options_,
|
||||||
*cfd->GetLatestMutableCFOptions(),
|
*cfd->GetLatestMutableCFOptions(), port::kMaxUint64 /* memtable_id */,
|
||||||
std::numeric_limits<uint64_t>::max() /* memtable_id */, env_options_,
|
env_options_, versions_.get(), &mutex_, &shutting_down_, {},
|
||||||
versions_.get(), &mutex_, &shutting_down_, {}, kMaxSequenceNumber,
|
kMaxSequenceNumber, snapshot_checker, &job_context, nullptr, nullptr,
|
||||||
snapshot_checker, &job_context, nullptr, nullptr, nullptr, kNoCompression,
|
nullptr, kNoCompression, db_options_.statistics.get(), &event_logger,
|
||||||
db_options_.statistics.get(), &event_logger, true,
|
true, true /* sync_output_directory */, true /* write_manifest */,
|
||||||
true /* sync_output_directory */, true /* write_manifest */,
|
|
||||||
Env::Priority::USER, nullptr /*IOTracer*/);
|
Env::Priority::USER, nullptr /*IOTracer*/);
|
||||||
|
|
||||||
HistogramData hist;
|
HistogramData hist;
|
||||||
@ -265,7 +264,7 @@ TEST_F(FlushJobTest, NonEmpty) {
|
|||||||
db_options_.statistics->histogramData(FLUSH_TIME, &hist);
|
db_options_.statistics->histogramData(FLUSH_TIME, &hist);
|
||||||
ASSERT_GT(hist.average, 0.0);
|
ASSERT_GT(hist.average, 0.0);
|
||||||
|
|
||||||
ASSERT_EQ(std::to_string(0), file_meta.smallest.user_key().ToString());
|
ASSERT_EQ(ToString(0), file_meta.smallest.user_key().ToString());
|
||||||
ASSERT_EQ("9999a", file_meta.largest.user_key().ToString());
|
ASSERT_EQ("9999a", file_meta.largest.user_key().ToString());
|
||||||
ASSERT_EQ(1, file_meta.fd.smallest_seqno);
|
ASSERT_EQ(1, file_meta.fd.smallest_seqno);
|
||||||
ASSERT_EQ(10006, file_meta.fd.largest_seqno);
|
ASSERT_EQ(10006, file_meta.fd.largest_seqno);
|
||||||
@ -291,7 +290,7 @@ TEST_F(FlushJobTest, FlushMemTablesSingleColumnFamily) {
|
|||||||
memtable_ids.push_back(mem->GetID());
|
memtable_ids.push_back(mem->GetID());
|
||||||
|
|
||||||
for (size_t j = 0; j < num_keys_per_table; ++j) {
|
for (size_t j = 0; j < num_keys_per_table; ++j) {
|
||||||
std::string key(std::to_string(j + i * num_keys_per_table));
|
std::string key(ToString(j + i * num_keys_per_table));
|
||||||
std::string value("value" + key);
|
std::string value("value" + key);
|
||||||
ASSERT_OK(mem->Add(SequenceNumber(j + i * num_keys_per_table), kTypeValue,
|
ASSERT_OK(mem->Add(SequenceNumber(j + i * num_keys_per_table), kTypeValue,
|
||||||
key, value, nullptr /* kv_prot_info */));
|
key, value, nullptr /* kv_prot_info */));
|
||||||
@ -326,7 +325,7 @@ TEST_F(FlushJobTest, FlushMemTablesSingleColumnFamily) {
|
|||||||
db_options_.statistics->histogramData(FLUSH_TIME, &hist);
|
db_options_.statistics->histogramData(FLUSH_TIME, &hist);
|
||||||
ASSERT_GT(hist.average, 0.0);
|
ASSERT_GT(hist.average, 0.0);
|
||||||
|
|
||||||
ASSERT_EQ(std::to_string(0), file_meta.smallest.user_key().ToString());
|
ASSERT_EQ(ToString(0), file_meta.smallest.user_key().ToString());
|
||||||
ASSERT_EQ("99", file_meta.largest.user_key().ToString());
|
ASSERT_EQ("99", file_meta.largest.user_key().ToString());
|
||||||
ASSERT_EQ(0, file_meta.fd.smallest_seqno);
|
ASSERT_EQ(0, file_meta.fd.smallest_seqno);
|
||||||
ASSERT_EQ(SequenceNumber(num_mems_to_flush * num_keys_per_table - 1),
|
ASSERT_EQ(SequenceNumber(num_mems_to_flush * num_keys_per_table - 1),
|
||||||
@ -364,7 +363,7 @@ TEST_F(FlushJobTest, FlushMemtablesMultipleColumnFamilies) {
|
|||||||
mem->Ref();
|
mem->Ref();
|
||||||
|
|
||||||
for (size_t j = 0; j != num_keys_per_memtable; ++j) {
|
for (size_t j = 0; j != num_keys_per_memtable; ++j) {
|
||||||
std::string key(std::to_string(j + i * num_keys_per_memtable));
|
std::string key(ToString(j + i * num_keys_per_memtable));
|
||||||
std::string value("value" + key);
|
std::string value("value" + key);
|
||||||
ASSERT_OK(mem->Add(curr_seqno++, kTypeValue, key, value,
|
ASSERT_OK(mem->Add(curr_seqno++, kTypeValue, key, value,
|
||||||
nullptr /* kv_prot_info */));
|
nullptr /* kv_prot_info */));
|
||||||
@ -439,7 +438,7 @@ TEST_F(FlushJobTest, FlushMemtablesMultipleColumnFamilies) {
|
|||||||
ASSERT_GT(hist.average, 0.0);
|
ASSERT_GT(hist.average, 0.0);
|
||||||
k = 0;
|
k = 0;
|
||||||
for (const auto& file_meta : file_metas) {
|
for (const auto& file_meta : file_metas) {
|
||||||
ASSERT_EQ(std::to_string(0), file_meta.smallest.user_key().ToString());
|
ASSERT_EQ(ToString(0), file_meta.smallest.user_key().ToString());
|
||||||
ASSERT_EQ("999", file_meta.largest.user_key()
|
ASSERT_EQ("999", file_meta.largest.user_key()
|
||||||
.ToString()); // max key by bytewise comparator
|
.ToString()); // max key by bytewise comparator
|
||||||
ASSERT_EQ(smallest_seqs[k], file_meta.fd.smallest_seqno);
|
ASSERT_EQ(smallest_seqs[k], file_meta.fd.smallest_seqno);
|
||||||
@ -480,7 +479,7 @@ TEST_F(FlushJobTest, Snapshots) {
|
|||||||
SequenceNumber current_seqno = 0;
|
SequenceNumber current_seqno = 0;
|
||||||
auto inserted_keys = mock::MakeMockFile();
|
auto inserted_keys = mock::MakeMockFile();
|
||||||
for (int i = 1; i < keys; ++i) {
|
for (int i = 1; i < keys; ++i) {
|
||||||
std::string key(std::to_string(i));
|
std::string key(ToString(i));
|
||||||
int insertions = rnd.Uniform(max_inserts_per_keys);
|
int insertions = rnd.Uniform(max_inserts_per_keys);
|
||||||
for (int j = 0; j < insertions; ++j) {
|
for (int j = 0; j < insertions; ++j) {
|
||||||
std::string value(rnd.HumanReadableString(10));
|
std::string value(rnd.HumanReadableString(10));
|
||||||
@ -510,12 +509,11 @@ TEST_F(FlushJobTest, Snapshots) {
|
|||||||
SnapshotChecker* snapshot_checker = nullptr; // not relavant
|
SnapshotChecker* snapshot_checker = nullptr; // not relavant
|
||||||
FlushJob flush_job(
|
FlushJob flush_job(
|
||||||
dbname_, versions_->GetColumnFamilySet()->GetDefault(), db_options_,
|
dbname_, versions_->GetColumnFamilySet()->GetDefault(), db_options_,
|
||||||
*cfd->GetLatestMutableCFOptions(),
|
*cfd->GetLatestMutableCFOptions(), port::kMaxUint64 /* memtable_id */,
|
||||||
std::numeric_limits<uint64_t>::max() /* memtable_id */, env_options_,
|
env_options_, versions_.get(), &mutex_, &shutting_down_, snapshots,
|
||||||
versions_.get(), &mutex_, &shutting_down_, snapshots, kMaxSequenceNumber,
|
kMaxSequenceNumber, snapshot_checker, &job_context, nullptr, nullptr,
|
||||||
snapshot_checker, &job_context, nullptr, nullptr, nullptr, kNoCompression,
|
nullptr, kNoCompression, db_options_.statistics.get(), &event_logger,
|
||||||
db_options_.statistics.get(), &event_logger, true,
|
true, true /* sync_output_directory */, true /* write_manifest */,
|
||||||
true /* sync_output_directory */, true /* write_manifest */,
|
|
||||||
Env::Priority::USER, nullptr /*IOTracer*/);
|
Env::Priority::USER, nullptr /*IOTracer*/);
|
||||||
mutex_.Lock();
|
mutex_.Lock();
|
||||||
flush_job.PickMemTable();
|
flush_job.PickMemTable();
|
||||||
@ -579,9 +577,9 @@ TEST_F(FlushJobTimestampTest, AllKeysExpired) {
|
|||||||
PutFixed64(&full_history_ts_low, std::numeric_limits<uint64_t>::max());
|
PutFixed64(&full_history_ts_low, std::numeric_limits<uint64_t>::max());
|
||||||
FlushJob flush_job(
|
FlushJob flush_job(
|
||||||
dbname_, cfd, db_options_, *cfd->GetLatestMutableCFOptions(),
|
dbname_, cfd, db_options_, *cfd->GetLatestMutableCFOptions(),
|
||||||
std::numeric_limits<uint64_t>::max() /* memtable_id */, env_options_,
|
port::kMaxUint64 /* memtable_id */, env_options_, versions_.get(),
|
||||||
versions_.get(), &mutex_, &shutting_down_, snapshots, kMaxSequenceNumber,
|
&mutex_, &shutting_down_, snapshots, kMaxSequenceNumber, snapshot_checker,
|
||||||
snapshot_checker, &job_context, nullptr, nullptr, nullptr, kNoCompression,
|
&job_context, nullptr, nullptr, nullptr, kNoCompression,
|
||||||
db_options_.statistics.get(), &event_logger, true,
|
db_options_.statistics.get(), &event_logger, true,
|
||||||
true /* sync_output_directory */, true /* write_manifest */,
|
true /* sync_output_directory */, true /* write_manifest */,
|
||||||
Env::Priority::USER, nullptr /*IOTracer*/, /*db_id=*/"",
|
Env::Priority::USER, nullptr /*IOTracer*/, /*db_id=*/"",
|
||||||
@ -630,9 +628,9 @@ TEST_F(FlushJobTimestampTest, NoKeyExpired) {
|
|||||||
PutFixed64(&full_history_ts_low, 0);
|
PutFixed64(&full_history_ts_low, 0);
|
||||||
FlushJob flush_job(
|
FlushJob flush_job(
|
||||||
dbname_, cfd, db_options_, *cfd->GetLatestMutableCFOptions(),
|
dbname_, cfd, db_options_, *cfd->GetLatestMutableCFOptions(),
|
||||||
std::numeric_limits<uint64_t>::max() /* memtable_id */, env_options_,
|
port::kMaxUint64 /* memtable_id */, env_options_, versions_.get(),
|
||||||
versions_.get(), &mutex_, &shutting_down_, snapshots, kMaxSequenceNumber,
|
&mutex_, &shutting_down_, snapshots, kMaxSequenceNumber, snapshot_checker,
|
||||||
snapshot_checker, &job_context, nullptr, nullptr, nullptr, kNoCompression,
|
&job_context, nullptr, nullptr, nullptr, kNoCompression,
|
||||||
db_options_.statistics.get(), &event_logger, true,
|
db_options_.statistics.get(), &event_logger, true,
|
||||||
true /* sync_output_directory */, true /* write_manifest */,
|
true /* sync_output_directory */, true /* write_manifest */,
|
||||||
Env::Priority::USER, nullptr /*IOTracer*/, /*db_id=*/"",
|
Env::Priority::USER, nullptr /*IOTracer*/, /*db_id=*/"",
|
||||||
|
@ -604,7 +604,7 @@ bool ForwardIterator::PrepareValue() {
|
|||||||
Status ForwardIterator::GetProperty(std::string prop_name, std::string* prop) {
|
Status ForwardIterator::GetProperty(std::string prop_name, std::string* prop) {
|
||||||
assert(prop != nullptr);
|
assert(prop != nullptr);
|
||||||
if (prop_name == "rocksdb.iterator.super-version-number") {
|
if (prop_name == "rocksdb.iterator.super-version-number") {
|
||||||
*prop = std::to_string(sv_->version_number);
|
*prop = ToString(sv_->version_number);
|
||||||
return Status::OK();
|
return Status::OK();
|
||||||
}
|
}
|
||||||
return Status::InvalidArgument();
|
return Status::InvalidArgument();
|
||||||
|
@ -704,19 +704,20 @@ void InternalStats::CacheEntryRoleStats::ToMap(
|
|||||||
auto& v = *values;
|
auto& v = *values;
|
||||||
v[BlockCacheEntryStatsMapKeys::CacheId()] = cache_id;
|
v[BlockCacheEntryStatsMapKeys::CacheId()] = cache_id;
|
||||||
v[BlockCacheEntryStatsMapKeys::CacheCapacityBytes()] =
|
v[BlockCacheEntryStatsMapKeys::CacheCapacityBytes()] =
|
||||||
std::to_string(cache_capacity);
|
ROCKSDB_NAMESPACE::ToString(cache_capacity);
|
||||||
v[BlockCacheEntryStatsMapKeys::LastCollectionDurationSeconds()] =
|
v[BlockCacheEntryStatsMapKeys::LastCollectionDurationSeconds()] =
|
||||||
std::to_string(GetLastDurationMicros() / 1000000.0);
|
ROCKSDB_NAMESPACE::ToString(GetLastDurationMicros() / 1000000.0);
|
||||||
v[BlockCacheEntryStatsMapKeys::LastCollectionAgeSeconds()] =
|
v[BlockCacheEntryStatsMapKeys::LastCollectionAgeSeconds()] =
|
||||||
std::to_string((clock->NowMicros() - last_end_time_micros_) / 1000000U);
|
ROCKSDB_NAMESPACE::ToString((clock->NowMicros() - last_end_time_micros_) /
|
||||||
|
1000000U);
|
||||||
for (size_t i = 0; i < kNumCacheEntryRoles; ++i) {
|
for (size_t i = 0; i < kNumCacheEntryRoles; ++i) {
|
||||||
auto role = static_cast<CacheEntryRole>(i);
|
auto role = static_cast<CacheEntryRole>(i);
|
||||||
v[BlockCacheEntryStatsMapKeys::EntryCount(role)] =
|
v[BlockCacheEntryStatsMapKeys::EntryCount(role)] =
|
||||||
std::to_string(entry_counts[i]);
|
ROCKSDB_NAMESPACE::ToString(entry_counts[i]);
|
||||||
v[BlockCacheEntryStatsMapKeys::UsedBytes(role)] =
|
v[BlockCacheEntryStatsMapKeys::UsedBytes(role)] =
|
||||||
std::to_string(total_charges[i]);
|
ROCKSDB_NAMESPACE::ToString(total_charges[i]);
|
||||||
v[BlockCacheEntryStatsMapKeys::UsedPercent(role)] =
|
v[BlockCacheEntryStatsMapKeys::UsedPercent(role)] =
|
||||||
std::to_string(100.0 * total_charges[i] / cache_capacity);
|
ROCKSDB_NAMESPACE::ToString(100.0 * total_charges[i] / cache_capacity);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -762,7 +763,7 @@ bool InternalStats::HandleLiveSstFilesSizeAtTemperature(std::string* value,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
*value = std::to_string(size);
|
*value = ToString(size);
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -918,7 +919,7 @@ bool InternalStats::HandleCompressionRatioAtLevelPrefix(std::string* value,
|
|||||||
if (!ok || level >= static_cast<uint64_t>(number_levels_)) {
|
if (!ok || level >= static_cast<uint64_t>(number_levels_)) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
*value = std::to_string(
|
*value = ToString(
|
||||||
vstorage->GetEstimatedCompressionRatioAtLevel(static_cast<int>(level)));
|
vstorage->GetEstimatedCompressionRatioAtLevel(static_cast<int>(level)));
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
@ -1005,7 +1006,7 @@ static std::map<std::string, std::string> MapUint64ValuesToString(
|
|||||||
const std::map<std::string, uint64_t>& from) {
|
const std::map<std::string, uint64_t>& from) {
|
||||||
std::map<std::string, std::string> to;
|
std::map<std::string, std::string> to;
|
||||||
for (const auto& e : from) {
|
for (const auto& e : from) {
|
||||||
to[e.first] = std::to_string(e.second);
|
to[e.first] = ToString(e.second);
|
||||||
}
|
}
|
||||||
return to;
|
return to;
|
||||||
}
|
}
|
||||||
@ -1499,7 +1500,7 @@ void InternalStats::DumpCFMapStats(
|
|||||||
DumpCFMapStats(vstorage, &levels_stats, &compaction_stats_sum);
|
DumpCFMapStats(vstorage, &levels_stats, &compaction_stats_sum);
|
||||||
for (auto const& level_ent : levels_stats) {
|
for (auto const& level_ent : levels_stats) {
|
||||||
auto level_str =
|
auto level_str =
|
||||||
level_ent.first == -1 ? "Sum" : "L" + std::to_string(level_ent.first);
|
level_ent.first == -1 ? "Sum" : "L" + ToString(level_ent.first);
|
||||||
for (auto const& stat_ent : level_ent.second) {
|
for (auto const& stat_ent : level_ent.second) {
|
||||||
auto stat_type = stat_ent.first;
|
auto stat_type = stat_ent.first;
|
||||||
auto key_str =
|
auto key_str =
|
||||||
@ -1650,8 +1651,7 @@ void InternalStats::DumpCFStatsNoFileHistogram(std::string* value) {
|
|||||||
DumpCFMapStats(vstorage, &levels_stats, &compaction_stats_sum);
|
DumpCFMapStats(vstorage, &levels_stats, &compaction_stats_sum);
|
||||||
for (int l = 0; l < number_levels_; ++l) {
|
for (int l = 0; l < number_levels_; ++l) {
|
||||||
if (levels_stats.find(l) != levels_stats.end()) {
|
if (levels_stats.find(l) != levels_stats.end()) {
|
||||||
PrintLevelStats(buf, sizeof(buf), "L" + std::to_string(l),
|
PrintLevelStats(buf, sizeof(buf), "L" + ToString(l), levels_stats[l]);
|
||||||
levels_stats[l]);
|
|
||||||
value->append(buf);
|
value->append(buf);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -436,10 +436,10 @@ TEST_F(EventListenerTest, MultiDBMultiListeners) {
|
|||||||
std::vector<std::vector<ColumnFamilyHandle *>> vec_handles;
|
std::vector<std::vector<ColumnFamilyHandle *>> vec_handles;
|
||||||
|
|
||||||
for (int d = 0; d < kNumDBs; ++d) {
|
for (int d = 0; d < kNumDBs; ++d) {
|
||||||
ASSERT_OK(DestroyDB(dbname_ + std::to_string(d), options));
|
ASSERT_OK(DestroyDB(dbname_ + ToString(d), options));
|
||||||
DB* db;
|
DB* db;
|
||||||
std::vector<ColumnFamilyHandle*> handles;
|
std::vector<ColumnFamilyHandle*> handles;
|
||||||
ASSERT_OK(DB::Open(options, dbname_ + std::to_string(d), &db));
|
ASSERT_OK(DB::Open(options, dbname_ + ToString(d), &db));
|
||||||
for (size_t c = 0; c < cf_names.size(); ++c) {
|
for (size_t c = 0; c < cf_names.size(); ++c) {
|
||||||
ColumnFamilyHandle* handle;
|
ColumnFamilyHandle* handle;
|
||||||
ASSERT_OK(db->CreateColumnFamily(cf_opts, cf_names[c], &handle));
|
ASSERT_OK(db->CreateColumnFamily(cf_opts, cf_names[c], &handle));
|
||||||
@ -527,8 +527,7 @@ TEST_F(EventListenerTest, DisableBGCompaction) {
|
|||||||
// keep writing until writes are forced to stop.
|
// keep writing until writes are forced to stop.
|
||||||
for (int i = 0; static_cast<int>(cf_meta.file_count) < kSlowdownTrigger * 10;
|
for (int i = 0; static_cast<int>(cf_meta.file_count) < kSlowdownTrigger * 10;
|
||||||
++i) {
|
++i) {
|
||||||
ASSERT_OK(
|
ASSERT_OK(Put(1, ToString(i), std::string(10000, 'x'), WriteOptions()));
|
||||||
Put(1, std::to_string(i), std::string(10000, 'x'), WriteOptions()));
|
|
||||||
FlushOptions fo;
|
FlushOptions fo;
|
||||||
fo.allow_write_stall = true;
|
fo.allow_write_stall = true;
|
||||||
ASSERT_OK(db_->Flush(fo, handles_[1]));
|
ASSERT_OK(db_->Flush(fo, handles_[1]));
|
||||||
|
@ -140,8 +140,8 @@ size_t MemTable::ApproximateMemoryUsage() {
|
|||||||
for (size_t usage : usages) {
|
for (size_t usage : usages) {
|
||||||
// If usage + total_usage >= kMaxSizet, return kMaxSizet.
|
// If usage + total_usage >= kMaxSizet, return kMaxSizet.
|
||||||
// the following variation is to avoid numeric overflow.
|
// the following variation is to avoid numeric overflow.
|
||||||
if (usage >= std::numeric_limits<size_t>::max() - total_usage) {
|
if (usage >= port::kMaxSizet - total_usage) {
|
||||||
return std::numeric_limits<size_t>::max();
|
return port::kMaxSizet;
|
||||||
}
|
}
|
||||||
total_usage += usage;
|
total_usage += usage;
|
||||||
}
|
}
|
||||||
|
@ -315,7 +315,7 @@ class MemTableList {
|
|||||||
// PickMemtablesToFlush() is called.
|
// PickMemtablesToFlush() is called.
|
||||||
void FlushRequested() {
|
void FlushRequested() {
|
||||||
flush_requested_ = true;
|
flush_requested_ = true;
|
||||||
// If there are some memtables stored in imm() that don't trigger
|
// If there are some memtables stored in imm() that dont trigger
|
||||||
// flush (eg: mempurge output memtable), then update imm_flush_needed.
|
// flush (eg: mempurge output memtable), then update imm_flush_needed.
|
||||||
// Note: if race condition and imm_flush_needed is set to true
|
// Note: if race condition and imm_flush_needed is set to true
|
||||||
// when there is num_flush_not_started_==0, then there is no
|
// when there is num_flush_not_started_==0, then there is no
|
||||||
|
@ -209,8 +209,7 @@ TEST_F(MemTableListTest, Empty) {
|
|||||||
ASSERT_FALSE(list.IsFlushPending());
|
ASSERT_FALSE(list.IsFlushPending());
|
||||||
|
|
||||||
autovector<MemTable*> mems;
|
autovector<MemTable*> mems;
|
||||||
list.PickMemtablesToFlush(
|
list.PickMemtablesToFlush(port::kMaxUint64 /* memtable_id */, &mems);
|
||||||
std::numeric_limits<uint64_t>::max() /* memtable_id */, &mems);
|
|
||||||
ASSERT_EQ(0, mems.size());
|
ASSERT_EQ(0, mems.size());
|
||||||
|
|
||||||
autovector<MemTable*> to_delete;
|
autovector<MemTable*> to_delete;
|
||||||
@ -419,8 +418,7 @@ TEST_F(MemTableListTest, GetFromHistoryTest) {
|
|||||||
// Flush this memtable from the list.
|
// Flush this memtable from the list.
|
||||||
// (It will then be a part of the memtable history).
|
// (It will then be a part of the memtable history).
|
||||||
autovector<MemTable*> to_flush;
|
autovector<MemTable*> to_flush;
|
||||||
list.PickMemtablesToFlush(
|
list.PickMemtablesToFlush(port::kMaxUint64 /* memtable_id */, &to_flush);
|
||||||
std::numeric_limits<uint64_t>::max() /* memtable_id */, &to_flush);
|
|
||||||
ASSERT_EQ(1, to_flush.size());
|
ASSERT_EQ(1, to_flush.size());
|
||||||
|
|
||||||
MutableCFOptions mutable_cf_options(options);
|
MutableCFOptions mutable_cf_options(options);
|
||||||
@ -474,8 +472,7 @@ TEST_F(MemTableListTest, GetFromHistoryTest) {
|
|||||||
ASSERT_EQ(0, to_delete.size());
|
ASSERT_EQ(0, to_delete.size());
|
||||||
|
|
||||||
to_flush.clear();
|
to_flush.clear();
|
||||||
list.PickMemtablesToFlush(
|
list.PickMemtablesToFlush(port::kMaxUint64 /* memtable_id */, &to_flush);
|
||||||
std::numeric_limits<uint64_t>::max() /* memtable_id */, &to_flush);
|
|
||||||
ASSERT_EQ(1, to_flush.size());
|
ASSERT_EQ(1, to_flush.size());
|
||||||
|
|
||||||
// Flush second memtable
|
// Flush second memtable
|
||||||
@ -578,15 +575,15 @@ TEST_F(MemTableListTest, FlushPendingTest) {
|
|||||||
std::string value;
|
std::string value;
|
||||||
MergeContext merge_context;
|
MergeContext merge_context;
|
||||||
|
|
||||||
ASSERT_OK(mem->Add(++seq, kTypeValue, "key1", std::to_string(i),
|
ASSERT_OK(mem->Add(++seq, kTypeValue, "key1", ToString(i),
|
||||||
nullptr /* kv_prot_info */));
|
nullptr /* kv_prot_info */));
|
||||||
ASSERT_OK(mem->Add(++seq, kTypeValue, "keyN" + std::to_string(i), "valueN",
|
ASSERT_OK(mem->Add(++seq, kTypeValue, "keyN" + ToString(i), "valueN",
|
||||||
nullptr /* kv_prot_info */));
|
nullptr /* kv_prot_info */));
|
||||||
ASSERT_OK(mem->Add(++seq, kTypeValue, "keyX" + std::to_string(i), "value",
|
ASSERT_OK(mem->Add(++seq, kTypeValue, "keyX" + ToString(i), "value",
|
||||||
nullptr /* kv_prot_info */));
|
nullptr /* kv_prot_info */));
|
||||||
ASSERT_OK(mem->Add(++seq, kTypeValue, "keyM" + std::to_string(i), "valueM",
|
ASSERT_OK(mem->Add(++seq, kTypeValue, "keyM" + ToString(i), "valueM",
|
||||||
nullptr /* kv_prot_info */));
|
nullptr /* kv_prot_info */));
|
||||||
ASSERT_OK(mem->Add(++seq, kTypeDeletion, "keyX" + std::to_string(i), "",
|
ASSERT_OK(mem->Add(++seq, kTypeDeletion, "keyX" + ToString(i), "",
|
||||||
nullptr /* kv_prot_info */));
|
nullptr /* kv_prot_info */));
|
||||||
|
|
||||||
tables.push_back(mem);
|
tables.push_back(mem);
|
||||||
@ -596,8 +593,7 @@ TEST_F(MemTableListTest, FlushPendingTest) {
|
|||||||
ASSERT_FALSE(list.IsFlushPending());
|
ASSERT_FALSE(list.IsFlushPending());
|
||||||
ASSERT_FALSE(list.imm_flush_needed.load(std::memory_order_acquire));
|
ASSERT_FALSE(list.imm_flush_needed.load(std::memory_order_acquire));
|
||||||
autovector<MemTable*> to_flush;
|
autovector<MemTable*> to_flush;
|
||||||
list.PickMemtablesToFlush(
|
list.PickMemtablesToFlush(port::kMaxUint64 /* memtable_id */, &to_flush);
|
||||||
std::numeric_limits<uint64_t>::max() /* memtable_id */, &to_flush);
|
|
||||||
ASSERT_EQ(0, to_flush.size());
|
ASSERT_EQ(0, to_flush.size());
|
||||||
|
|
||||||
// Request a flush even though there is nothing to flush
|
// Request a flush even though there is nothing to flush
|
||||||
@ -606,8 +602,7 @@ TEST_F(MemTableListTest, FlushPendingTest) {
|
|||||||
ASSERT_FALSE(list.imm_flush_needed.load(std::memory_order_acquire));
|
ASSERT_FALSE(list.imm_flush_needed.load(std::memory_order_acquire));
|
||||||
|
|
||||||
// Attempt to 'flush' to clear request for flush
|
// Attempt to 'flush' to clear request for flush
|
||||||
list.PickMemtablesToFlush(
|
list.PickMemtablesToFlush(port::kMaxUint64 /* memtable_id */, &to_flush);
|
||||||
std::numeric_limits<uint64_t>::max() /* memtable_id */, &to_flush);
|
|
||||||
ASSERT_EQ(0, to_flush.size());
|
ASSERT_EQ(0, to_flush.size());
|
||||||
ASSERT_FALSE(list.IsFlushPending());
|
ASSERT_FALSE(list.IsFlushPending());
|
||||||
ASSERT_FALSE(list.imm_flush_needed.load(std::memory_order_acquire));
|
ASSERT_FALSE(list.imm_flush_needed.load(std::memory_order_acquire));
|
||||||
@ -631,8 +626,7 @@ TEST_F(MemTableListTest, FlushPendingTest) {
|
|||||||
ASSERT_TRUE(list.imm_flush_needed.load(std::memory_order_acquire));
|
ASSERT_TRUE(list.imm_flush_needed.load(std::memory_order_acquire));
|
||||||
|
|
||||||
// Pick tables to flush
|
// Pick tables to flush
|
||||||
list.PickMemtablesToFlush(
|
list.PickMemtablesToFlush(port::kMaxUint64 /* memtable_id */, &to_flush);
|
||||||
std::numeric_limits<uint64_t>::max() /* memtable_id */, &to_flush);
|
|
||||||
ASSERT_EQ(2, to_flush.size());
|
ASSERT_EQ(2, to_flush.size());
|
||||||
ASSERT_EQ(2, list.NumNotFlushed());
|
ASSERT_EQ(2, list.NumNotFlushed());
|
||||||
ASSERT_FALSE(list.IsFlushPending());
|
ASSERT_FALSE(list.IsFlushPending());
|
||||||
@ -653,8 +647,7 @@ TEST_F(MemTableListTest, FlushPendingTest) {
|
|||||||
ASSERT_EQ(0, to_delete.size());
|
ASSERT_EQ(0, to_delete.size());
|
||||||
|
|
||||||
// Pick tables to flush
|
// Pick tables to flush
|
||||||
list.PickMemtablesToFlush(
|
list.PickMemtablesToFlush(port::kMaxUint64 /* memtable_id */, &to_flush);
|
||||||
std::numeric_limits<uint64_t>::max() /* memtable_id */, &to_flush);
|
|
||||||
ASSERT_EQ(3, to_flush.size());
|
ASSERT_EQ(3, to_flush.size());
|
||||||
ASSERT_EQ(3, list.NumNotFlushed());
|
ASSERT_EQ(3, list.NumNotFlushed());
|
||||||
ASSERT_FALSE(list.IsFlushPending());
|
ASSERT_FALSE(list.IsFlushPending());
|
||||||
@ -662,8 +655,7 @@ TEST_F(MemTableListTest, FlushPendingTest) {
|
|||||||
|
|
||||||
// Pick tables to flush again
|
// Pick tables to flush again
|
||||||
autovector<MemTable*> to_flush2;
|
autovector<MemTable*> to_flush2;
|
||||||
list.PickMemtablesToFlush(
|
list.PickMemtablesToFlush(port::kMaxUint64 /* memtable_id */, &to_flush2);
|
||||||
std::numeric_limits<uint64_t>::max() /* memtable_id */, &to_flush2);
|
|
||||||
ASSERT_EQ(0, to_flush2.size());
|
ASSERT_EQ(0, to_flush2.size());
|
||||||
ASSERT_EQ(3, list.NumNotFlushed());
|
ASSERT_EQ(3, list.NumNotFlushed());
|
||||||
ASSERT_FALSE(list.IsFlushPending());
|
ASSERT_FALSE(list.IsFlushPending());
|
||||||
@ -681,8 +673,7 @@ TEST_F(MemTableListTest, FlushPendingTest) {
|
|||||||
ASSERT_TRUE(list.imm_flush_needed.load(std::memory_order_acquire));
|
ASSERT_TRUE(list.imm_flush_needed.load(std::memory_order_acquire));
|
||||||
|
|
||||||
// Pick tables to flush again
|
// Pick tables to flush again
|
||||||
list.PickMemtablesToFlush(
|
list.PickMemtablesToFlush(port::kMaxUint64 /* memtable_id */, &to_flush2);
|
||||||
std::numeric_limits<uint64_t>::max() /* memtable_id */, &to_flush2);
|
|
||||||
ASSERT_EQ(1, to_flush2.size());
|
ASSERT_EQ(1, to_flush2.size());
|
||||||
ASSERT_EQ(4, list.NumNotFlushed());
|
ASSERT_EQ(4, list.NumNotFlushed());
|
||||||
ASSERT_FALSE(list.IsFlushPending());
|
ASSERT_FALSE(list.IsFlushPending());
|
||||||
@ -703,8 +694,7 @@ TEST_F(MemTableListTest, FlushPendingTest) {
|
|||||||
ASSERT_EQ(0, to_delete.size());
|
ASSERT_EQ(0, to_delete.size());
|
||||||
|
|
||||||
// Pick tables to flush
|
// Pick tables to flush
|
||||||
list.PickMemtablesToFlush(
|
list.PickMemtablesToFlush(port::kMaxUint64 /* memtable_id */, &to_flush);
|
||||||
std::numeric_limits<uint64_t>::max() /* memtable_id */, &to_flush);
|
|
||||||
// Should pick 4 of 5 since 1 table has been picked in to_flush2
|
// Should pick 4 of 5 since 1 table has been picked in to_flush2
|
||||||
ASSERT_EQ(4, to_flush.size());
|
ASSERT_EQ(4, to_flush.size());
|
||||||
ASSERT_EQ(5, list.NumNotFlushed());
|
ASSERT_EQ(5, list.NumNotFlushed());
|
||||||
@ -713,8 +703,7 @@ TEST_F(MemTableListTest, FlushPendingTest) {
|
|||||||
|
|
||||||
// Pick tables to flush again
|
// Pick tables to flush again
|
||||||
autovector<MemTable*> to_flush3;
|
autovector<MemTable*> to_flush3;
|
||||||
list.PickMemtablesToFlush(
|
list.PickMemtablesToFlush(port::kMaxUint64 /* memtable_id */, &to_flush3);
|
||||||
std::numeric_limits<uint64_t>::max() /* memtable_id */, &to_flush3);
|
|
||||||
ASSERT_EQ(0, to_flush3.size()); // nothing not in progress of being flushed
|
ASSERT_EQ(0, to_flush3.size()); // nothing not in progress of being flushed
|
||||||
ASSERT_EQ(5, list.NumNotFlushed());
|
ASSERT_EQ(5, list.NumNotFlushed());
|
||||||
ASSERT_FALSE(list.IsFlushPending());
|
ASSERT_FALSE(list.IsFlushPending());
|
||||||
@ -860,15 +849,15 @@ TEST_F(MemTableListTest, AtomicFlusTest) {
|
|||||||
|
|
||||||
std::string value;
|
std::string value;
|
||||||
|
|
||||||
ASSERT_OK(mem->Add(++seq, kTypeValue, "key1", std::to_string(i),
|
ASSERT_OK(mem->Add(++seq, kTypeValue, "key1", ToString(i),
|
||||||
nullptr /* kv_prot_info */));
|
nullptr /* kv_prot_info */));
|
||||||
ASSERT_OK(mem->Add(++seq, kTypeValue, "keyN" + std::to_string(i),
|
ASSERT_OK(mem->Add(++seq, kTypeValue, "keyN" + ToString(i), "valueN",
|
||||||
"valueN", nullptr /* kv_prot_info */));
|
|
||||||
ASSERT_OK(mem->Add(++seq, kTypeValue, "keyX" + std::to_string(i), "value",
|
|
||||||
nullptr /* kv_prot_info */));
|
nullptr /* kv_prot_info */));
|
||||||
ASSERT_OK(mem->Add(++seq, kTypeValue, "keyM" + std::to_string(i),
|
ASSERT_OK(mem->Add(++seq, kTypeValue, "keyX" + ToString(i), "value",
|
||||||
"valueM", nullptr /* kv_prot_info */));
|
nullptr /* kv_prot_info */));
|
||||||
ASSERT_OK(mem->Add(++seq, kTypeDeletion, "keyX" + std::to_string(i), "",
|
ASSERT_OK(mem->Add(++seq, kTypeValue, "keyM" + ToString(i), "valueM",
|
||||||
|
nullptr /* kv_prot_info */));
|
||||||
|
ASSERT_OK(mem->Add(++seq, kTypeDeletion, "keyX" + ToString(i), "",
|
||||||
nullptr /* kv_prot_info */));
|
nullptr /* kv_prot_info */));
|
||||||
|
|
||||||
elem.push_back(mem);
|
elem.push_back(mem);
|
||||||
@ -883,8 +872,7 @@ TEST_F(MemTableListTest, AtomicFlusTest) {
|
|||||||
auto* list = lists[i];
|
auto* list = lists[i];
|
||||||
ASSERT_FALSE(list->IsFlushPending());
|
ASSERT_FALSE(list->IsFlushPending());
|
||||||
ASSERT_FALSE(list->imm_flush_needed.load(std::memory_order_acquire));
|
ASSERT_FALSE(list->imm_flush_needed.load(std::memory_order_acquire));
|
||||||
list->PickMemtablesToFlush(
|
list->PickMemtablesToFlush(port::kMaxUint64 /* memtable_id */,
|
||||||
std::numeric_limits<uint64_t>::max() /* memtable_id */,
|
|
||||||
&flush_candidates[i]);
|
&flush_candidates[i]);
|
||||||
ASSERT_EQ(0, flush_candidates[i].size());
|
ASSERT_EQ(0, flush_candidates[i].size());
|
||||||
}
|
}
|
||||||
|
@ -41,7 +41,7 @@ class ObsoleteFilesTest : public DBTestBase {
|
|||||||
WriteOptions options;
|
WriteOptions options;
|
||||||
options.sync = false;
|
options.sync = false;
|
||||||
for (int i = startkey; i < (numkeys + startkey) ; i++) {
|
for (int i = startkey; i < (numkeys + startkey) ; i++) {
|
||||||
std::string temp = std::to_string(i);
|
std::string temp = ToString(i);
|
||||||
Slice key(temp);
|
Slice key(temp);
|
||||||
Slice value(temp);
|
Slice value(temp);
|
||||||
ASSERT_OK(db_->Put(options, key, value));
|
ASSERT_OK(db_->Put(options, key, value));
|
||||||
|
@ -75,21 +75,21 @@ TEST_F(PerfContextTest, SeekIntoDeletion) {
|
|||||||
ReadOptions read_options;
|
ReadOptions read_options;
|
||||||
|
|
||||||
for (int i = 0; i < FLAGS_total_keys; ++i) {
|
for (int i = 0; i < FLAGS_total_keys; ++i) {
|
||||||
std::string key = "k" + std::to_string(i);
|
std::string key = "k" + ToString(i);
|
||||||
std::string value = "v" + std::to_string(i);
|
std::string value = "v" + ToString(i);
|
||||||
|
|
||||||
ASSERT_OK(db->Put(write_options, key, value));
|
ASSERT_OK(db->Put(write_options, key, value));
|
||||||
}
|
}
|
||||||
|
|
||||||
for (int i = 0; i < FLAGS_total_keys -1 ; ++i) {
|
for (int i = 0; i < FLAGS_total_keys -1 ; ++i) {
|
||||||
std::string key = "k" + std::to_string(i);
|
std::string key = "k" + ToString(i);
|
||||||
ASSERT_OK(db->Delete(write_options, key));
|
ASSERT_OK(db->Delete(write_options, key));
|
||||||
}
|
}
|
||||||
|
|
||||||
HistogramImpl hist_get;
|
HistogramImpl hist_get;
|
||||||
HistogramImpl hist_get_time;
|
HistogramImpl hist_get_time;
|
||||||
for (int i = 0; i < FLAGS_total_keys - 1; ++i) {
|
for (int i = 0; i < FLAGS_total_keys - 1; ++i) {
|
||||||
std::string key = "k" + std::to_string(i);
|
std::string key = "k" + ToString(i);
|
||||||
std::string value;
|
std::string value;
|
||||||
|
|
||||||
get_perf_context()->Reset();
|
get_perf_context()->Reset();
|
||||||
@ -130,7 +130,7 @@ TEST_F(PerfContextTest, SeekIntoDeletion) {
|
|||||||
HistogramImpl hist_seek;
|
HistogramImpl hist_seek;
|
||||||
for (int i = 0; i < FLAGS_total_keys; ++i) {
|
for (int i = 0; i < FLAGS_total_keys; ++i) {
|
||||||
std::unique_ptr<Iterator> iter(db->NewIterator(read_options));
|
std::unique_ptr<Iterator> iter(db->NewIterator(read_options));
|
||||||
std::string key = "k" + std::to_string(i);
|
std::string key = "k" + ToString(i);
|
||||||
|
|
||||||
get_perf_context()->Reset();
|
get_perf_context()->Reset();
|
||||||
StopWatchNano timer(SystemClock::Default().get(), true);
|
StopWatchNano timer(SystemClock::Default().get(), true);
|
||||||
@ -265,8 +265,8 @@ void ProfileQueries(bool enabled_time = false) {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
std::string key = "k" + std::to_string(i);
|
std::string key = "k" + ToString(i);
|
||||||
std::string value = "v" + std::to_string(i);
|
std::string value = "v" + ToString(i);
|
||||||
|
|
||||||
std::vector<std::string> values;
|
std::vector<std::string> values;
|
||||||
|
|
||||||
@ -297,8 +297,8 @@ void ProfileQueries(bool enabled_time = false) {
|
|||||||
if (i == kFlushFlag) {
|
if (i == kFlushFlag) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
std::string key = "k" + std::to_string(i);
|
std::string key = "k" + ToString(i);
|
||||||
std::string expected_value = "v" + std::to_string(i);
|
std::string expected_value = "v" + ToString(i);
|
||||||
std::string value;
|
std::string value;
|
||||||
|
|
||||||
std::vector<Slice> multiget_keys = {Slice(key)};
|
std::vector<Slice> multiget_keys = {Slice(key)};
|
||||||
@ -415,8 +415,8 @@ void ProfileQueries(bool enabled_time = false) {
|
|||||||
if (i == kFlushFlag) {
|
if (i == kFlushFlag) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
std::string key = "k" + std::to_string(i);
|
std::string key = "k" + ToString(i);
|
||||||
std::string expected_value = "v" + std::to_string(i);
|
std::string expected_value = "v" + ToString(i);
|
||||||
std::string value;
|
std::string value;
|
||||||
|
|
||||||
std::vector<Slice> multiget_keys = {Slice(key)};
|
std::vector<Slice> multiget_keys = {Slice(key)};
|
||||||
@ -543,8 +543,8 @@ TEST_F(PerfContextTest, SeekKeyComparison) {
|
|||||||
SetPerfLevel(kEnableTime);
|
SetPerfLevel(kEnableTime);
|
||||||
StopWatchNano timer(SystemClock::Default().get());
|
StopWatchNano timer(SystemClock::Default().get());
|
||||||
for (const int i : keys) {
|
for (const int i : keys) {
|
||||||
std::string key = "k" + std::to_string(i);
|
std::string key = "k" + ToString(i);
|
||||||
std::string value = "v" + std::to_string(i);
|
std::string value = "v" + ToString(i);
|
||||||
|
|
||||||
get_perf_context()->Reset();
|
get_perf_context()->Reset();
|
||||||
timer.Start();
|
timer.Start();
|
||||||
@ -565,8 +565,8 @@ TEST_F(PerfContextTest, SeekKeyComparison) {
|
|||||||
HistogramImpl hist_next;
|
HistogramImpl hist_next;
|
||||||
|
|
||||||
for (int i = 0; i < FLAGS_total_keys; ++i) {
|
for (int i = 0; i < FLAGS_total_keys; ++i) {
|
||||||
std::string key = "k" + std::to_string(i);
|
std::string key = "k" + ToString(i);
|
||||||
std::string value = "v" + std::to_string(i);
|
std::string value = "v" + ToString(i);
|
||||||
|
|
||||||
std::unique_ptr<Iterator> iter(db->NewIterator(read_options));
|
std::unique_ptr<Iterator> iter(db->NewIterator(read_options));
|
||||||
get_perf_context()->Reset();
|
get_perf_context()->Reset();
|
||||||
@ -841,7 +841,7 @@ TEST_F(PerfContextTest, CPUTimer) {
|
|||||||
|
|
||||||
std::string max_str = "0";
|
std::string max_str = "0";
|
||||||
for (int i = 0; i < FLAGS_total_keys; ++i) {
|
for (int i = 0; i < FLAGS_total_keys; ++i) {
|
||||||
std::string i_str = std::to_string(i);
|
std::string i_str = ToString(i);
|
||||||
std::string key = "k" + i_str;
|
std::string key = "k" + i_str;
|
||||||
std::string value = "v" + i_str;
|
std::string value = "v" + i_str;
|
||||||
max_str = max_str > i_str ? max_str : i_str;
|
max_str = max_str > i_str ? max_str : i_str;
|
||||||
@ -935,9 +935,9 @@ TEST_F(PerfContextTest, CPUTimer) {
|
|||||||
get_perf_context()->Reset();
|
get_perf_context()->Reset();
|
||||||
auto count = get_perf_context()->iter_seek_cpu_nanos;
|
auto count = get_perf_context()->iter_seek_cpu_nanos;
|
||||||
for (int i = 0; i < FLAGS_total_keys; ++i) {
|
for (int i = 0; i < FLAGS_total_keys; ++i) {
|
||||||
iter->Seek("k" + std::to_string(i));
|
iter->Seek("k" + ToString(i));
|
||||||
ASSERT_TRUE(iter->Valid());
|
ASSERT_TRUE(iter->Valid());
|
||||||
ASSERT_EQ("v" + std::to_string(i), iter->value().ToString());
|
ASSERT_EQ("v" + ToString(i), iter->value().ToString());
|
||||||
auto next_count = get_perf_context()->iter_seek_cpu_nanos;
|
auto next_count = get_perf_context()->iter_seek_cpu_nanos;
|
||||||
ASSERT_GT(next_count, count);
|
ASSERT_GT(next_count, count);
|
||||||
count = next_count;
|
count = next_count;
|
||||||
|
@ -220,8 +220,8 @@ class PlainTableDBTest : public testing::Test,
|
|||||||
|
|
||||||
int NumTableFilesAtLevel(int level) {
|
int NumTableFilesAtLevel(int level) {
|
||||||
std::string property;
|
std::string property;
|
||||||
EXPECT_TRUE(db_->GetProperty(
|
EXPECT_TRUE(db_->GetProperty("rocksdb.num-files-at-level" + ToString(level),
|
||||||
"rocksdb.num-files-at-level" + std::to_string(level), &property));
|
&property));
|
||||||
return atoi(property.c_str());
|
return atoi(property.c_str());
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -889,7 +889,7 @@ TEST_P(PlainTableDBTest, IteratorLargeKeys) {
|
|||||||
};
|
};
|
||||||
|
|
||||||
for (size_t i = 0; i < 7; i++) {
|
for (size_t i = 0; i < 7; i++) {
|
||||||
ASSERT_OK(Put(key_list[i], std::to_string(i)));
|
ASSERT_OK(Put(key_list[i], ToString(i)));
|
||||||
}
|
}
|
||||||
|
|
||||||
ASSERT_OK(dbfull()->TEST_FlushMemTable());
|
ASSERT_OK(dbfull()->TEST_FlushMemTable());
|
||||||
@ -900,7 +900,7 @@ TEST_P(PlainTableDBTest, IteratorLargeKeys) {
|
|||||||
for (size_t i = 0; i < 7; i++) {
|
for (size_t i = 0; i < 7; i++) {
|
||||||
ASSERT_TRUE(iter->Valid());
|
ASSERT_TRUE(iter->Valid());
|
||||||
ASSERT_EQ(key_list[i], iter->key().ToString());
|
ASSERT_EQ(key_list[i], iter->key().ToString());
|
||||||
ASSERT_EQ(std::to_string(i), iter->value().ToString());
|
ASSERT_EQ(ToString(i), iter->value().ToString());
|
||||||
iter->Next();
|
iter->Next();
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -937,7 +937,7 @@ TEST_P(PlainTableDBTest, IteratorLargeKeysWithPrefix) {
|
|||||||
MakeLongKeyWithPrefix(26, '6')};
|
MakeLongKeyWithPrefix(26, '6')};
|
||||||
|
|
||||||
for (size_t i = 0; i < 7; i++) {
|
for (size_t i = 0; i < 7; i++) {
|
||||||
ASSERT_OK(Put(key_list[i], std::to_string(i)));
|
ASSERT_OK(Put(key_list[i], ToString(i)));
|
||||||
}
|
}
|
||||||
|
|
||||||
ASSERT_OK(dbfull()->TEST_FlushMemTable());
|
ASSERT_OK(dbfull()->TEST_FlushMemTable());
|
||||||
@ -948,7 +948,7 @@ TEST_P(PlainTableDBTest, IteratorLargeKeysWithPrefix) {
|
|||||||
for (size_t i = 0; i < 7; i++) {
|
for (size_t i = 0; i < 7; i++) {
|
||||||
ASSERT_TRUE(iter->Valid());
|
ASSERT_TRUE(iter->Valid());
|
||||||
ASSERT_EQ(key_list[i], iter->key().ToString());
|
ASSERT_EQ(key_list[i], iter->key().ToString());
|
||||||
ASSERT_EQ(std::to_string(i), iter->value().ToString());
|
ASSERT_EQ(ToString(i), iter->value().ToString());
|
||||||
iter->Next();
|
iter->Next();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -628,7 +628,7 @@ TEST_F(PrefixTest, DynamicPrefixIterator) {
|
|||||||
TestKey test_key(prefix, FLAGS_items_per_prefix / 2);
|
TestKey test_key(prefix, FLAGS_items_per_prefix / 2);
|
||||||
std::string s;
|
std::string s;
|
||||||
Slice key = TestKeyToSlice(s, test_key);
|
Slice key = TestKeyToSlice(s, test_key);
|
||||||
std::string value = "v" + std::to_string(0);
|
std::string value = "v" + ToString(0);
|
||||||
|
|
||||||
get_perf_context()->Reset();
|
get_perf_context()->Reset();
|
||||||
StopWatchNano timer(SystemClock::Default().get(), true);
|
StopWatchNano timer(SystemClock::Default().get(), true);
|
||||||
|
@ -147,7 +147,7 @@ class Repairer {
|
|||||||
const auto* cf_opts = GetColumnFamilyOptions(cf_name);
|
const auto* cf_opts = GetColumnFamilyOptions(cf_name);
|
||||||
if (cf_opts == nullptr) {
|
if (cf_opts == nullptr) {
|
||||||
return Status::Corruption("Encountered unknown column family with name=" +
|
return Status::Corruption("Encountered unknown column family with name=" +
|
||||||
cf_name + ", id=" + std::to_string(cf_id));
|
cf_name + ", id=" + ToString(cf_id));
|
||||||
}
|
}
|
||||||
Options opts(db_options_, *cf_opts);
|
Options opts(db_options_, *cf_opts);
|
||||||
MutableCFOptions mut_cf_opts(opts);
|
MutableCFOptions mut_cf_opts(opts);
|
||||||
|
@ -289,7 +289,7 @@ TEST_F(RepairTest, RepairMultipleColumnFamilies) {
|
|||||||
CreateAndReopenWithCF({"pikachu1", "pikachu2"}, CurrentOptions());
|
CreateAndReopenWithCF({"pikachu1", "pikachu2"}, CurrentOptions());
|
||||||
for (int i = 0; i < kNumCfs; ++i) {
|
for (int i = 0; i < kNumCfs; ++i) {
|
||||||
for (int j = 0; j < kEntriesPerCf; ++j) {
|
for (int j = 0; j < kEntriesPerCf; ++j) {
|
||||||
ASSERT_OK(Put(i, "key" + std::to_string(j), "val" + std::to_string(j)));
|
ASSERT_OK(Put(i, "key" + ToString(j), "val" + ToString(j)));
|
||||||
if (j == kEntriesPerCf - 1 && i == kNumCfs - 1) {
|
if (j == kEntriesPerCf - 1 && i == kNumCfs - 1) {
|
||||||
// Leave one unflushed so we can verify WAL entries are properly
|
// Leave one unflushed so we can verify WAL entries are properly
|
||||||
// associated with column families.
|
// associated with column families.
|
||||||
@ -313,7 +313,7 @@ TEST_F(RepairTest, RepairMultipleColumnFamilies) {
|
|||||||
CurrentOptions());
|
CurrentOptions());
|
||||||
for (int i = 0; i < kNumCfs; ++i) {
|
for (int i = 0; i < kNumCfs; ++i) {
|
||||||
for (int j = 0; j < kEntriesPerCf; ++j) {
|
for (int j = 0; j < kEntriesPerCf; ++j) {
|
||||||
ASSERT_EQ(Get(i, "key" + std::to_string(j)), "val" + std::to_string(j));
|
ASSERT_EQ(Get(i, "key" + ToString(j)), "val" + ToString(j));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -334,7 +334,7 @@ TEST_F(RepairTest, RepairColumnFamilyOptions) {
|
|||||||
std::vector<Options>{opts, rev_opts});
|
std::vector<Options>{opts, rev_opts});
|
||||||
for (int i = 0; i < kNumCfs; ++i) {
|
for (int i = 0; i < kNumCfs; ++i) {
|
||||||
for (int j = 0; j < kEntriesPerCf; ++j) {
|
for (int j = 0; j < kEntriesPerCf; ++j) {
|
||||||
ASSERT_OK(Put(i, "key" + std::to_string(j), "val" + std::to_string(j)));
|
ASSERT_OK(Put(i, "key" + ToString(j), "val" + ToString(j)));
|
||||||
if (i == kNumCfs - 1 && j == kEntriesPerCf - 1) {
|
if (i == kNumCfs - 1 && j == kEntriesPerCf - 1) {
|
||||||
// Leave one unflushed so we can verify RepairDB's flush logic
|
// Leave one unflushed so we can verify RepairDB's flush logic
|
||||||
continue;
|
continue;
|
||||||
@ -352,7 +352,7 @@ TEST_F(RepairTest, RepairColumnFamilyOptions) {
|
|||||||
std::vector<Options>{opts, rev_opts}));
|
std::vector<Options>{opts, rev_opts}));
|
||||||
for (int i = 0; i < kNumCfs; ++i) {
|
for (int i = 0; i < kNumCfs; ++i) {
|
||||||
for (int j = 0; j < kEntriesPerCf; ++j) {
|
for (int j = 0; j < kEntriesPerCf; ++j) {
|
||||||
ASSERT_EQ(Get(i, "key" + std::to_string(j)), "val" + std::to_string(j));
|
ASSERT_EQ(Get(i, "key" + ToString(j)), "val" + ToString(j));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -377,7 +377,7 @@ TEST_F(RepairTest, RepairColumnFamilyOptions) {
|
|||||||
std::vector<Options>{opts, rev_opts}));
|
std::vector<Options>{opts, rev_opts}));
|
||||||
for (int i = 0; i < kNumCfs; ++i) {
|
for (int i = 0; i < kNumCfs; ++i) {
|
||||||
for (int j = 0; j < kEntriesPerCf; ++j) {
|
for (int j = 0; j < kEntriesPerCf; ++j) {
|
||||||
ASSERT_EQ(Get(i, "key" + std::to_string(j)), "val" + std::to_string(j));
|
ASSERT_EQ(Get(i, "key" + ToString(j)), "val" + ToString(j));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -33,9 +33,10 @@ class DisableGCSnapshotChecker : public SnapshotChecker {
|
|||||||
// By returning kNotInSnapshot, we prevent all the values from being GCed
|
// By returning kNotInSnapshot, we prevent all the values from being GCed
|
||||||
return SnapshotCheckerResult::kNotInSnapshot;
|
return SnapshotCheckerResult::kNotInSnapshot;
|
||||||
}
|
}
|
||||||
static DisableGCSnapshotChecker* Instance();
|
static DisableGCSnapshotChecker* Instance() { return &instance_; }
|
||||||
|
|
||||||
protected:
|
protected:
|
||||||
|
static DisableGCSnapshotChecker instance_;
|
||||||
explicit DisableGCSnapshotChecker() {}
|
explicit DisableGCSnapshotChecker() {}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -468,7 +468,8 @@ Status TableCache::Get(
|
|||||||
#ifndef ROCKSDB_LITE
|
#ifndef ROCKSDB_LITE
|
||||||
// Put the replay log in row cache only if something was found.
|
// Put the replay log in row cache only if something was found.
|
||||||
if (!done && s.ok() && row_cache_entry && !row_cache_entry->empty()) {
|
if (!done && s.ok() && row_cache_entry && !row_cache_entry->empty()) {
|
||||||
size_t charge = row_cache_entry->capacity() + sizeof(std::string);
|
size_t charge =
|
||||||
|
row_cache_key.Size() + row_cache_entry->size() + sizeof(std::string);
|
||||||
void* row_ptr = new std::string(std::move(*row_cache_entry));
|
void* row_ptr = new std::string(std::move(*row_cache_entry));
|
||||||
// If row cache is full, it's OK to continue.
|
// If row cache is full, it's OK to continue.
|
||||||
ioptions_.row_cache
|
ioptions_.row_cache
|
||||||
@ -591,7 +592,8 @@ Status TableCache::MultiGet(
|
|||||||
user_key.size());
|
user_key.size());
|
||||||
// Put the replay log in row cache only if something was found.
|
// Put the replay log in row cache only if something was found.
|
||||||
if (s.ok() && !row_cache_entry.empty()) {
|
if (s.ok() && !row_cache_entry.empty()) {
|
||||||
size_t charge = row_cache_entry.capacity() + sizeof(std::string);
|
size_t charge =
|
||||||
|
row_cache_key.Size() + row_cache_entry.size() + sizeof(std::string);
|
||||||
void* row_ptr = new std::string(std::move(row_cache_entry));
|
void* row_ptr = new std::string(std::move(row_cache_entry));
|
||||||
// If row cache is full, it's OK.
|
// If row cache is full, it's OK.
|
||||||
ioptions_.row_cache
|
ioptions_.row_cache
|
||||||
|
@ -1144,7 +1144,7 @@ class VersionBuilder::Rep {
|
|||||||
|
|
||||||
size_t table_cache_capacity = table_cache_->get_cache()->GetCapacity();
|
size_t table_cache_capacity = table_cache_->get_cache()->GetCapacity();
|
||||||
bool always_load = (table_cache_capacity == TableCache::kInfiniteCapacity);
|
bool always_load = (table_cache_capacity == TableCache::kInfiniteCapacity);
|
||||||
size_t max_load = std::numeric_limits<size_t>::max();
|
size_t max_load = port::kMaxSizet;
|
||||||
|
|
||||||
if (!always_load) {
|
if (!always_load) {
|
||||||
// If it is initial loading and not set to always loading all the
|
// If it is initial loading and not set to always loading all the
|
||||||
|
@ -1702,9 +1702,11 @@ TEST_F(VersionBuilderTest, EstimatedActiveKeys) {
|
|||||||
const uint32_t kDeletionsPerFile = 100;
|
const uint32_t kDeletionsPerFile = 100;
|
||||||
for (uint32_t i = 0; i < kNumFiles; ++i) {
|
for (uint32_t i = 0; i < kNumFiles; ++i) {
|
||||||
Add(static_cast<int>(i / kFilesPerLevel), i + 1,
|
Add(static_cast<int>(i / kFilesPerLevel), i + 1,
|
||||||
std::to_string((i + 100) * 1000).c_str(),
|
ToString((i + 100) * 1000).c_str(),
|
||||||
std::to_string((i + 100) * 1000 + 999).c_str(), 100U, 0, 100, 100,
|
ToString((i + 100) * 1000 + 999).c_str(),
|
||||||
kEntriesPerFile, kDeletionsPerFile, (i < kTotalSamples));
|
100U, 0, 100, 100,
|
||||||
|
kEntriesPerFile, kDeletionsPerFile,
|
||||||
|
(i < kTotalSamples));
|
||||||
}
|
}
|
||||||
// minus 2X for the number of deletion entries because:
|
// minus 2X for the number of deletion entries because:
|
||||||
// 1x for deletion entry does not count as a data entry.
|
// 1x for deletion entry does not count as a data entry.
|
||||||
|
@ -817,7 +817,7 @@ std::string VersionEdit::DebugString(bool hex_key) const {
|
|||||||
r.append(" temperature: ");
|
r.append(" temperature: ");
|
||||||
// Maybe change to human readable format whenthe feature becomes
|
// Maybe change to human readable format whenthe feature becomes
|
||||||
// permanent
|
// permanent
|
||||||
r.append(std::to_string(static_cast<int>(f.temperature)));
|
r.append(ToString(static_cast<int>(f.temperature)));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -928,7 +928,7 @@ std::string VersionEdit::DebugJSON(int edit_num, bool hex_key) const {
|
|||||||
jw << "FileChecksum" << Slice(f.file_checksum).ToString(true);
|
jw << "FileChecksum" << Slice(f.file_checksum).ToString(true);
|
||||||
jw << "FileChecksumFuncName" << f.file_checksum_func_name;
|
jw << "FileChecksumFuncName" << f.file_checksum_func_name;
|
||||||
if (f.temperature != Temperature::kUnknown) {
|
if (f.temperature != Temperature::kUnknown) {
|
||||||
jw << "temperature" << std::to_string(static_cast<int>(f.temperature));
|
jw << "temperature" << ToString(static_cast<int>(f.temperature));
|
||||||
}
|
}
|
||||||
if (f.oldest_blob_file_number != kInvalidBlobFileNumber) {
|
if (f.oldest_blob_file_number != kInvalidBlobFileNumber) {
|
||||||
jw << "OldestBlobFile" << f.oldest_blob_file_number;
|
jw << "OldestBlobFile" << f.oldest_blob_file_number;
|
||||||
|
@ -1517,7 +1517,7 @@ uint64_t Version::GetSstFilesSize() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
void Version::GetCreationTimeOfOldestFile(uint64_t* creation_time) {
|
void Version::GetCreationTimeOfOldestFile(uint64_t* creation_time) {
|
||||||
uint64_t oldest_time = std::numeric_limits<uint64_t>::max();
|
uint64_t oldest_time = port::kMaxUint64;
|
||||||
for (int level = 0; level < storage_info_.num_non_empty_levels_; level++) {
|
for (int level = 0; level < storage_info_.num_non_empty_levels_; level++) {
|
||||||
for (FileMetaData* meta : storage_info_.LevelFiles(level)) {
|
for (FileMetaData* meta : storage_info_.LevelFiles(level)) {
|
||||||
assert(meta->fd.table_reader != nullptr);
|
assert(meta->fd.table_reader != nullptr);
|
||||||
@ -3986,7 +3986,7 @@ std::string Version::DebugString(bool hex, bool print_stats) const {
|
|||||||
}
|
}
|
||||||
if (print_stats) {
|
if (print_stats) {
|
||||||
r.append("(");
|
r.append("(");
|
||||||
r.append(std::to_string(
|
r.append(ToString(
|
||||||
files[i]->stats.num_reads_sampled.load(std::memory_order_relaxed)));
|
files[i]->stats.num_reads_sampled.load(std::memory_order_relaxed)));
|
||||||
r.append(")");
|
r.append(")");
|
||||||
}
|
}
|
||||||
|
@ -1213,7 +1213,7 @@ class VersionSet {
|
|||||||
// new_log_number_for_empty_cf.
|
// new_log_number_for_empty_cf.
|
||||||
uint64_t PreComputeMinLogNumberWithUnflushedData(
|
uint64_t PreComputeMinLogNumberWithUnflushedData(
|
||||||
uint64_t new_log_number_for_empty_cf) const {
|
uint64_t new_log_number_for_empty_cf) const {
|
||||||
uint64_t min_log_num = std::numeric_limits<uint64_t>::max();
|
uint64_t min_log_num = port::kMaxUint64;
|
||||||
for (auto cfd : *column_family_set_) {
|
for (auto cfd : *column_family_set_) {
|
||||||
// It's safe to ignore dropped column families here:
|
// It's safe to ignore dropped column families here:
|
||||||
// cfd->IsDropped() becomes true after the drop is persisted in MANIFEST.
|
// cfd->IsDropped() becomes true after the drop is persisted in MANIFEST.
|
||||||
@ -1229,7 +1229,7 @@ class VersionSet {
|
|||||||
// file, except data from `cfd_to_skip`.
|
// file, except data from `cfd_to_skip`.
|
||||||
uint64_t PreComputeMinLogNumberWithUnflushedData(
|
uint64_t PreComputeMinLogNumberWithUnflushedData(
|
||||||
const ColumnFamilyData* cfd_to_skip) const {
|
const ColumnFamilyData* cfd_to_skip) const {
|
||||||
uint64_t min_log_num = std::numeric_limits<uint64_t>::max();
|
uint64_t min_log_num = port::kMaxUint64;
|
||||||
for (auto cfd : *column_family_set_) {
|
for (auto cfd : *column_family_set_) {
|
||||||
if (cfd == cfd_to_skip) {
|
if (cfd == cfd_to_skip) {
|
||||||
continue;
|
continue;
|
||||||
@ -1246,7 +1246,7 @@ class VersionSet {
|
|||||||
// file, except data from `cfds_to_skip`.
|
// file, except data from `cfds_to_skip`.
|
||||||
uint64_t PreComputeMinLogNumberWithUnflushedData(
|
uint64_t PreComputeMinLogNumberWithUnflushedData(
|
||||||
const std::unordered_set<const ColumnFamilyData*>& cfds_to_skip) const {
|
const std::unordered_set<const ColumnFamilyData*>& cfds_to_skip) const {
|
||||||
uint64_t min_log_num = std::numeric_limits<uint64_t>::max();
|
uint64_t min_log_num = port::kMaxUint64;
|
||||||
for (auto cfd : *column_family_set_) {
|
for (auto cfd : *column_family_set_) {
|
||||||
if (cfds_to_skip.count(cfd)) {
|
if (cfds_to_skip.count(cfd)) {
|
||||||
continue;
|
continue;
|
||||||
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user