2016-02-10 00:12:00 +01:00
|
|
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
2017-07-16 01:03:42 +02:00
|
|
|
// This source code is licensed under both the GPLv2 (found in the
|
|
|
|
// COPYING file in the root directory) and Apache 2.0 License
|
|
|
|
// (found in the LICENSE.Apache file in the root directory).
|
2015-07-14 01:53:38 +02:00
|
|
|
//
|
|
|
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
|
|
|
// Use of this source code is governed by a BSD-style license that can be
|
|
|
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
|
|
|
|
2015-10-12 22:05:42 +02:00
|
|
|
#include "db/db_test_util.h"
|
2020-07-09 23:33:42 +02:00
|
|
|
|
2016-12-16 20:00:03 +01:00
|
|
|
#include "db/forward_iterator.h"
|
2020-09-16 00:12:58 +02:00
|
|
|
#include "rocksdb/convenience.h"
|
2017-06-27 01:52:06 +02:00
|
|
|
#include "rocksdb/env_encryption.h"
|
2019-08-10 00:08:36 +02:00
|
|
|
#include "rocksdb/utilities/object_registry.h"
|
2020-07-09 23:33:42 +02:00
|
|
|
#include "util/random.h"
|
2015-07-14 01:53:38 +02:00
|
|
|
|
2020-02-20 21:07:53 +01:00
|
|
|
namespace ROCKSDB_NAMESPACE {
|
2015-07-14 01:53:38 +02:00
|
|
|
|
2020-05-05 22:09:12 +02:00
|
|
|
namespace {
|
|
|
|
int64_t MaybeCurrentTime(Env* env) {
|
|
|
|
int64_t time = 1337346000; // arbitrary fallback default
|
2020-08-25 01:41:42 +02:00
|
|
|
env->GetCurrentTime(&time).PermitUncheckedError();
|
2020-05-05 22:09:12 +02:00
|
|
|
return time;
|
|
|
|
}
|
|
|
|
} // namespace
|
|
|
|
|
2015-07-14 01:53:38 +02:00
|
|
|
// Special Env used to delay background operations
|
|
|
|
|
Fix+clean up handling of mock sleeps (#7101)
Summary:
We have a number of tests hanging on MacOS and windows due to
mishandling of code for mock sleeps. In addition, the code was in
terrible shape because the same variable (addon_time_) would sometimes
refer to microseconds and sometimes to seconds. One test even assumed it
was nanoseconds but was written to pass anyway.
This has been cleaned up so that DB tests generally use a SpecialEnv
function to mock sleep, for either some number of microseconds or seconds
depending on the function called. But to call one of these, the test must first
call SetMockSleep (precondition enforced with assertion), which also turns
sleeps in RocksDB into mock sleeps. To also removes accounting for actual
clock time, call SetTimeElapseOnlySleepOnReopen, which implies
SetMockSleep (on DB re-open). This latter setting only works by applying
on DB re-open, otherwise havoc can ensue if Env goes back in time with
DB open.
More specifics:
Removed some unused test classes, and updated comments on the general
problem.
Fixed DBSSTTest.GetTotalSstFilesSize using a sync point callback instead
of mock time. For this we have the only modification to production code,
inserting a sync point callback in flush_job.cc, which is not a change to
production behavior.
Removed unnecessary resetting of mock times to 0 in many tests. RocksDB
deals in relative time. Any behaviors relying on absolute date/time are likely
a bug. (The above test DBSSTTest.GetTotalSstFilesSize was the only one
clearly injecting a specific absolute time for actual testing convenience.) Just
in case I misunderstood some test, I put this note in each replacement:
// NOTE: Presumed unnecessary and removed: resetting mock time in env
Strengthened some tests like MergeTestTime, MergeCompactionTimeTest, and
FilterCompactionTimeTest in db_test.cc
stats_history_test and blob_db_test are each their own beast, rather deeply
dependent on MockTimeEnv. Each gets its own variant of a work-around for
TimedWait in a mock time environment. (Reduces redundancy and
inconsistency in stats_history_test.)
Intended follow-up:
Remove TimedWait from the public API of InstrumentedCondVar, and only
make that accessible through Env by passing in an InstrumentedCondVar and
a deadline. Then the Env implementations mocking time can fix this problem
without using sync points. (Test infrastructure using sync points interferes
with individual tests' control over sync points.)
With that change, we can simplify/consolidate the scattered work-arounds.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7101
Test Plan: make check on Linux and MacOS
Reviewed By: zhichao-cao
Differential Revision: D23032815
Pulled By: pdillinger
fbshipit-source-id: 7f33967ada8b83011fb54e8279365c008bd6610b
2020-08-11 21:39:49 +02:00
|
|
|
SpecialEnv::SpecialEnv(Env* base, bool time_elapse_only_sleep)
|
2015-07-14 01:53:38 +02:00
|
|
|
: EnvWrapper(base),
|
2020-05-05 22:09:12 +02:00
|
|
|
maybe_starting_time_(MaybeCurrentTime(base)),
|
2015-07-14 01:53:38 +02:00
|
|
|
rnd_(301),
|
|
|
|
sleep_counter_(this),
|
Fix+clean up handling of mock sleeps (#7101)
Summary:
We have a number of tests hanging on MacOS and windows due to
mishandling of code for mock sleeps. In addition, the code was in
terrible shape because the same variable (addon_time_) would sometimes
refer to microseconds and sometimes to seconds. One test even assumed it
was nanoseconds but was written to pass anyway.
This has been cleaned up so that DB tests generally use a SpecialEnv
function to mock sleep, for either some number of microseconds or seconds
depending on the function called. But to call one of these, the test must first
call SetMockSleep (precondition enforced with assertion), which also turns
sleeps in RocksDB into mock sleeps. To also removes accounting for actual
clock time, call SetTimeElapseOnlySleepOnReopen, which implies
SetMockSleep (on DB re-open). This latter setting only works by applying
on DB re-open, otherwise havoc can ensue if Env goes back in time with
DB open.
More specifics:
Removed some unused test classes, and updated comments on the general
problem.
Fixed DBSSTTest.GetTotalSstFilesSize using a sync point callback instead
of mock time. For this we have the only modification to production code,
inserting a sync point callback in flush_job.cc, which is not a change to
production behavior.
Removed unnecessary resetting of mock times to 0 in many tests. RocksDB
deals in relative time. Any behaviors relying on absolute date/time are likely
a bug. (The above test DBSSTTest.GetTotalSstFilesSize was the only one
clearly injecting a specific absolute time for actual testing convenience.) Just
in case I misunderstood some test, I put this note in each replacement:
// NOTE: Presumed unnecessary and removed: resetting mock time in env
Strengthened some tests like MergeTestTime, MergeCompactionTimeTest, and
FilterCompactionTimeTest in db_test.cc
stats_history_test and blob_db_test are each their own beast, rather deeply
dependent on MockTimeEnv. Each gets its own variant of a work-around for
TimedWait in a mock time environment. (Reduces redundancy and
inconsistency in stats_history_test.)
Intended follow-up:
Remove TimedWait from the public API of InstrumentedCondVar, and only
make that accessible through Env by passing in an InstrumentedCondVar and
a deadline. Then the Env implementations mocking time can fix this problem
without using sync points. (Test infrastructure using sync points interferes
with individual tests' control over sync points.)
With that change, we can simplify/consolidate the scattered work-arounds.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7101
Test Plan: make check on Linux and MacOS
Reviewed By: zhichao-cao
Differential Revision: D23032815
Pulled By: pdillinger
fbshipit-source-id: 7f33967ada8b83011fb54e8279365c008bd6610b
2020-08-11 21:39:49 +02:00
|
|
|
time_elapse_only_sleep_(time_elapse_only_sleep),
|
|
|
|
no_slowdown_(time_elapse_only_sleep) {
|
2015-07-14 01:53:38 +02:00
|
|
|
delay_sstable_sync_.store(false, std::memory_order_release);
|
|
|
|
drop_writes_.store(false, std::memory_order_release);
|
|
|
|
no_space_.store(false, std::memory_order_release);
|
|
|
|
non_writable_.store(false, std::memory_order_release);
|
|
|
|
count_random_reads_ = false;
|
|
|
|
count_sequential_reads_ = false;
|
|
|
|
manifest_sync_error_.store(false, std::memory_order_release);
|
|
|
|
manifest_write_error_.store(false, std::memory_order_release);
|
|
|
|
log_write_error_.store(false, std::memory_order_release);
|
2015-08-04 22:48:16 +02:00
|
|
|
random_file_open_counter_.store(0, std::memory_order_relaxed);
|
2016-08-04 02:42:06 +02:00
|
|
|
delete_count_.store(0, std::memory_order_relaxed);
|
|
|
|
num_open_wal_file_.store(0);
|
2015-07-14 01:53:38 +02:00
|
|
|
log_write_slowdown_ = 0;
|
|
|
|
bytes_written_ = 0;
|
|
|
|
sync_counter_ = 0;
|
|
|
|
non_writeable_rate_ = 0;
|
|
|
|
new_writable_count_ = 0;
|
|
|
|
non_writable_count_ = 0;
|
|
|
|
table_write_callback_ = nullptr;
|
|
|
|
}
|
2020-08-18 03:41:20 +02:00
|
|
|
DBTestBase::DBTestBase(const std::string path, bool env_do_fsync)
|
2019-09-20 21:00:55 +02:00
|
|
|
: mem_env_(nullptr), encrypted_env_(nullptr), option_config_(kDefault) {
|
2019-08-10 00:08:36 +02:00
|
|
|
Env* base_env = Env::Default();
|
2019-08-11 04:12:09 +02:00
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
const char* test_env_uri = getenv("TEST_ENV_URI");
|
2019-08-10 00:08:36 +02:00
|
|
|
if (test_env_uri) {
|
2019-12-17 05:00:43 +01:00
|
|
|
Env* test_env = nullptr;
|
|
|
|
Status s = Env::LoadEnv(test_env_uri, &test_env, &env_guard_);
|
|
|
|
base_env = test_env;
|
2019-08-10 00:08:36 +02:00
|
|
|
EXPECT_OK(s);
|
|
|
|
EXPECT_NE(Env::Default(), base_env);
|
|
|
|
}
|
2019-08-11 04:12:09 +02:00
|
|
|
#endif // !ROCKSDB_LITE
|
2019-08-10 00:08:36 +02:00
|
|
|
EXPECT_NE(nullptr, base_env);
|
|
|
|
if (getenv("MEM_ENV")) {
|
|
|
|
mem_env_ = new MockEnv(base_env);
|
|
|
|
}
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
if (getenv("ENCRYPTED_ENV")) {
|
2020-09-16 00:12:58 +02:00
|
|
|
std::shared_ptr<EncryptionProvider> provider;
|
|
|
|
Status s = EncryptionProvider::CreateFromString(
|
|
|
|
ConfigOptions(), std::string("test://") + getenv("ENCRYPTED_ENV"),
|
|
|
|
&provider);
|
|
|
|
encrypted_env_ = NewEncryptedEnv(mem_env_ ? mem_env_ : base_env, provider);
|
2019-08-10 00:08:36 +02:00
|
|
|
}
|
|
|
|
#endif // !ROCKSDB_LITE
|
|
|
|
env_ = new SpecialEnv(encrypted_env_ ? encrypted_env_
|
|
|
|
: (mem_env_ ? mem_env_ : base_env));
|
2015-07-14 01:53:38 +02:00
|
|
|
env_->SetBackgroundThreads(1, Env::LOW);
|
|
|
|
env_->SetBackgroundThreads(1, Env::HIGH);
|
2020-08-18 03:41:20 +02:00
|
|
|
env_->skip_fsync_ = !env_do_fsync;
|
2018-07-14 02:18:39 +02:00
|
|
|
dbname_ = test::PerThreadDBPath(env_, path);
|
2015-07-14 01:53:38 +02:00
|
|
|
alternative_wal_dir_ = dbname_ + "/wal";
|
2015-07-22 02:22:49 +02:00
|
|
|
alternative_db_log_dir_ = dbname_ + "/db_log_dir";
|
2015-07-14 01:53:38 +02:00
|
|
|
auto options = CurrentOptions();
|
2017-03-13 17:41:30 +01:00
|
|
|
options.env = env_;
|
2015-07-14 01:53:38 +02:00
|
|
|
auto delete_options = options;
|
|
|
|
delete_options.wal_dir = alternative_wal_dir_;
|
|
|
|
EXPECT_OK(DestroyDB(dbname_, delete_options));
|
|
|
|
// Destroy it for not alternative WAL dir is used.
|
|
|
|
EXPECT_OK(DestroyDB(dbname_, options));
|
|
|
|
db_ = nullptr;
|
|
|
|
Reopen(options);
|
2015-11-06 17:07:08 +01:00
|
|
|
Random::GetTLSInstance()->Reset(0xdeadbeef);
|
2015-07-14 01:53:38 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
DBTestBase::~DBTestBase() {
|
2020-02-20 21:07:53 +01:00
|
|
|
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
|
|
|
|
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({});
|
|
|
|
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
|
2015-07-14 01:53:38 +02:00
|
|
|
Close();
|
|
|
|
Options options;
|
|
|
|
options.db_paths.emplace_back(dbname_, 0);
|
|
|
|
options.db_paths.emplace_back(dbname_ + "_2", 0);
|
|
|
|
options.db_paths.emplace_back(dbname_ + "_3", 0);
|
|
|
|
options.db_paths.emplace_back(dbname_ + "_4", 0);
|
2017-03-13 17:41:30 +01:00
|
|
|
options.env = env_;
|
2017-01-17 22:38:40 +01:00
|
|
|
|
|
|
|
if (getenv("KEEP_DB")) {
|
|
|
|
printf("DB is still at %s\n", dbname_.c_str());
|
|
|
|
} else {
|
|
|
|
EXPECT_OK(DestroyDB(dbname_, options));
|
|
|
|
}
|
2015-07-14 01:53:38 +02:00
|
|
|
delete env_;
|
|
|
|
}
|
|
|
|
|
2016-01-22 20:02:43 +01:00
|
|
|
bool DBTestBase::ShouldSkipOptions(int option_config, int skip_mask) {
|
2015-10-14 02:33:23 +02:00
|
|
|
#ifdef ROCKSDB_LITE
|
|
|
|
// These options are not supported in ROCKSDB_LITE
|
2019-02-08 01:06:40 +01:00
|
|
|
if (option_config == kHashSkipList ||
|
|
|
|
option_config == kPlainTableFirstBytePrefix ||
|
|
|
|
option_config == kPlainTableCappedPrefix ||
|
|
|
|
option_config == kPlainTableCappedPrefixNonMmap ||
|
|
|
|
option_config == kPlainTableAllBytesPrefix ||
|
|
|
|
option_config == kVectorRep || option_config == kHashLinkList ||
|
|
|
|
option_config == kUniversalCompaction ||
|
|
|
|
option_config == kUniversalCompactionMultiLevel ||
|
|
|
|
option_config == kUniversalSubcompactions ||
|
|
|
|
option_config == kFIFOCompaction ||
|
|
|
|
option_config == kConcurrentSkipList) {
|
|
|
|
return true;
|
2015-10-14 02:33:23 +02:00
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2015-07-14 01:53:38 +02:00
|
|
|
if ((skip_mask & kSkipUniversalCompaction) &&
|
2016-01-22 20:02:43 +01:00
|
|
|
(option_config == kUniversalCompaction ||
|
2018-07-13 20:08:57 +02:00
|
|
|
option_config == kUniversalCompactionMultiLevel ||
|
|
|
|
option_config == kUniversalSubcompactions)) {
|
2016-01-22 20:02:43 +01:00
|
|
|
return true;
|
2015-07-14 01:53:38 +02:00
|
|
|
}
|
2016-01-22 20:02:43 +01:00
|
|
|
if ((skip_mask & kSkipMergePut) && option_config == kMergePut) {
|
|
|
|
return true;
|
2015-07-14 01:53:38 +02:00
|
|
|
}
|
|
|
|
if ((skip_mask & kSkipNoSeekToLast) &&
|
2016-01-22 20:02:43 +01:00
|
|
|
(option_config == kHashLinkList || option_config == kHashSkipList)) {
|
|
|
|
return true;
|
2015-07-14 01:53:38 +02:00
|
|
|
}
|
|
|
|
if ((skip_mask & kSkipPlainTable) &&
|
2016-01-22 20:02:43 +01:00
|
|
|
(option_config == kPlainTableAllBytesPrefix ||
|
|
|
|
option_config == kPlainTableFirstBytePrefix ||
|
|
|
|
option_config == kPlainTableCappedPrefix ||
|
|
|
|
option_config == kPlainTableCappedPrefixNonMmap)) {
|
|
|
|
return true;
|
2015-07-14 01:53:38 +02:00
|
|
|
}
|
|
|
|
if ((skip_mask & kSkipHashIndex) &&
|
2016-01-22 20:02:43 +01:00
|
|
|
(option_config == kBlockBasedTableWithPrefixHashIndex ||
|
|
|
|
option_config == kBlockBasedTableWithWholeKeyHashIndex)) {
|
|
|
|
return true;
|
2015-07-14 01:53:38 +02:00
|
|
|
}
|
2016-01-22 20:02:43 +01:00
|
|
|
if ((skip_mask & kSkipFIFOCompaction) && option_config == kFIFOCompaction) {
|
|
|
|
return true;
|
2015-07-14 01:53:38 +02:00
|
|
|
}
|
2016-01-22 20:02:43 +01:00
|
|
|
if ((skip_mask & kSkipMmapReads) && option_config == kWalDirAndMmapReads) {
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Switch to a fresh database with the next option configuration to
|
|
|
|
// test. Return false if there are no more configurations to test.
|
|
|
|
bool DBTestBase::ChangeOptions(int skip_mask) {
|
|
|
|
for (option_config_++; option_config_ < kEnd; option_config_++) {
|
|
|
|
if (ShouldSkipOptions(option_config_, skip_mask)) {
|
2015-07-14 01:53:38 +02:00
|
|
|
continue;
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (option_config_ >= kEnd) {
|
|
|
|
Destroy(last_options_);
|
|
|
|
return false;
|
|
|
|
} else {
|
|
|
|
auto options = CurrentOptions();
|
|
|
|
options.create_if_missing = true;
|
|
|
|
DestroyAndReopen(options);
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-08-31 21:59:02 +02:00
|
|
|
// Switch between different compaction styles.
|
2015-07-14 01:53:38 +02:00
|
|
|
bool DBTestBase::ChangeCompactOptions() {
|
|
|
|
if (option_config_ == kDefault) {
|
|
|
|
option_config_ = kUniversalCompaction;
|
|
|
|
Destroy(last_options_);
|
|
|
|
auto options = CurrentOptions();
|
|
|
|
options.create_if_missing = true;
|
|
|
|
TryReopen(options);
|
|
|
|
return true;
|
|
|
|
} else if (option_config_ == kUniversalCompaction) {
|
|
|
|
option_config_ = kUniversalCompactionMultiLevel;
|
|
|
|
Destroy(last_options_);
|
|
|
|
auto options = CurrentOptions();
|
|
|
|
options.create_if_missing = true;
|
|
|
|
TryReopen(options);
|
|
|
|
return true;
|
2015-08-05 07:19:07 +02:00
|
|
|
} else if (option_config_ == kUniversalCompactionMultiLevel) {
|
|
|
|
option_config_ = kLevelSubcompactions;
|
|
|
|
Destroy(last_options_);
|
|
|
|
auto options = CurrentOptions();
|
2015-08-31 21:59:02 +02:00
|
|
|
assert(options.max_subcompactions > 1);
|
|
|
|
TryReopen(options);
|
|
|
|
return true;
|
|
|
|
} else if (option_config_ == kLevelSubcompactions) {
|
|
|
|
option_config_ = kUniversalSubcompactions;
|
|
|
|
Destroy(last_options_);
|
|
|
|
auto options = CurrentOptions();
|
|
|
|
assert(options.max_subcompactions > 1);
|
2015-08-05 07:19:07 +02:00
|
|
|
TryReopen(options);
|
|
|
|
return true;
|
2015-07-14 01:53:38 +02:00
|
|
|
} else {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-03-01 02:32:58 +01:00
|
|
|
// Switch between different WAL settings
|
|
|
|
bool DBTestBase::ChangeWalOptions() {
|
|
|
|
if (option_config_ == kDefault) {
|
|
|
|
option_config_ = kDBLogDir;
|
|
|
|
Destroy(last_options_);
|
|
|
|
auto options = CurrentOptions();
|
|
|
|
Destroy(options);
|
|
|
|
options.create_if_missing = true;
|
|
|
|
TryReopen(options);
|
|
|
|
return true;
|
|
|
|
} else if (option_config_ == kDBLogDir) {
|
|
|
|
option_config_ = kWalDirAndMmapReads;
|
|
|
|
Destroy(last_options_);
|
|
|
|
auto options = CurrentOptions();
|
|
|
|
Destroy(options);
|
|
|
|
options.create_if_missing = true;
|
|
|
|
TryReopen(options);
|
|
|
|
return true;
|
|
|
|
} else if (option_config_ == kWalDirAndMmapReads) {
|
|
|
|
option_config_ = kRecycleLogFiles;
|
|
|
|
Destroy(last_options_);
|
|
|
|
auto options = CurrentOptions();
|
|
|
|
Destroy(options);
|
|
|
|
TryReopen(options);
|
|
|
|
return true;
|
|
|
|
} else {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-07-14 01:53:38 +02:00
|
|
|
// Switch between different filter policy
|
|
|
|
// Jump from kDefault to kFilter to kFullFilter
|
|
|
|
bool DBTestBase::ChangeFilterOptions() {
|
|
|
|
if (option_config_ == kDefault) {
|
|
|
|
option_config_ = kFilter;
|
|
|
|
} else if (option_config_ == kFilter) {
|
2015-08-21 07:33:25 +02:00
|
|
|
option_config_ = kFullFilterWithNewTableReaderForCompactions;
|
2017-03-22 17:11:23 +01:00
|
|
|
} else if (option_config_ == kFullFilterWithNewTableReaderForCompactions) {
|
|
|
|
option_config_ = kPartitionedFilterWithNewTableReaderForCompactions;
|
2015-07-14 01:53:38 +02:00
|
|
|
} else {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
Destroy(last_options_);
|
|
|
|
|
|
|
|
auto options = CurrentOptions();
|
|
|
|
options.create_if_missing = true;
|
|
|
|
TryReopen(options);
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2018-11-02 00:21:30 +01:00
|
|
|
// Switch between different DB options for file ingestion tests.
|
|
|
|
bool DBTestBase::ChangeOptionsForFileIngestionTest() {
|
|
|
|
if (option_config_ == kDefault) {
|
|
|
|
option_config_ = kUniversalCompaction;
|
|
|
|
Destroy(last_options_);
|
|
|
|
auto options = CurrentOptions();
|
|
|
|
options.create_if_missing = true;
|
|
|
|
TryReopen(options);
|
|
|
|
return true;
|
|
|
|
} else if (option_config_ == kUniversalCompaction) {
|
|
|
|
option_config_ = kUniversalCompactionMultiLevel;
|
|
|
|
Destroy(last_options_);
|
|
|
|
auto options = CurrentOptions();
|
|
|
|
options.create_if_missing = true;
|
|
|
|
TryReopen(options);
|
|
|
|
return true;
|
|
|
|
} else if (option_config_ == kUniversalCompactionMultiLevel) {
|
|
|
|
option_config_ = kLevelSubcompactions;
|
|
|
|
Destroy(last_options_);
|
|
|
|
auto options = CurrentOptions();
|
|
|
|
assert(options.max_subcompactions > 1);
|
|
|
|
TryReopen(options);
|
|
|
|
return true;
|
|
|
|
} else if (option_config_ == kLevelSubcompactions) {
|
|
|
|
option_config_ = kUniversalSubcompactions;
|
|
|
|
Destroy(last_options_);
|
|
|
|
auto options = CurrentOptions();
|
|
|
|
assert(options.max_subcompactions > 1);
|
|
|
|
TryReopen(options);
|
|
|
|
return true;
|
|
|
|
} else if (option_config_ == kUniversalSubcompactions) {
|
|
|
|
option_config_ = kDirectIO;
|
|
|
|
Destroy(last_options_);
|
|
|
|
auto options = CurrentOptions();
|
|
|
|
TryReopen(options);
|
|
|
|
return true;
|
|
|
|
} else {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-07-14 01:53:38 +02:00
|
|
|
// Return the current option configuration.
|
|
|
|
Options DBTestBase::CurrentOptions(
|
2017-05-31 19:45:47 +02:00
|
|
|
const anon::OptionsOverride& options_override) const {
|
|
|
|
return GetOptions(option_config_, GetDefaultOptions(), options_override);
|
|
|
|
}
|
|
|
|
|
|
|
|
Options DBTestBase::CurrentOptions(
|
|
|
|
const Options& default_options,
|
|
|
|
const anon::OptionsOverride& options_override) const {
|
|
|
|
return GetOptions(option_config_, default_options, options_override);
|
|
|
|
}
|
|
|
|
|
Fix many tests to run with MEM_ENV and ENCRYPTED_ENV; Introduce a MemoryFileSystem class (#7566)
Summary:
This PR does a few things:
1. The MockFileSystem class was split out from the MockEnv. This change would theoretically allow a MockFileSystem to be used by other Environments as well (if we created a means of constructing one). The MockFileSystem implements a FileSystem in its entirety and does not rely on any Wrapper implementation.
2. Make the RocksDB test suite work when MOCK_ENV=1 and ENCRYPTED_ENV=1 are set. To accomplish this, a few things were needed:
- The tests that tried to use the "wrong" environment (Env::Default() instead of env_) were updated
- The MockFileSystem was changed to support the features it was missing or mishandled (such as recursively deleting files in a directory or supporting renaming of a directory).
3. Updated the test framework to have a ROCKSDB_GTEST_SKIP macro. This can be used to flag tests that are skipped. Currently, this defaults to doing nothing (marks the test as SUCCESS) but will mark the tests as SKIPPED when RocksDB is upgraded to a version of gtest that supports this (gtest-1.10).
I have run a full "make check" with MEM_ENV, ENCRYPTED_ENV, both, and neither under both MacOS and RedHat. A few tests were disabled/skipped for the MEM/ENCRYPTED cases. The error_handler_fs_test fails/hangs for MEM_ENV (presumably a timing problem) and I will introduce another PR/issue to track that problem. (I will also push a change to disable those tests soon). There is one more test in DBTest2 that also fails which I need to investigate or skip before this PR is merged.
Theoretically, this PR should also allow the test suite to run against an Env loaded from the registry, though I do not have one to try it with currently.
Finally, once this is accepted, it would be nice if there was a CircleCI job to run these tests on a checkin so this effort does not become stale. I do not know how to do that, so if someone could write that job, it would be appreciated :)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7566
Reviewed By: zhichao-cao
Differential Revision: D24408980
Pulled By: jay-zhuang
fbshipit-source-id: 911b1554a4d0da06fd51feca0c090a4abdcb4a5f
2020-10-27 18:31:34 +01:00
|
|
|
Options DBTestBase::GetDefaultOptions() const {
|
2015-07-14 01:53:38 +02:00
|
|
|
Options options;
|
2015-08-26 23:19:31 +02:00
|
|
|
options.write_buffer_size = 4090 * 4096;
|
2016-03-24 20:45:50 +01:00
|
|
|
options.target_file_size_base = 2 * 1024 * 1024;
|
|
|
|
options.max_bytes_for_level_base = 10 * 1024 * 1024;
|
2016-04-08 02:40:42 +02:00
|
|
|
options.max_open_files = 5000;
|
|
|
|
options.wal_recovery_mode = WALRecoveryMode::kTolerateCorruptedTailRecords;
|
|
|
|
options.compaction_pri = CompactionPri::kByCompensatedSize;
|
Fix many tests to run with MEM_ENV and ENCRYPTED_ENV; Introduce a MemoryFileSystem class (#7566)
Summary:
This PR does a few things:
1. The MockFileSystem class was split out from the MockEnv. This change would theoretically allow a MockFileSystem to be used by other Environments as well (if we created a means of constructing one). The MockFileSystem implements a FileSystem in its entirety and does not rely on any Wrapper implementation.
2. Make the RocksDB test suite work when MOCK_ENV=1 and ENCRYPTED_ENV=1 are set. To accomplish this, a few things were needed:
- The tests that tried to use the "wrong" environment (Env::Default() instead of env_) were updated
- The MockFileSystem was changed to support the features it was missing or mishandled (such as recursively deleting files in a directory or supporting renaming of a directory).
3. Updated the test framework to have a ROCKSDB_GTEST_SKIP macro. This can be used to flag tests that are skipped. Currently, this defaults to doing nothing (marks the test as SUCCESS) but will mark the tests as SKIPPED when RocksDB is upgraded to a version of gtest that supports this (gtest-1.10).
I have run a full "make check" with MEM_ENV, ENCRYPTED_ENV, both, and neither under both MacOS and RedHat. A few tests were disabled/skipped for the MEM/ENCRYPTED cases. The error_handler_fs_test fails/hangs for MEM_ENV (presumably a timing problem) and I will introduce another PR/issue to track that problem. (I will also push a change to disable those tests soon). There is one more test in DBTest2 that also fails which I need to investigate or skip before this PR is merged.
Theoretically, this PR should also allow the test suite to run against an Env loaded from the registry, though I do not have one to try it with currently.
Finally, once this is accepted, it would be nice if there was a CircleCI job to run these tests on a checkin so this effort does not become stale. I do not know how to do that, so if someone could write that job, it would be appreciated :)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7566
Reviewed By: zhichao-cao
Differential Revision: D24408980
Pulled By: jay-zhuang
fbshipit-source-id: 911b1554a4d0da06fd51feca0c090a4abdcb4a5f
2020-10-27 18:31:34 +01:00
|
|
|
options.env = env_;
|
2017-05-31 19:45:47 +02:00
|
|
|
return options;
|
2015-07-14 01:53:38 +02:00
|
|
|
}
|
|
|
|
|
2017-05-31 19:45:47 +02:00
|
|
|
Options DBTestBase::GetOptions(
|
|
|
|
int option_config, const Options& default_options,
|
|
|
|
const anon::OptionsOverride& options_override) const {
|
2015-08-25 21:29:44 +02:00
|
|
|
// this redundant copy is to minimize code change w/o having lint error.
|
2017-05-31 19:45:47 +02:00
|
|
|
Options options = default_options;
|
2015-07-14 01:53:38 +02:00
|
|
|
BlockBasedTableOptions table_options;
|
|
|
|
bool set_block_based_table_factory = true;
|
2017-08-31 23:02:06 +02:00
|
|
|
#if !defined(OS_MACOSX) && !defined(OS_WIN) && !defined(OS_SOLARIS) && \
|
|
|
|
!defined(OS_AIX)
|
2020-02-20 21:07:53 +01:00
|
|
|
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearCallBack(
|
2017-05-23 03:40:41 +02:00
|
|
|
"NewRandomAccessFile:O_DIRECT");
|
2020-02-20 21:07:53 +01:00
|
|
|
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearCallBack(
|
|
|
|
"NewWritableFile:O_DIRECT");
|
2017-05-23 03:40:41 +02:00
|
|
|
#endif
|
|
|
|
|
2017-06-27 01:52:06 +02:00
|
|
|
bool can_allow_mmap = IsMemoryMappedAccessSupported();
|
2017-05-31 19:45:47 +02:00
|
|
|
switch (option_config) {
|
2015-10-14 02:33:23 +02:00
|
|
|
#ifndef ROCKSDB_LITE
|
2015-07-14 01:53:38 +02:00
|
|
|
case kHashSkipList:
|
|
|
|
options.prefix_extractor.reset(NewFixedPrefixTransform(1));
|
2015-10-13 00:06:38 +02:00
|
|
|
options.memtable_factory.reset(NewHashSkipListRepFactory(16));
|
2016-11-16 18:24:52 +01:00
|
|
|
options.allow_concurrent_memtable_write = false;
|
2019-05-14 02:43:47 +02:00
|
|
|
options.unordered_write = false;
|
2015-07-14 01:53:38 +02:00
|
|
|
break;
|
|
|
|
case kPlainTableFirstBytePrefix:
|
Fix many tests to run with MEM_ENV and ENCRYPTED_ENV; Introduce a MemoryFileSystem class (#7566)
Summary:
This PR does a few things:
1. The MockFileSystem class was split out from the MockEnv. This change would theoretically allow a MockFileSystem to be used by other Environments as well (if we created a means of constructing one). The MockFileSystem implements a FileSystem in its entirety and does not rely on any Wrapper implementation.
2. Make the RocksDB test suite work when MOCK_ENV=1 and ENCRYPTED_ENV=1 are set. To accomplish this, a few things were needed:
- The tests that tried to use the "wrong" environment (Env::Default() instead of env_) were updated
- The MockFileSystem was changed to support the features it was missing or mishandled (such as recursively deleting files in a directory or supporting renaming of a directory).
3. Updated the test framework to have a ROCKSDB_GTEST_SKIP macro. This can be used to flag tests that are skipped. Currently, this defaults to doing nothing (marks the test as SUCCESS) but will mark the tests as SKIPPED when RocksDB is upgraded to a version of gtest that supports this (gtest-1.10).
I have run a full "make check" with MEM_ENV, ENCRYPTED_ENV, both, and neither under both MacOS and RedHat. A few tests were disabled/skipped for the MEM/ENCRYPTED cases. The error_handler_fs_test fails/hangs for MEM_ENV (presumably a timing problem) and I will introduce another PR/issue to track that problem. (I will also push a change to disable those tests soon). There is one more test in DBTest2 that also fails which I need to investigate or skip before this PR is merged.
Theoretically, this PR should also allow the test suite to run against an Env loaded from the registry, though I do not have one to try it with currently.
Finally, once this is accepted, it would be nice if there was a CircleCI job to run these tests on a checkin so this effort does not become stale. I do not know how to do that, so if someone could write that job, it would be appreciated :)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7566
Reviewed By: zhichao-cao
Differential Revision: D24408980
Pulled By: jay-zhuang
fbshipit-source-id: 911b1554a4d0da06fd51feca0c090a4abdcb4a5f
2020-10-27 18:31:34 +01:00
|
|
|
options.table_factory.reset(NewPlainTableFactory());
|
2015-07-14 01:53:38 +02:00
|
|
|
options.prefix_extractor.reset(NewFixedPrefixTransform(1));
|
2017-06-27 01:52:06 +02:00
|
|
|
options.allow_mmap_reads = can_allow_mmap;
|
2015-07-14 01:53:38 +02:00
|
|
|
options.max_sequential_skip_in_iterations = 999999;
|
|
|
|
set_block_based_table_factory = false;
|
|
|
|
break;
|
|
|
|
case kPlainTableCappedPrefix:
|
Fix many tests to run with MEM_ENV and ENCRYPTED_ENV; Introduce a MemoryFileSystem class (#7566)
Summary:
This PR does a few things:
1. The MockFileSystem class was split out from the MockEnv. This change would theoretically allow a MockFileSystem to be used by other Environments as well (if we created a means of constructing one). The MockFileSystem implements a FileSystem in its entirety and does not rely on any Wrapper implementation.
2. Make the RocksDB test suite work when MOCK_ENV=1 and ENCRYPTED_ENV=1 are set. To accomplish this, a few things were needed:
- The tests that tried to use the "wrong" environment (Env::Default() instead of env_) were updated
- The MockFileSystem was changed to support the features it was missing or mishandled (such as recursively deleting files in a directory or supporting renaming of a directory).
3. Updated the test framework to have a ROCKSDB_GTEST_SKIP macro. This can be used to flag tests that are skipped. Currently, this defaults to doing nothing (marks the test as SUCCESS) but will mark the tests as SKIPPED when RocksDB is upgraded to a version of gtest that supports this (gtest-1.10).
I have run a full "make check" with MEM_ENV, ENCRYPTED_ENV, both, and neither under both MacOS and RedHat. A few tests were disabled/skipped for the MEM/ENCRYPTED cases. The error_handler_fs_test fails/hangs for MEM_ENV (presumably a timing problem) and I will introduce another PR/issue to track that problem. (I will also push a change to disable those tests soon). There is one more test in DBTest2 that also fails which I need to investigate or skip before this PR is merged.
Theoretically, this PR should also allow the test suite to run against an Env loaded from the registry, though I do not have one to try it with currently.
Finally, once this is accepted, it would be nice if there was a CircleCI job to run these tests on a checkin so this effort does not become stale. I do not know how to do that, so if someone could write that job, it would be appreciated :)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7566
Reviewed By: zhichao-cao
Differential Revision: D24408980
Pulled By: jay-zhuang
fbshipit-source-id: 911b1554a4d0da06fd51feca0c090a4abdcb4a5f
2020-10-27 18:31:34 +01:00
|
|
|
options.table_factory.reset(NewPlainTableFactory());
|
2015-07-14 01:53:38 +02:00
|
|
|
options.prefix_extractor.reset(NewCappedPrefixTransform(8));
|
2017-06-27 01:52:06 +02:00
|
|
|
options.allow_mmap_reads = can_allow_mmap;
|
2015-07-14 01:53:38 +02:00
|
|
|
options.max_sequential_skip_in_iterations = 999999;
|
|
|
|
set_block_based_table_factory = false;
|
|
|
|
break;
|
2015-09-17 01:57:43 +02:00
|
|
|
case kPlainTableCappedPrefixNonMmap:
|
Fix many tests to run with MEM_ENV and ENCRYPTED_ENV; Introduce a MemoryFileSystem class (#7566)
Summary:
This PR does a few things:
1. The MockFileSystem class was split out from the MockEnv. This change would theoretically allow a MockFileSystem to be used by other Environments as well (if we created a means of constructing one). The MockFileSystem implements a FileSystem in its entirety and does not rely on any Wrapper implementation.
2. Make the RocksDB test suite work when MOCK_ENV=1 and ENCRYPTED_ENV=1 are set. To accomplish this, a few things were needed:
- The tests that tried to use the "wrong" environment (Env::Default() instead of env_) were updated
- The MockFileSystem was changed to support the features it was missing or mishandled (such as recursively deleting files in a directory or supporting renaming of a directory).
3. Updated the test framework to have a ROCKSDB_GTEST_SKIP macro. This can be used to flag tests that are skipped. Currently, this defaults to doing nothing (marks the test as SUCCESS) but will mark the tests as SKIPPED when RocksDB is upgraded to a version of gtest that supports this (gtest-1.10).
I have run a full "make check" with MEM_ENV, ENCRYPTED_ENV, both, and neither under both MacOS and RedHat. A few tests were disabled/skipped for the MEM/ENCRYPTED cases. The error_handler_fs_test fails/hangs for MEM_ENV (presumably a timing problem) and I will introduce another PR/issue to track that problem. (I will also push a change to disable those tests soon). There is one more test in DBTest2 that also fails which I need to investigate or skip before this PR is merged.
Theoretically, this PR should also allow the test suite to run against an Env loaded from the registry, though I do not have one to try it with currently.
Finally, once this is accepted, it would be nice if there was a CircleCI job to run these tests on a checkin so this effort does not become stale. I do not know how to do that, so if someone could write that job, it would be appreciated :)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7566
Reviewed By: zhichao-cao
Differential Revision: D24408980
Pulled By: jay-zhuang
fbshipit-source-id: 911b1554a4d0da06fd51feca0c090a4abdcb4a5f
2020-10-27 18:31:34 +01:00
|
|
|
options.table_factory.reset(NewPlainTableFactory());
|
2015-09-17 01:57:43 +02:00
|
|
|
options.prefix_extractor.reset(NewCappedPrefixTransform(8));
|
|
|
|
options.allow_mmap_reads = false;
|
|
|
|
options.max_sequential_skip_in_iterations = 999999;
|
|
|
|
set_block_based_table_factory = false;
|
|
|
|
break;
|
2015-07-14 01:53:38 +02:00
|
|
|
case kPlainTableAllBytesPrefix:
|
Fix many tests to run with MEM_ENV and ENCRYPTED_ENV; Introduce a MemoryFileSystem class (#7566)
Summary:
This PR does a few things:
1. The MockFileSystem class was split out from the MockEnv. This change would theoretically allow a MockFileSystem to be used by other Environments as well (if we created a means of constructing one). The MockFileSystem implements a FileSystem in its entirety and does not rely on any Wrapper implementation.
2. Make the RocksDB test suite work when MOCK_ENV=1 and ENCRYPTED_ENV=1 are set. To accomplish this, a few things were needed:
- The tests that tried to use the "wrong" environment (Env::Default() instead of env_) were updated
- The MockFileSystem was changed to support the features it was missing or mishandled (such as recursively deleting files in a directory or supporting renaming of a directory).
3. Updated the test framework to have a ROCKSDB_GTEST_SKIP macro. This can be used to flag tests that are skipped. Currently, this defaults to doing nothing (marks the test as SUCCESS) but will mark the tests as SKIPPED when RocksDB is upgraded to a version of gtest that supports this (gtest-1.10).
I have run a full "make check" with MEM_ENV, ENCRYPTED_ENV, both, and neither under both MacOS and RedHat. A few tests were disabled/skipped for the MEM/ENCRYPTED cases. The error_handler_fs_test fails/hangs for MEM_ENV (presumably a timing problem) and I will introduce another PR/issue to track that problem. (I will also push a change to disable those tests soon). There is one more test in DBTest2 that also fails which I need to investigate or skip before this PR is merged.
Theoretically, this PR should also allow the test suite to run against an Env loaded from the registry, though I do not have one to try it with currently.
Finally, once this is accepted, it would be nice if there was a CircleCI job to run these tests on a checkin so this effort does not become stale. I do not know how to do that, so if someone could write that job, it would be appreciated :)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7566
Reviewed By: zhichao-cao
Differential Revision: D24408980
Pulled By: jay-zhuang
fbshipit-source-id: 911b1554a4d0da06fd51feca0c090a4abdcb4a5f
2020-10-27 18:31:34 +01:00
|
|
|
options.table_factory.reset(NewPlainTableFactory());
|
2015-07-14 01:53:38 +02:00
|
|
|
options.prefix_extractor.reset(NewNoopTransform());
|
2017-06-27 01:52:06 +02:00
|
|
|
options.allow_mmap_reads = can_allow_mmap;
|
2015-07-14 01:53:38 +02:00
|
|
|
options.max_sequential_skip_in_iterations = 999999;
|
|
|
|
set_block_based_table_factory = false;
|
|
|
|
break;
|
2015-10-14 02:33:23 +02:00
|
|
|
case kVectorRep:
|
|
|
|
options.memtable_factory.reset(new VectorRepFactory(100));
|
2016-11-16 18:24:52 +01:00
|
|
|
options.allow_concurrent_memtable_write = false;
|
2019-05-14 02:43:47 +02:00
|
|
|
options.unordered_write = false;
|
2015-10-14 02:33:23 +02:00
|
|
|
break;
|
|
|
|
case kHashLinkList:
|
|
|
|
options.prefix_extractor.reset(NewFixedPrefixTransform(1));
|
|
|
|
options.memtable_factory.reset(
|
|
|
|
NewHashLinkListRepFactory(4, 0, 3, true, 4));
|
2016-11-16 18:24:52 +01:00
|
|
|
options.allow_concurrent_memtable_write = false;
|
2019-05-14 02:43:47 +02:00
|
|
|
options.unordered_write = false;
|
2015-10-14 02:33:23 +02:00
|
|
|
break;
|
2018-06-02 05:28:19 +02:00
|
|
|
case kDirectIO: {
|
|
|
|
options.use_direct_reads = true;
|
|
|
|
options.use_direct_io_for_flush_and_compaction = true;
|
|
|
|
options.compaction_readahead_size = 2 * 1024 * 1024;
|
2020-07-09 23:33:42 +02:00
|
|
|
SetupSyncPointsToMockDirectIO();
|
2018-06-02 05:28:19 +02:00
|
|
|
break;
|
|
|
|
}
|
2015-10-14 02:33:23 +02:00
|
|
|
#endif // ROCKSDB_LITE
|
2015-07-14 01:53:38 +02:00
|
|
|
case kMergePut:
|
|
|
|
options.merge_operator = MergeOperators::CreatePutOperator();
|
|
|
|
break;
|
|
|
|
case kFilter:
|
|
|
|
table_options.filter_policy.reset(NewBloomFilterPolicy(10, true));
|
|
|
|
break;
|
2015-08-21 07:33:25 +02:00
|
|
|
case kFullFilterWithNewTableReaderForCompactions:
|
2015-07-14 01:53:38 +02:00
|
|
|
table_options.filter_policy.reset(NewBloomFilterPolicy(10, false));
|
2015-08-21 07:33:25 +02:00
|
|
|
options.new_table_reader_for_compaction_inputs = true;
|
2015-08-27 00:25:59 +02:00
|
|
|
options.compaction_readahead_size = 10 * 1024 * 1024;
|
2015-07-14 01:53:38 +02:00
|
|
|
break;
|
2017-03-22 17:11:23 +01:00
|
|
|
case kPartitionedFilterWithNewTableReaderForCompactions:
|
|
|
|
table_options.filter_policy.reset(NewBloomFilterPolicy(10, false));
|
|
|
|
table_options.partition_filters = true;
|
|
|
|
table_options.index_type =
|
|
|
|
BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch;
|
|
|
|
options.new_table_reader_for_compaction_inputs = true;
|
|
|
|
options.compaction_readahead_size = 10 * 1024 * 1024;
|
|
|
|
break;
|
2015-07-14 01:53:38 +02:00
|
|
|
case kUncompressed:
|
|
|
|
options.compression = kNoCompression;
|
|
|
|
break;
|
|
|
|
case kNumLevel_3:
|
|
|
|
options.num_levels = 3;
|
|
|
|
break;
|
|
|
|
case kDBLogDir:
|
2015-07-22 02:22:49 +02:00
|
|
|
options.db_log_dir = alternative_db_log_dir_;
|
2015-07-14 01:53:38 +02:00
|
|
|
break;
|
|
|
|
case kWalDirAndMmapReads:
|
|
|
|
options.wal_dir = alternative_wal_dir_;
|
|
|
|
// mmap reads should be orthogonal to WalDir setting, so we piggyback to
|
|
|
|
// this option config to test mmap reads as well
|
2017-06-27 01:52:06 +02:00
|
|
|
options.allow_mmap_reads = can_allow_mmap;
|
2015-07-14 01:53:38 +02:00
|
|
|
break;
|
|
|
|
case kManifestFileSize:
|
|
|
|
options.max_manifest_file_size = 50; // 50 bytes
|
2016-12-14 03:30:21 +01:00
|
|
|
break;
|
2015-07-15 21:28:22 +02:00
|
|
|
case kPerfOptions:
|
2015-07-14 01:53:38 +02:00
|
|
|
options.soft_rate_limit = 2.0;
|
|
|
|
options.delayed_write_rate = 8 * 1024 * 1024;
|
2016-04-14 22:56:29 +02:00
|
|
|
options.report_bg_io_stats = true;
|
2015-07-14 01:53:38 +02:00
|
|
|
// TODO(3.13) -- test more options
|
|
|
|
break;
|
|
|
|
case kUniversalCompaction:
|
|
|
|
options.compaction_style = kCompactionStyleUniversal;
|
|
|
|
options.num_levels = 1;
|
|
|
|
break;
|
|
|
|
case kUniversalCompactionMultiLevel:
|
|
|
|
options.compaction_style = kCompactionStyleUniversal;
|
|
|
|
options.num_levels = 8;
|
|
|
|
break;
|
|
|
|
case kCompressedBlockCache:
|
2017-06-27 01:52:06 +02:00
|
|
|
options.allow_mmap_writes = can_allow_mmap;
|
2015-10-13 00:06:38 +02:00
|
|
|
table_options.block_cache_compressed = NewLRUCache(8 * 1024 * 1024);
|
2015-07-14 01:53:38 +02:00
|
|
|
break;
|
|
|
|
case kInfiniteMaxOpenFiles:
|
|
|
|
options.max_open_files = -1;
|
|
|
|
break;
|
|
|
|
case kxxHashChecksum: {
|
|
|
|
table_options.checksum = kxxHash;
|
|
|
|
break;
|
|
|
|
}
|
2018-11-01 23:39:40 +01:00
|
|
|
case kxxHash64Checksum: {
|
|
|
|
table_options.checksum = kxxHash64;
|
|
|
|
break;
|
|
|
|
}
|
2015-07-14 01:53:38 +02:00
|
|
|
case kFIFOCompaction: {
|
|
|
|
options.compaction_style = kCompactionStyleFIFO;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
case kBlockBasedTableWithPrefixHashIndex: {
|
|
|
|
table_options.index_type = BlockBasedTableOptions::kHashSearch;
|
|
|
|
options.prefix_extractor.reset(NewFixedPrefixTransform(1));
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
case kBlockBasedTableWithWholeKeyHashIndex: {
|
|
|
|
table_options.index_type = BlockBasedTableOptions::kHashSearch;
|
|
|
|
options.prefix_extractor.reset(NewNoopTransform());
|
|
|
|
break;
|
|
|
|
}
|
2017-02-07 01:29:29 +01:00
|
|
|
case kBlockBasedTableWithPartitionedIndex: {
|
|
|
|
table_options.index_type = BlockBasedTableOptions::kTwoLevelIndexSearch;
|
|
|
|
options.prefix_extractor.reset(NewNoopTransform());
|
|
|
|
break;
|
|
|
|
}
|
2018-08-10 01:49:45 +02:00
|
|
|
case kBlockBasedTableWithPartitionedIndexFormat4: {
|
|
|
|
table_options.format_version = 4;
|
2018-09-18 02:13:58 +02:00
|
|
|
// Format 4 changes the binary index format. Since partitioned index is a
|
2018-06-05 04:59:44 +02:00
|
|
|
// super-set of simple indexes, we are also using kTwoLevelIndexSearch to
|
|
|
|
// test this format.
|
2018-06-07 01:44:52 +02:00
|
|
|
table_options.index_type = BlockBasedTableOptions::kTwoLevelIndexSearch;
|
2018-09-18 02:13:58 +02:00
|
|
|
// The top-level index in partition filters are also affected by format 4.
|
2018-06-07 01:44:52 +02:00
|
|
|
table_options.filter_policy.reset(NewBloomFilterPolicy(10, false));
|
|
|
|
table_options.partition_filters = true;
|
2018-09-18 02:13:58 +02:00
|
|
|
table_options.index_block_restart_interval = 8;
|
2018-06-05 04:59:44 +02:00
|
|
|
break;
|
|
|
|
}
|
2016-02-05 19:22:37 +01:00
|
|
|
case kBlockBasedTableWithIndexRestartInterval: {
|
|
|
|
table_options.index_block_restart_interval = 8;
|
|
|
|
break;
|
|
|
|
}
|
2015-07-14 01:53:38 +02:00
|
|
|
case kOptimizeFiltersForHits: {
|
|
|
|
options.optimize_filters_for_hits = true;
|
|
|
|
set_block_based_table_factory = true;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
case kRowCache: {
|
|
|
|
options.row_cache = NewLRUCache(1024 * 1024);
|
|
|
|
break;
|
|
|
|
}
|
2015-10-01 23:14:20 +02:00
|
|
|
case kRecycleLogFiles: {
|
|
|
|
options.recycle_log_file_num = 2;
|
|
|
|
break;
|
|
|
|
}
|
Parallelize L0-L1 Compaction: Restructure Compaction Job
Summary:
As of now compactions involving files from Level 0 and Level 1 are single
threaded because the files in L0, although sorted, are not range partitioned like
the other levels. This means that during L0-L1 compaction each file from L1
needs to be merged with potentially all the files from L0.
This attempt to parallelize the L0-L1 compaction assigns a thread and a
corresponding iterator to each L1 file that then considers only the key range
found in that L1 file and only the L0 files that have those keys (and only the
specific portion of those L0 files in which those keys are found). In this way
the overlap is minimized and potentially eliminated between different iterators
focusing on the same files.
The first step is to restructure the compaction logic to break L0-L1 compactions
into multiple, smaller, sequential compactions. Eventually each of these smaller
jobs will be run simultaneously. Areas to pay extra attention to are
# Correct aggregation of compaction job statistics across multiple threads
# Proper opening/closing of output files (make sure each thread's is unique)
# Keys that span multiple L1 files
# Skewed distributions of keys within L0 files
Test Plan: Make and run db_test (newer version has separate compaction tests) and compaction_job_stats_test
Reviewers: igor, noetzli, anthony, sdong, yhchiang
Reviewed By: yhchiang
Subscribers: MarkCallaghan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D42699
2015-08-03 20:32:14 +02:00
|
|
|
case kLevelSubcompactions: {
|
2015-08-31 21:59:02 +02:00
|
|
|
options.max_subcompactions = 4;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
case kUniversalSubcompactions: {
|
|
|
|
options.compaction_style = kCompactionStyleUniversal;
|
|
|
|
options.num_levels = 8;
|
|
|
|
options.max_subcompactions = 4;
|
Parallelize L0-L1 Compaction: Restructure Compaction Job
Summary:
As of now compactions involving files from Level 0 and Level 1 are single
threaded because the files in L0, although sorted, are not range partitioned like
the other levels. This means that during L0-L1 compaction each file from L1
needs to be merged with potentially all the files from L0.
This attempt to parallelize the L0-L1 compaction assigns a thread and a
corresponding iterator to each L1 file that then considers only the key range
found in that L1 file and only the L0 files that have those keys (and only the
specific portion of those L0 files in which those keys are found). In this way
the overlap is minimized and potentially eliminated between different iterators
focusing on the same files.
The first step is to restructure the compaction logic to break L0-L1 compactions
into multiple, smaller, sequential compactions. Eventually each of these smaller
jobs will be run simultaneously. Areas to pay extra attention to are
# Correct aggregation of compaction job statistics across multiple threads
# Proper opening/closing of output files (make sure each thread's is unique)
# Keys that span multiple L1 files
# Skewed distributions of keys within L0 files
Test Plan: Make and run db_test (newer version has separate compaction tests) and compaction_job_stats_test
Reviewers: igor, noetzli, anthony, sdong, yhchiang
Reviewed By: yhchiang
Subscribers: MarkCallaghan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D42699
2015-08-03 20:32:14 +02:00
|
|
|
break;
|
|
|
|
}
|
2016-01-31 19:48:43 +01:00
|
|
|
case kConcurrentSkipList: {
|
|
|
|
options.allow_concurrent_memtable_write = true;
|
|
|
|
options.enable_write_thread_adaptive_yield = true;
|
|
|
|
break;
|
|
|
|
}
|
2017-05-31 19:45:47 +02:00
|
|
|
case kPipelinedWrite: {
|
|
|
|
options.enable_pipelined_write = true;
|
|
|
|
break;
|
|
|
|
}
|
2017-06-24 23:06:43 +02:00
|
|
|
case kConcurrentWALWrites: {
|
|
|
|
// This options optimize 2PC commit path
|
2017-11-11 02:18:01 +01:00
|
|
|
options.two_write_queues = true;
|
2017-06-24 23:06:43 +02:00
|
|
|
options.manual_wal_flush = true;
|
|
|
|
break;
|
|
|
|
}
|
2019-05-14 02:43:47 +02:00
|
|
|
case kUnorderedWrite: {
|
|
|
|
options.allow_concurrent_memtable_write = false;
|
|
|
|
options.unordered_write = false;
|
|
|
|
break;
|
|
|
|
}
|
2015-07-14 01:53:38 +02:00
|
|
|
|
|
|
|
default:
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (options_override.filter_policy) {
|
|
|
|
table_options.filter_policy = options_override.filter_policy;
|
2017-03-22 17:11:23 +01:00
|
|
|
table_options.partition_filters = options_override.partition_filters;
|
2017-03-28 20:56:56 +02:00
|
|
|
table_options.metadata_block_size = options_override.metadata_block_size;
|
2015-07-14 01:53:38 +02:00
|
|
|
}
|
|
|
|
if (set_block_based_table_factory) {
|
|
|
|
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
|
|
|
|
}
|
|
|
|
options.env = env_;
|
|
|
|
options.create_if_missing = true;
|
2015-11-11 07:58:01 +01:00
|
|
|
options.fail_if_options_file_error = true;
|
2015-07-14 01:53:38 +02:00
|
|
|
return options;
|
|
|
|
}
|
|
|
|
|
|
|
|
void DBTestBase::CreateColumnFamilies(const std::vector<std::string>& cfs,
|
2015-10-13 00:06:38 +02:00
|
|
|
const Options& options) {
|
2015-07-14 01:53:38 +02:00
|
|
|
ColumnFamilyOptions cf_opts(options);
|
|
|
|
size_t cfi = handles_.size();
|
|
|
|
handles_.resize(cfi + cfs.size());
|
|
|
|
for (auto cf : cfs) {
|
2019-09-19 01:56:25 +02:00
|
|
|
Status s = db_->CreateColumnFamily(cf_opts, cf, &handles_[cfi++]);
|
|
|
|
ASSERT_OK(s);
|
2015-07-14 01:53:38 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
void DBTestBase::CreateAndReopenWithCF(const std::vector<std::string>& cfs,
|
2015-10-13 00:06:38 +02:00
|
|
|
const Options& options) {
|
2015-07-14 01:53:38 +02:00
|
|
|
CreateColumnFamilies(cfs, options);
|
|
|
|
std::vector<std::string> cfs_plus_default = cfs;
|
|
|
|
cfs_plus_default.insert(cfs_plus_default.begin(), kDefaultColumnFamilyName);
|
|
|
|
ReopenWithColumnFamilies(cfs_plus_default, options);
|
|
|
|
}
|
|
|
|
|
|
|
|
void DBTestBase::ReopenWithColumnFamilies(const std::vector<std::string>& cfs,
|
2015-10-13 00:06:38 +02:00
|
|
|
const std::vector<Options>& options) {
|
2015-07-14 01:53:38 +02:00
|
|
|
ASSERT_OK(TryReopenWithColumnFamilies(cfs, options));
|
|
|
|
}
|
|
|
|
|
|
|
|
void DBTestBase::ReopenWithColumnFamilies(const std::vector<std::string>& cfs,
|
2015-10-13 00:06:38 +02:00
|
|
|
const Options& options) {
|
2015-07-14 01:53:38 +02:00
|
|
|
ASSERT_OK(TryReopenWithColumnFamilies(cfs, options));
|
|
|
|
}
|
|
|
|
|
Fix+clean up handling of mock sleeps (#7101)
Summary:
We have a number of tests hanging on MacOS and windows due to
mishandling of code for mock sleeps. In addition, the code was in
terrible shape because the same variable (addon_time_) would sometimes
refer to microseconds and sometimes to seconds. One test even assumed it
was nanoseconds but was written to pass anyway.
This has been cleaned up so that DB tests generally use a SpecialEnv
function to mock sleep, for either some number of microseconds or seconds
depending on the function called. But to call one of these, the test must first
call SetMockSleep (precondition enforced with assertion), which also turns
sleeps in RocksDB into mock sleeps. To also removes accounting for actual
clock time, call SetTimeElapseOnlySleepOnReopen, which implies
SetMockSleep (on DB re-open). This latter setting only works by applying
on DB re-open, otherwise havoc can ensue if Env goes back in time with
DB open.
More specifics:
Removed some unused test classes, and updated comments on the general
problem.
Fixed DBSSTTest.GetTotalSstFilesSize using a sync point callback instead
of mock time. For this we have the only modification to production code,
inserting a sync point callback in flush_job.cc, which is not a change to
production behavior.
Removed unnecessary resetting of mock times to 0 in many tests. RocksDB
deals in relative time. Any behaviors relying on absolute date/time are likely
a bug. (The above test DBSSTTest.GetTotalSstFilesSize was the only one
clearly injecting a specific absolute time for actual testing convenience.) Just
in case I misunderstood some test, I put this note in each replacement:
// NOTE: Presumed unnecessary and removed: resetting mock time in env
Strengthened some tests like MergeTestTime, MergeCompactionTimeTest, and
FilterCompactionTimeTest in db_test.cc
stats_history_test and blob_db_test are each their own beast, rather deeply
dependent on MockTimeEnv. Each gets its own variant of a work-around for
TimedWait in a mock time environment. (Reduces redundancy and
inconsistency in stats_history_test.)
Intended follow-up:
Remove TimedWait from the public API of InstrumentedCondVar, and only
make that accessible through Env by passing in an InstrumentedCondVar and
a deadline. Then the Env implementations mocking time can fix this problem
without using sync points. (Test infrastructure using sync points interferes
with individual tests' control over sync points.)
With that change, we can simplify/consolidate the scattered work-arounds.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7101
Test Plan: make check on Linux and MacOS
Reviewed By: zhichao-cao
Differential Revision: D23032815
Pulled By: pdillinger
fbshipit-source-id: 7f33967ada8b83011fb54e8279365c008bd6610b
2020-08-11 21:39:49 +02:00
|
|
|
void DBTestBase::SetTimeElapseOnlySleepOnReopen(DBOptions* options) {
|
|
|
|
time_elapse_only_sleep_on_reopen_ = true;
|
|
|
|
|
|
|
|
// Need to disable stats dumping and persisting which also use
|
|
|
|
// RepeatableThread, which uses InstrumentedCondVar::TimedWaitInternal.
|
|
|
|
// With time_elapse_only_sleep_, this can hang on some platforms (MacOS)
|
|
|
|
// because (a) on some platforms, pthread_cond_timedwait does not appear
|
|
|
|
// to release the lock for other threads to operate if the deadline time
|
|
|
|
// is already passed, and (b) TimedWait calls are currently a bad abstraction
|
|
|
|
// because the deadline parameter is usually computed from Env time,
|
|
|
|
// but is interpreted in real clock time.
|
|
|
|
options->stats_dump_period_sec = 0;
|
|
|
|
options->stats_persist_period_sec = 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
void DBTestBase::MaybeInstallTimeElapseOnlySleep(const DBOptions& options) {
|
|
|
|
if (time_elapse_only_sleep_on_reopen_) {
|
|
|
|
assert(options.env == env_ ||
|
|
|
|
static_cast_with_check<CompositeEnvWrapper>(options.env)
|
|
|
|
->env_target() == env_);
|
|
|
|
assert(options.stats_dump_period_sec == 0);
|
|
|
|
assert(options.stats_persist_period_sec == 0);
|
|
|
|
// We cannot set these before destroying the last DB because they might
|
|
|
|
// cause a deadlock or similar without the appropriate options set in
|
|
|
|
// the DB.
|
|
|
|
env_->time_elapse_only_sleep_ = true;
|
|
|
|
env_->no_slowdown_ = true;
|
|
|
|
} else {
|
|
|
|
// Going back in same test run is not yet supported, so no
|
|
|
|
// reset in this case.
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-07-14 01:53:38 +02:00
|
|
|
Status DBTestBase::TryReopenWithColumnFamilies(
|
2015-10-13 00:06:38 +02:00
|
|
|
const std::vector<std::string>& cfs, const std::vector<Options>& options) {
|
2015-07-14 01:53:38 +02:00
|
|
|
Close();
|
|
|
|
EXPECT_EQ(cfs.size(), options.size());
|
|
|
|
std::vector<ColumnFamilyDescriptor> column_families;
|
|
|
|
for (size_t i = 0; i < cfs.size(); ++i) {
|
|
|
|
column_families.push_back(ColumnFamilyDescriptor(cfs[i], options[i]));
|
|
|
|
}
|
|
|
|
DBOptions db_opts = DBOptions(options[0]);
|
2018-12-29 03:00:00 +01:00
|
|
|
last_options_ = options[0];
|
Fix+clean up handling of mock sleeps (#7101)
Summary:
We have a number of tests hanging on MacOS and windows due to
mishandling of code for mock sleeps. In addition, the code was in
terrible shape because the same variable (addon_time_) would sometimes
refer to microseconds and sometimes to seconds. One test even assumed it
was nanoseconds but was written to pass anyway.
This has been cleaned up so that DB tests generally use a SpecialEnv
function to mock sleep, for either some number of microseconds or seconds
depending on the function called. But to call one of these, the test must first
call SetMockSleep (precondition enforced with assertion), which also turns
sleeps in RocksDB into mock sleeps. To also removes accounting for actual
clock time, call SetTimeElapseOnlySleepOnReopen, which implies
SetMockSleep (on DB re-open). This latter setting only works by applying
on DB re-open, otherwise havoc can ensue if Env goes back in time with
DB open.
More specifics:
Removed some unused test classes, and updated comments on the general
problem.
Fixed DBSSTTest.GetTotalSstFilesSize using a sync point callback instead
of mock time. For this we have the only modification to production code,
inserting a sync point callback in flush_job.cc, which is not a change to
production behavior.
Removed unnecessary resetting of mock times to 0 in many tests. RocksDB
deals in relative time. Any behaviors relying on absolute date/time are likely
a bug. (The above test DBSSTTest.GetTotalSstFilesSize was the only one
clearly injecting a specific absolute time for actual testing convenience.) Just
in case I misunderstood some test, I put this note in each replacement:
// NOTE: Presumed unnecessary and removed: resetting mock time in env
Strengthened some tests like MergeTestTime, MergeCompactionTimeTest, and
FilterCompactionTimeTest in db_test.cc
stats_history_test and blob_db_test are each their own beast, rather deeply
dependent on MockTimeEnv. Each gets its own variant of a work-around for
TimedWait in a mock time environment. (Reduces redundancy and
inconsistency in stats_history_test.)
Intended follow-up:
Remove TimedWait from the public API of InstrumentedCondVar, and only
make that accessible through Env by passing in an InstrumentedCondVar and
a deadline. Then the Env implementations mocking time can fix this problem
without using sync points. (Test infrastructure using sync points interferes
with individual tests' control over sync points.)
With that change, we can simplify/consolidate the scattered work-arounds.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7101
Test Plan: make check on Linux and MacOS
Reviewed By: zhichao-cao
Differential Revision: D23032815
Pulled By: pdillinger
fbshipit-source-id: 7f33967ada8b83011fb54e8279365c008bd6610b
2020-08-11 21:39:49 +02:00
|
|
|
MaybeInstallTimeElapseOnlySleep(db_opts);
|
2015-07-14 01:53:38 +02:00
|
|
|
return DB::Open(db_opts, dbname_, column_families, &handles_, &db_);
|
|
|
|
}
|
|
|
|
|
|
|
|
Status DBTestBase::TryReopenWithColumnFamilies(
|
2015-10-13 00:06:38 +02:00
|
|
|
const std::vector<std::string>& cfs, const Options& options) {
|
2015-07-14 01:53:38 +02:00
|
|
|
Close();
|
|
|
|
std::vector<Options> v_opts(cfs.size(), options);
|
|
|
|
return TryReopenWithColumnFamilies(cfs, v_opts);
|
|
|
|
}
|
|
|
|
|
|
|
|
void DBTestBase::Reopen(const Options& options) {
|
|
|
|
ASSERT_OK(TryReopen(options));
|
|
|
|
}
|
|
|
|
|
|
|
|
void DBTestBase::Close() {
|
|
|
|
for (auto h : handles_) {
|
2020-09-17 00:45:30 +02:00
|
|
|
EXPECT_OK(db_->DestroyColumnFamilyHandle(h));
|
2015-07-14 01:53:38 +02:00
|
|
|
}
|
|
|
|
handles_.clear();
|
|
|
|
delete db_;
|
|
|
|
db_ = nullptr;
|
|
|
|
}
|
|
|
|
|
|
|
|
void DBTestBase::DestroyAndReopen(const Options& options) {
|
|
|
|
// Destroy using last options
|
|
|
|
Destroy(last_options_);
|
|
|
|
ASSERT_OK(TryReopen(options));
|
|
|
|
}
|
|
|
|
|
2018-04-06 04:49:06 +02:00
|
|
|
void DBTestBase::Destroy(const Options& options, bool delete_cf_paths) {
|
|
|
|
std::vector<ColumnFamilyDescriptor> column_families;
|
|
|
|
if (delete_cf_paths) {
|
|
|
|
for (size_t i = 0; i < handles_.size(); ++i) {
|
|
|
|
ColumnFamilyDescriptor cfdescriptor;
|
2020-10-06 23:40:37 +02:00
|
|
|
// GetDescriptor is not implemented for ROCKSDB_LITE
|
|
|
|
handles_[i]->GetDescriptor(&cfdescriptor).PermitUncheckedError();
|
2018-04-06 04:49:06 +02:00
|
|
|
column_families.push_back(cfdescriptor);
|
|
|
|
}
|
|
|
|
}
|
2015-07-14 01:53:38 +02:00
|
|
|
Close();
|
2018-04-06 04:49:06 +02:00
|
|
|
ASSERT_OK(DestroyDB(dbname_, options, column_families));
|
2015-07-14 01:53:38 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
Status DBTestBase::ReadOnlyReopen(const Options& options) {
|
Fix+clean up handling of mock sleeps (#7101)
Summary:
We have a number of tests hanging on MacOS and windows due to
mishandling of code for mock sleeps. In addition, the code was in
terrible shape because the same variable (addon_time_) would sometimes
refer to microseconds and sometimes to seconds. One test even assumed it
was nanoseconds but was written to pass anyway.
This has been cleaned up so that DB tests generally use a SpecialEnv
function to mock sleep, for either some number of microseconds or seconds
depending on the function called. But to call one of these, the test must first
call SetMockSleep (precondition enforced with assertion), which also turns
sleeps in RocksDB into mock sleeps. To also removes accounting for actual
clock time, call SetTimeElapseOnlySleepOnReopen, which implies
SetMockSleep (on DB re-open). This latter setting only works by applying
on DB re-open, otherwise havoc can ensue if Env goes back in time with
DB open.
More specifics:
Removed some unused test classes, and updated comments on the general
problem.
Fixed DBSSTTest.GetTotalSstFilesSize using a sync point callback instead
of mock time. For this we have the only modification to production code,
inserting a sync point callback in flush_job.cc, which is not a change to
production behavior.
Removed unnecessary resetting of mock times to 0 in many tests. RocksDB
deals in relative time. Any behaviors relying on absolute date/time are likely
a bug. (The above test DBSSTTest.GetTotalSstFilesSize was the only one
clearly injecting a specific absolute time for actual testing convenience.) Just
in case I misunderstood some test, I put this note in each replacement:
// NOTE: Presumed unnecessary and removed: resetting mock time in env
Strengthened some tests like MergeTestTime, MergeCompactionTimeTest, and
FilterCompactionTimeTest in db_test.cc
stats_history_test and blob_db_test are each their own beast, rather deeply
dependent on MockTimeEnv. Each gets its own variant of a work-around for
TimedWait in a mock time environment. (Reduces redundancy and
inconsistency in stats_history_test.)
Intended follow-up:
Remove TimedWait from the public API of InstrumentedCondVar, and only
make that accessible through Env by passing in an InstrumentedCondVar and
a deadline. Then the Env implementations mocking time can fix this problem
without using sync points. (Test infrastructure using sync points interferes
with individual tests' control over sync points.)
With that change, we can simplify/consolidate the scattered work-arounds.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7101
Test Plan: make check on Linux and MacOS
Reviewed By: zhichao-cao
Differential Revision: D23032815
Pulled By: pdillinger
fbshipit-source-id: 7f33967ada8b83011fb54e8279365c008bd6610b
2020-08-11 21:39:49 +02:00
|
|
|
MaybeInstallTimeElapseOnlySleep(options);
|
2015-07-14 01:53:38 +02:00
|
|
|
return DB::OpenForReadOnly(options, dbname_, &db_);
|
|
|
|
}
|
|
|
|
|
|
|
|
Status DBTestBase::TryReopen(const Options& options) {
|
|
|
|
Close();
|
2017-03-06 03:01:28 +01:00
|
|
|
last_options_.table_factory.reset();
|
2018-11-09 20:17:34 +01:00
|
|
|
// Note: operator= is an unsafe approach here since it destructs
|
|
|
|
// std::shared_ptr in the same order of their creation, in contrast to
|
|
|
|
// destructors which destructs them in the opposite order of creation. One
|
Fix many tests to run with MEM_ENV and ENCRYPTED_ENV; Introduce a MemoryFileSystem class (#7566)
Summary:
This PR does a few things:
1. The MockFileSystem class was split out from the MockEnv. This change would theoretically allow a MockFileSystem to be used by other Environments as well (if we created a means of constructing one). The MockFileSystem implements a FileSystem in its entirety and does not rely on any Wrapper implementation.
2. Make the RocksDB test suite work when MOCK_ENV=1 and ENCRYPTED_ENV=1 are set. To accomplish this, a few things were needed:
- The tests that tried to use the "wrong" environment (Env::Default() instead of env_) were updated
- The MockFileSystem was changed to support the features it was missing or mishandled (such as recursively deleting files in a directory or supporting renaming of a directory).
3. Updated the test framework to have a ROCKSDB_GTEST_SKIP macro. This can be used to flag tests that are skipped. Currently, this defaults to doing nothing (marks the test as SUCCESS) but will mark the tests as SKIPPED when RocksDB is upgraded to a version of gtest that supports this (gtest-1.10).
I have run a full "make check" with MEM_ENV, ENCRYPTED_ENV, both, and neither under both MacOS and RedHat. A few tests were disabled/skipped for the MEM/ENCRYPTED cases. The error_handler_fs_test fails/hangs for MEM_ENV (presumably a timing problem) and I will introduce another PR/issue to track that problem. (I will also push a change to disable those tests soon). There is one more test in DBTest2 that also fails which I need to investigate or skip before this PR is merged.
Theoretically, this PR should also allow the test suite to run against an Env loaded from the registry, though I do not have one to try it with currently.
Finally, once this is accepted, it would be nice if there was a CircleCI job to run these tests on a checkin so this effort does not become stale. I do not know how to do that, so if someone could write that job, it would be appreciated :)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7566
Reviewed By: zhichao-cao
Differential Revision: D24408980
Pulled By: jay-zhuang
fbshipit-source-id: 911b1554a4d0da06fd51feca0c090a4abdcb4a5f
2020-10-27 18:31:34 +01:00
|
|
|
// particular problem is that the cache destructor might invoke callback
|
2018-11-09 20:17:34 +01:00
|
|
|
// functions that use Option members such as statistics. To work around this
|
Fix many tests to run with MEM_ENV and ENCRYPTED_ENV; Introduce a MemoryFileSystem class (#7566)
Summary:
This PR does a few things:
1. The MockFileSystem class was split out from the MockEnv. This change would theoretically allow a MockFileSystem to be used by other Environments as well (if we created a means of constructing one). The MockFileSystem implements a FileSystem in its entirety and does not rely on any Wrapper implementation.
2. Make the RocksDB test suite work when MOCK_ENV=1 and ENCRYPTED_ENV=1 are set. To accomplish this, a few things were needed:
- The tests that tried to use the "wrong" environment (Env::Default() instead of env_) were updated
- The MockFileSystem was changed to support the features it was missing or mishandled (such as recursively deleting files in a directory or supporting renaming of a directory).
3. Updated the test framework to have a ROCKSDB_GTEST_SKIP macro. This can be used to flag tests that are skipped. Currently, this defaults to doing nothing (marks the test as SUCCESS) but will mark the tests as SKIPPED when RocksDB is upgraded to a version of gtest that supports this (gtest-1.10).
I have run a full "make check" with MEM_ENV, ENCRYPTED_ENV, both, and neither under both MacOS and RedHat. A few tests were disabled/skipped for the MEM/ENCRYPTED cases. The error_handler_fs_test fails/hangs for MEM_ENV (presumably a timing problem) and I will introduce another PR/issue to track that problem. (I will also push a change to disable those tests soon). There is one more test in DBTest2 that also fails which I need to investigate or skip before this PR is merged.
Theoretically, this PR should also allow the test suite to run against an Env loaded from the registry, though I do not have one to try it with currently.
Finally, once this is accepted, it would be nice if there was a CircleCI job to run these tests on a checkin so this effort does not become stale. I do not know how to do that, so if someone could write that job, it would be appreciated :)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7566
Reviewed By: zhichao-cao
Differential Revision: D24408980
Pulled By: jay-zhuang
fbshipit-source-id: 911b1554a4d0da06fd51feca0c090a4abdcb4a5f
2020-10-27 18:31:34 +01:00
|
|
|
// problem, we manually call destructor of table_factory which eventually
|
2018-11-09 20:17:34 +01:00
|
|
|
// clears the block cache.
|
2015-07-14 01:53:38 +02:00
|
|
|
last_options_ = options;
|
Fix+clean up handling of mock sleeps (#7101)
Summary:
We have a number of tests hanging on MacOS and windows due to
mishandling of code for mock sleeps. In addition, the code was in
terrible shape because the same variable (addon_time_) would sometimes
refer to microseconds and sometimes to seconds. One test even assumed it
was nanoseconds but was written to pass anyway.
This has been cleaned up so that DB tests generally use a SpecialEnv
function to mock sleep, for either some number of microseconds or seconds
depending on the function called. But to call one of these, the test must first
call SetMockSleep (precondition enforced with assertion), which also turns
sleeps in RocksDB into mock sleeps. To also removes accounting for actual
clock time, call SetTimeElapseOnlySleepOnReopen, which implies
SetMockSleep (on DB re-open). This latter setting only works by applying
on DB re-open, otherwise havoc can ensue if Env goes back in time with
DB open.
More specifics:
Removed some unused test classes, and updated comments on the general
problem.
Fixed DBSSTTest.GetTotalSstFilesSize using a sync point callback instead
of mock time. For this we have the only modification to production code,
inserting a sync point callback in flush_job.cc, which is not a change to
production behavior.
Removed unnecessary resetting of mock times to 0 in many tests. RocksDB
deals in relative time. Any behaviors relying on absolute date/time are likely
a bug. (The above test DBSSTTest.GetTotalSstFilesSize was the only one
clearly injecting a specific absolute time for actual testing convenience.) Just
in case I misunderstood some test, I put this note in each replacement:
// NOTE: Presumed unnecessary and removed: resetting mock time in env
Strengthened some tests like MergeTestTime, MergeCompactionTimeTest, and
FilterCompactionTimeTest in db_test.cc
stats_history_test and blob_db_test are each their own beast, rather deeply
dependent on MockTimeEnv. Each gets its own variant of a work-around for
TimedWait in a mock time environment. (Reduces redundancy and
inconsistency in stats_history_test.)
Intended follow-up:
Remove TimedWait from the public API of InstrumentedCondVar, and only
make that accessible through Env by passing in an InstrumentedCondVar and
a deadline. Then the Env implementations mocking time can fix this problem
without using sync points. (Test infrastructure using sync points interferes
with individual tests' control over sync points.)
With that change, we can simplify/consolidate the scattered work-arounds.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7101
Test Plan: make check on Linux and MacOS
Reviewed By: zhichao-cao
Differential Revision: D23032815
Pulled By: pdillinger
fbshipit-source-id: 7f33967ada8b83011fb54e8279365c008bd6610b
2020-08-11 21:39:49 +02:00
|
|
|
MaybeInstallTimeElapseOnlySleep(options);
|
2015-07-14 01:53:38 +02:00
|
|
|
return DB::Open(options, dbname_, &db_);
|
|
|
|
}
|
|
|
|
|
2016-12-22 21:51:29 +01:00
|
|
|
bool DBTestBase::IsDirectIOSupported() {
|
2018-11-13 20:14:41 +01:00
|
|
|
return test::IsDirectIOSupported(env_, dbname_);
|
2016-12-22 21:51:29 +01:00
|
|
|
}
|
|
|
|
|
2017-06-27 01:52:06 +02:00
|
|
|
bool DBTestBase::IsMemoryMappedAccessSupported() const {
|
|
|
|
return (!encrypted_env_);
|
|
|
|
}
|
|
|
|
|
2015-07-14 01:53:38 +02:00
|
|
|
Status DBTestBase::Flush(int cf) {
|
|
|
|
if (cf == 0) {
|
|
|
|
return db_->Flush(FlushOptions());
|
|
|
|
} else {
|
|
|
|
return db_->Flush(FlushOptions(), handles_[cf]);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-10-27 00:06:44 +02:00
|
|
|
Status DBTestBase::Flush(const std::vector<int>& cf_ids) {
|
|
|
|
std::vector<ColumnFamilyHandle*> cfhs;
|
|
|
|
std::for_each(cf_ids.begin(), cf_ids.end(),
|
|
|
|
[&cfhs, this](int id) { cfhs.emplace_back(handles_[id]); });
|
|
|
|
return db_->Flush(FlushOptions(), cfhs);
|
|
|
|
}
|
|
|
|
|
2015-07-14 01:53:38 +02:00
|
|
|
Status DBTestBase::Put(const Slice& k, const Slice& v, WriteOptions wo) {
|
|
|
|
if (kMergePut == option_config_) {
|
|
|
|
return db_->Merge(wo, k, v);
|
|
|
|
} else {
|
|
|
|
return db_->Put(wo, k, v);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
Status DBTestBase::Put(int cf, const Slice& k, const Slice& v,
|
2015-10-13 00:06:38 +02:00
|
|
|
WriteOptions wo) {
|
2015-07-14 01:53:38 +02:00
|
|
|
if (kMergePut == option_config_) {
|
|
|
|
return db_->Merge(wo, handles_[cf], k, v);
|
|
|
|
} else {
|
|
|
|
return db_->Put(wo, handles_[cf], k, v);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-12-16 20:00:03 +01:00
|
|
|
Status DBTestBase::Merge(const Slice& k, const Slice& v, WriteOptions wo) {
|
|
|
|
return db_->Merge(wo, k, v);
|
|
|
|
}
|
|
|
|
|
|
|
|
Status DBTestBase::Merge(int cf, const Slice& k, const Slice& v,
|
|
|
|
WriteOptions wo) {
|
|
|
|
return db_->Merge(wo, handles_[cf], k, v);
|
|
|
|
}
|
|
|
|
|
2015-07-14 01:53:38 +02:00
|
|
|
Status DBTestBase::Delete(const std::string& k) {
|
|
|
|
return db_->Delete(WriteOptions(), k);
|
|
|
|
}
|
|
|
|
|
|
|
|
Status DBTestBase::Delete(int cf, const std::string& k) {
|
|
|
|
return db_->Delete(WriteOptions(), handles_[cf], k);
|
|
|
|
}
|
|
|
|
|
Support for SingleDelete()
Summary:
This patch fixes #7460559. It introduces SingleDelete as a new database
operation. This operation can be used to delete keys that were never
overwritten (no put following another put of the same key). If an overwritten
key is single deleted the behavior is undefined. Single deletion of a
non-existent key has no effect but multiple consecutive single deletions are
not allowed (see limitations).
In contrast to the conventional Delete() operation, the deletion entry is
removed along with the value when the two are lined up in a compaction. Note:
The semantics are similar to @igor's prototype that allowed to have this
behavior on the granularity of a column family (
https://reviews.facebook.net/D42093 ). This new patch, however, is more
aggressive when it comes to removing tombstones: It removes the SingleDelete
together with the value whenever there is no snapshot between them while the
older patch only did this when the sequence number of the deletion was older
than the earliest snapshot.
Most of the complex additions are in the Compaction Iterator, all other changes
should be relatively straightforward. The patch also includes basic support for
single deletions in db_stress and db_bench.
Limitations:
- Not compatible with cuckoo hash tables
- Single deletions cannot be used in combination with merges and normal
deletions on the same key (other keys are not affected by this)
- Consecutive single deletions are currently not allowed (and older version of
this patch supported this so it could be resurrected if needed)
Test Plan: make all check
Reviewers: yhchiang, sdong, rven, anthony, yoshinorim, igor
Reviewed By: igor
Subscribers: maykov, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D43179
2015-09-17 20:42:56 +02:00
|
|
|
Status DBTestBase::SingleDelete(const std::string& k) {
|
|
|
|
return db_->SingleDelete(WriteOptions(), k);
|
|
|
|
}
|
|
|
|
|
|
|
|
Status DBTestBase::SingleDelete(int cf, const std::string& k) {
|
|
|
|
return db_->SingleDelete(WriteOptions(), handles_[cf], k);
|
|
|
|
}
|
|
|
|
|
Added support for differential snapshots
Summary:
The motivation for this PR is to add to RocksDB support for differential (incremental) snapshots, as snapshot of the DB changes between two points in time (one can think of it as diff between to sequence numbers, or the diff D which can be thought of as an SST file or just set of KVs that can be applied to sequence number S1 to get the database to the state at sequence number S2).
This feature would be useful for various distributed storages layers built on top of RocksDB, as it should help reduce resources (time and network bandwidth) needed to recover and rebuilt DB instances as replicas in the context of distributed storages.
From the API standpoint that would like client app requesting iterator between (start seqnum) and current DB state, and reading the "diff".
This is a very draft PR for initial review in the discussion on the approach, i'm going to rework some parts and keep updating the PR.
For now, what's done here according to initial discussions:
Preserving deletes:
- We want to be able to optionally preserve recent deletes for some defined period of time, so that if a delete came in recently and might need to be included in the next incremental snapshot it would't get dropped by a compaction. This is done by adding new param to Options (preserve deletes flag) and new variable to DB Impl where we keep track of the sequence number after which we don't want to drop tombstones, even if they are otherwise eligible for deletion.
- I also added a new API call for clients to be able to advance this cutoff seqnum after which we drop deletes; i assume it's more flexible to let clients control this, since otherwise we'd need to keep some kind of timestamp < -- > seqnum mapping inside the DB, which sounds messy and painful to support. Clients could make use of it by periodically calling GetLatestSequenceNumber(), noting the timestamp, doing some calculation and figuring out by how much we need to advance the cutoff seqnum.
- Compaction codepath in compaction_iterator.cc has been modified to avoid dropping tombstones with seqnum > cutoff seqnum.
Iterator changes:
- couple params added to ReadOptions, to optionally allow client to request internal keys instead of user keys (so that client can get the latest value of a key, be it delete marker or a put), as well as min timestamp and min seqnum.
TableCache changes:
- I modified table_cache code to be able to quickly exclude SST files from iterators heep if creation_time on the file is less then iter_start_ts as passed in ReadOptions. That would help a lot in some DB settings (like reading very recent data only or using FIFO compactions), but not so much for universal compaction with more or less long iterator time span.
What's left:
- Still looking at how to best plug that inside DBIter codepath. So far it seems that FindNextUserKeyInternal only parses values as UserKeys, and iter->key() call generally returns user key. Can we add new API to DBIter as internal_key(), and modify this internal method to optionally set saved_key_ to point to the full internal key? I don't need to store actual seqnum there, but I do need to store type.
Closes https://github.com/facebook/rocksdb/pull/2999
Differential Revision: D6175602
Pulled By: mikhail-antonov
fbshipit-source-id: c779a6696ee2d574d86c69cec866a3ae095aa900
2017-11-02 02:43:29 +01:00
|
|
|
bool DBTestBase::SetPreserveDeletesSequenceNumber(SequenceNumber sn) {
|
|
|
|
return db_->SetPreserveDeletesSequenceNumber(sn);
|
|
|
|
}
|
|
|
|
|
2015-07-14 01:53:38 +02:00
|
|
|
std::string DBTestBase::Get(const std::string& k, const Snapshot* snapshot) {
|
|
|
|
ReadOptions options;
|
|
|
|
options.verify_checksums = true;
|
|
|
|
options.snapshot = snapshot;
|
|
|
|
std::string result;
|
|
|
|
Status s = db_->Get(options, k, &result);
|
|
|
|
if (s.IsNotFound()) {
|
|
|
|
result = "NOT_FOUND";
|
|
|
|
} else if (!s.ok()) {
|
|
|
|
result = s.ToString();
|
|
|
|
}
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
|
|
|
std::string DBTestBase::Get(int cf, const std::string& k,
|
2015-10-13 00:06:38 +02:00
|
|
|
const Snapshot* snapshot) {
|
2015-07-14 01:53:38 +02:00
|
|
|
ReadOptions options;
|
|
|
|
options.verify_checksums = true;
|
|
|
|
options.snapshot = snapshot;
|
|
|
|
std::string result;
|
|
|
|
Status s = db_->Get(options, handles_[cf], k, &result);
|
|
|
|
if (s.IsNotFound()) {
|
|
|
|
result = "NOT_FOUND";
|
|
|
|
} else if (!s.ok()) {
|
|
|
|
result = s.ToString();
|
|
|
|
}
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2019-01-02 20:40:12 +01:00
|
|
|
std::vector<std::string> DBTestBase::MultiGet(std::vector<int> cfs,
|
|
|
|
const std::vector<std::string>& k,
|
2019-11-12 22:51:18 +01:00
|
|
|
const Snapshot* snapshot,
|
|
|
|
const bool batched) {
|
2019-01-02 20:40:12 +01:00
|
|
|
ReadOptions options;
|
|
|
|
options.verify_checksums = true;
|
|
|
|
options.snapshot = snapshot;
|
|
|
|
std::vector<ColumnFamilyHandle*> handles;
|
|
|
|
std::vector<Slice> keys;
|
|
|
|
std::vector<std::string> result;
|
|
|
|
|
|
|
|
for (unsigned int i = 0; i < cfs.size(); ++i) {
|
|
|
|
handles.push_back(handles_[cfs[i]]);
|
|
|
|
keys.push_back(k[i]);
|
|
|
|
}
|
2019-11-12 22:51:18 +01:00
|
|
|
std::vector<Status> s;
|
|
|
|
if (!batched) {
|
|
|
|
s = db_->MultiGet(options, handles, keys, &result);
|
|
|
|
for (unsigned int i = 0; i < s.size(); ++i) {
|
|
|
|
if (s[i].IsNotFound()) {
|
|
|
|
result[i] = "NOT_FOUND";
|
|
|
|
} else if (!s[i].ok()) {
|
|
|
|
result[i] = s[i].ToString();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
std::vector<PinnableSlice> pin_values(cfs.size());
|
|
|
|
result.resize(cfs.size());
|
|
|
|
s.resize(cfs.size());
|
|
|
|
db_->MultiGet(options, cfs.size(), handles.data(), keys.data(),
|
|
|
|
pin_values.data(), s.data());
|
|
|
|
for (unsigned int i = 0; i < s.size(); ++i) {
|
|
|
|
if (s[i].IsNotFound()) {
|
|
|
|
result[i] = "NOT_FOUND";
|
|
|
|
} else if (!s[i].ok()) {
|
|
|
|
result[i] = s[i].ToString();
|
|
|
|
} else {
|
|
|
|
result[i].assign(pin_values[i].data(), pin_values[i].size());
|
|
|
|
}
|
2019-01-02 20:40:12 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
Introduce a new MultiGet batching implementation (#5011)
Summary:
This PR introduces a new MultiGet() API, with the underlying implementation grouping keys based on SST file and batching lookups in a file. The reason for the new API is twofold - the definition allows callers to allocate storage for status and values on stack instead of std::vector, as well as return values as PinnableSlices in order to avoid copying, and it keeps the original MultiGet() implementation intact while we experiment with batching.
Batching is useful when there is some spatial locality to the keys being queries, as well as larger batch sizes. The main benefits are due to -
1. Fewer function calls, especially to BlockBasedTableReader::MultiGet() and FullFilterBlockReader::KeysMayMatch()
2. Bloom filter cachelines can be prefetched, hiding the cache miss latency
The next step is to optimize the binary searches in the level_storage_info, index blocks and data blocks, since we could reduce the number of key comparisons if the keys are relatively close to each other. The batching optimizations also need to be extended to other formats, such as PlainTable and filter formats. This also needs to be added to db_stress.
Benchmark results from db_bench for various batch size/locality of reference combinations are given below. Locality was simulated by offsetting the keys in a batch by a stride length. Each SST file is about 8.6MB uncompressed and key/value size is 16/100 uncompressed. To focus on the cpu benefit of batching, the runs were single threaded and bound to the same cpu to eliminate interference from other system events. The results show a 10-25% improvement in micros/op from smaller to larger batch sizes (4 - 32).
Batch Sizes
1 | 2 | 4 | 8 | 16 | 32
Random pattern (Stride length 0)
4.158 | 4.109 | 4.026 | 4.05 | 4.1 | 4.074 - Get
4.438 | 4.302 | 4.165 | 4.122 | 4.096 | 4.075 - MultiGet (no batching)
4.461 | 4.256 | 4.277 | 4.11 | 4.182 | 4.14 - MultiGet (w/ batching)
Good locality (Stride length 16)
4.048 | 3.659 | 3.248 | 2.99 | 2.84 | 2.753
4.429 | 3.728 | 3.406 | 3.053 | 2.911 | 2.781
4.452 | 3.45 | 2.833 | 2.451 | 2.233 | 2.135
Good locality (Stride length 256)
4.066 | 3.786 | 3.581 | 3.447 | 3.415 | 3.232
4.406 | 4.005 | 3.644 | 3.49 | 3.381 | 3.268
4.393 | 3.649 | 3.186 | 2.882 | 2.676 | 2.62
Medium locality (Stride length 4096)
4.012 | 3.922 | 3.768 | 3.61 | 3.582 | 3.555
4.364 | 4.057 | 3.791 | 3.65 | 3.57 | 3.465
4.479 | 3.758 | 3.316 | 3.077 | 2.959 | 2.891
dbbench command used (on a DB with 4 levels, 12 million keys)-
TEST_TMPDIR=/dev/shm numactl -C 10 ./db_bench.tmp -use_existing_db=true -benchmarks="readseq,multireadrandom" -write_buffer_size=4194304 -target_file_size_base=4194304 -max_bytes_for_level_base=16777216 -num=12000000 -reads=12000000 -duration=90 -threads=1 -compression_type=none -cache_size=4194304000 -batch_size=32 -disable_auto_compactions=true -bloom_bits=10 -cache_index_and_filter_blocks=true -pin_l0_filter_and_index_blocks_in_cache=true -multiread_batched=true -multiread_stride=4
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5011
Differential Revision: D14348703
Pulled By: anand1976
fbshipit-source-id: 774406dab3776d979c809522a67bedac6c17f84b
2019-04-11 23:24:09 +02:00
|
|
|
std::vector<std::string> DBTestBase::MultiGet(const std::vector<std::string>& k,
|
|
|
|
const Snapshot* snapshot) {
|
|
|
|
ReadOptions options;
|
|
|
|
options.verify_checksums = true;
|
|
|
|
options.snapshot = snapshot;
|
|
|
|
std::vector<Slice> keys;
|
|
|
|
std::vector<std::string> result;
|
|
|
|
std::vector<Status> statuses(k.size());
|
|
|
|
std::vector<PinnableSlice> pin_values(k.size());
|
|
|
|
|
|
|
|
for (unsigned int i = 0; i < k.size(); ++i) {
|
|
|
|
keys.push_back(k[i]);
|
|
|
|
}
|
|
|
|
db_->MultiGet(options, dbfull()->DefaultColumnFamily(), keys.size(),
|
|
|
|
keys.data(), pin_values.data(), statuses.data());
|
|
|
|
result.resize(k.size());
|
|
|
|
for (auto iter = result.begin(); iter != result.end(); ++iter) {
|
|
|
|
iter->assign(pin_values[iter - result.begin()].data(),
|
|
|
|
pin_values[iter - result.begin()].size());
|
|
|
|
}
|
|
|
|
for (unsigned int i = 0; i < statuses.size(); ++i) {
|
|
|
|
if (statuses[i].IsNotFound()) {
|
|
|
|
result[i] = "NOT_FOUND";
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2017-07-17 23:53:15 +02:00
|
|
|
Status DBTestBase::Get(const std::string& k, PinnableSlice* v) {
|
|
|
|
ReadOptions options;
|
|
|
|
options.verify_checksums = true;
|
|
|
|
Status s = dbfull()->Get(options, dbfull()->DefaultColumnFamily(), k, v);
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
2015-07-14 01:53:38 +02:00
|
|
|
uint64_t DBTestBase::GetNumSnapshots() {
|
|
|
|
uint64_t int_num;
|
|
|
|
EXPECT_TRUE(dbfull()->GetIntProperty("rocksdb.num-snapshots", &int_num));
|
|
|
|
return int_num;
|
|
|
|
}
|
|
|
|
|
|
|
|
uint64_t DBTestBase::GetTimeOldestSnapshots() {
|
|
|
|
uint64_t int_num;
|
|
|
|
EXPECT_TRUE(
|
|
|
|
dbfull()->GetIntProperty("rocksdb.oldest-snapshot-time", &int_num));
|
|
|
|
return int_num;
|
|
|
|
}
|
|
|
|
|
2020-01-07 17:35:12 +01:00
|
|
|
uint64_t DBTestBase::GetSequenceOldestSnapshots() {
|
|
|
|
uint64_t int_num;
|
|
|
|
EXPECT_TRUE(
|
|
|
|
dbfull()->GetIntProperty("rocksdb.oldest-snapshot-sequence", &int_num));
|
|
|
|
return int_num;
|
|
|
|
}
|
|
|
|
|
2015-07-14 01:53:38 +02:00
|
|
|
// Return a string that contains all key,value pairs in order,
|
|
|
|
// formatted like "(k1->v1)(k2->v2)".
|
|
|
|
std::string DBTestBase::Contents(int cf) {
|
|
|
|
std::vector<std::string> forward;
|
|
|
|
std::string result;
|
|
|
|
Iterator* iter = (cf == 0) ? db_->NewIterator(ReadOptions())
|
|
|
|
: db_->NewIterator(ReadOptions(), handles_[cf]);
|
|
|
|
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
|
|
|
|
std::string s = IterStatus(iter);
|
|
|
|
result.push_back('(');
|
|
|
|
result.append(s);
|
|
|
|
result.push_back(')');
|
|
|
|
forward.push_back(s);
|
|
|
|
}
|
|
|
|
|
|
|
|
// Check reverse iteration results are the reverse of forward results
|
|
|
|
unsigned int matched = 0;
|
|
|
|
for (iter->SeekToLast(); iter->Valid(); iter->Prev()) {
|
|
|
|
EXPECT_LT(matched, forward.size());
|
|
|
|
EXPECT_EQ(IterStatus(iter), forward[forward.size() - matched - 1]);
|
|
|
|
matched++;
|
|
|
|
}
|
|
|
|
EXPECT_EQ(matched, forward.size());
|
|
|
|
|
|
|
|
delete iter;
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
|
|
|
std::string DBTestBase::AllEntriesFor(const Slice& user_key, int cf) {
|
|
|
|
Arena arena;
|
2016-11-04 19:53:38 +01:00
|
|
|
auto options = CurrentOptions();
|
2016-11-21 21:07:09 +01:00
|
|
|
InternalKeyComparator icmp(options.comparator);
|
2018-12-18 02:26:56 +01:00
|
|
|
ReadRangeDelAggregator range_del_agg(&icmp,
|
|
|
|
kMaxSequenceNumber /* upper_bound */);
|
2020-08-04 00:21:56 +02:00
|
|
|
ReadOptions read_options;
|
2015-07-14 01:53:38 +02:00
|
|
|
ScopedArenaIterator iter;
|
|
|
|
if (cf == 0) {
|
2020-08-04 00:21:56 +02:00
|
|
|
iter.set(dbfull()->NewInternalIterator(read_options, &arena, &range_del_agg,
|
2018-11-29 00:26:56 +01:00
|
|
|
kMaxSequenceNumber));
|
2015-07-14 01:53:38 +02:00
|
|
|
} else {
|
2020-08-04 00:21:56 +02:00
|
|
|
iter.set(dbfull()->NewInternalIterator(read_options, &arena, &range_del_agg,
|
2018-11-29 00:26:56 +01:00
|
|
|
kMaxSequenceNumber, handles_[cf]));
|
2015-07-14 01:53:38 +02:00
|
|
|
}
|
|
|
|
InternalKey target(user_key, kMaxSequenceNumber, kTypeValue);
|
|
|
|
iter->Seek(target.Encode());
|
|
|
|
std::string result;
|
|
|
|
if (!iter->status().ok()) {
|
|
|
|
result = iter->status().ToString();
|
|
|
|
} else {
|
|
|
|
result = "[ ";
|
|
|
|
bool first = true;
|
|
|
|
while (iter->Valid()) {
|
|
|
|
ParsedInternalKey ikey(Slice(), 0, kTypeValue);
|
2020-10-28 18:11:13 +01:00
|
|
|
if (ParseInternalKey(iter->key(), &ikey, true /* log_err_key */) !=
|
|
|
|
Status::OK()) {
|
2015-07-14 01:53:38 +02:00
|
|
|
result += "CORRUPTED";
|
|
|
|
} else {
|
2015-09-09 00:30:49 +02:00
|
|
|
if (!last_options_.comparator->Equal(ikey.user_key, user_key)) {
|
2015-07-14 01:53:38 +02:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
if (!first) {
|
|
|
|
result += ", ";
|
|
|
|
}
|
|
|
|
first = false;
|
|
|
|
switch (ikey.type) {
|
|
|
|
case kTypeValue:
|
|
|
|
result += iter->value().ToString();
|
|
|
|
break;
|
|
|
|
case kTypeMerge:
|
|
|
|
// keep it the same as kTypeValue for testing kMergePut
|
|
|
|
result += iter->value().ToString();
|
|
|
|
break;
|
|
|
|
case kTypeDeletion:
|
|
|
|
result += "DEL";
|
|
|
|
break;
|
Support for SingleDelete()
Summary:
This patch fixes #7460559. It introduces SingleDelete as a new database
operation. This operation can be used to delete keys that were never
overwritten (no put following another put of the same key). If an overwritten
key is single deleted the behavior is undefined. Single deletion of a
non-existent key has no effect but multiple consecutive single deletions are
not allowed (see limitations).
In contrast to the conventional Delete() operation, the deletion entry is
removed along with the value when the two are lined up in a compaction. Note:
The semantics are similar to @igor's prototype that allowed to have this
behavior on the granularity of a column family (
https://reviews.facebook.net/D42093 ). This new patch, however, is more
aggressive when it comes to removing tombstones: It removes the SingleDelete
together with the value whenever there is no snapshot between them while the
older patch only did this when the sequence number of the deletion was older
than the earliest snapshot.
Most of the complex additions are in the Compaction Iterator, all other changes
should be relatively straightforward. The patch also includes basic support for
single deletions in db_stress and db_bench.
Limitations:
- Not compatible with cuckoo hash tables
- Single deletions cannot be used in combination with merges and normal
deletions on the same key (other keys are not affected by this)
- Consecutive single deletions are currently not allowed (and older version of
this patch supported this so it could be resurrected if needed)
Test Plan: make all check
Reviewers: yhchiang, sdong, rven, anthony, yoshinorim, igor
Reviewed By: igor
Subscribers: maykov, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D43179
2015-09-17 20:42:56 +02:00
|
|
|
case kTypeSingleDeletion:
|
|
|
|
result += "SDEL";
|
|
|
|
break;
|
2015-07-14 01:53:38 +02:00
|
|
|
default:
|
|
|
|
assert(false);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
iter->Next();
|
|
|
|
}
|
|
|
|
if (!first) {
|
|
|
|
result += " ";
|
|
|
|
}
|
|
|
|
result += "]";
|
|
|
|
}
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2015-10-14 02:33:23 +02:00
|
|
|
#ifndef ROCKSDB_LITE
|
2015-07-14 01:53:38 +02:00
|
|
|
int DBTestBase::NumSortedRuns(int cf) {
|
|
|
|
ColumnFamilyMetaData cf_meta;
|
|
|
|
if (cf == 0) {
|
|
|
|
db_->GetColumnFamilyMetaData(&cf_meta);
|
|
|
|
} else {
|
|
|
|
db_->GetColumnFamilyMetaData(handles_[cf], &cf_meta);
|
|
|
|
}
|
|
|
|
int num_sr = static_cast<int>(cf_meta.levels[0].files.size());
|
|
|
|
for (size_t i = 1U; i < cf_meta.levels.size(); i++) {
|
|
|
|
if (cf_meta.levels[i].files.size() > 0) {
|
|
|
|
num_sr++;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return num_sr;
|
|
|
|
}
|
|
|
|
|
|
|
|
uint64_t DBTestBase::TotalSize(int cf) {
|
|
|
|
ColumnFamilyMetaData cf_meta;
|
|
|
|
if (cf == 0) {
|
|
|
|
db_->GetColumnFamilyMetaData(&cf_meta);
|
|
|
|
} else {
|
|
|
|
db_->GetColumnFamilyMetaData(handles_[cf], &cf_meta);
|
|
|
|
}
|
|
|
|
return cf_meta.size;
|
|
|
|
}
|
|
|
|
|
|
|
|
uint64_t DBTestBase::SizeAtLevel(int level) {
|
|
|
|
std::vector<LiveFileMetaData> metadata;
|
|
|
|
db_->GetLiveFilesMetaData(&metadata);
|
|
|
|
uint64_t sum = 0;
|
|
|
|
for (const auto& m : metadata) {
|
|
|
|
if (m.level == level) {
|
|
|
|
sum += m.size;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return sum;
|
|
|
|
}
|
|
|
|
|
2015-11-15 19:49:14 +01:00
|
|
|
size_t DBTestBase::TotalLiveFiles(int cf) {
|
2015-07-14 01:53:38 +02:00
|
|
|
ColumnFamilyMetaData cf_meta;
|
|
|
|
if (cf == 0) {
|
|
|
|
db_->GetColumnFamilyMetaData(&cf_meta);
|
|
|
|
} else {
|
|
|
|
db_->GetColumnFamilyMetaData(handles_[cf], &cf_meta);
|
|
|
|
}
|
2015-11-15 19:49:14 +01:00
|
|
|
size_t num_files = 0;
|
2015-07-14 01:53:38 +02:00
|
|
|
for (auto& level : cf_meta.levels) {
|
|
|
|
num_files += level.files.size();
|
|
|
|
}
|
|
|
|
return num_files;
|
|
|
|
}
|
|
|
|
|
2015-10-14 02:33:23 +02:00
|
|
|
size_t DBTestBase::CountLiveFiles() {
|
|
|
|
std::vector<LiveFileMetaData> metadata;
|
|
|
|
db_->GetLiveFilesMetaData(&metadata);
|
|
|
|
return metadata.size();
|
|
|
|
}
|
|
|
|
|
|
|
|
int DBTestBase::NumTableFilesAtLevel(int level, int cf) {
|
|
|
|
std::string property;
|
|
|
|
if (cf == 0) {
|
|
|
|
// default cfd
|
|
|
|
EXPECT_TRUE(db_->GetProperty(
|
|
|
|
"rocksdb.num-files-at-level" + NumberToString(level), &property));
|
|
|
|
} else {
|
|
|
|
EXPECT_TRUE(db_->GetProperty(
|
|
|
|
handles_[cf], "rocksdb.num-files-at-level" + NumberToString(level),
|
|
|
|
&property));
|
|
|
|
}
|
|
|
|
return atoi(property.c_str());
|
|
|
|
}
|
|
|
|
|
2016-04-21 03:46:54 +02:00
|
|
|
double DBTestBase::CompressionRatioAtLevel(int level, int cf) {
|
|
|
|
std::string property;
|
|
|
|
if (cf == 0) {
|
|
|
|
// default cfd
|
|
|
|
EXPECT_TRUE(db_->GetProperty(
|
|
|
|
"rocksdb.compression-ratio-at-level" + NumberToString(level),
|
|
|
|
&property));
|
|
|
|
} else {
|
|
|
|
EXPECT_TRUE(db_->GetProperty(
|
|
|
|
handles_[cf],
|
|
|
|
"rocksdb.compression-ratio-at-level" + NumberToString(level),
|
|
|
|
&property));
|
|
|
|
}
|
|
|
|
return std::stod(property);
|
|
|
|
}
|
|
|
|
|
2015-07-14 01:53:38 +02:00
|
|
|
int DBTestBase::TotalTableFiles(int cf, int levels) {
|
|
|
|
if (levels == -1) {
|
2016-08-10 21:37:43 +02:00
|
|
|
levels = (cf == 0) ? db_->NumberLevels() : db_->NumberLevels(handles_[1]);
|
2015-07-14 01:53:38 +02:00
|
|
|
}
|
|
|
|
int result = 0;
|
|
|
|
for (int level = 0; level < levels; level++) {
|
|
|
|
result += NumTableFilesAtLevel(level, cf);
|
|
|
|
}
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Return spread of files per level
|
|
|
|
std::string DBTestBase::FilesPerLevel(int cf) {
|
|
|
|
int num_levels =
|
|
|
|
(cf == 0) ? db_->NumberLevels() : db_->NumberLevels(handles_[1]);
|
|
|
|
std::string result;
|
|
|
|
size_t last_non_zero_offset = 0;
|
|
|
|
for (int level = 0; level < num_levels; level++) {
|
|
|
|
int f = NumTableFilesAtLevel(level, cf);
|
|
|
|
char buf[100];
|
|
|
|
snprintf(buf, sizeof(buf), "%s%d", (level ? "," : ""), f);
|
|
|
|
result += buf;
|
|
|
|
if (f > 0) {
|
|
|
|
last_non_zero_offset = result.size();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
result.resize(last_non_zero_offset);
|
|
|
|
return result;
|
|
|
|
}
|
2018-01-17 02:26:29 +01:00
|
|
|
#endif // !ROCKSDB_LITE
|
2015-07-14 01:53:38 +02:00
|
|
|
|
|
|
|
size_t DBTestBase::CountFiles() {
|
|
|
|
std::vector<std::string> files;
|
|
|
|
env_->GetChildren(dbname_, &files);
|
|
|
|
|
|
|
|
std::vector<std::string> logfiles;
|
|
|
|
if (dbname_ != last_options_.wal_dir) {
|
|
|
|
env_->GetChildren(last_options_.wal_dir, &logfiles);
|
|
|
|
}
|
|
|
|
|
|
|
|
return files.size() + logfiles.size();
|
|
|
|
}
|
|
|
|
|
|
|
|
uint64_t DBTestBase::Size(const Slice& start, const Slice& limit, int cf) {
|
|
|
|
Range r(start, limit);
|
|
|
|
uint64_t size;
|
|
|
|
if (cf == 0) {
|
|
|
|
db_->GetApproximateSizes(&r, 1, &size);
|
|
|
|
} else {
|
|
|
|
db_->GetApproximateSizes(handles_[1], &r, 1, &size);
|
|
|
|
}
|
|
|
|
return size;
|
|
|
|
}
|
|
|
|
|
|
|
|
void DBTestBase::Compact(int cf, const Slice& start, const Slice& limit,
|
2015-10-13 00:06:38 +02:00
|
|
|
uint32_t target_path_id) {
|
2015-07-14 01:53:38 +02:00
|
|
|
CompactRangeOptions compact_options;
|
|
|
|
compact_options.target_path_id = target_path_id;
|
|
|
|
ASSERT_OK(db_->CompactRange(compact_options, handles_[cf], &start, &limit));
|
|
|
|
}
|
|
|
|
|
|
|
|
void DBTestBase::Compact(int cf, const Slice& start, const Slice& limit) {
|
|
|
|
ASSERT_OK(
|
|
|
|
db_->CompactRange(CompactRangeOptions(), handles_[cf], &start, &limit));
|
|
|
|
}
|
|
|
|
|
|
|
|
void DBTestBase::Compact(const Slice& start, const Slice& limit) {
|
|
|
|
ASSERT_OK(db_->CompactRange(CompactRangeOptions(), &start, &limit));
|
|
|
|
}
|
|
|
|
|
|
|
|
// Do n memtable compactions, each of which produces an sstable
|
|
|
|
// covering the range [small,large].
|
2015-10-13 00:06:38 +02:00
|
|
|
void DBTestBase::MakeTables(int n, const std::string& small,
|
|
|
|
const std::string& large, int cf) {
|
2015-07-14 01:53:38 +02:00
|
|
|
for (int i = 0; i < n; i++) {
|
|
|
|
ASSERT_OK(Put(cf, small, "begin"));
|
|
|
|
ASSERT_OK(Put(cf, large, "end"));
|
|
|
|
ASSERT_OK(Flush(cf));
|
2015-07-17 21:02:52 +02:00
|
|
|
MoveFilesToLevel(n - i - 1, cf);
|
2015-07-14 01:53:38 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Prevent pushing of new sstables into deeper levels by adding
|
|
|
|
// tables that cover a specified range to all levels.
|
2015-10-13 00:06:38 +02:00
|
|
|
void DBTestBase::FillLevels(const std::string& smallest,
|
|
|
|
const std::string& largest, int cf) {
|
2015-07-14 01:53:38 +02:00
|
|
|
MakeTables(db_->NumberLevels(handles_[cf]), smallest, largest, cf);
|
|
|
|
}
|
|
|
|
|
2015-07-17 21:02:52 +02:00
|
|
|
void DBTestBase::MoveFilesToLevel(int level, int cf) {
|
|
|
|
for (int l = 0; l < level; ++l) {
|
|
|
|
if (cf > 0) {
|
2020-09-29 18:47:33 +02:00
|
|
|
EXPECT_OK(dbfull()->TEST_CompactRange(l, nullptr, nullptr, handles_[cf]));
|
2015-07-17 21:02:52 +02:00
|
|
|
} else {
|
2020-09-29 18:47:33 +02:00
|
|
|
EXPECT_OK(dbfull()->TEST_CompactRange(l, nullptr, nullptr));
|
2015-07-17 21:02:52 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-01-17 02:26:29 +01:00
|
|
|
#ifndef ROCKSDB_LITE
|
2015-07-14 01:53:38 +02:00
|
|
|
void DBTestBase::DumpFileCounts(const char* label) {
|
|
|
|
fprintf(stderr, "---\n%s:\n", label);
|
|
|
|
fprintf(stderr, "maxoverlap: %" PRIu64 "\n",
|
2015-10-13 00:06:38 +02:00
|
|
|
dbfull()->TEST_MaxNextLevelOverlappingBytes());
|
2015-07-14 01:53:38 +02:00
|
|
|
for (int level = 0; level < db_->NumberLevels(); level++) {
|
|
|
|
int num = NumTableFilesAtLevel(level);
|
|
|
|
if (num > 0) {
|
|
|
|
fprintf(stderr, " level %3d : %d files\n", level, num);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2018-01-17 02:26:29 +01:00
|
|
|
#endif // !ROCKSDB_LITE
|
2015-07-14 01:53:38 +02:00
|
|
|
|
|
|
|
std::string DBTestBase::DumpSSTableList() {
|
|
|
|
std::string property;
|
|
|
|
db_->GetProperty("rocksdb.sstables", &property);
|
|
|
|
return property;
|
|
|
|
}
|
|
|
|
|
2018-04-06 04:49:06 +02:00
|
|
|
void DBTestBase::GetSstFiles(Env* env, std::string path,
|
2015-10-07 02:46:22 +02:00
|
|
|
std::vector<std::string>* files) {
|
2020-10-02 22:33:50 +02:00
|
|
|
EXPECT_OK(env->GetChildren(path, files));
|
2015-10-07 02:46:22 +02:00
|
|
|
|
|
|
|
files->erase(
|
|
|
|
std::remove_if(files->begin(), files->end(), [](std::string name) {
|
|
|
|
uint64_t number;
|
|
|
|
FileType type;
|
|
|
|
return !(ParseFileName(name, &number, &type) && type == kTableFile);
|
|
|
|
}), files->end());
|
|
|
|
}
|
|
|
|
|
2015-07-14 01:53:38 +02:00
|
|
|
int DBTestBase::GetSstFileCount(std::string path) {
|
|
|
|
std::vector<std::string> files;
|
2018-04-06 04:49:06 +02:00
|
|
|
DBTestBase::GetSstFiles(env_, path, &files);
|
2015-10-07 02:46:22 +02:00
|
|
|
return static_cast<int>(files.size());
|
2015-07-14 01:53:38 +02:00
|
|
|
}
|
|
|
|
|
2015-08-26 23:19:31 +02:00
|
|
|
// this will generate non-overlapping files since it keeps increasing key_idx
|
|
|
|
void DBTestBase::GenerateNewFile(int cf, Random* rnd, int* key_idx,
|
|
|
|
bool nowait) {
|
2015-12-04 04:05:13 +01:00
|
|
|
for (int i = 0; i < KNumKeysByGenerateNewFile; i++) {
|
2020-07-09 23:33:42 +02:00
|
|
|
ASSERT_OK(Put(cf, Key(*key_idx), rnd->RandomString((i == 99) ? 1 : 990)));
|
2015-08-26 23:19:31 +02:00
|
|
|
(*key_idx)++;
|
|
|
|
}
|
|
|
|
if (!nowait) {
|
2020-10-02 02:45:52 +02:00
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
|
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
2015-08-26 23:19:31 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-07-14 01:53:38 +02:00
|
|
|
// this will generate non-overlapping files since it keeps increasing key_idx
|
|
|
|
void DBTestBase::GenerateNewFile(Random* rnd, int* key_idx, bool nowait) {
|
2015-12-04 04:05:13 +01:00
|
|
|
for (int i = 0; i < KNumKeysByGenerateNewFile; i++) {
|
2020-07-09 23:33:42 +02:00
|
|
|
ASSERT_OK(Put(Key(*key_idx), rnd->RandomString((i == 99) ? 1 : 990)));
|
2015-07-14 01:53:38 +02:00
|
|
|
(*key_idx)++;
|
|
|
|
}
|
|
|
|
if (!nowait) {
|
2020-10-02 02:45:52 +02:00
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
|
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
2015-07-14 01:53:38 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-11-30 20:41:53 +01:00
|
|
|
const int DBTestBase::kNumKeysByGenerateNewRandomFile = 51;
|
|
|
|
|
2015-07-14 01:53:38 +02:00
|
|
|
void DBTestBase::GenerateNewRandomFile(Random* rnd, bool nowait) {
|
2015-11-30 20:41:53 +01:00
|
|
|
for (int i = 0; i < kNumKeysByGenerateNewRandomFile; i++) {
|
2020-07-09 23:33:42 +02:00
|
|
|
ASSERT_OK(Put("key" + rnd->RandomString(7), rnd->RandomString(2000)));
|
2015-07-14 01:53:38 +02:00
|
|
|
}
|
2020-07-09 23:33:42 +02:00
|
|
|
ASSERT_OK(Put("key" + rnd->RandomString(7), rnd->RandomString(200)));
|
2015-07-14 01:53:38 +02:00
|
|
|
if (!nowait) {
|
|
|
|
dbfull()->TEST_WaitForFlushMemTable();
|
|
|
|
dbfull()->TEST_WaitForCompact();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
std::string DBTestBase::IterStatus(Iterator* iter) {
|
|
|
|
std::string result;
|
|
|
|
if (iter->Valid()) {
|
|
|
|
result = iter->key().ToString() + "->" + iter->value().ToString();
|
|
|
|
} else {
|
|
|
|
result = "(invalid)";
|
|
|
|
}
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
|
|
|
Options DBTestBase::OptionsForLogIterTest() {
|
|
|
|
Options options = CurrentOptions();
|
|
|
|
options.create_if_missing = true;
|
|
|
|
options.WAL_ttl_seconds = 1000;
|
|
|
|
return options;
|
|
|
|
}
|
|
|
|
|
|
|
|
std::string DBTestBase::DummyString(size_t len, char c) {
|
|
|
|
return std::string(len, c);
|
|
|
|
}
|
|
|
|
|
|
|
|
void DBTestBase::VerifyIterLast(std::string expected_key, int cf) {
|
|
|
|
Iterator* iter;
|
|
|
|
ReadOptions ro;
|
|
|
|
if (cf == 0) {
|
|
|
|
iter = db_->NewIterator(ro);
|
|
|
|
} else {
|
|
|
|
iter = db_->NewIterator(ro, handles_[cf]);
|
|
|
|
}
|
|
|
|
iter->SeekToLast();
|
|
|
|
ASSERT_EQ(IterStatus(iter), expected_key);
|
|
|
|
delete iter;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Used to test InplaceUpdate
|
|
|
|
|
|
|
|
// If previous value is nullptr or delta is > than previous value,
|
|
|
|
// sets newValue with delta
|
|
|
|
// If previous value is not empty,
|
|
|
|
// updates previous value with 'b' string of previous value size - 1.
|
2015-10-13 00:06:38 +02:00
|
|
|
UpdateStatus DBTestBase::updateInPlaceSmallerSize(char* prevValue,
|
|
|
|
uint32_t* prevSize,
|
|
|
|
Slice delta,
|
|
|
|
std::string* newValue) {
|
2015-07-14 01:53:38 +02:00
|
|
|
if (prevValue == nullptr) {
|
|
|
|
*newValue = std::string(delta.size(), 'c');
|
|
|
|
return UpdateStatus::UPDATED;
|
|
|
|
} else {
|
|
|
|
*prevSize = *prevSize - 1;
|
|
|
|
std::string str_b = std::string(*prevSize, 'b');
|
|
|
|
memcpy(prevValue, str_b.c_str(), str_b.size());
|
|
|
|
return UpdateStatus::UPDATED_INPLACE;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-10-13 00:06:38 +02:00
|
|
|
UpdateStatus DBTestBase::updateInPlaceSmallerVarintSize(char* prevValue,
|
|
|
|
uint32_t* prevSize,
|
|
|
|
Slice delta,
|
|
|
|
std::string* newValue) {
|
2015-07-14 01:53:38 +02:00
|
|
|
if (prevValue == nullptr) {
|
|
|
|
*newValue = std::string(delta.size(), 'c');
|
|
|
|
return UpdateStatus::UPDATED;
|
|
|
|
} else {
|
|
|
|
*prevSize = 1;
|
|
|
|
std::string str_b = std::string(*prevSize, 'b');
|
|
|
|
memcpy(prevValue, str_b.c_str(), str_b.size());
|
|
|
|
return UpdateStatus::UPDATED_INPLACE;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-03-05 22:08:17 +01:00
|
|
|
UpdateStatus DBTestBase::updateInPlaceLargerSize(char* /*prevValue*/,
|
|
|
|
uint32_t* /*prevSize*/,
|
2015-10-13 00:06:38 +02:00
|
|
|
Slice delta,
|
|
|
|
std::string* newValue) {
|
2015-07-14 01:53:38 +02:00
|
|
|
*newValue = std::string(delta.size(), 'c');
|
|
|
|
return UpdateStatus::UPDATED;
|
|
|
|
}
|
|
|
|
|
2018-03-05 22:08:17 +01:00
|
|
|
UpdateStatus DBTestBase::updateInPlaceNoAction(char* /*prevValue*/,
|
|
|
|
uint32_t* /*prevSize*/,
|
|
|
|
Slice /*delta*/,
|
|
|
|
std::string* /*newValue*/) {
|
2015-07-14 01:53:38 +02:00
|
|
|
return UpdateStatus::UPDATE_FAILED;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Utility method to test InplaceUpdate
|
|
|
|
void DBTestBase::validateNumberOfEntries(int numValues, int cf) {
|
|
|
|
Arena arena;
|
2016-11-04 19:53:38 +01:00
|
|
|
auto options = CurrentOptions();
|
2016-11-21 21:07:09 +01:00
|
|
|
InternalKeyComparator icmp(options.comparator);
|
2018-12-18 02:26:56 +01:00
|
|
|
ReadRangeDelAggregator range_del_agg(&icmp,
|
|
|
|
kMaxSequenceNumber /* upper_bound */);
|
2017-12-18 23:13:45 +01:00
|
|
|
// This should be defined after range_del_agg so that it destructs the
|
|
|
|
// assigned iterator before it range_del_agg is already destructed.
|
2020-08-04 00:21:56 +02:00
|
|
|
ReadOptions read_options;
|
2017-12-18 23:13:45 +01:00
|
|
|
ScopedArenaIterator iter;
|
2015-07-14 01:53:38 +02:00
|
|
|
if (cf != 0) {
|
2020-08-04 00:21:56 +02:00
|
|
|
iter.set(dbfull()->NewInternalIterator(read_options, &arena, &range_del_agg,
|
2018-11-29 00:26:56 +01:00
|
|
|
kMaxSequenceNumber, handles_[cf]));
|
2015-07-14 01:53:38 +02:00
|
|
|
} else {
|
2020-08-04 00:21:56 +02:00
|
|
|
iter.set(dbfull()->NewInternalIterator(read_options, &arena, &range_del_agg,
|
2018-11-29 00:26:56 +01:00
|
|
|
kMaxSequenceNumber));
|
2015-07-14 01:53:38 +02:00
|
|
|
}
|
|
|
|
iter->SeekToFirst();
|
2020-10-28 18:11:13 +01:00
|
|
|
ASSERT_OK(iter->status());
|
2015-07-14 01:53:38 +02:00
|
|
|
int seq = numValues;
|
|
|
|
while (iter->Valid()) {
|
|
|
|
ParsedInternalKey ikey;
|
2017-10-19 19:48:47 +02:00
|
|
|
ikey.clear();
|
2020-10-28 18:11:13 +01:00
|
|
|
ASSERT_OK(ParseInternalKey(iter->key(), &ikey, true /* log_err_key */));
|
2015-07-14 01:53:38 +02:00
|
|
|
|
|
|
|
// checks sequence number for updates
|
|
|
|
ASSERT_EQ(ikey.sequence, (unsigned)seq--);
|
|
|
|
iter->Next();
|
|
|
|
}
|
|
|
|
ASSERT_EQ(0, seq);
|
|
|
|
}
|
|
|
|
|
2015-10-13 00:06:38 +02:00
|
|
|
void DBTestBase::CopyFile(const std::string& source,
|
|
|
|
const std::string& destination, uint64_t size) {
|
2015-07-14 01:53:38 +02:00
|
|
|
const EnvOptions soptions;
|
2018-11-09 20:17:34 +01:00
|
|
|
std::unique_ptr<SequentialFile> srcfile;
|
2015-07-14 01:53:38 +02:00
|
|
|
ASSERT_OK(env_->NewSequentialFile(source, &srcfile, soptions));
|
2018-11-09 20:17:34 +01:00
|
|
|
std::unique_ptr<WritableFile> destfile;
|
2015-07-14 01:53:38 +02:00
|
|
|
ASSERT_OK(env_->NewWritableFile(destination, &destfile, soptions));
|
|
|
|
|
|
|
|
if (size == 0) {
|
|
|
|
// default argument means copy everything
|
|
|
|
ASSERT_OK(env_->GetFileSize(source, &size));
|
|
|
|
}
|
|
|
|
|
|
|
|
char buffer[4096];
|
|
|
|
Slice slice;
|
|
|
|
while (size > 0) {
|
|
|
|
uint64_t one = std::min(uint64_t(sizeof(buffer)), size);
|
|
|
|
ASSERT_OK(srcfile->Read(one, &slice, buffer));
|
|
|
|
ASSERT_OK(destfile->Append(slice));
|
|
|
|
size -= slice.size();
|
|
|
|
}
|
|
|
|
ASSERT_OK(destfile->Close());
|
|
|
|
}
|
|
|
|
|
2020-09-30 01:28:42 +02:00
|
|
|
Status DBTestBase::GetAllSSTFiles(
|
|
|
|
std::unordered_map<std::string, uint64_t>* sst_files,
|
|
|
|
uint64_t* total_size /* = nullptr */) {
|
2016-02-18 00:20:23 +01:00
|
|
|
if (total_size) {
|
|
|
|
*total_size = 0;
|
|
|
|
}
|
2016-01-29 03:35:01 +01:00
|
|
|
std::vector<std::string> files;
|
2020-09-30 01:28:42 +02:00
|
|
|
Status s = env_->GetChildren(dbname_, &files);
|
|
|
|
if (s.ok()) {
|
|
|
|
for (auto& file_name : files) {
|
|
|
|
uint64_t number;
|
|
|
|
FileType type;
|
|
|
|
if (ParseFileName(file_name, &number, &type) && type == kTableFile) {
|
|
|
|
std::string file_path = dbname_ + "/" + file_name;
|
|
|
|
uint64_t file_size = 0;
|
|
|
|
s = env_->GetFileSize(file_path, &file_size);
|
|
|
|
if (!s.ok()) {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
(*sst_files)[file_path] = file_size;
|
|
|
|
if (total_size) {
|
|
|
|
*total_size += file_size;
|
|
|
|
}
|
2016-02-18 00:20:23 +01:00
|
|
|
}
|
2016-01-29 03:35:01 +01:00
|
|
|
}
|
|
|
|
}
|
2020-09-30 01:28:42 +02:00
|
|
|
return s;
|
2016-01-29 03:35:01 +01:00
|
|
|
}
|
|
|
|
|
2016-04-18 18:42:50 +02:00
|
|
|
std::vector<std::uint64_t> DBTestBase::ListTableFiles(Env* env,
|
|
|
|
const std::string& path) {
|
|
|
|
std::vector<std::string> files;
|
|
|
|
std::vector<uint64_t> file_numbers;
|
2020-09-29 18:47:33 +02:00
|
|
|
EXPECT_OK(env->GetChildren(path, &files));
|
2016-04-18 18:42:50 +02:00
|
|
|
uint64_t number;
|
|
|
|
FileType type;
|
|
|
|
for (size_t i = 0; i < files.size(); ++i) {
|
|
|
|
if (ParseFileName(files[i], &number, &type)) {
|
|
|
|
if (type == kTableFile) {
|
|
|
|
file_numbers.push_back(number);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return file_numbers;
|
|
|
|
}
|
|
|
|
|
2016-08-12 04:10:16 +02:00
|
|
|
void DBTestBase::VerifyDBFromMap(std::map<std::string, std::string> true_data,
|
2016-12-16 20:00:03 +01:00
|
|
|
size_t* total_reads_res, bool tailing_iter,
|
|
|
|
std::map<std::string, Status> status) {
|
2016-08-12 04:10:16 +02:00
|
|
|
size_t total_reads = 0;
|
|
|
|
|
Introduce FullMergeV2 (eliminate memcpy from merge operators)
Summary:
This diff update the code to pin the merge operator operands while the merge operation is done, so that we can eliminate the memcpy cost, to do that we need a new public API for FullMerge that replace the std::deque<std::string> with std::vector<Slice>
This diff is stacked on top of D56493 and D56511
In this diff we
- Update FullMergeV2 arguments to be encapsulated in MergeOperationInput and MergeOperationOutput which will make it easier to add new arguments in the future
- Replace std::deque<std::string> with std::vector<Slice> to pass operands
- Replace MergeContext std::deque with std::vector (based on a simple benchmark I ran https://gist.github.com/IslamAbdelRahman/78fc86c9ab9f52b1df791e58943fb187)
- Allow FullMergeV2 output to be an existing operand
```
[Everything in Memtable | 10K operands | 10 KB each | 1 operand per key]
DEBUG_LEVEL=0 make db_bench -j64 && ./db_bench --benchmarks="mergerandom,readseq,readseq,readseq,readseq,readseq" --merge_operator="max" --merge_keys=10000 --num=10000 --disable_auto_compactions --value_size=10240 --write_buffer_size=1000000000
[FullMergeV2]
readseq : 0.607 micros/op 1648235 ops/sec; 16121.2 MB/s
readseq : 0.478 micros/op 2091546 ops/sec; 20457.2 MB/s
readseq : 0.252 micros/op 3972081 ops/sec; 38850.5 MB/s
readseq : 0.237 micros/op 4218328 ops/sec; 41259.0 MB/s
readseq : 0.247 micros/op 4043927 ops/sec; 39553.2 MB/s
[master]
readseq : 3.935 micros/op 254140 ops/sec; 2485.7 MB/s
readseq : 3.722 micros/op 268657 ops/sec; 2627.7 MB/s
readseq : 3.149 micros/op 317605 ops/sec; 3106.5 MB/s
readseq : 3.125 micros/op 320024 ops/sec; 3130.1 MB/s
readseq : 4.075 micros/op 245374 ops/sec; 2400.0 MB/s
```
```
[Everything in Memtable | 10K operands | 10 KB each | 10 operand per key]
DEBUG_LEVEL=0 make db_bench -j64 && ./db_bench --benchmarks="mergerandom,readseq,readseq,readseq,readseq,readseq" --merge_operator="max" --merge_keys=1000 --num=10000 --disable_auto_compactions --value_size=10240 --write_buffer_size=1000000000
[FullMergeV2]
readseq : 3.472 micros/op 288018 ops/sec; 2817.1 MB/s
readseq : 2.304 micros/op 434027 ops/sec; 4245.2 MB/s
readseq : 1.163 micros/op 859845 ops/sec; 8410.0 MB/s
readseq : 1.192 micros/op 838926 ops/sec; 8205.4 MB/s
readseq : 1.250 micros/op 800000 ops/sec; 7824.7 MB/s
[master]
readseq : 24.025 micros/op 41623 ops/sec; 407.1 MB/s
readseq : 18.489 micros/op 54086 ops/sec; 529.0 MB/s
readseq : 18.693 micros/op 53495 ops/sec; 523.2 MB/s
readseq : 23.621 micros/op 42335 ops/sec; 414.1 MB/s
readseq : 18.775 micros/op 53262 ops/sec; 521.0 MB/s
```
```
[Everything in Block cache | 10K operands | 10 KB each | 1 operand per key]
[FullMergeV2]
$ DEBUG_LEVEL=0 make db_bench -j64 && ./db_bench --benchmarks="readseq,readseq,readseq,readseq,readseq" --merge_operator="max" --num=100000 --db="/dev/shm/merge-random-10K-10KB" --cache_size=1000000000 --use_existing_db --disable_auto_compactions
readseq : 14.741 micros/op 67837 ops/sec; 663.5 MB/s
readseq : 1.029 micros/op 971446 ops/sec; 9501.6 MB/s
readseq : 0.974 micros/op 1026229 ops/sec; 10037.4 MB/s
readseq : 0.965 micros/op 1036080 ops/sec; 10133.8 MB/s
readseq : 0.943 micros/op 1060657 ops/sec; 10374.2 MB/s
[master]
readseq : 16.735 micros/op 59755 ops/sec; 584.5 MB/s
readseq : 3.029 micros/op 330151 ops/sec; 3229.2 MB/s
readseq : 3.136 micros/op 318883 ops/sec; 3119.0 MB/s
readseq : 3.065 micros/op 326245 ops/sec; 3191.0 MB/s
readseq : 3.014 micros/op 331813 ops/sec; 3245.4 MB/s
```
```
[Everything in Block cache | 10K operands | 10 KB each | 10 operand per key]
DEBUG_LEVEL=0 make db_bench -j64 && ./db_bench --benchmarks="readseq,readseq,readseq,readseq,readseq" --merge_operator="max" --num=100000 --db="/dev/shm/merge-random-10-operands-10K-10KB" --cache_size=1000000000 --use_existing_db --disable_auto_compactions
[FullMergeV2]
readseq : 24.325 micros/op 41109 ops/sec; 402.1 MB/s
readseq : 1.470 micros/op 680272 ops/sec; 6653.7 MB/s
readseq : 1.231 micros/op 812347 ops/sec; 7945.5 MB/s
readseq : 1.091 micros/op 916590 ops/sec; 8965.1 MB/s
readseq : 1.109 micros/op 901713 ops/sec; 8819.6 MB/s
[master]
readseq : 27.257 micros/op 36687 ops/sec; 358.8 MB/s
readseq : 4.443 micros/op 225073 ops/sec; 2201.4 MB/s
readseq : 5.830 micros/op 171526 ops/sec; 1677.7 MB/s
readseq : 4.173 micros/op 239635 ops/sec; 2343.8 MB/s
readseq : 4.150 micros/op 240963 ops/sec; 2356.8 MB/s
```
Test Plan: COMPILE_WITH_ASAN=1 make check -j64
Reviewers: yhchiang, andrewkr, sdong
Reviewed By: sdong
Subscribers: lovro, andrewkr, dhruba
Differential Revision: https://reviews.facebook.net/D57075
2016-07-20 18:49:03 +02:00
|
|
|
for (auto& kv : true_data) {
|
2016-12-16 20:00:03 +01:00
|
|
|
Status s = status[kv.first];
|
|
|
|
if (s.ok()) {
|
|
|
|
ASSERT_EQ(Get(kv.first), kv.second);
|
|
|
|
} else {
|
|
|
|
std::string value;
|
|
|
|
ASSERT_EQ(s, db_->Get(ReadOptions(), kv.first, &value));
|
|
|
|
}
|
2016-08-12 04:10:16 +02:00
|
|
|
total_reads++;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Normal Iterator
|
|
|
|
{
|
|
|
|
int iter_cnt = 0;
|
|
|
|
ReadOptions ro;
|
|
|
|
ro.total_order_seek = true;
|
|
|
|
Iterator* iter = db_->NewIterator(ro);
|
|
|
|
// Verify Iterator::Next()
|
|
|
|
iter_cnt = 0;
|
|
|
|
auto data_iter = true_data.begin();
|
2016-12-16 20:00:03 +01:00
|
|
|
Status s;
|
2016-08-12 04:10:16 +02:00
|
|
|
for (iter->SeekToFirst(); iter->Valid(); iter->Next(), data_iter++) {
|
|
|
|
ASSERT_EQ(iter->key().ToString(), data_iter->first);
|
2016-12-16 20:00:03 +01:00
|
|
|
Status current_status = status[data_iter->first];
|
|
|
|
if (!current_status.ok()) {
|
|
|
|
s = current_status;
|
|
|
|
}
|
|
|
|
ASSERT_EQ(iter->status(), s);
|
|
|
|
if (current_status.ok()) {
|
|
|
|
ASSERT_EQ(iter->value().ToString(), data_iter->second);
|
|
|
|
}
|
2016-08-12 04:10:16 +02:00
|
|
|
iter_cnt++;
|
|
|
|
total_reads++;
|
|
|
|
}
|
|
|
|
ASSERT_EQ(data_iter, true_data.end()) << iter_cnt << " / "
|
|
|
|
<< true_data.size();
|
2016-12-16 20:00:03 +01:00
|
|
|
delete iter;
|
2016-08-12 04:10:16 +02:00
|
|
|
|
|
|
|
// Verify Iterator::Prev()
|
2016-12-16 20:00:03 +01:00
|
|
|
// Use a new iterator to make sure its status is clean.
|
|
|
|
iter = db_->NewIterator(ro);
|
2016-08-12 04:10:16 +02:00
|
|
|
iter_cnt = 0;
|
2016-12-16 20:00:03 +01:00
|
|
|
s = Status::OK();
|
2016-08-12 04:10:16 +02:00
|
|
|
auto data_rev = true_data.rbegin();
|
|
|
|
for (iter->SeekToLast(); iter->Valid(); iter->Prev(), data_rev++) {
|
|
|
|
ASSERT_EQ(iter->key().ToString(), data_rev->first);
|
2016-12-16 20:00:03 +01:00
|
|
|
Status current_status = status[data_rev->first];
|
|
|
|
if (!current_status.ok()) {
|
|
|
|
s = current_status;
|
|
|
|
}
|
|
|
|
ASSERT_EQ(iter->status(), s);
|
|
|
|
if (current_status.ok()) {
|
|
|
|
ASSERT_EQ(iter->value().ToString(), data_rev->second);
|
|
|
|
}
|
2016-08-12 04:10:16 +02:00
|
|
|
iter_cnt++;
|
|
|
|
total_reads++;
|
|
|
|
}
|
|
|
|
ASSERT_EQ(data_rev, true_data.rend()) << iter_cnt << " / "
|
|
|
|
<< true_data.size();
|
|
|
|
|
|
|
|
// Verify Iterator::Seek()
|
|
|
|
for (auto kv : true_data) {
|
|
|
|
iter->Seek(kv.first);
|
|
|
|
ASSERT_EQ(kv.first, iter->key().ToString());
|
|
|
|
ASSERT_EQ(kv.second, iter->value().ToString());
|
|
|
|
total_reads++;
|
|
|
|
}
|
|
|
|
delete iter;
|
Introduce FullMergeV2 (eliminate memcpy from merge operators)
Summary:
This diff update the code to pin the merge operator operands while the merge operation is done, so that we can eliminate the memcpy cost, to do that we need a new public API for FullMerge that replace the std::deque<std::string> with std::vector<Slice>
This diff is stacked on top of D56493 and D56511
In this diff we
- Update FullMergeV2 arguments to be encapsulated in MergeOperationInput and MergeOperationOutput which will make it easier to add new arguments in the future
- Replace std::deque<std::string> with std::vector<Slice> to pass operands
- Replace MergeContext std::deque with std::vector (based on a simple benchmark I ran https://gist.github.com/IslamAbdelRahman/78fc86c9ab9f52b1df791e58943fb187)
- Allow FullMergeV2 output to be an existing operand
```
[Everything in Memtable | 10K operands | 10 KB each | 1 operand per key]
DEBUG_LEVEL=0 make db_bench -j64 && ./db_bench --benchmarks="mergerandom,readseq,readseq,readseq,readseq,readseq" --merge_operator="max" --merge_keys=10000 --num=10000 --disable_auto_compactions --value_size=10240 --write_buffer_size=1000000000
[FullMergeV2]
readseq : 0.607 micros/op 1648235 ops/sec; 16121.2 MB/s
readseq : 0.478 micros/op 2091546 ops/sec; 20457.2 MB/s
readseq : 0.252 micros/op 3972081 ops/sec; 38850.5 MB/s
readseq : 0.237 micros/op 4218328 ops/sec; 41259.0 MB/s
readseq : 0.247 micros/op 4043927 ops/sec; 39553.2 MB/s
[master]
readseq : 3.935 micros/op 254140 ops/sec; 2485.7 MB/s
readseq : 3.722 micros/op 268657 ops/sec; 2627.7 MB/s
readseq : 3.149 micros/op 317605 ops/sec; 3106.5 MB/s
readseq : 3.125 micros/op 320024 ops/sec; 3130.1 MB/s
readseq : 4.075 micros/op 245374 ops/sec; 2400.0 MB/s
```
```
[Everything in Memtable | 10K operands | 10 KB each | 10 operand per key]
DEBUG_LEVEL=0 make db_bench -j64 && ./db_bench --benchmarks="mergerandom,readseq,readseq,readseq,readseq,readseq" --merge_operator="max" --merge_keys=1000 --num=10000 --disable_auto_compactions --value_size=10240 --write_buffer_size=1000000000
[FullMergeV2]
readseq : 3.472 micros/op 288018 ops/sec; 2817.1 MB/s
readseq : 2.304 micros/op 434027 ops/sec; 4245.2 MB/s
readseq : 1.163 micros/op 859845 ops/sec; 8410.0 MB/s
readseq : 1.192 micros/op 838926 ops/sec; 8205.4 MB/s
readseq : 1.250 micros/op 800000 ops/sec; 7824.7 MB/s
[master]
readseq : 24.025 micros/op 41623 ops/sec; 407.1 MB/s
readseq : 18.489 micros/op 54086 ops/sec; 529.0 MB/s
readseq : 18.693 micros/op 53495 ops/sec; 523.2 MB/s
readseq : 23.621 micros/op 42335 ops/sec; 414.1 MB/s
readseq : 18.775 micros/op 53262 ops/sec; 521.0 MB/s
```
```
[Everything in Block cache | 10K operands | 10 KB each | 1 operand per key]
[FullMergeV2]
$ DEBUG_LEVEL=0 make db_bench -j64 && ./db_bench --benchmarks="readseq,readseq,readseq,readseq,readseq" --merge_operator="max" --num=100000 --db="/dev/shm/merge-random-10K-10KB" --cache_size=1000000000 --use_existing_db --disable_auto_compactions
readseq : 14.741 micros/op 67837 ops/sec; 663.5 MB/s
readseq : 1.029 micros/op 971446 ops/sec; 9501.6 MB/s
readseq : 0.974 micros/op 1026229 ops/sec; 10037.4 MB/s
readseq : 0.965 micros/op 1036080 ops/sec; 10133.8 MB/s
readseq : 0.943 micros/op 1060657 ops/sec; 10374.2 MB/s
[master]
readseq : 16.735 micros/op 59755 ops/sec; 584.5 MB/s
readseq : 3.029 micros/op 330151 ops/sec; 3229.2 MB/s
readseq : 3.136 micros/op 318883 ops/sec; 3119.0 MB/s
readseq : 3.065 micros/op 326245 ops/sec; 3191.0 MB/s
readseq : 3.014 micros/op 331813 ops/sec; 3245.4 MB/s
```
```
[Everything in Block cache | 10K operands | 10 KB each | 10 operand per key]
DEBUG_LEVEL=0 make db_bench -j64 && ./db_bench --benchmarks="readseq,readseq,readseq,readseq,readseq" --merge_operator="max" --num=100000 --db="/dev/shm/merge-random-10-operands-10K-10KB" --cache_size=1000000000 --use_existing_db --disable_auto_compactions
[FullMergeV2]
readseq : 24.325 micros/op 41109 ops/sec; 402.1 MB/s
readseq : 1.470 micros/op 680272 ops/sec; 6653.7 MB/s
readseq : 1.231 micros/op 812347 ops/sec; 7945.5 MB/s
readseq : 1.091 micros/op 916590 ops/sec; 8965.1 MB/s
readseq : 1.109 micros/op 901713 ops/sec; 8819.6 MB/s
[master]
readseq : 27.257 micros/op 36687 ops/sec; 358.8 MB/s
readseq : 4.443 micros/op 225073 ops/sec; 2201.4 MB/s
readseq : 5.830 micros/op 171526 ops/sec; 1677.7 MB/s
readseq : 4.173 micros/op 239635 ops/sec; 2343.8 MB/s
readseq : 4.150 micros/op 240963 ops/sec; 2356.8 MB/s
```
Test Plan: COMPILE_WITH_ASAN=1 make check -j64
Reviewers: yhchiang, andrewkr, sdong
Reviewed By: sdong
Subscribers: lovro, andrewkr, dhruba
Differential Revision: https://reviews.facebook.net/D57075
2016-07-20 18:49:03 +02:00
|
|
|
}
|
|
|
|
|
2016-10-21 02:05:32 +02:00
|
|
|
if (tailing_iter) {
|
2016-08-12 04:10:16 +02:00
|
|
|
#ifndef ROCKSDB_LITE
|
2016-10-21 02:05:32 +02:00
|
|
|
// Tailing iterator
|
|
|
|
int iter_cnt = 0;
|
|
|
|
ReadOptions ro;
|
|
|
|
ro.tailing = true;
|
|
|
|
ro.total_order_seek = true;
|
|
|
|
Iterator* iter = db_->NewIterator(ro);
|
Introduce FullMergeV2 (eliminate memcpy from merge operators)
Summary:
This diff update the code to pin the merge operator operands while the merge operation is done, so that we can eliminate the memcpy cost, to do that we need a new public API for FullMerge that replace the std::deque<std::string> with std::vector<Slice>
This diff is stacked on top of D56493 and D56511
In this diff we
- Update FullMergeV2 arguments to be encapsulated in MergeOperationInput and MergeOperationOutput which will make it easier to add new arguments in the future
- Replace std::deque<std::string> with std::vector<Slice> to pass operands
- Replace MergeContext std::deque with std::vector (based on a simple benchmark I ran https://gist.github.com/IslamAbdelRahman/78fc86c9ab9f52b1df791e58943fb187)
- Allow FullMergeV2 output to be an existing operand
```
[Everything in Memtable | 10K operands | 10 KB each | 1 operand per key]
DEBUG_LEVEL=0 make db_bench -j64 && ./db_bench --benchmarks="mergerandom,readseq,readseq,readseq,readseq,readseq" --merge_operator="max" --merge_keys=10000 --num=10000 --disable_auto_compactions --value_size=10240 --write_buffer_size=1000000000
[FullMergeV2]
readseq : 0.607 micros/op 1648235 ops/sec; 16121.2 MB/s
readseq : 0.478 micros/op 2091546 ops/sec; 20457.2 MB/s
readseq : 0.252 micros/op 3972081 ops/sec; 38850.5 MB/s
readseq : 0.237 micros/op 4218328 ops/sec; 41259.0 MB/s
readseq : 0.247 micros/op 4043927 ops/sec; 39553.2 MB/s
[master]
readseq : 3.935 micros/op 254140 ops/sec; 2485.7 MB/s
readseq : 3.722 micros/op 268657 ops/sec; 2627.7 MB/s
readseq : 3.149 micros/op 317605 ops/sec; 3106.5 MB/s
readseq : 3.125 micros/op 320024 ops/sec; 3130.1 MB/s
readseq : 4.075 micros/op 245374 ops/sec; 2400.0 MB/s
```
```
[Everything in Memtable | 10K operands | 10 KB each | 10 operand per key]
DEBUG_LEVEL=0 make db_bench -j64 && ./db_bench --benchmarks="mergerandom,readseq,readseq,readseq,readseq,readseq" --merge_operator="max" --merge_keys=1000 --num=10000 --disable_auto_compactions --value_size=10240 --write_buffer_size=1000000000
[FullMergeV2]
readseq : 3.472 micros/op 288018 ops/sec; 2817.1 MB/s
readseq : 2.304 micros/op 434027 ops/sec; 4245.2 MB/s
readseq : 1.163 micros/op 859845 ops/sec; 8410.0 MB/s
readseq : 1.192 micros/op 838926 ops/sec; 8205.4 MB/s
readseq : 1.250 micros/op 800000 ops/sec; 7824.7 MB/s
[master]
readseq : 24.025 micros/op 41623 ops/sec; 407.1 MB/s
readseq : 18.489 micros/op 54086 ops/sec; 529.0 MB/s
readseq : 18.693 micros/op 53495 ops/sec; 523.2 MB/s
readseq : 23.621 micros/op 42335 ops/sec; 414.1 MB/s
readseq : 18.775 micros/op 53262 ops/sec; 521.0 MB/s
```
```
[Everything in Block cache | 10K operands | 10 KB each | 1 operand per key]
[FullMergeV2]
$ DEBUG_LEVEL=0 make db_bench -j64 && ./db_bench --benchmarks="readseq,readseq,readseq,readseq,readseq" --merge_operator="max" --num=100000 --db="/dev/shm/merge-random-10K-10KB" --cache_size=1000000000 --use_existing_db --disable_auto_compactions
readseq : 14.741 micros/op 67837 ops/sec; 663.5 MB/s
readseq : 1.029 micros/op 971446 ops/sec; 9501.6 MB/s
readseq : 0.974 micros/op 1026229 ops/sec; 10037.4 MB/s
readseq : 0.965 micros/op 1036080 ops/sec; 10133.8 MB/s
readseq : 0.943 micros/op 1060657 ops/sec; 10374.2 MB/s
[master]
readseq : 16.735 micros/op 59755 ops/sec; 584.5 MB/s
readseq : 3.029 micros/op 330151 ops/sec; 3229.2 MB/s
readseq : 3.136 micros/op 318883 ops/sec; 3119.0 MB/s
readseq : 3.065 micros/op 326245 ops/sec; 3191.0 MB/s
readseq : 3.014 micros/op 331813 ops/sec; 3245.4 MB/s
```
```
[Everything in Block cache | 10K operands | 10 KB each | 10 operand per key]
DEBUG_LEVEL=0 make db_bench -j64 && ./db_bench --benchmarks="readseq,readseq,readseq,readseq,readseq" --merge_operator="max" --num=100000 --db="/dev/shm/merge-random-10-operands-10K-10KB" --cache_size=1000000000 --use_existing_db --disable_auto_compactions
[FullMergeV2]
readseq : 24.325 micros/op 41109 ops/sec; 402.1 MB/s
readseq : 1.470 micros/op 680272 ops/sec; 6653.7 MB/s
readseq : 1.231 micros/op 812347 ops/sec; 7945.5 MB/s
readseq : 1.091 micros/op 916590 ops/sec; 8965.1 MB/s
readseq : 1.109 micros/op 901713 ops/sec; 8819.6 MB/s
[master]
readseq : 27.257 micros/op 36687 ops/sec; 358.8 MB/s
readseq : 4.443 micros/op 225073 ops/sec; 2201.4 MB/s
readseq : 5.830 micros/op 171526 ops/sec; 1677.7 MB/s
readseq : 4.173 micros/op 239635 ops/sec; 2343.8 MB/s
readseq : 4.150 micros/op 240963 ops/sec; 2356.8 MB/s
```
Test Plan: COMPILE_WITH_ASAN=1 make check -j64
Reviewers: yhchiang, andrewkr, sdong
Reviewed By: sdong
Subscribers: lovro, andrewkr, dhruba
Differential Revision: https://reviews.facebook.net/D57075
2016-07-20 18:49:03 +02:00
|
|
|
|
2016-10-21 02:05:32 +02:00
|
|
|
// Verify ForwardIterator::Next()
|
|
|
|
iter_cnt = 0;
|
|
|
|
auto data_iter = true_data.begin();
|
|
|
|
for (iter->SeekToFirst(); iter->Valid(); iter->Next(), data_iter++) {
|
|
|
|
ASSERT_EQ(iter->key().ToString(), data_iter->first);
|
|
|
|
ASSERT_EQ(iter->value().ToString(), data_iter->second);
|
|
|
|
iter_cnt++;
|
|
|
|
total_reads++;
|
|
|
|
}
|
|
|
|
ASSERT_EQ(data_iter, true_data.end()) << iter_cnt << " / "
|
|
|
|
<< true_data.size();
|
|
|
|
|
|
|
|
// Verify ForwardIterator::Seek()
|
|
|
|
for (auto kv : true_data) {
|
|
|
|
iter->Seek(kv.first);
|
|
|
|
ASSERT_EQ(kv.first, iter->key().ToString());
|
|
|
|
ASSERT_EQ(kv.second, iter->value().ToString());
|
|
|
|
total_reads++;
|
|
|
|
}
|
|
|
|
|
|
|
|
delete iter;
|
2016-08-12 04:10:16 +02:00
|
|
|
#endif // ROCKSDB_LITE
|
2016-10-21 02:05:32 +02:00
|
|
|
}
|
2016-08-12 04:10:16 +02:00
|
|
|
|
|
|
|
if (total_reads_res) {
|
|
|
|
*total_reads_res = total_reads;
|
|
|
|
}
|
Introduce FullMergeV2 (eliminate memcpy from merge operators)
Summary:
This diff update the code to pin the merge operator operands while the merge operation is done, so that we can eliminate the memcpy cost, to do that we need a new public API for FullMerge that replace the std::deque<std::string> with std::vector<Slice>
This diff is stacked on top of D56493 and D56511
In this diff we
- Update FullMergeV2 arguments to be encapsulated in MergeOperationInput and MergeOperationOutput which will make it easier to add new arguments in the future
- Replace std::deque<std::string> with std::vector<Slice> to pass operands
- Replace MergeContext std::deque with std::vector (based on a simple benchmark I ran https://gist.github.com/IslamAbdelRahman/78fc86c9ab9f52b1df791e58943fb187)
- Allow FullMergeV2 output to be an existing operand
```
[Everything in Memtable | 10K operands | 10 KB each | 1 operand per key]
DEBUG_LEVEL=0 make db_bench -j64 && ./db_bench --benchmarks="mergerandom,readseq,readseq,readseq,readseq,readseq" --merge_operator="max" --merge_keys=10000 --num=10000 --disable_auto_compactions --value_size=10240 --write_buffer_size=1000000000
[FullMergeV2]
readseq : 0.607 micros/op 1648235 ops/sec; 16121.2 MB/s
readseq : 0.478 micros/op 2091546 ops/sec; 20457.2 MB/s
readseq : 0.252 micros/op 3972081 ops/sec; 38850.5 MB/s
readseq : 0.237 micros/op 4218328 ops/sec; 41259.0 MB/s
readseq : 0.247 micros/op 4043927 ops/sec; 39553.2 MB/s
[master]
readseq : 3.935 micros/op 254140 ops/sec; 2485.7 MB/s
readseq : 3.722 micros/op 268657 ops/sec; 2627.7 MB/s
readseq : 3.149 micros/op 317605 ops/sec; 3106.5 MB/s
readseq : 3.125 micros/op 320024 ops/sec; 3130.1 MB/s
readseq : 4.075 micros/op 245374 ops/sec; 2400.0 MB/s
```
```
[Everything in Memtable | 10K operands | 10 KB each | 10 operand per key]
DEBUG_LEVEL=0 make db_bench -j64 && ./db_bench --benchmarks="mergerandom,readseq,readseq,readseq,readseq,readseq" --merge_operator="max" --merge_keys=1000 --num=10000 --disable_auto_compactions --value_size=10240 --write_buffer_size=1000000000
[FullMergeV2]
readseq : 3.472 micros/op 288018 ops/sec; 2817.1 MB/s
readseq : 2.304 micros/op 434027 ops/sec; 4245.2 MB/s
readseq : 1.163 micros/op 859845 ops/sec; 8410.0 MB/s
readseq : 1.192 micros/op 838926 ops/sec; 8205.4 MB/s
readseq : 1.250 micros/op 800000 ops/sec; 7824.7 MB/s
[master]
readseq : 24.025 micros/op 41623 ops/sec; 407.1 MB/s
readseq : 18.489 micros/op 54086 ops/sec; 529.0 MB/s
readseq : 18.693 micros/op 53495 ops/sec; 523.2 MB/s
readseq : 23.621 micros/op 42335 ops/sec; 414.1 MB/s
readseq : 18.775 micros/op 53262 ops/sec; 521.0 MB/s
```
```
[Everything in Block cache | 10K operands | 10 KB each | 1 operand per key]
[FullMergeV2]
$ DEBUG_LEVEL=0 make db_bench -j64 && ./db_bench --benchmarks="readseq,readseq,readseq,readseq,readseq" --merge_operator="max" --num=100000 --db="/dev/shm/merge-random-10K-10KB" --cache_size=1000000000 --use_existing_db --disable_auto_compactions
readseq : 14.741 micros/op 67837 ops/sec; 663.5 MB/s
readseq : 1.029 micros/op 971446 ops/sec; 9501.6 MB/s
readseq : 0.974 micros/op 1026229 ops/sec; 10037.4 MB/s
readseq : 0.965 micros/op 1036080 ops/sec; 10133.8 MB/s
readseq : 0.943 micros/op 1060657 ops/sec; 10374.2 MB/s
[master]
readseq : 16.735 micros/op 59755 ops/sec; 584.5 MB/s
readseq : 3.029 micros/op 330151 ops/sec; 3229.2 MB/s
readseq : 3.136 micros/op 318883 ops/sec; 3119.0 MB/s
readseq : 3.065 micros/op 326245 ops/sec; 3191.0 MB/s
readseq : 3.014 micros/op 331813 ops/sec; 3245.4 MB/s
```
```
[Everything in Block cache | 10K operands | 10 KB each | 10 operand per key]
DEBUG_LEVEL=0 make db_bench -j64 && ./db_bench --benchmarks="readseq,readseq,readseq,readseq,readseq" --merge_operator="max" --num=100000 --db="/dev/shm/merge-random-10-operands-10K-10KB" --cache_size=1000000000 --use_existing_db --disable_auto_compactions
[FullMergeV2]
readseq : 24.325 micros/op 41109 ops/sec; 402.1 MB/s
readseq : 1.470 micros/op 680272 ops/sec; 6653.7 MB/s
readseq : 1.231 micros/op 812347 ops/sec; 7945.5 MB/s
readseq : 1.091 micros/op 916590 ops/sec; 8965.1 MB/s
readseq : 1.109 micros/op 901713 ops/sec; 8819.6 MB/s
[master]
readseq : 27.257 micros/op 36687 ops/sec; 358.8 MB/s
readseq : 4.443 micros/op 225073 ops/sec; 2201.4 MB/s
readseq : 5.830 micros/op 171526 ops/sec; 1677.7 MB/s
readseq : 4.173 micros/op 239635 ops/sec; 2343.8 MB/s
readseq : 4.150 micros/op 240963 ops/sec; 2356.8 MB/s
```
Test Plan: COMPILE_WITH_ASAN=1 make check -j64
Reviewers: yhchiang, andrewkr, sdong
Reviewed By: sdong
Subscribers: lovro, andrewkr, dhruba
Differential Revision: https://reviews.facebook.net/D57075
2016-07-20 18:49:03 +02:00
|
|
|
}
|
|
|
|
|
2016-12-16 20:00:03 +01:00
|
|
|
void DBTestBase::VerifyDBInternal(
|
|
|
|
std::vector<std::pair<std::string, std::string>> true_data) {
|
|
|
|
Arena arena;
|
|
|
|
InternalKeyComparator icmp(last_options_.comparator);
|
2018-12-18 02:26:56 +01:00
|
|
|
ReadRangeDelAggregator range_del_agg(&icmp,
|
|
|
|
kMaxSequenceNumber /* upper_bound */);
|
2020-08-04 00:21:56 +02:00
|
|
|
ReadOptions read_options;
|
|
|
|
auto iter = dbfull()->NewInternalIterator(read_options, &arena,
|
|
|
|
&range_del_agg, kMaxSequenceNumber);
|
2016-12-16 20:00:03 +01:00
|
|
|
iter->SeekToFirst();
|
|
|
|
for (auto p : true_data) {
|
|
|
|
ASSERT_TRUE(iter->Valid());
|
|
|
|
ParsedInternalKey ikey;
|
2020-10-28 18:11:13 +01:00
|
|
|
ASSERT_OK(ParseInternalKey(iter->key(), &ikey, true /* log_err_key */));
|
2016-12-16 20:00:03 +01:00
|
|
|
ASSERT_EQ(p.first, ikey.user_key);
|
|
|
|
ASSERT_EQ(p.second, iter->value());
|
|
|
|
iter->Next();
|
|
|
|
};
|
|
|
|
ASSERT_FALSE(iter->Valid());
|
|
|
|
iter->~InternalIterator();
|
|
|
|
}
|
|
|
|
|
2016-04-18 18:42:50 +02:00
|
|
|
#ifndef ROCKSDB_LITE
|
2016-06-22 02:57:59 +02:00
|
|
|
|
2016-04-18 18:42:50 +02:00
|
|
|
uint64_t DBTestBase::GetNumberOfSstFilesForColumnFamily(
|
|
|
|
DB* db, std::string column_family_name) {
|
|
|
|
std::vector<LiveFileMetaData> metadata;
|
|
|
|
db->GetLiveFilesMetaData(&metadata);
|
|
|
|
uint64_t result = 0;
|
|
|
|
for (auto& fileMetadata : metadata) {
|
|
|
|
result += (fileMetadata.column_family_name == column_family_name);
|
|
|
|
}
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
#endif // ROCKSDB_LITE
|
|
|
|
|
2020-02-20 21:07:53 +01:00
|
|
|
} // namespace ROCKSDB_NAMESPACE
|