andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Track WAL in MANIFEST: add option track_and_verify_wals_in_manifest (#7275)

Browse Source 
Summary:
This option determines whether WALs will be tracked in MANIFEST and verified on recovery.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/7275

Test Plan:
db_options_test
options_test

Reviewed By: pdillinger

Differential Revision: D23181418

Pulled By: cheng-chang

fbshipit-source-id: 5dd1cdc166f3dfc1c93c094df4a2f7734e3b4547
This commit is contained in:
Cheng Chang 2020-10-09 16:40:25 -07:00 committed by Facebook GitHub Bot
parent 9dd25487cc
commit 12b78e40bd
8 changed files with 51 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
db/db_options_test.cc
View File

@ -79,6 +79,21 @@ class DBOptionsTest : public DBTestBase {
#endif // ROCKSDB_LITE
};
TEST_F(DBOptionsTest, ImmutableTrackAndVerifyWalsInManifest) {
Options options;
options.track_and_verify_wals_in_manifest = true;
ImmutableDBOptions db_options(options);
ASSERT_TRUE(db_options.track_and_verify_wals_in_manifest);
Reopen(options);
ASSERT_TRUE(dbfull()->GetDBOptions().track_and_verify_wals_in_manifest);
Status s =
dbfull()->SetDBOptions({{"track_and_verify_wals_in_manifest", "false"}});
ASSERT_FALSE(s.ok());
}
// RocksDB lite don't support dynamic options.
#ifndef ROCKSDB_LITE

14
include/rocksdb/options.h
View File

@ -390,6 +390,20 @@ struct DBOptions {
// Default: true
bool paranoid_checks = true;
// If true, track WALs in MANIFEST and verify them on recovery.
//
// If a WAL is tracked in MANIFEST but is missing from disk on recovery,
// or the size of the tracked WAL is larger than the WAL's on-disk size,
// an error is reported and recovery is aborted.
//
// If a WAL is not tracked in MANIFEST, then no verification will happen
// during recovery.
//
// Default: false
// FIXME(cheng): This option is part of a work in progress and does not yet
// work
bool track_and_verify_wals_in_manifest = false;
// Use the specified object to interact with the environment,
// e.g. to read/write files, schedule background work, etc. In the near
// future, support for doing storage operations such as read/write files

11
options/db_options.cc
View File

@ -198,6 +198,11 @@ static std::unordered_map<std::string, OptionTypeInfo>
{offsetof(struct ImmutableDBOptions, paranoid_checks),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone}},
{"track_and_verify_wals_in_manifest",
{offsetof(struct ImmutableDBOptions,
track_and_verify_wals_in_manifest),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone}},
{"skip_log_error_on_recovery",
{0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
OptionTypeFlags::kNone}},
@ -497,6 +502,8 @@ ImmutableDBOptions::ImmutableDBOptions(const DBOptions& options)
create_missing_column_families(options.create_missing_column_families),
error_if_exists(options.error_if_exists),
paranoid_checks(options.paranoid_checks),
track_and_verify_wals_in_manifest(
options.track_and_verify_wals_in_manifest),
env(options.env),
fs(options.env->GetFileSystem()),
rate_limiter(options.rate_limiter),
@ -579,6 +586,10 @@ void ImmutableDBOptions::Dump(Logger* log) const {
create_if_missing);
ROCKS_LOG_HEADER(log, " Options.paranoid_checks: %d",
paranoid_checks);
ROCKS_LOG_HEADER(log,
" "
"Options.track_and_verify_wals_in_manifest: %d",
track_and_verify_wals_in_manifest);
ROCKS_LOG_HEADER(log, " Options.env: %p",
env);
ROCKS_LOG_HEADER(log, " Options.fs: %s",

1
options/db_options.h
View File

@ -23,6 +23,7 @@ struct ImmutableDBOptions {
bool create_missing_column_families;
bool error_if_exists;
bool paranoid_checks;
bool track_and_verify_wals_in_manifest;
Env* env;
std::shared_ptr<FileSystem> fs;
std::shared_ptr<RateLimiter> rate_limiter;

2
options/options_helper.cc
View File

@ -51,6 +51,8 @@ DBOptions BuildDBOptions(const ImmutableDBOptions& immutable_db_options,
immutable_db_options.create_missing_column_families;
options.error_if_exists = immutable_db_options.error_if_exists;
options.paranoid_checks = immutable_db_options.paranoid_checks;
options.track_and_verify_wals_in_manifest =
immutable_db_options.track_and_verify_wals_in_manifest;
options.env = immutable_db_options.env;
options.rate_limiter = immutable_db_options.rate_limiter;
options.sst_file_manager = immutable_db_options.sst_file_manager;

1
options/options_settable_test.cc
View File

@ -278,6 +278,7 @@ TEST_F(OptionsSettableTest, DBOptionsAllFieldsSettable) {
"skip_log_error_on_recovery=true;"
"writable_file_max_buffer_size=1048576;"
"paranoid_checks=true;"
"track_and_verify_wals_in_manifest=true;"
"is_fd_close_on_exec=false;"
"bytes_per_sync=4295013613;"
"strict_bytes_per_sync=true;"

6
options/options_test.cc
View File

@ -109,6 +109,7 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) {
{"create_missing_column_families", "true"},
{"error_if_exists", "false"},
{"paranoid_checks", "true"},
{"track_and_verify_wals_in_manifest", "true"},
{"max_open_files", "32"},
{"max_total_wal_size", "33"},
{"use_fsync", "true"},
@ -263,6 +264,7 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) {
ASSERT_EQ(new_db_opt.create_missing_column_families, true);
ASSERT_EQ(new_db_opt.error_if_exists, false);
ASSERT_EQ(new_db_opt.paranoid_checks, true);
ASSERT_EQ(new_db_opt.track_and_verify_wals_in_manifest, true);
ASSERT_EQ(new_db_opt.max_open_files, 32);
ASSERT_EQ(new_db_opt.max_total_wal_size, static_cast<uint64_t>(33));
ASSERT_EQ(new_db_opt.use_fsync, true);
@ -774,6 +776,7 @@ TEST_F(OptionsTest, OldInterfaceTest) {
{"create_missing_column_families", "true"},
{"error_if_exists", "false"},
{"paranoid_checks", "true"},
{"track_and_verify_wals_in_manifest", "true"},
{"max_open_files", "32"},
};
ASSERT_OK(GetDBOptionsFromMap(base_db_opt, db_options_map, &new_db_opt));
@ -781,6 +784,7 @@ TEST_F(OptionsTest, OldInterfaceTest) {
ASSERT_EQ(new_db_opt.create_missing_column_families, true);
ASSERT_EQ(new_db_opt.error_if_exists, false);
ASSERT_EQ(new_db_opt.paranoid_checks, true);
ASSERT_EQ(new_db_opt.track_and_verify_wals_in_manifest, true);
ASSERT_EQ(new_db_opt.max_open_files, 32);
db_options_map["unknown_option"] = "1";
ASSERT_NOK(GetDBOptionsFromMap(base_db_opt, db_options_map, &new_db_opt));
@ -1620,6 +1624,7 @@ TEST_F(OptionsOldApiTest, GetOptionsFromMapTest) {
{"create_missing_column_families", "true"},
{"error_if_exists", "false"},
{"paranoid_checks", "true"},
{"track_and_verify_wals_in_manifest", "true"},
{"max_open_files", "32"},
{"max_total_wal_size", "33"},
{"use_fsync", "true"},
@ -1768,6 +1773,7 @@ TEST_F(OptionsOldApiTest, GetOptionsFromMapTest) {
ASSERT_EQ(new_db_opt.create_missing_column_families, true);
ASSERT_EQ(new_db_opt.error_if_exists, false);
ASSERT_EQ(new_db_opt.paranoid_checks, true);
ASSERT_EQ(new_db_opt.track_and_verify_wals_in_manifest, true);
ASSERT_EQ(new_db_opt.max_open_files, 32);
ASSERT_EQ(new_db_opt.max_total_wal_size, static_cast<uint64_t>(33));
ASSERT_EQ(new_db_opt.use_fsync, true);

1
test_util/testutil.cc
View File

@ -309,6 +309,7 @@ void RandomInitDBOptions(DBOptions* db_opt, Random* rnd) {
db_opt->error_if_exists = rnd->Uniform(2);
db_opt->is_fd_close_on_exec = rnd->Uniform(2);
db_opt->paranoid_checks = rnd->Uniform(2);
db_opt->track_and_verify_wals_in_manifest = rnd->Uniform(2);
db_opt->skip_log_error_on_recovery = rnd->Uniform(2);
db_opt->skip_stats_update_on_db_open = rnd->Uniform(2);
db_opt->skip_checking_sst_file_sizes_on_db_open = rnd->Uniform(2);