From 12b78e40bdba980f3692fd2018cc905eef2127f8 Mon Sep 17 00:00:00 2001 From: Cheng Chang Date: Fri, 9 Oct 2020 16:40:25 -0700 Subject: [PATCH] Track WAL in MANIFEST: add option track_and_verify_wals_in_manifest (#7275) Summary: This option determines whether WALs will be tracked in MANIFEST and verified on recovery. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7275 Test Plan: db_options_test options_test Reviewed By: pdillinger Differential Revision: D23181418 Pulled By: cheng-chang fbshipit-source-id: 5dd1cdc166f3dfc1c93c094df4a2f7734e3b4547 --- db/db_options_test.cc | 15 +++++++++++++++ include/rocksdb/options.h | 14 ++++++++++++++ options/db_options.cc | 11 +++++++++++ options/db_options.h | 1 + options/options_helper.cc | 2 ++ options/options_settable_test.cc | 1 + options/options_test.cc | 6 ++++++ test_util/testutil.cc | 1 + 8 files changed, 51 insertions(+) diff --git a/db/db_options_test.cc b/db/db_options_test.cc index 0ab06489c..a49b76939 100644 --- a/db/db_options_test.cc +++ b/db/db_options_test.cc @@ -79,6 +79,21 @@ class DBOptionsTest : public DBTestBase { #endif // ROCKSDB_LITE }; +TEST_F(DBOptionsTest, ImmutableTrackAndVerifyWalsInManifest) { + Options options; + options.track_and_verify_wals_in_manifest = true; + + ImmutableDBOptions db_options(options); + ASSERT_TRUE(db_options.track_and_verify_wals_in_manifest); + + Reopen(options); + ASSERT_TRUE(dbfull()->GetDBOptions().track_and_verify_wals_in_manifest); + + Status s = + dbfull()->SetDBOptions({{"track_and_verify_wals_in_manifest", "false"}}); + ASSERT_FALSE(s.ok()); +} + // RocksDB lite don't support dynamic options. #ifndef ROCKSDB_LITE diff --git a/include/rocksdb/options.h b/include/rocksdb/options.h index 53c444a30..a85fdf4a6 100644 --- a/include/rocksdb/options.h +++ b/include/rocksdb/options.h @@ -390,6 +390,20 @@ struct DBOptions { // Default: true bool paranoid_checks = true; + // If true, track WALs in MANIFEST and verify them on recovery. + // + // If a WAL is tracked in MANIFEST but is missing from disk on recovery, + // or the size of the tracked WAL is larger than the WAL's on-disk size, + // an error is reported and recovery is aborted. + // + // If a WAL is not tracked in MANIFEST, then no verification will happen + // during recovery. + // + // Default: false + // FIXME(cheng): This option is part of a work in progress and does not yet + // work + bool track_and_verify_wals_in_manifest = false; + // Use the specified object to interact with the environment, // e.g. to read/write files, schedule background work, etc. In the near // future, support for doing storage operations such as read/write files diff --git a/options/db_options.cc b/options/db_options.cc index 2b97c37db..c61154a4b 100644 --- a/options/db_options.cc +++ b/options/db_options.cc @@ -198,6 +198,11 @@ static std::unordered_map {offsetof(struct ImmutableDBOptions, paranoid_checks), OptionType::kBoolean, OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, + {"track_and_verify_wals_in_manifest", + {offsetof(struct ImmutableDBOptions, + track_and_verify_wals_in_manifest), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone}}, {"skip_log_error_on_recovery", {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, OptionTypeFlags::kNone}}, @@ -497,6 +502,8 @@ ImmutableDBOptions::ImmutableDBOptions(const DBOptions& options) create_missing_column_families(options.create_missing_column_families), error_if_exists(options.error_if_exists), paranoid_checks(options.paranoid_checks), + track_and_verify_wals_in_manifest( + options.track_and_verify_wals_in_manifest), env(options.env), fs(options.env->GetFileSystem()), rate_limiter(options.rate_limiter), @@ -579,6 +586,10 @@ void ImmutableDBOptions::Dump(Logger* log) const { create_if_missing); ROCKS_LOG_HEADER(log, " Options.paranoid_checks: %d", paranoid_checks); + ROCKS_LOG_HEADER(log, + " " + "Options.track_and_verify_wals_in_manifest: %d", + track_and_verify_wals_in_manifest); ROCKS_LOG_HEADER(log, " Options.env: %p", env); ROCKS_LOG_HEADER(log, " Options.fs: %s", diff --git a/options/db_options.h b/options/db_options.h index 2684e01b5..631d38ac0 100644 --- a/options/db_options.h +++ b/options/db_options.h @@ -23,6 +23,7 @@ struct ImmutableDBOptions { bool create_missing_column_families; bool error_if_exists; bool paranoid_checks; + bool track_and_verify_wals_in_manifest; Env* env; std::shared_ptr fs; std::shared_ptr rate_limiter; diff --git a/options/options_helper.cc b/options/options_helper.cc index 5240c4b94..6bec5e82b 100644 --- a/options/options_helper.cc +++ b/options/options_helper.cc @@ -51,6 +51,8 @@ DBOptions BuildDBOptions(const ImmutableDBOptions& immutable_db_options, immutable_db_options.create_missing_column_families; options.error_if_exists = immutable_db_options.error_if_exists; options.paranoid_checks = immutable_db_options.paranoid_checks; + options.track_and_verify_wals_in_manifest = + immutable_db_options.track_and_verify_wals_in_manifest; options.env = immutable_db_options.env; options.rate_limiter = immutable_db_options.rate_limiter; options.sst_file_manager = immutable_db_options.sst_file_manager; diff --git a/options/options_settable_test.cc b/options/options_settable_test.cc index 28e25b576..3dc87da16 100644 --- a/options/options_settable_test.cc +++ b/options/options_settable_test.cc @@ -278,6 +278,7 @@ TEST_F(OptionsSettableTest, DBOptionsAllFieldsSettable) { "skip_log_error_on_recovery=true;" "writable_file_max_buffer_size=1048576;" "paranoid_checks=true;" + "track_and_verify_wals_in_manifest=true;" "is_fd_close_on_exec=false;" "bytes_per_sync=4295013613;" "strict_bytes_per_sync=true;" diff --git a/options/options_test.cc b/options/options_test.cc index 2ad551eed..510b9ee1e 100644 --- a/options/options_test.cc +++ b/options/options_test.cc @@ -109,6 +109,7 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) { {"create_missing_column_families", "true"}, {"error_if_exists", "false"}, {"paranoid_checks", "true"}, + {"track_and_verify_wals_in_manifest", "true"}, {"max_open_files", "32"}, {"max_total_wal_size", "33"}, {"use_fsync", "true"}, @@ -263,6 +264,7 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) { ASSERT_EQ(new_db_opt.create_missing_column_families, true); ASSERT_EQ(new_db_opt.error_if_exists, false); ASSERT_EQ(new_db_opt.paranoid_checks, true); + ASSERT_EQ(new_db_opt.track_and_verify_wals_in_manifest, true); ASSERT_EQ(new_db_opt.max_open_files, 32); ASSERT_EQ(new_db_opt.max_total_wal_size, static_cast(33)); ASSERT_EQ(new_db_opt.use_fsync, true); @@ -774,6 +776,7 @@ TEST_F(OptionsTest, OldInterfaceTest) { {"create_missing_column_families", "true"}, {"error_if_exists", "false"}, {"paranoid_checks", "true"}, + {"track_and_verify_wals_in_manifest", "true"}, {"max_open_files", "32"}, }; ASSERT_OK(GetDBOptionsFromMap(base_db_opt, db_options_map, &new_db_opt)); @@ -781,6 +784,7 @@ TEST_F(OptionsTest, OldInterfaceTest) { ASSERT_EQ(new_db_opt.create_missing_column_families, true); ASSERT_EQ(new_db_opt.error_if_exists, false); ASSERT_EQ(new_db_opt.paranoid_checks, true); + ASSERT_EQ(new_db_opt.track_and_verify_wals_in_manifest, true); ASSERT_EQ(new_db_opt.max_open_files, 32); db_options_map["unknown_option"] = "1"; ASSERT_NOK(GetDBOptionsFromMap(base_db_opt, db_options_map, &new_db_opt)); @@ -1620,6 +1624,7 @@ TEST_F(OptionsOldApiTest, GetOptionsFromMapTest) { {"create_missing_column_families", "true"}, {"error_if_exists", "false"}, {"paranoid_checks", "true"}, + {"track_and_verify_wals_in_manifest", "true"}, {"max_open_files", "32"}, {"max_total_wal_size", "33"}, {"use_fsync", "true"}, @@ -1768,6 +1773,7 @@ TEST_F(OptionsOldApiTest, GetOptionsFromMapTest) { ASSERT_EQ(new_db_opt.create_missing_column_families, true); ASSERT_EQ(new_db_opt.error_if_exists, false); ASSERT_EQ(new_db_opt.paranoid_checks, true); + ASSERT_EQ(new_db_opt.track_and_verify_wals_in_manifest, true); ASSERT_EQ(new_db_opt.max_open_files, 32); ASSERT_EQ(new_db_opt.max_total_wal_size, static_cast(33)); ASSERT_EQ(new_db_opt.use_fsync, true); diff --git a/test_util/testutil.cc b/test_util/testutil.cc index d05bb766e..24005e782 100644 --- a/test_util/testutil.cc +++ b/test_util/testutil.cc @@ -309,6 +309,7 @@ void RandomInitDBOptions(DBOptions* db_opt, Random* rnd) { db_opt->error_if_exists = rnd->Uniform(2); db_opt->is_fd_close_on_exec = rnd->Uniform(2); db_opt->paranoid_checks = rnd->Uniform(2); + db_opt->track_and_verify_wals_in_manifest = rnd->Uniform(2); db_opt->skip_log_error_on_recovery = rnd->Uniform(2); db_opt->skip_stats_update_on_db_open = rnd->Uniform(2); db_opt->skip_checking_sst_file_sizes_on_db_open = rnd->Uniform(2);