andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

RocksDBOptionsParser::Parse()'s ignore_unknown_options argument only ingores options from higher version.

Browse Source 
Summary:
RocksDB should always be able to parse an option file generated using the same or lower version. Unknown option should only happen if it is from a higher version. Change the behavior of RocksDBOptionsParser::Parse()'s behavior with ignore_unknown_options=true so that unknown option from a lower or the same version will never be skipped.
Closes https://github.com/facebook/rocksdb/pull/3527

Differential Revision: D7048851

Pulled By: siying

fbshipit-source-id: e261caea12f6515611a4a29f39acf2b619df2361
This commit is contained in:
Siying Dong 2018-02-22 13:23:53 -08:00 committed by Facebook Github Bot
parent aba3409740
commit 4624edc440
3 changed files with 82 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
HISTORY.md
View File

@ -5,6 +5,7 @@
* Add `include_end` option to make the range end exclusive when `include_end == false` in `DeleteFilesInRange()`. * Add `include_end` option to make the range end exclusive when `include_end == false` in `DeleteFilesInRange()`.
* Add `CompactRangeOptions::allow_write_stall`, which makes `CompactRange` start working immediately, even if it causes user writes to stall. The default value is false, meaning we add delay to `CompactRange` calls until stalling can be avoided when possible. Note this delay is not present in previous RocksDB versions. * Add `CompactRangeOptions::allow_write_stall`, which makes `CompactRange` start working immediately, even if it causes user writes to stall. The default value is false, meaning we add delay to `CompactRange` calls until stalling can be avoided when possible. Note this delay is not present in previous RocksDB versions.
* Creating checkpoint with empty directory now returns `Status::InvalidArgument`; previously, it returned `Status::IOError`. * Creating checkpoint with empty directory now returns `Status::InvalidArgument`; previously, it returned `Status::IOError`.
* RocksDBOptionsParser::Parse()'s `ignore_unknown_options` argument will only be effective if the option file shows it is generated using a higher version of RocksDB than the current version.
### New Features ### New Features
* Improve the performance of iterators doing long range scans by using readahead. * Improve the performance of iterators doing long range scans by using readahead.

10
options/options_parser.cc
View File

@ -268,6 +268,16 @@ Status RocksDBOptionsParser::Parse(const std::string& file_name, Env* env,
if (!s.ok()) { if (!s.ok()) {
return s; return s;
} }
// If the option file is not generated by a higher minor version,
// there shouldn't be any unknown option.
if (ignore_unknown_options && section == kOptionSectionVersion) {
if (db_version[0] < ROCKSDB_MAJOR || (db_version[0] == ROCKSDB_MAJOR &&
db_version[1] <= ROCKSDB_MINOR)) {
ignore_unknown_options = false;
}
}
s = ParseSection(&section, &title, &argument, line, line_num); s = ParseSection(&section, &title, &argument, line, line_num);
if (!s.ok()) { if (!s.ok()) {
return s; return s;

100
options/options_test.cc
View File

@ -1216,37 +1216,79 @@ TEST_F(OptionsParserTest, DuplicateCFOptions) {
} }
TEST_F(OptionsParserTest, IgnoreUnknownOptions) { TEST_F(OptionsParserTest, IgnoreUnknownOptions) {
DBOptions db_opt; for (int case_id = 0; case_id < 5; case_id++) {
db_opt.max_open_files = 12345; DBOptions db_opt;
db_opt.max_background_flushes = 301; db_opt.max_open_files = 12345;
db_opt.max_total_wal_size = 1024; db_opt.max_background_flushes = 301;
ColumnFamilyOptions cf_opt; db_opt.max_total_wal_size = 1024;
ColumnFamilyOptions cf_opt;
std::string options_file_content = std::string version_string;
"# This is a testing option string.\n" bool should_ignore = true;
"# Currently we only support \"#\" styled comment.\n" if (case_id == 0) {
"\n" // same version
"[Version]\n" should_ignore = false;
" rocksdb_version=3.14.0\n" version_string =
" options_file_version=1\n" ToString(ROCKSDB_MAJOR) + "." + ToString(ROCKSDB_MINOR) + ".0";
"[DBOptions]\n" } else if (case_id == 1) {
" max_open_files=12345\n" // higher minor version
" max_background_flushes=301\n" should_ignore = true;
" max_total_wal_size=1024 # keep_log_file_num=1000\n" version_string =
" unknown_db_option1=321\n" ToString(ROCKSDB_MAJOR) + "." + ToString(ROCKSDB_MINOR + 1) + ".0";
" unknown_db_option2=false\n" } else if (case_id == 2) {
"[CFOptions \"default\"]\n" // higher major version.
" unknown_cf_option1=hello\n" should_ignore = true;
"[CFOptions \"something_else\"]\n" version_string = ToString(ROCKSDB_MAJOR + 1) + ".0.0";
" unknown_cf_option2=world\n" } else if (case_id == 3) {
" # if a section is blank, we will use the default\n"; // lower minor version
#if ROCKSDB_MINOR == 0
continue;
#else
version_string =
ToString(ROCKSDB_MAJOR) + "." + ToString(ROCKSDB_MINOR - 1) + ".0";
should_ignore = false;
#endif
} else {
// lower major version
should_ignore = false;
version_string =
ToString(ROCKSDB_MAJOR - 1) + "." + ToString(ROCKSDB_MINOR) + ".0";
}
const std::string kTestFileName = "test-rocksdb-options.ini"; std::string options_file_content =
env_->WriteToNewFile(kTestFileName, options_file_content); "# This is a testing option string.\n"
RocksDBOptionsParser parser; "# Currently we only support \"#\" styled comment.\n"
ASSERT_NOK(parser.Parse(kTestFileName, env_.get())); "\n"
ASSERT_OK(parser.Parse(kTestFileName, env_.get(), "[Version]\n"
true /* ignore_unknown_options */)); " rocksdb_version=" +
version_string +
"\n"
" options_file_version=1\n"
"[DBOptions]\n"
" max_open_files=12345\n"
" max_background_flushes=301\n"
" max_total_wal_size=1024 # keep_log_file_num=1000\n"
" unknown_db_option1=321\n"
" unknown_db_option2=false\n"
"[CFOptions \"default\"]\n"
" unknown_cf_option1=hello\n"
"[CFOptions \"something_else\"]\n"
" unknown_cf_option2=world\n"
" # if a section is blank, we will use the default\n";
const std::string kTestFileName = "test-rocksdb-options.ini";
env_->DeleteFile(kTestFileName);
env_->WriteToNewFile(kTestFileName, options_file_content);
RocksDBOptionsParser parser;
ASSERT_NOK(parser.Parse(kTestFileName, env_.get()));
if (should_ignore) {
ASSERT_OK(parser.Parse(kTestFileName, env_.get(),
true /* ignore_unknown_options */));
} else {
ASSERT_NOK(parser.Parse(kTestFileName, env_.get(),
true /* ignore_unknown_options */));
}
}
} }
TEST_F(OptionsParserTest, ParseVersion) { TEST_F(OptionsParserTest, ParseVersion) {