andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add OptionsSanityCheckLevel

Browse Source 
Summary:
This patch introduces OptionsSanityCheckLevel internally to enable
sanity check rocksdb options.

Utilities API will be added in the follow-up diffs.

Test Plan: Added more tests in options_test

Reviewers: igor, IslamAbdelRahman, sdong, anthony

Reviewed By: anthony

Subscribers: dhruba, leveldb

Differential Revision: https://reviews.facebook.net/D49515
This commit is contained in:
Yueh-Hsuan Chiang 2015-11-04 18:53:30 -08:00
parent dba5e00741
commit 183cadfc87
7 changed files with 330 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CMakeLists.txt
View File

@ -215,6 +215,7 @@ set(SOURCES
util/options_builder.cc
util/options_helper.cc
util/options_parser.cc
util/options_sanity_check.cc
util/perf_context.cc
util/perf_level.cc
util/rate_limiter.cc

3
src.mk
View File

@ -138,10 +138,11 @@ LIB_SOURCES = \
util/memenv.cc \
util/murmurhash.cc \
util/mutable_cf_options.cc \
util/options_builder.cc \
util/options.cc \
util/options_builder.cc \
util/options_helper.cc \
util/options_parser.cc \
util/options_sanity_check.cc \
util/perf_context.cc \
util/perf_level.cc \
util/rate_limiter.cc \

142
util/options_parser.cc
View File

@ -581,7 +581,8 @@ bool AreEqualOptions(
Status RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
const DBOptions& db_opt, const std::vector<std::string>& cf_names,
const std::vector<ColumnFamilyOptions>& cf_opts,
const std::string& file_name, Env* env) {
const std::string& file_name, Env* env,
OptionsSanityCheckLevel sanity_check_level) {
RocksDBOptionsParser parser;
std::unique_ptr<SequentialFile> seq_file;
Status s = parser.Parse(file_name, env);
@ -590,20 +591,28 @@ Status RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
}
// Verify DBOptions
s = VerifyDBOptions(db_opt, *parser.db_opt(), parser.db_opt_map());
s = VerifyDBOptions(db_opt, *parser.db_opt(), parser.db_opt_map(),
sanity_check_level);
if (!s.ok()) {
return s;
}
// Verify ColumnFamily Name
if (cf_names.size() != parser.cf_names()->size()) {
return Status::Corruption(
"[RocksDBOptionParser Error] The persisted options does not have"
"the same number of column family names as the db instance.");
if (sanity_check_level >= kSanityLevelLooselyCompatible) {
return Status::InvalidArgument(
"[RocksDBOptionParser Error] The persisted options does not have "
"the same number of column family names as the db instance.");
} else if (cf_opts.size() > parser.cf_opts()->size()) {
return Status::InvalidArgument(
"[RocksDBOptionsParser Error]",
"The persisted options file has less number of column family "
"names than that of the specified one.");
}
}
for (size_t i = 0; i < cf_names.size(); ++i) {
if (cf_names[i] != parser.cf_names()->at(i)) {
return Status::Corruption(
return Status::InvalidArgument(
"[RocksDBOptionParser Error] The persisted options and the db"
"instance does not have the same name for column family ",
ToString(i));
@ -612,18 +621,27 @@ Status RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
// Verify Column Family Options
if (cf_opts.size() != parser.cf_opts()->size()) {
return Status::Corruption(
"[RocksDBOptionParser Error] The persisted options does not have"
"the same number of column families as the db instance.");
if (sanity_check_level >= kSanityLevelLooselyCompatible) {
return Status::InvalidArgument(
"[RocksDBOptionsParser Error]",
"The persisted options does not have the same number of "
"column families as the db instance.");
} else if (cf_opts.size() > parser.cf_opts()->size()) {
return Status::InvalidArgument(
"[RocksDBOptionsParser Error]",
"The persisted options file has less number of column families "
"than that of the specified number.");
}
}
for (size_t i = 0; i < cf_opts.size(); ++i) {
s = VerifyCFOptions(cf_opts[i], parser.cf_opts()->at(i),
&(parser.cf_opt_maps()->at(i)));
&(parser.cf_opt_maps()->at(i)), sanity_check_level);
if (!s.ok()) {
return s;
}
s = VerifyTableFactory(cf_opts[i].table_factory.get(),
parser.cf_opts()->at(i).table_factory.get());
parser.cf_opts()->at(i).table_factory.get(),
sanity_check_level);
if (!s.ok()) {
return s;
}
@ -633,42 +651,75 @@ Status RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
}
Status RocksDBOptionsParser::VerifyDBOptions(
const DBOptions& base_opt, const DBOptions& new_opt,
const std::unordered_map<std::string, std::string>* opt_map) {
const DBOptions& base_opt, const DBOptions& persisted_opt,
const std::unordered_map<std::string, std::string>* opt_map,
OptionsSanityCheckLevel sanity_check_level) {
for (auto pair : db_options_type_info) {
if (pair.second.verification == OptionVerificationType::kDeprecated) {
// We skip checking deprecated variables as they might
// contain random values since they might not be initialized
continue;
}
if (!AreEqualOptions(reinterpret_cast<const char*>(&base_opt),
reinterpret_cast<const char*>(&new_opt), pair.second,
pair.first, nullptr)) {
return Status::Corruption(
"[RocksDBOptionsParser]: "
"failed the verification on DBOptions::",
pair.first);
if (DBOptionSanityCheckLevel(pair.first) <= sanity_check_level) {
if (!AreEqualOptions(reinterpret_cast<const char*>(&base_opt),
reinterpret_cast<const char*>(&persisted_opt),
pair.second, pair.first, nullptr)) {
const size_t kBufferSize = 2048;
char buffer[kBufferSize];
std::string base_value;
std::string persisted_value;
SerializeSingleOptionHelper(
reinterpret_cast<const char*>(&base_opt) + pair.second.offset,
pair.second.type, &base_value);
SerializeSingleOptionHelper(
reinterpret_cast<const char*>(&persisted_opt) + pair.second.offset,
pair.second.type, &persisted_value);
snprintf(buffer, sizeof(buffer),
"[RocksDBOptionsParser]: "
"failed the verification on DBOptions::%s --- "
"The specified one is %s while the persisted one is %s.\n",
pair.first.c_str(), base_value.c_str(),
persisted_value.c_str());
return Status::InvalidArgument(Slice(buffer, strlen(buffer)));
}
}
}
return Status::OK();
}
Status RocksDBOptionsParser::VerifyCFOptions(
const ColumnFamilyOptions& base_opt, const ColumnFamilyOptions& new_opt,
const std::unordered_map<std::string, std::string>* new_opt_map) {
const ColumnFamilyOptions& base_opt,
const ColumnFamilyOptions& persisted_opt,
const std::unordered_map<std::string, std::string>* persisted_opt_map,
OptionsSanityCheckLevel sanity_check_level) {
for (auto& pair : cf_options_type_info) {
if (pair.second.verification == OptionVerificationType::kDeprecated) {
// We skip checking deprecated variables as they might
// contain random values since they might not be initialized
continue;
}
if (!AreEqualOptions(reinterpret_cast<const char*>(&base_opt),
reinterpret_cast<const char*>(&new_opt), pair.second,
pair.first, new_opt_map)) {
return Status::Corruption(
"[RocksDBOptionsParser]: "
"failed the verification on ColumnFamilyOptions::",
pair.first);
if (CFOptionSanityCheckLevel(pair.first) <= sanity_check_level) {
if (!AreEqualOptions(reinterpret_cast<const char*>(&base_opt),
reinterpret_cast<const char*>(&persisted_opt),
pair.second, pair.first, persisted_opt_map)) {
const size_t kBufferSize = 2048;
char buffer[kBufferSize];
std::string base_value;
std::string persisted_value;
SerializeSingleOptionHelper(
reinterpret_cast<const char*>(&base_opt) + pair.second.offset,
pair.second.type, &base_value);
SerializeSingleOptionHelper(
reinterpret_cast<const char*>(&persisted_opt) + pair.second.offset,
pair.second.type, &persisted_value);
snprintf(buffer, sizeof(buffer),
"[RocksDBOptionsParser]: "
"failed the verification on ColumnFamilyOptions::%s --- "
"The specified one is %s while the persisted one is %s.\n",
pair.first.c_str(), base_value.c_str(),
persisted_value.c_str());
return Status::InvalidArgument(Slice(buffer, sizeof(buffer)));
}
}
}
return Status::OK();
@ -676,8 +727,10 @@ Status RocksDBOptionsParser::VerifyCFOptions(
Status RocksDBOptionsParser::VerifyBlockBasedTableFactory(
const BlockBasedTableFactory* base_tf,
const BlockBasedTableFactory* file_tf) {
if ((base_tf != nullptr) != (file_tf != nullptr)) {
const BlockBasedTableFactory* file_tf,
OptionsSanityCheckLevel sanity_check_level) {
if ((base_tf != nullptr) != (file_tf != nullptr) &&
sanity_check_level > kSanityLevelNone) {
return Status::Corruption(
"[RocksDBOptionsParser]: Inconsistent TableFactory class type");
}
@ -694,29 +747,34 @@ Status RocksDBOptionsParser::VerifyBlockBasedTableFactory(
// contain random values since they might not be initialized
continue;
}
if (!AreEqualOptions(reinterpret_cast<const char*>(&base_opt),
reinterpret_cast<const char*>(&file_opt), pair.second,
pair.first, nullptr)) {
return Status::Corruption(
"[RocksDBOptionsParser]: "
"failed the verification on BlockBasedTableOptions::",
pair.first);
if (BBTOptionSanityCheckLevel(pair.first) <= sanity_check_level) {
if (!AreEqualOptions(reinterpret_cast<const char*>(&base_opt),
reinterpret_cast<const char*>(&file_opt),
pair.second, pair.first, nullptr)) {
return Status::Corruption(
"[RocksDBOptionsParser]: "
"failed the verification on BlockBasedTableOptions::",
pair.first);
}
}
}
return Status::OK();
}
Status RocksDBOptionsParser::VerifyTableFactory(const TableFactory* base_tf,
const TableFactory* file_tf) {
Status RocksDBOptionsParser::VerifyTableFactory(
const TableFactory* base_tf, const TableFactory* file_tf,
OptionsSanityCheckLevel sanity_check_level) {
if (base_tf && file_tf) {
if (base_tf->Name() != file_tf->Name()) {
if (sanity_check_level > kSanityLevelNone &&
base_tf->Name() != file_tf->Name()) {
return Status::Corruption(
"[RocksDBOptionsParser]: "
"failed the verification on TableFactory->Name()");
}
auto s = VerifyBlockBasedTableFactory(
dynamic_cast<const BlockBasedTableFactory*>(base_tf),
dynamic_cast<const BlockBasedTableFactory*>(file_tf));
dynamic_cast<const BlockBasedTableFactory*>(file_tf),
sanity_check_level);
if (!s.ok()) {
return s;
}

19
util/options_parser.h
View File

@ -11,6 +11,8 @@
#include "rocksdb/env.h"
#include "rocksdb/options.h"
#include "util/options_helper.h"
#include "util/options_sanity_check.h"
#include "table/block_based_table_factory.h"
namespace rocksdb {
@ -65,24 +67,27 @@ class RocksDBOptionsParser {
static Status VerifyRocksDBOptionsFromFile(
const DBOptions& db_opt, const std::vector<std::string>& cf_names,
const std::vector<ColumnFamilyOptions>& cf_opts,
const std::string& file_name, Env* env);
const std::string& file_name, Env* env,
OptionsSanityCheckLevel sanity_check_level = kSanityLevelExactMatch);
static Status VerifyDBOptions(
const DBOptions& base_opt, const DBOptions& new_opt,
const std::unordered_map<std::string, std::string>* new_opt_map =
nullptr);
const std::unordered_map<std::string, std::string>* new_opt_map = nullptr,
OptionsSanityCheckLevel sanity_check_level = kSanityLevelExactMatch);
static Status VerifyCFOptions(
const ColumnFamilyOptions& base_opt, const ColumnFamilyOptions& new_opt,
const std::unordered_map<std::string, std::string>* new_opt_map =
nullptr);
const std::unordered_map<std::string, std::string>* new_opt_map = nullptr,
OptionsSanityCheckLevel sanity_check_level = kSanityLevelExactMatch);
static Status VerifyTableFactory(const TableFactory* base_tf,
const TableFactory* file_tf);
const TableFactory* file_tf,
OptionsSanityCheckLevel sanity_check_level);
static Status VerifyBlockBasedTableFactory(
const BlockBasedTableFactory* base_tf,
const BlockBasedTableFactory* file_tf);
const BlockBasedTableFactory* file_tf,
OptionsSanityCheckLevel sanity_check_level);
static Status ExtraParserCheck(const RocksDBOptionsParser& input_parser);

38
util/options_sanity_check.cc Normal file
View File

@ -0,0 +1,38 @@
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
#ifndef ROCKSDB_LITE
#include "util/options_sanity_check.h"
namespace rocksdb {
namespace {
OptionsSanityCheckLevel SanityCheckLevelHelper(
const std::unordered_map<std::string, OptionsSanityCheckLevel>& smap,
const std::string& name) {
auto iter = smap.find(name);
return iter != smap.end() ? iter->second : kSanityLevelExactMatch;
}
}
OptionsSanityCheckLevel DBOptionSanityCheckLevel(
const std::string& option_name) {
return SanityCheckLevelHelper(sanity_level_db_options, option_name);
}
OptionsSanityCheckLevel CFOptionSanityCheckLevel(
const std::string& option_name) {
return SanityCheckLevelHelper(sanity_level_cf_options, option_name);
}
OptionsSanityCheckLevel BBTOptionSanityCheckLevel(
const std::string& option_name) {
return SanityCheckLevelHelper(sanity_level_bbt_options, option_name);
}
} // namespace rocksdb
#endif // !ROCKSDB_LITE

49
util/options_sanity_check.h Normal file
View File

@ -0,0 +1,49 @@
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
#pragma once
#include <string>
#include <unordered_map>
#ifndef ROCKSDB_LITE
namespace rocksdb {
// This enum defines the RocksDB options sanity level.
enum OptionsSanityCheckLevel : unsigned char {
// Performs no sanity check at all.
kSanityLevelNone = 0x00,
// Performs minimum check to ensure the RocksDB instance can be
// opened without corrupting / mis-interpreting the data.
kSanityLevelLooselyCompatible = 0x01,
// Perform exact match sanity check.
kSanityLevelExactMatch = 0xFF,
};
// The sanity check level for DB options
static const std::unordered_map<std::string, OptionsSanityCheckLevel>
sanity_level_db_options = {};
// The sanity check level for column-family options
static const std::unordered_map<std::string, OptionsSanityCheckLevel>
sanity_level_cf_options = {
{"comparator", kSanityLevelLooselyCompatible},
{"prefix_extractor", kSanityLevelLooselyCompatible},
{"table_factory", kSanityLevelLooselyCompatible},
{"merge_operator", kSanityLevelLooselyCompatible}};
// The sanity check level for block-based table options
static const std::unordered_map<std::string, OptionsSanityCheckLevel>
sanity_level_bbt_options = {};
OptionsSanityCheckLevel DBOptionSanityCheckLevel(
const std::string& options_name);
OptionsSanityCheckLevel CFOptionSanityCheckLevel(
const std::string& options_name);
OptionsSanityCheckLevel BBTOptionSanityCheckLevel(
const std::string& options_name);
} // namespace rocksdb
#endif // !ROCKSDB_LITE

128
util/options_test.cc
View File

@ -26,6 +26,7 @@
#include "table/plain_table_factory.h"
#include "util/options_helper.h"
#include "util/options_parser.h"
#include "util/options_sanity_check.h"
#include "util/random.h"
#include "util/testharness.h"
#include "util/testutil.h"
@ -1644,6 +1645,133 @@ TEST_F(OptionsParserTest, DifferentDefault) {
ASSERT_OK(parser.Parse(kOptionsFileName, env_.get()));
}
class OptionsSanityCheckTest : public OptionsParserTest {
public:
OptionsSanityCheckTest() {}
protected:
Status SanityCheckCFOptions(const ColumnFamilyOptions& cf_opts,
OptionsSanityCheckLevel level) {
return RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
DBOptions(), {"default"}, {cf_opts}, kOptionsFileName, env_.get(),
level);
}
Status PersistCFOptions(const ColumnFamilyOptions& cf_opts) {
Status s = env_->DeleteFile(kOptionsFileName);
if (!s.ok()) {
return s;
}
return PersistRocksDBOptions(DBOptions(), {"default"}, {cf_opts},
kOptionsFileName, env_.get());
}
const std::string kOptionsFileName = "OPTIONS";
};
TEST_F(OptionsSanityCheckTest, SanityCheck) {
ColumnFamilyOptions opts;
Random rnd(301);
// default ColumnFamilyOptions
{
ASSERT_OK(PersistCFOptions(opts));
ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
}
// prefix_extractor
{
// change the prefix extractor and expect only pass when
// sanity-level == kSanityLevelNone
opts.prefix_extractor.reset(NewCappedPrefixTransform(10));
ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible));
ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone));
// persist the change
ASSERT_OK(PersistCFOptions(opts));
ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
// use same prefix extractor but with different parameter
opts.prefix_extractor.reset(NewCappedPrefixTransform(15));
// expect pass only in kSanityLevelNone
ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible));
ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone));
// repeat the test with FixedPrefixTransform
opts.prefix_extractor.reset(NewFixedPrefixTransform(10));
ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible));
ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone));
// persist the change of prefix_extractor
ASSERT_OK(PersistCFOptions(opts));
ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
// use same prefix extractor but with different parameter
opts.prefix_extractor.reset(NewFixedPrefixTransform(15));
// expect pass only in kSanityLevelNone
ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible));
ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone));
}
// table_factory
{
for (int tb = 2; tb >= 0; --tb) {
// change the table factory
opts.table_factory.reset(RandomTableFactory(&rnd, tb));
ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible));
ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone));
// persist the change
ASSERT_OK(PersistCFOptions(opts));
ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
}
}
// merge_operator
{
for (int test = 0; test < 5; ++test) {
// change the merge operator
opts.merge_operator.reset(RandomMergeOperator(&rnd));
ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible));
ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone));
// persist the change
ASSERT_OK(PersistCFOptions(opts));
ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
}
}
// compaction_filter
{
for (int test = 0; test < 5; ++test) {
// change the compaction filter
opts.compaction_filter = RandomCompactionFilter(&rnd);
ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible));
// persist the change
ASSERT_OK(PersistCFOptions(opts));
ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
delete opts.compaction_filter;
opts.compaction_filter = nullptr;
}
}
// compaction_filter_factory
{
for (int test = 0; test < 5; ++test) {
// change the compaction filter factory
opts.compaction_filter_factory.reset(RandomCompactionFilterFactory(&rnd));
ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible));
// persist the change
ASSERT_OK(PersistCFOptions(opts));
ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
}
}
}
namespace {
bool IsEscapedString(const std::string& str) {
for (size_t i = 0; i < str.size(); ++i) {