andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #797 from SherlockNoMad/optionHelper

Browse Source 
Support PlainTableOption in option_helper.cc
This commit is contained in:
Siying Dong 2015-10-30 09:44:11 -07:00
commit 66a3a87ab3
6 changed files with 226 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
include/rocksdb/convenience.h
View File

@ -43,6 +43,12 @@ Status GetBlockBasedTableOptionsFromMap(
BlockBasedTableOptions* new_table_options,
bool input_strings_escaped = false);
Status GetPlainTableOptionsFromMap(
const PlainTableOptions& table_options,
const std::unordered_map<std::string, std::string>& opts_map,
PlainTableOptions* new_table_options,
bool input_strings_escaped = false);
// Take a string representation of option names and values, apply them into the
// base_options, and return the new options as a result. The string has the
// following format:
@ -74,6 +80,11 @@ Status GetBlockBasedTableOptionsFromString(
const std::string& opts_str,
BlockBasedTableOptions* new_table_options);
Status GetPlainTableOptionsFromString(
const PlainTableOptions& table_options,
const std::string& opts_str,
PlainTableOptions* new_table_options);
Status GetOptionsFromString(const Options& base_options,
const std::string& opts_str, Options* new_options);

32
table/plain_table_factory.cc
View File

@ -21,8 +21,9 @@ Status PlainTableFactory::NewTableReader(
return PlainTableReader::Open(
table_reader_options.ioptions, table_reader_options.env_options,
table_reader_options.internal_comparator, std::move(file), file_size,
table, bloom_bits_per_key_, hash_table_ratio_, index_sparseness_,
huge_page_tlb_size_, full_scan_mode_);
table, table_options_.bloom_bits_per_key, table_options_.hash_table_ratio,
table_options_.index_sparseness, table_options_.huge_page_tlb_size,
table_options_.full_scan_mode);
}
TableBuilder* PlainTableFactory::NewTableBuilder(
@ -35,9 +36,10 @@ TableBuilder* PlainTableFactory::NewTableBuilder(
return new PlainTableBuilder(
table_builder_options.ioptions,
table_builder_options.int_tbl_prop_collector_factories, column_family_id,
file, user_key_len_, encoding_type_, index_sparseness_,
bloom_bits_per_key_, 6, huge_page_tlb_size_, hash_table_ratio_,
store_index_in_file_);
file, table_options_.user_key_len, table_options_.encoding_type,
table_options_.index_sparseness, table_options_.bloom_bits_per_key, 6,
table_options_.huge_page_tlb_size, table_options_.hash_table_ratio,
table_options_.store_index_in_file);
}
std::string PlainTableFactory::GetPrintableTableOptions() const {
@ -47,32 +49,36 @@ std::string PlainTableFactory::GetPrintableTableOptions() const {
char buffer[kBufferSize];
snprintf(buffer, kBufferSize, " user_key_len: %u\n",
user_key_len_);
table_options_.user_key_len);
ret.append(buffer);
snprintf(buffer, kBufferSize, " bloom_bits_per_key: %d\n",
bloom_bits_per_key_);
table_options_.bloom_bits_per_key);
ret.append(buffer);
snprintf(buffer, kBufferSize, " hash_table_ratio: %lf\n",
hash_table_ratio_);
table_options_.hash_table_ratio);
ret.append(buffer);
snprintf(buffer, kBufferSize, " index_sparseness: %" ROCKSDB_PRIszt "\n",
index_sparseness_);
table_options_.index_sparseness);
ret.append(buffer);
snprintf(buffer, kBufferSize, " huge_page_tlb_size: %" ROCKSDB_PRIszt "\n",
huge_page_tlb_size_);
table_options_.huge_page_tlb_size);
ret.append(buffer);
snprintf(buffer, kBufferSize, " encoding_type: %d\n",
encoding_type_);
table_options_.encoding_type);
ret.append(buffer);
snprintf(buffer, kBufferSize, " full_scan_mode: %d\n",
full_scan_mode_);
table_options_.full_scan_mode);
ret.append(buffer);
snprintf(buffer, kBufferSize, " store_index_in_file: %d\n",
store_index_in_file_);
table_options_.store_index_in_file);
ret.append(buffer);
return ret;
}
const PlainTableOptions& PlainTableFactory::GetTableOptions() const {
return table_options_;
}
extern TableFactory* NewPlainTableFactory(const PlainTableOptions& options) {
return new PlainTableFactory(options);
}

26
table/plain_table_factory.h
View File

@ -142,27 +142,24 @@ class PlainTableFactory : public TableFactory {
// huge_page_tlb_size determines whether to allocate hash indexes from huge
// page TLB and the page size if allocating from there. See comments of
// Arena::AllocateAligned() for details.
explicit PlainTableFactory(const PlainTableOptions& options =
PlainTableOptions())
: user_key_len_(options.user_key_len),
bloom_bits_per_key_(options.bloom_bits_per_key),
hash_table_ratio_(options.hash_table_ratio),
index_sparseness_(options.index_sparseness),
huge_page_tlb_size_(options.huge_page_tlb_size),
encoding_type_(options.encoding_type),
full_scan_mode_(options.full_scan_mode),
store_index_in_file_(options.store_index_in_file) {}
explicit PlainTableFactory(const PlainTableOptions& table_options =
PlainTableOptions())
: table_options_(table_options) {};
const char* Name() const override { return "PlainTable"; }
Status NewTableReader(const TableReaderOptions& table_reader_options,
unique_ptr<RandomAccessFileReader>&& file,
uint64_t file_size,
unique_ptr<TableReader>* table) const override;
TableBuilder* NewTableBuilder(
const TableBuilderOptions& table_builder_options,
uint32_t column_family_id, WritableFileWriter* file) const override;
std::string GetPrintableTableOptions() const override;
const PlainTableOptions& GetTableOptions() const;
static const char kValueTypeSeqId0 = 0xFF;
// Sanitizes the specified DB Options.
@ -172,14 +169,7 @@ class PlainTableFactory : public TableFactory {
}
private:
uint32_t user_key_len_;
int bloom_bits_per_key_;
double hash_table_ratio_;
size_t index_sparseness_;
size_t huge_page_tlb_size_;
EncodingType encoding_type_;
bool full_scan_mode_;
bool store_index_in_file_;
PlainTableOptions table_options_;
};
} // namespace rocksdb

145
util/options_helper.cc
View File

@ -20,6 +20,7 @@
#include "rocksdb/slice_transform.h"
#include "rocksdb/table.h"
#include "table/block_based_table_factory.h"
#include "table/plain_table_factory.h"
#include "util/logging.h"
#include "util/string_util.h"
@ -200,6 +201,31 @@ bool ParseBlockBasedTableIndexType(const std::string& type,
return true;
}
bool SerializeEncodingType(
const EncodingType& type, std::string* value) {
switch (type) {
case EncodingType::kPlain:
*value = "kPlain";
return true;
case EncodingType::kPrefix:
*value = "kPrefix";
return true;
default:
return false;
}
}
bool PraseEncodingType(const std::string& type, EncodingType* value) {
if (type == "kPlain") {
*value = EncodingType::kPlain;
} else if (type == "kPrefix") {
*value = EncodingType::kPrefix;
} else {
return false;
}
return true;
}
static std::unordered_map<std::string, ChecksumType> checksum_type_map = {
{"kNoChecksum", kNoChecksum}, {"kCRC32c", kCRC32c}, {"kxxHash", kxxHash}};
@ -448,6 +474,10 @@ bool ParseOptionHelper(char* opt_address, const OptionType& opt_type,
return ParseBlockBasedTableIndexType(
value,
reinterpret_cast<BlockBasedTableOptions::IndexType*>(opt_address));
case OptionType::kEncodingType:
return PraseEncodingType(
value,
reinterpret_cast<EncodingType*>(opt_address));
default:
return false;
}
@ -569,6 +599,9 @@ bool SerializeSingleOptionHelper(const char* opt_address,
*value = ptr->get() ? ptr->get()->Name() : kNullptrString;
break;
}
case OptionType::kEncodingType:
return SerializeEncodingType(
*reinterpret_cast<const EncodingType*>(opt_address), value);
default:
return false;
}
@ -774,9 +807,9 @@ Status StringToMap(const std::string& opts_str,
bool ParseColumnFamilyOption(const std::string& name,
const std::string& org_value,
ColumnFamilyOptions* new_options,
bool input_string_escaped = false) {
bool input_strings_escaped = false) {
const std::string& value =
input_string_escaped ? UnescapeOptionString(org_value) : org_value;
input_strings_escaped ? UnescapeOptionString(org_value) : org_value;
try {
if (name == "max_bytes_for_level_multiplier_additional") {
new_options->max_bytes_for_level_multiplier_additional.clear();
@ -807,6 +840,20 @@ bool ParseColumnFamilyOption(const std::string& name,
return false;
}
new_options->table_factory.reset(NewBlockBasedTableFactory(table_opt));
} else if (name == "plain_table_factory") {
// Nested options
PlainTableOptions table_opt, base_table_options;
auto plain_table_factory = dynamic_cast<PlainTableFactory*>(
new_options->table_factory.get());
if (plain_table_factory != nullptr) {
base_table_options = plain_table_factory->GetTableOptions();
}
Status table_opt_s = GetPlainTableOptionsFromString(
base_table_options, value, &table_opt);
if (!table_opt_s.ok()) {
return false;
}
new_options->table_factory.reset(NewPlainTableFactory(table_opt));
} else if (name == "compression_opts") {
size_t start = 0;
size_t end = value.find(':');
@ -991,9 +1038,9 @@ Status GetStringFromTableFactory(std::string* opts_str, const TableFactory* tf,
}
bool ParseDBOption(const std::string& name, const std::string& org_value,
DBOptions* new_options, bool input_string_escaped = false) {
DBOptions* new_options, bool input_strings_escaped = false) {
const std::string& value =
input_string_escaped ? UnescapeOptionString(org_value) : org_value;
input_strings_escaped ? UnescapeOptionString(org_value) : org_value;
try {
if (name == "rate_limiter_bytes_per_sec") {
new_options->rate_limiter.reset(
@ -1020,10 +1067,10 @@ bool ParseDBOption(const std::string& name, const std::string& org_value,
std::string ParseBlockBasedTableOption(const std::string& name,
const std::string& org_value,
BlockBasedTableOptions* new_options,
bool input_string_escaped = false) {
bool input_strings_escaped = false) {
const std::string& value =
input_string_escaped ? UnescapeOptionString(org_value) : org_value;
if (!input_string_escaped) {
input_strings_escaped ? UnescapeOptionString(org_value) : org_value;
if (!input_strings_escaped) {
// if the input string is not escaped, it means this function is
// invoked from SetOptions, which takes the old format.
if (name == "block_cache") {
@ -1064,6 +1111,24 @@ std::string ParseBlockBasedTableOption(const std::string& name,
return "";
}
std::string ParsePlainTableOptions(const std::string& name,
const std::string& org_value,
PlainTableOptions* new_option,
bool input_strings_escaped = false) {
const std::string& value =
input_strings_escaped ? UnescapeOptionString(org_value) : org_value;
const auto iter = plain_table_type_info.find(name);
if (iter == plain_table_type_info.end()) {
return "Unrecognized option";
}
const auto& opt_info = iter->second;
if (!ParseOptionHelper(reinterpret_cast<char*>(new_option) + opt_info.offset,
opt_info.type, value)) {
return "Invalid value";
}
return "";
}
Status GetBlockBasedTableOptionsFromMap(
const BlockBasedTableOptions& table_options,
const std::unordered_map<std::string, std::string>& opts_map,
@ -1105,46 +1170,41 @@ Status GetBlockBasedTableOptionsFromString(
Status GetPlainTableOptionsFromMap(
const PlainTableOptions& table_options,
const std::unordered_map<std::string, std::string>& opts_map,
PlainTableOptions* new_table_options) {
PlainTableOptions* new_table_options, bool input_strings_escaped) {
assert(new_table_options);
*new_table_options = table_options;
for (const auto& o : opts_map) {
try {
if (o.first == "user_key_len") {
new_table_options->user_key_len = ParseUint32(o.second);
} else if (o.first == "bloom_bits_per_key") {
new_table_options->bloom_bits_per_key = ParseInt(o.second);
} else if (o.first == "hash_table_ratio") {
new_table_options->hash_table_ratio = ParseDouble(o.second);
} else if (o.first == "index_sparseness") {
new_table_options->index_sparseness = ParseSizeT(o.second);
} else if (o.first == "huge_page_tlb_size") {
new_table_options->huge_page_tlb_size = ParseSizeT(o.second);
} else if (o.first == "encoding_type") {
if (o.second == "kPlain") {
new_table_options->encoding_type = kPlain;
} else if (o.second == "kPrefix") {
new_table_options->encoding_type = kPrefix;
} else {
throw std::invalid_argument("Unknown encoding_type: " + o.second);
}
} else if (o.first == "full_scan_mode") {
new_table_options->full_scan_mode = ParseBoolean(o.first, o.second);
} else if (o.first == "store_index_in_file") {
new_table_options->store_index_in_file =
ParseBoolean(o.first, o.second);
} else {
return Status::InvalidArgument("Unrecognized option: " + o.first);
auto error_message = ParsePlainTableOptions(
o.first, o.second, new_table_options, input_strings_escaped);
if (error_message != "") {
const auto iter = plain_table_type_info.find(o.first);
if (iter == plain_table_type_info.end() ||
!input_strings_escaped ||// !input_strings_escaped indicates
// the old API, where everything is
// parsable.
(iter->second.verification != OptionVerificationType::kByName &&
iter->second.verification != OptionVerificationType::kDeprecated)) {
return Status::InvalidArgument("Can't parse PlainTableOptions:",
o.first + " " + error_message);
}
} catch (std::exception& e) {
return Status::InvalidArgument("error parsing " + o.first + ":" +
std::string(e.what()));
}
}
return Status::OK();
}
Status GetPlainTableOptionsFromString(
const PlainTableOptions& table_options,
const std::string& opts_str,
PlainTableOptions* new_table_options) {
std::unordered_map<std::string, std::string> opts_map;
Status s = StringToMap(opts_str, &opts_map);
if (!s.ok()) {
return s;
}
return GetPlainTableOptionsFromMap(table_options, opts_map,
new_table_options);
}
Status GetColumnFamilyOptionsFromMap(
const ColumnFamilyOptions& base_options,
const std::unordered_map<std::string, std::string>& opts_map,
@ -1239,6 +1299,15 @@ Status GetTableFactoryFromMap(
}
table_factory->reset(new BlockBasedTableFactory(bbt_opt));
return Status::OK();
} else if (factory_name == PlainTableFactory().Name()) {
PlainTableOptions pt_opt;
s = GetPlainTableOptionsFromMap(PlainTableOptions(), opt_map,
&pt_opt, true);
if (!s.ok()) {
return s;
}
table_factory->reset(new PlainTableFactory(pt_opt));
return Status::OK();
}
// Return OK for not supported table factories as TableFactory
// Deserialization is optional.

29
util/options_helper.h
View File

@ -87,6 +87,7 @@ enum class OptionType {
kFilterPolicy,
kFlushBlockPolicyFactory,
kChecksumType,
kEncodingType,
kUnknown
};
@ -462,6 +463,34 @@ static std::unordered_map<std::string,
{"format_version",
{offsetof(struct BlockBasedTableOptions, format_version),
OptionType::kUInt32T, OptionVerificationType::kNormal}}};
static std::unordered_map<std::string,
OptionTypeInfo> plain_table_type_info = {
{"user_key_len",
{offsetof(struct PlainTableOptions, user_key_len),
OptionType::kUInt32T, OptionVerificationType::kNormal}},
{"bloom_bits_per_key",
{offsetof(struct PlainTableOptions, bloom_bits_per_key),
OptionType::kInt, OptionVerificationType::kNormal}},
{"hash_table_ratio",
{offsetof(struct PlainTableOptions, hash_table_ratio),
OptionType::kDouble, OptionVerificationType::kNormal}},
{"index_sparseness",
{offsetof(struct PlainTableOptions, index_sparseness),
OptionType::kSizeT, OptionVerificationType::kNormal}},
{"huge_page_tlb_size",
{offsetof(struct PlainTableOptions, huge_page_tlb_size),
OptionType::kSizeT, OptionVerificationType::kNormal}},
{"encoding_type",
{offsetof(struct PlainTableOptions, encoding_type),
OptionType::kEncodingType, OptionVerificationType::kByName}},
{"full_scan_mode",
{offsetof(struct PlainTableOptions, full_scan_mode),
OptionType::kBoolean, OptionVerificationType::kNormal}},
{"store_index_in_file",
{offsetof(struct PlainTableOptions, store_index_in_file),
OptionType::kBoolean, OptionVerificationType::kNormal}}};
} // namespace rocksdb
#endif // !ROCKSDB_LITE

52
util/options_test.cc
View File

@ -23,6 +23,7 @@
#include "rocksdb/table.h"
#include "rocksdb/utilities/leveldb_options.h"
#include "table/block_based_table_factory.h"
#include "table/plain_table_factory.h"
#include "util/options_helper.h"
#include "util/options_parser.h"
#include "util/random.h"
@ -590,6 +591,23 @@ TEST_F(OptionsTest, GetColumnFamilyOptionsFromStringTest) {
ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
"optimize_filters_for_hits=junk",
&new_cf_opt));
// Nested plain table options
// Emtpy
ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
"write_buffer_size=10;max_write_buffer_number=16;"
"plain_table_factory={};arena_block_size=1024",
&new_cf_opt));
ASSERT_TRUE(new_cf_opt.table_factory != nullptr);
ASSERT_EQ(new_cf_opt.table_factory->Name(), "PlainTable");
// Non-empty
ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
"write_buffer_size=10;max_write_buffer_number=16;"
"plain_table_factory={user_key_len=66;bloom_bits_per_key=20;};"
"arena_block_size=1024",
&new_cf_opt));
ASSERT_TRUE(new_cf_opt.table_factory != nullptr);
ASSERT_EQ(new_cf_opt.table_factory->Name(), "PlainTable");
}
#endif // !ROCKSDB_LITE
@ -648,6 +666,40 @@ TEST_F(OptionsTest, GetBlockBasedTableOptionsFromString) {
}
#endif // !ROCKSDB_LITE
#ifndef ROCKSDB_LITE // GetPlainTableOptionsFromString is not supported
TEST_F(OptionsTest, GetPlainTableOptionsFromString) {
PlainTableOptions table_opt;
PlainTableOptions new_opt;
// make sure default values are overwritten by something else
ASSERT_OK(GetPlainTableOptionsFromString(table_opt,
"user_key_len=66;bloom_bits_per_key=20;hash_table_ratio=0.5;"
"index_sparseness=8;huge_page_tlb_size=4;encoding_type=kPrefix;"
"full_scan_mode=true;store_index_in_file=true",
&new_opt));
ASSERT_EQ(new_opt.user_key_len, 66);
ASSERT_EQ(new_opt.bloom_bits_per_key, 20);
ASSERT_EQ(new_opt.hash_table_ratio, 0.5);
ASSERT_EQ(new_opt.index_sparseness, 8);
ASSERT_EQ(new_opt.huge_page_tlb_size, 4);
ASSERT_EQ(new_opt.encoding_type, EncodingType::kPrefix);
ASSERT_TRUE(new_opt.full_scan_mode);
ASSERT_TRUE(new_opt.store_index_in_file);
// unknown option
ASSERT_NOK(GetPlainTableOptionsFromString(table_opt,
"user_key_len=66;bloom_bits_per_key=20;hash_table_ratio=0.5;"
"bad_option=1",
&new_opt));
// unrecognized EncodingType
ASSERT_NOK(GetPlainTableOptionsFromString(table_opt,
"user_key_len=66;bloom_bits_per_key=20;hash_table_ratio=0.5;"
"encoding_type=kPrefixXX",
&new_opt));
}
#endif // !ROCKSDB_LITE
#ifndef ROCKSDB_LITE // GetOptionsFromString is not supported in RocksDB Lite
TEST_F(OptionsTest, GetOptionsFromStringTest) {
Options base_options, new_options;