andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Option type info functions (#9411)

Browse Source 
Summary:
Add methods to set the various functions (Parse, Serialize, Equals) to the OptionTypeInfo.  These methods simplify the number of constructors required for OptionTypeInfo and make the code a little clearer.

Add functions to the OptionTypeInfo for Prepare and Validate.  These methods allow types other than Configurable and Customizable to have Prepare and Validate logic.  These methods could be used by an option to guarantee that its settings were in a range or that a value was initialized.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/9411

Reviewed By: pdillinger

Differential Revision: D36174849

Pulled By: mrambacher

fbshipit-source-id: 72517d8c6bab4723788a4c1a9e16590bff870125
This commit is contained in:
mrambacher 2022-05-13 04:57:08 -07:00 committed by Facebook GitHub Bot
parent cdaa9576bb
commit bfc6a8ee4a
13 changed files with 550 additions and 326 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
HISTORY.md
View File

@ -15,6 +15,7 @@
* EXPERIMENTAL: Add new API AbortIO in file_system to abort the read requests submitted asynchronously.
* CompactionFilter::Decision has a new value: kRemoveWithSingleDelete. If CompactionFilter returns this decision, then CompactionIterator will use `SingleDelete` to mark a key as removed.
* Renamed CompactionFilter::Decision::kRemoveWithSingleDelete to kPurge since the latter sounds more general and hides the implementation details of how compaction iterator handles keys.
* Added ability to specify functions for Prepare and Validate to OptionsTypeInfo. Added methods to OptionTypeInfo to set the functions via an API. These methods are intended for RocksDB plugin developers for configuration management.
### Bug Fixes
* RocksDB calls FileSystem::Poll API during FilePrefetchBuffer destruction which impacts performance as it waits for read requets completion which is not needed anymore. Calling FileSystem::AbortIO to abort those requests instead fixes that performance issue.

1
db/compaction/compaction_job.cc
View File

@ -2952,6 +2952,7 @@ static std::unordered_map<std::string, OptionTypeInfo> cs_result_type_info = {
const void* addr1, const void* addr2, std::string* mismatch) {
const auto status1 = static_cast<const Status*>(addr1);
const auto status2 = static_cast<const Status*>(addr2);
StatusSerializationAdapter adatper1(*status1);
StatusSerializationAdapter adapter2(*status2);
return OptionTypeInfo::TypesAreEqual(opts, status_adapter_type_info,

93
env/composite_env.cc vendored
View File

@ -5,6 +5,7 @@
//
#include "env/composite_env_wrapper.h"
#include "rocksdb/utilities/options_type.h"
#include "util/string_util.h"
namespace ROCKSDB_NAMESPACE {
namespace {
@ -382,19 +383,49 @@ Status CompositeEnv::NewDirectory(const std::string& name,
}
namespace {
static std::unordered_map<std::string, OptionTypeInfo>
composite_env_wrapper_type_info = {
static std::unordered_map<std::string, OptionTypeInfo> env_wrapper_type_info = {
#ifndef ROCKSDB_LITE
{"target",
{0, OptionType::kCustomizable, OptionVerificationType::kByName,
OptionTypeFlags::kDontSerialize | OptionTypeFlags::kRawPointer,
[](const ConfigOptions& opts, const std::string& /*name*/,
const std::string& value, void* addr) {
OptionTypeInfo(0, OptionType::kUnknown, OptionVerificationType::kByName,
OptionTypeFlags::kDontSerialize)
.SetParseFunc([](const ConfigOptions& opts,
const std::string& /*name*/, const std::string& value,
void* addr) {
auto target = static_cast<EnvWrapper::Target*>(addr);
return Env::CreateFromString(opts, value, &(target->env),
&(target->guard));
},
nullptr, nullptr}},
})
.SetEqualsFunc([](const ConfigOptions& opts,
const std::string& /*name*/, const void* addr1,
const void* addr2, std::string* mismatch) {
const auto target1 = static_cast<const EnvWrapper::Target*>(addr1);
const auto target2 = static_cast<const EnvWrapper::Target*>(addr2);
if (target1->env != nullptr) {
return target1->env->AreEquivalent(opts, target2->env, mismatch);
} else {
return (target2->env == nullptr);
}
})
.SetPrepareFunc([](const ConfigOptions& opts,
const std::string& /*name*/, void* addr) {
auto target = static_cast<EnvWrapper::Target*>(addr);
if (target->guard.get() != nullptr) {
target->env = target->guard.get();
} else if (target->env == nullptr) {
target->env = Env::Default();
}
return target->env->PrepareOptions(opts);
})
.SetValidateFunc([](const DBOptions& db_opts,
const ColumnFamilyOptions& cf_opts,
const std::string& /*name*/, const void* addr) {
const auto target = static_cast<const EnvWrapper::Target*>(addr);
if (target->env == nullptr) {
return Status::InvalidArgument("Target Env not specified");
} else {
return target->env->ValidateOptions(db_opts, cf_opts);
}
})},
#endif // ROCKSDB_LITE
};
static std::unordered_map<std::string, OptionTypeInfo>
@ -425,7 +456,7 @@ CompositeEnvWrapper::CompositeEnvWrapper(Env* env,
const std::shared_ptr<FileSystem>& fs,
const std::shared_ptr<SystemClock>& sc)
: CompositeEnv(fs, sc), target_(env) {
RegisterOptions("", &target_, &composite_env_wrapper_type_info);
RegisterOptions("", &target_, &env_wrapper_type_info);
RegisterOptions("", &file_system_, &composite_fs_wrapper_type_info);
RegisterOptions("", &system_clock_, &composite_clock_wrapper_type_info);
}
@ -434,7 +465,7 @@ CompositeEnvWrapper::CompositeEnvWrapper(const std::shared_ptr<Env>& env,
const std::shared_ptr<FileSystem>& fs,
const std::shared_ptr<SystemClock>& sc)
: CompositeEnv(fs, sc), target_(env) {
RegisterOptions("", &target_, &composite_env_wrapper_type_info);
RegisterOptions("", &target_, &env_wrapper_type_info);
RegisterOptions("", &file_system_, &composite_fs_wrapper_type_info);
RegisterOptions("", &system_clock_, &composite_clock_wrapper_type_info);
}
@ -461,4 +492,46 @@ std::string CompositeEnvWrapper::SerializeOptions(
return options;
}
#endif // ROCKSDB_LITE
EnvWrapper::EnvWrapper(Env* t) : target_(t) {
RegisterOptions("", &target_, &env_wrapper_type_info);
}
EnvWrapper::EnvWrapper(std::unique_ptr<Env>&& t) : target_(std::move(t)) {
RegisterOptions("", &target_, &env_wrapper_type_info);
}
EnvWrapper::EnvWrapper(const std::shared_ptr<Env>& t) : target_(t) {
RegisterOptions("", &target_, &env_wrapper_type_info);
}
EnvWrapper::~EnvWrapper() {}
Status EnvWrapper::PrepareOptions(const ConfigOptions& options) {
target_.Prepare();
return Env::PrepareOptions(options);
}
#ifndef ROCKSDB_LITE
std::string EnvWrapper::SerializeOptions(const ConfigOptions& config_options,
const std::string& header) const {
auto parent = Env::SerializeOptions(config_options, "");
if (config_options.IsShallow() || target_.env == nullptr ||
target_.env == Env::Default()) {
return parent;
} else {
std::string result = header;
if (!StartsWith(parent, OptionTypeInfo::kIdPropName())) {
result.append(OptionTypeInfo::kIdPropName()).append("=");
}
result.append(parent);
if (!EndsWith(result, config_options.delimiter)) {
result.append(config_options.delimiter);
}
result.append("target=").append(target_.env->ToString(config_options));
return result;
}
}
#endif // ROCKSDB_LITE
} // namespace ROCKSDB_NAMESPACE

60
env/env.cc vendored
View File

@ -26,7 +26,6 @@
#include "rocksdb/utilities/object_registry.h"
#include "rocksdb/utilities/options_type.h"
#include "util/autovector.h"
#include "util/string_util.h"
namespace ROCKSDB_NAMESPACE {
namespace {
@ -1084,65 +1083,6 @@ Status ReadFileToString(Env* env, const std::string& fname, std::string* data) {
return ReadFileToString(fs.get(), fname, data);
}
namespace {
static std::unordered_map<std::string, OptionTypeInfo> env_wrapper_type_info = {
#ifndef ROCKSDB_LITE
{"target",
{0, OptionType::kCustomizable, OptionVerificationType::kByName,
OptionTypeFlags::kDontSerialize | OptionTypeFlags::kRawPointer,
[](const ConfigOptions& opts, const std::string& /*name*/,
const std::string& value, void* addr) {
EnvWrapper::Target* target = static_cast<EnvWrapper::Target*>(addr);
return Env::CreateFromString(opts, value, &(target->env),
&(target->guard));
},
nullptr, nullptr}},
#endif // ROCKSDB_LITE
};
} // namespace
EnvWrapper::EnvWrapper(Env* t) : target_(t) {
RegisterOptions("", &target_, &env_wrapper_type_info);
}
EnvWrapper::EnvWrapper(std::unique_ptr<Env>&& t) : target_(std::move(t)) {
RegisterOptions("", &target_, &env_wrapper_type_info);
}
EnvWrapper::EnvWrapper(const std::shared_ptr<Env>& t) : target_(t) {
RegisterOptions("", &target_, &env_wrapper_type_info);
}
EnvWrapper::~EnvWrapper() {
}
Status EnvWrapper::PrepareOptions(const ConfigOptions& options) {
target_.Prepare();
return Env::PrepareOptions(options);
}
#ifndef ROCKSDB_LITE
std::string EnvWrapper::SerializeOptions(const ConfigOptions& config_options,
const std::string& header) const {
auto parent = Env::SerializeOptions(config_options, "");
if (config_options.IsShallow() || target_.env == nullptr ||
target_.env == Env::Default()) {
return parent;
} else {
std::string result = header;
if (!StartsWith(parent, OptionTypeInfo::kIdPropName())) {
result.append(OptionTypeInfo::kIdPropName()).append("=");
}
result.append(parent);
if (!EndsWith(result, config_options.delimiter)) {
result.append(config_options.delimiter);
}
result.append("target=").append(target_.env->ToString(config_options));
return result;
}
}
#endif // ROCKSDB_LITE
namespace { // anonymous namespace
void AssignEnvOptions(EnvOptions* env_options, const DBOptions& options) {

21
env/env_test.cc vendored
View File

@ -44,6 +44,7 @@
#include "env/unique_id_gen.h"
#include "logging/log_buffer.h"
#include "logging/logging.h"
#include "options/options_helper.h"
#include "port/malloc.h"
#include "port/port.h"
#include "port/stack_trace.h"
@ -2937,7 +2938,7 @@ TEST_F(EnvTest, FailureToCreateLockFile) {
ASSERT_OK(DestroyDir(env, dir));
}
TEST_F(EnvTest, CreateDefaultEnv) {
TEST_F(CreateEnvTest, CreateDefaultEnv) {
ConfigOptions options;
options.ignore_unsupported_options = false;
@ -2989,7 +2990,7 @@ class WrappedEnv : public EnvWrapper {
}
};
} // namespace
TEST_F(EnvTest, CreateMockEnv) {
TEST_F(CreateEnvTest, CreateMockEnv) {
ConfigOptions options;
options.ignore_unsupported_options = false;
WrappedEnv::Register(*(options.registry->AddLibrary("test")), "");
@ -3017,7 +3018,7 @@ TEST_F(EnvTest, CreateMockEnv) {
opt_str = copy->ToString(options);
}
TEST_F(EnvTest, CreateWrappedEnv) {
TEST_F(CreateEnvTest, CreateWrappedEnv) {
ConfigOptions options;
options.ignore_unsupported_options = false;
WrappedEnv::Register(*(options.registry->AddLibrary("test")), "");
@ -3054,7 +3055,7 @@ TEST_F(EnvTest, CreateWrappedEnv) {
ASSERT_TRUE(guard->AreEquivalent(options, copy.get(), &mismatch));
}
TEST_F(EnvTest, CreateCompositeEnv) {
TEST_F(CreateEnvTest, CreateCompositeEnv) {
ConfigOptions options;
options.ignore_unsupported_options = false;
std::shared_ptr<Env> guard, copy;
@ -3109,6 +3110,18 @@ TEST_F(EnvTest, CreateCompositeEnv) {
ASSERT_NE(env, nullptr);
ASSERT_NE(env, Env::Default());
ASSERT_TRUE(guard->AreEquivalent(options, copy.get(), &mismatch));
guard.reset(new CompositeEnvWrapper(nullptr, timed_fs, clock));
ColumnFamilyOptions cf_opts;
DBOptions db_opts;
db_opts.env = guard.get();
auto comp = db_opts.env->CheckedCast<CompositeEnvWrapper>();
ASSERT_NE(comp, nullptr);
ASSERT_EQ(comp->Inner(), nullptr);
ASSERT_NOK(ValidateOptions(db_opts, cf_opts));
ASSERT_OK(db_opts.env->PrepareOptions(options));
ASSERT_NE(comp->Inner(), nullptr);
ASSERT_OK(ValidateOptions(db_opts, cf_opts));
}
#endif // ROCKSDB_LITE

9
include/rocksdb/utilities/customizable_util.h
View File

@ -2,6 +2,15 @@
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
//
// The methods in this file are used to instantiate new Customizable
// instances of objects. These methods are most typically used by
// the "CreateFromString" method of a customizable class.
// If not developing a new Type of customizable class, you probably
// do not need the methods in this file.
//
// See https://github.com/facebook/rocksdb/wiki/RocksDB-Configurable-Objects
// for more information on how to develop and use customizable objects
#pragma once
#include <functional>

276
include/rocksdb/utilities/options_type.h
View File

@ -2,6 +2,15 @@
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
//
// The OptionTypeInfo and related classes provide a framework for
// configuring and validating RocksDB classes via the Options framework.
// This file is part of the public API to allow developers who wish to
// write their own extensions and plugins to take use the Options
// framework in their custom implementations.
//
// See https://github.com/facebook/rocksdb/wiki/RocksDB-Configurable-Objects
// for more information on how to develop and use custom extensions
#pragma once
@ -15,6 +24,8 @@
namespace ROCKSDB_NAMESPACE {
class OptionTypeInfo;
struct ColumnFamilyOptions;
struct DBOptions;
// The underlying "class/type" of the option.
// This enum is used to determine how the option should
@ -196,6 +207,16 @@ using EqualsFunc = std::function<bool(
const ConfigOptions& /*opts*/, const std::string& /*name*/,
const void* /*addr1*/, const void* /*addr2*/, std::string* mismatch)>;
// Function for preparing/initializing an option.
using PrepareFunc =
std::function<Status(const ConfigOptions& /*opts*/,
const std::string& /*name*/, void* /*addr*/)>;
// Function for validating an option.
using ValidateFunc = std::function<Status(
const DBOptions& /*db_opts*/, const ColumnFamilyOptions& /*cf_opts*/,
const std::string& /*name*/, const void* /*addr*/)>;
// A struct for storing constant option information such as option name,
// option type, and offset.
class OptionTypeInfo {
@ -259,8 +280,9 @@ class OptionTypeInfo {
static OptionTypeInfo Enum(
int offset, const std::unordered_map<std::string, T>* const map,
OptionTypeFlags flags = OptionTypeFlags::kNone) {
return OptionTypeInfo(
offset, OptionType::kEnum, OptionVerificationType::kNormal, flags,
OptionTypeInfo info(offset, OptionType::kEnum,
OptionVerificationType::kNormal, flags);
info.SetParseFunc(
// Uses the map argument to convert the input string into
// its corresponding enum value. If value is found in the map,
// addr is updated to the corresponding map entry.
@ -275,7 +297,8 @@ class OptionTypeInfo {
} else {
return Status::InvalidArgument("No mapping for enum ", name);
}
},
});
info.SetSerializeFunc(
// Uses the map argument to convert the input enum into
// its corresponding string value. If enum value is found in the map,
// value is updated to the corresponding string value in the map.
@ -291,7 +314,8 @@ class OptionTypeInfo {
} else {
return Status::InvalidArgument("No mapping for enum ", name);
}
},
});
info.SetEqualsFunc(
// Casts addr1 and addr2 to the enum type and returns true if
// they are equal, false otherwise.
[](const ConfigOptions&, const std::string&, const void* addr1,
@ -299,6 +323,7 @@ class OptionTypeInfo {
return (*static_cast<const T*>(addr1) ==
*static_cast<const T*>(addr2));
});
return info;
} // End OptionTypeInfo::Enum
// Creates an OptionTypeInfo for a Struct type. Structs have a
@ -327,21 +352,23 @@ class OptionTypeInfo {
const std::string& struct_name,
const std::unordered_map<std::string, OptionTypeInfo>* struct_map,
int offset, OptionVerificationType verification, OptionTypeFlags flags) {
return OptionTypeInfo(
offset, OptionType::kStruct, verification, flags,
OptionTypeInfo info(offset, OptionType::kStruct, verification, flags);
info.SetParseFunc(
// Parses the struct and updates the fields at addr
[struct_name, struct_map](const ConfigOptions& opts,
const std::string& name,
const std::string& value, void* addr) {
return ParseStruct(opts, struct_name, struct_map, name, value, addr);
},
});
info.SetSerializeFunc(
// Serializes the struct options into value
[struct_name, struct_map](const ConfigOptions& opts,
const std::string& name, const void* addr,
std::string* value) {
return SerializeStruct(opts, struct_name, struct_map, name, addr,
value);
},
});
info.SetEqualsFunc(
// Compares the struct fields of addr1 and addr2 for equality
[struct_name, struct_map](const ConfigOptions& opts,
const std::string& name, const void* addr1,
@ -349,26 +376,16 @@ class OptionTypeInfo {
return StructsAreEqual(opts, struct_name, struct_map, name, addr1,
addr2, mismatch);
});
return info;
}
static OptionTypeInfo Struct(
const std::string& struct_name,
const std::unordered_map<std::string, OptionTypeInfo>* struct_map,
int offset, OptionVerificationType verification, OptionTypeFlags flags,
const ParseFunc& parse_func) {
return OptionTypeInfo(
offset, OptionType::kStruct, verification, flags, parse_func,
[struct_name, struct_map](const ConfigOptions& opts,
const std::string& name, const void* addr,
std::string* value) {
return SerializeStruct(opts, struct_name, struct_map, name, addr,
value);
},
[struct_name, struct_map](const ConfigOptions& opts,
const std::string& name, const void* addr1,
const void* addr2, std::string* mismatch) {
return StructsAreEqual(opts, struct_name, struct_map, name, addr1,
addr2, mismatch);
});
OptionTypeInfo info(
Struct(struct_name, struct_map, offset, verification, flags));
return info.SetParseFunc(parse_func);
}
template <typename T>
@ -377,30 +394,28 @@ class OptionTypeInfo {
OptionTypeFlags _flags,
const OptionTypeInfo& elem_info,
char separator = ':') {
return OptionTypeInfo(
_offset, OptionType::kVector, _verification, _flags,
[elem_info, separator](const ConfigOptions& opts,
const std::string& name,
OptionTypeInfo info(_offset, OptionType::kVector, _verification, _flags);
info.SetParseFunc([elem_info, separator](
const ConfigOptions& opts, const std::string& name,
const std::string& value, void* addr) {
auto result = static_cast<std::vector<T>*>(addr);
return ParseVector<T>(opts, elem_info, separator, name, value,
result);
},
[elem_info, separator](const ConfigOptions& opts,
const std::string& name, const void* addr,
return ParseVector<T>(opts, elem_info, separator, name, value, result);
});
info.SetSerializeFunc([elem_info, separator](const ConfigOptions& opts,
const std::string& name,
const void* addr,
std::string* value) {
const auto& vec = *(static_cast<const std::vector<T>*>(addr));
return SerializeVector<T>(opts, elem_info, separator, name, vec,
value);
},
[elem_info](const ConfigOptions& opts, const std::string& name,
const void* addr1, const void* addr2,
std::string* mismatch) {
return SerializeVector<T>(opts, elem_info, separator, name, vec, value);
});
info.SetEqualsFunc([elem_info](const ConfigOptions& opts,
const std::string& name, const void* addr1,
const void* addr2, std::string* mismatch) {
const auto& vec1 = *(static_cast<const std::vector<T>*>(addr1));
const auto& vec2 = *(static_cast<const std::vector<T>*>(addr2));
return VectorsAreEqual<T>(opts, elem_info, name, vec1, vec2,
mismatch);
return VectorsAreEqual<T>(opts, elem_info, name, vec1, vec2, mismatch);
});
return info;
}
// Create a new std::shared_ptr<Customizable> OptionTypeInfo
@ -416,19 +431,10 @@ class OptionTypeInfo {
static OptionTypeInfo AsCustomSharedPtr(int offset,
OptionVerificationType ovt,
OptionTypeFlags flags) {
return AsCustomSharedPtr<T>(offset, ovt, flags, nullptr, nullptr);
}
template <typename T>
static OptionTypeInfo AsCustomSharedPtr(int offset,
OptionVerificationType ovt,
OptionTypeFlags flags,
const SerializeFunc& serialize_func,
const EqualsFunc& equals_func) {
return OptionTypeInfo(
offset, OptionType::kCustomizable, ovt,
flags | OptionTypeFlags::kShared,
[](const ConfigOptions& opts, const std::string& name,
OptionTypeInfo info(offset, OptionType::kCustomizable, ovt,
flags | OptionTypeFlags::kShared);
return info.SetParseFunc([](const ConfigOptions& opts,
const std::string& name,
const std::string& value, void* addr) {
auto* shared = static_cast<std::shared_ptr<T>*>(addr);
if (name == kIdPropName() && value.empty()) {
@ -437,8 +443,19 @@ class OptionTypeInfo {
} else {
return T::CreateFromString(opts, value, shared);
}
},
serialize_func, equals_func);
});
}
template <typename T>
static OptionTypeInfo AsCustomSharedPtr(int offset,
OptionVerificationType ovt,
OptionTypeFlags flags,
const SerializeFunc& serialize_func,
const EqualsFunc& equals_func) {
OptionTypeInfo info(AsCustomSharedPtr<T>(offset, ovt, flags));
info.SetSerializeFunc(serialize_func);
info.SetEqualsFunc(equals_func);
return info;
}
// Create a new std::unique_ptr<Customizable> OptionTypeInfo
@ -454,19 +471,10 @@ class OptionTypeInfo {
static OptionTypeInfo AsCustomUniquePtr(int offset,
OptionVerificationType ovt,
OptionTypeFlags flags) {
return AsCustomUniquePtr<T>(offset, ovt, flags, nullptr, nullptr);
}
template <typename T>
static OptionTypeInfo AsCustomUniquePtr(int offset,
OptionVerificationType ovt,
OptionTypeFlags flags,
const SerializeFunc& serialize_func,
const EqualsFunc& equals_func) {
return OptionTypeInfo(
offset, OptionType::kCustomizable, ovt,
flags | OptionTypeFlags::kUnique,
[](const ConfigOptions& opts, const std::string& name,
OptionTypeInfo info(offset, OptionType::kCustomizable, ovt,
flags | OptionTypeFlags::kUnique);
return info.SetParseFunc([](const ConfigOptions& opts,
const std::string& name,
const std::string& value, void* addr) {
auto* unique = static_cast<std::unique_ptr<T>*>(addr);
if (name == kIdPropName() && value.empty()) {
@ -475,8 +483,19 @@ class OptionTypeInfo {
} else {
return T::CreateFromString(opts, value, unique);
}
},
serialize_func, equals_func);
});
}
template <typename T>
static OptionTypeInfo AsCustomUniquePtr(int offset,
OptionVerificationType ovt,
OptionTypeFlags flags,
const SerializeFunc& serialize_func,
const EqualsFunc& equals_func) {
OptionTypeInfo info(AsCustomUniquePtr<T>(offset, ovt, flags));
info.SetSerializeFunc(serialize_func);
info.SetEqualsFunc(equals_func);
return info;
}
// Create a new Customizable* OptionTypeInfo
@ -491,18 +510,10 @@ class OptionTypeInfo {
template <typename T>
static OptionTypeInfo AsCustomRawPtr(int offset, OptionVerificationType ovt,
OptionTypeFlags flags) {
return AsCustomRawPtr<T>(offset, ovt, flags, nullptr, nullptr);
}
template <typename T>
static OptionTypeInfo AsCustomRawPtr(int offset, OptionVerificationType ovt,
OptionTypeFlags flags,
const SerializeFunc& serialize_func,
const EqualsFunc& equals_func) {
return OptionTypeInfo(
offset, OptionType::kCustomizable, ovt,
flags | OptionTypeFlags::kRawPointer,
[](const ConfigOptions& opts, const std::string& name,
OptionTypeInfo info(offset, OptionType::kCustomizable, ovt,
flags | OptionTypeFlags::kRawPointer);
return info.SetParseFunc([](const ConfigOptions& opts,
const std::string& name,
const std::string& value, void* addr) {
auto** pointer = static_cast<T**>(addr);
if (name == kIdPropName() && value.empty()) {
@ -511,8 +522,42 @@ class OptionTypeInfo {
} else {
return T::CreateFromString(opts, value, pointer);
}
},
serialize_func, equals_func);
});
}
template <typename T>
static OptionTypeInfo AsCustomRawPtr(int offset, OptionVerificationType ovt,
OptionTypeFlags flags,
const SerializeFunc& serialize_func,
const EqualsFunc& equals_func) {
OptionTypeInfo info(AsCustomRawPtr<T>(offset, ovt, flags));
info.SetSerializeFunc(serialize_func);
info.SetEqualsFunc(equals_func);
return info;
}
OptionTypeInfo& SetParseFunc(const ParseFunc& f) {
parse_func_ = f;
return *this;
}
OptionTypeInfo& SetSerializeFunc(const SerializeFunc& f) {
serialize_func_ = f;
return *this;
}
OptionTypeInfo& SetEqualsFunc(const EqualsFunc& f) {
equals_func_ = f;
return *this;
}
OptionTypeInfo& SetPrepareFunc(const PrepareFunc& f) {
prepare_func_ = f;
return *this;
}
OptionTypeInfo& SetValidateFunc(const ValidateFunc& f) {
validate_func_ = f;
return *this;
}
bool IsEnabled(OptionTypeFlags otf) const { return (flags_ & otf) == otf; }
@ -569,6 +614,24 @@ class OptionTypeInfo {
}
}
bool ShouldPrepare() const {
if (IsDeprecated() || IsAlias()) {
return false;
} else if (IsEnabled(OptionTypeFlags::kDontPrepare)) {
return false;
} else {
return (prepare_func_ != nullptr || IsConfigurable());
}
}
bool ShouldValidate() const {
if (IsDeprecated() || IsAlias()) {
return false;
} else {
return (validate_func_ != nullptr || IsConfigurable());
}
}
// Returns true if the option is allowed to be null.
// Options can be null if the verification type is allow from null
// or if the flags specify allow null.
@ -599,6 +662,26 @@ class OptionTypeInfo {
bool IsCustomizable() const { return (type_ == OptionType::kCustomizable); }
inline const void* GetOffset(const void* base) const {
return static_cast<const char*>(base) + offset_;
}
inline void* GetOffset(void* base) const {
return static_cast<char*>(base) + offset_;
}
template <typename T>
const T* GetOffsetAs(const void* base) const {
const void* addr = GetOffset(base);
return static_cast<const T*>(addr);
}
template <typename T>
T* GetOffsetAs(void* base) const {
void* addr = GetOffset(base);
return static_cast<T*>(addr);
}
// Returns the underlying pointer for the type at base_addr
// The value returned is the underlying "raw" pointer, offset from base.
template <typename T>
@ -606,20 +689,17 @@ class OptionTypeInfo {
if (base_addr == nullptr) {
return nullptr;
}
const void* opt_addr = static_cast<const char*>(base_addr) + offset_;
if (IsUniquePtr()) {
const std::unique_ptr<T>* ptr =
static_cast<const std::unique_ptr<T>*>(opt_addr);
const auto ptr = GetOffsetAs<std::unique_ptr<T>>(base_addr);
return ptr->get();
} else if (IsSharedPtr()) {
const std::shared_ptr<T>* ptr =
static_cast<const std::shared_ptr<T>*>(opt_addr);
const auto ptr = GetOffsetAs<std::shared_ptr<T>>(base_addr);
return ptr->get();
} else if (IsRawPtr()) {
const T* const* ptr = static_cast<const T* const*>(opt_addr);
const T* const* ptr = GetOffsetAs<T* const>(base_addr);
return *ptr;
} else {
return static_cast<const T*>(opt_addr);
return GetOffsetAs<T>(base_addr);
}
}
@ -630,18 +710,17 @@ class OptionTypeInfo {
if (base_addr == nullptr) {
return nullptr;
}
void* opt_addr = static_cast<char*>(base_addr) + offset_;
if (IsUniquePtr()) {
std::unique_ptr<T>* ptr = static_cast<std::unique_ptr<T>*>(opt_addr);
auto ptr = GetOffsetAs<std::unique_ptr<T>>(base_addr);
return ptr->get();
} else if (IsSharedPtr()) {
std::shared_ptr<T>* ptr = static_cast<std::shared_ptr<T>*>(opt_addr);
auto ptr = GetOffsetAs<std::shared_ptr<T>>(base_addr);
return ptr->get();
} else if (IsRawPtr()) {
T** ptr = static_cast<T**>(opt_addr);
auto ptr = GetOffsetAs<T*>(base_addr);
return *ptr;
} else {
return static_cast<T*>(opt_addr);
return GetOffsetAs<T>(base_addr);
}
}
@ -675,6 +754,11 @@ class OptionTypeInfo {
const std::string& opt_name, const void* const this_ptr,
const std::string& that_value) const;
Status Prepare(const ConfigOptions& config_options, const std::string& name,
void* opt_ptr) const;
Status Validate(const DBOptions& db_opts, const ColumnFamilyOptions& cf_opts,
const std::string& name, const void* opt_ptr) const;
// Parses the input opts_map according to the type_map for the opt_addr
// For each name-value pair in opts_map, find the corresponding name in
// type_map If the name is found:
@ -802,6 +886,8 @@ class OptionTypeInfo {
// The optional function to match two option values
EqualsFunc equals_func_;
PrepareFunc prepare_func_;
ValidateFunc validate_func_;
OptionType type_;
OptionVerificationType verification_;
OptionTypeFlags flags_;

25
options/cf_options.cc
View File

@ -364,9 +364,10 @@ static std::unordered_map<std::string, OptionTypeInfo>
OptionTypeInfo::Struct(
"compaction_options_fifo", &fifo_compaction_options_type_info,
offsetof(struct MutableCFOptions, compaction_options_fifo),
OptionVerificationType::kNormal, OptionTypeFlags::kMutable,
[](const ConfigOptions& opts, const std::string& name,
const std::string& value, void* addr) {
OptionVerificationType::kNormal, OptionTypeFlags::kMutable)
.SetParseFunc([](const ConfigOptions& opts,
const std::string& name, const std::string& value,
void* addr) {
// This is to handle backward compatibility, where
// compaction_options_fifo could be assigned a single scalar
// value, say, like "23", which would be assigned to
@ -556,16 +557,17 @@ static std::unordered_map<std::string, OptionTypeInfo>
{"comparator",
OptionTypeInfo::AsCustomRawPtr<const Comparator>(
offsetof(struct ImmutableCFOptions, user_comparator),
OptionVerificationType::kByName, OptionTypeFlags::kCompareLoose,
OptionVerificationType::kByName, OptionTypeFlags::kCompareLoose)
.SetSerializeFunc(
// Serializes a Comparator
[](const ConfigOptions& opts, const std::string&, const void* addr,
std::string* value) {
[](const ConfigOptions& opts, const std::string&,
const void* addr, std::string* value) {
// it's a const pointer of const Comparator*
const auto* ptr = static_cast<const Comparator* const*>(addr);
const auto* ptr =
static_cast<const Comparator* const*>(addr);
// Since the user-specified comparator will be wrapped by
// InternalKeyComparator, we should persist the user-specified
// one instead of InternalKeyComparator.
// InternalKeyComparator, we should persist the
// user-specified one instead of InternalKeyComparator.
if (*ptr == nullptr) {
*value = kNullptrString;
} else if (opts.mutable_options_only) {
@ -578,8 +580,7 @@ static std::unordered_map<std::string, OptionTypeInfo>
*value = root_comp->ToString(opts);
}
return Status::OK();
},
/* Use the default match function*/ nullptr)},
})},
{"memtable_insert_with_hint_prefix_extractor",
OptionTypeInfo::AsCustomSharedPtr<const SliceTransform>(
offsetof(struct ImmutableCFOptions,

28
options/configurable.cc
View File

@ -46,17 +46,8 @@ Status Configurable::PrepareOptions(const ConfigOptions& opts) {
if (opt_iter.type_map != nullptr) {
for (auto map_iter : *(opt_iter.type_map)) {
auto& opt_info = map_iter.second;
if (!opt_info.IsDeprecated() && !opt_info.IsAlias() &&
opt_info.IsConfigurable()) {
if (!opt_info.IsEnabled(OptionTypeFlags::kDontPrepare)) {
Configurable* config =
opt_info.AsRawPointer<Configurable>(opt_iter.opt_ptr);
if (config != nullptr) {
status = config->PrepareOptions(opts);
} else if (!opt_info.CanBeNull()) {
status = Status::NotFound("Missing configurable object",
map_iter.first);
}
if (opt_info.ShouldPrepare()) {
status = opt_info.Prepare(opts, map_iter.first, opt_iter.opt_ptr);
if (!status.ok()) {
return status;
}
@ -64,7 +55,6 @@ Status Configurable::PrepareOptions(const ConfigOptions& opts) {
}
}
}
}
#else
(void)opts;
#endif // ROCKSDB_LITE
@ -79,16 +69,9 @@ Status Configurable::ValidateOptions(const DBOptions& db_opts,
if (opt_iter.type_map != nullptr) {
for (auto map_iter : *(opt_iter.type_map)) {
auto& opt_info = map_iter.second;
if (!opt_info.IsDeprecated() && !opt_info.IsAlias()) {
if (opt_info.IsConfigurable()) {
const Configurable* config =
opt_info.AsRawPointer<Configurable>(opt_iter.opt_ptr);
if (config != nullptr) {
status = config->ValidateOptions(db_opts, cf_opts);
} else if (!opt_info.CanBeNull()) {
status = Status::NotFound("Missing configurable object",
map_iter.first);
}
if (opt_info.ShouldValidate()) {
status = opt_info.Validate(db_opts, cf_opts, map_iter.first,
opt_iter.opt_ptr);
if (!status.ok()) {
return status;
}
@ -96,7 +79,6 @@ Status Configurable::ValidateOptions(const DBOptions& db_opts,
}
}
}
}
#else
(void)db_opts;
(void)cf_opts;

4
options/customizable.cc
View File

@ -76,7 +76,9 @@ bool Customizable::AreEquivalent(const ConfigOptions& config_options,
if (config_options.sanity_level > ConfigOptions::kSanityLevelNone &&
this != other) {
const Customizable* custom = reinterpret_cast<const Customizable*>(other);
if (GetId() != custom->GetId()) {
if (custom == nullptr) { // Cast failed
return false;
} else if (GetId() != custom->GetId()) {
*mismatch = OptionTypeInfo::kIdPropName();
return false;
} else if (config_options.sanity_level >

26
options/db_options.cc
View File

@ -439,13 +439,15 @@ static std::unordered_map<std::string, OptionTypeInfo>
static_cast<int64_t>(ParseUint64(value))));
return Status::OK();
}}},
{"env",
{offsetof(struct ImmutableDBOptions, env), OptionType::kUnknown,
{"env", //**TODO: Should this be kCustomizable?
OptionTypeInfo(
offsetof(struct ImmutableDBOptions, env), OptionType::kUnknown,
OptionVerificationType::kNormal,
(OptionTypeFlags::kDontSerialize | OptionTypeFlags::kCompareNever),
// Parse the input value as an Env
[](const ConfigOptions& opts, const std::string& /*name*/,
(OptionTypeFlags::kDontSerialize | OptionTypeFlags::kCompareNever))
.SetParseFunc([](const ConfigOptions& opts,
const std::string& /*name*/,
const std::string& value, void* addr) {
// Parse the input value as an Env
auto old_env = static_cast<Env**>(addr); // Get the old value
Env* new_env = *old_env; // Set new to old
Status s = Env::CreateFromString(opts, value,
@ -454,7 +456,19 @@ static std::unordered_map<std::string, OptionTypeInfo>
*old_env = new_env; // Update the old one
}
return s;
}}},
})
.SetPrepareFunc([](const ConfigOptions& opts,
const std::string& /*name*/, void* addr) {
auto env = static_cast<Env**>(addr);
return (*env)->PrepareOptions(opts);
})
.SetValidateFunc([](const DBOptions& db_opts,
const ColumnFamilyOptions& cf_opts,
const std::string& /*name*/,
const void* addr) {
const auto env = static_cast<const Env* const*>(addr);
return (*env)->ValidateOptions(db_opts, cf_opts);
})},
{"allow_data_in_errors",
{offsetof(struct ImmutableDBOptions, allow_data_in_errors),
OptionType::kBoolean, OptionVerificationType::kNormal,

69
options/options_helper.cc
View File

@ -898,18 +898,18 @@ Status OptionTypeInfo::Parse(const ConfigOptions& config_options,
return Status::OK();
}
try {
void* opt_addr = static_cast<char*>(opt_ptr) + offset_;
const std::string& opt_value = config_options.input_strings_escaped
? UnescapeOptionString(value)
: value;
if (opt_addr == nullptr) {
if (opt_ptr == nullptr) {
return Status::NotFound("Could not find option", opt_name);
} else if (parse_func_ != nullptr) {
ConfigOptions copy = config_options;
copy.invoke_prepare_options = false;
void* opt_addr = GetOffset(opt_ptr);
return parse_func_(copy, opt_name, opt_value, opt_addr);
} else if (ParseOptionHelper(opt_addr, type_, opt_value)) {
} else if (ParseOptionHelper(GetOffset(opt_ptr), type_, opt_value)) {
return Status::OK();
} else if (IsConfigurable()) {
// The option is <config>.<name>
@ -1021,12 +1021,12 @@ Status OptionTypeInfo::Serialize(const ConfigOptions& config_options,
std::string* opt_value) const {
// If the option is no longer used in rocksdb and marked as deprecated,
// we skip it in the serialization.
const void* opt_addr = static_cast<const char*>(opt_ptr) + offset_;
if (opt_addr == nullptr || IsDeprecated()) {
if (opt_ptr == nullptr || IsDeprecated()) {
return Status::OK();
} else if (IsEnabled(OptionTypeFlags::kDontSerialize)) {
return Status::NotSupported("Cannot serialize option: ", opt_name);
} else if (serialize_func_ != nullptr) {
const void* opt_addr = GetOffset(opt_ptr);
return serialize_func_(config_options, opt_name, opt_addr, opt_value);
} else if (IsCustomizable()) {
const Customizable* custom = AsRawPointer<Customizable>(opt_ptr);
@ -1074,7 +1074,8 @@ Status OptionTypeInfo::Serialize(const ConfigOptions& config_options,
return Status::OK();
} else if (config_options.mutable_options_only && !IsMutable()) {
return Status::OK();
} else if (SerializeSingleOptionHelper(opt_addr, type_, opt_value)) {
} else if (SerializeSingleOptionHelper(GetOffset(opt_ptr), type_,
opt_value)) {
return Status::OK();
} else {
return Status::InvalidArgument("Cannot serialize option: ", opt_name);
@ -1223,18 +1224,21 @@ bool OptionTypeInfo::AreEqual(const ConfigOptions& config_options,
if (!config_options.IsCheckEnabled(level)) {
return true; // If the sanity level is not being checked, skip it
}
const void* this_addr = static_cast<const char*>(this_ptr) + offset_;
const void* that_addr = static_cast<const char*>(that_ptr) + offset_;
if (this_addr == nullptr || that_addr == nullptr) {
if (this_addr == that_addr) {
if (this_ptr == nullptr || that_ptr == nullptr) {
if (this_ptr == that_ptr) {
return true;
}
} else if (equals_func_ != nullptr) {
const void* this_addr = GetOffset(this_ptr);
const void* that_addr = GetOffset(that_ptr);
if (equals_func_(config_options, opt_name, this_addr, that_addr,
mismatch)) {
return true;
}
} else if (AreOptionsEqual(type_, this_addr, that_addr)) {
} else {
const void* this_addr = GetOffset(this_ptr);
const void* that_addr = GetOffset(that_ptr);
if (AreOptionsEqual(type_, this_addr, that_addr)) {
return true;
} else if (IsConfigurable()) {
const auto* this_config = AsRawPointer<Configurable>(this_ptr);
@ -1249,8 +1253,8 @@ bool OptionTypeInfo::AreEqual(const ConfigOptions& config_options,
copy.sanity_level = level;
matches = this_config->AreEquivalent(copy, that_config, &bad_name);
} else {
matches =
this_config->AreEquivalent(config_options, that_config, &bad_name);
matches = this_config->AreEquivalent(config_options, that_config,
&bad_name);
}
if (!matches) {
*mismatch = opt_name + "." + bad_name;
@ -1258,6 +1262,7 @@ bool OptionTypeInfo::AreEqual(const ConfigOptions& config_options,
return matches;
}
}
}
if (mismatch->empty()) {
*mismatch = opt_name;
}
@ -1379,6 +1384,44 @@ bool OptionTypeInfo::AreEqualByName(const ConfigOptions& config_options,
return (this_value == that_value);
}
Status OptionTypeInfo::Prepare(const ConfigOptions& config_options,
const std::string& name, void* opt_ptr) const {
if (ShouldPrepare()) {
if (prepare_func_ != nullptr) {
void* opt_addr = GetOffset(opt_ptr);
return prepare_func_(config_options, name, opt_addr);
} else if (IsConfigurable()) {
Configurable* config = AsRawPointer<Configurable>(opt_ptr);
if (config != nullptr) {
return config->PrepareOptions(config_options);
} else if (!CanBeNull()) {
return Status::NotFound("Missing configurable object", name);
}
}
}
return Status::OK();
}
Status OptionTypeInfo::Validate(const DBOptions& db_opts,
const ColumnFamilyOptions& cf_opts,
const std::string& name,
const void* opt_ptr) const {
if (ShouldValidate()) {
if (validate_func_ != nullptr) {
const void* opt_addr = GetOffset(opt_ptr);
return validate_func_(db_opts, cf_opts, name, opt_addr);
} else if (IsConfigurable()) {
const Configurable* config = AsRawPointer<Configurable>(opt_ptr);
if (config != nullptr) {
return config->ValidateOptions(db_opts, cf_opts);
} else if (!CanBeNull()) {
return Status::NotFound("Missing configurable object", name);
}
}
}
return Status::OK();
}
const OptionTypeInfo* OptionTypeInfo::Find(
const std::string& opt_name,
const std::unordered_map<std::string, OptionTypeInfo>& opt_map,

93
options/options_test.cc
View File

@ -4266,11 +4266,12 @@ TEST_F(OptionTypeInfoTest, TestInvalidArgs) {
}
TEST_F(OptionTypeInfoTest, TestParseFunc) {
OptionTypeInfo opt_info(
0, OptionType::kUnknown, OptionVerificationType::kNormal,
OptionTypeFlags::kNone,
[](const ConfigOptions& /*opts*/, const std::string& name,
const std::string& value, void* addr) {
OptionTypeInfo opt_info(0, OptionType::kUnknown,
OptionVerificationType::kNormal,
OptionTypeFlags::kNone);
opt_info.SetParseFunc([](const ConfigOptions& /*opts*/,
const std::string& name, const std::string& value,
void* addr) {
auto ptr = static_cast<std::string*>(addr);
if (name == "Oops") {
return Status::InvalidArgument(value);
@ -4287,19 +4288,19 @@ TEST_F(OptionTypeInfoTest, TestParseFunc) {
}
TEST_F(OptionTypeInfoTest, TestSerializeFunc) {
OptionTypeInfo opt_info(
0, OptionType::kString, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, nullptr,
[](const ConfigOptions& /*opts*/, const std::string& name,
const void* /*addr*/, std::string* value) {
OptionTypeInfo opt_info(0, OptionType::kString,
OptionVerificationType::kNormal,
OptionTypeFlags::kNone);
opt_info.SetSerializeFunc([](const ConfigOptions& /*opts*/,
const std::string& name, const void* /*addr*/,
std::string* value) {
if (name == "Oops") {
return Status::InvalidArgument(name);
} else {
*value = name;
return Status::OK();
}
},
nullptr);
});
ConfigOptions config_options;
std::string base;
std::string value;
@ -4309,11 +4310,11 @@ TEST_F(OptionTypeInfoTest, TestSerializeFunc) {
}
TEST_F(OptionTypeInfoTest, TestEqualsFunc) {
OptionTypeInfo opt_info(
0, OptionType::kInt, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, nullptr, nullptr,
[](const ConfigOptions& /*opts*/, const std::string& name,
const void* addr1, const void* addr2, std::string* mismatch) {
OptionTypeInfo opt_info(0, OptionType::kInt, OptionVerificationType::kNormal,
OptionTypeFlags::kNone);
opt_info.SetEqualsFunc([](const ConfigOptions& /*opts*/,
const std::string& name, const void* addr1,
const void* addr2, std::string* mismatch) {
auto i1 = *(static_cast<const int*>(addr1));
auto i2 = *(static_cast<const int*>(addr2));
if (name == "LT") {
@ -4342,6 +4343,64 @@ TEST_F(OptionTypeInfoTest, TestEqualsFunc) {
ASSERT_EQ(mismatch, "NO???");
}
TEST_F(OptionTypeInfoTest, TestPrepareFunc) {
OptionTypeInfo opt_info(0, OptionType::kInt, OptionVerificationType::kNormal,
OptionTypeFlags::kNone);
opt_info.SetPrepareFunc(
[](const ConfigOptions& /*opts*/, const std::string& name, void* addr) {
auto i1 = static_cast<int*>(addr);
if (name == "x2") {
*i1 *= 2;
} else if (name == "/2") {
*i1 /= 2;
} else {
return Status::InvalidArgument("Bad Argument", name);
}
return Status::OK();
});
ConfigOptions config_options;
int int1 = 100;
ASSERT_OK(opt_info.Prepare(config_options, "x2", &int1));
ASSERT_EQ(int1, 200);
ASSERT_OK(opt_info.Prepare(config_options, "/2", &int1));
ASSERT_EQ(int1, 100);
ASSERT_NOK(opt_info.Prepare(config_options, "??", &int1));
ASSERT_EQ(int1, 100);
}
TEST_F(OptionTypeInfoTest, TestValidateFunc) {
OptionTypeInfo opt_info(0, OptionType::kSizeT,
OptionVerificationType::kNormal,
OptionTypeFlags::kNone);
opt_info.SetValidateFunc([](const DBOptions& db_opts,
const ColumnFamilyOptions& cf_opts,
const std::string& name, const void* addr) {
const auto sz = static_cast<const size_t*>(addr);
bool is_valid = false;
if (name == "keep_log_file_num") {
is_valid = (*sz == db_opts.keep_log_file_num);
} else if (name == "write_buffer_size") {
is_valid = (*sz == cf_opts.write_buffer_size);
}
if (is_valid) {
return Status::OK();
} else {
return Status::InvalidArgument("Mismatched value", name);
}
});
ConfigOptions config_options;
DBOptions db_options;
ColumnFamilyOptions cf_options;
ASSERT_OK(opt_info.Validate(db_options, cf_options, "keep_log_file_num",
&db_options.keep_log_file_num));
ASSERT_OK(opt_info.Validate(db_options, cf_options, "write_buffer_size",
&cf_options.write_buffer_size));
ASSERT_NOK(opt_info.Validate(db_options, cf_options, "keep_log_file_num",
&cf_options.write_buffer_size));
ASSERT_NOK(opt_info.Validate(db_options, cf_options, "write_buffer_size",
&db_options.keep_log_file_num));
}
TEST_F(OptionTypeInfoTest, TestOptionFlags) {
OptionTypeInfo opt_none(0, OptionType::kString,
OptionVerificationType::kNormal,