Make SliceTransform into a Customizable class (#8641)
Summary: Made SliceTransform into a Customizable class. Would be nice to write a test that stored and used a custom transform in an SST table. There are a set of tests (DBBlockFliterTest.PrefixExtractor*, SamePrefixTest.InDomainTest, PrefixTest.PrefixAndWholeKeyTest that run the same with or without a SliceTransform/PrefixFilter. Is this expected? Pull Request resolved: https://github.com/facebook/rocksdb/pull/8641 Reviewed By: zhichao-cao Differential Revision: D31142793 Pulled By: mrambacher fbshipit-source-id: bb08672fccbfdc263dcae21f25a62307e1facda1
This commit is contained in:
parent
b92cef2d1d
commit
e0f697d2bd
@ -5,6 +5,7 @@
|
|||||||
### New Features
|
### New Features
|
||||||
### Public API change
|
### Public API change
|
||||||
* Made SystemClock extend the Customizable class and added a CreateFromString method. Implementations need to be registered with the ObjectRegistry and to implement a Name() method in order to be created via this method.
|
* Made SystemClock extend the Customizable class and added a CreateFromString method. Implementations need to be registered with the ObjectRegistry and to implement a Name() method in order to be created via this method.
|
||||||
|
* Made SliceTransform extend the Customizable class and added a CreateFromString method. Implementations need to be registered with the ObjectRegistry and to implement a Name() method in order to be created via this method. The Capped and Prefixed transform classes return a short name (no length); use GetId for the fully qualified name.
|
||||||
|
|
||||||
## 6.25.0 (2021-09-20)
|
## 6.25.0 (2021-09-20)
|
||||||
### Bug Fixes
|
### Bug Fixes
|
||||||
|
@ -1820,8 +1820,8 @@ TEST_F(DBBloomFilterTest, DynamicBloomFilterUpperBound) {
|
|||||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
|
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
|
||||||
}
|
}
|
||||||
ASSERT_OK(dbfull()->SetOptions({{"prefix_extractor", "fixed:5"}}));
|
ASSERT_OK(dbfull()->SetOptions({{"prefix_extractor", "fixed:5"}}));
|
||||||
ASSERT_EQ(0, strcmp(dbfull()->GetOptions().prefix_extractor->Name(),
|
ASSERT_EQ(dbfull()->GetOptions().prefix_extractor->AsString(),
|
||||||
"rocksdb.FixedPrefix.5"));
|
"rocksdb.FixedPrefix.5");
|
||||||
{
|
{
|
||||||
// BF changed, [abcdxx00, abce) is a valid bound, will trigger BF read
|
// BF changed, [abcdxx00, abce) is a valid bound, will trigger BF read
|
||||||
Slice upper_bound("abce");
|
Slice upper_bound("abce");
|
||||||
@ -1940,8 +1940,8 @@ TEST_F(DBBloomFilterTest, DynamicBloomFilterMultipleSST) {
|
|||||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_CHECKED), 1);
|
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_CHECKED), 1);
|
||||||
|
|
||||||
ASSERT_OK(dbfull()->SetOptions({{"prefix_extractor", "capped:3"}}));
|
ASSERT_OK(dbfull()->SetOptions({{"prefix_extractor", "capped:3"}}));
|
||||||
ASSERT_EQ(0, strcmp(dbfull()->GetOptions().prefix_extractor->Name(),
|
ASSERT_EQ(dbfull()->GetOptions().prefix_extractor->AsString(),
|
||||||
"rocksdb.CappedPrefix.3"));
|
"rocksdb.CappedPrefix.3");
|
||||||
read_options.iterate_upper_bound = &upper_bound;
|
read_options.iterate_upper_bound = &upper_bound;
|
||||||
std::unique_ptr<Iterator> iter(db_->NewIterator(read_options));
|
std::unique_ptr<Iterator> iter(db_->NewIterator(read_options));
|
||||||
ASSERT_EQ(CountIter(iter, "foo"), 2);
|
ASSERT_EQ(CountIter(iter, "foo"), 2);
|
||||||
@ -1974,8 +1974,8 @@ TEST_F(DBBloomFilterTest, DynamicBloomFilterMultipleSST) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
ASSERT_OK(dbfull()->SetOptions({{"prefix_extractor", "fixed:2"}}));
|
ASSERT_OK(dbfull()->SetOptions({{"prefix_extractor", "fixed:2"}}));
|
||||||
ASSERT_EQ(0, strcmp(dbfull()->GetOptions().prefix_extractor->Name(),
|
ASSERT_EQ(dbfull()->GetOptions().prefix_extractor->AsString(),
|
||||||
"rocksdb.FixedPrefix.2"));
|
"rocksdb.FixedPrefix.2");
|
||||||
// third SST with fixed:2 BF
|
// third SST with fixed:2 BF
|
||||||
ASSERT_OK(Put("foo6", "bar6"));
|
ASSERT_OK(Put("foo6", "bar6"));
|
||||||
ASSERT_OK(Put("foo7", "bar7"));
|
ASSERT_OK(Put("foo7", "bar7"));
|
||||||
@ -2022,8 +2022,8 @@ TEST_F(DBBloomFilterTest, DynamicBloomFilterMultipleSST) {
|
|||||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_USEFUL), 3);
|
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_USEFUL), 3);
|
||||||
}
|
}
|
||||||
ASSERT_OK(dbfull()->SetOptions({{"prefix_extractor", "capped:3"}}));
|
ASSERT_OK(dbfull()->SetOptions({{"prefix_extractor", "capped:3"}}));
|
||||||
ASSERT_EQ(0, strcmp(dbfull()->GetOptions().prefix_extractor->Name(),
|
ASSERT_EQ(dbfull()->GetOptions().prefix_extractor->AsString(),
|
||||||
"rocksdb.CappedPrefix.3"));
|
"rocksdb.CappedPrefix.3");
|
||||||
{
|
{
|
||||||
std::unique_ptr<Iterator> iter_all(db_->NewIterator(read_options));
|
std::unique_ptr<Iterator> iter_all(db_->NewIterator(read_options));
|
||||||
ASSERT_EQ(CountIter(iter_all, "foo"), 6);
|
ASSERT_EQ(CountIter(iter_all, "foo"), 6);
|
||||||
@ -2063,9 +2063,8 @@ TEST_F(DBBloomFilterTest, DynamicBloomFilterNewColumnFamily) {
|
|||||||
// create a new CF and set prefix_extractor dynamically
|
// create a new CF and set prefix_extractor dynamically
|
||||||
options.prefix_extractor.reset(NewCappedPrefixTransform(3));
|
options.prefix_extractor.reset(NewCappedPrefixTransform(3));
|
||||||
CreateColumnFamilies({"ramen_dojo_" + std::to_string(iteration)}, options);
|
CreateColumnFamilies({"ramen_dojo_" + std::to_string(iteration)}, options);
|
||||||
ASSERT_EQ(0,
|
ASSERT_EQ(dbfull()->GetOptions(handles_[2]).prefix_extractor->AsString(),
|
||||||
strcmp(dbfull()->GetOptions(handles_[2]).prefix_extractor->Name(),
|
"rocksdb.CappedPrefix.3");
|
||||||
"rocksdb.CappedPrefix.3"));
|
|
||||||
ASSERT_OK(Put(2, "foo3", "bar3"));
|
ASSERT_OK(Put(2, "foo3", "bar3"));
|
||||||
ASSERT_OK(Put(2, "foo4", "bar4"));
|
ASSERT_OK(Put(2, "foo4", "bar4"));
|
||||||
ASSERT_OK(Put(2, "foo5", "bar5"));
|
ASSERT_OK(Put(2, "foo5", "bar5"));
|
||||||
@ -2081,9 +2080,8 @@ TEST_F(DBBloomFilterTest, DynamicBloomFilterNewColumnFamily) {
|
|||||||
}
|
}
|
||||||
ASSERT_OK(
|
ASSERT_OK(
|
||||||
dbfull()->SetOptions(handles_[2], {{"prefix_extractor", "fixed:2"}}));
|
dbfull()->SetOptions(handles_[2], {{"prefix_extractor", "fixed:2"}}));
|
||||||
ASSERT_EQ(0,
|
ASSERT_EQ(dbfull()->GetOptions(handles_[2]).prefix_extractor->AsString(),
|
||||||
strcmp(dbfull()->GetOptions(handles_[2]).prefix_extractor->Name(),
|
"rocksdb.FixedPrefix.2");
|
||||||
"rocksdb.FixedPrefix.2"));
|
|
||||||
{
|
{
|
||||||
std::unique_ptr<Iterator> iter(
|
std::unique_ptr<Iterator> iter(
|
||||||
db_->NewIterator(read_options, handles_[2]));
|
db_->NewIterator(read_options, handles_[2]));
|
||||||
@ -2148,8 +2146,8 @@ TEST_F(DBBloomFilterTest, DynamicBloomFilterOptions) {
|
|||||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
|
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
|
||||||
|
|
||||||
ASSERT_OK(dbfull()->SetOptions({{"prefix_extractor", "capped:3"}}));
|
ASSERT_OK(dbfull()->SetOptions({{"prefix_extractor", "capped:3"}}));
|
||||||
ASSERT_EQ(0, strcmp(dbfull()->GetOptions().prefix_extractor->Name(),
|
ASSERT_EQ(dbfull()->GetOptions().prefix_extractor->AsString(),
|
||||||
"rocksdb.CappedPrefix.3"));
|
"rocksdb.CappedPrefix.3");
|
||||||
{
|
{
|
||||||
std::unique_ptr<Iterator> iter(db_->NewIterator(read_options));
|
std::unique_ptr<Iterator> iter(db_->NewIterator(read_options));
|
||||||
// "fp*" should be skipped
|
// "fp*" should be skipped
|
||||||
|
@ -14,13 +14,16 @@
|
|||||||
|
|
||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
|
#include <memory>
|
||||||
#include <string>
|
#include <string>
|
||||||
|
|
||||||
|
#include "rocksdb/customizable.h"
|
||||||
#include "rocksdb/rocksdb_namespace.h"
|
#include "rocksdb/rocksdb_namespace.h"
|
||||||
|
|
||||||
namespace ROCKSDB_NAMESPACE {
|
namespace ROCKSDB_NAMESPACE {
|
||||||
|
|
||||||
class Slice;
|
class Slice;
|
||||||
|
struct ConfigOptions;
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* A SliceTransform is a generic pluggable way of transforming one string
|
* A SliceTransform is a generic pluggable way of transforming one string
|
||||||
@ -28,12 +31,22 @@ class Slice;
|
|||||||
* to store prefix blooms by setting prefix_extractor in
|
* to store prefix blooms by setting prefix_extractor in
|
||||||
* ColumnFamilyOptions.
|
* ColumnFamilyOptions.
|
||||||
*/
|
*/
|
||||||
class SliceTransform {
|
class SliceTransform : public Customizable {
|
||||||
public:
|
public:
|
||||||
virtual ~SliceTransform(){};
|
virtual ~SliceTransform(){};
|
||||||
|
|
||||||
// Return the name of this transformation.
|
// Return the name of this transformation.
|
||||||
virtual const char* Name() const = 0;
|
virtual const char* Name() const = 0;
|
||||||
|
static const char* Type() { return "SliceTransform"; }
|
||||||
|
|
||||||
|
// Creates and configures a new SliceTransform from the input options and id.
|
||||||
|
static Status CreateFromString(const ConfigOptions& config_options,
|
||||||
|
const std::string& id,
|
||||||
|
std::shared_ptr<const SliceTransform>* result);
|
||||||
|
|
||||||
|
// Returns a string representation of this SliceTransform, representing the ID
|
||||||
|
// and any additional properties
|
||||||
|
std::string AsString() const;
|
||||||
|
|
||||||
// Extract a prefix from a specified key. This method is called when
|
// Extract a prefix from a specified key. This method is called when
|
||||||
// a key is inserted into the db, and the returned slice is used to
|
// a key is inserted into the db, and the returned slice is used to
|
||||||
|
@ -33,7 +33,6 @@ enum class OptionType {
|
|||||||
kDouble,
|
kDouble,
|
||||||
kCompactionStyle,
|
kCompactionStyle,
|
||||||
kCompactionPri,
|
kCompactionPri,
|
||||||
kSliceTransform,
|
|
||||||
kCompressionType,
|
kCompressionType,
|
||||||
kCompactionStopStyle,
|
kCompactionStopStyle,
|
||||||
kFilterPolicy,
|
kFilterPolicy,
|
||||||
@ -575,6 +574,7 @@ class OptionTypeInfo {
|
|||||||
// or if the flags specify allow null.
|
// or if the flags specify allow null.
|
||||||
bool CanBeNull() const {
|
bool CanBeNull() const {
|
||||||
return (IsEnabled(OptionTypeFlags::kAllowNull) ||
|
return (IsEnabled(OptionTypeFlags::kAllowNull) ||
|
||||||
|
IsEnabled(OptionVerificationType::kByNameAllowNull) ||
|
||||||
IsEnabled(OptionVerificationType::kByNameAllowFromNull));
|
IsEnabled(OptionVerificationType::kByNameAllowFromNull));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -360,9 +360,10 @@ static std::unordered_map<std::string, OptionTypeInfo>
|
|||||||
OptionType::kCompressionType, OptionVerificationType::kNormal,
|
OptionType::kCompressionType, OptionVerificationType::kNormal,
|
||||||
OptionTypeFlags::kMutable}},
|
OptionTypeFlags::kMutable}},
|
||||||
{"prefix_extractor",
|
{"prefix_extractor",
|
||||||
{offsetof(struct MutableCFOptions, prefix_extractor),
|
OptionTypeInfo::AsCustomSharedPtr<const SliceTransform>(
|
||||||
OptionType::kSliceTransform, OptionVerificationType::kByNameAllowNull,
|
offsetof(struct MutableCFOptions, prefix_extractor),
|
||||||
OptionTypeFlags::kMutable}},
|
OptionVerificationType::kByNameAllowNull,
|
||||||
|
(OptionTypeFlags::kMutable | OptionTypeFlags::kAllowNull))},
|
||||||
{"compaction_options_fifo",
|
{"compaction_options_fifo",
|
||||||
OptionTypeInfo::Struct(
|
OptionTypeInfo::Struct(
|
||||||
"compaction_options_fifo", &fifo_compaction_options_type_info,
|
"compaction_options_fifo", &fifo_compaction_options_type_info,
|
||||||
@ -567,10 +568,10 @@ static std::unordered_map<std::string, OptionTypeInfo>
|
|||||||
},
|
},
|
||||||
/* Use the default match function*/ nullptr)},
|
/* Use the default match function*/ nullptr)},
|
||||||
{"memtable_insert_with_hint_prefix_extractor",
|
{"memtable_insert_with_hint_prefix_extractor",
|
||||||
{offset_of(
|
OptionTypeInfo::AsCustomSharedPtr<const SliceTransform>(
|
||||||
&ImmutableCFOptions::memtable_insert_with_hint_prefix_extractor),
|
offset_of(&ImmutableCFOptions::
|
||||||
OptionType::kSliceTransform, OptionVerificationType::kByNameAllowNull,
|
memtable_insert_with_hint_prefix_extractor),
|
||||||
OptionTypeFlags::kNone}},
|
OptionVerificationType::kByNameAllowNull, OptionTypeFlags::kNone)},
|
||||||
{"memtable_factory",
|
{"memtable_factory",
|
||||||
{offset_of(&ImmutableCFOptions::memtable_factory),
|
{offset_of(&ImmutableCFOptions::memtable_factory),
|
||||||
OptionType::kCustomizable, OptionVerificationType::kByName,
|
OptionType::kCustomizable, OptionVerificationType::kByName,
|
||||||
@ -935,9 +936,10 @@ void MutableCFOptions::Dump(Logger* log) const {
|
|||||||
ROCKS_LOG_INFO(log,
|
ROCKS_LOG_INFO(log,
|
||||||
" inplace_update_num_locks: %" ROCKSDB_PRIszt,
|
" inplace_update_num_locks: %" ROCKSDB_PRIszt,
|
||||||
inplace_update_num_locks);
|
inplace_update_num_locks);
|
||||||
ROCKS_LOG_INFO(
|
ROCKS_LOG_INFO(log, " prefix_extractor: %s",
|
||||||
log, " prefix_extractor: %s",
|
prefix_extractor == nullptr
|
||||||
prefix_extractor == nullptr ? "nullptr" : prefix_extractor->Name());
|
? "nullptr"
|
||||||
|
: prefix_extractor->GetId().c_str());
|
||||||
ROCKS_LOG_INFO(log, " disable_auto_compactions: %d",
|
ROCKS_LOG_INFO(log, " disable_auto_compactions: %d",
|
||||||
disable_auto_compactions);
|
disable_auto_compactions);
|
||||||
ROCKS_LOG_INFO(log, " soft_pending_compaction_bytes_limit: %" PRIu64,
|
ROCKS_LOG_INFO(log, " soft_pending_compaction_bytes_limit: %" PRIu64,
|
||||||
|
@ -22,6 +22,7 @@
|
|||||||
#include "rocksdb/env_encryption.h"
|
#include "rocksdb/env_encryption.h"
|
||||||
#include "rocksdb/flush_block_policy.h"
|
#include "rocksdb/flush_block_policy.h"
|
||||||
#include "rocksdb/secondary_cache.h"
|
#include "rocksdb/secondary_cache.h"
|
||||||
|
#include "rocksdb/slice_transform.h"
|
||||||
#include "rocksdb/statistics.h"
|
#include "rocksdb/statistics.h"
|
||||||
#include "rocksdb/utilities/customizable_util.h"
|
#include "rocksdb/utilities/customizable_util.h"
|
||||||
#include "rocksdb/utilities/object_registry.h"
|
#include "rocksdb/utilities/object_registry.h"
|
||||||
@ -1241,6 +1242,18 @@ class TestFlushBlockPolicyFactory : public FlushBlockPolicyFactory {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
class MockSliceTransform : public SliceTransform {
|
||||||
|
public:
|
||||||
|
const char* Name() const override { return kClassName(); }
|
||||||
|
static const char* kClassName() { return "Mock"; }
|
||||||
|
|
||||||
|
Slice Transform(const Slice& /*key*/) const override { return Slice(); }
|
||||||
|
|
||||||
|
bool InDomain(const Slice& /*key*/) const override { return false; }
|
||||||
|
|
||||||
|
bool InRange(const Slice& /*key*/) const override { return false; }
|
||||||
|
};
|
||||||
|
|
||||||
#ifndef ROCKSDB_LITE
|
#ifndef ROCKSDB_LITE
|
||||||
class MockEncryptionProvider : public EncryptionProvider {
|
class MockEncryptionProvider : public EncryptionProvider {
|
||||||
public:
|
public:
|
||||||
@ -1309,6 +1322,14 @@ static int RegisterLocalObjects(ObjectLibrary& library,
|
|||||||
return guard->get();
|
return guard->get();
|
||||||
});
|
});
|
||||||
// Load any locally defined objects here
|
// Load any locally defined objects here
|
||||||
|
library.Register<const SliceTransform>(
|
||||||
|
MockSliceTransform::kClassName(),
|
||||||
|
[](const std::string& /*uri*/,
|
||||||
|
std::unique_ptr<const SliceTransform>* guard,
|
||||||
|
std::string* /* errmsg */) {
|
||||||
|
guard->reset(new MockSliceTransform());
|
||||||
|
return guard->get();
|
||||||
|
});
|
||||||
library.Register<Statistics>(
|
library.Register<Statistics>(
|
||||||
TestStatistics::kClassName(),
|
TestStatistics::kClassName(),
|
||||||
[](const std::string& /*uri*/, std::unique_ptr<Statistics>* guard,
|
[](const std::string& /*uri*/, std::unique_ptr<Statistics>* guard,
|
||||||
@ -1447,6 +1468,37 @@ TEST_F(LoadCustomizableTest, LoadComparatorTest) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
TEST_F(LoadCustomizableTest, LoadSliceTransformFactoryTest) {
|
||||||
|
std::shared_ptr<const SliceTransform> result;
|
||||||
|
ASSERT_NOK(
|
||||||
|
SliceTransform::CreateFromString(config_options_, "Mock", &result));
|
||||||
|
ASSERT_OK(
|
||||||
|
SliceTransform::CreateFromString(config_options_, "fixed:16", &result));
|
||||||
|
ASSERT_NE(result.get(), nullptr);
|
||||||
|
ASSERT_TRUE(result->IsInstanceOf("fixed"));
|
||||||
|
ASSERT_OK(SliceTransform::CreateFromString(
|
||||||
|
config_options_, "rocksdb.FixedPrefix.22", &result));
|
||||||
|
ASSERT_NE(result.get(), nullptr);
|
||||||
|
ASSERT_TRUE(result->IsInstanceOf("fixed"));
|
||||||
|
|
||||||
|
ASSERT_OK(
|
||||||
|
SliceTransform::CreateFromString(config_options_, "capped:16", &result));
|
||||||
|
ASSERT_NE(result.get(), nullptr);
|
||||||
|
ASSERT_TRUE(result->IsInstanceOf("capped"));
|
||||||
|
|
||||||
|
ASSERT_OK(SliceTransform::CreateFromString(
|
||||||
|
config_options_, "rocksdb.CappedPrefix.11", &result));
|
||||||
|
ASSERT_NE(result.get(), nullptr);
|
||||||
|
ASSERT_TRUE(result->IsInstanceOf("capped"));
|
||||||
|
|
||||||
|
if (RegisterTests("Test")) {
|
||||||
|
ASSERT_OK(
|
||||||
|
SliceTransform::CreateFromString(config_options_, "Mock", &result));
|
||||||
|
ASSERT_NE(result, nullptr);
|
||||||
|
ASSERT_STREQ(result->Name(), "Mock");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
TEST_F(LoadCustomizableTest, LoadStatisticsTest) {
|
TEST_F(LoadCustomizableTest, LoadStatisticsTest) {
|
||||||
std::shared_ptr<Statistics> stats;
|
std::shared_ptr<Statistics> stats;
|
||||||
ASSERT_NOK(Statistics::CreateFromString(
|
ASSERT_NOK(Statistics::CreateFromString(
|
||||||
|
@ -364,61 +364,6 @@ std::vector<CompressionType> GetSupportedDictCompressions() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#ifndef ROCKSDB_LITE
|
#ifndef ROCKSDB_LITE
|
||||||
bool ParseSliceTransformHelper(
|
|
||||||
const std::string& kFixedPrefixName, const std::string& kCappedPrefixName,
|
|
||||||
const std::string& value,
|
|
||||||
std::shared_ptr<const SliceTransform>* slice_transform) {
|
|
||||||
const char* no_op_name = "rocksdb.Noop";
|
|
||||||
size_t no_op_length = strlen(no_op_name);
|
|
||||||
auto& pe_value = value;
|
|
||||||
if (pe_value.size() > kFixedPrefixName.size() &&
|
|
||||||
pe_value.compare(0, kFixedPrefixName.size(), kFixedPrefixName) == 0) {
|
|
||||||
int prefix_length = ParseInt(trim(value.substr(kFixedPrefixName.size())));
|
|
||||||
slice_transform->reset(NewFixedPrefixTransform(prefix_length));
|
|
||||||
} else if (pe_value.size() > kCappedPrefixName.size() &&
|
|
||||||
pe_value.compare(0, kCappedPrefixName.size(), kCappedPrefixName) ==
|
|
||||||
0) {
|
|
||||||
int prefix_length =
|
|
||||||
ParseInt(trim(pe_value.substr(kCappedPrefixName.size())));
|
|
||||||
slice_transform->reset(NewCappedPrefixTransform(prefix_length));
|
|
||||||
} else if (pe_value.size() == no_op_length &&
|
|
||||||
pe_value.compare(0, no_op_length, no_op_name) == 0) {
|
|
||||||
const SliceTransform* no_op_transform = NewNoopTransform();
|
|
||||||
slice_transform->reset(no_op_transform);
|
|
||||||
} else if (value == kNullptrString) {
|
|
||||||
slice_transform->reset();
|
|
||||||
} else {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
bool ParseSliceTransform(
|
|
||||||
const std::string& value,
|
|
||||||
std::shared_ptr<const SliceTransform>* slice_transform) {
|
|
||||||
// While we normally don't convert the string representation of a
|
|
||||||
// pointer-typed option into its instance, here we do so for backward
|
|
||||||
// compatibility as we allow this action in SetOption().
|
|
||||||
|
|
||||||
// TODO(yhchiang): A possible better place for these serialization /
|
|
||||||
// deserialization is inside the class definition of pointer-typed
|
|
||||||
// option itself, but this requires a bigger change of public API.
|
|
||||||
bool result =
|
|
||||||
ParseSliceTransformHelper("fixed:", "capped:", value, slice_transform);
|
|
||||||
if (result) {
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
result = ParseSliceTransformHelper(
|
|
||||||
"rocksdb.FixedPrefix.", "rocksdb.CappedPrefix.", value, slice_transform);
|
|
||||||
if (result) {
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
// TODO(yhchiang): we can further support other default
|
|
||||||
// SliceTransforms here.
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
static bool ParseOptionHelper(void* opt_address, const OptionType& opt_type,
|
static bool ParseOptionHelper(void* opt_address, const OptionType& opt_type,
|
||||||
const std::string& value) {
|
const std::string& value) {
|
||||||
switch (opt_type) {
|
switch (opt_type) {
|
||||||
@ -466,10 +411,6 @@ static bool ParseOptionHelper(void* opt_address, const OptionType& opt_type,
|
|||||||
return ParseEnum<CompressionType>(
|
return ParseEnum<CompressionType>(
|
||||||
compression_type_string_map, value,
|
compression_type_string_map, value,
|
||||||
static_cast<CompressionType*>(opt_address));
|
static_cast<CompressionType*>(opt_address));
|
||||||
case OptionType::kSliceTransform:
|
|
||||||
return ParseSliceTransform(
|
|
||||||
value,
|
|
||||||
static_cast<std::shared_ptr<const SliceTransform>*>(opt_address));
|
|
||||||
case OptionType::kChecksumType:
|
case OptionType::kChecksumType:
|
||||||
return ParseEnum<ChecksumType>(checksum_type_string_map, value,
|
return ParseEnum<ChecksumType>(checksum_type_string_map, value,
|
||||||
static_cast<ChecksumType*>(opt_address));
|
static_cast<ChecksumType*>(opt_address));
|
||||||
@ -554,14 +495,7 @@ bool SerializeSingleOptionHelper(const void* opt_address,
|
|||||||
return SerializeEnum<CompressionType>(
|
return SerializeEnum<CompressionType>(
|
||||||
compression_type_string_map,
|
compression_type_string_map,
|
||||||
*(static_cast<const CompressionType*>(opt_address)), value);
|
*(static_cast<const CompressionType*>(opt_address)), value);
|
||||||
case OptionType::kSliceTransform: {
|
|
||||||
const auto* slice_transform_ptr =
|
|
||||||
static_cast<const std::shared_ptr<const SliceTransform>*>(
|
|
||||||
opt_address);
|
|
||||||
*value = slice_transform_ptr->get() ? slice_transform_ptr->get()->Name()
|
|
||||||
: kNullptrString;
|
|
||||||
break;
|
break;
|
||||||
}
|
|
||||||
case OptionType::kFilterPolicy: {
|
case OptionType::kFilterPolicy: {
|
||||||
const auto* ptr =
|
const auto* ptr =
|
||||||
static_cast<const std::shared_ptr<FilterPolicy>*>(opt_address);
|
static_cast<const std::shared_ptr<FilterPolicy>*>(opt_address);
|
||||||
@ -1068,6 +1002,7 @@ Status OptionTypeInfo::Serialize(const ConfigOptions& config_options,
|
|||||||
return serialize_func_(config_options, opt_name, opt_addr, opt_value);
|
return serialize_func_(config_options, opt_name, opt_addr, opt_value);
|
||||||
} else if (IsCustomizable()) {
|
} else if (IsCustomizable()) {
|
||||||
const Customizable* custom = AsRawPointer<Customizable>(opt_ptr);
|
const Customizable* custom = AsRawPointer<Customizable>(opt_ptr);
|
||||||
|
opt_value->clear();
|
||||||
if (custom == nullptr) {
|
if (custom == nullptr) {
|
||||||
// We do not have a custom object to serialize.
|
// We do not have a custom object to serialize.
|
||||||
// If the option is not mutable and we are doing only mutable options,
|
// If the option is not mutable and we are doing only mutable options,
|
||||||
@ -1081,7 +1016,9 @@ Status OptionTypeInfo::Serialize(const ConfigOptions& config_options,
|
|||||||
}
|
}
|
||||||
} else if (IsEnabled(OptionTypeFlags::kStringNameOnly) &&
|
} else if (IsEnabled(OptionTypeFlags::kStringNameOnly) &&
|
||||||
!config_options.IsDetailed()) {
|
!config_options.IsDetailed()) {
|
||||||
*opt_value = custom->GetId();
|
if (!config_options.mutable_options_only || IsMutable()) {
|
||||||
|
*opt_value = custom->GetId();
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
ConfigOptions embedded = config_options;
|
ConfigOptions embedded = config_options;
|
||||||
embedded.delimiter = ";";
|
embedded.delimiter = ";";
|
||||||
|
@ -54,11 +54,6 @@ std::unique_ptr<Configurable> CFOptionsAsConfigurable(
|
|||||||
const ColumnFamilyOptions& opts,
|
const ColumnFamilyOptions& opts,
|
||||||
const std::unordered_map<std::string, std::string>* opt_map = nullptr);
|
const std::unordered_map<std::string, std::string>* opt_map = nullptr);
|
||||||
|
|
||||||
|
|
||||||
bool ParseSliceTransform(
|
|
||||||
const std::string& value,
|
|
||||||
std::shared_ptr<const SliceTransform>* slice_transform);
|
|
||||||
|
|
||||||
extern Status StringToMap(
|
extern Status StringToMap(
|
||||||
const std::string& opts_str,
|
const std::string& opts_str,
|
||||||
std::unordered_map<std::string, std::string>* opts_map);
|
std::unordered_map<std::string, std::string>* opts_map);
|
||||||
|
@ -231,8 +231,7 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) {
|
|||||||
ASSERT_EQ(new_cf_opt.max_successive_merges, 30U);
|
ASSERT_EQ(new_cf_opt.max_successive_merges, 30U);
|
||||||
ASSERT_TRUE(new_cf_opt.prefix_extractor != nullptr);
|
ASSERT_TRUE(new_cf_opt.prefix_extractor != nullptr);
|
||||||
ASSERT_EQ(new_cf_opt.optimize_filters_for_hits, true);
|
ASSERT_EQ(new_cf_opt.optimize_filters_for_hits, true);
|
||||||
ASSERT_EQ(std::string(new_cf_opt.prefix_extractor->Name()),
|
ASSERT_EQ(new_cf_opt.prefix_extractor->AsString(), "rocksdb.FixedPrefix.31");
|
||||||
"rocksdb.FixedPrefix.31");
|
|
||||||
ASSERT_EQ(new_cf_opt.enable_blob_files, true);
|
ASSERT_EQ(new_cf_opt.enable_blob_files, true);
|
||||||
ASSERT_EQ(new_cf_opt.min_blob_size, 1ULL << 10);
|
ASSERT_EQ(new_cf_opt.min_blob_size, 1ULL << 10);
|
||||||
ASSERT_EQ(new_cf_opt.blob_file_size, 1ULL << 30);
|
ASSERT_EQ(new_cf_opt.blob_file_size, 1ULL << 30);
|
||||||
@ -456,8 +455,7 @@ TEST_F(OptionsTest, GetColumnFamilyOptionsFromStringTest) {
|
|||||||
ASSERT_EQ(new_cf_opt.write_buffer_size, 18 * giga);
|
ASSERT_EQ(new_cf_opt.write_buffer_size, 18 * giga);
|
||||||
ASSERT_EQ(new_cf_opt.arena_block_size, 19 * giga);
|
ASSERT_EQ(new_cf_opt.arena_block_size, 19 * giga);
|
||||||
ASSERT_TRUE(new_cf_opt.prefix_extractor.get() != nullptr);
|
ASSERT_TRUE(new_cf_opt.prefix_extractor.get() != nullptr);
|
||||||
std::string prefix_name(new_cf_opt.prefix_extractor->Name());
|
ASSERT_EQ(new_cf_opt.prefix_extractor->AsString(), "rocksdb.CappedPrefix.8");
|
||||||
ASSERT_EQ(prefix_name, "rocksdb.CappedPrefix.8");
|
|
||||||
|
|
||||||
// Units (t)
|
// Units (t)
|
||||||
ASSERT_OK(GetColumnFamilyOptionsFromString(
|
ASSERT_OK(GetColumnFamilyOptionsFromString(
|
||||||
@ -2254,8 +2252,7 @@ TEST_F(OptionsOldApiTest, GetOptionsFromMapTest) {
|
|||||||
ASSERT_EQ(new_cf_opt.max_successive_merges, 30U);
|
ASSERT_EQ(new_cf_opt.max_successive_merges, 30U);
|
||||||
ASSERT_TRUE(new_cf_opt.prefix_extractor != nullptr);
|
ASSERT_TRUE(new_cf_opt.prefix_extractor != nullptr);
|
||||||
ASSERT_EQ(new_cf_opt.optimize_filters_for_hits, true);
|
ASSERT_EQ(new_cf_opt.optimize_filters_for_hits, true);
|
||||||
ASSERT_EQ(std::string(new_cf_opt.prefix_extractor->Name()),
|
ASSERT_EQ(new_cf_opt.prefix_extractor->AsString(), "rocksdb.FixedPrefix.31");
|
||||||
"rocksdb.FixedPrefix.31");
|
|
||||||
ASSERT_EQ(new_cf_opt.enable_blob_files, true);
|
ASSERT_EQ(new_cf_opt.enable_blob_files, true);
|
||||||
ASSERT_EQ(new_cf_opt.min_blob_size, 1ULL << 10);
|
ASSERT_EQ(new_cf_opt.min_blob_size, 1ULL << 10);
|
||||||
ASSERT_EQ(new_cf_opt.blob_file_size, 1ULL << 30);
|
ASSERT_EQ(new_cf_opt.blob_file_size, 1ULL << 30);
|
||||||
@ -2448,8 +2445,7 @@ TEST_F(OptionsOldApiTest, GetColumnFamilyOptionsFromStringTest) {
|
|||||||
ASSERT_EQ(new_cf_opt.write_buffer_size, 18 * giga);
|
ASSERT_EQ(new_cf_opt.write_buffer_size, 18 * giga);
|
||||||
ASSERT_EQ(new_cf_opt.arena_block_size, 19 * giga);
|
ASSERT_EQ(new_cf_opt.arena_block_size, 19 * giga);
|
||||||
ASSERT_TRUE(new_cf_opt.prefix_extractor.get() != nullptr);
|
ASSERT_TRUE(new_cf_opt.prefix_extractor.get() != nullptr);
|
||||||
std::string prefix_name(new_cf_opt.prefix_extractor->Name());
|
ASSERT_EQ(new_cf_opt.prefix_extractor->AsString(), "rocksdb.CappedPrefix.8");
|
||||||
ASSERT_EQ(prefix_name, "rocksdb.CappedPrefix.8");
|
|
||||||
|
|
||||||
// Units (t)
|
// Units (t)
|
||||||
ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
|
ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
|
||||||
@ -2551,6 +2547,67 @@ TEST_F(OptionsOldApiTest, GetColumnFamilyOptionsFromStringTest) {
|
|||||||
ASSERT_TRUE(new_cf_opt.memtable_factory->IsInstanceOf("SkipListFactory"));
|
ASSERT_TRUE(new_cf_opt.memtable_factory->IsInstanceOf("SkipListFactory"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
TEST_F(OptionsTest, SliceTransformCreateFromString) {
|
||||||
|
std::shared_ptr<const SliceTransform> transform = nullptr;
|
||||||
|
ConfigOptions config_options;
|
||||||
|
config_options.ignore_unsupported_options = false;
|
||||||
|
config_options.ignore_unknown_options = false;
|
||||||
|
|
||||||
|
ASSERT_OK(
|
||||||
|
SliceTransform::CreateFromString(config_options, "fixed:31", &transform));
|
||||||
|
ASSERT_NE(transform, nullptr);
|
||||||
|
ASSERT_FALSE(transform->IsInstanceOf("capped"));
|
||||||
|
ASSERT_TRUE(transform->IsInstanceOf("fixed"));
|
||||||
|
ASSERT_TRUE(transform->IsInstanceOf("rocksdb.FixedPrefix"));
|
||||||
|
ASSERT_EQ(transform->GetId(), "rocksdb.FixedPrefix.31");
|
||||||
|
ASSERT_OK(SliceTransform::CreateFromString(
|
||||||
|
config_options, "rocksdb.FixedPrefix.42", &transform));
|
||||||
|
ASSERT_NE(transform, nullptr);
|
||||||
|
ASSERT_EQ(transform->GetId(), "rocksdb.FixedPrefix.42");
|
||||||
|
|
||||||
|
ASSERT_OK(SliceTransform::CreateFromString(config_options, "capped:16",
|
||||||
|
&transform));
|
||||||
|
ASSERT_NE(transform, nullptr);
|
||||||
|
ASSERT_FALSE(transform->IsInstanceOf("fixed"));
|
||||||
|
ASSERT_TRUE(transform->IsInstanceOf("capped"));
|
||||||
|
ASSERT_TRUE(transform->IsInstanceOf("rocksdb.CappedPrefix"));
|
||||||
|
ASSERT_EQ(transform->GetId(), "rocksdb.CappedPrefix.16");
|
||||||
|
ASSERT_OK(SliceTransform::CreateFromString(
|
||||||
|
config_options, "rocksdb.CappedPrefix.42", &transform));
|
||||||
|
ASSERT_NE(transform, nullptr);
|
||||||
|
ASSERT_EQ(transform->GetId(), "rocksdb.CappedPrefix.42");
|
||||||
|
|
||||||
|
ASSERT_OK(SliceTransform::CreateFromString(config_options, "rocksdb.Noop",
|
||||||
|
&transform));
|
||||||
|
ASSERT_NE(transform, nullptr);
|
||||||
|
|
||||||
|
ASSERT_NOK(SliceTransform::CreateFromString(config_options,
|
||||||
|
"fixed:21:invalid", &transform));
|
||||||
|
ASSERT_NOK(SliceTransform::CreateFromString(config_options,
|
||||||
|
"capped:21:invalid", &transform));
|
||||||
|
ASSERT_NOK(
|
||||||
|
SliceTransform::CreateFromString(config_options, "fixed", &transform));
|
||||||
|
ASSERT_NOK(
|
||||||
|
SliceTransform::CreateFromString(config_options, "capped", &transform));
|
||||||
|
ASSERT_NOK(SliceTransform::CreateFromString(
|
||||||
|
config_options, "rocksdb.FixedPrefix:42", &transform));
|
||||||
|
ASSERT_NOK(SliceTransform::CreateFromString(
|
||||||
|
config_options, "rocksdb.CappedPrefix:42", &transform));
|
||||||
|
ASSERT_NOK(
|
||||||
|
SliceTransform::CreateFromString(config_options, "invalid", &transform));
|
||||||
|
|
||||||
|
#ifndef ROCKSDB_LITE
|
||||||
|
ASSERT_OK(SliceTransform::CreateFromString(
|
||||||
|
config_options, "id=rocksdb.CappedPrefix; length=11", &transform));
|
||||||
|
ASSERT_NE(transform, nullptr);
|
||||||
|
ASSERT_EQ(transform->GetId(), "rocksdb.CappedPrefix.11");
|
||||||
|
|
||||||
|
ASSERT_NOK(SliceTransform::CreateFromString(
|
||||||
|
config_options, "id=rocksdb.CappedPrefix; length=11; invalid=true",
|
||||||
|
&transform));
|
||||||
|
#endif // ROCKSDB_LITE
|
||||||
|
}
|
||||||
|
|
||||||
TEST_F(OptionsOldApiTest, GetBlockBasedTableOptionsFromString) {
|
TEST_F(OptionsOldApiTest, GetBlockBasedTableOptionsFromString) {
|
||||||
BlockBasedTableOptions table_opt;
|
BlockBasedTableOptions table_opt;
|
||||||
BlockBasedTableOptions new_opt;
|
BlockBasedTableOptions new_opt;
|
||||||
|
@ -1670,9 +1670,8 @@ void BlockBasedTableBuilder::WritePropertiesBlock(
|
|||||||
CompressionOptionsToString(rep_->compression_opts);
|
CompressionOptionsToString(rep_->compression_opts);
|
||||||
rep_->props.prefix_extractor_name =
|
rep_->props.prefix_extractor_name =
|
||||||
rep_->moptions.prefix_extractor != nullptr
|
rep_->moptions.prefix_extractor != nullptr
|
||||||
? rep_->moptions.prefix_extractor->Name()
|
? rep_->moptions.prefix_extractor->AsString()
|
||||||
: "nullptr";
|
: "nullptr";
|
||||||
|
|
||||||
std::string property_collectors_names = "[";
|
std::string property_collectors_names = "[";
|
||||||
for (size_t i = 0;
|
for (size_t i = 0;
|
||||||
i < rep_->ioptions.table_properties_collector_factories.size(); ++i) {
|
i < rep_->ioptions.table_properties_collector_factories.size(); ++i) {
|
||||||
|
@ -23,10 +23,10 @@
|
|||||||
#include "file/file_util.h"
|
#include "file/file_util.h"
|
||||||
#include "file/random_access_file_reader.h"
|
#include "file/random_access_file_reader.h"
|
||||||
#include "monitoring/perf_context_imp.h"
|
#include "monitoring/perf_context_imp.h"
|
||||||
#include "options/options_helper.h"
|
|
||||||
#include "port/lang.h"
|
#include "port/lang.h"
|
||||||
#include "rocksdb/cache.h"
|
#include "rocksdb/cache.h"
|
||||||
#include "rocksdb/comparator.h"
|
#include "rocksdb/comparator.h"
|
||||||
|
#include "rocksdb/convenience.h"
|
||||||
#include "rocksdb/env.h"
|
#include "rocksdb/env.h"
|
||||||
#include "rocksdb/file_system.h"
|
#include "rocksdb/file_system.h"
|
||||||
#include "rocksdb/filter_policy.h"
|
#include "rocksdb/filter_policy.h"
|
||||||
@ -134,8 +134,7 @@ bool PrefixExtractorChanged(const TableProperties* table_properties,
|
|||||||
}
|
}
|
||||||
|
|
||||||
// prefix_extractor and prefix_extractor_block are both non-empty
|
// prefix_extractor and prefix_extractor_block are both non-empty
|
||||||
if (table_properties->prefix_extractor_name.compare(
|
if (table_properties->prefix_extractor_name != prefix_extractor->AsString()) {
|
||||||
prefix_extractor->Name()) != 0) {
|
|
||||||
return true;
|
return true;
|
||||||
} else {
|
} else {
|
||||||
return false;
|
return false;
|
||||||
@ -816,8 +815,19 @@ Status BlockBasedTable::ReadPropertiesBlock(
|
|||||||
}
|
}
|
||||||
#ifndef ROCKSDB_LITE
|
#ifndef ROCKSDB_LITE
|
||||||
if (rep_->table_properties) {
|
if (rep_->table_properties) {
|
||||||
ParseSliceTransform(rep_->table_properties->prefix_extractor_name,
|
//**TODO: If/When the DBOptions has a registry in it, the ConfigOptions
|
||||||
&(rep_->table_prefix_extractor));
|
// will need to use it
|
||||||
|
ConfigOptions config_options;
|
||||||
|
Status st = SliceTransform::CreateFromString(
|
||||||
|
config_options, rep_->table_properties->prefix_extractor_name,
|
||||||
|
&(rep_->table_prefix_extractor));
|
||||||
|
if (!st.ok()) {
|
||||||
|
//**TODO: Should this be error be returned or swallowed?
|
||||||
|
ROCKS_LOG_ERROR(rep_->ioptions.logger,
|
||||||
|
"Failed to create prefix extractor[%s]: %s",
|
||||||
|
rep_->table_properties->prefix_extractor_name.c_str(),
|
||||||
|
st.ToString().c_str());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
#endif // ROCKSDB_LITE
|
#endif // ROCKSDB_LITE
|
||||||
|
|
||||||
@ -2223,8 +2233,8 @@ bool BlockBasedTable::FullFilterKeyMayMatch(
|
|||||||
filter->KeyMayMatch(user_key_without_ts, prefix_extractor, kNotValid,
|
filter->KeyMayMatch(user_key_without_ts, prefix_extractor, kNotValid,
|
||||||
no_io, const_ikey_ptr, get_context, lookup_context);
|
no_io, const_ikey_ptr, get_context, lookup_context);
|
||||||
} else if (!read_options.total_order_seek && prefix_extractor &&
|
} else if (!read_options.total_order_seek && prefix_extractor &&
|
||||||
rep_->table_properties->prefix_extractor_name.compare(
|
rep_->table_properties->prefix_extractor_name ==
|
||||||
prefix_extractor->Name()) == 0 &&
|
prefix_extractor->AsString() &&
|
||||||
prefix_extractor->InDomain(user_key_without_ts) &&
|
prefix_extractor->InDomain(user_key_without_ts) &&
|
||||||
!filter->PrefixMayMatch(
|
!filter->PrefixMayMatch(
|
||||||
prefix_extractor->Transform(user_key_without_ts),
|
prefix_extractor->Transform(user_key_without_ts),
|
||||||
@ -2265,8 +2275,8 @@ void BlockBasedTable::FullFilterKeysMayMatch(
|
|||||||
rep_->level);
|
rep_->level);
|
||||||
}
|
}
|
||||||
} else if (!read_options.total_order_seek && prefix_extractor &&
|
} else if (!read_options.total_order_seek && prefix_extractor &&
|
||||||
rep_->table_properties->prefix_extractor_name.compare(
|
rep_->table_properties->prefix_extractor_name ==
|
||||||
prefix_extractor->Name()) == 0) {
|
prefix_extractor->AsString()) {
|
||||||
filter->PrefixesMayMatch(range, prefix_extractor, kNotValid, false,
|
filter->PrefixesMayMatch(range, prefix_extractor, kNotValid, false,
|
||||||
lookup_context);
|
lookup_context);
|
||||||
RecordTick(rep_->ioptions.stats, BLOOM_FILTER_PREFIX_CHECKED, before_keys);
|
RecordTick(rep_->ioptions.stats, BLOOM_FILTER_PREFIX_CHECKED, before_keys);
|
||||||
|
@ -104,9 +104,10 @@ PlainTableBuilder::PlainTableBuilder(
|
|||||||
ROCKS_LOG_INFO(ioptions_.logger, "db_host_id property will not be set");
|
ROCKS_LOG_INFO(ioptions_.logger, "db_host_id property will not be set");
|
||||||
}
|
}
|
||||||
properties_.orig_file_number = file_number;
|
properties_.orig_file_number = file_number;
|
||||||
properties_.prefix_extractor_name = moptions_.prefix_extractor != nullptr
|
properties_.prefix_extractor_name =
|
||||||
? moptions_.prefix_extractor->Name()
|
moptions_.prefix_extractor != nullptr
|
||||||
: "nullptr";
|
? moptions_.prefix_extractor->AsString()
|
||||||
|
: "nullptr";
|
||||||
|
|
||||||
std::string val;
|
std::string val;
|
||||||
PutFixed32(&val, static_cast<uint32_t>(encoder_.GetEncodingType()));
|
PutFixed32(&val, static_cast<uint32_t>(encoder_.GetEncodingType()));
|
||||||
|
@ -149,8 +149,7 @@ Status PlainTableReader::Open(
|
|||||||
return Status::InvalidArgument(
|
return Status::InvalidArgument(
|
||||||
"Prefix extractor is missing when opening a PlainTable built "
|
"Prefix extractor is missing when opening a PlainTable built "
|
||||||
"using a prefix extractor");
|
"using a prefix extractor");
|
||||||
} else if (prefix_extractor_in_file.compare(prefix_extractor->Name()) !=
|
} else if (prefix_extractor_in_file != prefix_extractor->AsString()) {
|
||||||
0) {
|
|
||||||
return Status::InvalidArgument(
|
return Status::InvalidArgument(
|
||||||
"Prefix extractor given doesn't match the one used to build "
|
"Prefix extractor given doesn't match the one used to build "
|
||||||
"PlainTable");
|
"PlainTable");
|
||||||
|
221
util/slice.cc
221
util/slice.cc
@ -7,32 +7,47 @@
|
|||||||
// Use of this source code is governed by a BSD-style license that can be
|
// Use of this source code is governed by a BSD-style license that can be
|
||||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||||
|
|
||||||
#include <algorithm>
|
|
||||||
#include "rocksdb/slice_transform.h"
|
|
||||||
#include "rocksdb/slice.h"
|
#include "rocksdb/slice.h"
|
||||||
#include "util/string_util.h"
|
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
|
|
||||||
|
#include <algorithm>
|
||||||
|
|
||||||
|
#include "rocksdb/convenience.h"
|
||||||
|
#include "rocksdb/slice_transform.h"
|
||||||
|
#include "rocksdb/utilities/object_registry.h"
|
||||||
|
#include "rocksdb/utilities/options_type.h"
|
||||||
|
#include "util/string_util.h"
|
||||||
|
|
||||||
namespace ROCKSDB_NAMESPACE {
|
namespace ROCKSDB_NAMESPACE {
|
||||||
|
|
||||||
namespace {
|
namespace {
|
||||||
|
static std::unordered_map<std::string, OptionTypeInfo>
|
||||||
|
slice_transform_length_info = {
|
||||||
|
#ifndef ROCKSDB_LITE
|
||||||
|
{"length",
|
||||||
|
{0, OptionType::kSizeT, OptionVerificationType::kNormal,
|
||||||
|
OptionTypeFlags::kDontSerialize | OptionTypeFlags::kCompareNever}},
|
||||||
|
#endif // ROCKSDB_LITE
|
||||||
|
};
|
||||||
|
|
||||||
class FixedPrefixTransform : public SliceTransform {
|
class FixedPrefixTransform : public SliceTransform {
|
||||||
private:
|
private:
|
||||||
size_t prefix_len_;
|
size_t prefix_len_;
|
||||||
std::string name_;
|
|
||||||
|
|
||||||
public:
|
public:
|
||||||
explicit FixedPrefixTransform(size_t prefix_len)
|
explicit FixedPrefixTransform(size_t prefix_len) : prefix_len_(prefix_len) {
|
||||||
: prefix_len_(prefix_len),
|
RegisterOptions(Name(), &prefix_len_, &slice_transform_length_info);
|
||||||
// Note that if any part of the name format changes, it will require
|
}
|
||||||
// changes on options_helper in order to make RocksDBOptionsParser work
|
|
||||||
// for the new change.
|
|
||||||
// TODO(yhchiang): move serialization / deserializaion code inside
|
|
||||||
// the class implementation itself.
|
|
||||||
name_("rocksdb.FixedPrefix." + ToString(prefix_len_)) {}
|
|
||||||
|
|
||||||
const char* Name() const override { return name_.c_str(); }
|
static const char* kClassName() { return "rocksdb.FixedPrefix"; }
|
||||||
|
static const char* kNickName() { return "fixed"; }
|
||||||
|
const char* Name() const override { return kClassName(); }
|
||||||
|
const char* NickName() const override { return kNickName(); }
|
||||||
|
|
||||||
|
std::string GetId() const override {
|
||||||
|
return std::string(Name()) + "." + ROCKSDB_NAMESPACE::ToString(prefix_len_);
|
||||||
|
}
|
||||||
|
|
||||||
Slice Transform(const Slice& src) const override {
|
Slice Transform(const Slice& src) const override {
|
||||||
assert(InDomain(src));
|
assert(InDomain(src));
|
||||||
@ -60,19 +75,19 @@ class FixedPrefixTransform : public SliceTransform {
|
|||||||
class CappedPrefixTransform : public SliceTransform {
|
class CappedPrefixTransform : public SliceTransform {
|
||||||
private:
|
private:
|
||||||
size_t cap_len_;
|
size_t cap_len_;
|
||||||
std::string name_;
|
|
||||||
|
|
||||||
public:
|
public:
|
||||||
explicit CappedPrefixTransform(size_t cap_len)
|
explicit CappedPrefixTransform(size_t cap_len) : cap_len_(cap_len) {
|
||||||
: cap_len_(cap_len),
|
RegisterOptions(Name(), &cap_len_, &slice_transform_length_info);
|
||||||
// Note that if any part of the name format changes, it will require
|
}
|
||||||
// changes on options_helper in order to make RocksDBOptionsParser work
|
|
||||||
// for the new change.
|
|
||||||
// TODO(yhchiang): move serialization / deserializaion code inside
|
|
||||||
// the class implementation itself.
|
|
||||||
name_("rocksdb.CappedPrefix." + ToString(cap_len_)) {}
|
|
||||||
|
|
||||||
const char* Name() const override { return name_.c_str(); }
|
static const char* kClassName() { return "rocksdb.CappedPrefix"; }
|
||||||
|
static const char* kNickName() { return "capped"; }
|
||||||
|
const char* Name() const override { return kClassName(); }
|
||||||
|
const char* NickName() const override { return kNickName(); }
|
||||||
|
std::string GetId() const override {
|
||||||
|
return std::string(Name()) + "." + ROCKSDB_NAMESPACE::ToString(cap_len_);
|
||||||
|
}
|
||||||
|
|
||||||
Slice Transform(const Slice& src) const override {
|
Slice Transform(const Slice& src) const override {
|
||||||
assert(InDomain(src));
|
assert(InDomain(src));
|
||||||
@ -99,7 +114,8 @@ class NoopTransform : public SliceTransform {
|
|||||||
public:
|
public:
|
||||||
explicit NoopTransform() { }
|
explicit NoopTransform() { }
|
||||||
|
|
||||||
const char* Name() const override { return "rocksdb.Noop"; }
|
static const char* kClassName() { return "rocksdb.Noop"; }
|
||||||
|
const char* Name() const override { return kClassName(); }
|
||||||
|
|
||||||
Slice Transform(const Slice& src) const override { return src; }
|
Slice Transform(const Slice& src) const override { return src; }
|
||||||
|
|
||||||
@ -112,6 +128,151 @@ class NoopTransform : public SliceTransform {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
} // end namespace
|
||||||
|
|
||||||
|
const SliceTransform* NewFixedPrefixTransform(size_t prefix_len) {
|
||||||
|
return new FixedPrefixTransform(prefix_len);
|
||||||
|
}
|
||||||
|
|
||||||
|
const SliceTransform* NewCappedPrefixTransform(size_t cap_len) {
|
||||||
|
return new CappedPrefixTransform(cap_len);
|
||||||
|
}
|
||||||
|
|
||||||
|
const SliceTransform* NewNoopTransform() { return new NoopTransform; }
|
||||||
|
|
||||||
|
#ifndef ROCKSDB_LITE
|
||||||
|
static int RegisterBuiltinSliceTransform(ObjectLibrary& library,
|
||||||
|
const std::string& /*arg*/) {
|
||||||
|
library.Register<const SliceTransform>(
|
||||||
|
NoopTransform::kClassName(),
|
||||||
|
[](const std::string& /*uri*/,
|
||||||
|
std::unique_ptr<const SliceTransform>* guard,
|
||||||
|
std::string* /*errmsg*/) {
|
||||||
|
guard->reset(NewNoopTransform());
|
||||||
|
return guard->get();
|
||||||
|
});
|
||||||
|
library.Register<const SliceTransform>(
|
||||||
|
std::string(FixedPrefixTransform::kNickName()) + ":[0-9]+",
|
||||||
|
[](const std::string& uri, std::unique_ptr<const SliceTransform>* guard,
|
||||||
|
std::string* /*errmsg*/) {
|
||||||
|
auto colon = uri.find(":");
|
||||||
|
auto len = ParseSizeT(uri.substr(colon + 1));
|
||||||
|
guard->reset(NewFixedPrefixTransform(len));
|
||||||
|
return guard->get();
|
||||||
|
});
|
||||||
|
library.Register<const SliceTransform>(
|
||||||
|
FixedPrefixTransform::kClassName(),
|
||||||
|
[](const std::string& /*uri*/,
|
||||||
|
std::unique_ptr<const SliceTransform>* guard,
|
||||||
|
std::string* /*errmsg*/) {
|
||||||
|
guard->reset(NewFixedPrefixTransform(0));
|
||||||
|
return guard->get();
|
||||||
|
});
|
||||||
|
library.Register<const SliceTransform>(
|
||||||
|
std::string(FixedPrefixTransform::kClassName()) + "\\.[0-9]+",
|
||||||
|
[](const std::string& uri, std::unique_ptr<const SliceTransform>* guard,
|
||||||
|
std::string* /*errmsg*/) {
|
||||||
|
auto len = ParseSizeT(
|
||||||
|
uri.substr(strlen(FixedPrefixTransform::kClassName()) + 1));
|
||||||
|
guard->reset(NewFixedPrefixTransform(len));
|
||||||
|
return guard->get();
|
||||||
|
});
|
||||||
|
library.Register<const SliceTransform>(
|
||||||
|
std::string(CappedPrefixTransform::kNickName()) + ":[0-9]+",
|
||||||
|
[](const std::string& uri, std::unique_ptr<const SliceTransform>* guard,
|
||||||
|
std::string* /*errmsg*/) {
|
||||||
|
auto colon = uri.find(":");
|
||||||
|
auto len = ParseSizeT(uri.substr(colon + 1));
|
||||||
|
guard->reset(NewCappedPrefixTransform(len));
|
||||||
|
return guard->get();
|
||||||
|
});
|
||||||
|
library.Register<const SliceTransform>(
|
||||||
|
std::string(CappedPrefixTransform::kClassName()) + "(\\.[0-9]+)?",
|
||||||
|
[](const std::string& uri, std::unique_ptr<const SliceTransform>* guard,
|
||||||
|
std::string* /*errmsg*/) {
|
||||||
|
if (uri == CappedPrefixTransform::kClassName()) {
|
||||||
|
guard->reset(NewCappedPrefixTransform(0));
|
||||||
|
} else { // Length + "."
|
||||||
|
auto len = ParseSizeT(
|
||||||
|
uri.substr(strlen(CappedPrefixTransform::kClassName()) + 1));
|
||||||
|
guard->reset(NewCappedPrefixTransform(len));
|
||||||
|
}
|
||||||
|
return guard->get();
|
||||||
|
});
|
||||||
|
return 5;
|
||||||
|
}
|
||||||
|
#endif // ROCKSDB_LITE
|
||||||
|
|
||||||
|
Status SliceTransform::CreateFromString(
|
||||||
|
const ConfigOptions& config_options, const std::string& value,
|
||||||
|
std::shared_ptr<const SliceTransform>* result) {
|
||||||
|
#ifndef ROCKSDB_LITE
|
||||||
|
static std::once_flag once;
|
||||||
|
std::call_once(once, [&]() {
|
||||||
|
RegisterBuiltinSliceTransform(*(ObjectLibrary::Default().get()), "");
|
||||||
|
});
|
||||||
|
#endif // ROCKSDB_LITE
|
||||||
|
std::string id;
|
||||||
|
std::unordered_map<std::string, std::string> opt_map;
|
||||||
|
Status status = Customizable::GetOptionsMap(config_options, result->get(),
|
||||||
|
value, &id, &opt_map);
|
||||||
|
if (!status.ok()) { // GetOptionsMap failed
|
||||||
|
return status;
|
||||||
|
}
|
||||||
|
#ifndef ROCKSDB_LITE
|
||||||
|
status = config_options.registry->NewSharedObject(id, result);
|
||||||
|
#else
|
||||||
|
auto Matches = [](const std::string& input, size_t size, const char* pattern,
|
||||||
|
char sep) {
|
||||||
|
auto plen = strlen(pattern);
|
||||||
|
return (size > plen + 2 && input[plen] == sep &&
|
||||||
|
StartsWith(input, pattern));
|
||||||
|
};
|
||||||
|
|
||||||
|
auto size = id.size();
|
||||||
|
if (id == NoopTransform::kClassName()) {
|
||||||
|
result->reset(NewNoopTransform());
|
||||||
|
} else if (Matches(id, size, FixedPrefixTransform::kNickName(), ':')) {
|
||||||
|
auto fixed = strlen(FixedPrefixTransform::kNickName());
|
||||||
|
auto len = ParseSizeT(id.substr(fixed + 1));
|
||||||
|
result->reset(NewFixedPrefixTransform(len));
|
||||||
|
} else if (Matches(id, size, CappedPrefixTransform::kNickName(), ':')) {
|
||||||
|
auto capped = strlen(CappedPrefixTransform::kNickName());
|
||||||
|
auto len = ParseSizeT(id.substr(capped + 1));
|
||||||
|
result->reset(NewCappedPrefixTransform(len));
|
||||||
|
} else if (Matches(id, size, CappedPrefixTransform::kClassName(), '.')) {
|
||||||
|
auto capped = strlen(CappedPrefixTransform::kClassName());
|
||||||
|
auto len = ParseSizeT(id.substr(capped + 1));
|
||||||
|
result->reset(NewCappedPrefixTransform(len));
|
||||||
|
} else if (Matches(id, size, FixedPrefixTransform::kClassName(), '.')) {
|
||||||
|
auto fixed = strlen(FixedPrefixTransform::kClassName());
|
||||||
|
auto len = ParseSizeT(id.substr(fixed + 1));
|
||||||
|
result->reset(NewFixedPrefixTransform(len));
|
||||||
|
} else {
|
||||||
|
status = Status::NotSupported("Cannot load object in LITE mode ", id);
|
||||||
|
}
|
||||||
|
#endif // ROCKSDB_LITE
|
||||||
|
if (!status.ok()) {
|
||||||
|
if (config_options.ignore_unsupported_options && status.IsNotSupported()) {
|
||||||
|
return Status::OK();
|
||||||
|
} else {
|
||||||
|
return status;
|
||||||
|
}
|
||||||
|
} else if (result->get() != nullptr) {
|
||||||
|
SliceTransform* transform = const_cast<SliceTransform*>(result->get());
|
||||||
|
status = transform->ConfigureFromMap(config_options, opt_map);
|
||||||
|
}
|
||||||
|
return status;
|
||||||
|
} // namespace ROCKSDB_NAMESPACE
|
||||||
|
|
||||||
|
std::string SliceTransform::AsString() const {
|
||||||
|
#ifndef ROCKSDB_LITE
|
||||||
|
ConfigOptions config_options;
|
||||||
|
config_options.delimiter = ";";
|
||||||
|
return ToString(config_options);
|
||||||
|
#else
|
||||||
|
return GetId();
|
||||||
|
#endif // ROCKSDB_LITE
|
||||||
}
|
}
|
||||||
|
|
||||||
// 2 small internal utility functions, for efficient hex conversions
|
// 2 small internal utility functions, for efficient hex conversions
|
||||||
@ -197,18 +358,6 @@ bool Slice::DecodeHex(std::string* result) const {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
const SliceTransform* NewFixedPrefixTransform(size_t prefix_len) {
|
|
||||||
return new FixedPrefixTransform(prefix_len);
|
|
||||||
}
|
|
||||||
|
|
||||||
const SliceTransform* NewCappedPrefixTransform(size_t cap_len) {
|
|
||||||
return new CappedPrefixTransform(cap_len);
|
|
||||||
}
|
|
||||||
|
|
||||||
const SliceTransform* NewNoopTransform() {
|
|
||||||
return new NoopTransform;
|
|
||||||
}
|
|
||||||
|
|
||||||
PinnableSlice::PinnableSlice(PinnableSlice&& other) {
|
PinnableSlice::PinnableSlice(PinnableSlice&& other) {
|
||||||
*this = std::move(other);
|
*this = std::move(other);
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user