Make WalFilter, SstPartitionerFactory, FileChecksumGenFactory, and TableProperties Customizable (#8638)

Summary: Pull Request resolved: https://github.com/facebook/rocksdb/pull/8638

Reviewed By: zhichao-cao

Differential Revision: D31024729

Pulled By: mrambacher

fbshipit-source-id: 954c04ccab0b8dee64050a27aadf78ed119106c0
This commit is contained in:
mrambacher 2021-09-28 05:30:32 -07:00 committed by Facebook GitHub Bot
parent b88109db19
commit 7fd68b7c39
21 changed files with 585 additions and 52 deletions

View File

@ -903,6 +903,7 @@ set(SOURCES
utilities/transactions/write_unprepared_txn.cc
utilities/transactions/write_unprepared_txn_db.cc
utilities/ttl/db_ttl_impl.cc
utilities/wal_filter.cc
utilities/write_batch_with_index/write_batch_with_index.cc
utilities/write_batch_with_index/write_batch_with_index_internal.cc)

View File

@ -9,6 +9,7 @@
### Public API change
* Made SystemClock extend the Customizable class and added a CreateFromString method. Implementations need to be registered with the ObjectRegistry and to implement a Name() method in order to be created via this method.
* Made SliceTransform extend the Customizable class and added a CreateFromString method. Implementations need to be registered with the ObjectRegistry and to implement a Name() method in order to be created via this method. The Capped and Prefixed transform classes return a short name (no length); use GetId for the fully qualified name.
* Made FileChecksumGenFactory, SstPartitionerFactory, TablePropertiesCollectorFactory, and WalFilter extend the Customizable class and added a CreateFromString method.
## 6.25.0 (2021-09-20)
### Bug Fixes

View File

@ -435,6 +435,7 @@ cpp_library(
"utilities/transactions/write_unprepared_txn.cc",
"utilities/transactions/write_unprepared_txn_db.cc",
"utilities/ttl/db_ttl_impl.cc",
"utilities/wal_filter.cc",
"utilities/write_batch_with_index/write_batch_with_index.cc",
"utilities/write_batch_with_index/write_batch_with_index_internal.cc",
],
@ -757,6 +758,7 @@ cpp_library(
"utilities/transactions/write_unprepared_txn.cc",
"utilities/transactions/write_unprepared_txn_db.cc",
"utilities/ttl/db_ttl_impl.cc",
"utilities/wal_filter.cc",
"utilities/write_batch_with_index/write_batch_with_index.cc",
"utilities/write_batch_with_index/write_batch_with_index_internal.cc",
],

View File

@ -8,7 +8,24 @@
#include <algorithm>
#include "rocksdb/utilities/customizable_util.h"
#include "rocksdb/utilities/object_registry.h"
#include "rocksdb/utilities/options_type.h"
namespace ROCKSDB_NAMESPACE {
static std::unordered_map<std::string, OptionTypeInfo>
sst_fixed_prefix_type_info = {
#ifndef ROCKSDB_LITE
{"length",
{0, OptionType::kSizeT, OptionVerificationType::kNormal,
OptionTypeFlags::kNone}},
#endif // ROCKSDB_LITE
};
SstPartitionerFixedPrefixFactory::SstPartitionerFixedPrefixFactory(size_t len)
: len_(len) {
RegisterOptions("Length", &len_, &sst_fixed_prefix_type_info);
}
PartitionerResult SstPartitionerFixedPrefix::ShouldPartition(
const PartitionerRequest& request) {
@ -41,4 +58,33 @@ std::shared_ptr<SstPartitionerFactory> NewSstPartitionerFixedPrefixFactory(
return std::make_shared<SstPartitionerFixedPrefixFactory>(prefix_len);
}
#ifndef ROCKSDB_LITE
namespace {
static int RegisterSstPartitionerFactories(ObjectLibrary& library,
const std::string& /*arg*/) {
library.Register<SstPartitionerFactory>(
SstPartitionerFixedPrefixFactory::kClassName(),
[](const std::string& /*uri*/,
std::unique_ptr<SstPartitionerFactory>* guard,
std::string* /* errmsg */) {
guard->reset(new SstPartitionerFixedPrefixFactory(0));
return guard->get();
});
return 1;
}
} // namespace
#endif // ROCKSDB_LITE
Status SstPartitionerFactory::CreateFromString(
const ConfigOptions& options, const std::string& value,
std::shared_ptr<SstPartitionerFactory>* result) {
#ifndef ROCKSDB_LITE
static std::once_flag once;
std::call_once(once, [&]() {
RegisterSstPartitionerFactories(*(ObjectLibrary::Default().get()), "");
});
#endif // ROCKSDB_LITE
return LoadSharedObject<SstPartitionerFactory>(options, value, nullptr,
result);
}
} // namespace ROCKSDB_NAMESPACE

View File

@ -94,6 +94,37 @@ TEST_F(DBTablePropertiesTest, GetPropertiesOfAllTablesTest) {
VerifyTableProperties(db_, 10 + 11 + 12 + 13);
}
TEST_F(DBTablePropertiesTest, CreateOnDeletionCollectorFactory) {
ConfigOptions options;
options.ignore_unsupported_options = false;
std::shared_ptr<TablePropertiesCollectorFactory> factory;
std::string id = CompactOnDeletionCollectorFactory::kClassName();
ASSERT_OK(
TablePropertiesCollectorFactory::CreateFromString(options, id, &factory));
auto del_factory = factory->CheckedCast<CompactOnDeletionCollectorFactory>();
ASSERT_NE(del_factory, nullptr);
ASSERT_EQ(0U, del_factory->GetWindowSize());
ASSERT_EQ(0U, del_factory->GetDeletionTrigger());
ASSERT_EQ(0.0, del_factory->GetDeletionRatio());
ASSERT_OK(TablePropertiesCollectorFactory::CreateFromString(
options, "window_size=100; deletion_trigger=90; id=" + id, &factory));
del_factory = factory->CheckedCast<CompactOnDeletionCollectorFactory>();
ASSERT_NE(del_factory, nullptr);
ASSERT_EQ(100U, del_factory->GetWindowSize());
ASSERT_EQ(90U, del_factory->GetDeletionTrigger());
ASSERT_EQ(0.0, del_factory->GetDeletionRatio());
ASSERT_OK(TablePropertiesCollectorFactory::CreateFromString(
options,
"window_size=100; deletion_trigger=90; deletion_ratio=0.5; id=" + id,
&factory));
del_factory = factory->CheckedCast<CompactOnDeletionCollectorFactory>();
ASSERT_NE(del_factory, nullptr);
ASSERT_EQ(100U, del_factory->GetWindowSize());
ASSERT_EQ(90U, del_factory->GetDeletionTrigger());
ASSERT_EQ(0.5, del_factory->GetDeletionRatio());
}
TablePropertiesCollection
DBTablePropertiesTest::TestGetPropertiesOfTablesInRange(
std::vector<Range> ranges, std::size_t* num_properties,

View File

@ -9,8 +9,11 @@
#pragma once
#include "rocksdb/env.h"
#include "rocksdb/statistics.h"
#include <stdint.h>
#include <string>
#include "rocksdb/rocksdb_namespace.h"
namespace ROCKSDB_NAMESPACE {

View File

@ -261,7 +261,6 @@ class Configurable {
virtual Status ValidateOptions(const DBOptions& db_opts,
const ColumnFamilyOptions& cf_opts) const;
// Splits the input opt_value into the ID field and the remaining options.
// The input opt_value can be in the form of "name" or "name=value
// [;name=value]". The first form uses the "name" as an id with no options The

View File

@ -14,6 +14,7 @@
#include <string>
#include <vector>
#include "rocksdb/customizable.h"
#include "rocksdb/status.h"
namespace ROCKSDB_NAMESPACE {
@ -63,9 +64,13 @@ class FileChecksumGenerator {
};
// Create the FileChecksumGenerator object for each SST file.
class FileChecksumGenFactory {
class FileChecksumGenFactory : public Customizable {
public:
virtual ~FileChecksumGenFactory() {}
static const char* Type() { return "FileChecksumGenFactory"; }
static Status CreateFromString(
const ConfigOptions& options, const std::string& value,
std::shared_ptr<FileChecksumGenFactory>* result);
// Create a new FileChecksumGenerator.
virtual std::unique_ptr<FileChecksumGenerator> CreateFileChecksumGenerator(

View File

@ -9,6 +9,7 @@
#include <memory>
#include <string>
#include "rocksdb/customizable.h"
#include "rocksdb/rocksdb_namespace.h"
#include "rocksdb/slice.h"
@ -77,9 +78,13 @@ class SstPartitioner {
};
};
class SstPartitionerFactory {
class SstPartitionerFactory : public Customizable {
public:
virtual ~SstPartitionerFactory() {}
static const char* Type() { return "SstPartitionerFactory"; }
static Status CreateFromString(
const ConfigOptions& options, const std::string& value,
std::shared_ptr<SstPartitionerFactory>* result);
virtual std::unique_ptr<SstPartitioner> CreatePartitioner(
const SstPartitioner::Context& context) const = 0;
@ -114,13 +119,12 @@ class SstPartitionerFixedPrefix : public SstPartitioner {
*/
class SstPartitionerFixedPrefixFactory : public SstPartitionerFactory {
public:
explicit SstPartitionerFixedPrefixFactory(size_t len) : len_(len) {}
explicit SstPartitionerFixedPrefixFactory(size_t len);
virtual ~SstPartitionerFixedPrefixFactory() {}
const char* Name() const override {
return "SstPartitionerFixedPrefixFactory";
}
static const char* kClassName() { return "SstPartitionerFixedPrefixFactory"; }
const char* Name() const override { return kClassName(); }
std::unique_ptr<SstPartitioner> CreatePartitioner(
const SstPartitioner::Context& /* context */) const override;

View File

@ -5,8 +5,12 @@
#pragma once
#include <stdint.h>
#include <map>
#include <memory>
#include <string>
#include "rocksdb/customizable.h"
#include "rocksdb/status.h"
#include "rocksdb/types.h"
@ -129,7 +133,7 @@ class TablePropertiesCollector {
// Constructs TablePropertiesCollector. Internals create a new
// TablePropertiesCollector for each new table
class TablePropertiesCollectorFactory {
class TablePropertiesCollectorFactory : public Customizable {
public:
struct Context {
uint32_t column_family_id;
@ -137,6 +141,11 @@ class TablePropertiesCollectorFactory {
};
virtual ~TablePropertiesCollectorFactory() {}
static const char* Type() { return "TablePropertiesCollectorFactory"; }
static Status CreateFromString(
const ConfigOptions& options, const std::string& value,
std::shared_ptr<TablePropertiesCollectorFactory>* result);
// has to be thread-safe
virtual TablePropertiesCollector* CreateTablePropertiesCollector(
TablePropertiesCollectorFactory::Context context) = 0;

View File

@ -19,40 +19,6 @@ namespace ROCKSDB_NAMESPACE {
class CompactOnDeletionCollectorFactory
: public TablePropertiesCollectorFactory {
public:
~CompactOnDeletionCollectorFactory() {}
TablePropertiesCollector* CreateTablePropertiesCollector(
TablePropertiesCollectorFactory::Context context) override;
// Change the value of sliding_window_size "N"
// Setting it to 0 disables the delete triggered compaction
void SetWindowSize(size_t sliding_window_size) {
sliding_window_size_.store(sliding_window_size);
}
// Change the value of deletion_trigger "D"
void SetDeletionTrigger(size_t deletion_trigger) {
deletion_trigger_.store(deletion_trigger);
}
// Change deletion ratio.
// @param deletion_ratio, if <= 0 or > 1, disable triggering compaction
// based on deletion ratio.
void SetDeletionRatio(double deletion_ratio) {
deletion_ratio_.store(deletion_ratio);
}
const char* Name() const override {
return "CompactOnDeletionCollector";
}
std::string ToString() const override;
private:
friend std::shared_ptr<CompactOnDeletionCollectorFactory>
NewCompactOnDeletionCollectorFactory(size_t sliding_window_size,
size_t deletion_trigger,
double deletion_ratio);
// A factory of a table property collector that marks a SST
// file as need-compaction when it observe at least "D" deletion
// entries in any "N" consecutive entries, or the ratio of tombstone
@ -64,11 +30,40 @@ class CompactOnDeletionCollectorFactory
// based on deletion ratio.
CompactOnDeletionCollectorFactory(size_t sliding_window_size,
size_t deletion_trigger,
double deletion_ratio)
: sliding_window_size_(sliding_window_size),
deletion_trigger_(deletion_trigger),
deletion_ratio_(deletion_ratio) {}
double deletion_ratio);
~CompactOnDeletionCollectorFactory() {}
TablePropertiesCollector* CreateTablePropertiesCollector(
TablePropertiesCollectorFactory::Context context) override;
// Change the value of sliding_window_size "N"
// Setting it to 0 disables the delete triggered compaction
void SetWindowSize(size_t sliding_window_size) {
sliding_window_size_.store(sliding_window_size);
}
size_t GetWindowSize() const { return sliding_window_size_.load(); }
// Change the value of deletion_trigger "D"
void SetDeletionTrigger(size_t deletion_trigger) {
deletion_trigger_.store(deletion_trigger);
}
size_t GetDeletionTrigger() const { return deletion_trigger_.load(); }
// Change deletion ratio.
// @param deletion_ratio, if <= 0 or > 1, disable triggering compaction
// based on deletion ratio.
void SetDeletionRatio(double deletion_ratio) {
deletion_ratio_.store(deletion_ratio);
}
double GetDeletionRatio() const { return deletion_ratio_.load(); }
static const char* kClassName() { return "CompactOnDeletionCollector"; }
const char* Name() const override { return kClassName(); }
std::string ToString() const override;
private:
std::atomic<size_t> sliding_window_size_;
std::atomic<size_t> deletion_trigger_;
std::atomic<double> deletion_ratio_;

View File

@ -8,17 +8,22 @@
#include <map>
#include <string>
#include "rocksdb/customizable.h"
#include "rocksdb/rocksdb_namespace.h"
namespace ROCKSDB_NAMESPACE {
class WriteBatch;
struct ConfigOptions;
// WALFilter allows an application to inspect write-ahead-log (WAL)
// records or modify their processing on recovery.
// Please see the details below.
class WalFilter {
class WalFilter : public Customizable {
public:
static const char* Type() { return "WalFilter"; }
static Status CreateFromString(const ConfigOptions& options,
const std::string& value, WalFilter** result);
enum class WalProcessingOption {
// Continue processing as usual
kContinueProcessing = 0,

View File

@ -673,6 +673,14 @@ static std::unordered_map<std::string, OptionTypeInfo>
return Status::NotFound("Mismatched table option: ", name);
}
}}},
{"table_properties_collectors",
OptionTypeInfo::Vector<
std::shared_ptr<TablePropertiesCollectorFactory>>(
offset_of(
&ImmutableCFOptions::table_properties_collector_factories),
OptionVerificationType::kByName, OptionTypeFlags::kNone,
OptionTypeInfo::AsCustomSharedPtr<TablePropertiesCollectorFactory>(
0, OptionVerificationType::kByName, OptionTypeFlags::kNone))},
{"compaction_filter",
OptionTypeInfo::AsCustomRawPtr<const CompactionFilter>(
offset_of(&ImmutableCFOptions::compaction_filter),
@ -694,6 +702,10 @@ static std::unordered_map<std::string, OptionTypeInfo>
{offset_of(&ImmutableCFOptions::compaction_pri),
OptionType::kCompactionPri, OptionVerificationType::kNormal,
OptionTypeFlags::kNone}},
{"sst_partitioner_factory",
OptionTypeInfo::AsCustomSharedPtr<SstPartitionerFactory>(
offset_of(&ImmutableCFOptions::sst_partitioner_factory),
OptionVerificationType::kByName, OptionTypeFlags::kAllowNull)},
};
const std::string OptionsHelper::kCFOptionsName = "ColumnFamilyOptions";

View File

@ -20,9 +20,11 @@
#include "port/stack_trace.h"
#include "rocksdb/convenience.h"
#include "rocksdb/env_encryption.h"
#include "rocksdb/file_checksum.h"
#include "rocksdb/flush_block_policy.h"
#include "rocksdb/secondary_cache.h"
#include "rocksdb/slice_transform.h"
#include "rocksdb/sst_partitioner.h"
#include "rocksdb/statistics.h"
#include "rocksdb/utilities/customizable_util.h"
#include "rocksdb/utilities/object_registry.h"
@ -32,6 +34,7 @@
#include "test_util/mock_time_env.h"
#include "test_util/testharness.h"
#include "test_util/testutil.h"
#include "util/file_checksum_helper.h"
#include "util/string_util.h"
#include "utilities/compaction_filters/remove_emptyvalue_compactionfilter.h"
@ -1297,6 +1300,41 @@ class MockCipher : public BlockCipher {
Status Decrypt(char* data) override { return Encrypt(data); }
};
#endif // ROCKSDB_LITE
class MockTablePropertiesCollectorFactory
: public TablePropertiesCollectorFactory {
private:
public:
TablePropertiesCollector* CreateTablePropertiesCollector(
TablePropertiesCollectorFactory::Context /*context*/) override {
return nullptr;
}
static const char* kClassName() { return "Mock"; }
const char* Name() const override { return kClassName(); }
};
class MockSstPartitionerFactory : public SstPartitionerFactory {
public:
static const char* kClassName() { return "Mock"; }
const char* Name() const override { return kClassName(); }
std::unique_ptr<SstPartitioner> CreatePartitioner(
const SstPartitioner::Context& /* context */) const override {
return nullptr;
}
};
class MockFileChecksumGenFactory : public FileChecksumGenFactory {
public:
static const char* kClassName() { return "Mock"; }
const char* Name() const override { return kClassName(); }
std::unique_ptr<FileChecksumGenerator> CreateFileChecksumGenerator(
const FileChecksumGenContext& /*context*/) override {
return nullptr;
}
};
#ifndef ROCKSDB_LITE
static int RegisterLocalObjects(ObjectLibrary& library,
const std::string& /*arg*/) {
size_t num_types;
@ -1367,6 +1405,33 @@ static int RegisterLocalObjects(ObjectLibrary& library,
guard->reset(new TestSecondaryCache());
return guard->get();
});
library.Register<SstPartitionerFactory>(
MockSstPartitionerFactory::kClassName(),
[](const std::string& /*uri*/,
std::unique_ptr<SstPartitionerFactory>* guard,
std::string* /* errmsg */) {
guard->reset(new MockSstPartitionerFactory());
return guard->get();
});
library.Register<FileChecksumGenFactory>(
MockFileChecksumGenFactory::kClassName(),
[](const std::string& /*uri*/,
std::unique_ptr<FileChecksumGenFactory>* guard,
std::string* /* errmsg */) {
guard->reset(new MockFileChecksumGenFactory());
return guard->get();
});
library.Register<TablePropertiesCollectorFactory>(
MockTablePropertiesCollectorFactory::kClassName(),
[](const std::string& /*uri*/,
std::unique_ptr<TablePropertiesCollectorFactory>* guard,
std::string* /* errmsg */) {
guard->reset(new MockTablePropertiesCollectorFactory());
return guard->get();
});
return static_cast<int>(library.GetFactoryCount(&num_types));
}
#endif // !ROCKSDB_LITE
@ -1443,6 +1508,58 @@ TEST_F(LoadCustomizableTest, LoadSecondaryCacheTest) {
}
}
#ifndef ROCKSDB_LITE
TEST_F(LoadCustomizableTest, LoadSstPartitionerFactoryTest) {
std::shared_ptr<SstPartitionerFactory> factory;
ASSERT_NOK(SstPartitionerFactory::CreateFromString(config_options_, "Mock",
&factory));
ASSERT_OK(SstPartitionerFactory::CreateFromString(
config_options_, SstPartitionerFixedPrefixFactory::kClassName(),
&factory));
ASSERT_NE(factory, nullptr);
ASSERT_STREQ(factory->Name(), SstPartitionerFixedPrefixFactory::kClassName());
if (RegisterTests("Test")) {
ASSERT_OK(SstPartitionerFactory::CreateFromString(config_options_, "Mock",
&factory));
ASSERT_NE(factory, nullptr);
ASSERT_STREQ(factory->Name(), "Mock");
}
}
#endif // ROCKSDB_LITE
TEST_F(LoadCustomizableTest, LoadChecksumGenFactoryTest) {
std::shared_ptr<FileChecksumGenFactory> factory;
ASSERT_NOK(FileChecksumGenFactory::CreateFromString(config_options_, "Mock",
&factory));
ASSERT_OK(FileChecksumGenFactory::CreateFromString(
config_options_, FileChecksumGenCrc32cFactory::kClassName(), &factory));
ASSERT_NE(factory, nullptr);
ASSERT_STREQ(factory->Name(), FileChecksumGenCrc32cFactory::kClassName());
if (RegisterTests("Test")) {
ASSERT_OK(FileChecksumGenFactory::CreateFromString(config_options_, "Mock",
&factory));
ASSERT_NE(factory, nullptr);
ASSERT_STREQ(factory->Name(), "Mock");
}
}
TEST_F(LoadCustomizableTest, LoadTablePropertiesCollectorFactoryTest) {
std::shared_ptr<TablePropertiesCollectorFactory> factory;
ASSERT_NOK(TablePropertiesCollectorFactory::CreateFromString(
config_options_, MockTablePropertiesCollectorFactory::kClassName(),
&factory));
if (RegisterTests("Test")) {
ASSERT_OK(TablePropertiesCollectorFactory::CreateFromString(
config_options_, MockTablePropertiesCollectorFactory::kClassName(),
&factory));
ASSERT_NE(factory, nullptr);
ASSERT_STREQ(factory->Name(),
MockTablePropertiesCollectorFactory::kClassName());
}
}
TEST_F(LoadCustomizableTest, LoadComparatorTest) {
const Comparator* bytewise = BytewiseComparator();
const Comparator* reverse = ReverseBytewiseComparator();

View File

@ -171,6 +171,11 @@ static std::unordered_map<std::string, OptionTypeInfo>
{"allow_2pc",
{offsetof(struct ImmutableDBOptions, allow_2pc), OptionType::kBoolean,
OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
{"wal_filter",
OptionTypeInfo::AsCustomRawPtr<WalFilter>(
offsetof(struct ImmutableDBOptions, wal_filter),
OptionVerificationType::kByName,
(OptionTypeFlags::kAllowNull | OptionTypeFlags::kCompareNever))},
{"create_if_missing",
{offsetof(struct ImmutableDBOptions, create_if_missing),
OptionType::kBoolean, OptionVerificationType::kNormal,
@ -444,6 +449,10 @@ static std::unordered_map<std::string, OptionTypeInfo>
{offsetof(struct ImmutableDBOptions, allow_data_in_errors),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone}},
{"file_checksum_gen_factory",
OptionTypeInfo::AsCustomSharedPtr<FileChecksumGenFactory>(
offsetof(struct ImmutableDBOptions, file_checksum_gen_factory),
OptionVerificationType::kByName, OptionTypeFlags::kAllowNull)},
{"statistics",
OptionTypeInfo::AsCustomSharedPtr<Statistics>(
// Statistics should not be compared and can be null

View File

@ -19,6 +19,7 @@
#include "port/port.h"
#include "rocksdb/cache.h"
#include "rocksdb/convenience.h"
#include "rocksdb/file_checksum.h"
#include "rocksdb/memtablerep.h"
#include "rocksdb/utilities/leveldb_options.h"
#include "rocksdb/utilities/object_registry.h"
@ -1960,6 +1961,133 @@ TEST_F(OptionsTest, OnlyMutableCFOptions) {
opt_str, &cf_opts));
delete cf_opts.compaction_filter;
}
TEST_F(OptionsTest, SstPartitionerTest) {
ConfigOptions cfg_opts;
ColumnFamilyOptions cf_opts, new_opt;
std::string opts_str, mismatch;
ASSERT_OK(SstPartitionerFactory::CreateFromString(
cfg_opts, SstPartitionerFixedPrefixFactory::kClassName(),
&cf_opts.sst_partitioner_factory));
ASSERT_NE(cf_opts.sst_partitioner_factory, nullptr);
ASSERT_STREQ(cf_opts.sst_partitioner_factory->Name(),
SstPartitionerFixedPrefixFactory::kClassName());
ASSERT_NOK(GetColumnFamilyOptionsFromString(
cfg_opts, ColumnFamilyOptions(),
std::string("sst_partitioner_factory={id=") +
SstPartitionerFixedPrefixFactory::kClassName() + "; unknown=10;}",
&cf_opts));
ASSERT_OK(GetColumnFamilyOptionsFromString(
cfg_opts, ColumnFamilyOptions(),
std::string("sst_partitioner_factory={id=") +
SstPartitionerFixedPrefixFactory::kClassName() + "; length=10;}",
&cf_opts));
ASSERT_NE(cf_opts.sst_partitioner_factory, nullptr);
ASSERT_STREQ(cf_opts.sst_partitioner_factory->Name(),
SstPartitionerFixedPrefixFactory::kClassName());
ASSERT_OK(GetStringFromColumnFamilyOptions(cfg_opts, cf_opts, &opts_str));
ASSERT_OK(
GetColumnFamilyOptionsFromString(cfg_opts, cf_opts, opts_str, &new_opt));
ASSERT_NE(new_opt.sst_partitioner_factory, nullptr);
ASSERT_STREQ(new_opt.sst_partitioner_factory->Name(),
SstPartitionerFixedPrefixFactory::kClassName());
ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(cfg_opts, cf_opts, new_opt));
ASSERT_TRUE(cf_opts.sst_partitioner_factory->AreEquivalent(
cfg_opts, new_opt.sst_partitioner_factory.get(), &mismatch));
}
TEST_F(OptionsTest, FileChecksumGenFactoryTest) {
ConfigOptions cfg_opts;
DBOptions db_opts, new_opt;
std::string opts_str, mismatch;
auto factory = GetFileChecksumGenCrc32cFactory();
cfg_opts.ignore_unsupported_options = false;
ASSERT_OK(GetStringFromDBOptions(cfg_opts, db_opts, &opts_str));
ASSERT_OK(GetDBOptionsFromString(cfg_opts, db_opts, opts_str, &new_opt));
ASSERT_NE(factory, nullptr);
ASSERT_OK(FileChecksumGenFactory::CreateFromString(
cfg_opts, factory->Name(), &db_opts.file_checksum_gen_factory));
ASSERT_NE(db_opts.file_checksum_gen_factory, nullptr);
ASSERT_STREQ(db_opts.file_checksum_gen_factory->Name(), factory->Name());
ASSERT_NOK(GetDBOptionsFromString(
cfg_opts, DBOptions(), "file_checksum_gen_factory=unknown", &db_opts));
ASSERT_OK(GetDBOptionsFromString(
cfg_opts, DBOptions(),
std::string("file_checksum_gen_factory=") + factory->Name(), &db_opts));
ASSERT_NE(db_opts.file_checksum_gen_factory, nullptr);
ASSERT_STREQ(db_opts.file_checksum_gen_factory->Name(), factory->Name());
ASSERT_OK(GetStringFromDBOptions(cfg_opts, db_opts, &opts_str));
ASSERT_OK(GetDBOptionsFromString(cfg_opts, db_opts, opts_str, &new_opt));
ASSERT_NE(new_opt.file_checksum_gen_factory, nullptr);
ASSERT_STREQ(new_opt.file_checksum_gen_factory->Name(), factory->Name());
ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(cfg_opts, db_opts, new_opt));
ASSERT_TRUE(factory->AreEquivalent(
cfg_opts, new_opt.file_checksum_gen_factory.get(), &mismatch));
ASSERT_TRUE(db_opts.file_checksum_gen_factory->AreEquivalent(
cfg_opts, new_opt.file_checksum_gen_factory.get(), &mismatch));
}
class TestTablePropertiesCollectorFactory
: public TablePropertiesCollectorFactory {
private:
std::string id_;
public:
explicit TestTablePropertiesCollectorFactory(const std::string& id)
: id_(id) {}
TablePropertiesCollector* CreateTablePropertiesCollector(
TablePropertiesCollectorFactory::Context /*context*/) override {
return nullptr;
}
static const char* kClassName() { return "TestCollector"; }
const char* Name() const override { return kClassName(); }
std::string GetId() const override {
return std::string(kClassName()) + ":" + id_;
}
};
TEST_F(OptionsTest, OptionTablePropertiesTest) {
ConfigOptions cfg_opts;
ColumnFamilyOptions orig, copy;
orig.table_properties_collector_factories.push_back(
std::make_shared<TestTablePropertiesCollectorFactory>("1"));
orig.table_properties_collector_factories.push_back(
std::make_shared<TestTablePropertiesCollectorFactory>("2"));
// Push two TablePropertiesCollectorFactories then create a new
// ColumnFamilyOptions based on those settings. The copy should
// have no properties but still match the original
std::string opts_str;
ASSERT_OK(GetStringFromColumnFamilyOptions(cfg_opts, orig, &opts_str));
ASSERT_OK(GetColumnFamilyOptionsFromString(cfg_opts, orig, opts_str, &copy));
ASSERT_EQ(copy.table_properties_collector_factories.size(), 0);
ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(cfg_opts, orig, copy));
// Now register a TablePropertiesCollectorFactory
// Repeat the experiment. The copy should have the same
// properties as the original
cfg_opts.registry->AddLibrary("collector")
->Register<TablePropertiesCollectorFactory>(
std::string(TestTablePropertiesCollectorFactory::kClassName()) +
":.*",
[](const std::string& name,
std::unique_ptr<TablePropertiesCollectorFactory>* guard,
std::string* /* errmsg */) {
std::string id = name.substr(
strlen(TestTablePropertiesCollectorFactory::kClassName()) + 1);
guard->reset(new TestTablePropertiesCollectorFactory(id));
return guard->get();
});
ASSERT_OK(GetColumnFamilyOptionsFromString(cfg_opts, orig, opts_str, &copy));
ASSERT_EQ(copy.table_properties_collector_factories.size(), 2);
ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(cfg_opts, orig, copy));
}
#endif // !ROCKSDB_LITE
TEST_F(OptionsTest, ConvertOptionsTest) {

1
src.mk
View File

@ -285,6 +285,7 @@ LIB_SOURCES = \
utilities/transactions/write_unprepared_txn.cc \
utilities/transactions/write_unprepared_txn_db.cc \
utilities/ttl/db_ttl_impl.cc \
utilities/wal_filter.cc \
utilities/write_batch_with_index/write_batch_with_index.cc \
utilities/write_batch_with_index/write_batch_with_index_internal.cc \

View File

@ -15,6 +15,7 @@
#include "db/version_edit.h"
#include "db/version_edit_handler.h"
#include "file/sequence_file_reader.h"
#include "rocksdb/utilities/customizable_util.h"
namespace ROCKSDB_NAMESPACE {
@ -133,4 +134,39 @@ Status GetFileChecksumsFromManifest(Env* src_env, const std::string& abs_path,
return retriever.status();
}
#ifndef ROCKSDB_LITE
namespace {
static int RegisterFileChecksumGenFactories(ObjectLibrary& library,
const std::string& /*arg*/) {
library.Register<FileChecksumGenFactory>(
FileChecksumGenCrc32cFactory::kClassName(),
[](const std::string& /*uri*/,
std::unique_ptr<FileChecksumGenFactory>* guard,
std::string* /* errmsg */) {
guard->reset(new FileChecksumGenCrc32cFactory());
return guard->get();
});
return 1;
}
} // namespace
#endif // !ROCKSDB_LITE
Status FileChecksumGenFactory::CreateFromString(
const ConfigOptions& options, const std::string& value,
std::shared_ptr<FileChecksumGenFactory>* result) {
#ifndef ROCKSDB_LITE
static std::once_flag once;
std::call_once(once, [&]() {
RegisterFileChecksumGenFactories(*(ObjectLibrary::Default().get()), "");
});
#endif // ROCKSDB_LITE
if (value == FileChecksumGenCrc32cFactory::kClassName()) {
*result = GetFileChecksumGenCrc32cFactory();
return Status::OK();
} else {
Status s = LoadSharedObject<FileChecksumGenFactory>(options, value, nullptr,
result);
return s;
}
}
} // namespace ROCKSDB_NAMESPACE

View File

@ -58,7 +58,8 @@ class FileChecksumGenCrc32cFactory : public FileChecksumGenFactory {
}
}
const char* Name() const override { return "FileChecksumGenCrc32cFactory"; }
static const char* kClassName() { return "FileChecksumGenCrc32cFactory"; }
const char* Name() const override { return kClassName(); }
};
// The default implementaion of FileChecksumList

View File

@ -3,14 +3,19 @@
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
#ifndef ROCKSDB_LITE
#include "utilities/table_properties_collectors/compact_on_deletion_collector.h"
#include <memory>
#include <sstream>
#include "rocksdb/utilities/customizable_util.h"
#include "rocksdb/utilities/object_registry.h"
#include "rocksdb/utilities/options_type.h"
#include "rocksdb/utilities/table_properties_collectors.h"
#include "util/string_util.h"
namespace ROCKSDB_NAMESPACE {
#ifndef ROCKSDB_LITE
CompactOnDeletionCollector::CompactOnDeletionCollector(
size_t sliding_window_size, size_t deletion_trigger, double deletion_ratio)
@ -93,6 +98,74 @@ Status CompactOnDeletionCollector::Finish(
finished_ = true;
return Status::OK();
}
static std::unordered_map<std::string, OptionTypeInfo>
on_deletion_collector_type_info = {
#ifndef ROCKSDB_LITE
{"window_size",
{0, OptionType::kUnknown, OptionVerificationType::kNormal,
OptionTypeFlags::kCompareNever | OptionTypeFlags::kMutable,
[](const ConfigOptions&, const std::string&, const std::string& value,
void* addr) {
auto* factory =
static_cast<CompactOnDeletionCollectorFactory*>(addr);
factory->SetWindowSize(ParseSizeT(value));
return Status::OK();
},
[](const ConfigOptions&, const std::string&, const void* addr,
std::string* value) {
const auto* factory =
static_cast<const CompactOnDeletionCollectorFactory*>(addr);
*value = ToString(factory->GetWindowSize());
return Status::OK();
},
nullptr}},
{"deletion_trigger",
{0, OptionType::kUnknown, OptionVerificationType::kNormal,
OptionTypeFlags::kCompareNever | OptionTypeFlags::kMutable,
[](const ConfigOptions&, const std::string&, const std::string& value,
void* addr) {
auto* factory =
static_cast<CompactOnDeletionCollectorFactory*>(addr);
factory->SetDeletionTrigger(ParseSizeT(value));
return Status::OK();
},
[](const ConfigOptions&, const std::string&, const void* addr,
std::string* value) {
const auto* factory =
static_cast<const CompactOnDeletionCollectorFactory*>(addr);
*value = ToString(factory->GetDeletionTrigger());
return Status::OK();
},
nullptr}},
{"deletion_ratio",
{0, OptionType::kUnknown, OptionVerificationType::kNormal,
OptionTypeFlags::kCompareNever | OptionTypeFlags::kMutable,
[](const ConfigOptions&, const std::string&, const std::string& value,
void* addr) {
auto* factory =
static_cast<CompactOnDeletionCollectorFactory*>(addr);
factory->SetDeletionRatio(ParseDouble(value));
return Status::OK();
},
[](const ConfigOptions&, const std::string&, const void* addr,
std::string* value) {
const auto* factory =
static_cast<const CompactOnDeletionCollectorFactory*>(addr);
*value = ToString(factory->GetDeletionRatio());
return Status::OK();
},
nullptr}},
#endif // ROCKSDB_LITE
};
CompactOnDeletionCollectorFactory::CompactOnDeletionCollectorFactory(
size_t sliding_window_size, size_t deletion_trigger, double deletion_ratio)
: sliding_window_size_(sliding_window_size),
deletion_trigger_(deletion_trigger),
deletion_ratio_(deletion_ratio) {
RegisterOptions("", this, &on_deletion_collector_type_info);
}
TablePropertiesCollector*
CompactOnDeletionCollectorFactory::CreateTablePropertiesCollector(
@ -118,5 +191,37 @@ NewCompactOnDeletionCollectorFactory(size_t sliding_window_size,
new CompactOnDeletionCollectorFactory(sliding_window_size,
deletion_trigger, deletion_ratio));
}
} // namespace ROCKSDB_NAMESPACE
namespace {
static int RegisterTablePropertiesCollectorFactories(
ObjectLibrary& library, const std::string& /*arg*/) {
library.Register<TablePropertiesCollectorFactory>(
CompactOnDeletionCollectorFactory::kClassName(),
[](const std::string& /*uri*/,
std::unique_ptr<TablePropertiesCollectorFactory>* guard,
std::string* /* errmsg */) {
// By default, create a CompactionOnDeletionCollector that is disabled.
// Users will need to provide configuration parameters or call the
// corresponding Setter to enable the factory.
guard->reset(new CompactOnDeletionCollectorFactory(0, 0, 0));
return guard->get();
});
return 1;
}
} // namespace
#endif // !ROCKSDB_LITE
Status TablePropertiesCollectorFactory::CreateFromString(
const ConfigOptions& options, const std::string& value,
std::shared_ptr<TablePropertiesCollectorFactory>* result) {
#ifndef ROCKSDB_LITE
static std::once_flag once;
std::call_once(once, [&]() {
RegisterTablePropertiesCollectorFactories(*(ObjectLibrary::Default().get()),
"");
});
#endif // ROCKSDB_LITE
return LoadSharedObject<TablePropertiesCollectorFactory>(options, value,
nullptr, result);
}
} // namespace ROCKSDB_NAMESPACE

23
utilities/wal_filter.cc Normal file
View File

@ -0,0 +1,23 @@
// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#include "rocksdb/wal_filter.h"
#include <memory>
#include "rocksdb/convenience.h"
#include "rocksdb/options.h"
#include "rocksdb/utilities/customizable_util.h"
namespace ROCKSDB_NAMESPACE {
Status WalFilter::CreateFromString(const ConfigOptions& config_options,
const std::string& value,
WalFilter** filter) {
Status s =
LoadStaticObject<WalFilter>(config_options, value, nullptr, filter);
return s;
}
} // namespace ROCKSDB_NAMESPACE