Add comparator, merge operator, property collectors to SST file properties (again)
Summary: This is the original diff that I have landed and reverted and now I want to land again https://reviews.facebook.net/D34269 For old SST files we will show ``` comparator name: N/A merge operator name: N/A property collectors names: N/A ``` For new SST files with no merge operator name and with no property collectors ``` comparator name: leveldb.BytewiseComparator merge operator name: nullptr property collectors names: [] ``` for new SST files with these properties ``` comparator name: leveldb.BytewiseComparator merge operator name: UInt64AddOperator property collectors names: [DummyPropertiesCollector1,DummyPropertiesCollector2] ``` Test Plan: unittests Reviewers: andrewkr, yhchiang, sdong Reviewed By: sdong Subscribers: andrewkr, dhruba Differential Revision: https://reviews.facebook.net/D56487
This commit is contained in:
parent
7a6045a3c7
commit
5bd4022fec
@ -41,6 +41,9 @@ struct TablePropertiesNames {
|
||||
static const std::string kFilterPolicy;
|
||||
static const std::string kColumnFamilyName;
|
||||
static const std::string kColumnFamilyId;
|
||||
static const std::string kComparator;
|
||||
static const std::string kMergeOperator;
|
||||
static const std::string kPropertyCollectors;
|
||||
};
|
||||
|
||||
extern const std::string kPropertiesBlock;
|
||||
@ -154,6 +157,18 @@ struct TableProperties {
|
||||
// If no filter policy is used, `filter_policy_name` will be an empty string.
|
||||
std::string filter_policy_name;
|
||||
|
||||
// The name of the comparator used in this table.
|
||||
std::string comparator_name;
|
||||
|
||||
// The name of the merge operator used in this table.
|
||||
// If no merge operator is used, `merge_operator_name` will be "nullptr".
|
||||
std::string merge_operator_name;
|
||||
|
||||
// The names of the property collectors factories used in this table
|
||||
// separated by commas
|
||||
// {collector_name[1]},{collector_name[2]},{collector_name[3]} ..
|
||||
std::string property_collectors_names;
|
||||
|
||||
// user collected properties
|
||||
UserCollectedProperties user_collected_properties;
|
||||
UserCollectedProperties readable_properties;
|
||||
|
@ -26,6 +26,7 @@
|
||||
#include "rocksdb/env.h"
|
||||
#include "rocksdb/filter_policy.h"
|
||||
#include "rocksdb/flush_block_policy.h"
|
||||
#include "rocksdb/merge_operator.h"
|
||||
#include "rocksdb/table.h"
|
||||
|
||||
#include "table/block.h"
|
||||
@ -810,6 +811,25 @@ Status BlockBasedTableBuilder::Finish() {
|
||||
r->table_options.filter_policy->Name() : "";
|
||||
r->props.index_size =
|
||||
r->index_builder->EstimatedSize() + kBlockTrailerSize;
|
||||
r->props.comparator_name = r->ioptions.comparator != nullptr
|
||||
? r->ioptions.comparator->Name()
|
||||
: "nullptr";
|
||||
r->props.merge_operator_name = r->ioptions.merge_operator != nullptr
|
||||
? r->ioptions.merge_operator->Name()
|
||||
: "nullptr";
|
||||
|
||||
std::string property_collectors_names = "[";
|
||||
property_collectors_names = "[";
|
||||
for (size_t i = 0;
|
||||
i < r->ioptions.table_properties_collector_factories.size(); ++i) {
|
||||
if (i != 0) {
|
||||
property_collectors_names += ",";
|
||||
}
|
||||
property_collectors_names +=
|
||||
r->ioptions.table_properties_collector_factories[i]->Name();
|
||||
}
|
||||
property_collectors_names += "]";
|
||||
r->props.property_collectors_names = property_collectors_names;
|
||||
|
||||
// Add basic properties
|
||||
property_block_builder.AddTableProperty(r->props);
|
||||
|
@ -72,8 +72,18 @@ void PropertyBlockBuilder::AddTableProperty(const TableProperties& props) {
|
||||
Add(TablePropertiesNames::kColumnFamilyId, props.column_family_id);
|
||||
|
||||
if (!props.filter_policy_name.empty()) {
|
||||
Add(TablePropertiesNames::kFilterPolicy,
|
||||
props.filter_policy_name);
|
||||
Add(TablePropertiesNames::kFilterPolicy, props.filter_policy_name);
|
||||
}
|
||||
if (!props.comparator_name.empty()) {
|
||||
Add(TablePropertiesNames::kComparator, props.comparator_name);
|
||||
}
|
||||
|
||||
if (!props.merge_operator_name.empty()) {
|
||||
Add(TablePropertiesNames::kMergeOperator, props.merge_operator_name);
|
||||
}
|
||||
if (!props.property_collectors_names.empty()) {
|
||||
Add(TablePropertiesNames::kPropertyCollectors,
|
||||
props.property_collectors_names);
|
||||
}
|
||||
if (!props.column_family_name.empty()) {
|
||||
Add(TablePropertiesNames::kColumnFamilyName, props.column_family_name);
|
||||
@ -212,6 +222,12 @@ Status ReadProperties(const Slice& handle_value, RandomAccessFileReader* file,
|
||||
new_table_properties->filter_policy_name = raw_val.ToString();
|
||||
} else if (key == TablePropertiesNames::kColumnFamilyName) {
|
||||
new_table_properties->column_family_name = raw_val.ToString();
|
||||
} else if (key == TablePropertiesNames::kComparator) {
|
||||
new_table_properties->comparator_name = raw_val.ToString();
|
||||
} else if (key == TablePropertiesNames::kMergeOperator) {
|
||||
new_table_properties->merge_operator_name = raw_val.ToString();
|
||||
} else if (key == TablePropertiesNames::kPropertyCollectors) {
|
||||
new_table_properties->property_collectors_names = raw_val.ToString();
|
||||
} else {
|
||||
// handle user-collected properties
|
||||
new_table_properties->user_collected_properties.insert(
|
||||
|
@ -86,6 +86,20 @@ std::string TableProperties::ToString(
|
||||
column_family_name.empty() ? std::string("N/A") : column_family_name,
|
||||
prop_delim, kv_delim);
|
||||
|
||||
AppendProperty(result, "comparator name",
|
||||
comparator_name.empty() ? std::string("N/A") : comparator_name,
|
||||
prop_delim, kv_delim);
|
||||
|
||||
AppendProperty(
|
||||
result, "merge operator name",
|
||||
merge_operator_name.empty() ? std::string("N/A") : merge_operator_name,
|
||||
prop_delim, kv_delim);
|
||||
|
||||
AppendProperty(result, "property collectors names",
|
||||
property_collectors_names.empty() ? std::string("N/A")
|
||||
: property_collectors_names,
|
||||
prop_delim, kv_delim);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
@ -123,6 +137,11 @@ const std::string TablePropertiesNames::kColumnFamilyId =
|
||||
"rocksdb.column.family.id";
|
||||
const std::string TablePropertiesNames::kColumnFamilyName =
|
||||
"rocksdb.column.family.name";
|
||||
const std::string TablePropertiesNames::kComparator = "rocksdb.comparator";
|
||||
const std::string TablePropertiesNames::kMergeOperator =
|
||||
"rocksdb.merge.operator";
|
||||
const std::string TablePropertiesNames::kPropertyCollectors =
|
||||
"rocksdb.property.collectors";
|
||||
|
||||
extern const std::string kPropertiesBlock = "rocksdb.properties";
|
||||
// Old property block name for backward compatibility
|
||||
|
@ -47,6 +47,7 @@
|
||||
#include "util/string_util.h"
|
||||
#include "util/testharness.h"
|
||||
#include "util/testutil.h"
|
||||
#include "utilities/merge_operators.h"
|
||||
|
||||
namespace rocksdb {
|
||||
|
||||
@ -57,6 +58,40 @@ extern const uint64_t kPlainTableMagicNumber;
|
||||
|
||||
namespace {
|
||||
|
||||
// DummyPropertiesCollector used to test BlockBasedTableProperties
|
||||
class DummyPropertiesCollector : public TablePropertiesCollector {
|
||||
public:
|
||||
const char* Name() const { return ""; }
|
||||
|
||||
Status Finish(UserCollectedProperties* properties) { return Status::OK(); }
|
||||
|
||||
Status Add(const Slice& user_key, const Slice& value) { return Status::OK(); }
|
||||
|
||||
virtual UserCollectedProperties GetReadableProperties() const {
|
||||
return UserCollectedProperties{};
|
||||
}
|
||||
};
|
||||
|
||||
class DummyPropertiesCollectorFactory1
|
||||
: public TablePropertiesCollectorFactory {
|
||||
public:
|
||||
virtual TablePropertiesCollector* CreateTablePropertiesCollector(
|
||||
TablePropertiesCollectorFactory::Context context) {
|
||||
return new DummyPropertiesCollector();
|
||||
}
|
||||
const char* Name() const { return "DummyPropertiesCollector1"; }
|
||||
};
|
||||
|
||||
class DummyPropertiesCollectorFactory2
|
||||
: public TablePropertiesCollectorFactory {
|
||||
public:
|
||||
virtual TablePropertiesCollector* CreateTablePropertiesCollector(
|
||||
TablePropertiesCollectorFactory::Context context) {
|
||||
return new DummyPropertiesCollector();
|
||||
}
|
||||
const char* Name() const { return "DummyPropertiesCollector2"; }
|
||||
};
|
||||
|
||||
// Return reverse of "key".
|
||||
// Used to test non-lexicographic comparators.
|
||||
std::string Reverse(const Slice& key) {
|
||||
@ -1028,6 +1063,59 @@ TEST_F(BlockBasedTableTest, BasicBlockBasedTableProperties) {
|
||||
c.ResetTableReader();
|
||||
}
|
||||
|
||||
TEST_F(BlockBasedTableTest, BlockBasedTableProperties2) {
|
||||
TableConstructor c(&reverse_key_comparator);
|
||||
std::vector<std::string> keys;
|
||||
stl_wrappers::KVMap kvmap;
|
||||
|
||||
{
|
||||
Options options;
|
||||
BlockBasedTableOptions table_options;
|
||||
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
|
||||
|
||||
const ImmutableCFOptions ioptions(options);
|
||||
c.Finish(options, ioptions, table_options,
|
||||
GetPlainInternalComparator(options.comparator), &keys, &kvmap);
|
||||
|
||||
auto& props = *c.GetTableReader()->GetTableProperties();
|
||||
|
||||
// Default comparator
|
||||
ASSERT_EQ("leveldb.BytewiseComparator", props.comparator_name);
|
||||
// No merge operator
|
||||
ASSERT_EQ("nullptr", props.merge_operator_name);
|
||||
// No property collectors
|
||||
ASSERT_EQ("[]", props.property_collectors_names);
|
||||
// No filter policy is used
|
||||
ASSERT_EQ("", props.filter_policy_name);
|
||||
c.ResetTableReader();
|
||||
}
|
||||
|
||||
{
|
||||
Options options;
|
||||
BlockBasedTableOptions table_options;
|
||||
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
|
||||
options.comparator = &reverse_key_comparator;
|
||||
options.merge_operator = MergeOperators::CreateUInt64AddOperator();
|
||||
options.table_properties_collector_factories.emplace_back(
|
||||
new DummyPropertiesCollectorFactory1());
|
||||
options.table_properties_collector_factories.emplace_back(
|
||||
new DummyPropertiesCollectorFactory2());
|
||||
|
||||
const ImmutableCFOptions ioptions(options);
|
||||
c.Finish(options, ioptions, table_options,
|
||||
GetPlainInternalComparator(options.comparator), &keys, &kvmap);
|
||||
|
||||
auto& props = *c.GetTableReader()->GetTableProperties();
|
||||
|
||||
ASSERT_EQ("rocksdb.ReverseBytewiseComparator", props.comparator_name);
|
||||
ASSERT_EQ("UInt64AddOperator", props.merge_operator_name);
|
||||
ASSERT_EQ("[DummyPropertiesCollector1,DummyPropertiesCollector2]",
|
||||
props.property_collectors_names);
|
||||
ASSERT_EQ("", props.filter_policy_name); // no filter policy is used
|
||||
c.ResetTableReader();
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(BlockBasedTableTest, FilterPolicyNameProperties) {
|
||||
TableConstructor c(BytewiseComparator(), true);
|
||||
c.Add("a1", "val1");
|
||||
|
Loading…
Reference in New Issue
Block a user