andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Extract metaindex block from block-based table

Browse Source 
Summary: This change will allow other table to reuse the code for meta blocks.

Test Plan: all existing unit tests passed

Reviewers: dhruba, haobo, sdong

CC: leveldb

Differential Revision: https://reviews.facebook.net/D14475
This commit is contained in:
kailiu 2013-12-05 13:09:13 -08:00
parent e1d92dfd2e
commit 90729f8b23
11 changed files with 432 additions and 293 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

91
db/table_properties_collector.cc
View File

@ -10,87 +10,6 @@
namespace rocksdb {
namespace {
void AppendProperty(
std::string& props,
const std::string& key,
const std::string& value,
const std::string& prop_delim,
const std::string& kv_delim) {
props.append(key);
props.append(kv_delim);
props.append(value);
props.append(prop_delim);
}
template <class TValue>
void AppendProperty(
std::string& props,
const std::string& key,
const TValue& value,
const std::string& prop_delim,
const std::string& kv_delim) {
AppendProperty(
props, key, std::to_string(value), prop_delim, kv_delim
);
}
}
std::string TableProperties::ToString(
const std::string& prop_delim,
const std::string& kv_delim) const {
std::string result;
result.reserve(1024);
// Basic Info
AppendProperty(
result, "# data blocks", num_data_blocks, prop_delim, kv_delim
);
AppendProperty(result, "# entries", num_entries, prop_delim, kv_delim);
AppendProperty(result, "raw key size", raw_key_size, prop_delim, kv_delim);
AppendProperty(
result,
"raw average key size",
num_entries != 0 ? 1.0 * raw_key_size / num_entries : 0.0,
prop_delim,
kv_delim
);
AppendProperty(
result, "raw value size", raw_value_size, prop_delim, kv_delim
);
AppendProperty(
result,
"raw average value size",
num_entries != 0 ? 1.0 * raw_value_size / num_entries : 0.0,
prop_delim,
kv_delim
);
AppendProperty(result, "data block size", data_size, prop_delim, kv_delim);
AppendProperty(result, "index block size", index_size, prop_delim, kv_delim);
AppendProperty(
result, "filter block size", filter_size, prop_delim, kv_delim
);
AppendProperty(
result,
"(estimated) table size",
data_size + index_size + filter_size,
prop_delim,
kv_delim
);
AppendProperty(
result,
"filter policy name",
filter_policy_name.empty() ? std::string("N/A") : filter_policy_name,
prop_delim,
kv_delim
);
return result;
}
Status InternalKeyPropertiesCollector::Add(
const Slice& key, const Slice& value) {
ParsedInternalKey ikey;
@ -106,7 +25,7 @@ Status InternalKeyPropertiesCollector::Add(
}
Status InternalKeyPropertiesCollector::Finish(
TableProperties::UserCollectedProperties* properties) {
UserCollectedProperties* properties) {
assert(properties);
assert(properties->find(
InternalKeyTablePropertiesNames::kDeletedKeys) == properties->end());
@ -118,7 +37,7 @@ Status InternalKeyPropertiesCollector::Finish(
return Status::OK();
}
TableProperties::UserCollectedProperties
UserCollectedProperties
InternalKeyPropertiesCollector::GetReadableProperties() const {
return {
{ "kDeletedKeys", std::to_string(deleted_keys_) }
@ -137,11 +56,11 @@ Status UserKeyTablePropertiesCollector::Add(
}
Status UserKeyTablePropertiesCollector::Finish(
TableProperties::UserCollectedProperties* properties) {
UserCollectedProperties* properties) {
return collector_->Finish(properties);
}
TableProperties::UserCollectedProperties
UserCollectedProperties
UserKeyTablePropertiesCollector::GetReadableProperties() const {
return collector_->GetReadableProperties();
}
@ -151,7 +70,7 @@ const std::string InternalKeyTablePropertiesNames::kDeletedKeys
= "rocksdb.deleted.keys";
uint64_t GetDeletedKeys(
const TableProperties::UserCollectedProperties& props) {
const UserCollectedProperties& props) {
auto pos = props.find(InternalKeyTablePropertiesNames::kDeletedKeys);
if (pos == props.end()) {
return 0;

12
db/table_properties_collector.h
View File

@ -24,15 +24,13 @@ class InternalKeyPropertiesCollector : public TablePropertiesCollector {
public:
virtual Status Add(const Slice& key, const Slice& value) override;
virtual Status Finish(
TableProperties::UserCollectedProperties* properties) override;
virtual Status Finish(UserCollectedProperties* properties) override;
virtual const char* Name() const override {
return "InternalKeyPropertiesCollector";
}
TableProperties::UserCollectedProperties
GetReadableProperties() const override;
UserCollectedProperties GetReadableProperties() const override;
private:
uint64_t deleted_keys_ = 0;
@ -61,13 +59,11 @@ class UserKeyTablePropertiesCollector : public TablePropertiesCollector {
virtual Status Add(const Slice& key, const Slice& value) override;
virtual Status Finish(
TableProperties::UserCollectedProperties* properties) override;
virtual Status Finish(UserCollectedProperties* properties) override;
virtual const char* Name() const override { return collector_->Name(); }
TableProperties::UserCollectedProperties
GetReadableProperties() const override;
UserCollectedProperties GetReadableProperties() const override;
protected:
std::shared_ptr<TablePropertiesCollector> collector_;

7
db/table_properties_collector_test.cc
View File

@ -114,10 +114,10 @@ class RegularKeysStartWithA: public TablePropertiesCollector {
public:
const char* Name() const { return "RegularKeysStartWithA"; }
Status Finish(TableProperties::UserCollectedProperties* properties) {
Status Finish(UserCollectedProperties* properties) {
std::string encoded;
PutVarint32(&encoded, count_);
*properties = TableProperties::UserCollectedProperties {
*properties = UserCollectedProperties {
{ "TablePropertiesTest", "Rocksdb" },
{ "Count", encoded }
};
@ -132,8 +132,7 @@ class RegularKeysStartWithA: public TablePropertiesCollector {
return Status::OK();
}
virtual TableProperties::UserCollectedProperties
GetReadableProperties() const {
virtual UserCollectedProperties GetReadableProperties() const {
return {};
}

5
include/rocksdb/options.h
View File

@ -611,8 +611,9 @@ struct Options {
// the tables.
// Default: emtpy vector -- no user-defined statistics collection will be
// performed.
std::vector<std::shared_ptr<TablePropertiesCollector>>
table_properties_collectors;
typedef std::vector<std::shared_ptr<TablePropertiesCollector>>
TablePropertiesCollectors;
TablePropertiesCollectors table_properties_collectors;
// Allows thread-safe inplace updates. Requires Updates iff
// * key exists in current memtable

38
include/rocksdb/table_properties.h
View File

@ -11,18 +11,18 @@
namespace rocksdb {
// Other than basic table properties, each table may also have the user
// collected properties.
// The value of the user-collected properties are encoded as raw bytes --
// users have to interprete these values by themselves.
typedef
std::unordered_map<std::string, std::string>
UserCollectedProperties;
// TableProperties contains a bunch of read-only properties of its associated
// table.
struct TableProperties {
public:
// Other than basic table properties, each table may also have the user
// collected properties.
// The value of the user-collected properties are encoded as raw bytes --
// users have to interprete these values by themselves.
typedef
std::unordered_map<std::string, std::string>
UserCollectedProperties;
// the total size of all data blocks.
uint64_t data_size = 0;
// the size of index block.
@ -52,6 +52,19 @@ struct TableProperties {
const std::string& kv_delim = "=") const;
};
// table properties' human-readable names in the property block.
struct TablePropertiesNames {
static const std::string kDataSize;
static const std::string kIndexSize;
static const std::string kFilterSize;
static const std::string kRawKeySize;
static const std::string kRawValueSize;
static const std::string kNumDataBlocks;
static const std::string kNumEntries;
static const std::string kFilterPolicy;
};
// `TablePropertiesCollector` provides the mechanism for users to collect
// their own interested properties. This class is essentially a collection
// of callback functions that will be invoked during table building.
@ -68,23 +81,20 @@ class TablePropertiesCollector {
// for writing the properties block.
// @params properties User will add their collected statistics to
// `properties`.
virtual Status Finish(
TableProperties::UserCollectedProperties* properties) = 0;
virtual Status Finish(UserCollectedProperties* properties) = 0;
// The name of the properties collector can be used for debugging purpose.
virtual const char* Name() const = 0;
// Return the human-readable properties, where the key is property name and
// the value is the human-readable form of value.
virtual TableProperties::UserCollectedProperties
GetReadableProperties() const = 0;
virtual UserCollectedProperties GetReadableProperties() const = 0;
};
// Extra properties
// Below is a list of non-basic properties that are collected by database
// itself. Especially some properties regarding to the internal keys (which
// is unknown to `table`).
extern uint64_t GetDeletedKeys(
const TableProperties::UserCollectedProperties& props);
extern uint64_t GetDeletedKeys(const UserCollectedProperties& props);
} // namespace rocksdb

176
table/block_based_table_builder.cc
View File

@ -26,6 +26,7 @@
#include "table/block_builder.h"
#include "table/filter_block.h"
#include "table/format.h"
#include "table/meta_blocks.h"
#include "util/coding.h"
#include "util/crc32c.h"
#include "util/stop_watch.h"
@ -34,47 +35,11 @@ namespace rocksdb {
namespace {
struct BytewiseLessThan {
bool operator()(const std::string& key1, const std::string& key2) const {
// smaller entries will be placed in front.
return comparator->Compare(key1, key2) <= 0;
}
const Comparator* comparator = BytewiseComparator();
};
// When writing to a block that requires entries to be sorted by
// `BytewiseComparator`, we can buffer the content to `BytewiseSortedMap`
// before writng to store.
typedef std::map<std::string, std::string, BytewiseLessThan> BytewiseSortedMap;
void AddProperties(BytewiseSortedMap& props, std::string name, uint64_t val) {
assert(props.find(name) == props.end());
std::string dst;
PutVarint64(&dst, val);
props.insert(
std::make_pair(name, dst)
);
}
static bool GoodCompressionRatio(size_t compressed_size, size_t raw_size) {
// Check to see if compressed less than 12.5%
return compressed_size < raw_size - (raw_size / 8u);
}
// Were we encounter any error occurs during user-defined statistics collection,
// we'll write the warning message to info log.
void LogPropertiesCollectionError(
Logger* info_log, const std::string& method, const std::string& name) {
assert(method == "Add" || method == "Finish");
std::string msg =
"[Warning] encountered error when calling TablePropertiesCollector::" +
method + "() with collector name: " + name;
Log(info_log, "%s", msg.c_str());
}
} // anonymous namespace
// kBlockBasedTableMagicNumber was picked by running
@ -186,16 +151,12 @@ void BlockBasedTableBuilder::Add(const Slice& key, const Slice& value) {
r->props.raw_key_size += key.size();
r->props.raw_value_size += value.size();
for (auto collector : r->options.table_properties_collectors) {
Status s = collector->Add(key, value);
if (!s.ok()) {
LogPropertiesCollectionError(
r->options.info_log.get(),
"Add", /* method */
collector->Name()
);
}
}
NotifyCollectTableCollectorsOnAdd(
key,
value,
r->options.table_properties_collectors,
r->options.info_log.get()
);
}
void BlockBasedTableBuilder::Flush() {
@ -389,14 +350,7 @@ Status BlockBasedTableBuilder::Finish() {
// 2. [meta block: properties]
// 3. [metaindex block]
if (ok()) {
// We use `BytewiseComparator` as the comparator for meta block.
BlockBuilder meta_index_block(
r->options.block_restart_interval,
BytewiseComparator()
);
// Key: meta block name
// Value: block handle to that meta block
BytewiseSortedMap meta_block_handles;
MetaIndexBuilder meta_index_builer;
// Write filter block.
if (r->filter_block != nullptr) {
@ -404,104 +358,43 @@ Status BlockBasedTableBuilder::Finish() {
// of filter data.
std::string key = BlockBasedTable::kFilterBlockPrefix;
key.append(r->options.filter_policy->Name());
std::string handle_encoding;
filter_block_handle.EncodeTo(&handle_encoding);
meta_block_handles.insert(
std::make_pair(key, handle_encoding)
);
meta_index_builer.Add(key, filter_block_handle);
}
// Write properties block.
{
BlockBuilder properties_block(
r->options.block_restart_interval,
BytewiseComparator()
);
BytewiseSortedMap properties;
// Add basic properties
AddProperties(
properties,
BlockBasedTablePropertiesNames::kRawKeySize,
r->props.raw_key_size
);
AddProperties(
properties,
BlockBasedTablePropertiesNames::kRawValueSize,
r->props.raw_value_size
);
AddProperties(
properties,
BlockBasedTablePropertiesNames::kDataSize,
r->props.data_size
);
PropertyBlockBuilder property_block_builder;
std::vector<std::string> failed_user_prop_collectors;
r->props.filter_policy_name = r->options.filter_policy != nullptr ?
r->options.filter_policy->Name() : "";
r->props.index_size =
r->index_block.CurrentSizeEstimate() + kBlockTrailerSize;
AddProperties(
properties,
BlockBasedTablePropertiesNames::kIndexSize,
r->props.index_size
);
AddProperties(
properties,
BlockBasedTablePropertiesNames::kNumEntries,
r->props.num_entries
);
AddProperties(
properties,
BlockBasedTablePropertiesNames::kNumDataBlocks,
r->props.num_data_blocks);
if (r->filter_block != nullptr) {
properties.insert({
BlockBasedTablePropertiesNames::kFilterPolicy,
r->options.filter_policy->Name()
});
}
AddProperties(
properties,
BlockBasedTablePropertiesNames::kFilterSize,
r->props.filter_size
);
for (auto collector : r->options.table_properties_collectors) {
TableProperties::UserCollectedProperties user_collected_properties;
Status s =
collector->Finish(&user_collected_properties);
// Add basic properties
property_block_builder.AddTableProperty(r->props);
if (!s.ok()) {
LogPropertiesCollectionError(
r->options.info_log.get(),
"Finish", /* method */
collector->Name()
);
} else {
properties.insert(
user_collected_properties.begin(),
user_collected_properties.end()
);
}
}
for (const auto& stat : properties) {
properties_block.Add(stat.first, stat.second);
}
NotifyCollectTableCollectorsOnFinish(
r->options.table_properties_collectors,
r->options.info_log.get(),
&property_block_builder
);
BlockHandle properties_block_handle;
WriteBlock(&properties_block, &properties_block_handle);
std::string handle_encoding;
properties_block_handle.EncodeTo(&handle_encoding);
meta_block_handles.insert(
{ BlockBasedTable::kPropertiesBlock, handle_encoding }
WriteRawBlock(
property_block_builder.Finish(),
kNoCompression,
&properties_block_handle
);
meta_index_builer.Add(BlockBasedTable::kPropertiesBlock,
properties_block_handle);
} // end of properties block writing
for (const auto& metablock : meta_block_handles) {
meta_index_block.Add(metablock.first, metablock.second);
}
WriteBlock(&meta_index_block, &metaindex_block_handle);
WriteRawBlock(
meta_index_builer.Finish(),
kNoCompression,
&metaindex_block_handle
);
} // meta blocks and metaindex block.
// Write index block
@ -563,4 +456,9 @@ uint64_t BlockBasedTableBuilder::FileSize() const {
return rep_->offset;
}
const std::string BlockBasedTable::kFilterBlockPrefix =
"filter.";
const std::string BlockBasedTable::kPropertiesBlock =
"rocksdb.properties";
} // namespace rocksdb

37
table/block_based_table_reader.cc
View File

@ -432,19 +432,19 @@ Status BlockBasedTable::ReadProperties(
// All pre-defined properties of type uint64_t
std::unordered_map<std::string, uint64_t*> predefined_uint64_properties = {
{ BlockBasedTablePropertiesNames::kDataSize,
{ TablePropertiesNames::kDataSize,
&table_properties->data_size },
{ BlockBasedTablePropertiesNames::kIndexSize,
{ TablePropertiesNames::kIndexSize,
&table_properties->index_size },
{ BlockBasedTablePropertiesNames::kFilterSize,
{ TablePropertiesNames::kFilterSize,
&table_properties->filter_size },
{ BlockBasedTablePropertiesNames::kRawKeySize,
{ TablePropertiesNames::kRawKeySize,
&table_properties->raw_key_size },
{ BlockBasedTablePropertiesNames::kRawValueSize,
{ TablePropertiesNames::kRawValueSize,
&table_properties->raw_value_size },
{ BlockBasedTablePropertiesNames::kNumDataBlocks,
{ TablePropertiesNames::kNumDataBlocks,
&table_properties->num_data_blocks },
{ BlockBasedTablePropertiesNames::kNumEntries,
{ TablePropertiesNames::kNumEntries,
&table_properties->num_entries },
};
@ -478,7 +478,7 @@ Status BlockBasedTable::ReadProperties(
continue;
}
*(pos->second) = val;
} else if (key == BlockBasedTablePropertiesNames::kFilterPolicy) {
} else if (key == TablePropertiesNames::kFilterPolicy) {
table_properties->filter_policy_name = raw_val.ToString();
} else {
// handle user-collected
@ -1062,25 +1062,4 @@ uint64_t BlockBasedTable::ApproximateOffsetOf(const Slice& key) {
return result;
}
const std::string BlockBasedTable::kFilterBlockPrefix =
"filter.";
const std::string BlockBasedTable::kPropertiesBlock =
"rocksdb.properties";
const std::string BlockBasedTablePropertiesNames::kDataSize =
"rocksdb.data.size";
const std::string BlockBasedTablePropertiesNames::kIndexSize =
"rocksdb.index.size";
const std::string BlockBasedTablePropertiesNames::kFilterSize =
"rocksdb.filter.size";
const std::string BlockBasedTablePropertiesNames::kRawKeySize =
"rocksdb.raw.key.size";
const std::string BlockBasedTablePropertiesNames::kRawValueSize =
"rocksdb.raw.value.size";
const std::string BlockBasedTablePropertiesNames::kNumDataBlocks =
"rocksdb.num.data.blocks";
const std::string BlockBasedTablePropertiesNames::kNumEntries =
"rocksdb.num.entries";
const std::string BlockBasedTablePropertiesNames::kFilterPolicy =
"rocksdb.filter.policy";
} // namespace rocksdb

11
table/block_based_table_reader.h
View File

@ -181,15 +181,4 @@ class BlockBasedTable : public TableReader {
void operator=(const TableReader&) = delete;
};
struct BlockBasedTablePropertiesNames {
static const std::string kDataSize;
static const std::string kIndexSize;
static const std::string kFilterSize;
static const std::string kRawKeySize;
static const std::string kRawValueSize;
static const std::string kNumDataBlocks;
static const std::string kNumEntries;
static const std::string kFilterPolicy;
};
} // namespace rocksdb

134
table/meta_blocks.cc Normal file
View File

@ -0,0 +1,134 @@
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
#include "table/meta_blocks.h"
#include <map>
#include "rocksdb/table_properties.h"
#include "table/format.h"
#include "util/coding.h"
namespace rocksdb {
MetaIndexBuilder::MetaIndexBuilder()
: meta_index_block_(
new BlockBuilder(1 /* restart interval */, BytewiseComparator())) {
}
void MetaIndexBuilder::Add(const std::string& key,
const BlockHandle& handle) {
std::string handle_encoding;
handle.EncodeTo(&handle_encoding);
meta_block_handles_.insert({key, handle_encoding});
}
Slice MetaIndexBuilder::Finish() {
for (const auto& metablock : meta_block_handles_) {
meta_index_block_->Add(metablock.first, metablock.second);
}
return meta_index_block_->Finish();
}
PropertyBlockBuilder::PropertyBlockBuilder()
: properties_block_(
new BlockBuilder(1 /* restart interval */, BytewiseComparator())) {
}
void PropertyBlockBuilder::Add(const std::string& name,
const std::string& val) {
props_.insert({name, val});
}
void PropertyBlockBuilder::Add(const std::string& name, uint64_t val) {
assert(props_.find(name) == props_.end());
std::string dst;
PutVarint64(&dst, val);
Add(name, dst);
}
void PropertyBlockBuilder::Add(
const UserCollectedProperties& user_collected_properties) {
for (const auto& prop : user_collected_properties) {
Add(prop.first, prop.second);
}
}
void PropertyBlockBuilder::AddTableProperty(const TableProperties& props) {
Add(TablePropertiesNames::kRawKeySize, props.raw_key_size);
Add(TablePropertiesNames::kRawValueSize, props.raw_value_size);
Add(TablePropertiesNames::kDataSize, props.data_size);
Add(TablePropertiesNames::kIndexSize, props.index_size);
Add(TablePropertiesNames::kNumEntries, props.num_entries);
Add(TablePropertiesNames::kNumDataBlocks, props.num_data_blocks);
Add(TablePropertiesNames::kFilterSize, props.filter_size);
if (!props.filter_policy_name.empty()) {
Add(TablePropertiesNames::kFilterPolicy,
props.filter_policy_name);
}
}
Slice PropertyBlockBuilder::Finish() {
for (const auto& prop : props_) {
properties_block_->Add(prop.first, prop.second);
}
return properties_block_->Finish();
}
void LogPropertiesCollectionError(
Logger* info_log, const std::string& method, const std::string& name) {
assert(method == "Add" || method == "Finish");
std::string msg =
"[Warning] encountered error when calling TablePropertiesCollector::" +
method + "() with collector name: " + name;
Log(info_log, "%s", msg.c_str());
}
bool NotifyCollectTableCollectorsOnAdd(
const Slice& key,
const Slice& value,
const Options::TablePropertiesCollectors& collectors,
Logger* info_log) {
bool all_succeeded = true;
for (auto collector : collectors) {
Status s = collector->Add(key, value);
all_succeeded = all_succeeded && s.ok();
if (!s.ok()) {
LogPropertiesCollectionError(
info_log, "Add", /* method */ collector->Name()
);
}
}
return all_succeeded;
}
bool NotifyCollectTableCollectorsOnFinish(
const Options::TablePropertiesCollectors& collectors,
Logger* info_log,
PropertyBlockBuilder* builder) {
bool all_succeeded = true;
for (auto collector : collectors) {
UserCollectedProperties user_collected_properties;
Status s = collector->Finish(&user_collected_properties);
all_succeeded = all_succeeded && s.ok();
if (!s.ok()) {
LogPropertiesCollectionError(
info_log, "Finish", /* method */ collector->Name()
);
} else {
builder->Add(user_collected_properties);
}
}
return all_succeeded;
}
} // namespace rocksdb

106
table/meta_blocks.h Normal file
View File

@ -0,0 +1,106 @@
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
#pragma once
#include <map>
#include <memory>
#include <string>
#include "rocksdb/comparator.h"
#include "rocksdb/options.h"
#include "rocksdb/slice.h"
#include "table/block_builder.h"
namespace rocksdb {
class BlockHandle;
class BlockBuilder;
class Logger;
struct TableProperties;
// An STL style comparator that does the bytewise comparator comparasion
// internally.
struct BytewiseLessThan {
bool operator()(const std::string& key1, const std::string& key2) const {
// smaller entries will be placed in front.
return comparator->Compare(key1, key2) <= 0;
}
const Comparator* comparator = BytewiseComparator();
};
// When writing to a block that requires entries to be sorted by
// `BytewiseComparator`, we can buffer the content to `BytewiseSortedMap`
// before writng to store.
typedef std::map<std::string, std::string, BytewiseLessThan> BytewiseSortedMap;
class MetaIndexBuilder {
public:
MetaIndexBuilder(const MetaIndexBuilder&) = delete;
MetaIndexBuilder& operator=(const MetaIndexBuilder&) = delete;
MetaIndexBuilder();
void Add(const std::string& key, const BlockHandle& handle);
// Write all the added key/value pairs to the block and return the contents
// of the block.
Slice Finish();
private:
// * Key: meta block name
// * Value: block handle to that meta block
struct Rep;
Rep* rep_;
// store the sorted key/handle of the metablocks.
BytewiseSortedMap meta_block_handles_;
std::unique_ptr<BlockBuilder> meta_index_block_;
};
class PropertyBlockBuilder {
public:
PropertyBlockBuilder(const PropertyBlockBuilder&) = delete;
PropertyBlockBuilder& operator=(const PropertyBlockBuilder&) = delete;
PropertyBlockBuilder();
void AddTableProperty(const TableProperties& props);
void Add(const std::string& key, uint64_t value);
void Add(const std::string& key, const std::string& value);
void Add(const UserCollectedProperties& user_collected_properties);
// Write all the added entries to the block and return the block contents
Slice Finish();
private:
std::unique_ptr<BlockBuilder> properties_block_;
BytewiseSortedMap props_;
};
// Were we encounter any error occurs during user-defined statistics collection,
// we'll write the warning message to info log.
void LogPropertiesCollectionError(
Logger* info_log, const std::string& method, const std::string& name);
// Utility functions help table builder to trigger batch events for user
// defined property collectors.
// Return value indicates if there is any error occurred; if error occurred,
// the warning message will be logged.
// NotifyCollectTableCollectorsOnAdd() triggers the `Add` event for all
// property collectors.
bool NotifyCollectTableCollectorsOnAdd(
const Slice& key,
const Slice& value,
const Options::TablePropertiesCollectors& collectors,
Logger* info_log);
// NotifyCollectTableCollectorsOnAdd() triggers the `Finish` event for all
// property collectors. The collected properties will be added to `builder`.
bool NotifyCollectTableCollectorsOnFinish(
const Options::TablePropertiesCollectors& collectors,
Logger* info_log,
PropertyBlockBuilder* builder);
} // namespace rocksdb

108
table/table_properties.cc Normal file
View File

@ -0,0 +1,108 @@
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
#include "rocksdb/table_properties.h"
namespace rocksdb {
namespace {
void AppendProperty(
std::string& props,
const std::string& key,
const std::string& value,
const std::string& prop_delim,
const std::string& kv_delim) {
props.append(key);
props.append(kv_delim);
props.append(value);
props.append(prop_delim);
}
template <class TValue>
void AppendProperty(
std::string& props,
const std::string& key,
const TValue& value,
const std::string& prop_delim,
const std::string& kv_delim) {
AppendProperty(
props, key, std::to_string(value), prop_delim, kv_delim
);
}
}
std::string TableProperties::ToString(
const std::string& prop_delim,
const std::string& kv_delim) const {
std::string result;
result.reserve(1024);
// Basic Info
AppendProperty(
result, "# data blocks", num_data_blocks, prop_delim, kv_delim
);
AppendProperty(result, "# entries", num_entries, prop_delim, kv_delim);
AppendProperty(result, "raw key size", raw_key_size, prop_delim, kv_delim);
AppendProperty(
result,
"raw average key size",
num_entries != 0 ? 1.0 * raw_key_size / num_entries : 0.0,
prop_delim,
kv_delim
);
AppendProperty(
result, "raw value size", raw_value_size, prop_delim, kv_delim
);
AppendProperty(
result,
"raw average value size",
num_entries != 0 ? 1.0 * raw_value_size / num_entries : 0.0,
prop_delim,
kv_delim
);
AppendProperty(result, "data block size", data_size, prop_delim, kv_delim);
AppendProperty(result, "index block size", index_size, prop_delim, kv_delim);
AppendProperty(
result, "filter block size", filter_size, prop_delim, kv_delim
);
AppendProperty(
result,
"(estimated) table size",
data_size + index_size + filter_size,
prop_delim,
kv_delim
);
AppendProperty(
result,
"filter policy name",
filter_policy_name.empty() ? std::string("N/A") : filter_policy_name,
prop_delim,
kv_delim
);
return result;
}
const std::string TablePropertiesNames::kDataSize =
"rocksdb.data.size";
const std::string TablePropertiesNames::kIndexSize =
"rocksdb.index.size";
const std::string TablePropertiesNames::kFilterSize =
"rocksdb.filter.size";
const std::string TablePropertiesNames::kRawKeySize =
"rocksdb.raw.key.size";
const std::string TablePropertiesNames::kRawValueSize =
"rocksdb.raw.value.size";
const std::string TablePropertiesNames::kNumDataBlocks =
"rocksdb.num.data.blocks";
const std::string TablePropertiesNames::kNumEntries =
"rocksdb.num.entries";
const std::string TablePropertiesNames::kFilterPolicy =
"rocksdb.filter.policy";
} // namespace rocksdb