andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Make Statistics a Customizable Class (#8637)

Browse Source 
Summary:
Make the Statistics object into a Customizable object.  Statistics can now be stored and created to/from the Options file.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/8637

Reviewed By: zhichao-cao

Differential Revision: D30530550

Pulled By: mrambacher

fbshipit-source-id: 5fc7d01d8431f37b2c205bbbd8342c9f697023bd
This commit is contained in:
mrambacher 2021-09-10 09:46:47 -07:00 committed by Facebook GitHub Bot
parent 12542488ef
commit dc0dc90cf5
8 changed files with 162 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
db/db_test2.cc
View File

@ -5588,6 +5588,7 @@ TEST_F(DBTest2, MultiDBParallelOpenTest) {
namespace {
class DummyOldStats : public Statistics {
public:
const char* Name() const override { return "DummyOldStats"; }
uint64_t getTickerCount(uint32_t /*ticker_type*/) const override { return 0; }
void recordTick(uint32_t /* ticker_type */, uint64_t /* count */) override {
num_rt++;

6
include/rocksdb/statistics.h
View File

@ -13,6 +13,7 @@
#include <string>
#include <vector>
#include "rocksdb/customizable.h"
#include "rocksdb/status.h"
namespace ROCKSDB_NAMESPACE {
@ -568,10 +569,13 @@ enum StatsLevel : uint8_t {
// options.statistics->getTickerCount(NUMBER_BLOCK_COMPRESSED);
// HistogramData hist;
// options.statistics->histogramData(FLUSH_TIME, &hist);
class Statistics {
class Statistics : public Customizable {
public:
virtual ~Statistics() {}
static const char* Type() { return "Statistics"; }
static Status CreateFromString(const ConfigOptions& opts,
const std::string& value,
std::shared_ptr<Statistics>* result);
virtual uint64_t getTickerCount(uint32_t tickerType) const = 0;
virtual void histogramData(uint32_t type,
HistogramData* const data) const = 0;

2
include/rocksdb/utilities/customizable_util.h
View File

@ -120,6 +120,8 @@ static Status NewManagedObject(
return object->ConfigureFromMap(config_options, opt_map);
});
#else
(void)result;
(void)opt_map;
status = Status::NotSupported("Cannot load object in LITE mode ", id);
#endif // ROCKSDB_LITE
if (config_options.ignore_unsupported_options && status.IsNotSupported()) {

51
monitoring/statistics.cc
View File

@ -8,7 +8,12 @@
#include <algorithm>
#include <cinttypes>
#include <cstdio>
#include "rocksdb/convenience.h"
#include "rocksdb/statistics.h"
#include "rocksdb/utilities/customizable_util.h"
#include "rocksdb/utilities/options_type.h"
#include "util/string_util.h"
namespace ROCKSDB_NAMESPACE {
@ -272,8 +277,52 @@ std::shared_ptr<Statistics> CreateDBStatistics() {
return std::make_shared<StatisticsImpl>(nullptr);
}
#ifndef ROCKSDB_LITE
static int RegisterBuiltinStatistics(ObjectLibrary& library,
const std::string& /*arg*/) {
library.Register<Statistics>(
StatisticsImpl::kClassName(),
[](const std::string& /*uri*/, std::unique_ptr<Statistics>* guard,
std::string* /* errmsg */) {
guard->reset(new StatisticsImpl(nullptr));
return guard->get();
});
return 1;
}
#endif // ROCKSDB_LITE
Status Statistics::CreateFromString(const ConfigOptions& config_options,
const std::string& id,
std::shared_ptr<Statistics>* result) {
#ifndef ROCKSDB_LITE
static std::once_flag once;
std::call_once(once, [&]() {
RegisterBuiltinStatistics(*(ObjectLibrary::Default().get()), "");
});
#endif // ROCKSDB_LITE
Status s;
if (id == "" || id == StatisticsImpl::kClassName()) {
result->reset(new StatisticsImpl(nullptr));
} else if (id == kNullptrString) {
result->reset();
} else {
s = LoadSharedObject<Statistics>(config_options, id, nullptr, result);
}
return s;
}
static std::unordered_map<std::string, OptionTypeInfo> stats_type_info = {
#ifndef ROCKSDB_LITE
{"inner", OptionTypeInfo::AsCustomSharedPtr<Statistics>(
0, OptionVerificationType::kByNameAllowFromNull,
OptionTypeFlags::kCompareNever)},
#endif // !ROCKSDB_LITE
};
StatisticsImpl::StatisticsImpl(std::shared_ptr<Statistics> stats)
: stats_(std::move(stats)) {}
: stats_(std::move(stats)) {
RegisterOptions("StatisticsOptions", &stats_, &stats_type_info);
}
StatisticsImpl::~StatisticsImpl() {}

4
monitoring/statistics.h
View File

@ -44,6 +44,8 @@ class StatisticsImpl : public Statistics {
public:
StatisticsImpl(std::shared_ptr<Statistics> stats);
virtual ~StatisticsImpl();
const char* Name() const override { return kClassName(); }
static const char* kClassName() { return "BasicStatistics"; }
virtual uint64_t getTickerCount(uint32_t ticker_type) const override;
virtual void histogramData(uint32_t histogram_type,
@ -68,6 +70,8 @@ class StatisticsImpl : public Statistics {
virtual bool getTickerMap(std::map<std::string, uint64_t>*) const override;
virtual bool HistEnabledForType(uint32_t type) const override;
const Customizable* Inner() const override { return stats_.get(); }
private:
// If non-nullptr, forwards updates to the object pointed to by `stats_`.
std::shared_ptr<Statistics> stats_;

107
options/customizable_test.cc
View File

@ -22,6 +22,7 @@
#include "rocksdb/env_encryption.h"
#include "rocksdb/flush_block_policy.h"
#include "rocksdb/secondary_cache.h"
#include "rocksdb/statistics.h"
#include "rocksdb/utilities/customizable_util.h"
#include "rocksdb/utilities/object_registry.h"
#include "rocksdb/utilities/options_type.h"
@ -1218,6 +1219,27 @@ class TestSecondaryCache : public SecondaryCache {
std::string GetPrintableOptions() const override { return ""; }
};
class TestStatistics : public StatisticsImpl {
public:
TestStatistics() : StatisticsImpl(nullptr) {}
const char* Name() const override { return kClassName(); }
static const char* kClassName() { return "Test"; }
};
class TestFlushBlockPolicyFactory : public FlushBlockPolicyFactory {
public:
TestFlushBlockPolicyFactory() {}
static const char* kClassName() { return "TestFlushBlockPolicyFactory"; }
const char* Name() const override { return kClassName(); }
FlushBlockPolicy* NewFlushBlockPolicy(
const BlockBasedTableOptions& /*table_options*/,
const BlockBuilder& /*data_block_builder*/) const override {
return nullptr;
}
};
#ifndef ROCKSDB_LITE
class MockEncryptionProvider : public EncryptionProvider {
public:
@ -1260,23 +1282,7 @@ class MockCipher : public BlockCipher {
Status Encrypt(char* /*data*/) override { return Status::NotSupported(); }
Status Decrypt(char* data) override { return Encrypt(data); }
};
#endif // ROCKSDB_LITE
class TestFlushBlockPolicyFactory : public FlushBlockPolicyFactory {
public:
TestFlushBlockPolicyFactory() {}
static const char* kClassName() { return "TestFlushBlockPolicyFactory"; }
const char* Name() const override { return kClassName(); }
FlushBlockPolicy* NewFlushBlockPolicy(
const BlockBasedTableOptions& /*table_options*/,
const BlockBuilder& /*data_block_builder*/) const override {
return nullptr;
}
};
#ifndef ROCKSDB_LITE
static int RegisterLocalObjects(ObjectLibrary& library,
const std::string& /*arg*/) {
size_t num_types;
@ -1302,6 +1308,14 @@ static int RegisterLocalObjects(ObjectLibrary& library,
return guard->get();
});
// Load any locally defined objects here
library.Register<Statistics>(
TestStatistics::kClassName(),
[](const std::string& /*uri*/, std::unique_ptr<Statistics>* guard,
std::string* /* errmsg */) {
guard->reset(new TestStatistics());
return guard->get();
});
library.Register<EncryptionProvider>(
"Mock(://test)?",
[](const std::string& uri, std::unique_ptr<EncryptionProvider>* guard,
@ -1432,6 +1446,56 @@ TEST_F(LoadCustomizableTest, LoadComparatorTest) {
}
}
TEST_F(LoadCustomizableTest, LoadStatisticsTest) {
std::shared_ptr<Statistics> stats;
ASSERT_NOK(Statistics::CreateFromString(
config_options_, TestStatistics::kClassName(), &stats));
ASSERT_OK(
Statistics::CreateFromString(config_options_, "BasicStatistics", &stats));
ASSERT_NE(stats, nullptr);
ASSERT_EQ(stats->Name(), std::string("BasicStatistics"));
#ifndef ROCKSDB_LITE
ASSERT_NOK(GetDBOptionsFromString(config_options_, db_opts_,
"statistics=Test", &db_opts_));
ASSERT_OK(GetDBOptionsFromString(config_options_, db_opts_,
"statistics=BasicStatistics", &db_opts_));
ASSERT_NE(db_opts_.statistics, nullptr);
ASSERT_STREQ(db_opts_.statistics->Name(), "BasicStatistics");
if (RegisterTests("test")) {
ASSERT_OK(Statistics::CreateFromString(
config_options_, TestStatistics::kClassName(), &stats));
ASSERT_NE(stats, nullptr);
ASSERT_STREQ(stats->Name(), TestStatistics::kClassName());
ASSERT_OK(GetDBOptionsFromString(config_options_, db_opts_,
"statistics=Test", &db_opts_));
ASSERT_NE(db_opts_.statistics, nullptr);
ASSERT_STREQ(db_opts_.statistics->Name(), TestStatistics::kClassName());
ASSERT_OK(GetDBOptionsFromString(
config_options_, db_opts_, "statistics={id=Test;inner=BasicStatistics}",
&db_opts_));
ASSERT_NE(db_opts_.statistics, nullptr);
ASSERT_STREQ(db_opts_.statistics->Name(), TestStatistics::kClassName());
auto* inner = db_opts_.statistics->GetOptions<std::shared_ptr<Statistics>>(
"StatisticsOptions");
ASSERT_NE(inner, nullptr);
ASSERT_NE(inner->get(), nullptr);
ASSERT_STREQ(inner->get()->Name(), "BasicStatistics");
ASSERT_OK(Statistics::CreateFromString(
config_options_, "id=BasicStatistics;inner=Test", &stats));
ASSERT_NE(stats, nullptr);
ASSERT_STREQ(stats->Name(), "BasicStatistics");
inner = stats->GetOptions<std::shared_ptr<Statistics>>("StatisticsOptions");
ASSERT_NE(inner, nullptr);
ASSERT_NE(inner->get(), nullptr);
ASSERT_STREQ(inner->get()->Name(), TestStatistics::kClassName());
}
#endif
}
TEST_F(LoadCustomizableTest, LoadMemTableRepFactoryTest) {
std::unique_ptr<MemTableRepFactory> result;
ASSERT_NOK(MemTableRepFactory::CreateFromString(
@ -1567,7 +1631,7 @@ TEST_F(LoadCustomizableTest, LoadFlushBlockPolicyFactoryTest) {
std::shared_ptr<TableFactory> table;
std::shared_ptr<FlushBlockPolicyFactory> result;
ASSERT_NOK(FlushBlockPolicyFactory::CreateFromString(
config_options_, "TestFlushBlockPolicyFactory", &result));
config_options_, TestFlushBlockPolicyFactory::kClassName(), &result));
ASSERT_OK(
FlushBlockPolicyFactory::CreateFromString(config_options_, "", &result));
@ -1595,16 +1659,17 @@ TEST_F(LoadCustomizableTest, LoadFlushBlockPolicyFactoryTest) {
FlushBlockEveryKeyPolicyFactory::kClassName());
if (RegisterTests("Test")) {
ASSERT_OK(FlushBlockPolicyFactory::CreateFromString(
config_options_, "TestFlushBlockPolicyFactory", &result));
config_options_, TestFlushBlockPolicyFactory::kClassName(), &result));
ASSERT_NE(result, nullptr);
ASSERT_STREQ(result->Name(), "TestFlushBlockPolicyFactory");
ASSERT_STREQ(result->Name(), TestFlushBlockPolicyFactory::kClassName());
ASSERT_OK(TableFactory::CreateFromString(
config_options_, table_opts + "TestFlushBlockPolicyFactory", &table));
config_options_, table_opts + TestFlushBlockPolicyFactory::kClassName(),
&table));
bbto = table->GetOptions<BlockBasedTableOptions>();
ASSERT_NE(bbto, nullptr);
ASSERT_NE(bbto->flush_block_policy_factory.get(), nullptr);
ASSERT_STREQ(bbto->flush_block_policy_factory->Name(),
"TestFlushBlockPolicyFactory");
TestFlushBlockPolicyFactory::kClassName());
}
#endif // ROCKSDB_LITE
}

10
options/db_options.cc
View File

@ -18,6 +18,7 @@
#include "rocksdb/listener.h"
#include "rocksdb/rate_limiter.h"
#include "rocksdb/sst_file_manager.h"
#include "rocksdb/statistics.h"
#include "rocksdb/system_clock.h"
#include "rocksdb/utilities/options_type.h"
#include "rocksdb/wal_filter.h"
@ -443,6 +444,15 @@ static std::unordered_map<std::string, OptionTypeInfo>
{offsetof(struct ImmutableDBOptions, allow_data_in_errors),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone}},
{"statistics",
OptionTypeInfo::AsCustomSharedPtr<Statistics>(
// Statistics should not be compared and can be null
// Statistics are maked "don't serialize" until they can be shared
// between DBs
offsetof(struct ImmutableDBOptions, statistics),
OptionVerificationType::kNormal,
OptionTypeFlags::kCompareNever | OptionTypeFlags::kDontSerialize |
OptionTypeFlags::kAllowNull)},
// Allow EventListeners that have a non-empty Name() to be read/written
// as options Each listener will either be
// - A simple name (e.g. "MyEventListener")

7
tools/db_bench_tool.cc
View File

@ -8017,6 +8017,7 @@ class Benchmark {
int db_bench_tool(int argc, char** argv) {
ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
ConfigOptions config_options;
static bool initialized = false;
if (!initialized) {
SetUsageMessage(std::string("\nUSAGE:\n") + std::string(argv[0]) +
@ -8033,8 +8034,8 @@ int db_bench_tool(int argc, char** argv) {
exit(1);
}
if (!FLAGS_statistics_string.empty()) {
Status s = ObjectRegistry::NewInstance()->NewSharedObject<Statistics>(
FLAGS_statistics_string, &dbstats);
Status s = Statistics::CreateFromString(config_options,
FLAGS_statistics_string, &dbstats);
if (dbstats == nullptr) {
fprintf(stderr,
"No Statistics registered matching string: %s status=%s\n",
@ -8080,7 +8081,7 @@ int db_bench_tool(int argc, char** argv) {
}
if (env_opts == 1) {
Status s = Env::CreateFromUri(ConfigOptions(), FLAGS_env_uri, FLAGS_fs_uri,
Status s = Env::CreateFromUri(config_options, FLAGS_env_uri, FLAGS_fs_uri,
&FLAGS_env, &env_guard);
if (!s.ok()) {
fprintf(stderr, "Failed creating env: %s\n", s.ToString().c_str());