Memory usage stats in C API (#4340)
Summary: Please consider this small PR providing access to the `MemoryUsage::GetApproximateMemoryUsageByType` function in plain C API. Actually I'm working on Go application and now trying to investigate the reasons of high memory consumption (#4313). Go [wrappers](https://github.com/tecbot/gorocksdb) are built on the top of Rocksdb C API. According to the #706, `MemoryUsage::GetApproximateMemoryUsageByType` is considered as the best option to get database internal memory usage stats, but it wasn't supported in C API yet. Pull Request resolved: https://github.com/facebook/rocksdb/pull/4340 Differential Revision: D9655135 Pulled By: ajkr fbshipit-source-id: a3d2f3f47c143ae75862fbcca2f571ea1b49e14a
This commit is contained in:
Vitaly Isaev
Facebook Github Bot
parent
9ea9007b50
commit
0bd2ede10e
101
db/c.cc
101
db/c.cc
@ -33,6 +33,7 @@
|
||||
#include "rocksdb/utilities/backupable_db.h"
|
||||
#include "rocksdb/utilities/checkpoint.h"
|
||||
#include "rocksdb/utilities/db_ttl.h"
|
||||
#include "rocksdb/utilities/memory_util.h"
|
||||
#include "rocksdb/utilities/optimistic_transaction_db.h"
|
||||
#include "rocksdb/utilities/transaction.h"
|
||||
#include "rocksdb/utilities/transaction_db.h"
|
||||
@ -41,6 +42,10 @@
|
||||
#include "rocksdb/perf_context.h"
|
||||
#include "utilities/merge_operators.h"
|
||||
|
||||
#include <vector>
|
||||
#include <unordered_set>
|
||||
#include <map>
|
||||
|
||||
using rocksdb::BytewiseComparator;
|
||||
using rocksdb::Cache;
|
||||
using rocksdb::ColumnFamilyDescriptor;
|
||||
@ -108,8 +113,12 @@ using rocksdb::TransactionLogIterator;
|
||||
using rocksdb::BatchResult;
|
||||
using rocksdb::PerfLevel;
|
||||
using rocksdb::PerfContext;
|
||||
using rocksdb::MemoryUtil;
|
||||
|
||||
using std::shared_ptr;
|
||||
using std::vector;
|
||||
using std::unordered_set;
|
||||
using std::map;
|
||||
|
||||
extern "C" {
|
||||
|
||||
@ -4101,6 +4110,98 @@ const char* rocksdb_pinnableslice_value(const rocksdb_pinnableslice_t* v,
|
||||
*vlen = v->rep.size();
|
||||
return v->rep.data();
|
||||
}
|
||||
|
||||
// container to keep databases and caches in order to use rocksdb::MemoryUtil
|
||||
struct rocksdb_memory_consumers_t {
|
||||
std::vector<rocksdb_t*> dbs;
|
||||
std::unordered_set<rocksdb_cache_t*> caches;
|
||||
};
|
||||
|
||||
// initializes new container of memory consumers
|
||||
rocksdb_memory_consumers_t* rocksdb_memory_consumers_create() {
|
||||
return new rocksdb_memory_consumers_t;
|
||||
}
|
||||
|
||||
// adds datatabase to the container of memory consumers
|
||||
void rocksdb_memory_consumers_add_db(rocksdb_memory_consumers_t* consumers,
|
||||
rocksdb_t* db) {
|
||||
consumers->dbs.push_back(db);
|
||||
}
|
||||
|
||||
// adds cache to the container of memory consumers
|
||||
void rocksdb_memory_consumers_add_cache(rocksdb_memory_consumers_t* consumers,
|
||||
rocksdb_cache_t* cache) {
|
||||
consumers->caches.insert(cache);
|
||||
}
|
||||
|
||||
// deletes container with memory consumers
|
||||
void rocksdb_memory_consumers_destroy(rocksdb_memory_consumers_t* consumers) {
|
||||
delete consumers;
|
||||
}
|
||||
|
||||
// contains memory usage statistics provided by rocksdb::MemoryUtil
|
||||
struct rocksdb_memory_usage_t {
|
||||
uint64_t mem_table_total;
|
||||
uint64_t mem_table_unflushed;
|
||||
uint64_t mem_table_readers_total;
|
||||
uint64_t cache_total;
|
||||
};
|
||||
|
||||
// estimates amount of memory occupied by consumers (dbs and caches)
|
||||
rocksdb_memory_usage_t* rocksdb_approximate_memory_usage_create(
|
||||
rocksdb_memory_consumers_t* consumers, char** errptr) {
|
||||
|
||||
vector<DB*> dbs;
|
||||
for (auto db : consumers->dbs) {
|
||||
dbs.push_back(db->rep);
|
||||
}
|
||||
|
||||
unordered_set<const Cache*> cache_set;
|
||||
for (auto cache : consumers->caches) {
|
||||
cache_set.insert(const_cast<const Cache*>(cache->rep.get()));
|
||||
}
|
||||
|
||||
std::map<rocksdb::MemoryUtil::UsageType, uint64_t> usage_by_type;
|
||||
|
||||
auto status = MemoryUtil::GetApproximateMemoryUsageByType(dbs, cache_set,
|
||||
&usage_by_type);
|
||||
if (SaveError(errptr, status)) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
auto result = new rocksdb_memory_usage_t;
|
||||
result->mem_table_total = usage_by_type[MemoryUtil::kMemTableTotal];
|
||||
result->mem_table_unflushed = usage_by_type[MemoryUtil::kMemTableUnFlushed];
|
||||
result->mem_table_readers_total = usage_by_type[MemoryUtil::kTableReadersTotal];
|
||||
result->cache_total = usage_by_type[MemoryUtil::kCacheTotal];
|
||||
return result;
|
||||
}
|
||||
|
||||
uint64_t rocksdb_approximate_memory_usage_get_mem_table_total(
|
||||
rocksdb_memory_usage_t* memory_usage) {
|
||||
return memory_usage->mem_table_total;
|
||||
}
|
||||
|
||||
uint64_t rocksdb_approximate_memory_usage_get_mem_table_unflushed(
|
||||
rocksdb_memory_usage_t* memory_usage) {
|
||||
return memory_usage->mem_table_unflushed;
|
||||
}
|
||||
|
||||
uint64_t rocksdb_approximate_memory_usage_get_mem_table_readers_total(
|
||||
rocksdb_memory_usage_t* memory_usage) {
|
||||
return memory_usage->mem_table_readers_total;
|
||||
}
|
||||
|
||||
uint64_t rocksdb_approximate_memory_usage_get_cache_total(
|
||||
rocksdb_memory_usage_t* memory_usage) {
|
||||
return memory_usage->cache_total;
|
||||
}
|
||||
|
||||
// deletes container with memory usage estimates
|
||||
void rocksdb_approximate_memory_usage_destroy(rocksdb_memory_usage_t* usage) {
|
||||
delete usage;
|
||||
}
|
||||
|
||||
} // end extern "C"
|
||||
|
||||
#endif // !ROCKSDB_LITE
|
||||
|
||||
43
db/c_test.c
43
db/c_test.c
@ -1334,6 +1334,47 @@ int main(int argc, char** argv) {
|
||||
rocksdb_destroy_db(options, dbname, &err);
|
||||
}
|
||||
|
||||
// Check memory usage stats
|
||||
StartPhase("approximate_memory_usage");
|
||||
{
|
||||
// Create database
|
||||
db = rocksdb_open(options, dbname, &err);
|
||||
CheckNoError(err);
|
||||
|
||||
rocksdb_memory_consumers_t* consumers;
|
||||
consumers = rocksdb_memory_consumers_create();
|
||||
rocksdb_memory_consumers_add_db(consumers, db);
|
||||
rocksdb_memory_consumers_add_cache(consumers, cache);
|
||||
|
||||
// take memory usage report before write-read operation
|
||||
rocksdb_memory_usage_t* mu1;
|
||||
mu1 = rocksdb_approximate_memory_usage_create(consumers, &err);
|
||||
CheckNoError(err);
|
||||
|
||||
// Put data (this should affect memtables)
|
||||
rocksdb_put(db, woptions, "memory", 6, "test", 4, &err);
|
||||
CheckNoError(err);
|
||||
CheckGet(db, roptions, "memory", "test");
|
||||
|
||||
// take memory usage report after write-read operation
|
||||
rocksdb_memory_usage_t* mu2;
|
||||
mu2 = rocksdb_approximate_memory_usage_create(consumers, &err);
|
||||
CheckNoError(err);
|
||||
|
||||
// amount of memory used within memtables should grow
|
||||
CheckCondition(rocksdb_approximate_memory_usage_get_mem_table_total(mu2) >=
|
||||
rocksdb_approximate_memory_usage_get_mem_table_total(mu1));
|
||||
CheckCondition(rocksdb_approximate_memory_usage_get_mem_table_unflushed(mu2) >=
|
||||
rocksdb_approximate_memory_usage_get_mem_table_unflushed(mu1));
|
||||
|
||||
rocksdb_memory_consumers_destroy(consumers);
|
||||
rocksdb_approximate_memory_usage_destroy(mu1);
|
||||
rocksdb_approximate_memory_usage_destroy(mu2);
|
||||
rocksdb_close(db);
|
||||
rocksdb_destroy_db(options, dbname, &err);
|
||||
CheckNoError(err);
|
||||
}
|
||||
|
||||
StartPhase("cuckoo_options");
|
||||
{
|
||||
rocksdb_cuckoo_table_options_t* cuckoo_options;
|
||||
@ -1675,7 +1716,7 @@ int main(int argc, char** argv) {
|
||||
db = rocksdb_open(options, dbname, &err);
|
||||
CheckNoError(err);
|
||||
}
|
||||
|
||||
|
||||
StartPhase("cleanup");
|
||||
rocksdb_close(db);
|
||||
rocksdb_options_destroy(options);
|
||||
|
||||
@ -123,6 +123,8 @@ typedef struct rocksdb_transaction_t rocksdb_transaction_t;
|
||||
typedef struct rocksdb_checkpoint_t rocksdb_checkpoint_t;
|
||||
typedef struct rocksdb_wal_iterator_t rocksdb_wal_iterator_t;
|
||||
typedef struct rocksdb_wal_readoptions_t rocksdb_wal_readoptions_t;
|
||||
typedef struct rocksdb_memory_consumers_t rocksdb_memory_consumers_t;
|
||||
typedef struct rocksdb_memory_usage_t rocksdb_memory_usage_t;
|
||||
|
||||
/* DB operations */
|
||||
|
||||
@ -1672,6 +1674,33 @@ extern ROCKSDB_LIBRARY_API void rocksdb_pinnableslice_destroy(
|
||||
extern ROCKSDB_LIBRARY_API const char* rocksdb_pinnableslice_value(
|
||||
const rocksdb_pinnableslice_t* t, size_t* vlen);
|
||||
|
||||
extern ROCKSDB_LIBRARY_API rocksdb_memory_consumers_t*
|
||||
rocksdb_memory_consumers_create();
|
||||
extern ROCKSDB_LIBRARY_API void rocksdb_memory_consumers_add_db(
|
||||
rocksdb_memory_consumers_t* consumers, rocksdb_t* db);
|
||||
extern ROCKSDB_LIBRARY_API void rocksdb_memory_consumers_add_cache(
|
||||
rocksdb_memory_consumers_t* consumers, rocksdb_cache_t* cache);
|
||||
extern ROCKSDB_LIBRARY_API void rocksdb_memory_consumers_destroy(
|
||||
rocksdb_memory_consumers_t* consumers);
|
||||
extern ROCKSDB_LIBRARY_API rocksdb_memory_usage_t*
|
||||
rocksdb_approximate_memory_usage_create(rocksdb_memory_consumers_t* consumers,
|
||||
char** errptr);
|
||||
extern ROCKSDB_LIBRARY_API void rocksdb_approximate_memory_usage_destroy(
|
||||
rocksdb_memory_usage_t* usage);
|
||||
|
||||
extern ROCKSDB_LIBRARY_API uint64_t
|
||||
rocksdb_approximate_memory_usage_get_mem_table_total(
|
||||
rocksdb_memory_usage_t* memory_usage);
|
||||
extern ROCKSDB_LIBRARY_API uint64_t
|
||||
rocksdb_approximate_memory_usage_get_mem_table_unflushed(
|
||||
rocksdb_memory_usage_t* memory_usage);
|
||||
extern ROCKSDB_LIBRARY_API uint64_t
|
||||
rocksdb_approximate_memory_usage_get_mem_table_readers_total(
|
||||
rocksdb_memory_usage_t* memory_usage);
|
||||
extern ROCKSDB_LIBRARY_API uint64_t
|
||||
rocksdb_approximate_memory_usage_get_cache_total(
|
||||
rocksdb_memory_usage_t* memory_usage);
|
||||
|
||||
#ifdef __cplusplus
|
||||
} /* end extern "C" */
|
||||
#endif
|
||||
|
||||
Loading…
Reference in New Issue
Block a user