andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add a DB Property For Number of Deletions in Memtables

Browse Source 
Summary: Add a DB property for number of deletions in memtables. It can sometimes help people debug slowness because of too many deletes.

Test Plan: Add test cases.

Reviewers: rven, yhchiang, kradhakrishnan, igor

Reviewed By: igor

Subscribers: leveldb, dhruba, yoshinorim

Differential Revision: https://reviews.facebook.net/D35247
This commit is contained in:
sdong 2015-03-18 16:11:02 -07:00
parent f7ed654641
commit 0831a35994
8 changed files with 80 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
db/db_test.cc
View File

@ -3059,6 +3059,35 @@ TEST_F(DBTest, NumImmutableMemTable) {
// "200" is the size of the metadata of an empty skiplist, this would
// break if we change the default skiplist implementation
ASSERT_EQ(num, "200");
uint64_t int_num;
uint64_t base_total_size;
ASSERT_TRUE(dbfull()->GetIntProperty(
handles_[1], "rocksdb.estimate-num-keys", &base_total_size));
ASSERT_OK(dbfull()->Delete(writeOpt, handles_[1], "k2"));
ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "k3", ""));
ASSERT_OK(dbfull()->Delete(writeOpt, handles_[1], "k3"));
ASSERT_TRUE(dbfull()->GetIntProperty(
handles_[1], "rocksdb.num-deletes-active-mem-table", &int_num));
ASSERT_EQ(int_num, 2U);
ASSERT_TRUE(dbfull()->GetIntProperty(
handles_[1], "rocksdb.num-entries-active-mem-table", &int_num));
ASSERT_EQ(int_num, 3U);
ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "k2", big_value));
ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "k2", big_value));
ASSERT_TRUE(dbfull()->GetIntProperty(
handles_[1], "rocksdb.num-entries-imm-mem-tables", &int_num));
ASSERT_EQ(int_num, 4U);
ASSERT_TRUE(dbfull()->GetIntProperty(
handles_[1], "rocksdb.num-deletes-imm-mem-tables", &int_num));
ASSERT_EQ(int_num, 2U);
ASSERT_TRUE(dbfull()->GetIntProperty(
handles_[1], "rocksdb.estimate-num-keys", &int_num));
ASSERT_EQ(int_num, base_total_size + 1);
SetPerfLevel(kDisable);
ASSERT_TRUE(GetPerfLevel() == kDisable);
} while (ChangeCompactOptions());
@ -3198,7 +3227,7 @@ TEST_F(DBTest, GetProperty) {
ASSERT_TRUE(dbfull()->GetProperty("rocksdb.compaction-pending", &num));
ASSERT_EQ(num, "0");
ASSERT_TRUE(dbfull()->GetProperty("rocksdb.estimate-num-keys", &num));
ASSERT_EQ(num, "4");
ASSERT_EQ(num, "2");
// Verify the same set of properties through GetIntProperty
ASSERT_TRUE(
dbfull()->GetIntProperty("rocksdb.num-immutable-mem-table", &int_num));
@ -3209,7 +3238,7 @@ TEST_F(DBTest, GetProperty) {
ASSERT_TRUE(dbfull()->GetIntProperty("rocksdb.compaction-pending", &int_num));
ASSERT_EQ(int_num, 0U);
ASSERT_TRUE(dbfull()->GetIntProperty("rocksdb.estimate-num-keys", &int_num));
ASSERT_EQ(int_num, 4U);
ASSERT_EQ(int_num, 2U);
ASSERT_TRUE(
dbfull()->GetIntProperty("rocksdb.estimate-table-readers-mem", &int_num));

21
db/internal_stats.cc
View File

@ -126,6 +126,10 @@ DBPropertyType GetPropertyType(const Slice& property, bool* is_int_property,
return kNumEntriesInMutableMemtable;
} else if (in == "num-entries-imm-mem-tables") {
return kNumEntriesInImmutableMemtable;
} else if (in == "num-deletes-active-mem-table") {
return kNumDeletesInMutableMemtable;
} else if (in == "num-deletes-imm-mem-tables") {
return kNumDeletesInImmutableMemtable;
} else if (in == "estimate-num-keys") {
return kEstimatedNumKeys;
} else if (in == "estimate-table-readers-mem") {
@ -256,17 +260,28 @@ bool InternalStats::GetIntProperty(DBPropertyType property_type,
return true;
case kNumEntriesInMutableMemtable:
// Current number of entires in the active memtable
*value = cfd_->mem()->GetNumEntries();
*value = cfd_->mem()->num_entries();
return true;
case kNumEntriesInImmutableMemtable:
// Current number of entries in the immutable memtables
*value = cfd_->imm()->current()->GetTotalNumEntries();
return true;
case kNumDeletesInMutableMemtable:
// Current number of entires in the active memtable
*value = cfd_->mem()->num_deletes();
return true;
case kNumDeletesInImmutableMemtable:
// Current number of entries in the immutable memtables
*value = cfd_->imm()->current()->GetTotalNumDeletes();
return true;
case kEstimatedNumKeys:
// Estimate number of entries in the column family:
// Use estimated entries in tables + total entries in memtables.
*value = cfd_->mem()->GetNumEntries() +
cfd_->imm()->current()->GetTotalNumEntries() +
*value = cfd_->mem()->num_entries() +
cfd_->imm()->current()->GetTotalNumEntries() -
(cfd_->mem()->num_deletes() +
cfd_->imm()->current()->GetTotalNumDeletes()) *
2 +
vstorage->GetEstimatedActiveKeys();
return true;
case kNumSnapshots:

6
db/internal_stats.h
View File

@ -40,10 +40,14 @@ enum DBPropertyType : uint32_t {
kCurSizeActiveMemTable, // Return current size of the active memtable
kCurSizeAllMemTables, // Return current size of all (active + immutable)
// memtables
kNumEntriesInMutableMemtable, // Return number of entries in the mutable
kNumEntriesInMutableMemtable, // Return number of deletes in the mutable
// memtable.
kNumEntriesInImmutableMemtable, // Return sum of number of entries in all
// the immutable mem tables.
kNumDeletesInMutableMemtable, // Return number of entries in the mutable
// memtable.
kNumDeletesInImmutableMemtable, // Return sum of number of deletes in all
// the immutable mem tables.
kEstimatedNumKeys, // Estimated total number of keys in the database.
kEstimatedUsageByTableReaders, // Estimated memory by table readers.
kIsFileDeletionEnabled, // Equals disable_delete_obsolete_files_,

9
db/memtable.cc
View File

@ -65,13 +65,15 @@ MemTable::MemTable(const InternalKeyComparator& cmp,
comparator_, &allocator_, ioptions.prefix_extractor,
ioptions.info_log)),
num_entries_(0),
num_deletes_(0),
flush_in_progress_(false),
flush_completed_(false),
file_number_(0),
first_seqno_(0),
mem_next_logfile_number_(0),
locks_(moptions_.inplace_update_support ?
moptions_.inplace_update_num_locks : 0),
locks_(moptions_.inplace_update_support
? moptions_.inplace_update_num_locks
: 0),
prefix_extractor_(ioptions.prefix_extractor),
should_flush_(ShouldFlushNow()),
flush_scheduled_(false) {
@ -313,6 +315,9 @@ void MemTable::Add(SequenceNumber s, ValueType type,
assert((unsigned)(p + val_size - buf) == (unsigned)encoded_len);
table_->Insert(handle);
num_entries_++;
if (type == kTypeDeletion) {
num_deletes_++;
}
if (prefix_bloom_) {
assert(prefix_extractor_);

5
db/memtable.h
View File

@ -164,7 +164,9 @@ class MemTable {
size_t CountSuccessiveMergeEntries(const LookupKey& key);
// Get total number of entries in the mem table.
uint64_t GetNumEntries() const { return num_entries_; }
uint64_t num_entries() const { return num_entries_; }
uint64_t num_deletes() const { return num_deletes_; }
// Returns the edits area that is needed for flushing the memtable
VersionEdit* GetEdits() { return &edit_; }
@ -227,6 +229,7 @@ class MemTable {
unique_ptr<MemTableRep> table_;
uint64_t num_entries_;
uint64_t num_deletes_;
// These are used to manage memtable flushes to storage
bool flush_in_progress_; // started the flush

10
db/memtable_list.cc
View File

@ -96,7 +96,15 @@ void MemTableListVersion::AddIterators(
uint64_t MemTableListVersion::GetTotalNumEntries() const {
uint64_t total_num = 0;
for (auto& m : memlist_) {
total_num += m->GetNumEntries();
total_num += m->num_entries();
}
return total_num;
}
uint64_t MemTableListVersion::GetTotalNumDeletes() const {
uint64_t total_num = 0;
for (auto& m : memlist_) {
total_num += m->num_deletes();
}
return total_num;
}

2
db/memtable_list.h
View File

@ -57,6 +57,8 @@ class MemTableListVersion {
uint64_t GetTotalNumEntries() const;
uint64_t GetTotalNumDeletes() const;
private:
// REQUIRE: m is mutable memtable
void Add(MemTable* m);

4
include/rocksdb/db.h
View File

@ -319,6 +319,8 @@ class DB {
// "rocksdb.cur-size-all-mem-tables"
// "rocksdb.num-entries-active-mem-table"
// "rocksdb.num-entries-imm-mem-tables"
// "rocksdb.num-deletes-active-mem-table"
// "rocksdb.num-deletes-imm-mem-tables"
// "rocksdb.estimate-num-keys" - estimated keys in the column family
// "rocksdb.estimate-table-readers-mem" - estimated memory used for reding
// SST tables, that is not counted as a part of block cache.
@ -346,6 +348,8 @@ class DB {
// "rocksdb.cur-size-all-mem-tables"
// "rocksdb.num-entries-active-mem-table"
// "rocksdb.num-entries-imm-mem-tables"
// "rocksdb.num-deletes-active-mem-table"
// "rocksdb.num-deletes-imm-mem-tables"
// "rocksdb.estimate-num-keys"
// "rocksdb.estimate-table-readers-mem"
// "rocksdb.is-file-deletions-enabled"