From 73895c9478297d7e1460696402cba29e83af0fb9 Mon Sep 17 00:00:00 2001 From: sdong Date: Tue, 22 Apr 2014 17:17:33 -0700 Subject: [PATCH] Expose number of entries in mem tables to users Summary: In this patch, two new DB properties are defined: rocksdb.num-immutable-mem-table and rocksdb.num-entries-imm-mem-tables, from where number of entries in mem tables can be exposed to users Test Plan: Cover the codes in db_test make all check Reviewers: haobo, ljin, igor Reviewed By: igor CC: nkg-, igor, yhchiang, dhruba, leveldb Differential Revision: https://reviews.facebook.net/D18207 Conflicts: db/db_test.cc --- db/db_test.cc | 32 ++++++++++++++++++++++++++++++++ db/internal_stats.cc | 14 +++++++++++++- db/internal_stats.h | 15 +++++++++------ db/memtable.cc | 2 ++ db/memtable.h | 5 +++++ db/memtable_list.cc | 8 ++++++++ db/memtable_list.h | 2 ++ 7 files changed, 71 insertions(+), 7 deletions(-) diff --git a/db/db_test.cc b/db/db_test.cc index 21079e02b..4a14be7cf 100644 --- a/db/db_test.cc +++ b/db/db_test.cc @@ -909,13 +909,29 @@ TEST(DBTest, Empty) { options.write_buffer_size = 100000; // Small write buffer Reopen(&options); + std::string num; + ASSERT_TRUE(dbfull()->GetProperty( + "rocksdb.num-entries-active-mem-table", &num)); + ASSERT_EQ("0", num); + ASSERT_OK(Put("foo", "v1")); ASSERT_EQ("v1", Get("foo")); + ASSERT_TRUE(dbfull()->GetProperty( + "rocksdb.num-entries-active-mem-table", &num)); + ASSERT_EQ("1", num); env_->delay_sstable_sync_.Release_Store(env_); // Block sync calls Put("k1", std::string(100000, 'x')); // Fill memtable + ASSERT_TRUE(dbfull()->GetProperty( + "rocksdb.num-entries-active-mem-table", &num)); + ASSERT_EQ("2", num); + Put("k2", std::string(100000, 'y')); // Trigger compaction ASSERT_EQ("v1", Get("foo")); + ASSERT_TRUE(dbfull()->GetProperty( + "rocksdb.num-entries-active-mem-table", &num)); + ASSERT_EQ("1", num); + env_->delay_sstable_sync_.Release_Store(nullptr); // Release sync calls } while (ChangeOptions()); } @@ -2088,6 +2104,9 @@ TEST(DBTest, NumImmutableMemTable) { ASSERT_OK(dbfull()->Put(writeOpt, "k1", big_value)); ASSERT_TRUE(dbfull()->GetProperty("rocksdb.num-immutable-mem-table", &num)); ASSERT_EQ(num, "0"); + ASSERT_TRUE(dbfull()->GetProperty( + "rocksdb.num-entries-active-mem-table", &num)); + ASSERT_EQ(num, "1"); perf_context.Reset(); Get("k1"); ASSERT_EQ(1, (int) perf_context.get_from_memtable_count); @@ -2095,6 +2114,13 @@ TEST(DBTest, NumImmutableMemTable) { ASSERT_OK(dbfull()->Put(writeOpt, "k2", big_value)); ASSERT_TRUE(dbfull()->GetProperty("rocksdb.num-immutable-mem-table", &num)); ASSERT_EQ(num, "1"); + ASSERT_TRUE(dbfull()->GetProperty( + "rocksdb.num-entries-active-mem-table", &num)); + ASSERT_EQ(num, "1"); + ASSERT_TRUE(dbfull()->GetProperty( + "rocksdb.num-entries-imm-mem-tables", &num)); + ASSERT_EQ(num, "1"); + perf_context.Reset(); Get("k1"); ASSERT_EQ(2, (int) perf_context.get_from_memtable_count); @@ -2107,6 +2133,12 @@ TEST(DBTest, NumImmutableMemTable) { &num)); ASSERT_TRUE(dbfull()->GetProperty("rocksdb.num-immutable-mem-table", &num)); ASSERT_EQ(num, "2"); + ASSERT_TRUE(dbfull()->GetProperty( + "rocksdb.num-entries-active-mem-table", &num)); + ASSERT_EQ(num, "1"); + ASSERT_TRUE(dbfull()->GetProperty( + "rocksdb.num-entries-imm-mem-tables", &num)); + ASSERT_EQ(num, "2"); perf_context.Reset(); Get("k2"); ASSERT_EQ(2, (int) perf_context.get_from_memtable_count); diff --git a/db/internal_stats.cc b/db/internal_stats.cc index 4cc049965..44c1489ce 100644 --- a/db/internal_stats.cc +++ b/db/internal_stats.cc @@ -38,6 +38,10 @@ DBPropertyType GetPropertyType(const Slice& property) { return kBackgroundErrors; } else if (in == "cur-size-active-mem-table") { return kCurSizeActiveMemTable; + } else if (in == "num-entries-active-mem-table") { + return kNumEntriesInMutableMemtable; + } else if (in == "num-entries-imm-mem-tables") { + return kNumEntriesInImmutableMemtable; } return kUnknown; } @@ -47,7 +51,7 @@ bool InternalStats::GetProperty(DBPropertyType property_type, DBImpl* db) { VersionSet* version_set = db->versions_.get(); Version* current = version_set->current(); - const MemTableList& imm = db->imm_; + MemTableList& imm = db->imm_; Slice in = property; switch (property_type) { @@ -353,6 +357,14 @@ bool InternalStats::GetProperty(DBPropertyType property_type, // Current size of the active memtable *value = std::to_string(db->mem_->ApproximateMemoryUsage()); return true; + case kNumEntriesInMutableMemtable: + // Current size of the active memtable + *value = std::to_string(db->mem_->GetNumEntries()); + return true; + case kNumEntriesInImmutableMemtable: + // Current size of the active memtable + *value = std::to_string(imm.current()->GetTotalNumEntries()); + return true; default: return false; } diff --git a/db/internal_stats.h b/db/internal_stats.h index e140e7280..4ff6e170c 100644 --- a/db/internal_stats.h +++ b/db/internal_stats.h @@ -26,13 +26,16 @@ enum DBPropertyType { kLevelStats, // Return number of files and total sizes of each level kStats, // Return general statitistics of DB kSsTables, // Return a human readable string of current SST files - kNumImmutableMemTable, // Return number of immutable mem tables - kMemtableFlushPending, // Return 1 if mem table flushing is pending, - // otherwise - // 0. - kCompactionPending, // Return 1 if a compaction is pending. Otherwise 0. - kBackgroundErrors, // Return accumulated background errors encountered. + kNumImmutableMemTable, // Return number of immutable mem tables + kMemtableFlushPending, // Return 1 if mem table flushing is pending, + // otherwise 0. + kCompactionPending, // Return 1 if a compaction is pending. Otherwise 0. + kBackgroundErrors, // Return accumulated background errors encountered. kCurSizeActiveMemTable, // Return current size of the active memtable + kNumEntriesInMutableMemtable, // Return number of entries in the mutable + // memtable. + kNumEntriesInImmutableMemtable, // Return sum of number of entries in all + // the immutable mem tables. kUnknown, }; diff --git a/db/memtable.cc b/db/memtable.cc index 0d949cb50..f3503fb65 100644 --- a/db/memtable.cc +++ b/db/memtable.cc @@ -37,6 +37,7 @@ MemTable::MemTable(const InternalKeyComparator& cmp, const Options& options) arena_(options.arena_block_size), table_(options.memtable_factory->CreateMemTableRep( comparator_, &arena_, options.prefix_extractor.get())), + num_entries_(0), flush_in_progress_(false), flush_completed_(false), file_number_(0), @@ -260,6 +261,7 @@ void MemTable::Add(SequenceNumber s, ValueType type, memcpy(p, value.data(), val_size); assert((unsigned)(p + val_size - buf) == (unsigned)encoded_len); table_->Insert(handle); + num_entries_++; if (prefix_bloom_) { assert(prefix_extractor_); diff --git a/db/memtable.h b/db/memtable.h index 451def38f..718a5c213 100644 --- a/db/memtable.h +++ b/db/memtable.h @@ -132,6 +132,9 @@ class MemTable { // key in the memtable. size_t CountSuccessiveMergeEntries(const LookupKey& key); + // Get total number of entries in the mem table. + uint64_t GetNumEntries() const { return num_entries_; } + // Returns the edits area that is needed for flushing the memtable VersionEdit* GetEdits() { return &edit_; } @@ -182,6 +185,8 @@ class MemTable { Arena arena_; unique_ptr table_; + uint64_t num_entries_; + // These are used to manage memtable flushes to storage bool flush_in_progress_; // started the flush bool flush_completed_; // finished the flush diff --git a/db/memtable_list.cc b/db/memtable_list.cc index 3c502c6de..85188d76a 100644 --- a/db/memtable_list.cc +++ b/db/memtable_list.cc @@ -77,6 +77,14 @@ void MemTableListVersion::AddIterators(const ReadOptions& options, } } +uint64_t MemTableListVersion::GetTotalNumEntries() const { + uint64_t total_num = 0; + for (auto& m : memlist_) { + total_num += m->GetNumEntries(); + } + return total_num; +} + // caller is responsible for referencing m void MemTableListVersion::Add(MemTable* m) { assert(refs_ == 1); // only when refs_ == 1 is MemTableListVersion mutable diff --git a/db/memtable_list.h b/db/memtable_list.h index 0bf376e55..3b00a2211 100644 --- a/db/memtable_list.h +++ b/db/memtable_list.h @@ -43,6 +43,8 @@ class MemTableListVersion { void AddIterators(const ReadOptions& options, std::vector* iterator_list); + uint64_t GetTotalNumEntries() const; + private: // REQUIRE: m is mutable memtable void Add(MemTable* m);