From 027ca5b2cd0c5392692935749ab2ebb5ade67caf Mon Sep 17 00:00:00 2001 From: Islam AbdelRahman Date: Thu, 20 Aug 2015 11:47:19 -0700 Subject: [PATCH] Total SST files size DB Property Summary: Add a new DB property that calculate the total size of files used by all RocksDB Versions Test Plan: Unittests for the new property Reviewers: igor, yhchiang, anthony, rven, kradhakrishnan, sdong Reviewed By: sdong Subscribers: dhruba Differential Revision: https://reviews.facebook.net/D44799 --- db/column_family.cc | 4 + db/column_family.h | 1 + db/db_test.cc | 177 +++++++++++++++++++++++++++++++++++++++++++ db/internal_stats.cc | 8 ++ db/internal_stats.h | 1 + db/version_set.cc | 18 +++++ db/version_set.h | 2 + include/rocksdb/db.h | 4 + 8 files changed, 215 insertions(+) diff --git a/db/column_family.cc b/db/column_family.cc index f7cb479cc..a8fec894f 100644 --- a/db/column_family.cc +++ b/db/column_family.cc @@ -478,6 +478,10 @@ uint64_t ColumnFamilyData::GetNumLiveVersions() const { return VersionSet::GetNumLiveVersions(dummy_versions_); } +uint64_t ColumnFamilyData::GetTotalSstFilesSize() const { + return VersionSet::GetTotalSstFilesSize(dummy_versions_); +} + MemTable* ColumnFamilyData::ConstructNewMemtable( const MutableCFOptions& mutable_cf_options, SequenceNumber earliest_seq) { assert(current_ != nullptr); diff --git a/db/column_family.h b/db/column_family.h index 6136b2566..e44873c7a 100644 --- a/db/column_family.h +++ b/db/column_family.h @@ -226,6 +226,7 @@ class ColumnFamilyData { Version* dummy_versions() { return dummy_versions_; } void SetCurrent(Version* current); uint64_t GetNumLiveVersions() const; // REQUIRE: DB mutex held + uint64_t GetTotalSstFilesSize() const; // REQUIRE: DB mutex held void SetMemtable(MemTable* new_mem) { mem_ = new_mem; } // See Memtable constructor for explanation of earliest_seq param. diff --git a/db/db_test.cc b/db/db_test.cc index d11c1b0bb..f7f0db3e6 100644 --- a/db/db_test.cc +++ b/db/db_test.cc @@ -8574,6 +8574,183 @@ TEST_F(DBTest, OpenDBWithInfiniteMaxOpenFiles) { } } +TEST_F(DBTest, GetTotalSstFilesSize) { + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + options.compression = kNoCompression; + DestroyAndReopen(options); + // Generate 5 files in L0 + for (int i = 0; i < 5; i++) { + for (int j = 0; j < 10; j++) { + std::string val = "val_file_" + ToString(i); + ASSERT_OK(Put(Key(j), val)); + } + Flush(); + } + ASSERT_EQ("5", FilesPerLevel(0)); + + std::vector live_files_meta; + dbfull()->GetLiveFilesMetaData(&live_files_meta); + ASSERT_EQ(live_files_meta.size(), 5); + uint64_t single_file_size = live_files_meta[0].size; + + uint64_t live_sst_files_size = 0; + uint64_t total_sst_files_size = 0; + for (const auto& file_meta : live_files_meta) { + live_sst_files_size += file_meta.size; + } + + ASSERT_TRUE(dbfull()->GetIntProperty("rocksdb.total-sst-files-size", + &total_sst_files_size)); + // Live SST files = 5 + // Total SST files = 5 + ASSERT_EQ(live_sst_files_size, 5 * single_file_size); + ASSERT_EQ(total_sst_files_size, 5 * single_file_size); + + // hold current version + std::unique_ptr iter1(dbfull()->NewIterator(ReadOptions())); + + // Compact 5 files into 1 file in L0 + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_EQ("0,1", FilesPerLevel(0)); + + live_files_meta.clear(); + dbfull()->GetLiveFilesMetaData(&live_files_meta); + ASSERT_EQ(live_files_meta.size(), 1); + + live_sst_files_size = 0; + total_sst_files_size = 0; + for (const auto& file_meta : live_files_meta) { + live_sst_files_size += file_meta.size; + } + ASSERT_TRUE(dbfull()->GetIntProperty("rocksdb.total-sst-files-size", + &total_sst_files_size)); + // Live SST files = 1 (compacted file) + // Total SST files = 6 (5 original files + compacted file) + ASSERT_EQ(live_sst_files_size, 1 * single_file_size); + ASSERT_EQ(total_sst_files_size, 6 * single_file_size); + + // hold current version + std::unique_ptr iter2(dbfull()->NewIterator(ReadOptions())); + + // Delete all keys and compact, this will delete all live files + for (int i = 0; i < 10; i++) { + ASSERT_OK(Delete(Key(i))); + } + Flush(); + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_EQ("", FilesPerLevel(0)); + + live_files_meta.clear(); + dbfull()->GetLiveFilesMetaData(&live_files_meta); + ASSERT_EQ(live_files_meta.size(), 0); + + ASSERT_TRUE(dbfull()->GetIntProperty("rocksdb.total-sst-files-size", + &total_sst_files_size)); + // Live SST files = 0 + // Total SST files = 6 (5 original files + compacted file) + ASSERT_EQ(total_sst_files_size, 6 * single_file_size); + + iter1.reset(); + ASSERT_TRUE(dbfull()->GetIntProperty("rocksdb.total-sst-files-size", + &total_sst_files_size)); + // Live SST files = 0 + // Total SST files = 1 (compacted file) + ASSERT_EQ(total_sst_files_size, 1 * single_file_size); + + iter2.reset(); + ASSERT_TRUE(dbfull()->GetIntProperty("rocksdb.total-sst-files-size", + &total_sst_files_size)); + // Live SST files = 0 + // Total SST files = 0 + ASSERT_EQ(total_sst_files_size, 0); +} + +TEST_F(DBTest, GetTotalSstFilesSizeVersionsFilesShared) { + Options options = CurrentOptions(); + options.disable_auto_compactions = true; + options.compression = kNoCompression; + DestroyAndReopen(options); + // Generate 5 files in L0 + for (int i = 0; i < 5; i++) { + ASSERT_OK(Put(Key(i), "val")); + Flush(); + } + ASSERT_EQ("5", FilesPerLevel(0)); + + std::vector live_files_meta; + dbfull()->GetLiveFilesMetaData(&live_files_meta); + ASSERT_EQ(live_files_meta.size(), 5); + uint64_t single_file_size = live_files_meta[0].size; + + uint64_t live_sst_files_size = 0; + uint64_t total_sst_files_size = 0; + for (const auto& file_meta : live_files_meta) { + live_sst_files_size += file_meta.size; + } + + ASSERT_TRUE(dbfull()->GetIntProperty("rocksdb.total-sst-files-size", + &total_sst_files_size)); + + // Live SST files = 5 + // Total SST files = 5 + ASSERT_EQ(live_sst_files_size, 5 * single_file_size); + ASSERT_EQ(total_sst_files_size, 5 * single_file_size); + + // hold current version + std::unique_ptr iter1(dbfull()->NewIterator(ReadOptions())); + + // Compaction will do trivial move from L0 to L1 + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_EQ("0,5", FilesPerLevel(0)); + + live_files_meta.clear(); + dbfull()->GetLiveFilesMetaData(&live_files_meta); + ASSERT_EQ(live_files_meta.size(), 5); + + live_sst_files_size = 0; + total_sst_files_size = 0; + for (const auto& file_meta : live_files_meta) { + live_sst_files_size += file_meta.size; + } + ASSERT_TRUE(dbfull()->GetIntProperty("rocksdb.total-sst-files-size", + &total_sst_files_size)); + // Live SST files = 5 + // Total SST files = 5 (used in 2 version) + ASSERT_EQ(live_sst_files_size, 5 * single_file_size); + ASSERT_EQ(total_sst_files_size, 5 * single_file_size); + + // hold current version + std::unique_ptr iter2(dbfull()->NewIterator(ReadOptions())); + + // Delete all keys and compact, this will delete all live files + for (int i = 0; i < 5; i++) { + ASSERT_OK(Delete(Key(i))); + } + Flush(); + ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_EQ("", FilesPerLevel(0)); + + live_files_meta.clear(); + dbfull()->GetLiveFilesMetaData(&live_files_meta); + ASSERT_EQ(live_files_meta.size(), 0); + + ASSERT_TRUE(dbfull()->GetIntProperty("rocksdb.total-sst-files-size", + &total_sst_files_size)); + // Live SST files = 0 + // Total SST files = 5 (used in 2 version) + ASSERT_EQ(total_sst_files_size, 5 * single_file_size); + + iter1.reset(); + iter2.reset(); + + ASSERT_TRUE(dbfull()->GetIntProperty("rocksdb.total-sst-files-size", + &total_sst_files_size)); + // Live SST files = 0 + // Total SST files = 0 + ASSERT_EQ(total_sst_files_size, 0); +} + INSTANTIATE_TEST_CASE_P(DBTestWithParam, DBTestWithParam, ::testing::Values(1, 4)); diff --git a/db/internal_stats.cc b/db/internal_stats.cc index 172aec6ef..5e9333a13 100644 --- a/db/internal_stats.cc +++ b/db/internal_stats.cc @@ -122,6 +122,7 @@ static const std::string oldest_snapshot_time = "oldest-snapshot-time"; static const std::string num_live_versions = "num-live-versions"; static const std::string estimate_live_data_size = "estimate-live-data-size"; static const std::string base_level = "base-level"; +static const std::string total_sst_files_size = "total-sst-files-size"; const std::string DB::Properties::kNumFilesAtLevelPrefix = rocksdb_prefix + num_files_at_level_prefix; @@ -165,6 +166,8 @@ const std::string DB::Properties::kNumLiveVersions = rocksdb_prefix + num_live_versions; const std::string DB::Properties::kEstimateLiveDataSize = rocksdb_prefix + estimate_live_data_size; +const std::string DB::Properties::kTotalSstFilesSize = + rocksdb_prefix + total_sst_files_size; DBPropertyType GetPropertyType(const Slice& property, bool* is_int_property, bool* need_out_of_mutex) { @@ -236,6 +239,8 @@ DBPropertyType GetPropertyType(const Slice& property, bool* is_int_property, return kEstimateLiveDataSize; } else if (in == base_level) { return kBaseLevel; + } else if (in == total_sst_files_size) { + return kTotalSstFilesSize; } return kUnknown; } @@ -401,6 +406,9 @@ bool InternalStats::GetIntProperty(DBPropertyType property_type, case kBaseLevel: *value = vstorage->base_level(); return true; + case kTotalSstFilesSize: + *value = cfd_->GetTotalSstFilesSize(); + return true; default: return false; } diff --git a/db/internal_stats.h b/db/internal_stats.h index 866d95b51..57d9e4ab5 100644 --- a/db/internal_stats.h +++ b/db/internal_stats.h @@ -61,6 +61,7 @@ enum DBPropertyType : uint32_t { kOldestSnapshotTime, // Unix timestamp of the first snapshot kNumLiveVersions, kEstimateLiveDataSize, // Estimated amount of live data in bytes + kTotalSstFilesSize, // Total size of all sst files. kBaseLevel, // The level that L0 data is compacted to }; diff --git a/db/version_set.cc b/db/version_set.cc index c6e0b52d9..e7284a0d6 100644 --- a/db/version_set.cc +++ b/db/version_set.cc @@ -3239,4 +3239,22 @@ uint64_t VersionSet::GetNumLiveVersions(Version* dummy_versions) { return count; } +uint64_t VersionSet::GetTotalSstFilesSize(Version* dummy_versions) { + std::unordered_set unique_files; + uint64_t total_files_size = 0; + for (Version* v = dummy_versions->next_; v != dummy_versions; v = v->next_) { + VersionStorageInfo* storage_info = v->storage_info(); + for (int level = 0; level < storage_info->num_levels_; level++) { + for (const auto& file_meta : storage_info->LevelFiles(level)) { + if (unique_files.find(file_meta->fd.packed_number_and_path_id) == + unique_files.end()) { + unique_files.insert(file_meta->fd.packed_number_and_path_id); + total_files_size += file_meta->fd.GetFileSize(); + } + } + } + } + return total_files_size; +} + } // namespace rocksdb diff --git a/db/version_set.h b/db/version_set.h index abc42e74b..39a7a2cf3 100644 --- a/db/version_set.h +++ b/db/version_set.h @@ -651,6 +651,8 @@ class VersionSet { static uint64_t GetNumLiveVersions(Version* dummy_versions); + static uint64_t GetTotalSstFilesSize(Version* dummy_versions); + private: struct ManifestWriter; diff --git a/include/rocksdb/db.h b/include/rocksdb/db.h index 624a8da82..311ed8983 100644 --- a/include/rocksdb/db.h +++ b/include/rocksdb/db.h @@ -328,6 +328,8 @@ class DB { // files are held from being deleted, by iterators or unfinished // compactions. // "rocksdb.estimate-live-data-size" +// "rocksdb.total-sst-files-size" - total size of all used sst files, this may +// slow down online queries if there are too many files. #ifndef ROCKSDB_LITE struct Properties { static const std::string kNumFilesAtLevelPrefix; @@ -353,6 +355,7 @@ class DB { static const std::string kOldestSnapshotTime; static const std::string kNumLiveVersions; static const std::string kEstimateLiveDataSize; + static const std::string kTotalSstFilesSize; }; #endif /* ROCKSDB_LITE */ @@ -383,6 +386,7 @@ class DB { // "rocksdb.oldest-snapshot-time" // "rocksdb.num-live-versions" // "rocksdb.estimate-live-data-size" + // "rocksdb.total-sst-files-size" virtual bool GetIntProperty(ColumnFamilyHandle* column_family, const Slice& property, uint64_t* value) = 0; virtual bool GetIntProperty(const Slice& property, uint64_t* value) {