diff --git a/HISTORY.md b/HISTORY.md index 2dc424cfb..fb27f9591 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -68,6 +68,7 @@ ## New Features * Introduced an option `BlockBasedTableBuilder::detect_filter_construct_corruption` for detecting corruption during Bloom Filter (format_version >= 5) and Ribbon Filter construction. * Improved the SstDumpTool to read the comparator from table properties and use it to read the SST File. +* Extended the column family statistics in the info log so the total amount of garbage in the blob files and the blob file space amplification factor are also logged. Also exposed the blob file space amp via the `rocksdb.blob-stats` DB property. ## 6.29.0 (01/21/2022) Note: The next release will be major release 7.0. See https://github.com/facebook/rocksdb/issues/9390 for more info. diff --git a/db/blob/db_blob_basic_test.cc b/db/blob/db_blob_basic_test.cc index 7f027fe3f..d3845eec6 100644 --- a/db/blob/db_blob_basic_test.cc +++ b/db/blob/db_blob_basic_test.cc @@ -694,18 +694,21 @@ TEST_F(DBBlobBasicTest, Properties) { constexpr char key3[] = "key3"; constexpr size_t key3_size = sizeof(key3) - 1; - constexpr char blob[] = "0000000000"; + constexpr char blob[] = "00000000000000"; constexpr size_t blob_size = sizeof(blob) - 1; + constexpr char longer_blob[] = "00000000000000000000"; + constexpr size_t longer_blob_size = sizeof(longer_blob) - 1; + ASSERT_OK(Put(key1, blob)); - ASSERT_OK(Put(key2, blob)); + ASSERT_OK(Put(key2, longer_blob)); ASSERT_OK(Flush()); constexpr size_t first_blob_file_expected_size = BlobLogHeader::kSize + BlobLogRecord::CalculateAdjustmentForRecordHeader(key1_size) + blob_size + - BlobLogRecord::CalculateAdjustmentForRecordHeader(key2_size) + blob_size + - BlobLogFooter::kSize; + BlobLogRecord::CalculateAdjustmentForRecordHeader(key2_size) + + longer_blob_size + BlobLogFooter::kSize; ASSERT_OK(Put(key3, blob)); ASSERT_OK(Flush()); @@ -747,7 +750,12 @@ TEST_F(DBBlobBasicTest, Properties) { ASSERT_OK(db_->CompactRange(CompactRangeOptions(), begin, end)); constexpr size_t expected_garbage_size = - BlobLogRecord::CalculateAdjustmentForRecordHeader(key2_size) + blob_size; + BlobLogRecord::CalculateAdjustmentForRecordHeader(key2_size) + + longer_blob_size; + + constexpr double expected_space_amp = + static_cast(total_expected_size) / + (total_expected_size - expected_garbage_size); // Blob file stats std::string blob_stats; @@ -757,7 +765,7 @@ TEST_F(DBBlobBasicTest, Properties) { oss << "Number of blob files: 2\nTotal size of blob files: " << total_expected_size << "\nTotal size of garbage in blob files: " << expected_garbage_size - << '\n'; + << "\nBlob file space amplification: " << expected_space_amp << '\n'; ASSERT_EQ(blob_stats, oss.str()); } diff --git a/db/internal_stats.cc b/db/internal_stats.cc index b7809a675..2df672379 100644 --- a/db/internal_stats.cc +++ b/db/internal_stats.cc @@ -781,24 +781,14 @@ bool InternalStats::HandleBlobStats(std::string* value, Slice /*suffix*/) { const auto* vstorage = current->storage_info(); assert(vstorage); - const auto& blob_files = vstorage->GetBlobFiles(); - - uint64_t total_file_size = 0; - uint64_t total_garbage_size = 0; - - for (const auto& meta : blob_files) { - assert(meta); - - total_file_size += meta->GetBlobFileSize(); - total_garbage_size += meta->GetGarbageBlobBytes(); - } + const auto blob_st = vstorage->GetBlobStats(); std::ostringstream oss; - oss << "Number of blob files: " << blob_files.size() - << "\nTotal size of blob files: " << total_file_size - << "\nTotal size of garbage in blob files: " << total_garbage_size - << '\n'; + oss << "Number of blob files: " << vstorage->GetBlobFiles().size() + << "\nTotal size of blob files: " << blob_st.total_file_size + << "\nTotal size of garbage in blob files: " << blob_st.total_garbage_size + << "\nBlob file space amplification: " << blob_st.space_amp << '\n'; value->append(oss.str()); @@ -824,7 +814,7 @@ bool InternalStats::HandleLiveBlobFileSize(uint64_t* value, DBImpl* /*db*/, const auto* vstorage = current->storage_info(); assert(vstorage); - *value = vstorage->GetTotalBlobFileSize(); + *value = vstorage->GetBlobStats().total_file_size; return true; } @@ -1681,10 +1671,13 @@ void InternalStats::DumpCFStatsNoFileHistogram(std::string* value) { } } + const auto blob_st = vstorage->GetBlobStats(); + snprintf(buf, sizeof(buf), - "\nBlob file count: %" ROCKSDB_PRIszt ", total size: %.1f GB\n\n", - vstorage->GetBlobFiles().size(), - vstorage->GetTotalBlobFileSize() / kGB); + "\nBlob file count: %" ROCKSDB_PRIszt + ", total size: %.1f GB, garbage size: %.1f GB, space amp: %.1f\n\n", + vstorage->GetBlobFiles().size(), blob_st.total_file_size / kGB, + blob_st.total_garbage_size / kGB, blob_st.space_amp); value->append(buf); uint64_t now_micros = clock_->NowMicros(); diff --git a/db/version_set.h b/db/version_set.h index 14026052e..e9d789954 100644 --- a/db/version_set.h +++ b/db/version_set.h @@ -356,16 +356,30 @@ class VersionStorageInfo { } // REQUIRES: This version has been saved (see VersionBuilder::SaveTo) - uint64_t GetTotalBlobFileSize() const { - uint64_t total_blob_bytes = 0; + struct BlobStats { + uint64_t total_file_size = 0; + uint64_t total_garbage_size = 0; + double space_amp = 0.0; + }; + + BlobStats GetBlobStats() const { + uint64_t total_file_size = 0; + uint64_t total_garbage_size = 0; for (const auto& meta : blob_files_) { assert(meta); - total_blob_bytes += meta->GetBlobFileSize(); + total_file_size += meta->GetBlobFileSize(); + total_garbage_size += meta->GetGarbageBlobBytes(); } - return total_blob_bytes; + double space_amp = 0.0; + if (total_file_size > total_garbage_size) { + space_amp = static_cast(total_file_size) / + (total_file_size - total_garbage_size); + } + + return BlobStats{total_file_size, total_garbage_size, space_amp}; } const ROCKSDB_NAMESPACE::LevelFilesBrief& LevelFilesBrief(int level) const {