andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Log blob file space amp and expose it via the rocksdb.blob-stats DB property (#9538)

Browse Source 
Summary:
Extend the periodic statistics in the info log with the total amount of garbage
in blob files and the space amplification pertaining to blob files, where the
latter is defined as `total_blob_file_size / (total_blob_file_size - total_blob_garbage_size)`.
Also expose the space amp via the `rocksdb.blob-stats` DB property.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/9538

Test Plan: `make check`

Reviewed By: riversand963

Differential Revision: D34126855

Pulled By: ltamasi

fbshipit-source-id: 3153e7a0fe0eca440322db273f4deaabaccc51b2
This commit is contained in:
Levi Tamasi 2022-02-10 12:41:19 -08:00 committed by Facebook GitHub Bot
parent b2423f8dde
commit 073ac54739
4 changed files with 45 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
HISTORY.md
View File

@ -68,6 +68,7 @@
## New Features ## New Features
* Introduced an option `BlockBasedTableBuilder::detect_filter_construct_corruption` for detecting corruption during Bloom Filter (format_version >= 5) and Ribbon Filter construction. * Introduced an option `BlockBasedTableBuilder::detect_filter_construct_corruption` for detecting corruption during Bloom Filter (format_version >= 5) and Ribbon Filter construction.
* Improved the SstDumpTool to read the comparator from table properties and use it to read the SST File. * Improved the SstDumpTool to read the comparator from table properties and use it to read the SST File.
* Extended the column family statistics in the info log so the total amount of garbage in the blob files and the blob file space amplification factor are also logged. Also exposed the blob file space amp via the `rocksdb.blob-stats` DB property.
## 6.29.0 (01/21/2022) ## 6.29.0 (01/21/2022)
Note: The next release will be major release 7.0. See https://github.com/facebook/rocksdb/issues/9390 for more info. Note: The next release will be major release 7.0. See https://github.com/facebook/rocksdb/issues/9390 for more info.

20
db/blob/db_blob_basic_test.cc
View File

@ -694,18 +694,21 @@ TEST_F(DBBlobBasicTest, Properties) {
constexpr char key3[] = "key3"; constexpr char key3[] = "key3";
constexpr size_t key3_size = sizeof(key3) - 1; constexpr size_t key3_size = sizeof(key3) - 1;
constexpr char blob[] = "0000000000"; constexpr char blob[] = "00000000000000";
constexpr size_t blob_size = sizeof(blob) - 1; constexpr size_t blob_size = sizeof(blob) - 1;
constexpr char longer_blob[] = "00000000000000000000";
constexpr size_t longer_blob_size = sizeof(longer_blob) - 1;
ASSERT_OK(Put(key1, blob)); ASSERT_OK(Put(key1, blob));
ASSERT_OK(Put(key2, blob)); ASSERT_OK(Put(key2, longer_blob));
ASSERT_OK(Flush()); ASSERT_OK(Flush());
constexpr size_t first_blob_file_expected_size = constexpr size_t first_blob_file_expected_size =
BlobLogHeader::kSize + BlobLogHeader::kSize +
BlobLogRecord::CalculateAdjustmentForRecordHeader(key1_size) + blob_size + BlobLogRecord::CalculateAdjustmentForRecordHeader(key1_size) + blob_size +
BlobLogRecord::CalculateAdjustmentForRecordHeader(key2_size) + blob_size + BlobLogRecord::CalculateAdjustmentForRecordHeader(key2_size) +
BlobLogFooter::kSize; longer_blob_size + BlobLogFooter::kSize;
ASSERT_OK(Put(key3, blob)); ASSERT_OK(Put(key3, blob));
ASSERT_OK(Flush()); ASSERT_OK(Flush());
@ -747,7 +750,12 @@ TEST_F(DBBlobBasicTest, Properties) {
ASSERT_OK(db_->CompactRange(CompactRangeOptions(), begin, end)); ASSERT_OK(db_->CompactRange(CompactRangeOptions(), begin, end));
constexpr size_t expected_garbage_size = constexpr size_t expected_garbage_size =
BlobLogRecord::CalculateAdjustmentForRecordHeader(key2_size) + blob_size; BlobLogRecord::CalculateAdjustmentForRecordHeader(key2_size) +
longer_blob_size;
constexpr double expected_space_amp =
static_cast<double>(total_expected_size) /
(total_expected_size - expected_garbage_size);
// Blob file stats // Blob file stats
std::string blob_stats; std::string blob_stats;
@ -757,7 +765,7 @@ TEST_F(DBBlobBasicTest, Properties) {
oss << "Number of blob files: 2\nTotal size of blob files: " oss << "Number of blob files: 2\nTotal size of blob files: "
<< total_expected_size << total_expected_size
<< "\nTotal size of garbage in blob files: " << expected_garbage_size << "\nTotal size of garbage in blob files: " << expected_garbage_size
<< '\n'; << "\nBlob file space amplification: " << expected_space_amp << '\n';
ASSERT_EQ(blob_stats, oss.str()); ASSERT_EQ(blob_stats, oss.str());
} }

31
db/internal_stats.cc
View File

@ -781,24 +781,14 @@ bool InternalStats::HandleBlobStats(std::string* value, Slice /*suffix*/) {
const auto* vstorage = current->storage_info(); const auto* vstorage = current->storage_info();
assert(vstorage); assert(vstorage);
const auto& blob_files = vstorage->GetBlobFiles(); const auto blob_st = vstorage->GetBlobStats();
uint64_t total_file_size = 0;
uint64_t total_garbage_size = 0;
for (const auto& meta : blob_files) {
assert(meta);
total_file_size += meta->GetBlobFileSize();
total_garbage_size += meta->GetGarbageBlobBytes();
}
std::ostringstream oss; std::ostringstream oss;
oss << "Number of blob files: " << blob_files.size() oss << "Number of blob files: " << vstorage->GetBlobFiles().size()
<< "\nTotal size of blob files: " << total_file_size << "\nTotal size of blob files: " << blob_st.total_file_size
<< "\nTotal size of garbage in blob files: " << total_garbage_size << "\nTotal size of garbage in blob files: " << blob_st.total_garbage_size
<< '\n'; << "\nBlob file space amplification: " << blob_st.space_amp << '\n';
value->append(oss.str()); value->append(oss.str());
@ -824,7 +814,7 @@ bool InternalStats::HandleLiveBlobFileSize(uint64_t* value, DBImpl* /*db*/,
const auto* vstorage = current->storage_info(); const auto* vstorage = current->storage_info();
assert(vstorage); assert(vstorage);
*value = vstorage->GetTotalBlobFileSize(); *value = vstorage->GetBlobStats().total_file_size;
return true; return true;
} }
@ -1681,10 +1671,13 @@ void InternalStats::DumpCFStatsNoFileHistogram(std::string* value) {
} }
} }
const auto blob_st = vstorage->GetBlobStats();
snprintf(buf, sizeof(buf), snprintf(buf, sizeof(buf),
"\nBlob file count: %" ROCKSDB_PRIszt ", total size: %.1f GB\n\n", "\nBlob file count: %" ROCKSDB_PRIszt
vstorage->GetBlobFiles().size(), ", total size: %.1f GB, garbage size: %.1f GB, space amp: %.1f\n\n",
vstorage->GetTotalBlobFileSize() / kGB); vstorage->GetBlobFiles().size(), blob_st.total_file_size / kGB,
blob_st.total_garbage_size / kGB, blob_st.space_amp);
value->append(buf); value->append(buf);
uint64_t now_micros = clock_->NowMicros(); uint64_t now_micros = clock_->NowMicros();

22
db/version_set.h
View File

@ -356,16 +356,30 @@ class VersionStorageInfo {
} }
// REQUIRES: This version has been saved (see VersionBuilder::SaveTo) // REQUIRES: This version has been saved (see VersionBuilder::SaveTo)
uint64_t GetTotalBlobFileSize() const { struct BlobStats {
uint64_t total_blob_bytes = 0; uint64_t total_file_size = 0;
uint64_t total_garbage_size = 0;
double space_amp = 0.0;
};
BlobStats GetBlobStats() const {
uint64_t total_file_size = 0;
uint64_t total_garbage_size = 0;
for (const auto& meta : blob_files_) { for (const auto& meta : blob_files_) {
assert(meta); assert(meta);
total_blob_bytes += meta->GetBlobFileSize(); total_file_size += meta->GetBlobFileSize();
total_garbage_size += meta->GetGarbageBlobBytes();
} }
return total_blob_bytes; double space_amp = 0.0;
if (total_file_size > total_garbage_size) {
space_amp = static_cast<double>(total_file_size) /
(total_file_size - total_garbage_size);
}
return BlobStats{total_file_size, total_garbage_size, space_amp};
} }
const ROCKSDB_NAMESPACE::LevelFilesBrief& LevelFilesBrief(int level) const { const ROCKSDB_NAMESPACE::LevelFilesBrief& LevelFilesBrief(int level) const {