andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

List blob files when using command - list_live_files_metadata (#8976)

Browse Source 
Summary:
The ldb list_live_files_metadata command does not print any information about blob files currently. We would like to add this functionality. Note that list_live_files_metadata has two different modes of operation: the one shown above, which shows the LSM tree structure, and another one, which can be enabled using the flag --sort_by_filename and simply lists the files in numerical order regardless of level. We would like to show blob files in both modes.

Changes:
1. Using GetAllColumnFamilyMetaData API instead of GetLiveFilesMetaData API for fetching live files data.

Testing:
1. Created a sample rocksdb instance using dbbench command (this creates both SST and blob files)
2. Checked if the blob files are listed or not by using ldb commands.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/8976

Reviewed By: ltamasi

Differential Revision: D31316061

Pulled By: pradeepambati

fbshipit-source-id: d15cdea192febf7a45f28deee2ba40615d3d84ab
This commit is contained in:
Pradeep Ambati 2021-09-30 15:11:49 -07:00 committed by Facebook GitHub Bot
parent 1953b63cdd
commit e5bfb91d09
2 changed files with 89 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
HISTORY.md
View File

@ -5,6 +5,7 @@
* Fix the incorrect disabling of SST rate limited deletion when the WAL and DB are in different directories. Only WAL rate limited deletion should be disabled if its in a different directory.
### New Features
* Print information about blob files when using "ldb list_live_files_metadata"
* Provided support for SingleDelete with user defined timestamp.
* Add remote compaction read/write bytes statistics: `REMOTE_COMPACT_READ_BYTES`, `REMOTE_COMPACT_WRITE_BYTES`.

165
tools/ldb_cmd.cc
View File

@ -3445,89 +3445,100 @@ void DBLiveFilesMetadataDumperCommand::DoCommand() {
}
Status s;
std::cout << "Live SST Files:" << std::endl;
std::vector<LiveFileMetaData> metadata;
db_->GetLiveFilesMetaData(&metadata);
std::vector<ColumnFamilyMetaData> metadata;
db_->GetAllColumnFamilyMetaData(&metadata);
if (sort_by_filename_) {
// Sort metadata vector by filename.
std::sort(metadata.begin(), metadata.end(),
[](const LiveFileMetaData& a, const LiveFileMetaData& b) -> bool {
std::string aName = a.db_path + a.name;
std::string bName = b.db_path + b.name;
return (aName.compare(bName) < 0);
});
for (auto& fileMetadata : metadata) {
// The fileMetada.name alwasy starts with "/",
// however fileMetada.db_path is the string provided by
// the user as an input. Therefore we check if we can
// concantenate the two string sdirectly or if we need to
// drop a possible extra "/" at the end of fileMetadata.db_path.
std::string filename = fileMetadata.db_path + "/" + fileMetadata.name;
// Drops any repeating '/' character that could happen during
// concatenation of db path and file name.
filename = NormalizePath(filename);
std::string cf = fileMetadata.column_family_name;
int level = fileMetadata.level;
std::cout << filename << " : level " << level << ", column family '" << cf
<< "'" << std::endl;
std::cout << "Live SST and Blob Files:" << std::endl;
// tuple of <file path, level, column family name>
std::vector<std::tuple<std::string, int, std::string>> all_files;
for (const auto& column_metadata : metadata) {
// Iterate Levels
const auto& levels = column_metadata.levels;
const std::string& cf = column_metadata.name;
for (const auto& level_metadata : levels) {
// Iterate SST files
const auto& sst_files = level_metadata.files;
int level = level_metadata.level;
for (const auto& sst_metadata : sst_files) {
// The SstFileMetaData.name always starts with "/",
// however SstFileMetaData.db_path is the string provided by
// the user as an input. Therefore we check if we can
// concantenate the two strings directly or if we need to
// drop a possible extra "/" at the end of SstFileMetaData.db_path.
std::string filename =
NormalizePath(sst_metadata.db_path + "/" + sst_metadata.name);
all_files.emplace_back(filename, level, cf);
} // End of for-loop over sst files
} // End of for-loop over levels
const auto& blob_files = column_metadata.blob_files;
for (const auto& blob_metadata : blob_files) {
// The BlobMetaData.blob_file_name always starts with "/",
// however BlobMetaData.blob_file_path is the string provided by
// the user as an input. Therefore we check if we can
// concantenate the two strings directly or if we need to
// drop a possible extra "/" at the end of BlobMetaData.blob_file_path.
std::string filename = NormalizePath(
blob_metadata.blob_file_path + "/" + blob_metadata.blob_file_name);
// Level for blob files is encoded as -1
all_files.emplace_back(filename, -1, cf);
} // End of for-loop over blob files
} // End of for-loop over column metadata
// Sort by filename (i.e. first entry in tuple)
std::sort(all_files.begin(), all_files.end());
for (const auto& item : all_files) {
const std::string& filename = std::get<0>(item);
int level = std::get<1>(item);
const std::string& cf = std::get<2>(item);
if (level == -1) { // Blob File
std::cout << filename << ", column family '" << cf << "'" << std::endl;
} else { // SST file
std::cout << filename << " : level " << level << ", column family '"
<< cf << "'" << std::endl;
}
}
} else {
std::map<std::string, std::map<int, std::vector<std::string>>>
filesPerLevelPerCf;
// Collect live files metadata.
// Store filenames into a 2D map, that will automatically
// sort by column family (first key) and by level (second key).
for (auto& fileMetadata : metadata) {
std::string cf = fileMetadata.column_family_name;
int level = fileMetadata.level;
if (filesPerLevelPerCf.find(cf) == filesPerLevelPerCf.end()) {
filesPerLevelPerCf.emplace(cf,
std::map<int, std::vector<std::string>>());
}
if (filesPerLevelPerCf[cf].find(level) == filesPerLevelPerCf[cf].end()) {
filesPerLevelPerCf[cf].emplace(level, std::vector<std::string>());
}
// The fileMetada.name alwasy starts with "/",
// however fileMetada.db_path is the string provided by
// the user as an input. Therefore we check if we can
// concantenate the two string sdirectly or if we need to
// drop a possible extra "/" at the end of fileMetadata.db_path.
std::string filename = fileMetadata.db_path + "/" + fileMetadata.name;
// Drops any repeating '/' character that could happen during
// concatenation of db path and file name.
filename = NormalizePath(filename);
filesPerLevelPerCf[cf][level].push_back(filename);
}
// For each column family,
// iterate through the levels and print out the live SST file names.
for (auto it = filesPerLevelPerCf.begin(); it != filesPerLevelPerCf.end();
it++) {
// it->first: Column Family name (string)
// it->second: map[level]={SST files...}.
std::cout << "===== Column Family: " << it->first
for (const auto& column_metadata : metadata) {
std::cout << "===== Column Family: " << column_metadata.name
<< " =====" << std::endl;
// For simplicity, create reference to the inner map (level={live SST
// files}).
std::map<int, std::vector<std::string>>& filesPerLevel = it->second;
int maxLevel = filesPerLevel.rbegin()->first;
std::cout << "Live SST Files:" << std::endl;
// Iterate levels
const auto& levels = column_metadata.levels;
for (const auto& level_metadata : levels) {
std::cout << "---------- level " << level_metadata.level
<< " ----------" << std::endl;
// Iterate SST files
const auto& sst_files = level_metadata.files;
for (const auto& sst_metadata : sst_files) {
// The SstFileMetaData.name always starts with "/",
// however SstFileMetaData.db_path is the string provided by
// the user as an input. Therefore we check if we can
// concantenate the two strings directly or if we need to
// drop a possible extra "/" at the end of SstFileMetaData.db_path.
std::string filename =
NormalizePath(sst_metadata.db_path + "/" + sst_metadata.name);
std::cout << filename << std::endl;
} // End of for-loop over sst files
} // End of for-loop over levels
// Even if the first few levels are empty, they are printed out.
for (int level = 0; level <= maxLevel; level++) {
std::cout << "---------- level " << level << " ----------" << std::endl;
if (filesPerLevel.find(level) != filesPerLevel.end()) {
std::vector<std::string>& fileList = filesPerLevel[level];
// Locally sort by filename for better information display.
std::sort(fileList.begin(), fileList.end());
for (const std::string& filename : fileList) {
std::cout << filename << std::endl;
}
}
} // End of for-loop over levels.
} // End of for-loop over filesPerLevelPerCf.
} // End of else ("not sort_by_filename").
std::cout << "Live Blob Files:" << std::endl;
const auto& blob_files = column_metadata.blob_files;
for (const auto& blob_metadata : blob_files) {
// The BlobMetaData.blob_file_name always starts with "/",
// however BlobMetaData.blob_file_path is the string provided by
// the user as an input. Therefore we check if we can
// concantenate the two strings directly or if we need to
// drop a possible extra "/" at the end of BlobMetaData.blob_file_path.
std::string filename = NormalizePath(
blob_metadata.blob_file_path + "/" + blob_metadata.blob_file_name);
std::cout << filename << std::endl;
} // End of for-loop over blob files
} // End of for-loop over column metadata
} // End of else ("not sort_by_filename")
std::cout << "------------------------------" << std::endl;
}