BlobDB: GetLiveFiles and GetLiveFilesMetadata return relative path (#4326)

Summary:
`GetLiveFiles` and `GetLiveFilesMetadata` should return path relative to db path.

It is a separate issue when `path_relative` is false how can we return relative path. But `DBImpl::GetLiveFiles` don't handle it as well when there are multiple `db_paths`.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4326

Differential Revision: D9545904

Pulled By: yiwu-arbug

fbshipit-source-id: 6762d879fcb561df2b612e6fdfb4a6b51db03f5d
This commit is contained in:
Yi Wu 2018-08-31 11:59:49 -07:00 committed by Facebook Github Bot
parent 1cf17ba53b
commit 462ed70d64
5 changed files with 37 additions and 14 deletions

View File

@ -950,14 +950,14 @@ class DB {
// GetLiveFiles followed by GetSortedWalFiles can generate a lossless backup // GetLiveFiles followed by GetSortedWalFiles can generate a lossless backup
// Retrieve the list of all files in the database. The files are // Retrieve the list of all files in the database. The files are
// relative to the dbname and are not absolute paths. The valid size of the // relative to the dbname and are not absolute paths. Despite being relative
// manifest file is returned in manifest_file_size. The manifest file is an // paths, the file names begin with "/". The valid size of the manifest file
// ever growing file, but only the portion specified by manifest_file_size is // is returned in manifest_file_size. The manifest file is an ever growing
// valid for this snapshot. // file, but only the portion specified by manifest_file_size is valid for
// Setting flush_memtable to true does Flush before recording the live files. // this snapshot. Setting flush_memtable to true does Flush before recording
// Setting flush_memtable to false is useful when we don't want to wait for // the live files. Setting flush_memtable to false is useful when we don't
// flush which may have to wait for compaction to complete taking an // want to wait for flush which may have to wait for compaction to complete
// indeterminate time. // taking an indeterminate time.
// //
// In case you have multiple column families, even if flush_memtable is true, // In case you have multiple column families, even if flush_memtable is true,
// you still need to call GetSortedWalFiles after GetLiveFiles to compensate // you still need to call GetSortedWalFiles after GetLiveFiles to compensate

View File

@ -80,6 +80,13 @@ std::string BlobFileName(const std::string& blobdirname, uint64_t number) {
return MakeFileName(blobdirname, number, kRocksDBBlobFileExt.c_str()); return MakeFileName(blobdirname, number, kRocksDBBlobFileExt.c_str());
} }
std::string BlobFileName(const std::string& dbname, const std::string& blob_dir,
uint64_t number) {
assert(number > 0);
return MakeFileName(dbname + "/" + blob_dir, number,
kRocksDBBlobFileExt.c_str());
}
std::string ArchivalDirectory(const std::string& dir) { std::string ArchivalDirectory(const std::string& dir) {
return dir + "/" + ARCHIVAL_DIR; return dir + "/" + ARCHIVAL_DIR;
} }

View File

@ -49,6 +49,9 @@ extern std::string LogFileName(const std::string& dbname, uint64_t number);
extern std::string BlobFileName(const std::string& bdirname, uint64_t number); extern std::string BlobFileName(const std::string& bdirname, uint64_t number);
extern std::string BlobFileName(const std::string& dbname,
const std::string& blob_dir, uint64_t number);
static const std::string ARCHIVAL_DIR = "archive"; static const std::string ARCHIVAL_DIR = "archive";
extern std::string ArchivalDirectory(const std::string& dbname); extern std::string ArchivalDirectory(const std::string& dbname);

View File

@ -7,6 +7,7 @@
#include "utilities/blob_db/blob_db_impl.h" #include "utilities/blob_db/blob_db_impl.h"
#include "util/filename.h"
#include "util/logging.h" #include "util/logging.h"
#include "util/mutexlock.h" #include "util/mutexlock.h"
@ -63,6 +64,10 @@ Status BlobDBImpl::EnableFileDeletions(bool force) {
Status BlobDBImpl::GetLiveFiles(std::vector<std::string>& ret, Status BlobDBImpl::GetLiveFiles(std::vector<std::string>& ret,
uint64_t* manifest_file_size, uint64_t* manifest_file_size,
bool flush_memtable) { bool flush_memtable) {
if (!bdb_options_.path_relative) {
return Status::NotSupported(
"Not able to get relative blob file path from absolute blob_dir.");
}
// Hold a lock in the beginning to avoid updates to base DB during the call // Hold a lock in the beginning to avoid updates to base DB during the call
ReadLock rl(&mutex_); ReadLock rl(&mutex_);
Status s = db_->GetLiveFiles(ret, manifest_file_size, flush_memtable); Status s = db_->GetLiveFiles(ret, manifest_file_size, flush_memtable);
@ -72,12 +77,16 @@ Status BlobDBImpl::GetLiveFiles(std::vector<std::string>& ret,
ret.reserve(ret.size() + blob_files_.size()); ret.reserve(ret.size() + blob_files_.size());
for (auto bfile_pair : blob_files_) { for (auto bfile_pair : blob_files_) {
auto blob_file = bfile_pair.second; auto blob_file = bfile_pair.second;
ret.emplace_back(blob_file->PathName()); // Path should be relative to db_name, but begin with slash.
ret.emplace_back(
BlobFileName("", bdb_options_.blob_dir, blob_file->BlobFileNumber()));
} }
return Status::OK(); return Status::OK();
} }
void BlobDBImpl::GetLiveFilesMetaData(std::vector<LiveFileMetaData>* metadata) { void BlobDBImpl::GetLiveFilesMetaData(std::vector<LiveFileMetaData>* metadata) {
// Path should be relative to db_name.
assert(bdb_options_.path_relative);
// Hold a lock in the beginning to avoid updates to base DB during the call // Hold a lock in the beginning to avoid updates to base DB during the call
ReadLock rl(&mutex_); ReadLock rl(&mutex_);
db_->GetLiveFilesMetaData(metadata); db_->GetLiveFilesMetaData(metadata);
@ -85,7 +94,9 @@ void BlobDBImpl::GetLiveFilesMetaData(std::vector<LiveFileMetaData>* metadata) {
auto blob_file = bfile_pair.second; auto blob_file = bfile_pair.second;
LiveFileMetaData filemetadata; LiveFileMetaData filemetadata;
filemetadata.size = blob_file->GetFileSize(); filemetadata.size = blob_file->GetFileSize();
filemetadata.name = blob_file->PathName(); // Path should be relative to db_name, but begin with slash.
filemetadata.name =
BlobFileName("", bdb_options_.blob_dir, blob_file->BlobFileNumber());
auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(DefaultColumnFamily()); auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(DefaultColumnFamily());
filemetadata.column_family_name = cfh->GetName(); filemetadata.column_family_name = cfh->GetName();
metadata->emplace_back(filemetadata); metadata->emplace_back(filemetadata);

View File

@ -834,6 +834,8 @@ TEST_F(BlobDBTest, ColumnFamilyNotSupported) {
TEST_F(BlobDBTest, GetLiveFilesMetaData) { TEST_F(BlobDBTest, GetLiveFilesMetaData) {
Random rnd(301); Random rnd(301);
BlobDBOptions bdb_options; BlobDBOptions bdb_options;
bdb_options.blob_dir = "blob_dir";
bdb_options.path_relative = true;
bdb_options.min_blob_size = 0; bdb_options.min_blob_size = 0;
bdb_options.disable_background_tasks = true; bdb_options.disable_background_tasks = true;
Open(bdb_options); Open(bdb_options);
@ -841,16 +843,16 @@ TEST_F(BlobDBTest, GetLiveFilesMetaData) {
for (size_t i = 0; i < 100; i++) { for (size_t i = 0; i < 100; i++) {
PutRandom("key" + ToString(i), &rnd, &data); PutRandom("key" + ToString(i), &rnd, &data);
} }
auto *bdb_impl = static_cast<BlobDBImpl *>(blob_db_);
std::vector<LiveFileMetaData> metadata; std::vector<LiveFileMetaData> metadata;
bdb_impl->GetLiveFilesMetaData(&metadata); blob_db_->GetLiveFilesMetaData(&metadata);
ASSERT_EQ(1U, metadata.size()); ASSERT_EQ(1U, metadata.size());
std::string filename = dbname_ + "/blob_dir/000001.blob"; // Path should be relative to db_name, but begin with slash.
std::string filename = "/blob_dir/000001.blob";
ASSERT_EQ(filename, metadata[0].name); ASSERT_EQ(filename, metadata[0].name);
ASSERT_EQ("default", metadata[0].column_family_name); ASSERT_EQ("default", metadata[0].column_family_name);
std::vector<std::string> livefile; std::vector<std::string> livefile;
uint64_t mfs; uint64_t mfs;
bdb_impl->GetLiveFiles(livefile, &mfs, false); ASSERT_OK(blob_db_->GetLiveFiles(livefile, &mfs, false));
ASSERT_EQ(4U, livefile.size()); ASSERT_EQ(4U, livefile.size());
ASSERT_EQ(filename, livefile[3]); ASSERT_EQ(filename, livefile[3]);
VerifyDB(data); VerifyDB(data);