andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

BlobDB::Open() should put all existing trash files to delete scheduler (#5103)

Browse Source 
Summary:
Right now, BlobDB::Open() fails to put all trash files to delete scheduler,
which causes some trash files permanently untracked.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5103

Differential Revision: D14606095

Pulled By: siying

fbshipit-source-id: 41a9437a2948abb235c0ed85f9a04612d0e50183
This commit is contained in:
Siying Dong 2019-03-26 10:43:22 -07:00 committed by Sagar Vemuri
parent bb2eb06720
commit 37eb632072
3 changed files with 69 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
utilities/blob_db/blob_db_impl.cc
View File

@ -31,6 +31,7 @@
#include "util/logging.h"
#include "util/mutexlock.h"
#include "util/random.h"
#include "util/sst_file_manager_impl.h"
#include "util/stop_watch.h"
#include "util/sync_point.h"
#include "util/timer_queue.h"
@ -180,6 +181,12 @@ Status BlobDBImpl::Open(std::vector<ColumnFamilyHandle*>* handles) {
return s;
}
db_impl_ = static_cast_with_check<DBImpl, DB>(db_->GetRootDB());
// Add trash files in blob dir to file delete scheduler.
SstFileManagerImpl* sfm = static_cast<SstFileManagerImpl*>(
db_impl_->immutable_db_options().sst_file_manager.get());
DeleteScheduler::CleanupDirectory(env_, sfm, blob_dir_);
UpdateLiveSSTSize();
// Start background jobs.

2
utilities/blob_db/blob_db_impl.h
View File

@ -197,6 +197,8 @@ class BlobDBImpl : public BlobDB {
void TEST_DeleteObsoleteFiles();
uint64_t TEST_live_sst_size();
const std::string& TEST_blob_dir() const { return blob_dir_; }
#endif // !NDEBUG
private:

61
utilities/blob_db/blob_db_test.cc
View File

@ -18,6 +18,7 @@
#include "rocksdb/utilities/debug.h"
#include "util/cast_util.h"
#include "util/fault_injection_test_env.h"
#include "util/file_util.h"
#include "util/random.h"
#include "util/sst_file_manager_impl.h"
#include "util/string_util.h"
@ -810,12 +811,70 @@ TEST_F(BlobDBTest, SstFileManager) {
ASSERT_EQ(0, files_deleted_directly);
Destroy();
// Make sure that DestroyBlobDB() also goes through delete scheduler.
ASSERT_GE(2, files_scheduled_to_delete);
ASSERT_GE(files_scheduled_to_delete, 2);
ASSERT_EQ(0, files_deleted_directly);
SyncPoint::GetInstance()->DisableProcessing();
sfm->WaitForEmptyTrash();
}
TEST_F(BlobDBTest, SstFileManagerRestart) {
int files_deleted_directly = 0;
int files_scheduled_to_delete = 0;
rocksdb::SyncPoint::GetInstance()->SetCallBack(
"SstFileManagerImpl::ScheduleFileDeletion",
[&](void * /*arg*/) { files_scheduled_to_delete++; });
rocksdb::SyncPoint::GetInstance()->SetCallBack(
"DeleteScheduler::DeleteFile",
[&](void * /*arg*/) { files_deleted_directly++; });
// run the same test for Get(), MultiGet() and Iterator each.
std::shared_ptr<SstFileManager> sst_file_manager(
NewSstFileManager(mock_env_.get()));
sst_file_manager->SetDeleteRateBytesPerSecond(1);
SstFileManagerImpl *sfm =
static_cast<SstFileManagerImpl *>(sst_file_manager.get());
BlobDBOptions bdb_options;
bdb_options.min_blob_size = 0;
Options db_options;
SyncPoint::GetInstance()->EnableProcessing();
db_options.sst_file_manager = sst_file_manager;
Open(bdb_options, db_options);
std::string blob_dir = blob_db_impl()->TEST_blob_dir();
blob_db_->Put(WriteOptions(), "foo", "bar");
Close();
// Create 3 dummy trash files under the blob_dir
CreateFile(db_options.env, blob_dir + "/000666.blob.trash", "", false);
CreateFile(db_options.env, blob_dir + "/000888.blob.trash", "", true);
CreateFile(db_options.env, blob_dir + "/something_not_match.trash", "",
false);
// Make sure that reopening the DB rescan the existing trash files
Open(bdb_options, db_options);
ASSERT_GE(files_scheduled_to_delete, 3);
ASSERT_EQ(0, files_deleted_directly);
sfm->WaitForEmptyTrash();
// There should be exact one file under the blob dir now.
std::vector<std::string> all_files;
ASSERT_OK(db_options.env->GetChildren(blob_dir, &all_files));
int nfiles = 0;
for (const auto &f : all_files) {
assert(!f.empty());
if (f[0] == '.') {
continue;
}
nfiles++;
}
ASSERT_EQ(nfiles, 1);
SyncPoint::GetInstance()->DisableProcessing();
}
TEST_F(BlobDBTest, SnapshotAndGarbageCollection) {
BlobDBOptions bdb_options;
bdb_options.min_blob_size = 0;