Limit trash directory to be 25% of total DB
Summary: Update DeleteScheduler to delete files immediately if trash directory is >= 25% of DB size Closes https://github.com/facebook/rocksdb/pull/2436 Differential Revision: D5230384 Pulled By: IslamAbdelRahman fbshipit-source-id: 5cbda8ac536a3cc72c774641621edc02c8202482
This commit is contained in:
parent
9bb91e9328
commit
d713471da8
@ -337,6 +337,7 @@ TEST_F(DBSSTTest, RateLimitedDelete) {
|
|||||||
ASSERT_OK(s);
|
ASSERT_OK(s);
|
||||||
options.sst_file_manager->SetDeleteRateBytesPerSecond(rate_bytes_per_sec);
|
options.sst_file_manager->SetDeleteRateBytesPerSecond(rate_bytes_per_sec);
|
||||||
auto sfm = static_cast<SstFileManagerImpl*>(options.sst_file_manager.get());
|
auto sfm = static_cast<SstFileManagerImpl*>(options.sst_file_manager.get());
|
||||||
|
sfm->delete_scheduler()->TEST_SetMaxTrashDBRatio(1.1);
|
||||||
|
|
||||||
ASSERT_OK(TryReopen(options));
|
ASSERT_OK(TryReopen(options));
|
||||||
// Create 4 files in L0
|
// Create 4 files in L0
|
||||||
@ -402,6 +403,7 @@ TEST_F(DBSSTTest, DeleteSchedulerMultipleDBPaths) {
|
|||||||
env_, nullptr, trash_dir, rate_bytes_per_sec, false, &s));
|
env_, nullptr, trash_dir, rate_bytes_per_sec, false, &s));
|
||||||
ASSERT_OK(s);
|
ASSERT_OK(s);
|
||||||
auto sfm = static_cast<SstFileManagerImpl*>(options.sst_file_manager.get());
|
auto sfm = static_cast<SstFileManagerImpl*>(options.sst_file_manager.get());
|
||||||
|
sfm->delete_scheduler()->TEST_SetMaxTrashDBRatio(1.1);
|
||||||
|
|
||||||
DestroyAndReopen(options);
|
DestroyAndReopen(options);
|
||||||
|
|
||||||
@ -456,9 +458,14 @@ TEST_F(DBSSTTest, DestroyDBWithRateLimitedDelete) {
|
|||||||
[&](void* arg) { bg_delete_file++; });
|
[&](void* arg) { bg_delete_file++; });
|
||||||
rocksdb::SyncPoint::GetInstance()->EnableProcessing();
|
rocksdb::SyncPoint::GetInstance()->EnableProcessing();
|
||||||
|
|
||||||
|
Status s;
|
||||||
Options options = CurrentOptions();
|
Options options = CurrentOptions();
|
||||||
options.disable_auto_compactions = true;
|
options.disable_auto_compactions = true;
|
||||||
options.env = env_;
|
options.env = env_;
|
||||||
|
std::string trash_dir = test::TmpDir(env_) + "/trash";
|
||||||
|
options.sst_file_manager.reset(
|
||||||
|
NewSstFileManager(env_, nullptr, trash_dir, 0, false, &s));
|
||||||
|
ASSERT_OK(s);
|
||||||
DestroyAndReopen(options);
|
DestroyAndReopen(options);
|
||||||
|
|
||||||
// Create 4 files in L0
|
// Create 4 files in L0
|
||||||
@ -471,15 +478,12 @@ TEST_F(DBSSTTest, DestroyDBWithRateLimitedDelete) {
|
|||||||
|
|
||||||
// Close DB and destroy it using DeleteScheduler
|
// Close DB and destroy it using DeleteScheduler
|
||||||
Close();
|
Close();
|
||||||
std::string trash_dir = test::TmpDir(env_) + "/trash";
|
|
||||||
int64_t rate_bytes_per_sec = 1024 * 1024; // 1 Mb / Sec
|
|
||||||
Status s;
|
|
||||||
options.sst_file_manager.reset(NewSstFileManager(
|
|
||||||
env_, nullptr, trash_dir, rate_bytes_per_sec, false, &s));
|
|
||||||
ASSERT_OK(s);
|
|
||||||
ASSERT_OK(DestroyDB(dbname_, options));
|
|
||||||
|
|
||||||
auto sfm = static_cast<SstFileManagerImpl*>(options.sst_file_manager.get());
|
auto sfm = static_cast<SstFileManagerImpl*>(options.sst_file_manager.get());
|
||||||
|
|
||||||
|
sfm->SetDeleteRateBytesPerSecond(1024 * 1024);
|
||||||
|
sfm->delete_scheduler()->TEST_SetMaxTrashDBRatio(1.1);
|
||||||
|
ASSERT_OK(DestroyDB(dbname_, options));
|
||||||
sfm->WaitForEmptyTrash();
|
sfm->WaitForEmptyTrash();
|
||||||
// We have deleted the 4 sst files in the delete_scheduler
|
// We have deleted the 4 sst files in the delete_scheduler
|
||||||
ASSERT_EQ(bg_delete_file, 4);
|
ASSERT_EQ(bg_delete_file, 4);
|
||||||
|
@ -26,12 +26,14 @@ DeleteScheduler::DeleteScheduler(Env* env, const std::string& trash_dir,
|
|||||||
SstFileManagerImpl* sst_file_manager)
|
SstFileManagerImpl* sst_file_manager)
|
||||||
: env_(env),
|
: env_(env),
|
||||||
trash_dir_(trash_dir),
|
trash_dir_(trash_dir),
|
||||||
|
total_trash_size_(0),
|
||||||
rate_bytes_per_sec_(rate_bytes_per_sec),
|
rate_bytes_per_sec_(rate_bytes_per_sec),
|
||||||
pending_files_(0),
|
pending_files_(0),
|
||||||
closing_(false),
|
closing_(false),
|
||||||
cv_(&mu_),
|
cv_(&mu_),
|
||||||
info_log_(info_log),
|
info_log_(info_log),
|
||||||
sst_file_manager_(sst_file_manager) {
|
sst_file_manager_(sst_file_manager) {
|
||||||
|
assert(sst_file_manager != nullptr);
|
||||||
bg_thread_.reset(
|
bg_thread_.reset(
|
||||||
new port::Thread(&DeleteScheduler::BackgroundEmptyTrash, this));
|
new port::Thread(&DeleteScheduler::BackgroundEmptyTrash, this));
|
||||||
}
|
}
|
||||||
@ -49,11 +51,14 @@ DeleteScheduler::~DeleteScheduler() {
|
|||||||
|
|
||||||
Status DeleteScheduler::DeleteFile(const std::string& file_path) {
|
Status DeleteScheduler::DeleteFile(const std::string& file_path) {
|
||||||
Status s;
|
Status s;
|
||||||
if (rate_bytes_per_sec_.load() <= 0) {
|
if (rate_bytes_per_sec_.load() <= 0 ||
|
||||||
// Rate limiting is disabled
|
total_trash_size_.load() >
|
||||||
|
sst_file_manager_->GetTotalSize() * max_trash_db_ratio_) {
|
||||||
|
// Rate limiting is disabled or trash size makes up more than
|
||||||
|
// max_trash_db_ratio_ (default 25%) of the total DB size
|
||||||
TEST_SYNC_POINT("DeleteScheduler::DeleteFile");
|
TEST_SYNC_POINT("DeleteScheduler::DeleteFile");
|
||||||
s = env_->DeleteFile(file_path);
|
s = env_->DeleteFile(file_path);
|
||||||
if (s.ok() && sst_file_manager_) {
|
if (s.ok()) {
|
||||||
sst_file_manager_->OnDeleteFile(file_path);
|
sst_file_manager_->OnDeleteFile(file_path);
|
||||||
}
|
}
|
||||||
return s;
|
return s;
|
||||||
@ -66,7 +71,7 @@ Status DeleteScheduler::DeleteFile(const std::string& file_path) {
|
|||||||
ROCKS_LOG_ERROR(info_log_, "Failed to move %s to trash directory (%s)",
|
ROCKS_LOG_ERROR(info_log_, "Failed to move %s to trash directory (%s)",
|
||||||
file_path.c_str(), trash_dir_.c_str());
|
file_path.c_str(), trash_dir_.c_str());
|
||||||
s = env_->DeleteFile(file_path);
|
s = env_->DeleteFile(file_path);
|
||||||
if (s.ok() && sst_file_manager_) {
|
if (s.ok()) {
|
||||||
sst_file_manager_->OnDeleteFile(file_path);
|
sst_file_manager_->OnDeleteFile(file_path);
|
||||||
}
|
}
|
||||||
return s;
|
return s;
|
||||||
@ -123,8 +128,10 @@ Status DeleteScheduler::MoveToTrash(const std::string& file_path,
|
|||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (s.ok() && sst_file_manager_) {
|
if (s.ok()) {
|
||||||
sst_file_manager_->OnMoveFile(file_path, *path_in_trash);
|
uint64_t trash_file_size = 0;
|
||||||
|
sst_file_manager_->OnMoveFile(file_path, *path_in_trash, &trash_file_size);
|
||||||
|
total_trash_size_.fetch_add(trash_file_size);
|
||||||
}
|
}
|
||||||
return s;
|
return s;
|
||||||
}
|
}
|
||||||
@ -210,10 +217,9 @@ Status DeleteScheduler::DeleteTrashFile(const std::string& path_in_trash,
|
|||||||
*deleted_bytes = 0;
|
*deleted_bytes = 0;
|
||||||
} else {
|
} else {
|
||||||
*deleted_bytes = file_size;
|
*deleted_bytes = file_size;
|
||||||
if (sst_file_manager_) {
|
total_trash_size_.fetch_sub(file_size);
|
||||||
sst_file_manager_->OnDeleteFile(path_in_trash);
|
sst_file_manager_->OnDeleteFile(path_in_trash);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
return s;
|
return s;
|
||||||
}
|
}
|
||||||
|
@ -59,6 +59,13 @@ class DeleteScheduler {
|
|||||||
// file_path => error status
|
// file_path => error status
|
||||||
std::map<std::string, Status> GetBackgroundErrors();
|
std::map<std::string, Status> GetBackgroundErrors();
|
||||||
|
|
||||||
|
uint64_t GetTotalTrashSize() { return total_trash_size_.load(); }
|
||||||
|
|
||||||
|
void TEST_SetMaxTrashDBRatio(double r) {
|
||||||
|
assert(r >= 0);
|
||||||
|
max_trash_db_ratio_ = r;
|
||||||
|
}
|
||||||
|
|
||||||
private:
|
private:
|
||||||
Status MoveToTrash(const std::string& file_path, std::string* path_in_trash);
|
Status MoveToTrash(const std::string& file_path, std::string* path_in_trash);
|
||||||
|
|
||||||
@ -70,6 +77,8 @@ class DeleteScheduler {
|
|||||||
Env* env_;
|
Env* env_;
|
||||||
// Path to the trash directory
|
// Path to the trash directory
|
||||||
std::string trash_dir_;
|
std::string trash_dir_;
|
||||||
|
// total size of trash directory
|
||||||
|
std::atomic<uint64_t> total_trash_size_;
|
||||||
// Maximum number of bytes that should be deleted per second
|
// Maximum number of bytes that should be deleted per second
|
||||||
std::atomic<int64_t> rate_bytes_per_sec_;
|
std::atomic<int64_t> rate_bytes_per_sec_;
|
||||||
// Mutex to protect queue_, pending_files_, bg_errors_, closing_
|
// Mutex to protect queue_, pending_files_, bg_errors_, closing_
|
||||||
@ -93,6 +102,9 @@ class DeleteScheduler {
|
|||||||
InstrumentedMutex file_move_mu_;
|
InstrumentedMutex file_move_mu_;
|
||||||
Logger* info_log_;
|
Logger* info_log_;
|
||||||
SstFileManagerImpl* sst_file_manager_;
|
SstFileManagerImpl* sst_file_manager_;
|
||||||
|
// If the trash size constitutes for more than 25% of the total DB size
|
||||||
|
// we will start deleting new files passed to DeleteScheduler immediately
|
||||||
|
double max_trash_db_ratio_ = 0.25;
|
||||||
static const uint64_t kMicrosInSecond = 1000 * 1000LL;
|
static const uint64_t kMicrosInSecond = 1000 * 1000LL;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -17,6 +17,7 @@
|
|||||||
#include "rocksdb/env.h"
|
#include "rocksdb/env.h"
|
||||||
#include "rocksdb/options.h"
|
#include "rocksdb/options.h"
|
||||||
#include "util/delete_scheduler.h"
|
#include "util/delete_scheduler.h"
|
||||||
|
#include "util/sst_file_manager_impl.h"
|
||||||
#include "util/string_util.h"
|
#include "util/string_util.h"
|
||||||
#include "util/sync_point.h"
|
#include "util/sync_point.h"
|
||||||
#include "util/testharness.h"
|
#include "util/testharness.h"
|
||||||
@ -61,20 +62,26 @@ class DeleteSchedulerTest : public testing::Test {
|
|||||||
std::string data(size, 'A');
|
std::string data(size, 'A');
|
||||||
EXPECT_OK(f->Append(data));
|
EXPECT_OK(f->Append(data));
|
||||||
EXPECT_OK(f->Close());
|
EXPECT_OK(f->Close());
|
||||||
|
sst_file_mgr_->OnAddFile(file_path);
|
||||||
return file_path;
|
return file_path;
|
||||||
}
|
}
|
||||||
|
|
||||||
void NewDeleteScheduler() {
|
void NewDeleteScheduler() {
|
||||||
ASSERT_OK(env_->CreateDirIfMissing(trash_dir_));
|
ASSERT_OK(env_->CreateDirIfMissing(trash_dir_));
|
||||||
delete_scheduler_.reset(new DeleteScheduler(
|
sst_file_mgr_.reset(
|
||||||
env_, trash_dir_, rate_bytes_per_sec_, nullptr, nullptr));
|
new SstFileManagerImpl(env_, nullptr, trash_dir_, rate_bytes_per_sec_));
|
||||||
|
delete_scheduler_ = sst_file_mgr_->delete_scheduler();
|
||||||
|
// Tests in this file are for DeleteScheduler component and dont create any
|
||||||
|
// DBs, so we need to use set this value to 100% (instead of default 25%)
|
||||||
|
delete_scheduler_->TEST_SetMaxTrashDBRatio(1.1);
|
||||||
}
|
}
|
||||||
|
|
||||||
Env* env_;
|
Env* env_;
|
||||||
std::string dummy_files_dir_;
|
std::string dummy_files_dir_;
|
||||||
std::string trash_dir_;
|
std::string trash_dir_;
|
||||||
int64_t rate_bytes_per_sec_;
|
int64_t rate_bytes_per_sec_;
|
||||||
std::shared_ptr<DeleteScheduler> delete_scheduler_;
|
DeleteScheduler* delete_scheduler_;
|
||||||
|
std::unique_ptr<SstFileManagerImpl> sst_file_mgr_;
|
||||||
};
|
};
|
||||||
|
|
||||||
// Test the basic functionality of DeleteScheduler (Rate Limiting).
|
// Test the basic functionality of DeleteScheduler (Rate Limiting).
|
||||||
@ -389,7 +396,7 @@ TEST_F(DeleteSchedulerTest, DestructorWithNonEmptyQueue) {
|
|||||||
|
|
||||||
// Deleting 100 files will need >28 hours to delete
|
// Deleting 100 files will need >28 hours to delete
|
||||||
// we will delete the DeleteScheduler while delete queue is not empty
|
// we will delete the DeleteScheduler while delete queue is not empty
|
||||||
delete_scheduler_.reset();
|
sst_file_mgr_.reset();
|
||||||
|
|
||||||
ASSERT_LT(bg_delete_file, 100);
|
ASSERT_LT(bg_delete_file, 100);
|
||||||
ASSERT_GT(CountFilesInDir(trash_dir_), 0);
|
ASSERT_GT(CountFilesInDir(trash_dir_), 0);
|
||||||
@ -508,6 +515,42 @@ TEST_F(DeleteSchedulerTest, DISABLED_DynamicRateLimiting1) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
TEST_F(DeleteSchedulerTest, ImmediateDeleteOn25PercDBSize) {
|
||||||
|
int bg_delete_file = 0;
|
||||||
|
int fg_delete_file = 0;
|
||||||
|
rocksdb::SyncPoint::GetInstance()->SetCallBack(
|
||||||
|
"DeleteScheduler::DeleteTrashFile:DeleteFile",
|
||||||
|
[&](void* arg) { bg_delete_file++; });
|
||||||
|
rocksdb::SyncPoint::GetInstance()->SetCallBack(
|
||||||
|
"DeleteScheduler::DeleteFile", [&](void* arg) { fg_delete_file++; });
|
||||||
|
|
||||||
|
rocksdb::SyncPoint::GetInstance()->EnableProcessing();
|
||||||
|
|
||||||
|
int num_files = 100; // 100 files
|
||||||
|
uint64_t file_size = 1024 * 10; // 100 KB as a file size
|
||||||
|
rate_bytes_per_sec_ = 1; // 1 byte per sec (very slow trash delete)
|
||||||
|
|
||||||
|
NewDeleteScheduler();
|
||||||
|
delete_scheduler_->TEST_SetMaxTrashDBRatio(0.25);
|
||||||
|
|
||||||
|
std::vector<std::string> generated_files;
|
||||||
|
for (int i = 0; i < num_files; i++) {
|
||||||
|
std::string file_name = "file" + ToString(i) + ".data";
|
||||||
|
generated_files.push_back(NewDummyFile(file_name, file_size));
|
||||||
|
}
|
||||||
|
|
||||||
|
for (std::string& file_name : generated_files) {
|
||||||
|
delete_scheduler_->DeleteFile(file_name);
|
||||||
|
}
|
||||||
|
|
||||||
|
// When we end up with 24 files in trash we will start
|
||||||
|
// deleting new files immediately
|
||||||
|
ASSERT_EQ(fg_delete_file, 74);
|
||||||
|
ASSERT_EQ(CountFilesInDir(trash_dir_), 25);
|
||||||
|
|
||||||
|
rocksdb::SyncPoint::GetInstance()->DisableProcessing();
|
||||||
|
}
|
||||||
|
|
||||||
} // namespace rocksdb
|
} // namespace rocksdb
|
||||||
|
|
||||||
int main(int argc, char** argv) {
|
int main(int argc, char** argv) {
|
||||||
|
@ -51,9 +51,13 @@ Status SstFileManagerImpl::OnDeleteFile(const std::string& file_path) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
Status SstFileManagerImpl::OnMoveFile(const std::string& old_path,
|
Status SstFileManagerImpl::OnMoveFile(const std::string& old_path,
|
||||||
const std::string& new_path) {
|
const std::string& new_path,
|
||||||
|
uint64_t* file_size) {
|
||||||
{
|
{
|
||||||
MutexLock l(&mu_);
|
MutexLock l(&mu_);
|
||||||
|
if (file_size != nullptr) {
|
||||||
|
*file_size = tracked_files_[old_path];
|
||||||
|
}
|
||||||
OnAddFileImpl(new_path, tracked_files_[old_path]);
|
OnAddFileImpl(new_path, tracked_files_[old_path]);
|
||||||
OnDeleteFileImpl(old_path);
|
OnDeleteFileImpl(old_path);
|
||||||
}
|
}
|
||||||
|
@ -39,7 +39,8 @@ class SstFileManagerImpl : public SstFileManager {
|
|||||||
Status OnDeleteFile(const std::string& file_path);
|
Status OnDeleteFile(const std::string& file_path);
|
||||||
|
|
||||||
// DB will call OnMoveFile whenever an sst file is move to a new path.
|
// DB will call OnMoveFile whenever an sst file is move to a new path.
|
||||||
Status OnMoveFile(const std::string& old_path, const std::string& new_path);
|
Status OnMoveFile(const std::string& old_path, const std::string& new_path,
|
||||||
|
uint64_t* file_size = nullptr);
|
||||||
|
|
||||||
// Update the maximum allowed space that should be used by RocksDB, if
|
// Update the maximum allowed space that should be used by RocksDB, if
|
||||||
// the total size of the SST files exceeds max_allowed_space, writes to
|
// the total size of the SST files exceeds max_allowed_space, writes to
|
||||||
@ -76,6 +77,8 @@ class SstFileManagerImpl : public SstFileManager {
|
|||||||
// destructor to be called.
|
// destructor to be called.
|
||||||
virtual void WaitForEmptyTrash();
|
virtual void WaitForEmptyTrash();
|
||||||
|
|
||||||
|
DeleteScheduler* delete_scheduler() { return &delete_scheduler_; }
|
||||||
|
|
||||||
private:
|
private:
|
||||||
// REQUIRES: mutex locked
|
// REQUIRES: mutex locked
|
||||||
void OnAddFileImpl(const std::string& file_path, uint64_t file_size);
|
void OnAddFileImpl(const std::string& file_path, uint64_t file_size);
|
||||||
|
Loading…
Reference in New Issue
Block a user