andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Don't ignore deletion rate limit if WAL dir is different (#8967)

Browse Source 
Summary:
If WAL dir is different from the DB dir, we should still honor the SstFileManager deletion rate limit for SST files.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/8967

Test Plan: Add a new unit test in db_sst_test

Reviewed By: pdillinger

Differential Revision: D31220116

Pulled By: anand1976

fbshipit-source-id: bcde8a53a7d728e15e597fb5d07ee86c1b38bd28
This commit is contained in:
anand76 2021-09-30 13:24:13 -07:00 committed by Facebook GitHub Bot
parent 2acffecca1
commit 532ff334d9
4 changed files with 28 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
HISTORY.md
View File

@ -2,6 +2,7 @@
## Unreleased ## Unreleased
### Bug Fixes ### Bug Fixes
* Fixes a bug in directed IO mode when calling MultiGet() for blobs in the same blob file. The bug is caused by not sorting the blob read requests by file offsets. * Fixes a bug in directed IO mode when calling MultiGet() for blobs in the same blob file. The bug is caused by not sorting the blob read requests by file offsets.
* Fix the incorrect disabling of SST rate limited deletion when the WAL and DB are in different directories. Only WAL rate limited deletion should be disabled if its in a different directory.
### New Features ### New Features
* Provided support for SingleDelete with user defined timestamp. * Provided support for SingleDelete with user defined timestamp.

6
db/db_impl/db_impl.cc
View File

@ -4059,8 +4059,10 @@ Status DestroyDB(const std::string& dbname, const Options& options,
del = DestroyDB(path_to_delete, options); del = DestroyDB(path_to_delete, options);
} else if (type == kTableFile || type == kWalFile || } else if (type == kTableFile || type == kWalFile ||
type == kBlobFile) { type == kBlobFile) {
del = DeleteDBFile(&soptions, path_to_delete, dbname, del = DeleteDBFile(
/*force_bg=*/false, /*force_fg=*/!wal_in_db_path); &soptions, path_to_delete, dbname,
/*force_bg=*/false,
/*force_fg=*/(type == kWalFile) ? !wal_in_db_path : false);
} else { } else {
del = env->DeleteFile(path_to_delete); del = env->DeleteFile(path_to_delete);
} }

8
db/db_impl/db_impl_files.cc
View File

@ -330,9 +330,11 @@ void DBImpl::DeleteObsoleteFileImpl(int job_id, const std::string& fname,
Status file_deletion_status; Status file_deletion_status;
if (type == kTableFile || type == kBlobFile || type == kWalFile) { if (type == kTableFile || type == kBlobFile || type == kWalFile) {
file_deletion_status = // Rate limit WAL deletion only if its in the DB dir
DeleteDBFile(&immutable_db_options_, fname, path_to_sync, file_deletion_status = DeleteDBFile(
/*force_bg=*/false, /*force_fg=*/!wal_in_db_path_); &immutable_db_options_, fname, path_to_sync,
/*force_bg=*/false,
/*force_fg=*/(type == kWalFile) ? !wal_in_db_path_ : false);
} else { } else {
file_deletion_status = env_->DeleteFile(fname); file_deletion_status = env_->DeleteFile(fname);
} }

20
db/db_sst_test.cc
View File

@ -602,7 +602,14 @@ TEST_F(DBSSTTest, DBWithSstFileManagerForBlobFilesWithGC) {
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
} }
TEST_F(DBSSTTest, RateLimitedDelete) { class DBSSTTestRateLimit : public DBSSTTest,
public ::testing::WithParamInterface<bool> {
public:
DBSSTTestRateLimit() : DBSSTTest() {}
~DBSSTTestRateLimit() override {}
};
TEST_P(DBSSTTestRateLimit, RateLimitedDelete) {
Destroy(last_options_); Destroy(last_options_);
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({ ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency({
{"DBSSTTest::RateLimitedDelete:1", {"DBSSTTest::RateLimitedDelete:1",
@ -640,11 +647,15 @@ TEST_F(DBSSTTest, RateLimitedDelete) {
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing(); ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
bool different_wal_dir = GetParam();
Options options = CurrentOptions(); Options options = CurrentOptions();
SetTimeElapseOnlySleepOnReopen(&options); SetTimeElapseOnlySleepOnReopen(&options);
options.disable_auto_compactions = true; options.disable_auto_compactions = true;
options.env = env_; options.env = env_;
options.statistics = CreateDBStatistics(); options.statistics = CreateDBStatistics();
if (different_wal_dir) {
options.wal_dir = alternative_wal_dir_;
}
int64_t rate_bytes_per_sec = 1024 * 10; // 10 Kbs / Sec int64_t rate_bytes_per_sec = 1024 * 10; // 10 Kbs / Sec
Status s; Status s;
@ -656,7 +667,9 @@ TEST_F(DBSSTTest, RateLimitedDelete) {
sfm->delete_scheduler()->SetMaxTrashDBRatio(1.1); sfm->delete_scheduler()->SetMaxTrashDBRatio(1.1);
WriteOptions wo; WriteOptions wo;
wo.disableWAL = true; if (!different_wal_dir) {
wo.disableWAL = true;
}
Reopen(options); Reopen(options);
// Create 4 files in L0 // Create 4 files in L0
for (char v = 'a'; v <= 'd'; v++) { for (char v = 'a'; v <= 'd'; v++) {
@ -701,6 +714,9 @@ TEST_F(DBSSTTest, RateLimitedDelete) {
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
} }
INSTANTIATE_TEST_CASE_P(RateLimitedDelete, DBSSTTestRateLimit,
::testing::Bool());
TEST_F(DBSSTTest, RateLimitedWALDelete) { TEST_F(DBSSTTest, RateLimitedWALDelete) {
Destroy(last_options_); Destroy(last_options_);