andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Refactor ObsoleteFilesTest to inherit from DBTestBase (#5820)

Browse Source 
Summary:
Make class ObsoleteFilesTest inherit from DBTestBase.

Test plan (on devserver):
```
$COMPILE_WITH_ASAN=1 make obsolete_files_test
$./obsolete_files_test
```
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5820

Differential Revision: D17452348

Pulled By: riversand963

fbshipit-source-id: b09f4581a18022ca2bfd79f2836c0bf7083f5f25
This commit is contained in:
Yanqin Jin 2019-09-18 11:49:41 -07:00 committed by Facebook Github Bot
parent 3a408eeae9
commit 6a279037cf
2 changed files with 62 additions and 108 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@ -1512,7 +1512,7 @@ options_file_test: db/options_file_test.o $(LIBOBJECTS) $(TESTHARNESS)
deletefile_test: db/deletefile_test.o $(LIBOBJECTS) $(TESTHARNESS) deletefile_test: db/deletefile_test.o $(LIBOBJECTS) $(TESTHARNESS)
$(AM_LINK) $(AM_LINK)
obsolete_files_test: db/obsolete_files_test.o $(LIBOBJECTS) $(TESTHARNESS) obsolete_files_test: db/obsolete_files_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
$(AM_LINK) $(AM_LINK)
rocksdb_dump: tools/dump/rocksdb_dump.o $(LIBOBJECTS) rocksdb_dump: tools/dump/rocksdb_dump.o $(LIBOBJECTS)

168
db/obsolete_files_test.cc
View File

@ -14,9 +14,11 @@
#include <string> #include <string>
#include <vector> #include <vector>
#include "db/db_impl/db_impl.h" #include "db/db_impl/db_impl.h"
#include "db/db_test_util.h"
#include "db/version_set.h" #include "db/version_set.h"
#include "db/write_batch_internal.h" #include "db/write_batch_internal.h"
#include "file/filename.h" #include "file/filename.h"
#include "port/stack_trace.h"
#include "rocksdb/db.h" #include "rocksdb/db.h"
#include "rocksdb/env.h" #include "rocksdb/env.h"
#include "rocksdb/transaction_log.h" #include "rocksdb/transaction_log.h"
@ -32,60 +34,10 @@ using std::flush;
namespace rocksdb { namespace rocksdb {
class ObsoleteFilesTest : public testing::Test { class ObsoleteFilesTest : public DBTestBase {
public: public:
std::string dbname_; ObsoleteFilesTest()
Options options_; : DBTestBase("/obsolete_files_test"), wal_dir_(dbname_ + "/wal_files") {}
DB* db_;
Env* env_;
int numlevels_;
ObsoleteFilesTest() {
db_ = nullptr;
env_ = Env::Default();
// Trigger compaction when the number of level 0 files reaches 2.
options_.level0_file_num_compaction_trigger = 2;
options_.disable_auto_compactions = false;
options_.delete_obsolete_files_period_micros = 0; // always do full purge
options_.enable_thread_tracking = true;
options_.write_buffer_size = 1024*1024*1000;
options_.target_file_size_base = 1024*1024*1000;
options_.max_bytes_for_level_base = 1024*1024*1000;
options_.WAL_ttl_seconds = 300; // Used to test log files
options_.WAL_size_limit_MB = 1024; // Used to test log files
dbname_ = test::PerThreadDBPath("obsolete_files_test");
options_.wal_dir = dbname_ + "/wal_files";
// clean up all the files that might have been there before
std::vector<std::string> old_files;
env_->GetChildren(dbname_, &old_files);
for (auto file : old_files) {
env_->DeleteFile(dbname_ + "/" + file);
}
env_->GetChildren(options_.wal_dir, &old_files);
for (auto file : old_files) {
env_->DeleteFile(options_.wal_dir + "/" + file);
}
DestroyDB(dbname_, options_);
numlevels_ = 7;
EXPECT_OK(ReopenDB(true));
}
Status ReopenDB(bool create) {
delete db_;
if (create) {
DestroyDB(dbname_, options_);
}
db_ = nullptr;
options_.create_if_missing = create;
return DB::Open(options_, dbname_, &db_);
}
void CloseDB() {
delete db_;
db_ = nullptr;
}
void AddKeys(int numkeys, int startkey) { void AddKeys(int numkeys, int startkey) {
WriteOptions options; WriteOptions options;
@ -98,50 +50,24 @@ class ObsoleteFilesTest : public testing::Test {
} }
} }
int numKeysInLevels(
std::vector<LiveFileMetaData> &metadata,
std::vector<int> *keysperlevel = nullptr) {
if (keysperlevel != nullptr) {
keysperlevel->resize(numlevels_);
}
int numKeys = 0;
for (size_t i = 0; i < metadata.size(); i++) {
int startkey = atoi(metadata[i].smallestkey.c_str());
int endkey = atoi(metadata[i].largestkey.c_str());
int numkeysinfile = (endkey - startkey + 1);
numKeys += numkeysinfile;
if (keysperlevel != nullptr) {
(*keysperlevel)[(int)metadata[i].level] += numkeysinfile;
}
fprintf(stderr, "level %d name %s smallest %s largest %s\n",
metadata[i].level, metadata[i].name.c_str(),
metadata[i].smallestkey.c_str(),
metadata[i].largestkey.c_str());
}
return numKeys;
}
void createLevel0Files(int numFiles, int numKeysPerFile) { void createLevel0Files(int numFiles, int numKeysPerFile) {
int startKey = 0; int startKey = 0;
DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
for (int i = 0; i < numFiles; i++) { for (int i = 0; i < numFiles; i++) {
AddKeys(numKeysPerFile, startKey); AddKeys(numKeysPerFile, startKey);
startKey += numKeysPerFile; startKey += numKeysPerFile;
ASSERT_OK(dbi->TEST_FlushMemTable()); ASSERT_OK(dbfull()->TEST_FlushMemTable());
ASSERT_OK(dbi->TEST_WaitForFlushMemTable()); ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
} }
} }
void CheckFileTypeCounts(std::string& dir, void CheckFileTypeCounts(const std::string& dir, int required_log,
int required_log, int required_sst, int required_manifest) {
int required_sst,
int required_manifest) {
std::vector<std::string> filenames; std::vector<std::string> filenames;
env_->GetChildren(dir, &filenames); env_->GetChildren(dir, &filenames);
int log_cnt = 0, sst_cnt = 0, manifest_cnt = 0; int log_cnt = 0;
int sst_cnt = 0;
int manifest_cnt = 0;
for (auto file : filenames) { for (auto file : filenames) {
uint64_t number; uint64_t number;
FileType type; FileType type;
@ -155,9 +81,31 @@ class ObsoleteFilesTest : public testing::Test {
ASSERT_EQ(required_sst, sst_cnt); ASSERT_EQ(required_sst, sst_cnt);
ASSERT_EQ(required_manifest, manifest_cnt); ASSERT_EQ(required_manifest, manifest_cnt);
} }
void ReopenDB() {
Options options = CurrentOptions();
// Trigger compaction when the number of level 0 files reaches 2.
options.create_if_missing = true;
options.level0_file_num_compaction_trigger = 2;
options.disable_auto_compactions = false;
options.delete_obsolete_files_period_micros = 0; // always do full purge
options.enable_thread_tracking = true;
options.write_buffer_size = 1024 * 1024 * 1000;
options.target_file_size_base = 1024 * 1024 * 1000;
options.max_bytes_for_level_base = 1024 * 1024 * 1000;
options.WAL_ttl_seconds = 300; // Used to test log files
options.WAL_size_limit_MB = 1024; // Used to test log files
options.wal_dir = wal_dir_;
Destroy(options);
Reopen(options);
}
const std::string wal_dir_;
}; };
TEST_F(ObsoleteFilesTest, RaceForObsoleteFileDeletion) { TEST_F(ObsoleteFilesTest, RaceForObsoleteFileDeletion) {
ReopenDB();
SyncPoint::GetInstance()->DisableProcessing();
SyncPoint::GetInstance()->LoadDependency({ SyncPoint::GetInstance()->LoadDependency({
{"DBImpl::BackgroundCallCompaction:FoundObsoleteFiles", {"DBImpl::BackgroundCallCompaction:FoundObsoleteFiles",
"ObsoleteFilesTest::RaceForObsoleteFileDeletion:1"}, "ObsoleteFilesTest::RaceForObsoleteFileDeletion:1"},
@ -178,28 +126,26 @@ TEST_F(ObsoleteFilesTest, RaceForObsoleteFileDeletion) {
SyncPoint::GetInstance()->EnableProcessing(); SyncPoint::GetInstance()->EnableProcessing();
createLevel0Files(2, 50000); createLevel0Files(2, 50000);
CheckFileTypeCounts(options_.wal_dir, 1, 0, 0); CheckFileTypeCounts(wal_dir_, 1, 0, 0);
DBImpl* dbi = reinterpret_cast<DBImpl*>(db_); port::Thread user_thread([this]() {
port::Thread user_thread([&]() {
JobContext jobCxt(0); JobContext jobCxt(0);
TEST_SYNC_POINT("ObsoleteFilesTest::RaceForObsoleteFileDeletion:1"); TEST_SYNC_POINT("ObsoleteFilesTest::RaceForObsoleteFileDeletion:1");
dbi->TEST_LockMutex(); dbfull()->TEST_LockMutex();
dbi->FindObsoleteFiles(&jobCxt, dbfull()->FindObsoleteFiles(&jobCxt, true /* force=true */,
true /* force=true */, false /* no_full_scan=false */); false /* no_full_scan=false */);
dbi->TEST_UnlockMutex(); dbfull()->TEST_UnlockMutex();
TEST_SYNC_POINT("ObsoleteFilesTest::RaceForObsoleteFileDeletion:2"); TEST_SYNC_POINT("ObsoleteFilesTest::RaceForObsoleteFileDeletion:2");
dbi->PurgeObsoleteFiles(jobCxt); dbfull()->PurgeObsoleteFiles(jobCxt);
jobCxt.Clean(); jobCxt.Clean();
}); });
user_thread.join(); user_thread.join();
CloseDB();
SyncPoint::GetInstance()->DisableProcessing();
} }
TEST_F(ObsoleteFilesTest, DeleteObsoleteOptionsFile) { TEST_F(ObsoleteFilesTest, DeleteObsoleteOptionsFile) {
ReopenDB();
SyncPoint::GetInstance()->DisableProcessing();
std::vector<uint64_t> optsfiles_nums; std::vector<uint64_t> optsfiles_nums;
std::vector<bool> optsfiles_keep; std::vector<bool> optsfiles_keep;
SyncPoint::GetInstance()->SetCallBack( SyncPoint::GetInstance()->SetCallBack(
@ -213,23 +159,22 @@ TEST_F(ObsoleteFilesTest, DeleteObsoleteOptionsFile) {
SyncPoint::GetInstance()->EnableProcessing(); SyncPoint::GetInstance()->EnableProcessing();
createLevel0Files(2, 50000); createLevel0Files(2, 50000);
CheckFileTypeCounts(options_.wal_dir, 1, 0, 0); CheckFileTypeCounts(wal_dir_, 1, 0, 0);
DBImpl* dbi = static_cast<DBImpl*>(db_); ASSERT_OK(dbfull()->DisableFileDeletions());
ASSERT_OK(dbi->DisableFileDeletions());
for (int i = 0; i != 4; ++i) { for (int i = 0; i != 4; ++i) {
if (i % 2) { if (i % 2) {
ASSERT_OK(dbi->SetOptions(dbi->DefaultColumnFamily(), ASSERT_OK(dbfull()->SetOptions(dbfull()->DefaultColumnFamily(),
{{"paranoid_file_checks", "false"}})); {{"paranoid_file_checks", "false"}}));
} else { } else {
ASSERT_OK(dbi->SetOptions(dbi->DefaultColumnFamily(), ASSERT_OK(dbfull()->SetOptions(dbfull()->DefaultColumnFamily(),
{{"paranoid_file_checks", "true"}})); {{"paranoid_file_checks", "true"}}));
} }
} }
ASSERT_OK(dbi->EnableFileDeletions(true /* force */)); ASSERT_OK(dbfull()->EnableFileDeletions(true /* force */));
ASSERT_EQ(optsfiles_nums.size(), optsfiles_keep.size()); ASSERT_EQ(optsfiles_nums.size(), optsfiles_keep.size());
CloseDB(); Close();
std::vector<std::string> files; std::vector<std::string> files;
int opts_file_count = 0; int opts_file_count = 0;
@ -246,13 +191,22 @@ TEST_F(ObsoleteFilesTest, DeleteObsoleteOptionsFile) {
} }
} }
ASSERT_EQ(2, opts_file_count); ASSERT_EQ(2, opts_file_count);
SyncPoint::GetInstance()->DisableProcessing();
} }
} //namespace rocksdb } //namespace rocksdb
#ifdef ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS
extern "C" {
void RegisterCustomObjects(int argc, char** argv);
}
#else
void RegisterCustomObjects(int /*argc*/, char** /*argv*/) {}
#endif // !ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS
int main(int argc, char** argv) { int main(int argc, char** argv) {
rocksdb::port::InstallStackTraceHandler();
::testing::InitGoogleTest(&argc, argv); ::testing::InitGoogleTest(&argc, argv);
RegisterCustomObjects(argc, argv);
return RUN_ALL_TESTS(); return RUN_ALL_TESTS();
} }