enhance dbstress to simulate hard crash

Summary:
dbstress has an option to reopen the database. Make it such that the
previous handle is not closed before we reopen, this simulates a
situation similar to a process crash.

Added new api to DMImpl to remove the lock file.

Test Plan: run db_stress

Reviewers: emayanke

Reviewed By: emayanke

CC: leveldb

Differential Revision: https://reviews.facebook.net/D6777
This commit is contained in:
Dhruba Borthakur 2012-11-16 15:28:14 -08:00
parent c3392c9fe0
commit 62e7583f94
3 changed files with 53 additions and 6 deletions

View File

@ -265,6 +265,29 @@ DBImpl::~DBImpl() {
delete logger_;
}
// Do not flush and close database elegantly. Simulate a crash.
void DBImpl::TEST_Destroy_DBImpl() {
// ensure that no new memtable flushes can occur
flush_on_destroy_ = false;
// wait till all background compactions are done.
mutex_.Lock();
while (bg_compaction_scheduled_ || bg_logstats_scheduled_) {
bg_cv_.Wait();
}
// Prevent new compactions from occuring.
const int LargeNumber = 10000000;
bg_compaction_scheduled_ += LargeNumber;
mutex_.Unlock();
// force release the lock file.
if (db_lock_ != NULL) {
env_->UnlockFile(db_lock_);
}
}
Status DBImpl::NewDB() {
VersionEdit new_db(NumberLevels());
new_db.SetComparatorName(user_comparator()->Name());

View File

@ -78,6 +78,9 @@ class DBImpl : public DB {
// file at a level >= 1.
int64_t TEST_MaxNextLevelOverlappingBytes();
// Simulate a db crash, no elegant closing of database.
void TEST_Destroy_DBImpl();
protected:
Env* const env_;

View File

@ -419,7 +419,8 @@ class StressTest {
filter_policy_(FLAGS_bloom_bits >= 0
? NewBloomFilterPolicy(FLAGS_bloom_bits)
: NULL),
db_(NULL) {
db_(NULL),
num_times_reopened_(0) {
std::vector<std::string> files;
FLAGS_env->GetChildren(FLAGS_db, &files);
for (unsigned int i = 0; i < files.size(); i++) {
@ -457,7 +458,9 @@ class StressTest {
shared.GetCondVar()->Wait();
}
fprintf(stdout, "Starting database operations\n");
double now = FLAGS_env->NowMicros();
fprintf(stdout, "%s Starting database operations\n",
FLAGS_env->TimeToString((uint64_t) now/1000000).c_str());
shared.SetStart();
shared.GetCondVar()->SignalAll();
@ -465,7 +468,10 @@ class StressTest {
shared.GetCondVar()->Wait();
}
fprintf(stdout, "Starting verification\n");
now = FLAGS_env->NowMicros();
fprintf(stdout, "%s Starting verification\n",
FLAGS_env->TimeToString((uint64_t) now/1000000).c_str());
shared.SetStartVerify();
shared.GetCondVar()->SignalAll();
while (!shared.AllDone()) {
@ -482,7 +488,10 @@ class StressTest {
delete threads[i];
threads[i] = NULL;
}
fprintf(stdout, "Verification successful\n");
double now = FLAGS_env->NowMicros();
fprintf(stdout, "%s Verification successful\n",
FLAGS_env->TimeToString((uint64_t) now/1000000).c_str());
PrintStatistics();
}
@ -542,6 +551,7 @@ class StressTest {
for (long i = 0; i < FLAGS_ops_per_thread; i++) {
if(i != 0 && (i % (FLAGS_ops_per_thread / (FLAGS_reopen + 1))) == 0) {
{
thread->stats.FinishedSingleOp();
MutexLock l(thread->shared->GetMutex());
thread->shared->IncVotedReopen();
if (thread->shared->AllVotedReopen()) {
@ -551,6 +561,7 @@ class StressTest {
else {
thread->shared->GetCondVar()->Wait();
}
thread->stats.Start();
}
}
long rand_key = thread->rand.Next() % max_key;
@ -667,7 +678,7 @@ class StressTest {
fprintf(stdout, "Num keys per lock : %d\n",
1 << FLAGS_log2_keys_per_lock);
char* compression = "";
char* compression = (char *)std::string("").c_str();
switch (FLAGS_compression_type) {
case leveldb::kNoCompression:
compression = (char *)std::string("none").c_str();
@ -722,7 +733,16 @@ class StressTest {
}
void Reopen() {
delete db_;
// do not close the db. Just delete the lock file. This
// simulates a crash-recovery kind of situation.
((DBImpl*) db_)->TEST_Destroy_DBImpl();
db_ = NULL;
num_times_reopened_++;
double now = FLAGS_env->NowMicros();
fprintf(stdout, "%s Reopening database for the %dth time\n",
FLAGS_env->TimeToString((uint64_t) now/1000000).c_str(),
num_times_reopened_);
Open();
}
@ -739,6 +759,7 @@ class StressTest {
Cache* cache_;
const FilterPolicy* filter_policy_;
DB* db_;
int num_times_reopened_;
};
} // namespace leveldb