andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Support backup and checkpoint in db_stress (#4005)

Browse Source 
Summary:
Add the `backup_one_in` and `checkpoint_one_in` options to periodically trigger backups and checkpoints. The directory names contain thread ID to avoid clashing with parallel backups/checkpoints. Enable checkpoint in crash test so our CI runs will use it. Didn't enable backup in crash test since it copies all the files which is too slow.
Closes https://github.com/facebook/rocksdb/pull/4005

Differential Revision: D8472275

Pulled By: ajkr

fbshipit-source-id: ff91bdc37caac4ffd97aea8df96b3983313ac1d5
This commit is contained in:
Andrew Kryczka 2018-06-18 19:23:42 -07:00 committed by Facebook Github Bot
parent de2c6fb158
commit 8585059ae0
2 changed files with 61 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
tools/db_crashtest.py
View File

@ -24,6 +24,7 @@ default_params = {
"acquire_snapshot_one_in": 10000,
"block_size": 16384,
"cache_size": 1048576,
"checkpoint_one_in": 1000000,
"clear_column_family_one_in": 0,
"compact_files_one_in": 1000000,
"compact_range_one_in": 1000000,

61
tools/db_stress.cc
View File

@ -54,6 +54,8 @@ int main() {
#include "rocksdb/slice.h"
#include "rocksdb/slice_transform.h"
#include "rocksdb/statistics.h"
#include "rocksdb/utilities/backupable_db.h"
#include "rocksdb/utilities/checkpoint.h"
#include "rocksdb/utilities/db_ttl.h"
#include "rocksdb/utilities/options_util.h"
#include "rocksdb/utilities/transaction.h"
@ -382,6 +384,16 @@ DEFINE_bool(use_txn, false,
"Use TransactionDB. Currently the default write policy is "
"TxnDBWritePolicy::WRITE_PREPARED");
DEFINE_int32(backup_one_in, 0,
"If non-zero, then CreateNewBackup() will be called once for "
"every N operations on average. 0 indicates CreateNewBackup() "
"is disabled.");
DEFINE_int32(checkpoint_one_in, 0,
"If non-zero, then CreateCheckpoint() will be called once for "
"every N operations on average. 0 indicates CreateCheckpoint() "
"is disabled.");
DEFINE_int32(compact_files_one_in, 0,
"If non-zero, then CompactFiles() will be called once for every N "
"operations on average. 0 indicates CompactFiles() is disabled.");
@ -1756,7 +1768,54 @@ class StressTest {
MaybeClearOneColumnFamily(thread);
#ifndef ROCKSDB_LITE // Lite does not support GetColumnFamilyMetaData
#ifndef ROCKSDB_LITE
if (FLAGS_checkpoint_one_in > 0 &&
thread->rand.Uniform(FLAGS_checkpoint_one_in) == 0) {
std::string checkpoint_dir =
FLAGS_db + "/.checkpoint" + ToString(thread->tid);
Checkpoint* checkpoint;
Status s = Checkpoint::Create(db_, &checkpoint);
if (s.ok()) {
s = checkpoint->CreateCheckpoint(checkpoint_dir);
}
std::vector<std::string> files;
if (s.ok()) {
s = FLAGS_env->GetChildren(checkpoint_dir, &files);
}
size_t file_idx = 0;
while (s.ok() && file_idx < files.size()) {
if (files[file_idx] != "." && files[file_idx] != "..") {
s = FLAGS_env->DeleteFile(checkpoint_dir + "/" + files[file_idx]);
}
++file_idx;
}
if (s.ok()) {
s = FLAGS_env->DeleteDir(checkpoint_dir);
}
if (!s.ok()) {
printf("A checkpoint operation failed with: %s\n",
s.ToString().c_str());
}
}
if (FLAGS_backup_one_in > 0 &&
thread->rand.Uniform(FLAGS_backup_one_in) == 0) {
std::string backup_dir = FLAGS_db + "/.backup" + ToString(thread->tid);
BackupableDBOptions backup_opts(backup_dir);
BackupEngine* backup_engine;
Status s = BackupEngine::Open(FLAGS_env, backup_opts, &backup_engine);
if (s.ok()) {
s = backup_engine->CreateNewBackup(db_);
}
if (s.ok()) {
s = backup_engine->PurgeOldBackups(0 /* num_backups_to_keep */);
}
if (!s.ok()) {
printf("A BackupEngine operation failed with: %s\n",
s.ToString().c_str());
}
}
if (FLAGS_compact_files_one_in > 0 &&
thread->rand.Uniform(FLAGS_compact_files_one_in) == 0) {
auto* random_cf =