andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

support SstFileManager in db_stress (#6454)

Browse Source 
Summary:
Add some flags for configuring an SstFileManager. An
SstFileManager is only created when one or more of these flags are
set.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6454

Test Plan:
- ran it a while:

```
$ python ./tools/db_crashtest.py blackbox --simple -max_key=100000 -write_buffer_size=131072 -target_file_size_base=131072 -max_bytes_for_level_base=524288 -value_size_mult=33 --interval=10 -max_background_compactions=4 -max_background_flushes=2 -sst_file_manager_bytes_per_sec=1048576
```

- verified with strace the SstFileManager is behaving as configured:

```
$ strace -fp `pidof db_stress` -e ftruncate,unlink
...
[pid 3074805]
ftruncate(9</tmp/rocksdb_crashtest_blackbox6OJywh/000070.sst.trash>,
67423) = 0
[pid 3074805]
ftruncate(9</tmp/rocksdb_crashtest_blackbox6OJywh/000070.sst.trash>,
51039) = 0
[pid 3074805]
ftruncate(9</tmp/rocksdb_crashtest_blackbox6OJywh/000070.sst.trash>,
34655) = 0
[pid 3074805]
ftruncate(9</tmp/rocksdb_crashtest_blackbox6OJywh/000070.sst.trash>,
18271) = 0
[pid 3074805]
ftruncate(9</tmp/rocksdb_crashtest_blackbox6OJywh/000070.sst.trash>,
1887) = 0
[pid 3074805]
unlink("/tmp/rocksdb_crashtest_blackbox6OJywh/000070.sst.trash") = 0
...
```

Differential Revision: D20103315

Pulled By: ajkr

fbshipit-source-id: b3e1092747157459d244b047947a979b85c98f48
This commit is contained in:
Andrew Kryczka 2020-02-25 16:43:33 -08:00 committed by Facebook Github Bot
parent 69679e7375
commit f52db84650
3 changed files with 26 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
db_stress_tool/db_stress_common.h
View File

@ -167,6 +167,8 @@ DECLARE_double(max_bytes_for_level_multiplier);
DECLARE_int32(range_deletion_width);
DECLARE_uint64(rate_limiter_bytes_per_sec);
DECLARE_bool(rate_limit_bg_reads);
DECLARE_uint64(sst_file_manager_bytes_per_sec);
DECLARE_uint64(sst_file_manager_bytes_per_truncate);
DECLARE_bool(use_txn);
DECLARE_uint64(txn_write_policy);
DECLARE_bool(unordered_write);

8
db_stress_tool/db_stress_gflags.cc
View File

@ -430,6 +430,14 @@ DEFINE_uint64(rate_limiter_bytes_per_sec, 0, "Set options.rate_limiter value.");
DEFINE_bool(rate_limit_bg_reads, false,
"Use options.rate_limiter on compaction reads");
DEFINE_uint64(sst_file_manager_bytes_per_sec, 0,
"Set `Options::sst_file_manager` to delete at this rate. By "
"default the deletion rate is unbounded.");
DEFINE_uint64(sst_file_manager_bytes_per_truncate, 0,
"Set `Options::sst_file_manager` to delete in chunks of this "
"many bytes. By default whole files will be deleted.");
DEFINE_bool(use_txn, false,
"Use TransactionDB. Currently the default write policy is "
"TxnDBWritePolicy::WRITE_PREPARED");

16
db_stress_tool/db_stress_test_base.cc
View File

@ -12,6 +12,7 @@
#include "db_stress_tool/db_stress_common.h"
#include "db_stress_tool/db_stress_driver.h"
#include "rocksdb/convenience.h"
#include "rocksdb/sst_file_manager.h"
namespace ROCKSDB_NAMESPACE {
StressTest::StressTest()
@ -1839,6 +1840,21 @@ void StressTest::Open() {
options_.new_table_reader_for_compaction_inputs = true;
}
}
if (FLAGS_sst_file_manager_bytes_per_sec > 0 ||
FLAGS_sst_file_manager_bytes_per_truncate > 0) {
Status status;
options_.sst_file_manager.reset(NewSstFileManager(
db_stress_env, options_.info_log, "" /* trash_dir */,
static_cast<int64_t>(FLAGS_sst_file_manager_bytes_per_sec),
true /* delete_existing_trash */, &status,
0.25 /* max_trash_db_ratio */,
FLAGS_sst_file_manager_bytes_per_truncate));
if (!status.ok()) {
fprintf(stderr, "SstFileManager creation failed: %s\n",
status.ToString().c_str());
exit(1);
}
}
if (FLAGS_prefix_size == 0 && FLAGS_rep_factory == kHashSkipList) {
fprintf(stderr,