andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Support BlobDB in db_stress (#6230)

Browse Source 
Summary:
The patch adds support for BlobDB to `db_stress`. Note that BlobDB currently does
not support (amongst other features) Column Families or the `SingleDelete` API,
so for now, those should be disabled on the command line when running `db_stress` in
BlobDB mode (using `-column_families=1` and `-nooverwritepercent=0`,
respectively). Also, some BlobDB features that do not go well with the verification logic
in `db_stress` like TTL and FIFO eviction are not supported currently.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6230

Test Plan:
```
./db_stress -max_key=100000 -use_blob_db -column_families=1 -nooverwritepercent=0 -reopen=1 -blob_db_file_size=1000000 -target_file_size_base=1000000 -blob_db_enable_gc -blob_db_gc_cutoff=0.1 -blob_db_min_blob_size=10 -blob_db_bytes_per_sync=16384
```

Differential Revision: D19191476

Pulled By: ltamasi

fbshipit-source-id: 35840452af8c5e6095249c7fd9a53a119a0985fc
This commit is contained in:
Levi Tamasi 2019-12-20 10:25:48 -08:00 committed by Facebook Github Bot
parent 670a916d01
commit 786c3d45ed
3 changed files with 76 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
db_stress_tool/db_stress_common.h
View File

@ -65,6 +65,7 @@
#include "util/mutexlock.h"
#include "util/random.h"
#include "util/string_util.h"
#include "utilities/blob_db/blob_db.h"
// SyncPoint is not supported in Released Windows Mode.
#if !(defined NDEBUG) || !defined(OS_WIN)
#include "test_util/sync_point.h"
@ -211,6 +212,15 @@ DECLARE_int32(verify_checksum_one_in);
DECLARE_int32(verify_db_one_in);
DECLARE_int32(continuous_verification_interval);
#ifndef ROCKSDB_LITE
DECLARE_bool(use_blob_db);
DECLARE_uint64(blob_db_min_blob_size);
DECLARE_uint64(blob_db_bytes_per_sync);
DECLARE_uint64(blob_db_file_size);
DECLARE_bool(blob_db_enable_gc);
DECLARE_double(blob_db_gc_cutoff);
#endif // !ROCKSDB_LITE
const long KB = 1024;
const int kRandomValueMaxFactor = 3;
const int kValueMaxLen = 100;

26
db_stress_tool/db_stress_gflags.cc
View File

@ -270,6 +270,32 @@ DEFINE_bool(allow_concurrent_memtable_write, false,
DEFINE_bool(enable_write_thread_adaptive_yield, true,
"Use a yielding spin loop for brief writer thread waits.");
#ifndef ROCKSDB_LITE
// BlobDB Options
DEFINE_bool(use_blob_db, false, "Use BlobDB.");
DEFINE_uint64(blob_db_min_blob_size,
rocksdb::blob_db::BlobDBOptions().min_blob_size,
"Smallest blob to store in a file. Blobs smaller than this "
"will be inlined with the key in the LSM tree.");
DEFINE_uint64(blob_db_bytes_per_sync,
rocksdb::blob_db::BlobDBOptions().bytes_per_sync,
"Sync blob files once per every N bytes written.");
DEFINE_uint64(blob_db_file_size,
rocksdb::blob_db::BlobDBOptions().blob_file_size,
"Target size of each blob file.");
DEFINE_bool(blob_db_enable_gc,
rocksdb::blob_db::BlobDBOptions().enable_garbage_collection,
"Enable BlobDB garbage collection.");
DEFINE_double(blob_db_gc_cutoff,
rocksdb::blob_db::BlobDBOptions().garbage_collection_cutoff,
"Cutoff ratio for BlobDB garbage collection.");
#endif // !ROCKSDB_LITE
static const bool FLAGS_subcompactions_dummy __attribute__((__unused__)) =
RegisterFlagValidator(&FLAGS_subcompactions, &ValidateUint32Range);

47
db_stress_tool/db_stress_test_base.cc
View File

@ -38,10 +38,20 @@ StressTest::StressTest()
FLAGS_env->DeleteFile(FLAGS_db + "/" + files[i]);
}
}
Options options;
// Remove files without preserving manfiest files
options.env = FLAGS_env->target();
Status s = DestroyDB(FLAGS_db, options);
#ifndef ROCKSDB_LITE
const Status s = !FLAGS_use_blob_db
? DestroyDB(FLAGS_db, options)
: blob_db::DestroyBlobDB(FLAGS_db, options,
blob_db::BlobDBOptions());
#else
const Status s = DestroyDB(FLAGS_db, options);
#endif // !ROCKSDB_LITE
if (!s.ok()) {
fprintf(stderr, "Cannot destroy original db: %s\n", s.ToString().c_str());
exit(1);
@ -1444,6 +1454,10 @@ void StressTest::PrintEnv() const {
fprintf(stdout, "Format version : %d\n", FLAGS_format_version);
fprintf(stdout, "TransactionDB : %s\n",
FLAGS_use_txn ? "true" : "false");
#ifndef ROCKSDB_LITE
fprintf(stdout, "BlobDB : %s\n",
FLAGS_use_blob_db ? "true" : "false");
#endif // !ROCKSDB_LITE
fprintf(stdout, "Read only mode : %s\n",
FLAGS_read_only ? "true" : "false");
fprintf(stdout, "Atomic flush : %s\n",
@ -1750,12 +1764,31 @@ void StressTest::Open() {
new DbStressListener(FLAGS_db, options_.db_paths, cf_descriptors));
options_.create_missing_column_families = true;
if (!FLAGS_use_txn) {
if (db_preload_finished_.load() && FLAGS_read_only) {
s = DB::OpenForReadOnly(DBOptions(options_), FLAGS_db, cf_descriptors,
&column_families_, &db_);
} else {
s = DB::Open(DBOptions(options_), FLAGS_db, cf_descriptors,
&column_families_, &db_);
#ifndef ROCKSDB_LITE
if (FLAGS_use_blob_db) {
blob_db::BlobDBOptions blob_db_options;
blob_db_options.min_blob_size = FLAGS_blob_db_min_blob_size;
blob_db_options.bytes_per_sync = FLAGS_blob_db_bytes_per_sync;
blob_db_options.blob_file_size = FLAGS_blob_db_file_size;
blob_db_options.enable_garbage_collection = FLAGS_blob_db_enable_gc;
blob_db_options.garbage_collection_cutoff = FLAGS_blob_db_gc_cutoff;
blob_db::BlobDB* blob_db = nullptr;
s = blob_db::BlobDB::Open(options_, blob_db_options, FLAGS_db,
cf_descriptors, &column_families_, &blob_db);
if (s.ok()) {
db_ = blob_db;
}
} else
#endif // !ROCKSDB_LITE
{
if (db_preload_finished_.load() && FLAGS_read_only) {
s = DB::OpenForReadOnly(DBOptions(options_), FLAGS_db, cf_descriptors,
&column_families_, &db_);
} else {
s = DB::Open(DBOptions(options_), FLAGS_db, cf_descriptors,
&column_families_, &db_);
}
}
} else {
#ifndef ROCKSDB_LITE