From bcc372c0c3c11fdfd11ef8d6cab33f88e472f5c7 Mon Sep 17 00:00:00 2001 From: sdong Date: Mon, 16 Dec 2019 15:24:26 -0800 Subject: [PATCH] Add some new options to crash_test (#6176) Summary: Several options are trivially added to crash test and random values are picked. Made simple test run non-dynamic level and normal test run dynamic level. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6176 Test Plan: Run crash_test and watch the printing Differential Revision: D19053955 fbshipit-source-id: 958cb43c968541ebd87ed4d91e778bd1d40e7502 --- db_stress_tool/db_stress_common.h | 4 ++++ db_stress_tool/db_stress_gflags.cc | 17 +++++++++++++++++ db_stress_tool/db_stress_test_base.cc | 15 +++++++++++++++ tools/db_crashtest.py | 19 +++++++++++++++++-- 4 files changed, 53 insertions(+), 2 deletions(-) diff --git a/db_stress_tool/db_stress_common.h b/db_stress_tool/db_stress_common.h index e77282c72..77841365b 100644 --- a/db_stress_tool/db_stress_common.h +++ b/db_stress_tool/db_stress_common.h @@ -203,6 +203,10 @@ DECLARE_int32(prefix_size); DECLARE_bool(use_merge); DECLARE_bool(use_full_merge_v1); DECLARE_int32(sync_wal_one_in); +DECLARE_bool(avoid_unnecessary_blocking_io); +DECLARE_bool(write_dbid_to_manifest); +DECLARE_uint64(max_write_batch_group_size_bytes); +DECLARE_bool(level_compaction_dynamic_level_bytes); const long KB = 1024; const int kRandomValueMaxFactor = 3; diff --git a/db_stress_tool/db_stress_gflags.cc b/db_stress_tool/db_stress_gflags.cc index 8dd6f6b05..bc9395866 100644 --- a/db_stress_tool/db_stress_gflags.cc +++ b/db_stress_tool/db_stress_gflags.cc @@ -560,4 +560,21 @@ DEFINE_bool(use_full_merge_v1, false, DEFINE_int32(sync_wal_one_in, 0, "If non-zero, then SyncWAL() will be called once for every N ops " "on average. 0 indicates that calls to SyncWAL() are disabled."); + +DEFINE_bool(avoid_unnecessary_blocking_io, + rocksdb::Options().avoid_unnecessary_blocking_io, + "If true, some expensive cleaning up operations will be moved from " + "user reads to high-pri background threads."); + +DEFINE_bool(write_dbid_to_manifest, rocksdb::Options().write_dbid_to_manifest, + "Write DB_ID to manifest"); + +DEFINE_uint64(max_write_batch_group_size_bytes, + rocksdb::Options().max_write_batch_group_size_bytes, + "Max write batch group size"); + +DEFINE_bool(level_compaction_dynamic_level_bytes, + rocksdb::Options().level_compaction_dynamic_level_bytes, + "Use dynamic level"); + #endif // GFLAGS diff --git a/db_stress_tool/db_stress_test_base.cc b/db_stress_tool/db_stress_test_base.cc index 46fd07fb3..0753c3dfc 100644 --- a/db_stress_tool/db_stress_test_base.cc +++ b/db_stress_tool/db_stress_test_base.cc @@ -1460,6 +1460,14 @@ void StressTest::PrintEnv() const { FLAGS_periodic_compaction_seconds); fprintf(stdout, "Compaction TTL : %" PRIu64 "\n", FLAGS_compaction_ttl); + fprintf(stdout, "Background Purge : %d\n", + static_cast(FLAGS_avoid_unnecessary_blocking_io)); + fprintf(stdout, "Write DB ID to manifest : %d\n", + static_cast(FLAGS_write_dbid_to_manifest)); + fprintf(stdout, "Max Write Batch Group Size: %" PRIu64 "\n", + FLAGS_max_write_batch_group_size_bytes); + fprintf(stdout, "Use dynamic level : %d\n", + static_cast(FLAGS_level_compaction_dynamic_level_bytes)); fprintf(stdout, "------------------------------------------------\n"); } @@ -1553,6 +1561,13 @@ void StressTest::Open() { options_.compaction_options_universal.max_size_amplification_percent = FLAGS_universal_max_size_amplification_percent; options_.atomic_flush = FLAGS_atomic_flush; + options_.avoid_unnecessary_blocking_io = + FLAGS_avoid_unnecessary_blocking_io; + options_.write_dbid_to_manifest = FLAGS_write_dbid_to_manifest; + options_.max_write_batch_group_size_bytes = + FLAGS_max_write_batch_group_size_bytes; + options_.level_compaction_dynamic_level_bytes = + FLAGS_level_compaction_dynamic_level_bytes; } else { #ifdef ROCKSDB_LITE fprintf(stderr, "--options_file not supported in lite mode\n"); diff --git a/tools/db_crashtest.py b/tools/db_crashtest.py index 2508b1036..4c0d94c2c 100644 --- a/tools/db_crashtest.py +++ b/tools/db_crashtest.py @@ -32,7 +32,9 @@ default_params = { "cache_index_and_filter_blocks": lambda: random.randint(0, 1), "cache_size": 1048576, "checkpoint_one_in": 1000000, - "compression_type": "snappy", + "compression_type": lambda: random.choice( + ["snappy", "none", "zlib"]), + "checksum_type" : lambda: random.choice(["kCRC32c", "kxxHash", "kxxHash64"]), "compression_max_dict_bytes": lambda: 16384 * random.randint(0, 1), "compression_zstd_max_train_bytes": lambda: 65536 * random.randint(0, 1), "clear_column_family_one_in": 0, @@ -81,7 +83,19 @@ default_params = { # Test small max_manifest_file_size in a smaller chance, as most of the # time we wnat manifest history to be preserved to help debug "max_manifest_file_size" : lambda : random.choice( - [t * 16384 if t < 3 else 1024 * 1024 * 1024 for t in range(1,30)]) + [t * 16384 if t < 3 else 1024 * 1024 * 1024 for t in range(1, 30)]), + # Sync mode might make test runs slower so running it in a smaller chance + "sync" : lambda : random.choice( + [0 if t == 0 else 1 for t in range(1, 20)]), + "compaction_readahead_size" : lambda : random.choice( + [0, 0, 1024 * 1024]), + "db_write_buffer_size" : lambda: random.choice( + [0, 0, 0, 1024 * 1024, 8 * 1024 * 1024, 128 * 1024 * 1024]), + "avoid_unnecessary_blocking_io" : random.randint(0, 1), + "write_dbid_to_manifest" : random.randint(0, 1), + "max_write_batch_group_size_bytes" : lambda: random.choice( + [16, 64, 1024 * 1024, 16 * 1024 * 1024]), + "level_compaction_dynamic_level_bytes" : True, } _TEST_DIR_ENV_VAR = 'TEST_TMPDIR' @@ -139,6 +153,7 @@ simple_default_params = { "target_file_size_multiplier": 1, "test_batches_snapshots": 0, "write_buffer_size": 32 * 1024 * 1024, + "level_compaction_dynamic_level_bytes": False, } blackbox_simple_default_params = {