Add some new options to crash_test (#6176)

Summary:
Several options are trivially added to crash test and random values are picked.
Made simple test run non-dynamic level and normal test run dynamic level.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6176

Test Plan: Run crash_test and watch the printing

Differential Revision: D19053955

fbshipit-source-id: 958cb43c968541ebd87ed4d91e778bd1d40e7502

db_stress_tool/db_stress_common.h

@@ -203,6 +203,10 @@ DECLARE_int32(prefix_size);
 DECLARE_bool(use_merge);
 DECLARE_bool(use_full_merge_v1);
 DECLARE_int32(sync_wal_one_in);
+DECLARE_bool(avoid_unnecessary_blocking_io);
+DECLARE_bool(write_dbid_to_manifest);
+DECLARE_uint64(max_write_batch_group_size_bytes);
+DECLARE_bool(level_compaction_dynamic_level_bytes);
 
 const long KB = 1024;
 const int kRandomValueMaxFactor = 3;

db_stress_tool/db_stress_gflags.cc

@@ -560,4 +560,21 @@ DEFINE_bool(use_full_merge_v1, false,
 DEFINE_int32(sync_wal_one_in, 0,
              "If non-zero, then SyncWAL() will be called once for every N ops "
              "on average. 0 indicates that calls to SyncWAL() are disabled.");
+
+DEFINE_bool(avoid_unnecessary_blocking_io,
+            rocksdb::Options().avoid_unnecessary_blocking_io,
+            "If true, some expensive cleaning up operations will be moved from "
+            "user reads to high-pri background threads.");
+
+DEFINE_bool(write_dbid_to_manifest, rocksdb::Options().write_dbid_to_manifest,
+            "Write DB_ID to manifest");
+
+DEFINE_uint64(max_write_batch_group_size_bytes,
+              rocksdb::Options().max_write_batch_group_size_bytes,
+              "Max write batch group size");
+
+DEFINE_bool(level_compaction_dynamic_level_bytes,
+            rocksdb::Options().level_compaction_dynamic_level_bytes,
+            "Use dynamic level");
+
 #endif  // GFLAGS

db_stress_tool/db_stress_test_base.cc

@@ -1460,6 +1460,14 @@ void StressTest::PrintEnv() const {
           FLAGS_periodic_compaction_seconds);
   fprintf(stdout, "Compaction TTL            : %" PRIu64 "\n",
           FLAGS_compaction_ttl);
+  fprintf(stdout, "Background Purge          : %d\n",
+          static_cast<int>(FLAGS_avoid_unnecessary_blocking_io));
+  fprintf(stdout, "Write DB ID to manifest   : %d\n",
+          static_cast<int>(FLAGS_write_dbid_to_manifest));
+  fprintf(stdout, "Max Write Batch Group Size: %" PRIu64 "\n",
+          FLAGS_max_write_batch_group_size_bytes);
+  fprintf(stdout, "Use dynamic level         : %d\n",
+          static_cast<int>(FLAGS_level_compaction_dynamic_level_bytes));
 
   fprintf(stdout, "------------------------------------------------\n");
 }
@@ -1553,6 +1561,13 @@ void StressTest::Open() {
     options_.compaction_options_universal.max_size_amplification_percent =
         FLAGS_universal_max_size_amplification_percent;
     options_.atomic_flush = FLAGS_atomic_flush;
+    options_.avoid_unnecessary_blocking_io =
+        FLAGS_avoid_unnecessary_blocking_io;
+    options_.write_dbid_to_manifest = FLAGS_write_dbid_to_manifest;
+    options_.max_write_batch_group_size_bytes =
+        FLAGS_max_write_batch_group_size_bytes;
+    options_.level_compaction_dynamic_level_bytes =
+        FLAGS_level_compaction_dynamic_level_bytes;
   } else {
 #ifdef ROCKSDB_LITE
     fprintf(stderr, "--options_file not supported in lite mode\n");

tools/db_crashtest.py

@@ -32,7 +32,9 @@ default_params = {
     "cache_index_and_filter_blocks": lambda: random.randint(0, 1),
     "cache_size": 1048576,
     "checkpoint_one_in": 1000000,
-    "compression_type": "snappy",
+    "compression_type": lambda: random.choice(
+        ["snappy", "none", "zlib"]),
+    "checksum_type" : lambda: random.choice(["kCRC32c", "kxxHash", "kxxHash64"]),
     "compression_max_dict_bytes": lambda: 16384 * random.randint(0, 1),
     "compression_zstd_max_train_bytes": lambda: 65536 * random.randint(0, 1),
     "clear_column_family_one_in": 0,
@@ -81,7 +83,19 @@ default_params = {
     # Test small max_manifest_file_size in a smaller chance, as most of the
     # time we wnat manifest history to be preserved to help debug
     "max_manifest_file_size" : lambda : random.choice(
-        [t * 16384 if t < 3 else 1024 * 1024 * 1024 for t in range(1,30)])
+        [t * 16384 if t < 3 else 1024 * 1024 * 1024 for t in range(1, 30)]),
+    # Sync mode might make test runs slower so running it in a smaller chance
+    "sync" : lambda : random.choice(
+        [0 if t == 0 else 1 for t in range(1, 20)]),
+    "compaction_readahead_size" : lambda : random.choice(
+        [0, 0, 1024 * 1024]),
+    "db_write_buffer_size" : lambda: random.choice(
+        [0, 0, 0, 1024 * 1024, 8 * 1024 * 1024, 128 * 1024 * 1024]),
+    "avoid_unnecessary_blocking_io" : random.randint(0, 1),
+    "write_dbid_to_manifest" : random.randint(0, 1),
+    "max_write_batch_group_size_bytes" : lambda: random.choice(
+        [16, 64, 1024 * 1024, 16 * 1024 * 1024]),
+    "level_compaction_dynamic_level_bytes" : True,
 }
 
 _TEST_DIR_ENV_VAR = 'TEST_TMPDIR'
@@ -139,6 +153,7 @@ simple_default_params = {
     "target_file_size_multiplier": 1,
     "test_batches_snapshots": 0,
     "write_buffer_size": 32 * 1024 * 1024,
+    "level_compaction_dynamic_level_bytes": False,
 }
 
 blackbox_simple_default_params = {