diff --git a/build_tools/rocksdb-lego-determinator b/build_tools/rocksdb-lego-determinator index ef8623656..5faa14070 100755 --- a/build_tools/rocksdb-lego-determinator +++ b/build_tools/rocksdb-lego-determinator @@ -88,6 +88,8 @@ CLANG="USE_CLANG=1" LITE="OPT=\"-DROCKSDB_LITE -g\"" TSAN="COMPILE_WITH_TSAN=1" UBSAN="COMPILE_WITH_UBSAN=1" +TSAN_CRASH="CRASH_TEST_EXT_ARGS='--compression_type=zstd --log2_keys_per_lock=22'" +NON_TSAN_CRASH="CRASH_TEST_EXT_ARGS=--compression_type=zstd" DISABLE_JEMALLOC="DISABLE_JEMALLOC=1" HTTP_PROXY="https_proxy=http://fwdproxy.29.prn1:8080 http_proxy=http://fwdproxy.29.prn1:8080 ftp_proxy=http://fwdproxy.29.prn1:8080" SETUP_JAVA_ENV="export $HTTP_PROXY; export JAVA_HOME=/usr/local/jdk-8u60-64/; export PATH=\$JAVA_HOME/bin:\$PATH" @@ -380,14 +382,14 @@ STRESS_CRASH_TEST_COMMANDS="[ $CLEANUP_ENV, { 'name':'Build and run RocksDB debug stress tests', - 'shell':'$SHM $DEBUG make J=1 db_stress || $CONTRUN_NAME=db_stress $TASK_CREATION_TOOL', + 'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 db_stress || $CONTRUN_NAME=db_stress $TASK_CREATION_TOOL', 'user':'root', $PARSER }, { 'name':'Build and run RocksDB debug crash tests', 'timeout': 86400, - 'shell':'$SHM $DEBUG make J=1 crash_test || $CONTRUN_NAME=crash_test $TASK_CREATION_TOOL', + 'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 crash_test || $CONTRUN_NAME=crash_test $TASK_CREATION_TOOL', 'user':'root', $PARSER } @@ -452,7 +454,7 @@ ASAN_CRASH_TEST_COMMANDS="[ { 'name':'Build and run RocksDB debug asan_crash_test', 'timeout': 86400, - 'shell':'$SHM $DEBUG make J=1 asan_crash_test || $CONTRUN_NAME=asan_crash_test $TASK_CREATION_TOOL', + 'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 asan_crash_test || $CONTRUN_NAME=asan_crash_test $TASK_CREATION_TOOL', 'user':'root', $PARSER }, @@ -494,7 +496,7 @@ UBSAN_CRASH_TEST_COMMANDS="[ { 'name':'Build and run RocksDB debug ubsan_crash_test', 'timeout': 86400, - 'shell':'$SHM $DEBUG make J=1 ubsan_crash_test || $CONTRUN_NAME=ubsan_crash_test $TASK_CREATION_TOOL', + 'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 ubsan_crash_test || $CONTRUN_NAME=ubsan_crash_test $TASK_CREATION_TOOL', 'user':'root', $PARSER }, @@ -560,7 +562,7 @@ TSAN_CRASH_TEST_COMMANDS="[ { 'name':'Compile and run', 'timeout': 86400, - 'shell':'set -o pipefail && $SHM $DEBUG $TSAN CRASH_TEST_KILL_ODD=1887 CRASH_TEST_EXT_ARGS=--log2_keys_per_lock=22 make J=1 crash_test || $CONTRUN_NAME=tsan_crash_test $TASK_CREATION_TOOL', + 'shell':'set -o pipefail && $SHM $DEBUG $TSAN $TSAN_CRASH CRASH_TEST_KILL_ODD=1887 make J=1 crash_test || $CONTRUN_NAME=tsan_crash_test $TASK_CREATION_TOOL', 'user':'root', $PARSER }, diff --git a/tools/db_crashtest.py b/tools/db_crashtest.py index 07f87df5f..6d1d7051a 100644 --- a/tools/db_crashtest.py +++ b/tools/db_crashtest.py @@ -23,6 +23,8 @@ default_params = { "block_size": 16384, "cache_size": 1048576, "checkpoint_one_in": 1000000, + "compression_max_dict_bytes": lambda: 16384 * random.randint(0, 1), + "compression_zstd_max_train_bytes": lambda: 65536 * random.randint(0, 1), "clear_column_family_one_in": 0, "compact_files_one_in": 1000000, "compact_range_one_in": 1000000, @@ -122,6 +124,9 @@ whitebox_simple_default_params = {} def finalize_and_sanitize(src_params): dest_params = dict([(k, v() if callable(v) else v) for (k, v) in src_params.items()]) + if dest_params.get("compression_type") != "zstd" or \ + dest_params.get("compression_max_dict_bytes") == 0: + dest_params["compression_zstd_max_train_bytes"] = 0 if dest_params.get("allow_concurrent_memtable_write", 1) == 1: dest_params["memtablerep"] = "skip_list" if dest_params["mmap_read"] == 1 or not is_direct_io_supported( diff --git a/tools/db_stress.cc b/tools/db_stress.cc index 1ea52cd24..d93ad5dbe 100644 --- a/tools/db_stress.cc +++ b/tools/db_stress.cc @@ -540,6 +540,14 @@ DEFINE_string(compression_type, "snappy", static enum rocksdb::CompressionType FLAGS_compression_type_e = rocksdb::kSnappyCompression; +DEFINE_int32(compression_max_dict_bytes, 0, + "Maximum size of dictionary used to prime the compression " + "library."); + +DEFINE_int32(compression_zstd_max_train_bytes, 0, + "Maximum size of training data passed to zstd's dictionary " + "trainer."); + DEFINE_string(checksum_type, "kCRC32c", "Algorithm to use to checksum blocks"); static enum rocksdb::ChecksumType FLAGS_checksum_type_e = rocksdb::kCRC32c; @@ -2247,6 +2255,10 @@ class StressTest { options_.level0_file_num_compaction_trigger = FLAGS_level0_file_num_compaction_trigger; options_.compression = FLAGS_compression_type_e; + options_.compression_opts.max_dict_bytes = + FLAGS_compression_max_dict_bytes; + options_.compression_opts.zstd_max_train_bytes = + FLAGS_compression_zstd_max_train_bytes; options_.create_if_missing = true; options_.max_manifest_file_size = FLAGS_max_manifest_file_size; options_.inplace_update_support = FLAGS_in_place_update;