crash_test to cover simply cases
Summary: crash_test now only runs complicated options, multiple column families, prefix hash, frequently changing options, many compaction threads, etc. These options are good to cover new features but we loss coverage in most common use cases. Furthermore, by running only for multiple column families, we are not able to create LSM trees that are large enough to cover some stress cases. Make half of crash_test runs the simply tests: single column family, default mem table, one compaction thread, no change options. Test Plan: Run crash_test Reviewers: rven, yhchiang, IslamAbdelRahman, kradhakrishnan Reviewed By: kradhakrishnan Subscribers: leveldb, dhruba Differential Revision: https://reviews.facebook.net/D43461
This commit is contained in:
parent
d5c0a6da6c
commit
1205bdbcee
2
Makefile
2
Makefile
@ -542,9 +542,11 @@ ldb_tests: ldb
|
||||
crash_test: whitebox_crash_test blackbox_crash_test
|
||||
|
||||
blackbox_crash_test: db_stress
|
||||
python -u tools/db_crashtest.py -s
|
||||
python -u tools/db_crashtest.py
|
||||
|
||||
whitebox_crash_test: db_stress
|
||||
python -u tools/db_crashtest2.py -s
|
||||
python -u tools/db_crashtest2.py
|
||||
|
||||
asan_check:
|
||||
|
@ -15,11 +15,11 @@ import shutil
|
||||
|
||||
def main(argv):
|
||||
try:
|
||||
opts, args = getopt.getopt(argv, "hd:t:i:o:b:")
|
||||
opts, args = getopt.getopt(argv, "hsd:t:i:o:b:")
|
||||
except getopt.GetoptError:
|
||||
print("db_crashtest.py -d <duration_test> -t <#threads> "
|
||||
"-i <interval for one run> -o <ops_per_thread> "
|
||||
"-b <write_buffer_size>\n")
|
||||
"-b <write_buffer_size> [-s (simple mode)]\n")
|
||||
sys.exit(2)
|
||||
|
||||
# default values, will be overridden by cmdline args
|
||||
@ -29,13 +29,19 @@ def main(argv):
|
||||
# since we will be killing anyway, use large value for ops_per_thread
|
||||
ops_per_thread = 100000000
|
||||
write_buf_size = 4 * 1024 * 1024
|
||||
|
||||
simple_mode = False
|
||||
write_buf_size_set = False
|
||||
for opt, arg in opts:
|
||||
if opt == '-h':
|
||||
print("db_crashtest.py -d <duration_test>"
|
||||
" -t <#threads> -i <interval for one run>"
|
||||
" -o <ops_per_thread> -b <write_buffer_size>\n")
|
||||
" -o <ops_per_thread> -b <write_buffer_size>"
|
||||
" [-s (simple mode)]\n")
|
||||
sys.exit()
|
||||
elif opt == '-s':
|
||||
simple_mode = True
|
||||
if not write_buf_size_set:
|
||||
write_buf_size = 32 * 1024 * 1024
|
||||
elif opt == "-d":
|
||||
duration = int(arg)
|
||||
elif opt == "-t":
|
||||
@ -46,6 +52,7 @@ def main(argv):
|
||||
ops_per_thread = int(arg)
|
||||
elif opt == "-b":
|
||||
write_buf_size = int(arg)
|
||||
write_buf_size_set = True
|
||||
else:
|
||||
print("db_crashtest.py -d <duration_test>"
|
||||
" -t <#threads> -i <interval for one run>"
|
||||
@ -66,45 +73,87 @@ def main(argv):
|
||||
run_had_errors = False
|
||||
killtime = time.time() + interval
|
||||
|
||||
cmd = re.sub('\s+', ' ', """
|
||||
./db_stress
|
||||
--test_batches_snapshots=1
|
||||
--ops_per_thread=%s
|
||||
--threads=%s
|
||||
--write_buffer_size=%s
|
||||
--destroy_db_initially=0
|
||||
--reopen=20
|
||||
--readpercent=45
|
||||
--prefixpercent=5
|
||||
--writepercent=35
|
||||
--delpercent=5
|
||||
--iterpercent=10
|
||||
--db=%s
|
||||
--max_key=100000000
|
||||
--mmap_read=%s
|
||||
--block_size=16384
|
||||
--cache_size=1048576
|
||||
--open_files=500000
|
||||
--verify_checksum=1
|
||||
--sync=0
|
||||
--progress_reports=0
|
||||
--disable_wal=0
|
||||
--disable_data_sync=1
|
||||
--target_file_size_base=2097152
|
||||
--target_file_size_multiplier=2
|
||||
--max_write_buffer_number=3
|
||||
--max_background_compactions=20
|
||||
--max_bytes_for_level_base=10485760
|
||||
--filter_deletes=%s
|
||||
--memtablerep=prefix_hash
|
||||
--prefix_size=7
|
||||
--set_options_one_in=10000
|
||||
""" % (ops_per_thread,
|
||||
threads,
|
||||
write_buf_size,
|
||||
dbname,
|
||||
random.randint(0, 1),
|
||||
random.randint(0, 1)))
|
||||
if simple_mode:
|
||||
cmd = re.sub('\s+', ' ', """
|
||||
./db_stress
|
||||
--column_families=1
|
||||
--test_batches_snapshots=0
|
||||
--ops_per_thread=%s
|
||||
--threads=%s
|
||||
--write_buffer_size=%s
|
||||
--destroy_db_initially=0
|
||||
--reopen=20
|
||||
--readpercent=50
|
||||
--prefixpercent=0
|
||||
--writepercent=35
|
||||
--delpercent=5
|
||||
--iterpercent=10
|
||||
--db=%s
|
||||
--max_key=100000000
|
||||
--mmap_read=%s
|
||||
--block_size=16384
|
||||
--cache_size=1048576
|
||||
--open_files=-1
|
||||
--verify_checksum=1
|
||||
--sync=0
|
||||
--progress_reports=0
|
||||
--disable_wal=0
|
||||
--disable_data_sync=1
|
||||
--target_file_size_base=16777216
|
||||
--target_file_size_multiplier=1
|
||||
--max_write_buffer_number=3
|
||||
--max_background_compactions=1
|
||||
--max_bytes_for_level_base=67108864
|
||||
--filter_deletes=%s
|
||||
--memtablerep=skip_list
|
||||
--prefix_size=0
|
||||
--set_options_one_in=0
|
||||
""" % (ops_per_thread,
|
||||
threads,
|
||||
write_buf_size,
|
||||
dbname,
|
||||
random.randint(0, 1),
|
||||
random.randint(0, 1)))
|
||||
else:
|
||||
cmd = re.sub('\s+', ' ', """
|
||||
./db_stress
|
||||
--test_batches_snapshots=1
|
||||
--ops_per_thread=%s
|
||||
--threads=%s
|
||||
--write_buffer_size=%s
|
||||
--destroy_db_initially=0
|
||||
--reopen=20
|
||||
--readpercent=45
|
||||
--prefixpercent=5
|
||||
--writepercent=35
|
||||
--delpercent=5
|
||||
--iterpercent=10
|
||||
--db=%s
|
||||
--max_key=100000000
|
||||
--mmap_read=%s
|
||||
--block_size=16384
|
||||
--cache_size=1048576
|
||||
--open_files=500000
|
||||
--verify_checksum=1
|
||||
--sync=0
|
||||
--progress_reports=0
|
||||
--disable_wal=0
|
||||
--disable_data_sync=1
|
||||
--target_file_size_base=2097152
|
||||
--target_file_size_multiplier=2
|
||||
--max_write_buffer_number=3
|
||||
--max_background_compactions=20
|
||||
--max_bytes_for_level_base=10485760
|
||||
--filter_deletes=%s
|
||||
--memtablerep=prefix_hash
|
||||
--prefix_size=7
|
||||
--set_options_one_in=10000
|
||||
""" % (ops_per_thread,
|
||||
threads,
|
||||
write_buf_size,
|
||||
dbname,
|
||||
random.randint(0, 1),
|
||||
random.randint(0, 1)))
|
||||
|
||||
child = subprocess.Popen([cmd],
|
||||
stderr=subprocess.PIPE, shell=True)
|
||||
|
@ -15,12 +15,12 @@ import shutil
|
||||
|
||||
def main(argv):
|
||||
try:
|
||||
opts, args = getopt.getopt(argv, "hd:t:k:o:b:")
|
||||
opts, args = getopt.getopt(argv, "hsd:t:k:o:b:")
|
||||
except getopt.GetoptError:
|
||||
print str(getopt.GetoptError)
|
||||
print "db_crashtest2.py -d <duration_test> -t <#threads> " \
|
||||
"-k <kills with prob 1/k> -o <ops_per_thread> "\
|
||||
"-b <write_buffer_size>\n"
|
||||
"-b <write_buffer_size> [-s (simple mode)]\n"
|
||||
sys.exit(2)
|
||||
|
||||
# default values, will be overridden by cmdline args
|
||||
@ -29,13 +29,19 @@ def main(argv):
|
||||
threads = 32
|
||||
ops_per_thread = 200000
|
||||
write_buf_size = 4 * 1024 * 1024
|
||||
simple_mode = False
|
||||
write_buf_size_set = False
|
||||
|
||||
for opt, arg in opts:
|
||||
if opt == '-h':
|
||||
print "db_crashtest2.py -d <duration_test> -t <#threads> " \
|
||||
"-k <kills with prob 1/k> -o <ops_per_thread> " \
|
||||
"-b <write_buffer_size>\n"
|
||||
"-b <write_buffer_size> [-s (simple mode)]\n"
|
||||
sys.exit()
|
||||
elif opt == '-s':
|
||||
simple_mode = True
|
||||
if not write_buf_size_set:
|
||||
write_buf_size = 32 * 1024 * 1024
|
||||
elif opt == "-d":
|
||||
duration = int(arg)
|
||||
elif opt == "-t":
|
||||
@ -46,6 +52,7 @@ def main(argv):
|
||||
ops_per_thread = int(arg)
|
||||
elif opt == "-b":
|
||||
write_buf_size = int(arg)
|
||||
write_buf_size_set = True
|
||||
else:
|
||||
print "unrecognized option " + str(opt) + "\n"
|
||||
print "db_crashtest2.py -d <duration_test> -t <#threads> " \
|
||||
@ -86,45 +93,85 @@ def main(argv):
|
||||
# normal run
|
||||
additional_opts = "--ops_per_thread=" + str(ops_per_thread)
|
||||
|
||||
cmd = re.sub('\s+', ' ', """
|
||||
./db_stress
|
||||
--test_batches_snapshots=%s
|
||||
--threads=%s
|
||||
--write_buffer_size=%s
|
||||
--destroy_db_initially=0
|
||||
--reopen=20
|
||||
--readpercent=45
|
||||
--prefixpercent=5
|
||||
--writepercent=35
|
||||
--delpercent=5
|
||||
--iterpercent=10
|
||||
--db=%s
|
||||
--max_key=100000000
|
||||
--mmap_read=%s
|
||||
--block_size=16384
|
||||
--cache_size=1048576
|
||||
--open_files=500000
|
||||
--verify_checksum=1
|
||||
--sync=0
|
||||
--progress_reports=0
|
||||
--disable_wal=0
|
||||
--disable_data_sync=1
|
||||
--target_file_size_base=2097152
|
||||
--target_file_size_multiplier=2
|
||||
--max_write_buffer_number=3
|
||||
--max_background_compactions=20
|
||||
--max_bytes_for_level_base=10485760
|
||||
--filter_deletes=%s
|
||||
--memtablerep=prefix_hash
|
||||
--prefix_size=7
|
||||
%s
|
||||
""" % (random.randint(0, 1),
|
||||
threads,
|
||||
write_buf_size,
|
||||
dbname,
|
||||
random.randint(0, 1),
|
||||
random.randint(0, 1),
|
||||
additional_opts))
|
||||
if simple_mode:
|
||||
cmd = re.sub('\s+', ' ', """
|
||||
./db_stress
|
||||
--column_families=1
|
||||
--threads=%s
|
||||
--write_buffer_size=%s
|
||||
--destroy_db_initially=0
|
||||
--reopen=20
|
||||
--prefixpercent=0
|
||||
--readpercent=50
|
||||
--writepercent=35
|
||||
--delpercent=5
|
||||
--iterpercent=10
|
||||
--db=%s
|
||||
--max_key=100000000
|
||||
--mmap_read=%s
|
||||
--block_size=16384
|
||||
--cache_size=1048576
|
||||
--open_files=500000
|
||||
--verify_checksum=1
|
||||
--sync=0
|
||||
--progress_reports=0
|
||||
--disable_wal=0
|
||||
--disable_data_sync=1
|
||||
--target_file_size_base=16777216
|
||||
--target_file_size_multiplier=1
|
||||
--max_write_buffer_number=3
|
||||
--max_background_compactions=1
|
||||
--max_bytes_for_level_base=67108864
|
||||
--filter_deletes=%s
|
||||
--memtablerep=skip_list
|
||||
--prefix_size=0
|
||||
%s
|
||||
""" % (threads,
|
||||
write_buf_size,
|
||||
dbname,
|
||||
random.randint(0, 1),
|
||||
random.randint(0, 1),
|
||||
additional_opts))
|
||||
else:
|
||||
cmd = re.sub('\s+', ' ', """
|
||||
./db_stress
|
||||
--test_batches_snapshots=%s
|
||||
--threads=%s
|
||||
--write_buffer_size=%s
|
||||
--destroy_db_initially=0
|
||||
--reopen=20
|
||||
--readpercent=45
|
||||
--prefixpercent=5
|
||||
--writepercent=35
|
||||
--delpercent=5
|
||||
--iterpercent=10
|
||||
--db=%s
|
||||
--max_key=100000000
|
||||
--mmap_read=%s
|
||||
--block_size=16384
|
||||
--cache_size=1048576
|
||||
--open_files=500000
|
||||
--verify_checksum=1
|
||||
--sync=0
|
||||
--progress_reports=0
|
||||
--disable_wal=0
|
||||
--disable_data_sync=1
|
||||
--target_file_size_base=2097152
|
||||
--target_file_size_multiplier=2
|
||||
--max_write_buffer_number=3
|
||||
--max_background_compactions=20
|
||||
--max_bytes_for_level_base=10485760
|
||||
--filter_deletes=%s
|
||||
--memtablerep=prefix_hash
|
||||
--prefix_size=7
|
||||
%s
|
||||
""" % (random.randint(0, 1),
|
||||
threads,
|
||||
write_buf_size,
|
||||
dbname,
|
||||
random.randint(0, 1),
|
||||
random.randint(0, 1),
|
||||
additional_opts))
|
||||
|
||||
print "Running:" + cmd + "\n"
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user