andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Update a few scripts to be python3 compatible (#6525)

Browse Source 
Summary:
There are a few scripts with python3 compatibility issues that were not
detected by automated tool before. Update them now.

Test Plan (devserver):
python2 tools/ldb_test.py
python3 tools/ldb_test.py

python2 tools/write_stress_runner.py --runtime_sec=30
python3 tools/write_stress_runner.py --runtime_sec=30

python2 tools/db_crashtest.py --simple --interval=2 --duration=10 blackbox
python3 tools/db_crashtest.py --simple --interval=2 --duration=10 blackbox

python2 tools/db_crashtest.py --simple --duration=10 --random_kill_odd=1000 --ops_per_thread=1000 whitebox
python3 tools/db_crashtest.py --simple --duration=10 --random_kill_odd=1000 --ops_per_thread=1000 whitebox
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6525

Reviewed By: cheng-chang

Differential Revision: D20627820

Pulled By: riversand963

fbshipit-source-id: 4b25a7bd4d001c7f868be8b640ef876523be6ca3
This commit is contained in:
Yanqin Jin 2020-03-24 20:57:53 -07:00 committed by Facebook GitHub Bot
parent 6fd0ed4993
commit ccf7676455
6 changed files with 90 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
Makefile
View File

@ -8,6 +8,7 @@
BASH_EXISTS := $(shell which bash) BASH_EXISTS := $(shell which bash)
SHELL := $(shell which bash) SHELL := $(shell which bash)
PYTHON?=$(shell which python)
CLEAN_FILES = # deliberately empty, so we can append below. CLEAN_FILES = # deliberately empty, so we can append below.
CFLAGS += ${EXTRA_CFLAGS} CFLAGS += ${EXTRA_CFLAGS}
@ -948,9 +949,9 @@ check: all
fi fi
rm -rf $(TMPD) rm -rf $(TMPD)
ifneq ($(PLATFORM), OS_AIX) ifneq ($(PLATFORM), OS_AIX)
python tools/check_all_python.py $(PYTHON) tools/check_all_python.py
ifeq ($(filter -DROCKSDB_LITE,$(OPT)),) ifeq ($(filter -DROCKSDB_LITE,$(OPT)),)
python tools/ldb_test.py $(PYTHON) tools/ldb_test.py
sh tools/rocksdb_dump_test.sh sh tools/rocksdb_dump_test.sh
endif endif
endif endif
@ -961,7 +962,7 @@ check_some: $(SUBSET)
.PHONY: ldb_tests .PHONY: ldb_tests
ldb_tests: ldb ldb_tests: ldb
python tools/ldb_test.py $(PYTHON) tools/ldb_test.py
crash_test: whitebox_crash_test blackbox_crash_test crash_test: whitebox_crash_test blackbox_crash_test
@ -970,31 +971,31 @@ crash_test_with_atomic_flush: whitebox_crash_test_with_atomic_flush blackbox_cra
crash_test_with_txn: whitebox_crash_test_with_txn blackbox_crash_test_with_txn crash_test_with_txn: whitebox_crash_test_with_txn blackbox_crash_test_with_txn
blackbox_crash_test: db_stress blackbox_crash_test: db_stress
python -u tools/db_crashtest.py --simple blackbox $(CRASH_TEST_EXT_ARGS) $(PYTHON) -u tools/db_crashtest.py --simple blackbox $(CRASH_TEST_EXT_ARGS)
python -u tools/db_crashtest.py blackbox $(CRASH_TEST_EXT_ARGS) $(PYTHON) -u tools/db_crashtest.py blackbox $(CRASH_TEST_EXT_ARGS)
blackbox_crash_test_with_atomic_flush: db_stress blackbox_crash_test_with_atomic_flush: db_stress
python -u tools/db_crashtest.py --cf_consistency blackbox $(CRASH_TEST_EXT_ARGS) $(PYTHON) -u tools/db_crashtest.py --cf_consistency blackbox $(CRASH_TEST_EXT_ARGS)
blackbox_crash_test_with_txn: db_stress blackbox_crash_test_with_txn: db_stress
python -u tools/db_crashtest.py --txn blackbox $(CRASH_TEST_EXT_ARGS) $(PYTHON) -u tools/db_crashtest.py --txn blackbox $(CRASH_TEST_EXT_ARGS)
ifeq ($(CRASH_TEST_KILL_ODD),) ifeq ($(CRASH_TEST_KILL_ODD),)
CRASH_TEST_KILL_ODD=888887 CRASH_TEST_KILL_ODD=888887
endif endif
whitebox_crash_test: db_stress whitebox_crash_test: db_stress
python -u tools/db_crashtest.py --simple whitebox --random_kill_odd \ $(PYTHON) -u tools/db_crashtest.py --simple whitebox --random_kill_odd \
$(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS) $(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS)
python -u tools/db_crashtest.py whitebox --random_kill_odd \ $(PYTHON) -u tools/db_crashtest.py whitebox --random_kill_odd \
$(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS) $(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS)
whitebox_crash_test_with_atomic_flush: db_stress whitebox_crash_test_with_atomic_flush: db_stress
python -u tools/db_crashtest.py --cf_consistency whitebox --random_kill_odd \ $(PYTHON) -u tools/db_crashtest.py --cf_consistency whitebox --random_kill_odd \
$(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS) $(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS)
whitebox_crash_test_with_txn: db_stress whitebox_crash_test_with_txn: db_stress
python -u tools/db_crashtest.py --txn whitebox --random_kill_odd \ $(PYTHON) -u tools/db_crashtest.py --txn whitebox --random_kill_odd \
$(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS) $(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS)
asan_check: asan_check:
@ -2093,7 +2094,7 @@ jtest_run:
jtest: rocksdbjava jtest: rocksdbjava
cd java;$(MAKE) sample;$(MAKE) test; cd java;$(MAKE) sample;$(MAKE) test;
python tools/check_all_python.py # TODO peterd: find a better place for this check in CI targets $(PYTHON) tools/check_all_python.py # TODO peterd: find a better place for this check in CI targets
jdb_bench: jdb_bench:
cd java;$(MAKE) db_bench; cd java;$(MAKE) db_bench;

7
coverage/coverage_test.sh
View File

@ -17,16 +17,19 @@ if [ -d /mnt/gvfs/third-party -a -z "$CXX" ]; then
else else
GCOV=$(which gcov) GCOV=$(which gcov)
fi fi
echo -e "Using $GCOV"
COVERAGE_DIR="$PWD/COVERAGE_REPORT" COVERAGE_DIR="$PWD/COVERAGE_REPORT"
mkdir -p $COVERAGE_DIR mkdir -p $COVERAGE_DIR
# Find all gcno files to generate the coverage report # Find all gcno files to generate the coverage report
PYTHON=${1:-`which python`}
echo -e "Using $PYTHON"
GCNO_FILES=`find $ROOT -name "*.gcno"` GCNO_FILES=`find $ROOT -name "*.gcno"`
$GCOV --preserve-paths --relative-only --no-output $GCNO_FILES 2>/dev/null | $GCOV --preserve-paths --relative-only --no-output $GCNO_FILES 2>/dev/null |
# Parse the raw gcov report to more human readable form. # Parse the raw gcov report to more human readable form.
python $ROOT/coverage/parse_gcov_output.py | $PYTHON $ROOT/coverage/parse_gcov_output.py |
# Write the output to both stdout and report file. # Write the output to both stdout and report file.
tee $COVERAGE_DIR/coverage_report_all.txt && tee $COVERAGE_DIR/coverage_report_all.txt &&
echo -e "Generated coverage report for all files: $COVERAGE_DIR/coverage_report_all.txt\n" echo -e "Generated coverage report for all files: $COVERAGE_DIR/coverage_report_all.txt\n"
@ -41,7 +44,7 @@ RECENT_REPORT=$COVERAGE_DIR/coverage_report_recent.txt
echo -e "Recently updated files: $LATEST_FILES\n" > $RECENT_REPORT echo -e "Recently updated files: $LATEST_FILES\n" > $RECENT_REPORT
$GCOV --preserve-paths --relative-only --no-output $GCNO_FILES 2>/dev/null | $GCOV --preserve-paths --relative-only --no-output $GCNO_FILES 2>/dev/null |
python $ROOT/coverage/parse_gcov_output.py -interested-files $LATEST_FILES | $PYTHON $ROOT/coverage/parse_gcov_output.py -interested-files $LATEST_FILES |
tee -a $RECENT_REPORT && tee -a $RECENT_REPORT &&
echo -e "Generated coverage report for recently updated files: $RECENT_REPORT\n" echo -e "Generated coverage report for recently updated files: $RECENT_REPORT\n"

22
coverage/parse_gcov_output.py
View File

@ -1,10 +1,12 @@
#!/usr/bin/env python2 #!/usr/bin/env python
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
from __future__ import print_function
import optparse
import re import re
import sys import sys
from optparse import OptionParser
# the gcov report follows certain pattern. Each file will have two lines # the gcov report follows certain pattern. Each file will have two lines
# of report, from which we can extract the file name, total lines and coverage # of report, from which we can extract the file name, total lines and coverage
# percentage. # percentage.
@ -48,7 +50,7 @@ def parse_gcov_report(gcov_input):
def get_option_parser(): def get_option_parser():
usage = "Parse the gcov output and generate more human-readable code " +\ usage = "Parse the gcov output and generate more human-readable code " +\
"coverage report." "coverage report."
parser = OptionParser(usage) parser = optparse.OptionParser(usage)
parser.add_option( parser.add_option(
"--interested-files", "-i", "--interested-files", "-i",
@ -73,8 +75,8 @@ def display_file_coverage(per_file_coverage, total_coverage):
header_template = \ header_template = \
"%" + str(max_file_name_length) + "s\t%s\t%s" "%" + str(max_file_name_length) + "s\t%s\t%s"
separator = "-" * (max_file_name_length + 10 + 20) separator = "-" * (max_file_name_length + 10 + 20)
print header_template % ("Filename", "Coverage", "Lines") # noqa: E999 T25377293 Grandfathered in print(header_template % ("Filename", "Coverage", "Lines")) # noqa: E999 T25377293 Grandfathered in
print separator print(separator)
# -- Print body # -- Print body
# template for printing coverage report for each file. # template for printing coverage report for each file.
@ -82,12 +84,12 @@ def display_file_coverage(per_file_coverage, total_coverage):
for fname, coverage_info in per_file_coverage.items(): for fname, coverage_info in per_file_coverage.items():
coverage, lines = coverage_info coverage, lines = coverage_info
print record_template % (fname, coverage, lines) print(record_template % (fname, coverage, lines))
# -- Print footer # -- Print footer
if total_coverage: if total_coverage:
print separator print(separator)
print record_template % ("Total", total_coverage[0], total_coverage[1]) print(record_template % ("Total", total_coverage[0], total_coverage[1]))
def report_coverage(): def report_coverage():
parser = get_option_parser() parser = get_option_parser()
@ -111,7 +113,7 @@ def report_coverage():
total_coverage = None total_coverage = None
if not len(per_file_coverage): if not len(per_file_coverage):
print >> sys.stderr, "Cannot find coverage info for the given files." print("Cannot find coverage info for the given files.", file=sys.stderr)
return return
display_file_coverage(per_file_coverage, total_coverage) display_file_coverage(per_file_coverage, total_coverage)

57
tools/db_crashtest.py
View File

@ -1,5 +1,7 @@
#!/usr/bin/env python2 #!/usr/bin/env python
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
from __future__ import absolute_import, division, print_function, unicode_literals
import os import os
import sys import sys
import time import time
@ -132,7 +134,7 @@ def is_direct_io_supported(dbname):
with tempfile.NamedTemporaryFile(dir=dbname) as f: with tempfile.NamedTemporaryFile(dir=dbname) as f:
try: try:
os.open(f.name, os.O_DIRECT) os.open(f.name, os.O_DIRECT)
except: except BaseException:
return False return False
return True return True
@ -297,8 +299,8 @@ def blackbox_crash_main(args, unknown_args):
killtime = time.time() + cmd_params['interval'] killtime = time.time() + cmd_params['interval']
cmd = gen_cmd(dict( cmd = gen_cmd(dict(
cmd_params.items() + list(cmd_params.items())
{'db': dbname}.items()), unknown_args) + list({'db': dbname}.items())), unknown_args)
child = subprocess.Popen(cmd, stderr=subprocess.PIPE) child = subprocess.Popen(cmd, stderr=subprocess.PIPE)
print("Running db_stress with pid=%d: %s\n\n" print("Running db_stress with pid=%d: %s\n\n"
@ -323,7 +325,7 @@ def blackbox_crash_main(args, unknown_args):
time.sleep(1) # time to stabilize after a kill time.sleep(1) # time to stabilize after a kill
while True: while True:
line = child.stderr.readline().strip() line = child.stderr.readline().strip().decode('utf-8')
if line == '': if line == '':
break break
elif not line.startswith('WARNING'): elif not line.startswith('WARNING'):
@ -348,7 +350,7 @@ def whitebox_crash_main(args, unknown_args):
cur_time = time.time() cur_time = time.time()
exit_time = cur_time + cmd_params['duration'] exit_time = cur_time + cmd_params['duration']
half_time = cur_time + cmd_params['duration'] / 2 half_time = cur_time + cmd_params['duration'] // 2
print("Running whitebox-crash-test with \n" print("Running whitebox-crash-test with \n"
+ "total-duration=" + str(cmd_params['duration']) + "\n") + "total-duration=" + str(cmd_params['duration']) + "\n")
@ -374,9 +376,9 @@ def whitebox_crash_main(args, unknown_args):
}) })
elif kill_mode == 1: elif kill_mode == 1:
if cmd_params.get('disable_wal', 0) == 1: if cmd_params.get('disable_wal', 0) == 1:
my_kill_odd = kill_random_test / 50 + 1 my_kill_odd = kill_random_test // 50 + 1
else: else:
my_kill_odd = kill_random_test / 10 + 1 my_kill_odd = kill_random_test // 10 + 1
additional_opts.update({ additional_opts.update({
"kill_random_test": my_kill_odd, "kill_random_test": my_kill_odd,
"kill_prefix_blacklist": "WritableFileWriter::Append," "kill_prefix_blacklist": "WritableFileWriter::Append,"
@ -386,7 +388,7 @@ def whitebox_crash_main(args, unknown_args):
# TODO: May need to adjust random odds if kill_random_test # TODO: May need to adjust random odds if kill_random_test
# is too small. # is too small.
additional_opts.update({ additional_opts.update({
"kill_random_test": (kill_random_test / 5000 + 1), "kill_random_test": (kill_random_test // 5000 + 1),
"kill_prefix_blacklist": "WritableFileWriter::Append," "kill_prefix_blacklist": "WritableFileWriter::Append,"
"WritableFileWriter::WriteBuffered," "WritableFileWriter::WriteBuffered,"
"PosixMmapFile::Allocate,WritableFileWriter::Flush", "PosixMmapFile::Allocate,WritableFileWriter::Flush",
@ -406,7 +408,7 @@ def whitebox_crash_main(args, unknown_args):
# style is quite a bit slower on reads with lot of files # style is quite a bit slower on reads with lot of files
additional_opts = { additional_opts = {
"kill_random_test": None, "kill_random_test": None,
"ops_per_thread": cmd_params['ops_per_thread'] / 5, "ops_per_thread": cmd_params['ops_per_thread'] // 5,
"compaction_style": 2, "compaction_style": 2,
} }
else: else:
@ -416,19 +418,24 @@ def whitebox_crash_main(args, unknown_args):
"ops_per_thread": cmd_params['ops_per_thread'], "ops_per_thread": cmd_params['ops_per_thread'],
} }
cmd = gen_cmd(dict(cmd_params.items() + additional_opts.items() cmd = gen_cmd(dict(list(cmd_params.items())
+ {'db': dbname}.items()), unknown_args) + list(additional_opts.items())
+ list({'db': dbname}.items())), unknown_args)
print "Running:" + ' '.join(cmd) + "\n" # noqa: E999 T25377293 Grandfathered in print("Running:" + ' '.join(cmd) + "\n") # noqa: E999 T25377293 Grandfathered in
popen = subprocess.Popen(cmd, stdout=subprocess.PIPE, popen = subprocess.Popen(cmd, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT) stderr=subprocess.STDOUT)
stdoutdata, stderrdata = popen.communicate() stdoutdata, stderrdata = popen.communicate()
if stdoutdata:
stdoutdata = stdoutdata.decode('utf-8')
if stderrdata:
stderrdata = stderrdata.decode('utf-8')
retncode = popen.returncode retncode = popen.returncode
msg = ("check_mode={0}, kill option={1}, exitcode={2}\n".format( msg = ("check_mode={0}, kill option={1}, exitcode={2}\n".format(
check_mode, additional_opts['kill_random_test'], retncode)) check_mode, additional_opts['kill_random_test'], retncode))
print msg print(msg)
print stdoutdata print(stdoutdata)
expected = False expected = False
if additional_opts['kill_random_test'] is None and (retncode == 0): if additional_opts['kill_random_test'] is None and (retncode == 0):
@ -440,19 +447,19 @@ def whitebox_crash_main(args, unknown_args):
expected = True expected = True
if not expected: if not expected:
print "TEST FAILED. See kill option and exit code above!!!\n" print("TEST FAILED. See kill option and exit code above!!!\n")
sys.exit(1) sys.exit(1)
stdoutdata = stdoutdata.lower() stdoutdata = stdoutdata.lower()
errorcount = (stdoutdata.count('error') - errorcount = (stdoutdata.count('error') -
stdoutdata.count('got errors 0 times')) stdoutdata.count('got errors 0 times'))
print "#times error occurred in output is " + str(errorcount) + "\n" print("#times error occurred in output is " + str(errorcount) + "\n")
if (errorcount > 0): if (errorcount > 0):
print "TEST FAILED. Output has 'error'!!!\n" print("TEST FAILED. Output has 'error'!!!\n")
sys.exit(2) sys.exit(2)
if (stdoutdata.find('fail') >= 0): if (stdoutdata.find('fail') >= 0):
print "TEST FAILED. Output has 'fail'!!!\n" print("TEST FAILED. Output has 'fail'!!!\n")
sys.exit(2) sys.exit(2)
# First half of the duration, keep doing kill test. For the next half, # First half of the duration, keep doing kill test. For the next half,
@ -476,12 +483,12 @@ def main():
parser.add_argument("--cf_consistency", action='store_true') parser.add_argument("--cf_consistency", action='store_true')
parser.add_argument("--txn", action='store_true') parser.add_argument("--txn", action='store_true')
all_params = dict(default_params.items() all_params = dict(list(default_params.items())
+ blackbox_default_params.items() + list(blackbox_default_params.items())
+ whitebox_default_params.items() + list(whitebox_default_params.items())
+ simple_default_params.items() + list(simple_default_params.items())
+ blackbox_simple_default_params.items() + list(blackbox_simple_default_params.items())
+ whitebox_simple_default_params.items()) + list(whitebox_simple_default_params.items()))
for k, v in all_params.items(): for k, v in all_params.items():
parser.add_argument("--" + k, type=type(v() if callable(v) else v)) parser.add_argument("--" + k, type=type(v() if callable(v) else v))

47
tools/ldb_test.py
View File

@ -1,5 +1,7 @@
#!/usr/bin/env python2 #!/usr/bin/env python
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
from __future__ import absolute_import, division, print_function, unicode_literals
import os import os
import glob import glob
import os.path import os.path
@ -27,7 +29,7 @@ def my_check_output(*popenargs, **kwargs):
cmd = popenargs[0] cmd = popenargs[0]
raise Exception("Exit code is not 0. It is %d. Command: %s" % raise Exception("Exit code is not 0. It is %d. Command: %s" %
(retcode, cmd)) (retcode, cmd))
return output return output.decode('utf-8')
def run_err_null(cmd): def run_err_null(cmd):
return os.system(cmd + " 2>/dev/null ") return os.system(cmd + " 2>/dev/null ")
@ -52,7 +54,6 @@ class LDBTestCase(unittest.TestCase):
""" """
All command-line params must be specified. All command-line params must be specified.
Allows full flexibility in testing; for example: missing db param. Allows full flexibility in testing; for example: missing db param.
""" """
output = my_check_output("./ldb %s |grep -v \"Created bg thread\"" % output = my_check_output("./ldb %s |grep -v \"Created bg thread\"" %
params, shell=True) params, shell=True)
@ -72,7 +73,6 @@ class LDBTestCase(unittest.TestCase):
""" """
All command-line params must be specified. All command-line params must be specified.
Allows full flexibility in testing; for example: missing db param. Allows full flexibility in testing; for example: missing db param.
""" """
try: try:
@ -87,7 +87,6 @@ class LDBTestCase(unittest.TestCase):
def assertRunOK(self, params, expectedOutput, unexpected=False): def assertRunOK(self, params, expectedOutput, unexpected=False):
""" """
Uses the default test db. Uses the default test db.
""" """
self.assertRunOKFull("%s %s" % (self.dbParam(self.DB_NAME), params), self.assertRunOKFull("%s %s" % (self.dbParam(self.DB_NAME), params),
expectedOutput, unexpected) expectedOutput, unexpected)
@ -99,7 +98,7 @@ class LDBTestCase(unittest.TestCase):
self.assertRunFAILFull("%s %s" % (self.dbParam(self.DB_NAME), params)) self.assertRunFAILFull("%s %s" % (self.dbParam(self.DB_NAME), params))
def testSimpleStringPutGet(self): def testSimpleStringPutGet(self):
print "Running testSimpleStringPutGet..." print("Running testSimpleStringPutGet...")
self.assertRunFAIL("put x1 y1") self.assertRunFAIL("put x1 y1")
self.assertRunOK("put --create_if_missing x1 y1", "OK") self.assertRunOK("put --create_if_missing x1 y1", "OK")
self.assertRunOK("get x1", "y1") self.assertRunOK("get x1", "y1")
@ -157,7 +156,7 @@ class LDBTestCase(unittest.TestCase):
% (inputSst, params)) % (inputSst, params))
def testStringBatchPut(self): def testStringBatchPut(self):
print "Running testStringBatchPut..." print("Running testStringBatchPut...")
self.assertRunOK("batchput x1 y1 --create_if_missing", "OK") self.assertRunOK("batchput x1 y1 --create_if_missing", "OK")
self.assertRunOK("scan", "x1 : y1") self.assertRunOK("scan", "x1 : y1")
self.assertRunOK("batchput x2 y2 x3 y3 \"x4 abc\" \"y4 xyz\"", "OK") self.assertRunOK("batchput x2 y2 x3 y3 \"x4 abc\" \"y4 xyz\"", "OK")
@ -167,7 +166,7 @@ class LDBTestCase(unittest.TestCase):
self.assertRunFAIL("batchput k1 v1 k2") self.assertRunFAIL("batchput k1 v1 k2")
def testCountDelimDump(self): def testCountDelimDump(self):
print "Running testCountDelimDump..." print("Running testCountDelimDump...")
self.assertRunOK("batchput x.1 x1 --create_if_missing", "OK") self.assertRunOK("batchput x.1 x1 --create_if_missing", "OK")
self.assertRunOK("batchput y.abc abc y.2 2 z.13c pqr", "OK") self.assertRunOK("batchput y.abc abc y.2 2 z.13c pqr", "OK")
self.assertRunOK("dump --count_delim", "x => count:1\tsize:5\ny => count:2\tsize:12\nz => count:1\tsize:8") self.assertRunOK("dump --count_delim", "x => count:1\tsize:5\ny => count:2\tsize:12\nz => count:1\tsize:8")
@ -176,7 +175,7 @@ class LDBTestCase(unittest.TestCase):
self.assertRunOK("dump --count_delim=\",\"", "x => count:2\tsize:14\nx.1 => count:1\tsize:5\ny.2 => count:1\tsize:4\ny.abc => count:1\tsize:8\nz.13c => count:1\tsize:8") self.assertRunOK("dump --count_delim=\",\"", "x => count:2\tsize:14\nx.1 => count:1\tsize:5\ny.2 => count:1\tsize:4\ny.abc => count:1\tsize:8\nz.13c => count:1\tsize:8")
def testCountDelimIDump(self): def testCountDelimIDump(self):
print "Running testCountDelimIDump..." print("Running testCountDelimIDump...")
self.assertRunOK("batchput x.1 x1 --create_if_missing", "OK") self.assertRunOK("batchput x.1 x1 --create_if_missing", "OK")
self.assertRunOK("batchput y.abc abc y.2 2 z.13c pqr", "OK") self.assertRunOK("batchput y.abc abc y.2 2 z.13c pqr", "OK")
self.assertRunOK("idump --count_delim", "x => count:1\tsize:5\ny => count:2\tsize:12\nz => count:1\tsize:8") self.assertRunOK("idump --count_delim", "x => count:1\tsize:5\ny => count:2\tsize:12\nz => count:1\tsize:8")
@ -185,7 +184,7 @@ class LDBTestCase(unittest.TestCase):
self.assertRunOK("idump --count_delim=\",\"", "x => count:2\tsize:14\nx.1 => count:1\tsize:5\ny.2 => count:1\tsize:4\ny.abc => count:1\tsize:8\nz.13c => count:1\tsize:8") self.assertRunOK("idump --count_delim=\",\"", "x => count:2\tsize:14\nx.1 => count:1\tsize:5\ny.2 => count:1\tsize:4\ny.abc => count:1\tsize:8\nz.13c => count:1\tsize:8")
def testInvalidCmdLines(self): def testInvalidCmdLines(self):
print "Running testInvalidCmdLines..." print("Running testInvalidCmdLines...")
# db not specified # db not specified
self.assertRunFAILFull("put 0x6133 0x6233 --hex --create_if_missing") self.assertRunFAILFull("put 0x6133 0x6233 --hex --create_if_missing")
# No param called he # No param called he
@ -195,7 +194,7 @@ class LDBTestCase(unittest.TestCase):
# hex has invalid boolean value # hex has invalid boolean value
def testHexPutGet(self): def testHexPutGet(self):
print "Running testHexPutGet..." print("Running testHexPutGet...")
self.assertRunOK("put a1 b1 --create_if_missing", "OK") self.assertRunOK("put a1 b1 --create_if_missing", "OK")
self.assertRunOK("scan", "a1 : b1") self.assertRunOK("scan", "a1 : b1")
self.assertRunOK("scan --hex", "0x6131 : 0x6231") self.assertRunOK("scan --hex", "0x6131 : 0x6231")
@ -225,7 +224,7 @@ class LDBTestCase(unittest.TestCase):
self.assertRunOK("checkconsistency", "OK") self.assertRunOK("checkconsistency", "OK")
def testTtlPutGet(self): def testTtlPutGet(self):
print "Running testTtlPutGet..." print("Running testTtlPutGet...")
self.assertRunOK("put a1 b1 --ttl --create_if_missing", "OK") self.assertRunOK("put a1 b1 --ttl --create_if_missing", "OK")
self.assertRunOK("scan --hex", "0x6131 : 0x6231", True) self.assertRunOK("scan --hex", "0x6131 : 0x6231", True)
self.assertRunOK("dump --ttl ", "a1 ==> b1", True) self.assertRunOK("dump --ttl ", "a1 ==> b1", True)
@ -240,7 +239,7 @@ class LDBTestCase(unittest.TestCase):
self.assertRunOK("checkconsistency", "OK") self.assertRunOK("checkconsistency", "OK")
def testInvalidCmdLines(self): # noqa: F811 T25377293 Grandfathered in def testInvalidCmdLines(self): # noqa: F811 T25377293 Grandfathered in
print "Running testInvalidCmdLines..." print("Running testInvalidCmdLines...")
# db not specified # db not specified
self.assertRunFAILFull("put 0x6133 0x6233 --hex --create_if_missing") self.assertRunFAILFull("put 0x6133 0x6233 --hex --create_if_missing")
# No param called he # No param called he
@ -251,7 +250,7 @@ class LDBTestCase(unittest.TestCase):
self.assertRunFAIL("put 0x6133 0x6233 --hex=Boo --create_if_missing") self.assertRunFAIL("put 0x6133 0x6233 --hex=Boo --create_if_missing")
def testDumpLoad(self): def testDumpLoad(self):
print "Running testDumpLoad..." print("Running testDumpLoad...")
self.assertRunOK("batchput --create_if_missing x1 y1 x2 y2 x3 y3 x4 y4", self.assertRunOK("batchput --create_if_missing x1 y1 x2 y2 x3 y3 x4 y4",
"OK") "OK")
self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4") self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4")
@ -342,7 +341,7 @@ class LDBTestCase(unittest.TestCase):
"--db=%s --create_if_missing" % origDbPath, dumpFilePath)) "--db=%s --create_if_missing" % origDbPath, dumpFilePath))
def testIDumpBasics(self): def testIDumpBasics(self):
print "Running testIDumpBasics..." print("Running testIDumpBasics...")
self.assertRunOK("put a val --create_if_missing", "OK") self.assertRunOK("put a val --create_if_missing", "OK")
self.assertRunOK("put b val", "OK") self.assertRunOK("put b val", "OK")
self.assertRunOK( self.assertRunOK(
@ -354,7 +353,7 @@ class LDBTestCase(unittest.TestCase):
"'a' seq:1, type:1 => val\nInternal keys in range: 1") "'a' seq:1, type:1 => val\nInternal keys in range: 1")
def testMiscAdminTask(self): def testMiscAdminTask(self):
print "Running testMiscAdminTask..." print("Running testMiscAdminTask...")
# These tests need to be improved; for example with asserts about # These tests need to be improved; for example with asserts about
# whether compaction or level reduction actually took place. # whether compaction or level reduction actually took place.
self.assertRunOK("batchput --create_if_missing x1 y1 x2 y2 x3 y3 x4 y4", self.assertRunOK("batchput --create_if_missing x1 y1 x2 y2 x3 y3 x4 y4",
@ -390,7 +389,7 @@ class LDBTestCase(unittest.TestCase):
self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4") self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4")
def testCheckConsistency(self): def testCheckConsistency(self):
print "Running testCheckConsistency..." print("Running testCheckConsistency...")
dbPath = os.path.join(self.TMP_DIR, self.DB_NAME) dbPath = os.path.join(self.TMP_DIR, self.DB_NAME)
self.assertRunOK("put x1 y1 --create_if_missing", "OK") self.assertRunOK("put x1 y1 --create_if_missing", "OK")
@ -414,7 +413,7 @@ class LDBTestCase(unittest.TestCase):
params, dumpFile)) params, dumpFile))
def testDumpLiveFiles(self): def testDumpLiveFiles(self):
print "Running testDumpLiveFiles..." print("Running testDumpLiveFiles...")
dbPath = os.path.join(self.TMP_DIR, self.DB_NAME) dbPath = os.path.join(self.TMP_DIR, self.DB_NAME)
self.assertRunOK("put x1 y1 --create_if_missing", "OK") self.assertRunOK("put x1 y1 --create_if_missing", "OK")
@ -439,7 +438,7 @@ class LDBTestCase(unittest.TestCase):
return 0 == run_err_null("cp " + src + " " + dest) return 0 == run_err_null("cp " + src + " " + dest)
def testManifestDump(self): def testManifestDump(self):
print "Running testManifestDump..." print("Running testManifestDump...")
dbPath = os.path.join(self.TMP_DIR, self.DB_NAME) dbPath = os.path.join(self.TMP_DIR, self.DB_NAME)
self.assertRunOK("put 1 1 --create_if_missing", "OK") self.assertRunOK("put 1 1 --create_if_missing", "OK")
self.assertRunOK("put 2 2", "OK") self.assertRunOK("put 2 2", "OK")
@ -475,7 +474,7 @@ class LDBTestCase(unittest.TestCase):
isPattern=True) isPattern=True)
def testSSTDump(self): def testSSTDump(self):
print "Running testSSTDump..." print("Running testSSTDump...")
dbPath = os.path.join(self.TMP_DIR, self.DB_NAME) dbPath = os.path.join(self.TMP_DIR, self.DB_NAME)
self.assertRunOK("put sst1 sst1_val --create_if_missing", "OK") self.assertRunOK("put sst1 sst1_val --create_if_missing", "OK")
@ -495,7 +494,7 @@ class LDBTestCase(unittest.TestCase):
isPattern=True) isPattern=True)
def testWALDump(self): def testWALDump(self):
print "Running testWALDump..." print("Running testWALDump...")
dbPath = os.path.join(self.TMP_DIR, self.DB_NAME) dbPath = os.path.join(self.TMP_DIR, self.DB_NAME)
self.assertRunOK("put wal1 wal1_val --create_if_missing", "OK") self.assertRunOK("put wal1 wal1_val --create_if_missing", "OK")
@ -515,7 +514,7 @@ class LDBTestCase(unittest.TestCase):
isPattern=True) isPattern=True)
def testListColumnFamilies(self): def testListColumnFamilies(self):
print "Running testListColumnFamilies..." print("Running testListColumnFamilies...")
self.assertRunOK("put x1 y1 --create_if_missing", "OK") self.assertRunOK("put x1 y1 --create_if_missing", "OK")
cmd = "list_column_families | grep -v \"Column families\"" cmd = "list_column_families | grep -v \"Column families\""
# Test on valid dbPath. # Test on valid dbPath.
@ -524,7 +523,7 @@ class LDBTestCase(unittest.TestCase):
self.assertRunFAIL(cmd) self.assertRunFAIL(cmd)
def testColumnFamilies(self): def testColumnFamilies(self):
print "Running testColumnFamilies..." print("Running testColumnFamilies...")
dbPath = os.path.join(self.TMP_DIR, self.DB_NAME) # noqa: F841 T25377293 Grandfathered in dbPath = os.path.join(self.TMP_DIR, self.DB_NAME) # noqa: F841 T25377293 Grandfathered in
self.assertRunOK("put cf1_1 1 --create_if_missing", "OK") self.assertRunOK("put cf1_1 1 --create_if_missing", "OK")
self.assertRunOK("put cf1_2 2 --create_if_missing", "OK") self.assertRunOK("put cf1_2 2 --create_if_missing", "OK")
@ -559,7 +558,7 @@ class LDBTestCase(unittest.TestCase):
self.assertRunFAIL("drop_column_family four") self.assertRunFAIL("drop_column_family four")
def testIngestExternalSst(self): def testIngestExternalSst(self):
print "Running testIngestExternalSst..." print("Running testIngestExternalSst...")
# Dump, load, write external sst and ingest it in another db # Dump, load, write external sst and ingest it in another db
dbPath = os.path.join(self.TMP_DIR, "db1") dbPath = os.path.join(self.TMP_DIR, "db1")

8
tools/write_stress_runner.py
View File

@ -1,5 +1,7 @@
#!/usr/bin/env python2 #!/usr/bin/env python
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
from __future__ import absolute_import, division, print_function, unicode_literals
import subprocess import subprocess
import argparse import argparse
import random import random
@ -10,7 +12,7 @@ import sys
def generate_runtimes(total_runtime): def generate_runtimes(total_runtime):
# combination of short runtimes and long runtimes, with heavier # combination of short runtimes and long runtimes, with heavier
# weight on short runtimes # weight on short runtimes
possible_runtimes_sec = range(1, 10) + range(1, 20) + [100, 1000] possible_runtimes_sec = list(range(1, 10)) + list(range(1, 20)) + [100, 1000]
runtimes = [] runtimes = []
while total_runtime > 0: while total_runtime > 0:
chosen = random.choice(possible_runtimes_sec) chosen = random.choice(possible_runtimes_sec)
@ -22,7 +24,7 @@ def generate_runtimes(total_runtime):
def main(args): def main(args):
runtimes = generate_runtimes(int(args.runtime_sec)) runtimes = generate_runtimes(int(args.runtime_sec))
print "Going to execute write stress for " + str(runtimes) # noqa: E999 T25377293 Grandfathered in print("Going to execute write stress for " + str(runtimes)) # noqa: E999 T25377293 Grandfathered in
first_time = True first_time = True
for runtime in runtimes: for runtime in runtimes: