andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

tools/check_format_compatible.sh to cover option file loading too

Browse Source 
Summary:
tools/check_format_compatible.sh will check a newer version of RocksDB can open option files generated by older version releases. In order to achieve that, a new parameter "--try_load_options" is added to ldb. With this parameter set, if option file exists, we load the option file and use it to open the DB. With this opiton set, we can validate option loading logic.
Closes https://github.com/facebook/rocksdb/pull/2178

Differential Revision: D4914989

Pulled By: siying

fbshipit-source-id: db114f7724fcb41e5e9483116d84d7c4b8389ca4
This commit is contained in:
Siying Dong 2017-04-20 10:16:13 -07:00 committed by Facebook Github Bot
parent 8f61967881
commit 97005dbd5d
8 changed files with 60 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
HISTORY.md
View File

@ -2,6 +2,7 @@
## Unreleased
### New Features
* DB::ResetStats() to reset internal stats.
* ldb add option --try_load_options, which will open DB with its own option file.
## 5.4.0 (04/11/2017)
### Public API Change

6
include/rocksdb/utilities/ldb_cmd.h
View File

@ -39,6 +39,7 @@ class LDBCommand {
static const std::string ARG_TTL_START;
static const std::string ARG_TTL_END;
static const std::string ARG_TIMESTAMP;
static const std::string ARG_TRY_LOAD_OPTIONS;
static const std::string ARG_FROM;
static const std::string ARG_TO;
static const std::string ARG_MAX_KEYS;
@ -143,6 +144,11 @@ class LDBCommand {
// If true, the kvs are output with their insert/modify timestamp in a ttl db
bool timestamp_;
// If true, try to construct options from DB's option files.
bool try_load_options_;
bool create_if_missing_;
/**
* Map of options passed on the command-line.
*/

6
tools/check_format_compatible.sh
View File

@ -58,7 +58,7 @@ generate_db()
compare_db()
{
set +e
$script_copy_dir/verify_random_db.sh $1 $2 $3
$script_copy_dir/verify_random_db.sh $1 $2 $3 $4
if [ $? -ne 0 ]; then
echo ==== Read different content from $1 and $2 or error happened. ====
exit 1
@ -95,7 +95,7 @@ generate_db $input_data_path $compare_base_db_dir
for checkout_obj in "${checkout_objs[@]}"
do
echo == Opening DB from "$checkout_obj" using debug build of $checkout_flag ...
compare_db $test_dir/$checkout_obj $compare_base_db_dir db_dump.txt
compare_db $test_dir/$checkout_obj $compare_base_db_dir db_dump.txt 1
done
for checkout_obj in "${forward_compatible_checkout_objs[@]}"
@ -104,7 +104,7 @@ do
git checkout origin/$checkout_obj
make clean
make ldb -j32
compare_db $test_dir/$checkout_obj $compare_base_db_dir forward_${checkout_obj}_dump.txt
compare_db $test_dir/$checkout_obj $compare_base_db_dir forward_${checkout_obj}_dump.txt 0
done
echo ==== Compatibility Test PASSED ====

42
tools/ldb_cmd.cc
View File

@ -22,6 +22,7 @@
#include "rocksdb/utilities/backupable_db.h"
#include "rocksdb/utilities/checkpoint.h"
#include "rocksdb/utilities/object_registry.h"
#include "rocksdb/utilities/options_util.h"
#include "rocksdb/write_batch.h"
#include "rocksdb/write_buffer_manager.h"
#include "table/scoped_arena_iterator.h"
@ -55,6 +56,7 @@ const std::string LDBCommand::ARG_TTL = "ttl";
const std::string LDBCommand::ARG_TTL_START = "start_time";
const std::string LDBCommand::ARG_TTL_END = "end_time";
const std::string LDBCommand::ARG_TIMESTAMP = "timestamp";
const std::string LDBCommand::ARG_TRY_LOAD_OPTIONS = "try_load_options";
const std::string LDBCommand::ARG_FROM = "from";
const std::string LDBCommand::ARG_TO = "to";
const std::string LDBCommand::ARG_MAX_KEYS = "max_keys";
@ -248,6 +250,12 @@ void LDBCommand::Run() {
if (db_ == nullptr && !NoDBOpen()) {
OpenDB();
if (exec_state_.IsFailed() && try_load_options_) {
// We don't always return if there is a failure because a WAL file or
// manifest file can be given to "dump" command so we should continue.
// --try_load_options is not valid in those cases.
return;
}
}
// We'll intentionally proceed even if the DB can't be opened because users
@ -272,6 +280,8 @@ LDBCommand::LDBCommand(const std::map<std::string, std::string>& options,
is_value_hex_(false),
is_db_ttl_(false),
timestamp_(false),
try_load_options_(false),
create_if_missing_(false),
option_map_(options),
flags_(flags),
valid_cmd_line_options_(valid_cmd_line_options) {
@ -291,10 +301,28 @@ LDBCommand::LDBCommand(const std::map<std::string, std::string>& options,
is_value_hex_ = IsValueHex(options, flags);
is_db_ttl_ = IsFlagPresent(flags, ARG_TTL);
timestamp_ = IsFlagPresent(flags, ARG_TIMESTAMP);
try_load_options_ = IsFlagPresent(flags, ARG_TRY_LOAD_OPTIONS);
}
void LDBCommand::OpenDB() {
Options opt = PrepareOptionsForOpenDB();
Options opt;
bool opt_set = false;
if (!create_if_missing_ && try_load_options_) {
Status s =
LoadLatestOptions(db_path_, Env::Default(), &opt, &column_families_);
if (s.ok()) {
opt_set = true;
} else if (!s.IsNotFound()) {
// Option file exists but load option file error.
std::string msg = s.ToString();
exec_state_ = LDBCommandExecuteResult::Failed(msg);
db_ = nullptr;
return;
}
}
if (!opt_set) {
opt = PrepareOptionsForOpenDB();
}
if (!exec_state_.IsNotStarted()) {
return;
}
@ -314,7 +342,7 @@ void LDBCommand::OpenDB() {
}
db_ = db_ttl_;
} else {
if (column_families_.empty()) {
if (!opt_set && column_families_.empty()) {
// Try to figure out column family lists
std::vector<std::string> cf_list;
st = DB::ListColumnFamilies(DBOptions(), db_path_, &cf_list);
@ -408,6 +436,7 @@ std::vector<std::string> LDBCommand::BuildCmdLineOptions(
ARG_WRITE_BUFFER_SIZE,
ARG_FILE_SIZE,
ARG_FIX_PREFIX_LEN,
ARG_TRY_LOAD_OPTIONS,
ARG_CF_NAME};
ret.insert(ret.end(), options.begin(), options.end());
return ret;
@ -807,7 +836,6 @@ DBLoaderCommand::DBLoaderCommand(
BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX, ARG_FROM,
ARG_TO, ARG_CREATE_IF_MISSING, ARG_DISABLE_WAL,
ARG_BULK_LOAD, ARG_COMPACT})),
create_if_missing_(false),
disable_wal_(false),
bulk_load_(false),
compact_(false) {
@ -1656,7 +1684,7 @@ void ReduceDBLevelsCommand::DoCommand() {
old_levels_ = old_level_num;
OpenDB();
if (!db_) {
if (exec_state_.IsFailed()) {
return;
}
// Compact the whole DB to put all files to the highest level.
@ -2140,6 +2168,7 @@ BatchPutCommand::BatchPutCommand(
is_value_hex_ ? HexToString(value) : value));
}
}
create_if_missing_ = IsFlagPresent(flags_, ARG_CREATE_IF_MISSING);
}
void BatchPutCommand::Help(std::string& ret) {
@ -2172,7 +2201,7 @@ void BatchPutCommand::DoCommand() {
Options BatchPutCommand::PrepareOptionsForOpenDB() {
Options opt = LDBCommand::PrepareOptionsForOpenDB();
opt.create_if_missing = IsFlagPresent(flags_, ARG_CREATE_IF_MISSING);
opt.create_if_missing = create_if_missing_;
return opt;
}
@ -2424,6 +2453,7 @@ PutCommand::PutCommand(const std::vector<std::string>& params,
if (is_value_hex_) {
value_ = HexToString(value_);
}
create_if_missing_ = IsFlagPresent(flags_, ARG_CREATE_IF_MISSING);
}
void PutCommand::Help(std::string& ret) {
@ -2449,7 +2479,7 @@ void PutCommand::DoCommand() {
Options PutCommand::PrepareOptionsForOpenDB() {
Options opt = LDBCommand::PrepareOptionsForOpenDB();
opt.create_if_missing = IsFlagPresent(flags_, ARG_CREATE_IF_MISSING);
opt.create_if_missing = create_if_missing_;
return opt;
}

1
tools/ldb_cmd_impl.h
View File

@ -139,7 +139,6 @@ class DBLoaderCommand : public LDBCommand {
virtual Options PrepareOptionsForOpenDB() override;
private:
bool create_if_missing_;
bool disable_wal_;
bool bulk_load_;
bool compact_;

5
tools/ldb_test.py
View File

@ -52,7 +52,6 @@ class LDBTestCase(unittest.TestCase):
Allows full flexibility in testing; for example: missing db param.
"""
output = my_check_output("./ldb %s |grep -v \"Created bg thread\"" %
params, shell=True)
if not unexpected:
@ -508,7 +507,7 @@ class LDBTestCase(unittest.TestCase):
dbPath = os.path.join(self.TMP_DIR, self.DB_NAME)
self.assertRunOK("put cf1_1 1 --create_if_missing", "OK")
self.assertRunOK("put cf1_2 2 --create_if_missing", "OK")
self.assertRunOK("put cf1_3 3", "OK")
self.assertRunOK("put cf1_3 3 --try_load_options", "OK")
# Given non-default column family to single CF DB.
self.assertRunFAIL("get cf1_1 --column_family=two")
self.assertRunOK("create_column_family two", "OK")
@ -525,6 +524,8 @@ class LDBTestCase(unittest.TestCase):
self.assertRunOK("get cf1_1 --column_family=default", "1")
self.assertRunOK("dump --column_family=two",
"cf2_1 ==> 1\nKeys in range: 1")
self.assertRunOK("dump --column_family=two --try_load_options",
"cf2_1 ==> 1\nKeys in range: 1")
self.assertRunOK("dump",
"cf1_1 ==> 1\ncf1_3 ==> 3\nKeys in range: 2")
self.assertRunOK("get cf2_1 --column_family=two",

2
tools/ldb_tool.cc
View File

@ -42,6 +42,8 @@ void LDBCommandRunner::PrintHelp(const LDBOptions& ldb_options,
ret.append(" --" + LDBCommand::ARG_TTL +
" with 'put','get','scan','dump','query','batchput'"
" : DB supports ttl and value is internally timestamp-suffixed\n");
ret.append(" --" + LDBCommand::ARG_TRY_LOAD_OPTIONS +
" : Try to load option file from DB.\n");
ret.append(" --" + LDBCommand::ARG_BLOOM_BITS + "=<int,e.g.:14>\n");
ret.append(" --" + LDBCommand::ARG_FIX_PREFIX_LEN + "=<int,e.g.:14>\n");
ret.append(" --" + LDBCommand::ARG_COMPRESSION_TYPE +

12
tools/verify_random_db.sh
View File

@ -7,21 +7,27 @@
scriptpath=`dirname $BASH_SOURCE`
if [ "$#" -lt 2 ]; then
echo "usage: $BASH_SOURCE <db_directory> <compare_base_db_directory> [dump_file_name]"
echo "usage: $BASH_SOURCE <db_directory> <compare_base_db_directory> [dump_file_name] [if_try_load_options]"
exit 1
fi
db_dir=$1
base_db_dir=$2
dump_file_name=${3:-"dump_file.txt"}
try_load_options=${4:-"1"}
db_dump=$db_dir"/"$dump_file_name
base_db_dump=$base_db_dir"/"$dump_file_name
extra_param=
if [ "$try_load_options" = "1" ]; then
extra_param=" --try_load_options "
fi
set -e
echo == Dumping data from $db_dir to $db_dump
./ldb dump --db=$db_dir > $db_dump
./ldb dump --db=$db_dir $extra_param > $db_dump
echo == Dumping data from $base_db_dir to $base_db_dump
./ldb dump --db=$base_db_dir > $base_db_dump
./ldb dump --db=$base_db_dir $extra_param > $base_db_dump
diff $db_dump $base_db_dir