tools: use provided options instead of the default (#4839)
Summary: The current implementation hardcode the default options in different places, which makes it impossible to support other environments (like encrypted environment). Pull Request resolved: https://github.com/facebook/rocksdb/pull/4839 Differential Revision: D13573578 Pulled By: sagar0 fbshipit-source-id: 76b58b4b758902798d10ff2f52d9f39abff015e7
This commit is contained in:
Huachao Huang
Facebook Github Bot
parent
e4feb78606
commit
74f7d7551e
@ -5,11 +5,13 @@
|
||||
#ifndef ROCKSDB_LITE
|
||||
#pragma once
|
||||
|
||||
#include "rocksdb/options.h"
|
||||
|
||||
namespace rocksdb {
|
||||
|
||||
class SSTDumpTool {
|
||||
public:
|
||||
int Run(int argc, char** argv);
|
||||
int Run(int argc, char** argv, Options options = Options());
|
||||
};
|
||||
|
||||
} // namespace rocksdb
|
||||
|
||||
@ -81,10 +81,12 @@ const char* LDBCommand::DELIM = " ==> ";
|
||||
|
||||
namespace {
|
||||
|
||||
void DumpWalFile(std::string wal_file, bool print_header, bool print_values,
|
||||
bool is_write_committed, LDBCommandExecuteResult* exec_state);
|
||||
void DumpWalFile(Options options, std::string wal_file, bool print_header,
|
||||
bool print_values, bool is_write_committed,
|
||||
LDBCommandExecuteResult* exec_state);
|
||||
|
||||
void DumpSstFile(std::string filename, bool output_hex, bool show_properties);
|
||||
void DumpSstFile(Options options, std::string filename, bool output_hex,
|
||||
bool show_properties);
|
||||
};
|
||||
|
||||
LDBCommand* LDBCommand::InitFromCmdLineArgs(
|
||||
@ -928,8 +930,8 @@ void DBLoaderCommand::DoCommand() {
|
||||
|
||||
namespace {
|
||||
|
||||
void DumpManifestFile(std::string file, bool verbose, bool hex, bool json) {
|
||||
Options options;
|
||||
void DumpManifestFile(Options options, std::string file, bool verbose, bool hex,
|
||||
bool json) {
|
||||
EnvOptions sopt;
|
||||
std::string dbname("dummy");
|
||||
std::shared_ptr<Cache> tc(NewLRUCache(options.max_open_files - 10,
|
||||
@ -1030,7 +1032,7 @@ void ManifestDumpCommand::DoCommand() {
|
||||
printf("Processing Manifest file %s\n", manifestfile.c_str());
|
||||
}
|
||||
|
||||
DumpManifestFile(manifestfile, verbose_, is_key_hex_, json_);
|
||||
DumpManifestFile(options_, manifestfile, verbose_, is_key_hex_, json_);
|
||||
|
||||
if (verbose_) {
|
||||
printf("Processing Manifest file %s done\n", manifestfile.c_str());
|
||||
@ -1425,14 +1427,15 @@ void DBDumperCommand::DoCommand() {
|
||||
switch (type) {
|
||||
case kLogFile:
|
||||
// TODO(myabandeh): allow configuring is_write_commited
|
||||
DumpWalFile(path_, /* print_header_ */ true, /* print_values_ */ true,
|
||||
true /* is_write_commited */, &exec_state_);
|
||||
DumpWalFile(options_, path_, /* print_header_ */ true,
|
||||
/* print_values_ */ true, true /* is_write_commited */,
|
||||
&exec_state_);
|
||||
break;
|
||||
case kTableFile:
|
||||
DumpSstFile(path_, is_key_hex_, /* show_properties */ true);
|
||||
DumpSstFile(options_, path_, is_key_hex_, /* show_properties */ true);
|
||||
break;
|
||||
case kDescriptorFile:
|
||||
DumpManifestFile(path_, /* verbose_ */ false, is_key_hex_,
|
||||
DumpManifestFile(options_, path_, /* verbose_ */ false, is_key_hex_,
|
||||
/* json_ */ false);
|
||||
break;
|
||||
default:
|
||||
@ -1960,16 +1963,17 @@ class InMemoryHandler : public WriteBatch::Handler {
|
||||
bool write_after_commit_;
|
||||
};
|
||||
|
||||
void DumpWalFile(std::string wal_file, bool print_header, bool print_values,
|
||||
bool is_write_committed, LDBCommandExecuteResult* exec_state) {
|
||||
Env* env_ = Env::Default();
|
||||
EnvOptions soptions;
|
||||
void DumpWalFile(Options options, std::string wal_file, bool print_header,
|
||||
bool print_values, bool is_write_committed,
|
||||
LDBCommandExecuteResult* exec_state) {
|
||||
Env* env = options.env;
|
||||
EnvOptions soptions(options);
|
||||
std::unique_ptr<SequentialFileReader> wal_file_reader;
|
||||
|
||||
Status status;
|
||||
{
|
||||
std::unique_ptr<SequentialFile> file;
|
||||
status = env_->NewSequentialFile(wal_file, &file, soptions);
|
||||
status = env->NewSequentialFile(wal_file, &file, soptions);
|
||||
if (status.ok()) {
|
||||
wal_file_reader.reset(
|
||||
new SequentialFileReader(std::move(file), wal_file));
|
||||
@ -1997,9 +2001,8 @@ void DumpWalFile(std::string wal_file, bool print_header, bool print_values,
|
||||
// bogus input, carry on as best we can
|
||||
log_number = 0;
|
||||
}
|
||||
DBOptions db_options;
|
||||
log::Reader reader(db_options.info_log, std::move(wal_file_reader),
|
||||
&reporter, true /* checksum */, log_number,
|
||||
log::Reader reader(options.info_log, std::move(wal_file_reader), &reporter,
|
||||
true /* checksum */, log_number,
|
||||
false /* retry_after_eof */);
|
||||
std::string scratch;
|
||||
WriteBatch batch;
|
||||
@ -2079,8 +2082,8 @@ void WALDumperCommand::Help(std::string& ret) {
|
||||
}
|
||||
|
||||
void WALDumperCommand::DoCommand() {
|
||||
DumpWalFile(wal_file_, print_header_, print_values_, is_write_committed_,
|
||||
&exec_state_);
|
||||
DumpWalFile(options_, wal_file_, print_header_, print_values_,
|
||||
is_write_committed_, &exec_state_);
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------
|
||||
@ -2836,7 +2839,8 @@ void RestoreCommand::DoCommand() {
|
||||
|
||||
namespace {
|
||||
|
||||
void DumpSstFile(std::string filename, bool output_hex, bool show_properties) {
|
||||
void DumpSstFile(Options options, std::string filename, bool output_hex,
|
||||
bool show_properties) {
|
||||
std::string from_key;
|
||||
std::string to_key;
|
||||
if (filename.length() <= 4 ||
|
||||
@ -2845,7 +2849,7 @@ void DumpSstFile(std::string filename, bool output_hex, bool show_properties) {
|
||||
return;
|
||||
}
|
||||
// no verification
|
||||
rocksdb::SstFileDumper dumper(filename, false, output_hex);
|
||||
rocksdb::SstFileDumper dumper(options, filename, false, output_hex);
|
||||
Status st = dumper.ReadSequential(true, std::numeric_limits<uint64_t>::max(),
|
||||
false, // has_from
|
||||
from_key, false, // has_to
|
||||
@ -2911,7 +2915,7 @@ void DBFileDumperCommand::DoCommand() {
|
||||
manifest_filename.resize(manifest_filename.size() - 1);
|
||||
std::string manifest_filepath = db_->GetName() + "/" + manifest_filename;
|
||||
std::cout << manifest_filepath << std::endl;
|
||||
DumpManifestFile(manifest_filepath, false, false, false);
|
||||
DumpManifestFile(options_, manifest_filepath, false, false, false);
|
||||
std::cout << std::endl;
|
||||
|
||||
std::cout << "SST Files" << std::endl;
|
||||
@ -2922,7 +2926,7 @@ void DBFileDumperCommand::DoCommand() {
|
||||
std::string filename = fileMetadata.db_path + fileMetadata.name;
|
||||
std::cout << filename << " level:" << fileMetadata.level << std::endl;
|
||||
std::cout << "------------------------------" << std::endl;
|
||||
DumpSstFile(filename, false, true);
|
||||
DumpSstFile(options_, filename, false, true);
|
||||
std::cout << std::endl;
|
||||
}
|
||||
std::cout << std::endl;
|
||||
@ -2939,7 +2943,7 @@ void DBFileDumperCommand::DoCommand() {
|
||||
std::string filename = db_->GetOptions().wal_dir + wal->PathName();
|
||||
std::cout << filename << std::endl;
|
||||
// TODO(myabandeh): allow configuring is_write_commited
|
||||
DumpWalFile(filename, true, true, true /* is_write_commited */,
|
||||
DumpWalFile(options_, filename, true, true, true /* is_write_commited */,
|
||||
&exec_state_);
|
||||
}
|
||||
}
|
||||
|
||||
@ -12,6 +12,8 @@ using std::string;
|
||||
using std::vector;
|
||||
using std::map;
|
||||
|
||||
namespace rocksdb {
|
||||
|
||||
class LdbCmdTest : public testing::Test {};
|
||||
|
||||
TEST_F(LdbCmdTest, HexToString) {
|
||||
@ -47,6 +49,40 @@ TEST_F(LdbCmdTest, HexToStringBadInputs) {
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(LdbCmdTest, MemEnv) {
|
||||
std::unique_ptr<Env> env(NewMemEnv(Env::Default()));
|
||||
Options opts;
|
||||
opts.env = env.get();
|
||||
opts.create_if_missing = true;
|
||||
|
||||
DB* db = nullptr;
|
||||
std::string dbname = test::TmpDir();
|
||||
ASSERT_OK(DB::Open(opts, dbname, &db));
|
||||
|
||||
WriteOptions wopts;
|
||||
for (int i = 0; i < 100; i++) {
|
||||
char buf[16];
|
||||
snprintf(buf, sizeof(buf), "%08d", i);
|
||||
ASSERT_OK(db->Put(wopts, buf, buf));
|
||||
}
|
||||
FlushOptions fopts;
|
||||
fopts.wait = true;
|
||||
ASSERT_OK(db->Flush(fopts));
|
||||
|
||||
delete db;
|
||||
|
||||
char arg1[] = "./ldb";
|
||||
char arg2[1024];
|
||||
snprintf(arg2, sizeof(arg2), "--db=%s", dbname.c_str());
|
||||
char arg3[] = "dump_live_files";
|
||||
char* argv[] = {arg1, arg2, arg3};
|
||||
|
||||
rocksdb::LDBTool tool;
|
||||
tool.Run(3, argv, opts);
|
||||
}
|
||||
|
||||
} // namespace rocksdb
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
::testing::InitGoogleTest(&argc, argv);
|
||||
return RUN_ALL_TESTS();
|
||||
|
||||
@ -38,24 +38,18 @@ static std::string MakeValue(int i) {
|
||||
return key.Encode().ToString();
|
||||
}
|
||||
|
||||
void createSST(const std::string& file_name,
|
||||
const BlockBasedTableOptions& table_options) {
|
||||
std::shared_ptr<rocksdb::TableFactory> tf;
|
||||
tf.reset(new rocksdb::BlockBasedTableFactory(table_options));
|
||||
|
||||
std::unique_ptr<WritableFile> file;
|
||||
Env* env = Env::Default();
|
||||
EnvOptions env_options;
|
||||
void createSST(const Options& opts, const std::string& file_name) {
|
||||
Env* env = opts.env;
|
||||
EnvOptions env_options(opts);
|
||||
ReadOptions read_options;
|
||||
Options opts;
|
||||
const ImmutableCFOptions imoptions(opts);
|
||||
const MutableCFOptions moptions(opts);
|
||||
rocksdb::InternalKeyComparator ikc(opts.comparator);
|
||||
std::unique_ptr<TableBuilder> tb;
|
||||
|
||||
std::unique_ptr<WritableFile> file;
|
||||
ASSERT_OK(env->NewWritableFile(file_name, &file, env_options));
|
||||
|
||||
opts.table_factory = tf;
|
||||
std::vector<std::unique_ptr<IntTblPropCollectorFactory> >
|
||||
int_tbl_prop_collector_factories;
|
||||
std::unique_ptr<WritableFileWriter> file_writer(
|
||||
@ -80,8 +74,8 @@ void createSST(const std::string& file_name,
|
||||
file_writer->Close();
|
||||
}
|
||||
|
||||
void cleanup(const std::string& file_name) {
|
||||
Env* env = Env::Default();
|
||||
void cleanup(const Options& opts, const std::string& file_name) {
|
||||
Env* env = opts.env;
|
||||
env->DeleteFile(file_name);
|
||||
std::string outfile_name = file_name.substr(0, file_name.length() - 4);
|
||||
outfile_name.append("_dump.txt");
|
||||
@ -94,8 +88,6 @@ class SSTDumpToolTest : public testing::Test {
|
||||
std::string testDir_;
|
||||
|
||||
public:
|
||||
BlockBasedTableOptions table_options_;
|
||||
|
||||
SSTDumpToolTest() { testDir_ = test::TmpDir(); }
|
||||
|
||||
~SSTDumpToolTest() {}
|
||||
@ -119,88 +111,121 @@ class SSTDumpToolTest : public testing::Test {
|
||||
};
|
||||
|
||||
TEST_F(SSTDumpToolTest, EmptyFilter) {
|
||||
Options opts;
|
||||
std::string file_path = MakeFilePath("rocksdb_sst_test.sst");
|
||||
createSST(file_path, table_options_);
|
||||
createSST(opts, file_path);
|
||||
|
||||
char* usage[3];
|
||||
PopulateCommandArgs(file_path, "--command=raw", usage);
|
||||
|
||||
rocksdb::SSTDumpTool tool;
|
||||
ASSERT_TRUE(!tool.Run(3, usage));
|
||||
ASSERT_TRUE(!tool.Run(3, usage, opts));
|
||||
|
||||
cleanup(file_path);
|
||||
cleanup(opts, file_path);
|
||||
for (int i = 0; i < 3; i++) {
|
||||
delete[] usage[i];
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(SSTDumpToolTest, FilterBlock) {
|
||||
table_options_.filter_policy.reset(rocksdb::NewBloomFilterPolicy(10, true));
|
||||
Options opts;
|
||||
BlockBasedTableOptions table_opts;
|
||||
table_opts.filter_policy.reset(rocksdb::NewBloomFilterPolicy(10, true));
|
||||
opts.table_factory.reset(new BlockBasedTableFactory(table_opts));
|
||||
std::string file_path = MakeFilePath("rocksdb_sst_test.sst");
|
||||
createSST(file_path, table_options_);
|
||||
createSST(opts, file_path);
|
||||
|
||||
char* usage[3];
|
||||
PopulateCommandArgs(file_path, "--command=raw", usage);
|
||||
|
||||
rocksdb::SSTDumpTool tool;
|
||||
ASSERT_TRUE(!tool.Run(3, usage));
|
||||
ASSERT_TRUE(!tool.Run(3, usage, opts));
|
||||
|
||||
cleanup(file_path);
|
||||
cleanup(opts, file_path);
|
||||
for (int i = 0; i < 3; i++) {
|
||||
delete[] usage[i];
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(SSTDumpToolTest, FullFilterBlock) {
|
||||
table_options_.filter_policy.reset(rocksdb::NewBloomFilterPolicy(10, false));
|
||||
Options opts;
|
||||
BlockBasedTableOptions table_opts;
|
||||
table_opts.filter_policy.reset(rocksdb::NewBloomFilterPolicy(10, false));
|
||||
opts.table_factory.reset(new BlockBasedTableFactory(table_opts));
|
||||
std::string file_path = MakeFilePath("rocksdb_sst_test.sst");
|
||||
createSST(file_path, table_options_);
|
||||
createSST(opts, file_path);
|
||||
|
||||
char* usage[3];
|
||||
PopulateCommandArgs(file_path, "--command=raw", usage);
|
||||
|
||||
rocksdb::SSTDumpTool tool;
|
||||
ASSERT_TRUE(!tool.Run(3, usage));
|
||||
ASSERT_TRUE(!tool.Run(3, usage, opts));
|
||||
|
||||
cleanup(file_path);
|
||||
cleanup(opts, file_path);
|
||||
for (int i = 0; i < 3; i++) {
|
||||
delete[] usage[i];
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(SSTDumpToolTest, GetProperties) {
|
||||
table_options_.filter_policy.reset(rocksdb::NewBloomFilterPolicy(10, false));
|
||||
Options opts;
|
||||
BlockBasedTableOptions table_opts;
|
||||
table_opts.filter_policy.reset(rocksdb::NewBloomFilterPolicy(10, false));
|
||||
opts.table_factory.reset(new BlockBasedTableFactory(table_opts));
|
||||
std::string file_path = MakeFilePath("rocksdb_sst_test.sst");
|
||||
createSST(file_path, table_options_);
|
||||
createSST(opts, file_path);
|
||||
|
||||
char* usage[3];
|
||||
PopulateCommandArgs(file_path, "--show_properties", usage);
|
||||
|
||||
rocksdb::SSTDumpTool tool;
|
||||
ASSERT_TRUE(!tool.Run(3, usage));
|
||||
ASSERT_TRUE(!tool.Run(3, usage, opts));
|
||||
|
||||
cleanup(file_path);
|
||||
cleanup(opts, file_path);
|
||||
for (int i = 0; i < 3; i++) {
|
||||
delete[] usage[i];
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(SSTDumpToolTest, CompressedSizes) {
|
||||
table_options_.filter_policy.reset(rocksdb::NewBloomFilterPolicy(10, false));
|
||||
Options opts;
|
||||
BlockBasedTableOptions table_opts;
|
||||
table_opts.filter_policy.reset(rocksdb::NewBloomFilterPolicy(10, false));
|
||||
opts.table_factory.reset(new BlockBasedTableFactory(table_opts));
|
||||
std::string file_path = MakeFilePath("rocksdb_sst_test.sst");
|
||||
createSST(file_path, table_options_);
|
||||
createSST(opts, file_path);
|
||||
|
||||
char* usage[3];
|
||||
PopulateCommandArgs(file_path, "--command=recompress", usage);
|
||||
|
||||
rocksdb::SSTDumpTool tool;
|
||||
ASSERT_TRUE(!tool.Run(3, usage));
|
||||
ASSERT_TRUE(!tool.Run(3, usage, opts));
|
||||
|
||||
cleanup(file_path);
|
||||
cleanup(opts, file_path);
|
||||
for (int i = 0; i < 3; i++) {
|
||||
delete[] usage[i];
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(SSTDumpToolTest, MemEnv) {
|
||||
std::unique_ptr<Env> env(NewMemEnv(Env::Default()));
|
||||
Options opts;
|
||||
opts.env = env.get();
|
||||
std::string file_path = MakeFilePath("rocksdb_sst_test.sst");
|
||||
createSST(opts, file_path);
|
||||
|
||||
char* usage[3];
|
||||
PopulateCommandArgs(file_path, "--command=verify_checksum", usage);
|
||||
|
||||
rocksdb::SSTDumpTool tool;
|
||||
ASSERT_TRUE(!tool.Run(3, usage, opts));
|
||||
|
||||
cleanup(opts, file_path);
|
||||
for (int i = 0; i < 3; i++) {
|
||||
delete[] usage[i];
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace rocksdb
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
|
||||
@ -43,12 +43,14 @@
|
||||
|
||||
namespace rocksdb {
|
||||
|
||||
SstFileDumper::SstFileDumper(const std::string& file_path, bool verify_checksum,
|
||||
SstFileDumper::SstFileDumper(const Options& options,
|
||||
const std::string& file_path, bool verify_checksum,
|
||||
bool output_hex)
|
||||
: file_name_(file_path),
|
||||
read_num_(0),
|
||||
verify_checksum_(verify_checksum),
|
||||
output_hex_(output_hex),
|
||||
options_(options),
|
||||
ioptions_(options_),
|
||||
moptions_(ColumnFamilyOptions(options_)),
|
||||
internal_comparator_(BytewiseComparator()) {
|
||||
@ -417,7 +419,7 @@ void print_help() {
|
||||
|
||||
} // namespace
|
||||
|
||||
int SSTDumpTool::Run(int argc, char** argv) {
|
||||
int SSTDumpTool::Run(int argc, char** argv, Options options) {
|
||||
const char* dir_or_file = nullptr;
|
||||
uint64_t read_num = std::numeric_limits<uint64_t>::max();
|
||||
std::string command;
|
||||
@ -545,7 +547,7 @@ int SSTDumpTool::Run(int argc, char** argv) {
|
||||
}
|
||||
|
||||
std::vector<std::string> filenames;
|
||||
rocksdb::Env* env = rocksdb::Env::Default();
|
||||
rocksdb::Env* env = options.env;
|
||||
rocksdb::Status st = env->GetChildren(dir_or_file, &filenames);
|
||||
bool dir = true;
|
||||
if (!st.ok()) {
|
||||
@ -570,7 +572,8 @@ int SSTDumpTool::Run(int argc, char** argv) {
|
||||
filename = std::string(dir_or_file) + "/" + filename;
|
||||
}
|
||||
|
||||
rocksdb::SstFileDumper dumper(filename, verify_checksum, output_hex);
|
||||
rocksdb::SstFileDumper dumper(options, filename, verify_checksum,
|
||||
output_hex);
|
||||
if (!dumper.getStatus().ok()) {
|
||||
fprintf(stderr, "%s: %s\n", filename.c_str(),
|
||||
dumper.getStatus().ToString().c_str());
|
||||
|
||||
@ -17,8 +17,8 @@ namespace rocksdb {
|
||||
|
||||
class SstFileDumper {
|
||||
public:
|
||||
explicit SstFileDumper(const std::string& file_name, bool verify_checksum,
|
||||
bool output_hex);
|
||||
explicit SstFileDumper(const Options& options, const std::string& file_name,
|
||||
bool verify_checksum, bool output_hex);
|
||||
|
||||
Status ReadSequential(bool print_kv, uint64_t read_num, bool has_from,
|
||||
const std::string& from_key, bool has_to,
|
||||
|
||||
Loading…
Reference in New Issue
Block a user