89ad9f3adb
Summary:
Added a flag, `ignore_unknown_options`, to skip unknown options when loading an options file (using `LoadLatestOptions`/`LoadOptionsFromFile`) or while verifying options (using `CheckOptionsCompatibility`). This will help in downgrading the db to an older version.
Also added `--ignore_unknown_options` flag to ldb
**Example Use case:**
In MyRocks, if copying from newer version to older version, it is often impossible to start because of new RocksDB options that don't exist in older version, even though data format is compatible.
MyRocks uses these load and verify functions in [ha_rocksdb.cc::check_rocksdb_options_compatibility](e004fd9f41/storage/rocksdb/ha_rocksdb.cc (L3348-L3401)
).
**Test Plan:**
Updated the unit tests.
`make check`
ldb:
$ ./ldb --db=/tmp/test_db --create_if_missing put a1 b1
OK
Now edit /tmp/test_db/<OPTIONS-file> and add an unknown option.
Try loading the options now, and it fails:
$ ./ldb --db=/tmp/test_db --try_load_options get a1
Failed: Invalid argument: Unrecognized option DBOptions:: abcd
Passes with the new --ignore_unknown_options flag
$ ./ldb --db=/tmp/test_db --try_load_options --ignore_unknown_options get a1
b1
Closes https://github.com/facebook/rocksdb/pull/2423
Differential Revision: D5212091
Pulled By: sagar0
fbshipit-source-id: 2ec17636feb47dc0351b53a77e5f15ef7cbf2ca7
147 lines
5.1 KiB
C++
147 lines
5.1 KiB
C++
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
|
// This source code is licensed under the BSD-style license found in the
|
|
// LICENSE file in the root directory of this source tree. An additional grant
|
|
// of patent rights can be found in the PATENTS file in the same directory.
|
|
|
|
#pragma once
|
|
|
|
#include <map>
|
|
#include <string>
|
|
#include <vector>
|
|
|
|
#include "options/options_helper.h"
|
|
#include "options/options_sanity_check.h"
|
|
#include "rocksdb/env.h"
|
|
#include "rocksdb/options.h"
|
|
#include "table/block_based_table_factory.h"
|
|
|
|
namespace rocksdb {
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
#define ROCKSDB_OPTION_FILE_MAJOR 1
|
|
#define ROCKSDB_OPTION_FILE_MINOR 1
|
|
|
|
enum OptionSection : char {
|
|
kOptionSectionVersion = 0,
|
|
kOptionSectionDBOptions,
|
|
kOptionSectionCFOptions,
|
|
kOptionSectionTableOptions,
|
|
kOptionSectionUnknown
|
|
};
|
|
|
|
static const std::string opt_section_titles[] = {
|
|
"Version", "DBOptions", "CFOptions", "TableOptions/", "Unknown"};
|
|
|
|
Status PersistRocksDBOptions(const DBOptions& db_opt,
|
|
const std::vector<std::string>& cf_names,
|
|
const std::vector<ColumnFamilyOptions>& cf_opts,
|
|
const std::string& file_name, Env* env);
|
|
|
|
class RocksDBOptionsParser {
|
|
public:
|
|
explicit RocksDBOptionsParser();
|
|
~RocksDBOptionsParser() {}
|
|
void Reset();
|
|
|
|
Status Parse(const std::string& file_name, Env* env,
|
|
bool ignore_unknown_options = false);
|
|
static std::string TrimAndRemoveComment(const std::string& line,
|
|
const bool trim_only = false);
|
|
|
|
const DBOptions* db_opt() const { return &db_opt_; }
|
|
const std::unordered_map<std::string, std::string>* db_opt_map() const {
|
|
return &db_opt_map_;
|
|
}
|
|
const std::vector<ColumnFamilyOptions>* cf_opts() const { return &cf_opts_; }
|
|
const std::vector<std::string>* cf_names() const { return &cf_names_; }
|
|
const std::vector<std::unordered_map<std::string, std::string>>* cf_opt_maps()
|
|
const {
|
|
return &cf_opt_maps_;
|
|
}
|
|
|
|
const ColumnFamilyOptions* GetCFOptions(const std::string& name) {
|
|
return GetCFOptionsImpl(name);
|
|
}
|
|
size_t NumColumnFamilies() { return cf_opts_.size(); }
|
|
|
|
static Status VerifyRocksDBOptionsFromFile(
|
|
const DBOptions& db_opt, const std::vector<std::string>& cf_names,
|
|
const std::vector<ColumnFamilyOptions>& cf_opts,
|
|
const std::string& file_name, Env* env,
|
|
OptionsSanityCheckLevel sanity_check_level = kSanityLevelExactMatch,
|
|
bool ignore_unknown_options = false);
|
|
|
|
static Status VerifyDBOptions(
|
|
const DBOptions& base_opt, const DBOptions& new_opt,
|
|
const std::unordered_map<std::string, std::string>* new_opt_map = nullptr,
|
|
OptionsSanityCheckLevel sanity_check_level = kSanityLevelExactMatch);
|
|
|
|
static Status VerifyCFOptions(
|
|
const ColumnFamilyOptions& base_opt, const ColumnFamilyOptions& new_opt,
|
|
const std::unordered_map<std::string, std::string>* new_opt_map = nullptr,
|
|
OptionsSanityCheckLevel sanity_check_level = kSanityLevelExactMatch);
|
|
|
|
static Status VerifyTableFactory(
|
|
const TableFactory* base_tf, const TableFactory* file_tf,
|
|
OptionsSanityCheckLevel sanity_check_level = kSanityLevelExactMatch);
|
|
|
|
static Status VerifyBlockBasedTableFactory(
|
|
const BlockBasedTableFactory* base_tf,
|
|
const BlockBasedTableFactory* file_tf,
|
|
OptionsSanityCheckLevel sanity_check_level);
|
|
|
|
static Status ExtraParserCheck(const RocksDBOptionsParser& input_parser);
|
|
|
|
protected:
|
|
bool IsSection(const std::string& line);
|
|
Status ParseSection(OptionSection* section, std::string* title,
|
|
std::string* argument, const std::string& line,
|
|
const int line_num);
|
|
|
|
Status CheckSection(const OptionSection section,
|
|
const std::string& section_arg, const int line_num);
|
|
|
|
Status ParseStatement(std::string* name, std::string* value,
|
|
const std::string& line, const int line_num);
|
|
|
|
Status EndSection(const OptionSection section, const std::string& title,
|
|
const std::string& section_arg,
|
|
const std::unordered_map<std::string, std::string>& opt_map,
|
|
bool ignore_unknown_options);
|
|
|
|
Status ValidityCheck();
|
|
|
|
Status InvalidArgument(const int line_num, const std::string& message);
|
|
|
|
Status ParseVersionNumber(const std::string& ver_name,
|
|
const std::string& ver_string, const int max_count,
|
|
int* version);
|
|
|
|
ColumnFamilyOptions* GetCFOptionsImpl(const std::string& name) {
|
|
assert(cf_names_.size() == cf_opts_.size());
|
|
for (size_t i = 0; i < cf_names_.size(); ++i) {
|
|
if (cf_names_[i] == name) {
|
|
return &cf_opts_[i];
|
|
}
|
|
}
|
|
return nullptr;
|
|
}
|
|
|
|
private:
|
|
DBOptions db_opt_;
|
|
std::unordered_map<std::string, std::string> db_opt_map_;
|
|
std::vector<std::string> cf_names_;
|
|
std::vector<ColumnFamilyOptions> cf_opts_;
|
|
std::vector<std::unordered_map<std::string, std::string>> cf_opt_maps_;
|
|
bool has_version_section_;
|
|
bool has_db_options_;
|
|
bool has_default_cf_options_;
|
|
int db_version[3];
|
|
int opt_file_version[3];
|
|
};
|
|
|
|
#endif // !ROCKSDB_LITE
|
|
|
|
} // namespace rocksdb
|