diff --git a/HISTORY.md b/HISTORY.md
index 6e9a6578d..893d46040 100644
--- a/HISTORY.md
+++ b/HISTORY.md
@@ -5,6 +5,7 @@
* Finish implementation of BlockBasedTableOptions::IndexType::kBinarySearchWithFirstKey. It's now ready for use. Significantly reduces read amplification in some setups, especially for iterator seeks.
### Public API Change
+* Add a ConfigOptions argument to the APIs dealing with converting options to and from strings and files. The ConfigOptions is meant to replace some of the options (such as input_strings_escaped and ignore_unknown_options) and allow for more parameters to be passed in the future without changing the function signature.
* Add NewFileChecksumGenCrc32cFactory to the file checksum public API, such that the builtin Crc32c based file checksum generator factory can be used by applications.
* Add IsDirectory to Env and FS to indicate if a path is a directory.
diff --git a/db/column_family_test.cc b/db/column_family_test.cc
index feee3dcf4..fa791506f 100644
--- a/db/column_family_test.cc
+++ b/db/column_family_test.cc
@@ -8,16 +8,16 @@
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#include
-#include
#include
#include
+#include
#include "db/db_impl/db_impl.h"
#include "db/db_test_util.h"
-#include "memtable/hash_skiplist_rep.h"
#include "options/options_parser.h"
#include "port/port.h"
#include "port/stack_trace.h"
+#include "rocksdb/convenience.h"
#include "rocksdb/db.h"
#include "rocksdb/env.h"
#include "rocksdb/iterator.h"
@@ -287,7 +287,9 @@ class ColumnFamilyTestBase : public testing::Test {
// Verify the CF options of the returned CF handle.
ColumnFamilyDescriptor desc;
ASSERT_OK(handles_[cfi]->GetDescriptor(&desc));
- RocksDBOptionsParser::VerifyCFOptions(desc.options, current_cf_opt);
+ RocksDBOptionsParser::VerifyCFOptions(ConfigOptions(), desc.options,
+ current_cf_opt);
+
#endif // !ROCKSDB_LITE
cfi++;
}
diff --git a/db/db_bloom_filter_test.cc b/db/db_bloom_filter_test.cc
index 00f2f07eb..194c87732 100644
--- a/db/db_bloom_filter_test.cc
+++ b/db/db_bloom_filter_test.cc
@@ -8,6 +8,7 @@
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#include "db/db_test_util.h"
+#include "options/options_helper.h"
#include "port/stack_trace.h"
#include "rocksdb/perf_context.h"
#include "table/block_based/filter_policy_internal.h"
diff --git a/db/db_options_test.cc b/db/db_options_test.cc
index 76ee27224..a884ac58e 100644
--- a/db/db_options_test.cc
+++ b/db/db_options_test.cc
@@ -33,7 +33,9 @@ class DBOptionsTest : public DBTestBase {
std::unordered_map GetMutableDBOptionsMap(
const DBOptions& options) {
std::string options_str;
- GetStringFromDBOptions(&options_str, options);
+ ConfigOptions config_options;
+ config_options.delimiter = "; ";
+ GetStringFromDBOptions(config_options, options, &options_str);
std::unordered_map options_map;
StringToMap(options_str, &options_map);
std::unordered_map mutable_map;
@@ -48,7 +50,10 @@ class DBOptionsTest : public DBTestBase {
std::unordered_map GetMutableCFOptionsMap(
const ColumnFamilyOptions& options) {
std::string options_str;
- GetStringFromColumnFamilyOptions(&options_str, options);
+ ConfigOptions config_options;
+ config_options.delimiter = "; ";
+
+ GetStringFromColumnFamilyOptions(config_options, options, &options_str);
std::unordered_map options_map;
StringToMap(options_str, &options_map);
std::unordered_map mutable_map;
diff --git a/examples/options_file_example.cc b/examples/options_file_example.cc
index e6a1d0e9a..30051d8d5 100644
--- a/examples/options_file_example.cc
+++ b/examples/options_file_example.cc
@@ -79,7 +79,8 @@ int main() {
// Load the options file.
DBOptions loaded_db_opt;
std::vector loaded_cf_descs;
- s = LoadLatestOptions(kDBPath, Env::Default(), &loaded_db_opt,
+ ConfigOptions config_options;
+ s = LoadLatestOptions(config_options, kDBPath, &loaded_db_opt,
&loaded_cf_descs);
assert(s.ok());
assert(loaded_db_opt.create_if_missing == db_opt.create_if_missing);
diff --git a/include/rocksdb/convenience.h b/include/rocksdb/convenience.h
index 442303d94..d8843cb76 100644
--- a/include/rocksdb/convenience.h
+++ b/include/rocksdb/convenience.h
@@ -10,12 +10,74 @@
#include
#include "rocksdb/db.h"
-#include "rocksdb/options.h"
+#include "rocksdb/status.h"
#include "rocksdb/table.h"
namespace ROCKSDB_NAMESPACE {
+class Env;
+struct ColumnFamilyOptions;
+struct DBOptions;
+struct Options;
+
+// ConfigOptions containing the parameters/controls for
+// comparing objects and converting to/from strings.
+// These settings control how the methods
+// treat errors (e.g. ignore_unknown_objects), the format
+// of the serialization (e.g. delimiter), and how to compare
+// options (sanity_level).
+struct ConfigOptions {
+ // This enum defines the RocksDB options sanity level.
+ enum SanityLevel : unsigned char {
+ kSanityLevelNone = 0x01, // Performs no sanity check at all.
+ // Performs minimum check to ensure the RocksDB instance can be
+ // opened without corrupting / mis-interpreting the data.
+ kSanityLevelLooselyCompatible = 0x02,
+ // Perform exact match sanity check.
+ kSanityLevelExactMatch = 0xFF,
+ };
+
+ enum Depth {
+ kDepthDefault, // Traverse nested options that are not flagged as "shallow"
+ kDepthShallow, // Do not traverse into any nested options
+ kDepthDetailed, // Traverse nested options, overriding the options shallow
+ // setting
+ };
+
+ // When true, any unused options will be ignored and OK will be returned
+ bool ignore_unknown_options = false;
+
+ // If the strings are escaped (old-style?)
+ bool input_strings_escaped = true;
+
+ // The separator between options when converting to a string
+ std::string delimiter = ";";
+
+ // Controls how to traverse options during print/match stages
+ Depth depth = Depth::kDepthDefault;
+
+ // Controls how options are serialized
+ // Controls how pedantic the comparison must be for equivalency
+ SanityLevel sanity_level = SanityLevel::kSanityLevelExactMatch;
+ // `file_readahead_size` is used for readahead for the option file.
+ size_t file_readahead_size = 512 * 1024;
+
+ // The environment to use for this option
+ Env* env = Env::Default();
+
+ bool IsShallow() const { return depth == Depth::kDepthShallow; }
+ bool IsDetailed() const { return depth == Depth::kDepthDetailed; }
+
+ bool IsCheckDisabled() const {
+ return sanity_level == SanityLevel::kSanityLevelNone;
+ }
+
+ bool IsCheckEnabled(SanityLevel level) const {
+ return (level > SanityLevel::kSanityLevelNone && level <= sanity_level);
+ }
+};
#ifndef ROCKSDB_LITE
+
// The following set of functions provide a way to construct RocksDB Options
// from a string or a string-to-string map. Here're the general rule of
// setting option values from strings by type. Some RocksDB types are also
@@ -134,13 +196,6 @@ namespace ROCKSDB_NAMESPACE {
// [Example]:
// * {"memtable", "vector:1024"} is equivalent to setting memtable
// to VectorRepFactory(1024).
-// - HashCuckooRepFactory:
-// Pass "cuckoo:" to use HashCuckooRepFactory with the
-// specified write buffer size, or simply "cuckoo" to use the default
-// HashCuckooRepFactory.
-// [Example]:
-// * {"memtable", "cuckoo:1024"} is equivalent to setting memtable
-// to NewHashCuckooRepFactory(1024).
//
// * compression_opts:
// Use "compression_opts" to config compression_opts. The value format
@@ -153,6 +208,12 @@ namespace ROCKSDB_NAMESPACE {
// cf_opt.compression_opts.strategy = 6;
// cf_opt.compression_opts.max_dict_bytes = 7;
//
+// The GetColumnFamilyOptionsFromMap(ConfigOptions, ...) should be used; the
+// alternative signature may be deprecated in a future release. The equivalent
+// functionality can be achieved by setting the corresponding options in
+// the ConfigOptions parameter.
+//
+// @param config_options controls how the map is processed.
// @param base_options the default options of the output "new_options".
// @param opts_map an option name to value map for specifying how "new_options"
// should be set.
@@ -165,6 +226,11 @@ namespace ROCKSDB_NAMESPACE {
// instead of resulting in an unknown-option error.
// @return Status::OK() on success. Otherwise, a non-ok status indicating
// error will be returned, and "new_options" will be set to "base_options".
+Status GetColumnFamilyOptionsFromMap(
+ const ConfigOptions& config_options,
+ const ColumnFamilyOptions& base_options,
+ const std::unordered_map& opts_map,
+ ColumnFamilyOptions* new_options);
Status GetColumnFamilyOptionsFromMap(
const ColumnFamilyOptions& base_options,
const std::unordered_map& opts_map,
@@ -184,6 +250,12 @@ Status GetColumnFamilyOptionsFromMap(
// - Passing {"rate_limiter_bytes_per_sec", "1024"} is equivalent to
// passing NewGenericRateLimiter(1024) to rate_limiter_bytes_per_sec.
//
+// The GetDBOptionsFromMap(ConfigOptions, ...) should be used; the
+// alternative signature may be deprecated in a future release. The equivalent
+// functionality can be achieved by setting the corresponding options in
+// the ConfigOptions parameter.
+//
+// @param config_options controls how the map is processed.
// @param base_options the default options of the output "new_options".
// @param opts_map an option name to value map for specifying how "new_options"
// should be set.
@@ -196,6 +268,10 @@ Status GetColumnFamilyOptionsFromMap(
// instead of resulting in an unknown-option error.
// @return Status::OK() on success. Otherwise, a non-ok status indicating
// error will be returned, and "new_options" will be set to "base_options".
+Status GetDBOptionsFromMap(
+ const ConfigOptions& cfg_options, const DBOptions& base_options,
+ const std::unordered_map& opts_map,
+ DBOptions* new_options);
Status GetDBOptionsFromMap(
const DBOptions& base_options,
const std::unordered_map& opts_map,
@@ -227,6 +303,12 @@ Status GetDBOptionsFromMap(
// - Passing {"block_cache", "1M"} in GetBlockBasedTableOptionsFromMap is
// equivalent to setting block_cache using NewLRUCache(1024 * 1024).
//
+// The GetBlockBasedTableOptionsFromMap(ConfigOptions, ...) should be used;
+// the alternative signature may be deprecated in a future release. The
+// equivalent functionality can be achieved by setting the corresponding
+// options in the ConfigOptions parameter.
+//
+// @param config_options controls how the map is processed.
// @param table_options the default options of the output "new_table_options".
// @param opts_map an option name to value map for specifying how
// "new_table_options" should be set.
@@ -240,6 +322,11 @@ Status GetDBOptionsFromMap(
// @return Status::OK() on success. Otherwise, a non-ok status indicating
// error will be returned, and "new_table_options" will be set to
// "table_options".
+Status GetBlockBasedTableOptionsFromMap(
+ const ConfigOptions& config_options,
+ const BlockBasedTableOptions& table_options,
+ const std::unordered_map& opts_map,
+ BlockBasedTableOptions* new_table_options);
Status GetBlockBasedTableOptionsFromMap(
const BlockBasedTableOptions& table_options,
const std::unordered_map& opts_map,
@@ -250,6 +337,12 @@ Status GetBlockBasedTableOptionsFromMap(
// map "opts_map" of option name to option value to construct the new
// PlainTableOptions "new_table_options".
//
+// The GetPlainTableOptionsFromMap(ConfigOptions, ...) should be used; the
+// alternative signature may be deprecated in a future release. The equivalent
+// functionality can be achieved by setting the corresponding options in
+// the ConfigOptions parameter.
+//
+// @param config_options controls how the map is processed.
// @param table_options the default options of the output "new_table_options".
// @param opts_map an option name to value map for specifying how
// "new_table_options" should be set.
@@ -263,13 +356,17 @@ Status GetBlockBasedTableOptionsFromMap(
// @return Status::OK() on success. Otherwise, a non-ok status indicating
// error will be returned, and "new_table_options" will be set to
// "table_options".
+Status GetPlainTableOptionsFromMap(
+ const ConfigOptions& config_options, const PlainTableOptions& table_options,
+ const std::unordered_map& opts_map,
+ PlainTableOptions* new_table_options);
Status GetPlainTableOptionsFromMap(
const PlainTableOptions& table_options,
const std::unordered_map& opts_map,
PlainTableOptions* new_table_options, bool input_strings_escaped = false,
bool ignore_unknown_options = false);
-// Take a string representation of option names and values, apply them into the
+// Take a string representation of option names and values, apply them into the
// base_options, and return the new options as a result. The string has the
// following format:
// "write_buffer_size=1024;max_write_buffer_number=2"
@@ -277,22 +374,43 @@ Status GetPlainTableOptionsFromMap(
// BlockBasedTableOptions as part of the string for block-based table factory:
// "write_buffer_size=1024;block_based_table_factory={block_size=4k};"
// "max_write_buffer_num=2"
+//
+//
+// The GetColumnFamilyOptionsFromString(ConfigOptions, ...) should be used; the
+// alternative signature may be deprecated in a future release. The equivalent
+// functionality can be achieved by setting the corresponding options in
+// the ConfigOptions parameter.
+Status GetColumnFamilyOptionsFromString(const ConfigOptions& config_options,
+ const ColumnFamilyOptions& base_options,
+ const std::string& opts_str,
+ ColumnFamilyOptions* new_options);
Status GetColumnFamilyOptionsFromString(const ColumnFamilyOptions& base_options,
const std::string& opts_str,
ColumnFamilyOptions* new_options);
+Status GetDBOptionsFromString(const ConfigOptions& config_options,
+ const DBOptions& base_options,
+ const std::string& opts_str,
+ DBOptions* new_options);
+
Status GetDBOptionsFromString(const DBOptions& base_options,
const std::string& opts_str,
DBOptions* new_options);
+Status GetStringFromDBOptions(const ConfigOptions& config_options,
+ const DBOptions& db_options,
+ std::string* opts_str);
+
Status GetStringFromDBOptions(std::string* opts_str,
const DBOptions& db_options,
const std::string& delimiter = "; ");
+Status GetStringFromColumnFamilyOptions(const ConfigOptions& config_options,
+ const ColumnFamilyOptions& cf_options,
+ std::string* opts_str);
Status GetStringFromColumnFamilyOptions(std::string* opts_str,
const ColumnFamilyOptions& cf_options,
const std::string& delimiter = "; ");
-
Status GetStringFromCompressionType(std::string* compression_str,
CompressionType compression_type);
@@ -301,10 +419,18 @@ std::vector GetSupportedCompressions();
Status GetBlockBasedTableOptionsFromString(
const BlockBasedTableOptions& table_options, const std::string& opts_str,
BlockBasedTableOptions* new_table_options);
+Status GetBlockBasedTableOptionsFromString(
+ const ConfigOptions& config_options,
+ const BlockBasedTableOptions& table_options, const std::string& opts_str,
+ BlockBasedTableOptions* new_table_options);
Status GetPlainTableOptionsFromString(const PlainTableOptions& table_options,
const std::string& opts_str,
PlainTableOptions* new_table_options);
+Status GetPlainTableOptionsFromString(const ConfigOptions& config_options,
+ const PlainTableOptions& table_options,
+ const std::string& opts_str,
+ PlainTableOptions* new_table_options);
Status GetMemTableRepFactoryFromString(
const std::string& opts_str,
@@ -312,6 +438,9 @@ Status GetMemTableRepFactoryFromString(
Status GetOptionsFromString(const Options& base_options,
const std::string& opts_str, Options* new_options);
+Status GetOptionsFromString(const ConfigOptions& config_options,
+ const Options& base_options,
+ const std::string& opts_str, Options* new_options);
Status StringToMap(const std::string& opts_str,
std::unordered_map* opts_map);
diff --git a/include/rocksdb/table.h b/include/rocksdb/table.h
index 7963513e0..86a6e1d65 100644
--- a/include/rocksdb/table.h
+++ b/include/rocksdb/table.h
@@ -25,20 +25,24 @@
#include "rocksdb/cache.h"
#include "rocksdb/env.h"
#include "rocksdb/iterator.h"
-#include "rocksdb/options.h"
#include "rocksdb/status.h"
namespace ROCKSDB_NAMESPACE {
// -- Block-based Table
+class FilterPolicy;
class FlushBlockPolicyFactory;
class PersistentCache;
class RandomAccessFile;
struct TableReaderOptions;
struct TableBuilderOptions;
class TableBuilder;
+class TableFactory;
class TableReader;
class WritableFileWriter;
+struct ColumnFamilyOptions;
+struct ConfigOptions;
+struct DBOptions;
struct EnvOptions;
struct Options;
@@ -555,8 +559,8 @@ class TableFactory {
// RocksDB prints configurations at DB Open().
virtual std::string GetPrintableTableOptions() const = 0;
- virtual Status GetOptionString(std::string* /*opt_string*/,
- const std::string& /*delimiter*/) const {
+ virtual Status GetOptionString(const ConfigOptions& /*config_options*/,
+ std::string* /*opt_string*/) const {
return Status::NotSupported(
"The table factory doesn't implement GetOptionString().");
}
diff --git a/include/rocksdb/utilities/ldb_cmd.h b/include/rocksdb/utilities/ldb_cmd.h
index 94548b538..0942db221 100644
--- a/include/rocksdb/utilities/ldb_cmd.h
+++ b/include/rocksdb/utilities/ldb_cmd.h
@@ -9,6 +9,7 @@
#include
#include
+
#include
#include
#include
*
+ * @param cfgOpts The ConfigOptions to control how the string is processed.
* @param properties {@link java.util.Properties} instance.
*
* @return {@link org.rocksdb.DBOptions instance}
@@ -80,22 +81,40 @@ public class DBOptions extends RocksObject
* {@link java.util.Properties} instance is passed to the method call.
*/
public static DBOptions getDBOptionsFromProps(
- final Properties properties) {
- if (properties == null || properties.size() == 0) {
- throw new IllegalArgumentException(
- "Properties value must contain at least one value.");
- }
+ final ConfigOptions cfgOpts, final Properties properties) {
DBOptions dbOptions = null;
- StringBuilder stringBuilder = new StringBuilder();
- for (final String name : properties.stringPropertyNames()){
- stringBuilder.append(name);
- stringBuilder.append("=");
- stringBuilder.append(properties.getProperty(name));
- stringBuilder.append(";");
+ final String optionsString = Options.getOptionStringFromProps(properties);
+ final long handle = getDBOptionsFromProps(cfgOpts.nativeHandle_, optionsString);
+ if (handle != 0) {
+ dbOptions = new DBOptions(handle);
}
- long handle = getDBOptionsFromProps(
- stringBuilder.toString());
- if (handle != 0){
+ return dbOptions;
+ }
+
+ /**
+ * Method to get a options instance by using pre-configured
+ * property values. If one or many values are undefined in
+ * the context of RocksDB the method will return a null
+ * value.
+ *
+ * Note: Property keys can be derived from
+ * getter methods within the options class. Example: the method
+ * {@code allowMmapReads()} has a property key:
+ * {@code allow_mmap_reads}.
+ *
+ * @param properties {@link java.util.Properties} instance.
+ *
+ * @return {@link org.rocksdb.DBOptions instance}
+ * or null.
+ *
+ * @throws java.lang.IllegalArgumentException if null or empty
+ * {@link java.util.Properties} instance is passed to the method call.
+ */
+ public static DBOptions getDBOptionsFromProps(final Properties properties) {
+ DBOptions dbOptions = null;
+ final String optionsString = Options.getOptionStringFromProps(properties);
+ final long handle = getDBOptionsFromProps(optionsString);
+ if (handle != 0) {
dbOptions = new DBOptions(handle);
}
return dbOptions;
@@ -1175,8 +1194,8 @@ public class DBOptions extends RocksObject
super(nativeHandle);
}
- private static native long getDBOptionsFromProps(
- String optString);
+ private static native long getDBOptionsFromProps(long cfgHandle, String optString);
+ private static native long getDBOptionsFromProps(String optString);
private static native long newDBOptions();
private static native long copyDBOptions(final long handle);
diff --git a/java/src/main/java/org/rocksdb/Options.java b/java/src/main/java/org/rocksdb/Options.java
index 8a534f503..349e191ee 100644
--- a/java/src/main/java/org/rocksdb/Options.java
+++ b/java/src/main/java/org/rocksdb/Options.java
@@ -6,10 +6,7 @@
package org.rocksdb;
import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
+import java.util.*;
/**
* Options to control the behavior of a database. It will be used
@@ -27,6 +24,25 @@ public class Options extends RocksObject
RocksDB.loadLibrary();
}
+ /**
+ * Converts the input properties into a Options-style formatted string
+ * @param properties The set of properties to convert
+ * @return The Options-style representation of those properties.
+ */
+ public static String getOptionStringFromProps(final Properties properties) {
+ if (properties == null || properties.size() == 0) {
+ throw new IllegalArgumentException("Properties value must contain at least one value.");
+ }
+ StringBuilder stringBuilder = new StringBuilder();
+ for (final String name : properties.stringPropertyNames()) {
+ stringBuilder.append(name);
+ stringBuilder.append("=");
+ stringBuilder.append(properties.getProperty(name));
+ stringBuilder.append(";");
+ }
+ return stringBuilder.toString();
+ }
+
/**
* Construct options for opening a RocksDB.
*
diff --git a/java/src/main/java/org/rocksdb/OptionsUtil.java b/java/src/main/java/org/rocksdb/OptionsUtil.java
index f153556ba..4f575f216 100644
--- a/java/src/main/java/org/rocksdb/OptionsUtil.java
+++ b/java/src/main/java/org/rocksdb/OptionsUtil.java
@@ -59,7 +59,7 @@ public class OptionsUtil {
* @param cfDescs A list of {@link org.rocksdb.ColumnFamilyDescriptor}'s be
* returned.
* @param ignoreUnknownOptions this flag can be set to true if you want to
- * ignore options that are from a newer version of the db, esentially for
+ * ignore options that are from a newer version of the db, essentially for
* forward compatibility.
*
* @throws RocksDBException thrown if error happens in underlying
@@ -71,6 +71,25 @@ public class OptionsUtil {
dbPath, env.nativeHandle_, dbOptions.nativeHandle_, cfDescs, ignoreUnknownOptions);
}
+ /**
+ * Similar to LoadLatestOptions, this function constructs the DBOptions
+ * and ColumnFamilyDescriptors based on the specified RocksDB Options file.
+ * See LoadLatestOptions above.
+ *
+ * @param dbPath the path to the RocksDB.
+ * @param configOptions {@link org.rocksdb.ConfigOptions} instance.
+ * @param dbOptions {@link org.rocksdb.DBOptions} instance. This will be
+ * filled and returned.
+ * @param cfDescs A list of {@link org.rocksdb.ColumnFamilyDescriptor}'s be
+ * returned.
+ * @throws RocksDBException thrown if error happens in underlying
+ * native library.
+ */
+ public static void loadLatestOptions(ConfigOptions configOptions, String dbPath,
+ DBOptions dbOptions, List cfDescs) throws RocksDBException {
+ loadLatestOptions(configOptions.nativeHandle_, dbPath, dbOptions.nativeHandle_, cfDescs);
+ }
+
/**
* Similar to LoadLatestOptions, this function constructs the DBOptions
* and ColumnFamilyDescriptors based on the specified RocksDB Options file.
@@ -111,6 +130,26 @@ public class OptionsUtil {
optionsFileName, env.nativeHandle_, dbOptions.nativeHandle_, cfDescs, ignoreUnknownOptions);
}
+ /**
+ * Similar to LoadLatestOptions, this function constructs the DBOptions
+ * and ColumnFamilyDescriptors based on the specified RocksDB Options file.
+ * See LoadLatestOptions above.
+ *
+ * @param optionsFileName the RocksDB options file path.
+ * @param configOptions {@link org.rocksdb.ConfigOptions} instance.
+ * @param dbOptions {@link org.rocksdb.DBOptions} instance. This will be
+ * filled and returned.
+ * @param cfDescs A list of {@link org.rocksdb.ColumnFamilyDescriptor}'s be
+ * returned.
+ * @throws RocksDBException thrown if error happens in underlying
+ * native library.
+ */
+ public static void loadOptionsFromFile(ConfigOptions configOptions, String optionsFileName,
+ DBOptions dbOptions, List cfDescs) throws RocksDBException {
+ loadOptionsFromFile(
+ configOptions.nativeHandle_, optionsFileName, dbOptions.nativeHandle_, cfDescs);
+ }
+
/**
* Returns the latest options file name under the specified RocksDB path.
*
@@ -134,9 +173,13 @@ public class OptionsUtil {
// native methods
private native static void loadLatestOptions(String dbPath, long envHandle, long dbOptionsHandle,
List cfDescs, boolean ignoreUnknownOptions) throws RocksDBException;
+ private native static void loadLatestOptions(long cfgHandle, String dbPath, long dbOptionsHandle,
+ List cfDescs) throws RocksDBException;
private native static void loadOptionsFromFile(String optionsFileName, long envHandle,
long dbOptionsHandle, List cfDescs, boolean ignoreUnknownOptions)
throws RocksDBException;
+ private native static void loadOptionsFromFile(long cfgHandle, String optionsFileName,
+ long dbOptionsHandle, List cfDescs) throws RocksDBException;
private native static String getLatestOptionsFileName(String dbPath, long envHandle)
throws RocksDBException;
}
diff --git a/java/src/main/java/org/rocksdb/SanityLevel.java b/java/src/main/java/org/rocksdb/SanityLevel.java
new file mode 100644
index 000000000..775611c07
--- /dev/null
+++ b/java/src/main/java/org/rocksdb/SanityLevel.java
@@ -0,0 +1,42 @@
+package org.rocksdb;
+
+public enum SanityLevel {
+ NONE((byte) 0x0),
+ LOOSELY_COMPATIBLE((byte) 0x1),
+ EXACT_MATCH((byte) 0xFF);
+
+ private final byte value;
+
+ SanityLevel(final byte value) {
+ this.value = value;
+ }
+
+ /**
+ * Get the internal representation value.
+ *
+ * @return the internal representation value.
+ */
+ // TODO(AR) should be made package-private
+ public byte getValue() {
+ return value;
+ }
+
+ /**
+ * Get the SanityLevel from the internal representation value.
+ *
+ * @param value the internal representation value.
+ *
+ * @return the SanityLevel
+ *
+ * @throws IllegalArgumentException if the value does not match a
+ * SanityLevel
+ */
+ static SanityLevel fromValue(final byte value) throws IllegalArgumentException {
+ for (final SanityLevel level : SanityLevel.values()) {
+ if (level.value == value) {
+ return level;
+ }
+ }
+ throw new IllegalArgumentException("Unknown value for SanityLevel: " + value);
+ }
+}
diff --git a/java/src/test/java/org/rocksdb/ColumnFamilyOptionsTest.java b/java/src/test/java/org/rocksdb/ColumnFamilyOptionsTest.java
index af67f4663..df01ee60b 100644
--- a/java/src/test/java/org/rocksdb/ColumnFamilyOptionsTest.java
+++ b/java/src/test/java/org/rocksdb/ColumnFamilyOptionsTest.java
@@ -54,6 +54,27 @@ public class ColumnFamilyOptionsTest {
}
}
+ @Test
+ public void getColumnFamilyOptionsFromPropsWithIgnoreIllegalValue() {
+ // setup sample properties
+ final Properties properties = new Properties();
+ properties.put("tomato", "1024");
+ properties.put("burger", "2");
+ properties.put("write_buffer_size", "112");
+ properties.put("max_write_buffer_number", "13");
+
+ try (final ConfigOptions cfgOpts = new ConfigOptions().setIgnoreUnknownOptions(true);
+ final ColumnFamilyOptions opt =
+ ColumnFamilyOptions.getColumnFamilyOptionsFromProps(cfgOpts, properties)) {
+ // setup sample properties
+ assertThat(opt).isNotNull();
+ assertThat(String.valueOf(opt.writeBufferSize()))
+ .isEqualTo(properties.get("write_buffer_size"));
+ assertThat(String.valueOf(opt.maxWriteBufferNumber()))
+ .isEqualTo(properties.get("max_write_buffer_number"));
+ }
+ }
+
@Test
public void failColumnFamilyOptionsFromPropsWithIllegalValue() {
// setup sample properties
diff --git a/options/cf_options.cc b/options/cf_options.cc
index 65b198510..ec1267030 100644
--- a/options/cf_options.cc
+++ b/options/cf_options.cc
@@ -347,12 +347,13 @@ std::unordered_map
OptionTypeFlags::kMutable,
offsetof(struct MutableCFOptions, sample_for_compression)}}};
-Status ParseColumnFamilyOption(const std::string& name,
+Status ParseColumnFamilyOption(const ConfigOptions& config_options,
+ const std::string& name,
const std::string& org_value,
- ColumnFamilyOptions* new_options,
- bool input_strings_escaped) {
- const std::string& value =
- input_strings_escaped ? UnescapeOptionString(org_value) : org_value;
+ ColumnFamilyOptions* new_options) {
+ const std::string& value = config_options.input_strings_escaped
+ ? UnescapeOptionString(org_value)
+ : org_value;
try {
if (name == "block_based_table_factory") {
// Nested options
diff --git a/options/options_helper.cc b/options/options_helper.cc
index e86925d5a..879697eb9 100644
--- a/options/options_helper.cc
+++ b/options/options_helper.cc
@@ -10,6 +10,7 @@
#include
#include
+#include "options/options_type.h"
#include "rocksdb/cache.h"
#include "rocksdb/compaction_filter.h"
#include "rocksdb/convenience.h"
@@ -353,10 +354,14 @@ bool FIFOCompactionOptionsSpecialCase(const std::string& opt_str,
}
static bool SerializeStruct(
- const void* const options, std::string* value,
+ const void* const opt_ptr, std::string* value,
const std::unordered_map& type_info_map) {
+ ConfigOptions config_options;
+ config_options.delimiter = ";";
+
std::string opt_str;
- Status s = GetStringFromStruct(&opt_str, options, type_info_map, ";");
+ Status s =
+ GetStringFromStruct(config_options, opt_ptr, type_info_map, &opt_str);
if (!s.ok()) {
return false;
}
@@ -1010,9 +1015,9 @@ Status StringToMap(const std::string& opts_str,
}
Status GetStringFromStruct(
- std::string* opt_string, const void* const options,
+ const ConfigOptions& cfg_options, const void* const opt_ptr,
const std::unordered_map& type_info,
- const std::string& delimiter) {
+ std::string* opt_string) {
assert(opt_string);
opt_string->clear();
for (const auto iter : type_info) {
@@ -1023,12 +1028,12 @@ Status GetStringFromStruct(
continue;
}
const char* opt_address =
- reinterpret_cast(options) + opt_info.offset;
+ reinterpret_cast(opt_ptr) + opt_info.offset;
std::string value;
bool result =
SerializeSingleOptionHelper(opt_address, opt_info.type, &value);
if (result) {
- opt_string->append(iter.first + "=" + value + delimiter);
+ opt_string->append(iter.first + "=" + value + cfg_options.delimiter);
} else {
return Status::InvalidArgument("failed to serialize %s\n",
iter.first.c_str());
@@ -1040,15 +1045,32 @@ Status GetStringFromStruct(
Status GetStringFromDBOptions(std::string* opt_string,
const DBOptions& db_options,
const std::string& delimiter) {
- return GetStringFromStruct(opt_string, &db_options, db_options_type_info,
- delimiter);
+ ConfigOptions config_options;
+ config_options.delimiter = delimiter;
+ return GetStringFromDBOptions(config_options, db_options, opt_string);
+}
+
+Status GetStringFromDBOptions(const ConfigOptions& cfg_options,
+ const DBOptions& db_options,
+ std::string* opt_string) {
+ return GetStringFromStruct(cfg_options, &db_options, db_options_type_info,
+ opt_string);
}
Status GetStringFromColumnFamilyOptions(std::string* opt_string,
const ColumnFamilyOptions& cf_options,
const std::string& delimiter) {
- return GetStringFromStruct(opt_string, &cf_options, cf_options_type_info,
- delimiter);
+ ConfigOptions config_options;
+ config_options.delimiter = delimiter;
+ return GetStringFromColumnFamilyOptions(config_options, cf_options,
+ opt_string);
+}
+
+Status GetStringFromColumnFamilyOptions(const ConfigOptions& config_options,
+ const ColumnFamilyOptions& cf_options,
+ std::string* opt_string) {
+ return GetStringFromStruct(config_options, &cf_options, cf_options_type_info,
+ opt_string);
}
Status GetStringFromCompressionType(std::string* compression_str,
@@ -1073,12 +1095,13 @@ std::vector GetSupportedCompressions() {
return supported_compressions;
}
-Status ParseDBOption(const std::string& name,
- const std::string& org_value,
- DBOptions* new_options,
- bool input_strings_escaped = false) {
- const std::string& value =
- input_strings_escaped ? UnescapeOptionString(org_value) : org_value;
+static Status ParseDBOption(const ConfigOptions& config_options,
+ const std::string& name,
+ const std::string& org_value,
+ DBOptions* new_options) {
+ const std::string& value = config_options.input_strings_escaped
+ ? UnescapeOptionString(org_value)
+ : org_value;
try {
if (name == "rate_limiter_bytes_per_sec") {
new_options->rate_limiter.reset(
@@ -1121,37 +1144,28 @@ Status GetColumnFamilyOptionsFromMap(
const std::unordered_map& opts_map,
ColumnFamilyOptions* new_options, bool input_strings_escaped,
bool ignore_unknown_options) {
- return GetColumnFamilyOptionsFromMapInternal(
- base_options, opts_map, new_options, input_strings_escaped, nullptr,
- ignore_unknown_options);
+ ConfigOptions config_options;
+ config_options.ignore_unknown_options = ignore_unknown_options;
+ config_options.input_strings_escaped = input_strings_escaped;
+ return GetColumnFamilyOptionsFromMap(config_options, base_options, opts_map,
+ new_options);
}
-Status GetColumnFamilyOptionsFromMapInternal(
+Status GetColumnFamilyOptionsFromMap(
+ const ConfigOptions& config_options,
const ColumnFamilyOptions& base_options,
const std::unordered_map& opts_map,
- ColumnFamilyOptions* new_options, bool input_strings_escaped,
- std::vector* unsupported_options_names,
- bool ignore_unknown_options) {
+ ColumnFamilyOptions* new_options) {
assert(new_options);
*new_options = base_options;
- if (unsupported_options_names) {
- unsupported_options_names->clear();
- }
for (const auto& o : opts_map) {
- auto s = ParseColumnFamilyOption(o.first, o.second, new_options,
- input_strings_escaped);
+ auto s =
+ ParseColumnFamilyOption(config_options, o.first, o.second, new_options);
if (!s.ok()) {
if (s.IsNotSupported()) {
- // If the deserialization of the specified option is not supported
- // and an output vector of unsupported_options is provided, then
- // we log the name of the unsupported option and proceed.
- if (unsupported_options_names != nullptr) {
- unsupported_options_names->push_back(o.first);
- }
- // Note that we still return Status::OK in such case to maintain
- // the backward compatibility in the old public API defined in
- // rocksdb/convenience.h
- } else if (s.IsInvalidArgument() && ignore_unknown_options) {
+ continue;
+ } else if (s.IsInvalidArgument() &&
+ config_options.ignore_unknown_options) {
continue;
} else {
// Restore "new_options" to the default "base_options".
@@ -1167,13 +1181,25 @@ Status GetColumnFamilyOptionsFromString(
const ColumnFamilyOptions& base_options,
const std::string& opts_str,
ColumnFamilyOptions* new_options) {
+ ConfigOptions config_options;
+ config_options.input_strings_escaped = false;
+ config_options.ignore_unknown_options = false;
+ return GetColumnFamilyOptionsFromString(config_options, base_options,
+ opts_str, new_options);
+}
+
+Status GetColumnFamilyOptionsFromString(const ConfigOptions& config_options,
+ const ColumnFamilyOptions& base_options,
+ const std::string& opts_str,
+ ColumnFamilyOptions* new_options) {
std::unordered_map opts_map;
Status s = StringToMap(opts_str, &opts_map);
if (!s.ok()) {
*new_options = base_options;
return s;
}
- return GetColumnFamilyOptionsFromMap(base_options, opts_map, new_options);
+ return GetColumnFamilyOptionsFromMap(config_options, base_options, opts_map,
+ new_options);
}
Status GetDBOptionsFromMap(
@@ -1181,25 +1207,33 @@ Status GetDBOptionsFromMap(
const std::unordered_map& opts_map,
DBOptions* new_options, bool input_strings_escaped,
bool ignore_unknown_options) {
- return GetDBOptionsFromMapInternal(base_options, opts_map, new_options,
- input_strings_escaped, nullptr,
- ignore_unknown_options);
+ ConfigOptions config_options;
+ config_options.input_strings_escaped = input_strings_escaped;
+ config_options.ignore_unknown_options = ignore_unknown_options;
+ return GetDBOptionsFromMap(config_options, base_options, opts_map,
+ new_options);
+}
+
+Status GetDBOptionsFromMap(
+ const ConfigOptions& config_options, const DBOptions& base_options,
+ const std::unordered_map& opts_map,
+ DBOptions* new_options) {
+ return GetDBOptionsFromMapInternal(config_options, base_options, opts_map,
+ new_options, nullptr);
}
Status GetDBOptionsFromMapInternal(
- const DBOptions& base_options,
+ const ConfigOptions& config_options, const DBOptions& base_options,
const std::unordered_map& opts_map,
- DBOptions* new_options, bool input_strings_escaped,
- std::vector* unsupported_options_names,
- bool ignore_unknown_options) {
+ DBOptions* new_options,
+ std::vector* unsupported_options_names) {
assert(new_options);
*new_options = base_options;
if (unsupported_options_names) {
unsupported_options_names->clear();
}
for (const auto& o : opts_map) {
- auto s = ParseDBOption(o.first, o.second,
- new_options, input_strings_escaped);
+ auto s = ParseDBOption(config_options, o.first, o.second, new_options);
if (!s.ok()) {
if (s.IsNotSupported()) {
// If the deserialization of the specified option is not supported
@@ -1211,7 +1245,8 @@ Status GetDBOptionsFromMapInternal(
// Note that we still return Status::OK in such case to maintain
// the backward compatibility in the old public API defined in
// rocksdb/convenience.h
- } else if (s.IsInvalidArgument() && ignore_unknown_options) {
+ } else if (s.IsInvalidArgument() &&
+ config_options.ignore_unknown_options) {
continue;
} else {
// Restore "new_options" to the default "base_options".
@@ -1223,21 +1258,44 @@ Status GetDBOptionsFromMapInternal(
return Status::OK();
}
-Status GetDBOptionsFromString(
- const DBOptions& base_options,
- const std::string& opts_str,
- DBOptions* new_options) {
+Status GetDBOptionsFromString(const DBOptions& base_options,
+ const std::string& opts_str,
+ DBOptions* new_options) {
+ ConfigOptions config_options;
+ config_options.input_strings_escaped = false;
+ config_options.ignore_unknown_options = false;
+
+ return GetDBOptionsFromString(config_options, base_options, opts_str,
+ new_options);
+}
+
+Status GetDBOptionsFromString(const ConfigOptions& config_options,
+ const DBOptions& base_options,
+ const std::string& opts_str,
+ DBOptions* new_options) {
std::unordered_map opts_map;
Status s = StringToMap(opts_str, &opts_map);
if (!s.ok()) {
*new_options = base_options;
return s;
}
- return GetDBOptionsFromMap(base_options, opts_map, new_options);
+ return GetDBOptionsFromMap(config_options, base_options, opts_map,
+ new_options);
}
Status GetOptionsFromString(const Options& base_options,
const std::string& opts_str, Options* new_options) {
+ ConfigOptions config_options;
+ config_options.input_strings_escaped = false;
+ config_options.ignore_unknown_options = false;
+
+ return GetOptionsFromString(config_options, base_options, opts_str,
+ new_options);
+}
+
+Status GetOptionsFromString(const ConfigOptions& config_options,
+ const Options& base_options,
+ const std::string& opts_str, Options* new_options) {
std::unordered_map opts_map;
Status s = StringToMap(opts_str, &opts_map);
if (!s.ok()) {
@@ -1246,9 +1304,11 @@ Status GetOptionsFromString(const Options& base_options,
DBOptions new_db_options(base_options);
ColumnFamilyOptions new_cf_options(base_options);
for (const auto& o : opts_map) {
- if (ParseDBOption(o.first, o.second, &new_db_options).ok()) {
- } else if (ParseColumnFamilyOption(
- o.first, o.second, &new_cf_options).ok()) {
+ if (ParseDBOption(config_options, o.first, o.second, &new_db_options)
+ .ok()) {
+ } else if (ParseColumnFamilyOption(config_options, o.first, o.second,
+ &new_cf_options)
+ .ok()) {
} else {
return Status::InvalidArgument("Can't parse option " + o.first);
}
@@ -1261,13 +1321,22 @@ Status GetTableFactoryFromMap(
const std::string& factory_name,
const std::unordered_map& opt_map,
std::shared_ptr* table_factory, bool ignore_unknown_options) {
+ ConfigOptions
+ config_options; // Use default for escaped(true) and check (exact)
+ config_options.ignore_unknown_options = ignore_unknown_options;
+ return GetTableFactoryFromMap(config_options, factory_name, opt_map,
+ table_factory);
+}
+
+Status GetTableFactoryFromMap(
+ const ConfigOptions& config_options, const std::string& factory_name,
+ const std::unordered_map& opt_map,
+ std::shared_ptr* table_factory) {
Status s;
if (factory_name == BlockBasedTableFactory().Name()) {
BlockBasedTableOptions bbt_opt;
- s = GetBlockBasedTableOptionsFromMap(BlockBasedTableOptions(), opt_map,
- &bbt_opt,
- true, /* input_strings_escaped */
- ignore_unknown_options);
+ s = GetBlockBasedTableOptionsFromMap(
+ config_options, BlockBasedTableOptions(), opt_map, &bbt_opt);
if (!s.ok()) {
return s;
}
@@ -1275,9 +1344,8 @@ Status GetTableFactoryFromMap(
return Status::OK();
} else if (factory_name == PlainTableFactory().Name()) {
PlainTableOptions pt_opt;
- s = GetPlainTableOptionsFromMap(PlainTableOptions(), opt_map, &pt_opt,
- true, /* input_strings_escaped */
- ignore_unknown_options);
+ s = GetPlainTableOptionsFromMap(config_options, PlainTableOptions(),
+ opt_map, &pt_opt);
if (!s.ok()) {
return s;
}
diff --git a/options/options_helper.h b/options/options_helper.h
index 6feca8f3d..399ba1034 100644
--- a/options/options_helper.h
+++ b/options/options_helper.h
@@ -19,6 +19,7 @@
#include "rocksdb/universal_compaction.h"
namespace ROCKSDB_NAMESPACE {
+struct ConfigOptions;
DBOptions BuildDBOptions(const ImmutableDBOptions& immutable_db_options,
const MutableDBOptions& mutable_db_options);
@@ -28,11 +29,15 @@ ColumnFamilyOptions BuildColumnFamilyOptions(
const MutableCFOptions& mutable_cf_options);
#ifndef ROCKSDB_LITE
+Status GetStringFromStruct(
+ const ConfigOptions& config_options, const void* const opt_ptr,
+ const std::unordered_map& type_info,
+ std::string* opt_string);
-Status ParseColumnFamilyOption(const std::string& name,
+Status ParseColumnFamilyOption(const ConfigOptions& config_options,
+ const std::string& name,
const std::string& org_value,
- ColumnFamilyOptions* new_options,
- bool input_strings_escaped = false);
+ ColumnFamilyOptions* new_options);
Status GetMutableOptionsFromStrings(
const MutableCFOptions& base_options,
@@ -54,6 +59,11 @@ Status ParseCompressionOptions(const std::string& value,
const std::string& name,
CompressionOptions& compression_opts);
+Status GetTableFactoryFromMap(
+ const ConfigOptions& config_options, const std::string& factory_name,
+ const std::unordered_map& opt_map,
+ std::shared_ptr* table_factory);
+
// A helper function that converts "opt_address" to a std::string
// based on the specified OptionType.
bool SerializeSingleOptionHelper(const char* opt_address,
@@ -63,21 +73,10 @@ bool SerializeSingleOptionHelper(const char* opt_address,
// this further takes an optional output vector "unsupported_options_names",
// which stores the name of all the unsupported options specified in "opts_map".
Status GetDBOptionsFromMapInternal(
- const DBOptions& base_options,
+ const ConfigOptions& config_options, const DBOptions& base_options,
const std::unordered_map& opts_map,
- DBOptions* new_options, bool input_strings_escaped,
- std::vector* unsupported_options_names = nullptr,
- bool ignore_unknown_options = false);
-
-// In addition to its public version defined in rocksdb/convenience.h,
-// this further takes an optional output vector "unsupported_options_names",
-// which stores the name of all the unsupported options specified in "opts_map".
-Status GetColumnFamilyOptionsFromMapInternal(
- const ColumnFamilyOptions& base_options,
- const std::unordered_map& opts_map,
- ColumnFamilyOptions* new_options, bool input_strings_escaped,
- std::vector* unsupported_options_names = nullptr,
- bool ignore_unknown_options = false);
+ DBOptions* new_options,
+ std::vector* unsupported_options_names = nullptr);
bool ParseSliceTransform(
const std::string& value,
@@ -89,10 +88,6 @@ extern Status StringToMap(
extern bool ParseOptionHelper(char* opt_address, const OptionType& opt_type,
const std::string& value);
-Status GetStringFromStruct(
- std::string* opt_string, const void* const options,
- const std::unordered_map& type_info,
- const std::string& delimiter);
#endif // !ROCKSDB_LITE
struct OptionsHelper {
diff --git a/options/options_parser.cc b/options/options_parser.cc
index 0f8eecc9a..50f88db1b 100644
--- a/options/options_parser.cc
+++ b/options/options_parser.cc
@@ -16,14 +16,15 @@
#include "file/read_write_util.h"
#include "file/writable_file_writer.h"
#include "options/options_helper.h"
+#include "options/options_sanity_check.h"
+#include "port/port.h"
#include "rocksdb/convenience.h"
#include "rocksdb/db.h"
+#include "table/block_based/block_based_table_factory.h"
#include "test_util/sync_point.h"
#include "util/cast_util.h"
#include "util/string_util.h"
-#include "port/port.h"
-
namespace ROCKSDB_NAMESPACE {
static const std::string option_file_header =
@@ -38,6 +39,25 @@ Status PersistRocksDBOptions(const DBOptions& db_opt,
const std::vector& cf_names,
const std::vector& cf_opts,
const std::string& file_name, FileSystem* fs) {
+ ConfigOptions
+ config_options; // Use default for escaped(true) and check (exact)
+ config_options.delimiter = "\n ";
+ // If a readahead size was set in the input options, use it
+ if (db_opt.log_readahead_size > 0) {
+ config_options.file_readahead_size = db_opt.log_readahead_size;
+ }
+ return PersistRocksDBOptions(config_options, db_opt, cf_names, cf_opts,
+ file_name, fs);
+}
+
+Status PersistRocksDBOptions(const ConfigOptions& config_options_in,
+ const DBOptions& db_opt,
+ const std::vector& cf_names,
+ const std::vector& cf_opts,
+ const std::string& file_name, FileSystem* fs) {
+ ConfigOptions config_options = config_options_in;
+ config_options.delimiter = "\n "; // Override the default to nl
+
TEST_SYNC_POINT("PersistRocksDBOptions:start");
if (cf_names.size() != cf_opts.size()) {
return Status::InvalidArgument(
@@ -68,7 +88,7 @@ Status PersistRocksDBOptions(const DBOptions& db_opt,
writable->Append("\n[" + opt_section_titles[kOptionSectionDBOptions] +
"]\n ");
- s = GetStringFromDBOptions(&options_file_content, db_opt, "\n ");
+ s = GetStringFromDBOptions(config_options, db_opt, &options_file_content);
if (!s.ok()) {
writable->Close();
return s;
@@ -79,8 +99,8 @@ Status PersistRocksDBOptions(const DBOptions& db_opt,
// CFOptions section
writable->Append("\n[" + opt_section_titles[kOptionSectionCFOptions] +
" \"" + EscapeOptionString(cf_names[i]) + "\"]\n ");
- s = GetStringFromColumnFamilyOptions(&options_file_content, cf_opts[i],
- "\n ");
+ s = GetStringFromColumnFamilyOptions(config_options, cf_opts[i],
+ &options_file_content);
if (!s.ok()) {
writable->Close();
return s;
@@ -93,7 +113,7 @@ Status PersistRocksDBOptions(const DBOptions& db_opt,
tf->Name() + " \"" + EscapeOptionString(cf_names[i]) +
"\"]\n ");
options_file_content.clear();
- s = tf->GetOptionString(&options_file_content, "\n ");
+ s = tf->GetOptionString(config_options, &options_file_content);
if (!s.ok()) {
return s;
}
@@ -104,7 +124,7 @@ Status PersistRocksDBOptions(const DBOptions& db_opt,
writable->Close();
return RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
- db_opt, cf_names, cf_opts, file_name, fs);
+ config_options, db_opt, cf_names, cf_opts, file_name, fs);
}
RocksDBOptionsParser::RocksDBOptionsParser() { Reset(); }
@@ -205,7 +225,20 @@ Status RocksDBOptionsParser::ParseStatement(std::string* name,
Status RocksDBOptionsParser::Parse(const std::string& file_name, FileSystem* fs,
bool ignore_unknown_options,
size_t file_readahead_size) {
+ ConfigOptions
+ config_options; // Use default for escaped(true) and check (exact)
+ config_options.ignore_unknown_options = ignore_unknown_options;
+ if (file_readahead_size > 0) {
+ config_options.file_readahead_size = file_readahead_size;
+ }
+ return Parse(config_options, file_name, fs);
+}
+
+Status RocksDBOptionsParser::Parse(const ConfigOptions& config_options_in,
+ const std::string& file_name,
+ FileSystem* fs) {
Reset();
+ ConfigOptions config_options = config_options_in;
std::unique_ptr seq_file;
Status s = fs->NewSequentialFile(file_name, FileOptions(), &seq_file,
@@ -213,9 +246,8 @@ Status RocksDBOptionsParser::Parse(const std::string& file_name, FileSystem* fs,
if (!s.ok()) {
return s;
}
-
SequentialFileReader sf_reader(std::move(seq_file), file_name,
- file_readahead_size);
+ config_options.file_readahead_size);
OptionSection section = kOptionSectionUnknown;
std::string title;
@@ -235,7 +267,7 @@ Status RocksDBOptionsParser::Parse(const std::string& file_name, FileSystem* fs,
continue;
}
if (IsSection(line)) {
- s = EndSection(section, title, argument, opt_map, ignore_unknown_options);
+ s = EndSection(config_options, section, title, argument, opt_map);
opt_map.clear();
if (!s.ok()) {
return s;
@@ -243,10 +275,11 @@ Status RocksDBOptionsParser::Parse(const std::string& file_name, FileSystem* fs,
// If the option file is not generated by a higher minor version,
// there shouldn't be any unknown option.
- if (ignore_unknown_options && section == kOptionSectionVersion) {
+ if (config_options.ignore_unknown_options &&
+ section == kOptionSectionVersion) {
if (db_version[0] < ROCKSDB_MAJOR || (db_version[0] == ROCKSDB_MAJOR &&
db_version[1] <= ROCKSDB_MINOR)) {
- ignore_unknown_options = false;
+ config_options.ignore_unknown_options = false;
}
}
@@ -265,7 +298,7 @@ Status RocksDBOptionsParser::Parse(const std::string& file_name, FileSystem* fs,
}
}
- s = EndSection(section, title, argument, opt_map, ignore_unknown_options);
+ s = EndSection(config_options, section, title, argument, opt_map);
opt_map.clear();
if (!s.ok()) {
return s;
@@ -372,14 +405,12 @@ Status RocksDBOptionsParser::ParseVersionNumber(const std::string& ver_name,
}
Status RocksDBOptionsParser::EndSection(
- const OptionSection section, const std::string& section_title,
- const std::string& section_arg,
- const std::unordered_map& opt_map,
- bool ignore_unknown_options) {
+ const ConfigOptions& config_options, const OptionSection section,
+ const std::string& section_title, const std::string& section_arg,
+ const std::unordered_map& opt_map) {
Status s;
if (section == kOptionSectionDBOptions) {
- s = GetDBOptionsFromMap(DBOptions(), opt_map, &db_opt_, true,
- ignore_unknown_options);
+ s = GetDBOptionsFromMap(config_options, DBOptions(), opt_map, &db_opt_);
if (!s.ok()) {
return s;
}
@@ -390,9 +421,8 @@ Status RocksDBOptionsParser::EndSection(
assert(GetCFOptions(section_arg) == nullptr);
cf_names_.emplace_back(section_arg);
cf_opts_.emplace_back();
- s = GetColumnFamilyOptionsFromMap(ColumnFamilyOptions(), opt_map,
- &cf_opts_.back(), true,
- ignore_unknown_options);
+ s = GetColumnFamilyOptionsFromMap(config_options, ColumnFamilyOptions(),
+ opt_map, &cf_opts_.back());
if (!s.ok()) {
return s;
}
@@ -409,9 +439,10 @@ Status RocksDBOptionsParser::EndSection(
}
// Ignore error as table factory deserialization is optional
s = GetTableFactoryFromMap(
+ config_options,
section_title.substr(
opt_section_titles[kOptionSectionTableOptions].size()),
- opt_map, &(cf_opt->table_factory), ignore_unknown_options);
+ opt_map, &(cf_opt->table_factory));
if (!s.ok()) {
return s;
}
@@ -652,36 +683,27 @@ bool AreEqualOptions(
}
Status RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
- const DBOptions& db_opt, const std::vector& cf_names,
+ const ConfigOptions& config_options, const DBOptions& db_opt,
+ const std::vector& cf_names,
const std::vector& cf_opts,
- const std::string& file_name, FileSystem* fs,
- OptionsSanityCheckLevel sanity_check_level, bool ignore_unknown_options) {
- // We infer option file readhead size from log readahead size.
- // If it is not given, use 512KB.
- size_t file_readahead_size = db_opt.log_readahead_size;
- if (file_readahead_size == 0) {
- const size_t kDefaultOptionFileReadAheadSize = 512 * 1024;
- file_readahead_size = kDefaultOptionFileReadAheadSize;
- }
-
+ const std::string& file_name, FileSystem* fs) {
RocksDBOptionsParser parser;
- Status s =
- parser.Parse(file_name, fs, ignore_unknown_options, file_readahead_size);
+ Status s = parser.Parse(config_options, file_name, fs);
if (!s.ok()) {
return s;
}
// Verify DBOptions
- s = VerifyDBOptions(db_opt, *parser.db_opt(), parser.db_opt_map(),
- sanity_check_level);
+ s = VerifyDBOptions(config_options, db_opt, *parser.db_opt(),
+ parser.db_opt_map());
if (!s.ok()) {
return s;
}
// Verify ColumnFamily Name
if (cf_names.size() != parser.cf_names()->size()) {
- if (sanity_check_level >=
- OptionsSanityCheckLevel::kSanityLevelLooselyCompatible) {
+ if (config_options.sanity_level >=
+ ConfigOptions::kSanityLevelLooselyCompatible) {
return Status::InvalidArgument(
"[RocksDBOptionParser Error] The persisted options does not have "
"the same number of column family names as the db instance.");
@@ -703,8 +725,8 @@ Status RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
// Verify Column Family Options
if (cf_opts.size() != parser.cf_opts()->size()) {
- if (sanity_check_level >=
- OptionsSanityCheckLevel::kSanityLevelLooselyCompatible) {
+ if (config_options.sanity_level >=
+ ConfigOptions::kSanityLevelLooselyCompatible) {
return Status::InvalidArgument(
"[RocksDBOptionsParser Error]",
"The persisted options does not have the same number of "
@@ -717,14 +739,13 @@ Status RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
}
}
for (size_t i = 0; i < cf_opts.size(); ++i) {
- s = VerifyCFOptions(cf_opts[i], parser.cf_opts()->at(i),
- &(parser.cf_opt_maps()->at(i)), sanity_check_level);
+ s = VerifyCFOptions(config_options, cf_opts[i], parser.cf_opts()->at(i),
+ &(parser.cf_opt_maps()->at(i)));
if (!s.ok()) {
return s;
}
- s = VerifyTableFactory(cf_opts[i].table_factory.get(),
- parser.cf_opts()->at(i).table_factory.get(),
- sanity_check_level);
+ s = VerifyTableFactory(config_options, cf_opts[i].table_factory.get(),
+ parser.cf_opts()->at(i).table_factory.get());
if (!s.ok()) {
return s;
}
@@ -734,16 +755,16 @@ Status RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
}
Status RocksDBOptionsParser::VerifyDBOptions(
- const DBOptions& base_opt, const DBOptions& persisted_opt,
- const std::unordered_map* /*opt_map*/,
- OptionsSanityCheckLevel sanity_check_level) {
+ const ConfigOptions& config_options, const DBOptions& base_opt,
+ const DBOptions& persisted_opt,
+ const std::unordered_map* /*opt_map*/) {
for (const auto& pair : db_options_type_info) {
if (pair.second.IsDeprecated()) {
// We skip checking deprecated variables as they might
// contain random values since they might not be initialized
continue;
}
- if (DBOptionSanityCheckLevel(pair.first) <= sanity_check_level) {
+ if (DBOptionSanityCheckLevel(pair.first) <= config_options.sanity_level) {
if (!AreEqualOptions(reinterpret_cast(&base_opt),
reinterpret_cast(&persisted_opt),
pair.second, pair.first, nullptr)) {
@@ -771,17 +792,16 @@ Status RocksDBOptionsParser::VerifyDBOptions(
}
Status RocksDBOptionsParser::VerifyCFOptions(
- const ColumnFamilyOptions& base_opt,
+ const ConfigOptions& config_options, const ColumnFamilyOptions& base_opt,
const ColumnFamilyOptions& persisted_opt,
- const std::unordered_map* persisted_opt_map,
- OptionsSanityCheckLevel sanity_check_level) {
+ const std::unordered_map* persisted_opt_map) {
for (const auto& pair : cf_options_type_info) {
if (pair.second.IsDeprecated()) {
// We skip checking deprecated variables as they might
// contain random values since they might not be initialized
continue;
}
- if (CFOptionSanityCheckLevel(pair.first) <= sanity_check_level) {
+ if (CFOptionSanityCheckLevel(pair.first) <= config_options.sanity_level) {
if (!AreEqualOptions(reinterpret_cast(&base_opt),
reinterpret_cast(&persisted_opt),
pair.second, pair.first, persisted_opt_map)) {
@@ -809,10 +829,10 @@ Status RocksDBOptionsParser::VerifyCFOptions(
}
Status RocksDBOptionsParser::VerifyTableFactory(
- const TableFactory* base_tf, const TableFactory* file_tf,
- OptionsSanityCheckLevel sanity_check_level) {
+ const ConfigOptions& config_options, const TableFactory* base_tf,
+ const TableFactory* file_tf) {
if (base_tf && file_tf) {
- if (sanity_check_level > OptionsSanityCheckLevel::kSanityLevelNone &&
+ if (config_options.sanity_level > ConfigOptions::kSanityLevelNone &&
std::string(base_tf->Name()) != std::string(file_tf->Name())) {
return Status::Corruption(
"[RocksDBOptionsParser]: "
@@ -820,11 +840,11 @@ Status RocksDBOptionsParser::VerifyTableFactory(
}
if (base_tf->Name() == BlockBasedTableFactory::kName) {
return VerifyBlockBasedTableFactory(
+ config_options,
static_cast_with_check(base_tf),
static_cast_with_check(file_tf),
- sanity_check_level);
+ const TableFactory>(file_tf));
}
// TODO(yhchiang): add checks for other table factory types
} else {
diff --git a/options/options_parser.h b/options/options_parser.h
index a88b806f8..2ebdec6d7 100644
--- a/options/options_parser.h
+++ b/options/options_parser.h
@@ -9,14 +9,15 @@
#include
#include
-#include "options/options_sanity_check.h"
#include "rocksdb/env.h"
#include "rocksdb/options.h"
-#include "table/block_based/block_based_table_factory.h"
namespace ROCKSDB_NAMESPACE {
#ifndef ROCKSDB_LITE
+struct ConfigOptions;
+class OptionTypeInfo;
+class TableFactory;
#define ROCKSDB_OPTION_FILE_MAJOR 1
#define ROCKSDB_OPTION_FILE_MINOR 1
@@ -36,6 +37,11 @@ Status PersistRocksDBOptions(const DBOptions& db_opt,
const std::vector& cf_names,
const std::vector& cf_opts,
const std::string& file_name, FileSystem* fs);
+Status PersistRocksDBOptions(const ConfigOptions& config_options,
+ const DBOptions& db_opt,
+ const std::vector& cf_names,
+ const std::vector& cf_opts,
+ const std::string& file_name, FileSystem* fs);
extern bool AreEqualOptions(
const char* opt1, const char* opt2, const OptionTypeInfo& type_info,
@@ -52,6 +58,10 @@ class RocksDBOptionsParser {
// If 0 is given, a default value will be used.
Status Parse(const std::string& file_name, FileSystem* fs,
bool ignore_unknown_options, size_t file_readahead_size);
+
+ Status Parse(const ConfigOptions& config_options,
+ const std::string& file_name, FileSystem* fs);
+
static std::string TrimAndRemoveComment(const std::string& line,
const bool trim_only = false);
@@ -70,27 +80,26 @@ class RocksDBOptionsParser {
return GetCFOptionsImpl(name);
}
size_t NumColumnFamilies() { return cf_opts_.size(); }
-
static Status VerifyRocksDBOptionsFromFile(
- const DBOptions& db_opt, const std::vector& cf_names,
+ const ConfigOptions& config_options, const DBOptions& db_opt,
+ const std::vector& cf_names,
const std::vector& cf_opts,
- const std::string& file_name, FileSystem* fs,
- OptionsSanityCheckLevel sanity_check_level = kSanityLevelExactMatch,
- bool ignore_unknown_options = false);
-
+ const std::string& file_name, FileSystem* fs);
static Status VerifyDBOptions(
- const DBOptions& base_opt, const DBOptions& new_opt,
- const std::unordered_map* new_opt_map = nullptr,
- OptionsSanityCheckLevel sanity_check_level = kSanityLevelExactMatch);
+ const ConfigOptions& config_options, const DBOptions& base_opt,
+ const DBOptions& new_opt,
+ const std::unordered_map* new_opt_map =
+ nullptr);
static Status VerifyCFOptions(
- const ColumnFamilyOptions& base_opt, const ColumnFamilyOptions& new_opt,
- const std::unordered_map* new_opt_map = nullptr,
- OptionsSanityCheckLevel sanity_check_level = kSanityLevelExactMatch);
+ const ConfigOptions& config_options, const ColumnFamilyOptions& base_opt,
+ const ColumnFamilyOptions& new_opt,
+ const std::unordered_map* new_opt_map =
+ nullptr);
- static Status VerifyTableFactory(
- const TableFactory* base_tf, const TableFactory* file_tf,
- OptionsSanityCheckLevel sanity_check_level = kSanityLevelExactMatch);
+ static Status VerifyTableFactory(const ConfigOptions& config_options,
+ const TableFactory* base_tf,
+ const TableFactory* file_tf);
static Status ExtraParserCheck(const RocksDBOptionsParser& input_parser);
@@ -106,10 +115,10 @@ class RocksDBOptionsParser {
Status ParseStatement(std::string* name, std::string* value,
const std::string& line, const int line_num);
- Status EndSection(const OptionSection section, const std::string& title,
- const std::string& section_arg,
- const std::unordered_map& opt_map,
- bool ignore_unknown_options);
+ Status EndSection(
+ const ConfigOptions& config_options, const OptionSection section,
+ const std::string& title, const std::string& section_arg,
+ const std::unordered_map& opt_map);
Status ValidityCheck();
diff --git a/options/options_sanity_check.cc b/options/options_sanity_check.cc
index 47cff31cf..f73555d6a 100644
--- a/options/options_sanity_check.cc
+++ b/options/options_sanity_check.cc
@@ -10,25 +10,26 @@
namespace ROCKSDB_NAMESPACE {
namespace {
-OptionsSanityCheckLevel SanityCheckLevelHelper(
- const std::unordered_map& smap,
+ConfigOptions::SanityLevel SanityCheckLevelHelper(
+ const std::unordered_map& smap,
const std::string& name) {
auto iter = smap.find(name);
- return iter != smap.end() ? iter->second : kSanityLevelExactMatch;
+ return iter != smap.end() ? iter->second
+ : ConfigOptions::kSanityLevelExactMatch;
}
}
-OptionsSanityCheckLevel DBOptionSanityCheckLevel(
+ConfigOptions::SanityLevel DBOptionSanityCheckLevel(
const std::string& option_name) {
return SanityCheckLevelHelper(sanity_level_db_options, option_name);
}
-OptionsSanityCheckLevel CFOptionSanityCheckLevel(
+ConfigOptions::SanityLevel CFOptionSanityCheckLevel(
const std::string& option_name) {
return SanityCheckLevelHelper(sanity_level_cf_options, option_name);
}
-OptionsSanityCheckLevel BBTOptionSanityCheckLevel(
+ConfigOptions::SanityLevel BBTOptionSanityCheckLevel(
const std::string& option_name) {
return SanityCheckLevelHelper(sanity_level_bbt_options, option_name);
}
diff --git a/options/options_sanity_check.h b/options/options_sanity_check.h
index ab8ec6d4d..64f8d9c50 100644
--- a/options/options_sanity_check.h
+++ b/options/options_sanity_check.h
@@ -8,41 +8,36 @@
#include
#include
+#include "rocksdb/convenience.h"
#include "rocksdb/rocksdb_namespace.h"
#ifndef ROCKSDB_LITE
namespace ROCKSDB_NAMESPACE {
// This enum defines the RocksDB options sanity level.
-enum OptionsSanityCheckLevel : unsigned char {
- // Performs no sanity check at all.
- kSanityLevelNone = 0x00,
- // Performs minimum check to ensure the RocksDB instance can be
- // opened without corrupting / mis-interpreting the data.
- kSanityLevelLooselyCompatible = 0x01,
- // Perform exact match sanity check.
- kSanityLevelExactMatch = 0xFF,
-};
// The sanity check level for DB options
-static const std::unordered_map
- sanity_level_db_options {};
+static const std::unordered_map
+ sanity_level_db_options{};
// The sanity check level for column-family options
-static const std::unordered_map
+static const std::unordered_map
sanity_level_cf_options = {
- {"comparator", kSanityLevelLooselyCompatible},
- {"table_factory", kSanityLevelLooselyCompatible},
- {"merge_operator", kSanityLevelLooselyCompatible}};
+ {"comparator",
+ ConfigOptions::SanityLevel::kSanityLevelLooselyCompatible},
+ {"table_factory",
+ ConfigOptions::SanityLevel::kSanityLevelLooselyCompatible},
+ {"merge_operator",
+ ConfigOptions::SanityLevel::kSanityLevelLooselyCompatible}};
// The sanity check level for block-based table options
-static const std::unordered_map
- sanity_level_bbt_options {};
+static const std::unordered_map
+ sanity_level_bbt_options{};
-OptionsSanityCheckLevel DBOptionSanityCheckLevel(
+ConfigOptions::SanityLevel DBOptionSanityCheckLevel(
const std::string& options_name);
-OptionsSanityCheckLevel CFOptionSanityCheckLevel(
+ConfigOptions::SanityLevel CFOptionSanityCheckLevel(
const std::string& options_name);
-OptionsSanityCheckLevel BBTOptionSanityCheckLevel(
+ConfigOptions::SanityLevel BBTOptionSanityCheckLevel(
const std::string& options_name);
} // namespace ROCKSDB_NAMESPACE
diff --git a/options/options_test.cc b/options/options_test.cc
index b795daa0e..78f39fd3c 100644
--- a/options/options_test.cc
+++ b/options/options_test.cc
@@ -145,8 +145,16 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) {
ColumnFamilyOptions base_cf_opt;
ColumnFamilyOptions new_cf_opt;
- ASSERT_OK(GetColumnFamilyOptionsFromMap(
- base_cf_opt, cf_options_map, &new_cf_opt));
+ ConfigOptions exact, loose;
+ exact.input_strings_escaped = false;
+ exact.ignore_unknown_options = false;
+ exact.sanity_level = ConfigOptions::kSanityLevelExactMatch;
+ loose.sanity_level = ConfigOptions::kSanityLevelLooselyCompatible;
+
+ loose.input_strings_escaped = false;
+ loose.ignore_unknown_options = true;
+ ASSERT_OK(GetColumnFamilyOptionsFromMap(exact, base_cf_opt, cf_options_map,
+ &new_cf_opt));
ASSERT_EQ(new_cf_opt.write_buffer_size, 1U);
ASSERT_EQ(new_cf_opt.max_write_buffer_number, 2);
ASSERT_EQ(new_cf_opt.min_write_buffer_number_to_merge, 3);
@@ -214,33 +222,33 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) {
"rocksdb.FixedPrefix.31");
cf_options_map["write_buffer_size"] = "hello";
- ASSERT_NOK(GetColumnFamilyOptionsFromMap(
- base_cf_opt, cf_options_map, &new_cf_opt));
- ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt));
+ ASSERT_NOK(GetColumnFamilyOptionsFromMap(exact, base_cf_opt, cf_options_map,
+ &new_cf_opt));
+ ASSERT_OK(
+ RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt));
cf_options_map["write_buffer_size"] = "1";
- ASSERT_OK(GetColumnFamilyOptionsFromMap(
- base_cf_opt, cf_options_map, &new_cf_opt));
+ ASSERT_OK(GetColumnFamilyOptionsFromMap(exact, base_cf_opt, cf_options_map,
+ &new_cf_opt));
cf_options_map["unknown_option"] = "1";
- ASSERT_NOK(GetColumnFamilyOptionsFromMap(
- base_cf_opt, cf_options_map, &new_cf_opt));
- ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt));
+ ASSERT_NOK(GetColumnFamilyOptionsFromMap(exact, base_cf_opt, cf_options_map,
+ &new_cf_opt));
+ ASSERT_OK(
+ RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt));
- ASSERT_OK(GetColumnFamilyOptionsFromMap(base_cf_opt, cf_options_map,
- &new_cf_opt,
- false, /* input_strings_escaped */
- true /* ignore_unknown_options */));
- ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(
- base_cf_opt, new_cf_opt, nullptr, /* new_opt_map */
- kSanityLevelLooselyCompatible /* from CheckOptionsCompatibility*/));
- ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(
- base_cf_opt, new_cf_opt, nullptr, /* new_opt_map */
- kSanityLevelExactMatch /* default for VerifyCFOptions */));
+ // ignore_unknown_options=true;input_strings_escaped=false
+ ASSERT_OK(GetColumnFamilyOptionsFromMap(loose, base_cf_opt, cf_options_map,
+ &new_cf_opt));
+ ASSERT_OK(
+ RocksDBOptionsParser::VerifyCFOptions(loose, base_cf_opt, new_cf_opt));
+ ASSERT_NOK(
+ RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt));
DBOptions base_db_opt;
DBOptions new_db_opt;
- ASSERT_OK(GetDBOptionsFromMap(base_db_opt, db_options_map, &new_db_opt));
+ ASSERT_OK(
+ GetDBOptionsFromMap(exact, base_db_opt, db_options_map, &new_db_opt));
ASSERT_EQ(new_db_opt.create_if_missing, false);
ASSERT_EQ(new_db_opt.create_missing_column_families, true);
ASSERT_EQ(new_db_opt.error_if_exists, false);
@@ -284,26 +292,26 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) {
ASSERT_EQ(new_db_opt.strict_bytes_per_sync, true);
db_options_map["max_open_files"] = "hello";
- ASSERT_NOK(GetDBOptionsFromMap(base_db_opt, db_options_map, &new_db_opt));
- ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(base_db_opt, new_db_opt));
- ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(
- base_db_opt, new_db_opt, nullptr, /* new_opt_map */
- kSanityLevelLooselyCompatible /* from CheckOptionsCompatibility */));
+ ASSERT_NOK(
+ GetDBOptionsFromMap(exact, base_db_opt, db_options_map, &new_db_opt));
+ ASSERT_OK(
+ RocksDBOptionsParser::VerifyDBOptions(exact, base_db_opt, new_db_opt));
+ ASSERT_OK(
+ RocksDBOptionsParser::VerifyDBOptions(loose, base_db_opt, new_db_opt));
// unknow options should fail parsing without ignore_unknown_options = true
db_options_map["unknown_db_option"] = "1";
- ASSERT_NOK(GetDBOptionsFromMap(base_db_opt, db_options_map, &new_db_opt));
- ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(base_db_opt, new_db_opt));
+ ASSERT_NOK(
+ GetDBOptionsFromMap(exact, base_db_opt, db_options_map, &new_db_opt));
+ ASSERT_OK(
+ RocksDBOptionsParser::VerifyDBOptions(exact, base_db_opt, new_db_opt));
- ASSERT_OK(GetDBOptionsFromMap(base_db_opt, db_options_map, &new_db_opt,
- false, /* input_strings_escaped */
- true /* ignore_unknown_options */));
- ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(
- base_db_opt, new_db_opt, nullptr, /* new_opt_map */
- kSanityLevelLooselyCompatible /* from CheckOptionsCompatibility */));
- ASSERT_NOK(RocksDBOptionsParser::VerifyDBOptions(
- base_db_opt, new_db_opt, nullptr, /* new_opt_mat */
- kSanityLevelExactMatch /* default for VerifyDBOptions */));
+ ASSERT_OK(
+ GetDBOptionsFromMap(loose, base_db_opt, db_options_map, &new_db_opt));
+ ASSERT_OK(
+ RocksDBOptionsParser::VerifyDBOptions(loose, base_db_opt, new_db_opt));
+ ASSERT_NOK(
+ RocksDBOptionsParser::VerifyDBOptions(exact, base_db_opt, new_db_opt));
}
#endif // !ROCKSDB_LITE
@@ -312,35 +320,42 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) {
TEST_F(OptionsTest, GetColumnFamilyOptionsFromStringTest) {
ColumnFamilyOptions base_cf_opt;
ColumnFamilyOptions new_cf_opt;
+ ConfigOptions config_options;
+ config_options.input_strings_escaped = false;
+ config_options.ignore_unknown_options = false;
+
base_cf_opt.table_factory.reset();
- ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt, "", &new_cf_opt));
- ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
- "write_buffer_size=5", &new_cf_opt));
+ ASSERT_OK(GetColumnFamilyOptionsFromString(config_options, base_cf_opt, "",
+ &new_cf_opt));
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ config_options, base_cf_opt, "write_buffer_size=5", &new_cf_opt));
ASSERT_EQ(new_cf_opt.write_buffer_size, 5U);
ASSERT_TRUE(new_cf_opt.table_factory == nullptr);
- ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
- "write_buffer_size=6;", &new_cf_opt));
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ config_options, base_cf_opt, "write_buffer_size=6;", &new_cf_opt));
ASSERT_EQ(new_cf_opt.write_buffer_size, 6U);
- ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
- " write_buffer_size = 7 ", &new_cf_opt));
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ config_options, base_cf_opt, " write_buffer_size = 7 ", &new_cf_opt));
ASSERT_EQ(new_cf_opt.write_buffer_size, 7U);
- ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
- " write_buffer_size = 8 ; ", &new_cf_opt));
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ config_options, base_cf_opt, " write_buffer_size = 8 ; ", &new_cf_opt));
ASSERT_EQ(new_cf_opt.write_buffer_size, 8U);
- ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
- "write_buffer_size=9;max_write_buffer_number=10", &new_cf_opt));
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ config_options, base_cf_opt,
+ "write_buffer_size=9;max_write_buffer_number=10", &new_cf_opt));
ASSERT_EQ(new_cf_opt.write_buffer_size, 9U);
ASSERT_EQ(new_cf_opt.max_write_buffer_number, 10);
- ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
- "write_buffer_size=11; max_write_buffer_number = 12 ;",
- &new_cf_opt));
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ config_options, base_cf_opt,
+ "write_buffer_size=11; max_write_buffer_number = 12 ;", &new_cf_opt));
ASSERT_EQ(new_cf_opt.write_buffer_size, 11U);
ASSERT_EQ(new_cf_opt.max_write_buffer_number, 12);
// Wrong name "max_write_buffer_number_"
- ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
- "write_buffer_size=13;max_write_buffer_number_=14;",
- &new_cf_opt));
- ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt));
+ ASSERT_NOK(GetColumnFamilyOptionsFromString(
+ config_options, base_cf_opt,
+ "write_buffer_size=13;max_write_buffer_number_=14;", &new_cf_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(config_options, base_cf_opt,
+ new_cf_opt));
// Comparator from object registry
std::string kCompName = "reverse_comp";
@@ -350,8 +365,9 @@ TEST_F(OptionsTest, GetColumnFamilyOptionsFromStringTest) {
std::unique_ptr* /*guard*/,
std::string* /* errmsg */) { return ReverseBytewiseComparator(); });
- ASSERT_OK(GetColumnFamilyOptionsFromString(
- base_cf_opt, "comparator=" + kCompName + ";", &new_cf_opt));
+ ASSERT_OK(GetColumnFamilyOptionsFromString(config_options, base_cf_opt,
+ "comparator=" + kCompName + ";",
+ &new_cf_opt));
ASSERT_EQ(new_cf_opt.comparator, ReverseBytewiseComparator());
// MergeOperator from object registry
@@ -365,24 +381,30 @@ TEST_F(OptionsTest, GetColumnFamilyOptionsFromStringTest) {
return guard->get();
});
- ASSERT_OK(GetColumnFamilyOptionsFromString(
- base_cf_opt, "merge_operator=" + kMoName + ";", &new_cf_opt));
+ ASSERT_OK(GetColumnFamilyOptionsFromString(config_options, base_cf_opt,
+ "merge_operator=" + kMoName + ";",
+ &new_cf_opt));
ASSERT_EQ(kMoName, std::string(new_cf_opt.merge_operator->Name()));
// Wrong key/value pair
- ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
- "write_buffer_size=13;max_write_buffer_number;", &new_cf_opt));
- ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt));
+ ASSERT_NOK(GetColumnFamilyOptionsFromString(
+ config_options, base_cf_opt,
+ "write_buffer_size=13;max_write_buffer_number;", &new_cf_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(config_options, base_cf_opt,
+ new_cf_opt));
// Error Paring value
- ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
- "write_buffer_size=13;max_write_buffer_number=;", &new_cf_opt));
- ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt));
+ ASSERT_NOK(GetColumnFamilyOptionsFromString(
+ config_options, base_cf_opt,
+ "write_buffer_size=13;max_write_buffer_number=;", &new_cf_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(config_options, base_cf_opt,
+ new_cf_opt));
// Missing option name
- ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
- "write_buffer_size=13; =100;", &new_cf_opt));
- ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt));
+ ASSERT_NOK(GetColumnFamilyOptionsFromString(
+ config_options, base_cf_opt, "write_buffer_size=13; =100;", &new_cf_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(config_options, base_cf_opt,
+ new_cf_opt));
const uint64_t kilo = 1024UL;
const uint64_t mega = 1024 * kilo;
@@ -391,17 +413,17 @@ TEST_F(OptionsTest, GetColumnFamilyOptionsFromStringTest) {
// Units (k)
ASSERT_OK(GetColumnFamilyOptionsFromString(
- base_cf_opt, "max_write_buffer_number=15K", &new_cf_opt));
+ config_options, base_cf_opt, "max_write_buffer_number=15K", &new_cf_opt));
ASSERT_EQ(new_cf_opt.max_write_buffer_number, 15 * kilo);
// Units (m)
- ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
- "max_write_buffer_number=16m;inplace_update_num_locks=17M",
- &new_cf_opt));
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ config_options, base_cf_opt,
+ "max_write_buffer_number=16m;inplace_update_num_locks=17M", &new_cf_opt));
ASSERT_EQ(new_cf_opt.max_write_buffer_number, 16 * mega);
ASSERT_EQ(new_cf_opt.inplace_update_num_locks, 17u * mega);
// Units (g)
ASSERT_OK(GetColumnFamilyOptionsFromString(
- base_cf_opt,
+ config_options, base_cf_opt,
"write_buffer_size=18g;prefix_extractor=capped:8;"
"arena_block_size=19G",
&new_cf_opt));
@@ -413,113 +435,212 @@ TEST_F(OptionsTest, GetColumnFamilyOptionsFromStringTest) {
ASSERT_EQ(prefix_name, "rocksdb.CappedPrefix.8");
// Units (t)
- ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
- "write_buffer_size=20t;arena_block_size=21T", &new_cf_opt));
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ config_options, base_cf_opt, "write_buffer_size=20t;arena_block_size=21T",
+ &new_cf_opt));
ASSERT_EQ(new_cf_opt.write_buffer_size, 20 * tera);
ASSERT_EQ(new_cf_opt.arena_block_size, 21 * tera);
// Nested block based table options
// Empty
- ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
- "write_buffer_size=10;max_write_buffer_number=16;"
- "block_based_table_factory={};arena_block_size=1024",
- &new_cf_opt));
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ config_options, base_cf_opt,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "block_based_table_factory={};arena_block_size=1024",
+ &new_cf_opt));
ASSERT_TRUE(new_cf_opt.table_factory != nullptr);
// Non-empty
- ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
- "write_buffer_size=10;max_write_buffer_number=16;"
- "block_based_table_factory={block_cache=1M;block_size=4;};"
- "arena_block_size=1024",
- &new_cf_opt));
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ config_options, base_cf_opt,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "block_based_table_factory={block_cache=1M;block_size=4;};"
+ "arena_block_size=1024",
+ &new_cf_opt));
ASSERT_TRUE(new_cf_opt.table_factory != nullptr);
// Last one
- ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
- "write_buffer_size=10;max_write_buffer_number=16;"
- "block_based_table_factory={block_cache=1M;block_size=4;}",
- &new_cf_opt));
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ config_options, base_cf_opt,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "block_based_table_factory={block_cache=1M;block_size=4;}",
+ &new_cf_opt));
ASSERT_TRUE(new_cf_opt.table_factory != nullptr);
// Mismatch curly braces
- ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
- "write_buffer_size=10;max_write_buffer_number=16;"
- "block_based_table_factory={{{block_size=4;};"
- "arena_block_size=1024",
- &new_cf_opt));
- ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt));
+ ASSERT_NOK(GetColumnFamilyOptionsFromString(
+ config_options, base_cf_opt,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "block_based_table_factory={{{block_size=4;};"
+ "arena_block_size=1024",
+ &new_cf_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(config_options, base_cf_opt,
+ new_cf_opt));
// Unexpected chars after closing curly brace
- ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
- "write_buffer_size=10;max_write_buffer_number=16;"
- "block_based_table_factory={block_size=4;}};"
- "arena_block_size=1024",
- &new_cf_opt));
- ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt));
+ ASSERT_NOK(GetColumnFamilyOptionsFromString(
+ config_options, base_cf_opt,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "block_based_table_factory={block_size=4;}};"
+ "arena_block_size=1024",
+ &new_cf_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(config_options, base_cf_opt,
+ new_cf_opt));
- ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
- "write_buffer_size=10;max_write_buffer_number=16;"
- "block_based_table_factory={block_size=4;}xdfa;"
- "arena_block_size=1024",
- &new_cf_opt));
- ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt));
+ ASSERT_NOK(GetColumnFamilyOptionsFromString(
+ config_options, base_cf_opt,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "block_based_table_factory={block_size=4;}xdfa;"
+ "arena_block_size=1024",
+ &new_cf_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(config_options, base_cf_opt,
+ new_cf_opt));
- ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
- "write_buffer_size=10;max_write_buffer_number=16;"
- "block_based_table_factory={block_size=4;}xdfa",
- &new_cf_opt));
- ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt));
+ ASSERT_NOK(GetColumnFamilyOptionsFromString(
+ config_options, base_cf_opt,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "block_based_table_factory={block_size=4;}xdfa",
+ &new_cf_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(config_options, base_cf_opt,
+ new_cf_opt));
// Invalid block based table option
- ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
- "write_buffer_size=10;max_write_buffer_number=16;"
- "block_based_table_factory={xx_block_size=4;}",
- &new_cf_opt));
- ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt));
+ ASSERT_NOK(GetColumnFamilyOptionsFromString(
+ config_options, base_cf_opt,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "block_based_table_factory={xx_block_size=4;}",
+ &new_cf_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(config_options, base_cf_opt,
+ new_cf_opt));
- ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
- "optimize_filters_for_hits=true",
- &new_cf_opt));
- ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
- "optimize_filters_for_hits=false",
- &new_cf_opt));
+ ASSERT_OK(GetColumnFamilyOptionsFromString(config_options, base_cf_opt,
+ "optimize_filters_for_hits=true",
+ &new_cf_opt));
+ ASSERT_OK(GetColumnFamilyOptionsFromString(config_options, base_cf_opt,
+ "optimize_filters_for_hits=false",
+ &new_cf_opt));
- ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
- "optimize_filters_for_hits=junk",
- &new_cf_opt));
- ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opt, new_cf_opt));
+ ASSERT_NOK(GetColumnFamilyOptionsFromString(config_options, base_cf_opt,
+ "optimize_filters_for_hits=junk",
+ &new_cf_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(config_options, base_cf_opt,
+ new_cf_opt));
// Nested plain table options
// Empty
- ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
- "write_buffer_size=10;max_write_buffer_number=16;"
- "plain_table_factory={};arena_block_size=1024",
- &new_cf_opt));
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ config_options, base_cf_opt,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "plain_table_factory={};arena_block_size=1024",
+ &new_cf_opt));
ASSERT_TRUE(new_cf_opt.table_factory != nullptr);
ASSERT_EQ(std::string(new_cf_opt.table_factory->Name()), "PlainTable");
// Non-empty
- ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
- "write_buffer_size=10;max_write_buffer_number=16;"
- "plain_table_factory={user_key_len=66;bloom_bits_per_key=20;};"
- "arena_block_size=1024",
- &new_cf_opt));
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ config_options, base_cf_opt,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "plain_table_factory={user_key_len=66;bloom_bits_per_key=20;};"
+ "arena_block_size=1024",
+ &new_cf_opt));
ASSERT_TRUE(new_cf_opt.table_factory != nullptr);
ASSERT_EQ(std::string(new_cf_opt.table_factory->Name()), "PlainTable");
// memtable factory
- ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
- "write_buffer_size=10;max_write_buffer_number=16;"
- "memtable=skip_list:10;arena_block_size=1024",
- &new_cf_opt));
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ config_options, base_cf_opt,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "memtable=skip_list:10;arena_block_size=1024",
+ &new_cf_opt));
ASSERT_TRUE(new_cf_opt.memtable_factory != nullptr);
ASSERT_EQ(std::string(new_cf_opt.memtable_factory->Name()), "SkipListFactory");
}
+
+TEST_F(OptionsTest, OldInterfaceTest) {
+ ColumnFamilyOptions base_cf_opt;
+ ColumnFamilyOptions new_cf_opt;
+ ConfigOptions exact;
+
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ base_cf_opt,
+ "write_buffer_size=18;prefix_extractor=capped:8;"
+ "arena_block_size=19",
+ &new_cf_opt));
+
+ ASSERT_EQ(new_cf_opt.write_buffer_size, 18);
+ ASSERT_EQ(new_cf_opt.arena_block_size, 19);
+ ASSERT_TRUE(new_cf_opt.prefix_extractor.get() != nullptr);
+
+ // And with a bad option
+ ASSERT_NOK(GetColumnFamilyOptionsFromString(
+ base_cf_opt,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "block_based_table_factory={xx_block_size=4;}",
+ &new_cf_opt));
+ ASSERT_OK(
+ RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt));
+
+ std::unordered_map cf_options_map = {
+ {"write_buffer_size", "1"},
+ {"max_write_buffer_number", "2"},
+ {"min_write_buffer_number_to_merge", "3"},
+ };
+ ASSERT_OK(
+ GetColumnFamilyOptionsFromMap(base_cf_opt, cf_options_map, &new_cf_opt));
+ cf_options_map["unknown_option"] = "1";
+ ASSERT_NOK(
+ GetColumnFamilyOptionsFromMap(base_cf_opt, cf_options_map, &new_cf_opt));
+ ASSERT_OK(
+ RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt));
+ ASSERT_OK(GetColumnFamilyOptionsFromMap(base_cf_opt, cf_options_map,
+ &new_cf_opt, true, true));
+
+ DBOptions base_db_opt;
+ DBOptions new_db_opt;
+ std::unordered_map db_options_map = {
+ {"create_if_missing", "false"},
+ {"create_missing_column_families", "true"},
+ {"error_if_exists", "false"},
+ {"paranoid_checks", "true"},
+ {"max_open_files", "32"},
+ };
+ ASSERT_OK(GetDBOptionsFromMap(base_db_opt, db_options_map, &new_db_opt));
+ ASSERT_EQ(new_db_opt.create_if_missing, false);
+ ASSERT_EQ(new_db_opt.create_missing_column_families, true);
+ ASSERT_EQ(new_db_opt.error_if_exists, false);
+ ASSERT_EQ(new_db_opt.paranoid_checks, true);
+ ASSERT_EQ(new_db_opt.max_open_files, 32);
+ db_options_map["unknown_option"] = "1";
+ ASSERT_NOK(GetDBOptionsFromMap(base_db_opt, db_options_map, &new_db_opt));
+ ASSERT_OK(
+ RocksDBOptionsParser::VerifyDBOptions(exact, base_db_opt, new_db_opt));
+ ASSERT_OK(GetDBOptionsFromMap(base_db_opt, db_options_map, &new_db_opt, true,
+ true));
+ ASSERT_OK(GetDBOptionsFromString(
+ base_db_opt,
+ "create_if_missing=false;error_if_exists=false;max_open_files=42;",
+ &new_db_opt));
+ ASSERT_EQ(new_db_opt.create_if_missing, false);
+ ASSERT_EQ(new_db_opt.error_if_exists, false);
+ ASSERT_EQ(new_db_opt.max_open_files, 42);
+ ASSERT_NOK(GetDBOptionsFromString(
+ base_db_opt,
+ "create_if_missing=false;error_if_exists=false;max_open_files=42;"
+ "unknown_option=1;",
+ &new_db_opt));
+ ASSERT_OK(
+ RocksDBOptionsParser::VerifyDBOptions(exact, base_db_opt, new_db_opt));
+}
+
#endif // !ROCKSDB_LITE
#ifndef ROCKSDB_LITE // GetBlockBasedTableOptionsFromString is not supported
TEST_F(OptionsTest, GetBlockBasedTableOptionsFromString) {
BlockBasedTableOptions table_opt;
BlockBasedTableOptions new_opt;
+ ConfigOptions config_options;
+ config_options.input_strings_escaped = false;
+ config_options.ignore_unknown_options = false;
+
// make sure default values are overwritten by something else
ASSERT_OK(GetBlockBasedTableOptionsFromString(
- table_opt,
+ config_options, table_opt,
"cache_index_and_filter_blocks=1;index_type=kHashSearch;"
"checksum=kxxHash;hash_index_allow_collision=1;no_block_cache=1;"
"block_cache=1M;block_cache_compressed=1k;block_size=1024;"
@@ -548,56 +669,60 @@ TEST_F(OptionsTest, GetBlockBasedTableOptionsFromString) {
EXPECT_EQ(bfp.GetWholeBitsPerKey(), 5);
// unknown option
- ASSERT_NOK(GetBlockBasedTableOptionsFromString(table_opt,
- "cache_index_and_filter_blocks=1;index_type=kBinarySearch;"
- "bad_option=1",
- &new_opt));
+ ASSERT_NOK(GetBlockBasedTableOptionsFromString(
+ config_options, table_opt,
+ "cache_index_and_filter_blocks=1;index_type=kBinarySearch;"
+ "bad_option=1",
+ &new_opt));
ASSERT_EQ(static_cast(table_opt.cache_index_and_filter_blocks),
new_opt.cache_index_and_filter_blocks);
ASSERT_EQ(table_opt.index_type, new_opt.index_type);
// unrecognized index type
- ASSERT_NOK(GetBlockBasedTableOptionsFromString(table_opt,
- "cache_index_and_filter_blocks=1;index_type=kBinarySearchXX",
- &new_opt));
+ ASSERT_NOK(GetBlockBasedTableOptionsFromString(
+ config_options, table_opt,
+ "cache_index_and_filter_blocks=1;index_type=kBinarySearchXX", &new_opt));
ASSERT_EQ(table_opt.cache_index_and_filter_blocks,
new_opt.cache_index_and_filter_blocks);
ASSERT_EQ(table_opt.index_type, new_opt.index_type);
// unrecognized checksum type
- ASSERT_NOK(GetBlockBasedTableOptionsFromString(table_opt,
- "cache_index_and_filter_blocks=1;checksum=kxxHashXX",
- &new_opt));
+ ASSERT_NOK(GetBlockBasedTableOptionsFromString(
+ config_options, table_opt,
+ "cache_index_and_filter_blocks=1;checksum=kxxHashXX", &new_opt));
ASSERT_EQ(table_opt.cache_index_and_filter_blocks,
new_opt.cache_index_and_filter_blocks);
ASSERT_EQ(table_opt.index_type, new_opt.index_type);
// unrecognized filter policy name
- ASSERT_NOK(GetBlockBasedTableOptionsFromString(table_opt,
- "cache_index_and_filter_blocks=1;"
- "filter_policy=bloomfilterxx:4:true",
- &new_opt));
+ ASSERT_NOK(
+ GetBlockBasedTableOptionsFromString(config_options, table_opt,
+ "cache_index_and_filter_blocks=1;"
+ "filter_policy=bloomfilterxx:4:true",
+ &new_opt));
ASSERT_EQ(table_opt.cache_index_and_filter_blocks,
new_opt.cache_index_and_filter_blocks);
ASSERT_EQ(table_opt.filter_policy, new_opt.filter_policy);
// unrecognized filter policy config
- ASSERT_NOK(GetBlockBasedTableOptionsFromString(table_opt,
- "cache_index_and_filter_blocks=1;"
- "filter_policy=bloomfilter:4",
- &new_opt));
+ ASSERT_NOK(
+ GetBlockBasedTableOptionsFromString(config_options, table_opt,
+ "cache_index_and_filter_blocks=1;"
+ "filter_policy=bloomfilter:4",
+ &new_opt));
ASSERT_EQ(table_opt.cache_index_and_filter_blocks,
new_opt.cache_index_and_filter_blocks);
ASSERT_EQ(table_opt.filter_policy, new_opt.filter_policy);
// Check block cache options are overwritten when specified
// in new format as a struct.
- ASSERT_OK(GetBlockBasedTableOptionsFromString(table_opt,
- "block_cache={capacity=1M;num_shard_bits=4;"
- "strict_capacity_limit=true;high_pri_pool_ratio=0.5;};"
- "block_cache_compressed={capacity=1M;num_shard_bits=4;"
- "strict_capacity_limit=true;high_pri_pool_ratio=0.5;}",
- &new_opt));
+ ASSERT_OK(GetBlockBasedTableOptionsFromString(
+ config_options, table_opt,
+ "block_cache={capacity=1M;num_shard_bits=4;"
+ "strict_capacity_limit=true;high_pri_pool_ratio=0.5;};"
+ "block_cache_compressed={capacity=1M;num_shard_bits=4;"
+ "strict_capacity_limit=true;high_pri_pool_ratio=0.5;}",
+ &new_opt));
ASSERT_TRUE(new_opt.block_cache != nullptr);
ASSERT_EQ(new_opt.block_cache->GetCapacity(), 1024UL*1024UL);
ASSERT_EQ(std::dynamic_pointer_cast(
@@ -616,10 +741,11 @@ TEST_F(OptionsTest, GetBlockBasedTableOptionsFromString) {
// Set only block cache capacity. Check other values are
// reset to default values.
- ASSERT_OK(GetBlockBasedTableOptionsFromString(table_opt,
- "block_cache={capacity=2M};"
- "block_cache_compressed={capacity=2M}",
- &new_opt));
+ ASSERT_OK(GetBlockBasedTableOptionsFromString(
+ config_options, table_opt,
+ "block_cache={capacity=2M};"
+ "block_cache_compressed={capacity=2M}",
+ &new_opt));
ASSERT_TRUE(new_opt.block_cache != nullptr);
ASSERT_EQ(new_opt.block_cache->GetCapacity(), 2*1024UL*1024UL);
// Default values
@@ -644,7 +770,7 @@ TEST_F(OptionsTest, GetBlockBasedTableOptionsFromString) {
// Set couple of block cache options.
ASSERT_OK(GetBlockBasedTableOptionsFromString(
- table_opt,
+ config_options, table_opt,
"block_cache={num_shard_bits=5;high_pri_pool_ratio=0.5;};"
"block_cache_compressed={num_shard_bits=5;"
"high_pri_pool_ratio=0.0;}",
@@ -665,12 +791,13 @@ TEST_F(OptionsTest, GetBlockBasedTableOptionsFromString) {
0.0);
// Set couple of block cache options.
- ASSERT_OK(GetBlockBasedTableOptionsFromString(table_opt,
- "block_cache={capacity=1M;num_shard_bits=4;"
- "strict_capacity_limit=true;};"
- "block_cache_compressed={capacity=1M;num_shard_bits=4;"
- "strict_capacity_limit=true;}",
- &new_opt));
+ ASSERT_OK(GetBlockBasedTableOptionsFromString(
+ config_options, table_opt,
+ "block_cache={capacity=1M;num_shard_bits=4;"
+ "strict_capacity_limit=true;};"
+ "block_cache_compressed={capacity=1M;num_shard_bits=4;"
+ "strict_capacity_limit=true;}",
+ &new_opt));
ASSERT_TRUE(new_opt.block_cache != nullptr);
ASSERT_EQ(new_opt.block_cache->GetCapacity(), 1024UL*1024UL);
ASSERT_EQ(std::dynamic_pointer_cast(
@@ -695,12 +822,16 @@ TEST_F(OptionsTest, GetBlockBasedTableOptionsFromString) {
TEST_F(OptionsTest, GetPlainTableOptionsFromString) {
PlainTableOptions table_opt;
PlainTableOptions new_opt;
+ ConfigOptions config_options;
+ config_options.input_strings_escaped = false;
+ config_options.ignore_unknown_options = false;
// make sure default values are overwritten by something else
- ASSERT_OK(GetPlainTableOptionsFromString(table_opt,
- "user_key_len=66;bloom_bits_per_key=20;hash_table_ratio=0.5;"
- "index_sparseness=8;huge_page_tlb_size=4;encoding_type=kPrefix;"
- "full_scan_mode=true;store_index_in_file=true",
- &new_opt));
+ ASSERT_OK(GetPlainTableOptionsFromString(
+ config_options, table_opt,
+ "user_key_len=66;bloom_bits_per_key=20;hash_table_ratio=0.5;"
+ "index_sparseness=8;huge_page_tlb_size=4;encoding_type=kPrefix;"
+ "full_scan_mode=true;store_index_in_file=true",
+ &new_opt));
ASSERT_EQ(new_opt.user_key_len, 66u);
ASSERT_EQ(new_opt.bloom_bits_per_key, 20);
ASSERT_EQ(new_opt.hash_table_ratio, 0.5);
@@ -711,16 +842,18 @@ TEST_F(OptionsTest, GetPlainTableOptionsFromString) {
ASSERT_TRUE(new_opt.store_index_in_file);
// unknown option
- ASSERT_NOK(GetPlainTableOptionsFromString(table_opt,
- "user_key_len=66;bloom_bits_per_key=20;hash_table_ratio=0.5;"
- "bad_option=1",
- &new_opt));
+ ASSERT_NOK(GetPlainTableOptionsFromString(
+ config_options, table_opt,
+ "user_key_len=66;bloom_bits_per_key=20;hash_table_ratio=0.5;"
+ "bad_option=1",
+ &new_opt));
// unrecognized EncodingType
- ASSERT_NOK(GetPlainTableOptionsFromString(table_opt,
- "user_key_len=66;bloom_bits_per_key=20;hash_table_ratio=0.5;"
- "encoding_type=kPrefixXX",
- &new_opt));
+ ASSERT_NOK(GetPlainTableOptionsFromString(
+ config_options, table_opt,
+ "user_key_len=66;bloom_bits_per_key=20;hash_table_ratio=0.5;"
+ "encoding_type=kPrefixXX",
+ &new_opt));
}
#endif // !ROCKSDB_LITE
@@ -766,6 +899,10 @@ TEST_F(OptionsTest, GetMemTableRepFactoryFromString) {
#ifndef ROCKSDB_LITE // GetOptionsFromString is not supported in RocksDB Lite
TEST_F(OptionsTest, GetOptionsFromStringTest) {
Options base_options, new_options;
+ ConfigOptions config_options;
+ config_options.input_strings_escaped = false;
+ config_options.ignore_unknown_options = false;
+
base_options.write_buffer_size = 20;
base_options.min_write_buffer_number_to_merge = 15;
BlockBasedTableOptions block_based_table_options;
@@ -789,7 +926,7 @@ TEST_F(OptionsTest, GetOptionsFromStringTest) {
});
ASSERT_OK(GetOptionsFromString(
- base_options,
+ config_options, base_options,
"write_buffer_size=10;max_write_buffer_number=16;"
"block_based_table_factory={block_cache=1M;block_size=4;};"
"compression_opts=4:5:6;create_if_missing=true;max_open_files=1;"
@@ -829,24 +966,38 @@ TEST_F(OptionsTest, GetOptionsFromStringTest) {
Env* newEnv = new_options.env;
ASSERT_OK(Env::LoadEnv(kCustomEnvName, &newEnv));
ASSERT_EQ(newEnv, new_options.env);
+
+ // Test the old interfaxe
+ ASSERT_OK(GetOptionsFromString(
+ base_options,
+ "write_buffer_size=22;max_write_buffer_number=33;max_open_files=44;",
+ &new_options));
+ ASSERT_EQ(new_options.write_buffer_size, 22U);
+ ASSERT_EQ(new_options.max_write_buffer_number, 33);
+ ASSERT_EQ(new_options.max_open_files, 44);
}
TEST_F(OptionsTest, DBOptionsSerialization) {
Options base_options, new_options;
Random rnd(301);
+ ConfigOptions config_options;
+ config_options.input_strings_escaped = false;
+ config_options.ignore_unknown_options = false;
// Phase 1: Make big change in base_options
test::RandomInitDBOptions(&base_options, &rnd);
// Phase 2: obtain a string from base_option
std::string base_options_file_content;
- ASSERT_OK(GetStringFromDBOptions(&base_options_file_content, base_options));
+ ASSERT_OK(GetStringFromDBOptions(config_options, base_options,
+ &base_options_file_content));
// Phase 3: Set new_options from the derived string and expect
// new_options == base_options
- ASSERT_OK(GetDBOptionsFromString(DBOptions(), base_options_file_content,
- &new_options));
- ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(base_options, new_options));
+ ASSERT_OK(GetDBOptionsFromString(config_options, DBOptions(),
+ base_options_file_content, &new_options));
+ ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(config_options, base_options,
+ new_options));
}
TEST_F(OptionsTest, OptionsComposeDecompose) {
@@ -854,6 +1005,9 @@ TEST_F(OptionsTest, OptionsComposeDecompose) {
// we get same constituent options.
DBOptions base_db_opts;
ColumnFamilyOptions base_cf_opts;
+ ConfigOptions
+ config_options; // Use default for ignore(false) and check (exact)
+ config_options.input_strings_escaped = false;
Random rnd(301);
test::RandomInitDBOptions(&base_db_opts, &rnd);
@@ -863,8 +1017,10 @@ TEST_F(OptionsTest, OptionsComposeDecompose) {
DBOptions new_db_opts(base_opts);
ColumnFamilyOptions new_cf_opts(base_opts);
- ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(base_db_opts, new_db_opts));
- ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_cf_opts, new_cf_opts));
+ ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(config_options, base_db_opts,
+ new_db_opts));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(config_options, base_cf_opts,
+ new_cf_opts));
delete new_cf_opts.compaction_filter;
}
@@ -872,20 +1028,25 @@ TEST_F(OptionsTest, ColumnFamilyOptionsSerialization) {
Options options;
ColumnFamilyOptions base_opt, new_opt;
Random rnd(302);
+ ConfigOptions config_options;
+ config_options.input_strings_escaped = false;
+
// Phase 1: randomly assign base_opt
// custom type options
test::RandomInitCFOptions(&base_opt, options, &rnd);
// Phase 2: obtain a string from base_opt
std::string base_options_file_content;
- ASSERT_OK(
- GetStringFromColumnFamilyOptions(&base_options_file_content, base_opt));
+ ASSERT_OK(GetStringFromColumnFamilyOptions(config_options, base_opt,
+ &base_options_file_content));
// Phase 3: Set new_opt from the derived string and expect
// new_opt == base_opt
- ASSERT_OK(GetColumnFamilyOptionsFromString(
- ColumnFamilyOptions(), base_options_file_content, &new_opt));
- ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_opt, new_opt));
+ ASSERT_OK(
+ GetColumnFamilyOptionsFromString(config_options, ColumnFamilyOptions(),
+ base_options_file_content, &new_opt));
+ ASSERT_OK(
+ RocksDBOptionsParser::VerifyCFOptions(config_options, base_opt, new_opt));
if (base_opt.compaction_filter) {
delete base_opt.compaction_filter;
}
@@ -1113,6 +1274,791 @@ TEST_F(OptionsTest, ConvertOptionsTest) {
ASSERT_EQ(table_opt.filter_policy.get(), leveldb_opt.filter_policy);
}
+#ifndef ROCKSDB_LITE
+// This test suite tests the old APIs into the Configure options methods.
+// Once those APIs are officially deprecated, this test suite can be deleted.
+class OptionsOldApiTest : public testing::Test {};
+
+TEST_F(OptionsOldApiTest, GetOptionsFromMapTest) {
+ std::unordered_map cf_options_map = {
+ {"write_buffer_size", "1"},
+ {"max_write_buffer_number", "2"},
+ {"min_write_buffer_number_to_merge", "3"},
+ {"max_write_buffer_number_to_maintain", "99"},
+ {"max_write_buffer_size_to_maintain", "-99999"},
+ {"compression", "kSnappyCompression"},
+ {"compression_per_level",
+ "kNoCompression:"
+ "kSnappyCompression:"
+ "kZlibCompression:"
+ "kBZip2Compression:"
+ "kLZ4Compression:"
+ "kLZ4HCCompression:"
+ "kXpressCompression:"
+ "kZSTD:"
+ "kZSTDNotFinalCompression"},
+ {"bottommost_compression", "kLZ4Compression"},
+ {"bottommost_compression_opts", "5:6:7:8:9:10:true"},
+ {"compression_opts", "4:5:6:7:8:9:true"},
+ {"num_levels", "8"},
+ {"level0_file_num_compaction_trigger", "8"},
+ {"level0_slowdown_writes_trigger", "9"},
+ {"level0_stop_writes_trigger", "10"},
+ {"target_file_size_base", "12"},
+ {"target_file_size_multiplier", "13"},
+ {"max_bytes_for_level_base", "14"},
+ {"level_compaction_dynamic_level_bytes", "true"},
+ {"max_bytes_for_level_multiplier", "15.0"},
+ {"max_bytes_for_level_multiplier_additional", "16:17:18"},
+ {"max_compaction_bytes", "21"},
+ {"soft_rate_limit", "1.1"},
+ {"hard_rate_limit", "2.1"},
+ {"hard_pending_compaction_bytes_limit", "211"},
+ {"arena_block_size", "22"},
+ {"disable_auto_compactions", "true"},
+ {"compaction_style", "kCompactionStyleLevel"},
+ {"compaction_pri", "kOldestSmallestSeqFirst"},
+ {"verify_checksums_in_compaction", "false"},
+ {"compaction_options_fifo", "23"},
+ {"max_sequential_skip_in_iterations", "24"},
+ {"inplace_update_support", "true"},
+ {"report_bg_io_stats", "true"},
+ {"compaction_measure_io_stats", "false"},
+ {"inplace_update_num_locks", "25"},
+ {"memtable_prefix_bloom_size_ratio", "0.26"},
+ {"memtable_whole_key_filtering", "true"},
+ {"memtable_huge_page_size", "28"},
+ {"bloom_locality", "29"},
+ {"max_successive_merges", "30"},
+ {"min_partial_merge_operands", "31"},
+ {"prefix_extractor", "fixed:31"},
+ {"optimize_filters_for_hits", "true"},
+ };
+
+ std::unordered_map db_options_map = {
+ {"create_if_missing", "false"},
+ {"create_missing_column_families", "true"},
+ {"error_if_exists", "false"},
+ {"paranoid_checks", "true"},
+ {"max_open_files", "32"},
+ {"max_total_wal_size", "33"},
+ {"use_fsync", "true"},
+ {"db_log_dir", "/db_log_dir"},
+ {"wal_dir", "/wal_dir"},
+ {"delete_obsolete_files_period_micros", "34"},
+ {"max_background_compactions", "35"},
+ {"max_background_flushes", "36"},
+ {"max_log_file_size", "37"},
+ {"log_file_time_to_roll", "38"},
+ {"keep_log_file_num", "39"},
+ {"recycle_log_file_num", "5"},
+ {"max_manifest_file_size", "40"},
+ {"table_cache_numshardbits", "41"},
+ {"WAL_ttl_seconds", "43"},
+ {"WAL_size_limit_MB", "44"},
+ {"manifest_preallocation_size", "45"},
+ {"allow_mmap_reads", "true"},
+ {"allow_mmap_writes", "false"},
+ {"use_direct_reads", "false"},
+ {"use_direct_io_for_flush_and_compaction", "false"},
+ {"is_fd_close_on_exec", "true"},
+ {"skip_log_error_on_recovery", "false"},
+ {"stats_dump_period_sec", "46"},
+ {"stats_persist_period_sec", "57"},
+ {"persist_stats_to_disk", "false"},
+ {"stats_history_buffer_size", "69"},
+ {"advise_random_on_open", "true"},
+ {"use_adaptive_mutex", "false"},
+ {"new_table_reader_for_compaction_inputs", "true"},
+ {"compaction_readahead_size", "100"},
+ {"random_access_max_buffer_size", "3145728"},
+ {"writable_file_max_buffer_size", "314159"},
+ {"bytes_per_sync", "47"},
+ {"wal_bytes_per_sync", "48"},
+ {"strict_bytes_per_sync", "true"},
+ };
+
+ ColumnFamilyOptions base_cf_opt;
+ ColumnFamilyOptions new_cf_opt;
+ ASSERT_OK(GetColumnFamilyOptionsFromMap(
+ base_cf_opt, cf_options_map, &new_cf_opt));
+ ASSERT_EQ(new_cf_opt.write_buffer_size, 1U);
+ ASSERT_EQ(new_cf_opt.max_write_buffer_number, 2);
+ ASSERT_EQ(new_cf_opt.min_write_buffer_number_to_merge, 3);
+ ASSERT_EQ(new_cf_opt.max_write_buffer_number_to_maintain, 99);
+ ASSERT_EQ(new_cf_opt.max_write_buffer_size_to_maintain, -99999);
+ ASSERT_EQ(new_cf_opt.compression, kSnappyCompression);
+ ASSERT_EQ(new_cf_opt.compression_per_level.size(), 9U);
+ ASSERT_EQ(new_cf_opt.compression_per_level[0], kNoCompression);
+ ASSERT_EQ(new_cf_opt.compression_per_level[1], kSnappyCompression);
+ ASSERT_EQ(new_cf_opt.compression_per_level[2], kZlibCompression);
+ ASSERT_EQ(new_cf_opt.compression_per_level[3], kBZip2Compression);
+ ASSERT_EQ(new_cf_opt.compression_per_level[4], kLZ4Compression);
+ ASSERT_EQ(new_cf_opt.compression_per_level[5], kLZ4HCCompression);
+ ASSERT_EQ(new_cf_opt.compression_per_level[6], kXpressCompression);
+ ASSERT_EQ(new_cf_opt.compression_per_level[7], kZSTD);
+ ASSERT_EQ(new_cf_opt.compression_per_level[8], kZSTDNotFinalCompression);
+ ASSERT_EQ(new_cf_opt.compression_opts.window_bits, 4);
+ ASSERT_EQ(new_cf_opt.compression_opts.level, 5);
+ ASSERT_EQ(new_cf_opt.compression_opts.strategy, 6);
+ ASSERT_EQ(new_cf_opt.compression_opts.max_dict_bytes, 7u);
+ ASSERT_EQ(new_cf_opt.compression_opts.zstd_max_train_bytes, 8u);
+ ASSERT_EQ(new_cf_opt.compression_opts.parallel_threads, 9u);
+ ASSERT_EQ(new_cf_opt.compression_opts.enabled, true);
+ ASSERT_EQ(new_cf_opt.bottommost_compression, kLZ4Compression);
+ ASSERT_EQ(new_cf_opt.bottommost_compression_opts.window_bits, 5);
+ ASSERT_EQ(new_cf_opt.bottommost_compression_opts.level, 6);
+ ASSERT_EQ(new_cf_opt.bottommost_compression_opts.strategy, 7);
+ ASSERT_EQ(new_cf_opt.bottommost_compression_opts.max_dict_bytes, 8u);
+ ASSERT_EQ(new_cf_opt.bottommost_compression_opts.zstd_max_train_bytes, 9u);
+ ASSERT_EQ(new_cf_opt.bottommost_compression_opts.parallel_threads, 10u);
+ ASSERT_EQ(new_cf_opt.bottommost_compression_opts.enabled, true);
+ ASSERT_EQ(new_cf_opt.num_levels, 8);
+ ASSERT_EQ(new_cf_opt.level0_file_num_compaction_trigger, 8);
+ ASSERT_EQ(new_cf_opt.level0_slowdown_writes_trigger, 9);
+ ASSERT_EQ(new_cf_opt.level0_stop_writes_trigger, 10);
+ ASSERT_EQ(new_cf_opt.target_file_size_base, static_cast(12));
+ ASSERT_EQ(new_cf_opt.target_file_size_multiplier, 13);
+ ASSERT_EQ(new_cf_opt.max_bytes_for_level_base, 14U);
+ ASSERT_EQ(new_cf_opt.level_compaction_dynamic_level_bytes, true);
+ ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier, 15.0);
+ ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier_additional.size(), 3U);
+ ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier_additional[0], 16);
+ ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier_additional[1], 17);
+ ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier_additional[2], 18);
+ ASSERT_EQ(new_cf_opt.max_compaction_bytes, 21);
+ ASSERT_EQ(new_cf_opt.hard_pending_compaction_bytes_limit, 211);
+ ASSERT_EQ(new_cf_opt.arena_block_size, 22U);
+ ASSERT_EQ(new_cf_opt.disable_auto_compactions, true);
+ ASSERT_EQ(new_cf_opt.compaction_style, kCompactionStyleLevel);
+ ASSERT_EQ(new_cf_opt.compaction_pri, kOldestSmallestSeqFirst);
+ ASSERT_EQ(new_cf_opt.compaction_options_fifo.max_table_files_size,
+ static_cast(23));
+ ASSERT_EQ(new_cf_opt.max_sequential_skip_in_iterations,
+ static_cast(24));
+ ASSERT_EQ(new_cf_opt.inplace_update_support, true);
+ ASSERT_EQ(new_cf_opt.inplace_update_num_locks, 25U);
+ ASSERT_EQ(new_cf_opt.memtable_prefix_bloom_size_ratio, 0.26);
+ ASSERT_EQ(new_cf_opt.memtable_whole_key_filtering, true);
+ ASSERT_EQ(new_cf_opt.memtable_huge_page_size, 28U);
+ ASSERT_EQ(new_cf_opt.bloom_locality, 29U);
+ ASSERT_EQ(new_cf_opt.max_successive_merges, 30U);
+ ASSERT_TRUE(new_cf_opt.prefix_extractor != nullptr);
+ ASSERT_EQ(new_cf_opt.optimize_filters_for_hits, true);
+ ASSERT_EQ(std::string(new_cf_opt.prefix_extractor->Name()),
+ "rocksdb.FixedPrefix.31");
+
+ cf_options_map["write_buffer_size"] = "hello";
+ ASSERT_NOK(GetColumnFamilyOptionsFromMap(
+ base_cf_opt, cf_options_map, &new_cf_opt));
+ ConfigOptions exact, loose;
+ exact.sanity_level = ConfigOptions::kSanityLevelExactMatch;
+ loose.sanity_level = ConfigOptions::kSanityLevelLooselyCompatible;
+
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt));
+
+ cf_options_map["write_buffer_size"] = "1";
+ ASSERT_OK(GetColumnFamilyOptionsFromMap(
+ base_cf_opt, cf_options_map, &new_cf_opt));
+
+ cf_options_map["unknown_option"] = "1";
+ ASSERT_NOK(GetColumnFamilyOptionsFromMap(
+ base_cf_opt, cf_options_map, &new_cf_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt));
+
+ ASSERT_OK(GetColumnFamilyOptionsFromMap(base_cf_opt, cf_options_map,
+ &new_cf_opt,
+ false, /* input_strings_escaped */
+ true /* ignore_unknown_options */));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(
+ loose, base_cf_opt, new_cf_opt, nullptr /* new_opt_map */));
+ ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(
+ exact /* default for VerifyCFOptions */, base_cf_opt, new_cf_opt, nullptr));
+
+ DBOptions base_db_opt;
+ DBOptions new_db_opt;
+ ASSERT_OK(GetDBOptionsFromMap(base_db_opt, db_options_map, &new_db_opt));
+ ASSERT_EQ(new_db_opt.create_if_missing, false);
+ ASSERT_EQ(new_db_opt.create_missing_column_families, true);
+ ASSERT_EQ(new_db_opt.error_if_exists, false);
+ ASSERT_EQ(new_db_opt.paranoid_checks, true);
+ ASSERT_EQ(new_db_opt.max_open_files, 32);
+ ASSERT_EQ(new_db_opt.max_total_wal_size, static_cast(33));
+ ASSERT_EQ(new_db_opt.use_fsync, true);
+ ASSERT_EQ(new_db_opt.db_log_dir, "/db_log_dir");
+ ASSERT_EQ(new_db_opt.wal_dir, "/wal_dir");
+ ASSERT_EQ(new_db_opt.delete_obsolete_files_period_micros,
+ static_cast(34));
+ ASSERT_EQ(new_db_opt.max_background_compactions, 35);
+ ASSERT_EQ(new_db_opt.max_background_flushes, 36);
+ ASSERT_EQ(new_db_opt.max_log_file_size, 37U);
+ ASSERT_EQ(new_db_opt.log_file_time_to_roll, 38U);
+ ASSERT_EQ(new_db_opt.keep_log_file_num, 39U);
+ ASSERT_EQ(new_db_opt.recycle_log_file_num, 5U);
+ ASSERT_EQ(new_db_opt.max_manifest_file_size, static_cast(40));
+ ASSERT_EQ(new_db_opt.table_cache_numshardbits, 41);
+ ASSERT_EQ(new_db_opt.WAL_ttl_seconds, static_cast(43));
+ ASSERT_EQ(new_db_opt.WAL_size_limit_MB, static_cast(44));
+ ASSERT_EQ(new_db_opt.manifest_preallocation_size, 45U);
+ ASSERT_EQ(new_db_opt.allow_mmap_reads, true);
+ ASSERT_EQ(new_db_opt.allow_mmap_writes, false);
+ ASSERT_EQ(new_db_opt.use_direct_reads, false);
+ ASSERT_EQ(new_db_opt.use_direct_io_for_flush_and_compaction, false);
+ ASSERT_EQ(new_db_opt.is_fd_close_on_exec, true);
+ ASSERT_EQ(new_db_opt.skip_log_error_on_recovery, false);
+ ASSERT_EQ(new_db_opt.stats_dump_period_sec, 46U);
+ ASSERT_EQ(new_db_opt.stats_persist_period_sec, 57U);
+ ASSERT_EQ(new_db_opt.persist_stats_to_disk, false);
+ ASSERT_EQ(new_db_opt.stats_history_buffer_size, 69U);
+ ASSERT_EQ(new_db_opt.advise_random_on_open, true);
+ ASSERT_EQ(new_db_opt.use_adaptive_mutex, false);
+ ASSERT_EQ(new_db_opt.new_table_reader_for_compaction_inputs, true);
+ ASSERT_EQ(new_db_opt.compaction_readahead_size, 100);
+ ASSERT_EQ(new_db_opt.random_access_max_buffer_size, 3145728);
+ ASSERT_EQ(new_db_opt.writable_file_max_buffer_size, 314159);
+ ASSERT_EQ(new_db_opt.bytes_per_sync, static_cast(47));
+ ASSERT_EQ(new_db_opt.wal_bytes_per_sync, static_cast(48));
+ ASSERT_EQ(new_db_opt.strict_bytes_per_sync, true);
+
+ db_options_map["max_open_files"] = "hello";
+ ASSERT_NOK(GetDBOptionsFromMap(base_db_opt, db_options_map, &new_db_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(exact, base_db_opt, new_db_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(loose, base_db_opt, new_db_opt));
+
+ // unknow options should fail parsing without ignore_unknown_options = true
+ db_options_map["unknown_db_option"] = "1";
+ ASSERT_NOK(GetDBOptionsFromMap(base_db_opt, db_options_map, &new_db_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(exact, base_db_opt, new_db_opt));
+
+ ASSERT_OK(GetDBOptionsFromMap(base_db_opt, db_options_map, &new_db_opt,
+ false, /* input_strings_escaped */
+ true /* ignore_unknown_options */));
+ ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(loose, base_db_opt, new_db_opt));
+ ASSERT_NOK(RocksDBOptionsParser::VerifyDBOptions(exact, base_db_opt, new_db_opt));
+}
+
+TEST_F(OptionsOldApiTest, GetColumnFamilyOptionsFromStringTest) {
+ ColumnFamilyOptions base_cf_opt;
+ ColumnFamilyOptions new_cf_opt;
+ base_cf_opt.table_factory.reset();
+ ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt, "", &new_cf_opt));
+ ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=5", &new_cf_opt));
+ ASSERT_EQ(new_cf_opt.write_buffer_size, 5U);
+ ASSERT_TRUE(new_cf_opt.table_factory == nullptr);
+ ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=6;", &new_cf_opt));
+ ASSERT_EQ(new_cf_opt.write_buffer_size, 6U);
+ ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ " write_buffer_size = 7 ", &new_cf_opt));
+ ASSERT_EQ(new_cf_opt.write_buffer_size, 7U);
+ ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ " write_buffer_size = 8 ; ", &new_cf_opt));
+ ASSERT_EQ(new_cf_opt.write_buffer_size, 8U);
+ ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=9;max_write_buffer_number=10", &new_cf_opt));
+ ASSERT_EQ(new_cf_opt.write_buffer_size, 9U);
+ ASSERT_EQ(new_cf_opt.max_write_buffer_number, 10);
+ ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=11; max_write_buffer_number = 12 ;",
+ &new_cf_opt));
+ ASSERT_EQ(new_cf_opt.write_buffer_size, 11U);
+ ASSERT_EQ(new_cf_opt.max_write_buffer_number, 12);
+ // Wrong name "max_write_buffer_number_"
+ ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=13;max_write_buffer_number_=14;",
+ &new_cf_opt));
+ ConfigOptions exact;
+ exact.sanity_level = ConfigOptions::kSanityLevelExactMatch;
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt));
+
+ // Comparator from object registry
+ std::string kCompName = "reverse_comp";
+ ObjectLibrary::Default()->Register(
+ kCompName,
+ [](const std::string& /*name*/,
+ std::unique_ptr* /*guard*/,
+ std::string* /* errmsg */) { return ReverseBytewiseComparator(); });
+
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ base_cf_opt, "comparator=" + kCompName + ";", &new_cf_opt));
+ ASSERT_EQ(new_cf_opt.comparator, ReverseBytewiseComparator());
+
+ // MergeOperator from object registry
+ std::unique_ptr bxo(new BytesXOROperator());
+ std::string kMoName = bxo->Name();
+ ObjectLibrary::Default()->Register(
+ kMoName,
+ [](const std::string& /*name*/, std::unique_ptr* guard,
+ std::string* /* errmsg */) {
+ guard->reset(new BytesXOROperator());
+ return guard->get();
+ });
+
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ base_cf_opt, "merge_operator=" + kMoName + ";", &new_cf_opt));
+ ASSERT_EQ(kMoName, std::string(new_cf_opt.merge_operator->Name()));
+
+ // Wrong key/value pair
+ ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=13;max_write_buffer_number;", &new_cf_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt));
+
+ // Error Paring value
+ ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=13;max_write_buffer_number=;", &new_cf_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt));
+
+ // Missing option name
+ ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=13; =100;", &new_cf_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt));
+
+ const uint64_t kilo = 1024UL;
+ const uint64_t mega = 1024 * kilo;
+ const uint64_t giga = 1024 * mega;
+ const uint64_t tera = 1024 * giga;
+
+ // Units (k)
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ base_cf_opt, "max_write_buffer_number=15K", &new_cf_opt));
+ ASSERT_EQ(new_cf_opt.max_write_buffer_number, 15 * kilo);
+ // Units (m)
+ ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "max_write_buffer_number=16m;inplace_update_num_locks=17M",
+ &new_cf_opt));
+ ASSERT_EQ(new_cf_opt.max_write_buffer_number, 16 * mega);
+ ASSERT_EQ(new_cf_opt.inplace_update_num_locks, 17u * mega);
+ // Units (g)
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ base_cf_opt,
+ "write_buffer_size=18g;prefix_extractor=capped:8;"
+ "arena_block_size=19G",
+ &new_cf_opt));
+
+ ASSERT_EQ(new_cf_opt.write_buffer_size, 18 * giga);
+ ASSERT_EQ(new_cf_opt.arena_block_size, 19 * giga);
+ ASSERT_TRUE(new_cf_opt.prefix_extractor.get() != nullptr);
+ std::string prefix_name(new_cf_opt.prefix_extractor->Name());
+ ASSERT_EQ(prefix_name, "rocksdb.CappedPrefix.8");
+
+ // Units (t)
+ ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=20t;arena_block_size=21T", &new_cf_opt));
+ ASSERT_EQ(new_cf_opt.write_buffer_size, 20 * tera);
+ ASSERT_EQ(new_cf_opt.arena_block_size, 21 * tera);
+
+ // Nested block based table options
+ // Empty
+ ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "block_based_table_factory={};arena_block_size=1024",
+ &new_cf_opt));
+ ASSERT_TRUE(new_cf_opt.table_factory != nullptr);
+ // Non-empty
+ ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "block_based_table_factory={block_cache=1M;block_size=4;};"
+ "arena_block_size=1024",
+ &new_cf_opt));
+ ASSERT_TRUE(new_cf_opt.table_factory != nullptr);
+ // Last one
+ ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "block_based_table_factory={block_cache=1M;block_size=4;}",
+ &new_cf_opt));
+ ASSERT_TRUE(new_cf_opt.table_factory != nullptr);
+ // Mismatch curly braces
+ ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "block_based_table_factory={{{block_size=4;};"
+ "arena_block_size=1024",
+ &new_cf_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt));
+
+ // Unexpected chars after closing curly brace
+ ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "block_based_table_factory={block_size=4;}};"
+ "arena_block_size=1024",
+ &new_cf_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt));
+
+ ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "block_based_table_factory={block_size=4;}xdfa;"
+ "arena_block_size=1024",
+ &new_cf_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt));
+
+ ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "block_based_table_factory={block_size=4;}xdfa",
+ &new_cf_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt));
+
+ // Invalid block based table option
+ ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "block_based_table_factory={xx_block_size=4;}",
+ &new_cf_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt));
+
+ ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "optimize_filters_for_hits=true",
+ &new_cf_opt));
+ ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "optimize_filters_for_hits=false",
+ &new_cf_opt));
+
+ ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "optimize_filters_for_hits=junk",
+ &new_cf_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(exact, base_cf_opt, new_cf_opt));
+
+ // Nested plain table options
+ // Empty
+ ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "plain_table_factory={};arena_block_size=1024",
+ &new_cf_opt));
+ ASSERT_TRUE(new_cf_opt.table_factory != nullptr);
+ ASSERT_EQ(std::string(new_cf_opt.table_factory->Name()), "PlainTable");
+ // Non-empty
+ ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "plain_table_factory={user_key_len=66;bloom_bits_per_key=20;};"
+ "arena_block_size=1024",
+ &new_cf_opt));
+ ASSERT_TRUE(new_cf_opt.table_factory != nullptr);
+ ASSERT_EQ(std::string(new_cf_opt.table_factory->Name()), "PlainTable");
+
+ // memtable factory
+ ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "memtable=skip_list:10;arena_block_size=1024",
+ &new_cf_opt));
+ ASSERT_TRUE(new_cf_opt.memtable_factory != nullptr);
+ ASSERT_EQ(std::string(new_cf_opt.memtable_factory->Name()), "SkipListFactory");
+}
+
+TEST_F(OptionsOldApiTest, GetBlockBasedTableOptionsFromString) {
+ BlockBasedTableOptions table_opt;
+ BlockBasedTableOptions new_opt;
+ // make sure default values are overwritten by something else
+ ASSERT_OK(GetBlockBasedTableOptionsFromString(
+ table_opt,
+ "cache_index_and_filter_blocks=1;index_type=kHashSearch;"
+ "checksum=kxxHash;hash_index_allow_collision=1;no_block_cache=1;"
+ "block_cache=1M;block_cache_compressed=1k;block_size=1024;"
+ "block_size_deviation=8;block_restart_interval=4;"
+ "format_version=5;whole_key_filtering=1;"
+ "filter_policy=bloomfilter:4.567:false;",
+ &new_opt));
+ ASSERT_TRUE(new_opt.cache_index_and_filter_blocks);
+ ASSERT_EQ(new_opt.index_type, BlockBasedTableOptions::kHashSearch);
+ ASSERT_EQ(new_opt.checksum, ChecksumType::kxxHash);
+ ASSERT_TRUE(new_opt.hash_index_allow_collision);
+ ASSERT_TRUE(new_opt.no_block_cache);
+ ASSERT_TRUE(new_opt.block_cache != nullptr);
+ ASSERT_EQ(new_opt.block_cache->GetCapacity(), 1024UL*1024UL);
+ ASSERT_TRUE(new_opt.block_cache_compressed != nullptr);
+ ASSERT_EQ(new_opt.block_cache_compressed->GetCapacity(), 1024UL);
+ ASSERT_EQ(new_opt.block_size, 1024UL);
+ ASSERT_EQ(new_opt.block_size_deviation, 8);
+ ASSERT_EQ(new_opt.block_restart_interval, 4);
+ ASSERT_EQ(new_opt.format_version, 5U);
+ ASSERT_EQ(new_opt.whole_key_filtering, true);
+ ASSERT_TRUE(new_opt.filter_policy != nullptr);
+ const BloomFilterPolicy& bfp =
+ dynamic_cast(*new_opt.filter_policy);
+ EXPECT_EQ(bfp.GetMillibitsPerKey(), 4567);
+ EXPECT_EQ(bfp.GetWholeBitsPerKey(), 5);
+
+ // unknown option
+ ASSERT_NOK(GetBlockBasedTableOptionsFromString(table_opt,
+ "cache_index_and_filter_blocks=1;index_type=kBinarySearch;"
+ "bad_option=1",
+ &new_opt));
+ ASSERT_EQ(static_cast(table_opt.cache_index_and_filter_blocks),
+ new_opt.cache_index_and_filter_blocks);
+ ASSERT_EQ(table_opt.index_type, new_opt.index_type);
+
+ // unrecognized index type
+ ASSERT_NOK(GetBlockBasedTableOptionsFromString(table_opt,
+ "cache_index_and_filter_blocks=1;index_type=kBinarySearchXX",
+ &new_opt));
+ ASSERT_EQ(table_opt.cache_index_and_filter_blocks,
+ new_opt.cache_index_and_filter_blocks);
+ ASSERT_EQ(table_opt.index_type, new_opt.index_type);
+
+ // unrecognized checksum type
+ ASSERT_NOK(GetBlockBasedTableOptionsFromString(table_opt,
+ "cache_index_and_filter_blocks=1;checksum=kxxHashXX",
+ &new_opt));
+ ASSERT_EQ(table_opt.cache_index_and_filter_blocks,
+ new_opt.cache_index_and_filter_blocks);
+ ASSERT_EQ(table_opt.index_type, new_opt.index_type);
+
+ // unrecognized filter policy name
+ ASSERT_NOK(GetBlockBasedTableOptionsFromString(table_opt,
+ "cache_index_and_filter_blocks=1;"
+ "filter_policy=bloomfilterxx:4:true",
+ &new_opt));
+ ASSERT_EQ(table_opt.cache_index_and_filter_blocks,
+ new_opt.cache_index_and_filter_blocks);
+ ASSERT_EQ(table_opt.filter_policy, new_opt.filter_policy);
+
+ // unrecognized filter policy config
+ ASSERT_NOK(GetBlockBasedTableOptionsFromString(table_opt,
+ "cache_index_and_filter_blocks=1;"
+ "filter_policy=bloomfilter:4",
+ &new_opt));
+ ASSERT_EQ(table_opt.cache_index_and_filter_blocks,
+ new_opt.cache_index_and_filter_blocks);
+ ASSERT_EQ(table_opt.filter_policy, new_opt.filter_policy);
+
+ // Check block cache options are overwritten when specified
+ // in new format as a struct.
+ ASSERT_OK(GetBlockBasedTableOptionsFromString(table_opt,
+ "block_cache={capacity=1M;num_shard_bits=4;"
+ "strict_capacity_limit=true;high_pri_pool_ratio=0.5;};"
+ "block_cache_compressed={capacity=1M;num_shard_bits=4;"
+ "strict_capacity_limit=true;high_pri_pool_ratio=0.5;}",
+ &new_opt));
+ ASSERT_TRUE(new_opt.block_cache != nullptr);
+ ASSERT_EQ(new_opt.block_cache->GetCapacity(), 1024UL*1024UL);
+ ASSERT_EQ(std::dynamic_pointer_cast(
+ new_opt.block_cache)->GetNumShardBits(), 4);
+ ASSERT_EQ(new_opt.block_cache->HasStrictCapacityLimit(), true);
+ ASSERT_EQ(std::dynamic_pointer_cast(
+ new_opt.block_cache)->GetHighPriPoolRatio(), 0.5);
+ ASSERT_TRUE(new_opt.block_cache_compressed != nullptr);
+ ASSERT_EQ(new_opt.block_cache_compressed->GetCapacity(), 1024UL*1024UL);
+ ASSERT_EQ(std::dynamic_pointer_cast(
+ new_opt.block_cache_compressed)->GetNumShardBits(), 4);
+ ASSERT_EQ(new_opt.block_cache_compressed->HasStrictCapacityLimit(), true);
+ ASSERT_EQ(std::dynamic_pointer_cast(
+ new_opt.block_cache_compressed)->GetHighPriPoolRatio(),
+ 0.5);
+
+ // Set only block cache capacity. Check other values are
+ // reset to default values.
+ ASSERT_OK(GetBlockBasedTableOptionsFromString(table_opt,
+ "block_cache={capacity=2M};"
+ "block_cache_compressed={capacity=2M}",
+ &new_opt));
+ ASSERT_TRUE(new_opt.block_cache != nullptr);
+ ASSERT_EQ(new_opt.block_cache->GetCapacity(), 2*1024UL*1024UL);
+ // Default values
+ ASSERT_EQ(std::dynamic_pointer_cast(
+ new_opt.block_cache)->GetNumShardBits(),
+ GetDefaultCacheShardBits(new_opt.block_cache->GetCapacity()));
+ ASSERT_EQ(new_opt.block_cache->HasStrictCapacityLimit(), false);
+ ASSERT_EQ(std::dynamic_pointer_cast(new_opt.block_cache)
+ ->GetHighPriPoolRatio(),
+ 0.5);
+ ASSERT_TRUE(new_opt.block_cache_compressed != nullptr);
+ ASSERT_EQ(new_opt.block_cache_compressed->GetCapacity(), 2*1024UL*1024UL);
+ // Default values
+ ASSERT_EQ(std::dynamic_pointer_cast(
+ new_opt.block_cache_compressed)->GetNumShardBits(),
+ GetDefaultCacheShardBits(
+ new_opt.block_cache_compressed->GetCapacity()));
+ ASSERT_EQ(new_opt.block_cache_compressed->HasStrictCapacityLimit(), false);
+ ASSERT_EQ(std::dynamic_pointer_cast(new_opt.block_cache_compressed)
+ ->GetHighPriPoolRatio(),
+ 0.5);
+
+ // Set couple of block cache options.
+ ASSERT_OK(GetBlockBasedTableOptionsFromString(
+ table_opt,
+ "block_cache={num_shard_bits=5;high_pri_pool_ratio=0.5;};"
+ "block_cache_compressed={num_shard_bits=5;"
+ "high_pri_pool_ratio=0.0;}",
+ &new_opt));
+ ASSERT_EQ(new_opt.block_cache->GetCapacity(), 0);
+ ASSERT_EQ(std::dynamic_pointer_cast(
+ new_opt.block_cache)->GetNumShardBits(), 5);
+ ASSERT_EQ(new_opt.block_cache->HasStrictCapacityLimit(), false);
+ ASSERT_EQ(std::dynamic_pointer_cast(
+ new_opt.block_cache)->GetHighPriPoolRatio(), 0.5);
+ ASSERT_TRUE(new_opt.block_cache_compressed != nullptr);
+ ASSERT_EQ(new_opt.block_cache_compressed->GetCapacity(), 0);
+ ASSERT_EQ(std::dynamic_pointer_cast(
+ new_opt.block_cache_compressed)->GetNumShardBits(), 5);
+ ASSERT_EQ(new_opt.block_cache_compressed->HasStrictCapacityLimit(), false);
+ ASSERT_EQ(std::dynamic_pointer_cast(new_opt.block_cache_compressed)
+ ->GetHighPriPoolRatio(),
+ 0.0);
+
+ // Set couple of block cache options.
+ ASSERT_OK(GetBlockBasedTableOptionsFromString(table_opt,
+ "block_cache={capacity=1M;num_shard_bits=4;"
+ "strict_capacity_limit=true;};"
+ "block_cache_compressed={capacity=1M;num_shard_bits=4;"
+ "strict_capacity_limit=true;}",
+ &new_opt));
+ ASSERT_TRUE(new_opt.block_cache != nullptr);
+ ASSERT_EQ(new_opt.block_cache->GetCapacity(), 1024UL*1024UL);
+ ASSERT_EQ(std::dynamic_pointer_cast(
+ new_opt.block_cache)->GetNumShardBits(), 4);
+ ASSERT_EQ(new_opt.block_cache->HasStrictCapacityLimit(), true);
+ ASSERT_EQ(std::dynamic_pointer_cast(new_opt.block_cache)
+ ->GetHighPriPoolRatio(),
+ 0.5);
+ ASSERT_TRUE(new_opt.block_cache_compressed != nullptr);
+ ASSERT_EQ(new_opt.block_cache_compressed->GetCapacity(), 1024UL*1024UL);
+ ASSERT_EQ(std::dynamic_pointer_cast(
+ new_opt.block_cache_compressed)->GetNumShardBits(), 4);
+ ASSERT_EQ(new_opt.block_cache_compressed->HasStrictCapacityLimit(), true);
+ ASSERT_EQ(std::dynamic_pointer_cast(new_opt.block_cache_compressed)
+ ->GetHighPriPoolRatio(),
+ 0.5);
+}
+
+TEST_F(OptionsOldApiTest, GetPlainTableOptionsFromString) {
+ PlainTableOptions table_opt;
+ PlainTableOptions new_opt;
+ // make sure default values are overwritten by something else
+ ASSERT_OK(GetPlainTableOptionsFromString(table_opt,
+ "user_key_len=66;bloom_bits_per_key=20;hash_table_ratio=0.5;"
+ "index_sparseness=8;huge_page_tlb_size=4;encoding_type=kPrefix;"
+ "full_scan_mode=true;store_index_in_file=true",
+ &new_opt));
+ ASSERT_EQ(new_opt.user_key_len, 66u);
+ ASSERT_EQ(new_opt.bloom_bits_per_key, 20);
+ ASSERT_EQ(new_opt.hash_table_ratio, 0.5);
+ ASSERT_EQ(new_opt.index_sparseness, 8);
+ ASSERT_EQ(new_opt.huge_page_tlb_size, 4);
+ ASSERT_EQ(new_opt.encoding_type, EncodingType::kPrefix);
+ ASSERT_TRUE(new_opt.full_scan_mode);
+ ASSERT_TRUE(new_opt.store_index_in_file);
+
+ // unknown option
+ ASSERT_NOK(GetPlainTableOptionsFromString(table_opt,
+ "user_key_len=66;bloom_bits_per_key=20;hash_table_ratio=0.5;"
+ "bad_option=1",
+ &new_opt));
+
+ // unrecognized EncodingType
+ ASSERT_NOK(GetPlainTableOptionsFromString(table_opt,
+ "user_key_len=66;bloom_bits_per_key=20;hash_table_ratio=0.5;"
+ "encoding_type=kPrefixXX",
+ &new_opt));
+}
+
+TEST_F(OptionsOldApiTest, GetOptionsFromStringTest) {
+ Options base_options, new_options;
+ base_options.write_buffer_size = 20;
+ base_options.min_write_buffer_number_to_merge = 15;
+ BlockBasedTableOptions block_based_table_options;
+ block_based_table_options.cache_index_and_filter_blocks = true;
+ base_options.table_factory.reset(
+ NewBlockBasedTableFactory(block_based_table_options));
+
+ // Register an Env with object registry.
+ const static char* kCustomEnvName = "CustomEnv";
+ class CustomEnv : public EnvWrapper {
+ public:
+ explicit CustomEnv(Env* _target) : EnvWrapper(_target) {}
+ };
+
+ ObjectLibrary::Default()->Register(
+ kCustomEnvName,
+ [](const std::string& /*name*/, std::unique_ptr* /*env_guard*/,
+ std::string* /* errmsg */) {
+ static CustomEnv env(Env::Default());
+ return &env;
+ });
+
+ ASSERT_OK(GetOptionsFromString(
+ base_options,
+ "write_buffer_size=10;max_write_buffer_number=16;"
+ "block_based_table_factory={block_cache=1M;block_size=4;};"
+ "compression_opts=4:5:6;create_if_missing=true;max_open_files=1;"
+ "bottommost_compression_opts=5:6:7;create_if_missing=true;max_open_files="
+ "1;"
+ "rate_limiter_bytes_per_sec=1024;env=CustomEnv",
+ &new_options));
+
+ ASSERT_EQ(new_options.compression_opts.window_bits, 4);
+ ASSERT_EQ(new_options.compression_opts.level, 5);
+ ASSERT_EQ(new_options.compression_opts.strategy, 6);
+ ASSERT_EQ(new_options.compression_opts.max_dict_bytes, 0u);
+ ASSERT_EQ(new_options.compression_opts.zstd_max_train_bytes, 0u);
+ ASSERT_EQ(new_options.compression_opts.parallel_threads, 1u);
+ ASSERT_EQ(new_options.compression_opts.enabled, false);
+ ASSERT_EQ(new_options.bottommost_compression, kDisableCompressionOption);
+ ASSERT_EQ(new_options.bottommost_compression_opts.window_bits, 5);
+ ASSERT_EQ(new_options.bottommost_compression_opts.level, 6);
+ ASSERT_EQ(new_options.bottommost_compression_opts.strategy, 7);
+ ASSERT_EQ(new_options.bottommost_compression_opts.max_dict_bytes, 0u);
+ ASSERT_EQ(new_options.bottommost_compression_opts.zstd_max_train_bytes, 0u);
+ ASSERT_EQ(new_options.bottommost_compression_opts.parallel_threads, 1u);
+ ASSERT_EQ(new_options.bottommost_compression_opts.enabled, false);
+ ASSERT_EQ(new_options.write_buffer_size, 10U);
+ ASSERT_EQ(new_options.max_write_buffer_number, 16);
+ BlockBasedTableOptions new_block_based_table_options =
+ dynamic_cast(new_options.table_factory.get())
+ ->table_options();
+ ASSERT_EQ(new_block_based_table_options.block_cache->GetCapacity(), 1U << 20);
+ ASSERT_EQ(new_block_based_table_options.block_size, 4U);
+ // don't overwrite block based table options
+ ASSERT_TRUE(new_block_based_table_options.cache_index_and_filter_blocks);
+
+ ASSERT_EQ(new_options.create_if_missing, true);
+ ASSERT_EQ(new_options.max_open_files, 1);
+ ASSERT_TRUE(new_options.rate_limiter.get() != nullptr);
+ Env* newEnv = new_options.env;
+ ASSERT_OK(Env::LoadEnv(kCustomEnvName, &newEnv));
+ ASSERT_EQ(newEnv, new_options.env);
+}
+
+TEST_F(OptionsOldApiTest, DBOptionsSerialization) {
+ Options base_options, new_options;
+ Random rnd(301);
+
+ // Phase 1: Make big change in base_options
+ test::RandomInitDBOptions(&base_options, &rnd);
+
+ // Phase 2: obtain a string from base_option
+ std::string base_options_file_content;
+ ASSERT_OK(GetStringFromDBOptions(&base_options_file_content, base_options));
+
+ // Phase 3: Set new_options from the derived string and expect
+ // new_options == base_options
+ ASSERT_OK(GetDBOptionsFromString(DBOptions(), base_options_file_content,
+ &new_options));
+ ConfigOptions config_options;
+ ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(config_options, base_options, new_options));
+}
+
+TEST_F(OptionsOldApiTest, ColumnFamilyOptionsSerialization) {
+ Options options;
+ ColumnFamilyOptions base_opt, new_opt;
+ Random rnd(302);
+ // Phase 1: randomly assign base_opt
+ // custom type options
+ test::RandomInitCFOptions(&base_opt, options, &rnd);
+
+ // Phase 2: obtain a string from base_opt
+ std::string base_options_file_content;
+ ASSERT_OK(
+ GetStringFromColumnFamilyOptions(&base_options_file_content, base_opt));
+
+ // Phase 3: Set new_opt from the derived string and expect
+ // new_opt == base_opt
+ ASSERT_OK(GetColumnFamilyOptionsFromString(
+ ColumnFamilyOptions(), base_options_file_content, &new_opt));
+ ConfigOptions config_options;
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(config_options, base_opt, new_opt));
+ if (base_opt.compaction_filter) {
+ delete base_opt.compaction_filter;
+ }
+}
+#endif // !ROCKSDB_LITE
+
#ifndef ROCKSDB_LITE
class OptionsParserTest : public testing::Test {
public:
@@ -1157,10 +2103,14 @@ TEST_F(OptionsParserTest, Comment) {
ASSERT_OK(
parser.Parse(kTestFileName, fs_.get(), false, 4096 /* readahead_size */));
- ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(*parser.db_opt(), db_opt));
+ ConfigOptions exact;
+ exact.input_strings_escaped = false;
+ exact.sanity_level = ConfigOptions::kSanityLevelExactMatch;
+ ASSERT_OK(
+ RocksDBOptionsParser::VerifyDBOptions(exact, *parser.db_opt(), db_opt));
ASSERT_EQ(parser.NumColumnFamilies(), 1U);
ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(
- *parser.GetCFOptions("default"), cf_opt));
+ exact, *parser.GetCFOptions("default"), cf_opt));
}
TEST_F(OptionsParserTest, ExtraSpace) {
@@ -1201,6 +2151,7 @@ TEST_F(OptionsParserTest, MissingDBOptions) {
RocksDBOptionsParser parser;
ASSERT_NOK(
parser.Parse(kTestFileName, fs_.get(), false, 4096 /* readahead_size */));
+ ;
}
TEST_F(OptionsParserTest, DoubleDBOptions) {
@@ -1448,8 +2399,10 @@ void VerifyCFPointerTypedOptions(
ColumnFamilyOptions* base_cf_opt, const ColumnFamilyOptions* new_cf_opt,
const std::unordered_map* new_cf_opt_map) {
std::string name_buffer;
- ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt,
- new_cf_opt_map));
+ ConfigOptions config_options;
+ config_options.input_strings_escaped = false;
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(config_options, *base_cf_opt,
+ *new_cf_opt, new_cf_opt_map));
// change the name of merge operator back-and-forth
{
@@ -1460,11 +2413,11 @@ void VerifyCFPointerTypedOptions(
// change the name and expect non-ok status
merge_operator->SetName("some-other-name");
ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(
- *base_cf_opt, *new_cf_opt, new_cf_opt_map));
+ config_options, *base_cf_opt, *new_cf_opt, new_cf_opt_map));
// change the name back and expect ok status
merge_operator->SetName(name_buffer);
- ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt,
- new_cf_opt_map));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(
+ config_options, *base_cf_opt, *new_cf_opt, new_cf_opt_map));
}
}
@@ -1478,11 +2431,11 @@ void VerifyCFPointerTypedOptions(
// change the name and expect non-ok status
compaction_filter_factory->SetName("some-other-name");
ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(
- *base_cf_opt, *new_cf_opt, new_cf_opt_map));
+ config_options, *base_cf_opt, *new_cf_opt, new_cf_opt_map));
// change the name back and expect ok status
compaction_filter_factory->SetName(name_buffer);
- ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt,
- new_cf_opt_map));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(
+ config_options, *base_cf_opt, *new_cf_opt, new_cf_opt_map));
}
}
@@ -1493,11 +2446,11 @@ void VerifyCFPointerTypedOptions(
base_cf_opt->compaction_filter = nullptr;
// set compaction_filter to nullptr and expect non-ok status
ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(
- *base_cf_opt, *new_cf_opt, new_cf_opt_map));
+ config_options, *base_cf_opt, *new_cf_opt, new_cf_opt_map));
// set the value back and expect ok status
base_cf_opt->compaction_filter = tmp_compaction_filter;
- ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt,
- new_cf_opt_map));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(
+ config_options, *base_cf_opt, *new_cf_opt, new_cf_opt_map));
}
}
@@ -1508,11 +2461,11 @@ void VerifyCFPointerTypedOptions(
base_cf_opt->table_factory.reset();
// set table_factory to nullptr and expect non-ok status
ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(
- *base_cf_opt, *new_cf_opt, new_cf_opt_map));
+ config_options, *base_cf_opt, *new_cf_opt, new_cf_opt_map));
// set the value back and expect ok status
base_cf_opt->table_factory = tmp_table_factory;
- ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt,
- new_cf_opt_map));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(
+ config_options, *base_cf_opt, *new_cf_opt, new_cf_opt_map));
}
}
@@ -1523,11 +2476,11 @@ void VerifyCFPointerTypedOptions(
base_cf_opt->memtable_factory.reset();
// set memtable_factory to nullptr and expect non-ok status
ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(
- *base_cf_opt, *new_cf_opt, new_cf_opt_map));
+ config_options, *base_cf_opt, *new_cf_opt, new_cf_opt_map));
// set the value back and expect ok status
base_cf_opt->memtable_factory = tmp_memtable_factory;
- ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt,
- new_cf_opt_map));
+ ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(
+ config_options, *base_cf_opt, *new_cf_opt, new_cf_opt_map));
}
}
}
@@ -1611,12 +2564,13 @@ TEST_F(OptionsParserTest, DumpAndParse) {
}
const std::string kOptionsFileName = "test-persisted-options.ini";
+ // Use default for escaped(true), unknown(false) and check (exact)
+ ConfigOptions config_options;
ASSERT_OK(PersistRocksDBOptions(base_db_opt, cf_names, base_cf_opts,
kOptionsFileName, fs_.get()));
RocksDBOptionsParser parser;
- ASSERT_OK(
- parser.Parse(kOptionsFileName, fs_.get(), false, 0 /* readahead_size */));
+ ASSERT_OK(parser.Parse(config_options, kOptionsFileName, fs_.get()));
// Make sure block-based table factory options was deserialized correctly
std::shared_ptr ttf = (*parser.cf_opts())[4].table_factory;
@@ -1628,15 +2582,17 @@ TEST_F(OptionsParserTest, DumpAndParse) {
parsed_bbto.cache_index_and_filter_blocks);
ASSERT_OK(RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
- base_db_opt, cf_names, base_cf_opts, kOptionsFileName, fs_.get()));
+ config_options, base_db_opt, cf_names, base_cf_opts, kOptionsFileName,
+ fs_.get()));
- ASSERT_OK(
- RocksDBOptionsParser::VerifyDBOptions(*parser.db_opt(), base_db_opt));
+ ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(
+ config_options, *parser.db_opt(), base_db_opt));
for (int c = 0; c < num_cf; ++c) {
const auto* cf_opt = parser.GetCFOptions(cf_names[c]);
ASSERT_NE(cf_opt, nullptr);
ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(
- base_cf_opts[c], *cf_opt, &(parser.cf_opt_maps()->at(c))));
+ config_options, base_cf_opts[c], *cf_opt,
+ &(parser.cf_opt_maps()->at(c))));
}
// Further verify pointer-typed options
@@ -1651,7 +2607,8 @@ TEST_F(OptionsParserTest, DumpAndParse) {
base_db_opt.max_open_files++;
ASSERT_NOK(RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
- base_db_opt, cf_names, base_cf_opts, kOptionsFileName, fs_.get()));
+ config_options, base_db_opt, cf_names, base_cf_opts, kOptionsFileName,
+ fs_.get()));
for (int c = 0; c < num_cf; ++c) {
if (base_cf_opts[c].compaction_filter) {
@@ -1675,8 +2632,8 @@ TEST_F(OptionsParserTest, DifferentDefault) {
kOptionsFileName, fs_.get()));
RocksDBOptionsParser parser;
- ASSERT_OK(
- parser.Parse(kOptionsFileName, fs_.get(), false, 0 /* readahead_size */));
+ ASSERT_OK(parser.Parse(kOptionsFileName, fs_.get(), false,
+ 4096 /* readahead_size */));
{
Options old_default_opts;
@@ -1757,10 +2714,16 @@ class OptionsSanityCheckTest : public OptionsParserTest {
protected:
Status SanityCheckCFOptions(const ColumnFamilyOptions& cf_opts,
- OptionsSanityCheckLevel level) {
+ ConfigOptions::SanityLevel level,
+ bool input_strings_escaped = true) {
+ ConfigOptions config_options;
+ config_options.sanity_level = level;
+ config_options.ignore_unknown_options = false;
+ config_options.input_strings_escaped = input_strings_escaped;
+
return RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
- DBOptions(), {"default"}, {cf_opts}, kOptionsFileName, fs_.get(),
- level);
+ config_options, DBOptions(), {"default"}, {cf_opts}, kOptionsFileName,
+ fs_.get());
}
Status PersistCFOptions(const ColumnFamilyOptions& cf_opts) {
@@ -1782,7 +2745,8 @@ TEST_F(OptionsSanityCheckTest, SanityCheck) {
// default ColumnFamilyOptions
{
ASSERT_OK(PersistCFOptions(opts));
- ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
+ ASSERT_OK(
+ SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelExactMatch));
}
// prefix_extractor
@@ -1790,59 +2754,73 @@ TEST_F(OptionsSanityCheckTest, SanityCheck) {
// Okay to change prefix_extractor form nullptr to non-nullptr
ASSERT_EQ(opts.prefix_extractor.get(), nullptr);
opts.prefix_extractor.reset(NewCappedPrefixTransform(10));
- ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible));
- ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone));
+ ASSERT_OK(SanityCheckCFOptions(
+ opts, ConfigOptions::kSanityLevelLooselyCompatible));
+ ASSERT_OK(SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelNone));
// persist the change
ASSERT_OK(PersistCFOptions(opts));
- ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
+ ASSERT_OK(
+ SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelExactMatch));
// use same prefix extractor but with different parameter
opts.prefix_extractor.reset(NewCappedPrefixTransform(15));
- // expect pass only in kSanityLevelLooselyCompatible
- ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
- ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible));
- ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone));
+ // expect pass only in
+ // ConfigOptions::kSanityLevelLooselyCompatible
+ ASSERT_NOK(
+ SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelExactMatch));
+ ASSERT_OK(SanityCheckCFOptions(
+ opts, ConfigOptions::kSanityLevelLooselyCompatible));
+ ASSERT_OK(SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelNone));
// repeat the test with FixedPrefixTransform
opts.prefix_extractor.reset(NewFixedPrefixTransform(10));
- ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
- ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible));
- ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone));
+ ASSERT_NOK(
+ SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelExactMatch));
+ ASSERT_OK(SanityCheckCFOptions(
+ opts, ConfigOptions::kSanityLevelLooselyCompatible));
+ ASSERT_OK(SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelNone));
// persist the change of prefix_extractor
ASSERT_OK(PersistCFOptions(opts));
- ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
+ ASSERT_OK(
+ SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelExactMatch));
// use same prefix extractor but with different parameter
opts.prefix_extractor.reset(NewFixedPrefixTransform(15));
- // expect pass only in kSanityLevelLooselyCompatible
- ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
- ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible));
- ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone));
+ // expect pass only in
+ // ConfigOptions::kSanityLevelLooselyCompatible
+ ASSERT_NOK(
+ SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelExactMatch));
+ ASSERT_OK(SanityCheckCFOptions(
+ opts, ConfigOptions::kSanityLevelLooselyCompatible));
+ ASSERT_OK(SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelNone));
// Change prefix extractor from non-nullptr to nullptr
opts.prefix_extractor.reset();
// expect pass as it's safe to change prefix_extractor
// from non-null to null
- ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible));
- ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone));
+ ASSERT_OK(SanityCheckCFOptions(
+ opts, ConfigOptions::kSanityLevelLooselyCompatible));
+ ASSERT_OK(SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelNone));
}
// persist the change
ASSERT_OK(PersistCFOptions(opts));
- ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
+ ASSERT_OK(SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelExactMatch));
// table_factory
{
for (int tb = 0; tb <= 2; ++tb) {
// change the table factory
opts.table_factory.reset(test::RandomTableFactory(&rnd, tb));
- ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible));
- ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone));
+ ASSERT_NOK(SanityCheckCFOptions(
+ opts, ConfigOptions::kSanityLevelLooselyCompatible));
+ ASSERT_OK(SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelNone));
// persist the change
ASSERT_OK(PersistCFOptions(opts));
- ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
+ ASSERT_OK(
+ SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelExactMatch));
}
}
@@ -1850,32 +2828,38 @@ TEST_F(OptionsSanityCheckTest, SanityCheck) {
{
// Test when going from nullptr -> merge operator
opts.merge_operator.reset(test::RandomMergeOperator(&rnd));
- ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible));
- ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone));
+ ASSERT_OK(SanityCheckCFOptions(
+ opts, ConfigOptions::kSanityLevelLooselyCompatible));
+ ASSERT_OK(SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelNone));
// persist the change
ASSERT_OK(PersistCFOptions(opts));
- ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
+ ASSERT_OK(
+ SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelExactMatch));
for (int test = 0; test < 5; ++test) {
// change the merge operator
opts.merge_operator.reset(test::RandomMergeOperator(&rnd));
- ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible));
- ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone));
+ ASSERT_NOK(SanityCheckCFOptions(
+ opts, ConfigOptions::kSanityLevelLooselyCompatible));
+ ASSERT_OK(SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelNone));
// persist the change
ASSERT_OK(PersistCFOptions(opts));
- ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
+ ASSERT_OK(
+ SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelExactMatch));
}
// Test when going from merge operator -> nullptr
opts.merge_operator = nullptr;
- ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible));
- ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone));
+ ASSERT_NOK(SanityCheckCFOptions(
+ opts, ConfigOptions::kSanityLevelLooselyCompatible));
+ ASSERT_OK(SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelNone));
// persist the change
ASSERT_OK(PersistCFOptions(opts));
- ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
+ ASSERT_OK(
+ SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelExactMatch));
}
// compaction_filter
@@ -1883,12 +2867,15 @@ TEST_F(OptionsSanityCheckTest, SanityCheck) {
for (int test = 0; test < 5; ++test) {
// change the compaction filter
opts.compaction_filter = test::RandomCompactionFilter(&rnd);
- ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
- ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible));
+ ASSERT_NOK(
+ SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelExactMatch));
+ ASSERT_OK(SanityCheckCFOptions(
+ opts, ConfigOptions::kSanityLevelLooselyCompatible));
// persist the change
ASSERT_OK(PersistCFOptions(opts));
- ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
+ ASSERT_OK(
+ SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelExactMatch));
delete opts.compaction_filter;
opts.compaction_filter = nullptr;
}
@@ -1900,12 +2887,15 @@ TEST_F(OptionsSanityCheckTest, SanityCheck) {
// change the compaction filter factory
opts.compaction_filter_factory.reset(
test::RandomCompactionFilterFactory(&rnd));
- ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
- ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible));
+ ASSERT_NOK(
+ SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelExactMatch));
+ ASSERT_OK(SanityCheckCFOptions(
+ opts, ConfigOptions::kSanityLevelLooselyCompatible));
// persist the change
ASSERT_OK(PersistCFOptions(opts));
- ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch));
+ ASSERT_OK(
+ SanityCheckCFOptions(opts, ConfigOptions::kSanityLevelExactMatch));
}
}
}
diff --git a/src.mk b/src.mk
index 825499146..a17e72949 100644
--- a/src.mk
+++ b/src.mk
@@ -496,6 +496,7 @@ JNI_NATIVE_SOURCES = \
java/rocksjni/comparator.cc \
java/rocksjni/comparatorjnicallback.cc \
java/rocksjni/compression_options.cc \
+ java/rocksjni/config_options.cc \
java/rocksjni/env.cc \
java/rocksjni/env_options.cc \
java/rocksjni/ingest_external_file_options.cc \
diff --git a/table/block_based/block_based_table_factory.cc b/table/block_based/block_based_table_factory.cc
index b79f8e7de..d2d507b3e 100644
--- a/table/block_based/block_based_table_factory.cc
+++ b/table/block_based/block_based_table_factory.cc
@@ -7,19 +7,22 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
-#include
-#include
+#include "table/block_based/block_based_table_factory.h"
+#include
+
+#include
#include
#include
#include "options/options_helper.h"
+#include "options/options_parser.h"
+#include "options/options_sanity_check.h"
#include "port/port.h"
#include "rocksdb/cache.h"
#include "rocksdb/convenience.h"
#include "rocksdb/flush_block_policy.h"
#include "table/block_based/block_based_table_builder.h"
-#include "table/block_based/block_based_table_factory.h"
#include "table/block_based/block_based_table_reader.h"
#include "table/format.h"
#include "util/mutexlock.h"
@@ -536,17 +539,16 @@ std::string BlockBasedTableFactory::GetPrintableTableOptions() const {
}
#ifndef ROCKSDB_LITE
-
Status BlockBasedTableFactory::GetOptionString(
- std::string* opt_string, const std::string& delimiter) const {
+ const ConfigOptions& config_options, std::string* opt_string) const {
assert(opt_string);
opt_string->clear();
- return GetStringFromStruct(opt_string, &table_options_,
- block_based_table_type_info, delimiter);
+ return GetStringFromStruct(config_options, &table_options_,
+ block_based_table_type_info, opt_string);
}
#else
Status BlockBasedTableFactory::GetOptionString(
- std::string* /*opt_string*/, const std::string& /*delimiter*/) const {
+ const ConfigOptions& /*opts*/, std::string* /*opt_string*/) const {
return Status::OK();
}
#endif // !ROCKSDB_LITE
@@ -557,14 +559,14 @@ const BlockBasedTableOptions& BlockBasedTableFactory::table_options() const {
#ifndef ROCKSDB_LITE
namespace {
-std::string ParseBlockBasedTableOption(const std::string& name,
+std::string ParseBlockBasedTableOption(const ConfigOptions& config_options,
+ const std::string& name,
const std::string& org_value,
- BlockBasedTableOptions* new_options,
- bool input_strings_escaped = false,
- bool ignore_unknown_options = false) {
- const std::string& value =
- input_strings_escaped ? UnescapeOptionString(org_value) : org_value;
- if (!input_strings_escaped) {
+ BlockBasedTableOptions* new_options) {
+ const std::string& value = config_options.input_strings_escaped
+ ? UnescapeOptionString(org_value)
+ : org_value;
+ if (!config_options.input_strings_escaped) {
// if the input string is not escaped, it means this function is
// invoked from SetOptions, which takes the old format.
if (name == "block_cache" || name == "block_cache_compressed") {
@@ -615,7 +617,7 @@ std::string ParseBlockBasedTableOption(const std::string& name,
}
const auto iter = block_based_table_type_info.find(name);
if (iter == block_based_table_type_info.end()) {
- if (ignore_unknown_options) {
+ if (config_options.ignore_unknown_options) {
return "";
} else {
return "Unrecognized option";
@@ -634,14 +636,23 @@ std::string ParseBlockBasedTableOption(const std::string& name,
Status GetBlockBasedTableOptionsFromString(
const BlockBasedTableOptions& table_options, const std::string& opts_str,
BlockBasedTableOptions* new_table_options) {
+ ConfigOptions config_options;
+ config_options.input_strings_escaped = false;
+ config_options.ignore_unknown_options = false;
+ return GetBlockBasedTableOptionsFromString(config_options, table_options,
+ opts_str, new_table_options);
+}
+Status GetBlockBasedTableOptionsFromString(
+ const ConfigOptions& config_options,
+ const BlockBasedTableOptions& table_options, const std::string& opts_str,
+ BlockBasedTableOptions* new_table_options) {
std::unordered_map opts_map;
Status s = StringToMap(opts_str, &opts_map);
if (!s.ok()) {
return s;
}
-
- return GetBlockBasedTableOptionsFromMap(table_options, opts_map,
- new_table_options);
+ return GetBlockBasedTableOptionsFromMap(config_options, table_options,
+ opts_map, new_table_options);
}
Status GetBlockBasedTableOptionsFromMap(
@@ -649,18 +660,31 @@ Status GetBlockBasedTableOptionsFromMap(
const std::unordered_map& opts_map,
BlockBasedTableOptions* new_table_options, bool input_strings_escaped,
bool ignore_unknown_options) {
+ ConfigOptions config_options;
+ config_options.input_strings_escaped = input_strings_escaped;
+ config_options.ignore_unknown_options = ignore_unknown_options;
+
+ return GetBlockBasedTableOptionsFromMap(config_options, table_options,
+ opts_map, new_table_options);
+}
+
+Status GetBlockBasedTableOptionsFromMap(
+ const ConfigOptions& config_options,
+ const BlockBasedTableOptions& table_options,
+ const std::unordered_map& opts_map,
+ BlockBasedTableOptions* new_table_options) {
assert(new_table_options);
*new_table_options = table_options;
for (const auto& o : opts_map) {
auto error_message = ParseBlockBasedTableOption(
- o.first, o.second, new_table_options, input_strings_escaped,
- ignore_unknown_options);
+ config_options, o.first, o.second, new_table_options);
if (error_message != "") {
const auto iter = block_based_table_type_info.find(o.first);
if (iter == block_based_table_type_info.end() ||
- !input_strings_escaped || // !input_strings_escaped indicates
- // the old API, where everything is
- // parsable.
+ !config_options
+ .input_strings_escaped || // !input_strings_escaped indicates
+ // the old API, where everything is
+ // parsable.
(!iter->second.IsByName() && !iter->second.IsDeprecated())) {
// Restore "new_options" to the default "base_options".
*new_table_options = table_options;
@@ -672,12 +696,11 @@ Status GetBlockBasedTableOptionsFromMap(
return Status::OK();
}
-Status VerifyBlockBasedTableFactory(
- const BlockBasedTableFactory* base_tf,
- const BlockBasedTableFactory* file_tf,
- OptionsSanityCheckLevel sanity_check_level) {
+Status VerifyBlockBasedTableFactory(const ConfigOptions& config_options,
+ const BlockBasedTableFactory* base_tf,
+ const BlockBasedTableFactory* file_tf) {
if ((base_tf != nullptr) != (file_tf != nullptr) &&
- sanity_check_level > OptionsSanityCheckLevel::kSanityLevelNone) {
+ config_options.sanity_level > ConfigOptions::kSanityLevelNone) {
return Status::Corruption(
"[RocksDBOptionsParser]: Inconsistent TableFactory class type");
}
@@ -695,7 +718,7 @@ Status VerifyBlockBasedTableFactory(
// contain random values since they might not be initialized
continue;
}
- if (BBTOptionSanityCheckLevel(pair.first) <= sanity_check_level) {
+ if (BBTOptionSanityCheckLevel(pair.first) <= config_options.sanity_level) {
if (!AreEqualOptions(reinterpret_cast(&base_opt),
reinterpret_cast(&file_opt),
pair.second, pair.first, nullptr)) {
diff --git a/table/block_based/block_based_table_factory.h b/table/block_based/block_based_table_factory.h
index 6bc01cee6..38bf2395b 100644
--- a/table/block_based/block_based_table_factory.h
+++ b/table/block_based/block_based_table_factory.h
@@ -14,13 +14,11 @@
#include
#include "db/dbformat.h"
-#include "options/options_helper.h"
-#include "options/options_parser.h"
#include "rocksdb/flush_block_policy.h"
#include "rocksdb/table.h"
namespace ROCKSDB_NAMESPACE {
-
+struct ConfigOptions;
struct EnvOptions;
class BlockBasedTableBuilder;
@@ -66,8 +64,8 @@ class BlockBasedTableFactory : public TableFactory {
std::string GetPrintableTableOptions() const override;
- Status GetOptionString(std::string* opt_string,
- const std::string& delimiter) const override;
+ Status GetOptionString(const ConfigOptions& config_options,
+ std::string* opt_string) const override;
const BlockBasedTableOptions& table_options() const;
@@ -89,9 +87,7 @@ extern const std::string kPropFalse;
#ifndef ROCKSDB_LITE
extern Status VerifyBlockBasedTableFactory(
- const BlockBasedTableFactory* base_tf,
- const BlockBasedTableFactory* file_tf,
- OptionsSanityCheckLevel sanity_check_level);
-
+ const ConfigOptions& config_options, const BlockBasedTableFactory* base_tf,
+ const BlockBasedTableFactory* file_tf);
#endif // !ROCKSDB_LITE
} // namespace ROCKSDB_NAMESPACE
diff --git a/table/block_based/block_based_table_reader.cc b/table/block_based/block_based_table_reader.cc
index 9a925de83..e7f40d079 100644
--- a/table/block_based/block_based_table_reader.cc
+++ b/table/block_based/block_based_table_reader.cc
@@ -7,6 +7,7 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#include "table/block_based/block_based_table_reader.h"
+
#include
#include
#include
@@ -16,10 +17,10 @@
#include "db/dbformat.h"
#include "db/pinned_iterators_manager.h"
-
#include "file/file_prefetch_buffer.h"
#include "file/random_access_file_reader.h"
-
+#include "monitoring/perf_context_imp.h"
+#include "options/options_helper.h"
#include "rocksdb/cache.h"
#include "rocksdb/comparator.h"
#include "rocksdb/env.h"
@@ -30,7 +31,6 @@
#include "rocksdb/statistics.h"
#include "rocksdb/table.h"
#include "rocksdb/table_properties.h"
-
#include "table/block_based/binary_search_index_reader.h"
#include "table/block_based/block.h"
#include "table/block_based/block_based_filter_block.h"
diff --git a/table/cuckoo/cuckoo_table_factory.h b/table/cuckoo/cuckoo_table_factory.h
index 1b5b6c200..6a8c36981 100644
--- a/table/cuckoo/cuckoo_table_factory.h
+++ b/table/cuckoo/cuckoo_table_factory.h
@@ -79,8 +79,8 @@ class CuckooTableFactory : public TableFactory {
void* GetOptions() override { return &table_options_; }
- Status GetOptionString(std::string* /*opt_string*/,
- const std::string& /*delimiter*/) const override {
+ Status GetOptionString(const ConfigOptions& /*config_options*/,
+ std::string* /*opt_string*/) const override {
return Status::OK();
}
diff --git a/table/plain/plain_table_factory.cc b/table/plain/plain_table_factory.cc
index df296cdff..4759eb975 100644
--- a/table/plain/plain_table_factory.cc
+++ b/table/plain/plain_table_factory.cc
@@ -7,7 +7,9 @@
#include "table/plain/plain_table_factory.h"
#include
+
#include
+
#include "db/dbformat.h"
#include "options/options_helper.h"
#include "port/port.h"
@@ -117,12 +119,24 @@ const PlainTableOptions& PlainTableFactory::table_options() const {
Status GetPlainTableOptionsFromString(const PlainTableOptions& table_options,
const std::string& opts_str,
PlainTableOptions* new_table_options) {
+ ConfigOptions config_options;
+ config_options.input_strings_escaped = false;
+ config_options.ignore_unknown_options = false;
+ return GetPlainTableOptionsFromString(config_options, table_options, opts_str,
+ new_table_options);
+}
+
+Status GetPlainTableOptionsFromString(const ConfigOptions& config_options,
+ const PlainTableOptions& table_options,
+ const std::string& opts_str,
+ PlainTableOptions* new_table_options) {
std::unordered_map opts_map;
Status s = StringToMap(opts_str, &opts_map);
if (!s.ok()) {
return s;
}
- return GetPlainTableOptionsFromMap(table_options, opts_map,
+
+ return GetPlainTableOptionsFromMap(config_options, table_options, opts_map,
new_table_options);
}
@@ -190,16 +204,16 @@ Status GetMemTableRepFactoryFromString(
return Status::OK();
}
-std::string ParsePlainTableOptions(const std::string& name,
+std::string ParsePlainTableOptions(const ConfigOptions& config_options,
+ const std::string& name,
const std::string& org_value,
- PlainTableOptions* new_options,
- bool input_strings_escaped = false,
- bool ignore_unknown_options = false) {
- const std::string& value =
- input_strings_escaped ? UnescapeOptionString(org_value) : org_value;
+ PlainTableOptions* new_options) {
+ const std::string& value = config_options.input_strings_escaped
+ ? UnescapeOptionString(org_value)
+ : org_value;
const auto iter = plain_table_type_info.find(name);
if (iter == plain_table_type_info.end()) {
- if (ignore_unknown_options) {
+ if (config_options.ignore_unknown_options) {
return "";
} else {
return "Unrecognized option";
@@ -218,18 +232,30 @@ Status GetPlainTableOptionsFromMap(
const PlainTableOptions& table_options,
const std::unordered_map& opts_map,
PlainTableOptions* new_table_options, bool input_strings_escaped,
- bool /*ignore_unknown_options*/) {
+ bool ignore_unknown_options) {
+ ConfigOptions config_options;
+ config_options.input_strings_escaped = input_strings_escaped;
+ config_options.ignore_unknown_options = ignore_unknown_options;
+ return GetPlainTableOptionsFromMap(config_options, table_options, opts_map,
+ new_table_options);
+}
+
+Status GetPlainTableOptionsFromMap(
+ const ConfigOptions& config_options, const PlainTableOptions& table_options,
+ const std::unordered_map& opts_map,
+ PlainTableOptions* new_table_options) {
assert(new_table_options);
*new_table_options = table_options;
for (const auto& o : opts_map) {
- auto error_message = ParsePlainTableOptions(
- o.first, o.second, new_table_options, input_strings_escaped);
+ auto error_message = ParsePlainTableOptions(config_options, o.first,
+ o.second, new_table_options);
if (error_message != "") {
const auto iter = plain_table_type_info.find(o.first);
if (iter == plain_table_type_info.end() ||
- !input_strings_escaped || // !input_strings_escaped indicates
- // the old API, where everything is
- // parsable.
+ !config_options
+ .input_strings_escaped || // !input_strings_escaped indicates
+ // the old API, where everything is
+ // parsable.
(!iter->second.IsByName() && !iter->second.IsDeprecated())) {
// Restore "new_options" to the default "base_options".
*new_table_options = table_options;
diff --git a/table/plain/plain_table_factory.h b/table/plain/plain_table_factory.h
index cbd892a9e..282a24a5b 100644
--- a/table/plain/plain_table_factory.h
+++ b/table/plain/plain_table_factory.h
@@ -184,8 +184,8 @@ class PlainTableFactory : public TableFactory {
void* GetOptions() override { return &table_options_; }
- Status GetOptionString(std::string* /*opt_string*/,
- const std::string& /*delimiter*/) const override {
+ Status GetOptionString(const ConfigOptions& /*config_options*/,
+ std::string* /*opt_string*/) const override {
return Status::OK();
}
diff --git a/tools/ldb_cmd.cc b/tools/ldb_cmd.cc
index bfcd118bf..56d52a340 100644
--- a/tools/ldb_cmd.cc
+++ b/tools/ldb_cmd.cc
@@ -330,7 +330,6 @@ LDBCommand::LDBCommand(const std::map& options,
is_db_ttl_(false),
timestamp_(false),
try_load_options_(false),
- ignore_unknown_options_(false),
create_if_missing_(false),
option_map_(options),
flags_(flags),
@@ -363,13 +362,15 @@ LDBCommand::LDBCommand(const std::map& options,
is_db_ttl_ = IsFlagPresent(flags, ARG_TTL);
timestamp_ = IsFlagPresent(flags, ARG_TIMESTAMP);
try_load_options_ = IsFlagPresent(flags, ARG_TRY_LOAD_OPTIONS);
- ignore_unknown_options_ = IsFlagPresent(flags, ARG_IGNORE_UNKNOWN_OPTIONS);
+ config_options_.ignore_unknown_options =
+ IsFlagPresent(flags, ARG_IGNORE_UNKNOWN_OPTIONS);
}
void LDBCommand::OpenDB() {
if (!create_if_missing_ && try_load_options_) {
- Status s = LoadLatestOptions(db_path_, options_.env, &options_,
- &column_families_, ignore_unknown_options_);
+ config_options_.env = options_.env;
+ Status s = LoadLatestOptions(config_options_, db_path_, &options_,
+ &column_families_);
if (!s.ok() && !s.IsNotFound()) {
// Option file exists but load option file error.
std::string msg = s.ToString();
diff --git a/utilities/options/options_util.cc b/utilities/options/options_util.cc
index 0719798e3..065211a84 100644
--- a/utilities/options/options_util.cc
+++ b/utilities/options/options_util.cc
@@ -10,7 +10,9 @@
#include "env/composite_env_wrapper.h"
#include "file/filename.h"
#include "options/options_parser.h"
+#include "rocksdb/convenience.h"
#include "rocksdb/options.h"
+#include "table/block_based/block_based_table_factory.h"
namespace ROCKSDB_NAMESPACE {
Status LoadOptionsFromFile(const std::string& file_name, Env* env,
@@ -18,10 +20,22 @@ Status LoadOptionsFromFile(const std::string& file_name, Env* env,
std::vector* cf_descs,
bool ignore_unknown_options,
std::shared_ptr* cache) {
+ ConfigOptions config_options;
+ config_options.ignore_unknown_options = ignore_unknown_options;
+ config_options.input_strings_escaped = true;
+ config_options.env = env;
+
+ return LoadOptionsFromFile(config_options, file_name, db_options, cf_descs,
+ cache);
+}
+
+Status LoadOptionsFromFile(const ConfigOptions& config_options,
+ const std::string& file_name, DBOptions* db_options,
+ std::vector* cf_descs,
+ std::shared_ptr* cache) {
RocksDBOptionsParser parser;
- LegacyFileSystemWrapper fs(env);
- Status s = parser.Parse(file_name, &fs, ignore_unknown_options,
- 0 /* file_readahead_size */);
+ LegacyFileSystemWrapper fs(config_options.env);
+ Status s = parser.Parse(config_options, file_name, &fs);
if (!s.ok()) {
return s;
}
@@ -76,21 +90,48 @@ Status LoadLatestOptions(const std::string& dbpath, Env* env,
std::vector* cf_descs,
bool ignore_unknown_options,
std::shared_ptr* cache) {
+ ConfigOptions config_options;
+ config_options.ignore_unknown_options = ignore_unknown_options;
+ config_options.input_strings_escaped = true;
+ config_options.env = env;
+
+ return LoadLatestOptions(config_options, dbpath, db_options, cf_descs, cache);
+}
+
+Status LoadLatestOptions(const ConfigOptions& config_options,
+ const std::string& dbpath, DBOptions* db_options,
+ std::vector* cf_descs,
+ std::shared_ptr* cache) {
std::string options_file_name;
- Status s = GetLatestOptionsFileName(dbpath, env, &options_file_name);
+ Status s =
+ GetLatestOptionsFileName(dbpath, config_options.env, &options_file_name);
if (!s.ok()) {
return s;
}
- return LoadOptionsFromFile(dbpath + "/" + options_file_name, env, db_options,
- cf_descs, ignore_unknown_options, cache);
+ return LoadOptionsFromFile(config_options, dbpath + "/" + options_file_name,
+ db_options, cf_descs, cache);
}
Status CheckOptionsCompatibility(
const std::string& dbpath, Env* env, const DBOptions& db_options,
const std::vector& cf_descs,
bool ignore_unknown_options) {
+ ConfigOptions config_options;
+ config_options.sanity_level = ConfigOptions::kSanityLevelLooselyCompatible;
+ config_options.ignore_unknown_options = ignore_unknown_options;
+ config_options.input_strings_escaped = true;
+ config_options.env = env;
+ return CheckOptionsCompatibility(config_options, dbpath, db_options,
+ cf_descs);
+}
+
+Status CheckOptionsCompatibility(
+ const ConfigOptions& config_options, const std::string& dbpath,
+ const DBOptions& db_options,
+ const std::vector& cf_descs) {
std::string options_file_name;
- Status s = GetLatestOptionsFileName(dbpath, env, &options_file_name);
+ Status s =
+ GetLatestOptionsFileName(dbpath, config_options.env, &options_file_name);
if (!s.ok()) {
return s;
}
@@ -102,12 +143,12 @@ Status CheckOptionsCompatibility(
cf_opts.push_back(cf_desc.options);
}
- const OptionsSanityCheckLevel kDefaultLevel = kSanityLevelLooselyCompatible;
- LegacyFileSystemWrapper fs(env);
+ LegacyFileSystemWrapper fs(config_options.env);
return RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
- db_options, cf_names, cf_opts, dbpath + "/" + options_file_name, &fs,
- kDefaultLevel, ignore_unknown_options);
+
+ config_options, db_options, cf_names, cf_opts,
+ dbpath + "/" + options_file_name, &fs);
}
} // namespace ROCKSDB_NAMESPACE
diff --git a/utilities/options/options_util_test.cc b/utilities/options/options_util_test.cc
index 30ad76a99..0062ad1cd 100644
--- a/utilities/options/options_util_test.cc
+++ b/utilities/options/options_util_test.cc
@@ -5,15 +5,16 @@
#ifndef ROCKSDB_LITE
-#include
+#include "rocksdb/utilities/options_util.h"
#include
+#include
#include
#include "options/options_parser.h"
+#include "rocksdb/convenience.h"
#include "rocksdb/db.h"
#include "rocksdb/table.h"
-#include "rocksdb/utilities/options_util.h"
#include "test_util/testharness.h"
#include "test_util/testutil.h"
#include "util/random.h"
@@ -67,23 +68,26 @@ TEST_F(OptionsUtilTest, SaveAndLoad) {
std::vector loaded_cf_descs;
ASSERT_OK(LoadOptionsFromFile(kFileName, env_.get(), &loaded_db_opt,
&loaded_cf_descs));
-
- ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(db_opt, loaded_db_opt));
+ ConfigOptions exact;
+ exact.sanity_level = ConfigOptions::kSanityLevelExactMatch;
+ ASSERT_OK(
+ RocksDBOptionsParser::VerifyDBOptions(exact, db_opt, loaded_db_opt));
test::RandomInitDBOptions(&db_opt, &rnd_);
- ASSERT_NOK(RocksDBOptionsParser::VerifyDBOptions(db_opt, loaded_db_opt));
+ ASSERT_NOK(
+ RocksDBOptionsParser::VerifyDBOptions(exact, db_opt, loaded_db_opt));
for (size_t i = 0; i < kCFCount; ++i) {
ASSERT_EQ(cf_names[i], loaded_cf_descs[i].name);
ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(
- cf_opts[i], loaded_cf_descs[i].options));
+ exact, cf_opts[i], loaded_cf_descs[i].options));
if (IsBlockBasedTableFactory(cf_opts[i].table_factory.get())) {
ASSERT_OK(RocksDBOptionsParser::VerifyTableFactory(
- cf_opts[i].table_factory.get(),
+ exact, cf_opts[i].table_factory.get(),
loaded_cf_descs[i].options.table_factory.get()));
}
test::RandomInitCFOptions(&cf_opts[i], db_opt, &rnd_);
ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(
- cf_opts[i], loaded_cf_descs[i].options));
+ exact, cf_opts[i], loaded_cf_descs[i].options));
}
for (size_t i = 0; i < kCFCount; ++i) {
@@ -125,6 +129,25 @@ TEST_F(OptionsUtilTest, SaveAndLoadWithCacheCheck) {
PersistRocksDBOptions(db_opt, cf_names, cf_opts, kFileName, fs_.get());
DBOptions loaded_db_opt;
std::vector loaded_cf_descs;
+
+ ConfigOptions config_options;
+ config_options.ignore_unknown_options = false;
+ config_options.input_strings_escaped = true;
+ config_options.env = env_.get();
+ ASSERT_OK(LoadOptionsFromFile(config_options, kFileName, &loaded_db_opt,
+ &loaded_cf_descs, &cache));
+ for (size_t i = 0; i < loaded_cf_descs.size(); i++) {
+ if (IsBlockBasedTableFactory(cf_opts[i].table_factory.get())) {
+ auto* loaded_bbt_opt = reinterpret_cast(
+ loaded_cf_descs[i].options.table_factory->GetOptions());
+ // Expect the same cache will be loaded
+ if (loaded_bbt_opt != nullptr) {
+ ASSERT_EQ(loaded_bbt_opt->block_cache.get(), cache.get());
+ }
+ }
+ }
+
+ // Test the old interface
ASSERT_OK(LoadOptionsFromFile(kFileName, env_.get(), &loaded_db_opt,
&loaded_cf_descs, false, &cache));
for (size_t i = 0; i < loaded_cf_descs.size(); i++) {
@@ -170,8 +193,8 @@ class DummyTableFactory : public TableFactory {
std::string GetPrintableTableOptions() const override { return ""; }
- Status GetOptionString(std::string* /*opt_string*/,
- const std::string& /*delimiter*/) const override {
+ Status GetOptionString(const ConfigOptions& /*opts*/,
+ std::string* /*opt_string*/) const override {
return Status::OK();
}
};
@@ -248,9 +271,13 @@ TEST_F(OptionsUtilTest, SanityCheck) {
}
delete db;
+ ConfigOptions config_options;
+ config_options.ignore_unknown_options = false;
+ config_options.input_strings_escaped = true;
+ config_options.sanity_level = ConfigOptions::kSanityLevelLooselyCompatible;
// perform sanity check
ASSERT_OK(
- CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs));
+ CheckOptionsCompatibility(config_options, dbname_, db_opt, cf_descs));
ASSERT_GE(kCFCount, 5);
// merge operator
@@ -261,15 +288,15 @@ TEST_F(OptionsUtilTest, SanityCheck) {
ASSERT_NE(merge_op.get(), nullptr);
cf_descs[0].options.merge_operator.reset();
ASSERT_NOK(
- CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs));
+ CheckOptionsCompatibility(config_options, dbname_, db_opt, cf_descs));
cf_descs[0].options.merge_operator.reset(new DummyMergeOperator());
ASSERT_NOK(
- CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs));
+ CheckOptionsCompatibility(config_options, dbname_, db_opt, cf_descs));
cf_descs[0].options.merge_operator = merge_op;
ASSERT_OK(
- CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs));
+ CheckOptionsCompatibility(config_options, dbname_, db_opt, cf_descs));
}
// prefix extractor
@@ -281,15 +308,15 @@ TEST_F(OptionsUtilTest, SanityCheck) {
ASSERT_NE(prefix_extractor, nullptr);
cf_descs[1].options.prefix_extractor.reset();
ASSERT_OK(
- CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs));
+ CheckOptionsCompatibility(config_options, dbname_, db_opt, cf_descs));
cf_descs[1].options.prefix_extractor.reset(new DummySliceTransform());
ASSERT_OK(
- CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs));
+ CheckOptionsCompatibility(config_options, dbname_, db_opt, cf_descs));
cf_descs[1].options.prefix_extractor = prefix_extractor;
ASSERT_OK(
- CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs));
+ CheckOptionsCompatibility(config_options, dbname_, db_opt, cf_descs));
}
// prefix extractor nullptr case
@@ -301,16 +328,16 @@ TEST_F(OptionsUtilTest, SanityCheck) {
ASSERT_EQ(prefix_extractor, nullptr);
cf_descs[0].options.prefix_extractor.reset();
ASSERT_OK(
- CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs));
+ CheckOptionsCompatibility(config_options, dbname_, db_opt, cf_descs));
// It's okay to change prefix_extractor from nullptr to non-nullptr
cf_descs[0].options.prefix_extractor.reset(new DummySliceTransform());
ASSERT_OK(
- CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs));
+ CheckOptionsCompatibility(config_options, dbname_, db_opt, cf_descs));
cf_descs[0].options.prefix_extractor = prefix_extractor;
ASSERT_OK(
- CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs));
+ CheckOptionsCompatibility(config_options, dbname_, db_opt, cf_descs));
}
// comparator
@@ -320,11 +347,11 @@ TEST_F(OptionsUtilTest, SanityCheck) {
auto* prev_comparator = cf_descs[2].options.comparator;
cf_descs[2].options.comparator = &comparator;
ASSERT_NOK(
- CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs));
+ CheckOptionsCompatibility(config_options, dbname_, db_opt, cf_descs));
cf_descs[2].options.comparator = prev_comparator;
ASSERT_OK(
- CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs));
+ CheckOptionsCompatibility(config_options, dbname_, db_opt, cf_descs));
}
// table factory
@@ -335,11 +362,11 @@ TEST_F(OptionsUtilTest, SanityCheck) {
ASSERT_NE(table_factory, nullptr);
cf_descs[3].options.table_factory.reset(new DummyTableFactory());
ASSERT_NOK(
- CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs));
+ CheckOptionsCompatibility(config_options, dbname_, db_opt, cf_descs));
cf_descs[3].options.table_factory = table_factory;
ASSERT_OK(
- CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs));
+ CheckOptionsCompatibility(config_options, dbname_, db_opt, cf_descs));
}
}