2014-05-16 19:35:41 +02:00
|
|
|
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
|
|
|
// This source code is licensed under the BSD-style license found in the
|
|
|
|
// LICENSE file in the root directory of this source tree. An additional grant
|
|
|
|
// of patent rights can be found in the PATENTS file in the same directory.
|
|
|
|
//
|
|
|
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
|
|
|
// Use of this source code is governed by a BSD-style license that can be
|
|
|
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
|
|
|
|
2014-09-05 08:14:37 +02:00
|
|
|
#ifndef __STDC_FORMAT_MACROS
|
2014-05-16 19:35:41 +02:00
|
|
|
#define __STDC_FORMAT_MACROS
|
2014-09-05 08:14:37 +02:00
|
|
|
#endif
|
|
|
|
|
RocksDB Options file format and its serialization / deserialization.
Summary:
This patch defines the format of RocksDB options file, which
follows the INI file format, and implements functions for its
serialization and deserialization. An example RocksDB options
file can be found in examples/rocksdb_option_file_example.ini.
A typical RocksDB options file has three sections, which are
Version, DBOptions, and more than one CFOptions. The RocksDB
options file in general follows the basic INI file format
with the following extensions / modifications:
* Escaped characters
We escaped the following characters:
- \n -- line feed - new line
- \r -- carriage return
- \\ -- backslash \
- \: -- colon symbol :
- \# -- hash tag #
* Comments
We support # style comments. Comments can appear at the ending
part of a line.
* Statements
A statement is of the form option_name = value.
Each statement contains a '=', where extra white-spaces
are supported. However, we don't support multi-lined statement.
Furthermore, each line can only contain at most one statement.
* Section
Sections are of the form [SecitonTitle "SectionArgument"],
where section argument is optional.
* List
We use colon-separated string to represent a list.
For instance, n1:n2:n3:n4 is a list containing four values.
Below is an example of a RocksDB options file:
[Version]
rocksdb_version=4.0.0
options_file_version=1.0
[DBOptions]
max_open_files=12345
max_background_flushes=301
[CFOptions "default"]
[CFOptions "the second column family"]
[CFOptions "the third column family"]
Test Plan: Added many tests in options_test.cc
Reviewers: igor, IslamAbdelRahman, sdong, anthony
Reviewed By: anthony
Subscribers: maykov, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D46059
2015-09-29 23:42:40 +02:00
|
|
|
#include <cctype>
|
2014-09-17 21:46:32 +02:00
|
|
|
#include <unordered_map>
|
2014-05-16 19:35:41 +02:00
|
|
|
#include <inttypes.h>
|
|
|
|
|
2014-12-22 22:18:57 +01:00
|
|
|
#include "rocksdb/cache.h"
|
2015-10-03 00:35:32 +02:00
|
|
|
#include "rocksdb/compaction_filter.h"
|
2015-07-15 23:51:51 +02:00
|
|
|
#include "rocksdb/convenience.h"
|
2015-10-03 00:35:32 +02:00
|
|
|
#include "rocksdb/merge_operator.h"
|
2014-05-16 19:35:41 +02:00
|
|
|
#include "rocksdb/options.h"
|
2014-11-21 04:24:39 +01:00
|
|
|
#include "rocksdb/table.h"
|
2014-12-22 22:18:57 +01:00
|
|
|
#include "rocksdb/utilities/leveldb_options.h"
|
2014-11-21 04:24:39 +01:00
|
|
|
#include "table/block_based_table_factory.h"
|
RocksDB Options file format and its serialization / deserialization.
Summary:
This patch defines the format of RocksDB options file, which
follows the INI file format, and implements functions for its
serialization and deserialization. An example RocksDB options
file can be found in examples/rocksdb_option_file_example.ini.
A typical RocksDB options file has three sections, which are
Version, DBOptions, and more than one CFOptions. The RocksDB
options file in general follows the basic INI file format
with the following extensions / modifications:
* Escaped characters
We escaped the following characters:
- \n -- line feed - new line
- \r -- carriage return
- \\ -- backslash \
- \: -- colon symbol :
- \# -- hash tag #
* Comments
We support # style comments. Comments can appear at the ending
part of a line.
* Statements
A statement is of the form option_name = value.
Each statement contains a '=', where extra white-spaces
are supported. However, we don't support multi-lined statement.
Furthermore, each line can only contain at most one statement.
* Section
Sections are of the form [SecitonTitle "SectionArgument"],
where section argument is optional.
* List
We use colon-separated string to represent a list.
For instance, n1:n2:n3:n4 is a list containing four values.
Below is an example of a RocksDB options file:
[Version]
rocksdb_version=4.0.0
options_file_version=1.0
[DBOptions]
max_open_files=12345
max_background_flushes=301
[CFOptions "default"]
[CFOptions "the second column family"]
[CFOptions "the third column family"]
Test Plan: Added many tests in options_test.cc
Reviewers: igor, IslamAbdelRahman, sdong, anthony
Reviewed By: anthony
Subscribers: maykov, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D46059
2015-09-29 23:42:40 +02:00
|
|
|
#include "util/options_helper.h"
|
|
|
|
#include "util/options_parser.h"
|
2015-03-20 01:06:02 +01:00
|
|
|
#include "util/random.h"
|
2014-05-16 19:35:41 +02:00
|
|
|
#include "util/testharness.h"
|
RocksDB Options file format and its serialization / deserialization.
Summary:
This patch defines the format of RocksDB options file, which
follows the INI file format, and implements functions for its
serialization and deserialization. An example RocksDB options
file can be found in examples/rocksdb_option_file_example.ini.
A typical RocksDB options file has three sections, which are
Version, DBOptions, and more than one CFOptions. The RocksDB
options file in general follows the basic INI file format
with the following extensions / modifications:
* Escaped characters
We escaped the following characters:
- \n -- line feed - new line
- \r -- carriage return
- \\ -- backslash \
- \: -- colon symbol :
- \# -- hash tag #
* Comments
We support # style comments. Comments can appear at the ending
part of a line.
* Statements
A statement is of the form option_name = value.
Each statement contains a '=', where extra white-spaces
are supported. However, we don't support multi-lined statement.
Furthermore, each line can only contain at most one statement.
* Section
Sections are of the form [SecitonTitle "SectionArgument"],
where section argument is optional.
* List
We use colon-separated string to represent a list.
For instance, n1:n2:n3:n4 is a list containing four values.
Below is an example of a RocksDB options file:
[Version]
rocksdb_version=4.0.0
options_file_version=1.0
[DBOptions]
max_open_files=12345
max_background_flushes=301
[CFOptions "default"]
[CFOptions "the second column family"]
[CFOptions "the third column family"]
Test Plan: Added many tests in options_test.cc
Reviewers: igor, IslamAbdelRahman, sdong, anthony
Reviewed By: anthony
Subscribers: maykov, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D46059
2015-09-29 23:42:40 +02:00
|
|
|
#include "util/testutil.h"
|
2014-05-16 19:35:41 +02:00
|
|
|
|
2014-11-24 21:53:23 +01:00
|
|
|
#ifndef GFLAGS
|
|
|
|
bool FLAGS_enable_print = false;
|
|
|
|
#else
|
|
|
|
#include <gflags/gflags.h>
|
2014-05-16 19:35:41 +02:00
|
|
|
using GFLAGS::ParseCommandLineFlags;
|
|
|
|
DEFINE_bool(enable_print, false, "Print options generated to console.");
|
2014-11-24 21:53:23 +01:00
|
|
|
#endif // GFLAGS
|
2014-05-16 19:35:41 +02:00
|
|
|
|
|
|
|
namespace rocksdb {
|
|
|
|
|
|
|
|
class StderrLogger : public Logger {
|
|
|
|
public:
|
2015-02-01 20:08:19 +01:00
|
|
|
using Logger::Logv;
|
2014-05-16 19:35:41 +02:00
|
|
|
virtual void Logv(const char* format, va_list ap) override {
|
|
|
|
vprintf(format, ap);
|
|
|
|
printf("\n");
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
Options PrintAndGetOptions(size_t total_write_buffer_limit,
|
|
|
|
int read_amplification_threshold,
|
|
|
|
int write_amplification_threshold,
|
|
|
|
uint64_t target_db_size = 68719476736) {
|
|
|
|
StderrLogger logger;
|
|
|
|
|
|
|
|
if (FLAGS_enable_print) {
|
2015-07-13 21:11:05 +02:00
|
|
|
printf("---- total_write_buffer_limit: %" ROCKSDB_PRIszt
|
|
|
|
" "
|
|
|
|
"read_amplification_threshold: %d write_amplification_threshold: %d "
|
|
|
|
"target_db_size %" PRIu64 " ----\n",
|
|
|
|
total_write_buffer_limit, read_amplification_threshold,
|
|
|
|
write_amplification_threshold, target_db_size);
|
2014-05-16 19:35:41 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
Options options =
|
|
|
|
GetOptions(total_write_buffer_limit, read_amplification_threshold,
|
|
|
|
write_amplification_threshold, target_db_size);
|
|
|
|
if (FLAGS_enable_print) {
|
|
|
|
options.Dump(&logger);
|
|
|
|
printf("-------------------------------------\n\n\n");
|
|
|
|
}
|
|
|
|
return options;
|
|
|
|
}
|
|
|
|
|
RocksDB Options file format and its serialization / deserialization.
Summary:
This patch defines the format of RocksDB options file, which
follows the INI file format, and implements functions for its
serialization and deserialization. An example RocksDB options
file can be found in examples/rocksdb_option_file_example.ini.
A typical RocksDB options file has three sections, which are
Version, DBOptions, and more than one CFOptions. The RocksDB
options file in general follows the basic INI file format
with the following extensions / modifications:
* Escaped characters
We escaped the following characters:
- \n -- line feed - new line
- \r -- carriage return
- \\ -- backslash \
- \: -- colon symbol :
- \# -- hash tag #
* Comments
We support # style comments. Comments can appear at the ending
part of a line.
* Statements
A statement is of the form option_name = value.
Each statement contains a '=', where extra white-spaces
are supported. However, we don't support multi-lined statement.
Furthermore, each line can only contain at most one statement.
* Section
Sections are of the form [SecitonTitle "SectionArgument"],
where section argument is optional.
* List
We use colon-separated string to represent a list.
For instance, n1:n2:n3:n4 is a list containing four values.
Below is an example of a RocksDB options file:
[Version]
rocksdb_version=4.0.0
options_file_version=1.0
[DBOptions]
max_open_files=12345
max_background_flushes=301
[CFOptions "default"]
[CFOptions "the second column family"]
[CFOptions "the third column family"]
Test Plan: Added many tests in options_test.cc
Reviewers: igor, IslamAbdelRahman, sdong, anthony
Reviewed By: anthony
Subscribers: maykov, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D46059
2015-09-29 23:42:40 +02:00
|
|
|
class StringEnv : public EnvWrapper {
|
|
|
|
public:
|
|
|
|
class SeqStringSource : public SequentialFile {
|
|
|
|
public:
|
|
|
|
explicit SeqStringSource(const std::string& data)
|
|
|
|
: data_(data), offset_(0) {}
|
|
|
|
~SeqStringSource() {}
|
|
|
|
Status Read(size_t n, Slice* result, char* scratch) override {
|
|
|
|
std::string output;
|
|
|
|
if (offset_ < data_.size()) {
|
|
|
|
n = std::min(data_.size() - offset_, n);
|
|
|
|
memcpy(scratch, data_.data() + offset_, n);
|
|
|
|
offset_ += n;
|
|
|
|
*result = Slice(scratch, n);
|
|
|
|
} else {
|
|
|
|
return Status::InvalidArgument(
|
|
|
|
"Attemp to read when it already reached eof.");
|
|
|
|
}
|
|
|
|
return Status::OK();
|
|
|
|
}
|
2015-09-29 23:56:54 +02:00
|
|
|
Status Skip(uint64_t n) override {
|
RocksDB Options file format and its serialization / deserialization.
Summary:
This patch defines the format of RocksDB options file, which
follows the INI file format, and implements functions for its
serialization and deserialization. An example RocksDB options
file can be found in examples/rocksdb_option_file_example.ini.
A typical RocksDB options file has three sections, which are
Version, DBOptions, and more than one CFOptions. The RocksDB
options file in general follows the basic INI file format
with the following extensions / modifications:
* Escaped characters
We escaped the following characters:
- \n -- line feed - new line
- \r -- carriage return
- \\ -- backslash \
- \: -- colon symbol :
- \# -- hash tag #
* Comments
We support # style comments. Comments can appear at the ending
part of a line.
* Statements
A statement is of the form option_name = value.
Each statement contains a '=', where extra white-spaces
are supported. However, we don't support multi-lined statement.
Furthermore, each line can only contain at most one statement.
* Section
Sections are of the form [SecitonTitle "SectionArgument"],
where section argument is optional.
* List
We use colon-separated string to represent a list.
For instance, n1:n2:n3:n4 is a list containing four values.
Below is an example of a RocksDB options file:
[Version]
rocksdb_version=4.0.0
options_file_version=1.0
[DBOptions]
max_open_files=12345
max_background_flushes=301
[CFOptions "default"]
[CFOptions "the second column family"]
[CFOptions "the third column family"]
Test Plan: Added many tests in options_test.cc
Reviewers: igor, IslamAbdelRahman, sdong, anthony
Reviewed By: anthony
Subscribers: maykov, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D46059
2015-09-29 23:42:40 +02:00
|
|
|
if (offset_ >= data_.size()) {
|
|
|
|
return Status::InvalidArgument(
|
|
|
|
"Attemp to read when it already reached eof.");
|
|
|
|
}
|
|
|
|
// TODO(yhchiang): Currently doesn't handle the overflow case.
|
|
|
|
offset_ += n;
|
|
|
|
return Status::OK();
|
|
|
|
}
|
|
|
|
|
|
|
|
private:
|
|
|
|
std::string data_;
|
|
|
|
size_t offset_;
|
|
|
|
};
|
|
|
|
|
|
|
|
class StringSink : public WritableFile {
|
|
|
|
public:
|
|
|
|
explicit StringSink(std::string* contents)
|
|
|
|
: WritableFile(), contents_(contents) {}
|
|
|
|
virtual Status Truncate(uint64_t size) override {
|
|
|
|
contents_->resize(size);
|
|
|
|
return Status::OK();
|
|
|
|
}
|
|
|
|
virtual Status Close() override { return Status::OK(); }
|
|
|
|
virtual Status Flush() override { return Status::OK(); }
|
|
|
|
virtual Status Sync() override { return Status::OK(); }
|
|
|
|
virtual Status Append(const Slice& slice) override {
|
|
|
|
contents_->append(slice.data(), slice.size());
|
|
|
|
return Status::OK();
|
|
|
|
}
|
|
|
|
|
|
|
|
private:
|
|
|
|
std::string* contents_;
|
|
|
|
};
|
|
|
|
|
|
|
|
explicit StringEnv(Env* t) : EnvWrapper(t) {}
|
|
|
|
virtual ~StringEnv() {}
|
|
|
|
|
|
|
|
const std::string& GetContent(const std::string& f) { return files_[f]; }
|
|
|
|
|
|
|
|
const Status WriteToNewFile(const std::string& file_name,
|
|
|
|
const std::string& content) {
|
|
|
|
unique_ptr<WritableFile> r;
|
|
|
|
auto s = NewWritableFile(file_name, &r, EnvOptions());
|
|
|
|
if (!s.ok()) {
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
r->Append(content);
|
|
|
|
r->Flush();
|
|
|
|
r->Close();
|
|
|
|
assert(files_[file_name] == content);
|
|
|
|
return Status::OK();
|
|
|
|
}
|
|
|
|
|
|
|
|
// The following text is boilerplate that forwards all methods to target()
|
|
|
|
Status NewSequentialFile(const std::string& f, unique_ptr<SequentialFile>* r,
|
|
|
|
const EnvOptions& options) override {
|
|
|
|
auto iter = files_.find(f);
|
|
|
|
if (iter == files_.end()) {
|
|
|
|
return Status::NotFound("The specified file does not exist", f);
|
|
|
|
}
|
|
|
|
r->reset(new SeqStringSource(iter->second));
|
|
|
|
return Status::OK();
|
|
|
|
}
|
|
|
|
Status NewRandomAccessFile(const std::string& f,
|
|
|
|
unique_ptr<RandomAccessFile>* r,
|
|
|
|
const EnvOptions& options) override {
|
|
|
|
return Status::NotSupported();
|
|
|
|
}
|
|
|
|
Status NewWritableFile(const std::string& f, unique_ptr<WritableFile>* r,
|
|
|
|
const EnvOptions& options) override {
|
|
|
|
auto iter = files_.find(f);
|
|
|
|
if (iter != files_.end()) {
|
|
|
|
return Status::IOError("The specified file already exists", f);
|
|
|
|
}
|
|
|
|
r->reset(new StringSink(&files_[f]));
|
|
|
|
return Status::OK();
|
|
|
|
}
|
|
|
|
virtual Status NewDirectory(const std::string& name,
|
|
|
|
unique_ptr<Directory>* result) override {
|
|
|
|
return Status::NotSupported();
|
|
|
|
}
|
|
|
|
Status FileExists(const std::string& f) override {
|
|
|
|
if (files_.find(f) == files_.end()) {
|
|
|
|
return Status::NotFound();
|
|
|
|
}
|
|
|
|
return Status::OK();
|
|
|
|
}
|
|
|
|
Status GetChildren(const std::string& dir,
|
|
|
|
std::vector<std::string>* r) override {
|
|
|
|
return Status::NotSupported();
|
|
|
|
}
|
|
|
|
Status DeleteFile(const std::string& f) override {
|
|
|
|
files_.erase(f);
|
|
|
|
return Status::OK();
|
|
|
|
}
|
|
|
|
Status CreateDir(const std::string& d) override {
|
|
|
|
return Status::NotSupported();
|
|
|
|
}
|
|
|
|
Status CreateDirIfMissing(const std::string& d) override {
|
|
|
|
return Status::NotSupported();
|
|
|
|
}
|
|
|
|
Status DeleteDir(const std::string& d) override {
|
|
|
|
return Status::NotSupported();
|
|
|
|
}
|
|
|
|
Status GetFileSize(const std::string& f, uint64_t* s) override {
|
|
|
|
auto iter = files_.find(f);
|
|
|
|
if (iter == files_.end()) {
|
|
|
|
return Status::NotFound("The specified file does not exist:", f);
|
|
|
|
}
|
|
|
|
*s = iter->second.size();
|
|
|
|
return Status::OK();
|
|
|
|
}
|
|
|
|
|
|
|
|
Status GetFileModificationTime(const std::string& fname,
|
|
|
|
uint64_t* file_mtime) override {
|
|
|
|
return Status::NotSupported();
|
|
|
|
}
|
|
|
|
|
|
|
|
Status RenameFile(const std::string& s, const std::string& t) override {
|
|
|
|
return Status::NotSupported();
|
|
|
|
}
|
|
|
|
|
|
|
|
Status LinkFile(const std::string& s, const std::string& t) override {
|
|
|
|
return Status::NotSupported();
|
|
|
|
}
|
|
|
|
|
|
|
|
Status LockFile(const std::string& f, FileLock** l) override {
|
|
|
|
return Status::NotSupported();
|
|
|
|
}
|
|
|
|
|
|
|
|
Status UnlockFile(FileLock* l) override { return Status::NotSupported(); }
|
|
|
|
|
|
|
|
protected:
|
|
|
|
std::unordered_map<std::string, std::string> files_;
|
|
|
|
};
|
|
|
|
|
|
|
|
class OptionsTest : public testing::Test {};
|
|
|
|
|
2015-03-17 22:08:00 +01:00
|
|
|
TEST_F(OptionsTest, LooseCondition) {
|
2014-05-16 19:35:41 +02:00
|
|
|
Options options;
|
|
|
|
PrintAndGetOptions(static_cast<size_t>(10) * 1024 * 1024 * 1024, 100, 100);
|
|
|
|
|
|
|
|
// Less mem table memory budget
|
|
|
|
PrintAndGetOptions(32 * 1024 * 1024, 100, 100);
|
|
|
|
|
|
|
|
// Tight read amplification
|
|
|
|
options = PrintAndGetOptions(128 * 1024 * 1024, 8, 100);
|
|
|
|
ASSERT_EQ(options.compaction_style, kCompactionStyleLevel);
|
|
|
|
|
2015-01-23 01:57:16 +01:00
|
|
|
#ifndef ROCKSDB_LITE // Universal compaction is not supported in ROCKSDB_LITE
|
2014-05-16 19:35:41 +02:00
|
|
|
// Tight write amplification
|
|
|
|
options = PrintAndGetOptions(128 * 1024 * 1024, 64, 10);
|
|
|
|
ASSERT_EQ(options.compaction_style, kCompactionStyleUniversal);
|
2015-01-23 01:57:16 +01:00
|
|
|
#endif // !ROCKSDB_LITE
|
2014-05-16 19:35:41 +02:00
|
|
|
|
|
|
|
// Both tight amplifications
|
|
|
|
PrintAndGetOptions(128 * 1024 * 1024, 4, 8);
|
|
|
|
}
|
2014-09-17 21:46:32 +02:00
|
|
|
|
2015-01-23 01:57:16 +01:00
|
|
|
#ifndef ROCKSDB_LITE // GetOptionsFromMap is not supported in ROCKSDB_LITE
|
2015-03-17 22:08:00 +01:00
|
|
|
TEST_F(OptionsTest, GetOptionsFromMapTest) {
|
2014-10-10 19:00:12 +02:00
|
|
|
std::unordered_map<std::string, std::string> cf_options_map = {
|
options.level_compaction_dynamic_level_bytes to allow RocksDB to pick size bases of levels dynamically.
Summary:
When having fixed max_bytes_for_level_base, the ratio of size of largest level and the second one can range from 0 to the multiplier. This makes LSM tree frequently irregular and unpredictable. It can also cause poor space amplification in some cases.
In this improvement (proposed by Igor Kabiljo), we introduce a parameter option.level_compaction_use_dynamic_max_bytes. When turning it on, RocksDB is free to pick a level base in the range of (options.max_bytes_for_level_base/options.max_bytes_for_level_multiplier, options.max_bytes_for_level_base] so that real level ratios are close to options.max_bytes_for_level_multiplier.
Test Plan: New unit tests and pass tests suites including valgrind.
Reviewers: MarkCallaghan, rven, yhchiang, igor, ikabiljo
Reviewed By: ikabiljo
Subscribers: yoshinorim, ikabiljo, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D31437
2015-02-05 20:44:17 +01:00
|
|
|
{"write_buffer_size", "1"},
|
|
|
|
{"max_write_buffer_number", "2"},
|
|
|
|
{"min_write_buffer_number_to_merge", "3"},
|
2015-07-03 02:23:41 +02:00
|
|
|
{"max_write_buffer_number_to_maintain", "99"},
|
options.level_compaction_dynamic_level_bytes to allow RocksDB to pick size bases of levels dynamically.
Summary:
When having fixed max_bytes_for_level_base, the ratio of size of largest level and the second one can range from 0 to the multiplier. This makes LSM tree frequently irregular and unpredictable. It can also cause poor space amplification in some cases.
In this improvement (proposed by Igor Kabiljo), we introduce a parameter option.level_compaction_use_dynamic_max_bytes. When turning it on, RocksDB is free to pick a level base in the range of (options.max_bytes_for_level_base/options.max_bytes_for_level_multiplier, options.max_bytes_for_level_base] so that real level ratios are close to options.max_bytes_for_level_multiplier.
Test Plan: New unit tests and pass tests suites including valgrind.
Reviewers: MarkCallaghan, rven, yhchiang, igor, ikabiljo
Reviewed By: ikabiljo
Subscribers: yoshinorim, ikabiljo, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D31437
2015-02-05 20:44:17 +01:00
|
|
|
{"compression", "kSnappyCompression"},
|
|
|
|
{"compression_per_level",
|
|
|
|
"kNoCompression:"
|
|
|
|
"kSnappyCompression:"
|
|
|
|
"kZlibCompression:"
|
|
|
|
"kBZip2Compression:"
|
|
|
|
"kLZ4Compression:"
|
2015-08-28 00:40:42 +02:00
|
|
|
"kLZ4HCCompression:"
|
|
|
|
"kZSTDNotFinalCompression"},
|
options.level_compaction_dynamic_level_bytes to allow RocksDB to pick size bases of levels dynamically.
Summary:
When having fixed max_bytes_for_level_base, the ratio of size of largest level and the second one can range from 0 to the multiplier. This makes LSM tree frequently irregular and unpredictable. It can also cause poor space amplification in some cases.
In this improvement (proposed by Igor Kabiljo), we introduce a parameter option.level_compaction_use_dynamic_max_bytes. When turning it on, RocksDB is free to pick a level base in the range of (options.max_bytes_for_level_base/options.max_bytes_for_level_multiplier, options.max_bytes_for_level_base] so that real level ratios are close to options.max_bytes_for_level_multiplier.
Test Plan: New unit tests and pass tests suites including valgrind.
Reviewers: MarkCallaghan, rven, yhchiang, igor, ikabiljo
Reviewed By: ikabiljo
Subscribers: yoshinorim, ikabiljo, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D31437
2015-02-05 20:44:17 +01:00
|
|
|
{"compression_opts", "4:5:6"},
|
|
|
|
{"num_levels", "7"},
|
|
|
|
{"level0_file_num_compaction_trigger", "8"},
|
|
|
|
{"level0_slowdown_writes_trigger", "9"},
|
|
|
|
{"level0_stop_writes_trigger", "10"},
|
|
|
|
{"target_file_size_base", "12"},
|
|
|
|
{"target_file_size_multiplier", "13"},
|
|
|
|
{"max_bytes_for_level_base", "14"},
|
|
|
|
{"level_compaction_dynamic_level_bytes", "true"},
|
|
|
|
{"max_bytes_for_level_multiplier", "15"},
|
|
|
|
{"max_bytes_for_level_multiplier_additional", "16:17:18"},
|
|
|
|
{"expanded_compaction_factor", "19"},
|
|
|
|
{"source_compaction_factor", "20"},
|
|
|
|
{"max_grandparent_overlap_factor", "21"},
|
|
|
|
{"soft_rate_limit", "1.1"},
|
|
|
|
{"hard_rate_limit", "2.1"},
|
2015-09-11 23:31:23 +02:00
|
|
|
{"hard_pending_compaction_bytes_limit", "211"},
|
options.level_compaction_dynamic_level_bytes to allow RocksDB to pick size bases of levels dynamically.
Summary:
When having fixed max_bytes_for_level_base, the ratio of size of largest level and the second one can range from 0 to the multiplier. This makes LSM tree frequently irregular and unpredictable. It can also cause poor space amplification in some cases.
In this improvement (proposed by Igor Kabiljo), we introduce a parameter option.level_compaction_use_dynamic_max_bytes. When turning it on, RocksDB is free to pick a level base in the range of (options.max_bytes_for_level_base/options.max_bytes_for_level_multiplier, options.max_bytes_for_level_base] so that real level ratios are close to options.max_bytes_for_level_multiplier.
Test Plan: New unit tests and pass tests suites including valgrind.
Reviewers: MarkCallaghan, rven, yhchiang, igor, ikabiljo
Reviewed By: ikabiljo
Subscribers: yoshinorim, ikabiljo, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D31437
2015-02-05 20:44:17 +01:00
|
|
|
{"arena_block_size", "22"},
|
|
|
|
{"disable_auto_compactions", "true"},
|
|
|
|
{"compaction_style", "kCompactionStyleLevel"},
|
|
|
|
{"verify_checksums_in_compaction", "false"},
|
|
|
|
{"compaction_options_fifo", "23"},
|
|
|
|
{"filter_deletes", "0"},
|
|
|
|
{"max_sequential_skip_in_iterations", "24"},
|
|
|
|
{"inplace_update_support", "true"},
|
Add options.compaction_measure_io_stats to print write I/O stats in compactions
Summary:
Add options.compaction_measure_io_stats to print out / pass to listener accumulated time spent on write calls. Example outputs in info logs:
2015/08/12-16:27:59.463944 7fd428bff700 (Original Log Time 2015/08/12-16:27:59.463922) EVENT_LOG_v1 {"time_micros": 1439422079463897, "job": 6, "event": "compaction_finished", "output_level": 1, "num_output_files": 4, "total_output_size": 6900525, "num_input_records": 111483, "num_output_records": 106877, "file_write_nanos": 15663206, "file_range_sync_nanos": 649588, "file_fsync_nanos": 349614797, "file_prepare_write_nanos": 1505812, "lsm_state": [2, 4, 0, 0, 0, 0, 0]}
Add two more counters in iostats_context.
Also add a parameter of db_bench.
Test Plan: Add a unit test. Also manually verify LOG outputs in db_bench
Subscribers: leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D44115
2015-08-13 02:24:45 +02:00
|
|
|
{"compaction_measure_io_stats", "true"},
|
options.level_compaction_dynamic_level_bytes to allow RocksDB to pick size bases of levels dynamically.
Summary:
When having fixed max_bytes_for_level_base, the ratio of size of largest level and the second one can range from 0 to the multiplier. This makes LSM tree frequently irregular and unpredictable. It can also cause poor space amplification in some cases.
In this improvement (proposed by Igor Kabiljo), we introduce a parameter option.level_compaction_use_dynamic_max_bytes. When turning it on, RocksDB is free to pick a level base in the range of (options.max_bytes_for_level_base/options.max_bytes_for_level_multiplier, options.max_bytes_for_level_base] so that real level ratios are close to options.max_bytes_for_level_multiplier.
Test Plan: New unit tests and pass tests suites including valgrind.
Reviewers: MarkCallaghan, rven, yhchiang, igor, ikabiljo
Reviewed By: ikabiljo
Subscribers: yoshinorim, ikabiljo, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D31437
2015-02-05 20:44:17 +01:00
|
|
|
{"inplace_update_num_locks", "25"},
|
|
|
|
{"memtable_prefix_bloom_bits", "26"},
|
|
|
|
{"memtable_prefix_bloom_probes", "27"},
|
|
|
|
{"memtable_prefix_bloom_huge_page_tlb_size", "28"},
|
|
|
|
{"bloom_locality", "29"},
|
|
|
|
{"max_successive_merges", "30"},
|
|
|
|
{"min_partial_merge_operands", "31"},
|
|
|
|
{"prefix_extractor", "fixed:31"},
|
|
|
|
{"optimize_filters_for_hits", "true"},
|
2014-10-10 19:00:12 +02:00
|
|
|
};
|
|
|
|
|
|
|
|
std::unordered_map<std::string, std::string> db_options_map = {
|
2015-08-21 07:33:25 +02:00
|
|
|
{"create_if_missing", "false"},
|
|
|
|
{"create_missing_column_families", "true"},
|
|
|
|
{"error_if_exists", "false"},
|
|
|
|
{"paranoid_checks", "true"},
|
|
|
|
{"max_open_files", "32"},
|
|
|
|
{"max_total_wal_size", "33"},
|
|
|
|
{"disable_data_sync", "false"},
|
|
|
|
{"use_fsync", "true"},
|
|
|
|
{"db_log_dir", "/db_log_dir"},
|
|
|
|
{"wal_dir", "/wal_dir"},
|
|
|
|
{"delete_obsolete_files_period_micros", "34"},
|
|
|
|
{"max_background_compactions", "35"},
|
|
|
|
{"max_background_flushes", "36"},
|
|
|
|
{"max_log_file_size", "37"},
|
|
|
|
{"log_file_time_to_roll", "38"},
|
|
|
|
{"keep_log_file_num", "39"},
|
2015-10-08 03:06:28 +02:00
|
|
|
{"recycle_log_file_num", "5"},
|
2015-08-21 07:33:25 +02:00
|
|
|
{"max_manifest_file_size", "40"},
|
|
|
|
{"table_cache_numshardbits", "41"},
|
|
|
|
{"WAL_ttl_seconds", "43"},
|
|
|
|
{"WAL_size_limit_MB", "44"},
|
|
|
|
{"manifest_preallocation_size", "45"},
|
|
|
|
{"allow_os_buffer", "false"},
|
|
|
|
{"allow_mmap_reads", "true"},
|
|
|
|
{"allow_mmap_writes", "false"},
|
|
|
|
{"is_fd_close_on_exec", "true"},
|
|
|
|
{"skip_log_error_on_recovery", "false"},
|
|
|
|
{"stats_dump_period_sec", "46"},
|
|
|
|
{"advise_random_on_open", "true"},
|
|
|
|
{"use_adaptive_mutex", "false"},
|
|
|
|
{"new_table_reader_for_compaction_inputs", "true"},
|
2015-08-27 00:25:59 +02:00
|
|
|
{"compaction_readahead_size", "100"},
|
2015-10-29 23:52:32 +01:00
|
|
|
{"random_access_max_buffer_size", "3145728"},
|
2015-08-21 07:33:25 +02:00
|
|
|
{"bytes_per_sync", "47"},
|
2015-10-08 03:06:28 +02:00
|
|
|
{"wal_bytes_per_sync", "48"},
|
|
|
|
};
|
2014-09-17 21:46:32 +02:00
|
|
|
|
2014-10-10 19:00:12 +02:00
|
|
|
ColumnFamilyOptions base_cf_opt;
|
|
|
|
ColumnFamilyOptions new_cf_opt;
|
2014-12-22 22:18:57 +01:00
|
|
|
ASSERT_OK(GetColumnFamilyOptionsFromMap(
|
|
|
|
base_cf_opt, cf_options_map, &new_cf_opt));
|
2014-10-10 19:00:12 +02:00
|
|
|
ASSERT_EQ(new_cf_opt.write_buffer_size, 1U);
|
|
|
|
ASSERT_EQ(new_cf_opt.max_write_buffer_number, 2);
|
|
|
|
ASSERT_EQ(new_cf_opt.min_write_buffer_number_to_merge, 3);
|
2015-07-03 02:23:41 +02:00
|
|
|
ASSERT_EQ(new_cf_opt.max_write_buffer_number_to_maintain, 99);
|
2014-10-10 19:00:12 +02:00
|
|
|
ASSERT_EQ(new_cf_opt.compression, kSnappyCompression);
|
2015-08-28 00:40:42 +02:00
|
|
|
ASSERT_EQ(new_cf_opt.compression_per_level.size(), 7U);
|
2014-10-10 19:00:12 +02:00
|
|
|
ASSERT_EQ(new_cf_opt.compression_per_level[0], kNoCompression);
|
|
|
|
ASSERT_EQ(new_cf_opt.compression_per_level[1], kSnappyCompression);
|
|
|
|
ASSERT_EQ(new_cf_opt.compression_per_level[2], kZlibCompression);
|
|
|
|
ASSERT_EQ(new_cf_opt.compression_per_level[3], kBZip2Compression);
|
|
|
|
ASSERT_EQ(new_cf_opt.compression_per_level[4], kLZ4Compression);
|
|
|
|
ASSERT_EQ(new_cf_opt.compression_per_level[5], kLZ4HCCompression);
|
2015-08-28 00:40:42 +02:00
|
|
|
ASSERT_EQ(new_cf_opt.compression_per_level[6], kZSTDNotFinalCompression);
|
2014-10-10 19:00:12 +02:00
|
|
|
ASSERT_EQ(new_cf_opt.compression_opts.window_bits, 4);
|
|
|
|
ASSERT_EQ(new_cf_opt.compression_opts.level, 5);
|
|
|
|
ASSERT_EQ(new_cf_opt.compression_opts.strategy, 6);
|
|
|
|
ASSERT_EQ(new_cf_opt.num_levels, 7);
|
|
|
|
ASSERT_EQ(new_cf_opt.level0_file_num_compaction_trigger, 8);
|
|
|
|
ASSERT_EQ(new_cf_opt.level0_slowdown_writes_trigger, 9);
|
|
|
|
ASSERT_EQ(new_cf_opt.level0_stop_writes_trigger, 10);
|
|
|
|
ASSERT_EQ(new_cf_opt.target_file_size_base, static_cast<uint64_t>(12));
|
|
|
|
ASSERT_EQ(new_cf_opt.target_file_size_multiplier, 13);
|
|
|
|
ASSERT_EQ(new_cf_opt.max_bytes_for_level_base, 14U);
|
options.level_compaction_dynamic_level_bytes to allow RocksDB to pick size bases of levels dynamically.
Summary:
When having fixed max_bytes_for_level_base, the ratio of size of largest level and the second one can range from 0 to the multiplier. This makes LSM tree frequently irregular and unpredictable. It can also cause poor space amplification in some cases.
In this improvement (proposed by Igor Kabiljo), we introduce a parameter option.level_compaction_use_dynamic_max_bytes. When turning it on, RocksDB is free to pick a level base in the range of (options.max_bytes_for_level_base/options.max_bytes_for_level_multiplier, options.max_bytes_for_level_base] so that real level ratios are close to options.max_bytes_for_level_multiplier.
Test Plan: New unit tests and pass tests suites including valgrind.
Reviewers: MarkCallaghan, rven, yhchiang, igor, ikabiljo
Reviewed By: ikabiljo
Subscribers: yoshinorim, ikabiljo, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D31437
2015-02-05 20:44:17 +01:00
|
|
|
ASSERT_EQ(new_cf_opt.level_compaction_dynamic_level_bytes, true);
|
2014-10-10 19:00:12 +02:00
|
|
|
ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier, 15);
|
|
|
|
ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier_additional.size(), 3U);
|
|
|
|
ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier_additional[0], 16);
|
|
|
|
ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier_additional[1], 17);
|
|
|
|
ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier_additional[2], 18);
|
|
|
|
ASSERT_EQ(new_cf_opt.expanded_compaction_factor, 19);
|
|
|
|
ASSERT_EQ(new_cf_opt.source_compaction_factor, 20);
|
|
|
|
ASSERT_EQ(new_cf_opt.max_grandparent_overlap_factor, 21);
|
|
|
|
ASSERT_EQ(new_cf_opt.soft_rate_limit, 1.1);
|
2015-09-11 23:31:23 +02:00
|
|
|
ASSERT_EQ(new_cf_opt.hard_pending_compaction_bytes_limit, 211);
|
2014-10-10 19:00:12 +02:00
|
|
|
ASSERT_EQ(new_cf_opt.arena_block_size, 22U);
|
|
|
|
ASSERT_EQ(new_cf_opt.disable_auto_compactions, true);
|
|
|
|
ASSERT_EQ(new_cf_opt.compaction_style, kCompactionStyleLevel);
|
|
|
|
ASSERT_EQ(new_cf_opt.verify_checksums_in_compaction, false);
|
|
|
|
ASSERT_EQ(new_cf_opt.compaction_options_fifo.max_table_files_size,
|
2014-09-18 00:40:25 +02:00
|
|
|
static_cast<uint64_t>(23));
|
2014-10-10 19:00:12 +02:00
|
|
|
ASSERT_EQ(new_cf_opt.filter_deletes, false);
|
|
|
|
ASSERT_EQ(new_cf_opt.max_sequential_skip_in_iterations,
|
2014-09-18 00:40:25 +02:00
|
|
|
static_cast<uint64_t>(24));
|
2014-10-10 19:00:12 +02:00
|
|
|
ASSERT_EQ(new_cf_opt.inplace_update_support, true);
|
|
|
|
ASSERT_EQ(new_cf_opt.inplace_update_num_locks, 25U);
|
|
|
|
ASSERT_EQ(new_cf_opt.memtable_prefix_bloom_bits, 26U);
|
|
|
|
ASSERT_EQ(new_cf_opt.memtable_prefix_bloom_probes, 27U);
|
|
|
|
ASSERT_EQ(new_cf_opt.memtable_prefix_bloom_huge_page_tlb_size, 28U);
|
|
|
|
ASSERT_EQ(new_cf_opt.bloom_locality, 29U);
|
|
|
|
ASSERT_EQ(new_cf_opt.max_successive_merges, 30U);
|
|
|
|
ASSERT_EQ(new_cf_opt.min_partial_merge_operands, 31U);
|
2015-01-16 00:33:12 +01:00
|
|
|
ASSERT_TRUE(new_cf_opt.prefix_extractor != nullptr);
|
2015-02-17 17:03:45 +01:00
|
|
|
ASSERT_EQ(new_cf_opt.optimize_filters_for_hits, true);
|
2015-01-16 00:33:12 +01:00
|
|
|
ASSERT_EQ(std::string(new_cf_opt.prefix_extractor->Name()),
|
|
|
|
"rocksdb.FixedPrefix.31");
|
2014-10-10 19:00:12 +02:00
|
|
|
|
|
|
|
cf_options_map["write_buffer_size"] = "hello";
|
2014-12-22 22:18:57 +01:00
|
|
|
ASSERT_NOK(GetColumnFamilyOptionsFromMap(
|
|
|
|
base_cf_opt, cf_options_map, &new_cf_opt));
|
2014-10-10 19:00:12 +02:00
|
|
|
cf_options_map["write_buffer_size"] = "1";
|
2014-12-22 22:18:57 +01:00
|
|
|
ASSERT_OK(GetColumnFamilyOptionsFromMap(
|
|
|
|
base_cf_opt, cf_options_map, &new_cf_opt));
|
2014-10-10 19:00:12 +02:00
|
|
|
cf_options_map["unknown_option"] = "1";
|
2014-12-22 22:18:57 +01:00
|
|
|
ASSERT_NOK(GetColumnFamilyOptionsFromMap(
|
|
|
|
base_cf_opt, cf_options_map, &new_cf_opt));
|
2014-10-10 19:00:12 +02:00
|
|
|
|
|
|
|
DBOptions base_db_opt;
|
|
|
|
DBOptions new_db_opt;
|
2014-12-22 22:18:57 +01:00
|
|
|
ASSERT_OK(GetDBOptionsFromMap(base_db_opt, db_options_map, &new_db_opt));
|
2014-10-10 19:00:12 +02:00
|
|
|
ASSERT_EQ(new_db_opt.create_if_missing, false);
|
|
|
|
ASSERT_EQ(new_db_opt.create_missing_column_families, true);
|
|
|
|
ASSERT_EQ(new_db_opt.error_if_exists, false);
|
|
|
|
ASSERT_EQ(new_db_opt.paranoid_checks, true);
|
|
|
|
ASSERT_EQ(new_db_opt.max_open_files, 32);
|
|
|
|
ASSERT_EQ(new_db_opt.max_total_wal_size, static_cast<uint64_t>(33));
|
|
|
|
ASSERT_EQ(new_db_opt.disableDataSync, false);
|
|
|
|
ASSERT_EQ(new_db_opt.use_fsync, true);
|
|
|
|
ASSERT_EQ(new_db_opt.db_log_dir, "/db_log_dir");
|
|
|
|
ASSERT_EQ(new_db_opt.wal_dir, "/wal_dir");
|
|
|
|
ASSERT_EQ(new_db_opt.delete_obsolete_files_period_micros,
|
2014-09-18 00:40:25 +02:00
|
|
|
static_cast<uint64_t>(34));
|
2014-10-10 19:00:12 +02:00
|
|
|
ASSERT_EQ(new_db_opt.max_background_compactions, 35);
|
|
|
|
ASSERT_EQ(new_db_opt.max_background_flushes, 36);
|
|
|
|
ASSERT_EQ(new_db_opt.max_log_file_size, 37U);
|
|
|
|
ASSERT_EQ(new_db_opt.log_file_time_to_roll, 38U);
|
|
|
|
ASSERT_EQ(new_db_opt.keep_log_file_num, 39U);
|
2015-10-08 03:06:28 +02:00
|
|
|
ASSERT_EQ(new_db_opt.recycle_log_file_num, 5U);
|
2014-10-10 19:00:12 +02:00
|
|
|
ASSERT_EQ(new_db_opt.max_manifest_file_size, static_cast<uint64_t>(40));
|
|
|
|
ASSERT_EQ(new_db_opt.table_cache_numshardbits, 41);
|
|
|
|
ASSERT_EQ(new_db_opt.WAL_ttl_seconds, static_cast<uint64_t>(43));
|
|
|
|
ASSERT_EQ(new_db_opt.WAL_size_limit_MB, static_cast<uint64_t>(44));
|
|
|
|
ASSERT_EQ(new_db_opt.manifest_preallocation_size, 45U);
|
|
|
|
ASSERT_EQ(new_db_opt.allow_os_buffer, false);
|
|
|
|
ASSERT_EQ(new_db_opt.allow_mmap_reads, true);
|
|
|
|
ASSERT_EQ(new_db_opt.allow_mmap_writes, false);
|
|
|
|
ASSERT_EQ(new_db_opt.is_fd_close_on_exec, true);
|
|
|
|
ASSERT_EQ(new_db_opt.skip_log_error_on_recovery, false);
|
|
|
|
ASSERT_EQ(new_db_opt.stats_dump_period_sec, 46U);
|
|
|
|
ASSERT_EQ(new_db_opt.advise_random_on_open, true);
|
|
|
|
ASSERT_EQ(new_db_opt.use_adaptive_mutex, false);
|
2015-08-21 07:33:25 +02:00
|
|
|
ASSERT_EQ(new_db_opt.new_table_reader_for_compaction_inputs, true);
|
2015-08-27 00:25:59 +02:00
|
|
|
ASSERT_EQ(new_db_opt.compaction_readahead_size, 100);
|
2015-10-27 22:44:16 +01:00
|
|
|
ASSERT_EQ(new_db_opt.random_access_max_buffer_size, 3145728);
|
2014-10-10 19:00:12 +02:00
|
|
|
ASSERT_EQ(new_db_opt.bytes_per_sync, static_cast<uint64_t>(47));
|
2015-05-19 02:03:59 +02:00
|
|
|
ASSERT_EQ(new_db_opt.wal_bytes_per_sync, static_cast<uint64_t>(48));
|
2014-10-10 19:00:12 +02:00
|
|
|
}
|
2015-01-23 01:57:16 +01:00
|
|
|
#endif // !ROCKSDB_LITE
|
2014-09-17 21:46:32 +02:00
|
|
|
|
2015-02-20 04:11:14 +01:00
|
|
|
#ifndef ROCKSDB_LITE // GetColumnFamilyOptionsFromString is not supported in
|
|
|
|
// ROCKSDB_LITE
|
2015-03-17 22:08:00 +01:00
|
|
|
TEST_F(OptionsTest, GetColumnFamilyOptionsFromStringTest) {
|
2014-10-10 19:00:12 +02:00
|
|
|
ColumnFamilyOptions base_cf_opt;
|
|
|
|
ColumnFamilyOptions new_cf_opt;
|
2014-12-22 22:18:57 +01:00
|
|
|
base_cf_opt.table_factory.reset();
|
|
|
|
ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt, "", &new_cf_opt));
|
|
|
|
ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
|
|
|
|
"write_buffer_size=5", &new_cf_opt));
|
2014-10-10 23:19:51 +02:00
|
|
|
ASSERT_EQ(new_cf_opt.write_buffer_size, 5U);
|
2014-12-22 22:18:57 +01:00
|
|
|
ASSERT_TRUE(new_cf_opt.table_factory == nullptr);
|
|
|
|
ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
|
|
|
|
"write_buffer_size=6;", &new_cf_opt));
|
2014-10-10 23:19:51 +02:00
|
|
|
ASSERT_EQ(new_cf_opt.write_buffer_size, 6U);
|
2014-12-22 22:18:57 +01:00
|
|
|
ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
|
|
|
|
" write_buffer_size = 7 ", &new_cf_opt));
|
2014-10-10 23:19:51 +02:00
|
|
|
ASSERT_EQ(new_cf_opt.write_buffer_size, 7U);
|
2014-12-22 22:18:57 +01:00
|
|
|
ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
|
|
|
|
" write_buffer_size = 8 ; ", &new_cf_opt));
|
2014-10-10 23:19:51 +02:00
|
|
|
ASSERT_EQ(new_cf_opt.write_buffer_size, 8U);
|
2014-12-22 22:18:57 +01:00
|
|
|
ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
|
|
|
|
"write_buffer_size=9;max_write_buffer_number=10", &new_cf_opt));
|
2014-10-10 23:19:51 +02:00
|
|
|
ASSERT_EQ(new_cf_opt.write_buffer_size, 9U);
|
2014-10-10 19:00:12 +02:00
|
|
|
ASSERT_EQ(new_cf_opt.max_write_buffer_number, 10);
|
2014-12-22 22:18:57 +01:00
|
|
|
ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
|
|
|
|
"write_buffer_size=11; max_write_buffer_number = 12 ;",
|
|
|
|
&new_cf_opt));
|
2014-10-10 23:19:51 +02:00
|
|
|
ASSERT_EQ(new_cf_opt.write_buffer_size, 11U);
|
2014-10-10 19:00:12 +02:00
|
|
|
ASSERT_EQ(new_cf_opt.max_write_buffer_number, 12);
|
|
|
|
// Wrong name "max_write_buffer_number_"
|
2014-12-22 22:18:57 +01:00
|
|
|
ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
|
|
|
|
"write_buffer_size=13;max_write_buffer_number_=14;",
|
2014-10-10 19:00:12 +02:00
|
|
|
&new_cf_opt));
|
|
|
|
// Wrong key/value pair
|
2014-12-22 22:18:57 +01:00
|
|
|
ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
|
|
|
|
"write_buffer_size=13;max_write_buffer_number;", &new_cf_opt));
|
2014-10-10 19:00:12 +02:00
|
|
|
// Error Paring value
|
2014-12-22 22:18:57 +01:00
|
|
|
ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
|
|
|
|
"write_buffer_size=13;max_write_buffer_number=;", &new_cf_opt));
|
2014-10-10 19:00:12 +02:00
|
|
|
// Missing option name
|
2014-12-22 22:18:57 +01:00
|
|
|
ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
|
|
|
|
"write_buffer_size=13; =100;", &new_cf_opt));
|
2015-07-02 01:13:49 +02:00
|
|
|
|
|
|
|
const int64_t kilo = 1024UL;
|
|
|
|
const int64_t mega = 1024 * kilo;
|
|
|
|
const int64_t giga = 1024 * mega;
|
|
|
|
const int64_t tera = 1024 * giga;
|
|
|
|
|
2014-11-17 22:47:51 +01:00
|
|
|
// Units (k)
|
2014-12-22 22:18:57 +01:00
|
|
|
ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
|
|
|
|
"memtable_prefix_bloom_bits=14k;max_write_buffer_number=-15K",
|
|
|
|
&new_cf_opt));
|
2015-07-13 21:11:05 +02:00
|
|
|
ASSERT_EQ(new_cf_opt.memtable_prefix_bloom_bits, 14UL * kilo);
|
|
|
|
ASSERT_EQ(new_cf_opt.max_write_buffer_number, -15 * kilo);
|
2014-11-17 22:47:51 +01:00
|
|
|
// Units (m)
|
2014-12-22 22:18:57 +01:00
|
|
|
ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
|
|
|
|
"max_write_buffer_number=16m;inplace_update_num_locks=17M",
|
|
|
|
&new_cf_opt));
|
2015-07-13 21:11:05 +02:00
|
|
|
ASSERT_EQ(new_cf_opt.max_write_buffer_number, 16 * mega);
|
|
|
|
ASSERT_EQ(new_cf_opt.inplace_update_num_locks, 17 * mega);
|
2014-11-17 22:47:51 +01:00
|
|
|
// Units (g)
|
2015-01-21 20:09:56 +01:00
|
|
|
ASSERT_OK(GetColumnFamilyOptionsFromString(
|
|
|
|
base_cf_opt,
|
|
|
|
"write_buffer_size=18g;prefix_extractor=capped:8;"
|
|
|
|
"arena_block_size=19G",
|
|
|
|
&new_cf_opt));
|
2015-07-02 01:13:49 +02:00
|
|
|
|
2015-07-13 21:11:05 +02:00
|
|
|
ASSERT_EQ(new_cf_opt.write_buffer_size, 18 * giga);
|
|
|
|
ASSERT_EQ(new_cf_opt.arena_block_size, 19 * giga);
|
2015-01-21 20:09:56 +01:00
|
|
|
ASSERT_TRUE(new_cf_opt.prefix_extractor.get() != nullptr);
|
|
|
|
std::string prefix_name(new_cf_opt.prefix_extractor->Name());
|
|
|
|
ASSERT_EQ(prefix_name, "rocksdb.CappedPrefix.8");
|
|
|
|
|
2014-11-17 22:47:51 +01:00
|
|
|
// Units (t)
|
2014-12-22 22:18:57 +01:00
|
|
|
ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
|
|
|
|
"write_buffer_size=20t;arena_block_size=21T", &new_cf_opt));
|
2015-07-13 21:11:05 +02:00
|
|
|
ASSERT_EQ(new_cf_opt.write_buffer_size, 20 * tera);
|
|
|
|
ASSERT_EQ(new_cf_opt.arena_block_size, 21 * tera);
|
2014-12-22 22:18:57 +01:00
|
|
|
|
|
|
|
// Nested block based table options
|
|
|
|
// Emtpy
|
|
|
|
ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
|
|
|
|
"write_buffer_size=10;max_write_buffer_number=16;"
|
|
|
|
"block_based_table_factory={};arena_block_size=1024",
|
|
|
|
&new_cf_opt));
|
|
|
|
ASSERT_TRUE(new_cf_opt.table_factory != nullptr);
|
|
|
|
// Non-empty
|
|
|
|
ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
|
|
|
|
"write_buffer_size=10;max_write_buffer_number=16;"
|
|
|
|
"block_based_table_factory={block_cache=1M;block_size=4;};"
|
|
|
|
"arena_block_size=1024",
|
|
|
|
&new_cf_opt));
|
|
|
|
ASSERT_TRUE(new_cf_opt.table_factory != nullptr);
|
|
|
|
// Last one
|
|
|
|
ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
|
|
|
|
"write_buffer_size=10;max_write_buffer_number=16;"
|
|
|
|
"block_based_table_factory={block_cache=1M;block_size=4;}",
|
|
|
|
&new_cf_opt));
|
|
|
|
ASSERT_TRUE(new_cf_opt.table_factory != nullptr);
|
|
|
|
// Mismatch curly braces
|
|
|
|
ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
|
|
|
|
"write_buffer_size=10;max_write_buffer_number=16;"
|
|
|
|
"block_based_table_factory={{{block_size=4;};"
|
|
|
|
"arena_block_size=1024",
|
|
|
|
&new_cf_opt));
|
|
|
|
// Unexpected chars after closing curly brace
|
|
|
|
ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
|
|
|
|
"write_buffer_size=10;max_write_buffer_number=16;"
|
|
|
|
"block_based_table_factory={block_size=4;}};"
|
|
|
|
"arena_block_size=1024",
|
|
|
|
&new_cf_opt));
|
|
|
|
ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
|
|
|
|
"write_buffer_size=10;max_write_buffer_number=16;"
|
|
|
|
"block_based_table_factory={block_size=4;}xdfa;"
|
|
|
|
"arena_block_size=1024",
|
|
|
|
&new_cf_opt));
|
|
|
|
ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
|
|
|
|
"write_buffer_size=10;max_write_buffer_number=16;"
|
|
|
|
"block_based_table_factory={block_size=4;}xdfa",
|
|
|
|
&new_cf_opt));
|
|
|
|
// Invalid block based table option
|
|
|
|
ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
|
|
|
|
"write_buffer_size=10;max_write_buffer_number=16;"
|
|
|
|
"block_based_table_factory={xx_block_size=4;}",
|
|
|
|
&new_cf_opt));
|
2015-02-17 17:03:45 +01:00
|
|
|
ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
|
|
|
|
"optimize_filters_for_hits=true",
|
|
|
|
&new_cf_opt));
|
|
|
|
ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
|
|
|
|
"optimize_filters_for_hits=false",
|
|
|
|
&new_cf_opt));
|
|
|
|
ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
|
|
|
|
"optimize_filters_for_hits=junk",
|
|
|
|
&new_cf_opt));
|
2014-12-22 22:18:57 +01:00
|
|
|
}
|
2015-01-23 01:57:16 +01:00
|
|
|
#endif // !ROCKSDB_LITE
|
2014-12-22 22:18:57 +01:00
|
|
|
|
2015-01-23 01:57:16 +01:00
|
|
|
#ifndef ROCKSDB_LITE // GetBlockBasedTableOptionsFromString is not supported
|
2015-03-17 22:08:00 +01:00
|
|
|
TEST_F(OptionsTest, GetBlockBasedTableOptionsFromString) {
|
2014-12-22 22:18:57 +01:00
|
|
|
BlockBasedTableOptions table_opt;
|
|
|
|
BlockBasedTableOptions new_opt;
|
|
|
|
// make sure default values are overwritten by something else
|
|
|
|
ASSERT_OK(GetBlockBasedTableOptionsFromString(table_opt,
|
|
|
|
"cache_index_and_filter_blocks=1;index_type=kHashSearch;"
|
|
|
|
"checksum=kxxHash;hash_index_allow_collision=1;no_block_cache=1;"
|
|
|
|
"block_cache=1M;block_cache_compressed=1k;block_size=1024;"
|
|
|
|
"block_size_deviation=8;block_restart_interval=4;"
|
|
|
|
"filter_policy=bloomfilter:4:true;whole_key_filtering=1",
|
|
|
|
&new_opt));
|
|
|
|
ASSERT_TRUE(new_opt.cache_index_and_filter_blocks);
|
|
|
|
ASSERT_EQ(new_opt.index_type, BlockBasedTableOptions::kHashSearch);
|
|
|
|
ASSERT_EQ(new_opt.checksum, ChecksumType::kxxHash);
|
|
|
|
ASSERT_TRUE(new_opt.hash_index_allow_collision);
|
|
|
|
ASSERT_TRUE(new_opt.no_block_cache);
|
|
|
|
ASSERT_TRUE(new_opt.block_cache != nullptr);
|
|
|
|
ASSERT_EQ(new_opt.block_cache->GetCapacity(), 1024UL*1024UL);
|
|
|
|
ASSERT_TRUE(new_opt.block_cache_compressed != nullptr);
|
|
|
|
ASSERT_EQ(new_opt.block_cache_compressed->GetCapacity(), 1024UL);
|
|
|
|
ASSERT_EQ(new_opt.block_size, 1024UL);
|
|
|
|
ASSERT_EQ(new_opt.block_size_deviation, 8);
|
|
|
|
ASSERT_EQ(new_opt.block_restart_interval, 4);
|
|
|
|
ASSERT_TRUE(new_opt.filter_policy != nullptr);
|
|
|
|
|
|
|
|
// unknown option
|
|
|
|
ASSERT_NOK(GetBlockBasedTableOptionsFromString(table_opt,
|
|
|
|
"cache_index_and_filter_blocks=1;index_type=kBinarySearch;"
|
|
|
|
"bad_option=1",
|
|
|
|
&new_opt));
|
|
|
|
|
|
|
|
// unrecognized index type
|
|
|
|
ASSERT_NOK(GetBlockBasedTableOptionsFromString(table_opt,
|
|
|
|
"cache_index_and_filter_blocks=1;index_type=kBinarySearchXX",
|
|
|
|
&new_opt));
|
|
|
|
|
|
|
|
// unrecognized checksum type
|
|
|
|
ASSERT_NOK(GetBlockBasedTableOptionsFromString(table_opt,
|
|
|
|
"cache_index_and_filter_blocks=1;checksum=kxxHashXX",
|
|
|
|
&new_opt));
|
|
|
|
|
|
|
|
// unrecognized filter policy name
|
|
|
|
ASSERT_NOK(GetBlockBasedTableOptionsFromString(table_opt,
|
|
|
|
"cache_index_and_filter_blocks=1;"
|
|
|
|
"filter_policy=bloomfilterxx:4:true",
|
|
|
|
&new_opt));
|
|
|
|
// unrecognized filter policy config
|
|
|
|
ASSERT_NOK(GetBlockBasedTableOptionsFromString(table_opt,
|
|
|
|
"cache_index_and_filter_blocks=1;"
|
|
|
|
"filter_policy=bloomfilter:4",
|
|
|
|
&new_opt));
|
|
|
|
}
|
2015-01-23 01:57:16 +01:00
|
|
|
#endif // !ROCKSDB_LITE
|
2014-12-22 22:18:57 +01:00
|
|
|
|
2015-02-20 04:11:14 +01:00
|
|
|
#ifndef ROCKSDB_LITE // GetOptionsFromString is not supported in RocksDB Lite
|
2015-03-17 22:08:00 +01:00
|
|
|
TEST_F(OptionsTest, GetOptionsFromStringTest) {
|
2015-02-20 04:11:14 +01:00
|
|
|
Options base_options, new_options;
|
|
|
|
base_options.write_buffer_size = 20;
|
|
|
|
base_options.min_write_buffer_number_to_merge = 15;
|
|
|
|
BlockBasedTableOptions block_based_table_options;
|
|
|
|
block_based_table_options.cache_index_and_filter_blocks = true;
|
|
|
|
base_options.table_factory.reset(
|
|
|
|
NewBlockBasedTableFactory(block_based_table_options));
|
|
|
|
ASSERT_OK(GetOptionsFromString(
|
|
|
|
base_options,
|
|
|
|
"write_buffer_size=10;max_write_buffer_number=16;"
|
|
|
|
"block_based_table_factory={block_cache=1M;block_size=4;};"
|
2015-03-06 23:21:15 +01:00
|
|
|
"create_if_missing=true;max_open_files=1;rate_limiter_bytes_per_sec=1024",
|
2015-02-20 04:11:14 +01:00
|
|
|
&new_options));
|
|
|
|
|
|
|
|
ASSERT_EQ(new_options.write_buffer_size, 10U);
|
2015-02-20 04:26:38 +01:00
|
|
|
ASSERT_EQ(new_options.max_write_buffer_number, 16);
|
2015-02-20 04:11:14 +01:00
|
|
|
BlockBasedTableOptions new_block_based_table_options =
|
|
|
|
dynamic_cast<BlockBasedTableFactory*>(new_options.table_factory.get())
|
|
|
|
->GetTableOptions();
|
|
|
|
ASSERT_EQ(new_block_based_table_options.block_cache->GetCapacity(), 1U << 20);
|
2015-02-20 04:26:38 +01:00
|
|
|
ASSERT_EQ(new_block_based_table_options.block_size, 4U);
|
2015-02-20 04:11:14 +01:00
|
|
|
// don't overwrite block based table options
|
|
|
|
ASSERT_TRUE(new_block_based_table_options.cache_index_and_filter_blocks);
|
|
|
|
|
|
|
|
ASSERT_EQ(new_options.create_if_missing, true);
|
|
|
|
ASSERT_EQ(new_options.max_open_files, 1);
|
2015-03-06 23:21:15 +01:00
|
|
|
ASSERT_TRUE(new_options.rate_limiter.get() != nullptr);
|
2015-02-20 04:11:14 +01:00
|
|
|
}
|
2015-08-18 22:30:18 +02:00
|
|
|
|
|
|
|
namespace {
|
RocksDB Options file format and its serialization / deserialization.
Summary:
This patch defines the format of RocksDB options file, which
follows the INI file format, and implements functions for its
serialization and deserialization. An example RocksDB options
file can be found in examples/rocksdb_option_file_example.ini.
A typical RocksDB options file has three sections, which are
Version, DBOptions, and more than one CFOptions. The RocksDB
options file in general follows the basic INI file format
with the following extensions / modifications:
* Escaped characters
We escaped the following characters:
- \n -- line feed - new line
- \r -- carriage return
- \\ -- backslash \
- \: -- colon symbol :
- \# -- hash tag #
* Comments
We support # style comments. Comments can appear at the ending
part of a line.
* Statements
A statement is of the form option_name = value.
Each statement contains a '=', where extra white-spaces
are supported. However, we don't support multi-lined statement.
Furthermore, each line can only contain at most one statement.
* Section
Sections are of the form [SecitonTitle "SectionArgument"],
where section argument is optional.
* List
We use colon-separated string to represent a list.
For instance, n1:n2:n3:n4 is a list containing four values.
Below is an example of a RocksDB options file:
[Version]
rocksdb_version=4.0.0
options_file_version=1.0
[DBOptions]
max_open_files=12345
max_background_flushes=301
[CFOptions "default"]
[CFOptions "the second column family"]
[CFOptions "the third column family"]
Test Plan: Added many tests in options_test.cc
Reviewers: igor, IslamAbdelRahman, sdong, anthony
Reviewed By: anthony
Subscribers: maykov, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D46059
2015-09-29 23:42:40 +02:00
|
|
|
void RandomInitDBOptions(DBOptions* db_opt, Random* rnd) {
|
2015-08-18 22:30:18 +02:00
|
|
|
// boolean options
|
RocksDB Options file format and its serialization / deserialization.
Summary:
This patch defines the format of RocksDB options file, which
follows the INI file format, and implements functions for its
serialization and deserialization. An example RocksDB options
file can be found in examples/rocksdb_option_file_example.ini.
A typical RocksDB options file has three sections, which are
Version, DBOptions, and more than one CFOptions. The RocksDB
options file in general follows the basic INI file format
with the following extensions / modifications:
* Escaped characters
We escaped the following characters:
- \n -- line feed - new line
- \r -- carriage return
- \\ -- backslash \
- \: -- colon symbol :
- \# -- hash tag #
* Comments
We support # style comments. Comments can appear at the ending
part of a line.
* Statements
A statement is of the form option_name = value.
Each statement contains a '=', where extra white-spaces
are supported. However, we don't support multi-lined statement.
Furthermore, each line can only contain at most one statement.
* Section
Sections are of the form [SecitonTitle "SectionArgument"],
where section argument is optional.
* List
We use colon-separated string to represent a list.
For instance, n1:n2:n3:n4 is a list containing four values.
Below is an example of a RocksDB options file:
[Version]
rocksdb_version=4.0.0
options_file_version=1.0
[DBOptions]
max_open_files=12345
max_background_flushes=301
[CFOptions "default"]
[CFOptions "the second column family"]
[CFOptions "the third column family"]
Test Plan: Added many tests in options_test.cc
Reviewers: igor, IslamAbdelRahman, sdong, anthony
Reviewed By: anthony
Subscribers: maykov, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D46059
2015-09-29 23:42:40 +02:00
|
|
|
db_opt->advise_random_on_open = rnd->Uniform(2);
|
|
|
|
db_opt->allow_mmap_reads = rnd->Uniform(2);
|
|
|
|
db_opt->allow_mmap_writes = rnd->Uniform(2);
|
|
|
|
db_opt->allow_os_buffer = rnd->Uniform(2);
|
|
|
|
db_opt->create_if_missing = rnd->Uniform(2);
|
|
|
|
db_opt->create_missing_column_families = rnd->Uniform(2);
|
|
|
|
db_opt->disableDataSync = rnd->Uniform(2);
|
|
|
|
db_opt->enable_thread_tracking = rnd->Uniform(2);
|
|
|
|
db_opt->error_if_exists = rnd->Uniform(2);
|
|
|
|
db_opt->is_fd_close_on_exec = rnd->Uniform(2);
|
|
|
|
db_opt->paranoid_checks = rnd->Uniform(2);
|
|
|
|
db_opt->skip_log_error_on_recovery = rnd->Uniform(2);
|
|
|
|
db_opt->skip_stats_update_on_db_open = rnd->Uniform(2);
|
|
|
|
db_opt->use_adaptive_mutex = rnd->Uniform(2);
|
|
|
|
db_opt->use_fsync = rnd->Uniform(2);
|
2015-10-08 03:06:28 +02:00
|
|
|
db_opt->recycle_log_file_num = rnd->Uniform(2);
|
2015-08-18 22:30:18 +02:00
|
|
|
|
|
|
|
// int options
|
RocksDB Options file format and its serialization / deserialization.
Summary:
This patch defines the format of RocksDB options file, which
follows the INI file format, and implements functions for its
serialization and deserialization. An example RocksDB options
file can be found in examples/rocksdb_option_file_example.ini.
A typical RocksDB options file has three sections, which are
Version, DBOptions, and more than one CFOptions. The RocksDB
options file in general follows the basic INI file format
with the following extensions / modifications:
* Escaped characters
We escaped the following characters:
- \n -- line feed - new line
- \r -- carriage return
- \\ -- backslash \
- \: -- colon symbol :
- \# -- hash tag #
* Comments
We support # style comments. Comments can appear at the ending
part of a line.
* Statements
A statement is of the form option_name = value.
Each statement contains a '=', where extra white-spaces
are supported. However, we don't support multi-lined statement.
Furthermore, each line can only contain at most one statement.
* Section
Sections are of the form [SecitonTitle "SectionArgument"],
where section argument is optional.
* List
We use colon-separated string to represent a list.
For instance, n1:n2:n3:n4 is a list containing four values.
Below is an example of a RocksDB options file:
[Version]
rocksdb_version=4.0.0
options_file_version=1.0
[DBOptions]
max_open_files=12345
max_background_flushes=301
[CFOptions "default"]
[CFOptions "the second column family"]
[CFOptions "the third column family"]
Test Plan: Added many tests in options_test.cc
Reviewers: igor, IslamAbdelRahman, sdong, anthony
Reviewed By: anthony
Subscribers: maykov, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D46059
2015-09-29 23:42:40 +02:00
|
|
|
db_opt->max_background_compactions = rnd->Uniform(100);
|
|
|
|
db_opt->max_background_flushes = rnd->Uniform(100);
|
|
|
|
db_opt->max_file_opening_threads = rnd->Uniform(100);
|
|
|
|
db_opt->max_open_files = rnd->Uniform(100);
|
|
|
|
db_opt->table_cache_numshardbits = rnd->Uniform(100);
|
2015-08-18 22:30:18 +02:00
|
|
|
|
|
|
|
// size_t options
|
RocksDB Options file format and its serialization / deserialization.
Summary:
This patch defines the format of RocksDB options file, which
follows the INI file format, and implements functions for its
serialization and deserialization. An example RocksDB options
file can be found in examples/rocksdb_option_file_example.ini.
A typical RocksDB options file has three sections, which are
Version, DBOptions, and more than one CFOptions. The RocksDB
options file in general follows the basic INI file format
with the following extensions / modifications:
* Escaped characters
We escaped the following characters:
- \n -- line feed - new line
- \r -- carriage return
- \\ -- backslash \
- \: -- colon symbol :
- \# -- hash tag #
* Comments
We support # style comments. Comments can appear at the ending
part of a line.
* Statements
A statement is of the form option_name = value.
Each statement contains a '=', where extra white-spaces
are supported. However, we don't support multi-lined statement.
Furthermore, each line can only contain at most one statement.
* Section
Sections are of the form [SecitonTitle "SectionArgument"],
where section argument is optional.
* List
We use colon-separated string to represent a list.
For instance, n1:n2:n3:n4 is a list containing four values.
Below is an example of a RocksDB options file:
[Version]
rocksdb_version=4.0.0
options_file_version=1.0
[DBOptions]
max_open_files=12345
max_background_flushes=301
[CFOptions "default"]
[CFOptions "the second column family"]
[CFOptions "the third column family"]
Test Plan: Added many tests in options_test.cc
Reviewers: igor, IslamAbdelRahman, sdong, anthony
Reviewed By: anthony
Subscribers: maykov, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D46059
2015-09-29 23:42:40 +02:00
|
|
|
db_opt->db_write_buffer_size = rnd->Uniform(10000);
|
|
|
|
db_opt->keep_log_file_num = rnd->Uniform(10000);
|
|
|
|
db_opt->log_file_time_to_roll = rnd->Uniform(10000);
|
|
|
|
db_opt->manifest_preallocation_size = rnd->Uniform(10000);
|
|
|
|
db_opt->max_log_file_size = rnd->Uniform(10000);
|
2015-08-18 22:30:18 +02:00
|
|
|
|
|
|
|
// std::string options
|
RocksDB Options file format and its serialization / deserialization.
Summary:
This patch defines the format of RocksDB options file, which
follows the INI file format, and implements functions for its
serialization and deserialization. An example RocksDB options
file can be found in examples/rocksdb_option_file_example.ini.
A typical RocksDB options file has three sections, which are
Version, DBOptions, and more than one CFOptions. The RocksDB
options file in general follows the basic INI file format
with the following extensions / modifications:
* Escaped characters
We escaped the following characters:
- \n -- line feed - new line
- \r -- carriage return
- \\ -- backslash \
- \: -- colon symbol :
- \# -- hash tag #
* Comments
We support # style comments. Comments can appear at the ending
part of a line.
* Statements
A statement is of the form option_name = value.
Each statement contains a '=', where extra white-spaces
are supported. However, we don't support multi-lined statement.
Furthermore, each line can only contain at most one statement.
* Section
Sections are of the form [SecitonTitle "SectionArgument"],
where section argument is optional.
* List
We use colon-separated string to represent a list.
For instance, n1:n2:n3:n4 is a list containing four values.
Below is an example of a RocksDB options file:
[Version]
rocksdb_version=4.0.0
options_file_version=1.0
[DBOptions]
max_open_files=12345
max_background_flushes=301
[CFOptions "default"]
[CFOptions "the second column family"]
[CFOptions "the third column family"]
Test Plan: Added many tests in options_test.cc
Reviewers: igor, IslamAbdelRahman, sdong, anthony
Reviewed By: anthony
Subscribers: maykov, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D46059
2015-09-29 23:42:40 +02:00
|
|
|
db_opt->db_log_dir = "path/to/db_log_dir";
|
|
|
|
db_opt->wal_dir = "path/to/wal_dir";
|
2015-08-18 22:30:18 +02:00
|
|
|
|
|
|
|
// uint32_t options
|
RocksDB Options file format and its serialization / deserialization.
Summary:
This patch defines the format of RocksDB options file, which
follows the INI file format, and implements functions for its
serialization and deserialization. An example RocksDB options
file can be found in examples/rocksdb_option_file_example.ini.
A typical RocksDB options file has three sections, which are
Version, DBOptions, and more than one CFOptions. The RocksDB
options file in general follows the basic INI file format
with the following extensions / modifications:
* Escaped characters
We escaped the following characters:
- \n -- line feed - new line
- \r -- carriage return
- \\ -- backslash \
- \: -- colon symbol :
- \# -- hash tag #
* Comments
We support # style comments. Comments can appear at the ending
part of a line.
* Statements
A statement is of the form option_name = value.
Each statement contains a '=', where extra white-spaces
are supported. However, we don't support multi-lined statement.
Furthermore, each line can only contain at most one statement.
* Section
Sections are of the form [SecitonTitle "SectionArgument"],
where section argument is optional.
* List
We use colon-separated string to represent a list.
For instance, n1:n2:n3:n4 is a list containing four values.
Below is an example of a RocksDB options file:
[Version]
rocksdb_version=4.0.0
options_file_version=1.0
[DBOptions]
max_open_files=12345
max_background_flushes=301
[CFOptions "default"]
[CFOptions "the second column family"]
[CFOptions "the third column family"]
Test Plan: Added many tests in options_test.cc
Reviewers: igor, IslamAbdelRahman, sdong, anthony
Reviewed By: anthony
Subscribers: maykov, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D46059
2015-09-29 23:42:40 +02:00
|
|
|
db_opt->max_subcompactions = rnd->Uniform(100000);
|
2015-08-18 22:30:18 +02:00
|
|
|
|
|
|
|
// uint64_t options
|
RocksDB Options file format and its serialization / deserialization.
Summary:
This patch defines the format of RocksDB options file, which
follows the INI file format, and implements functions for its
serialization and deserialization. An example RocksDB options
file can be found in examples/rocksdb_option_file_example.ini.
A typical RocksDB options file has three sections, which are
Version, DBOptions, and more than one CFOptions. The RocksDB
options file in general follows the basic INI file format
with the following extensions / modifications:
* Escaped characters
We escaped the following characters:
- \n -- line feed - new line
- \r -- carriage return
- \\ -- backslash \
- \: -- colon symbol :
- \# -- hash tag #
* Comments
We support # style comments. Comments can appear at the ending
part of a line.
* Statements
A statement is of the form option_name = value.
Each statement contains a '=', where extra white-spaces
are supported. However, we don't support multi-lined statement.
Furthermore, each line can only contain at most one statement.
* Section
Sections are of the form [SecitonTitle "SectionArgument"],
where section argument is optional.
* List
We use colon-separated string to represent a list.
For instance, n1:n2:n3:n4 is a list containing four values.
Below is an example of a RocksDB options file:
[Version]
rocksdb_version=4.0.0
options_file_version=1.0
[DBOptions]
max_open_files=12345
max_background_flushes=301
[CFOptions "default"]
[CFOptions "the second column family"]
[CFOptions "the third column family"]
Test Plan: Added many tests in options_test.cc
Reviewers: igor, IslamAbdelRahman, sdong, anthony
Reviewed By: anthony
Subscribers: maykov, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D46059
2015-09-29 23:42:40 +02:00
|
|
|
static const uint64_t uint_max = static_cast<uint64_t>(UINT_MAX);
|
|
|
|
db_opt->WAL_size_limit_MB = uint_max + rnd->Uniform(100000);
|
|
|
|
db_opt->WAL_ttl_seconds = uint_max + rnd->Uniform(100000);
|
|
|
|
db_opt->bytes_per_sync = uint_max + rnd->Uniform(100000);
|
|
|
|
db_opt->delayed_write_rate = uint_max + rnd->Uniform(100000);
|
|
|
|
db_opt->delete_obsolete_files_period_micros = uint_max + rnd->Uniform(100000);
|
|
|
|
db_opt->max_manifest_file_size = uint_max + rnd->Uniform(100000);
|
|
|
|
db_opt->max_total_wal_size = uint_max + rnd->Uniform(100000);
|
|
|
|
db_opt->wal_bytes_per_sync = uint_max + rnd->Uniform(100000);
|
2015-08-18 22:30:18 +02:00
|
|
|
|
|
|
|
// unsigned int options
|
RocksDB Options file format and its serialization / deserialization.
Summary:
This patch defines the format of RocksDB options file, which
follows the INI file format, and implements functions for its
serialization and deserialization. An example RocksDB options
file can be found in examples/rocksdb_option_file_example.ini.
A typical RocksDB options file has three sections, which are
Version, DBOptions, and more than one CFOptions. The RocksDB
options file in general follows the basic INI file format
with the following extensions / modifications:
* Escaped characters
We escaped the following characters:
- \n -- line feed - new line
- \r -- carriage return
- \\ -- backslash \
- \: -- colon symbol :
- \# -- hash tag #
* Comments
We support # style comments. Comments can appear at the ending
part of a line.
* Statements
A statement is of the form option_name = value.
Each statement contains a '=', where extra white-spaces
are supported. However, we don't support multi-lined statement.
Furthermore, each line can only contain at most one statement.
* Section
Sections are of the form [SecitonTitle "SectionArgument"],
where section argument is optional.
* List
We use colon-separated string to represent a list.
For instance, n1:n2:n3:n4 is a list containing four values.
Below is an example of a RocksDB options file:
[Version]
rocksdb_version=4.0.0
options_file_version=1.0
[DBOptions]
max_open_files=12345
max_background_flushes=301
[CFOptions "default"]
[CFOptions "the second column family"]
[CFOptions "the third column family"]
Test Plan: Added many tests in options_test.cc
Reviewers: igor, IslamAbdelRahman, sdong, anthony
Reviewed By: anthony
Subscribers: maykov, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D46059
2015-09-29 23:42:40 +02:00
|
|
|
db_opt->stats_dump_period_sec = rnd->Uniform(100000);
|
2015-08-18 22:30:18 +02:00
|
|
|
}
|
RocksDB Options file format and its serialization / deserialization.
Summary:
This patch defines the format of RocksDB options file, which
follows the INI file format, and implements functions for its
serialization and deserialization. An example RocksDB options
file can be found in examples/rocksdb_option_file_example.ini.
A typical RocksDB options file has three sections, which are
Version, DBOptions, and more than one CFOptions. The RocksDB
options file in general follows the basic INI file format
with the following extensions / modifications:
* Escaped characters
We escaped the following characters:
- \n -- line feed - new line
- \r -- carriage return
- \\ -- backslash \
- \: -- colon symbol :
- \# -- hash tag #
* Comments
We support # style comments. Comments can appear at the ending
part of a line.
* Statements
A statement is of the form option_name = value.
Each statement contains a '=', where extra white-spaces
are supported. However, we don't support multi-lined statement.
Furthermore, each line can only contain at most one statement.
* Section
Sections are of the form [SecitonTitle "SectionArgument"],
where section argument is optional.
* List
We use colon-separated string to represent a list.
For instance, n1:n2:n3:n4 is a list containing four values.
Below is an example of a RocksDB options file:
[Version]
rocksdb_version=4.0.0
options_file_version=1.0
[DBOptions]
max_open_files=12345
max_background_flushes=301
[CFOptions "default"]
[CFOptions "the second column family"]
[CFOptions "the third column family"]
Test Plan: Added many tests in options_test.cc
Reviewers: igor, IslamAbdelRahman, sdong, anthony
Reviewed By: anthony
Subscribers: maykov, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D46059
2015-09-29 23:42:40 +02:00
|
|
|
|
2015-08-18 22:30:18 +02:00
|
|
|
} // namespace
|
|
|
|
|
|
|
|
TEST_F(OptionsTest, DBOptionsSerialization) {
|
|
|
|
Options base_options, new_options;
|
|
|
|
Random rnd(301);
|
|
|
|
|
|
|
|
// Phase 1: Make big change in base_options
|
RocksDB Options file format and its serialization / deserialization.
Summary:
This patch defines the format of RocksDB options file, which
follows the INI file format, and implements functions for its
serialization and deserialization. An example RocksDB options
file can be found in examples/rocksdb_option_file_example.ini.
A typical RocksDB options file has three sections, which are
Version, DBOptions, and more than one CFOptions. The RocksDB
options file in general follows the basic INI file format
with the following extensions / modifications:
* Escaped characters
We escaped the following characters:
- \n -- line feed - new line
- \r -- carriage return
- \\ -- backslash \
- \: -- colon symbol :
- \# -- hash tag #
* Comments
We support # style comments. Comments can appear at the ending
part of a line.
* Statements
A statement is of the form option_name = value.
Each statement contains a '=', where extra white-spaces
are supported. However, we don't support multi-lined statement.
Furthermore, each line can only contain at most one statement.
* Section
Sections are of the form [SecitonTitle "SectionArgument"],
where section argument is optional.
* List
We use colon-separated string to represent a list.
For instance, n1:n2:n3:n4 is a list containing four values.
Below is an example of a RocksDB options file:
[Version]
rocksdb_version=4.0.0
options_file_version=1.0
[DBOptions]
max_open_files=12345
max_background_flushes=301
[CFOptions "default"]
[CFOptions "the second column family"]
[CFOptions "the third column family"]
Test Plan: Added many tests in options_test.cc
Reviewers: igor, IslamAbdelRahman, sdong, anthony
Reviewed By: anthony
Subscribers: maykov, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D46059
2015-09-29 23:42:40 +02:00
|
|
|
RandomInitDBOptions(&base_options, &rnd);
|
2015-08-18 22:30:18 +02:00
|
|
|
|
|
|
|
// Phase 2: obtain a string from base_option
|
RocksDB Options file format and its serialization / deserialization.
Summary:
This patch defines the format of RocksDB options file, which
follows the INI file format, and implements functions for its
serialization and deserialization. An example RocksDB options
file can be found in examples/rocksdb_option_file_example.ini.
A typical RocksDB options file has three sections, which are
Version, DBOptions, and more than one CFOptions. The RocksDB
options file in general follows the basic INI file format
with the following extensions / modifications:
* Escaped characters
We escaped the following characters:
- \n -- line feed - new line
- \r -- carriage return
- \\ -- backslash \
- \: -- colon symbol :
- \# -- hash tag #
* Comments
We support # style comments. Comments can appear at the ending
part of a line.
* Statements
A statement is of the form option_name = value.
Each statement contains a '=', where extra white-spaces
are supported. However, we don't support multi-lined statement.
Furthermore, each line can only contain at most one statement.
* Section
Sections are of the form [SecitonTitle "SectionArgument"],
where section argument is optional.
* List
We use colon-separated string to represent a list.
For instance, n1:n2:n3:n4 is a list containing four values.
Below is an example of a RocksDB options file:
[Version]
rocksdb_version=4.0.0
options_file_version=1.0
[DBOptions]
max_open_files=12345
max_background_flushes=301
[CFOptions "default"]
[CFOptions "the second column family"]
[CFOptions "the third column family"]
Test Plan: Added many tests in options_test.cc
Reviewers: igor, IslamAbdelRahman, sdong, anthony
Reviewed By: anthony
Subscribers: maykov, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D46059
2015-09-29 23:42:40 +02:00
|
|
|
std::string base_options_file_content;
|
|
|
|
ASSERT_OK(GetStringFromDBOptions(&base_options_file_content, base_options));
|
2015-08-18 22:30:18 +02:00
|
|
|
|
|
|
|
// Phase 3: Set new_options from the derived string and expect
|
|
|
|
// new_options == base_options
|
RocksDB Options file format and its serialization / deserialization.
Summary:
This patch defines the format of RocksDB options file, which
follows the INI file format, and implements functions for its
serialization and deserialization. An example RocksDB options
file can be found in examples/rocksdb_option_file_example.ini.
A typical RocksDB options file has three sections, which are
Version, DBOptions, and more than one CFOptions. The RocksDB
options file in general follows the basic INI file format
with the following extensions / modifications:
* Escaped characters
We escaped the following characters:
- \n -- line feed - new line
- \r -- carriage return
- \\ -- backslash \
- \: -- colon symbol :
- \# -- hash tag #
* Comments
We support # style comments. Comments can appear at the ending
part of a line.
* Statements
A statement is of the form option_name = value.
Each statement contains a '=', where extra white-spaces
are supported. However, we don't support multi-lined statement.
Furthermore, each line can only contain at most one statement.
* Section
Sections are of the form [SecitonTitle "SectionArgument"],
where section argument is optional.
* List
We use colon-separated string to represent a list.
For instance, n1:n2:n3:n4 is a list containing four values.
Below is an example of a RocksDB options file:
[Version]
rocksdb_version=4.0.0
options_file_version=1.0
[DBOptions]
max_open_files=12345
max_background_flushes=301
[CFOptions "default"]
[CFOptions "the second column family"]
[CFOptions "the third column family"]
Test Plan: Added many tests in options_test.cc
Reviewers: igor, IslamAbdelRahman, sdong, anthony
Reviewed By: anthony
Subscribers: maykov, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D46059
2015-09-29 23:42:40 +02:00
|
|
|
ASSERT_OK(GetDBOptionsFromString(DBOptions(), base_options_file_content,
|
|
|
|
&new_options));
|
|
|
|
ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(base_options, new_options));
|
2015-08-18 22:30:18 +02:00
|
|
|
}
|
2015-08-27 01:13:56 +02:00
|
|
|
|
|
|
|
namespace {
|
2015-10-03 00:35:32 +02:00
|
|
|
CompressionType RandomCompressionType(Random* rnd) {
|
|
|
|
return static_cast<CompressionType>(rnd->Uniform(6));
|
|
|
|
}
|
|
|
|
|
|
|
|
void RandomCompressionTypeVector(const size_t count,
|
|
|
|
std::vector<CompressionType>* types,
|
|
|
|
Random* rnd) {
|
|
|
|
types->clear();
|
|
|
|
for (size_t i = 0; i < count; ++i) {
|
|
|
|
types->emplace_back(RandomCompressionType(rnd));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
const SliceTransform* RandomSliceTransform(Random* rnd, int pre_defined = -1) {
|
|
|
|
int random_num = pre_defined >= 0 ? pre_defined : rnd->Uniform(4);
|
|
|
|
switch (random_num) {
|
|
|
|
case 0:
|
|
|
|
return NewFixedPrefixTransform(rnd->Uniform(20) + 1);
|
|
|
|
case 1:
|
|
|
|
return NewCappedPrefixTransform(rnd->Uniform(20) + 1);
|
|
|
|
case 2:
|
|
|
|
return NewNoopTransform();
|
|
|
|
default:
|
|
|
|
return nullptr;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
TableFactory* RandomTableFactory(Random* rnd, int pre_defined = -1) {
|
|
|
|
int random_num = pre_defined >= 0 ? pre_defined : rnd->Uniform(3);
|
|
|
|
switch (random_num) {
|
|
|
|
case 0:
|
|
|
|
return NewPlainTableFactory();
|
|
|
|
case 1:
|
|
|
|
return NewCuckooTableFactory();
|
|
|
|
default:
|
|
|
|
return NewBlockBasedTableFactory();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
std::string RandomString(Random* rnd, const size_t len) {
|
|
|
|
std::stringstream ss;
|
|
|
|
for (size_t i = 0; i < len; ++i) {
|
|
|
|
ss << static_cast<char>(rnd->Uniform(26) + 'a');
|
|
|
|
}
|
|
|
|
return ss.str();
|
|
|
|
}
|
|
|
|
|
|
|
|
class ChanglingMergeOperator : public MergeOperator {
|
|
|
|
public:
|
|
|
|
explicit ChanglingMergeOperator(const std::string& name)
|
|
|
|
: name_(name + "MergeOperator") {}
|
|
|
|
~ChanglingMergeOperator() {}
|
|
|
|
|
|
|
|
void SetName(const std::string& name) { name_ = name; }
|
|
|
|
|
|
|
|
virtual bool FullMerge(const Slice& key, const Slice* existing_value,
|
|
|
|
const std::deque<std::string>& operand_list,
|
|
|
|
std::string* new_value,
|
|
|
|
Logger* logger) const override {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
virtual bool PartialMergeMulti(const Slice& key,
|
|
|
|
const std::deque<Slice>& operand_list,
|
|
|
|
std::string* new_value,
|
|
|
|
Logger* logger) const override {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
virtual const char* Name() const override { return name_.c_str(); }
|
|
|
|
|
|
|
|
protected:
|
|
|
|
std::string name_;
|
|
|
|
};
|
|
|
|
|
|
|
|
MergeOperator* RandomMergeOperator(Random* rnd) {
|
|
|
|
return new ChanglingMergeOperator(RandomString(rnd, 10));
|
|
|
|
}
|
|
|
|
|
|
|
|
class ChanglingCompactionFilter : public CompactionFilter {
|
|
|
|
public:
|
|
|
|
explicit ChanglingCompactionFilter(const std::string& name)
|
|
|
|
: name_(name + "CompactionFilter") {}
|
|
|
|
~ChanglingCompactionFilter() {}
|
|
|
|
|
|
|
|
void SetName(const std::string& name) { name_ = name; }
|
|
|
|
|
|
|
|
bool Filter(int level, const Slice& key, const Slice& existing_value,
|
|
|
|
std::string* new_value, bool* value_changed) const override {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
const char* Name() const override { return name_.c_str(); }
|
|
|
|
|
|
|
|
private:
|
|
|
|
std::string name_;
|
|
|
|
};
|
|
|
|
|
|
|
|
CompactionFilter* RandomCompactionFilter(Random* rnd) {
|
|
|
|
return new ChanglingCompactionFilter(RandomString(rnd, 10));
|
|
|
|
}
|
|
|
|
|
|
|
|
class ChanglingCompactionFilterFactory : public CompactionFilterFactory {
|
|
|
|
public:
|
|
|
|
explicit ChanglingCompactionFilterFactory(const std::string& name)
|
|
|
|
: name_(name + "CompactionFilterFactory") {}
|
|
|
|
~ChanglingCompactionFilterFactory() {}
|
|
|
|
|
|
|
|
void SetName(const std::string& name) { name_ = name; }
|
|
|
|
|
|
|
|
std::unique_ptr<CompactionFilter> CreateCompactionFilter(
|
|
|
|
const CompactionFilter::Context& context) override {
|
|
|
|
return std::unique_ptr<CompactionFilter>();
|
|
|
|
}
|
2015-08-27 01:13:56 +02:00
|
|
|
|
2015-10-03 00:35:32 +02:00
|
|
|
// Returns a name that identifies this compaction filter factory.
|
|
|
|
const char* Name() const override { return name_.c_str(); }
|
|
|
|
|
|
|
|
protected:
|
|
|
|
std::string name_;
|
|
|
|
};
|
|
|
|
|
|
|
|
CompactionFilterFactory* RandomCompactionFilterFactory(Random* rnd) {
|
|
|
|
return new ChanglingCompactionFilterFactory(RandomString(rnd, 10));
|
|
|
|
}
|
|
|
|
|
|
|
|
// Note that the caller is responsible for releasing non-null
|
|
|
|
// cf_opt->compaction_filter.
|
RocksDB Options file format and its serialization / deserialization.
Summary:
This patch defines the format of RocksDB options file, which
follows the INI file format, and implements functions for its
serialization and deserialization. An example RocksDB options
file can be found in examples/rocksdb_option_file_example.ini.
A typical RocksDB options file has three sections, which are
Version, DBOptions, and more than one CFOptions. The RocksDB
options file in general follows the basic INI file format
with the following extensions / modifications:
* Escaped characters
We escaped the following characters:
- \n -- line feed - new line
- \r -- carriage return
- \\ -- backslash \
- \: -- colon symbol :
- \# -- hash tag #
* Comments
We support # style comments. Comments can appear at the ending
part of a line.
* Statements
A statement is of the form option_name = value.
Each statement contains a '=', where extra white-spaces
are supported. However, we don't support multi-lined statement.
Furthermore, each line can only contain at most one statement.
* Section
Sections are of the form [SecitonTitle "SectionArgument"],
where section argument is optional.
* List
We use colon-separated string to represent a list.
For instance, n1:n2:n3:n4 is a list containing four values.
Below is an example of a RocksDB options file:
[Version]
rocksdb_version=4.0.0
options_file_version=1.0
[DBOptions]
max_open_files=12345
max_background_flushes=301
[CFOptions "default"]
[CFOptions "the second column family"]
[CFOptions "the third column family"]
Test Plan: Added many tests in options_test.cc
Reviewers: igor, IslamAbdelRahman, sdong, anthony
Reviewed By: anthony
Subscribers: maykov, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D46059
2015-09-29 23:42:40 +02:00
|
|
|
void RandomInitCFOptions(ColumnFamilyOptions* cf_opt, Random* rnd) {
|
|
|
|
cf_opt->compaction_style = (CompactionStyle)(rnd->Uniform(4));
|
2015-08-27 01:13:56 +02:00
|
|
|
|
|
|
|
// boolean options
|
RocksDB Options file format and its serialization / deserialization.
Summary:
This patch defines the format of RocksDB options file, which
follows the INI file format, and implements functions for its
serialization and deserialization. An example RocksDB options
file can be found in examples/rocksdb_option_file_example.ini.
A typical RocksDB options file has three sections, which are
Version, DBOptions, and more than one CFOptions. The RocksDB
options file in general follows the basic INI file format
with the following extensions / modifications:
* Escaped characters
We escaped the following characters:
- \n -- line feed - new line
- \r -- carriage return
- \\ -- backslash \
- \: -- colon symbol :
- \# -- hash tag #
* Comments
We support # style comments. Comments can appear at the ending
part of a line.
* Statements
A statement is of the form option_name = value.
Each statement contains a '=', where extra white-spaces
are supported. However, we don't support multi-lined statement.
Furthermore, each line can only contain at most one statement.
* Section
Sections are of the form [SecitonTitle "SectionArgument"],
where section argument is optional.
* List
We use colon-separated string to represent a list.
For instance, n1:n2:n3:n4 is a list containing four values.
Below is an example of a RocksDB options file:
[Version]
rocksdb_version=4.0.0
options_file_version=1.0
[DBOptions]
max_open_files=12345
max_background_flushes=301
[CFOptions "default"]
[CFOptions "the second column family"]
[CFOptions "the third column family"]
Test Plan: Added many tests in options_test.cc
Reviewers: igor, IslamAbdelRahman, sdong, anthony
Reviewed By: anthony
Subscribers: maykov, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D46059
2015-09-29 23:42:40 +02:00
|
|
|
cf_opt->compaction_measure_io_stats = rnd->Uniform(2);
|
|
|
|
cf_opt->disable_auto_compactions = rnd->Uniform(2);
|
|
|
|
cf_opt->filter_deletes = rnd->Uniform(2);
|
|
|
|
cf_opt->inplace_update_support = rnd->Uniform(2);
|
|
|
|
cf_opt->level_compaction_dynamic_level_bytes = rnd->Uniform(2);
|
|
|
|
cf_opt->optimize_filters_for_hits = rnd->Uniform(2);
|
|
|
|
cf_opt->paranoid_file_checks = rnd->Uniform(2);
|
|
|
|
cf_opt->purge_redundant_kvs_while_flush = rnd->Uniform(2);
|
|
|
|
cf_opt->verify_checksums_in_compaction = rnd->Uniform(2);
|
2015-08-27 01:13:56 +02:00
|
|
|
|
|
|
|
// double options
|
RocksDB Options file format and its serialization / deserialization.
Summary:
This patch defines the format of RocksDB options file, which
follows the INI file format, and implements functions for its
serialization and deserialization. An example RocksDB options
file can be found in examples/rocksdb_option_file_example.ini.
A typical RocksDB options file has three sections, which are
Version, DBOptions, and more than one CFOptions. The RocksDB
options file in general follows the basic INI file format
with the following extensions / modifications:
* Escaped characters
We escaped the following characters:
- \n -- line feed - new line
- \r -- carriage return
- \\ -- backslash \
- \: -- colon symbol :
- \# -- hash tag #
* Comments
We support # style comments. Comments can appear at the ending
part of a line.
* Statements
A statement is of the form option_name = value.
Each statement contains a '=', where extra white-spaces
are supported. However, we don't support multi-lined statement.
Furthermore, each line can only contain at most one statement.
* Section
Sections are of the form [SecitonTitle "SectionArgument"],
where section argument is optional.
* List
We use colon-separated string to represent a list.
For instance, n1:n2:n3:n4 is a list containing four values.
Below is an example of a RocksDB options file:
[Version]
rocksdb_version=4.0.0
options_file_version=1.0
[DBOptions]
max_open_files=12345
max_background_flushes=301
[CFOptions "default"]
[CFOptions "the second column family"]
[CFOptions "the third column family"]
Test Plan: Added many tests in options_test.cc
Reviewers: igor, IslamAbdelRahman, sdong, anthony
Reviewed By: anthony
Subscribers: maykov, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D46059
2015-09-29 23:42:40 +02:00
|
|
|
cf_opt->hard_rate_limit = static_cast<double>(rnd->Uniform(10000)) / 13;
|
|
|
|
cf_opt->soft_rate_limit = static_cast<double>(rnd->Uniform(10000)) / 13;
|
2015-08-27 01:13:56 +02:00
|
|
|
|
|
|
|
// int options
|
RocksDB Options file format and its serialization / deserialization.
Summary:
This patch defines the format of RocksDB options file, which
follows the INI file format, and implements functions for its
serialization and deserialization. An example RocksDB options
file can be found in examples/rocksdb_option_file_example.ini.
A typical RocksDB options file has three sections, which are
Version, DBOptions, and more than one CFOptions. The RocksDB
options file in general follows the basic INI file format
with the following extensions / modifications:
* Escaped characters
We escaped the following characters:
- \n -- line feed - new line
- \r -- carriage return
- \\ -- backslash \
- \: -- colon symbol :
- \# -- hash tag #
* Comments
We support # style comments. Comments can appear at the ending
part of a line.
* Statements
A statement is of the form option_name = value.
Each statement contains a '=', where extra white-spaces
are supported. However, we don't support multi-lined statement.
Furthermore, each line can only contain at most one statement.
* Section
Sections are of the form [SecitonTitle "SectionArgument"],
where section argument is optional.
* List
We use colon-separated string to represent a list.
For instance, n1:n2:n3:n4 is a list containing four values.
Below is an example of a RocksDB options file:
[Version]
rocksdb_version=4.0.0
options_file_version=1.0
[DBOptions]
max_open_files=12345
max_background_flushes=301
[CFOptions "default"]
[CFOptions "the second column family"]
[CFOptions "the third column family"]
Test Plan: Added many tests in options_test.cc
Reviewers: igor, IslamAbdelRahman, sdong, anthony
Reviewed By: anthony
Subscribers: maykov, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D46059
2015-09-29 23:42:40 +02:00
|
|
|
cf_opt->expanded_compaction_factor = rnd->Uniform(100);
|
|
|
|
cf_opt->level0_file_num_compaction_trigger = rnd->Uniform(100);
|
|
|
|
cf_opt->level0_slowdown_writes_trigger = rnd->Uniform(100);
|
|
|
|
cf_opt->level0_stop_writes_trigger = rnd->Uniform(100);
|
|
|
|
cf_opt->max_bytes_for_level_multiplier = rnd->Uniform(100);
|
|
|
|
cf_opt->max_grandparent_overlap_factor = rnd->Uniform(100);
|
|
|
|
cf_opt->max_mem_compaction_level = rnd->Uniform(100);
|
|
|
|
cf_opt->max_write_buffer_number = rnd->Uniform(100);
|
|
|
|
cf_opt->max_write_buffer_number_to_maintain = rnd->Uniform(100);
|
|
|
|
cf_opt->min_write_buffer_number_to_merge = rnd->Uniform(100);
|
|
|
|
cf_opt->num_levels = rnd->Uniform(100);
|
|
|
|
cf_opt->source_compaction_factor = rnd->Uniform(100);
|
|
|
|
cf_opt->target_file_size_multiplier = rnd->Uniform(100);
|
2015-08-27 01:13:56 +02:00
|
|
|
|
|
|
|
// size_t options
|
RocksDB Options file format and its serialization / deserialization.
Summary:
This patch defines the format of RocksDB options file, which
follows the INI file format, and implements functions for its
serialization and deserialization. An example RocksDB options
file can be found in examples/rocksdb_option_file_example.ini.
A typical RocksDB options file has three sections, which are
Version, DBOptions, and more than one CFOptions. The RocksDB
options file in general follows the basic INI file format
with the following extensions / modifications:
* Escaped characters
We escaped the following characters:
- \n -- line feed - new line
- \r -- carriage return
- \\ -- backslash \
- \: -- colon symbol :
- \# -- hash tag #
* Comments
We support # style comments. Comments can appear at the ending
part of a line.
* Statements
A statement is of the form option_name = value.
Each statement contains a '=', where extra white-spaces
are supported. However, we don't support multi-lined statement.
Furthermore, each line can only contain at most one statement.
* Section
Sections are of the form [SecitonTitle "SectionArgument"],
where section argument is optional.
* List
We use colon-separated string to represent a list.
For instance, n1:n2:n3:n4 is a list containing four values.
Below is an example of a RocksDB options file:
[Version]
rocksdb_version=4.0.0
options_file_version=1.0
[DBOptions]
max_open_files=12345
max_background_flushes=301
[CFOptions "default"]
[CFOptions "the second column family"]
[CFOptions "the third column family"]
Test Plan: Added many tests in options_test.cc
Reviewers: igor, IslamAbdelRahman, sdong, anthony
Reviewed By: anthony
Subscribers: maykov, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D46059
2015-09-29 23:42:40 +02:00
|
|
|
cf_opt->arena_block_size = rnd->Uniform(10000);
|
|
|
|
cf_opt->inplace_update_num_locks = rnd->Uniform(10000);
|
|
|
|
cf_opt->max_successive_merges = rnd->Uniform(10000);
|
|
|
|
cf_opt->memtable_prefix_bloom_huge_page_tlb_size = rnd->Uniform(10000);
|
|
|
|
cf_opt->write_buffer_size = rnd->Uniform(10000);
|
2015-08-27 01:13:56 +02:00
|
|
|
|
|
|
|
// uint32_t options
|
RocksDB Options file format and its serialization / deserialization.
Summary:
This patch defines the format of RocksDB options file, which
follows the INI file format, and implements functions for its
serialization and deserialization. An example RocksDB options
file can be found in examples/rocksdb_option_file_example.ini.
A typical RocksDB options file has three sections, which are
Version, DBOptions, and more than one CFOptions. The RocksDB
options file in general follows the basic INI file format
with the following extensions / modifications:
* Escaped characters
We escaped the following characters:
- \n -- line feed - new line
- \r -- carriage return
- \\ -- backslash \
- \: -- colon symbol :
- \# -- hash tag #
* Comments
We support # style comments. Comments can appear at the ending
part of a line.
* Statements
A statement is of the form option_name = value.
Each statement contains a '=', where extra white-spaces
are supported. However, we don't support multi-lined statement.
Furthermore, each line can only contain at most one statement.
* Section
Sections are of the form [SecitonTitle "SectionArgument"],
where section argument is optional.
* List
We use colon-separated string to represent a list.
For instance, n1:n2:n3:n4 is a list containing four values.
Below is an example of a RocksDB options file:
[Version]
rocksdb_version=4.0.0
options_file_version=1.0
[DBOptions]
max_open_files=12345
max_background_flushes=301
[CFOptions "default"]
[CFOptions "the second column family"]
[CFOptions "the third column family"]
Test Plan: Added many tests in options_test.cc
Reviewers: igor, IslamAbdelRahman, sdong, anthony
Reviewed By: anthony
Subscribers: maykov, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D46059
2015-09-29 23:42:40 +02:00
|
|
|
cf_opt->bloom_locality = rnd->Uniform(10000);
|
|
|
|
cf_opt->memtable_prefix_bloom_bits = rnd->Uniform(10000);
|
|
|
|
cf_opt->memtable_prefix_bloom_probes = rnd->Uniform(10000);
|
|
|
|
cf_opt->min_partial_merge_operands = rnd->Uniform(10000);
|
|
|
|
cf_opt->max_bytes_for_level_base = rnd->Uniform(10000);
|
2015-08-27 01:13:56 +02:00
|
|
|
|
|
|
|
// uint64_t options
|
RocksDB Options file format and its serialization / deserialization.
Summary:
This patch defines the format of RocksDB options file, which
follows the INI file format, and implements functions for its
serialization and deserialization. An example RocksDB options
file can be found in examples/rocksdb_option_file_example.ini.
A typical RocksDB options file has three sections, which are
Version, DBOptions, and more than one CFOptions. The RocksDB
options file in general follows the basic INI file format
with the following extensions / modifications:
* Escaped characters
We escaped the following characters:
- \n -- line feed - new line
- \r -- carriage return
- \\ -- backslash \
- \: -- colon symbol :
- \# -- hash tag #
* Comments
We support # style comments. Comments can appear at the ending
part of a line.
* Statements
A statement is of the form option_name = value.
Each statement contains a '=', where extra white-spaces
are supported. However, we don't support multi-lined statement.
Furthermore, each line can only contain at most one statement.
* Section
Sections are of the form [SecitonTitle "SectionArgument"],
where section argument is optional.
* List
We use colon-separated string to represent a list.
For instance, n1:n2:n3:n4 is a list containing four values.
Below is an example of a RocksDB options file:
[Version]
rocksdb_version=4.0.0
options_file_version=1.0
[DBOptions]
max_open_files=12345
max_background_flushes=301
[CFOptions "default"]
[CFOptions "the second column family"]
[CFOptions "the third column family"]
Test Plan: Added many tests in options_test.cc
Reviewers: igor, IslamAbdelRahman, sdong, anthony
Reviewed By: anthony
Subscribers: maykov, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D46059
2015-09-29 23:42:40 +02:00
|
|
|
static const uint64_t uint_max = static_cast<uint64_t>(UINT_MAX);
|
|
|
|
cf_opt->max_sequential_skip_in_iterations = uint_max + rnd->Uniform(10000);
|
|
|
|
cf_opt->target_file_size_base = uint_max + rnd->Uniform(10000);
|
2015-08-27 01:13:56 +02:00
|
|
|
|
|
|
|
// unsigned int options
|
RocksDB Options file format and its serialization / deserialization.
Summary:
This patch defines the format of RocksDB options file, which
follows the INI file format, and implements functions for its
serialization and deserialization. An example RocksDB options
file can be found in examples/rocksdb_option_file_example.ini.
A typical RocksDB options file has three sections, which are
Version, DBOptions, and more than one CFOptions. The RocksDB
options file in general follows the basic INI file format
with the following extensions / modifications:
* Escaped characters
We escaped the following characters:
- \n -- line feed - new line
- \r -- carriage return
- \\ -- backslash \
- \: -- colon symbol :
- \# -- hash tag #
* Comments
We support # style comments. Comments can appear at the ending
part of a line.
* Statements
A statement is of the form option_name = value.
Each statement contains a '=', where extra white-spaces
are supported. However, we don't support multi-lined statement.
Furthermore, each line can only contain at most one statement.
* Section
Sections are of the form [SecitonTitle "SectionArgument"],
where section argument is optional.
* List
We use colon-separated string to represent a list.
For instance, n1:n2:n3:n4 is a list containing four values.
Below is an example of a RocksDB options file:
[Version]
rocksdb_version=4.0.0
options_file_version=1.0
[DBOptions]
max_open_files=12345
max_background_flushes=301
[CFOptions "default"]
[CFOptions "the second column family"]
[CFOptions "the third column family"]
Test Plan: Added many tests in options_test.cc
Reviewers: igor, IslamAbdelRahman, sdong, anthony
Reviewed By: anthony
Subscribers: maykov, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D46059
2015-09-29 23:42:40 +02:00
|
|
|
cf_opt->rate_limit_delay_max_milliseconds = rnd->Uniform(10000);
|
2015-10-03 00:35:32 +02:00
|
|
|
|
|
|
|
// pointer typed options
|
|
|
|
cf_opt->prefix_extractor.reset(RandomSliceTransform(rnd));
|
|
|
|
cf_opt->table_factory.reset(RandomTableFactory(rnd));
|
|
|
|
cf_opt->merge_operator.reset(RandomMergeOperator(rnd));
|
|
|
|
if (cf_opt->compaction_filter) {
|
|
|
|
delete cf_opt->compaction_filter;
|
|
|
|
}
|
|
|
|
cf_opt->compaction_filter = RandomCompactionFilter(rnd);
|
|
|
|
cf_opt->compaction_filter_factory.reset(RandomCompactionFilterFactory(rnd));
|
|
|
|
|
|
|
|
// custom typed options
|
|
|
|
cf_opt->compression = RandomCompressionType(rnd);
|
|
|
|
RandomCompressionTypeVector(cf_opt->num_levels,
|
|
|
|
&cf_opt->compression_per_level, rnd);
|
2015-08-27 01:13:56 +02:00
|
|
|
}
|
RocksDB Options file format and its serialization / deserialization.
Summary:
This patch defines the format of RocksDB options file, which
follows the INI file format, and implements functions for its
serialization and deserialization. An example RocksDB options
file can be found in examples/rocksdb_option_file_example.ini.
A typical RocksDB options file has three sections, which are
Version, DBOptions, and more than one CFOptions. The RocksDB
options file in general follows the basic INI file format
with the following extensions / modifications:
* Escaped characters
We escaped the following characters:
- \n -- line feed - new line
- \r -- carriage return
- \\ -- backslash \
- \: -- colon symbol :
- \# -- hash tag #
* Comments
We support # style comments. Comments can appear at the ending
part of a line.
* Statements
A statement is of the form option_name = value.
Each statement contains a '=', where extra white-spaces
are supported. However, we don't support multi-lined statement.
Furthermore, each line can only contain at most one statement.
* Section
Sections are of the form [SecitonTitle "SectionArgument"],
where section argument is optional.
* List
We use colon-separated string to represent a list.
For instance, n1:n2:n3:n4 is a list containing four values.
Below is an example of a RocksDB options file:
[Version]
rocksdb_version=4.0.0
options_file_version=1.0
[DBOptions]
max_open_files=12345
max_background_flushes=301
[CFOptions "default"]
[CFOptions "the second column family"]
[CFOptions "the third column family"]
Test Plan: Added many tests in options_test.cc
Reviewers: igor, IslamAbdelRahman, sdong, anthony
Reviewed By: anthony
Subscribers: maykov, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D46059
2015-09-29 23:42:40 +02:00
|
|
|
|
2015-08-27 01:13:56 +02:00
|
|
|
} // namespace
|
|
|
|
|
|
|
|
TEST_F(OptionsTest, ColumnFamilyOptionsSerialization) {
|
|
|
|
ColumnFamilyOptions base_opt, new_opt;
|
|
|
|
Random rnd(302);
|
|
|
|
// Phase 1: randomly assign base_opt
|
|
|
|
// custom type options
|
RocksDB Options file format and its serialization / deserialization.
Summary:
This patch defines the format of RocksDB options file, which
follows the INI file format, and implements functions for its
serialization and deserialization. An example RocksDB options
file can be found in examples/rocksdb_option_file_example.ini.
A typical RocksDB options file has three sections, which are
Version, DBOptions, and more than one CFOptions. The RocksDB
options file in general follows the basic INI file format
with the following extensions / modifications:
* Escaped characters
We escaped the following characters:
- \n -- line feed - new line
- \r -- carriage return
- \\ -- backslash \
- \: -- colon symbol :
- \# -- hash tag #
* Comments
We support # style comments. Comments can appear at the ending
part of a line.
* Statements
A statement is of the form option_name = value.
Each statement contains a '=', where extra white-spaces
are supported. However, we don't support multi-lined statement.
Furthermore, each line can only contain at most one statement.
* Section
Sections are of the form [SecitonTitle "SectionArgument"],
where section argument is optional.
* List
We use colon-separated string to represent a list.
For instance, n1:n2:n3:n4 is a list containing four values.
Below is an example of a RocksDB options file:
[Version]
rocksdb_version=4.0.0
options_file_version=1.0
[DBOptions]
max_open_files=12345
max_background_flushes=301
[CFOptions "default"]
[CFOptions "the second column family"]
[CFOptions "the third column family"]
Test Plan: Added many tests in options_test.cc
Reviewers: igor, IslamAbdelRahman, sdong, anthony
Reviewed By: anthony
Subscribers: maykov, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D46059
2015-09-29 23:42:40 +02:00
|
|
|
RandomInitCFOptions(&base_opt, &rnd);
|
2015-08-27 01:13:56 +02:00
|
|
|
|
|
|
|
// Phase 2: obtain a string from base_opt
|
RocksDB Options file format and its serialization / deserialization.
Summary:
This patch defines the format of RocksDB options file, which
follows the INI file format, and implements functions for its
serialization and deserialization. An example RocksDB options
file can be found in examples/rocksdb_option_file_example.ini.
A typical RocksDB options file has three sections, which are
Version, DBOptions, and more than one CFOptions. The RocksDB
options file in general follows the basic INI file format
with the following extensions / modifications:
* Escaped characters
We escaped the following characters:
- \n -- line feed - new line
- \r -- carriage return
- \\ -- backslash \
- \: -- colon symbol :
- \# -- hash tag #
* Comments
We support # style comments. Comments can appear at the ending
part of a line.
* Statements
A statement is of the form option_name = value.
Each statement contains a '=', where extra white-spaces
are supported. However, we don't support multi-lined statement.
Furthermore, each line can only contain at most one statement.
* Section
Sections are of the form [SecitonTitle "SectionArgument"],
where section argument is optional.
* List
We use colon-separated string to represent a list.
For instance, n1:n2:n3:n4 is a list containing four values.
Below is an example of a RocksDB options file:
[Version]
rocksdb_version=4.0.0
options_file_version=1.0
[DBOptions]
max_open_files=12345
max_background_flushes=301
[CFOptions "default"]
[CFOptions "the second column family"]
[CFOptions "the third column family"]
Test Plan: Added many tests in options_test.cc
Reviewers: igor, IslamAbdelRahman, sdong, anthony
Reviewed By: anthony
Subscribers: maykov, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D46059
2015-09-29 23:42:40 +02:00
|
|
|
std::string base_options_file_content;
|
|
|
|
ASSERT_OK(
|
|
|
|
GetStringFromColumnFamilyOptions(&base_options_file_content, base_opt));
|
2015-08-27 01:13:56 +02:00
|
|
|
|
|
|
|
// Phase 3: Set new_opt from the derived string and expect
|
|
|
|
// new_opt == base_opt
|
RocksDB Options file format and its serialization / deserialization.
Summary:
This patch defines the format of RocksDB options file, which
follows the INI file format, and implements functions for its
serialization and deserialization. An example RocksDB options
file can be found in examples/rocksdb_option_file_example.ini.
A typical RocksDB options file has three sections, which are
Version, DBOptions, and more than one CFOptions. The RocksDB
options file in general follows the basic INI file format
with the following extensions / modifications:
* Escaped characters
We escaped the following characters:
- \n -- line feed - new line
- \r -- carriage return
- \\ -- backslash \
- \: -- colon symbol :
- \# -- hash tag #
* Comments
We support # style comments. Comments can appear at the ending
part of a line.
* Statements
A statement is of the form option_name = value.
Each statement contains a '=', where extra white-spaces
are supported. However, we don't support multi-lined statement.
Furthermore, each line can only contain at most one statement.
* Section
Sections are of the form [SecitonTitle "SectionArgument"],
where section argument is optional.
* List
We use colon-separated string to represent a list.
For instance, n1:n2:n3:n4 is a list containing four values.
Below is an example of a RocksDB options file:
[Version]
rocksdb_version=4.0.0
options_file_version=1.0
[DBOptions]
max_open_files=12345
max_background_flushes=301
[CFOptions "default"]
[CFOptions "the second column family"]
[CFOptions "the third column family"]
Test Plan: Added many tests in options_test.cc
Reviewers: igor, IslamAbdelRahman, sdong, anthony
Reviewed By: anthony
Subscribers: maykov, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D46059
2015-09-29 23:42:40 +02:00
|
|
|
ASSERT_OK(GetColumnFamilyOptionsFromString(
|
|
|
|
ColumnFamilyOptions(), base_options_file_content, &new_opt));
|
|
|
|
ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_opt, new_opt));
|
2015-10-03 00:35:32 +02:00
|
|
|
if (base_opt.compaction_filter) {
|
|
|
|
delete base_opt.compaction_filter;
|
|
|
|
}
|
2015-08-27 01:13:56 +02:00
|
|
|
}
|
|
|
|
|
2015-02-20 04:11:14 +01:00
|
|
|
#endif // !ROCKSDB_LITE
|
|
|
|
|
|
|
|
|
2014-12-22 22:18:57 +01:00
|
|
|
Status StringToMap(
|
|
|
|
const std::string& opts_str,
|
|
|
|
std::unordered_map<std::string, std::string>* opts_map);
|
|
|
|
|
2015-01-23 01:57:16 +01:00
|
|
|
#ifndef ROCKSDB_LITE // StringToMap is not supported in ROCKSDB_LITE
|
2015-03-17 22:08:00 +01:00
|
|
|
TEST_F(OptionsTest, StringToMapTest) {
|
2014-12-22 22:18:57 +01:00
|
|
|
std::unordered_map<std::string, std::string> opts_map;
|
|
|
|
// Regular options
|
|
|
|
ASSERT_OK(StringToMap("k1=v1;k2=v2;k3=v3", &opts_map));
|
|
|
|
ASSERT_EQ(opts_map["k1"], "v1");
|
|
|
|
ASSERT_EQ(opts_map["k2"], "v2");
|
|
|
|
ASSERT_EQ(opts_map["k3"], "v3");
|
|
|
|
// Value with '='
|
|
|
|
opts_map.clear();
|
|
|
|
ASSERT_OK(StringToMap("k1==v1;k2=v2=;", &opts_map));
|
|
|
|
ASSERT_EQ(opts_map["k1"], "=v1");
|
|
|
|
ASSERT_EQ(opts_map["k2"], "v2=");
|
|
|
|
// Overwrriten option
|
|
|
|
opts_map.clear();
|
|
|
|
ASSERT_OK(StringToMap("k1=v1;k1=v2;k3=v3", &opts_map));
|
|
|
|
ASSERT_EQ(opts_map["k1"], "v2");
|
|
|
|
ASSERT_EQ(opts_map["k3"], "v3");
|
|
|
|
// Empty value
|
|
|
|
opts_map.clear();
|
|
|
|
ASSERT_OK(StringToMap("k1=v1;k2=;k3=v3;k4=", &opts_map));
|
|
|
|
ASSERT_EQ(opts_map["k1"], "v1");
|
|
|
|
ASSERT_TRUE(opts_map.find("k2") != opts_map.end());
|
|
|
|
ASSERT_EQ(opts_map["k2"], "");
|
|
|
|
ASSERT_EQ(opts_map["k3"], "v3");
|
|
|
|
ASSERT_TRUE(opts_map.find("k4") != opts_map.end());
|
|
|
|
ASSERT_EQ(opts_map["k4"], "");
|
|
|
|
opts_map.clear();
|
|
|
|
ASSERT_OK(StringToMap("k1=v1;k2=;k3=v3;k4= ", &opts_map));
|
|
|
|
ASSERT_EQ(opts_map["k1"], "v1");
|
|
|
|
ASSERT_TRUE(opts_map.find("k2") != opts_map.end());
|
|
|
|
ASSERT_EQ(opts_map["k2"], "");
|
|
|
|
ASSERT_EQ(opts_map["k3"], "v3");
|
|
|
|
ASSERT_TRUE(opts_map.find("k4") != opts_map.end());
|
|
|
|
ASSERT_EQ(opts_map["k4"], "");
|
|
|
|
opts_map.clear();
|
|
|
|
ASSERT_OK(StringToMap("k1=v1;k2=;k3=", &opts_map));
|
|
|
|
ASSERT_EQ(opts_map["k1"], "v1");
|
|
|
|
ASSERT_TRUE(opts_map.find("k2") != opts_map.end());
|
|
|
|
ASSERT_EQ(opts_map["k2"], "");
|
|
|
|
ASSERT_TRUE(opts_map.find("k3") != opts_map.end());
|
|
|
|
ASSERT_EQ(opts_map["k3"], "");
|
|
|
|
opts_map.clear();
|
|
|
|
ASSERT_OK(StringToMap("k1=v1;k2=;k3=;", &opts_map));
|
|
|
|
ASSERT_EQ(opts_map["k1"], "v1");
|
|
|
|
ASSERT_TRUE(opts_map.find("k2") != opts_map.end());
|
|
|
|
ASSERT_EQ(opts_map["k2"], "");
|
|
|
|
ASSERT_TRUE(opts_map.find("k3") != opts_map.end());
|
|
|
|
ASSERT_EQ(opts_map["k3"], "");
|
|
|
|
// Regular nested options
|
|
|
|
opts_map.clear();
|
|
|
|
ASSERT_OK(StringToMap("k1=v1;k2={nk1=nv1;nk2=nv2};k3=v3", &opts_map));
|
|
|
|
ASSERT_EQ(opts_map["k1"], "v1");
|
|
|
|
ASSERT_EQ(opts_map["k2"], "nk1=nv1;nk2=nv2");
|
|
|
|
ASSERT_EQ(opts_map["k3"], "v3");
|
|
|
|
// Multi-level nested options
|
|
|
|
opts_map.clear();
|
|
|
|
ASSERT_OK(StringToMap("k1=v1;k2={nk1=nv1;nk2={nnk1=nnk2}};"
|
|
|
|
"k3={nk1={nnk1={nnnk1=nnnv1;nnnk2;nnnv2}}};k4=v4",
|
|
|
|
&opts_map));
|
|
|
|
ASSERT_EQ(opts_map["k1"], "v1");
|
|
|
|
ASSERT_EQ(opts_map["k2"], "nk1=nv1;nk2={nnk1=nnk2}");
|
|
|
|
ASSERT_EQ(opts_map["k3"], "nk1={nnk1={nnnk1=nnnv1;nnnk2;nnnv2}}");
|
|
|
|
ASSERT_EQ(opts_map["k4"], "v4");
|
|
|
|
// Garbage inside curly braces
|
|
|
|
opts_map.clear();
|
|
|
|
ASSERT_OK(StringToMap("k1=v1;k2={dfad=};k3={=};k4=v4",
|
|
|
|
&opts_map));
|
|
|
|
ASSERT_EQ(opts_map["k1"], "v1");
|
|
|
|
ASSERT_EQ(opts_map["k2"], "dfad=");
|
|
|
|
ASSERT_EQ(opts_map["k3"], "=");
|
|
|
|
ASSERT_EQ(opts_map["k4"], "v4");
|
|
|
|
// Empty nested options
|
|
|
|
opts_map.clear();
|
|
|
|
ASSERT_OK(StringToMap("k1=v1;k2={};", &opts_map));
|
|
|
|
ASSERT_EQ(opts_map["k1"], "v1");
|
|
|
|
ASSERT_EQ(opts_map["k2"], "");
|
|
|
|
opts_map.clear();
|
|
|
|
ASSERT_OK(StringToMap("k1=v1;k2={{{{}}}{}{}};", &opts_map));
|
|
|
|
ASSERT_EQ(opts_map["k1"], "v1");
|
|
|
|
ASSERT_EQ(opts_map["k2"], "{{{}}}{}{}");
|
|
|
|
// With random spaces
|
|
|
|
opts_map.clear();
|
|
|
|
ASSERT_OK(StringToMap(" k1 = v1 ; k2= {nk1=nv1; nk2={nnk1=nnk2}} ; "
|
|
|
|
"k3={ { } }; k4= v4 ",
|
|
|
|
&opts_map));
|
|
|
|
ASSERT_EQ(opts_map["k1"], "v1");
|
|
|
|
ASSERT_EQ(opts_map["k2"], "nk1=nv1; nk2={nnk1=nnk2}");
|
|
|
|
ASSERT_EQ(opts_map["k3"], "{ }");
|
|
|
|
ASSERT_EQ(opts_map["k4"], "v4");
|
|
|
|
|
|
|
|
// Empty key
|
|
|
|
ASSERT_NOK(StringToMap("k1=v1;k2=v2;=", &opts_map));
|
|
|
|
ASSERT_NOK(StringToMap("=v1;k2=v2", &opts_map));
|
|
|
|
ASSERT_NOK(StringToMap("k1=v1;k2v2;", &opts_map));
|
|
|
|
ASSERT_NOK(StringToMap("k1=v1;k2=v2;fadfa", &opts_map));
|
|
|
|
ASSERT_NOK(StringToMap("k1=v1;k2=v2;;", &opts_map));
|
|
|
|
// Mismatch curly braces
|
|
|
|
ASSERT_NOK(StringToMap("k1=v1;k2={;k3=v3", &opts_map));
|
|
|
|
ASSERT_NOK(StringToMap("k1=v1;k2={{};k3=v3", &opts_map));
|
|
|
|
ASSERT_NOK(StringToMap("k1=v1;k2={}};k3=v3", &opts_map));
|
|
|
|
ASSERT_NOK(StringToMap("k1=v1;k2={{}{}}};k3=v3", &opts_map));
|
|
|
|
// However this is valid!
|
|
|
|
opts_map.clear();
|
|
|
|
ASSERT_OK(StringToMap("k1=v1;k2=};k3=v3", &opts_map));
|
|
|
|
ASSERT_EQ(opts_map["k1"], "v1");
|
|
|
|
ASSERT_EQ(opts_map["k2"], "}");
|
|
|
|
ASSERT_EQ(opts_map["k3"], "v3");
|
|
|
|
|
|
|
|
// Invalid chars after closing curly brace
|
|
|
|
ASSERT_NOK(StringToMap("k1=v1;k2={{}}{};k3=v3", &opts_map));
|
|
|
|
ASSERT_NOK(StringToMap("k1=v1;k2={{}}cfda;k3=v3", &opts_map));
|
|
|
|
ASSERT_NOK(StringToMap("k1=v1;k2={{}} cfda;k3=v3", &opts_map));
|
|
|
|
ASSERT_NOK(StringToMap("k1=v1;k2={{}} cfda", &opts_map));
|
|
|
|
ASSERT_NOK(StringToMap("k1=v1;k2={{}}{}", &opts_map));
|
|
|
|
ASSERT_NOK(StringToMap("k1=v1;k2={{dfdl}adfa}{}", &opts_map));
|
|
|
|
}
|
2015-01-23 01:57:16 +01:00
|
|
|
#endif // ROCKSDB_LITE
|
2014-12-22 22:18:57 +01:00
|
|
|
|
2015-01-23 01:57:16 +01:00
|
|
|
#ifndef ROCKSDB_LITE // StringToMap is not supported in ROCKSDB_LITE
|
2015-03-17 22:08:00 +01:00
|
|
|
TEST_F(OptionsTest, StringToMapRandomTest) {
|
2014-12-22 22:18:57 +01:00
|
|
|
std::unordered_map<std::string, std::string> opts_map;
|
|
|
|
// Make sure segfault is not hit by semi-random strings
|
|
|
|
|
|
|
|
std::vector<std::string> bases = {
|
|
|
|
"a={aa={};tt={xxx={}}};c=defff",
|
|
|
|
"a={aa={};tt={xxx={}}};c=defff;d={{}yxx{}3{xx}}",
|
|
|
|
"abc={{}{}{}{{{}}}{{}{}{}{}{}{}{}"};
|
|
|
|
|
|
|
|
for (std::string base : bases) {
|
|
|
|
for (int rand_seed = 301; rand_seed < 401; rand_seed++) {
|
|
|
|
Random rnd(rand_seed);
|
|
|
|
for (int attempt = 0; attempt < 10; attempt++) {
|
|
|
|
std::string str = base;
|
|
|
|
// Replace random position to space
|
|
|
|
size_t pos = static_cast<size_t>(
|
|
|
|
rnd.Uniform(static_cast<int>(base.size())));
|
|
|
|
str[pos] = ' ';
|
|
|
|
Status s = StringToMap(str, &opts_map);
|
|
|
|
ASSERT_TRUE(s.ok() || s.IsInvalidArgument());
|
|
|
|
opts_map.clear();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Random Construct a string
|
|
|
|
std::vector<char> chars = {'{', '}', ' ', '=', ';', 'c'};
|
|
|
|
for (int rand_seed = 301; rand_seed < 1301; rand_seed++) {
|
|
|
|
Random rnd(rand_seed);
|
|
|
|
int len = rnd.Uniform(30);
|
|
|
|
std::string str = "";
|
|
|
|
for (int attempt = 0; attempt < len; attempt++) {
|
|
|
|
// Add a random character
|
|
|
|
size_t pos = static_cast<size_t>(
|
|
|
|
rnd.Uniform(static_cast<int>(chars.size())));
|
|
|
|
str.append(1, chars[pos]);
|
|
|
|
}
|
|
|
|
Status s = StringToMap(str, &opts_map);
|
|
|
|
ASSERT_TRUE(s.ok() || s.IsInvalidArgument());
|
|
|
|
s = StringToMap("name=" + str, &opts_map);
|
|
|
|
ASSERT_TRUE(s.ok() || s.IsInvalidArgument());
|
|
|
|
opts_map.clear();
|
|
|
|
}
|
2014-09-17 21:46:32 +02:00
|
|
|
}
|
2015-01-23 01:57:16 +01:00
|
|
|
#endif // !ROCKSDB_LITE
|
2014-09-17 21:46:32 +02:00
|
|
|
|
2015-03-17 22:08:00 +01:00
|
|
|
TEST_F(OptionsTest, ConvertOptionsTest) {
|
2014-11-21 04:24:39 +01:00
|
|
|
LevelDBOptions leveldb_opt;
|
|
|
|
Options converted_opt = ConvertOptions(leveldb_opt);
|
|
|
|
|
|
|
|
ASSERT_EQ(converted_opt.create_if_missing, leveldb_opt.create_if_missing);
|
|
|
|
ASSERT_EQ(converted_opt.error_if_exists, leveldb_opt.error_if_exists);
|
|
|
|
ASSERT_EQ(converted_opt.paranoid_checks, leveldb_opt.paranoid_checks);
|
|
|
|
ASSERT_EQ(converted_opt.env, leveldb_opt.env);
|
|
|
|
ASSERT_EQ(converted_opt.info_log.get(), leveldb_opt.info_log);
|
|
|
|
ASSERT_EQ(converted_opt.write_buffer_size, leveldb_opt.write_buffer_size);
|
|
|
|
ASSERT_EQ(converted_opt.max_open_files, leveldb_opt.max_open_files);
|
|
|
|
ASSERT_EQ(converted_opt.compression, leveldb_opt.compression);
|
|
|
|
|
|
|
|
std::shared_ptr<BlockBasedTableFactory> table_factory =
|
|
|
|
std::dynamic_pointer_cast<BlockBasedTableFactory>(
|
|
|
|
converted_opt.table_factory);
|
|
|
|
|
|
|
|
ASSERT_TRUE(table_factory.get() != nullptr);
|
|
|
|
|
|
|
|
const BlockBasedTableOptions table_opt = table_factory->GetTableOptions();
|
|
|
|
|
|
|
|
ASSERT_EQ(table_opt.block_cache->GetCapacity(), 8UL << 20);
|
|
|
|
ASSERT_EQ(table_opt.block_size, leveldb_opt.block_size);
|
|
|
|
ASSERT_EQ(table_opt.block_restart_interval,
|
|
|
|
leveldb_opt.block_restart_interval);
|
|
|
|
ASSERT_EQ(table_opt.filter_policy.get(), leveldb_opt.filter_policy);
|
|
|
|
}
|
|
|
|
|
RocksDB Options file format and its serialization / deserialization.
Summary:
This patch defines the format of RocksDB options file, which
follows the INI file format, and implements functions for its
serialization and deserialization. An example RocksDB options
file can be found in examples/rocksdb_option_file_example.ini.
A typical RocksDB options file has three sections, which are
Version, DBOptions, and more than one CFOptions. The RocksDB
options file in general follows the basic INI file format
with the following extensions / modifications:
* Escaped characters
We escaped the following characters:
- \n -- line feed - new line
- \r -- carriage return
- \\ -- backslash \
- \: -- colon symbol :
- \# -- hash tag #
* Comments
We support # style comments. Comments can appear at the ending
part of a line.
* Statements
A statement is of the form option_name = value.
Each statement contains a '=', where extra white-spaces
are supported. However, we don't support multi-lined statement.
Furthermore, each line can only contain at most one statement.
* Section
Sections are of the form [SecitonTitle "SectionArgument"],
where section argument is optional.
* List
We use colon-separated string to represent a list.
For instance, n1:n2:n3:n4 is a list containing four values.
Below is an example of a RocksDB options file:
[Version]
rocksdb_version=4.0.0
options_file_version=1.0
[DBOptions]
max_open_files=12345
max_background_flushes=301
[CFOptions "default"]
[CFOptions "the second column family"]
[CFOptions "the third column family"]
Test Plan: Added many tests in options_test.cc
Reviewers: igor, IslamAbdelRahman, sdong, anthony
Reviewed By: anthony
Subscribers: maykov, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D46059
2015-09-29 23:42:40 +02:00
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
class OptionsParserTest : public testing::Test {
|
|
|
|
public:
|
|
|
|
OptionsParserTest() { env_.reset(new StringEnv(Env::Default())); }
|
|
|
|
|
|
|
|
protected:
|
|
|
|
std::unique_ptr<StringEnv> env_;
|
|
|
|
};
|
|
|
|
|
|
|
|
TEST_F(OptionsParserTest, Comment) {
|
|
|
|
DBOptions db_opt;
|
|
|
|
db_opt.max_open_files = 12345;
|
|
|
|
db_opt.max_background_flushes = 301;
|
|
|
|
db_opt.max_total_wal_size = 1024;
|
|
|
|
ColumnFamilyOptions cf_opt;
|
|
|
|
|
|
|
|
std::string options_file_content =
|
|
|
|
"# This is a testing option string.\n"
|
|
|
|
"# Currently we only support \"#\" styled comment.\n"
|
|
|
|
"\n"
|
|
|
|
"[Version]\n"
|
|
|
|
" rocksdb_version=3.14.0\n"
|
|
|
|
" options_file_version=1\n"
|
|
|
|
"[ DBOptions ]\n"
|
|
|
|
" # note that we don't support space around \"=\"\n"
|
|
|
|
" max_open_files=12345;\n"
|
|
|
|
" max_background_flushes=301 # comment after a statement is fine\n"
|
|
|
|
" # max_background_flushes=1000 # this line would be ignored\n"
|
|
|
|
" # max_background_compactions=2000 # so does this one\n"
|
|
|
|
" max_total_wal_size=1024 # keep_log_file_num=1000\n"
|
|
|
|
"[CFOptions \"default\"] # column family must be specified\n"
|
|
|
|
" # in the correct order\n"
|
|
|
|
" # if a section is blank, we will use the default\n";
|
|
|
|
|
|
|
|
const std::string kTestFileName = "test-rocksdb-options.ini";
|
|
|
|
env_->WriteToNewFile(kTestFileName, options_file_content);
|
|
|
|
RocksDBOptionsParser parser;
|
|
|
|
ASSERT_OK(parser.Parse(kTestFileName, env_.get()));
|
|
|
|
|
|
|
|
ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(*parser.db_opt(), db_opt));
|
|
|
|
ASSERT_EQ(parser.NumColumnFamilies(), 1U);
|
|
|
|
ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(
|
|
|
|
*parser.GetCFOptions("default"), cf_opt));
|
|
|
|
}
|
|
|
|
|
|
|
|
TEST_F(OptionsParserTest, ExtraSpace) {
|
|
|
|
std::string options_file_content =
|
|
|
|
"# This is a testing option string.\n"
|
|
|
|
"# Currently we only support \"#\" styled comment.\n"
|
|
|
|
"\n"
|
|
|
|
"[ Version ]\n"
|
|
|
|
" rocksdb_version = 3.14.0 \n"
|
|
|
|
" options_file_version=1 # some comment\n"
|
|
|
|
"[DBOptions ] # some comment\n"
|
|
|
|
"max_open_files=12345 \n"
|
|
|
|
" max_background_flushes = 301 \n"
|
|
|
|
" max_total_wal_size = 1024 # keep_log_file_num=1000\n"
|
|
|
|
" [CFOptions \"default\" ]\n"
|
|
|
|
" # if a section is blank, we will use the default\n";
|
|
|
|
|
|
|
|
const std::string kTestFileName = "test-rocksdb-options.ini";
|
|
|
|
env_->WriteToNewFile(kTestFileName, options_file_content);
|
|
|
|
RocksDBOptionsParser parser;
|
|
|
|
ASSERT_OK(parser.Parse(kTestFileName, env_.get()));
|
|
|
|
}
|
|
|
|
|
|
|
|
TEST_F(OptionsParserTest, MissingDBOptions) {
|
|
|
|
std::string options_file_content =
|
|
|
|
"# This is a testing option string.\n"
|
|
|
|
"# Currently we only support \"#\" styled comment.\n"
|
|
|
|
"\n"
|
|
|
|
"[Version]\n"
|
|
|
|
" rocksdb_version=3.14.0\n"
|
|
|
|
" options_file_version=1\n"
|
|
|
|
"[CFOptions \"default\"]\n"
|
|
|
|
" # if a section is blank, we will use the default\n";
|
|
|
|
|
|
|
|
const std::string kTestFileName = "test-rocksdb-options.ini";
|
|
|
|
env_->WriteToNewFile(kTestFileName, options_file_content);
|
|
|
|
RocksDBOptionsParser parser;
|
|
|
|
ASSERT_NOK(parser.Parse(kTestFileName, env_.get()));
|
|
|
|
}
|
|
|
|
|
|
|
|
TEST_F(OptionsParserTest, DoubleDBOptions) {
|
|
|
|
DBOptions db_opt;
|
|
|
|
db_opt.max_open_files = 12345;
|
|
|
|
db_opt.max_background_flushes = 301;
|
|
|
|
db_opt.max_total_wal_size = 1024;
|
|
|
|
ColumnFamilyOptions cf_opt;
|
|
|
|
|
|
|
|
std::string options_file_content =
|
|
|
|
"# This is a testing option string.\n"
|
|
|
|
"# Currently we only support \"#\" styled comment.\n"
|
|
|
|
"\n"
|
|
|
|
"[Version]\n"
|
|
|
|
" rocksdb_version=3.14.0\n"
|
|
|
|
" options_file_version=1\n"
|
|
|
|
"[DBOptions]\n"
|
|
|
|
" max_open_files=12345\n"
|
|
|
|
" max_background_flushes=301\n"
|
|
|
|
" max_total_wal_size=1024 # keep_log_file_num=1000\n"
|
|
|
|
"[DBOptions]\n"
|
|
|
|
"[CFOptions \"default\"]\n"
|
|
|
|
" # if a section is blank, we will use the default\n";
|
|
|
|
|
|
|
|
const std::string kTestFileName = "test-rocksdb-options.ini";
|
|
|
|
env_->WriteToNewFile(kTestFileName, options_file_content);
|
|
|
|
RocksDBOptionsParser parser;
|
|
|
|
ASSERT_NOK(parser.Parse(kTestFileName, env_.get()));
|
|
|
|
}
|
|
|
|
|
|
|
|
TEST_F(OptionsParserTest, NoDefaultCFOptions) {
|
|
|
|
DBOptions db_opt;
|
|
|
|
db_opt.max_open_files = 12345;
|
|
|
|
db_opt.max_background_flushes = 301;
|
|
|
|
db_opt.max_total_wal_size = 1024;
|
|
|
|
ColumnFamilyOptions cf_opt;
|
|
|
|
|
|
|
|
std::string options_file_content =
|
|
|
|
"# This is a testing option string.\n"
|
|
|
|
"# Currently we only support \"#\" styled comment.\n"
|
|
|
|
"\n"
|
|
|
|
"[Version]\n"
|
|
|
|
" rocksdb_version=3.14.0\n"
|
|
|
|
" options_file_version=1\n"
|
|
|
|
"[DBOptions]\n"
|
|
|
|
" max_open_files=12345\n"
|
|
|
|
" max_background_flushes=301\n"
|
|
|
|
" max_total_wal_size=1024 # keep_log_file_num=1000\n"
|
|
|
|
"[CFOptions \"something_else\"]\n"
|
|
|
|
" # if a section is blank, we will use the default\n";
|
|
|
|
|
|
|
|
const std::string kTestFileName = "test-rocksdb-options.ini";
|
|
|
|
env_->WriteToNewFile(kTestFileName, options_file_content);
|
|
|
|
RocksDBOptionsParser parser;
|
|
|
|
ASSERT_NOK(parser.Parse(kTestFileName, env_.get()));
|
|
|
|
}
|
|
|
|
|
|
|
|
TEST_F(OptionsParserTest, DefaultCFOptionsMustBeTheFirst) {
|
|
|
|
DBOptions db_opt;
|
|
|
|
db_opt.max_open_files = 12345;
|
|
|
|
db_opt.max_background_flushes = 301;
|
|
|
|
db_opt.max_total_wal_size = 1024;
|
|
|
|
ColumnFamilyOptions cf_opt;
|
|
|
|
|
|
|
|
std::string options_file_content =
|
|
|
|
"# This is a testing option string.\n"
|
|
|
|
"# Currently we only support \"#\" styled comment.\n"
|
|
|
|
"\n"
|
|
|
|
"[Version]\n"
|
|
|
|
" rocksdb_version=3.14.0\n"
|
|
|
|
" options_file_version=1\n"
|
|
|
|
"[DBOptions]\n"
|
|
|
|
" max_open_files=12345\n"
|
|
|
|
" max_background_flushes=301\n"
|
|
|
|
" max_total_wal_size=1024 # keep_log_file_num=1000\n"
|
|
|
|
"[CFOptions \"something_else\"]\n"
|
|
|
|
" # if a section is blank, we will use the default\n"
|
|
|
|
"[CFOptions \"default\"]\n"
|
|
|
|
" # if a section is blank, we will use the default\n";
|
|
|
|
|
|
|
|
const std::string kTestFileName = "test-rocksdb-options.ini";
|
|
|
|
env_->WriteToNewFile(kTestFileName, options_file_content);
|
|
|
|
RocksDBOptionsParser parser;
|
|
|
|
ASSERT_NOK(parser.Parse(kTestFileName, env_.get()));
|
|
|
|
}
|
|
|
|
|
|
|
|
TEST_F(OptionsParserTest, DuplicateCFOptions) {
|
|
|
|
DBOptions db_opt;
|
|
|
|
db_opt.max_open_files = 12345;
|
|
|
|
db_opt.max_background_flushes = 301;
|
|
|
|
db_opt.max_total_wal_size = 1024;
|
|
|
|
ColumnFamilyOptions cf_opt;
|
|
|
|
|
|
|
|
std::string options_file_content =
|
|
|
|
"# This is a testing option string.\n"
|
|
|
|
"# Currently we only support \"#\" styled comment.\n"
|
|
|
|
"\n"
|
|
|
|
"[Version]\n"
|
|
|
|
" rocksdb_version=3.14.0\n"
|
|
|
|
" options_file_version=1\n"
|
|
|
|
"[DBOptions]\n"
|
|
|
|
" max_open_files=12345\n"
|
|
|
|
" max_background_flushes=301\n"
|
|
|
|
" max_total_wal_size=1024 # keep_log_file_num=1000\n"
|
|
|
|
"[CFOptions \"default\"]\n"
|
|
|
|
"[CFOptions \"something_else\"]\n"
|
|
|
|
"[CFOptions \"something_else\"]\n";
|
|
|
|
|
|
|
|
const std::string kTestFileName = "test-rocksdb-options.ini";
|
|
|
|
env_->WriteToNewFile(kTestFileName, options_file_content);
|
|
|
|
RocksDBOptionsParser parser;
|
|
|
|
ASSERT_NOK(parser.Parse(kTestFileName, env_.get()));
|
|
|
|
}
|
|
|
|
|
|
|
|
TEST_F(OptionsParserTest, ParseVersion) {
|
|
|
|
DBOptions db_opt;
|
|
|
|
db_opt.max_open_files = 12345;
|
|
|
|
db_opt.max_background_flushes = 301;
|
|
|
|
db_opt.max_total_wal_size = 1024;
|
|
|
|
ColumnFamilyOptions cf_opt;
|
|
|
|
|
|
|
|
std::string file_template =
|
|
|
|
"# This is a testing option string.\n"
|
|
|
|
"# Currently we only support \"#\" styled comment.\n"
|
|
|
|
"\n"
|
|
|
|
"[Version]\n"
|
|
|
|
" rocksdb_version=3.13.1\n"
|
|
|
|
" options_file_version=%s\n"
|
|
|
|
"[DBOptions]\n"
|
|
|
|
"[CFOptions \"default\"]\n";
|
|
|
|
const int kLength = 1000;
|
|
|
|
char buffer[kLength];
|
|
|
|
RocksDBOptionsParser parser;
|
|
|
|
|
|
|
|
const std::vector<std::string> invalid_versions = {
|
|
|
|
"a.b.c", "3.2.2b", "3.-12", "3. 1", // only digits and dots are allowed
|
|
|
|
"1.2.3.4",
|
|
|
|
"1.2.3" // can only contains at most one dot.
|
|
|
|
"0", // options_file_version must be at least one
|
|
|
|
"3..2",
|
|
|
|
".", ".1.2", // must have at least one digit before each dot
|
|
|
|
"1.2.", "1.", "2.34."}; // must have at least one digit after each dot
|
|
|
|
for (auto iv : invalid_versions) {
|
|
|
|
snprintf(buffer, kLength - 1, file_template.c_str(), iv.c_str());
|
|
|
|
|
|
|
|
parser.Reset();
|
|
|
|
env_->WriteToNewFile(iv, buffer);
|
|
|
|
ASSERT_NOK(parser.Parse(iv, env_.get()));
|
|
|
|
}
|
|
|
|
|
|
|
|
const std::vector<std::string> valid_versions = {
|
|
|
|
"1.232", "100", "3.12", "1", "12.3 ", " 1.25 "};
|
|
|
|
for (auto vv : valid_versions) {
|
|
|
|
snprintf(buffer, kLength - 1, file_template.c_str(), vv.c_str());
|
|
|
|
parser.Reset();
|
|
|
|
env_->WriteToNewFile(vv, buffer);
|
|
|
|
ASSERT_OK(parser.Parse(vv, env_.get()));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-10-03 00:35:32 +02:00
|
|
|
void VerifyCFPointerTypedOptions(
|
|
|
|
ColumnFamilyOptions* base_cf_opt, const ColumnFamilyOptions* new_cf_opt,
|
|
|
|
const std::unordered_map<std::string, std::string>* new_cf_opt_map) {
|
|
|
|
std::string name_buffer;
|
|
|
|
ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt,
|
|
|
|
new_cf_opt_map));
|
|
|
|
|
|
|
|
// change the name of merge operator back-and-forth
|
|
|
|
{
|
|
|
|
auto* merge_operator = dynamic_cast<ChanglingMergeOperator*>(
|
|
|
|
base_cf_opt->merge_operator.get());
|
|
|
|
if (merge_operator != nullptr) {
|
|
|
|
name_buffer = merge_operator->Name();
|
|
|
|
// change the name and expect non-ok status
|
|
|
|
merge_operator->SetName("some-other-name");
|
|
|
|
ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(
|
|
|
|
*base_cf_opt, *new_cf_opt, new_cf_opt_map));
|
|
|
|
// change the name back and expect ok status
|
|
|
|
merge_operator->SetName(name_buffer);
|
|
|
|
ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt,
|
|
|
|
new_cf_opt_map));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// change the name of the compaction filter factory back-and-forth
|
|
|
|
{
|
|
|
|
auto* compaction_filter_factory =
|
|
|
|
dynamic_cast<ChanglingCompactionFilterFactory*>(
|
|
|
|
base_cf_opt->compaction_filter_factory.get());
|
|
|
|
if (compaction_filter_factory != nullptr) {
|
|
|
|
name_buffer = compaction_filter_factory->Name();
|
|
|
|
// change the name and expect non-ok status
|
|
|
|
compaction_filter_factory->SetName("some-other-name");
|
|
|
|
ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(
|
|
|
|
*base_cf_opt, *new_cf_opt, new_cf_opt_map));
|
|
|
|
// change the name back and expect ok status
|
|
|
|
compaction_filter_factory->SetName(name_buffer);
|
|
|
|
ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt,
|
|
|
|
new_cf_opt_map));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// test by setting compaction_filter to nullptr
|
|
|
|
{
|
|
|
|
auto* tmp_compaction_filter = base_cf_opt->compaction_filter;
|
|
|
|
if (tmp_compaction_filter != nullptr) {
|
|
|
|
base_cf_opt->compaction_filter = nullptr;
|
|
|
|
// set compaction_filter to nullptr and expect non-ok status
|
|
|
|
ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(
|
|
|
|
*base_cf_opt, *new_cf_opt, new_cf_opt_map));
|
|
|
|
// set the value back and expect ok status
|
|
|
|
base_cf_opt->compaction_filter = tmp_compaction_filter;
|
|
|
|
ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt,
|
|
|
|
new_cf_opt_map));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// test by setting table_factory to nullptr
|
|
|
|
{
|
|
|
|
auto tmp_table_factory = base_cf_opt->table_factory;
|
|
|
|
if (tmp_table_factory != nullptr) {
|
|
|
|
base_cf_opt->table_factory.reset();
|
|
|
|
// set table_factory to nullptr and expect non-ok status
|
|
|
|
ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(
|
|
|
|
*base_cf_opt, *new_cf_opt, new_cf_opt_map));
|
|
|
|
// set the value back and expect ok status
|
|
|
|
base_cf_opt->table_factory = tmp_table_factory;
|
|
|
|
ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt,
|
|
|
|
new_cf_opt_map));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// test by setting memtable_factory to nullptr
|
|
|
|
{
|
|
|
|
auto tmp_memtable_factory = base_cf_opt->memtable_factory;
|
|
|
|
if (tmp_memtable_factory != nullptr) {
|
|
|
|
base_cf_opt->memtable_factory.reset();
|
|
|
|
// set memtable_factory to nullptr and expect non-ok status
|
|
|
|
ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(
|
|
|
|
*base_cf_opt, *new_cf_opt, new_cf_opt_map));
|
|
|
|
// set the value back and expect ok status
|
|
|
|
base_cf_opt->memtable_factory = tmp_memtable_factory;
|
|
|
|
ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt,
|
|
|
|
new_cf_opt_map));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
RocksDB Options file format and its serialization / deserialization.
Summary:
This patch defines the format of RocksDB options file, which
follows the INI file format, and implements functions for its
serialization and deserialization. An example RocksDB options
file can be found in examples/rocksdb_option_file_example.ini.
A typical RocksDB options file has three sections, which are
Version, DBOptions, and more than one CFOptions. The RocksDB
options file in general follows the basic INI file format
with the following extensions / modifications:
* Escaped characters
We escaped the following characters:
- \n -- line feed - new line
- \r -- carriage return
- \\ -- backslash \
- \: -- colon symbol :
- \# -- hash tag #
* Comments
We support # style comments. Comments can appear at the ending
part of a line.
* Statements
A statement is of the form option_name = value.
Each statement contains a '=', where extra white-spaces
are supported. However, we don't support multi-lined statement.
Furthermore, each line can only contain at most one statement.
* Section
Sections are of the form [SecitonTitle "SectionArgument"],
where section argument is optional.
* List
We use colon-separated string to represent a list.
For instance, n1:n2:n3:n4 is a list containing four values.
Below is an example of a RocksDB options file:
[Version]
rocksdb_version=4.0.0
options_file_version=1.0
[DBOptions]
max_open_files=12345
max_background_flushes=301
[CFOptions "default"]
[CFOptions "the second column family"]
[CFOptions "the third column family"]
Test Plan: Added many tests in options_test.cc
Reviewers: igor, IslamAbdelRahman, sdong, anthony
Reviewed By: anthony
Subscribers: maykov, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D46059
2015-09-29 23:42:40 +02:00
|
|
|
TEST_F(OptionsParserTest, DumpAndParse) {
|
|
|
|
DBOptions base_db_opt;
|
|
|
|
std::vector<ColumnFamilyOptions> base_cf_opts;
|
2015-10-03 00:35:32 +02:00
|
|
|
std::vector<std::string> cf_names = {"default", "cf1", "cf2", "cf3",
|
|
|
|
"c:f:4:4:4"
|
|
|
|
"p\\i\\k\\a\\chu\\\\\\",
|
|
|
|
"###rocksdb#1-testcf#2###"};
|
RocksDB Options file format and its serialization / deserialization.
Summary:
This patch defines the format of RocksDB options file, which
follows the INI file format, and implements functions for its
serialization and deserialization. An example RocksDB options
file can be found in examples/rocksdb_option_file_example.ini.
A typical RocksDB options file has three sections, which are
Version, DBOptions, and more than one CFOptions. The RocksDB
options file in general follows the basic INI file format
with the following extensions / modifications:
* Escaped characters
We escaped the following characters:
- \n -- line feed - new line
- \r -- carriage return
- \\ -- backslash \
- \: -- colon symbol :
- \# -- hash tag #
* Comments
We support # style comments. Comments can appear at the ending
part of a line.
* Statements
A statement is of the form option_name = value.
Each statement contains a '=', where extra white-spaces
are supported. However, we don't support multi-lined statement.
Furthermore, each line can only contain at most one statement.
* Section
Sections are of the form [SecitonTitle "SectionArgument"],
where section argument is optional.
* List
We use colon-separated string to represent a list.
For instance, n1:n2:n3:n4 is a list containing four values.
Below is an example of a RocksDB options file:
[Version]
rocksdb_version=4.0.0
options_file_version=1.0
[DBOptions]
max_open_files=12345
max_background_flushes=301
[CFOptions "default"]
[CFOptions "the second column family"]
[CFOptions "the third column family"]
Test Plan: Added many tests in options_test.cc
Reviewers: igor, IslamAbdelRahman, sdong, anthony
Reviewed By: anthony
Subscribers: maykov, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D46059
2015-09-29 23:42:40 +02:00
|
|
|
const int num_cf = static_cast<int>(cf_names.size());
|
|
|
|
Random rnd(302);
|
|
|
|
RandomInitDBOptions(&base_db_opt, &rnd);
|
|
|
|
base_db_opt.db_log_dir += "/#odd #but #could #happen #path #/\\\\#OMG";
|
|
|
|
for (int c = 0; c < num_cf; ++c) {
|
|
|
|
ColumnFamilyOptions cf_opt;
|
|
|
|
Random cf_rnd(0xFB + c);
|
|
|
|
RandomInitCFOptions(&cf_opt, &cf_rnd);
|
2015-10-03 00:35:32 +02:00
|
|
|
if (c < 4) {
|
|
|
|
cf_opt.prefix_extractor.reset(RandomSliceTransform(&rnd, c));
|
|
|
|
}
|
|
|
|
if (c < 3) {
|
|
|
|
cf_opt.table_factory.reset(RandomTableFactory(&rnd, c));
|
|
|
|
}
|
RocksDB Options file format and its serialization / deserialization.
Summary:
This patch defines the format of RocksDB options file, which
follows the INI file format, and implements functions for its
serialization and deserialization. An example RocksDB options
file can be found in examples/rocksdb_option_file_example.ini.
A typical RocksDB options file has three sections, which are
Version, DBOptions, and more than one CFOptions. The RocksDB
options file in general follows the basic INI file format
with the following extensions / modifications:
* Escaped characters
We escaped the following characters:
- \n -- line feed - new line
- \r -- carriage return
- \\ -- backslash \
- \: -- colon symbol :
- \# -- hash tag #
* Comments
We support # style comments. Comments can appear at the ending
part of a line.
* Statements
A statement is of the form option_name = value.
Each statement contains a '=', where extra white-spaces
are supported. However, we don't support multi-lined statement.
Furthermore, each line can only contain at most one statement.
* Section
Sections are of the form [SecitonTitle "SectionArgument"],
where section argument is optional.
* List
We use colon-separated string to represent a list.
For instance, n1:n2:n3:n4 is a list containing four values.
Below is an example of a RocksDB options file:
[Version]
rocksdb_version=4.0.0
options_file_version=1.0
[DBOptions]
max_open_files=12345
max_background_flushes=301
[CFOptions "default"]
[CFOptions "the second column family"]
[CFOptions "the third column family"]
Test Plan: Added many tests in options_test.cc
Reviewers: igor, IslamAbdelRahman, sdong, anthony
Reviewed By: anthony
Subscribers: maykov, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D46059
2015-09-29 23:42:40 +02:00
|
|
|
base_cf_opts.emplace_back(cf_opt);
|
|
|
|
}
|
|
|
|
|
|
|
|
const std::string kOptionsFileName = "test-persisted-options.ini";
|
|
|
|
ASSERT_OK(PersistRocksDBOptions(base_db_opt, cf_names, base_cf_opts,
|
|
|
|
kOptionsFileName, env_.get()));
|
|
|
|
|
|
|
|
RocksDBOptionsParser parser;
|
|
|
|
ASSERT_OK(parser.Parse(kOptionsFileName, env_.get()));
|
|
|
|
|
|
|
|
ASSERT_OK(RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
|
|
|
|
base_db_opt, cf_names, base_cf_opts, kOptionsFileName, env_.get()));
|
|
|
|
|
|
|
|
ASSERT_OK(
|
|
|
|
RocksDBOptionsParser::VerifyDBOptions(*parser.db_opt(), base_db_opt));
|
|
|
|
for (int c = 0; c < num_cf; ++c) {
|
|
|
|
const auto* cf_opt = parser.GetCFOptions(cf_names[c]);
|
|
|
|
ASSERT_NE(cf_opt, nullptr);
|
2015-10-03 00:35:32 +02:00
|
|
|
ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(
|
|
|
|
base_cf_opts[c], *cf_opt, &(parser.cf_opt_maps()->at(c))));
|
RocksDB Options file format and its serialization / deserialization.
Summary:
This patch defines the format of RocksDB options file, which
follows the INI file format, and implements functions for its
serialization and deserialization. An example RocksDB options
file can be found in examples/rocksdb_option_file_example.ini.
A typical RocksDB options file has three sections, which are
Version, DBOptions, and more than one CFOptions. The RocksDB
options file in general follows the basic INI file format
with the following extensions / modifications:
* Escaped characters
We escaped the following characters:
- \n -- line feed - new line
- \r -- carriage return
- \\ -- backslash \
- \: -- colon symbol :
- \# -- hash tag #
* Comments
We support # style comments. Comments can appear at the ending
part of a line.
* Statements
A statement is of the form option_name = value.
Each statement contains a '=', where extra white-spaces
are supported. However, we don't support multi-lined statement.
Furthermore, each line can only contain at most one statement.
* Section
Sections are of the form [SecitonTitle "SectionArgument"],
where section argument is optional.
* List
We use colon-separated string to represent a list.
For instance, n1:n2:n3:n4 is a list containing four values.
Below is an example of a RocksDB options file:
[Version]
rocksdb_version=4.0.0
options_file_version=1.0
[DBOptions]
max_open_files=12345
max_background_flushes=301
[CFOptions "default"]
[CFOptions "the second column family"]
[CFOptions "the third column family"]
Test Plan: Added many tests in options_test.cc
Reviewers: igor, IslamAbdelRahman, sdong, anthony
Reviewed By: anthony
Subscribers: maykov, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D46059
2015-09-29 23:42:40 +02:00
|
|
|
}
|
2015-10-03 00:35:32 +02:00
|
|
|
|
|
|
|
// Further verify pointer-typed options
|
|
|
|
for (int c = 0; c < num_cf; ++c) {
|
|
|
|
const auto* cf_opt = parser.GetCFOptions(cf_names[c]);
|
|
|
|
ASSERT_NE(cf_opt, nullptr);
|
|
|
|
VerifyCFPointerTypedOptions(&base_cf_opts[c], cf_opt,
|
|
|
|
&(parser.cf_opt_maps()->at(c)));
|
|
|
|
}
|
|
|
|
|
RocksDB Options file format and its serialization / deserialization.
Summary:
This patch defines the format of RocksDB options file, which
follows the INI file format, and implements functions for its
serialization and deserialization. An example RocksDB options
file can be found in examples/rocksdb_option_file_example.ini.
A typical RocksDB options file has three sections, which are
Version, DBOptions, and more than one CFOptions. The RocksDB
options file in general follows the basic INI file format
with the following extensions / modifications:
* Escaped characters
We escaped the following characters:
- \n -- line feed - new line
- \r -- carriage return
- \\ -- backslash \
- \: -- colon symbol :
- \# -- hash tag #
* Comments
We support # style comments. Comments can appear at the ending
part of a line.
* Statements
A statement is of the form option_name = value.
Each statement contains a '=', where extra white-spaces
are supported. However, we don't support multi-lined statement.
Furthermore, each line can only contain at most one statement.
* Section
Sections are of the form [SecitonTitle "SectionArgument"],
where section argument is optional.
* List
We use colon-separated string to represent a list.
For instance, n1:n2:n3:n4 is a list containing four values.
Below is an example of a RocksDB options file:
[Version]
rocksdb_version=4.0.0
options_file_version=1.0
[DBOptions]
max_open_files=12345
max_background_flushes=301
[CFOptions "default"]
[CFOptions "the second column family"]
[CFOptions "the third column family"]
Test Plan: Added many tests in options_test.cc
Reviewers: igor, IslamAbdelRahman, sdong, anthony
Reviewed By: anthony
Subscribers: maykov, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D46059
2015-09-29 23:42:40 +02:00
|
|
|
ASSERT_EQ(parser.GetCFOptions("does not exist"), nullptr);
|
|
|
|
|
|
|
|
base_db_opt.max_open_files++;
|
|
|
|
ASSERT_NOK(RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
|
|
|
|
base_db_opt, cf_names, base_cf_opts, kOptionsFileName, env_.get()));
|
2015-10-03 00:35:32 +02:00
|
|
|
|
|
|
|
for (int c = 0; c < num_cf; ++c) {
|
|
|
|
if (base_cf_opts[c].compaction_filter) {
|
|
|
|
delete base_cf_opts[c].compaction_filter;
|
|
|
|
}
|
|
|
|
}
|
RocksDB Options file format and its serialization / deserialization.
Summary:
This patch defines the format of RocksDB options file, which
follows the INI file format, and implements functions for its
serialization and deserialization. An example RocksDB options
file can be found in examples/rocksdb_option_file_example.ini.
A typical RocksDB options file has three sections, which are
Version, DBOptions, and more than one CFOptions. The RocksDB
options file in general follows the basic INI file format
with the following extensions / modifications:
* Escaped characters
We escaped the following characters:
- \n -- line feed - new line
- \r -- carriage return
- \\ -- backslash \
- \: -- colon symbol :
- \# -- hash tag #
* Comments
We support # style comments. Comments can appear at the ending
part of a line.
* Statements
A statement is of the form option_name = value.
Each statement contains a '=', where extra white-spaces
are supported. However, we don't support multi-lined statement.
Furthermore, each line can only contain at most one statement.
* Section
Sections are of the form [SecitonTitle "SectionArgument"],
where section argument is optional.
* List
We use colon-separated string to represent a list.
For instance, n1:n2:n3:n4 is a list containing four values.
Below is an example of a RocksDB options file:
[Version]
rocksdb_version=4.0.0
options_file_version=1.0
[DBOptions]
max_open_files=12345
max_background_flushes=301
[CFOptions "default"]
[CFOptions "the second column family"]
[CFOptions "the third column family"]
Test Plan: Added many tests in options_test.cc
Reviewers: igor, IslamAbdelRahman, sdong, anthony
Reviewed By: anthony
Subscribers: maykov, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D46059
2015-09-29 23:42:40 +02:00
|
|
|
}
|
|
|
|
|
2015-10-11 21:17:42 +02:00
|
|
|
TEST_F(OptionsParserTest, DifferentDefault) {
|
|
|
|
const std::string kOptionsFileName = "test-persisted-options.ini";
|
|
|
|
|
|
|
|
ColumnFamilyOptions cf_level_opts;
|
|
|
|
cf_level_opts.OptimizeLevelStyleCompaction();
|
|
|
|
|
|
|
|
ColumnFamilyOptions cf_univ_opts;
|
|
|
|
cf_univ_opts.OptimizeUniversalStyleCompaction();
|
|
|
|
|
|
|
|
ASSERT_OK(PersistRocksDBOptions(DBOptions(), {"default", "universal"},
|
|
|
|
{cf_level_opts, cf_univ_opts},
|
|
|
|
kOptionsFileName, env_.get()));
|
|
|
|
|
|
|
|
RocksDBOptionsParser parser;
|
|
|
|
ASSERT_OK(parser.Parse(kOptionsFileName, env_.get()));
|
|
|
|
}
|
|
|
|
|
RocksDB Options file format and its serialization / deserialization.
Summary:
This patch defines the format of RocksDB options file, which
follows the INI file format, and implements functions for its
serialization and deserialization. An example RocksDB options
file can be found in examples/rocksdb_option_file_example.ini.
A typical RocksDB options file has three sections, which are
Version, DBOptions, and more than one CFOptions. The RocksDB
options file in general follows the basic INI file format
with the following extensions / modifications:
* Escaped characters
We escaped the following characters:
- \n -- line feed - new line
- \r -- carriage return
- \\ -- backslash \
- \: -- colon symbol :
- \# -- hash tag #
* Comments
We support # style comments. Comments can appear at the ending
part of a line.
* Statements
A statement is of the form option_name = value.
Each statement contains a '=', where extra white-spaces
are supported. However, we don't support multi-lined statement.
Furthermore, each line can only contain at most one statement.
* Section
Sections are of the form [SecitonTitle "SectionArgument"],
where section argument is optional.
* List
We use colon-separated string to represent a list.
For instance, n1:n2:n3:n4 is a list containing four values.
Below is an example of a RocksDB options file:
[Version]
rocksdb_version=4.0.0
options_file_version=1.0
[DBOptions]
max_open_files=12345
max_background_flushes=301
[CFOptions "default"]
[CFOptions "the second column family"]
[CFOptions "the third column family"]
Test Plan: Added many tests in options_test.cc
Reviewers: igor, IslamAbdelRahman, sdong, anthony
Reviewed By: anthony
Subscribers: maykov, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D46059
2015-09-29 23:42:40 +02:00
|
|
|
namespace {
|
|
|
|
bool IsEscapedString(const std::string& str) {
|
|
|
|
for (size_t i = 0; i < str.size(); ++i) {
|
|
|
|
if (str[i] == '\\') {
|
|
|
|
// since we already handle those two consecutive '\'s in
|
|
|
|
// the next if-then branch, any '\' appear at the end
|
|
|
|
// of an escaped string in such case is not valid.
|
|
|
|
if (i == str.size() - 1) {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
if (str[i + 1] == '\\') {
|
|
|
|
// if there're two consecutive '\'s, skip the second one.
|
|
|
|
i++;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
switch (str[i + 1]) {
|
|
|
|
case ':':
|
|
|
|
case '\\':
|
|
|
|
case '#':
|
|
|
|
continue;
|
|
|
|
default:
|
|
|
|
// if true, '\' together with str[i + 1] is not a valid escape.
|
|
|
|
if (UnescapeChar(str[i + 1]) == str[i + 1]) {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
} else if (isSpecialChar(str[i]) && (i == 0 || str[i - 1] != '\\')) {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
} // namespace
|
|
|
|
|
|
|
|
TEST_F(OptionsParserTest, EscapeOptionString) {
|
|
|
|
ASSERT_EQ(UnescapeOptionString(
|
|
|
|
"This is a test string with \\# \\: and \\\\ escape chars."),
|
|
|
|
"This is a test string with # : and \\ escape chars.");
|
|
|
|
|
|
|
|
ASSERT_EQ(
|
|
|
|
EscapeOptionString("This is a test string with # : and \\ escape chars."),
|
|
|
|
"This is a test string with \\# \\: and \\\\ escape chars.");
|
|
|
|
|
|
|
|
std::string readible_chars =
|
|
|
|
"A String like this \"1234567890-=_)(*&^%$#@!ertyuiop[]{POIU"
|
|
|
|
"YTREWQasdfghjkl;':LKJHGFDSAzxcvbnm,.?>"
|
|
|
|
"<MNBVCXZ\\\" should be okay to \\#\\\\\\:\\#\\#\\#\\ "
|
|
|
|
"be serialized and deserialized";
|
|
|
|
|
|
|
|
std::string escaped_string = EscapeOptionString(readible_chars);
|
|
|
|
ASSERT_TRUE(IsEscapedString(escaped_string));
|
|
|
|
// This two transformations should be canceled and should output
|
|
|
|
// the original input.
|
|
|
|
ASSERT_EQ(UnescapeOptionString(escaped_string), readible_chars);
|
|
|
|
|
|
|
|
std::string all_chars;
|
|
|
|
for (unsigned char c = 0;; ++c) {
|
|
|
|
all_chars += c;
|
|
|
|
if (c == 255) {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
escaped_string = EscapeOptionString(all_chars);
|
|
|
|
ASSERT_TRUE(IsEscapedString(escaped_string));
|
|
|
|
ASSERT_EQ(UnescapeOptionString(escaped_string), all_chars);
|
|
|
|
|
|
|
|
ASSERT_EQ(RocksDBOptionsParser::TrimAndRemoveComment(
|
|
|
|
" A simple statement with a comment. # like this :)"),
|
|
|
|
"A simple statement with a comment.");
|
|
|
|
|
|
|
|
ASSERT_EQ(RocksDBOptionsParser::TrimAndRemoveComment(
|
|
|
|
"Escape \\# and # comment together ."),
|
|
|
|
"Escape \\# and");
|
|
|
|
}
|
|
|
|
|
|
|
|
#endif // !ROCKSDB_LITE
|
|
|
|
|
2014-05-16 19:35:41 +02:00
|
|
|
} // namespace rocksdb
|
|
|
|
|
|
|
|
int main(int argc, char** argv) {
|
2015-03-17 22:08:00 +01:00
|
|
|
::testing::InitGoogleTest(&argc, argv);
|
2014-11-24 21:53:23 +01:00
|
|
|
#ifdef GFLAGS
|
2014-05-16 19:35:41 +02:00
|
|
|
ParseCommandLineFlags(&argc, &argv, true);
|
2014-11-24 21:53:23 +01:00
|
|
|
#endif // GFLAGS
|
2015-03-17 22:08:00 +01:00
|
|
|
return RUN_ALL_TESTS();
|
2014-05-16 19:35:41 +02:00
|
|
|
}
|