andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Enable RocksDB to persist Options file.

Browse Source 
Summary:
This patch allows rocksdb to persist options into a file on
DB::Open, SetOptions, and Create / Drop ColumnFamily.
Options files are created under the same directory as the rocksdb
instance.

In addition, this patch also adds a fail_if_missing_options_file in DBOptions
that makes any function call return non-ok status when it is not able to
persist options properly.

  // If true, then DB::Open / CreateColumnFamily / DropColumnFamily
  // / SetOptions will fail if options file is not detected or properly
  // persisted.
  //
  // DEFAULT: false
  bool fail_if_missing_options_file;

Options file names are formatted as OPTIONS-<number>, and RocksDB
will always keep the latest two options files.

Test Plan:
Add options_file_test.

options_test
column_family_test

Reviewers: igor, IslamAbdelRahman, sdong, anthony

Reviewed By: anthony

Subscribers: dhruba

Differential Revision: https://reviews.facebook.net/D48285
This commit is contained in:
Yueh-Hsuan Chiang 2015-11-10 22:58:01 -08:00
parent 7ed2c3e45b
commit e114f0abb8
18 changed files with 455 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CMakeLists.txt
View File

@ -329,6 +329,7 @@ set(TESTS
db/memtable_list_test.cc
db/merge_test.cc
db/merge_helper_test.cc
db/options_file_test.cc
db/perf_context_test.cc
db/plain_table_db_test.cc
db/prefix_test.cc

1
HISTORY.md
View File

@ -5,6 +5,7 @@
* Introduce CreateLoggerFromOptions(), this function create a Logger for provided DBOptions.
* Add GetAggregatedIntProperty(), which returns the sum of the GetIntProperty of all the column families.
* Add MemoryUtil in rocksdb/utilities/memory.h. It currently offers a way to get the memory usage by type from a list rocksdb instances.
* RocksDB will now persist options under the same directory as the RocksDB database on successful DB::Open, CreateColumnFamily, DropColumnFamily, and SetOptions.
### Public API Changes
* CompactionFilter::Context includes information of Column Family ID
* The need-compaction hint given by TablePropertiesCollector::NeedCompact() will be persistent and recoverable after DB recovery. This introduces a breaking format change. If you use this experimental feature, including NewCompactOnDeletionCollectorFactory() in the new version, you may not be able to directly downgrade the DB back to version 4.0 or lower.

4
Makefile
View File

@ -275,6 +275,7 @@ TESTS = \
memory_test \
merge_test \
merger_test \
options_file_test \
redis_test \
reduce_levels_test \
plain_table_db_test \
@ -892,6 +893,9 @@ merge_test: db/merge_test.o $(LIBOBJECTS) $(TESTHARNESS)
merger_test: table/merger_test.o $(LIBOBJECTS) $(TESTHARNESS)
$(AM_LINK)
options_file_test: db/options_file_test.o $(LIBOBJECTS) $(TESTHARNESS)
$(AM_LINK)
deletefile_test: db/deletefile_test.o $(LIBOBJECTS) $(TESTHARNESS)
$(AM_LINK)

1
db/column_family_test.cc
View File

@ -57,6 +57,7 @@ class ColumnFamilyTest : public testing::Test {
env_ = new EnvCounter(Env::Default());
dbname_ = test::TmpDir() + "/column_family_test";
db_options_.create_if_missing = true;
db_options_.fail_if_options_file_error = true;
db_options_.env = env_;
DestroyDB(dbname_, Options(db_options_, column_family_options_));
}

170
db/db_impl.cc
View File

@ -19,6 +19,7 @@
#include <algorithm>
#include <climits>
#include <cstdio>
#include <map>
#include <set>
#include <stdexcept>
#include <string>
@ -84,6 +85,8 @@
#include "util/log_buffer.h"
#include "util/logging.h"
#include "util/mutexlock.h"
#include "util/options_helper.h"
#include "util/options_parser.h"
#include "util/perf_context_imp.h"
#include "util/stop_watch.h"
#include "util/string_util.h"
@ -734,8 +737,12 @@ void DBImpl::PurgeObsoleteFiles(const JobContext& state) {
// Also, SetCurrentFile creates a temp file when writing out new
// manifest, which is equal to state.pending_manifest_file_number. We
// should not delete that file
//
// TODO(yhchiang): carefully modify the third condition to safely
// remove the temp options files.
keep = (sst_live_map.find(number) != sst_live_map.end()) ||
(number == state.pending_manifest_file_number);
(number == state.pending_manifest_file_number) ||
(to_delete.find(kOptionsFileNamePrefix) != std::string::npos);
break;
case kInfoLogFile:
keep = true;
@ -747,6 +754,7 @@ void DBImpl::PurgeObsoleteFiles(const JobContext& state) {
case kDBLockFile:
case kIdentityFile:
case kMetaDatabase:
case kOptionsFile:
keep = true;
break;
}
@ -1922,6 +1930,19 @@ Status DBImpl::SetOptions(ColumnFamilyHandle* column_family,
new_options = *cfd->GetLatestMutableCFOptions();
}
}
if (s.ok()) {
Status persist_options_status = WriteOptionsFile();
if (!persist_options_status.ok()) {
if (db_options_.fail_if_options_file_error) {
s = Status::IOError(
"SetOptions succeeded, but unable to persist options",
persist_options_status.ToString());
}
Warn(db_options_.info_log,
"Unable to persist options in SetOptions() -- %s",
persist_options_status.ToString().c_str());
}
}
Log(InfoLogLevel::INFO_LEVEL, db_options_.info_log,
"SetOptions() on column family [%s], inputs:",
@ -3458,6 +3479,18 @@ Status DBImpl::CreateColumnFamily(const ColumnFamilyOptions& cf_options,
// this is outside the mutex
if (s.ok()) {
Status persist_options_status = WriteOptionsFile();
if (!persist_options_status.ok()) {
if (db_options_.fail_if_options_file_error) {
s = Status::IOError(
"ColumnFamily has been created, but unable to persist"
"options in CreateColumnFamily()",
persist_options_status.ToString().c_str());
}
Warn(db_options_.info_log,
"Unable to persist options in CreateColumnFamily() -- %s",
persist_options_status.ToString().c_str());
}
NewThreadStatusCfInfo(
reinterpret_cast<ColumnFamilyHandleImpl*>(*handle)->cfd());
}
@ -3515,6 +3548,18 @@ Status DBImpl::DropColumnFamily(ColumnFamilyHandle* column_family) {
auto* mutable_cf_options = cfd->GetLatestMutableCFOptions();
max_total_in_memory_state_ -= mutable_cf_options->write_buffer_size *
mutable_cf_options->max_write_buffer_number;
auto options_persist_status = WriteOptionsFile();
if (!options_persist_status.ok()) {
if (db_options_.fail_if_options_file_error) {
s = Status::IOError(
"ColumnFamily has been dropped, but unable to persist "
"options in DropColumnFamily()",
options_persist_status.ToString().c_str());
}
Warn(db_options_.info_log,
"Unable to persist options in DropColumnFamily() -- %s",
options_persist_status.ToString().c_str());
}
Log(InfoLogLevel::INFO_LEVEL, db_options_.info_log,
"Dropped column family with id %u\n",
cfd->GetID());
@ -4931,6 +4976,19 @@ Status DB::Open(const DBOptions& db_options, const std::string& dbname,
impl);
LogFlush(impl->db_options_.info_log);
auto persist_options_status = impl->WriteOptionsFile();
if (!persist_options_status.ok()) {
if (db_options.fail_if_options_file_error) {
s = Status::IOError(
"DB::Open() failed --- Unable to persist Options file",
persist_options_status.ToString());
}
Warn(impl->db_options_.info_log,
"Unable to persist options in DB::Open() -- %s",
persist_options_status.ToString().c_str());
}
}
if (s.ok()) {
*dbptr = impl;
} else {
for (auto* h : *handles) {
@ -4938,6 +4996,7 @@ Status DB::Open(const DBOptions& db_options, const std::string& dbname,
}
handles->clear();
delete impl;
*dbptr = nullptr;
}
return s;
}
@ -5034,6 +5093,7 @@ Status DestroyDB(const std::string& dbname, const Options& options) {
}
}
}
// ignore case where no archival directory is present.
env->DeleteDir(archivedir);
@ -5045,6 +5105,114 @@ Status DestroyDB(const std::string& dbname, const Options& options) {
return result;
}
Status DBImpl::WriteOptionsFile() {
#ifndef ROCKSDB_LITE
std::string file_name;
Status s = WriteOptionsToTempFile(&file_name);
if (!s.ok()) {
return s;
}
s = RenameTempFileToOptionsFile(file_name);
return s;
#else
return Status::OK();
#endif // !ROCKSDB_LITE
}
Status DBImpl::WriteOptionsToTempFile(std::string* file_name) {
#ifndef ROCKSDB_LITE
std::vector<std::string> cf_names;
std::vector<ColumnFamilyOptions> cf_opts;
{
InstrumentedMutexLock l(&mutex_);
// This part requires mutex to protect the column family options
for (auto cfd : *versions_->GetColumnFamilySet()) {
if (cfd->IsDropped()) {
continue;
}
cf_names.push_back(cfd->GetName());
cf_opts.push_back(BuildColumnFamilyOptions(
*cfd->options(), *cfd->GetLatestMutableCFOptions()));
}
}
*file_name = TempOptionsFileName(GetName(), versions_->NewFileNumber());
Status s = PersistRocksDBOptions(GetDBOptions(), cf_names, cf_opts,
*file_name, GetEnv());
return s;
#else
return Status::OK();
#endif // !ROCKSDB_LITE
}
#ifndef ROCKSDB_LITE
namespace {
void DeleteOptionsFilesHelper(const std::map<uint64_t, std::string>& filenames,
const size_t num_files_to_keep,
const std::shared_ptr<Logger>& info_log,
Env* env) {
if (filenames.size() <= num_files_to_keep) {
return;
}
for (auto iter = std::next(filenames.begin(), num_files_to_keep);
iter != filenames.end(); ++iter) {
if (!env->DeleteFile(iter->second).ok()) {
Warn(info_log, "Unable to delete options file %s", iter->second.c_str());
}
}
}
} // namespace
#endif // !ROCKSDB_LITE
Status DBImpl::DeleteObsoleteOptionsFiles() {
#ifndef ROCKSDB_LITE
options_files_mutex_.AssertHeld();
std::vector<std::string> filenames;
// use ordered map to store keep the filenames sorted from the newest
// to the oldest.
std::map<uint64_t, std::string> options_filenames;
Status s;
s = GetEnv()->GetChildren(GetName(), &filenames);
if (!s.ok()) {
return s;
}
for (auto& filename : filenames) {
uint64_t file_number;
FileType type;
if (ParseFileName(filename, &file_number, &type) && type == kOptionsFile) {
options_filenames.insert(
{std::numeric_limits<uint64_t>::max() - file_number,
GetName() + "/" + filename});
}
}
// Keeps the latest 2 Options file
const size_t kNumOptionsFilesKept = 2;
DeleteOptionsFilesHelper(options_filenames, kNumOptionsFilesKept,
db_options_.info_log, GetEnv());
return Status::OK();
#else
return Status::OK();
#endif // !ROCKSDB_LITE
}
Status DBImpl::RenameTempFileToOptionsFile(const std::string& file_name) {
#ifndef ROCKSDB_LITE
InstrumentedMutexLock l(&options_files_mutex_);
Status s;
std::string options_file_name =
OptionsFileName(GetName(), versions_->NewFileNumber());
// Retry if the file name happen to conflict with an existing one.
s = GetEnv()->RenameFile(file_name, options_file_name);
DeleteObsoleteOptionsFiles();
return s;
#else
return Status::OK();
#endif // !ROCKSDB_LITE
}
#if ROCKSDB_USING_THREAD_STATUS
void DBImpl::NewThreadStatusCfInfo(

12
db/db_impl.h
View File

@ -398,6 +398,13 @@ class DBImpl : public DB {
SuperVersion* super_version,
Arena* arena);
// The following options file related functions should not be
// called while DB mutex is held.
Status WriteOptionsFile();
Status WriteOptionsToTempFile(std::string* file_name);
Status RenameTempFileToOptionsFile(const std::string& file_name);
Status DeleteObsoleteOptionsFiles();
void NotifyOnFlushCompleted(ColumnFamilyData* cfd, FileMetaData* file_meta,
const MutableCFOptions& mutable_cf_options,
int job_id, TableProperties prop);
@ -552,8 +559,13 @@ class DBImpl : public DB {
// Lock over the persistent DB state. Non-nullptr iff successfully acquired.
FileLock* db_lock_;
// The mutex for options file related operations.
// NOTE: should never acquire options_file_mutex_ and mutex_ at the
// same time.
InstrumentedMutex options_files_mutex_;
// State below is protected by mutex_
InstrumentedMutex mutex_;
std::atomic<bool> shutting_down_;
// This condition variable is signaled on these conditions:
// * whenever bg_compaction_scheduled_ goes down to 0

1
db/db_test_util.cc
View File

@ -368,6 +368,7 @@ Options DBTestBase::CurrentOptions(
}
options.env = env_;
options.create_if_missing = true;
options.fail_if_options_file_error = true;
return options;
}

40
db/filename.cc
View File

@ -21,6 +21,7 @@
#include "util/file_reader_writer.h"
#include "util/logging.h"
#include "util/stop_watch.h"
#include "util/string_util.h"
#include "util/sync_point.h"
namespace rocksdb {
@ -47,8 +48,9 @@ static size_t GetInfoLogPrefix(const std::string& path, char* dest, int len) {
path[i] == '_'){
dest[write_idx++] = path[i];
} else {
if (i > 0)
if (i > 0) {
dest[write_idx++] = '_';
}
}
i++;
}
@ -146,7 +148,7 @@ std::string LockFileName(const std::string& dbname) {
}
std::string TempFileName(const std::string& dbname, uint64_t number) {
return MakeFileName(dbname, number, "dbtmp");
return MakeFileName(dbname, number, kTempFileNameSuffix.c_str());
}
InfoLogPrefix::InfoLogPrefix(bool has_log_dir,
@ -186,6 +188,21 @@ std::string OldInfoLogFileName(const std::string& dbname, uint64_t ts,
return log_dir + "/" + info_log_prefix.buf + ".old." + buf;
}
std::string OptionsFileName(const std::string& dbname, uint64_t file_num) {
char buffer[256];
snprintf(buffer, sizeof(buffer), "%s%06" PRIu64,
kOptionsFileNamePrefix.c_str(), file_num);
return dbname + "/" + buffer;
}
std::string TempOptionsFileName(const std::string& dbname, uint64_t file_num) {
char buffer[256];
snprintf(buffer, sizeof(buffer), "%s%06" PRIu64 ".%s",
kOptionsFileNamePrefix.c_str(), file_num,
kTempFileNameSuffix.c_str());
return dbname + "/" + buffer;
}
std::string MetaDatabaseName(const std::string& dbname, uint64_t number) {
char buf[100];
snprintf(buf, sizeof(buf), "/METADB-%llu",
@ -206,6 +223,8 @@ std::string IdentityFileName(const std::string& dbname) {
// dbname/MANIFEST-[0-9]+
// dbname/[0-9]+.(log|sst)
// dbname/METADB-[0-9]+
// dbname/OPTIONS-[0-9]+
// dbname/OPTIONS-[0-9]+.dbtmp
// Disregards / at the beginning
bool ParseFileName(const std::string& fname,
uint64_t* number,
@ -268,6 +287,21 @@ bool ParseFileName(const std::string& fname, uint64_t* number,
}
*type = kMetaDatabase;
*number = num;
} else if (rest.starts_with(kOptionsFileNamePrefix)) {
uint64_t ts_suffix;
bool is_temp_file = false;
rest.remove_prefix(kOptionsFileNamePrefix.size());
const std::string kTempFileNameSuffixWithDot =
std::string(".") + kTempFileNameSuffix;
if (rest.ends_with(kTempFileNameSuffixWithDot)) {
rest.remove_suffix(kTempFileNameSuffixWithDot.size());
is_temp_file = true;
}
if (!ConsumeDecimalNumber(&rest, &ts_suffix)) {
return false;
}
*number = ts_suffix;
*type = is_temp_file ? kTempFile : kOptionsFile;
} else {
// Avoid strtoull() to keep filename format independent of the
// current locale
@ -302,7 +336,7 @@ bool ParseFileName(const std::string& fname, uint64_t* number,
} else if (suffix == Slice(kRocksDbTFileExt) ||
suffix == Slice(kLevelDbTFileExt)) {
*type = kTableFile;
} else if (suffix == Slice("dbtmp")) {
} else if (suffix == Slice(kTempFileNameSuffix)) {
*type = kTempFile;
} else {
return false;

16
db/filename.h
View File

@ -36,7 +36,8 @@ enum FileType {
kTempFile,
kInfoLogFile, // Either the current one, or an old one
kMetaDatabase,
kIdentityFile
kIdentityFile,
kOptionsFile
};
// Return the name of the log file with the specified number
@ -114,6 +115,19 @@ extern std::string OldInfoLogFileName(const std::string& dbname, uint64_t ts,
const std::string& db_path = "",
const std::string& log_dir = "");
static const std::string kOptionsFileNamePrefix = "OPTIONS-";
static const std::string kTempFileNameSuffix = "dbtmp";
// Return a options file name given the "dbname" and file number.
// Format: OPTIONS-[number].dbtmp
extern std::string OptionsFileName(const std::string& dbname,
uint64_t file_num);
// Return a temp options file name given the "dbname" and file number.
// Format: OPTIONS-[number]
extern std::string TempOptionsFileName(const std::string& dbname,
uint64_t file_num);
// Return the name to use for a metadatabase. The result will be prefixed with
// "dbname".
extern std::string MetaDatabaseName(const std::string& dbname,

116
db/options_file_test.cc Normal file
View File

@ -0,0 +1,116 @@
// Copyright (c) 2015, Facebook, Inc. All rights reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
#ifndef ROCKSDB_LITE
#include <string>
#include "db/db_impl.h"
#include "db/db_test_util.h"
#include "rocksdb/options.h"
#include "rocksdb/table.h"
#include "util/testharness.h"
namespace rocksdb {
class OptionsFileTest : public testing::Test {
public:
OptionsFileTest() : dbname_(test::TmpDir() + "/options_file_test") {}
std::string dbname_;
};
namespace {
void UpdateOptionsFiles(DB* db,
std::unordered_set<std::string>* filename_history,
int* options_files_count) {
std::vector<std::string> filenames;
db->GetEnv()->GetChildren(db->GetName(), &filenames);
uint64_t number;
FileType type;
*options_files_count = 0;
for (auto filename : filenames) {
if (ParseFileName(filename, &number, &type) && type == kOptionsFile) {
filename_history->insert(filename);
(*options_files_count)++;
}
}
}
// Verify whether the current Options Files are the latest ones.
void VerifyOptionsFileName(
DB* db, const std::unordered_set<std::string>& past_filenames) {
std::vector<std::string> filenames;
std::unordered_set<std::string> current_filenames;
db->GetEnv()->GetChildren(db->GetName(), &filenames);
uint64_t number;
FileType type;
for (auto filename : filenames) {
if (ParseFileName(filename, &number, &type) && type == kOptionsFile) {
current_filenames.insert(filename);
}
}
for (auto past_filename : past_filenames) {
if (current_filenames.find(past_filename) != current_filenames.end()) {
continue;
}
for (auto filename : current_filenames) {
ASSERT_GT(filename, past_filename);
}
}
}
} // namespace
TEST_F(OptionsFileTest, NumberOfOptionsFiles) {
const int kReopenCount = 20;
Options opt;
opt.create_if_missing = true;
DestroyDB(dbname_, opt);
std::unordered_set<std::string> filename_history;
DB* db;
for (int i = 0; i < kReopenCount; ++i) {
ASSERT_OK(DB::Open(opt, dbname_, &db));
int num_options_files = 0;
UpdateOptionsFiles(db, &filename_history, &num_options_files);
ASSERT_GT(num_options_files, 0);
ASSERT_LE(num_options_files, 2);
// Make sure we always keep the latest option files.
VerifyOptionsFileName(db, filename_history);
delete db;
}
}
TEST_F(OptionsFileTest, OptionsFileName) {
const uint64_t kOptionsFileNum = 12345;
uint64_t number;
FileType type;
auto options_file_name = OptionsFileName("", kOptionsFileNum);
ASSERT_TRUE(ParseFileName(options_file_name, &number, &type, nullptr));
ASSERT_EQ(type, kOptionsFile);
ASSERT_EQ(number, kOptionsFileNum);
const uint64_t kTempOptionsFileNum = 54352;
auto temp_options_file_name = TempOptionsFileName("", kTempOptionsFileNum);
ASSERT_TRUE(ParseFileName(temp_options_file_name, &number, &type, nullptr));
ASSERT_NE(temp_options_file_name.find(kTempFileNameSuffix),
std::string::npos);
ASSERT_EQ(type, kTempFile);
ASSERT_EQ(number, kTempOptionsFileNum);
}
} // namespace rocksdb
int main(int argc, char** argv) {
#if !(defined NDEBUG) || !defined(OS_WIN)
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
#else
return 0;
#endif // !(defined NDEBUG) || !defined(OS_WIN)
}
#else
int main(int argc, char** argv) {
printf("Skipped as Options file is not supported in RocksDBLite.\n");
return 0;
}
#endif // !ROCKSDB_LITE

7
include/rocksdb/options.h
View File

@ -1172,6 +1172,13 @@ struct DBOptions {
// currently.
const WalFilter* wal_filter;
#endif // ROCKSDB_LITE
// If true, then DB::Open / CreateColumnFamily / DropColumnFamily
// / SetOptions will fail if options file is not detected or properly
// persisted.
//
// DEFAULT: false
bool fail_if_options_file_error;
};
// Options to control the behavior of a database (passed to DB::Open)

10
include/rocksdb/slice.h
View File

@ -73,6 +73,11 @@ class Slice {
size_ -= n;
}
void remove_suffix(size_t n) {
assert(n <= size());
size_ -= n;
}
// Return a string that contains the copy of the referenced data.
std::string ToString(bool hex = false) const;
@ -88,6 +93,11 @@ class Slice {
(memcmp(data_, x.data_, x.size_) == 0));
}
bool ends_with(const Slice& x) const {
return ((size_ >= x.size_) &&
(memcmp(data_ + size_ - x.size_, x.data_, x.size_) == 0));
}
// Compare two slices and returns the first byte where they differ
size_t difference_offset(const Slice& b) const;

2
include/rocksdb/status.h
View File

@ -48,7 +48,7 @@ class Status {
kAborted = 10,
kBusy = 11,
kExpired = 12,
kTryAgain = 13
kTryAgain = 13,
};
Code code() const { return code_; }

1
src.mk
View File

@ -206,6 +206,7 @@ TEST_BENCH_SOURCES = \
db/manual_compaction_test.cc \
db/memtablerep_bench.cc \
db/merge_test.cc \
db/options_file_test.cc \
db/perf_context_test.cc \
db/plain_table_db_test.cc \
db/prefix_test.cc \

14
util/options.cc
View File

@ -259,12 +259,11 @@ DBOptions::DBOptions()
enable_thread_tracking(false),
delayed_write_rate(1024U * 1024U),
skip_stats_update_on_db_open(false),
wal_recovery_mode(WALRecoveryMode::kTolerateCorruptedTailRecords)
wal_recovery_mode(WALRecoveryMode::kTolerateCorruptedTailRecords),
#ifndef ROCKSDB_LITE
,
wal_filter(nullptr)
wal_filter(nullptr),
#endif // ROCKSDB_LITE
{
fail_if_options_file_error(false) {
}
DBOptions::DBOptions(const Options& options)
@ -323,12 +322,11 @@ DBOptions::DBOptions(const Options& options)
delayed_write_rate(options.delayed_write_rate),
skip_stats_update_on_db_open(options.skip_stats_update_on_db_open),
wal_recovery_mode(options.wal_recovery_mode),
row_cache(options.row_cache)
row_cache(options.row_cache),
#ifndef ROCKSDB_LITE
,
wal_filter(options.wal_filter)
wal_filter(options.wal_filter),
#endif // ROCKSDB_LITE
{
fail_if_options_file_error(options.fail_if_options_file_error) {
}
static const char* const access_hints[] = {

66
util/options_helper.cc
View File

@ -1315,5 +1315,71 @@ Status GetTableFactoryFromMap(
return Status::OK();
}
ColumnFamilyOptions BuildColumnFamilyOptions(
const Options& options, const MutableCFOptions& mutable_cf_options) {
ColumnFamilyOptions cf_opts(options);
// Memtable related options
cf_opts.write_buffer_size = mutable_cf_options.write_buffer_size;
cf_opts.max_write_buffer_number = mutable_cf_options.max_write_buffer_number;
cf_opts.arena_block_size = mutable_cf_options.arena_block_size;
cf_opts.memtable_prefix_bloom_bits =
mutable_cf_options.memtable_prefix_bloom_bits;
cf_opts.memtable_prefix_bloom_probes =
mutable_cf_options.memtable_prefix_bloom_probes;
cf_opts.memtable_prefix_bloom_huge_page_tlb_size =
mutable_cf_options.memtable_prefix_bloom_huge_page_tlb_size;
cf_opts.max_successive_merges = mutable_cf_options.max_successive_merges;
cf_opts.filter_deletes = mutable_cf_options.filter_deletes;
cf_opts.inplace_update_num_locks =
mutable_cf_options.inplace_update_num_locks;
// Compaction related options
cf_opts.disable_auto_compactions =
mutable_cf_options.disable_auto_compactions;
cf_opts.soft_rate_limit = mutable_cf_options.soft_rate_limit;
cf_opts.level0_file_num_compaction_trigger =
mutable_cf_options.level0_file_num_compaction_trigger;
cf_opts.level0_slowdown_writes_trigger =
mutable_cf_options.level0_slowdown_writes_trigger;
cf_opts.level0_stop_writes_trigger =
mutable_cf_options.level0_stop_writes_trigger;
cf_opts.max_grandparent_overlap_factor =
mutable_cf_options.max_grandparent_overlap_factor;
cf_opts.expanded_compaction_factor =
mutable_cf_options.expanded_compaction_factor;
cf_opts.source_compaction_factor =
mutable_cf_options.source_compaction_factor;
cf_opts.target_file_size_base = mutable_cf_options.target_file_size_base;
cf_opts.target_file_size_multiplier =
mutable_cf_options.target_file_size_multiplier;
cf_opts.max_bytes_for_level_base =
mutable_cf_options.max_bytes_for_level_base;
cf_opts.max_bytes_for_level_multiplier =
mutable_cf_options.max_bytes_for_level_multiplier;
cf_opts.max_bytes_for_level_multiplier_additional.clear();
for (auto value :
mutable_cf_options.max_bytes_for_level_multiplier_additional) {
cf_opts.max_bytes_for_level_multiplier_additional.emplace_back(value);
}
cf_opts.verify_checksums_in_compaction =
mutable_cf_options.verify_checksums_in_compaction;
// Misc options
cf_opts.max_sequential_skip_in_iterations =
mutable_cf_options.max_sequential_skip_in_iterations;
cf_opts.paranoid_file_checks = mutable_cf_options.paranoid_file_checks;
cf_opts.compaction_measure_io_stats =
mutable_cf_options.compaction_measure_io_stats;
cf_opts.table_factory = options.table_factory;
// TODO(yhchiang): find some way to handle the following derived options
// * max_file_size
return cf_opts;
}
#endif // !ROCKSDB_LITE
} // namespace rocksdb

3
util/options_helper.h
View File

@ -64,6 +64,9 @@ Status GetTableFactoryFromMap(
Status GetStringFromTableFactory(std::string* opts_str, const TableFactory* tf,
const std::string& delimiter = "; ");
ColumnFamilyOptions BuildColumnFamilyOptions(
const Options& options, const MutableCFOptions& mutable_cf_options);
enum class OptionType {
kBoolean,
kInt,

8
utilities/backupable/backupable_db_test.cc
View File

@ -1198,18 +1198,18 @@ TEST_F(BackupableDBTest, GarbageCollectionBeforeBackup) {
OpenDBAndBackupEngine(true);
env_->CreateDirIfMissing(backupdir_ + "/shared");
std::string file_five = backupdir_ + "/shared/000005.sst";
std::string file_five = backupdir_ + "/shared/000007.sst";
std::string file_five_contents = "I'm not really a sst file";
// this depends on the fact that 00005.sst is the first file created by the DB
// this depends on the fact that 00007.sst is the first file created by the DB
ASSERT_OK(file_manager_->WriteToFile(file_five, file_five_contents));
FillDB(db_.get(), 0, 100);
// backup overwrites file 000005.sst
// backup overwrites file 000007.sst
ASSERT_TRUE(backup_engine_->CreateNewBackup(db_.get(), true).ok());
std::string new_file_five_contents;
ASSERT_OK(ReadFileToString(env_, file_five, &new_file_five_contents));
// file 000005.sst was overwritten
// file 000007.sst was overwritten
ASSERT_TRUE(new_file_five_contents != file_five_contents);
CloseDBAndBackupEngine();