andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Backup Options

Browse Source 
Summary: Backup options file to private directory

Test Plan:
backupable_db_test.cc, BackupOptions
	   Modify DB options by calling OpenDB for 3 times. Check the latest options file is in the right place. Also check no redundent files are backuped.

Reviewers: andrewkr

Reviewed By: andrewkr

Subscribers: leveldb, dhruba, andrewkr

Differential Revision: https://reviews.facebook.net/D59373
This commit is contained in:
Wanning Jiang 2016-06-09 19:03:10 -07:00
parent a683d4aba9
commit 56887f6cb8
8 changed files with 72 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
db/db_filesnapshot.cc
View File

@ -14,19 +14,20 @@
#endif
#include <inttypes.h>
#include <stdint.h>
#include <algorithm>
#include <string>
#include <stdint.h>
#include "db/db_impl.h"
#include "db/filename.h"
#include "db/job_context.h"
#include "db/version_set.h"
#include "port/port.h"
#include "rocksdb/db.h"
#include "rocksdb/env.h"
#include "port/port.h"
#include "util/file_util.h"
#include "util/mutexlock.h"
#include "util/sync_point.h"
#include "util/file_util.h"
#include "util/testharness.h"
namespace rocksdb {
@ -83,7 +84,6 @@ int DBImpl::IsFileDeletionsEnabled() const {
Status DBImpl::GetLiveFiles(std::vector<std::string>& ret,
uint64_t* manifest_file_size,
bool flush_memtable) {
*manifest_file_size = 0;
mutex_.Lock();
@ -126,7 +126,7 @@ Status DBImpl::GetLiveFiles(std::vector<std::string>& ret,
}
ret.clear();
ret.reserve(live.size() + 2); //*.sst + CURRENT + MANIFEST
ret.reserve(live.size() + 3); // *.sst + CURRENT + MANIFEST + OPTIONS
// create names of the live files. The names are not absolute
// paths, instead they are relative to dbname_;
@ -136,6 +136,7 @@ Status DBImpl::GetLiveFiles(std::vector<std::string>& ret,
ret.push_back(CurrentFileName(""));
ret.push_back(DescriptorFileName("", versions_->manifest_file_number()));
ret.push_back(OptionsFileName("", versions_->options_file_number()));
// find length of manifest file while holding the mutex lock
*manifest_file_size = versions_->manifest_file_size();

4
db/db_impl.cc
View File

@ -6005,8 +6005,10 @@ Status DBImpl::DeleteObsoleteOptionsFiles() {
Status DBImpl::RenameTempFileToOptionsFile(const std::string& file_name) {
#ifndef ROCKSDB_LITE
Status s;
versions_->options_file_number_ = versions_->NewFileNumber();
std::string options_file_name =
OptionsFileName(GetName(), versions_->NewFileNumber());
OptionsFileName(GetName(), versions_->options_file_number_);
// Retry if the file name happen to conflict with an existing one.
s = GetEnv()->RenameFile(file_name, options_file_name);

2
db/db_impl.h
View File

@ -521,7 +521,7 @@ class DBImpl : public DB {
Compaction *c, const Status &st,
const CompactionJobStats& job_stats,
int job_id);
void NotifyOnMemTableSealed(ColumnFamilyData* cfd,
void NotifyOnMemTableSealed(ColumnFamilyData* cfd,
const MemTableInfo& mem_table_info);
void NewThreadStatusCfInfo(ColumnFamilyData* cfd) const;

4
db/db_test.cc
View File

@ -2067,8 +2067,8 @@ TEST_F(DBTest, SnapshotFiles) {
dbfull()->DisableFileDeletions();
dbfull()->GetLiveFiles(files, &manifest_size);
// CURRENT, MANIFEST, *.sst files (one for each CF)
ASSERT_EQ(files.size(), 4U);
// CURRENT, MANIFEST, OPTIONS, *.sst files (one for each CF)
ASSERT_EQ(files.size(), 5U);
uint64_t number = 0;
FileType type;

3
db/version_set.h
View File

@ -627,6 +627,8 @@ class VersionSet {
// Return the current manifest file number
uint64_t manifest_file_number() const { return manifest_file_number_; }
uint64_t options_file_number() const { return options_file_number_; }
uint64_t pending_manifest_file_number() const {
return pending_manifest_file_number_;
}
@ -743,6 +745,7 @@ class VersionSet {
const DBOptions* const db_options_;
std::atomic<uint64_t> next_file_number_;
uint64_t manifest_file_number_;
uint64_t options_file_number_;
uint64_t pending_manifest_file_number_;
std::atomic<uint64_t> last_sequence_;
uint64_t prev_log_number_; // 0 or backing store for memtable being compacted

12
utilities/backupable/backupable_db.cc
View File

@ -29,17 +29,17 @@
#include <inttypes.h>
#include <stdlib.h>
#include <algorithm>
#include <vector>
#include <atomic>
#include <future>
#include <limits>
#include <map>
#include <mutex>
#include <sstream>
#include <string>
#include <limits>
#include <atomic>
#include <future>
#include <thread>
#include <unordered_map>
#include <unordered_set>
#include <vector>
namespace rocksdb {
@ -694,6 +694,7 @@ Status BackupEngineImpl::CreateNewBackupWithMetadata(
TEST_SYNC_POINT("BackupEngineImpl::CreateNewBackup:SavedLiveFiles2");
BackupID new_backup_id = latest_backup_id_ + 1;
assert(backups_.find(new_backup_id) == backups_.end());
auto ret = backups_.insert(
std::make_pair(new_backup_id, unique_ptr<BackupMeta>(new BackupMeta(
@ -745,7 +746,8 @@ Status BackupEngineImpl::CreateNewBackupWithMetadata(
}
// we should only get sst, manifest and current files here
assert(type == kTableFile || type == kDescriptorFile ||
type == kCurrentFile);
type == kCurrentFile || type == kOptionsFile);
if (type == kCurrentFile) {
// We will craft the current file manually to ensure it's consistent with
// the manifest number. This is necessary because current's file contents

60
utilities/backupable/backupable_db_test.cc
View File

@ -20,10 +20,12 @@
#include "rocksdb/transaction_log.h"
#include "rocksdb/types.h"
#include "rocksdb/utilities/backupable_db.h"
#include "rocksdb/utilities/options_util.h"
#include "util/env_chroot.h"
#include "util/file_reader_writer.h"
#include "util/mutexlock.h"
#include "util/random.h"
#include "util/stderr_logger.h"
#include "util/string_util.h"
#include "util/sync_point.h"
#include "util/testharness.h"
@ -200,6 +202,7 @@ class TestEnv : public EnvWrapper {
MutexLock l(&mutex_);
std::sort(should_have_written.begin(), should_have_written.end());
std::sort(written_files_.begin(), written_files_.end());
ASSERT_TRUE(written_files_ == should_have_written);
}
@ -756,8 +759,8 @@ TEST_F(BackupableDBTest, NoDoubleCopy) {
test_backup_env_->SetLimitWrittenFiles(7);
test_backup_env_->ClearWrittenFiles();
test_db_env_->SetLimitWrittenFiles(0);
dummy_db_->live_files_ = { "/00010.sst", "/00011.sst",
"/CURRENT", "/MANIFEST-01" };
dummy_db_->live_files_ = {"/00010.sst", "/00011.sst", "/CURRENT",
"/MANIFEST-01"};
dummy_db_->wal_files_ = {{"/00011.log", true}, {"/00012.log", false}};
test_db_env_->SetFilenamesForMockedAttrs(dummy_db_->live_files_);
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), false));
@ -773,20 +776,20 @@ TEST_F(BackupableDBTest, NoDoubleCopy) {
// should not write/copy 00010.sst, since it's already there!
test_backup_env_->SetLimitWrittenFiles(6);
test_backup_env_->ClearWrittenFiles();
dummy_db_->live_files_ = { "/00010.sst", "/00015.sst",
"/CURRENT", "/MANIFEST-01" };
dummy_db_->live_files_ = {"/00010.sst", "/00015.sst", "/CURRENT",
"/MANIFEST-01"};
dummy_db_->wal_files_ = {{"/00011.log", true}, {"/00012.log", false}};
test_db_env_->SetFilenamesForMockedAttrs(dummy_db_->live_files_);
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), false));
// should not open 00010.sst - it's already there
should_have_written = {
"/shared/00015.sst.tmp",
"/private/2.tmp/CURRENT",
"/private/2.tmp/MANIFEST-01",
"/private/2.tmp/00011.log",
"/meta/2.tmp",
"/LATEST_BACKUP.tmp"
};
should_have_written = {"/shared/00015.sst.tmp",
"/private/2.tmp/CURRENT",
"/private/2.tmp/MANIFEST-01",
"/private/2.tmp/00011.log",
"/meta/2.tmp",
"/LATEST_BACKUP.tmp"};
AppendPath(backupdir_, should_have_written);
test_backup_env_->AssertWrittenFiles(should_have_written);
@ -943,6 +946,39 @@ TEST_F(BackupableDBTest, CorruptionsTest) {
AssertBackupConsistency(2, 0, keys_iteration * 2, keys_iteration * 5);
}
inline std::string OptionsPath(std::string ret, int backupID) {
ret += "/private/";
ret += std::to_string(backupID);
ret += "/";
return ret;
}
// Backup the LATEST options file to
// "<backup_dir>/private/<backup_id>/OPTIONS<number>"
TEST_F(BackupableDBTest, BackupOptions) {
OpenDBAndBackupEngine(true);
for (int i = 1; i < 5; i++) {
std::string name;
std::vector<std::string> filenames;
// Must reset() before reset(OpenDB()) again.
// Calling OpenDB() while *db_ is existing will cause LOCK issue
db_.reset();
db_.reset(OpenDB());
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), true));
rocksdb::GetLatestOptionsFileName(db_->GetName(), options_.env, &name);
ASSERT_OK(file_manager_->FileExists(OptionsPath(backupdir_, i) + name));
backup_chroot_env_->GetChildren(OptionsPath(backupdir_, i), &filenames);
for (auto fn : filenames) {
if (fn.compare(0, 7, "OPTIONS") == 0) {
ASSERT_EQ(name, fn);
}
}
}
CloseDBAndBackupEngine();
}
// This test verifies we don't delete the latest backup when read-only option is
// set
TEST_F(BackupableDBTest, NoDeleteWithReadOnly) {

4
utilities/checkpoint/checkpoint.cc
View File

@ -110,9 +110,9 @@ Status CheckpointImpl::CreateCheckpoint(const std::string& checkpoint_dir) {
s = Status::Corruption("Can't parse file name. This is very bad");
break;
}
// we should only get sst, manifest and current files here
// we should only get sst, options, manifest and current files here
assert(type == kTableFile || type == kDescriptorFile ||
type == kCurrentFile);
type == kCurrentFile || type == kOptionsFile);
assert(live_files[i].size() > 0 && live_files[i][0] == '/');
if (type == kCurrentFile) {
// We will craft the current file manually to ensure it's consistent with