andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

utilities/backupable: backup should limit the copy size of wal.

Browse Source 
Summary:
Since the backup work as snapshot, we should only copy
 the bytes of the wal while we get the alive files.
Closes https://github.com/facebook/rocksdb/pull/1733

Differential Revision: D4373457

Pulled By: ajkr

fbshipit-source-id: 389318f
This commit is contained in:
Vincent Lee 2016-12-31 10:35:38 -08:00 committed by Facebook Github Bot
parent 0712d541d1
commit e425ec1162
2 changed files with 12 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
db/c.cc
View File

@ -448,7 +448,11 @@ rocksdb_backup_engine_t* rocksdb_backup_engine_open(
const rocksdb_options_t* options, const char* path, char** errptr) {
BackupEngine* be;
if (SaveError(errptr, BackupEngine::Open(options->rep.env,
BackupableDBOptions(path), &be))) {
BackupableDBOptions(path,
nullptr,
true,
options->rep.info_log.get()),
&be))) {
return nullptr;
}
rocksdb_backup_engine_t* result = new rocksdb_backup_engine_t;

26
utilities/backupable/backupable_db.cc
View File

@ -781,36 +781,24 @@ Status BackupEngineImpl::CreateNewBackupWithMetadata(
manifest_fname.size(), 0 /* size_limit */, false /* shared_checksum */,
progress_callback, manifest_fname.substr(1) + "\n");
}
// Pre-fetch sizes for WAL files
std::unordered_map<std::string, uint64_t> wal_path_to_size;
if (s.ok()) {
if (db->GetOptions().wal_dir != "") {
s = InsertPathnameToSizeBytes(db->GetOptions().wal_dir, db_env_,
&wal_path_to_size);
} else {
wal_path_to_size = std::move(data_path_to_size);
}
}
Log(options_.info_log, "begin add wal files for backup -- %" ROCKSDB_PRIszt,
live_wal_files.size());
// Add a CopyOrCreateWorkItem to the channel for each WAL file
for (size_t i = 0; s.ok() && i < live_wal_files.size(); ++i) {
auto wal_path_to_size_iter =
wal_path_to_size.find(live_wal_files[i]->PathName());
uint64_t size_bytes = wal_path_to_size_iter == wal_path_to_size.end()
? port::kMaxUint64
: wal_path_to_size_iter->second;
uint64_t size_bytes = live_wal_files[i]->SizeFileBytes();
if (live_wal_files[i]->Type() == kAliveLogFile) {
Log(options_.info_log, "add wal file for backup %s -- %" PRIu64,
live_wal_files[i]->PathName().c_str(), size_bytes);
// we only care about live log files
// copy the file into backup_dir/files/<new backup>/
s = AddBackupFileWorkItem(live_dst_paths, backup_items_to_finish,
new_backup_id, false, /* not shared */
db->GetOptions().wal_dir,
live_wal_files[i]->PathName(), rate_limiter,
size_bytes);
size_bytes, size_bytes);
}
}
Log(options_.info_log, "add files for backup done, wait finish.");
Status item_status;
for (auto& item : backup_items_to_finish) {
item.result.wait();