andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

backup garbage collect shared_checksum tmp files

Browse Source 
Summary:
previously we only cleaned up .tmp files under "shared/" and "private/" directories in case the previous backup failed. we need to do the same for "shared_checksum/"; otherwise, the subsequent backup will fail if it tries to backup at least one of the same files.
Closes https://github.com/facebook/rocksdb/pull/2062

Differential Revision: D4805599

Pulled By: ajkr

fbshipit-source-id: eaa6088
This commit is contained in:
Andrew Kryczka 2017-03-30 14:44:56 -07:00 committed by Facebook Github Bot
parent da175f7ecc
commit a9c86f51b7
2 changed files with 48 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
utilities/backupable/backupable_db.cc
View File

@ -1458,7 +1458,12 @@ Status BackupEngineImpl::GarbageCollect() {
// delete obsolete shared files // delete obsolete shared files
std::vector<std::string> shared_children; std::vector<std::string> shared_children;
{ {
auto shared_path = GetAbsolutePath(GetSharedFileRel()); std::string shared_path;
if (options_.share_files_with_checksum) {
shared_path = GetAbsolutePath(GetSharedFileWithChecksumRel());
} else {
shared_path = GetAbsolutePath(GetSharedFileRel());
}
auto s = backup_env_->FileExists(shared_path); auto s = backup_env_->FileExists(shared_path);
if (s.ok()) { if (s.ok()) {
s = backup_env_->GetChildren(shared_path, &shared_children); s = backup_env_->GetChildren(shared_path, &shared_children);
@ -1470,7 +1475,12 @@ Status BackupEngineImpl::GarbageCollect() {
} }
} }
for (auto& child : shared_children) { for (auto& child : shared_children) {
std::string rel_fname = GetSharedFileRel(child); std::string rel_fname;
if (options_.share_files_with_checksum) {
rel_fname = GetSharedFileWithChecksumRel(child);
} else {
rel_fname = GetSharedFileRel(child);
}
auto child_itr = backuped_file_infos_.find(rel_fname); auto child_itr = backuped_file_infos_.find(rel_fname);
// if it's not refcounted, delete it // if it's not refcounted, delete it
if (child_itr == backuped_file_infos_.end() || if (child_itr == backuped_file_infos_.end() ||

52
utilities/backupable/backupable_db_test.cc
View File

@ -1126,22 +1126,42 @@ TEST_F(BackupableDBTest, ShareTableFilesWithChecksumsTransition) {
} }
TEST_F(BackupableDBTest, DeleteTmpFiles) { TEST_F(BackupableDBTest, DeleteTmpFiles) {
OpenDBAndBackupEngine(); for (bool shared_checksum : {false, true}) {
CloseDBAndBackupEngine(); if (shared_checksum) {
std::string shared_tmp = backupdir_ + "/shared/00006.sst.tmp"; OpenDBAndBackupEngineShareWithChecksum(
std::string private_tmp_dir = backupdir_ + "/private/10.tmp"; false /* destroy_old_data */, false /* dummy */,
std::string private_tmp_file = private_tmp_dir + "/00003.sst"; true /* share_table_files */, true /* share_with_checksums */);
file_manager_->WriteToFile(shared_tmp, "tmp"); } else {
file_manager_->CreateDir(private_tmp_dir); OpenDBAndBackupEngine();
file_manager_->WriteToFile(private_tmp_file, "tmp"); }
ASSERT_OK(file_manager_->FileExists(private_tmp_dir)); CloseDBAndBackupEngine();
OpenDBAndBackupEngine(); std::string shared_tmp = backupdir_;
// Need to call this explicitly to delete tmp files if (shared_checksum) {
(void)backup_engine_->GarbageCollect(); shared_tmp += "/shared_checksum";
CloseDBAndBackupEngine(); } else {
ASSERT_EQ(Status::NotFound(), file_manager_->FileExists(shared_tmp)); shared_tmp += "/shared";
ASSERT_EQ(Status::NotFound(), file_manager_->FileExists(private_tmp_file)); }
ASSERT_EQ(Status::NotFound(), file_manager_->FileExists(private_tmp_dir)); shared_tmp += "/00006.sst.tmp";
std::string private_tmp_dir = backupdir_ + "/private/10.tmp";
std::string private_tmp_file = private_tmp_dir + "/00003.sst";
file_manager_->WriteToFile(shared_tmp, "tmp");
file_manager_->CreateDir(private_tmp_dir);
file_manager_->WriteToFile(private_tmp_file, "tmp");
ASSERT_OK(file_manager_->FileExists(private_tmp_dir));
if (shared_checksum) {
OpenDBAndBackupEngineShareWithChecksum(
false /* destroy_old_data */, false /* dummy */,
true /* share_table_files */, true /* share_with_checksums */);
} else {
OpenDBAndBackupEngine();
}
// Need to call this explicitly to delete tmp files
(void)backup_engine_->GarbageCollect();
CloseDBAndBackupEngine();
ASSERT_EQ(Status::NotFound(), file_manager_->FileExists(shared_tmp));
ASSERT_EQ(Status::NotFound(), file_manager_->FileExists(private_tmp_file));
ASSERT_EQ(Status::NotFound(), file_manager_->FileExists(private_tmp_dir));
}
} }
TEST_F(BackupableDBTest, KeepLogFiles) { TEST_F(BackupableDBTest, KeepLogFiles) {