andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Small changes in Deleting obsolete files

Browse Source 
Summary:
@haobo's suggestions from https://reviews.facebook.net/D13827

Renaming some variables, deprecating purge_log_after_flush, changing for loop into auto for loop.

I have not implemented deleting objects outside of mutex yet because it would require a big code change - we would delete object in db_impl, which currently does not know anything about object because it's defined in version_edit.h (FileMetaData). We should do it at some point, though.

Test Plan: Ran deletefile_test

Reviewers: haobo

Reviewed By: haobo

CC: leveldb, haobo

Differential Revision: https://reviews.facebook.net/D14025
This commit is contained in:
Igor Canadi 2013-11-12 11:53:26 -08:00
parent dad425562f
commit 9bc4a26f56
6 changed files with 51 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
db/db_impl.cc
View File

@ -435,8 +435,8 @@ void DBImpl::MaybeDumpStats() {
} }
} }
// Returns the list of live files in 'sstlive' and the list // Returns the list of live files in 'sst_live' and the list
// of all files in the filesystem in 'allfiles'. // of all files in the filesystem in 'all_files'.
void DBImpl::FindObsoleteFiles(DeletionState& deletion_state, bool force) { void DBImpl::FindObsoleteFiles(DeletionState& deletion_state, bool force) {
mutex_.AssertHeld(); mutex_.AssertHeld();
@ -464,19 +464,19 @@ void DBImpl::FindObsoleteFiles(DeletionState& deletion_state, bool force) {
// Make a list of all of the live files; set is slow, should not // Make a list of all of the live files; set is slow, should not
// be used. // be used.
deletion_state.sstlive.assign(pending_outputs_.begin(), deletion_state.sst_live.assign(pending_outputs_.begin(),
pending_outputs_.end()); pending_outputs_.end());
versions_->AddLiveFiles(&deletion_state.sstlive); versions_->AddLiveFiles(&deletion_state.sst_live);
// set of all files in the directory // set of all files in the directory
env_->GetChildren(dbname_, &deletion_state.allfiles); // Ignore errors env_->GetChildren(dbname_, &deletion_state.all_files); // Ignore errors
//Add log files in wal_dir //Add log files in wal_dir
if (options_.wal_dir != dbname_) { if (options_.wal_dir != dbname_) {
std::vector<std::string> log_files; std::vector<std::string> log_files;
env_->GetChildren(options_.wal_dir, &log_files); // Ignore errors env_->GetChildren(options_.wal_dir, &log_files); // Ignore errors
deletion_state.allfiles.insert( deletion_state.all_files.insert(
deletion_state.allfiles.end(), deletion_state.all_files.end(),
log_files.begin(), log_files.begin(),
log_files.end() log_files.end()
); );
@ -485,7 +485,7 @@ void DBImpl::FindObsoleteFiles(DeletionState& deletion_state, bool force) {
// Diffs the files listed in filenames and those that do not // Diffs the files listed in filenames and those that do not
// belong to live files are posibly removed. Also, removes all the // belong to live files are posibly removed. Also, removes all the
// files in sstdeletefiles and logdeletefiles. // files in sst_delete_files and log_delete_files.
// It is not necessary to hold the mutex when invoking this method. // It is not necessary to hold the mutex when invoking this method.
void DBImpl::PurgeObsoleteFiles(DeletionState& state) { void DBImpl::PurgeObsoleteFiles(DeletionState& state) {
// if deletion is disabled, do nothing // if deletion is disabled, do nothing
@ -499,23 +499,26 @@ void DBImpl::PurgeObsoleteFiles(DeletionState& state) {
// Now, convert live list to an unordered set, WITHOUT mutex held; // Now, convert live list to an unordered set, WITHOUT mutex held;
// set is slow. // set is slow.
std::unordered_set<uint64_t> live_set(state.sstlive.begin(), std::unordered_set<uint64_t> live_set(state.sst_live.begin(),
state.sstlive.end()); state.sst_live.end());
state.allfiles.reserve(state.allfiles.size() + state.sstdeletefiles.size()); state.all_files.reserve(state.all_files.size() +
for (auto filenum : state.sstdeletefiles) { state.sst_delete_files.size());
state.allfiles.push_back(TableFileName("", filenum)); for (auto file : state.sst_delete_files) {
state.all_files.push_back(TableFileName("", file->number));
delete file;
} }
state.allfiles.reserve(state.allfiles.size() + state.logdeletefiles.size()); state.all_files.reserve(state.all_files.size() +
for (auto filenum : state.logdeletefiles) { state.log_delete_files.size());
for (auto filenum : state.log_delete_files) {
if (filenum > 0) { if (filenum > 0) {
state.allfiles.push_back(LogFileName("", filenum)); state.all_files.push_back(LogFileName("", filenum));
} }
} }
for (size_t i = 0; i < state.allfiles.size(); i++) { for (size_t i = 0; i < state.all_files.size(); i++) {
if (ParseFileName(state.allfiles[i], &number, &type)) { if (ParseFileName(state.all_files[i], &number, &type)) {
bool keep = true; bool keep = true;
switch (type) { switch (type) {
case kLogFile: case kLogFile:
@ -538,7 +541,7 @@ void DBImpl::PurgeObsoleteFiles(DeletionState& state) {
case kInfoLogFile: case kInfoLogFile:
keep = true; keep = true;
if (number != 0) { if (number != 0) {
old_log_files.push_back(state.allfiles[i]); old_log_files.push_back(state.all_files[i]);
} }
break; break;
case kCurrentFile: case kCurrentFile:
@ -559,14 +562,14 @@ void DBImpl::PurgeObsoleteFiles(DeletionState& state) {
Status st; Status st;
if (type == kLogFile && (options_.WAL_ttl_seconds > 0 || if (type == kLogFile && (options_.WAL_ttl_seconds > 0 ||
options_.WAL_size_limit_MB > 0)) { options_.WAL_size_limit_MB > 0)) {
st = env_->RenameFile(dbname_ + "/" + state.allfiles[i], st = env_->RenameFile(dbname_ + "/" + state.all_files[i],
ArchivedLogFileName(options_.wal_dir, ArchivedLogFileName(options_.wal_dir,
number)); number));
if (!st.ok()) { if (!st.ok()) {
Log(options_.info_log, "RenameFile logfile #%lu FAILED", number); Log(options_.info_log, "RenameFile logfile #%lu FAILED", number);
} }
} else { } else {
st = env_->DeleteFile(dbname_ + "/" + state.allfiles[i]); st = env_->DeleteFile(dbname_ + "/" + state.all_files[i]);
if (!st.ok()) { if (!st.ok()) {
Log(options_.info_log, "Delete type=%d #%lu FAILED\n", Log(options_.info_log, "Delete type=%d #%lu FAILED\n",
int(type), number); int(type), number);
@ -1132,12 +1135,12 @@ Status DBImpl::FlushMemTableToOutputFile(bool* madeProgress,
MaybeScheduleLogDBDeployStats(); MaybeScheduleLogDBDeployStats();
if (options_.purge_log_after_memtable_flush && if (!disable_delete_obsolete_files_) {
!disable_delete_obsolete_files_) {
// add to deletion state // add to deletion state
deletion_state.logdeletefiles.insert(deletion_state.logdeletefiles.end(), deletion_state.log_delete_files.insert(
logs_to_delete.begin(), deletion_state.log_delete_files.end(),
logs_to_delete.end()); logs_to_delete.begin(),
logs_to_delete.end());
} }
} }
return s; return s;
@ -1776,7 +1779,7 @@ Status DBImpl::BackgroundCompaction(bool* madeProgress,
CleanupCompaction(compact, status); CleanupCompaction(compact, status);
versions_->ReleaseCompactionFiles(c.get(), status); versions_->ReleaseCompactionFiles(c.get(), status);
c->ReleaseInputs(); c->ReleaseInputs();
versions_->GetAndFreeObsoleteFiles(&deletion_state.sstdeletefiles); versions_->GetObsoleteFiles(&deletion_state.sst_delete_files);
*madeProgress = true; *madeProgress = true;
} }
c.reset(); c.reset();
@ -3366,7 +3369,7 @@ Status DBImpl::DeleteFile(std::string name) {
edit.DeleteFile(level, number); edit.DeleteFile(level, number);
status = versions_->LogAndApply(&edit, &mutex_); status = versions_->LogAndApply(&edit, &mutex_);
if (status.ok()) { if (status.ok()) {
versions_->GetAndFreeObsoleteFiles(&deletion_state.sstdeletefiles); versions_->GetObsoleteFiles(&deletion_state.sst_delete_files);
} }
FindObsoleteFiles(deletion_state, false); FindObsoleteFiles(deletion_state, false);
} // lock released here } // lock released here

17
db/db_impl.h
View File

@ -14,6 +14,7 @@
#include "db/dbformat.h" #include "db/dbformat.h"
#include "db/log_writer.h" #include "db/log_writer.h"
#include "db/snapshot.h" #include "db/snapshot.h"
#include "db/version_edit.h"
#include "rocksdb/db.h" #include "rocksdb/db.h"
#include "rocksdb/env.h" #include "rocksdb/env.h"
#include "rocksdb/memtablerep.h" #include "rocksdb/memtablerep.h"
@ -159,22 +160,24 @@ class DBImpl : public DB {
struct DeletionState { struct DeletionState {
inline bool HaveSomethingToDelete() const { inline bool HaveSomethingToDelete() const {
return allfiles.size() || sstdeletefiles.size() || logdeletefiles.size(); return all_files.size() ||
sst_delete_files.size() ||
log_delete_files.size();
} }
// a list of all files that we'll consider deleting // a list of all files that we'll consider deleting
// (every once in a while this is filled up with all files // (every once in a while this is filled up with all files
// in the DB directory) // in the DB directory)
std::vector<std::string> allfiles; std::vector<std::string> all_files;
// the list of all live sst files that cannot be deleted // the list of all live sst files that cannot be deleted
std::vector<uint64_t> sstlive; std::vector<uint64_t> sst_live;
// a list of sst files that we need to delete // a list of sst files that we need to delete
std::vector<uint64_t> sstdeletefiles; std::vector<FileMetaData*> sst_delete_files;
// a list of log files that we need to delete // a list of log files that we need to delete
std::vector<uint64_t> logdeletefiles; std::vector<uint64_t> log_delete_files;
// the current manifest_file_number, log_number and prev_log_number // the current manifest_file_number, log_number and prev_log_number
// that corresponds to the set of files in 'live'. // that corresponds to the set of files in 'live'.
@ -241,7 +244,7 @@ class DBImpl : public DB {
void ReleaseCompactionUnusedFileNumbers(CompactionState* compact); void ReleaseCompactionUnusedFileNumbers(CompactionState* compact);
// Returns the list of live files in 'live' and the list // Returns the list of live files in 'live' and the list
// of all files in the filesystem in 'allfiles'. // of all files in the filesystem in 'all_files'.
// If force == false and the last call was less than // If force == false and the last call was less than
// options_.delete_obsolete_files_period_micros microseconds ago, // options_.delete_obsolete_files_period_micros microseconds ago,
// it will not fill up the deletion_state // it will not fill up the deletion_state
@ -249,7 +252,7 @@ class DBImpl : public DB {
// Diffs the files listed in filenames and those that do not // Diffs the files listed in filenames and those that do not
// belong to live files are posibly removed. Also, removes all the // belong to live files are posibly removed. Also, removes all the
// files in sstdeletefiles and logdeletefiles. // files in sst_delete_files and log_delete_files.
// It is not necessary to hold the mutex when invoking this method. // It is not necessary to hold the mutex when invoking this method.
void PurgeObsoleteFiles(DeletionState& deletion_state); void PurgeObsoleteFiles(DeletionState& deletion_state);

19
db/version_set.cc
View File

@ -1161,7 +1161,10 @@ VersionSet::VersionSet(const std::string& dbname,
VersionSet::~VersionSet() { VersionSet::~VersionSet() {
current_->Unref(); current_->Unref();
assert(dummy_versions_.next_ == &dummy_versions_); // List must be empty assert(dummy_versions_.next_ == &dummy_versions_); // List must be empty
GetAndFreeObsoleteFiles(nullptr); for (auto file : obsolete_files_) {
delete file;
}
obsolete_files_.clear();
delete[] compact_pointer_; delete[] compact_pointer_;
delete[] max_file_size_; delete[] max_file_size_;
delete[] level_max_bytes_; delete[] level_max_bytes_;
@ -2861,16 +2864,10 @@ void VersionSet::GetLiveFilesMetaData(
} }
} }
void VersionSet::GetAndFreeObsoleteFiles(std::vector<uint64_t>* files) { void VersionSet::GetObsoleteFiles(std::vector<FileMetaData*>* files) {
if (files != nullptr) { files->insert(files->end(),
files->reserve(files->size() + obsolete_files_.size()); obsolete_files_.begin(),
} obsolete_files_.end());
for (size_t i = 0; i < obsolete_files_.size(); i++) {
if (files != nullptr) {
files->push_back(obsolete_files_[i]->number);
}
delete obsolete_files_[i];
}
obsolete_files_.clear(); obsolete_files_.clear();
} }

2
db/version_set.h
View File

@ -431,7 +431,7 @@ class VersionSet {
void GetLiveFilesMetaData( void GetLiveFilesMetaData(
std::vector<LiveFileMetaData> *metadata); std::vector<LiveFileMetaData> *metadata);
void GetAndFreeObsoleteFiles(std::vector<uint64_t>* files); void GetObsoleteFiles(std::vector<FileMetaData*>* files);
private: private:
class Builder; class Builder;

5
include/rocksdb/options.h
View File

@ -609,11 +609,6 @@ struct Options {
// Default: a factory that doesn't provide any object // Default: a factory that doesn't provide any object
std::shared_ptr<CompactionFilterFactory> compaction_filter_factory; std::shared_ptr<CompactionFilterFactory> compaction_filter_factory;
// Remove the log file immediately after the corresponding memtable is flushed
// to data file.
// Default: true
bool purge_log_after_memtable_flush;
// This option allows user to to collect their own interested statistics of // This option allows user to to collect their own interested statistics of
// the tables. // the tables.
// Default: emtpy vector -- no user-defined statistics collection will be // Default: emtpy vector -- no user-defined statistics collection will be

3
util/options.cc
View File

@ -101,7 +101,6 @@ Options::Options()
compaction_filter_factory( compaction_filter_factory(
std::shared_ptr<CompactionFilterFactory>( std::shared_ptr<CompactionFilterFactory>(
new DefaultCompactionFilterFactory())), new DefaultCompactionFilterFactory())),
purge_log_after_memtable_flush(true),
inplace_update_support(false), inplace_update_support(false),
inplace_update_num_locks(10000) { inplace_update_num_locks(10000) {
assert(memtable_factory.get() != nullptr); assert(memtable_factory.get() != nullptr);
@ -280,8 +279,6 @@ Options::Dump(Logger* log) const
Log(log,"Options.compaction_options_universal." Log(log,"Options.compaction_options_universal."
"max_size_amplification_percent: %u", "max_size_amplification_percent: %u",
compaction_options_universal.max_size_amplification_percent); compaction_options_universal.max_size_amplification_percent);
Log(log," Options.purge_log_after_memtable_flush: %d",
purge_log_after_memtable_flush);
std::string collector_names; std::string collector_names;
for (auto collector : table_stats_collectors) { for (auto collector : table_stats_collectors) {
collector_names.append(collector->Name()); collector_names.append(collector->Name());