andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Use StopWatch to do statistic job in db_impl_add_file.cc

Browse Source 
Summary:
patch for diff https://reviews.facebook.net/D58587
Also change StopWatch class to add a fifth param named overwrite which decides whether to overwrite *elapse or add on it.

Test Plan: make all check -j64

Reviewers: sdong, IslamAbdelRahman

Reviewed By: IslamAbdelRahman

Subscribers: andrewkr, dhruba, leveldb

Differential Revision: https://reviews.facebook.net/D61239
This commit is contained in:
Aaron Gao 2016-08-02 14:53:29 -07:00
parent cdc4eb6892
commit 343304e1d3
3 changed files with 26 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
db/db_impl_add_file.cc
View File

@ -22,21 +22,6 @@
namespace rocksdb { namespace rocksdb {
namespace {
// RAII Timer
class StatsTimer {
public:
explicit StatsTimer(Env* env, uint64_t* counter)
: env_(env), counter_(counter), start_micros_(env_->NowMicros()) {}
~StatsTimer() { *counter_ += env_->NowMicros() - start_micros_; }
private:
Env* env_;
uint64_t* counter_;
uint64_t start_micros_;
};
} // anonymous namespace
#ifndef ROCKSDB_LITE #ifndef ROCKSDB_LITE
Status DBImpl::ReadExternalSstFileInfo(ColumnFamilyHandle* column_family, Status DBImpl::ReadExternalSstFileInfo(ColumnFamilyHandle* column_family,
@ -127,7 +112,6 @@ Status DBImpl::AddFile(ColumnFamilyHandle* column_family,
return Status::InvalidArgument("The list of files is empty"); return Status::InvalidArgument("The list of files is empty");
} }
std::vector<ExternalSstFileInfo> file_infos(num_files);
std::vector<ExternalSstFileInfo> file_info_list(num_files); std::vector<ExternalSstFileInfo> file_info_list(num_files);
for (size_t i = 0; i < num_files; i++) { for (size_t i = 0; i < num_files; i++) {
status = ReadExternalSstFileInfo(column_family, file_path_list[i], status = ReadExternalSstFileInfo(column_family, file_path_list[i],
@ -178,7 +162,7 @@ Status DBImpl::AddFile(ColumnFamilyHandle* column_family,
std::vector<uint64_t> micro_list(num_files, 0); std::vector<uint64_t> micro_list(num_files, 0);
std::vector<FileMetaData> meta_list(num_files); std::vector<FileMetaData> meta_list(num_files);
for (size_t i = 0; i < num_files; i++) { for (size_t i = 0; i < num_files; i++) {
StatsTimer t(env_, &micro_list[i]); StopWatch sw(env_, nullptr, 0, &micro_list[i], false);
if (file_info_list[i].num_entries == 0) { if (file_info_list[i].num_entries == 0) {
return Status::InvalidArgument("File contain no entries"); return Status::InvalidArgument("File contain no entries");
} }
@ -212,7 +196,7 @@ Status DBImpl::AddFile(ColumnFamilyHandle* column_family,
{ {
InstrumentedMutexLock l(&mutex_); InstrumentedMutexLock l(&mutex_);
for (size_t i = 0; i < num_files; i++) { for (size_t i = 0; i < num_files; i++) {
StatsTimer t(env_, &micro_list[i]); StopWatch sw(env_, nullptr, 0, &micro_list[i], false);
pending_outputs_inserted_elem_list[i] = pending_outputs_inserted_elem_list[i] =
CaptureCurrentFileNumberInPendingOutputs(); CaptureCurrentFileNumberInPendingOutputs();
meta_list[i].fd = FileDescriptor(versions_->NewFileNumber(), 0, meta_list[i].fd = FileDescriptor(versions_->NewFileNumber(), 0,
@ -224,7 +208,7 @@ Status DBImpl::AddFile(ColumnFamilyHandle* column_family,
std::vector<std::string> db_fname_list(num_files); std::vector<std::string> db_fname_list(num_files);
size_t j = 0; size_t j = 0;
for (; j < num_files; j++) { for (; j < num_files; j++) {
StatsTimer t(env_, &micro_list[j]); StopWatch sw(env_, nullptr, 0, &micro_list[j], false);
db_fname_list[j] = db_fname_list[j] =
TableFileName(db_options_.db_paths, meta_list[j].fd.GetNumber(), TableFileName(db_options_.db_paths, meta_list[j].fd.GetNumber(),
meta_list[j].fd.GetPathId()); meta_list[j].fd.GetPathId());
@ -275,7 +259,7 @@ Status DBImpl::AddFile(ColumnFamilyHandle* column_family,
ScopedArenaIterator iter(NewInternalIterator(ro, cfd, sv, &arena)); ScopedArenaIterator iter(NewInternalIterator(ro, cfd, sv, &arena));
for (size_t i = 0; i < num_files; i++) { for (size_t i = 0; i < num_files; i++) {
StatsTimer t(env_, &micro_list[i]); StopWatch sw(env_, nullptr, 0, &micro_list[i], false);
InternalKey range_start(file_info_list[i].smallest_key, InternalKey range_start(file_info_list[i].smallest_key,
kMaxSequenceNumber, kTypeValue); kMaxSequenceNumber, kTypeValue);
iter->Seek(range_start.Encode()); iter->Seek(range_start.Encode());
@ -298,14 +282,13 @@ Status DBImpl::AddFile(ColumnFamilyHandle* column_family,
} }
} }
// The level the file will be ingested into // The levels that the files will be ingested into
std::vector<int> target_level_list(num_files, 0); std::vector<int> target_level_list(num_files, 0);
if (status.ok()) { if (status.ok()) {
// Add files to L0
VersionEdit edit; VersionEdit edit;
edit.SetColumnFamily(cfd->GetID()); edit.SetColumnFamily(cfd->GetID());
for (size_t i = 0; i < num_files; i++) { for (size_t i = 0; i < num_files; i++) {
StatsTimer t(env_, &micro_list[i]); StopWatch sw(env_, nullptr, 0, &micro_list[i], false);
// Add file to the lowest possible level // Add file to the lowest possible level
target_level_list[i] = PickLevelForIngestedFile(cfd, file_info_list[i]); target_level_list[i] = PickLevelForIngestedFile(cfd, file_info_list[i]);
edit.AddFile(target_level_list[i], meta_list[i].fd.GetNumber(), edit.AddFile(target_level_list[i], meta_list[i].fd.GetNumber(),

4
include/rocksdb/db.h
View File

@ -810,9 +810,9 @@ class DB {
// //
// Current Requirements: // Current Requirements:
// (1) The key ranges of the files don't overlap with each other // (1) The key ranges of the files don't overlap with each other
// (1) The key range of any file in list doesn't overlap with // (2) The key range of any file in list doesn't overlap with
// existing keys or tombstones in DB. // existing keys or tombstones in DB.
// (2) No snapshots are held. // (3) No snapshots are held.
// //
// Notes: We will try to ingest the files to the lowest possible level // Notes: We will try to ingest the files to the lowest possible level
// even if the file compression dont match the level compression // even if the file compression dont match the level compression

32
util/stop_watch.h
View File

@ -10,25 +10,28 @@
namespace rocksdb { namespace rocksdb {
// Auto-scoped. // Auto-scoped.
// Records the measure time into the corresponding histogram if statistics // Records the measure time into the corresponding histogram if statistics
// is not nullptr. It is also saved into *elapsed if the pointer is not nullptr. // is not nullptr. It is also saved into *elapsed if the pointer is not nullptr
// and overwrite is true, it will be added to *elapsed if overwrite is false.
class StopWatch { class StopWatch {
public: public:
StopWatch(Env * const env, Statistics* statistics, StopWatch(Env* const env, Statistics* statistics, const uint32_t hist_type,
const uint32_t hist_type, uint64_t* elapsed = nullptr, bool overwrite = true)
uint64_t* elapsed = nullptr) : env_(env),
: env_(env), statistics_(statistics),
statistics_(statistics), hist_type_(hist_type),
hist_type_(hist_type), elapsed_(elapsed),
elapsed_(elapsed), overwrite_(overwrite),
stats_enabled_(statistics && statistics->HistEnabledForType(hist_type)), stats_enabled_(statistics && statistics->HistEnabledForType(hist_type)),
start_time_((stats_enabled_ || elapsed != nullptr) ? start_time_((stats_enabled_ || elapsed != nullptr) ? env->NowMicros()
env->NowMicros() : 0) { : 0) {}
}
~StopWatch() { ~StopWatch() {
if (elapsed_) { if (elapsed_) {
*elapsed_ = env_->NowMicros() - start_time_; if (overwrite_) {
*elapsed_ = env_->NowMicros() - start_time_;
} else {
*elapsed_ += env_->NowMicros() - start_time_;
}
} }
if (stats_enabled_) { if (stats_enabled_) {
statistics_->measureTime(hist_type_, statistics_->measureTime(hist_type_,
@ -42,6 +45,7 @@ class StopWatch {
Statistics* statistics_; Statistics* statistics_;
const uint32_t hist_type_; const uint32_t hist_type_;
uint64_t* elapsed_; uint64_t* elapsed_;
bool overwrite_;
bool stats_enabled_; bool stats_enabled_;
const uint64_t start_time_; const uint64_t start_time_;
}; };