andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add --wal_bytes_per_sync for db_bench and more IO stats

Browse Source 
Summary:
See https://gist.github.com/mdcallag/89ebb2b8cbd331854865 for the IO stats.
I added "Cumulative compaction:" and "Interval compaction:" lines. The IO rates
can be confusing. Rates fro per-level stats lines, Wr(MB/s) & Rd(MB/s), are computed
using the duration of the compaction job. If the job reads 10MB, writes 9MB and the job
(IO & merging) takes 1 second then the rates are 10MB/s for read and 9MB/s for writes.
The IO rates in the Cumulative compaction line uses the total uptime. The IO rates in the
Interval compaction line uses the interval uptime. So these Cumalative & Interval
compaction IO rates cannot be compared to the per-level IO rates. But both forms of
the rates are useful for debugging perf.

Task ID: #

Blame Rev:

Test Plan:
run db_bench

Revert Plan:

Database Impact:

Memcache Impact:

Other Notes:

EImportant:

- begin *PUBLIC* platform impact section -
Bugzilla: #
- end platform impact -

Reviewers: igor

Reviewed By: igor

Subscribers: dhruba

Differential Revision: https://reviews.facebook.net/D38667
This commit is contained in:
Mark Callaghan 2015-05-19 16:19:30 -07:00
parent d5de04d20e
commit 944043d683
3 changed files with 58 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
db/db_bench.cc
View File

@ -558,9 +558,15 @@ DEFINE_bool(use_adaptive_mutex, rocksdb::Options().use_adaptive_mutex,
"Use adaptive mutex");
DEFINE_uint64(bytes_per_sync, rocksdb::Options().bytes_per_sync,
"Allows OS to incrementally sync files to disk while they are"
"Allows OS to incrementally sync SST files to disk while they are"
" being written, in the background. Issue one request for every"
" bytes_per_sync written. 0 turns it off.");
DEFINE_uint64(wal_bytes_per_sync, rocksdb::Options().wal_bytes_per_sync,
"Allows OS to incrementally sync WAL files to disk while they are"
" being written, in the background. Issue one request for every"
" wal_bytes_per_sync written. 0 turns it off.");
DEFINE_bool(filter_deletes, false, " On true, deletes use bloom-filter and drop"
" the delete if key not present");
@ -2224,6 +2230,7 @@ class Benchmark {
options.access_hint_on_compaction_start = FLAGS_compaction_fadvice_e;
options.use_adaptive_mutex = FLAGS_use_adaptive_mutex;
options.bytes_per_sync = FLAGS_bytes_per_sync;
options.wal_bytes_per_sync = FLAGS_wal_bytes_per_sync;
// merge operator options
options.merge_operator = MergeOperators::CreateFromStringId(

43
db/internal_stats.cc
View File

@ -397,6 +397,10 @@ void InternalStats::DumpDBStats(std::string* value) {
uint64_t wal_synced = db_stats_[InternalStats::WAL_FILE_SYNCED];
uint64_t write_with_wal = db_stats_[InternalStats::WRITE_WITH_WAL];
uint64_t write_stall_micros = db_stats_[InternalStats::WRITE_STALL_MICROS];
uint64_t compact_bytes_read = 0;
uint64_t compact_bytes_write = 0;
uint64_t compact_micros = 0;
const int kHumanMicrosLen = 32;
char human_micros[kHumanMicrosLen];
@ -427,6 +431,22 @@ void InternalStats::DumpDBStats(std::string* value) {
write_with_wal / static_cast<double>(wal_synced + 1),
wal_bytes / kGB, wal_bytes / kMB / seconds_up);
value->append(buf);
// Compact
for (int level = 0; level < number_levels_; level++) {
compact_bytes_read += comp_stats_[level].bytes_readnp1 +
comp_stats_[level].bytes_readn;
compact_bytes_write += comp_stats_[level].bytes_written;
compact_micros += comp_stats_[level].micros;
}
snprintf(buf, sizeof(buf),
"Cumulative compaction: %.2f GB write, %.2f MB/s write, "
"%.2f GB read, %.2f MB/s read, %.1f seconds\n",
compact_bytes_write / kGB,
compact_bytes_write / kMB / seconds_up,
compact_bytes_read / kGB,
compact_bytes_read / kMB / seconds_up,
compact_micros / 1000000.0);
value->append(buf);
// Stall
AppendHumanMicros(write_stall_micros, human_micros, kHumanMicrosLen, true);
snprintf(buf, sizeof(buf),
@ -471,6 +491,26 @@ void InternalStats::DumpDBStats(std::string* value) {
interval_wal_bytes / kMB / std::max(interval_seconds_up, 0.001));
value->append(buf);
// Compaction
uint64_t interval_compact_bytes_write =
compact_bytes_write - db_stats_snapshot_.compact_bytes_write;
uint64_t interval_compact_bytes_read =
compact_bytes_read - db_stats_snapshot_.compact_bytes_read;
uint64_t interval_compact_micros =
compact_micros - db_stats_snapshot_.compact_micros;
snprintf(buf, sizeof(buf),
"Interval compaction: %.2f GB write, %.2f MB/s write, "
"%.2f GB read, %.2f MB/s read, %.1f seconds\n",
interval_compact_bytes_write / kGB,
interval_compact_bytes_write / kMB /
std::max(interval_seconds_up, 0.001),
interval_compact_bytes_read / kGB,
interval_compact_bytes_read / kMB /
std::max(interval_seconds_up, 0.001),
interval_compact_micros / 1000000.0);
value->append(buf);
// Stall
AppendHumanMicros(
write_stall_micros - db_stats_snapshot_.write_stall_micros,
@ -492,6 +532,9 @@ void InternalStats::DumpDBStats(std::string* value) {
db_stats_snapshot_.wal_synced = wal_synced;
db_stats_snapshot_.write_with_wal = write_with_wal;
db_stats_snapshot_.write_stall_micros = write_stall_micros;
db_stats_snapshot_.compact_bytes_write = compact_bytes_write;
db_stats_snapshot_.compact_bytes_read = compact_bytes_read;
db_stats_snapshot_.compact_micros = compact_micros;
}
void InternalStats::DumpCFStats(std::string* value) {

7
db/internal_stats.h
View File

@ -279,6 +279,10 @@ class InternalStats {
// another thread.
uint64_t write_other;
uint64_t write_self;
// Stats from compaction jobs - bytes written, bytes read, duration.
uint64_t compact_bytes_write;
uint64_t compact_bytes_read;
uint64_t compact_micros;
// Total number of keys written. write_self and write_other measure number
// of write requests written, Each of the write request can contain updates
// to multiple keys. num_keys_written is total number of keys updated by all
@ -295,6 +299,9 @@ class InternalStats {
write_with_wal(0),
write_other(0),
write_self(0),
compact_bytes_write(0),
compact_bytes_read(0),
compact_micros(0),
num_keys_written(0),
write_stall_micros(0),
seconds_up(0) {}