rocksdb/util/logging.cc
Mark Callaghan 99ec2412e5 Make the benchmark scripts configurable and add tests
Summary:
This makes run_flash_bench.sh configurable. Previously it was hardwired for 1B keys and tests
ran for 12 hours each. That kept me from using it. This makes it configuable, adds more tests,
makes the duration per-test configurable and refactors the test scripts.

Adds the seekrandomwhilemerging test to db_bench which is the same as seekrandomwhilewriting except
the writer thread does Merge rather than Put.

Forces the stall-time column in compaction IO stats to use a fixed format (H:M:S) which makes
it easier to scrape and parse. Also adds an option to AppendHumanMicros to force a fixed format.
Sometimes automation and humans want different format.

Calls thread->stats.AddBytes(bytes); in db_bench for more tests to get the MB/sec summary
stats in the output at test end.

Adds the average ingest rate to compaction IO stats. Output now looks like:
https://gist.github.com/mdcallag/2bd64d18be1b93adc494

More information on the benchmark output is at https://gist.github.com/mdcallag/db43a58bd5ac624f01e1

For benchmark.sh changes default RocksDB configuration to reduce stalls:
* min_level_to_compress from 2 to 3
* hard_rate_limit from 2 to 3
* max_grandparent_overlap_factor and max_bytes_for_level_multiplier from 10 to 8
* L0 file count triggers from 4,8,12 to 4,12,20 for (start,stall,stop)

Task ID: #6596829

Blame Rev:

Test Plan:
run tools/run_flash_bench.sh

Revert Plan:

Database Impact:

Memcache Impact:

Other Notes:

EImportant:

- begin *PUBLIC* platform impact section -
Bugzilla: #
- end platform impact -

Reviewers: igor

Reviewed By: igor

Subscribers: dhruba

Differential Revision: https://reviews.facebook.net/D36075
2015-03-30 11:28:25 -07:00

146 lines
4.3 KiB
C++

// Copyright (c) 2013, Facebook, Inc. All rights reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
//
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#include "util/logging.h"
#ifndef __STDC_FORMAT_MACROS
#define __STDC_FORMAT_MACROS
#endif
#include <cmath>
#include <inttypes.h>
#include <errno.h>
#include <stdarg.h>
#include <stdio.h>
#include <stdlib.h>
#include "rocksdb/env.h"
#include "rocksdb/slice.h"
namespace rocksdb {
// for micros < 10ms, print "XX us".
// for micros < 10sec, print "XX ms".
// for micros >= 10 sec, print "XX sec".
// for micros <= 1 hour, print Y:X M:S".
// for micros > 1 hour, print Z:Y:X H:M:S".
int AppendHumanMicros(uint64_t micros, char* output, int len,
bool fixed_format) {
if (micros < 10000 && !fixed_format) {
return snprintf(output, len, "%" PRIu64 " us", micros);
} else if (micros < 10000000 && !fixed_format) {
return snprintf(output, len, "%.3lf ms",
static_cast<double>(micros) / 1000);
} else if (micros < 1000000l * 60 && !fixed_format) {
return snprintf(output, len, "%.3lf sec",
static_cast<double>(micros) / 1000000);
} else if (micros < 1000000ll * 60 * 60 && !fixed_format) {
return snprintf(output, len, "%02" PRIu64 ":%05.3f M:S",
micros / 1000000 / 60,
static_cast<double>(micros % 60000000) / 1000000);
} else {
return snprintf(output, len,
"%02" PRIu64 ":%02" PRIu64 ":%05.3f H:M:S",
micros / 1000000 / 3600,
(micros / 1000000 / 60) % 60,
static_cast<double>(micros % 60000000) / 1000000);
}
}
// for sizes >=10TB, print "XXTB"
// for sizes >=10GB, print "XXGB"
// etc.
// append file size summary to output and return the len
int AppendHumanBytes(uint64_t bytes, char* output, int len) {
const uint64_t ull10 = 10;
if (bytes >= ull10 << 40) {
return snprintf(output, len, "%" PRIu64 "TB", bytes >> 40);
} else if (bytes >= ull10 << 30) {
return snprintf(output, len, "%" PRIu64 "GB", bytes >> 30);
} else if (bytes >= ull10 << 20) {
return snprintf(output, len, "%" PRIu64 "MB", bytes >> 20);
} else if (bytes >= ull10 << 10) {
return snprintf(output, len, "%" PRIu64 "KB", bytes >> 10);
} else {
return snprintf(output, len, "%" PRIu64 "B", bytes);
}
}
void AppendNumberTo(std::string* str, uint64_t num) {
char buf[30];
snprintf(buf, sizeof(buf), "%" PRIu64, num);
str->append(buf);
}
void AppendEscapedStringTo(std::string* str, const Slice& value) {
for (size_t i = 0; i < value.size(); i++) {
char c = value[i];
if (c >= ' ' && c <= '~') {
str->push_back(c);
} else {
char buf[10];
snprintf(buf, sizeof(buf), "\\x%02x",
static_cast<unsigned int>(c) & 0xff);
str->append(buf);
}
}
}
std::string NumberToString(uint64_t num) {
std::string r;
AppendNumberTo(&r, num);
return r;
}
std::string NumberToHumanString(int64_t num) {
char buf[16];
int64_t absnum = num < 0 ? -num : num;
if (absnum < 10000) {
snprintf(buf, sizeof(buf), "%" PRIi64, num);
} else if (absnum < 10000000) {
snprintf(buf, sizeof(buf), "%" PRIi64 "K", num / 1000);
} else if (absnum < 10000000000LL) {
snprintf(buf, sizeof(buf), "%" PRIi64 "M", num / 1000000);
} else {
snprintf(buf, sizeof(buf), "%" PRIi64 "G", num / 1000000000);
}
return std::string(buf);
}
std::string EscapeString(const Slice& value) {
std::string r;
AppendEscapedStringTo(&r, value);
return r;
}
bool ConsumeDecimalNumber(Slice* in, uint64_t* val) {
uint64_t v = 0;
int digits = 0;
while (!in->empty()) {
char c = (*in)[0];
if (c >= '0' && c <= '9') {
++digits;
const unsigned int delta = (c - '0');
static const uint64_t kMaxUint64 = ~static_cast<uint64_t>(0);
if (v > kMaxUint64/10 ||
(v == kMaxUint64/10 && delta > kMaxUint64%10)) {
// Overflow
return false;
}
v = (v * 10) + delta;
in->remove_prefix(1);
} else {
break;
}
}
*val = v;
return (digits > 0);
}
} // namespace rocksdb