Add an option to table_reader_bench to access the table from DB And Iterating non-existing prefix case.
Summary: This patch adds an option to table_reader_bench that queries run against DB level (which has one table). It is useful if user wants to see the extra costs DB level introduces. Test Plan: Run the benchmark with and without the new parameter Reviewers: haobo, dhruba, kailiu Reviewed By: kailiu CC: leveldb Differential Revision: https://reviews.facebook.net/D13863
This commit is contained in:
parent
3d56b0698c
commit
55baa3d955
@ -7,7 +7,10 @@
|
||||
|
||||
#include "rocksdb/db.h"
|
||||
#include "rocksdb/table.h"
|
||||
#include "rocksdb/slice_transform.h"
|
||||
#include "db/db_impl.h"
|
||||
#include "db/dbformat.h"
|
||||
#include "port/atomic_pointer.h"
|
||||
#include "table/block_based_table_factory.h"
|
||||
#include "util/histogram.h"
|
||||
#include "util/testharness.h"
|
||||
@ -16,10 +19,18 @@
|
||||
namespace rocksdb {
|
||||
// Make a key that i determines the first 4 characters and j determines the
|
||||
// last 4 characters.
|
||||
static std::string MakeKey(int i, int j) {
|
||||
static std::string MakeKey(int i, int j, bool through_db) {
|
||||
char buf[100];
|
||||
snprintf(buf, sizeof(buf), "%04d__key___%04d ", i, j);
|
||||
return std::string(buf);
|
||||
snprintf(buf, sizeof(buf), "%04d__key___%04d", i, j);
|
||||
if (through_db) {
|
||||
return std::string(buf);
|
||||
}
|
||||
// If we directly query table, which operates on internal keys
|
||||
// instead of user keys, we need to add 8 bytes of internal
|
||||
// information (row type etc) to user key to make an internal
|
||||
// key.
|
||||
InternalKey key(std::string(buf), 0, ValueType::kTypeValue);
|
||||
return key.Encode().ToString();
|
||||
}
|
||||
|
||||
static bool DummySaveValue(void* arg, const Slice& ikey, const Slice& v,
|
||||
@ -35,41 +46,67 @@ static bool DummySaveValue(void* arg, const Slice& ikey, const Slice& v,
|
||||
// times randomly.
|
||||
// If if_query_empty_keys = true, query numKey1 * numKey2 random empty keys.
|
||||
// Print out the total time.
|
||||
// If through_db=true, a full DB will be created and queries will be against
|
||||
// it. Otherwise, operations will be directly through table level.
|
||||
//
|
||||
// If for_terator=true, instead of just query one key each time, it queries
|
||||
// a range sharing the same prefix.
|
||||
void TableReaderBenchmark(Options& opts, EnvOptions& env_options,
|
||||
ReadOptions& read_options, TableFactory* tf,
|
||||
int num_keys1, int num_keys2, int num_iter,
|
||||
bool if_query_empty_keys, bool for_iterator) {
|
||||
ReadOptions& read_options, int num_keys1,
|
||||
int num_keys2, int num_iter, int prefix_len,
|
||||
bool if_query_empty_keys, bool for_iterator,
|
||||
bool through_db) {
|
||||
Slice prefix = Slice();
|
||||
|
||||
std::string file_name = test::TmpDir()
|
||||
+ "/rocksdb_table_reader_benchmark";
|
||||
std::string dbname = test::TmpDir() + "/rocksdb_table_reader_bench_db";
|
||||
ReadOptions ro;
|
||||
WriteOptions wo;
|
||||
unique_ptr<WritableFile> file;
|
||||
Env* env = Env::Default();
|
||||
env->NewWritableFile(file_name, &file, env_options);
|
||||
TableBuilder* tb = tf->GetTableBuilder(opts, file.get(),
|
||||
CompressionType::kNoCompression);
|
||||
|
||||
TableBuilder* tb = nullptr;
|
||||
DB* db = nullptr;
|
||||
Status s;
|
||||
if (!through_db) {
|
||||
env->NewWritableFile(file_name, &file, env_options);
|
||||
tb = opts.table_factory->GetTableBuilder(opts, file.get(),
|
||||
CompressionType::kNoCompression);
|
||||
} else {
|
||||
s = DB::Open(opts, dbname, &db);
|
||||
ASSERT_OK(s);
|
||||
ASSERT_TRUE(db != nullptr);
|
||||
}
|
||||
// Populate slightly more than 1M keys
|
||||
for (int i = 0; i < num_keys1; i++) {
|
||||
for (int j = 0; j < num_keys2; j++) {
|
||||
std::string key = MakeKey(i * 2, j);
|
||||
tb->Add(key, key);
|
||||
std::string key = MakeKey(i * 2, j, through_db);
|
||||
if (!through_db) {
|
||||
tb->Add(key, key);
|
||||
} else {
|
||||
db->Put(wo, key, key);
|
||||
}
|
||||
}
|
||||
}
|
||||
tb->Finish();
|
||||
file->Close();
|
||||
if (!through_db) {
|
||||
tb->Finish();
|
||||
file->Close();
|
||||
} else {
|
||||
db->Flush(FlushOptions());
|
||||
}
|
||||
|
||||
unique_ptr<TableReader> table_reader;
|
||||
unique_ptr<RandomAccessFile> raf;
|
||||
Status s = env->NewRandomAccessFile(file_name, &raf, env_options);
|
||||
uint64_t file_size;
|
||||
env->GetFileSize(file_name, &file_size);
|
||||
s = tf->GetTableReader(opts, env_options, std::move(raf), file_size,
|
||||
&table_reader);
|
||||
if (!through_db) {
|
||||
Status s = env->NewRandomAccessFile(file_name, &raf, env_options);
|
||||
uint64_t file_size;
|
||||
env->GetFileSize(file_name, &file_size);
|
||||
s = opts.table_factory->GetTableReader(opts, env_options, std::move(raf),
|
||||
file_size, &table_reader);
|
||||
}
|
||||
|
||||
Random rnd(301);
|
||||
std::string result;
|
||||
HistogramImpl hist;
|
||||
|
||||
void* arg = nullptr;
|
||||
@ -78,31 +115,57 @@ void TableReaderBenchmark(Options& opts, EnvOptions& env_options,
|
||||
for (int j = 0; j < num_keys2; j++) {
|
||||
int r1 = rnd.Uniform(num_keys1) * 2;
|
||||
int r2 = rnd.Uniform(num_keys2);
|
||||
if (if_query_empty_keys) {
|
||||
r1++;
|
||||
r2 = num_keys2 * 2 - r2;
|
||||
}
|
||||
|
||||
if (!for_iterator) {
|
||||
if (if_query_empty_keys) {
|
||||
r1++;
|
||||
r2 = num_keys2 * 2 - r2;
|
||||
}
|
||||
// Query one existing key;
|
||||
std::string key = MakeKey(r1, r2);
|
||||
std::string key = MakeKey(r1, r2, through_db);
|
||||
uint64_t start_micros = env->NowMicros();
|
||||
s = table_reader->Get(ro, key, arg, DummySaveValue, nullptr);
|
||||
port::MemoryBarrier();
|
||||
if (!through_db) {
|
||||
s = table_reader->Get(ro, key, arg, DummySaveValue, nullptr);
|
||||
} else {
|
||||
s = db->Get(ro, key, &result);
|
||||
}
|
||||
port::MemoryBarrier();
|
||||
hist.Add(env->NowMicros() - start_micros);
|
||||
} else {
|
||||
int r2_len = rnd.Uniform(num_keys2) + 1;
|
||||
if (r2_len + r2 > num_keys2) {
|
||||
r2_len = num_keys2 - r2;
|
||||
int r2_len;
|
||||
if (if_query_empty_keys) {
|
||||
r2_len = 0;
|
||||
} else {
|
||||
r2_len = rnd.Uniform(num_keys2) + 1;
|
||||
if (r2_len + r2 > num_keys2) {
|
||||
r2_len = num_keys2 - r2;
|
||||
}
|
||||
}
|
||||
std::string start_key = MakeKey(r1, r2, through_db);
|
||||
std::string end_key = MakeKey(r1, r2 + r2_len, through_db);
|
||||
if (prefix_len < 16) {
|
||||
prefix = Slice(start_key.data(), prefix_len);
|
||||
read_options.prefix = &prefix;
|
||||
}
|
||||
std::string start_key = MakeKey(r1, r2);
|
||||
std::string end_key = MakeKey(r1, r2 + r2_len);
|
||||
uint64_t total_time = 0;
|
||||
uint64_t start_micros = env->NowMicros();
|
||||
Iterator* iter = table_reader->NewIterator(read_options);
|
||||
port::MemoryBarrier();
|
||||
Iterator* iter;
|
||||
if (!through_db) {
|
||||
iter = table_reader->NewIterator(read_options);
|
||||
} else {
|
||||
iter = db->NewIterator(read_options);
|
||||
}
|
||||
int count = 0;
|
||||
for(iter->Seek(start_key); iter->Valid(); iter->Next()) {
|
||||
if (if_query_empty_keys) {
|
||||
break;
|
||||
}
|
||||
// verify key;
|
||||
port::MemoryBarrier();
|
||||
total_time += env->NowMicros() - start_micros;
|
||||
assert(Slice(MakeKey(r1, r2 + count)) == iter->key());
|
||||
assert(Slice(MakeKey(r1, r2 + count, through_db)) == iter->key());
|
||||
start_micros = env->NowMicros();
|
||||
if (++count >= r2_len) {
|
||||
break;
|
||||
@ -115,6 +178,7 @@ void TableReaderBenchmark(Options& opts, EnvOptions& env_options,
|
||||
assert(false);
|
||||
}
|
||||
delete iter;
|
||||
port::MemoryBarrier();
|
||||
total_time += env->NowMicros() - start_micros;
|
||||
hist.Add(total_time);
|
||||
}
|
||||
@ -131,10 +195,16 @@ void TableReaderBenchmark(Options& opts, EnvOptions& env_options,
|
||||
"==================================================="
|
||||
"===================================================="
|
||||
"\nHistogram (unit: microseconds): \n%s",
|
||||
tf->Name(), num_keys1, num_keys2,
|
||||
opts.table_factory->Name(), num_keys1, num_keys2,
|
||||
for_iterator? "iterator" : (if_query_empty_keys ? "empty" : "non_empty"),
|
||||
hist.ToString().c_str());
|
||||
env->DeleteFile(file_name);
|
||||
if (!through_db) {
|
||||
env->DeleteFile(file_name);
|
||||
} else {
|
||||
delete db;
|
||||
db = nullptr;
|
||||
DestroyDB(dbname, opts);
|
||||
}
|
||||
}
|
||||
} // namespace rocksdb
|
||||
|
||||
@ -143,21 +213,32 @@ DEFINE_bool(query_empty, false, "query non-existing keys instead of existing "
|
||||
DEFINE_int32(num_keys1, 4096, "number of distinguish prefix of keys");
|
||||
DEFINE_int32(num_keys2, 512, "number of distinguish keys for each prefix");
|
||||
DEFINE_int32(iter, 3, "query non-existing keys instead of existing ones");
|
||||
DEFINE_int32(prefix_len, 16, "Prefix length used for iterators and indexes");
|
||||
DEFINE_bool(iterator, false, "For test iterator");
|
||||
DEFINE_bool(through_db, false, "If enable, a DB instance will be created and "
|
||||
"the query will be against DB. Otherwise, will be directly against "
|
||||
"a table reader.");
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
google::SetUsageMessage(std::string("\nUSAGE:\n") + std::string(argv[0]) +
|
||||
" [OPTIONS]...");
|
||||
google::ParseCommandLineFlags(&argc, &argv, true);
|
||||
|
||||
rocksdb::TableFactory* tf;
|
||||
rocksdb::TableFactory* tf = new rocksdb::BlockBasedTableFactory();
|
||||
rocksdb::Options options;
|
||||
if (FLAGS_prefix_len < 16) {
|
||||
options.prefix_extractor = rocksdb::NewFixedPrefixTransform(
|
||||
FLAGS_prefix_len);
|
||||
}
|
||||
options.SetUpDefaultFlushBlockPolicyFactory();
|
||||
rocksdb::ReadOptions ro;
|
||||
rocksdb::EnvOptions env_options;
|
||||
tf = new rocksdb::BlockBasedTableFactory();
|
||||
TableReaderBenchmark(options, env_options, ro, tf, FLAGS_num_keys1,
|
||||
FLAGS_num_keys2, FLAGS_iter, FLAGS_query_empty,
|
||||
FLAGS_iterator);
|
||||
options.create_if_missing = true;
|
||||
options.table_factory =
|
||||
std::shared_ptr<rocksdb::TableFactory>(tf);
|
||||
TableReaderBenchmark(options, env_options, ro, FLAGS_num_keys1,
|
||||
FLAGS_num_keys2, FLAGS_iter, FLAGS_prefix_len,
|
||||
FLAGS_query_empty, FLAGS_iterator, FLAGS_through_db);
|
||||
delete tf;
|
||||
return 0;
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user