Add db_bench options of data block hash index (#4281)

Summary:
Add `--data_block_index_type` and `--data_block_hash_table_util_ratio` option to `db_bench`.

`--data_block_index_type` can be either of `binary` (default) or `binary_and_hash`;
`--data_block_hash_table_util_ratio` will be a double. The default value is `0.75`.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4281

Differential Revision: D9361476

Pulled By: fgwu

fbshipit-source-id: dc53e01acef9db81b9eec5e8a96f3bc8ed718c10
This commit is contained in:
Fenggang Wu 2018-08-16 18:29:13 -07:00 committed by Facebook Github Bot
parent 889a0553c8
commit 9d646a6311
10 changed files with 56 additions and 45 deletions

View File

@ -102,8 +102,8 @@ struct BlockBasedTableOptions {
// The index type that will be used for the data block.
enum DataBlockIndexType : char {
kDataBlockBinarySearch = 0, // traditional block type
kDataBlockBinaryAndHash = 1, // additional hash index
kDataBlockBinarySearch = 0, // traditional block type
kDataBlockBinaryAndHash = 1, // additional hash index
};
DataBlockIndexType data_block_index_type = kDataBlockBinarySearch;
@ -112,7 +112,6 @@ struct BlockBasedTableOptions {
// kDataBlockBinaryAndHash.
double data_block_hash_table_util_ratio = 0.75;
// This option is now deprecated. No matter what value it is set to,
// it will behave as if hash_index_allow_collision=true.
bool hash_index_allow_collision = true;

View File

@ -262,7 +262,7 @@ bool DataBlockIter::SeekForGetImpl(const Slice& target) {
// Even if the user_key is not found in the hash map, the caller still
// have to conntinue searching the next block. So we invalidate the
// iterator to tell the caller to go on.
current_ = restarts_; // Invalidate the iter
current_ = restarts_; // Invalidate the iter
return true;
}
@ -781,7 +781,7 @@ Block::Block(BlockContents&& contents, SequenceNumber _global_seqno,
break;
case BlockBasedTableOptions::kDataBlockBinaryAndHash:
if (size_ < sizeof(uint32_t) /* block footer */ +
sizeof(uint16_t) /* NUM_BUCK */) {
sizeof(uint16_t) /* NUM_BUCK */) {
size_ = 0;
break;
}
@ -789,9 +789,9 @@ Block::Block(BlockContents&& contents, SequenceNumber _global_seqno,
uint16_t map_offset;
data_block_hash_index_.Initialize(
contents.data.data(),
static_cast<uint16_t>(
contents.data.size() - sizeof(uint32_t)), /*chop off
NUM_RESTARTS*/
static_cast<uint16_t>(contents.data.size() -
sizeof(uint32_t)), /*chop off
NUM_RESTARTS*/
&map_offset);
restart_offset_ = map_offset - num_restarts_ * sizeof(uint32_t);

View File

@ -260,7 +260,7 @@ Status BlockBasedTableFactory::SanitizeOptions(
"Block alignment requested but block size is not a power of 2");
}
if (table_options_.data_block_index_type ==
BlockBasedTableOptions::kDataBlockBinaryAndHash &&
BlockBasedTableOptions::kDataBlockBinaryAndHash &&
table_options_.data_block_hash_table_util_ratio <= 0) {
return Status::InvalidArgument(
"data_block_hash_table_util_ratio should be greater than 0 when "

View File

@ -33,8 +33,8 @@
#include "table/block_builder.h"
#include <algorithm>
#include <assert.h>
#include <algorithm>
#include "db/dbformat.h"
#include "rocksdb/comparator.h"
#include "table/data_block_footer.h"
@ -118,7 +118,7 @@ Slice BlockBuilder::Finish() {
uint32_t num_restarts = static_cast<uint32_t>(restarts_.size());
BlockBasedTableOptions::DataBlockIndexType index_type =
BlockBasedTableOptions::kDataBlockBinarySearch;
BlockBasedTableOptions::kDataBlockBinarySearch;
if (data_block_hash_index_builder_.Valid() &&
CurrentSizeEstimate() <= kMaxBlockSizeSupportedByHashIndex) {
data_block_hash_index_builder_.Finish(buffer_);
@ -126,8 +126,7 @@ Slice BlockBuilder::Finish() {
}
// footer is a packed format of data_block_index_type and num_restarts
uint32_t block_footer = PackIndexTypeAndNumRestarts(
index_type, num_restarts);
uint32_t block_footer = PackIndexTypeAndNumRestarts(index_type, num_restarts);
PutFixed32(&buffer_, block_footer);
finished_ = true;

View File

@ -32,13 +32,12 @@ uint32_t PackIndexTypeAndNumRestarts(
if (index_type == BlockBasedTableOptions::kDataBlockBinaryAndHash) {
block_footer |= 1u << kDataBlockIndexTypeBitShift;
} else if (index_type != BlockBasedTableOptions::kDataBlockBinarySearch) {
assert(0);
assert(0);
}
return block_footer;
}
void UnPackIndexTypeAndNumRestarts(
uint32_t block_footer,
BlockBasedTableOptions::DataBlockIndexType* index_type,
@ -57,4 +56,4 @@ void UnPackIndexTypeAndNumRestarts(
}
}
} // namespace rocksdb
} // namespace rocksdb

View File

@ -17,10 +17,9 @@ uint32_t PackIndexTypeAndNumRestarts(
BlockBasedTableOptions::DataBlockIndexType index_type,
uint32_t num_restarts);
void UnPackIndexTypeAndNumRestarts(
uint32_t block_footer,
BlockBasedTableOptions::DataBlockIndexType* index_type,
uint32_t* num_restarts);
} // namespace rocksdb
} // namespace rocksdb

View File

@ -54,9 +54,9 @@ void DataBlockHashIndexBuilder::Finish(std::string& buffer) {
}
for (uint8_t restart_index : buckets) {
buffer.append(const_cast<const char*>(
reinterpret_cast<char*>(&restart_index)),
sizeof(restart_index));
buffer.append(
const_cast<const char*>(reinterpret_cast<char*>(&restart_index)),
sizeof(restart_index));
}
// write NUM_BUCK
@ -81,7 +81,7 @@ void DataBlockHashIndex::Initialize(const char* data, uint16_t size,
}
uint8_t DataBlockHashIndex::Lookup(const char* data, uint32_t map_offset,
const Slice& key) const {
const Slice& key) const {
uint32_t hash_value = GetSliceHash(key);
uint16_t idx = static_cast<uint16_t>(hash_value % num_buckets_);
const char* bucket_table = data + map_offset;

View File

@ -9,10 +9,10 @@
#include "rocksdb/slice.h"
#include "table/block.h"
#include "table/block_based_table_reader.h"
#include "table/block_builder.h"
#include "table/data_block_hash_index.h"
#include "table/get_context.h"
#include "table/block_based_table_reader.h"
#include "util/testharness.h"
#include "util/testutil.h"
@ -40,9 +40,9 @@ static std::string RandomString(Random* rnd, int len) {
return r;
}
std::string GenerateKey(int primary_key, int secondary_key, int padding_size,
Random *rnd) {
Random* rnd) {
char buf[50];
char *p = &buf[0];
char* p = &buf[0];
snprintf(buf, sizeof(buf), "%6d%4d", primary_key, secondary_key);
std::string k(p);
if (padding_size) {
@ -55,8 +55,8 @@ std::string GenerateKey(int primary_key, int secondary_key, int padding_size,
// Generate random key value pairs.
// The generated key will be sorted. You can tune the parameters to generated
// different kinds of test key/value pairs for different scenario.
void GenerateRandomKVs(std::vector<std::string> *keys,
std::vector<std::string> *values, const int from,
void GenerateRandomKVs(std::vector<std::string>* keys,
std::vector<std::string>* values, const int from,
const int len, const int step = 1,
const int padding_size = 0,
const int keys_share_prefix = 1) {
@ -93,8 +93,7 @@ TEST(DataBlockHashIndex, DataBlockHashTestSmall) {
ASSERT_EQ(buffer.size(), estimated_size);
buffer2 = buffer; // test for the correctness of relative offset
buffer2 = buffer; // test for the correctness of relative offset
Slice s(buffer2);
DataBlockHashIndex index;
@ -290,7 +289,6 @@ TEST(DataBlockHashIndex, BlockRestartIndexExceedMax) {
BlockBasedTableOptions::kDataBlockBinaryAndHash);
}
builder.Reset();
// #restarts > 253. HashIndex is not used
@ -343,7 +341,7 @@ TEST(DataBlockHashIndex, BlockSizeExceedMax) {
Block reader(std::move(contents), kDisableGlobalSequenceNumber);
ASSERT_EQ(reader.IndexType(),
BlockBasedTableOptions::kDataBlockBinaryAndHash);
BlockBasedTableOptions::kDataBlockBinaryAndHash);
}
builder.Reset();
@ -404,8 +402,8 @@ TEST(DataBlockHashIndex, BlockTestSingleKey) {
may_exist = iter->SeekForGet(seek_ikey.Encode().ToString());
ASSERT_TRUE(may_exist);
ASSERT_TRUE(iter->Valid());
ASSERT_EQ(options.comparator->Compare(
iter->key(), ikey.Encode().ToString()), 0);
ASSERT_EQ(
options.comparator->Compare(iter->key(), ikey.Encode().ToString()), 0);
ASSERT_EQ(iter->value(), value);
}
@ -419,8 +417,8 @@ TEST(DataBlockHashIndex, BlockTestSingleKey) {
ASSERT_TRUE(iter->Valid());
// user key should match
ASSERT_EQ(options.comparator->Compare(
ExtractUserKey(iter->key()), ukey), 0);
ASSERT_EQ(options.comparator->Compare(ExtractUserKey(iter->key()), ukey),
0);
// seek_key seqno number should be greater than that of iter result
ASSERT_GT(GetInternalKeySeqno(seek_ikey.Encode()),
@ -601,7 +599,7 @@ void TestBoundary(InternalKey& ik1, std::string& v1, InternalKey& ik2,
ReadOptions ro;
ASSERT_OK(table_reader->Get(ro, seek_ikey.Encode().ToString(), &get_context,
moptions.prefix_extractor.get()));
moptions.prefix_extractor.get()));
}
TEST(DataBlockHashIndex, BlockBoundary) {

View File

@ -3660,7 +3660,7 @@ TEST_P(BlockBasedTableTest, DataBlockHashIndex) {
BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
table_options.data_block_index_type =
BlockBasedTableOptions::kDataBlockBinaryAndHash;
BlockBasedTableOptions::kDataBlockBinaryAndHash;
Options options;
options.comparator = BytewiseComparator();
@ -3685,14 +3685,12 @@ TEST_P(BlockBasedTableTest, DataBlockHashIndex) {
c.Finish(options, ioptions, moptions, table_options, internal_comparator,
&keys, &kvmap);
auto reader = c.GetTableReader();
std::unique_ptr<InternalIterator> seek_iter;
seek_iter.reset(reader->NewIterator(ReadOptions(),
moptions.prefix_extractor.get()));
seek_iter.reset(
reader->NewIterator(ReadOptions(), moptions.prefix_extractor.get()));
for (int i = 0; i < 2; ++i) {
ReadOptions ro;
// for every kv, we seek using two method: Get() and Seek()
// Get() will use the SuffixIndexHash in Block. For non-existent key it
@ -3727,17 +3725,17 @@ TEST_P(BlockBasedTableTest, DataBlockHashIndex) {
// Search for non-existent keys
for (auto& kv : kvmap) {
std::string user_key = ExtractUserKey(kv.first).ToString();
user_key.back() = '0'; // make it non-existent key
user_key.back() = '0'; // make it non-existent key
InternalKey internal_key(user_key, 0, kTypeValue);
std::string encoded_key = internal_key.Encode().ToString();
if (i == 0) { // Search using Seek()
if (i == 0) { // Search using Seek()
seek_iter->Seek(encoded_key);
ASSERT_OK(seek_iter->status());
if (seek_iter->Valid()){
if (seek_iter->Valid()) {
ASSERT_TRUE(BytewiseComparator()->Compare(
user_key, ExtractUserKey(seek_iter->key())) < 0);
}
} else { // Search using Get()
} else { // Search using Get()
PinnableSlice value;
GetContext get_context(options.comparator, nullptr, nullptr, nullptr,
GetContext::kNotFound, user_key, &value, nullptr,

View File

@ -466,6 +466,16 @@ DEFINE_bool(enable_index_compression,
DEFINE_bool(block_align, rocksdb::BlockBasedTableOptions().block_align,
"Align data blocks on page size");
DEFINE_bool(use_data_block_hash_index, false,
"if use kDataBlockBinaryAndHash "
"instead of kDataBlockBinarySearch. "
"This is valid if only we use BlockTable");
DEFINE_double(data_block_hash_table_util_ratio, 0.75,
"util ratio for data block hash index table. "
"This is only valid if use_data_block_hash_index is "
"set to true");
DEFINE_int64(compressed_cache_size, -1,
"Number of bytes to use as a cache of compressed data.");
@ -3265,6 +3275,15 @@ void VerifyDBFromDB(std::string& truth_db_name) {
block_based_options.enable_index_compression =
FLAGS_enable_index_compression;
block_based_options.block_align = FLAGS_block_align;
if (FLAGS_use_data_block_hash_index) {
block_based_options.data_block_index_type =
rocksdb::BlockBasedTableOptions::kDataBlockBinaryAndHash;
} else {
block_based_options.data_block_index_type =
rocksdb::BlockBasedTableOptions::kDataBlockBinarySearch;
}
block_based_options.data_block_hash_table_util_ratio =
FLAGS_data_block_hash_table_util_ratio;
if (FLAGS_read_cache_path != "") {
#ifndef ROCKSDB_LITE
Status rc_status;