Add db_bench options of data block hash index (#4281)
Summary: Add `--data_block_index_type` and `--data_block_hash_table_util_ratio` option to `db_bench`. `--data_block_index_type` can be either of `binary` (default) or `binary_and_hash`; `--data_block_hash_table_util_ratio` will be a double. The default value is `0.75`. Pull Request resolved: https://github.com/facebook/rocksdb/pull/4281 Differential Revision: D9361476 Pulled By: fgwu fbshipit-source-id: dc53e01acef9db81b9eec5e8a96f3bc8ed718c10
This commit is contained in:
parent
889a0553c8
commit
9d646a6311
@ -112,7 +112,6 @@ struct BlockBasedTableOptions {
|
||||
// kDataBlockBinaryAndHash.
|
||||
double data_block_hash_table_util_ratio = 0.75;
|
||||
|
||||
|
||||
// This option is now deprecated. No matter what value it is set to,
|
||||
// it will behave as if hash_index_allow_collision=true.
|
||||
bool hash_index_allow_collision = true;
|
||||
|
@ -789,8 +789,8 @@ Block::Block(BlockContents&& contents, SequenceNumber _global_seqno,
|
||||
uint16_t map_offset;
|
||||
data_block_hash_index_.Initialize(
|
||||
contents.data.data(),
|
||||
static_cast<uint16_t>(
|
||||
contents.data.size() - sizeof(uint32_t)), /*chop off
|
||||
static_cast<uint16_t>(contents.data.size() -
|
||||
sizeof(uint32_t)), /*chop off
|
||||
NUM_RESTARTS*/
|
||||
&map_offset);
|
||||
|
||||
|
@ -33,8 +33,8 @@
|
||||
|
||||
#include "table/block_builder.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <assert.h>
|
||||
#include <algorithm>
|
||||
#include "db/dbformat.h"
|
||||
#include "rocksdb/comparator.h"
|
||||
#include "table/data_block_footer.h"
|
||||
@ -126,8 +126,7 @@ Slice BlockBuilder::Finish() {
|
||||
}
|
||||
|
||||
// footer is a packed format of data_block_index_type and num_restarts
|
||||
uint32_t block_footer = PackIndexTypeAndNumRestarts(
|
||||
index_type, num_restarts);
|
||||
uint32_t block_footer = PackIndexTypeAndNumRestarts(index_type, num_restarts);
|
||||
|
||||
PutFixed32(&buffer_, block_footer);
|
||||
finished_ = true;
|
||||
|
@ -38,7 +38,6 @@ uint32_t PackIndexTypeAndNumRestarts(
|
||||
return block_footer;
|
||||
}
|
||||
|
||||
|
||||
void UnPackIndexTypeAndNumRestarts(
|
||||
uint32_t block_footer,
|
||||
BlockBasedTableOptions::DataBlockIndexType* index_type,
|
||||
|
@ -17,7 +17,6 @@ uint32_t PackIndexTypeAndNumRestarts(
|
||||
BlockBasedTableOptions::DataBlockIndexType index_type,
|
||||
uint32_t num_restarts);
|
||||
|
||||
|
||||
void UnPackIndexTypeAndNumRestarts(
|
||||
uint32_t block_footer,
|
||||
BlockBasedTableOptions::DataBlockIndexType* index_type,
|
||||
|
@ -54,8 +54,8 @@ void DataBlockHashIndexBuilder::Finish(std::string& buffer) {
|
||||
}
|
||||
|
||||
for (uint8_t restart_index : buckets) {
|
||||
buffer.append(const_cast<const char*>(
|
||||
reinterpret_cast<char*>(&restart_index)),
|
||||
buffer.append(
|
||||
const_cast<const char*>(reinterpret_cast<char*>(&restart_index)),
|
||||
sizeof(restart_index));
|
||||
}
|
||||
|
||||
|
@ -9,10 +9,10 @@
|
||||
|
||||
#include "rocksdb/slice.h"
|
||||
#include "table/block.h"
|
||||
#include "table/block_based_table_reader.h"
|
||||
#include "table/block_builder.h"
|
||||
#include "table/data_block_hash_index.h"
|
||||
#include "table/get_context.h"
|
||||
#include "table/block_based_table_reader.h"
|
||||
#include "util/testharness.h"
|
||||
#include "util/testutil.h"
|
||||
|
||||
@ -95,7 +95,6 @@ TEST(DataBlockHashIndex, DataBlockHashTestSmall) {
|
||||
|
||||
buffer2 = buffer; // test for the correctness of relative offset
|
||||
|
||||
|
||||
Slice s(buffer2);
|
||||
DataBlockHashIndex index;
|
||||
uint16_t map_offset;
|
||||
@ -290,7 +289,6 @@ TEST(DataBlockHashIndex, BlockRestartIndexExceedMax) {
|
||||
BlockBasedTableOptions::kDataBlockBinaryAndHash);
|
||||
}
|
||||
|
||||
|
||||
builder.Reset();
|
||||
|
||||
// #restarts > 253. HashIndex is not used
|
||||
@ -404,8 +402,8 @@ TEST(DataBlockHashIndex, BlockTestSingleKey) {
|
||||
may_exist = iter->SeekForGet(seek_ikey.Encode().ToString());
|
||||
ASSERT_TRUE(may_exist);
|
||||
ASSERT_TRUE(iter->Valid());
|
||||
ASSERT_EQ(options.comparator->Compare(
|
||||
iter->key(), ikey.Encode().ToString()), 0);
|
||||
ASSERT_EQ(
|
||||
options.comparator->Compare(iter->key(), ikey.Encode().ToString()), 0);
|
||||
ASSERT_EQ(iter->value(), value);
|
||||
}
|
||||
|
||||
@ -419,8 +417,8 @@ TEST(DataBlockHashIndex, BlockTestSingleKey) {
|
||||
ASSERT_TRUE(iter->Valid());
|
||||
|
||||
// user key should match
|
||||
ASSERT_EQ(options.comparator->Compare(
|
||||
ExtractUserKey(iter->key()), ukey), 0);
|
||||
ASSERT_EQ(options.comparator->Compare(ExtractUserKey(iter->key()), ukey),
|
||||
0);
|
||||
|
||||
// seek_key seqno number should be greater than that of iter result
|
||||
ASSERT_GT(GetInternalKeySeqno(seek_ikey.Encode()),
|
||||
|
@ -3685,14 +3685,12 @@ TEST_P(BlockBasedTableTest, DataBlockHashIndex) {
|
||||
c.Finish(options, ioptions, moptions, table_options, internal_comparator,
|
||||
&keys, &kvmap);
|
||||
|
||||
|
||||
auto reader = c.GetTableReader();
|
||||
|
||||
std::unique_ptr<InternalIterator> seek_iter;
|
||||
seek_iter.reset(reader->NewIterator(ReadOptions(),
|
||||
moptions.prefix_extractor.get()));
|
||||
seek_iter.reset(
|
||||
reader->NewIterator(ReadOptions(), moptions.prefix_extractor.get()));
|
||||
for (int i = 0; i < 2; ++i) {
|
||||
|
||||
ReadOptions ro;
|
||||
// for every kv, we seek using two method: Get() and Seek()
|
||||
// Get() will use the SuffixIndexHash in Block. For non-existent key it
|
||||
|
@ -466,6 +466,16 @@ DEFINE_bool(enable_index_compression,
|
||||
DEFINE_bool(block_align, rocksdb::BlockBasedTableOptions().block_align,
|
||||
"Align data blocks on page size");
|
||||
|
||||
DEFINE_bool(use_data_block_hash_index, false,
|
||||
"if use kDataBlockBinaryAndHash "
|
||||
"instead of kDataBlockBinarySearch. "
|
||||
"This is valid if only we use BlockTable");
|
||||
|
||||
DEFINE_double(data_block_hash_table_util_ratio, 0.75,
|
||||
"util ratio for data block hash index table. "
|
||||
"This is only valid if use_data_block_hash_index is "
|
||||
"set to true");
|
||||
|
||||
DEFINE_int64(compressed_cache_size, -1,
|
||||
"Number of bytes to use as a cache of compressed data.");
|
||||
|
||||
@ -3265,6 +3275,15 @@ void VerifyDBFromDB(std::string& truth_db_name) {
|
||||
block_based_options.enable_index_compression =
|
||||
FLAGS_enable_index_compression;
|
||||
block_based_options.block_align = FLAGS_block_align;
|
||||
if (FLAGS_use_data_block_hash_index) {
|
||||
block_based_options.data_block_index_type =
|
||||
rocksdb::BlockBasedTableOptions::kDataBlockBinaryAndHash;
|
||||
} else {
|
||||
block_based_options.data_block_index_type =
|
||||
rocksdb::BlockBasedTableOptions::kDataBlockBinarySearch;
|
||||
}
|
||||
block_based_options.data_block_hash_table_util_ratio =
|
||||
FLAGS_data_block_hash_table_util_ratio;
|
||||
if (FLAGS_read_cache_path != "") {
|
||||
#ifndef ROCKSDB_LITE
|
||||
Status rc_status;
|
||||
|
Loading…
Reference in New Issue
Block a user