Add PlainTableOptions
Summary: Since we have a lot of options for PlainTable, add a struct PlainTableOptions to manage them Test Plan: make all check Reviewers: sdong Reviewed By: sdong Subscribers: leveldb Differential Revision: https://reviews.facebook.net/D20175
This commit is contained in:
parent
052ddbe0e2
commit
92d73cbe78
@ -6,6 +6,7 @@
|
|||||||
|
|
||||||
### Public API changes
|
### Public API changes
|
||||||
* DBOptions.db_paths now is a vector of a DBPath structure which indicates both of path and target size
|
* DBOptions.db_paths now is a vector of a DBPath structure which indicates both of path and target size
|
||||||
|
* NewPlainTableFactory instead of bunch of parameters now accepts PlainTableOptions, which is defined in include/rocksdb/table.h
|
||||||
|
|
||||||
|
|
||||||
## 3.3.0 (7/10/2014)
|
## 3.3.0 (7/10/2014)
|
||||||
|
10
db/c.cc
10
db/c.cc
@ -1428,9 +1428,13 @@ void rocksdb_options_set_plain_table_factory(
|
|||||||
double hash_table_ratio, size_t index_sparseness) {
|
double hash_table_ratio, size_t index_sparseness) {
|
||||||
static rocksdb::TableFactory* factory = 0;
|
static rocksdb::TableFactory* factory = 0;
|
||||||
if (!factory) {
|
if (!factory) {
|
||||||
factory = rocksdb::NewPlainTableFactory(
|
rocksdb::PlainTableOptions options;
|
||||||
user_key_len, bloom_bits_per_key,
|
options.user_key_len = user_key_len;
|
||||||
hash_table_ratio, index_sparseness);
|
options.bloom_bits_per_key = bloom_bits_per_key;
|
||||||
|
options.hash_table_ratio = hash_table_ratio;
|
||||||
|
options.index_sparseness = index_sparseness;
|
||||||
|
|
||||||
|
factory = rocksdb::NewPlainTableFactory(options);
|
||||||
}
|
}
|
||||||
opt->rep.table_factory.reset(factory);
|
opt->rep.table_factory.reset(factory);
|
||||||
}
|
}
|
||||||
|
@ -1688,8 +1688,13 @@ class Benchmark {
|
|||||||
if (bloom_bits_per_key < 0) {
|
if (bloom_bits_per_key < 0) {
|
||||||
bloom_bits_per_key = 0;
|
bloom_bits_per_key = 0;
|
||||||
}
|
}
|
||||||
options.table_factory.reset(
|
|
||||||
NewPlainTableFactory(FLAGS_key_size, bloom_bits_per_key, 0.75));
|
PlainTableOptions plain_table_options;
|
||||||
|
plain_table_options.user_key_len = FLAGS_key_size;
|
||||||
|
plain_table_options.bloom_bits_per_key = bloom_bits_per_key;
|
||||||
|
plain_table_options.hash_table_ratio = 0.75;
|
||||||
|
options.table_factory = std::shared_ptr<TableFactory>(
|
||||||
|
NewPlainTableFactory(plain_table_options));
|
||||||
} else {
|
} else {
|
||||||
BlockBasedTableOptions block_based_options;
|
BlockBasedTableOptions block_based_options;
|
||||||
if (FLAGS_use_hash_search) {
|
if (FLAGS_use_hash_search) {
|
||||||
|
@ -61,7 +61,18 @@ class PlainTableDBTest {
|
|||||||
// Return the current option configuration.
|
// Return the current option configuration.
|
||||||
Options CurrentOptions() {
|
Options CurrentOptions() {
|
||||||
Options options;
|
Options options;
|
||||||
options.table_factory.reset(NewPlainTableFactory(0, 2, 0.8, 3, 0, kPrefix));
|
|
||||||
|
PlainTableOptions plain_table_options;
|
||||||
|
plain_table_options.user_key_len = 0;
|
||||||
|
plain_table_options.bloom_bits_per_key = 2;
|
||||||
|
plain_table_options.hash_table_ratio = 0.8;
|
||||||
|
plain_table_options.index_sparseness = 3;
|
||||||
|
plain_table_options.huge_page_tlb_size = 0;
|
||||||
|
plain_table_options.encoding_type = kPrefix;
|
||||||
|
plain_table_options.full_scan_mode = false;
|
||||||
|
|
||||||
|
options.table_factory.reset(NewPlainTableFactory(plain_table_options));
|
||||||
|
|
||||||
options.memtable_factory.reset(NewHashLinkListRepFactory(4, 0, 3, true, 3));
|
options.memtable_factory.reset(NewHashLinkListRepFactory(4, 0, 3, true, 3));
|
||||||
options.prefix_extractor.reset(NewFixedPrefixTransform(8));
|
options.prefix_extractor.reset(NewFixedPrefixTransform(8));
|
||||||
options.allow_mmap_reads = true;
|
options.allow_mmap_reads = true;
|
||||||
@ -212,16 +223,11 @@ extern const uint64_t kPlainTableMagicNumber;
|
|||||||
class TestPlainTableFactory : public PlainTableFactory {
|
class TestPlainTableFactory : public PlainTableFactory {
|
||||||
public:
|
public:
|
||||||
explicit TestPlainTableFactory(bool* expect_bloom_not_match,
|
explicit TestPlainTableFactory(bool* expect_bloom_not_match,
|
||||||
uint32_t user_key_len, int bloom_bits_per_key,
|
const PlainTableOptions& options)
|
||||||
double hash_table_ratio,
|
: PlainTableFactory(options),
|
||||||
size_t index_sparseness,
|
bloom_bits_per_key_(options.bloom_bits_per_key),
|
||||||
size_t huge_page_tlb_size,
|
hash_table_ratio_(options.hash_table_ratio),
|
||||||
EncodingType encoding_type)
|
index_sparseness_(options.index_sparseness),
|
||||||
: PlainTableFactory(user_key_len, bloom_bits_per_key, hash_table_ratio,
|
|
||||||
index_sparseness, huge_page_tlb_size, encoding_type),
|
|
||||||
bloom_bits_per_key_(bloom_bits_per_key),
|
|
||||||
hash_table_ratio_(hash_table_ratio),
|
|
||||||
index_sparseness_(index_sparseness),
|
|
||||||
expect_bloom_not_match_(expect_bloom_not_match) {}
|
expect_bloom_not_match_(expect_bloom_not_match) {}
|
||||||
|
|
||||||
Status NewTableReader(const Options& options, const EnvOptions& soptions,
|
Status NewTableReader(const Options& options, const EnvOptions& soptions,
|
||||||
@ -268,11 +274,30 @@ TEST(PlainTableDBTest, Flush) {
|
|||||||
// Test index interval for the same prefix to be 1, 2 and 4
|
// Test index interval for the same prefix to be 1, 2 and 4
|
||||||
if (total_order) {
|
if (total_order) {
|
||||||
options.prefix_extractor.reset();
|
options.prefix_extractor.reset();
|
||||||
options.table_factory.reset(NewPlainTableFactory(
|
|
||||||
0, bloom_bits, 0, 2, huge_page_tlb_size, encoding_type));
|
PlainTableOptions plain_table_options;
|
||||||
|
plain_table_options.user_key_len = 0;
|
||||||
|
plain_table_options.bloom_bits_per_key = bloom_bits;
|
||||||
|
plain_table_options.hash_table_ratio = 0;
|
||||||
|
plain_table_options.index_sparseness = 2;
|
||||||
|
plain_table_options.huge_page_tlb_size = huge_page_tlb_size;
|
||||||
|
plain_table_options.encoding_type = encoding_type;
|
||||||
|
plain_table_options.full_scan_mode = false;
|
||||||
|
|
||||||
|
options.table_factory.reset(
|
||||||
|
NewPlainTableFactory(plain_table_options));
|
||||||
} else {
|
} else {
|
||||||
options.table_factory.reset(NewPlainTableFactory(
|
PlainTableOptions plain_table_options;
|
||||||
0, bloom_bits, 0.75, 16, huge_page_tlb_size, encoding_type));
|
plain_table_options.user_key_len = 0;
|
||||||
|
plain_table_options.bloom_bits_per_key = bloom_bits;
|
||||||
|
plain_table_options.hash_table_ratio = 0.75;
|
||||||
|
plain_table_options.index_sparseness = 16;
|
||||||
|
plain_table_options.huge_page_tlb_size = huge_page_tlb_size;
|
||||||
|
plain_table_options.encoding_type = encoding_type;
|
||||||
|
plain_table_options.full_scan_mode = false;
|
||||||
|
|
||||||
|
options.table_factory.reset(
|
||||||
|
NewPlainTableFactory(plain_table_options));
|
||||||
}
|
}
|
||||||
DestroyAndReopen(&options);
|
DestroyAndReopen(&options);
|
||||||
|
|
||||||
@ -315,13 +340,27 @@ TEST(PlainTableDBTest, Flush2) {
|
|||||||
// Test index interval for the same prefix to be 1, 2 and 4
|
// Test index interval for the same prefix to be 1, 2 and 4
|
||||||
if (total_order) {
|
if (total_order) {
|
||||||
options.prefix_extractor = nullptr;
|
options.prefix_extractor = nullptr;
|
||||||
|
PlainTableOptions plain_table_options;
|
||||||
|
plain_table_options.user_key_len = 0;
|
||||||
|
plain_table_options.bloom_bits_per_key = bloom_bits;
|
||||||
|
plain_table_options.hash_table_ratio = 0;
|
||||||
|
plain_table_options.index_sparseness = 2;
|
||||||
|
plain_table_options.huge_page_tlb_size = huge_page_tlb_size;
|
||||||
|
plain_table_options.encoding_type = encoding_type;
|
||||||
|
|
||||||
options.table_factory.reset(new TestPlainTableFactory(
|
options.table_factory.reset(new TestPlainTableFactory(
|
||||||
&expect_bloom_not_match, 0, bloom_bits, 0, 2, huge_page_tlb_size,
|
&expect_bloom_not_match, plain_table_options));
|
||||||
encoding_type));
|
|
||||||
} else {
|
} else {
|
||||||
|
PlainTableOptions plain_table_options;
|
||||||
|
plain_table_options.user_key_len = 0;
|
||||||
|
plain_table_options.bloom_bits_per_key = bloom_bits;
|
||||||
|
plain_table_options.hash_table_ratio = 0.75;
|
||||||
|
plain_table_options.index_sparseness = 16;
|
||||||
|
plain_table_options.huge_page_tlb_size = huge_page_tlb_size;
|
||||||
|
plain_table_options.encoding_type = encoding_type;
|
||||||
|
|
||||||
options.table_factory.reset(new TestPlainTableFactory(
|
options.table_factory.reset(new TestPlainTableFactory(
|
||||||
&expect_bloom_not_match, 0, bloom_bits, 0.75, 16,
|
&expect_bloom_not_match, plain_table_options));
|
||||||
huge_page_tlb_size, encoding_type));
|
|
||||||
}
|
}
|
||||||
DestroyAndReopen(&options);
|
DestroyAndReopen(&options);
|
||||||
ASSERT_OK(Put("0000000000000bar", "b"));
|
ASSERT_OK(Put("0000000000000bar", "b"));
|
||||||
@ -380,13 +419,28 @@ TEST(PlainTableDBTest, Iterator) {
|
|||||||
// Test index interval for the same prefix to be 1, 2 and 4
|
// Test index interval for the same prefix to be 1, 2 and 4
|
||||||
if (total_order) {
|
if (total_order) {
|
||||||
options.prefix_extractor = nullptr;
|
options.prefix_extractor = nullptr;
|
||||||
|
|
||||||
|
PlainTableOptions plain_table_options;
|
||||||
|
plain_table_options.user_key_len = 16;
|
||||||
|
plain_table_options.bloom_bits_per_key = bloom_bits;
|
||||||
|
plain_table_options.hash_table_ratio = 0;
|
||||||
|
plain_table_options.index_sparseness = 2;
|
||||||
|
plain_table_options.huge_page_tlb_size = huge_page_tlb_size;
|
||||||
|
plain_table_options.encoding_type = encoding_type;
|
||||||
|
|
||||||
options.table_factory.reset(new TestPlainTableFactory(
|
options.table_factory.reset(new TestPlainTableFactory(
|
||||||
&expect_bloom_not_match, 16, bloom_bits, 0, 2, huge_page_tlb_size,
|
&expect_bloom_not_match, plain_table_options));
|
||||||
encoding_type));
|
|
||||||
} else {
|
} else {
|
||||||
|
PlainTableOptions plain_table_options;
|
||||||
|
plain_table_options.user_key_len = 16;
|
||||||
|
plain_table_options.bloom_bits_per_key = bloom_bits;
|
||||||
|
plain_table_options.hash_table_ratio = 0.75;
|
||||||
|
plain_table_options.index_sparseness = 16;
|
||||||
|
plain_table_options.huge_page_tlb_size = huge_page_tlb_size;
|
||||||
|
plain_table_options.encoding_type = encoding_type;
|
||||||
|
|
||||||
options.table_factory.reset(new TestPlainTableFactory(
|
options.table_factory.reset(new TestPlainTableFactory(
|
||||||
&expect_bloom_not_match, 16, bloom_bits, 0.75, 16,
|
&expect_bloom_not_match, plain_table_options));
|
||||||
huge_page_tlb_size, encoding_type));
|
|
||||||
}
|
}
|
||||||
DestroyAndReopen(&options);
|
DestroyAndReopen(&options);
|
||||||
|
|
||||||
@ -485,7 +539,13 @@ std::string MakeLongKey(size_t length, char c) {
|
|||||||
|
|
||||||
TEST(PlainTableDBTest, IteratorLargeKeys) {
|
TEST(PlainTableDBTest, IteratorLargeKeys) {
|
||||||
Options options = CurrentOptions();
|
Options options = CurrentOptions();
|
||||||
options.table_factory.reset(NewPlainTableFactory(0, 0, 0));
|
|
||||||
|
PlainTableOptions plain_table_options;
|
||||||
|
plain_table_options.user_key_len = 0;
|
||||||
|
plain_table_options.bloom_bits_per_key = 0;
|
||||||
|
plain_table_options.hash_table_ratio = 0;
|
||||||
|
|
||||||
|
options.table_factory.reset(NewPlainTableFactory(plain_table_options));
|
||||||
options.create_if_missing = true;
|
options.create_if_missing = true;
|
||||||
options.prefix_extractor.reset();
|
options.prefix_extractor.reset();
|
||||||
DestroyAndReopen(&options);
|
DestroyAndReopen(&options);
|
||||||
@ -529,7 +589,16 @@ std::string MakeLongKeyWithPrefix(size_t length, char c) {
|
|||||||
|
|
||||||
TEST(PlainTableDBTest, IteratorLargeKeysWithPrefix) {
|
TEST(PlainTableDBTest, IteratorLargeKeysWithPrefix) {
|
||||||
Options options = CurrentOptions();
|
Options options = CurrentOptions();
|
||||||
options.table_factory.reset(NewPlainTableFactory(16, 0, 0.8, 3, 0, kPrefix));
|
|
||||||
|
PlainTableOptions plain_table_options;
|
||||||
|
plain_table_options.user_key_len = 16;
|
||||||
|
plain_table_options.bloom_bits_per_key = 0;
|
||||||
|
plain_table_options.hash_table_ratio = 0.8;
|
||||||
|
plain_table_options.index_sparseness = 3;
|
||||||
|
plain_table_options.huge_page_tlb_size = 0;
|
||||||
|
plain_table_options.encoding_type = kPrefix;
|
||||||
|
|
||||||
|
options.table_factory.reset(NewPlainTableFactory(plain_table_options));
|
||||||
options.create_if_missing = true;
|
options.create_if_missing = true;
|
||||||
DestroyAndReopen(&options);
|
DestroyAndReopen(&options);
|
||||||
|
|
||||||
@ -665,8 +734,16 @@ TEST(PlainTableDBTest, HashBucketConflict) {
|
|||||||
options.create_if_missing = true;
|
options.create_if_missing = true;
|
||||||
// Set only one bucket to force bucket conflict.
|
// Set only one bucket to force bucket conflict.
|
||||||
// Test index interval for the same prefix to be 1, 2 and 4
|
// Test index interval for the same prefix to be 1, 2 and 4
|
||||||
options.table_factory.reset(
|
|
||||||
NewPlainTableFactory(16, 0, 0, 2 ^ i, huge_page_tlb_size));
|
PlainTableOptions plain_table_options;
|
||||||
|
plain_table_options.user_key_len = 16;
|
||||||
|
plain_table_options.bloom_bits_per_key = 0;
|
||||||
|
plain_table_options.hash_table_ratio = 0;
|
||||||
|
plain_table_options.index_sparseness = 2 ^ i;
|
||||||
|
plain_table_options.huge_page_tlb_size = huge_page_tlb_size;
|
||||||
|
|
||||||
|
options.table_factory.reset(NewPlainTableFactory(plain_table_options));
|
||||||
|
|
||||||
DestroyAndReopen(&options);
|
DestroyAndReopen(&options);
|
||||||
ASSERT_OK(Put("5000000000000fo0", "v1"));
|
ASSERT_OK(Put("5000000000000fo0", "v1"));
|
||||||
ASSERT_OK(Put("5000000000000fo1", "v2"));
|
ASSERT_OK(Put("5000000000000fo1", "v2"));
|
||||||
@ -752,8 +829,15 @@ TEST(PlainTableDBTest, HashBucketConflictReverseSuffixComparator) {
|
|||||||
options.comparator = ∁
|
options.comparator = ∁
|
||||||
// Set only one bucket to force bucket conflict.
|
// Set only one bucket to force bucket conflict.
|
||||||
// Test index interval for the same prefix to be 1, 2 and 4
|
// Test index interval for the same prefix to be 1, 2 and 4
|
||||||
options.table_factory.reset(
|
|
||||||
NewPlainTableFactory(16, 0, 0, 2 ^ i, huge_page_tlb_size));
|
PlainTableOptions plain_table_options;
|
||||||
|
plain_table_options.user_key_len = 16;
|
||||||
|
plain_table_options.bloom_bits_per_key = 0;
|
||||||
|
plain_table_options.hash_table_ratio = 0;
|
||||||
|
plain_table_options.index_sparseness = 2 ^ i;
|
||||||
|
plain_table_options.huge_page_tlb_size = huge_page_tlb_size;
|
||||||
|
|
||||||
|
options.table_factory.reset(NewPlainTableFactory(plain_table_options));
|
||||||
DestroyAndReopen(&options);
|
DestroyAndReopen(&options);
|
||||||
ASSERT_OK(Put("5000000000000fo0", "v1"));
|
ASSERT_OK(Put("5000000000000fo0", "v1"));
|
||||||
ASSERT_OK(Put("5000000000000fo1", "v2"));
|
ASSERT_OK(Put("5000000000000fo1", "v2"));
|
||||||
@ -833,7 +917,13 @@ TEST(PlainTableDBTest, NonExistingKeyToNonEmptyBucket) {
|
|||||||
options.create_if_missing = true;
|
options.create_if_missing = true;
|
||||||
// Set only one bucket to force bucket conflict.
|
// Set only one bucket to force bucket conflict.
|
||||||
// Test index interval for the same prefix to be 1, 2 and 4
|
// Test index interval for the same prefix to be 1, 2 and 4
|
||||||
options.table_factory.reset(NewPlainTableFactory(16, 0, 0, 5));
|
PlainTableOptions plain_table_options;
|
||||||
|
plain_table_options.user_key_len = 16;
|
||||||
|
plain_table_options.bloom_bits_per_key = 0;
|
||||||
|
plain_table_options.hash_table_ratio = 0;
|
||||||
|
plain_table_options.index_sparseness = 5;
|
||||||
|
|
||||||
|
options.table_factory.reset(NewPlainTableFactory(plain_table_options));
|
||||||
DestroyAndReopen(&options);
|
DestroyAndReopen(&options);
|
||||||
ASSERT_OK(Put("5000000000000fo0", "v1"));
|
ASSERT_OK(Put("5000000000000fo0", "v1"));
|
||||||
ASSERT_OK(Put("5000000000000fo1", "v2"));
|
ASSERT_OK(Put("5000000000000fo1", "v2"));
|
||||||
|
@ -213,7 +213,14 @@ TEST(TablePropertiesTest, CustomizedTablePropertiesCollector) {
|
|||||||
Options options;
|
Options options;
|
||||||
options.table_properties_collector_factories.emplace_back(
|
options.table_properties_collector_factories.emplace_back(
|
||||||
new RegularKeysStartWithAFactory());
|
new RegularKeysStartWithAFactory());
|
||||||
options.table_factory = std::make_shared<PlainTableFactory>(8, 8, 0);
|
|
||||||
|
PlainTableOptions plain_table_options;
|
||||||
|
plain_table_options.user_key_len = 8;
|
||||||
|
plain_table_options.bloom_bits_per_key = 8;
|
||||||
|
plain_table_options.hash_table_ratio = 0;
|
||||||
|
|
||||||
|
options.table_factory =
|
||||||
|
std::make_shared<PlainTableFactory>(plain_table_options);
|
||||||
test::PlainInternalKeyComparator ikc(options.comparator);
|
test::PlainInternalKeyComparator ikc(options.comparator);
|
||||||
TestCustomizedTablePropertiesCollector(kPlainTableMagicNumber, true, options,
|
TestCustomizedTablePropertiesCollector(kPlainTableMagicNumber, true, options,
|
||||||
ikc);
|
ikc);
|
||||||
@ -299,11 +306,15 @@ TEST(TablePropertiesTest, InternalKeyPropertiesCollector) {
|
|||||||
true /* not sanitize */,
|
true /* not sanitize */,
|
||||||
std::make_shared<BlockBasedTableFactory>()
|
std::make_shared<BlockBasedTableFactory>()
|
||||||
);
|
);
|
||||||
|
|
||||||
|
PlainTableOptions plain_table_options;
|
||||||
|
plain_table_options.user_key_len = 8;
|
||||||
|
plain_table_options.bloom_bits_per_key = 8;
|
||||||
|
plain_table_options.hash_table_ratio = 0;
|
||||||
|
|
||||||
TestInternalKeyPropertiesCollector(
|
TestInternalKeyPropertiesCollector(
|
||||||
kPlainTableMagicNumber,
|
kPlainTableMagicNumber, false /* not sanitize */,
|
||||||
false /* not sanitize */,
|
std::make_shared<PlainTableFactory>(plain_table_options));
|
||||||
std::make_shared<PlainTableFactory>(8, 8, 0)
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
} // namespace rocksdb
|
} // namespace rocksdb
|
||||||
|
@ -121,31 +121,39 @@ struct PlainTablePropertyNames {
|
|||||||
static const std::string kEncodingType;
|
static const std::string kEncodingType;
|
||||||
};
|
};
|
||||||
|
|
||||||
// -- Plain Table with prefix-only seek
|
const uint32_t kPlainTableVariableLength = 0;
|
||||||
// For this factory, you need to set Options.prefix_extrator properly to make it
|
|
||||||
// work. Look-up will starts with prefix hash lookup for key prefix. Inside the
|
struct PlainTableOptions {
|
||||||
// hash bucket found, a binary search is executed for hash conflicts. Finally,
|
|
||||||
// a linear search is used.
|
|
||||||
// @user_key_len: plain table has optimization for fix-sized keys, which can be
|
// @user_key_len: plain table has optimization for fix-sized keys, which can be
|
||||||
// specified via user_key_len. Alternatively, you can pass
|
// specified via user_key_len. Alternatively, you can pass
|
||||||
// `kPlainTableVariableLength` if your keys have variable
|
// `kPlainTableVariableLength` if your keys have variable
|
||||||
// lengths.
|
// lengths.
|
||||||
|
uint32_t user_key_len = kPlainTableVariableLength;
|
||||||
|
|
||||||
// @bloom_bits_per_key: the number of bits used for bloom filer per prefix. You
|
// @bloom_bits_per_key: the number of bits used for bloom filer per prefix. You
|
||||||
// may disable it by passing a zero.
|
// may disable it by passing a zero.
|
||||||
|
int bloom_bits_per_key = 10;
|
||||||
|
|
||||||
// @hash_table_ratio: the desired utilization of the hash table used for prefix
|
// @hash_table_ratio: the desired utilization of the hash table used for prefix
|
||||||
// hashing. hash_table_ratio = number of prefixes / #buckets
|
// hashing. hash_table_ratio = number of prefixes / #buckets
|
||||||
// in the hash table
|
// in the hash table
|
||||||
|
double hash_table_ratio = 0.75;
|
||||||
|
|
||||||
// @index_sparseness: inside each prefix, need to build one index record for how
|
// @index_sparseness: inside each prefix, need to build one index record for how
|
||||||
// many keys for binary search inside each hash bucket.
|
// many keys for binary search inside each hash bucket.
|
||||||
// For encoding type kPrefix, the value will be used when
|
// For encoding type kPrefix, the value will be used when
|
||||||
// writing to determine an interval to rewrite the full key.
|
// writing to determine an interval to rewrite the full key.
|
||||||
// It will also be used as a suggestion and satisfied when
|
// It will also be used as a suggestion and satisfied when
|
||||||
// possible.
|
// possible.
|
||||||
|
size_t index_sparseness = 16;
|
||||||
|
|
||||||
// @huge_page_tlb_size: if <=0, allocate hash indexes and blooms from malloc.
|
// @huge_page_tlb_size: if <=0, allocate hash indexes and blooms from malloc.
|
||||||
// Otherwise from huge page TLB. The user needs to reserve
|
// Otherwise from huge page TLB. The user needs to reserve
|
||||||
// huge pages for it to be allocated, like:
|
// huge pages for it to be allocated, like:
|
||||||
// sysctl -w vm.nr_hugepages=20
|
// sysctl -w vm.nr_hugepages=20
|
||||||
// See linux doc Documentation/vm/hugetlbpage.txt
|
// See linux doc Documentation/vm/hugetlbpage.txt
|
||||||
|
size_t huge_page_tlb_size = 0;
|
||||||
|
|
||||||
// @encoding_type: how to encode the keys. See enum EncodingType above for
|
// @encoding_type: how to encode the keys. See enum EncodingType above for
|
||||||
// the choices. The value will determine how to encode keys
|
// the choices. The value will determine how to encode keys
|
||||||
// when writing to a new SST file. This value will be stored
|
// when writing to a new SST file. This value will be stored
|
||||||
@ -153,13 +161,21 @@ struct PlainTablePropertyNames {
|
|||||||
// file, which makes it possible for users to choose different
|
// file, which makes it possible for users to choose different
|
||||||
// encoding type when reopening a DB. Files with different
|
// encoding type when reopening a DB. Files with different
|
||||||
// encoding types can co-exist in the same DB and can be read.
|
// encoding types can co-exist in the same DB and can be read.
|
||||||
|
EncodingType encoding_type = kPlain;
|
||||||
|
|
||||||
const uint32_t kPlainTableVariableLength = 0;
|
// @full_scan_mode: mode for reading the whole file one record by one without
|
||||||
extern TableFactory* NewPlainTableFactory(
|
// using the index.
|
||||||
uint32_t user_key_len = kPlainTableVariableLength,
|
bool full_scan_mode = false;
|
||||||
int bloom_bits_per_prefix = 10, double hash_table_ratio = 0.75,
|
};
|
||||||
size_t index_sparseness = 16, size_t huge_page_tlb_size = 0,
|
|
||||||
EncodingType encoding_type = kPlain, bool full_scan_mode = false);
|
// -- Plain Table with prefix-only seek
|
||||||
|
// For this factory, you need to set Options.prefix_extrator properly to make it
|
||||||
|
// work. Look-up will starts with prefix hash lookup for key prefix. Inside the
|
||||||
|
// hash bucket found, a binary search is executed for hash conflicts. Finally,
|
||||||
|
// a linear search is used.
|
||||||
|
|
||||||
|
extern TableFactory* NewPlainTableFactory(const PlainTableOptions& options =
|
||||||
|
PlainTableOptions());
|
||||||
|
|
||||||
#endif // ROCKSDB_LITE
|
#endif // ROCKSDB_LITE
|
||||||
|
|
||||||
|
@ -33,13 +33,8 @@ TableBuilder* PlainTableFactory::NewTableBuilder(
|
|||||||
index_sparseness_);
|
index_sparseness_);
|
||||||
}
|
}
|
||||||
|
|
||||||
extern TableFactory* NewPlainTableFactory(
|
extern TableFactory* NewPlainTableFactory(const PlainTableOptions& options) {
|
||||||
uint32_t user_key_len, int bloom_bits_per_key, double hash_table_ratio,
|
return new PlainTableFactory(options);
|
||||||
size_t index_sparseness, size_t huge_page_tlb_size,
|
|
||||||
EncodingType encoding_type, bool full_scan_mode) {
|
|
||||||
return new PlainTableFactory(
|
|
||||||
user_key_len, bloom_bits_per_key, hash_table_ratio, index_sparseness,
|
|
||||||
huge_page_tlb_size, encoding_type, full_scan_mode);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const std::string PlainTablePropertyNames::kPrefixExtractorName =
|
const std::string PlainTablePropertyNames::kPrefixExtractorName =
|
||||||
|
@ -143,20 +143,15 @@ class PlainTableFactory : public TableFactory {
|
|||||||
// huge_page_tlb_size determines whether to allocate hash indexes from huge
|
// huge_page_tlb_size determines whether to allocate hash indexes from huge
|
||||||
// page TLB and the page size if allocating from there. See comments of
|
// page TLB and the page size if allocating from there. See comments of
|
||||||
// Arena::AllocateAligned() for details.
|
// Arena::AllocateAligned() for details.
|
||||||
explicit PlainTableFactory(uint32_t user_key_len = kPlainTableVariableLength,
|
explicit PlainTableFactory(const PlainTableOptions& options =
|
||||||
int bloom_bits_per_key = 0,
|
PlainTableOptions())
|
||||||
double hash_table_ratio = 0.75,
|
: user_key_len_(options.user_key_len),
|
||||||
size_t index_sparseness = 16,
|
bloom_bits_per_key_(options.bloom_bits_per_key),
|
||||||
size_t huge_page_tlb_size = 0,
|
hash_table_ratio_(options.hash_table_ratio),
|
||||||
EncodingType encoding_type = kPlain,
|
index_sparseness_(options.index_sparseness),
|
||||||
bool full_scan_mode = false)
|
huge_page_tlb_size_(options.huge_page_tlb_size),
|
||||||
: user_key_len_(user_key_len),
|
encoding_type_(options.encoding_type),
|
||||||
bloom_bits_per_key_(bloom_bits_per_key),
|
full_scan_mode_(options.full_scan_mode) {}
|
||||||
hash_table_ratio_(hash_table_ratio),
|
|
||||||
index_sparseness_(index_sparseness),
|
|
||||||
huge_page_tlb_size_(huge_page_tlb_size),
|
|
||||||
encoding_type_(encoding_type),
|
|
||||||
full_scan_mode_(full_scan_mode) {}
|
|
||||||
const char* Name() const override { return "PlainTable"; }
|
const char* Name() const override { return "PlainTable"; }
|
||||||
Status NewTableReader(const Options& options, const EnvOptions& soptions,
|
Status NewTableReader(const Options& options, const EnvOptions& soptions,
|
||||||
const InternalKeyComparator& internal_comparator,
|
const InternalKeyComparator& internal_comparator,
|
||||||
|
@ -445,9 +445,10 @@ Status PlainTableReader::PopulateIndex(TableProperties* props,
|
|||||||
size_t huge_page_tlb_size) {
|
size_t huge_page_tlb_size) {
|
||||||
assert(props != nullptr);
|
assert(props != nullptr);
|
||||||
table_properties_.reset(props);
|
table_properties_.reset(props);
|
||||||
|
// options.prefix_extractor is requried for a hash-based look-up.
|
||||||
|
if ((options_.prefix_extractor.get() == nullptr) && (hash_table_ratio != 0)) {
|
||||||
|
|
||||||
// options.prefix_extractor is requried for a hash-based look-up.
|
// options.prefix_extractor is requried for a hash-based look-up.
|
||||||
if (options_.prefix_extractor.get() == nullptr && hash_table_ratio != 0) {
|
|
||||||
return Status::NotSupported(
|
return Status::NotSupported(
|
||||||
"PlainTable requires a prefix extractor enable prefix hash mode.");
|
"PlainTable requires a prefix extractor enable prefix hash mode.");
|
||||||
}
|
}
|
||||||
|
@ -261,8 +261,13 @@ int main(int argc, char** argv) {
|
|||||||
if (FLAGS_plain_table) {
|
if (FLAGS_plain_table) {
|
||||||
options.allow_mmap_reads = true;
|
options.allow_mmap_reads = true;
|
||||||
env_options.use_mmap_reads = true;
|
env_options.use_mmap_reads = true;
|
||||||
tf = new rocksdb::PlainTableFactory(16, (FLAGS_prefix_len == 16) ? 0 : 8,
|
|
||||||
0.75);
|
rocksdb::PlainTableOptions plain_table_options;
|
||||||
|
plain_table_options.user_key_len = 16;
|
||||||
|
plain_table_options.bloom_bits_per_key = (FLAGS_prefix_len == 16) ? 0 : 8;
|
||||||
|
plain_table_options.hash_table_ratio = 0.75;
|
||||||
|
|
||||||
|
tf = new rocksdb::PlainTableFactory(plain_table_options);
|
||||||
options.prefix_extractor.reset(rocksdb::NewFixedPrefixTransform(
|
options.prefix_extractor.reset(rocksdb::NewFixedPrefixTransform(
|
||||||
FLAGS_prefix_len));
|
FLAGS_prefix_len));
|
||||||
} else {
|
} else {
|
||||||
|
@ -718,8 +718,16 @@ class Harness {
|
|||||||
only_support_prefix_seek_ = false;
|
only_support_prefix_seek_ = false;
|
||||||
options_.prefix_extractor = nullptr;
|
options_.prefix_extractor = nullptr;
|
||||||
options_.allow_mmap_reads = true;
|
options_.allow_mmap_reads = true;
|
||||||
|
|
||||||
|
{
|
||||||
|
PlainTableOptions plain_table_options;
|
||||||
|
plain_table_options.user_key_len = kPlainTableVariableLength;
|
||||||
|
plain_table_options.bloom_bits_per_key = 0;
|
||||||
|
plain_table_options.hash_table_ratio = 0;
|
||||||
|
|
||||||
options_.table_factory.reset(
|
options_.table_factory.reset(
|
||||||
NewPlainTableFactory(kPlainTableVariableLength, 0, 0));
|
NewPlainTableFactory(plain_table_options));
|
||||||
|
}
|
||||||
constructor_ = new TableConstructor(options_.comparator, true);
|
constructor_ = new TableConstructor(options_.comparator, true);
|
||||||
internal_comparator_.reset(
|
internal_comparator_.reset(
|
||||||
new InternalKeyComparator(options_.comparator));
|
new InternalKeyComparator(options_.comparator));
|
||||||
@ -1493,7 +1501,12 @@ TEST(BlockBasedTableTest, BlockCacheLeak) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
TEST(PlainTableTest, BasicPlainTableProperties) {
|
TEST(PlainTableTest, BasicPlainTableProperties) {
|
||||||
PlainTableFactory factory(8, 8, 0);
|
PlainTableOptions plain_table_options;
|
||||||
|
plain_table_options.user_key_len = 8;
|
||||||
|
plain_table_options.bloom_bits_per_key = 8;
|
||||||
|
plain_table_options.hash_table_ratio = 0;
|
||||||
|
|
||||||
|
PlainTableFactory factory(plain_table_options);
|
||||||
StringSink sink;
|
StringSink sink;
|
||||||
Options options;
|
Options options;
|
||||||
InternalKeyComparator ikc(options.comparator);
|
InternalKeyComparator ikc(options.comparator);
|
||||||
|
@ -157,8 +157,17 @@ Status SstFileReader::SetTableOptionsByMagicNumber(
|
|||||||
} else if (table_magic_number == kPlainTableMagicNumber ||
|
} else if (table_magic_number == kPlainTableMagicNumber ||
|
||||||
table_magic_number == kLegacyPlainTableMagicNumber) {
|
table_magic_number == kLegacyPlainTableMagicNumber) {
|
||||||
options_.allow_mmap_reads = true;
|
options_.allow_mmap_reads = true;
|
||||||
options_.table_factory.reset(NewPlainTableFactory(
|
|
||||||
kPlainTableVariableLength, 0, 0, 1, 0, kPlain, true));
|
PlainTableOptions plain_table_options;
|
||||||
|
plain_table_options.user_key_len = kPlainTableVariableLength;
|
||||||
|
plain_table_options.bloom_bits_per_key = 0;
|
||||||
|
plain_table_options.hash_table_ratio = 0;
|
||||||
|
plain_table_options.index_sparseness = 1;
|
||||||
|
plain_table_options.huge_page_tlb_size = 0;
|
||||||
|
plain_table_options.encoding_type = kPlain;
|
||||||
|
plain_table_options.full_scan_mode = true;
|
||||||
|
|
||||||
|
options_.table_factory.reset(NewPlainTableFactory(plain_table_options));
|
||||||
fprintf(stdout, "Sst file format: plain table\n");
|
fprintf(stdout, "Sst file format: plain table\n");
|
||||||
} else {
|
} else {
|
||||||
char error_msg_buffer[80];
|
char error_msg_buffer[80];
|
||||||
|
Loading…
Reference in New Issue
Block a user