7528130e38
Summary: This allows tombstone fragmenting to only be performed when the table is opened, and cached for subsequent accesses. On the same DB used in #4449, running `readrandom` results in the following: ``` readrandom : 0.983 micros/op 1017076 ops/sec; 78.3 MB/s (63103 of 100000 found) ``` Now that Get performance in the presence of range tombstones is reasonable, I also compared the performance between a DB with range tombstones, "expanded" range tombstones (several point tombstones that cover the same keys the equivalent range tombstone would cover, a common workaround for DeleteRange), and no range tombstones. The created DBs had 5 million keys each, and DeleteRange was called at regular intervals (depending on the total number of range tombstones being written) after 4.5 million Puts. The table below summarizes the results of a `readwhilewriting` benchmark (in order to provide somewhat more realistic results): ``` Tombstones? | avg micros/op | stddev micros/op | avg ops/s | stddev ops/s ----------------- | ------------- | ---------------- | ------------ | ------------ None | 0.6186 | 0.04637 | 1,625,252.90 | 124,679.41 500 Expanded | 0.6019 | 0.03628 | 1,666,670.40 | 101,142.65 500 Unexpanded | 0.6435 | 0.03994 | 1,559,979.40 | 104,090.52 1k Expanded | 0.6034 | 0.04349 | 1,665,128.10 | 125,144.57 1k Unexpanded | 0.6261 | 0.03093 | 1,600,457.50 | 79,024.94 5k Expanded | 0.6163 | 0.05926 | 1,636,668.80 | 154,888.85 5k Unexpanded | 0.6402 | 0.04002 | 1,567,804.70 | 100,965.55 10k Expanded | 0.6036 | 0.05105 | 1,667,237.70 | 142,830.36 10k Unexpanded | 0.6128 | 0.02598 | 1,634,633.40 | 72,161.82 25k Expanded | 0.6198 | 0.04542 | 1,620,980.50 | 116,662.93 25k Unexpanded | 0.5478 | 0.0362 | 1,833,059.10 | 121,233.81 50k Expanded | 0.5104 | 0.04347 | 1,973,107.90 | 184,073.49 50k Unexpanded | 0.4528 | 0.03387 | 2,219,034.50 | 170,984.32 ``` After a large enough quantity of range tombstones are written, range tombstone Gets can become faster than reading from an equivalent DB with several point tombstones. Pull Request resolved: https://github.com/facebook/rocksdb/pull/4493 Differential Revision: D10842844 Pulled By: abhimadan fbshipit-source-id: a7d44534f8120e6aabb65779d26c6b9df954c509
314 lines
13 KiB
C++
314 lines
13 KiB
C++
// Copyright (c) 2018-present, Facebook, Inc. All rights reserved.
|
|
// This source code is licensed under both the GPLv2 (found in the
|
|
// COPYING file in the root directory) and Apache 2.0 License
|
|
// (found in the LICENSE.Apache file in the root directory).
|
|
|
|
#include "db/range_tombstone_fragmenter.h"
|
|
|
|
#include "db/db_test_util.h"
|
|
#include "rocksdb/comparator.h"
|
|
#include "util/testutil.h"
|
|
|
|
namespace rocksdb {
|
|
|
|
class RangeTombstoneFragmenterTest : public testing::Test {};
|
|
|
|
namespace {
|
|
|
|
static auto bytewise_icmp = InternalKeyComparator(BytewiseComparator());
|
|
|
|
std::unique_ptr<InternalIterator> MakeRangeDelIter(
|
|
const std::vector<RangeTombstone>& range_dels) {
|
|
std::vector<std::string> keys, values;
|
|
for (const auto& range_del : range_dels) {
|
|
auto key_and_value = range_del.Serialize();
|
|
keys.push_back(key_and_value.first.Encode().ToString());
|
|
values.push_back(key_and_value.second.ToString());
|
|
}
|
|
return std::unique_ptr<test::VectorIterator>(
|
|
new test::VectorIterator(keys, values));
|
|
}
|
|
|
|
void VerifyFragmentedRangeDels(
|
|
FragmentedRangeTombstoneIterator* iter,
|
|
const std::vector<RangeTombstone>& expected_tombstones) {
|
|
iter->SeekToFirst();
|
|
for (size_t i = 0; i < expected_tombstones.size() && iter->Valid();
|
|
i++, iter->Next()) {
|
|
EXPECT_EQ(ExtractUserKey(iter->key()), expected_tombstones[i].start_key_);
|
|
EXPECT_EQ(iter->value(), expected_tombstones[i].end_key_);
|
|
EXPECT_EQ(GetInternalKeySeqno(iter->key()), expected_tombstones[i].seq_);
|
|
}
|
|
EXPECT_FALSE(iter->Valid());
|
|
}
|
|
|
|
struct SeekForPrevTestCase {
|
|
Slice seek_target;
|
|
RangeTombstone expected_position;
|
|
bool out_of_range;
|
|
};
|
|
|
|
void VerifySeekForPrev(FragmentedRangeTombstoneIterator* iter,
|
|
const std::vector<SeekForPrevTestCase>& cases) {
|
|
for (const auto& testcase : cases) {
|
|
InternalKey ikey_seek_target(testcase.seek_target, 0, kTypeRangeDeletion);
|
|
iter->SeekForPrev(ikey_seek_target.Encode());
|
|
if (testcase.out_of_range) {
|
|
ASSERT_FALSE(iter->Valid());
|
|
} else {
|
|
ASSERT_TRUE(iter->Valid());
|
|
EXPECT_EQ(ExtractUserKey(iter->key()),
|
|
testcase.expected_position.start_key_);
|
|
EXPECT_EQ(iter->value(), testcase.expected_position.end_key_);
|
|
EXPECT_EQ(GetInternalKeySeqno(iter->key()),
|
|
testcase.expected_position.seq_);
|
|
}
|
|
}
|
|
}
|
|
|
|
struct MaxCoveringTombstoneSeqnumTestCase {
|
|
Slice user_key;
|
|
int result;
|
|
};
|
|
|
|
void VerifyMaxCoveringTombstoneSeqnum(
|
|
FragmentedRangeTombstoneIterator* iter, const Comparator* ucmp,
|
|
const std::vector<MaxCoveringTombstoneSeqnumTestCase>& cases) {
|
|
for (const auto& testcase : cases) {
|
|
InternalKey key_and_snapshot(testcase.user_key, kMaxSequenceNumber,
|
|
kTypeValue);
|
|
EXPECT_EQ(testcase.result, MaxCoveringTombstoneSeqnum(
|
|
iter, key_and_snapshot.Encode(), ucmp));
|
|
}
|
|
}
|
|
|
|
} // anonymous namespace
|
|
|
|
TEST_F(RangeTombstoneFragmenterTest, NonOverlappingTombstones) {
|
|
auto range_del_iter = MakeRangeDelIter({{"a", "b", 10}, {"c", "d", 5}});
|
|
|
|
FragmentedRangeTombstoneList fragment_list(
|
|
std::move(range_del_iter), bytewise_icmp, true /* one_time_use */);
|
|
FragmentedRangeTombstoneIterator iter(&fragment_list, bytewise_icmp);
|
|
VerifyFragmentedRangeDels(&iter, {{"a", "b", 10}, {"c", "d", 5}});
|
|
VerifyMaxCoveringTombstoneSeqnum(&iter, bytewise_icmp.user_comparator(),
|
|
{{"", 0}, {"a", 10}, {"b", 0}, {"c", 5}});
|
|
}
|
|
|
|
TEST_F(RangeTombstoneFragmenterTest, OverlappingTombstones) {
|
|
auto range_del_iter = MakeRangeDelIter({{"a", "e", 10}, {"c", "g", 15}});
|
|
|
|
FragmentedRangeTombstoneList fragment_list(
|
|
std::move(range_del_iter), bytewise_icmp, true /* one_time_use */);
|
|
FragmentedRangeTombstoneIterator iter(&fragment_list, bytewise_icmp);
|
|
VerifyFragmentedRangeDels(&iter,
|
|
{{"a", "c", 10}, {"c", "e", 15}, {"e", "g", 15}});
|
|
VerifyMaxCoveringTombstoneSeqnum(&iter, bytewise_icmp.user_comparator(),
|
|
{{"a", 10}, {"c", 15}, {"e", 15}, {"g", 0}});
|
|
}
|
|
|
|
TEST_F(RangeTombstoneFragmenterTest, ContiguousTombstones) {
|
|
auto range_del_iter = MakeRangeDelIter(
|
|
{{"a", "c", 10}, {"c", "e", 20}, {"c", "e", 5}, {"e", "g", 15}});
|
|
|
|
FragmentedRangeTombstoneList fragment_list(
|
|
std::move(range_del_iter), bytewise_icmp, true /* one_time_use */);
|
|
FragmentedRangeTombstoneIterator iter(&fragment_list, bytewise_icmp);
|
|
VerifyFragmentedRangeDels(&iter,
|
|
{{"a", "c", 10}, {"c", "e", 20}, {"e", "g", 15}});
|
|
VerifyMaxCoveringTombstoneSeqnum(&iter, bytewise_icmp.user_comparator(),
|
|
{{"a", 10}, {"c", 20}, {"e", 15}, {"g", 0}});
|
|
}
|
|
|
|
TEST_F(RangeTombstoneFragmenterTest, RepeatedStartAndEndKey) {
|
|
auto range_del_iter =
|
|
MakeRangeDelIter({{"a", "c", 10}, {"a", "c", 7}, {"a", "c", 3}});
|
|
|
|
FragmentedRangeTombstoneList fragment_list(
|
|
std::move(range_del_iter), bytewise_icmp, true /* one_time_use */);
|
|
FragmentedRangeTombstoneIterator iter(&fragment_list, bytewise_icmp);
|
|
VerifyFragmentedRangeDels(&iter, {{"a", "c", 10}});
|
|
VerifyMaxCoveringTombstoneSeqnum(&iter, bytewise_icmp.user_comparator(),
|
|
{{"a", 10}, {"b", 10}, {"c", 0}});
|
|
}
|
|
|
|
TEST_F(RangeTombstoneFragmenterTest, RepeatedStartKeyDifferentEndKeys) {
|
|
auto range_del_iter =
|
|
MakeRangeDelIter({{"a", "e", 10}, {"a", "g", 7}, {"a", "c", 3}});
|
|
|
|
FragmentedRangeTombstoneList fragment_list(
|
|
std::move(range_del_iter), bytewise_icmp, true /* one_time_use */);
|
|
FragmentedRangeTombstoneIterator iter(&fragment_list, bytewise_icmp);
|
|
VerifyFragmentedRangeDels(&iter,
|
|
{{"a", "c", 10}, {"c", "e", 10}, {"e", "g", 7}});
|
|
VerifyMaxCoveringTombstoneSeqnum(&iter, bytewise_icmp.user_comparator(),
|
|
{{"a", 10}, {"c", 10}, {"e", 7}, {"g", 0}});
|
|
}
|
|
|
|
TEST_F(RangeTombstoneFragmenterTest, RepeatedStartKeyMixedEndKeys) {
|
|
auto range_del_iter = MakeRangeDelIter({{"a", "c", 30},
|
|
{"a", "g", 20},
|
|
{"a", "e", 10},
|
|
{"a", "g", 7},
|
|
{"a", "c", 3}});
|
|
|
|
FragmentedRangeTombstoneList fragment_list(
|
|
std::move(range_del_iter), bytewise_icmp, true /* one_time_use */);
|
|
FragmentedRangeTombstoneIterator iter(&fragment_list, bytewise_icmp);
|
|
VerifyFragmentedRangeDels(&iter,
|
|
{{"a", "c", 30}, {"c", "e", 20}, {"e", "g", 20}});
|
|
VerifyMaxCoveringTombstoneSeqnum(&iter, bytewise_icmp.user_comparator(),
|
|
{{"a", 30}, {"c", 20}, {"e", 20}, {"g", 0}});
|
|
}
|
|
|
|
TEST_F(RangeTombstoneFragmenterTest, OverlapAndRepeatedStartKey) {
|
|
auto range_del_iter = MakeRangeDelIter({{"a", "e", 10},
|
|
{"c", "g", 8},
|
|
{"c", "i", 6},
|
|
{"j", "n", 4},
|
|
{"j", "l", 2}});
|
|
|
|
FragmentedRangeTombstoneList fragment_list(
|
|
std::move(range_del_iter), bytewise_icmp, true /* one_time_use */);
|
|
FragmentedRangeTombstoneIterator iter(&fragment_list, bytewise_icmp);
|
|
VerifyFragmentedRangeDels(&iter, {{"a", "c", 10},
|
|
{"c", "e", 10},
|
|
{"e", "g", 8},
|
|
{"g", "i", 6},
|
|
{"j", "l", 4},
|
|
{"l", "n", 4}});
|
|
VerifyMaxCoveringTombstoneSeqnum(
|
|
&iter, bytewise_icmp.user_comparator(),
|
|
{{"a", 10}, {"c", 10}, {"e", 8}, {"i", 0}, {"j", 4}, {"m", 4}});
|
|
}
|
|
|
|
TEST_F(RangeTombstoneFragmenterTest, OverlapAndRepeatedStartKeyWithSnapshot) {
|
|
auto range_del_iter = MakeRangeDelIter({{"a", "e", 10},
|
|
{"c", "g", 8},
|
|
{"c", "i", 6},
|
|
{"j", "n", 4},
|
|
{"j", "l", 2}});
|
|
|
|
FragmentedRangeTombstoneList fragment_list(
|
|
std::move(range_del_iter), bytewise_icmp, true /* one_time_use */, 9);
|
|
FragmentedRangeTombstoneIterator iter(&fragment_list, bytewise_icmp);
|
|
VerifyFragmentedRangeDels(
|
|
&iter, {{"c", "g", 8}, {"g", "i", 6}, {"j", "l", 4}, {"l", "n", 4}});
|
|
VerifyMaxCoveringTombstoneSeqnum(
|
|
&iter, bytewise_icmp.user_comparator(),
|
|
{{"a", 0}, {"c", 8}, {"e", 8}, {"i", 0}, {"j", 4}, {"m", 4}});
|
|
}
|
|
|
|
TEST_F(RangeTombstoneFragmenterTest, OverlapAndRepeatedStartKeyUnordered) {
|
|
auto range_del_iter = MakeRangeDelIter({{"a", "e", 10},
|
|
{"j", "n", 4},
|
|
{"c", "i", 6},
|
|
{"c", "g", 8},
|
|
{"j", "l", 2}});
|
|
|
|
FragmentedRangeTombstoneList fragment_list(
|
|
std::move(range_del_iter), bytewise_icmp, true /* one_time_use */, 9);
|
|
FragmentedRangeTombstoneIterator iter(&fragment_list, bytewise_icmp);
|
|
VerifyFragmentedRangeDels(
|
|
&iter, {{"c", "g", 8}, {"g", "i", 6}, {"j", "l", 4}, {"l", "n", 4}});
|
|
VerifyMaxCoveringTombstoneSeqnum(
|
|
&iter, bytewise_icmp.user_comparator(),
|
|
{{"a", 0}, {"c", 8}, {"e", 8}, {"i", 0}, {"j", 4}, {"m", 4}});
|
|
}
|
|
|
|
TEST_F(RangeTombstoneFragmenterTest, OverlapAndRepeatedStartKeyMultiUse) {
|
|
auto range_del_iter = MakeRangeDelIter({{"a", "e", 10},
|
|
{"c", "g", 8},
|
|
{"c", "i", 6},
|
|
{"j", "n", 4},
|
|
{"j", "l", 2}});
|
|
|
|
FragmentedRangeTombstoneList fragment_list(
|
|
std::move(range_del_iter), bytewise_icmp, false /* one_time_use */);
|
|
FragmentedRangeTombstoneIterator iter(&fragment_list, bytewise_icmp);
|
|
VerifyFragmentedRangeDels(&iter, {{"a", "c", 10},
|
|
{"c", "e", 10},
|
|
{"c", "e", 8},
|
|
{"c", "e", 6},
|
|
{"e", "g", 8},
|
|
{"e", "g", 6},
|
|
{"g", "i", 6},
|
|
{"j", "l", 4},
|
|
{"j", "l", 2},
|
|
{"l", "n", 4}});
|
|
VerifyMaxCoveringTombstoneSeqnum(
|
|
&iter, bytewise_icmp.user_comparator(),
|
|
{{"a", 10}, {"c", 10}, {"e", 8}, {"i", 0}, {"j", 4}, {"m", 4}});
|
|
}
|
|
|
|
TEST_F(RangeTombstoneFragmenterTest, SeekForPrevStartKey) {
|
|
// Same tombstones as OverlapAndRepeatedStartKey.
|
|
auto range_del_iter = MakeRangeDelIter({{"a", "e", 10},
|
|
{"c", "g", 8},
|
|
{"c", "i", 6},
|
|
{"j", "n", 4},
|
|
{"j", "l", 2}});
|
|
|
|
FragmentedRangeTombstoneList fragment_list(
|
|
std::move(range_del_iter), bytewise_icmp, true /* one_time_use */);
|
|
FragmentedRangeTombstoneIterator iter(&fragment_list, bytewise_icmp);
|
|
VerifySeekForPrev(
|
|
&iter,
|
|
{{"a", {"a", "c", 10}}, {"e", {"e", "g", 8}}, {"l", {"l", "n", 4}}});
|
|
}
|
|
|
|
TEST_F(RangeTombstoneFragmenterTest, SeekForPrevCovered) {
|
|
// Same tombstones as OverlapAndRepeatedStartKey.
|
|
auto range_del_iter = MakeRangeDelIter({{"a", "e", 10},
|
|
{"c", "g", 8},
|
|
{"c", "i", 6},
|
|
{"j", "n", 4},
|
|
{"j", "l", 2}});
|
|
|
|
FragmentedRangeTombstoneList fragment_list(
|
|
std::move(range_del_iter), bytewise_icmp, true /* one_time_use */);
|
|
FragmentedRangeTombstoneIterator iter(&fragment_list, bytewise_icmp);
|
|
VerifySeekForPrev(
|
|
&iter,
|
|
{{"b", {"a", "c", 10}}, {"f", {"e", "g", 8}}, {"m", {"l", "n", 4}}});
|
|
}
|
|
|
|
TEST_F(RangeTombstoneFragmenterTest, SeekForPrevEndKey) {
|
|
// Same tombstones as OverlapAndRepeatedStartKey.
|
|
auto range_del_iter = MakeRangeDelIter({{"a", "e", 10},
|
|
{"c", "g", 8},
|
|
{"c", "i", 6},
|
|
{"j", "n", 4},
|
|
{"j", "l", 2}});
|
|
|
|
FragmentedRangeTombstoneList fragment_list(
|
|
std::move(range_del_iter), bytewise_icmp, true /* one_time_use */);
|
|
FragmentedRangeTombstoneIterator iter(&fragment_list, bytewise_icmp);
|
|
VerifySeekForPrev(&iter, {{"c", {"c", "e", 10}},
|
|
{"g", {"g", "i", 6}},
|
|
{"i", {"g", "i", 6}},
|
|
{"n", {"l", "n", 4}}});
|
|
}
|
|
|
|
TEST_F(RangeTombstoneFragmenterTest, SeekForPrevOutOfBounds) {
|
|
// Same tombstones as OverlapAndRepeatedStartKey.
|
|
auto range_del_iter = MakeRangeDelIter({{"a", "e", 10},
|
|
{"c", "g", 8},
|
|
{"c", "i", 6},
|
|
{"j", "n", 4},
|
|
{"j", "l", 2}});
|
|
|
|
FragmentedRangeTombstoneList fragment_list(
|
|
std::move(range_del_iter), bytewise_icmp, true /* one_time_use */);
|
|
FragmentedRangeTombstoneIterator iter(&fragment_list, bytewise_icmp);
|
|
VerifySeekForPrev(&iter,
|
|
{{"", {}, true /* out of range */}, {"z", {"l", "n", 4}}});
|
|
}
|
|
|
|
} // namespace rocksdb
|
|
|
|
int main(int argc, char** argv) {
|
|
::testing::InitGoogleTest(&argc, argv);
|
|
return RUN_ALL_TESTS();
|
|
}
|