2016-02-10 00:12:00 +01:00
|
|
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
2017-07-16 01:03:42 +02:00
|
|
|
// This source code is licensed under both the GPLv2 (found in the
|
|
|
|
// COPYING file in the root directory) and Apache 2.0 License
|
|
|
|
// (found in the LICENSE.Apache file in the root directory).
|
2013-10-16 23:59:46 +02:00
|
|
|
//
|
2011-03-18 23:37:00 +01:00
|
|
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
|
|
|
// Use of this source code is governed by a BSD-style license that can be
|
|
|
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
2016-09-08 23:45:32 +02:00
|
|
|
|
2017-02-03 01:38:40 +01:00
|
|
|
#include "table/merging_iterator.h"
|
2016-10-11 22:54:26 +02:00
|
|
|
#include <string>
|
2014-04-10 06:17:14 +02:00
|
|
|
#include <vector>
|
2017-09-11 20:53:22 +02:00
|
|
|
#include "db/dbformat.h"
|
2016-04-26 21:41:07 +02:00
|
|
|
#include "db/pinned_iterators_manager.h"
|
2017-04-06 04:02:00 +02:00
|
|
|
#include "monitoring/perf_context_imp.h"
|
2013-08-23 17:38:13 +02:00
|
|
|
#include "rocksdb/comparator.h"
|
|
|
|
#include "rocksdb/iterator.h"
|
2013-11-18 20:32:54 +01:00
|
|
|
#include "rocksdb/options.h"
|
2015-10-13 00:06:38 +02:00
|
|
|
#include "table/internal_iterator.h"
|
2012-12-26 20:51:36 +01:00
|
|
|
#include "table/iter_heap.h"
|
2011-03-18 23:37:00 +01:00
|
|
|
#include "table/iterator_wrapper.h"
|
In DB::NewIterator(), try to allocate the whole iterator tree in an arena
Summary:
In this patch, try to allocate the whole iterator tree starting from DBIter from an arena
1. ArenaWrappedDBIter is created when serves as the entry point of an iterator tree, with an arena in it.
2. Add an option to create iterator from arena for following iterators: DBIter, MergingIterator, MemtableIterator, all mem table's iterators, all table reader's iterators and two level iterator.
3. MergeIteratorBuilder is created to incrementally build the tree of internal iterators. It is passed to mem table list and version set and add iterators to it.
Limitations:
(1) Only DB::NewIterator() without tailing uses the arena. Other cases, including readonly DB and compactions are still from malloc
(2) Two level iterator itself is allocated in arena, but not iterators inside it.
Test Plan: make all check
Reviewers: ljin, haobo
Reviewed By: haobo
Subscribers: leveldb, dhruba, yhchiang, igor
Differential Revision: https://reviews.facebook.net/D18513
2014-06-03 01:38:00 +02:00
|
|
|
#include "util/arena.h"
|
2016-04-26 21:41:07 +02:00
|
|
|
#include "util/autovector.h"
|
2015-07-06 13:24:09 +02:00
|
|
|
#include "util/heap.h"
|
2013-11-18 20:32:54 +01:00
|
|
|
#include "util/stop_watch.h"
|
2015-08-06 22:23:02 +02:00
|
|
|
#include "util/sync_point.h"
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2013-10-04 06:49:15 +02:00
|
|
|
namespace rocksdb {
|
2014-11-13 20:39:30 +01:00
|
|
|
// Without anonymous namespace here, we fail the warning -Wmissing-prototypes
|
|
|
|
namespace {
|
2015-07-06 13:24:09 +02:00
|
|
|
typedef BinaryHeap<IteratorWrapper*, MaxIteratorComparator> MergerMaxIterHeap;
|
|
|
|
typedef BinaryHeap<IteratorWrapper*, MinIteratorComparator> MergerMinIterHeap;
|
2014-11-13 20:39:30 +01:00
|
|
|
} // namespace
|
2014-04-10 06:17:14 +02:00
|
|
|
|
2014-05-08 22:32:45 +02:00
|
|
|
const size_t kNumIterReserve = 4;
|
|
|
|
|
2015-10-13 00:06:38 +02:00
|
|
|
class MergingIterator : public InternalIterator {
|
2011-03-18 23:37:00 +01:00
|
|
|
public:
|
2017-09-11 20:53:22 +02:00
|
|
|
MergingIterator(const InternalKeyComparator* comparator,
|
|
|
|
InternalIterator** children, int n, bool is_arena_mode,
|
|
|
|
bool prefix_seek_mode)
|
2016-04-26 21:41:07 +02:00
|
|
|
: is_arena_mode_(is_arena_mode),
|
In DB::NewIterator(), try to allocate the whole iterator tree in an arena
Summary:
In this patch, try to allocate the whole iterator tree starting from DBIter from an arena
1. ArenaWrappedDBIter is created when serves as the entry point of an iterator tree, with an arena in it.
2. Add an option to create iterator from arena for following iterators: DBIter, MergingIterator, MemtableIterator, all mem table's iterators, all table reader's iterators and two level iterator.
3. MergeIteratorBuilder is created to incrementally build the tree of internal iterators. It is passed to mem table list and version set and add iterators to it.
Limitations:
(1) Only DB::NewIterator() without tailing uses the arena. Other cases, including readonly DB and compactions are still from malloc
(2) Two level iterator itself is allocated in arena, but not iterators inside it.
Test Plan: make all check
Reviewers: ljin, haobo
Reviewed By: haobo
Subscribers: leveldb, dhruba, yhchiang, igor
Differential Revision: https://reviews.facebook.net/D18513
2014-06-03 01:38:00 +02:00
|
|
|
comparator_(comparator),
|
2013-03-01 03:04:58 +01:00
|
|
|
current_(nullptr),
|
2012-12-26 20:51:36 +01:00
|
|
|
direction_(kForward),
|
2016-04-26 21:41:07 +02:00
|
|
|
minHeap_(comparator_),
|
2016-10-24 22:13:01 +02:00
|
|
|
prefix_seek_mode_(prefix_seek_mode),
|
2016-04-26 21:41:07 +02:00
|
|
|
pinned_iters_mgr_(nullptr) {
|
2014-05-08 22:32:45 +02:00
|
|
|
children_.resize(n);
|
2011-03-18 23:37:00 +01:00
|
|
|
for (int i = 0; i < n; i++) {
|
|
|
|
children_[i].Set(children[i]);
|
|
|
|
}
|
2013-08-21 07:58:16 +02:00
|
|
|
for (auto& child : children_) {
|
|
|
|
if (child.Valid()) {
|
|
|
|
minHeap_.push(&child);
|
2012-12-26 20:51:36 +01:00
|
|
|
}
|
|
|
|
}
|
2015-07-06 13:24:09 +02:00
|
|
|
current_ = CurrentForward();
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2015-10-13 00:06:38 +02:00
|
|
|
virtual void AddIterator(InternalIterator* iter) {
|
In DB::NewIterator(), try to allocate the whole iterator tree in an arena
Summary:
In this patch, try to allocate the whole iterator tree starting from DBIter from an arena
1. ArenaWrappedDBIter is created when serves as the entry point of an iterator tree, with an arena in it.
2. Add an option to create iterator from arena for following iterators: DBIter, MergingIterator, MemtableIterator, all mem table's iterators, all table reader's iterators and two level iterator.
3. MergeIteratorBuilder is created to incrementally build the tree of internal iterators. It is passed to mem table list and version set and add iterators to it.
Limitations:
(1) Only DB::NewIterator() without tailing uses the arena. Other cases, including readonly DB and compactions are still from malloc
(2) Two level iterator itself is allocated in arena, but not iterators inside it.
Test Plan: make all check
Reviewers: ljin, haobo
Reviewed By: haobo
Subscribers: leveldb, dhruba, yhchiang, igor
Differential Revision: https://reviews.facebook.net/D18513
2014-06-03 01:38:00 +02:00
|
|
|
assert(direction_ == kForward);
|
|
|
|
children_.emplace_back(iter);
|
2016-04-26 21:41:07 +02:00
|
|
|
if (pinned_iters_mgr_) {
|
|
|
|
iter->SetPinnedItersMgr(pinned_iters_mgr_);
|
2015-12-16 21:08:30 +01:00
|
|
|
}
|
In DB::NewIterator(), try to allocate the whole iterator tree in an arena
Summary:
In this patch, try to allocate the whole iterator tree starting from DBIter from an arena
1. ArenaWrappedDBIter is created when serves as the entry point of an iterator tree, with an arena in it.
2. Add an option to create iterator from arena for following iterators: DBIter, MergingIterator, MemtableIterator, all mem table's iterators, all table reader's iterators and two level iterator.
3. MergeIteratorBuilder is created to incrementally build the tree of internal iterators. It is passed to mem table list and version set and add iterators to it.
Limitations:
(1) Only DB::NewIterator() without tailing uses the arena. Other cases, including readonly DB and compactions are still from malloc
(2) Two level iterator itself is allocated in arena, but not iterators inside it.
Test Plan: make all check
Reviewers: ljin, haobo
Reviewed By: haobo
Subscribers: leveldb, dhruba, yhchiang, igor
Differential Revision: https://reviews.facebook.net/D18513
2014-06-03 01:38:00 +02:00
|
|
|
auto new_wrapper = children_.back();
|
|
|
|
if (new_wrapper.Valid()) {
|
|
|
|
minHeap_.push(&new_wrapper);
|
2015-07-06 13:24:09 +02:00
|
|
|
current_ = CurrentForward();
|
In DB::NewIterator(), try to allocate the whole iterator tree in an arena
Summary:
In this patch, try to allocate the whole iterator tree starting from DBIter from an arena
1. ArenaWrappedDBIter is created when serves as the entry point of an iterator tree, with an arena in it.
2. Add an option to create iterator from arena for following iterators: DBIter, MergingIterator, MemtableIterator, all mem table's iterators, all table reader's iterators and two level iterator.
3. MergeIteratorBuilder is created to incrementally build the tree of internal iterators. It is passed to mem table list and version set and add iterators to it.
Limitations:
(1) Only DB::NewIterator() without tailing uses the arena. Other cases, including readonly DB and compactions are still from malloc
(2) Two level iterator itself is allocated in arena, but not iterators inside it.
Test Plan: make all check
Reviewers: ljin, haobo
Reviewed By: haobo
Subscribers: leveldb, dhruba, yhchiang, igor
Differential Revision: https://reviews.facebook.net/D18513
2014-06-03 01:38:00 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
virtual ~MergingIterator() {
|
|
|
|
for (auto& child : children_) {
|
|
|
|
child.DeleteIter(is_arena_mode_);
|
|
|
|
}
|
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2015-02-26 20:28:41 +01:00
|
|
|
virtual bool Valid() const override { return (current_ != nullptr); }
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2015-02-26 20:28:41 +01:00
|
|
|
virtual void SeekToFirst() override {
|
2012-12-26 20:51:36 +01:00
|
|
|
ClearHeaps();
|
2013-08-21 07:58:16 +02:00
|
|
|
for (auto& child : children_) {
|
|
|
|
child.SeekToFirst();
|
|
|
|
if (child.Valid()) {
|
|
|
|
minHeap_.push(&child);
|
2012-12-26 20:51:36 +01:00
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
2011-03-21 20:40:57 +01:00
|
|
|
direction_ = kForward;
|
2015-07-06 13:24:09 +02:00
|
|
|
current_ = CurrentForward();
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2015-02-26 20:28:41 +01:00
|
|
|
virtual void SeekToLast() override {
|
2012-12-26 20:51:36 +01:00
|
|
|
ClearHeaps();
|
2015-07-06 13:24:09 +02:00
|
|
|
InitMaxHeap();
|
2013-08-21 07:58:16 +02:00
|
|
|
for (auto& child : children_) {
|
|
|
|
child.SeekToLast();
|
|
|
|
if (child.Valid()) {
|
2015-07-06 13:24:09 +02:00
|
|
|
maxHeap_->push(&child);
|
2012-12-26 20:51:36 +01:00
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
2011-03-21 20:40:57 +01:00
|
|
|
direction_ = kReverse;
|
2015-07-06 13:24:09 +02:00
|
|
|
current_ = CurrentReverse();
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2015-02-26 20:28:41 +01:00
|
|
|
virtual void Seek(const Slice& target) override {
|
2015-07-06 13:24:09 +02:00
|
|
|
ClearHeaps();
|
2013-08-21 07:58:16 +02:00
|
|
|
for (auto& child : children_) {
|
2014-08-23 00:28:58 +02:00
|
|
|
{
|
|
|
|
PERF_TIMER_GUARD(seek_child_seek_time);
|
|
|
|
child.Seek(target);
|
|
|
|
}
|
2016-08-30 05:55:39 +02:00
|
|
|
PERF_COUNTER_ADD(seek_child_seek_count, 1);
|
2016-09-08 23:45:32 +02:00
|
|
|
|
2013-08-21 07:58:16 +02:00
|
|
|
if (child.Valid()) {
|
2015-07-06 13:24:09 +02:00
|
|
|
PERF_TIMER_GUARD(seek_min_heap_time);
|
|
|
|
minHeap_.push(&child);
|
2012-12-26 20:51:36 +01:00
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
2015-07-06 13:24:09 +02:00
|
|
|
direction_ = kForward;
|
|
|
|
{
|
2014-08-23 00:28:58 +02:00
|
|
|
PERF_TIMER_GUARD(seek_min_heap_time);
|
2015-07-06 13:24:09 +02:00
|
|
|
current_ = CurrentForward();
|
2013-11-19 00:39:42 +01:00
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2016-09-28 03:20:57 +02:00
|
|
|
virtual void SeekForPrev(const Slice& target) override {
|
|
|
|
ClearHeaps();
|
|
|
|
InitMaxHeap();
|
|
|
|
|
|
|
|
for (auto& child : children_) {
|
|
|
|
{
|
|
|
|
PERF_TIMER_GUARD(seek_child_seek_time);
|
|
|
|
child.SeekForPrev(target);
|
|
|
|
}
|
|
|
|
PERF_COUNTER_ADD(seek_child_seek_count, 1);
|
|
|
|
|
|
|
|
if (child.Valid()) {
|
|
|
|
PERF_TIMER_GUARD(seek_max_heap_time);
|
|
|
|
maxHeap_->push(&child);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
direction_ = kReverse;
|
|
|
|
{
|
|
|
|
PERF_TIMER_GUARD(seek_max_heap_time);
|
|
|
|
current_ = CurrentReverse();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-02-26 20:28:41 +01:00
|
|
|
virtual void Next() override {
|
2011-03-18 23:37:00 +01:00
|
|
|
assert(Valid());
|
2016-09-08 23:45:32 +02:00
|
|
|
|
2011-03-21 20:40:57 +01:00
|
|
|
// Ensure that all children are positioned after key().
|
|
|
|
// If we are moving in the forward direction, it is already
|
2015-07-06 13:24:09 +02:00
|
|
|
// true for all of the non-current children since current_ is
|
|
|
|
// the smallest child and key() == current_->key().
|
2011-03-21 20:40:57 +01:00
|
|
|
if (direction_ != kForward) {
|
2015-07-06 13:24:09 +02:00
|
|
|
// Otherwise, advance the non-current children. We advance current_
|
|
|
|
// just after the if-block.
|
2012-12-26 20:51:36 +01:00
|
|
|
ClearHeaps();
|
2013-08-21 07:58:16 +02:00
|
|
|
for (auto& child : children_) {
|
|
|
|
if (&child != current_) {
|
2016-10-14 02:36:48 +02:00
|
|
|
child.Seek(key());
|
2016-09-08 23:45:32 +02:00
|
|
|
if (child.Valid() && comparator_->Equal(key(), child.key())) {
|
2013-08-21 07:58:16 +02:00
|
|
|
child.Next();
|
2011-03-21 20:40:57 +01:00
|
|
|
}
|
2015-07-06 13:24:09 +02:00
|
|
|
}
|
|
|
|
if (child.Valid()) {
|
2016-09-08 23:45:32 +02:00
|
|
|
minHeap_.push(&child);
|
2011-03-21 20:40:57 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
direction_ = kForward;
|
2015-07-06 13:24:09 +02:00
|
|
|
// The loop advanced all non-current children to be > key() so current_
|
|
|
|
// should still be strictly the smallest key.
|
|
|
|
assert(current_ == CurrentForward());
|
2011-03-21 20:40:57 +01:00
|
|
|
}
|
|
|
|
|
2015-07-06 13:24:09 +02:00
|
|
|
// For the heap modifications below to be correct, current_ must be the
|
|
|
|
// current top of the heap.
|
|
|
|
assert(current_ == CurrentForward());
|
|
|
|
|
2012-12-26 20:51:36 +01:00
|
|
|
// as the current points to the current record. move the iterator forward.
|
2011-03-18 23:37:00 +01:00
|
|
|
current_->Next();
|
2015-07-06 13:24:09 +02:00
|
|
|
if (current_->Valid()) {
|
|
|
|
// current is still valid after the Next() call above. Call
|
|
|
|
// replace_top() to restore the heap property. When the same child
|
|
|
|
// iterator yields a sequence of keys, this is cheap.
|
|
|
|
minHeap_.replace_top(current_);
|
|
|
|
} else {
|
|
|
|
// current stopped being valid, remove it from the heap.
|
|
|
|
minHeap_.pop();
|
2012-12-26 20:51:36 +01:00
|
|
|
}
|
2015-07-06 13:24:09 +02:00
|
|
|
current_ = CurrentForward();
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2015-02-26 20:28:41 +01:00
|
|
|
virtual void Prev() override {
|
2011-03-18 23:37:00 +01:00
|
|
|
assert(Valid());
|
2011-03-21 20:40:57 +01:00
|
|
|
// Ensure that all children are positioned before key().
|
|
|
|
// If we are moving in the reverse direction, it is already
|
2015-07-06 13:24:09 +02:00
|
|
|
// true for all of the non-current children since current_ is
|
|
|
|
// the largest child and key() == current_->key().
|
2011-03-21 20:40:57 +01:00
|
|
|
if (direction_ != kReverse) {
|
2015-07-06 13:24:09 +02:00
|
|
|
// Otherwise, retreat the non-current children. We retreat current_
|
|
|
|
// just after the if-block.
|
2012-12-26 20:51:36 +01:00
|
|
|
ClearHeaps();
|
2015-07-06 13:24:09 +02:00
|
|
|
InitMaxHeap();
|
2013-08-21 07:58:16 +02:00
|
|
|
for (auto& child : children_) {
|
|
|
|
if (&child != current_) {
|
2016-10-24 22:13:01 +02:00
|
|
|
if (!prefix_seek_mode_) {
|
|
|
|
child.Seek(key());
|
|
|
|
if (child.Valid()) {
|
|
|
|
// Child is at first entry >= key(). Step back one to be < key()
|
|
|
|
TEST_SYNC_POINT_CALLBACK("MergeIterator::Prev:BeforePrev",
|
|
|
|
&child);
|
|
|
|
child.Prev();
|
|
|
|
} else {
|
|
|
|
// Child has no entries >= key(). Position at last entry.
|
|
|
|
TEST_SYNC_POINT("MergeIterator::Prev:BeforeSeekToLast");
|
|
|
|
child.SeekToLast();
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
child.SeekForPrev(key());
|
|
|
|
if (child.Valid() && comparator_->Equal(key(), child.key())) {
|
|
|
|
child.Prev();
|
|
|
|
}
|
2011-03-21 20:40:57 +01:00
|
|
|
}
|
2015-07-06 13:24:09 +02:00
|
|
|
}
|
|
|
|
if (child.Valid()) {
|
2016-09-08 23:45:32 +02:00
|
|
|
maxHeap_->push(&child);
|
2011-03-21 20:40:57 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
direction_ = kReverse;
|
2016-10-24 22:13:01 +02:00
|
|
|
if (!prefix_seek_mode_) {
|
|
|
|
// Note that we don't do assert(current_ == CurrentReverse()) here
|
|
|
|
// because it is possible to have some keys larger than the seek-key
|
|
|
|
// inserted between Seek() and SeekToLast(), which makes current_ not
|
|
|
|
// equal to CurrentReverse().
|
|
|
|
current_ = CurrentReverse();
|
|
|
|
}
|
2016-10-11 22:54:26 +02:00
|
|
|
// The loop advanced all non-current children to be < key() so current_
|
|
|
|
// should still be strictly the smallest key.
|
|
|
|
assert(current_ == CurrentReverse());
|
2011-03-21 20:40:57 +01:00
|
|
|
}
|
|
|
|
|
2015-07-06 13:24:09 +02:00
|
|
|
// For the heap modifications below to be correct, current_ must be the
|
|
|
|
// current top of the heap.
|
|
|
|
assert(current_ == CurrentReverse());
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
current_->Prev();
|
2012-12-26 20:51:36 +01:00
|
|
|
if (current_->Valid()) {
|
2015-07-06 13:24:09 +02:00
|
|
|
// current is still valid after the Prev() call above. Call
|
|
|
|
// replace_top() to restore the heap property. When the same child
|
|
|
|
// iterator yields a sequence of keys, this is cheap.
|
|
|
|
maxHeap_->replace_top(current_);
|
|
|
|
} else {
|
|
|
|
// current stopped being valid, remove it from the heap.
|
|
|
|
maxHeap_->pop();
|
2012-12-26 20:51:36 +01:00
|
|
|
}
|
2015-07-06 13:24:09 +02:00
|
|
|
current_ = CurrentReverse();
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2015-02-26 20:28:41 +01:00
|
|
|
virtual Slice key() const override {
|
2011-03-18 23:37:00 +01:00
|
|
|
assert(Valid());
|
|
|
|
return current_->key();
|
|
|
|
}
|
|
|
|
|
2015-02-26 20:28:41 +01:00
|
|
|
virtual Slice value() const override {
|
2011-03-18 23:37:00 +01:00
|
|
|
assert(Valid());
|
|
|
|
return current_->value();
|
|
|
|
}
|
|
|
|
|
2015-02-26 20:28:41 +01:00
|
|
|
virtual Status status() const override {
|
2014-11-06 20:14:28 +01:00
|
|
|
Status s;
|
2013-08-21 07:58:16 +02:00
|
|
|
for (auto& child : children_) {
|
2014-11-06 20:14:28 +01:00
|
|
|
s = child.status();
|
|
|
|
if (!s.ok()) {
|
2011-03-18 23:37:00 +01:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
2014-11-06 20:14:28 +01:00
|
|
|
return s;
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2016-04-26 21:41:07 +02:00
|
|
|
virtual void SetPinnedItersMgr(
|
|
|
|
PinnedIteratorsManager* pinned_iters_mgr) override {
|
|
|
|
pinned_iters_mgr_ = pinned_iters_mgr;
|
2015-12-16 21:08:30 +01:00
|
|
|
for (auto& child : children_) {
|
2016-04-26 21:41:07 +02:00
|
|
|
child.SetPinnedItersMgr(pinned_iters_mgr);
|
2015-12-16 21:08:30 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
virtual bool IsKeyPinned() const override {
|
|
|
|
assert(Valid());
|
2016-04-26 21:41:07 +02:00
|
|
|
return pinned_iters_mgr_ && pinned_iters_mgr_->PinningEnabled() &&
|
|
|
|
current_->IsKeyPinned();
|
2015-12-16 21:08:30 +01:00
|
|
|
}
|
|
|
|
|
Introduce FullMergeV2 (eliminate memcpy from merge operators)
Summary:
This diff update the code to pin the merge operator operands while the merge operation is done, so that we can eliminate the memcpy cost, to do that we need a new public API for FullMerge that replace the std::deque<std::string> with std::vector<Slice>
This diff is stacked on top of D56493 and D56511
In this diff we
- Update FullMergeV2 arguments to be encapsulated in MergeOperationInput and MergeOperationOutput which will make it easier to add new arguments in the future
- Replace std::deque<std::string> with std::vector<Slice> to pass operands
- Replace MergeContext std::deque with std::vector (based on a simple benchmark I ran https://gist.github.com/IslamAbdelRahman/78fc86c9ab9f52b1df791e58943fb187)
- Allow FullMergeV2 output to be an existing operand
```
[Everything in Memtable | 10K operands | 10 KB each | 1 operand per key]
DEBUG_LEVEL=0 make db_bench -j64 && ./db_bench --benchmarks="mergerandom,readseq,readseq,readseq,readseq,readseq" --merge_operator="max" --merge_keys=10000 --num=10000 --disable_auto_compactions --value_size=10240 --write_buffer_size=1000000000
[FullMergeV2]
readseq : 0.607 micros/op 1648235 ops/sec; 16121.2 MB/s
readseq : 0.478 micros/op 2091546 ops/sec; 20457.2 MB/s
readseq : 0.252 micros/op 3972081 ops/sec; 38850.5 MB/s
readseq : 0.237 micros/op 4218328 ops/sec; 41259.0 MB/s
readseq : 0.247 micros/op 4043927 ops/sec; 39553.2 MB/s
[master]
readseq : 3.935 micros/op 254140 ops/sec; 2485.7 MB/s
readseq : 3.722 micros/op 268657 ops/sec; 2627.7 MB/s
readseq : 3.149 micros/op 317605 ops/sec; 3106.5 MB/s
readseq : 3.125 micros/op 320024 ops/sec; 3130.1 MB/s
readseq : 4.075 micros/op 245374 ops/sec; 2400.0 MB/s
```
```
[Everything in Memtable | 10K operands | 10 KB each | 10 operand per key]
DEBUG_LEVEL=0 make db_bench -j64 && ./db_bench --benchmarks="mergerandom,readseq,readseq,readseq,readseq,readseq" --merge_operator="max" --merge_keys=1000 --num=10000 --disable_auto_compactions --value_size=10240 --write_buffer_size=1000000000
[FullMergeV2]
readseq : 3.472 micros/op 288018 ops/sec; 2817.1 MB/s
readseq : 2.304 micros/op 434027 ops/sec; 4245.2 MB/s
readseq : 1.163 micros/op 859845 ops/sec; 8410.0 MB/s
readseq : 1.192 micros/op 838926 ops/sec; 8205.4 MB/s
readseq : 1.250 micros/op 800000 ops/sec; 7824.7 MB/s
[master]
readseq : 24.025 micros/op 41623 ops/sec; 407.1 MB/s
readseq : 18.489 micros/op 54086 ops/sec; 529.0 MB/s
readseq : 18.693 micros/op 53495 ops/sec; 523.2 MB/s
readseq : 23.621 micros/op 42335 ops/sec; 414.1 MB/s
readseq : 18.775 micros/op 53262 ops/sec; 521.0 MB/s
```
```
[Everything in Block cache | 10K operands | 10 KB each | 1 operand per key]
[FullMergeV2]
$ DEBUG_LEVEL=0 make db_bench -j64 && ./db_bench --benchmarks="readseq,readseq,readseq,readseq,readseq" --merge_operator="max" --num=100000 --db="/dev/shm/merge-random-10K-10KB" --cache_size=1000000000 --use_existing_db --disable_auto_compactions
readseq : 14.741 micros/op 67837 ops/sec; 663.5 MB/s
readseq : 1.029 micros/op 971446 ops/sec; 9501.6 MB/s
readseq : 0.974 micros/op 1026229 ops/sec; 10037.4 MB/s
readseq : 0.965 micros/op 1036080 ops/sec; 10133.8 MB/s
readseq : 0.943 micros/op 1060657 ops/sec; 10374.2 MB/s
[master]
readseq : 16.735 micros/op 59755 ops/sec; 584.5 MB/s
readseq : 3.029 micros/op 330151 ops/sec; 3229.2 MB/s
readseq : 3.136 micros/op 318883 ops/sec; 3119.0 MB/s
readseq : 3.065 micros/op 326245 ops/sec; 3191.0 MB/s
readseq : 3.014 micros/op 331813 ops/sec; 3245.4 MB/s
```
```
[Everything in Block cache | 10K operands | 10 KB each | 10 operand per key]
DEBUG_LEVEL=0 make db_bench -j64 && ./db_bench --benchmarks="readseq,readseq,readseq,readseq,readseq" --merge_operator="max" --num=100000 --db="/dev/shm/merge-random-10-operands-10K-10KB" --cache_size=1000000000 --use_existing_db --disable_auto_compactions
[FullMergeV2]
readseq : 24.325 micros/op 41109 ops/sec; 402.1 MB/s
readseq : 1.470 micros/op 680272 ops/sec; 6653.7 MB/s
readseq : 1.231 micros/op 812347 ops/sec; 7945.5 MB/s
readseq : 1.091 micros/op 916590 ops/sec; 8965.1 MB/s
readseq : 1.109 micros/op 901713 ops/sec; 8819.6 MB/s
[master]
readseq : 27.257 micros/op 36687 ops/sec; 358.8 MB/s
readseq : 4.443 micros/op 225073 ops/sec; 2201.4 MB/s
readseq : 5.830 micros/op 171526 ops/sec; 1677.7 MB/s
readseq : 4.173 micros/op 239635 ops/sec; 2343.8 MB/s
readseq : 4.150 micros/op 240963 ops/sec; 2356.8 MB/s
```
Test Plan: COMPILE_WITH_ASAN=1 make check -j64
Reviewers: yhchiang, andrewkr, sdong
Reviewed By: sdong
Subscribers: lovro, andrewkr, dhruba
Differential Revision: https://reviews.facebook.net/D57075
2016-07-20 18:49:03 +02:00
|
|
|
virtual bool IsValuePinned() const override {
|
|
|
|
assert(Valid());
|
|
|
|
return pinned_iters_mgr_ && pinned_iters_mgr_->PinningEnabled() &&
|
|
|
|
current_->IsValuePinned();
|
|
|
|
}
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
private:
|
2015-07-06 13:24:09 +02:00
|
|
|
// Clears heaps for both directions, used when changing direction or seeking
|
2012-12-26 20:51:36 +01:00
|
|
|
void ClearHeaps();
|
2015-07-06 13:24:09 +02:00
|
|
|
// Ensures that maxHeap_ is initialized when starting to go in the reverse
|
|
|
|
// direction
|
|
|
|
void InitMaxHeap();
|
2011-03-18 23:37:00 +01:00
|
|
|
|
In DB::NewIterator(), try to allocate the whole iterator tree in an arena
Summary:
In this patch, try to allocate the whole iterator tree starting from DBIter from an arena
1. ArenaWrappedDBIter is created when serves as the entry point of an iterator tree, with an arena in it.
2. Add an option to create iterator from arena for following iterators: DBIter, MergingIterator, MemtableIterator, all mem table's iterators, all table reader's iterators and two level iterator.
3. MergeIteratorBuilder is created to incrementally build the tree of internal iterators. It is passed to mem table list and version set and add iterators to it.
Limitations:
(1) Only DB::NewIterator() without tailing uses the arena. Other cases, including readonly DB and compactions are still from malloc
(2) Two level iterator itself is allocated in arena, but not iterators inside it.
Test Plan: make all check
Reviewers: ljin, haobo
Reviewed By: haobo
Subscribers: leveldb, dhruba, yhchiang, igor
Differential Revision: https://reviews.facebook.net/D18513
2014-06-03 01:38:00 +02:00
|
|
|
bool is_arena_mode_;
|
2017-09-11 20:53:22 +02:00
|
|
|
const InternalKeyComparator* comparator_;
|
2014-05-08 22:32:45 +02:00
|
|
|
autovector<IteratorWrapper, kNumIterReserve> children_;
|
2015-07-06 13:24:09 +02:00
|
|
|
|
|
|
|
// Cached pointer to child iterator with the current key, or nullptr if no
|
|
|
|
// child iterators are valid. This is the top of minHeap_ or maxHeap_
|
|
|
|
// depending on the direction.
|
2011-03-18 23:37:00 +01:00
|
|
|
IteratorWrapper* current_;
|
2011-03-21 20:40:57 +01:00
|
|
|
// Which direction is the iterator moving?
|
|
|
|
enum Direction {
|
|
|
|
kForward,
|
|
|
|
kReverse
|
|
|
|
};
|
|
|
|
Direction direction_;
|
2014-11-13 20:39:30 +01:00
|
|
|
MergerMinIterHeap minHeap_;
|
2016-10-24 22:13:01 +02:00
|
|
|
bool prefix_seek_mode_;
|
|
|
|
|
2015-07-06 13:24:09 +02:00
|
|
|
// Max heap is used for reverse iteration, which is way less common than
|
|
|
|
// forward. Lazily initialize it to save memory.
|
|
|
|
std::unique_ptr<MergerMaxIterHeap> maxHeap_;
|
2016-04-26 21:41:07 +02:00
|
|
|
PinnedIteratorsManager* pinned_iters_mgr_;
|
2012-12-26 20:51:36 +01:00
|
|
|
|
2015-07-06 13:24:09 +02:00
|
|
|
IteratorWrapper* CurrentForward() const {
|
|
|
|
assert(direction_ == kForward);
|
|
|
|
return !minHeap_.empty() ? minHeap_.top() : nullptr;
|
2015-07-06 13:24:09 +02:00
|
|
|
}
|
|
|
|
|
2015-07-06 13:24:09 +02:00
|
|
|
IteratorWrapper* CurrentReverse() const {
|
|
|
|
assert(direction_ == kReverse);
|
|
|
|
assert(maxHeap_);
|
|
|
|
return !maxHeap_->empty() ? maxHeap_->top() : nullptr;
|
2015-07-06 13:24:09 +02:00
|
|
|
}
|
2015-07-06 13:24:09 +02:00
|
|
|
};
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2015-07-07 23:45:20 +02:00
|
|
|
void MergingIterator::ClearHeaps() {
|
2015-07-06 13:24:09 +02:00
|
|
|
minHeap_.clear();
|
|
|
|
if (maxHeap_) {
|
|
|
|
maxHeap_->clear();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
void MergingIterator::InitMaxHeap() {
|
|
|
|
if (!maxHeap_) {
|
|
|
|
maxHeap_.reset(new MergerMaxIterHeap(comparator_));
|
|
|
|
}
|
2015-07-07 23:45:20 +02:00
|
|
|
}
|
|
|
|
|
2017-09-11 20:53:22 +02:00
|
|
|
InternalIterator* NewMergingIterator(const InternalKeyComparator* cmp,
|
2016-09-08 23:45:32 +02:00
|
|
|
InternalIterator** list, int n,
|
2016-10-24 22:13:01 +02:00
|
|
|
Arena* arena, bool prefix_seek_mode) {
|
2011-03-18 23:37:00 +01:00
|
|
|
assert(n >= 0);
|
|
|
|
if (n == 0) {
|
2015-10-13 00:06:38 +02:00
|
|
|
return NewEmptyInternalIterator(arena);
|
2011-03-18 23:37:00 +01:00
|
|
|
} else if (n == 1) {
|
|
|
|
return list[0];
|
|
|
|
} else {
|
In DB::NewIterator(), try to allocate the whole iterator tree in an arena
Summary:
In this patch, try to allocate the whole iterator tree starting from DBIter from an arena
1. ArenaWrappedDBIter is created when serves as the entry point of an iterator tree, with an arena in it.
2. Add an option to create iterator from arena for following iterators: DBIter, MergingIterator, MemtableIterator, all mem table's iterators, all table reader's iterators and two level iterator.
3. MergeIteratorBuilder is created to incrementally build the tree of internal iterators. It is passed to mem table list and version set and add iterators to it.
Limitations:
(1) Only DB::NewIterator() without tailing uses the arena. Other cases, including readonly DB and compactions are still from malloc
(2) Two level iterator itself is allocated in arena, but not iterators inside it.
Test Plan: make all check
Reviewers: ljin, haobo
Reviewed By: haobo
Subscribers: leveldb, dhruba, yhchiang, igor
Differential Revision: https://reviews.facebook.net/D18513
2014-06-03 01:38:00 +02:00
|
|
|
if (arena == nullptr) {
|
2016-10-24 22:13:01 +02:00
|
|
|
return new MergingIterator(cmp, list, n, false, prefix_seek_mode);
|
In DB::NewIterator(), try to allocate the whole iterator tree in an arena
Summary:
In this patch, try to allocate the whole iterator tree starting from DBIter from an arena
1. ArenaWrappedDBIter is created when serves as the entry point of an iterator tree, with an arena in it.
2. Add an option to create iterator from arena for following iterators: DBIter, MergingIterator, MemtableIterator, all mem table's iterators, all table reader's iterators and two level iterator.
3. MergeIteratorBuilder is created to incrementally build the tree of internal iterators. It is passed to mem table list and version set and add iterators to it.
Limitations:
(1) Only DB::NewIterator() without tailing uses the arena. Other cases, including readonly DB and compactions are still from malloc
(2) Two level iterator itself is allocated in arena, but not iterators inside it.
Test Plan: make all check
Reviewers: ljin, haobo
Reviewed By: haobo
Subscribers: leveldb, dhruba, yhchiang, igor
Differential Revision: https://reviews.facebook.net/D18513
2014-06-03 01:38:00 +02:00
|
|
|
} else {
|
|
|
|
auto mem = arena->AllocateAligned(sizeof(MergingIterator));
|
2016-10-24 22:13:01 +02:00
|
|
|
return new (mem) MergingIterator(cmp, list, n, true, prefix_seek_mode);
|
In DB::NewIterator(), try to allocate the whole iterator tree in an arena
Summary:
In this patch, try to allocate the whole iterator tree starting from DBIter from an arena
1. ArenaWrappedDBIter is created when serves as the entry point of an iterator tree, with an arena in it.
2. Add an option to create iterator from arena for following iterators: DBIter, MergingIterator, MemtableIterator, all mem table's iterators, all table reader's iterators and two level iterator.
3. MergeIteratorBuilder is created to incrementally build the tree of internal iterators. It is passed to mem table list and version set and add iterators to it.
Limitations:
(1) Only DB::NewIterator() without tailing uses the arena. Other cases, including readonly DB and compactions are still from malloc
(2) Two level iterator itself is allocated in arena, but not iterators inside it.
Test Plan: make all check
Reviewers: ljin, haobo
Reviewed By: haobo
Subscribers: leveldb, dhruba, yhchiang, igor
Differential Revision: https://reviews.facebook.net/D18513
2014-06-03 01:38:00 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-09-11 20:53:22 +02:00
|
|
|
MergeIteratorBuilder::MergeIteratorBuilder(
|
|
|
|
const InternalKeyComparator* comparator, Arena* a, bool prefix_seek_mode)
|
In DB::NewIterator(), try to allocate the whole iterator tree in an arena
Summary:
In this patch, try to allocate the whole iterator tree starting from DBIter from an arena
1. ArenaWrappedDBIter is created when serves as the entry point of an iterator tree, with an arena in it.
2. Add an option to create iterator from arena for following iterators: DBIter, MergingIterator, MemtableIterator, all mem table's iterators, all table reader's iterators and two level iterator.
3. MergeIteratorBuilder is created to incrementally build the tree of internal iterators. It is passed to mem table list and version set and add iterators to it.
Limitations:
(1) Only DB::NewIterator() without tailing uses the arena. Other cases, including readonly DB and compactions are still from malloc
(2) Two level iterator itself is allocated in arena, but not iterators inside it.
Test Plan: make all check
Reviewers: ljin, haobo
Reviewed By: haobo
Subscribers: leveldb, dhruba, yhchiang, igor
Differential Revision: https://reviews.facebook.net/D18513
2014-06-03 01:38:00 +02:00
|
|
|
: first_iter(nullptr), use_merging_iter(false), arena(a) {
|
|
|
|
auto mem = arena->AllocateAligned(sizeof(MergingIterator));
|
2016-10-24 22:13:01 +02:00
|
|
|
merge_iter =
|
|
|
|
new (mem) MergingIterator(comparator, nullptr, 0, true, prefix_seek_mode);
|
In DB::NewIterator(), try to allocate the whole iterator tree in an arena
Summary:
In this patch, try to allocate the whole iterator tree starting from DBIter from an arena
1. ArenaWrappedDBIter is created when serves as the entry point of an iterator tree, with an arena in it.
2. Add an option to create iterator from arena for following iterators: DBIter, MergingIterator, MemtableIterator, all mem table's iterators, all table reader's iterators and two level iterator.
3. MergeIteratorBuilder is created to incrementally build the tree of internal iterators. It is passed to mem table list and version set and add iterators to it.
Limitations:
(1) Only DB::NewIterator() without tailing uses the arena. Other cases, including readonly DB and compactions are still from malloc
(2) Two level iterator itself is allocated in arena, but not iterators inside it.
Test Plan: make all check
Reviewers: ljin, haobo
Reviewed By: haobo
Subscribers: leveldb, dhruba, yhchiang, igor
Differential Revision: https://reviews.facebook.net/D18513
2014-06-03 01:38:00 +02:00
|
|
|
}
|
|
|
|
|
2015-10-13 00:06:38 +02:00
|
|
|
void MergeIteratorBuilder::AddIterator(InternalIterator* iter) {
|
In DB::NewIterator(), try to allocate the whole iterator tree in an arena
Summary:
In this patch, try to allocate the whole iterator tree starting from DBIter from an arena
1. ArenaWrappedDBIter is created when serves as the entry point of an iterator tree, with an arena in it.
2. Add an option to create iterator from arena for following iterators: DBIter, MergingIterator, MemtableIterator, all mem table's iterators, all table reader's iterators and two level iterator.
3. MergeIteratorBuilder is created to incrementally build the tree of internal iterators. It is passed to mem table list and version set and add iterators to it.
Limitations:
(1) Only DB::NewIterator() without tailing uses the arena. Other cases, including readonly DB and compactions are still from malloc
(2) Two level iterator itself is allocated in arena, but not iterators inside it.
Test Plan: make all check
Reviewers: ljin, haobo
Reviewed By: haobo
Subscribers: leveldb, dhruba, yhchiang, igor
Differential Revision: https://reviews.facebook.net/D18513
2014-06-03 01:38:00 +02:00
|
|
|
if (!use_merging_iter && first_iter != nullptr) {
|
|
|
|
merge_iter->AddIterator(first_iter);
|
|
|
|
use_merging_iter = true;
|
|
|
|
}
|
|
|
|
if (use_merging_iter) {
|
|
|
|
merge_iter->AddIterator(iter);
|
|
|
|
} else {
|
|
|
|
first_iter = iter;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-10-13 00:06:38 +02:00
|
|
|
InternalIterator* MergeIteratorBuilder::Finish() {
|
In DB::NewIterator(), try to allocate the whole iterator tree in an arena
Summary:
In this patch, try to allocate the whole iterator tree starting from DBIter from an arena
1. ArenaWrappedDBIter is created when serves as the entry point of an iterator tree, with an arena in it.
2. Add an option to create iterator from arena for following iterators: DBIter, MergingIterator, MemtableIterator, all mem table's iterators, all table reader's iterators and two level iterator.
3. MergeIteratorBuilder is created to incrementally build the tree of internal iterators. It is passed to mem table list and version set and add iterators to it.
Limitations:
(1) Only DB::NewIterator() without tailing uses the arena. Other cases, including readonly DB and compactions are still from malloc
(2) Two level iterator itself is allocated in arena, but not iterators inside it.
Test Plan: make all check
Reviewers: ljin, haobo
Reviewed By: haobo
Subscribers: leveldb, dhruba, yhchiang, igor
Differential Revision: https://reviews.facebook.net/D18513
2014-06-03 01:38:00 +02:00
|
|
|
if (!use_merging_iter) {
|
|
|
|
return first_iter;
|
|
|
|
} else {
|
|
|
|
auto ret = merge_iter;
|
|
|
|
merge_iter = nullptr;
|
|
|
|
return ret;
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-10-04 06:49:15 +02:00
|
|
|
} // namespace rocksdb
|