2016-02-10 00:12:00 +01:00
|
|
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
2017-07-16 01:03:42 +02:00
|
|
|
// This source code is licensed under both the GPLv2 (found in the
|
|
|
|
// COPYING file in the root directory) and Apache 2.0 License
|
|
|
|
// (found in the LICENSE.Apache file in the root directory).
|
2013-10-16 23:59:46 +02:00
|
|
|
//
|
2011-03-18 23:37:00 +01:00
|
|
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
|
|
|
// Use of this source code is governed by a BSD-style license that can be
|
|
|
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
|
|
|
//
|
|
|
|
// Decodes the blocks generated by block_builder.cc.
|
|
|
|
|
|
|
|
#include "table/block.h"
|
|
|
|
#include <algorithm>
|
2014-04-10 23:19:43 +02:00
|
|
|
#include <string>
|
|
|
|
#include <unordered_map>
|
|
|
|
#include <vector>
|
|
|
|
|
2017-04-06 04:02:00 +02:00
|
|
|
#include "monitoring/perf_context_imp.h"
|
2016-08-27 03:55:58 +02:00
|
|
|
#include "port/port.h"
|
|
|
|
#include "port/stack_trace.h"
|
2013-08-23 17:38:13 +02:00
|
|
|
#include "rocksdb/comparator.h"
|
2014-06-13 04:03:22 +02:00
|
|
|
#include "table/block_prefix_index.h"
|
2016-08-27 03:55:58 +02:00
|
|
|
#include "table/format.h"
|
2011-03-18 23:37:00 +01:00
|
|
|
#include "util/coding.h"
|
|
|
|
#include "util/logging.h"
|
|
|
|
|
2013-10-04 06:49:15 +02:00
|
|
|
namespace rocksdb {
|
2011-03-18 23:37:00 +01:00
|
|
|
|
|
|
|
// Helper routine: decode the next block entry starting at "p",
|
|
|
|
// storing the number of shared key bytes, non_shared key bytes,
|
|
|
|
// and the length of the value in "*shared", "*non_shared", and
|
|
|
|
// "*value_length", respectively. Will not derefence past "limit".
|
|
|
|
//
|
2013-03-01 03:04:58 +01:00
|
|
|
// If any errors are detected, returns nullptr. Otherwise, returns a
|
2011-03-18 23:37:00 +01:00
|
|
|
// pointer to the key delta (just past the three decoded values).
|
2018-08-10 01:49:45 +02:00
|
|
|
struct DecodeEntry {
|
|
|
|
inline const char* operator()(const char* p, const char* limit,
|
|
|
|
uint32_t* shared, uint32_t* non_shared,
|
|
|
|
uint32_t* value_length) {
|
|
|
|
// We need 2 bytes for shared and non_shared size. We also need one more
|
|
|
|
// byte either for value size or the actual value in case of value delta
|
|
|
|
// encoding.
|
|
|
|
assert(limit - p >= 3);
|
|
|
|
*shared = reinterpret_cast<const unsigned char*>(p)[0];
|
|
|
|
*non_shared = reinterpret_cast<const unsigned char*>(p)[1];
|
|
|
|
*value_length = reinterpret_cast<const unsigned char*>(p)[2];
|
|
|
|
if ((*shared | *non_shared | *value_length) < 128) {
|
|
|
|
// Fast path: all three values are encoded in one byte each
|
|
|
|
p += 3;
|
|
|
|
} else {
|
|
|
|
if ((p = GetVarint32Ptr(p, limit, shared)) == nullptr) return nullptr;
|
|
|
|
if ((p = GetVarint32Ptr(p, limit, non_shared)) == nullptr) return nullptr;
|
|
|
|
if ((p = GetVarint32Ptr(p, limit, value_length)) == nullptr) {
|
|
|
|
return nullptr;
|
|
|
|
}
|
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2018-08-10 01:49:45 +02:00
|
|
|
// Using an assert in place of "return null" since we should not pay the
|
|
|
|
// cost of checking for corruption on every single key decoding
|
|
|
|
assert(!(static_cast<uint32_t>(limit - p) < (*non_shared + *value_length)));
|
|
|
|
return p;
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
struct DecodeKey {
|
|
|
|
inline const char* operator()(const char* p, const char* limit,
|
|
|
|
uint32_t* shared, uint32_t* non_shared) {
|
|
|
|
uint32_t value_length;
|
|
|
|
return DecodeEntry()(p, limit, shared, non_shared, &value_length);
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
// In format_version 4, which is used by index blocks, the value size is not
|
|
|
|
// encoded before the entry, as the value is known to be the handle with the
|
|
|
|
// known size.
|
|
|
|
struct DecodeKeyV4 {
|
|
|
|
inline const char* operator()(const char* p, const char* limit,
|
|
|
|
uint32_t* shared, uint32_t* non_shared) {
|
|
|
|
// We need 2 bytes for shared and non_shared size. We also need one more
|
|
|
|
// byte either for value size or the actual value in case of value delta
|
|
|
|
// encoding.
|
|
|
|
if (limit - p < 3) return nullptr;
|
|
|
|
*shared = reinterpret_cast<const unsigned char*>(p)[0];
|
|
|
|
*non_shared = reinterpret_cast<const unsigned char*>(p)[1];
|
|
|
|
if ((*shared | *non_shared) < 128) {
|
|
|
|
// Fast path: all three values are encoded in one byte each
|
|
|
|
p += 2;
|
|
|
|
} else {
|
|
|
|
if ((p = GetVarint32Ptr(p, limit, shared)) == nullptr) return nullptr;
|
|
|
|
if ((p = GetVarint32Ptr(p, limit, non_shared)) == nullptr) return nullptr;
|
|
|
|
}
|
|
|
|
return p;
|
2011-04-21 00:48:11 +02:00
|
|
|
}
|
2018-08-10 01:49:45 +02:00
|
|
|
};
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2018-07-17 02:10:44 +02:00
|
|
|
void DataBlockIter::Next() {
|
2014-07-31 01:34:35 +02:00
|
|
|
assert(Valid());
|
2018-07-17 02:10:44 +02:00
|
|
|
ParseNextDataKey();
|
2014-07-31 01:34:35 +02:00
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2018-07-17 02:10:44 +02:00
|
|
|
void IndexBlockIter::Next() {
|
|
|
|
assert(Valid());
|
|
|
|
ParseNextIndexKey();
|
|
|
|
}
|
|
|
|
|
|
|
|
void IndexBlockIter::Prev() {
|
|
|
|
assert(Valid());
|
|
|
|
// Scan backwards to a restart point before current_
|
|
|
|
const uint32_t original = current_;
|
|
|
|
while (GetRestartPoint(restart_index_) >= original) {
|
|
|
|
if (restart_index_ == 0) {
|
|
|
|
// No more entries
|
|
|
|
current_ = restarts_;
|
|
|
|
restart_index_ = num_restarts_;
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
restart_index_--;
|
|
|
|
}
|
|
|
|
SeekToRestartPoint(restart_index_);
|
|
|
|
do {
|
|
|
|
if (!ParseNextIndexKey()) {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
// Loop until end of current entry hits the start of original entry
|
|
|
|
} while (NextEntryOffset() < original);
|
|
|
|
}
|
|
|
|
|
|
|
|
// Similar to IndexBlockIter::Prev but also caches the prev entries
|
|
|
|
void DataBlockIter::Prev() {
|
2014-07-31 01:34:35 +02:00
|
|
|
assert(Valid());
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2016-06-14 21:27:46 +02:00
|
|
|
assert(prev_entries_idx_ == -1 ||
|
|
|
|
static_cast<size_t>(prev_entries_idx_) < prev_entries_.size());
|
|
|
|
// Check if we can use cached prev_entries_
|
|
|
|
if (prev_entries_idx_ > 0 &&
|
|
|
|
prev_entries_[prev_entries_idx_].offset == current_) {
|
|
|
|
// Read cached CachedPrevEntry
|
|
|
|
prev_entries_idx_--;
|
|
|
|
const CachedPrevEntry& current_prev_entry =
|
|
|
|
prev_entries_[prev_entries_idx_];
|
|
|
|
|
2016-07-09 02:50:51 +02:00
|
|
|
const char* key_ptr = nullptr;
|
2016-06-14 21:27:46 +02:00
|
|
|
if (current_prev_entry.key_ptr != nullptr) {
|
|
|
|
// The key is not delta encoded and stored in the data block
|
|
|
|
key_ptr = current_prev_entry.key_ptr;
|
|
|
|
key_pinned_ = true;
|
|
|
|
} else {
|
|
|
|
// The key is delta encoded and stored in prev_entries_keys_buff_
|
|
|
|
key_ptr = prev_entries_keys_buff_.data() + current_prev_entry.key_offset;
|
|
|
|
key_pinned_ = false;
|
|
|
|
}
|
|
|
|
const Slice current_key(key_ptr, current_prev_entry.key_size);
|
|
|
|
|
|
|
|
current_ = current_prev_entry.offset;
|
2018-07-17 02:10:44 +02:00
|
|
|
key_.SetKey(current_key, false /* copy */);
|
2016-06-14 21:27:46 +02:00
|
|
|
value_ = current_prev_entry.value;
|
|
|
|
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Clear prev entries cache
|
|
|
|
prev_entries_idx_ = -1;
|
|
|
|
prev_entries_.clear();
|
|
|
|
prev_entries_keys_buff_.clear();
|
|
|
|
|
2014-07-31 01:34:35 +02:00
|
|
|
// Scan backwards to a restart point before current_
|
|
|
|
const uint32_t original = current_;
|
|
|
|
while (GetRestartPoint(restart_index_) >= original) {
|
|
|
|
if (restart_index_ == 0) {
|
|
|
|
// No more entries
|
|
|
|
current_ = restarts_;
|
|
|
|
restart_index_ = num_restarts_;
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
restart_index_--;
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2014-07-31 01:34:35 +02:00
|
|
|
SeekToRestartPoint(restart_index_);
|
2016-06-14 21:27:46 +02:00
|
|
|
|
2014-07-31 01:34:35 +02:00
|
|
|
do {
|
2018-07-17 02:10:44 +02:00
|
|
|
if (!ParseNextDataKey()) {
|
2016-06-14 21:27:46 +02:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
Slice current_key = key();
|
|
|
|
|
|
|
|
if (key_.IsKeyPinned()) {
|
|
|
|
// The key is not delta encoded
|
|
|
|
prev_entries_.emplace_back(current_, current_key.data(), 0,
|
|
|
|
current_key.size(), value());
|
|
|
|
} else {
|
|
|
|
// The key is delta encoded, cache decoded key in buffer
|
|
|
|
size_t new_key_offset = prev_entries_keys_buff_.size();
|
|
|
|
prev_entries_keys_buff_.append(current_key.data(), current_key.size());
|
|
|
|
|
|
|
|
prev_entries_.emplace_back(current_, nullptr, new_key_offset,
|
|
|
|
current_key.size(), value());
|
|
|
|
}
|
2014-07-31 01:34:35 +02:00
|
|
|
// Loop until end of current entry hits the start of original entry
|
2016-06-14 21:27:46 +02:00
|
|
|
} while (NextEntryOffset() < original);
|
2016-06-15 09:24:33 +02:00
|
|
|
prev_entries_idx_ = static_cast<int32_t>(prev_entries_.size()) - 1;
|
2014-07-31 01:34:35 +02:00
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2018-07-16 18:58:58 +02:00
|
|
|
void DataBlockIter::Seek(const Slice& target) {
|
2018-07-13 02:19:57 +02:00
|
|
|
Slice seek_key = target;
|
|
|
|
PERF_TIMER_GUARD(block_seek_nanos);
|
|
|
|
if (data_ == nullptr) { // Not init yet
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
uint32_t index = 0;
|
2018-08-10 01:49:45 +02:00
|
|
|
bool ok = BinarySeek<DecodeKey>(seek_key, 0, num_restarts_ - 1, &index,
|
|
|
|
comparator_);
|
2018-07-13 02:19:57 +02:00
|
|
|
|
|
|
|
if (!ok) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
SeekToRestartPoint(index);
|
|
|
|
// Linear search (within restart block) for first key >= target
|
|
|
|
|
|
|
|
while (true) {
|
2018-07-17 02:10:44 +02:00
|
|
|
if (!ParseNextDataKey() || Compare(key_, seek_key) >= 0) {
|
2018-07-13 02:19:57 +02:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
void IndexBlockIter::Seek(const Slice& target) {
|
2018-05-26 03:41:31 +02:00
|
|
|
Slice seek_key = target;
|
|
|
|
if (!key_includes_seq_) {
|
|
|
|
seek_key = ExtractUserKey(target);
|
|
|
|
}
|
2015-07-09 01:34:48 +02:00
|
|
|
PERF_TIMER_GUARD(block_seek_nanos);
|
2014-07-31 01:34:35 +02:00
|
|
|
if (data_ == nullptr) { // Not init yet
|
|
|
|
return;
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
2014-07-31 01:34:35 +02:00
|
|
|
uint32_t index = 0;
|
|
|
|
bool ok = false;
|
|
|
|
if (prefix_index_) {
|
|
|
|
ok = PrefixSeek(target, &index);
|
2018-08-10 01:49:45 +02:00
|
|
|
} else if (value_delta_encoded_) {
|
|
|
|
ok = BinarySeek<DecodeKeyV4>(seek_key, 0, num_restarts_ - 1, &index,
|
|
|
|
active_comparator_);
|
2014-07-31 01:34:35 +02:00
|
|
|
} else {
|
2018-08-10 01:49:45 +02:00
|
|
|
ok = BinarySeek<DecodeKey>(seek_key, 0, num_restarts_ - 1, &index,
|
|
|
|
active_comparator_);
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2014-07-31 01:34:35 +02:00
|
|
|
if (!ok) {
|
|
|
|
return;
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
2014-07-31 01:34:35 +02:00
|
|
|
SeekToRestartPoint(index);
|
|
|
|
// Linear search (within restart block) for first key >= target
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2014-07-31 01:34:35 +02:00
|
|
|
while (true) {
|
2018-07-17 02:10:44 +02:00
|
|
|
if (!ParseNextIndexKey() || Compare(key_, seek_key) >= 0) {
|
2014-04-10 23:19:43 +02:00
|
|
|
return;
|
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
2014-07-31 01:34:35 +02:00
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2018-07-16 18:58:58 +02:00
|
|
|
void DataBlockIter::SeekForPrev(const Slice& target) {
|
2016-09-28 03:20:57 +02:00
|
|
|
PERF_TIMER_GUARD(block_seek_nanos);
|
2018-05-26 03:41:31 +02:00
|
|
|
Slice seek_key = target;
|
2016-09-28 03:20:57 +02:00
|
|
|
if (data_ == nullptr) { // Not init yet
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
uint32_t index = 0;
|
2018-08-10 01:49:45 +02:00
|
|
|
bool ok = BinarySeek<DecodeKey>(seek_key, 0, num_restarts_ - 1, &index,
|
|
|
|
comparator_);
|
2016-09-28 03:20:57 +02:00
|
|
|
|
|
|
|
if (!ok) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
SeekToRestartPoint(index);
|
2018-05-26 03:41:31 +02:00
|
|
|
// Linear search (within restart block) for first key >= seek_key
|
2016-09-28 03:20:57 +02:00
|
|
|
|
2018-07-17 02:10:44 +02:00
|
|
|
while (ParseNextDataKey() && Compare(key_, seek_key) < 0) {
|
2016-09-28 03:20:57 +02:00
|
|
|
}
|
|
|
|
if (!Valid()) {
|
|
|
|
SeekToLast();
|
|
|
|
} else {
|
2018-05-26 03:41:31 +02:00
|
|
|
while (Valid() && Compare(key_, seek_key) > 0) {
|
2016-09-28 03:20:57 +02:00
|
|
|
Prev();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-07-17 02:10:44 +02:00
|
|
|
void DataBlockIter::SeekToFirst() {
|
|
|
|
if (data_ == nullptr) { // Not init yet
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
SeekToRestartPoint(0);
|
|
|
|
ParseNextDataKey();
|
|
|
|
}
|
|
|
|
|
|
|
|
void IndexBlockIter::SeekToFirst() {
|
2014-07-31 01:34:35 +02:00
|
|
|
if (data_ == nullptr) { // Not init yet
|
|
|
|
return;
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
2014-07-31 01:34:35 +02:00
|
|
|
SeekToRestartPoint(0);
|
2018-07-17 02:10:44 +02:00
|
|
|
ParseNextIndexKey();
|
2014-07-31 01:34:35 +02:00
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2018-07-17 02:10:44 +02:00
|
|
|
void DataBlockIter::SeekToLast() {
|
2014-07-31 01:34:35 +02:00
|
|
|
if (data_ == nullptr) { // Not init yet
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
SeekToRestartPoint(num_restarts_ - 1);
|
2018-07-17 02:10:44 +02:00
|
|
|
while (ParseNextDataKey() && NextEntryOffset() < restarts_) {
|
|
|
|
// Keep skipping
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
void IndexBlockIter::SeekToLast() {
|
|
|
|
if (data_ == nullptr) { // Not init yet
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
SeekToRestartPoint(num_restarts_ - 1);
|
|
|
|
while (ParseNextIndexKey() && NextEntryOffset() < restarts_) {
|
2014-07-31 01:34:35 +02:00
|
|
|
// Keep skipping
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
2014-07-31 01:34:35 +02:00
|
|
|
}
|
|
|
|
|
2018-08-10 01:49:45 +02:00
|
|
|
template <class TValue>
|
|
|
|
void BlockIter<TValue>::CorruptionError() {
|
2014-07-31 01:34:35 +02:00
|
|
|
current_ = restarts_;
|
|
|
|
restart_index_ = num_restarts_;
|
|
|
|
status_ = Status::Corruption("bad entry in block");
|
|
|
|
key_.Clear();
|
|
|
|
value_.clear();
|
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2018-07-17 02:10:44 +02:00
|
|
|
bool DataBlockIter::ParseNextDataKey() {
|
2016-09-28 03:20:57 +02:00
|
|
|
current_ = NextEntryOffset();
|
|
|
|
const char* p = data_ + current_;
|
|
|
|
const char* limit = data_ + restarts_; // Restarts come right after data
|
|
|
|
if (p >= limit) {
|
|
|
|
// No more entries to return. Mark as invalid.
|
|
|
|
current_ = restarts_;
|
|
|
|
restart_index_ = num_restarts_;
|
|
|
|
return false;
|
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2016-09-28 03:20:57 +02:00
|
|
|
// Decode next entry
|
|
|
|
uint32_t shared, non_shared, value_length;
|
2018-08-10 01:49:45 +02:00
|
|
|
p = DecodeEntry()(p, limit, &shared, &non_shared, &value_length);
|
2016-09-28 03:20:57 +02:00
|
|
|
if (p == nullptr || key_.Size() < shared) {
|
|
|
|
CorruptionError();
|
|
|
|
return false;
|
|
|
|
} else {
|
|
|
|
if (shared == 0) {
|
|
|
|
// If this key dont share any bytes with prev key then we dont need
|
|
|
|
// to decode it and can use it's address in the block directly.
|
2018-07-17 02:10:44 +02:00
|
|
|
key_.SetKey(Slice(p, non_shared), false /* copy */);
|
2016-09-28 03:20:57 +02:00
|
|
|
key_pinned_ = true;
|
2011-03-18 23:37:00 +01:00
|
|
|
} else {
|
2016-09-28 03:20:57 +02:00
|
|
|
// This key share `shared` bytes with prev key, we need to decode it
|
|
|
|
key_.TrimAppend(shared, p, non_shared);
|
|
|
|
key_pinned_ = false;
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
2016-10-19 01:59:37 +02:00
|
|
|
|
|
|
|
if (global_seqno_ != kDisableGlobalSequenceNumber) {
|
|
|
|
// If we are reading a file with a global sequence number we should
|
2017-05-26 21:05:19 +02:00
|
|
|
// expect that all encoded sequence numbers are zeros and any value
|
2018-07-14 07:40:23 +02:00
|
|
|
// type is kTypeValue, kTypeMerge, kTypeDeletion, or kTypeRangeDeletion.
|
2017-04-04 23:17:16 +02:00
|
|
|
assert(GetInternalKeySeqno(key_.GetInternalKey()) == 0);
|
2017-05-26 21:05:19 +02:00
|
|
|
|
2018-07-17 02:10:44 +02:00
|
|
|
ValueType value_type = ExtractValueType(key_.GetKey());
|
2017-05-26 21:05:19 +02:00
|
|
|
assert(value_type == ValueType::kTypeValue ||
|
|
|
|
value_type == ValueType::kTypeMerge ||
|
2018-07-14 07:40:23 +02:00
|
|
|
value_type == ValueType::kTypeDeletion ||
|
|
|
|
value_type == ValueType::kTypeRangeDeletion);
|
2016-10-19 01:59:37 +02:00
|
|
|
|
|
|
|
if (key_pinned_) {
|
|
|
|
// TODO(tec): Investigate updating the seqno in the loaded block
|
|
|
|
// directly instead of doing a copy and update.
|
|
|
|
|
|
|
|
// We cannot use the key address in the block directly because
|
|
|
|
// we have a global_seqno_ that will overwrite the encoded one.
|
|
|
|
key_.OwnKey();
|
|
|
|
key_pinned_ = false;
|
|
|
|
}
|
|
|
|
|
2017-05-26 21:05:19 +02:00
|
|
|
key_.UpdateInternalKey(global_seqno_, value_type);
|
2016-10-19 01:59:37 +02:00
|
|
|
}
|
|
|
|
|
2016-09-28 03:20:57 +02:00
|
|
|
value_ = Slice(p + non_shared, value_length);
|
2018-08-10 01:49:45 +02:00
|
|
|
if (shared == 0) {
|
|
|
|
while (restart_index_ + 1 < num_restarts_ &&
|
|
|
|
GetRestartPoint(restart_index_ + 1) < current_) {
|
|
|
|
++restart_index_;
|
|
|
|
}
|
2016-09-28 03:20:57 +02:00
|
|
|
}
|
2018-08-10 01:49:45 +02:00
|
|
|
// else we are in the middle of a restart interval and the restart_index_
|
|
|
|
// thus has not changed
|
2016-09-28 03:20:57 +02:00
|
|
|
return true;
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
2016-09-28 03:20:57 +02:00
|
|
|
}
|
2014-06-13 04:03:22 +02:00
|
|
|
|
2018-07-17 02:10:44 +02:00
|
|
|
bool IndexBlockIter::ParseNextIndexKey() {
|
|
|
|
current_ = NextEntryOffset();
|
|
|
|
const char* p = data_ + current_;
|
|
|
|
const char* limit = data_ + restarts_; // Restarts come right after data
|
|
|
|
if (p >= limit) {
|
|
|
|
// No more entries to return. Mark as invalid.
|
|
|
|
current_ = restarts_;
|
|
|
|
restart_index_ = num_restarts_;
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Decode next entry
|
|
|
|
uint32_t shared, non_shared, value_length;
|
2018-08-10 01:49:45 +02:00
|
|
|
if (value_delta_encoded_) {
|
|
|
|
p = DecodeKeyV4()(p, limit, &shared, &non_shared);
|
|
|
|
value_length = 0;
|
|
|
|
} else {
|
|
|
|
p = DecodeEntry()(p, limit, &shared, &non_shared, &value_length);
|
|
|
|
}
|
2018-07-17 02:10:44 +02:00
|
|
|
if (p == nullptr || key_.Size() < shared) {
|
|
|
|
CorruptionError();
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
if (shared == 0) {
|
|
|
|
// If this key dont share any bytes with prev key then we dont need
|
|
|
|
// to decode it and can use it's address in the block directly.
|
|
|
|
key_.SetKey(Slice(p, non_shared), false /* copy */);
|
|
|
|
key_pinned_ = true;
|
|
|
|
} else {
|
|
|
|
// This key share `shared` bytes with prev key, we need to decode it
|
|
|
|
key_.TrimAppend(shared, p, non_shared);
|
|
|
|
key_pinned_ = false;
|
|
|
|
}
|
|
|
|
value_ = Slice(p + non_shared, value_length);
|
2018-08-10 01:49:45 +02:00
|
|
|
if (shared == 0) {
|
|
|
|
while (restart_index_ + 1 < num_restarts_ &&
|
|
|
|
GetRestartPoint(restart_index_ + 1) < current_) {
|
|
|
|
++restart_index_;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
// else we are in the middle of a restart interval and the restart_index_
|
|
|
|
// thus has not changed
|
|
|
|
if (value_delta_encoded_) {
|
|
|
|
assert(value_length == 0);
|
|
|
|
DecodeCurrentValue(shared);
|
2018-07-17 02:10:44 +02:00
|
|
|
}
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2018-08-10 01:49:45 +02:00
|
|
|
// The format:
|
|
|
|
// restart_point 0: k, v (off, sz), k, v (delta-sz), ..., k, v (delta-sz)
|
|
|
|
// restart_point 1: k, v (off, sz), k, v (delta-sz), ..., k, v (delta-sz)
|
|
|
|
// ...
|
|
|
|
// restart_point n-1: k, v (off, sz), k, v (delta-sz), ..., k, v (delta-sz)
|
|
|
|
// where, k is key, v is value, and its encoding is in parenthesis.
|
|
|
|
// The format of each key is (shared_size, non_shared_size, shared, non_shared)
|
|
|
|
// The format of each value, i.e., block hanlde, is (offset, size) whenever the
|
|
|
|
// shared_size is 0, which included the first entry in each restart point.
|
|
|
|
// Otherwise the format is delta-size = block handle size - size of last block
|
|
|
|
// handle.
|
|
|
|
void IndexBlockIter::DecodeCurrentValue(uint32_t shared) {
|
|
|
|
assert(value_delta_encoded_);
|
|
|
|
const char* limit = data_ + restarts_;
|
|
|
|
if (shared == 0) {
|
|
|
|
uint64_t o, s;
|
|
|
|
const char* newp = GetVarint64Ptr(value_.data(), limit, &o);
|
|
|
|
newp = GetVarint64Ptr(newp, limit, &s);
|
|
|
|
decoded_value_ = BlockHandle(o, s);
|
|
|
|
value_ = Slice(value_.data(), newp - value_.data());
|
|
|
|
} else {
|
|
|
|
uint64_t next_value_base =
|
|
|
|
decoded_value_.offset() + decoded_value_.size() + kBlockTrailerSize;
|
|
|
|
int64_t delta;
|
|
|
|
const char* newp = GetVarsignedint64Ptr(value_.data(), limit, &delta);
|
|
|
|
decoded_value_ =
|
|
|
|
BlockHandle(next_value_base, decoded_value_.size() + delta);
|
|
|
|
value_ = Slice(value_.data(), newp - value_.data());
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-09-28 03:20:57 +02:00
|
|
|
// Binary search in restart array to find the first restart point that
|
|
|
|
// is either the last restart point with a key less than target,
|
|
|
|
// which means the key of next restart point is larger than target, or
|
|
|
|
// the first restart point with a key = target
|
2018-08-10 01:49:45 +02:00
|
|
|
template <class TValue>
|
|
|
|
template <typename DecodeKeyFunc>
|
|
|
|
bool BlockIter<TValue>::BinarySeek(const Slice& target, uint32_t left,
|
|
|
|
uint32_t right, uint32_t* index,
|
|
|
|
const Comparator* comp) {
|
2014-07-31 01:34:35 +02:00
|
|
|
assert(left <= right);
|
2014-04-10 23:19:43 +02:00
|
|
|
|
2014-07-31 01:34:35 +02:00
|
|
|
while (left < right) {
|
|
|
|
uint32_t mid = (left + right + 1) / 2;
|
|
|
|
uint32_t region_offset = GetRestartPoint(mid);
|
2018-08-10 01:49:45 +02:00
|
|
|
uint32_t shared, non_shared;
|
|
|
|
const char* key_ptr = DecodeKeyFunc()(
|
|
|
|
data_ + region_offset, data_ + restarts_, &shared, &non_shared);
|
Block::Iter::PrefixSeek() to have an extra check to filter out some false matches
Summary:
In block based table's hash index checking, when looking for a key that doesn't exist, there is a high chance that a false block is returned because of hash bucket conflicts. In this revision, another check is done to filter out some of those cases: comparing previous key of the block boundary to see whether the target block is what we are looking for.
In a favored test setting (bloom filter disabled, 8 L0 files), I saw about 80% improvements. In a non-favored test setting (bloom filter enabled, files are all in L1, files are all cached), I see the performance penalty is less than 3%.
Test Plan: make all check
Reviewers: haobo, ljin
Reviewed By: ljin
Subscribers: wuj, leveldb, zagfox, yhchiang
Differential Revision: https://reviews.facebook.net/D20595
2014-07-25 23:38:18 +02:00
|
|
|
if (key_ptr == nullptr || (shared != 0)) {
|
|
|
|
CorruptionError();
|
2014-07-31 01:34:35 +02:00
|
|
|
return false;
|
|
|
|
}
|
|
|
|
Slice mid_key(key_ptr, non_shared);
|
2018-07-16 18:58:58 +02:00
|
|
|
int cmp = comp->Compare(mid_key, target);
|
2014-07-31 01:34:35 +02:00
|
|
|
if (cmp < 0) {
|
|
|
|
// Key at "mid" is smaller than "target". Therefore all
|
|
|
|
// blocks before "mid" are uninteresting.
|
|
|
|
left = mid;
|
|
|
|
} else if (cmp > 0) {
|
|
|
|
// Key at "mid" is >= "target". Therefore all blocks at or
|
|
|
|
// after "mid" are uninteresting.
|
|
|
|
right = mid - 1;
|
|
|
|
} else {
|
|
|
|
left = right = mid;
|
Block::Iter::PrefixSeek() to have an extra check to filter out some false matches
Summary:
In block based table's hash index checking, when looking for a key that doesn't exist, there is a high chance that a false block is returned because of hash bucket conflicts. In this revision, another check is done to filter out some of those cases: comparing previous key of the block boundary to see whether the target block is what we are looking for.
In a favored test setting (bloom filter disabled, 8 L0 files), I saw about 80% improvements. In a non-favored test setting (bloom filter enabled, files are all in L1, files are all cached), I see the performance penalty is less than 3%.
Test Plan: make all check
Reviewers: haobo, ljin
Reviewed By: ljin
Subscribers: wuj, leveldb, zagfox, yhchiang
Differential Revision: https://reviews.facebook.net/D20595
2014-07-25 23:38:18 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-07-31 01:34:35 +02:00
|
|
|
*index = left;
|
|
|
|
return true;
|
|
|
|
}
|
2014-06-13 04:03:22 +02:00
|
|
|
|
2014-07-31 01:34:35 +02:00
|
|
|
// Compare target key and the block key of the block of `block_index`.
|
2018-07-17 02:10:44 +02:00
|
|
|
// Return -1 if error.
|
2018-07-13 02:19:57 +02:00
|
|
|
int IndexBlockIter::CompareBlockKey(uint32_t block_index, const Slice& target) {
|
2014-07-31 01:34:35 +02:00
|
|
|
uint32_t region_offset = GetRestartPoint(block_index);
|
2018-08-10 01:49:45 +02:00
|
|
|
uint32_t shared, non_shared;
|
|
|
|
const char* key_ptr =
|
|
|
|
value_delta_encoded_
|
|
|
|
? DecodeKeyV4()(data_ + region_offset, data_ + restarts_, &shared,
|
|
|
|
&non_shared)
|
|
|
|
: DecodeKey()(data_ + region_offset, data_ + restarts_, &shared,
|
|
|
|
&non_shared);
|
2014-07-31 01:34:35 +02:00
|
|
|
if (key_ptr == nullptr || (shared != 0)) {
|
|
|
|
CorruptionError();
|
|
|
|
return 1; // Return target is smaller
|
|
|
|
}
|
|
|
|
Slice block_key(key_ptr, non_shared);
|
|
|
|
return Compare(block_key, target);
|
|
|
|
}
|
Block::Iter::PrefixSeek() to have an extra check to filter out some false matches
Summary:
In block based table's hash index checking, when looking for a key that doesn't exist, there is a high chance that a false block is returned because of hash bucket conflicts. In this revision, another check is done to filter out some of those cases: comparing previous key of the block boundary to see whether the target block is what we are looking for.
In a favored test setting (bloom filter disabled, 8 L0 files), I saw about 80% improvements. In a non-favored test setting (bloom filter enabled, files are all in L1, files are all cached), I see the performance penalty is less than 3%.
Test Plan: make all check
Reviewers: haobo, ljin
Reviewed By: ljin
Subscribers: wuj, leveldb, zagfox, yhchiang
Differential Revision: https://reviews.facebook.net/D20595
2014-07-25 23:38:18 +02:00
|
|
|
|
2014-07-31 01:34:35 +02:00
|
|
|
// Binary search in block_ids to find the first block
|
|
|
|
// with a key >= target
|
2018-07-13 02:19:57 +02:00
|
|
|
bool IndexBlockIter::BinaryBlockIndexSeek(const Slice& target,
|
|
|
|
uint32_t* block_ids, uint32_t left,
|
|
|
|
uint32_t right, uint32_t* index) {
|
2014-07-31 01:34:35 +02:00
|
|
|
assert(left <= right);
|
|
|
|
uint32_t left_bound = left;
|
2014-06-13 04:03:22 +02:00
|
|
|
|
2014-07-31 01:34:35 +02:00
|
|
|
while (left <= right) {
|
2016-09-28 03:20:57 +02:00
|
|
|
uint32_t mid = (right + left) / 2;
|
Block::Iter::PrefixSeek() to have an extra check to filter out some false matches
Summary:
In block based table's hash index checking, when looking for a key that doesn't exist, there is a high chance that a false block is returned because of hash bucket conflicts. In this revision, another check is done to filter out some of those cases: comparing previous key of the block boundary to see whether the target block is what we are looking for.
In a favored test setting (bloom filter disabled, 8 L0 files), I saw about 80% improvements. In a non-favored test setting (bloom filter enabled, files are all in L1, files are all cached), I see the performance penalty is less than 3%.
Test Plan: make all check
Reviewers: haobo, ljin
Reviewed By: ljin
Subscribers: wuj, leveldb, zagfox, yhchiang
Differential Revision: https://reviews.facebook.net/D20595
2014-07-25 23:38:18 +02:00
|
|
|
|
2014-07-31 01:34:35 +02:00
|
|
|
int cmp = CompareBlockKey(block_ids[mid], target);
|
|
|
|
if (!status_.ok()) {
|
2014-06-13 04:03:22 +02:00
|
|
|
return false;
|
|
|
|
}
|
2014-07-31 01:34:35 +02:00
|
|
|
if (cmp < 0) {
|
|
|
|
// Key at "target" is larger than "mid". Therefore all
|
|
|
|
// blocks before or at "mid" are uninteresting.
|
|
|
|
left = mid + 1;
|
|
|
|
} else {
|
|
|
|
// Key at "target" is <= "mid". Therefore all blocks
|
|
|
|
// after "mid" are uninteresting.
|
|
|
|
// If there is only one block left, we found it.
|
|
|
|
if (left == right) break;
|
|
|
|
right = mid;
|
|
|
|
}
|
2014-06-13 04:03:22 +02:00
|
|
|
}
|
|
|
|
|
2014-07-31 01:34:35 +02:00
|
|
|
if (left == right) {
|
|
|
|
// In one of the two following cases:
|
|
|
|
// (1) left is the first one of block_ids
|
|
|
|
// (2) there is a gap of blocks between block of `left` and `left-1`.
|
|
|
|
// we can further distinguish the case of key in the block or key not
|
|
|
|
// existing, by comparing the target key and the key of the previous
|
|
|
|
// block to the left of the block found.
|
|
|
|
if (block_ids[left] > 0 &&
|
|
|
|
(left == left_bound || block_ids[left - 1] != block_ids[left] - 1) &&
|
|
|
|
CompareBlockKey(block_ids[left] - 1, target) > 0) {
|
2014-04-10 23:19:43 +02:00
|
|
|
current_ = restarts_;
|
2014-06-13 04:03:22 +02:00
|
|
|
return false;
|
2014-04-10 23:19:43 +02:00
|
|
|
}
|
|
|
|
|
2014-07-31 01:34:35 +02:00
|
|
|
*index = block_ids[left];
|
|
|
|
return true;
|
|
|
|
} else {
|
|
|
|
assert(left > right);
|
|
|
|
// Mark iterator invalid
|
|
|
|
current_ = restarts_;
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-07-13 02:19:57 +02:00
|
|
|
bool IndexBlockIter::PrefixSeek(const Slice& target, uint32_t* index) {
|
2014-07-31 01:34:35 +02:00
|
|
|
assert(prefix_index_);
|
2018-05-26 03:41:31 +02:00
|
|
|
Slice seek_key = target;
|
|
|
|
if (!key_includes_seq_) {
|
|
|
|
seek_key = ExtractUserKey(target);
|
|
|
|
}
|
2014-07-31 01:34:35 +02:00
|
|
|
uint32_t* block_ids = nullptr;
|
|
|
|
uint32_t num_blocks = prefix_index_->GetBlocks(target, &block_ids);
|
2014-06-13 04:03:22 +02:00
|
|
|
|
2014-07-31 01:34:35 +02:00
|
|
|
if (num_blocks == 0) {
|
|
|
|
current_ = restarts_;
|
|
|
|
return false;
|
|
|
|
} else {
|
2018-05-26 03:41:31 +02:00
|
|
|
return BinaryBlockIndexSeek(seek_key, block_ids, 0, num_blocks - 1, index);
|
2014-07-31 01:34:35 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
uint32_t Block::NumRestarts() const {
|
|
|
|
assert(size_ >= 2*sizeof(uint32_t));
|
|
|
|
return DecodeFixed32(data_ + size_ - sizeof(uint32_t));
|
|
|
|
}
|
|
|
|
|
2016-10-19 01:59:37 +02:00
|
|
|
Block::Block(BlockContents&& contents, SequenceNumber _global_seqno,
|
|
|
|
size_t read_amp_bytes_per_bit, Statistics* statistics)
|
2014-09-18 01:45:58 +02:00
|
|
|
: contents_(std::move(contents)),
|
|
|
|
data_(contents_.data.data()),
|
2016-10-19 01:59:37 +02:00
|
|
|
size_(contents_.data.size()),
|
2017-12-07 20:50:49 +01:00
|
|
|
restart_offset_(0),
|
2018-05-18 21:54:09 +02:00
|
|
|
num_restarts_(0),
|
2016-10-19 01:59:37 +02:00
|
|
|
global_seqno_(_global_seqno) {
|
2014-07-31 01:34:35 +02:00
|
|
|
if (size_ < sizeof(uint32_t)) {
|
|
|
|
size_ = 0; // Error marker
|
|
|
|
} else {
|
2018-06-16 04:24:21 +02:00
|
|
|
// Should only decode restart points for uncompressed blocks
|
|
|
|
if (compression_type() == kNoCompression) {
|
|
|
|
num_restarts_ = NumRestarts();
|
|
|
|
restart_offset_ =
|
|
|
|
static_cast<uint32_t>(size_) - (1 + num_restarts_) * sizeof(uint32_t);
|
|
|
|
if (restart_offset_ > size_ - sizeof(uint32_t)) {
|
|
|
|
// The size is too small for NumRestarts() and therefore
|
|
|
|
// restart_offset_ wrapped around.
|
|
|
|
size_ = 0;
|
|
|
|
}
|
2014-06-13 04:03:22 +02:00
|
|
|
}
|
|
|
|
}
|
2016-08-27 03:55:58 +02:00
|
|
|
if (read_amp_bytes_per_bit != 0 && statistics && size_ != 0) {
|
|
|
|
read_amp_bitmap_.reset(new BlockReadAmpBitmap(
|
|
|
|
restart_offset_, read_amp_bytes_per_bit, statistics));
|
|
|
|
}
|
2014-07-31 01:34:35 +02:00
|
|
|
}
|
|
|
|
|
2018-07-13 02:19:57 +02:00
|
|
|
template <>
|
|
|
|
DataBlockIter* Block::NewIterator(const Comparator* cmp, const Comparator* ucmp,
|
|
|
|
DataBlockIter* iter, Statistics* stats,
|
|
|
|
bool /*total_order_seek*/,
|
|
|
|
bool /*key_includes_seq*/,
|
2018-08-10 01:49:45 +02:00
|
|
|
bool /*value_is_full*/,
|
2018-07-13 02:19:57 +02:00
|
|
|
BlockPrefixIndex* /*prefix_index*/) {
|
|
|
|
DataBlockIter* ret_iter;
|
|
|
|
if (iter != nullptr) {
|
|
|
|
ret_iter = iter;
|
|
|
|
} else {
|
|
|
|
ret_iter = new DataBlockIter;
|
|
|
|
}
|
|
|
|
if (size_ < 2 * sizeof(uint32_t)) {
|
|
|
|
ret_iter->Invalidate(Status::Corruption("bad block contents"));
|
|
|
|
return ret_iter;
|
|
|
|
}
|
|
|
|
if (num_restarts_ == 0) {
|
|
|
|
// Empty block.
|
|
|
|
ret_iter->Invalidate(Status::OK());
|
|
|
|
return ret_iter;
|
|
|
|
} else {
|
|
|
|
ret_iter->Initialize(cmp, ucmp, data_, restart_offset_, num_restarts_,
|
|
|
|
global_seqno_, read_amp_bitmap_.get(), cachable());
|
2016-08-27 03:55:58 +02:00
|
|
|
if (read_amp_bitmap_) {
|
|
|
|
if (read_amp_bitmap_->GetStatistics() != stats) {
|
|
|
|
// DB changed the Statistics pointer, we need to notify read_amp_bitmap_
|
|
|
|
read_amp_bitmap_->SetStatistics(stats);
|
|
|
|
}
|
2014-07-31 01:34:35 +02:00
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
2014-07-31 01:34:35 +02:00
|
|
|
|
2018-01-25 23:43:43 +01:00
|
|
|
return ret_iter;
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2018-07-13 02:19:57 +02:00
|
|
|
template <>
|
|
|
|
IndexBlockIter* Block::NewIterator(const Comparator* cmp,
|
|
|
|
const Comparator* ucmp, IndexBlockIter* iter,
|
|
|
|
Statistics* /*stats*/, bool total_order_seek,
|
2018-08-10 01:49:45 +02:00
|
|
|
bool key_includes_seq, bool value_is_full,
|
2018-07-13 02:19:57 +02:00
|
|
|
BlockPrefixIndex* prefix_index) {
|
|
|
|
IndexBlockIter* ret_iter;
|
|
|
|
if (iter != nullptr) {
|
|
|
|
ret_iter = iter;
|
|
|
|
} else {
|
|
|
|
ret_iter = new IndexBlockIter;
|
|
|
|
}
|
|
|
|
if (size_ < 2 * sizeof(uint32_t)) {
|
|
|
|
ret_iter->Invalidate(Status::Corruption("bad block contents"));
|
|
|
|
return ret_iter;
|
|
|
|
}
|
|
|
|
if (num_restarts_ == 0) {
|
|
|
|
// Empty block.
|
|
|
|
ret_iter->Invalidate(Status::OK());
|
|
|
|
return ret_iter;
|
|
|
|
} else {
|
|
|
|
BlockPrefixIndex* prefix_index_ptr =
|
|
|
|
total_order_seek ? nullptr : prefix_index;
|
|
|
|
ret_iter->Initialize(cmp, ucmp, data_, restart_offset_, num_restarts_,
|
2018-08-10 01:49:45 +02:00
|
|
|
prefix_index_ptr, key_includes_seq, value_is_full,
|
|
|
|
cachable());
|
2018-07-13 02:19:57 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
return ret_iter;
|
2014-06-13 04:03:22 +02:00
|
|
|
}
|
|
|
|
|
2014-08-05 20:27:34 +02:00
|
|
|
size_t Block::ApproximateMemoryUsage() const {
|
Use malloc_usable_size() for accounting block cache size
Summary:
Currently, when we insert something into block cache, we say that the block cache capacity decreased by the size of the block. However, size of the block might be less than the actual memory used by this object. For example, 4.5KB block will actually use 8KB of memory. So even if we configure block cache to 10GB, our actually memory usage of block cache will be 20GB!
This problem showed up a lot in testing and just recently also showed up in MongoRocks production where we were using 30GB more memory than expected.
This diff will fix the problem. Instead of counting the block size, we will count memory used by the block. That way, a block cache configured to be 10GB will actually use only 10GB of memory.
I'm using non-portable function and I couldn't find info on portability on Google. However, it seems to work on Linux, which will cover majority of our use-cases.
Test Plan:
1. fill up mongo instance with 80GB of data
2. restart mongo with block cache size configured to 10GB
3. do a table scan in mongo
4. memory usage before the diff: 12GB. memory usage after the diff: 10.5GB
Reviewers: sdong, MarkCallaghan, rven, yhchiang
Reviewed By: yhchiang
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D40635
2015-06-26 20:48:09 +02:00
|
|
|
size_t usage = usable_size();
|
2018-06-29 17:55:33 +02:00
|
|
|
#ifdef ROCKSDB_MALLOC_USABLE_SIZE
|
|
|
|
usage += malloc_usable_size((void*)this);
|
|
|
|
#else
|
|
|
|
usage += sizeof(*this);
|
|
|
|
#endif // ROCKSDB_MALLOC_USABLE_SIZE
|
|
|
|
if (read_amp_bitmap_) {
|
|
|
|
usage += read_amp_bitmap_->ApproximateMemoryUsage();
|
|
|
|
}
|
2014-08-05 20:27:34 +02:00
|
|
|
return usage;
|
|
|
|
}
|
|
|
|
|
2013-10-04 06:49:15 +02:00
|
|
|
} // namespace rocksdb
|