2016-02-10 00:12:00 +01:00
|
|
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
2017-07-16 01:03:42 +02:00
|
|
|
// This source code is licensed under both the GPLv2 (found in the
|
|
|
|
// COPYING file in the root directory) and Apache 2.0 License
|
|
|
|
// (found in the LICENSE.Apache file in the root directory).
|
2013-10-16 23:59:46 +02:00
|
|
|
//
|
2011-03-18 23:37:00 +01:00
|
|
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
|
|
|
// Use of this source code is governed by a BSD-style license that can be
|
|
|
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
|
|
|
|
2013-10-05 07:32:05 +02:00
|
|
|
#pragma once
|
2011-03-18 23:37:00 +01:00
|
|
|
#include <stddef.h>
|
|
|
|
#include <stdint.h>
|
2016-06-14 21:27:46 +02:00
|
|
|
#include <string>
|
|
|
|
#include <vector>
|
Use malloc_usable_size() for accounting block cache size
Summary:
Currently, when we insert something into block cache, we say that the block cache capacity decreased by the size of the block. However, size of the block might be less than the actual memory used by this object. For example, 4.5KB block will actually use 8KB of memory. So even if we configure block cache to 10GB, our actually memory usage of block cache will be 20GB!
This problem showed up a lot in testing and just recently also showed up in MongoRocks production where we were using 30GB more memory than expected.
This diff will fix the problem. Instead of counting the block size, we will count memory used by the block. That way, a block cache configured to be 10GB will actually use only 10GB of memory.
I'm using non-portable function and I couldn't find info on portability on Google. However, it seems to work on Linux, which will cover majority of our use-cases.
Test Plan:
1. fill up mongo instance with 80GB of data
2. restart mongo with block cache size configured to 10GB
3. do a table scan in mongo
4. memory usage before the diff: 12GB. memory usage after the diff: 10.5GB
Reviewers: sdong, MarkCallaghan, rven, yhchiang
Reviewed By: yhchiang
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D40635
2015-06-26 20:48:09 +02:00
|
|
|
#ifdef ROCKSDB_MALLOC_USABLE_SIZE
|
2016-10-28 19:44:52 +02:00
|
|
|
#ifdef OS_FREEBSD
|
|
|
|
#include <malloc_np.h>
|
|
|
|
#else
|
Use malloc_usable_size() for accounting block cache size
Summary:
Currently, when we insert something into block cache, we say that the block cache capacity decreased by the size of the block. However, size of the block might be less than the actual memory used by this object. For example, 4.5KB block will actually use 8KB of memory. So even if we configure block cache to 10GB, our actually memory usage of block cache will be 20GB!
This problem showed up a lot in testing and just recently also showed up in MongoRocks production where we were using 30GB more memory than expected.
This diff will fix the problem. Instead of counting the block size, we will count memory used by the block. That way, a block cache configured to be 10GB will actually use only 10GB of memory.
I'm using non-portable function and I couldn't find info on portability on Google. However, it seems to work on Linux, which will cover majority of our use-cases.
Test Plan:
1. fill up mongo instance with 80GB of data
2. restart mongo with block cache size configured to 10GB
3. do a table scan in mongo
4. memory usage before the diff: 12GB. memory usage after the diff: 10.5GB
Reviewers: sdong, MarkCallaghan, rven, yhchiang
Reviewed By: yhchiang
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D40635
2015-06-26 20:48:09 +02:00
|
|
|
#include <malloc.h>
|
|
|
|
#endif
|
2016-10-28 19:44:52 +02:00
|
|
|
#endif
|
2014-04-10 23:19:43 +02:00
|
|
|
|
2016-04-26 21:41:07 +02:00
|
|
|
#include "db/dbformat.h"
|
|
|
|
#include "db/pinned_iterators_manager.h"
|
2013-08-23 17:38:13 +02:00
|
|
|
#include "rocksdb/iterator.h"
|
2013-09-02 08:23:40 +02:00
|
|
|
#include "rocksdb/options.h"
|
2016-08-27 03:55:58 +02:00
|
|
|
#include "rocksdb/statistics.h"
|
2018-08-15 23:27:47 +02:00
|
|
|
#include "rocksdb/table.h"
|
2019-05-30 23:47:29 +02:00
|
|
|
#include "table/block_based/block_prefix_index.h"
|
|
|
|
#include "table/block_based/data_block_hash_index.h"
|
2019-05-31 02:39:43 +02:00
|
|
|
#include "table/format.h"
|
2015-10-13 00:06:38 +02:00
|
|
|
#include "table/internal_iterator.h"
|
2019-05-30 20:21:38 +02:00
|
|
|
#include "test_util/sync_point.h"
|
2019-05-31 02:39:43 +02:00
|
|
|
#include "util/random.h"
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2013-10-04 06:49:15 +02:00
|
|
|
namespace rocksdb {
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
struct BlockContents;
|
2011-03-18 23:37:00 +01:00
|
|
|
class Comparator;
|
2018-08-10 01:49:45 +02:00
|
|
|
template <class TValue>
|
2014-07-31 01:34:35 +02:00
|
|
|
class BlockIter;
|
2018-07-13 02:19:57 +02:00
|
|
|
class DataBlockIter;
|
|
|
|
class IndexBlockIter;
|
2014-06-13 04:03:22 +02:00
|
|
|
class BlockPrefixIndex;
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2016-08-27 03:55:58 +02:00
|
|
|
// BlockReadAmpBitmap is a bitmap that map the rocksdb::Block data bytes to
|
|
|
|
// a bitmap with ratio bytes_per_bit. Whenever we access a range of bytes in
|
|
|
|
// the Block we update the bitmap and increment READ_AMP_ESTIMATE_USEFUL_BYTES.
|
|
|
|
class BlockReadAmpBitmap {
|
|
|
|
public:
|
|
|
|
explicit BlockReadAmpBitmap(size_t block_size, size_t bytes_per_bit,
|
|
|
|
Statistics* statistics)
|
unbiase readamp bitmap
Summary:
Consider BlockReadAmpBitmap with bytes_per_bit = 32. Suppose bytes [a, b) were used, while bytes [a-32, a)
and [b+1, b+33) weren't used; more formally, the union of ranges passed to BlockReadAmpBitmap::Mark() contains [a, b) and doesn't intersect with [a-32, a) and [b+1, b+33). Then bits [floor(a/32), ceil(b/32)] will be set, and so the number of useful bytes will be estimated as (ceil(b/32) - floor(a/32)) * 32, which is on average equal to b-a+31.
An extreme example: if we use 1 byte from each block, it'll be counted as 32 bytes from each block.
It's easy to remove this bias by slightly changing the semantics of the bitmap. Currently each bit represents a byte range [i*32, (i+1)*32).
This diff makes each bit represent a single byte: i*32 + X, where X is a random number in [0, 31] generated when bitmap is created. So, e.g., if you read a single byte at random, with probability 31/32 it won't be counted at all, and with probability 1/32 it will be counted as 32 bytes; so, on average it's counted as 1 byte.
*But there is one exception: the last bit will always set with the old way.*
(*) - assuming read_amp_bytes_per_bit = 32.
Closes https://github.com/facebook/rocksdb/pull/2259
Differential Revision: D5035652
Pulled By: lightmark
fbshipit-source-id: bd98b1b9b49fbe61f9e3781d07f624e3cbd92356
2017-05-10 10:32:52 +02:00
|
|
|
: bitmap_(nullptr),
|
|
|
|
bytes_per_bit_pow_(0),
|
|
|
|
statistics_(statistics),
|
2019-03-28 00:13:08 +01:00
|
|
|
rnd_(Random::GetTLSInstance()->Uniform(
|
|
|
|
static_cast<int>(bytes_per_bit))) {
|
unbiase readamp bitmap
Summary:
Consider BlockReadAmpBitmap with bytes_per_bit = 32. Suppose bytes [a, b) were used, while bytes [a-32, a)
and [b+1, b+33) weren't used; more formally, the union of ranges passed to BlockReadAmpBitmap::Mark() contains [a, b) and doesn't intersect with [a-32, a) and [b+1, b+33). Then bits [floor(a/32), ceil(b/32)] will be set, and so the number of useful bytes will be estimated as (ceil(b/32) - floor(a/32)) * 32, which is on average equal to b-a+31.
An extreme example: if we use 1 byte from each block, it'll be counted as 32 bytes from each block.
It's easy to remove this bias by slightly changing the semantics of the bitmap. Currently each bit represents a byte range [i*32, (i+1)*32).
This diff makes each bit represent a single byte: i*32 + X, where X is a random number in [0, 31] generated when bitmap is created. So, e.g., if you read a single byte at random, with probability 31/32 it won't be counted at all, and with probability 1/32 it will be counted as 32 bytes; so, on average it's counted as 1 byte.
*But there is one exception: the last bit will always set with the old way.*
(*) - assuming read_amp_bytes_per_bit = 32.
Closes https://github.com/facebook/rocksdb/pull/2259
Differential Revision: D5035652
Pulled By: lightmark
fbshipit-source-id: bd98b1b9b49fbe61f9e3781d07f624e3cbd92356
2017-05-10 10:32:52 +02:00
|
|
|
TEST_SYNC_POINT_CALLBACK("BlockReadAmpBitmap:rnd", &rnd_);
|
2016-08-27 03:55:58 +02:00
|
|
|
assert(block_size > 0 && bytes_per_bit > 0);
|
|
|
|
|
|
|
|
// convert bytes_per_bit to be a power of 2
|
|
|
|
while (bytes_per_bit >>= 1) {
|
|
|
|
bytes_per_bit_pow_++;
|
|
|
|
}
|
|
|
|
|
|
|
|
// num_bits_needed = ceil(block_size / bytes_per_bit)
|
2019-03-28 00:13:08 +01:00
|
|
|
size_t num_bits_needed = ((block_size - 1) >> bytes_per_bit_pow_) + 1;
|
unbiase readamp bitmap
Summary:
Consider BlockReadAmpBitmap with bytes_per_bit = 32. Suppose bytes [a, b) were used, while bytes [a-32, a)
and [b+1, b+33) weren't used; more formally, the union of ranges passed to BlockReadAmpBitmap::Mark() contains [a, b) and doesn't intersect with [a-32, a) and [b+1, b+33). Then bits [floor(a/32), ceil(b/32)] will be set, and so the number of useful bytes will be estimated as (ceil(b/32) - floor(a/32)) * 32, which is on average equal to b-a+31.
An extreme example: if we use 1 byte from each block, it'll be counted as 32 bytes from each block.
It's easy to remove this bias by slightly changing the semantics of the bitmap. Currently each bit represents a byte range [i*32, (i+1)*32).
This diff makes each bit represent a single byte: i*32 + X, where X is a random number in [0, 31] generated when bitmap is created. So, e.g., if you read a single byte at random, with probability 31/32 it won't be counted at all, and with probability 1/32 it will be counted as 32 bytes; so, on average it's counted as 1 byte.
*But there is one exception: the last bit will always set with the old way.*
(*) - assuming read_amp_bytes_per_bit = 32.
Closes https://github.com/facebook/rocksdb/pull/2259
Differential Revision: D5035652
Pulled By: lightmark
fbshipit-source-id: bd98b1b9b49fbe61f9e3781d07f624e3cbd92356
2017-05-10 10:32:52 +02:00
|
|
|
assert(num_bits_needed > 0);
|
2016-08-27 03:55:58 +02:00
|
|
|
|
|
|
|
// bitmap_size = ceil(num_bits_needed / kBitsPerEntry)
|
unbiase readamp bitmap
Summary:
Consider BlockReadAmpBitmap with bytes_per_bit = 32. Suppose bytes [a, b) were used, while bytes [a-32, a)
and [b+1, b+33) weren't used; more formally, the union of ranges passed to BlockReadAmpBitmap::Mark() contains [a, b) and doesn't intersect with [a-32, a) and [b+1, b+33). Then bits [floor(a/32), ceil(b/32)] will be set, and so the number of useful bytes will be estimated as (ceil(b/32) - floor(a/32)) * 32, which is on average equal to b-a+31.
An extreme example: if we use 1 byte from each block, it'll be counted as 32 bytes from each block.
It's easy to remove this bias by slightly changing the semantics of the bitmap. Currently each bit represents a byte range [i*32, (i+1)*32).
This diff makes each bit represent a single byte: i*32 + X, where X is a random number in [0, 31] generated when bitmap is created. So, e.g., if you read a single byte at random, with probability 31/32 it won't be counted at all, and with probability 1/32 it will be counted as 32 bytes; so, on average it's counted as 1 byte.
*But there is one exception: the last bit will always set with the old way.*
(*) - assuming read_amp_bytes_per_bit = 32.
Closes https://github.com/facebook/rocksdb/pull/2259
Differential Revision: D5035652
Pulled By: lightmark
fbshipit-source-id: bd98b1b9b49fbe61f9e3781d07f624e3cbd92356
2017-05-10 10:32:52 +02:00
|
|
|
size_t bitmap_size = (num_bits_needed - 1) / kBitsPerEntry + 1;
|
2016-08-27 03:55:58 +02:00
|
|
|
|
|
|
|
// Create bitmap and set all the bits to 0
|
2017-07-17 19:33:12 +02:00
|
|
|
bitmap_ = new std::atomic<uint32_t>[bitmap_size]();
|
2016-08-27 03:55:58 +02:00
|
|
|
|
unbiase readamp bitmap
Summary:
Consider BlockReadAmpBitmap with bytes_per_bit = 32. Suppose bytes [a, b) were used, while bytes [a-32, a)
and [b+1, b+33) weren't used; more formally, the union of ranges passed to BlockReadAmpBitmap::Mark() contains [a, b) and doesn't intersect with [a-32, a) and [b+1, b+33). Then bits [floor(a/32), ceil(b/32)] will be set, and so the number of useful bytes will be estimated as (ceil(b/32) - floor(a/32)) * 32, which is on average equal to b-a+31.
An extreme example: if we use 1 byte from each block, it'll be counted as 32 bytes from each block.
It's easy to remove this bias by slightly changing the semantics of the bitmap. Currently each bit represents a byte range [i*32, (i+1)*32).
This diff makes each bit represent a single byte: i*32 + X, where X is a random number in [0, 31] generated when bitmap is created. So, e.g., if you read a single byte at random, with probability 31/32 it won't be counted at all, and with probability 1/32 it will be counted as 32 bytes; so, on average it's counted as 1 byte.
*But there is one exception: the last bit will always set with the old way.*
(*) - assuming read_amp_bytes_per_bit = 32.
Closes https://github.com/facebook/rocksdb/pull/2259
Differential Revision: D5035652
Pulled By: lightmark
fbshipit-source-id: bd98b1b9b49fbe61f9e3781d07f624e3cbd92356
2017-05-10 10:32:52 +02:00
|
|
|
RecordTick(GetStatistics(), READ_AMP_TOTAL_READ_BYTES, block_size);
|
2016-08-27 03:55:58 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
~BlockReadAmpBitmap() { delete[] bitmap_; }
|
|
|
|
|
|
|
|
void Mark(uint32_t start_offset, uint32_t end_offset) {
|
|
|
|
assert(end_offset >= start_offset);
|
unbiase readamp bitmap
Summary:
Consider BlockReadAmpBitmap with bytes_per_bit = 32. Suppose bytes [a, b) were used, while bytes [a-32, a)
and [b+1, b+33) weren't used; more formally, the union of ranges passed to BlockReadAmpBitmap::Mark() contains [a, b) and doesn't intersect with [a-32, a) and [b+1, b+33). Then bits [floor(a/32), ceil(b/32)] will be set, and so the number of useful bytes will be estimated as (ceil(b/32) - floor(a/32)) * 32, which is on average equal to b-a+31.
An extreme example: if we use 1 byte from each block, it'll be counted as 32 bytes from each block.
It's easy to remove this bias by slightly changing the semantics of the bitmap. Currently each bit represents a byte range [i*32, (i+1)*32).
This diff makes each bit represent a single byte: i*32 + X, where X is a random number in [0, 31] generated when bitmap is created. So, e.g., if you read a single byte at random, with probability 31/32 it won't be counted at all, and with probability 1/32 it will be counted as 32 bytes; so, on average it's counted as 1 byte.
*But there is one exception: the last bit will always set with the old way.*
(*) - assuming read_amp_bytes_per_bit = 32.
Closes https://github.com/facebook/rocksdb/pull/2259
Differential Revision: D5035652
Pulled By: lightmark
fbshipit-source-id: bd98b1b9b49fbe61f9e3781d07f624e3cbd92356
2017-05-10 10:32:52 +02:00
|
|
|
// Index of first bit in mask
|
|
|
|
uint32_t start_bit =
|
|
|
|
(start_offset + (1 << bytes_per_bit_pow_) - rnd_ - 1) >>
|
|
|
|
bytes_per_bit_pow_;
|
|
|
|
// Index of last bit in mask + 1
|
|
|
|
uint32_t exclusive_end_bit =
|
|
|
|
(end_offset + (1 << bytes_per_bit_pow_) - rnd_) >> bytes_per_bit_pow_;
|
|
|
|
if (start_bit >= exclusive_end_bit) {
|
|
|
|
return;
|
2016-08-27 03:55:58 +02:00
|
|
|
}
|
unbiase readamp bitmap
Summary:
Consider BlockReadAmpBitmap with bytes_per_bit = 32. Suppose bytes [a, b) were used, while bytes [a-32, a)
and [b+1, b+33) weren't used; more formally, the union of ranges passed to BlockReadAmpBitmap::Mark() contains [a, b) and doesn't intersect with [a-32, a) and [b+1, b+33). Then bits [floor(a/32), ceil(b/32)] will be set, and so the number of useful bytes will be estimated as (ceil(b/32) - floor(a/32)) * 32, which is on average equal to b-a+31.
An extreme example: if we use 1 byte from each block, it'll be counted as 32 bytes from each block.
It's easy to remove this bias by slightly changing the semantics of the bitmap. Currently each bit represents a byte range [i*32, (i+1)*32).
This diff makes each bit represent a single byte: i*32 + X, where X is a random number in [0, 31] generated when bitmap is created. So, e.g., if you read a single byte at random, with probability 31/32 it won't be counted at all, and with probability 1/32 it will be counted as 32 bytes; so, on average it's counted as 1 byte.
*But there is one exception: the last bit will always set with the old way.*
(*) - assuming read_amp_bytes_per_bit = 32.
Closes https://github.com/facebook/rocksdb/pull/2259
Differential Revision: D5035652
Pulled By: lightmark
fbshipit-source-id: bd98b1b9b49fbe61f9e3781d07f624e3cbd92356
2017-05-10 10:32:52 +02:00
|
|
|
assert(exclusive_end_bit > 0);
|
2016-08-27 03:55:58 +02:00
|
|
|
|
|
|
|
if (GetAndSet(start_bit) == 0) {
|
unbiase readamp bitmap
Summary:
Consider BlockReadAmpBitmap with bytes_per_bit = 32. Suppose bytes [a, b) were used, while bytes [a-32, a)
and [b+1, b+33) weren't used; more formally, the union of ranges passed to BlockReadAmpBitmap::Mark() contains [a, b) and doesn't intersect with [a-32, a) and [b+1, b+33). Then bits [floor(a/32), ceil(b/32)] will be set, and so the number of useful bytes will be estimated as (ceil(b/32) - floor(a/32)) * 32, which is on average equal to b-a+31.
An extreme example: if we use 1 byte from each block, it'll be counted as 32 bytes from each block.
It's easy to remove this bias by slightly changing the semantics of the bitmap. Currently each bit represents a byte range [i*32, (i+1)*32).
This diff makes each bit represent a single byte: i*32 + X, where X is a random number in [0, 31] generated when bitmap is created. So, e.g., if you read a single byte at random, with probability 31/32 it won't be counted at all, and with probability 1/32 it will be counted as 32 bytes; so, on average it's counted as 1 byte.
*But there is one exception: the last bit will always set with the old way.*
(*) - assuming read_amp_bytes_per_bit = 32.
Closes https://github.com/facebook/rocksdb/pull/2259
Differential Revision: D5035652
Pulled By: lightmark
fbshipit-source-id: bd98b1b9b49fbe61f9e3781d07f624e3cbd92356
2017-05-10 10:32:52 +02:00
|
|
|
uint32_t new_useful_bytes = (exclusive_end_bit - start_bit)
|
|
|
|
<< bytes_per_bit_pow_;
|
2016-08-27 03:55:58 +02:00
|
|
|
RecordTick(GetStatistics(), READ_AMP_ESTIMATE_USEFUL_BYTES,
|
|
|
|
new_useful_bytes);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
Statistics* GetStatistics() {
|
|
|
|
return statistics_.load(std::memory_order_relaxed);
|
|
|
|
}
|
|
|
|
|
|
|
|
void SetStatistics(Statistics* stats) { statistics_.store(stats); }
|
|
|
|
|
|
|
|
uint32_t GetBytesPerBit() { return 1 << bytes_per_bit_pow_; }
|
|
|
|
|
2018-06-29 17:55:33 +02:00
|
|
|
size_t ApproximateMemoryUsage() const {
|
|
|
|
#ifdef ROCKSDB_MALLOC_USABLE_SIZE
|
|
|
|
return malloc_usable_size((void*)this);
|
|
|
|
#endif // ROCKSDB_MALLOC_USABLE_SIZE
|
|
|
|
return sizeof(*this);
|
|
|
|
}
|
|
|
|
|
2016-08-27 03:55:58 +02:00
|
|
|
private:
|
|
|
|
// Get the current value of bit at `bit_idx` and set it to 1
|
|
|
|
inline bool GetAndSet(uint32_t bit_idx) {
|
|
|
|
const uint32_t byte_idx = bit_idx / kBitsPerEntry;
|
|
|
|
const uint32_t bit_mask = 1 << (bit_idx % kBitsPerEntry);
|
|
|
|
|
|
|
|
return bitmap_[byte_idx].fetch_or(bit_mask, std::memory_order_relaxed) &
|
|
|
|
bit_mask;
|
|
|
|
}
|
|
|
|
|
|
|
|
const uint32_t kBytesPersEntry = sizeof(uint32_t); // 4 bytes
|
|
|
|
const uint32_t kBitsPerEntry = kBytesPersEntry * 8; // 32 bits
|
|
|
|
|
|
|
|
// Bitmap used to record the bytes that we read, use atomic to protect
|
|
|
|
// against multiple threads updating the same bit
|
|
|
|
std::atomic<uint32_t>* bitmap_;
|
|
|
|
// (1 << bytes_per_bit_pow_) is bytes_per_bit. Use power of 2 to optimize
|
|
|
|
// muliplication and division
|
|
|
|
uint8_t bytes_per_bit_pow_;
|
|
|
|
// Pointer to DB Statistics object, Since this bitmap may outlive the DB
|
|
|
|
// this pointer maybe invalid, but the DB will update it to a valid pointer
|
|
|
|
// by using SetStatistics() before calling Mark()
|
|
|
|
std::atomic<Statistics*> statistics_;
|
unbiase readamp bitmap
Summary:
Consider BlockReadAmpBitmap with bytes_per_bit = 32. Suppose bytes [a, b) were used, while bytes [a-32, a)
and [b+1, b+33) weren't used; more formally, the union of ranges passed to BlockReadAmpBitmap::Mark() contains [a, b) and doesn't intersect with [a-32, a) and [b+1, b+33). Then bits [floor(a/32), ceil(b/32)] will be set, and so the number of useful bytes will be estimated as (ceil(b/32) - floor(a/32)) * 32, which is on average equal to b-a+31.
An extreme example: if we use 1 byte from each block, it'll be counted as 32 bytes from each block.
It's easy to remove this bias by slightly changing the semantics of the bitmap. Currently each bit represents a byte range [i*32, (i+1)*32).
This diff makes each bit represent a single byte: i*32 + X, where X is a random number in [0, 31] generated when bitmap is created. So, e.g., if you read a single byte at random, with probability 31/32 it won't be counted at all, and with probability 1/32 it will be counted as 32 bytes; so, on average it's counted as 1 byte.
*But there is one exception: the last bit will always set with the old way.*
(*) - assuming read_amp_bytes_per_bit = 32.
Closes https://github.com/facebook/rocksdb/pull/2259
Differential Revision: D5035652
Pulled By: lightmark
fbshipit-source-id: bd98b1b9b49fbe61f9e3781d07f624e3cbd92356
2017-05-10 10:32:52 +02:00
|
|
|
uint32_t rnd_;
|
2016-08-27 03:55:58 +02:00
|
|
|
};
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
class Block {
|
|
|
|
public:
|
|
|
|
// Initialize the block with the specified contents.
|
2016-10-19 01:59:37 +02:00
|
|
|
explicit Block(BlockContents&& contents, SequenceNumber _global_seqno,
|
|
|
|
size_t read_amp_bytes_per_bit = 0,
|
2016-08-27 03:55:58 +02:00
|
|
|
Statistics* statistics = nullptr);
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2018-08-14 02:31:58 +02:00
|
|
|
~Block();
|
2011-03-18 23:37:00 +01:00
|
|
|
|
|
|
|
size_t size() const { return size_; }
|
2014-04-10 23:19:43 +02:00
|
|
|
const char* data() const { return data_; }
|
2018-06-29 17:55:33 +02:00
|
|
|
// The additional memory space taken by the block data.
|
|
|
|
size_t usable_size() const { return contents_.usable_size(); }
|
2014-04-10 23:19:43 +02:00
|
|
|
uint32_t NumRestarts() const;
|
2018-11-14 02:00:49 +01:00
|
|
|
bool own_bytes() const { return contents_.own_bytes(); }
|
|
|
|
|
2018-08-15 23:27:47 +02:00
|
|
|
BlockBasedTableOptions::DataBlockIndexType IndexType() const;
|
2014-04-10 23:19:43 +02:00
|
|
|
|
2018-05-26 03:41:31 +02:00
|
|
|
// If comparator is InternalKeyComparator, user_comparator is its user
|
|
|
|
// comparator; they are equal otherwise.
|
|
|
|
//
|
2014-07-31 01:34:35 +02:00
|
|
|
// If iter is null, return new Iterator
|
|
|
|
// If iter is not null, update this one and return it as Iterator*
|
2014-08-26 01:14:30 +02:00
|
|
|
//
|
2018-08-10 01:49:45 +02:00
|
|
|
// key_includes_seq, default true, means that the keys are in internal key
|
|
|
|
// format.
|
2018-10-05 05:44:43 +02:00
|
|
|
// value_is_full, default true, means that no delta encoding is
|
2018-08-10 01:49:45 +02:00
|
|
|
// applied to values.
|
|
|
|
//
|
2018-07-13 02:19:57 +02:00
|
|
|
// NewIterator<DataBlockIter>
|
|
|
|
// Same as above but also updates read_amp_bitmap_ if it is not nullptr.
|
|
|
|
//
|
|
|
|
// NewIterator<IndexBlockIter>
|
|
|
|
// If `prefix_index` is not nullptr this block will do hash lookup for the key
|
|
|
|
// prefix. If total_order_seek is true, prefix_index_ is ignored.
|
|
|
|
//
|
2018-11-14 02:00:49 +01:00
|
|
|
// If `block_contents_pinned` is true, the caller will guarantee that when
|
|
|
|
// the cleanup functions are transferred from the iterator to other
|
|
|
|
// classes, e.g. PinnableSlice, the pointer to the bytes will still be
|
|
|
|
// valid. Either the iterator holds cache handle or ownership of some resource
|
|
|
|
// and release them in a release function, or caller is sure that the data
|
|
|
|
// will not go away (for example, it's from mmapped file which will not be
|
|
|
|
// closed).
|
|
|
|
//
|
2018-07-13 02:19:57 +02:00
|
|
|
// NOTE: for the hash based lookup, if a key prefix doesn't match any key,
|
|
|
|
// the iterator will simply be set as "invalid", rather than returning
|
|
|
|
// the key that is just pass the target key.
|
|
|
|
template <typename TBlockIter>
|
2018-08-10 01:49:45 +02:00
|
|
|
TBlockIter* NewIterator(
|
|
|
|
const Comparator* comparator, const Comparator* user_comparator,
|
|
|
|
TBlockIter* iter = nullptr, Statistics* stats = nullptr,
|
|
|
|
bool total_order_seek = true, bool key_includes_seq = true,
|
2018-11-14 02:00:49 +01:00
|
|
|
bool value_is_full = true, bool block_contents_pinned = false,
|
|
|
|
BlockPrefixIndex* prefix_index = nullptr);
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2014-08-05 20:27:34 +02:00
|
|
|
// Report an approximation of how much memory has been used.
|
|
|
|
size_t ApproximateMemoryUsage() const;
|
|
|
|
|
2016-10-19 01:59:37 +02:00
|
|
|
SequenceNumber global_seqno() const { return global_seqno_; }
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
private:
|
2014-08-16 00:05:09 +02:00
|
|
|
BlockContents contents_;
|
2019-03-28 00:13:08 +01:00
|
|
|
const char* data_; // contents_.data.data()
|
|
|
|
size_t size_; // contents_.data.size()
|
|
|
|
uint32_t restart_offset_; // Offset in data_ of restart array
|
2018-05-18 21:54:09 +02:00
|
|
|
uint32_t num_restarts_;
|
2016-08-27 03:55:58 +02:00
|
|
|
std::unique_ptr<BlockReadAmpBitmap> read_amp_bitmap_;
|
2016-10-19 01:59:37 +02:00
|
|
|
// All keys in the block will have seqno = global_seqno_, regardless of
|
|
|
|
// the encoded value (kDisableGlobalSequenceNumber means disabled)
|
|
|
|
const SequenceNumber global_seqno_;
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2018-08-15 23:27:47 +02:00
|
|
|
DataBlockHashIndex data_block_hash_index_;
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
// No copying allowed
|
2018-01-25 23:43:43 +01:00
|
|
|
Block(const Block&) = delete;
|
|
|
|
void operator=(const Block&) = delete;
|
2014-07-31 01:34:35 +02:00
|
|
|
};
|
|
|
|
|
2018-08-10 01:49:45 +02:00
|
|
|
template <class TValue>
|
|
|
|
class BlockIter : public InternalIteratorBase<TValue> {
|
2014-07-31 01:34:35 +02:00
|
|
|
public:
|
2018-07-16 18:58:58 +02:00
|
|
|
void InitializeBase(const Comparator* comparator, const char* data,
|
2018-07-13 02:19:57 +02:00
|
|
|
uint32_t restarts, uint32_t num_restarts,
|
2018-07-17 02:10:44 +02:00
|
|
|
SequenceNumber global_seqno, bool block_contents_pinned) {
|
2019-03-28 00:13:08 +01:00
|
|
|
assert(data_ == nullptr); // Ensure it is called only once
|
|
|
|
assert(num_restarts > 0); // Ensure the param is valid
|
2014-07-31 01:34:35 +02:00
|
|
|
|
|
|
|
comparator_ = comparator;
|
|
|
|
data_ = data;
|
|
|
|
restarts_ = restarts;
|
|
|
|
num_restarts_ = num_restarts;
|
|
|
|
current_ = restarts_;
|
|
|
|
restart_index_ = num_restarts_;
|
2016-10-19 01:59:37 +02:00
|
|
|
global_seqno_ = global_seqno;
|
Copy Get() result when file reads use mmap
Summary:
For iterator reads, a `SuperVersion` is pinned to preserve a snapshot of SST files, and `Block`s are pinned to allow `key()` and `value()` to return pointers directly into a RocksDB memory region. This works for both non-mmap reads, where the block owns the memory region, and mmap reads, where the file owns the memory region.
For point reads with `PinnableSlice`, only the `Block` object is pinned. This works for non-mmap reads because the block owns the memory region, so even if the file is deleted after compaction, the memory region survives. However, for mmap reads, file deletion causes the memory region to which the `PinnableSlice` refers to be unmapped. The result is usually a segfault upon accessing the `PinnableSlice`, although sometimes it returned wrong results (I repro'd this a bunch of times with `db_stress`).
This PR copies the value into the `PinnableSlice` when it comes from mmap'd memory. We can tell whether the `Block` owns its memory using `Block::cachable()`, which is unset when reads do not use the provided buffer as is the case with mmap file reads. When that is false we ensure the result of `Get()` is copied.
This feels like a short-term solution as ideally we'd have the `PinnableSlice` pin the mmap'd memory so we can do zero-copy reads. It seemed hard so I chose this approach to fix correctness in the meantime.
Closes https://github.com/facebook/rocksdb/pull/3881
Differential Revision: D8076288
Pulled By: ajkr
fbshipit-source-id: 31d78ec010198723522323dbc6ea325122a46b08
2018-06-02 01:46:32 +02:00
|
|
|
block_contents_pinned_ = block_contents_pinned;
|
2019-06-10 22:28:18 +02:00
|
|
|
cache_handle_ = nullptr;
|
2014-07-31 01:34:35 +02:00
|
|
|
}
|
|
|
|
|
Change and clarify the relationship between Valid(), status() and Seek*() for all iterators. Also fix some bugs
Summary:
Before this PR, Iterator/InternalIterator may simultaneously have non-ok status() and Valid() = true. That state means that the last operation failed, but the iterator is nevertheless positioned on some unspecified record. Likely intended uses of that are:
* If some sst files are corrupted, a normal iterator can be used to read the data from files that are not corrupted.
* When using read_tier = kBlockCacheTier, read the data that's in block cache, skipping over the data that is not.
However, this behavior wasn't documented well (and until recently the wiki on github had misleading incorrect information). In the code there's a lot of confusion about the relationship between status() and Valid(), and about whether Seek()/SeekToLast()/etc reset the status or not. There were a number of bugs caused by this confusion, both inside rocksdb and in the code that uses rocksdb (including ours).
This PR changes the convention to:
* If status() is not ok, Valid() always returns false.
* Any seek operation resets status. (Before the PR, it depended on iterator type and on particular error.)
This does sacrifice the two use cases listed above, but siying said it's ok.
Overview of the changes:
* A commit that adds missing status checks in MergingIterator. This fixes a bug that actually affects us, and we need it fixed. `DBIteratorTest.NonBlockingIterationBugRepro` explains the scenario.
* Changes to lots of iterator types to make all of them conform to the new convention. Some bug fixes along the way. By far the biggest changes are in DBIter, which is a big messy piece of code; I tried to make it less big and messy but mostly failed.
* A stress-test for DBIter, to gain some confidence that I didn't break it. It does a few million random operations on the iterator, while occasionally modifying the underlying data (like ForwardIterator does) and occasionally returning non-ok status from internal iterator.
To find the iterator types that needed changes I searched for "public .*Iterator" in the code. Here's an overview of all 27 iterator types:
Iterators that didn't need changes:
* status() is always ok(), or Valid() is always false: MemTableIterator, ModelIter, TestIterator, KVIter (2 classes with this name anonymous namespaces), LoggingForwardVectorIterator, VectorIterator, MockTableIterator, EmptyIterator, EmptyInternalIterator.
* Thin wrappers that always pass through Valid() and status(): ArenaWrappedDBIter, TtlIterator, InternalIteratorFromIterator.
Iterators with changes (see inline comments for details):
* DBIter - an overhaul:
- It used to silently skip corrupted keys (`FindParseableKey()`), which seems dangerous. This PR makes it just stop immediately after encountering a corrupted key, just like it would for other kinds of corruption. Let me know if there was actually some deeper meaning in this behavior and I should put it back.
- It had a few code paths silently discarding subiterator's status. The stress test caught a few.
- The backwards iteration code path was expecting the internal iterator's set of keys to be immutable. It's probably always true in practice at the moment, since ForwardIterator doesn't support backwards iteration, but this PR fixes it anyway. See added DBIteratorTest.ReverseToForwardBug for an example.
- Some parts of backwards iteration code path even did things like `assert(iter_->Valid())` after a seek, which is never a safe assumption.
- It used to not reset status on seek for some types of errors.
- Some simplifications and better comments.
- Some things got more complicated from the added error handling. I'm open to ideas for how to make it nicer.
* MergingIterator - check status after every operation on every subiterator, and in some places assert that valid subiterators have ok status.
* ForwardIterator - changed to the new convention, also slightly simplified.
* ForwardLevelIterator - fixed some bugs and simplified.
* LevelIterator - simplified.
* TwoLevelIterator - changed to the new convention. Also fixed a bug that would make SeekForPrev() sometimes silently ignore errors from first_level_iter_.
* BlockBasedTableIterator - minor changes.
* BlockIter - replaced `SetStatus()` with `Invalidate()` to make sure non-ok BlockIter is always invalid.
* PlainTableIterator - some seeks used to not reset status.
* CuckooTableIterator - tiny code cleanup.
* ManagedIterator - fixed some bugs.
* BaseDeltaIterator - changed to the new convention and fixed a bug.
* BlobDBIterator - seeks used to not reset status.
* KeyConvertingIterator - some small change.
Closes https://github.com/facebook/rocksdb/pull/3810
Differential Revision: D7888019
Pulled By: al13n321
fbshipit-source-id: 4aaf6d3421c545d16722a815b2fa2e7912bc851d
2018-05-17 11:44:14 +02:00
|
|
|
// Makes Valid() return false, status() return `s`, and Seek()/Prev()/etc do
|
2018-06-05 20:32:07 +02:00
|
|
|
// nothing. Calls cleanup functions.
|
2018-07-17 02:10:44 +02:00
|
|
|
void InvalidateBase(Status s) {
|
Change and clarify the relationship between Valid(), status() and Seek*() for all iterators. Also fix some bugs
Summary:
Before this PR, Iterator/InternalIterator may simultaneously have non-ok status() and Valid() = true. That state means that the last operation failed, but the iterator is nevertheless positioned on some unspecified record. Likely intended uses of that are:
* If some sst files are corrupted, a normal iterator can be used to read the data from files that are not corrupted.
* When using read_tier = kBlockCacheTier, read the data that's in block cache, skipping over the data that is not.
However, this behavior wasn't documented well (and until recently the wiki on github had misleading incorrect information). In the code there's a lot of confusion about the relationship between status() and Valid(), and about whether Seek()/SeekToLast()/etc reset the status or not. There were a number of bugs caused by this confusion, both inside rocksdb and in the code that uses rocksdb (including ours).
This PR changes the convention to:
* If status() is not ok, Valid() always returns false.
* Any seek operation resets status. (Before the PR, it depended on iterator type and on particular error.)
This does sacrifice the two use cases listed above, but siying said it's ok.
Overview of the changes:
* A commit that adds missing status checks in MergingIterator. This fixes a bug that actually affects us, and we need it fixed. `DBIteratorTest.NonBlockingIterationBugRepro` explains the scenario.
* Changes to lots of iterator types to make all of them conform to the new convention. Some bug fixes along the way. By far the biggest changes are in DBIter, which is a big messy piece of code; I tried to make it less big and messy but mostly failed.
* A stress-test for DBIter, to gain some confidence that I didn't break it. It does a few million random operations on the iterator, while occasionally modifying the underlying data (like ForwardIterator does) and occasionally returning non-ok status from internal iterator.
To find the iterator types that needed changes I searched for "public .*Iterator" in the code. Here's an overview of all 27 iterator types:
Iterators that didn't need changes:
* status() is always ok(), or Valid() is always false: MemTableIterator, ModelIter, TestIterator, KVIter (2 classes with this name anonymous namespaces), LoggingForwardVectorIterator, VectorIterator, MockTableIterator, EmptyIterator, EmptyInternalIterator.
* Thin wrappers that always pass through Valid() and status(): ArenaWrappedDBIter, TtlIterator, InternalIteratorFromIterator.
Iterators with changes (see inline comments for details):
* DBIter - an overhaul:
- It used to silently skip corrupted keys (`FindParseableKey()`), which seems dangerous. This PR makes it just stop immediately after encountering a corrupted key, just like it would for other kinds of corruption. Let me know if there was actually some deeper meaning in this behavior and I should put it back.
- It had a few code paths silently discarding subiterator's status. The stress test caught a few.
- The backwards iteration code path was expecting the internal iterator's set of keys to be immutable. It's probably always true in practice at the moment, since ForwardIterator doesn't support backwards iteration, but this PR fixes it anyway. See added DBIteratorTest.ReverseToForwardBug for an example.
- Some parts of backwards iteration code path even did things like `assert(iter_->Valid())` after a seek, which is never a safe assumption.
- It used to not reset status on seek for some types of errors.
- Some simplifications and better comments.
- Some things got more complicated from the added error handling. I'm open to ideas for how to make it nicer.
* MergingIterator - check status after every operation on every subiterator, and in some places assert that valid subiterators have ok status.
* ForwardIterator - changed to the new convention, also slightly simplified.
* ForwardLevelIterator - fixed some bugs and simplified.
* LevelIterator - simplified.
* TwoLevelIterator - changed to the new convention. Also fixed a bug that would make SeekForPrev() sometimes silently ignore errors from first_level_iter_.
* BlockBasedTableIterator - minor changes.
* BlockIter - replaced `SetStatus()` with `Invalidate()` to make sure non-ok BlockIter is always invalid.
* PlainTableIterator - some seeks used to not reset status.
* CuckooTableIterator - tiny code cleanup.
* ManagedIterator - fixed some bugs.
* BaseDeltaIterator - changed to the new convention and fixed a bug.
* BlobDBIterator - seeks used to not reset status.
* KeyConvertingIterator - some small change.
Closes https://github.com/facebook/rocksdb/pull/3810
Differential Revision: D7888019
Pulled By: al13n321
fbshipit-source-id: 4aaf6d3421c545d16722a815b2fa2e7912bc851d
2018-05-17 11:44:14 +02:00
|
|
|
// Assert that the BlockIter is never deleted while Pinning is Enabled.
|
|
|
|
assert(!pinned_iters_mgr_ ||
|
|
|
|
(pinned_iters_mgr_ && !pinned_iters_mgr_->PinningEnabled()));
|
|
|
|
|
|
|
|
data_ = nullptr;
|
|
|
|
current_ = restarts_;
|
2014-07-31 01:34:35 +02:00
|
|
|
status_ = s;
|
Change and clarify the relationship between Valid(), status() and Seek*() for all iterators. Also fix some bugs
Summary:
Before this PR, Iterator/InternalIterator may simultaneously have non-ok status() and Valid() = true. That state means that the last operation failed, but the iterator is nevertheless positioned on some unspecified record. Likely intended uses of that are:
* If some sst files are corrupted, a normal iterator can be used to read the data from files that are not corrupted.
* When using read_tier = kBlockCacheTier, read the data that's in block cache, skipping over the data that is not.
However, this behavior wasn't documented well (and until recently the wiki on github had misleading incorrect information). In the code there's a lot of confusion about the relationship between status() and Valid(), and about whether Seek()/SeekToLast()/etc reset the status or not. There were a number of bugs caused by this confusion, both inside rocksdb and in the code that uses rocksdb (including ours).
This PR changes the convention to:
* If status() is not ok, Valid() always returns false.
* Any seek operation resets status. (Before the PR, it depended on iterator type and on particular error.)
This does sacrifice the two use cases listed above, but siying said it's ok.
Overview of the changes:
* A commit that adds missing status checks in MergingIterator. This fixes a bug that actually affects us, and we need it fixed. `DBIteratorTest.NonBlockingIterationBugRepro` explains the scenario.
* Changes to lots of iterator types to make all of them conform to the new convention. Some bug fixes along the way. By far the biggest changes are in DBIter, which is a big messy piece of code; I tried to make it less big and messy but mostly failed.
* A stress-test for DBIter, to gain some confidence that I didn't break it. It does a few million random operations on the iterator, while occasionally modifying the underlying data (like ForwardIterator does) and occasionally returning non-ok status from internal iterator.
To find the iterator types that needed changes I searched for "public .*Iterator" in the code. Here's an overview of all 27 iterator types:
Iterators that didn't need changes:
* status() is always ok(), or Valid() is always false: MemTableIterator, ModelIter, TestIterator, KVIter (2 classes with this name anonymous namespaces), LoggingForwardVectorIterator, VectorIterator, MockTableIterator, EmptyIterator, EmptyInternalIterator.
* Thin wrappers that always pass through Valid() and status(): ArenaWrappedDBIter, TtlIterator, InternalIteratorFromIterator.
Iterators with changes (see inline comments for details):
* DBIter - an overhaul:
- It used to silently skip corrupted keys (`FindParseableKey()`), which seems dangerous. This PR makes it just stop immediately after encountering a corrupted key, just like it would for other kinds of corruption. Let me know if there was actually some deeper meaning in this behavior and I should put it back.
- It had a few code paths silently discarding subiterator's status. The stress test caught a few.
- The backwards iteration code path was expecting the internal iterator's set of keys to be immutable. It's probably always true in practice at the moment, since ForwardIterator doesn't support backwards iteration, but this PR fixes it anyway. See added DBIteratorTest.ReverseToForwardBug for an example.
- Some parts of backwards iteration code path even did things like `assert(iter_->Valid())` after a seek, which is never a safe assumption.
- It used to not reset status on seek for some types of errors.
- Some simplifications and better comments.
- Some things got more complicated from the added error handling. I'm open to ideas for how to make it nicer.
* MergingIterator - check status after every operation on every subiterator, and in some places assert that valid subiterators have ok status.
* ForwardIterator - changed to the new convention, also slightly simplified.
* ForwardLevelIterator - fixed some bugs and simplified.
* LevelIterator - simplified.
* TwoLevelIterator - changed to the new convention. Also fixed a bug that would make SeekForPrev() sometimes silently ignore errors from first_level_iter_.
* BlockBasedTableIterator - minor changes.
* BlockIter - replaced `SetStatus()` with `Invalidate()` to make sure non-ok BlockIter is always invalid.
* PlainTableIterator - some seeks used to not reset status.
* CuckooTableIterator - tiny code cleanup.
* ManagedIterator - fixed some bugs.
* BaseDeltaIterator - changed to the new convention and fixed a bug.
* BlobDBIterator - seeks used to not reset status.
* KeyConvertingIterator - some small change.
Closes https://github.com/facebook/rocksdb/pull/3810
Differential Revision: D7888019
Pulled By: al13n321
fbshipit-source-id: 4aaf6d3421c545d16722a815b2fa2e7912bc851d
2018-05-17 11:44:14 +02:00
|
|
|
|
2018-06-05 20:32:07 +02:00
|
|
|
// Call cleanup callbacks.
|
|
|
|
Cleanable::Reset();
|
2014-07-31 01:34:35 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
virtual bool Valid() const override { return current_ < restarts_; }
|
|
|
|
virtual Status status() const override { return status_; }
|
|
|
|
virtual Slice key() const override {
|
|
|
|
assert(Valid());
|
2018-07-17 02:10:44 +02:00
|
|
|
return key_.GetKey();
|
2014-07-31 01:34:35 +02:00
|
|
|
}
|
|
|
|
|
2016-04-26 21:41:07 +02:00
|
|
|
#ifndef NDEBUG
|
2018-07-13 02:19:57 +02:00
|
|
|
virtual ~BlockIter() {
|
2016-04-26 21:41:07 +02:00
|
|
|
// Assert that the BlockIter is never deleted while Pinning is Enabled.
|
|
|
|
assert(!pinned_iters_mgr_ ||
|
|
|
|
(pinned_iters_mgr_ && !pinned_iters_mgr_->PinningEnabled()));
|
2015-12-16 21:08:30 +01:00
|
|
|
}
|
2016-04-26 21:41:07 +02:00
|
|
|
virtual void SetPinnedItersMgr(
|
|
|
|
PinnedIteratorsManager* pinned_iters_mgr) override {
|
|
|
|
pinned_iters_mgr_ = pinned_iters_mgr;
|
2015-12-16 21:08:30 +01:00
|
|
|
}
|
2016-04-26 21:41:07 +02:00
|
|
|
PinnedIteratorsManager* pinned_iters_mgr_ = nullptr;
|
|
|
|
#endif
|
2015-12-16 21:08:30 +01:00
|
|
|
|
Copy Get() result when file reads use mmap
Summary:
For iterator reads, a `SuperVersion` is pinned to preserve a snapshot of SST files, and `Block`s are pinned to allow `key()` and `value()` to return pointers directly into a RocksDB memory region. This works for both non-mmap reads, where the block owns the memory region, and mmap reads, where the file owns the memory region.
For point reads with `PinnableSlice`, only the `Block` object is pinned. This works for non-mmap reads because the block owns the memory region, so even if the file is deleted after compaction, the memory region survives. However, for mmap reads, file deletion causes the memory region to which the `PinnableSlice` refers to be unmapped. The result is usually a segfault upon accessing the `PinnableSlice`, although sometimes it returned wrong results (I repro'd this a bunch of times with `db_stress`).
This PR copies the value into the `PinnableSlice` when it comes from mmap'd memory. We can tell whether the `Block` owns its memory using `Block::cachable()`, which is unset when reads do not use the provided buffer as is the case with mmap file reads. When that is false we ensure the result of `Get()` is copied.
This feels like a short-term solution as ideally we'd have the `PinnableSlice` pin the mmap'd memory so we can do zero-copy reads. It seemed hard so I chose this approach to fix correctness in the meantime.
Closes https://github.com/facebook/rocksdb/pull/3881
Differential Revision: D8076288
Pulled By: ajkr
fbshipit-source-id: 31d78ec010198723522323dbc6ea325122a46b08
2018-06-02 01:46:32 +02:00
|
|
|
virtual bool IsKeyPinned() const override {
|
|
|
|
return block_contents_pinned_ && key_pinned_;
|
|
|
|
}
|
2015-12-16 21:08:30 +01:00
|
|
|
|
Copy Get() result when file reads use mmap
Summary:
For iterator reads, a `SuperVersion` is pinned to preserve a snapshot of SST files, and `Block`s are pinned to allow `key()` and `value()` to return pointers directly into a RocksDB memory region. This works for both non-mmap reads, where the block owns the memory region, and mmap reads, where the file owns the memory region.
For point reads with `PinnableSlice`, only the `Block` object is pinned. This works for non-mmap reads because the block owns the memory region, so even if the file is deleted after compaction, the memory region survives. However, for mmap reads, file deletion causes the memory region to which the `PinnableSlice` refers to be unmapped. The result is usually a segfault upon accessing the `PinnableSlice`, although sometimes it returned wrong results (I repro'd this a bunch of times with `db_stress`).
This PR copies the value into the `PinnableSlice` when it comes from mmap'd memory. We can tell whether the `Block` owns its memory using `Block::cachable()`, which is unset when reads do not use the provided buffer as is the case with mmap file reads. When that is false we ensure the result of `Get()` is copied.
This feels like a short-term solution as ideally we'd have the `PinnableSlice` pin the mmap'd memory so we can do zero-copy reads. It seemed hard so I chose this approach to fix correctness in the meantime.
Closes https://github.com/facebook/rocksdb/pull/3881
Differential Revision: D8076288
Pulled By: ajkr
fbshipit-source-id: 31d78ec010198723522323dbc6ea325122a46b08
2018-06-02 01:46:32 +02:00
|
|
|
virtual bool IsValuePinned() const override { return block_contents_pinned_; }
|
Introduce FullMergeV2 (eliminate memcpy from merge operators)
Summary:
This diff update the code to pin the merge operator operands while the merge operation is done, so that we can eliminate the memcpy cost, to do that we need a new public API for FullMerge that replace the std::deque<std::string> with std::vector<Slice>
This diff is stacked on top of D56493 and D56511
In this diff we
- Update FullMergeV2 arguments to be encapsulated in MergeOperationInput and MergeOperationOutput which will make it easier to add new arguments in the future
- Replace std::deque<std::string> with std::vector<Slice> to pass operands
- Replace MergeContext std::deque with std::vector (based on a simple benchmark I ran https://gist.github.com/IslamAbdelRahman/78fc86c9ab9f52b1df791e58943fb187)
- Allow FullMergeV2 output to be an existing operand
```
[Everything in Memtable | 10K operands | 10 KB each | 1 operand per key]
DEBUG_LEVEL=0 make db_bench -j64 && ./db_bench --benchmarks="mergerandom,readseq,readseq,readseq,readseq,readseq" --merge_operator="max" --merge_keys=10000 --num=10000 --disable_auto_compactions --value_size=10240 --write_buffer_size=1000000000
[FullMergeV2]
readseq : 0.607 micros/op 1648235 ops/sec; 16121.2 MB/s
readseq : 0.478 micros/op 2091546 ops/sec; 20457.2 MB/s
readseq : 0.252 micros/op 3972081 ops/sec; 38850.5 MB/s
readseq : 0.237 micros/op 4218328 ops/sec; 41259.0 MB/s
readseq : 0.247 micros/op 4043927 ops/sec; 39553.2 MB/s
[master]
readseq : 3.935 micros/op 254140 ops/sec; 2485.7 MB/s
readseq : 3.722 micros/op 268657 ops/sec; 2627.7 MB/s
readseq : 3.149 micros/op 317605 ops/sec; 3106.5 MB/s
readseq : 3.125 micros/op 320024 ops/sec; 3130.1 MB/s
readseq : 4.075 micros/op 245374 ops/sec; 2400.0 MB/s
```
```
[Everything in Memtable | 10K operands | 10 KB each | 10 operand per key]
DEBUG_LEVEL=0 make db_bench -j64 && ./db_bench --benchmarks="mergerandom,readseq,readseq,readseq,readseq,readseq" --merge_operator="max" --merge_keys=1000 --num=10000 --disable_auto_compactions --value_size=10240 --write_buffer_size=1000000000
[FullMergeV2]
readseq : 3.472 micros/op 288018 ops/sec; 2817.1 MB/s
readseq : 2.304 micros/op 434027 ops/sec; 4245.2 MB/s
readseq : 1.163 micros/op 859845 ops/sec; 8410.0 MB/s
readseq : 1.192 micros/op 838926 ops/sec; 8205.4 MB/s
readseq : 1.250 micros/op 800000 ops/sec; 7824.7 MB/s
[master]
readseq : 24.025 micros/op 41623 ops/sec; 407.1 MB/s
readseq : 18.489 micros/op 54086 ops/sec; 529.0 MB/s
readseq : 18.693 micros/op 53495 ops/sec; 523.2 MB/s
readseq : 23.621 micros/op 42335 ops/sec; 414.1 MB/s
readseq : 18.775 micros/op 53262 ops/sec; 521.0 MB/s
```
```
[Everything in Block cache | 10K operands | 10 KB each | 1 operand per key]
[FullMergeV2]
$ DEBUG_LEVEL=0 make db_bench -j64 && ./db_bench --benchmarks="readseq,readseq,readseq,readseq,readseq" --merge_operator="max" --num=100000 --db="/dev/shm/merge-random-10K-10KB" --cache_size=1000000000 --use_existing_db --disable_auto_compactions
readseq : 14.741 micros/op 67837 ops/sec; 663.5 MB/s
readseq : 1.029 micros/op 971446 ops/sec; 9501.6 MB/s
readseq : 0.974 micros/op 1026229 ops/sec; 10037.4 MB/s
readseq : 0.965 micros/op 1036080 ops/sec; 10133.8 MB/s
readseq : 0.943 micros/op 1060657 ops/sec; 10374.2 MB/s
[master]
readseq : 16.735 micros/op 59755 ops/sec; 584.5 MB/s
readseq : 3.029 micros/op 330151 ops/sec; 3229.2 MB/s
readseq : 3.136 micros/op 318883 ops/sec; 3119.0 MB/s
readseq : 3.065 micros/op 326245 ops/sec; 3191.0 MB/s
readseq : 3.014 micros/op 331813 ops/sec; 3245.4 MB/s
```
```
[Everything in Block cache | 10K operands | 10 KB each | 10 operand per key]
DEBUG_LEVEL=0 make db_bench -j64 && ./db_bench --benchmarks="readseq,readseq,readseq,readseq,readseq" --merge_operator="max" --num=100000 --db="/dev/shm/merge-random-10-operands-10K-10KB" --cache_size=1000000000 --use_existing_db --disable_auto_compactions
[FullMergeV2]
readseq : 24.325 micros/op 41109 ops/sec; 402.1 MB/s
readseq : 1.470 micros/op 680272 ops/sec; 6653.7 MB/s
readseq : 1.231 micros/op 812347 ops/sec; 7945.5 MB/s
readseq : 1.091 micros/op 916590 ops/sec; 8965.1 MB/s
readseq : 1.109 micros/op 901713 ops/sec; 8819.6 MB/s
[master]
readseq : 27.257 micros/op 36687 ops/sec; 358.8 MB/s
readseq : 4.443 micros/op 225073 ops/sec; 2201.4 MB/s
readseq : 5.830 micros/op 171526 ops/sec; 1677.7 MB/s
readseq : 4.173 micros/op 239635 ops/sec; 2343.8 MB/s
readseq : 4.150 micros/op 240963 ops/sec; 2356.8 MB/s
```
Test Plan: COMPILE_WITH_ASAN=1 make check -j64
Reviewers: yhchiang, andrewkr, sdong
Reviewed By: sdong
Subscribers: lovro, andrewkr, dhruba
Differential Revision: https://reviews.facebook.net/D57075
2016-07-20 18:49:03 +02:00
|
|
|
|
2016-08-27 03:55:58 +02:00
|
|
|
size_t TEST_CurrentEntrySize() { return NextEntryOffset() - current_; }
|
|
|
|
|
2016-10-19 01:59:37 +02:00
|
|
|
uint32_t ValueOffset() const {
|
|
|
|
return static_cast<uint32_t>(value_.data() - data_);
|
|
|
|
}
|
|
|
|
|
2019-06-10 22:28:18 +02:00
|
|
|
void SetCacheHandle(Cache::Handle* handle) { cache_handle_ = handle; }
|
|
|
|
|
|
|
|
Cache::Handle* cache_handle() { return cache_handle_; }
|
|
|
|
|
2018-07-13 02:19:57 +02:00
|
|
|
protected:
|
2018-05-26 03:41:31 +02:00
|
|
|
// Note: The type could be changed to InternalKeyComparator but we see a weird
|
|
|
|
// performance drop by that.
|
2014-07-31 01:34:35 +02:00
|
|
|
const Comparator* comparator_;
|
|
|
|
const char* data_; // underlying block contents
|
|
|
|
uint32_t num_restarts_; // Number of uint32_t entries in restart array
|
|
|
|
|
2018-08-10 01:49:45 +02:00
|
|
|
// Index of restart block in which current_ or current_-1 falls
|
|
|
|
uint32_t restart_index_;
|
2019-03-28 00:13:08 +01:00
|
|
|
uint32_t restarts_; // Offset of restart array (list of fixed32)
|
2014-07-31 01:34:35 +02:00
|
|
|
// current_ is offset in data_ of current entry. >= restarts_ if !Valid
|
|
|
|
uint32_t current_;
|
|
|
|
IterKey key_;
|
|
|
|
Slice value_;
|
|
|
|
Status status_;
|
2016-06-14 21:27:46 +02:00
|
|
|
bool key_pinned_;
|
2018-11-14 02:00:49 +01:00
|
|
|
// Whether the block data is guaranteed to outlive this iterator, and
|
|
|
|
// as long as the cleanup functions are transferred to another class,
|
|
|
|
// e.g. PinnableSlice, the pointer to the bytes will still be valid.
|
2018-07-13 02:19:57 +02:00
|
|
|
bool block_contents_pinned_;
|
2016-10-19 01:59:37 +02:00
|
|
|
SequenceNumber global_seqno_;
|
2016-06-14 21:27:46 +02:00
|
|
|
|
2019-06-10 22:28:18 +02:00
|
|
|
private:
|
|
|
|
// Store the cache handle, if the block is cached. We need this since the
|
|
|
|
// only other place the handle is stored is as an argument to the Cleanable
|
|
|
|
// function callback, which is hard to retrieve. When multiple value
|
|
|
|
// PinnableSlices reference the block, they need the cache handle in order
|
|
|
|
// to bump up the ref count
|
|
|
|
Cache::Handle* cache_handle_;
|
|
|
|
|
2018-07-13 02:19:57 +02:00
|
|
|
public:
|
2014-07-31 01:34:35 +02:00
|
|
|
// Return the offset in data_ just past the end of the current entry.
|
|
|
|
inline uint32_t NextEntryOffset() const {
|
2016-11-04 19:19:09 +01:00
|
|
|
// NOTE: We don't support blocks bigger than 2GB
|
2014-11-11 22:47:22 +01:00
|
|
|
return static_cast<uint32_t>((value_.data() + value_.size()) - data_);
|
2014-07-31 01:34:35 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
uint32_t GetRestartPoint(uint32_t index) {
|
|
|
|
assert(index < num_restarts_);
|
|
|
|
return DecodeFixed32(data_ + restarts_ + index * sizeof(uint32_t));
|
|
|
|
}
|
|
|
|
|
|
|
|
void SeekToRestartPoint(uint32_t index) {
|
|
|
|
key_.Clear();
|
|
|
|
restart_index_ = index;
|
|
|
|
// current_ will be fixed by ParseNextKey();
|
|
|
|
|
|
|
|
// ParseNextKey() starts at the end of value_, so set value_ accordingly
|
|
|
|
uint32_t offset = GetRestartPoint(index);
|
|
|
|
value_ = Slice(data_ + offset, 0);
|
|
|
|
}
|
|
|
|
|
|
|
|
void CorruptionError();
|
|
|
|
|
2018-08-10 01:49:45 +02:00
|
|
|
template <typename DecodeKeyFunc>
|
|
|
|
inline bool BinarySeek(const Slice& target, uint32_t left, uint32_t right,
|
|
|
|
uint32_t* index, const Comparator* comp);
|
2018-07-13 02:19:57 +02:00
|
|
|
};
|
2014-07-31 01:34:35 +02:00
|
|
|
|
2018-08-10 01:49:45 +02:00
|
|
|
class DataBlockIter final : public BlockIter<Slice> {
|
2018-07-13 02:19:57 +02:00
|
|
|
public:
|
|
|
|
DataBlockIter()
|
|
|
|
: BlockIter(), read_amp_bitmap_(nullptr), last_bitmap_offset_(0) {}
|
|
|
|
DataBlockIter(const Comparator* comparator, const Comparator* user_comparator,
|
|
|
|
const char* data, uint32_t restarts, uint32_t num_restarts,
|
|
|
|
SequenceNumber global_seqno,
|
2018-08-15 23:27:47 +02:00
|
|
|
BlockReadAmpBitmap* read_amp_bitmap, bool block_contents_pinned,
|
|
|
|
DataBlockHashIndex* data_block_hash_index)
|
2018-07-13 02:19:57 +02:00
|
|
|
: DataBlockIter() {
|
|
|
|
Initialize(comparator, user_comparator, data, restarts, num_restarts,
|
2018-08-15 23:27:47 +02:00
|
|
|
global_seqno, read_amp_bitmap, block_contents_pinned,
|
|
|
|
data_block_hash_index);
|
2018-07-13 02:19:57 +02:00
|
|
|
}
|
|
|
|
void Initialize(const Comparator* comparator,
|
2018-08-15 23:27:47 +02:00
|
|
|
const Comparator* user_comparator, const char* data,
|
2018-07-13 02:19:57 +02:00
|
|
|
uint32_t restarts, uint32_t num_restarts,
|
|
|
|
SequenceNumber global_seqno,
|
|
|
|
BlockReadAmpBitmap* read_amp_bitmap,
|
2018-08-15 23:27:47 +02:00
|
|
|
bool block_contents_pinned,
|
|
|
|
DataBlockHashIndex* data_block_hash_index) {
|
2018-07-16 18:58:58 +02:00
|
|
|
InitializeBase(comparator, data, restarts, num_restarts, global_seqno,
|
2018-07-17 02:10:44 +02:00
|
|
|
block_contents_pinned);
|
2018-08-15 23:27:47 +02:00
|
|
|
user_comparator_ = user_comparator;
|
2018-07-17 02:10:44 +02:00
|
|
|
key_.SetIsUserKey(false);
|
2018-07-13 02:19:57 +02:00
|
|
|
read_amp_bitmap_ = read_amp_bitmap;
|
|
|
|
last_bitmap_offset_ = current_ + 1;
|
2018-08-15 23:27:47 +02:00
|
|
|
data_block_hash_index_ = data_block_hash_index;
|
2018-07-13 02:19:57 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
virtual Slice value() const override {
|
|
|
|
assert(Valid());
|
|
|
|
if (read_amp_bitmap_ && current_ < restarts_ &&
|
|
|
|
current_ != last_bitmap_offset_) {
|
|
|
|
read_amp_bitmap_->Mark(current_ /* current entry offset */,
|
|
|
|
NextEntryOffset() - 1);
|
|
|
|
last_bitmap_offset_ = current_;
|
|
|
|
}
|
|
|
|
return value_;
|
|
|
|
}
|
|
|
|
|
2018-07-16 18:58:58 +02:00
|
|
|
virtual void Seek(const Slice& target) override;
|
|
|
|
|
2018-08-15 23:27:47 +02:00
|
|
|
inline bool SeekForGet(const Slice& target) {
|
|
|
|
if (!data_block_hash_index_) {
|
|
|
|
Seek(target);
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
return SeekForGetImpl(target);
|
|
|
|
}
|
|
|
|
|
2018-07-16 18:58:58 +02:00
|
|
|
virtual void SeekForPrev(const Slice& target) override;
|
|
|
|
|
2018-07-17 02:10:44 +02:00
|
|
|
virtual void Prev() override;
|
|
|
|
|
2019-04-18 20:08:33 +02:00
|
|
|
virtual void Next() final override;
|
2018-07-17 02:10:44 +02:00
|
|
|
|
2019-02-11 20:37:07 +01:00
|
|
|
// Try to advance to the next entry in the block. If there is data corruption
|
|
|
|
// or error, report it to the caller instead of aborting the process. May
|
|
|
|
// incur higher CPU overhead because we need to perform check on every entry.
|
|
|
|
void NextOrReport();
|
|
|
|
|
2018-07-17 02:10:44 +02:00
|
|
|
virtual void SeekToFirst() override;
|
|
|
|
|
2019-02-11 20:37:07 +01:00
|
|
|
// Try to seek to the first entry in the block. If there is data corruption
|
|
|
|
// or error, report it to caller instead of aborting the process. May incur
|
|
|
|
// higher CPU overhead because we need to perform check on every entry.
|
|
|
|
void SeekToFirstOrReport();
|
|
|
|
|
2018-07-17 02:10:44 +02:00
|
|
|
virtual void SeekToLast() override;
|
|
|
|
|
|
|
|
void Invalidate(Status s) {
|
|
|
|
InvalidateBase(s);
|
|
|
|
// Clear prev entries cache.
|
|
|
|
prev_entries_keys_buff_.clear();
|
|
|
|
prev_entries_.clear();
|
|
|
|
prev_entries_idx_ = -1;
|
|
|
|
}
|
|
|
|
|
2018-07-13 02:19:57 +02:00
|
|
|
private:
|
|
|
|
// read-amp bitmap
|
|
|
|
BlockReadAmpBitmap* read_amp_bitmap_;
|
|
|
|
// last `current_` value we report to read-amp bitmp
|
|
|
|
mutable uint32_t last_bitmap_offset_;
|
2018-07-17 02:10:44 +02:00
|
|
|
struct CachedPrevEntry {
|
|
|
|
explicit CachedPrevEntry(uint32_t _offset, const char* _key_ptr,
|
|
|
|
size_t _key_offset, size_t _key_size, Slice _value)
|
|
|
|
: offset(_offset),
|
|
|
|
key_ptr(_key_ptr),
|
|
|
|
key_offset(_key_offset),
|
|
|
|
key_size(_key_size),
|
|
|
|
value(_value) {}
|
|
|
|
|
|
|
|
// offset of entry in block
|
|
|
|
uint32_t offset;
|
|
|
|
// Pointer to key data in block (nullptr if key is delta-encoded)
|
|
|
|
const char* key_ptr;
|
|
|
|
// offset of key in prev_entries_keys_buff_ (0 if key_ptr is not nullptr)
|
|
|
|
size_t key_offset;
|
|
|
|
// size of key
|
|
|
|
size_t key_size;
|
|
|
|
// value slice pointing to data in block
|
|
|
|
Slice value;
|
|
|
|
};
|
|
|
|
std::string prev_entries_keys_buff_;
|
|
|
|
std::vector<CachedPrevEntry> prev_entries_;
|
|
|
|
int32_t prev_entries_idx_ = -1;
|
|
|
|
|
2018-08-15 23:27:47 +02:00
|
|
|
DataBlockHashIndex* data_block_hash_index_;
|
|
|
|
const Comparator* user_comparator_;
|
|
|
|
|
2019-02-11 20:37:07 +01:00
|
|
|
template <typename DecodeEntryFunc>
|
2018-08-15 23:27:47 +02:00
|
|
|
inline bool ParseNextDataKey(const char* limit = nullptr);
|
2018-07-16 18:58:58 +02:00
|
|
|
|
|
|
|
inline int Compare(const IterKey& ikey, const Slice& b) const {
|
|
|
|
return comparator_->Compare(ikey.GetInternalKey(), b);
|
|
|
|
}
|
2018-08-15 23:27:47 +02:00
|
|
|
|
|
|
|
bool SeekForGetImpl(const Slice& target);
|
2018-07-13 02:19:57 +02:00
|
|
|
};
|
2014-07-31 01:34:35 +02:00
|
|
|
|
2018-08-10 01:49:45 +02:00
|
|
|
class IndexBlockIter final : public BlockIter<BlockHandle> {
|
2018-07-13 02:19:57 +02:00
|
|
|
public:
|
|
|
|
IndexBlockIter() : BlockIter(), prefix_index_(nullptr) {}
|
|
|
|
|
|
|
|
virtual Slice key() const override {
|
|
|
|
assert(Valid());
|
2018-07-17 02:10:44 +02:00
|
|
|
return key_.GetKey();
|
2018-07-13 02:19:57 +02:00
|
|
|
}
|
2018-08-10 01:49:45 +02:00
|
|
|
// key_includes_seq, default true, means that the keys are in internal key
|
|
|
|
// format.
|
2018-10-05 05:44:43 +02:00
|
|
|
// value_is_full, default true, means that no delta encoding is
|
2018-08-10 01:49:45 +02:00
|
|
|
// applied to values.
|
2018-07-13 02:19:57 +02:00
|
|
|
IndexBlockIter(const Comparator* comparator,
|
|
|
|
const Comparator* user_comparator, const char* data,
|
|
|
|
uint32_t restarts, uint32_t num_restarts,
|
|
|
|
BlockPrefixIndex* prefix_index, bool key_includes_seq,
|
2018-08-10 01:49:45 +02:00
|
|
|
bool value_is_full, bool block_contents_pinned)
|
2018-07-13 02:19:57 +02:00
|
|
|
: IndexBlockIter() {
|
|
|
|
Initialize(comparator, user_comparator, data, restarts, num_restarts,
|
2018-08-10 01:49:45 +02:00
|
|
|
prefix_index, key_includes_seq, block_contents_pinned,
|
2018-08-15 23:27:47 +02:00
|
|
|
value_is_full, nullptr /* data_block_hash_index */);
|
2018-07-13 02:19:57 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
void Initialize(const Comparator* comparator,
|
|
|
|
const Comparator* user_comparator, const char* data,
|
|
|
|
uint32_t restarts, uint32_t num_restarts,
|
|
|
|
BlockPrefixIndex* prefix_index, bool key_includes_seq,
|
2018-08-15 23:27:47 +02:00
|
|
|
bool value_is_full, bool block_contents_pinned,
|
|
|
|
DataBlockHashIndex* /*data_block_hash_index*/) {
|
2018-09-12 18:42:31 +02:00
|
|
|
InitializeBase(key_includes_seq ? comparator : user_comparator, data,
|
|
|
|
restarts, num_restarts, kDisableGlobalSequenceNumber,
|
|
|
|
block_contents_pinned);
|
2018-07-17 02:10:44 +02:00
|
|
|
key_includes_seq_ = key_includes_seq;
|
|
|
|
key_.SetIsUserKey(!key_includes_seq_);
|
2018-07-13 02:19:57 +02:00
|
|
|
prefix_index_ = prefix_index;
|
2018-08-10 01:49:45 +02:00
|
|
|
value_delta_encoded_ = !value_is_full;
|
|
|
|
}
|
|
|
|
|
2019-04-16 20:32:03 +02:00
|
|
|
Slice user_key() const override {
|
|
|
|
if (key_includes_seq_) {
|
|
|
|
return ExtractUserKey(key());
|
|
|
|
}
|
|
|
|
return key();
|
|
|
|
}
|
|
|
|
|
2018-08-10 01:49:45 +02:00
|
|
|
virtual BlockHandle value() const override {
|
|
|
|
assert(Valid());
|
|
|
|
if (value_delta_encoded_) {
|
|
|
|
return decoded_value_;
|
|
|
|
} else {
|
|
|
|
BlockHandle handle;
|
|
|
|
Slice v = value_;
|
|
|
|
Status decode_s __attribute__((__unused__)) = handle.DecodeFrom(&v);
|
|
|
|
assert(decode_s.ok());
|
|
|
|
return handle;
|
|
|
|
}
|
2018-07-13 02:19:57 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
virtual void Seek(const Slice& target) override;
|
|
|
|
|
2018-07-16 18:58:58 +02:00
|
|
|
virtual void SeekForPrev(const Slice&) override {
|
|
|
|
assert(false);
|
|
|
|
current_ = restarts_;
|
|
|
|
restart_index_ = num_restarts_;
|
|
|
|
status_ = Status::InvalidArgument(
|
2018-07-17 02:10:44 +02:00
|
|
|
"RocksDB internal error: should never call SeekForPrev() on index "
|
2018-07-16 18:58:58 +02:00
|
|
|
"blocks");
|
|
|
|
key_.Clear();
|
|
|
|
value_.clear();
|
|
|
|
}
|
|
|
|
|
2018-07-17 02:10:44 +02:00
|
|
|
virtual void Prev() override;
|
|
|
|
|
|
|
|
virtual void Next() override;
|
|
|
|
|
|
|
|
virtual void SeekToFirst() override;
|
|
|
|
|
|
|
|
virtual void SeekToLast() override;
|
|
|
|
|
|
|
|
void Invalidate(Status s) { InvalidateBase(s); }
|
|
|
|
|
2018-07-13 02:19:57 +02:00
|
|
|
private:
|
2018-08-10 01:49:45 +02:00
|
|
|
// Key is in InternalKey format
|
|
|
|
bool key_includes_seq_;
|
|
|
|
bool value_delta_encoded_;
|
|
|
|
BlockPrefixIndex* prefix_index_;
|
|
|
|
// Whether the value is delta encoded. In that case the value is assumed to be
|
|
|
|
// BlockHandle. The first value in each restart interval is the full encoded
|
|
|
|
// BlockHandle; the restart of encoded size part of the BlockHandle. The
|
|
|
|
// offset of delta encoded BlockHandles is computed by adding the size of
|
|
|
|
// previous delta encoded values in the same restart interval to the offset of
|
|
|
|
// the first value in that restart interval.
|
|
|
|
BlockHandle decoded_value_;
|
|
|
|
|
2018-07-13 02:19:57 +02:00
|
|
|
bool PrefixSeek(const Slice& target, uint32_t* index);
|
2014-07-31 01:34:35 +02:00
|
|
|
bool BinaryBlockIndexSeek(const Slice& target, uint32_t* block_ids,
|
2019-03-28 00:13:08 +01:00
|
|
|
uint32_t left, uint32_t right, uint32_t* index);
|
2018-08-10 01:49:45 +02:00
|
|
|
inline int CompareBlockKey(uint32_t block_index, const Slice& target);
|
2014-07-31 01:34:35 +02:00
|
|
|
|
2018-07-16 18:58:58 +02:00
|
|
|
inline int Compare(const Slice& a, const Slice& b) const {
|
2018-09-12 18:42:31 +02:00
|
|
|
return comparator_->Compare(a, b);
|
2018-07-16 18:58:58 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
inline int Compare(const IterKey& ikey, const Slice& b) const {
|
2018-09-12 18:42:31 +02:00
|
|
|
return comparator_->Compare(ikey.GetKey(), b);
|
2018-07-16 18:58:58 +02:00
|
|
|
}
|
|
|
|
|
2018-08-10 01:49:45 +02:00
|
|
|
inline bool ParseNextIndexKey();
|
2018-07-17 02:10:44 +02:00
|
|
|
|
2018-08-10 01:49:45 +02:00
|
|
|
// When value_delta_encoded_ is enabled it decodes the value which is assumed
|
|
|
|
// to be BlockHandle and put it to decoded_value_
|
|
|
|
inline void DecodeCurrentValue(uint32_t shared);
|
2011-03-18 23:37:00 +01:00
|
|
|
};
|
|
|
|
|
2013-10-04 06:49:15 +02:00
|
|
|
} // namespace rocksdb
|