2013-10-29 04:34:02 +01:00
|
|
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
|
|
|
// Use of this source code is governed by a BSD-style license that can be
|
|
|
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
|
|
|
|
|
|
|
#pragma once
|
|
|
|
#include <unordered_map>
|
|
|
|
#include <memory>
|
2014-01-25 06:10:19 +01:00
|
|
|
#include <vector>
|
2014-01-27 22:53:22 +01:00
|
|
|
#include <string>
|
2013-10-29 04:34:02 +01:00
|
|
|
#include <stdint.h>
|
2014-01-28 06:58:46 +01:00
|
|
|
|
2014-01-27 22:53:22 +01:00
|
|
|
#include "db/dbformat.h"
|
2013-10-29 04:34:02 +01:00
|
|
|
#include "rocksdb/env.h"
|
|
|
|
#include "rocksdb/iterator.h"
|
2014-01-28 19:35:48 +01:00
|
|
|
#include "rocksdb/slice_transform.h"
|
2014-01-28 06:58:46 +01:00
|
|
|
#include "rocksdb/table.h"
|
|
|
|
#include "rocksdb/table_properties.h"
|
|
|
|
#include "table/table_reader.h"
|
|
|
|
#include "table/plain_table_factory.h"
|
2013-10-29 04:34:02 +01:00
|
|
|
|
|
|
|
namespace rocksdb {
|
|
|
|
|
|
|
|
class Block;
|
|
|
|
class BlockHandle;
|
|
|
|
class Footer;
|
|
|
|
struct Options;
|
|
|
|
class RandomAccessFile;
|
|
|
|
struct ReadOptions;
|
|
|
|
class TableCache;
|
|
|
|
class TableReader;
|
2013-12-20 18:35:24 +01:00
|
|
|
class DynamicBloom;
|
2014-01-27 22:53:22 +01:00
|
|
|
class InternalKeyComparator;
|
2013-10-29 04:34:02 +01:00
|
|
|
|
|
|
|
using std::unique_ptr;
|
|
|
|
using std::unordered_map;
|
2014-01-28 06:58:46 +01:00
|
|
|
extern const uint32_t kPlainTableVariableLength;
|
2013-10-29 04:34:02 +01:00
|
|
|
|
2013-12-20 18:35:24 +01:00
|
|
|
// Based on following output file format shown in plain_table_factory.h
|
2013-10-29 04:34:02 +01:00
|
|
|
// When opening the output file, IndexedTableReader creates a hash table
|
|
|
|
// from key prefixes to offset of the output file. IndexedTable will decide
|
|
|
|
// whether it points to the data offset of the first key with the key prefix
|
|
|
|
// or the offset of it. If there are too many keys share this prefix, it will
|
|
|
|
// create a binary search-able index from the suffix to offset on disk.
|
|
|
|
//
|
|
|
|
// The implementation of IndexedTableReader requires output file is mmaped
|
|
|
|
class PlainTableReader: public TableReader {
|
2014-01-25 06:10:19 +01:00
|
|
|
public:
|
2013-10-29 04:34:02 +01:00
|
|
|
static Status Open(const Options& options, const EnvOptions& soptions,
|
2014-01-27 22:53:22 +01:00
|
|
|
const InternalKeyComparator& internal_comparator,
|
2014-01-28 19:35:48 +01:00
|
|
|
unique_ptr<RandomAccessFile>&& file, uint64_t file_size,
|
2014-01-28 06:58:46 +01:00
|
|
|
unique_ptr<TableReader>* table,
|
|
|
|
const int bloom_bits_per_key, double hash_table_ratio);
|
2013-10-29 04:34:02 +01:00
|
|
|
|
|
|
|
bool PrefixMayMatch(const Slice& internal_prefix);
|
|
|
|
|
|
|
|
Iterator* NewIterator(const ReadOptions&);
|
|
|
|
|
2014-01-27 22:53:22 +01:00
|
|
|
Status Get(const ReadOptions&, const Slice& key, void* arg,
|
|
|
|
bool (*result_handler)(void* arg, const ParsedInternalKey& k,
|
|
|
|
const Slice& v, bool),
|
|
|
|
void (*mark_key_may_exist)(void*) = nullptr);
|
2013-10-29 04:34:02 +01:00
|
|
|
|
|
|
|
uint64_t ApproximateOffsetOf(const Slice& key);
|
|
|
|
|
|
|
|
void SetupForCompaction();
|
|
|
|
|
2014-02-08 04:26:49 +01:00
|
|
|
std::shared_ptr<const TableProperties> GetTableProperties() const {
|
|
|
|
return table_properties_;
|
|
|
|
}
|
2013-10-29 04:34:02 +01:00
|
|
|
|
2014-01-27 22:53:22 +01:00
|
|
|
PlainTableReader(const EnvOptions& storage_options,
|
|
|
|
const InternalKeyComparator& internal_comparator,
|
|
|
|
uint64_t file_size, int bloom_num_bits,
|
|
|
|
double hash_table_ratio,
|
2014-02-08 04:26:49 +01:00
|
|
|
const TableProperties* table_properties);
|
2013-10-29 04:34:02 +01:00
|
|
|
~PlainTableReader();
|
|
|
|
|
2014-01-25 06:10:19 +01:00
|
|
|
private:
|
2013-12-20 18:35:24 +01:00
|
|
|
struct IndexRecord;
|
|
|
|
class IndexRecordList;
|
|
|
|
|
2013-12-06 01:51:26 +01:00
|
|
|
uint32_t* hash_table_ = nullptr;
|
2014-01-25 06:10:19 +01:00
|
|
|
int hash_table_size_ = 0;
|
2013-12-20 18:35:24 +01:00
|
|
|
char* sub_index_ = nullptr;
|
2013-10-29 04:34:02 +01:00
|
|
|
|
|
|
|
Options options_;
|
|
|
|
const EnvOptions& soptions_;
|
2014-01-27 22:53:22 +01:00
|
|
|
const InternalKeyComparator internal_comparator_;
|
2013-10-29 04:34:02 +01:00
|
|
|
Status status_;
|
|
|
|
unique_ptr<RandomAccessFile> file_;
|
|
|
|
|
2013-11-21 20:11:02 +01:00
|
|
|
Slice file_data_;
|
|
|
|
uint32_t version_;
|
|
|
|
uint32_t file_size_;
|
2013-12-20 18:35:24 +01:00
|
|
|
|
2014-01-25 06:10:19 +01:00
|
|
|
const double kHashTableRatio;
|
|
|
|
const int kBloomBitsPerKey;
|
|
|
|
DynamicBloom* bloom_ = nullptr;
|
2013-10-29 04:34:02 +01:00
|
|
|
|
2014-02-08 04:26:49 +01:00
|
|
|
std::shared_ptr<const TableProperties> table_properties_;
|
2014-01-25 06:10:19 +01:00
|
|
|
const uint32_t data_start_offset_ = 0;
|
2013-12-20 18:35:24 +01:00
|
|
|
const uint32_t data_end_offset_;
|
|
|
|
const size_t user_key_len_;
|
2013-10-29 04:34:02 +01:00
|
|
|
|
|
|
|
static const size_t kNumInternalBytes = 8;
|
2013-11-21 20:11:02 +01:00
|
|
|
static const uint32_t kSubIndexMask = 0x80000000;
|
|
|
|
static const size_t kOffsetLen = sizeof(uint32_t);
|
2014-01-25 06:10:19 +01:00
|
|
|
static const uint64_t kMaxFileSize = 1u << 31;
|
|
|
|
static const size_t kRecordsPerGroup = 256;
|
|
|
|
// To speed up the search for keys with same prefix, we'll add index key for
|
|
|
|
// every N keys, where the "N" is determined by
|
|
|
|
// kIndexIntervalForSamePrefixKeys
|
|
|
|
static const size_t kIndexIntervalForSamePrefixKeys = 16;
|
|
|
|
|
|
|
|
bool IsFixedLength() const {
|
2014-01-28 06:58:46 +01:00
|
|
|
return user_key_len_ != kPlainTableVariableLength;
|
2013-12-20 18:35:24 +01:00
|
|
|
}
|
|
|
|
|
2014-01-25 06:10:19 +01:00
|
|
|
size_t GetFixedInternalKeyLength() const {
|
2013-12-20 18:35:24 +01:00
|
|
|
return user_key_len_ + kNumInternalBytes;
|
2013-10-29 04:34:02 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
friend class TableCache;
|
|
|
|
friend class PlainTableIterator;
|
|
|
|
|
2013-12-20 18:35:24 +01:00
|
|
|
// Internal helper function to generate an IndexRecordList object from all
|
|
|
|
// the rows, which contains index records as a list.
|
2014-01-25 06:10:19 +01:00
|
|
|
int PopulateIndexRecordList(IndexRecordList* record_list);
|
2013-12-20 18:35:24 +01:00
|
|
|
|
|
|
|
// Internal helper function to allocate memory for indexes and bloom filters
|
2014-01-25 06:10:19 +01:00
|
|
|
void AllocateIndexAndBloom(int num_prefixes);
|
2013-12-20 18:35:24 +01:00
|
|
|
|
|
|
|
// Internal helper function to bucket index record list to hash buckets.
|
2014-01-25 06:10:19 +01:00
|
|
|
// hash_to_offsets is sized of of hash_table_size_, each contains a linked
|
|
|
|
// list
|
2013-12-20 18:35:24 +01:00
|
|
|
// of offsets for the hash, in reversed order.
|
|
|
|
// bucket_count is sized of hash_table_size_. The value is how many index
|
2014-01-25 06:10:19 +01:00
|
|
|
// records are there in hash_to_offsets for the same bucket.
|
2013-12-20 18:35:24 +01:00
|
|
|
size_t BucketizeIndexesAndFillBloom(
|
|
|
|
IndexRecordList& record_list, int num_prefixes,
|
2014-01-25 06:10:19 +01:00
|
|
|
std::vector<IndexRecord*>* hash_to_offsets,
|
|
|
|
std::vector<uint32_t>* bucket_count);
|
2013-12-20 18:35:24 +01:00
|
|
|
|
|
|
|
// Internal helper class to fill the indexes and bloom filters to internal
|
2014-01-25 06:10:19 +01:00
|
|
|
// data structures. hash_to_offsets and bucket_count are bucketized indexes
|
|
|
|
// and counts generated by BucketizeIndexesAndFillBloom().
|
2013-12-20 18:35:24 +01:00
|
|
|
void FillIndexes(size_t sub_index_size_needed,
|
2014-01-25 06:10:19 +01:00
|
|
|
const std::vector<IndexRecord*>& hash_to_offsets,
|
|
|
|
const std::vector<uint32_t>& bucket_count);
|
|
|
|
|
|
|
|
// PopulateIndex() builds index of keys. It must be called before any query
|
|
|
|
// to the table.
|
|
|
|
//
|
|
|
|
// hash_table_ contains buckets size of hash_table_size_, each is a 32-bit
|
|
|
|
// integer. The lower 31 bits contain an offset value (explained below) and
|
|
|
|
// the first bit of the integer indicates type of the offset.
|
|
|
|
//
|
|
|
|
// +--------------+------------------------------------------------------+
|
|
|
|
// | Flag (1 bit) | Offset to binary search buffer or file (31 bits) +
|
|
|
|
// +--------------+------------------------------------------------------+
|
|
|
|
//
|
|
|
|
// Explanation for the "flag bit":
|
|
|
|
//
|
|
|
|
// 0 indicates that the bucket contains only one prefix (no conflict when
|
|
|
|
// hashing this prefix), whose first row starts from this offset of the
|
|
|
|
// file.
|
|
|
|
// 1 indicates that the bucket contains more than one prefixes, or there
|
|
|
|
// are too many rows for one prefix so we need a binary search for it. In
|
|
|
|
// this case, the offset indicates the offset of sub_index_ holding the
|
|
|
|
// binary search indexes of keys for those rows. Those binary search indexes
|
|
|
|
// are organized in this way:
|
|
|
|
//
|
|
|
|
// The first 4 bytes, indicate how many indexes (N) are stored after it. After
|
|
|
|
// it, there are N 32-bit integers, each points of an offset of the file,
|
|
|
|
// which
|
|
|
|
// points to starting of a row. Those offsets need to be guaranteed to be in
|
|
|
|
// ascending order so the keys they are pointing to are also in ascending
|
|
|
|
// order
|
|
|
|
// to make sure we can use them to do binary searches. Below is visual
|
|
|
|
// presentation of a bucket.
|
|
|
|
//
|
|
|
|
// <begin>
|
|
|
|
// number_of_records: varint32
|
|
|
|
// record 1 file offset: fixedint32
|
|
|
|
// record 2 file offset: fixedint32
|
|
|
|
// ....
|
|
|
|
// record N file offset: fixedint32
|
|
|
|
// <end>
|
2013-12-06 01:51:26 +01:00
|
|
|
Status PopulateIndex();
|
2013-11-21 20:11:02 +01:00
|
|
|
|
2013-12-20 18:35:24 +01:00
|
|
|
// Check bloom filter to see whether it might contain this prefix.
|
|
|
|
// The hash of the prefix is given, since it can be reused for index lookup
|
|
|
|
// too.
|
|
|
|
bool MayHavePrefix(uint32_t hash);
|
2013-10-29 04:34:02 +01:00
|
|
|
|
2014-01-27 22:53:22 +01:00
|
|
|
Status ReadKey(const char* row_ptr, ParsedInternalKey* key,
|
|
|
|
size_t& bytes_read);
|
2013-11-21 20:11:02 +01:00
|
|
|
// Read the key and value at offset to key and value.
|
|
|
|
// tmp_slice is a tmp slice.
|
|
|
|
// return next_offset as the offset for the next key.
|
2014-01-27 22:53:22 +01:00
|
|
|
Status Next(uint32_t offset, ParsedInternalKey* key, Slice* value,
|
|
|
|
uint32_t& next_offset);
|
2013-11-21 20:11:02 +01:00
|
|
|
// Get file offset for key target.
|
|
|
|
// return value prefix_matched is set to true if the offset is confirmed
|
|
|
|
// for a key with the same prefix as target.
|
2013-12-20 18:35:24 +01:00
|
|
|
Status GetOffset(const Slice& target, const Slice& prefix,
|
|
|
|
uint32_t prefix_hash, bool& prefix_matched,
|
|
|
|
uint32_t& ret_offset);
|
|
|
|
|
|
|
|
Slice GetPrefix(const Slice& target) {
|
2014-01-28 19:35:48 +01:00
|
|
|
assert(target.size() >= 8); // target is internal key
|
2013-12-20 18:35:24 +01:00
|
|
|
return options_.prefix_extractor->Transform(
|
|
|
|
Slice(target.data(), target.size() - 8));
|
|
|
|
}
|
2013-11-21 20:11:02 +01:00
|
|
|
|
2014-01-27 22:53:22 +01:00
|
|
|
Slice GetPrefix(const ParsedInternalKey& target);
|
|
|
|
|
2013-10-29 04:34:02 +01:00
|
|
|
// No copying allowed
|
|
|
|
explicit PlainTableReader(const TableReader&) = delete;
|
|
|
|
void operator=(const TableReader&) = delete;
|
|
|
|
};
|
|
|
|
} // namespace rocksdb
|