Summary: According to https://github.com/facebook/rocksdb/issues/5907, each filter partition "should include the bloom of the prefix of the last key in the previous partition" so that SeekForPrev() in prefix mode can return correct result. The prefix of the last key in the previous partition does not necessarily have the same prefix as the first key in the current partition. Regardless of the first key in current partition, the prefix of the last key in the previous partition should be added. The existing code, however, does not follow this. Furthermore, there is another issue: when finishing current filter partition, `FullFilterBlockBuilder::AddPrefix()` is called for the first key in next filter partition, which effectively overwrites `last_prefix_str_` prematurely. Consequently, when the filter block builder proceeds to the next partition, `last_prefix_str_` will be the prefix of its first key, leaving no way of adding the bloom of the prefix of the last key of the previous partition. Prefix extractor is FixedLength.2. ``` [ filter part 1 ] [ filter part 2 ] abc d ``` When SeekForPrev("abcd"), checking the filter partition will land on filter part 2 because "abcd" > "abc" but smaller than "d". If the filter in filter part 2 happens to return false for the test for "ab", then SeekForPrev("abcd") will build incorrect iterator tree in non-total-order mode. Also fix a unit test which starts to fail following this PR. `InDomain` should not fail due to assertion error when checking on an arbitrary key. Pull Request resolved: https://github.com/facebook/rocksdb/pull/8137 Test Plan: ``` make check ``` Without this fix, the following command will fail pretty soon. ``` ./db_stress --acquire_snapshot_one_in=10000 --avoid_flush_during_recovery=0 \ --avoid_unnecessary_blocking_io=0 --backup_max_size=104857600 --backup_one_in=0 \ --batch_protection_bytes_per_key=0 --block_size=16384 --bloom_bits=17 \ --bottommost_compression_type=disable --cache_index_and_filter_blocks=1 --cache_size=1048576 \ --checkpoint_one_in=0 --checksum_type=kxxHash64 --clear_column_family_one_in=0 \ --compact_files_one_in=1000000 --compact_range_one_in=1000000 --compaction_ttl=0 \ --compression_max_dict_buffer_bytes=0 --compression_max_dict_bytes=0 \ --compression_parallel_threads=1 --compression_type=zstd --compression_zstd_max_train_bytes=0 \ --continuous_verification_interval=0 --db=/dev/shm/rocksdb/rocksdb_crashtest_whitebox \ --db_write_buffer_size=8388608 --delpercent=5 --delrangepercent=0 --destroy_db_initially=0 --enable_blob_files=0 \ --enable_compaction_filter=0 --enable_pipelined_write=1 --file_checksum_impl=big --flush_one_in=1000000 \ --format_version=5 --get_current_wal_file_one_in=0 --get_live_files_one_in=1000000 --get_property_one_in=1000000 \ --get_sorted_wal_files_one_in=0 --index_block_restart_interval=4 --index_type=2 --ingest_external_file_one_in=0 \ --iterpercent=10 --key_len_percent_dist=1,30,69 --level_compaction_dynamic_level_bytes=True \ --log2_keys_per_lock=10 --long_running_snapshots=1 --mark_for_compaction_one_file_in=0 \ --max_background_compactions=20 --max_bytes_for_level_base=10485760 --max_key=100000000 --max_key_len=3 \ --max_manifest_file_size=1073741824 --max_write_batch_group_size_bytes=16777216 --max_write_buffer_number=3 \ --max_write_buffer_size_to_maintain=8388608 --memtablerep=skip_list --mmap_read=1 --mock_direct_io=False \ --nooverwritepercent=0 --open_files=500000 --ops_per_thread=20000000 --optimize_filters_for_memory=0 --paranoid_file_checks=1 --partition_filters=1 --partition_pinning=0 --pause_background_one_in=1000000 \ --periodic_compaction_seconds=0 --prefixpercent=5 --progress_reports=0 --read_fault_one_in=0 --read_only=0 \ --readpercent=45 --recycle_log_file_num=0 --reopen=20 --secondary_catch_up_one_in=0 \ --snapshot_hold_ops=100000 --sst_file_manager_bytes_per_sec=104857600 \ --sst_file_manager_bytes_per_truncate=0 --subcompactions=2 --sync=0 --sync_fault_injection=False \ --target_file_size_base=2097152 --target_file_size_multiplier=2 --test_batches_snapshots=0 --test_cf_consistency=0 \ --top_level_index_pinning=0 --unpartitioned_pinning=1 --use_blob_db=0 --use_block_based_filter=0 \ --use_direct_io_for_flush_and_compaction=0 --use_direct_reads=0 --use_full_merge_v1=0 --use_merge=0 \ --use_multiget=0 --use_ribbon_filter=0 --use_txn=0 --user_timestamp_size=8 --verify_checksum=1 \ --verify_checksum_one_in=1000000 --verify_db_one_in=100000 --write_buffer_size=4194304 \ --write_dbid_to_manifest=1 --writepercent=35 ``` Reviewed By: pdillinger Differential Revision: D27553054 Pulled By: riversand963 fbshipit-source-id: 60e391e4a2d8d98a9a3172ec5d6176b90ec3de98
145 lines
5.9 KiB
C++
145 lines
5.9 KiB
C++
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
|
// This source code is licensed under both the GPLv2 (found in the
|
|
// COPYING file in the root directory) and Apache 2.0 License
|
|
// (found in the LICENSE.Apache file in the root directory).
|
|
|
|
#pragma once
|
|
|
|
#include <stddef.h>
|
|
#include <stdint.h>
|
|
#include <memory>
|
|
#include <string>
|
|
#include <vector>
|
|
|
|
#include "db/dbformat.h"
|
|
#include "rocksdb/options.h"
|
|
#include "rocksdb/slice.h"
|
|
#include "rocksdb/slice_transform.h"
|
|
#include "table/block_based/filter_block_reader_common.h"
|
|
#include "table/block_based/parsed_full_filter_block.h"
|
|
#include "util/hash.h"
|
|
|
|
namespace ROCKSDB_NAMESPACE {
|
|
|
|
class FilterPolicy;
|
|
class FilterBitsBuilder;
|
|
class FilterBitsReader;
|
|
|
|
// A FullFilterBlockBuilder is used to construct a full filter for a
|
|
// particular Table. It generates a single string which is stored as
|
|
// a special block in the Table.
|
|
// The format of full filter block is:
|
|
// +----------------------------------------------------------------+
|
|
// | full filter for all keys in sst file |
|
|
// +----------------------------------------------------------------+
|
|
// The full filter can be very large. At the end of it, we put
|
|
// num_probes: how many hash functions are used in bloom filter
|
|
//
|
|
class FullFilterBlockBuilder : public FilterBlockBuilder {
|
|
public:
|
|
explicit FullFilterBlockBuilder(const SliceTransform* prefix_extractor,
|
|
bool whole_key_filtering,
|
|
FilterBitsBuilder* filter_bits_builder);
|
|
// No copying allowed
|
|
FullFilterBlockBuilder(const FullFilterBlockBuilder&) = delete;
|
|
void operator=(const FullFilterBlockBuilder&) = delete;
|
|
|
|
// bits_builder is created in filter_policy, it should be passed in here
|
|
// directly. and be deleted here
|
|
~FullFilterBlockBuilder() {}
|
|
|
|
virtual bool IsBlockBased() override { return false; }
|
|
virtual void StartBlock(uint64_t /*block_offset*/) override {}
|
|
virtual void Add(const Slice& key_without_ts) override;
|
|
virtual size_t NumAdded() const override { return num_added_; }
|
|
virtual Slice Finish(const BlockHandle& tmp, Status* status) override;
|
|
using FilterBlockBuilder::Finish;
|
|
|
|
protected:
|
|
virtual void AddKey(const Slice& key);
|
|
std::unique_ptr<FilterBitsBuilder> filter_bits_builder_;
|
|
virtual void Reset();
|
|
void AddPrefix(const Slice& key);
|
|
const SliceTransform* prefix_extractor() { return prefix_extractor_; }
|
|
const std::string& last_prefix_str() const { return last_prefix_str_; }
|
|
|
|
private:
|
|
// important: all of these might point to invalid addresses
|
|
// at the time of destruction of this filter block. destructor
|
|
// should NOT dereference them.
|
|
const SliceTransform* prefix_extractor_;
|
|
bool whole_key_filtering_;
|
|
bool last_whole_key_recorded_;
|
|
std::string last_whole_key_str_;
|
|
bool last_prefix_recorded_;
|
|
std::string last_prefix_str_;
|
|
// Whether prefix_extractor_->InDomain(last_whole_key_) is true.
|
|
// Used in partitioned filters so that the last prefix from the previous
|
|
// filter partition will be added to the current partition if
|
|
// last_key_in_domain_ is true, regardless of the current key.
|
|
bool last_key_in_domain_;
|
|
|
|
uint32_t num_added_;
|
|
std::unique_ptr<const char[]> filter_data_;
|
|
};
|
|
|
|
// A FilterBlockReader is used to parse filter from SST table.
|
|
// KeyMayMatch and PrefixMayMatch would trigger filter checking
|
|
class FullFilterBlockReader
|
|
: public FilterBlockReaderCommon<ParsedFullFilterBlock> {
|
|
public:
|
|
FullFilterBlockReader(const BlockBasedTable* t,
|
|
CachableEntry<ParsedFullFilterBlock>&& filter_block);
|
|
|
|
static std::unique_ptr<FilterBlockReader> Create(
|
|
const BlockBasedTable* table, const ReadOptions& ro,
|
|
FilePrefetchBuffer* prefetch_buffer, bool use_cache, bool prefetch,
|
|
bool pin, BlockCacheLookupContext* lookup_context);
|
|
|
|
bool IsBlockBased() override { return false; }
|
|
|
|
bool KeyMayMatch(const Slice& key, const SliceTransform* prefix_extractor,
|
|
uint64_t block_offset, const bool no_io,
|
|
const Slice* const const_ikey_ptr, GetContext* get_context,
|
|
BlockCacheLookupContext* lookup_context) override;
|
|
|
|
bool PrefixMayMatch(const Slice& prefix,
|
|
const SliceTransform* prefix_extractor,
|
|
uint64_t block_offset, const bool no_io,
|
|
const Slice* const const_ikey_ptr,
|
|
GetContext* get_context,
|
|
BlockCacheLookupContext* lookup_context) override;
|
|
|
|
void KeysMayMatch(MultiGetRange* range,
|
|
const SliceTransform* prefix_extractor,
|
|
uint64_t block_offset, const bool no_io,
|
|
BlockCacheLookupContext* lookup_context) override;
|
|
|
|
void PrefixesMayMatch(MultiGetRange* range,
|
|
const SliceTransform* prefix_extractor,
|
|
uint64_t block_offset, const bool no_io,
|
|
BlockCacheLookupContext* lookup_context) override;
|
|
size_t ApproximateMemoryUsage() const override;
|
|
bool RangeMayExist(const Slice* iterate_upper_bound, const Slice& user_key,
|
|
const SliceTransform* prefix_extractor,
|
|
const Comparator* comparator,
|
|
const Slice* const const_ikey_ptr, bool* filter_checked,
|
|
bool need_upper_bound_check, bool no_io,
|
|
BlockCacheLookupContext* lookup_context) override;
|
|
|
|
private:
|
|
bool MayMatch(const Slice& entry, bool no_io, GetContext* get_context,
|
|
BlockCacheLookupContext* lookup_context) const;
|
|
void MayMatch(MultiGetRange* range, bool no_io,
|
|
const SliceTransform* prefix_extractor,
|
|
BlockCacheLookupContext* lookup_context) const;
|
|
bool IsFilterCompatible(const Slice* iterate_upper_bound, const Slice& prefix,
|
|
const Comparator* comparator) const;
|
|
|
|
private:
|
|
bool full_length_enabled_;
|
|
size_t prefix_extractor_full_length_;
|
|
};
|
|
|
|
} // namespace ROCKSDB_NAMESPACE
|