2019-10-08 05:09:27 +02:00
|
|
|
// Copyright (c) 2019-present, Facebook, Inc. All rights reserved.
|
|
|
|
// This source code is licensed under both the GPLv2 (found in the
|
|
|
|
// COPYING file in the root directory) and Apache 2.0 License
|
|
|
|
// (found in the LICENSE.Apache file in the root directory).
|
|
|
|
#pragma once
|
|
|
|
|
|
|
|
#include "rocksdb/filter_policy.h"
|
2019-11-27 03:18:29 +01:00
|
|
|
#include "table/block_based/block_based_filter_block.h"
|
2019-10-08 05:09:27 +02:00
|
|
|
#include "table/block_based/block_based_table_reader.h"
|
2019-11-27 03:18:29 +01:00
|
|
|
#include "table/block_based/filter_policy_internal.h"
|
2019-10-08 05:09:27 +02:00
|
|
|
|
2020-02-20 21:07:53 +01:00
|
|
|
namespace ROCKSDB_NAMESPACE {
|
2019-10-08 05:09:27 +02:00
|
|
|
namespace mock {
|
|
|
|
|
|
|
|
class MockBlockBasedTable : public BlockBasedTable {
|
|
|
|
public:
|
|
|
|
explicit MockBlockBasedTable(Rep* rep)
|
|
|
|
: BlockBasedTable(rep, nullptr /* block_cache_tracer */) {}
|
|
|
|
};
|
|
|
|
|
|
|
|
class MockBlockBasedTableTester {
|
2019-11-27 03:18:29 +01:00
|
|
|
static constexpr int kMockLevel = 0;
|
|
|
|
|
2019-10-08 05:09:27 +02:00
|
|
|
public:
|
|
|
|
Options options_;
|
|
|
|
ImmutableCFOptions ioptions_;
|
|
|
|
EnvOptions env_options_;
|
|
|
|
BlockBasedTableOptions table_options_;
|
|
|
|
InternalKeyComparator icomp_;
|
|
|
|
std::unique_ptr<BlockBasedTable> table_;
|
|
|
|
|
|
|
|
MockBlockBasedTableTester(const FilterPolicy *filter_policy)
|
|
|
|
: ioptions_(options_),
|
|
|
|
env_options_(options_),
|
|
|
|
icomp_(options_.comparator) {
|
|
|
|
table_options_.filter_policy.reset(filter_policy);
|
|
|
|
|
|
|
|
constexpr bool skip_filters = false;
|
|
|
|
constexpr bool immortal_table = false;
|
2019-11-27 03:18:29 +01:00
|
|
|
table_.reset(new MockBlockBasedTable(new BlockBasedTable::Rep(
|
|
|
|
ioptions_, env_options_, table_options_, icomp_, skip_filters,
|
For ApproximateSizes, pro-rate table metadata size over data blocks (#6784)
Summary:
The implementation of GetApproximateSizes was inconsistent in
its treatment of the size of non-data blocks of SST files, sometimes
including and sometimes now. This was at its worst with large portion
of table file used by filters and querying a small range that crossed
a table boundary: the size estimate would include large filter size.
It's conceivable that someone might want only to know the size in terms
of data blocks, but I believe that's unlikely enough to ignore for now.
Similarly, there's no evidence the internal function AppoximateOffsetOf
is used for anything other than a one-sided ApproximateSize, so I intend
to refactor to remove redundancy in a follow-up commit.
So to fix this, GetApproximateSizes (and implementation details
ApproximateSize and ApproximateOffsetOf) now consistently include in
their returned sizes a portion of table file metadata (incl filters
and indexes) based on the size portion of the data blocks in range. In
other words, if a key range covers data blocks that are X% by size of all
the table's data blocks, returned approximate size is X% of the total
file size. It would technically be more accurate to attribute metadata
based on number of keys, but that's not computationally efficient with
data available and rarely a meaningful difference.
Also includes miscellaneous comment improvements / clarifications.
Also included is a new approximatesizerandom benchmark for db_bench.
No significant performance difference seen with this change, whether ~700 ops/sec with cache_index_and_filter_blocks and small cache or ~150k ops/sec without cache_index_and_filter_blocks.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6784
Test Plan:
Test added to DBTest.ApproximateSizesFilesWithErrorMargin.
Old code running new test...
[ RUN ] DBTest.ApproximateSizesFilesWithErrorMargin
db/db_test.cc:1562: Failure
Expected: (size) <= (11 * 100), actual: 9478 vs 1100
Other tests updated to reflect consistent accounting of metadata.
Reviewed By: siying
Differential Revision: D21334706
Pulled By: pdillinger
fbshipit-source-id: 6f86870e45213334fedbe9c73b4ebb1d8d611185
2020-06-02 21:27:59 +02:00
|
|
|
12345 /*file_size*/, kMockLevel, immortal_table)));
|
2019-11-27 03:18:29 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
FilterBitsBuilder* GetBuilder() const {
|
|
|
|
FilterBuildingContext context(table_options_);
|
|
|
|
context.column_family_name = "mock_cf";
|
|
|
|
context.compaction_style = ioptions_.compaction_style;
|
|
|
|
context.level_at_creation = kMockLevel;
|
2020-01-18 04:36:09 +01:00
|
|
|
context.info_log = ioptions_.info_log;
|
2019-11-27 03:18:29 +01:00
|
|
|
return BloomFilterPolicy::GetBuilderFromContext(context);
|
2019-10-08 05:09:27 +02:00
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
} // namespace mock
|
2020-02-20 21:07:53 +01:00
|
|
|
} // namespace ROCKSDB_NAMESPACE
|