2016-02-10 00:12:00 +01:00
|
|
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
2013-10-16 23:59:46 +02:00
|
|
|
// This source code is licensed under the BSD-style license found in the
|
|
|
|
// LICENSE file in the root directory of this source tree. An additional grant
|
|
|
|
// of patent rights can be found in the PATENTS file in the same directory.
|
2017-04-28 02:50:56 +02:00
|
|
|
// This source code is also licensed under the GPLv2 license found in the
|
|
|
|
// COPYING file in the root directory of this source tree.
|
2013-10-16 23:59:46 +02:00
|
|
|
//
|
2011-03-18 23:37:00 +01:00
|
|
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
|
|
|
// Use of this source code is governed by a BSD-style license that can be
|
|
|
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
|
|
|
|
2013-10-29 01:54:09 +01:00
|
|
|
#include "table/block_based_table_builder.h"
|
2011-03-18 23:37:00 +01:00
|
|
|
|
|
|
|
#include <assert.h>
|
2013-11-20 01:29:42 +01:00
|
|
|
#include <inttypes.h>
|
|
|
|
#include <stdio.h>
|
2013-10-10 20:43:24 +02:00
|
|
|
|
2017-02-07 01:29:29 +01:00
|
|
|
#include <list>
|
2014-03-01 03:19:07 +01:00
|
|
|
#include <map>
|
|
|
|
#include <memory>
|
2014-05-15 23:09:03 +02:00
|
|
|
#include <string>
|
|
|
|
#include <unordered_map>
|
2014-08-16 00:05:09 +02:00
|
|
|
#include <utility>
|
2014-03-01 03:19:07 +01:00
|
|
|
|
|
|
|
#include "db/dbformat.h"
|
|
|
|
|
2013-09-02 08:23:40 +02:00
|
|
|
#include "rocksdb/cache.h"
|
2013-08-23 17:38:13 +02:00
|
|
|
#include "rocksdb/comparator.h"
|
|
|
|
#include "rocksdb/env.h"
|
|
|
|
#include "rocksdb/filter_policy.h"
|
2014-03-01 03:19:07 +01:00
|
|
|
#include "rocksdb/flush_block_policy.h"
|
2016-04-21 19:16:28 +02:00
|
|
|
#include "rocksdb/merge_operator.h"
|
2014-03-01 03:19:07 +01:00
|
|
|
#include "rocksdb/table.h"
|
|
|
|
|
2013-09-02 08:23:40 +02:00
|
|
|
#include "table/block.h"
|
2017-03-04 03:09:43 +01:00
|
|
|
#include "table/block_based_filter_block.h"
|
|
|
|
#include "table/block_based_table_factory.h"
|
2014-03-01 03:19:07 +01:00
|
|
|
#include "table/block_based_table_reader.h"
|
2011-03-18 23:37:00 +01:00
|
|
|
#include "table/block_builder.h"
|
2012-04-17 17:36:46 +02:00
|
|
|
#include "table/filter_block.h"
|
2011-03-18 23:37:00 +01:00
|
|
|
#include "table/format.h"
|
2017-03-04 03:09:43 +01:00
|
|
|
#include "table/full_filter_block.h"
|
2013-12-05 22:09:13 +01:00
|
|
|
#include "table/meta_blocks.h"
|
2014-03-01 03:19:07 +01:00
|
|
|
#include "table/table_builder.h"
|
|
|
|
|
Add more table properties to EventLogger
Summary:
Example output:
{"time_micros": 1431463794310521, "job": 353, "event": "table_file_creation", "file_number": 387, "file_size": 86937, "table_info": {"data_size": "81801", "index_size": "9751", "filter_size": "0", "raw_key_size": "23448", "raw_average_key_size": "24.000000", "raw_value_size": "990571", "raw_average_value_size": "1013.890481", "num_data_blocks": "245", "num_entries": "977", "filter_policy_name": "", "kDeletedKeys": "0"}}
Also fixed a bug where BuildTable() in recovery was passing Env::IOHigh argument into paranoid_checks_file parameter.
Test Plan: make check + check out the output in the log
Reviewers: sdong, rven, yhchiang
Reviewed By: yhchiang
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D38343
2015-05-13 00:53:55 +02:00
|
|
|
#include "util/string_util.h"
|
2011-03-18 23:37:00 +01:00
|
|
|
#include "util/coding.h"
|
2015-01-09 21:57:11 +01:00
|
|
|
#include "util/compression.h"
|
2011-03-18 23:37:00 +01:00
|
|
|
#include "util/crc32c.h"
|
2013-06-17 19:11:10 +02:00
|
|
|
#include "util/stop_watch.h"
|
2014-05-01 20:09:32 +02:00
|
|
|
#include "util/xxhash.h"
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2017-03-07 22:48:02 +01:00
|
|
|
#include "table/index_builder.h"
|
|
|
|
#include "table/partitioned_filter_block.h"
|
|
|
|
|
2013-10-04 06:49:15 +02:00
|
|
|
namespace rocksdb {
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2014-05-15 23:09:03 +02:00
|
|
|
extern const std::string kHashIndexPrefixesBlock;
|
|
|
|
extern const std::string kHashIndexPrefixesMetadataBlock;
|
2013-10-10 20:43:24 +02:00
|
|
|
|
2014-03-01 03:19:07 +01:00
|
|
|
typedef BlockBasedTableOptions::IndexType IndexType;
|
2014-04-10 23:19:43 +02:00
|
|
|
|
2014-11-13 20:39:30 +01:00
|
|
|
// Without anonymous namespace here, we fail the warning -Wmissing-prototypes
|
|
|
|
namespace {
|
|
|
|
|
2017-03-07 22:48:02 +01:00
|
|
|
// Create a filter block builder based on its type.
|
|
|
|
FilterBlockBuilder* CreateFilterBlockBuilder(
|
|
|
|
const ImmutableCFOptions& opt, const BlockBasedTableOptions& table_opt,
|
|
|
|
PartitionedIndexBuilder* const p_index_builder) {
|
2014-09-08 19:37:05 +02:00
|
|
|
if (table_opt.filter_policy == nullptr) return nullptr;
|
|
|
|
|
|
|
|
FilterBitsBuilder* filter_bits_builder =
|
|
|
|
table_opt.filter_policy->GetFilterBitsBuilder();
|
|
|
|
if (filter_bits_builder == nullptr) {
|
|
|
|
return new BlockBasedFilterBlockBuilder(opt.prefix_extractor, table_opt);
|
|
|
|
} else {
|
2017-03-07 22:48:02 +01:00
|
|
|
if (table_opt.partition_filters) {
|
|
|
|
assert(p_index_builder != nullptr);
|
|
|
|
return new PartitionedFilterBlockBuilder(
|
|
|
|
opt.prefix_extractor, table_opt.whole_key_filtering,
|
|
|
|
filter_bits_builder, table_opt.index_block_restart_interval,
|
|
|
|
p_index_builder);
|
|
|
|
} else {
|
|
|
|
return new FullFilterBlockBuilder(opt.prefix_extractor,
|
|
|
|
table_opt.whole_key_filtering,
|
|
|
|
filter_bits_builder);
|
|
|
|
}
|
2014-09-08 19:37:05 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-03-01 03:19:07 +01:00
|
|
|
bool GoodCompressionRatio(size_t compressed_size, size_t raw_size) {
|
2013-10-10 20:43:24 +02:00
|
|
|
// Check to see if compressed less than 12.5%
|
|
|
|
return compressed_size < raw_size - (raw_size / 8u);
|
|
|
|
}
|
|
|
|
|
2016-08-01 23:50:19 +02:00
|
|
|
} // namespace
|
|
|
|
|
2015-01-15 01:24:24 +01:00
|
|
|
// format_version is the block format as defined in include/rocksdb/table.h
|
2014-03-01 03:19:07 +01:00
|
|
|
Slice CompressBlock(const Slice& raw,
|
|
|
|
const CompressionOptions& compression_options,
|
2015-01-15 01:24:24 +01:00
|
|
|
CompressionType* type, uint32_t format_version,
|
Shared dictionary compression using reference block
Summary:
This adds a new metablock containing a shared dictionary that is used
to compress all data blocks in the SST file. The size of the shared dictionary
is configurable in CompressionOptions and defaults to 0. It's currently only
used for zlib/lz4/lz4hc, but the block will be stored in the SST regardless of
the compression type if the user chooses a nonzero dictionary size.
During compaction, computes the dictionary by randomly sampling the first
output file in each subcompaction. It pre-computes the intervals to sample
by assuming the output file will have the maximum allowable length. In case
the file is smaller, some of the pre-computed sampling intervals can be beyond
end-of-file, in which case we skip over those samples and the dictionary will
be a bit smaller. After the dictionary is generated using the first file in a
subcompaction, it is loaded into the compression library before writing each
block in each subsequent file of that subcompaction.
On the read path, gets the dictionary from the metablock, if it exists. Then,
loads that dictionary into the compression library before reading each block.
Test Plan: new unit test
Reviewers: yhchiang, IslamAbdelRahman, cyan, sdong
Reviewed By: sdong
Subscribers: andrewkr, yoshinorim, kradhakrishnan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D52287
2016-04-28 02:36:03 +02:00
|
|
|
const Slice& compression_dict,
|
2015-01-15 01:24:24 +01:00
|
|
|
std::string* compressed_output) {
|
2014-03-01 03:19:07 +01:00
|
|
|
if (*type == kNoCompression) {
|
|
|
|
return raw;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Will return compressed block contents if (1) the compression method is
|
|
|
|
// supported in this platform and (2) the compression rate is "good enough".
|
|
|
|
switch (*type) {
|
|
|
|
case kSnappyCompression:
|
2015-01-09 21:57:11 +01:00
|
|
|
if (Snappy_Compress(compression_options, raw.data(), raw.size(),
|
|
|
|
compressed_output) &&
|
2014-03-01 03:19:07 +01:00
|
|
|
GoodCompressionRatio(compressed_output->size(), raw.size())) {
|
|
|
|
return *compressed_output;
|
|
|
|
}
|
|
|
|
break; // fall back to no compression.
|
|
|
|
case kZlibCompression:
|
2015-01-15 01:24:24 +01:00
|
|
|
if (Zlib_Compress(
|
|
|
|
compression_options,
|
|
|
|
GetCompressFormatForVersion(kZlibCompression, format_version),
|
Shared dictionary compression using reference block
Summary:
This adds a new metablock containing a shared dictionary that is used
to compress all data blocks in the SST file. The size of the shared dictionary
is configurable in CompressionOptions and defaults to 0. It's currently only
used for zlib/lz4/lz4hc, but the block will be stored in the SST regardless of
the compression type if the user chooses a nonzero dictionary size.
During compaction, computes the dictionary by randomly sampling the first
output file in each subcompaction. It pre-computes the intervals to sample
by assuming the output file will have the maximum allowable length. In case
the file is smaller, some of the pre-computed sampling intervals can be beyond
end-of-file, in which case we skip over those samples and the dictionary will
be a bit smaller. After the dictionary is generated using the first file in a
subcompaction, it is loaded into the compression library before writing each
block in each subsequent file of that subcompaction.
On the read path, gets the dictionary from the metablock, if it exists. Then,
loads that dictionary into the compression library before reading each block.
Test Plan: new unit test
Reviewers: yhchiang, IslamAbdelRahman, cyan, sdong
Reviewed By: sdong
Subscribers: andrewkr, yoshinorim, kradhakrishnan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D52287
2016-04-28 02:36:03 +02:00
|
|
|
raw.data(), raw.size(), compressed_output, compression_dict) &&
|
2014-03-01 03:19:07 +01:00
|
|
|
GoodCompressionRatio(compressed_output->size(), raw.size())) {
|
|
|
|
return *compressed_output;
|
|
|
|
}
|
|
|
|
break; // fall back to no compression.
|
|
|
|
case kBZip2Compression:
|
2015-01-15 01:24:24 +01:00
|
|
|
if (BZip2_Compress(
|
|
|
|
compression_options,
|
|
|
|
GetCompressFormatForVersion(kBZip2Compression, format_version),
|
|
|
|
raw.data(), raw.size(), compressed_output) &&
|
2014-03-01 03:19:07 +01:00
|
|
|
GoodCompressionRatio(compressed_output->size(), raw.size())) {
|
|
|
|
return *compressed_output;
|
|
|
|
}
|
|
|
|
break; // fall back to no compression.
|
|
|
|
case kLZ4Compression:
|
2015-01-15 01:24:24 +01:00
|
|
|
if (LZ4_Compress(
|
|
|
|
compression_options,
|
|
|
|
GetCompressFormatForVersion(kLZ4Compression, format_version),
|
Shared dictionary compression using reference block
Summary:
This adds a new metablock containing a shared dictionary that is used
to compress all data blocks in the SST file. The size of the shared dictionary
is configurable in CompressionOptions and defaults to 0. It's currently only
used for zlib/lz4/lz4hc, but the block will be stored in the SST regardless of
the compression type if the user chooses a nonzero dictionary size.
During compaction, computes the dictionary by randomly sampling the first
output file in each subcompaction. It pre-computes the intervals to sample
by assuming the output file will have the maximum allowable length. In case
the file is smaller, some of the pre-computed sampling intervals can be beyond
end-of-file, in which case we skip over those samples and the dictionary will
be a bit smaller. After the dictionary is generated using the first file in a
subcompaction, it is loaded into the compression library before writing each
block in each subsequent file of that subcompaction.
On the read path, gets the dictionary from the metablock, if it exists. Then,
loads that dictionary into the compression library before reading each block.
Test Plan: new unit test
Reviewers: yhchiang, IslamAbdelRahman, cyan, sdong
Reviewed By: sdong
Subscribers: andrewkr, yoshinorim, kradhakrishnan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D52287
2016-04-28 02:36:03 +02:00
|
|
|
raw.data(), raw.size(), compressed_output, compression_dict) &&
|
2014-03-01 03:19:07 +01:00
|
|
|
GoodCompressionRatio(compressed_output->size(), raw.size())) {
|
|
|
|
return *compressed_output;
|
|
|
|
}
|
|
|
|
break; // fall back to no compression.
|
|
|
|
case kLZ4HCCompression:
|
2015-01-15 01:24:24 +01:00
|
|
|
if (LZ4HC_Compress(
|
|
|
|
compression_options,
|
|
|
|
GetCompressFormatForVersion(kLZ4HCCompression, format_version),
|
Shared dictionary compression using reference block
Summary:
This adds a new metablock containing a shared dictionary that is used
to compress all data blocks in the SST file. The size of the shared dictionary
is configurable in CompressionOptions and defaults to 0. It's currently only
used for zlib/lz4/lz4hc, but the block will be stored in the SST regardless of
the compression type if the user chooses a nonzero dictionary size.
During compaction, computes the dictionary by randomly sampling the first
output file in each subcompaction. It pre-computes the intervals to sample
by assuming the output file will have the maximum allowable length. In case
the file is smaller, some of the pre-computed sampling intervals can be beyond
end-of-file, in which case we skip over those samples and the dictionary will
be a bit smaller. After the dictionary is generated using the first file in a
subcompaction, it is loaded into the compression library before writing each
block in each subsequent file of that subcompaction.
On the read path, gets the dictionary from the metablock, if it exists. Then,
loads that dictionary into the compression library before reading each block.
Test Plan: new unit test
Reviewers: yhchiang, IslamAbdelRahman, cyan, sdong
Reviewed By: sdong
Subscribers: andrewkr, yoshinorim, kradhakrishnan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D52287
2016-04-28 02:36:03 +02:00
|
|
|
raw.data(), raw.size(), compressed_output, compression_dict) &&
|
2014-03-01 03:19:07 +01:00
|
|
|
GoodCompressionRatio(compressed_output->size(), raw.size())) {
|
|
|
|
return *compressed_output;
|
|
|
|
}
|
|
|
|
break; // fall back to no compression.
|
2016-04-20 07:54:24 +02:00
|
|
|
case kXpressCompression:
|
|
|
|
if (XPRESS_Compress(raw.data(), raw.size(),
|
|
|
|
compressed_output) &&
|
|
|
|
GoodCompressionRatio(compressed_output->size(), raw.size())) {
|
|
|
|
return *compressed_output;
|
|
|
|
}
|
|
|
|
break;
|
2016-09-02 00:28:40 +02:00
|
|
|
case kZSTD:
|
2015-08-28 00:40:42 +02:00
|
|
|
case kZSTDNotFinalCompression:
|
|
|
|
if (ZSTD_Compress(compression_options, raw.data(), raw.size(),
|
Shared dictionary compression using reference block
Summary:
This adds a new metablock containing a shared dictionary that is used
to compress all data blocks in the SST file. The size of the shared dictionary
is configurable in CompressionOptions and defaults to 0. It's currently only
used for zlib/lz4/lz4hc, but the block will be stored in the SST regardless of
the compression type if the user chooses a nonzero dictionary size.
During compaction, computes the dictionary by randomly sampling the first
output file in each subcompaction. It pre-computes the intervals to sample
by assuming the output file will have the maximum allowable length. In case
the file is smaller, some of the pre-computed sampling intervals can be beyond
end-of-file, in which case we skip over those samples and the dictionary will
be a bit smaller. After the dictionary is generated using the first file in a
subcompaction, it is loaded into the compression library before writing each
block in each subsequent file of that subcompaction.
On the read path, gets the dictionary from the metablock, if it exists. Then,
loads that dictionary into the compression library before reading each block.
Test Plan: new unit test
Reviewers: yhchiang, IslamAbdelRahman, cyan, sdong
Reviewed By: sdong
Subscribers: andrewkr, yoshinorim, kradhakrishnan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D52287
2016-04-28 02:36:03 +02:00
|
|
|
compressed_output, compression_dict) &&
|
2015-08-28 00:40:42 +02:00
|
|
|
GoodCompressionRatio(compressed_output->size(), raw.size())) {
|
|
|
|
return *compressed_output;
|
|
|
|
}
|
|
|
|
break; // fall back to no compression.
|
2014-03-01 03:19:07 +01:00
|
|
|
default: {} // Do not recognize this compression type
|
|
|
|
}
|
|
|
|
|
|
|
|
// Compression method is not supported, or not good compression ratio, so just
|
|
|
|
// fall back to uncompressed form.
|
|
|
|
*type = kNoCompression;
|
|
|
|
return raw;
|
|
|
|
}
|
|
|
|
|
2013-12-05 01:35:48 +01:00
|
|
|
// kBlockBasedTableMagicNumber was picked by running
|
2014-05-01 20:09:32 +02:00
|
|
|
// echo rocksdb.table.block_based | sha1sum
|
2013-12-05 00:09:41 +01:00
|
|
|
// and taking the leading 64 bits.
|
2015-07-13 21:11:05 +02:00
|
|
|
// Please note that kBlockBasedTableMagicNumber may also be accessed by other
|
|
|
|
// .cc files
|
|
|
|
// for that reason we declare it extern in the header but to get the space
|
|
|
|
// allocated
|
2015-07-02 01:13:49 +02:00
|
|
|
// it must be not extern in one place.
|
|
|
|
const uint64_t kBlockBasedTableMagicNumber = 0x88e241b785f4cff7ull;
|
2014-05-01 20:09:32 +02:00
|
|
|
// We also support reading and writing legacy block based table format (for
|
|
|
|
// backwards compatibility)
|
2015-07-02 01:13:49 +02:00
|
|
|
const uint64_t kLegacyBlockBasedTableMagicNumber = 0xdb4775248b80fb57ull;
|
2013-12-05 00:09:41 +01:00
|
|
|
|
2014-03-01 03:19:07 +01:00
|
|
|
// A collector that collects properties of interest to block-based table.
|
|
|
|
// For now this class looks heavy-weight since we only write one additional
|
|
|
|
// property.
|
2015-04-25 11:14:27 +02:00
|
|
|
// But in the foreseeable future, we will add more and more properties that are
|
2014-03-01 03:19:07 +01:00
|
|
|
// specific to block-based table.
|
|
|
|
class BlockBasedTableBuilder::BlockBasedTablePropertiesCollector
|
A new call back to TablePropertiesCollector to allow users know the entry is add, delete or merge
Summary:
Currently users have no idea a key is add, delete or merge from TablePropertiesCollector call back. Add a new function to add it.
Also refactor the codes so that
(1) make table property collector and internal table property collector two separate data structures with the later one now exposed
(2) table builders only receive internal table properties
Test Plan: Add cases in table_properties_collector_test to cover both of old and new ways of using TablePropertiesCollector.
Reviewers: yhchiang, igor.sugak, rven, igor
Reviewed By: rven, igor
Subscribers: meyering, yoshinorim, maykov, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D35373
2015-04-06 19:04:30 +02:00
|
|
|
: public IntTblPropCollector {
|
2014-03-01 03:19:07 +01:00
|
|
|
public:
|
2014-05-15 23:09:03 +02:00
|
|
|
explicit BlockBasedTablePropertiesCollector(
|
2015-02-05 02:03:57 +01:00
|
|
|
BlockBasedTableOptions::IndexType index_type, bool whole_key_filtering,
|
|
|
|
bool prefix_filtering)
|
|
|
|
: index_type_(index_type),
|
|
|
|
whole_key_filtering_(whole_key_filtering),
|
|
|
|
prefix_filtering_(prefix_filtering) {}
|
2014-03-01 03:19:07 +01:00
|
|
|
|
A new call back to TablePropertiesCollector to allow users know the entry is add, delete or merge
Summary:
Currently users have no idea a key is add, delete or merge from TablePropertiesCollector call back. Add a new function to add it.
Also refactor the codes so that
(1) make table property collector and internal table property collector two separate data structures with the later one now exposed
(2) table builders only receive internal table properties
Test Plan: Add cases in table_properties_collector_test to cover both of old and new ways of using TablePropertiesCollector.
Reviewers: yhchiang, igor.sugak, rven, igor
Reviewed By: rven, igor
Subscribers: meyering, yoshinorim, maykov, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D35373
2015-04-06 19:04:30 +02:00
|
|
|
virtual Status InternalAdd(const Slice& key, const Slice& value,
|
|
|
|
uint64_t file_size) override {
|
2014-03-01 03:19:07 +01:00
|
|
|
// Intentionally left blank. Have no interest in collecting stats for
|
|
|
|
// individual key/value pairs.
|
|
|
|
return Status::OK();
|
|
|
|
}
|
|
|
|
|
2015-02-26 20:28:41 +01:00
|
|
|
virtual Status Finish(UserCollectedProperties* properties) override {
|
2014-03-01 03:19:07 +01:00
|
|
|
std::string val;
|
|
|
|
PutFixed32(&val, static_cast<uint32_t>(index_type_));
|
|
|
|
properties->insert({BlockBasedTablePropertyNames::kIndexType, val});
|
2015-02-05 02:03:57 +01:00
|
|
|
properties->insert({BlockBasedTablePropertyNames::kWholeKeyFiltering,
|
|
|
|
whole_key_filtering_ ? kPropTrue : kPropFalse});
|
|
|
|
properties->insert({BlockBasedTablePropertyNames::kPrefixFiltering,
|
|
|
|
prefix_filtering_ ? kPropTrue : kPropFalse});
|
2014-03-01 03:19:07 +01:00
|
|
|
return Status::OK();
|
|
|
|
}
|
|
|
|
|
|
|
|
// The name of the properties collector can be used for debugging purpose.
|
2015-02-26 20:28:41 +01:00
|
|
|
virtual const char* Name() const override {
|
2014-03-01 03:19:07 +01:00
|
|
|
return "BlockBasedTablePropertiesCollector";
|
|
|
|
}
|
|
|
|
|
2015-02-26 20:28:41 +01:00
|
|
|
virtual UserCollectedProperties GetReadableProperties() const override {
|
2014-03-01 03:19:07 +01:00
|
|
|
// Intentionally left blank.
|
|
|
|
return UserCollectedProperties();
|
|
|
|
}
|
|
|
|
|
|
|
|
private:
|
|
|
|
BlockBasedTableOptions::IndexType index_type_;
|
2015-02-05 02:03:57 +01:00
|
|
|
bool whole_key_filtering_;
|
|
|
|
bool prefix_filtering_;
|
2014-03-01 03:19:07 +01:00
|
|
|
};
|
|
|
|
|
2013-10-29 01:54:09 +01:00
|
|
|
struct BlockBasedTableBuilder::Rep {
|
2014-09-05 01:18:36 +02:00
|
|
|
const ImmutableCFOptions ioptions;
|
2014-08-25 23:22:05 +02:00
|
|
|
const BlockBasedTableOptions table_options;
|
2014-01-27 22:53:22 +01:00
|
|
|
const InternalKeyComparator& internal_comparator;
|
Move rate_limiter, write buffering, most perf context instrumentation and most random kill out of Env
Summary: We want to keep Env a think layer for better portability. Less platform dependent codes should be moved out of Env. In this patch, I create a wrapper of file readers and writers, and put rate limiting, write buffering, as well as most perf context instrumentation and random kill out of Env. It will make it easier to maintain multiple Env in the future.
Test Plan: Run all existing unit tests.
Reviewers: anthony, kradhakrishnan, IslamAbdelRahman, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D42321
2015-07-18 01:16:11 +02:00
|
|
|
WritableFileWriter* file;
|
2013-10-10 20:43:24 +02:00
|
|
|
uint64_t offset = 0;
|
2011-03-18 23:37:00 +01:00
|
|
|
Status status;
|
|
|
|
BlockBuilder data_block;
|
2016-08-20 00:10:31 +02:00
|
|
|
BlockBuilder range_del_block;
|
2014-05-15 23:09:03 +02:00
|
|
|
|
|
|
|
InternalKeySliceTransform internal_prefix_transform;
|
2014-03-01 03:19:07 +01:00
|
|
|
std::unique_ptr<IndexBuilder> index_builder;
|
2017-06-13 19:59:22 +02:00
|
|
|
PartitionedIndexBuilder* p_index_builder_ = nullptr;
|
2014-03-01 03:19:07 +01:00
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
std::string last_key;
|
2014-09-05 01:18:36 +02:00
|
|
|
const CompressionType compression_type;
|
|
|
|
const CompressionOptions compression_opts;
|
Shared dictionary compression using reference block
Summary:
This adds a new metablock containing a shared dictionary that is used
to compress all data blocks in the SST file. The size of the shared dictionary
is configurable in CompressionOptions and defaults to 0. It's currently only
used for zlib/lz4/lz4hc, but the block will be stored in the SST regardless of
the compression type if the user chooses a nonzero dictionary size.
During compaction, computes the dictionary by randomly sampling the first
output file in each subcompaction. It pre-computes the intervals to sample
by assuming the output file will have the maximum allowable length. In case
the file is smaller, some of the pre-computed sampling intervals can be beyond
end-of-file, in which case we skip over those samples and the dictionary will
be a bit smaller. After the dictionary is generated using the first file in a
subcompaction, it is loaded into the compression library before writing each
block in each subsequent file of that subcompaction.
On the read path, gets the dictionary from the metablock, if it exists. Then,
loads that dictionary into the compression library before reading each block.
Test Plan: new unit test
Reviewers: yhchiang, IslamAbdelRahman, cyan, sdong
Reviewed By: sdong
Subscribers: andrewkr, yoshinorim, kradhakrishnan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D52287
2016-04-28 02:36:03 +02:00
|
|
|
// Data for presetting the compression library's dictionary, or nullptr.
|
|
|
|
const std::string* compression_dict;
|
2013-11-20 01:29:42 +01:00
|
|
|
TableProperties props;
|
2013-10-10 20:43:24 +02:00
|
|
|
|
|
|
|
bool closed = false; // Either Finish() or Abandon() has been called.
|
2017-03-07 22:48:02 +01:00
|
|
|
std::unique_ptr<FilterBlockBuilder> filter_builder;
|
2013-09-02 08:23:40 +02:00
|
|
|
char compressed_cache_key_prefix[BlockBasedTable::kMaxCacheKeyPrefixSize];
|
|
|
|
size_t compressed_cache_key_prefix_size;
|
2011-03-18 23:37:00 +01:00
|
|
|
|
|
|
|
BlockHandle pending_handle; // Handle to add to index block
|
|
|
|
|
|
|
|
std::string compressed_output;
|
2013-11-08 06:27:21 +01:00
|
|
|
std::unique_ptr<FlushBlockPolicy> flush_block_policy;
|
2016-04-07 08:10:32 +02:00
|
|
|
uint32_t column_family_id;
|
|
|
|
const std::string& column_family_name;
|
2011-03-18 23:37:00 +01:00
|
|
|
|
A new call back to TablePropertiesCollector to allow users know the entry is add, delete or merge
Summary:
Currently users have no idea a key is add, delete or merge from TablePropertiesCollector call back. Add a new function to add it.
Also refactor the codes so that
(1) make table property collector and internal table property collector two separate data structures with the later one now exposed
(2) table builders only receive internal table properties
Test Plan: Add cases in table_properties_collector_test to cover both of old and new ways of using TablePropertiesCollector.
Reviewers: yhchiang, igor.sugak, rven, igor
Reviewed By: rven, igor
Subscribers: meyering, yoshinorim, maykov, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D35373
2015-04-06 19:04:30 +02:00
|
|
|
std::vector<std::unique_ptr<IntTblPropCollector>> table_properties_collectors;
|
TablePropertiesCollectorFactory
Summary:
This diff addresses task #4296714 and rethinks how users provide us with TablePropertiesCollectors as part of Options.
Here's description of task #4296714:
I'm debugging #4295529 and noticed that our count of user properties kDeletedKeys is wrong. We're sharing one single InternalKeyPropertiesCollector with all Table Builders. In LOG Files, we're outputting number of kDeletedKeys as connected with a single table, while it's actually the total count of deleted keys since creation of the DB.
For example, this table has 3155 entries and 1391828 deleted keys.
The problem with current approach that we call methods on a single TablePropertiesCollector for all the tables we create. Even worse, we could do it from multiple threads at the same time and TablePropertiesCollector has no way of knowing which table we're calling it for.
Good part: Looks like nobody inside Facebook is using Options::table_properties_collectors. This means we should be able to painfully change the API.
In this change, I introduce TablePropertiesCollectorFactory. For every table we create, we call `CreateTablePropertiesCollector`, which creates a TablePropertiesCollector for a single table. We then use it sequentially from a single thread, which means it doesn't have to be thread-safe.
Test Plan:
Added a test in table_properties_collector_test that fails on master (build two tables, assert that kDeletedKeys count is correct for the second one).
Also, all other tests
Reviewers: sdong, dhruba, haobo, kailiu
Reviewed By: kailiu
CC: leveldb
Differential Revision: https://reviews.facebook.net/D18579
2014-05-13 21:30:55 +02:00
|
|
|
|
2014-10-31 19:59:54 +01:00
|
|
|
Rep(const ImmutableCFOptions& _ioptions,
|
2014-09-05 01:18:36 +02:00
|
|
|
const BlockBasedTableOptions& table_opt,
|
A new call back to TablePropertiesCollector to allow users know the entry is add, delete or merge
Summary:
Currently users have no idea a key is add, delete or merge from TablePropertiesCollector call back. Add a new function to add it.
Also refactor the codes so that
(1) make table property collector and internal table property collector two separate data structures with the later one now exposed
(2) table builders only receive internal table properties
Test Plan: Add cases in table_properties_collector_test to cover both of old and new ways of using TablePropertiesCollector.
Reviewers: yhchiang, igor.sugak, rven, igor
Reviewed By: rven, igor
Subscribers: meyering, yoshinorim, maykov, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D35373
2015-04-06 19:04:30 +02:00
|
|
|
const InternalKeyComparator& icomparator,
|
|
|
|
const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
|
|
|
|
int_tbl_prop_collector_factories,
|
2016-04-07 08:10:32 +02:00
|
|
|
uint32_t _column_family_id, WritableFileWriter* f,
|
2015-10-09 01:57:35 +02:00
|
|
|
const CompressionType _compression_type,
|
Shared dictionary compression using reference block
Summary:
This adds a new metablock containing a shared dictionary that is used
to compress all data blocks in the SST file. The size of the shared dictionary
is configurable in CompressionOptions and defaults to 0. It's currently only
used for zlib/lz4/lz4hc, but the block will be stored in the SST regardless of
the compression type if the user chooses a nonzero dictionary size.
During compaction, computes the dictionary by randomly sampling the first
output file in each subcompaction. It pre-computes the intervals to sample
by assuming the output file will have the maximum allowable length. In case
the file is smaller, some of the pre-computed sampling intervals can be beyond
end-of-file, in which case we skip over those samples and the dictionary will
be a bit smaller. After the dictionary is generated using the first file in a
subcompaction, it is loaded into the compression library before writing each
block in each subsequent file of that subcompaction.
On the read path, gets the dictionary from the metablock, if it exists. Then,
loads that dictionary into the compression library before reading each block.
Test Plan: new unit test
Reviewers: yhchiang, IslamAbdelRahman, cyan, sdong
Reviewed By: sdong
Subscribers: andrewkr, yoshinorim, kradhakrishnan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D52287
2016-04-28 02:36:03 +02:00
|
|
|
const CompressionOptions& _compression_opts,
|
|
|
|
const std::string* _compression_dict, const bool skip_filters,
|
2016-04-07 08:10:32 +02:00
|
|
|
const std::string& _column_family_name)
|
2014-10-31 19:59:54 +01:00
|
|
|
: ioptions(_ioptions),
|
2014-08-25 23:22:05 +02:00
|
|
|
table_options(table_opt),
|
2014-01-27 22:53:22 +01:00
|
|
|
internal_comparator(icomparator),
|
2011-03-18 23:37:00 +01:00
|
|
|
file(f),
|
2015-12-16 21:08:30 +01:00
|
|
|
data_block(table_options.block_restart_interval,
|
|
|
|
table_options.use_delta_encoding),
|
2016-08-20 00:10:31 +02:00
|
|
|
range_del_block(1), // TODO(andrewkr): restart_interval unnecessary
|
2014-10-31 19:59:54 +01:00
|
|
|
internal_prefix_transform(_ioptions.prefix_extractor),
|
|
|
|
compression_type(_compression_type),
|
|
|
|
compression_opts(_compression_opts),
|
Shared dictionary compression using reference block
Summary:
This adds a new metablock containing a shared dictionary that is used
to compress all data blocks in the SST file. The size of the shared dictionary
is configurable in CompressionOptions and defaults to 0. It's currently only
used for zlib/lz4/lz4hc, but the block will be stored in the SST regardless of
the compression type if the user chooses a nonzero dictionary size.
During compaction, computes the dictionary by randomly sampling the first
output file in each subcompaction. It pre-computes the intervals to sample
by assuming the output file will have the maximum allowable length. In case
the file is smaller, some of the pre-computed sampling intervals can be beyond
end-of-file, in which case we skip over those samples and the dictionary will
be a bit smaller. After the dictionary is generated using the first file in a
subcompaction, it is loaded into the compression library before writing each
block in each subsequent file of that subcompaction.
On the read path, gets the dictionary from the metablock, if it exists. Then,
loads that dictionary into the compression library before reading each block.
Test Plan: new unit test
Reviewers: yhchiang, IslamAbdelRahman, cyan, sdong
Reviewed By: sdong
Subscribers: andrewkr, yoshinorim, kradhakrishnan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D52287
2016-04-28 02:36:03 +02:00
|
|
|
compression_dict(_compression_dict),
|
2014-08-25 23:22:05 +02:00
|
|
|
flush_block_policy(
|
|
|
|
table_options.flush_block_policy_factory->NewFlushBlockPolicy(
|
2016-04-07 08:10:32 +02:00
|
|
|
table_options, data_block)),
|
|
|
|
column_family_id(_column_family_id),
|
|
|
|
column_family_name(_column_family_name) {
|
2017-03-07 22:48:02 +01:00
|
|
|
if (table_options.index_type ==
|
|
|
|
BlockBasedTableOptions::kTwoLevelIndexSearch) {
|
2017-06-13 19:59:22 +02:00
|
|
|
p_index_builder_ = PartitionedIndexBuilder::CreateIndexBuilder(
|
2017-03-07 22:48:02 +01:00
|
|
|
&internal_comparator, table_options);
|
2017-06-13 19:59:22 +02:00
|
|
|
index_builder.reset(p_index_builder_);
|
2017-03-07 22:48:02 +01:00
|
|
|
} else {
|
|
|
|
index_builder.reset(IndexBuilder::CreateIndexBuilder(
|
|
|
|
table_options.index_type, &internal_comparator,
|
|
|
|
&this->internal_prefix_transform, table_options));
|
|
|
|
}
|
|
|
|
if (skip_filters) {
|
|
|
|
filter_builder = nullptr;
|
|
|
|
} else {
|
|
|
|
filter_builder.reset(
|
2017-06-13 19:59:22 +02:00
|
|
|
CreateFilterBlockBuilder(_ioptions, table_options, p_index_builder_));
|
2017-03-07 22:48:02 +01:00
|
|
|
}
|
|
|
|
|
A new call back to TablePropertiesCollector to allow users know the entry is add, delete or merge
Summary:
Currently users have no idea a key is add, delete or merge from TablePropertiesCollector call back. Add a new function to add it.
Also refactor the codes so that
(1) make table property collector and internal table property collector two separate data structures with the later one now exposed
(2) table builders only receive internal table properties
Test Plan: Add cases in table_properties_collector_test to cover both of old and new ways of using TablePropertiesCollector.
Reviewers: yhchiang, igor.sugak, rven, igor
Reviewed By: rven, igor
Subscribers: meyering, yoshinorim, maykov, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D35373
2015-04-06 19:04:30 +02:00
|
|
|
for (auto& collector_factories : *int_tbl_prop_collector_factories) {
|
TablePropertiesCollectorFactory
Summary:
This diff addresses task #4296714 and rethinks how users provide us with TablePropertiesCollectors as part of Options.
Here's description of task #4296714:
I'm debugging #4295529 and noticed that our count of user properties kDeletedKeys is wrong. We're sharing one single InternalKeyPropertiesCollector with all Table Builders. In LOG Files, we're outputting number of kDeletedKeys as connected with a single table, while it's actually the total count of deleted keys since creation of the DB.
For example, this table has 3155 entries and 1391828 deleted keys.
The problem with current approach that we call methods on a single TablePropertiesCollector for all the tables we create. Even worse, we could do it from multiple threads at the same time and TablePropertiesCollector has no way of knowing which table we're calling it for.
Good part: Looks like nobody inside Facebook is using Options::table_properties_collectors. This means we should be able to painfully change the API.
In this change, I introduce TablePropertiesCollectorFactory. For every table we create, we call `CreateTablePropertiesCollector`, which creates a TablePropertiesCollector for a single table. We then use it sequentially from a single thread, which means it doesn't have to be thread-safe.
Test Plan:
Added a test in table_properties_collector_test that fails on master (build two tables, assert that kDeletedKeys count is correct for the second one).
Also, all other tests
Reviewers: sdong, dhruba, haobo, kailiu
Reviewed By: kailiu
CC: leveldb
Differential Revision: https://reviews.facebook.net/D18579
2014-05-13 21:30:55 +02:00
|
|
|
table_properties_collectors.emplace_back(
|
2015-10-09 01:57:35 +02:00
|
|
|
collector_factories->CreateIntTblPropCollector(column_family_id));
|
TablePropertiesCollectorFactory
Summary:
This diff addresses task #4296714 and rethinks how users provide us with TablePropertiesCollectors as part of Options.
Here's description of task #4296714:
I'm debugging #4295529 and noticed that our count of user properties kDeletedKeys is wrong. We're sharing one single InternalKeyPropertiesCollector with all Table Builders. In LOG Files, we're outputting number of kDeletedKeys as connected with a single table, while it's actually the total count of deleted keys since creation of the DB.
For example, this table has 3155 entries and 1391828 deleted keys.
The problem with current approach that we call methods on a single TablePropertiesCollector for all the tables we create. Even worse, we could do it from multiple threads at the same time and TablePropertiesCollector has no way of knowing which table we're calling it for.
Good part: Looks like nobody inside Facebook is using Options::table_properties_collectors. This means we should be able to painfully change the API.
In this change, I introduce TablePropertiesCollectorFactory. For every table we create, we call `CreateTablePropertiesCollector`, which creates a TablePropertiesCollector for a single table. We then use it sequentially from a single thread, which means it doesn't have to be thread-safe.
Test Plan:
Added a test in table_properties_collector_test that fails on master (build two tables, assert that kDeletedKeys count is correct for the second one).
Also, all other tests
Reviewers: sdong, dhruba, haobo, kailiu
Reviewed By: kailiu
CC: leveldb
Differential Revision: https://reviews.facebook.net/D18579
2014-05-13 21:30:55 +02:00
|
|
|
}
|
|
|
|
table_properties_collectors.emplace_back(
|
2015-02-05 02:03:57 +01:00
|
|
|
new BlockBasedTablePropertiesCollector(
|
|
|
|
table_options.index_type, table_options.whole_key_filtering,
|
|
|
|
_ioptions.prefix_extractor != nullptr));
|
2014-03-01 03:19:07 +01:00
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
};
|
|
|
|
|
2013-11-20 07:00:48 +01:00
|
|
|
BlockBasedTableBuilder::BlockBasedTableBuilder(
|
2014-09-05 01:18:36 +02:00
|
|
|
const ImmutableCFOptions& ioptions,
|
|
|
|
const BlockBasedTableOptions& table_options,
|
A new call back to TablePropertiesCollector to allow users know the entry is add, delete or merge
Summary:
Currently users have no idea a key is add, delete or merge from TablePropertiesCollector call back. Add a new function to add it.
Also refactor the codes so that
(1) make table property collector and internal table property collector two separate data structures with the later one now exposed
(2) table builders only receive internal table properties
Test Plan: Add cases in table_properties_collector_test to cover both of old and new ways of using TablePropertiesCollector.
Reviewers: yhchiang, igor.sugak, rven, igor
Reviewed By: rven, igor
Subscribers: meyering, yoshinorim, maykov, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D35373
2015-04-06 19:04:30 +02:00
|
|
|
const InternalKeyComparator& internal_comparator,
|
|
|
|
const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
|
|
|
|
int_tbl_prop_collector_factories,
|
2015-10-09 01:57:35 +02:00
|
|
|
uint32_t column_family_id, WritableFileWriter* file,
|
|
|
|
const CompressionType compression_type,
|
Shared dictionary compression using reference block
Summary:
This adds a new metablock containing a shared dictionary that is used
to compress all data blocks in the SST file. The size of the shared dictionary
is configurable in CompressionOptions and defaults to 0. It's currently only
used for zlib/lz4/lz4hc, but the block will be stored in the SST regardless of
the compression type if the user chooses a nonzero dictionary size.
During compaction, computes the dictionary by randomly sampling the first
output file in each subcompaction. It pre-computes the intervals to sample
by assuming the output file will have the maximum allowable length. In case
the file is smaller, some of the pre-computed sampling intervals can be beyond
end-of-file, in which case we skip over those samples and the dictionary will
be a bit smaller. After the dictionary is generated using the first file in a
subcompaction, it is loaded into the compression library before writing each
block in each subsequent file of that subcompaction.
On the read path, gets the dictionary from the metablock, if it exists. Then,
loads that dictionary into the compression library before reading each block.
Test Plan: new unit test
Reviewers: yhchiang, IslamAbdelRahman, cyan, sdong
Reviewed By: sdong
Subscribers: andrewkr, yoshinorim, kradhakrishnan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D52287
2016-04-28 02:36:03 +02:00
|
|
|
const CompressionOptions& compression_opts,
|
|
|
|
const std::string* compression_dict, const bool skip_filters,
|
2016-04-07 08:10:32 +02:00
|
|
|
const std::string& column_family_name) {
|
2015-01-13 23:33:04 +01:00
|
|
|
BlockBasedTableOptions sanitized_table_options(table_options);
|
|
|
|
if (sanitized_table_options.format_version == 0 &&
|
|
|
|
sanitized_table_options.checksum != kCRC32c) {
|
2017-03-16 03:22:52 +01:00
|
|
|
ROCKS_LOG_WARN(
|
|
|
|
ioptions.info_log,
|
2015-01-13 23:33:04 +01:00
|
|
|
"Silently converting format_version to 1 because checksum is "
|
|
|
|
"non-default");
|
|
|
|
// silently convert format_version to 1 to keep consistent with current
|
|
|
|
// behavior
|
|
|
|
sanitized_table_options.format_version = 1;
|
|
|
|
}
|
|
|
|
|
A new call back to TablePropertiesCollector to allow users know the entry is add, delete or merge
Summary:
Currently users have no idea a key is add, delete or merge from TablePropertiesCollector call back. Add a new function to add it.
Also refactor the codes so that
(1) make table property collector and internal table property collector two separate data structures with the later one now exposed
(2) table builders only receive internal table properties
Test Plan: Add cases in table_properties_collector_test to cover both of old and new ways of using TablePropertiesCollector.
Reviewers: yhchiang, igor.sugak, rven, igor
Reviewed By: rven, igor
Subscribers: meyering, yoshinorim, maykov, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D35373
2015-04-06 19:04:30 +02:00
|
|
|
rep_ = new Rep(ioptions, sanitized_table_options, internal_comparator,
|
2015-10-09 01:57:35 +02:00
|
|
|
int_tbl_prop_collector_factories, column_family_id, file,
|
Shared dictionary compression using reference block
Summary:
This adds a new metablock containing a shared dictionary that is used
to compress all data blocks in the SST file. The size of the shared dictionary
is configurable in CompressionOptions and defaults to 0. It's currently only
used for zlib/lz4/lz4hc, but the block will be stored in the SST regardless of
the compression type if the user chooses a nonzero dictionary size.
During compaction, computes the dictionary by randomly sampling the first
output file in each subcompaction. It pre-computes the intervals to sample
by assuming the output file will have the maximum allowable length. In case
the file is smaller, some of the pre-computed sampling intervals can be beyond
end-of-file, in which case we skip over those samples and the dictionary will
be a bit smaller. After the dictionary is generated using the first file in a
subcompaction, it is loaded into the compression library before writing each
block in each subsequent file of that subcompaction.
On the read path, gets the dictionary from the metablock, if it exists. Then,
loads that dictionary into the compression library before reading each block.
Test Plan: new unit test
Reviewers: yhchiang, IslamAbdelRahman, cyan, sdong
Reviewed By: sdong
Subscribers: andrewkr, yoshinorim, kradhakrishnan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D52287
2016-04-28 02:36:03 +02:00
|
|
|
compression_type, compression_opts, compression_dict,
|
|
|
|
skip_filters, column_family_name);
|
2015-02-17 17:03:45 +01:00
|
|
|
|
2017-03-07 22:48:02 +01:00
|
|
|
if (rep_->filter_builder != nullptr) {
|
|
|
|
rep_->filter_builder->StartBlock(0);
|
2012-04-17 17:36:46 +02:00
|
|
|
}
|
2014-08-25 23:22:05 +02:00
|
|
|
if (table_options.block_cache_compressed.get() != nullptr) {
|
2013-12-03 20:17:58 +01:00
|
|
|
BlockBasedTable::GenerateCachePrefix(
|
Move rate_limiter, write buffering, most perf context instrumentation and most random kill out of Env
Summary: We want to keep Env a think layer for better portability. Less platform dependent codes should be moved out of Env. In this patch, I create a wrapper of file readers and writers, and put rate limiting, write buffering, as well as most perf context instrumentation and random kill out of Env. It will make it easier to maintain multiple Env in the future.
Test Plan: Run all existing unit tests.
Reviewers: anthony, kradhakrishnan, IslamAbdelRahman, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D42321
2015-07-18 01:16:11 +02:00
|
|
|
table_options.block_cache_compressed.get(), file->writable_file(),
|
2013-12-03 20:17:58 +01:00
|
|
|
&rep_->compressed_cache_key_prefix[0],
|
|
|
|
&rep_->compressed_cache_key_prefix_size);
|
2013-09-02 08:23:40 +02:00
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2013-10-29 01:54:09 +01:00
|
|
|
BlockBasedTableBuilder::~BlockBasedTableBuilder() {
|
2011-03-18 23:37:00 +01:00
|
|
|
assert(rep_->closed); // Catch errors where caller forgot to call Finish()
|
|
|
|
delete rep_;
|
|
|
|
}
|
|
|
|
|
2013-10-29 01:54:09 +01:00
|
|
|
void BlockBasedTableBuilder::Add(const Slice& key, const Slice& value) {
|
2011-03-18 23:37:00 +01:00
|
|
|
Rep* r = rep_;
|
|
|
|
assert(!r->closed);
|
|
|
|
if (!ok()) return;
|
2016-08-20 00:10:31 +02:00
|
|
|
ValueType value_type = ExtractValueType(key);
|
|
|
|
if (IsValueType(value_type)) {
|
|
|
|
if (r->props.num_entries > 0) {
|
|
|
|
assert(r->internal_comparator.Compare(key, Slice(r->last_key)) > 0);
|
2013-11-08 06:27:21 +01:00
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2016-08-20 00:10:31 +02:00
|
|
|
auto should_flush = r->flush_block_policy->Update(key, value);
|
|
|
|
if (should_flush) {
|
|
|
|
assert(!r->data_block.empty());
|
|
|
|
Flush();
|
|
|
|
|
|
|
|
// Add item to index block.
|
|
|
|
// We do not emit the index entry for a block until we have seen the
|
|
|
|
// first key for the next data block. This allows us to use shorter
|
|
|
|
// keys in the index block. For example, consider a block boundary
|
|
|
|
// between the keys "the quick brown fox" and "the who". We can use
|
|
|
|
// "the r" as the key for the index block entry since it is >= all
|
|
|
|
// entries in the first block and < all entries in subsequent
|
|
|
|
// blocks.
|
|
|
|
if (ok()) {
|
|
|
|
r->index_builder->AddIndexEntry(&r->last_key, &key, r->pending_handle);
|
|
|
|
}
|
|
|
|
}
|
2012-04-17 17:36:46 +02:00
|
|
|
|
2017-03-07 22:48:02 +01:00
|
|
|
// Note: PartitionedFilterBlockBuilder requires key being added to filter
|
|
|
|
// builder after being added to index builder.
|
|
|
|
if (r->filter_builder != nullptr) {
|
|
|
|
r->filter_builder->Add(ExtractUserKey(key));
|
2016-08-20 00:10:31 +02:00
|
|
|
}
|
2013-10-16 20:50:50 +02:00
|
|
|
|
2016-08-20 00:10:31 +02:00
|
|
|
r->last_key.assign(key.data(), key.size());
|
|
|
|
r->data_block.Add(key, value);
|
|
|
|
r->props.num_entries++;
|
|
|
|
r->props.raw_key_size += key.size();
|
|
|
|
r->props.raw_value_size += value.size();
|
|
|
|
|
|
|
|
r->index_builder->OnKeyAdded(key);
|
|
|
|
NotifyCollectTableCollectorsOnAdd(key, value, r->offset,
|
|
|
|
r->table_properties_collectors,
|
|
|
|
r->ioptions.info_log);
|
|
|
|
|
|
|
|
} else if (value_type == kTypeRangeDeletion) {
|
|
|
|
// TODO(wanning&andrewkr) add num_tomestone to table properties
|
|
|
|
r->range_del_block.Add(key, value);
|
|
|
|
++r->props.num_entries;
|
2016-09-12 23:14:40 +02:00
|
|
|
r->props.raw_key_size += key.size();
|
|
|
|
r->props.raw_value_size += value.size();
|
|
|
|
NotifyCollectTableCollectorsOnAdd(key, value, r->offset,
|
|
|
|
r->table_properties_collectors,
|
|
|
|
r->ioptions.info_log);
|
2016-08-20 00:10:31 +02:00
|
|
|
} else {
|
|
|
|
assert(false);
|
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2013-10-29 01:54:09 +01:00
|
|
|
void BlockBasedTableBuilder::Flush() {
|
2011-03-18 23:37:00 +01:00
|
|
|
Rep* r = rep_;
|
|
|
|
assert(!r->closed);
|
|
|
|
if (!ok()) return;
|
|
|
|
if (r->data_block.empty()) return;
|
Shared dictionary compression using reference block
Summary:
This adds a new metablock containing a shared dictionary that is used
to compress all data blocks in the SST file. The size of the shared dictionary
is configurable in CompressionOptions and defaults to 0. It's currently only
used for zlib/lz4/lz4hc, but the block will be stored in the SST regardless of
the compression type if the user chooses a nonzero dictionary size.
During compaction, computes the dictionary by randomly sampling the first
output file in each subcompaction. It pre-computes the intervals to sample
by assuming the output file will have the maximum allowable length. In case
the file is smaller, some of the pre-computed sampling intervals can be beyond
end-of-file, in which case we skip over those samples and the dictionary will
be a bit smaller. After the dictionary is generated using the first file in a
subcompaction, it is loaded into the compression library before writing each
block in each subsequent file of that subcompaction.
On the read path, gets the dictionary from the metablock, if it exists. Then,
loads that dictionary into the compression library before reading each block.
Test Plan: new unit test
Reviewers: yhchiang, IslamAbdelRahman, cyan, sdong
Reviewed By: sdong
Subscribers: andrewkr, yoshinorim, kradhakrishnan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D52287
2016-04-28 02:36:03 +02:00
|
|
|
WriteBlock(&r->data_block, &r->pending_handle, true /* is_data_block */);
|
2017-03-07 22:48:02 +01:00
|
|
|
if (r->filter_builder != nullptr) {
|
|
|
|
r->filter_builder->StartBlock(r->offset);
|
2012-04-17 17:36:46 +02:00
|
|
|
}
|
2013-11-20 01:29:42 +01:00
|
|
|
r->props.data_size = r->offset;
|
2015-11-02 23:12:29 +01:00
|
|
|
++r->props.num_data_blocks;
|
2012-06-28 08:41:33 +02:00
|
|
|
}
|
|
|
|
|
2013-10-29 01:54:09 +01:00
|
|
|
void BlockBasedTableBuilder::WriteBlock(BlockBuilder* block,
|
Shared dictionary compression using reference block
Summary:
This adds a new metablock containing a shared dictionary that is used
to compress all data blocks in the SST file. The size of the shared dictionary
is configurable in CompressionOptions and defaults to 0. It's currently only
used for zlib/lz4/lz4hc, but the block will be stored in the SST regardless of
the compression type if the user chooses a nonzero dictionary size.
During compaction, computes the dictionary by randomly sampling the first
output file in each subcompaction. It pre-computes the intervals to sample
by assuming the output file will have the maximum allowable length. In case
the file is smaller, some of the pre-computed sampling intervals can be beyond
end-of-file, in which case we skip over those samples and the dictionary will
be a bit smaller. After the dictionary is generated using the first file in a
subcompaction, it is loaded into the compression library before writing each
block in each subsequent file of that subcompaction.
On the read path, gets the dictionary from the metablock, if it exists. Then,
loads that dictionary into the compression library before reading each block.
Test Plan: new unit test
Reviewers: yhchiang, IslamAbdelRahman, cyan, sdong
Reviewed By: sdong
Subscribers: andrewkr, yoshinorim, kradhakrishnan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D52287
2016-04-28 02:36:03 +02:00
|
|
|
BlockHandle* handle,
|
|
|
|
bool is_data_block) {
|
|
|
|
WriteBlock(block->Finish(), handle, is_data_block);
|
2014-03-01 03:19:07 +01:00
|
|
|
block->Reset();
|
|
|
|
}
|
|
|
|
|
|
|
|
void BlockBasedTableBuilder::WriteBlock(const Slice& raw_block_contents,
|
Shared dictionary compression using reference block
Summary:
This adds a new metablock containing a shared dictionary that is used
to compress all data blocks in the SST file. The size of the shared dictionary
is configurable in CompressionOptions and defaults to 0. It's currently only
used for zlib/lz4/lz4hc, but the block will be stored in the SST regardless of
the compression type if the user chooses a nonzero dictionary size.
During compaction, computes the dictionary by randomly sampling the first
output file in each subcompaction. It pre-computes the intervals to sample
by assuming the output file will have the maximum allowable length. In case
the file is smaller, some of the pre-computed sampling intervals can be beyond
end-of-file, in which case we skip over those samples and the dictionary will
be a bit smaller. After the dictionary is generated using the first file in a
subcompaction, it is loaded into the compression library before writing each
block in each subsequent file of that subcompaction.
On the read path, gets the dictionary from the metablock, if it exists. Then,
loads that dictionary into the compression library before reading each block.
Test Plan: new unit test
Reviewers: yhchiang, IslamAbdelRahman, cyan, sdong
Reviewed By: sdong
Subscribers: andrewkr, yoshinorim, kradhakrishnan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D52287
2016-04-28 02:36:03 +02:00
|
|
|
BlockHandle* handle,
|
|
|
|
bool is_data_block) {
|
2011-03-18 23:37:00 +01:00
|
|
|
// File format contains a sequence of blocks where each block has:
|
|
|
|
// block_data: uint8[n]
|
|
|
|
// type: uint8
|
|
|
|
// crc: uint32
|
|
|
|
assert(ok());
|
|
|
|
Rep* r = rep_;
|
|
|
|
|
2014-03-01 03:19:07 +01:00
|
|
|
auto type = r->compression_type;
|
2014-06-09 21:26:09 +02:00
|
|
|
Slice block_contents;
|
2016-06-11 03:20:54 +02:00
|
|
|
bool abort_compression = false;
|
2016-08-20 00:10:31 +02:00
|
|
|
|
2016-07-19 18:44:03 +02:00
|
|
|
StopWatchNano timer(r->ioptions.env,
|
|
|
|
ShouldReportDetailedTime(r->ioptions.env, r->ioptions.statistics));
|
|
|
|
|
2014-06-09 21:26:09 +02:00
|
|
|
if (raw_block_contents.size() < kCompressionSizeLimit) {
|
Shared dictionary compression using reference block
Summary:
This adds a new metablock containing a shared dictionary that is used
to compress all data blocks in the SST file. The size of the shared dictionary
is configurable in CompressionOptions and defaults to 0. It's currently only
used for zlib/lz4/lz4hc, but the block will be stored in the SST regardless of
the compression type if the user chooses a nonzero dictionary size.
During compaction, computes the dictionary by randomly sampling the first
output file in each subcompaction. It pre-computes the intervals to sample
by assuming the output file will have the maximum allowable length. In case
the file is smaller, some of the pre-computed sampling intervals can be beyond
end-of-file, in which case we skip over those samples and the dictionary will
be a bit smaller. After the dictionary is generated using the first file in a
subcompaction, it is loaded into the compression library before writing each
block in each subsequent file of that subcompaction.
On the read path, gets the dictionary from the metablock, if it exists. Then,
loads that dictionary into the compression library before reading each block.
Test Plan: new unit test
Reviewers: yhchiang, IslamAbdelRahman, cyan, sdong
Reviewed By: sdong
Subscribers: andrewkr, yoshinorim, kradhakrishnan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D52287
2016-04-28 02:36:03 +02:00
|
|
|
Slice compression_dict;
|
|
|
|
if (is_data_block && r->compression_dict && r->compression_dict->size()) {
|
|
|
|
compression_dict = *r->compression_dict;
|
|
|
|
}
|
2016-07-19 18:44:03 +02:00
|
|
|
|
Shared dictionary compression using reference block
Summary:
This adds a new metablock containing a shared dictionary that is used
to compress all data blocks in the SST file. The size of the shared dictionary
is configurable in CompressionOptions and defaults to 0. It's currently only
used for zlib/lz4/lz4hc, but the block will be stored in the SST regardless of
the compression type if the user chooses a nonzero dictionary size.
During compaction, computes the dictionary by randomly sampling the first
output file in each subcompaction. It pre-computes the intervals to sample
by assuming the output file will have the maximum allowable length. In case
the file is smaller, some of the pre-computed sampling intervals can be beyond
end-of-file, in which case we skip over those samples and the dictionary will
be a bit smaller. After the dictionary is generated using the first file in a
subcompaction, it is loaded into the compression library before writing each
block in each subsequent file of that subcompaction.
On the read path, gets the dictionary from the metablock, if it exists. Then,
loads that dictionary into the compression library before reading each block.
Test Plan: new unit test
Reviewers: yhchiang, IslamAbdelRahman, cyan, sdong
Reviewed By: sdong
Subscribers: andrewkr, yoshinorim, kradhakrishnan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D52287
2016-04-28 02:36:03 +02:00
|
|
|
block_contents = CompressBlock(raw_block_contents, r->compression_opts,
|
|
|
|
&type, r->table_options.format_version,
|
|
|
|
compression_dict, &r->compressed_output);
|
2016-06-11 03:20:54 +02:00
|
|
|
|
|
|
|
// Some of the compression algorithms are known to be unreliable. If
|
|
|
|
// the verify_compression flag is set then try to de-compress the
|
|
|
|
// compressed data and compare to the input.
|
|
|
|
if (type != kNoCompression && r->table_options.verify_compression) {
|
|
|
|
// Retrieve the uncompressed contents into a new buffer
|
|
|
|
BlockContents contents;
|
|
|
|
Status stat = UncompressBlockContentsForCompressionType(
|
|
|
|
block_contents.data(), block_contents.size(), &contents,
|
2016-07-19 18:44:03 +02:00
|
|
|
r->table_options.format_version, compression_dict, type,
|
|
|
|
r->ioptions);
|
2016-06-11 03:20:54 +02:00
|
|
|
|
|
|
|
if (stat.ok()) {
|
|
|
|
bool compressed_ok = contents.data.compare(raw_block_contents) == 0;
|
|
|
|
if (!compressed_ok) {
|
|
|
|
// The result of the compression was invalid. abort.
|
|
|
|
abort_compression = true;
|
2017-03-16 03:22:52 +01:00
|
|
|
ROCKS_LOG_ERROR(r->ioptions.info_log,
|
|
|
|
"Decompressed block did not match raw block");
|
2016-06-11 03:20:54 +02:00
|
|
|
r->status =
|
|
|
|
Status::Corruption("Decompressed block did not match raw block");
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
// Decompression reported an error. abort.
|
|
|
|
r->status = Status::Corruption("Could not decompress");
|
|
|
|
abort_compression = true;
|
|
|
|
}
|
|
|
|
}
|
2014-06-09 21:26:09 +02:00
|
|
|
} else {
|
2016-06-11 03:20:54 +02:00
|
|
|
// Block is too big to be compressed.
|
|
|
|
abort_compression = true;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Abort compression if the block is too big, or did not pass
|
|
|
|
// verification.
|
|
|
|
if (abort_compression) {
|
2014-09-05 01:18:36 +02:00
|
|
|
RecordTick(r->ioptions.statistics, NUMBER_BLOCK_NOT_COMPRESSED);
|
2014-06-09 21:26:09 +02:00
|
|
|
type = kNoCompression;
|
|
|
|
block_contents = raw_block_contents;
|
2016-08-20 00:10:31 +02:00
|
|
|
} else if (type != kNoCompression &&
|
|
|
|
ShouldReportDetailedTime(r->ioptions.env,
|
|
|
|
r->ioptions.statistics)) {
|
|
|
|
MeasureTime(r->ioptions.statistics, COMPRESSION_TIMES_NANOS,
|
|
|
|
timer.ElapsedNanos());
|
|
|
|
MeasureTime(r->ioptions.statistics, BYTES_COMPRESSED,
|
|
|
|
raw_block_contents.size());
|
2016-07-19 18:44:03 +02:00
|
|
|
RecordTick(r->ioptions.statistics, NUMBER_BLOCK_COMPRESSED);
|
|
|
|
}
|
2016-06-11 03:20:54 +02:00
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
WriteRawBlock(block_contents, type, handle);
|
|
|
|
r->compressed_output.clear();
|
|
|
|
}
|
|
|
|
|
2013-10-29 01:54:09 +01:00
|
|
|
void BlockBasedTableBuilder::WriteRawBlock(const Slice& block_contents,
|
|
|
|
CompressionType type,
|
|
|
|
BlockHandle* handle) {
|
2012-04-17 17:36:46 +02:00
|
|
|
Rep* r = rep_;
|
2014-09-05 01:18:36 +02:00
|
|
|
StopWatch sw(r->ioptions.env, r->ioptions.statistics, WRITE_RAW_BLOCK_MICROS);
|
2011-03-18 23:37:00 +01:00
|
|
|
handle->set_offset(r->offset);
|
|
|
|
handle->set_size(block_contents.size());
|
2017-06-26 22:15:55 +02:00
|
|
|
assert(r->status.ok());
|
2011-03-18 23:37:00 +01:00
|
|
|
r->status = r->file->Append(block_contents);
|
|
|
|
if (r->status.ok()) {
|
|
|
|
char trailer[kBlockTrailerSize];
|
|
|
|
trailer[0] = type;
|
2014-05-01 20:09:32 +02:00
|
|
|
char* trailer_without_type = trailer + 1;
|
2014-08-25 23:22:05 +02:00
|
|
|
switch (r->table_options.checksum) {
|
2014-05-01 20:09:32 +02:00
|
|
|
case kNoChecksum:
|
|
|
|
// we don't support no checksum yet
|
|
|
|
assert(false);
|
2016-12-13 22:50:22 +01:00
|
|
|
// intentional fallthrough
|
2014-05-01 20:09:32 +02:00
|
|
|
case kCRC32c: {
|
|
|
|
auto crc = crc32c::Value(block_contents.data(), block_contents.size());
|
|
|
|
crc = crc32c::Extend(crc, trailer, 1); // Extend to cover block type
|
|
|
|
EncodeFixed32(trailer_without_type, crc32c::Mask(crc));
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
case kxxHash: {
|
|
|
|
void* xxh = XXH32_init(0);
|
2014-11-11 22:47:22 +01:00
|
|
|
XXH32_update(xxh, block_contents.data(),
|
|
|
|
static_cast<uint32_t>(block_contents.size()));
|
2014-05-01 20:09:32 +02:00
|
|
|
XXH32_update(xxh, trailer, 1); // Extend to cover block type
|
|
|
|
EncodeFixed32(trailer_without_type, XXH32_digest(xxh));
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-06-26 22:15:55 +02:00
|
|
|
assert(r->status.ok());
|
2011-03-18 23:37:00 +01:00
|
|
|
r->status = r->file->Append(Slice(trailer, kBlockTrailerSize));
|
2013-09-02 08:23:40 +02:00
|
|
|
if (r->status.ok()) {
|
|
|
|
r->status = InsertBlockInCache(block_contents, type, handle);
|
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
if (r->status.ok()) {
|
|
|
|
r->offset += block_contents.size() + kBlockTrailerSize;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-10-29 01:54:09 +01:00
|
|
|
Status BlockBasedTableBuilder::status() const {
|
2011-03-18 23:37:00 +01:00
|
|
|
return rep_->status;
|
|
|
|
}
|
|
|
|
|
2013-09-02 08:23:40 +02:00
|
|
|
static void DeleteCachedBlock(const Slice& key, void* value) {
|
|
|
|
Block* block = reinterpret_cast<Block*>(value);
|
|
|
|
delete block;
|
|
|
|
}
|
|
|
|
|
|
|
|
//
|
|
|
|
// Make a copy of the block contents and insert into compressed block cache
|
|
|
|
//
|
|
|
|
Status BlockBasedTableBuilder::InsertBlockInCache(const Slice& block_contents,
|
2014-07-16 15:45:49 +02:00
|
|
|
const CompressionType type,
|
|
|
|
const BlockHandle* handle) {
|
2013-09-02 08:23:40 +02:00
|
|
|
Rep* r = rep_;
|
2014-08-25 23:22:05 +02:00
|
|
|
Cache* block_cache_compressed = r->table_options.block_cache_compressed.get();
|
2013-09-02 08:23:40 +02:00
|
|
|
|
|
|
|
if (type != kNoCompression && block_cache_compressed != nullptr) {
|
|
|
|
|
|
|
|
size_t size = block_contents.size();
|
|
|
|
|
2014-09-18 00:08:19 +02:00
|
|
|
std::unique_ptr<char[]> ubuf(new char[size + 1]);
|
2014-08-16 00:05:09 +02:00
|
|
|
memcpy(ubuf.get(), block_contents.data(), size);
|
2014-07-16 15:45:49 +02:00
|
|
|
ubuf[size] = type;
|
2013-09-02 08:23:40 +02:00
|
|
|
|
2014-08-16 00:05:09 +02:00
|
|
|
BlockContents results(std::move(ubuf), size, true, type);
|
2013-09-02 08:23:40 +02:00
|
|
|
|
2016-10-19 01:59:37 +02:00
|
|
|
Block* block = new Block(std::move(results), kDisableGlobalSequenceNumber);
|
2013-09-02 08:23:40 +02:00
|
|
|
|
|
|
|
// make cache key by appending the file offset to the cache prefix id
|
|
|
|
char* end = EncodeVarint64(
|
|
|
|
r->compressed_cache_key_prefix +
|
|
|
|
r->compressed_cache_key_prefix_size,
|
|
|
|
handle->offset());
|
|
|
|
Slice key(r->compressed_cache_key_prefix, static_cast<size_t>
|
|
|
|
(end - r->compressed_cache_key_prefix));
|
|
|
|
|
|
|
|
// Insert into compressed block cache.
|
2016-03-11 02:35:19 +01:00
|
|
|
block_cache_compressed->Insert(key, block, block->usable_size(),
|
|
|
|
&DeleteCachedBlock);
|
2013-09-02 08:23:40 +02:00
|
|
|
|
|
|
|
// Invalidate OS cache.
|
2014-11-13 20:39:30 +01:00
|
|
|
r->file->InvalidateCache(static_cast<size_t>(r->offset), size);
|
2013-09-02 08:23:40 +02:00
|
|
|
}
|
|
|
|
return Status::OK();
|
|
|
|
}
|
|
|
|
|
2013-10-29 01:54:09 +01:00
|
|
|
Status BlockBasedTableBuilder::Finish() {
|
2011-03-18 23:37:00 +01:00
|
|
|
Rep* r = rep_;
|
2013-11-08 06:27:21 +01:00
|
|
|
bool empty_data_block = r->data_block.empty();
|
2011-03-18 23:37:00 +01:00
|
|
|
Flush();
|
|
|
|
assert(!r->closed);
|
|
|
|
r->closed = true;
|
2012-04-17 17:36:46 +02:00
|
|
|
|
2013-11-20 01:29:42 +01:00
|
|
|
// To make sure properties block is able to keep the accurate size of index
|
2013-10-10 20:43:24 +02:00
|
|
|
// block, we will finish writing all index entries here and flush them
|
|
|
|
// to storage after metaindex block is written.
|
2013-11-08 06:27:21 +01:00
|
|
|
if (ok() && !empty_data_block) {
|
2014-05-15 23:09:03 +02:00
|
|
|
r->index_builder->AddIndexEntry(
|
|
|
|
&r->last_key, nullptr /* no next data block */, r->pending_handle);
|
|
|
|
}
|
|
|
|
|
2017-03-07 22:48:02 +01:00
|
|
|
BlockHandle filter_block_handle, metaindex_block_handle, index_block_handle,
|
|
|
|
compression_dict_block_handle, range_del_block_handle;
|
|
|
|
// Write filter block
|
|
|
|
if (ok() && r->filter_builder != nullptr) {
|
|
|
|
Status s = Status::Incomplete();
|
|
|
|
while (s.IsIncomplete()) {
|
|
|
|
Slice filter_content = r->filter_builder->Finish(filter_block_handle, &s);
|
|
|
|
assert(s.ok() || s.IsIncomplete());
|
|
|
|
r->props.filter_size += filter_content.size();
|
|
|
|
WriteRawBlock(filter_content, kNoCompression, &filter_block_handle);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-05-15 23:09:03 +02:00
|
|
|
IndexBuilder::IndexBlocks index_blocks;
|
2017-02-07 01:29:29 +01:00
|
|
|
auto index_builder_status = r->index_builder->Finish(&index_blocks);
|
|
|
|
if (index_builder_status.IsIncomplete()) {
|
|
|
|
// We we have more than one index partition then meta_blocks are not
|
|
|
|
// supported for the index. Currently meta_blocks are used only by
|
|
|
|
// HashIndexBuilder which is not multi-partition.
|
|
|
|
assert(index_blocks.meta_blocks.empty());
|
|
|
|
} else if (!index_builder_status.ok()) {
|
|
|
|
return index_builder_status;
|
2013-10-10 20:43:24 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// Write meta blocks and metaindex block with the following order.
|
|
|
|
// 1. [meta block: filter]
|
2016-08-20 00:10:31 +02:00
|
|
|
// 2. [meta block: properties]
|
|
|
|
// 3. [meta block: compression dictionary]
|
|
|
|
// 4. [meta block: range deletion tombstone]
|
|
|
|
// 5. [metaindex block]
|
2014-05-15 23:09:03 +02:00
|
|
|
// write meta blocks
|
|
|
|
MetaIndexBuilder meta_index_builder;
|
|
|
|
for (const auto& item : index_blocks.meta_blocks) {
|
|
|
|
BlockHandle block_handle;
|
Shared dictionary compression using reference block
Summary:
This adds a new metablock containing a shared dictionary that is used
to compress all data blocks in the SST file. The size of the shared dictionary
is configurable in CompressionOptions and defaults to 0. It's currently only
used for zlib/lz4/lz4hc, but the block will be stored in the SST regardless of
the compression type if the user chooses a nonzero dictionary size.
During compaction, computes the dictionary by randomly sampling the first
output file in each subcompaction. It pre-computes the intervals to sample
by assuming the output file will have the maximum allowable length. In case
the file is smaller, some of the pre-computed sampling intervals can be beyond
end-of-file, in which case we skip over those samples and the dictionary will
be a bit smaller. After the dictionary is generated using the first file in a
subcompaction, it is loaded into the compression library before writing each
block in each subsequent file of that subcompaction.
On the read path, gets the dictionary from the metablock, if it exists. Then,
loads that dictionary into the compression library before reading each block.
Test Plan: new unit test
Reviewers: yhchiang, IslamAbdelRahman, cyan, sdong
Reviewed By: sdong
Subscribers: andrewkr, yoshinorim, kradhakrishnan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D52287
2016-04-28 02:36:03 +02:00
|
|
|
WriteBlock(item.second, &block_handle, false /* is_data_block */);
|
2014-05-15 23:09:03 +02:00
|
|
|
meta_index_builder.Add(item.first, block_handle);
|
|
|
|
}
|
2013-10-10 20:43:24 +02:00
|
|
|
|
2014-05-15 23:09:03 +02:00
|
|
|
if (ok()) {
|
2017-03-07 22:48:02 +01:00
|
|
|
if (r->filter_builder != nullptr) {
|
2013-10-10 20:43:24 +02:00
|
|
|
// Add mapping from "<filter_block_prefix>.Name" to location
|
|
|
|
// of filter data.
|
2014-09-08 19:37:05 +02:00
|
|
|
std::string key;
|
2017-03-07 22:48:02 +01:00
|
|
|
if (r->filter_builder->IsBlockBased()) {
|
2014-09-08 19:37:05 +02:00
|
|
|
key = BlockBasedTable::kFilterBlockPrefix;
|
|
|
|
} else {
|
2017-03-07 22:48:02 +01:00
|
|
|
key = r->table_options.partition_filters
|
|
|
|
? BlockBasedTable::kPartitionedFilterBlockPrefix
|
|
|
|
: BlockBasedTable::kFullFilterBlockPrefix;
|
2014-09-08 19:37:05 +02:00
|
|
|
}
|
2014-08-25 23:22:05 +02:00
|
|
|
key.append(r->table_options.filter_policy->Name());
|
2014-05-15 23:09:03 +02:00
|
|
|
meta_index_builder.Add(key, filter_block_handle);
|
2013-10-10 20:43:24 +02:00
|
|
|
}
|
|
|
|
|
Shared dictionary compression using reference block
Summary:
This adds a new metablock containing a shared dictionary that is used
to compress all data blocks in the SST file. The size of the shared dictionary
is configurable in CompressionOptions and defaults to 0. It's currently only
used for zlib/lz4/lz4hc, but the block will be stored in the SST regardless of
the compression type if the user chooses a nonzero dictionary size.
During compaction, computes the dictionary by randomly sampling the first
output file in each subcompaction. It pre-computes the intervals to sample
by assuming the output file will have the maximum allowable length. In case
the file is smaller, some of the pre-computed sampling intervals can be beyond
end-of-file, in which case we skip over those samples and the dictionary will
be a bit smaller. After the dictionary is generated using the first file in a
subcompaction, it is loaded into the compression library before writing each
block in each subsequent file of that subcompaction.
On the read path, gets the dictionary from the metablock, if it exists. Then,
loads that dictionary into the compression library before reading each block.
Test Plan: new unit test
Reviewers: yhchiang, IslamAbdelRahman, cyan, sdong
Reviewed By: sdong
Subscribers: andrewkr, yoshinorim, kradhakrishnan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D52287
2016-04-28 02:36:03 +02:00
|
|
|
// Write properties and compression dictionary blocks.
|
2013-10-10 20:43:24 +02:00
|
|
|
{
|
2013-12-05 22:09:13 +01:00
|
|
|
PropertyBlockBuilder property_block_builder;
|
2016-04-07 08:10:32 +02:00
|
|
|
r->props.column_family_id = r->column_family_id;
|
|
|
|
r->props.column_family_name = r->column_family_name;
|
2014-08-25 23:22:05 +02:00
|
|
|
r->props.filter_policy_name = r->table_options.filter_policy != nullptr ?
|
|
|
|
r->table_options.filter_policy->Name() : "";
|
2013-11-20 01:29:42 +01:00
|
|
|
r->props.index_size =
|
2014-03-01 03:19:07 +01:00
|
|
|
r->index_builder->EstimatedSize() + kBlockTrailerSize;
|
2016-10-21 20:31:42 +02:00
|
|
|
r->props.comparator_name = r->ioptions.user_comparator != nullptr
|
|
|
|
? r->ioptions.user_comparator->Name()
|
2016-04-21 19:16:28 +02:00
|
|
|
: "nullptr";
|
|
|
|
r->props.merge_operator_name = r->ioptions.merge_operator != nullptr
|
|
|
|
? r->ioptions.merge_operator->Name()
|
|
|
|
: "nullptr";
|
2016-05-12 18:47:16 +02:00
|
|
|
r->props.compression_name = CompressionTypeToString(r->compression_type);
|
2016-08-26 20:46:32 +02:00
|
|
|
r->props.prefix_extractor_name =
|
|
|
|
r->ioptions.prefix_extractor != nullptr
|
|
|
|
? r->ioptions.prefix_extractor->Name()
|
|
|
|
: "nullptr";
|
2016-04-21 19:16:28 +02:00
|
|
|
|
|
|
|
std::string property_collectors_names = "[";
|
|
|
|
property_collectors_names = "[";
|
|
|
|
for (size_t i = 0;
|
|
|
|
i < r->ioptions.table_properties_collector_factories.size(); ++i) {
|
|
|
|
if (i != 0) {
|
|
|
|
property_collectors_names += ",";
|
|
|
|
}
|
|
|
|
property_collectors_names +=
|
|
|
|
r->ioptions.table_properties_collector_factories[i]->Name();
|
|
|
|
}
|
|
|
|
property_collectors_names += "]";
|
|
|
|
r->props.property_collectors_names = property_collectors_names;
|
2017-06-13 19:59:22 +02:00
|
|
|
if (r->table_options.index_type ==
|
|
|
|
BlockBasedTableOptions::kTwoLevelIndexSearch) {
|
|
|
|
assert(r->p_index_builder_ != nullptr);
|
|
|
|
r->props.index_partitions = r->p_index_builder_->NumPartitions();
|
|
|
|
r->props.top_level_index_size =
|
|
|
|
r->p_index_builder_->EstimateTopLevelIndexSize(r->offset);
|
|
|
|
}
|
2013-10-10 20:43:24 +02:00
|
|
|
|
2013-12-05 22:09:13 +01:00
|
|
|
// Add basic properties
|
|
|
|
property_block_builder.AddTableProperty(r->props);
|
2013-10-16 20:50:50 +02:00
|
|
|
|
2014-03-01 03:19:07 +01:00
|
|
|
// Add use collected properties
|
TablePropertiesCollectorFactory
Summary:
This diff addresses task #4296714 and rethinks how users provide us with TablePropertiesCollectors as part of Options.
Here's description of task #4296714:
I'm debugging #4295529 and noticed that our count of user properties kDeletedKeys is wrong. We're sharing one single InternalKeyPropertiesCollector with all Table Builders. In LOG Files, we're outputting number of kDeletedKeys as connected with a single table, while it's actually the total count of deleted keys since creation of the DB.
For example, this table has 3155 entries and 1391828 deleted keys.
The problem with current approach that we call methods on a single TablePropertiesCollector for all the tables we create. Even worse, we could do it from multiple threads at the same time and TablePropertiesCollector has no way of knowing which table we're calling it for.
Good part: Looks like nobody inside Facebook is using Options::table_properties_collectors. This means we should be able to painfully change the API.
In this change, I introduce TablePropertiesCollectorFactory. For every table we create, we call `CreateTablePropertiesCollector`, which creates a TablePropertiesCollector for a single table. We then use it sequentially from a single thread, which means it doesn't have to be thread-safe.
Test Plan:
Added a test in table_properties_collector_test that fails on master (build two tables, assert that kDeletedKeys count is correct for the second one).
Also, all other tests
Reviewers: sdong, dhruba, haobo, kailiu
Reviewed By: kailiu
CC: leveldb
Differential Revision: https://reviews.facebook.net/D18579
2014-05-13 21:30:55 +02:00
|
|
|
NotifyCollectTableCollectorsOnFinish(r->table_properties_collectors,
|
2014-09-05 01:18:36 +02:00
|
|
|
r->ioptions.info_log,
|
TablePropertiesCollectorFactory
Summary:
This diff addresses task #4296714 and rethinks how users provide us with TablePropertiesCollectors as part of Options.
Here's description of task #4296714:
I'm debugging #4295529 and noticed that our count of user properties kDeletedKeys is wrong. We're sharing one single InternalKeyPropertiesCollector with all Table Builders. In LOG Files, we're outputting number of kDeletedKeys as connected with a single table, while it's actually the total count of deleted keys since creation of the DB.
For example, this table has 3155 entries and 1391828 deleted keys.
The problem with current approach that we call methods on a single TablePropertiesCollector for all the tables we create. Even worse, we could do it from multiple threads at the same time and TablePropertiesCollector has no way of knowing which table we're calling it for.
Good part: Looks like nobody inside Facebook is using Options::table_properties_collectors. This means we should be able to painfully change the API.
In this change, I introduce TablePropertiesCollectorFactory. For every table we create, we call `CreateTablePropertiesCollector`, which creates a TablePropertiesCollector for a single table. We then use it sequentially from a single thread, which means it doesn't have to be thread-safe.
Test Plan:
Added a test in table_properties_collector_test that fails on master (build two tables, assert that kDeletedKeys count is correct for the second one).
Also, all other tests
Reviewers: sdong, dhruba, haobo, kailiu
Reviewed By: kailiu
CC: leveldb
Differential Revision: https://reviews.facebook.net/D18579
2014-05-13 21:30:55 +02:00
|
|
|
&property_block_builder);
|
2013-10-10 20:43:24 +02:00
|
|
|
|
2013-11-20 01:29:42 +01:00
|
|
|
BlockHandle properties_block_handle;
|
2013-12-05 22:09:13 +01:00
|
|
|
WriteRawBlock(
|
|
|
|
property_block_builder.Finish(),
|
|
|
|
kNoCompression,
|
|
|
|
&properties_block_handle
|
2013-10-10 20:43:24 +02:00
|
|
|
);
|
2014-05-15 23:09:03 +02:00
|
|
|
meta_index_builder.Add(kPropertiesBlock, properties_block_handle);
|
Shared dictionary compression using reference block
Summary:
This adds a new metablock containing a shared dictionary that is used
to compress all data blocks in the SST file. The size of the shared dictionary
is configurable in CompressionOptions and defaults to 0. It's currently only
used for zlib/lz4/lz4hc, but the block will be stored in the SST regardless of
the compression type if the user chooses a nonzero dictionary size.
During compaction, computes the dictionary by randomly sampling the first
output file in each subcompaction. It pre-computes the intervals to sample
by assuming the output file will have the maximum allowable length. In case
the file is smaller, some of the pre-computed sampling intervals can be beyond
end-of-file, in which case we skip over those samples and the dictionary will
be a bit smaller. After the dictionary is generated using the first file in a
subcompaction, it is loaded into the compression library before writing each
block in each subsequent file of that subcompaction.
On the read path, gets the dictionary from the metablock, if it exists. Then,
loads that dictionary into the compression library before reading each block.
Test Plan: new unit test
Reviewers: yhchiang, IslamAbdelRahman, cyan, sdong
Reviewed By: sdong
Subscribers: andrewkr, yoshinorim, kradhakrishnan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D52287
2016-04-28 02:36:03 +02:00
|
|
|
|
|
|
|
// Write compression dictionary block
|
|
|
|
if (r->compression_dict && r->compression_dict->size()) {
|
|
|
|
WriteRawBlock(*r->compression_dict, kNoCompression,
|
|
|
|
&compression_dict_block_handle);
|
|
|
|
meta_index_builder.Add(kCompressionDictBlock,
|
|
|
|
compression_dict_block_handle);
|
|
|
|
}
|
|
|
|
} // end of properties/compression dictionary block writing
|
2016-08-20 00:10:31 +02:00
|
|
|
|
|
|
|
if (ok() && !r->range_del_block.empty()) {
|
|
|
|
WriteRawBlock(r->range_del_block.Finish(), kNoCompression,
|
|
|
|
&range_del_block_handle);
|
|
|
|
meta_index_builder.Add(kRangeDelBlock, range_del_block_handle);
|
|
|
|
} // range deletion tombstone meta block
|
2014-05-15 23:09:03 +02:00
|
|
|
} // meta blocks
|
2012-04-17 17:36:46 +02:00
|
|
|
|
|
|
|
// Write index block
|
2011-03-18 23:37:00 +01:00
|
|
|
if (ok()) {
|
2014-05-15 23:09:03 +02:00
|
|
|
// flush the meta index block
|
|
|
|
WriteRawBlock(meta_index_builder.Finish(), kNoCompression,
|
|
|
|
&metaindex_block_handle);
|
2017-02-07 01:29:29 +01:00
|
|
|
|
|
|
|
const bool is_data_block = true;
|
Shared dictionary compression using reference block
Summary:
This adds a new metablock containing a shared dictionary that is used
to compress all data blocks in the SST file. The size of the shared dictionary
is configurable in CompressionOptions and defaults to 0. It's currently only
used for zlib/lz4/lz4hc, but the block will be stored in the SST regardless of
the compression type if the user chooses a nonzero dictionary size.
During compaction, computes the dictionary by randomly sampling the first
output file in each subcompaction. It pre-computes the intervals to sample
by assuming the output file will have the maximum allowable length. In case
the file is smaller, some of the pre-computed sampling intervals can be beyond
end-of-file, in which case we skip over those samples and the dictionary will
be a bit smaller. After the dictionary is generated using the first file in a
subcompaction, it is loaded into the compression library before writing each
block in each subsequent file of that subcompaction.
On the read path, gets the dictionary from the metablock, if it exists. Then,
loads that dictionary into the compression library before reading each block.
Test Plan: new unit test
Reviewers: yhchiang, IslamAbdelRahman, cyan, sdong
Reviewed By: sdong
Subscribers: andrewkr, yoshinorim, kradhakrishnan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D52287
2016-04-28 02:36:03 +02:00
|
|
|
WriteBlock(index_blocks.index_block_contents, &index_block_handle,
|
2017-02-07 01:29:29 +01:00
|
|
|
!is_data_block);
|
|
|
|
// If there are more index partitions, finish them and write them out
|
|
|
|
Status& s = index_builder_status;
|
|
|
|
while (s.IsIncomplete()) {
|
|
|
|
s = r->index_builder->Finish(&index_blocks, index_block_handle);
|
|
|
|
if (!s.ok() && !s.IsIncomplete()) {
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
WriteBlock(index_blocks.index_block_contents, &index_block_handle,
|
|
|
|
!is_data_block);
|
|
|
|
// The last index_block_handle will be for the partition index block
|
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
2012-04-17 17:36:46 +02:00
|
|
|
|
|
|
|
// Write footer
|
2011-03-18 23:37:00 +01:00
|
|
|
if (ok()) {
|
2014-05-01 20:09:32 +02:00
|
|
|
// No need to write out new footer if we're using default checksum.
|
|
|
|
// We're writing legacy magic number because we want old versions of RocksDB
|
|
|
|
// be able to read files generated with new release (just in case if
|
|
|
|
// somebody wants to roll back after an upgrade)
|
|
|
|
// TODO(icanadi) at some point in the future, when we're absolutely sure
|
|
|
|
// nobody will roll back to RocksDB 2.x versions, retire the legacy magic
|
|
|
|
// number and always write new table files with new magic number
|
2015-01-13 23:33:04 +01:00
|
|
|
bool legacy = (r->table_options.format_version == 0);
|
|
|
|
// this is guaranteed by BlockBasedTableBuilder's constructor
|
|
|
|
assert(r->table_options.checksum == kCRC32c ||
|
|
|
|
r->table_options.format_version != 0);
|
2014-05-01 20:09:32 +02:00
|
|
|
Footer footer(legacy ? kLegacyBlockBasedTableMagicNumber
|
2015-01-13 23:33:04 +01:00
|
|
|
: kBlockBasedTableMagicNumber,
|
|
|
|
r->table_options.format_version);
|
2011-03-18 23:37:00 +01:00
|
|
|
footer.set_metaindex_handle(metaindex_block_handle);
|
|
|
|
footer.set_index_handle(index_block_handle);
|
2014-08-25 23:22:05 +02:00
|
|
|
footer.set_checksum(r->table_options.checksum);
|
2011-03-18 23:37:00 +01:00
|
|
|
std::string footer_encoding;
|
|
|
|
footer.EncodeTo(&footer_encoding);
|
2017-06-26 22:15:55 +02:00
|
|
|
assert(r->status.ok());
|
2011-03-18 23:37:00 +01:00
|
|
|
r->status = r->file->Append(footer_encoding);
|
|
|
|
if (r->status.ok()) {
|
|
|
|
r->offset += footer_encoding.size();
|
|
|
|
}
|
|
|
|
}
|
2013-11-20 01:29:42 +01:00
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
return r->status;
|
|
|
|
}
|
|
|
|
|
2013-10-29 01:54:09 +01:00
|
|
|
void BlockBasedTableBuilder::Abandon() {
|
2011-03-18 23:37:00 +01:00
|
|
|
Rep* r = rep_;
|
|
|
|
assert(!r->closed);
|
|
|
|
r->closed = true;
|
|
|
|
}
|
|
|
|
|
2013-10-29 01:54:09 +01:00
|
|
|
uint64_t BlockBasedTableBuilder::NumEntries() const {
|
2013-11-20 01:29:42 +01:00
|
|
|
return rep_->props.num_entries;
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2013-10-29 01:54:09 +01:00
|
|
|
uint64_t BlockBasedTableBuilder::FileSize() const {
|
2011-03-18 23:37:00 +01:00
|
|
|
return rep_->offset;
|
|
|
|
}
|
|
|
|
|
2015-06-04 21:03:40 +02:00
|
|
|
bool BlockBasedTableBuilder::NeedCompact() const {
|
|
|
|
for (const auto& collector : rep_->table_properties_collectors) {
|
|
|
|
if (collector->NeedCompact()) {
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
Add more table properties to EventLogger
Summary:
Example output:
{"time_micros": 1431463794310521, "job": 353, "event": "table_file_creation", "file_number": 387, "file_size": 86937, "table_info": {"data_size": "81801", "index_size": "9751", "filter_size": "0", "raw_key_size": "23448", "raw_average_key_size": "24.000000", "raw_value_size": "990571", "raw_average_value_size": "1013.890481", "num_data_blocks": "245", "num_entries": "977", "filter_policy_name": "", "kDeletedKeys": "0"}}
Also fixed a bug where BuildTable() in recovery was passing Env::IOHigh argument into paranoid_checks_file parameter.
Test Plan: make check + check out the output in the log
Reviewers: sdong, rven, yhchiang
Reviewed By: yhchiang
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D38343
2015-05-13 00:53:55 +02:00
|
|
|
TableProperties BlockBasedTableBuilder::GetTableProperties() const {
|
|
|
|
TableProperties ret = rep_->props;
|
|
|
|
for (const auto& collector : rep_->table_properties_collectors) {
|
|
|
|
for (const auto& prop : collector->GetReadableProperties()) {
|
2015-09-15 18:03:08 +02:00
|
|
|
ret.readable_properties.insert(prop);
|
Add more table properties to EventLogger
Summary:
Example output:
{"time_micros": 1431463794310521, "job": 353, "event": "table_file_creation", "file_number": 387, "file_size": 86937, "table_info": {"data_size": "81801", "index_size": "9751", "filter_size": "0", "raw_key_size": "23448", "raw_average_key_size": "24.000000", "raw_value_size": "990571", "raw_average_value_size": "1013.890481", "num_data_blocks": "245", "num_entries": "977", "filter_policy_name": "", "kDeletedKeys": "0"}}
Also fixed a bug where BuildTable() in recovery was passing Env::IOHigh argument into paranoid_checks_file parameter.
Test Plan: make check + check out the output in the log
Reviewers: sdong, rven, yhchiang
Reviewed By: yhchiang
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D38343
2015-05-13 00:53:55 +02:00
|
|
|
}
|
2015-09-15 18:03:08 +02:00
|
|
|
collector->Finish(&ret.user_collected_properties);
|
Add more table properties to EventLogger
Summary:
Example output:
{"time_micros": 1431463794310521, "job": 353, "event": "table_file_creation", "file_number": 387, "file_size": 86937, "table_info": {"data_size": "81801", "index_size": "9751", "filter_size": "0", "raw_key_size": "23448", "raw_average_key_size": "24.000000", "raw_value_size": "990571", "raw_average_value_size": "1013.890481", "num_data_blocks": "245", "num_entries": "977", "filter_policy_name": "", "kDeletedKeys": "0"}}
Also fixed a bug where BuildTable() in recovery was passing Env::IOHigh argument into paranoid_checks_file parameter.
Test Plan: make check + check out the output in the log
Reviewers: sdong, rven, yhchiang
Reviewed By: yhchiang
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D38343
2015-05-13 00:53:55 +02:00
|
|
|
}
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2014-05-15 23:09:03 +02:00
|
|
|
const std::string BlockBasedTable::kFilterBlockPrefix = "filter.";
|
2014-09-08 19:37:05 +02:00
|
|
|
const std::string BlockBasedTable::kFullFilterBlockPrefix = "fullfilter.";
|
2017-03-07 22:48:02 +01:00
|
|
|
const std::string BlockBasedTable::kPartitionedFilterBlockPrefix =
|
|
|
|
"partitionedfilter.";
|
2013-10-04 06:49:15 +02:00
|
|
|
} // namespace rocksdb
|