RocksJava: Add more flags to BlockBasedTableConfig (#4589)
Summary: Punch through more flags for BlockBasedTableConfig, mostly around caching index + filter blocks and partitioned filters. sagar0 adamretter Pull Request resolved: https://github.com/facebook/rocksdb/pull/4589 Differential Revision: D12840626 Pulled By: sagar0 fbshipit-source-id: 3c289d367ceb2a012023aa791b990a437dd1393a
This commit is contained in:
parent
9da88a8321
commit
72afdf20fc
@ -37,7 +37,7 @@ jlong Java_org_rocksdb_PlainTableConfig_newTableFactoryHandle(
|
||||
/*
|
||||
* Class: org_rocksdb_BlockBasedTableConfig
|
||||
* Method: newTableFactoryHandle
|
||||
* Signature: (ZJIJJIIZIZZZJIBBI)J
|
||||
* Signature: (ZJIJJIIZJZZZZJZZJIBBI)J
|
||||
*/
|
||||
jlong Java_org_rocksdb_BlockBasedTableConfig_newTableFactoryHandle(
|
||||
JNIEnv * /*env*/, jobject /*jobj*/, jboolean no_block_cache,
|
||||
@ -45,7 +45,10 @@ jlong Java_org_rocksdb_BlockBasedTableConfig_newTableFactoryHandle(
|
||||
jlong block_size, jint block_size_deviation, jint block_restart_interval,
|
||||
jboolean whole_key_filtering, jlong jfilter_policy,
|
||||
jboolean cache_index_and_filter_blocks,
|
||||
jboolean cache_index_and_filter_blocks_with_high_priority,
|
||||
jboolean pin_l0_filter_and_index_blocks_in_cache,
|
||||
jboolean partition_filters, jlong metadata_block_size,
|
||||
jboolean pin_top_level_index_and_filter,
|
||||
jboolean hash_index_allow_collision, jlong block_cache_compressed_size,
|
||||
jint block_cache_compressd_num_shard_bits, jbyte jchecksum_type,
|
||||
jbyte jindex_type, jint jformat_version) {
|
||||
@ -77,8 +80,13 @@ jlong Java_org_rocksdb_BlockBasedTableConfig_newTableFactoryHandle(
|
||||
options.filter_policy = *pFilterPolicy;
|
||||
}
|
||||
options.cache_index_and_filter_blocks = cache_index_and_filter_blocks;
|
||||
options.cache_index_and_filter_blocks_with_high_priority =
|
||||
cache_index_and_filter_blocks_with_high_priority;
|
||||
options.pin_l0_filter_and_index_blocks_in_cache =
|
||||
pin_l0_filter_and_index_blocks_in_cache;
|
||||
options.partition_filters = partition_filters;
|
||||
options.metadata_block_size = metadata_block_size;
|
||||
options.pin_top_level_index_and_filter = pin_top_level_index_and_filter;
|
||||
options.hash_index_allow_collision = hash_index_allow_collision;
|
||||
if (block_cache_compressed_size > 0) {
|
||||
if (block_cache_compressd_num_shard_bits > 0) {
|
||||
|
@ -22,7 +22,11 @@ public class BlockBasedTableConfig extends TableFormatConfig {
|
||||
wholeKeyFiltering_ = true;
|
||||
filter_ = null;
|
||||
cacheIndexAndFilterBlocks_ = false;
|
||||
cacheIndexAndFilterBlocksWithHighPriority_ = false;
|
||||
pinL0FilterAndIndexBlocksInCache_ = false;
|
||||
partitionFilters_ = false;
|
||||
metadataBlockSize_ = 4096;
|
||||
pinTopLevelIndexAndFilter_ = true;
|
||||
hashIndexAllowCollision_ = true;
|
||||
blockCacheCompressedSize_ = 0;
|
||||
blockCacheCompressedNumShardBits_ = 0;
|
||||
@ -246,6 +250,31 @@ public class BlockBasedTableConfig extends TableFormatConfig {
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Indicates if index and filter blocks will be treated as high-priority in the block cache.
|
||||
* See note below about applicability. If not specified, defaults to false.
|
||||
*
|
||||
* @return if index and filter blocks will be treated as high-priority.
|
||||
*/
|
||||
public boolean cacheIndexAndFilterBlocksWithHighPriority() {
|
||||
return cacheIndexAndFilterBlocksWithHighPriority_;
|
||||
}
|
||||
|
||||
/**
|
||||
* If true, cache index and filter blocks with high priority. If set to true,
|
||||
* depending on implementation of block cache, index and filter blocks may be
|
||||
* less likely to be evicted than data blocks.
|
||||
*
|
||||
* @param cacheIndexAndFilterBlocksWithHighPriority if index and filter blocks
|
||||
* will be treated as high-priority.
|
||||
* @return the reference to the current config.
|
||||
*/
|
||||
public BlockBasedTableConfig setCacheIndexAndFilterBlocksWithHighPriority(
|
||||
final boolean cacheIndexAndFilterBlocksWithHighPriority) {
|
||||
cacheIndexAndFilterBlocksWithHighPriority_ = cacheIndexAndFilterBlocksWithHighPriority;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Indicating if we'd like to pin L0 index/filter blocks to the block cache.
|
||||
If not specified, defaults to false.
|
||||
@ -269,6 +298,70 @@ public class BlockBasedTableConfig extends TableFormatConfig {
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Indicating if we're using partitioned filters. Defaults to false.
|
||||
*
|
||||
* @return if we're using partition filters.
|
||||
*/
|
||||
public boolean partitionFilters() {
|
||||
return partitionFilters_;
|
||||
}
|
||||
|
||||
/**
|
||||
* Use partitioned full filters for each SST file. This option is incompatible with
|
||||
* block-based filters.
|
||||
*
|
||||
* @param partitionFilters use partition filters.
|
||||
* @return the reference to the current config.
|
||||
*/
|
||||
public BlockBasedTableConfig setPartitionFilters(final boolean partitionFilters) {
|
||||
partitionFilters_ = partitionFilters;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return block size for partitioned metadata.
|
||||
*/
|
||||
public long metadataBlockSize() {
|
||||
return metadataBlockSize_;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set block size for partitioned metadata.
|
||||
*
|
||||
* @param metadataBlockSize Partitioned metadata block size.
|
||||
* @return the reference to the current config.
|
||||
*/
|
||||
public BlockBasedTableConfig setMetadataBlockSize(
|
||||
final long metadataBlockSize) {
|
||||
metadataBlockSize_ = metadataBlockSize;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Indicates if top-level index and filter blocks should be pinned.
|
||||
*
|
||||
* @return if top-level index and filter blocks should be pinned.
|
||||
*/
|
||||
public boolean pinTopLevelIndexAndFilter() {
|
||||
return pinTopLevelIndexAndFilter_;
|
||||
}
|
||||
|
||||
/**
|
||||
* If cacheIndexAndFilterBlocks is true and the below is true, then
|
||||
* the top-level index of partitioned filter and index blocks are stored in
|
||||
* the cache, but a reference is held in the "table reader" object so the
|
||||
* blocks are pinned and only evicted from cache when the table reader is
|
||||
* freed. This is not limited to l0 in LSM tree.
|
||||
*
|
||||
* @param pinTopLevelIndexAndFilter if top-level index and filter blocks should be pinned.
|
||||
* @return the reference to the current config.
|
||||
*/
|
||||
public BlockBasedTableConfig setPinTopLevelIndexAndFilter(final boolean pinTopLevelIndexAndFilter) {
|
||||
pinTopLevelIndexAndFilter_ = pinTopLevelIndexAndFilter;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Influence the behavior when kHashSearch is used.
|
||||
if false, stores a precise prefix to block range mapping
|
||||
@ -440,20 +533,27 @@ public class BlockBasedTableConfig extends TableFormatConfig {
|
||||
return newTableFactoryHandle(noBlockCache_, blockCacheSize_, blockCacheNumShardBits_,
|
||||
blockCacheHandle, blockSize_, blockSizeDeviation_, blockRestartInterval_,
|
||||
wholeKeyFiltering_, filterHandle, cacheIndexAndFilterBlocks_,
|
||||
pinL0FilterAndIndexBlocksInCache_, hashIndexAllowCollision_, blockCacheCompressedSize_,
|
||||
blockCacheCompressedNumShardBits_, checksumType_.getValue(), indexType_.getValue(),
|
||||
formatVersion_);
|
||||
cacheIndexAndFilterBlocksWithHighPriority_, pinL0FilterAndIndexBlocksInCache_,
|
||||
partitionFilters_, metadataBlockSize_, pinTopLevelIndexAndFilter_,
|
||||
hashIndexAllowCollision_, blockCacheCompressedSize_, blockCacheCompressedNumShardBits_,
|
||||
checksumType_.getValue(), indexType_.getValue(), formatVersion_);
|
||||
}
|
||||
|
||||
private native long newTableFactoryHandle(boolean noBlockCache, long blockCacheSize,
|
||||
int blockCacheNumShardBits, long blockCacheHandle, long blockSize, int blockSizeDeviation,
|
||||
int blockRestartInterval, boolean wholeKeyFiltering, long filterPolicyHandle,
|
||||
boolean cacheIndexAndFilterBlocks, boolean pinL0FilterAndIndexBlocksInCache,
|
||||
boolean hashIndexAllowCollision, long blockCacheCompressedSize,
|
||||
int blockCacheCompressedNumShardBits, byte checkSumType, byte indexType, int formatVersion);
|
||||
boolean cacheIndexAndFilterBlocks, boolean cacheIndexAndFilterBlocksWithHighPriority,
|
||||
boolean pinL0FilterAndIndexBlocksInCache, boolean partitionFilters, long metadataBlockSize,
|
||||
boolean pinTopLevelIndexAndFilter, boolean hashIndexAllowCollision,
|
||||
long blockCacheCompressedSize, int blockCacheCompressedNumShardBits,
|
||||
byte checkSumType, byte indexType, int formatVersion);
|
||||
|
||||
private boolean cacheIndexAndFilterBlocks_;
|
||||
private boolean cacheIndexAndFilterBlocksWithHighPriority_;
|
||||
private boolean pinL0FilterAndIndexBlocksInCache_;
|
||||
private boolean partitionFilters_;
|
||||
private long metadataBlockSize_;
|
||||
private boolean pinTopLevelIndexAndFilter_;
|
||||
private IndexType indexType_;
|
||||
private boolean hashIndexAllowCollision_;
|
||||
private ChecksumType checksumType_;
|
||||
|
@ -95,6 +95,46 @@ public class BlockBasedTableConfigTest {
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void cacheIndexAndFilterBlocksWithHighPriority() {
|
||||
BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
|
||||
blockBasedTableConfig.setCacheIndexAndFilterBlocksWithHighPriority(true);
|
||||
assertThat(blockBasedTableConfig.cacheIndexAndFilterBlocksWithHighPriority()).
|
||||
isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void pinL0FilterAndIndexBlocksInCache() {
|
||||
BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
|
||||
blockBasedTableConfig.setPinL0FilterAndIndexBlocksInCache(true);
|
||||
assertThat(blockBasedTableConfig.pinL0FilterAndIndexBlocksInCache()).
|
||||
isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void partitionFilters() {
|
||||
BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
|
||||
blockBasedTableConfig.setPartitionFilters(true);
|
||||
assertThat(blockBasedTableConfig.partitionFilters()).
|
||||
isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void metadataBlockSize() {
|
||||
BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
|
||||
blockBasedTableConfig.setMetadataBlockSize(1024);
|
||||
assertThat(blockBasedTableConfig.metadataBlockSize()).
|
||||
isEqualTo(1024);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void pinTopLevelIndexAndFilter() {
|
||||
BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
|
||||
blockBasedTableConfig.setPinTopLevelIndexAndFilter(false);
|
||||
assertThat(blockBasedTableConfig.pinTopLevelIndexAndFilter()).
|
||||
isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void hashIndexAllowCollision() {
|
||||
BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
|
||||
|
Loading…
Reference in New Issue
Block a user