Merge pull request #356 from fyrz/RocksJava-TableOptions-3.6

[RocksJava] BlockBasedTableConfig & PlainTableConfig enhancements
This commit is contained in:
Yueh-Hsuan Chiang 2014-10-30 13:17:53 -07:00
commit fda592d902
9 changed files with 422 additions and 24 deletions

View File

@ -39,11 +39,13 @@ test: java
javac org/rocksdb/test/*.java
java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.WriteBatchTest
java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.BackupableDBTest
java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.BlockBasedTableConfigTest
java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.ColumnFamilyTest
java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.FilterTest
java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.KeyMayExistTest
java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.MemTableTest
java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.OptionsTest
java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.PlainTableConfigTest
java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.ReadOnlyTest
java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.MergeTest
java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.ReadOptionsTest

View File

@ -14,6 +14,7 @@ public class BlockBasedTableConfig extends TableFormatConfig {
public BlockBasedTableConfig() {
noBlockCache_ = false;
blockCacheSize_ = 8 * 1024 * 1024;
blockCacheNumShardBits_ = 0;
blockSize_ = 4 * 1024;
blockSizeDeviation_ = 10;
blockRestartInterval_ = 16;
@ -22,6 +23,9 @@ public class BlockBasedTableConfig extends TableFormatConfig {
cacheIndexAndFilterBlocks_ = false;
hashIndexAllowCollision_ = true;
blockCacheCompressedSize_ = 0;
blockCacheCompressedNumShardBits_ = 0;
checksumType_ = ChecksumType.kCRC32c;
indexType_ = IndexType.kBinarySearch;
}
/**
@ -293,6 +297,44 @@ public class BlockBasedTableConfig extends TableFormatConfig {
return this;
}
/**
* Sets the checksum type to be used with this table.
*
* @param checksumType {@link org.rocksdb.ChecksumType} value.
* @return the reference to the current option.
*/
public BlockBasedTableConfig setChecksumType(ChecksumType checksumType) {
checksumType_ = checksumType;
return this;
}
/**
*
* @return the currently set checksum type
*/
public ChecksumType checksumType() {
return checksumType_;
}
/**
* Sets the index type to used with this table.
*
* @param indexType {@link org.rocksdb.IndexType} value
* @return the reference to the current option.
*/
public BlockBasedTableConfig setIndexType(IndexType indexType) {
indexType_ = indexType;
return this;
}
/**
*
* @return the currently set index type
*/
public IndexType indexType() {
return indexType_;
}
@Override protected long newTableFactoryHandle() {
long filterHandle = 0;
if (filter_ != null) {
@ -304,7 +346,8 @@ public class BlockBasedTableConfig extends TableFormatConfig {
blockRestartInterval_, wholeKeyFiltering_,
filterHandle, cacheIndexAndFilterBlocks_,
hashIndexAllowCollision_, blockCacheCompressedSize_,
blockCacheCompressedNumShardBits_);
blockCacheCompressedNumShardBits_,
checksumType_.getValue(), indexType_.getValue());
}
private native long newTableFactoryHandle(
@ -312,19 +355,21 @@ public class BlockBasedTableConfig extends TableFormatConfig {
long blockSize, int blockSizeDeviation, int blockRestartInterval,
boolean wholeKeyFiltering, long filterPolicyHandle,
boolean cacheIndexAndFilterBlocks, boolean hashIndexAllowCollision,
long blockCacheCompressedSize, int blockCacheCompressedNumShardBits);
long blockCacheCompressedSize, int blockCacheCompressedNumShardBits,
byte checkSumType, byte indexType);
private boolean cacheIndexAndFilterBlocks_;
private IndexType indexType_;
private boolean hashIndexAllowCollision_;
private ChecksumType checksumType_;
private boolean noBlockCache_;
private long blockSize_;
private long blockCacheSize_;
private int blockCacheNumShardBits_;
private long shard;
private long blockSize_;
private int blockSizeDeviation_;
private int blockRestartInterval_;
private boolean wholeKeyFiltering_;
private Filter filter_;
private boolean cacheIndexAndFilterBlocks_;
private boolean hashIndexAllowCollision_;
private long blockCacheCompressedSize_;
private int blockCacheCompressedNumShardBits_;
private int blockSizeDeviation_;
private int blockRestartInterval_;
private Filter filter_;
private boolean wholeKeyFiltering_;
}

View File

@ -0,0 +1,39 @@
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
package org.rocksdb;
/**
* Checksum types used in conjunction with BlockBasedTable.
*/
public enum ChecksumType {
/**
* Not implemented yet.
*/
kNoChecksum((byte) 0),
/**
* CRC32 Checksum
*/
kCRC32c((byte) 1),
/**
* XX Hash
*/
kxxHash((byte) 2);
/**
* Returns the byte value of the enumerations value
*
* @return byte representation
*/
public byte getValue() {
return value_;
}
private ChecksumType(byte value) {
value_ = value;
}
private final byte value_;
}

View File

@ -0,0 +1,55 @@
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
package org.rocksdb;
/**
* EncodingType
*
* <p>The value will determine how to encode keys
* when writing to a new SST file.</p>
*
* <p>This value will be stored
* inside the SST file which will be used when reading from
* the file, which makes it possible for users to choose
* different encoding type when reopening a DB. Files with
* different encoding types can co-exist in the same DB and
* can be read.</p>
*/
public enum EncodingType {
/**
* Always write full keys without any special encoding.
*/
kPlain((byte) 0),
/**
* <p>Find opportunity to write the same prefix once for multiple rows.
* In some cases, when a key follows a previous key with the same prefix,
* instead of writing out the full key, it just writes out the size of the
* shared prefix, as well as other bytes, to save some bytes.</p>
*
* <p>When using this option, the user is required to use the same prefix
* extractor to make sure the same prefix will be extracted from the same key.
* The Name() value of the prefix extractor will be stored in the file. When
* reopening the file, the name of the options.prefix_extractor given will be
* bitwise compared to the prefix extractors stored in the file. An error
* will be returned if the two don't match.</p>
*/
kPrefix((byte) 1);
/**
* Returns the byte value of the enumerations value
*
* @return byte representation
*/
public byte getValue() {
return value_;
}
private EncodingType(byte value) {
value_ = value;
}
private final byte value_;
}

View File

@ -0,0 +1,37 @@
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
package org.rocksdb;
/**
* IndexType used in conjunction with BlockBasedTable.
*/
public enum IndexType {
/**
* A space efficient index block that is optimized for
* binary-search-based index.
*/
kBinarySearch((byte) 0),
/**
* The hash index, if enabled, will do the hash lookup when
* {@code Options.prefix_extractor} is provided.
*/
kHashSearch((byte) 1);
/**
* Returns the byte value of the enumerations value
*
* @return byte representation
*/
public byte getValue() {
return value_;
}
private IndexType(byte value) {
value_ = value;
}
private final byte value_;
}

View File

@ -7,28 +7,43 @@ package org.rocksdb;
/**
* The config for plain table sst format.
*
* PlainTable is a RocksDB's SST file format optimized for low query latency
* on pure-memory or really low-latency media. It also support prefix
* hash feature.
* <p>PlainTable is a RocksDB's SST file format optimized for low query
* latency on pure-memory or really low-latency media.</p>
*
* <p>It also support prefix hash feature.</p>
*/
public class PlainTableConfig extends TableFormatConfig {
public static final int VARIABLE_LENGTH = 0;
public static final int DEFAULT_BLOOM_BITS_PER_KEY = 10;
public static final double DEFAULT_HASH_TABLE_RATIO = 0.75;
public static final int DEFAULT_INDEX_SPARSENESS = 16;
public static final int DEFAULT_HUGE_TLB_SIZE = 0;
public static final EncodingType DEFAULT_ENCODING_TYPE =
EncodingType.kPlain;
public static final boolean DEFAULT_FULL_SCAN_MODE = false;
public static final boolean DEFAULT_STORE_INDEX_IN_FILE
= false;
public PlainTableConfig() {
keySize_ = VARIABLE_LENGTH;
bloomBitsPerKey_ = DEFAULT_BLOOM_BITS_PER_KEY;
hashTableRatio_ = DEFAULT_HASH_TABLE_RATIO;
indexSparseness_ = DEFAULT_INDEX_SPARSENESS;
hugePageTlbSize_ = DEFAULT_HUGE_TLB_SIZE;
encodingType_ = DEFAULT_ENCODING_TYPE;
fullScanMode_ = DEFAULT_FULL_SCAN_MODE;
storeIndexInFile_ = DEFAULT_STORE_INDEX_IN_FILE;
}
/**
* Set the length of the user key. If it is set to be VARIABLE_LENGTH,
* then it indicates the user keys are variable-lengthed. Otherwise,
* all the keys need to have the same length in byte.
* DEFAULT: VARIABLE_LENGTH
* <p>Set the length of the user key. If it is set to be
* VARIABLE_LENGTH, then it indicates the user keys are
* of variable length.</p>
*
* <p>Otherwise,all the keys need to have the same length
* in byte.</p>
*
* <p>DEFAULT: VARIABLE_LENGTH</p>
*
* @param keySize the length of the user key.
* @return the reference to the current config.
@ -103,21 +118,134 @@ public class PlainTableConfig extends TableFormatConfig {
/**
* @return the index sparseness.
*/
public int indexSparseness() {
public long indexSparseness() {
return indexSparseness_;
}
/**
* <p>huge_page_tlb_size: if <=0, allocate hash indexes and blooms
* from malloc otherwise from huge page TLB.</p>
*
* <p>The user needs to reserve huge pages for it to be allocated,
* like: {@code sysctl -w vm.nr_hugepages=20}</p>
*
* <p>See linux doc Documentation/vm/hugetlbpage.txt</p>
*
* @param hugePageTlbSize
* @return the reference to the current config.
*/
public PlainTableConfig setHugePageTlbSize(int hugePageTlbSize) {
this.hugePageTlbSize_ = hugePageTlbSize;
return this;
}
/**
* Returns the value for huge page tlb size
*
* @return hugePageTlbSize
*/
public int hugePageTlbSize() {
return hugePageTlbSize_;
}
/**
* Sets the encoding type.
*
* <p>This setting determines how to encode
* the keys. See enum {@link EncodingType} for
* the choices.</p>
*
* <p>The value will determine how to encode keys
* when writing to a new SST file. This value will be stored
* inside the SST file which will be used when reading from
* the file, which makes it possible for users to choose
* different encoding type when reopening a DB. Files with
* different encoding types can co-exist in the same DB and
* can be read.</p>
*
* @param encodingType {@link org.rocksdb.EncodingType} value.
* @return the reference to the current config.
*/
public PlainTableConfig setEncodingType(EncodingType encodingType) {
this.encodingType_ = encodingType;
return this;
}
/**
* Returns the active EncodingType
*
* @return currently set encoding type
*/
public EncodingType encodingType() {
return encodingType_;
}
/**
* Set full scan mode, if true the whole file will be read
* one record by one without using the index.
*
* @param fullScanMode boolean value indicating if full
* scan mode shall be enabled.
* @return the reference to the current config.
*/
public PlainTableConfig setFullScanMode(boolean fullScanMode) {
this.fullScanMode_ = fullScanMode;
return this;
}
/**
* Return if full scan mode is active
* @return boolean value indicating if the full scan mode is
* enabled.
*/
public boolean fullScanMode() {
return fullScanMode_;
}
/**
* <p>If set to true: compute plain table index and bloom
* filter during file building and store it in file.
* When reading file, index will be mmaped instead
* of doing recomputation.</p>
*
* @param storeIndexInFile value indicating if index shall
* be stored in a file
* @return the reference to the current config.
*/
public PlainTableConfig setStoreIndexInFile(boolean storeIndexInFile) {
this.storeIndexInFile_ = storeIndexInFile;
return this;
}
/**
* Return a boolean value indicating if index shall be stored
* in a file.
*
* @return currently set value for store index in file.
*/
public boolean storeIndexInFile() {
return storeIndexInFile_;
}
@Override protected long newTableFactoryHandle() {
return newTableFactoryHandle(keySize_, bloomBitsPerKey_,
hashTableRatio_, indexSparseness_);
hashTableRatio_, indexSparseness_, hugePageTlbSize_,
encodingType_.getValue(), fullScanMode_,
storeIndexInFile_);
}
private native long newTableFactoryHandle(
int keySize, int bloomBitsPerKey,
double hashTableRatio, int indexSparseness);
double hashTableRatio, int indexSparseness,
int hugePageTlbSize, byte encodingType,
boolean fullScanMode, boolean storeIndexInFile);
private int keySize_;
private int bloomBitsPerKey_;
private double hashTableRatio_;
private int indexSparseness_;
private int hugePageTlbSize_;
private EncodingType encodingType_;
private boolean fullScanMode_;
private boolean storeIndexInFile_;
}

View File

@ -0,0 +1,47 @@
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
package org.rocksdb.test;
import org.rocksdb.BlockBasedTableConfig;
import org.rocksdb.ChecksumType;
import org.rocksdb.IndexType;
public class BlockBasedTableConfigTest {
public static void main(String[] args) {
BlockBasedTableConfig blockBasedTableConfig =
new BlockBasedTableConfig();
blockBasedTableConfig.setNoBlockCache(true);
assert(blockBasedTableConfig.noBlockCache());
blockBasedTableConfig.setBlockCacheSize(8*1024);
assert(blockBasedTableConfig.blockCacheSize() == (8*1024));
blockBasedTableConfig.setBlockSizeDeviation(12);
assert(blockBasedTableConfig.blockSizeDeviation() == 12);
blockBasedTableConfig.setBlockRestartInterval(15);
assert(blockBasedTableConfig.blockRestartInterval() == 15);
blockBasedTableConfig.setWholeKeyFiltering(false);
assert(!blockBasedTableConfig.wholeKeyFiltering());
blockBasedTableConfig.setCacheIndexAndFilterBlocks(true);
assert(blockBasedTableConfig.cacheIndexAndFilterBlocks());
blockBasedTableConfig.setHashIndexAllowCollision(false);
assert(!blockBasedTableConfig.hashIndexAllowCollision());
blockBasedTableConfig.setBlockCacheCompressedSize(40);
assert(blockBasedTableConfig.blockCacheCompressedSize() == 40);
blockBasedTableConfig.setChecksumType(ChecksumType.kNoChecksum);
blockBasedTableConfig.setChecksumType(ChecksumType.kxxHash);
assert(blockBasedTableConfig.checksumType().equals(
ChecksumType.kxxHash));
blockBasedTableConfig.setIndexType(IndexType.kHashSearch);
assert(blockBasedTableConfig.indexType().equals(
IndexType.kHashSearch));
blockBasedTableConfig.setBlockCacheCompressedNumShardBits(4);
assert(blockBasedTableConfig.blockCacheCompressedNumShardBits()
== 4);
blockBasedTableConfig.setCacheNumShardBits(5);
assert(blockBasedTableConfig.cacheNumShardBits() == 5);
System.out.println("BlockBasedTableConfig test passed");
}
}

View File

@ -0,0 +1,34 @@
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
package org.rocksdb.test;
import org.rocksdb.EncodingType;
import org.rocksdb.PlainTableConfig;
public class PlainTableConfigTest {
public static void main(String[] args) {
PlainTableConfig plainTableConfig = new PlainTableConfig();
plainTableConfig.setKeySize(5);
assert(plainTableConfig.keySize() == 5);
plainTableConfig.setBloomBitsPerKey(11);
assert(plainTableConfig.bloomBitsPerKey() == 11);
plainTableConfig.setHashTableRatio(0.95);
assert(plainTableConfig.hashTableRatio() == 0.95);
plainTableConfig.setIndexSparseness(18);
assert(plainTableConfig.indexSparseness() == 18);
plainTableConfig.setHugePageTlbSize(1);
assert(plainTableConfig.hugePageTlbSize() == 1);
plainTableConfig.setEncodingType(EncodingType.kPrefix);
assert(plainTableConfig.encodingType().equals(
EncodingType.kPrefix));
plainTableConfig.setFullScanMode(true);
assert(plainTableConfig.fullScanMode());
plainTableConfig.setStoreIndexInFile(true);
assert(plainTableConfig.storeIndexInFile());
System.out.println("PlainTableConfig test passed");
}
}

View File

@ -15,23 +15,30 @@
/*
* Class: org_rocksdb_PlainTableConfig
* Method: newTableFactoryHandle
* Signature: (IIDI)J
* Signature: (IIDIIBZZ)J
*/
jlong Java_org_rocksdb_PlainTableConfig_newTableFactoryHandle(
JNIEnv* env, jobject jobj, jint jkey_size, jint jbloom_bits_per_key,
jdouble jhash_table_ratio, jint jindex_sparseness) {
jdouble jhash_table_ratio, jint jindex_sparseness,
jint jhuge_page_tlb_size, jbyte jencoding_type,
jboolean jfull_scan_mode, jboolean jstore_index_in_file) {
rocksdb::PlainTableOptions options = rocksdb::PlainTableOptions();
options.user_key_len = jkey_size;
options.bloom_bits_per_key = jbloom_bits_per_key;
options.hash_table_ratio = jhash_table_ratio;
options.index_sparseness = jindex_sparseness;
options.huge_page_tlb_size = jhuge_page_tlb_size;
options.encoding_type = static_cast<rocksdb::EncodingType>(
jencoding_type);
options.full_scan_mode = jfull_scan_mode;
options.store_index_in_file = jstore_index_in_file;
return reinterpret_cast<jlong>(rocksdb::NewPlainTableFactory(options));
}
/*
* Class: org_rocksdb_BlockBasedTableConfig
* Method: newTableFactoryHandle
* Signature: (ZJIJIIZIZZJI)J
* Signature: (ZJIJIIZIZZJIBB)J
*/
jlong Java_org_rocksdb_BlockBasedTableConfig_newTableFactoryHandle(
JNIEnv* env, jobject jobj, jboolean no_block_cache, jlong block_cache_size,
@ -39,7 +46,8 @@ jlong Java_org_rocksdb_BlockBasedTableConfig_newTableFactoryHandle(
jint block_restart_interval, jboolean whole_key_filtering,
jlong jfilterPolicy, jboolean cache_index_and_filter_blocks,
jboolean hash_index_allow_collision, jlong block_cache_compressed_size,
jint block_cache_compressd_num_shard_bits) {
jint block_cache_compressd_num_shard_bits, jbyte jchecksum_type,
jbyte jindex_type) {
rocksdb::BlockBasedTableOptions options;
options.no_block_cache = no_block_cache;
@ -72,6 +80,9 @@ jlong Java_org_rocksdb_BlockBasedTableConfig_newTableFactoryHandle(
options.block_cache = rocksdb::NewLRUCache(block_cache_compressed_size);
}
}
options.checksum = static_cast<rocksdb::ChecksumType>(jchecksum_type);
options.index_type = static_cast<
rocksdb::BlockBasedTableOptions::IndexType>(jindex_type);
return reinterpret_cast<jlong>(rocksdb::NewBlockBasedTableFactory(options));
}