Merge pull request #354 from fyrz/RocksJava-memtables-3.6

[RocksJava] Memtables update to 3.6
This commit is contained in:
Yueh-Hsuan Chiang 2014-10-24 16:53:55 -07:00
commit 42f0eaceb7
7 changed files with 298 additions and 15 deletions

View File

@ -42,6 +42,7 @@ test: java
java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.ColumnFamilyTest
java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.FilterTest
java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.KeyMayExistTest
java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.MemTableTest
java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.OptionsTest
java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.ReadOnlyTest
java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.ReadOptionsTest

View File

@ -15,9 +15,21 @@ package org.rocksdb;
*/
public class HashLinkedListMemTableConfig extends MemTableConfig {
public static final long DEFAULT_BUCKET_COUNT = 50000;
public static final long DEFAULT_HUGE_PAGE_TLB_SIZE = 0;
public static final int DEFAULT_BUCKET_ENTRIES_LOG_THRES = 4096;
public static final boolean
DEFAULT_IF_LOG_BUCKET_DIST_WHEN_FLUSH = true;
public static final int DEFAUL_THRESHOLD_USE_SKIPLIST = 256;
/**
* HashLinkedListMemTableConfig constructor
*/
public HashLinkedListMemTableConfig() {
bucketCount_ = DEFAULT_BUCKET_COUNT;
hugePageTlbSize_ = DEFAULT_HUGE_PAGE_TLB_SIZE;
bucketEntriesLoggingThreshold_ = DEFAULT_BUCKET_ENTRIES_LOG_THRES;
ifLogBucketDistWhenFlush_ = DEFAULT_IF_LOG_BUCKET_DIST_WHEN_FLUSH;
thresholdUseSkiplist_ = DEFAUL_THRESHOLD_USE_SKIPLIST;
}
/**
@ -42,13 +54,119 @@ public class HashLinkedListMemTableConfig extends MemTableConfig {
return bucketCount_;
}
@Override protected long newMemTableFactoryHandle()
throws RocksDBException {
return newMemTableFactoryHandle(bucketCount_);
/**
* <p>Set the size of huge tlb or allocate the hashtable bytes from
* malloc if {@code size <= 0}.</p>
*
* <p>The user needs to reserve huge pages for it to be allocated,
* like: {@code sysctl -w vm.nr_hugepages=20}</p>
*
* <p>See linux documentation/vm/hugetlbpage.txt</p>
*
* @param size if set to {@code <= 0} hashtable bytes from malloc
* @return the reference to the current HashLinkedListMemTableConfig.
*/
public HashLinkedListMemTableConfig setHugePageTlbSize(long size) {
hugePageTlbSize_ = size;
return this;
}
private native long newMemTableFactoryHandle(long bucketCount)
/**
* Returns the size value of hugePageTlbSize.
*
* @return the hugePageTlbSize.
*/
public long hugePageTlbSize() {
return hugePageTlbSize_;
}
/**
* If number of entries in one bucket exceeds that setting, log
* about it.
*
* @param threshold - number of entries in a single bucket before
* logging starts.
* @return the reference to the current HashLinkedListMemTableConfig.
*/
public HashLinkedListMemTableConfig
setBucketEntriesLoggingThreshold(int threshold) {
bucketEntriesLoggingThreshold_ = threshold;
return this;
}
/**
* Returns the maximum number of entries in one bucket before
* logging starts.
*
* @return maximum number of entries in one bucket before logging
* starts.
*/
public int bucketEntriesLoggingThreshold() {
return bucketEntriesLoggingThreshold_;
}
/**
* If true the distrubition of number of entries will be logged.
*
* @param logDistribution - boolean parameter indicating if number
* of entry distribution shall be logged.
* @return the reference to the current HashLinkedListMemTableConfig.
*/
public HashLinkedListMemTableConfig
setIfLogBucketDistWhenFlush(boolean logDistribution) {
ifLogBucketDistWhenFlush_ = logDistribution;
return this;
}
/**
* Returns information about logging the distribution of
* number of entries on flush.
*
* @return if distrubtion of number of entries shall be logged.
*/
public boolean ifLogBucketDistWhenFlush() {
return ifLogBucketDistWhenFlush_;
}
/**
* Set maximum number of entries in one bucket. Exceeding this val
* leads to a switch from LinkedList to SkipList.
*
* @param threshold maximum number of entries before SkipList is
* used.
* @return the reference to the current HashLinkedListMemTableConfig.
*/
public HashLinkedListMemTableConfig
setThresholdUseSkiplist(int threshold) {
thresholdUseSkiplist_ = threshold;
return this;
}
/**
* Returns entries per bucket threshold before LinkedList is
* replaced by SkipList usage for that bucket.
*
* @return entries per bucket threshold before SkipList is used.
*/
public int thresholdUseSkiplist() {
return thresholdUseSkiplist_;
}
@Override protected long newMemTableFactoryHandle()
throws RocksDBException {
return newMemTableFactoryHandle(bucketCount_, hugePageTlbSize_,
bucketEntriesLoggingThreshold_, ifLogBucketDistWhenFlush_,
thresholdUseSkiplist_);
}
private native long newMemTableFactoryHandle(long bucketCount,
long hugePageTlbSize, int bucketEntriesLoggingThreshold,
boolean ifLogBucketDistWhenFlush, int thresholdUseSkiplist)
throws RocksDBException;
private long bucketCount_;
private long hugePageTlbSize_;
private int bucketEntriesLoggingThreshold_;
private boolean ifLogBucketDistWhenFlush_;
private int thresholdUseSkiplist_;
}

View File

@ -18,6 +18,9 @@ public class HashSkipListMemTableConfig extends MemTableConfig {
public static final int DEFAULT_BRANCHING_FACTOR = 4;
public static final int DEFAULT_HEIGHT = 4;
/**
* HashSkipListMemTableConfig constructor
*/
public HashSkipListMemTableConfig() {
bucketCount_ = DEFAULT_BUCKET_COUNT;
branchingFactor_ = DEFAULT_BRANCHING_FACTOR;

View File

@ -4,12 +4,46 @@ package org.rocksdb;
* The config for skip-list memtable representation.
*/
public class SkipListMemTableConfig extends MemTableConfig {
public static final long DEFAULT_LOOKAHEAD = 0;
/**
* SkipListMemTableConfig constructor
*/
public SkipListMemTableConfig() {
lookahead_ = DEFAULT_LOOKAHEAD;
}
/**
* Sets lookahead for SkipList
*
* @param lookahead If non-zero, each iterator's seek operation
* will start the search from the previously visited record
* (doing at most 'lookahead' steps). This is an
* optimization for the access pattern including many
* seeks with consecutive keys.
* @return the current instance of SkipListMemTableConfig
*/
public SkipListMemTableConfig setLookahead(long lookahead) {
lookahead_ = lookahead;
return this;
}
/**
* Returns the currently set lookahead value.
*
* @return lookahead value
*/
public long lookahead() {
return lookahead_;
}
@Override protected long newMemTableFactoryHandle() {
return newMemTableFactoryHandle0();
return newMemTableFactoryHandle0(lookahead_);
}
private native long newMemTableFactoryHandle0();
private native long newMemTableFactoryHandle0(long lookahead);
private long lookahead_;
}

View File

@ -5,6 +5,10 @@ package org.rocksdb;
*/
public class VectorMemTableConfig extends MemTableConfig {
public static final int DEFAULT_RESERVED_SIZE = 0;
/**
* VectorMemTableConfig constructor
*/
public VectorMemTableConfig() {
reservedSize_ = DEFAULT_RESERVED_SIZE;
}

View File

@ -0,0 +1,107 @@
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
package org.rocksdb.test;
import org.rocksdb.*;
public class MemTableTest {
static {
RocksDB.loadLibrary();
}
public static void main(String[] args) {
Options options = new Options();
// Test HashSkipListMemTableConfig
HashSkipListMemTableConfig memTableConfig =
new HashSkipListMemTableConfig();
assert(memTableConfig.bucketCount() == 1000000);
memTableConfig.setBucketCount(2000000);
assert(memTableConfig.bucketCount() == 2000000);
assert(memTableConfig.height() == 4);
memTableConfig.setHeight(5);
assert(memTableConfig.height() == 5);
assert(memTableConfig.branchingFactor() == 4);
memTableConfig.setBranchingFactor(6);
assert(memTableConfig.branchingFactor() == 6);
try {
options.setMemTableConfig(memTableConfig);
} catch (RocksDBException e) {
assert(false);
}
memTableConfig = null;
options.dispose();
System.gc();
System.runFinalization();
// Test SkipList
options = new Options();
SkipListMemTableConfig skipMemTableConfig =
new SkipListMemTableConfig();
assert(skipMemTableConfig.lookahead() == 0);
skipMemTableConfig.setLookahead(20);
assert(skipMemTableConfig.lookahead() == 20);
try {
options.setMemTableConfig(skipMemTableConfig);
} catch (RocksDBException e) {
assert(false);
}
skipMemTableConfig = null;
options.dispose();
System.gc();
System.runFinalization();
// Test HashLinkedListMemTableConfig
options = new Options();
HashLinkedListMemTableConfig hashLinkedListMemTableConfig =
new HashLinkedListMemTableConfig();
assert(hashLinkedListMemTableConfig.bucketCount() == 50000);
hashLinkedListMemTableConfig.setBucketCount(100000);
assert(hashLinkedListMemTableConfig.bucketCount() == 100000);
assert(hashLinkedListMemTableConfig.hugePageTlbSize() == 0);
hashLinkedListMemTableConfig.setHugePageTlbSize(1);
assert(hashLinkedListMemTableConfig.hugePageTlbSize() == 1);
assert(hashLinkedListMemTableConfig.
bucketEntriesLoggingThreshold() == 4096);
hashLinkedListMemTableConfig.
setBucketEntriesLoggingThreshold(200);
assert(hashLinkedListMemTableConfig.
bucketEntriesLoggingThreshold() == 200);
assert(hashLinkedListMemTableConfig.
ifLogBucketDistWhenFlush() == true);
hashLinkedListMemTableConfig.
setIfLogBucketDistWhenFlush(false);
assert(hashLinkedListMemTableConfig.
ifLogBucketDistWhenFlush() == false);
assert(hashLinkedListMemTableConfig.
thresholdUseSkiplist() == 256);
hashLinkedListMemTableConfig.setThresholdUseSkiplist(29);
assert(hashLinkedListMemTableConfig.
thresholdUseSkiplist() == 29);
try {
options.setMemTableConfig(hashLinkedListMemTableConfig);
} catch (RocksDBException e) {
assert(false);
}
hashLinkedListMemTableConfig = null;
options.dispose();
System.gc();
System.runFinalization();
// test VectorMemTableConfig
options = new Options();
VectorMemTableConfig vectorMemTableConfig =
new VectorMemTableConfig();
assert(vectorMemTableConfig.reservedSize() == 0);
vectorMemTableConfig.setReservedSize(123);
assert(vectorMemTableConfig.reservedSize() == 123);
try {
options.setMemTableConfig(vectorMemTableConfig);
} catch (RocksDBException e) {
assert(false);
}
vectorMemTableConfig = null;
options.dispose();
System.gc();
System.runFinalization();
System.out.println("Mem-table test passed");
}
}

View File

@ -34,16 +34,26 @@ jlong Java_org_rocksdb_HashSkipListMemTableConfig_newMemTableFactoryHandle(
/*
* Class: org_rocksdb_HashLinkedListMemTableConfig
* Method: newMemTableFactoryHandle
* Signature: (J)J
* Signature: (JJIZI)J
*/
jlong Java_org_rocksdb_HashLinkedListMemTableConfig_newMemTableFactoryHandle(
JNIEnv* env, jobject jobj, jlong jbucket_count) {
rocksdb::Status s = rocksdb::check_if_jlong_fits_size_t(jbucket_count);
if (s.ok()) {
JNIEnv* env, jobject jobj, jlong jbucket_count, jlong jhuge_page_tlb_size,
jint jbucket_entries_logging_threshold,
jboolean jif_log_bucket_dist_when_flash, jint jthreshold_use_skiplist) {
rocksdb::Status statusBucketCount =
rocksdb::check_if_jlong_fits_size_t(jbucket_count);
rocksdb::Status statusHugePageTlb =
rocksdb::check_if_jlong_fits_size_t(jhuge_page_tlb_size);
if (statusBucketCount.ok() && statusHugePageTlb.ok()) {
return reinterpret_cast<jlong>(rocksdb::NewHashLinkListRepFactory(
static_cast<size_t>(jbucket_count)));
static_cast<size_t>(jbucket_count),
static_cast<size_t>(jhuge_page_tlb_size),
static_cast<int32_t>(jbucket_entries_logging_threshold),
static_cast<bool>(jif_log_bucket_dist_when_flash),
static_cast<int32_t>(jthreshold_use_skiplist)));
}
rocksdb::RocksDBExceptionJni::ThrowNew(env, s);
rocksdb::RocksDBExceptionJni::ThrowNew(env,
!statusBucketCount.ok()?statusBucketCount:statusHugePageTlb);
return 0;
}
@ -66,9 +76,15 @@ jlong Java_org_rocksdb_VectorMemTableConfig_newMemTableFactoryHandle(
/*
* Class: org_rocksdb_SkipListMemTableConfig
* Method: newMemTableFactoryHandle0
* Signature: ()J
* Signature: (J)J
*/
jlong Java_org_rocksdb_SkipListMemTableConfig_newMemTableFactoryHandle0(
JNIEnv* env, jobject jobj) {
return reinterpret_cast<jlong>(new rocksdb::SkipListFactory());
JNIEnv* env, jobject jobj, jlong jlookahead) {
rocksdb::Status s = rocksdb::check_if_jlong_fits_size_t(jlookahead);
if (s.ok()) {
return reinterpret_cast<jlong>(new rocksdb::SkipListFactory(
static_cast<size_t>(jlookahead)));
}
rocksdb::RocksDBExceptionJni::ThrowNew(env, s);
return 0;
}