From 9aa9668a83cfebe865212a5468cc747fc800872c Mon Sep 17 00:00:00 2001 From: fyrz Date: Thu, 16 Oct 2014 20:39:26 +0200 Subject: [PATCH] [RocksJava] Memtables update to 3.6 - Adjusted HashLinkedList to 3.6.0 - Adjusted SkipList to 3.6.0 - Introduced a memtable test --- java/Makefile | 1 + .../rocksdb/HashLinkedListMemTableConfig.java | 126 +++++++++++++++++- .../rocksdb/HashSkipListMemTableConfig.java | 3 + java/org/rocksdb/SkipListMemTableConfig.java | 38 +++++- java/org/rocksdb/VectorMemTableConfig.java | 4 + java/org/rocksdb/test/MemTableTest.java | 107 +++++++++++++++ java/rocksjni/memtablejni.cc | 34 +++-- 7 files changed, 298 insertions(+), 15 deletions(-) create mode 100644 java/org/rocksdb/test/MemTableTest.java diff --git a/java/Makefile b/java/Makefile index 697df5175..504060bc3 100644 --- a/java/Makefile +++ b/java/Makefile @@ -42,6 +42,7 @@ test: java java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.ColumnFamilyTest java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.FilterTest java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.KeyMayExistTest + java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.MemTableTest java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.OptionsTest java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.ReadOnlyTest java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.ReadOptionsTest diff --git a/java/org/rocksdb/HashLinkedListMemTableConfig.java b/java/org/rocksdb/HashLinkedListMemTableConfig.java index 381a16f49..78a4e8661 100644 --- a/java/org/rocksdb/HashLinkedListMemTableConfig.java +++ b/java/org/rocksdb/HashLinkedListMemTableConfig.java @@ -15,9 +15,21 @@ package org.rocksdb; */ public class HashLinkedListMemTableConfig extends MemTableConfig { public static final long DEFAULT_BUCKET_COUNT = 50000; + public static final long DEFAULT_HUGE_PAGE_TLB_SIZE = 0; + public static final int DEFAULT_BUCKET_ENTRIES_LOG_THRES = 4096; + public static final boolean + DEFAULT_IF_LOG_BUCKET_DIST_WHEN_FLUSH = true; + public static final int DEFAUL_THRESHOLD_USE_SKIPLIST = 256; + /** + * HashLinkedListMemTableConfig constructor + */ public HashLinkedListMemTableConfig() { bucketCount_ = DEFAULT_BUCKET_COUNT; + hugePageTlbSize_ = DEFAULT_HUGE_PAGE_TLB_SIZE; + bucketEntriesLoggingThreshold_ = DEFAULT_BUCKET_ENTRIES_LOG_THRES; + ifLogBucketDistWhenFlush_ = DEFAULT_IF_LOG_BUCKET_DIST_WHEN_FLUSH; + thresholdUseSkiplist_ = DEFAUL_THRESHOLD_USE_SKIPLIST; } /** @@ -42,13 +54,119 @@ public class HashLinkedListMemTableConfig extends MemTableConfig { return bucketCount_; } - @Override protected long newMemTableFactoryHandle() - throws RocksDBException { - return newMemTableFactoryHandle(bucketCount_); + /** + *

Set the size of huge tlb or allocate the hashtable bytes from + * malloc if {@code size <= 0}.

+ * + *

The user needs to reserve huge pages for it to be allocated, + * like: {@code sysctl -w vm.nr_hugepages=20}

+ * + *

See linux documentation/vm/hugetlbpage.txt

+ * + * @param size if set to {@code <= 0} hashtable bytes from malloc + * @return the reference to the current HashLinkedListMemTableConfig. + */ + public HashLinkedListMemTableConfig setHugePageTlbSize(long size) { + hugePageTlbSize_ = size; + return this; } - private native long newMemTableFactoryHandle(long bucketCount) + /** + * Returns the size value of hugePageTlbSize. + * + * @return the hugePageTlbSize. + */ + public long hugePageTlbSize() { + return hugePageTlbSize_; + } + + /** + * If number of entries in one bucket exceeds that setting, log + * about it. + * + * @param threshold - number of entries in a single bucket before + * logging starts. + * @return the reference to the current HashLinkedListMemTableConfig. + */ + public HashLinkedListMemTableConfig + setBucketEntriesLoggingThreshold(int threshold) { + bucketEntriesLoggingThreshold_ = threshold; + return this; + } + + /** + * Returns the maximum number of entries in one bucket before + * logging starts. + * + * @return maximum number of entries in one bucket before logging + * starts. + */ + public int bucketEntriesLoggingThreshold() { + return bucketEntriesLoggingThreshold_; + } + + /** + * If true the distrubition of number of entries will be logged. + * + * @param logDistribution - boolean parameter indicating if number + * of entry distribution shall be logged. + * @return the reference to the current HashLinkedListMemTableConfig. + */ + public HashLinkedListMemTableConfig + setIfLogBucketDistWhenFlush(boolean logDistribution) { + ifLogBucketDistWhenFlush_ = logDistribution; + return this; + } + + /** + * Returns information about logging the distribution of + * number of entries on flush. + * + * @return if distrubtion of number of entries shall be logged. + */ + public boolean ifLogBucketDistWhenFlush() { + return ifLogBucketDistWhenFlush_; + } + + /** + * Set maximum number of entries in one bucket. Exceeding this val + * leads to a switch from LinkedList to SkipList. + * + * @param threshold maximum number of entries before SkipList is + * used. + * @return the reference to the current HashLinkedListMemTableConfig. + */ + public HashLinkedListMemTableConfig + setThresholdUseSkiplist(int threshold) { + thresholdUseSkiplist_ = threshold; + return this; + } + + /** + * Returns entries per bucket threshold before LinkedList is + * replaced by SkipList usage for that bucket. + * + * @return entries per bucket threshold before SkipList is used. + */ + public int thresholdUseSkiplist() { + return thresholdUseSkiplist_; + } + + @Override protected long newMemTableFactoryHandle() + throws RocksDBException { + return newMemTableFactoryHandle(bucketCount_, hugePageTlbSize_, + bucketEntriesLoggingThreshold_, ifLogBucketDistWhenFlush_, + thresholdUseSkiplist_); + } + + private native long newMemTableFactoryHandle(long bucketCount, + long hugePageTlbSize, int bucketEntriesLoggingThreshold, + boolean ifLogBucketDistWhenFlush, int thresholdUseSkiplist) throws RocksDBException; private long bucketCount_; + private long hugePageTlbSize_; + private int bucketEntriesLoggingThreshold_; + private boolean ifLogBucketDistWhenFlush_; + private int thresholdUseSkiplist_; } diff --git a/java/org/rocksdb/HashSkipListMemTableConfig.java b/java/org/rocksdb/HashSkipListMemTableConfig.java index 100f16c82..ad2120f18 100644 --- a/java/org/rocksdb/HashSkipListMemTableConfig.java +++ b/java/org/rocksdb/HashSkipListMemTableConfig.java @@ -18,6 +18,9 @@ public class HashSkipListMemTableConfig extends MemTableConfig { public static final int DEFAULT_BRANCHING_FACTOR = 4; public static final int DEFAULT_HEIGHT = 4; + /** + * HashSkipListMemTableConfig constructor + */ public HashSkipListMemTableConfig() { bucketCount_ = DEFAULT_BUCKET_COUNT; branchingFactor_ = DEFAULT_BRANCHING_FACTOR; diff --git a/java/org/rocksdb/SkipListMemTableConfig.java b/java/org/rocksdb/SkipListMemTableConfig.java index 7f9f5cb5f..d26fd9d32 100644 --- a/java/org/rocksdb/SkipListMemTableConfig.java +++ b/java/org/rocksdb/SkipListMemTableConfig.java @@ -4,12 +4,46 @@ package org.rocksdb; * The config for skip-list memtable representation. */ public class SkipListMemTableConfig extends MemTableConfig { + + public static final long DEFAULT_LOOKAHEAD = 0; + + /** + * SkipListMemTableConfig constructor + */ public SkipListMemTableConfig() { + lookahead_ = DEFAULT_LOOKAHEAD; } + /** + * Sets lookahead for SkipList + * + * @param lookahead If non-zero, each iterator's seek operation + * will start the search from the previously visited record + * (doing at most 'lookahead' steps). This is an + * optimization for the access pattern including many + * seeks with consecutive keys. + * @return the current instance of SkipListMemTableConfig + */ + public SkipListMemTableConfig setLookahead(long lookahead) { + lookahead_ = lookahead; + return this; + } + + /** + * Returns the currently set lookahead value. + * + * @return lookahead value + */ + public long lookahead() { + return lookahead_; + } + + @Override protected long newMemTableFactoryHandle() { - return newMemTableFactoryHandle0(); + return newMemTableFactoryHandle0(lookahead_); } - private native long newMemTableFactoryHandle0(); + private native long newMemTableFactoryHandle0(long lookahead); + + private long lookahead_; } diff --git a/java/org/rocksdb/VectorMemTableConfig.java b/java/org/rocksdb/VectorMemTableConfig.java index b7a413f19..ba1be3e77 100644 --- a/java/org/rocksdb/VectorMemTableConfig.java +++ b/java/org/rocksdb/VectorMemTableConfig.java @@ -5,6 +5,10 @@ package org.rocksdb; */ public class VectorMemTableConfig extends MemTableConfig { public static final int DEFAULT_RESERVED_SIZE = 0; + + /** + * VectorMemTableConfig constructor + */ public VectorMemTableConfig() { reservedSize_ = DEFAULT_RESERVED_SIZE; } diff --git a/java/org/rocksdb/test/MemTableTest.java b/java/org/rocksdb/test/MemTableTest.java new file mode 100644 index 000000000..0d1e4d54a --- /dev/null +++ b/java/org/rocksdb/test/MemTableTest.java @@ -0,0 +1,107 @@ +// Copyright (c) 2014, Facebook, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +package org.rocksdb.test; + +import org.rocksdb.*; + +public class MemTableTest { + static { + RocksDB.loadLibrary(); + } + public static void main(String[] args) { + Options options = new Options(); + // Test HashSkipListMemTableConfig + HashSkipListMemTableConfig memTableConfig = + new HashSkipListMemTableConfig(); + assert(memTableConfig.bucketCount() == 1000000); + memTableConfig.setBucketCount(2000000); + assert(memTableConfig.bucketCount() == 2000000); + assert(memTableConfig.height() == 4); + memTableConfig.setHeight(5); + assert(memTableConfig.height() == 5); + assert(memTableConfig.branchingFactor() == 4); + memTableConfig.setBranchingFactor(6); + assert(memTableConfig.branchingFactor() == 6); + try { + options.setMemTableConfig(memTableConfig); + } catch (RocksDBException e) { + assert(false); + } + memTableConfig = null; + options.dispose(); + System.gc(); + System.runFinalization(); + // Test SkipList + options = new Options(); + SkipListMemTableConfig skipMemTableConfig = + new SkipListMemTableConfig(); + assert(skipMemTableConfig.lookahead() == 0); + skipMemTableConfig.setLookahead(20); + assert(skipMemTableConfig.lookahead() == 20); + try { + options.setMemTableConfig(skipMemTableConfig); + } catch (RocksDBException e) { + assert(false); + } + skipMemTableConfig = null; + options.dispose(); + System.gc(); + System.runFinalization(); + // Test HashLinkedListMemTableConfig + options = new Options(); + HashLinkedListMemTableConfig hashLinkedListMemTableConfig = + new HashLinkedListMemTableConfig(); + assert(hashLinkedListMemTableConfig.bucketCount() == 50000); + hashLinkedListMemTableConfig.setBucketCount(100000); + assert(hashLinkedListMemTableConfig.bucketCount() == 100000); + assert(hashLinkedListMemTableConfig.hugePageTlbSize() == 0); + hashLinkedListMemTableConfig.setHugePageTlbSize(1); + assert(hashLinkedListMemTableConfig.hugePageTlbSize() == 1); + assert(hashLinkedListMemTableConfig. + bucketEntriesLoggingThreshold() == 4096); + hashLinkedListMemTableConfig. + setBucketEntriesLoggingThreshold(200); + assert(hashLinkedListMemTableConfig. + bucketEntriesLoggingThreshold() == 200); + assert(hashLinkedListMemTableConfig. + ifLogBucketDistWhenFlush() == true); + hashLinkedListMemTableConfig. + setIfLogBucketDistWhenFlush(false); + assert(hashLinkedListMemTableConfig. + ifLogBucketDistWhenFlush() == false); + assert(hashLinkedListMemTableConfig. + thresholdUseSkiplist() == 256); + hashLinkedListMemTableConfig.setThresholdUseSkiplist(29); + assert(hashLinkedListMemTableConfig. + thresholdUseSkiplist() == 29); + try { + options.setMemTableConfig(hashLinkedListMemTableConfig); + } catch (RocksDBException e) { + assert(false); + } + hashLinkedListMemTableConfig = null; + options.dispose(); + System.gc(); + System.runFinalization(); + // test VectorMemTableConfig + options = new Options(); + VectorMemTableConfig vectorMemTableConfig = + new VectorMemTableConfig(); + assert(vectorMemTableConfig.reservedSize() == 0); + vectorMemTableConfig.setReservedSize(123); + assert(vectorMemTableConfig.reservedSize() == 123); + try { + options.setMemTableConfig(vectorMemTableConfig); + } catch (RocksDBException e) { + assert(false); + } + vectorMemTableConfig = null; + options.dispose(); + System.gc(); + System.runFinalization(); + System.out.println("Mem-table test passed"); + } +} diff --git a/java/rocksjni/memtablejni.cc b/java/rocksjni/memtablejni.cc index 4be03d491..fe83885c2 100644 --- a/java/rocksjni/memtablejni.cc +++ b/java/rocksjni/memtablejni.cc @@ -34,16 +34,26 @@ jlong Java_org_rocksdb_HashSkipListMemTableConfig_newMemTableFactoryHandle( /* * Class: org_rocksdb_HashLinkedListMemTableConfig * Method: newMemTableFactoryHandle - * Signature: (J)J + * Signature: (JJIZI)J */ jlong Java_org_rocksdb_HashLinkedListMemTableConfig_newMemTableFactoryHandle( - JNIEnv* env, jobject jobj, jlong jbucket_count) { - rocksdb::Status s = rocksdb::check_if_jlong_fits_size_t(jbucket_count); - if (s.ok()) { + JNIEnv* env, jobject jobj, jlong jbucket_count, jlong jhuge_page_tlb_size, + jint jbucket_entries_logging_threshold, + jboolean jif_log_bucket_dist_when_flash, jint jthreshold_use_skiplist) { + rocksdb::Status statusBucketCount = + rocksdb::check_if_jlong_fits_size_t(jbucket_count); + rocksdb::Status statusHugePageTlb = + rocksdb::check_if_jlong_fits_size_t(jhuge_page_tlb_size); + if (statusBucketCount.ok() && statusHugePageTlb.ok()) { return reinterpret_cast(rocksdb::NewHashLinkListRepFactory( - static_cast(jbucket_count))); + static_cast(jbucket_count), + static_cast(jhuge_page_tlb_size), + static_cast(jbucket_entries_logging_threshold), + static_cast(jif_log_bucket_dist_when_flash), + static_cast(jthreshold_use_skiplist))); } - rocksdb::RocksDBExceptionJni::ThrowNew(env, s); + rocksdb::RocksDBExceptionJni::ThrowNew(env, + !statusBucketCount.ok()?statusBucketCount:statusHugePageTlb); return 0; } @@ -66,9 +76,15 @@ jlong Java_org_rocksdb_VectorMemTableConfig_newMemTableFactoryHandle( /* * Class: org_rocksdb_SkipListMemTableConfig * Method: newMemTableFactoryHandle0 - * Signature: ()J + * Signature: (J)J */ jlong Java_org_rocksdb_SkipListMemTableConfig_newMemTableFactoryHandle0( - JNIEnv* env, jobject jobj) { - return reinterpret_cast(new rocksdb::SkipListFactory()); + JNIEnv* env, jobject jobj, jlong jlookahead) { + rocksdb::Status s = rocksdb::check_if_jlong_fits_size_t(jlookahead); + if (s.ok()) { + return reinterpret_cast(new rocksdb::SkipListFactory( + static_cast(jlookahead))); + } + rocksdb::RocksDBExceptionJni::ThrowNew(env, s); + return 0; }