Merge pull request #354 from fyrz/RocksJava-memtables-3.6

[RocksJava] Memtables update to 3.6

java/Makefile

@@ -42,6 +42,7 @@ test: java
 	java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.ColumnFamilyTest
 	java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.FilterTest
 	java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.KeyMayExistTest
+	java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.MemTableTest
 	java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.OptionsTest
 	java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.ReadOnlyTest
 	java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.ReadOptionsTest

java/org/rocksdb/HashLinkedListMemTableConfig.java

@@ -15,9 +15,21 @@ package org.rocksdb;
  */
 public class HashLinkedListMemTableConfig extends MemTableConfig {
   public static final long DEFAULT_BUCKET_COUNT = 50000;
+  public static final long DEFAULT_HUGE_PAGE_TLB_SIZE = 0;
+  public static final int DEFAULT_BUCKET_ENTRIES_LOG_THRES = 4096;
+  public static final boolean
+      DEFAULT_IF_LOG_BUCKET_DIST_WHEN_FLUSH = true;
+  public static final int DEFAUL_THRESHOLD_USE_SKIPLIST = 256;
 
+  /**
+   * HashLinkedListMemTableConfig constructor
+   */
   public HashLinkedListMemTableConfig() {
     bucketCount_ = DEFAULT_BUCKET_COUNT;
+    hugePageTlbSize_ = DEFAULT_HUGE_PAGE_TLB_SIZE;
+    bucketEntriesLoggingThreshold_ = DEFAULT_BUCKET_ENTRIES_LOG_THRES;
+    ifLogBucketDistWhenFlush_ = DEFAULT_IF_LOG_BUCKET_DIST_WHEN_FLUSH;
+    thresholdUseSkiplist_ = DEFAUL_THRESHOLD_USE_SKIPLIST;
   }
 
   /**
@@ -42,13 +54,119 @@ public class HashLinkedListMemTableConfig extends MemTableConfig {
     return bucketCount_;
   }
 
-  @Override protected long newMemTableFactoryHandle()
-       throws RocksDBException {
-    return newMemTableFactoryHandle(bucketCount_);
+  /**
+   * <p>Set the size of huge tlb or allocate the hashtable bytes from
+   * malloc if {@code size <= 0}.</p>
+   *
+   * <p>The user needs to reserve huge pages for it to be allocated,
+   * like: {@code sysctl -w vm.nr_hugepages=20}</p>
+   *
+   * <p>See linux documentation/vm/hugetlbpage.txt</p>
+   *
+   * @param size if set to {@code <= 0} hashtable bytes from malloc
+   * @return the reference to the current HashLinkedListMemTableConfig.
+   */
+  public HashLinkedListMemTableConfig setHugePageTlbSize(long size) {
+    hugePageTlbSize_ = size;
+    return this;
   }
 
-  private native long newMemTableFactoryHandle(long bucketCount)
+  /**
+   * Returns the size value of hugePageTlbSize.
+   *
+   * @return the hugePageTlbSize.
+   */
+  public long hugePageTlbSize() {
+    return hugePageTlbSize_;
+  }
+
+  /**
+   * If number of entries in one bucket exceeds that setting, log
+   * about it.
+   *
+   * @param threshold - number of entries in a single bucket before
+   *     logging starts.
+   * @return the reference to the current HashLinkedListMemTableConfig.
+   */
+  public HashLinkedListMemTableConfig
+      setBucketEntriesLoggingThreshold(int threshold) {
+    bucketEntriesLoggingThreshold_ = threshold;
+    return this;
+  }
+
+  /**
+   * Returns the maximum number of entries in one bucket before
+   * logging starts.
+   *
+   * @return maximum number of entries in one bucket before logging
+   *     starts.
+   */
+  public int bucketEntriesLoggingThreshold() {
+    return bucketEntriesLoggingThreshold_;
+  }
+
+  /**
+   * If true the distrubition of number of entries will be logged.
+   *
+   * @param logDistribution - boolean parameter indicating if number
+   *     of entry distribution shall be logged.
+   * @return the reference to the current HashLinkedListMemTableConfig.
+   */
+  public HashLinkedListMemTableConfig
+      setIfLogBucketDistWhenFlush(boolean logDistribution) {
+    ifLogBucketDistWhenFlush_ = logDistribution;
+    return this;
+  }
+
+  /**
+   * Returns information about logging the distribution of
+   *  number of entries on flush.
+   *
+   * @return if distrubtion of number of entries shall be logged.
+   */
+  public boolean ifLogBucketDistWhenFlush() {
+    return ifLogBucketDistWhenFlush_;
+  }
+
+  /**
+   * Set maximum number of entries in one bucket. Exceeding this val
+   * leads to a switch from LinkedList to SkipList.
+   *
+   * @param threshold maximum number of entries before SkipList is
+   *     used.
+   * @return the reference to the current HashLinkedListMemTableConfig.
+   */
+  public HashLinkedListMemTableConfig
+      setThresholdUseSkiplist(int threshold) {
+    thresholdUseSkiplist_ = threshold;
+    return this;
+  }
+
+  /**
+   * Returns entries per bucket threshold before LinkedList is
+   * replaced by SkipList usage for that bucket.
+   *
+   * @return entries per bucket threshold before SkipList is used.
+   */
+  public int thresholdUseSkiplist() {
+    return thresholdUseSkiplist_;
+  }
+
+  @Override protected long newMemTableFactoryHandle()
+       throws RocksDBException {
+    return newMemTableFactoryHandle(bucketCount_, hugePageTlbSize_,
+        bucketEntriesLoggingThreshold_, ifLogBucketDistWhenFlush_,
+        thresholdUseSkiplist_);
+  }
+
+  private native long newMemTableFactoryHandle(long bucketCount,
+      long hugePageTlbSize, int bucketEntriesLoggingThreshold,
+      boolean ifLogBucketDistWhenFlush, int thresholdUseSkiplist)
       throws RocksDBException;
 
   private long bucketCount_;
+  private long hugePageTlbSize_;
+  private int bucketEntriesLoggingThreshold_;
+  private boolean ifLogBucketDistWhenFlush_;
+  private int thresholdUseSkiplist_;
 }

java/org/rocksdb/HashSkipListMemTableConfig.java

@@ -18,6 +18,9 @@ public class HashSkipListMemTableConfig extends MemTableConfig {
   public static final int DEFAULT_BRANCHING_FACTOR = 4;
   public static final int DEFAULT_HEIGHT = 4;
 
+  /**
+   * HashSkipListMemTableConfig constructor
+   */
   public HashSkipListMemTableConfig() {
     bucketCount_ = DEFAULT_BUCKET_COUNT;
     branchingFactor_ = DEFAULT_BRANCHING_FACTOR;

java/org/rocksdb/SkipListMemTableConfig.java

@@ -4,12 +4,46 @@ package org.rocksdb;
  * The config for skip-list memtable representation.
  */
 public class SkipListMemTableConfig extends MemTableConfig {
+
+  public static final long DEFAULT_LOOKAHEAD = 0;
+
+  /**
+   * SkipListMemTableConfig constructor
+   */
   public SkipListMemTableConfig() {
+    lookahead_ = DEFAULT_LOOKAHEAD;
   }
 
+  /**
+   * Sets lookahead for SkipList
+   *
+   * @param lookahead If non-zero, each iterator's seek operation
+   *     will start the search from the previously visited record
+   *     (doing at most 'lookahead' steps). This is an
+   *     optimization for the access pattern including many
+   *     seeks with consecutive keys.
+   * @return the current instance of SkipListMemTableConfig
+   */
+  public SkipListMemTableConfig setLookahead(long lookahead) {
+    lookahead_ = lookahead;
+    return this;
+  }
+
+  /**
+   * Returns the currently set lookahead value.
+   *
+   * @return lookahead value
+   */
+  public long lookahead() {
+    return lookahead_;
+  }
+
+
   @Override protected long newMemTableFactoryHandle() {
-    return newMemTableFactoryHandle0();
+    return newMemTableFactoryHandle0(lookahead_);
   }
 
-  private native long newMemTableFactoryHandle0();
+  private native long newMemTableFactoryHandle0(long lookahead);
+
+  private long lookahead_;
 }

java/org/rocksdb/VectorMemTableConfig.java

@@ -5,6 +5,10 @@ package org.rocksdb;
  */
 public class VectorMemTableConfig extends MemTableConfig {
   public static final int DEFAULT_RESERVED_SIZE = 0;
+
+  /**
+   * VectorMemTableConfig constructor
+   */
   public VectorMemTableConfig() {
     reservedSize_ = DEFAULT_RESERVED_SIZE;
   }

java/org/rocksdb/test/MemTableTest.java

@@ -0,0 +1,107 @@
+// Copyright (c) 2014, Facebook, Inc.  All rights reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+
+package org.rocksdb.test;
+
+import org.rocksdb.*;
+
+public class MemTableTest {
+  static {
+    RocksDB.loadLibrary();
+  }
+  public static void main(String[] args) {
+    Options options = new Options();
+    // Test HashSkipListMemTableConfig
+    HashSkipListMemTableConfig memTableConfig =
+        new HashSkipListMemTableConfig();
+    assert(memTableConfig.bucketCount() == 1000000);
+    memTableConfig.setBucketCount(2000000);
+    assert(memTableConfig.bucketCount() == 2000000);
+    assert(memTableConfig.height() == 4);
+    memTableConfig.setHeight(5);
+    assert(memTableConfig.height() == 5);
+    assert(memTableConfig.branchingFactor() == 4);
+    memTableConfig.setBranchingFactor(6);
+    assert(memTableConfig.branchingFactor() == 6);
+    try {
+      options.setMemTableConfig(memTableConfig);
+    } catch (RocksDBException e) {
+      assert(false);
+    }
+    memTableConfig = null;
+    options.dispose();
+    System.gc();
+    System.runFinalization();
+    // Test SkipList
+    options = new Options();
+    SkipListMemTableConfig skipMemTableConfig =
+        new SkipListMemTableConfig();
+    assert(skipMemTableConfig.lookahead() == 0);
+    skipMemTableConfig.setLookahead(20);
+    assert(skipMemTableConfig.lookahead() == 20);
+    try {
+      options.setMemTableConfig(skipMemTableConfig);
+    } catch (RocksDBException e) {
+      assert(false);
+    }
+    skipMemTableConfig = null;
+    options.dispose();
+    System.gc();
+    System.runFinalization();
+    // Test HashLinkedListMemTableConfig
+    options = new Options();
+    HashLinkedListMemTableConfig hashLinkedListMemTableConfig =
+        new HashLinkedListMemTableConfig();
+    assert(hashLinkedListMemTableConfig.bucketCount() == 50000);
+    hashLinkedListMemTableConfig.setBucketCount(100000);
+    assert(hashLinkedListMemTableConfig.bucketCount() == 100000);
+    assert(hashLinkedListMemTableConfig.hugePageTlbSize() == 0);
+    hashLinkedListMemTableConfig.setHugePageTlbSize(1);
+    assert(hashLinkedListMemTableConfig.hugePageTlbSize() == 1);
+    assert(hashLinkedListMemTableConfig.
+       bucketEntriesLoggingThreshold() == 4096);
+    hashLinkedListMemTableConfig.
+        setBucketEntriesLoggingThreshold(200);
+    assert(hashLinkedListMemTableConfig.
+       bucketEntriesLoggingThreshold() == 200);
+    assert(hashLinkedListMemTableConfig.
+        ifLogBucketDistWhenFlush() == true);
+    hashLinkedListMemTableConfig.
+        setIfLogBucketDistWhenFlush(false);
+    assert(hashLinkedListMemTableConfig.
+        ifLogBucketDistWhenFlush() == false);
+    assert(hashLinkedListMemTableConfig.
+        thresholdUseSkiplist() == 256);
+    hashLinkedListMemTableConfig.setThresholdUseSkiplist(29);
+    assert(hashLinkedListMemTableConfig.
+        thresholdUseSkiplist() == 29);
+    try {
+      options.setMemTableConfig(hashLinkedListMemTableConfig);
+    } catch (RocksDBException e) {
+      assert(false);
+    }
+    hashLinkedListMemTableConfig = null;
+    options.dispose();
+    System.gc();
+    System.runFinalization();
+    // test VectorMemTableConfig
+    options = new Options();
+    VectorMemTableConfig vectorMemTableConfig =
+        new VectorMemTableConfig();
+    assert(vectorMemTableConfig.reservedSize() == 0);
+    vectorMemTableConfig.setReservedSize(123);
+    assert(vectorMemTableConfig.reservedSize() == 123);
+    try {
+      options.setMemTableConfig(vectorMemTableConfig);
+    } catch (RocksDBException e) {
+      assert(false);
+    }
+    vectorMemTableConfig = null;
+    options.dispose();
+    System.gc();
+    System.runFinalization();
+    System.out.println("Mem-table test passed");
+  }
+}

java/rocksjni/memtablejni.cc

@@ -34,16 +34,26 @@ jlong Java_org_rocksdb_HashSkipListMemTableConfig_newMemTableFactoryHandle(
 /*
  * Class:     org_rocksdb_HashLinkedListMemTableConfig
  * Method:    newMemTableFactoryHandle
- * Signature: (J)J
+ * Signature: (JJIZI)J
  */
 jlong Java_org_rocksdb_HashLinkedListMemTableConfig_newMemTableFactoryHandle(
-    JNIEnv* env, jobject jobj, jlong jbucket_count) {
-  rocksdb::Status s = rocksdb::check_if_jlong_fits_size_t(jbucket_count);
-  if (s.ok()) {
+    JNIEnv* env, jobject jobj, jlong jbucket_count, jlong jhuge_page_tlb_size,
+    jint jbucket_entries_logging_threshold,
+    jboolean jif_log_bucket_dist_when_flash, jint jthreshold_use_skiplist) {
+  rocksdb::Status statusBucketCount =
+      rocksdb::check_if_jlong_fits_size_t(jbucket_count);
+  rocksdb::Status statusHugePageTlb =
+      rocksdb::check_if_jlong_fits_size_t(jhuge_page_tlb_size);
+  if (statusBucketCount.ok() && statusHugePageTlb.ok()) {
     return reinterpret_cast<jlong>(rocksdb::NewHashLinkListRepFactory(
-        static_cast<size_t>(jbucket_count)));
+        static_cast<size_t>(jbucket_count),
+        static_cast<size_t>(jhuge_page_tlb_size),
+        static_cast<int32_t>(jbucket_entries_logging_threshold),
+        static_cast<bool>(jif_log_bucket_dist_when_flash),
+        static_cast<int32_t>(jthreshold_use_skiplist)));
   }
-  rocksdb::RocksDBExceptionJni::ThrowNew(env, s);
+  rocksdb::RocksDBExceptionJni::ThrowNew(env,
+      !statusBucketCount.ok()?statusBucketCount:statusHugePageTlb);
   return 0;
 }
 
@@ -66,9 +76,15 @@ jlong Java_org_rocksdb_VectorMemTableConfig_newMemTableFactoryHandle(
 /*
  * Class:     org_rocksdb_SkipListMemTableConfig
  * Method:    newMemTableFactoryHandle0
- * Signature: ()J
+ * Signature: (J)J
  */
 jlong Java_org_rocksdb_SkipListMemTableConfig_newMemTableFactoryHandle0(
-    JNIEnv* env, jobject jobj) {
-  return reinterpret_cast<jlong>(new rocksdb::SkipListFactory());
+    JNIEnv* env, jobject jobj, jlong jlookahead) {
+  rocksdb::Status s = rocksdb::check_if_jlong_fits_size_t(jlookahead);
+  if (s.ok()) {
+    return reinterpret_cast<jlong>(new rocksdb::SkipListFactory(
+        static_cast<size_t>(jlookahead)));
+  }
+  rocksdb::RocksDBExceptionJni::ThrowNew(env, s);
+  return 0;
 }