From 12350115da0e3b7b10c760d9fa54ce3d21642635 Mon Sep 17 00:00:00 2001 From: fyrz Date: Wed, 11 Mar 2015 21:50:10 +0100 Subject: [PATCH] [RocksJava] Added LevelCompactionDynamicLevelBytes to Options Summary: Added LevelCompactionDynamicLevelBytes to Options. Test Plan: make clean jclean rocksdbjava jtest mvn -f rocksjni.pom package Reviewers: adamretter, ankgup87, yhchiang Subscribers: dhruba Differential Revision: https://reviews.facebook.net/D34857 --- java/rocksjni/options.cc | 48 +++++++++++ .../java/org/rocksdb/ColumnFamilyOptions.java | 17 ++++ .../rocksdb/ColumnFamilyOptionsInterface.java | 85 ++++++++++++++++++- java/src/main/java/org/rocksdb/Options.java | 21 ++++- .../org/rocksdb/ColumnFamilyOptionsTest.java | 16 ++++ .../test/java/org/rocksdb/OptionsTest.java | 16 ++++ 6 files changed, 200 insertions(+), 3 deletions(-) diff --git a/java/rocksjni/options.cc b/java/rocksjni/options.cc index 07aadeab2..58da3632c 100644 --- a/java/rocksjni/options.cc +++ b/java/rocksjni/options.cc @@ -1264,6 +1264,30 @@ void Java_org_rocksdb_Options_setMaxBytesForLevelBase( static_cast(jmax_bytes_for_level_base); } +/* + * Class: org_rocksdb_Options + * Method: levelCompactionDynamicLevelBytes + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_Options_levelCompactionDynamicLevelBytes( + JNIEnv* env, jobject jobj, jlong jhandle) { + return reinterpret_cast( + jhandle)->level_compaction_dynamic_level_bytes; +} + +/* + * Class: org_rocksdb_Options + * Method: setLevelCompactionDynamicLevelBytes + * Signature: (JZ)V + */ +void Java_org_rocksdb_Options_setLevelCompactionDynamicLevelBytes( + JNIEnv* env, jobject jobj, jlong jhandle, + jboolean jenable_dynamic_level_bytes) { + reinterpret_cast( + jhandle)->level_compaction_dynamic_level_bytes = + (jenable_dynamic_level_bytes); +} + /* * Class: org_rocksdb_Options * Method: maxBytesForLevelMultiplier @@ -2329,6 +2353,30 @@ void Java_org_rocksdb_ColumnFamilyOptions_setMaxBytesForLevelBase( static_cast(jmax_bytes_for_level_base); } +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: levelCompactionDynamicLevelBytes + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_ColumnFamilyOptions_levelCompactionDynamicLevelBytes( + JNIEnv* env, jobject jobj, jlong jhandle) { + return reinterpret_cast( + jhandle)->level_compaction_dynamic_level_bytes; +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setLevelCompactionDynamicLevelBytes + * Signature: (JZ)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setLevelCompactionDynamicLevelBytes( + JNIEnv* env, jobject jobj, jlong jhandle, + jboolean jenable_dynamic_level_bytes) { + reinterpret_cast( + jhandle)->level_compaction_dynamic_level_bytes = + (jenable_dynamic_level_bytes); +} + /* * Class: org_rocksdb_ColumnFamilyOptions * Method: maxBytesForLevelMultiplier diff --git a/java/src/main/java/org/rocksdb/ColumnFamilyOptions.java b/java/src/main/java/org/rocksdb/ColumnFamilyOptions.java index a449a0b75..06324745e 100644 --- a/java/src/main/java/org/rocksdb/ColumnFamilyOptions.java +++ b/java/src/main/java/org/rocksdb/ColumnFamilyOptions.java @@ -296,6 +296,19 @@ public class ColumnFamilyOptions extends RocksObject return maxBytesForLevelBase(nativeHandle_); } + @Override + public ColumnFamilyOptions setLevelCompactionDynamicLevelBytes( + final boolean enableLevelCompactionDynamicLevelBytes) { + setLevelCompactionDynamicLevelBytes(nativeHandle_, + enableLevelCompactionDynamicLevelBytes); + return this; + } + + @Override + public boolean levelCompactionDynamicLevelBytes() { + return levelCompactionDynamicLevelBytes(nativeHandle_); + } + @Override public ColumnFamilyOptions setMaxBytesForLevelMultiplier( final int multiplier) { @@ -667,6 +680,10 @@ public class ColumnFamilyOptions extends RocksObject private native void setMaxBytesForLevelBase( long handle, long maxBytesForLevelBase); private native long maxBytesForLevelBase(long handle); + private native void setLevelCompactionDynamicLevelBytes( + long handle, boolean enableLevelCompactionDynamicLevelBytes); + private native boolean levelCompactionDynamicLevelBytes( + long handle); private native void setMaxBytesForLevelMultiplier( long handle, int multiplier); private native int maxBytesForLevelMultiplier(long handle); diff --git a/java/src/main/java/org/rocksdb/ColumnFamilyOptionsInterface.java b/java/src/main/java/org/rocksdb/ColumnFamilyOptionsInterface.java index c79c8f9ba..182653277 100644 --- a/java/src/main/java/org/rocksdb/ColumnFamilyOptionsInterface.java +++ b/java/src/main/java/org/rocksdb/ColumnFamilyOptionsInterface.java @@ -435,11 +435,94 @@ public interface ColumnFamilyOptionsInterface { * and total file size for level-3 will be 2GB. * by default 'maxBytesForLevelBase' is 10MB. * - * @return the upper-bound of the total size of leve-1 files in bytes. + * @return the upper-bound of the total size of level-1 files + * in bytes. * @see #maxBytesForLevelMultiplier() */ long maxBytesForLevelBase(); + /** + *

If {@code true}, RocksDB will pick target size of each level + * dynamically. We will pick a base level b >= 1. L0 will be + * directly merged into level b, instead of always into level 1. + * Level 1 to b-1 need to be empty. We try to pick b and its target + * size so that

+ * + *
    + *
  1. target size is in the range of + * (max_bytes_for_level_base / max_bytes_for_level_multiplier, + * max_bytes_for_level_base]
    2. + *
  2. target size of the last level (level num_levels-1) equals to extra size + * of the level.
    3. + *
+ * + *

At the same time max_bytes_for_level_multiplier and + * max_bytes_for_level_multiplier_additional are still satisfied.

+ * + *

With this option on, from an empty DB, we make last level the base + * level, which means merging L0 data into the last level, until it exceeds + * max_bytes_for_level_base. And then we make the second last level to be + * base level, to start to merge L0 data to second last level, with its + * target size to be {@code 1/max_bytes_for_level_multiplier} of the last + * levels extra size. After the data accumulates more so that we need to + * move the base level to the third last one, and so on.

+ * + *

Example

+ *

For example, assume {@code max_bytes_for_level_multiplier=10}, + * {@code num_levels=6}, and {@code max_bytes_for_level_base=10MB}.

+ * + *

Target sizes of level 1 to 5 starts with:

+ * {@code [- - - - 10MB]} + *

with base level is level. Target sizes of level 1 to 4 are not applicable + * because they will not be used. + * Until the size of Level 5 grows to more than 10MB, say 11MB, we make + * base target to level 4 and now the targets looks like:

+ * {@code [- - - 1.1MB 11MB]} + *

While data are accumulated, size targets are tuned based on actual data + * of level 5. When level 5 has 50MB of data, the target is like:

+ * {@code [- - - 5MB 50MB]} + *

Until level 5's actual size is more than 100MB, say 101MB. Now if we + * keep level 4 to be the base level, its target size needs to be 10.1MB, + * which doesn't satisfy the target size range. So now we make level 3 + * the target size and the target sizes of the levels look like:

+ * {@code [- - 1.01MB 10.1MB 101MB]} + *

In the same way, while level 5 further grows, all levels' targets grow, + * like

+ * {@code [- - 5MB 50MB 500MB]} + *

Until level 5 exceeds 1000MB and becomes 1001MB, we make level 2 the + * base level and make levels' target sizes like this:

+ * {@code [- 1.001MB 10.01MB 100.1MB 1001MB]} + *

and go on...

+ * + *

By doing it, we give {@code max_bytes_for_level_multiplier} a priority + * against {@code max_bytes_for_level_base}, for a more predictable LSM tree + * shape. It is useful to limit worse case space amplification.

+ * + *

{@code max_bytes_for_level_multiplier_additional} is ignored with + * this flag on.

+ * + *

Turning this feature on or off for an existing DB can cause unexpected + * LSM tree structure so it's not recommended.

+ * + *

Caution: this option is experimental

+ * + *

Default: false

+ */ + Object setLevelCompactionDynamicLevelBytes( + boolean enableLevelCompactionDynamicLevelBytes); + + /** + *

Return if {@code LevelCompactionDynamicLevelBytes} is enabled. + *

+ * + *

For further information see + * {@link #setLevelCompactionDynamicLevelBytes(boolean)}

+ * + * @return boolean value indicating if + * {@code levelCompactionDynamicLevelBytes} is enabled. + */ + boolean levelCompactionDynamicLevelBytes(); + /** * The ratio between the total size of level-(L+1) files and the total * size of level-L files for all L. diff --git a/java/src/main/java/org/rocksdb/Options.java b/java/src/main/java/org/rocksdb/Options.java index b530bc795..2662020f3 100644 --- a/java/src/main/java/org/rocksdb/Options.java +++ b/java/src/main/java/org/rocksdb/Options.java @@ -790,17 +790,30 @@ public class Options extends RocksObject return this; } + @Override + public Options setMaxBytesForLevelBase(final long maxBytesForLevelBase) { + setMaxBytesForLevelBase(nativeHandle_, maxBytesForLevelBase); + return this; + } + @Override public long maxBytesForLevelBase() { return maxBytesForLevelBase(nativeHandle_); } @Override - public Options setMaxBytesForLevelBase(final long maxBytesForLevelBase) { - setMaxBytesForLevelBase(nativeHandle_, maxBytesForLevelBase); + public Options setLevelCompactionDynamicLevelBytes( + final boolean enableLevelCompactionDynamicLevelBytes) { + setLevelCompactionDynamicLevelBytes(nativeHandle_, + enableLevelCompactionDynamicLevelBytes); return this; } + @Override + public boolean levelCompactionDynamicLevelBytes() { + return levelCompactionDynamicLevelBytes(nativeHandle_); + } + @Override public int maxBytesForLevelMultiplier() { return maxBytesForLevelMultiplier(nativeHandle_); @@ -1223,6 +1236,10 @@ public class Options extends RocksObject private native void setMaxBytesForLevelBase( long handle, long maxBytesForLevelBase); private native long maxBytesForLevelBase(long handle); + private native void setLevelCompactionDynamicLevelBytes( + long handle, boolean enableLevelCompactionDynamicLevelBytes); + private native boolean levelCompactionDynamicLevelBytes( + long handle); private native void setMaxBytesForLevelMultiplier( long handle, int multiplier); private native int maxBytesForLevelMultiplier(long handle); diff --git a/java/src/test/java/org/rocksdb/ColumnFamilyOptionsTest.java b/java/src/test/java/org/rocksdb/ColumnFamilyOptionsTest.java index ad1a66617..8d3db188c 100644 --- a/java/src/test/java/org/rocksdb/ColumnFamilyOptionsTest.java +++ b/java/src/test/java/org/rocksdb/ColumnFamilyOptionsTest.java @@ -238,6 +238,22 @@ public class ColumnFamilyOptionsTest { } } + @Test + public void levelCompactionDynamicLevelBytes() { + ColumnFamilyOptions opt = null; + try { + opt = new ColumnFamilyOptions(); + boolean boolValue = rand.nextBoolean(); + opt.setLevelCompactionDynamicLevelBytes(boolValue); + assertThat(opt.levelCompactionDynamicLevelBytes()) + .isEqualTo(boolValue); + } finally { + if (opt != null) { + opt.dispose(); + } + } + } + @Test public void maxBytesForLevelMultiplier() { ColumnFamilyOptions opt = null; diff --git a/java/src/test/java/org/rocksdb/OptionsTest.java b/java/src/test/java/org/rocksdb/OptionsTest.java index 97da9cfb1..fd426479f 100644 --- a/java/src/test/java/org/rocksdb/OptionsTest.java +++ b/java/src/test/java/org/rocksdb/OptionsTest.java @@ -200,6 +200,22 @@ public class OptionsTest { } } + @Test + public void levelCompactionDynamicLevelBytes() { + Options opt = null; + try { + opt = new Options(); + boolean boolValue = rand.nextBoolean(); + opt.setLevelCompactionDynamicLevelBytes(boolValue); + assertThat(opt.levelCompactionDynamicLevelBytes()) + .isEqualTo(boolValue); + } finally { + if (opt != null) { + opt.dispose(); + } + } + } + @Test public void maxBytesForLevelMultiplier() { Options opt = null;