andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Java wrapper for blob_gc_force_threshold as blobGarbageCollectionForceThreshold (#9109)

Browse Source 
Summary:
Extra option added as a supplement to https://github.com/facebook/rocksdb/pull/8999

Closes https://github.com/facebook/rocksdb/issues/8221

Pull Request resolved: https://github.com/facebook/rocksdb/pull/9109

Reviewed By: mrambacher

Differential Revision: D32065039

Pulled By: ltamasi

fbshipit-source-id: 6c484050a30fe0523850a8a3c95dc85b0a501362
This commit is contained in:
Alan Paxton 2021-11-01 11:58:03 -07:00 committed by Facebook GitHub Bot
parent 2b70224f82
commit 73e6b89fad
8 changed files with 171 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
java/rocksjni/options.cc
View File

@ -3837,6 +3837,30 @@ jdouble Java_org_rocksdb_Options_blobGarbageCollectionAgeCutoff(JNIEnv*,
return static_cast<jdouble>(opts->blob_garbage_collection_age_cutoff);
}
/*
* Class: org_rocksdb_Options
* Method: setBlobGarbageCollectionForceThreshold
* Signature: (JD)V
*/
void Java_org_rocksdb_Options_setBlobGarbageCollectionForceThreshold(
JNIEnv*, jobject, jlong jhandle,
jdouble jblob_garbage_collection_force_threshold) {
auto* opts = reinterpret_cast<ROCKSDB_NAMESPACE::Options*>(jhandle);
opts->blob_garbage_collection_force_threshold =
static_cast<double>(jblob_garbage_collection_force_threshold);
}
/*
* Class: org_rocksdb_Options
* Method: blobGarbageCollectionForceThreshold
* Signature: (J)D
*/
jdouble Java_org_rocksdb_Options_blobGarbageCollectionForceThreshold(
JNIEnv*, jobject, jlong jhandle) {
auto* opts = reinterpret_cast<ROCKSDB_NAMESPACE::Options*>(jhandle);
return static_cast<jdouble>(opts->blob_garbage_collection_force_threshold);
}
//////////////////////////////////////////////////////////////////////////////
// ROCKSDB_NAMESPACE::ColumnFamilyOptions
@ -5545,6 +5569,33 @@ jdouble Java_org_rocksdb_ColumnFamilyOptions_blobGarbageCollectionAgeCutoff(
return static_cast<jdouble>(opts->blob_garbage_collection_age_cutoff);
}
/*
* Class: org_rocksdb_ColumnFamilyOptions
* Method: setBlobGarbageCollectionForceThreshold
* Signature: (JD)V
*/
void Java_org_rocksdb_ColumnFamilyOptions_setBlobGarbageCollectionForceThreshold(
JNIEnv*, jobject, jlong jhandle,
jdouble jblob_garbage_collection_force_threshold) {
auto* opts =
reinterpret_cast<ROCKSDB_NAMESPACE::ColumnFamilyOptions*>(jhandle);
opts->blob_garbage_collection_force_threshold =
static_cast<double>(jblob_garbage_collection_force_threshold);
}
/*
* Class: org_rocksdb_ColumnFamilyOptions
* Method: blobGarbageCollectionAgeCutoff
* Signature: (J)D
*/
jdouble
Java_org_rocksdb_ColumnFamilyOptions_blobGarbageCollectionForceThreshold(
JNIEnv*, jobject, jlong jhandle) {
auto* opts =
reinterpret_cast<ROCKSDB_NAMESPACE::ColumnFamilyOptions*>(jhandle);
return static_cast<jdouble>(opts->blob_garbage_collection_force_threshold);
}
/////////////////////////////////////////////////////////////////////
// ROCKSDB_NAMESPACE::DBOptions

25
java/src/main/java/org/rocksdb/AdvancedMutableColumnFamilyOptionsInterface.java
View File

@ -687,6 +687,31 @@ public interface AdvancedMutableColumnFamilyOptionsInterface<
*/
double blobGarbageCollectionAgeCutoff();
/**
* If the ratio of garbage in the oldest blob files exceeds this threshold,
* targeted compactions are scheduled in order to force garbage collecting
* the blob files in question, assuming they are all eligible based on the
* value of {@link #blobGarbageCollectionAgeCutoff} above. This option is
* currently only supported with leveled compactions.
*
* Note that {@link #enableBlobGarbageCollection} has to be set in order for this
* option to have any effect.
*
* Default: 1.0
*
* Dynamically changeable through the SetOptions() API
*
* @param blobGarbageCollectionForceThreshold new value for the threshold
* @return the reference to the current options
*/
T setBlobGarbageCollectionForceThreshold(double blobGarbageCollectionForceThreshold);
/**
* Get the current value for the {@link #blobGarbageCollectionForceThreshold}
* @return the current threshold at which garbage collection of blobs is forced
*/
double blobGarbageCollectionForceThreshold();
//
// END options for blobs (integrated BlobDB)
//

36
java/src/main/java/org/rocksdb/ColumnFamilyOptions.java
View File

@ -1164,6 +1164,39 @@ public class ColumnFamilyOptions extends RocksObject
return blobGarbageCollectionAgeCutoff(nativeHandle_);
}
/**
* If the ratio of garbage in the oldest blob files exceeds this threshold,
* targeted compactions are scheduled in order to force garbage collecting
* the blob files in question, assuming they are all eligible based on the
* value of {@link #blobGarbageCollectionAgeCutoff} above. This option is
* currently only supported with leveled compactions.
*
* Note that {@link #enableBlobGarbageCollection} has to be set in order for this
* option to have any effect.
*
* Default: 1.0
*
* Dynamically changeable through the SetOptions() API
*
* @param blobGarbageCollectionForceThreshold new value for the threshold
* @return the reference to the current options
*/
@Override
public ColumnFamilyOptions setBlobGarbageCollectionForceThreshold(
final double blobGarbageCollectionForceThreshold) {
setBlobGarbageCollectionForceThreshold(nativeHandle_, blobGarbageCollectionForceThreshold);
return this;
}
/**
* Get the current value for the {@link #blobGarbageCollectionForceThreshold}
* @return the current threshold at which garbage collection of blobs is forced
*/
@Override
public double blobGarbageCollectionForceThreshold() {
return blobGarbageCollectionForceThreshold(nativeHandle_);
}
//
// END options for blobs (integrated BlobDB)
//
@ -1358,6 +1391,9 @@ public class ColumnFamilyOptions extends RocksObject
private native void setBlobGarbageCollectionAgeCutoff(
final long nativeHandle_, final double blobGarbageCollectionAgeCutoff);
private native double blobGarbageCollectionAgeCutoff(final long nativeHandle_);
private native void setBlobGarbageCollectionForceThreshold(
final long nativeHandle_, final double blobGarbageCollectionForceThreshold);
private native double blobGarbageCollectionForceThreshold(final long nativeHandle_);
// instance variables
// NOTE: If you add new member variables, please update the copy constructor above!

15
java/src/main/java/org/rocksdb/MutableColumnFamilyOptions.java
View File

@ -119,7 +119,8 @@ public class MutableColumnFamilyOptions
blob_file_size(ValueType.LONG),
blob_compression_type(ValueType.ENUM),
enable_blob_garbage_collection(ValueType.BOOLEAN),
blob_garbage_collection_age_cutoff(ValueType.DOUBLE);
blob_garbage_collection_age_cutoff(ValueType.DOUBLE),
blob_garbage_collection_force_threshold(ValueType.DOUBLE);
private final ValueType valueType;
BlobOption(final ValueType valueType) {
@ -546,5 +547,17 @@ public class MutableColumnFamilyOptions
public double blobGarbageCollectionAgeCutoff() {
return getDouble(BlobOption.blob_garbage_collection_age_cutoff);
}
@Override
public MutableColumnFamilyOptionsBuilder setBlobGarbageCollectionForceThreshold(
final double blobGarbageCollectionForceThreshold) {
return setDouble(
BlobOption.blob_garbage_collection_force_threshold, blobGarbageCollectionForceThreshold);
}
@Override
public double blobGarbageCollectionForceThreshold() {
return getDouble(BlobOption.blob_garbage_collection_force_threshold);
}
}
}

19
java/src/main/java/org/rocksdb/Options.java
View File

@ -2077,7 +2077,7 @@ public class Options extends RocksObject
}
@Override
public Options setBlobGarbageCollectionAgeCutoff(double blobGarbageCollectionAgeCutoff) {
public Options setBlobGarbageCollectionAgeCutoff(final double blobGarbageCollectionAgeCutoff) {
setBlobGarbageCollectionAgeCutoff(nativeHandle_, blobGarbageCollectionAgeCutoff);
return this;
}
@ -2087,6 +2087,18 @@ public class Options extends RocksObject
return blobGarbageCollectionAgeCutoff(nativeHandle_);
}
@Override
public Options setBlobGarbageCollectionForceThreshold(
final double blobGarbageCollectionForceThreshold) {
setBlobGarbageCollectionForceThreshold(nativeHandle_, blobGarbageCollectionForceThreshold);
return this;
}
@Override
public double blobGarbageCollectionForceThreshold() {
return blobGarbageCollectionForceThreshold(nativeHandle_);
}
//
// END options for blobs (integrated BlobDB)
//
@ -2517,8 +2529,11 @@ public class Options extends RocksObject
final long nativeHandle_, final boolean enableBlobGarbageCollection);
private native boolean enableBlobGarbageCollection(final long nativeHandle_);
private native void setBlobGarbageCollectionAgeCutoff(
final long nativeHandle_, double blobGarbageCollectionAgeCutoff);
final long nativeHandle_, final double blobGarbageCollectionAgeCutoff);
private native double blobGarbageCollectionAgeCutoff(final long nativeHandle_);
private native void setBlobGarbageCollectionForceThreshold(
final long nativeHandle_, final double blobGarbageCollectionForceThreshold);
private native double blobGarbageCollectionForceThreshold(final long nativeHandle_);
// instance variables
// NOTE: If you add new member variables, please update the copy constructor above!

16
java/src/test/java/org/rocksdb/BlobOptionsTest.java
View File

@ -77,6 +77,7 @@ public class BlobOptionsTest {
assertThat(options.enableBlobGarbageCollection()).isEqualTo(false);
assertThat(options.blobFileSize()).isEqualTo(268435456L);
assertThat(options.blobGarbageCollectionAgeCutoff()).isEqualTo(0.25);
assertThat(options.blobGarbageCollectionForceThreshold()).isEqualTo(1.0);
assertThat(options.setEnableBlobFiles(true)).isEqualTo(options);
assertThat(options.setMinBlobSize(132768L)).isEqualTo(options);
@ -85,6 +86,7 @@ public class BlobOptionsTest {
assertThat(options.setEnableBlobGarbageCollection(true)).isEqualTo(options);
assertThat(options.setBlobFileSize(132768L)).isEqualTo(options);
assertThat(options.setBlobGarbageCollectionAgeCutoff(0.89)).isEqualTo(options);
assertThat(options.setBlobGarbageCollectionForceThreshold(0.80)).isEqualTo(options);
assertThat(options.enableBlobFiles()).isEqualTo(true);
assertThat(options.minBlobSize()).isEqualTo(132768L);
@ -92,6 +94,7 @@ public class BlobOptionsTest {
assertThat(options.enableBlobGarbageCollection()).isEqualTo(true);
assertThat(options.blobFileSize()).isEqualTo(132768L);
assertThat(options.blobGarbageCollectionAgeCutoff()).isEqualTo(0.89);
assertThat(options.blobGarbageCollectionForceThreshold()).isEqualTo(0.80);
}
}
@ -105,6 +108,7 @@ public class BlobOptionsTest {
assertThat(columnFamilyOptions.enableBlobGarbageCollection()).isEqualTo(false);
assertThat(columnFamilyOptions.blobFileSize()).isEqualTo(268435456L);
assertThat(columnFamilyOptions.blobGarbageCollectionAgeCutoff()).isEqualTo(0.25);
assertThat(columnFamilyOptions.blobGarbageCollectionForceThreshold()).isEqualTo(1.0);
assertThat(columnFamilyOptions.setEnableBlobFiles(true)).isEqualTo(columnFamilyOptions);
assertThat(columnFamilyOptions.setMinBlobSize(132768L)).isEqualTo(columnFamilyOptions);
@ -115,6 +119,8 @@ public class BlobOptionsTest {
assertThat(columnFamilyOptions.setBlobFileSize(132768L)).isEqualTo(columnFamilyOptions);
assertThat(columnFamilyOptions.setBlobGarbageCollectionAgeCutoff(0.89))
.isEqualTo(columnFamilyOptions);
assertThat(columnFamilyOptions.setBlobGarbageCollectionForceThreshold(0.80))
.isEqualTo(columnFamilyOptions);
assertThat(columnFamilyOptions.enableBlobFiles()).isEqualTo(true);
assertThat(columnFamilyOptions.minBlobSize()).isEqualTo(132768L);
@ -123,6 +129,7 @@ public class BlobOptionsTest {
assertThat(columnFamilyOptions.enableBlobGarbageCollection()).isEqualTo(true);
assertThat(columnFamilyOptions.blobFileSize()).isEqualTo(132768L);
assertThat(columnFamilyOptions.blobGarbageCollectionAgeCutoff()).isEqualTo(0.89);
assertThat(columnFamilyOptions.blobGarbageCollectionForceThreshold()).isEqualTo(0.80);
}
}
@ -135,6 +142,7 @@ public class BlobOptionsTest {
.setBlobCompressionType(CompressionType.BZLIB2_COMPRESSION)
.setEnableBlobGarbageCollection(true)
.setBlobGarbageCollectionAgeCutoff(0.89)
.setBlobGarbageCollectionForceThreshold(0.80)
.setBlobFileSize(132768);
assertThat(builder.enableBlobFiles()).isEqualTo(true);
@ -142,6 +150,7 @@ public class BlobOptionsTest {
assertThat(builder.blobCompressionType()).isEqualTo(CompressionType.BZLIB2_COMPRESSION);
assertThat(builder.enableBlobGarbageCollection()).isEqualTo(true);
assertThat(builder.blobGarbageCollectionAgeCutoff()).isEqualTo(0.89);
assertThat(builder.blobGarbageCollectionForceThreshold()).isEqualTo(0.80);
assertThat(builder.blobFileSize()).isEqualTo(132768);
builder.setEnableBlobFiles(false)
@ -149,6 +158,7 @@ public class BlobOptionsTest {
.setBlobCompressionType(CompressionType.LZ4_COMPRESSION)
.setEnableBlobGarbageCollection(false)
.setBlobGarbageCollectionAgeCutoff(0.91)
.setBlobGarbageCollectionForceThreshold(0.96)
.setBlobFileSize(2048);
assertThat(builder.enableBlobFiles()).isEqualTo(false);
@ -156,15 +166,17 @@ public class BlobOptionsTest {
assertThat(builder.blobCompressionType()).isEqualTo(CompressionType.LZ4_COMPRESSION);
assertThat(builder.enableBlobGarbageCollection()).isEqualTo(false);
assertThat(builder.blobGarbageCollectionAgeCutoff()).isEqualTo(0.91);
assertThat(builder.blobGarbageCollectionForceThreshold()).isEqualTo(0.96);
assertThat(builder.blobFileSize()).isEqualTo(2048);
final MutableColumnFamilyOptions options = builder.build();
assertThat(options.getKeys())
.isEqualTo(new String[] {"enable_blob_files", "min_blob_size", "blob_compression_type",
"enable_blob_garbage_collection", "blob_garbage_collection_age_cutoff",
"blob_file_size"});
"blob_garbage_collection_force_threshold", "blob_file_size"});
assertThat(options.getValues())
.isEqualTo(new String[] {"false", "4096", "LZ4_COMPRESSION", "false", "0.91", "2048"});
.isEqualTo(
new String[] {"false", "4096", "LZ4_COMPRESSION", "false", "0.91", "0.96", "2048"});
}
/**

3
java/src/test/java/org/rocksdb/MutableColumnFamilyOptionsTest.java
View File

@ -96,7 +96,7 @@ public class MutableColumnFamilyOptionsTest {
public void mutableColumnFamilyOptions_parse_getOptions_output() {
final String optionsString =
"bottommost_compression=kDisableCompressionOption; sample_for_compression=0; "
+ "blob_garbage_collection_age_cutoff=0.250000; arena_block_size=1048576; enable_blob_garbage_collection=false; "
+ "blob_garbage_collection_age_cutoff=0.250000; blob_garbage_collection_force_threshold=0.800000; arena_block_size=1048576; enable_blob_garbage_collection=false; "
+ "level0_stop_writes_trigger=36; min_blob_size=65536; "
+ "compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;"
+ "compression_size_percent=-1;max_size_amplification_percent=200;max_merge_width=4294967295;size_ratio=1;}; "
@ -126,6 +126,7 @@ public class MutableColumnFamilyOptionsTest {
// Check the values from the parsed string which are column family options
assertThat(cf.blobGarbageCollectionAgeCutoff()).isEqualTo(0.25);
assertThat(cf.blobGarbageCollectionForceThreshold()).isEqualTo(0.80);
assertThat(cf.arenaBlockSize()).isEqualTo(1048576);
assertThat(cf.enableBlobGarbageCollection()).isEqualTo(false);
assertThat(cf.level0StopWritesTrigger()).isEqualTo(36);

12
java/src/test/java/org/rocksdb/MutableOptionsGetSetTest.java
View File

@ -44,6 +44,8 @@ public class MutableOptionsGetSetTest {
new ColumnFamilyOptions()
.setMinBlobSize(minBlobSize)
.setEnableBlobFiles(true)
.setBlobGarbageCollectionAgeCutoff(0.25)
.setBlobGarbageCollectionForceThreshold(0.80)
.setArenaBlockSize(42)
.setMemtablePrefixBloomSizeRatio(0.17)
.setMemtableHugePageSize(3)
@ -98,6 +100,8 @@ public class MutableOptionsGetSetTest {
final MutableColumnFamilyOptions.MutableColumnFamilyOptionsBuilder builder1 =
db.getOptions(columnFamilyHandle1);
assertThat(builder1.enableBlobFiles()).isEqualTo(true);
assertThat(builder1.blobGarbageCollectionAgeCutoff()).isEqualTo(0.25);
assertThat(builder1.blobGarbageCollectionForceThreshold()).isEqualTo(0.80);
assertThat(builder1.minBlobSize()).isEqualTo(minBlobSize);
assertThat(builder1.arenaBlockSize()).isEqualTo(42);
assertThat(builder1.memtableHugePageSize()).isEqualTo(3);
@ -184,6 +188,8 @@ public class MutableOptionsGetSetTest {
MutableColumnFamilyOptions.builder()
.setMinBlobSize(minBlobSize)
.setEnableBlobFiles(true)
.setBlobGarbageCollectionAgeCutoff(0.25)
.setBlobGarbageCollectionForceThreshold(0.80)
.setArenaBlockSize(42)
.setMemtablePrefixBloomSizeRatio(0.17)
.setMemtableHugePageSize(3)
@ -205,6 +211,8 @@ public class MutableOptionsGetSetTest {
final MutableColumnFamilyOptions.MutableColumnFamilyOptionsBuilder builder1 =
db.getOptions(columnFamilyHandle1);
assertThat(builder1.enableBlobFiles()).isEqualTo(true);
assertThat(builder1.blobGarbageCollectionAgeCutoff()).isEqualTo(0.25);
assertThat(builder1.blobGarbageCollectionForceThreshold()).isEqualTo(0.80);
assertThat(builder1.minBlobSize()).isEqualTo(minBlobSize);
assertThat(builder1.arenaBlockSize()).isEqualTo(42);
assertThat(builder1.memtableHugePageSize()).isEqualTo(3);
@ -294,6 +302,8 @@ public class MutableOptionsGetSetTest {
MutableColumnFamilyOptions.builder()
.setMinBlobSize(minBlobSize)
.setEnableBlobFiles(true)
.setBlobGarbageCollectionAgeCutoff(0.25)
.setBlobGarbageCollectionForceThreshold(0.80)
.setArenaBlockSize(42)
.setMemtablePrefixBloomSizeRatio(0.17)
.setMemtableHugePageSize(3)
@ -314,6 +324,8 @@ public class MutableOptionsGetSetTest {
// Check the getOptions() brings back the creation options for CF1
final MutableColumnFamilyOptions.MutableColumnFamilyOptionsBuilder builder1 = db.getOptions();
assertThat(builder1.enableBlobFiles()).isEqualTo(true);
assertThat(builder1.blobGarbageCollectionAgeCutoff()).isEqualTo(0.25);
assertThat(builder1.blobGarbageCollectionForceThreshold()).isEqualTo(0.80);
assertThat(builder1.minBlobSize()).isEqualTo(minBlobSize);
assertThat(builder1.arenaBlockSize()).isEqualTo(42);
assertThat(builder1.memtableHugePageSize()).isEqualTo(3);