Added support for SstFileManager to RocksJava
Summary: Closes https://github.com/facebook/rocksdb/pull/3666 Differential Revision: D7457634 Pulled By: sagar0 fbshipit-source-id: 47741e2ee66e9255c580f4e38cfb86b284c27c2f
This commit is contained in:
parent
74767deec3
commit
ca87aef82d
@ -13,6 +13,7 @@
|
|||||||
|
|
||||||
### Java API Changes
|
### Java API Changes
|
||||||
* Add `BlockBasedTableConfig.setBlockCache` to allow sharing a block cache across DB instances.
|
* Add `BlockBasedTableConfig.setBlockCache` to allow sharing a block cache across DB instances.
|
||||||
|
* Added SstFileManager to the Java API to allow managing SST files across DB instances.
|
||||||
|
|
||||||
## 5.13.0 (3/20/2018)
|
## 5.13.0 (3/20/2018)
|
||||||
### Public API Change
|
### Public API Change
|
||||||
|
@ -37,6 +37,7 @@ set(JNI_NATIVE_SOURCES
|
|||||||
rocksjni/rocksjni.cc
|
rocksjni/rocksjni.cc
|
||||||
rocksjni/slice.cc
|
rocksjni/slice.cc
|
||||||
rocksjni/snapshot.cc
|
rocksjni/snapshot.cc
|
||||||
|
rocksjni/sst_file_manager.cc
|
||||||
rocksjni/sst_file_writerjni.cc
|
rocksjni/sst_file_writerjni.cc
|
||||||
rocksjni/statistics.cc
|
rocksjni/statistics.cc
|
||||||
rocksjni/statisticsjni.cc
|
rocksjni/statisticsjni.cc
|
||||||
@ -111,6 +112,7 @@ set(NATIVE_JAVA_CLASSES
|
|||||||
org.rocksdb.Slice
|
org.rocksdb.Slice
|
||||||
org.rocksdb.Snapshot
|
org.rocksdb.Snapshot
|
||||||
org.rocksdb.SnapshotTest
|
org.rocksdb.SnapshotTest
|
||||||
|
org.rocksdb.SstFileManager
|
||||||
org.rocksdb.SstFileWriter
|
org.rocksdb.SstFileWriter
|
||||||
org.rocksdb.Statistics
|
org.rocksdb.Statistics
|
||||||
org.rocksdb.StringAppendOperator
|
org.rocksdb.StringAppendOperator
|
||||||
@ -232,6 +234,7 @@ add_jar(
|
|||||||
src/main/java/org/rocksdb/SkipListMemTableConfig.java
|
src/main/java/org/rocksdb/SkipListMemTableConfig.java
|
||||||
src/main/java/org/rocksdb/Slice.java
|
src/main/java/org/rocksdb/Slice.java
|
||||||
src/main/java/org/rocksdb/Snapshot.java
|
src/main/java/org/rocksdb/Snapshot.java
|
||||||
|
src/main/java/org/rocksdb/SstFileManager.java
|
||||||
src/main/java/org/rocksdb/SstFileWriter.java
|
src/main/java/org/rocksdb/SstFileWriter.java
|
||||||
src/main/java/org/rocksdb/Statistics.java
|
src/main/java/org/rocksdb/Statistics.java
|
||||||
src/main/java/org/rocksdb/StatsLevel.java
|
src/main/java/org/rocksdb/StatsLevel.java
|
||||||
|
@ -47,6 +47,7 @@ NATIVE_JAVA_CLASSES = org.rocksdb.AbstractCompactionFilter\
|
|||||||
org.rocksdb.RocksMemEnv\
|
org.rocksdb.RocksMemEnv\
|
||||||
org.rocksdb.SkipListMemTableConfig\
|
org.rocksdb.SkipListMemTableConfig\
|
||||||
org.rocksdb.Slice\
|
org.rocksdb.Slice\
|
||||||
|
org.rocksdb.SstFileManager\
|
||||||
org.rocksdb.SstFileWriter\
|
org.rocksdb.SstFileWriter\
|
||||||
org.rocksdb.Statistics\
|
org.rocksdb.Statistics\
|
||||||
org.rocksdb.Transaction\
|
org.rocksdb.Transaction\
|
||||||
@ -132,6 +133,7 @@ JAVA_TESTS = org.rocksdb.BackupableDBOptionsTest\
|
|||||||
org.rocksdb.util.SizeUnitTest\
|
org.rocksdb.util.SizeUnitTest\
|
||||||
org.rocksdb.SliceTest\
|
org.rocksdb.SliceTest\
|
||||||
org.rocksdb.SnapshotTest\
|
org.rocksdb.SnapshotTest\
|
||||||
|
org.rocksdb.SstFileManagerTest\
|
||||||
org.rocksdb.SstFileWriterTest\
|
org.rocksdb.SstFileWriterTest\
|
||||||
org.rocksdb.TransactionTest\
|
org.rocksdb.TransactionTest\
|
||||||
org.rocksdb.TransactionDBTest\
|
org.rocksdb.TransactionDBTest\
|
||||||
|
@ -895,6 +895,20 @@ void Java_org_rocksdb_Options_setRateLimiter(
|
|||||||
rate_limiter = *pRateLimiter;
|
rate_limiter = *pRateLimiter;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Class: org_rocksdb_Options
|
||||||
|
* Method: setSstFileManager
|
||||||
|
* Signature: (JJ)V
|
||||||
|
*/
|
||||||
|
void Java_org_rocksdb_Options_setSstFileManager(
|
||||||
|
JNIEnv* env, jobject job, jlong jhandle,
|
||||||
|
jlong jsst_file_manager_handle) {
|
||||||
|
auto* sptr_sst_file_manager =
|
||||||
|
reinterpret_cast<std::shared_ptr<rocksdb::SstFileManager> *>(jsst_file_manager_handle);
|
||||||
|
reinterpret_cast<rocksdb::Options*>(jhandle)->
|
||||||
|
sst_file_manager = *sptr_sst_file_manager;
|
||||||
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Class: org_rocksdb_Options
|
* Class: org_rocksdb_Options
|
||||||
* Method: setLogger
|
* Method: setLogger
|
||||||
@ -4399,6 +4413,20 @@ void Java_org_rocksdb_DBOptions_setRateLimiter(
|
|||||||
reinterpret_cast<rocksdb::DBOptions*>(jhandle)->rate_limiter = *pRateLimiter;
|
reinterpret_cast<rocksdb::DBOptions*>(jhandle)->rate_limiter = *pRateLimiter;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Class: org_rocksdb_DBOptions
|
||||||
|
* Method: setSstFileManager
|
||||||
|
* Signature: (JJ)V
|
||||||
|
*/
|
||||||
|
void Java_org_rocksdb_DBOptions_setSstFileManager(
|
||||||
|
JNIEnv* env, jobject jobj, jlong jhandle,
|
||||||
|
jlong jsst_file_manager_handle) {
|
||||||
|
auto* sptr_sst_file_manager =
|
||||||
|
reinterpret_cast<std::shared_ptr<rocksdb::SstFileManager> *>(jsst_file_manager_handle);
|
||||||
|
reinterpret_cast<rocksdb::DBOptions*>(jhandle)->
|
||||||
|
sst_file_manager = *sptr_sst_file_manager;
|
||||||
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Class: org_rocksdb_DBOptions
|
* Class: org_rocksdb_DBOptions
|
||||||
* Method: setLogger
|
* Method: setLogger
|
||||||
|
217
java/rocksjni/sst_file_manager.cc
Normal file
217
java/rocksjni/sst_file_manager.cc
Normal file
@ -0,0 +1,217 @@
|
|||||||
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under both the GPLv2 (found in the
|
||||||
|
// COPYING file in the root directory) and Apache 2.0 License
|
||||||
|
// (found in the LICENSE.Apache file in the root directory).
|
||||||
|
//
|
||||||
|
// This file implements the "bridge" between Java and C++ and enables
|
||||||
|
// calling C++ rocksdb::SstFileManager methods
|
||||||
|
// from Java side.
|
||||||
|
|
||||||
|
#include <jni.h>
|
||||||
|
#include <memory>
|
||||||
|
|
||||||
|
#include "include/org_rocksdb_SstFileManager.h"
|
||||||
|
#include "rocksdb/sst_file_manager.h"
|
||||||
|
#include "rocksjni/portal.h"
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Class: org_rocksdb_SstFileManager
|
||||||
|
* Method: newSstFileManager
|
||||||
|
* Signature: (JJJDJ)J
|
||||||
|
*/
|
||||||
|
jlong Java_org_rocksdb_SstFileManager_newSstFileManager(
|
||||||
|
JNIEnv* jnienv, jclass jcls, jlong jenv_handle, jlong jlogger_handle,
|
||||||
|
jlong jrate_bytes, jdouble jmax_trash_db_ratio,
|
||||||
|
jlong jmax_delete_chunk_bytes) {
|
||||||
|
|
||||||
|
auto* env = reinterpret_cast<rocksdb::Env*>(jenv_handle);
|
||||||
|
rocksdb::Status s;
|
||||||
|
rocksdb::SstFileManager* sst_file_manager = nullptr;
|
||||||
|
|
||||||
|
if (jlogger_handle != 0) {
|
||||||
|
auto* sptr_logger =
|
||||||
|
reinterpret_cast<std::shared_ptr<rocksdb::Logger> *>(jlogger_handle);
|
||||||
|
sst_file_manager = rocksdb::NewSstFileManager(env, *sptr_logger, "",
|
||||||
|
jrate_bytes, true, &s, jmax_trash_db_ratio,
|
||||||
|
jmax_delete_chunk_bytes);
|
||||||
|
} else {
|
||||||
|
sst_file_manager = rocksdb::NewSstFileManager(env, nullptr, "",
|
||||||
|
jrate_bytes, true, &s, jmax_trash_db_ratio,
|
||||||
|
jmax_delete_chunk_bytes);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!s.ok()) {
|
||||||
|
if (sst_file_manager != nullptr) {
|
||||||
|
delete sst_file_manager;
|
||||||
|
}
|
||||||
|
rocksdb::RocksDBExceptionJni::ThrowNew(jnienv, s);
|
||||||
|
}
|
||||||
|
auto* sptr_sst_file_manager
|
||||||
|
= new std::shared_ptr<rocksdb::SstFileManager>(sst_file_manager);
|
||||||
|
|
||||||
|
return reinterpret_cast<jlong>(sptr_sst_file_manager);
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Class: org_rocksdb_SstFileManager
|
||||||
|
* Method: setMaxAllowedSpaceUsage
|
||||||
|
* Signature: (JJ)V
|
||||||
|
*/
|
||||||
|
void Java_org_rocksdb_SstFileManager_setMaxAllowedSpaceUsage(
|
||||||
|
JNIEnv* env, jobject jobj, jlong jhandle, jlong jmax_allowed_space) {
|
||||||
|
auto* sptr_sst_file_manager =
|
||||||
|
reinterpret_cast<std::shared_ptr<rocksdb::SstFileManager> *>(jhandle);
|
||||||
|
sptr_sst_file_manager->get()->SetMaxAllowedSpaceUsage(jmax_allowed_space);
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Class: org_rocksdb_SstFileManager
|
||||||
|
* Method: setCompactionBufferSize
|
||||||
|
* Signature: (JJ)V
|
||||||
|
*/
|
||||||
|
void Java_org_rocksdb_SstFileManager_setCompactionBufferSize(
|
||||||
|
JNIEnv* env, jobject jobj, jlong jhandle, jlong jcompaction_buffer_size) {
|
||||||
|
auto* sptr_sst_file_manager =
|
||||||
|
reinterpret_cast<std::shared_ptr<rocksdb::SstFileManager> *>(jhandle);
|
||||||
|
sptr_sst_file_manager->get()->SetCompactionBufferSize(jcompaction_buffer_size);
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Class: org_rocksdb_SstFileManager
|
||||||
|
* Method: isMaxAllowedSpaceReached
|
||||||
|
* Signature: (J)Z
|
||||||
|
*/
|
||||||
|
jboolean Java_org_rocksdb_SstFileManager_isMaxAllowedSpaceReached(
|
||||||
|
JNIEnv* env, jobject jobj, jlong jhandle) {
|
||||||
|
auto* sptr_sst_file_manager =
|
||||||
|
reinterpret_cast<std::shared_ptr<rocksdb::SstFileManager> *>(jhandle);
|
||||||
|
return sptr_sst_file_manager->get()->IsMaxAllowedSpaceReached();
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Class: org_rocksdb_SstFileManager
|
||||||
|
* Method: isMaxAllowedSpaceReachedIncludingCompactions
|
||||||
|
* Signature: (J)Z
|
||||||
|
*/
|
||||||
|
jboolean Java_org_rocksdb_SstFileManager_isMaxAllowedSpaceReachedIncludingCompactions(
|
||||||
|
JNIEnv* env, jobject jobj, jlong jhandle) {
|
||||||
|
auto* sptr_sst_file_manager =
|
||||||
|
reinterpret_cast<std::shared_ptr<rocksdb::SstFileManager> *>(jhandle);
|
||||||
|
return sptr_sst_file_manager->get()->IsMaxAllowedSpaceReachedIncludingCompactions();
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Class: org_rocksdb_SstFileManager
|
||||||
|
* Method: getTotalSize
|
||||||
|
* Signature: (J)J
|
||||||
|
*/
|
||||||
|
jlong Java_org_rocksdb_SstFileManager_getTotalSize(
|
||||||
|
JNIEnv* env, jobject jobj, jlong jhandle) {
|
||||||
|
auto* sptr_sst_file_manager =
|
||||||
|
reinterpret_cast<std::shared_ptr<rocksdb::SstFileManager> *>(jhandle);
|
||||||
|
return sptr_sst_file_manager->get()->GetTotalSize();
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Class: org_rocksdb_SstFileManager
|
||||||
|
* Method: getTrackedFiles
|
||||||
|
* Signature: (J)Ljava/util/Map;
|
||||||
|
*/
|
||||||
|
jobject Java_org_rocksdb_SstFileManager_getTrackedFiles(
|
||||||
|
JNIEnv* env, jobject jobj, jlong jhandle) {
|
||||||
|
auto* sptr_sst_file_manager =
|
||||||
|
reinterpret_cast<std::shared_ptr<rocksdb::SstFileManager> *>(jhandle);
|
||||||
|
auto tracked_files = sptr_sst_file_manager->get()->GetTrackedFiles();
|
||||||
|
|
||||||
|
const jobject jtracked_files = rocksdb::HashMapJni::construct(env,
|
||||||
|
static_cast<uint32_t>(tracked_files.size()));
|
||||||
|
if (jtracked_files == nullptr) {
|
||||||
|
// exception occurred
|
||||||
|
return nullptr;
|
||||||
|
}
|
||||||
|
|
||||||
|
const rocksdb::HashMapJni::FnMapKV<const std::string, const uint64_t> fn_map_kv =
|
||||||
|
[env, &tracked_files](const std::pair<const std::string, const uint64_t>& pair) {
|
||||||
|
const jstring jtracked_file_path = env->NewStringUTF(pair.first.c_str());
|
||||||
|
if (jtracked_file_path == nullptr) {
|
||||||
|
// an error occurred
|
||||||
|
return std::unique_ptr<std::pair<jobject, jobject>>(nullptr);
|
||||||
|
}
|
||||||
|
const jobject jtracked_file_size =
|
||||||
|
rocksdb::LongJni::valueOf(env, pair.second);
|
||||||
|
if (jtracked_file_size == nullptr) {
|
||||||
|
// an error occurred
|
||||||
|
return std::unique_ptr<std::pair<jobject, jobject>>(nullptr);
|
||||||
|
}
|
||||||
|
return std::unique_ptr<std::pair<jobject, jobject>>(new std::pair<jobject, jobject>(jtracked_file_path,
|
||||||
|
jtracked_file_size));
|
||||||
|
};
|
||||||
|
|
||||||
|
if(!rocksdb::HashMapJni::putAll(env, jtracked_files,
|
||||||
|
tracked_files.begin(), tracked_files.end(), fn_map_kv)) {
|
||||||
|
// exception occcurred
|
||||||
|
return nullptr;
|
||||||
|
}
|
||||||
|
|
||||||
|
return jtracked_files;
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Class: org_rocksdb_SstFileManager
|
||||||
|
* Method: getDeleteRateBytesPerSecond
|
||||||
|
* Signature: (J)J
|
||||||
|
*/
|
||||||
|
jlong Java_org_rocksdb_SstFileManager_getDeleteRateBytesPerSecond(
|
||||||
|
JNIEnv* env, jobject jobj, jlong jhandle) {
|
||||||
|
auto* sptr_sst_file_manager =
|
||||||
|
reinterpret_cast<std::shared_ptr<rocksdb::SstFileManager> *>(jhandle);
|
||||||
|
return sptr_sst_file_manager->get()->GetDeleteRateBytesPerSecond();
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Class: org_rocksdb_SstFileManager
|
||||||
|
* Method: setDeleteRateBytesPerSecond
|
||||||
|
* Signature: (JJ)V
|
||||||
|
*/
|
||||||
|
void Java_org_rocksdb_SstFileManager_setDeleteRateBytesPerSecond(
|
||||||
|
JNIEnv* env, jobject jobj, jlong jhandle, jlong jdelete_rate) {
|
||||||
|
auto* sptr_sst_file_manager =
|
||||||
|
reinterpret_cast<std::shared_ptr<rocksdb::SstFileManager> *>(jhandle);
|
||||||
|
sptr_sst_file_manager->get()->SetDeleteRateBytesPerSecond(jdelete_rate);
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Class: org_rocksdb_SstFileManager
|
||||||
|
* Method: getMaxTrashDBRatio
|
||||||
|
* Signature: (J)D
|
||||||
|
*/
|
||||||
|
jdouble Java_org_rocksdb_SstFileManager_getMaxTrashDBRatio(
|
||||||
|
JNIEnv* env, jobject jobj, jlong jhandle) {
|
||||||
|
auto* sptr_sst_file_manager =
|
||||||
|
reinterpret_cast<std::shared_ptr<rocksdb::SstFileManager> *>(jhandle);
|
||||||
|
return sptr_sst_file_manager->get()->GetMaxTrashDBRatio();
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Class: org_rocksdb_SstFileManager
|
||||||
|
* Method: setMaxTrashDBRatio
|
||||||
|
* Signature: (JD)V
|
||||||
|
*/
|
||||||
|
void Java_org_rocksdb_SstFileManager_setMaxTrashDBRatio(
|
||||||
|
JNIEnv* env, jobject jobj, jlong jhandle, jdouble jratio) {
|
||||||
|
auto* sptr_sst_file_manager =
|
||||||
|
reinterpret_cast<std::shared_ptr<rocksdb::SstFileManager> *>(jhandle);
|
||||||
|
sptr_sst_file_manager->get()->SetMaxTrashDBRatio(jratio);
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Class: org_rocksdb_SstFileManager
|
||||||
|
* Method: disposeInternal
|
||||||
|
* Signature: (J)V
|
||||||
|
*/
|
||||||
|
void Java_org_rocksdb_SstFileManager_disposeInternal(
|
||||||
|
JNIEnv* env, jobject jobj, jlong jhandle) {
|
||||||
|
auto* sptr_sst_file_manager =
|
||||||
|
reinterpret_cast<std::shared_ptr<rocksdb::SstFileManager> *>(jhandle);
|
||||||
|
delete sptr_sst_file_manager;
|
||||||
|
}
|
@ -12,6 +12,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
|
|||||||
* {@link AbstractNativeReference} which have an immutable reference to the
|
* {@link AbstractNativeReference} which have an immutable reference to the
|
||||||
* underlying native C++ object
|
* underlying native C++ object
|
||||||
*/
|
*/
|
||||||
|
//@ThreadSafe
|
||||||
public abstract class AbstractImmutableNativeReference
|
public abstract class AbstractImmutableNativeReference
|
||||||
extends AbstractNativeReference {
|
extends AbstractNativeReference {
|
||||||
|
|
||||||
|
@ -178,6 +178,13 @@ public class DBOptions
|
|||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public DBOptions setSstFileManager(final SstFileManager sstFileManager) {
|
||||||
|
assert(isOwningHandle());
|
||||||
|
setSstFileManager(nativeHandle_, sstFileManager.nativeHandle_);
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public DBOptions setLogger(final Logger logger) {
|
public DBOptions setLogger(final Logger logger) {
|
||||||
assert(isOwningHandle());
|
assert(isOwningHandle());
|
||||||
@ -988,6 +995,8 @@ public class DBOptions
|
|||||||
private native boolean paranoidChecks(long handle);
|
private native boolean paranoidChecks(long handle);
|
||||||
private native void setRateLimiter(long handle,
|
private native void setRateLimiter(long handle,
|
||||||
long rateLimiterHandle);
|
long rateLimiterHandle);
|
||||||
|
private native void setSstFileManager(final long handle,
|
||||||
|
final long sstFileManagerHandle);
|
||||||
private native void setLogger(long handle,
|
private native void setLogger(long handle,
|
||||||
long loggerHandle);
|
long loggerHandle);
|
||||||
private native void setInfoLogLevel(long handle, byte logLevel);
|
private native void setInfoLogLevel(long handle, byte logLevel);
|
||||||
|
@ -158,6 +158,25 @@ public interface DBOptionsInterface<T extends DBOptionsInterface> {
|
|||||||
*/
|
*/
|
||||||
T setRateLimiter(RateLimiter rateLimiter);
|
T setRateLimiter(RateLimiter rateLimiter);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Use to track SST files and control their file deletion rate.
|
||||||
|
*
|
||||||
|
* Features:
|
||||||
|
* - Throttle the deletion rate of the SST files.
|
||||||
|
* - Keep track the total size of all SST files.
|
||||||
|
* - Set a maximum allowed space limit for SST files that when reached
|
||||||
|
* the DB wont do any further flushes or compactions and will set the
|
||||||
|
* background error.
|
||||||
|
* - Can be shared between multiple dbs.
|
||||||
|
*
|
||||||
|
* Limitations:
|
||||||
|
* - Only track and throttle deletes of SST files in
|
||||||
|
* first db_path (db_name if db_paths is empty).
|
||||||
|
*
|
||||||
|
* @param sstFileManager The SST File Manager for the db.
|
||||||
|
*/
|
||||||
|
T setSstFileManager(SstFileManager sstFileManager);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* <p>Any internal progress/error information generated by
|
* <p>Any internal progress/error information generated by
|
||||||
* the db will be written to the Logger if it is non-nullptr,
|
* the db will be written to the Logger if it is non-nullptr,
|
||||||
|
@ -1032,6 +1032,13 @@ public class Options extends RocksObject
|
|||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Options setSstFileManager(final SstFileManager sstFileManager) {
|
||||||
|
assert(isOwningHandle());
|
||||||
|
setSstFileManager(nativeHandle_, sstFileManager.nativeHandle_);
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Options setLogger(final Logger logger) {
|
public Options setLogger(final Logger logger) {
|
||||||
assert(isOwningHandle());
|
assert(isOwningHandle());
|
||||||
@ -1588,6 +1595,8 @@ public class Options extends RocksObject
|
|||||||
private native boolean paranoidChecks(long handle);
|
private native boolean paranoidChecks(long handle);
|
||||||
private native void setRateLimiter(long handle,
|
private native void setRateLimiter(long handle,
|
||||||
long rateLimiterHandle);
|
long rateLimiterHandle);
|
||||||
|
private native void setSstFileManager(final long handle,
|
||||||
|
final long sstFileManagerHandle);
|
||||||
private native void setLogger(long handle,
|
private native void setLogger(long handle,
|
||||||
long loggerHandle);
|
long loggerHandle);
|
||||||
private native void setInfoLogLevel(long handle, byte logLevel);
|
private native void setInfoLogLevel(long handle, byte logLevel);
|
||||||
|
241
java/src/main/java/org/rocksdb/SstFileManager.java
Normal file
241
java/src/main/java/org/rocksdb/SstFileManager.java
Normal file
@ -0,0 +1,241 @@
|
|||||||
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under both the GPLv2 (found in the
|
||||||
|
// COPYING file in the root directory) and Apache 2.0 License
|
||||||
|
// (found in the LICENSE.Apache file in the root directory).
|
||||||
|
|
||||||
|
package org.rocksdb;
|
||||||
|
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* SstFileManager is used to track SST files in the DB and control their
|
||||||
|
* deletion rate.
|
||||||
|
*
|
||||||
|
* All SstFileManager public functions are thread-safe.
|
||||||
|
*
|
||||||
|
* SstFileManager is not extensible.
|
||||||
|
*/
|
||||||
|
//@ThreadSafe
|
||||||
|
public final class SstFileManager extends RocksObject {
|
||||||
|
|
||||||
|
public static final long RATE_BYTES_PER_SEC_DEFAULT = 0;
|
||||||
|
public static final boolean DELETE_EXISTING_TRASH_DEFAULT = true;
|
||||||
|
public static final double MAX_TRASH_DB_RATION_DEFAULT = 0.25;
|
||||||
|
public static final long BYTES_MAX_DELETE_CHUNK_DEFAULT = 64 * 1024 * 1024;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a new SstFileManager that can be shared among multiple RocksDB
|
||||||
|
* instances to track SST file and control there deletion rate.
|
||||||
|
*
|
||||||
|
* @param env the environment.
|
||||||
|
*/
|
||||||
|
public SstFileManager(final Env env) throws RocksDBException {
|
||||||
|
this(env, null);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a new SstFileManager that can be shared among multiple RocksDB
|
||||||
|
* instances to track SST file and control there deletion rate.
|
||||||
|
*
|
||||||
|
* @param env the environment.
|
||||||
|
* @param logger if not null, the logger will be used to log errors.
|
||||||
|
*/
|
||||||
|
public SstFileManager(final Env env, /*@Nullable*/ final Logger logger)
|
||||||
|
throws RocksDBException {
|
||||||
|
this(env, logger, RATE_BYTES_PER_SEC_DEFAULT);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a new SstFileManager that can be shared among multiple RocksDB
|
||||||
|
* instances to track SST file and control there deletion rate.
|
||||||
|
*
|
||||||
|
* @param env the environment.
|
||||||
|
* @param logger if not null, the logger will be used to log errors.
|
||||||
|
*
|
||||||
|
* == Deletion rate limiting specific arguments ==
|
||||||
|
* @param rateBytesPerSec how many bytes should be deleted per second, If
|
||||||
|
* this value is set to 1024 (1 Kb / sec) and we deleted a file of size
|
||||||
|
* 4 Kb in 1 second, we will wait for another 3 seconds before we delete
|
||||||
|
* other files, Set to 0 to disable deletion rate limiting.
|
||||||
|
*/
|
||||||
|
public SstFileManager(final Env env, /*@Nullable*/ final Logger logger,
|
||||||
|
final long rateBytesPerSec) throws RocksDBException {
|
||||||
|
this(env, logger, rateBytesPerSec, MAX_TRASH_DB_RATION_DEFAULT);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a new SstFileManager that can be shared among multiple RocksDB
|
||||||
|
* instances to track SST file and control there deletion rate.
|
||||||
|
*
|
||||||
|
* @param env the environment.
|
||||||
|
* @param logger if not null, the logger will be used to log errors.
|
||||||
|
*
|
||||||
|
* == Deletion rate limiting specific arguments ==
|
||||||
|
* @param rateBytesPerSec how many bytes should be deleted per second, If
|
||||||
|
* this value is set to 1024 (1 Kb / sec) and we deleted a file of size
|
||||||
|
* 4 Kb in 1 second, we will wait for another 3 seconds before we delete
|
||||||
|
* other files, Set to 0 to disable deletion rate limiting.
|
||||||
|
* @param maxTrashDbRatio if the trash size constitutes for more than this
|
||||||
|
* fraction of the total DB size we will start deleting new files passed
|
||||||
|
* to DeleteScheduler immediately.
|
||||||
|
*/
|
||||||
|
public SstFileManager(final Env env, /*@Nullable*/ final Logger logger,
|
||||||
|
final long rateBytesPerSec, final double maxTrashDbRatio)
|
||||||
|
throws RocksDBException {
|
||||||
|
this(env, logger, rateBytesPerSec, maxTrashDbRatio,
|
||||||
|
BYTES_MAX_DELETE_CHUNK_DEFAULT);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a new SstFileManager that can be shared among multiple RocksDB
|
||||||
|
* instances to track SST file and control there deletion rate.
|
||||||
|
*
|
||||||
|
* @param env the environment.
|
||||||
|
* @param logger if not null, the logger will be used to log errors.
|
||||||
|
*
|
||||||
|
* == Deletion rate limiting specific arguments ==
|
||||||
|
* @param rateBytesPerSec how many bytes should be deleted per second, If
|
||||||
|
* this value is set to 1024 (1 Kb / sec) and we deleted a file of size
|
||||||
|
* 4 Kb in 1 second, we will wait for another 3 seconds before we delete
|
||||||
|
* other files, Set to 0 to disable deletion rate limiting.
|
||||||
|
* @param maxTrashDbRatio if the trash size constitutes for more than this
|
||||||
|
* fraction of the total DB size we will start deleting new files passed
|
||||||
|
* to DeleteScheduler immediately.
|
||||||
|
* @param bytesMaxDeleteChunk if a single file is larger than delete chunk,
|
||||||
|
* ftruncate the file by this size each time, rather than dropping the whole
|
||||||
|
* file. 0 means to always delete the whole file.
|
||||||
|
*/
|
||||||
|
public SstFileManager(final Env env, /*@Nullable*/final Logger logger,
|
||||||
|
final long rateBytesPerSec, final double maxTrashDbRatio,
|
||||||
|
final long bytesMaxDeleteChunk) throws RocksDBException {
|
||||||
|
super(newSstFileManager(env.nativeHandle_,
|
||||||
|
logger != null ? logger.nativeHandle_ : 0,
|
||||||
|
rateBytesPerSec, maxTrashDbRatio, bytesMaxDeleteChunk));
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Update the maximum allowed space that should be used by RocksDB, if
|
||||||
|
* the total size of the SST files exceeds {@code maxAllowedSpace}, writes to
|
||||||
|
* RocksDB will fail.
|
||||||
|
*
|
||||||
|
* Setting {@code maxAllowedSpace} to 0 will disable this feature;
|
||||||
|
* maximum allowed space will be infinite (Default value).
|
||||||
|
*
|
||||||
|
* @param maxAllowedSpace the maximum allowed space that should be used by
|
||||||
|
* RocksDB.
|
||||||
|
*/
|
||||||
|
public void setMaxAllowedSpaceUsage(final long maxAllowedSpace) {
|
||||||
|
setMaxAllowedSpaceUsage(nativeHandle_, maxAllowedSpace);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Set the amount of buffer room each compaction should be able to leave.
|
||||||
|
* In other words, at its maximum disk space consumption, the compaction
|
||||||
|
* should still leave {@code compactionBufferSize} available on the disk so
|
||||||
|
* that other background functions may continue, such as logging and flushing.
|
||||||
|
*
|
||||||
|
* @param compactionBufferSize the amount of buffer room each compaction
|
||||||
|
* should be able to leave.
|
||||||
|
*/
|
||||||
|
public void setCompactionBufferSize(final long compactionBufferSize) {
|
||||||
|
setCompactionBufferSize(nativeHandle_, compactionBufferSize);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Determines if the total size of SST files exceeded the maximum allowed
|
||||||
|
* space usage.
|
||||||
|
*
|
||||||
|
* @return true when the maximum allows space usage has been exceeded.
|
||||||
|
*/
|
||||||
|
public boolean isMaxAllowedSpaceReached() {
|
||||||
|
return isMaxAllowedSpaceReached(nativeHandle_);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Determines if the total size of SST files as well as estimated size
|
||||||
|
* of ongoing compactions exceeds the maximums allowed space usage.
|
||||||
|
*
|
||||||
|
* @return true when the total size of SST files as well as estimated size
|
||||||
|
* of ongoing compactions exceeds the maximums allowed space usage.
|
||||||
|
*/
|
||||||
|
public boolean isMaxAllowedSpaceReachedIncludingCompactions() {
|
||||||
|
return isMaxAllowedSpaceReachedIncludingCompactions(nativeHandle_);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the total size of all tracked files.
|
||||||
|
*
|
||||||
|
* @return the total size of all tracked files.
|
||||||
|
*/
|
||||||
|
public long getTotalSize() {
|
||||||
|
return getTotalSize(nativeHandle_);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Gets all tracked files and their corresponding sizes.
|
||||||
|
*
|
||||||
|
* @return a map containing all tracked files and there corresponding sizes.
|
||||||
|
*/
|
||||||
|
public Map<String, Long> getTrackedFiles() {
|
||||||
|
return getTrackedFiles(nativeHandle_);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Gets the delete rate limit.
|
||||||
|
*
|
||||||
|
* @return the delete rate limit (in bytes per second).
|
||||||
|
*/
|
||||||
|
public long getDeleteRateBytesPerSecond() {
|
||||||
|
return getDeleteRateBytesPerSecond(nativeHandle_);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Set the delete rate limit.
|
||||||
|
*
|
||||||
|
* Zero means disable delete rate limiting and delete files immediately.
|
||||||
|
*
|
||||||
|
* @param deleteRate the delete rate limit (in bytes per second).
|
||||||
|
*/
|
||||||
|
public void setDeleteRateBytesPerSecond(final long deleteRate) {
|
||||||
|
setDeleteRateBytesPerSecond(nativeHandle_, deleteRate);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the trash/DB size ratio where new files will be deleted immediately.
|
||||||
|
*
|
||||||
|
* @return the trash/DB size ratio.
|
||||||
|
*/
|
||||||
|
public double getMaxTrashDBRatio() {
|
||||||
|
return getMaxTrashDBRatio(nativeHandle_);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Set the trash/DB size ratio where new files will be deleted immediately.
|
||||||
|
*
|
||||||
|
* @param ratio the trash/DB size ratio.
|
||||||
|
*/
|
||||||
|
public void setMaxTrashDBRatio(final double ratio) {
|
||||||
|
setMaxTrashDBRatio(nativeHandle_, ratio);
|
||||||
|
}
|
||||||
|
|
||||||
|
private native static long newSstFileManager(final long handle,
|
||||||
|
final long logger_handle, final long rateBytesPerSec,
|
||||||
|
final double maxTrashDbRatio, final long bytesMaxDeleteChunk)
|
||||||
|
throws RocksDBException;
|
||||||
|
private native void setMaxAllowedSpaceUsage(final long handle,
|
||||||
|
final long maxAllowedSpace);
|
||||||
|
private native void setCompactionBufferSize(final long handle,
|
||||||
|
final long compactionBufferSize);
|
||||||
|
private native boolean isMaxAllowedSpaceReached(final long handle);
|
||||||
|
private native boolean isMaxAllowedSpaceReachedIncludingCompactions(
|
||||||
|
final long handle);
|
||||||
|
private native long getTotalSize(final long handle);
|
||||||
|
private native Map<String, Long> getTrackedFiles(final long handle);
|
||||||
|
private native long getDeleteRateBytesPerSecond(final long handle);
|
||||||
|
private native void setDeleteRateBytesPerSecond(final long handle,
|
||||||
|
final long deleteRate);
|
||||||
|
private native double getMaxTrashDBRatio(final long handle);
|
||||||
|
private native void setMaxTrashDBRatio(final long handle, final double ratio);
|
||||||
|
@Override protected final native void disposeInternal(final long handle);
|
||||||
|
}
|
@ -643,6 +643,15 @@ public class DBOptionsTest {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void sstFileManager() throws RocksDBException {
|
||||||
|
try (final DBOptions options = new DBOptions();
|
||||||
|
final SstFileManager sstFileManager =
|
||||||
|
new SstFileManager(Env.getDefault())) {
|
||||||
|
options.setSstFileManager(sstFileManager);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void statistics() {
|
public void statistics() {
|
||||||
try(final DBOptions options = new DBOptions()) {
|
try(final DBOptions options = new DBOptions()) {
|
||||||
|
@ -999,6 +999,15 @@ public class OptionsTest {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void sstFileManager() throws RocksDBException {
|
||||||
|
try (final Options options = new Options();
|
||||||
|
final SstFileManager sstFileManager =
|
||||||
|
new SstFileManager(Env.getDefault())) {
|
||||||
|
options.setSstFileManager(sstFileManager);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void shouldSetTestPrefixExtractor() {
|
public void shouldSetTestPrefixExtractor() {
|
||||||
try (final Options options = new Options()) {
|
try (final Options options = new Options()) {
|
||||||
|
66
java/src/test/java/org/rocksdb/SstFileManagerTest.java
Normal file
66
java/src/test/java/org/rocksdb/SstFileManagerTest.java
Normal file
@ -0,0 +1,66 @@
|
|||||||
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under both the GPLv2 (found in the
|
||||||
|
// COPYING file in the root directory) and Apache 2.0 License
|
||||||
|
// (found in the LICENSE.Apache file in the root directory).
|
||||||
|
|
||||||
|
package org.rocksdb;
|
||||||
|
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
|
import java.util.Collections;
|
||||||
|
|
||||||
|
import static org.assertj.core.api.Assertions.*;
|
||||||
|
|
||||||
|
public class SstFileManagerTest {
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void maxAllowedSpaceUsage() throws RocksDBException {
|
||||||
|
try (final SstFileManager sstFileManager = new SstFileManager(Env.getDefault())) {
|
||||||
|
sstFileManager.setMaxAllowedSpaceUsage(1024 * 1024 * 64);
|
||||||
|
assertThat(sstFileManager.isMaxAllowedSpaceReached()).isFalse();
|
||||||
|
assertThat(sstFileManager.isMaxAllowedSpaceReachedIncludingCompactions()).isFalse();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void compactionBufferSize() throws RocksDBException {
|
||||||
|
try (final SstFileManager sstFileManager = new SstFileManager(Env.getDefault())) {
|
||||||
|
sstFileManager.setCompactionBufferSize(1024 * 1024 * 10);
|
||||||
|
assertThat(sstFileManager.isMaxAllowedSpaceReachedIncludingCompactions()).isFalse();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void totalSize() throws RocksDBException {
|
||||||
|
try (final SstFileManager sstFileManager = new SstFileManager(Env.getDefault())) {
|
||||||
|
assertThat(sstFileManager.getTotalSize()).isEqualTo(0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void trackedFiles() throws RocksDBException {
|
||||||
|
try (final SstFileManager sstFileManager = new SstFileManager(Env.getDefault())) {
|
||||||
|
assertThat(sstFileManager.getTrackedFiles()).isEqualTo(Collections.emptyMap());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void deleteRateBytesPerSecond() throws RocksDBException {
|
||||||
|
try (final SstFileManager sstFileManager = new SstFileManager(Env.getDefault())) {
|
||||||
|
assertThat(sstFileManager.getDeleteRateBytesPerSecond()).isEqualTo(SstFileManager.RATE_BYTES_PER_SEC_DEFAULT);
|
||||||
|
final long ratePerSecond = 1024 * 1024 * 52;
|
||||||
|
sstFileManager.setDeleteRateBytesPerSecond(ratePerSecond);
|
||||||
|
assertThat(sstFileManager.getDeleteRateBytesPerSecond()).isEqualTo(ratePerSecond);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void maxTrashDBRatio() throws RocksDBException {
|
||||||
|
try (final SstFileManager sstFileManager = new SstFileManager(Env.getDefault())) {
|
||||||
|
assertThat(sstFileManager.getMaxTrashDBRatio()).isEqualTo(SstFileManager.MAX_TRASH_DB_RATION_DEFAULT);
|
||||||
|
final double trashRatio = 0.2;
|
||||||
|
sstFileManager.setMaxTrashDBRatio(trashRatio);
|
||||||
|
assertThat(sstFileManager.getMaxTrashDBRatio()).isEqualTo(trashRatio);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
1
src.mk
1
src.mk
@ -415,6 +415,7 @@ JNI_NATIVE_SOURCES = \
|
|||||||
java/rocksjni/rocksdb_exception_test.cc \
|
java/rocksjni/rocksdb_exception_test.cc \
|
||||||
java/rocksjni/slice.cc \
|
java/rocksjni/slice.cc \
|
||||||
java/rocksjni/snapshot.cc \
|
java/rocksjni/snapshot.cc \
|
||||||
|
java/rocksjni/sst_file_manager.cc \
|
||||||
java/rocksjni/sst_file_writerjni.cc \
|
java/rocksjni/sst_file_writerjni.cc \
|
||||||
java/rocksjni/statistics.cc \
|
java/rocksjni/statistics.cc \
|
||||||
java/rocksjni/statisticsjni.cc \
|
java/rocksjni/statisticsjni.cc \
|
||||||
|
Loading…
Reference in New Issue
Block a user