Added support for SstFileReader JNI interface (#5556)

Summary:
Feature request as per https://github.com/facebook/rocksdb/issues/5538 issue.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5556

Differential Revision: D17219008

fbshipit-source-id: e31f18dec318416eac9dea8213bab31da96e1f3a
This commit is contained in:
Richard He 2019-09-09 18:11:02 -07:00 committed by Facebook Github Bot
parent 7af6ced14b
commit 699e1b5ede
8 changed files with 592 additions and 0 deletions

View File

@ -51,6 +51,8 @@ set(JNI_NATIVE_SOURCES
rocksjni/snapshot.cc
rocksjni/sst_file_manager.cc
rocksjni/sst_file_writerjni.cc
rocksjni/sst_file_readerjni.cc
rocksjni/sst_file_reader_iterator.cc
rocksjni/statistics.cc
rocksjni/statisticsjni.cc
rocksjni/table.cc
@ -194,6 +196,8 @@ set(JAVA_MAIN_CLASSES
src/main/java/org/rocksdb/SstFileManager.java
src/main/java/org/rocksdb/SstFileMetaData.java
src/main/java/org/rocksdb/SstFileWriter.java
src/main/java/org/rocksdb/SstFileReader.java
src/main/java/org/rocksdb/SstFileReaderIterator.java
src/main/java/org/rocksdb/StateType.java
src/main/java/org/rocksdb/StatisticsCollectorCallback.java
src/main/java/org/rocksdb/StatisticsCollector.java
@ -436,6 +440,8 @@ if(${CMAKE_VERSION} VERSION_LESS "3.11.4" OR (${Java_VERSION_MINOR} STREQUAL "7"
org.rocksdb.Snapshot
org.rocksdb.SstFileManager
org.rocksdb.SstFileWriter
org.rocksdb.SstFileReader
org.rocksdb.SstFileReaderIterator
org.rocksdb.Statistics
org.rocksdb.StringAppendOperator
org.rocksdb.TableFormatConfig

View File

@ -60,6 +60,8 @@ NATIVE_JAVA_CLASSES = org.rocksdb.AbstractCompactionFilter\
org.rocksdb.Slice\
org.rocksdb.SstFileManager\
org.rocksdb.SstFileWriter\
org.rocksdb.SstFileReader\
org.rocksdb.SstFileReaderIterator\
org.rocksdb.Statistics\
org.rocksdb.ThreadStatus\
org.rocksdb.TimedEnv\
@ -156,6 +158,7 @@ JAVA_TESTS = org.rocksdb.BackupableDBOptionsTest\
org.rocksdb.SnapshotTest\
org.rocksdb.SstFileManagerTest\
org.rocksdb.SstFileWriterTest\
org.rocksdb.SstFileReaderTest\
org.rocksdb.TableFilterTest\
org.rocksdb.TimedEnvTest\
org.rocksdb.TransactionTest\

View File

@ -0,0 +1,186 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
//
// This file implements the "bridge" between Java and C++ and enables
// calling c++ rocksdb::Iterator methods from Java side.
#include <stdio.h>
#include <stdlib.h>
#include <jni.h>
#include "include/org_rocksdb_SstFileReaderIterator.h"
#include "rocksjni/portal.h"
#include "rocksdb/iterator.h"
/*
* Class: org_rocksdb_SstFileReaderIterator
* Method: disposeInternal
* Signature: (J)V
*/
void Java_org_rocksdb_SstFileReaderIterator_disposeInternal(JNIEnv* /*env*/,
jobject /*jobj*/,
jlong handle) {
auto* it = reinterpret_cast<rocksdb::Iterator*>(handle);
assert(it != nullptr);
delete it;
}
/*
* Class: org_rocksdb_SstFileReaderIterator
* Method: isValid0
* Signature: (J)Z
*/
jboolean Java_org_rocksdb_SstFileReaderIterator_isValid0(JNIEnv* /*env*/,
jobject /*jobj*/,
jlong handle) {
return reinterpret_cast<rocksdb::Iterator*>(handle)->Valid();
}
/*
* Class: org_rocksdb_SstFileReaderIterator
* Method: seekToFirst0
* Signature: (J)V
*/
void Java_org_rocksdb_SstFileReaderIterator_seekToFirst0(JNIEnv* /*env*/,
jobject /*jobj*/,
jlong handle) {
reinterpret_cast<rocksdb::Iterator*>(handle)->SeekToFirst();
}
/*
* Class: org_rocksdb_SstFileReaderIterator
* Method: seekToLast0
* Signature: (J)V
*/
void Java_org_rocksdb_SstFileReaderIterator_seekToLast0(JNIEnv* /*env*/,
jobject /*jobj*/,
jlong handle) {
reinterpret_cast<rocksdb::Iterator*>(handle)->SeekToLast();
}
/*
* Class: org_rocksdb_SstFileReaderIterator
* Method: next0
* Signature: (J)V
*/
void Java_org_rocksdb_SstFileReaderIterator_next0(JNIEnv* /*env*/, jobject /*jobj*/,
jlong handle) {
reinterpret_cast<rocksdb::Iterator*>(handle)->Next();
}
/*
* Class: org_rocksdb_SstFileReaderIterator
* Method: prev0
* Signature: (J)V
*/
void Java_org_rocksdb_SstFileReaderIterator_prev0(JNIEnv* /*env*/, jobject /*jobj*/,
jlong handle) {
reinterpret_cast<rocksdb::Iterator*>(handle)->Prev();
}
/*
* Class: org_rocksdb_SstFileReaderIterator
* Method: seek0
* Signature: (J[BI)V
*/
void Java_org_rocksdb_SstFileReaderIterator_seek0(JNIEnv* env, jobject /*jobj*/,
jlong handle, jbyteArray jtarget,
jint jtarget_len) {
jbyte* target = env->GetByteArrayElements(jtarget, nullptr);
if(target == nullptr) {
// exception thrown: OutOfMemoryError
return;
}
rocksdb::Slice target_slice(
reinterpret_cast<char*>(target), jtarget_len);
auto* it = reinterpret_cast<rocksdb::Iterator*>(handle);
it->Seek(target_slice);
env->ReleaseByteArrayElements(jtarget, target, JNI_ABORT);
}
/*
* Class: org_rocksdb_SstFileReaderIterator
* Method: seekForPrev0
* Signature: (J[BI)V
*/
void Java_org_rocksdb_SstFileReaderIterator_seekForPrev0(JNIEnv* env, jobject /*jobj*/,
jlong handle,
jbyteArray jtarget,
jint jtarget_len) {
jbyte* target = env->GetByteArrayElements(jtarget, nullptr);
if(target == nullptr) {
// exception thrown: OutOfMemoryError
return;
}
rocksdb::Slice target_slice(
reinterpret_cast<char*>(target), jtarget_len);
auto* it = reinterpret_cast<rocksdb::Iterator*>(handle);
it->SeekForPrev(target_slice);
env->ReleaseByteArrayElements(jtarget, target, JNI_ABORT);
}
/*
* Class: org_rocksdb_SstFileReaderIterator
* Method: status0
* Signature: (J)V
*/
void Java_org_rocksdb_SstFileReaderIterator_status0(JNIEnv* env, jobject /*jobj*/,
jlong handle) {
auto* it = reinterpret_cast<rocksdb::Iterator*>(handle);
rocksdb::Status s = it->status();
if (s.ok()) {
return;
}
rocksdb::RocksDBExceptionJni::ThrowNew(env, s);
}
/*
* Class: org_rocksdb_SstFileReaderIterator
* Method: key0
* Signature: (J)[B
*/
jbyteArray Java_org_rocksdb_SstFileReaderIterator_key0(JNIEnv* env, jobject /*jobj*/,
jlong handle) {
auto* it = reinterpret_cast<rocksdb::Iterator*>(handle);
rocksdb::Slice key_slice = it->key();
jbyteArray jkey = env->NewByteArray(static_cast<jsize>(key_slice.size()));
if(jkey == nullptr) {
// exception thrown: OutOfMemoryError
return nullptr;
}
env->SetByteArrayRegion(jkey, 0, static_cast<jsize>(key_slice.size()),
const_cast<jbyte*>(reinterpret_cast<const jbyte*>(key_slice.data())));
return jkey;
}
/*
* Class: org_rocksdb_SstFileReaderIterator
* Method: value0
* Signature: (J)[B
*/
jbyteArray Java_org_rocksdb_SstFileReaderIterator_value0(JNIEnv* env, jobject /*jobj*/,
jlong handle) {
auto* it = reinterpret_cast<rocksdb::Iterator*>(handle);
rocksdb::Slice value_slice = it->value();
jbyteArray jkeyValue =
env->NewByteArray(static_cast<jsize>(value_slice.size()));
if(jkeyValue == nullptr) {
// exception thrown: OutOfMemoryError
return nullptr;
}
env->SetByteArrayRegion(jkeyValue, 0, static_cast<jsize>(value_slice.size()),
const_cast<jbyte*>(reinterpret_cast<const jbyte*>(value_slice.data())));
return jkeyValue;
}

View File

@ -0,0 +1,110 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
//
// This file implements the "bridge" between Java and C++ and enables
// calling C++ rocksdb::SstFileReader methods
// from Java side.
#include <jni.h>
#include <string>
#include "include/org_rocksdb_SstFileReader.h"
#include "rocksdb/comparator.h"
#include "rocksdb/env.h"
#include "rocksdb/options.h"
#include "rocksdb/sst_file_reader.h"
#include "rocksjni/portal.h"
/*
* Class: org_rocksdb_SstFileReader
* Method: newSstFileReader
* Signature: (J)J
*/
jlong Java_org_rocksdb_SstFileReader_newSstFileReader(JNIEnv * /*env*/,
jclass /*jcls*/,
jlong joptions) {
auto *options = reinterpret_cast<const rocksdb::Options *>(joptions);
rocksdb::SstFileReader *sst_file_reader =
new rocksdb::SstFileReader(*options);
return reinterpret_cast<jlong>(sst_file_reader);
}
/*
* Class: org_rocksdb_SstFileReader
* Method: open
* Signature: (JLjava/lang/String;)V
*/
void Java_org_rocksdb_SstFileReader_open(JNIEnv *env, jobject /*jobj*/,
jlong jhandle, jstring jfile_path) {
const char *file_path = env->GetStringUTFChars(jfile_path, nullptr);
if (file_path == nullptr) {
// exception thrown: OutOfMemoryError
return;
}
rocksdb::Status s =
reinterpret_cast<rocksdb::SstFileReader *>(jhandle)->Open(file_path);
env->ReleaseStringUTFChars(jfile_path, file_path);
if (!s.ok()) {
rocksdb::RocksDBExceptionJni::ThrowNew(env, s);
}
}
/*
* Class: org_rocksdb_SstFileReader
* Method: newIterator
* Signature: (JJ)J
*/
jlong Java_org_rocksdb_SstFileReader_newIterator(JNIEnv* /*env*/,
jobject /*jobj*/,
jlong jhandle,
jlong jread_options_handle) {
auto* sst_file_reader = reinterpret_cast<rocksdb::SstFileReader*>(jhandle);
auto* read_options =
reinterpret_cast<rocksdb::ReadOptions*>(jread_options_handle);
return reinterpret_cast<jlong>(sst_file_reader->NewIterator(*read_options));
}
/*
* Class: org_rocksdb_SstFileReader
* Method: disposeInternal
* Signature: (J)V
*/
void Java_org_rocksdb_SstFileReader_disposeInternal(JNIEnv * /*env*/,
jobject /*jobj*/,
jlong jhandle) {
delete reinterpret_cast<rocksdb::SstFileReader *>(jhandle);
}
/*
* Class: org_rocksdb_SstFileReader
* Method: verifyChecksum
* Signature: (J)V
*/
void Java_org_rocksdb_SstFileReader_verifyChecksum(JNIEnv *env,
jobject /*jobj*/,
jlong jhandle) {
auto* sst_file_reader = reinterpret_cast<rocksdb::SstFileReader*>(jhandle);
auto s = sst_file_reader->VerifyChecksum();
if (!s.ok()) {
rocksdb::RocksDBExceptionJni::ThrowNew(env, s);
}
}
/*
* Class: org_rocksdb_SstFileReader
* Method: getTableProperties
* Signature: (J)J
*/
jobject Java_org_rocksdb_SstFileReader_getTableProperties(JNIEnv *env,
jobject /*jobj*/,
jlong jhandle) {
auto* sst_file_reader = reinterpret_cast<rocksdb::SstFileReader*>(jhandle);
std::shared_ptr<const rocksdb::TableProperties> tp = sst_file_reader->GetTableProperties();
jobject jtable_properties = rocksdb::TablePropertiesJni::fromCppTableProperties(
env, *(tp.get()));
return jtable_properties;
}

View File

@ -0,0 +1,83 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
package org.rocksdb;
public class SstFileReader extends RocksObject {
static {
RocksDB.loadLibrary();
}
public SstFileReader(final Options options) {
super(newSstFileReader(options.nativeHandle_));
}
/**
* Returns an iterator that will iterate on all keys in the default
* column family including both keys in the DB and uncommitted keys in this
* transaction.
*
* Setting {@link ReadOptions#setSnapshot(Snapshot)} will affect what is read
* from the DB but will NOT change which keys are read from this transaction
* (the keys in this transaction do not yet belong to any snapshot and will be
* fetched regardless).
*
* Caller is responsible for deleting the returned Iterator.
*
* @param readOptions Read options.
*
* @return instance of iterator object.
*/
public SstFileReaderIterator newIterator(final ReadOptions readOptions) {
assert(isOwningHandle());
long iter = newIterator(nativeHandle_,
readOptions.nativeHandle_);
return new SstFileReaderIterator(this, iter);
}
/**
* Prepare SstFileReader to read a file.
*
* @param filePath the location of file
*
* @throws RocksDBException thrown if error happens in underlying
* native library.
*/
public void open(final String filePath) throws RocksDBException {
open(nativeHandle_, filePath);
}
/**
* Verify checksum
*
* @throws RocksDBException if the checksum is not valid
*/
public void verifyChecksum() throws RocksDBException {
verifyChecksum(nativeHandle_);
}
/**
* Get the properties of the table.
*
*
* @return the properties
*/
public TableProperties getTableProperties() throws RocksDBException {
return getTableProperties(nativeHandle_);
}
@Override protected final native void disposeInternal(final long handle);
private native long newIterator(final long handle,
final long readOptionsHandle);
private native void open(final long handle, final String filePath)
throws RocksDBException;
private native static long newSstFileReader(final long optionsHandle);
private native void verifyChecksum(final long handle) throws RocksDBException;
private native TableProperties getTableProperties(final long handle) throws RocksDBException;
}

View File

@ -0,0 +1,65 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
package org.rocksdb;
/**
* <p>An iterator that yields a sequence of key/value pairs from a source.
* Multiple implementations are provided by this library.
* In particular, iterators are provided
* to access the contents of a Table or a DB.</p>
*
* <p>Multiple threads can invoke const methods on an RocksIterator without
* external synchronization, but if any of the threads may call a
* non-const method, all threads accessing the same RocksIterator must use
* external synchronization.</p>
*
* @see RocksObject
*/
public class SstFileReaderIterator extends AbstractRocksIterator<SstFileReader> {
protected SstFileReaderIterator(SstFileReader reader, long nativeHandle) {
super(reader, nativeHandle);
}
/**
* <p>Return the key for the current entry. The underlying storage for
* the returned slice is valid only until the next modification of
* the iterator.</p>
*
* <p>REQUIRES: {@link #isValid()}</p>
*
* @return key for the current entry.
*/
public byte[] key() {
assert(isOwningHandle());
return key0(nativeHandle_);
}
/**
* <p>Return the value for the current entry. The underlying storage for
* the returned slice is valid only until the next modification of
* the iterator.</p>
*
* <p>REQUIRES: !AtEnd() &amp;&amp; !AtStart()</p>
* @return value for the current entry.
*/
public byte[] value() {
assert(isOwningHandle());
return value0(nativeHandle_);
}
@Override protected final native void disposeInternal(final long handle);
@Override final native boolean isValid0(long handle);
@Override final native void seekToFirst0(long handle);
@Override final native void seekToLast0(long handle);
@Override final native void next0(long handle);
@Override final native void prev0(long handle);
@Override final native void seek0(long handle, byte[] target, int targetLen);
@Override final native void seekForPrev0(long handle, byte[] target, int targetLen);
@Override final native void status0(long handle) throws RocksDBException;
private native byte[] key0(long handle);
private native byte[] value0(long handle);
}

View File

@ -0,0 +1,137 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
package org.rocksdb;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.rocksdb.util.BytewiseComparator;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
public class SstFileReaderTest {
private static final String SST_FILE_NAME = "test.sst";
class KeyValueWithOp {
KeyValueWithOp(String key, String value, OpType opType) {
this.key = key;
this.value = value;
this.opType = opType;
}
String getKey() {
return key;
}
String getValue() {
return value;
}
OpType getOpType() {
return opType;
}
private String key;
private String value;
private OpType opType;
}
@Rule public TemporaryFolder parentFolder = new TemporaryFolder();
enum OpType { PUT, PUT_BYTES, MERGE, MERGE_BYTES, DELETE, DELETE_BYTES}
private File newSstFile(final List<KeyValueWithOp> keyValues) throws IOException, RocksDBException {
final EnvOptions envOptions = new EnvOptions();
final StringAppendOperator stringAppendOperator = new StringAppendOperator();
final Options options = new Options().setMergeOperator(stringAppendOperator);
SstFileWriter sstFileWriter;
sstFileWriter = new SstFileWriter(envOptions, options);
final File sstFile = parentFolder.newFile(SST_FILE_NAME);
try {
sstFileWriter.open(sstFile.getAbsolutePath());
for (KeyValueWithOp keyValue : keyValues) {
Slice keySlice = new Slice(keyValue.getKey());
Slice valueSlice = new Slice(keyValue.getValue());
byte[] keyBytes = keyValue.getKey().getBytes();
byte[] valueBytes = keyValue.getValue().getBytes();
switch (keyValue.getOpType()) {
case PUT:
sstFileWriter.put(keySlice, valueSlice);
break;
case PUT_BYTES:
sstFileWriter.put(keyBytes, valueBytes);
break;
case MERGE:
sstFileWriter.merge(keySlice, valueSlice);
break;
case MERGE_BYTES:
sstFileWriter.merge(keyBytes, valueBytes);
break;
case DELETE:
sstFileWriter.delete(keySlice);
break;
case DELETE_BYTES:
sstFileWriter.delete(keyBytes);
break;
default:
fail("Unsupported op type");
}
keySlice.close();
valueSlice.close();
}
sstFileWriter.finish();
} finally {
assertThat(sstFileWriter).isNotNull();
sstFileWriter.close();
options.close();
envOptions.close();
}
return sstFile;
}
@Test
public void readSstFile() throws RocksDBException, IOException {
final List<KeyValueWithOp> keyValues = new ArrayList<>();
keyValues.add(new KeyValueWithOp("key1", "value1", OpType.PUT));
final File sstFile = newSstFile(keyValues);
try(final StringAppendOperator stringAppendOperator =
new StringAppendOperator();
final Options options = new Options()
.setCreateIfMissing(true)
.setMergeOperator(stringAppendOperator);
final SstFileReader reader = new SstFileReader(options)
) {
// Open the sst file and iterator
reader.open(sstFile.getAbsolutePath());
final ReadOptions readOptions = new ReadOptions();
final SstFileReaderIterator iterator = reader.newIterator(readOptions);
// Use the iterator to read sst file
iterator.seekToFirst();
// Verify Checksum
reader.verifyChecksum();
// Verify Table Properties
assertEquals(reader.getTableProperties().getNumEntries(), 1);
// Check key and value
assertThat(iterator.key()).isEqualTo("key1".getBytes());
assertThat(iterator.value()).isEqualTo("value1".getBytes());
}
}
}

2
src.mk
View File

@ -476,6 +476,8 @@ JNI_NATIVE_SOURCES = \
java/rocksjni/snapshot.cc \
java/rocksjni/sst_file_manager.cc \
java/rocksjni/sst_file_writerjni.cc \
java/rocksjni/sst_file_readerjni.cc \
java/rocksjni/sst_file_reader_iterator.cc \
java/rocksjni/statistics.cc \
java/rocksjni/statisticsjni.cc \
java/rocksjni/table.cc \