andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Adding support for Windows JNI build

Browse Source
This commit is contained in:
Tomas Kolda 2016-01-28 15:44:31 +01:00
parent 9c2cf9479b
commit e2972803a6
15 changed files with 257 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
CMakeLists.txt
View File

@ -13,8 +13,8 @@
# cd build
# 3. Run cmake to generate project files for Windows, add more options to enable required third-party libraries.
# See thirdparty.inc for more information.
# sample command: cmake -G "Visual Studio 12 Win64" -DGFLAGS=1 -DSNAPPY=1 -DJEMALLOC=1 ..
# OR for VS Studio 15 cmake -G "Visual Studio 14 Win64" -DGFLAGS=1 -DSNAPPY=1 -DJEMALLOC=1 ..
# sample command: cmake -G "Visual Studio 12 Win64" -DGFLAGS=1 -DSNAPPY=1 -DJEMALLOC=1 -DJNI=1 ..
# OR for VS Studio 15 cmake -G "Visual Studio 14 Win64" -DGFLAGS=1 -DSNAPPY=1 -DJEMALLOC=1 -DJNI=1 ..
# 4. Then build the project in debug mode (you may want to add /m[:<N>] flag to run msbuild in <N> parallel threads
# or simply /m ot use all avail cores)
# msbuild rocksdb.sln
@ -243,6 +243,7 @@ set(SOURCES
util/xxhash.cc
utilities/backupable/backupable_db.cc
utilities/checkpoint/checkpoint.cc
utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc
utilities/document/document_db.cc
utilities/document/json_document.cc
utilities/document/json_document_builder.cc
@ -288,6 +289,17 @@ set_target_properties(rocksdb${ARTIFACT_SUFFIX} PROPERTIES COMPILE_FLAGS "-DROCK
add_dependencies(rocksdb${ARTIFACT_SUFFIX} GenerateBuildVersion)
target_link_libraries(rocksdb${ARTIFACT_SUFFIX} ${LIBS})
if (DEFINED JNI)
if (${JNI} EQUAL 1)
message(STATUS "JNI library is enabled")
add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/java)
else()
message(STATUS "JNI library is disabled")
endif()
else()
message(STATUS "JNI library is disabled")
endif()
set(APPS
db/db_bench.cc
db/memtablerep_bench.cc

155
java/CMakeLists.txt Normal file
View File

@ -0,0 +1,155 @@
set(JNI_NATIVE_SOURCES
rocksjni/backupenginejni.cc
rocksjni/backupablejni.cc
rocksjni/checkpoint.cc
rocksjni/columnfamilyhandle.cc
rocksjni/compaction_filter.cc
rocksjni/comparator.cc
rocksjni/comparatorjnicallback.cc
rocksjni/env.cc
rocksjni/filter.cc
rocksjni/iterator.cc
rocksjni/loggerjnicallback.cc
rocksjni/memtablejni.cc
rocksjni/merge_operator.cc
rocksjni/options.cc
rocksjni/ratelimiterjni.cc
rocksjni/remove_emptyvalue_compactionfilterjni.cc
rocksjni/restorejni.cc
rocksjni/rocksjni.cc
rocksjni/slice.cc
rocksjni/snapshot.cc
rocksjni/statistics.cc
rocksjni/table.cc
rocksjni/transaction_log.cc
rocksjni/ttl.cc
rocksjni/write_batch.cc
rocksjni/writebatchhandlerjnicallback.cc
rocksjni/write_batch_with_index.cc
rocksjni/write_batch_test.cc
)
set(NATIVE_JAVA_CLASSES
org.rocksdb.AbstractCompactionFilter
org.rocksdb.AbstractComparator
org.rocksdb.AbstractSlice
org.rocksdb.BackupEngine
org.rocksdb.BackupableDB
org.rocksdb.BackupableDBOptions
org.rocksdb.BlockBasedTableConfig
org.rocksdb.BloomFilter
org.rocksdb.Checkpoint
org.rocksdb.ColumnFamilyHandle
org.rocksdb.ColumnFamilyOptions
org.rocksdb.Comparator
org.rocksdb.ComparatorOptions
org.rocksdb.DBOptions
org.rocksdb.DirectComparator
org.rocksdb.DirectSlice
org.rocksdb.Env
org.rocksdb.FlushOptions
org.rocksdb.Filter
org.rocksdb.GenericRateLimiterConfig
org.rocksdb.HashLinkedListMemTableConfig
org.rocksdb.HashSkipListMemTableConfig
org.rocksdb.Logger
org.rocksdb.MergeOperator
org.rocksdb.Options
org.rocksdb.PlainTableConfig
org.rocksdb.ReadOptions
org.rocksdb.RemoveEmptyValueCompactionFilter
org.rocksdb.RestoreBackupableDB
org.rocksdb.RestoreOptions
org.rocksdb.RocksDB
org.rocksdb.RocksEnv
org.rocksdb.RocksIterator
org.rocksdb.RocksMemEnv
org.rocksdb.SkipListMemTableConfig
org.rocksdb.Slice
org.rocksdb.Statistics
org.rocksdb.TransactionLogIterator
org.rocksdb.TtlDB
org.rocksdb.VectorMemTableConfig
org.rocksdb.Snapshot
org.rocksdb.StringAppendOperator
org.rocksdb.WriteBatch
org.rocksdb.WriteBatch.Handler
org.rocksdb.WriteOptions
org.rocksdb.WriteBatchWithIndex
org.rocksdb.WBWIRocksIterator
org.rocksdb.WriteBatchTest
org.rocksdb.WriteBatchTestInternalHelper
)
include_directories($ENV{JAVA_HOME}/include)
include_directories($ENV{JAVA_HOME}/include/win32)
include_directories(${PROJECT_SOURCE_DIR}/java)
set(JAVA_TEST_LIBDIR ${PROJECT_SOURCE_DIR}/java/test-libs)
set(JAVA_TMP_JAR ${JAVA_TEST_LIBDIR}/tmp.jar)
set(JAVA_JUNIT_JAR ${JAVA_TEST_LIBDIR}/junit-4.12.jar)
set(JAVA_HAMCR_JAR ${JAVA_TEST_LIBDIR}/hamcrest-core-1.3.jar)
set(JAVA_MOCKITO_JAR ${JAVA_TEST_LIBDIR}/mockito-all-1.10.19.jar)
set(JAVA_CGLIB_JAR ${JAVA_TEST_LIBDIR}/cglib-2.2.2.jar)
set(JAVA_ASSERTJ_JAR ${JAVA_TEST_LIBDIR}/assertj-core-1.7.1.jar)
set(JAVA_TESTCLASSPATH "${JAVA_JUNIT_JAR}\;${JAVA_HAMCR_JAR}\;${JAVA_MOCKITO_JAR}\;${JAVA_CGLIB_JAR}\;${JAVA_ASSERTJ_JAR}")
if(NOT EXISTS ${PROJECT_SOURCE_DIR}/java/classes)
execute_process(COMMAND mkdir ${PROJECT_SOURCE_DIR}/java/classes)
endif()
if(NOT EXISTS ${JAVA_TEST_LIBDIR})
execute_process(COMMAND mkdir ${JAVA_TEST_LIBDIR})
endif()
if(NOT EXISTS ${JAVA_JUNIT_JAR})
message("Downloading ${JAVA_JUNIT_JAR}")
file(DOWNLOAD http://search.maven.org/remotecontent?filepath=junit/junit/4.12/junit-4.12.jar ${JAVA_TMP_JAR} STATUS downloadStatus)
list(GET downloadStatus 0 error_code)
if(NOT error_code EQUAL 0)
message(FATAL_ERROR "Failed downloading ${JAVA_JUNIT_JAR}")
endif()
file(RENAME ${JAVA_TMP_JAR} ${JAVA_JUNIT_JAR})
endif()
if(NOT EXISTS ${JAVA_HAMCR_JAR})
message("Downloading ${JAVA_HAMCR_JAR}")
file(DOWNLOAD http://search.maven.org/remotecontent?filepath=org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar ${JAVA_TMP_JAR} STATUS downloadStatus)
list(GET downloadStatus 0 error_code)
if(NOT error_code EQUAL 0)
message(FATAL_ERROR "Failed downloading ${JAVA_HAMCR_JAR}")
endif()
file(RENAME ${JAVA_TMP_JAR} ${JAVA_HAMCR_JAR})
endif()
if(NOT EXISTS ${JAVA_MOCKITO_JAR})
message("Downloading ${JAVA_MOCKITO_JAR}")
file(DOWNLOAD http://search.maven.org/remotecontent?filepath=org/mockito/mockito-all/1.10.19/mockito-all-1.10.19.jar ${JAVA_TMP_JAR} STATUS downloadStatus)
list(GET downloadStatus 0 error_code)
if(NOT error_code EQUAL 0)
message(FATAL_ERROR "Failed downloading ${JAVA_MOCKITO_JAR}")
endif()
file(RENAME ${JAVA_TMP_JAR} ${JAVA_MOCKITO_JAR})
endif()
if(NOT EXISTS ${JAVA_CGLIB_JAR})
message("Downloading ${JAVA_CGLIB_JAR}")
file(DOWNLOAD http://search.maven.org/remotecontent?filepath=cglib/cglib/2.2.2/cglib-2.2.2.jar ${JAVA_TMP_JAR} STATUS downloadStatus)
list(GET downloadStatus 0 error_code)
if(NOT error_code EQUAL 0)
message(FATAL_ERROR "Failed downloading ${JAVA_CGLIB_JAR}")
endif()
file(RENAME ${JAVA_TMP_JAR} ${JAVA_CGLIB_JAR})
endif()
if(NOT EXISTS ${JAVA_ASSERTJ_JAR})
message("Downloading ${JAVA_ASSERTJ_JAR}")
file(DOWNLOAD http://central.maven.org/maven2/org/assertj/assertj-core/1.7.1/assertj-core-1.7.1.jar ${JAVA_TMP_JAR} STATUS downloadStatus)
list(GET downloadStatus 0 error_code)
if(NOT error_code EQUAL 0)
message(FATAL_ERROR "Failed downloading ${JAVA_ASSERTJ_JAR}")
endif()
file(RENAME ${JAVA_TMP_JAR} ${JAVA_ASSERTJ_JAR})
endif()
execute_process(COMMAND javac -cp ${JAVA_TESTCLASSPATH} -d ${PROJECT_SOURCE_DIR}/java/classes ${PROJECT_SOURCE_DIR}/java/src/main/java/org/rocksdb/util/*.java ${PROJECT_SOURCE_DIR}/java/src/main/java/org/rocksdb/*.java ${PROJECT_SOURCE_DIR}/java/src/test/java/org/rocksdb/*.java)
execute_process(COMMAND javah -cp ${PROJECT_SOURCE_DIR}/java/classes -d ${PROJECT_SOURCE_DIR}/java/include -jni ${NATIVE_JAVA_CLASSES})
add_library(rocksdbjni${ARTIFACT_SUFFIX} SHARED ${JNI_NATIVE_SOURCES})
set_target_properties(rocksdbjni${ARTIFACT_SUFFIX} PROPERTIES COMPILE_FLAGS "/Fd${CMAKE_CFG_INTDIR}/rocksdbjni${ARTIFACT_SUFFIX}.pdb")
target_link_libraries(rocksdbjni${ARTIFACT_SUFFIX} rocksdblib${ARTIFACT_SUFFIX} ${LIBS})

12
java/rocksjni/backupablejni.cc
View File

@ -103,20 +103,14 @@ jintArray Java_org_rocksdb_BackupableDB_getCorruptedBackups(
reinterpret_cast<rocksdb::BackupableDB*>(jhandle)->
GetCorruptedBackups(&backup_ids);
// store backupids in int array
const std::vector<rocksdb::BackupID>::size_type
kIdSize = backup_ids.size();
int int_backup_ids[kIdSize];
for (std::vector<rocksdb::BackupID>::size_type i = 0;
i != kIdSize; i++) {
int_backup_ids[i] = backup_ids[i];
}
std::vector<jint> int_backup_ids(backup_ids.begin(), backup_ids.end());
// Store ints in java array
jintArray ret_backup_ids;
// Its ok to loose precision here (64->32)
jsize ret_backup_ids_size = static_cast<jsize>(kIdSize);
jsize ret_backup_ids_size = static_cast<jsize>(backup_ids.size());
ret_backup_ids = env->NewIntArray(ret_backup_ids_size);
env->SetIntArrayRegion(ret_backup_ids, 0, ret_backup_ids_size,
int_backup_ids);
int_backup_ids.data());
return ret_backup_ids;
}

12
java/rocksjni/backupenginejni.cc
View File

@ -81,20 +81,14 @@ jintArray Java_org_rocksdb_BackupEngine_getCorruptedBackups(
std::vector<rocksdb::BackupID> backup_ids;
backup_engine->GetCorruptedBackups(&backup_ids);
// store backupids in int array
const std::vector<rocksdb::BackupID>::size_type
kIdSize = backup_ids.size();
int int_backup_ids[kIdSize];
for (std::vector<rocksdb::BackupID>::size_type i = 0;
i != kIdSize; i++) {
int_backup_ids[i] = backup_ids[i];
}
std::vector<jint> int_backup_ids(backup_ids.begin(), backup_ids.end());
// Store ints in java array
jintArray ret_backup_ids;
// Its ok to loose precision here (64->32)
jsize ret_backup_ids_size = static_cast<jsize>(kIdSize);
jsize ret_backup_ids_size = static_cast<jsize>(backup_ids.size());
ret_backup_ids = env->NewIntArray(ret_backup_ids_size);
env->SetIntArrayRegion(ret_backup_ids, 0, ret_backup_ids_size,
int_backup_ids);
int_backup_ids.data());
return ret_backup_ids;
}

5
java/rocksjni/options.cc
View File

@ -8,7 +8,6 @@
#include <stdio.h>
#include <stdlib.h>
#include <jni.h>
#include <strings.h>
#include <memory>
#include "include/org_rocksdb_Options.h"
@ -1180,7 +1179,7 @@ jbyte Java_org_rocksdb_Options_compactionStyle(
void Java_org_rocksdb_Options_setMaxTableFilesSizeFIFO(
JNIEnv* env, jobject jobj, jlong jhandle, jlong jmax_table_files_size) {
reinterpret_cast<rocksdb::Options*>(jhandle)->compaction_options_fifo.max_table_files_size =
static_cast<long>(jmax_table_files_size);
static_cast<uint64_t>(jmax_table_files_size);
}
/*
@ -2339,7 +2338,7 @@ jbyte Java_org_rocksdb_ColumnFamilyOptions_compactionStyle(
void Java_org_rocksdb_ColumnFamilyOptions_setMaxTableFilesSizeFIFO(
JNIEnv* env, jobject jobj, jlong jhandle, jlong jmax_table_files_size) {
reinterpret_cast<rocksdb::ColumnFamilyOptions*>(jhandle)->compaction_options_fifo.max_table_files_size =
static_cast<long>(jmax_table_files_size);
static_cast<uint64_t>(jmax_table_files_size);
}
/*

5
java/rocksjni/portal.h
View File

@ -24,6 +24,11 @@
#include "rocksjni/loggerjnicallback.h"
#include "rocksjni/writebatchhandlerjnicallback.h"
// Remove macro on windows
#ifdef DELETE
#undef DELETE
#endif
namespace rocksdb {
// Detect if jlong overflows size_t

13
java/rocksjni/restorejni.cc
View File

@ -156,21 +156,14 @@ jintArray Java_org_rocksdb_RestoreBackupableDB_getCorruptedBackups(
reinterpret_cast<rocksdb::RestoreBackupableDB*>(jhandle)->
GetCorruptedBackups(&backup_ids);
// store backupids in int array
const std::vector<rocksdb::BackupID>::size_type
kIdSize = backup_ids.size();
int int_backup_ids[kIdSize];
for (std::vector<rocksdb::BackupID>::size_type i = 0;
i != kIdSize; i++) {
int_backup_ids[i] = backup_ids[i];
}
std::vector<jint> int_backup_ids(backup_ids.begin(), backup_ids.end());
// Store ints in java array
jintArray ret_backup_ids;
// Its ok to loose precision here (64->32)
jsize ret_backup_ids_size = static_cast<jsize>(kIdSize);
jsize ret_backup_ids_size = static_cast<jsize>(backup_ids.size());
ret_backup_ids = env->NewIntArray(ret_backup_ids_size);
env->SetIntArrayRegion(ret_backup_ids, 0, ret_backup_ids_size,
int_backup_ids);
int_backup_ids.data());
return ret_backup_ids;
}

9
java/rocksjni/rocksjni.cc
View File

@ -12,6 +12,7 @@
#include <memory>
#include <string>
#include <vector>
#include <algorithm>
#include "include/org_rocksdb_RocksDB.h"
#include "rocksdb/db.h"
@ -19,6 +20,10 @@
#include "rocksdb/types.h"
#include "rocksjni/portal.h"
#ifdef min
#undef min
#endif
//////////////////////////////////////////////////////////////////////////////
// rocksdb::DB::Open
@ -688,8 +693,8 @@ jint rocksdb_get_helper(
return kStatusError;
}
int cvalue_len = static_cast<int>(cvalue.size());
int length = std::min(jentry_value_len, cvalue_len);
jint cvalue_len = static_cast<jint>(cvalue.size());
jint length = std::min(jentry_value_len, cvalue_len);
env->SetByteArrayRegion(
jentry_value, 0, length,

4
java/rocksjni/statistics.cc
View File

@ -20,7 +20,7 @@
* Signature: (IJ)J
*/
jlong Java_org_rocksdb_Statistics_getTickerCount0(
JNIEnv* env, jobject jobj, int tickerType, jlong handle) {
JNIEnv* env, jobject jobj, jint tickerType, jlong handle) {
auto st = reinterpret_cast<rocksdb::Statistics*>(handle);
assert(st != nullptr);
@ -33,7 +33,7 @@ jlong Java_org_rocksdb_Statistics_getTickerCount0(
* Signature: (IJ)Lorg/rocksdb/HistogramData;
*/
jobject Java_org_rocksdb_Statistics_geHistogramData0(
JNIEnv* env, jobject jobj, int histogramType, jlong handle) {
JNIEnv* env, jobject jobj, jint histogramType, jlong handle) {
auto st = reinterpret_cast<rocksdb::Statistics*>(handle);
assert(st != nullptr);

16
java/src/main/java/org/rocksdb/NativeLibraryLoader.java
View File

@ -19,7 +19,7 @@ public class NativeLibraryLoader {
private static final String jniLibraryName = Environment.getJniLibraryName("rocksdb");
private static final String jniLibraryFileName = Environment.getJniLibraryFileName("rocksdb");
private static final String tempFilePrefix = "librocksdbjni";
private static final String tempFileSuffix = "." + Environment.getJniLibraryExtension();
private static final String tempFileSuffix = Environment.getJniLibraryExtension();
/**
* Get a reference to the NativeLibraryLoader
@ -75,8 +75,15 @@ public class NativeLibraryLoader {
void loadLibraryFromJar(final String tmpDir)
throws IOException {
if (!initialized) {
System.load(loadLibraryFromJarToTemp(tmpDir).getAbsolutePath());
initialized = true;
}
}
File loadLibraryFromJarToTemp(final String tmpDir)
throws IOException {
final File temp;
if (tmpDir == null || tmpDir.equals("")) {
if (tmpDir == null || tmpDir.isEmpty()) {
temp = File.createTempFile(tempFilePrefix, tempFileSuffix);
} else {
temp = new File(tmpDir, jniLibraryFileName);
@ -102,10 +109,9 @@ public class NativeLibraryLoader {
}
}
System.load(temp.getAbsolutePath());
initialized = true;
}
return temp;
}
/**
* Private constructor to disallow instantiation
*/

7
java/src/main/java/org/rocksdb/util/Environment.java
View File

@ -42,6 +42,8 @@ public class Environment {
return String.format("%sjni-osx", name);
} else if (isSolaris()) {
return String.format("%sjni-solaris%d", name, is64Bit() ? 64 : 32);
} else if (isWindows() && is64Bit()) {
return String.format("%sjni-win64", name);
}
throw new UnsupportedOperationException();
}
@ -55,11 +57,16 @@ public class Environment {
return libraryFileName + ".so";
} else if (isMac()) {
return libraryFileName + (shared ? ".dylib" : ".jnilib");
} else if (isWindows()) {
return libraryFileName + ".dll";
}
throw new UnsupportedOperationException();
}
public static String getJniLibraryExtension() {
if (isWindows()) {
return ".dll";
}
return (isMac()) ? ".jnilib" : ".so";
}
}

10
java/src/test/java/org/rocksdb/BackupableDBOptionsTest.java
View File

@ -5,18 +5,18 @@
package org.rocksdb;
import static org.assertj.core.api.Assertions.assertThat;
import java.util.Random;
import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import java.util.Random;
import static org.assertj.core.api.Assertions.assertThat;
public class BackupableDBOptionsTest {
private final static String ARBITRARY_PATH = "/tmp";
private final static String ARBITRARY_PATH = System.getProperty("java.io.tmpdir");
@ClassRule
public static final RocksMemoryResource rocksMemoryResource =

3
java/src/test/java/org/rocksdb/InfoLogLevelTest.java
View File

@ -4,6 +4,7 @@ import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.rocksdb.util.Environment;
import java.io.IOException;
@ -113,7 +114,7 @@ public class InfoLogLevelTest {
* @throws IOException if file is not found.
*/
private String getLogContentsWithoutHeader() throws IOException {
final String separator = System.getProperty("line.separator");
final String separator = Environment.isWindows() ? "\n" : System.getProperty("line.separator");
final String[] lines = new String(readAllBytes(get(
dbFolder.getRoot().getAbsolutePath()+ "/LOG"))).split(separator);

2
java/src/test/java/org/rocksdb/NativeLibraryLoaderTest.java
View File

@ -21,7 +21,7 @@ public class NativeLibraryLoaderTest {
@Test
public void tempFolder() throws IOException {
NativeLibraryLoader.getInstance().loadLibraryFromJar(
NativeLibraryLoader.getInstance().loadLibraryFromJarToTemp(
temporaryFolder.getRoot().getAbsolutePath());
Path path = Paths.get(temporaryFolder.getRoot().getAbsolutePath(),
Environment.getJniLibraryFileName("rocksdb"));

18
java/src/test/java/org/rocksdb/util/EnvironmentTest.java
View File

@ -117,16 +117,22 @@ public class EnvironmentTest {
assertThat(Environment.isWindows()).isTrue();
}
@Test(expected = UnsupportedOperationException.class)
public void failWinJniLibraryName(){
@Test
public void win64() {
setEnvironmentClassFields("win", "x64");
Environment.getJniLibraryFileName("rocksdb");
assertThat(Environment.isWindows()).isTrue();
assertThat(Environment.getJniLibraryExtension()).
isEqualTo(".dll");
assertThat(Environment.getJniLibraryFileName("rocksdb")).
isEqualTo("librocksdbjni-win64.dll");
assertThat(Environment.getSharedLibraryFileName("rocksdb")).
isEqualTo("librocksdbjni.dll");
}
@Test(expected = UnsupportedOperationException.class)
public void failWinSharedLibrary(){
setEnvironmentClassFields("win", "x64");
Environment.getSharedLibraryFileName("rocksdb");
public void win32(){
setEnvironmentClassFields("win", "32");
Environment.getJniLibraryFileName("rocksdb");
}
private void setEnvironmentClassFields(String osName,