From e96e71becf44ca45a38fd0286ef39f78c9a4a777 Mon Sep 17 00:00:00 2001 From: Ankit Gupta Date: Fri, 25 Apr 2014 13:57:20 -0700 Subject: [PATCH 1/5] Multi-get impl and formatting --- Makefile | 2 +- java/Makefile | 2 +- java/RocksDBSample.java | 2 + java/org/rocksdb/RocksDB.java | 62 +++++++++++++++++++++++++ java/rocksjni/portal.h | 64 ++++++++++++++++++++++++++ java/rocksjni/rocksjni.cc | 86 +++++++++++++++++++++++++++++++++++ 6 files changed, 216 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 80b819304..36083f7d5 100644 --- a/Makefile +++ b/Makefile @@ -434,7 +434,7 @@ ROCKSDBJNILIB = ./java/librocksdbjni.jnilib JAVA_INCLUDE = -I/System/Library/Frameworks/JavaVM.framework/Headers/ endif -rocksdbjava: clean +rocksdbjava: OPT="-fPIC -DNDEBUG -O2" $(MAKE) $(LIBRARY) -j32 cd java;$(MAKE) java; rm -f $(ROCKSDBJNILIB) diff --git a/java/Makefile b/java/Makefile index 1a199e1df..1013d99e5 100644 --- a/java/Makefile +++ b/java/Makefile @@ -16,7 +16,7 @@ sample: java javac -cp $(ROCKSDB_JAR) RocksDBSample.java @rm -rf /tmp/rocksdbjni @rm -rf /tmp/rocksdbjni_not_found - java -ea -Djava.library.path=.:../ -cp ".:./*" -Xcheck:jni RocksDBSample /tmp/rocksdbjni + java -ea -Djava.library.path=. -cp ".:./*" -Xcheck:jni RocksDBSample /tmp/rocksdbjni @rm -rf /tmp/rocksdbjni @rm -rf /tmp/rocksdbjni_not_found diff --git a/java/RocksDBSample.java b/java/RocksDBSample.java index 7e5974722..741a88422 100644 --- a/java/RocksDBSample.java +++ b/java/RocksDBSample.java @@ -4,6 +4,8 @@ // of patent rights can be found in the PATENTS file in the same directory. import java.util.Arrays; +import java.util.List; +import java.util.ArrayList; import org.rocksdb.*; import org.rocksdb.util.SizeUnit; import java.io.IOException; diff --git a/java/org/rocksdb/RocksDB.java b/java/org/rocksdb/RocksDB.java index 0b78450d5..289638d65 100644 --- a/java/org/rocksdb/RocksDB.java +++ b/java/org/rocksdb/RocksDB.java @@ -5,6 +5,9 @@ package org.rocksdb; +import java.util.List; +import java.util.Map; +import java.util.HashMap; import java.io.Closeable; import java.io.IOException; @@ -142,6 +145,7 @@ public class RocksDB { * returned if the specified key is not found. * * @param key the key retrieve the value. + * @param opt Read options. * @return a byte array storing the value associated with the input key if * any. null if it does not find the specified key. * @@ -150,6 +154,60 @@ public class RocksDB { public byte[] get(ReadOptions opt, byte[] key) throws RocksDBException { return get(nativeHandle_, opt.nativeHandle_, key, key.length); } + + /** + * Returns a map of keys for which values were found in DB. + * + * @param keys List of keys for which values need to be retrieved. + * @return Map where key of map is the key passed by user and value for map + * entry is the corresponding value in DB. + * + * @see RocksDBException + */ + public Map multiGet(List keys) + throws RocksDBException { + List values = multiGet( + nativeHandle_, keys, keys.size()); + + Map keyValueMap = new HashMap(); + for(int i = 0; i < values.size(); i++) { + if(values.get(i) == null) { + continue; + } + + keyValueMap.put(keys.get(i), values.get(i)); + } + + return keyValueMap; + } + + + /** + * Returns a map of keys for which values were found in DB. + * + * @param List of keys for which values need to be retrieved. + * @param opt Read options. + * @return Map where key of map is the key passed by user and value for map + * entry is the corresponding value in DB. + * + * @see RocksDBException + */ + public Map multiGet(ReadOptions opt, List keys) + throws RocksDBException { + List values = multiGet( + nativeHandle_, opt.nativeHandle_, keys, keys.size()); + + Map keyValueMap = new HashMap(); + for(int i = 0; i < values.size(); i++) { + if(values.get(i) == null) { + continue; + } + + keyValueMap.put(keys.get(i), values.get(i)); + } + + return keyValueMap; + } /** * Remove the database entry (if any) for "key". Returns OK on @@ -213,6 +271,10 @@ public class RocksDB { protected native int get( long handle, long readOptHandle, byte[] key, int keyLen, byte[] value, int valueLen) throws RocksDBException; + protected native List multiGet( + long dbHandle, List keys, int keysCount); + protected native List multiGet( + long dbHandle, long rOptHandle, List keys, int keysCount); protected native byte[] get( long handle, byte[] key, int keyLen) throws RocksDBException; protected native byte[] get( diff --git a/java/rocksjni/portal.h b/java/rocksjni/portal.h index 4c4444329..11764eee8 100644 --- a/java/rocksjni/portal.h +++ b/java/rocksjni/portal.h @@ -315,5 +315,69 @@ class FilterJni { reinterpret_cast(op)); } }; + +class ListJni { + public: + // Get the java class id of java.util.List. + static jclass getListClass(JNIEnv* env) { + static jclass jclazz = env->FindClass("java/util/List"); + assert(jclazz != nullptr); + return jclazz; + } + + // Get the java class id of java.util.ArrayList. + static jclass getArrayListClass(JNIEnv* env) { + static jclass jclazz = env->FindClass("java/util/ArrayList"); + assert(jclazz != nullptr); + return jclazz; + } + + // Get the java class id of java.util.Iterator. + static jclass getIteratorClass(JNIEnv* env) { + static jclass jclazz = env->FindClass("java/util/Iterator"); + assert(jclazz != nullptr); + return jclazz; + } + + // Get the java method id of java.util.List.iterator(). + static jmethodID getIteratorMethod(JNIEnv* env) { + static jmethodID mid = env->GetMethodID( + getListClass(env), "iterator", "()Ljava/util/Iterator;"); + assert(mid != nullptr); + return mid; + } + + // Get the java method id of java.util.Iterator.hasNext(). + static jmethodID getHasNextMethod(JNIEnv* env) { + static jmethodID mid = env->GetMethodID( + getIteratorClass(env), "hasNext", "()Z"); + assert(mid != nullptr); + return mid; + } + + // Get the java method id of java.util.Iterator.next(). + static jmethodID getNextMethod(JNIEnv* env) { + static jmethodID mid = env->GetMethodID( + getIteratorClass(env), "next", "()Ljava/lang/Object;"); + assert(mid != nullptr); + return mid; + } + + // Get the java method id of arrayList constructor. + static jmethodID getArrayListConstructorMethodId(JNIEnv* env, jclass jclazz) { + static jmethodID mid = env->GetMethodID( + jclazz, "", "(I)V"); + assert(mid != nullptr); + return mid; + } + + // Get the java method id of java.util.List.add(). + static jmethodID getListAddMethodId(JNIEnv* env) { + static jmethodID mid = env->GetMethodID( + getListClass(env), "add", "(Ljava/lang/Object;)Z"); + assert(mid != nullptr); + return mid; + } +}; } // namespace rocksdb #endif // JAVA_ROCKSJNI_PORTAL_H_ diff --git a/java/rocksjni/rocksjni.cc b/java/rocksjni/rocksjni.cc index 17c7b8b10..a8e6dfbb0 100644 --- a/java/rocksjni/rocksjni.cc +++ b/java/rocksjni/rocksjni.cc @@ -10,6 +10,7 @@ #include #include #include +#include #include "include/org_rocksdb_RocksDB.h" #include "rocksjni/portal.h" @@ -244,6 +245,91 @@ jint rocksdb_get_helper( return cvalue_len; } +jobject multi_get_helper(JNIEnv* env, jobject jdb, rocksdb::DB* db, + const rocksdb::ReadOptions& rOpt, jobject jkey_list, jint jkeys_count) { + std::vector keys; + std::vector keys_to_free; + + // get iterator + jobject iteratorObj = env->CallObjectMethod( + jkey_list, rocksdb::ListJni::getIteratorMethod(env)); + + // iterate over keys and convert java byte array to slice + while(env->CallBooleanMethod( + iteratorObj, rocksdb::ListJni::getHasNextMethod(env)) == JNI_TRUE) { + jbyteArray jkey = (jbyteArray) env->CallObjectMethod( + iteratorObj, rocksdb::ListJni::getNextMethod(env)); + jint key_length = env->GetArrayLength(jkey); + + jbyte* key = new jbyte[key_length]; + env->GetByteArrayRegion(jkey, 0, key_length, key); + // store allocated jbyte to free it after multiGet call + keys_to_free.push_back(key); + + rocksdb::Slice key_slice( + reinterpret_cast(key), key_length); + keys.push_back(key_slice); + } + + std::vector values; + std::vector s = db->MultiGet(rOpt, keys, &values); + + // Don't reuse class pointer + jclass jclazz = env->FindClass("java/util/ArrayList"); + jmethodID mid = rocksdb::ListJni::getArrayListConstructorMethodId( + env, jclazz); + jobject jvalue_list = env->NewObject(jclazz, mid, jkeys_count); + + // insert in java list + for(std::vector::size_type i = 0; i != s.size(); i++) { + if(s[i].ok()) { + jbyteArray jvalue = env->NewByteArray(values[i].size()); + env->SetByteArrayRegion( + jvalue, 0, values[i].size(), + reinterpret_cast(values[i].c_str())); + env->CallBooleanMethod( + jvalue_list, rocksdb::ListJni::getListAddMethodId(env), jvalue); + } + else { + env->CallBooleanMethod( + jvalue_list, rocksdb::ListJni::getListAddMethodId(env), nullptr); + } + } + + // free up allocated byte arrays + for(std::vector::size_type i = 0; i != keys_to_free.size(); i++) { + delete[] keys_to_free[i]; + } + keys_to_free.clear(); + + return jvalue_list; +} + +/* + * Class: org_rocksdb_RocksDB + * Method: multiGet + * Signature: (JLjava/util/List;I)Ljava/util/List; + */ +jobject Java_org_rocksdb_RocksDB_multiGet( + JNIEnv* env, jobject jdb, jlong jdb_handle, + jobject jkey_list, jint jkeys_count) { + return multi_get_helper(env, jdb, reinterpret_cast(jdb_handle), + rocksdb::ReadOptions(), jkey_list, jkeys_count); +} + +/* + * Class: org_rocksdb_RocksDB + * Method: multiGet + * Signature: (JJLjava/util/List;I)Ljava/util/List; + */ +jobject Java_org_rocksdb_RocksDB_multiGet( + JNIEnv* env, jobject jdb, jlong jdb_handle, + jlong jropt_handle, jobject jkey_list, jint jkeys_count) { + return multi_get_helper(env, jdb, reinterpret_cast(jdb_handle), + *reinterpret_cast(jropt_handle), jkey_list, + jkeys_count); +} + /* * Class: org_rocksdb_RocksDB * Method: get From 3f8b4129ef8f887716d3fcafa3e88b4b64eff51e Mon Sep 17 00:00:00 2001 From: Ankit Gupta Date: Fri, 25 Apr 2014 20:59:16 -0700 Subject: [PATCH 2/5] Fix formatting --- java/org/rocksdb/RocksDB.java | 38 +++++++++++++++++------------------ java/rocksjni/portal.h | 12 +++++------ java/rocksjni/rocksjni.cc | 24 +++++++++++----------- 3 files changed, 37 insertions(+), 37 deletions(-) diff --git a/java/org/rocksdb/RocksDB.java b/java/org/rocksdb/RocksDB.java index 289638d65..649433b8c 100644 --- a/java/org/rocksdb/RocksDB.java +++ b/java/org/rocksdb/RocksDB.java @@ -154,58 +154,58 @@ public class RocksDB { public byte[] get(ReadOptions opt, byte[] key) throws RocksDBException { return get(nativeHandle_, opt.nativeHandle_, key, key.length); } - + /** * Returns a map of keys for which values were found in DB. - * + * * @param keys List of keys for which values need to be retrieved. - * @return Map where key of map is the key passed by user and value for map + * @return Map where key of map is the key passed by user and value for map * entry is the corresponding value in DB. - * - * @see RocksDBException + * + * @see RocksDBException */ public Map multiGet(List keys) throws RocksDBException { List values = multiGet( nativeHandle_, keys, keys.size()); - - Map keyValueMap = new HashMap(); + + Map keyValueMap = new HashMap(); for(int i = 0; i < values.size(); i++) { if(values.get(i) == null) { continue; } - + keyValueMap.put(keys.get(i), values.get(i)); } - + return keyValueMap; } - - + + /** * Returns a map of keys for which values were found in DB. - * + * * @param List of keys for which values need to be retrieved. * @param opt Read options. - * @return Map where key of map is the key passed by user and value for map + * @return Map where key of map is the key passed by user and value for map * entry is the corresponding value in DB. - * - * @see RocksDBException + * + * @see RocksDBException */ public Map multiGet(ReadOptions opt, List keys) throws RocksDBException { List values = multiGet( nativeHandle_, opt.nativeHandle_, keys, keys.size()); - - Map keyValueMap = new HashMap(); + + Map keyValueMap = new HashMap(); for(int i = 0; i < values.size(); i++) { if(values.get(i) == null) { continue; } - + keyValueMap.put(keys.get(i), values.get(i)); } - + return keyValueMap; } diff --git a/java/rocksjni/portal.h b/java/rocksjni/portal.h index 11764eee8..7d70eecae 100644 --- a/java/rocksjni/portal.h +++ b/java/rocksjni/portal.h @@ -324,7 +324,7 @@ class ListJni { assert(jclazz != nullptr); return jclazz; } - + // Get the java class id of java.util.ArrayList. static jclass getArrayListClass(JNIEnv* env) { static jclass jclazz = env->FindClass("java/util/ArrayList"); @@ -338,7 +338,7 @@ class ListJni { assert(jclazz != nullptr); return jclazz; } - + // Get the java method id of java.util.List.iterator(). static jmethodID getIteratorMethod(JNIEnv* env) { static jmethodID mid = env->GetMethodID( @@ -346,7 +346,7 @@ class ListJni { assert(mid != nullptr); return mid; } - + // Get the java method id of java.util.Iterator.hasNext(). static jmethodID getHasNextMethod(JNIEnv* env) { static jmethodID mid = env->GetMethodID( @@ -354,7 +354,7 @@ class ListJni { assert(mid != nullptr); return mid; } - + // Get the java method id of java.util.Iterator.next(). static jmethodID getNextMethod(JNIEnv* env) { static jmethodID mid = env->GetMethodID( @@ -362,7 +362,7 @@ class ListJni { assert(mid != nullptr); return mid; } - + // Get the java method id of arrayList constructor. static jmethodID getArrayListConstructorMethodId(JNIEnv* env, jclass jclazz) { static jmethodID mid = env->GetMethodID( @@ -370,7 +370,7 @@ class ListJni { assert(mid != nullptr); return mid; } - + // Get the java method id of java.util.List.add(). static jmethodID getListAddMethodId(JNIEnv* env) { static jmethodID mid = env->GetMethodID( diff --git a/java/rocksjni/rocksjni.cc b/java/rocksjni/rocksjni.cc index a8e6dfbb0..639c54ade 100644 --- a/java/rocksjni/rocksjni.cc +++ b/java/rocksjni/rocksjni.cc @@ -249,37 +249,37 @@ jobject multi_get_helper(JNIEnv* env, jobject jdb, rocksdb::DB* db, const rocksdb::ReadOptions& rOpt, jobject jkey_list, jint jkeys_count) { std::vector keys; std::vector keys_to_free; - + // get iterator jobject iteratorObj = env->CallObjectMethod( jkey_list, rocksdb::ListJni::getIteratorMethod(env)); - + // iterate over keys and convert java byte array to slice while(env->CallBooleanMethod( iteratorObj, rocksdb::ListJni::getHasNextMethod(env)) == JNI_TRUE) { jbyteArray jkey = (jbyteArray) env->CallObjectMethod( iteratorObj, rocksdb::ListJni::getNextMethod(env)); jint key_length = env->GetArrayLength(jkey); - + jbyte* key = new jbyte[key_length]; env->GetByteArrayRegion(jkey, 0, key_length, key); // store allocated jbyte to free it after multiGet call - keys_to_free.push_back(key); - + keys_to_free.push_back(key); + rocksdb::Slice key_slice( reinterpret_cast(key), key_length); keys.push_back(key_slice); } - + std::vector values; std::vector s = db->MultiGet(rOpt, keys, &values); - + // Don't reuse class pointer - jclass jclazz = env->FindClass("java/util/ArrayList"); + jclass jclazz = env->FindClass("java/util/ArrayList"); jmethodID mid = rocksdb::ListJni::getArrayListConstructorMethodId( - env, jclazz); + env, jclazz); jobject jvalue_list = env->NewObject(jclazz, mid, jkeys_count); - + // insert in java list for(std::vector::size_type i = 0; i != s.size(); i++) { if(s[i].ok()) { @@ -295,13 +295,13 @@ jobject multi_get_helper(JNIEnv* env, jobject jdb, rocksdb::DB* db, jvalue_list, rocksdb::ListJni::getListAddMethodId(env), nullptr); } } - + // free up allocated byte arrays for(std::vector::size_type i = 0; i != keys_to_free.size(); i++) { delete[] keys_to_free[i]; } keys_to_free.clear(); - + return jvalue_list; } From 9e0ab427b8356d450312cd89ccc76b96f8247d6c Mon Sep 17 00:00:00 2001 From: Ankit Gupta Date: Fri, 25 Apr 2014 22:39:33 -0700 Subject: [PATCH 3/5] Fix bug + formatting --- java/Makefile | 2 +- java/RocksDBSample.java | 20 ++++++++++++++++++++ java/rocksjni/rocksjni.cc | 4 ++-- 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/java/Makefile b/java/Makefile index 1013d99e5..1a199e1df 100644 --- a/java/Makefile +++ b/java/Makefile @@ -16,7 +16,7 @@ sample: java javac -cp $(ROCKSDB_JAR) RocksDBSample.java @rm -rf /tmp/rocksdbjni @rm -rf /tmp/rocksdbjni_not_found - java -ea -Djava.library.path=. -cp ".:./*" -Xcheck:jni RocksDBSample /tmp/rocksdbjni + java -ea -Djava.library.path=.:../ -cp ".:./*" -Xcheck:jni RocksDBSample /tmp/rocksdbjni @rm -rf /tmp/rocksdbjni @rm -rf /tmp/rocksdbjni_not_found diff --git a/java/RocksDBSample.java b/java/RocksDBSample.java index 741a88422..f8c1c6038 100644 --- a/java/RocksDBSample.java +++ b/java/RocksDBSample.java @@ -5,6 +5,7 @@ import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.ArrayList; import org.rocksdb.*; import org.rocksdb.util.SizeUnit; @@ -219,6 +220,25 @@ public class RocksDBSample { iterator.close(); System.out.println("iterator tests passed."); + + iterator = db.newIterator(); + List keys = new ArrayList(); + for (iterator.seekToLast(); iterator.isValid(); iterator.prev()) { + keys.add(iterator.key()); + } + iterator.close(); + + Map values = db.multiGet(keys); + assert(values.size() == keys.size()); + for(byte[] value1 : values.values()) { + assert(value1 != null); + } + + values = db.multiGet(new ReadOptions(), keys); + assert(values.size() == keys.size()); + for(byte[] value1 : values.values()) { + assert(value1 != null); + } } catch (RocksDBException e) { System.err.println(e); } diff --git a/java/rocksjni/rocksjni.cc b/java/rocksjni/rocksjni.cc index 639c54ade..94c41392d 100644 --- a/java/rocksjni/rocksjni.cc +++ b/java/rocksjni/rocksjni.cc @@ -310,7 +310,7 @@ jobject multi_get_helper(JNIEnv* env, jobject jdb, rocksdb::DB* db, * Method: multiGet * Signature: (JLjava/util/List;I)Ljava/util/List; */ -jobject Java_org_rocksdb_RocksDB_multiGet( +jobject Java_org_rocksdb_RocksDB_multiGet__JLjava_util_List_2I( JNIEnv* env, jobject jdb, jlong jdb_handle, jobject jkey_list, jint jkeys_count) { return multi_get_helper(env, jdb, reinterpret_cast(jdb_handle), @@ -322,7 +322,7 @@ jobject Java_org_rocksdb_RocksDB_multiGet( * Method: multiGet * Signature: (JJLjava/util/List;I)Ljava/util/List; */ -jobject Java_org_rocksdb_RocksDB_multiGet( +jobject Java_org_rocksdb_RocksDB_multiGet__JJLjava_util_List_2I( JNIEnv* env, jobject jdb, jlong jdb_handle, jlong jropt_handle, jobject jkey_list, jint jkeys_count) { return multi_get_helper(env, jdb, reinterpret_cast(jdb_handle), From a515d236516c1555396c28d685f187aab4e4537e Mon Sep 17 00:00:00 2001 From: Ankit Gupta Date: Fri, 25 Apr 2014 22:40:16 -0700 Subject: [PATCH 4/5] Revert makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 36083f7d5..80b819304 100644 --- a/Makefile +++ b/Makefile @@ -434,7 +434,7 @@ ROCKSDBJNILIB = ./java/librocksdbjni.jnilib JAVA_INCLUDE = -I/System/Library/Frameworks/JavaVM.framework/Headers/ endif -rocksdbjava: +rocksdbjava: clean OPT="-fPIC -DNDEBUG -O2" $(MAKE) $(LIBRARY) -j32 cd java;$(MAKE) java; rm -f $(ROCKSDBJNILIB) From d5cb6e7bb5f8a76e7ecaab50b022489d8d662cec Mon Sep 17 00:00:00 2001 From: Ankit Gupta Date: Tue, 29 Apr 2014 08:02:52 -0700 Subject: [PATCH 5/5] Assert on key size in multiget --- java/org/rocksdb/RocksDB.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/java/org/rocksdb/RocksDB.java b/java/org/rocksdb/RocksDB.java index 728798ade..9ff45707f 100644 --- a/java/org/rocksdb/RocksDB.java +++ b/java/org/rocksdb/RocksDB.java @@ -171,6 +171,8 @@ public class RocksDB { */ public Map multiGet(List keys) throws RocksDBException { + assert(keys.size() != 0); + List values = multiGet( nativeHandle_, keys, keys.size()); @@ -199,6 +201,8 @@ public class RocksDB { */ public Map multiGet(ReadOptions opt, List keys) throws RocksDBException { + assert(keys.size() != 0); + List values = multiGet( nativeHandle_, opt.nativeHandle_, keys, keys.size());