From 036e323f7a30d3353cfac78b02479d3a241e49c4 Mon Sep 17 00:00:00 2001 From: Ankit Gupta Date: Tue, 13 May 2014 22:20:58 -0700 Subject: [PATCH 1/8] Add RestoreBackupableDB and RestoreOptions --- java/org/rocksdb/RestoreBackupableDB.java | 85 ++++++++++++ java/org/rocksdb/RestoreOptions.java | 38 ++++++ java/rocksjni/restorejni.cc | 149 ++++++++++++++++++++++ 3 files changed, 272 insertions(+) create mode 100644 java/org/rocksdb/RestoreBackupableDB.java create mode 100644 java/org/rocksdb/RestoreOptions.java create mode 100644 java/rocksjni/restorejni.cc diff --git a/java/org/rocksdb/RestoreBackupableDB.java b/java/org/rocksdb/RestoreBackupableDB.java new file mode 100644 index 000000000..117881f33 --- /dev/null +++ b/java/org/rocksdb/RestoreBackupableDB.java @@ -0,0 +1,85 @@ +// Copyright (c) 2014, Facebook, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +package org.rocksdb; + +/** + * This class is used to access information about backups and restore from them. + * + * Note that dispose() must be called before this instance become out-of-scope + * to release the allocated memory in c++. + * + * @param options Instance of BackupableDBOptions. + */ +public class RestoreBackupableDB extends RocksObject { + public RestoreBackupableDB(BackupableDBOptions options) { + super(); + newRestoreBackupableDB(options.nativeHandle_); + } + + /** + * Restore from backup with backup_id + * IMPORTANT -- if options_.share_table_files == true and you restore DB + * from some backup that is not the latest, and you start creating new + * backups from the new DB, they will probably fail. + * + * Example: Let's say you have backups 1, 2, 3, 4, 5 and you restore 3. + * If you add new data to the DB and try creating a new backup now, the + * database will diverge from backups 4 and 5 and the new backup will fail. + * If you want to create new backup, you will first have to delete backups 4 + * and 5. + */ + public void restoreDBFromBackup(long backupId, String dbDir, String walDir, + RestoreOptions restoreOptions) throws RocksDBException { + restoreDBFromBackup0(nativeHandle_, backupId, dbDir, walDir, + restoreOptions.nativeHandle_); + } + + /** + * Restore from the latest backup. + */ + public void restoreDBFromLatestBackup(String dbDir, String walDir, + RestoreOptions restoreOptions) throws RocksDBException { + restoreDBFromLatestBackup0(nativeHandle_, dbDir, walDir, + restoreOptions.nativeHandle_); + } + + /** + * Deletes old backups, keeping latest numBackupsToKeep alive. + * + * @param Number of latest backups to keep + */ + public void purgeOldBackups(int numBackupsToKeep) throws RocksDBException { + purgeOldBackups0(nativeHandle_, numBackupsToKeep); + } + + /** + * Deletes a specific backup. + * + * @param ID of backup to delete. + */ + public void deleteBackup(long backupId) throws RocksDBException { + deleteBackup0(nativeHandle_, backupId); + } + + /** + * Release the memory allocated for the current instance + * in the c++ side. + */ + @Override public synchronized void dispose() { + if (isInitialized()) { + dispose(nativeHandle_); + } + } + + private native void newRestoreBackupableDB(long options); + private native void restoreDBFromBackup0(long nativeHandle, long backupId, + String dbDir, String walDir, long restoreOptions) throws RocksDBException; + private native void restoreDBFromLatestBackup0(long nativeHandle, + String dbDir, String walDir, long restoreOptions) throws RocksDBException; + private native void purgeOldBackups0(long nativeHandle, int numBackupsToKeep); + private native void deleteBackup0(long nativeHandle, long backupId); + private native void dispose(long nativeHandle); +} \ No newline at end of file diff --git a/java/org/rocksdb/RestoreOptions.java b/java/org/rocksdb/RestoreOptions.java new file mode 100644 index 000000000..a451bd2dd --- /dev/null +++ b/java/org/rocksdb/RestoreOptions.java @@ -0,0 +1,38 @@ +// Copyright (c) 2014, Facebook, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +package org.rocksdb; + +/** + * RestoreOptions to control the behavior of restore. + * + * Note that dispose() must be called before this instance become out-of-scope + * to release the allocated memory in c++. + * + * @param If true, restore won't overwrite the existing log files in wal_dir. It + * will also move all log files from archive directory to wal_dir. Use this + * option in combination with BackupableDBOptions::backup_log_files = false + * for persisting in-memory databases. + * Default: false + */ +public class RestoreOptions extends RocksObject { + public RestoreOptions(boolean keepLogFiles) { + super(); + newRestoreOptions(keepLogFiles); + } + + /** + * Release the memory allocated for the current instance + * in the c++ side. + */ + @Override public synchronized void dispose() { + if (isInitialized()) { + dispose(nativeHandle_); + } + } + + private native void newRestoreOptions(boolean keepLogFiles); + private native void dispose(long handle); +} \ No newline at end of file diff --git a/java/rocksjni/restorejni.cc b/java/rocksjni/restorejni.cc new file mode 100644 index 000000000..185b780a5 --- /dev/null +++ b/java/rocksjni/restorejni.cc @@ -0,0 +1,149 @@ +// Copyright (c) 2014, Facebook, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. +// +// This file implements the "bridge" between Java and C++ and enables +// calling c++ rocksdb::RestoreBackupableDB and rocksdb::RestoreOptions methods +// from Java side. + +#include +#include +#include +#include +#include + +#include "include/org_rocksdb_RestoreOptions.h" +#include "include/org_rocksdb_RestoreBackupableDB.h" +#include "rocksjni/portal.h" +#include "utilities/backupable_db.h" +/* + * Class: org_rocksdb_RestoreOptions + * Method: newRestoreOptions + * Signature: (Z)V + */ +void Java_org_rocksdb_RestoreOptions_newRestoreOptions(JNIEnv* env, + jobject jobj, jboolean keep_log_files) { + auto ropt = new rocksdb::RestoreOptions(keep_log_files); + rocksdb::RestoreOptionsJni::setHandle(env, jobj, ropt); +} + +/* + * Class: org_rocksdb_RestoreOptions + * Method: dispose + * Signature: (J)V + */ +void Java_org_rocksdb_RestoreOptions_dispose(JNIEnv* env, jobject jobj, + jlong jhandle) { + auto ropt = reinterpret_cast(jhandle); + assert(ropt); + delete ropt; + + rocksdb::RestoreOptionsJni::setHandle(env, jobj, nullptr); +} + +/* + * Class: org_rocksdb_RestoreBackupableDB + * Method: newRestoreBackupableDB + * Signature: (J)V + */ +void Java_org_rocksdb_RestoreBackupableDB_newRestoreBackupableDB(JNIEnv* env, + jobject jobj, jlong jopt_handle) { + auto opt = reinterpret_cast(jopt_handle); + auto rdb = new rocksdb::RestoreBackupableDB(rocksdb::Env::Default(), *opt); + rocksdb::RestoreBackupableDBJni::setHandle(env, jobj, rdb); +} + +/* + * Class: org_rocksdb_RestoreBackupableDB + * Method: restoreDBFromBackup0 + * Signature: (JJLjava/lang/String;Ljava/lang/String;J)V + */ +void Java_org_rocksdb_RestoreBackupableDB_restoreDBFromBackup0(JNIEnv* env, + jobject jobj, jlong jhandle, jlong jbackup_id, jstring jdb_dir, + jstring jwal_dir, jlong jopt_handle) { + auto opt = reinterpret_cast(jopt_handle); + + const char* cdb_dir = env->GetStringUTFChars(jdb_dir, 0); + const char* cwal_dir = env->GetStringUTFChars(jwal_dir, 0); + + auto rdb = reinterpret_cast(jhandle); + rocksdb::Status s = + rdb->RestoreDBFromBackup(jbackup_id, cdb_dir, cwal_dir, *opt); + + env->ReleaseStringUTFChars(jdb_dir, cdb_dir); + env->ReleaseStringUTFChars(jwal_dir, cwal_dir); + + if(!s.ok()) { + rocksdb::RocksDBExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_RestoreBackupableDB + * Method: restoreDBFromLatestBackup0 + * Signature: (JLjava/lang/String;Ljava/lang/String;J)V + */ +void Java_org_rocksdb_RestoreBackupableDB_restoreDBFromLatestBackup0( + JNIEnv* env, jobject jobj, jlong jhandle, jstring jdb_dir, jstring jwal_dir, + jlong jopt_handle) { + auto opt = reinterpret_cast(jopt_handle); + + const char* cdb_dir = env->GetStringUTFChars(jdb_dir, 0); + const char* cwal_dir = env->GetStringUTFChars(jwal_dir, 0); + + auto rdb = reinterpret_cast(jhandle); + rocksdb::Status s = + rdb->RestoreDBFromLatestBackup(cdb_dir, cwal_dir, *opt); + + env->ReleaseStringUTFChars(jdb_dir, cdb_dir); + env->ReleaseStringUTFChars(jwal_dir, cwal_dir); + + if(!s.ok()) { + rocksdb::RocksDBExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_RestoreBackupableDB + * Method: purgeOldBackups0 + * Signature: (JI)V + */ +void Java_org_rocksdb_RestoreBackupableDB_purgeOldBackups0(JNIEnv* env, + jobject jobj, jlong jhandle, jint jnum_backups_to_keep) { + auto rdb = reinterpret_cast(jhandle); + rocksdb::Status s = rdb->PurgeOldBackups(jnum_backups_to_keep); + + if(!s.ok()) { + rocksdb::RocksDBExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_RestoreBackupableDB + * Method: deleteBackup0 + * Signature: (JJ)V + */ +void Java_org_rocksdb_RestoreBackupableDB_deleteBackup0(JNIEnv* env, + jobject jobj, jlong jhandle, jlong jbackup_id) { + auto rdb = reinterpret_cast(jhandle); + rocksdb::Status s = rdb->DeleteBackup(jbackup_id); + + if(!s.ok()) { + rocksdb::RocksDBExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_RestoreBackupableDB + * Method: dispose + * Signature: (J)V + */ +void Java_org_rocksdb_RestoreBackupableDB_dispose(JNIEnv* env, jobject jobj, + jlong jhandle) { + auto ropt = reinterpret_cast(jhandle); + assert(ropt); + delete ropt; + + rocksdb::RestoreBackupableDBJni::setHandle(env, jobj, nullptr); +} \ No newline at end of file From cf0b773a2964dcc863b7980e3f3b9eaf4b574c22 Mon Sep 17 00:00:00 2001 From: Ankit Gupta Date: Tue, 13 May 2014 22:22:21 -0700 Subject: [PATCH 2/8] Add more options to backupable DB --- java/Makefile | 3 +- java/org/rocksdb/BackupableDBOptions.java | 11 +++- java/org/rocksdb/test/BackupableDBTest.java | 28 +++++++-- java/rocksjni/backupablejni.cc | 13 +++- java/rocksjni/portal.h | 67 +++++++++++++++++++++ util/hash_cuckoo_rep.cc | 1 - 6 files changed, 111 insertions(+), 12 deletions(-) diff --git a/java/Makefile b/java/Makefile index feacb8346..d124689f6 100644 --- a/java/Makefile +++ b/java/Makefile @@ -1,4 +1,5 @@ -NATIVE_JAVA_CLASSES = org.rocksdb.RocksDB org.rocksdb.Options org.rocksdb.WriteBatch org.rocksdb.WriteBatchInternal org.rocksdb.WriteBatchTest org.rocksdb.WriteOptions org.rocksdb.BackupableDB org.rocksdb.BackupableDBOptions org.rocksdb.Statistics org.rocksdb.RocksIterator org.rocksdb.VectorMemTableConfig org.rocksdb.SkipListMemTableConfig org.rocksdb.HashLinkedListMemTableConfig org.rocksdb.HashSkipListMemTableConfig org.rocksdb.PlainTableConfig org.rocksdb.ReadOptions org.rocksdb.Filter org.rocksdb.BloomFilter +NATIVE_JAVA_CLASSES = org.rocksdb.RocksDB org.rocksdb.Options org.rocksdb.WriteBatch org.rocksdb.WriteBatchInternal org.rocksdb.WriteBatchTest org.rocksdb.WriteOptions org.rocksdb.BackupableDB org.rocksdb.BackupableDBOptions org.rocksdb.Statistics org.rocksdb.RocksIterator org.rocksdb.VectorMemTableConfig org.rocksdb.SkipListMemTableConfig org.rocksdb.HashLinkedListMemTableConfig org.rocksdb.HashSkipListMemTableConfig org.rocksdb.PlainTableConfig org.rocksdb.ReadOptions org.rocksdb.Filter org.rocksdb.BloomFilter org.rocksdb.RestoreOptions org.rocksdb.RestoreBackupableDB + NATIVE_INCLUDE = ./include ROCKSDB_JAR = rocksdbjni.jar diff --git a/java/org/rocksdb/BackupableDBOptions.java b/java/org/rocksdb/BackupableDBOptions.java index 2c64b60a3..18c3953f4 100644 --- a/java/org/rocksdb/BackupableDBOptions.java +++ b/java/org/rocksdb/BackupableDBOptions.java @@ -13,9 +13,12 @@ package org.rocksdb; * become out-of-scope to release the allocated memory in c++. */ public class BackupableDBOptions extends RocksObject { - public BackupableDBOptions(String path) { + public BackupableDBOptions(String path, boolean shareTableFiles, boolean sync, + boolean destroyOldData, boolean backupLogFiles, long backupRateLimit, + long restoreRateLimit) { super(); - newBackupableDBOptions(path); + newBackupableDBOptions(path, shareTableFiles, sync, destroyOldData, + backupLogFiles, backupRateLimit, restoreRateLimit); } /** @@ -38,7 +41,9 @@ public class BackupableDBOptions extends RocksObject { } } - private native void newBackupableDBOptions(String path); + private native void newBackupableDBOptions(String path, + boolean shareTableFiles, boolean sync, boolean destroyOldData, + boolean backupLogFiles, long backupRateLimit, long restoreRateLimit); private native String backupDir(long handle); private native void dispose(long handle); } diff --git a/java/org/rocksdb/test/BackupableDBTest.java b/java/org/rocksdb/test/BackupableDBTest.java index f0fc3d501..3ef88dc8c 100644 --- a/java/org/rocksdb/test/BackupableDBTest.java +++ b/java/org/rocksdb/test/BackupableDBTest.java @@ -18,15 +18,35 @@ public class BackupableDBTest { Options opt = new Options(); opt.setCreateIfMissing(true); - BackupableDBOptions bopt = new BackupableDBOptions(backup_path); + BackupableDBOptions bopt = new BackupableDBOptions(backup_path, false, + true, false, true, 0, 0); BackupableDB bdb = null; try { bdb = BackupableDB.open(opt, bopt, db_path); - bdb.put("hello".getBytes(), "BackupableDB".getBytes()); + + bdb.put("abc".getBytes(), "def".getBytes()); + bdb.put("ghi".getBytes(), "jkl".getBytes()); bdb.createNewBackup(true); - byte[] value = bdb.get("hello".getBytes()); - assert(new String(value).equals("BackupableDB")); + + // delete record after backup + bdb.remove("abc".getBytes()); + byte[] value = bdb.get("abc".getBytes()); + assert(value == null); + bdb.close(); + + // restore from backup + RestoreBackupableDB rdb = new RestoreBackupableDB(bopt); + rdb.restoreDBFromLatestBackup(db_path, db_path, + new RestoreOptions(false)); + rdb.dispose(); + + // verify that backed up data contains deleted record + bdb = BackupableDB.open(opt, bopt, db_path); + value = bdb.get("abc".getBytes()); + assert(new String(value).equals("def")); + + System.out.println("Backup and restore test passed"); } catch (RocksDBException e) { System.err.format("[ERROR]: %s%n", e); e.printStackTrace(); diff --git a/java/rocksjni/backupablejni.cc b/java/rocksjni/backupablejni.cc index 8b57a0c62..9fee7cfcc 100644 --- a/java/rocksjni/backupablejni.cc +++ b/java/rocksjni/backupablejni.cc @@ -4,7 +4,8 @@ // of patent rights can be found in the PATENTS file in the same directory. // // This file implements the "bridge" between Java and C++ and enables -// calling c++ rocksdb::DB methods from Java side. +// calling c++ rocksdb::BackupableDB and rocksdb::BackupableDBOptions methods +// from Java side. #include #include @@ -51,9 +52,15 @@ void Java_org_rocksdb_BackupableDB_createNewBackup( * Signature: (Ljava/lang/String;)V */ void Java_org_rocksdb_BackupableDBOptions_newBackupableDBOptions( - JNIEnv* env, jobject jobj, jstring jpath) { + JNIEnv* env, jobject jobj, jstring jpath, jboolean jshare_table_files, + jboolean jsync, jboolean jdestroy_old_data, jboolean jbackup_log_files, + jlong jbackup_rate_limit, jlong jrestore_rate_limit) { const char* cpath = env->GetStringUTFChars(jpath, 0); - auto bopt = new rocksdb::BackupableDBOptions(cpath); + + auto bopt = new rocksdb::BackupableDBOptions(cpath, nullptr, + jshare_table_files, nullptr, jsync, jdestroy_old_data, jbackup_log_files, + jbackup_rate_limit, jrestore_rate_limit); + env->ReleaseStringUTFChars(jpath, cpath); rocksdb::BackupableDBOptionsJni::setHandle(env, jobj, bopt); diff --git a/java/rocksjni/portal.h b/java/rocksjni/portal.h index bd37bffcf..181b67705 100644 --- a/java/rocksjni/portal.h +++ b/java/rocksjni/portal.h @@ -217,6 +217,7 @@ class HistogramDataJni { return mid; } }; + class BackupableDBOptionsJni { public: // Get the java class id of org.rocksdb.BackupableDBOptions. @@ -250,6 +251,72 @@ class BackupableDBOptionsJni { } }; +class RestoreOptionsJni { + public: + // Get the java class id of org.rocksdb.RestoreOptions. + static jclass getJClass(JNIEnv* env) { + static jclass jclazz = env->FindClass("org/rocksdb/RestoreOptions"); + assert(jclazz != nullptr); + return jclazz; + } + + // Get the field id of the member variable of org.rocksdb.RestoreOptions + // that stores the pointer to rocksdb::RestoreOptions + static jfieldID getHandleFieldID(JNIEnv* env) { + static jfieldID fid = env->GetFieldID( + getJClass(env), "nativeHandle_", "J"); + assert(fid != nullptr); + return fid; + } + + // Get the pointer to rocksdb::RestoreOptions + static rocksdb::RestoreOptions* getHandle(JNIEnv* env, jobject jobj) { + return reinterpret_cast( + env->GetLongField(jobj, getHandleFieldID(env))); + } + + // Pass the rocksdb::RestoreOptions pointer to the java side. + static void setHandle( + JNIEnv* env, jobject jobj, rocksdb::RestoreOptions* op) { + env->SetLongField( + jobj, getHandleFieldID(env), + reinterpret_cast(op)); + } +}; + +class RestoreBackupableDBJni { + public: + // Get the java class id of org.rocksdb.RestoreBackupableDB. + static jclass getJClass(JNIEnv* env) { + static jclass jclazz = env->FindClass("org/rocksdb/RestoreBackupableDB"); + assert(jclazz != nullptr); + return jclazz; + } + + // Get the field id of the member variable of org.rocksdb.RestoreBackupableDB + // that stores the pointer to rocksdb::RestoreBackupableDB + static jfieldID getHandleFieldID(JNIEnv* env) { + static jfieldID fid = env->GetFieldID( + getJClass(env), "nativeHandle_", "J"); + assert(fid != nullptr); + return fid; + } + + // Get the pointer to rocksdb::RestoreBackupableDB + static rocksdb::RestoreBackupableDB* getHandle(JNIEnv* env, jobject jobj) { + return reinterpret_cast( + env->GetLongField(jobj, getHandleFieldID(env))); + } + + // Pass the rocksdb::RestoreBackupableDB pointer to the java side. + static void setHandle( + JNIEnv* env, jobject jobj, rocksdb::RestoreBackupableDB* op) { + env->SetLongField( + jobj, getHandleFieldID(env), + reinterpret_cast(op)); + } +}; + class IteratorJni { public: // Get the java class id of org.rocksdb.Iteartor. diff --git a/util/hash_cuckoo_rep.cc b/util/hash_cuckoo_rep.cc index a8864692f..0161690dd 100644 --- a/util/hash_cuckoo_rep.cc +++ b/util/hash_cuckoo_rep.cc @@ -1,4 +1,3 @@ - // Copyright (c) 2014, Facebook, Inc. All rights reserved. // This source code is licensed under the BSD-style license found in the // LICENSE file in the root directory of this source tree. An additional grant From b830bb937f9f323dde43a4ca784a1fff6b051d5a Mon Sep 17 00:00:00 2001 From: Ankit Gupta Date: Tue, 13 May 2014 22:34:29 -0700 Subject: [PATCH 3/8] Add newline at end of file --- java/org/rocksdb/RestoreBackupableDB.java | 2 +- java/org/rocksdb/RestoreOptions.java | 2 +- java/rocksjni/restorejni.cc | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/java/org/rocksdb/RestoreBackupableDB.java b/java/org/rocksdb/RestoreBackupableDB.java index 117881f33..f6a4ed0ab 100644 --- a/java/org/rocksdb/RestoreBackupableDB.java +++ b/java/org/rocksdb/RestoreBackupableDB.java @@ -82,4 +82,4 @@ public class RestoreBackupableDB extends RocksObject { private native void purgeOldBackups0(long nativeHandle, int numBackupsToKeep); private native void deleteBackup0(long nativeHandle, long backupId); private native void dispose(long nativeHandle); -} \ No newline at end of file +} diff --git a/java/org/rocksdb/RestoreOptions.java b/java/org/rocksdb/RestoreOptions.java index a451bd2dd..51ffcbc73 100644 --- a/java/org/rocksdb/RestoreOptions.java +++ b/java/org/rocksdb/RestoreOptions.java @@ -35,4 +35,4 @@ public class RestoreOptions extends RocksObject { private native void newRestoreOptions(boolean keepLogFiles); private native void dispose(long handle); -} \ No newline at end of file +} diff --git a/java/rocksjni/restorejni.cc b/java/rocksjni/restorejni.cc index 185b780a5..d351d4066 100644 --- a/java/rocksjni/restorejni.cc +++ b/java/rocksjni/restorejni.cc @@ -146,4 +146,4 @@ void Java_org_rocksdb_RestoreBackupableDB_dispose(JNIEnv* env, jobject jobj, delete ropt; rocksdb::RestoreBackupableDBJni::setHandle(env, jobj, nullptr); -} \ No newline at end of file +} From 29eef1f87de3a5b7a5297447bc48dd709bdc377e Mon Sep 17 00:00:00 2001 From: Ankit Gupta Date: Tue, 13 May 2014 22:40:01 -0700 Subject: [PATCH 4/8] Add javadoc for BackupableDBOptions constructor params --- java/org/rocksdb/BackupableDBOptions.java | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/java/org/rocksdb/BackupableDBOptions.java b/java/org/rocksdb/BackupableDBOptions.java index 18c3953f4..8bcc2cf18 100644 --- a/java/org/rocksdb/BackupableDBOptions.java +++ b/java/org/rocksdb/BackupableDBOptions.java @@ -11,6 +11,27 @@ package org.rocksdb; * * Note that dispose() must be called before an Options instance * become out-of-scope to release the allocated memory in c++. + * + * @param path Where to keep the backup files. Has to be different than dbname. + Best to set this to dbname_ + "/backups" + * @param shareTableFiles If share_table_files == true, backup will assume that + * table files with same name have the same contents. This enables + * incremental backups and avoids unnecessary data copies. If + * share_table_files == false, each backup will be on its own and will not + * share any data with other backups. default: true + * @param sync If sync == true, we can guarantee you'll get consistent backup + * even on a machine crash/reboot. Backup process is slower with sync + * enabled. If sync == false, we don't guarantee anything on machine reboot. + * However, chances are some of the backups are consistent. Default: true + * @param destroyOldData If true, it will delete whatever backups there are + * already. Default: false + * @param backupLogFiles If false, we won't backup log files. This option can be + * useful for backing up in-memory databases where log file are persisted, + * but table files are in memory. Default: true + * @param backupRateLimit Max bytes that can be transferred in a second during + * backup. If 0, go as fast as you can. Default: 0 + * @param restoreRateLimit Max bytes that can be transferred in a second during + * restore. If 0, go as fast as you can. Default: 0 */ public class BackupableDBOptions extends RocksObject { public BackupableDBOptions(String path, boolean shareTableFiles, boolean sync, From e87973cde12596657c665c44a86e412f6759389a Mon Sep 17 00:00:00 2001 From: Ankit Gupta Date: Sun, 18 May 2014 22:40:48 -0700 Subject: [PATCH 5/8] Removing code from portal.h for setting handle of RestoreOptions and RestoreBackupableDB --- java/org/rocksdb/BackupableDBOptions.java | 22 +++---- java/org/rocksdb/RestoreBackupableDB.java | 29 ++++----- java/org/rocksdb/RestoreOptions.java | 15 ++--- java/org/rocksdb/test/BackupableDBTest.java | 4 +- java/rocksjni/portal.h | 66 --------------------- java/rocksjni/restorejni.cc | 38 ++++++------ 6 files changed, 54 insertions(+), 120 deletions(-) diff --git a/java/org/rocksdb/BackupableDBOptions.java b/java/org/rocksdb/BackupableDBOptions.java index 8bcc2cf18..0989d39a8 100644 --- a/java/org/rocksdb/BackupableDBOptions.java +++ b/java/org/rocksdb/BackupableDBOptions.java @@ -11,26 +11,26 @@ package org.rocksdb; * * Note that dispose() must be called before an Options instance * become out-of-scope to release the allocated memory in c++. - * + * * @param path Where to keep the backup files. Has to be different than dbname. Best to set this to dbname_ + "/backups" - * @param shareTableFiles If share_table_files == true, backup will assume that - * table files with same name have the same contents. This enables - * incremental backups and avoids unnecessary data copies. If - * share_table_files == false, each backup will be on its own and will not + * @param shareTableFiles If share_table_files == true, backup will assume that + * table files with same name have the same contents. This enables + * incremental backups and avoids unnecessary data copies. If + * share_table_files == false, each backup will be on its own and will not * share any data with other backups. default: true - * @param sync If sync == true, we can guarantee you'll get consistent backup - * even on a machine crash/reboot. Backup process is slower with sync + * @param sync If sync == true, we can guarantee you'll get consistent backup + * even on a machine crash/reboot. Backup process is slower with sync * enabled. If sync == false, we don't guarantee anything on machine reboot. * However, chances are some of the backups are consistent. Default: true - * @param destroyOldData If true, it will delete whatever backups there are + * @param destroyOldData If true, it will delete whatever backups there are * already. Default: false * @param backupLogFiles If false, we won't backup log files. This option can be - * useful for backing up in-memory databases where log file are persisted, + * useful for backing up in-memory databases where log file are persisted, * but table files are in memory. Default: true - * @param backupRateLimit Max bytes that can be transferred in a second during + * @param backupRateLimit Max bytes that can be transferred in a second during * backup. If 0, go as fast as you can. Default: 0 - * @param restoreRateLimit Max bytes that can be transferred in a second during + * @param restoreRateLimit Max bytes that can be transferred in a second during * restore. If 0, go as fast as you can. Default: 0 */ public class BackupableDBOptions extends RocksObject { diff --git a/java/org/rocksdb/RestoreBackupableDB.java b/java/org/rocksdb/RestoreBackupableDB.java index f6a4ed0ab..8c6dbf531 100644 --- a/java/org/rocksdb/RestoreBackupableDB.java +++ b/java/org/rocksdb/RestoreBackupableDB.java @@ -7,24 +7,24 @@ package org.rocksdb; /** * This class is used to access information about backups and restore from them. - * - * Note that dispose() must be called before this instance become out-of-scope + * + * Note that dispose() must be called before this instance become out-of-scope * to release the allocated memory in c++. - * + * * @param options Instance of BackupableDBOptions. */ public class RestoreBackupableDB extends RocksObject { public RestoreBackupableDB(BackupableDBOptions options) { super(); - newRestoreBackupableDB(options.nativeHandle_); + nativeHandle_ = newRestoreBackupableDB(options.nativeHandle_); } - + /** * Restore from backup with backup_id * IMPORTANT -- if options_.share_table_files == true and you restore DB * from some backup that is not the latest, and you start creating new * backups from the new DB, they will probably fail. - * + * * Example: Let's say you have backups 1, 2, 3, 4, 5 and you restore 3. * If you add new data to the DB and try creating a new backup now, the * database will diverge from backups 4 and 5 and the new backup will fail. @@ -36,7 +36,7 @@ public class RestoreBackupableDB extends RocksObject { restoreDBFromBackup0(nativeHandle_, backupId, dbDir, walDir, restoreOptions.nativeHandle_); } - + /** * Restore from the latest backup. */ @@ -45,25 +45,25 @@ public class RestoreBackupableDB extends RocksObject { restoreDBFromLatestBackup0(nativeHandle_, dbDir, walDir, restoreOptions.nativeHandle_); } - + /** * Deletes old backups, keeping latest numBackupsToKeep alive. - * + * * @param Number of latest backups to keep */ public void purgeOldBackups(int numBackupsToKeep) throws RocksDBException { purgeOldBackups0(nativeHandle_, numBackupsToKeep); } - + /** * Deletes a specific backup. - * + * * @param ID of backup to delete. */ public void deleteBackup(long backupId) throws RocksDBException { deleteBackup0(nativeHandle_, backupId); } - + /** * Release the memory allocated for the current instance * in the c++ side. @@ -71,10 +71,11 @@ public class RestoreBackupableDB extends RocksObject { @Override public synchronized void dispose() { if (isInitialized()) { dispose(nativeHandle_); + nativeHandle_ = 0; } } - - private native void newRestoreBackupableDB(long options); + + private native long newRestoreBackupableDB(long options); private native void restoreDBFromBackup0(long nativeHandle, long backupId, String dbDir, String walDir, long restoreOptions) throws RocksDBException; private native void restoreDBFromLatestBackup0(long nativeHandle, diff --git a/java/org/rocksdb/RestoreOptions.java b/java/org/rocksdb/RestoreOptions.java index 51ffcbc73..0321c3b9c 100644 --- a/java/org/rocksdb/RestoreOptions.java +++ b/java/org/rocksdb/RestoreOptions.java @@ -8,21 +8,21 @@ package org.rocksdb; /** * RestoreOptions to control the behavior of restore. * - * Note that dispose() must be called before this instance become out-of-scope + * Note that dispose() must be called before this instance become out-of-scope * to release the allocated memory in c++. - * + * * @param If true, restore won't overwrite the existing log files in wal_dir. It * will also move all log files from archive directory to wal_dir. Use this - * option in combination with BackupableDBOptions::backup_log_files = false + * option in combination with BackupableDBOptions::backup_log_files = false * for persisting in-memory databases. * Default: false */ public class RestoreOptions extends RocksObject { public RestoreOptions(boolean keepLogFiles) { super(); - newRestoreOptions(keepLogFiles); + nativeHandle_ = newRestoreOptions(keepLogFiles); } - + /** * Release the memory allocated for the current instance * in the c++ side. @@ -30,9 +30,10 @@ public class RestoreOptions extends RocksObject { @Override public synchronized void dispose() { if (isInitialized()) { dispose(nativeHandle_); + nativeHandle_ = 0; } } - - private native void newRestoreOptions(boolean keepLogFiles); + + private native long newRestoreOptions(boolean keepLogFiles); private native void dispose(long handle); } diff --git a/java/org/rocksdb/test/BackupableDBTest.java b/java/org/rocksdb/test/BackupableDBTest.java index 3ef88dc8c..9d3a64c06 100644 --- a/java/org/rocksdb/test/BackupableDBTest.java +++ b/java/org/rocksdb/test/BackupableDBTest.java @@ -36,10 +36,12 @@ public class BackupableDBTest { bdb.close(); // restore from backup + RestoreOptions ropt = new RestoreOptions(false); RestoreBackupableDB rdb = new RestoreBackupableDB(bopt); rdb.restoreDBFromLatestBackup(db_path, db_path, - new RestoreOptions(false)); + ropt); rdb.dispose(); + ropt.dispose(); // verify that backed up data contains deleted record bdb = BackupableDB.open(opt, bopt, db_path); diff --git a/java/rocksjni/portal.h b/java/rocksjni/portal.h index 181b67705..b4f874330 100644 --- a/java/rocksjni/portal.h +++ b/java/rocksjni/portal.h @@ -251,72 +251,6 @@ class BackupableDBOptionsJni { } }; -class RestoreOptionsJni { - public: - // Get the java class id of org.rocksdb.RestoreOptions. - static jclass getJClass(JNIEnv* env) { - static jclass jclazz = env->FindClass("org/rocksdb/RestoreOptions"); - assert(jclazz != nullptr); - return jclazz; - } - - // Get the field id of the member variable of org.rocksdb.RestoreOptions - // that stores the pointer to rocksdb::RestoreOptions - static jfieldID getHandleFieldID(JNIEnv* env) { - static jfieldID fid = env->GetFieldID( - getJClass(env), "nativeHandle_", "J"); - assert(fid != nullptr); - return fid; - } - - // Get the pointer to rocksdb::RestoreOptions - static rocksdb::RestoreOptions* getHandle(JNIEnv* env, jobject jobj) { - return reinterpret_cast( - env->GetLongField(jobj, getHandleFieldID(env))); - } - - // Pass the rocksdb::RestoreOptions pointer to the java side. - static void setHandle( - JNIEnv* env, jobject jobj, rocksdb::RestoreOptions* op) { - env->SetLongField( - jobj, getHandleFieldID(env), - reinterpret_cast(op)); - } -}; - -class RestoreBackupableDBJni { - public: - // Get the java class id of org.rocksdb.RestoreBackupableDB. - static jclass getJClass(JNIEnv* env) { - static jclass jclazz = env->FindClass("org/rocksdb/RestoreBackupableDB"); - assert(jclazz != nullptr); - return jclazz; - } - - // Get the field id of the member variable of org.rocksdb.RestoreBackupableDB - // that stores the pointer to rocksdb::RestoreBackupableDB - static jfieldID getHandleFieldID(JNIEnv* env) { - static jfieldID fid = env->GetFieldID( - getJClass(env), "nativeHandle_", "J"); - assert(fid != nullptr); - return fid; - } - - // Get the pointer to rocksdb::RestoreBackupableDB - static rocksdb::RestoreBackupableDB* getHandle(JNIEnv* env, jobject jobj) { - return reinterpret_cast( - env->GetLongField(jobj, getHandleFieldID(env))); - } - - // Pass the rocksdb::RestoreBackupableDB pointer to the java side. - static void setHandle( - JNIEnv* env, jobject jobj, rocksdb::RestoreBackupableDB* op) { - env->SetLongField( - jobj, getHandleFieldID(env), - reinterpret_cast(op)); - } -}; - class IteratorJni { public: // Get the java class id of org.rocksdb.Iteartor. diff --git a/java/rocksjni/restorejni.cc b/java/rocksjni/restorejni.cc index d351d4066..a6ade81a7 100644 --- a/java/rocksjni/restorejni.cc +++ b/java/rocksjni/restorejni.cc @@ -4,7 +4,7 @@ // of patent rights can be found in the PATENTS file in the same directory. // // This file implements the "bridge" between Java and C++ and enables -// calling c++ rocksdb::RestoreBackupableDB and rocksdb::RestoreOptions methods +// calling c++ rocksdb::RestoreBackupableDB and rocksdb::RestoreOptions methods // from Java side. #include @@ -20,12 +20,12 @@ /* * Class: org_rocksdb_RestoreOptions * Method: newRestoreOptions - * Signature: (Z)V + * Signature: (Z)J */ -void Java_org_rocksdb_RestoreOptions_newRestoreOptions(JNIEnv* env, +jlong Java_org_rocksdb_RestoreOptions_newRestoreOptions(JNIEnv* env, jobject jobj, jboolean keep_log_files) { auto ropt = new rocksdb::RestoreOptions(keep_log_files); - rocksdb::RestoreOptionsJni::setHandle(env, jobj, ropt); + return reinterpret_cast(ropt); } /* @@ -38,20 +38,18 @@ void Java_org_rocksdb_RestoreOptions_dispose(JNIEnv* env, jobject jobj, auto ropt = reinterpret_cast(jhandle); assert(ropt); delete ropt; - - rocksdb::RestoreOptionsJni::setHandle(env, jobj, nullptr); } /* * Class: org_rocksdb_RestoreBackupableDB * Method: newRestoreBackupableDB - * Signature: (J)V + * Signature: (J)J */ -void Java_org_rocksdb_RestoreBackupableDB_newRestoreBackupableDB(JNIEnv* env, +jlong Java_org_rocksdb_RestoreBackupableDB_newRestoreBackupableDB(JNIEnv* env, jobject jobj, jlong jopt_handle) { auto opt = reinterpret_cast(jopt_handle); auto rdb = new rocksdb::RestoreBackupableDB(rocksdb::Env::Default(), *opt); - rocksdb::RestoreBackupableDBJni::setHandle(env, jobj, rdb); + return reinterpret_cast(rdb); } /* @@ -63,17 +61,17 @@ void Java_org_rocksdb_RestoreBackupableDB_restoreDBFromBackup0(JNIEnv* env, jobject jobj, jlong jhandle, jlong jbackup_id, jstring jdb_dir, jstring jwal_dir, jlong jopt_handle) { auto opt = reinterpret_cast(jopt_handle); - + const char* cdb_dir = env->GetStringUTFChars(jdb_dir, 0); const char* cwal_dir = env->GetStringUTFChars(jwal_dir, 0); - + auto rdb = reinterpret_cast(jhandle); rocksdb::Status s = rdb->RestoreDBFromBackup(jbackup_id, cdb_dir, cwal_dir, *opt); - + env->ReleaseStringUTFChars(jdb_dir, cdb_dir); env->ReleaseStringUTFChars(jwal_dir, cwal_dir); - + if(!s.ok()) { rocksdb::RocksDBExceptionJni::ThrowNew(env, s); } @@ -88,17 +86,17 @@ void Java_org_rocksdb_RestoreBackupableDB_restoreDBFromLatestBackup0( JNIEnv* env, jobject jobj, jlong jhandle, jstring jdb_dir, jstring jwal_dir, jlong jopt_handle) { auto opt = reinterpret_cast(jopt_handle); - + const char* cdb_dir = env->GetStringUTFChars(jdb_dir, 0); const char* cwal_dir = env->GetStringUTFChars(jwal_dir, 0); - + auto rdb = reinterpret_cast(jhandle); rocksdb::Status s = rdb->RestoreDBFromLatestBackup(cdb_dir, cwal_dir, *opt); - + env->ReleaseStringUTFChars(jdb_dir, cdb_dir); env->ReleaseStringUTFChars(jwal_dir, cwal_dir); - + if(!s.ok()) { rocksdb::RocksDBExceptionJni::ThrowNew(env, s); } @@ -113,7 +111,7 @@ void Java_org_rocksdb_RestoreBackupableDB_purgeOldBackups0(JNIEnv* env, jobject jobj, jlong jhandle, jint jnum_backups_to_keep) { auto rdb = reinterpret_cast(jhandle); rocksdb::Status s = rdb->PurgeOldBackups(jnum_backups_to_keep); - + if(!s.ok()) { rocksdb::RocksDBExceptionJni::ThrowNew(env, s); } @@ -128,7 +126,7 @@ void Java_org_rocksdb_RestoreBackupableDB_deleteBackup0(JNIEnv* env, jobject jobj, jlong jhandle, jlong jbackup_id) { auto rdb = reinterpret_cast(jhandle); rocksdb::Status s = rdb->DeleteBackup(jbackup_id); - + if(!s.ok()) { rocksdb::RocksDBExceptionJni::ThrowNew(env, s); } @@ -144,6 +142,4 @@ void Java_org_rocksdb_RestoreBackupableDB_dispose(JNIEnv* env, jobject jobj, auto ropt = reinterpret_cast(jhandle); assert(ropt); delete ropt; - - rocksdb::RestoreBackupableDBJni::setHandle(env, jobj, nullptr); } From d271cc5b4e2171dbf4dd193caa3672e14b4119ee Mon Sep 17 00:00:00 2001 From: Ankit Gupta Date: Wed, 21 May 2014 07:54:22 -0700 Subject: [PATCH 6/8] Treat negative values as no limit --- java/org/rocksdb/BackupableDBOptions.java | 7 +++++-- java/rocksjni/backupablejni.cc | 3 +++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/java/org/rocksdb/BackupableDBOptions.java b/java/org/rocksdb/BackupableDBOptions.java index 0989d39a8..5d71b144f 100644 --- a/java/org/rocksdb/BackupableDBOptions.java +++ b/java/org/rocksdb/BackupableDBOptions.java @@ -29,14 +29,17 @@ package org.rocksdb; * useful for backing up in-memory databases where log file are persisted, * but table files are in memory. Default: true * @param backupRateLimit Max bytes that can be transferred in a second during - * backup. If 0, go as fast as you can. Default: 0 + * backup. If 0 or negative, then go as fast as you can. Default: 0 * @param restoreRateLimit Max bytes that can be transferred in a second during - * restore. If 0, go as fast as you can. Default: 0 + * restore. If 0 or negative, then go as fast as you can. Default: 0 */ public class BackupableDBOptions extends RocksObject { public BackupableDBOptions(String path, boolean shareTableFiles, boolean sync, boolean destroyOldData, boolean backupLogFiles, long backupRateLimit, long restoreRateLimit) { + backupRateLimit = (backupRateLimit <= 0) ? 0 : backupRateLimit; + restoreRateLimit = (restoreRateLimit <= 0) ? 0 : restoreRateLimit; + super(); newBackupableDBOptions(path, shareTableFiles, sync, destroyOldData, backupLogFiles, backupRateLimit, restoreRateLimit); diff --git a/java/rocksjni/backupablejni.cc b/java/rocksjni/backupablejni.cc index 9fee7cfcc..1ddcbd036 100644 --- a/java/rocksjni/backupablejni.cc +++ b/java/rocksjni/backupablejni.cc @@ -55,6 +55,9 @@ void Java_org_rocksdb_BackupableDBOptions_newBackupableDBOptions( JNIEnv* env, jobject jobj, jstring jpath, jboolean jshare_table_files, jboolean jsync, jboolean jdestroy_old_data, jboolean jbackup_log_files, jlong jbackup_rate_limit, jlong jrestore_rate_limit) { + jbackup_rate_limit = (jbackup_rate_limit <= 0) ? 0 : jbackup_rate_limit; + jrestore_rate_limit = (jrestore_rate_limit <= 0) ? 0 : jrestore_rate_limit; + const char* cpath = env->GetStringUTFChars(jpath, 0); auto bopt = new rocksdb::BackupableDBOptions(cpath, nullptr, From 2fa0a993b8539f0d336399a9a6b6884b1655c0b8 Mon Sep 17 00:00:00 2001 From: Ankit Gupta Date: Thu, 5 Jun 2014 13:25:08 +0100 Subject: [PATCH 7/8] Fix build --- java/org/rocksdb/BackupableDBOptions.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/java/org/rocksdb/BackupableDBOptions.java b/java/org/rocksdb/BackupableDBOptions.java index 3132dbcd8..c72b44f8c 100644 --- a/java/org/rocksdb/BackupableDBOptions.java +++ b/java/org/rocksdb/BackupableDBOptions.java @@ -37,10 +37,11 @@ public class BackupableDBOptions extends RocksObject { public BackupableDBOptions(String path, boolean shareTableFiles, boolean sync, boolean destroyOldData, boolean backupLogFiles, long backupRateLimit, long restoreRateLimit) { + super(); + backupRateLimit = (backupRateLimit <= 0) ? 0 : backupRateLimit; restoreRateLimit = (restoreRateLimit <= 0) ? 0 : restoreRateLimit; - - super(); + newBackupableDBOptions(path, shareTableFiles, sync, destroyOldData, backupLogFiles, backupRateLimit, restoreRateLimit); } From 1a6c1a5ddd5379c205fa0db9175b2a53d1e1cdc1 Mon Sep 17 00:00:00 2001 From: Ankit Gupta Date: Thu, 5 Jun 2014 13:34:38 +0100 Subject: [PATCH 8/8] Fix build --- java/org/rocksdb/RestoreBackupableDB.java | 8 +++----- java/org/rocksdb/RestoreOptions.java | 8 +++----- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/java/org/rocksdb/RestoreBackupableDB.java b/java/org/rocksdb/RestoreBackupableDB.java index 8c6dbf531..dbde447a0 100644 --- a/java/org/rocksdb/RestoreBackupableDB.java +++ b/java/org/rocksdb/RestoreBackupableDB.java @@ -68,11 +68,9 @@ public class RestoreBackupableDB extends RocksObject { * Release the memory allocated for the current instance * in the c++ side. */ - @Override public synchronized void dispose() { - if (isInitialized()) { - dispose(nativeHandle_); - nativeHandle_ = 0; - } + @Override public synchronized void disposeInternal() { + assert(isInitialized()); + dispose(nativeHandle_); } private native long newRestoreBackupableDB(long options); diff --git a/java/org/rocksdb/RestoreOptions.java b/java/org/rocksdb/RestoreOptions.java index 0321c3b9c..77a2b99bc 100644 --- a/java/org/rocksdb/RestoreOptions.java +++ b/java/org/rocksdb/RestoreOptions.java @@ -27,11 +27,9 @@ public class RestoreOptions extends RocksObject { * Release the memory allocated for the current instance * in the c++ side. */ - @Override public synchronized void dispose() { - if (isInitialized()) { - dispose(nativeHandle_); - nativeHandle_ = 0; - } + @Override public synchronized void disposeInternal() { + assert(isInitialized()); + dispose(nativeHandle_); } private native long newRestoreOptions(boolean keepLogFiles);