Merge pull request #157 from ankgup87/master

[Java] Add RestoreBackupableDB
This commit is contained in:
Yueh-Hsuan Chiang 2014-06-05 09:24:55 -07:00
commit 166cc5b456
9 changed files with 341 additions and 12 deletions

View File

@ -1,4 +1,5 @@
NATIVE_JAVA_CLASSES = org.rocksdb.RocksDB org.rocksdb.Options org.rocksdb.WriteBatch org.rocksdb.WriteBatchInternal org.rocksdb.WriteBatchTest org.rocksdb.WriteOptions org.rocksdb.BackupableDB org.rocksdb.BackupableDBOptions org.rocksdb.Statistics org.rocksdb.RocksIterator org.rocksdb.VectorMemTableConfig org.rocksdb.SkipListMemTableConfig org.rocksdb.HashLinkedListMemTableConfig org.rocksdb.HashSkipListMemTableConfig org.rocksdb.PlainTableConfig org.rocksdb.ReadOptions org.rocksdb.Filter org.rocksdb.BloomFilter NATIVE_JAVA_CLASSES = org.rocksdb.RocksDB org.rocksdb.Options org.rocksdb.WriteBatch org.rocksdb.WriteBatchInternal org.rocksdb.WriteBatchTest org.rocksdb.WriteOptions org.rocksdb.BackupableDB org.rocksdb.BackupableDBOptions org.rocksdb.Statistics org.rocksdb.RocksIterator org.rocksdb.VectorMemTableConfig org.rocksdb.SkipListMemTableConfig org.rocksdb.HashLinkedListMemTableConfig org.rocksdb.HashSkipListMemTableConfig org.rocksdb.PlainTableConfig org.rocksdb.ReadOptions org.rocksdb.Filter org.rocksdb.BloomFilter org.rocksdb.RestoreOptions org.rocksdb.RestoreBackupableDB
NATIVE_INCLUDE = ./include NATIVE_INCLUDE = ./include
ROCKSDB_JAR = rocksdbjni.jar ROCKSDB_JAR = rocksdbjni.jar

View File

@ -11,11 +11,39 @@ package org.rocksdb;
* *
* Note that dispose() must be called before an Options instance * Note that dispose() must be called before an Options instance
* become out-of-scope to release the allocated memory in c++. * become out-of-scope to release the allocated memory in c++.
*
* @param path Where to keep the backup files. Has to be different than dbname.
Best to set this to dbname_ + "/backups"
* @param shareTableFiles If share_table_files == true, backup will assume that
* table files with same name have the same contents. This enables
* incremental backups and avoids unnecessary data copies. If
* share_table_files == false, each backup will be on its own and will not
* share any data with other backups. default: true
* @param sync If sync == true, we can guarantee you'll get consistent backup
* even on a machine crash/reboot. Backup process is slower with sync
* enabled. If sync == false, we don't guarantee anything on machine reboot.
* However, chances are some of the backups are consistent. Default: true
* @param destroyOldData If true, it will delete whatever backups there are
* already. Default: false
* @param backupLogFiles If false, we won't backup log files. This option can be
* useful for backing up in-memory databases where log file are persisted,
* but table files are in memory. Default: true
* @param backupRateLimit Max bytes that can be transferred in a second during
* backup. If 0 or negative, then go as fast as you can. Default: 0
* @param restoreRateLimit Max bytes that can be transferred in a second during
* restore. If 0 or negative, then go as fast as you can. Default: 0
*/ */
public class BackupableDBOptions extends RocksObject { public class BackupableDBOptions extends RocksObject {
public BackupableDBOptions(String path) { public BackupableDBOptions(String path, boolean shareTableFiles, boolean sync,
boolean destroyOldData, boolean backupLogFiles, long backupRateLimit,
long restoreRateLimit) {
super(); super();
newBackupableDBOptions(path);
backupRateLimit = (backupRateLimit <= 0) ? 0 : backupRateLimit;
restoreRateLimit = (restoreRateLimit <= 0) ? 0 : restoreRateLimit;
newBackupableDBOptions(path, shareTableFiles, sync, destroyOldData,
backupLogFiles, backupRateLimit, restoreRateLimit);
} }
/** /**
@ -37,7 +65,9 @@ public class BackupableDBOptions extends RocksObject {
disposeInternal(nativeHandle_); disposeInternal(nativeHandle_);
} }
private native void newBackupableDBOptions(String path); private native void newBackupableDBOptions(String path,
boolean shareTableFiles, boolean sync, boolean destroyOldData,
boolean backupLogFiles, long backupRateLimit, long restoreRateLimit);
private native String backupDir(long handle); private native String backupDir(long handle);
private native void disposeInternal(long handle); private native void disposeInternal(long handle);
} }

View File

@ -0,0 +1,84 @@
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
package org.rocksdb;
/**
* This class is used to access information about backups and restore from them.
*
* Note that dispose() must be called before this instance become out-of-scope
* to release the allocated memory in c++.
*
* @param options Instance of BackupableDBOptions.
*/
public class RestoreBackupableDB extends RocksObject {
public RestoreBackupableDB(BackupableDBOptions options) {
super();
nativeHandle_ = newRestoreBackupableDB(options.nativeHandle_);
}
/**
* Restore from backup with backup_id
* IMPORTANT -- if options_.share_table_files == true and you restore DB
* from some backup that is not the latest, and you start creating new
* backups from the new DB, they will probably fail.
*
* Example: Let's say you have backups 1, 2, 3, 4, 5 and you restore 3.
* If you add new data to the DB and try creating a new backup now, the
* database will diverge from backups 4 and 5 and the new backup will fail.
* If you want to create new backup, you will first have to delete backups 4
* and 5.
*/
public void restoreDBFromBackup(long backupId, String dbDir, String walDir,
RestoreOptions restoreOptions) throws RocksDBException {
restoreDBFromBackup0(nativeHandle_, backupId, dbDir, walDir,
restoreOptions.nativeHandle_);
}
/**
* Restore from the latest backup.
*/
public void restoreDBFromLatestBackup(String dbDir, String walDir,
RestoreOptions restoreOptions) throws RocksDBException {
restoreDBFromLatestBackup0(nativeHandle_, dbDir, walDir,
restoreOptions.nativeHandle_);
}
/**
* Deletes old backups, keeping latest numBackupsToKeep alive.
*
* @param Number of latest backups to keep
*/
public void purgeOldBackups(int numBackupsToKeep) throws RocksDBException {
purgeOldBackups0(nativeHandle_, numBackupsToKeep);
}
/**
* Deletes a specific backup.
*
* @param ID of backup to delete.
*/
public void deleteBackup(long backupId) throws RocksDBException {
deleteBackup0(nativeHandle_, backupId);
}
/**
* Release the memory allocated for the current instance
* in the c++ side.
*/
@Override public synchronized void disposeInternal() {
assert(isInitialized());
dispose(nativeHandle_);
}
private native long newRestoreBackupableDB(long options);
private native void restoreDBFromBackup0(long nativeHandle, long backupId,
String dbDir, String walDir, long restoreOptions) throws RocksDBException;
private native void restoreDBFromLatestBackup0(long nativeHandle,
String dbDir, String walDir, long restoreOptions) throws RocksDBException;
private native void purgeOldBackups0(long nativeHandle, int numBackupsToKeep);
private native void deleteBackup0(long nativeHandle, long backupId);
private native void dispose(long nativeHandle);
}

View File

@ -0,0 +1,37 @@
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
package org.rocksdb;
/**
* RestoreOptions to control the behavior of restore.
*
* Note that dispose() must be called before this instance become out-of-scope
* to release the allocated memory in c++.
*
* @param If true, restore won't overwrite the existing log files in wal_dir. It
* will also move all log files from archive directory to wal_dir. Use this
* option in combination with BackupableDBOptions::backup_log_files = false
* for persisting in-memory databases.
* Default: false
*/
public class RestoreOptions extends RocksObject {
public RestoreOptions(boolean keepLogFiles) {
super();
nativeHandle_ = newRestoreOptions(keepLogFiles);
}
/**
* Release the memory allocated for the current instance
* in the c++ side.
*/
@Override public synchronized void disposeInternal() {
assert(isInitialized());
dispose(nativeHandle_);
}
private native long newRestoreOptions(boolean keepLogFiles);
private native void dispose(long handle);
}

View File

@ -18,15 +18,37 @@ public class BackupableDBTest {
Options opt = new Options(); Options opt = new Options();
opt.setCreateIfMissing(true); opt.setCreateIfMissing(true);
BackupableDBOptions bopt = new BackupableDBOptions(backup_path); BackupableDBOptions bopt = new BackupableDBOptions(backup_path, false,
true, false, true, 0, 0);
BackupableDB bdb = null; BackupableDB bdb = null;
try { try {
bdb = BackupableDB.open(opt, bopt, db_path); bdb = BackupableDB.open(opt, bopt, db_path);
bdb.put("hello".getBytes(), "BackupableDB".getBytes());
bdb.put("abc".getBytes(), "def".getBytes());
bdb.put("ghi".getBytes(), "jkl".getBytes());
bdb.createNewBackup(true); bdb.createNewBackup(true);
byte[] value = bdb.get("hello".getBytes());
assert(new String(value).equals("BackupableDB")); // delete record after backup
bdb.remove("abc".getBytes());
byte[] value = bdb.get("abc".getBytes());
assert(value == null);
bdb.close();
// restore from backup
RestoreOptions ropt = new RestoreOptions(false);
RestoreBackupableDB rdb = new RestoreBackupableDB(bopt);
rdb.restoreDBFromLatestBackup(db_path, db_path,
ropt);
rdb.dispose();
ropt.dispose();
// verify that backed up data contains deleted record
bdb = BackupableDB.open(opt, bopt, db_path);
value = bdb.get("abc".getBytes());
assert(new String(value).equals("def"));
System.out.println("Backup and restore test passed");
} catch (RocksDBException e) { } catch (RocksDBException e) {
System.err.format("[ERROR]: %s%n", e); System.err.format("[ERROR]: %s%n", e);
e.printStackTrace(); e.printStackTrace();

View File

@ -4,7 +4,8 @@
// of patent rights can be found in the PATENTS file in the same directory. // of patent rights can be found in the PATENTS file in the same directory.
// //
// This file implements the "bridge" between Java and C++ and enables // This file implements the "bridge" between Java and C++ and enables
// calling c++ rocksdb::DB methods from Java side. // calling c++ rocksdb::BackupableDB and rocksdb::BackupableDBOptions methods
// from Java side.
#include <stdio.h> #include <stdio.h>
#include <stdlib.h> #include <stdlib.h>
@ -51,9 +52,18 @@ void Java_org_rocksdb_BackupableDB_createNewBackup(
* Signature: (Ljava/lang/String;)V * Signature: (Ljava/lang/String;)V
*/ */
void Java_org_rocksdb_BackupableDBOptions_newBackupableDBOptions( void Java_org_rocksdb_BackupableDBOptions_newBackupableDBOptions(
JNIEnv* env, jobject jobj, jstring jpath) { JNIEnv* env, jobject jobj, jstring jpath, jboolean jshare_table_files,
jboolean jsync, jboolean jdestroy_old_data, jboolean jbackup_log_files,
jlong jbackup_rate_limit, jlong jrestore_rate_limit) {
jbackup_rate_limit = (jbackup_rate_limit <= 0) ? 0 : jbackup_rate_limit;
jrestore_rate_limit = (jrestore_rate_limit <= 0) ? 0 : jrestore_rate_limit;
const char* cpath = env->GetStringUTFChars(jpath, 0); const char* cpath = env->GetStringUTFChars(jpath, 0);
auto bopt = new rocksdb::BackupableDBOptions(cpath);
auto bopt = new rocksdb::BackupableDBOptions(cpath, nullptr,
jshare_table_files, nullptr, jsync, jdestroy_old_data, jbackup_log_files,
jbackup_rate_limit, jrestore_rate_limit);
env->ReleaseStringUTFChars(jpath, cpath); env->ReleaseStringUTFChars(jpath, cpath);
rocksdb::BackupableDBOptionsJni::setHandle(env, jobj, bopt); rocksdb::BackupableDBOptionsJni::setHandle(env, jobj, bopt);

View File

@ -217,6 +217,7 @@ class HistogramDataJni {
return mid; return mid;
} }
}; };
class BackupableDBOptionsJni { class BackupableDBOptionsJni {
public: public:
// Get the java class id of org.rocksdb.BackupableDBOptions. // Get the java class id of org.rocksdb.BackupableDBOptions.

145
java/rocksjni/restorejni.cc Normal file
View File

@ -0,0 +1,145 @@
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
//
// This file implements the "bridge" between Java and C++ and enables
// calling c++ rocksdb::RestoreBackupableDB and rocksdb::RestoreOptions methods
// from Java side.
#include <stdio.h>
#include <stdlib.h>
#include <jni.h>
#include <iostream>
#include <string>
#include "include/org_rocksdb_RestoreOptions.h"
#include "include/org_rocksdb_RestoreBackupableDB.h"
#include "rocksjni/portal.h"
#include "utilities/backupable_db.h"
/*
* Class: org_rocksdb_RestoreOptions
* Method: newRestoreOptions
* Signature: (Z)J
*/
jlong Java_org_rocksdb_RestoreOptions_newRestoreOptions(JNIEnv* env,
jobject jobj, jboolean keep_log_files) {
auto ropt = new rocksdb::RestoreOptions(keep_log_files);
return reinterpret_cast<jlong>(ropt);
}
/*
* Class: org_rocksdb_RestoreOptions
* Method: dispose
* Signature: (J)V
*/
void Java_org_rocksdb_RestoreOptions_dispose(JNIEnv* env, jobject jobj,
jlong jhandle) {
auto ropt = reinterpret_cast<rocksdb::RestoreOptions*>(jhandle);
assert(ropt);
delete ropt;
}
/*
* Class: org_rocksdb_RestoreBackupableDB
* Method: newRestoreBackupableDB
* Signature: (J)J
*/
jlong Java_org_rocksdb_RestoreBackupableDB_newRestoreBackupableDB(JNIEnv* env,
jobject jobj, jlong jopt_handle) {
auto opt = reinterpret_cast<rocksdb::BackupableDBOptions*>(jopt_handle);
auto rdb = new rocksdb::RestoreBackupableDB(rocksdb::Env::Default(), *opt);
return reinterpret_cast<jlong>(rdb);
}
/*
* Class: org_rocksdb_RestoreBackupableDB
* Method: restoreDBFromBackup0
* Signature: (JJLjava/lang/String;Ljava/lang/String;J)V
*/
void Java_org_rocksdb_RestoreBackupableDB_restoreDBFromBackup0(JNIEnv* env,
jobject jobj, jlong jhandle, jlong jbackup_id, jstring jdb_dir,
jstring jwal_dir, jlong jopt_handle) {
auto opt = reinterpret_cast<rocksdb::RestoreOptions*>(jopt_handle);
const char* cdb_dir = env->GetStringUTFChars(jdb_dir, 0);
const char* cwal_dir = env->GetStringUTFChars(jwal_dir, 0);
auto rdb = reinterpret_cast<rocksdb::RestoreBackupableDB*>(jhandle);
rocksdb::Status s =
rdb->RestoreDBFromBackup(jbackup_id, cdb_dir, cwal_dir, *opt);
env->ReleaseStringUTFChars(jdb_dir, cdb_dir);
env->ReleaseStringUTFChars(jwal_dir, cwal_dir);
if(!s.ok()) {
rocksdb::RocksDBExceptionJni::ThrowNew(env, s);
}
}
/*
* Class: org_rocksdb_RestoreBackupableDB
* Method: restoreDBFromLatestBackup0
* Signature: (JLjava/lang/String;Ljava/lang/String;J)V
*/
void Java_org_rocksdb_RestoreBackupableDB_restoreDBFromLatestBackup0(
JNIEnv* env, jobject jobj, jlong jhandle, jstring jdb_dir, jstring jwal_dir,
jlong jopt_handle) {
auto opt = reinterpret_cast<rocksdb::RestoreOptions*>(jopt_handle);
const char* cdb_dir = env->GetStringUTFChars(jdb_dir, 0);
const char* cwal_dir = env->GetStringUTFChars(jwal_dir, 0);
auto rdb = reinterpret_cast<rocksdb::RestoreBackupableDB*>(jhandle);
rocksdb::Status s =
rdb->RestoreDBFromLatestBackup(cdb_dir, cwal_dir, *opt);
env->ReleaseStringUTFChars(jdb_dir, cdb_dir);
env->ReleaseStringUTFChars(jwal_dir, cwal_dir);
if(!s.ok()) {
rocksdb::RocksDBExceptionJni::ThrowNew(env, s);
}
}
/*
* Class: org_rocksdb_RestoreBackupableDB
* Method: purgeOldBackups0
* Signature: (JI)V
*/
void Java_org_rocksdb_RestoreBackupableDB_purgeOldBackups0(JNIEnv* env,
jobject jobj, jlong jhandle, jint jnum_backups_to_keep) {
auto rdb = reinterpret_cast<rocksdb::RestoreBackupableDB*>(jhandle);
rocksdb::Status s = rdb->PurgeOldBackups(jnum_backups_to_keep);
if(!s.ok()) {
rocksdb::RocksDBExceptionJni::ThrowNew(env, s);
}
}
/*
* Class: org_rocksdb_RestoreBackupableDB
* Method: deleteBackup0
* Signature: (JJ)V
*/
void Java_org_rocksdb_RestoreBackupableDB_deleteBackup0(JNIEnv* env,
jobject jobj, jlong jhandle, jlong jbackup_id) {
auto rdb = reinterpret_cast<rocksdb::RestoreBackupableDB*>(jhandle);
rocksdb::Status s = rdb->DeleteBackup(jbackup_id);
if(!s.ok()) {
rocksdb::RocksDBExceptionJni::ThrowNew(env, s);
}
}
/*
* Class: org_rocksdb_RestoreBackupableDB
* Method: dispose
* Signature: (J)V
*/
void Java_org_rocksdb_RestoreBackupableDB_dispose(JNIEnv* env, jobject jobj,
jlong jhandle) {
auto ropt = reinterpret_cast<rocksdb::RestoreBackupableDB*>(jhandle);
assert(ropt);
delete ropt;
}

View File

@ -1,4 +1,3 @@
// Copyright (c) 2014, Facebook, Inc. All rights reserved. // Copyright (c) 2014, Facebook, Inc. All rights reserved.
// This source code is licensed under the BSD-style license found in the // This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant // LICENSE file in the root directory of this source tree. An additional grant