andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

RocksJava - Add errorIfLogFileExists parameter to RocksDB.openReadOnly (#7046)

Browse Source 
Summary:
Expose from C++ API to Java API.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/7046

Reviewed By: riversand963

Differential Revision: D23726297

Pulled By: pdillinger

fbshipit-source-id: fc66bf626ce6fe9797e7d021ac849eacab91bf6d
This commit is contained in:
Adam Retter 2020-09-17 15:39:25 -07:00 committed by Facebook GitHub Bot
parent b9750c7c3c
commit 3ac07a12fe
12 changed files with 178 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
db/c.cc
View File

@ -504,13 +504,13 @@ rocksdb_t* rocksdb_open_with_ttl(
return result;
}
rocksdb_t* rocksdb_open_for_read_only(
const rocksdb_options_t* options,
const char* name,
unsigned char error_if_log_file_exist,
char** errptr) {
rocksdb_t* rocksdb_open_for_read_only(const rocksdb_options_t* options,
const char* name,
unsigned char error_if_wal_file_exists,
char** errptr) {
DB* db;
if (SaveError(errptr, DB::OpenForReadOnly(options->rep, std::string(name), &db, error_if_log_file_exist))) {
if (SaveError(errptr, DB::OpenForReadOnly(options->rep, std::string(name),
&db, error_if_wal_file_exists))) {
return nullptr;
}
rocksdb_t* result = new rocksdb_t;
@ -747,7 +747,7 @@ rocksdb_t* rocksdb_open_for_read_only_column_families(
int num_column_families, const char* const* column_family_names,
const rocksdb_options_t* const* column_family_options,
rocksdb_column_family_handle_t** column_family_handles,
unsigned char error_if_log_file_exist, char** errptr) {
unsigned char error_if_wal_file_exists, char** errptr) {
std::vector<ColumnFamilyDescriptor> column_families;
for (int i = 0; i < num_column_families; i++) {
column_families.push_back(ColumnFamilyDescriptor(
@ -757,8 +757,10 @@ rocksdb_t* rocksdb_open_for_read_only_column_families(
DB* db;
std::vector<ColumnFamilyHandle*> handles;
if (SaveError(errptr, DB::OpenForReadOnly(DBOptions(db_options->rep),
std::string(name), column_families, &handles, &db, error_if_log_file_exist))) {
if (SaveError(errptr,
DB::OpenForReadOnly(DBOptions(db_options->rep),
std::string(name), column_families,
&handles, &db, error_if_wal_file_exists))) {
return nullptr;
}

4
db/db_impl/db_impl.h
View File

@ -1186,8 +1186,8 @@ class DBImpl : public DB {
// skipped.
virtual Status Recover(
const std::vector<ColumnFamilyDescriptor>& column_families,
bool read_only = false, bool error_if_log_file_exist = false,
bool error_if_data_exists_in_logs = false,
bool read_only = false, bool error_if_wal_file_exists = false,
bool error_if_data_exists_in_wals = false,
uint64_t* recovered_seq = nullptr);
virtual bool OwnTablesAndLogs() const { return true; }

14
db/db_impl/db_impl_open.cc
View File

@ -364,7 +364,7 @@ IOStatus Directories::SetDirectories(FileSystem* fs, const std::string& dbname,
Status DBImpl::Recover(
const std::vector<ColumnFamilyDescriptor>& column_families, bool read_only,
bool error_if_log_file_exist, bool error_if_data_exists_in_logs,
bool error_if_wal_file_exists, bool error_if_data_exists_in_wals,
uint64_t* recovered_seq) {
mutex_.AssertHeld();
@ -588,11 +588,11 @@ Status DBImpl::Recover(
}
if (logs.size() > 0) {
if (error_if_log_file_exist) {
if (error_if_wal_file_exists) {
return Status::Corruption(
"The db was opened in readonly mode with error_if_log_file_exist"
"flag but a log file already exists");
} else if (error_if_data_exists_in_logs) {
"The db was opened in readonly mode with error_if_wal_file_exists"
"flag but a WAL file already exists");
} else if (error_if_data_exists_in_wals) {
for (auto& log : logs) {
std::string fname = LogFileName(immutable_db_options_.wal_dir, log);
uint64_t bytes;
@ -600,8 +600,8 @@ Status DBImpl::Recover(
if (s.ok()) {
if (bytes > 0) {
return Status::Corruption(
"error_if_data_exists_in_logs is set but there are data "
" in log files.");
"error_if_data_exists_in_wals is set but there are data "
" in WAL files.");
}
}
}

14
db/db_impl/db_impl_readonly.cc
View File

@ -151,7 +151,7 @@ Status OpenForReadOnlyCheckExistence(const DBOptions& db_options,
} // namespace
Status DB::OpenForReadOnly(const Options& options, const std::string& dbname,
DB** dbptr, bool /*error_if_log_file_exist*/) {
DB** dbptr, bool /*error_if_wal_file_exists*/) {
// If dbname does not exist in the file system, should not do anything
Status s = OpenForReadOnlyCheckExistence(options, dbname);
if (!s.ok()) {
@ -188,7 +188,7 @@ Status DB::OpenForReadOnly(
const DBOptions& db_options, const std::string& dbname,
const std::vector<ColumnFamilyDescriptor>& column_families,
std::vector<ColumnFamilyHandle*>* handles, DB** dbptr,
bool error_if_log_file_exist) {
bool error_if_wal_file_exists) {
// If dbname does not exist in the file system, should not do anything
Status s = OpenForReadOnlyCheckExistence(db_options, dbname);
if (!s.ok()) {
@ -197,14 +197,14 @@ Status DB::OpenForReadOnly(
return DBImplReadOnly::OpenForReadOnlyWithoutCheck(
db_options, dbname, column_families, handles, dbptr,
error_if_log_file_exist);
error_if_wal_file_exists);
}
Status DBImplReadOnly::OpenForReadOnlyWithoutCheck(
const DBOptions& db_options, const std::string& dbname,
const std::vector<ColumnFamilyDescriptor>& column_families,
std::vector<ColumnFamilyHandle*>* handles, DB** dbptr,
bool error_if_log_file_exist) {
bool error_if_wal_file_exists) {
*dbptr = nullptr;
handles->clear();
@ -212,7 +212,7 @@ Status DBImplReadOnly::OpenForReadOnlyWithoutCheck(
DBImplReadOnly* impl = new DBImplReadOnly(db_options, dbname);
impl->mutex_.Lock();
Status s = impl->Recover(column_families, true /* read only */,
error_if_log_file_exist);
error_if_wal_file_exists);
if (s.ok()) {
// set column family handles
for (auto cf : column_families) {
@ -253,7 +253,7 @@ Status DBImplReadOnly::OpenForReadOnlyWithoutCheck(
Status DB::OpenForReadOnly(const Options& /*options*/,
const std::string& /*dbname*/, DB** /*dbptr*/,
bool /*error_if_log_file_exist*/) {
bool /*error_if_wal_file_exists*/) {
return Status::NotSupported("Not supported in ROCKSDB_LITE.");
}
@ -261,7 +261,7 @@ Status DB::OpenForReadOnly(
const DBOptions& /*db_options*/, const std::string& /*dbname*/,
const std::vector<ColumnFamilyDescriptor>& /*column_families*/,
std::vector<ColumnFamilyHandle*>* /*handles*/, DB** /*dbptr*/,
bool /*error_if_log_file_exist*/) {
bool /*error_if_wal_file_exists*/) {
return Status::NotSupported("Not supported in ROCKSDB_LITE.");
}
#endif // !ROCKSDB_LITE

2
db/db_impl/db_impl_readonly.h
View File

@ -138,7 +138,7 @@ class DBImplReadOnly : public DBImpl {
const DBOptions& db_options, const std::string& dbname,
const std::vector<ColumnFamilyDescriptor>& column_families,
std::vector<ColumnFamilyHandle*>* handles, DB** dbptr,
bool error_if_log_file_exist = false);
bool error_if_wal_file_exists = false);
friend class DB;
};
} // namespace ROCKSDB_NAMESPACE

4
db/db_impl/db_impl_secondary.cc
View File

@ -28,8 +28,8 @@ DBImplSecondary::~DBImplSecondary() {}
Status DBImplSecondary::Recover(
const std::vector<ColumnFamilyDescriptor>& column_families,
bool /*readonly*/, bool /*error_if_log_file_exist*/,
bool /*error_if_data_exists_in_logs*/, uint64_t*) {
bool /*readonly*/, bool /*error_if_wal_file_exists*/,
bool /*error_if_data_exists_in_wals*/, uint64_t*) {
mutex_.AssertHeld();
JobContext job_context(0);

4
db/db_impl/db_impl_secondary.h
View File

@ -77,8 +77,8 @@ class DBImplSecondary : public DBImpl {
// Recover by replaying MANIFEST and WAL. Also initialize manifest_reader_
// and log_readers_ to facilitate future operations.
Status Recover(const std::vector<ColumnFamilyDescriptor>& column_families,
bool read_only, bool error_if_log_file_exist,
bool error_if_data_exists_in_logs,
bool read_only, bool error_if_wal_file_exists,
bool error_if_data_exists_in_wals,
uint64_t* = nullptr) override;
// Implementations of the DB interface

4
include/rocksdb/c.h
View File

@ -136,7 +136,7 @@ extern ROCKSDB_LIBRARY_API rocksdb_t* rocksdb_open_with_ttl(
extern ROCKSDB_LIBRARY_API rocksdb_t* rocksdb_open_for_read_only(
const rocksdb_options_t* options, const char* name,
unsigned char error_if_log_file_exist, char** errptr);
unsigned char error_if_wal_file_exists, char** errptr);
extern ROCKSDB_LIBRARY_API rocksdb_t* rocksdb_open_as_secondary(
const rocksdb_options_t* options, const char* name,
@ -232,7 +232,7 @@ rocksdb_open_for_read_only_column_families(
const char* const* column_family_names,
const rocksdb_options_t* const* column_family_options,
rocksdb_column_family_handle_t** column_family_handles,
unsigned char error_if_log_file_exist, char** errptr);
unsigned char error_if_wal_file_exists, char** errptr);
extern ROCKSDB_LIBRARY_API rocksdb_t* rocksdb_open_as_secondary_column_families(
const rocksdb_options_t* options, const char* name,

4
include/rocksdb/db.h
View File

@ -157,7 +157,7 @@ class DB {
// return Status::NotSupported.
static Status OpenForReadOnly(const Options& options, const std::string& name,
DB** dbptr,
bool error_if_log_file_exist = false);
bool error_if_wal_file_exists = false);
// Open the database for read only with column families. When opening DB with
// read only, you can specify only a subset of column families in the
@ -171,7 +171,7 @@ class DB {
const DBOptions& db_options, const std::string& name,
const std::vector<ColumnFamilyDescriptor>& column_families,
std::vector<ColumnFamilyHandle*>* handles, DB** dbptr,
bool error_if_log_file_exist = false);
bool error_if_wal_file_exists = false);
// The following OpenAsSecondary functions create a secondary instance that
// can dynamically tail the MANIFEST of a primary that must have already been

40
java/rocksjni/rocksjni.cc
View File

@ -72,15 +72,19 @@ jlong Java_org_rocksdb_RocksDB_open__JLjava_lang_String_2(
/*
* Class: org_rocksdb_RocksDB
* Method: openROnly
* Signature: (JLjava/lang/String;)J
* Signature: (JLjava/lang/String;Z)J
*/
jlong Java_org_rocksdb_RocksDB_openROnly__JLjava_lang_String_2(
JNIEnv* env, jclass, jlong jopt_handle, jstring jdb_path) {
jlong Java_org_rocksdb_RocksDB_openROnly__JLjava_lang_String_2Z(
JNIEnv* env, jclass, jlong jopt_handle, jstring jdb_path,
jboolean jerror_if_wal_file_exists) {
const bool error_if_wal_file_exists = jerror_if_wal_file_exists == JNI_TRUE;
return rocksdb_open_helper(
env, jopt_handle, jdb_path,
[](const ROCKSDB_NAMESPACE::Options& options, const std::string& db_path,
ROCKSDB_NAMESPACE::DB** db) {
return ROCKSDB_NAMESPACE::DB::OpenForReadOnly(options, db_path, db);
[error_if_wal_file_exists](const ROCKSDB_NAMESPACE::Options& options,
const std::string& db_path,
ROCKSDB_NAMESPACE::DB** db) {
return ROCKSDB_NAMESPACE::DB::OpenForReadOnly(options, db_path, db,
error_if_wal_file_exists);
});
}
@ -172,21 +176,25 @@ jlongArray rocksdb_open_helper(
/*
* Class: org_rocksdb_RocksDB
* Method: openROnly
* Signature: (JLjava/lang/String;[[B[J)[J
* Signature: (JLjava/lang/String;[[B[JZ)[J
*/
jlongArray Java_org_rocksdb_RocksDB_openROnly__JLjava_lang_String_2_3_3B_3J(
jlongArray Java_org_rocksdb_RocksDB_openROnly__JLjava_lang_String_2_3_3B_3JZ(
JNIEnv* env, jclass, jlong jopt_handle, jstring jdb_path,
jobjectArray jcolumn_names, jlongArray jcolumn_options) {
jobjectArray jcolumn_names, jlongArray jcolumn_options,
jboolean jerror_if_wal_file_exists) {
const bool error_if_wal_file_exists = jerror_if_wal_file_exists == JNI_TRUE;
return rocksdb_open_helper(
env, jopt_handle, jdb_path, jcolumn_names, jcolumn_options,
[](const ROCKSDB_NAMESPACE::DBOptions& options,
const std::string& db_path,
const std::vector<ROCKSDB_NAMESPACE::ColumnFamilyDescriptor>&
column_families,
std::vector<ROCKSDB_NAMESPACE::ColumnFamilyHandle*>* handles,
ROCKSDB_NAMESPACE::DB** db) {
[error_if_wal_file_exists](
const ROCKSDB_NAMESPACE::DBOptions& options,
const std::string& db_path,
const std::vector<ROCKSDB_NAMESPACE::ColumnFamilyDescriptor>&
column_families,
std::vector<ROCKSDB_NAMESPACE::ColumnFamilyHandle*>* handles,
ROCKSDB_NAMESPACE::DB** db) {
return ROCKSDB_NAMESPACE::DB::OpenForReadOnly(
options, db_path, column_families, handles, db);
options, db_path, column_families, handles, db,
error_if_wal_file_exists);
});
}

130
java/src/main/java/org/rocksdb/RocksDB.java
View File

@ -329,10 +329,61 @@ public class RocksDB extends RocksObject {
throws RocksDBException {
// This allows to use the rocksjni default Options instead of
// the c++ one.
Options options = new Options();
final Options options = new Options();
return openReadOnly(options, path);
}
/**
* The factory constructor of RocksDB that opens a RocksDB instance in
* Read-Only mode given the path to the database using the specified
* options and db path.
*
* Options instance *should* not be disposed before all DBs using this options
* instance have been closed. If user doesn't call options dispose explicitly,
* then this options instance will be GC'd automatically.
*
* @param options {@link Options} instance.
* @param path the path to the RocksDB.
* @return a {@link RocksDB} instance on success, null if the specified
* {@link RocksDB} can not be opened.
*
* @throws RocksDBException thrown if error happens in underlying
* native library.
*/
public static RocksDB openReadOnly(final Options options, final String path)
throws RocksDBException {
return openReadOnly(options, path, false);
}
/**
* The factory constructor of RocksDB that opens a RocksDB instance in
* Read-Only mode given the path to the database using the specified
* options and db path.
*
* Options instance *should* not be disposed before all DBs using this options
* instance have been closed. If user doesn't call options dispose explicitly,
* then this options instance will be GC'd automatically.
*
* @param options {@link Options} instance.
* @param path the path to the RocksDB.
* @param errorIfWalFileExists true to raise an error when opening the db
* if a Write Ahead Log file exists, false otherwise.
* @return a {@link RocksDB} instance on success, null if the specified
* {@link RocksDB} can not be opened.
*
* @throws RocksDBException thrown if error happens in underlying
* native library.
*/
public static RocksDB openReadOnly(final Options options, final String path,
final boolean errorIfWalFileExists) throws RocksDBException {
// when non-default Options is used, keeping an Options reference
// in RocksDB can prevent Java to GC during the life-time of
// the currently-created RocksDB.
final RocksDB db = new RocksDB(openROnly(options.nativeHandle_, path, errorIfWalFileExists));
db.storeOptionsInstance(options);
return db;
}
/**
* The factory constructor of RocksDB that opens a RocksDB instance in
* Read-Only mode given the path to the database using the default
@ -355,35 +406,7 @@ public class RocksDB extends RocksObject {
// This allows to use the rocksjni default Options instead of
// the c++ one.
final DBOptions options = new DBOptions();
return openReadOnly(options, path, columnFamilyDescriptors,
columnFamilyHandles);
}
/**
* The factory constructor of RocksDB that opens a RocksDB instance in
* Read-Only mode given the path to the database using the specified
* options and db path.
*
* Options instance *should* not be disposed before all DBs using this options
* instance have been closed. If user doesn't call options dispose explicitly,
* then this options instance will be GC'd automatically.
*
* @param options {@link Options} instance.
* @param path the path to the RocksDB.
* @return a {@link RocksDB} instance on success, null if the specified
* {@link RocksDB} can not be opened.
*
* @throws RocksDBException thrown if error happens in underlying
* native library.
*/
public static RocksDB openReadOnly(final Options options, final String path)
throws RocksDBException {
// when non-default Options is used, keeping an Options reference
// in RocksDB can prevent Java to GC during the life-time of
// the currently-created RocksDB.
final RocksDB db = new RocksDB(openROnly(options.nativeHandle_, path));
db.storeOptionsInstance(options);
return db;
return openReadOnly(options, path, columnFamilyDescriptors, columnFamilyHandles, false);
}
/**
@ -410,7 +433,37 @@ public class RocksDB extends RocksObject {
*/
public static RocksDB openReadOnly(final DBOptions options, final String path,
final List<ColumnFamilyDescriptor> columnFamilyDescriptors,
final List<ColumnFamilyHandle> columnFamilyHandles)
final List<ColumnFamilyHandle> columnFamilyHandles) throws RocksDBException {
return openReadOnly(options, path, columnFamilyDescriptors, columnFamilyHandles, false);
}
/**
* The factory constructor of RocksDB that opens a RocksDB instance in
* Read-Only mode given the path to the database using the specified
* options and db path.
*
* <p>This open method allows to open RocksDB using a subset of available
* column families</p>
* <p>Options instance *should* not be disposed before all DBs using this
* options instance have been closed. If user doesn't call options dispose
* explicitly,then this options instance will be GC'd automatically.</p>
*
* @param options {@link DBOptions} instance.
* @param path the path to the RocksDB.
* @param columnFamilyDescriptors list of column family descriptors
* @param columnFamilyHandles will be filled with ColumnFamilyHandle instances
* on open.
* @param errorIfWalFileExists true to raise an error when opening the db
* if a Write Ahead Log file exists, false otherwise.
* @return a {@link RocksDB} instance on success, null if the specified
* {@link RocksDB} can not be opened.
*
* @throws RocksDBException thrown if error happens in underlying
* native library.
*/
public static RocksDB openReadOnly(final DBOptions options, final String path,
final List<ColumnFamilyDescriptor> columnFamilyDescriptors,
final List<ColumnFamilyHandle> columnFamilyHandles, final boolean errorIfWalFileExists)
throws RocksDBException {
// when non-default Options is used, keeping an Options reference
// in RocksDB can prevent Java to GC during the life-time of
@ -425,8 +478,8 @@ public class RocksDB extends RocksObject {
cfOptionHandles[i] = cfDescriptor.getOptions().nativeHandle_;
}
final long[] handles = openROnly(options.nativeHandle_, path, cfNames,
cfOptionHandles);
final long[] handles =
openROnly(options.nativeHandle_, path, cfNames, cfOptionHandles, errorIfWalFileExists);
final RocksDB db = new RocksDB(handles[0]);
db.storeOptionsInstance(options);
@ -4381,8 +4434,8 @@ public class RocksDB extends RocksObject {
final String path, final byte[][] columnFamilyNames,
final long[] columnFamilyOptions) throws RocksDBException;
private native static long openROnly(final long optionsHandle,
final String path) throws RocksDBException;
private native static long openROnly(final long optionsHandle, final String path,
final boolean errorIfWalFileExists) throws RocksDBException;
/**
* @param optionsHandle Native handle pointing to an Options object
@ -4396,10 +4449,9 @@ public class RocksDB extends RocksObject {
*
* @throws RocksDBException thrown if the database could not be opened
*/
private native static long[] openROnly(final long optionsHandle,
final String path, final byte[][] columnFamilyNames,
final long[] columnFamilyOptions
) throws RocksDBException;
private native static long[] openROnly(final long optionsHandle, final String path,
final byte[][] columnFamilyNames, final long[] columnFamilyOptions,
final boolean errorIfWalFileExists) throws RocksDBException;
private native static long openAsSecondary(final long optionsHandle, final String path,
final String secondaryPath) throws RocksDBException;

27
java/src/test/java/org/rocksdb/ReadOnlyTest.java
View File

@ -302,4 +302,31 @@ public class ReadOnlyTest {
}
}
}
@Test(expected = RocksDBException.class)
public void errorIfWalFileExists() throws RocksDBException {
try (final Options options = new Options().setCreateIfMissing(true);
final RocksDB db = RocksDB.open(options, dbFolder.getRoot().getAbsolutePath())) {
// no-op
}
try (final ColumnFamilyOptions cfOpts = new ColumnFamilyOptions()) {
final List<ColumnFamilyDescriptor> cfDescriptors =
Arrays.asList(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, cfOpts));
final List<ColumnFamilyHandle> readOnlyColumnFamilyHandleList = new ArrayList<>();
try (final DBOptions options = new DBOptions();
final RocksDB rDb = RocksDB.openReadOnly(options, dbFolder.getRoot().getAbsolutePath(),
cfDescriptors, readOnlyColumnFamilyHandleList, true);) {
try {
// no-op... should have raised an error as errorIfWalFileExists=true
} finally {
for (final ColumnFamilyHandle columnFamilyHandle : readOnlyColumnFamilyHandleList) {
columnFamilyHandle.close();
}
}
}
}
}
}