expose IngestExternalFile to c abi
Summary: IngestExternalFile is very useful when doing bulk load. This pr expose this API to c so many bindings can benefit from it too. Closes https://github.com/facebook/rocksdb/pull/1454 Differential Revision: D4113420 Pulled By: yiwu-arbug fbshipit-source-id: 307c6ae
This commit is contained in:
Jay Lee
Facebook Github Bot
parent
ce22ea99a0
commit
16fb04434f
107
db/c.cc
107
db/c.cc
@ -47,10 +47,12 @@ using rocksdb::WALRecoveryMode;
|
||||
using rocksdb::DB;
|
||||
using rocksdb::DBOptions;
|
||||
using rocksdb::Env;
|
||||
using rocksdb::EnvOptions;
|
||||
using rocksdb::InfoLogLevel;
|
||||
using rocksdb::FileLock;
|
||||
using rocksdb::FilterPolicy;
|
||||
using rocksdb::FlushOptions;
|
||||
using rocksdb::IngestExternalFileOptions;
|
||||
using rocksdb::Iterator;
|
||||
using rocksdb::Logger;
|
||||
using rocksdb::MergeOperator;
|
||||
@ -68,6 +70,7 @@ using rocksdb::Slice;
|
||||
using rocksdb::SliceParts;
|
||||
using rocksdb::SliceTransform;
|
||||
using rocksdb::Snapshot;
|
||||
using rocksdb::SstFileWriter;
|
||||
using rocksdb::Status;
|
||||
using rocksdb::WritableFile;
|
||||
using rocksdb::WriteBatch;
|
||||
@ -108,6 +111,9 @@ struct rocksdb_logger_t { shared_ptr<Logger> rep; };
|
||||
struct rocksdb_cache_t { shared_ptr<Cache> rep; };
|
||||
struct rocksdb_livefiles_t { std::vector<LiveFileMetaData> rep; };
|
||||
struct rocksdb_column_family_handle_t { ColumnFamilyHandle* rep; };
|
||||
struct rocksdb_envoptions_t { EnvOptions rep; };
|
||||
struct rocksdb_ingestexternalfileoptions_t { IngestExternalFileOptions rep; };
|
||||
struct rocksdb_sstfilewriter_t { SstFileWriter* rep; };
|
||||
|
||||
struct rocksdb_compactionfiltercontext_t {
|
||||
CompactionFilter::Context rep;
|
||||
@ -2187,6 +2193,107 @@ void rocksdb_env_destroy(rocksdb_env_t* env) {
|
||||
delete env;
|
||||
}
|
||||
|
||||
rocksdb_envoptions_t* rocksdb_envoptions_create() {
|
||||
rocksdb_envoptions_t* opt = new rocksdb_envoptions_t;
|
||||
return opt;
|
||||
}
|
||||
|
||||
void rocksdb_envoptions_destroy(rocksdb_envoptions_t* opt) { delete opt; }
|
||||
|
||||
rocksdb_sstfilewriter_t* rocksdb_sstfilewriter_create(
|
||||
const rocksdb_envoptions_t* env, const rocksdb_options_t* io_options) {
|
||||
rocksdb_sstfilewriter_t* writer = new rocksdb_sstfilewriter_t;
|
||||
writer->rep =
|
||||
new SstFileWriter(env->rep, io_options->rep, io_options->rep.comparator);
|
||||
return writer;
|
||||
}
|
||||
|
||||
rocksdb_sstfilewriter_t* rocksdb_sstfilewriter_create_with_comparator(
|
||||
const rocksdb_envoptions_t* env, const rocksdb_options_t* io_options,
|
||||
const rocksdb_comparator_t* comparator) {
|
||||
rocksdb_sstfilewriter_t* writer = new rocksdb_sstfilewriter_t;
|
||||
writer->rep =
|
||||
new SstFileWriter(env->rep, io_options->rep, io_options->rep.comparator);
|
||||
return writer;
|
||||
}
|
||||
|
||||
void rocksdb_sstfilewriter_open(rocksdb_sstfilewriter_t* writer,
|
||||
const char* name, char** errptr) {
|
||||
SaveError(errptr, writer->rep->Open(std::string(name)));
|
||||
}
|
||||
|
||||
void rocksdb_sstfilewriter_add(rocksdb_sstfilewriter_t* writer, const char* key,
|
||||
size_t keylen, const char* val, size_t vallen,
|
||||
char** errptr) {
|
||||
SaveError(errptr, writer->rep->Add(Slice(key, keylen), Slice(val, vallen)));
|
||||
}
|
||||
|
||||
void rocksdb_sstfilewriter_finish(rocksdb_sstfilewriter_t* writer,
|
||||
char** errptr) {
|
||||
SaveError(errptr, writer->rep->Finish(NULL));
|
||||
}
|
||||
|
||||
void rocksdb_sstfilewriter_destroy(rocksdb_sstfilewriter_t* writer) {
|
||||
delete writer->rep;
|
||||
delete writer;
|
||||
}
|
||||
|
||||
rocksdb_ingestexternalfileoptions_t*
|
||||
rocksdb_ingestexternalfileoptions_create() {
|
||||
rocksdb_ingestexternalfileoptions_t* opt =
|
||||
new rocksdb_ingestexternalfileoptions_t;
|
||||
return opt;
|
||||
}
|
||||
|
||||
void rocksdb_ingestexternalfileoptions_set_move_files(
|
||||
rocksdb_ingestexternalfileoptions_t* opt, unsigned char move_files) {
|
||||
opt->rep.move_files = move_files;
|
||||
}
|
||||
|
||||
void rocksdb_ingestexternalfileoptions_set_snapshot_consistency(
|
||||
rocksdb_ingestexternalfileoptions_t* opt,
|
||||
unsigned char snapshot_consistency) {
|
||||
opt->rep.snapshot_consistency = snapshot_consistency;
|
||||
}
|
||||
|
||||
void rocksdb_ingestexternalfileoptions_set_allow_global_seqno(
|
||||
rocksdb_ingestexternalfileoptions_t* opt,
|
||||
unsigned char allow_global_seqno) {
|
||||
opt->rep.allow_global_seqno = allow_global_seqno;
|
||||
}
|
||||
|
||||
void rocksdb_ingestexternalfileoptions_set_allow_blocking_flush(
|
||||
rocksdb_ingestexternalfileoptions_t* opt,
|
||||
unsigned char allow_blocking_flush) {
|
||||
opt->rep.allow_blocking_flush = allow_blocking_flush;
|
||||
}
|
||||
|
||||
void rocksdb_ingestexternalfileoptions_destroy(
|
||||
rocksdb_ingestexternalfileoptions_t* opt) {
|
||||
delete opt;
|
||||
}
|
||||
|
||||
void rocksdb_ingest_external_file(
|
||||
rocksdb_t* db, const char* const* file_list, const size_t list_len,
|
||||
const rocksdb_ingestexternalfileoptions_t* opt, char** errptr) {
|
||||
std::vector<std::string> files(list_len);
|
||||
for (size_t i = 0; i < list_len; ++i) {
|
||||
files[i] = std::string(file_list[i]);
|
||||
}
|
||||
SaveError(errptr, db->rep->IngestExternalFile(files, opt->rep));
|
||||
}
|
||||
|
||||
void rocksdb_ingest_external_file_cf(
|
||||
rocksdb_t* db, rocksdb_column_family_handle_t* handle,
|
||||
const char* const* file_list, const size_t list_len,
|
||||
const rocksdb_ingestexternalfileoptions_t* opt, char** errptr) {
|
||||
std::vector<std::string> files(list_len);
|
||||
for (size_t i = 0; i < list_len; ++i) {
|
||||
files[i] = std::string(file_list[i]);
|
||||
}
|
||||
SaveError(errptr, db->rep->IngestExternalFile(handle->rep, files, opt->rep));
|
||||
}
|
||||
|
||||
rocksdb_slicetransform_t* rocksdb_slicetransform_create(
|
||||
void* state,
|
||||
void (*destructor)(void*),
|
||||
|
||||
61
db/c_test.c
61
db/c_test.c
@ -25,7 +25,6 @@
|
||||
|
||||
// Ok for uniqueness
|
||||
int geteuid() {
|
||||
|
||||
int result = 0;
|
||||
|
||||
result = ((int)GetCurrentProcessId() << 16);
|
||||
@ -38,13 +37,13 @@ int geteuid() {
|
||||
|
||||
const char* phase = "";
|
||||
static char dbname[200];
|
||||
static char sstfilename[200];
|
||||
static char dbbackupname[200];
|
||||
|
||||
static void StartPhase(const char* name) {
|
||||
fprintf(stderr, "=== Test %s\n", name);
|
||||
phase = name;
|
||||
}
|
||||
|
||||
static const char* GetTempDir(void) {
|
||||
const char* ret = getenv("TEST_TMPDIR");
|
||||
if (ret == NULL || ret[0] == '\0')
|
||||
@ -304,6 +303,11 @@ int main(int argc, char** argv) {
|
||||
GetTempDir(),
|
||||
((int) geteuid()));
|
||||
|
||||
snprintf(sstfilename, sizeof(sstfilename),
|
||||
"%s/rocksdb_c_test-%d-sst",
|
||||
GetTempDir(),
|
||||
((int)geteuid()));
|
||||
|
||||
StartPhase("create_objects");
|
||||
cmp = rocksdb_comparator_create(NULL, CmpDestroy, CmpCompare, CmpName);
|
||||
env = rocksdb_create_default_env();
|
||||
@ -565,6 +569,59 @@ int main(int argc, char** argv) {
|
||||
rocksdb_release_snapshot(db, snap);
|
||||
}
|
||||
|
||||
StartPhase("addfile");
|
||||
{
|
||||
rocksdb_envoptions_t* env_opt = rocksdb_envoptions_create();
|
||||
rocksdb_options_t* io_options = rocksdb_options_create();
|
||||
rocksdb_sstfilewriter_t* writer =
|
||||
rocksdb_sstfilewriter_create(env_opt, io_options);
|
||||
|
||||
unlink(sstfilename);
|
||||
rocksdb_sstfilewriter_open(writer, sstfilename, &err);
|
||||
CheckNoError(err);
|
||||
rocksdb_sstfilewriter_add(writer, "sstk1", 5, "v1", 2, &err);
|
||||
CheckNoError(err);
|
||||
rocksdb_sstfilewriter_add(writer, "sstk2", 5, "v2", 2, &err);
|
||||
CheckNoError(err);
|
||||
rocksdb_sstfilewriter_add(writer, "sstk3", 5, "v3", 2, &err);
|
||||
CheckNoError(err);
|
||||
rocksdb_sstfilewriter_finish(writer, &err);
|
||||
CheckNoError(err);
|
||||
|
||||
rocksdb_ingestexternalfileoptions_t* ing_opt =
|
||||
rocksdb_ingestexternalfileoptions_create();
|
||||
const char* file_list[1] = {sstfilename};
|
||||
rocksdb_ingest_external_file(db, file_list, 1, ing_opt, &err);
|
||||
CheckNoError(err);
|
||||
CheckGet(db, roptions, "sstk1", "v1");
|
||||
CheckGet(db, roptions, "sstk2", "v2");
|
||||
CheckGet(db, roptions, "sstk3", "v3");
|
||||
|
||||
unlink(sstfilename);
|
||||
rocksdb_sstfilewriter_open(writer, sstfilename, &err);
|
||||
CheckNoError(err);
|
||||
rocksdb_sstfilewriter_add(writer, "sstk2", 5, "v4", 2, &err);
|
||||
CheckNoError(err);
|
||||
rocksdb_sstfilewriter_add(writer, "sstk22", 6, "v5", 2, &err);
|
||||
CheckNoError(err);
|
||||
rocksdb_sstfilewriter_add(writer, "sstk3", 5, "v6", 2, &err);
|
||||
CheckNoError(err);
|
||||
rocksdb_sstfilewriter_finish(writer, &err);
|
||||
CheckNoError(err);
|
||||
|
||||
rocksdb_ingest_external_file(db, file_list, 1, ing_opt, &err);
|
||||
CheckNoError(err);
|
||||
CheckGet(db, roptions, "sstk1", "v1");
|
||||
CheckGet(db, roptions, "sstk2", "v4");
|
||||
CheckGet(db, roptions, "sstk22", "v5");
|
||||
CheckGet(db, roptions, "sstk3", "v6");
|
||||
|
||||
rocksdb_ingestexternalfileoptions_destroy(ing_opt);
|
||||
rocksdb_sstfilewriter_destroy(writer);
|
||||
rocksdb_options_destroy(io_options);
|
||||
rocksdb_envoptions_destroy(env_opt);
|
||||
}
|
||||
|
||||
StartPhase("repair");
|
||||
{
|
||||
// If we do not compact here, then the lazy deletion of
|
||||
|
||||
@ -105,6 +105,9 @@ typedef struct rocksdb_writeoptions_t rocksdb_writeoptions_t;
|
||||
typedef struct rocksdb_universal_compaction_options_t rocksdb_universal_compaction_options_t;
|
||||
typedef struct rocksdb_livefiles_t rocksdb_livefiles_t;
|
||||
typedef struct rocksdb_column_family_handle_t rocksdb_column_family_handle_t;
|
||||
typedef struct rocksdb_envoptions_t rocksdb_envoptions_t;
|
||||
typedef struct rocksdb_ingestexternalfileoptions_t rocksdb_ingestexternalfileoptions_t;
|
||||
typedef struct rocksdb_sstfilewriter_t rocksdb_sstfilewriter_t;
|
||||
|
||||
/* DB operations */
|
||||
|
||||
@ -865,6 +868,56 @@ extern ROCKSDB_LIBRARY_API void rocksdb_env_join_all_threads(
|
||||
rocksdb_env_t* env);
|
||||
extern ROCKSDB_LIBRARY_API void rocksdb_env_destroy(rocksdb_env_t*);
|
||||
|
||||
extern ROCKSDB_LIBRARY_API rocksdb_envoptions_t* rocksdb_envoptions_create();
|
||||
extern ROCKSDB_LIBRARY_API void rocksdb_envoptions_destroy(
|
||||
rocksdb_envoptions_t* opt);
|
||||
|
||||
/* SstFile */
|
||||
|
||||
extern ROCKSDB_LIBRARY_API rocksdb_sstfilewriter_t*
|
||||
rocksdb_sstfilewriter_create(const rocksdb_envoptions_t* env,
|
||||
const rocksdb_options_t* io_options);
|
||||
extern ROCKSDB_LIBRARY_API rocksdb_sstfilewriter_t*
|
||||
rocksdb_sstfilewriter_create_with_comparator(
|
||||
const rocksdb_envoptions_t* env, const rocksdb_options_t* io_options,
|
||||
const rocksdb_comparator_t* comparator);
|
||||
extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_open(
|
||||
rocksdb_sstfilewriter_t* writer, const char* name, char** errptr);
|
||||
extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_add(
|
||||
rocksdb_sstfilewriter_t* writer, const char* key, size_t keylen,
|
||||
const char* val, size_t vallen, char** errptr);
|
||||
extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_finish(
|
||||
rocksdb_sstfilewriter_t* writer, char** errptr);
|
||||
extern ROCKSDB_LIBRARY_API void rocksdb_sstfilewriter_destroy(
|
||||
rocksdb_sstfilewriter_t* writer);
|
||||
|
||||
extern ROCKSDB_LIBRARY_API rocksdb_ingestexternalfileoptions_t*
|
||||
rocksdb_ingestexternalfileoptions_create();
|
||||
extern ROCKSDB_LIBRARY_API void
|
||||
rocksdb_ingestexternalfileoptions_set_move_files(
|
||||
rocksdb_ingestexternalfileoptions_t* opt, unsigned char move_files);
|
||||
extern ROCKSDB_LIBRARY_API void
|
||||
rocksdb_ingestexternalfileoptions_set_snapshot_consistency(
|
||||
rocksdb_ingestexternalfileoptions_t* opt,
|
||||
unsigned char snapshot_consistency);
|
||||
extern ROCKSDB_LIBRARY_API void
|
||||
rocksdb_ingestexternalfileoptions_set_allow_global_seqno(
|
||||
rocksdb_ingestexternalfileoptions_t* opt, unsigned char allow_global_seqno);
|
||||
extern ROCKSDB_LIBRARY_API void
|
||||
rocksdb_ingestexternalfileoptions_set_allow_blocking_flush(
|
||||
rocksdb_ingestexternalfileoptions_t* opt,
|
||||
unsigned char allow_blocking_flush);
|
||||
extern ROCKSDB_LIBRARY_API void rocksdb_ingestexternalfileoptions_destroy(
|
||||
rocksdb_ingestexternalfileoptions_t* opt);
|
||||
|
||||
extern ROCKSDB_LIBRARY_API void rocksdb_ingest_external_file(
|
||||
rocksdb_t* db, const char* const* file_list, const size_t list_len,
|
||||
const rocksdb_ingestexternalfileoptions_t* opt, char** errptr);
|
||||
extern ROCKSDB_LIBRARY_API void rocksdb_ingest_external_file_cf(
|
||||
rocksdb_t* db, rocksdb_column_family_handle_t* handle,
|
||||
const char* const* file_list, const size_t list_len,
|
||||
const rocksdb_ingestexternalfileoptions_t* opt, char** errptr);
|
||||
|
||||
/* SliceTransform */
|
||||
|
||||
extern ROCKSDB_LIBRARY_API rocksdb_slicetransform_t*
|
||||
|
||||
Loading…
Reference in New Issue
Block a user