andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Expose new configration variables via the thrift api

Browse Source 
Summary: Expose new configration variables via the thrift api. when compiing for fbcode, always build thrift server

Test Plan: none.

Reviewers: heyongqiang

Reviewed By: heyongqiang

Differential Revision: https://reviews.facebook.net/D4689
This commit is contained in:
Dhruba Borthakur 2012-08-16 11:03:07 -07:00
parent 7c0b5ec54a
commit b56ff5ea38
7 changed files with 317 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
fbcode.sh
View File

@ -8,6 +8,9 @@ TOOLCHAIN_REV=d28c90311ca14f9f0b2bb720f4e34b285513d4f4
TOOLCHAIN_EXECUTABLES="/mnt/gvfs/third-party/$TOOLCHAIN_REV/centos5.2-native"
TOOLCHAIN_LIB_BASE="/mnt/gvfs/third-party/$TOOLCHAIN_REV/gcc-4.6.2-glibc-2.13"
# always build thrift server
export USE_THRIFT=1
# location of libhdfs libraries
if test "$USE_HDFS"; then
JAVA_HOME="/usr/local/jdk-6u22-64"

2
thrift/README
View File

@ -21,5 +21,5 @@ You can run the leveldb server unit tests by
You can regenerate the thrift cpp files by doing the following
cd ./thrift
thrift --gen cpp if/leveldb.thrift
bin/thrift --gen cpp if/leveldb.thrift

BIN
thrift/bin/thrift Executable file
View File

Binary file not shown.

209
thrift/gen-cpp/leveldb_types.cpp
View File

@ -422,6 +422,76 @@ void reflectionInitializer_6731746507948871532(::apache::thrift::reflection::Sch
f.name = "compression";
dt.fields[7] = f;
}
{
::apache::thrift::reflection::StructField f;
f.isRequired = true;
f.type = 5U;
f.name = "num_levels";
dt.fields[8] = f;
}
{
::apache::thrift::reflection::StructField f;
f.isRequired = true;
f.type = 5U;
f.name = "level0_file_num_compaction_trigger";
dt.fields[9] = f;
}
{
::apache::thrift::reflection::StructField f;
f.isRequired = true;
f.type = 5U;
f.name = "level0_slowdown_writes_trigger";
dt.fields[10] = f;
}
{
::apache::thrift::reflection::StructField f;
f.isRequired = true;
f.type = 5U;
f.name = "level0_stop_writes_trigger";
dt.fields[11] = f;
}
{
::apache::thrift::reflection::StructField f;
f.isRequired = true;
f.type = 5U;
f.name = "target_file_size_base";
dt.fields[12] = f;
}
{
::apache::thrift::reflection::StructField f;
f.isRequired = true;
f.type = 5U;
f.name = "target_file_size_multiplier";
dt.fields[13] = f;
}
{
::apache::thrift::reflection::StructField f;
f.isRequired = true;
f.type = 5U;
f.name = "max_bytes_for_level_base";
dt.fields[14] = f;
}
{
::apache::thrift::reflection::StructField f;
f.isRequired = true;
f.type = 5U;
f.name = "max_bytes_for_level_multiplier";
dt.fields[15] = f;
}
{
::apache::thrift::reflection::StructField f;
f.isRequired = true;
f.type = 5U;
f.name = "max_grandparent_overlap_factor";
dt.fields[16] = f;
}
{
::apache::thrift::reflection::StructField f;
f.isRequired = true;
f.type = 2U;
f.name = "disableDataSync";
dt.fields[17] = f;
}
schema.dataTypes[id] = dt;
schema.names[dt.name] = id;
}
@ -509,6 +579,86 @@ uint32_t DBOptions::read(apache::thrift::protocol::TProtocol* iprot) {
xfer += iprot->skip(ftype);
}
break;
case 8:
if (ftype == apache::thrift::protocol::T_I32) {
xfer += iprot->readI32(this->num_levels);
this->__isset.num_levels = true;
} else {
xfer += iprot->skip(ftype);
}
break;
case 9:
if (ftype == apache::thrift::protocol::T_I32) {
xfer += iprot->readI32(this->level0_file_num_compaction_trigger);
this->__isset.level0_file_num_compaction_trigger = true;
} else {
xfer += iprot->skip(ftype);
}
break;
case 10:
if (ftype == apache::thrift::protocol::T_I32) {
xfer += iprot->readI32(this->level0_slowdown_writes_trigger);
this->__isset.level0_slowdown_writes_trigger = true;
} else {
xfer += iprot->skip(ftype);
}
break;
case 11:
if (ftype == apache::thrift::protocol::T_I32) {
xfer += iprot->readI32(this->level0_stop_writes_trigger);
this->__isset.level0_stop_writes_trigger = true;
} else {
xfer += iprot->skip(ftype);
}
break;
case 12:
if (ftype == apache::thrift::protocol::T_I32) {
xfer += iprot->readI32(this->target_file_size_base);
this->__isset.target_file_size_base = true;
} else {
xfer += iprot->skip(ftype);
}
break;
case 13:
if (ftype == apache::thrift::protocol::T_I32) {
xfer += iprot->readI32(this->target_file_size_multiplier);
this->__isset.target_file_size_multiplier = true;
} else {
xfer += iprot->skip(ftype);
}
break;
case 14:
if (ftype == apache::thrift::protocol::T_I32) {
xfer += iprot->readI32(this->max_bytes_for_level_base);
this->__isset.max_bytes_for_level_base = true;
} else {
xfer += iprot->skip(ftype);
}
break;
case 15:
if (ftype == apache::thrift::protocol::T_I32) {
xfer += iprot->readI32(this->max_bytes_for_level_multiplier);
this->__isset.max_bytes_for_level_multiplier = true;
} else {
xfer += iprot->skip(ftype);
}
break;
case 16:
if (ftype == apache::thrift::protocol::T_I32) {
xfer += iprot->readI32(this->max_grandparent_overlap_factor);
this->__isset.max_grandparent_overlap_factor = true;
} else {
xfer += iprot->skip(ftype);
}
break;
case 17:
if (ftype == apache::thrift::protocol::T_BOOL) {
xfer += iprot->readBool(this->disableDataSync);
this->__isset.disableDataSync = true;
} else {
xfer += iprot->skip(ftype);
}
break;
default:
xfer += iprot->skip(ftype);
break;
@ -545,6 +695,36 @@ uint32_t DBOptions::write(apache::thrift::protocol::TProtocol* oprot) const {
xfer += oprot->writeFieldBegin("compression", apache::thrift::protocol::T_I32, 7);
xfer += oprot->writeI32((int32_t)this->compression);
xfer += oprot->writeFieldEnd();
xfer += oprot->writeFieldBegin("num_levels", apache::thrift::protocol::T_I32, 8);
xfer += oprot->writeI32(this->num_levels);
xfer += oprot->writeFieldEnd();
xfer += oprot->writeFieldBegin("level0_file_num_compaction_trigger", apache::thrift::protocol::T_I32, 9);
xfer += oprot->writeI32(this->level0_file_num_compaction_trigger);
xfer += oprot->writeFieldEnd();
xfer += oprot->writeFieldBegin("level0_slowdown_writes_trigger", apache::thrift::protocol::T_I32, 10);
xfer += oprot->writeI32(this->level0_slowdown_writes_trigger);
xfer += oprot->writeFieldEnd();
xfer += oprot->writeFieldBegin("level0_stop_writes_trigger", apache::thrift::protocol::T_I32, 11);
xfer += oprot->writeI32(this->level0_stop_writes_trigger);
xfer += oprot->writeFieldEnd();
xfer += oprot->writeFieldBegin("target_file_size_base", apache::thrift::protocol::T_I32, 12);
xfer += oprot->writeI32(this->target_file_size_base);
xfer += oprot->writeFieldEnd();
xfer += oprot->writeFieldBegin("target_file_size_multiplier", apache::thrift::protocol::T_I32, 13);
xfer += oprot->writeI32(this->target_file_size_multiplier);
xfer += oprot->writeFieldEnd();
xfer += oprot->writeFieldBegin("max_bytes_for_level_base", apache::thrift::protocol::T_I32, 14);
xfer += oprot->writeI32(this->max_bytes_for_level_base);
xfer += oprot->writeFieldEnd();
xfer += oprot->writeFieldBegin("max_bytes_for_level_multiplier", apache::thrift::protocol::T_I32, 15);
xfer += oprot->writeI32(this->max_bytes_for_level_multiplier);
xfer += oprot->writeFieldEnd();
xfer += oprot->writeFieldBegin("max_grandparent_overlap_factor", apache::thrift::protocol::T_I32, 16);
xfer += oprot->writeI32(this->max_grandparent_overlap_factor);
xfer += oprot->writeFieldEnd();
xfer += oprot->writeFieldBegin("disableDataSync", apache::thrift::protocol::T_BOOL, 17);
xfer += oprot->writeBool(this->disableDataSync);
xfer += oprot->writeFieldEnd();
xfer += oprot->writeFieldStop();
xfer += oprot->writeStructEnd();
return xfer;
@ -561,6 +741,16 @@ void swap(DBOptions &a, DBOptions &b) {
swap(a.block_size, b.block_size);
swap(a.block_restart_interval, b.block_restart_interval);
swap(a.compression, b.compression);
swap(a.num_levels, b.num_levels);
swap(a.level0_file_num_compaction_trigger, b.level0_file_num_compaction_trigger);
swap(a.level0_slowdown_writes_trigger, b.level0_slowdown_writes_trigger);
swap(a.level0_stop_writes_trigger, b.level0_stop_writes_trigger);
swap(a.target_file_size_base, b.target_file_size_base);
swap(a.target_file_size_multiplier, b.target_file_size_multiplier);
swap(a.max_bytes_for_level_base, b.max_bytes_for_level_base);
swap(a.max_bytes_for_level_multiplier, b.max_bytes_for_level_multiplier);
swap(a.max_grandparent_overlap_factor, b.max_grandparent_overlap_factor);
swap(a.disableDataSync, b.disableDataSync);
swap(a.__isset, b.__isset);
}
@ -579,6 +769,13 @@ void reflectionInitializer_8830325115029814540(::apache::thrift::reflection::Sch
f.name = "sync";
dt.fields[1] = f;
}
{
::apache::thrift::reflection::StructField f;
f.isRequired = true;
f.type = 2U;
f.name = "disableWAL";
dt.fields[2] = f;
}
schema.dataTypes[id] = dt;
schema.names[dt.name] = id;
}
@ -616,6 +813,14 @@ uint32_t WriteOptions::read(apache::thrift::protocol::TProtocol* iprot) {
xfer += iprot->skip(ftype);
}
break;
case 2:
if (ftype == apache::thrift::protocol::T_BOOL) {
xfer += iprot->readBool(this->disableWAL);
this->__isset.disableWAL = true;
} else {
xfer += iprot->skip(ftype);
}
break;
default:
xfer += iprot->skip(ftype);
break;
@ -634,6 +839,9 @@ uint32_t WriteOptions::write(apache::thrift::protocol::TProtocol* oprot) const {
xfer += oprot->writeFieldBegin("sync", apache::thrift::protocol::T_BOOL, 1);
xfer += oprot->writeBool(this->sync);
xfer += oprot->writeFieldEnd();
xfer += oprot->writeFieldBegin("disableWAL", apache::thrift::protocol::T_BOOL, 2);
xfer += oprot->writeBool(this->disableWAL);
xfer += oprot->writeFieldEnd();
xfer += oprot->writeFieldStop();
xfer += oprot->writeStructEnd();
return xfer;
@ -644,6 +852,7 @@ void swap(WriteOptions &a, WriteOptions &b) {
(void)a;
(void)b;
swap(a.sync, b.sync);
swap(a.disableWAL, b.disableWAL);
swap(a.__isset, b.__isset);
}

70
thrift/gen-cpp/leveldb_types.h
View File

@ -238,7 +238,7 @@ class DBOptions {
static const uint64_t _reflection_id = 6731746507948871532U;
static void _reflection_register(::apache::thrift::reflection::Schema&);
DBOptions() : create_if_missing(0), error_if_exists(0), write_buffer_size(0), max_open_files(0), block_size(0), block_restart_interval(0), compression(static_cast<CompressionType>(0)) {
DBOptions() : create_if_missing(0), error_if_exists(0), write_buffer_size(0), max_open_files(0), block_size(0), block_restart_interval(0), compression(static_cast<CompressionType>(0)), num_levels(0), level0_file_num_compaction_trigger(0), level0_slowdown_writes_trigger(0), level0_stop_writes_trigger(0), target_file_size_base(0), target_file_size_multiplier(0), max_bytes_for_level_base(0), max_bytes_for_level_multiplier(0), max_grandparent_overlap_factor(0), disableDataSync(0) {
}
DBOptions(const DBOptions&) = default;
@ -254,6 +254,16 @@ class DBOptions {
block_size = 0;
block_restart_interval = 0;
compression = static_cast<CompressionType>(0);
num_levels = 0;
level0_file_num_compaction_trigger = 0;
level0_slowdown_writes_trigger = 0;
level0_stop_writes_trigger = 0;
target_file_size_base = 0;
target_file_size_multiplier = 0;
max_bytes_for_level_base = 0;
max_bytes_for_level_multiplier = 0;
max_grandparent_overlap_factor = 0;
disableDataSync = 0;
__isset.__clear();
}
@ -266,6 +276,16 @@ class DBOptions {
int32_t block_size;
int32_t block_restart_interval;
CompressionType compression;
int32_t num_levels;
int32_t level0_file_num_compaction_trigger;
int32_t level0_slowdown_writes_trigger;
int32_t level0_stop_writes_trigger;
int32_t target_file_size_base;
int32_t target_file_size_multiplier;
int32_t max_bytes_for_level_base;
int32_t max_bytes_for_level_multiplier;
int32_t max_grandparent_overlap_factor;
bool disableDataSync;
struct __isset {
__isset() { __clear(); }
@ -277,6 +297,16 @@ class DBOptions {
block_size = false;
block_restart_interval = false;
compression = false;
num_levels = false;
level0_file_num_compaction_trigger = false;
level0_slowdown_writes_trigger = false;
level0_stop_writes_trigger = false;
target_file_size_base = false;
target_file_size_multiplier = false;
max_bytes_for_level_base = false;
max_bytes_for_level_multiplier = false;
max_grandparent_overlap_factor = false;
disableDataSync = false;
}
bool create_if_missing;
bool error_if_exists;
@ -285,6 +315,16 @@ class DBOptions {
bool block_size;
bool block_restart_interval;
bool compression;
bool num_levels;
bool level0_file_num_compaction_trigger;
bool level0_slowdown_writes_trigger;
bool level0_stop_writes_trigger;
bool target_file_size_base;
bool target_file_size_multiplier;
bool max_bytes_for_level_base;
bool max_bytes_for_level_multiplier;
bool max_grandparent_overlap_factor;
bool disableDataSync;
} __isset;
bool operator == (const DBOptions & rhs) const
@ -303,6 +343,26 @@ class DBOptions {
return false;
if (!(this->compression == rhs.compression))
return false;
if (!(this->num_levels == rhs.num_levels))
return false;
if (!(this->level0_file_num_compaction_trigger == rhs.level0_file_num_compaction_trigger))
return false;
if (!(this->level0_slowdown_writes_trigger == rhs.level0_slowdown_writes_trigger))
return false;
if (!(this->level0_stop_writes_trigger == rhs.level0_stop_writes_trigger))
return false;
if (!(this->target_file_size_base == rhs.target_file_size_base))
return false;
if (!(this->target_file_size_multiplier == rhs.target_file_size_multiplier))
return false;
if (!(this->max_bytes_for_level_base == rhs.max_bytes_for_level_base))
return false;
if (!(this->max_bytes_for_level_multiplier == rhs.max_bytes_for_level_multiplier))
return false;
if (!(this->max_grandparent_overlap_factor == rhs.max_grandparent_overlap_factor))
return false;
if (!(this->disableDataSync == rhs.disableDataSync))
return false;
return true;
}
bool operator != (const DBOptions &rhs) const {
@ -324,7 +384,7 @@ class WriteOptions {
static const uint64_t _reflection_id = 8830325115029814540U;
static void _reflection_register(::apache::thrift::reflection::Schema&);
WriteOptions() : sync(0) {
WriteOptions() : sync(0), disableWAL(0) {
}
WriteOptions(const WriteOptions&) = default;
@ -334,25 +394,31 @@ class WriteOptions {
void __clear() {
sync = 0;
disableWAL = 0;
__isset.__clear();
}
virtual ~WriteOptions() throw() {}
bool sync;
bool disableWAL;
struct __isset {
__isset() { __clear(); }
void __clear() {
sync = false;
disableWAL = false;
}
bool sync;
bool disableWAL;
} __isset;
bool operator == (const WriteOptions & rhs) const
{
if (!(this->sync == rhs.sync))
return false;
if (!(this->disableWAL == rhs.disableWAL))
return false;
return true;
}
bool operator != (const WriteOptions &rhs) const {

15
thrift/if/leveldb.thrift
View File

@ -47,12 +47,23 @@ struct DBOptions {
4:i32 max_open_files;
5:i32 block_size;
6:i32 block_restart_interval;
7:CompressionType compression
7:CompressionType compression,
8:i32 num_levels,
9:i32 level0_file_num_compaction_trigger,
10:i32 level0_slowdown_writes_trigger,
11:i32 level0_stop_writes_trigger,
12:i32 target_file_size_base,
13:i32 target_file_size_multiplier,
14:i32 max_bytes_for_level_base,
15:i32 max_bytes_for_level_multiplier,
16:i32 max_grandparent_overlap_factor,
17:bool disableDataSync
}
// Options for writing
struct WriteOptions {
1:bool sync
1:bool sync,
2:bool disableWAL
}
struct Snapshot {

23
thrift/server_utils.cpp
View File

@ -75,6 +75,26 @@ class DBHandler : virtual public DBIf {
} else if (dboptions.compression == kSnappyCompression) {
options.compression = leveldb::kSnappyCompression;
}
if (dboptions.num_levels > 0)
options.num_levels = dboptions.num_levels;
if (dboptions.level0_file_num_compaction_trigger > 0)
options.level0_file_num_compaction_trigger = dboptions.level0_file_num_compaction_trigger;
if (dboptions.level0_slowdown_writes_trigger > 0)
options.level0_slowdown_writes_trigger = dboptions.level0_slowdown_writes_trigger;
if (dboptions.level0_stop_writes_trigger)
options.level0_stop_writes_trigger = dboptions.level0_stop_writes_trigger;
if (dboptions.target_file_size_base > 0)
options.target_file_size_base = dboptions.target_file_size_base;
if (dboptions.target_file_size_multiplier > 0)
options.target_file_size_multiplier = dboptions.target_file_size_multiplier;
if (dboptions.max_bytes_for_level_base)
options.max_bytes_for_level_base = dboptions.max_bytes_for_level_base;
if (dboptions.max_bytes_for_level_multiplier)
options.max_bytes_for_level_multiplier = dboptions.max_bytes_for_level_multiplier;
if (dboptions.max_grandparent_overlap_factor)
options.max_grandparent_overlap_factor = dboptions.max_grandparent_overlap_factor;
if (dboptions.disableDataSync)
options.disableDataSync = dboptions.disableDataSync;
openHandles->add(options, dbname, dbdir);
_return.dbname = dbname;
}
@ -91,6 +111,7 @@ class DBHandler : virtual public DBIf {
const WriteOptions& options) {
leveldb::WriteOptions woptions;
woptions.sync = options.sync;
woptions.disableWAL = options.disableWAL;
leveldb::Slice key, value;
key.data_ = kv.key.data.data();
key.size_ = kv.key.size;
@ -111,6 +132,7 @@ class DBHandler : virtual public DBIf {
const WriteOptions& options) {
leveldb::WriteOptions woptions;
woptions.sync = options.sync;
woptions.disableWAL = options.disableWAL;
leveldb::Slice key;
key.data_ = kv.data.data();
key.size_ = kv.size;
@ -130,6 +152,7 @@ class DBHandler : virtual public DBIf {
leveldb::WriteOptions woptions;
leveldb::WriteBatch lbatch;
woptions.sync = options.sync;
woptions.disableWAL = options.disableWAL;
leveldb::Slice key, value;
for (unsigned int i = 0; i < batch.size(); i++) {
kv one = batch[i];