diff --git a/fbcode.sh b/fbcode.sh index 0f8cc3853..9e9cfe672 100644 --- a/fbcode.sh +++ b/fbcode.sh @@ -8,6 +8,9 @@ TOOLCHAIN_REV=d28c90311ca14f9f0b2bb720f4e34b285513d4f4 TOOLCHAIN_EXECUTABLES="/mnt/gvfs/third-party/$TOOLCHAIN_REV/centos5.2-native" TOOLCHAIN_LIB_BASE="/mnt/gvfs/third-party/$TOOLCHAIN_REV/gcc-4.6.2-glibc-2.13" +# always build thrift server +export USE_THRIFT=1 + # location of libhdfs libraries if test "$USE_HDFS"; then JAVA_HOME="/usr/local/jdk-6u22-64" diff --git a/thrift/README b/thrift/README index 6405a063a..f76c2bbf9 100644 --- a/thrift/README +++ b/thrift/README @@ -21,5 +21,5 @@ You can run the leveldb server unit tests by You can regenerate the thrift cpp files by doing the following cd ./thrift -thrift --gen cpp if/leveldb.thrift +bin/thrift --gen cpp if/leveldb.thrift diff --git a/thrift/bin/thrift b/thrift/bin/thrift new file mode 100755 index 000000000..a28684503 Binary files /dev/null and b/thrift/bin/thrift differ diff --git a/thrift/gen-cpp/leveldb_types.cpp b/thrift/gen-cpp/leveldb_types.cpp index 14465f87b..4654ee213 100644 --- a/thrift/gen-cpp/leveldb_types.cpp +++ b/thrift/gen-cpp/leveldb_types.cpp @@ -422,6 +422,76 @@ void reflectionInitializer_6731746507948871532(::apache::thrift::reflection::Sch f.name = "compression"; dt.fields[7] = f; } + { + ::apache::thrift::reflection::StructField f; + f.isRequired = true; + f.type = 5U; + f.name = "num_levels"; + dt.fields[8] = f; + } + { + ::apache::thrift::reflection::StructField f; + f.isRequired = true; + f.type = 5U; + f.name = "level0_file_num_compaction_trigger"; + dt.fields[9] = f; + } + { + ::apache::thrift::reflection::StructField f; + f.isRequired = true; + f.type = 5U; + f.name = "level0_slowdown_writes_trigger"; + dt.fields[10] = f; + } + { + ::apache::thrift::reflection::StructField f; + f.isRequired = true; + f.type = 5U; + f.name = "level0_stop_writes_trigger"; + dt.fields[11] = f; + } + { + ::apache::thrift::reflection::StructField f; + f.isRequired = true; + f.type = 5U; + f.name = "target_file_size_base"; + dt.fields[12] = f; + } + { + ::apache::thrift::reflection::StructField f; + f.isRequired = true; + f.type = 5U; + f.name = "target_file_size_multiplier"; + dt.fields[13] = f; + } + { + ::apache::thrift::reflection::StructField f; + f.isRequired = true; + f.type = 5U; + f.name = "max_bytes_for_level_base"; + dt.fields[14] = f; + } + { + ::apache::thrift::reflection::StructField f; + f.isRequired = true; + f.type = 5U; + f.name = "max_bytes_for_level_multiplier"; + dt.fields[15] = f; + } + { + ::apache::thrift::reflection::StructField f; + f.isRequired = true; + f.type = 5U; + f.name = "max_grandparent_overlap_factor"; + dt.fields[16] = f; + } + { + ::apache::thrift::reflection::StructField f; + f.isRequired = true; + f.type = 2U; + f.name = "disableDataSync"; + dt.fields[17] = f; + } schema.dataTypes[id] = dt; schema.names[dt.name] = id; } @@ -509,6 +579,86 @@ uint32_t DBOptions::read(apache::thrift::protocol::TProtocol* iprot) { xfer += iprot->skip(ftype); } break; + case 8: + if (ftype == apache::thrift::protocol::T_I32) { + xfer += iprot->readI32(this->num_levels); + this->__isset.num_levels = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 9: + if (ftype == apache::thrift::protocol::T_I32) { + xfer += iprot->readI32(this->level0_file_num_compaction_trigger); + this->__isset.level0_file_num_compaction_trigger = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 10: + if (ftype == apache::thrift::protocol::T_I32) { + xfer += iprot->readI32(this->level0_slowdown_writes_trigger); + this->__isset.level0_slowdown_writes_trigger = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 11: + if (ftype == apache::thrift::protocol::T_I32) { + xfer += iprot->readI32(this->level0_stop_writes_trigger); + this->__isset.level0_stop_writes_trigger = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 12: + if (ftype == apache::thrift::protocol::T_I32) { + xfer += iprot->readI32(this->target_file_size_base); + this->__isset.target_file_size_base = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 13: + if (ftype == apache::thrift::protocol::T_I32) { + xfer += iprot->readI32(this->target_file_size_multiplier); + this->__isset.target_file_size_multiplier = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 14: + if (ftype == apache::thrift::protocol::T_I32) { + xfer += iprot->readI32(this->max_bytes_for_level_base); + this->__isset.max_bytes_for_level_base = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 15: + if (ftype == apache::thrift::protocol::T_I32) { + xfer += iprot->readI32(this->max_bytes_for_level_multiplier); + this->__isset.max_bytes_for_level_multiplier = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 16: + if (ftype == apache::thrift::protocol::T_I32) { + xfer += iprot->readI32(this->max_grandparent_overlap_factor); + this->__isset.max_grandparent_overlap_factor = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 17: + if (ftype == apache::thrift::protocol::T_BOOL) { + xfer += iprot->readBool(this->disableDataSync); + this->__isset.disableDataSync = true; + } else { + xfer += iprot->skip(ftype); + } + break; default: xfer += iprot->skip(ftype); break; @@ -545,6 +695,36 @@ uint32_t DBOptions::write(apache::thrift::protocol::TProtocol* oprot) const { xfer += oprot->writeFieldBegin("compression", apache::thrift::protocol::T_I32, 7); xfer += oprot->writeI32((int32_t)this->compression); xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("num_levels", apache::thrift::protocol::T_I32, 8); + xfer += oprot->writeI32(this->num_levels); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("level0_file_num_compaction_trigger", apache::thrift::protocol::T_I32, 9); + xfer += oprot->writeI32(this->level0_file_num_compaction_trigger); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("level0_slowdown_writes_trigger", apache::thrift::protocol::T_I32, 10); + xfer += oprot->writeI32(this->level0_slowdown_writes_trigger); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("level0_stop_writes_trigger", apache::thrift::protocol::T_I32, 11); + xfer += oprot->writeI32(this->level0_stop_writes_trigger); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("target_file_size_base", apache::thrift::protocol::T_I32, 12); + xfer += oprot->writeI32(this->target_file_size_base); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("target_file_size_multiplier", apache::thrift::protocol::T_I32, 13); + xfer += oprot->writeI32(this->target_file_size_multiplier); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("max_bytes_for_level_base", apache::thrift::protocol::T_I32, 14); + xfer += oprot->writeI32(this->max_bytes_for_level_base); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("max_bytes_for_level_multiplier", apache::thrift::protocol::T_I32, 15); + xfer += oprot->writeI32(this->max_bytes_for_level_multiplier); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("max_grandparent_overlap_factor", apache::thrift::protocol::T_I32, 16); + xfer += oprot->writeI32(this->max_grandparent_overlap_factor); + xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("disableDataSync", apache::thrift::protocol::T_BOOL, 17); + xfer += oprot->writeBool(this->disableDataSync); + xfer += oprot->writeFieldEnd(); xfer += oprot->writeFieldStop(); xfer += oprot->writeStructEnd(); return xfer; @@ -561,6 +741,16 @@ void swap(DBOptions &a, DBOptions &b) { swap(a.block_size, b.block_size); swap(a.block_restart_interval, b.block_restart_interval); swap(a.compression, b.compression); + swap(a.num_levels, b.num_levels); + swap(a.level0_file_num_compaction_trigger, b.level0_file_num_compaction_trigger); + swap(a.level0_slowdown_writes_trigger, b.level0_slowdown_writes_trigger); + swap(a.level0_stop_writes_trigger, b.level0_stop_writes_trigger); + swap(a.target_file_size_base, b.target_file_size_base); + swap(a.target_file_size_multiplier, b.target_file_size_multiplier); + swap(a.max_bytes_for_level_base, b.max_bytes_for_level_base); + swap(a.max_bytes_for_level_multiplier, b.max_bytes_for_level_multiplier); + swap(a.max_grandparent_overlap_factor, b.max_grandparent_overlap_factor); + swap(a.disableDataSync, b.disableDataSync); swap(a.__isset, b.__isset); } @@ -579,6 +769,13 @@ void reflectionInitializer_8830325115029814540(::apache::thrift::reflection::Sch f.name = "sync"; dt.fields[1] = f; } + { + ::apache::thrift::reflection::StructField f; + f.isRequired = true; + f.type = 2U; + f.name = "disableWAL"; + dt.fields[2] = f; + } schema.dataTypes[id] = dt; schema.names[dt.name] = id; } @@ -616,6 +813,14 @@ uint32_t WriteOptions::read(apache::thrift::protocol::TProtocol* iprot) { xfer += iprot->skip(ftype); } break; + case 2: + if (ftype == apache::thrift::protocol::T_BOOL) { + xfer += iprot->readBool(this->disableWAL); + this->__isset.disableWAL = true; + } else { + xfer += iprot->skip(ftype); + } + break; default: xfer += iprot->skip(ftype); break; @@ -634,6 +839,9 @@ uint32_t WriteOptions::write(apache::thrift::protocol::TProtocol* oprot) const { xfer += oprot->writeFieldBegin("sync", apache::thrift::protocol::T_BOOL, 1); xfer += oprot->writeBool(this->sync); xfer += oprot->writeFieldEnd(); + xfer += oprot->writeFieldBegin("disableWAL", apache::thrift::protocol::T_BOOL, 2); + xfer += oprot->writeBool(this->disableWAL); + xfer += oprot->writeFieldEnd(); xfer += oprot->writeFieldStop(); xfer += oprot->writeStructEnd(); return xfer; @@ -644,6 +852,7 @@ void swap(WriteOptions &a, WriteOptions &b) { (void)a; (void)b; swap(a.sync, b.sync); + swap(a.disableWAL, b.disableWAL); swap(a.__isset, b.__isset); } diff --git a/thrift/gen-cpp/leveldb_types.h b/thrift/gen-cpp/leveldb_types.h index 6b85d4d20..117ed4f71 100644 --- a/thrift/gen-cpp/leveldb_types.h +++ b/thrift/gen-cpp/leveldb_types.h @@ -238,7 +238,7 @@ class DBOptions { static const uint64_t _reflection_id = 6731746507948871532U; static void _reflection_register(::apache::thrift::reflection::Schema&); - DBOptions() : create_if_missing(0), error_if_exists(0), write_buffer_size(0), max_open_files(0), block_size(0), block_restart_interval(0), compression(static_cast(0)) { + DBOptions() : create_if_missing(0), error_if_exists(0), write_buffer_size(0), max_open_files(0), block_size(0), block_restart_interval(0), compression(static_cast(0)), num_levels(0), level0_file_num_compaction_trigger(0), level0_slowdown_writes_trigger(0), level0_stop_writes_trigger(0), target_file_size_base(0), target_file_size_multiplier(0), max_bytes_for_level_base(0), max_bytes_for_level_multiplier(0), max_grandparent_overlap_factor(0), disableDataSync(0) { } DBOptions(const DBOptions&) = default; @@ -254,6 +254,16 @@ class DBOptions { block_size = 0; block_restart_interval = 0; compression = static_cast(0); + num_levels = 0; + level0_file_num_compaction_trigger = 0; + level0_slowdown_writes_trigger = 0; + level0_stop_writes_trigger = 0; + target_file_size_base = 0; + target_file_size_multiplier = 0; + max_bytes_for_level_base = 0; + max_bytes_for_level_multiplier = 0; + max_grandparent_overlap_factor = 0; + disableDataSync = 0; __isset.__clear(); } @@ -266,6 +276,16 @@ class DBOptions { int32_t block_size; int32_t block_restart_interval; CompressionType compression; + int32_t num_levels; + int32_t level0_file_num_compaction_trigger; + int32_t level0_slowdown_writes_trigger; + int32_t level0_stop_writes_trigger; + int32_t target_file_size_base; + int32_t target_file_size_multiplier; + int32_t max_bytes_for_level_base; + int32_t max_bytes_for_level_multiplier; + int32_t max_grandparent_overlap_factor; + bool disableDataSync; struct __isset { __isset() { __clear(); } @@ -277,6 +297,16 @@ class DBOptions { block_size = false; block_restart_interval = false; compression = false; + num_levels = false; + level0_file_num_compaction_trigger = false; + level0_slowdown_writes_trigger = false; + level0_stop_writes_trigger = false; + target_file_size_base = false; + target_file_size_multiplier = false; + max_bytes_for_level_base = false; + max_bytes_for_level_multiplier = false; + max_grandparent_overlap_factor = false; + disableDataSync = false; } bool create_if_missing; bool error_if_exists; @@ -285,6 +315,16 @@ class DBOptions { bool block_size; bool block_restart_interval; bool compression; + bool num_levels; + bool level0_file_num_compaction_trigger; + bool level0_slowdown_writes_trigger; + bool level0_stop_writes_trigger; + bool target_file_size_base; + bool target_file_size_multiplier; + bool max_bytes_for_level_base; + bool max_bytes_for_level_multiplier; + bool max_grandparent_overlap_factor; + bool disableDataSync; } __isset; bool operator == (const DBOptions & rhs) const @@ -303,6 +343,26 @@ class DBOptions { return false; if (!(this->compression == rhs.compression)) return false; + if (!(this->num_levels == rhs.num_levels)) + return false; + if (!(this->level0_file_num_compaction_trigger == rhs.level0_file_num_compaction_trigger)) + return false; + if (!(this->level0_slowdown_writes_trigger == rhs.level0_slowdown_writes_trigger)) + return false; + if (!(this->level0_stop_writes_trigger == rhs.level0_stop_writes_trigger)) + return false; + if (!(this->target_file_size_base == rhs.target_file_size_base)) + return false; + if (!(this->target_file_size_multiplier == rhs.target_file_size_multiplier)) + return false; + if (!(this->max_bytes_for_level_base == rhs.max_bytes_for_level_base)) + return false; + if (!(this->max_bytes_for_level_multiplier == rhs.max_bytes_for_level_multiplier)) + return false; + if (!(this->max_grandparent_overlap_factor == rhs.max_grandparent_overlap_factor)) + return false; + if (!(this->disableDataSync == rhs.disableDataSync)) + return false; return true; } bool operator != (const DBOptions &rhs) const { @@ -324,7 +384,7 @@ class WriteOptions { static const uint64_t _reflection_id = 8830325115029814540U; static void _reflection_register(::apache::thrift::reflection::Schema&); - WriteOptions() : sync(0) { + WriteOptions() : sync(0), disableWAL(0) { } WriteOptions(const WriteOptions&) = default; @@ -334,25 +394,31 @@ class WriteOptions { void __clear() { sync = 0; + disableWAL = 0; __isset.__clear(); } virtual ~WriteOptions() throw() {} bool sync; + bool disableWAL; struct __isset { __isset() { __clear(); } void __clear() { sync = false; + disableWAL = false; } bool sync; + bool disableWAL; } __isset; bool operator == (const WriteOptions & rhs) const { if (!(this->sync == rhs.sync)) return false; + if (!(this->disableWAL == rhs.disableWAL)) + return false; return true; } bool operator != (const WriteOptions &rhs) const { diff --git a/thrift/if/leveldb.thrift b/thrift/if/leveldb.thrift index c78c04541..1768bb897 100644 --- a/thrift/if/leveldb.thrift +++ b/thrift/if/leveldb.thrift @@ -47,12 +47,23 @@ struct DBOptions { 4:i32 max_open_files; 5:i32 block_size; 6:i32 block_restart_interval; - 7:CompressionType compression + 7:CompressionType compression, + 8:i32 num_levels, + 9:i32 level0_file_num_compaction_trigger, + 10:i32 level0_slowdown_writes_trigger, + 11:i32 level0_stop_writes_trigger, + 12:i32 target_file_size_base, + 13:i32 target_file_size_multiplier, + 14:i32 max_bytes_for_level_base, + 15:i32 max_bytes_for_level_multiplier, + 16:i32 max_grandparent_overlap_factor, + 17:bool disableDataSync } // Options for writing struct WriteOptions { - 1:bool sync + 1:bool sync, + 2:bool disableWAL } struct Snapshot { diff --git a/thrift/server_utils.cpp b/thrift/server_utils.cpp index d155bb74f..120737f8b 100644 --- a/thrift/server_utils.cpp +++ b/thrift/server_utils.cpp @@ -75,6 +75,26 @@ class DBHandler : virtual public DBIf { } else if (dboptions.compression == kSnappyCompression) { options.compression = leveldb::kSnappyCompression; } + if (dboptions.num_levels > 0) + options.num_levels = dboptions.num_levels; + if (dboptions.level0_file_num_compaction_trigger > 0) + options.level0_file_num_compaction_trigger = dboptions.level0_file_num_compaction_trigger; + if (dboptions.level0_slowdown_writes_trigger > 0) + options.level0_slowdown_writes_trigger = dboptions.level0_slowdown_writes_trigger; + if (dboptions.level0_stop_writes_trigger) + options.level0_stop_writes_trigger = dboptions.level0_stop_writes_trigger; + if (dboptions.target_file_size_base > 0) + options.target_file_size_base = dboptions.target_file_size_base; + if (dboptions.target_file_size_multiplier > 0) + options.target_file_size_multiplier = dboptions.target_file_size_multiplier; + if (dboptions.max_bytes_for_level_base) + options.max_bytes_for_level_base = dboptions.max_bytes_for_level_base; + if (dboptions.max_bytes_for_level_multiplier) + options.max_bytes_for_level_multiplier = dboptions.max_bytes_for_level_multiplier; + if (dboptions.max_grandparent_overlap_factor) + options.max_grandparent_overlap_factor = dboptions.max_grandparent_overlap_factor; + if (dboptions.disableDataSync) + options.disableDataSync = dboptions.disableDataSync; openHandles->add(options, dbname, dbdir); _return.dbname = dbname; } @@ -91,6 +111,7 @@ class DBHandler : virtual public DBIf { const WriteOptions& options) { leveldb::WriteOptions woptions; woptions.sync = options.sync; + woptions.disableWAL = options.disableWAL; leveldb::Slice key, value; key.data_ = kv.key.data.data(); key.size_ = kv.key.size; @@ -111,6 +132,7 @@ class DBHandler : virtual public DBIf { const WriteOptions& options) { leveldb::WriteOptions woptions; woptions.sync = options.sync; + woptions.disableWAL = options.disableWAL; leveldb::Slice key; key.data_ = kv.data.data(); key.size_ = kv.size; @@ -130,6 +152,7 @@ class DBHandler : virtual public DBIf { leveldb::WriteOptions woptions; leveldb::WriteBatch lbatch; woptions.sync = options.sync; + woptions.disableWAL = options.disableWAL; leveldb::Slice key, value; for (unsigned int i = 0; i < batch.size(); i++) { kv one = batch[i];