andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Change max_bytes_for_level_multiplier to double

Browse Source 
Summary: Closes https://github.com/facebook/rocksdb/pull/1427

Differential Revision: D4094732

Pulled By: yiwu-arbug

fbshipit-source-id: b9b79e9
This commit is contained in:
Benoit Girard 2016-11-01 21:05:32 -07:00 committed by Facebook Github Bot
parent 16fb04434f
commit 2b16d664cb
23 changed files with 77 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
HISTORY.md
View File

@ -1,4 +1,8 @@
# Rocksdb Change Log
## Unreleased
### Public API Change
* Options::max_bytes_for_level_multiplier is now a double along with all getters and setters.
## 4.13.0 (10/18/2016)
### Public API Change
* DB::GetOptions() reflect dynamic changed options (i.e. through DB::SetOptions()) and return copy of options instead of reference.

4
db/c.cc
View File

@ -1517,8 +1517,8 @@ void rocksdb_options_set_max_bytes_for_level_base(
opt->rep.max_bytes_for_level_base = n;
}
void rocksdb_options_set_max_bytes_for_level_multiplier(
rocksdb_options_t* opt, int n) {
void rocksdb_options_set_max_bytes_for_level_multiplier(rocksdb_options_t* opt,
double n) {
opt->rep.max_bytes_for_level_multiplier = n;
}

3
db/compaction_picker.cc
View File

@ -1184,7 +1184,8 @@ uint32_t LevelCompactionPicker::GetPathId(
return p;
} else {
current_path_size -= level_size;
level_size *= mutable_cf_options.max_bytes_for_level_multiplier;
level_size = static_cast<uint64_t>(
level_size * mutable_cf_options.max_bytes_for_level_multiplier);
cur_level++;
continue;
}

6
db/db_compaction_test.cc
View File

@ -1259,7 +1259,8 @@ TEST_F(DBCompactionTest, ManualPartialFill) {
uint64_t target_size = 4 * options.max_bytes_for_level_base;
for (int32_t i = 1; i < options.num_levels; i++) {
ASSERT_LE(SizeAtLevel(i), target_size);
target_size *= options.max_bytes_for_level_multiplier;
target_size = static_cast<uint64_t>(target_size *
options.max_bytes_for_level_multiplier);
}
TEST_SYNC_POINT("DBCompaction::PartialFill:2");
@ -1336,7 +1337,8 @@ TEST_F(DBCompactionTest, DeleteFileRange) {
uint64_t target_size = 4 * options.max_bytes_for_level_base;
for (int32_t i = 1; i < options.num_levels; i++) {
ASSERT_LE(SizeAtLevel(i), target_size);
target_size *= options.max_bytes_for_level_multiplier;
target_size = static_cast<uint64_t>(target_size *
options.max_bytes_for_level_multiplier);
}
size_t old_num_files = CountFiles();

11
db/version_set.cc
View File

@ -1980,14 +1980,15 @@ void VersionStorageInfo::CalculateBaseBytes(const ImmutableCFOptions& ioptions,
base_level_ = num_levels_ - 1;
} else {
uint64_t base_bytes_max = options.max_bytes_for_level_base;
uint64_t base_bytes_min =
base_bytes_max / options.max_bytes_for_level_multiplier;
uint64_t base_bytes_min = static_cast<uint64_t>(
base_bytes_max / options.max_bytes_for_level_multiplier);
// Try whether we can make last level's target size to be max_level_size
uint64_t cur_level_size = max_level_size;
for (int i = num_levels_ - 2; i >= first_non_empty_level; i--) {
// Round up after dividing
cur_level_size /= options.max_bytes_for_level_multiplier;
cur_level_size = static_cast<uint64_t>(
cur_level_size / options.max_bytes_for_level_multiplier);
}
// Calculate base level and its size.
@ -2006,8 +2007,8 @@ void VersionStorageInfo::CalculateBaseBytes(const ImmutableCFOptions& ioptions,
base_level_ = first_non_empty_level;
while (base_level_ > 1 && cur_level_size > base_bytes_max) {
--base_level_;
cur_level_size =
cur_level_size / options.max_bytes_for_level_multiplier;
cur_level_size = static_cast<uint64_t>(
cur_level_size / options.max_bytes_for_level_multiplier);
}
if (cur_level_size > base_bytes_max) {
// Even L1 will be too large

2
include/rocksdb/c.h
View File

@ -561,7 +561,7 @@ extern ROCKSDB_LIBRARY_API void rocksdb_options_set_target_file_size_multiplier(
extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_bytes_for_level_base(
rocksdb_options_t*, uint64_t);
extern ROCKSDB_LIBRARY_API void
rocksdb_options_set_max_bytes_for_level_multiplier(rocksdb_options_t*, int);
rocksdb_options_set_max_bytes_for_level_multiplier(rocksdb_options_t*, double);
extern ROCKSDB_LIBRARY_API void
rocksdb_options_set_max_bytes_for_level_multiplier_additional(
rocksdb_options_t*, int* level_values, size_t num_levels);

2
include/rocksdb/options.h
View File

@ -527,7 +527,7 @@ struct ColumnFamilyOptions {
// Default: 10.
//
// Dynamically changeable through SetOptions() API
int max_bytes_for_level_multiplier;
double max_bytes_for_level_multiplier;
// Different max-size multipliers for different levels.
// These are multiplied by max_bytes_for_level_multiplier to arrive

8
java/benchmark/src/main/java/org/rocksdb/benchmark/DbBenchmark.java
View File

@ -574,12 +574,10 @@ public class DbBenchmark {
(Integer)flags_.get(Flag.num_levels));
options.setTargetFileSizeBase(
(Integer)flags_.get(Flag.target_file_size_base));
options.setTargetFileSizeMultiplier(
(Integer)flags_.get(Flag.target_file_size_multiplier));
options.setTargetFileSizeMultiplier((Double) flags_.get(Flag.target_file_size_multiplier));
options.setMaxBytesForLevelBase(
(Integer)flags_.get(Flag.max_bytes_for_level_base));
options.setMaxBytesForLevelMultiplier(
(Integer)flags_.get(Flag.max_bytes_for_level_multiplier));
options.setMaxBytesForLevelMultiplier((Double) flags_.get(Flag.max_bytes_for_level_multiplier));
options.setLevelZeroStopWritesTrigger(
(Integer)flags_.get(Flag.level0_stop_writes_trigger));
options.setLevelZeroSlowdownWritesTrigger(
@ -1268,7 +1266,7 @@ public class DbBenchmark {
max_bytes_for_level_multiplier(10,
"A multiplier to compute max bytes for level-N (N >= 2)") {
@Override public Object parseValue(String value) {
return Integer.parseInt(value);
return Double.parseDouble(value);
}
},
level0_stop_writes_trigger(12,"Number of files in level-0\n" +

30
java/rocksjni/options.cc
View File

@ -1522,10 +1522,11 @@ void Java_org_rocksdb_Options_setLevelCompactionDynamicLevelBytes(
/*
* Class: org_rocksdb_Options
* Method: maxBytesForLevelMultiplier
* Signature: (J)I
* Signature: (J)D
*/
jint Java_org_rocksdb_Options_maxBytesForLevelMultiplier(
JNIEnv* env, jobject jobj, jlong jhandle) {
jdouble Java_org_rocksdb_Options_maxBytesForLevelMultiplier(JNIEnv* env,
jobject jobj,
jlong jhandle) {
return reinterpret_cast<rocksdb::Options*>(
jhandle)->max_bytes_for_level_multiplier;
}
@ -1533,14 +1534,13 @@ jint Java_org_rocksdb_Options_maxBytesForLevelMultiplier(
/*
* Class: org_rocksdb_Options
* Method: setMaxBytesForLevelMultiplier
* Signature: (JI)V
* Signature: (JD)V
*/
void Java_org_rocksdb_Options_setMaxBytesForLevelMultiplier(
JNIEnv* env, jobject jobj, jlong jhandle,
jint jmax_bytes_for_level_multiplier) {
reinterpret_cast<rocksdb::Options*>(
jhandle)->max_bytes_for_level_multiplier =
static_cast<int>(jmax_bytes_for_level_multiplier);
jdouble jmax_bytes_for_level_multiplier) {
reinterpret_cast<rocksdb::Options*>(jhandle)->max_bytes_for_level_multiplier =
static_cast<double>(jmax_bytes_for_level_multiplier);
}
/*
@ -2823,9 +2823,9 @@ void Java_org_rocksdb_ColumnFamilyOptions_setLevelCompactionDynamicLevelBytes(
/*
* Class: org_rocksdb_ColumnFamilyOptions
* Method: maxBytesForLevelMultiplier
* Signature: (J)I
* Signature: (J)D
*/
jint Java_org_rocksdb_ColumnFamilyOptions_maxBytesForLevelMultiplier(
jdouble Java_org_rocksdb_ColumnFamilyOptions_maxBytesForLevelMultiplier(
JNIEnv* env, jobject jobj, jlong jhandle) {
return reinterpret_cast<rocksdb::ColumnFamilyOptions*>(
jhandle)->max_bytes_for_level_multiplier;
@ -2834,14 +2834,14 @@ jint Java_org_rocksdb_ColumnFamilyOptions_maxBytesForLevelMultiplier(
/*
* Class: org_rocksdb_ColumnFamilyOptions
* Method: setMaxBytesForLevelMultiplier
* Signature: (JI)V
* Signature: (JD)V
*/
void Java_org_rocksdb_ColumnFamilyOptions_setMaxBytesForLevelMultiplier(
JNIEnv* env, jobject jobj, jlong jhandle,
jint jmax_bytes_for_level_multiplier) {
reinterpret_cast<rocksdb::ColumnFamilyOptions*>(
jhandle)->max_bytes_for_level_multiplier =
static_cast<int>(jmax_bytes_for_level_multiplier);
jdouble jmax_bytes_for_level_multiplier) {
reinterpret_cast<rocksdb::ColumnFamilyOptions*>(jhandle)
->max_bytes_for_level_multiplier =
static_cast<double>(jmax_bytes_for_level_multiplier);
}
/*

10
java/src/main/java/org/rocksdb/ColumnFamilyOptions.java
View File

@ -352,14 +352,13 @@ public class ColumnFamilyOptions extends RocksObject
}
@Override
public ColumnFamilyOptions setMaxBytesForLevelMultiplier(
final int multiplier) {
public ColumnFamilyOptions setMaxBytesForLevelMultiplier(final double multiplier) {
setMaxBytesForLevelMultiplier(nativeHandle_, multiplier);
return this;
}
@Override
public int maxBytesForLevelMultiplier() {
public double maxBytesForLevelMultiplier() {
return maxBytesForLevelMultiplier(nativeHandle_);
}
@ -775,9 +774,8 @@ public class ColumnFamilyOptions extends RocksObject
long handle, boolean enableLevelCompactionDynamicLevelBytes);
private native boolean levelCompactionDynamicLevelBytes(
long handle);
private native void setMaxBytesForLevelMultiplier(
long handle, int multiplier);
private native int maxBytesForLevelMultiplier(long handle);
private native void setMaxBytesForLevelMultiplier(long handle, double multiplier);
private native double maxBytesForLevelMultiplier(long handle);
private native void setMaxCompactionBytes(long handle, long maxCompactionBytes);
private native long maxCompactionBytes(long handle);
private native void setSoftRateLimit(

4
java/src/main/java/org/rocksdb/ColumnFamilyOptionsInterface.java
View File

@ -456,7 +456,7 @@ public interface ColumnFamilyOptionsInterface {
* files and the total size of level-L files for all L.
* @return the reference to the current option.
*/
Object setMaxBytesForLevelMultiplier(int multiplier);
Object setMaxBytesForLevelMultiplier(double multiplier);
/**
* The ratio between the total size of level-(L+1) files and the total
@ -466,7 +466,7 @@ public interface ColumnFamilyOptionsInterface {
* @return the ratio between the total size of level-(L+1) files and
* the total size of level-L files for all L.
*/
int maxBytesForLevelMultiplier();
double maxBytesForLevelMultiplier();
/**
* Maximum size of each compaction (not guarantee)

9
java/src/main/java/org/rocksdb/MutableColumnFamilyOptions.java
View File

@ -843,14 +843,13 @@ public class MutableColumnFamilyOptions {
@Override
public MutableColumnFamilyOptionsBuilder setMaxBytesForLevelMultiplier(
final int maxBytesForLevelMultiplier) {
return setInt(CompactionOption.max_bytes_for_level_multiplier,
maxBytesForLevelMultiplier);
final double maxBytesForLevelMultiplier) {
return setDouble(CompactionOption.max_bytes_for_level_multiplier, maxBytesForLevelMultiplier);
}
@Override
public int maxBytesForLevelMultiplier() {
return getInt(CompactionOption.max_bytes_for_level_multiplier);
public double maxBytesForLevelMultiplier() {
return getDouble(CompactionOption.max_bytes_for_level_multiplier);
}
@Override

7
java/src/main/java/org/rocksdb/MutableColumnFamilyOptionsInterface.java
View File

@ -474,7 +474,7 @@ public interface MutableColumnFamilyOptionsInterface {
* @param maxBytesForLevelBase maximum bytes for level base.
*
* @return the reference to the current option.
* @see #setMaxBytesForLevelMultiplier(int)
* @see #setMaxBytesForLevelMultiplier(double)
*/
MutableColumnFamilyOptionsInterface setMaxBytesForLevelBase(
long maxBytesForLevelBase);
@ -505,8 +505,7 @@ public interface MutableColumnFamilyOptionsInterface {
* @return the reference to the current option.
* @see #setMaxBytesForLevelBase(long)
*/
MutableColumnFamilyOptionsInterface setMaxBytesForLevelMultiplier(
int multiplier);
MutableColumnFamilyOptionsInterface setMaxBytesForLevelMultiplier(double multiplier);
/**
* The ratio between the total size of level-(L+1) files and the total
@ -517,7 +516,7 @@ public interface MutableColumnFamilyOptionsInterface {
* the total size of level-L files for all L.
* @see #maxBytesForLevelBase()
*/
int maxBytesForLevelMultiplier();
double maxBytesForLevelMultiplier();
/**
* Different max-size multipliers for different levels.

9
java/src/main/java/org/rocksdb/Options.java
View File

@ -904,12 +904,12 @@ public class Options extends RocksObject
}
@Override
public int maxBytesForLevelMultiplier() {
public double maxBytesForLevelMultiplier() {
return maxBytesForLevelMultiplier(nativeHandle_);
}
@Override
public Options setMaxBytesForLevelMultiplier(final int multiplier) {
public Options setMaxBytesForLevelMultiplier(final double multiplier) {
setMaxBytesForLevelMultiplier(nativeHandle_, multiplier);
return this;
}
@ -1382,9 +1382,8 @@ public class Options extends RocksObject
long handle, boolean enableLevelCompactionDynamicLevelBytes);
private native boolean levelCompactionDynamicLevelBytes(
long handle);
private native void setMaxBytesForLevelMultiplier(
long handle, int multiplier);
private native int maxBytesForLevelMultiplier(long handle);
private native void setMaxBytesForLevelMultiplier(long handle, double multiplier);
private native double maxBytesForLevelMultiplier(long handle);
private native void setMaxCompactionBytes(long handle, long maxCompactionBytes);
private native long maxCompactionBytes(long handle);
private native void setSoftRateLimit(

6
java/src/test/java/org/rocksdb/ColumnFamilyOptionsTest.java
View File

@ -172,9 +172,9 @@ public class ColumnFamilyOptionsTest {
@Test
public void maxBytesForLevelMultiplier() {
try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) {
final int intValue = rand.nextInt();
opt.setMaxBytesForLevelMultiplier(intValue);
assertThat(opt.maxBytesForLevelMultiplier()).isEqualTo(intValue);
final double doubleValue = rand.nextDouble();
opt.setMaxBytesForLevelMultiplier(doubleValue);
assertThat(opt.maxBytesForLevelMultiplier()).isEqualTo(doubleValue);
}
}

6
java/src/test/java/org/rocksdb/OptionsTest.java
View File

@ -135,9 +135,9 @@ public class OptionsTest {
@Test
public void maxBytesForLevelMultiplier() {
try (final Options opt = new Options()) {
final int intValue = rand.nextInt();
opt.setMaxBytesForLevelMultiplier(intValue);
assertThat(opt.maxBytesForLevelMultiplier()).isEqualTo(intValue);
final double doubleValue = rand.nextDouble();
opt.setMaxBytesForLevelMultiplier(doubleValue);
assertThat(opt.maxBytesForLevelMultiplier()).isEqualTo(doubleValue);
}
}

4
tools/db_bench_tool.cc
View File

@ -482,8 +482,8 @@ DEFINE_uint64(max_bytes_for_level_base, 10 * 1048576, "Max bytes for level-1");
DEFINE_bool(level_compaction_dynamic_level_bytes, false,
"Whether level size base is dynamic");
DEFINE_int32(max_bytes_for_level_multiplier, 10,
"A multiplier to compute max bytes for level-N (N >= 2)");
DEFINE_double(max_bytes_for_level_multiplier, 10,
"A multiplier to compute max bytes for level-N (N >= 2)");
static std::vector<int> FLAGS_max_bytes_for_level_multiplier_additional_v;
DEFINE_string(max_bytes_for_level_multiplier_additional, "",

4
tools/db_stress.cc
View File

@ -303,8 +303,8 @@ DEFINE_int32(target_file_size_multiplier, 1,
DEFINE_uint64(max_bytes_for_level_base, 256 * KB, "Max bytes for level-1");
DEFINE_int32(max_bytes_for_level_multiplier, 2,
"A multiplier to compute max bytes for level-N (N >= 2)");
DEFINE_double(max_bytes_for_level_multiplier, 2,
"A multiplier to compute max bytes for level-N (N >= 2)");
// Temporarily disable this to allows it to detect new bugs
DEFINE_int32(compact_files_one_in, 0,

14
util/cf_options.cc
View File

@ -75,18 +75,14 @@ ImmutableCFOptions::ImmutableCFOptions(const ImmutableDBOptions& db_options,
max_subcompactions(db_options.max_subcompactions) {}
// Multiple two operands. If they overflow, return op1.
uint64_t MultiplyCheckOverflow(uint64_t op1, int op2) {
if (op1 == 0) {
uint64_t MultiplyCheckOverflow(uint64_t op1, double op2) {
if (op1 == 0 || op2 <= 0) {
return 0;
}
if (op2 <= 0) {
if (port::kMaxUint64 / op1 < op2) {
return op1;
}
uint64_t casted_op2 = (uint64_t) op2;
if (std::numeric_limits<uint64_t>::max() / op1 < casted_op2) {
return op1;
}
return op1 * casted_op2;
return static_cast<uint64_t>(op1 * op2);
}
void MutableCFOptions::RefreshDerivedOptions(int num_levels,
@ -146,7 +142,7 @@ void MutableCFOptions::Dump(Logger* log) const {
target_file_size_multiplier);
Log(log, " max_bytes_for_level_base: %" PRIu64,
max_bytes_for_level_base);
Log(log, " max_bytes_for_level_multiplier: %d",
Log(log, " max_bytes_for_level_multiplier: %f",
max_bytes_for_level_multiplier);
std::string result;
char buf[10];

4
util/cf_options.h
View File

@ -220,7 +220,7 @@ struct MutableCFOptions {
uint64_t target_file_size_base;
int target_file_size_multiplier;
uint64_t max_bytes_for_level_base;
int max_bytes_for_level_multiplier;
double max_bytes_for_level_multiplier;
std::vector<int> max_bytes_for_level_multiplier_additional;
bool verify_checksums_in_compaction;
@ -236,6 +236,6 @@ struct MutableCFOptions {
std::vector<uint64_t> max_file_size;
};
uint64_t MultiplyCheckOverflow(uint64_t op1, int op2);
uint64_t MultiplyCheckOverflow(uint64_t op1, double op2);
} // namespace rocksdb

4
util/options.cc
View File

@ -489,8 +489,8 @@ void ColumnFamilyOptions::Dump(Logger* log) const {
max_bytes_for_level_base);
Header(log, "Options.level_compaction_dynamic_level_bytes: %d",
level_compaction_dynamic_level_bytes);
Header(log, " Options.max_bytes_for_level_multiplier: %d",
max_bytes_for_level_multiplier);
Header(log, " Options.max_bytes_for_level_multiplier: %f",
max_bytes_for_level_multiplier);
for (size_t i = 0; i < max_bytes_for_level_multiplier_additional.size();
i++) {
Header(log,

2
util/options_helper.h
View File

@ -490,7 +490,7 @@ static std::unordered_map<std::string, OptionTypeInfo> cf_options_type_info = {
offsetof(struct MutableCFOptions, max_bytes_for_level_base)}},
{"max_bytes_for_level_multiplier",
{offsetof(struct ColumnFamilyOptions, max_bytes_for_level_multiplier),
OptionType::kInt, OptionVerificationType::kNormal, true,
OptionType::kDouble, OptionVerificationType::kNormal, true,
offsetof(struct MutableCFOptions, max_bytes_for_level_multiplier)}},
{"max_bytes_for_level_multiplier_additional",
{offsetof(struct ColumnFamilyOptions,

4
util/options_test.cc
View File

@ -68,7 +68,7 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) {
{"target_file_size_multiplier", "13"},
{"max_bytes_for_level_base", "14"},
{"level_compaction_dynamic_level_bytes", "true"},
{"max_bytes_for_level_multiplier", "15"},
{"max_bytes_for_level_multiplier", "15.0"},
{"max_bytes_for_level_multiplier_additional", "16:17:18"},
{"max_compaction_bytes", "21"},
{"soft_rate_limit", "1.1"},
@ -164,7 +164,7 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) {
ASSERT_EQ(new_cf_opt.target_file_size_multiplier, 13);
ASSERT_EQ(new_cf_opt.max_bytes_for_level_base, 14U);
ASSERT_EQ(new_cf_opt.level_compaction_dynamic_level_bytes, true);
ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier, 15);
ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier, 15.0);
ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier_additional.size(), 3U);
ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier_additional[0], 16);
ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier_additional[1], 17);