andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fix listener_test when using ROCKSDB_MALLOC_USABLE_SIZE

Browse Source 
Summary:
Flushes in listener_test happened to early when ROCKSDB_MALLOC_USABLE_SIZE was
active (e.g. when compiling with ROCKSDB_FBCODE_BUILD_WITH_481=1) due to
malloc_usable_size() reporting a better estimate (similar to
https://reviews.facebook.net/D43317 ). This patch grows the write buffer size
slightly to compensate for this.

Test Plan: ROCKSDB_FBCODE_BUILD_WITH_481=1 make listener_test && ./listener_test

Reviewers: rven, anthony, yhchiang, igor, sdong

Reviewed By: igor

Subscribers: dhruba, leveldb

Differential Revision: https://reviews.facebook.net/D45921
This commit is contained in:
Andres Notzli 2015-08-31 23:11:12 -07:00
parent 40cd91b7fe
commit b722007778
3 changed files with 17 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
db/db_compaction_test.cc
View File

@ -505,7 +505,7 @@ TEST_F(DBCompactionTest, DisableStatsUpdateReopen) {
TEST_P(DBCompactionTestWithParam, CompactionTrigger) {
Options options;
options.write_buffer_size = 110 << 10; // 100KB
options.write_buffer_size = 110 << 10; // 110KB
options.arena_block_size = 4 << 10;
options.num_levels = 3;
options.level0_file_num_compaction_trigger = 3;
@ -1008,7 +1008,7 @@ TEST_P(DBCompactionTestWithParam, LevelCompactionPathUse) {
options.db_paths.emplace_back(dbname_ + "_2", 4 * 1024 * 1024);
options.db_paths.emplace_back(dbname_ + "_3", 1024 * 1024 * 1024);
options.compaction_style = kCompactionStyleLevel;
options.write_buffer_size = 110 << 10; // 100KB
options.write_buffer_size = 110 << 10; // 110KB
options.arena_block_size = 4 << 10;
options.level0_file_num_compaction_trigger = 2;
options.num_levels = 4;
@ -1639,7 +1639,7 @@ TEST_P(DBCompactionTestWithParam, CompressLevelCompaction) {
}
Options options = CurrentOptions();
options.compaction_style = kCompactionStyleLevel;
options.write_buffer_size = 110 << 10; // 100KB
options.write_buffer_size = 110 << 10; // 110KB
options.arena_block_size = 4 << 10;
options.level0_file_num_compaction_trigger = 2;
options.num_levels = 4;

23
db/listener_test.cc
View File

@ -2,8 +2,9 @@
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
#include "db/dbformat.h"
#include "db/db_impl.h"
#include "db/dbformat.h"
#include "db/filename.h"
#include "db/version_set.h"
#include "db/write_batch_internal.h"
@ -12,25 +13,25 @@
#include "rocksdb/db.h"
#include "rocksdb/env.h"
#include "rocksdb/filter_policy.h"
#include "rocksdb/options.h"
#include "rocksdb/perf_context.h"
#include "rocksdb/slice.h"
#include "rocksdb/slice_transform.h"
#include "rocksdb/table.h"
#include "rocksdb/options.h"
#include "rocksdb/table_properties.h"
#include "table/block_based_table_factory.h"
#include "table/plain_table_factory.h"
#include "util/hash.h"
#include "util/hash_linklist_rep.h"
#include "utilities/merge_operators.h"
#include "util/logging.h"
#include "util/mutexlock.h"
#include "util/rate_limiter.h"
#include "util/statistics.h"
#include "util/string_util.h"
#include "util/testharness.h"
#include "util/sync_point.h"
#include "util/testharness.h"
#include "util/testutil.h"
#include "utilities/merge_operators.h"
#ifndef ROCKSDB_LITE
@ -134,7 +135,7 @@ class EventListenerTest : public testing::Test {
return db_->Put(wo, handles_[cf], k, v);
}
Status Flush(int cf = 0) {
Status Flush(size_t cf = 0) {
FlushOptions opt = FlushOptions();
opt.wait = true;
if (cf == 0) {
@ -144,6 +145,8 @@ class EventListenerTest : public testing::Test {
}
}
const size_t k110KB = 110 << 10;
DB* db_;
std::string dbname_;
std::vector<ColumnFamilyHandle*> handles_;
@ -198,7 +201,7 @@ TEST_F(EventListenerTest, OnSingleDBCompactionTest) {
ASSERT_OK(Put(6, "alyosha", std::string(90000, 'a')));
ASSERT_OK(Put(7, "popovich", std::string(90000, 'p')));
for (size_t i = 1; i < 8; ++i) {
ASSERT_OK(Flush(static_cast<int>(i)));
ASSERT_OK(Flush(i));
const Slice kStart = "a";
const Slice kEnd = "z";
ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), handles_[i],
@ -287,7 +290,7 @@ class TestFlushListener : public EventListener {
TEST_F(EventListenerTest, OnSingleDBFlushTest) {
Options options;
options.write_buffer_size = 100000;
options.write_buffer_size = k110KB;
#if ROCKSDB_USING_THREAD_STATUS
options.enable_thread_tracking = true;
#endif // ROCKSDB_USING_THREAD_STATUS
@ -306,7 +309,7 @@ TEST_F(EventListenerTest, OnSingleDBFlushTest) {
ASSERT_OK(Put(6, "alyosha", std::string(90000, 'a')));
ASSERT_OK(Put(7, "popovich", std::string(90000, 'p')));
for (size_t i = 1; i < 8; ++i) {
ASSERT_OK(Flush(static_cast<int>(i)));
ASSERT_OK(Flush(i));
dbfull()->TEST_WaitForFlushMemTable();
ASSERT_EQ(listener->flushed_dbs_.size(), i);
ASSERT_EQ(listener->flushed_column_family_names_.size(), i);
@ -321,7 +324,7 @@ TEST_F(EventListenerTest, OnSingleDBFlushTest) {
TEST_F(EventListenerTest, MultiCF) {
Options options;
options.write_buffer_size = 100000;
options.write_buffer_size = k110KB;
#if ROCKSDB_USING_THREAD_STATUS
options.enable_thread_tracking = true;
#endif // ROCKSDB_USING_THREAD_STATUS
@ -340,7 +343,7 @@ TEST_F(EventListenerTest, MultiCF) {
ASSERT_OK(Put(6, "alyosha", std::string(90000, 'a')));
ASSERT_OK(Put(7, "popovich", std::string(90000, 'p')));
for (size_t i = 1; i < 8; ++i) {
ASSERT_OK(Flush(static_cast<int>(i)));
ASSERT_OK(Flush(i));
ASSERT_EQ(listener->flushed_dbs_.size(), i);
ASSERT_EQ(listener->flushed_column_family_names_.size(), i);
}

2
db/memtable.cc
View File

@ -112,7 +112,7 @@ bool MemTable::ShouldFlushNow() const {
// In a lot of times, we cannot allocate arena blocks that exactly matches the
// buffer size. Thus we have to decide if we should over-allocate or
// under-allocate.
// This constant avariable can be interpreted as: if we still have more than
// This constant variable can be interpreted as: if we still have more than
// "kAllowOverAllocationRatio * kArenaBlockSize" space left, we'd try to over
// allocate one more block.
const double kAllowOverAllocationRatio = 0.6;