andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

using ThreadLocalPtr to hide ROCKSDB_SUPPORT_THREAD_LOCAL from public…

Browse Source 
Summary:
… headers

https://github.com/facebook/rocksdb/pull/2199 should not reference RocksDB-specific macros (like ROCKSDB_SUPPORT_THREAD_LOCAL in this case) to public headers, `iostats_context.h` and `perf_context.h`. We shouldn't do that because users have to provide these compiler flags when building their binary with RocksDB.

We should hide the thread local global variable inside our implementation and just expose a function api to retrieve these variables. It may break some users for now but good for long term.

make check -j64
Closes https://github.com/facebook/rocksdb/pull/2380

Differential Revision: D5177896

Pulled By: lightmark

fbshipit-source-id: 6fcdfac57f2e2dcfe60992b7385c5403f6dcb390
This commit is contained in:
Aaron Gao 2017-06-02 17:12:39 -07:00 committed by Facebook Github Bot
parent 138b87eae4
commit 7f6c02dda1
20 changed files with 257 additions and 263 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
HISTORY.md
View File

@ -4,6 +4,7 @@
* Scheduling flushes and compactions in the same thread pool is no longer supported by setting `max_background_flushes=0`. Instead, users can achieve this by configuring their high-pri thread pool to have zero threads. * Scheduling flushes and compactions in the same thread pool is no longer supported by setting `max_background_flushes=0`. Instead, users can achieve this by configuring their high-pri thread pool to have zero threads.
* Replace `Options::max_background_flushes`, `Options::max_background_compactions`, and `Options::base_background_compactions` all with `Options::max_background_jobs`, which automatically decides how many threads to allocate towards flush/compaction. * Replace `Options::max_background_flushes`, `Options::max_background_compactions`, and `Options::base_background_compactions` all with `Options::max_background_jobs`, which automatically decides how many threads to allocate towards flush/compaction.
* options.delayed_write_rate by default take the value of options.rate_limiter rate. * options.delayed_write_rate by default take the value of options.rate_limiter rate.
* Replace global variable `IOStatsContext iostats_context` with `IOStatsContext* get_iostats_context()`; replace global variable `PerfContext perf_context` with `PerfContext* get_perf_context()`.
### New Features ### New Features
* Change ticker/histogram statistics implementations to use core-local storage. This improves aggregation speed compared to our previous thread-local approach, particularly for applications with many threads. * Change ticker/histogram statistics implementations to use core-local storage. This improves aggregation speed compared to our previous thread-local approach, particularly for applications with many threads.

8
db/db_basic_test.cc
View File

@ -366,9 +366,9 @@ TEST_F(DBBasicTest, FLUSH) {
ASSERT_OK(Flush(1)); ASSERT_OK(Flush(1));
ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "bar", "v1")); ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "bar", "v1"));
perf_context.Reset(); get_perf_context()->Reset();
Get(1, "foo"); Get(1, "foo");
ASSERT_TRUE((int)perf_context.get_from_output_files_time > 0); ASSERT_TRUE((int)get_perf_context()->get_from_output_files_time > 0);
ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions()); ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions());
ASSERT_EQ("v1", Get(1, "foo")); ASSERT_EQ("v1", Get(1, "foo"));
@ -381,9 +381,9 @@ TEST_F(DBBasicTest, FLUSH) {
ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions()); ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions());
ASSERT_EQ("v2", Get(1, "bar")); ASSERT_EQ("v2", Get(1, "bar"));
perf_context.Reset(); get_perf_context()->Reset();
ASSERT_EQ("v2", Get(1, "foo")); ASSERT_EQ("v2", Get(1, "foo"));
ASSERT_TRUE((int)perf_context.get_from_output_files_time > 0); ASSERT_TRUE((int)get_perf_context()->get_from_output_files_time > 0);
writeOpt.disableWAL = false; writeOpt.disableWAL = false;
ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "bar", "v3")); ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "bar", "v3"));

48
db/db_bloom_filter_test.cc
View File

@ -688,12 +688,12 @@ class BloomStatsTestWithParam
} }
options_.env = env_; options_.env = env_;
perf_context.Reset(); get_perf_context()->Reset();
DestroyAndReopen(options_); DestroyAndReopen(options_);
} }
~BloomStatsTestWithParam() { ~BloomStatsTestWithParam() {
perf_context.Reset(); get_perf_context()->Reset();
Destroy(options_); Destroy(options_);
} }
@ -726,33 +726,33 @@ TEST_P(BloomStatsTestWithParam, BloomStatsTest) {
// check memtable bloom stats // check memtable bloom stats
ASSERT_EQ(value1, Get(key1)); ASSERT_EQ(value1, Get(key1));
ASSERT_EQ(1, perf_context.bloom_memtable_hit_count); ASSERT_EQ(1, get_perf_context()->bloom_memtable_hit_count);
ASSERT_EQ(value3, Get(key3)); ASSERT_EQ(value3, Get(key3));
ASSERT_EQ(2, perf_context.bloom_memtable_hit_count); ASSERT_EQ(2, get_perf_context()->bloom_memtable_hit_count);
ASSERT_EQ(0, perf_context.bloom_memtable_miss_count); ASSERT_EQ(0, get_perf_context()->bloom_memtable_miss_count);
ASSERT_EQ("NOT_FOUND", Get(key2)); ASSERT_EQ("NOT_FOUND", Get(key2));
ASSERT_EQ(1, perf_context.bloom_memtable_miss_count); ASSERT_EQ(1, get_perf_context()->bloom_memtable_miss_count);
ASSERT_EQ(2, perf_context.bloom_memtable_hit_count); ASSERT_EQ(2, get_perf_context()->bloom_memtable_hit_count);
// sanity checks // sanity checks
ASSERT_EQ(0, perf_context.bloom_sst_hit_count); ASSERT_EQ(0, get_perf_context()->bloom_sst_hit_count);
ASSERT_EQ(0, perf_context.bloom_sst_miss_count); ASSERT_EQ(0, get_perf_context()->bloom_sst_miss_count);
Flush(); Flush();
// sanity checks // sanity checks
ASSERT_EQ(0, perf_context.bloom_sst_hit_count); ASSERT_EQ(0, get_perf_context()->bloom_sst_hit_count);
ASSERT_EQ(0, perf_context.bloom_sst_miss_count); ASSERT_EQ(0, get_perf_context()->bloom_sst_miss_count);
// check SST bloom stats // check SST bloom stats
ASSERT_EQ(value1, Get(key1)); ASSERT_EQ(value1, Get(key1));
ASSERT_EQ(1, perf_context.bloom_sst_hit_count); ASSERT_EQ(1, get_perf_context()->bloom_sst_hit_count);
ASSERT_EQ(value3, Get(key3)); ASSERT_EQ(value3, Get(key3));
ASSERT_EQ(2, perf_context.bloom_sst_hit_count); ASSERT_EQ(2, get_perf_context()->bloom_sst_hit_count);
ASSERT_EQ("NOT_FOUND", Get(key2)); ASSERT_EQ("NOT_FOUND", Get(key2));
ASSERT_EQ(1, perf_context.bloom_sst_miss_count); ASSERT_EQ(1, get_perf_context()->bloom_sst_miss_count);
} }
// Same scenario as in BloomStatsTest but using an iterator // Same scenario as in BloomStatsTest but using an iterator
@ -773,21 +773,21 @@ TEST_P(BloomStatsTestWithParam, BloomStatsTestWithIter) {
ASSERT_OK(iter->status()); ASSERT_OK(iter->status());
ASSERT_TRUE(iter->Valid()); ASSERT_TRUE(iter->Valid());
ASSERT_EQ(value1, iter->value().ToString()); ASSERT_EQ(value1, iter->value().ToString());
ASSERT_EQ(1, perf_context.bloom_memtable_hit_count); ASSERT_EQ(1, get_perf_context()->bloom_memtable_hit_count);
ASSERT_EQ(0, perf_context.bloom_memtable_miss_count); ASSERT_EQ(0, get_perf_context()->bloom_memtable_miss_count);
iter->Seek(key3); iter->Seek(key3);
ASSERT_OK(iter->status()); ASSERT_OK(iter->status());
ASSERT_TRUE(iter->Valid()); ASSERT_TRUE(iter->Valid());
ASSERT_EQ(value3, iter->value().ToString()); ASSERT_EQ(value3, iter->value().ToString());
ASSERT_EQ(2, perf_context.bloom_memtable_hit_count); ASSERT_EQ(2, get_perf_context()->bloom_memtable_hit_count);
ASSERT_EQ(0, perf_context.bloom_memtable_miss_count); ASSERT_EQ(0, get_perf_context()->bloom_memtable_miss_count);
iter->Seek(key2); iter->Seek(key2);
ASSERT_OK(iter->status()); ASSERT_OK(iter->status());
ASSERT_TRUE(!iter->Valid()); ASSERT_TRUE(!iter->Valid());
ASSERT_EQ(1, perf_context.bloom_memtable_miss_count); ASSERT_EQ(1, get_perf_context()->bloom_memtable_miss_count);
ASSERT_EQ(2, perf_context.bloom_memtable_hit_count); ASSERT_EQ(2, get_perf_context()->bloom_memtable_hit_count);
Flush(); Flush();
@ -798,19 +798,19 @@ TEST_P(BloomStatsTestWithParam, BloomStatsTestWithIter) {
ASSERT_OK(iter->status()); ASSERT_OK(iter->status());
ASSERT_TRUE(iter->Valid()); ASSERT_TRUE(iter->Valid());
ASSERT_EQ(value1, iter->value().ToString()); ASSERT_EQ(value1, iter->value().ToString());
ASSERT_EQ(1, perf_context.bloom_sst_hit_count); ASSERT_EQ(1, get_perf_context()->bloom_sst_hit_count);
iter->Seek(key3); iter->Seek(key3);
ASSERT_OK(iter->status()); ASSERT_OK(iter->status());
ASSERT_TRUE(iter->Valid()); ASSERT_TRUE(iter->Valid());
ASSERT_EQ(value3, iter->value().ToString()); ASSERT_EQ(value3, iter->value().ToString());
ASSERT_EQ(2, perf_context.bloom_sst_hit_count); ASSERT_EQ(2, get_perf_context()->bloom_sst_hit_count);
iter->Seek(key2); iter->Seek(key2);
ASSERT_OK(iter->status()); ASSERT_OK(iter->status());
ASSERT_TRUE(!iter->Valid()); ASSERT_TRUE(!iter->Valid());
ASSERT_EQ(1, perf_context.bloom_sst_miss_count); ASSERT_EQ(1, get_perf_context()->bloom_sst_miss_count);
ASSERT_EQ(2, perf_context.bloom_sst_hit_count); ASSERT_EQ(2, get_perf_context()->bloom_sst_hit_count);
} }
INSTANTIATE_TEST_CASE_P(BloomStatsTestWithParam, BloomStatsTestWithParam, INSTANTIATE_TEST_CASE_P(BloomStatsTestWithParam, BloomStatsTestWithParam,

8
db/db_iter_test.cc
View File

@ -354,11 +354,11 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
SetPerfLevel(kEnableCount); SetPerfLevel(kEnableCount);
ASSERT_TRUE(GetPerfLevel() == kEnableCount); ASSERT_TRUE(GetPerfLevel() == kEnableCount);
perf_context.Reset(); get_perf_context()->Reset();
db_iter->SeekToLast(); db_iter->SeekToLast();
ASSERT_TRUE(db_iter->Valid()); ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(static_cast<int>(perf_context.internal_key_skipped_count), 7); ASSERT_EQ(static_cast<int>(get_perf_context()->internal_key_skipped_count), 7);
ASSERT_EQ(db_iter->key().ToString(), "b"); ASSERT_EQ(db_iter->key().ToString(), "b");
SetPerfLevel(kDisable); SetPerfLevel(kDisable);
@ -473,11 +473,11 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
SetPerfLevel(kEnableCount); SetPerfLevel(kEnableCount);
ASSERT_TRUE(GetPerfLevel() == kEnableCount); ASSERT_TRUE(GetPerfLevel() == kEnableCount);
perf_context.Reset(); get_perf_context()->Reset();
db_iter->SeekToLast(); db_iter->SeekToLast();
ASSERT_TRUE(db_iter->Valid()); ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(static_cast<int>(perf_context.internal_delete_skipped_count), 1); ASSERT_EQ(static_cast<int>(get_perf_context()->internal_delete_skipped_count), 1);
ASSERT_EQ(db_iter->key().ToString(), "b"); ASSERT_EQ(db_iter->key().ToString(), "b");
SetPerfLevel(kDisable); SetPerfLevel(kDisable);

28
db/db_iterator_test.cc
View File

@ -974,11 +974,11 @@ TEST_F(DBIteratorTest, DBIteratorBoundTest) {
ASSERT_TRUE(iter->Valid()); ASSERT_TRUE(iter->Valid());
ASSERT_EQ(iter->key().compare(("b1")), 0); ASSERT_EQ(iter->key().compare(("b1")), 0);
perf_context.Reset(); get_perf_context()->Reset();
iter->Next(); iter->Next();
ASSERT_TRUE(iter->Valid()); ASSERT_TRUE(iter->Valid());
ASSERT_EQ(static_cast<int>(perf_context.internal_delete_skipped_count), 2); ASSERT_EQ(static_cast<int>(get_perf_context()->internal_delete_skipped_count), 2);
// now testing with iterate_bound // now testing with iterate_bound
Slice prefix("c"); Slice prefix("c");
@ -986,7 +986,7 @@ TEST_F(DBIteratorTest, DBIteratorBoundTest) {
iter.reset(db_->NewIterator(ro)); iter.reset(db_->NewIterator(ro));
perf_context.Reset(); get_perf_context()->Reset();
iter->Seek("b"); iter->Seek("b");
ASSERT_TRUE(iter->Valid()); ASSERT_TRUE(iter->Valid());
@ -1001,7 +1001,7 @@ TEST_F(DBIteratorTest, DBIteratorBoundTest) {
// even though the key is deleted // even though the key is deleted
// hence internal_delete_skipped_count should be 0 // hence internal_delete_skipped_count should be 0
ASSERT_TRUE(!iter->Valid()); ASSERT_TRUE(!iter->Valid());
ASSERT_EQ(static_cast<int>(perf_context.internal_delete_skipped_count), 0); ASSERT_EQ(static_cast<int>(get_perf_context()->internal_delete_skipped_count), 0);
} }
} }
@ -1888,7 +1888,7 @@ TEST_F(DBIteratorTest, DBIteratorSkipRecentDuplicatesTest) {
#endif #endif
// Seek iterator to a smaller key. // Seek iterator to a smaller key.
perf_context.Reset(); get_perf_context()->Reset();
iter->Seek("a"); iter->Seek("a");
ASSERT_TRUE(iter->Valid()); ASSERT_TRUE(iter->Valid());
EXPECT_EQ("b", iter->key().ToString()); EXPECT_EQ("b", iter->key().ToString());
@ -1896,17 +1896,17 @@ TEST_F(DBIteratorTest, DBIteratorSkipRecentDuplicatesTest) {
// Check that the seek didn't do too much work. // Check that the seek didn't do too much work.
// Checks are not tight, just make sure that everything is well below 100. // Checks are not tight, just make sure that everything is well below 100.
EXPECT_LT(perf_context.internal_key_skipped_count, 4); EXPECT_LT(get_perf_context()->internal_key_skipped_count, 4);
EXPECT_LT(perf_context.internal_recent_skipped_count, 8); EXPECT_LT(get_perf_context()->internal_recent_skipped_count, 8);
EXPECT_LT(perf_context.seek_on_memtable_count, 10); EXPECT_LT(get_perf_context()->seek_on_memtable_count, 10);
EXPECT_LT(perf_context.next_on_memtable_count, 10); EXPECT_LT(get_perf_context()->next_on_memtable_count, 10);
EXPECT_LT(perf_context.prev_on_memtable_count, 10); EXPECT_LT(get_perf_context()->prev_on_memtable_count, 10);
// Check that iterator did something like what we expect. // Check that iterator did something like what we expect.
EXPECT_EQ(perf_context.internal_delete_skipped_count, 0); EXPECT_EQ(get_perf_context()->internal_delete_skipped_count, 0);
EXPECT_EQ(perf_context.internal_merge_count, 0); EXPECT_EQ(get_perf_context()->internal_merge_count, 0);
EXPECT_GE(perf_context.internal_recent_skipped_count, 2); EXPECT_GE(get_perf_context()->internal_recent_skipped_count, 2);
EXPECT_GE(perf_context.seek_on_memtable_count, 2); EXPECT_GE(get_perf_context()->seek_on_memtable_count, 2);
EXPECT_EQ(1, options.statistics->getTickerCount( EXPECT_EQ(1, options.statistics->getTickerCount(
NUMBER_OF_RESEEKS_IN_ITERATION)); NUMBER_OF_RESEEKS_IN_ITERATION));
} }

38
db/db_properties_test.cc
View File

@ -533,9 +533,9 @@ TEST_F(DBPropertiesTest, NumImmutableMemTable) {
ASSERT_TRUE(dbfull()->GetProperty( ASSERT_TRUE(dbfull()->GetProperty(
handles_[1], "rocksdb.num-entries-active-mem-table", &num)); handles_[1], "rocksdb.num-entries-active-mem-table", &num));
ASSERT_EQ(num, "1"); ASSERT_EQ(num, "1");
perf_context.Reset(); get_perf_context()->Reset();
Get(1, "k1"); Get(1, "k1");
ASSERT_EQ(1, static_cast<int>(perf_context.get_from_memtable_count)); ASSERT_EQ(1, static_cast<int>(get_perf_context()->get_from_memtable_count));
ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "k2", big_value)); ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "k2", big_value));
ASSERT_TRUE(dbfull()->GetProperty(handles_[1], ASSERT_TRUE(dbfull()->GetProperty(handles_[1],
@ -548,12 +548,12 @@ TEST_F(DBPropertiesTest, NumImmutableMemTable) {
handles_[1], "rocksdb.num-entries-imm-mem-tables", &num)); handles_[1], "rocksdb.num-entries-imm-mem-tables", &num));
ASSERT_EQ(num, "1"); ASSERT_EQ(num, "1");
perf_context.Reset(); get_perf_context()->Reset();
Get(1, "k1"); Get(1, "k1");
ASSERT_EQ(2, static_cast<int>(perf_context.get_from_memtable_count)); ASSERT_EQ(2, static_cast<int>(get_perf_context()->get_from_memtable_count));
perf_context.Reset(); get_perf_context()->Reset();
Get(1, "k2"); Get(1, "k2");
ASSERT_EQ(1, static_cast<int>(perf_context.get_from_memtable_count)); ASSERT_EQ(1, static_cast<int>(get_perf_context()->get_from_memtable_count));
ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "k3", big_value)); ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "k3", big_value));
ASSERT_TRUE(dbfull()->GetProperty( ASSERT_TRUE(dbfull()->GetProperty(
@ -567,15 +567,15 @@ TEST_F(DBPropertiesTest, NumImmutableMemTable) {
ASSERT_TRUE(dbfull()->GetProperty( ASSERT_TRUE(dbfull()->GetProperty(
handles_[1], "rocksdb.num-entries-imm-mem-tables", &num)); handles_[1], "rocksdb.num-entries-imm-mem-tables", &num));
ASSERT_EQ(num, "2"); ASSERT_EQ(num, "2");
perf_context.Reset(); get_perf_context()->Reset();
Get(1, "k2"); Get(1, "k2");
ASSERT_EQ(2, static_cast<int>(perf_context.get_from_memtable_count)); ASSERT_EQ(2, static_cast<int>(get_perf_context()->get_from_memtable_count));
perf_context.Reset(); get_perf_context()->Reset();
Get(1, "k3"); Get(1, "k3");
ASSERT_EQ(1, static_cast<int>(perf_context.get_from_memtable_count)); ASSERT_EQ(1, static_cast<int>(get_perf_context()->get_from_memtable_count));
perf_context.Reset(); get_perf_context()->Reset();
Get(1, "k1"); Get(1, "k1");
ASSERT_EQ(3, static_cast<int>(perf_context.get_from_memtable_count)); ASSERT_EQ(3, static_cast<int>(get_perf_context()->get_from_memtable_count));
ASSERT_OK(Flush(1)); ASSERT_OK(Flush(1));
ASSERT_TRUE(dbfull()->GetProperty(handles_[1], ASSERT_TRUE(dbfull()->GetProperty(handles_[1],
@ -670,7 +670,7 @@ TEST_F(DBPropertiesTest, DISABLED_GetProperty) {
ASSERT_EQ(num, "0"); ASSERT_EQ(num, "0");
ASSERT_TRUE(dbfull()->GetProperty("rocksdb.estimate-num-keys", &num)); ASSERT_TRUE(dbfull()->GetProperty("rocksdb.estimate-num-keys", &num));
ASSERT_EQ(num, "1"); ASSERT_EQ(num, "1");
perf_context.Reset(); get_perf_context()->Reset();
ASSERT_OK(dbfull()->Put(writeOpt, "k2", big_value)); ASSERT_OK(dbfull()->Put(writeOpt, "k2", big_value));
ASSERT_TRUE(dbfull()->GetProperty("rocksdb.num-immutable-mem-table", &num)); ASSERT_TRUE(dbfull()->GetProperty("rocksdb.num-immutable-mem-table", &num));
@ -1228,7 +1228,7 @@ TEST_F(DBPropertiesTest, TablePropertiesNeedCompactTest) {
{ {
SetPerfLevel(kEnableCount); SetPerfLevel(kEnableCount);
perf_context.Reset(); get_perf_context()->Reset();
int c = 0; int c = 0;
std::unique_ptr<Iterator> iter(db_->NewIterator(ReadOptions())); std::unique_ptr<Iterator> iter(db_->NewIterator(ReadOptions()));
iter->Seek(Key(kMaxKey - 100)); iter->Seek(Key(kMaxKey - 100));
@ -1236,8 +1236,8 @@ TEST_F(DBPropertiesTest, TablePropertiesNeedCompactTest) {
iter->Next(); iter->Next();
} }
ASSERT_EQ(c, 0); ASSERT_EQ(c, 0);
ASSERT_LT(perf_context.internal_delete_skipped_count, 30u); ASSERT_LT(get_perf_context()->internal_delete_skipped_count, 30u);
ASSERT_LT(perf_context.internal_key_skipped_count, 30u); ASSERT_LT(get_perf_context()->internal_key_skipped_count, 30u);
SetPerfLevel(kDisable); SetPerfLevel(kDisable);
} }
} }
@ -1284,16 +1284,16 @@ TEST_F(DBPropertiesTest, NeedCompactHintPersistentTest) {
ASSERT_EQ(NumTableFilesAtLevel(0), 0); ASSERT_EQ(NumTableFilesAtLevel(0), 0);
{ {
SetPerfLevel(kEnableCount); SetPerfLevel(kEnableCount);
perf_context.Reset(); get_perf_context()->Reset();
int c = 0; int c = 0;
std::unique_ptr<Iterator> iter(db_->NewIterator(ReadOptions())); std::unique_ptr<Iterator> iter(db_->NewIterator(ReadOptions()));
for (iter->Seek(Key(0)); iter->Valid(); iter->Next()) { for (iter->Seek(Key(0)); iter->Valid(); iter->Next()) {
c++; c++;
} }
ASSERT_EQ(c, 2); ASSERT_EQ(c, 2);
ASSERT_EQ(perf_context.internal_delete_skipped_count, 0); ASSERT_EQ(get_perf_context()->internal_delete_skipped_count, 0);
// We iterate every key twice. Is it a bug? // We iterate every key twice. Is it a bug?
ASSERT_LE(perf_context.internal_key_skipped_count, 2); ASSERT_LE(get_perf_context()->internal_key_skipped_count, 2);
SetPerfLevel(kDisable); SetPerfLevel(kDisable);
} }
} }

142
db/perf_context_test.cc
View File

@ -89,13 +89,13 @@ TEST_F(PerfContextTest, SeekIntoDeletion) {
std::string key = "k" + ToString(i); std::string key = "k" + ToString(i);
std::string value; std::string value;
perf_context.Reset(); get_perf_context()->Reset();
StopWatchNano timer(Env::Default()); StopWatchNano timer(Env::Default());
timer.Start(); timer.Start();
auto status = db->Get(read_options, key, &value); auto status = db->Get(read_options, key, &value);
auto elapsed_nanos = timer.ElapsedNanos(); auto elapsed_nanos = timer.ElapsedNanos();
ASSERT_TRUE(status.IsNotFound()); ASSERT_TRUE(status.IsNotFound());
hist_get.Add(perf_context.user_key_comparison_count); hist_get.Add(get_perf_context()->user_key_comparison_count);
hist_get_time.Add(elapsed_nanos); hist_get_time.Add(elapsed_nanos);
} }
@ -108,19 +108,19 @@ TEST_F(PerfContextTest, SeekIntoDeletion) {
HistogramImpl hist_seek_to_first; HistogramImpl hist_seek_to_first;
std::unique_ptr<Iterator> iter(db->NewIterator(read_options)); std::unique_ptr<Iterator> iter(db->NewIterator(read_options));
perf_context.Reset(); get_perf_context()->Reset();
StopWatchNano timer(Env::Default(), true); StopWatchNano timer(Env::Default(), true);
iter->SeekToFirst(); iter->SeekToFirst();
hist_seek_to_first.Add(perf_context.user_key_comparison_count); hist_seek_to_first.Add(get_perf_context()->user_key_comparison_count);
auto elapsed_nanos = timer.ElapsedNanos(); auto elapsed_nanos = timer.ElapsedNanos();
if (FLAGS_verbose) { if (FLAGS_verbose) {
std::cout << "SeekToFirst uesr key comparison: \n" std::cout << "SeekToFirst uesr key comparison: \n"
<< hist_seek_to_first.ToString() << hist_seek_to_first.ToString()
<< "ikey skipped: " << perf_context.internal_key_skipped_count << "ikey skipped: " << get_perf_context()->internal_key_skipped_count
<< "\n" << "\n"
<< "idelete skipped: " << "idelete skipped: "
<< perf_context.internal_delete_skipped_count << "\n" << get_perf_context()->internal_delete_skipped_count << "\n"
<< "elapsed: " << elapsed_nanos << "\n"; << "elapsed: " << elapsed_nanos << "\n";
} }
} }
@ -130,26 +130,26 @@ TEST_F(PerfContextTest, SeekIntoDeletion) {
std::unique_ptr<Iterator> iter(db->NewIterator(read_options)); std::unique_ptr<Iterator> iter(db->NewIterator(read_options));
std::string key = "k" + ToString(i); std::string key = "k" + ToString(i);
perf_context.Reset(); get_perf_context()->Reset();
StopWatchNano timer(Env::Default(), true); StopWatchNano timer(Env::Default(), true);
iter->Seek(key); iter->Seek(key);
auto elapsed_nanos = timer.ElapsedNanos(); auto elapsed_nanos = timer.ElapsedNanos();
hist_seek.Add(perf_context.user_key_comparison_count); hist_seek.Add(get_perf_context()->user_key_comparison_count);
if (FLAGS_verbose) { if (FLAGS_verbose) {
std::cout << "seek cmp: " << perf_context.user_key_comparison_count std::cout << "seek cmp: " << get_perf_context()->user_key_comparison_count
<< " ikey skipped " << perf_context.internal_key_skipped_count << " ikey skipped " << get_perf_context()->internal_key_skipped_count
<< " idelete skipped " << " idelete skipped "
<< perf_context.internal_delete_skipped_count << get_perf_context()->internal_delete_skipped_count
<< " elapsed: " << elapsed_nanos << "ns\n"; << " elapsed: " << elapsed_nanos << "ns\n";
} }
perf_context.Reset(); get_perf_context()->Reset();
ASSERT_TRUE(iter->Valid()); ASSERT_TRUE(iter->Valid());
StopWatchNano timer2(Env::Default(), true); StopWatchNano timer2(Env::Default(), true);
iter->Next(); iter->Next();
auto elapsed_nanos2 = timer2.ElapsedNanos(); auto elapsed_nanos2 = timer2.ElapsedNanos();
if (FLAGS_verbose) { if (FLAGS_verbose) {
std::cout << "next cmp: " << perf_context.user_key_comparison_count std::cout << "next cmp: " << get_perf_context()->user_key_comparison_count
<< "elapsed: " << elapsed_nanos2 << "ns\n"; << "elapsed: " << elapsed_nanos2 << "ns\n";
} }
} }
@ -265,18 +265,18 @@ void ProfileQueries(bool enabled_time = false) {
std::vector<std::string> values; std::vector<std::string> values;
perf_context.Reset(); get_perf_context()->Reset();
db->Put(write_options, key, value); db->Put(write_options, key, value);
if (++num_mutex_waited > 3) { if (++num_mutex_waited > 3) {
#ifndef NDEBUG #ifndef NDEBUG
ThreadStatusUtil::TEST_SetStateDelay(ThreadStatus::STATE_MUTEX_WAIT, 0U); ThreadStatusUtil::TEST_SetStateDelay(ThreadStatus::STATE_MUTEX_WAIT, 0U);
#endif #endif
} }
hist_write_pre_post.Add(perf_context.write_pre_and_post_process_time); hist_write_pre_post.Add(get_perf_context()->write_pre_and_post_process_time);
hist_write_wal_time.Add(perf_context.write_wal_time); hist_write_wal_time.Add(get_perf_context()->write_wal_time);
hist_write_memtable_time.Add(perf_context.write_memtable_time); hist_write_memtable_time.Add(get_perf_context()->write_memtable_time);
hist_put.Add(perf_context.user_key_comparison_count); hist_put.Add(get_perf_context()->user_key_comparison_count);
total_db_mutex_nanos += perf_context.db_mutex_lock_nanos; total_db_mutex_nanos += get_perf_context()->db_mutex_lock_nanos;
} }
#ifndef NDEBUG #ifndef NDEBUG
ThreadStatusUtil::TEST_SetStateDelay(ThreadStatus::STATE_MUTEX_WAIT, 0U); ThreadStatusUtil::TEST_SetStateDelay(ThreadStatus::STATE_MUTEX_WAIT, 0U);
@ -293,24 +293,24 @@ void ProfileQueries(bool enabled_time = false) {
std::vector<Slice> multiget_keys = {Slice(key)}; std::vector<Slice> multiget_keys = {Slice(key)};
std::vector<std::string> values; std::vector<std::string> values;
perf_context.Reset(); get_perf_context()->Reset();
ASSERT_OK(db->Get(read_options, key, &value)); ASSERT_OK(db->Get(read_options, key, &value));
ASSERT_EQ(expected_value, value); ASSERT_EQ(expected_value, value);
hist_get_snapshot.Add(perf_context.get_snapshot_time); hist_get_snapshot.Add(get_perf_context()->get_snapshot_time);
hist_get_memtable.Add(perf_context.get_from_memtable_time); hist_get_memtable.Add(get_perf_context()->get_from_memtable_time);
hist_get_files.Add(perf_context.get_from_output_files_time); hist_get_files.Add(get_perf_context()->get_from_output_files_time);
hist_num_memtable_checked.Add(perf_context.get_from_memtable_count); hist_num_memtable_checked.Add(get_perf_context()->get_from_memtable_count);
hist_get_post_process.Add(perf_context.get_post_process_time); hist_get_post_process.Add(get_perf_context()->get_post_process_time);
hist_get.Add(perf_context.user_key_comparison_count); hist_get.Add(get_perf_context()->user_key_comparison_count);
perf_context.Reset(); get_perf_context()->Reset();
db->MultiGet(read_options, multiget_keys, &values); db->MultiGet(read_options, multiget_keys, &values);
hist_mget_snapshot.Add(perf_context.get_snapshot_time); hist_mget_snapshot.Add(get_perf_context()->get_snapshot_time);
hist_mget_memtable.Add(perf_context.get_from_memtable_time); hist_mget_memtable.Add(get_perf_context()->get_from_memtable_time);
hist_mget_files.Add(perf_context.get_from_output_files_time); hist_mget_files.Add(get_perf_context()->get_from_output_files_time);
hist_mget_num_memtable_checked.Add(perf_context.get_from_memtable_count); hist_mget_num_memtable_checked.Add(get_perf_context()->get_from_memtable_count);
hist_mget_post_process.Add(perf_context.get_post_process_time); hist_mget_post_process.Add(get_perf_context()->get_post_process_time);
hist_mget.Add(perf_context.user_key_comparison_count); hist_mget.Add(get_perf_context()->user_key_comparison_count);
} }
if (FLAGS_verbose) { if (FLAGS_verbose) {
@ -394,24 +394,24 @@ void ProfileQueries(bool enabled_time = false) {
std::vector<Slice> multiget_keys = {Slice(key)}; std::vector<Slice> multiget_keys = {Slice(key)};
std::vector<std::string> values; std::vector<std::string> values;
perf_context.Reset(); get_perf_context()->Reset();
ASSERT_OK(db->Get(read_options, key, &value)); ASSERT_OK(db->Get(read_options, key, &value));
ASSERT_EQ(expected_value, value); ASSERT_EQ(expected_value, value);
hist_get_snapshot.Add(perf_context.get_snapshot_time); hist_get_snapshot.Add(get_perf_context()->get_snapshot_time);
hist_get_memtable.Add(perf_context.get_from_memtable_time); hist_get_memtable.Add(get_perf_context()->get_from_memtable_time);
hist_get_files.Add(perf_context.get_from_output_files_time); hist_get_files.Add(get_perf_context()->get_from_output_files_time);
hist_num_memtable_checked.Add(perf_context.get_from_memtable_count); hist_num_memtable_checked.Add(get_perf_context()->get_from_memtable_count);
hist_get_post_process.Add(perf_context.get_post_process_time); hist_get_post_process.Add(get_perf_context()->get_post_process_time);
hist_get.Add(perf_context.user_key_comparison_count); hist_get.Add(get_perf_context()->user_key_comparison_count);
perf_context.Reset(); get_perf_context()->Reset();
db->MultiGet(read_options, multiget_keys, &values); db->MultiGet(read_options, multiget_keys, &values);
hist_mget_snapshot.Add(perf_context.get_snapshot_time); hist_mget_snapshot.Add(get_perf_context()->get_snapshot_time);
hist_mget_memtable.Add(perf_context.get_from_memtable_time); hist_mget_memtable.Add(get_perf_context()->get_from_memtable_time);
hist_mget_files.Add(perf_context.get_from_output_files_time); hist_mget_files.Add(get_perf_context()->get_from_output_files_time);
hist_mget_num_memtable_checked.Add(perf_context.get_from_memtable_count); hist_mget_num_memtable_checked.Add(get_perf_context()->get_from_memtable_count);
hist_mget_post_process.Add(perf_context.get_post_process_time); hist_mget_post_process.Add(get_perf_context()->get_post_process_time);
hist_mget.Add(perf_context.user_key_comparison_count); hist_mget.Add(get_perf_context()->user_key_comparison_count);
} }
if (FLAGS_verbose) { if (FLAGS_verbose) {
@ -514,13 +514,13 @@ TEST_F(PerfContextTest, SeekKeyComparison) {
std::string key = "k" + ToString(i); std::string key = "k" + ToString(i);
std::string value = "v" + ToString(i); std::string value = "v" + ToString(i);
perf_context.Reset(); get_perf_context()->Reset();
timer.Start(); timer.Start();
db->Put(write_options, key, value); db->Put(write_options, key, value);
auto put_time = timer.ElapsedNanos(); auto put_time = timer.ElapsedNanos();
hist_put_time.Add(put_time); hist_put_time.Add(put_time);
hist_wal_time.Add(perf_context.write_wal_time); hist_wal_time.Add(get_perf_context()->write_wal_time);
hist_time_diff.Add(put_time - perf_context.write_wal_time); hist_time_diff.Add(put_time - get_perf_context()->write_wal_time);
} }
if (FLAGS_verbose) { if (FLAGS_verbose) {
@ -537,18 +537,18 @@ TEST_F(PerfContextTest, SeekKeyComparison) {
std::string value = "v" + ToString(i); std::string value = "v" + ToString(i);
std::unique_ptr<Iterator> iter(db->NewIterator(read_options)); std::unique_ptr<Iterator> iter(db->NewIterator(read_options));
perf_context.Reset(); get_perf_context()->Reset();
iter->Seek(key); iter->Seek(key);
ASSERT_TRUE(iter->Valid()); ASSERT_TRUE(iter->Valid());
ASSERT_EQ(iter->value().ToString(), value); ASSERT_EQ(iter->value().ToString(), value);
hist_seek.Add(perf_context.user_key_comparison_count); hist_seek.Add(get_perf_context()->user_key_comparison_count);
} }
std::unique_ptr<Iterator> iter(db->NewIterator(read_options)); std::unique_ptr<Iterator> iter(db->NewIterator(read_options));
for (iter->SeekToFirst(); iter->Valid();) { for (iter->SeekToFirst(); iter->Valid();) {
perf_context.Reset(); get_perf_context()->Reset();
iter->Next(); iter->Next();
hist_next.Add(perf_context.user_key_comparison_count); hist_next.Add(get_perf_context()->user_key_comparison_count);
} }
if (FLAGS_verbose) { if (FLAGS_verbose) {
@ -566,16 +566,16 @@ TEST_F(PerfContextTest, DBMutexLockCounter) {
mutex.Lock(); mutex.Lock();
rocksdb::port::Thread child_thread([&] { rocksdb::port::Thread child_thread([&] {
SetPerfLevel(perf_level); SetPerfLevel(perf_level);
perf_context.Reset(); get_perf_context()->Reset();
ASSERT_EQ(perf_context.db_mutex_lock_nanos, 0); ASSERT_EQ(get_perf_context()->db_mutex_lock_nanos, 0);
mutex.Lock(); mutex.Lock();
mutex.Unlock(); mutex.Unlock();
if (perf_level == PerfLevel::kEnableTimeExceptForMutex || if (perf_level == PerfLevel::kEnableTimeExceptForMutex ||
stats_code[c] != DB_MUTEX_WAIT_MICROS) { stats_code[c] != DB_MUTEX_WAIT_MICROS) {
ASSERT_EQ(perf_context.db_mutex_lock_nanos, 0); ASSERT_EQ(get_perf_context()->db_mutex_lock_nanos, 0);
} else { } else {
// increment the counter only when it's a DB Mutex // increment the counter only when it's a DB Mutex
ASSERT_GT(perf_context.db_mutex_lock_nanos, 0); ASSERT_GT(get_perf_context()->db_mutex_lock_nanos, 0);
} }
}); });
Env::Default()->SleepForMicroseconds(100); Env::Default()->SleepForMicroseconds(100);
@ -591,28 +591,28 @@ TEST_F(PerfContextTest, FalseDBMutexWait) {
for (int c = 0; c < 2; ++c) { for (int c = 0; c < 2; ++c) {
InstrumentedMutex mutex(nullptr, Env::Default(), stats_code[c]); InstrumentedMutex mutex(nullptr, Env::Default(), stats_code[c]);
InstrumentedCondVar lock(&mutex); InstrumentedCondVar lock(&mutex);
perf_context.Reset(); get_perf_context()->Reset();
mutex.Lock(); mutex.Lock();
lock.TimedWait(100); lock.TimedWait(100);
mutex.Unlock(); mutex.Unlock();
if (stats_code[c] == static_cast<int>(DB_MUTEX_WAIT_MICROS)) { if (stats_code[c] == static_cast<int>(DB_MUTEX_WAIT_MICROS)) {
// increment the counter only when it's a DB Mutex // increment the counter only when it's a DB Mutex
ASSERT_GT(perf_context.db_condition_wait_nanos, 0); ASSERT_GT(get_perf_context()->db_condition_wait_nanos, 0);
} else { } else {
ASSERT_EQ(perf_context.db_condition_wait_nanos, 0); ASSERT_EQ(get_perf_context()->db_condition_wait_nanos, 0);
} }
} }
} }
TEST_F(PerfContextTest, ToString) { TEST_F(PerfContextTest, ToString) {
perf_context.Reset(); get_perf_context()->Reset();
perf_context.block_read_count = 12345; get_perf_context()->block_read_count = 12345;
std::string zero_included = perf_context.ToString(); std::string zero_included = get_perf_context()->ToString();
ASSERT_NE(std::string::npos, zero_included.find("= 0")); ASSERT_NE(std::string::npos, zero_included.find("= 0"));
ASSERT_NE(std::string::npos, zero_included.find("= 12345")); ASSERT_NE(std::string::npos, zero_included.find("= 12345"));
std::string zero_excluded = perf_context.ToString(true); std::string zero_excluded = get_perf_context()->ToString(true);
ASSERT_EQ(std::string::npos, zero_excluded.find("= 0")); ASSERT_EQ(std::string::npos, zero_excluded.find("= 0"));
ASSERT_NE(std::string::npos, zero_excluded.find("= 12345")); ASSERT_NE(std::string::npos, zero_excluded.find("= 12345"));
} }
@ -633,36 +633,36 @@ TEST_F(PerfContextTest, MergeOperatorTime) {
ASSERT_OK(db->Merge(WriteOptions(), "k1", "val4")); ASSERT_OK(db->Merge(WriteOptions(), "k1", "val4"));
SetPerfLevel(kEnableTime); SetPerfLevel(kEnableTime);
perf_context.Reset(); get_perf_context()->Reset();
ASSERT_OK(db->Get(ReadOptions(), "k1", &val)); ASSERT_OK(db->Get(ReadOptions(), "k1", &val));
#ifdef OS_SOLARIS #ifdef OS_SOLARIS
for (int i = 0; i < 100; i++) { for (int i = 0; i < 100; i++) {
ASSERT_OK(db->Get(ReadOptions(), "k1", &val)); ASSERT_OK(db->Get(ReadOptions(), "k1", &val));
} }
#endif #endif
EXPECT_GT(perf_context.merge_operator_time_nanos, 0); EXPECT_GT(get_perf_context()->merge_operator_time_nanos, 0);
ASSERT_OK(db->Flush(FlushOptions())); ASSERT_OK(db->Flush(FlushOptions()));
perf_context.Reset(); get_perf_context()->Reset();
ASSERT_OK(db->Get(ReadOptions(), "k1", &val)); ASSERT_OK(db->Get(ReadOptions(), "k1", &val));
#ifdef OS_SOLARIS #ifdef OS_SOLARIS
for (int i = 0; i < 100; i++) { for (int i = 0; i < 100; i++) {
ASSERT_OK(db->Get(ReadOptions(), "k1", &val)); ASSERT_OK(db->Get(ReadOptions(), "k1", &val));
} }
#endif #endif
EXPECT_GT(perf_context.merge_operator_time_nanos, 0); EXPECT_GT(get_perf_context()->merge_operator_time_nanos, 0);
ASSERT_OK(db->CompactRange(CompactRangeOptions(), nullptr, nullptr)); ASSERT_OK(db->CompactRange(CompactRangeOptions(), nullptr, nullptr));
perf_context.Reset(); get_perf_context()->Reset();
ASSERT_OK(db->Get(ReadOptions(), "k1", &val)); ASSERT_OK(db->Get(ReadOptions(), "k1", &val));
#ifdef OS_SOLARIS #ifdef OS_SOLARIS
for (int i = 0; i < 100; i++) { for (int i = 0; i < 100; i++) {
ASSERT_OK(db->Get(ReadOptions(), "k1", &val)); ASSERT_OK(db->Get(ReadOptions(), "k1", &val));
} }
#endif #endif
EXPECT_GT(perf_context.merge_operator_time_nanos, 0); EXPECT_GT(get_perf_context()->merge_operator_time_nanos, 0);
delete db; delete db;
} }

12
db/prefix_test.cc
View File

@ -605,11 +605,11 @@ TEST_F(PrefixTest, DynamicPrefixIterator) {
Slice key = TestKeyToSlice(s, test_key); Slice key = TestKeyToSlice(s, test_key);
std::string value(FLAGS_value_size, 0); std::string value(FLAGS_value_size, 0);
perf_context.Reset(); get_perf_context()->Reset();
StopWatchNano timer(Env::Default(), true); StopWatchNano timer(Env::Default(), true);
ASSERT_OK(db->Put(write_options, key, value)); ASSERT_OK(db->Put(write_options, key, value));
hist_put_time.Add(timer.ElapsedNanos()); hist_put_time.Add(timer.ElapsedNanos());
hist_put_comparison.Add(perf_context.user_key_comparison_count); hist_put_comparison.Add(get_perf_context()->user_key_comparison_count);
} }
} }
@ -628,7 +628,7 @@ TEST_F(PrefixTest, DynamicPrefixIterator) {
Slice key = TestKeyToSlice(s, test_key); Slice key = TestKeyToSlice(s, test_key);
std::string value = "v" + ToString(0); std::string value = "v" + ToString(0);
perf_context.Reset(); get_perf_context()->Reset();
StopWatchNano timer(Env::Default(), true); StopWatchNano timer(Env::Default(), true);
auto key_prefix = options.prefix_extractor->Transform(key); auto key_prefix = options.prefix_extractor->Transform(key);
uint64_t total_keys = 0; uint64_t total_keys = 0;
@ -642,7 +642,7 @@ TEST_F(PrefixTest, DynamicPrefixIterator) {
total_keys++; total_keys++;
} }
hist_seek_time.Add(timer.ElapsedNanos()); hist_seek_time.Add(timer.ElapsedNanos());
hist_seek_comparison.Add(perf_context.user_key_comparison_count); hist_seek_comparison.Add(get_perf_context()->user_key_comparison_count);
ASSERT_EQ(total_keys, FLAGS_items_per_prefix - FLAGS_items_per_prefix/2); ASSERT_EQ(total_keys, FLAGS_items_per_prefix - FLAGS_items_per_prefix/2);
} }
@ -662,11 +662,11 @@ TEST_F(PrefixTest, DynamicPrefixIterator) {
std::string s; std::string s;
Slice key = TestKeyToSlice(s, test_key); Slice key = TestKeyToSlice(s, test_key);
perf_context.Reset(); get_perf_context()->Reset();
StopWatchNano timer(Env::Default(), true); StopWatchNano timer(Env::Default(), true);
iter->Seek(key); iter->Seek(key);
hist_no_seek_time.Add(timer.ElapsedNanos()); hist_no_seek_time.Add(timer.ElapsedNanos());
hist_no_seek_comparison.Add(perf_context.user_key_comparison_count); hist_no_seek_comparison.Add(get_perf_context()->user_key_comparison_count);
ASSERT_TRUE(!iter->Valid()); ASSERT_TRUE(!iter->Valid());
} }

10
include/rocksdb/iostats_context.h
View File

@ -46,13 +46,7 @@ struct IOStatsContext {
uint64_t logger_nanos; uint64_t logger_nanos;
}; };
#ifdef ROCKSDB_SUPPORT_THREAD_LOCAL // Get Thread-local IOStatsContext object pointer
#if defined(_MSC_VER) && !defined(__thread) IOStatsContext* get_iostats_context();
// Thread local storage on Linux
// There is thread_local in C++11
#define __thread __declspec(thread)
#endif
extern __thread IOStatsContext iostats_context;
#endif
} // namespace rocksdb } // namespace rocksdb

19
include/rocksdb/perf_context.h
View File

@ -150,21 +150,10 @@ struct PerfContext {
uint64_t env_new_logger_nanos; uint64_t env_new_logger_nanos;
}; };
#if defined(NPERF_CONTEXT) || !defined(ROCKSDB_SUPPORT_THREAD_LOCAL) // Get Thread-local PerfContext object pointer
extern PerfContext perf_context; // if defined(NPERF_CONTEXT), then the pointer is not thread-local
#else PerfContext* get_perf_context();
#if defined(OS_SOLARIS)
PerfContext *getPerfContext();
#define perf_context (*getPerfContext())
#else
#if defined(_MSC_VER) && !defined(__thread)
// Thread local storage on Linux
// There is thread_local in C++11
#define __thread __declspec(thread)
#endif
extern __thread PerfContext perf_context;
#endif
#endif
} }

1
include/rocksdb/perf_level.h
View File

@ -12,7 +12,6 @@
namespace rocksdb { namespace rocksdb {
// How much perf stats to collect. Affects perf_context and iostats_context. // How much perf stats to collect. Affects perf_context and iostats_context.
enum PerfLevel : unsigned char { enum PerfLevel : unsigned char {
kUninitialized = 0, // unknown setting kUninitialized = 0, // unknown setting
kDisable = 1, // disable perf stats kDisable = 1, // disable perf stats

15
monitoring/iostats_context.cc
View File

@ -6,12 +6,21 @@
#include <sstream> #include <sstream>
#include "monitoring/iostats_context_imp.h" #include "monitoring/iostats_context_imp.h"
#include "rocksdb/env.h" #include "rocksdb/env.h"
#include "util/thread_local.h"
namespace rocksdb { namespace rocksdb {
#ifdef ROCKSDB_SUPPORT_THREAD_LOCAL ThreadLocalPtr iostats_context([](void* ptr) {
__thread IOStatsContext iostats_context; auto* p = static_cast<IOStatsContext*>(ptr);
#endif delete p;
});
IOStatsContext* get_iostats_context() {
if (iostats_context.Get() == nullptr) {
iostats_context.Reset(static_cast<void*>(new IOStatsContext()));
}
return static_cast<IOStatsContext*>(iostats_context.Get());
}
void IOStatsContext::Reset() { void IOStatsContext::Reset() {
thread_pool_id = Env::Priority::TOTAL; thread_pool_id = Env::Priority::TOTAL;

14
monitoring/iostats_context_imp.h
View File

@ -13,7 +13,7 @@
// increment a specific counter by the specified value // increment a specific counter by the specified value
#define IOSTATS_ADD(metric, value) \ #define IOSTATS_ADD(metric, value) \
(iostats_context.metric += value) (get_iostats_context()->metric += value)
// Increase metric value only when it is positive // Increase metric value only when it is positive
#define IOSTATS_ADD_IF_POSITIVE(metric, value) \ #define IOSTATS_ADD_IF_POSITIVE(metric, value) \
@ -21,24 +21,24 @@
// reset a specific counter to zero // reset a specific counter to zero
#define IOSTATS_RESET(metric) \ #define IOSTATS_RESET(metric) \
(iostats_context.metric = 0) (get_iostats_context()->metric = 0)
// reset all counters to zero // reset all counters to zero
#define IOSTATS_RESET_ALL() \ #define IOSTATS_RESET_ALL() \
(iostats_context.Reset()) (get_iostats_context()->Reset())
#define IOSTATS_SET_THREAD_POOL_ID(value) \ #define IOSTATS_SET_THREAD_POOL_ID(value) \
(iostats_context.thread_pool_id = value) (get_iostats_context()->thread_pool_id = value)
#define IOSTATS_THREAD_POOL_ID() \ #define IOSTATS_THREAD_POOL_ID() \
(iostats_context.thread_pool_id) (get_iostats_context()->thread_pool_id)
#define IOSTATS(metric) \ #define IOSTATS(metric) \
(iostats_context.metric) (get_iostats_context()->metric)
// Declare and set start time of the timer // Declare and set start time of the timer
#define IOSTATS_TIMER_GUARD(metric) \ #define IOSTATS_TIMER_GUARD(metric) \
PerfStepTimer iostats_step_timer_ ## metric(&(iostats_context.metric)); \ PerfStepTimer iostats_step_timer_##metric(&(get_iostats_context()->metric)); \
iostats_step_timer_##metric.Start(); iostats_step_timer_##metric.Start();
#else // ROCKSDB_SUPPORT_THREAD_LOCAL #else // ROCKSDB_SUPPORT_THREAD_LOCAL

8
monitoring/iostats_context_test.cc
View File

@ -9,14 +9,14 @@
namespace rocksdb { namespace rocksdb {
TEST(IOStatsContextTest, ToString) { TEST(IOStatsContextTest, ToString) {
iostats_context.Reset(); get_iostats_context()->Reset();
iostats_context.bytes_read = 12345; get_iostats_context()->bytes_read = 12345;
std::string zero_included = iostats_context.ToString(); std::string zero_included = get_iostats_context()->ToString();
ASSERT_NE(std::string::npos, zero_included.find("= 0")); ASSERT_NE(std::string::npos, zero_included.find("= 0"));
ASSERT_NE(std::string::npos, zero_included.find("= 12345")); ASSERT_NE(std::string::npos, zero_included.find("= 12345"));
std::string zero_excluded = iostats_context.ToString(true); std::string zero_excluded = get_iostats_context()->ToString(true);
ASSERT_EQ(std::string::npos, zero_excluded.find("= 0")); ASSERT_EQ(std::string::npos, zero_excluded.find("= 0"));
ASSERT_NE(std::string::npos, zero_excluded.find("= 12345")); ASSERT_NE(std::string::npos, zero_excluded.find("= 12345"));
} }

33
monitoring/perf_context.cc
View File

@ -8,21 +8,32 @@
#include <sstream> #include <sstream>
#include "monitoring/perf_context_imp.h" #include "monitoring/perf_context_imp.h"
#include "util/thread_local.h"
namespace rocksdb { namespace rocksdb {
#if defined(NPERF_CONTEXT) || !defined(ROCKSDB_SUPPORT_THREAD_LOCAL) #ifdef NPERF_CONTEXT
PerfContext perf_context; PerfContext perf_context;
#else #else
#if defined(OS_SOLARIS) ThreadLocalPtr perf_context([](void* ptr) {
__thread PerfContext perf_context_; auto* p = static_cast<PerfContext*>(ptr);
#else delete p;
__thread PerfContext perf_context; });
#endif
#endif #endif
PerfContext* get_perf_context() {
#ifdef NPERF_CONTEXT
return &perf_context;
#else
if (perf_context.Get() == nullptr) {
perf_context.Reset(static_cast<void*>(new PerfContext()));
}
return static_cast<PerfContext*>(perf_context.Get());
#endif
}
void PerfContext::Reset() { void PerfContext::Reset() {
#if !defined(NPERF_CONTEXT) && defined(ROCKSDB_SUPPORT_THREAD_LOCAL) #ifndef NPERF_CONTEXT
user_key_comparison_count = 0; user_key_comparison_count = 0;
block_cache_hit_count = 0; block_cache_hit_count = 0;
block_read_count = 0; block_read_count = 0;
@ -96,7 +107,7 @@ void PerfContext::Reset() {
} }
std::string PerfContext::ToString(bool exclude_zero_counters) const { std::string PerfContext::ToString(bool exclude_zero_counters) const {
#if defined(NPERF_CONTEXT) || !defined(ROCKSDB_SUPPORT_THREAD_LOCAL) #ifdef NPERF_CONTEXT
return ""; return "";
#else #else
std::ostringstream ss; std::ostringstream ss;
@ -166,10 +177,4 @@ std::string PerfContext::ToString(bool exclude_zero_counters) const {
#endif #endif
} }
#if defined(OS_SOLARIS)
PerfContext *getPerfContext() {
return &perf_context_;
}
#endif
} }

18
monitoring/perf_context_imp.h
View File

@ -12,7 +12,7 @@
namespace rocksdb { namespace rocksdb {
#if defined(NPERF_CONTEXT) || !defined(ROCKSDB_SUPPORT_THREAD_LOCAL) #if defined(NPERF_CONTEXT)
#define PERF_TIMER_GUARD(metric) #define PERF_TIMER_GUARD(metric)
#define PERF_CONDITIONAL_TIMER_FOR_MUTEX_GUARD(metric, condition) #define PERF_CONDITIONAL_TIMER_FOR_MUTEX_GUARD(metric, condition)
@ -24,31 +24,27 @@ namespace rocksdb {
#else #else
// Stop the timer and update the metric // Stop the timer and update the metric
#define PERF_TIMER_STOP(metric) \ #define PERF_TIMER_STOP(metric) perf_step_timer_##metric.Stop();
perf_step_timer_ ## metric.Stop();
#define PERF_TIMER_START(metric) \ #define PERF_TIMER_START(metric) perf_step_timer_##metric.Start();
perf_step_timer_ ## metric.Start();
// Declare and set start time of the timer // Declare and set start time of the timer
#define PERF_TIMER_GUARD(metric) \ #define PERF_TIMER_GUARD(metric) \
PerfStepTimer perf_step_timer_ ## metric(&(perf_context.metric)); \ PerfStepTimer perf_step_timer_##metric(&(get_perf_context()->metric)); \
perf_step_timer_##metric.Start(); perf_step_timer_##metric.Start();
#define PERF_CONDITIONAL_TIMER_FOR_MUTEX_GUARD(metric, condition) \ #define PERF_CONDITIONAL_TIMER_FOR_MUTEX_GUARD(metric, condition) \
PerfStepTimer perf_step_timer_##metric(&(perf_context.metric), true); \ PerfStepTimer perf_step_timer_##metric(&(get_perf_context()->metric), true); \
if ((condition)) { \ if ((condition)) { \
perf_step_timer_##metric.Start(); \ perf_step_timer_##metric.Start(); \
} }
// Update metric with time elapsed since last START. start time is reset // Update metric with time elapsed since last START. start time is reset
// to current timestamp. // to current timestamp.
#define PERF_TIMER_MEASURE(metric) \ #define PERF_TIMER_MEASURE(metric) perf_step_timer_##metric.Measure();
perf_step_timer_ ## metric.Measure();
// Increase metric value // Increase metric value
#define PERF_COUNTER_ADD(metric, value) \ #define PERF_COUNTER_ADD(metric, value) get_perf_context()->metric += value;
perf_context.metric += value;
#endif #endif

14
table/table_test.cc
View File

@ -2085,14 +2085,14 @@ TEST_F(BlockBasedTableTest, BlockReadCountTest) {
GetContext get_context(options.comparator, nullptr, nullptr, nullptr, GetContext get_context(options.comparator, nullptr, nullptr, nullptr,
GetContext::kNotFound, user_key, &value, nullptr, GetContext::kNotFound, user_key, &value, nullptr,
nullptr, nullptr, nullptr); nullptr, nullptr, nullptr);
perf_context.Reset(); get_perf_context()->Reset();
ASSERT_OK(reader->Get(ReadOptions(), encoded_key, &get_context)); ASSERT_OK(reader->Get(ReadOptions(), encoded_key, &get_context));
if (index_and_filter_in_cache) { if (index_and_filter_in_cache) {
// data, index and filter block // data, index and filter block
ASSERT_EQ(perf_context.block_read_count, 3); ASSERT_EQ(get_perf_context()->block_read_count, 3);
} else { } else {
// just the data block // just the data block
ASSERT_EQ(perf_context.block_read_count, 1); ASSERT_EQ(get_perf_context()->block_read_count, 1);
} }
ASSERT_EQ(get_context.State(), GetContext::kFound); ASSERT_EQ(get_context.State(), GetContext::kFound);
ASSERT_STREQ(value.data(), "hello"); ASSERT_STREQ(value.data(), "hello");
@ -2106,22 +2106,22 @@ TEST_F(BlockBasedTableTest, BlockReadCountTest) {
get_context = GetContext(options.comparator, nullptr, nullptr, nullptr, get_context = GetContext(options.comparator, nullptr, nullptr, nullptr,
GetContext::kNotFound, user_key, &value, nullptr, GetContext::kNotFound, user_key, &value, nullptr,
nullptr, nullptr, nullptr); nullptr, nullptr, nullptr);
perf_context.Reset(); get_perf_context()->Reset();
ASSERT_OK(reader->Get(ReadOptions(), encoded_key, &get_context)); ASSERT_OK(reader->Get(ReadOptions(), encoded_key, &get_context));
ASSERT_EQ(get_context.State(), GetContext::kNotFound); ASSERT_EQ(get_context.State(), GetContext::kNotFound);
if (index_and_filter_in_cache) { if (index_and_filter_in_cache) {
if (bloom_filter_type == 0) { if (bloom_filter_type == 0) {
// with block-based, we read index and then the filter // with block-based, we read index and then the filter
ASSERT_EQ(perf_context.block_read_count, 2); ASSERT_EQ(get_perf_context()->block_read_count, 2);
} else { } else {
// with full-filter, we read filter first and then we stop // with full-filter, we read filter first and then we stop
ASSERT_EQ(perf_context.block_read_count, 1); ASSERT_EQ(get_perf_context()->block_read_count, 1);
} }
} else { } else {
// filter is already in memory and it figures out that the key doesn't // filter is already in memory and it figures out that the key doesn't
// exist // exist
ASSERT_EQ(perf_context.block_read_count, 0); ASSERT_EQ(get_perf_context()->block_read_count, 0);
} }
} }
} }

12
tools/db_bench_tool.cc
View File

@ -3840,7 +3840,7 @@ void VerifyDBFromDB(std::string& truth_db_name) {
delete iter; delete iter;
thread->stats.AddBytes(bytes); thread->stats.AddBytes(bytes);
if (FLAGS_perf_level > rocksdb::PerfLevel::kDisable) { if (FLAGS_perf_level > rocksdb::PerfLevel::kDisable) {
thread->stats.AddMessage(perf_context.ToString()); thread->stats.AddMessage(get_perf_context()->ToString());
} }
} }
@ -3921,7 +3921,7 @@ void VerifyDBFromDB(std::string& truth_db_name) {
thread->stats.AddMessage(msg); thread->stats.AddMessage(msg);
if (FLAGS_perf_level > rocksdb::PerfLevel::kDisable) { if (FLAGS_perf_level > rocksdb::PerfLevel::kDisable) {
thread->stats.AddMessage(perf_context.ToString()); thread->stats.AddMessage(get_perf_context()->ToString());
} }
} }
@ -4006,7 +4006,7 @@ void VerifyDBFromDB(std::string& truth_db_name) {
thread->stats.AddMessage(msg); thread->stats.AddMessage(msg);
if (FLAGS_perf_level > rocksdb::PerfLevel::kDisable) { if (FLAGS_perf_level > rocksdb::PerfLevel::kDisable) {
thread->stats.AddMessage(perf_context.ToString()); thread->stats.AddMessage(get_perf_context()->ToString());
} }
} }
@ -4162,7 +4162,7 @@ void VerifyDBFromDB(std::string& truth_db_name) {
thread->stats.AddBytes(bytes); thread->stats.AddBytes(bytes);
thread->stats.AddMessage(msg); thread->stats.AddMessage(msg);
if (FLAGS_perf_level > rocksdb::PerfLevel::kDisable) { if (FLAGS_perf_level > rocksdb::PerfLevel::kDisable) {
thread->stats.AddMessage(perf_context.ToString()); thread->stats.AddMessage(get_perf_context()->ToString());
} }
} }
@ -4816,7 +4816,7 @@ void VerifyDBFromDB(std::string& truth_db_name) {
thread->stats.AddMessage(msg); thread->stats.AddMessage(msg);
if (FLAGS_perf_level > rocksdb::PerfLevel::kDisable) { if (FLAGS_perf_level > rocksdb::PerfLevel::kDisable) {
thread->stats.AddMessage(perf_context.ToString()); thread->stats.AddMessage(get_perf_context()->ToString());
} }
} }
@ -4977,7 +4977,7 @@ void VerifyDBFromDB(std::string& truth_db_name) {
thread->stats.AddBytes(bytes); thread->stats.AddBytes(bytes);
thread->stats.AddMessage(msg); thread->stats.AddMessage(msg);
if (FLAGS_perf_level > rocksdb::PerfLevel::kDisable) { if (FLAGS_perf_level > rocksdb::PerfLevel::kDisable) {
thread->stats.AddMessage(perf_context.ToString()); thread->stats.AddMessage(get_perf_context()->ToString());
} }
} }

73
util/thread_local_test.cc
View File

@ -67,53 +67,54 @@ TEST_F(ThreadLocalTest, UniqueIdTest) {
port::Mutex mu; port::Mutex mu;
port::CondVar cv(&mu); port::CondVar cv(&mu);
ASSERT_EQ(IDChecker::PeekId(), 0u); // perf_context and iostats_context take 2 ids
ASSERT_EQ(IDChecker::PeekId(), 2u);
// New ThreadLocal instance bumps id by 1 // New ThreadLocal instance bumps id by 1
{ {
// Id used 0
Params p1(&mu, &cv, nullptr, 1u);
ASSERT_EQ(IDChecker::PeekId(), 1u);
// Id used 1
Params p2(&mu, &cv, nullptr, 1u);
ASSERT_EQ(IDChecker::PeekId(), 2u);
// Id used 2 // Id used 2
Params p3(&mu, &cv, nullptr, 1u); Params p1(&mu, &cv, nullptr, 1u);
ASSERT_EQ(IDChecker::PeekId(), 3u); ASSERT_EQ(IDChecker::PeekId(), 3u);
// Id used 3 // Id used 3
Params p4(&mu, &cv, nullptr, 1u); Params p2(&mu, &cv, nullptr, 1u);
ASSERT_EQ(IDChecker::PeekId(), 4u); ASSERT_EQ(IDChecker::PeekId(), 4u);
} // Id used 4
// id 3, 2, 1, 0 are in the free queue in order
ASSERT_EQ(IDChecker::PeekId(), 0u);
// pick up 0
Params p1(&mu, &cv, nullptr, 1u);
ASSERT_EQ(IDChecker::PeekId(), 1u);
// pick up 1
Params* p2 = new Params(&mu, &cv, nullptr, 1u);
ASSERT_EQ(IDChecker::PeekId(), 2u);
// pick up 2
Params p3(&mu, &cv, nullptr, 1u); Params p3(&mu, &cv, nullptr, 1u);
ASSERT_EQ(IDChecker::PeekId(), 3u); ASSERT_EQ(IDChecker::PeekId(), 5u);
// return up 1 // Id used 5
delete p2;
ASSERT_EQ(IDChecker::PeekId(), 1u);
// Now we have 3, 1 in queue
// pick up 1
Params p4(&mu, &cv, nullptr, 1u); Params p4(&mu, &cv, nullptr, 1u);
ASSERT_EQ(IDChecker::PeekId(), 6u);
}
// id 5, 4, 3, 2 are in the free queue in order
ASSERT_EQ(IDChecker::PeekId(), 2u);
// pick up 2
Params p1(&mu, &cv, nullptr, 1u);
ASSERT_EQ(IDChecker::PeekId(), 3u); ASSERT_EQ(IDChecker::PeekId(), 3u);
// pick up 3 // pick up 3
Params* p2 = new Params(&mu, &cv, nullptr, 1u);
ASSERT_EQ(IDChecker::PeekId(), 4u);
// pick up 4
Params p3(&mu, &cv, nullptr, 1u);
ASSERT_EQ(IDChecker::PeekId(), 5u);
// return up 3
delete p2;
ASSERT_EQ(IDChecker::PeekId(), 3u);
// Now we have 4, 2 in queue
// pick up 3
Params p4(&mu, &cv, nullptr, 1u);
ASSERT_EQ(IDChecker::PeekId(), 5u);
// pick up 5
Params p5(&mu, &cv, nullptr, 1u); Params p5(&mu, &cv, nullptr, 1u);
// next new id // next new id
ASSERT_EQ(IDChecker::PeekId(), 4u); ASSERT_EQ(IDChecker::PeekId(), 6u);
// After exit, id sequence in queue: // After exit, id sequence in queue:
// 3, 1, 2, 0 // 5, 4, 3, 2(, 1, 0)
} }
#endif // __clang_analyzer__ #endif // __clang_analyzer__
TEST_F(ThreadLocalTest, SequentialReadWriteTest) { TEST_F(ThreadLocalTest, SequentialReadWriteTest) {
// global id list carries over 3, 1, 2, 0 // global id list carries over 5, 4, 3, 2
ASSERT_EQ(IDChecker::PeekId(), 0u); ASSERT_EQ(IDChecker::PeekId(), 2u);
port::Mutex mu; port::Mutex mu;
port::CondVar cv(&mu); port::CondVar cv(&mu);
@ -143,7 +144,7 @@ TEST_F(ThreadLocalTest, SequentialReadWriteTest) {
}; };
for (int iter = 0; iter < 1024; ++iter) { for (int iter = 0; iter < 1024; ++iter) {
ASSERT_EQ(IDChecker::PeekId(), 1u); ASSERT_EQ(IDChecker::PeekId(), 3u);
// Another new thread, read/write should not see value from previous thread // Another new thread, read/write should not see value from previous thread
env_->StartThread(func, static_cast<void*>(&p)); env_->StartThread(func, static_cast<void*>(&p));
mu.Lock(); mu.Lock();
@ -151,13 +152,13 @@ TEST_F(ThreadLocalTest, SequentialReadWriteTest) {
cv.Wait(); cv.Wait();
} }
mu.Unlock(); mu.Unlock();
ASSERT_EQ(IDChecker::PeekId(), 1u); ASSERT_EQ(IDChecker::PeekId(), 3u);
} }
} }
TEST_F(ThreadLocalTest, ConcurrentReadWriteTest) { TEST_F(ThreadLocalTest, ConcurrentReadWriteTest) {
// global id list carries over 3, 1, 2, 0 // global id list carries over 5, 4, 3, 2
ASSERT_EQ(IDChecker::PeekId(), 0u); ASSERT_EQ(IDChecker::PeekId(), 2u);
ThreadLocalPtr tls2; ThreadLocalPtr tls2;
port::Mutex mu1; port::Mutex mu1;
@ -238,11 +239,11 @@ TEST_F(ThreadLocalTest, ConcurrentReadWriteTest) {
} }
mu2.Unlock(); mu2.Unlock();
ASSERT_EQ(IDChecker::PeekId(), 3u); ASSERT_EQ(IDChecker::PeekId(), 5u);
} }
TEST_F(ThreadLocalTest, Unref) { TEST_F(ThreadLocalTest, Unref) {
ASSERT_EQ(IDChecker::PeekId(), 0u); ASSERT_EQ(IDChecker::PeekId(), 2u);
auto unref = [](void* ptr) { auto unref = [](void* ptr) {
auto& p = *static_cast<Params*>(ptr); auto& p = *static_cast<Params*>(ptr);

4
utilities/env_timed_test.cc
View File

@ -16,14 +16,14 @@ class TimedEnvTest : public testing::Test {
TEST_F(TimedEnvTest, BasicTest) { TEST_F(TimedEnvTest, BasicTest) {
SetPerfLevel(PerfLevel::kEnableTime); SetPerfLevel(PerfLevel::kEnableTime);
ASSERT_EQ(0, perf_context.env_new_writable_file_nanos); ASSERT_EQ(0, get_perf_context()->env_new_writable_file_nanos);
std::unique_ptr<Env> mem_env(NewMemEnv(Env::Default())); std::unique_ptr<Env> mem_env(NewMemEnv(Env::Default()));
std::unique_ptr<Env> timed_env(NewTimedEnv(mem_env.get())); std::unique_ptr<Env> timed_env(NewTimedEnv(mem_env.get()));
std::unique_ptr<WritableFile> writable_file; std::unique_ptr<WritableFile> writable_file;
timed_env->NewWritableFile("f", &writable_file, EnvOptions()); timed_env->NewWritableFile("f", &writable_file, EnvOptions());
ASSERT_GT(perf_context.env_new_writable_file_nanos, 0); ASSERT_GT(get_perf_context()->env_new_writable_file_nanos, 0);
} }
} // namespace rocksdb } // namespace rocksdb