andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Change default number of cache shard bit to be 6 and max_file_opening_threads to be 16.

Browse Source 
Summary: Cache shard bit 4 is sometimes too small and 6 is a more common value picked by users. Make that default. It shouldn't hurt much to change options.max_file_opening_threads default to be 16, which will reduce the worst case DB open time.

Test Plan: Run all existing tests.

Reviewers: IslamAbdelRahman, yhchiang, kradhakrishnan, igor, andrewkr

Reviewed By: andrewkr

Subscribers: andrewkr, MarkCallaghan, leveldb, dhruba

Differential Revision: https://reviews.facebook.net/D55047
This commit is contained in:
sdong 2016-04-07 13:51:47 -07:00
parent ada88b63f5
commit 1518b733eb
5 changed files with 18 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
DEFAULT_OPTIONS_HISTORY.md
View File

@ -3,4 +3,6 @@
* options.target_file_size_base changes from 2MB to 64MB * options.target_file_size_base changes from 2MB to 64MB
* options.max_bytes_for_level_base changes from 10MB to 256MB * options.max_bytes_for_level_base changes from 10MB to 256MB
* options.soft_pending_compaction_bytes_limit changes from 0 (disabled) to 64GB * options.soft_pending_compaction_bytes_limit changes from 0 (disabled) to 64GB
* options.hard_pending_compaction_bytes_limit changes from 0 (disabled) to 256GB * options.hard_pending_compaction_bytes_limit changes from 0 (disabled) to 256GB
* table_cache_numshardbits changes from 4 to 6
* max_file_opening_threads changes from 1 to 16

18
table/table_test.cc
View File

@ -1065,7 +1065,7 @@ void PrefetchRange(TableConstructor* c, Options* opt,
const std::vector<std::string>& keys_not_in_cache, const std::vector<std::string>& keys_not_in_cache,
const Status expected_status = Status::OK()) { const Status expected_status = Status::OK()) {
// reset the cache and reopen the table // reset the cache and reopen the table
table_options->block_cache = NewLRUCache(16 * 1024 * 1024); table_options->block_cache = NewLRUCache(16 * 1024 * 1024, 4);
opt->table_factory.reset(NewBlockBasedTableFactory(*table_options)); opt->table_factory.reset(NewBlockBasedTableFactory(*table_options));
const ImmutableCFOptions ioptions2(*opt); const ImmutableCFOptions ioptions2(*opt);
ASSERT_OK(c->Reopen(ioptions2)); ASSERT_OK(c->Reopen(ioptions2));
@ -1094,7 +1094,7 @@ TEST_F(BlockBasedTableTest, PrefetchTest) {
BlockBasedTableOptions table_options; BlockBasedTableOptions table_options;
table_options.block_size = 1024; table_options.block_size = 1024;
// big enough so we don't ever lose cached values. // big enough so we don't ever lose cached values.
table_options.block_cache = NewLRUCache(16 * 1024 * 1024); table_options.block_cache = NewLRUCache(16 * 1024 * 1024, 4);
opt.table_factory.reset(NewBlockBasedTableFactory(table_options)); opt.table_factory.reset(NewBlockBasedTableFactory(table_options));
TableConstructor c(BytewiseComparator()); TableConstructor c(BytewiseComparator());
@ -1328,7 +1328,7 @@ TEST_F(TableTest, HashIndexTest) {
table_options.index_type = BlockBasedTableOptions::kHashSearch; table_options.index_type = BlockBasedTableOptions::kHashSearch;
table_options.hash_index_allow_collision = true; table_options.hash_index_allow_collision = true;
table_options.block_size = 1700; table_options.block_size = 1700;
table_options.block_cache = NewLRUCache(1024); table_options.block_cache = NewLRUCache(1024, 4);
options.table_factory.reset(NewBlockBasedTableFactory(table_options)); options.table_factory.reset(NewBlockBasedTableFactory(table_options));
std::unique_ptr<InternalKeyComparator> comparator( std::unique_ptr<InternalKeyComparator> comparator(
@ -1550,7 +1550,7 @@ TEST_F(BlockBasedTableTest, BlockCacheDisabledTest) {
options.create_if_missing = true; options.create_if_missing = true;
options.statistics = CreateDBStatistics(); options.statistics = CreateDBStatistics();
BlockBasedTableOptions table_options; BlockBasedTableOptions table_options;
table_options.block_cache = NewLRUCache(1024); table_options.block_cache = NewLRUCache(1024, 4);
table_options.filter_policy.reset(NewBloomFilterPolicy(10)); table_options.filter_policy.reset(NewBloomFilterPolicy(10));
options.table_factory.reset(new BlockBasedTableFactory(table_options)); options.table_factory.reset(new BlockBasedTableFactory(table_options));
std::vector<std::string> keys; std::vector<std::string> keys;
@ -1596,7 +1596,7 @@ TEST_F(BlockBasedTableTest, FilterBlockInBlockCache) {
// Enable the cache for index/filter blocks // Enable the cache for index/filter blocks
BlockBasedTableOptions table_options; BlockBasedTableOptions table_options;
table_options.block_cache = NewLRUCache(1024); table_options.block_cache = NewLRUCache(1024, 4);
table_options.cache_index_and_filter_blocks = true; table_options.cache_index_and_filter_blocks = true;
options.table_factory.reset(new BlockBasedTableFactory(table_options)); options.table_factory.reset(new BlockBasedTableFactory(table_options));
std::vector<std::string> keys; std::vector<std::string> keys;
@ -1679,7 +1679,7 @@ TEST_F(BlockBasedTableTest, FilterBlockInBlockCache) {
// -- PART 2: Open with very small block cache // -- PART 2: Open with very small block cache
// In this test, no block will ever get hit since the block cache is // In this test, no block will ever get hit since the block cache is
// too small to fit even one entry. // too small to fit even one entry.
table_options.block_cache = NewLRUCache(1); table_options.block_cache = NewLRUCache(1, 4);
options.statistics = CreateDBStatistics(); options.statistics = CreateDBStatistics();
options.table_factory.reset(new BlockBasedTableFactory(table_options)); options.table_factory.reset(new BlockBasedTableFactory(table_options));
const ImmutableCFOptions ioptions2(options); const ImmutableCFOptions ioptions2(options);
@ -1719,7 +1719,7 @@ TEST_F(BlockBasedTableTest, FilterBlockInBlockCache) {
c.ResetTableReader(); c.ResetTableReader();
// -- PART 3: Open table with bloom filter enabled but not in SST file // -- PART 3: Open table with bloom filter enabled but not in SST file
table_options.block_cache = NewLRUCache(4096); table_options.block_cache = NewLRUCache(4096, 4);
table_options.cache_index_and_filter_blocks = false; table_options.cache_index_and_filter_blocks = false;
options.table_factory.reset(NewBlockBasedTableFactory(table_options)); options.table_factory.reset(NewBlockBasedTableFactory(table_options));
@ -1881,7 +1881,7 @@ TEST_F(BlockBasedTableTest, BlockCacheLeak) {
BlockBasedTableOptions table_options; BlockBasedTableOptions table_options;
table_options.block_size = 1024; table_options.block_size = 1024;
// big enough so we don't ever lose cached values. // big enough so we don't ever lose cached values.
table_options.block_cache = NewLRUCache(16 * 1024 * 1024); table_options.block_cache = NewLRUCache(16 * 1024 * 1024, 4);
opt.table_factory.reset(NewBlockBasedTableFactory(table_options)); opt.table_factory.reset(NewBlockBasedTableFactory(table_options));
TableConstructor c(BytewiseComparator()); TableConstructor c(BytewiseComparator());
@ -1915,7 +1915,7 @@ TEST_F(BlockBasedTableTest, BlockCacheLeak) {
c.ResetTableReader(); c.ResetTableReader();
// rerun with different block cache // rerun with different block cache
table_options.block_cache = NewLRUCache(16 * 1024 * 1024); table_options.block_cache = NewLRUCache(16 * 1024 * 1024, 4);
opt.table_factory.reset(NewBlockBasedTableFactory(table_options)); opt.table_factory.reset(NewBlockBasedTableFactory(table_options));
const ImmutableCFOptions ioptions2(opt); const ImmutableCFOptions ioptions2(opt);
ASSERT_OK(c.Reopen(ioptions2)); ASSERT_OK(c.Reopen(ioptions2));

2
util/cache.cc
View File

@ -520,7 +520,7 @@ void LRUCache::Erase(const Slice& key, uint32_t hash) {
} }
} }
static int kNumShardBits = 4; // default values, can be overridden static int kNumShardBits = 6; // default values, can be overridden
class ShardedLRUCache : public Cache { class ShardedLRUCache : public Cache {
private: private:

6
util/options.cc
View File

@ -221,7 +221,7 @@ DBOptions::DBOptions()
info_log_level(DEBUG_LEVEL), info_log_level(DEBUG_LEVEL),
#endif // NDEBUG #endif // NDEBUG
max_open_files(5000), max_open_files(5000),
max_file_opening_threads(1), max_file_opening_threads(16),
max_total_wal_size(0), max_total_wal_size(0),
statistics(nullptr), statistics(nullptr),
disableDataSync(false), disableDataSync(false),
@ -238,7 +238,7 @@ DBOptions::DBOptions()
keep_log_file_num(1000), keep_log_file_num(1000),
recycle_log_file_num(0), recycle_log_file_num(0),
max_manifest_file_size(std::numeric_limits<uint64_t>::max()), max_manifest_file_size(std::numeric_limits<uint64_t>::max()),
table_cache_numshardbits(4), table_cache_numshardbits(6),
WAL_ttl_seconds(0), WAL_ttl_seconds(0),
WAL_size_limit_MB(0), WAL_size_limit_MB(0),
manifest_preallocation_size(4 * 1024 * 1024), manifest_preallocation_size(4 * 1024 * 1024),
@ -673,6 +673,8 @@ Options* Options::OldDefaults(int rocksdb_major_version,
DBOptions* DBOptions::OldDefaults(int rocksdb_major_version, DBOptions* DBOptions::OldDefaults(int rocksdb_major_version,
int rocksdb_minor_version) { int rocksdb_minor_version) {
max_file_opening_threads = 1;
table_cache_numshardbits = 4;
return this; return this;
} }

1
util/options_test.cc
View File

@ -1270,6 +1270,7 @@ TEST_F(OptionsParserTest, DifferentDefault) {
Options old_default_opts46; Options old_default_opts46;
old_default_opts46.OldDefaults(); old_default_opts46.OldDefaults();
ASSERT_EQ(10 * 1048576, old_default_opts46.max_bytes_for_level_base); ASSERT_EQ(10 * 1048576, old_default_opts46.max_bytes_for_level_base);
ASSERT_EQ(4, old_default_opts46.table_cache_numshardbits);
ColumnFamilyOptions old_default_cf_opts; ColumnFamilyOptions old_default_cf_opts;
old_default_cf_opts.OldDefaults(); old_default_cf_opts.OldDefaults();