andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Support for block_cache num_shards and other config via option string.

Browse Source 
Summary:
Problem: Option string accepts only cache_size as parameter for block_cache which is specified as "block_cache=1M".
It doesn't accept other parameters like num_shards etc.

Changes :
1) ParseBlockBasedTableOption in block_based_table_factory is edited to accept cache options in the format "block_cache=<cache_size>:<num_shard_bits>:<strict_capacity_limit>:<high_pri_pool_ratio>".
Options other than cache_size are optional to maintain backward compatibility. The changes are valid for block_cache_compressed as well.
For example, "block_cache=1M:6:true:0.5", "block_cache=1M:6:true", "block_cache=1M:6" and "block_cache=1M" are all valid option strings.

2) Corresponding unit tests are added.
Closes https://github.com/facebook/rocksdb/pull/3108

Differential Revision: D6420997

Pulled By: sagar0

fbshipit-source-id: cdea8b785688d2802907974af27225ccc1c0cd43
This commit is contained in:
Phani Shekhar Mantripragada 2017-11-28 10:35:17 -08:00 committed by Facebook Github Bot
parent 2f09524762
commit 4b65cfc723
7 changed files with 207 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
cache/lru_cache.cc vendored
View File

@ -169,6 +169,11 @@ size_t LRUCacheShard::TEST_GetLRUSize() {
return lru_size;
}
double LRUCacheShard::GetHighPriPoolRatio() {
MutexLock l(&mutex_);
return high_pri_pool_ratio_;
}
void LRUCacheShard::LRU_Remove(LRUHandle* e) {
assert(e->next != nullptr);
assert(e->prev != nullptr);
@ -513,6 +518,20 @@ size_t LRUCache::TEST_GetLRUSize() {
return lru_size_of_all_shards;
}
double LRUCache::GetHighPriPoolRatio() {
double result = 0.0;
if (num_shards_ > 0) {
result = shards_[0].GetHighPriPoolRatio();
}
return result;
}
std::shared_ptr<Cache> NewLRUCache(const LRUCacheOptions& cache_opts) {
return NewLRUCache(cache_opts.capacity, cache_opts.num_shard_bits,
cache_opts.strict_capacity_limit,
cache_opts.high_pri_pool_ratio);
}
std::shared_ptr<Cache> NewLRUCache(size_t capacity, int num_shard_bits,
bool strict_capacity_limit,
double high_pri_pool_ratio) {

5
cache/lru_cache.h vendored
View File

@ -202,6 +202,9 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard : public CacheShard {
// not threadsafe
size_t TEST_GetLRUSize();
// Retrives high pri pool ratio
double GetHighPriPoolRatio();
// Overloading to aligned it to cache line size
void* operator new(size_t);
@ -293,6 +296,8 @@ class LRUCache : public ShardedCache {
// Retrieves number of elements in LRU, for unit test purpose only
size_t TEST_GetLRUSize();
// Retrives high pri pool ratio
double GetHighPriPoolRatio();
private:
LRUCacheShard* shards_;

27
include/rocksdb/cache.h
View File

@ -33,6 +33,31 @@ namespace rocksdb {
class Cache;
struct LRUCacheOptions {
// Capacity of the cache.
size_t capacity = 0;
// Cache is sharded into 2^num_shard_bits shards,
// by hash of key. Refer to NewLRUCache for further
// information.
int num_shard_bits = -1;
// If strict_capacity_limit is set,
// insert to the cache will fail when cache is full.
bool strict_capacity_limit = false;
// Percentage of cache reserved for high priority entries.
double high_pri_pool_ratio = 0.0;
LRUCacheOptions() {}
LRUCacheOptions(size_t _capacity, int _num_shard_bits,
bool _strict_capacity_limit, double _high_pri_pool_ratio)
: capacity(_capacity),
num_shard_bits(_num_shard_bits),
strict_capacity_limit(_strict_capacity_limit),
high_pri_pool_ratio(_high_pri_pool_ratio) {}
};
// Create a new cache with a fixed size capacity. The cache is sharded
// to 2^num_shard_bits shards, by hash of the key. The total capacity
// is divided and evenly assigned to each shard. If strict_capacity_limit
@ -46,6 +71,8 @@ extern std::shared_ptr<Cache> NewLRUCache(size_t capacity,
bool strict_capacity_limit = false,
double high_pri_pool_ratio = 0.0);
extern std::shared_ptr<Cache> NewLRUCache(const LRUCacheOptions& cache_opts);
// Similar to NewLRUCache, but create a cache based on CLOCK algorithm with
// better concurrent performance in some cases. See util/clock_cache.cc for
// more detail.

28
options/options_helper.cc
View File

@ -491,6 +491,11 @@ bool ParseOptionHelper(char* opt_address, const OptionType& opt_type,
}
return true;
}
case OptionType::kLRUCacheOptions: {
return ParseStructOptions<LRUCacheOptions>(value,
reinterpret_cast<LRUCacheOptions*>(opt_address),
lru_cache_options_type_info);
}
default:
return false;
}
@ -1519,6 +1524,7 @@ std::unordered_map<std::string, InfoLogLevel>
ColumnFamilyOptions OptionsHelper::dummy_cf_options;
CompactionOptionsFIFO OptionsHelper::dummy_comp_options;
LRUCacheOptions OptionsHelper::dummy_lru_cache_options;
// offset_of is used to get the offset of a class data member
// ex: offset_of(&ColumnFamilyOptions::num_levels)
@ -1544,6 +1550,11 @@ int offset_of(T1 CompactionOptionsFIFO::*member) {
return int(size_t(&(OptionsHelper::dummy_comp_options.*member)) -
size_t(&OptionsHelper::dummy_comp_options));
}
template <typename T1>
int offset_of(T1 LRUCacheOptions::*member) {
return int(size_t(&(OptionsHelper::dummy_lru_cache_options.*member)) -
size_t(&OptionsHelper::dummy_lru_cache_options));
}
std::unordered_map<std::string, OptionTypeInfo>
OptionsHelper::cf_options_type_info = {
@ -1788,6 +1799,23 @@ std::unordered_map<std::string, OptionTypeInfo>
OptionType::kBoolean, OptionVerificationType::kNormal, true,
offsetof(struct CompactionOptionsFIFO, allow_compaction)}}};
std::unordered_map<std::string, OptionTypeInfo>
OptionsHelper::lru_cache_options_type_info = {
{"capacity", {offset_of(&LRUCacheOptions::capacity),
OptionType::kSizeT, OptionVerificationType::kNormal, true,
offsetof(struct LRUCacheOptions, capacity)}},
{"num_shard_bits", {offset_of(&LRUCacheOptions::num_shard_bits),
OptionType::kInt, OptionVerificationType::kNormal, true,
offsetof(struct LRUCacheOptions, num_shard_bits)}},
{"strict_capacity_limit",
{offset_of(&LRUCacheOptions::strict_capacity_limit),
OptionType::kBoolean, OptionVerificationType::kNormal, true,
offsetof(struct LRUCacheOptions, strict_capacity_limit)}},
{"high_pri_pool_ratio",
{offset_of(&LRUCacheOptions::high_pri_pool_ratio),
OptionType::kDouble, OptionVerificationType::kNormal, true,
offsetof(struct LRUCacheOptions, high_pri_pool_ratio)}}};
#endif // !ROCKSDB_LITE
} // namespace rocksdb

6
options/options_helper.h
View File

@ -73,6 +73,7 @@ enum class OptionType {
kWALRecoveryMode,
kAccessHint,
kInfoLogLevel,
kLRUCacheOptions,
kUnknown
};
@ -145,6 +146,8 @@ struct OptionsHelper {
static std::unordered_map<std::string, OptionTypeInfo>
fifo_compaction_options_type_info;
static std::unordered_map<std::string, OptionTypeInfo> db_options_type_info;
static std::unordered_map<std::string, OptionTypeInfo>
lru_cache_options_type_info;
static std::unordered_map<std::string, CompressionType>
compression_type_string_map;
static std::unordered_map<std::string, BlockBasedTableOptions::IndexType>
@ -162,6 +165,7 @@ struct OptionsHelper {
info_log_level_string_map;
static ColumnFamilyOptions dummy_cf_options;
static CompactionOptionsFIFO dummy_comp_options;
static LRUCacheOptions dummy_lru_cache_options;
#endif // !ROCKSDB_LITE
};
@ -177,6 +181,8 @@ static auto& cf_options_type_info = OptionsHelper::cf_options_type_info;
static auto& fifo_compaction_options_type_info =
OptionsHelper::fifo_compaction_options_type_info;
static auto& db_options_type_info = OptionsHelper::db_options_type_info;
static auto& lru_cache_options_type_info =
OptionsHelper::lru_cache_options_type_info;
static auto& compression_type_string_map =
OptionsHelper::compression_type_string_map;
static auto& block_base_table_index_type_string_map =

97
options/options_test.cc
View File

@ -16,6 +16,8 @@
#include <unordered_map>
#include <inttypes.h>
#include "cache/lru_cache.h"
#include "cache/sharded_cache.h"
#include "options/options_helper.h"
#include "options/options_parser.h"
#include "options/options_sanity_check.h"
@ -529,6 +531,101 @@ TEST_F(OptionsTest, GetBlockBasedTableOptionsFromString) {
ASSERT_EQ(table_opt.cache_index_and_filter_blocks,
new_opt.cache_index_and_filter_blocks);
ASSERT_EQ(table_opt.filter_policy, new_opt.filter_policy);
// Check block cache options are overwritten when specified
// in new format as a struct.
ASSERT_OK(GetBlockBasedTableOptionsFromString(table_opt,
"block_cache={capacity=1M;num_shard_bits=4;"
"strict_capacity_limit=true;high_pri_pool_ratio=0.5;};"
"block_cache_compressed={capacity=1M;num_shard_bits=4;"
"strict_capacity_limit=true;high_pri_pool_ratio=0.5;}",
&new_opt));
ASSERT_TRUE(new_opt.block_cache != nullptr);
ASSERT_EQ(new_opt.block_cache->GetCapacity(), 1024UL*1024UL);
ASSERT_EQ(std::dynamic_pointer_cast<ShardedCache>(
new_opt.block_cache)->GetNumShardBits(), 4);
ASSERT_EQ(new_opt.block_cache->HasStrictCapacityLimit(), true);
ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>(
new_opt.block_cache)->GetHighPriPoolRatio(), 0.5);
ASSERT_TRUE(new_opt.block_cache_compressed != nullptr);
ASSERT_EQ(new_opt.block_cache_compressed->GetCapacity(), 1024UL*1024UL);
ASSERT_EQ(std::dynamic_pointer_cast<ShardedCache>(
new_opt.block_cache_compressed)->GetNumShardBits(), 4);
ASSERT_EQ(new_opt.block_cache_compressed->HasStrictCapacityLimit(), true);
ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>(
new_opt.block_cache_compressed)->GetHighPriPoolRatio(),
0.5);
// Set only block cache capacity. Check other values are
// reset to default values.
ASSERT_OK(GetBlockBasedTableOptionsFromString(table_opt,
"block_cache={capacity=2M};"
"block_cache_compressed={capacity=2M}",
&new_opt));
ASSERT_TRUE(new_opt.block_cache != nullptr);
ASSERT_EQ(new_opt.block_cache->GetCapacity(), 2*1024UL*1024UL);
// Default values
ASSERT_EQ(std::dynamic_pointer_cast<ShardedCache>(
new_opt.block_cache)->GetNumShardBits(),
GetDefaultCacheShardBits(new_opt.block_cache->GetCapacity()));
ASSERT_EQ(new_opt.block_cache->HasStrictCapacityLimit(), false);
ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>(
new_opt.block_cache)->GetHighPriPoolRatio(), 0.0);
ASSERT_TRUE(new_opt.block_cache_compressed != nullptr);
ASSERT_EQ(new_opt.block_cache_compressed->GetCapacity(), 2*1024UL*1024UL);
// Default values
ASSERT_EQ(std::dynamic_pointer_cast<ShardedCache>(
new_opt.block_cache_compressed)->GetNumShardBits(),
GetDefaultCacheShardBits(
new_opt.block_cache_compressed->GetCapacity()));
ASSERT_EQ(new_opt.block_cache_compressed->HasStrictCapacityLimit(), false);
ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>(
new_opt.block_cache_compressed)->GetHighPriPoolRatio(),
0.0);
// Set couple of block cache options.
ASSERT_OK(GetBlockBasedTableOptionsFromString(table_opt,
"block_cache={num_shard_bits=5;high_pri_pool_ratio=0.5;};"
"block_cache_compressed={num_shard_bits=5;"
"high_pri_pool_ratio=0.5;}",
&new_opt));
ASSERT_EQ(new_opt.block_cache->GetCapacity(), 0);
ASSERT_EQ(std::dynamic_pointer_cast<ShardedCache>(
new_opt.block_cache)->GetNumShardBits(), 5);
ASSERT_EQ(new_opt.block_cache->HasStrictCapacityLimit(), false);
ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>(
new_opt.block_cache)->GetHighPriPoolRatio(), 0.5);
ASSERT_TRUE(new_opt.block_cache_compressed != nullptr);
ASSERT_EQ(new_opt.block_cache_compressed->GetCapacity(), 0);
ASSERT_EQ(std::dynamic_pointer_cast<ShardedCache>(
new_opt.block_cache_compressed)->GetNumShardBits(), 5);
ASSERT_EQ(new_opt.block_cache_compressed->HasStrictCapacityLimit(), false);
ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>(
new_opt.block_cache_compressed)->GetHighPriPoolRatio(),
0.5);
// Set couple of block cache options.
ASSERT_OK(GetBlockBasedTableOptionsFromString(table_opt,
"block_cache={capacity=1M;num_shard_bits=4;"
"strict_capacity_limit=true;};"
"block_cache_compressed={capacity=1M;num_shard_bits=4;"
"strict_capacity_limit=true;}",
&new_opt));
ASSERT_TRUE(new_opt.block_cache != nullptr);
ASSERT_EQ(new_opt.block_cache->GetCapacity(), 1024UL*1024UL);
ASSERT_EQ(std::dynamic_pointer_cast<ShardedCache>(
new_opt.block_cache)->GetNumShardBits(), 4);
ASSERT_EQ(new_opt.block_cache->HasStrictCapacityLimit(), true);
ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>(
new_opt.block_cache)->GetHighPriPoolRatio(), 0.0);
ASSERT_TRUE(new_opt.block_cache_compressed != nullptr);
ASSERT_EQ(new_opt.block_cache_compressed->GetCapacity(), 1024UL*1024UL);
ASSERT_EQ(std::dynamic_pointer_cast<ShardedCache>(
new_opt.block_cache_compressed)->GetNumShardBits(), 4);
ASSERT_EQ(new_opt.block_cache_compressed->HasStrictCapacityLimit(), true);
ASSERT_EQ(std::dynamic_pointer_cast<LRUCache>(
new_opt.block_cache_compressed)->GetHighPriPoolRatio(),
0.0);
}
#endif // !ROCKSDB_LITE

30
table/block_based_table_factory.cc
View File

@ -291,11 +291,31 @@ std::string ParseBlockBasedTableOption(const std::string& name,
if (!input_strings_escaped) {
// if the input string is not escaped, it means this function is
// invoked from SetOptions, which takes the old format.
if (name == "block_cache") {
new_options->block_cache = NewLRUCache(ParseSizeT(value));
return "";
} else if (name == "block_cache_compressed") {
new_options->block_cache_compressed = NewLRUCache(ParseSizeT(value));
if (name == "block_cache" || name == "block_cache_compressed") {
// cache options can be specified in the following format
// "block_cache={capacity=1M;num_shard_bits=4;
// strict_capacity_limit=true;high_pri_pool_ratio=0.5;}"
// To support backward compatibility, the following format
// is also supported.
// "block_cache=1M"
std::shared_ptr<Cache> cache;
// block_cache is specified in format block_cache=<cache_size>.
if (value.find('=') == std::string::npos) {
cache = NewLRUCache(ParseSizeT(value));
} else {
LRUCacheOptions cache_opts;
if(!ParseOptionHelper(reinterpret_cast<char*>(&cache_opts),
OptionType::kLRUCacheOptions, value)) {
return "Invalid cache options";
}
cache = NewLRUCache(cache_opts);
}
if (name == "block_cache") {
new_options->block_cache = cache;
} else {
new_options->block_cache_compressed = cache;
}
return "";
} else if (name == "filter_policy") {
// Expect the following format