andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add options for integrated blob GC (#7661)

Browse Source 
Summary:
This patch simply adds a couple of options that will enable users to
configure garbage collection when using the integrated BlobDB
implementation. The actual GC logic will be added in a separate step.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/7661

Test Plan: `make check`

Reviewed By: riversand963

Differential Revision: D24906544

Pulled By: ltamasi

fbshipit-source-id: ee0e056a712a4b4475cd90de8b27d969bd61b7e1
This commit is contained in:
Levi Tamasi 2020-11-12 18:57:20 -08:00 committed by Facebook GitHub Bot
parent 76ef894f9f
commit bbbb5a280d
8 changed files with 69 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
include/rocksdb/advanced_options.h
View File

@ -733,7 +733,8 @@ struct AdvancedColumnFamilyOptions {
// only pointers to them are stored in SST files. This can reduce write
// amplification for large-value use cases at the cost of introducing a level
// of indirection for reads. See also the options min_blob_size,
// blob_file_size, and blob_compression_type below.
// blob_file_size, blob_compression_type, enable_blob_garbage_collection,
// and blob_garbage_collection_age_cutoff below.
//
// Default: false
//
@ -773,6 +774,30 @@ struct AdvancedColumnFamilyOptions {
// Dynamically changeable through the SetOptions() API
CompressionType blob_compression_type = kNoCompression;
// UNDER CONSTRUCTION -- DO NOT USE
// Enables garbage collection of blobs. Blob GC is performed as part of
// compaction. Valid blobs residing in blob files older than a cutoff get
// relocated to new files as they are encountered during compaction, which
// makes it possible to clean up blob files once they contain nothing but
// obsolete/garbage blobs. See also blob_garbage_collection_age_cutoff below.
//
// Default: false
//
// Dynamically changeable through the SetOptions() API
bool enable_blob_garbage_collection = false;
// UNDER CONSTRUCTION -- DO NOT USE
// The cutoff in terms of blob file age for garbage collection. Blobs in
// the oldest N blob files will be relocated when encountered during
// compaction, where N = garbage_collection_cutoff * number_of_blob_files.
// Note that enable_blob_garbage_collection has to be set in order for this
// option to have any effect.
//
// Default: 0.25
//
// Dynamically changeable through the SetOptions() API
double blob_garbage_collection_age_cutoff = 0.25;
// Create ColumnFamilyOptions with default values for all fields
AdvancedColumnFamilyOptions();
// Create ColumnFamilyOptions from Options

12
options/cf_options.cc
View File

@ -408,6 +408,14 @@ static std::unordered_map<std::string, OptionTypeInfo>
{offsetof(struct MutableCFOptions, blob_compression_type),
OptionType::kCompressionType, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"enable_blob_garbage_collection",
{offsetof(struct MutableCFOptions, enable_blob_garbage_collection),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"blob_garbage_collection_age_cutoff",
{offsetof(struct MutableCFOptions, blob_garbage_collection_age_cutoff),
OptionType::kDouble, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"sample_for_compression",
{offsetof(struct MutableCFOptions, sample_for_compression),
OptionType::kUInt64T, OptionVerificationType::kNormal,
@ -1006,6 +1014,10 @@ void MutableCFOptions::Dump(Logger* log) const {
blob_file_size);
ROCKS_LOG_INFO(log, " blob_compression_type: %s",
CompressionTypeToString(blob_compression_type).c_str());
ROCKS_LOG_INFO(log, " enable_blob_garbage_collection: %s",
enable_blob_garbage_collection ? "true" : "false");
ROCKS_LOG_INFO(log, " blob_garbage_collection_age_cutoff: %f",
blob_garbage_collection_age_cutoff);
}
MutableCFOptions::MutableCFOptions(const Options& options)

7
options/cf_options.h
View File

@ -165,6 +165,9 @@ struct MutableCFOptions {
min_blob_size(options.min_blob_size),
blob_file_size(options.blob_file_size),
blob_compression_type(options.blob_compression_type),
enable_blob_garbage_collection(options.enable_blob_garbage_collection),
blob_garbage_collection_age_cutoff(
options.blob_garbage_collection_age_cutoff),
max_sequential_skip_in_iterations(
options.max_sequential_skip_in_iterations),
check_flush_compaction_key_order(
@ -208,6 +211,8 @@ struct MutableCFOptions {
min_blob_size(0),
blob_file_size(0),
blob_compression_type(kNoCompression),
enable_blob_garbage_collection(false),
blob_garbage_collection_age_cutoff(0.0),
max_sequential_skip_in_iterations(0),
check_flush_compaction_key_order(true),
paranoid_file_checks(false),
@ -269,6 +274,8 @@ struct MutableCFOptions {
uint64_t min_blob_size;
uint64_t blob_file_size;
CompressionType blob_compression_type;
bool enable_blob_garbage_collection;
double blob_garbage_collection_age_cutoff;
// Misc options
uint64_t max_sequential_skip_in_iterations;

9
options/options.cc
View File

@ -92,7 +92,10 @@ AdvancedColumnFamilyOptions::AdvancedColumnFamilyOptions(const Options& options)
enable_blob_files(options.enable_blob_files),
min_blob_size(options.min_blob_size),
blob_file_size(options.blob_file_size),
blob_compression_type(options.blob_compression_type) {
blob_compression_type(options.blob_compression_type),
enable_blob_garbage_collection(options.enable_blob_garbage_collection),
blob_garbage_collection_age_cutoff(
options.blob_garbage_collection_age_cutoff) {
assert(memtable_factory.get() != nullptr);
if (max_bytes_for_level_multiplier_additional.size() <
static_cast<unsigned int>(num_levels)) {
@ -383,6 +386,10 @@ void ColumnFamilyOptions::Dump(Logger* log) const {
blob_file_size);
ROCKS_LOG_HEADER(log, " Options.blob_compression_type: %s",
CompressionTypeToString(blob_compression_type).c_str());
ROCKS_LOG_HEADER(log, " Options.enable_blob_garbage_collection: %s",
enable_blob_garbage_collection ? "true" : "false");
ROCKS_LOG_HEADER(log, " Options.blob_garbage_collection_age_cutoff: %f",
blob_garbage_collection_age_cutoff);
} // ColumnFamilyOptions::Dump
void Options::Dump(Logger* log) const {

4
options/options_helper.cc
View File

@ -229,6 +229,10 @@ ColumnFamilyOptions BuildColumnFamilyOptions(
cf_opts.min_blob_size = mutable_cf_options.min_blob_size;
cf_opts.blob_file_size = mutable_cf_options.blob_file_size;
cf_opts.blob_compression_type = mutable_cf_options.blob_compression_type;
cf_opts.enable_blob_garbage_collection =
mutable_cf_options.enable_blob_garbage_collection;
cf_opts.blob_garbage_collection_age_cutoff =
mutable_cf_options.blob_garbage_collection_age_cutoff;
// Misc options
cf_opts.max_sequential_skip_in_iterations =

2
options/options_settable_test.cc
View File

@ -504,6 +504,8 @@ TEST_F(OptionsSettableTest, ColumnFamilyOptionsAllFieldsSettable) {
"min_blob_size=256;"
"blob_file_size=1000000;"
"blob_compression_type=kBZip2Compression;"
"enable_blob_garbage_collection=true;"
"blob_garbage_collection_age_cutoff=0.5;"
"compaction_options_fifo={max_table_files_size=3;allow_"
"compaction=false;};",
new_options));

8
options/options_test.cc
View File

@ -102,6 +102,8 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) {
{"min_blob_size", "1K"},
{"blob_file_size", "1G"},
{"blob_compression_type", "kZSTD"},
{"enable_blob_garbage_collection", "true"},
{"blob_garbage_collection_age_cutoff", "0.5"},
};
std::unordered_map<std::string, std::string> db_options_map = {
@ -231,6 +233,8 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) {
ASSERT_EQ(new_cf_opt.min_blob_size, 1ULL << 10);
ASSERT_EQ(new_cf_opt.blob_file_size, 1ULL << 30);
ASSERT_EQ(new_cf_opt.blob_compression_type, kZSTD);
ASSERT_EQ(new_cf_opt.enable_blob_garbage_collection, true);
ASSERT_EQ(new_cf_opt.blob_garbage_collection_age_cutoff, 0.5);
cf_options_map["write_buffer_size"] = "hello";
ASSERT_NOK(GetColumnFamilyOptionsFromMap(exact, base_cf_opt, cf_options_map,
@ -1649,6 +1653,8 @@ TEST_F(OptionsOldApiTest, GetOptionsFromMapTest) {
{"min_blob_size", "1K"},
{"blob_file_size", "1G"},
{"blob_compression_type", "kZSTD"},
{"enable_blob_garbage_collection", "true"},
{"blob_garbage_collection_age_cutoff", "0.5"},
};
std::unordered_map<std::string, std::string> db_options_map = {
@ -1770,6 +1776,8 @@ TEST_F(OptionsOldApiTest, GetOptionsFromMapTest) {
ASSERT_EQ(new_cf_opt.min_blob_size, 1ULL << 10);
ASSERT_EQ(new_cf_opt.blob_file_size, 1ULL << 30);
ASSERT_EQ(new_cf_opt.blob_compression_type, kZSTD);
ASSERT_EQ(new_cf_opt.enable_blob_garbage_collection, true);
ASSERT_EQ(new_cf_opt.blob_garbage_collection_age_cutoff, 0.5);
cf_options_map["write_buffer_size"] = "hello";
ASSERT_NOK(GetColumnFamilyOptionsFromMap(

2
test_util/testutil.cc
View File

@ -381,12 +381,14 @@ void RandomInitCFOptions(ColumnFamilyOptions* cf_opt, DBOptions& db_options,
cf_opt->compaction_options_fifo.allow_compaction = rnd->Uniform(2);
cf_opt->memtable_whole_key_filtering = rnd->Uniform(2);
cf_opt->enable_blob_files = rnd->Uniform(2);
cf_opt->enable_blob_garbage_collection = rnd->Uniform(2);
// double options
cf_opt->hard_rate_limit = static_cast<double>(rnd->Uniform(10000)) / 13;
cf_opt->soft_rate_limit = static_cast<double>(rnd->Uniform(10000)) / 13;
cf_opt->memtable_prefix_bloom_size_ratio =
static_cast<double>(rnd->Uniform(10000)) / 20000.0;
cf_opt->blob_garbage_collection_age_cutoff = rnd->Uniform(10000) / 10000.0;
// int options
cf_opt->level0_file_num_compaction_trigger = rnd->Uniform(100);