andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
c8bae6e29c
rocksdb/options/cf_options.cc
Changyu Bi cc23b46da1 Support using ZDICT_finalizeDictionary to generate zstd dictionary (#9857) 
Summary:
An untrained dictionary is currently simply the concatenation of several samples. The ZSTD API, ZDICT_finalizeDictionary(), can improve such a dictionary's effectiveness at low cost. This PR changes how dictionary is created by calling the ZSTD ZDICT_finalizeDictionary() API instead of creating raw content dictionary (when max_dict_buffer_bytes > 0), and pass in all buffered uncompressed data blocks as samples.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/9857

Test Plan:
#### db_bench test for cpu/memory of compression+decompression and space saving on synthetic data:
Set up: change the parameter [here](fb9a167a55/tools/db_bench_tool.cc (L1766)) to 16384 to make synthetic data more compressible.
```
# linked local ZSTD with version 1.5.2
# DEBUG_LEVEL=0 ROCKSDB_NO_FBCODE=1 ROCKSDB_DISABLE_ZSTD=1  EXTRA_CXXFLAGS="-DZSTD_STATIC_LINKING_ONLY -DZSTD -I/data/users/changyubi/install/include/" EXTRA_LDFLAGS="-L/data/users/changyubi/install/lib/ -l:libzstd.a" make -j32 db_bench

dict_bytes=16384
train_bytes=1048576
echo "========== No Dictionary =========="
TEST_TMPDIR=/dev/shm ./db_bench -benchmarks=filluniquerandom,compact -num=10000000 -compression_type=zstd -compression_max_dict_bytes=0 -block_size=4096 -max_background_jobs=24 -memtablerep=vector -allow_concurrent_memtable_write=false -disable_wal=true -max_write_buffer_number=8 >/dev/null 2>&1
TEST_TMPDIR=/dev/shm /usr/bin/time ./db_bench -use_existing_db=true -benchmarks=compact -compression_type=zstd -compression_max_dict_bytes=0 -block_size=4096 2>&1 | grep elapsed
du -hc /dev/shm/dbbench/*sst | grep total

echo "========== Raw Content Dictionary =========="
TEST_TMPDIR=/dev/shm ./db_bench_main -benchmarks=filluniquerandom,compact -num=10000000 -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -block_size=4096 -max_background_jobs=24 -memtablerep=vector -allow_concurrent_memtable_write=false -disable_wal=true -max_write_buffer_number=8 >/dev/null 2>&1
TEST_TMPDIR=/dev/shm /usr/bin/time ./db_bench_main -use_existing_db=true -benchmarks=compact -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -block_size=4096 2>&1 | grep elapsed
du -hc /dev/shm/dbbench/*sst | grep total

echo "========== FinalizeDictionary =========="
TEST_TMPDIR=/dev/shm ./db_bench -benchmarks=filluniquerandom,compact -num=10000000 -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -compression_zstd_max_train_bytes=$train_bytes -compression_use_zstd_dict_trainer=false -block_size=4096 -max_background_jobs=24 -memtablerep=vector -allow_concurrent_memtable_write=false -disable_wal=true -max_write_buffer_number=8 >/dev/null 2>&1
TEST_TMPDIR=/dev/shm /usr/bin/time ./db_bench -use_existing_db=true -benchmarks=compact -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -compression_zstd_max_train_bytes=$train_bytes -compression_use_zstd_dict_trainer=false -block_size=4096 2>&1 | grep elapsed
du -hc /dev/shm/dbbench/*sst | grep total

echo "========== TrainDictionary =========="
TEST_TMPDIR=/dev/shm ./db_bench -benchmarks=filluniquerandom,compact -num=10000000 -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -compression_zstd_max_train_bytes=$train_bytes -block_size=4096 -max_background_jobs=24 -memtablerep=vector -allow_concurrent_memtable_write=false -disable_wal=true -max_write_buffer_number=8 >/dev/null 2>&1
TEST_TMPDIR=/dev/shm /usr/bin/time ./db_bench -use_existing_db=true -benchmarks=compact -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -compression_zstd_max_train_bytes=$train_bytes -block_size=4096 2>&1 | grep elapsed
du -hc /dev/shm/dbbench/*sst | grep total

# Result: TrainDictionary is much better on space saving, but FinalizeDictionary seems to use less memory.
# before compression data size: 1.2GB
dict_bytes=16384
max_dict_buffer_bytes =  1048576
                    space   cpu/memory
No Dictionary       468M    14.93user 1.00system 0:15.92elapsed 100%CPU (0avgtext+0avgdata 23904maxresident)k
Raw Dictionary      251M    15.81user 0.80system 0:16.56elapsed 100%CPU (0avgtext+0avgdata 156808maxresident)k
FinalizeDictionary  236M    11.93user 0.64system 0:12.56elapsed 100%CPU (0avgtext+0avgdata 89548maxresident)k
TrainDictionary     84M     7.29user 0.45system 0:07.75elapsed 100%CPU (0avgtext+0avgdata 97288maxresident)k
```

#### Benchmark on 10 sample SST files for spacing saving and CPU time on compression:
FinalizeDictionary is comparable to TrainDictionary in terms of space saving, and takes less time in compression.
```
dict_bytes=16384
train_bytes=1048576

for sst_file in `ls ../temp/myrock-sst/`
do
  echo "********** $sst_file **********"
  echo "========== No Dictionary =========="
  ./sst_dump --file="../temp/myrock-sst/$sst_file" --command=recompress --compression_level_from=6 --compression_level_to=6 --compression_types=kZSTD

  echo "========== Raw Content Dictionary =========="
  ./sst_dump --file="../temp/myrock-sst/$sst_file" --command=recompress --compression_level_from=6 --compression_level_to=6 --compression_types=kZSTD --compression_max_dict_bytes=$dict_bytes

  echo "========== FinalizeDictionary =========="
  ./sst_dump --file="../temp/myrock-sst/$sst_file" --command=recompress --compression_level_from=6 --compression_level_to=6 --compression_types=kZSTD --compression_max_dict_bytes=$dict_bytes --compression_zstd_max_train_bytes=$train_bytes --compression_use_zstd_finalize_dict

  echo "========== TrainDictionary =========="
  ./sst_dump --file="../temp/myrock-sst/$sst_file" --command=recompress --compression_level_from=6 --compression_level_to=6 --compression_types=kZSTD --compression_max_dict_bytes=$dict_bytes --compression_zstd_max_train_bytes=$train_bytes
done

                         010240.sst (Size/Time) 011029.sst              013184.sst              021552.sst              185054.sst              185137.sst              191666.sst              7560381.sst             7604174.sst             7635312.sst
No Dictionary           28165569 / 2614419      32899411 / 2976832      32977848 / 3055542      31966329 / 2004590      33614351 / 1755877      33429029 / 1717042      33611933 / 1776936      33634045 / 2771417      33789721 / 2205414      33592194 / 388254
Raw Content Dictionary  28019950 / 2697961      33748665 / 3572422      33896373 / 3534701      26418431 / 2259658      28560825 / 1839168      28455030 / 1846039      28494319 / 1861349      32391599 / 3095649      33772142 / 2407843      33592230 / 474523
FinalizeDictionary      27896012 / 2650029      33763886 / 3719427      33904283 / 3552793      26008225 / 2198033      28111872 / 1869530      28014374 / 1789771      28047706 / 1848300      32296254 / 3204027      33698698 / 2381468      33592344 / 517433
TrainDictionary         28046089 / 2740037      33706480 / 3679019      33885741 / 3629351      25087123 / 2204558      27194353 / 1970207      27234229 / 1896811      27166710 / 1903119      32011041 / 3322315      32730692 / 2406146      33608631 / 570593
```

#### Decompression/Read test:
With FinalizeDictionary/TrainDictionary, some data structure used for decompression are in stored in dictionary, so they are expected to be faster in terms of decompression/reads.
```
dict_bytes=16384
train_bytes=1048576
echo "No Dictionary"
TEST_TMPDIR=/dev/shm/ ./db_bench -benchmarks=filluniquerandom,compact -compression_type=zstd -compression_max_dict_bytes=0 > /dev/null 2>&1
TEST_TMPDIR=/dev/shm/ ./db_bench -use_existing_db=true -benchmarks=readrandom -cache_size=0 -compression_type=zstd -compression_max_dict_bytes=0 2>&1 | grep MB/s

echo "Raw Dictionary"
TEST_TMPDIR=/dev/shm/ ./db_bench -benchmarks=filluniquerandom,compact -compression_type=zstd -compression_max_dict_bytes=$dict_bytes > /dev/null 2>&1
TEST_TMPDIR=/dev/shm/ ./db_bench -use_existing_db=true -benchmarks=readrandom -cache_size=0 -compression_type=zstd  -compression_max_dict_bytes=$dict_bytes 2>&1 | grep MB/s

echo "FinalizeDict"
TEST_TMPDIR=/dev/shm/ ./db_bench -benchmarks=filluniquerandom,compact -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -compression_zstd_max_train_bytes=$train_bytes -compression_use_zstd_dict_trainer=false  > /dev/null 2>&1
TEST_TMPDIR=/dev/shm/ ./db_bench -use_existing_db=true -benchmarks=readrandom -cache_size=0 -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -compression_zstd_max_train_bytes=$train_bytes -compression_use_zstd_dict_trainer=false 2>&1 | grep MB/s

echo "Train Dictionary"
TEST_TMPDIR=/dev/shm/ ./db_bench -benchmarks=filluniquerandom,compact -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -compression_zstd_max_train_bytes=$train_bytes > /dev/null 2>&1
TEST_TMPDIR=/dev/shm/ ./db_bench -use_existing_db=true -benchmarks=readrandom -cache_size=0 -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -compression_zstd_max_train_bytes=$train_bytes 2>&1 | grep MB/s

No Dictionary
readrandom   :      12.183 micros/op 82082 ops/sec 12.183 seconds 1000000 operations;    9.1 MB/s (1000000 of 1000000 found)
Raw Dictionary
readrandom   :      12.314 micros/op 81205 ops/sec 12.314 seconds 1000000 operations;    9.0 MB/s (1000000 of 1000000 found)
FinalizeDict
readrandom   :       9.787 micros/op 102180 ops/sec 9.787 seconds 1000000 operations;   11.3 MB/s (1000000 of 1000000 found)
Train Dictionary
readrandom   :       9.698 micros/op 103108 ops/sec 9.699 seconds 1000000 operations;   11.4 MB/s (1000000 of 1000000 found)
```

Reviewed By: ajkr

Differential Revision: D35720026

Pulled By: cbi42

fbshipit-source-id: 24d230fdff0fd28a1bb650658798f00dfcfb2a1f
2022-05-20 12:09:09 -07:00

1104 lines
50 KiB
C++
Raw Blame History

// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
#include "options/cf_options.h"
#include <cassert>
#include <cinttypes>
#include <limits>
#include <string>
#include "logging/logging.h"
#include "options/configurable_helper.h"
#include "options/db_options.h"
#include "options/options_helper.h"
#include "options/options_parser.h"
#include "port/port.h"
#include "rocksdb/compaction_filter.h"
#include "rocksdb/concurrent_task_limiter.h"
#include "rocksdb/configurable.h"
#include "rocksdb/convenience.h"
#include "rocksdb/env.h"
#include "rocksdb/file_system.h"
#include "rocksdb/merge_operator.h"
#include "rocksdb/options.h"
#include "rocksdb/table.h"
#include "rocksdb/utilities/object_registry.h"
#include "rocksdb/utilities/options_type.h"
#include "util/cast_util.h"
// NOTE: in this file, many option flags that were deprecated
// and removed from the rest of the code have to be kept here
// and marked as kDeprecated in order to be able to read old
// OPTIONS files.
namespace ROCKSDB_NAMESPACE {
#ifndef ROCKSDB_LITE
static Status ParseCompressionOptions(const std::string& value,
const std::string& name,
CompressionOptions& compression_opts) {
const char kDelimiter = ':';
std::istringstream field_stream(value);
std::string field;
if (!std::getline(field_stream, field, kDelimiter)) {
return Status::InvalidArgument("unable to parse the specified CF option " +
name);
}
compression_opts.window_bits = ParseInt(field);
if (!std::getline(field_stream, field, kDelimiter)) {
return Status::InvalidArgument("unable to parse the specified CF option " +
name);
}
compression_opts.level = ParseInt(field);
if (!std::getline(field_stream, field, kDelimiter)) {
return Status::InvalidArgument("unable to parse the specified CF option " +
name);
}
compression_opts.strategy = ParseInt(field);
// max_dict_bytes is optional for backwards compatibility
if (!field_stream.eof()) {
if (!std::getline(field_stream, field, kDelimiter)) {
return Status::InvalidArgument(
"unable to parse the specified CF option " + name);
}
compression_opts.max_dict_bytes = ParseInt(field);
}
// zstd_max_train_bytes is optional for backwards compatibility
if (!field_stream.eof()) {
if (!std::getline(field_stream, field, kDelimiter)) {
return Status::InvalidArgument(
"unable to parse the specified CF option " + name);
}
compression_opts.zstd_max_train_bytes = ParseInt(field);
}
// parallel_threads is optional for backwards compatibility
if (!field_stream.eof()) {
if (!std::getline(field_stream, field, kDelimiter)) {
return Status::InvalidArgument(
"unable to parse the specified CF option " + name);
}
// Since parallel_threads comes before enabled but was added optionally
// later, we need to check if this is the final token (meaning it is the
// enabled bit), or if there are more tokens (meaning this one is
// parallel_threads).
if (!field_stream.eof()) {
compression_opts.parallel_threads = ParseInt(field);
} else {
// parallel_threads is not serialized with this format, but enabled is
compression_opts.enabled = ParseBoolean("", field);
}
}
// enabled is optional for backwards compatibility
if (!field_stream.eof()) {
if (!std::getline(field_stream, field, kDelimiter)) {
return Status::InvalidArgument(
"unable to parse the specified CF option " + name);
}
compression_opts.enabled = ParseBoolean("", field);
}
// max_dict_buffer_bytes is optional for backwards compatibility
if (!field_stream.eof()) {
if (!std::getline(field_stream, field, kDelimiter)) {
return Status::InvalidArgument(
"unable to parse the specified CF option " + name);
}
compression_opts.max_dict_buffer_bytes = ParseUint64(field);
}
// use_zstd_dict_trainer is optional for backwards compatibility
if (!field_stream.eof()) {
if (!std::getline(field_stream, field, kDelimiter)) {
return Status::InvalidArgument(
"unable to parse the specified CF option " + name);
}
compression_opts.use_zstd_dict_trainer = ParseBoolean("", field);
}
if (!field_stream.eof()) {
return Status::InvalidArgument("unable to parse the specified CF option " +
name);
}
return Status::OK();
}
const std::string kOptNameBMCompOpts = "bottommost_compression_opts";
const std::string kOptNameCompOpts = "compression_opts";
// OptionTypeInfo map for CompressionOptions
static std::unordered_map<std::string, OptionTypeInfo>
compression_options_type_info = {
{"window_bits",
{offsetof(struct CompressionOptions, window_bits), OptionType::kInt,
OptionVerificationType::kNormal, OptionTypeFlags::kMutable}},
{"level",
{offsetof(struct CompressionOptions, level), OptionType::kInt,
OptionVerificationType::kNormal, OptionTypeFlags::kMutable}},
{"strategy",
{offsetof(struct CompressionOptions, strategy), OptionType::kInt,
OptionVerificationType::kNormal, OptionTypeFlags::kMutable}},
{"max_dict_bytes",
{offsetof(struct CompressionOptions, max_dict_bytes), OptionType::kInt,
OptionVerificationType::kNormal, OptionTypeFlags::kMutable}},
{"zstd_max_train_bytes",
{offsetof(struct CompressionOptions, zstd_max_train_bytes),
OptionType::kUInt32T, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"parallel_threads",
{offsetof(struct CompressionOptions, parallel_threads),
OptionType::kUInt32T, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"enabled",
{offsetof(struct CompressionOptions, enabled), OptionType::kBoolean,
OptionVerificationType::kNormal, OptionTypeFlags::kMutable}},
{"max_dict_buffer_bytes",
{offsetof(struct CompressionOptions, max_dict_buffer_bytes),
OptionType::kUInt64T, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"use_zstd_dict_trainer",
{offsetof(struct CompressionOptions, use_zstd_dict_trainer),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
};
static std::unordered_map<std::string, OptionTypeInfo>
fifo_compaction_options_type_info = {
{"max_table_files_size",
{offsetof(struct CompactionOptionsFIFO, max_table_files_size),
OptionType::kUInt64T, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"age_for_warm",
{offsetof(struct CompactionOptionsFIFO, age_for_warm),
OptionType::kUInt64T, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"ttl",
{0, OptionType::kUInt64T, OptionVerificationType::kDeprecated,
OptionTypeFlags::kNone}},
{"allow_compaction",
{offsetof(struct CompactionOptionsFIFO, allow_compaction),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
};
static std::unordered_map<std::string, OptionTypeInfo>
universal_compaction_options_type_info = {
{"size_ratio",
{offsetof(class CompactionOptionsUniversal, size_ratio),
OptionType::kUInt, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"min_merge_width",
{offsetof(class CompactionOptionsUniversal, min_merge_width),
OptionType::kUInt, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"max_merge_width",
{offsetof(class CompactionOptionsUniversal, max_merge_width),
OptionType::kUInt, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"max_size_amplification_percent",
{offsetof(class CompactionOptionsUniversal,
max_size_amplification_percent),
OptionType::kUInt, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"compression_size_percent",
{offsetof(class CompactionOptionsUniversal, compression_size_percent),
OptionType::kInt, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"stop_style",
{offsetof(class CompactionOptionsUniversal, stop_style),
OptionType::kCompactionStopStyle, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"incremental",
{offsetof(class CompactionOptionsUniversal, incremental),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"allow_trivial_move",
{offsetof(class CompactionOptionsUniversal, allow_trivial_move),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}}};
static std::unordered_map<std::string, OptionTypeInfo>
cf_mutable_options_type_info = {
{"report_bg_io_stats",
{offsetof(struct MutableCFOptions, report_bg_io_stats),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"disable_auto_compactions",
{offsetof(struct MutableCFOptions, disable_auto_compactions),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"filter_deletes",
{0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
OptionTypeFlags::kMutable}},
{"check_flush_compaction_key_order",
{offsetof(struct MutableCFOptions, check_flush_compaction_key_order),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"paranoid_file_checks",
{offsetof(struct MutableCFOptions, paranoid_file_checks),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"verify_checksums_in_compaction",
{0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
OptionTypeFlags::kMutable}},
{"soft_pending_compaction_bytes_limit",
{offsetof(struct MutableCFOptions,
soft_pending_compaction_bytes_limit),
OptionType::kUInt64T, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"hard_pending_compaction_bytes_limit",
{offsetof(struct MutableCFOptions,
hard_pending_compaction_bytes_limit),
OptionType::kUInt64T, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"hard_rate_limit",
{0, OptionType::kDouble, OptionVerificationType::kDeprecated,
OptionTypeFlags::kMutable}},
{"soft_rate_limit",
{0, OptionType::kDouble, OptionVerificationType::kDeprecated,
OptionTypeFlags::kMutable}},
{"max_compaction_bytes",
{offsetof(struct MutableCFOptions, max_compaction_bytes),
OptionType::kUInt64T, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"expanded_compaction_factor",
{0, OptionType::kInt, OptionVerificationType::kDeprecated,
OptionTypeFlags::kMutable}},
{"level0_file_num_compaction_trigger",
{offsetof(struct MutableCFOptions, level0_file_num_compaction_trigger),
OptionType::kInt, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"level0_slowdown_writes_trigger",
{offsetof(struct MutableCFOptions, level0_slowdown_writes_trigger),
OptionType::kInt, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"level0_stop_writes_trigger",
{offsetof(struct MutableCFOptions, level0_stop_writes_trigger),
OptionType::kInt, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"max_grandparent_overlap_factor",
{0, OptionType::kInt, OptionVerificationType::kDeprecated,
OptionTypeFlags::kMutable}},
{"max_write_buffer_number",
{offsetof(struct MutableCFOptions, max_write_buffer_number),
OptionType::kInt, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"source_compaction_factor",
{0, OptionType::kInt, OptionVerificationType::kDeprecated,
OptionTypeFlags::kMutable}},
{"target_file_size_multiplier",
{offsetof(struct MutableCFOptions, target_file_size_multiplier),
OptionType::kInt, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"arena_block_size",
{offsetof(struct MutableCFOptions, arena_block_size),
OptionType::kSizeT, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"inplace_update_num_locks",
{offsetof(struct MutableCFOptions, inplace_update_num_locks),
OptionType::kSizeT, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"max_successive_merges",
{offsetof(struct MutableCFOptions, max_successive_merges),
OptionType::kSizeT, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"memtable_huge_page_size",
{offsetof(struct MutableCFOptions, memtable_huge_page_size),
OptionType::kSizeT, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"memtable_prefix_bloom_huge_page_tlb_size",
{0, OptionType::kSizeT, OptionVerificationType::kDeprecated,
OptionTypeFlags::kMutable}},
{"write_buffer_size",
{offsetof(struct MutableCFOptions, write_buffer_size),
OptionType::kSizeT, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"memtable_prefix_bloom_bits",
{0, OptionType::kUInt32T, OptionVerificationType::kDeprecated,
OptionTypeFlags::kMutable}},
{"memtable_prefix_bloom_size_ratio",
{offsetof(struct MutableCFOptions, memtable_prefix_bloom_size_ratio),
OptionType::kDouble, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"memtable_prefix_bloom_probes",
{0, OptionType::kUInt32T, OptionVerificationType::kDeprecated,
OptionTypeFlags::kMutable}},
{"memtable_whole_key_filtering",
{offsetof(struct MutableCFOptions, memtable_whole_key_filtering),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"min_partial_merge_operands",
{0, OptionType::kUInt32T, OptionVerificationType::kDeprecated,
OptionTypeFlags::kMutable}},
{"max_bytes_for_level_base",
{offsetof(struct MutableCFOptions, max_bytes_for_level_base),
OptionType::kUInt64T, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"snap_refresh_nanos",
{0, OptionType::kUInt64T, OptionVerificationType::kDeprecated,
OptionTypeFlags::kMutable}},
{"max_bytes_for_level_multiplier",
{offsetof(struct MutableCFOptions, max_bytes_for_level_multiplier),
OptionType::kDouble, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"max_bytes_for_level_multiplier_additional",
OptionTypeInfo::Vector<int>(
offsetof(struct MutableCFOptions,
max_bytes_for_level_multiplier_additional),
OptionVerificationType::kNormal, OptionTypeFlags::kMutable,
{0, OptionType::kInt})},
{"max_sequential_skip_in_iterations",
{offsetof(struct MutableCFOptions, max_sequential_skip_in_iterations),
OptionType::kUInt64T, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"target_file_size_base",
{offsetof(struct MutableCFOptions, target_file_size_base),
OptionType::kUInt64T, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"compression",
{offsetof(struct MutableCFOptions, compression),
OptionType::kCompressionType, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"prefix_extractor",
OptionTypeInfo::AsCustomSharedPtr<const SliceTransform>(
offsetof(struct MutableCFOptions, prefix_extractor),
OptionVerificationType::kByNameAllowNull,
(OptionTypeFlags::kMutable | OptionTypeFlags::kAllowNull))},
{"compaction_options_fifo",
OptionTypeInfo::Struct(
"compaction_options_fifo", &fifo_compaction_options_type_info,
offsetof(struct MutableCFOptions, compaction_options_fifo),
OptionVerificationType::kNormal, OptionTypeFlags::kMutable)
.SetParseFunc([](const ConfigOptions& opts,
const std::string& name, const std::string& value,
void* addr) {
// This is to handle backward compatibility, where
// compaction_options_fifo could be assigned a single scalar
// value, say, like "23", which would be assigned to
// max_table_files_size.
if (name == "compaction_options_fifo" &&
value.find("=") == std::string::npos) {
// Old format. Parse just a single uint64_t value.
auto options = static_cast<CompactionOptionsFIFO*>(addr);
options->max_table_files_size = ParseUint64(value);
return Status::OK();
} else {
return OptionTypeInfo::ParseStruct(
opts, "compaction_options_fifo",
&fifo_compaction_options_type_info, name, value, addr);
}
})},
{"compaction_options_universal",
OptionTypeInfo::Struct(
"compaction_options_universal",
&universal_compaction_options_type_info,
offsetof(struct MutableCFOptions, compaction_options_universal),
OptionVerificationType::kNormal, OptionTypeFlags::kMutable)},
{"ttl",
{offsetof(struct MutableCFOptions, ttl), OptionType::kUInt64T,
OptionVerificationType::kNormal, OptionTypeFlags::kMutable}},
{"periodic_compaction_seconds",
{offsetof(struct MutableCFOptions, periodic_compaction_seconds),
OptionType::kUInt64T, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"bottommost_temperature",
{offsetof(struct MutableCFOptions, bottommost_temperature),
OptionType::kTemperature, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"enable_blob_files",
{offsetof(struct MutableCFOptions, enable_blob_files),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"min_blob_size",
{offsetof(struct MutableCFOptions, min_blob_size),
OptionType::kUInt64T, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"blob_file_size",
{offsetof(struct MutableCFOptions, blob_file_size),
OptionType::kUInt64T, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"blob_compression_type",
{offsetof(struct MutableCFOptions, blob_compression_type),
OptionType::kCompressionType, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"enable_blob_garbage_collection",
{offsetof(struct MutableCFOptions, enable_blob_garbage_collection),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"blob_garbage_collection_age_cutoff",
{offsetof(struct MutableCFOptions, blob_garbage_collection_age_cutoff),
OptionType::kDouble, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"blob_garbage_collection_force_threshold",
{offsetof(struct MutableCFOptions,
blob_garbage_collection_force_threshold),
OptionType::kDouble, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"blob_compaction_readahead_size",
{offsetof(struct MutableCFOptions, blob_compaction_readahead_size),
OptionType::kUInt64T, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"sample_for_compression",
{offsetof(struct MutableCFOptions, sample_for_compression),
OptionType::kUInt64T, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"bottommost_compression",
{offsetof(struct MutableCFOptions, bottommost_compression),
OptionType::kCompressionType, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"compression_per_level",
OptionTypeInfo::Vector<CompressionType>(
offsetof(struct MutableCFOptions, compression_per_level),
OptionVerificationType::kNormal, OptionTypeFlags::kMutable,
{0, OptionType::kCompressionType})},
{kOptNameCompOpts,
OptionTypeInfo::Struct(
kOptNameCompOpts, &compression_options_type_info,
offsetof(struct MutableCFOptions, compression_opts),
OptionVerificationType::kNormal,
(OptionTypeFlags::kMutable | OptionTypeFlags::kCompareNever),
[](const ConfigOptions& opts, const std::string& name,
const std::string& value, void* addr) {
// This is to handle backward compatibility, where
// compression_options was a ":" separated list.
if (name == kOptNameCompOpts &&
value.find("=") == std::string::npos) {
auto* compression = static_cast<CompressionOptions*>(addr);
return ParseCompressionOptions(value, name, *compression);
} else {
return OptionTypeInfo::ParseStruct(
opts, kOptNameCompOpts, &compression_options_type_info,
name, value, addr);
}
})},
{kOptNameBMCompOpts,
OptionTypeInfo::Struct(
kOptNameBMCompOpts, &compression_options_type_info,
offsetof(struct MutableCFOptions, bottommost_compression_opts),
OptionVerificationType::kNormal,
(OptionTypeFlags::kMutable | OptionTypeFlags::kCompareNever),
[](const ConfigOptions& opts, const std::string& name,
const std::string& value, void* addr) {
// This is to handle backward compatibility, where
// compression_options was a ":" separated list.
if (name == kOptNameBMCompOpts &&
value.find("=") == std::string::npos) {
auto* compression = static_cast<CompressionOptions*>(addr);
return ParseCompressionOptions(value, name, *compression);
} else {
return OptionTypeInfo::ParseStruct(
opts, kOptNameBMCompOpts, &compression_options_type_info,
name, value, addr);
}
})},
// End special case properties
};
static std::unordered_map<std::string, OptionTypeInfo>
cf_immutable_options_type_info = {
/* not yet supported
CompressionOptions compression_opts;
TablePropertiesCollectorFactories table_properties_collector_factories;
using TablePropertiesCollectorFactories =
std::vector<std::shared_ptr<TablePropertiesCollectorFactory>>;
UpdateStatus (*inplace_callback)(char* existing_value,
uint34_t* existing_value_size,
Slice delta_value,
std::string* merged_value);
std::vector<DbPath> cf_paths;
*/
{"compaction_measure_io_stats",
{0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
OptionTypeFlags::kNone}},
{"purge_redundant_kvs_while_flush",
{0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
OptionTypeFlags::kNone}},
{"inplace_update_support",
{offsetof(struct ImmutableCFOptions, inplace_update_support),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone}},
{"level_compaction_dynamic_level_bytes",
{offsetof(struct ImmutableCFOptions,
level_compaction_dynamic_level_bytes),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone}},
{"optimize_filters_for_hits",
{offsetof(struct ImmutableCFOptions, optimize_filters_for_hits),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone}},
{"force_consistency_checks",
{offsetof(struct ImmutableCFOptions, force_consistency_checks),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone}},
// Need to keep this around to be able to read old OPTIONS files.
{"max_mem_compaction_level",
{0, OptionType::kInt, OptionVerificationType::kDeprecated,
OptionTypeFlags::kNone}},
{"max_write_buffer_number_to_maintain",
{offsetof(struct ImmutableCFOptions,
max_write_buffer_number_to_maintain),
OptionType::kInt, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}},
{"max_write_buffer_size_to_maintain",
{offsetof(struct ImmutableCFOptions,
max_write_buffer_size_to_maintain),
OptionType::kInt64T, OptionVerificationType::kNormal,
OptionTypeFlags::kNone}},
{"min_write_buffer_number_to_merge",
{offsetof(struct ImmutableCFOptions, min_write_buffer_number_to_merge),
OptionType::kInt, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}},
{"num_levels",
{offsetof(struct ImmutableCFOptions, num_levels), OptionType::kInt,
OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
{"bloom_locality",
{offsetof(struct ImmutableCFOptions, bloom_locality),
OptionType::kUInt32T, OptionVerificationType::kNormal,
OptionTypeFlags::kNone}},
{"rate_limit_delay_max_milliseconds",
{0, OptionType::kUInt, OptionVerificationType::kDeprecated,
OptionTypeFlags::kNone}},
{"comparator",
OptionTypeInfo::AsCustomRawPtr<const Comparator>(
offsetof(struct ImmutableCFOptions, user_comparator),
OptionVerificationType::kByName, OptionTypeFlags::kCompareLoose)
.SetSerializeFunc(
// Serializes a Comparator
[](const ConfigOptions& opts, const std::string&,
const void* addr, std::string* value) {
// it's a const pointer of const Comparator*
const auto* ptr =
static_cast<const Comparator* const*>(addr);
// Since the user-specified comparator will be wrapped by
// InternalKeyComparator, we should persist the
// user-specified one instead of InternalKeyComparator.
if (*ptr == nullptr) {
*value = kNullptrString;
} else if (opts.mutable_options_only) {
*value = "";
} else {
const Comparator* root_comp = (*ptr)->GetRootComparator();
if (root_comp == nullptr) {
root_comp = (*ptr);
}
*value = root_comp->ToString(opts);
}
return Status::OK();
})},
{"memtable_insert_with_hint_prefix_extractor",
OptionTypeInfo::AsCustomSharedPtr<const SliceTransform>(
offsetof(struct ImmutableCFOptions,
memtable_insert_with_hint_prefix_extractor),
OptionVerificationType::kByNameAllowNull, OptionTypeFlags::kNone)},
{"memtable_factory",
{offsetof(struct ImmutableCFOptions, memtable_factory),
OptionType::kCustomizable, OptionVerificationType::kByName,
OptionTypeFlags::kShared,
[](const ConfigOptions& opts, const std::string&,
const std::string& value, void* addr) {
std::unique_ptr<MemTableRepFactory> factory;
auto* shared =
static_cast<std::shared_ptr<MemTableRepFactory>*>(addr);
Status s =
MemTableRepFactory::CreateFromString(opts, value, shared);
return s;
}}},
{"memtable",
{offsetof(struct ImmutableCFOptions, memtable_factory),
OptionType::kCustomizable, OptionVerificationType::kAlias,
OptionTypeFlags::kShared,
[](const ConfigOptions& opts, const std::string&,
const std::string& value, void* addr) {
std::unique_ptr<MemTableRepFactory> factory;
auto* shared =
static_cast<std::shared_ptr<MemTableRepFactory>*>(addr);
Status s =
MemTableRepFactory::CreateFromString(opts, value, shared);
return s;
}}},
{"table_factory",
OptionTypeInfo::AsCustomSharedPtr<TableFactory>(
offsetof(struct ImmutableCFOptions, table_factory),
OptionVerificationType::kByName,
(OptionTypeFlags::kCompareLoose |
OptionTypeFlags::kStringNameOnly |
OptionTypeFlags::kDontPrepare))},
{"block_based_table_factory",
{offsetof(struct ImmutableCFOptions, table_factory),
OptionType::kCustomizable, OptionVerificationType::kAlias,
OptionTypeFlags::kShared | OptionTypeFlags::kCompareLoose,
// Parses the input value and creates a BlockBasedTableFactory
[](const ConfigOptions& opts, const std::string& name,
const std::string& value, void* addr) {
BlockBasedTableOptions* old_opts = nullptr;
auto table_factory =
static_cast<std::shared_ptr<TableFactory>*>(addr);
if (table_factory->get() != nullptr) {
old_opts =
table_factory->get()->GetOptions<BlockBasedTableOptions>();
}
if (name == "block_based_table_factory") {
std::unique_ptr<TableFactory> new_factory;
if (old_opts != nullptr) {
new_factory.reset(NewBlockBasedTableFactory(*old_opts));
} else {
new_factory.reset(NewBlockBasedTableFactory());
}
Status s = new_factory->ConfigureFromString(opts, value);
if (s.ok()) {
table_factory->reset(new_factory.release());
}
return s;
} else if (old_opts != nullptr) {
return table_factory->get()->ConfigureOption(opts, name, value);
} else {
return Status::NotFound("Mismatched table option: ", name);
}
}}},
{"plain_table_factory",
{offsetof(struct ImmutableCFOptions, table_factory),
OptionType::kCustomizable, OptionVerificationType::kAlias,
OptionTypeFlags::kShared | OptionTypeFlags::kCompareLoose,
// Parses the input value and creates a PlainTableFactory
[](const ConfigOptions& opts, const std::string& name,
const std::string& value, void* addr) {
PlainTableOptions* old_opts = nullptr;
auto table_factory =
static_cast<std::shared_ptr<TableFactory>*>(addr);
if (table_factory->get() != nullptr) {
old_opts = table_factory->get()->GetOptions<PlainTableOptions>();
}
if (name == "plain_table_factory") {
std::unique_ptr<TableFactory> new_factory;
if (old_opts != nullptr) {
new_factory.reset(NewPlainTableFactory(*old_opts));
} else {
new_factory.reset(NewPlainTableFactory());
}
Status s = new_factory->ConfigureFromString(opts, value);
if (s.ok()) {
table_factory->reset(new_factory.release());
}
return s;
} else if (old_opts != nullptr) {
return table_factory->get()->ConfigureOption(opts, name, value);
} else {
return Status::NotFound("Mismatched table option: ", name);
}
}}},
{"table_properties_collectors",
OptionTypeInfo::Vector<
std::shared_ptr<TablePropertiesCollectorFactory>>(
offsetof(struct ImmutableCFOptions,
table_properties_collector_factories),
OptionVerificationType::kByName, OptionTypeFlags::kNone,
OptionTypeInfo::AsCustomSharedPtr<TablePropertiesCollectorFactory>(
0, OptionVerificationType::kByName, OptionTypeFlags::kNone))},
{"compaction_filter",
OptionTypeInfo::AsCustomRawPtr<const CompactionFilter>(
offsetof(struct ImmutableCFOptions, compaction_filter),
OptionVerificationType::kByName, OptionTypeFlags::kAllowNull)},
{"compaction_filter_factory",
OptionTypeInfo::AsCustomSharedPtr<CompactionFilterFactory>(
offsetof(struct ImmutableCFOptions, compaction_filter_factory),
OptionVerificationType::kByName, OptionTypeFlags::kAllowNull)},
{"merge_operator",
OptionTypeInfo::AsCustomSharedPtr<MergeOperator>(
offsetof(struct ImmutableCFOptions, merge_operator),
OptionVerificationType::kByNameAllowFromNull,
OptionTypeFlags::kCompareLoose | OptionTypeFlags::kAllowNull)},
{"compaction_style",
{offsetof(struct ImmutableCFOptions, compaction_style),
OptionType::kCompactionStyle, OptionVerificationType::kNormal,
OptionTypeFlags::kNone}},
{"compaction_pri",
{offsetof(struct ImmutableCFOptions, compaction_pri),
OptionType::kCompactionPri, OptionVerificationType::kNormal,
OptionTypeFlags::kNone}},
{"sst_partitioner_factory",
OptionTypeInfo::AsCustomSharedPtr<SstPartitionerFactory>(
offsetof(struct ImmutableCFOptions, sst_partitioner_factory),
OptionVerificationType::kByName, OptionTypeFlags::kAllowNull)},
};
const std::string OptionsHelper::kCFOptionsName = "ColumnFamilyOptions";
class ConfigurableMutableCFOptions : public Configurable {
public:
explicit ConfigurableMutableCFOptions(const MutableCFOptions& mcf) {
mutable_ = mcf;
RegisterOptions(&mutable_, &cf_mutable_options_type_info);
}
protected:
MutableCFOptions mutable_;
};
class ConfigurableCFOptions : public ConfigurableMutableCFOptions {
public:
ConfigurableCFOptions(const ColumnFamilyOptions& opts,
const std::unordered_map<std::string, std::string>* map)
: ConfigurableMutableCFOptions(MutableCFOptions(opts)),
immutable_(opts),
cf_options_(opts),
opt_map_(map) {
RegisterOptions(&immutable_, &cf_immutable_options_type_info);
}
protected:
Status ConfigureOptions(
const ConfigOptions& config_options,
const std::unordered_map<std::string, std::string>& opts_map,
std::unordered_map<std::string, std::string>* unused) override {
Status s = Configurable::ConfigureOptions(config_options, opts_map, unused);
if (s.ok()) {
UpdateColumnFamilyOptions(mutable_, &cf_options_);
UpdateColumnFamilyOptions(immutable_, &cf_options_);
s = PrepareOptions(config_options);
}
return s;
}
virtual const void* GetOptionsPtr(const std::string& name) const override {
if (name == OptionsHelper::kCFOptionsName) {
return &cf_options_;
} else {
return ConfigurableMutableCFOptions::GetOptionsPtr(name);
}
}
bool OptionsAreEqual(const ConfigOptions& config_options,
const OptionTypeInfo& opt_info,
const std::string& opt_name, const void* const this_ptr,
const void* const that_ptr,
std::string* mismatch) const override {
bool equals = opt_info.AreEqual(config_options, opt_name, this_ptr,
that_ptr, mismatch);
if (!equals && opt_info.IsByName()) {
if (opt_map_ == nullptr) {
equals = true;
} else {
const auto& iter = opt_map_->find(opt_name);
if (iter == opt_map_->end()) {
equals = true;
} else {
equals = opt_info.AreEqualByName(config_options, opt_name, this_ptr,
iter->second);
}
}
if (equals) { // False alarm, clear mismatch
*mismatch = "";
}
}
if (equals && opt_info.IsConfigurable() && opt_map_ != nullptr) {
const auto* this_config = opt_info.AsRawPointer<Configurable>(this_ptr);
if (this_config == nullptr) {
const auto& iter = opt_map_->find(opt_name);
// If the name exists in the map and is not empty/null,
// then the this_config should be set.
if (iter != opt_map_->end() && !iter->second.empty() &&
iter->second != kNullptrString) {
*mismatch = opt_name;
equals = false;
}
}
}
return equals;
}
private:
ImmutableCFOptions immutable_;
ColumnFamilyOptions cf_options_;
const std::unordered_map<std::string, std::string>* opt_map_;
};
std::unique_ptr<Configurable> CFOptionsAsConfigurable(
const MutableCFOptions& opts) {
std::unique_ptr<Configurable> ptr(new ConfigurableMutableCFOptions(opts));
return ptr;
}
std::unique_ptr<Configurable> CFOptionsAsConfigurable(
const ColumnFamilyOptions& opts,
const std::unordered_map<std::string, std::string>* opt_map) {
std::unique_ptr<Configurable> ptr(new ConfigurableCFOptions(opts, opt_map));
return ptr;
}
#endif // ROCKSDB_LITE
ImmutableCFOptions::ImmutableCFOptions() : ImmutableCFOptions(Options()) {}
ImmutableCFOptions::ImmutableCFOptions(const ColumnFamilyOptions& cf_options)
: compaction_style(cf_options.compaction_style),
compaction_pri(cf_options.compaction_pri),
user_comparator(cf_options.comparator),
internal_comparator(InternalKeyComparator(cf_options.comparator)),
merge_operator(cf_options.merge_operator),
compaction_filter(cf_options.compaction_filter),
compaction_filter_factory(cf_options.compaction_filter_factory),
min_write_buffer_number_to_merge(
cf_options.min_write_buffer_number_to_merge),
max_write_buffer_number_to_maintain(
cf_options.max_write_buffer_number_to_maintain),
max_write_buffer_size_to_maintain(
cf_options.max_write_buffer_size_to_maintain),
inplace_update_support(cf_options.inplace_update_support),
inplace_callback(cf_options.inplace_callback),
memtable_factory(cf_options.memtable_factory),
table_factory(cf_options.table_factory),
table_properties_collector_factories(
cf_options.table_properties_collector_factories),
bloom_locality(cf_options.bloom_locality),
level_compaction_dynamic_level_bytes(
cf_options.level_compaction_dynamic_level_bytes),
num_levels(cf_options.num_levels),
optimize_filters_for_hits(cf_options.optimize_filters_for_hits),
force_consistency_checks(cf_options.force_consistency_checks),
memtable_insert_with_hint_prefix_extractor(
cf_options.memtable_insert_with_hint_prefix_extractor),
cf_paths(cf_options.cf_paths),
compaction_thread_limiter(cf_options.compaction_thread_limiter),
sst_partitioner_factory(cf_options.sst_partitioner_factory) {}
ImmutableOptions::ImmutableOptions() : ImmutableOptions(Options()) {}
ImmutableOptions::ImmutableOptions(const Options& options)
: ImmutableOptions(options, options) {}
ImmutableOptions::ImmutableOptions(const DBOptions& db_options,
const ColumnFamilyOptions& cf_options)
: ImmutableDBOptions(db_options), ImmutableCFOptions(cf_options) {}
ImmutableOptions::ImmutableOptions(const DBOptions& db_options,
const ImmutableCFOptions& cf_options)
: ImmutableDBOptions(db_options), ImmutableCFOptions(cf_options) {}
ImmutableOptions::ImmutableOptions(const ImmutableDBOptions& db_options,
const ColumnFamilyOptions& cf_options)
: ImmutableDBOptions(db_options), ImmutableCFOptions(cf_options) {}
ImmutableOptions::ImmutableOptions(const ImmutableDBOptions& db_options,
const ImmutableCFOptions& cf_options)
: ImmutableDBOptions(db_options), ImmutableCFOptions(cf_options) {}
// Multiple two operands. If they overflow, return op1.
uint64_t MultiplyCheckOverflow(uint64_t op1, double op2) {
if (op1 == 0 || op2 <= 0) {
return 0;
}
if (std::numeric_limits<uint64_t>::max() / op1 < op2) {
return op1;
}
return static_cast<uint64_t>(op1 * op2);
}
// when level_compaction_dynamic_level_bytes is true and leveled compaction
// is used, the base level is not always L1, so precomupted max_file_size can
// no longer be used. Recompute file_size_for_level from base level.
uint64_t MaxFileSizeForLevel(const MutableCFOptions& cf_options,
int level, CompactionStyle compaction_style, int base_level,
bool level_compaction_dynamic_level_bytes) {
if (!level_compaction_dynamic_level_bytes || level < base_level ||
compaction_style != kCompactionStyleLevel) {
assert(level >= 0);
assert(level < (int)cf_options.max_file_size.size());
return cf_options.max_file_size[level];
} else {
assert(level >= 0 && base_level >= 0);
assert(level - base_level < (int)cf_options.max_file_size.size());
return cf_options.max_file_size[level - base_level];
}
}
size_t MaxFileSizeForL0MetaPin(const MutableCFOptions& cf_options) {
// We do not want to pin meta-blocks that almost certainly came from intra-L0
// or a former larger `write_buffer_size` value to avoid surprising users with
// pinned memory usage. We use a factor of 1.5 to account for overhead
// introduced during flush in most cases.
if (std::numeric_limits<size_t>::max() / 3 <
cf_options.write_buffer_size / 2) {
return std::numeric_limits<size_t>::max();
}
return cf_options.write_buffer_size / 2 * 3;
}
void MutableCFOptions::RefreshDerivedOptions(int num_levels,
CompactionStyle compaction_style) {
max_file_size.resize(num_levels);
for (int i = 0; i < num_levels; ++i) {
if (i == 0 && compaction_style == kCompactionStyleUniversal) {
max_file_size[i] = ULLONG_MAX;
} else if (i > 1) {
max_file_size[i] = MultiplyCheckOverflow(max_file_size[i - 1],
target_file_size_multiplier);
} else {
max_file_size[i] = target_file_size_base;
}
}
}
void MutableCFOptions::Dump(Logger* log) const {
// Memtable related options
ROCKS_LOG_INFO(log,
" write_buffer_size: %" ROCKSDB_PRIszt,
write_buffer_size);
ROCKS_LOG_INFO(log, " max_write_buffer_number: %d",
max_write_buffer_number);
ROCKS_LOG_INFO(log,
" arena_block_size: %" ROCKSDB_PRIszt,
arena_block_size);
ROCKS_LOG_INFO(log, " memtable_prefix_bloom_ratio: %f",
memtable_prefix_bloom_size_ratio);
ROCKS_LOG_INFO(log, " memtable_whole_key_filtering: %d",
memtable_whole_key_filtering);
ROCKS_LOG_INFO(log,
" memtable_huge_page_size: %" ROCKSDB_PRIszt,
memtable_huge_page_size);
ROCKS_LOG_INFO(log,
" max_successive_merges: %" ROCKSDB_PRIszt,
max_successive_merges);
ROCKS_LOG_INFO(log,
" inplace_update_num_locks: %" ROCKSDB_PRIszt,
inplace_update_num_locks);
ROCKS_LOG_INFO(log, " prefix_extractor: %s",
prefix_extractor == nullptr
? "nullptr"
: prefix_extractor->GetId().c_str());
ROCKS_LOG_INFO(log, " disable_auto_compactions: %d",
disable_auto_compactions);
ROCKS_LOG_INFO(log, " soft_pending_compaction_bytes_limit: %" PRIu64,
soft_pending_compaction_bytes_limit);
ROCKS_LOG_INFO(log, " hard_pending_compaction_bytes_limit: %" PRIu64,
hard_pending_compaction_bytes_limit);
ROCKS_LOG_INFO(log, " level0_file_num_compaction_trigger: %d",
level0_file_num_compaction_trigger);
ROCKS_LOG_INFO(log, " level0_slowdown_writes_trigger: %d",
level0_slowdown_writes_trigger);
ROCKS_LOG_INFO(log, " level0_stop_writes_trigger: %d",
level0_stop_writes_trigger);
ROCKS_LOG_INFO(log, " max_compaction_bytes: %" PRIu64,
max_compaction_bytes);
ROCKS_LOG_INFO(log, " target_file_size_base: %" PRIu64,
target_file_size_base);
ROCKS_LOG_INFO(log, " target_file_size_multiplier: %d",
target_file_size_multiplier);
ROCKS_LOG_INFO(log, " max_bytes_for_level_base: %" PRIu64,
max_bytes_for_level_base);
ROCKS_LOG_INFO(log, " max_bytes_for_level_multiplier: %f",
max_bytes_for_level_multiplier);
ROCKS_LOG_INFO(log, " ttl: %" PRIu64,
ttl);
ROCKS_LOG_INFO(log, " periodic_compaction_seconds: %" PRIu64,
periodic_compaction_seconds);
std::string result;
char buf[10];
for (const auto m : max_bytes_for_level_multiplier_additional) {
snprintf(buf, sizeof(buf), "%d, ", m);
result += buf;
}
if (result.size() >= 2) {
result.resize(result.size() - 2);
} else {
result = "";
}
ROCKS_LOG_INFO(log, "max_bytes_for_level_multiplier_additional: %s",
result.c_str());
ROCKS_LOG_INFO(log, " max_sequential_skip_in_iterations: %" PRIu64,
max_sequential_skip_in_iterations);
ROCKS_LOG_INFO(log, " check_flush_compaction_key_order: %d",
check_flush_compaction_key_order);
ROCKS_LOG_INFO(log, " paranoid_file_checks: %d",
paranoid_file_checks);
ROCKS_LOG_INFO(log, " report_bg_io_stats: %d",
report_bg_io_stats);
ROCKS_LOG_INFO(log, " compression: %d",
static_cast<int>(compression));
// Universal Compaction Options
ROCKS_LOG_INFO(log, "compaction_options_universal.size_ratio : %d",
compaction_options_universal.size_ratio);
ROCKS_LOG_INFO(log, "compaction_options_universal.min_merge_width : %d",
compaction_options_universal.min_merge_width);
ROCKS_LOG_INFO(log, "compaction_options_universal.max_merge_width : %d",
compaction_options_universal.max_merge_width);
ROCKS_LOG_INFO(
log, "compaction_options_universal.max_size_amplification_percent : %d",
compaction_options_universal.max_size_amplification_percent);
ROCKS_LOG_INFO(log,
"compaction_options_universal.compression_size_percent : %d",
compaction_options_universal.compression_size_percent);
ROCKS_LOG_INFO(log, "compaction_options_universal.stop_style : %d",
compaction_options_universal.stop_style);
ROCKS_LOG_INFO(
log, "compaction_options_universal.allow_trivial_move : %d",
static_cast<int>(compaction_options_universal.allow_trivial_move));
ROCKS_LOG_INFO(log, "compaction_options_universal.incremental : %d",
static_cast<int>(compaction_options_universal.incremental));
// FIFO Compaction Options
ROCKS_LOG_INFO(log, "compaction_options_fifo.max_table_files_size : %" PRIu64,
compaction_options_fifo.max_table_files_size);
ROCKS_LOG_INFO(log, "compaction_options_fifo.allow_compaction : %d",
compaction_options_fifo.allow_compaction);
// Blob file related options
ROCKS_LOG_INFO(log, " enable_blob_files: %s",
enable_blob_files ? "true" : "false");
ROCKS_LOG_INFO(log, " min_blob_size: %" PRIu64,
min_blob_size);
ROCKS_LOG_INFO(log, " blob_file_size: %" PRIu64,
blob_file_size);
ROCKS_LOG_INFO(log, " blob_compression_type: %s",
CompressionTypeToString(blob_compression_type).c_str());
ROCKS_LOG_INFO(log, " enable_blob_garbage_collection: %s",
enable_blob_garbage_collection ? "true" : "false");
ROCKS_LOG_INFO(log, " blob_garbage_collection_age_cutoff: %f",
blob_garbage_collection_age_cutoff);
ROCKS_LOG_INFO(log, " blob_garbage_collection_force_threshold: %f",
blob_garbage_collection_force_threshold);
ROCKS_LOG_INFO(log, " blob_compaction_readahead_size: %" PRIu64,
blob_compaction_readahead_size);
ROCKS_LOG_INFO(log, " bottommost_temperature: %d",
static_cast<int>(bottommost_temperature));
}
MutableCFOptions::MutableCFOptions(const Options& options)
: MutableCFOptions(ColumnFamilyOptions(options)) {}
#ifndef ROCKSDB_LITE
Status GetMutableOptionsFromStrings(
const MutableCFOptions& base_options,
const std::unordered_map<std::string, std::string>& options_map,
Logger* /*info_log*/, MutableCFOptions* new_options) {
assert(new_options);
*new_options = base_options;
ConfigOptions config_options;
Status s = OptionTypeInfo::ParseType(
config_options, options_map, cf_mutable_options_type_info, new_options);
if (!s.ok()) {
*new_options = base_options;
}
return s;
}
Status GetStringFromMutableCFOptions(const ConfigOptions& config_options,
const MutableCFOptions& mutable_opts,
std::string* opt_string) {
assert(opt_string);
opt_string->clear();
return OptionTypeInfo::SerializeType(
config_options, cf_mutable_options_type_info, &mutable_opts, opt_string);
}
#endif // ROCKSDB_LITE
} // namespace ROCKSDB_NAMESPACE
Reference in New Issue View Git Blame Copy Permalink