2013-10-16 23:59:46 +02:00
|
|
|
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
|
|
|
// This source code is licensed under the BSD-style license found in the
|
|
|
|
// LICENSE file in the root directory of this source tree. An additional grant
|
|
|
|
// of patent rights can be found in the PATENTS file in the same directory.
|
|
|
|
//
|
2011-03-18 23:37:00 +01:00
|
|
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
|
|
|
// Use of this source code is governed by a BSD-style license that can be
|
|
|
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
|
|
|
|
2013-08-23 17:38:13 +02:00
|
|
|
#include "rocksdb/options.h"
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2013-01-11 02:18:50 +01:00
|
|
|
#include <limits>
|
|
|
|
|
2013-08-23 17:38:13 +02:00
|
|
|
#include "rocksdb/cache.h"
|
|
|
|
#include "rocksdb/compaction_filter.h"
|
|
|
|
#include "rocksdb/comparator.h"
|
|
|
|
#include "rocksdb/env.h"
|
|
|
|
#include "rocksdb/filter_policy.h"
|
2014-01-25 01:15:05 +01:00
|
|
|
#include "rocksdb/memtablerep.h"
|
2014-01-28 06:58:46 +01:00
|
|
|
#include "rocksdb/merge_operator.h"
|
2014-01-25 01:15:05 +01:00
|
|
|
#include "rocksdb/slice.h"
|
|
|
|
#include "rocksdb/slice_transform.h"
|
2014-01-28 06:58:46 +01:00
|
|
|
#include "rocksdb/table.h"
|
2014-01-25 01:15:05 +01:00
|
|
|
#include "rocksdb/table_properties.h"
|
2013-10-29 01:54:09 +01:00
|
|
|
#include "table/block_based_table_factory.h"
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2013-10-04 06:49:15 +02:00
|
|
|
namespace rocksdb {
|
2011-03-18 23:37:00 +01:00
|
|
|
|
|
|
|
Options::Options()
|
|
|
|
: comparator(BytewiseComparator()),
|
2013-03-21 23:59:47 +01:00
|
|
|
merge_operator(nullptr),
|
2013-05-12 11:36:59 +02:00
|
|
|
compaction_filter(nullptr),
|
2014-03-25 01:57:13 +01:00
|
|
|
compaction_filter_factory(std::shared_ptr<CompactionFilterFactory>(
|
|
|
|
new DefaultCompactionFilterFactory())),
|
2014-03-27 16:22:59 +01:00
|
|
|
compaction_filter_factory_v2(new DefaultCompactionFilterFactoryV2()),
|
2011-03-18 23:37:00 +01:00
|
|
|
create_if_missing(false),
|
|
|
|
error_if_exists(false),
|
|
|
|
paranoid_checks(false),
|
|
|
|
env(Env::Default()),
|
2013-03-01 03:04:58 +01:00
|
|
|
info_log(nullptr),
|
2014-03-05 03:10:14 +01:00
|
|
|
info_log_level(INFO),
|
2014-03-25 01:57:13 +01:00
|
|
|
write_buffer_size(4 << 20),
|
2012-10-19 23:00:53 +02:00
|
|
|
max_write_buffer_number(2),
|
2013-06-11 23:23:58 +02:00
|
|
|
min_write_buffer_number_to_merge(1),
|
2011-03-18 23:37:00 +01:00
|
|
|
max_open_files(1000),
|
2013-09-02 08:23:40 +02:00
|
|
|
block_cache(nullptr),
|
|
|
|
block_cache_compressed(nullptr),
|
2011-04-12 21:38:58 +02:00
|
|
|
block_size(4096),
|
2011-03-18 23:37:00 +01:00
|
|
|
block_restart_interval(16),
|
2012-04-17 17:36:46 +02:00
|
|
|
compression(kSnappyCompression),
|
2013-03-01 03:04:58 +01:00
|
|
|
filter_policy(nullptr),
|
2013-08-13 23:04:56 +02:00
|
|
|
prefix_extractor(nullptr),
|
|
|
|
whole_key_filtering(true),
|
2012-06-23 04:30:03 +02:00
|
|
|
num_levels(7),
|
|
|
|
level0_file_num_compaction_trigger(4),
|
|
|
|
level0_slowdown_writes_trigger(8),
|
|
|
|
level0_stop_writes_trigger(12),
|
|
|
|
max_mem_compaction_level(2),
|
|
|
|
target_file_size_base(2 * 1048576),
|
2012-08-22 02:33:46 +02:00
|
|
|
target_file_size_multiplier(1),
|
2012-07-03 02:30:32 +02:00
|
|
|
max_bytes_for_level_base(10 * 1048576),
|
2012-06-23 04:30:03 +02:00
|
|
|
max_bytes_for_level_multiplier(10),
|
2013-05-21 20:37:06 +02:00
|
|
|
max_bytes_for_level_multiplier_additional(num_levels, 1),
|
2012-06-23 04:30:03 +02:00
|
|
|
expanded_compaction_factor(25),
|
2012-11-21 08:07:41 +01:00
|
|
|
source_compaction_factor(1),
|
2012-06-23 04:30:03 +02:00
|
|
|
max_grandparent_overlap_factor(10),
|
2012-08-15 00:20:36 +02:00
|
|
|
disableDataSync(false),
|
2012-08-27 21:10:26 +02:00
|
|
|
use_fsync(false),
|
2012-09-06 02:44:13 +02:00
|
|
|
db_stats_log_interval(1800),
|
2012-09-17 22:35:57 +02:00
|
|
|
db_log_dir(""),
|
2013-10-01 23:46:52 +02:00
|
|
|
wal_dir(""),
|
2012-10-16 17:53:46 +02:00
|
|
|
disable_seek_compaction(false),
|
2013-11-09 00:23:46 +01:00
|
|
|
delete_obsolete_files_period_micros(6 * 60 * 60 * 1000000UL),
|
2012-11-06 01:51:55 +01:00
|
|
|
max_background_compactions(1),
|
2013-09-13 23:38:37 +02:00
|
|
|
max_background_flushes(0),
|
2012-10-29 22:18:00 +01:00
|
|
|
max_log_file_size(0),
|
2013-02-05 04:42:40 +01:00
|
|
|
log_file_time_to_roll(0),
|
|
|
|
keep_log_file_num(1000),
|
2013-08-06 00:43:49 +02:00
|
|
|
soft_rate_limit(0.0),
|
|
|
|
hard_rate_limit(0.0),
|
|
|
|
rate_limit_delay_max_milliseconds(1000),
|
2013-01-11 02:18:50 +01:00
|
|
|
max_manifest_file_size(std::numeric_limits<uint64_t>::max()),
|
2012-11-06 21:02:18 +01:00
|
|
|
no_block_cache(false),
|
|
|
|
table_cache_numshardbits(4),
|
2013-10-10 02:04:40 +02:00
|
|
|
table_cache_remove_scan_count_limit(16),
|
Make arena block size configurable
Summary:
Add an option for arena block size, default value 4096 bytes. Arena will allocate blocks with such size.
I am not sure about passing parameter to skiplist in the new virtualized framework, though I talked to Jim a bit. So add Jim as reviewer.
Test Plan:
new unit test, I am running db_test.
For passing paramter from configured option to Arena, I tried tests like:
TEST(DBTest, Arena_Option) {
std::string dbname = test::TmpDir() + "/db_arena_option_test";
DestroyDB(dbname, Options());
DB* db = nullptr;
Options opts;
opts.create_if_missing = true;
opts.arena_block_size = 1000000; // tested 99, 999999
Status s = DB::Open(opts, dbname, &db);
db->Put(WriteOptions(), "a", "123");
}
and printed some debug info. The results look good. Any suggestion for such a unit-test?
Reviewers: haobo, dhruba, emayanke, jpaton
Reviewed By: dhruba
CC: leveldb, zshao
Differential Revision: https://reviews.facebook.net/D11799
2013-07-31 21:42:23 +02:00
|
|
|
arena_block_size(0),
|
2012-11-26 22:56:45 +01:00
|
|
|
disable_auto_compactions(false),
|
2013-01-15 23:05:42 +01:00
|
|
|
WAL_ttl_seconds(0),
|
2013-11-07 03:46:28 +01:00
|
|
|
WAL_size_limit_MB(0),
|
2013-02-28 23:09:30 +01:00
|
|
|
manifest_preallocation_size(4 * 1024 * 1024),
|
2013-03-15 01:00:04 +01:00
|
|
|
purge_redundant_kvs_while_flush(true),
|
|
|
|
allow_os_buffer(true),
|
|
|
|
allow_mmap_reads(false),
|
2014-03-27 19:53:05 +01:00
|
|
|
allow_mmap_writes(false),
|
2013-05-21 20:53:33 +02:00
|
|
|
is_fd_close_on_exec(true),
|
2013-05-11 00:21:04 +02:00
|
|
|
skip_log_error_on_recovery(false),
|
2013-05-15 19:34:02 +02:00
|
|
|
stats_dump_period_sec(3600),
|
2014-03-25 01:57:13 +01:00
|
|
|
block_size_deviation(10),
|
2013-05-18 00:53:01 +02:00
|
|
|
advise_random_on_open(true),
|
2013-06-01 01:30:17 +02:00
|
|
|
access_hint_on_compaction_start(NORMAL),
|
2013-06-14 07:49:46 +02:00
|
|
|
use_adaptive_mutex(false),
|
2013-07-06 03:49:18 +02:00
|
|
|
bytes_per_sync(0),
|
2013-07-18 00:08:56 +02:00
|
|
|
compaction_style(kCompactionStyleLevel),
|
2014-03-10 18:06:34 +01:00
|
|
|
verify_checksums_in_compaction(true),
|
2013-07-23 23:42:27 +02:00
|
|
|
filter_deletes(false),
|
2013-07-28 20:53:08 +02:00
|
|
|
max_sequential_skip_in_iterations(8),
|
2013-08-13 19:56:20 +02:00
|
|
|
memtable_factory(std::shared_ptr<SkipListFactory>(new SkipListFactory)),
|
2013-10-29 01:54:09 +01:00
|
|
|
table_factory(
|
2014-03-25 01:57:13 +01:00
|
|
|
std::shared_ptr<TableFactory>(new BlockBasedTableFactory())),
|
In-place updates for equal keys and similar sized values
Summary:
Currently for each put, a fresh memory is allocated, and a new entry is added to the memtable with a new sequence number irrespective of whether the key already exists in the memtable. This diff is an attempt to update the value inplace for existing keys. It currently handles a very simple case:
1. Key already exists in the current memtable. Does not inplace update values in immutable memtable or snapshot
2. Latest value type is a 'put' ie kTypeValue
3. New value size is less than existing value, to avoid reallocating memory
TODO: For a put of an existing key, deallocate memory take by values, for other value types till a kTypeValue is found, ie. remove kTypeMerge.
TODO: Update the transaction log, to allow consistent reload of the memtable.
Test Plan: Added a unit test verifying the inplace update. But some other unit tests broken due to invalid sequence number checks. WIll fix them next.
Reviewers: xinyaohu, sumeet, haobo, dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D12423
Automatic commit by arc
2013-08-19 23:12:47 +02:00
|
|
|
inplace_update_support(false),
|
2013-11-27 23:27:02 +01:00
|
|
|
inplace_update_num_locks(10000),
|
2014-01-14 16:55:16 +01:00
|
|
|
inplace_callback(nullptr),
|
2013-11-27 23:27:02 +01:00
|
|
|
memtable_prefix_bloom_bits(0),
|
2014-01-16 08:12:31 +01:00
|
|
|
memtable_prefix_bloom_probes(6),
|
2014-02-27 20:38:55 +01:00
|
|
|
max_successive_merges(0),
|
2014-03-25 01:57:13 +01:00
|
|
|
min_partial_merge_operands(2),
|
2014-02-27 20:38:55 +01:00
|
|
|
allow_thread_local(true) {
|
2013-07-23 23:42:27 +02:00
|
|
|
assert(memtable_factory.get() != nullptr);
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2013-05-18 00:53:01 +02:00
|
|
|
static const char* const access_hints[] = {
|
|
|
|
"NONE", "NORMAL", "SEQUENTIAL", "WILLNEED"
|
|
|
|
};
|
|
|
|
|
2012-08-22 20:43:53 +02:00
|
|
|
void
|
2013-01-20 11:07:13 +01:00
|
|
|
Options::Dump(Logger* log) const
|
2012-08-22 20:43:53 +02:00
|
|
|
{
|
2012-10-19 23:00:53 +02:00
|
|
|
Log(log," Options.comparator: %s", comparator->Name());
|
2013-03-21 23:59:47 +01:00
|
|
|
Log(log," Options.merge_operator: %s",
|
|
|
|
merge_operator? merge_operator->Name() : "None");
|
2013-05-12 11:36:59 +02:00
|
|
|
Log(log," Options.compaction_filter: %s",
|
|
|
|
compaction_filter? compaction_filter->Name() : "None");
|
2013-08-13 19:56:20 +02:00
|
|
|
Log(log," Options.compaction_filter_factory: %s",
|
|
|
|
compaction_filter_factory->Name());
|
2014-01-10 02:52:11 +01:00
|
|
|
Log(log, " Options.compaction_filter_factory_v2: %s",
|
|
|
|
compaction_filter_factory_v2->Name());
|
2013-09-27 07:53:01 +02:00
|
|
|
Log(log," Options.memtable_factory: %s",
|
|
|
|
memtable_factory->Name());
|
2013-10-29 01:54:09 +01:00
|
|
|
Log(log," Options.table_factory: %s", table_factory->Name());
|
2012-10-19 23:00:53 +02:00
|
|
|
Log(log," Options.error_if_exists: %d", error_if_exists);
|
2013-08-14 18:06:10 +02:00
|
|
|
Log(log," Options.create_if_missing: %d", create_if_missing);
|
2012-10-19 23:00:53 +02:00
|
|
|
Log(log," Options.paranoid_checks: %d", paranoid_checks);
|
|
|
|
Log(log," Options.env: %p", env);
|
2013-01-20 11:07:13 +01:00
|
|
|
Log(log," Options.info_log: %p", info_log.get());
|
2012-10-19 23:00:53 +02:00
|
|
|
Log(log," Options.write_buffer_size: %zd", write_buffer_size);
|
2012-11-08 00:11:37 +01:00
|
|
|
Log(log," Options.max_write_buffer_number: %d", max_write_buffer_number);
|
2012-10-19 23:00:53 +02:00
|
|
|
Log(log," Options.max_open_files: %d", max_open_files);
|
2013-01-20 11:07:13 +01:00
|
|
|
Log(log," Options.block_cache: %p", block_cache.get());
|
2013-09-02 08:23:40 +02:00
|
|
|
Log(log," Options.block_cache_compressed: %p",
|
|
|
|
block_cache_compressed.get());
|
2012-11-01 01:02:24 +01:00
|
|
|
if (block_cache) {
|
2013-09-02 08:23:40 +02:00
|
|
|
Log(log," Options.block_cache_size: %zd",
|
2012-11-01 01:02:24 +01:00
|
|
|
block_cache->GetCapacity());
|
|
|
|
}
|
2013-09-02 08:23:40 +02:00
|
|
|
if (block_cache_compressed) {
|
|
|
|
Log(log,"Options.block_cache_compressed_size: %zd",
|
|
|
|
block_cache_compressed->GetCapacity());
|
|
|
|
}
|
2012-10-19 23:00:53 +02:00
|
|
|
Log(log," Options.block_size: %zd", block_size);
|
|
|
|
Log(log," Options.block_restart_interval: %d", block_restart_interval);
|
2013-01-24 19:54:26 +01:00
|
|
|
if (!compression_per_level.empty()) {
|
2013-03-15 02:32:01 +01:00
|
|
|
for (unsigned int i = 0; i < compression_per_level.size(); i++) {
|
2012-11-01 18:50:08 +01:00
|
|
|
Log(log," Options.compression[%d]: %d",
|
2012-10-28 07:13:17 +01:00
|
|
|
i, compression_per_level[i]);
|
|
|
|
}
|
|
|
|
} else {
|
2012-10-29 22:18:00 +01:00
|
|
|
Log(log," Options.compression: %d", compression);
|
2012-10-28 07:13:17 +01:00
|
|
|
}
|
2012-08-27 21:10:26 +02:00
|
|
|
Log(log," Options.filter_policy: %s",
|
2013-03-01 03:04:58 +01:00
|
|
|
filter_policy == nullptr ? "nullptr" : filter_policy->Name());
|
2013-08-14 18:06:10 +02:00
|
|
|
Log(log," Options.prefix_extractor: %s",
|
|
|
|
prefix_extractor == nullptr ? "nullptr" : prefix_extractor->Name());
|
|
|
|
Log(log," Options.whole_key_filtering: %d", whole_key_filtering);
|
2012-08-27 21:10:26 +02:00
|
|
|
Log(log," Options.num_levels: %d", num_levels);
|
|
|
|
Log(log," Options.disableDataSync: %d", disableDataSync);
|
|
|
|
Log(log," Options.use_fsync: %d", use_fsync);
|
2014-02-03 19:21:58 +01:00
|
|
|
Log(log," Options.max_log_file_size: %zu", max_log_file_size);
|
2013-11-13 06:02:03 +01:00
|
|
|
Log(log,"Options.max_manifest_file_size: %lu",
|
|
|
|
(unsigned long)max_manifest_file_size);
|
2014-02-03 19:21:58 +01:00
|
|
|
Log(log," Options.log_file_time_to_roll: %zu", log_file_time_to_roll);
|
|
|
|
Log(log," Options.keep_log_file_num: %zu", keep_log_file_num);
|
2012-10-26 23:55:02 +02:00
|
|
|
Log(log," Options.db_stats_log_interval: %d",
|
2012-08-27 21:10:26 +02:00
|
|
|
db_stats_log_interval);
|
2013-03-15 01:00:04 +01:00
|
|
|
Log(log," Options.allow_os_buffer: %d", allow_os_buffer);
|
|
|
|
Log(log," Options.allow_mmap_reads: %d", allow_mmap_reads);
|
|
|
|
Log(log," Options.allow_mmap_writes: %d", allow_mmap_writes);
|
2013-06-11 23:23:58 +02:00
|
|
|
Log(log," Options.min_write_buffer_number_to_merge: %d",
|
|
|
|
min_write_buffer_number_to_merge);
|
2013-02-28 23:09:30 +01:00
|
|
|
Log(log," Options.purge_redundant_kvs_while_flush: %d",
|
|
|
|
purge_redundant_kvs_while_flush);
|
2012-11-01 18:50:08 +01:00
|
|
|
Log(log," Options.compression_opts.window_bits: %d",
|
|
|
|
compression_opts.window_bits);
|
|
|
|
Log(log," Options.compression_opts.level: %d",
|
|
|
|
compression_opts.level);
|
|
|
|
Log(log," Options.compression_opts.strategy: %d",
|
|
|
|
compression_opts.strategy);
|
2012-08-27 21:10:26 +02:00
|
|
|
Log(log," Options.level0_file_num_compaction_trigger: %d",
|
|
|
|
level0_file_num_compaction_trigger);
|
|
|
|
Log(log," Options.level0_slowdown_writes_trigger: %d",
|
|
|
|
level0_slowdown_writes_trigger);
|
|
|
|
Log(log," Options.level0_stop_writes_trigger: %d",
|
|
|
|
level0_stop_writes_trigger);
|
|
|
|
Log(log," Options.max_mem_compaction_level: %d",
|
|
|
|
max_mem_compaction_level);
|
|
|
|
Log(log," Options.target_file_size_base: %d",
|
|
|
|
target_file_size_base);
|
|
|
|
Log(log," Options.target_file_size_multiplier: %d",
|
|
|
|
target_file_size_multiplier);
|
2013-11-13 06:02:03 +01:00
|
|
|
Log(log," Options.max_bytes_for_level_base: %lu",
|
|
|
|
(unsigned long)max_bytes_for_level_base);
|
2012-08-27 21:10:26 +02:00
|
|
|
Log(log," Options.max_bytes_for_level_multiplier: %d",
|
|
|
|
max_bytes_for_level_multiplier);
|
2013-05-21 20:37:06 +02:00
|
|
|
for (int i = 0; i < num_levels; i++) {
|
|
|
|
Log(log,"Options.max_bytes_for_level_multiplier_addtl[%d]: %d",
|
|
|
|
i, max_bytes_for_level_multiplier_additional[i]);
|
|
|
|
}
|
2013-11-13 06:02:03 +01:00
|
|
|
Log(log," Options.max_sequential_skip_in_iterations: %lu",
|
|
|
|
(unsigned long)max_sequential_skip_in_iterations);
|
2012-08-27 21:10:26 +02:00
|
|
|
Log(log," Options.expanded_compaction_factor: %d",
|
|
|
|
expanded_compaction_factor);
|
2012-11-21 08:07:41 +01:00
|
|
|
Log(log," Options.source_compaction_factor: %d",
|
|
|
|
source_compaction_factor);
|
2012-08-27 21:10:26 +02:00
|
|
|
Log(log," Options.max_grandparent_overlap_factor: %d",
|
|
|
|
max_grandparent_overlap_factor);
|
2012-10-19 23:00:53 +02:00
|
|
|
Log(log," Options.db_log_dir: %s",
|
2012-09-06 02:44:13 +02:00
|
|
|
db_log_dir.c_str());
|
2013-10-01 23:46:52 +02:00
|
|
|
Log(log," Options.wal_dir: %s",
|
|
|
|
wal_dir.c_str());
|
2012-10-19 23:00:53 +02:00
|
|
|
Log(log," Options.disable_seek_compaction: %d",
|
2012-09-17 22:35:57 +02:00
|
|
|
disable_seek_compaction);
|
2012-11-01 01:02:24 +01:00
|
|
|
Log(log," Options.no_block_cache: %d",
|
|
|
|
no_block_cache);
|
|
|
|
Log(log," Options.table_cache_numshardbits: %d",
|
2013-05-11 00:21:04 +02:00
|
|
|
table_cache_numshardbits);
|
2013-10-10 02:04:40 +02:00
|
|
|
Log(log," Options.table_cache_remove_scan_count_limit: %d",
|
|
|
|
table_cache_remove_scan_count_limit);
|
2014-02-03 19:21:58 +01:00
|
|
|
Log(log," Options.arena_block_size: %zu",
|
Make arena block size configurable
Summary:
Add an option for arena block size, default value 4096 bytes. Arena will allocate blocks with such size.
I am not sure about passing parameter to skiplist in the new virtualized framework, though I talked to Jim a bit. So add Jim as reviewer.
Test Plan:
new unit test, I am running db_test.
For passing paramter from configured option to Arena, I tried tests like:
TEST(DBTest, Arena_Option) {
std::string dbname = test::TmpDir() + "/db_arena_option_test";
DestroyDB(dbname, Options());
DB* db = nullptr;
Options opts;
opts.create_if_missing = true;
opts.arena_block_size = 1000000; // tested 99, 999999
Status s = DB::Open(opts, dbname, &db);
db->Put(WriteOptions(), "a", "123");
}
and printed some debug info. The results look good. Any suggestion for such a unit-test?
Reviewers: haobo, dhruba, emayanke, jpaton
Reviewed By: dhruba
CC: leveldb, zshao
Differential Revision: https://reviews.facebook.net/D11799
2013-07-31 21:42:23 +02:00
|
|
|
arena_block_size);
|
2013-11-13 06:02:03 +01:00
|
|
|
Log(log," Options.delete_obsolete_files_period_micros: %lu",
|
|
|
|
(unsigned long)delete_obsolete_files_period_micros);
|
2012-10-19 23:00:53 +02:00
|
|
|
Log(log," Options.max_background_compactions: %d",
|
|
|
|
max_background_compactions);
|
2013-09-13 23:38:37 +02:00
|
|
|
Log(log," Options.max_background_flushes: %d",
|
|
|
|
max_background_flushes);
|
2013-08-14 18:06:10 +02:00
|
|
|
Log(log," Options.soft_rate_limit: %.2f",
|
|
|
|
soft_rate_limit);
|
2013-08-06 00:43:49 +02:00
|
|
|
Log(log," Options.hard_rate_limit: %.2f",
|
|
|
|
hard_rate_limit);
|
2013-08-02 20:46:47 +02:00
|
|
|
Log(log," Options.rate_limit_delay_max_milliseconds: %u",
|
2013-08-06 00:43:49 +02:00
|
|
|
rate_limit_delay_max_milliseconds);
|
2012-11-21 00:45:41 +01:00
|
|
|
Log(log," Options.disable_auto_compactions: %d",
|
|
|
|
disable_auto_compactions);
|
2013-11-13 06:02:03 +01:00
|
|
|
Log(log," Options.WAL_ttl_seconds: %lu",
|
|
|
|
(unsigned long)WAL_ttl_seconds);
|
|
|
|
Log(log," Options.WAL_size_limit_MB: %lu",
|
|
|
|
(unsigned long)WAL_size_limit_MB);
|
2014-02-03 19:21:58 +01:00
|
|
|
Log(log," Options.manifest_preallocation_size: %zu",
|
2013-01-15 23:05:42 +01:00
|
|
|
manifest_preallocation_size);
|
2013-04-10 04:42:07 +02:00
|
|
|
Log(log," Options.purge_redundant_kvs_while_flush: %d",
|
|
|
|
purge_redundant_kvs_while_flush);
|
|
|
|
Log(log," Options.allow_os_buffer: %d",
|
|
|
|
allow_os_buffer);
|
|
|
|
Log(log," Options.allow_mmap_reads: %d",
|
|
|
|
allow_mmap_reads);
|
|
|
|
Log(log," Options.allow_mmap_writes: %d",
|
|
|
|
allow_mmap_writes);
|
|
|
|
Log(log," Options.is_fd_close_on_exec: %d",
|
|
|
|
is_fd_close_on_exec);
|
2013-05-21 20:53:33 +02:00
|
|
|
Log(log," Options.skip_log_error_on_recovery: %d",
|
|
|
|
skip_log_error_on_recovery);
|
2013-08-02 20:46:47 +02:00
|
|
|
Log(log," Options.stats_dump_period_sec: %u",
|
2013-05-11 00:21:04 +02:00
|
|
|
stats_dump_period_sec);
|
2013-05-15 19:34:02 +02:00
|
|
|
Log(log," Options.block_size_deviation: %d",
|
|
|
|
block_size_deviation);
|
2013-05-18 00:53:01 +02:00
|
|
|
Log(log," Options.advise_random_on_open: %d",
|
|
|
|
advise_random_on_open);
|
|
|
|
Log(log," Options.access_hint_on_compaction_start: %s",
|
|
|
|
access_hints[access_hint_on_compaction_start]);
|
2013-06-01 01:30:17 +02:00
|
|
|
Log(log," Options.use_adaptive_mutex: %d",
|
|
|
|
use_adaptive_mutex);
|
2013-11-13 06:02:03 +01:00
|
|
|
Log(log," Options.bytes_per_sync: %lu",
|
|
|
|
(unsigned long)bytes_per_sync);
|
2013-07-13 01:56:52 +02:00
|
|
|
Log(log," Options.filter_deletes: %d",
|
|
|
|
filter_deletes);
|
2014-03-10 18:06:34 +01:00
|
|
|
Log(log, " Options.verify_checksums_in_compaction: %d",
|
|
|
|
verify_checksums_in_compaction);
|
2013-07-04 00:32:49 +02:00
|
|
|
Log(log," Options.compaction_style: %d",
|
|
|
|
compaction_style);
|
In-place updates for equal keys and similar sized values
Summary:
Currently for each put, a fresh memory is allocated, and a new entry is added to the memtable with a new sequence number irrespective of whether the key already exists in the memtable. This diff is an attempt to update the value inplace for existing keys. It currently handles a very simple case:
1. Key already exists in the current memtable. Does not inplace update values in immutable memtable or snapshot
2. Latest value type is a 'put' ie kTypeValue
3. New value size is less than existing value, to avoid reallocating memory
TODO: For a put of an existing key, deallocate memory take by values, for other value types till a kTypeValue is found, ie. remove kTypeMerge.
TODO: Update the transaction log, to allow consistent reload of the memtable.
Test Plan: Added a unit test verifying the inplace update. But some other unit tests broken due to invalid sequence number checks. WIll fix them next.
Reviewers: xinyaohu, sumeet, haobo, dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D12423
Automatic commit by arc
2013-08-19 23:12:47 +02:00
|
|
|
Log(log," Options.compaction_options_universal.size_ratio: %u",
|
2013-07-04 00:32:49 +02:00
|
|
|
compaction_options_universal.size_ratio);
|
In-place updates for equal keys and similar sized values
Summary:
Currently for each put, a fresh memory is allocated, and a new entry is added to the memtable with a new sequence number irrespective of whether the key already exists in the memtable. This diff is an attempt to update the value inplace for existing keys. It currently handles a very simple case:
1. Key already exists in the current memtable. Does not inplace update values in immutable memtable or snapshot
2. Latest value type is a 'put' ie kTypeValue
3. New value size is less than existing value, to avoid reallocating memory
TODO: For a put of an existing key, deallocate memory take by values, for other value types till a kTypeValue is found, ie. remove kTypeMerge.
TODO: Update the transaction log, to allow consistent reload of the memtable.
Test Plan: Added a unit test verifying the inplace update. But some other unit tests broken due to invalid sequence number checks. WIll fix them next.
Reviewers: xinyaohu, sumeet, haobo, dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D12423
Automatic commit by arc
2013-08-19 23:12:47 +02:00
|
|
|
Log(log,"Options.compaction_options_universal.min_merge_width: %u",
|
2013-07-04 00:32:49 +02:00
|
|
|
compaction_options_universal.min_merge_width);
|
In-place updates for equal keys and similar sized values
Summary:
Currently for each put, a fresh memory is allocated, and a new entry is added to the memtable with a new sequence number irrespective of whether the key already exists in the memtable. This diff is an attempt to update the value inplace for existing keys. It currently handles a very simple case:
1. Key already exists in the current memtable. Does not inplace update values in immutable memtable or snapshot
2. Latest value type is a 'put' ie kTypeValue
3. New value size is less than existing value, to avoid reallocating memory
TODO: For a put of an existing key, deallocate memory take by values, for other value types till a kTypeValue is found, ie. remove kTypeMerge.
TODO: Update the transaction log, to allow consistent reload of the memtable.
Test Plan: Added a unit test verifying the inplace update. But some other unit tests broken due to invalid sequence number checks. WIll fix them next.
Reviewers: xinyaohu, sumeet, haobo, dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D12423
Automatic commit by arc
2013-08-19 23:12:47 +02:00
|
|
|
Log(log,"Options.compaction_options_universal.max_merge_width: %u",
|
2013-07-04 00:32:49 +02:00
|
|
|
compaction_options_universal.max_merge_width);
|
2013-09-10 01:06:10 +02:00
|
|
|
Log(log,"Options.compaction_options_universal."
|
|
|
|
"max_size_amplification_percent: %u",
|
|
|
|
compaction_options_universal.max_size_amplification_percent);
|
2013-12-03 21:32:07 +01:00
|
|
|
Log(log,
|
|
|
|
"Options.compaction_options_universal.compression_size_percent: %u",
|
|
|
|
compaction_options_universal.compression_size_percent);
|
2013-10-16 20:50:50 +02:00
|
|
|
std::string collector_names;
|
2013-11-20 01:29:42 +01:00
|
|
|
for (auto collector : table_properties_collectors) {
|
2013-10-16 20:50:50 +02:00
|
|
|
collector_names.append(collector->Name());
|
|
|
|
collector_names.append("; ");
|
|
|
|
}
|
2013-11-20 01:29:42 +01:00
|
|
|
Log(log, " Options.table_properties_collectors: %s",
|
2013-10-16 20:50:50 +02:00
|
|
|
collector_names.c_str());
|
2013-11-08 06:27:21 +01:00
|
|
|
Log(log, " Options.inplace_update_support: %d",
|
In-place updates for equal keys and similar sized values
Summary:
Currently for each put, a fresh memory is allocated, and a new entry is added to the memtable with a new sequence number irrespective of whether the key already exists in the memtable. This diff is an attempt to update the value inplace for existing keys. It currently handles a very simple case:
1. Key already exists in the current memtable. Does not inplace update values in immutable memtable or snapshot
2. Latest value type is a 'put' ie kTypeValue
3. New value size is less than existing value, to avoid reallocating memory
TODO: For a put of an existing key, deallocate memory take by values, for other value types till a kTypeValue is found, ie. remove kTypeMerge.
TODO: Update the transaction log, to allow consistent reload of the memtable.
Test Plan: Added a unit test verifying the inplace update. But some other unit tests broken due to invalid sequence number checks. WIll fix them next.
Reviewers: xinyaohu, sumeet, haobo, dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D12423
Automatic commit by arc
2013-08-19 23:12:47 +02:00
|
|
|
inplace_update_support);
|
2013-11-08 06:27:21 +01:00
|
|
|
Log(log, " Options.inplace_update_num_locks: %zd",
|
In-place updates for equal keys and similar sized values
Summary:
Currently for each put, a fresh memory is allocated, and a new entry is added to the memtable with a new sequence number irrespective of whether the key already exists in the memtable. This diff is an attempt to update the value inplace for existing keys. It currently handles a very simple case:
1. Key already exists in the current memtable. Does not inplace update values in immutable memtable or snapshot
2. Latest value type is a 'put' ie kTypeValue
3. New value size is less than existing value, to avoid reallocating memory
TODO: For a put of an existing key, deallocate memory take by values, for other value types till a kTypeValue is found, ie. remove kTypeMerge.
TODO: Update the transaction log, to allow consistent reload of the memtable.
Test Plan: Added a unit test verifying the inplace update. But some other unit tests broken due to invalid sequence number checks. WIll fix them next.
Reviewers: xinyaohu, sumeet, haobo, dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D12423
Automatic commit by arc
2013-08-19 23:12:47 +02:00
|
|
|
inplace_update_num_locks);
|
2014-03-25 01:57:13 +01:00
|
|
|
Log(log, " Options.min_partial_merge_operands: %u",
|
|
|
|
min_partial_merge_operands);
|
2013-11-27 23:27:02 +01:00
|
|
|
// TODO: easier config for bloom (maybe based on avg key/value size)
|
|
|
|
Log(log, " Options.memtable_prefix_bloom_bits: %d",
|
|
|
|
memtable_prefix_bloom_bits);
|
|
|
|
Log(log, " Options.memtable_prefix_bloom_probes: %d",
|
|
|
|
memtable_prefix_bloom_probes);
|
2014-01-11 02:33:56 +01:00
|
|
|
Log(log, " Options.max_successive_merges: %zd",
|
|
|
|
max_successive_merges);
|
2012-08-22 20:43:53 +02:00
|
|
|
} // Options::Dump
|
|
|
|
|
2013-03-08 18:19:24 +01:00
|
|
|
//
|
|
|
|
// The goal of this method is to create a configuration that
|
|
|
|
// allows an application to write all files into L0 and
|
|
|
|
// then do a single compaction to output all files into L1.
|
2013-02-26 07:57:37 +01:00
|
|
|
Options*
|
|
|
|
Options::PrepareForBulkLoad()
|
|
|
|
{
|
2013-03-08 18:19:24 +01:00
|
|
|
// never slowdown ingest.
|
2013-02-26 07:57:37 +01:00
|
|
|
level0_file_num_compaction_trigger = (1<<30);
|
|
|
|
level0_slowdown_writes_trigger = (1<<30);
|
|
|
|
level0_stop_writes_trigger = (1<<30);
|
2013-03-08 18:19:24 +01:00
|
|
|
|
|
|
|
// no auto compactions please. The application should issue a
|
|
|
|
// manual compaction after all data is loaded into L0.
|
2013-02-26 07:57:37 +01:00
|
|
|
disable_auto_compactions = true;
|
|
|
|
disable_seek_compaction = true;
|
|
|
|
disableDataSync = true;
|
2013-03-08 18:19:24 +01:00
|
|
|
|
|
|
|
// A manual compaction run should pick all files in L0 in
|
|
|
|
// a single compaction run.
|
2013-02-26 07:57:37 +01:00
|
|
|
source_compaction_factor = (1<<30);
|
|
|
|
|
2013-03-08 18:19:24 +01:00
|
|
|
// It is better to have only 2 levels, otherwise a manual
|
|
|
|
// compaction would compact at every possible level, thereby
|
|
|
|
// increasing the total time needed for compactions.
|
|
|
|
num_levels = 2;
|
|
|
|
|
|
|
|
// Prevent a memtable flush to automatically promote files
|
|
|
|
// to L1. This is helpful so that all files that are
|
|
|
|
// input to the manual compaction are all at L0.
|
|
|
|
max_background_compactions = 2;
|
|
|
|
|
|
|
|
// The compaction would create large files in L1.
|
|
|
|
target_file_size_base = 256 * 1024 * 1024;
|
2013-02-26 07:57:37 +01:00
|
|
|
return this;
|
|
|
|
}
|
|
|
|
|
2013-10-04 06:49:15 +02:00
|
|
|
} // namespace rocksdb
|