2016-02-10 00:12:00 +01:00
|
|
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
2013-10-16 23:59:46 +02:00
|
|
|
// This source code is licensed under the BSD-style license found in the
|
|
|
|
// LICENSE file in the root directory of this source tree. An additional grant
|
|
|
|
// of patent rights can be found in the PATENTS file in the same directory.
|
|
|
|
//
|
2011-03-18 23:37:00 +01:00
|
|
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
|
|
|
// Use of this source code is governed by a BSD-style license that can be
|
|
|
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
|
|
|
#include "db/db_impl.h"
|
|
|
|
|
2014-09-05 08:14:37 +02:00
|
|
|
#ifndef __STDC_FORMAT_MACROS
|
2014-02-26 23:16:23 +01:00
|
|
|
#define __STDC_FORMAT_MACROS
|
2014-09-05 08:14:37 +02:00
|
|
|
#endif
|
2014-02-26 23:16:23 +01:00
|
|
|
#include <inttypes.h>
|
2015-09-02 22:58:22 +02:00
|
|
|
#include <stdint.h>
|
2016-01-19 07:17:31 +01:00
|
|
|
#ifdef OS_SOLARIS
|
2016-01-19 05:45:21 +01:00
|
|
|
#include <alloca.h>
|
2016-01-19 07:17:31 +01:00
|
|
|
#endif
|
2016-04-28 01:23:33 +02:00
|
|
|
#ifdef ROCKSDB_JEMALLOC
|
|
|
|
#include "jemalloc/jemalloc.h"
|
|
|
|
#endif
|
2015-09-02 22:58:22 +02:00
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
#include <algorithm>
|
2012-11-26 22:56:45 +01:00
|
|
|
#include <climits>
|
|
|
|
#include <cstdio>
|
2015-11-11 07:58:01 +01:00
|
|
|
#include <map>
|
2011-03-18 23:37:00 +01:00
|
|
|
#include <set>
|
2013-06-21 01:58:59 +02:00
|
|
|
#include <stdexcept>
|
2013-10-16 20:50:50 +02:00
|
|
|
#include <string>
|
2014-01-28 20:05:04 +01:00
|
|
|
#include <unordered_map>
|
2015-09-02 22:58:22 +02:00
|
|
|
#include <unordered_set>
|
2014-01-17 06:56:26 +01:00
|
|
|
#include <utility>
|
2013-10-16 20:50:50 +02:00
|
|
|
#include <vector>
|
2012-11-26 22:56:45 +01:00
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
#include "db/builder.h"
|
2014-11-01 00:31:25 +01:00
|
|
|
#include "db/compaction_job.h"
|
2016-01-26 02:49:58 +01:00
|
|
|
#include "db/db_info_dumper.h"
|
2013-10-16 20:50:50 +02:00
|
|
|
#include "db/db_iter.h"
|
2013-12-31 03:33:57 +01:00
|
|
|
#include "db/dbformat.h"
|
2015-05-28 22:37:47 +02:00
|
|
|
#include "db/event_helpers.h"
|
2016-10-21 02:05:32 +02:00
|
|
|
#include "db/external_sst_file_ingestion_job.h"
|
2015-09-02 22:58:22 +02:00
|
|
|
#include "db/flush_job.h"
|
|
|
|
#include "db/forward_iterator.h"
|
2014-11-15 00:43:10 +01:00
|
|
|
#include "db/job_context.h"
|
2011-03-18 23:37:00 +01:00
|
|
|
#include "db/log_reader.h"
|
|
|
|
#include "db/log_writer.h"
|
2015-02-18 20:49:31 +01:00
|
|
|
#include "db/managed_iterator.h"
|
2011-03-18 23:37:00 +01:00
|
|
|
#include "db/memtable.h"
|
2013-12-11 04:03:13 +01:00
|
|
|
#include "db/memtable_list.h"
|
2013-12-03 03:34:05 +01:00
|
|
|
#include "db/merge_context.h"
|
2013-03-21 23:59:47 +01:00
|
|
|
#include "db/merge_helper.h"
|
2016-11-04 02:40:23 +01:00
|
|
|
#include "db/range_del_aggregator.h"
|
2011-03-18 23:37:00 +01:00
|
|
|
#include "db/table_cache.h"
|
2013-11-20 01:29:42 +01:00
|
|
|
#include "db/table_properties_collector.h"
|
2013-10-16 20:50:50 +02:00
|
|
|
#include "db/transaction_log_impl.h"
|
2011-03-18 23:37:00 +01:00
|
|
|
#include "db/version_set.h"
|
|
|
|
#include "db/write_batch_internal.h"
|
2015-05-29 23:36:35 +02:00
|
|
|
#include "db/write_callback.h"
|
2015-10-16 23:10:33 +02:00
|
|
|
#include "memtable/hash_linklist_rep.h"
|
|
|
|
#include "memtable/hash_skiplist_rep.h"
|
2017-04-06 04:02:00 +02:00
|
|
|
#include "monitoring/iostats_context_imp.h"
|
|
|
|
#include "monitoring/perf_context_imp.h"
|
|
|
|
#include "monitoring/thread_status_updater.h"
|
|
|
|
#include "monitoring/thread_status_util.h"
|
|
|
|
#include "options/cf_options.h"
|
|
|
|
#include "options/options_helper.h"
|
|
|
|
#include "options/options_parser.h"
|
2015-09-02 22:58:22 +02:00
|
|
|
#include "port/likely.h"
|
2013-10-16 20:50:50 +02:00
|
|
|
#include "port/port.h"
|
[CF] Rethink table cache
Summary:
Adapting table cache to column families is interesting. We want table cache to be global LRU, so if some column families are use not as often as others, we want them to be evicted from cache. However, current TableCache object also constructs tables on its own. If table is not found in the cache, TableCache automatically creates new table. We want each column family to be able to specify different table factory.
To solve the problem, we still have a single LRU, but we provide the LRUCache object to TableCache on construction. We have one TableCache per column family, but the underyling cache is shared by all TableCache objects.
This allows us to have a global LRU, but still be able to support different table factories for different column families. Also, in the future it will also be able to support different directories for different column families.
Test Plan: make check
Reviewers: dhruba, haobo, kailiu, sdong
CC: leveldb
Differential Revision: https://reviews.facebook.net/D15915
2014-02-05 18:07:55 +01:00
|
|
|
#include "rocksdb/cache.h"
|
2013-08-23 17:38:13 +02:00
|
|
|
#include "rocksdb/compaction_filter.h"
|
|
|
|
#include "rocksdb/db.h"
|
|
|
|
#include "rocksdb/env.h"
|
|
|
|
#include "rocksdb/merge_operator.h"
|
|
|
|
#include "rocksdb/statistics.h"
|
|
|
|
#include "rocksdb/status.h"
|
2013-10-29 01:54:09 +01:00
|
|
|
#include "rocksdb/table.h"
|
2015-09-02 22:58:22 +02:00
|
|
|
#include "rocksdb/version.h"
|
2016-06-21 03:01:03 +02:00
|
|
|
#include "rocksdb/write_buffer_manager.h"
|
2011-03-18 23:37:00 +01:00
|
|
|
#include "table/block.h"
|
2013-11-20 07:00:48 +01:00
|
|
|
#include "table/block_based_table_factory.h"
|
2017-02-03 01:38:40 +01:00
|
|
|
#include "table/merging_iterator.h"
|
2014-01-28 06:58:46 +01:00
|
|
|
#include "table/table_builder.h"
|
2011-03-18 23:37:00 +01:00
|
|
|
#include "table/two_level_iterator.h"
|
2017-04-04 03:27:24 +02:00
|
|
|
#include "util/auto_roll_logger.h"
|
2014-01-28 06:58:46 +01:00
|
|
|
#include "util/autovector.h"
|
2013-02-15 20:53:17 +01:00
|
|
|
#include "util/build_version.h"
|
2011-03-18 23:37:00 +01:00
|
|
|
#include "util/coding.h"
|
2015-04-06 21:50:44 +02:00
|
|
|
#include "util/compression.h"
|
2015-07-11 02:35:27 +02:00
|
|
|
#include "util/crc32c.h"
|
Move rate_limiter, write buffering, most perf context instrumentation and most random kill out of Env
Summary: We want to keep Env a think layer for better portability. Less platform dependent codes should be moved out of Env. In this patch, I create a wrapper of file readers and writers, and put rate limiting, write buffering, as well as most perf context instrumentation and random kill out of Env. It will make it easier to maintain multiple Env in the future.
Test Plan: Run all existing unit tests.
Reviewers: anthony, kradhakrishnan, IslamAbdelRahman, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D42321
2015-07-18 01:16:11 +02:00
|
|
|
#include "util/file_reader_writer.h"
|
2014-12-16 06:48:16 +01:00
|
|
|
#include "util/file_util.h"
|
2017-04-04 03:27:24 +02:00
|
|
|
#include "util/filename.h"
|
2014-03-10 06:01:13 +01:00
|
|
|
#include "util/log_buffer.h"
|
2015-09-02 22:58:22 +02:00
|
|
|
#include "util/logging.h"
|
2011-03-18 23:37:00 +01:00
|
|
|
#include "util/mutexlock.h"
|
2016-04-18 20:11:51 +02:00
|
|
|
#include "util/sst_file_manager_impl.h"
|
2013-02-15 20:53:17 +01:00
|
|
|
#include "util/stop_watch.h"
|
2014-11-25 05:44:49 +01:00
|
|
|
#include "util/string_util.h"
|
2015-09-02 22:58:22 +02:00
|
|
|
#include "util/sync_point.h"
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2013-10-04 06:49:15 +02:00
|
|
|
namespace rocksdb {
|
2014-04-09 18:56:17 +02:00
|
|
|
const std::string kDefaultColumnFamilyName("default");
|
2014-10-07 19:40:57 +02:00
|
|
|
void DumpRocksDBBuildVersion(Logger * log);
|
2012-09-18 22:18:32 +02:00
|
|
|
|
2016-05-17 22:11:56 +02:00
|
|
|
CompressionType GetCompressionFlush(
|
|
|
|
const ImmutableCFOptions& ioptions,
|
|
|
|
const MutableCFOptions& mutable_cf_options) {
|
2013-12-19 19:02:53 +01:00
|
|
|
// Compressing memtable flushes might not help unless the sequential load
|
|
|
|
// optimization is used for leveled compaction. Otherwise the CPU and
|
|
|
|
// latency overhead is not offset by saving much space.
|
2014-09-09 00:04:34 +02:00
|
|
|
if (ioptions.compaction_style == kCompactionStyleUniversal) {
|
2016-10-29 02:28:57 +02:00
|
|
|
if (ioptions.compaction_options_universal.compression_size_percent < 0) {
|
|
|
|
return mutable_cf_options.compression;
|
|
|
|
} else {
|
|
|
|
return kNoCompression;
|
|
|
|
}
|
|
|
|
} else if (!ioptions.compression_per_level.empty()) {
|
|
|
|
// For leveled compress when min_level_to_compress != 0.
|
|
|
|
return ioptions.compression_per_level[0];
|
2013-12-19 19:02:53 +01:00
|
|
|
} else {
|
2016-05-17 22:11:56 +02:00
|
|
|
return mutable_cf_options.compression;
|
2013-12-19 19:02:53 +01:00
|
|
|
}
|
|
|
|
}
|
2015-03-26 19:22:20 +01:00
|
|
|
|
2017-04-06 02:14:05 +02:00
|
|
|
namespace {
|
2015-07-11 02:35:27 +02:00
|
|
|
void DumpSupportInfo(Logger* logger) {
|
2017-03-16 03:22:52 +01:00
|
|
|
ROCKS_LOG_HEADER(logger, "Compression algorithms supported:");
|
|
|
|
ROCKS_LOG_HEADER(logger, "\tSnappy supported: %d", Snappy_Supported());
|
|
|
|
ROCKS_LOG_HEADER(logger, "\tZlib supported: %d", Zlib_Supported());
|
|
|
|
ROCKS_LOG_HEADER(logger, "\tBzip supported: %d", BZip2_Supported());
|
|
|
|
ROCKS_LOG_HEADER(logger, "\tLZ4 supported: %d", LZ4_Supported());
|
|
|
|
ROCKS_LOG_HEADER(logger, "\tZSTD supported: %d", ZSTD_Supported());
|
|
|
|
ROCKS_LOG_HEADER(logger, "Fast CRC32 supported: %d",
|
|
|
|
crc32c::IsFastCrc32Supported());
|
2015-03-26 19:22:20 +01:00
|
|
|
}
|
2017-04-06 02:14:05 +02:00
|
|
|
} // namespace
|
2013-12-19 19:02:53 +01:00
|
|
|
|
2014-02-05 22:12:23 +01:00
|
|
|
DBImpl::DBImpl(const DBOptions& options, const std::string& dbname)
|
2011-03-18 23:37:00 +01:00
|
|
|
: env_(options.env),
|
2012-11-06 04:18:49 +01:00
|
|
|
dbname_(dbname),
|
2016-10-14 21:25:39 +02:00
|
|
|
initial_db_options_(SanitizeOptions(dbname, options)),
|
|
|
|
immutable_db_options_(initial_db_options_),
|
|
|
|
mutable_db_options_(initial_db_options_),
|
2016-09-24 01:34:04 +02:00
|
|
|
stats_(immutable_db_options_.statistics.get()),
|
2013-02-15 20:53:17 +01:00
|
|
|
db_lock_(nullptr),
|
2016-09-24 01:34:04 +02:00
|
|
|
mutex_(stats_, env_, DB_MUTEX_WAIT_MICROS,
|
|
|
|
immutable_db_options_.use_adaptive_mutex),
|
2014-10-27 22:50:21 +01:00
|
|
|
shutting_down_(false),
|
2011-03-18 23:37:00 +01:00
|
|
|
bg_cv_(&mutex_),
|
2011-06-22 04:36:45 +02:00
|
|
|
logfile_number_(0),
|
2015-01-27 00:48:59 +01:00
|
|
|
log_dir_synced_(false),
|
2014-04-15 18:57:25 +02:00
|
|
|
log_empty_(true),
|
2014-02-11 02:04:44 +01:00
|
|
|
default_cf_handle_(nullptr),
|
[wal changes 2/3] write with sync=true syncs previous unsynced wals to prevent illegal data loss
Summary:
I'll just copy internal task summary here:
"
This sequence will cause data loss in the middle after an sync write:
non-sync write key 1
flush triggered, not yet scheduled
sync write key 2
system crash
After rebooting, users might see key 2 but not key 1, which violates the API of sync write.
This can be reproduced using unit test FaultInjectionTest::DISABLED_WriteOptionSyncTest.
One way to fix it is for a sync write, if there is outstanding unsynced log files, we need to syc them too.
"
This diff should be considered together with the next diff D40905; in isolation this fix probably could be a little simpler.
Test Plan: `make check`; added a test for that (DBTest.SyncingPreviousLogs) before noticing FaultInjectionTest.WriteOptionSyncTest (keeping both since mine asserts a bit more); both tests fail without this diff; for D40905 stacked on top of this diff, ran tests with ASAN, TSAN and valgrind
Reviewers: rven, yhchiang, IslamAbdelRahman, anthony, kradhakrishnan, igor, sdong
Reviewed By: sdong
Subscribers: dhruba
Differential Revision: https://reviews.facebook.net/D40899
2015-07-22 12:28:08 +02:00
|
|
|
log_sync_cv_(&mutex_),
|
2014-04-30 20:33:40 +02:00
|
|
|
total_log_size_(0),
|
|
|
|
max_total_in_memory_state_(0),
|
2014-12-11 03:39:09 +01:00
|
|
|
is_snapshot_supported_(true),
|
2016-09-24 01:34:04 +02:00
|
|
|
write_buffer_manager_(immutable_db_options_.write_buffer_manager.get()),
|
|
|
|
write_thread_(immutable_db_options_.enable_write_thread_adaptive_yield
|
|
|
|
? immutable_db_options_.write_thread_max_yield_usec
|
support for concurrent adds to memtable
Summary:
This diff adds support for concurrent adds to the skiplist memtable
implementations. Memory allocation is made thread-safe by the addition of
a spinlock, with small per-core buffers to avoid contention. Concurrent
memtable writes are made via an additional method and don't impose a
performance overhead on the non-concurrent case, so parallelism can be
selected on a per-batch basis.
Write thread synchronization is an increasing bottleneck for higher levels
of concurrency, so this diff adds --enable_write_thread_adaptive_yield
(default off). This feature causes threads joining a write batch
group to spin for a short time (default 100 usec) using sched_yield,
rather than going to sleep on a mutex. If the timing of the yield calls
indicates that another thread has actually run during the yield then
spinning is avoided. This option improves performance for concurrent
situations even without parallel adds, although it has the potential to
increase CPU usage (and the heuristic adaptation is not yet mature).
Parallel writes are not currently compatible with
inplace updates, update callbacks, or delete filtering.
Enable it with --allow_concurrent_memtable_write (and
--enable_write_thread_adaptive_yield). Parallel memtable writes
are performance neutral when there is no actual parallelism, and in
my experiments (SSD server-class Linux and varying contention and key
sizes for fillrandom) they are always a performance win when there is
more than one thread.
Statistics are updated earlier in the write path, dropping the number
of DB mutex acquisitions from 2 to 1 for almost all cases.
This diff was motivated and inspired by Yahoo's cLSM work. It is more
conservative than cLSM: RocksDB's write batch group leader role is
preserved (along with all of the existing flush and write throttling
logic) and concurrent writers are blocked until all memtable insertions
have completed and the sequence number has been advanced, to preserve
linearizability.
My test config is "db_bench -benchmarks=fillrandom -threads=$T
-batch_size=1 -memtablerep=skip_list -value_size=100 --num=1000000/$T
-level0_slowdown_writes_trigger=9999 -level0_stop_writes_trigger=9999
-disable_auto_compactions --max_write_buffer_number=8
-max_background_flushes=8 --disable_wal --write_buffer_size=160000000
--block_size=16384 --allow_concurrent_memtable_write" on a two-socket
Xeon E5-2660 @ 2.2Ghz with lots of memory and an SSD hard drive. With 1
thread I get ~440Kops/sec. Peak performance for 1 socket (numactl
-N1) is slightly more than 1Mops/sec, at 16 threads. Peak performance
across both sockets happens at 30 threads, and is ~900Kops/sec, although
with fewer threads there is less performance loss when the system has
background work.
Test Plan:
1. concurrent stress tests for InlineSkipList and DynamicBloom
2. make clean; make check
3. make clean; DISABLE_JEMALLOC=1 make valgrind_check; valgrind db_bench
4. make clean; COMPILE_WITH_TSAN=1 make all check; db_bench
5. make clean; COMPILE_WITH_ASAN=1 make all check; db_bench
6. make clean; OPT=-DROCKSDB_LITE make check
7. verify no perf regressions when disabled
Reviewers: igor, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, IslamAbdelRahman, anthony, yhchiang, rven, sdong, guyg8, kradhakrishnan, dhruba
Differential Revision: https://reviews.facebook.net/D50589
2015-08-15 01:59:07 +02:00
|
|
|
: 0,
|
2016-09-24 01:34:04 +02:00
|
|
|
immutable_db_options_.write_thread_slow_yield_usec),
|
2016-11-13 00:43:33 +01:00
|
|
|
write_controller_(mutable_db_options_.delayed_write_rate),
|
2015-05-16 00:52:51 +02:00
|
|
|
last_batch_group_size_(0),
|
Rewritten system for scheduling background work
Summary:
When scaling to higher number of column families, the worst bottleneck was MaybeScheduleFlushOrCompaction(), which did a for loop over all column families while holding a mutex. This patch addresses the issue.
The approach is similar to our earlier efforts: instead of a pull-model, where we do something for every column family, we can do a push-based model -- when we detect that column family is ready to be flushed/compacted, we add it to the flush_queue_/compaction_queue_. That way we don't need to loop over every column family in MaybeScheduleFlushOrCompaction.
Here are the performance results:
Command:
./db_bench --write_buffer_size=268435456 --db_write_buffer_size=268435456 --db=/fast-rocksdb-tmp/rocks_lots_of_cf --use_existing_db=0 --open_files=55000 --statistics=1 --histogram=1 --disable_data_sync=1 --max_write_buffer_number=2 --sync=0 --benchmarks=fillrandom --threads=16 --num_column_families=5000 --disable_wal=1 --max_background_flushes=16 --max_background_compactions=16 --level0_file_num_compaction_trigger=2 --level0_slowdown_writes_trigger=2 --level0_stop_writes_trigger=3 --hard_rate_limit=1 --num=33333333 --writes=33333333
Before the patch:
fillrandom : 26.950 micros/op 37105 ops/sec; 4.1 MB/s
After the patch:
fillrandom : 17.404 micros/op 57456 ops/sec; 6.4 MB/s
Next bottleneck is VersionSet::AddLiveFiles, which is painfully slow when we have a lot of files. This is coming in the next patch, but when I removed that code, here's what I got:
fillrandom : 7.590 micros/op 131758 ops/sec; 14.6 MB/s
Test Plan:
make check
two stress tests:
Big number of compactions and flushes:
./db_stress --threads=30 --ops_per_thread=20000000 --max_key=10000 --column_families=20 --clear_column_family_one_in=10000000 --verify_before_write=0 --reopen=15 --max_background_compactions=10 --max_background_flushes=10 --db=/fast-rocksdb-tmp/db_stress --prefixpercent=0 --iterpercent=0 --writepercent=75 --db_write_buffer_size=2000000
max_background_flushes=0, to verify that this case also works correctly
./db_stress --threads=30 --ops_per_thread=2000000 --max_key=10000 --column_families=20 --clear_column_family_one_in=10000000 --verify_before_write=0 --reopen=3 --max_background_compactions=3 --max_background_flushes=0 --db=/fast-rocksdb-tmp/db_stress --prefixpercent=0 --iterpercent=0 --writepercent=75 --db_write_buffer_size=2000000
Reviewers: ljin, rven, yhchiang, sdong
Reviewed By: sdong
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D30123
2014-12-19 20:38:12 +01:00
|
|
|
unscheduled_flushes_(0),
|
|
|
|
unscheduled_compactions_(0),
|
2012-10-19 23:00:53 +02:00
|
|
|
bg_compaction_scheduled_(0),
|
2015-10-17 09:16:36 +02:00
|
|
|
num_running_compactions_(0),
|
2013-09-13 23:38:37 +02:00
|
|
|
bg_flush_scheduled_(0),
|
2015-10-17 09:16:36 +02:00
|
|
|
num_running_flushes_(0),
|
2016-06-22 03:41:23 +02:00
|
|
|
bg_purge_scheduled_(0),
|
2014-01-02 12:33:42 +01:00
|
|
|
disable_delete_obsolete_files_(0),
|
2016-12-05 23:09:35 +01:00
|
|
|
delete_obsolete_files_last_run_(env_->NowMicros()),
|
2013-05-11 00:21:04 +02:00
|
|
|
last_stats_dump_time_microsec_(0),
|
2015-02-12 18:54:48 +01:00
|
|
|
next_job_id_(1),
|
2016-02-09 20:20:22 +01:00
|
|
|
has_unpersisted_data_(false),
|
2017-01-20 00:21:07 +01:00
|
|
|
unable_to_flush_oldest_log_(false),
|
2016-09-24 01:34:04 +02:00
|
|
|
env_options_(BuildDBOptions(immutable_db_options_, mutable_db_options_)),
|
2016-10-21 02:05:32 +02:00
|
|
|
num_running_ingest_file_(0),
|
2014-10-30 01:43:37 +01:00
|
|
|
#ifndef ROCKSDB_LITE
|
2016-09-24 01:34:04 +02:00
|
|
|
wal_manager_(immutable_db_options_, env_options_),
|
2014-10-30 01:43:37 +01:00
|
|
|
#endif // ROCKSDB_LITE
|
2016-09-24 01:34:04 +02:00
|
|
|
event_logger_(immutable_db_options_.info_log.get()),
|
2015-10-02 22:17:34 +02:00
|
|
|
bg_work_paused_(0),
|
Fix a bug where flush does not happen when a manual compaction is running
Summary:
Currently, when rocksdb tries to run manual compaction to refit data into a level,
there's a ReFitLevel() process that requires no bg work is currently running.
When RocksDB plans to ReFitLevel(), it will do the following:
1. pause scheduling new bg work.
2. wait until all bg work finished
3. do the ReFitLevel()
4. unpause scheduling new bg work.
However, as it pause scheduling new bg work at step one and waiting for all bg work
finished in step 2, RocksDB will stop flushing until all bg work is done (which
could take a long time.)
This patch fix this issue by changing the way ReFitLevel() pause the background work:
1. pause scheduling compaction.
2. wait until all bg work finished.
3. pause scheduling flush
4. do ReFitLevel()
5. unpause both flush and compaction.
The major difference is that. We only pause scheduling compaction in step 1 and wait
for all bg work finished in step 2. This prevent flush being blocked for a long time.
Although there's a very rare case that ReFitLevel() might be in starvation in step 2,
but it's less likely the case as flush typically finish very fast.
Test Plan: existing test.
Reviewers: anthony, IslamAbdelRahman, kradhakrishnan, sdong
Reviewed By: sdong
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D55029
2016-03-04 23:24:52 +01:00
|
|
|
bg_compaction_paused_(0),
|
2014-02-27 20:38:55 +01:00
|
|
|
refitting_level_(false),
|
2015-06-02 00:32:23 +02:00
|
|
|
opened_successfully_(false) {
|
2012-09-06 02:44:13 +02:00
|
|
|
env_->GetAbsolutePath(dbname, &db_absolute_path_);
|
2013-03-02 21:56:04 +01:00
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
// Reserve ten files or so for other uses and give the rest to TableCache.
|
2014-01-07 05:29:17 +01:00
|
|
|
// Give a large number for setting of "infinite" open files.
|
2016-09-24 01:34:04 +02:00
|
|
|
const int table_cache_size = (immutable_db_options_.max_open_files == -1)
|
|
|
|
? 4194304
|
|
|
|
: immutable_db_options_.max_open_files - 10;
|
|
|
|
table_cache_ = NewLRUCache(table_cache_size,
|
|
|
|
immutable_db_options_.table_cache_numshardbits);
|
2014-02-07 00:42:16 +01:00
|
|
|
|
2016-09-24 01:34:04 +02:00
|
|
|
versions_.reset(new VersionSet(dbname_, &immutable_db_options_, env_options_,
|
2016-06-21 03:01:03 +02:00
|
|
|
table_cache_.get(), write_buffer_manager_,
|
2014-12-02 21:09:20 +01:00
|
|
|
&write_controller_));
|
support for concurrent adds to memtable
Summary:
This diff adds support for concurrent adds to the skiplist memtable
implementations. Memory allocation is made thread-safe by the addition of
a spinlock, with small per-core buffers to avoid contention. Concurrent
memtable writes are made via an additional method and don't impose a
performance overhead on the non-concurrent case, so parallelism can be
selected on a per-batch basis.
Write thread synchronization is an increasing bottleneck for higher levels
of concurrency, so this diff adds --enable_write_thread_adaptive_yield
(default off). This feature causes threads joining a write batch
group to spin for a short time (default 100 usec) using sched_yield,
rather than going to sleep on a mutex. If the timing of the yield calls
indicates that another thread has actually run during the yield then
spinning is avoided. This option improves performance for concurrent
situations even without parallel adds, although it has the potential to
increase CPU usage (and the heuristic adaptation is not yet mature).
Parallel writes are not currently compatible with
inplace updates, update callbacks, or delete filtering.
Enable it with --allow_concurrent_memtable_write (and
--enable_write_thread_adaptive_yield). Parallel memtable writes
are performance neutral when there is no actual parallelism, and in
my experiments (SSD server-class Linux and varying contention and key
sizes for fillrandom) they are always a performance win when there is
more than one thread.
Statistics are updated earlier in the write path, dropping the number
of DB mutex acquisitions from 2 to 1 for almost all cases.
This diff was motivated and inspired by Yahoo's cLSM work. It is more
conservative than cLSM: RocksDB's write batch group leader role is
preserved (along with all of the existing flush and write throttling
logic) and concurrent writers are blocked until all memtable insertions
have completed and the sequence number has been advanced, to preserve
linearizability.
My test config is "db_bench -benchmarks=fillrandom -threads=$T
-batch_size=1 -memtablerep=skip_list -value_size=100 --num=1000000/$T
-level0_slowdown_writes_trigger=9999 -level0_stop_writes_trigger=9999
-disable_auto_compactions --max_write_buffer_number=8
-max_background_flushes=8 --disable_wal --write_buffer_size=160000000
--block_size=16384 --allow_concurrent_memtable_write" on a two-socket
Xeon E5-2660 @ 2.2Ghz with lots of memory and an SSD hard drive. With 1
thread I get ~440Kops/sec. Peak performance for 1 socket (numactl
-N1) is slightly more than 1Mops/sec, at 16 threads. Peak performance
across both sockets happens at 30 threads, and is ~900Kops/sec, although
with fewer threads there is less performance loss when the system has
background work.
Test Plan:
1. concurrent stress tests for InlineSkipList and DynamicBloom
2. make clean; make check
3. make clean; DISABLE_JEMALLOC=1 make valgrind_check; valgrind db_bench
4. make clean; COMPILE_WITH_TSAN=1 make all check; db_bench
5. make clean; COMPILE_WITH_ASAN=1 make all check; db_bench
6. make clean; OPT=-DROCKSDB_LITE make check
7. verify no perf regressions when disabled
Reviewers: igor, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, IslamAbdelRahman, anthony, yhchiang, rven, sdong, guyg8, kradhakrishnan, dhruba
Differential Revision: https://reviews.facebook.net/D50589
2015-08-15 01:59:07 +02:00
|
|
|
column_family_memtables_.reset(
|
|
|
|
new ColumnFamilyMemTablesImpl(versions_->GetColumnFamilySet()));
|
2012-08-15 00:20:36 +02:00
|
|
|
|
2016-09-24 01:34:04 +02:00
|
|
|
DumpRocksDBBuildVersion(immutable_db_options_.info_log.get());
|
|
|
|
DumpDBFileSummary(immutable_db_options_, dbname_);
|
|
|
|
immutable_db_options_.Dump(immutable_db_options_.info_log.get());
|
|
|
|
mutable_db_options_.Dump(immutable_db_options_.info_log.get());
|
|
|
|
DumpSupportInfo(immutable_db_options_.info_log.get());
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2015-06-25 23:43:25 +02:00
|
|
|
// Will lock the mutex_, will wait for completion if wait is true
|
2015-05-06 01:54:47 +02:00
|
|
|
void DBImpl::CancelAllBackgroundWork(bool wait) {
|
2015-06-25 23:43:25 +02:00
|
|
|
InstrumentedMutexLock l(&mutex_);
|
CompactFiles, EventListener and GetDatabaseMetaData
Summary:
This diff adds three sets of APIs to RocksDB.
= GetColumnFamilyMetaData =
* This APIs allow users to obtain the current state of a RocksDB instance on one column family.
* See GetColumnFamilyMetaData in include/rocksdb/db.h
= EventListener =
* A virtual class that allows users to implement a set of
call-back functions which will be called when specific
events of a RocksDB instance happens.
* To register EventListener, simply insert an EventListener to ColumnFamilyOptions::listeners
= CompactFiles =
* CompactFiles API inputs a set of file numbers and an output level, and RocksDB
will try to compact those files into the specified level.
= Example =
* Example code can be found in example/compact_files_example.cc, which implements
a simple external compactor using EventListener, GetColumnFamilyMetaData, and
CompactFiles API.
Test Plan:
listener_test
compactor_test
example/compact_files_example
export ROCKSDB_TESTS=CompactFiles
db_test
export ROCKSDB_TESTS=MetaData
db_test
Reviewers: ljin, igor, rven, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D24705
2014-11-07 23:45:18 +01:00
|
|
|
|
2017-03-16 03:22:52 +01:00
|
|
|
ROCKS_LOG_INFO(immutable_db_options_.info_log,
|
|
|
|
"Shutdown: canceling all background work");
|
2017-01-12 00:01:21 +01:00
|
|
|
|
2016-02-09 20:20:22 +01:00
|
|
|
if (!shutting_down_.load(std::memory_order_acquire) &&
|
2017-02-14 03:44:21 +01:00
|
|
|
has_unpersisted_data_.load(std::memory_order_relaxed) &&
|
2016-11-02 23:22:13 +01:00
|
|
|
!mutable_db_options_.avoid_flush_during_shutdown) {
|
2014-03-12 19:50:10 +01:00
|
|
|
for (auto cfd : *versions_->GetColumnFamilySet()) {
|
2015-03-20 01:04:29 +01:00
|
|
|
if (!cfd->IsDropped() && !cfd->mem()->IsEmpty()) {
|
2014-03-12 19:50:10 +01:00
|
|
|
cfd->Ref();
|
|
|
|
mutex_.Unlock();
|
|
|
|
FlushMemTable(cfd, FlushOptions());
|
|
|
|
mutex_.Lock();
|
2014-04-07 23:21:25 +02:00
|
|
|
cfd->Unref();
|
2014-03-12 19:50:10 +01:00
|
|
|
}
|
|
|
|
}
|
2014-04-07 23:21:25 +02:00
|
|
|
versions_->GetColumnFamilySet()->FreeDeadColumnFamilies();
|
2012-11-06 20:21:57 +01:00
|
|
|
}
|
2016-08-25 21:24:22 +02:00
|
|
|
|
|
|
|
shutting_down_.store(true, std::memory_order_release);
|
|
|
|
bg_cv_.SignalAll();
|
|
|
|
if (!wait) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
// Wait for background work to finish
|
|
|
|
while (bg_compaction_scheduled_ || bg_flush_scheduled_) {
|
|
|
|
bg_cv_.Wait();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
DBImpl::~DBImpl() {
|
2015-06-25 23:43:25 +02:00
|
|
|
// CancelAllBackgroundWork called with false means we just set the shutdown
|
|
|
|
// marker. After this we do a variant of the waiting and unschedule work
|
2015-05-06 01:54:47 +02:00
|
|
|
// (to consider: moving all the waiting into CancelAllBackgroundWork(true))
|
|
|
|
CancelAllBackgroundWork(false);
|
2015-03-17 02:49:14 +01:00
|
|
|
int compactions_unscheduled = env_->UnSchedule(this, Env::Priority::LOW);
|
|
|
|
int flushes_unscheduled = env_->UnSchedule(this, Env::Priority::HIGH);
|
|
|
|
mutex_.Lock();
|
|
|
|
bg_compaction_scheduled_ -= compactions_unscheduled;
|
|
|
|
bg_flush_scheduled_ -= flushes_unscheduled;
|
|
|
|
|
|
|
|
// Wait for background work to finish
|
2016-06-22 03:41:23 +02:00
|
|
|
while (bg_compaction_scheduled_ || bg_flush_scheduled_ ||
|
|
|
|
bg_purge_scheduled_) {
|
|
|
|
TEST_SYNC_POINT("DBImpl::~DBImpl:WaitJob");
|
2015-03-17 02:49:14 +01:00
|
|
|
bg_cv_.Wait();
|
|
|
|
}
|
2015-06-11 23:18:02 +02:00
|
|
|
EraseThreadStatusDbInfo();
|
2014-09-11 03:46:09 +02:00
|
|
|
flush_scheduler_.Clear();
|
|
|
|
|
Rewritten system for scheduling background work
Summary:
When scaling to higher number of column families, the worst bottleneck was MaybeScheduleFlushOrCompaction(), which did a for loop over all column families while holding a mutex. This patch addresses the issue.
The approach is similar to our earlier efforts: instead of a pull-model, where we do something for every column family, we can do a push-based model -- when we detect that column family is ready to be flushed/compacted, we add it to the flush_queue_/compaction_queue_. That way we don't need to loop over every column family in MaybeScheduleFlushOrCompaction.
Here are the performance results:
Command:
./db_bench --write_buffer_size=268435456 --db_write_buffer_size=268435456 --db=/fast-rocksdb-tmp/rocks_lots_of_cf --use_existing_db=0 --open_files=55000 --statistics=1 --histogram=1 --disable_data_sync=1 --max_write_buffer_number=2 --sync=0 --benchmarks=fillrandom --threads=16 --num_column_families=5000 --disable_wal=1 --max_background_flushes=16 --max_background_compactions=16 --level0_file_num_compaction_trigger=2 --level0_slowdown_writes_trigger=2 --level0_stop_writes_trigger=3 --hard_rate_limit=1 --num=33333333 --writes=33333333
Before the patch:
fillrandom : 26.950 micros/op 37105 ops/sec; 4.1 MB/s
After the patch:
fillrandom : 17.404 micros/op 57456 ops/sec; 6.4 MB/s
Next bottleneck is VersionSet::AddLiveFiles, which is painfully slow when we have a lot of files. This is coming in the next patch, but when I removed that code, here's what I got:
fillrandom : 7.590 micros/op 131758 ops/sec; 14.6 MB/s
Test Plan:
make check
two stress tests:
Big number of compactions and flushes:
./db_stress --threads=30 --ops_per_thread=20000000 --max_key=10000 --column_families=20 --clear_column_family_one_in=10000000 --verify_before_write=0 --reopen=15 --max_background_compactions=10 --max_background_flushes=10 --db=/fast-rocksdb-tmp/db_stress --prefixpercent=0 --iterpercent=0 --writepercent=75 --db_write_buffer_size=2000000
max_background_flushes=0, to verify that this case also works correctly
./db_stress --threads=30 --ops_per_thread=2000000 --max_key=10000 --column_families=20 --clear_column_family_one_in=10000000 --verify_before_write=0 --reopen=3 --max_background_compactions=3 --max_background_flushes=0 --db=/fast-rocksdb-tmp/db_stress --prefixpercent=0 --iterpercent=0 --writepercent=75 --db_write_buffer_size=2000000
Reviewers: ljin, rven, yhchiang, sdong
Reviewed By: sdong
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D30123
2014-12-19 20:38:12 +01:00
|
|
|
while (!flush_queue_.empty()) {
|
|
|
|
auto cfd = PopFirstFromFlushQueue();
|
|
|
|
if (cfd->Unref()) {
|
|
|
|
delete cfd;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
while (!compaction_queue_.empty()) {
|
|
|
|
auto cfd = PopFirstFromCompactionQueue();
|
|
|
|
if (cfd->Unref()) {
|
|
|
|
delete cfd;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-03-11 22:52:17 +01:00
|
|
|
if (default_cf_handle_ != nullptr) {
|
|
|
|
// we need to delete handle outside of lock because it does its own locking
|
|
|
|
mutex_.Unlock();
|
|
|
|
delete default_cf_handle_;
|
|
|
|
mutex_.Lock();
|
2014-03-04 18:03:56 +01:00
|
|
|
}
|
|
|
|
|
2014-09-24 22:12:16 +02:00
|
|
|
// Clean up obsolete files due to SuperVersion release.
|
|
|
|
// (1) Need to delete to obsolete files before closing because RepairDB()
|
|
|
|
// scans all existing files in the file system and builds manifest file.
|
|
|
|
// Keeping obsolete files confuses the repair process.
|
|
|
|
// (2) Need to check if we Open()/Recover() the DB successfully before
|
|
|
|
// deleting because if VersionSet recover fails (may be due to corrupted
|
|
|
|
// manifest file), it is not able to identify live files correctly. As a
|
|
|
|
// result, all "live" files can get deleted by accident. However, corrupted
|
|
|
|
// manifest is recoverable by RepairDB().
|
|
|
|
if (opened_successfully_) {
|
2015-02-12 18:54:48 +01:00
|
|
|
JobContext job_context(next_job_id_.fetch_add(1));
|
2014-10-28 19:54:33 +01:00
|
|
|
FindObsoleteFiles(&job_context, true);
|
2015-06-04 04:57:01 +02:00
|
|
|
|
|
|
|
mutex_.Unlock();
|
2014-09-24 22:12:16 +02:00
|
|
|
// manifest number starting from 2
|
2014-10-28 19:54:33 +01:00
|
|
|
job_context.manifest_file_number = 1;
|
|
|
|
if (job_context.HaveSomethingToDelete()) {
|
|
|
|
PurgeObsoleteFiles(job_context);
|
2014-02-27 20:38:55 +01:00
|
|
|
}
|
2014-11-15 00:43:10 +01:00
|
|
|
job_context.Clean();
|
2015-06-04 04:57:01 +02:00
|
|
|
mutex_.Lock();
|
2014-02-27 20:38:55 +01:00
|
|
|
}
|
|
|
|
|
2015-03-30 21:04:10 +02:00
|
|
|
for (auto l : logs_to_free_) {
|
|
|
|
delete l;
|
|
|
|
}
|
2015-08-06 05:51:27 +02:00
|
|
|
for (auto& log : logs_) {
|
|
|
|
log.ClearWriter();
|
|
|
|
}
|
[wal changes 2/3] write with sync=true syncs previous unsynced wals to prevent illegal data loss
Summary:
I'll just copy internal task summary here:
"
This sequence will cause data loss in the middle after an sync write:
non-sync write key 1
flush triggered, not yet scheduled
sync write key 2
system crash
After rebooting, users might see key 2 but not key 1, which violates the API of sync write.
This can be reproduced using unit test FaultInjectionTest::DISABLED_WriteOptionSyncTest.
One way to fix it is for a sync write, if there is outstanding unsynced log files, we need to syc them too.
"
This diff should be considered together with the next diff D40905; in isolation this fix probably could be a little simpler.
Test Plan: `make check`; added a test for that (DBTest.SyncingPreviousLogs) before noticing FaultInjectionTest.WriteOptionSyncTest (keeping both since mine asserts a bit more); both tests fail without this diff; for D40905 stacked on top of this diff, ran tests with ASAN, TSAN and valgrind
Reviewers: rven, yhchiang, IslamAbdelRahman, anthony, kradhakrishnan, igor, sdong
Reviewed By: sdong
Subscribers: dhruba
Differential Revision: https://reviews.facebook.net/D40899
2015-07-22 12:28:08 +02:00
|
|
|
logs_.clear();
|
2015-03-30 21:04:10 +02:00
|
|
|
|
Adding pin_l0_filter_and_index_blocks_in_cache feature and related fixes.
Summary:
When a block based table file is opened, if prefetch_index_and_filter is true, it will prefetch the index and filter blocks, putting them into the block cache.
What this feature adds: when a L0 block based table file is opened, if pin_l0_filter_and_index_blocks_in_cache is true in the options (and prefetch_index_and_filter is true), then the filter and index blocks aren't released back to the block cache at the end of BlockBasedTableReader::Open(). Instead the table reader takes ownership of them, hence pinning them, ie. the LRU cache will never push them out. Meanwhile in the table reader, further accesses will not hit the block cache, thus avoiding lock contention.
Test Plan:
'export TEST_TMPDIR=/dev/shm/ && DISABLE_JEMALLOC=1 OPT=-g make all valgrind_check -j32' is OK.
I didn't run the Java tests, I don't have Java set up on my devserver.
Reviewers: sdong
Reviewed By: sdong
Subscribers: andrewkr, dhruba
Differential Revision: https://reviews.facebook.net/D56133
2016-04-01 19:42:39 +02:00
|
|
|
// Table cache may have table handles holding blocks from the block cache.
|
|
|
|
// We need to release them before the block cache is destroyed. The block
|
|
|
|
// cache may be destroyed inside versions_.reset(), when column family data
|
|
|
|
// list is destroyed, so leaving handles in table cache after
|
|
|
|
// versions_.reset() may cause issues.
|
|
|
|
// Here we clean all unreferenced handles in table cache.
|
|
|
|
// Now we assume all user queries have finished, so only version set itself
|
|
|
|
// can possibly hold the blocks from block cache. After releasing unreferenced
|
|
|
|
// handles here, only handles held by version set left and inside
|
|
|
|
// versions_.reset(), we will release them. There, we need to make sure every
|
|
|
|
// time a handle is released, we erase it from the cache too. By doing that,
|
|
|
|
// we can guarantee that after versions_.reset(), table cache is empty
|
|
|
|
// so the cache can be safely destroyed.
|
|
|
|
table_cache_->EraseUnRefEntries();
|
|
|
|
|
2016-05-17 01:32:55 +02:00
|
|
|
for (auto& txn_entry : recovered_transactions_) {
|
|
|
|
delete txn_entry.second;
|
|
|
|
}
|
|
|
|
|
2014-02-07 00:42:16 +01:00
|
|
|
// versions need to be destroyed before table_cache since it can hold
|
2014-01-07 05:29:17 +01:00
|
|
|
// references to table_cache.
|
|
|
|
versions_.reset();
|
2014-03-04 18:03:56 +01:00
|
|
|
mutex_.Unlock();
|
2014-03-11 22:52:17 +01:00
|
|
|
if (db_lock_ != nullptr) {
|
|
|
|
env_->UnlockFile(db_lock_);
|
|
|
|
}
|
2014-02-07 00:42:16 +01:00
|
|
|
|
2017-03-16 03:22:52 +01:00
|
|
|
ROCKS_LOG_INFO(immutable_db_options_.info_log, "Shutdown complete");
|
2016-09-24 01:34:04 +02:00
|
|
|
LogFlush(immutable_db_options_.info_log);
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
void DBImpl::MaybeIgnoreError(Status* s) const {
|
2016-09-24 01:34:04 +02:00
|
|
|
if (s->ok() || immutable_db_options_.paranoid_checks) {
|
2011-03-18 23:37:00 +01:00
|
|
|
// No change needed
|
|
|
|
} else {
|
2017-03-16 03:22:52 +01:00
|
|
|
ROCKS_LOG_WARN(immutable_db_options_.info_log, "Ignoring error %s",
|
|
|
|
s->ToString().c_str());
|
2011-03-18 23:37:00 +01:00
|
|
|
*s = Status::OK();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2012-11-26 22:56:45 +01:00
|
|
|
const Status DBImpl::CreateArchivalDirectory() {
|
2016-09-24 01:34:04 +02:00
|
|
|
if (immutable_db_options_.wal_ttl_seconds > 0 ||
|
|
|
|
immutable_db_options_.wal_size_limit_mb > 0) {
|
|
|
|
std::string archivalPath = ArchivalDirectory(immutable_db_options_.wal_dir);
|
2012-11-26 22:56:45 +01:00
|
|
|
return env_->CreateDirIfMissing(archivalPath);
|
|
|
|
}
|
|
|
|
return Status::OK();
|
|
|
|
}
|
|
|
|
|
2013-05-28 21:35:43 +02:00
|
|
|
void DBImpl::PrintStatistics() {
|
2016-09-24 01:34:04 +02:00
|
|
|
auto dbstats = immutable_db_options_.statistics.get();
|
2013-05-28 21:35:43 +02:00
|
|
|
if (dbstats) {
|
2017-03-16 03:22:52 +01:00
|
|
|
ROCKS_LOG_WARN(immutable_db_options_.info_log, "STATISTICS:\n %s",
|
|
|
|
dbstats->ToString().c_str());
|
2013-05-28 21:35:43 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-04-28 03:25:19 +02:00
|
|
|
#ifndef ROCKSDB_LITE
|
2016-04-28 01:23:33 +02:00
|
|
|
#ifdef ROCKSDB_JEMALLOC
|
|
|
|
typedef struct {
|
|
|
|
char* cur;
|
|
|
|
char* end;
|
|
|
|
} MallocStatus;
|
|
|
|
|
|
|
|
static void GetJemallocStatus(void* mstat_arg, const char* status) {
|
|
|
|
MallocStatus* mstat = reinterpret_cast<MallocStatus*>(mstat_arg);
|
|
|
|
size_t status_len = status ? strlen(status) : 0;
|
|
|
|
size_t buf_size = (size_t)(mstat->end - mstat->cur);
|
|
|
|
if (!status_len || status_len > buf_size) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2016-04-29 01:04:31 +02:00
|
|
|
snprintf(mstat->cur, buf_size, "%s", status);
|
2016-04-28 01:23:33 +02:00
|
|
|
mstat->cur += status_len;
|
|
|
|
}
|
|
|
|
#endif // ROCKSDB_JEMALLOC
|
|
|
|
|
|
|
|
static void DumpMallocStats(std::string* stats) {
|
|
|
|
#ifdef ROCKSDB_JEMALLOC
|
|
|
|
MallocStatus mstat;
|
2017-04-03 20:19:29 +02:00
|
|
|
const unsigned int kMallocStatusLen = 1000000;
|
2017-02-28 02:22:50 +01:00
|
|
|
std::unique_ptr<char[]> buf{new char[kMallocStatusLen + 1]};
|
2016-04-28 01:23:33 +02:00
|
|
|
mstat.cur = buf.get();
|
|
|
|
mstat.end = buf.get() + kMallocStatusLen;
|
2017-04-06 22:54:49 +02:00
|
|
|
je_malloc_stats_print(GetJemallocStatus, &mstat, "");
|
2016-04-28 01:23:33 +02:00
|
|
|
stats->append(buf.get());
|
|
|
|
#endif // ROCKSDB_JEMALLOC
|
|
|
|
}
|
2016-04-28 03:25:19 +02:00
|
|
|
#endif // !ROCKSDB_LITE
|
2016-04-28 01:23:33 +02:00
|
|
|
|
2013-05-11 00:21:04 +02:00
|
|
|
void DBImpl::MaybeDumpStats() {
|
2017-03-21 06:50:56 +01:00
|
|
|
mutex_.Lock();
|
|
|
|
unsigned int stats_dump_period_sec =
|
|
|
|
mutable_db_options_.stats_dump_period_sec;
|
|
|
|
mutex_.Unlock();
|
|
|
|
if (stats_dump_period_sec == 0) return;
|
2013-05-24 21:52:45 +02:00
|
|
|
|
|
|
|
const uint64_t now_micros = env_->NowMicros();
|
|
|
|
|
2017-03-21 06:50:56 +01:00
|
|
|
if (last_stats_dump_time_microsec_ + stats_dump_period_sec * 1000000 <=
|
2016-09-24 01:34:04 +02:00
|
|
|
now_micros) {
|
2013-05-24 21:52:45 +02:00
|
|
|
// Multiple threads could race in here simultaneously.
|
|
|
|
// However, the last one will update last_stats_dump_time_microsec_
|
|
|
|
// atomically. We could see more than one dump during one dump
|
|
|
|
// period in rare cases.
|
|
|
|
last_stats_dump_time_microsec_ = now_micros;
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 21:57:29 +02:00
|
|
|
|
2015-04-10 02:05:17 +02:00
|
|
|
#ifndef ROCKSDB_LITE
|
Eliminate duplicated property constants
Summary:
Before this diff, there were duplicated constants to refer to properties (user-
facing API had strings and InternalStats had an enum). I noticed these were
inconsistent in terms of which constants are provided, names of constants, and
documentation of constants. Overall it seemed annoying/error-prone to maintain
these duplicated constants.
So, this diff gets rid of InternalStats's constants and replaces them with a map
keyed on the user-facing constant. The value in that map contains a function
pointer to get the property value, so we don't need to do string matching while
holding db->mutex_. This approach has a side benefit of making many small
handler functions rather than a giant switch-statement.
Test Plan: db_properties_test passes, running "make commit-prereq -j32"
Reviewers: sdong, yhchiang, kradhakrishnan, IslamAbdelRahman, rven, anthony
Reviewed By: anthony
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D53253
2016-02-03 04:14:56 +01:00
|
|
|
const DBPropertyInfo* cf_property_info =
|
|
|
|
GetPropertyInfo(DB::Properties::kCFStats);
|
|
|
|
assert(cf_property_info != nullptr);
|
|
|
|
const DBPropertyInfo* db_property_info =
|
|
|
|
GetPropertyInfo(DB::Properties::kDBStats);
|
|
|
|
assert(db_property_info != nullptr);
|
|
|
|
|
2013-05-24 21:52:45 +02:00
|
|
|
std::string stats;
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 21:57:29 +02:00
|
|
|
{
|
2015-02-05 06:39:45 +01:00
|
|
|
InstrumentedMutexLock l(&mutex_);
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 21:57:29 +02:00
|
|
|
for (auto cfd : *versions_->GetColumnFamilySet()) {
|
Eliminate duplicated property constants
Summary:
Before this diff, there were duplicated constants to refer to properties (user-
facing API had strings and InternalStats had an enum). I noticed these were
inconsistent in terms of which constants are provided, names of constants, and
documentation of constants. Overall it seemed annoying/error-prone to maintain
these duplicated constants.
So, this diff gets rid of InternalStats's constants and replaces them with a map
keyed on the user-facing constant. The value in that map contains a function
pointer to get the property value, so we don't need to do string matching while
holding db->mutex_. This approach has a side benefit of making many small
handler functions rather than a giant switch-statement.
Test Plan: db_properties_test passes, running "make commit-prereq -j32"
Reviewers: sdong, yhchiang, kradhakrishnan, IslamAbdelRahman, rven, anthony
Reviewed By: anthony
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D53253
2016-02-03 04:14:56 +01:00
|
|
|
cfd->internal_stats()->GetStringProperty(
|
|
|
|
*cf_property_info, DB::Properties::kCFStats, &stats);
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 21:57:29 +02:00
|
|
|
}
|
Eliminate duplicated property constants
Summary:
Before this diff, there were duplicated constants to refer to properties (user-
facing API had strings and InternalStats had an enum). I noticed these were
inconsistent in terms of which constants are provided, names of constants, and
documentation of constants. Overall it seemed annoying/error-prone to maintain
these duplicated constants.
So, this diff gets rid of InternalStats's constants and replaces them with a map
keyed on the user-facing constant. The value in that map contains a function
pointer to get the property value, so we don't need to do string matching while
holding db->mutex_. This approach has a side benefit of making many small
handler functions rather than a giant switch-statement.
Test Plan: db_properties_test passes, running "make commit-prereq -j32"
Reviewers: sdong, yhchiang, kradhakrishnan, IslamAbdelRahman, rven, anthony
Reviewed By: anthony
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D53253
2016-02-03 04:14:56 +01:00
|
|
|
default_cf_internal_stats_->GetStringProperty(
|
|
|
|
*db_property_info, DB::Properties::kDBStats, &stats);
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 21:57:29 +02:00
|
|
|
}
|
2017-03-16 03:22:52 +01:00
|
|
|
ROCKS_LOG_WARN(immutable_db_options_.info_log,
|
|
|
|
"------- DUMPING STATS -------");
|
|
|
|
ROCKS_LOG_WARN(immutable_db_options_.info_log, "%s", stats.c_str());
|
2016-10-30 01:01:30 +02:00
|
|
|
if (immutable_db_options_.dump_malloc_stats) {
|
|
|
|
stats.clear();
|
|
|
|
DumpMallocStats(&stats);
|
|
|
|
if (!stats.empty()) {
|
2017-03-16 03:22:52 +01:00
|
|
|
ROCKS_LOG_WARN(immutable_db_options_.info_log,
|
|
|
|
"------- Malloc STATS -------");
|
|
|
|
ROCKS_LOG_WARN(immutable_db_options_.info_log, "%s", stats.c_str());
|
2016-10-30 01:01:30 +02:00
|
|
|
}
|
|
|
|
}
|
2015-04-10 02:05:17 +02:00
|
|
|
#endif // !ROCKSDB_LITE
|
make internal stats independent of statistics
Summary:
also make it aware of column family
output from db_bench
```
** Compaction Stats [default] **
Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) Stall(sec) Stall(cnt) Avg(ms)
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
L0 14 956 0.9 0.0 0.0 0.0 2.7 2.7 0.0 0.0 0.0 111.6 0 0 0 0 24 40 0.612 75.20 492387 0.15
L1 21 2001 2.0 5.7 2.0 3.7 5.3 1.6 5.4 2.6 71.2 65.7 31 43 55 12 82 2 41.242 43.72 41183 1.06
L2 217 18974 1.9 16.5 2.0 14.4 15.1 0.7 15.6 7.4 70.1 64.3 17 182 185 3 241 16 15.052 0.00 0 0.00
L3 1641 188245 1.8 9.1 1.1 8.0 8.5 0.5 15.4 7.4 61.3 57.2 9 75 76 1 152 9 16.887 0.00 0 0.00
L4 4447 449025 0.4 13.4 4.8 8.6 9.1 0.5 4.7 1.9 77.8 52.7 38 79 100 21 176 38 4.639 0.00 0 0.00
Sum 6340 659201 0.0 44.7 10.0 34.7 40.6 6.0 32.0 15.2 67.7 61.6 95 379 416 37 676 105 6.439 118.91 533570 0.22
Int 0 0 0.0 1.2 0.4 0.8 1.3 0.5 5.2 2.7 59.1 65.6 3 7 9 2 20 10 2.003 0.00 0 0.00
Stalls(secs): 75.197 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 43.717 leveln_slowdown
Stalls(count): 492387 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 41183 leveln_slowdown
** DB Stats **
Uptime(secs): 202.1 total, 13.5 interval
Cumulative writes: 6291456 writes, 6291456 batches, 1.0 writes per batch, 4.90 ingest GB
Cumulative WAL: 6291456 writes, 6291456 syncs, 1.00 writes per sync, 4.90 GB written
Interval writes: 1048576 writes, 1048576 batches, 1.0 writes per batch, 836.0 ingest MB
Interval WAL: 1048576 writes, 1048576 syncs, 1.00 writes per sync, 0.82 MB written
Test Plan: ran it
Reviewers: sdong, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D19917
2014-07-21 21:57:29 +02:00
|
|
|
|
2013-05-28 21:35:43 +02:00
|
|
|
PrintStatistics();
|
2013-05-11 00:21:04 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-08-04 02:42:06 +02:00
|
|
|
void DBImpl::ScheduleBgLogWriterClose(JobContext* job_context) {
|
|
|
|
if (!job_context->logs_to_free.empty()) {
|
|
|
|
for (auto l : job_context->logs_to_free) {
|
|
|
|
AddToLogsToFreeQueue(l);
|
|
|
|
}
|
|
|
|
job_context->logs_to_free.clear();
|
|
|
|
SchedulePurge();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-04-06 02:14:05 +02:00
|
|
|
Directory* DBImpl::Directories::GetDataDir(size_t path_id) {
|
|
|
|
assert(path_id < data_dirs_.size());
|
|
|
|
Directory* ret_dir = data_dirs_[path_id].get();
|
|
|
|
if (ret_dir == nullptr) {
|
|
|
|
// Should use db_dir_
|
|
|
|
return db_dir_.get();
|
2016-12-28 20:53:29 +01:00
|
|
|
}
|
2017-04-06 02:14:05 +02:00
|
|
|
return ret_dir;
|
2016-12-28 20:53:29 +01:00
|
|
|
}
|
|
|
|
|
2017-04-06 02:14:05 +02:00
|
|
|
Status DBImpl::SetOptions(ColumnFamilyHandle* column_family,
|
|
|
|
const std::unordered_map<std::string, std::string>& options_map) {
|
|
|
|
#ifdef ROCKSDB_LITE
|
|
|
|
return Status::NotSupported("Not supported in ROCKSDB LITE");
|
|
|
|
#else
|
|
|
|
auto* cfd = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family)->cfd();
|
|
|
|
if (options_map.empty()) {
|
|
|
|
ROCKS_LOG_WARN(immutable_db_options_.info_log,
|
|
|
|
"SetOptions() on column family [%s], empty input",
|
|
|
|
cfd->GetName().c_str());
|
|
|
|
return Status::InvalidArgument("empty input");
|
2012-09-15 02:11:35 +02:00
|
|
|
}
|
|
|
|
|
2017-04-06 02:14:05 +02:00
|
|
|
MutableCFOptions new_options;
|
|
|
|
Status s;
|
|
|
|
Status persist_options_status;
|
|
|
|
WriteThread::Writer w;
|
|
|
|
{
|
|
|
|
InstrumentedMutexLock l(&mutex_);
|
|
|
|
s = cfd->SetOptions(options_map);
|
|
|
|
if (s.ok()) {
|
|
|
|
new_options = *cfd->GetLatestMutableCFOptions();
|
|
|
|
// Append new version to recompute compaction score.
|
|
|
|
VersionEdit dummy_edit;
|
|
|
|
versions_->LogAndApply(cfd, new_options, &dummy_edit, &mutex_,
|
|
|
|
directories_.GetDbDir());
|
|
|
|
// Trigger possible flush/compactions. This has to be before we persist
|
|
|
|
// options to file, otherwise there will be a deadlock with writer
|
|
|
|
// thread.
|
|
|
|
auto* old_sv =
|
|
|
|
InstallSuperVersionAndScheduleWork(cfd, nullptr, new_options);
|
|
|
|
delete old_sv;
|
2013-11-15 07:42:02 +01:00
|
|
|
|
2017-04-06 02:14:05 +02:00
|
|
|
write_thread_.EnterUnbatched(&w, &mutex_);
|
|
|
|
persist_options_status = WriteOptionsFile();
|
|
|
|
write_thread_.ExitUnbatched(&w);
|
2013-11-15 07:42:02 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-04-06 02:14:05 +02:00
|
|
|
ROCKS_LOG_INFO(immutable_db_options_.info_log,
|
|
|
|
"SetOptions() on column family [%s], inputs:",
|
|
|
|
cfd->GetName().c_str());
|
|
|
|
for (const auto& o : options_map) {
|
|
|
|
ROCKS_LOG_INFO(immutable_db_options_.info_log, "%s: %s\n", o.first.c_str(),
|
|
|
|
o.second.c_str());
|
2016-10-13 08:13:09 +02:00
|
|
|
}
|
2017-04-06 02:14:05 +02:00
|
|
|
if (s.ok()) {
|
|
|
|
ROCKS_LOG_INFO(immutable_db_options_.info_log,
|
|
|
|
"[%s] SetOptions() succeeded", cfd->GetName().c_str());
|
|
|
|
new_options.Dump(immutable_db_options_.info_log.get());
|
|
|
|
if (!persist_options_status.ok()) {
|
|
|
|
if (immutable_db_options_.fail_if_options_file_error) {
|
|
|
|
s = Status::IOError(
|
|
|
|
"SetOptions() succeeded, but unable to persist options",
|
|
|
|
persist_options_status.ToString());
|
2015-09-14 23:01:13 +02:00
|
|
|
}
|
2017-04-06 02:14:05 +02:00
|
|
|
ROCKS_LOG_WARN(immutable_db_options_.info_log,
|
|
|
|
"Unable to persist options in SetOptions() -- %s",
|
|
|
|
persist_options_status.ToString().c_str());
|
2015-09-14 23:01:13 +02:00
|
|
|
}
|
2014-07-02 18:54:20 +02:00
|
|
|
} else {
|
2017-04-06 02:14:05 +02:00
|
|
|
ROCKS_LOG_WARN(immutable_db_options_.info_log, "[%s] SetOptions() failed",
|
|
|
|
cfd->GetName().c_str());
|
2016-06-22 03:41:23 +02:00
|
|
|
}
|
2017-04-06 02:14:05 +02:00
|
|
|
LogFlush(immutable_db_options_.info_log);
|
|
|
|
return s;
|
|
|
|
#endif // ROCKSDB_LITE
|
2016-06-22 03:41:23 +02:00
|
|
|
}
|
|
|
|
|
2017-04-06 02:14:05 +02:00
|
|
|
Status DBImpl::SetDBOptions(
|
|
|
|
const std::unordered_map<std::string, std::string>& options_map) {
|
|
|
|
#ifdef ROCKSDB_LITE
|
|
|
|
return Status::NotSupported("Not supported in ROCKSDB LITE");
|
|
|
|
#else
|
|
|
|
if (options_map.empty()) {
|
|
|
|
ROCKS_LOG_WARN(immutable_db_options_.info_log,
|
|
|
|
"SetDBOptions(), empty input.");
|
|
|
|
return Status::InvalidArgument("empty input");
|
2013-11-09 00:23:46 +01:00
|
|
|
}
|
2014-03-09 23:11:34 +01:00
|
|
|
|
2017-04-06 02:14:05 +02:00
|
|
|
MutableDBOptions new_options;
|
|
|
|
Status s;
|
|
|
|
Status persist_options_status;
|
|
|
|
WriteThread::Writer w;
|
|
|
|
WriteContext write_context;
|
|
|
|
{
|
|
|
|
InstrumentedMutexLock l(&mutex_);
|
|
|
|
s = GetMutableDBOptionsFromStrings(mutable_db_options_, options_map,
|
|
|
|
&new_options);
|
|
|
|
if (s.ok()) {
|
|
|
|
if (new_options.max_background_compactions >
|
|
|
|
mutable_db_options_.max_background_compactions) {
|
|
|
|
env_->IncBackgroundThreadsIfNeeded(
|
|
|
|
new_options.max_background_compactions, Env::Priority::LOW);
|
|
|
|
MaybeScheduleFlushOrCompaction();
|
|
|
|
}
|
2016-10-14 21:25:39 +02:00
|
|
|
|
2016-11-13 00:43:33 +01:00
|
|
|
write_controller_.set_max_delayed_write_rate(new_options.delayed_write_rate);
|
|
|
|
|
2016-10-14 21:25:39 +02:00
|
|
|
mutable_db_options_ = new_options;
|
|
|
|
|
2017-04-04 19:19:33 +02:00
|
|
|
write_thread_.EnterUnbatched(&w, &mutex_);
|
2016-11-15 07:45:16 +01:00
|
|
|
if (total_log_size_ > GetMaxTotalWalSize()) {
|
2017-04-04 19:19:33 +02:00
|
|
|
Status purge_wal_status = HandleWALFull(&write_context);
|
|
|
|
if (!purge_wal_status.ok()) {
|
|
|
|
ROCKS_LOG_WARN(immutable_db_options_.info_log,
|
|
|
|
"Unable to purge WAL files in SetDBOptions() -- %s",
|
|
|
|
purge_wal_status.ToString().c_str());
|
|
|
|
}
|
2016-11-15 07:45:16 +01:00
|
|
|
}
|
2017-04-04 19:19:33 +02:00
|
|
|
persist_options_status = WriteOptionsFile();
|
|
|
|
write_thread_.ExitUnbatched(&w);
|
2016-10-14 21:25:39 +02:00
|
|
|
}
|
|
|
|
}
|
2017-03-16 03:22:52 +01:00
|
|
|
ROCKS_LOG_INFO(immutable_db_options_.info_log, "SetDBOptions(), inputs:");
|
2016-10-14 21:25:39 +02:00
|
|
|
for (const auto& o : options_map) {
|
2017-03-16 03:22:52 +01:00
|
|
|
ROCKS_LOG_INFO(immutable_db_options_.info_log, "%s: %s\n", o.first.c_str(),
|
|
|
|
o.second.c_str());
|
2016-10-14 21:25:39 +02:00
|
|
|
}
|
|
|
|
if (s.ok()) {
|
2017-03-16 03:22:52 +01:00
|
|
|
ROCKS_LOG_INFO(immutable_db_options_.info_log, "SetDBOptions() succeeded");
|
2016-10-14 21:25:39 +02:00
|
|
|
new_options.Dump(immutable_db_options_.info_log.get());
|
|
|
|
if (!persist_options_status.ok()) {
|
|
|
|
if (immutable_db_options_.fail_if_options_file_error) {
|
|
|
|
s = Status::IOError(
|
|
|
|
"SetDBOptions() succeeded, but unable to persist options",
|
|
|
|
persist_options_status.ToString());
|
|
|
|
}
|
2017-03-16 03:22:52 +01:00
|
|
|
ROCKS_LOG_WARN(immutable_db_options_.info_log,
|
|
|
|
"Unable to persist options in SetDBOptions() -- %s",
|
|
|
|
persist_options_status.ToString().c_str());
|
2016-10-14 21:25:39 +02:00
|
|
|
}
|
|
|
|
} else {
|
2017-03-16 03:22:52 +01:00
|
|
|
ROCKS_LOG_WARN(immutable_db_options_.info_log, "SetDBOptions failed");
|
2014-10-17 02:22:28 +02:00
|
|
|
}
|
2016-09-24 01:34:04 +02:00
|
|
|
LogFlush(immutable_db_options_.info_log);
|
2014-11-05 01:23:05 +01:00
|
|
|
return s;
|
2014-11-13 22:45:33 +01:00
|
|
|
#endif // ROCKSDB_LITE
|
2014-09-17 21:49:13 +02:00
|
|
|
}
|
|
|
|
|
2013-06-30 08:21:36 +02:00
|
|
|
// return the same level if it cannot be moved
|
2014-10-02 01:19:16 +02:00
|
|
|
int DBImpl::FindMinimumEmptyLevelFitting(ColumnFamilyData* cfd,
|
|
|
|
const MutableCFOptions& mutable_cf_options, int level) {
|
2013-06-30 08:21:36 +02:00
|
|
|
mutex_.AssertHeld();
|
2014-10-31 16:48:19 +01:00
|
|
|
const auto* vstorage = cfd->current()->storage_info();
|
2013-06-30 08:21:36 +02:00
|
|
|
int minimum_level = level;
|
2013-07-22 21:19:46 +02:00
|
|
|
for (int i = level - 1; i > 0; --i) {
|
2013-06-30 08:21:36 +02:00
|
|
|
// stop if level i is not empty
|
2014-10-27 23:49:46 +01:00
|
|
|
if (vstorage->NumLevelFiles(i) > 0) break;
|
2013-06-30 08:21:36 +02:00
|
|
|
// stop if level i is too small (cannot fit the level files)
|
options.level_compaction_dynamic_level_bytes to allow RocksDB to pick size bases of levels dynamically.
Summary:
When having fixed max_bytes_for_level_base, the ratio of size of largest level and the second one can range from 0 to the multiplier. This makes LSM tree frequently irregular and unpredictable. It can also cause poor space amplification in some cases.
In this improvement (proposed by Igor Kabiljo), we introduce a parameter option.level_compaction_use_dynamic_max_bytes. When turning it on, RocksDB is free to pick a level base in the range of (options.max_bytes_for_level_base/options.max_bytes_for_level_multiplier, options.max_bytes_for_level_base] so that real level ratios are close to options.max_bytes_for_level_multiplier.
Test Plan: New unit tests and pass tests suites including valgrind.
Reviewers: MarkCallaghan, rven, yhchiang, igor, ikabiljo
Reviewed By: ikabiljo
Subscribers: yoshinorim, ikabiljo, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D31437
2015-02-05 20:44:17 +01:00
|
|
|
if (vstorage->MaxBytesForLevel(i) < vstorage->NumLevelBytes(level)) {
|
2014-02-01 00:30:27 +01:00
|
|
|
break;
|
|
|
|
}
|
2013-06-30 08:21:36 +02:00
|
|
|
|
|
|
|
minimum_level = i;
|
|
|
|
}
|
|
|
|
return minimum_level;
|
|
|
|
}
|
|
|
|
|
[wal changes 3/3] method in DB to sync WAL without blocking writers
Summary:
Subj. We really need this feature.
Previous diff D40899 has most of the changes to make this possible, this diff just adds the method.
Test Plan: `make check`, the new test fails without this diff; ran with ASAN, TSAN and valgrind.
Reviewers: igor, rven, IslamAbdelRahman, anthony, kradhakrishnan, tnovak, yhchiang, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, maykov, hermanlee4, yoshinorim, tnovak, dhruba
Differential Revision: https://reviews.facebook.net/D40905
2015-08-05 15:06:39 +02:00
|
|
|
Status DBImpl::SyncWAL() {
|
|
|
|
autovector<log::Writer*, 1> logs_to_sync;
|
|
|
|
bool need_log_dir_sync;
|
|
|
|
uint64_t current_log_number;
|
|
|
|
|
|
|
|
{
|
|
|
|
InstrumentedMutexLock l(&mutex_);
|
|
|
|
assert(!logs_.empty());
|
|
|
|
|
|
|
|
// This SyncWAL() call only cares about logs up to this number.
|
|
|
|
current_log_number = logfile_number_;
|
|
|
|
|
|
|
|
while (logs_.front().number <= current_log_number &&
|
|
|
|
logs_.front().getting_synced) {
|
|
|
|
log_sync_cv_.Wait();
|
|
|
|
}
|
|
|
|
// First check that logs are safe to sync in background.
|
|
|
|
for (auto it = logs_.begin();
|
|
|
|
it != logs_.end() && it->number <= current_log_number; ++it) {
|
|
|
|
if (!it->writer->file()->writable_file()->IsSyncThreadSafe()) {
|
|
|
|
return Status::NotSupported(
|
2016-09-24 01:34:04 +02:00
|
|
|
"SyncWAL() is not supported for this implementation of WAL file",
|
|
|
|
immutable_db_options_.allow_mmap_writes
|
|
|
|
? "try setting Options::allow_mmap_writes to false"
|
|
|
|
: Slice());
|
[wal changes 3/3] method in DB to sync WAL without blocking writers
Summary:
Subj. We really need this feature.
Previous diff D40899 has most of the changes to make this possible, this diff just adds the method.
Test Plan: `make check`, the new test fails without this diff; ran with ASAN, TSAN and valgrind.
Reviewers: igor, rven, IslamAbdelRahman, anthony, kradhakrishnan, tnovak, yhchiang, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, maykov, hermanlee4, yoshinorim, tnovak, dhruba
Differential Revision: https://reviews.facebook.net/D40905
2015-08-05 15:06:39 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
for (auto it = logs_.begin();
|
|
|
|
it != logs_.end() && it->number <= current_log_number; ++it) {
|
|
|
|
auto& log = *it;
|
2017-04-06 02:14:05 +02:00
|
|
|
assert(!log.getting_synced);
|
|
|
|
log.getting_synced = true;
|
|
|
|
logs_to_sync.push_back(log.writer);
|
|
|
|
}
|
Rewritten system for scheduling background work
Summary:
When scaling to higher number of column families, the worst bottleneck was MaybeScheduleFlushOrCompaction(), which did a for loop over all column families while holding a mutex. This patch addresses the issue.
The approach is similar to our earlier efforts: instead of a pull-model, where we do something for every column family, we can do a push-based model -- when we detect that column family is ready to be flushed/compacted, we add it to the flush_queue_/compaction_queue_. That way we don't need to loop over every column family in MaybeScheduleFlushOrCompaction.
Here are the performance results:
Command:
./db_bench --write_buffer_size=268435456 --db_write_buffer_size=268435456 --db=/fast-rocksdb-tmp/rocks_lots_of_cf --use_existing_db=0 --open_files=55000 --statistics=1 --histogram=1 --disable_data_sync=1 --max_write_buffer_number=2 --sync=0 --benchmarks=fillrandom --threads=16 --num_column_families=5000 --disable_wal=1 --max_background_flushes=16 --max_background_compactions=16 --level0_file_num_compaction_trigger=2 --level0_slowdown_writes_trigger=2 --level0_stop_writes_trigger=3 --hard_rate_limit=1 --num=33333333 --writes=33333333
Before the patch:
fillrandom : 26.950 micros/op 37105 ops/sec; 4.1 MB/s
After the patch:
fillrandom : 17.404 micros/op 57456 ops/sec; 6.4 MB/s
Next bottleneck is VersionSet::AddLiveFiles, which is painfully slow when we have a lot of files. This is coming in the next patch, but when I removed that code, here's what I got:
fillrandom : 7.590 micros/op 131758 ops/sec; 14.6 MB/s
Test Plan:
make check
two stress tests:
Big number of compactions and flushes:
./db_stress --threads=30 --ops_per_thread=20000000 --max_key=10000 --column_families=20 --clear_column_family_one_in=10000000 --verify_before_write=0 --reopen=15 --max_background_compactions=10 --max_background_flushes=10 --db=/fast-rocksdb-tmp/db_stress --prefixpercent=0 --iterpercent=0 --writepercent=75 --db_write_buffer_size=2000000
max_background_flushes=0, to verify that this case also works correctly
./db_stress --threads=30 --ops_per_thread=2000000 --max_key=10000 --column_families=20 --clear_column_family_one_in=10000000 --verify_before_write=0 --reopen=3 --max_background_compactions=3 --max_background_flushes=0 --db=/fast-rocksdb-tmp/db_stress --prefixpercent=0 --iterpercent=0 --writepercent=75 --db_write_buffer_size=2000000
Reviewers: ljin, rven, yhchiang, sdong
Reviewed By: sdong
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D30123
2014-12-19 20:38:12 +01:00
|
|
|
|
2017-04-06 02:14:05 +02:00
|
|
|
need_log_dir_sync = !log_dir_synced_;
|
Rewritten system for scheduling background work
Summary:
When scaling to higher number of column families, the worst bottleneck was MaybeScheduleFlushOrCompaction(), which did a for loop over all column families while holding a mutex. This patch addresses the issue.
The approach is similar to our earlier efforts: instead of a pull-model, where we do something for every column family, we can do a push-based model -- when we detect that column family is ready to be flushed/compacted, we add it to the flush_queue_/compaction_queue_. That way we don't need to loop over every column family in MaybeScheduleFlushOrCompaction.
Here are the performance results:
Command:
./db_bench --write_buffer_size=268435456 --db_write_buffer_size=268435456 --db=/fast-rocksdb-tmp/rocks_lots_of_cf --use_existing_db=0 --open_files=55000 --statistics=1 --histogram=1 --disable_data_sync=1 --max_write_buffer_number=2 --sync=0 --benchmarks=fillrandom --threads=16 --num_column_families=5000 --disable_wal=1 --max_background_flushes=16 --max_background_compactions=16 --level0_file_num_compaction_trigger=2 --level0_slowdown_writes_trigger=2 --level0_stop_writes_trigger=3 --hard_rate_limit=1 --num=33333333 --writes=33333333
Before the patch:
fillrandom : 26.950 micros/op 37105 ops/sec; 4.1 MB/s
After the patch:
fillrandom : 17.404 micros/op 57456 ops/sec; 6.4 MB/s
Next bottleneck is VersionSet::AddLiveFiles, which is painfully slow when we have a lot of files. This is coming in the next patch, but when I removed that code, here's what I got:
fillrandom : 7.590 micros/op 131758 ops/sec; 14.6 MB/s
Test Plan:
make check
two stress tests:
Big number of compactions and flushes:
./db_stress --threads=30 --ops_per_thread=20000000 --max_key=10000 --column_families=20 --clear_column_family_one_in=10000000 --verify_before_write=0 --reopen=15 --max_background_compactions=10 --max_background_flushes=10 --db=/fast-rocksdb-tmp/db_stress --prefixpercent=0 --iterpercent=0 --writepercent=75 --db_write_buffer_size=2000000
max_background_flushes=0, to verify that this case also works correctly
./db_stress --threads=30 --ops_per_thread=2000000 --max_key=10000 --column_families=20 --clear_column_family_one_in=10000000 --verify_before_write=0 --reopen=3 --max_background_compactions=3 --max_background_flushes=0 --db=/fast-rocksdb-tmp/db_stress --prefixpercent=0 --iterpercent=0 --writepercent=75 --db_write_buffer_size=2000000
Reviewers: ljin, rven, yhchiang, sdong
Reviewed By: sdong
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D30123
2014-12-19 20:38:12 +01:00
|
|
|
}
|
|
|
|
|
2017-04-06 02:14:05 +02:00
|
|
|
RecordTick(stats_, WAL_FILE_SYNCED);
|
|
|
|
Status status;
|
|
|
|
for (log::Writer* log : logs_to_sync) {
|
|
|
|
status = log->file()->SyncWithoutFlush(immutable_db_options_.use_fsync);
|
|
|
|
if (!status.ok()) {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (status.ok() && need_log_dir_sync) {
|
|
|
|
status = directories_.GetWalDir()->Fsync();
|
2013-09-13 23:38:37 +02:00
|
|
|
}
|
|
|
|
|
2017-04-06 02:14:05 +02:00
|
|
|
TEST_SYNC_POINT("DBImpl::SyncWAL:BeforeMarkLogsSynced:1");
|
|
|
|
{
|
|
|
|
InstrumentedMutexLock l(&mutex_);
|
|
|
|
MarkLogsSynced(current_log_number, need_log_dir_sync, status);
|
|
|
|
}
|
|
|
|
TEST_SYNC_POINT("DBImpl::SyncWAL:BeforeMarkLogsSynced:2");
|
2016-06-22 03:41:23 +02:00
|
|
|
|
2017-04-06 02:14:05 +02:00
|
|
|
return status;
|
2013-09-13 23:38:37 +02:00
|
|
|
}
|
|
|
|
|
2017-04-06 02:14:05 +02:00
|
|
|
void DBImpl::MarkLogsSynced(
|
|
|
|
uint64_t up_to, bool synced_dir, const Status& status) {
|
|
|
|
mutex_.AssertHeld();
|
|
|
|
if (synced_dir &&
|
|
|
|
logfile_number_ == up_to &&
|
|
|
|
status.ok()) {
|
|
|
|
log_dir_synced_ = true;
|
|
|
|
}
|
|
|
|
for (auto it = logs_.begin(); it != logs_.end() && it->number <= up_to;) {
|
|
|
|
auto& log = *it;
|
|
|
|
assert(log.getting_synced);
|
|
|
|
if (status.ok() && logs_.size() > 1) {
|
|
|
|
logs_to_free_.push_back(log.ReleaseWriter());
|
|
|
|
it = logs_.erase(it);
|
|
|
|
} else {
|
|
|
|
log.getting_synced = false;
|
|
|
|
++it;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
assert(!status.ok() || logs_.empty() || logs_[0].number > up_to ||
|
|
|
|
(logs_.size() == 1 && !logs_[0].getting_synced));
|
|
|
|
log_sync_cv_.SignalAll();
|
Running manual compactions in parallel with other automatic or manual compactions in restricted cases
Summary:
This diff provides a framework for doing manual
compactions in parallel with other compactions. We now have a deque of manual compactions. We also pass manual compactions as an argument from RunManualCompactions down to
BackgroundCompactions, so that RunManualCompactions can be reentrant.
Parallelism is controlled by the two routines
ConflictingManualCompaction to allow/disallow new parallel/manual
compactions based on already existing ManualCompactions. In this diff, by default manual compactions still have to run exclusive of other compactions. However, by setting the compaction option, exclusive_manual_compaction to false, it is possible to run other compactions in parallel with a manual compaction. However, we are still restricted to one manual compaction per column family at a time. All of these restrictions will be relaxed in future diffs.
I will be adding more tests later.
Test Plan: Rocksdb regression + new tests + valgrind
Reviewers: igor, anthony, IslamAbdelRahman, kradhakrishnan, yhchiang, sdong
Reviewed By: sdong
Subscribers: yoshinorim, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D47973
2015-12-14 20:20:34 +01:00
|
|
|
}
|
|
|
|
|
2017-04-06 02:14:05 +02:00
|
|
|
SequenceNumber DBImpl::GetLatestSequenceNumber() const {
|
|
|
|
return versions_->LastSequence();
|
2016-06-22 03:41:23 +02:00
|
|
|
}
|
|
|
|
|
2017-04-06 02:14:05 +02:00
|
|
|
InternalIterator* DBImpl::NewInternalIterator(
|
|
|
|
Arena* arena, RangeDelAggregator* range_del_agg,
|
|
|
|
ColumnFamilyHandle* column_family) {
|
|
|
|
ColumnFamilyData* cfd;
|
|
|
|
if (column_family == nullptr) {
|
|
|
|
cfd = default_cf_handle_->cfd();
|
|
|
|
} else {
|
|
|
|
auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
|
|
|
|
cfd = cfh->cfd();
|
Running manual compactions in parallel with other automatic or manual compactions in restricted cases
Summary:
This diff provides a framework for doing manual
compactions in parallel with other compactions. We now have a deque of manual compactions. We also pass manual compactions as an argument from RunManualCompactions down to
BackgroundCompactions, so that RunManualCompactions can be reentrant.
Parallelism is controlled by the two routines
ConflictingManualCompaction to allow/disallow new parallel/manual
compactions based on already existing ManualCompactions. In this diff, by default manual compactions still have to run exclusive of other compactions. However, by setting the compaction option, exclusive_manual_compaction to false, it is possible to run other compactions in parallel with a manual compaction. However, we are still restricted to one manual compaction per column family at a time. All of these restrictions will be relaxed in future diffs.
I will be adding more tests later.
Test Plan: Rocksdb regression + new tests + valgrind
Reviewers: igor, anthony, IslamAbdelRahman, kradhakrishnan, yhchiang, sdong
Reviewed By: sdong
Subscribers: yoshinorim, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D47973
2015-12-14 20:20:34 +01:00
|
|
|
}
|
2013-09-13 23:38:37 +02:00
|
|
|
|
2017-04-06 02:14:05 +02:00
|
|
|
mutex_.Lock();
|
|
|
|
SuperVersion* super_version = cfd->GetSuperVersion()->Ref();
|
|
|
|
mutex_.Unlock();
|
|
|
|
ReadOptions roptions;
|
|
|
|
return NewInternalIterator(roptions, cfd, super_version, arena,
|
|
|
|
range_del_agg);
|
|
|
|
}
|
2017-04-08 01:58:07 +02:00
|
|
|
|
2017-04-06 02:14:05 +02:00
|
|
|
void DBImpl::SchedulePurge() {
|
|
|
|
mutex_.AssertHeld();
|
|
|
|
assert(opened_successfully_);
|
2017-04-08 01:58:07 +02:00
|
|
|
|
2017-04-06 02:14:05 +02:00
|
|
|
// Purge operations are put into High priority queue
|
|
|
|
bg_purge_scheduled_++;
|
|
|
|
env_->Schedule(&DBImpl::BGWorkPurge, this, Env::Priority::HIGH, nullptr);
|
2017-04-08 01:58:07 +02:00
|
|
|
}
|
|
|
|
|
2016-06-22 03:41:23 +02:00
|
|
|
void DBImpl::BackgroundCallPurge() {
|
|
|
|
mutex_.Lock();
|
|
|
|
|
2016-08-04 02:42:06 +02:00
|
|
|
// We use one single loop to clear both queues so that after existing the loop
|
|
|
|
// both queues are empty. This is stricter than what is needed, but can make
|
|
|
|
// it easier for us to reason the correctness.
|
|
|
|
while (!purge_queue_.empty() || !logs_to_free_queue_.empty()) {
|
|
|
|
if (!purge_queue_.empty()) {
|
|
|
|
auto purge_file = purge_queue_.begin();
|
|
|
|
auto fname = purge_file->fname;
|
|
|
|
auto type = purge_file->type;
|
|
|
|
auto number = purge_file->number;
|
|
|
|
auto path_id = purge_file->path_id;
|
|
|
|
auto job_id = purge_file->job_id;
|
|
|
|
purge_queue_.pop_front();
|
2016-06-22 03:41:23 +02:00
|
|
|
|
2016-08-04 02:42:06 +02:00
|
|
|
mutex_.Unlock();
|
|
|
|
Status file_deletion_status;
|
|
|
|
DeleteObsoleteFileImpl(file_deletion_status, job_id, fname, type, number,
|
|
|
|
path_id);
|
|
|
|
mutex_.Lock();
|
|
|
|
} else {
|
|
|
|
assert(!logs_to_free_queue_.empty());
|
|
|
|
log::Writer* log_writer = *(logs_to_free_queue_.begin());
|
|
|
|
logs_to_free_queue_.pop_front();
|
|
|
|
mutex_.Unlock();
|
|
|
|
delete log_writer;
|
|
|
|
mutex_.Lock();
|
|
|
|
}
|
2016-06-22 03:41:23 +02:00
|
|
|
}
|
|
|
|
bg_purge_scheduled_--;
|
|
|
|
|
|
|
|
bg_cv_.SignalAll();
|
|
|
|
// IMPORTANT:there should be no code after calling SignalAll. This call may
|
|
|
|
// signal the DB destructor that it's OK to proceed with destruction. In
|
|
|
|
// that case, all DB variables will be dealloacated and referencing them
|
|
|
|
// will cause trouble.
|
|
|
|
mutex_.Unlock();
|
|
|
|
}
|
|
|
|
|
2011-05-28 02:53:58 +02:00
|
|
|
namespace {
|
|
|
|
struct IterState {
|
2016-06-22 03:41:23 +02:00
|
|
|
IterState(DBImpl* _db, InstrumentedMutex* _mu, SuperVersion* _super_version,
|
2016-07-05 20:57:14 +02:00
|
|
|
bool _background_purge)
|
2016-06-22 03:41:23 +02:00
|
|
|
: db(_db),
|
|
|
|
mu(_mu),
|
|
|
|
super_version(_super_version),
|
2016-07-05 20:57:14 +02:00
|
|
|
background_purge(_background_purge) {}
|
2014-02-03 22:13:36 +01:00
|
|
|
|
|
|
|
DBImpl* db;
|
2015-02-05 06:39:45 +01:00
|
|
|
InstrumentedMutex* mu;
|
2014-02-03 22:44:47 +01:00
|
|
|
SuperVersion* super_version;
|
2016-07-05 20:57:14 +02:00
|
|
|
bool background_purge;
|
2011-05-28 02:53:58 +02:00
|
|
|
};
|
|
|
|
|
|
|
|
static void CleanupIteratorState(void* arg1, void* arg2) {
|
|
|
|
IterState* state = reinterpret_cast<IterState*>(arg1);
|
2014-02-03 22:13:36 +01:00
|
|
|
|
2014-04-14 18:34:59 +02:00
|
|
|
if (state->super_version->Unref()) {
|
2015-02-12 18:54:48 +01:00
|
|
|
// Job id == 0 means that this is not our background process, but rather
|
|
|
|
// user thread
|
|
|
|
JobContext job_context(0);
|
2014-03-05 03:17:27 +01:00
|
|
|
|
2014-02-03 22:13:36 +01:00
|
|
|
state->mu->Lock();
|
|
|
|
state->super_version->Cleanup();
|
2014-10-28 19:54:33 +01:00
|
|
|
state->db->FindObsoleteFiles(&job_context, false, true);
|
2016-08-04 02:42:06 +02:00
|
|
|
if (state->background_purge) {
|
|
|
|
state->db->ScheduleBgLogWriterClose(&job_context);
|
|
|
|
}
|
2014-02-03 22:13:36 +01:00
|
|
|
state->mu->Unlock();
|
|
|
|
|
|
|
|
delete state->super_version;
|
2014-10-28 19:54:33 +01:00
|
|
|
if (job_context.HaveSomethingToDelete()) {
|
2016-07-05 20:57:14 +02:00
|
|
|
if (state->background_purge) {
|
2016-06-22 03:41:23 +02:00
|
|
|
// PurgeObsoleteFiles here does not delete files. Instead, it adds the
|
|
|
|
// files to be deleted to a job queue, and deletes it in a separate
|
|
|
|
// background thread.
|
|
|
|
state->db->PurgeObsoleteFiles(job_context, true /* schedule only */);
|
|
|
|
state->mu->Lock();
|
|
|
|
state->db->SchedulePurge();
|
|
|
|
state->mu->Unlock();
|
|
|
|
} else {
|
|
|
|
state->db->PurgeObsoleteFiles(job_context);
|
|
|
|
}
|
2014-03-10 23:42:14 +01:00
|
|
|
}
|
2014-11-15 00:43:10 +01:00
|
|
|
job_context.Clean();
|
MemTableListVersion
Summary:
MemTableListVersion is to MemTableList what Version is to VersionSet. I took almost the same ideas to develop MemTableListVersion. The reason is to have copying std::list done in background, while flushing, rather than in foreground (MultiGet() and NewIterator()) under a mutex! Also, whenever we copied MemTableList, we copied also some MemTableList metadata (flush_requested_, commit_in_progress_, etc.), which was wasteful.
This diff avoids std::list copy under a mutex in both MultiGet() and NewIterator(). I created a small database with some number of immutable memtables, and creating 100.000 iterators in a single-thread (!) decreased from {188739, 215703, 198028} to {154352, 164035, 159817}. A lot of the savings come from code under a mutex, so we should see much higher savings with multiple threads. Creating new iterator is very important to LogDevice team.
I also think this diff will make SuperVersion obsolete for performance reasons. I will try it in the next diff. SuperVersion gave us huge savings on Get() code path, but I think that most of the savings came from copying MemTableList under a mutex. If we had MemTableListVersion, we would never need to copy the entire object (like we still do in NewIterator() and MultiGet())
Test Plan: `make check` works. I will also do `make valgrind_check` before commit
Reviewers: dhruba, haobo, kailiu, sdong, emayanke, tnovak
Reviewed By: kailiu
CC: leveldb
Differential Revision: https://reviews.facebook.net/D15255
2014-01-24 23:52:08 +01:00
|
|
|
}
|
2014-01-17 06:56:26 +01:00
|
|
|
|
2011-05-28 02:53:58 +02:00
|
|
|
delete state;
|
|
|
|
}
|
2011-10-31 18:22:06 +01:00
|
|
|
} // namespace
|
2011-05-28 02:53:58 +02:00
|
|
|
|
2016-11-04 19:53:38 +01:00
|
|
|
InternalIterator* DBImpl::NewInternalIterator(
|
|
|
|
const ReadOptions& read_options, ColumnFamilyData* cfd,
|
|
|
|
SuperVersion* super_version, Arena* arena,
|
|
|
|
RangeDelAggregator* range_del_agg) {
|
2015-10-13 00:06:38 +02:00
|
|
|
InternalIterator* internal_iter;
|
2014-09-05 02:40:41 +02:00
|
|
|
assert(arena != nullptr);
|
2016-11-04 19:53:38 +01:00
|
|
|
assert(range_del_agg != nullptr);
|
2014-09-05 02:40:41 +02:00
|
|
|
// Need to create internal iterator from the arena.
|
2016-10-24 22:13:01 +02:00
|
|
|
MergeIteratorBuilder merge_iter_builder(
|
|
|
|
&cfd->internal_comparator(), arena,
|
|
|
|
!read_options.total_order_seek &&
|
|
|
|
cfd->ioptions()->prefix_extractor != nullptr);
|
2014-09-05 02:40:41 +02:00
|
|
|
// Collect iterator for mutable mem
|
|
|
|
merge_iter_builder.AddIterator(
|
2014-10-17 01:57:59 +02:00
|
|
|
super_version->mem->NewIterator(read_options, arena));
|
2016-11-19 23:14:35 +01:00
|
|
|
std::unique_ptr<InternalIterator> range_del_iter;
|
2016-11-04 19:53:38 +01:00
|
|
|
Status s;
|
|
|
|
if (!read_options.ignore_range_deletions) {
|
2016-11-19 23:14:35 +01:00
|
|
|
range_del_iter.reset(
|
|
|
|
super_version->mem->NewRangeTombstoneIterator(read_options));
|
2016-11-04 19:53:38 +01:00
|
|
|
s = range_del_agg->AddTombstones(std::move(range_del_iter));
|
|
|
|
}
|
2014-09-05 02:40:41 +02:00
|
|
|
// Collect all needed child iterators for immutable memtables
|
2016-11-04 19:53:38 +01:00
|
|
|
if (s.ok()) {
|
|
|
|
super_version->imm->AddIterators(read_options, &merge_iter_builder);
|
|
|
|
if (!read_options.ignore_range_deletions) {
|
|
|
|
s = super_version->imm->AddRangeTombstoneIterators(read_options, arena,
|
|
|
|
range_del_agg);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (s.ok()) {
|
|
|
|
// Collect iterators for files in L0 - Ln
|
2017-03-07 20:50:02 +01:00
|
|
|
if (read_options.read_tier != kMemtableTier) {
|
|
|
|
super_version->current->AddIterators(read_options, env_options_,
|
|
|
|
&merge_iter_builder, range_del_agg);
|
|
|
|
}
|
2016-11-04 19:53:38 +01:00
|
|
|
internal_iter = merge_iter_builder.Finish();
|
|
|
|
IterState* cleanup =
|
|
|
|
new IterState(this, &mutex_, super_version,
|
|
|
|
read_options.background_purge_on_iterator_cleanup);
|
|
|
|
internal_iter->RegisterCleanup(CleanupIteratorState, cleanup, nullptr);
|
|
|
|
|
|
|
|
return internal_iter;
|
|
|
|
}
|
|
|
|
return NewErrorInternalIterator(s);
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2014-02-11 02:04:44 +01:00
|
|
|
ColumnFamilyHandle* DBImpl::DefaultColumnFamily() const {
|
|
|
|
return default_cf_handle_;
|
|
|
|
}
|
|
|
|
|
2014-10-17 01:57:59 +02:00
|
|
|
Status DBImpl::Get(const ReadOptions& read_options,
|
2017-04-06 02:14:05 +02:00
|
|
|
ColumnFamilyHandle* column_family, const Slice& key,
|
|
|
|
PinnableSlice* value) {
|
|
|
|
return GetImpl(read_options, column_family, key, value);
|
2014-10-17 01:57:59 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
Status DBImpl::GetImpl(const ReadOptions& read_options,
|
2014-02-11 02:04:44 +01:00
|
|
|
ColumnFamilyHandle* column_family, const Slice& key,
|
2017-03-13 19:44:50 +01:00
|
|
|
PinnableSlice* pinnable_val, bool* value_found) {
|
|
|
|
assert(pinnable_val != nullptr);
|
2014-07-28 21:05:36 +02:00
|
|
|
StopWatch sw(env_, stats_, DB_GET);
|
2014-08-23 00:28:58 +02:00
|
|
|
PERF_TIMER_GUARD(get_snapshot_time);
|
2014-01-28 20:05:04 +01:00
|
|
|
|
2014-02-11 02:04:44 +01:00
|
|
|
auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
|
|
|
|
auto cfd = cfh->cfd();
|
|
|
|
|
2017-02-03 20:35:22 +01:00
|
|
|
// Acquire SuperVersion
|
|
|
|
SuperVersion* sv = GetAndRefSuperVersion(cfd);
|
|
|
|
|
|
|
|
TEST_SYNC_POINT("DBImpl::GetImpl:1");
|
|
|
|
TEST_SYNC_POINT("DBImpl::GetImpl:2");
|
|
|
|
|
2011-06-22 04:36:45 +02:00
|
|
|
SequenceNumber snapshot;
|
2014-10-17 01:57:59 +02:00
|
|
|
if (read_options.snapshot != nullptr) {
|
|
|
|
snapshot = reinterpret_cast<const SnapshotImpl*>(
|
|
|
|
read_options.snapshot)->number_;
|
2011-06-22 04:36:45 +02:00
|
|
|
} else {
|
2017-02-03 20:35:22 +01:00
|
|
|
// Since we get and reference the super version before getting
|
|
|
|
// the snapshot number, without a mutex protection, it is possible
|
|
|
|
// that a memtable switch happened in the middle and not all the
|
|
|
|
// data for this snapshot is available. But it will contain all
|
|
|
|
// the data available in the super version we have, which is also
|
|
|
|
// a valid snapshot to read from.
|
|
|
|
// We shouldn't get snapshot before finding and referencing the
|
|
|
|
// super versipon because a flush happening in between may compact
|
|
|
|
// away data for the snapshot, but the snapshot is earlier than the
|
|
|
|
// data overwriting it, so users may see wrong results.
|
2011-06-22 04:36:45 +02:00
|
|
|
snapshot = versions_->LastSequence();
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
2017-02-03 20:35:22 +01:00
|
|
|
TEST_SYNC_POINT("DBImpl::GetImpl:3");
|
|
|
|
TEST_SYNC_POINT("DBImpl::GetImpl:4");
|
2017-01-27 01:25:19 +01:00
|
|
|
|
[RocksDB] [MergeOperator] The new Merge Interface! Uses merge sequences.
Summary:
Here are the major changes to the Merge Interface. It has been expanded
to handle cases where the MergeOperator is not associative. It does so by stacking
up merge operations while scanning through the key history (i.e.: during Get() or
Compaction), until a valid Put/Delete/end-of-history is encountered; it then
applies all of the merge operations in the correct sequence starting with the
base/sentinel value.
I have also introduced an "AssociativeMerge" function which allows the user to
take advantage of associative merge operations (such as in the case of counters).
The implementation will always attempt to merge the operations/operands themselves
together when they are encountered, and will resort to the "stacking" method if
and only if the "associative-merge" fails.
This implementation is conjectured to allow MergeOperator to handle the general
case, while still providing the user with the ability to take advantage of certain
efficiencies in their own merge-operator / data-structure.
NOTE: This is a preliminary diff. This must still go through a lot of review,
revision, and testing. Feedback welcome!
Test Plan:
-This is a preliminary diff. I have only just begun testing/debugging it.
-I will be testing this with the existing MergeOperator use-cases and unit-tests
(counters, string-append, and redis-lists)
-I will be "desk-checking" and walking through the code with the help gdb.
-I will find a way of stress-testing the new interface / implementation using
db_bench, db_test, merge_test, and/or db_stress.
-I will ensure that my tests cover all cases: Get-Memtable,
Get-Immutable-Memtable, Get-from-Disk, Iterator-Range-Scan, Flush-Memtable-to-L0,
Compaction-L0-L1, Compaction-Ln-L(n+1), Put/Delete found, Put/Delete not-found,
end-of-history, end-of-file, etc.
-A lot of feedback from the reviewers.
Reviewers: haobo, dhruba, zshao, emayanke
Reviewed By: haobo
CC: leveldb
Differential Revision: https://reviews.facebook.net/D11499
2013-08-06 05:14:32 +02:00
|
|
|
// Prepare to store a list of merge operations if merge occurs.
|
2013-12-03 03:34:05 +01:00
|
|
|
MergeContext merge_context;
|
2016-11-19 01:54:09 +01:00
|
|
|
RangeDelAggregator range_del_agg(cfd->internal_comparator(), snapshot);
|
[RocksDB] [MergeOperator] The new Merge Interface! Uses merge sequences.
Summary:
Here are the major changes to the Merge Interface. It has been expanded
to handle cases where the MergeOperator is not associative. It does so by stacking
up merge operations while scanning through the key history (i.e.: during Get() or
Compaction), until a valid Put/Delete/end-of-history is encountered; it then
applies all of the merge operations in the correct sequence starting with the
base/sentinel value.
I have also introduced an "AssociativeMerge" function which allows the user to
take advantage of associative merge operations (such as in the case of counters).
The implementation will always attempt to merge the operations/operands themselves
together when they are encountered, and will resort to the "stacking" method if
and only if the "associative-merge" fails.
This implementation is conjectured to allow MergeOperator to handle the general
case, while still providing the user with the ability to take advantage of certain
efficiencies in their own merge-operator / data-structure.
NOTE: This is a preliminary diff. This must still go through a lot of review,
revision, and testing. Feedback welcome!
Test Plan:
-This is a preliminary diff. I have only just begun testing/debugging it.
-I will be testing this with the existing MergeOperator use-cases and unit-tests
(counters, string-append, and redis-lists)
-I will be "desk-checking" and walking through the code with the help gdb.
-I will find a way of stress-testing the new interface / implementation using
db_bench, db_test, merge_test, and/or db_stress.
-I will ensure that my tests cover all cases: Get-Memtable,
Get-Immutable-Memtable, Get-from-Disk, Iterator-Range-Scan, Flush-Memtable-to-L0,
Compaction-L0-L1, Compaction-Ln-L(n+1), Put/Delete found, Put/Delete not-found,
end-of-history, end-of-file, etc.
-A lot of feedback from the reviewers.
Reviewers: haobo, dhruba, zshao, emayanke
Reviewed By: haobo
CC: leveldb
Differential Revision: https://reviews.facebook.net/D11499
2013-08-06 05:14:32 +02:00
|
|
|
|
2014-01-28 20:05:04 +01:00
|
|
|
Status s;
|
2013-03-26 02:01:47 +01:00
|
|
|
// First look in the memtable, then in the immutable memtable (if any).
|
2013-03-21 23:59:47 +01:00
|
|
|
// s is both in/out. When in, s could either be OK or MergeInProgress.
|
[RocksDB] [MergeOperator] The new Merge Interface! Uses merge sequences.
Summary:
Here are the major changes to the Merge Interface. It has been expanded
to handle cases where the MergeOperator is not associative. It does so by stacking
up merge operations while scanning through the key history (i.e.: during Get() or
Compaction), until a valid Put/Delete/end-of-history is encountered; it then
applies all of the merge operations in the correct sequence starting with the
base/sentinel value.
I have also introduced an "AssociativeMerge" function which allows the user to
take advantage of associative merge operations (such as in the case of counters).
The implementation will always attempt to merge the operations/operands themselves
together when they are encountered, and will resort to the "stacking" method if
and only if the "associative-merge" fails.
This implementation is conjectured to allow MergeOperator to handle the general
case, while still providing the user with the ability to take advantage of certain
efficiencies in their own merge-operator / data-structure.
NOTE: This is a preliminary diff. This must still go through a lot of review,
revision, and testing. Feedback welcome!
Test Plan:
-This is a preliminary diff. I have only just begun testing/debugging it.
-I will be testing this with the existing MergeOperator use-cases and unit-tests
(counters, string-append, and redis-lists)
-I will be "desk-checking" and walking through the code with the help gdb.
-I will find a way of stress-testing the new interface / implementation using
db_bench, db_test, merge_test, and/or db_stress.
-I will ensure that my tests cover all cases: Get-Memtable,
Get-Immutable-Memtable, Get-from-Disk, Iterator-Range-Scan, Flush-Memtable-to-L0,
Compaction-L0-L1, Compaction-Ln-L(n+1), Put/Delete found, Put/Delete not-found,
end-of-history, end-of-file, etc.
-A lot of feedback from the reviewers.
Reviewers: haobo, dhruba, zshao, emayanke
Reviewed By: haobo
CC: leveldb
Differential Revision: https://reviews.facebook.net/D11499
2013-08-06 05:14:32 +02:00
|
|
|
// merge_operands will contain the sequence of merges in the latter case.
|
2013-03-26 02:01:47 +01:00
|
|
|
LookupKey lkey(key, snapshot);
|
2014-04-08 19:58:07 +02:00
|
|
|
PERF_TIMER_STOP(get_snapshot_time);
|
2014-08-23 00:28:58 +02:00
|
|
|
|
2017-02-14 03:44:21 +01:00
|
|
|
bool skip_memtable = (read_options.read_tier == kPersistedTier &&
|
|
|
|
has_unpersisted_data_.load(std::memory_order_relaxed));
|
2016-02-09 20:20:22 +01:00
|
|
|
bool done = false;
|
|
|
|
if (!skip_memtable) {
|
2017-03-13 19:44:50 +01:00
|
|
|
if (sv->mem->Get(lkey, pinnable_val->GetSelf(), &s, &merge_context,
|
|
|
|
&range_del_agg, read_options)) {
|
2016-02-09 20:20:22 +01:00
|
|
|
done = true;
|
2017-03-13 19:44:50 +01:00
|
|
|
pinnable_val->PinSelf();
|
2016-02-09 20:20:22 +01:00
|
|
|
RecordTick(stats_, MEMTABLE_HIT);
|
2016-11-04 02:40:23 +01:00
|
|
|
} else if ((s.ok() || s.IsMergeInProgress()) &&
|
2017-03-13 19:44:50 +01:00
|
|
|
sv->imm->Get(lkey, pinnable_val->GetSelf(), &s, &merge_context,
|
|
|
|
&range_del_agg, read_options)) {
|
2016-02-09 20:20:22 +01:00
|
|
|
done = true;
|
2017-03-13 19:44:50 +01:00
|
|
|
pinnable_val->PinSelf();
|
2016-02-09 20:20:22 +01:00
|
|
|
RecordTick(stats_, MEMTABLE_HIT);
|
|
|
|
}
|
2016-11-04 02:40:23 +01:00
|
|
|
if (!done && !s.ok() && !s.IsMergeInProgress()) {
|
|
|
|
return s;
|
|
|
|
}
|
2016-02-09 20:20:22 +01:00
|
|
|
}
|
|
|
|
if (!done) {
|
2014-08-23 00:28:58 +02:00
|
|
|
PERF_TIMER_GUARD(get_from_output_files_time);
|
2017-03-13 19:44:50 +01:00
|
|
|
sv->current->Get(read_options, lkey, pinnable_val, &s, &merge_context,
|
2016-11-04 02:40:23 +01:00
|
|
|
&range_del_agg, value_found);
|
2014-07-28 21:05:36 +02:00
|
|
|
RecordTick(stats_, MEMTABLE_MISS);
|
2011-06-22 04:36:45 +02:00
|
|
|
}
|
2011-08-22 23:08:51 +02:00
|
|
|
|
2014-08-23 00:28:58 +02:00
|
|
|
{
|
|
|
|
PERF_TIMER_GUARD(get_post_process_time);
|
2011-08-22 23:08:51 +02:00
|
|
|
|
2014-08-23 00:28:58 +02:00
|
|
|
ReturnAndCleanupSuperVersion(cfd, sv);
|
2013-11-25 20:55:36 +01:00
|
|
|
|
2014-08-23 00:28:58 +02:00
|
|
|
RecordTick(stats_, NUMBER_KEYS_READ);
|
2017-03-13 19:44:50 +01:00
|
|
|
size_t size = pinnable_val->size();
|
|
|
|
RecordTick(stats_, BYTES_READ, size);
|
|
|
|
MeasureTime(stats_, BYTES_PER_READ, size);
|
2014-08-23 00:28:58 +02:00
|
|
|
}
|
2011-06-22 04:36:45 +02:00
|
|
|
return s;
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 20:14:09 +01:00
|
|
|
std::vector<Status> DBImpl::MultiGet(
|
2014-10-17 01:57:59 +02:00
|
|
|
const ReadOptions& read_options,
|
2014-02-11 02:04:44 +01:00
|
|
|
const std::vector<ColumnFamilyHandle*>& column_family,
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 20:14:09 +01:00
|
|
|
const std::vector<Slice>& keys, std::vector<std::string>* values) {
|
2013-06-05 20:22:38 +02:00
|
|
|
|
2014-07-28 21:05:36 +02:00
|
|
|
StopWatch sw(env_, stats_, DB_MULTIGET);
|
2014-08-23 00:28:58 +02:00
|
|
|
PERF_TIMER_GUARD(get_snapshot_time);
|
2013-11-27 20:47:40 +01:00
|
|
|
|
2013-06-05 20:22:38 +02:00
|
|
|
SequenceNumber snapshot;
|
2013-11-25 20:55:36 +01:00
|
|
|
|
2014-02-04 00:28:03 +01:00
|
|
|
struct MultiGetColumnFamilyData {
|
2014-02-05 22:12:23 +01:00
|
|
|
ColumnFamilyData* cfd;
|
2014-02-04 00:28:03 +01:00
|
|
|
SuperVersion* super_version;
|
|
|
|
};
|
|
|
|
std::unordered_map<uint32_t, MultiGetColumnFamilyData*> multiget_cf_data;
|
|
|
|
// fill up and allocate outside of mutex
|
|
|
|
for (auto cf : column_family) {
|
2014-02-11 02:04:44 +01:00
|
|
|
auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(cf);
|
|
|
|
auto cfd = cfh->cfd();
|
|
|
|
if (multiget_cf_data.find(cfd->GetID()) == multiget_cf_data.end()) {
|
|
|
|
auto mgcfd = new MultiGetColumnFamilyData();
|
|
|
|
mgcfd->cfd = cfd;
|
|
|
|
multiget_cf_data.insert({cfd->GetID(), mgcfd});
|
2014-02-04 00:28:03 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-08-27 07:23:47 +02:00
|
|
|
mutex_.Lock();
|
2014-10-17 01:57:59 +02:00
|
|
|
if (read_options.snapshot != nullptr) {
|
|
|
|
snapshot = reinterpret_cast<const SnapshotImpl*>(
|
|
|
|
read_options.snapshot)->number_;
|
2013-06-05 20:22:38 +02:00
|
|
|
} else {
|
|
|
|
snapshot = versions_->LastSequence();
|
|
|
|
}
|
2014-02-04 00:28:03 +01:00
|
|
|
for (auto mgd_iter : multiget_cf_data) {
|
2014-02-11 02:04:44 +01:00
|
|
|
mgd_iter.second->super_version =
|
|
|
|
mgd_iter.second->cfd->GetSuperVersion()->Ref();
|
2014-02-04 00:28:03 +01:00
|
|
|
}
|
2013-06-05 20:22:38 +02:00
|
|
|
mutex_.Unlock();
|
2014-02-03 22:13:36 +01:00
|
|
|
|
2013-12-03 03:34:05 +01:00
|
|
|
// Contain a list of merge operations if merge occurs.
|
|
|
|
MergeContext merge_context;
|
[RocksDB] [MergeOperator] The new Merge Interface! Uses merge sequences.
Summary:
Here are the major changes to the Merge Interface. It has been expanded
to handle cases where the MergeOperator is not associative. It does so by stacking
up merge operations while scanning through the key history (i.e.: during Get() or
Compaction), until a valid Put/Delete/end-of-history is encountered; it then
applies all of the merge operations in the correct sequence starting with the
base/sentinel value.
I have also introduced an "AssociativeMerge" function which allows the user to
take advantage of associative merge operations (such as in the case of counters).
The implementation will always attempt to merge the operations/operands themselves
together when they are encountered, and will resort to the "stacking" method if
and only if the "associative-merge" fails.
This implementation is conjectured to allow MergeOperator to handle the general
case, while still providing the user with the ability to take advantage of certain
efficiencies in their own merge-operator / data-structure.
NOTE: This is a preliminary diff. This must still go through a lot of review,
revision, and testing. Feedback welcome!
Test Plan:
-This is a preliminary diff. I have only just begun testing/debugging it.
-I will be testing this with the existing MergeOperator use-cases and unit-tests
(counters, string-append, and redis-lists)
-I will be "desk-checking" and walking through the code with the help gdb.
-I will find a way of stress-testing the new interface / implementation using
db_bench, db_test, merge_test, and/or db_stress.
-I will ensure that my tests cover all cases: Get-Memtable,
Get-Immutable-Memtable, Get-from-Disk, Iterator-Range-Scan, Flush-Memtable-to-L0,
Compaction-L0-L1, Compaction-Ln-L(n+1), Put/Delete found, Put/Delete not-found,
end-of-history, end-of-file, etc.
-A lot of feedback from the reviewers.
Reviewers: haobo, dhruba, zshao, emayanke
Reviewed By: haobo
CC: leveldb
Differential Revision: https://reviews.facebook.net/D11499
2013-08-06 05:14:32 +02:00
|
|
|
|
2013-06-05 20:22:38 +02:00
|
|
|
// Note: this always resizes the values array
|
2014-02-04 00:28:03 +01:00
|
|
|
size_t num_keys = keys.size();
|
|
|
|
std::vector<Status> stat_list(num_keys);
|
|
|
|
values->resize(num_keys);
|
2013-06-05 20:22:38 +02:00
|
|
|
|
|
|
|
// Keep track of bytes that we read for statistics-recording later
|
2014-02-04 00:28:03 +01:00
|
|
|
uint64_t bytes_read = 0;
|
2014-04-08 19:58:07 +02:00
|
|
|
PERF_TIMER_STOP(get_snapshot_time);
|
2013-06-05 20:22:38 +02:00
|
|
|
|
|
|
|
// For each of the given keys, apply the entire "get" process as follows:
|
|
|
|
// First look in the memtable, then in the immutable memtable (if any).
|
|
|
|
// s is both in/out. When in, s could either be OK or MergeInProgress.
|
[RocksDB] [MergeOperator] The new Merge Interface! Uses merge sequences.
Summary:
Here are the major changes to the Merge Interface. It has been expanded
to handle cases where the MergeOperator is not associative. It does so by stacking
up merge operations while scanning through the key history (i.e.: during Get() or
Compaction), until a valid Put/Delete/end-of-history is encountered; it then
applies all of the merge operations in the correct sequence starting with the
base/sentinel value.
I have also introduced an "AssociativeMerge" function which allows the user to
take advantage of associative merge operations (such as in the case of counters).
The implementation will always attempt to merge the operations/operands themselves
together when they are encountered, and will resort to the "stacking" method if
and only if the "associative-merge" fails.
This implementation is conjectured to allow MergeOperator to handle the general
case, while still providing the user with the ability to take advantage of certain
efficiencies in their own merge-operator / data-structure.
NOTE: This is a preliminary diff. This must still go through a lot of review,
revision, and testing. Feedback welcome!
Test Plan:
-This is a preliminary diff. I have only just begun testing/debugging it.
-I will be testing this with the existing MergeOperator use-cases and unit-tests
(counters, string-append, and redis-lists)
-I will be "desk-checking" and walking through the code with the help gdb.
-I will find a way of stress-testing the new interface / implementation using
db_bench, db_test, merge_test, and/or db_stress.
-I will ensure that my tests cover all cases: Get-Memtable,
Get-Immutable-Memtable, Get-from-Disk, Iterator-Range-Scan, Flush-Memtable-to-L0,
Compaction-L0-L1, Compaction-Ln-L(n+1), Put/Delete found, Put/Delete not-found,
end-of-history, end-of-file, etc.
-A lot of feedback from the reviewers.
Reviewers: haobo, dhruba, zshao, emayanke
Reviewed By: haobo
CC: leveldb
Differential Revision: https://reviews.facebook.net/D11499
2013-08-06 05:14:32 +02:00
|
|
|
// merge_operands will contain the sequence of merges in the latter case.
|
2014-02-04 00:28:03 +01:00
|
|
|
for (size_t i = 0; i < num_keys; ++i) {
|
2013-12-03 03:34:05 +01:00
|
|
|
merge_context.Clear();
|
2014-02-04 00:28:03 +01:00
|
|
|
Status& s = stat_list[i];
|
2013-06-05 20:22:38 +02:00
|
|
|
std::string* value = &(*values)[i];
|
|
|
|
|
|
|
|
LookupKey lkey(keys[i], snapshot);
|
2014-02-11 02:04:44 +01:00
|
|
|
auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family[i]);
|
2016-11-04 02:40:23 +01:00
|
|
|
RangeDelAggregator range_del_agg(cfh->cfd()->internal_comparator(),
|
2016-11-19 01:54:09 +01:00
|
|
|
snapshot);
|
2014-02-11 02:04:44 +01:00
|
|
|
auto mgd_iter = multiget_cf_data.find(cfh->cfd()->GetID());
|
2014-02-04 00:28:03 +01:00
|
|
|
assert(mgd_iter != multiget_cf_data.end());
|
|
|
|
auto mgd = mgd_iter->second;
|
|
|
|
auto super_version = mgd->super_version;
|
2016-02-09 20:20:22 +01:00
|
|
|
bool skip_memtable =
|
2017-02-14 03:44:21 +01:00
|
|
|
(read_options.read_tier == kPersistedTier &&
|
|
|
|
has_unpersisted_data_.load(std::memory_order_relaxed));
|
2016-02-09 20:20:22 +01:00
|
|
|
bool done = false;
|
|
|
|
if (!skip_memtable) {
|
2017-01-08 23:08:51 +01:00
|
|
|
if (super_version->mem->Get(lkey, value, &s, &merge_context,
|
2016-11-04 02:40:23 +01:00
|
|
|
&range_del_agg, read_options)) {
|
2016-02-09 20:20:22 +01:00
|
|
|
done = true;
|
|
|
|
// TODO(?): RecordTick(stats_, MEMTABLE_HIT)?
|
2017-01-08 23:08:51 +01:00
|
|
|
} else if (super_version->imm->Get(lkey, value, &s, &merge_context,
|
2016-11-04 02:40:23 +01:00
|
|
|
&range_del_agg, read_options)) {
|
2016-02-09 20:20:22 +01:00
|
|
|
done = true;
|
|
|
|
// TODO(?): RecordTick(stats_, MEMTABLE_HIT)?
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (!done) {
|
2017-03-13 19:44:50 +01:00
|
|
|
PinnableSlice pinnable_val;
|
2014-10-03 02:02:30 +02:00
|
|
|
PERF_TIMER_GUARD(get_from_output_files_time);
|
2017-03-13 19:44:50 +01:00
|
|
|
super_version->current->Get(read_options, lkey, &pinnable_val, &s,
|
|
|
|
&merge_context, &range_del_agg);
|
|
|
|
value->assign(pinnable_val.data(), pinnable_val.size());
|
2016-02-09 20:20:22 +01:00
|
|
|
// TODO(?): RecordTick(stats_, MEMTABLE_MISS)?
|
2013-06-05 20:22:38 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
if (s.ok()) {
|
2017-01-08 23:08:51 +01:00
|
|
|
bytes_read += value->size();
|
2013-06-05 20:22:38 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Post processing (decrement reference counts and record statistics)
|
2014-08-23 00:28:58 +02:00
|
|
|
PERF_TIMER_GUARD(get_post_process_time);
|
2014-02-04 00:28:03 +01:00
|
|
|
autovector<SuperVersion*> superversions_to_delete;
|
|
|
|
|
2014-06-20 10:23:02 +02:00
|
|
|
// TODO(icanadi) do we need lock here or just around Cleanup()?
|
2014-02-04 00:28:03 +01:00
|
|
|
mutex_.Lock();
|
|
|
|
for (auto mgd_iter : multiget_cf_data) {
|
|
|
|
auto mgd = mgd_iter.second;
|
|
|
|
if (mgd->super_version->Unref()) {
|
|
|
|
mgd->super_version->Cleanup();
|
|
|
|
superversions_to_delete.push_back(mgd->super_version);
|
2014-02-03 22:13:36 +01:00
|
|
|
}
|
|
|
|
}
|
2014-02-04 00:28:03 +01:00
|
|
|
mutex_.Unlock();
|
|
|
|
|
|
|
|
for (auto td : superversions_to_delete) {
|
|
|
|
delete td;
|
|
|
|
}
|
|
|
|
for (auto mgd : multiget_cf_data) {
|
|
|
|
delete mgd.second;
|
2013-06-05 20:22:38 +02:00
|
|
|
}
|
2013-11-25 20:55:36 +01:00
|
|
|
|
2014-07-28 21:05:36 +02:00
|
|
|
RecordTick(stats_, NUMBER_MULTIGET_CALLS);
|
|
|
|
RecordTick(stats_, NUMBER_MULTIGET_KEYS_READ, num_keys);
|
|
|
|
RecordTick(stats_, NUMBER_MULTIGET_BYTES_READ, bytes_read);
|
2016-02-01 03:09:24 +01:00
|
|
|
MeasureTime(stats_, BYTES_PER_MULTIGET, bytes_read);
|
2014-04-08 19:58:07 +02:00
|
|
|
PERF_TIMER_STOP(get_post_process_time);
|
2013-06-05 20:22:38 +02:00
|
|
|
|
2014-02-04 00:28:03 +01:00
|
|
|
return stat_list;
|
2013-06-05 20:22:38 +02:00
|
|
|
}
|
|
|
|
|
2014-10-17 01:57:59 +02:00
|
|
|
Status DBImpl::CreateColumnFamily(const ColumnFamilyOptions& cf_options,
|
2014-01-06 22:31:06 +01:00
|
|
|
const std::string& column_family_name,
|
2014-02-11 02:04:44 +01:00
|
|
|
ColumnFamilyHandle** handle) {
|
2014-11-20 19:49:32 +01:00
|
|
|
Status s;
|
2015-12-09 02:01:02 +01:00
|
|
|
Status persist_options_status;
|
2014-02-28 19:29:37 +01:00
|
|
|
*handle = nullptr;
|
2015-06-18 23:55:05 +02:00
|
|
|
|
|
|
|
s = CheckCompressionSupported(cf_options);
|
2016-09-24 01:34:04 +02:00
|
|
|
if (s.ok() && immutable_db_options_.allow_concurrent_memtable_write) {
|
support for concurrent adds to memtable
Summary:
This diff adds support for concurrent adds to the skiplist memtable
implementations. Memory allocation is made thread-safe by the addition of
a spinlock, with small per-core buffers to avoid contention. Concurrent
memtable writes are made via an additional method and don't impose a
performance overhead on the non-concurrent case, so parallelism can be
selected on a per-batch basis.
Write thread synchronization is an increasing bottleneck for higher levels
of concurrency, so this diff adds --enable_write_thread_adaptive_yield
(default off). This feature causes threads joining a write batch
group to spin for a short time (default 100 usec) using sched_yield,
rather than going to sleep on a mutex. If the timing of the yield calls
indicates that another thread has actually run during the yield then
spinning is avoided. This option improves performance for concurrent
situations even without parallel adds, although it has the potential to
increase CPU usage (and the heuristic adaptation is not yet mature).
Parallel writes are not currently compatible with
inplace updates, update callbacks, or delete filtering.
Enable it with --allow_concurrent_memtable_write (and
--enable_write_thread_adaptive_yield). Parallel memtable writes
are performance neutral when there is no actual parallelism, and in
my experiments (SSD server-class Linux and varying contention and key
sizes for fillrandom) they are always a performance win when there is
more than one thread.
Statistics are updated earlier in the write path, dropping the number
of DB mutex acquisitions from 2 to 1 for almost all cases.
This diff was motivated and inspired by Yahoo's cLSM work. It is more
conservative than cLSM: RocksDB's write batch group leader role is
preserved (along with all of the existing flush and write throttling
logic) and concurrent writers are blocked until all memtable insertions
have completed and the sequence number has been advanced, to preserve
linearizability.
My test config is "db_bench -benchmarks=fillrandom -threads=$T
-batch_size=1 -memtablerep=skip_list -value_size=100 --num=1000000/$T
-level0_slowdown_writes_trigger=9999 -level0_stop_writes_trigger=9999
-disable_auto_compactions --max_write_buffer_number=8
-max_background_flushes=8 --disable_wal --write_buffer_size=160000000
--block_size=16384 --allow_concurrent_memtable_write" on a two-socket
Xeon E5-2660 @ 2.2Ghz with lots of memory and an SSD hard drive. With 1
thread I get ~440Kops/sec. Peak performance for 1 socket (numactl
-N1) is slightly more than 1Mops/sec, at 16 threads. Peak performance
across both sockets happens at 30 threads, and is ~900Kops/sec, although
with fewer threads there is less performance loss when the system has
background work.
Test Plan:
1. concurrent stress tests for InlineSkipList and DynamicBloom
2. make clean; make check
3. make clean; DISABLE_JEMALLOC=1 make valgrind_check; valgrind db_bench
4. make clean; COMPILE_WITH_TSAN=1 make all check; db_bench
5. make clean; COMPILE_WITH_ASAN=1 make all check; db_bench
6. make clean; OPT=-DROCKSDB_LITE make check
7. verify no perf regressions when disabled
Reviewers: igor, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, IslamAbdelRahman, anthony, yhchiang, rven, sdong, guyg8, kradhakrishnan, dhruba
Differential Revision: https://reviews.facebook.net/D50589
2015-08-15 01:59:07 +02:00
|
|
|
s = CheckConcurrentWritesSupported(cf_options);
|
|
|
|
}
|
2015-06-18 23:55:05 +02:00
|
|
|
if (!s.ok()) {
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
2014-11-20 19:49:32 +01:00
|
|
|
{
|
2015-02-05 06:39:45 +01:00
|
|
|
InstrumentedMutexLock l(&mutex_);
|
2014-02-28 19:29:37 +01:00
|
|
|
|
2014-11-20 19:49:32 +01:00
|
|
|
if (versions_->GetColumnFamilySet()->GetColumnFamily(column_family_name) !=
|
|
|
|
nullptr) {
|
|
|
|
return Status::InvalidArgument("Column family already exists");
|
|
|
|
}
|
|
|
|
VersionEdit edit;
|
|
|
|
edit.AddColumnFamily(column_family_name);
|
|
|
|
uint32_t new_id = versions_->GetColumnFamilySet()->GetNextColumnFamilyID();
|
|
|
|
edit.SetColumnFamily(new_id);
|
|
|
|
edit.SetLogNumber(logfile_number_);
|
|
|
|
edit.SetComparatorName(cf_options.comparator->Name());
|
|
|
|
|
|
|
|
// LogAndApply will both write the creation in MANIFEST and create
|
|
|
|
// ColumnFamilyData object
|
2015-01-06 21:44:21 +01:00
|
|
|
{ // write thread
|
2015-08-06 01:56:28 +02:00
|
|
|
WriteThread::Writer w;
|
|
|
|
write_thread_.EnterUnbatched(&w, &mutex_);
|
2015-01-06 21:44:21 +01:00
|
|
|
// LogAndApply will both write the creation in MANIFEST and create
|
|
|
|
// ColumnFamilyData object
|
2016-09-24 01:34:04 +02:00
|
|
|
s = versions_->LogAndApply(nullptr, MutableCFOptions(cf_options), &edit,
|
|
|
|
&mutex_, directories_.GetDbDir(), false,
|
|
|
|
&cf_options);
|
2015-12-09 02:01:02 +01:00
|
|
|
|
|
|
|
if (s.ok()) {
|
|
|
|
// If the column family was created successfully, we then persist
|
|
|
|
// the updated RocksDB options under the same single write thread
|
|
|
|
persist_options_status = WriteOptionsFile();
|
|
|
|
}
|
2015-08-06 01:56:28 +02:00
|
|
|
write_thread_.ExitUnbatched(&w);
|
2015-01-06 21:44:21 +01:00
|
|
|
}
|
2014-11-20 19:49:32 +01:00
|
|
|
if (s.ok()) {
|
|
|
|
single_column_family_mode_ = false;
|
|
|
|
auto* cfd =
|
|
|
|
versions_->GetColumnFamilySet()->GetColumnFamily(column_family_name);
|
|
|
|
assert(cfd != nullptr);
|
2015-06-17 21:37:59 +02:00
|
|
|
delete InstallSuperVersionAndScheduleWork(
|
2014-11-20 19:49:32 +01:00
|
|
|
cfd, nullptr, *cfd->GetLatestMutableCFOptions());
|
2014-12-11 03:39:09 +01:00
|
|
|
|
|
|
|
if (!cfd->mem()->IsSnapshotSupported()) {
|
|
|
|
is_snapshot_supported_ = false;
|
|
|
|
}
|
|
|
|
|
2014-11-20 19:49:32 +01:00
|
|
|
*handle = new ColumnFamilyHandleImpl(cfd, this, &mutex_);
|
2017-03-16 03:22:52 +01:00
|
|
|
ROCKS_LOG_INFO(immutable_db_options_.info_log,
|
|
|
|
"Created column family [%s] (ID %u)",
|
|
|
|
column_family_name.c_str(), (unsigned)cfd->GetID());
|
2014-11-20 19:49:32 +01:00
|
|
|
} else {
|
2017-03-16 03:22:52 +01:00
|
|
|
ROCKS_LOG_ERROR(immutable_db_options_.info_log,
|
|
|
|
"Creating column family [%s] FAILED -- %s",
|
|
|
|
column_family_name.c_str(), s.ToString().c_str());
|
2014-11-20 19:49:32 +01:00
|
|
|
}
|
2015-02-05 06:39:45 +01:00
|
|
|
} // InstrumentedMutexLock l(&mutex_)
|
2014-11-20 19:49:32 +01:00
|
|
|
|
|
|
|
// this is outside the mutex
|
2014-01-06 22:31:06 +01:00
|
|
|
if (s.ok()) {
|
2015-12-09 02:01:02 +01:00
|
|
|
NewThreadStatusCfInfo(
|
|
|
|
reinterpret_cast<ColumnFamilyHandleImpl*>(*handle)->cfd());
|
2015-11-11 07:58:01 +01:00
|
|
|
if (!persist_options_status.ok()) {
|
2016-09-24 01:34:04 +02:00
|
|
|
if (immutable_db_options_.fail_if_options_file_error) {
|
2015-11-11 07:58:01 +01:00
|
|
|
s = Status::IOError(
|
|
|
|
"ColumnFamily has been created, but unable to persist"
|
|
|
|
"options in CreateColumnFamily()",
|
|
|
|
persist_options_status.ToString().c_str());
|
|
|
|
}
|
2017-03-16 03:22:52 +01:00
|
|
|
ROCKS_LOG_WARN(immutable_db_options_.info_log,
|
|
|
|
"Unable to persist options in CreateColumnFamily() -- %s",
|
|
|
|
persist_options_status.ToString().c_str());
|
2015-11-11 07:58:01 +01:00
|
|
|
}
|
2014-02-28 01:44:26 +01:00
|
|
|
}
|
2014-01-06 22:31:06 +01:00
|
|
|
return s;
|
2014-01-02 18:08:12 +01:00
|
|
|
}
|
|
|
|
|
2014-02-11 02:04:44 +01:00
|
|
|
Status DBImpl::DropColumnFamily(ColumnFamilyHandle* column_family) {
|
|
|
|
auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
|
|
|
|
auto cfd = cfh->cfd();
|
|
|
|
if (cfd->GetID() == 0) {
|
2014-01-11 00:12:34 +01:00
|
|
|
return Status::InvalidArgument("Can't drop default column family");
|
|
|
|
}
|
2014-02-11 02:04:44 +01:00
|
|
|
|
2014-12-11 03:39:09 +01:00
|
|
|
bool cf_support_snapshot = cfd->mem()->IsSnapshotSupported();
|
|
|
|
|
2014-01-22 20:44:53 +01:00
|
|
|
VersionEdit edit;
|
|
|
|
edit.DropColumnFamily();
|
2014-02-11 02:04:44 +01:00
|
|
|
edit.SetColumnFamily(cfd->GetID());
|
|
|
|
|
2014-02-28 01:44:26 +01:00
|
|
|
Status s;
|
2015-12-09 02:01:02 +01:00
|
|
|
Status options_persist_status;
|
2014-03-05 02:16:40 +01:00
|
|
|
{
|
2015-02-05 06:39:45 +01:00
|
|
|
InstrumentedMutexLock l(&mutex_);
|
2014-03-05 02:16:40 +01:00
|
|
|
if (cfd->IsDropped()) {
|
|
|
|
s = Status::InvalidArgument("Column family already dropped!\n");
|
|
|
|
}
|
|
|
|
if (s.ok()) {
|
2014-09-06 00:20:05 +02:00
|
|
|
// we drop column family from a single write thread
|
2015-08-06 01:56:28 +02:00
|
|
|
WriteThread::Writer w;
|
|
|
|
write_thread_.EnterUnbatched(&w, &mutex_);
|
2014-10-02 01:19:16 +02:00
|
|
|
s = versions_->LogAndApply(cfd, *cfd->GetLatestMutableCFOptions(),
|
|
|
|
&edit, &mutex_);
|
2015-12-09 02:01:02 +01:00
|
|
|
if (s.ok()) {
|
|
|
|
// If the column family was dropped successfully, we then persist
|
|
|
|
// the updated RocksDB options under the same single write thread
|
|
|
|
options_persist_status = WriteOptionsFile();
|
|
|
|
}
|
2015-08-06 01:56:28 +02:00
|
|
|
write_thread_.ExitUnbatched(&w);
|
2014-03-05 02:16:40 +01:00
|
|
|
}
|
2014-12-11 03:39:09 +01:00
|
|
|
|
|
|
|
if (!cf_support_snapshot) {
|
|
|
|
// Dropped Column Family doesn't support snapshot. Need to recalculate
|
|
|
|
// is_snapshot_supported_.
|
|
|
|
bool new_is_snapshot_supported = true;
|
|
|
|
for (auto c : *versions_->GetColumnFamilySet()) {
|
2015-03-20 01:04:29 +01:00
|
|
|
if (!c->IsDropped() && !c->mem()->IsSnapshotSupported()) {
|
2014-12-11 03:39:09 +01:00
|
|
|
new_is_snapshot_supported = false;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
is_snapshot_supported_ = new_is_snapshot_supported;
|
|
|
|
}
|
2014-02-11 02:04:44 +01:00
|
|
|
}
|
2014-02-06 20:44:50 +01:00
|
|
|
|
2014-02-28 01:44:26 +01:00
|
|
|
if (s.ok()) {
|
2014-11-20 19:49:32 +01:00
|
|
|
// Note that here we erase the associated cf_info of the to-be-dropped
|
|
|
|
// cfd before its ref-count goes to zero to avoid having to erase cf_info
|
|
|
|
// later inside db_mutex.
|
|
|
|
EraseThreadStatusCfInfo(cfd);
|
2014-03-11 22:52:17 +01:00
|
|
|
assert(cfd->IsDropped());
|
2014-10-17 01:57:59 +02:00
|
|
|
auto* mutable_cf_options = cfd->GetLatestMutableCFOptions();
|
|
|
|
max_total_in_memory_state_ -= mutable_cf_options->write_buffer_size *
|
|
|
|
mutable_cf_options->max_write_buffer_number;
|
2017-03-16 03:22:52 +01:00
|
|
|
ROCKS_LOG_INFO(immutable_db_options_.info_log,
|
|
|
|
"Dropped column family with id %u\n", cfd->GetID());
|
2015-12-09 02:01:02 +01:00
|
|
|
|
2015-11-11 07:58:01 +01:00
|
|
|
if (!options_persist_status.ok()) {
|
2016-09-24 01:34:04 +02:00
|
|
|
if (immutable_db_options_.fail_if_options_file_error) {
|
2015-11-11 07:58:01 +01:00
|
|
|
s = Status::IOError(
|
|
|
|
"ColumnFamily has been dropped, but unable to persist "
|
|
|
|
"options in DropColumnFamily()",
|
|
|
|
options_persist_status.ToString().c_str());
|
|
|
|
}
|
2017-03-16 03:22:52 +01:00
|
|
|
ROCKS_LOG_WARN(immutable_db_options_.info_log,
|
|
|
|
"Unable to persist options in DropColumnFamily() -- %s",
|
|
|
|
options_persist_status.ToString().c_str());
|
2015-11-11 07:58:01 +01:00
|
|
|
}
|
2014-02-28 01:44:26 +01:00
|
|
|
} else {
|
2017-03-16 03:22:52 +01:00
|
|
|
ROCKS_LOG_ERROR(immutable_db_options_.info_log,
|
|
|
|
"Dropping column family with id %u FAILED -- %s\n",
|
|
|
|
cfd->GetID(), s.ToString().c_str());
|
2014-02-28 01:44:26 +01:00
|
|
|
}
|
|
|
|
|
2014-01-06 22:31:06 +01:00
|
|
|
return s;
|
2014-01-02 18:08:12 +01:00
|
|
|
}
|
|
|
|
|
2014-10-17 01:57:59 +02:00
|
|
|
bool DBImpl::KeyMayExist(const ReadOptions& read_options,
|
2014-02-11 02:04:44 +01:00
|
|
|
ColumnFamilyHandle* column_family, const Slice& key,
|
|
|
|
std::string* value, bool* value_found) {
|
2017-03-13 19:44:50 +01:00
|
|
|
assert(value != nullptr);
|
2013-07-26 21:57:01 +02:00
|
|
|
if (value_found != nullptr) {
|
2013-11-17 07:59:22 +01:00
|
|
|
// falsify later if key-may-exist but can't fetch value
|
|
|
|
*value_found = true;
|
2013-07-26 21:57:01 +02:00
|
|
|
}
|
2014-10-17 01:57:59 +02:00
|
|
|
ReadOptions roptions = read_options;
|
2013-08-25 07:48:51 +02:00
|
|
|
roptions.read_tier = kBlockCacheTier; // read from block cache only
|
2017-03-13 19:44:50 +01:00
|
|
|
PinnableSlice pinnable_val;
|
|
|
|
auto s = GetImpl(roptions, column_family, key, &pinnable_val, value_found);
|
|
|
|
value->assign(pinnable_val.data(), pinnable_val.size());
|
2013-11-17 07:59:22 +01:00
|
|
|
|
2014-08-25 23:22:05 +02:00
|
|
|
// If block_cache is enabled and the index block of the table didn't
|
2013-11-17 07:59:22 +01:00
|
|
|
// not present in block_cache, the return value will be Status::Incomplete.
|
|
|
|
// In this case, key may still exist in the table.
|
|
|
|
return s.ok() || s.IsIncomplete();
|
2013-07-06 03:49:18 +02:00
|
|
|
}
|
|
|
|
|
2014-09-09 00:04:34 +02:00
|
|
|
Iterator* DBImpl::NewIterator(const ReadOptions& read_options,
|
2014-02-11 02:04:44 +01:00
|
|
|
ColumnFamilyHandle* column_family) {
|
2016-02-09 20:20:22 +01:00
|
|
|
if (read_options.read_tier == kPersistedTier) {
|
|
|
|
return NewErrorIterator(Status::NotSupported(
|
|
|
|
"ReadTier::kPersistedData is not yet supported in iterators."));
|
|
|
|
}
|
2014-02-11 02:04:44 +01:00
|
|
|
auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
|
|
|
|
auto cfd = cfh->cfd();
|
2015-02-18 20:49:31 +01:00
|
|
|
if (read_options.managed) {
|
|
|
|
#ifdef ROCKSDB_LITE
|
|
|
|
// not supported in lite version
|
|
|
|
return NewErrorIterator(Status::InvalidArgument(
|
|
|
|
"Managed Iterators not supported in RocksDBLite."));
|
|
|
|
#else
|
|
|
|
if ((read_options.tailing) || (read_options.snapshot != nullptr) ||
|
|
|
|
(is_snapshot_supported_)) {
|
|
|
|
return new ManagedIterator(this, read_options, cfd);
|
|
|
|
}
|
|
|
|
// Managed iter not supported
|
|
|
|
return NewErrorIterator(Status::InvalidArgument(
|
|
|
|
"Managed Iterators not supported without snapshots."));
|
|
|
|
#endif
|
|
|
|
} else if (read_options.tailing) {
|
2014-04-15 22:39:26 +02:00
|
|
|
#ifdef ROCKSDB_LITE
|
|
|
|
// not supported in lite version
|
|
|
|
return nullptr;
|
|
|
|
#else
|
2014-10-24 00:34:21 +02:00
|
|
|
SuperVersion* sv = cfd->GetReferencedSuperVersion(&mutex_);
|
|
|
|
auto iter = new ForwardIterator(this, read_options, cfd, sv);
|
2015-11-05 22:24:05 +01:00
|
|
|
return NewDBIterator(
|
2017-04-10 20:13:23 +02:00
|
|
|
env_, read_options, *cfd->ioptions(), cfd->user_comparator(), iter,
|
2014-10-24 00:34:21 +02:00
|
|
|
kMaxSequenceNumber,
|
|
|
|
sv->mutable_cf_options.max_sequential_skip_in_iterations,
|
2017-04-10 20:13:23 +02:00
|
|
|
sv->version_number);
|
2014-04-15 22:39:26 +02:00
|
|
|
#endif
|
2014-01-17 06:56:26 +01:00
|
|
|
} else {
|
2014-04-14 18:34:59 +02:00
|
|
|
SequenceNumber latest_snapshot = versions_->LastSequence();
|
2014-10-24 00:34:21 +02:00
|
|
|
SuperVersion* sv = cfd->GetReferencedSuperVersion(&mutex_);
|
2014-04-14 18:34:59 +02:00
|
|
|
|
2014-03-08 01:12:34 +01:00
|
|
|
auto snapshot =
|
2014-09-09 00:04:34 +02:00
|
|
|
read_options.snapshot != nullptr
|
|
|
|
? reinterpret_cast<const SnapshotImpl*>(
|
|
|
|
read_options.snapshot)->number_
|
2014-03-08 01:12:34 +01:00
|
|
|
: latest_snapshot;
|
2014-01-17 06:56:26 +01:00
|
|
|
|
In DB::NewIterator(), try to allocate the whole iterator tree in an arena
Summary:
In this patch, try to allocate the whole iterator tree starting from DBIter from an arena
1. ArenaWrappedDBIter is created when serves as the entry point of an iterator tree, with an arena in it.
2. Add an option to create iterator from arena for following iterators: DBIter, MergingIterator, MemtableIterator, all mem table's iterators, all table reader's iterators and two level iterator.
3. MergeIteratorBuilder is created to incrementally build the tree of internal iterators. It is passed to mem table list and version set and add iterators to it.
Limitations:
(1) Only DB::NewIterator() without tailing uses the arena. Other cases, including readonly DB and compactions are still from malloc
(2) Two level iterator itself is allocated in arena, but not iterators inside it.
Test Plan: make all check
Reviewers: ljin, haobo
Reviewed By: haobo
Subscribers: leveldb, dhruba, yhchiang, igor
Differential Revision: https://reviews.facebook.net/D18513
2014-06-03 01:38:00 +02:00
|
|
|
// Try to generate a DB iterator tree in continuous memory area to be
|
|
|
|
// cache friendly. Here is an example of result:
|
|
|
|
// +-------------------------------+
|
|
|
|
// | |
|
|
|
|
// | ArenaWrappedDBIter |
|
|
|
|
// | + |
|
|
|
|
// | +---> Inner Iterator ------------+
|
|
|
|
// | | | |
|
|
|
|
// | | +-- -- -- -- -- -- -- --+ |
|
|
|
|
// | +--- | Arena | |
|
|
|
|
// | | | |
|
|
|
|
// | Allocated Memory: | |
|
|
|
|
// | | +-------------------+ |
|
|
|
|
// | | | DBIter | <---+
|
|
|
|
// | | + |
|
|
|
|
// | | | +-> iter_ ------------+
|
|
|
|
// | | | | |
|
|
|
|
// | | +-------------------+ |
|
|
|
|
// | | | MergingIterator | <---+
|
|
|
|
// | | + |
|
|
|
|
// | | | +->child iter1 ------------+
|
|
|
|
// | | | | | |
|
|
|
|
// | | +->child iter2 ----------+ |
|
|
|
|
// | | | | | | |
|
|
|
|
// | | | +->child iter3 --------+ | |
|
|
|
|
// | | | | | |
|
|
|
|
// | | +-------------------+ | | |
|
|
|
|
// | | | Iterator1 | <--------+
|
|
|
|
// | | +-------------------+ | |
|
|
|
|
// | | | Iterator2 | <------+
|
|
|
|
// | | +-------------------+ |
|
|
|
|
// | | | Iterator3 | <----+
|
|
|
|
// | | +-------------------+
|
|
|
|
// | | |
|
|
|
|
// +-------+-----------------------+
|
|
|
|
//
|
2015-09-02 22:58:22 +02:00
|
|
|
// ArenaWrappedDBIter inlines an arena area where all the iterators in
|
|
|
|
// the iterator tree are allocated in the order of being accessed when
|
In DB::NewIterator(), try to allocate the whole iterator tree in an arena
Summary:
In this patch, try to allocate the whole iterator tree starting from DBIter from an arena
1. ArenaWrappedDBIter is created when serves as the entry point of an iterator tree, with an arena in it.
2. Add an option to create iterator from arena for following iterators: DBIter, MergingIterator, MemtableIterator, all mem table's iterators, all table reader's iterators and two level iterator.
3. MergeIteratorBuilder is created to incrementally build the tree of internal iterators. It is passed to mem table list and version set and add iterators to it.
Limitations:
(1) Only DB::NewIterator() without tailing uses the arena. Other cases, including readonly DB and compactions are still from malloc
(2) Two level iterator itself is allocated in arena, but not iterators inside it.
Test Plan: make all check
Reviewers: ljin, haobo
Reviewed By: haobo
Subscribers: leveldb, dhruba, yhchiang, igor
Differential Revision: https://reviews.facebook.net/D18513
2014-06-03 01:38:00 +02:00
|
|
|
// querying.
|
|
|
|
// Laying out the iterators in the order of being accessed makes it more
|
|
|
|
// likely that any iterator pointer is close to the iterator it points to so
|
|
|
|
// that they are likely to be in the same cache line and/or page.
|
|
|
|
ArenaWrappedDBIter* db_iter = NewArenaWrappedDbIterator(
|
2017-04-10 20:13:23 +02:00
|
|
|
env_, read_options, *cfd->ioptions(), cfd->user_comparator(), snapshot,
|
2017-04-06 02:14:05 +02:00
|
|
|
sv->mutable_cf_options.max_sequential_skip_in_iterations,
|
2017-04-10 20:13:23 +02:00
|
|
|
sv->version_number);
|
2016-11-15 07:45:16 +01:00
|
|
|
|
2017-04-06 02:14:05 +02:00
|
|
|
InternalIterator* internal_iter =
|
|
|
|
NewInternalIterator(read_options, cfd, sv, db_iter->GetArena(),
|
|
|
|
db_iter->GetRangeDelAggregator());
|
|
|
|
db_iter->SetIterUnderDBIter(internal_iter);
|
2017-01-20 00:21:07 +01:00
|
|
|
|
2017-04-06 02:14:05 +02:00
|
|
|
return db_iter;
|
2016-11-15 07:45:16 +01:00
|
|
|
}
|
2017-04-06 02:14:05 +02:00
|
|
|
// To stop compiler from complaining
|
|
|
|
return nullptr;
|
2017-04-04 19:19:33 +02:00
|
|
|
}
|
|
|
|
|
2017-04-06 02:14:05 +02:00
|
|
|
Status DBImpl::NewIterators(
|
|
|
|
const ReadOptions& read_options,
|
|
|
|
const std::vector<ColumnFamilyHandle*>& column_families,
|
|
|
|
std::vector<Iterator*>* iterators) {
|
|
|
|
if (read_options.read_tier == kPersistedTier) {
|
|
|
|
return Status::NotSupported(
|
|
|
|
"ReadTier::kPersistedData is not yet supported in iterators.");
|
|
|
|
}
|
|
|
|
iterators->clear();
|
|
|
|
iterators->reserve(column_families.size());
|
|
|
|
if (read_options.managed) {
|
|
|
|
#ifdef ROCKSDB_LITE
|
|
|
|
return Status::InvalidArgument(
|
|
|
|
"Managed interator not supported in RocksDB lite");
|
|
|
|
#else
|
|
|
|
if ((!read_options.tailing) && (read_options.snapshot == nullptr) &&
|
|
|
|
(!is_snapshot_supported_)) {
|
|
|
|
return Status::InvalidArgument(
|
|
|
|
"Managed interator not supported without snapshots");
|
2017-04-04 19:19:33 +02:00
|
|
|
}
|
2017-04-06 02:14:05 +02:00
|
|
|
for (auto cfh : column_families) {
|
|
|
|
auto cfd = reinterpret_cast<ColumnFamilyHandleImpl*>(cfh)->cfd();
|
|
|
|
auto iter = new ManagedIterator(this, read_options, cfd);
|
|
|
|
iterators->push_back(iter);
|
2017-04-04 19:19:33 +02:00
|
|
|
}
|
2017-04-06 02:14:05 +02:00
|
|
|
#endif
|
|
|
|
} else if (read_options.tailing) {
|
|
|
|
#ifdef ROCKSDB_LITE
|
|
|
|
return Status::InvalidArgument(
|
|
|
|
"Tailing interator not supported in RocksDB lite");
|
|
|
|
#else
|
|
|
|
for (auto cfh : column_families) {
|
|
|
|
auto cfd = reinterpret_cast<ColumnFamilyHandleImpl*>(cfh)->cfd();
|
|
|
|
SuperVersion* sv = cfd->GetReferencedSuperVersion(&mutex_);
|
|
|
|
auto iter = new ForwardIterator(this, read_options, cfd, sv);
|
|
|
|
iterators->push_back(NewDBIterator(
|
2017-04-10 20:13:23 +02:00
|
|
|
env_, read_options, *cfd->ioptions(), cfd->user_comparator(), iter,
|
2017-04-06 02:14:05 +02:00
|
|
|
kMaxSequenceNumber,
|
|
|
|
sv->mutable_cf_options.max_sequential_skip_in_iterations,
|
2017-04-10 20:13:23 +02:00
|
|
|
sv->version_number));
|
2017-04-04 19:19:33 +02:00
|
|
|
}
|
2017-04-06 02:14:05 +02:00
|
|
|
#endif
|
|
|
|
} else {
|
|
|
|
SequenceNumber latest_snapshot = versions_->LastSequence();
|
2017-03-17 02:10:37 +01:00
|
|
|
|
2017-04-06 02:14:05 +02:00
|
|
|
for (size_t i = 0; i < column_families.size(); ++i) {
|
|
|
|
auto* cfd = reinterpret_cast<ColumnFamilyHandleImpl*>(
|
|
|
|
column_families[i])->cfd();
|
|
|
|
SuperVersion* sv = cfd->GetReferencedSuperVersion(&mutex_);
|
2017-03-17 02:10:37 +01:00
|
|
|
|
2017-04-06 02:14:05 +02:00
|
|
|
auto snapshot =
|
|
|
|
read_options.snapshot != nullptr
|
|
|
|
? reinterpret_cast<const SnapshotImpl*>(
|
|
|
|
read_options.snapshot)->number_
|
|
|
|
: latest_snapshot;
|
Push- instead of pull-model for managing Write stalls
Summary:
Introducing WriteController, which is a source of truth about per-DB write delays. Let's define an DB epoch as a period where there are no flushes and compactions (i.e. new epoch is started when flush or compaction finishes). Each epoch can either:
* proceed with all writes without delay
* delay all writes by fixed time
* stop all writes
The three modes are recomputed at each epoch change (flush, compaction), rather than on every write (which is currently the case).
When we have a lot of column families, our current pull behavior adds a big overhead, since we need to loop over every column family for every write. With new push model, overhead on Write code-path is minimal.
This is just the start. Next step is to also take care of stalls introduced by slow memtable flushes. The final goal is to eliminate function MakeRoomForWrite(), which currently needs to be called for every column family by every write.
Test Plan: make check for now. I'll add some unit tests later. Also, perf test.
Reviewers: dhruba, yhchiang, MarkCallaghan, sdong, ljin
Reviewed By: ljin
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D22791
2014-09-08 20:20:25 +02:00
|
|
|
|
2017-04-06 02:14:05 +02:00
|
|
|
ArenaWrappedDBIter* db_iter = NewArenaWrappedDbIterator(
|
2017-04-10 20:13:23 +02:00
|
|
|
env_, read_options, *cfd->ioptions(), cfd->user_comparator(),
|
|
|
|
snapshot, sv->mutable_cf_options.max_sequential_skip_in_iterations,
|
|
|
|
sv->version_number);
|
2017-04-06 02:14:05 +02:00
|
|
|
InternalIterator* internal_iter =
|
|
|
|
NewInternalIterator(read_options, cfd, sv, db_iter->GetArena(),
|
|
|
|
db_iter->GetRangeDelAggregator());
|
|
|
|
db_iter->SetIterUnderDBIter(internal_iter);
|
|
|
|
iterators->push_back(db_iter);
|
Push- instead of pull-model for managing Write stalls
Summary:
Introducing WriteController, which is a source of truth about per-DB write delays. Let's define an DB epoch as a period where there are no flushes and compactions (i.e. new epoch is started when flush or compaction finishes). Each epoch can either:
* proceed with all writes without delay
* delay all writes by fixed time
* stop all writes
The three modes are recomputed at each epoch change (flush, compaction), rather than on every write (which is currently the case).
When we have a lot of column families, our current pull behavior adds a big overhead, since we need to loop over every column family for every write. With new push model, overhead on Write code-path is minimal.
This is just the start. Next step is to also take care of stalls introduced by slow memtable flushes. The final goal is to eliminate function MakeRoomForWrite(), which currently needs to be called for every column family by every write.
Test Plan: make check for now. I'll add some unit tests later. Also, perf test.
Reviewers: dhruba, yhchiang, MarkCallaghan, sdong, ljin
Reviewed By: ljin
Subscribers: leveldb
Differential Revision: https://reviews.facebook.net/D22791
2014-09-08 20:20:25 +02:00
|
|
|
}
|
|
|
|
}
|
2013-08-01 01:20:48 +02:00
|
|
|
|
2014-09-11 03:46:09 +02:00
|
|
|
return Status::OK();
|
2014-08-12 07:10:32 +02:00
|
|
|
}
|
|
|
|
|
2017-04-06 02:14:05 +02:00
|
|
|
const Snapshot* DBImpl::GetSnapshot() { return GetSnapshotImpl(false); }
|
2016-06-02 20:57:31 +02:00
|
|
|
|
2017-04-06 02:14:05 +02:00
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
const Snapshot* DBImpl::GetSnapshotForWriteConflictBoundary() {
|
|
|
|
return GetSnapshotImpl(true);
|
2016-06-02 20:57:31 +02:00
|
|
|
}
|
|
|
|
#endif // ROCKSDB_LITE
|
|
|
|
|
2017-04-06 02:14:05 +02:00
|
|
|
const Snapshot* DBImpl::GetSnapshotImpl(bool is_write_conflict_boundary) {
|
|
|
|
int64_t unix_time = 0;
|
|
|
|
env_->GetCurrentTime(&unix_time); // Ignore error
|
|
|
|
SnapshotImpl* s = new SnapshotImpl;
|
|
|
|
|
|
|
|
InstrumentedMutexLock l(&mutex_);
|
|
|
|
// returns null if the underlying memtable does not support snapshot.
|
|
|
|
if (!is_snapshot_supported_) {
|
|
|
|
delete s;
|
|
|
|
return nullptr;
|
2015-10-01 15:12:04 +02:00
|
|
|
}
|
2017-04-06 02:14:05 +02:00
|
|
|
return snapshots_.New(s, versions_->LastSequence(), unix_time,
|
|
|
|
is_write_conflict_boundary);
|
|
|
|
}
|
2016-06-02 20:57:31 +02:00
|
|
|
|
2017-04-06 02:14:05 +02:00
|
|
|
void DBImpl::ReleaseSnapshot(const Snapshot* s) {
|
|
|
|
const SnapshotImpl* casted_s = reinterpret_cast<const SnapshotImpl*>(s);
|
2014-08-12 07:10:32 +02:00
|
|
|
{
|
2017-04-06 02:14:05 +02:00
|
|
|
InstrumentedMutexLock l(&mutex_);
|
|
|
|
snapshots_.Delete(casted_s);
|
2011-04-12 21:38:58 +02:00
|
|
|
}
|
2017-04-06 02:14:05 +02:00
|
|
|
delete casted_s;
|
2011-04-12 21:38:58 +02:00
|
|
|
}
|
|
|
|
|
2014-04-15 22:39:26 +02:00
|
|
|
#ifndef ROCKSDB_LITE
|
2014-02-15 02:02:10 +01:00
|
|
|
Status DBImpl::GetPropertiesOfAllTables(ColumnFamilyHandle* column_family,
|
|
|
|
TablePropertiesCollection* props) {
|
|
|
|
auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
|
|
|
|
auto cfd = cfh->cfd();
|
|
|
|
|
2014-02-14 01:28:21 +01:00
|
|
|
// Increment the ref count
|
|
|
|
mutex_.Lock();
|
2014-02-15 02:02:10 +01:00
|
|
|
auto version = cfd->current();
|
2014-02-14 01:28:21 +01:00
|
|
|
version->Ref();
|
|
|
|
mutex_.Unlock();
|
|
|
|
|
|
|
|
auto s = version->GetPropertiesOfAllTables(props);
|
|
|
|
|
|
|
|
// Decrement the ref count
|
|
|
|
mutex_.Lock();
|
|
|
|
version->Unref();
|
|
|
|
mutex_.Unlock();
|
|
|
|
|
|
|
|
return s;
|
|
|
|
}
|
2015-10-13 23:24:45 +02:00
|
|
|
|
|
|
|
Status DBImpl::GetPropertiesOfTablesInRange(ColumnFamilyHandle* column_family,
|
2015-10-19 19:34:55 +02:00
|
|
|
const Range* range, std::size_t n,
|
2015-10-13 23:24:45 +02:00
|
|
|
TablePropertiesCollection* props) {
|
|
|
|
auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
|
|
|
|
auto cfd = cfh->cfd();
|
|
|
|
|
|
|
|
// Increment the ref count
|
|
|
|
mutex_.Lock();
|
|
|
|
auto version = cfd->current();
|
|
|
|
version->Ref();
|
|
|
|
mutex_.Unlock();
|
|
|
|
|
|
|
|
auto s = version->GetPropertiesOfTablesInRange(range, n, props);
|
|
|
|
|
|
|
|
// Decrement the ref count
|
|
|
|
mutex_.Lock();
|
|
|
|
version->Unref();
|
|
|
|
mutex_.Unlock();
|
|
|
|
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
2014-04-15 22:39:26 +02:00
|
|
|
#endif // ROCKSDB_LITE
|
2014-02-14 01:28:21 +01:00
|
|
|
|
[RocksDB] BackupableDB
Summary:
In this diff I present you BackupableDB v1. You can easily use it to backup your DB and it will do incremental snapshots for you.
Let's first describe how you would use BackupableDB. It's inheriting StackableDB interface so you can easily construct it with your DB object -- it will add a method RollTheSnapshot() to the DB object. When you call RollTheSnapshot(), current snapshot of the DB will be stored in the backup dir. To restore, you can just call RestoreDBFromBackup() on a BackupableDB (which is a static method) and it will restore all files from the backup dir. In the next version, it will even support automatic backuping every X minutes.
There are multiple things you can configure:
1. backup_env and db_env can be different, which is awesome because then you can easily backup to HDFS or wherever you feel like.
2. sync - if true, it *guarantees* backup consistency on machine reboot
3. number of snapshots to keep - this will keep last N snapshots around if you want, for some reason, be able to restore from an earlier snapshot. All the backuping is done in incremental fashion - if we already have 00010.sst, we will not copy it again. *IMPORTANT* -- This is based on assumption that 00010.sst never changes - two files named 00010.sst from the same DB will always be exactly the same. Is this true? I always copy manifest, current and log files.
4. You can decide if you want to flush the memtables before you backup, or you're fine with backing up the log files -- either way, you get a complete and consistent view of the database at a time of backup.
5. More things you can find in BackupableDBOptions
Here is the directory structure I use:
backup_dir/CURRENT_SNAPSHOT - just 4 bytes holding the latest snapshot
0, 1, 2, ... - files containing serialized version of each snapshot - containing a list of files
files/*.sst - sst files shared between snapshots - if one snapshot references 00010.sst and another one needs to backup it from the DB, it will just reference the same file
files/ 0/, 1/, 2/, ... - snapshot directories containing private snapshot files - current, manifest and log files
All the files are ref counted and deleted immediatelly when they get out of scope.
Some other stuff in this diff:
1. Added GetEnv() method to the DB. Discussed with @haobo and we agreed that it seems right thing to do.
2. Fixed StackableDB interface. The way it was set up before, I was not able to implement BackupableDB.
Test Plan:
I have a unittest, but please don't look at this yet. I just hacked it up to help me with debugging. I will write a lot of good tests and update the diff.
Also, `make asan_check`
Reviewers: dhruba, haobo, emayanke
Reviewed By: dhruba
CC: leveldb, haobo
Differential Revision: https://reviews.facebook.net/D14295
2013-12-09 23:06:52 +01:00
|
|
|
const std::string& DBImpl::GetName() const {
|
|
|
|
return dbname_;
|
|
|
|
}
|
|
|
|
|
2013-11-25 21:39:23 +01:00
|
|
|
Env* DBImpl::GetEnv() const {
|
|
|
|
return env_;
|
|
|
|
}
|
|
|
|
|
2016-09-15 07:10:28 +02:00
|
|
|
Options DBImpl::GetOptions(ColumnFamilyHandle* column_family) const {
|
|
|
|
InstrumentedMutexLock l(&mutex_);
|
2014-02-11 02:04:44 +01:00
|
|
|
auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
|
2016-09-24 01:34:04 +02:00
|
|
|
return Options(BuildDBOptions(immutable_db_options_, mutable_db_options_),
|
|
|
|
cfh->cfd()->GetLatestCFOptions());
|
2013-11-26 00:51:50 +01:00
|
|
|
}
|
|
|
|
|
2016-09-24 01:34:04 +02:00
|
|
|
DBOptions DBImpl::GetDBOptions() const {
|
|
|
|
InstrumentedMutexLock l(&mutex_);
|
|
|
|
return BuildDBOptions(immutable_db_options_, mutable_db_options_);
|
|
|
|
}
|
2015-05-11 23:51:51 +02:00
|
|
|
|
2014-02-11 02:04:44 +01:00
|
|
|
bool DBImpl::GetProperty(ColumnFamilyHandle* column_family,
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 20:14:09 +01:00
|
|
|
const Slice& property, std::string* value) {
|
Eliminate duplicated property constants
Summary:
Before this diff, there were duplicated constants to refer to properties (user-
facing API had strings and InternalStats had an enum). I noticed these were
inconsistent in terms of which constants are provided, names of constants, and
documentation of constants. Overall it seemed annoying/error-prone to maintain
these duplicated constants.
So, this diff gets rid of InternalStats's constants and replaces them with a map
keyed on the user-facing constant. The value in that map contains a function
pointer to get the property value, so we don't need to do string matching while
holding db->mutex_. This approach has a side benefit of making many small
handler functions rather than a giant switch-statement.
Test Plan: db_properties_test passes, running "make commit-prereq -j32"
Reviewers: sdong, yhchiang, kradhakrishnan, IslamAbdelRahman, rven, anthony
Reviewed By: anthony
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D53253
2016-02-03 04:14:56 +01:00
|
|
|
const DBPropertyInfo* property_info = GetPropertyInfo(property);
|
2011-04-12 21:38:58 +02:00
|
|
|
value->clear();
|
2015-11-04 00:54:18 +01:00
|
|
|
auto cfd = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family)->cfd();
|
Eliminate duplicated property constants
Summary:
Before this diff, there were duplicated constants to refer to properties (user-
facing API had strings and InternalStats had an enum). I noticed these were
inconsistent in terms of which constants are provided, names of constants, and
documentation of constants. Overall it seemed annoying/error-prone to maintain
these duplicated constants.
So, this diff gets rid of InternalStats's constants and replaces them with a map
keyed on the user-facing constant. The value in that map contains a function
pointer to get the property value, so we don't need to do string matching while
holding db->mutex_. This approach has a side benefit of making many small
handler functions rather than a giant switch-statement.
Test Plan: db_properties_test passes, running "make commit-prereq -j32"
Reviewers: sdong, yhchiang, kradhakrishnan, IslamAbdelRahman, rven, anthony
Reviewed By: anthony
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D53253
2016-02-03 04:14:56 +01:00
|
|
|
if (property_info == nullptr) {
|
|
|
|
return false;
|
|
|
|
} else if (property_info->handle_int) {
|
2014-08-05 20:27:34 +02:00
|
|
|
uint64_t int_value;
|
Eliminate duplicated property constants
Summary:
Before this diff, there were duplicated constants to refer to properties (user-
facing API had strings and InternalStats had an enum). I noticed these were
inconsistent in terms of which constants are provided, names of constants, and
documentation of constants. Overall it seemed annoying/error-prone to maintain
these duplicated constants.
So, this diff gets rid of InternalStats's constants and replaces them with a map
keyed on the user-facing constant. The value in that map contains a function
pointer to get the property value, so we don't need to do string matching while
holding db->mutex_. This approach has a side benefit of making many small
handler functions rather than a giant switch-statement.
Test Plan: db_properties_test passes, running "make commit-prereq -j32"
Reviewers: sdong, yhchiang, kradhakrishnan, IslamAbdelRahman, rven, anthony
Reviewed By: anthony
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D53253
2016-02-03 04:14:56 +01:00
|
|
|
bool ret_value =
|
|
|
|
GetIntPropertyInternal(cfd, *property_info, false, &int_value);
|
2014-08-05 20:27:34 +02:00
|
|
|
if (ret_value) {
|
2014-11-25 05:44:49 +01:00
|
|
|
*value = ToString(int_value);
|
2014-08-05 20:27:34 +02:00
|
|
|
}
|
|
|
|
return ret_value;
|
Eliminate duplicated property constants
Summary:
Before this diff, there were duplicated constants to refer to properties (user-
facing API had strings and InternalStats had an enum). I noticed these were
inconsistent in terms of which constants are provided, names of constants, and
documentation of constants. Overall it seemed annoying/error-prone to maintain
these duplicated constants.
So, this diff gets rid of InternalStats's constants and replaces them with a map
keyed on the user-facing constant. The value in that map contains a function
pointer to get the property value, so we don't need to do string matching while
holding db->mutex_. This approach has a side benefit of making many small
handler functions rather than a giant switch-statement.
Test Plan: db_properties_test passes, running "make commit-prereq -j32"
Reviewers: sdong, yhchiang, kradhakrishnan, IslamAbdelRahman, rven, anthony
Reviewed By: anthony
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D53253
2016-02-03 04:14:56 +01:00
|
|
|
} else if (property_info->handle_string) {
|
2015-02-05 06:39:45 +01:00
|
|
|
InstrumentedMutexLock l(&mutex_);
|
Eliminate duplicated property constants
Summary:
Before this diff, there were duplicated constants to refer to properties (user-
facing API had strings and InternalStats had an enum). I noticed these were
inconsistent in terms of which constants are provided, names of constants, and
documentation of constants. Overall it seemed annoying/error-prone to maintain
these duplicated constants.
So, this diff gets rid of InternalStats's constants and replaces them with a map
keyed on the user-facing constant. The value in that map contains a function
pointer to get the property value, so we don't need to do string matching while
holding db->mutex_. This approach has a side benefit of making many small
handler functions rather than a giant switch-statement.
Test Plan: db_properties_test passes, running "make commit-prereq -j32"
Reviewers: sdong, yhchiang, kradhakrishnan, IslamAbdelRahman, rven, anthony
Reviewed By: anthony
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D53253
2016-02-03 04:14:56 +01:00
|
|
|
return cfd->internal_stats()->GetStringProperty(*property_info, property,
|
2014-08-05 20:27:34 +02:00
|
|
|
value);
|
|
|
|
}
|
Eliminate duplicated property constants
Summary:
Before this diff, there were duplicated constants to refer to properties (user-
facing API had strings and InternalStats had an enum). I noticed these were
inconsistent in terms of which constants are provided, names of constants, and
documentation of constants. Overall it seemed annoying/error-prone to maintain
these duplicated constants.
So, this diff gets rid of InternalStats's constants and replaces them with a map
keyed on the user-facing constant. The value in that map contains a function
pointer to get the property value, so we don't need to do string matching while
holding db->mutex_. This approach has a side benefit of making many small
handler functions rather than a giant switch-statement.
Test Plan: db_properties_test passes, running "make commit-prereq -j32"
Reviewers: sdong, yhchiang, kradhakrishnan, IslamAbdelRahman, rven, anthony
Reviewed By: anthony
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D53253
2016-02-03 04:14:56 +01:00
|
|
|
// Shouldn't reach here since exactly one of handle_string and handle_int
|
|
|
|
// should be non-nullptr.
|
|
|
|
assert(false);
|
|
|
|
return false;
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2016-11-12 05:45:47 +01:00
|
|
|
bool DBImpl::GetMapProperty(ColumnFamilyHandle* column_family,
|
|
|
|
const Slice& property,
|
|
|
|
std::map<std::string, double>* value) {
|
|
|
|
const DBPropertyInfo* property_info = GetPropertyInfo(property);
|
|
|
|
value->clear();
|
|
|
|
auto cfd = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family)->cfd();
|
|
|
|
if (property_info == nullptr) {
|
|
|
|
return false;
|
|
|
|
} else if (property_info->handle_map) {
|
|
|
|
InstrumentedMutexLock l(&mutex_);
|
|
|
|
return cfd->internal_stats()->GetMapProperty(*property_info, property,
|
|
|
|
value);
|
|
|
|
}
|
|
|
|
// If we reach this point it means that handle_map is not provided for the
|
|
|
|
// requested property
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2014-07-29 00:28:53 +02:00
|
|
|
bool DBImpl::GetIntProperty(ColumnFamilyHandle* column_family,
|
|
|
|
const Slice& property, uint64_t* value) {
|
Eliminate duplicated property constants
Summary:
Before this diff, there were duplicated constants to refer to properties (user-
facing API had strings and InternalStats had an enum). I noticed these were
inconsistent in terms of which constants are provided, names of constants, and
documentation of constants. Overall it seemed annoying/error-prone to maintain
these duplicated constants.
So, this diff gets rid of InternalStats's constants and replaces them with a map
keyed on the user-facing constant. The value in that map contains a function
pointer to get the property value, so we don't need to do string matching while
holding db->mutex_. This approach has a side benefit of making many small
handler functions rather than a giant switch-statement.
Test Plan: db_properties_test passes, running "make commit-prereq -j32"
Reviewers: sdong, yhchiang, kradhakrishnan, IslamAbdelRahman, rven, anthony
Reviewed By: anthony
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D53253
2016-02-03 04:14:56 +01:00
|
|
|
const DBPropertyInfo* property_info = GetPropertyInfo(property);
|
|
|
|
if (property_info == nullptr || property_info->handle_int == nullptr) {
|
2014-08-05 20:27:34 +02:00
|
|
|
return false;
|
|
|
|
}
|
2015-11-04 00:54:18 +01:00
|
|
|
auto cfd = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family)->cfd();
|
Eliminate duplicated property constants
Summary:
Before this diff, there were duplicated constants to refer to properties (user-
facing API had strings and InternalStats had an enum). I noticed these were
inconsistent in terms of which constants are provided, names of constants, and
documentation of constants. Overall it seemed annoying/error-prone to maintain
these duplicated constants.
So, this diff gets rid of InternalStats's constants and replaces them with a map
keyed on the user-facing constant. The value in that map contains a function
pointer to get the property value, so we don't need to do string matching while
holding db->mutex_. This approach has a side benefit of making many small
handler functions rather than a giant switch-statement.
Test Plan: db_properties_test passes, running "make commit-prereq -j32"
Reviewers: sdong, yhchiang, kradhakrishnan, IslamAbdelRahman, rven, anthony
Reviewed By: anthony
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D53253
2016-02-03 04:14:56 +01:00
|
|
|
return GetIntPropertyInternal(cfd, *property_info, false, value);
|
2014-08-05 20:27:34 +02:00
|
|
|
}
|
|
|
|
|
2015-11-04 00:54:18 +01:00
|
|
|
bool DBImpl::GetIntPropertyInternal(ColumnFamilyData* cfd,
|
Eliminate duplicated property constants
Summary:
Before this diff, there were duplicated constants to refer to properties (user-
facing API had strings and InternalStats had an enum). I noticed these were
inconsistent in terms of which constants are provided, names of constants, and
documentation of constants. Overall it seemed annoying/error-prone to maintain
these duplicated constants.
So, this diff gets rid of InternalStats's constants and replaces them with a map
keyed on the user-facing constant. The value in that map contains a function
pointer to get the property value, so we don't need to do string matching while
holding db->mutex_. This approach has a side benefit of making many small
handler functions rather than a giant switch-statement.
Test Plan: db_properties_test passes, running "make commit-prereq -j32"
Reviewers: sdong, yhchiang, kradhakrishnan, IslamAbdelRahman, rven, anthony
Reviewed By: anthony
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D53253
2016-02-03 04:14:56 +01:00
|
|
|
const DBPropertyInfo& property_info,
|
|
|
|
bool is_locked, uint64_t* value) {
|
|
|
|
assert(property_info.handle_int != nullptr);
|
|
|
|
if (!property_info.need_out_of_mutex) {
|
2015-11-04 00:54:18 +01:00
|
|
|
if (is_locked) {
|
|
|
|
mutex_.AssertHeld();
|
Eliminate duplicated property constants
Summary:
Before this diff, there were duplicated constants to refer to properties (user-
facing API had strings and InternalStats had an enum). I noticed these were
inconsistent in terms of which constants are provided, names of constants, and
documentation of constants. Overall it seemed annoying/error-prone to maintain
these duplicated constants.
So, this diff gets rid of InternalStats's constants and replaces them with a map
keyed on the user-facing constant. The value in that map contains a function
pointer to get the property value, so we don't need to do string matching while
holding db->mutex_. This approach has a side benefit of making many small
handler functions rather than a giant switch-statement.
Test Plan: db_properties_test passes, running "make commit-prereq -j32"
Reviewers: sdong, yhchiang, kradhakrishnan, IslamAbdelRahman, rven, anthony
Reviewed By: anthony
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D53253
2016-02-03 04:14:56 +01:00
|
|
|
return cfd->internal_stats()->GetIntProperty(property_info, value, this);
|
2015-11-04 00:54:18 +01:00
|
|
|
} else {
|
|
|
|
InstrumentedMutexLock l(&mutex_);
|
Eliminate duplicated property constants
Summary:
Before this diff, there were duplicated constants to refer to properties (user-
facing API had strings and InternalStats had an enum). I noticed these were
inconsistent in terms of which constants are provided, names of constants, and
documentation of constants. Overall it seemed annoying/error-prone to maintain
these duplicated constants.
So, this diff gets rid of InternalStats's constants and replaces them with a map
keyed on the user-facing constant. The value in that map contains a function
pointer to get the property value, so we don't need to do string matching while
holding db->mutex_. This approach has a side benefit of making many small
handler functions rather than a giant switch-statement.
Test Plan: db_properties_test passes, running "make commit-prereq -j32"
Reviewers: sdong, yhchiang, kradhakrishnan, IslamAbdelRahman, rven, anthony
Reviewed By: anthony
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D53253
2016-02-03 04:14:56 +01:00
|
|
|
return cfd->internal_stats()->GetIntProperty(property_info, value, this);
|
2015-11-04 00:54:18 +01:00
|
|
|
}
|
2014-08-05 20:27:34 +02:00
|
|
|
} else {
|
2015-11-04 00:54:18 +01:00
|
|
|
SuperVersion* sv = nullptr;
|
|
|
|
if (!is_locked) {
|
|
|
|
sv = GetAndRefSuperVersion(cfd);
|
|
|
|
} else {
|
|
|
|
sv = cfd->GetSuperVersion();
|
|
|
|
}
|
2014-08-05 20:27:34 +02:00
|
|
|
|
|
|
|
bool ret = cfd->internal_stats()->GetIntPropertyOutOfMutex(
|
Eliminate duplicated property constants
Summary:
Before this diff, there were duplicated constants to refer to properties (user-
facing API had strings and InternalStats had an enum). I noticed these were
inconsistent in terms of which constants are provided, names of constants, and
documentation of constants. Overall it seemed annoying/error-prone to maintain
these duplicated constants.
So, this diff gets rid of InternalStats's constants and replaces them with a map
keyed on the user-facing constant. The value in that map contains a function
pointer to get the property value, so we don't need to do string matching while
holding db->mutex_. This approach has a side benefit of making many small
handler functions rather than a giant switch-statement.
Test Plan: db_properties_test passes, running "make commit-prereq -j32"
Reviewers: sdong, yhchiang, kradhakrishnan, IslamAbdelRahman, rven, anthony
Reviewed By: anthony
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D53253
2016-02-03 04:14:56 +01:00
|
|
|
property_info, sv->current, value);
|
2014-08-05 20:27:34 +02:00
|
|
|
|
2015-11-04 00:54:18 +01:00
|
|
|
if (!is_locked) {
|
|
|
|
ReturnAndCleanupSuperVersion(cfd, sv);
|
|
|
|
}
|
2014-08-05 20:27:34 +02:00
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-11-04 00:54:18 +01:00
|
|
|
bool DBImpl::GetAggregatedIntProperty(const Slice& property,
|
|
|
|
uint64_t* aggregated_value) {
|
Eliminate duplicated property constants
Summary:
Before this diff, there were duplicated constants to refer to properties (user-
facing API had strings and InternalStats had an enum). I noticed these were
inconsistent in terms of which constants are provided, names of constants, and
documentation of constants. Overall it seemed annoying/error-prone to maintain
these duplicated constants.
So, this diff gets rid of InternalStats's constants and replaces them with a map
keyed on the user-facing constant. The value in that map contains a function
pointer to get the property value, so we don't need to do string matching while
holding db->mutex_. This approach has a side benefit of making many small
handler functions rather than a giant switch-statement.
Test Plan: db_properties_test passes, running "make commit-prereq -j32"
Reviewers: sdong, yhchiang, kradhakrishnan, IslamAbdelRahman, rven, anthony
Reviewed By: anthony
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D53253
2016-02-03 04:14:56 +01:00
|
|
|
const DBPropertyInfo* property_info = GetPropertyInfo(property);
|
|
|
|
if (property_info == nullptr || property_info->handle_int == nullptr) {
|
2015-11-04 00:54:18 +01:00
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
uint64_t sum = 0;
|
|
|
|
{
|
|
|
|
// Needs mutex to protect the list of column families.
|
|
|
|
InstrumentedMutexLock l(&mutex_);
|
|
|
|
uint64_t value;
|
|
|
|
for (auto* cfd : *versions_->GetColumnFamilySet()) {
|
Eliminate duplicated property constants
Summary:
Before this diff, there were duplicated constants to refer to properties (user-
facing API had strings and InternalStats had an enum). I noticed these were
inconsistent in terms of which constants are provided, names of constants, and
documentation of constants. Overall it seemed annoying/error-prone to maintain
these duplicated constants.
So, this diff gets rid of InternalStats's constants and replaces them with a map
keyed on the user-facing constant. The value in that map contains a function
pointer to get the property value, so we don't need to do string matching while
holding db->mutex_. This approach has a side benefit of making many small
handler functions rather than a giant switch-statement.
Test Plan: db_properties_test passes, running "make commit-prereq -j32"
Reviewers: sdong, yhchiang, kradhakrishnan, IslamAbdelRahman, rven, anthony
Reviewed By: anthony
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D53253
2016-02-03 04:14:56 +01:00
|
|
|
if (GetIntPropertyInternal(cfd, *property_info, true, &value)) {
|
2015-11-04 00:54:18 +01:00
|
|
|
sum += value;
|
|
|
|
} else {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
*aggregated_value = sum;
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2014-08-05 20:27:34 +02:00
|
|
|
SuperVersion* DBImpl::GetAndRefSuperVersion(ColumnFamilyData* cfd) {
|
|
|
|
// TODO(ljin): consider using GetReferencedSuperVersion() directly
|
2014-09-24 22:12:16 +02:00
|
|
|
return cfd->GetThreadLocalSuperVersion(&mutex_);
|
2014-08-05 20:27:34 +02:00
|
|
|
}
|
|
|
|
|
2015-05-29 23:36:35 +02:00
|
|
|
// REQUIRED: this function should only be called on the write thread or if the
|
|
|
|
// mutex is held.
|
|
|
|
SuperVersion* DBImpl::GetAndRefSuperVersion(uint32_t column_family_id) {
|
|
|
|
auto column_family_set = versions_->GetColumnFamilySet();
|
|
|
|
auto cfd = column_family_set->GetColumnFamily(column_family_id);
|
|
|
|
if (!cfd) {
|
|
|
|
return nullptr;
|
|
|
|
}
|
|
|
|
|
|
|
|
return GetAndRefSuperVersion(cfd);
|
|
|
|
}
|
|
|
|
|
2014-08-05 20:27:34 +02:00
|
|
|
void DBImpl::ReturnAndCleanupSuperVersion(ColumnFamilyData* cfd,
|
|
|
|
SuperVersion* sv) {
|
2014-09-24 22:12:16 +02:00
|
|
|
bool unref_sv = !cfd->ReturnThreadLocalSuperVersion(sv);
|
2014-08-05 20:27:34 +02:00
|
|
|
|
|
|
|
if (unref_sv) {
|
|
|
|
// Release SuperVersion
|
|
|
|
if (sv->Unref()) {
|
|
|
|
{
|
2015-02-05 06:39:45 +01:00
|
|
|
InstrumentedMutexLock l(&mutex_);
|
2014-08-05 20:27:34 +02:00
|
|
|
sv->Cleanup();
|
|
|
|
}
|
|
|
|
delete sv;
|
|
|
|
RecordTick(stats_, NUMBER_SUPERVERSION_CLEANUPS);
|
|
|
|
}
|
|
|
|
RecordTick(stats_, NUMBER_SUPERVERSION_RELEASES);
|
|
|
|
}
|
2014-07-29 00:28:53 +02:00
|
|
|
}
|
|
|
|
|
2015-05-29 23:36:35 +02:00
|
|
|
// REQUIRED: this function should only be called on the write thread.
|
|
|
|
void DBImpl::ReturnAndCleanupSuperVersion(uint32_t column_family_id,
|
|
|
|
SuperVersion* sv) {
|
|
|
|
auto column_family_set = versions_->GetColumnFamilySet();
|
|
|
|
auto cfd = column_family_set->GetColumnFamily(column_family_id);
|
|
|
|
|
|
|
|
// If SuperVersion is held, and we successfully fetched a cfd using
|
|
|
|
// GetAndRefSuperVersion(), it must still exist.
|
|
|
|
assert(cfd != nullptr);
|
|
|
|
ReturnAndCleanupSuperVersion(cfd, sv);
|
|
|
|
}
|
|
|
|
|
|
|
|
// REQUIRED: this function should only be called on the write thread or if the
|
|
|
|
// mutex is held.
|
|
|
|
ColumnFamilyHandle* DBImpl::GetColumnFamilyHandle(uint32_t column_family_id) {
|
|
|
|
ColumnFamilyMemTables* cf_memtables = column_family_memtables_.get();
|
|
|
|
|
|
|
|
if (!cf_memtables->Seek(column_family_id)) {
|
|
|
|
return nullptr;
|
|
|
|
}
|
|
|
|
|
|
|
|
return cf_memtables->GetColumnFamilyHandle();
|
|
|
|
}
|
|
|
|
|
2017-02-06 23:42:38 +01:00
|
|
|
void DBImpl::GetApproximateMemTableStats(ColumnFamilyHandle* column_family,
|
|
|
|
const Range& range,
|
|
|
|
uint64_t* const count,
|
|
|
|
uint64_t* const size) {
|
|
|
|
ColumnFamilyHandleImpl* cfh =
|
|
|
|
reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
|
|
|
|
ColumnFamilyData* cfd = cfh->cfd();
|
|
|
|
SuperVersion* sv = GetAndRefSuperVersion(cfd);
|
|
|
|
|
|
|
|
// Convert user_key into a corresponding internal key.
|
|
|
|
InternalKey k1(range.start, kMaxSequenceNumber, kValueTypeForSeek);
|
|
|
|
InternalKey k2(range.limit, kMaxSequenceNumber, kValueTypeForSeek);
|
|
|
|
MemTable::MemTableStats memStats =
|
|
|
|
sv->mem->ApproximateStats(k1.Encode(), k2.Encode());
|
|
|
|
MemTable::MemTableStats immStats =
|
|
|
|
sv->imm->ApproximateStats(k1.Encode(), k2.Encode());
|
|
|
|
*count = memStats.count + immStats.count;
|
|
|
|
*size = memStats.size + immStats.size;
|
|
|
|
|
|
|
|
ReturnAndCleanupSuperVersion(cfd, sv);
|
|
|
|
}
|
|
|
|
|
2014-02-11 02:04:44 +01:00
|
|
|
void DBImpl::GetApproximateSizes(ColumnFamilyHandle* column_family,
|
2015-06-13 03:04:30 +02:00
|
|
|
const Range* range, int n, uint64_t* sizes,
|
2017-01-20 18:26:07 +01:00
|
|
|
uint8_t include_flags) {
|
|
|
|
assert(include_flags & DB::SizeApproximationFlags::INCLUDE_FILES ||
|
|
|
|
include_flags & DB::SizeApproximationFlags::INCLUDE_MEMTABLES);
|
2011-03-18 23:37:00 +01:00
|
|
|
Version* v;
|
2014-02-11 02:04:44 +01:00
|
|
|
auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
|
|
|
|
auto cfd = cfh->cfd();
|
2015-06-13 03:04:30 +02:00
|
|
|
SuperVersion* sv = GetAndRefSuperVersion(cfd);
|
|
|
|
v = sv->current;
|
2011-03-18 23:37:00 +01:00
|
|
|
|
|
|
|
for (int i = 0; i < n; i++) {
|
|
|
|
// Convert user_key into a corresponding internal key.
|
2015-04-30 00:36:21 +02:00
|
|
|
InternalKey k1(range[i].start, kMaxSequenceNumber, kValueTypeForSeek);
|
|
|
|
InternalKey k2(range[i].limit, kMaxSequenceNumber, kValueTypeForSeek);
|
2017-01-21 00:51:21 +01:00
|
|
|
sizes[i] = 0;
|
2017-01-20 18:26:07 +01:00
|
|
|
if (include_flags & DB::SizeApproximationFlags::INCLUDE_FILES) {
|
2017-01-21 00:51:21 +01:00
|
|
|
sizes[i] += versions_->ApproximateSize(v, k1.Encode(), k2.Encode());
|
2017-01-20 18:26:07 +01:00
|
|
|
}
|
|
|
|
if (include_flags & DB::SizeApproximationFlags::INCLUDE_MEMTABLES) {
|
2017-02-06 23:42:38 +01:00
|
|
|
sizes[i] += sv->mem->ApproximateStats(k1.Encode(), k2.Encode()).size;
|
|
|
|
sizes[i] += sv->imm->ApproximateStats(k1.Encode(), k2.Encode()).size;
|
2015-06-13 03:04:30 +02:00
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2015-06-13 03:04:30 +02:00
|
|
|
ReturnAndCleanupSuperVersion(cfd, sv);
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2014-11-07 20:50:34 +01:00
|
|
|
std::list<uint64_t>::iterator
|
|
|
|
DBImpl::CaptureCurrentFileNumberInPendingOutputs() {
|
|
|
|
// We need to remember the iterator of our insert, because after the
|
|
|
|
// background job is done, we need to remove that element from
|
|
|
|
// pending_outputs_.
|
|
|
|
pending_outputs_.push_back(versions_->current_next_file_number());
|
|
|
|
auto pending_outputs_inserted_elem = pending_outputs_.end();
|
|
|
|
--pending_outputs_inserted_elem;
|
|
|
|
return pending_outputs_inserted_elem;
|
|
|
|
}
|
|
|
|
|
|
|
|
void DBImpl::ReleaseFileNumberFromPendingOutputs(
|
|
|
|
std::list<uint64_t>::iterator v) {
|
|
|
|
pending_outputs_.erase(v);
|
|
|
|
}
|
|
|
|
|
2014-04-15 22:39:26 +02:00
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
Status DBImpl::GetUpdatesSince(
|
|
|
|
SequenceNumber seq, unique_ptr<TransactionLogIterator>* iter,
|
|
|
|
const TransactionLogIterator::ReadOptions& read_options) {
|
|
|
|
|
2014-07-28 21:05:36 +02:00
|
|
|
RecordTick(stats_, GET_UPDATES_SINCE_CALLS);
|
2014-04-15 22:39:26 +02:00
|
|
|
if (seq > versions_->LastSequence()) {
|
|
|
|
return Status::NotFound("Requested sequence not yet written in the db");
|
|
|
|
}
|
2014-10-30 01:43:37 +01:00
|
|
|
return wal_manager_.GetUpdatesSince(seq, iter, read_options, versions_.get());
|
2014-04-15 22:39:26 +02:00
|
|
|
}
|
|
|
|
|
2013-08-22 23:32:53 +02:00
|
|
|
Status DBImpl::DeleteFile(std::string name) {
|
|
|
|
uint64_t number;
|
|
|
|
FileType type;
|
2013-10-24 08:39:23 +02:00
|
|
|
WalFileType log_type;
|
|
|
|
if (!ParseFileName(name, &number, &type, &log_type) ||
|
|
|
|
(type != kTableFile && type != kLogFile)) {
|
2017-03-16 03:22:52 +01:00
|
|
|
ROCKS_LOG_ERROR(immutable_db_options_.info_log, "DeleteFile %s failed.\n",
|
|
|
|
name.c_str());
|
2013-08-22 23:32:53 +02:00
|
|
|
return Status::InvalidArgument("Invalid file name");
|
|
|
|
}
|
|
|
|
|
2013-10-24 08:39:23 +02:00
|
|
|
Status status;
|
|
|
|
if (type == kLogFile) {
|
|
|
|
// Only allow deleting archived log files
|
|
|
|
if (log_type != kArchivedLogFile) {
|
2017-03-16 03:22:52 +01:00
|
|
|
ROCKS_LOG_ERROR(immutable_db_options_.info_log,
|
|
|
|
"DeleteFile %s failed - not archived log.\n",
|
|
|
|
name.c_str());
|
2013-10-24 08:39:23 +02:00
|
|
|
return Status::NotSupported("Delete only supported for archived logs");
|
|
|
|
}
|
2016-09-24 01:34:04 +02:00
|
|
|
status =
|
|
|
|
env_->DeleteFile(immutable_db_options_.wal_dir + "/" + name.c_str());
|
2013-10-24 08:39:23 +02:00
|
|
|
if (!status.ok()) {
|
2017-03-16 03:22:52 +01:00
|
|
|
ROCKS_LOG_ERROR(immutable_db_options_.info_log,
|
|
|
|
"DeleteFile %s failed -- %s.\n", name.c_str(),
|
|
|
|
status.ToString().c_str());
|
2013-10-24 08:39:23 +02:00
|
|
|
}
|
|
|
|
return status;
|
|
|
|
}
|
|
|
|
|
2013-08-22 23:32:53 +02:00
|
|
|
int level;
|
2014-04-08 00:38:53 +02:00
|
|
|
FileMetaData* metadata;
|
2014-01-27 23:33:50 +01:00
|
|
|
ColumnFamilyData* cfd;
|
2014-01-15 00:27:09 +01:00
|
|
|
VersionEdit edit;
|
2015-02-12 18:54:48 +01:00
|
|
|
JobContext job_context(next_job_id_.fetch_add(1), true);
|
2013-08-27 23:54:06 +02:00
|
|
|
{
|
2015-02-05 06:39:45 +01:00
|
|
|
InstrumentedMutexLock l(&mutex_);
|
2014-01-27 23:33:50 +01:00
|
|
|
status = versions_->GetMetadataForFile(number, &level, &metadata, &cfd);
|
2013-08-27 23:54:06 +02:00
|
|
|
if (!status.ok()) {
|
2017-03-16 03:22:52 +01:00
|
|
|
ROCKS_LOG_WARN(immutable_db_options_.info_log,
|
|
|
|
"DeleteFile %s failed. File not found\n", name.c_str());
|
2014-11-15 01:20:24 +01:00
|
|
|
job_context.Clean();
|
2013-08-27 23:54:06 +02:00
|
|
|
return Status::InvalidArgument("File not found");
|
|
|
|
}
|
2014-10-21 20:23:06 +02:00
|
|
|
assert(level < cfd->NumberLevels());
|
2013-08-22 23:32:53 +02:00
|
|
|
|
2013-08-27 23:54:06 +02:00
|
|
|
// If the file is being compacted no need to delete.
|
2014-01-07 05:29:17 +01:00
|
|
|
if (metadata->being_compacted) {
|
2017-03-16 03:22:52 +01:00
|
|
|
ROCKS_LOG_INFO(immutable_db_options_.info_log,
|
|
|
|
"DeleteFile %s Skipped. File about to be compacted\n",
|
|
|
|
name.c_str());
|
2014-11-15 01:20:24 +01:00
|
|
|
job_context.Clean();
|
2013-08-27 23:54:06 +02:00
|
|
|
return Status::OK();
|
2013-08-22 23:32:53 +02:00
|
|
|
}
|
|
|
|
|
2013-08-27 23:54:06 +02:00
|
|
|
// Only the files in the last level can be deleted externally.
|
|
|
|
// This is to make sure that any deletion tombstones are not
|
|
|
|
// lost. Check that the level passed is the last level.
|
2014-10-31 16:48:19 +01:00
|
|
|
auto* vstoreage = cfd->current()->storage_info();
|
2014-02-03 21:08:33 +01:00
|
|
|
for (int i = level + 1; i < cfd->NumberLevels(); i++) {
|
2014-10-27 23:49:46 +01:00
|
|
|
if (vstoreage->NumLevelFiles(i) != 0) {
|
2017-03-16 03:22:52 +01:00
|
|
|
ROCKS_LOG_WARN(immutable_db_options_.info_log,
|
|
|
|
"DeleteFile %s FAILED. File not in last level\n",
|
|
|
|
name.c_str());
|
2014-11-15 01:20:24 +01:00
|
|
|
job_context.Clean();
|
2013-08-27 23:54:06 +02:00
|
|
|
return Status::InvalidArgument("File not in last level");
|
|
|
|
}
|
|
|
|
}
|
2014-10-21 20:23:06 +02:00
|
|
|
// if level == 0, it has to be the oldest file
|
2014-10-31 16:48:19 +01:00
|
|
|
if (level == 0 &&
|
|
|
|
vstoreage->LevelFiles(0).back()->fd.GetNumber() != number) {
|
2017-03-16 03:22:52 +01:00
|
|
|
ROCKS_LOG_WARN(immutable_db_options_.info_log,
|
|
|
|
"DeleteFile %s failed ---"
|
|
|
|
" target file in level 0 must be the oldest.",
|
|
|
|
name.c_str());
|
2014-11-15 01:20:24 +01:00
|
|
|
job_context.Clean();
|
2014-10-21 20:23:06 +02:00
|
|
|
return Status::InvalidArgument("File in level 0, but not oldest");
|
|
|
|
}
|
|
|
|
edit.SetColumnFamily(cfd->GetID());
|
2013-08-27 23:54:06 +02:00
|
|
|
edit.DeleteFile(level, number);
|
2014-10-02 01:19:16 +02:00
|
|
|
status = versions_->LogAndApply(cfd, *cfd->GetLatestMutableCFOptions(),
|
2015-01-26 22:59:38 +01:00
|
|
|
&edit, &mutex_, directories_.GetDbDir());
|
2013-12-20 18:57:58 +01:00
|
|
|
if (status.ok()) {
|
2015-06-17 21:37:59 +02:00
|
|
|
InstallSuperVersionAndScheduleWorkWrapper(
|
|
|
|
cfd, &job_context, *cfd->GetLatestMutableCFOptions());
|
2013-12-20 18:57:58 +01:00
|
|
|
}
|
2014-10-28 19:54:33 +01:00
|
|
|
FindObsoleteFiles(&job_context, false);
|
|
|
|
} // lock released here
|
2015-06-04 04:57:01 +02:00
|
|
|
|
2016-09-24 01:34:04 +02:00
|
|
|
LogFlush(immutable_db_options_.info_log);
|
2015-12-29 22:22:13 +01:00
|
|
|
// remove files outside the db-lock
|
|
|
|
if (job_context.HaveSomethingToDelete()) {
|
|
|
|
// Call PurgeObsoleteFiles() without holding mutex.
|
|
|
|
PurgeObsoleteFiles(job_context);
|
|
|
|
}
|
|
|
|
job_context.Clean();
|
|
|
|
return status;
|
|
|
|
}
|
|
|
|
|
|
|
|
Status DBImpl::DeleteFilesInRange(ColumnFamilyHandle* column_family,
|
|
|
|
const Slice* begin, const Slice* end) {
|
|
|
|
Status status;
|
|
|
|
auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
|
|
|
|
ColumnFamilyData* cfd = cfh->cfd();
|
|
|
|
VersionEdit edit;
|
2016-01-07 23:48:45 +01:00
|
|
|
std::vector<FileMetaData*> deleted_files;
|
2015-12-29 22:22:13 +01:00
|
|
|
JobContext job_context(next_job_id_.fetch_add(1), true);
|
|
|
|
{
|
|
|
|
InstrumentedMutexLock l(&mutex_);
|
2016-01-07 23:48:45 +01:00
|
|
|
Version* input_version = cfd->current();
|
2015-12-29 22:22:13 +01:00
|
|
|
|
2016-01-07 23:48:45 +01:00
|
|
|
auto* vstorage = input_version->storage_info();
|
|
|
|
for (int i = 1; i < cfd->NumberLevels(); i++) {
|
2015-12-29 22:22:13 +01:00
|
|
|
if (vstorage->LevelFiles(i).empty() ||
|
|
|
|
!vstorage->OverlapInLevel(i, begin, end)) {
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
std::vector<FileMetaData*> level_files;
|
|
|
|
InternalKey begin_storage, end_storage, *begin_key, *end_key;
|
|
|
|
if (begin == nullptr) {
|
|
|
|
begin_key = nullptr;
|
|
|
|
} else {
|
|
|
|
begin_storage.SetMaxPossibleForUserKey(*begin);
|
|
|
|
begin_key = &begin_storage;
|
|
|
|
}
|
|
|
|
if (end == nullptr) {
|
|
|
|
end_key = nullptr;
|
|
|
|
} else {
|
|
|
|
end_storage.SetMinPossibleForUserKey(*end);
|
|
|
|
end_key = &end_storage;
|
|
|
|
}
|
|
|
|
|
|
|
|
vstorage->GetOverlappingInputs(i, begin_key, end_key, &level_files, -1,
|
|
|
|
nullptr, false);
|
2016-01-07 23:48:45 +01:00
|
|
|
FileMetaData* level_file;
|
|
|
|
for (uint32_t j = 0; j < level_files.size(); j++) {
|
|
|
|
level_file = level_files[j];
|
2015-12-29 22:22:13 +01:00
|
|
|
if (((begin == nullptr) ||
|
|
|
|
(cfd->internal_comparator().user_comparator()->Compare(
|
|
|
|
level_file->smallest.user_key(), *begin) >= 0)) &&
|
|
|
|
((end == nullptr) ||
|
|
|
|
(cfd->internal_comparator().user_comparator()->Compare(
|
|
|
|
level_file->largest.user_key(), *end) <= 0))) {
|
|
|
|
if (level_file->being_compacted) {
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
edit.SetColumnFamily(cfd->GetID());
|
|
|
|
edit.DeleteFile(i, level_file->fd.GetNumber());
|
2016-01-07 23:48:45 +01:00
|
|
|
deleted_files.push_back(level_file);
|
|
|
|
level_file->being_compacted = true;
|
2015-12-29 22:22:13 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (edit.GetDeletedFiles().empty()) {
|
2016-01-04 19:55:31 +01:00
|
|
|
job_context.Clean();
|
2015-12-29 22:22:13 +01:00
|
|
|
return Status::OK();
|
|
|
|
}
|
2016-01-07 23:48:45 +01:00
|
|
|
input_version->Ref();
|
2015-12-29 22:22:13 +01:00
|
|
|
status = versions_->LogAndApply(cfd, *cfd->GetLatestMutableCFOptions(),
|
|
|
|
&edit, &mutex_, directories_.GetDbDir());
|
|
|
|
if (status.ok()) {
|
|
|
|
InstallSuperVersionAndScheduleWorkWrapper(
|
|
|
|
cfd, &job_context, *cfd->GetLatestMutableCFOptions());
|
|
|
|
}
|
2016-01-07 23:48:45 +01:00
|
|
|
for (auto* deleted_file : deleted_files) {
|
|
|
|
deleted_file->being_compacted = false;
|
|
|
|
}
|
|
|
|
input_version->Unref();
|
2015-12-29 22:22:13 +01:00
|
|
|
FindObsoleteFiles(&job_context, false);
|
|
|
|
} // lock released here
|
|
|
|
|
2016-09-24 01:34:04 +02:00
|
|
|
LogFlush(immutable_db_options_.info_log);
|
2013-12-20 18:57:58 +01:00
|
|
|
// remove files outside the db-lock
|
2014-10-28 19:54:33 +01:00
|
|
|
if (job_context.HaveSomethingToDelete()) {
|
2015-06-04 04:57:01 +02:00
|
|
|
// Call PurgeObsoleteFiles() without holding mutex.
|
2014-10-28 19:54:33 +01:00
|
|
|
PurgeObsoleteFiles(job_context);
|
2014-03-10 23:42:14 +01:00
|
|
|
}
|
2014-11-15 00:43:10 +01:00
|
|
|
job_context.Clean();
|
2013-08-22 23:32:53 +02:00
|
|
|
return status;
|
|
|
|
}
|
|
|
|
|
2014-02-11 02:04:44 +01:00
|
|
|
void DBImpl::GetLiveFilesMetaData(std::vector<LiveFileMetaData>* metadata) {
|
2015-02-05 06:39:45 +01:00
|
|
|
InstrumentedMutexLock l(&mutex_);
|
2014-02-11 02:04:44 +01:00
|
|
|
versions_->GetLiveFilesMetaData(metadata);
|
2013-08-22 23:32:53 +02:00
|
|
|
}
|
CompactFiles, EventListener and GetDatabaseMetaData
Summary:
This diff adds three sets of APIs to RocksDB.
= GetColumnFamilyMetaData =
* This APIs allow users to obtain the current state of a RocksDB instance on one column family.
* See GetColumnFamilyMetaData in include/rocksdb/db.h
= EventListener =
* A virtual class that allows users to implement a set of
call-back functions which will be called when specific
events of a RocksDB instance happens.
* To register EventListener, simply insert an EventListener to ColumnFamilyOptions::listeners
= CompactFiles =
* CompactFiles API inputs a set of file numbers and an output level, and RocksDB
will try to compact those files into the specified level.
= Example =
* Example code can be found in example/compact_files_example.cc, which implements
a simple external compactor using EventListener, GetColumnFamilyMetaData, and
CompactFiles API.
Test Plan:
listener_test
compactor_test
example/compact_files_example
export ROCKSDB_TESTS=CompactFiles
db_test
export ROCKSDB_TESTS=MetaData
db_test
Reviewers: ljin, igor, rven, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D24705
2014-11-07 23:45:18 +01:00
|
|
|
|
|
|
|
void DBImpl::GetColumnFamilyMetaData(
|
|
|
|
ColumnFamilyHandle* column_family,
|
|
|
|
ColumnFamilyMetaData* cf_meta) {
|
|
|
|
assert(column_family);
|
|
|
|
auto* cfd = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family)->cfd();
|
|
|
|
auto* sv = GetAndRefSuperVersion(cfd);
|
|
|
|
sv->current->GetColumnFamilyMetaData(cf_meta);
|
|
|
|
ReturnAndCleanupSuperVersion(cfd, sv);
|
|
|
|
}
|
|
|
|
|
2014-04-15 22:39:26 +02:00
|
|
|
#endif // ROCKSDB_LITE
|
2013-08-22 23:32:53 +02:00
|
|
|
|
2014-03-20 22:18:29 +01:00
|
|
|
Status DBImpl::CheckConsistency() {
|
|
|
|
mutex_.AssertHeld();
|
|
|
|
std::vector<LiveFileMetaData> metadata;
|
|
|
|
versions_->GetLiveFilesMetaData(&metadata);
|
|
|
|
|
|
|
|
std::string corruption_messages;
|
|
|
|
for (const auto& md : metadata) {
|
2015-01-17 00:44:55 +01:00
|
|
|
// md.name has a leading "/".
|
|
|
|
std::string file_path = md.db_path + md.name;
|
2014-07-02 18:54:20 +02:00
|
|
|
|
2014-03-20 22:18:29 +01:00
|
|
|
uint64_t fsize = 0;
|
|
|
|
Status s = env_->GetFileSize(file_path, &fsize);
|
2015-10-07 02:46:22 +02:00
|
|
|
if (!s.ok() &&
|
|
|
|
env_->GetFileSize(Rocks2LevelTableFileName(file_path), &fsize).ok()) {
|
|
|
|
s = Status::OK();
|
|
|
|
}
|
2014-03-20 22:18:29 +01:00
|
|
|
if (!s.ok()) {
|
|
|
|
corruption_messages +=
|
|
|
|
"Can't access " + md.name + ": " + s.ToString() + "\n";
|
|
|
|
} else if (fsize != md.size) {
|
2014-07-02 18:54:20 +02:00
|
|
|
corruption_messages += "Sst file size mismatch: " + file_path +
|
2014-03-20 22:18:29 +01:00
|
|
|
". Size recorded in manifest " +
|
2014-11-25 05:44:49 +01:00
|
|
|
ToString(md.size) + ", actual size " +
|
|
|
|
ToString(fsize) + "\n";
|
2014-03-20 22:18:29 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
if (corruption_messages.size() == 0) {
|
|
|
|
return Status::OK();
|
|
|
|
} else {
|
|
|
|
return Status::Corruption(corruption_messages);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-05-21 20:01:48 +02:00
|
|
|
Status DBImpl::GetDbIdentity(std::string& identity) const {
|
2013-12-03 15:39:07 +01:00
|
|
|
std::string idfilename = IdentityFileName(dbname_);
|
|
|
|
const EnvOptions soptions;
|
Move rate_limiter, write buffering, most perf context instrumentation and most random kill out of Env
Summary: We want to keep Env a think layer for better portability. Less platform dependent codes should be moved out of Env. In this patch, I create a wrapper of file readers and writers, and put rate limiting, write buffering, as well as most perf context instrumentation and random kill out of Env. It will make it easier to maintain multiple Env in the future.
Test Plan: Run all existing unit tests.
Reviewers: anthony, kradhakrishnan, IslamAbdelRahman, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D42321
2015-07-18 01:16:11 +02:00
|
|
|
unique_ptr<SequentialFileReader> id_file_reader;
|
|
|
|
Status s;
|
|
|
|
{
|
|
|
|
unique_ptr<SequentialFile> idfile;
|
|
|
|
s = env_->NewSequentialFile(idfilename, &idfile, soptions);
|
|
|
|
if (!s.ok()) {
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
id_file_reader.reset(new SequentialFileReader(std::move(idfile)));
|
2013-12-03 15:39:07 +01:00
|
|
|
}
|
Move rate_limiter, write buffering, most perf context instrumentation and most random kill out of Env
Summary: We want to keep Env a think layer for better portability. Less platform dependent codes should be moved out of Env. In this patch, I create a wrapper of file readers and writers, and put rate limiting, write buffering, as well as most perf context instrumentation and random kill out of Env. It will make it easier to maintain multiple Env in the future.
Test Plan: Run all existing unit tests.
Reviewers: anthony, kradhakrishnan, IslamAbdelRahman, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D42321
2015-07-18 01:16:11 +02:00
|
|
|
|
2013-12-03 15:39:07 +01:00
|
|
|
uint64_t file_size;
|
|
|
|
s = env_->GetFileSize(idfilename, &file_size);
|
|
|
|
if (!s.ok()) {
|
|
|
|
return s;
|
|
|
|
}
|
2015-07-02 01:13:49 +02:00
|
|
|
char* buffer = reinterpret_cast<char*>(alloca(file_size));
|
2013-12-03 15:39:07 +01:00
|
|
|
Slice id;
|
Move rate_limiter, write buffering, most perf context instrumentation and most random kill out of Env
Summary: We want to keep Env a think layer for better portability. Less platform dependent codes should be moved out of Env. In this patch, I create a wrapper of file readers and writers, and put rate limiting, write buffering, as well as most perf context instrumentation and random kill out of Env. It will make it easier to maintain multiple Env in the future.
Test Plan: Run all existing unit tests.
Reviewers: anthony, kradhakrishnan, IslamAbdelRahman, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D42321
2015-07-18 01:16:11 +02:00
|
|
|
s = id_file_reader->Read(static_cast<size_t>(file_size), &id, buffer);
|
2013-12-03 15:39:07 +01:00
|
|
|
if (!s.ok()) {
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
identity.assign(id.ToString());
|
|
|
|
// If last character is '\n' remove it from identity
|
|
|
|
if (identity.size() > 0 && identity.back() == '\n') {
|
|
|
|
identity.pop_back();
|
|
|
|
}
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
2014-01-02 18:08:12 +01:00
|
|
|
// Default implementation -- returns not supported status
|
2014-10-17 01:57:59 +02:00
|
|
|
Status DB::CreateColumnFamily(const ColumnFamilyOptions& cf_options,
|
2014-01-06 22:31:06 +01:00
|
|
|
const std::string& column_family_name,
|
2014-02-11 02:04:44 +01:00
|
|
|
ColumnFamilyHandle** handle) {
|
2014-01-02 18:08:12 +01:00
|
|
|
return Status::NotSupported("");
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 20:14:09 +01:00
|
|
|
}
|
2014-02-11 02:04:44 +01:00
|
|
|
Status DB::DropColumnFamily(ColumnFamilyHandle* column_family) {
|
2014-01-02 18:08:12 +01:00
|
|
|
return Status::NotSupported("");
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 20:14:09 +01:00
|
|
|
}
|
2016-07-14 02:59:25 +02:00
|
|
|
Status DB::DestroyColumnFamilyHandle(ColumnFamilyHandle* column_family) {
|
|
|
|
delete column_family;
|
|
|
|
return Status::OK();
|
|
|
|
}
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 20:14:09 +01:00
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
DB::~DB() { }
|
|
|
|
|
2014-01-02 18:08:12 +01:00
|
|
|
Status DB::ListColumnFamilies(const DBOptions& db_options,
|
|
|
|
const std::string& name,
|
|
|
|
std::vector<std::string>* column_families) {
|
2014-01-22 20:44:53 +01:00
|
|
|
return VersionSet::ListColumnFamilies(column_families, name, db_options.env);
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 20:14:09 +01:00
|
|
|
}
|
|
|
|
|
2011-05-21 04:17:43 +02:00
|
|
|
Snapshot::~Snapshot() {
|
|
|
|
}
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
Status DestroyDB(const std::string& dbname, const Options& options) {
|
2016-09-24 01:34:04 +02:00
|
|
|
const ImmutableDBOptions soptions(SanitizeOptions(dbname, options));
|
2013-10-01 23:46:52 +02:00
|
|
|
Env* env = soptions.env;
|
2011-03-18 23:37:00 +01:00
|
|
|
std::vector<std::string> filenames;
|
2012-12-10 20:02:07 +01:00
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
// Ignore error in case directory does not exist
|
|
|
|
env->GetChildren(dbname, &filenames);
|
2013-10-01 23:46:52 +02:00
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
FileLock* lock;
|
2011-07-15 02:20:57 +02:00
|
|
|
const std::string lockname = LockFileName(dbname);
|
|
|
|
Status result = env->LockFile(lockname, &lock);
|
2011-03-18 23:37:00 +01:00
|
|
|
if (result.ok()) {
|
|
|
|
uint64_t number;
|
|
|
|
FileType type;
|
2016-09-24 01:34:04 +02:00
|
|
|
InfoLogPrefix info_log_prefix(!soptions.db_log_dir.empty(), dbname);
|
2011-04-21 00:48:11 +02:00
|
|
|
for (size_t i = 0; i < filenames.size(); i++) {
|
2014-08-20 19:29:32 +02:00
|
|
|
if (ParseFileName(filenames[i], &number, info_log_prefix.prefix, &type) &&
|
2012-03-09 16:51:04 +01:00
|
|
|
type != kDBLockFile) { // Lock file will be deleted at end
|
2012-12-17 20:26:59 +01:00
|
|
|
Status del;
|
2015-08-20 00:02:17 +02:00
|
|
|
std::string path_to_delete = dbname + "/" + filenames[i];
|
2012-12-17 20:26:59 +01:00
|
|
|
if (type == kMetaDatabase) {
|
2015-08-20 00:02:17 +02:00
|
|
|
del = DestroyDB(path_to_delete, options);
|
|
|
|
} else if (type == kTableFile) {
|
2016-09-24 01:34:04 +02:00
|
|
|
del = DeleteSSTFile(&soptions, path_to_delete, 0);
|
2012-12-17 20:26:59 +01:00
|
|
|
} else {
|
2015-08-20 00:02:17 +02:00
|
|
|
del = env->DeleteFile(path_to_delete);
|
2012-12-17 20:26:59 +01:00
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
if (result.ok() && !del.ok()) {
|
|
|
|
result = del;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2012-12-10 20:02:07 +01:00
|
|
|
|
2015-08-20 00:02:17 +02:00
|
|
|
for (size_t path_id = 0; path_id < options.db_paths.size(); path_id++) {
|
|
|
|
const auto& db_path = options.db_paths[path_id];
|
2014-07-15 00:34:30 +02:00
|
|
|
env->GetChildren(db_path.path, &filenames);
|
2014-07-02 18:54:20 +02:00
|
|
|
for (size_t i = 0; i < filenames.size(); i++) {
|
|
|
|
if (ParseFileName(filenames[i], &number, &type) &&
|
|
|
|
type == kTableFile) { // Lock file will be deleted at end
|
2015-08-20 00:02:17 +02:00
|
|
|
std::string table_path = db_path.path + "/" + filenames[i];
|
2016-09-24 01:34:04 +02:00
|
|
|
Status del = DeleteSSTFile(&soptions, table_path,
|
2016-01-29 03:35:01 +01:00
|
|
|
static_cast<uint32_t>(path_id));
|
2014-07-02 18:54:20 +02:00
|
|
|
if (result.ok() && !del.ok()) {
|
|
|
|
result = del;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-02-06 05:09:42 +01:00
|
|
|
std::vector<std::string> walDirFiles;
|
|
|
|
std::string archivedir = ArchivalDirectory(dbname);
|
|
|
|
if (dbname != soptions.wal_dir) {
|
|
|
|
env->GetChildren(soptions.wal_dir, &walDirFiles);
|
|
|
|
archivedir = ArchivalDirectory(soptions.wal_dir);
|
|
|
|
}
|
|
|
|
|
|
|
|
// Delete log files in the WAL dir
|
|
|
|
for (const auto& file : walDirFiles) {
|
|
|
|
if (ParseFileName(file, &number, &type) && type == kLogFile) {
|
|
|
|
Status del = env->DeleteFile(soptions.wal_dir + "/" + file);
|
|
|
|
if (result.ok() && !del.ok()) {
|
|
|
|
result = del;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
std::vector<std::string> archiveFiles;
|
2013-10-01 23:46:52 +02:00
|
|
|
env->GetChildren(archivedir, &archiveFiles);
|
2012-12-10 20:02:07 +01:00
|
|
|
// Delete archival files.
|
|
|
|
for (size_t i = 0; i < archiveFiles.size(); ++i) {
|
2013-10-11 03:02:10 +02:00
|
|
|
if (ParseFileName(archiveFiles[i], &number, &type) &&
|
|
|
|
type == kLogFile) {
|
2013-10-01 23:46:52 +02:00
|
|
|
Status del = env->DeleteFile(archivedir + "/" + archiveFiles[i]);
|
2012-12-10 20:02:07 +01:00
|
|
|
if (result.ok() && !del.ok()) {
|
|
|
|
result = del;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2015-11-11 07:58:01 +01:00
|
|
|
|
2017-04-08 01:58:07 +02:00
|
|
|
// ignore case where no archival directory is present
|
2013-10-01 23:46:52 +02:00
|
|
|
env->DeleteDir(archivedir);
|
2012-12-10 20:02:07 +01:00
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
env->UnlockFile(lock); // Ignore error since state is already gone
|
2011-07-15 02:20:57 +02:00
|
|
|
env->DeleteFile(lockname);
|
2011-03-18 23:37:00 +01:00
|
|
|
env->DeleteDir(dbname); // Ignore error in case dir contains other files
|
2013-10-01 23:46:52 +02:00
|
|
|
env->DeleteDir(soptions.wal_dir);
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2015-11-11 07:58:01 +01:00
|
|
|
Status DBImpl::WriteOptionsFile() {
|
|
|
|
#ifndef ROCKSDB_LITE
|
2015-12-09 02:01:02 +01:00
|
|
|
mutex_.AssertHeld();
|
2015-11-11 07:58:01 +01:00
|
|
|
|
|
|
|
std::vector<std::string> cf_names;
|
|
|
|
std::vector<ColumnFamilyOptions> cf_opts;
|
2015-12-09 02:01:02 +01:00
|
|
|
|
|
|
|
// This part requires mutex to protect the column family options
|
|
|
|
for (auto cfd : *versions_->GetColumnFamilySet()) {
|
|
|
|
if (cfd->IsDropped()) {
|
|
|
|
continue;
|
2015-11-11 07:58:01 +01:00
|
|
|
}
|
2015-12-09 02:01:02 +01:00
|
|
|
cf_names.push_back(cfd->GetName());
|
2016-09-15 07:10:28 +02:00
|
|
|
cf_opts.push_back(cfd->GetLatestCFOptions());
|
2015-11-11 07:58:01 +01:00
|
|
|
}
|
|
|
|
|
2015-12-09 02:01:02 +01:00
|
|
|
// Unlock during expensive operations. New writes cannot get here
|
|
|
|
// because the single write thread ensures all new writes get queued.
|
2016-09-24 01:34:04 +02:00
|
|
|
DBOptions db_options =
|
|
|
|
BuildDBOptions(immutable_db_options_, mutable_db_options_);
|
2015-12-09 02:01:02 +01:00
|
|
|
mutex_.Unlock();
|
|
|
|
|
|
|
|
std::string file_name =
|
|
|
|
TempOptionsFileName(GetName(), versions_->NewFileNumber());
|
2016-09-24 01:34:04 +02:00
|
|
|
Status s =
|
|
|
|
PersistRocksDBOptions(db_options, cf_names, cf_opts, file_name, GetEnv());
|
2015-12-09 02:01:02 +01:00
|
|
|
|
|
|
|
if (s.ok()) {
|
|
|
|
s = RenameTempFileToOptionsFile(file_name);
|
|
|
|
}
|
|
|
|
mutex_.Lock();
|
2015-11-11 07:58:01 +01:00
|
|
|
return s;
|
|
|
|
#else
|
|
|
|
return Status::OK();
|
|
|
|
#endif // !ROCKSDB_LITE
|
|
|
|
}
|
|
|
|
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
namespace {
|
|
|
|
void DeleteOptionsFilesHelper(const std::map<uint64_t, std::string>& filenames,
|
|
|
|
const size_t num_files_to_keep,
|
|
|
|
const std::shared_ptr<Logger>& info_log,
|
|
|
|
Env* env) {
|
|
|
|
if (filenames.size() <= num_files_to_keep) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
for (auto iter = std::next(filenames.begin(), num_files_to_keep);
|
|
|
|
iter != filenames.end(); ++iter) {
|
|
|
|
if (!env->DeleteFile(iter->second).ok()) {
|
2017-03-16 03:22:52 +01:00
|
|
|
ROCKS_LOG_WARN(info_log, "Unable to delete options file %s",
|
|
|
|
iter->second.c_str());
|
2015-11-11 07:58:01 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
} // namespace
|
|
|
|
#endif // !ROCKSDB_LITE
|
|
|
|
|
|
|
|
Status DBImpl::DeleteObsoleteOptionsFiles() {
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
std::vector<std::string> filenames;
|
|
|
|
// use ordered map to store keep the filenames sorted from the newest
|
|
|
|
// to the oldest.
|
|
|
|
std::map<uint64_t, std::string> options_filenames;
|
|
|
|
Status s;
|
|
|
|
s = GetEnv()->GetChildren(GetName(), &filenames);
|
|
|
|
if (!s.ok()) {
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
for (auto& filename : filenames) {
|
|
|
|
uint64_t file_number;
|
|
|
|
FileType type;
|
|
|
|
if (ParseFileName(filename, &file_number, &type) && type == kOptionsFile) {
|
|
|
|
options_filenames.insert(
|
|
|
|
{std::numeric_limits<uint64_t>::max() - file_number,
|
|
|
|
GetName() + "/" + filename});
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Keeps the latest 2 Options file
|
|
|
|
const size_t kNumOptionsFilesKept = 2;
|
|
|
|
DeleteOptionsFilesHelper(options_filenames, kNumOptionsFilesKept,
|
2016-09-24 01:34:04 +02:00
|
|
|
immutable_db_options_.info_log, GetEnv());
|
2015-11-11 07:58:01 +01:00
|
|
|
return Status::OK();
|
|
|
|
#else
|
|
|
|
return Status::OK();
|
|
|
|
#endif // !ROCKSDB_LITE
|
|
|
|
}
|
|
|
|
|
|
|
|
Status DBImpl::RenameTempFileToOptionsFile(const std::string& file_name) {
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
Status s;
|
2016-06-10 04:03:10 +02:00
|
|
|
|
|
|
|
versions_->options_file_number_ = versions_->NewFileNumber();
|
2015-11-11 07:58:01 +01:00
|
|
|
std::string options_file_name =
|
2016-06-10 04:03:10 +02:00
|
|
|
OptionsFileName(GetName(), versions_->options_file_number_);
|
2015-11-11 07:58:01 +01:00
|
|
|
// Retry if the file name happen to conflict with an existing one.
|
|
|
|
s = GetEnv()->RenameFile(file_name, options_file_name);
|
|
|
|
|
|
|
|
DeleteObsoleteOptionsFiles();
|
|
|
|
return s;
|
|
|
|
#else
|
|
|
|
return Status::OK();
|
|
|
|
#endif // !ROCKSDB_LITE
|
|
|
|
}
|
|
|
|
|
2016-12-14 03:22:00 +01:00
|
|
|
#ifdef ROCKSDB_USING_THREAD_STATUS
|
2014-12-22 21:20:17 +01:00
|
|
|
|
2014-11-20 19:49:32 +01:00
|
|
|
void DBImpl::NewThreadStatusCfInfo(
|
|
|
|
ColumnFamilyData* cfd) const {
|
2016-09-24 01:34:04 +02:00
|
|
|
if (immutable_db_options_.enable_thread_tracking) {
|
2016-01-26 01:26:53 +01:00
|
|
|
ThreadStatusUtil::NewColumnFamilyInfo(this, cfd, cfd->GetName(),
|
|
|
|
cfd->ioptions()->env);
|
2014-11-21 06:13:18 +01:00
|
|
|
}
|
2014-11-20 19:49:32 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
void DBImpl::EraseThreadStatusCfInfo(
|
|
|
|
ColumnFamilyData* cfd) const {
|
2016-09-24 01:34:04 +02:00
|
|
|
if (immutable_db_options_.enable_thread_tracking) {
|
2014-12-22 21:20:17 +01:00
|
|
|
ThreadStatusUtil::EraseColumnFamilyInfo(cfd);
|
2014-11-21 06:13:18 +01:00
|
|
|
}
|
2014-11-20 19:49:32 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
void DBImpl::EraseThreadStatusDbInfo() const {
|
2016-09-24 01:34:04 +02:00
|
|
|
if (immutable_db_options_.enable_thread_tracking) {
|
2014-12-22 21:20:17 +01:00
|
|
|
ThreadStatusUtil::EraseDatabaseInfo(this);
|
2014-11-21 06:13:18 +01:00
|
|
|
}
|
2014-11-20 19:49:32 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
#else
|
|
|
|
void DBImpl::NewThreadStatusCfInfo(
|
|
|
|
ColumnFamilyData* cfd) const {
|
|
|
|
}
|
|
|
|
|
|
|
|
void DBImpl::EraseThreadStatusCfInfo(
|
|
|
|
ColumnFamilyData* cfd) const {
|
|
|
|
}
|
|
|
|
|
|
|
|
void DBImpl::EraseThreadStatusDbInfo() const {
|
|
|
|
}
|
|
|
|
#endif // ROCKSDB_USING_THREAD_STATUS
|
|
|
|
|
2012-08-21 09:33:21 +02:00
|
|
|
//
|
|
|
|
// A global method that can dump out the build version
|
2014-10-07 19:40:57 +02:00
|
|
|
void DumpRocksDBBuildVersion(Logger * log) {
|
2014-04-11 19:19:58 +02:00
|
|
|
#if !defined(IOS_CROSS_COMPILE)
|
2014-10-07 19:40:57 +02:00
|
|
|
// if we compile with Xcode, we don't run build_detect_vesion, so we don't
|
|
|
|
// generate util/build_version.cc
|
2017-03-16 03:22:52 +01:00
|
|
|
ROCKS_LOG_HEADER(log, "RocksDB version: %d.%d.%d\n", ROCKSDB_MAJOR,
|
|
|
|
ROCKSDB_MINOR, ROCKSDB_PATCH);
|
|
|
|
ROCKS_LOG_HEADER(log, "Git sha %s", rocksdb_build_git_sha);
|
|
|
|
ROCKS_LOG_HEADER(log, "Compile date %s", rocksdb_build_compile_date);
|
2014-04-11 19:19:58 +02:00
|
|
|
#endif
|
2012-08-21 09:33:21 +02:00
|
|
|
}
|
|
|
|
|
2015-05-29 23:36:35 +02:00
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
SequenceNumber DBImpl::GetEarliestMemTableSequenceNumber(SuperVersion* sv,
|
|
|
|
bool include_history) {
|
|
|
|
// Find the earliest sequence number that we know we can rely on reading
|
|
|
|
// from the memtable without needing to check sst files.
|
|
|
|
SequenceNumber earliest_seq =
|
|
|
|
sv->imm->GetEarliestSequenceNumber(include_history);
|
|
|
|
if (earliest_seq == kMaxSequenceNumber) {
|
|
|
|
earliest_seq = sv->mem->GetEarliestSequenceNumber();
|
|
|
|
}
|
|
|
|
assert(sv->mem->GetEarliestSequenceNumber() >= earliest_seq);
|
|
|
|
|
|
|
|
return earliest_seq;
|
|
|
|
}
|
|
|
|
#endif // ROCKSDB_LITE
|
|
|
|
|
|
|
|
#ifndef ROCKSDB_LITE
|
Use SST files for Transaction conflict detection
Summary:
Currently, transactions can fail even if there is no actual write conflict. This is due to relying on only the memtables to check for write-conflicts. Users have to tune memtable settings to try to avoid this, but it's hard to figure out exactly how to tune these settings.
With this diff, TransactionDB will use both memtables and SST files to determine if there are any write conflicts. This relies on the fact that BlockBasedTable stores sequence numbers for all writes that happen after any open snapshot. Also, D50295 is needed to prevent SingleDelete from disappearing writes (the TODOs in this test code will be fixed once the other diff is approved and merged).
Note that Optimistic transactions will still rely on tuning memtable settings as we do not want to read from SST while on the write thread. Also, memtable settings can still be used to reduce how often TransactionDB needs to read SST files.
Test Plan: unit tests, db bench
Reviewers: rven, yhchiang, kradhakrishnan, IslamAbdelRahman, sdong
Reviewed By: sdong
Subscribers: dhruba, leveldb, yoshinorim
Differential Revision: https://reviews.facebook.net/D50475
2015-10-16 01:37:15 +02:00
|
|
|
Status DBImpl::GetLatestSequenceForKey(SuperVersion* sv, const Slice& key,
|
|
|
|
bool cache_only, SequenceNumber* seq,
|
|
|
|
bool* found_record_for_key) {
|
2015-05-29 23:36:35 +02:00
|
|
|
Status s;
|
|
|
|
MergeContext merge_context;
|
2016-11-19 01:54:09 +01:00
|
|
|
RangeDelAggregator range_del_agg(sv->mem->GetInternalKeyComparator(),
|
|
|
|
kMaxSequenceNumber);
|
2015-05-29 23:36:35 +02:00
|
|
|
|
2016-11-04 02:40:23 +01:00
|
|
|
ReadOptions read_options;
|
2015-05-29 23:36:35 +02:00
|
|
|
SequenceNumber current_seq = versions_->LastSequence();
|
|
|
|
LookupKey lkey(key, current_seq);
|
|
|
|
|
|
|
|
*seq = kMaxSequenceNumber;
|
Use SST files for Transaction conflict detection
Summary:
Currently, transactions can fail even if there is no actual write conflict. This is due to relying on only the memtables to check for write-conflicts. Users have to tune memtable settings to try to avoid this, but it's hard to figure out exactly how to tune these settings.
With this diff, TransactionDB will use both memtables and SST files to determine if there are any write conflicts. This relies on the fact that BlockBasedTable stores sequence numbers for all writes that happen after any open snapshot. Also, D50295 is needed to prevent SingleDelete from disappearing writes (the TODOs in this test code will be fixed once the other diff is approved and merged).
Note that Optimistic transactions will still rely on tuning memtable settings as we do not want to read from SST while on the write thread. Also, memtable settings can still be used to reduce how often TransactionDB needs to read SST files.
Test Plan: unit tests, db bench
Reviewers: rven, yhchiang, kradhakrishnan, IslamAbdelRahman, sdong
Reviewed By: sdong
Subscribers: dhruba, leveldb, yoshinorim
Differential Revision: https://reviews.facebook.net/D50475
2015-10-16 01:37:15 +02:00
|
|
|
*found_record_for_key = false;
|
|
|
|
|
2015-05-29 23:36:35 +02:00
|
|
|
// Check if there is a record for this key in the latest memtable
|
2017-01-08 23:08:51 +01:00
|
|
|
sv->mem->Get(lkey, nullptr, &s, &merge_context, &range_del_agg, seq,
|
2016-11-04 02:40:23 +01:00
|
|
|
read_options);
|
2015-05-29 23:36:35 +02:00
|
|
|
|
|
|
|
if (!(s.ok() || s.IsNotFound() || s.IsMergeInProgress())) {
|
|
|
|
// unexpected error reading memtable.
|
2017-03-16 03:22:52 +01:00
|
|
|
ROCKS_LOG_ERROR(immutable_db_options_.info_log,
|
|
|
|
"Unexpected status returned from MemTable::Get: %s\n",
|
|
|
|
s.ToString().c_str());
|
2015-05-29 23:36:35 +02:00
|
|
|
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (*seq != kMaxSequenceNumber) {
|
|
|
|
// Found a sequence number, no need to check immutable memtables
|
Use SST files for Transaction conflict detection
Summary:
Currently, transactions can fail even if there is no actual write conflict. This is due to relying on only the memtables to check for write-conflicts. Users have to tune memtable settings to try to avoid this, but it's hard to figure out exactly how to tune these settings.
With this diff, TransactionDB will use both memtables and SST files to determine if there are any write conflicts. This relies on the fact that BlockBasedTable stores sequence numbers for all writes that happen after any open snapshot. Also, D50295 is needed to prevent SingleDelete from disappearing writes (the TODOs in this test code will be fixed once the other diff is approved and merged).
Note that Optimistic transactions will still rely on tuning memtable settings as we do not want to read from SST while on the write thread. Also, memtable settings can still be used to reduce how often TransactionDB needs to read SST files.
Test Plan: unit tests, db bench
Reviewers: rven, yhchiang, kradhakrishnan, IslamAbdelRahman, sdong
Reviewed By: sdong
Subscribers: dhruba, leveldb, yoshinorim
Differential Revision: https://reviews.facebook.net/D50475
2015-10-16 01:37:15 +02:00
|
|
|
*found_record_for_key = true;
|
2015-05-29 23:36:35 +02:00
|
|
|
return Status::OK();
|
|
|
|
}
|
|
|
|
|
|
|
|
// Check if there is a record for this key in the immutable memtables
|
2017-01-08 23:08:51 +01:00
|
|
|
sv->imm->Get(lkey, nullptr, &s, &merge_context, &range_del_agg, seq,
|
2016-11-04 02:40:23 +01:00
|
|
|
read_options);
|
2015-05-29 23:36:35 +02:00
|
|
|
|
|
|
|
if (!(s.ok() || s.IsNotFound() || s.IsMergeInProgress())) {
|
|
|
|
// unexpected error reading memtable.
|
2017-03-16 03:22:52 +01:00
|
|
|
ROCKS_LOG_ERROR(immutable_db_options_.info_log,
|
|
|
|
"Unexpected status returned from MemTableList::Get: %s\n",
|
|
|
|
s.ToString().c_str());
|
2015-05-29 23:36:35 +02:00
|
|
|
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (*seq != kMaxSequenceNumber) {
|
|
|
|
// Found a sequence number, no need to check memtable history
|
Use SST files for Transaction conflict detection
Summary:
Currently, transactions can fail even if there is no actual write conflict. This is due to relying on only the memtables to check for write-conflicts. Users have to tune memtable settings to try to avoid this, but it's hard to figure out exactly how to tune these settings.
With this diff, TransactionDB will use both memtables and SST files to determine if there are any write conflicts. This relies on the fact that BlockBasedTable stores sequence numbers for all writes that happen after any open snapshot. Also, D50295 is needed to prevent SingleDelete from disappearing writes (the TODOs in this test code will be fixed once the other diff is approved and merged).
Note that Optimistic transactions will still rely on tuning memtable settings as we do not want to read from SST while on the write thread. Also, memtable settings can still be used to reduce how often TransactionDB needs to read SST files.
Test Plan: unit tests, db bench
Reviewers: rven, yhchiang, kradhakrishnan, IslamAbdelRahman, sdong
Reviewed By: sdong
Subscribers: dhruba, leveldb, yoshinorim
Differential Revision: https://reviews.facebook.net/D50475
2015-10-16 01:37:15 +02:00
|
|
|
*found_record_for_key = true;
|
2015-05-29 23:36:35 +02:00
|
|
|
return Status::OK();
|
|
|
|
}
|
|
|
|
|
|
|
|
// Check if there is a record for this key in the immutable memtables
|
2016-11-04 02:40:23 +01:00
|
|
|
sv->imm->GetFromHistory(lkey, nullptr, &s, &merge_context, &range_del_agg,
|
|
|
|
seq, read_options);
|
2015-05-29 23:36:35 +02:00
|
|
|
|
|
|
|
if (!(s.ok() || s.IsNotFound() || s.IsMergeInProgress())) {
|
|
|
|
// unexpected error reading memtable.
|
2017-03-16 03:22:52 +01:00
|
|
|
ROCKS_LOG_ERROR(
|
|
|
|
immutable_db_options_.info_log,
|
2015-05-29 23:36:35 +02:00
|
|
|
"Unexpected status returned from MemTableList::GetFromHistory: %s\n",
|
|
|
|
s.ToString().c_str());
|
|
|
|
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
Use SST files for Transaction conflict detection
Summary:
Currently, transactions can fail even if there is no actual write conflict. This is due to relying on only the memtables to check for write-conflicts. Users have to tune memtable settings to try to avoid this, but it's hard to figure out exactly how to tune these settings.
With this diff, TransactionDB will use both memtables and SST files to determine if there are any write conflicts. This relies on the fact that BlockBasedTable stores sequence numbers for all writes that happen after any open snapshot. Also, D50295 is needed to prevent SingleDelete from disappearing writes (the TODOs in this test code will be fixed once the other diff is approved and merged).
Note that Optimistic transactions will still rely on tuning memtable settings as we do not want to read from SST while on the write thread. Also, memtable settings can still be used to reduce how often TransactionDB needs to read SST files.
Test Plan: unit tests, db bench
Reviewers: rven, yhchiang, kradhakrishnan, IslamAbdelRahman, sdong
Reviewed By: sdong
Subscribers: dhruba, leveldb, yoshinorim
Differential Revision: https://reviews.facebook.net/D50475
2015-10-16 01:37:15 +02:00
|
|
|
if (*seq != kMaxSequenceNumber) {
|
|
|
|
// Found a sequence number, no need to check SST files
|
|
|
|
*found_record_for_key = true;
|
|
|
|
return Status::OK();
|
|
|
|
}
|
|
|
|
|
|
|
|
// TODO(agiardullo): possible optimization: consider checking cached
|
|
|
|
// SST files if cache_only=true?
|
|
|
|
if (!cache_only) {
|
|
|
|
// Check tables
|
2015-12-22 01:57:04 +01:00
|
|
|
sv->current->Get(read_options, lkey, nullptr, &s, &merge_context,
|
2016-11-04 02:40:23 +01:00
|
|
|
&range_del_agg, nullptr /* value_found */,
|
|
|
|
found_record_for_key, seq);
|
Use SST files for Transaction conflict detection
Summary:
Currently, transactions can fail even if there is no actual write conflict. This is due to relying on only the memtables to check for write-conflicts. Users have to tune memtable settings to try to avoid this, but it's hard to figure out exactly how to tune these settings.
With this diff, TransactionDB will use both memtables and SST files to determine if there are any write conflicts. This relies on the fact that BlockBasedTable stores sequence numbers for all writes that happen after any open snapshot. Also, D50295 is needed to prevent SingleDelete from disappearing writes (the TODOs in this test code will be fixed once the other diff is approved and merged).
Note that Optimistic transactions will still rely on tuning memtable settings as we do not want to read from SST while on the write thread. Also, memtable settings can still be used to reduce how often TransactionDB needs to read SST files.
Test Plan: unit tests, db bench
Reviewers: rven, yhchiang, kradhakrishnan, IslamAbdelRahman, sdong
Reviewed By: sdong
Subscribers: dhruba, leveldb, yoshinorim
Differential Revision: https://reviews.facebook.net/D50475
2015-10-16 01:37:15 +02:00
|
|
|
|
|
|
|
if (!(s.ok() || s.IsNotFound() || s.IsMergeInProgress())) {
|
|
|
|
// unexpected error reading SST files
|
2017-03-16 03:22:52 +01:00
|
|
|
ROCKS_LOG_ERROR(immutable_db_options_.info_log,
|
|
|
|
"Unexpected status returned from Version::Get: %s\n",
|
|
|
|
s.ToString().c_str());
|
Use SST files for Transaction conflict detection
Summary:
Currently, transactions can fail even if there is no actual write conflict. This is due to relying on only the memtables to check for write-conflicts. Users have to tune memtable settings to try to avoid this, but it's hard to figure out exactly how to tune these settings.
With this diff, TransactionDB will use both memtables and SST files to determine if there are any write conflicts. This relies on the fact that BlockBasedTable stores sequence numbers for all writes that happen after any open snapshot. Also, D50295 is needed to prevent SingleDelete from disappearing writes (the TODOs in this test code will be fixed once the other diff is approved and merged).
Note that Optimistic transactions will still rely on tuning memtable settings as we do not want to read from SST while on the write thread. Also, memtable settings can still be used to reduce how often TransactionDB needs to read SST files.
Test Plan: unit tests, db bench
Reviewers: rven, yhchiang, kradhakrishnan, IslamAbdelRahman, sdong
Reviewed By: sdong
Subscribers: dhruba, leveldb, yoshinorim
Differential Revision: https://reviews.facebook.net/D50475
2015-10-16 01:37:15 +02:00
|
|
|
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-05-29 23:36:35 +02:00
|
|
|
return Status::OK();
|
|
|
|
}
|
2016-10-21 02:05:32 +02:00
|
|
|
|
|
|
|
Status DBImpl::IngestExternalFile(
|
|
|
|
ColumnFamilyHandle* column_family,
|
|
|
|
const std::vector<std::string>& external_files,
|
|
|
|
const IngestExternalFileOptions& ingestion_options) {
|
|
|
|
Status status;
|
|
|
|
auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
|
|
|
|
auto cfd = cfh->cfd();
|
|
|
|
|
|
|
|
ExternalSstFileIngestionJob ingestion_job(env_, versions_.get(), cfd,
|
|
|
|
immutable_db_options_, env_options_,
|
|
|
|
&snapshots_, ingestion_options);
|
|
|
|
|
|
|
|
std::list<uint64_t>::iterator pending_output_elem;
|
|
|
|
{
|
|
|
|
InstrumentedMutexLock l(&mutex_);
|
2017-02-17 22:36:46 +01:00
|
|
|
if (!bg_error_.ok()) {
|
|
|
|
// Don't ingest files when there is a bg_error
|
|
|
|
return bg_error_;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Make sure that bg cleanup wont delete the files that we are ingesting
|
2016-10-21 02:05:32 +02:00
|
|
|
pending_output_elem = CaptureCurrentFileNumberInPendingOutputs();
|
|
|
|
}
|
|
|
|
|
|
|
|
status = ingestion_job.Prepare(external_files);
|
|
|
|
if (!status.ok()) {
|
|
|
|
return status;
|
|
|
|
}
|
|
|
|
|
2016-11-09 19:37:51 +01:00
|
|
|
TEST_SYNC_POINT("DBImpl::AddFile:Start");
|
2016-10-21 02:05:32 +02:00
|
|
|
{
|
|
|
|
// Lock db mutex
|
|
|
|
InstrumentedMutexLock l(&mutex_);
|
|
|
|
TEST_SYNC_POINT("DBImpl::AddFile:MutexLock");
|
|
|
|
|
|
|
|
// Stop writes to the DB
|
|
|
|
WriteThread::Writer w;
|
|
|
|
write_thread_.EnterUnbatched(&w, &mutex_);
|
|
|
|
|
2016-11-09 19:37:51 +01:00
|
|
|
num_running_ingest_file_++;
|
|
|
|
|
2016-12-13 09:47:52 +01:00
|
|
|
// We cannot ingest a file into a dropped CF
|
|
|
|
if (cfd->IsDropped()) {
|
|
|
|
status = Status::InvalidArgument(
|
|
|
|
"Cannot ingest an external file into a dropped CF");
|
|
|
|
}
|
|
|
|
|
2016-10-21 02:05:32 +02:00
|
|
|
// Figure out if we need to flush the memtable first
|
2016-12-13 09:47:52 +01:00
|
|
|
if (status.ok()) {
|
|
|
|
bool need_flush = false;
|
|
|
|
status = ingestion_job.NeedsFlush(&need_flush);
|
|
|
|
|
|
|
|
if (status.ok() && need_flush) {
|
|
|
|
mutex_.Unlock();
|
|
|
|
status = FlushMemTable(cfd, FlushOptions(), true /* writes_stopped */);
|
|
|
|
mutex_.Lock();
|
|
|
|
}
|
2016-10-21 02:05:32 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// Run the ingestion job
|
|
|
|
if (status.ok()) {
|
|
|
|
status = ingestion_job.Run();
|
|
|
|
}
|
|
|
|
|
|
|
|
// Install job edit [Mutex will be unlocked here]
|
|
|
|
auto mutable_cf_options = cfd->GetLatestMutableCFOptions();
|
|
|
|
if (status.ok()) {
|
|
|
|
status =
|
|
|
|
versions_->LogAndApply(cfd, *mutable_cf_options, ingestion_job.edit(),
|
|
|
|
&mutex_, directories_.GetDbDir());
|
|
|
|
}
|
|
|
|
if (status.ok()) {
|
|
|
|
delete InstallSuperVersionAndScheduleWork(cfd, nullptr,
|
|
|
|
*mutable_cf_options);
|
|
|
|
}
|
|
|
|
|
|
|
|
// Resume writes to the DB
|
|
|
|
write_thread_.ExitUnbatched(&w);
|
|
|
|
|
|
|
|
// Update stats
|
|
|
|
if (status.ok()) {
|
|
|
|
ingestion_job.UpdateStats();
|
|
|
|
}
|
|
|
|
|
|
|
|
ReleaseFileNumberFromPendingOutputs(pending_output_elem);
|
|
|
|
|
|
|
|
num_running_ingest_file_--;
|
|
|
|
if (num_running_ingest_file_ == 0) {
|
|
|
|
bg_cv_.SignalAll();
|
|
|
|
}
|
|
|
|
|
|
|
|
TEST_SYNC_POINT("DBImpl::AddFile:MutexUnlock");
|
|
|
|
}
|
|
|
|
// mutex_ is unlocked here
|
|
|
|
|
|
|
|
// Cleanup
|
|
|
|
ingestion_job.Cleanup(status);
|
|
|
|
|
2016-12-06 22:56:17 +01:00
|
|
|
if (status.ok()) {
|
|
|
|
NotifyOnExternalFileIngested(cfd, ingestion_job);
|
|
|
|
}
|
|
|
|
|
2016-10-21 02:05:32 +02:00
|
|
|
return status;
|
|
|
|
}
|
|
|
|
|
2016-12-06 22:56:17 +01:00
|
|
|
void DBImpl::NotifyOnExternalFileIngested(
|
|
|
|
ColumnFamilyData* cfd, const ExternalSstFileIngestionJob& ingestion_job) {
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
if (immutable_db_options_.listeners.empty()) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
for (const IngestedFileInfo& f : ingestion_job.files_to_ingest()) {
|
|
|
|
ExternalFileIngestionInfo info;
|
|
|
|
info.cf_name = cfd->GetName();
|
|
|
|
info.external_file_path = f.external_file_path;
|
|
|
|
info.internal_file_path = f.internal_file_path;
|
|
|
|
info.global_seqno = f.assigned_seqno;
|
|
|
|
info.table_properties = f.table_properties;
|
|
|
|
for (auto listener : immutable_db_options_.listeners) {
|
|
|
|
listener->OnExternalFileIngested(this, info);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
2016-10-21 02:05:32 +02:00
|
|
|
void DBImpl::WaitForIngestFile() {
|
|
|
|
mutex_.AssertHeld();
|
|
|
|
while (num_running_ingest_file_ > 0) {
|
|
|
|
bg_cv_.Wait();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-05-29 23:36:35 +02:00
|
|
|
#endif // ROCKSDB_LITE
|
|
|
|
|
2013-10-04 06:49:15 +02:00
|
|
|
} // namespace rocksdb
|