2013-10-16 23:59:46 +02:00
|
|
|
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
|
|
|
// This source code is licensed under the BSD-style license found in the
|
|
|
|
// LICENSE file in the root directory of this source tree. An additional grant
|
|
|
|
// of patent rights can be found in the PATENTS file in the same directory.
|
|
|
|
//
|
2011-03-18 23:37:00 +01:00
|
|
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
|
|
|
// Use of this source code is governed by a BSD-style license that can be
|
|
|
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
|
|
|
|
|
|
|
#include "db/db_impl.h"
|
|
|
|
|
|
|
|
#include <algorithm>
|
2012-11-26 22:56:45 +01:00
|
|
|
#include <climits>
|
|
|
|
#include <cstdio>
|
2011-03-18 23:37:00 +01:00
|
|
|
#include <set>
|
2013-06-21 01:58:59 +02:00
|
|
|
#include <stdexcept>
|
2013-10-16 20:50:50 +02:00
|
|
|
#include <stdint.h>
|
|
|
|
#include <string>
|
[RocksDB] [Performance] Speed up FindObsoleteFiles
Summary:
FindObsoleteFiles was slow, holding the single big lock, resulted in bad p99 behavior.
Didn't profile anything, but several things could be improved:
1. VersionSet::AddLiveFiles works with std::set, which is by itself slow (a tree).
You also don't know how many dynamic allocations occur just for building up this tree.
switched to std::vector, also added logic to pre-calculate total size and do just one allocation
2. Don't see why env_->GetChildren() needs to be mutex proteced, moved to PurgeObsoleteFiles where
mutex could be unlocked.
3. switched std::set to std:unordered_set, the conversion from vector is also inside PurgeObsoleteFiles
I have a feeling this should pretty much fix it.
Test Plan: make check; db_stress
Reviewers: dhruba, heyongqiang, MarkCallaghan
Reviewed By: dhruba
CC: leveldb, zshao
Differential Revision: https://reviews.facebook.net/D10197
2013-04-12 01:49:53 +02:00
|
|
|
#include <unordered_set>
|
2014-01-28 20:05:04 +01:00
|
|
|
#include <unordered_map>
|
2014-01-17 06:56:26 +01:00
|
|
|
#include <utility>
|
2013-10-16 20:50:50 +02:00
|
|
|
#include <vector>
|
2012-11-26 22:56:45 +01:00
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
#include "db/builder.h"
|
|
|
|
#include "db/dbformat.h"
|
2013-10-16 20:50:50 +02:00
|
|
|
#include "db/db_iter.h"
|
2011-03-18 23:37:00 +01:00
|
|
|
#include "db/filename.h"
|
|
|
|
#include "db/log_reader.h"
|
|
|
|
#include "db/log_writer.h"
|
|
|
|
#include "db/memtable.h"
|
2012-10-19 23:00:53 +02:00
|
|
|
#include "db/memtablelist.h"
|
2013-12-03 03:34:05 +01:00
|
|
|
#include "db/merge_context.h"
|
2013-03-21 23:59:47 +01:00
|
|
|
#include "db/merge_helper.h"
|
2013-08-13 23:04:56 +02:00
|
|
|
#include "db/prefix_filter_iterator.h"
|
2011-03-18 23:37:00 +01:00
|
|
|
#include "db/table_cache.h"
|
2013-11-20 01:29:42 +01:00
|
|
|
#include "db/table_properties_collector.h"
|
2014-01-17 06:56:26 +01:00
|
|
|
#include "db/tailing_iter.h"
|
2013-10-16 20:50:50 +02:00
|
|
|
#include "db/transaction_log_impl.h"
|
2011-03-18 23:37:00 +01:00
|
|
|
#include "db/version_set.h"
|
|
|
|
#include "db/write_batch_internal.h"
|
2013-10-16 20:50:50 +02:00
|
|
|
#include "port/port.h"
|
2013-08-23 17:38:13 +02:00
|
|
|
#include "rocksdb/compaction_filter.h"
|
|
|
|
#include "rocksdb/db.h"
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 20:14:09 +01:00
|
|
|
#include "rocksdb/column_family.h"
|
2013-08-23 17:38:13 +02:00
|
|
|
#include "rocksdb/env.h"
|
|
|
|
#include "rocksdb/merge_operator.h"
|
|
|
|
#include "rocksdb/statistics.h"
|
|
|
|
#include "rocksdb/status.h"
|
2013-10-29 01:54:09 +01:00
|
|
|
#include "rocksdb/table.h"
|
|
|
|
#include "port/port.h"
|
2011-03-18 23:37:00 +01:00
|
|
|
#include "table/block.h"
|
2013-11-20 07:00:48 +01:00
|
|
|
#include "table/block_based_table_factory.h"
|
2011-03-18 23:37:00 +01:00
|
|
|
#include "table/merger.h"
|
|
|
|
#include "table/two_level_iterator.h"
|
2013-02-15 20:53:17 +01:00
|
|
|
#include "util/auto_roll_logger.h"
|
|
|
|
#include "util/build_version.h"
|
2011-03-18 23:37:00 +01:00
|
|
|
#include "util/coding.h"
|
2013-12-03 21:42:15 +01:00
|
|
|
#include "util/hash_skiplist_rep.h"
|
2011-03-18 23:37:00 +01:00
|
|
|
#include "util/logging.h"
|
|
|
|
#include "util/mutexlock.h"
|
2013-10-23 08:26:51 +02:00
|
|
|
#include "util/perf_context_imp.h"
|
2013-02-15 20:53:17 +01:00
|
|
|
#include "util/stop_watch.h"
|
2014-01-14 23:49:31 +01:00
|
|
|
#include "util/autovector.h"
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2013-10-04 06:49:15 +02:00
|
|
|
namespace rocksdb {
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2014-01-06 22:31:06 +01:00
|
|
|
const std::string default_column_family_name("default");
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 20:14:09 +01:00
|
|
|
|
2012-09-18 22:18:32 +02:00
|
|
|
void dumpLeveldbBuildVersion(Logger * log);
|
|
|
|
|
2012-03-09 01:23:21 +01:00
|
|
|
// Information kept for every waiting writer
|
|
|
|
struct DBImpl::Writer {
|
|
|
|
Status status;
|
|
|
|
WriteBatch* batch;
|
|
|
|
bool sync;
|
2012-07-05 22:39:28 +02:00
|
|
|
bool disableWAL;
|
2012-03-09 01:23:21 +01:00
|
|
|
bool done;
|
|
|
|
port::CondVar cv;
|
|
|
|
|
|
|
|
explicit Writer(port::Mutex* mu) : cv(mu) { }
|
|
|
|
};
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
struct DBImpl::CompactionState {
|
|
|
|
Compaction* const compaction;
|
|
|
|
|
2012-11-27 06:16:21 +01:00
|
|
|
// If there were two snapshots with seq numbers s1 and
|
|
|
|
// s2 and s1 < s2, and if we find two instances of a key k1 then lies
|
|
|
|
// entirely within s1 and s2, then the earlier version of k1 can be safely
|
|
|
|
// deleted because that version is not visible in any snapshot.
|
|
|
|
std::vector<SequenceNumber> existing_snapshots;
|
2011-03-18 23:37:00 +01:00
|
|
|
|
|
|
|
// Files produced by compaction
|
|
|
|
struct Output {
|
|
|
|
uint64_t number;
|
|
|
|
uint64_t file_size;
|
|
|
|
InternalKey smallest, largest;
|
2013-06-14 07:09:08 +02:00
|
|
|
SequenceNumber smallest_seqno, largest_seqno;
|
2011-03-18 23:37:00 +01:00
|
|
|
};
|
|
|
|
std::vector<Output> outputs;
|
2012-10-19 23:00:53 +02:00
|
|
|
std::list<uint64_t> allocated_file_numbers;
|
2011-03-18 23:37:00 +01:00
|
|
|
|
|
|
|
// State kept for output being generated
|
2013-01-20 11:07:13 +01:00
|
|
|
unique_ptr<WritableFile> outfile;
|
|
|
|
unique_ptr<TableBuilder> builder;
|
2011-03-18 23:37:00 +01:00
|
|
|
|
|
|
|
uint64_t total_bytes;
|
|
|
|
|
|
|
|
Output* current_output() { return &outputs[outputs.size()-1]; }
|
|
|
|
|
|
|
|
explicit CompactionState(Compaction* c)
|
|
|
|
: compaction(c),
|
|
|
|
total_bytes(0) {
|
|
|
|
}
|
2013-10-27 07:01:26 +01:00
|
|
|
|
|
|
|
// Create a client visible context of this compaction
|
|
|
|
CompactionFilter::Context GetFilterContext() {
|
|
|
|
CompactionFilter::Context context;
|
|
|
|
context.is_full_compaction = compaction->IsFullCompaction();
|
|
|
|
return context;
|
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
};
|
|
|
|
|
|
|
|
// Fix user-supplied options to be reasonable
|
2013-06-05 20:22:38 +02:00
|
|
|
template <class T, class V>
|
2011-03-18 23:37:00 +01:00
|
|
|
static void ClipToRange(T* ptr, V minvalue, V maxvalue) {
|
2011-04-21 00:48:11 +02:00
|
|
|
if (static_cast<V>(*ptr) > maxvalue) *ptr = maxvalue;
|
|
|
|
if (static_cast<V>(*ptr) < minvalue) *ptr = minvalue;
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
Options SanitizeOptions(const std::string& dbname,
|
|
|
|
const InternalKeyComparator* icmp,
|
2012-04-17 17:36:46 +02:00
|
|
|
const InternalFilterPolicy* ipolicy,
|
2011-03-18 23:37:00 +01:00
|
|
|
const Options& src) {
|
|
|
|
Options result = src;
|
|
|
|
result.comparator = icmp;
|
2013-02-15 20:53:17 +01:00
|
|
|
result.filter_policy = (src.filter_policy != nullptr) ? ipolicy : nullptr;
|
2013-02-26 09:13:17 +01:00
|
|
|
ClipToRange(&result.max_open_files, 20, 1000000);
|
2013-05-23 20:56:19 +02:00
|
|
|
ClipToRange(&result.write_buffer_size, ((size_t)64)<<10,
|
|
|
|
((size_t)64)<<30);
|
2012-04-17 17:36:46 +02:00
|
|
|
ClipToRange(&result.block_size, 1<<10, 4<<20);
|
2013-06-11 23:23:58 +02:00
|
|
|
|
Make arena block size configurable
Summary:
Add an option for arena block size, default value 4096 bytes. Arena will allocate blocks with such size.
I am not sure about passing parameter to skiplist in the new virtualized framework, though I talked to Jim a bit. So add Jim as reviewer.
Test Plan:
new unit test, I am running db_test.
For passing paramter from configured option to Arena, I tried tests like:
TEST(DBTest, Arena_Option) {
std::string dbname = test::TmpDir() + "/db_arena_option_test";
DestroyDB(dbname, Options());
DB* db = nullptr;
Options opts;
opts.create_if_missing = true;
opts.arena_block_size = 1000000; // tested 99, 999999
Status s = DB::Open(opts, dbname, &db);
db->Put(WriteOptions(), "a", "123");
}
and printed some debug info. The results look good. Any suggestion for such a unit-test?
Reviewers: haobo, dhruba, emayanke, jpaton
Reviewed By: dhruba
CC: leveldb, zshao
Differential Revision: https://reviews.facebook.net/D11799
2013-07-31 21:42:23 +02:00
|
|
|
// if user sets arena_block_size, we trust user to use this value. Otherwise,
|
|
|
|
// calculate a proper value from writer_buffer_size;
|
|
|
|
if (result.arena_block_size <= 0) {
|
|
|
|
result.arena_block_size = result.write_buffer_size / 10;
|
|
|
|
}
|
|
|
|
|
2013-06-11 23:23:58 +02:00
|
|
|
result.min_write_buffer_number_to_merge = std::min(
|
|
|
|
result.min_write_buffer_number_to_merge, result.max_write_buffer_number-1);
|
2013-02-15 20:53:17 +01:00
|
|
|
if (result.info_log == nullptr) {
|
2013-02-05 04:42:40 +01:00
|
|
|
Status s = CreateLoggerFromOptions(dbname, result.db_log_dir, src.env,
|
|
|
|
result, &result.info_log);
|
2011-03-18 23:37:00 +01:00
|
|
|
if (!s.ok()) {
|
|
|
|
// No place suitable for logging
|
2013-02-15 20:53:17 +01:00
|
|
|
result.info_log = nullptr;
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
}
|
2013-02-15 20:53:17 +01:00
|
|
|
if (result.block_cache == nullptr && !result.no_block_cache) {
|
2011-04-12 21:38:58 +02:00
|
|
|
result.block_cache = NewLRUCache(8 << 20);
|
|
|
|
}
|
2013-01-24 19:54:26 +01:00
|
|
|
result.compression_per_level = src.compression_per_level;
|
2013-05-15 19:34:02 +02:00
|
|
|
if (result.block_size_deviation < 0 || result.block_size_deviation > 100) {
|
|
|
|
result.block_size_deviation = 0;
|
|
|
|
}
|
2013-06-09 05:15:10 +02:00
|
|
|
if (result.max_mem_compaction_level >= result.num_levels) {
|
|
|
|
result.max_mem_compaction_level = result.num_levels - 1;
|
|
|
|
}
|
2013-08-06 00:43:49 +02:00
|
|
|
if (result.soft_rate_limit > result.hard_rate_limit) {
|
|
|
|
result.soft_rate_limit = result.hard_rate_limit;
|
|
|
|
}
|
2013-10-27 07:01:26 +01:00
|
|
|
if (result.compaction_filter) {
|
|
|
|
Log(result.info_log, "Compaction filter specified, ignore factory");
|
2013-08-13 19:56:20 +02:00
|
|
|
}
|
2013-08-23 08:10:02 +02:00
|
|
|
if (result.prefix_extractor) {
|
2013-12-03 21:42:15 +01:00
|
|
|
// If a prefix extractor has been supplied and a HashSkipListRepFactory is
|
2013-08-23 08:10:02 +02:00
|
|
|
// being used, make sure that the latter uses the former as its transform
|
|
|
|
// function.
|
2013-12-03 21:42:15 +01:00
|
|
|
auto factory = dynamic_cast<HashSkipListRepFactory*>(
|
2013-08-23 08:10:02 +02:00
|
|
|
result.memtable_factory.get());
|
2013-09-13 23:38:37 +02:00
|
|
|
if (factory &&
|
2013-08-29 02:45:38 +02:00
|
|
|
factory->GetTransform() != result.prefix_extractor) {
|
2013-08-23 08:10:02 +02:00
|
|
|
Log(result.info_log, "A prefix hash representation factory was supplied "
|
|
|
|
"whose prefix extractor does not match options.prefix_extractor. "
|
|
|
|
"Falling back to skip list representation factory");
|
|
|
|
result.memtable_factory = std::make_shared<SkipListFactory>();
|
2013-09-13 23:38:37 +02:00
|
|
|
} else if (factory) {
|
|
|
|
Log(result.info_log, "Prefix hash memtable rep is in use.");
|
2013-08-23 08:10:02 +02:00
|
|
|
}
|
|
|
|
}
|
2013-10-01 23:46:52 +02:00
|
|
|
|
|
|
|
if (result.wal_dir.empty()) {
|
|
|
|
// Use dbname as default
|
|
|
|
result.wal_dir = dbname;
|
|
|
|
}
|
2013-10-16 20:50:50 +02:00
|
|
|
|
2013-11-20 01:29:42 +01:00
|
|
|
// -- Sanitize the table properties collector
|
|
|
|
// All user defined properties collectors will be wrapped by
|
|
|
|
// UserKeyTablePropertiesCollector since for them they only have the
|
|
|
|
// knowledge of the user keys; internal keys are invisible to them.
|
|
|
|
auto& collectors = result.table_properties_collectors;
|
|
|
|
for (size_t i = 0; i < result.table_properties_collectors.size(); ++i) {
|
2013-10-16 20:50:50 +02:00
|
|
|
assert(collectors[i]);
|
|
|
|
collectors[i] =
|
2013-11-20 01:29:42 +01:00
|
|
|
std::make_shared<UserKeyTablePropertiesCollector>(collectors[i]);
|
2013-10-16 20:50:50 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// Add collector to collect internal key statistics
|
|
|
|
collectors.push_back(
|
2013-11-20 01:29:42 +01:00
|
|
|
std::make_shared<InternalKeyPropertiesCollector>()
|
2013-10-16 20:50:50 +02:00
|
|
|
);
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2013-10-30 18:52:33 +01:00
|
|
|
CompressionType GetCompressionType(const Options& options, int level,
|
|
|
|
const bool enable_compression) {
|
|
|
|
if (!enable_compression) {
|
|
|
|
// disable compression
|
|
|
|
return kNoCompression;
|
|
|
|
}
|
|
|
|
// If the use has specified a different compression level for each level,
|
|
|
|
// then pick the compresison for that level.
|
|
|
|
if (!options.compression_per_level.empty()) {
|
|
|
|
const int n = options.compression_per_level.size() - 1;
|
|
|
|
// It is possible for level_ to be -1; in that case, we use level
|
|
|
|
// 0's compression. This occurs mostly in backwards compatibility
|
|
|
|
// situations when the builder doesn't know what level the file
|
|
|
|
// belongs to. Likewise, if level_ is beyond the end of the
|
|
|
|
// specified compression levels, use the last value.
|
|
|
|
return options.compression_per_level[std::max(0, std::min(level, n))];
|
|
|
|
} else {
|
|
|
|
return options.compression;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-12-19 19:02:53 +01:00
|
|
|
CompressionType GetCompressionFlush(const Options& options) {
|
|
|
|
// Compressing memtable flushes might not help unless the sequential load
|
|
|
|
// optimization is used for leveled compaction. Otherwise the CPU and
|
|
|
|
// latency overhead is not offset by saving much space.
|
|
|
|
|
|
|
|
bool can_compress;
|
|
|
|
|
|
|
|
if (options.compaction_style == kCompactionStyleUniversal) {
|
|
|
|
can_compress =
|
|
|
|
(options.compaction_options_universal.compression_size_percent < 0);
|
|
|
|
} else {
|
|
|
|
// For leveled compress when min_level_to_compress == 0.
|
|
|
|
can_compress = (GetCompressionType(options, 0, true) != kNoCompression);
|
|
|
|
}
|
|
|
|
|
|
|
|
if (can_compress) {
|
|
|
|
return options.compression;
|
|
|
|
} else {
|
|
|
|
return kNoCompression;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
DBImpl::DBImpl(const Options& options, const std::string& dbname)
|
|
|
|
: env_(options.env),
|
2012-11-06 04:18:49 +01:00
|
|
|
dbname_(dbname),
|
2011-03-18 23:37:00 +01:00
|
|
|
internal_comparator_(options.comparator),
|
2014-01-15 00:27:09 +01:00
|
|
|
options_(SanitizeOptions(dbname, &internal_comparator_,
|
|
|
|
&internal_filter_policy_, options)),
|
2012-11-06 04:18:49 +01:00
|
|
|
internal_filter_policy_(options.filter_policy),
|
2011-03-18 23:37:00 +01:00
|
|
|
owns_info_log_(options_.info_log != options.info_log),
|
2013-02-15 20:53:17 +01:00
|
|
|
db_lock_(nullptr),
|
2013-06-01 01:30:17 +02:00
|
|
|
mutex_(options.use_adaptive_mutex),
|
2013-02-15 20:53:17 +01:00
|
|
|
shutting_down_(nullptr),
|
2011-03-18 23:37:00 +01:00
|
|
|
bg_cv_(&mutex_),
|
2011-06-22 04:36:45 +02:00
|
|
|
logfile_number_(0),
|
2013-03-28 23:19:28 +01:00
|
|
|
tmp_batch_(),
|
2012-10-19 23:00:53 +02:00
|
|
|
bg_compaction_scheduled_(0),
|
Fix a deadlock in CompactRange()
Summary:
The way DBImpl::TEST_CompactRange() throttles down the number of bg compactions
can cause it to deadlock when CompactRange() is called concurrently from
multiple threads. Imagine a following scenario with only two threads
(max_background_compactions is 10 and bg_compaction_scheduled_ is initially 0):
1. Thread #1 increments bg_compaction_scheduled_ (to LargeNumber), sets
bg_compaction_scheduled_ to 9 (newvalue), schedules the compaction
(bg_compaction_scheduled_ is now 10) and waits for it to complete.
2. Thread #2 calls TEST_CompactRange(), increments bg_compaction_scheduled_
(now LargeNumber + 10) and waits on a cv for bg_compaction_scheduled_ to
drop to LargeNumber.
3. BG thread completes the first manual compaction, decrements
bg_compaction_scheduled_ and wakes up all threads waiting on bg_cv_.
Thread #1 runs, increments bg_compaction_scheduled_ by LargeNumber again
(now 2*LargeNumber + 9). Since that's more than LargeNumber + newvalue,
thread #2 also goes to sleep (waiting on bg_cv_), without resetting
bg_compaction_scheduled_.
This diff attempts to address the problem by introducing a new counter
bg_manual_only_ (when positive, MaybeScheduleFlushOrCompaction() will only
schedule manual compactions).
Test Plan:
I could pretty much consistently reproduce the deadlock with a program that
calls CompactRange(nullptr, nullptr) immediately after Write() from multiple
threads. This no longer happens with this patch.
Tests (make check) pass.
Reviewers: dhruba, igor, sdong, haobo
Reviewed By: igor
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14799
2013-12-22 00:10:39 +01:00
|
|
|
bg_manual_only_(0),
|
2013-09-13 23:38:37 +02:00
|
|
|
bg_flush_scheduled_(0),
|
2012-08-27 09:50:26 +02:00
|
|
|
bg_logstats_scheduled_(false),
|
2013-02-15 20:53:17 +01:00
|
|
|
manual_compaction_(nullptr),
|
|
|
|
logger_(nullptr),
|
2014-01-02 12:33:42 +01:00
|
|
|
disable_delete_obsolete_files_(0),
|
2013-11-15 03:03:57 +01:00
|
|
|
delete_obsolete_files_last_run_(options.env->NowMicros()),
|
2013-05-06 20:41:01 +02:00
|
|
|
purge_wal_files_last_run_(0),
|
2013-05-11 00:21:04 +02:00
|
|
|
last_stats_dump_time_microsec_(0),
|
2013-11-07 03:46:28 +01:00
|
|
|
default_interval_to_delete_obsolete_WAL_(600),
|
2012-10-23 19:34:09 +02:00
|
|
|
stall_level0_slowdown_(0),
|
|
|
|
stall_memtable_compaction_(0),
|
|
|
|
stall_level0_num_files_(0),
|
2013-07-29 19:34:23 +02:00
|
|
|
stall_level0_slowdown_count_(0),
|
|
|
|
stall_memtable_compaction_count_(0),
|
|
|
|
stall_level0_num_files_count_(0),
|
2012-10-26 17:57:56 +02:00
|
|
|
started_at_(options.env->NowMicros()),
|
2013-02-19 07:32:07 +01:00
|
|
|
flush_on_destroy_(false),
|
2013-03-28 23:19:28 +01:00
|
|
|
stats_(options.num_levels),
|
2013-03-04 19:44:04 +01:00
|
|
|
delayed_writes_(0),
|
2013-06-30 08:21:36 +02:00
|
|
|
storage_options_(options),
|
|
|
|
bg_work_gate_closed_(false),
|
|
|
|
refitting_level_(false) {
|
2013-03-04 19:44:04 +01:00
|
|
|
|
2012-09-06 02:44:13 +02:00
|
|
|
env_->GetAbsolutePath(dbname, &db_absolute_path_);
|
2013-03-02 21:56:04 +01:00
|
|
|
|
|
|
|
stall_leveln_slowdown_.resize(options.num_levels);
|
2013-07-29 19:34:23 +02:00
|
|
|
stall_leveln_slowdown_count_.resize(options.num_levels);
|
|
|
|
for (int i = 0; i < options.num_levels; ++i) {
|
2013-03-02 21:56:04 +01:00
|
|
|
stall_leveln_slowdown_[i] = 0;
|
2013-07-29 19:34:23 +02:00
|
|
|
stall_leveln_slowdown_count_[i] = 0;
|
|
|
|
}
|
2013-03-02 21:56:04 +01:00
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
// Reserve ten files or so for other uses and give the rest to TableCache.
|
2012-08-22 20:43:53 +02:00
|
|
|
const int table_cache_size = options_.max_open_files - 10;
|
2013-03-15 01:00:04 +01:00
|
|
|
table_cache_.reset(new TableCache(dbname_, &options_,
|
|
|
|
storage_options_, table_cache_size));
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2013-03-15 01:00:04 +01:00
|
|
|
versions_.reset(new VersionSet(dbname_, &options_, storage_options_,
|
|
|
|
table_cache_.get(), &internal_comparator_));
|
2014-01-28 20:05:04 +01:00
|
|
|
column_family_memtables_.reset(
|
|
|
|
new ColumnFamilyMemTablesImpl(versions_->GetColumnFamilySet()));
|
2012-08-15 00:20:36 +02:00
|
|
|
|
2013-01-20 11:07:13 +01:00
|
|
|
dumpLeveldbBuildVersion(options_.info_log.get());
|
|
|
|
options_.Dump(options_.info_log.get());
|
2012-08-22 20:43:53 +02:00
|
|
|
|
2012-08-15 00:20:36 +02:00
|
|
|
char name[100];
|
2012-09-12 18:54:22 +02:00
|
|
|
Status st = env_->GetHostName(name, 100L);
|
[RocksDB] [MergeOperator] The new Merge Interface! Uses merge sequences.
Summary:
Here are the major changes to the Merge Interface. It has been expanded
to handle cases where the MergeOperator is not associative. It does so by stacking
up merge operations while scanning through the key history (i.e.: during Get() or
Compaction), until a valid Put/Delete/end-of-history is encountered; it then
applies all of the merge operations in the correct sequence starting with the
base/sentinel value.
I have also introduced an "AssociativeMerge" function which allows the user to
take advantage of associative merge operations (such as in the case of counters).
The implementation will always attempt to merge the operations/operands themselves
together when they are encountered, and will resort to the "stacking" method if
and only if the "associative-merge" fails.
This implementation is conjectured to allow MergeOperator to handle the general
case, while still providing the user with the ability to take advantage of certain
efficiencies in their own merge-operator / data-structure.
NOTE: This is a preliminary diff. This must still go through a lot of review,
revision, and testing. Feedback welcome!
Test Plan:
-This is a preliminary diff. I have only just begun testing/debugging it.
-I will be testing this with the existing MergeOperator use-cases and unit-tests
(counters, string-append, and redis-lists)
-I will be "desk-checking" and walking through the code with the help gdb.
-I will find a way of stress-testing the new interface / implementation using
db_bench, db_test, merge_test, and/or db_stress.
-I will ensure that my tests cover all cases: Get-Memtable,
Get-Immutable-Memtable, Get-from-Disk, Iterator-Range-Scan, Flush-Memtable-to-L0,
Compaction-L0-L1, Compaction-Ln-L(n+1), Put/Delete found, Put/Delete not-found,
end-of-history, end-of-file, etc.
-A lot of feedback from the reviewers.
Reviewers: haobo, dhruba, zshao, emayanke
Reviewed By: haobo
CC: leveldb
Differential Revision: https://reviews.facebook.net/D11499
2013-08-06 05:14:32 +02:00
|
|
|
if (st.ok()) {
|
2012-08-15 00:20:36 +02:00
|
|
|
host_name_ = name;
|
|
|
|
} else {
|
|
|
|
Log(options_.info_log, "Can't get hostname, use localhost as host name.");
|
|
|
|
host_name_ = "localhost";
|
|
|
|
}
|
|
|
|
last_log_ts = 0;
|
2012-11-26 22:56:45 +01:00
|
|
|
|
2013-11-07 20:31:56 +01:00
|
|
|
LogFlush(options_.info_log);
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
DBImpl::~DBImpl() {
|
|
|
|
// Wait for background work to finish
|
2014-01-29 22:28:50 +01:00
|
|
|
if (flush_on_destroy_ && default_cfd_->mem()->GetFirstSequenceNumber() != 0) {
|
2012-11-06 20:21:57 +01:00
|
|
|
FlushMemTable(FlushOptions());
|
|
|
|
}
|
2012-11-09 03:45:19 +01:00
|
|
|
mutex_.Lock();
|
2013-02-15 20:53:17 +01:00
|
|
|
shutting_down_.Release_Store(this); // Any non-nullptr value is ok
|
2013-09-13 23:38:37 +02:00
|
|
|
while (bg_compaction_scheduled_ ||
|
|
|
|
bg_flush_scheduled_ ||
|
|
|
|
bg_logstats_scheduled_) {
|
2011-06-07 16:40:26 +02:00
|
|
|
bg_cv_.Wait();
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
mutex_.Unlock();
|
|
|
|
|
2013-02-15 20:53:17 +01:00
|
|
|
if (db_lock_ != nullptr) {
|
2011-03-18 23:37:00 +01:00
|
|
|
env_->UnlockFile(db_lock_);
|
|
|
|
}
|
|
|
|
|
2013-11-07 20:31:56 +01:00
|
|
|
LogFlush(options_.info_log);
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2012-11-29 01:42:36 +01:00
|
|
|
// Do not flush and close database elegantly. Simulate a crash.
|
2012-11-17 00:28:14 +01:00
|
|
|
void DBImpl::TEST_Destroy_DBImpl() {
|
|
|
|
// ensure that no new memtable flushes can occur
|
|
|
|
flush_on_destroy_ = false;
|
|
|
|
|
|
|
|
// wait till all background compactions are done.
|
|
|
|
mutex_.Lock();
|
2013-09-13 23:38:37 +02:00
|
|
|
while (bg_compaction_scheduled_ ||
|
|
|
|
bg_flush_scheduled_ ||
|
|
|
|
bg_logstats_scheduled_) {
|
2012-11-17 00:28:14 +01:00
|
|
|
bg_cv_.Wait();
|
|
|
|
}
|
|
|
|
|
|
|
|
// Prevent new compactions from occuring.
|
2013-09-13 23:38:37 +02:00
|
|
|
bg_work_gate_closed_ = true;
|
2012-11-17 00:28:14 +01:00
|
|
|
const int LargeNumber = 10000000;
|
|
|
|
bg_compaction_scheduled_ += LargeNumber;
|
2013-09-13 23:38:37 +02:00
|
|
|
|
2012-11-17 00:28:14 +01:00
|
|
|
mutex_.Unlock();
|
2013-11-07 20:31:56 +01:00
|
|
|
LogFlush(options_.info_log);
|
2012-11-17 00:28:14 +01:00
|
|
|
|
|
|
|
// force release the lock file.
|
2013-02-15 20:53:17 +01:00
|
|
|
if (db_lock_ != nullptr) {
|
2012-11-17 00:28:14 +01:00
|
|
|
env_->UnlockFile(db_lock_);
|
|
|
|
}
|
2013-01-20 11:07:13 +01:00
|
|
|
|
|
|
|
log_.reset();
|
|
|
|
versions_.reset();
|
|
|
|
table_cache_.reset();
|
2012-11-17 00:28:14 +01:00
|
|
|
}
|
|
|
|
|
2013-01-11 02:18:50 +01:00
|
|
|
uint64_t DBImpl::TEST_Current_Manifest_FileNo() {
|
|
|
|
return versions_->ManifestFileNumber();
|
|
|
|
}
|
2012-11-17 00:28:14 +01:00
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
Status DBImpl::NewDB() {
|
2014-01-15 00:27:09 +01:00
|
|
|
VersionEdit new_db;
|
2011-03-18 23:37:00 +01:00
|
|
|
new_db.SetComparatorName(user_comparator()->Name());
|
2011-04-12 21:38:58 +02:00
|
|
|
new_db.SetLogNumber(0);
|
2011-03-18 23:37:00 +01:00
|
|
|
new_db.SetNextFile(2);
|
|
|
|
new_db.SetLastSequence(0);
|
|
|
|
|
|
|
|
const std::string manifest = DescriptorFileName(dbname_, 1);
|
2013-01-20 11:07:13 +01:00
|
|
|
unique_ptr<WritableFile> file;
|
2013-03-15 01:00:04 +01:00
|
|
|
Status s = env_->NewWritableFile(manifest, &file, storage_options_);
|
2011-03-18 23:37:00 +01:00
|
|
|
if (!s.ok()) {
|
|
|
|
return s;
|
|
|
|
}
|
2013-01-15 23:05:42 +01:00
|
|
|
file->SetPreallocationBlockSize(options_.manifest_preallocation_size);
|
2011-03-18 23:37:00 +01:00
|
|
|
{
|
2013-01-20 11:07:13 +01:00
|
|
|
log::Writer log(std::move(file));
|
2011-03-18 23:37:00 +01:00
|
|
|
std::string record;
|
|
|
|
new_db.EncodeTo(&record);
|
|
|
|
s = log.AddRecord(record);
|
|
|
|
}
|
|
|
|
if (s.ok()) {
|
|
|
|
// Make "CURRENT" file that points to the new manifest file.
|
|
|
|
s = SetCurrentFile(env_, dbname_, 1);
|
|
|
|
} else {
|
|
|
|
env_->DeleteFile(manifest);
|
|
|
|
}
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
|
|
|
void DBImpl::MaybeIgnoreError(Status* s) const {
|
|
|
|
if (s->ok() || options_.paranoid_checks) {
|
|
|
|
// No change needed
|
|
|
|
} else {
|
2011-07-21 04:40:18 +02:00
|
|
|
Log(options_.info_log, "Ignoring error %s", s->ToString().c_str());
|
2011-03-18 23:37:00 +01:00
|
|
|
*s = Status::OK();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2012-11-26 22:56:45 +01:00
|
|
|
const Status DBImpl::CreateArchivalDirectory() {
|
2013-11-07 03:46:28 +01:00
|
|
|
if (options_.WAL_ttl_seconds > 0 || options_.WAL_size_limit_MB > 0) {
|
2013-10-01 23:46:52 +02:00
|
|
|
std::string archivalPath = ArchivalDirectory(options_.wal_dir);
|
2012-11-26 22:56:45 +01:00
|
|
|
return env_->CreateDirIfMissing(archivalPath);
|
|
|
|
}
|
|
|
|
return Status::OK();
|
|
|
|
}
|
|
|
|
|
2013-05-28 21:35:43 +02:00
|
|
|
void DBImpl::PrintStatistics() {
|
2013-11-22 23:14:05 +01:00
|
|
|
auto dbstats = options_.statistics.get();
|
2013-05-28 21:35:43 +02:00
|
|
|
if (dbstats) {
|
|
|
|
Log(options_.info_log,
|
2013-06-19 05:28:41 +02:00
|
|
|
"STATISTCS:\n %s",
|
|
|
|
dbstats->ToString().c_str());
|
2013-05-28 21:35:43 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-05-11 00:21:04 +02:00
|
|
|
void DBImpl::MaybeDumpStats() {
|
2013-05-24 21:52:45 +02:00
|
|
|
if (options_.stats_dump_period_sec == 0) return;
|
|
|
|
|
|
|
|
const uint64_t now_micros = env_->NowMicros();
|
|
|
|
|
|
|
|
if (last_stats_dump_time_microsec_ +
|
|
|
|
options_.stats_dump_period_sec * 1000000
|
|
|
|
<= now_micros) {
|
|
|
|
// Multiple threads could race in here simultaneously.
|
|
|
|
// However, the last one will update last_stats_dump_time_microsec_
|
|
|
|
// atomically. We could see more than one dump during one dump
|
|
|
|
// period in rare cases.
|
|
|
|
last_stats_dump_time_microsec_ = now_micros;
|
|
|
|
std::string stats;
|
2013-10-05 07:32:05 +02:00
|
|
|
GetProperty("rocksdb.stats", &stats);
|
2013-05-24 21:52:45 +02:00
|
|
|
Log(options_.info_log, "%s", stats.c_str());
|
2013-05-28 21:35:43 +02:00
|
|
|
PrintStatistics();
|
2013-05-11 00:21:04 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-11-12 20:53:26 +01:00
|
|
|
// Returns the list of live files in 'sst_live' and the list
|
|
|
|
// of all files in the filesystem in 'all_files'.
|
2013-11-15 03:03:57 +01:00
|
|
|
// no_full_scan = true -- never do the full scan using GetChildren()
|
|
|
|
// force = false -- don't force the full scan, except every
|
|
|
|
// options_.delete_obsolete_files_period_micros
|
|
|
|
// force = true -- force the full scan
|
|
|
|
void DBImpl::FindObsoleteFiles(DeletionState& deletion_state,
|
|
|
|
bool force,
|
|
|
|
bool no_full_scan) {
|
2012-10-21 10:49:48 +02:00
|
|
|
mutex_.AssertHeld();
|
|
|
|
|
2012-09-15 02:11:35 +02:00
|
|
|
// if deletion is disabled, do nothing
|
2014-01-02 12:33:42 +01:00
|
|
|
if (disable_delete_obsolete_files_ > 0) {
|
2012-09-15 02:11:35 +02:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2013-11-15 07:42:02 +01:00
|
|
|
bool doing_the_full_scan = false;
|
|
|
|
|
|
|
|
// logic for figurint out if we're doing the full scan
|
|
|
|
if (no_full_scan) {
|
|
|
|
doing_the_full_scan = false;
|
|
|
|
} else if (force || options_.delete_obsolete_files_period_micros == 0) {
|
|
|
|
doing_the_full_scan = true;
|
|
|
|
} else {
|
|
|
|
const uint64_t now_micros = env_->NowMicros();
|
|
|
|
if (delete_obsolete_files_last_run_ +
|
|
|
|
options_.delete_obsolete_files_period_micros < now_micros) {
|
|
|
|
doing_the_full_scan = true;
|
|
|
|
delete_obsolete_files_last_run_ = now_micros;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-11-15 03:03:57 +01:00
|
|
|
// get obsolete files
|
|
|
|
versions_->GetObsoleteFiles(&deletion_state.sst_delete_files);
|
|
|
|
|
2013-11-09 00:23:46 +01:00
|
|
|
// store the current filenum, lognum, etc
|
|
|
|
deletion_state.manifest_file_number = versions_->ManifestFileNumber();
|
2014-01-28 20:05:04 +01:00
|
|
|
deletion_state.log_number = versions_->MinLogNumber();
|
2013-11-09 00:23:46 +01:00
|
|
|
deletion_state.prev_log_number = versions_->PrevLogNumber();
|
|
|
|
|
2013-11-15 07:42:02 +01:00
|
|
|
if (!doing_the_full_scan && !deletion_state.HaveSomethingToDelete()) {
|
|
|
|
// avoid filling up sst_live if we're sure that we
|
|
|
|
// are not going to do the full scan and that we don't have
|
|
|
|
// anything to delete at the moment
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
// don't delete live files
|
2013-11-15 03:03:57 +01:00
|
|
|
deletion_state.sst_live.assign(pending_outputs_.begin(),
|
|
|
|
pending_outputs_.end());
|
|
|
|
versions_->AddLiveFiles(&deletion_state.sst_live);
|
|
|
|
|
2013-11-15 07:42:02 +01:00
|
|
|
if (doing_the_full_scan) {
|
|
|
|
// set of all files in the directory
|
|
|
|
env_->GetChildren(dbname_, &deletion_state.all_files); // Ignore errors
|
|
|
|
|
|
|
|
//Add log files in wal_dir
|
|
|
|
if (options_.wal_dir != dbname_) {
|
|
|
|
std::vector<std::string> log_files;
|
|
|
|
env_->GetChildren(options_.wal_dir, &log_files); // Ignore errors
|
|
|
|
deletion_state.all_files.insert(
|
|
|
|
deletion_state.all_files.end(),
|
|
|
|
log_files.begin(),
|
|
|
|
log_files.end()
|
|
|
|
);
|
2012-10-16 17:53:46 +02:00
|
|
|
}
|
2013-10-01 23:46:52 +02:00
|
|
|
}
|
2013-07-16 20:56:46 +02:00
|
|
|
}
|
|
|
|
|
2012-10-21 10:49:48 +02:00
|
|
|
// Diffs the files listed in filenames and those that do not
|
2013-11-09 00:23:46 +01:00
|
|
|
// belong to live files are posibly removed. Also, removes all the
|
2013-11-12 20:53:26 +01:00
|
|
|
// files in sst_delete_files and log_delete_files.
|
2013-10-01 23:46:52 +02:00
|
|
|
// It is not necessary to hold the mutex when invoking this method.
|
2012-10-21 10:49:48 +02:00
|
|
|
void DBImpl::PurgeObsoleteFiles(DeletionState& state) {
|
2013-11-27 23:56:20 +01:00
|
|
|
|
|
|
|
// check if there is anything to do
|
|
|
|
if (!state.all_files.size() &&
|
|
|
|
!state.sst_delete_files.size() &&
|
|
|
|
!state.log_delete_files.size()) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2013-11-15 03:03:57 +01:00
|
|
|
// this checks if FindObsoleteFiles() was run before. If not, don't do
|
|
|
|
// PurgeObsoleteFiles(). If FindObsoleteFiles() was run, we need to also
|
|
|
|
// run PurgeObsoleteFiles(), even if disable_delete_obsolete_files_ is true
|
|
|
|
if (state.manifest_file_number == 0) {
|
2013-11-09 00:23:46 +01:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
uint64_t number;
|
|
|
|
FileType type;
|
2012-09-06 02:44:13 +02:00
|
|
|
std::vector<std::string> old_log_files;
|
2012-11-26 22:56:45 +01:00
|
|
|
|
[RocksDB] [Performance] Speed up FindObsoleteFiles
Summary:
FindObsoleteFiles was slow, holding the single big lock, resulted in bad p99 behavior.
Didn't profile anything, but several things could be improved:
1. VersionSet::AddLiveFiles works with std::set, which is by itself slow (a tree).
You also don't know how many dynamic allocations occur just for building up this tree.
switched to std::vector, also added logic to pre-calculate total size and do just one allocation
2. Don't see why env_->GetChildren() needs to be mutex proteced, moved to PurgeObsoleteFiles where
mutex could be unlocked.
3. switched std::set to std:unordered_set, the conversion from vector is also inside PurgeObsoleteFiles
I have a feeling this should pretty much fix it.
Test Plan: make check; db_stress
Reviewers: dhruba, heyongqiang, MarkCallaghan
Reviewed By: dhruba
CC: leveldb, zshao
Differential Revision: https://reviews.facebook.net/D10197
2013-04-12 01:49:53 +02:00
|
|
|
// Now, convert live list to an unordered set, WITHOUT mutex held;
|
|
|
|
// set is slow.
|
2013-11-12 20:53:26 +01:00
|
|
|
std::unordered_set<uint64_t> live_set(state.sst_live.begin(),
|
|
|
|
state.sst_live.end());
|
2013-11-09 00:23:46 +01:00
|
|
|
|
2013-11-12 20:53:26 +01:00
|
|
|
state.all_files.reserve(state.all_files.size() +
|
|
|
|
state.sst_delete_files.size());
|
|
|
|
for (auto file : state.sst_delete_files) {
|
2013-11-13 05:32:07 +01:00
|
|
|
state.all_files.push_back(TableFileName("", file->number).substr(1));
|
2013-11-12 20:53:26 +01:00
|
|
|
delete file;
|
2013-11-09 00:23:46 +01:00
|
|
|
}
|
|
|
|
|
2013-11-12 20:53:26 +01:00
|
|
|
state.all_files.reserve(state.all_files.size() +
|
|
|
|
state.log_delete_files.size());
|
|
|
|
for (auto filenum : state.log_delete_files) {
|
2013-11-09 00:23:46 +01:00
|
|
|
if (filenum > 0) {
|
2013-11-13 05:32:07 +01:00
|
|
|
state.all_files.push_back(LogFileName("", filenum).substr(1));
|
2013-11-09 00:23:46 +01:00
|
|
|
}
|
|
|
|
}
|
[RocksDB] [Performance] Speed up FindObsoleteFiles
Summary:
FindObsoleteFiles was slow, holding the single big lock, resulted in bad p99 behavior.
Didn't profile anything, but several things could be improved:
1. VersionSet::AddLiveFiles works with std::set, which is by itself slow (a tree).
You also don't know how many dynamic allocations occur just for building up this tree.
switched to std::vector, also added logic to pre-calculate total size and do just one allocation
2. Don't see why env_->GetChildren() needs to be mutex proteced, moved to PurgeObsoleteFiles where
mutex could be unlocked.
3. switched std::set to std:unordered_set, the conversion from vector is also inside PurgeObsoleteFiles
I have a feeling this should pretty much fix it.
Test Plan: make check; db_stress
Reviewers: dhruba, heyongqiang, MarkCallaghan
Reviewed By: dhruba
CC: leveldb, zshao
Differential Revision: https://reviews.facebook.net/D10197
2013-04-12 01:49:53 +02:00
|
|
|
|
2013-11-13 05:32:07 +01:00
|
|
|
// dedup state.all_files so we don't try to delete the same
|
|
|
|
// file twice
|
|
|
|
sort(state.all_files.begin(), state.all_files.end());
|
|
|
|
auto unique_end = unique(state.all_files.begin(), state.all_files.end());
|
|
|
|
|
|
|
|
for (size_t i = 0; state.all_files.begin() + i < unique_end; i++) {
|
2013-11-12 20:53:26 +01:00
|
|
|
if (ParseFileName(state.all_files[i], &number, &type)) {
|
2011-03-18 23:37:00 +01:00
|
|
|
bool keep = true;
|
|
|
|
switch (type) {
|
|
|
|
case kLogFile:
|
2013-11-09 00:23:46 +01:00
|
|
|
keep = ((number >= state.log_number) ||
|
|
|
|
(number == state.prev_log_number));
|
2011-03-18 23:37:00 +01:00
|
|
|
break;
|
|
|
|
case kDescriptorFile:
|
|
|
|
// Keep my manifest file, and any newer incarnations'
|
|
|
|
// (in case there is a race that allows other incarnations)
|
2013-11-09 00:23:46 +01:00
|
|
|
keep = (number >= state.manifest_file_number);
|
2011-03-18 23:37:00 +01:00
|
|
|
break;
|
|
|
|
case kTableFile:
|
[RocksDB] [Performance] Speed up FindObsoleteFiles
Summary:
FindObsoleteFiles was slow, holding the single big lock, resulted in bad p99 behavior.
Didn't profile anything, but several things could be improved:
1. VersionSet::AddLiveFiles works with std::set, which is by itself slow (a tree).
You also don't know how many dynamic allocations occur just for building up this tree.
switched to std::vector, also added logic to pre-calculate total size and do just one allocation
2. Don't see why env_->GetChildren() needs to be mutex proteced, moved to PurgeObsoleteFiles where
mutex could be unlocked.
3. switched std::set to std:unordered_set, the conversion from vector is also inside PurgeObsoleteFiles
I have a feeling this should pretty much fix it.
Test Plan: make check; db_stress
Reviewers: dhruba, heyongqiang, MarkCallaghan
Reviewed By: dhruba
CC: leveldb, zshao
Differential Revision: https://reviews.facebook.net/D10197
2013-04-12 01:49:53 +02:00
|
|
|
keep = (live_set.find(number) != live_set.end());
|
2011-03-18 23:37:00 +01:00
|
|
|
break;
|
|
|
|
case kTempFile:
|
|
|
|
// Any temp files that are currently being written to must
|
|
|
|
// be recorded in pending_outputs_, which is inserted into "live"
|
[RocksDB] [Performance] Speed up FindObsoleteFiles
Summary:
FindObsoleteFiles was slow, holding the single big lock, resulted in bad p99 behavior.
Didn't profile anything, but several things could be improved:
1. VersionSet::AddLiveFiles works with std::set, which is by itself slow (a tree).
You also don't know how many dynamic allocations occur just for building up this tree.
switched to std::vector, also added logic to pre-calculate total size and do just one allocation
2. Don't see why env_->GetChildren() needs to be mutex proteced, moved to PurgeObsoleteFiles where
mutex could be unlocked.
3. switched std::set to std:unordered_set, the conversion from vector is also inside PurgeObsoleteFiles
I have a feeling this should pretty much fix it.
Test Plan: make check; db_stress
Reviewers: dhruba, heyongqiang, MarkCallaghan
Reviewed By: dhruba
CC: leveldb, zshao
Differential Revision: https://reviews.facebook.net/D10197
2013-04-12 01:49:53 +02:00
|
|
|
keep = (live_set.find(number) != live_set.end());
|
2011-03-18 23:37:00 +01:00
|
|
|
break;
|
2012-08-18 01:06:05 +02:00
|
|
|
case kInfoLogFile:
|
|
|
|
keep = true;
|
|
|
|
if (number != 0) {
|
2013-11-12 20:53:26 +01:00
|
|
|
old_log_files.push_back(state.all_files[i]);
|
2012-08-18 01:06:05 +02:00
|
|
|
}
|
|
|
|
break;
|
2011-03-18 23:37:00 +01:00
|
|
|
case kCurrentFile:
|
|
|
|
case kDBLockFile:
|
2013-10-18 23:50:54 +02:00
|
|
|
case kIdentityFile:
|
2012-12-17 20:26:59 +01:00
|
|
|
case kMetaDatabase:
|
2011-03-18 23:37:00 +01:00
|
|
|
keep = true;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!keep) {
|
|
|
|
if (type == kTableFile) {
|
2013-11-09 00:23:46 +01:00
|
|
|
// evict from cache
|
|
|
|
table_cache_->Evict(number);
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
2013-11-13 23:54:54 +01:00
|
|
|
std::string fname = ((type == kLogFile) ? options_.wal_dir : dbname_) +
|
|
|
|
"/" + state.all_files[i];
|
2013-11-13 06:02:03 +01:00
|
|
|
Log(options_.info_log,
|
|
|
|
"Delete type=%d #%lu",
|
|
|
|
int(type),
|
|
|
|
(unsigned long)number);
|
2013-05-06 20:41:01 +02:00
|
|
|
|
2013-11-09 00:23:46 +01:00
|
|
|
Status st;
|
|
|
|
if (type == kLogFile && (options_.WAL_ttl_seconds > 0 ||
|
|
|
|
options_.WAL_size_limit_MB > 0)) {
|
2013-11-13 05:32:07 +01:00
|
|
|
st = env_->RenameFile(fname,
|
|
|
|
ArchivedLogFileName(options_.wal_dir, number));
|
2013-11-09 00:23:46 +01:00
|
|
|
if (!st.ok()) {
|
2013-11-13 06:02:03 +01:00
|
|
|
Log(options_.info_log,
|
2013-11-13 23:54:54 +01:00
|
|
|
"RenameFile logfile #%lu FAILED -- %s\n",
|
|
|
|
(unsigned long)number, st.ToString().c_str());
|
2013-11-09 00:23:46 +01:00
|
|
|
}
|
2012-11-26 22:56:45 +01:00
|
|
|
} else {
|
2013-11-13 05:32:07 +01:00
|
|
|
st = env_->DeleteFile(fname);
|
[RocksDB] [MergeOperator] The new Merge Interface! Uses merge sequences.
Summary:
Here are the major changes to the Merge Interface. It has been expanded
to handle cases where the MergeOperator is not associative. It does so by stacking
up merge operations while scanning through the key history (i.e.: during Get() or
Compaction), until a valid Put/Delete/end-of-history is encountered; it then
applies all of the merge operations in the correct sequence starting with the
base/sentinel value.
I have also introduced an "AssociativeMerge" function which allows the user to
take advantage of associative merge operations (such as in the case of counters).
The implementation will always attempt to merge the operations/operands themselves
together when they are encountered, and will resort to the "stacking" method if
and only if the "associative-merge" fails.
This implementation is conjectured to allow MergeOperator to handle the general
case, while still providing the user with the ability to take advantage of certain
efficiencies in their own merge-operator / data-structure.
NOTE: This is a preliminary diff. This must still go through a lot of review,
revision, and testing. Feedback welcome!
Test Plan:
-This is a preliminary diff. I have only just begun testing/debugging it.
-I will be testing this with the existing MergeOperator use-cases and unit-tests
(counters, string-append, and redis-lists)
-I will be "desk-checking" and walking through the code with the help gdb.
-I will find a way of stress-testing the new interface / implementation using
db_bench, db_test, merge_test, and/or db_stress.
-I will ensure that my tests cover all cases: Get-Memtable,
Get-Immutable-Memtable, Get-from-Disk, Iterator-Range-Scan, Flush-Memtable-to-L0,
Compaction-L0-L1, Compaction-Ln-L(n+1), Put/Delete found, Put/Delete not-found,
end-of-history, end-of-file, etc.
-A lot of feedback from the reviewers.
Reviewers: haobo, dhruba, zshao, emayanke
Reviewed By: haobo
CC: leveldb
Differential Revision: https://reviews.facebook.net/D11499
2013-08-06 05:14:32 +02:00
|
|
|
if (!st.ok()) {
|
2013-11-13 23:54:54 +01:00
|
|
|
Log(options_.info_log, "Delete type=%d #%lu FAILED -- %s\n",
|
|
|
|
int(type), (unsigned long)number, st.ToString().c_str());
|
2012-11-26 22:56:45 +01:00
|
|
|
}
|
2012-08-23 04:15:06 +02:00
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2012-08-18 01:06:05 +02:00
|
|
|
|
2013-07-16 20:56:46 +02:00
|
|
|
// Delete old info log files.
|
2013-02-05 04:42:40 +01:00
|
|
|
size_t old_log_file_count = old_log_files.size();
|
|
|
|
// NOTE: Currently we only support log purge when options_.db_log_dir is
|
|
|
|
// located in `dbname` directory.
|
|
|
|
if (old_log_file_count >= options_.keep_log_file_num &&
|
|
|
|
options_.db_log_dir.empty()) {
|
2012-09-06 02:44:13 +02:00
|
|
|
std::sort(old_log_files.begin(), old_log_files.end());
|
2013-02-05 04:42:40 +01:00
|
|
|
size_t end = old_log_file_count - options_.keep_log_file_num;
|
2013-03-15 02:32:01 +01:00
|
|
|
for (unsigned int i = 0; i <= end; i++) {
|
2012-09-06 02:44:13 +02:00
|
|
|
std::string& to_delete = old_log_files.at(i);
|
2012-10-21 10:49:48 +02:00
|
|
|
// Log(options_.info_log, "Delete type=%d %s\n",
|
|
|
|
// int(kInfoLogFile), to_delete.c_str());
|
2012-08-18 01:06:05 +02:00
|
|
|
env_->DeleteFile(dbname_ + "/" + to_delete);
|
|
|
|
}
|
|
|
|
}
|
2013-05-06 20:41:01 +02:00
|
|
|
PurgeObsoleteWALFiles();
|
2013-11-09 00:23:46 +01:00
|
|
|
LogFlush(options_.info_log);
|
2012-10-21 10:49:48 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
void DBImpl::DeleteObsoleteFiles() {
|
|
|
|
mutex_.AssertHeld();
|
|
|
|
DeletionState deletion_state;
|
2013-11-09 00:23:46 +01:00
|
|
|
FindObsoleteFiles(deletion_state, true);
|
2012-10-21 10:49:48 +02:00
|
|
|
PurgeObsoleteFiles(deletion_state);
|
2012-11-26 22:56:45 +01:00
|
|
|
}
|
|
|
|
|
2013-11-07 03:46:28 +01:00
|
|
|
// 1. Go through all archived files and
|
|
|
|
// a. if ttl is enabled, delete outdated files
|
|
|
|
// b. if archive size limit is enabled, delete empty files,
|
|
|
|
// compute file number and size.
|
|
|
|
// 2. If size limit is enabled:
|
|
|
|
// a. compute how many files should be deleted
|
|
|
|
// b. get sorted non-empty archived logs
|
|
|
|
// c. delete what should be deleted
|
2012-11-26 22:56:45 +01:00
|
|
|
void DBImpl::PurgeObsoleteWALFiles() {
|
2013-11-07 03:46:28 +01:00
|
|
|
bool const ttl_enabled = options_.WAL_ttl_seconds > 0;
|
|
|
|
bool const size_limit_enabled = options_.WAL_size_limit_MB > 0;
|
|
|
|
if (!ttl_enabled && !size_limit_enabled) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2013-05-06 20:41:01 +02:00
|
|
|
int64_t current_time;
|
|
|
|
Status s = env_->GetCurrentTime(¤t_time);
|
2013-11-07 03:46:28 +01:00
|
|
|
if (!s.ok()) {
|
|
|
|
Log(options_.info_log, "Can't get current time: %s", s.ToString().c_str());
|
|
|
|
assert(false);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
uint64_t const now_seconds = static_cast<uint64_t>(current_time);
|
|
|
|
uint64_t const time_to_check = (ttl_enabled && !size_limit_enabled) ?
|
|
|
|
options_.WAL_ttl_seconds / 2 : default_interval_to_delete_obsolete_WAL_;
|
2013-05-06 20:41:01 +02:00
|
|
|
|
2013-11-07 03:46:28 +01:00
|
|
|
if (purge_wal_files_last_run_ + time_to_check > now_seconds) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
purge_wal_files_last_run_ = now_seconds;
|
|
|
|
|
|
|
|
std::string archival_dir = ArchivalDirectory(options_.wal_dir);
|
|
|
|
std::vector<std::string> files;
|
|
|
|
s = env_->GetChildren(archival_dir, &files);
|
|
|
|
if (!s.ok()) {
|
|
|
|
Log(options_.info_log, "Can't get archive files: %s", s.ToString().c_str());
|
|
|
|
assert(false);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
size_t log_files_num = 0;
|
|
|
|
uint64_t log_file_size = 0;
|
|
|
|
|
|
|
|
for (auto& f : files) {
|
|
|
|
uint64_t number;
|
|
|
|
FileType type;
|
|
|
|
if (ParseFileName(f, &number, &type) && type == kLogFile) {
|
|
|
|
std::string const file_path = archival_dir + "/" + f;
|
|
|
|
if (ttl_enabled) {
|
|
|
|
uint64_t file_m_time;
|
|
|
|
Status const s = env_->GetFileModificationTime(file_path,
|
|
|
|
&file_m_time);
|
|
|
|
if (!s.ok()) {
|
|
|
|
Log(options_.info_log, "Can't get file mod time: %s: %s",
|
|
|
|
file_path.c_str(), s.ToString().c_str());
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
if (now_seconds - file_m_time > options_.WAL_ttl_seconds) {
|
|
|
|
Status const s = env_->DeleteFile(file_path);
|
|
|
|
if (!s.ok()) {
|
|
|
|
Log(options_.info_log, "Can't delete file: %s: %s",
|
|
|
|
file_path.c_str(), s.ToString().c_str());
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
continue;
|
2012-11-26 22:56:45 +01:00
|
|
|
}
|
2013-11-07 03:46:28 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
if (size_limit_enabled) {
|
|
|
|
uint64_t file_size;
|
|
|
|
Status const s = env_->GetFileSize(file_path, &file_size);
|
|
|
|
if (!s.ok()) {
|
|
|
|
Log(options_.info_log, "Can't get file size: %s: %s",
|
|
|
|
file_path.c_str(), s.ToString().c_str());
|
|
|
|
return;
|
|
|
|
} else {
|
|
|
|
if (file_size > 0) {
|
|
|
|
log_file_size = std::max(log_file_size, file_size);
|
|
|
|
++log_files_num;
|
|
|
|
} else {
|
|
|
|
Status s = env_->DeleteFile(file_path);
|
|
|
|
if (!s.ok()) {
|
|
|
|
Log(options_.info_log, "Can't delete file: %s: %s",
|
|
|
|
file_path.c_str(), s.ToString().c_str());
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (0 == log_files_num || !size_limit_enabled) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
size_t const files_keep_num = options_.WAL_size_limit_MB *
|
|
|
|
1024 * 1024 / log_file_size;
|
|
|
|
if (log_files_num <= files_keep_num) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
size_t files_del_num = log_files_num - files_keep_num;
|
|
|
|
VectorLogPtr archived_logs;
|
|
|
|
AppendSortedWalsOfType(archival_dir, archived_logs, kArchivedLogFile);
|
|
|
|
|
|
|
|
if (files_del_num > archived_logs.size()) {
|
|
|
|
Log(options_.info_log, "Trying to delete more archived log files than "
|
|
|
|
"exist. Deleting all");
|
|
|
|
files_del_num = archived_logs.size();
|
|
|
|
}
|
|
|
|
|
|
|
|
for (size_t i = 0; i < files_del_num; ++i) {
|
|
|
|
std::string const file_path = archived_logs[i]->PathName();
|
|
|
|
Status const s = DeleteFile(file_path);
|
|
|
|
if (!s.ok()) {
|
|
|
|
Log(options_.info_log, "Can't delete file: %s: %s",
|
|
|
|
file_path.c_str(), s.ToString().c_str());
|
|
|
|
continue;
|
2012-11-26 22:56:45 +01:00
|
|
|
}
|
|
|
|
}
|
2012-10-21 10:49:48 +02:00
|
|
|
}
|
|
|
|
|
2014-01-06 22:31:06 +01:00
|
|
|
Status DBImpl::Recover(
|
2014-01-22 19:59:07 +01:00
|
|
|
const std::vector<ColumnFamilyDescriptor>& column_families, bool read_only,
|
|
|
|
bool error_if_log_file_exist) {
|
2011-03-18 23:37:00 +01:00
|
|
|
mutex_.AssertHeld();
|
|
|
|
|
2013-02-15 20:53:17 +01:00
|
|
|
assert(db_lock_ == nullptr);
|
Refactor Recover() code
Summary:
This diff does two things:
* Rethinks how we call Recover() with read_only option. Before, we call it with pointer to memtable where we'd like to apply those changes to. This memtable is set in db_impl_readonly.cc and it's actually DBImpl::mem_. Why don't we just apply updates to mem_ right away? It seems more intuitive.
* Changes when we apply updates to manifest. Before, the process is to recover all the logs, flush it to sst files and then do one giant commit that atomically adds all recovered sst files and sets the next log number. This works good enough, but causes some small troubles for my column family approach, since I can't have one VersionEdit apply to more than single column family[1]. The change here is to commit the files recovered from logs right away. Here is the state of the world before the change:
1. Recover log 5, add new sst files to edit
2. Recover log 7, add new sst files to edit
3. Recover log 8, add new sst files to edit
4. Commit all added sst files to manifest and mark log files 5, 7 and 8 as recoverd (via SetLogNumber(9) function)
After the change, we'll do:
1. Recover log 5, commit the new sst files and set log 5 as recovered
2. Recover log 7, commit the new sst files and set log 7 as recovered
3. Recover log 8, commit the new sst files and set log 8 as recovered
The added (small) benefit is that if we fail after (2), the new recovery will only have to recover log 8. In previous case, we'll have to restart the recovery from the beginning. The bigger benefit will be to enable easier integration of multiple column families in Recovery code path.
[1] I'm happy to dicuss this decison, but I believe this is the cleanest way to go. It also makes backward compatibility much easier. We don't have a requirement of adding multiple column families atomically.
Test Plan: make check
Reviewers: dhruba, haobo, kailiu, sdong
Reviewed By: kailiu
CC: leveldb
Differential Revision: https://reviews.facebook.net/D15237
2014-01-22 19:45:26 +01:00
|
|
|
if (!read_only) {
|
2013-01-07 19:11:18 +01:00
|
|
|
// We call CreateDirIfMissing() as the directory may already exist (if we
|
|
|
|
// are reopening a DB), when this happens we don't want creating the
|
|
|
|
// directory to cause an error. However, we need to check if creating the
|
|
|
|
// directory fails or else we may get an obscure message about the lock
|
|
|
|
// file not existing. One real-world example of this occurring is if
|
|
|
|
// env->CreateDirIfMissing() doesn't create intermediate directories, e.g.
|
|
|
|
// when dbname_ is "dir/db" but when "dir" doesn't exist.
|
|
|
|
Status s = env_->CreateDirIfMissing(dbname_);
|
|
|
|
if (!s.ok()) {
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
2014-01-27 20:02:21 +01:00
|
|
|
s = env_->NewDirectory(dbname_, &db_directory_);
|
|
|
|
if (!s.ok()) {
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
2013-01-07 19:11:18 +01:00
|
|
|
s = env_->LockFile(LockFileName(dbname_), &db_lock_);
|
2012-12-17 06:01:02 +01:00
|
|
|
if (!s.ok()) {
|
|
|
|
return s;
|
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2012-12-17 06:01:02 +01:00
|
|
|
if (!env_->FileExists(CurrentFileName(dbname_))) {
|
|
|
|
if (options_.create_if_missing) {
|
2013-03-21 23:59:47 +01:00
|
|
|
// TODO: add merge_operator name check
|
2012-12-17 06:01:02 +01:00
|
|
|
s = NewDB();
|
|
|
|
if (!s.ok()) {
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
return Status::InvalidArgument(
|
|
|
|
dbname_, "does not exist (create_if_missing is false)");
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
} else {
|
2012-12-17 06:01:02 +01:00
|
|
|
if (options_.error_if_exists) {
|
|
|
|
return Status::InvalidArgument(
|
|
|
|
dbname_, "exists (error_if_exists is true)");
|
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
2013-10-18 23:50:54 +02:00
|
|
|
// Check for the IDENTITY file and create it if not there
|
|
|
|
if (!env_->FileExists(IdentityFileName(dbname_))) {
|
|
|
|
s = SetIdentityFile(env_, dbname_);
|
|
|
|
if (!s.ok()) {
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2014-01-22 20:44:53 +01:00
|
|
|
Status s = versions_->Recover(column_families);
|
2011-03-18 23:37:00 +01:00
|
|
|
if (s.ok()) {
|
|
|
|
SequenceNumber max_sequence(0);
|
2014-01-24 23:30:28 +01:00
|
|
|
default_cfd_ = versions_->GetColumnFamilySet()->GetDefault();
|
2011-06-22 04:36:45 +02:00
|
|
|
|
|
|
|
// Recover from all newer log files than the ones named in the
|
|
|
|
// descriptor (new log files may have been added by the previous
|
|
|
|
// incarnation without registering them in the descriptor).
|
|
|
|
//
|
|
|
|
// Note that PrevLogNumber() is no longer used, but we pay
|
|
|
|
// attention to it in case we are recovering a database
|
2013-10-05 07:32:05 +02:00
|
|
|
// produced by an older version of rocksdb.
|
2014-01-28 20:05:04 +01:00
|
|
|
const uint64_t min_log = versions_->MinLogNumber();
|
2011-06-22 04:36:45 +02:00
|
|
|
const uint64_t prev_log = versions_->PrevLogNumber();
|
|
|
|
std::vector<std::string> filenames;
|
2013-10-01 23:46:52 +02:00
|
|
|
s = env_->GetChildren(options_.wal_dir, &filenames);
|
2011-06-22 04:36:45 +02:00
|
|
|
if (!s.ok()) {
|
|
|
|
return s;
|
2011-04-12 21:38:58 +02:00
|
|
|
}
|
2011-06-22 04:36:45 +02:00
|
|
|
uint64_t number;
|
|
|
|
FileType type;
|
|
|
|
std::vector<uint64_t> logs;
|
|
|
|
for (size_t i = 0; i < filenames.size(); i++) {
|
|
|
|
if (ParseFileName(filenames[i], &number, &type)
|
|
|
|
&& type == kLogFile
|
|
|
|
&& ((number >= min_log) || (number == prev_log))) {
|
|
|
|
logs.push_back(number);
|
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
2011-06-22 04:36:45 +02:00
|
|
|
|
2012-11-06 04:18:49 +01:00
|
|
|
if (logs.size() > 0 && error_if_log_file_exist) {
|
|
|
|
return Status::Corruption(""
|
|
|
|
"The db was opened in readonly mode with error_if_log_file_exist"
|
|
|
|
"flag but a log file already exists");
|
|
|
|
}
|
|
|
|
|
2011-06-22 04:36:45 +02:00
|
|
|
// Recover in the order in which the logs were generated
|
|
|
|
std::sort(logs.begin(), logs.end());
|
Refactor Recover() code
Summary:
This diff does two things:
* Rethinks how we call Recover() with read_only option. Before, we call it with pointer to memtable where we'd like to apply those changes to. This memtable is set in db_impl_readonly.cc and it's actually DBImpl::mem_. Why don't we just apply updates to mem_ right away? It seems more intuitive.
* Changes when we apply updates to manifest. Before, the process is to recover all the logs, flush it to sst files and then do one giant commit that atomically adds all recovered sst files and sets the next log number. This works good enough, but causes some small troubles for my column family approach, since I can't have one VersionEdit apply to more than single column family[1]. The change here is to commit the files recovered from logs right away. Here is the state of the world before the change:
1. Recover log 5, add new sst files to edit
2. Recover log 7, add new sst files to edit
3. Recover log 8, add new sst files to edit
4. Commit all added sst files to manifest and mark log files 5, 7 and 8 as recoverd (via SetLogNumber(9) function)
After the change, we'll do:
1. Recover log 5, commit the new sst files and set log 5 as recovered
2. Recover log 7, commit the new sst files and set log 7 as recovered
3. Recover log 8, commit the new sst files and set log 8 as recovered
The added (small) benefit is that if we fail after (2), the new recovery will only have to recover log 8. In previous case, we'll have to restart the recovery from the beginning. The bigger benefit will be to enable easier integration of multiple column families in Recovery code path.
[1] I'm happy to dicuss this decison, but I believe this is the cleanest way to go. It also makes backward compatibility much easier. We don't have a requirement of adding multiple column families atomically.
Test Plan: make check
Reviewers: dhruba, haobo, kailiu, sdong
Reviewed By: kailiu
CC: leveldb
Differential Revision: https://reviews.facebook.net/D15237
2014-01-22 19:45:26 +01:00
|
|
|
for (size_t i = 0; s.ok() && i < logs.size(); i++) {
|
2011-09-01 21:08:02 +02:00
|
|
|
// The previous incarnation may not have written any MANIFEST
|
|
|
|
// records after allocating this log number. So we manually
|
|
|
|
// update the file number allocation counter in VersionSet.
|
|
|
|
versions_->MarkFileNumberUsed(logs[i]);
|
Refactor Recover() code
Summary:
This diff does two things:
* Rethinks how we call Recover() with read_only option. Before, we call it with pointer to memtable where we'd like to apply those changes to. This memtable is set in db_impl_readonly.cc and it's actually DBImpl::mem_. Why don't we just apply updates to mem_ right away? It seems more intuitive.
* Changes when we apply updates to manifest. Before, the process is to recover all the logs, flush it to sst files and then do one giant commit that atomically adds all recovered sst files and sets the next log number. This works good enough, but causes some small troubles for my column family approach, since I can't have one VersionEdit apply to more than single column family[1]. The change here is to commit the files recovered from logs right away. Here is the state of the world before the change:
1. Recover log 5, add new sst files to edit
2. Recover log 7, add new sst files to edit
3. Recover log 8, add new sst files to edit
4. Commit all added sst files to manifest and mark log files 5, 7 and 8 as recoverd (via SetLogNumber(9) function)
After the change, we'll do:
1. Recover log 5, commit the new sst files and set log 5 as recovered
2. Recover log 7, commit the new sst files and set log 7 as recovered
3. Recover log 8, commit the new sst files and set log 8 as recovered
The added (small) benefit is that if we fail after (2), the new recovery will only have to recover log 8. In previous case, we'll have to restart the recovery from the beginning. The bigger benefit will be to enable easier integration of multiple column families in Recovery code path.
[1] I'm happy to dicuss this decison, but I believe this is the cleanest way to go. It also makes backward compatibility much easier. We don't have a requirement of adding multiple column families atomically.
Test Plan: make check
Reviewers: dhruba, haobo, kailiu, sdong
Reviewed By: kailiu
CC: leveldb
Differential Revision: https://reviews.facebook.net/D15237
2014-01-22 19:45:26 +01:00
|
|
|
s = RecoverLogFile(logs[i], &max_sequence, read_only);
|
2011-06-22 04:36:45 +02:00
|
|
|
}
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
if (s.ok()) {
|
2011-04-12 21:38:58 +02:00
|
|
|
if (versions_->LastSequence() < max_sequence) {
|
|
|
|
versions_->SetLastSequence(max_sequence);
|
|
|
|
}
|
2013-11-22 23:14:05 +01:00
|
|
|
SetTickerCount(options_.statistics.get(), SEQUENCE_NUMBER,
|
2013-08-16 07:06:26 +02:00
|
|
|
versions_->LastSequence());
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
Refactor Recover() code
Summary:
This diff does two things:
* Rethinks how we call Recover() with read_only option. Before, we call it with pointer to memtable where we'd like to apply those changes to. This memtable is set in db_impl_readonly.cc and it's actually DBImpl::mem_. Why don't we just apply updates to mem_ right away? It seems more intuitive.
* Changes when we apply updates to manifest. Before, the process is to recover all the logs, flush it to sst files and then do one giant commit that atomically adds all recovered sst files and sets the next log number. This works good enough, but causes some small troubles for my column family approach, since I can't have one VersionEdit apply to more than single column family[1]. The change here is to commit the files recovered from logs right away. Here is the state of the world before the change:
1. Recover log 5, add new sst files to edit
2. Recover log 7, add new sst files to edit
3. Recover log 8, add new sst files to edit
4. Commit all added sst files to manifest and mark log files 5, 7 and 8 as recoverd (via SetLogNumber(9) function)
After the change, we'll do:
1. Recover log 5, commit the new sst files and set log 5 as recovered
2. Recover log 7, commit the new sst files and set log 7 as recovered
3. Recover log 8, commit the new sst files and set log 8 as recovered
The added (small) benefit is that if we fail after (2), the new recovery will only have to recover log 8. In previous case, we'll have to restart the recovery from the beginning. The bigger benefit will be to enable easier integration of multiple column families in Recovery code path.
[1] I'm happy to dicuss this decison, but I believe this is the cleanest way to go. It also makes backward compatibility much easier. We don't have a requirement of adding multiple column families atomically.
Test Plan: make check
Reviewers: dhruba, haobo, kailiu, sdong
Reviewed By: kailiu
CC: leveldb
Differential Revision: https://reviews.facebook.net/D15237
2014-01-22 19:45:26 +01:00
|
|
|
Status DBImpl::RecoverLogFile(uint64_t log_number, SequenceNumber* max_sequence,
|
|
|
|
bool read_only) {
|
2011-03-18 23:37:00 +01:00
|
|
|
struct LogReporter : public log::Reader::Reporter {
|
|
|
|
Env* env;
|
2011-07-21 04:40:18 +02:00
|
|
|
Logger* info_log;
|
2011-03-18 23:37:00 +01:00
|
|
|
const char* fname;
|
2013-05-21 20:53:33 +02:00
|
|
|
Status* status; // nullptr if options_.paranoid_checks==false or
|
|
|
|
// options_.skip_log_error_on_recovery==true
|
2011-03-18 23:37:00 +01:00
|
|
|
virtual void Corruption(size_t bytes, const Status& s) {
|
2011-07-21 04:40:18 +02:00
|
|
|
Log(info_log, "%s%s: dropping %d bytes; %s",
|
2013-02-15 20:53:17 +01:00
|
|
|
(this->status == nullptr ? "(ignoring error) " : ""),
|
2011-03-18 23:37:00 +01:00
|
|
|
fname, static_cast<int>(bytes), s.ToString().c_str());
|
2013-02-15 20:53:17 +01:00
|
|
|
if (this->status != nullptr && this->status->ok()) *this->status = s;
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
mutex_.AssertHeld();
|
|
|
|
|
2014-01-28 20:05:04 +01:00
|
|
|
std::unordered_map<int, VersionEdit> version_edits;
|
|
|
|
for (auto cfd : *versions_->GetColumnFamilySet()) {
|
|
|
|
VersionEdit edit;
|
2014-01-29 22:28:50 +01:00
|
|
|
edit.SetColumnFamily(cfd->GetID());
|
|
|
|
version_edits.insert({cfd->GetID(), edit});
|
2014-01-28 20:05:04 +01:00
|
|
|
}
|
Refactor Recover() code
Summary:
This diff does two things:
* Rethinks how we call Recover() with read_only option. Before, we call it with pointer to memtable where we'd like to apply those changes to. This memtable is set in db_impl_readonly.cc and it's actually DBImpl::mem_. Why don't we just apply updates to mem_ right away? It seems more intuitive.
* Changes when we apply updates to manifest. Before, the process is to recover all the logs, flush it to sst files and then do one giant commit that atomically adds all recovered sst files and sets the next log number. This works good enough, but causes some small troubles for my column family approach, since I can't have one VersionEdit apply to more than single column family[1]. The change here is to commit the files recovered from logs right away. Here is the state of the world before the change:
1. Recover log 5, add new sst files to edit
2. Recover log 7, add new sst files to edit
3. Recover log 8, add new sst files to edit
4. Commit all added sst files to manifest and mark log files 5, 7 and 8 as recoverd (via SetLogNumber(9) function)
After the change, we'll do:
1. Recover log 5, commit the new sst files and set log 5 as recovered
2. Recover log 7, commit the new sst files and set log 7 as recovered
3. Recover log 8, commit the new sst files and set log 8 as recovered
The added (small) benefit is that if we fail after (2), the new recovery will only have to recover log 8. In previous case, we'll have to restart the recovery from the beginning. The bigger benefit will be to enable easier integration of multiple column families in Recovery code path.
[1] I'm happy to dicuss this decison, but I believe this is the cleanest way to go. It also makes backward compatibility much easier. We don't have a requirement of adding multiple column families atomically.
Test Plan: make check
Reviewers: dhruba, haobo, kailiu, sdong
Reviewed By: kailiu
CC: leveldb
Differential Revision: https://reviews.facebook.net/D15237
2014-01-22 19:45:26 +01:00
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
// Open the log file
|
2013-10-01 23:46:52 +02:00
|
|
|
std::string fname = LogFileName(options_.wal_dir, log_number);
|
2013-01-20 11:07:13 +01:00
|
|
|
unique_ptr<SequentialFile> file;
|
2013-03-15 01:00:04 +01:00
|
|
|
Status status = env_->NewSequentialFile(fname, &file, storage_options_);
|
2011-03-18 23:37:00 +01:00
|
|
|
if (!status.ok()) {
|
|
|
|
MaybeIgnoreError(&status);
|
|
|
|
return status;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Create the log reader.
|
|
|
|
LogReporter reporter;
|
|
|
|
reporter.env = env_;
|
2013-01-20 11:07:13 +01:00
|
|
|
reporter.info_log = options_.info_log.get();
|
2011-03-18 23:37:00 +01:00
|
|
|
reporter.fname = fname.c_str();
|
2013-05-21 20:53:33 +02:00
|
|
|
reporter.status = (options_.paranoid_checks &&
|
|
|
|
!options_.skip_log_error_on_recovery ? &status : nullptr);
|
2011-03-18 23:37:00 +01:00
|
|
|
// We intentially make log::Reader do checksumming even if
|
|
|
|
// paranoid_checks==false so that corruptions cause entire commits
|
|
|
|
// to be skipped instead of propagating bad information (like overly
|
|
|
|
// large sequence numbers).
|
2013-01-20 11:07:13 +01:00
|
|
|
log::Reader reader(std::move(file), &reporter, true/*checksum*/,
|
2011-05-21 04:17:43 +02:00
|
|
|
0/*initial_offset*/);
|
2013-11-13 06:02:03 +01:00
|
|
|
Log(options_.info_log, "Recovering log #%lu",
|
|
|
|
(unsigned long) log_number);
|
2011-03-18 23:37:00 +01:00
|
|
|
|
|
|
|
// Read all the records and add to a memtable
|
|
|
|
std::string scratch;
|
|
|
|
Slice record;
|
|
|
|
WriteBatch batch;
|
Refactor Recover() code
Summary:
This diff does two things:
* Rethinks how we call Recover() with read_only option. Before, we call it with pointer to memtable where we'd like to apply those changes to. This memtable is set in db_impl_readonly.cc and it's actually DBImpl::mem_. Why don't we just apply updates to mem_ right away? It seems more intuitive.
* Changes when we apply updates to manifest. Before, the process is to recover all the logs, flush it to sst files and then do one giant commit that atomically adds all recovered sst files and sets the next log number. This works good enough, but causes some small troubles for my column family approach, since I can't have one VersionEdit apply to more than single column family[1]. The change here is to commit the files recovered from logs right away. Here is the state of the world before the change:
1. Recover log 5, add new sst files to edit
2. Recover log 7, add new sst files to edit
3. Recover log 8, add new sst files to edit
4. Commit all added sst files to manifest and mark log files 5, 7 and 8 as recoverd (via SetLogNumber(9) function)
After the change, we'll do:
1. Recover log 5, commit the new sst files and set log 5 as recovered
2. Recover log 7, commit the new sst files and set log 7 as recovered
3. Recover log 8, commit the new sst files and set log 8 as recovered
The added (small) benefit is that if we fail after (2), the new recovery will only have to recover log 8. In previous case, we'll have to restart the recovery from the beginning. The bigger benefit will be to enable easier integration of multiple column families in Recovery code path.
[1] I'm happy to dicuss this decison, but I believe this is the cleanest way to go. It also makes backward compatibility much easier. We don't have a requirement of adding multiple column families atomically.
Test Plan: make check
Reviewers: dhruba, haobo, kailiu, sdong
Reviewed By: kailiu
CC: leveldb
Differential Revision: https://reviews.facebook.net/D15237
2014-01-22 19:45:26 +01:00
|
|
|
while (reader.ReadRecord(&record, &scratch)) {
|
2011-03-18 23:37:00 +01:00
|
|
|
if (record.size() < 12) {
|
|
|
|
reporter.Corruption(
|
|
|
|
record.size(), Status::Corruption("log record too small"));
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
WriteBatchInternal::SetContents(&batch, record);
|
|
|
|
|
2014-01-28 20:05:04 +01:00
|
|
|
// filter out all the column families that have already
|
|
|
|
// flushed memtables with log_number
|
|
|
|
column_family_memtables_->SetLogNumber(log_number);
|
|
|
|
status = WriteBatchInternal::InsertInto(
|
|
|
|
&batch, column_family_memtables_.get(), &options_);
|
|
|
|
column_family_memtables_->SetLogNumber(0);
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
MaybeIgnoreError(&status);
|
|
|
|
if (!status.ok()) {
|
Refactor Recover() code
Summary:
This diff does two things:
* Rethinks how we call Recover() with read_only option. Before, we call it with pointer to memtable where we'd like to apply those changes to. This memtable is set in db_impl_readonly.cc and it's actually DBImpl::mem_. Why don't we just apply updates to mem_ right away? It seems more intuitive.
* Changes when we apply updates to manifest. Before, the process is to recover all the logs, flush it to sst files and then do one giant commit that atomically adds all recovered sst files and sets the next log number. This works good enough, but causes some small troubles for my column family approach, since I can't have one VersionEdit apply to more than single column family[1]. The change here is to commit the files recovered from logs right away. Here is the state of the world before the change:
1. Recover log 5, add new sst files to edit
2. Recover log 7, add new sst files to edit
3. Recover log 8, add new sst files to edit
4. Commit all added sst files to manifest and mark log files 5, 7 and 8 as recoverd (via SetLogNumber(9) function)
After the change, we'll do:
1. Recover log 5, commit the new sst files and set log 5 as recovered
2. Recover log 7, commit the new sst files and set log 7 as recovered
3. Recover log 8, commit the new sst files and set log 8 as recovered
The added (small) benefit is that if we fail after (2), the new recovery will only have to recover log 8. In previous case, we'll have to restart the recovery from the beginning. The bigger benefit will be to enable easier integration of multiple column families in Recovery code path.
[1] I'm happy to dicuss this decison, but I believe this is the cleanest way to go. It also makes backward compatibility much easier. We don't have a requirement of adding multiple column families atomically.
Test Plan: make check
Reviewers: dhruba, haobo, kailiu, sdong
Reviewed By: kailiu
CC: leveldb
Differential Revision: https://reviews.facebook.net/D15237
2014-01-22 19:45:26 +01:00
|
|
|
return status;
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
const SequenceNumber last_seq =
|
|
|
|
WriteBatchInternal::Sequence(&batch) +
|
|
|
|
WriteBatchInternal::Count(&batch) - 1;
|
|
|
|
if (last_seq > *max_sequence) {
|
|
|
|
*max_sequence = last_seq;
|
|
|
|
}
|
|
|
|
|
2014-01-28 20:05:04 +01:00
|
|
|
if (!read_only) {
|
|
|
|
for (auto cfd : *versions_->GetColumnFamilySet()) {
|
2014-01-29 22:28:50 +01:00
|
|
|
if (cfd->mem()->ApproximateMemoryUsage() >
|
|
|
|
cfd->options()->write_buffer_size) {
|
|
|
|
auto iter = version_edits.find(cfd->GetID());
|
2014-01-28 20:05:04 +01:00
|
|
|
assert(iter != version_edits.end());
|
|
|
|
VersionEdit* edit = &iter->second;
|
2014-01-29 22:28:50 +01:00
|
|
|
status = WriteLevel0TableForRecovery(cfd->mem(), edit);
|
2014-01-28 20:05:04 +01:00
|
|
|
// we still want to clear the memtable, even if the recovery failed
|
|
|
|
cfd->CreateNewMemtable();
|
|
|
|
if (!status.ok()) {
|
|
|
|
// Reflect errors immediately so that conditions like full
|
|
|
|
// file-systems cause the DB::Open() to fail.
|
|
|
|
return status;
|
|
|
|
}
|
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-01-28 20:05:04 +01:00
|
|
|
if (!read_only) {
|
|
|
|
for (auto cfd : *versions_->GetColumnFamilySet()) {
|
2014-01-29 22:28:50 +01:00
|
|
|
auto iter = version_edits.find(cfd->GetID());
|
2014-01-28 20:05:04 +01:00
|
|
|
assert(iter != version_edits.end());
|
|
|
|
VersionEdit* edit = &iter->second;
|
|
|
|
|
|
|
|
// flush the final memtable
|
2014-01-29 22:28:50 +01:00
|
|
|
status = WriteLevel0TableForRecovery(cfd->mem(), edit);
|
2014-01-28 20:05:04 +01:00
|
|
|
// we still want to clear the memtable, even if the recovery failed
|
|
|
|
cfd->CreateNewMemtable();
|
|
|
|
if (!status.ok()) {
|
|
|
|
return status;
|
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2014-01-28 20:05:04 +01:00
|
|
|
// write MANIFEST with update
|
|
|
|
// writing log number in the manifest means that any log file
|
|
|
|
// with number strongly less than (log_number + 1) is already
|
|
|
|
// recovered and should be ignored on next reincarnation.
|
|
|
|
// Since we already recovered log_number, we want all logs
|
|
|
|
// with numbers `<= log_number` (includes this one) to be ignored
|
|
|
|
edit->SetLogNumber(log_number + 1);
|
|
|
|
status = versions_->LogAndApply(cfd, edit, &mutex_);
|
|
|
|
if (!status.ok()) {
|
|
|
|
return status;
|
|
|
|
}
|
|
|
|
}
|
2013-11-25 20:55:36 +01:00
|
|
|
}
|
Refactor Recover() code
Summary:
This diff does two things:
* Rethinks how we call Recover() with read_only option. Before, we call it with pointer to memtable where we'd like to apply those changes to. This memtable is set in db_impl_readonly.cc and it's actually DBImpl::mem_. Why don't we just apply updates to mem_ right away? It seems more intuitive.
* Changes when we apply updates to manifest. Before, the process is to recover all the logs, flush it to sst files and then do one giant commit that atomically adds all recovered sst files and sets the next log number. This works good enough, but causes some small troubles for my column family approach, since I can't have one VersionEdit apply to more than single column family[1]. The change here is to commit the files recovered from logs right away. Here is the state of the world before the change:
1. Recover log 5, add new sst files to edit
2. Recover log 7, add new sst files to edit
3. Recover log 8, add new sst files to edit
4. Commit all added sst files to manifest and mark log files 5, 7 and 8 as recoverd (via SetLogNumber(9) function)
After the change, we'll do:
1. Recover log 5, commit the new sst files and set log 5 as recovered
2. Recover log 7, commit the new sst files and set log 7 as recovered
3. Recover log 8, commit the new sst files and set log 8 as recovered
The added (small) benefit is that if we fail after (2), the new recovery will only have to recover log 8. In previous case, we'll have to restart the recovery from the beginning. The bigger benefit will be to enable easier integration of multiple column families in Recovery code path.
[1] I'm happy to dicuss this decison, but I believe this is the cleanest way to go. It also makes backward compatibility much easier. We don't have a requirement of adding multiple column families atomically.
Test Plan: make check
Reviewers: dhruba, haobo, kailiu, sdong
Reviewed By: kailiu
CC: leveldb
Differential Revision: https://reviews.facebook.net/D15237
2014-01-22 19:45:26 +01:00
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
return status;
|
|
|
|
}
|
|
|
|
|
2012-10-19 23:00:53 +02:00
|
|
|
Status DBImpl::WriteLevel0TableForRecovery(MemTable* mem, VersionEdit* edit) {
|
2011-03-18 23:37:00 +01:00
|
|
|
mutex_.AssertHeld();
|
2011-04-12 21:38:58 +02:00
|
|
|
const uint64_t start_micros = env_->NowMicros();
|
2011-03-18 23:37:00 +01:00
|
|
|
FileMetaData meta;
|
|
|
|
meta.number = versions_->NewFileNumber();
|
|
|
|
pending_outputs_.insert(meta.number);
|
|
|
|
Iterator* iter = mem->NewIterator();
|
2013-02-28 23:09:30 +01:00
|
|
|
const SequenceNumber newest_snapshot = snapshots_.GetNewest();
|
|
|
|
const SequenceNumber earliest_seqno_in_memtable =
|
|
|
|
mem->GetFirstSequenceNumber();
|
2013-11-13 06:02:03 +01:00
|
|
|
Log(options_.info_log, "Level-0 table #%lu: started",
|
|
|
|
(unsigned long) meta.number);
|
2011-04-12 21:38:58 +02:00
|
|
|
|
|
|
|
Status s;
|
|
|
|
{
|
|
|
|
mutex_.Unlock();
|
2013-03-15 01:00:04 +01:00
|
|
|
s = BuildTable(dbname_, env_, options_, storage_options_,
|
|
|
|
table_cache_.get(), iter, &meta,
|
2013-02-28 23:09:30 +01:00
|
|
|
user_comparator(), newest_snapshot,
|
2013-12-19 19:02:53 +01:00
|
|
|
earliest_seqno_in_memtable,
|
|
|
|
GetCompressionFlush(options_));
|
2013-11-07 20:31:56 +01:00
|
|
|
LogFlush(options_.info_log);
|
2011-04-12 21:38:58 +02:00
|
|
|
mutex_.Lock();
|
|
|
|
}
|
|
|
|
|
2013-11-13 06:02:03 +01:00
|
|
|
Log(options_.info_log, "Level-0 table #%lu: %lu bytes %s",
|
|
|
|
(unsigned long) meta.number,
|
|
|
|
(unsigned long) meta.file_size,
|
2011-03-18 23:37:00 +01:00
|
|
|
s.ToString().c_str());
|
|
|
|
delete iter;
|
2011-04-12 21:38:58 +02:00
|
|
|
|
2012-10-19 23:00:53 +02:00
|
|
|
pending_outputs_.erase(meta.number);
|
2011-06-22 04:36:45 +02:00
|
|
|
|
|
|
|
// Note that if file_size is zero, the file has been deleted and
|
|
|
|
// should not be added to the manifest.
|
|
|
|
int level = 0;
|
|
|
|
if (s.ok() && meta.file_size > 0) {
|
|
|
|
edit->AddFile(level, meta.number, meta.file_size,
|
2013-06-14 07:09:08 +02:00
|
|
|
meta.smallest, meta.largest,
|
|
|
|
meta.smallest_seqno, meta.largest_seqno);
|
2011-06-22 04:36:45 +02:00
|
|
|
}
|
|
|
|
|
2011-04-12 21:38:58 +02:00
|
|
|
CompactionStats stats;
|
|
|
|
stats.micros = env_->NowMicros() - start_micros;
|
|
|
|
stats.bytes_written = meta.file_size;
|
2012-10-23 19:34:09 +02:00
|
|
|
stats.files_out_levelnp1 = 1;
|
2011-06-22 04:36:45 +02:00
|
|
|
stats_[level].Add(stats);
|
Add monitoring for universal compaction and add counters for compaction IO
Summary:
Adds these counters
{ WAL_FILE_SYNCED, "rocksdb.wal.synced" }
number of writes that request a WAL sync
{ WAL_FILE_BYTES, "rocksdb.wal.bytes" },
number of bytes written to the WAL
{ WRITE_DONE_BY_SELF, "rocksdb.write.self" },
number of writes processed by the calling thread
{ WRITE_DONE_BY_OTHER, "rocksdb.write.other" },
number of writes not processed by the calling thread. Instead these were
processed by the current holder of the write lock
{ WRITE_WITH_WAL, "rocksdb.write.wal" },
number of writes that request WAL logging
{ COMPACT_READ_BYTES, "rocksdb.compact.read.bytes" },
number of bytes read during compaction
{ COMPACT_WRITE_BYTES, "rocksdb.compact.write.bytes" },
number of bytes written during compaction
Per-interval stats output was updated with WAL stats and correct stats for universal compaction
including a correct value for write-amplification. It now looks like:
Compactions
Level Files Size(MB) Score Time(sec) Read(MB) Write(MB) Rn(MB) Rnp1(MB) Wnew(MB) RW-Amplify Read(MB/s) Write(MB/s) Rn Rnp1 Wnp1 NewW Count Ln-stall Stall-cnt
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
0 7 464 46.4 281 3411 3875 3411 0 3875 2.1 12.1 13.8 621 0 240 240 628 0.0 0
Uptime(secs): 310.8 total, 2.0 interval
Writes cumulative: 9999999 total, 9999999 batches, 1.0 per batch, 1.22 ingest GB
WAL cumulative: 9999999 WAL writes, 9999999 WAL syncs, 1.00 writes per sync, 1.22 GB written
Compaction IO cumulative (GB): 1.22 new, 3.33 read, 3.78 write, 7.12 read+write
Compaction IO cumulative (MB/sec): 4.0 new, 11.0 read, 12.5 write, 23.4 read+write
Amplification cumulative: 4.1 write, 6.8 compaction
Writes interval: 100000 total, 100000 batches, 1.0 per batch, 12.5 ingest MB
WAL interval: 100000 WAL writes, 100000 WAL syncs, 1.00 writes per sync, 0.01 MB written
Compaction IO interval (MB): 12.49 new, 14.98 read, 21.50 write, 36.48 read+write
Compaction IO interval (MB/sec): 6.4 new, 7.6 read, 11.0 write, 18.6 read+write
Amplification interval: 101.7 write, 102.9 compaction
Stalls(secs): 142.924 level0_slowdown, 0.000 level0_numfiles, 0.805 memtable_compaction, 0.000 leveln_slowdown
Stalls(count): 132461 level0_slowdown, 0 level0_numfiles, 3 memtable_compaction, 0 leveln_slowdown
Task ID: #3329644, #3301695
Blame Rev:
Test Plan:
Revert Plan:
Database Impact:
Memcache Impact:
Other Notes:
EImportant:
- begin *PUBLIC* platform impact section -
Bugzilla: #
- end platform impact -
Reviewers: dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14583
2013-12-09 22:43:34 +01:00
|
|
|
RecordTick(options_.statistics.get(), COMPACT_WRITE_BYTES, meta.file_size);
|
2011-03-18 23:37:00 +01:00
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
2012-10-19 23:00:53 +02:00
|
|
|
|
2013-06-11 23:23:58 +02:00
|
|
|
Status DBImpl::WriteLevel0Table(std::vector<MemTable*> &mems, VersionEdit* edit,
|
2012-10-19 23:00:53 +02:00
|
|
|
uint64_t* filenumber) {
|
2011-03-18 23:37:00 +01:00
|
|
|
mutex_.AssertHeld();
|
2012-10-19 23:00:53 +02:00
|
|
|
const uint64_t start_micros = env_->NowMicros();
|
|
|
|
FileMetaData meta;
|
|
|
|
meta.number = versions_->NewFileNumber();
|
|
|
|
*filenumber = meta.number;
|
|
|
|
pending_outputs_.insert(meta.number);
|
2013-06-11 23:23:58 +02:00
|
|
|
|
2013-02-28 23:09:30 +01:00
|
|
|
const SequenceNumber newest_snapshot = snapshots_.GetNewest();
|
|
|
|
const SequenceNumber earliest_seqno_in_memtable =
|
2013-06-11 23:23:58 +02:00
|
|
|
mems[0]->GetFirstSequenceNumber();
|
2014-01-29 22:28:50 +01:00
|
|
|
Version* base = default_cfd_->current();
|
2013-06-11 23:23:58 +02:00
|
|
|
base->Ref(); // it is likely that we do not need this reference
|
2012-10-19 23:00:53 +02:00
|
|
|
Status s;
|
|
|
|
{
|
|
|
|
mutex_.Unlock();
|
2013-12-11 05:03:27 +01:00
|
|
|
std::vector<Iterator*> list;
|
|
|
|
for (MemTable* m : mems) {
|
|
|
|
Log(options_.info_log,
|
|
|
|
"Flushing memtable with log file: %lu\n",
|
|
|
|
(unsigned long)m->GetLogNumber());
|
|
|
|
list.push_back(m->NewIterator());
|
|
|
|
}
|
|
|
|
Iterator* iter = NewMergingIterator(&internal_comparator_, &list[0],
|
|
|
|
list.size());
|
|
|
|
Log(options_.info_log,
|
|
|
|
"Level-0 flush table #%lu: started",
|
|
|
|
(unsigned long)meta.number);
|
2013-12-19 19:02:53 +01:00
|
|
|
|
2013-03-15 01:00:04 +01:00
|
|
|
s = BuildTable(dbname_, env_, options_, storage_options_,
|
|
|
|
table_cache_.get(), iter, &meta,
|
2013-02-28 23:09:30 +01:00
|
|
|
user_comparator(), newest_snapshot,
|
2013-12-19 19:02:53 +01:00
|
|
|
earliest_seqno_in_memtable, GetCompressionFlush(options_));
|
2013-11-07 20:31:56 +01:00
|
|
|
LogFlush(options_.info_log);
|
2013-12-11 05:03:27 +01:00
|
|
|
delete iter;
|
|
|
|
Log(options_.info_log, "Level-0 flush table #%lu: %lu bytes %s",
|
|
|
|
(unsigned long) meta.number,
|
|
|
|
(unsigned long) meta.file_size,
|
|
|
|
s.ToString().c_str());
|
2014-01-27 20:02:21 +01:00
|
|
|
if (!options_.disableDataSync) {
|
|
|
|
db_directory_->Fsync();
|
|
|
|
}
|
2012-10-19 23:00:53 +02:00
|
|
|
mutex_.Lock();
|
|
|
|
}
|
2011-06-22 04:36:45 +02:00
|
|
|
base->Unref();
|
|
|
|
|
2012-10-19 23:00:53 +02:00
|
|
|
|
|
|
|
// re-acquire the most current version
|
2014-01-29 22:28:50 +01:00
|
|
|
base = default_cfd_->current();
|
2012-10-19 23:00:53 +02:00
|
|
|
|
|
|
|
// There could be multiple threads writing to its own level-0 file.
|
|
|
|
// The pending_outputs cannot be cleared here, otherwise this newly
|
|
|
|
// created file might not be considered as a live-file by another
|
|
|
|
// compaction thread that is concurrently deleting obselete files.
|
|
|
|
// The pending_outputs can be cleared only after the new version is
|
2012-11-29 01:42:36 +01:00
|
|
|
// committed so that other threads can recognize this file as a
|
2012-10-19 23:00:53 +02:00
|
|
|
// valid one.
|
|
|
|
// pending_outputs_.erase(meta.number);
|
|
|
|
|
|
|
|
// Note that if file_size is zero, the file has been deleted and
|
|
|
|
// should not be added to the manifest.
|
|
|
|
int level = 0;
|
|
|
|
if (s.ok() && meta.file_size > 0) {
|
|
|
|
const Slice min_user_key = meta.smallest.user_key();
|
|
|
|
const Slice max_user_key = meta.largest.user_key();
|
|
|
|
// if we have more than 1 background thread, then we cannot
|
|
|
|
// insert files directly into higher levels because some other
|
|
|
|
// threads could be concurrently producing compacted files for
|
|
|
|
// that key range.
|
2013-06-14 07:09:08 +02:00
|
|
|
if (base != nullptr && options_.max_background_compactions <= 1 &&
|
2013-07-04 00:32:49 +02:00
|
|
|
options_.compaction_style == kCompactionStyleLevel) {
|
2012-10-19 23:00:53 +02:00
|
|
|
level = base->PickLevelForMemTableOutput(min_user_key, max_user_key);
|
|
|
|
}
|
|
|
|
edit->AddFile(level, meta.number, meta.file_size,
|
2013-06-14 07:09:08 +02:00
|
|
|
meta.smallest, meta.largest,
|
|
|
|
meta.smallest_seqno, meta.largest_seqno);
|
2012-10-19 23:00:53 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
CompactionStats stats;
|
|
|
|
stats.micros = env_->NowMicros() - start_micros;
|
|
|
|
stats.bytes_written = meta.file_size;
|
|
|
|
stats_[level].Add(stats);
|
Add monitoring for universal compaction and add counters for compaction IO
Summary:
Adds these counters
{ WAL_FILE_SYNCED, "rocksdb.wal.synced" }
number of writes that request a WAL sync
{ WAL_FILE_BYTES, "rocksdb.wal.bytes" },
number of bytes written to the WAL
{ WRITE_DONE_BY_SELF, "rocksdb.write.self" },
number of writes processed by the calling thread
{ WRITE_DONE_BY_OTHER, "rocksdb.write.other" },
number of writes not processed by the calling thread. Instead these were
processed by the current holder of the write lock
{ WRITE_WITH_WAL, "rocksdb.write.wal" },
number of writes that request WAL logging
{ COMPACT_READ_BYTES, "rocksdb.compact.read.bytes" },
number of bytes read during compaction
{ COMPACT_WRITE_BYTES, "rocksdb.compact.write.bytes" },
number of bytes written during compaction
Per-interval stats output was updated with WAL stats and correct stats for universal compaction
including a correct value for write-amplification. It now looks like:
Compactions
Level Files Size(MB) Score Time(sec) Read(MB) Write(MB) Rn(MB) Rnp1(MB) Wnew(MB) RW-Amplify Read(MB/s) Write(MB/s) Rn Rnp1 Wnp1 NewW Count Ln-stall Stall-cnt
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
0 7 464 46.4 281 3411 3875 3411 0 3875 2.1 12.1 13.8 621 0 240 240 628 0.0 0
Uptime(secs): 310.8 total, 2.0 interval
Writes cumulative: 9999999 total, 9999999 batches, 1.0 per batch, 1.22 ingest GB
WAL cumulative: 9999999 WAL writes, 9999999 WAL syncs, 1.00 writes per sync, 1.22 GB written
Compaction IO cumulative (GB): 1.22 new, 3.33 read, 3.78 write, 7.12 read+write
Compaction IO cumulative (MB/sec): 4.0 new, 11.0 read, 12.5 write, 23.4 read+write
Amplification cumulative: 4.1 write, 6.8 compaction
Writes interval: 100000 total, 100000 batches, 1.0 per batch, 12.5 ingest MB
WAL interval: 100000 WAL writes, 100000 WAL syncs, 1.00 writes per sync, 0.01 MB written
Compaction IO interval (MB): 12.49 new, 14.98 read, 21.50 write, 36.48 read+write
Compaction IO interval (MB/sec): 6.4 new, 7.6 read, 11.0 write, 18.6 read+write
Amplification interval: 101.7 write, 102.9 compaction
Stalls(secs): 142.924 level0_slowdown, 0.000 level0_numfiles, 0.805 memtable_compaction, 0.000 leveln_slowdown
Stalls(count): 132461 level0_slowdown, 0 level0_numfiles, 3 memtable_compaction, 0 leveln_slowdown
Task ID: #3329644, #3301695
Blame Rev:
Test Plan:
Revert Plan:
Database Impact:
Memcache Impact:
Other Notes:
EImportant:
- begin *PUBLIC* platform impact section -
Bugzilla: #
- end platform impact -
Reviewers: dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14583
2013-12-09 22:43:34 +01:00
|
|
|
RecordTick(options_.statistics.get(), COMPACT_WRITE_BYTES, meta.file_size);
|
2012-10-19 23:00:53 +02:00
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
2013-11-09 00:23:46 +01:00
|
|
|
Status DBImpl::FlushMemTableToOutputFile(bool* madeProgress,
|
|
|
|
DeletionState& deletion_state) {
|
2012-10-19 23:00:53 +02:00
|
|
|
mutex_.AssertHeld();
|
2014-01-29 22:28:50 +01:00
|
|
|
assert(default_cfd_->imm()->size() != 0);
|
2012-10-19 23:00:53 +02:00
|
|
|
|
2014-01-29 22:28:50 +01:00
|
|
|
if (!default_cfd_->imm()->IsFlushPending()) {
|
2013-10-15 00:12:15 +02:00
|
|
|
Log(options_.info_log, "FlushMemTableToOutputFile already in progress");
|
|
|
|
Status s = Status::IOError("FlushMemTableToOutputFile already in progress");
|
2012-10-19 23:00:53 +02:00
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Save the contents of the earliest memtable as a new Table
|
|
|
|
uint64_t file_number;
|
2013-06-11 23:23:58 +02:00
|
|
|
std::vector<MemTable*> mems;
|
2014-01-29 22:28:50 +01:00
|
|
|
default_cfd_->imm()->PickMemtablesToFlush(&mems);
|
2013-06-11 23:23:58 +02:00
|
|
|
if (mems.empty()) {
|
2012-10-19 23:00:53 +02:00
|
|
|
Log(options_.info_log, "Nothing in memstore to flush");
|
|
|
|
Status s = Status::IOError("Nothing in memstore to flush");
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
|
|
|
// record the logfile_number_ before we release the mutex
|
2013-10-25 00:58:00 +02:00
|
|
|
// entries mems are (implicitly) sorted in ascending order by their created
|
|
|
|
// time. We will use the first memtable's `edit` to keep the meta info for
|
|
|
|
// this flush.
|
2013-06-11 23:23:58 +02:00
|
|
|
MemTable* m = mems[0];
|
2012-10-19 23:00:53 +02:00
|
|
|
VersionEdit* edit = m->GetEdits();
|
|
|
|
edit->SetPrevLogNumber(0);
|
2013-10-25 00:58:00 +02:00
|
|
|
// SetLogNumber(log_num) indicates logs with number smaller than log_num
|
|
|
|
// will no longer be picked up for recovery.
|
|
|
|
edit->SetLogNumber(
|
|
|
|
mems.back()->GetNextLogNumber()
|
|
|
|
);
|
|
|
|
|
|
|
|
std::vector<uint64_t> logs_to_delete;
|
|
|
|
for (auto mem : mems) {
|
|
|
|
logs_to_delete.push_back(mem->GetLogNumber());
|
|
|
|
}
|
2012-10-19 23:00:53 +02:00
|
|
|
|
2013-07-16 20:56:46 +02:00
|
|
|
// This will release and re-acquire the mutex.
|
2013-06-11 23:23:58 +02:00
|
|
|
Status s = WriteLevel0Table(mems, edit, &file_number);
|
2012-10-19 23:00:53 +02:00
|
|
|
|
2011-06-22 04:36:45 +02:00
|
|
|
if (s.ok() && shutting_down_.Acquire_Load()) {
|
2013-04-16 00:27:15 +02:00
|
|
|
s = Status::IOError(
|
|
|
|
"Database shutdown started during memtable compaction"
|
|
|
|
);
|
2011-06-22 04:36:45 +02:00
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2011-04-12 21:38:58 +02:00
|
|
|
// Replace immutable memtable with the generated Table
|
2014-01-29 22:28:50 +01:00
|
|
|
s = default_cfd_->imm()->InstallMemtableFlushResults(
|
2014-01-27 22:55:47 +01:00
|
|
|
default_cfd_, mems, versions_.get(), s, &mutex_, options_.info_log.get(),
|
|
|
|
file_number, pending_outputs_, &deletion_state.memtables_to_free,
|
|
|
|
db_directory_.get());
|
2011-03-18 23:37:00 +01:00
|
|
|
|
|
|
|
if (s.ok()) {
|
2014-01-24 23:30:28 +01:00
|
|
|
InstallSuperVersion(default_cfd_, deletion_state);
|
2012-10-19 23:00:53 +02:00
|
|
|
if (madeProgress) {
|
|
|
|
*madeProgress = 1;
|
|
|
|
}
|
2013-07-16 20:56:46 +02:00
|
|
|
|
2012-08-15 00:20:36 +02:00
|
|
|
MaybeScheduleLogDBDeployStats();
|
2013-11-09 00:23:46 +01:00
|
|
|
|
2014-01-02 12:33:42 +01:00
|
|
|
if (disable_delete_obsolete_files_ == 0) {
|
2013-11-09 00:23:46 +01:00
|
|
|
// add to deletion state
|
2013-11-12 20:53:26 +01:00
|
|
|
deletion_state.log_delete_files.insert(
|
|
|
|
deletion_state.log_delete_files.end(),
|
|
|
|
logs_to_delete.begin(),
|
|
|
|
logs_to_delete.end());
|
2013-07-16 20:56:46 +02:00
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
2014-01-24 18:27:29 +01:00
|
|
|
Status DBImpl::CompactRange(const ColumnFamilyHandle& column_family,
|
|
|
|
const Slice* begin, const Slice* end,
|
|
|
|
bool reduce_level, int target_level) {
|
2014-01-22 21:46:24 +01:00
|
|
|
Status s = FlushMemTable(FlushOptions());
|
|
|
|
if (!s.ok()) {
|
|
|
|
LogFlush(options_.info_log);
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
2011-10-06 01:30:28 +02:00
|
|
|
int max_level_with_files = 1;
|
|
|
|
{
|
|
|
|
MutexLock l(&mutex_);
|
2014-01-29 22:28:50 +01:00
|
|
|
Version* base = default_cfd_->current();
|
2012-06-23 04:30:03 +02:00
|
|
|
for (int level = 1; level < NumberLevels(); level++) {
|
2011-10-06 01:30:28 +02:00
|
|
|
if (base->OverlapInLevel(level, begin, end)) {
|
|
|
|
max_level_with_files = level;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2014-01-15 01:19:09 +01:00
|
|
|
for (int level = 0; level <= max_level_with_files; level++) {
|
|
|
|
// in case the compaction is unversal or if we're compacting the
|
|
|
|
// bottom-most level, the output level will be the same as input one
|
|
|
|
if (options_.compaction_style == kCompactionStyleUniversal ||
|
|
|
|
level == max_level_with_files) {
|
2014-01-22 21:46:24 +01:00
|
|
|
s = RunManualCompaction(level, level, begin, end);
|
2014-01-15 01:19:09 +01:00
|
|
|
} else {
|
2014-01-22 21:46:24 +01:00
|
|
|
s = RunManualCompaction(level, level + 1, begin, end);
|
|
|
|
}
|
|
|
|
if (!s.ok()) {
|
|
|
|
LogFlush(options_.info_log);
|
|
|
|
return s;
|
2014-01-15 01:19:09 +01:00
|
|
|
}
|
2011-10-06 01:30:28 +02:00
|
|
|
}
|
2013-06-30 08:21:36 +02:00
|
|
|
|
|
|
|
if (reduce_level) {
|
2014-01-22 21:46:24 +01:00
|
|
|
s = ReFitLevel(max_level_with_files, target_level);
|
2013-06-30 08:21:36 +02:00
|
|
|
}
|
2013-11-07 20:31:56 +01:00
|
|
|
LogFlush(options_.info_log);
|
2014-01-22 21:46:24 +01:00
|
|
|
|
|
|
|
return s;
|
2013-06-30 08:21:36 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// return the same level if it cannot be moved
|
|
|
|
int DBImpl::FindMinimumEmptyLevelFitting(int level) {
|
|
|
|
mutex_.AssertHeld();
|
2014-01-29 22:28:50 +01:00
|
|
|
Version* current = default_cfd_->current();
|
2013-06-30 08:21:36 +02:00
|
|
|
int minimum_level = level;
|
2013-07-22 21:19:46 +02:00
|
|
|
for (int i = level - 1; i > 0; --i) {
|
2013-06-30 08:21:36 +02:00
|
|
|
// stop if level i is not empty
|
2014-01-16 01:18:04 +01:00
|
|
|
if (current->NumLevelFiles(i) > 0) break;
|
2013-06-30 08:21:36 +02:00
|
|
|
// stop if level i is too small (cannot fit the level files)
|
2014-01-16 01:18:04 +01:00
|
|
|
if (versions_->MaxBytesForLevel(i) < current->NumLevelBytes(level)) break;
|
2013-06-30 08:21:36 +02:00
|
|
|
|
|
|
|
minimum_level = i;
|
|
|
|
}
|
|
|
|
return minimum_level;
|
|
|
|
}
|
|
|
|
|
2014-01-22 21:46:24 +01:00
|
|
|
Status DBImpl::ReFitLevel(int level, int target_level) {
|
2013-06-30 08:21:36 +02:00
|
|
|
assert(level < NumberLevels());
|
|
|
|
|
2013-12-20 18:57:58 +01:00
|
|
|
SuperVersion* superversion_to_free = nullptr;
|
|
|
|
SuperVersion* new_superversion =
|
|
|
|
new SuperVersion(options_.max_write_buffer_number);
|
|
|
|
|
|
|
|
mutex_.Lock();
|
2013-06-30 08:21:36 +02:00
|
|
|
|
|
|
|
// only allow one thread refitting
|
|
|
|
if (refitting_level_) {
|
2013-12-20 18:57:58 +01:00
|
|
|
mutex_.Unlock();
|
2013-06-30 08:21:36 +02:00
|
|
|
Log(options_.info_log, "ReFitLevel: another thread is refitting");
|
2013-12-20 18:57:58 +01:00
|
|
|
delete new_superversion;
|
2014-01-22 21:46:24 +01:00
|
|
|
return Status::NotSupported("another thread is refitting");
|
2013-06-30 08:21:36 +02:00
|
|
|
}
|
|
|
|
refitting_level_ = true;
|
|
|
|
|
|
|
|
// wait for all background threads to stop
|
|
|
|
bg_work_gate_closed_ = true;
|
2013-09-13 23:38:37 +02:00
|
|
|
while (bg_compaction_scheduled_ > 0 || bg_flush_scheduled_) {
|
2013-06-30 08:21:36 +02:00
|
|
|
Log(options_.info_log,
|
2013-09-13 23:38:37 +02:00
|
|
|
"RefitLevel: waiting for background threads to stop: %d %d",
|
|
|
|
bg_compaction_scheduled_, bg_flush_scheduled_);
|
2013-06-30 08:21:36 +02:00
|
|
|
bg_cv_.Wait();
|
|
|
|
}
|
|
|
|
|
|
|
|
// move to a smaller level
|
2013-09-04 22:13:08 +02:00
|
|
|
int to_level = target_level;
|
|
|
|
if (target_level < 0) {
|
|
|
|
to_level = FindMinimumEmptyLevelFitting(level);
|
|
|
|
}
|
2013-06-30 08:21:36 +02:00
|
|
|
|
|
|
|
assert(to_level <= level);
|
|
|
|
|
2014-01-22 21:46:24 +01:00
|
|
|
Status status;
|
2013-06-30 08:21:36 +02:00
|
|
|
if (to_level < level) {
|
|
|
|
Log(options_.info_log, "Before refitting:\n%s",
|
2014-01-29 22:28:50 +01:00
|
|
|
default_cfd_->current()->DebugString().data());
|
2013-06-30 08:21:36 +02:00
|
|
|
|
2014-01-15 00:27:09 +01:00
|
|
|
VersionEdit edit;
|
2014-01-29 22:28:50 +01:00
|
|
|
for (const auto& f : default_cfd_->current()->files_[level]) {
|
2013-06-30 08:21:36 +02:00
|
|
|
edit.DeleteFile(level, f->number);
|
2013-08-02 19:22:08 +02:00
|
|
|
edit.AddFile(to_level, f->number, f->file_size, f->smallest, f->largest,
|
|
|
|
f->smallest_seqno, f->largest_seqno);
|
2013-06-30 08:21:36 +02:00
|
|
|
}
|
|
|
|
Log(options_.info_log, "Apply version edit:\n%s",
|
|
|
|
edit.DebugString().data());
|
|
|
|
|
2014-01-27 22:55:47 +01:00
|
|
|
status = versions_->LogAndApply(default_cfd_, &edit, &mutex_,
|
|
|
|
db_directory_.get());
|
2014-01-29 22:28:50 +01:00
|
|
|
superversion_to_free = default_cfd_->InstallSuperVersion(new_superversion);
|
2013-12-20 18:57:58 +01:00
|
|
|
new_superversion = nullptr;
|
2013-06-30 08:21:36 +02:00
|
|
|
|
|
|
|
Log(options_.info_log, "LogAndApply: %s\n", status.ToString().data());
|
|
|
|
|
|
|
|
if (status.ok()) {
|
|
|
|
Log(options_.info_log, "After refitting:\n%s",
|
2014-01-29 22:28:50 +01:00
|
|
|
default_cfd_->current()->DebugString().data());
|
2013-06-30 08:21:36 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
refitting_level_ = false;
|
|
|
|
bg_work_gate_closed_ = false;
|
2013-12-20 18:57:58 +01:00
|
|
|
|
|
|
|
mutex_.Unlock();
|
|
|
|
delete superversion_to_free;
|
|
|
|
delete new_superversion;
|
2014-01-22 21:46:24 +01:00
|
|
|
return status;
|
2011-10-06 01:30:28 +02:00
|
|
|
}
|
|
|
|
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 20:14:09 +01:00
|
|
|
int DBImpl::NumberLevels(const ColumnFamilyHandle& column_family) {
|
2012-08-15 00:20:36 +02:00
|
|
|
return options_.num_levels;
|
2012-06-23 04:30:03 +02:00
|
|
|
}
|
|
|
|
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 20:14:09 +01:00
|
|
|
int DBImpl::MaxMemCompactionLevel(const ColumnFamilyHandle& column_family) {
|
2012-08-15 00:20:36 +02:00
|
|
|
return options_.max_mem_compaction_level;
|
2012-06-23 04:30:03 +02:00
|
|
|
}
|
|
|
|
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 20:14:09 +01:00
|
|
|
int DBImpl::Level0StopWriteTrigger(const ColumnFamilyHandle& column_family) {
|
2012-08-15 00:20:36 +02:00
|
|
|
return options_.level0_stop_writes_trigger;
|
2012-06-23 04:30:03 +02:00
|
|
|
}
|
|
|
|
|
2014-01-17 06:56:26 +01:00
|
|
|
uint64_t DBImpl::CurrentVersionNumber() const {
|
2014-01-29 22:28:50 +01:00
|
|
|
return default_cfd_->GetSuperVersionNumber();
|
2014-01-17 06:56:26 +01:00
|
|
|
}
|
|
|
|
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 20:14:09 +01:00
|
|
|
Status DBImpl::Flush(const FlushOptions& options,
|
|
|
|
const ColumnFamilyHandle& column_family) {
|
2012-07-06 20:42:09 +02:00
|
|
|
Status status = FlushMemTable(options);
|
|
|
|
return status;
|
|
|
|
}
|
|
|
|
|
2013-10-25 04:09:02 +02:00
|
|
|
SequenceNumber DBImpl::GetLatestSequenceNumber() const {
|
2012-12-11 00:37:00 +01:00
|
|
|
return versions_->LastSequence();
|
|
|
|
}
|
|
|
|
|
2012-11-30 02:28:37 +01:00
|
|
|
Status DBImpl::GetUpdatesSince(SequenceNumber seq,
|
2013-01-24 19:54:26 +01:00
|
|
|
unique_ptr<TransactionLogIterator>* iter) {
|
2012-11-30 02:28:37 +01:00
|
|
|
|
2013-11-22 23:14:05 +01:00
|
|
|
RecordTick(options_.statistics.get(), GET_UPDATES_SINCE_CALLS);
|
2013-10-25 04:09:02 +02:00
|
|
|
if (seq > versions_->LastSequence()) {
|
2013-08-06 21:54:37 +02:00
|
|
|
return Status::IOError("Requested sequence not yet written in the db");
|
|
|
|
}
|
|
|
|
// Get all sorted Wal Files.
|
2012-11-30 02:28:37 +01:00
|
|
|
// Do binary search and open files and find the seq number.
|
|
|
|
|
2013-08-06 21:54:37 +02:00
|
|
|
std::unique_ptr<VectorLogPtr> wal_files(new VectorLogPtr);
|
|
|
|
Status s = GetSortedWalFiles(*wal_files);
|
2012-11-30 02:28:37 +01:00
|
|
|
if (!s.ok()) {
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
2013-08-06 21:54:37 +02:00
|
|
|
s = RetainProbableWalFiles(*wal_files, seq);
|
2012-12-10 20:02:07 +01:00
|
|
|
if (!s.ok()) {
|
|
|
|
return s;
|
2012-11-30 02:28:37 +01:00
|
|
|
}
|
2013-01-24 19:54:26 +01:00
|
|
|
iter->reset(
|
2013-10-01 23:46:52 +02:00
|
|
|
new TransactionLogIteratorImpl(options_.wal_dir,
|
2013-03-04 19:44:04 +01:00
|
|
|
&options_,
|
2013-03-15 01:00:04 +01:00
|
|
|
storage_options_,
|
2013-03-04 19:44:04 +01:00
|
|
|
seq,
|
2013-08-06 21:54:37 +02:00
|
|
|
std::move(wal_files),
|
2013-10-25 04:09:02 +02:00
|
|
|
this));
|
2013-10-14 00:28:24 +02:00
|
|
|
return (*iter)->status();
|
2012-11-30 02:28:37 +01:00
|
|
|
}
|
|
|
|
|
2013-08-06 21:54:37 +02:00
|
|
|
Status DBImpl::RetainProbableWalFiles(VectorLogPtr& all_logs,
|
|
|
|
const SequenceNumber target) {
|
2012-12-10 20:02:07 +01:00
|
|
|
long start = 0; // signed to avoid overflow when target is < first file.
|
2013-08-06 21:54:37 +02:00
|
|
|
long end = static_cast<long>(all_logs.size()) - 1;
|
2012-11-30 02:28:37 +01:00
|
|
|
// Binary Search. avoid opening all files.
|
2012-12-10 20:02:07 +01:00
|
|
|
while (end >= start) {
|
|
|
|
long mid = start + (end - start) / 2; // Avoid overflow.
|
2013-08-06 21:54:37 +02:00
|
|
|
SequenceNumber current_seq_num = all_logs.at(mid)->StartSequence();
|
|
|
|
if (current_seq_num == target) {
|
2012-11-30 02:28:37 +01:00
|
|
|
end = mid;
|
2012-12-10 20:02:07 +01:00
|
|
|
break;
|
2013-08-06 21:54:37 +02:00
|
|
|
} else if (current_seq_num < target) {
|
2012-12-10 20:02:07 +01:00
|
|
|
start = mid + 1;
|
2012-11-30 02:28:37 +01:00
|
|
|
} else {
|
2012-12-10 20:02:07 +01:00
|
|
|
end = mid - 1;
|
2012-11-30 02:28:37 +01:00
|
|
|
}
|
|
|
|
}
|
2013-08-06 21:54:37 +02:00
|
|
|
size_t start_index = std::max(0l, end); // end could be -ve.
|
|
|
|
// The last wal file is always included
|
|
|
|
all_logs.erase(all_logs.begin(), all_logs.begin() + start_index);
|
2012-11-30 02:28:37 +01:00
|
|
|
return Status::OK();
|
|
|
|
}
|
|
|
|
|
2013-08-06 21:54:37 +02:00
|
|
|
bool DBImpl::CheckWalFileExistsAndEmpty(const WalFileType type,
|
|
|
|
const uint64_t number) {
|
|
|
|
const std::string fname = (type == kAliveLogFile) ?
|
2013-10-01 23:46:52 +02:00
|
|
|
LogFileName(options_.wal_dir, number) :
|
|
|
|
ArchivedLogFileName(options_.wal_dir, number);
|
2013-03-18 22:50:59 +01:00
|
|
|
uint64_t file_size;
|
|
|
|
Status s = env_->GetFileSize(fname, &file_size);
|
2013-08-06 21:54:37 +02:00
|
|
|
return (s.ok() && (file_size == 0));
|
2013-03-18 22:50:59 +01:00
|
|
|
}
|
|
|
|
|
2013-08-06 21:54:37 +02:00
|
|
|
Status DBImpl::ReadFirstRecord(const WalFileType type, const uint64_t number,
|
|
|
|
WriteBatch* const result) {
|
2012-11-30 02:28:37 +01:00
|
|
|
|
2013-08-06 21:54:37 +02:00
|
|
|
if (type == kAliveLogFile) {
|
2013-10-01 23:46:52 +02:00
|
|
|
std::string fname = LogFileName(options_.wal_dir, number);
|
2012-11-30 02:28:37 +01:00
|
|
|
Status status = ReadFirstLine(fname, result);
|
|
|
|
if (!status.ok()) {
|
|
|
|
// check if the file got moved to archive.
|
2013-10-01 23:46:52 +02:00
|
|
|
std::string archived_file =
|
|
|
|
ArchivedLogFileName(options_.wal_dir, number);
|
2013-08-06 21:54:37 +02:00
|
|
|
Status s = ReadFirstLine(archived_file, result);
|
2012-11-30 02:28:37 +01:00
|
|
|
if (!s.ok()) {
|
2013-10-01 23:46:52 +02:00
|
|
|
return Status::IOError("Log File has been deleted: " + archived_file);
|
2012-11-30 02:28:37 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
return Status::OK();
|
2013-08-06 21:54:37 +02:00
|
|
|
} else if (type == kArchivedLogFile) {
|
2013-10-01 23:46:52 +02:00
|
|
|
std::string fname = ArchivedLogFileName(options_.wal_dir, number);
|
2012-11-30 02:28:37 +01:00
|
|
|
Status status = ReadFirstLine(fname, result);
|
|
|
|
return status;
|
|
|
|
}
|
2013-11-13 05:05:28 +01:00
|
|
|
return Status::NotSupported("File Type Not Known: " + std::to_string(type));
|
2012-11-30 02:28:37 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
Status DBImpl::ReadFirstLine(const std::string& fname,
|
|
|
|
WriteBatch* const batch) {
|
|
|
|
struct LogReporter : public log::Reader::Reporter {
|
|
|
|
Env* env;
|
|
|
|
Logger* info_log;
|
|
|
|
const char* fname;
|
2013-02-15 20:53:17 +01:00
|
|
|
Status* status; // nullptr if options_.paranoid_checks==false
|
2012-11-30 02:28:37 +01:00
|
|
|
virtual void Corruption(size_t bytes, const Status& s) {
|
|
|
|
Log(info_log, "%s%s: dropping %d bytes; %s",
|
2013-02-15 20:53:17 +01:00
|
|
|
(this->status == nullptr ? "(ignoring error) " : ""),
|
2012-11-30 02:28:37 +01:00
|
|
|
fname, static_cast<int>(bytes), s.ToString().c_str());
|
2013-02-15 20:53:17 +01:00
|
|
|
if (this->status != nullptr && this->status->ok()) *this->status = s;
|
2012-11-30 02:28:37 +01:00
|
|
|
}
|
|
|
|
};
|
|
|
|
|
2013-01-20 11:07:13 +01:00
|
|
|
unique_ptr<SequentialFile> file;
|
2013-03-15 01:00:04 +01:00
|
|
|
Status status = env_->NewSequentialFile(fname, &file, storage_options_);
|
2012-11-30 02:28:37 +01:00
|
|
|
|
|
|
|
if (!status.ok()) {
|
|
|
|
return status;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
LogReporter reporter;
|
|
|
|
reporter.env = env_;
|
2013-01-20 11:07:13 +01:00
|
|
|
reporter.info_log = options_.info_log.get();
|
2012-11-30 02:28:37 +01:00
|
|
|
reporter.fname = fname.c_str();
|
2013-02-15 20:53:17 +01:00
|
|
|
reporter.status = (options_.paranoid_checks ? &status : nullptr);
|
2013-01-20 11:07:13 +01:00
|
|
|
log::Reader reader(std::move(file), &reporter, true/*checksum*/,
|
2012-11-30 02:28:37 +01:00
|
|
|
0/*initial_offset*/);
|
|
|
|
std::string scratch;
|
|
|
|
Slice record;
|
2013-08-06 21:54:37 +02:00
|
|
|
|
2012-11-30 02:28:37 +01:00
|
|
|
if (reader.ReadRecord(&record, &scratch) && status.ok()) {
|
|
|
|
if (record.size() < 12) {
|
|
|
|
reporter.Corruption(
|
|
|
|
record.size(), Status::Corruption("log record too small"));
|
|
|
|
return Status::IOError("Corruption noted");
|
|
|
|
// TODO read record's till the first no corrupt entry?
|
|
|
|
}
|
|
|
|
WriteBatchInternal::SetContents(batch, record);
|
|
|
|
return Status::OK();
|
|
|
|
}
|
|
|
|
return Status::IOError("Error reading from file " + fname);
|
|
|
|
}
|
|
|
|
|
2013-08-06 21:54:37 +02:00
|
|
|
struct CompareLogByPointer {
|
|
|
|
bool operator() (const unique_ptr<LogFile>& a,
|
|
|
|
const unique_ptr<LogFile>& b) {
|
|
|
|
LogFileImpl* a_impl = dynamic_cast<LogFileImpl*>(a.get());
|
|
|
|
LogFileImpl* b_impl = dynamic_cast<LogFileImpl*>(b.get());
|
|
|
|
return *a_impl < *b_impl;
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
Status DBImpl::AppendSortedWalsOfType(const std::string& path,
|
|
|
|
VectorLogPtr& log_files, WalFileType log_type) {
|
|
|
|
std::vector<std::string> all_files;
|
|
|
|
const Status status = env_->GetChildren(path, &all_files);
|
2012-11-30 02:28:37 +01:00
|
|
|
if (!status.ok()) {
|
|
|
|
return status;
|
|
|
|
}
|
2013-08-06 21:54:37 +02:00
|
|
|
log_files.reserve(log_files.size() + all_files.size());
|
2013-08-29 23:30:52 +02:00
|
|
|
VectorLogPtr::iterator pos_start;
|
|
|
|
if (!log_files.empty()) {
|
|
|
|
pos_start = log_files.end() - 1;
|
|
|
|
} else {
|
|
|
|
pos_start = log_files.begin();
|
|
|
|
}
|
2013-08-06 21:54:37 +02:00
|
|
|
for (const auto& f : all_files) {
|
2012-11-30 02:28:37 +01:00
|
|
|
uint64_t number;
|
|
|
|
FileType type;
|
2013-03-29 02:01:06 +01:00
|
|
|
if (ParseFileName(f, &number, &type) && type == kLogFile){
|
2013-08-06 21:54:37 +02:00
|
|
|
|
|
|
|
WriteBatch batch;
|
|
|
|
Status s = ReadFirstRecord(log_type, number, &batch);
|
|
|
|
if (!s.ok()) {
|
|
|
|
if (CheckWalFileExistsAndEmpty(log_type, number)) {
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
|
|
|
uint64_t size_bytes;
|
|
|
|
s = env_->GetFileSize(LogFileName(path, number), &size_bytes);
|
|
|
|
if (!s.ok()) {
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
|
|
|
log_files.push_back(std::move(unique_ptr<LogFile>(new LogFileImpl(
|
|
|
|
number, log_type, WriteBatchInternal::Sequence(&batch), size_bytes))));
|
2012-11-30 02:28:37 +01:00
|
|
|
}
|
|
|
|
}
|
2013-08-06 21:54:37 +02:00
|
|
|
CompareLogByPointer compare_log_files;
|
2013-08-29 23:30:52 +02:00
|
|
|
std::sort(pos_start, log_files.end(), compare_log_files);
|
2012-11-30 02:28:37 +01:00
|
|
|
return status;
|
|
|
|
}
|
|
|
|
|
2014-01-22 21:46:24 +01:00
|
|
|
Status DBImpl::RunManualCompaction(int input_level,
|
|
|
|
int output_level,
|
|
|
|
const Slice* begin,
|
|
|
|
const Slice* end) {
|
2014-01-15 01:19:09 +01:00
|
|
|
assert(input_level >= 0);
|
2011-06-22 04:36:45 +02:00
|
|
|
|
2011-10-06 01:30:28 +02:00
|
|
|
InternalKey begin_storage, end_storage;
|
|
|
|
|
2011-06-07 16:40:26 +02:00
|
|
|
ManualCompaction manual;
|
2014-01-15 01:19:09 +01:00
|
|
|
manual.input_level = input_level;
|
|
|
|
manual.output_level = output_level;
|
2011-10-06 01:30:28 +02:00
|
|
|
manual.done = false;
|
2012-10-19 23:00:53 +02:00
|
|
|
manual.in_progress = false;
|
2013-08-08 00:20:41 +02:00
|
|
|
// For universal compaction, we enforce every manual compaction to compact
|
|
|
|
// all files.
|
|
|
|
if (begin == nullptr ||
|
|
|
|
options_.compaction_style == kCompactionStyleUniversal) {
|
2013-02-15 20:53:17 +01:00
|
|
|
manual.begin = nullptr;
|
2011-10-06 01:30:28 +02:00
|
|
|
} else {
|
|
|
|
begin_storage = InternalKey(*begin, kMaxSequenceNumber, kValueTypeForSeek);
|
|
|
|
manual.begin = &begin_storage;
|
|
|
|
}
|
2013-08-08 00:20:41 +02:00
|
|
|
if (end == nullptr ||
|
|
|
|
options_.compaction_style == kCompactionStyleUniversal) {
|
2013-02-15 20:53:17 +01:00
|
|
|
manual.end = nullptr;
|
2011-10-06 01:30:28 +02:00
|
|
|
} else {
|
|
|
|
end_storage = InternalKey(*end, 0, static_cast<ValueType>(0));
|
|
|
|
manual.end = &end_storage;
|
|
|
|
}
|
|
|
|
|
|
|
|
MutexLock l(&mutex_);
|
2012-10-19 23:00:53 +02:00
|
|
|
|
Fix a deadlock in CompactRange()
Summary:
The way DBImpl::TEST_CompactRange() throttles down the number of bg compactions
can cause it to deadlock when CompactRange() is called concurrently from
multiple threads. Imagine a following scenario with only two threads
(max_background_compactions is 10 and bg_compaction_scheduled_ is initially 0):
1. Thread #1 increments bg_compaction_scheduled_ (to LargeNumber), sets
bg_compaction_scheduled_ to 9 (newvalue), schedules the compaction
(bg_compaction_scheduled_ is now 10) and waits for it to complete.
2. Thread #2 calls TEST_CompactRange(), increments bg_compaction_scheduled_
(now LargeNumber + 10) and waits on a cv for bg_compaction_scheduled_ to
drop to LargeNumber.
3. BG thread completes the first manual compaction, decrements
bg_compaction_scheduled_ and wakes up all threads waiting on bg_cv_.
Thread #1 runs, increments bg_compaction_scheduled_ by LargeNumber again
(now 2*LargeNumber + 9). Since that's more than LargeNumber + newvalue,
thread #2 also goes to sleep (waiting on bg_cv_), without resetting
bg_compaction_scheduled_.
This diff attempts to address the problem by introducing a new counter
bg_manual_only_ (when positive, MaybeScheduleFlushOrCompaction() will only
schedule manual compactions).
Test Plan:
I could pretty much consistently reproduce the deadlock with a program that
calls CompactRange(nullptr, nullptr) immediately after Write() from multiple
threads. This no longer happens with this patch.
Tests (make check) pass.
Reviewers: dhruba, igor, sdong, haobo
Reviewed By: igor
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14799
2013-12-22 00:10:39 +01:00
|
|
|
// When a manual compaction arrives, temporarily disable scheduling of
|
|
|
|
// non-manual compactions and wait until the number of scheduled compaction
|
|
|
|
// jobs drops to zero. This is needed to ensure that this manual compaction
|
|
|
|
// can compact any range of keys/files.
|
|
|
|
//
|
|
|
|
// bg_manual_only_ is non-zero when at least one thread is inside
|
2014-01-15 01:19:09 +01:00
|
|
|
// RunManualCompaction(), i.e. during that time no other compaction will
|
Fix a deadlock in CompactRange()
Summary:
The way DBImpl::TEST_CompactRange() throttles down the number of bg compactions
can cause it to deadlock when CompactRange() is called concurrently from
multiple threads. Imagine a following scenario with only two threads
(max_background_compactions is 10 and bg_compaction_scheduled_ is initially 0):
1. Thread #1 increments bg_compaction_scheduled_ (to LargeNumber), sets
bg_compaction_scheduled_ to 9 (newvalue), schedules the compaction
(bg_compaction_scheduled_ is now 10) and waits for it to complete.
2. Thread #2 calls TEST_CompactRange(), increments bg_compaction_scheduled_
(now LargeNumber + 10) and waits on a cv for bg_compaction_scheduled_ to
drop to LargeNumber.
3. BG thread completes the first manual compaction, decrements
bg_compaction_scheduled_ and wakes up all threads waiting on bg_cv_.
Thread #1 runs, increments bg_compaction_scheduled_ by LargeNumber again
(now 2*LargeNumber + 9). Since that's more than LargeNumber + newvalue,
thread #2 also goes to sleep (waiting on bg_cv_), without resetting
bg_compaction_scheduled_.
This diff attempts to address the problem by introducing a new counter
bg_manual_only_ (when positive, MaybeScheduleFlushOrCompaction() will only
schedule manual compactions).
Test Plan:
I could pretty much consistently reproduce the deadlock with a program that
calls CompactRange(nullptr, nullptr) immediately after Write() from multiple
threads. This no longer happens with this patch.
Tests (make check) pass.
Reviewers: dhruba, igor, sdong, haobo
Reviewed By: igor
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14799
2013-12-22 00:10:39 +01:00
|
|
|
// get scheduled (see MaybeScheduleFlushOrCompaction).
|
|
|
|
//
|
|
|
|
// Note that the following loop doesn't stop more that one thread calling
|
2014-01-15 01:19:09 +01:00
|
|
|
// RunManualCompaction() from getting to the second while loop below.
|
Fix a deadlock in CompactRange()
Summary:
The way DBImpl::TEST_CompactRange() throttles down the number of bg compactions
can cause it to deadlock when CompactRange() is called concurrently from
multiple threads. Imagine a following scenario with only two threads
(max_background_compactions is 10 and bg_compaction_scheduled_ is initially 0):
1. Thread #1 increments bg_compaction_scheduled_ (to LargeNumber), sets
bg_compaction_scheduled_ to 9 (newvalue), schedules the compaction
(bg_compaction_scheduled_ is now 10) and waits for it to complete.
2. Thread #2 calls TEST_CompactRange(), increments bg_compaction_scheduled_
(now LargeNumber + 10) and waits on a cv for bg_compaction_scheduled_ to
drop to LargeNumber.
3. BG thread completes the first manual compaction, decrements
bg_compaction_scheduled_ and wakes up all threads waiting on bg_cv_.
Thread #1 runs, increments bg_compaction_scheduled_ by LargeNumber again
(now 2*LargeNumber + 9). Since that's more than LargeNumber + newvalue,
thread #2 also goes to sleep (waiting on bg_cv_), without resetting
bg_compaction_scheduled_.
This diff attempts to address the problem by introducing a new counter
bg_manual_only_ (when positive, MaybeScheduleFlushOrCompaction() will only
schedule manual compactions).
Test Plan:
I could pretty much consistently reproduce the deadlock with a program that
calls CompactRange(nullptr, nullptr) immediately after Write() from multiple
threads. This no longer happens with this patch.
Tests (make check) pass.
Reviewers: dhruba, igor, sdong, haobo
Reviewed By: igor
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14799
2013-12-22 00:10:39 +01:00
|
|
|
// However, only one of them will actually schedule compaction, while
|
|
|
|
// others will wait on a condition variable until it completes.
|
|
|
|
|
|
|
|
++bg_manual_only_;
|
|
|
|
while (bg_compaction_scheduled_ > 0) {
|
|
|
|
Log(options_.info_log,
|
|
|
|
"Manual compaction waiting for all other scheduled background "
|
|
|
|
"compactions to finish");
|
2012-10-19 23:00:53 +02:00
|
|
|
bg_cv_.Wait();
|
|
|
|
}
|
Fix a deadlock in CompactRange()
Summary:
The way DBImpl::TEST_CompactRange() throttles down the number of bg compactions
can cause it to deadlock when CompactRange() is called concurrently from
multiple threads. Imagine a following scenario with only two threads
(max_background_compactions is 10 and bg_compaction_scheduled_ is initially 0):
1. Thread #1 increments bg_compaction_scheduled_ (to LargeNumber), sets
bg_compaction_scheduled_ to 9 (newvalue), schedules the compaction
(bg_compaction_scheduled_ is now 10) and waits for it to complete.
2. Thread #2 calls TEST_CompactRange(), increments bg_compaction_scheduled_
(now LargeNumber + 10) and waits on a cv for bg_compaction_scheduled_ to
drop to LargeNumber.
3. BG thread completes the first manual compaction, decrements
bg_compaction_scheduled_ and wakes up all threads waiting on bg_cv_.
Thread #1 runs, increments bg_compaction_scheduled_ by LargeNumber again
(now 2*LargeNumber + 9). Since that's more than LargeNumber + newvalue,
thread #2 also goes to sleep (waiting on bg_cv_), without resetting
bg_compaction_scheduled_.
This diff attempts to address the problem by introducing a new counter
bg_manual_only_ (when positive, MaybeScheduleFlushOrCompaction() will only
schedule manual compactions).
Test Plan:
I could pretty much consistently reproduce the deadlock with a program that
calls CompactRange(nullptr, nullptr) immediately after Write() from multiple
threads. This no longer happens with this patch.
Tests (make check) pass.
Reviewers: dhruba, igor, sdong, haobo
Reviewed By: igor
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14799
2013-12-22 00:10:39 +01:00
|
|
|
|
2012-10-19 23:00:53 +02:00
|
|
|
Log(options_.info_log, "Manual compaction starting");
|
|
|
|
|
Fix a deadlock in CompactRange()
Summary:
The way DBImpl::TEST_CompactRange() throttles down the number of bg compactions
can cause it to deadlock when CompactRange() is called concurrently from
multiple threads. Imagine a following scenario with only two threads
(max_background_compactions is 10 and bg_compaction_scheduled_ is initially 0):
1. Thread #1 increments bg_compaction_scheduled_ (to LargeNumber), sets
bg_compaction_scheduled_ to 9 (newvalue), schedules the compaction
(bg_compaction_scheduled_ is now 10) and waits for it to complete.
2. Thread #2 calls TEST_CompactRange(), increments bg_compaction_scheduled_
(now LargeNumber + 10) and waits on a cv for bg_compaction_scheduled_ to
drop to LargeNumber.
3. BG thread completes the first manual compaction, decrements
bg_compaction_scheduled_ and wakes up all threads waiting on bg_cv_.
Thread #1 runs, increments bg_compaction_scheduled_ by LargeNumber again
(now 2*LargeNumber + 9). Since that's more than LargeNumber + newvalue,
thread #2 also goes to sleep (waiting on bg_cv_), without resetting
bg_compaction_scheduled_.
This diff attempts to address the problem by introducing a new counter
bg_manual_only_ (when positive, MaybeScheduleFlushOrCompaction() will only
schedule manual compactions).
Test Plan:
I could pretty much consistently reproduce the deadlock with a program that
calls CompactRange(nullptr, nullptr) immediately after Write() from multiple
threads. This no longer happens with this patch.
Tests (make check) pass.
Reviewers: dhruba, igor, sdong, haobo
Reviewed By: igor
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14799
2013-12-22 00:10:39 +01:00
|
|
|
while (!manual.done && !shutting_down_.Acquire_Load() && bg_error_.ok()) {
|
|
|
|
assert(bg_manual_only_ > 0);
|
|
|
|
if (manual_compaction_ != nullptr) {
|
|
|
|
// Running either this or some other manual compaction
|
2011-10-06 01:30:28 +02:00
|
|
|
bg_cv_.Wait();
|
Fix a deadlock in CompactRange()
Summary:
The way DBImpl::TEST_CompactRange() throttles down the number of bg compactions
can cause it to deadlock when CompactRange() is called concurrently from
multiple threads. Imagine a following scenario with only two threads
(max_background_compactions is 10 and bg_compaction_scheduled_ is initially 0):
1. Thread #1 increments bg_compaction_scheduled_ (to LargeNumber), sets
bg_compaction_scheduled_ to 9 (newvalue), schedules the compaction
(bg_compaction_scheduled_ is now 10) and waits for it to complete.
2. Thread #2 calls TEST_CompactRange(), increments bg_compaction_scheduled_
(now LargeNumber + 10) and waits on a cv for bg_compaction_scheduled_ to
drop to LargeNumber.
3. BG thread completes the first manual compaction, decrements
bg_compaction_scheduled_ and wakes up all threads waiting on bg_cv_.
Thread #1 runs, increments bg_compaction_scheduled_ by LargeNumber again
(now 2*LargeNumber + 9). Since that's more than LargeNumber + newvalue,
thread #2 also goes to sleep (waiting on bg_cv_), without resetting
bg_compaction_scheduled_.
This diff attempts to address the problem by introducing a new counter
bg_manual_only_ (when positive, MaybeScheduleFlushOrCompaction() will only
schedule manual compactions).
Test Plan:
I could pretty much consistently reproduce the deadlock with a program that
calls CompactRange(nullptr, nullptr) immediately after Write() from multiple
threads. This no longer happens with this patch.
Tests (make check) pass.
Reviewers: dhruba, igor, sdong, haobo
Reviewed By: igor
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14799
2013-12-22 00:10:39 +01:00
|
|
|
} else {
|
|
|
|
manual_compaction_ = &manual;
|
|
|
|
MaybeScheduleFlushOrCompaction();
|
2011-10-06 01:30:28 +02:00
|
|
|
}
|
2011-06-07 16:40:26 +02:00
|
|
|
}
|
2012-10-19 23:00:53 +02:00
|
|
|
|
Fix a deadlock in CompactRange()
Summary:
The way DBImpl::TEST_CompactRange() throttles down the number of bg compactions
can cause it to deadlock when CompactRange() is called concurrently from
multiple threads. Imagine a following scenario with only two threads
(max_background_compactions is 10 and bg_compaction_scheduled_ is initially 0):
1. Thread #1 increments bg_compaction_scheduled_ (to LargeNumber), sets
bg_compaction_scheduled_ to 9 (newvalue), schedules the compaction
(bg_compaction_scheduled_ is now 10) and waits for it to complete.
2. Thread #2 calls TEST_CompactRange(), increments bg_compaction_scheduled_
(now LargeNumber + 10) and waits on a cv for bg_compaction_scheduled_ to
drop to LargeNumber.
3. BG thread completes the first manual compaction, decrements
bg_compaction_scheduled_ and wakes up all threads waiting on bg_cv_.
Thread #1 runs, increments bg_compaction_scheduled_ by LargeNumber again
(now 2*LargeNumber + 9). Since that's more than LargeNumber + newvalue,
thread #2 also goes to sleep (waiting on bg_cv_), without resetting
bg_compaction_scheduled_.
This diff attempts to address the problem by introducing a new counter
bg_manual_only_ (when positive, MaybeScheduleFlushOrCompaction() will only
schedule manual compactions).
Test Plan:
I could pretty much consistently reproduce the deadlock with a program that
calls CompactRange(nullptr, nullptr) immediately after Write() from multiple
threads. This no longer happens with this patch.
Tests (make check) pass.
Reviewers: dhruba, igor, sdong, haobo
Reviewed By: igor
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14799
2013-12-22 00:10:39 +01:00
|
|
|
assert(!manual.in_progress);
|
|
|
|
assert(bg_manual_only_ > 0);
|
|
|
|
--bg_manual_only_;
|
2014-01-22 21:46:24 +01:00
|
|
|
return manual.status;
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2014-01-22 21:46:24 +01:00
|
|
|
Status DBImpl::TEST_CompactRange(int level,
|
|
|
|
const Slice* begin,
|
|
|
|
const Slice* end) {
|
2014-01-15 01:19:09 +01:00
|
|
|
int output_level = (options_.compaction_style == kCompactionStyleUniversal)
|
|
|
|
? level
|
|
|
|
: level + 1;
|
2014-01-22 21:46:24 +01:00
|
|
|
return RunManualCompaction(level, output_level, begin, end);
|
2014-01-15 01:19:09 +01:00
|
|
|
}
|
|
|
|
|
2012-07-06 20:42:09 +02:00
|
|
|
Status DBImpl::FlushMemTable(const FlushOptions& options) {
|
2013-02-15 20:53:17 +01:00
|
|
|
// nullptr batch means just wait for earlier writes to be done
|
|
|
|
Status s = Write(WriteOptions(), nullptr);
|
2012-07-06 20:42:09 +02:00
|
|
|
if (s.ok() && options.wait) {
|
2011-04-12 21:38:58 +02:00
|
|
|
// Wait until the compaction completes
|
2013-10-15 00:12:15 +02:00
|
|
|
s = WaitForFlushMemTable();
|
2011-04-12 21:38:58 +02:00
|
|
|
}
|
|
|
|
return s;
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2013-10-15 00:12:15 +02:00
|
|
|
Status DBImpl::WaitForFlushMemTable() {
|
2012-08-15 00:20:36 +02:00
|
|
|
Status s;
|
|
|
|
// Wait until the compaction completes
|
|
|
|
MutexLock l(&mutex_);
|
2014-01-29 22:28:50 +01:00
|
|
|
while (default_cfd_->imm()->size() > 0 && bg_error_.ok()) {
|
2012-08-15 00:20:36 +02:00
|
|
|
bg_cv_.Wait();
|
|
|
|
}
|
2014-01-29 22:28:50 +01:00
|
|
|
if (default_cfd_->imm()->size() != 0) {
|
2012-08-15 00:20:36 +02:00
|
|
|
s = bg_error_;
|
|
|
|
}
|
|
|
|
return s;
|
2012-07-06 20:42:09 +02:00
|
|
|
}
|
|
|
|
|
2013-10-15 00:12:15 +02:00
|
|
|
Status DBImpl::TEST_FlushMemTable() {
|
2012-07-06 20:42:09 +02:00
|
|
|
return FlushMemTable(FlushOptions());
|
|
|
|
}
|
|
|
|
|
2013-10-15 00:12:15 +02:00
|
|
|
Status DBImpl::TEST_WaitForFlushMemTable() {
|
|
|
|
return WaitForFlushMemTable();
|
2012-06-23 04:30:03 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
Status DBImpl::TEST_WaitForCompact() {
|
2012-08-15 00:20:36 +02:00
|
|
|
// Wait until the compaction completes
|
2013-10-15 00:12:15 +02:00
|
|
|
|
|
|
|
// TODO: a bug here. This function actually does not necessarily
|
|
|
|
// wait for compact. It actually waits for scheduled compaction
|
|
|
|
// OR flush to finish.
|
|
|
|
|
2012-08-15 00:20:36 +02:00
|
|
|
MutexLock l(&mutex_);
|
2013-09-13 23:38:37 +02:00
|
|
|
while ((bg_compaction_scheduled_ || bg_flush_scheduled_) &&
|
|
|
|
bg_error_.ok()) {
|
2012-08-15 00:20:36 +02:00
|
|
|
bg_cv_.Wait();
|
|
|
|
}
|
|
|
|
return bg_error_;
|
2012-06-23 04:30:03 +02:00
|
|
|
}
|
|
|
|
|
2013-10-15 00:12:15 +02:00
|
|
|
void DBImpl::MaybeScheduleFlushOrCompaction() {
|
2011-03-18 23:37:00 +01:00
|
|
|
mutex_.AssertHeld();
|
2013-06-30 08:21:36 +02:00
|
|
|
if (bg_work_gate_closed_) {
|
|
|
|
// gate closed for backgrond work
|
2011-03-18 23:37:00 +01:00
|
|
|
} else if (shutting_down_.Acquire_Load()) {
|
|
|
|
// DB is being deleted; no more background compactions
|
|
|
|
} else {
|
2014-01-29 22:28:50 +01:00
|
|
|
bool is_flush_pending = default_cfd_->imm()->IsFlushPending();
|
2013-09-13 23:38:37 +02:00
|
|
|
if (is_flush_pending &&
|
|
|
|
(bg_flush_scheduled_ < options_.max_background_flushes)) {
|
|
|
|
// memtable flush needed
|
|
|
|
bg_flush_scheduled_++;
|
|
|
|
env_->Schedule(&DBImpl::BGWorkFlush, this, Env::Priority::HIGH);
|
|
|
|
}
|
|
|
|
|
Fix a deadlock in CompactRange()
Summary:
The way DBImpl::TEST_CompactRange() throttles down the number of bg compactions
can cause it to deadlock when CompactRange() is called concurrently from
multiple threads. Imagine a following scenario with only two threads
(max_background_compactions is 10 and bg_compaction_scheduled_ is initially 0):
1. Thread #1 increments bg_compaction_scheduled_ (to LargeNumber), sets
bg_compaction_scheduled_ to 9 (newvalue), schedules the compaction
(bg_compaction_scheduled_ is now 10) and waits for it to complete.
2. Thread #2 calls TEST_CompactRange(), increments bg_compaction_scheduled_
(now LargeNumber + 10) and waits on a cv for bg_compaction_scheduled_ to
drop to LargeNumber.
3. BG thread completes the first manual compaction, decrements
bg_compaction_scheduled_ and wakes up all threads waiting on bg_cv_.
Thread #1 runs, increments bg_compaction_scheduled_ by LargeNumber again
(now 2*LargeNumber + 9). Since that's more than LargeNumber + newvalue,
thread #2 also goes to sleep (waiting on bg_cv_), without resetting
bg_compaction_scheduled_.
This diff attempts to address the problem by introducing a new counter
bg_manual_only_ (when positive, MaybeScheduleFlushOrCompaction() will only
schedule manual compactions).
Test Plan:
I could pretty much consistently reproduce the deadlock with a program that
calls CompactRange(nullptr, nullptr) immediately after Write() from multiple
threads. This no longer happens with this patch.
Tests (make check) pass.
Reviewers: dhruba, igor, sdong, haobo
Reviewed By: igor
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14799
2013-12-22 00:10:39 +01:00
|
|
|
// Schedule BGWorkCompaction if there's a compaction pending (or a memtable
|
|
|
|
// flush, but the HIGH pool is not enabled). Do it only if
|
|
|
|
// max_background_compactions hasn't been reached and, in case
|
|
|
|
// bg_manual_only_ > 0, if it's a manual compaction.
|
2014-01-29 22:28:50 +01:00
|
|
|
if ((manual_compaction_ || default_cfd_->current()->NeedsCompaction() ||
|
2013-09-13 23:38:37 +02:00
|
|
|
(is_flush_pending && (options_.max_background_flushes <= 0))) &&
|
Fix a deadlock in CompactRange()
Summary:
The way DBImpl::TEST_CompactRange() throttles down the number of bg compactions
can cause it to deadlock when CompactRange() is called concurrently from
multiple threads. Imagine a following scenario with only two threads
(max_background_compactions is 10 and bg_compaction_scheduled_ is initially 0):
1. Thread #1 increments bg_compaction_scheduled_ (to LargeNumber), sets
bg_compaction_scheduled_ to 9 (newvalue), schedules the compaction
(bg_compaction_scheduled_ is now 10) and waits for it to complete.
2. Thread #2 calls TEST_CompactRange(), increments bg_compaction_scheduled_
(now LargeNumber + 10) and waits on a cv for bg_compaction_scheduled_ to
drop to LargeNumber.
3. BG thread completes the first manual compaction, decrements
bg_compaction_scheduled_ and wakes up all threads waiting on bg_cv_.
Thread #1 runs, increments bg_compaction_scheduled_ by LargeNumber again
(now 2*LargeNumber + 9). Since that's more than LargeNumber + newvalue,
thread #2 also goes to sleep (waiting on bg_cv_), without resetting
bg_compaction_scheduled_.
This diff attempts to address the problem by introducing a new counter
bg_manual_only_ (when positive, MaybeScheduleFlushOrCompaction() will only
schedule manual compactions).
Test Plan:
I could pretty much consistently reproduce the deadlock with a program that
calls CompactRange(nullptr, nullptr) immediately after Write() from multiple
threads. This no longer happens with this patch.
Tests (make check) pass.
Reviewers: dhruba, igor, sdong, haobo
Reviewed By: igor
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14799
2013-12-22 00:10:39 +01:00
|
|
|
bg_compaction_scheduled_ < options_.max_background_compactions &&
|
|
|
|
(!bg_manual_only_ || manual_compaction_)) {
|
|
|
|
|
2013-09-13 23:38:37 +02:00
|
|
|
bg_compaction_scheduled_++;
|
|
|
|
env_->Schedule(&DBImpl::BGWorkCompaction, this, Env::Priority::LOW);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
void DBImpl::BGWorkFlush(void* db) {
|
|
|
|
reinterpret_cast<DBImpl*>(db)->BackgroundCallFlush();
|
|
|
|
}
|
|
|
|
|
|
|
|
void DBImpl::BGWorkCompaction(void* db) {
|
|
|
|
reinterpret_cast<DBImpl*>(db)->BackgroundCallCompaction();
|
|
|
|
}
|
|
|
|
|
2013-11-09 00:23:46 +01:00
|
|
|
Status DBImpl::BackgroundFlush(bool* madeProgress,
|
|
|
|
DeletionState& deletion_state) {
|
2013-09-13 23:38:37 +02:00
|
|
|
Status stat;
|
2014-01-29 22:28:50 +01:00
|
|
|
while (stat.ok() && default_cfd_->imm()->IsFlushPending()) {
|
2013-09-13 23:38:37 +02:00
|
|
|
Log(options_.info_log,
|
2013-10-15 00:12:15 +02:00
|
|
|
"BackgroundCallFlush doing FlushMemTableToOutputFile, flush slots available %d",
|
2013-09-13 23:38:37 +02:00
|
|
|
options_.max_background_flushes - bg_flush_scheduled_);
|
2013-11-09 00:23:46 +01:00
|
|
|
stat = FlushMemTableToOutputFile(madeProgress, deletion_state);
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
2013-09-13 23:38:37 +02:00
|
|
|
return stat;
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2013-09-13 23:38:37 +02:00
|
|
|
void DBImpl::BackgroundCallFlush() {
|
2013-10-16 22:32:53 +02:00
|
|
|
bool madeProgress = false;
|
2013-12-20 18:57:58 +01:00
|
|
|
DeletionState deletion_state(options_.max_write_buffer_number, true);
|
2013-09-13 23:38:37 +02:00
|
|
|
assert(bg_flush_scheduled_);
|
|
|
|
MutexLock l(&mutex_);
|
|
|
|
|
2013-11-12 06:41:32 +01:00
|
|
|
Status s;
|
2013-09-13 23:38:37 +02:00
|
|
|
if (!shutting_down_.Acquire_Load()) {
|
2013-11-12 06:41:32 +01:00
|
|
|
s = BackgroundFlush(&madeProgress, deletion_state);
|
2013-09-13 23:38:37 +02:00
|
|
|
if (!s.ok()) {
|
|
|
|
// Wait a little bit before retrying background compaction in
|
|
|
|
// case this is an environmental problem and we do not want to
|
|
|
|
// chew up resources for failed compactions for the duration of
|
|
|
|
// the problem.
|
|
|
|
bg_cv_.SignalAll(); // In case a waiter can proceed despite the error
|
|
|
|
Log(options_.info_log, "Waiting after background flush error: %s",
|
|
|
|
s.ToString().c_str());
|
|
|
|
mutex_.Unlock();
|
2013-11-07 20:31:56 +01:00
|
|
|
LogFlush(options_.info_log);
|
2013-09-13 23:38:37 +02:00
|
|
|
env_->SleepForMicroseconds(1000000);
|
|
|
|
mutex_.Lock();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-11-12 06:41:32 +01:00
|
|
|
// If !s.ok(), this means that Flush failed. In that case, we want
|
|
|
|
// to delete all obsolete files and we force FindObsoleteFiles()
|
|
|
|
FindObsoleteFiles(deletion_state, !s.ok());
|
2013-11-12 19:07:55 +01:00
|
|
|
// delete unnecessary files if any, this is done outside the mutex
|
2013-11-09 00:23:46 +01:00
|
|
|
if (deletion_state.HaveSomethingToDelete()) {
|
|
|
|
mutex_.Unlock();
|
|
|
|
PurgeObsoleteFiles(deletion_state);
|
|
|
|
mutex_.Lock();
|
|
|
|
}
|
|
|
|
|
2013-09-13 23:38:37 +02:00
|
|
|
bg_flush_scheduled_--;
|
2013-10-16 22:32:53 +02:00
|
|
|
if (madeProgress) {
|
|
|
|
MaybeScheduleFlushOrCompaction();
|
|
|
|
}
|
2013-09-13 23:38:37 +02:00
|
|
|
bg_cv_.SignalAll();
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2013-09-13 23:38:37 +02:00
|
|
|
|
2013-05-06 20:41:01 +02:00
|
|
|
void DBImpl::TEST_PurgeObsoleteteWAL() {
|
|
|
|
PurgeObsoleteWALFiles();
|
|
|
|
}
|
|
|
|
|
2014-01-16 01:18:04 +01:00
|
|
|
uint64_t DBImpl::TEST_GetLevel0TotalSize() {
|
|
|
|
MutexLock l(&mutex_);
|
2014-01-29 22:28:50 +01:00
|
|
|
return default_cfd_->current()->NumLevelBytes(0);
|
2014-01-16 01:18:04 +01:00
|
|
|
}
|
|
|
|
|
2013-09-13 23:38:37 +02:00
|
|
|
void DBImpl::BackgroundCallCompaction() {
|
2013-01-08 21:16:40 +01:00
|
|
|
bool madeProgress = false;
|
2013-12-20 18:57:58 +01:00
|
|
|
DeletionState deletion_state(options_.max_write_buffer_number, true);
|
2013-05-24 21:52:45 +02:00
|
|
|
|
|
|
|
MaybeDumpStats();
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
MutexLock l(&mutex_);
|
2012-10-19 23:00:53 +02:00
|
|
|
// Log(options_.info_log, "XXX BG Thread %llx process new work item", pthread_self());
|
2011-03-18 23:37:00 +01:00
|
|
|
assert(bg_compaction_scheduled_);
|
2013-11-12 06:41:32 +01:00
|
|
|
Status s;
|
2011-06-07 16:40:26 +02:00
|
|
|
if (!shutting_down_.Acquire_Load()) {
|
2013-11-12 06:41:32 +01:00
|
|
|
s = BackgroundCompaction(&madeProgress, deletion_state);
|
2012-08-23 01:57:51 +02:00
|
|
|
if (!s.ok()) {
|
|
|
|
// Wait a little bit before retrying background compaction in
|
|
|
|
// case this is an environmental problem and we do not want to
|
|
|
|
// chew up resources for failed compactions for the duration of
|
|
|
|
// the problem.
|
|
|
|
bg_cv_.SignalAll(); // In case a waiter can proceed despite the error
|
|
|
|
Log(options_.info_log, "Waiting after background compaction error: %s",
|
|
|
|
s.ToString().c_str());
|
|
|
|
mutex_.Unlock();
|
2013-11-07 20:31:56 +01:00
|
|
|
LogFlush(options_.info_log);
|
2012-08-23 01:57:51 +02:00
|
|
|
env_->SleepForMicroseconds(1000000);
|
|
|
|
mutex_.Lock();
|
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
2012-08-23 01:57:51 +02:00
|
|
|
|
2013-11-12 06:41:32 +01:00
|
|
|
// If !s.ok(), this means that Compaction failed. In that case, we want
|
|
|
|
// to delete all obsolete files we might have created and we force
|
|
|
|
// FindObsoleteFiles(). This is because deletion_state does not catch
|
|
|
|
// all created files if compaction failed.
|
|
|
|
FindObsoleteFiles(deletion_state, !s.ok());
|
2013-11-27 23:56:20 +01:00
|
|
|
|
2013-11-12 19:07:55 +01:00
|
|
|
// delete unnecessary files if any, this is done outside the mutex
|
2013-11-09 00:23:46 +01:00
|
|
|
if (deletion_state.HaveSomethingToDelete()) {
|
2012-10-21 10:49:48 +02:00
|
|
|
mutex_.Unlock();
|
|
|
|
PurgeObsoleteFiles(deletion_state);
|
2012-11-01 06:01:57 +01:00
|
|
|
mutex_.Lock();
|
2012-10-21 10:49:48 +02:00
|
|
|
}
|
|
|
|
|
2012-10-19 23:00:53 +02:00
|
|
|
bg_compaction_scheduled_--;
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2012-08-15 00:20:36 +02:00
|
|
|
MaybeScheduleLogDBDeployStats();
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
// Previous compaction may have produced too many files in a level,
|
2012-11-29 01:42:36 +01:00
|
|
|
// So reschedule another compaction if we made progress in the
|
2012-10-19 23:00:53 +02:00
|
|
|
// last compaction.
|
|
|
|
if (madeProgress) {
|
2013-10-15 00:12:15 +02:00
|
|
|
MaybeScheduleFlushOrCompaction();
|
2012-10-19 23:00:53 +02:00
|
|
|
}
|
2011-06-07 16:40:26 +02:00
|
|
|
bg_cv_.SignalAll();
|
2013-05-11 00:21:04 +02:00
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2012-11-29 01:42:36 +01:00
|
|
|
Status DBImpl::BackgroundCompaction(bool* madeProgress,
|
2013-12-20 18:57:58 +01:00
|
|
|
DeletionState& deletion_state) {
|
2012-10-19 23:00:53 +02:00
|
|
|
*madeProgress = false;
|
2011-03-18 23:37:00 +01:00
|
|
|
mutex_.AssertHeld();
|
2011-04-12 21:38:58 +02:00
|
|
|
|
2013-09-13 23:38:37 +02:00
|
|
|
// TODO: remove memtable flush from formal compaction
|
2014-01-29 22:28:50 +01:00
|
|
|
while (default_cfd_->imm()->IsFlushPending()) {
|
2012-11-29 01:42:36 +01:00
|
|
|
Log(options_.info_log,
|
2013-10-15 00:12:15 +02:00
|
|
|
"BackgroundCompaction doing FlushMemTableToOutputFile, compaction slots "
|
|
|
|
"available %d",
|
2012-10-19 23:00:53 +02:00
|
|
|
options_.max_background_compactions - bg_compaction_scheduled_);
|
2013-11-09 00:23:46 +01:00
|
|
|
Status stat = FlushMemTableToOutputFile(madeProgress, deletion_state);
|
2012-10-19 23:00:53 +02:00
|
|
|
if (!stat.ok()) {
|
|
|
|
return stat;
|
|
|
|
}
|
2011-04-12 21:38:58 +02:00
|
|
|
}
|
|
|
|
|
2013-01-20 11:07:13 +01:00
|
|
|
unique_ptr<Compaction> c;
|
2013-02-15 20:53:17 +01:00
|
|
|
bool is_manual = (manual_compaction_ != nullptr) &&
|
2012-10-19 23:00:53 +02:00
|
|
|
(manual_compaction_->in_progress == false);
|
2014-01-15 01:19:09 +01:00
|
|
|
InternalKey manual_end_storage;
|
|
|
|
InternalKey* manual_end = &manual_end_storage;
|
2011-06-07 16:40:26 +02:00
|
|
|
if (is_manual) {
|
2011-10-06 01:30:28 +02:00
|
|
|
ManualCompaction* m = manual_compaction_;
|
2012-10-19 23:00:53 +02:00
|
|
|
assert(!m->in_progress);
|
|
|
|
m->in_progress = true; // another thread cannot pick up the same work
|
2014-01-15 01:19:09 +01:00
|
|
|
c.reset(versions_->CompactRange(
|
|
|
|
m->input_level, m->output_level, m->begin, m->end, &manual_end));
|
|
|
|
if (!c) {
|
2013-01-20 11:07:13 +01:00
|
|
|
m->done = true;
|
2011-10-06 01:30:28 +02:00
|
|
|
}
|
|
|
|
Log(options_.info_log,
|
2014-01-15 01:19:09 +01:00
|
|
|
"Manual compaction from level-%d to level-%d from %s .. %s; will stop "
|
|
|
|
"at %s\n",
|
|
|
|
m->input_level,
|
|
|
|
m->output_level,
|
2011-10-06 01:30:28 +02:00
|
|
|
(m->begin ? m->begin->DebugString().c_str() : "(begin)"),
|
|
|
|
(m->end ? m->end->DebugString().c_str() : "(end)"),
|
2014-01-15 01:19:09 +01:00
|
|
|
((m->done || manual_end == nullptr)
|
|
|
|
? "(end)"
|
|
|
|
: manual_end->DebugString().c_str()));
|
2012-11-21 00:45:41 +01:00
|
|
|
} else if (!options_.disable_auto_compactions) {
|
2013-01-20 11:07:13 +01:00
|
|
|
c.reset(versions_->PickCompaction());
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
Status status;
|
2013-01-20 11:07:13 +01:00
|
|
|
if (!c) {
|
2011-06-07 16:40:26 +02:00
|
|
|
// Nothing to do
|
2012-10-19 23:00:53 +02:00
|
|
|
Log(options_.info_log, "Compaction nothing to do");
|
2011-06-07 16:40:26 +02:00
|
|
|
} else if (!is_manual && c->IsTrivialMove()) {
|
2011-03-18 23:37:00 +01:00
|
|
|
// Move file to next level
|
2011-03-22 19:32:49 +01:00
|
|
|
assert(c->num_input_files(0) == 1);
|
2011-03-18 23:37:00 +01:00
|
|
|
FileMetaData* f = c->input(0, 0);
|
|
|
|
c->edit()->DeleteFile(c->level(), f->number);
|
|
|
|
c->edit()->AddFile(c->level() + 1, f->number, f->file_size,
|
2013-06-14 07:09:08 +02:00
|
|
|
f->smallest, f->largest,
|
|
|
|
f->smallest_seqno, f->largest_seqno);
|
2014-01-27 22:55:47 +01:00
|
|
|
status = versions_->LogAndApply(default_cfd_, c->edit(), &mutex_,
|
|
|
|
db_directory_.get());
|
2014-01-24 23:30:28 +01:00
|
|
|
InstallSuperVersion(default_cfd_, deletion_state);
|
|
|
|
|
2014-01-16 01:18:04 +01:00
|
|
|
Version::LevelSummaryStorage tmp;
|
2011-07-21 04:40:18 +02:00
|
|
|
Log(options_.info_log, "Moved #%lld to level-%d %lld bytes %s: %s\n",
|
2014-01-16 01:18:04 +01:00
|
|
|
static_cast<unsigned long long>(f->number), c->level() + 1,
|
2011-03-18 23:37:00 +01:00
|
|
|
static_cast<unsigned long long>(f->file_size),
|
2014-01-29 22:28:50 +01:00
|
|
|
status.ToString().c_str(), default_cfd_->current()->LevelSummary(&tmp));
|
2013-01-20 11:07:13 +01:00
|
|
|
versions_->ReleaseCompactionFiles(c.get(), status);
|
2012-10-19 23:00:53 +02:00
|
|
|
*madeProgress = true;
|
2011-03-18 23:37:00 +01:00
|
|
|
} else {
|
2013-10-15 00:12:15 +02:00
|
|
|
MaybeScheduleFlushOrCompaction(); // do more compaction work in parallel.
|
2013-01-20 11:07:13 +01:00
|
|
|
CompactionState* compact = new CompactionState(c.get());
|
2013-11-09 00:23:46 +01:00
|
|
|
status = DoCompactionWork(compact, deletion_state);
|
2013-09-02 08:23:40 +02:00
|
|
|
CleanupCompaction(compact, status);
|
2013-01-20 11:07:13 +01:00
|
|
|
versions_->ReleaseCompactionFiles(c.get(), status);
|
2012-01-25 23:56:52 +01:00
|
|
|
c->ReleaseInputs();
|
2012-10-19 23:00:53 +02:00
|
|
|
*madeProgress = true;
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
2013-01-20 11:07:13 +01:00
|
|
|
c.reset();
|
2011-03-18 23:37:00 +01:00
|
|
|
|
|
|
|
if (status.ok()) {
|
|
|
|
// Done
|
|
|
|
} else if (shutting_down_.Acquire_Load()) {
|
|
|
|
// Ignore compaction errors found during shutting down
|
|
|
|
} else {
|
2011-07-21 04:40:18 +02:00
|
|
|
Log(options_.info_log,
|
2011-03-18 23:37:00 +01:00
|
|
|
"Compaction error: %s", status.ToString().c_str());
|
|
|
|
if (options_.paranoid_checks && bg_error_.ok()) {
|
|
|
|
bg_error_ = status;
|
|
|
|
}
|
|
|
|
}
|
2011-06-07 16:40:26 +02:00
|
|
|
|
|
|
|
if (is_manual) {
|
2011-10-06 01:30:28 +02:00
|
|
|
ManualCompaction* m = manual_compaction_;
|
2012-01-25 23:56:52 +01:00
|
|
|
if (!status.ok()) {
|
2014-01-22 21:46:24 +01:00
|
|
|
m->status = status;
|
2012-01-25 23:56:52 +01:00
|
|
|
m->done = true;
|
|
|
|
}
|
2013-08-02 20:46:47 +02:00
|
|
|
// For universal compaction:
|
|
|
|
// Because universal compaction always happens at level 0, so one
|
|
|
|
// compaction will pick up all overlapped files. No files will be
|
|
|
|
// filtered out due to size limit and left for a successive compaction.
|
|
|
|
// So we can safely conclude the current compaction.
|
|
|
|
//
|
|
|
|
// Also note that, if we don't stop here, then the current compaction
|
|
|
|
// writes a new file back to level 0, which will be used in successive
|
|
|
|
// compaction. Hence the manual compaction will never finish.
|
2014-01-15 01:19:09 +01:00
|
|
|
//
|
|
|
|
// Stop the compaction if manual_end points to nullptr -- this means
|
|
|
|
// that we compacted the whole range. manual_end should always point
|
|
|
|
// to nullptr in case of universal compaction
|
|
|
|
if (manual_end == nullptr) {
|
2013-08-02 20:46:47 +02:00
|
|
|
m->done = true;
|
|
|
|
}
|
2011-10-06 01:30:28 +02:00
|
|
|
if (!m->done) {
|
|
|
|
// We only compacted part of the requested range. Update *m
|
|
|
|
// to the range that is left to be compacted.
|
2014-01-15 01:19:09 +01:00
|
|
|
// Universal compaction should always compact the whole range
|
|
|
|
assert(options_.compaction_style != kCompactionStyleUniversal);
|
|
|
|
m->tmp_storage = *manual_end;
|
2011-10-06 01:30:28 +02:00
|
|
|
m->begin = &m->tmp_storage;
|
|
|
|
}
|
2012-10-19 23:00:53 +02:00
|
|
|
m->in_progress = false; // not being processed anymore
|
2013-02-15 20:53:17 +01:00
|
|
|
manual_compaction_ = nullptr;
|
2011-06-07 16:40:26 +02:00
|
|
|
}
|
2012-08-23 01:57:51 +02:00
|
|
|
return status;
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2013-09-02 08:23:40 +02:00
|
|
|
void DBImpl::CleanupCompaction(CompactionState* compact, Status status) {
|
2011-03-18 23:37:00 +01:00
|
|
|
mutex_.AssertHeld();
|
2013-02-15 20:53:17 +01:00
|
|
|
if (compact->builder != nullptr) {
|
2011-03-18 23:37:00 +01:00
|
|
|
// May happen if we get a shutdown call in the middle of compaction
|
|
|
|
compact->builder->Abandon();
|
2013-01-20 11:07:13 +01:00
|
|
|
compact->builder.reset();
|
2011-03-18 23:37:00 +01:00
|
|
|
} else {
|
2013-02-15 20:53:17 +01:00
|
|
|
assert(compact->outfile == nullptr);
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
2011-04-21 00:48:11 +02:00
|
|
|
for (size_t i = 0; i < compact->outputs.size(); i++) {
|
2011-03-18 23:37:00 +01:00
|
|
|
const CompactionState::Output& out = compact->outputs[i];
|
|
|
|
pending_outputs_.erase(out.number);
|
2013-09-02 08:23:40 +02:00
|
|
|
|
|
|
|
// If this file was inserted into the table cache then remove
|
|
|
|
// them here because this compaction was not committed.
|
|
|
|
if (!status.ok()) {
|
|
|
|
table_cache_->Evict(out.number);
|
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
delete compact;
|
|
|
|
}
|
|
|
|
|
2012-10-19 23:00:53 +02:00
|
|
|
// Allocate the file numbers for the output file. We allocate as
|
2014-01-15 01:19:09 +01:00
|
|
|
// many output file numbers as there are files in level+1 (at least one)
|
2012-10-19 23:00:53 +02:00
|
|
|
// Insert them into pending_outputs so that they do not get deleted.
|
|
|
|
void DBImpl::AllocateCompactionOutputFileNumbers(CompactionState* compact) {
|
|
|
|
mutex_.AssertHeld();
|
2013-02-15 20:53:17 +01:00
|
|
|
assert(compact != nullptr);
|
|
|
|
assert(compact->builder == nullptr);
|
2012-10-19 23:00:53 +02:00
|
|
|
int filesNeeded = compact->compaction->num_input_files(1);
|
2014-01-15 01:19:09 +01:00
|
|
|
for (int i = 0; i < std::max(filesNeeded, 1); i++) {
|
2012-10-19 23:00:53 +02:00
|
|
|
uint64_t file_number = versions_->NewFileNumber();
|
|
|
|
pending_outputs_.insert(file_number);
|
|
|
|
compact->allocated_file_numbers.push_back(file_number);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Frees up unused file number.
|
|
|
|
void DBImpl::ReleaseCompactionUnusedFileNumbers(CompactionState* compact) {
|
|
|
|
mutex_.AssertHeld();
|
2013-03-29 02:01:06 +01:00
|
|
|
for (const auto file_number : compact->allocated_file_numbers) {
|
2012-10-19 23:00:53 +02:00
|
|
|
pending_outputs_.erase(file_number);
|
|
|
|
// Log(options_.info_log, "XXX releasing unused file num %d", file_number);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
Status DBImpl::OpenCompactionOutputFile(CompactionState* compact) {
|
2013-02-15 20:53:17 +01:00
|
|
|
assert(compact != nullptr);
|
|
|
|
assert(compact->builder == nullptr);
|
2011-03-18 23:37:00 +01:00
|
|
|
uint64_t file_number;
|
2012-10-19 23:00:53 +02:00
|
|
|
// If we have not yet exhausted the pre-allocated file numbers,
|
|
|
|
// then use the one from the front. Otherwise, we have to acquire
|
|
|
|
// the heavyweight lock and allocate a new file number.
|
|
|
|
if (!compact->allocated_file_numbers.empty()) {
|
|
|
|
file_number = compact->allocated_file_numbers.front();
|
|
|
|
compact->allocated_file_numbers.pop_front();
|
|
|
|
} else {
|
2011-03-18 23:37:00 +01:00
|
|
|
mutex_.Lock();
|
|
|
|
file_number = versions_->NewFileNumber();
|
|
|
|
pending_outputs_.insert(file_number);
|
|
|
|
mutex_.Unlock();
|
|
|
|
}
|
2012-10-19 23:00:53 +02:00
|
|
|
CompactionState::Output out;
|
|
|
|
out.number = file_number;
|
|
|
|
out.smallest.Clear();
|
|
|
|
out.largest.Clear();
|
2013-06-14 07:09:08 +02:00
|
|
|
out.smallest_seqno = out.largest_seqno = 0;
|
2012-10-19 23:00:53 +02:00
|
|
|
compact->outputs.push_back(out);
|
2011-03-18 23:37:00 +01:00
|
|
|
|
|
|
|
// Make the output file
|
|
|
|
std::string fname = TableFileName(dbname_, file_number);
|
2013-03-15 01:00:04 +01:00
|
|
|
Status s = env_->NewWritableFile(fname, &compact->outfile, storage_options_);
|
2013-01-15 23:05:42 +01:00
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
if (s.ok()) {
|
2013-04-10 17:37:03 +02:00
|
|
|
// Over-estimate slightly so we don't end up just barely crossing
|
|
|
|
// the threshold.
|
|
|
|
compact->outfile->SetPreallocationBlockSize(
|
2013-06-14 07:09:08 +02:00
|
|
|
1.1 * versions_->MaxFileSizeForLevel(compact->compaction->output_level()));
|
2013-04-10 17:37:03 +02:00
|
|
|
|
2013-10-30 18:52:33 +01:00
|
|
|
CompressionType compression_type = GetCompressionType(
|
|
|
|
options_, compact->compaction->output_level(),
|
|
|
|
compact->compaction->enable_compression());
|
|
|
|
|
2013-10-29 01:54:09 +01:00
|
|
|
compact->builder.reset(
|
2013-10-30 18:52:33 +01:00
|
|
|
GetTableBuilder(options_, compact->outfile.get(), compression_type));
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
2013-11-07 20:31:56 +01:00
|
|
|
LogFlush(options_.info_log);
|
2011-03-18 23:37:00 +01:00
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
|
|
|
Status DBImpl::FinishCompactionOutputFile(CompactionState* compact,
|
|
|
|
Iterator* input) {
|
2013-02-15 20:53:17 +01:00
|
|
|
assert(compact != nullptr);
|
2013-01-20 11:07:13 +01:00
|
|
|
assert(compact->outfile);
|
2013-02-15 20:53:17 +01:00
|
|
|
assert(compact->builder != nullptr);
|
2011-03-18 23:37:00 +01:00
|
|
|
|
|
|
|
const uint64_t output_number = compact->current_output()->number;
|
|
|
|
assert(output_number != 0);
|
|
|
|
|
|
|
|
// Check for iterator errors
|
|
|
|
Status s = input->status();
|
|
|
|
const uint64_t current_entries = compact->builder->NumEntries();
|
|
|
|
if (s.ok()) {
|
|
|
|
s = compact->builder->Finish();
|
|
|
|
} else {
|
|
|
|
compact->builder->Abandon();
|
|
|
|
}
|
|
|
|
const uint64_t current_bytes = compact->builder->FileSize();
|
|
|
|
compact->current_output()->file_size = current_bytes;
|
|
|
|
compact->total_bytes += current_bytes;
|
2013-01-20 11:07:13 +01:00
|
|
|
compact->builder.reset();
|
2011-03-18 23:37:00 +01:00
|
|
|
|
|
|
|
// Finish and check for file errors
|
2012-08-04 00:20:58 +02:00
|
|
|
if (s.ok() && !options_.disableDataSync) {
|
2012-08-27 21:10:26 +02:00
|
|
|
if (options_.use_fsync) {
|
2013-11-22 23:14:05 +01:00
|
|
|
StopWatch sw(env_, options_.statistics.get(),
|
2014-01-07 23:33:15 +01:00
|
|
|
COMPACTION_OUTFILE_SYNC_MICROS, false);
|
2012-08-27 21:10:26 +02:00
|
|
|
s = compact->outfile->Fsync();
|
|
|
|
} else {
|
2013-11-22 23:14:05 +01:00
|
|
|
StopWatch sw(env_, options_.statistics.get(),
|
2014-01-07 23:33:15 +01:00
|
|
|
COMPACTION_OUTFILE_SYNC_MICROS, false);
|
2012-08-27 21:10:26 +02:00
|
|
|
s = compact->outfile->Sync();
|
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
if (s.ok()) {
|
|
|
|
s = compact->outfile->Close();
|
|
|
|
}
|
2013-01-20 11:07:13 +01:00
|
|
|
compact->outfile.reset();
|
2011-03-18 23:37:00 +01:00
|
|
|
|
|
|
|
if (s.ok() && current_entries > 0) {
|
|
|
|
// Verify that the table is usable
|
2011-03-28 22:43:44 +02:00
|
|
|
Iterator* iter = table_cache_->NewIterator(ReadOptions(),
|
2013-03-15 01:00:04 +01:00
|
|
|
storage_options_,
|
2011-03-28 22:43:44 +02:00
|
|
|
output_number,
|
|
|
|
current_bytes);
|
2011-03-18 23:37:00 +01:00
|
|
|
s = iter->status();
|
|
|
|
delete iter;
|
|
|
|
if (s.ok()) {
|
2011-07-21 04:40:18 +02:00
|
|
|
Log(options_.info_log,
|
2013-11-13 06:02:03 +01:00
|
|
|
"Generated table #%lu: %lu keys, %lu bytes",
|
|
|
|
(unsigned long) output_number,
|
|
|
|
(unsigned long) current_entries,
|
|
|
|
(unsigned long) current_bytes);
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
Status DBImpl::InstallCompactionResults(CompactionState* compact) {
|
|
|
|
mutex_.AssertHeld();
|
2012-10-19 23:00:53 +02:00
|
|
|
|
|
|
|
// paranoia: verify that the files that we started with
|
|
|
|
// still exist in the current version and in the same original level.
|
|
|
|
// This ensures that a concurrent compaction did not erroneously
|
|
|
|
// pick the same files to compact.
|
2013-03-06 22:28:54 +01:00
|
|
|
if (!versions_->VerifyCompactionFileConsistency(compact->compaction)) {
|
2012-10-19 23:00:53 +02:00
|
|
|
Log(options_.info_log, "Compaction %d@%d + %d@%d files aborted",
|
|
|
|
compact->compaction->num_input_files(0),
|
|
|
|
compact->compaction->level(),
|
|
|
|
compact->compaction->num_input_files(1),
|
|
|
|
compact->compaction->level() + 1);
|
|
|
|
return Status::IOError("Compaction input files inconsistent");
|
|
|
|
}
|
|
|
|
|
2011-07-21 04:40:18 +02:00
|
|
|
Log(options_.info_log, "Compacted %d@%d + %d@%d files => %lld bytes",
|
2011-03-18 23:37:00 +01:00
|
|
|
compact->compaction->num_input_files(0),
|
|
|
|
compact->compaction->level(),
|
|
|
|
compact->compaction->num_input_files(1),
|
|
|
|
compact->compaction->level() + 1,
|
|
|
|
static_cast<long long>(compact->total_bytes));
|
|
|
|
|
|
|
|
// Add compaction outputs
|
|
|
|
compact->compaction->AddInputDeletions(compact->compaction->edit());
|
2011-04-21 00:48:11 +02:00
|
|
|
for (size_t i = 0; i < compact->outputs.size(); i++) {
|
2011-03-18 23:37:00 +01:00
|
|
|
const CompactionState::Output& out = compact->outputs[i];
|
|
|
|
compact->compaction->edit()->AddFile(
|
2014-01-15 01:19:09 +01:00
|
|
|
compact->compaction->output_level(), out.number, out.file_size,
|
|
|
|
out.smallest, out.largest, out.smallest_seqno, out.largest_seqno);
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
2014-01-27 22:55:47 +01:00
|
|
|
return versions_->LogAndApply(default_cfd_, compact->compaction->edit(),
|
|
|
|
&mutex_, db_directory_.get());
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2012-11-27 06:16:21 +01:00
|
|
|
//
|
|
|
|
// Given a sequence number, return the sequence number of the
|
|
|
|
// earliest snapshot that this sequence number is visible in.
|
|
|
|
// The snapshots themselves are arranged in ascending order of
|
|
|
|
// sequence numbers.
|
|
|
|
// Employ a sequential search because the total number of
|
|
|
|
// snapshots are typically small.
|
|
|
|
inline SequenceNumber DBImpl::findEarliestVisibleSnapshot(
|
2013-03-21 23:59:47 +01:00
|
|
|
SequenceNumber in, std::vector<SequenceNumber>& snapshots,
|
|
|
|
SequenceNumber* prev_snapshot) {
|
2013-01-14 21:39:24 +01:00
|
|
|
SequenceNumber prev __attribute__((unused)) = 0;
|
2013-03-29 02:01:06 +01:00
|
|
|
for (const auto cur : snapshots) {
|
|
|
|
assert(prev <= cur);
|
|
|
|
if (cur >= in) {
|
2013-03-21 23:59:47 +01:00
|
|
|
*prev_snapshot = prev;
|
2013-03-29 02:01:06 +01:00
|
|
|
return cur;
|
2012-11-27 06:16:21 +01:00
|
|
|
}
|
2013-03-29 02:01:06 +01:00
|
|
|
prev = cur; // assignment
|
|
|
|
assert(prev);
|
2012-11-27 06:16:21 +01:00
|
|
|
}
|
|
|
|
Log(options_.info_log,
|
2013-11-13 06:02:03 +01:00
|
|
|
"Looking for seqid %lu but maxseqid is %lu",
|
|
|
|
(unsigned long)in,
|
|
|
|
(unsigned long)snapshots[snapshots.size()-1]);
|
2012-11-27 06:16:21 +01:00
|
|
|
assert(0);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2013-11-09 00:23:46 +01:00
|
|
|
Status DBImpl::DoCompactionWork(CompactionState* compact,
|
|
|
|
DeletionState& deletion_state) {
|
2013-10-27 07:01:26 +01:00
|
|
|
assert(compact);
|
2011-04-12 21:38:58 +02:00
|
|
|
int64_t imm_micros = 0; // Micros spent doing imm_ compactions
|
2012-11-29 01:42:36 +01:00
|
|
|
Log(options_.info_log,
|
2012-12-04 18:47:05 +01:00
|
|
|
"Compacting %d@%d + %d@%d files, score %.2f slots available %d",
|
2011-03-18 23:37:00 +01:00
|
|
|
compact->compaction->num_input_files(0),
|
|
|
|
compact->compaction->level(),
|
|
|
|
compact->compaction->num_input_files(1),
|
2014-01-15 01:19:09 +01:00
|
|
|
compact->compaction->output_level(),
|
2012-12-04 18:47:05 +01:00
|
|
|
compact->compaction->score(),
|
2012-10-19 23:00:53 +02:00
|
|
|
options_.max_background_compactions - bg_compaction_scheduled_);
|
2012-09-21 19:47:08 +02:00
|
|
|
char scratch[256];
|
|
|
|
compact->compaction->Summary(scratch, sizeof(scratch));
|
2012-08-18 03:10:09 +02:00
|
|
|
Log(options_.info_log, "Compaction start summary: %s\n", scratch);
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2014-01-27 23:33:50 +01:00
|
|
|
assert(compact->compaction->input_version()->NumLevelFiles(
|
|
|
|
compact->compaction->level()) > 0);
|
2013-02-15 20:53:17 +01:00
|
|
|
assert(compact->builder == nullptr);
|
2013-01-20 11:07:13 +01:00
|
|
|
assert(!compact->outfile);
|
2012-11-27 06:16:21 +01:00
|
|
|
|
|
|
|
SequenceNumber visible_at_tip = 0;
|
|
|
|
SequenceNumber earliest_snapshot;
|
2013-06-06 19:31:37 +02:00
|
|
|
SequenceNumber latest_snapshot = 0;
|
2012-11-27 06:16:21 +01:00
|
|
|
snapshots_.getAll(compact->existing_snapshots);
|
|
|
|
if (compact->existing_snapshots.size() == 0) {
|
|
|
|
// optimize for fast path if there are no snapshots
|
|
|
|
visible_at_tip = versions_->LastSequence();
|
|
|
|
earliest_snapshot = visible_at_tip;
|
2011-03-18 23:37:00 +01:00
|
|
|
} else {
|
2013-06-06 19:31:37 +02:00
|
|
|
latest_snapshot = compact->existing_snapshots.back();
|
2012-11-27 06:16:21 +01:00
|
|
|
// Add the current seqno as the 'latest' virtual
|
|
|
|
// snapshot to the end of this list.
|
|
|
|
compact->existing_snapshots.push_back(versions_->LastSequence());
|
|
|
|
earliest_snapshot = compact->existing_snapshots[0];
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2013-02-15 23:31:24 +01:00
|
|
|
// Is this compaction producing files at the bottommost level?
|
2013-08-08 00:25:00 +02:00
|
|
|
bool bottommost_level = compact->compaction->BottomMostLevel();
|
2013-02-15 23:31:24 +01:00
|
|
|
|
2012-10-19 23:00:53 +02:00
|
|
|
// Allocate the output file numbers before we release the lock
|
|
|
|
AllocateCompactionOutputFileNumbers(compact);
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
// Release mutex while we're actually doing the compaction work
|
|
|
|
mutex_.Unlock();
|
|
|
|
|
2012-10-19 23:00:53 +02:00
|
|
|
const uint64_t start_micros = env_->NowMicros();
|
2013-01-20 11:07:13 +01:00
|
|
|
unique_ptr<Iterator> input(versions_->MakeInputIterator(compact->compaction));
|
2011-03-18 23:37:00 +01:00
|
|
|
input->SeekToFirst();
|
|
|
|
Status status;
|
|
|
|
ParsedInternalKey ikey;
|
|
|
|
std::string current_user_key;
|
|
|
|
bool has_current_user_key = false;
|
2013-01-14 21:39:24 +01:00
|
|
|
SequenceNumber last_sequence_for_key __attribute__((unused)) =
|
|
|
|
kMaxSequenceNumber;
|
2012-11-27 06:16:21 +01:00
|
|
|
SequenceNumber visible_in_snapshot = kMaxSequenceNumber;
|
2013-04-18 06:30:21 +02:00
|
|
|
std::string compaction_filter_value;
|
2013-06-06 19:31:37 +02:00
|
|
|
std::vector<char> delete_key; // for compaction filter
|
2013-08-20 22:35:28 +02:00
|
|
|
MergeHelper merge(user_comparator(), options_.merge_operator.get(),
|
2013-03-21 23:59:47 +01:00
|
|
|
options_.info_log.get(),
|
|
|
|
false /* internal key corruption is expected */);
|
2013-08-13 19:56:20 +02:00
|
|
|
auto compaction_filter = options_.compaction_filter;
|
|
|
|
std::unique_ptr<CompactionFilter> compaction_filter_from_factory = nullptr;
|
|
|
|
if (!compaction_filter) {
|
2013-10-27 07:01:26 +01:00
|
|
|
auto context = compact->GetFilterContext();
|
|
|
|
compaction_filter_from_factory =
|
|
|
|
options_.compaction_filter_factory->CreateCompactionFilter(context);
|
2013-08-13 19:56:20 +02:00
|
|
|
compaction_filter = compaction_filter_from_factory.get();
|
|
|
|
}
|
2013-10-27 07:01:26 +01:00
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
for (; input->Valid() && !shutting_down_.Acquire_Load(); ) {
|
2011-04-12 21:38:58 +02:00
|
|
|
// Prioritize immutable compaction work
|
2013-09-13 23:38:37 +02:00
|
|
|
// TODO: remove memtable flush from normal compaction work
|
2014-01-29 22:28:50 +01:00
|
|
|
if (default_cfd_->imm()->imm_flush_needed.NoBarrier_Load() != nullptr) {
|
2011-04-12 21:38:58 +02:00
|
|
|
const uint64_t imm_start = env_->NowMicros();
|
2013-11-07 20:31:56 +01:00
|
|
|
LogFlush(options_.info_log);
|
2011-04-12 21:38:58 +02:00
|
|
|
mutex_.Lock();
|
2014-01-29 22:28:50 +01:00
|
|
|
if (default_cfd_->imm()->IsFlushPending()) {
|
2013-11-09 00:23:46 +01:00
|
|
|
FlushMemTableToOutputFile(nullptr, deletion_state);
|
2011-06-07 16:40:26 +02:00
|
|
|
bg_cv_.SignalAll(); // Wakeup MakeRoomForWrite() if necessary
|
2011-04-12 21:38:58 +02:00
|
|
|
}
|
|
|
|
mutex_.Unlock();
|
|
|
|
imm_micros += (env_->NowMicros() - imm_start);
|
|
|
|
}
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
Slice key = input->key();
|
2012-10-29 09:13:41 +01:00
|
|
|
Slice value = input->value();
|
2013-04-18 06:30:21 +02:00
|
|
|
|
2011-05-21 04:17:43 +02:00
|
|
|
if (compact->compaction->ShouldStopBefore(key) &&
|
2013-02-15 20:53:17 +01:00
|
|
|
compact->builder != nullptr) {
|
2013-01-20 11:07:13 +01:00
|
|
|
status = FinishCompactionOutputFile(compact, input.get());
|
2011-03-22 19:32:49 +01:00
|
|
|
if (!status.ok()) {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Handle key/value, add to state, etc.
|
2011-03-18 23:37:00 +01:00
|
|
|
bool drop = false;
|
[RocksDB] [MergeOperator] The new Merge Interface! Uses merge sequences.
Summary:
Here are the major changes to the Merge Interface. It has been expanded
to handle cases where the MergeOperator is not associative. It does so by stacking
up merge operations while scanning through the key history (i.e.: during Get() or
Compaction), until a valid Put/Delete/end-of-history is encountered; it then
applies all of the merge operations in the correct sequence starting with the
base/sentinel value.
I have also introduced an "AssociativeMerge" function which allows the user to
take advantage of associative merge operations (such as in the case of counters).
The implementation will always attempt to merge the operations/operands themselves
together when they are encountered, and will resort to the "stacking" method if
and only if the "associative-merge" fails.
This implementation is conjectured to allow MergeOperator to handle the general
case, while still providing the user with the ability to take advantage of certain
efficiencies in their own merge-operator / data-structure.
NOTE: This is a preliminary diff. This must still go through a lot of review,
revision, and testing. Feedback welcome!
Test Plan:
-This is a preliminary diff. I have only just begun testing/debugging it.
-I will be testing this with the existing MergeOperator use-cases and unit-tests
(counters, string-append, and redis-lists)
-I will be "desk-checking" and walking through the code with the help gdb.
-I will find a way of stress-testing the new interface / implementation using
db_bench, db_test, merge_test, and/or db_stress.
-I will ensure that my tests cover all cases: Get-Memtable,
Get-Immutable-Memtable, Get-from-Disk, Iterator-Range-Scan, Flush-Memtable-to-L0,
Compaction-L0-L1, Compaction-Ln-L(n+1), Put/Delete found, Put/Delete not-found,
end-of-history, end-of-file, etc.
-A lot of feedback from the reviewers.
Reviewers: haobo, dhruba, zshao, emayanke
Reviewed By: haobo
CC: leveldb
Differential Revision: https://reviews.facebook.net/D11499
2013-08-06 05:14:32 +02:00
|
|
|
bool current_entry_is_merging = false;
|
2011-03-18 23:37:00 +01:00
|
|
|
if (!ParseInternalKey(key, &ikey)) {
|
|
|
|
// Do not hide error keys
|
2013-03-21 23:59:47 +01:00
|
|
|
// TODO: error key stays in db forever? Figure out the intention/rationale
|
|
|
|
// v10 error v8 : we cannot hide v8 even though it's pretty obvious.
|
2011-03-18 23:37:00 +01:00
|
|
|
current_user_key.clear();
|
|
|
|
has_current_user_key = false;
|
|
|
|
last_sequence_for_key = kMaxSequenceNumber;
|
2012-11-27 06:16:21 +01:00
|
|
|
visible_in_snapshot = kMaxSequenceNumber;
|
2011-03-18 23:37:00 +01:00
|
|
|
} else {
|
|
|
|
if (!has_current_user_key ||
|
|
|
|
user_comparator()->Compare(ikey.user_key,
|
|
|
|
Slice(current_user_key)) != 0) {
|
|
|
|
// First occurrence of this user key
|
|
|
|
current_user_key.assign(ikey.user_key.data(), ikey.user_key.size());
|
|
|
|
has_current_user_key = true;
|
|
|
|
last_sequence_for_key = kMaxSequenceNumber;
|
2012-11-27 06:16:21 +01:00
|
|
|
visible_in_snapshot = kMaxSequenceNumber;
|
2013-06-06 19:31:37 +02:00
|
|
|
|
|
|
|
// apply the compaction filter to the first occurrence of the user key
|
2013-08-13 19:56:20 +02:00
|
|
|
if (compaction_filter &&
|
2013-06-06 19:31:37 +02:00
|
|
|
ikey.type == kTypeValue &&
|
|
|
|
(visible_at_tip || ikey.sequence > latest_snapshot)) {
|
|
|
|
// If the user has specified a compaction filter and the sequence
|
|
|
|
// number is greater than any external snapshot, then invoke the
|
|
|
|
// filter.
|
|
|
|
// If the return value of the compaction filter is true, replace
|
|
|
|
// the entry with a delete marker.
|
|
|
|
bool value_changed = false;
|
|
|
|
compaction_filter_value.clear();
|
|
|
|
bool to_delete =
|
2013-08-13 19:56:20 +02:00
|
|
|
compaction_filter->Filter(compact->compaction->level(),
|
2013-10-29 01:54:09 +01:00
|
|
|
ikey.user_key, value,
|
|
|
|
&compaction_filter_value,
|
|
|
|
&value_changed);
|
2013-06-06 19:31:37 +02:00
|
|
|
if (to_delete) {
|
|
|
|
// make a copy of the original key
|
|
|
|
delete_key.assign(key.data(), key.data() + key.size());
|
|
|
|
// convert it to a delete
|
|
|
|
UpdateInternalKey(&delete_key[0], delete_key.size(),
|
|
|
|
ikey.sequence, kTypeDeletion);
|
|
|
|
// anchor the key again
|
|
|
|
key = Slice(&delete_key[0], delete_key.size());
|
|
|
|
// needed because ikey is backed by key
|
|
|
|
ParseInternalKey(key, &ikey);
|
|
|
|
// no value associated with delete
|
|
|
|
value.clear();
|
2013-11-22 23:14:05 +01:00
|
|
|
RecordTick(options_.statistics.get(), COMPACTION_KEY_DROP_USER);
|
2013-06-06 19:31:37 +02:00
|
|
|
} else if (value_changed) {
|
|
|
|
value = compaction_filter_value;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2012-11-27 06:16:21 +01:00
|
|
|
// If there are no snapshots, then this kv affect visibility at tip.
|
|
|
|
// Otherwise, search though all existing snapshots to find
|
|
|
|
// the earlist snapshot that is affected by this kv.
|
2013-03-21 23:59:47 +01:00
|
|
|
SequenceNumber prev_snapshot = 0; // 0 means no previous snapshot
|
|
|
|
SequenceNumber visible = visible_at_tip ?
|
|
|
|
visible_at_tip :
|
|
|
|
findEarliestVisibleSnapshot(ikey.sequence,
|
|
|
|
compact->existing_snapshots,
|
|
|
|
&prev_snapshot);
|
2012-11-27 06:16:21 +01:00
|
|
|
|
|
|
|
if (visible_in_snapshot == visible) {
|
|
|
|
// If the earliest snapshot is which this key is visible in
|
|
|
|
// is the same as the visibily of a previous instance of the
|
|
|
|
// same key, then this kv is not visible in any snapshot.
|
2011-03-18 23:37:00 +01:00
|
|
|
// Hidden by an newer entry for same user key
|
2013-03-21 23:59:47 +01:00
|
|
|
// TODO: why not > ?
|
2012-11-27 06:16:21 +01:00
|
|
|
assert(last_sequence_for_key >= ikey.sequence);
|
2011-03-18 23:37:00 +01:00
|
|
|
drop = true; // (A)
|
2013-11-22 23:14:05 +01:00
|
|
|
RecordTick(options_.statistics.get(), COMPACTION_KEY_DROP_NEWER_ENTRY);
|
2011-03-18 23:37:00 +01:00
|
|
|
} else if (ikey.type == kTypeDeletion &&
|
2012-11-27 06:16:21 +01:00
|
|
|
ikey.sequence <= earliest_snapshot &&
|
2011-03-18 23:37:00 +01:00
|
|
|
compact->compaction->IsBaseLevelForKey(ikey.user_key)) {
|
|
|
|
// For this user key:
|
|
|
|
// (1) there is no data in higher levels
|
|
|
|
// (2) data in lower levels will have larger sequence numbers
|
|
|
|
// (3) data in layers that are being compacted here and have
|
|
|
|
// smaller sequence numbers will be dropped in the next
|
|
|
|
// few iterations of this loop (by rule (A) above).
|
|
|
|
// Therefore this deletion marker is obsolete and can be dropped.
|
|
|
|
drop = true;
|
2013-11-22 23:14:05 +01:00
|
|
|
RecordTick(options_.statistics.get(), COMPACTION_KEY_DROP_OBSOLETE);
|
2013-03-21 23:59:47 +01:00
|
|
|
} else if (ikey.type == kTypeMerge) {
|
|
|
|
// We know the merge type entry is not hidden, otherwise we would
|
|
|
|
// have hit (A)
|
|
|
|
// We encapsulate the merge related state machine in a different
|
|
|
|
// object to minimize change to the existing flow. Turn out this
|
|
|
|
// logic could also be nicely re-used for memtable flush purge
|
|
|
|
// optimization in BuildTable.
|
2013-08-09 08:07:36 +02:00
|
|
|
merge.MergeUntil(input.get(), prev_snapshot, bottommost_level,
|
2013-11-22 23:14:05 +01:00
|
|
|
options_.statistics.get());
|
[RocksDB] [MergeOperator] The new Merge Interface! Uses merge sequences.
Summary:
Here are the major changes to the Merge Interface. It has been expanded
to handle cases where the MergeOperator is not associative. It does so by stacking
up merge operations while scanning through the key history (i.e.: during Get() or
Compaction), until a valid Put/Delete/end-of-history is encountered; it then
applies all of the merge operations in the correct sequence starting with the
base/sentinel value.
I have also introduced an "AssociativeMerge" function which allows the user to
take advantage of associative merge operations (such as in the case of counters).
The implementation will always attempt to merge the operations/operands themselves
together when they are encountered, and will resort to the "stacking" method if
and only if the "associative-merge" fails.
This implementation is conjectured to allow MergeOperator to handle the general
case, while still providing the user with the ability to take advantage of certain
efficiencies in their own merge-operator / data-structure.
NOTE: This is a preliminary diff. This must still go through a lot of review,
revision, and testing. Feedback welcome!
Test Plan:
-This is a preliminary diff. I have only just begun testing/debugging it.
-I will be testing this with the existing MergeOperator use-cases and unit-tests
(counters, string-append, and redis-lists)
-I will be "desk-checking" and walking through the code with the help gdb.
-I will find a way of stress-testing the new interface / implementation using
db_bench, db_test, merge_test, and/or db_stress.
-I will ensure that my tests cover all cases: Get-Memtable,
Get-Immutable-Memtable, Get-from-Disk, Iterator-Range-Scan, Flush-Memtable-to-L0,
Compaction-L0-L1, Compaction-Ln-L(n+1), Put/Delete found, Put/Delete not-found,
end-of-history, end-of-file, etc.
-A lot of feedback from the reviewers.
Reviewers: haobo, dhruba, zshao, emayanke
Reviewed By: haobo
CC: leveldb
Differential Revision: https://reviews.facebook.net/D11499
2013-08-06 05:14:32 +02:00
|
|
|
current_entry_is_merging = true;
|
|
|
|
if (merge.IsSuccess()) {
|
|
|
|
// Successfully found Put/Delete/(end-of-key-range) while merging
|
|
|
|
// Get the merge result
|
|
|
|
key = merge.key();
|
|
|
|
ParseInternalKey(key, &ikey);
|
|
|
|
value = merge.value();
|
|
|
|
} else {
|
|
|
|
// Did not find a Put/Delete/(end-of-key-range) while merging
|
|
|
|
// We now have some stack of merge operands to write out.
|
|
|
|
// NOTE: key,value, and ikey are now referring to old entries.
|
|
|
|
// These will be correctly set below.
|
|
|
|
assert(!merge.keys().empty());
|
|
|
|
assert(merge.keys().size() == merge.values().size());
|
|
|
|
|
|
|
|
// Hack to make sure last_sequence_for_key is correct
|
|
|
|
ParseInternalKey(merge.keys().front(), &ikey);
|
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
last_sequence_for_key = ikey.sequence;
|
2012-11-27 06:16:21 +01:00
|
|
|
visible_in_snapshot = visible;
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
#if 0
|
2011-07-21 04:40:18 +02:00
|
|
|
Log(options_.info_log,
|
2011-03-18 23:37:00 +01:00
|
|
|
" Compact: %s, seq %d, type: %d %d, drop: %d, is_base: %d, "
|
2013-02-15 23:31:24 +01:00
|
|
|
"%d smallest_snapshot: %d level: %d bottommost %d",
|
2011-03-18 23:37:00 +01:00
|
|
|
ikey.user_key.ToString().c_str(),
|
2011-04-21 00:48:11 +02:00
|
|
|
(int)ikey.sequence, ikey.type, kTypeValue, drop,
|
2011-03-18 23:37:00 +01:00
|
|
|
compact->compaction->IsBaseLevelForKey(ikey.user_key),
|
2013-02-15 23:31:24 +01:00
|
|
|
(int)last_sequence_for_key, (int)earliest_snapshot,
|
|
|
|
compact->compaction->level(), bottommost_level);
|
2011-03-18 23:37:00 +01:00
|
|
|
#endif
|
|
|
|
|
|
|
|
if (!drop) {
|
[RocksDB] [MergeOperator] The new Merge Interface! Uses merge sequences.
Summary:
Here are the major changes to the Merge Interface. It has been expanded
to handle cases where the MergeOperator is not associative. It does so by stacking
up merge operations while scanning through the key history (i.e.: during Get() or
Compaction), until a valid Put/Delete/end-of-history is encountered; it then
applies all of the merge operations in the correct sequence starting with the
base/sentinel value.
I have also introduced an "AssociativeMerge" function which allows the user to
take advantage of associative merge operations (such as in the case of counters).
The implementation will always attempt to merge the operations/operands themselves
together when they are encountered, and will resort to the "stacking" method if
and only if the "associative-merge" fails.
This implementation is conjectured to allow MergeOperator to handle the general
case, while still providing the user with the ability to take advantage of certain
efficiencies in their own merge-operator / data-structure.
NOTE: This is a preliminary diff. This must still go through a lot of review,
revision, and testing. Feedback welcome!
Test Plan:
-This is a preliminary diff. I have only just begun testing/debugging it.
-I will be testing this with the existing MergeOperator use-cases and unit-tests
(counters, string-append, and redis-lists)
-I will be "desk-checking" and walking through the code with the help gdb.
-I will find a way of stress-testing the new interface / implementation using
db_bench, db_test, merge_test, and/or db_stress.
-I will ensure that my tests cover all cases: Get-Memtable,
Get-Immutable-Memtable, Get-from-Disk, Iterator-Range-Scan, Flush-Memtable-to-L0,
Compaction-L0-L1, Compaction-Ln-L(n+1), Put/Delete found, Put/Delete not-found,
end-of-history, end-of-file, etc.
-A lot of feedback from the reviewers.
Reviewers: haobo, dhruba, zshao, emayanke
Reviewed By: haobo
CC: leveldb
Differential Revision: https://reviews.facebook.net/D11499
2013-08-06 05:14:32 +02:00
|
|
|
// We may write a single key (e.g.: for Put/Delete or successful merge).
|
|
|
|
// Or we may instead have to write a sequence/list of keys.
|
|
|
|
// We have to write a sequence iff we have an unsuccessful merge
|
|
|
|
bool has_merge_list = current_entry_is_merging && !merge.IsSuccess();
|
|
|
|
const std::deque<std::string>* keys = nullptr;
|
|
|
|
const std::deque<std::string>* values = nullptr;
|
|
|
|
std::deque<std::string>::const_reverse_iterator key_iter;
|
|
|
|
std::deque<std::string>::const_reverse_iterator value_iter;
|
|
|
|
if (has_merge_list) {
|
|
|
|
keys = &merge.keys();
|
|
|
|
values = &merge.values();
|
|
|
|
key_iter = keys->rbegin(); // The back (*rbegin()) is the first key
|
|
|
|
value_iter = values->rbegin();
|
|
|
|
|
|
|
|
key = Slice(*key_iter);
|
|
|
|
value = Slice(*value_iter);
|
2013-02-15 23:31:24 +01:00
|
|
|
}
|
2013-02-21 04:22:13 +01:00
|
|
|
|
[RocksDB] [MergeOperator] The new Merge Interface! Uses merge sequences.
Summary:
Here are the major changes to the Merge Interface. It has been expanded
to handle cases where the MergeOperator is not associative. It does so by stacking
up merge operations while scanning through the key history (i.e.: during Get() or
Compaction), until a valid Put/Delete/end-of-history is encountered; it then
applies all of the merge operations in the correct sequence starting with the
base/sentinel value.
I have also introduced an "AssociativeMerge" function which allows the user to
take advantage of associative merge operations (such as in the case of counters).
The implementation will always attempt to merge the operations/operands themselves
together when they are encountered, and will resort to the "stacking" method if
and only if the "associative-merge" fails.
This implementation is conjectured to allow MergeOperator to handle the general
case, while still providing the user with the ability to take advantage of certain
efficiencies in their own merge-operator / data-structure.
NOTE: This is a preliminary diff. This must still go through a lot of review,
revision, and testing. Feedback welcome!
Test Plan:
-This is a preliminary diff. I have only just begun testing/debugging it.
-I will be testing this with the existing MergeOperator use-cases and unit-tests
(counters, string-append, and redis-lists)
-I will be "desk-checking" and walking through the code with the help gdb.
-I will find a way of stress-testing the new interface / implementation using
db_bench, db_test, merge_test, and/or db_stress.
-I will ensure that my tests cover all cases: Get-Memtable,
Get-Immutable-Memtable, Get-from-Disk, Iterator-Range-Scan, Flush-Memtable-to-L0,
Compaction-L0-L1, Compaction-Ln-L(n+1), Put/Delete found, Put/Delete not-found,
end-of-history, end-of-file, etc.
-A lot of feedback from the reviewers.
Reviewers: haobo, dhruba, zshao, emayanke
Reviewed By: haobo
CC: leveldb
Differential Revision: https://reviews.facebook.net/D11499
2013-08-06 05:14:32 +02:00
|
|
|
// If we have a list of keys to write, traverse the list.
|
|
|
|
// If we have a single key to write, simply write that key.
|
|
|
|
while (true) {
|
|
|
|
// Invariant: key,value,ikey will always be the next entry to write
|
|
|
|
char* kptr = (char*)key.data();
|
|
|
|
std::string kstr;
|
|
|
|
|
|
|
|
// Zeroing out the sequence number leads to better compression.
|
|
|
|
// If this is the bottommost level (no files in lower levels)
|
|
|
|
// and the earliest snapshot is larger than this seqno
|
|
|
|
// then we can squash the seqno to zero.
|
2013-08-07 20:58:06 +02:00
|
|
|
if (options_.compaction_style == kCompactionStyleLevel &&
|
|
|
|
bottommost_level && ikey.sequence < earliest_snapshot &&
|
[RocksDB] [MergeOperator] The new Merge Interface! Uses merge sequences.
Summary:
Here are the major changes to the Merge Interface. It has been expanded
to handle cases where the MergeOperator is not associative. It does so by stacking
up merge operations while scanning through the key history (i.e.: during Get() or
Compaction), until a valid Put/Delete/end-of-history is encountered; it then
applies all of the merge operations in the correct sequence starting with the
base/sentinel value.
I have also introduced an "AssociativeMerge" function which allows the user to
take advantage of associative merge operations (such as in the case of counters).
The implementation will always attempt to merge the operations/operands themselves
together when they are encountered, and will resort to the "stacking" method if
and only if the "associative-merge" fails.
This implementation is conjectured to allow MergeOperator to handle the general
case, while still providing the user with the ability to take advantage of certain
efficiencies in their own merge-operator / data-structure.
NOTE: This is a preliminary diff. This must still go through a lot of review,
revision, and testing. Feedback welcome!
Test Plan:
-This is a preliminary diff. I have only just begun testing/debugging it.
-I will be testing this with the existing MergeOperator use-cases and unit-tests
(counters, string-append, and redis-lists)
-I will be "desk-checking" and walking through the code with the help gdb.
-I will find a way of stress-testing the new interface / implementation using
db_bench, db_test, merge_test, and/or db_stress.
-I will ensure that my tests cover all cases: Get-Memtable,
Get-Immutable-Memtable, Get-from-Disk, Iterator-Range-Scan, Flush-Memtable-to-L0,
Compaction-L0-L1, Compaction-Ln-L(n+1), Put/Delete found, Put/Delete not-found,
end-of-history, end-of-file, etc.
-A lot of feedback from the reviewers.
Reviewers: haobo, dhruba, zshao, emayanke
Reviewed By: haobo
CC: leveldb
Differential Revision: https://reviews.facebook.net/D11499
2013-08-06 05:14:32 +02:00
|
|
|
ikey.type != kTypeMerge) {
|
|
|
|
assert(ikey.type != kTypeDeletion);
|
|
|
|
// make a copy because updating in place would cause problems
|
|
|
|
// with the priority queue that is managing the input key iterator
|
|
|
|
kstr.assign(key.data(), key.size());
|
|
|
|
kptr = (char *)kstr.c_str();
|
|
|
|
UpdateInternalKey(kptr, key.size(), (uint64_t)0, ikey.type);
|
|
|
|
}
|
|
|
|
|
|
|
|
Slice newkey(kptr, key.size());
|
|
|
|
assert((key.clear(), 1)); // we do not need 'key' anymore
|
2013-03-06 06:53:28 +01:00
|
|
|
|
[RocksDB] [MergeOperator] The new Merge Interface! Uses merge sequences.
Summary:
Here are the major changes to the Merge Interface. It has been expanded
to handle cases where the MergeOperator is not associative. It does so by stacking
up merge operations while scanning through the key history (i.e.: during Get() or
Compaction), until a valid Put/Delete/end-of-history is encountered; it then
applies all of the merge operations in the correct sequence starting with the
base/sentinel value.
I have also introduced an "AssociativeMerge" function which allows the user to
take advantage of associative merge operations (such as in the case of counters).
The implementation will always attempt to merge the operations/operands themselves
together when they are encountered, and will resort to the "stacking" method if
and only if the "associative-merge" fails.
This implementation is conjectured to allow MergeOperator to handle the general
case, while still providing the user with the ability to take advantage of certain
efficiencies in their own merge-operator / data-structure.
NOTE: This is a preliminary diff. This must still go through a lot of review,
revision, and testing. Feedback welcome!
Test Plan:
-This is a preliminary diff. I have only just begun testing/debugging it.
-I will be testing this with the existing MergeOperator use-cases and unit-tests
(counters, string-append, and redis-lists)
-I will be "desk-checking" and walking through the code with the help gdb.
-I will find a way of stress-testing the new interface / implementation using
db_bench, db_test, merge_test, and/or db_stress.
-I will ensure that my tests cover all cases: Get-Memtable,
Get-Immutable-Memtable, Get-from-Disk, Iterator-Range-Scan, Flush-Memtable-to-L0,
Compaction-L0-L1, Compaction-Ln-L(n+1), Put/Delete found, Put/Delete not-found,
end-of-history, end-of-file, etc.
-A lot of feedback from the reviewers.
Reviewers: haobo, dhruba, zshao, emayanke
Reviewed By: haobo
CC: leveldb
Differential Revision: https://reviews.facebook.net/D11499
2013-08-06 05:14:32 +02:00
|
|
|
// Open output file if necessary
|
|
|
|
if (compact->builder == nullptr) {
|
|
|
|
status = OpenCompactionOutputFile(compact);
|
|
|
|
if (!status.ok()) {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
2013-08-07 20:58:06 +02:00
|
|
|
|
|
|
|
SequenceNumber seqno = GetInternalKeySeqno(newkey);
|
[RocksDB] [MergeOperator] The new Merge Interface! Uses merge sequences.
Summary:
Here are the major changes to the Merge Interface. It has been expanded
to handle cases where the MergeOperator is not associative. It does so by stacking
up merge operations while scanning through the key history (i.e.: during Get() or
Compaction), until a valid Put/Delete/end-of-history is encountered; it then
applies all of the merge operations in the correct sequence starting with the
base/sentinel value.
I have also introduced an "AssociativeMerge" function which allows the user to
take advantage of associative merge operations (such as in the case of counters).
The implementation will always attempt to merge the operations/operands themselves
together when they are encountered, and will resort to the "stacking" method if
and only if the "associative-merge" fails.
This implementation is conjectured to allow MergeOperator to handle the general
case, while still providing the user with the ability to take advantage of certain
efficiencies in their own merge-operator / data-structure.
NOTE: This is a preliminary diff. This must still go through a lot of review,
revision, and testing. Feedback welcome!
Test Plan:
-This is a preliminary diff. I have only just begun testing/debugging it.
-I will be testing this with the existing MergeOperator use-cases and unit-tests
(counters, string-append, and redis-lists)
-I will be "desk-checking" and walking through the code with the help gdb.
-I will find a way of stress-testing the new interface / implementation using
db_bench, db_test, merge_test, and/or db_stress.
-I will ensure that my tests cover all cases: Get-Memtable,
Get-Immutable-Memtable, Get-from-Disk, Iterator-Range-Scan, Flush-Memtable-to-L0,
Compaction-L0-L1, Compaction-Ln-L(n+1), Put/Delete found, Put/Delete not-found,
end-of-history, end-of-file, etc.
-A lot of feedback from the reviewers.
Reviewers: haobo, dhruba, zshao, emayanke
Reviewed By: haobo
CC: leveldb
Differential Revision: https://reviews.facebook.net/D11499
2013-08-06 05:14:32 +02:00
|
|
|
if (compact->builder->NumEntries() == 0) {
|
|
|
|
compact->current_output()->smallest.DecodeFrom(newkey);
|
2013-08-07 20:58:06 +02:00
|
|
|
compact->current_output()->smallest_seqno = seqno;
|
|
|
|
} else {
|
|
|
|
compact->current_output()->smallest_seqno =
|
|
|
|
std::min(compact->current_output()->smallest_seqno, seqno);
|
[RocksDB] [MergeOperator] The new Merge Interface! Uses merge sequences.
Summary:
Here are the major changes to the Merge Interface. It has been expanded
to handle cases where the MergeOperator is not associative. It does so by stacking
up merge operations while scanning through the key history (i.e.: during Get() or
Compaction), until a valid Put/Delete/end-of-history is encountered; it then
applies all of the merge operations in the correct sequence starting with the
base/sentinel value.
I have also introduced an "AssociativeMerge" function which allows the user to
take advantage of associative merge operations (such as in the case of counters).
The implementation will always attempt to merge the operations/operands themselves
together when they are encountered, and will resort to the "stacking" method if
and only if the "associative-merge" fails.
This implementation is conjectured to allow MergeOperator to handle the general
case, while still providing the user with the ability to take advantage of certain
efficiencies in their own merge-operator / data-structure.
NOTE: This is a preliminary diff. This must still go through a lot of review,
revision, and testing. Feedback welcome!
Test Plan:
-This is a preliminary diff. I have only just begun testing/debugging it.
-I will be testing this with the existing MergeOperator use-cases and unit-tests
(counters, string-append, and redis-lists)
-I will be "desk-checking" and walking through the code with the help gdb.
-I will find a way of stress-testing the new interface / implementation using
db_bench, db_test, merge_test, and/or db_stress.
-I will ensure that my tests cover all cases: Get-Memtable,
Get-Immutable-Memtable, Get-from-Disk, Iterator-Range-Scan, Flush-Memtable-to-L0,
Compaction-L0-L1, Compaction-Ln-L(n+1), Put/Delete found, Put/Delete not-found,
end-of-history, end-of-file, etc.
-A lot of feedback from the reviewers.
Reviewers: haobo, dhruba, zshao, emayanke
Reviewed By: haobo
CC: leveldb
Differential Revision: https://reviews.facebook.net/D11499
2013-08-06 05:14:32 +02:00
|
|
|
}
|
|
|
|
compact->current_output()->largest.DecodeFrom(newkey);
|
|
|
|
compact->builder->Add(newkey, value);
|
2013-08-07 20:58:06 +02:00
|
|
|
compact->current_output()->largest_seqno =
|
|
|
|
std::max(compact->current_output()->largest_seqno, seqno);
|
[RocksDB] [MergeOperator] The new Merge Interface! Uses merge sequences.
Summary:
Here are the major changes to the Merge Interface. It has been expanded
to handle cases where the MergeOperator is not associative. It does so by stacking
up merge operations while scanning through the key history (i.e.: during Get() or
Compaction), until a valid Put/Delete/end-of-history is encountered; it then
applies all of the merge operations in the correct sequence starting with the
base/sentinel value.
I have also introduced an "AssociativeMerge" function which allows the user to
take advantage of associative merge operations (such as in the case of counters).
The implementation will always attempt to merge the operations/operands themselves
together when they are encountered, and will resort to the "stacking" method if
and only if the "associative-merge" fails.
This implementation is conjectured to allow MergeOperator to handle the general
case, while still providing the user with the ability to take advantage of certain
efficiencies in their own merge-operator / data-structure.
NOTE: This is a preliminary diff. This must still go through a lot of review,
revision, and testing. Feedback welcome!
Test Plan:
-This is a preliminary diff. I have only just begun testing/debugging it.
-I will be testing this with the existing MergeOperator use-cases and unit-tests
(counters, string-append, and redis-lists)
-I will be "desk-checking" and walking through the code with the help gdb.
-I will find a way of stress-testing the new interface / implementation using
db_bench, db_test, merge_test, and/or db_stress.
-I will ensure that my tests cover all cases: Get-Memtable,
Get-Immutable-Memtable, Get-from-Disk, Iterator-Range-Scan, Flush-Memtable-to-L0,
Compaction-L0-L1, Compaction-Ln-L(n+1), Put/Delete found, Put/Delete not-found,
end-of-history, end-of-file, etc.
-A lot of feedback from the reviewers.
Reviewers: haobo, dhruba, zshao, emayanke
Reviewed By: haobo
CC: leveldb
Differential Revision: https://reviews.facebook.net/D11499
2013-08-06 05:14:32 +02:00
|
|
|
|
|
|
|
// Close output file if it is big enough
|
|
|
|
if (compact->builder->FileSize() >=
|
|
|
|
compact->compaction->MaxOutputFileSize()) {
|
|
|
|
status = FinishCompactionOutputFile(compact, input.get());
|
|
|
|
if (!status.ok()) {
|
|
|
|
break;
|
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
[RocksDB] [MergeOperator] The new Merge Interface! Uses merge sequences.
Summary:
Here are the major changes to the Merge Interface. It has been expanded
to handle cases where the MergeOperator is not associative. It does so by stacking
up merge operations while scanning through the key history (i.e.: during Get() or
Compaction), until a valid Put/Delete/end-of-history is encountered; it then
applies all of the merge operations in the correct sequence starting with the
base/sentinel value.
I have also introduced an "AssociativeMerge" function which allows the user to
take advantage of associative merge operations (such as in the case of counters).
The implementation will always attempt to merge the operations/operands themselves
together when they are encountered, and will resort to the "stacking" method if
and only if the "associative-merge" fails.
This implementation is conjectured to allow MergeOperator to handle the general
case, while still providing the user with the ability to take advantage of certain
efficiencies in their own merge-operator / data-structure.
NOTE: This is a preliminary diff. This must still go through a lot of review,
revision, and testing. Feedback welcome!
Test Plan:
-This is a preliminary diff. I have only just begun testing/debugging it.
-I will be testing this with the existing MergeOperator use-cases and unit-tests
(counters, string-append, and redis-lists)
-I will be "desk-checking" and walking through the code with the help gdb.
-I will find a way of stress-testing the new interface / implementation using
db_bench, db_test, merge_test, and/or db_stress.
-I will ensure that my tests cover all cases: Get-Memtable,
Get-Immutable-Memtable, Get-from-Disk, Iterator-Range-Scan, Flush-Memtable-to-L0,
Compaction-L0-L1, Compaction-Ln-L(n+1), Put/Delete found, Put/Delete not-found,
end-of-history, end-of-file, etc.
-A lot of feedback from the reviewers.
Reviewers: haobo, dhruba, zshao, emayanke
Reviewed By: haobo
CC: leveldb
Differential Revision: https://reviews.facebook.net/D11499
2013-08-06 05:14:32 +02:00
|
|
|
// If we have a list of entries, move to next element
|
|
|
|
// If we only had one entry, then break the loop.
|
|
|
|
if (has_merge_list) {
|
|
|
|
++key_iter;
|
|
|
|
++value_iter;
|
|
|
|
|
|
|
|
// If at end of list
|
|
|
|
if (key_iter == keys->rend() || value_iter == values->rend()) {
|
|
|
|
// Sanity Check: if one ends, then both end
|
|
|
|
assert(key_iter == keys->rend() && value_iter == values->rend());
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Otherwise not at end of list. Update key, value, and ikey.
|
|
|
|
key = Slice(*key_iter);
|
|
|
|
value = Slice(*value_iter);
|
|
|
|
ParseInternalKey(key, &ikey);
|
|
|
|
|
|
|
|
} else{
|
|
|
|
// Only had one item to begin with (Put/Delete)
|
2011-03-18 23:37:00 +01:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-03-21 23:59:47 +01:00
|
|
|
// MergeUntil has moved input to the next entry
|
[RocksDB] [MergeOperator] The new Merge Interface! Uses merge sequences.
Summary:
Here are the major changes to the Merge Interface. It has been expanded
to handle cases where the MergeOperator is not associative. It does so by stacking
up merge operations while scanning through the key history (i.e.: during Get() or
Compaction), until a valid Put/Delete/end-of-history is encountered; it then
applies all of the merge operations in the correct sequence starting with the
base/sentinel value.
I have also introduced an "AssociativeMerge" function which allows the user to
take advantage of associative merge operations (such as in the case of counters).
The implementation will always attempt to merge the operations/operands themselves
together when they are encountered, and will resort to the "stacking" method if
and only if the "associative-merge" fails.
This implementation is conjectured to allow MergeOperator to handle the general
case, while still providing the user with the ability to take advantage of certain
efficiencies in their own merge-operator / data-structure.
NOTE: This is a preliminary diff. This must still go through a lot of review,
revision, and testing. Feedback welcome!
Test Plan:
-This is a preliminary diff. I have only just begun testing/debugging it.
-I will be testing this with the existing MergeOperator use-cases and unit-tests
(counters, string-append, and redis-lists)
-I will be "desk-checking" and walking through the code with the help gdb.
-I will find a way of stress-testing the new interface / implementation using
db_bench, db_test, merge_test, and/or db_stress.
-I will ensure that my tests cover all cases: Get-Memtable,
Get-Immutable-Memtable, Get-from-Disk, Iterator-Range-Scan, Flush-Memtable-to-L0,
Compaction-L0-L1, Compaction-Ln-L(n+1), Put/Delete found, Put/Delete not-found,
end-of-history, end-of-file, etc.
-A lot of feedback from the reviewers.
Reviewers: haobo, dhruba, zshao, emayanke
Reviewed By: haobo
CC: leveldb
Differential Revision: https://reviews.facebook.net/D11499
2013-08-06 05:14:32 +02:00
|
|
|
if (!current_entry_is_merging) {
|
2013-03-21 23:59:47 +01:00
|
|
|
input->Next();
|
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
if (status.ok() && shutting_down_.Acquire_Load()) {
|
2013-04-16 00:27:15 +02:00
|
|
|
status = Status::IOError("Database shutdown started during compaction");
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
2013-02-15 20:53:17 +01:00
|
|
|
if (status.ok() && compact->builder != nullptr) {
|
2013-01-20 11:07:13 +01:00
|
|
|
status = FinishCompactionOutputFile(compact, input.get());
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
if (status.ok()) {
|
|
|
|
status = input->status();
|
|
|
|
}
|
2013-01-20 11:07:13 +01:00
|
|
|
input.reset();
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2014-01-27 20:02:21 +01:00
|
|
|
if (!options_.disableDataSync) {
|
|
|
|
db_directory_->Fsync();
|
|
|
|
}
|
2011-04-12 21:38:58 +02:00
|
|
|
CompactionStats stats;
|
|
|
|
stats.micros = env_->NowMicros() - start_micros - imm_micros;
|
2014-01-17 21:46:06 +01:00
|
|
|
MeasureTime(options_.statistics.get(), COMPACTION_TIME, stats.micros);
|
2012-10-23 19:34:09 +02:00
|
|
|
stats.files_in_leveln = compact->compaction->num_input_files(0);
|
|
|
|
stats.files_in_levelnp1 = compact->compaction->num_input_files(1);
|
2013-01-17 19:04:45 +01:00
|
|
|
|
|
|
|
int num_output_files = compact->outputs.size();
|
2013-02-15 20:53:17 +01:00
|
|
|
if (compact->builder != nullptr) {
|
2013-11-16 12:21:34 +01:00
|
|
|
// An error occurred so ignore the last output.
|
2013-01-17 19:04:45 +01:00
|
|
|
assert(num_output_files > 0);
|
|
|
|
--num_output_files;
|
|
|
|
}
|
|
|
|
stats.files_out_levelnp1 = num_output_files;
|
2012-10-23 19:34:09 +02:00
|
|
|
|
Add monitoring for universal compaction and add counters for compaction IO
Summary:
Adds these counters
{ WAL_FILE_SYNCED, "rocksdb.wal.synced" }
number of writes that request a WAL sync
{ WAL_FILE_BYTES, "rocksdb.wal.bytes" },
number of bytes written to the WAL
{ WRITE_DONE_BY_SELF, "rocksdb.write.self" },
number of writes processed by the calling thread
{ WRITE_DONE_BY_OTHER, "rocksdb.write.other" },
number of writes not processed by the calling thread. Instead these were
processed by the current holder of the write lock
{ WRITE_WITH_WAL, "rocksdb.write.wal" },
number of writes that request WAL logging
{ COMPACT_READ_BYTES, "rocksdb.compact.read.bytes" },
number of bytes read during compaction
{ COMPACT_WRITE_BYTES, "rocksdb.compact.write.bytes" },
number of bytes written during compaction
Per-interval stats output was updated with WAL stats and correct stats for universal compaction
including a correct value for write-amplification. It now looks like:
Compactions
Level Files Size(MB) Score Time(sec) Read(MB) Write(MB) Rn(MB) Rnp1(MB) Wnew(MB) RW-Amplify Read(MB/s) Write(MB/s) Rn Rnp1 Wnp1 NewW Count Ln-stall Stall-cnt
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
0 7 464 46.4 281 3411 3875 3411 0 3875 2.1 12.1 13.8 621 0 240 240 628 0.0 0
Uptime(secs): 310.8 total, 2.0 interval
Writes cumulative: 9999999 total, 9999999 batches, 1.0 per batch, 1.22 ingest GB
WAL cumulative: 9999999 WAL writes, 9999999 WAL syncs, 1.00 writes per sync, 1.22 GB written
Compaction IO cumulative (GB): 1.22 new, 3.33 read, 3.78 write, 7.12 read+write
Compaction IO cumulative (MB/sec): 4.0 new, 11.0 read, 12.5 write, 23.4 read+write
Amplification cumulative: 4.1 write, 6.8 compaction
Writes interval: 100000 total, 100000 batches, 1.0 per batch, 12.5 ingest MB
WAL interval: 100000 WAL writes, 100000 WAL syncs, 1.00 writes per sync, 0.01 MB written
Compaction IO interval (MB): 12.49 new, 14.98 read, 21.50 write, 36.48 read+write
Compaction IO interval (MB/sec): 6.4 new, 7.6 read, 11.0 write, 18.6 read+write
Amplification interval: 101.7 write, 102.9 compaction
Stalls(secs): 142.924 level0_slowdown, 0.000 level0_numfiles, 0.805 memtable_compaction, 0.000 leveln_slowdown
Stalls(count): 132461 level0_slowdown, 0 level0_numfiles, 3 memtable_compaction, 0 leveln_slowdown
Task ID: #3329644, #3301695
Blame Rev:
Test Plan:
Revert Plan:
Database Impact:
Memcache Impact:
Other Notes:
EImportant:
- begin *PUBLIC* platform impact section -
Bugzilla: #
- end platform impact -
Reviewers: dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14583
2013-12-09 22:43:34 +01:00
|
|
|
for (int i = 0; i < compact->compaction->num_input_files(0); i++) {
|
2012-10-23 19:34:09 +02:00
|
|
|
stats.bytes_readn += compact->compaction->input(0, i)->file_size;
|
Add monitoring for universal compaction and add counters for compaction IO
Summary:
Adds these counters
{ WAL_FILE_SYNCED, "rocksdb.wal.synced" }
number of writes that request a WAL sync
{ WAL_FILE_BYTES, "rocksdb.wal.bytes" },
number of bytes written to the WAL
{ WRITE_DONE_BY_SELF, "rocksdb.write.self" },
number of writes processed by the calling thread
{ WRITE_DONE_BY_OTHER, "rocksdb.write.other" },
number of writes not processed by the calling thread. Instead these were
processed by the current holder of the write lock
{ WRITE_WITH_WAL, "rocksdb.write.wal" },
number of writes that request WAL logging
{ COMPACT_READ_BYTES, "rocksdb.compact.read.bytes" },
number of bytes read during compaction
{ COMPACT_WRITE_BYTES, "rocksdb.compact.write.bytes" },
number of bytes written during compaction
Per-interval stats output was updated with WAL stats and correct stats for universal compaction
including a correct value for write-amplification. It now looks like:
Compactions
Level Files Size(MB) Score Time(sec) Read(MB) Write(MB) Rn(MB) Rnp1(MB) Wnew(MB) RW-Amplify Read(MB/s) Write(MB/s) Rn Rnp1 Wnp1 NewW Count Ln-stall Stall-cnt
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
0 7 464 46.4 281 3411 3875 3411 0 3875 2.1 12.1 13.8 621 0 240 240 628 0.0 0
Uptime(secs): 310.8 total, 2.0 interval
Writes cumulative: 9999999 total, 9999999 batches, 1.0 per batch, 1.22 ingest GB
WAL cumulative: 9999999 WAL writes, 9999999 WAL syncs, 1.00 writes per sync, 1.22 GB written
Compaction IO cumulative (GB): 1.22 new, 3.33 read, 3.78 write, 7.12 read+write
Compaction IO cumulative (MB/sec): 4.0 new, 11.0 read, 12.5 write, 23.4 read+write
Amplification cumulative: 4.1 write, 6.8 compaction
Writes interval: 100000 total, 100000 batches, 1.0 per batch, 12.5 ingest MB
WAL interval: 100000 WAL writes, 100000 WAL syncs, 1.00 writes per sync, 0.01 MB written
Compaction IO interval (MB): 12.49 new, 14.98 read, 21.50 write, 36.48 read+write
Compaction IO interval (MB/sec): 6.4 new, 7.6 read, 11.0 write, 18.6 read+write
Amplification interval: 101.7 write, 102.9 compaction
Stalls(secs): 142.924 level0_slowdown, 0.000 level0_numfiles, 0.805 memtable_compaction, 0.000 leveln_slowdown
Stalls(count): 132461 level0_slowdown, 0 level0_numfiles, 3 memtable_compaction, 0 leveln_slowdown
Task ID: #3329644, #3301695
Blame Rev:
Test Plan:
Revert Plan:
Database Impact:
Memcache Impact:
Other Notes:
EImportant:
- begin *PUBLIC* platform impact section -
Bugzilla: #
- end platform impact -
Reviewers: dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14583
2013-12-09 22:43:34 +01:00
|
|
|
RecordTick(options_.statistics.get(), COMPACT_READ_BYTES,
|
|
|
|
compact->compaction->input(0, i)->file_size);
|
|
|
|
}
|
2012-10-23 19:34:09 +02:00
|
|
|
|
Add monitoring for universal compaction and add counters for compaction IO
Summary:
Adds these counters
{ WAL_FILE_SYNCED, "rocksdb.wal.synced" }
number of writes that request a WAL sync
{ WAL_FILE_BYTES, "rocksdb.wal.bytes" },
number of bytes written to the WAL
{ WRITE_DONE_BY_SELF, "rocksdb.write.self" },
number of writes processed by the calling thread
{ WRITE_DONE_BY_OTHER, "rocksdb.write.other" },
number of writes not processed by the calling thread. Instead these were
processed by the current holder of the write lock
{ WRITE_WITH_WAL, "rocksdb.write.wal" },
number of writes that request WAL logging
{ COMPACT_READ_BYTES, "rocksdb.compact.read.bytes" },
number of bytes read during compaction
{ COMPACT_WRITE_BYTES, "rocksdb.compact.write.bytes" },
number of bytes written during compaction
Per-interval stats output was updated with WAL stats and correct stats for universal compaction
including a correct value for write-amplification. It now looks like:
Compactions
Level Files Size(MB) Score Time(sec) Read(MB) Write(MB) Rn(MB) Rnp1(MB) Wnew(MB) RW-Amplify Read(MB/s) Write(MB/s) Rn Rnp1 Wnp1 NewW Count Ln-stall Stall-cnt
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
0 7 464 46.4 281 3411 3875 3411 0 3875 2.1 12.1 13.8 621 0 240 240 628 0.0 0
Uptime(secs): 310.8 total, 2.0 interval
Writes cumulative: 9999999 total, 9999999 batches, 1.0 per batch, 1.22 ingest GB
WAL cumulative: 9999999 WAL writes, 9999999 WAL syncs, 1.00 writes per sync, 1.22 GB written
Compaction IO cumulative (GB): 1.22 new, 3.33 read, 3.78 write, 7.12 read+write
Compaction IO cumulative (MB/sec): 4.0 new, 11.0 read, 12.5 write, 23.4 read+write
Amplification cumulative: 4.1 write, 6.8 compaction
Writes interval: 100000 total, 100000 batches, 1.0 per batch, 12.5 ingest MB
WAL interval: 100000 WAL writes, 100000 WAL syncs, 1.00 writes per sync, 0.01 MB written
Compaction IO interval (MB): 12.49 new, 14.98 read, 21.50 write, 36.48 read+write
Compaction IO interval (MB/sec): 6.4 new, 7.6 read, 11.0 write, 18.6 read+write
Amplification interval: 101.7 write, 102.9 compaction
Stalls(secs): 142.924 level0_slowdown, 0.000 level0_numfiles, 0.805 memtable_compaction, 0.000 leveln_slowdown
Stalls(count): 132461 level0_slowdown, 0 level0_numfiles, 3 memtable_compaction, 0 leveln_slowdown
Task ID: #3329644, #3301695
Blame Rev:
Test Plan:
Revert Plan:
Database Impact:
Memcache Impact:
Other Notes:
EImportant:
- begin *PUBLIC* platform impact section -
Bugzilla: #
- end platform impact -
Reviewers: dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14583
2013-12-09 22:43:34 +01:00
|
|
|
for (int i = 0; i < compact->compaction->num_input_files(1); i++) {
|
2012-10-23 19:34:09 +02:00
|
|
|
stats.bytes_readnp1 += compact->compaction->input(1, i)->file_size;
|
Add monitoring for universal compaction and add counters for compaction IO
Summary:
Adds these counters
{ WAL_FILE_SYNCED, "rocksdb.wal.synced" }
number of writes that request a WAL sync
{ WAL_FILE_BYTES, "rocksdb.wal.bytes" },
number of bytes written to the WAL
{ WRITE_DONE_BY_SELF, "rocksdb.write.self" },
number of writes processed by the calling thread
{ WRITE_DONE_BY_OTHER, "rocksdb.write.other" },
number of writes not processed by the calling thread. Instead these were
processed by the current holder of the write lock
{ WRITE_WITH_WAL, "rocksdb.write.wal" },
number of writes that request WAL logging
{ COMPACT_READ_BYTES, "rocksdb.compact.read.bytes" },
number of bytes read during compaction
{ COMPACT_WRITE_BYTES, "rocksdb.compact.write.bytes" },
number of bytes written during compaction
Per-interval stats output was updated with WAL stats and correct stats for universal compaction
including a correct value for write-amplification. It now looks like:
Compactions
Level Files Size(MB) Score Time(sec) Read(MB) Write(MB) Rn(MB) Rnp1(MB) Wnew(MB) RW-Amplify Read(MB/s) Write(MB/s) Rn Rnp1 Wnp1 NewW Count Ln-stall Stall-cnt
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
0 7 464 46.4 281 3411 3875 3411 0 3875 2.1 12.1 13.8 621 0 240 240 628 0.0 0
Uptime(secs): 310.8 total, 2.0 interval
Writes cumulative: 9999999 total, 9999999 batches, 1.0 per batch, 1.22 ingest GB
WAL cumulative: 9999999 WAL writes, 9999999 WAL syncs, 1.00 writes per sync, 1.22 GB written
Compaction IO cumulative (GB): 1.22 new, 3.33 read, 3.78 write, 7.12 read+write
Compaction IO cumulative (MB/sec): 4.0 new, 11.0 read, 12.5 write, 23.4 read+write
Amplification cumulative: 4.1 write, 6.8 compaction
Writes interval: 100000 total, 100000 batches, 1.0 per batch, 12.5 ingest MB
WAL interval: 100000 WAL writes, 100000 WAL syncs, 1.00 writes per sync, 0.01 MB written
Compaction IO interval (MB): 12.49 new, 14.98 read, 21.50 write, 36.48 read+write
Compaction IO interval (MB/sec): 6.4 new, 7.6 read, 11.0 write, 18.6 read+write
Amplification interval: 101.7 write, 102.9 compaction
Stalls(secs): 142.924 level0_slowdown, 0.000 level0_numfiles, 0.805 memtable_compaction, 0.000 leveln_slowdown
Stalls(count): 132461 level0_slowdown, 0 level0_numfiles, 3 memtable_compaction, 0 leveln_slowdown
Task ID: #3329644, #3301695
Blame Rev:
Test Plan:
Revert Plan:
Database Impact:
Memcache Impact:
Other Notes:
EImportant:
- begin *PUBLIC* platform impact section -
Bugzilla: #
- end platform impact -
Reviewers: dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14583
2013-12-09 22:43:34 +01:00
|
|
|
RecordTick(options_.statistics.get(), COMPACT_READ_BYTES,
|
|
|
|
compact->compaction->input(1, i)->file_size);
|
|
|
|
}
|
2012-10-23 19:34:09 +02:00
|
|
|
|
2013-01-17 19:04:45 +01:00
|
|
|
for (int i = 0; i < num_output_files; i++) {
|
2011-04-12 21:38:58 +02:00
|
|
|
stats.bytes_written += compact->outputs[i].file_size;
|
Add monitoring for universal compaction and add counters for compaction IO
Summary:
Adds these counters
{ WAL_FILE_SYNCED, "rocksdb.wal.synced" }
number of writes that request a WAL sync
{ WAL_FILE_BYTES, "rocksdb.wal.bytes" },
number of bytes written to the WAL
{ WRITE_DONE_BY_SELF, "rocksdb.write.self" },
number of writes processed by the calling thread
{ WRITE_DONE_BY_OTHER, "rocksdb.write.other" },
number of writes not processed by the calling thread. Instead these were
processed by the current holder of the write lock
{ WRITE_WITH_WAL, "rocksdb.write.wal" },
number of writes that request WAL logging
{ COMPACT_READ_BYTES, "rocksdb.compact.read.bytes" },
number of bytes read during compaction
{ COMPACT_WRITE_BYTES, "rocksdb.compact.write.bytes" },
number of bytes written during compaction
Per-interval stats output was updated with WAL stats and correct stats for universal compaction
including a correct value for write-amplification. It now looks like:
Compactions
Level Files Size(MB) Score Time(sec) Read(MB) Write(MB) Rn(MB) Rnp1(MB) Wnew(MB) RW-Amplify Read(MB/s) Write(MB/s) Rn Rnp1 Wnp1 NewW Count Ln-stall Stall-cnt
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
0 7 464 46.4 281 3411 3875 3411 0 3875 2.1 12.1 13.8 621 0 240 240 628 0.0 0
Uptime(secs): 310.8 total, 2.0 interval
Writes cumulative: 9999999 total, 9999999 batches, 1.0 per batch, 1.22 ingest GB
WAL cumulative: 9999999 WAL writes, 9999999 WAL syncs, 1.00 writes per sync, 1.22 GB written
Compaction IO cumulative (GB): 1.22 new, 3.33 read, 3.78 write, 7.12 read+write
Compaction IO cumulative (MB/sec): 4.0 new, 11.0 read, 12.5 write, 23.4 read+write
Amplification cumulative: 4.1 write, 6.8 compaction
Writes interval: 100000 total, 100000 batches, 1.0 per batch, 12.5 ingest MB
WAL interval: 100000 WAL writes, 100000 WAL syncs, 1.00 writes per sync, 0.01 MB written
Compaction IO interval (MB): 12.49 new, 14.98 read, 21.50 write, 36.48 read+write
Compaction IO interval (MB/sec): 6.4 new, 7.6 read, 11.0 write, 18.6 read+write
Amplification interval: 101.7 write, 102.9 compaction
Stalls(secs): 142.924 level0_slowdown, 0.000 level0_numfiles, 0.805 memtable_compaction, 0.000 leveln_slowdown
Stalls(count): 132461 level0_slowdown, 0 level0_numfiles, 3 memtable_compaction, 0 leveln_slowdown
Task ID: #3329644, #3301695
Blame Rev:
Test Plan:
Revert Plan:
Database Impact:
Memcache Impact:
Other Notes:
EImportant:
- begin *PUBLIC* platform impact section -
Bugzilla: #
- end platform impact -
Reviewers: dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14583
2013-12-09 22:43:34 +01:00
|
|
|
RecordTick(options_.statistics.get(), COMPACT_WRITE_BYTES,
|
|
|
|
compact->outputs[i].file_size);
|
2011-04-12 21:38:58 +02:00
|
|
|
}
|
|
|
|
|
2013-11-07 20:31:56 +01:00
|
|
|
LogFlush(options_.info_log);
|
2011-03-18 23:37:00 +01:00
|
|
|
mutex_.Lock();
|
2013-10-03 01:20:17 +02:00
|
|
|
stats_[compact->compaction->output_level()].Add(stats);
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2012-10-19 23:00:53 +02:00
|
|
|
// if there were any unused file number (mostly in case of
|
|
|
|
// compaction error), free up the entry from pending_putputs
|
|
|
|
ReleaseCompactionUnusedFileNumbers(compact);
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
if (status.ok()) {
|
|
|
|
status = InstallCompactionResults(compact);
|
2014-01-24 23:30:28 +01:00
|
|
|
InstallSuperVersion(default_cfd_, deletion_state);
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
2014-01-16 01:18:04 +01:00
|
|
|
Version::LevelSummaryStorage tmp;
|
2011-07-21 04:40:18 +02:00
|
|
|
Log(options_.info_log,
|
2012-10-23 19:34:09 +02:00
|
|
|
"compacted to: %s, %.1f MB/sec, level %d, files in(%d, %d) out(%d) "
|
2013-08-23 07:37:13 +02:00
|
|
|
"MB in(%.1f, %.1f) out(%.1f), read-write-amplify(%.1f) "
|
|
|
|
"write-amplify(%.1f) %s\n",
|
2014-01-27 23:33:50 +01:00
|
|
|
compact->compaction->input_version()->LevelSummary(&tmp),
|
2012-10-23 19:34:09 +02:00
|
|
|
(stats.bytes_readn + stats.bytes_readnp1 + stats.bytes_written) /
|
2014-01-16 01:18:04 +01:00
|
|
|
(double)stats.micros,
|
|
|
|
compact->compaction->output_level(), stats.files_in_leveln,
|
|
|
|
stats.files_in_levelnp1, stats.files_out_levelnp1,
|
|
|
|
stats.bytes_readn / 1048576.0, stats.bytes_readnp1 / 1048576.0,
|
2012-10-23 19:34:09 +02:00
|
|
|
stats.bytes_written / 1048576.0,
|
2013-08-23 07:37:13 +02:00
|
|
|
(stats.bytes_written + stats.bytes_readnp1 + stats.bytes_readn) /
|
2014-01-16 01:18:04 +01:00
|
|
|
(double)stats.bytes_readn,
|
|
|
|
stats.bytes_written / (double)stats.bytes_readn,
|
2012-11-28 07:10:20 +01:00
|
|
|
status.ToString().c_str());
|
2012-10-23 19:34:09 +02:00
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
return status;
|
|
|
|
}
|
|
|
|
|
2011-05-28 02:53:58 +02:00
|
|
|
namespace {
|
|
|
|
struct IterState {
|
|
|
|
port::Mutex* mu;
|
MemTableListVersion
Summary:
MemTableListVersion is to MemTableList what Version is to VersionSet. I took almost the same ideas to develop MemTableListVersion. The reason is to have copying std::list done in background, while flushing, rather than in foreground (MultiGet() and NewIterator()) under a mutex! Also, whenever we copied MemTableList, we copied also some MemTableList metadata (flush_requested_, commit_in_progress_, etc.), which was wasteful.
This diff avoids std::list copy under a mutex in both MultiGet() and NewIterator(). I created a small database with some number of immutable memtables, and creating 100.000 iterators in a single-thread (!) decreased from {188739, 215703, 198028} to {154352, 164035, 159817}. A lot of the savings come from code under a mutex, so we should see much higher savings with multiple threads. Creating new iterator is very important to LogDevice team.
I also think this diff will make SuperVersion obsolete for performance reasons. I will try it in the next diff. SuperVersion gave us huge savings on Get() code path, but I think that most of the savings came from copying MemTableList under a mutex. If we had MemTableListVersion, we would never need to copy the entire object (like we still do in NewIterator() and MultiGet())
Test Plan: `make check` works. I will also do `make valgrind_check` before commit
Reviewers: dhruba, haobo, kailiu, sdong, emayanke, tnovak
Reviewed By: kailiu
CC: leveldb
Differential Revision: https://reviews.facebook.net/D15255
2014-01-24 23:52:08 +01:00
|
|
|
Version* version = nullptr;
|
|
|
|
MemTable* mem = nullptr;
|
|
|
|
MemTableListVersion* imm = nullptr;
|
2013-11-15 03:03:57 +01:00
|
|
|
DBImpl *db;
|
2011-05-28 02:53:58 +02:00
|
|
|
};
|
|
|
|
|
|
|
|
static void CleanupIteratorState(void* arg1, void* arg2) {
|
|
|
|
IterState* state = reinterpret_cast<IterState*>(arg1);
|
2013-11-27 23:56:20 +01:00
|
|
|
DBImpl::DeletionState deletion_state(state->db->GetOptions().
|
|
|
|
max_write_buffer_number);
|
2011-05-28 02:53:58 +02:00
|
|
|
state->mu->Lock();
|
2014-01-25 00:51:01 +01:00
|
|
|
if (state->mem) { // not set for immutable iterator
|
|
|
|
MemTable* m = state->mem->Unref();
|
|
|
|
if (m != nullptr) {
|
|
|
|
deletion_state.memtables_to_free.push_back(m);
|
|
|
|
}
|
2012-10-19 23:00:53 +02:00
|
|
|
}
|
2014-01-17 06:56:26 +01:00
|
|
|
if (state->version) { // not set for memtable-only iterator
|
|
|
|
state->version->Unref();
|
|
|
|
}
|
MemTableListVersion
Summary:
MemTableListVersion is to MemTableList what Version is to VersionSet. I took almost the same ideas to develop MemTableListVersion. The reason is to have copying std::list done in background, while flushing, rather than in foreground (MultiGet() and NewIterator()) under a mutex! Also, whenever we copied MemTableList, we copied also some MemTableList metadata (flush_requested_, commit_in_progress_, etc.), which was wasteful.
This diff avoids std::list copy under a mutex in both MultiGet() and NewIterator(). I created a small database with some number of immutable memtables, and creating 100.000 iterators in a single-thread (!) decreased from {188739, 215703, 198028} to {154352, 164035, 159817}. A lot of the savings come from code under a mutex, so we should see much higher savings with multiple threads. Creating new iterator is very important to LogDevice team.
I also think this diff will make SuperVersion obsolete for performance reasons. I will try it in the next diff. SuperVersion gave us huge savings on Get() code path, but I think that most of the savings came from copying MemTableList under a mutex. If we had MemTableListVersion, we would never need to copy the entire object (like we still do in NewIterator() and MultiGet())
Test Plan: `make check` works. I will also do `make valgrind_check` before commit
Reviewers: dhruba, haobo, kailiu, sdong, emayanke, tnovak
Reviewed By: kailiu
CC: leveldb
Differential Revision: https://reviews.facebook.net/D15255
2014-01-24 23:52:08 +01:00
|
|
|
if (state->imm) { // not set for memtable-only iterator
|
|
|
|
state->imm->Unref(&deletion_state.memtables_to_free);
|
|
|
|
}
|
2013-11-15 03:03:57 +01:00
|
|
|
// fast path FindObsoleteFiles
|
|
|
|
state->db->FindObsoleteFiles(deletion_state, false, true);
|
2011-05-28 02:53:58 +02:00
|
|
|
state->mu->Unlock();
|
2013-11-15 03:03:57 +01:00
|
|
|
state->db->PurgeObsoleteFiles(deletion_state);
|
2014-01-17 06:56:26 +01:00
|
|
|
|
2011-05-28 02:53:58 +02:00
|
|
|
delete state;
|
|
|
|
}
|
2011-10-31 18:22:06 +01:00
|
|
|
} // namespace
|
2011-05-28 02:53:58 +02:00
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
Iterator* DBImpl::NewInternalIterator(const ReadOptions& options,
|
|
|
|
SequenceNumber* latest_snapshot) {
|
2011-05-28 02:53:58 +02:00
|
|
|
IterState* cleanup = new IterState;
|
2013-12-12 19:54:03 +01:00
|
|
|
MemTable* mutable_mem;
|
MemTableListVersion
Summary:
MemTableListVersion is to MemTableList what Version is to VersionSet. I took almost the same ideas to develop MemTableListVersion. The reason is to have copying std::list done in background, while flushing, rather than in foreground (MultiGet() and NewIterator()) under a mutex! Also, whenever we copied MemTableList, we copied also some MemTableList metadata (flush_requested_, commit_in_progress_, etc.), which was wasteful.
This diff avoids std::list copy under a mutex in both MultiGet() and NewIterator(). I created a small database with some number of immutable memtables, and creating 100.000 iterators in a single-thread (!) decreased from {188739, 215703, 198028} to {154352, 164035, 159817}. A lot of the savings come from code under a mutex, so we should see much higher savings with multiple threads. Creating new iterator is very important to LogDevice team.
I also think this diff will make SuperVersion obsolete for performance reasons. I will try it in the next diff. SuperVersion gave us huge savings on Get() code path, but I think that most of the savings came from copying MemTableList under a mutex. If we had MemTableListVersion, we would never need to copy the entire object (like we still do in NewIterator() and MultiGet())
Test Plan: `make check` works. I will also do `make valgrind_check` before commit
Reviewers: dhruba, haobo, kailiu, sdong, emayanke, tnovak
Reviewed By: kailiu
CC: leveldb
Differential Revision: https://reviews.facebook.net/D15255
2014-01-24 23:52:08 +01:00
|
|
|
MemTableListVersion* immutable_mems;
|
2013-12-12 19:54:03 +01:00
|
|
|
Version* version;
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2012-10-19 23:00:53 +02:00
|
|
|
// Collect together all needed child iterators for mem
|
2013-12-12 19:54:03 +01:00
|
|
|
mutex_.Lock();
|
|
|
|
*latest_snapshot = versions_->LastSequence();
|
2014-01-29 22:28:50 +01:00
|
|
|
mutable_mem = default_cfd_->mem();
|
2014-01-24 23:30:28 +01:00
|
|
|
mutable_mem->Ref();
|
2014-01-29 22:28:50 +01:00
|
|
|
immutable_mems = default_cfd_->imm()->current();
|
MemTableListVersion
Summary:
MemTableListVersion is to MemTableList what Version is to VersionSet. I took almost the same ideas to develop MemTableListVersion. The reason is to have copying std::list done in background, while flushing, rather than in foreground (MultiGet() and NewIterator()) under a mutex! Also, whenever we copied MemTableList, we copied also some MemTableList metadata (flush_requested_, commit_in_progress_, etc.), which was wasteful.
This diff avoids std::list copy under a mutex in both MultiGet() and NewIterator(). I created a small database with some number of immutable memtables, and creating 100.000 iterators in a single-thread (!) decreased from {188739, 215703, 198028} to {154352, 164035, 159817}. A lot of the savings come from code under a mutex, so we should see much higher savings with multiple threads. Creating new iterator is very important to LogDevice team.
I also think this diff will make SuperVersion obsolete for performance reasons. I will try it in the next diff. SuperVersion gave us huge savings on Get() code path, but I think that most of the savings came from copying MemTableList under a mutex. If we had MemTableListVersion, we would never need to copy the entire object (like we still do in NewIterator() and MultiGet())
Test Plan: `make check` works. I will also do `make valgrind_check` before commit
Reviewers: dhruba, haobo, kailiu, sdong, emayanke, tnovak
Reviewed By: kailiu
CC: leveldb
Differential Revision: https://reviews.facebook.net/D15255
2014-01-24 23:52:08 +01:00
|
|
|
immutable_mems->Ref();
|
2014-01-29 22:28:50 +01:00
|
|
|
version = default_cfd_->current();
|
2014-01-27 23:33:50 +01:00
|
|
|
version->Ref();
|
2013-12-12 19:54:03 +01:00
|
|
|
mutex_.Unlock();
|
|
|
|
|
MemTableListVersion
Summary:
MemTableListVersion is to MemTableList what Version is to VersionSet. I took almost the same ideas to develop MemTableListVersion. The reason is to have copying std::list done in background, while flushing, rather than in foreground (MultiGet() and NewIterator()) under a mutex! Also, whenever we copied MemTableList, we copied also some MemTableList metadata (flush_requested_, commit_in_progress_, etc.), which was wasteful.
This diff avoids std::list copy under a mutex in both MultiGet() and NewIterator(). I created a small database with some number of immutable memtables, and creating 100.000 iterators in a single-thread (!) decreased from {188739, 215703, 198028} to {154352, 164035, 159817}. A lot of the savings come from code under a mutex, so we should see much higher savings with multiple threads. Creating new iterator is very important to LogDevice team.
I also think this diff will make SuperVersion obsolete for performance reasons. I will try it in the next diff. SuperVersion gave us huge savings on Get() code path, but I think that most of the savings came from copying MemTableList under a mutex. If we had MemTableListVersion, we would never need to copy the entire object (like we still do in NewIterator() and MultiGet())
Test Plan: `make check` works. I will also do `make valgrind_check` before commit
Reviewers: dhruba, haobo, kailiu, sdong, emayanke, tnovak
Reviewed By: kailiu
CC: leveldb
Differential Revision: https://reviews.facebook.net/D15255
2014-01-24 23:52:08 +01:00
|
|
|
std::vector<Iterator*> iterator_list;
|
|
|
|
iterator_list.push_back(mutable_mem->NewIterator(options));
|
|
|
|
cleanup->mem = mutable_mem;
|
|
|
|
cleanup->imm = immutable_mems;
|
2014-01-17 06:56:26 +01:00
|
|
|
// Collect all needed child iterators for immutable memtables
|
MemTableListVersion
Summary:
MemTableListVersion is to MemTableList what Version is to VersionSet. I took almost the same ideas to develop MemTableListVersion. The reason is to have copying std::list done in background, while flushing, rather than in foreground (MultiGet() and NewIterator()) under a mutex! Also, whenever we copied MemTableList, we copied also some MemTableList metadata (flush_requested_, commit_in_progress_, etc.), which was wasteful.
This diff avoids std::list copy under a mutex in both MultiGet() and NewIterator(). I created a small database with some number of immutable memtables, and creating 100.000 iterators in a single-thread (!) decreased from {188739, 215703, 198028} to {154352, 164035, 159817}. A lot of the savings come from code under a mutex, so we should see much higher savings with multiple threads. Creating new iterator is very important to LogDevice team.
I also think this diff will make SuperVersion obsolete for performance reasons. I will try it in the next diff. SuperVersion gave us huge savings on Get() code path, but I think that most of the savings came from copying MemTableList under a mutex. If we had MemTableListVersion, we would never need to copy the entire object (like we still do in NewIterator() and MultiGet())
Test Plan: `make check` works. I will also do `make valgrind_check` before commit
Reviewers: dhruba, haobo, kailiu, sdong, emayanke, tnovak
Reviewed By: kailiu
CC: leveldb
Differential Revision: https://reviews.facebook.net/D15255
2014-01-24 23:52:08 +01:00
|
|
|
immutable_mems->AddIterators(options, &iterator_list);
|
2014-01-17 06:56:26 +01:00
|
|
|
// Collect iterators for files in L0 - Ln
|
MemTableListVersion
Summary:
MemTableListVersion is to MemTableList what Version is to VersionSet. I took almost the same ideas to develop MemTableListVersion. The reason is to have copying std::list done in background, while flushing, rather than in foreground (MultiGet() and NewIterator()) under a mutex! Also, whenever we copied MemTableList, we copied also some MemTableList metadata (flush_requested_, commit_in_progress_, etc.), which was wasteful.
This diff avoids std::list copy under a mutex in both MultiGet() and NewIterator(). I created a small database with some number of immutable memtables, and creating 100.000 iterators in a single-thread (!) decreased from {188739, 215703, 198028} to {154352, 164035, 159817}. A lot of the savings come from code under a mutex, so we should see much higher savings with multiple threads. Creating new iterator is very important to LogDevice team.
I also think this diff will make SuperVersion obsolete for performance reasons. I will try it in the next diff. SuperVersion gave us huge savings on Get() code path, but I think that most of the savings came from copying MemTableList under a mutex. If we had MemTableListVersion, we would never need to copy the entire object (like we still do in NewIterator() and MultiGet())
Test Plan: `make check` works. I will also do `make valgrind_check` before commit
Reviewers: dhruba, haobo, kailiu, sdong, emayanke, tnovak
Reviewed By: kailiu
CC: leveldb
Differential Revision: https://reviews.facebook.net/D15255
2014-01-24 23:52:08 +01:00
|
|
|
version->AddIterators(options, storage_options_, &iterator_list);
|
|
|
|
Iterator* internal_iter = NewMergingIterator(
|
|
|
|
&internal_comparator_, &iterator_list[0], iterator_list.size());
|
2013-12-12 19:54:03 +01:00
|
|
|
cleanup->version = version;
|
2011-05-28 02:53:58 +02:00
|
|
|
cleanup->mu = &mutex_;
|
2013-11-15 03:03:57 +01:00
|
|
|
cleanup->db = this;
|
2013-02-15 20:53:17 +01:00
|
|
|
internal_iter->RegisterCleanup(CleanupIteratorState, cleanup, nullptr);
|
2011-03-18 23:37:00 +01:00
|
|
|
|
|
|
|
return internal_iter;
|
|
|
|
}
|
|
|
|
|
|
|
|
Iterator* DBImpl::TEST_NewInternalIterator() {
|
|
|
|
SequenceNumber ignored;
|
|
|
|
return NewInternalIterator(ReadOptions(), &ignored);
|
|
|
|
}
|
|
|
|
|
2014-01-17 06:56:26 +01:00
|
|
|
std::pair<Iterator*, Iterator*> DBImpl::GetTailingIteratorPair(
|
|
|
|
const ReadOptions& options,
|
|
|
|
uint64_t* superversion_number) {
|
|
|
|
|
|
|
|
MemTable* mutable_mem;
|
MemTableListVersion
Summary:
MemTableListVersion is to MemTableList what Version is to VersionSet. I took almost the same ideas to develop MemTableListVersion. The reason is to have copying std::list done in background, while flushing, rather than in foreground (MultiGet() and NewIterator()) under a mutex! Also, whenever we copied MemTableList, we copied also some MemTableList metadata (flush_requested_, commit_in_progress_, etc.), which was wasteful.
This diff avoids std::list copy under a mutex in both MultiGet() and NewIterator(). I created a small database with some number of immutable memtables, and creating 100.000 iterators in a single-thread (!) decreased from {188739, 215703, 198028} to {154352, 164035, 159817}. A lot of the savings come from code under a mutex, so we should see much higher savings with multiple threads. Creating new iterator is very important to LogDevice team.
I also think this diff will make SuperVersion obsolete for performance reasons. I will try it in the next diff. SuperVersion gave us huge savings on Get() code path, but I think that most of the savings came from copying MemTableList under a mutex. If we had MemTableListVersion, we would never need to copy the entire object (like we still do in NewIterator() and MultiGet())
Test Plan: `make check` works. I will also do `make valgrind_check` before commit
Reviewers: dhruba, haobo, kailiu, sdong, emayanke, tnovak
Reviewed By: kailiu
CC: leveldb
Differential Revision: https://reviews.facebook.net/D15255
2014-01-24 23:52:08 +01:00
|
|
|
MemTableListVersion* immutable_mems;
|
2014-01-17 06:56:26 +01:00
|
|
|
Version* version;
|
|
|
|
|
|
|
|
// get all child iterators and bump their refcounts under lock
|
|
|
|
mutex_.Lock();
|
2014-01-29 22:28:50 +01:00
|
|
|
mutable_mem = default_cfd_->mem();
|
2014-01-17 06:56:26 +01:00
|
|
|
mutable_mem->Ref();
|
2014-01-29 22:28:50 +01:00
|
|
|
immutable_mems = default_cfd_->imm()->current();
|
MemTableListVersion
Summary:
MemTableListVersion is to MemTableList what Version is to VersionSet. I took almost the same ideas to develop MemTableListVersion. The reason is to have copying std::list done in background, while flushing, rather than in foreground (MultiGet() and NewIterator()) under a mutex! Also, whenever we copied MemTableList, we copied also some MemTableList metadata (flush_requested_, commit_in_progress_, etc.), which was wasteful.
This diff avoids std::list copy under a mutex in both MultiGet() and NewIterator(). I created a small database with some number of immutable memtables, and creating 100.000 iterators in a single-thread (!) decreased from {188739, 215703, 198028} to {154352, 164035, 159817}. A lot of the savings come from code under a mutex, so we should see much higher savings with multiple threads. Creating new iterator is very important to LogDevice team.
I also think this diff will make SuperVersion obsolete for performance reasons. I will try it in the next diff. SuperVersion gave us huge savings on Get() code path, but I think that most of the savings came from copying MemTableList under a mutex. If we had MemTableListVersion, we would never need to copy the entire object (like we still do in NewIterator() and MultiGet())
Test Plan: `make check` works. I will also do `make valgrind_check` before commit
Reviewers: dhruba, haobo, kailiu, sdong, emayanke, tnovak
Reviewed By: kailiu
CC: leveldb
Differential Revision: https://reviews.facebook.net/D15255
2014-01-24 23:52:08 +01:00
|
|
|
immutable_mems->Ref();
|
2014-01-29 22:28:50 +01:00
|
|
|
version = default_cfd_->current();
|
2014-01-17 06:56:26 +01:00
|
|
|
version->Ref();
|
|
|
|
if (superversion_number != nullptr) {
|
|
|
|
*superversion_number = CurrentVersionNumber();
|
|
|
|
}
|
|
|
|
mutex_.Unlock();
|
|
|
|
|
|
|
|
Iterator* mutable_iter = mutable_mem->NewIterator(options);
|
|
|
|
IterState* mutable_cleanup = new IterState();
|
MemTableListVersion
Summary:
MemTableListVersion is to MemTableList what Version is to VersionSet. I took almost the same ideas to develop MemTableListVersion. The reason is to have copying std::list done in background, while flushing, rather than in foreground (MultiGet() and NewIterator()) under a mutex! Also, whenever we copied MemTableList, we copied also some MemTableList metadata (flush_requested_, commit_in_progress_, etc.), which was wasteful.
This diff avoids std::list copy under a mutex in both MultiGet() and NewIterator(). I created a small database with some number of immutable memtables, and creating 100.000 iterators in a single-thread (!) decreased from {188739, 215703, 198028} to {154352, 164035, 159817}. A lot of the savings come from code under a mutex, so we should see much higher savings with multiple threads. Creating new iterator is very important to LogDevice team.
I also think this diff will make SuperVersion obsolete for performance reasons. I will try it in the next diff. SuperVersion gave us huge savings on Get() code path, but I think that most of the savings came from copying MemTableList under a mutex. If we had MemTableListVersion, we would never need to copy the entire object (like we still do in NewIterator() and MultiGet())
Test Plan: `make check` works. I will also do `make valgrind_check` before commit
Reviewers: dhruba, haobo, kailiu, sdong, emayanke, tnovak
Reviewed By: kailiu
CC: leveldb
Differential Revision: https://reviews.facebook.net/D15255
2014-01-24 23:52:08 +01:00
|
|
|
mutable_cleanup->mem = mutable_mem;
|
2014-01-17 06:56:26 +01:00
|
|
|
mutable_cleanup->db = this;
|
|
|
|
mutable_cleanup->mu = &mutex_;
|
|
|
|
mutable_iter->RegisterCleanup(CleanupIteratorState, mutable_cleanup, nullptr);
|
|
|
|
|
|
|
|
// create a DBIter that only uses memtable content; see NewIterator()
|
|
|
|
mutable_iter = NewDBIterator(&dbname_, env_, options_, user_comparator(),
|
|
|
|
mutable_iter, kMaxSequenceNumber);
|
|
|
|
|
|
|
|
Iterator* immutable_iter;
|
|
|
|
IterState* immutable_cleanup = new IterState();
|
|
|
|
std::vector<Iterator*> list;
|
MemTableListVersion
Summary:
MemTableListVersion is to MemTableList what Version is to VersionSet. I took almost the same ideas to develop MemTableListVersion. The reason is to have copying std::list done in background, while flushing, rather than in foreground (MultiGet() and NewIterator()) under a mutex! Also, whenever we copied MemTableList, we copied also some MemTableList metadata (flush_requested_, commit_in_progress_, etc.), which was wasteful.
This diff avoids std::list copy under a mutex in both MultiGet() and NewIterator(). I created a small database with some number of immutable memtables, and creating 100.000 iterators in a single-thread (!) decreased from {188739, 215703, 198028} to {154352, 164035, 159817}. A lot of the savings come from code under a mutex, so we should see much higher savings with multiple threads. Creating new iterator is very important to LogDevice team.
I also think this diff will make SuperVersion obsolete for performance reasons. I will try it in the next diff. SuperVersion gave us huge savings on Get() code path, but I think that most of the savings came from copying MemTableList under a mutex. If we had MemTableListVersion, we would never need to copy the entire object (like we still do in NewIterator() and MultiGet())
Test Plan: `make check` works. I will also do `make valgrind_check` before commit
Reviewers: dhruba, haobo, kailiu, sdong, emayanke, tnovak
Reviewed By: kailiu
CC: leveldb
Differential Revision: https://reviews.facebook.net/D15255
2014-01-24 23:52:08 +01:00
|
|
|
immutable_mems->AddIterators(options, &list);
|
|
|
|
immutable_cleanup->imm = immutable_mems;
|
2014-01-17 06:56:26 +01:00
|
|
|
version->AddIterators(options, storage_options_, &list);
|
|
|
|
immutable_cleanup->version = version;
|
|
|
|
immutable_cleanup->db = this;
|
|
|
|
immutable_cleanup->mu = &mutex_;
|
|
|
|
|
|
|
|
immutable_iter =
|
|
|
|
NewMergingIterator(&internal_comparator_, &list[0], list.size());
|
|
|
|
immutable_iter->RegisterCleanup(CleanupIteratorState, immutable_cleanup,
|
|
|
|
nullptr);
|
|
|
|
|
|
|
|
// create a DBIter that only uses memtable content; see NewIterator()
|
|
|
|
immutable_iter = NewDBIterator(&dbname_, env_, options_, user_comparator(),
|
|
|
|
immutable_iter, kMaxSequenceNumber);
|
|
|
|
|
|
|
|
return std::make_pair(mutable_iter, immutable_iter);
|
|
|
|
}
|
|
|
|
|
2011-03-23 00:24:02 +01:00
|
|
|
int64_t DBImpl::TEST_MaxNextLevelOverlappingBytes() {
|
2011-03-22 19:32:49 +01:00
|
|
|
MutexLock l(&mutex_);
|
2014-01-29 22:28:50 +01:00
|
|
|
return default_cfd_->current()->MaxNextLevelOverlappingBytes();
|
2011-03-22 19:32:49 +01:00
|
|
|
}
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
Status DBImpl::Get(const ReadOptions& options,
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 20:14:09 +01:00
|
|
|
const ColumnFamilyHandle& column_family, const Slice& key,
|
2011-03-18 23:37:00 +01:00
|
|
|
std::string* value) {
|
2014-01-28 20:05:04 +01:00
|
|
|
return GetImpl(options, column_family, key, value);
|
2013-07-06 03:49:18 +02:00
|
|
|
}
|
|
|
|
|
2013-12-20 18:57:58 +01:00
|
|
|
// DeletionState gets created and destructed outside of the lock -- we
|
|
|
|
// use this convinently to:
|
|
|
|
// * malloc one SuperVersion() outside of the lock -- new_superversion
|
|
|
|
// * delete one SuperVersion() outside of the lock -- superversion_to_free
|
|
|
|
//
|
|
|
|
// However, if InstallSuperVersion() gets called twice with the same,
|
|
|
|
// deletion_state, we can't reuse the SuperVersion() that got malloced because
|
|
|
|
// first call already used it. In that rare case, we take a hit and create a
|
|
|
|
// new SuperVersion() inside of the mutex. We do similar thing
|
|
|
|
// for superversion_to_free
|
2014-01-24 23:30:28 +01:00
|
|
|
void DBImpl::InstallSuperVersion(ColumnFamilyData* cfd,
|
|
|
|
DeletionState& deletion_state) {
|
2014-01-29 22:28:50 +01:00
|
|
|
mutex_.AssertHeld();
|
2013-12-20 18:57:58 +01:00
|
|
|
// if new_superversion == nullptr, it means somebody already used it
|
|
|
|
SuperVersion* new_superversion =
|
|
|
|
(deletion_state.new_superversion != nullptr) ?
|
|
|
|
deletion_state.new_superversion : new SuperVersion();
|
2014-01-29 22:28:50 +01:00
|
|
|
SuperVersion* old_superversion = cfd->InstallSuperVersion(new_superversion);
|
2013-12-20 18:57:58 +01:00
|
|
|
deletion_state.new_superversion = nullptr;
|
|
|
|
if (deletion_state.superversion_to_free != nullptr) {
|
|
|
|
// somebody already put it there
|
|
|
|
delete old_superversion;
|
|
|
|
} else {
|
|
|
|
deletion_state.superversion_to_free = old_superversion;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-07-06 03:49:18 +02:00
|
|
|
Status DBImpl::GetImpl(const ReadOptions& options,
|
2014-01-28 20:05:04 +01:00
|
|
|
const ColumnFamilyHandle& column_family,
|
|
|
|
const Slice& key, std::string* value,
|
2013-07-26 21:57:01 +02:00
|
|
|
bool* value_found) {
|
2014-01-07 23:33:15 +01:00
|
|
|
StopWatch sw(env_, options_.statistics.get(), DB_GET, false);
|
2014-01-28 20:05:04 +01:00
|
|
|
|
|
|
|
mutex_.Lock();
|
|
|
|
auto cfd = versions_->GetColumnFamilySet()->GetColumnFamily(column_family.id);
|
|
|
|
// this is asserting because client calling Get() with undefined
|
|
|
|
// ColumnFamilyHandle is undefined behavior.
|
|
|
|
assert(cfd != nullptr);
|
2014-01-29 22:28:50 +01:00
|
|
|
SuperVersion* get_version = cfd->GetSuperVersion()->Ref();
|
2014-01-28 20:05:04 +01:00
|
|
|
mutex_.Unlock();
|
|
|
|
|
2011-06-22 04:36:45 +02:00
|
|
|
SequenceNumber snapshot;
|
2013-02-15 20:53:17 +01:00
|
|
|
if (options.snapshot != nullptr) {
|
2011-06-22 04:36:45 +02:00
|
|
|
snapshot = reinterpret_cast<const SnapshotImpl*>(options.snapshot)->number_;
|
|
|
|
} else {
|
|
|
|
snapshot = versions_->LastSequence();
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
2011-06-22 04:36:45 +02:00
|
|
|
|
2011-08-22 23:08:51 +02:00
|
|
|
bool have_stat_update = false;
|
2011-06-22 04:36:45 +02:00
|
|
|
Version::GetStats stats;
|
2013-03-21 23:59:47 +01:00
|
|
|
|
[RocksDB] [MergeOperator] The new Merge Interface! Uses merge sequences.
Summary:
Here are the major changes to the Merge Interface. It has been expanded
to handle cases where the MergeOperator is not associative. It does so by stacking
up merge operations while scanning through the key history (i.e.: during Get() or
Compaction), until a valid Put/Delete/end-of-history is encountered; it then
applies all of the merge operations in the correct sequence starting with the
base/sentinel value.
I have also introduced an "AssociativeMerge" function which allows the user to
take advantage of associative merge operations (such as in the case of counters).
The implementation will always attempt to merge the operations/operands themselves
together when they are encountered, and will resort to the "stacking" method if
and only if the "associative-merge" fails.
This implementation is conjectured to allow MergeOperator to handle the general
case, while still providing the user with the ability to take advantage of certain
efficiencies in their own merge-operator / data-structure.
NOTE: This is a preliminary diff. This must still go through a lot of review,
revision, and testing. Feedback welcome!
Test Plan:
-This is a preliminary diff. I have only just begun testing/debugging it.
-I will be testing this with the existing MergeOperator use-cases and unit-tests
(counters, string-append, and redis-lists)
-I will be "desk-checking" and walking through the code with the help gdb.
-I will find a way of stress-testing the new interface / implementation using
db_bench, db_test, merge_test, and/or db_stress.
-I will ensure that my tests cover all cases: Get-Memtable,
Get-Immutable-Memtable, Get-from-Disk, Iterator-Range-Scan, Flush-Memtable-to-L0,
Compaction-L0-L1, Compaction-Ln-L(n+1), Put/Delete found, Put/Delete not-found,
end-of-history, end-of-file, etc.
-A lot of feedback from the reviewers.
Reviewers: haobo, dhruba, zshao, emayanke
Reviewed By: haobo
CC: leveldb
Differential Revision: https://reviews.facebook.net/D11499
2013-08-06 05:14:32 +02:00
|
|
|
// Prepare to store a list of merge operations if merge occurs.
|
2013-12-03 03:34:05 +01:00
|
|
|
MergeContext merge_context;
|
[RocksDB] [MergeOperator] The new Merge Interface! Uses merge sequences.
Summary:
Here are the major changes to the Merge Interface. It has been expanded
to handle cases where the MergeOperator is not associative. It does so by stacking
up merge operations while scanning through the key history (i.e.: during Get() or
Compaction), until a valid Put/Delete/end-of-history is encountered; it then
applies all of the merge operations in the correct sequence starting with the
base/sentinel value.
I have also introduced an "AssociativeMerge" function which allows the user to
take advantage of associative merge operations (such as in the case of counters).
The implementation will always attempt to merge the operations/operands themselves
together when they are encountered, and will resort to the "stacking" method if
and only if the "associative-merge" fails.
This implementation is conjectured to allow MergeOperator to handle the general
case, while still providing the user with the ability to take advantage of certain
efficiencies in their own merge-operator / data-structure.
NOTE: This is a preliminary diff. This must still go through a lot of review,
revision, and testing. Feedback welcome!
Test Plan:
-This is a preliminary diff. I have only just begun testing/debugging it.
-I will be testing this with the existing MergeOperator use-cases and unit-tests
(counters, string-append, and redis-lists)
-I will be "desk-checking" and walking through the code with the help gdb.
-I will find a way of stress-testing the new interface / implementation using
db_bench, db_test, merge_test, and/or db_stress.
-I will ensure that my tests cover all cases: Get-Memtable,
Get-Immutable-Memtable, Get-from-Disk, Iterator-Range-Scan, Flush-Memtable-to-L0,
Compaction-L0-L1, Compaction-Ln-L(n+1), Put/Delete found, Put/Delete not-found,
end-of-history, end-of-file, etc.
-A lot of feedback from the reviewers.
Reviewers: haobo, dhruba, zshao, emayanke
Reviewed By: haobo
CC: leveldb
Differential Revision: https://reviews.facebook.net/D11499
2013-08-06 05:14:32 +02:00
|
|
|
|
2014-01-28 20:05:04 +01:00
|
|
|
Status s;
|
2013-03-26 02:01:47 +01:00
|
|
|
// First look in the memtable, then in the immutable memtable (if any).
|
2013-03-21 23:59:47 +01:00
|
|
|
// s is both in/out. When in, s could either be OK or MergeInProgress.
|
[RocksDB] [MergeOperator] The new Merge Interface! Uses merge sequences.
Summary:
Here are the major changes to the Merge Interface. It has been expanded
to handle cases where the MergeOperator is not associative. It does so by stacking
up merge operations while scanning through the key history (i.e.: during Get() or
Compaction), until a valid Put/Delete/end-of-history is encountered; it then
applies all of the merge operations in the correct sequence starting with the
base/sentinel value.
I have also introduced an "AssociativeMerge" function which allows the user to
take advantage of associative merge operations (such as in the case of counters).
The implementation will always attempt to merge the operations/operands themselves
together when they are encountered, and will resort to the "stacking" method if
and only if the "associative-merge" fails.
This implementation is conjectured to allow MergeOperator to handle the general
case, while still providing the user with the ability to take advantage of certain
efficiencies in their own merge-operator / data-structure.
NOTE: This is a preliminary diff. This must still go through a lot of review,
revision, and testing. Feedback welcome!
Test Plan:
-This is a preliminary diff. I have only just begun testing/debugging it.
-I will be testing this with the existing MergeOperator use-cases and unit-tests
(counters, string-append, and redis-lists)
-I will be "desk-checking" and walking through the code with the help gdb.
-I will find a way of stress-testing the new interface / implementation using
db_bench, db_test, merge_test, and/or db_stress.
-I will ensure that my tests cover all cases: Get-Memtable,
Get-Immutable-Memtable, Get-from-Disk, Iterator-Range-Scan, Flush-Memtable-to-L0,
Compaction-L0-L1, Compaction-Ln-L(n+1), Put/Delete found, Put/Delete not-found,
end-of-history, end-of-file, etc.
-A lot of feedback from the reviewers.
Reviewers: haobo, dhruba, zshao, emayanke
Reviewed By: haobo
CC: leveldb
Differential Revision: https://reviews.facebook.net/D11499
2013-08-06 05:14:32 +02:00
|
|
|
// merge_operands will contain the sequence of merges in the latter case.
|
2013-03-26 02:01:47 +01:00
|
|
|
LookupKey lkey(key, snapshot);
|
2013-12-20 18:57:58 +01:00
|
|
|
if (get_version->mem->Get(lkey, value, &s, merge_context, options_)) {
|
2013-03-26 02:01:47 +01:00
|
|
|
// Done
|
2013-12-03 21:59:53 +01:00
|
|
|
RecordTick(options_.statistics.get(), MEMTABLE_HIT);
|
MemTableListVersion
Summary:
MemTableListVersion is to MemTableList what Version is to VersionSet. I took almost the same ideas to develop MemTableListVersion. The reason is to have copying std::list done in background, while flushing, rather than in foreground (MultiGet() and NewIterator()) under a mutex! Also, whenever we copied MemTableList, we copied also some MemTableList metadata (flush_requested_, commit_in_progress_, etc.), which was wasteful.
This diff avoids std::list copy under a mutex in both MultiGet() and NewIterator(). I created a small database with some number of immutable memtables, and creating 100.000 iterators in a single-thread (!) decreased from {188739, 215703, 198028} to {154352, 164035, 159817}. A lot of the savings come from code under a mutex, so we should see much higher savings with multiple threads. Creating new iterator is very important to LogDevice team.
I also think this diff will make SuperVersion obsolete for performance reasons. I will try it in the next diff. SuperVersion gave us huge savings on Get() code path, but I think that most of the savings came from copying MemTableList under a mutex. If we had MemTableListVersion, we would never need to copy the entire object (like we still do in NewIterator() and MultiGet())
Test Plan: `make check` works. I will also do `make valgrind_check` before commit
Reviewers: dhruba, haobo, kailiu, sdong, emayanke, tnovak
Reviewed By: kailiu
CC: leveldb
Differential Revision: https://reviews.facebook.net/D15255
2014-01-24 23:52:08 +01:00
|
|
|
} else if (get_version->imm->Get(lkey, value, &s, merge_context, options_)) {
|
2013-03-26 02:01:47 +01:00
|
|
|
// Done
|
2013-12-03 21:59:53 +01:00
|
|
|
RecordTick(options_.statistics.get(), MEMTABLE_HIT);
|
2013-03-26 02:01:47 +01:00
|
|
|
} else {
|
2013-12-20 18:57:58 +01:00
|
|
|
get_version->current->Get(options, lkey, value, &s, &merge_context, &stats,
|
|
|
|
options_, value_found);
|
2013-03-26 02:01:47 +01:00
|
|
|
have_stat_update = true;
|
2013-12-03 21:59:53 +01:00
|
|
|
RecordTick(options_.statistics.get(), MEMTABLE_MISS);
|
2011-06-22 04:36:45 +02:00
|
|
|
}
|
2011-08-22 23:08:51 +02:00
|
|
|
|
2013-12-20 18:57:58 +01:00
|
|
|
bool delete_get_version = false;
|
|
|
|
if (!options_.disable_seek_compaction && have_stat_update) {
|
|
|
|
mutex_.Lock();
|
|
|
|
if (get_version->current->UpdateStats(stats)) {
|
|
|
|
MaybeScheduleFlushOrCompaction();
|
|
|
|
}
|
|
|
|
if (get_version->Unref()) {
|
|
|
|
get_version->Cleanup();
|
|
|
|
delete_get_version = true;
|
|
|
|
}
|
|
|
|
mutex_.Unlock();
|
|
|
|
} else {
|
|
|
|
if (get_version->Unref()) {
|
|
|
|
mutex_.Lock();
|
|
|
|
get_version->Cleanup();
|
|
|
|
mutex_.Unlock();
|
|
|
|
delete_get_version = true;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (delete_get_version) {
|
|
|
|
delete get_version;
|
2011-06-22 04:36:45 +02:00
|
|
|
}
|
2013-11-25 20:55:36 +01:00
|
|
|
|
2013-08-27 07:23:47 +02:00
|
|
|
// Note, tickers are atomic now - no lock protection needed any more.
|
2013-11-22 23:14:05 +01:00
|
|
|
RecordTick(options_.statistics.get(), NUMBER_KEYS_READ);
|
|
|
|
RecordTick(options_.statistics.get(), BYTES_READ, value->size());
|
2011-06-22 04:36:45 +02:00
|
|
|
return s;
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 20:14:09 +01:00
|
|
|
std::vector<Status> DBImpl::MultiGet(
|
|
|
|
const ReadOptions& options,
|
|
|
|
const std::vector<ColumnFamilyHandle>& column_family,
|
|
|
|
const std::vector<Slice>& keys, std::vector<std::string>* values) {
|
2013-06-05 20:22:38 +02:00
|
|
|
|
2014-01-07 23:33:15 +01:00
|
|
|
StopWatch sw(env_, options_.statistics.get(), DB_MULTIGET, false);
|
2013-06-05 20:22:38 +02:00
|
|
|
SequenceNumber snapshot;
|
2013-11-25 20:55:36 +01:00
|
|
|
std::vector<MemTable*> to_delete;
|
|
|
|
|
2013-08-27 07:23:47 +02:00
|
|
|
mutex_.Lock();
|
2013-06-05 20:22:38 +02:00
|
|
|
if (options.snapshot != nullptr) {
|
|
|
|
snapshot = reinterpret_cast<const SnapshotImpl*>(options.snapshot)->number_;
|
|
|
|
} else {
|
|
|
|
snapshot = versions_->LastSequence();
|
|
|
|
}
|
|
|
|
|
2014-01-24 23:30:28 +01:00
|
|
|
// TODO only works for default column family
|
2014-01-29 22:28:50 +01:00
|
|
|
MemTable* mem = default_cfd_->mem();
|
|
|
|
MemTableListVersion* imm = default_cfd_->imm()->current();
|
|
|
|
Version* current = default_cfd_->current();
|
2013-06-05 20:22:38 +02:00
|
|
|
mem->Ref();
|
MemTableListVersion
Summary:
MemTableListVersion is to MemTableList what Version is to VersionSet. I took almost the same ideas to develop MemTableListVersion. The reason is to have copying std::list done in background, while flushing, rather than in foreground (MultiGet() and NewIterator()) under a mutex! Also, whenever we copied MemTableList, we copied also some MemTableList metadata (flush_requested_, commit_in_progress_, etc.), which was wasteful.
This diff avoids std::list copy under a mutex in both MultiGet() and NewIterator(). I created a small database with some number of immutable memtables, and creating 100.000 iterators in a single-thread (!) decreased from {188739, 215703, 198028} to {154352, 164035, 159817}. A lot of the savings come from code under a mutex, so we should see much higher savings with multiple threads. Creating new iterator is very important to LogDevice team.
I also think this diff will make SuperVersion obsolete for performance reasons. I will try it in the next diff. SuperVersion gave us huge savings on Get() code path, but I think that most of the savings came from copying MemTableList under a mutex. If we had MemTableListVersion, we would never need to copy the entire object (like we still do in NewIterator() and MultiGet())
Test Plan: `make check` works. I will also do `make valgrind_check` before commit
Reviewers: dhruba, haobo, kailiu, sdong, emayanke, tnovak
Reviewed By: kailiu
CC: leveldb
Differential Revision: https://reviews.facebook.net/D15255
2014-01-24 23:52:08 +01:00
|
|
|
imm->Ref();
|
2013-06-05 20:22:38 +02:00
|
|
|
current->Ref();
|
|
|
|
|
|
|
|
// Unlock while reading from files and memtables
|
|
|
|
|
|
|
|
mutex_.Unlock();
|
|
|
|
bool have_stat_update = false;
|
|
|
|
Version::GetStats stats;
|
|
|
|
|
2013-12-03 03:34:05 +01:00
|
|
|
// Contain a list of merge operations if merge occurs.
|
|
|
|
MergeContext merge_context;
|
[RocksDB] [MergeOperator] The new Merge Interface! Uses merge sequences.
Summary:
Here are the major changes to the Merge Interface. It has been expanded
to handle cases where the MergeOperator is not associative. It does so by stacking
up merge operations while scanning through the key history (i.e.: during Get() or
Compaction), until a valid Put/Delete/end-of-history is encountered; it then
applies all of the merge operations in the correct sequence starting with the
base/sentinel value.
I have also introduced an "AssociativeMerge" function which allows the user to
take advantage of associative merge operations (such as in the case of counters).
The implementation will always attempt to merge the operations/operands themselves
together when they are encountered, and will resort to the "stacking" method if
and only if the "associative-merge" fails.
This implementation is conjectured to allow MergeOperator to handle the general
case, while still providing the user with the ability to take advantage of certain
efficiencies in their own merge-operator / data-structure.
NOTE: This is a preliminary diff. This must still go through a lot of review,
revision, and testing. Feedback welcome!
Test Plan:
-This is a preliminary diff. I have only just begun testing/debugging it.
-I will be testing this with the existing MergeOperator use-cases and unit-tests
(counters, string-append, and redis-lists)
-I will be "desk-checking" and walking through the code with the help gdb.
-I will find a way of stress-testing the new interface / implementation using
db_bench, db_test, merge_test, and/or db_stress.
-I will ensure that my tests cover all cases: Get-Memtable,
Get-Immutable-Memtable, Get-from-Disk, Iterator-Range-Scan, Flush-Memtable-to-L0,
Compaction-L0-L1, Compaction-Ln-L(n+1), Put/Delete found, Put/Delete not-found,
end-of-history, end-of-file, etc.
-A lot of feedback from the reviewers.
Reviewers: haobo, dhruba, zshao, emayanke
Reviewed By: haobo
CC: leveldb
Differential Revision: https://reviews.facebook.net/D11499
2013-08-06 05:14:32 +02:00
|
|
|
|
2013-06-05 20:22:38 +02:00
|
|
|
// Note: this always resizes the values array
|
|
|
|
int numKeys = keys.size();
|
|
|
|
std::vector<Status> statList(numKeys);
|
|
|
|
values->resize(numKeys);
|
|
|
|
|
|
|
|
// Keep track of bytes that we read for statistics-recording later
|
|
|
|
uint64_t bytesRead = 0;
|
|
|
|
|
|
|
|
// For each of the given keys, apply the entire "get" process as follows:
|
|
|
|
// First look in the memtable, then in the immutable memtable (if any).
|
|
|
|
// s is both in/out. When in, s could either be OK or MergeInProgress.
|
[RocksDB] [MergeOperator] The new Merge Interface! Uses merge sequences.
Summary:
Here are the major changes to the Merge Interface. It has been expanded
to handle cases where the MergeOperator is not associative. It does so by stacking
up merge operations while scanning through the key history (i.e.: during Get() or
Compaction), until a valid Put/Delete/end-of-history is encountered; it then
applies all of the merge operations in the correct sequence starting with the
base/sentinel value.
I have also introduced an "AssociativeMerge" function which allows the user to
take advantage of associative merge operations (such as in the case of counters).
The implementation will always attempt to merge the operations/operands themselves
together when they are encountered, and will resort to the "stacking" method if
and only if the "associative-merge" fails.
This implementation is conjectured to allow MergeOperator to handle the general
case, while still providing the user with the ability to take advantage of certain
efficiencies in their own merge-operator / data-structure.
NOTE: This is a preliminary diff. This must still go through a lot of review,
revision, and testing. Feedback welcome!
Test Plan:
-This is a preliminary diff. I have only just begun testing/debugging it.
-I will be testing this with the existing MergeOperator use-cases and unit-tests
(counters, string-append, and redis-lists)
-I will be "desk-checking" and walking through the code with the help gdb.
-I will find a way of stress-testing the new interface / implementation using
db_bench, db_test, merge_test, and/or db_stress.
-I will ensure that my tests cover all cases: Get-Memtable,
Get-Immutable-Memtable, Get-from-Disk, Iterator-Range-Scan, Flush-Memtable-to-L0,
Compaction-L0-L1, Compaction-Ln-L(n+1), Put/Delete found, Put/Delete not-found,
end-of-history, end-of-file, etc.
-A lot of feedback from the reviewers.
Reviewers: haobo, dhruba, zshao, emayanke
Reviewed By: haobo
CC: leveldb
Differential Revision: https://reviews.facebook.net/D11499
2013-08-06 05:14:32 +02:00
|
|
|
// merge_operands will contain the sequence of merges in the latter case.
|
|
|
|
for (int i=0; i<numKeys; ++i) {
|
2013-12-03 03:34:05 +01:00
|
|
|
merge_context.Clear();
|
2013-06-05 20:22:38 +02:00
|
|
|
Status& s = statList[i];
|
|
|
|
std::string* value = &(*values)[i];
|
|
|
|
|
|
|
|
LookupKey lkey(keys[i], snapshot);
|
2013-12-03 03:34:05 +01:00
|
|
|
if (mem->Get(lkey, value, &s, merge_context, options_)) {
|
2013-06-05 20:22:38 +02:00
|
|
|
// Done
|
MemTableListVersion
Summary:
MemTableListVersion is to MemTableList what Version is to VersionSet. I took almost the same ideas to develop MemTableListVersion. The reason is to have copying std::list done in background, while flushing, rather than in foreground (MultiGet() and NewIterator()) under a mutex! Also, whenever we copied MemTableList, we copied also some MemTableList metadata (flush_requested_, commit_in_progress_, etc.), which was wasteful.
This diff avoids std::list copy under a mutex in both MultiGet() and NewIterator(). I created a small database with some number of immutable memtables, and creating 100.000 iterators in a single-thread (!) decreased from {188739, 215703, 198028} to {154352, 164035, 159817}. A lot of the savings come from code under a mutex, so we should see much higher savings with multiple threads. Creating new iterator is very important to LogDevice team.
I also think this diff will make SuperVersion obsolete for performance reasons. I will try it in the next diff. SuperVersion gave us huge savings on Get() code path, but I think that most of the savings came from copying MemTableList under a mutex. If we had MemTableListVersion, we would never need to copy the entire object (like we still do in NewIterator() and MultiGet())
Test Plan: `make check` works. I will also do `make valgrind_check` before commit
Reviewers: dhruba, haobo, kailiu, sdong, emayanke, tnovak
Reviewed By: kailiu
CC: leveldb
Differential Revision: https://reviews.facebook.net/D15255
2014-01-24 23:52:08 +01:00
|
|
|
} else if (imm->Get(lkey, value, &s, merge_context, options_)) {
|
2013-06-05 20:22:38 +02:00
|
|
|
// Done
|
|
|
|
} else {
|
2013-12-03 03:34:05 +01:00
|
|
|
current->Get(options, lkey, value, &s, &merge_context, &stats, options_);
|
2013-06-05 20:22:38 +02:00
|
|
|
have_stat_update = true;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (s.ok()) {
|
|
|
|
bytesRead += value->size();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Post processing (decrement reference counts and record statistics)
|
|
|
|
mutex_.Lock();
|
|
|
|
if (!options_.disable_seek_compaction &&
|
|
|
|
have_stat_update && current->UpdateStats(stats)) {
|
2013-10-15 00:12:15 +02:00
|
|
|
MaybeScheduleFlushOrCompaction();
|
2013-06-05 20:22:38 +02:00
|
|
|
}
|
2013-11-25 20:55:36 +01:00
|
|
|
MemTable* m = mem->Unref();
|
MemTableListVersion
Summary:
MemTableListVersion is to MemTableList what Version is to VersionSet. I took almost the same ideas to develop MemTableListVersion. The reason is to have copying std::list done in background, while flushing, rather than in foreground (MultiGet() and NewIterator()) under a mutex! Also, whenever we copied MemTableList, we copied also some MemTableList metadata (flush_requested_, commit_in_progress_, etc.), which was wasteful.
This diff avoids std::list copy under a mutex in both MultiGet() and NewIterator(). I created a small database with some number of immutable memtables, and creating 100.000 iterators in a single-thread (!) decreased from {188739, 215703, 198028} to {154352, 164035, 159817}. A lot of the savings come from code under a mutex, so we should see much higher savings with multiple threads. Creating new iterator is very important to LogDevice team.
I also think this diff will make SuperVersion obsolete for performance reasons. I will try it in the next diff. SuperVersion gave us huge savings on Get() code path, but I think that most of the savings came from copying MemTableList under a mutex. If we had MemTableListVersion, we would never need to copy the entire object (like we still do in NewIterator() and MultiGet())
Test Plan: `make check` works. I will also do `make valgrind_check` before commit
Reviewers: dhruba, haobo, kailiu, sdong, emayanke, tnovak
Reviewed By: kailiu
CC: leveldb
Differential Revision: https://reviews.facebook.net/D15255
2014-01-24 23:52:08 +01:00
|
|
|
imm->Unref(&to_delete);
|
2013-06-05 20:22:38 +02:00
|
|
|
current->Unref();
|
2013-08-27 07:23:47 +02:00
|
|
|
mutex_.Unlock();
|
|
|
|
|
2013-11-25 20:55:36 +01:00
|
|
|
// free up all obsolete memtables outside the mutex
|
|
|
|
delete m;
|
|
|
|
for (MemTable* v: to_delete) delete v;
|
|
|
|
|
2013-11-22 23:14:05 +01:00
|
|
|
RecordTick(options_.statistics.get(), NUMBER_MULTIGET_CALLS);
|
|
|
|
RecordTick(options_.statistics.get(), NUMBER_MULTIGET_KEYS_READ, numKeys);
|
|
|
|
RecordTick(options_.statistics.get(), NUMBER_MULTIGET_BYTES_READ, bytesRead);
|
2013-06-05 20:22:38 +02:00
|
|
|
|
|
|
|
return statList;
|
|
|
|
}
|
|
|
|
|
2014-01-02 18:08:12 +01:00
|
|
|
Status DBImpl::CreateColumnFamily(const ColumnFamilyOptions& options,
|
2014-01-06 22:31:06 +01:00
|
|
|
const std::string& column_family_name,
|
2014-01-02 18:08:12 +01:00
|
|
|
ColumnFamilyHandle* handle) {
|
2014-01-24 23:30:28 +01:00
|
|
|
MutexLock l(&mutex_);
|
2014-01-24 18:10:24 +01:00
|
|
|
if (versions_->GetColumnFamilySet()->Exists(column_family_name)) {
|
2014-01-22 20:44:53 +01:00
|
|
|
return Status::InvalidArgument("Column family already exists");
|
|
|
|
}
|
2014-01-22 02:01:52 +01:00
|
|
|
VersionEdit edit;
|
2014-01-06 22:31:06 +01:00
|
|
|
edit.AddColumnFamily(column_family_name);
|
2014-01-22 20:44:53 +01:00
|
|
|
handle->id = versions_->GetColumnFamilySet()->GetNextColumnFamilyID();
|
2014-01-02 18:08:12 +01:00
|
|
|
edit.SetColumnFamily(handle->id);
|
2014-01-27 22:55:47 +01:00
|
|
|
Status s = versions_->LogAndApply(default_cfd_, &edit, &mutex_);
|
2014-01-06 22:31:06 +01:00
|
|
|
if (s.ok()) {
|
|
|
|
// add to internal data structures
|
2014-01-11 00:12:34 +01:00
|
|
|
versions_->CreateColumnFamily(options, &edit);
|
2014-01-06 22:31:06 +01:00
|
|
|
}
|
2014-01-28 20:05:04 +01:00
|
|
|
Log(options_.info_log, "Created column family %s\n",
|
|
|
|
column_family_name.c_str());
|
2014-01-06 22:31:06 +01:00
|
|
|
return s;
|
2014-01-02 18:08:12 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
Status DBImpl::DropColumnFamily(const ColumnFamilyHandle& column_family) {
|
2014-01-11 00:12:34 +01:00
|
|
|
if (column_family.id == 0) {
|
|
|
|
return Status::InvalidArgument("Can't drop default column family");
|
|
|
|
}
|
2014-01-29 20:50:49 +01:00
|
|
|
mutex_.Lock();
|
2014-01-22 20:44:53 +01:00
|
|
|
if (!versions_->GetColumnFamilySet()->Exists(column_family.id)) {
|
2014-01-06 22:31:06 +01:00
|
|
|
return Status::NotFound("Column family not found");
|
|
|
|
}
|
2014-01-22 20:44:53 +01:00
|
|
|
VersionEdit edit;
|
|
|
|
edit.DropColumnFamily();
|
|
|
|
edit.SetColumnFamily(column_family.id);
|
2014-01-27 22:55:47 +01:00
|
|
|
Status s = versions_->LogAndApply(default_cfd_, &edit, &mutex_);
|
2014-01-06 22:31:06 +01:00
|
|
|
if (s.ok()) {
|
|
|
|
// remove from internal data structures
|
2014-01-11 00:12:34 +01:00
|
|
|
versions_->DropColumnFamily(&edit);
|
2014-01-06 22:31:06 +01:00
|
|
|
}
|
2014-01-29 20:50:49 +01:00
|
|
|
DeletionState deletion_state;
|
|
|
|
FindObsoleteFiles(deletion_state, false, true);
|
|
|
|
mutex_.Unlock();
|
|
|
|
PurgeObsoleteFiles(deletion_state);
|
2014-01-28 20:05:04 +01:00
|
|
|
Log(options_.info_log, "Dropped column family with id %u\n",
|
|
|
|
column_family.id);
|
2014-01-06 22:31:06 +01:00
|
|
|
return s;
|
2014-01-02 18:08:12 +01:00
|
|
|
}
|
|
|
|
|
2013-07-26 21:57:01 +02:00
|
|
|
bool DBImpl::KeyMayExist(const ReadOptions& options,
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 20:14:09 +01:00
|
|
|
const ColumnFamilyHandle& column_family,
|
|
|
|
const Slice& key, std::string* value,
|
2013-07-26 21:57:01 +02:00
|
|
|
bool* value_found) {
|
|
|
|
if (value_found != nullptr) {
|
2013-11-17 07:59:22 +01:00
|
|
|
// falsify later if key-may-exist but can't fetch value
|
|
|
|
*value_found = true;
|
2013-07-26 21:57:01 +02:00
|
|
|
}
|
2013-08-25 07:48:51 +02:00
|
|
|
ReadOptions roptions = options;
|
|
|
|
roptions.read_tier = kBlockCacheTier; // read from block cache only
|
2014-01-28 20:05:04 +01:00
|
|
|
auto s = GetImpl(roptions, column_family, key, value, value_found);
|
2013-11-17 07:59:22 +01:00
|
|
|
|
|
|
|
// If options.block_cache != nullptr and the index block of the table didn't
|
|
|
|
// not present in block_cache, the return value will be Status::Incomplete.
|
|
|
|
// In this case, key may still exist in the table.
|
|
|
|
return s.ok() || s.IsIncomplete();
|
2013-07-06 03:49:18 +02:00
|
|
|
}
|
|
|
|
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 20:14:09 +01:00
|
|
|
Iterator* DBImpl::NewIterator(const ReadOptions& options,
|
|
|
|
const ColumnFamilyHandle& column_family) {
|
2014-01-17 06:56:26 +01:00
|
|
|
Iterator* iter;
|
|
|
|
|
|
|
|
if (options.tailing) {
|
|
|
|
iter = new TailingIterator(this, options, user_comparator());
|
|
|
|
} else {
|
|
|
|
SequenceNumber latest_snapshot;
|
|
|
|
iter = NewInternalIterator(options, &latest_snapshot);
|
|
|
|
|
|
|
|
iter = NewDBIterator(
|
|
|
|
&dbname_, env_, options_, user_comparator(), iter,
|
|
|
|
(options.snapshot != nullptr
|
|
|
|
? reinterpret_cast<const SnapshotImpl*>(options.snapshot)->number_
|
|
|
|
: latest_snapshot));
|
|
|
|
}
|
|
|
|
|
2013-08-13 23:04:56 +02:00
|
|
|
if (options.prefix) {
|
|
|
|
// use extra wrapper to exclude any keys from the results which
|
|
|
|
// don't begin with the prefix
|
|
|
|
iter = new PrefixFilterIterator(iter, *options.prefix,
|
|
|
|
options_.prefix_extractor);
|
|
|
|
}
|
|
|
|
return iter;
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 20:14:09 +01:00
|
|
|
Status DBImpl::NewIterators(
|
|
|
|
const ReadOptions& options,
|
|
|
|
const std::vector<ColumnFamilyHandle>& column_family,
|
|
|
|
std::vector<Iterator*>* iterators) {
|
|
|
|
// TODO
|
|
|
|
return Status::NotSupported("Not yet!");
|
|
|
|
}
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
const Snapshot* DBImpl::GetSnapshot() {
|
|
|
|
MutexLock l(&mutex_);
|
2011-04-12 21:38:58 +02:00
|
|
|
return snapshots_.New(versions_->LastSequence());
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
void DBImpl::ReleaseSnapshot(const Snapshot* s) {
|
|
|
|
MutexLock l(&mutex_);
|
2011-05-21 04:17:43 +02:00
|
|
|
snapshots_.Delete(reinterpret_cast<const SnapshotImpl*>(s));
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
// Convenience methods
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 20:14:09 +01:00
|
|
|
Status DBImpl::Put(const WriteOptions& o,
|
|
|
|
const ColumnFamilyHandle& column_family, const Slice& key,
|
|
|
|
const Slice& val) {
|
|
|
|
return DB::Put(o, column_family, key, val);
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 20:14:09 +01:00
|
|
|
Status DBImpl::Merge(const WriteOptions& o,
|
|
|
|
const ColumnFamilyHandle& column_family, const Slice& key,
|
2013-03-21 23:59:47 +01:00
|
|
|
const Slice& val) {
|
|
|
|
if (!options_.merge_operator) {
|
|
|
|
return Status::NotSupported("Provide a merge_operator when opening DB");
|
|
|
|
} else {
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 20:14:09 +01:00
|
|
|
return DB::Merge(o, column_family, key, val);
|
2013-03-21 23:59:47 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 20:14:09 +01:00
|
|
|
Status DBImpl::Delete(const WriteOptions& options,
|
|
|
|
const ColumnFamilyHandle& column_family,
|
|
|
|
const Slice& key) {
|
|
|
|
return DB::Delete(options, column_family, key);
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2012-03-09 01:23:21 +01:00
|
|
|
Status DBImpl::Write(const WriteOptions& options, WriteBatch* my_batch) {
|
|
|
|
Writer w(&mutex_);
|
|
|
|
w.batch = my_batch;
|
|
|
|
w.sync = options.sync;
|
2012-07-05 22:39:28 +02:00
|
|
|
w.disableWAL = options.disableWAL;
|
2012-03-09 01:23:21 +01:00
|
|
|
w.done = false;
|
2011-09-01 21:08:02 +02:00
|
|
|
|
2014-01-07 23:33:15 +01:00
|
|
|
StopWatch sw(env_, options_.statistics.get(), DB_WRITE, false);
|
2013-12-20 18:57:58 +01:00
|
|
|
mutex_.Lock();
|
2013-07-24 19:01:13 +02:00
|
|
|
writers_.push_back(&w);
|
|
|
|
while (!w.done && &w != writers_.front()) {
|
|
|
|
w.cv.Wait();
|
|
|
|
}
|
Add monitoring for universal compaction and add counters for compaction IO
Summary:
Adds these counters
{ WAL_FILE_SYNCED, "rocksdb.wal.synced" }
number of writes that request a WAL sync
{ WAL_FILE_BYTES, "rocksdb.wal.bytes" },
number of bytes written to the WAL
{ WRITE_DONE_BY_SELF, "rocksdb.write.self" },
number of writes processed by the calling thread
{ WRITE_DONE_BY_OTHER, "rocksdb.write.other" },
number of writes not processed by the calling thread. Instead these were
processed by the current holder of the write lock
{ WRITE_WITH_WAL, "rocksdb.write.wal" },
number of writes that request WAL logging
{ COMPACT_READ_BYTES, "rocksdb.compact.read.bytes" },
number of bytes read during compaction
{ COMPACT_WRITE_BYTES, "rocksdb.compact.write.bytes" },
number of bytes written during compaction
Per-interval stats output was updated with WAL stats and correct stats for universal compaction
including a correct value for write-amplification. It now looks like:
Compactions
Level Files Size(MB) Score Time(sec) Read(MB) Write(MB) Rn(MB) Rnp1(MB) Wnew(MB) RW-Amplify Read(MB/s) Write(MB/s) Rn Rnp1 Wnp1 NewW Count Ln-stall Stall-cnt
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
0 7 464 46.4 281 3411 3875 3411 0 3875 2.1 12.1 13.8 621 0 240 240 628 0.0 0
Uptime(secs): 310.8 total, 2.0 interval
Writes cumulative: 9999999 total, 9999999 batches, 1.0 per batch, 1.22 ingest GB
WAL cumulative: 9999999 WAL writes, 9999999 WAL syncs, 1.00 writes per sync, 1.22 GB written
Compaction IO cumulative (GB): 1.22 new, 3.33 read, 3.78 write, 7.12 read+write
Compaction IO cumulative (MB/sec): 4.0 new, 11.0 read, 12.5 write, 23.4 read+write
Amplification cumulative: 4.1 write, 6.8 compaction
Writes interval: 100000 total, 100000 batches, 1.0 per batch, 12.5 ingest MB
WAL interval: 100000 WAL writes, 100000 WAL syncs, 1.00 writes per sync, 0.01 MB written
Compaction IO interval (MB): 12.49 new, 14.98 read, 21.50 write, 36.48 read+write
Compaction IO interval (MB/sec): 6.4 new, 7.6 read, 11.0 write, 18.6 read+write
Amplification interval: 101.7 write, 102.9 compaction
Stalls(secs): 142.924 level0_slowdown, 0.000 level0_numfiles, 0.805 memtable_compaction, 0.000 leveln_slowdown
Stalls(count): 132461 level0_slowdown, 0 level0_numfiles, 3 memtable_compaction, 0 leveln_slowdown
Task ID: #3329644, #3301695
Blame Rev:
Test Plan:
Revert Plan:
Database Impact:
Memcache Impact:
Other Notes:
EImportant:
- begin *PUBLIC* platform impact section -
Bugzilla: #
- end platform impact -
Reviewers: dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14583
2013-12-09 22:43:34 +01:00
|
|
|
|
|
|
|
if (!options.disableWAL) {
|
|
|
|
RecordTick(options_.statistics.get(), WRITE_WITH_WAL, 1);
|
|
|
|
}
|
|
|
|
|
2013-07-24 19:01:13 +02:00
|
|
|
if (w.done) {
|
2013-12-20 18:57:58 +01:00
|
|
|
mutex_.Unlock();
|
Add monitoring for universal compaction and add counters for compaction IO
Summary:
Adds these counters
{ WAL_FILE_SYNCED, "rocksdb.wal.synced" }
number of writes that request a WAL sync
{ WAL_FILE_BYTES, "rocksdb.wal.bytes" },
number of bytes written to the WAL
{ WRITE_DONE_BY_SELF, "rocksdb.write.self" },
number of writes processed by the calling thread
{ WRITE_DONE_BY_OTHER, "rocksdb.write.other" },
number of writes not processed by the calling thread. Instead these were
processed by the current holder of the write lock
{ WRITE_WITH_WAL, "rocksdb.write.wal" },
number of writes that request WAL logging
{ COMPACT_READ_BYTES, "rocksdb.compact.read.bytes" },
number of bytes read during compaction
{ COMPACT_WRITE_BYTES, "rocksdb.compact.write.bytes" },
number of bytes written during compaction
Per-interval stats output was updated with WAL stats and correct stats for universal compaction
including a correct value for write-amplification. It now looks like:
Compactions
Level Files Size(MB) Score Time(sec) Read(MB) Write(MB) Rn(MB) Rnp1(MB) Wnew(MB) RW-Amplify Read(MB/s) Write(MB/s) Rn Rnp1 Wnp1 NewW Count Ln-stall Stall-cnt
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
0 7 464 46.4 281 3411 3875 3411 0 3875 2.1 12.1 13.8 621 0 240 240 628 0.0 0
Uptime(secs): 310.8 total, 2.0 interval
Writes cumulative: 9999999 total, 9999999 batches, 1.0 per batch, 1.22 ingest GB
WAL cumulative: 9999999 WAL writes, 9999999 WAL syncs, 1.00 writes per sync, 1.22 GB written
Compaction IO cumulative (GB): 1.22 new, 3.33 read, 3.78 write, 7.12 read+write
Compaction IO cumulative (MB/sec): 4.0 new, 11.0 read, 12.5 write, 23.4 read+write
Amplification cumulative: 4.1 write, 6.8 compaction
Writes interval: 100000 total, 100000 batches, 1.0 per batch, 12.5 ingest MB
WAL interval: 100000 WAL writes, 100000 WAL syncs, 1.00 writes per sync, 0.01 MB written
Compaction IO interval (MB): 12.49 new, 14.98 read, 21.50 write, 36.48 read+write
Compaction IO interval (MB/sec): 6.4 new, 7.6 read, 11.0 write, 18.6 read+write
Amplification interval: 101.7 write, 102.9 compaction
Stalls(secs): 142.924 level0_slowdown, 0.000 level0_numfiles, 0.805 memtable_compaction, 0.000 leveln_slowdown
Stalls(count): 132461 level0_slowdown, 0 level0_numfiles, 3 memtable_compaction, 0 leveln_slowdown
Task ID: #3329644, #3301695
Blame Rev:
Test Plan:
Revert Plan:
Database Impact:
Memcache Impact:
Other Notes:
EImportant:
- begin *PUBLIC* platform impact section -
Bugzilla: #
- end platform impact -
Reviewers: dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14583
2013-12-09 22:43:34 +01:00
|
|
|
RecordTick(options_.statistics.get(), WRITE_DONE_BY_OTHER, 1);
|
2013-07-24 19:01:13 +02:00
|
|
|
return w.status;
|
Add monitoring for universal compaction and add counters for compaction IO
Summary:
Adds these counters
{ WAL_FILE_SYNCED, "rocksdb.wal.synced" }
number of writes that request a WAL sync
{ WAL_FILE_BYTES, "rocksdb.wal.bytes" },
number of bytes written to the WAL
{ WRITE_DONE_BY_SELF, "rocksdb.write.self" },
number of writes processed by the calling thread
{ WRITE_DONE_BY_OTHER, "rocksdb.write.other" },
number of writes not processed by the calling thread. Instead these were
processed by the current holder of the write lock
{ WRITE_WITH_WAL, "rocksdb.write.wal" },
number of writes that request WAL logging
{ COMPACT_READ_BYTES, "rocksdb.compact.read.bytes" },
number of bytes read during compaction
{ COMPACT_WRITE_BYTES, "rocksdb.compact.write.bytes" },
number of bytes written during compaction
Per-interval stats output was updated with WAL stats and correct stats for universal compaction
including a correct value for write-amplification. It now looks like:
Compactions
Level Files Size(MB) Score Time(sec) Read(MB) Write(MB) Rn(MB) Rnp1(MB) Wnew(MB) RW-Amplify Read(MB/s) Write(MB/s) Rn Rnp1 Wnp1 NewW Count Ln-stall Stall-cnt
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
0 7 464 46.4 281 3411 3875 3411 0 3875 2.1 12.1 13.8 621 0 240 240 628 0.0 0
Uptime(secs): 310.8 total, 2.0 interval
Writes cumulative: 9999999 total, 9999999 batches, 1.0 per batch, 1.22 ingest GB
WAL cumulative: 9999999 WAL writes, 9999999 WAL syncs, 1.00 writes per sync, 1.22 GB written
Compaction IO cumulative (GB): 1.22 new, 3.33 read, 3.78 write, 7.12 read+write
Compaction IO cumulative (MB/sec): 4.0 new, 11.0 read, 12.5 write, 23.4 read+write
Amplification cumulative: 4.1 write, 6.8 compaction
Writes interval: 100000 total, 100000 batches, 1.0 per batch, 12.5 ingest MB
WAL interval: 100000 WAL writes, 100000 WAL syncs, 1.00 writes per sync, 0.01 MB written
Compaction IO interval (MB): 12.49 new, 14.98 read, 21.50 write, 36.48 read+write
Compaction IO interval (MB/sec): 6.4 new, 7.6 read, 11.0 write, 18.6 read+write
Amplification interval: 101.7 write, 102.9 compaction
Stalls(secs): 142.924 level0_slowdown, 0.000 level0_numfiles, 0.805 memtable_compaction, 0.000 leveln_slowdown
Stalls(count): 132461 level0_slowdown, 0 level0_numfiles, 3 memtable_compaction, 0 leveln_slowdown
Task ID: #3329644, #3301695
Blame Rev:
Test Plan:
Revert Plan:
Database Impact:
Memcache Impact:
Other Notes:
EImportant:
- begin *PUBLIC* platform impact section -
Bugzilla: #
- end platform impact -
Reviewers: dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14583
2013-12-09 22:43:34 +01:00
|
|
|
} else {
|
|
|
|
RecordTick(options_.statistics.get(), WRITE_DONE_BY_SELF, 1);
|
2012-03-09 01:23:21 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
// May temporarily unlock and wait.
|
2013-12-20 18:57:58 +01:00
|
|
|
SuperVersion* superversion_to_free = nullptr;
|
|
|
|
Status status = MakeRoomForWrite(my_batch == nullptr, &superversion_to_free);
|
2011-04-21 00:48:11 +02:00
|
|
|
uint64_t last_sequence = versions_->LastSequence();
|
2012-03-09 01:23:21 +01:00
|
|
|
Writer* last_writer = &w;
|
2013-02-15 20:53:17 +01:00
|
|
|
if (status.ok() && my_batch != nullptr) { // nullptr batch is for compactions
|
2014-01-14 23:49:31 +01:00
|
|
|
autovector<WriteBatch*> write_batch_group;
|
|
|
|
BuildBatchGroup(&last_writer, &write_batch_group);
|
2013-08-27 07:23:47 +02:00
|
|
|
|
2012-03-09 01:23:21 +01:00
|
|
|
// Add to log and apply to memtable. We can release the lock
|
|
|
|
// during this phase since &w is currently responsible for logging
|
|
|
|
// and protects against concurrent loggers and concurrent writes
|
2014-01-28 20:05:04 +01:00
|
|
|
// into memtables
|
2011-09-01 21:08:02 +02:00
|
|
|
{
|
2013-07-24 19:01:13 +02:00
|
|
|
mutex_.Unlock();
|
2014-01-14 23:49:31 +01:00
|
|
|
WriteBatch* updates = nullptr;
|
|
|
|
if (write_batch_group.size() == 1) {
|
|
|
|
updates = write_batch_group[0];
|
|
|
|
} else {
|
|
|
|
updates = &tmp_batch_;
|
|
|
|
for (size_t i = 0; i < write_batch_group.size(); ++i) {
|
|
|
|
WriteBatchInternal::Append(updates, write_batch_group[i]);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-08-27 07:23:47 +02:00
|
|
|
const SequenceNumber current_sequence = last_sequence + 1;
|
|
|
|
WriteBatchInternal::SetSequence(updates, current_sequence);
|
|
|
|
int my_batch_count = WriteBatchInternal::Count(updates);
|
|
|
|
last_sequence += my_batch_count;
|
|
|
|
// Record statistics
|
2013-11-22 23:14:05 +01:00
|
|
|
RecordTick(options_.statistics.get(),
|
|
|
|
NUMBER_KEYS_WRITTEN, my_batch_count);
|
|
|
|
RecordTick(options_.statistics.get(),
|
2013-08-27 07:23:47 +02:00
|
|
|
BYTES_WRITTEN,
|
|
|
|
WriteBatchInternal::ByteSize(updates));
|
2012-11-06 20:21:57 +01:00
|
|
|
if (options.disableWAL) {
|
|
|
|
flush_on_destroy_ = true;
|
2013-07-24 19:01:13 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
if (!options.disableWAL) {
|
2013-10-23 08:26:51 +02:00
|
|
|
StopWatchNano timer(env_);
|
|
|
|
StartPerfTimer(&timer);
|
Add monitoring for universal compaction and add counters for compaction IO
Summary:
Adds these counters
{ WAL_FILE_SYNCED, "rocksdb.wal.synced" }
number of writes that request a WAL sync
{ WAL_FILE_BYTES, "rocksdb.wal.bytes" },
number of bytes written to the WAL
{ WRITE_DONE_BY_SELF, "rocksdb.write.self" },
number of writes processed by the calling thread
{ WRITE_DONE_BY_OTHER, "rocksdb.write.other" },
number of writes not processed by the calling thread. Instead these were
processed by the current holder of the write lock
{ WRITE_WITH_WAL, "rocksdb.write.wal" },
number of writes that request WAL logging
{ COMPACT_READ_BYTES, "rocksdb.compact.read.bytes" },
number of bytes read during compaction
{ COMPACT_WRITE_BYTES, "rocksdb.compact.write.bytes" },
number of bytes written during compaction
Per-interval stats output was updated with WAL stats and correct stats for universal compaction
including a correct value for write-amplification. It now looks like:
Compactions
Level Files Size(MB) Score Time(sec) Read(MB) Write(MB) Rn(MB) Rnp1(MB) Wnew(MB) RW-Amplify Read(MB/s) Write(MB/s) Rn Rnp1 Wnp1 NewW Count Ln-stall Stall-cnt
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
0 7 464 46.4 281 3411 3875 3411 0 3875 2.1 12.1 13.8 621 0 240 240 628 0.0 0
Uptime(secs): 310.8 total, 2.0 interval
Writes cumulative: 9999999 total, 9999999 batches, 1.0 per batch, 1.22 ingest GB
WAL cumulative: 9999999 WAL writes, 9999999 WAL syncs, 1.00 writes per sync, 1.22 GB written
Compaction IO cumulative (GB): 1.22 new, 3.33 read, 3.78 write, 7.12 read+write
Compaction IO cumulative (MB/sec): 4.0 new, 11.0 read, 12.5 write, 23.4 read+write
Amplification cumulative: 4.1 write, 6.8 compaction
Writes interval: 100000 total, 100000 batches, 1.0 per batch, 12.5 ingest MB
WAL interval: 100000 WAL writes, 100000 WAL syncs, 1.00 writes per sync, 0.01 MB written
Compaction IO interval (MB): 12.49 new, 14.98 read, 21.50 write, 36.48 read+write
Compaction IO interval (MB/sec): 6.4 new, 7.6 read, 11.0 write, 18.6 read+write
Amplification interval: 101.7 write, 102.9 compaction
Stalls(secs): 142.924 level0_slowdown, 0.000 level0_numfiles, 0.805 memtable_compaction, 0.000 leveln_slowdown
Stalls(count): 132461 level0_slowdown, 0 level0_numfiles, 3 memtable_compaction, 0 leveln_slowdown
Task ID: #3329644, #3301695
Blame Rev:
Test Plan:
Revert Plan:
Database Impact:
Memcache Impact:
Other Notes:
EImportant:
- begin *PUBLIC* platform impact section -
Bugzilla: #
- end platform impact -
Reviewers: dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14583
2013-12-09 22:43:34 +01:00
|
|
|
Slice log_entry = WriteBatchInternal::Contents(updates);
|
|
|
|
status = log_->AddRecord(log_entry);
|
|
|
|
RecordTick(options_.statistics.get(), WAL_FILE_SYNCED, 1);
|
|
|
|
RecordTick(options_.statistics.get(), WAL_FILE_BYTES, log_entry.size());
|
2013-11-22 02:40:39 +01:00
|
|
|
BumpPerfTime(&perf_context.wal_write_time, &timer);
|
2012-07-05 22:39:28 +02:00
|
|
|
if (status.ok() && options.sync) {
|
2012-08-27 21:10:26 +02:00
|
|
|
if (options_.use_fsync) {
|
2013-11-22 23:14:05 +01:00
|
|
|
StopWatch(env_, options_.statistics.get(), WAL_FILE_SYNC_MICROS);
|
2013-01-20 11:07:13 +01:00
|
|
|
status = log_->file()->Fsync();
|
2012-08-27 21:10:26 +02:00
|
|
|
} else {
|
2013-11-22 23:14:05 +01:00
|
|
|
StopWatch(env_, options_.statistics.get(), WAL_FILE_SYNC_MICROS);
|
2013-01-20 11:07:13 +01:00
|
|
|
status = log_->file()->Sync();
|
2012-08-27 21:10:26 +02:00
|
|
|
}
|
2012-07-05 22:39:28 +02:00
|
|
|
}
|
2011-09-01 21:08:02 +02:00
|
|
|
}
|
|
|
|
if (status.ok()) {
|
2014-01-28 20:05:04 +01:00
|
|
|
// TODO(icanadi) this accesses column_family_set_ without any lock.
|
|
|
|
// We'll need to add a spinlock for reading that we also lock when we
|
|
|
|
// write to a column family (only on column family add/drop, which is
|
|
|
|
// a very rare action)
|
|
|
|
status = WriteBatchInternal::InsertInto(
|
|
|
|
updates, column_family_memtables_.get(), &options_, this,
|
|
|
|
options_.filter_deletes);
|
|
|
|
|
2013-06-21 01:58:59 +02:00
|
|
|
if (!status.ok()) {
|
|
|
|
// Panic for in-memory corruptions
|
|
|
|
// Note that existing logic was not sound. Any partial failure writing
|
|
|
|
// into the memtable would result in a state that some write ops might
|
|
|
|
// have succeeded in memtable but Status reports error for all writes.
|
|
|
|
throw std::runtime_error("In memory WriteBatch corruption!");
|
|
|
|
}
|
2014-01-17 21:46:06 +01:00
|
|
|
SetTickerCount(options_.statistics.get(), SEQUENCE_NUMBER,
|
|
|
|
last_sequence);
|
2013-09-27 01:04:12 +02:00
|
|
|
}
|
2014-01-14 23:49:31 +01:00
|
|
|
if (updates == &tmp_batch_) tmp_batch_.Clear();
|
2013-09-27 01:04:12 +02:00
|
|
|
mutex_.Lock();
|
|
|
|
if (status.ok()) {
|
2013-06-21 01:58:59 +02:00
|
|
|
versions_->SetLastSequence(last_sequence);
|
2011-09-01 21:08:02 +02:00
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
}
|
2013-10-28 20:36:02 +01:00
|
|
|
if (options_.paranoid_checks && !status.ok() && bg_error_.ok()) {
|
|
|
|
bg_error_ = status; // stop compaction & fail any further writes
|
|
|
|
}
|
2012-03-09 01:23:21 +01:00
|
|
|
|
2013-07-24 19:01:13 +02:00
|
|
|
while (true) {
|
|
|
|
Writer* ready = writers_.front();
|
|
|
|
writers_.pop_front();
|
|
|
|
if (ready != &w) {
|
|
|
|
ready->status = status;
|
|
|
|
ready->done = true;
|
|
|
|
ready->cv.Signal();
|
2012-03-09 01:23:21 +01:00
|
|
|
}
|
2013-07-24 19:01:13 +02:00
|
|
|
if (ready == last_writer) break;
|
|
|
|
}
|
2012-03-09 01:23:21 +01:00
|
|
|
|
2013-07-24 19:01:13 +02:00
|
|
|
// Notify new head of write queue
|
|
|
|
if (!writers_.empty()) {
|
|
|
|
writers_.front()->cv.Signal();
|
2012-03-09 01:23:21 +01:00
|
|
|
}
|
2013-12-20 18:57:58 +01:00
|
|
|
mutex_.Unlock();
|
|
|
|
delete superversion_to_free;
|
2011-03-18 23:37:00 +01:00
|
|
|
return status;
|
|
|
|
}
|
|
|
|
|
2012-03-09 01:23:21 +01:00
|
|
|
// REQUIRES: Writer list must be non-empty
|
2013-02-15 20:53:17 +01:00
|
|
|
// REQUIRES: First writer must have a non-nullptr batch
|
2014-01-14 23:49:31 +01:00
|
|
|
void DBImpl::BuildBatchGroup(Writer** last_writer,
|
|
|
|
autovector<WriteBatch*>* write_batch_group) {
|
2012-03-09 01:23:21 +01:00
|
|
|
assert(!writers_.empty());
|
|
|
|
Writer* first = writers_.front();
|
2014-01-14 23:49:31 +01:00
|
|
|
assert(first->batch != nullptr);
|
2012-03-09 01:23:21 +01:00
|
|
|
|
|
|
|
size_t size = WriteBatchInternal::ByteSize(first->batch);
|
2014-01-14 23:49:31 +01:00
|
|
|
write_batch_group->push_back(first->batch);
|
2012-03-09 01:23:21 +01:00
|
|
|
|
|
|
|
// Allow the group to grow up to a maximum size, but if the
|
|
|
|
// original write is small, limit the growth so we do not slow
|
|
|
|
// down the small write too much.
|
|
|
|
size_t max_size = 1 << 20;
|
|
|
|
if (size <= (128<<10)) {
|
|
|
|
max_size = size + (128<<10);
|
|
|
|
}
|
|
|
|
|
|
|
|
*last_writer = first;
|
|
|
|
std::deque<Writer*>::iterator iter = writers_.begin();
|
|
|
|
++iter; // Advance past "first"
|
|
|
|
for (; iter != writers_.end(); ++iter) {
|
|
|
|
Writer* w = *iter;
|
|
|
|
if (w->sync && !first->sync) {
|
|
|
|
// Do not include a sync write into a batch handled by a non-sync write.
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2012-07-05 22:39:28 +02:00
|
|
|
if (!w->disableWAL && first->disableWAL) {
|
|
|
|
// Do not include a write that needs WAL into a batch that has
|
|
|
|
// WAL disabled.
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2013-02-15 20:53:17 +01:00
|
|
|
if (w->batch != nullptr) {
|
2012-03-09 01:23:21 +01:00
|
|
|
size += WriteBatchInternal::ByteSize(w->batch);
|
|
|
|
if (size > max_size) {
|
|
|
|
// Do not make batch too big
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2014-01-14 23:49:31 +01:00
|
|
|
write_batch_group->push_back(w->batch);
|
2012-03-09 01:23:21 +01:00
|
|
|
}
|
|
|
|
*last_writer = w;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-08-01 01:20:48 +02:00
|
|
|
// This function computes the amount of time in microseconds by which a write
|
|
|
|
// should be delayed based on the number of level-0 files according to the
|
|
|
|
// following formula:
|
2013-08-06 00:43:49 +02:00
|
|
|
// if n < bottom, return 0;
|
|
|
|
// if n >= top, return 1000;
|
|
|
|
// otherwise, let r = (n - bottom) /
|
|
|
|
// (top - bottom)
|
2013-08-01 01:20:48 +02:00
|
|
|
// and return r^2 * 1000.
|
|
|
|
// The goal of this formula is to gradually increase the rate at which writes
|
|
|
|
// are slowed. We also tried linear delay (r * 1000), but it seemed to do
|
|
|
|
// slightly worse. There is no other particular reason for choosing quadratic.
|
Fix SlowdownAmount
Summary:
This had a few bugs.
1) bottom and top were reversed. top is for the max value but the callers were passing the max
value to bottom. The result is that the max sleep is used when n >= bottom.
2) one of the callers passed values with type double and these values are frequently between
1.0 and 2.0 so rounding will do some bad things
3) sometimes the function returned 0 when there should be a stall
With this change and one other diff (out for review soon) there are slightly fewer stalls on one workload.
With the fix.
Stalls(secs): 160.166 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 58.495 leveln_slowdown
Stalls(count): 910261 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 54526 leveln_slowdown
Without the fix.
Stalls(secs): 172.227 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 56.538 leveln_slowdown
Stalls(count): 160831 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 52845 leveln_slowdown
Task ID: #
Blame Rev:
Test Plan:
run db_bench for --benchmarks=overwrite with IO-bound database
Revert Plan:
Database Impact:
Memcache Impact:
Other Notes:
EImportant:
- begin *PUBLIC* platform impact section -
Bugzilla: #
- end platform impact -
Reviewers: haobo
Reviewed By: haobo
CC: leveldb
Differential Revision: https://reviews.facebook.net/D15243
2014-01-17 03:44:23 +01:00
|
|
|
uint64_t DBImpl::SlowdownAmount(int n, double bottom, double top) {
|
2013-08-01 01:20:48 +02:00
|
|
|
uint64_t delay;
|
2013-08-06 00:43:49 +02:00
|
|
|
if (n >= top) {
|
2013-08-01 01:20:48 +02:00
|
|
|
delay = 1000;
|
|
|
|
}
|
2013-08-06 00:43:49 +02:00
|
|
|
else if (n < bottom) {
|
2013-08-01 01:20:48 +02:00
|
|
|
delay = 0;
|
|
|
|
}
|
|
|
|
else {
|
|
|
|
// If we are here, we know that:
|
2013-08-06 00:43:49 +02:00
|
|
|
// level0_start_slowdown <= n < level0_slowdown
|
2013-08-01 01:20:48 +02:00
|
|
|
// since the previous two conditions are false.
|
Fix SlowdownAmount
Summary:
This had a few bugs.
1) bottom and top were reversed. top is for the max value but the callers were passing the max
value to bottom. The result is that the max sleep is used when n >= bottom.
2) one of the callers passed values with type double and these values are frequently between
1.0 and 2.0 so rounding will do some bad things
3) sometimes the function returned 0 when there should be a stall
With this change and one other diff (out for review soon) there are slightly fewer stalls on one workload.
With the fix.
Stalls(secs): 160.166 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 58.495 leveln_slowdown
Stalls(count): 910261 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 54526 leveln_slowdown
Without the fix.
Stalls(secs): 172.227 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 56.538 leveln_slowdown
Stalls(count): 160831 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 52845 leveln_slowdown
Task ID: #
Blame Rev:
Test Plan:
run db_bench for --benchmarks=overwrite with IO-bound database
Revert Plan:
Database Impact:
Memcache Impact:
Other Notes:
EImportant:
- begin *PUBLIC* platform impact section -
Bugzilla: #
- end platform impact -
Reviewers: haobo
Reviewed By: haobo
CC: leveldb
Differential Revision: https://reviews.facebook.net/D15243
2014-01-17 03:44:23 +01:00
|
|
|
double how_much =
|
|
|
|
(double) (n - bottom) /
|
2013-08-06 00:43:49 +02:00
|
|
|
(top - bottom);
|
Fix SlowdownAmount
Summary:
This had a few bugs.
1) bottom and top were reversed. top is for the max value but the callers were passing the max
value to bottom. The result is that the max sleep is used when n >= bottom.
2) one of the callers passed values with type double and these values are frequently between
1.0 and 2.0 so rounding will do some bad things
3) sometimes the function returned 0 when there should be a stall
With this change and one other diff (out for review soon) there are slightly fewer stalls on one workload.
With the fix.
Stalls(secs): 160.166 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 58.495 leveln_slowdown
Stalls(count): 910261 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 54526 leveln_slowdown
Without the fix.
Stalls(secs): 172.227 level0_slowdown, 0.000 level0_numfiles, 0.000 memtable_compaction, 56.538 leveln_slowdown
Stalls(count): 160831 level0_slowdown, 0 level0_numfiles, 0 memtable_compaction, 52845 leveln_slowdown
Task ID: #
Blame Rev:
Test Plan:
run db_bench for --benchmarks=overwrite with IO-bound database
Revert Plan:
Database Impact:
Memcache Impact:
Other Notes:
EImportant:
- begin *PUBLIC* platform impact section -
Bugzilla: #
- end platform impact -
Reviewers: haobo
Reviewed By: haobo
CC: leveldb
Differential Revision: https://reviews.facebook.net/D15243
2014-01-17 03:44:23 +01:00
|
|
|
delay = std::max(how_much * how_much * 1000, 100.0);
|
2013-08-01 01:20:48 +02:00
|
|
|
}
|
|
|
|
assert(delay <= 1000);
|
|
|
|
return delay;
|
|
|
|
}
|
|
|
|
|
2011-09-01 21:08:02 +02:00
|
|
|
// REQUIRES: mutex_ is held
|
2012-03-09 01:23:21 +01:00
|
|
|
// REQUIRES: this thread is currently at the front of the writer queue
|
2013-12-20 18:57:58 +01:00
|
|
|
Status DBImpl::MakeRoomForWrite(bool force,
|
|
|
|
SuperVersion** superversion_to_free) {
|
2011-04-12 21:38:58 +02:00
|
|
|
mutex_.AssertHeld();
|
2013-07-24 19:01:13 +02:00
|
|
|
assert(!writers_.empty());
|
2011-05-21 04:17:43 +02:00
|
|
|
bool allow_delay = !force;
|
2013-08-06 00:43:49 +02:00
|
|
|
bool allow_hard_rate_limit_delay = !force;
|
|
|
|
bool allow_soft_rate_limit_delay = !force;
|
2013-03-02 21:56:04 +01:00
|
|
|
uint64_t rate_limit_delay_millis = 0;
|
2011-04-12 21:38:58 +02:00
|
|
|
Status s;
|
2012-10-26 22:37:21 +02:00
|
|
|
double score;
|
2013-12-20 18:57:58 +01:00
|
|
|
*superversion_to_free = nullptr;
|
2012-06-23 04:30:03 +02:00
|
|
|
|
2011-04-12 21:38:58 +02:00
|
|
|
while (true) {
|
|
|
|
if (!bg_error_.ok()) {
|
|
|
|
// Yield previous error
|
|
|
|
s = bg_error_;
|
|
|
|
break;
|
2014-01-16 01:18:04 +01:00
|
|
|
} else if (allow_delay && versions_->NeedSlowdownForNumLevel0Files()) {
|
2011-05-21 04:17:43 +02:00
|
|
|
// We are getting close to hitting a hard limit on the number of
|
|
|
|
// L0 files. Rather than delaying a single write by several
|
|
|
|
// seconds when we hit the hard limit, start delaying each
|
2013-08-01 01:20:48 +02:00
|
|
|
// individual write by 0-1ms to reduce latency variance. Also,
|
2011-05-21 04:17:43 +02:00
|
|
|
// this delay hands over some CPU to the compaction thread in
|
|
|
|
// case it is sharing the same core as the writer.
|
2014-01-18 01:38:54 +01:00
|
|
|
uint64_t slowdown =
|
2014-01-29 22:28:50 +01:00
|
|
|
SlowdownAmount(default_cfd_->current()->NumLevelFiles(0),
|
2014-01-18 01:38:54 +01:00
|
|
|
options_.level0_slowdown_writes_trigger,
|
|
|
|
options_.level0_stop_writes_trigger);
|
2011-05-21 04:17:43 +02:00
|
|
|
mutex_.Unlock();
|
2013-07-30 00:46:36 +02:00
|
|
|
uint64_t delayed;
|
2013-07-29 19:34:23 +02:00
|
|
|
{
|
2013-11-22 23:14:05 +01:00
|
|
|
StopWatch sw(env_, options_.statistics.get(), STALL_L0_SLOWDOWN_COUNT);
|
2014-01-18 01:38:54 +01:00
|
|
|
env_->SleepForMicroseconds(slowdown);
|
2013-07-30 00:46:36 +02:00
|
|
|
delayed = sw.ElapsedMicros();
|
2013-07-29 19:34:23 +02:00
|
|
|
}
|
2013-11-22 23:14:05 +01:00
|
|
|
RecordTick(options_.statistics.get(), STALL_L0_SLOWDOWN_MICROS, delayed);
|
2012-10-29 20:04:27 +01:00
|
|
|
stall_level0_slowdown_ += delayed;
|
2013-07-29 19:34:23 +02:00
|
|
|
stall_level0_slowdown_count_++;
|
2011-05-21 04:17:43 +02:00
|
|
|
allow_delay = false; // Do not delay a single write more than once
|
|
|
|
mutex_.Lock();
|
2012-10-19 23:00:53 +02:00
|
|
|
delayed_writes_++;
|
2014-01-29 22:28:50 +01:00
|
|
|
} else if (!force && (default_cfd_->mem()->ApproximateMemoryUsage() <=
|
2014-01-24 23:30:28 +01:00
|
|
|
options_.write_buffer_size)) {
|
2011-04-12 21:38:58 +02:00
|
|
|
// There is room in current memtable
|
2012-10-19 23:00:53 +02:00
|
|
|
if (allow_delay) {
|
|
|
|
DelayLoggingAndReset();
|
|
|
|
}
|
2011-04-12 21:38:58 +02:00
|
|
|
break;
|
2014-01-29 22:28:50 +01:00
|
|
|
} else if (default_cfd_->imm()->size() ==
|
2014-01-24 23:30:28 +01:00
|
|
|
options_.max_write_buffer_number - 1) {
|
2011-04-12 21:38:58 +02:00
|
|
|
// We have filled up the current memtable, but the previous
|
2012-10-19 23:00:53 +02:00
|
|
|
// ones are still being compacted, so we wait.
|
|
|
|
DelayLoggingAndReset();
|
2012-08-23 01:57:51 +02:00
|
|
|
Log(options_.info_log, "wait for memtable compaction...\n");
|
2013-07-30 00:46:36 +02:00
|
|
|
uint64_t stall;
|
2013-07-29 19:34:23 +02:00
|
|
|
{
|
2013-11-22 23:14:05 +01:00
|
|
|
StopWatch sw(env_, options_.statistics.get(),
|
2013-07-29 19:34:23 +02:00
|
|
|
STALL_MEMTABLE_COMPACTION_COUNT);
|
|
|
|
bg_cv_.Wait();
|
2013-07-30 00:46:36 +02:00
|
|
|
stall = sw.ElapsedMicros();
|
2013-07-29 19:34:23 +02:00
|
|
|
}
|
2013-11-22 23:14:05 +01:00
|
|
|
RecordTick(options_.statistics.get(),
|
|
|
|
STALL_MEMTABLE_COMPACTION_MICROS, stall);
|
2013-05-10 23:41:45 +02:00
|
|
|
stall_memtable_compaction_ += stall;
|
2013-07-29 19:34:23 +02:00
|
|
|
stall_memtable_compaction_count_++;
|
2014-01-29 22:28:50 +01:00
|
|
|
} else if (default_cfd_->current()->NumLevelFiles(0) >=
|
2012-11-27 06:16:21 +01:00
|
|
|
options_.level0_stop_writes_trigger) {
|
2011-05-21 04:17:43 +02:00
|
|
|
// There are too many level-0 files.
|
2012-10-19 23:00:53 +02:00
|
|
|
DelayLoggingAndReset();
|
|
|
|
Log(options_.info_log, "wait for fewer level0 files...\n");
|
2013-07-30 00:46:36 +02:00
|
|
|
uint64_t stall;
|
2013-07-29 19:34:23 +02:00
|
|
|
{
|
2013-11-22 23:14:05 +01:00
|
|
|
StopWatch sw(env_, options_.statistics.get(),
|
|
|
|
STALL_L0_NUM_FILES_COUNT);
|
2013-07-29 19:34:23 +02:00
|
|
|
bg_cv_.Wait();
|
2013-07-30 00:46:36 +02:00
|
|
|
stall = sw.ElapsedMicros();
|
2013-07-29 19:34:23 +02:00
|
|
|
}
|
2013-11-22 23:14:05 +01:00
|
|
|
RecordTick(options_.statistics.get(), STALL_L0_NUM_FILES_MICROS, stall);
|
2013-05-10 23:41:45 +02:00
|
|
|
stall_level0_num_files_ += stall;
|
2013-07-29 19:34:23 +02:00
|
|
|
stall_level0_num_files_count_++;
|
2014-01-27 18:59:00 +01:00
|
|
|
} else if (allow_hard_rate_limit_delay && options_.hard_rate_limit > 1.0 &&
|
2014-01-29 22:28:50 +01:00
|
|
|
(score = default_cfd_->current()->MaxCompactionScore()) >
|
2014-01-27 18:59:00 +01:00
|
|
|
options_.hard_rate_limit) {
|
2012-10-26 22:37:21 +02:00
|
|
|
// Delay a write when the compaction score for any level is too large.
|
2014-01-29 22:28:50 +01:00
|
|
|
int max_level = default_cfd_->current()->MaxCompactionScoreLevel();
|
2012-10-26 22:37:21 +02:00
|
|
|
mutex_.Unlock();
|
2013-07-30 00:46:36 +02:00
|
|
|
uint64_t delayed;
|
2013-07-29 19:34:23 +02:00
|
|
|
{
|
2013-11-22 23:14:05 +01:00
|
|
|
StopWatch sw(env_, options_.statistics.get(),
|
|
|
|
HARD_RATE_LIMIT_DELAY_COUNT);
|
2013-07-29 19:34:23 +02:00
|
|
|
env_->SleepForMicroseconds(1000);
|
2013-07-30 00:46:36 +02:00
|
|
|
delayed = sw.ElapsedMicros();
|
2013-07-29 19:34:23 +02:00
|
|
|
}
|
2013-03-02 21:56:04 +01:00
|
|
|
stall_leveln_slowdown_[max_level] += delayed;
|
2013-07-29 19:34:23 +02:00
|
|
|
stall_leveln_slowdown_count_[max_level]++;
|
2013-03-02 21:56:04 +01:00
|
|
|
// Make sure the following value doesn't round to zero.
|
2013-05-10 23:41:45 +02:00
|
|
|
uint64_t rate_limit = std::max((delayed / 1000), (uint64_t) 1);
|
|
|
|
rate_limit_delay_millis += rate_limit;
|
2013-11-22 23:14:05 +01:00
|
|
|
RecordTick(options_.statistics.get(),
|
|
|
|
RATE_LIMIT_DELAY_MILLIS, rate_limit);
|
2013-08-06 00:43:49 +02:00
|
|
|
if (options_.rate_limit_delay_max_milliseconds > 0 &&
|
|
|
|
rate_limit_delay_millis >=
|
|
|
|
(unsigned)options_.rate_limit_delay_max_milliseconds) {
|
|
|
|
allow_hard_rate_limit_delay = false;
|
2013-03-02 21:56:04 +01:00
|
|
|
}
|
2012-10-26 22:37:21 +02:00
|
|
|
mutex_.Lock();
|
2014-01-27 18:59:00 +01:00
|
|
|
} else if (allow_soft_rate_limit_delay && options_.soft_rate_limit > 0.0 &&
|
2014-01-29 22:28:50 +01:00
|
|
|
(score = default_cfd_->current()->MaxCompactionScore()) >
|
2014-01-27 18:59:00 +01:00
|
|
|
options_.soft_rate_limit) {
|
2013-08-06 00:43:49 +02:00
|
|
|
// Delay a write when the compaction score for any level is too large.
|
|
|
|
// TODO: add statistics
|
|
|
|
mutex_.Unlock();
|
2013-08-06 03:45:23 +02:00
|
|
|
{
|
2013-11-22 23:14:05 +01:00
|
|
|
StopWatch sw(env_, options_.statistics.get(),
|
|
|
|
SOFT_RATE_LIMIT_DELAY_COUNT);
|
2013-08-06 03:45:23 +02:00
|
|
|
env_->SleepForMicroseconds(SlowdownAmount(
|
|
|
|
score,
|
|
|
|
options_.soft_rate_limit,
|
|
|
|
options_.hard_rate_limit)
|
|
|
|
);
|
|
|
|
rate_limit_delay_millis += sw.ElapsedMicros();
|
|
|
|
}
|
2013-08-06 00:43:49 +02:00
|
|
|
allow_soft_rate_limit_delay = false;
|
|
|
|
mutex_.Lock();
|
2013-11-22 21:52:33 +01:00
|
|
|
|
2011-04-12 21:38:58 +02:00
|
|
|
} else {
|
2013-11-22 21:52:33 +01:00
|
|
|
unique_ptr<WritableFile> lfile;
|
|
|
|
MemTable* memtmp = nullptr;
|
|
|
|
|
|
|
|
// Attempt to switch to a new memtable and trigger compaction of old.
|
|
|
|
// Do this without holding the dbmutex lock.
|
2011-04-12 21:38:58 +02:00
|
|
|
assert(versions_->PrevLogNumber() == 0);
|
|
|
|
uint64_t new_log_number = versions_->NewFileNumber();
|
2013-12-20 18:57:58 +01:00
|
|
|
SuperVersion* new_superversion = nullptr;
|
2013-11-22 21:52:33 +01:00
|
|
|
mutex_.Unlock();
|
|
|
|
{
|
|
|
|
EnvOptions soptions(storage_options_);
|
|
|
|
soptions.use_mmap_writes = false;
|
|
|
|
DelayLoggingAndReset();
|
2014-01-15 00:27:09 +01:00
|
|
|
s = env_->NewWritableFile(LogFileName(options_.wal_dir, new_log_number),
|
|
|
|
&lfile, soptions);
|
2013-11-22 21:52:33 +01:00
|
|
|
if (s.ok()) {
|
|
|
|
// Our final size should be less than write_buffer_size
|
|
|
|
// (compression, etc) but err on the side of caution.
|
|
|
|
lfile->SetPreallocationBlockSize(1.1 * options_.write_buffer_size);
|
2014-01-15 00:27:09 +01:00
|
|
|
memtmp = new MemTable(internal_comparator_, options_);
|
2013-12-20 18:57:58 +01:00
|
|
|
new_superversion = new SuperVersion(options_.max_write_buffer_number);
|
2013-11-22 21:52:33 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
mutex_.Lock();
|
2011-04-12 21:38:58 +02:00
|
|
|
if (!s.ok()) {
|
2012-08-27 08:45:35 +02:00
|
|
|
// Avoid chewing through file number space in a tight loop.
|
2013-01-20 11:07:13 +01:00
|
|
|
versions_->ReuseFileNumber(new_log_number);
|
2013-11-22 21:52:33 +01:00
|
|
|
assert (!memtmp);
|
2011-04-12 21:38:58 +02:00
|
|
|
break;
|
|
|
|
}
|
2011-06-22 04:36:45 +02:00
|
|
|
logfile_number_ = new_log_number;
|
2013-01-20 11:07:13 +01:00
|
|
|
log_.reset(new log::Writer(std::move(lfile)));
|
2014-01-29 22:28:50 +01:00
|
|
|
default_cfd_->mem()->SetNextLogNumber(logfile_number_);
|
|
|
|
default_cfd_->imm()->Add(default_cfd_->mem());
|
2013-09-05 02:24:35 +02:00
|
|
|
if (force) {
|
2014-01-29 22:28:50 +01:00
|
|
|
default_cfd_->imm()->FlushRequested();
|
2013-09-05 02:24:35 +02:00
|
|
|
}
|
2014-01-29 22:28:50 +01:00
|
|
|
memtmp->Ref();
|
|
|
|
memtmp->SetLogNumber(logfile_number_);
|
|
|
|
default_cfd_->SetMemtable(memtmp);
|
|
|
|
Log(options_.info_log, "New memtable created with log file: #%lu\n",
|
2013-11-13 06:02:03 +01:00
|
|
|
(unsigned long)logfile_number_);
|
2011-04-12 21:38:58 +02:00
|
|
|
force = false; // Do not force another compaction if have room
|
2013-10-15 00:12:15 +02:00
|
|
|
MaybeScheduleFlushOrCompaction();
|
2014-01-24 23:30:28 +01:00
|
|
|
*superversion_to_free =
|
2014-01-29 22:28:50 +01:00
|
|
|
default_cfd_->InstallSuperVersion(new_superversion);
|
2011-04-12 21:38:58 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
[RocksDB] BackupableDB
Summary:
In this diff I present you BackupableDB v1. You can easily use it to backup your DB and it will do incremental snapshots for you.
Let's first describe how you would use BackupableDB. It's inheriting StackableDB interface so you can easily construct it with your DB object -- it will add a method RollTheSnapshot() to the DB object. When you call RollTheSnapshot(), current snapshot of the DB will be stored in the backup dir. To restore, you can just call RestoreDBFromBackup() on a BackupableDB (which is a static method) and it will restore all files from the backup dir. In the next version, it will even support automatic backuping every X minutes.
There are multiple things you can configure:
1. backup_env and db_env can be different, which is awesome because then you can easily backup to HDFS or wherever you feel like.
2. sync - if true, it *guarantees* backup consistency on machine reboot
3. number of snapshots to keep - this will keep last N snapshots around if you want, for some reason, be able to restore from an earlier snapshot. All the backuping is done in incremental fashion - if we already have 00010.sst, we will not copy it again. *IMPORTANT* -- This is based on assumption that 00010.sst never changes - two files named 00010.sst from the same DB will always be exactly the same. Is this true? I always copy manifest, current and log files.
4. You can decide if you want to flush the memtables before you backup, or you're fine with backing up the log files -- either way, you get a complete and consistent view of the database at a time of backup.
5. More things you can find in BackupableDBOptions
Here is the directory structure I use:
backup_dir/CURRENT_SNAPSHOT - just 4 bytes holding the latest snapshot
0, 1, 2, ... - files containing serialized version of each snapshot - containing a list of files
files/*.sst - sst files shared between snapshots - if one snapshot references 00010.sst and another one needs to backup it from the DB, it will just reference the same file
files/ 0/, 1/, 2/, ... - snapshot directories containing private snapshot files - current, manifest and log files
All the files are ref counted and deleted immediatelly when they get out of scope.
Some other stuff in this diff:
1. Added GetEnv() method to the DB. Discussed with @haobo and we agreed that it seems right thing to do.
2. Fixed StackableDB interface. The way it was set up before, I was not able to implement BackupableDB.
Test Plan:
I have a unittest, but please don't look at this yet. I just hacked it up to help me with debugging. I will write a lot of good tests and update the diff.
Also, `make asan_check`
Reviewers: dhruba, haobo, emayanke
Reviewed By: dhruba
CC: leveldb, haobo
Differential Revision: https://reviews.facebook.net/D14295
2013-12-09 23:06:52 +01:00
|
|
|
const std::string& DBImpl::GetName() const {
|
|
|
|
return dbname_;
|
|
|
|
}
|
|
|
|
|
2013-11-25 21:39:23 +01:00
|
|
|
Env* DBImpl::GetEnv() const {
|
|
|
|
return env_;
|
|
|
|
}
|
|
|
|
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 20:14:09 +01:00
|
|
|
const Options& DBImpl::GetOptions(const ColumnFamilyHandle& column_family)
|
|
|
|
const {
|
2013-11-26 00:51:50 +01:00
|
|
|
return options_;
|
|
|
|
}
|
|
|
|
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 20:14:09 +01:00
|
|
|
bool DBImpl::GetProperty(const ColumnFamilyHandle& column_family,
|
|
|
|
const Slice& property, std::string* value) {
|
2011-04-12 21:38:58 +02:00
|
|
|
value->clear();
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
MutexLock l(&mutex_);
|
2014-01-29 22:28:50 +01:00
|
|
|
Version* current = default_cfd_->current();
|
2011-03-18 23:37:00 +01:00
|
|
|
Slice in = property;
|
2013-10-05 07:32:05 +02:00
|
|
|
Slice prefix("rocksdb.");
|
2011-03-18 23:37:00 +01:00
|
|
|
if (!in.starts_with(prefix)) return false;
|
|
|
|
in.remove_prefix(prefix.size());
|
|
|
|
|
|
|
|
if (in.starts_with("num-files-at-level")) {
|
|
|
|
in.remove_prefix(strlen("num-files-at-level"));
|
|
|
|
uint64_t level;
|
|
|
|
bool ok = ConsumeDecimalNumber(&in, &level) && in.empty();
|
2012-08-29 21:29:43 +02:00
|
|
|
if (!ok || (int)level >= NumberLevels()) {
|
2011-03-18 23:37:00 +01:00
|
|
|
return false;
|
|
|
|
} else {
|
2011-04-12 21:38:58 +02:00
|
|
|
char buf[100];
|
2011-04-21 00:48:11 +02:00
|
|
|
snprintf(buf, sizeof(buf), "%d",
|
2014-01-16 01:15:43 +01:00
|
|
|
current->NumLevelFiles(static_cast<int>(level)));
|
2011-04-12 21:38:58 +02:00
|
|
|
*value = buf;
|
2011-03-18 23:37:00 +01:00
|
|
|
return true;
|
|
|
|
}
|
2013-03-19 14:36:03 +01:00
|
|
|
} else if (in == "levelstats") {
|
|
|
|
char buf[1000];
|
|
|
|
snprintf(buf, sizeof(buf),
|
|
|
|
"Level Files Size(MB)\n"
|
|
|
|
"--------------------\n");
|
|
|
|
value->append(buf);
|
|
|
|
|
|
|
|
for (int level = 0; level < NumberLevels(); level++) {
|
|
|
|
snprintf(buf, sizeof(buf),
|
|
|
|
"%3d %8d %8.0f\n",
|
|
|
|
level,
|
2014-01-16 01:15:43 +01:00
|
|
|
current->NumLevelFiles(level),
|
2014-01-16 01:18:04 +01:00
|
|
|
current->NumLevelBytes(level) / 1048576.0);
|
2013-03-19 14:36:03 +01:00
|
|
|
value->append(buf);
|
|
|
|
}
|
|
|
|
return true;
|
|
|
|
|
2011-04-12 21:38:58 +02:00
|
|
|
} else if (in == "stats") {
|
2012-10-23 19:34:09 +02:00
|
|
|
char buf[1000];
|
Add monitoring for universal compaction and add counters for compaction IO
Summary:
Adds these counters
{ WAL_FILE_SYNCED, "rocksdb.wal.synced" }
number of writes that request a WAL sync
{ WAL_FILE_BYTES, "rocksdb.wal.bytes" },
number of bytes written to the WAL
{ WRITE_DONE_BY_SELF, "rocksdb.write.self" },
number of writes processed by the calling thread
{ WRITE_DONE_BY_OTHER, "rocksdb.write.other" },
number of writes not processed by the calling thread. Instead these were
processed by the current holder of the write lock
{ WRITE_WITH_WAL, "rocksdb.write.wal" },
number of writes that request WAL logging
{ COMPACT_READ_BYTES, "rocksdb.compact.read.bytes" },
number of bytes read during compaction
{ COMPACT_WRITE_BYTES, "rocksdb.compact.write.bytes" },
number of bytes written during compaction
Per-interval stats output was updated with WAL stats and correct stats for universal compaction
including a correct value for write-amplification. It now looks like:
Compactions
Level Files Size(MB) Score Time(sec) Read(MB) Write(MB) Rn(MB) Rnp1(MB) Wnew(MB) RW-Amplify Read(MB/s) Write(MB/s) Rn Rnp1 Wnp1 NewW Count Ln-stall Stall-cnt
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
0 7 464 46.4 281 3411 3875 3411 0 3875 2.1 12.1 13.8 621 0 240 240 628 0.0 0
Uptime(secs): 310.8 total, 2.0 interval
Writes cumulative: 9999999 total, 9999999 batches, 1.0 per batch, 1.22 ingest GB
WAL cumulative: 9999999 WAL writes, 9999999 WAL syncs, 1.00 writes per sync, 1.22 GB written
Compaction IO cumulative (GB): 1.22 new, 3.33 read, 3.78 write, 7.12 read+write
Compaction IO cumulative (MB/sec): 4.0 new, 11.0 read, 12.5 write, 23.4 read+write
Amplification cumulative: 4.1 write, 6.8 compaction
Writes interval: 100000 total, 100000 batches, 1.0 per batch, 12.5 ingest MB
WAL interval: 100000 WAL writes, 100000 WAL syncs, 1.00 writes per sync, 0.01 MB written
Compaction IO interval (MB): 12.49 new, 14.98 read, 21.50 write, 36.48 read+write
Compaction IO interval (MB/sec): 6.4 new, 7.6 read, 11.0 write, 18.6 read+write
Amplification interval: 101.7 write, 102.9 compaction
Stalls(secs): 142.924 level0_slowdown, 0.000 level0_numfiles, 0.805 memtable_compaction, 0.000 leveln_slowdown
Stalls(count): 132461 level0_slowdown, 0 level0_numfiles, 3 memtable_compaction, 0 leveln_slowdown
Task ID: #3329644, #3301695
Blame Rev:
Test Plan:
Revert Plan:
Database Impact:
Memcache Impact:
Other Notes:
EImportant:
- begin *PUBLIC* platform impact section -
Bugzilla: #
- end platform impact -
Reviewers: dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14583
2013-12-09 22:43:34 +01:00
|
|
|
|
|
|
|
uint64_t wal_bytes = 0;
|
|
|
|
uint64_t wal_synced = 0;
|
|
|
|
uint64_t user_bytes_written = 0;
|
|
|
|
uint64_t write_other = 0;
|
|
|
|
uint64_t write_self = 0;
|
|
|
|
uint64_t write_with_wal = 0;
|
Improve output for GetProperty('leveldb.stats')
Summary:
Display separate values for read, write & total compaction IO.
Display compaction amplification and write amplification.
Add similar values for the period since the last call to GetProperty. Results since the server started
are reported as "cumulative" stats. Results since the last call to GetProperty are reported as
"interval" stats.
Level Files Size(MB) Time(sec) Read(MB) Write(MB) Rn(MB) Rnp1(MB) Wnew(MB) Amplify Read(MB/s) Write(MB/s) Rn Rnp1 Wnp1 NewW Count Ln-stall
----------------------------------------------------------------------------------------------------------------------------------------------------------------------
0 7 13 21 0 211 0 0 211 0.0 0.0 10.1 0 0 0 0 113 0.0
1 79 157 88 993 989 198 795 194 9.0 11.3 11.2 106 405 502 97 14 0.0
2 19 36 5 63 63 37 27 36 2.4 12.3 12.2 19 14 32 18 12 0.0
>>>>>>>>>>>>>>>>>>>>>>>>> text below has been is new and/or reformatted
Uptime(secs): 122.2 total, 0.9 interval
Compaction IO cumulative (GB): 0.21 new, 1.03 read, 1.23 write, 2.26 read+write
Compaction IO cumulative (MB/sec): 1.7 new, 8.6 read, 10.3 write, 19.0 read+write
Amplification cumulative: 6.0 write, 11.0 compaction
Compaction IO interval (MB): 5.59 new, 0.00 read, 5.59 write, 5.59 read+write
Compaction IO interval (MB/sec): 6.5 new, 0.0 read, 6.5 write, 6.5 read+write
Amplification interval: 1.0 write, 1.0 compaction
>>>>>>>>>>>>>>>>>>>>>>>> text above is new and/or reformatted
Stalls(secs): 90.574 level0_slowdown, 0.000 level0_numfiles, 10.165 memtable_compaction, 0.000 leveln_slowdown
Task ID: #
Blame Rev:
Test Plan:
make check, run db_bench
Revert Plan:
Database Impact:
Memcache Impact:
Other Notes:
EImportant:
- begin *PUBLIC* platform impact section -
Bugzilla: #
- end platform impact -
Reviewers: haobo
Reviewed By: haobo
CC: leveldb
Differential Revision: https://reviews.facebook.net/D11049
2013-06-03 17:16:16 +02:00
|
|
|
uint64_t total_bytes_written = 0;
|
|
|
|
uint64_t total_bytes_read = 0;
|
2012-10-23 19:34:09 +02:00
|
|
|
uint64_t micros_up = env_->NowMicros() - started_at_;
|
Improve output for GetProperty('leveldb.stats')
Summary:
Display separate values for read, write & total compaction IO.
Display compaction amplification and write amplification.
Add similar values for the period since the last call to GetProperty. Results since the server started
are reported as "cumulative" stats. Results since the last call to GetProperty are reported as
"interval" stats.
Level Files Size(MB) Time(sec) Read(MB) Write(MB) Rn(MB) Rnp1(MB) Wnew(MB) Amplify Read(MB/s) Write(MB/s) Rn Rnp1 Wnp1 NewW Count Ln-stall
----------------------------------------------------------------------------------------------------------------------------------------------------------------------
0 7 13 21 0 211 0 0 211 0.0 0.0 10.1 0 0 0 0 113 0.0
1 79 157 88 993 989 198 795 194 9.0 11.3 11.2 106 405 502 97 14 0.0
2 19 36 5 63 63 37 27 36 2.4 12.3 12.2 19 14 32 18 12 0.0
>>>>>>>>>>>>>>>>>>>>>>>>> text below has been is new and/or reformatted
Uptime(secs): 122.2 total, 0.9 interval
Compaction IO cumulative (GB): 0.21 new, 1.03 read, 1.23 write, 2.26 read+write
Compaction IO cumulative (MB/sec): 1.7 new, 8.6 read, 10.3 write, 19.0 read+write
Amplification cumulative: 6.0 write, 11.0 compaction
Compaction IO interval (MB): 5.59 new, 0.00 read, 5.59 write, 5.59 read+write
Compaction IO interval (MB/sec): 6.5 new, 0.0 read, 6.5 write, 6.5 read+write
Amplification interval: 1.0 write, 1.0 compaction
>>>>>>>>>>>>>>>>>>>>>>>> text above is new and/or reformatted
Stalls(secs): 90.574 level0_slowdown, 0.000 level0_numfiles, 10.165 memtable_compaction, 0.000 leveln_slowdown
Task ID: #
Blame Rev:
Test Plan:
make check, run db_bench
Revert Plan:
Database Impact:
Memcache Impact:
Other Notes:
EImportant:
- begin *PUBLIC* platform impact section -
Bugzilla: #
- end platform impact -
Reviewers: haobo
Reviewed By: haobo
CC: leveldb
Differential Revision: https://reviews.facebook.net/D11049
2013-06-03 17:16:16 +02:00
|
|
|
// Add "+1" to make sure seconds_up is > 0 and avoid NaN later
|
|
|
|
double seconds_up = (micros_up + 1) / 1000000.0;
|
2013-03-02 21:56:04 +01:00
|
|
|
uint64_t total_slowdown = 0;
|
2013-07-29 19:34:23 +02:00
|
|
|
uint64_t total_slowdown_count = 0;
|
Improve output for GetProperty('leveldb.stats')
Summary:
Display separate values for read, write & total compaction IO.
Display compaction amplification and write amplification.
Add similar values for the period since the last call to GetProperty. Results since the server started
are reported as "cumulative" stats. Results since the last call to GetProperty are reported as
"interval" stats.
Level Files Size(MB) Time(sec) Read(MB) Write(MB) Rn(MB) Rnp1(MB) Wnew(MB) Amplify Read(MB/s) Write(MB/s) Rn Rnp1 Wnp1 NewW Count Ln-stall
----------------------------------------------------------------------------------------------------------------------------------------------------------------------
0 7 13 21 0 211 0 0 211 0.0 0.0 10.1 0 0 0 0 113 0.0
1 79 157 88 993 989 198 795 194 9.0 11.3 11.2 106 405 502 97 14 0.0
2 19 36 5 63 63 37 27 36 2.4 12.3 12.2 19 14 32 18 12 0.0
>>>>>>>>>>>>>>>>>>>>>>>>> text below has been is new and/or reformatted
Uptime(secs): 122.2 total, 0.9 interval
Compaction IO cumulative (GB): 0.21 new, 1.03 read, 1.23 write, 2.26 read+write
Compaction IO cumulative (MB/sec): 1.7 new, 8.6 read, 10.3 write, 19.0 read+write
Amplification cumulative: 6.0 write, 11.0 compaction
Compaction IO interval (MB): 5.59 new, 0.00 read, 5.59 write, 5.59 read+write
Compaction IO interval (MB/sec): 6.5 new, 0.0 read, 6.5 write, 6.5 read+write
Amplification interval: 1.0 write, 1.0 compaction
>>>>>>>>>>>>>>>>>>>>>>>> text above is new and/or reformatted
Stalls(secs): 90.574 level0_slowdown, 0.000 level0_numfiles, 10.165 memtable_compaction, 0.000 leveln_slowdown
Task ID: #
Blame Rev:
Test Plan:
make check, run db_bench
Revert Plan:
Database Impact:
Memcache Impact:
Other Notes:
EImportant:
- begin *PUBLIC* platform impact section -
Bugzilla: #
- end platform impact -
Reviewers: haobo
Reviewed By: haobo
CC: leveldb
Differential Revision: https://reviews.facebook.net/D11049
2013-06-03 17:16:16 +02:00
|
|
|
uint64_t interval_bytes_written = 0;
|
|
|
|
uint64_t interval_bytes_read = 0;
|
|
|
|
uint64_t interval_bytes_new = 0;
|
|
|
|
double interval_seconds_up = 0;
|
2012-10-23 19:34:09 +02:00
|
|
|
|
Add monitoring for universal compaction and add counters for compaction IO
Summary:
Adds these counters
{ WAL_FILE_SYNCED, "rocksdb.wal.synced" }
number of writes that request a WAL sync
{ WAL_FILE_BYTES, "rocksdb.wal.bytes" },
number of bytes written to the WAL
{ WRITE_DONE_BY_SELF, "rocksdb.write.self" },
number of writes processed by the calling thread
{ WRITE_DONE_BY_OTHER, "rocksdb.write.other" },
number of writes not processed by the calling thread. Instead these were
processed by the current holder of the write lock
{ WRITE_WITH_WAL, "rocksdb.write.wal" },
number of writes that request WAL logging
{ COMPACT_READ_BYTES, "rocksdb.compact.read.bytes" },
number of bytes read during compaction
{ COMPACT_WRITE_BYTES, "rocksdb.compact.write.bytes" },
number of bytes written during compaction
Per-interval stats output was updated with WAL stats and correct stats for universal compaction
including a correct value for write-amplification. It now looks like:
Compactions
Level Files Size(MB) Score Time(sec) Read(MB) Write(MB) Rn(MB) Rnp1(MB) Wnew(MB) RW-Amplify Read(MB/s) Write(MB/s) Rn Rnp1 Wnp1 NewW Count Ln-stall Stall-cnt
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
0 7 464 46.4 281 3411 3875 3411 0 3875 2.1 12.1 13.8 621 0 240 240 628 0.0 0
Uptime(secs): 310.8 total, 2.0 interval
Writes cumulative: 9999999 total, 9999999 batches, 1.0 per batch, 1.22 ingest GB
WAL cumulative: 9999999 WAL writes, 9999999 WAL syncs, 1.00 writes per sync, 1.22 GB written
Compaction IO cumulative (GB): 1.22 new, 3.33 read, 3.78 write, 7.12 read+write
Compaction IO cumulative (MB/sec): 4.0 new, 11.0 read, 12.5 write, 23.4 read+write
Amplification cumulative: 4.1 write, 6.8 compaction
Writes interval: 100000 total, 100000 batches, 1.0 per batch, 12.5 ingest MB
WAL interval: 100000 WAL writes, 100000 WAL syncs, 1.00 writes per sync, 0.01 MB written
Compaction IO interval (MB): 12.49 new, 14.98 read, 21.50 write, 36.48 read+write
Compaction IO interval (MB/sec): 6.4 new, 7.6 read, 11.0 write, 18.6 read+write
Amplification interval: 101.7 write, 102.9 compaction
Stalls(secs): 142.924 level0_slowdown, 0.000 level0_numfiles, 0.805 memtable_compaction, 0.000 leveln_slowdown
Stalls(count): 132461 level0_slowdown, 0 level0_numfiles, 3 memtable_compaction, 0 leveln_slowdown
Task ID: #3329644, #3301695
Blame Rev:
Test Plan:
Revert Plan:
Database Impact:
Memcache Impact:
Other Notes:
EImportant:
- begin *PUBLIC* platform impact section -
Bugzilla: #
- end platform impact -
Reviewers: dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14583
2013-12-09 22:43:34 +01:00
|
|
|
Statistics* s = options_.statistics.get();
|
|
|
|
if (s) {
|
|
|
|
wal_bytes = s->getTickerCount(WAL_FILE_BYTES);
|
|
|
|
wal_synced = s->getTickerCount(WAL_FILE_SYNCED);
|
|
|
|
user_bytes_written = s->getTickerCount(BYTES_WRITTEN);
|
|
|
|
write_other = s->getTickerCount(WRITE_DONE_BY_OTHER);
|
|
|
|
write_self = s->getTickerCount(WRITE_DONE_BY_SELF);
|
|
|
|
write_with_wal = s->getTickerCount(WRITE_WITH_WAL);
|
|
|
|
}
|
|
|
|
|
2012-10-23 19:34:09 +02:00
|
|
|
// Pardon the long line but I think it is easier to read this way.
|
2011-04-12 21:38:58 +02:00
|
|
|
snprintf(buf, sizeof(buf),
|
|
|
|
" Compactions\n"
|
2014-01-22 23:37:20 +01:00
|
|
|
"Level Files Size(MB) Score Time(sec) Read(MB) Write(MB) Rn(MB) Rnp1(MB) Wnew(MB) RW-Amplify Read(MB/s) Write(MB/s) Rn Rnp1 Wnp1 NewW Count msComp msStall Ln-stall Stall-cnt\n"
|
|
|
|
"------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------\n"
|
2011-04-12 21:38:58 +02:00
|
|
|
);
|
|
|
|
value->append(buf);
|
2014-01-16 01:18:04 +01:00
|
|
|
for (int level = 0; level < current->NumberLevels(); level++) {
|
2014-01-16 01:15:43 +01:00
|
|
|
int files = current->NumLevelFiles(level);
|
2011-04-12 21:38:58 +02:00
|
|
|
if (stats_[level].micros > 0 || files > 0) {
|
2012-10-23 19:34:09 +02:00
|
|
|
int64_t bytes_read = stats_[level].bytes_readn +
|
|
|
|
stats_[level].bytes_readnp1;
|
|
|
|
int64_t bytes_new = stats_[level].bytes_written -
|
|
|
|
stats_[level].bytes_readnp1;
|
|
|
|
double amplify = (stats_[level].bytes_readn == 0)
|
|
|
|
? 0.0
|
2013-08-23 07:37:13 +02:00
|
|
|
: (stats_[level].bytes_written +
|
|
|
|
stats_[level].bytes_readnp1 +
|
|
|
|
stats_[level].bytes_readn) /
|
2012-10-23 19:34:09 +02:00
|
|
|
(double) stats_[level].bytes_readn;
|
|
|
|
|
Improve output for GetProperty('leveldb.stats')
Summary:
Display separate values for read, write & total compaction IO.
Display compaction amplification and write amplification.
Add similar values for the period since the last call to GetProperty. Results since the server started
are reported as "cumulative" stats. Results since the last call to GetProperty are reported as
"interval" stats.
Level Files Size(MB) Time(sec) Read(MB) Write(MB) Rn(MB) Rnp1(MB) Wnew(MB) Amplify Read(MB/s) Write(MB/s) Rn Rnp1 Wnp1 NewW Count Ln-stall
----------------------------------------------------------------------------------------------------------------------------------------------------------------------
0 7 13 21 0 211 0 0 211 0.0 0.0 10.1 0 0 0 0 113 0.0
1 79 157 88 993 989 198 795 194 9.0 11.3 11.2 106 405 502 97 14 0.0
2 19 36 5 63 63 37 27 36 2.4 12.3 12.2 19 14 32 18 12 0.0
>>>>>>>>>>>>>>>>>>>>>>>>> text below has been is new and/or reformatted
Uptime(secs): 122.2 total, 0.9 interval
Compaction IO cumulative (GB): 0.21 new, 1.03 read, 1.23 write, 2.26 read+write
Compaction IO cumulative (MB/sec): 1.7 new, 8.6 read, 10.3 write, 19.0 read+write
Amplification cumulative: 6.0 write, 11.0 compaction
Compaction IO interval (MB): 5.59 new, 0.00 read, 5.59 write, 5.59 read+write
Compaction IO interval (MB/sec): 6.5 new, 0.0 read, 6.5 write, 6.5 read+write
Amplification interval: 1.0 write, 1.0 compaction
>>>>>>>>>>>>>>>>>>>>>>>> text above is new and/or reformatted
Stalls(secs): 90.574 level0_slowdown, 0.000 level0_numfiles, 10.165 memtable_compaction, 0.000 leveln_slowdown
Task ID: #
Blame Rev:
Test Plan:
make check, run db_bench
Revert Plan:
Database Impact:
Memcache Impact:
Other Notes:
EImportant:
- begin *PUBLIC* platform impact section -
Bugzilla: #
- end platform impact -
Reviewers: haobo
Reviewed By: haobo
CC: leveldb
Differential Revision: https://reviews.facebook.net/D11049
2013-06-03 17:16:16 +02:00
|
|
|
total_bytes_read += bytes_read;
|
|
|
|
total_bytes_written += stats_[level].bytes_written;
|
|
|
|
|
2014-01-22 23:37:20 +01:00
|
|
|
uint64_t stalls = level == 0 ?
|
|
|
|
(stall_level0_slowdown_count_ +
|
|
|
|
stall_level0_num_files_count_ +
|
|
|
|
stall_memtable_compaction_count_) :
|
|
|
|
stall_leveln_slowdown_count_[level];
|
|
|
|
|
|
|
|
double stall_us = level == 0 ?
|
|
|
|
(stall_level0_slowdown_ +
|
|
|
|
stall_level0_num_files_ +
|
|
|
|
stall_memtable_compaction_) :
|
|
|
|
stall_leveln_slowdown_[level];
|
|
|
|
|
2011-04-12 21:38:58 +02:00
|
|
|
snprintf(
|
|
|
|
buf, sizeof(buf),
|
2014-01-22 23:37:20 +01:00
|
|
|
"%3d %8d %8.0f %5.1f %9.0f %9.0f %9.0f %9.0f %9.0f %9.0f %10.1f %9.1f %11.1f %8d %8d %8d %8d %8d %8d %9.1f %9.1f %9lu\n",
|
2011-04-12 21:38:58 +02:00
|
|
|
level,
|
|
|
|
files,
|
2014-01-16 01:18:04 +01:00
|
|
|
current->NumLevelBytes(level) / 1048576.0,
|
|
|
|
current->NumLevelBytes(level) /
|
2013-08-23 07:37:13 +02:00
|
|
|
versions_->MaxBytesForLevel(level),
|
2011-04-12 21:38:58 +02:00
|
|
|
stats_[level].micros / 1e6,
|
2012-10-23 19:34:09 +02:00
|
|
|
bytes_read / 1048576.0,
|
|
|
|
stats_[level].bytes_written / 1048576.0,
|
|
|
|
stats_[level].bytes_readn / 1048576.0,
|
|
|
|
stats_[level].bytes_readnp1 / 1048576.0,
|
|
|
|
bytes_new / 1048576.0,
|
|
|
|
amplify,
|
2013-03-19 14:36:03 +01:00
|
|
|
// +1 to avoid division by 0
|
|
|
|
(bytes_read / 1048576.0) / ((stats_[level].micros+1) / 1000000.0),
|
2012-10-26 22:00:42 +02:00
|
|
|
(stats_[level].bytes_written / 1048576.0) /
|
2013-03-19 14:36:03 +01:00
|
|
|
((stats_[level].micros+1) / 1000000.0),
|
2012-10-23 19:34:09 +02:00
|
|
|
stats_[level].files_in_leveln,
|
|
|
|
stats_[level].files_in_levelnp1,
|
|
|
|
stats_[level].files_out_levelnp1,
|
|
|
|
stats_[level].files_out_levelnp1 - stats_[level].files_in_levelnp1,
|
2013-03-02 21:56:04 +01:00
|
|
|
stats_[level].count,
|
2014-01-22 23:37:20 +01:00
|
|
|
(int) ((double) stats_[level].micros /
|
|
|
|
1000.0 /
|
|
|
|
(stats_[level].count + 1)),
|
|
|
|
(double) stall_us / 1000.0 / (stalls + 1),
|
|
|
|
stall_us / 1000000.0,
|
|
|
|
(unsigned long) stalls);
|
|
|
|
|
2013-03-02 21:56:04 +01:00
|
|
|
total_slowdown += stall_leveln_slowdown_[level];
|
2013-07-29 19:34:23 +02:00
|
|
|
total_slowdown_count += stall_leveln_slowdown_count_[level];
|
2011-04-12 21:38:58 +02:00
|
|
|
value->append(buf);
|
|
|
|
}
|
|
|
|
}
|
2012-10-23 19:34:09 +02:00
|
|
|
|
Add monitoring for universal compaction and add counters for compaction IO
Summary:
Adds these counters
{ WAL_FILE_SYNCED, "rocksdb.wal.synced" }
number of writes that request a WAL sync
{ WAL_FILE_BYTES, "rocksdb.wal.bytes" },
number of bytes written to the WAL
{ WRITE_DONE_BY_SELF, "rocksdb.write.self" },
number of writes processed by the calling thread
{ WRITE_DONE_BY_OTHER, "rocksdb.write.other" },
number of writes not processed by the calling thread. Instead these were
processed by the current holder of the write lock
{ WRITE_WITH_WAL, "rocksdb.write.wal" },
number of writes that request WAL logging
{ COMPACT_READ_BYTES, "rocksdb.compact.read.bytes" },
number of bytes read during compaction
{ COMPACT_WRITE_BYTES, "rocksdb.compact.write.bytes" },
number of bytes written during compaction
Per-interval stats output was updated with WAL stats and correct stats for universal compaction
including a correct value for write-amplification. It now looks like:
Compactions
Level Files Size(MB) Score Time(sec) Read(MB) Write(MB) Rn(MB) Rnp1(MB) Wnew(MB) RW-Amplify Read(MB/s) Write(MB/s) Rn Rnp1 Wnp1 NewW Count Ln-stall Stall-cnt
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
0 7 464 46.4 281 3411 3875 3411 0 3875 2.1 12.1 13.8 621 0 240 240 628 0.0 0
Uptime(secs): 310.8 total, 2.0 interval
Writes cumulative: 9999999 total, 9999999 batches, 1.0 per batch, 1.22 ingest GB
WAL cumulative: 9999999 WAL writes, 9999999 WAL syncs, 1.00 writes per sync, 1.22 GB written
Compaction IO cumulative (GB): 1.22 new, 3.33 read, 3.78 write, 7.12 read+write
Compaction IO cumulative (MB/sec): 4.0 new, 11.0 read, 12.5 write, 23.4 read+write
Amplification cumulative: 4.1 write, 6.8 compaction
Writes interval: 100000 total, 100000 batches, 1.0 per batch, 12.5 ingest MB
WAL interval: 100000 WAL writes, 100000 WAL syncs, 1.00 writes per sync, 0.01 MB written
Compaction IO interval (MB): 12.49 new, 14.98 read, 21.50 write, 36.48 read+write
Compaction IO interval (MB/sec): 6.4 new, 7.6 read, 11.0 write, 18.6 read+write
Amplification interval: 101.7 write, 102.9 compaction
Stalls(secs): 142.924 level0_slowdown, 0.000 level0_numfiles, 0.805 memtable_compaction, 0.000 leveln_slowdown
Stalls(count): 132461 level0_slowdown, 0 level0_numfiles, 3 memtable_compaction, 0 leveln_slowdown
Task ID: #3329644, #3301695
Blame Rev:
Test Plan:
Revert Plan:
Database Impact:
Memcache Impact:
Other Notes:
EImportant:
- begin *PUBLIC* platform impact section -
Bugzilla: #
- end platform impact -
Reviewers: dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14583
2013-12-09 22:43:34 +01:00
|
|
|
interval_bytes_new = user_bytes_written - last_stats_.ingest_bytes_;
|
|
|
|
interval_bytes_read = total_bytes_read - last_stats_.compaction_bytes_read_;
|
|
|
|
interval_bytes_written =
|
|
|
|
total_bytes_written - last_stats_.compaction_bytes_written_;
|
Improve output for GetProperty('leveldb.stats')
Summary:
Display separate values for read, write & total compaction IO.
Display compaction amplification and write amplification.
Add similar values for the period since the last call to GetProperty. Results since the server started
are reported as "cumulative" stats. Results since the last call to GetProperty are reported as
"interval" stats.
Level Files Size(MB) Time(sec) Read(MB) Write(MB) Rn(MB) Rnp1(MB) Wnew(MB) Amplify Read(MB/s) Write(MB/s) Rn Rnp1 Wnp1 NewW Count Ln-stall
----------------------------------------------------------------------------------------------------------------------------------------------------------------------
0 7 13 21 0 211 0 0 211 0.0 0.0 10.1 0 0 0 0 113 0.0
1 79 157 88 993 989 198 795 194 9.0 11.3 11.2 106 405 502 97 14 0.0
2 19 36 5 63 63 37 27 36 2.4 12.3 12.2 19 14 32 18 12 0.0
>>>>>>>>>>>>>>>>>>>>>>>>> text below has been is new and/or reformatted
Uptime(secs): 122.2 total, 0.9 interval
Compaction IO cumulative (GB): 0.21 new, 1.03 read, 1.23 write, 2.26 read+write
Compaction IO cumulative (MB/sec): 1.7 new, 8.6 read, 10.3 write, 19.0 read+write
Amplification cumulative: 6.0 write, 11.0 compaction
Compaction IO interval (MB): 5.59 new, 0.00 read, 5.59 write, 5.59 read+write
Compaction IO interval (MB/sec): 6.5 new, 0.0 read, 6.5 write, 6.5 read+write
Amplification interval: 1.0 write, 1.0 compaction
>>>>>>>>>>>>>>>>>>>>>>>> text above is new and/or reformatted
Stalls(secs): 90.574 level0_slowdown, 0.000 level0_numfiles, 10.165 memtable_compaction, 0.000 leveln_slowdown
Task ID: #
Blame Rev:
Test Plan:
make check, run db_bench
Revert Plan:
Database Impact:
Memcache Impact:
Other Notes:
EImportant:
- begin *PUBLIC* platform impact section -
Bugzilla: #
- end platform impact -
Reviewers: haobo
Reviewed By: haobo
CC: leveldb
Differential Revision: https://reviews.facebook.net/D11049
2013-06-03 17:16:16 +02:00
|
|
|
interval_seconds_up = seconds_up - last_stats_.seconds_up_;
|
|
|
|
|
|
|
|
snprintf(buf, sizeof(buf), "Uptime(secs): %.1f total, %.1f interval\n",
|
|
|
|
seconds_up, interval_seconds_up);
|
|
|
|
value->append(buf);
|
|
|
|
|
Add monitoring for universal compaction and add counters for compaction IO
Summary:
Adds these counters
{ WAL_FILE_SYNCED, "rocksdb.wal.synced" }
number of writes that request a WAL sync
{ WAL_FILE_BYTES, "rocksdb.wal.bytes" },
number of bytes written to the WAL
{ WRITE_DONE_BY_SELF, "rocksdb.write.self" },
number of writes processed by the calling thread
{ WRITE_DONE_BY_OTHER, "rocksdb.write.other" },
number of writes not processed by the calling thread. Instead these were
processed by the current holder of the write lock
{ WRITE_WITH_WAL, "rocksdb.write.wal" },
number of writes that request WAL logging
{ COMPACT_READ_BYTES, "rocksdb.compact.read.bytes" },
number of bytes read during compaction
{ COMPACT_WRITE_BYTES, "rocksdb.compact.write.bytes" },
number of bytes written during compaction
Per-interval stats output was updated with WAL stats and correct stats for universal compaction
including a correct value for write-amplification. It now looks like:
Compactions
Level Files Size(MB) Score Time(sec) Read(MB) Write(MB) Rn(MB) Rnp1(MB) Wnew(MB) RW-Amplify Read(MB/s) Write(MB/s) Rn Rnp1 Wnp1 NewW Count Ln-stall Stall-cnt
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
0 7 464 46.4 281 3411 3875 3411 0 3875 2.1 12.1 13.8 621 0 240 240 628 0.0 0
Uptime(secs): 310.8 total, 2.0 interval
Writes cumulative: 9999999 total, 9999999 batches, 1.0 per batch, 1.22 ingest GB
WAL cumulative: 9999999 WAL writes, 9999999 WAL syncs, 1.00 writes per sync, 1.22 GB written
Compaction IO cumulative (GB): 1.22 new, 3.33 read, 3.78 write, 7.12 read+write
Compaction IO cumulative (MB/sec): 4.0 new, 11.0 read, 12.5 write, 23.4 read+write
Amplification cumulative: 4.1 write, 6.8 compaction
Writes interval: 100000 total, 100000 batches, 1.0 per batch, 12.5 ingest MB
WAL interval: 100000 WAL writes, 100000 WAL syncs, 1.00 writes per sync, 0.01 MB written
Compaction IO interval (MB): 12.49 new, 14.98 read, 21.50 write, 36.48 read+write
Compaction IO interval (MB/sec): 6.4 new, 7.6 read, 11.0 write, 18.6 read+write
Amplification interval: 101.7 write, 102.9 compaction
Stalls(secs): 142.924 level0_slowdown, 0.000 level0_numfiles, 0.805 memtable_compaction, 0.000 leveln_slowdown
Stalls(count): 132461 level0_slowdown, 0 level0_numfiles, 3 memtable_compaction, 0 leveln_slowdown
Task ID: #3329644, #3301695
Blame Rev:
Test Plan:
Revert Plan:
Database Impact:
Memcache Impact:
Other Notes:
EImportant:
- begin *PUBLIC* platform impact section -
Bugzilla: #
- end platform impact -
Reviewers: dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14583
2013-12-09 22:43:34 +01:00
|
|
|
snprintf(buf, sizeof(buf),
|
|
|
|
"Writes cumulative: %llu total, %llu batches, "
|
|
|
|
"%.1f per batch, %.2f ingest GB\n",
|
|
|
|
(unsigned long long) (write_other + write_self),
|
|
|
|
(unsigned long long) write_self,
|
|
|
|
(write_other + write_self) / (double) (write_self + 1),
|
|
|
|
user_bytes_written / (1048576.0 * 1024));
|
|
|
|
value->append(buf);
|
|
|
|
|
|
|
|
snprintf(buf, sizeof(buf),
|
|
|
|
"WAL cumulative: %llu WAL writes, %llu WAL syncs, "
|
|
|
|
"%.2f writes per sync, %.2f GB written\n",
|
|
|
|
(unsigned long long) write_with_wal,
|
|
|
|
(unsigned long long ) wal_synced,
|
|
|
|
write_with_wal / (double) (wal_synced + 1),
|
|
|
|
wal_bytes / (1048576.0 * 1024));
|
|
|
|
value->append(buf);
|
|
|
|
|
2012-10-23 19:34:09 +02:00
|
|
|
snprintf(buf, sizeof(buf),
|
Improve output for GetProperty('leveldb.stats')
Summary:
Display separate values for read, write & total compaction IO.
Display compaction amplification and write amplification.
Add similar values for the period since the last call to GetProperty. Results since the server started
are reported as "cumulative" stats. Results since the last call to GetProperty are reported as
"interval" stats.
Level Files Size(MB) Time(sec) Read(MB) Write(MB) Rn(MB) Rnp1(MB) Wnew(MB) Amplify Read(MB/s) Write(MB/s) Rn Rnp1 Wnp1 NewW Count Ln-stall
----------------------------------------------------------------------------------------------------------------------------------------------------------------------
0 7 13 21 0 211 0 0 211 0.0 0.0 10.1 0 0 0 0 113 0.0
1 79 157 88 993 989 198 795 194 9.0 11.3 11.2 106 405 502 97 14 0.0
2 19 36 5 63 63 37 27 36 2.4 12.3 12.2 19 14 32 18 12 0.0
>>>>>>>>>>>>>>>>>>>>>>>>> text below has been is new and/or reformatted
Uptime(secs): 122.2 total, 0.9 interval
Compaction IO cumulative (GB): 0.21 new, 1.03 read, 1.23 write, 2.26 read+write
Compaction IO cumulative (MB/sec): 1.7 new, 8.6 read, 10.3 write, 19.0 read+write
Amplification cumulative: 6.0 write, 11.0 compaction
Compaction IO interval (MB): 5.59 new, 0.00 read, 5.59 write, 5.59 read+write
Compaction IO interval (MB/sec): 6.5 new, 0.0 read, 6.5 write, 6.5 read+write
Amplification interval: 1.0 write, 1.0 compaction
>>>>>>>>>>>>>>>>>>>>>>>> text above is new and/or reformatted
Stalls(secs): 90.574 level0_slowdown, 0.000 level0_numfiles, 10.165 memtable_compaction, 0.000 leveln_slowdown
Task ID: #
Blame Rev:
Test Plan:
make check, run db_bench
Revert Plan:
Database Impact:
Memcache Impact:
Other Notes:
EImportant:
- begin *PUBLIC* platform impact section -
Bugzilla: #
- end platform impact -
Reviewers: haobo
Reviewed By: haobo
CC: leveldb
Differential Revision: https://reviews.facebook.net/D11049
2013-06-03 17:16:16 +02:00
|
|
|
"Compaction IO cumulative (GB): "
|
|
|
|
"%.2f new, %.2f read, %.2f write, %.2f read+write\n",
|
Add monitoring for universal compaction and add counters for compaction IO
Summary:
Adds these counters
{ WAL_FILE_SYNCED, "rocksdb.wal.synced" }
number of writes that request a WAL sync
{ WAL_FILE_BYTES, "rocksdb.wal.bytes" },
number of bytes written to the WAL
{ WRITE_DONE_BY_SELF, "rocksdb.write.self" },
number of writes processed by the calling thread
{ WRITE_DONE_BY_OTHER, "rocksdb.write.other" },
number of writes not processed by the calling thread. Instead these were
processed by the current holder of the write lock
{ WRITE_WITH_WAL, "rocksdb.write.wal" },
number of writes that request WAL logging
{ COMPACT_READ_BYTES, "rocksdb.compact.read.bytes" },
number of bytes read during compaction
{ COMPACT_WRITE_BYTES, "rocksdb.compact.write.bytes" },
number of bytes written during compaction
Per-interval stats output was updated with WAL stats and correct stats for universal compaction
including a correct value for write-amplification. It now looks like:
Compactions
Level Files Size(MB) Score Time(sec) Read(MB) Write(MB) Rn(MB) Rnp1(MB) Wnew(MB) RW-Amplify Read(MB/s) Write(MB/s) Rn Rnp1 Wnp1 NewW Count Ln-stall Stall-cnt
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
0 7 464 46.4 281 3411 3875 3411 0 3875 2.1 12.1 13.8 621 0 240 240 628 0.0 0
Uptime(secs): 310.8 total, 2.0 interval
Writes cumulative: 9999999 total, 9999999 batches, 1.0 per batch, 1.22 ingest GB
WAL cumulative: 9999999 WAL writes, 9999999 WAL syncs, 1.00 writes per sync, 1.22 GB written
Compaction IO cumulative (GB): 1.22 new, 3.33 read, 3.78 write, 7.12 read+write
Compaction IO cumulative (MB/sec): 4.0 new, 11.0 read, 12.5 write, 23.4 read+write
Amplification cumulative: 4.1 write, 6.8 compaction
Writes interval: 100000 total, 100000 batches, 1.0 per batch, 12.5 ingest MB
WAL interval: 100000 WAL writes, 100000 WAL syncs, 1.00 writes per sync, 0.01 MB written
Compaction IO interval (MB): 12.49 new, 14.98 read, 21.50 write, 36.48 read+write
Compaction IO interval (MB/sec): 6.4 new, 7.6 read, 11.0 write, 18.6 read+write
Amplification interval: 101.7 write, 102.9 compaction
Stalls(secs): 142.924 level0_slowdown, 0.000 level0_numfiles, 0.805 memtable_compaction, 0.000 leveln_slowdown
Stalls(count): 132461 level0_slowdown, 0 level0_numfiles, 3 memtable_compaction, 0 leveln_slowdown
Task ID: #3329644, #3301695
Blame Rev:
Test Plan:
Revert Plan:
Database Impact:
Memcache Impact:
Other Notes:
EImportant:
- begin *PUBLIC* platform impact section -
Bugzilla: #
- end platform impact -
Reviewers: dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14583
2013-12-09 22:43:34 +01:00
|
|
|
user_bytes_written / (1048576.0 * 1024),
|
Improve output for GetProperty('leveldb.stats')
Summary:
Display separate values for read, write & total compaction IO.
Display compaction amplification and write amplification.
Add similar values for the period since the last call to GetProperty. Results since the server started
are reported as "cumulative" stats. Results since the last call to GetProperty are reported as
"interval" stats.
Level Files Size(MB) Time(sec) Read(MB) Write(MB) Rn(MB) Rnp1(MB) Wnew(MB) Amplify Read(MB/s) Write(MB/s) Rn Rnp1 Wnp1 NewW Count Ln-stall
----------------------------------------------------------------------------------------------------------------------------------------------------------------------
0 7 13 21 0 211 0 0 211 0.0 0.0 10.1 0 0 0 0 113 0.0
1 79 157 88 993 989 198 795 194 9.0 11.3 11.2 106 405 502 97 14 0.0
2 19 36 5 63 63 37 27 36 2.4 12.3 12.2 19 14 32 18 12 0.0
>>>>>>>>>>>>>>>>>>>>>>>>> text below has been is new and/or reformatted
Uptime(secs): 122.2 total, 0.9 interval
Compaction IO cumulative (GB): 0.21 new, 1.03 read, 1.23 write, 2.26 read+write
Compaction IO cumulative (MB/sec): 1.7 new, 8.6 read, 10.3 write, 19.0 read+write
Amplification cumulative: 6.0 write, 11.0 compaction
Compaction IO interval (MB): 5.59 new, 0.00 read, 5.59 write, 5.59 read+write
Compaction IO interval (MB/sec): 6.5 new, 0.0 read, 6.5 write, 6.5 read+write
Amplification interval: 1.0 write, 1.0 compaction
>>>>>>>>>>>>>>>>>>>>>>>> text above is new and/or reformatted
Stalls(secs): 90.574 level0_slowdown, 0.000 level0_numfiles, 10.165 memtable_compaction, 0.000 leveln_slowdown
Task ID: #
Blame Rev:
Test Plan:
make check, run db_bench
Revert Plan:
Database Impact:
Memcache Impact:
Other Notes:
EImportant:
- begin *PUBLIC* platform impact section -
Bugzilla: #
- end platform impact -
Reviewers: haobo
Reviewed By: haobo
CC: leveldb
Differential Revision: https://reviews.facebook.net/D11049
2013-06-03 17:16:16 +02:00
|
|
|
total_bytes_read / (1048576.0 * 1024),
|
|
|
|
total_bytes_written / (1048576.0 * 1024),
|
|
|
|
(total_bytes_read + total_bytes_written) / (1048576.0 * 1024));
|
|
|
|
value->append(buf);
|
|
|
|
|
|
|
|
snprintf(buf, sizeof(buf),
|
|
|
|
"Compaction IO cumulative (MB/sec): "
|
|
|
|
"%.1f new, %.1f read, %.1f write, %.1f read+write\n",
|
Add monitoring for universal compaction and add counters for compaction IO
Summary:
Adds these counters
{ WAL_FILE_SYNCED, "rocksdb.wal.synced" }
number of writes that request a WAL sync
{ WAL_FILE_BYTES, "rocksdb.wal.bytes" },
number of bytes written to the WAL
{ WRITE_DONE_BY_SELF, "rocksdb.write.self" },
number of writes processed by the calling thread
{ WRITE_DONE_BY_OTHER, "rocksdb.write.other" },
number of writes not processed by the calling thread. Instead these were
processed by the current holder of the write lock
{ WRITE_WITH_WAL, "rocksdb.write.wal" },
number of writes that request WAL logging
{ COMPACT_READ_BYTES, "rocksdb.compact.read.bytes" },
number of bytes read during compaction
{ COMPACT_WRITE_BYTES, "rocksdb.compact.write.bytes" },
number of bytes written during compaction
Per-interval stats output was updated with WAL stats and correct stats for universal compaction
including a correct value for write-amplification. It now looks like:
Compactions
Level Files Size(MB) Score Time(sec) Read(MB) Write(MB) Rn(MB) Rnp1(MB) Wnew(MB) RW-Amplify Read(MB/s) Write(MB/s) Rn Rnp1 Wnp1 NewW Count Ln-stall Stall-cnt
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
0 7 464 46.4 281 3411 3875 3411 0 3875 2.1 12.1 13.8 621 0 240 240 628 0.0 0
Uptime(secs): 310.8 total, 2.0 interval
Writes cumulative: 9999999 total, 9999999 batches, 1.0 per batch, 1.22 ingest GB
WAL cumulative: 9999999 WAL writes, 9999999 WAL syncs, 1.00 writes per sync, 1.22 GB written
Compaction IO cumulative (GB): 1.22 new, 3.33 read, 3.78 write, 7.12 read+write
Compaction IO cumulative (MB/sec): 4.0 new, 11.0 read, 12.5 write, 23.4 read+write
Amplification cumulative: 4.1 write, 6.8 compaction
Writes interval: 100000 total, 100000 batches, 1.0 per batch, 12.5 ingest MB
WAL interval: 100000 WAL writes, 100000 WAL syncs, 1.00 writes per sync, 0.01 MB written
Compaction IO interval (MB): 12.49 new, 14.98 read, 21.50 write, 36.48 read+write
Compaction IO interval (MB/sec): 6.4 new, 7.6 read, 11.0 write, 18.6 read+write
Amplification interval: 101.7 write, 102.9 compaction
Stalls(secs): 142.924 level0_slowdown, 0.000 level0_numfiles, 0.805 memtable_compaction, 0.000 leveln_slowdown
Stalls(count): 132461 level0_slowdown, 0 level0_numfiles, 3 memtable_compaction, 0 leveln_slowdown
Task ID: #3329644, #3301695
Blame Rev:
Test Plan:
Revert Plan:
Database Impact:
Memcache Impact:
Other Notes:
EImportant:
- begin *PUBLIC* platform impact section -
Bugzilla: #
- end platform impact -
Reviewers: dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14583
2013-12-09 22:43:34 +01:00
|
|
|
user_bytes_written / 1048576.0 / seconds_up,
|
Improve output for GetProperty('leveldb.stats')
Summary:
Display separate values for read, write & total compaction IO.
Display compaction amplification and write amplification.
Add similar values for the period since the last call to GetProperty. Results since the server started
are reported as "cumulative" stats. Results since the last call to GetProperty are reported as
"interval" stats.
Level Files Size(MB) Time(sec) Read(MB) Write(MB) Rn(MB) Rnp1(MB) Wnew(MB) Amplify Read(MB/s) Write(MB/s) Rn Rnp1 Wnp1 NewW Count Ln-stall
----------------------------------------------------------------------------------------------------------------------------------------------------------------------
0 7 13 21 0 211 0 0 211 0.0 0.0 10.1 0 0 0 0 113 0.0
1 79 157 88 993 989 198 795 194 9.0 11.3 11.2 106 405 502 97 14 0.0
2 19 36 5 63 63 37 27 36 2.4 12.3 12.2 19 14 32 18 12 0.0
>>>>>>>>>>>>>>>>>>>>>>>>> text below has been is new and/or reformatted
Uptime(secs): 122.2 total, 0.9 interval
Compaction IO cumulative (GB): 0.21 new, 1.03 read, 1.23 write, 2.26 read+write
Compaction IO cumulative (MB/sec): 1.7 new, 8.6 read, 10.3 write, 19.0 read+write
Amplification cumulative: 6.0 write, 11.0 compaction
Compaction IO interval (MB): 5.59 new, 0.00 read, 5.59 write, 5.59 read+write
Compaction IO interval (MB/sec): 6.5 new, 0.0 read, 6.5 write, 6.5 read+write
Amplification interval: 1.0 write, 1.0 compaction
>>>>>>>>>>>>>>>>>>>>>>>> text above is new and/or reformatted
Stalls(secs): 90.574 level0_slowdown, 0.000 level0_numfiles, 10.165 memtable_compaction, 0.000 leveln_slowdown
Task ID: #
Blame Rev:
Test Plan:
make check, run db_bench
Revert Plan:
Database Impact:
Memcache Impact:
Other Notes:
EImportant:
- begin *PUBLIC* platform impact section -
Bugzilla: #
- end platform impact -
Reviewers: haobo
Reviewed By: haobo
CC: leveldb
Differential Revision: https://reviews.facebook.net/D11049
2013-06-03 17:16:16 +02:00
|
|
|
total_bytes_read / 1048576.0 / seconds_up,
|
|
|
|
total_bytes_written / 1048576.0 / seconds_up,
|
|
|
|
(total_bytes_read + total_bytes_written) / 1048576.0 / seconds_up);
|
|
|
|
value->append(buf);
|
|
|
|
|
|
|
|
// +1 to avoid divide by 0 and NaN
|
|
|
|
snprintf(buf, sizeof(buf),
|
|
|
|
"Amplification cumulative: %.1f write, %.1f compaction\n",
|
Add monitoring for universal compaction and add counters for compaction IO
Summary:
Adds these counters
{ WAL_FILE_SYNCED, "rocksdb.wal.synced" }
number of writes that request a WAL sync
{ WAL_FILE_BYTES, "rocksdb.wal.bytes" },
number of bytes written to the WAL
{ WRITE_DONE_BY_SELF, "rocksdb.write.self" },
number of writes processed by the calling thread
{ WRITE_DONE_BY_OTHER, "rocksdb.write.other" },
number of writes not processed by the calling thread. Instead these were
processed by the current holder of the write lock
{ WRITE_WITH_WAL, "rocksdb.write.wal" },
number of writes that request WAL logging
{ COMPACT_READ_BYTES, "rocksdb.compact.read.bytes" },
number of bytes read during compaction
{ COMPACT_WRITE_BYTES, "rocksdb.compact.write.bytes" },
number of bytes written during compaction
Per-interval stats output was updated with WAL stats and correct stats for universal compaction
including a correct value for write-amplification. It now looks like:
Compactions
Level Files Size(MB) Score Time(sec) Read(MB) Write(MB) Rn(MB) Rnp1(MB) Wnew(MB) RW-Amplify Read(MB/s) Write(MB/s) Rn Rnp1 Wnp1 NewW Count Ln-stall Stall-cnt
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
0 7 464 46.4 281 3411 3875 3411 0 3875 2.1 12.1 13.8 621 0 240 240 628 0.0 0
Uptime(secs): 310.8 total, 2.0 interval
Writes cumulative: 9999999 total, 9999999 batches, 1.0 per batch, 1.22 ingest GB
WAL cumulative: 9999999 WAL writes, 9999999 WAL syncs, 1.00 writes per sync, 1.22 GB written
Compaction IO cumulative (GB): 1.22 new, 3.33 read, 3.78 write, 7.12 read+write
Compaction IO cumulative (MB/sec): 4.0 new, 11.0 read, 12.5 write, 23.4 read+write
Amplification cumulative: 4.1 write, 6.8 compaction
Writes interval: 100000 total, 100000 batches, 1.0 per batch, 12.5 ingest MB
WAL interval: 100000 WAL writes, 100000 WAL syncs, 1.00 writes per sync, 0.01 MB written
Compaction IO interval (MB): 12.49 new, 14.98 read, 21.50 write, 36.48 read+write
Compaction IO interval (MB/sec): 6.4 new, 7.6 read, 11.0 write, 18.6 read+write
Amplification interval: 101.7 write, 102.9 compaction
Stalls(secs): 142.924 level0_slowdown, 0.000 level0_numfiles, 0.805 memtable_compaction, 0.000 leveln_slowdown
Stalls(count): 132461 level0_slowdown, 0 level0_numfiles, 3 memtable_compaction, 0 leveln_slowdown
Task ID: #3329644, #3301695
Blame Rev:
Test Plan:
Revert Plan:
Database Impact:
Memcache Impact:
Other Notes:
EImportant:
- begin *PUBLIC* platform impact section -
Bugzilla: #
- end platform impact -
Reviewers: dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14583
2013-12-09 22:43:34 +01:00
|
|
|
(double) (total_bytes_written + wal_bytes)
|
|
|
|
/ (user_bytes_written + 1),
|
|
|
|
(double) (total_bytes_written + total_bytes_read + wal_bytes)
|
|
|
|
/ (user_bytes_written + 1));
|
|
|
|
value->append(buf);
|
|
|
|
|
|
|
|
uint64_t interval_write_other = write_other - last_stats_.write_other_;
|
|
|
|
uint64_t interval_write_self = write_self - last_stats_.write_self_;
|
|
|
|
|
|
|
|
snprintf(buf, sizeof(buf),
|
|
|
|
"Writes interval: %llu total, %llu batches, "
|
|
|
|
"%.1f per batch, %.1f ingest MB\n",
|
|
|
|
(unsigned long long) (interval_write_other + interval_write_self),
|
|
|
|
(unsigned long long) interval_write_self,
|
|
|
|
(double) (interval_write_other + interval_write_self)
|
|
|
|
/ (interval_write_self + 1),
|
|
|
|
(user_bytes_written - last_stats_.ingest_bytes_) / 1048576.0);
|
|
|
|
value->append(buf);
|
|
|
|
|
|
|
|
uint64_t interval_write_with_wal =
|
|
|
|
write_with_wal - last_stats_.write_with_wal_;
|
|
|
|
|
|
|
|
uint64_t interval_wal_synced = wal_synced - last_stats_.wal_synced_;
|
|
|
|
uint64_t interval_wal_bytes = wal_bytes - last_stats_.wal_bytes_;
|
|
|
|
|
|
|
|
snprintf(buf, sizeof(buf),
|
|
|
|
"WAL interval: %llu WAL writes, %llu WAL syncs, "
|
|
|
|
"%.2f writes per sync, %.2f MB written\n",
|
|
|
|
(unsigned long long) interval_write_with_wal,
|
|
|
|
(unsigned long long ) interval_wal_synced,
|
|
|
|
interval_write_with_wal / (double) (interval_wal_synced + 1),
|
|
|
|
interval_wal_bytes / (1048576.0 * 1024));
|
2012-10-23 19:34:09 +02:00
|
|
|
value->append(buf);
|
|
|
|
|
Improve output for GetProperty('leveldb.stats')
Summary:
Display separate values for read, write & total compaction IO.
Display compaction amplification and write amplification.
Add similar values for the period since the last call to GetProperty. Results since the server started
are reported as "cumulative" stats. Results since the last call to GetProperty are reported as
"interval" stats.
Level Files Size(MB) Time(sec) Read(MB) Write(MB) Rn(MB) Rnp1(MB) Wnew(MB) Amplify Read(MB/s) Write(MB/s) Rn Rnp1 Wnp1 NewW Count Ln-stall
----------------------------------------------------------------------------------------------------------------------------------------------------------------------
0 7 13 21 0 211 0 0 211 0.0 0.0 10.1 0 0 0 0 113 0.0
1 79 157 88 993 989 198 795 194 9.0 11.3 11.2 106 405 502 97 14 0.0
2 19 36 5 63 63 37 27 36 2.4 12.3 12.2 19 14 32 18 12 0.0
>>>>>>>>>>>>>>>>>>>>>>>>> text below has been is new and/or reformatted
Uptime(secs): 122.2 total, 0.9 interval
Compaction IO cumulative (GB): 0.21 new, 1.03 read, 1.23 write, 2.26 read+write
Compaction IO cumulative (MB/sec): 1.7 new, 8.6 read, 10.3 write, 19.0 read+write
Amplification cumulative: 6.0 write, 11.0 compaction
Compaction IO interval (MB): 5.59 new, 0.00 read, 5.59 write, 5.59 read+write
Compaction IO interval (MB/sec): 6.5 new, 0.0 read, 6.5 write, 6.5 read+write
Amplification interval: 1.0 write, 1.0 compaction
>>>>>>>>>>>>>>>>>>>>>>>> text above is new and/or reformatted
Stalls(secs): 90.574 level0_slowdown, 0.000 level0_numfiles, 10.165 memtable_compaction, 0.000 leveln_slowdown
Task ID: #
Blame Rev:
Test Plan:
make check, run db_bench
Revert Plan:
Database Impact:
Memcache Impact:
Other Notes:
EImportant:
- begin *PUBLIC* platform impact section -
Bugzilla: #
- end platform impact -
Reviewers: haobo
Reviewed By: haobo
CC: leveldb
Differential Revision: https://reviews.facebook.net/D11049
2013-06-03 17:16:16 +02:00
|
|
|
snprintf(buf, sizeof(buf),
|
|
|
|
"Compaction IO interval (MB): "
|
|
|
|
"%.2f new, %.2f read, %.2f write, %.2f read+write\n",
|
|
|
|
interval_bytes_new / 1048576.0,
|
|
|
|
interval_bytes_read/ 1048576.0,
|
|
|
|
interval_bytes_written / 1048576.0,
|
|
|
|
(interval_bytes_read + interval_bytes_written) / 1048576.0);
|
|
|
|
value->append(buf);
|
|
|
|
|
|
|
|
snprintf(buf, sizeof(buf),
|
|
|
|
"Compaction IO interval (MB/sec): "
|
|
|
|
"%.1f new, %.1f read, %.1f write, %.1f read+write\n",
|
|
|
|
interval_bytes_new / 1048576.0 / interval_seconds_up,
|
|
|
|
interval_bytes_read / 1048576.0 / interval_seconds_up,
|
|
|
|
interval_bytes_written / 1048576.0 / interval_seconds_up,
|
|
|
|
(interval_bytes_read + interval_bytes_written)
|
|
|
|
/ 1048576.0 / interval_seconds_up);
|
|
|
|
value->append(buf);
|
|
|
|
|
|
|
|
// +1 to avoid divide by 0 and NaN
|
|
|
|
snprintf(buf, sizeof(buf),
|
|
|
|
"Amplification interval: %.1f write, %.1f compaction\n",
|
Add monitoring for universal compaction and add counters for compaction IO
Summary:
Adds these counters
{ WAL_FILE_SYNCED, "rocksdb.wal.synced" }
number of writes that request a WAL sync
{ WAL_FILE_BYTES, "rocksdb.wal.bytes" },
number of bytes written to the WAL
{ WRITE_DONE_BY_SELF, "rocksdb.write.self" },
number of writes processed by the calling thread
{ WRITE_DONE_BY_OTHER, "rocksdb.write.other" },
number of writes not processed by the calling thread. Instead these were
processed by the current holder of the write lock
{ WRITE_WITH_WAL, "rocksdb.write.wal" },
number of writes that request WAL logging
{ COMPACT_READ_BYTES, "rocksdb.compact.read.bytes" },
number of bytes read during compaction
{ COMPACT_WRITE_BYTES, "rocksdb.compact.write.bytes" },
number of bytes written during compaction
Per-interval stats output was updated with WAL stats and correct stats for universal compaction
including a correct value for write-amplification. It now looks like:
Compactions
Level Files Size(MB) Score Time(sec) Read(MB) Write(MB) Rn(MB) Rnp1(MB) Wnew(MB) RW-Amplify Read(MB/s) Write(MB/s) Rn Rnp1 Wnp1 NewW Count Ln-stall Stall-cnt
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
0 7 464 46.4 281 3411 3875 3411 0 3875 2.1 12.1 13.8 621 0 240 240 628 0.0 0
Uptime(secs): 310.8 total, 2.0 interval
Writes cumulative: 9999999 total, 9999999 batches, 1.0 per batch, 1.22 ingest GB
WAL cumulative: 9999999 WAL writes, 9999999 WAL syncs, 1.00 writes per sync, 1.22 GB written
Compaction IO cumulative (GB): 1.22 new, 3.33 read, 3.78 write, 7.12 read+write
Compaction IO cumulative (MB/sec): 4.0 new, 11.0 read, 12.5 write, 23.4 read+write
Amplification cumulative: 4.1 write, 6.8 compaction
Writes interval: 100000 total, 100000 batches, 1.0 per batch, 12.5 ingest MB
WAL interval: 100000 WAL writes, 100000 WAL syncs, 1.00 writes per sync, 0.01 MB written
Compaction IO interval (MB): 12.49 new, 14.98 read, 21.50 write, 36.48 read+write
Compaction IO interval (MB/sec): 6.4 new, 7.6 read, 11.0 write, 18.6 read+write
Amplification interval: 101.7 write, 102.9 compaction
Stalls(secs): 142.924 level0_slowdown, 0.000 level0_numfiles, 0.805 memtable_compaction, 0.000 leveln_slowdown
Stalls(count): 132461 level0_slowdown, 0 level0_numfiles, 3 memtable_compaction, 0 leveln_slowdown
Task ID: #3329644, #3301695
Blame Rev:
Test Plan:
Revert Plan:
Database Impact:
Memcache Impact:
Other Notes:
EImportant:
- begin *PUBLIC* platform impact section -
Bugzilla: #
- end platform impact -
Reviewers: dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14583
2013-12-09 22:43:34 +01:00
|
|
|
(double) (interval_bytes_written + wal_bytes)
|
|
|
|
/ (interval_bytes_new + 1),
|
|
|
|
(double) (interval_bytes_written + interval_bytes_read + wal_bytes)
|
|
|
|
/ (interval_bytes_new + 1));
|
2012-10-23 19:34:09 +02:00
|
|
|
value->append(buf);
|
|
|
|
|
|
|
|
snprintf(buf, sizeof(buf),
|
|
|
|
"Stalls(secs): %.3f level0_slowdown, %.3f level0_numfiles, "
|
2012-10-26 22:37:21 +02:00
|
|
|
"%.3f memtable_compaction, %.3f leveln_slowdown\n",
|
2012-10-23 19:34:09 +02:00
|
|
|
stall_level0_slowdown_ / 1000000.0,
|
|
|
|
stall_level0_num_files_ / 1000000.0,
|
2012-10-26 22:37:21 +02:00
|
|
|
stall_memtable_compaction_ / 1000000.0,
|
2013-03-02 21:56:04 +01:00
|
|
|
total_slowdown / 1000000.0);
|
2012-10-23 19:34:09 +02:00
|
|
|
value->append(buf);
|
|
|
|
|
2013-07-29 19:34:23 +02:00
|
|
|
snprintf(buf, sizeof(buf),
|
|
|
|
"Stalls(count): %lu level0_slowdown, %lu level0_numfiles, "
|
|
|
|
"%lu memtable_compaction, %lu leveln_slowdown\n",
|
|
|
|
(unsigned long) stall_level0_slowdown_count_,
|
|
|
|
(unsigned long) stall_level0_num_files_count_,
|
|
|
|
(unsigned long) stall_memtable_compaction_count_,
|
|
|
|
(unsigned long) total_slowdown_count);
|
|
|
|
value->append(buf);
|
|
|
|
|
Add monitoring for universal compaction and add counters for compaction IO
Summary:
Adds these counters
{ WAL_FILE_SYNCED, "rocksdb.wal.synced" }
number of writes that request a WAL sync
{ WAL_FILE_BYTES, "rocksdb.wal.bytes" },
number of bytes written to the WAL
{ WRITE_DONE_BY_SELF, "rocksdb.write.self" },
number of writes processed by the calling thread
{ WRITE_DONE_BY_OTHER, "rocksdb.write.other" },
number of writes not processed by the calling thread. Instead these were
processed by the current holder of the write lock
{ WRITE_WITH_WAL, "rocksdb.write.wal" },
number of writes that request WAL logging
{ COMPACT_READ_BYTES, "rocksdb.compact.read.bytes" },
number of bytes read during compaction
{ COMPACT_WRITE_BYTES, "rocksdb.compact.write.bytes" },
number of bytes written during compaction
Per-interval stats output was updated with WAL stats and correct stats for universal compaction
including a correct value for write-amplification. It now looks like:
Compactions
Level Files Size(MB) Score Time(sec) Read(MB) Write(MB) Rn(MB) Rnp1(MB) Wnew(MB) RW-Amplify Read(MB/s) Write(MB/s) Rn Rnp1 Wnp1 NewW Count Ln-stall Stall-cnt
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
0 7 464 46.4 281 3411 3875 3411 0 3875 2.1 12.1 13.8 621 0 240 240 628 0.0 0
Uptime(secs): 310.8 total, 2.0 interval
Writes cumulative: 9999999 total, 9999999 batches, 1.0 per batch, 1.22 ingest GB
WAL cumulative: 9999999 WAL writes, 9999999 WAL syncs, 1.00 writes per sync, 1.22 GB written
Compaction IO cumulative (GB): 1.22 new, 3.33 read, 3.78 write, 7.12 read+write
Compaction IO cumulative (MB/sec): 4.0 new, 11.0 read, 12.5 write, 23.4 read+write
Amplification cumulative: 4.1 write, 6.8 compaction
Writes interval: 100000 total, 100000 batches, 1.0 per batch, 12.5 ingest MB
WAL interval: 100000 WAL writes, 100000 WAL syncs, 1.00 writes per sync, 0.01 MB written
Compaction IO interval (MB): 12.49 new, 14.98 read, 21.50 write, 36.48 read+write
Compaction IO interval (MB/sec): 6.4 new, 7.6 read, 11.0 write, 18.6 read+write
Amplification interval: 101.7 write, 102.9 compaction
Stalls(secs): 142.924 level0_slowdown, 0.000 level0_numfiles, 0.805 memtable_compaction, 0.000 leveln_slowdown
Stalls(count): 132461 level0_slowdown, 0 level0_numfiles, 3 memtable_compaction, 0 leveln_slowdown
Task ID: #3329644, #3301695
Blame Rev:
Test Plan:
Revert Plan:
Database Impact:
Memcache Impact:
Other Notes:
EImportant:
- begin *PUBLIC* platform impact section -
Bugzilla: #
- end platform impact -
Reviewers: dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14583
2013-12-09 22:43:34 +01:00
|
|
|
last_stats_.compaction_bytes_read_ = total_bytes_read;
|
|
|
|
last_stats_.compaction_bytes_written_ = total_bytes_written;
|
|
|
|
last_stats_.ingest_bytes_ = user_bytes_written;
|
Improve output for GetProperty('leveldb.stats')
Summary:
Display separate values for read, write & total compaction IO.
Display compaction amplification and write amplification.
Add similar values for the period since the last call to GetProperty. Results since the server started
are reported as "cumulative" stats. Results since the last call to GetProperty are reported as
"interval" stats.
Level Files Size(MB) Time(sec) Read(MB) Write(MB) Rn(MB) Rnp1(MB) Wnew(MB) Amplify Read(MB/s) Write(MB/s) Rn Rnp1 Wnp1 NewW Count Ln-stall
----------------------------------------------------------------------------------------------------------------------------------------------------------------------
0 7 13 21 0 211 0 0 211 0.0 0.0 10.1 0 0 0 0 113 0.0
1 79 157 88 993 989 198 795 194 9.0 11.3 11.2 106 405 502 97 14 0.0
2 19 36 5 63 63 37 27 36 2.4 12.3 12.2 19 14 32 18 12 0.0
>>>>>>>>>>>>>>>>>>>>>>>>> text below has been is new and/or reformatted
Uptime(secs): 122.2 total, 0.9 interval
Compaction IO cumulative (GB): 0.21 new, 1.03 read, 1.23 write, 2.26 read+write
Compaction IO cumulative (MB/sec): 1.7 new, 8.6 read, 10.3 write, 19.0 read+write
Amplification cumulative: 6.0 write, 11.0 compaction
Compaction IO interval (MB): 5.59 new, 0.00 read, 5.59 write, 5.59 read+write
Compaction IO interval (MB/sec): 6.5 new, 0.0 read, 6.5 write, 6.5 read+write
Amplification interval: 1.0 write, 1.0 compaction
>>>>>>>>>>>>>>>>>>>>>>>> text above is new and/or reformatted
Stalls(secs): 90.574 level0_slowdown, 0.000 level0_numfiles, 10.165 memtable_compaction, 0.000 leveln_slowdown
Task ID: #
Blame Rev:
Test Plan:
make check, run db_bench
Revert Plan:
Database Impact:
Memcache Impact:
Other Notes:
EImportant:
- begin *PUBLIC* platform impact section -
Bugzilla: #
- end platform impact -
Reviewers: haobo
Reviewed By: haobo
CC: leveldb
Differential Revision: https://reviews.facebook.net/D11049
2013-06-03 17:16:16 +02:00
|
|
|
last_stats_.seconds_up_ = seconds_up;
|
Add monitoring for universal compaction and add counters for compaction IO
Summary:
Adds these counters
{ WAL_FILE_SYNCED, "rocksdb.wal.synced" }
number of writes that request a WAL sync
{ WAL_FILE_BYTES, "rocksdb.wal.bytes" },
number of bytes written to the WAL
{ WRITE_DONE_BY_SELF, "rocksdb.write.self" },
number of writes processed by the calling thread
{ WRITE_DONE_BY_OTHER, "rocksdb.write.other" },
number of writes not processed by the calling thread. Instead these were
processed by the current holder of the write lock
{ WRITE_WITH_WAL, "rocksdb.write.wal" },
number of writes that request WAL logging
{ COMPACT_READ_BYTES, "rocksdb.compact.read.bytes" },
number of bytes read during compaction
{ COMPACT_WRITE_BYTES, "rocksdb.compact.write.bytes" },
number of bytes written during compaction
Per-interval stats output was updated with WAL stats and correct stats for universal compaction
including a correct value for write-amplification. It now looks like:
Compactions
Level Files Size(MB) Score Time(sec) Read(MB) Write(MB) Rn(MB) Rnp1(MB) Wnew(MB) RW-Amplify Read(MB/s) Write(MB/s) Rn Rnp1 Wnp1 NewW Count Ln-stall Stall-cnt
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
0 7 464 46.4 281 3411 3875 3411 0 3875 2.1 12.1 13.8 621 0 240 240 628 0.0 0
Uptime(secs): 310.8 total, 2.0 interval
Writes cumulative: 9999999 total, 9999999 batches, 1.0 per batch, 1.22 ingest GB
WAL cumulative: 9999999 WAL writes, 9999999 WAL syncs, 1.00 writes per sync, 1.22 GB written
Compaction IO cumulative (GB): 1.22 new, 3.33 read, 3.78 write, 7.12 read+write
Compaction IO cumulative (MB/sec): 4.0 new, 11.0 read, 12.5 write, 23.4 read+write
Amplification cumulative: 4.1 write, 6.8 compaction
Writes interval: 100000 total, 100000 batches, 1.0 per batch, 12.5 ingest MB
WAL interval: 100000 WAL writes, 100000 WAL syncs, 1.00 writes per sync, 0.01 MB written
Compaction IO interval (MB): 12.49 new, 14.98 read, 21.50 write, 36.48 read+write
Compaction IO interval (MB/sec): 6.4 new, 7.6 read, 11.0 write, 18.6 read+write
Amplification interval: 101.7 write, 102.9 compaction
Stalls(secs): 142.924 level0_slowdown, 0.000 level0_numfiles, 0.805 memtable_compaction, 0.000 leveln_slowdown
Stalls(count): 132461 level0_slowdown, 0 level0_numfiles, 3 memtable_compaction, 0 leveln_slowdown
Task ID: #3329644, #3301695
Blame Rev:
Test Plan:
Revert Plan:
Database Impact:
Memcache Impact:
Other Notes:
EImportant:
- begin *PUBLIC* platform impact section -
Bugzilla: #
- end platform impact -
Reviewers: dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14583
2013-12-09 22:43:34 +01:00
|
|
|
last_stats_.wal_bytes_ = wal_bytes;
|
|
|
|
last_stats_.wal_synced_ = wal_synced;
|
|
|
|
last_stats_.write_with_wal_ = write_with_wal;
|
|
|
|
last_stats_.write_other_ = write_other;
|
|
|
|
last_stats_.write_self_ = write_self;
|
Improve output for GetProperty('leveldb.stats')
Summary:
Display separate values for read, write & total compaction IO.
Display compaction amplification and write amplification.
Add similar values for the period since the last call to GetProperty. Results since the server started
are reported as "cumulative" stats. Results since the last call to GetProperty are reported as
"interval" stats.
Level Files Size(MB) Time(sec) Read(MB) Write(MB) Rn(MB) Rnp1(MB) Wnew(MB) Amplify Read(MB/s) Write(MB/s) Rn Rnp1 Wnp1 NewW Count Ln-stall
----------------------------------------------------------------------------------------------------------------------------------------------------------------------
0 7 13 21 0 211 0 0 211 0.0 0.0 10.1 0 0 0 0 113 0.0
1 79 157 88 993 989 198 795 194 9.0 11.3 11.2 106 405 502 97 14 0.0
2 19 36 5 63 63 37 27 36 2.4 12.3 12.2 19 14 32 18 12 0.0
>>>>>>>>>>>>>>>>>>>>>>>>> text below has been is new and/or reformatted
Uptime(secs): 122.2 total, 0.9 interval
Compaction IO cumulative (GB): 0.21 new, 1.03 read, 1.23 write, 2.26 read+write
Compaction IO cumulative (MB/sec): 1.7 new, 8.6 read, 10.3 write, 19.0 read+write
Amplification cumulative: 6.0 write, 11.0 compaction
Compaction IO interval (MB): 5.59 new, 0.00 read, 5.59 write, 5.59 read+write
Compaction IO interval (MB/sec): 6.5 new, 0.0 read, 6.5 write, 6.5 read+write
Amplification interval: 1.0 write, 1.0 compaction
>>>>>>>>>>>>>>>>>>>>>>>> text above is new and/or reformatted
Stalls(secs): 90.574 level0_slowdown, 0.000 level0_numfiles, 10.165 memtable_compaction, 0.000 leveln_slowdown
Task ID: #
Blame Rev:
Test Plan:
make check, run db_bench
Revert Plan:
Database Impact:
Memcache Impact:
Other Notes:
EImportant:
- begin *PUBLIC* platform impact section -
Bugzilla: #
- end platform impact -
Reviewers: haobo
Reviewed By: haobo
CC: leveldb
Differential Revision: https://reviews.facebook.net/D11049
2013-06-03 17:16:16 +02:00
|
|
|
|
2011-04-12 21:38:58 +02:00
|
|
|
return true;
|
2011-10-06 01:30:28 +02:00
|
|
|
} else if (in == "sstables") {
|
2014-01-29 22:28:50 +01:00
|
|
|
*value = default_cfd_->current()->DebugString();
|
2011-10-06 01:30:28 +02:00
|
|
|
return true;
|
2013-10-05 00:17:54 +02:00
|
|
|
} else if (in == "num-immutable-mem-table") {
|
2014-01-29 22:28:50 +01:00
|
|
|
*value = std::to_string(default_cfd_->imm()->size());
|
2013-10-05 00:17:54 +02:00
|
|
|
return true;
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
2011-04-12 21:38:58 +02:00
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 20:14:09 +01:00
|
|
|
void DBImpl::GetApproximateSizes(const ColumnFamilyHandle& column_family,
|
|
|
|
const Range* range, int n, uint64_t* sizes) {
|
2011-03-18 23:37:00 +01:00
|
|
|
// TODO(opt): better implementation
|
|
|
|
Version* v;
|
|
|
|
{
|
|
|
|
MutexLock l(&mutex_);
|
2014-01-29 22:28:50 +01:00
|
|
|
v = default_cfd_->current();
|
2014-01-27 23:33:50 +01:00
|
|
|
v->Ref();
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
for (int i = 0; i < n; i++) {
|
|
|
|
// Convert user_key into a corresponding internal key.
|
|
|
|
InternalKey k1(range[i].start, kMaxSequenceNumber, kValueTypeForSeek);
|
|
|
|
InternalKey k2(range[i].limit, kMaxSequenceNumber, kValueTypeForSeek);
|
|
|
|
uint64_t start = versions_->ApproximateOffsetOf(v, k1);
|
|
|
|
uint64_t limit = versions_->ApproximateOffsetOf(v, k2);
|
|
|
|
sizes[i] = (limit >= start ? limit - start : 0);
|
|
|
|
}
|
|
|
|
|
|
|
|
{
|
|
|
|
MutexLock l(&mutex_);
|
|
|
|
v->Unref();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2012-10-19 23:00:53 +02:00
|
|
|
inline void DBImpl::DelayLoggingAndReset() {
|
|
|
|
if (delayed_writes_ > 0) {
|
|
|
|
Log(options_.info_log, "delayed %d write...\n", delayed_writes_ );
|
|
|
|
delayed_writes_ = 0;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-08-22 23:32:53 +02:00
|
|
|
Status DBImpl::DeleteFile(std::string name) {
|
|
|
|
uint64_t number;
|
|
|
|
FileType type;
|
2013-10-24 08:39:23 +02:00
|
|
|
WalFileType log_type;
|
|
|
|
if (!ParseFileName(name, &number, &type, &log_type) ||
|
|
|
|
(type != kTableFile && type != kLogFile)) {
|
|
|
|
Log(options_.info_log, "DeleteFile %s failed.\n", name.c_str());
|
2013-08-22 23:32:53 +02:00
|
|
|
return Status::InvalidArgument("Invalid file name");
|
|
|
|
}
|
|
|
|
|
2013-10-24 08:39:23 +02:00
|
|
|
Status status;
|
|
|
|
if (type == kLogFile) {
|
|
|
|
// Only allow deleting archived log files
|
|
|
|
if (log_type != kArchivedLogFile) {
|
|
|
|
Log(options_.info_log, "DeleteFile %s failed.\n", name.c_str());
|
|
|
|
return Status::NotSupported("Delete only supported for archived logs");
|
|
|
|
}
|
|
|
|
status = env_->DeleteFile(options_.wal_dir + "/" + name.c_str());
|
|
|
|
if (!status.ok()) {
|
|
|
|
Log(options_.info_log, "DeleteFile %s failed.\n", name.c_str());
|
|
|
|
}
|
|
|
|
return status;
|
|
|
|
}
|
|
|
|
|
2013-08-22 23:32:53 +02:00
|
|
|
int level;
|
|
|
|
FileMetaData metadata;
|
|
|
|
int maxlevel = NumberLevels();
|
2014-01-27 23:33:50 +01:00
|
|
|
ColumnFamilyData* cfd;
|
2014-01-15 00:27:09 +01:00
|
|
|
VersionEdit edit;
|
2013-12-20 18:57:58 +01:00
|
|
|
DeletionState deletion_state(0, true);
|
2013-08-27 23:54:06 +02:00
|
|
|
{
|
|
|
|
MutexLock l(&mutex_);
|
2014-01-27 23:33:50 +01:00
|
|
|
status = versions_->GetMetadataForFile(number, &level, &metadata, &cfd);
|
2013-08-27 23:54:06 +02:00
|
|
|
if (!status.ok()) {
|
2013-10-24 08:39:23 +02:00
|
|
|
Log(options_.info_log, "DeleteFile %s failed. File not found\n",
|
|
|
|
name.c_str());
|
2013-08-27 23:54:06 +02:00
|
|
|
return Status::InvalidArgument("File not found");
|
|
|
|
}
|
|
|
|
assert((level > 0) && (level < maxlevel));
|
2013-08-22 23:32:53 +02:00
|
|
|
|
2013-08-27 23:54:06 +02:00
|
|
|
// If the file is being compacted no need to delete.
|
|
|
|
if (metadata.being_compacted) {
|
2013-08-22 23:32:53 +02:00
|
|
|
Log(options_.info_log,
|
2013-10-24 08:39:23 +02:00
|
|
|
"DeleteFile %s Skipped. File about to be compacted\n", name.c_str());
|
2013-08-27 23:54:06 +02:00
|
|
|
return Status::OK();
|
2013-08-22 23:32:53 +02:00
|
|
|
}
|
|
|
|
|
2013-08-27 23:54:06 +02:00
|
|
|
// Only the files in the last level can be deleted externally.
|
|
|
|
// This is to make sure that any deletion tombstones are not
|
|
|
|
// lost. Check that the level passed is the last level.
|
|
|
|
for (int i = level + 1; i < maxlevel; i++) {
|
2014-01-29 22:28:50 +01:00
|
|
|
if (cfd->current()->NumLevelFiles(i) != 0) {
|
2013-08-27 23:54:06 +02:00
|
|
|
Log(options_.info_log,
|
2013-10-24 08:39:23 +02:00
|
|
|
"DeleteFile %s FAILED. File not in last level\n", name.c_str());
|
2013-08-27 23:54:06 +02:00
|
|
|
return Status::InvalidArgument("File not in last level");
|
|
|
|
}
|
|
|
|
}
|
|
|
|
edit.DeleteFile(level, number);
|
2014-01-27 23:33:50 +01:00
|
|
|
status = versions_->LogAndApply(cfd, &edit, &mutex_, db_directory_.get());
|
2013-12-20 18:57:58 +01:00
|
|
|
if (status.ok()) {
|
2014-01-27 23:33:50 +01:00
|
|
|
InstallSuperVersion(cfd, deletion_state);
|
2013-12-20 18:57:58 +01:00
|
|
|
}
|
2013-11-12 06:03:41 +01:00
|
|
|
FindObsoleteFiles(deletion_state, false);
|
2013-08-27 23:54:06 +02:00
|
|
|
} // lock released here
|
2013-11-07 20:31:56 +01:00
|
|
|
LogFlush(options_.info_log);
|
2013-12-20 18:57:58 +01:00
|
|
|
// remove files outside the db-lock
|
|
|
|
PurgeObsoleteFiles(deletion_state);
|
2013-08-22 23:32:53 +02:00
|
|
|
return status;
|
|
|
|
}
|
|
|
|
|
2013-09-01 10:52:32 +02:00
|
|
|
void DBImpl::GetLiveFilesMetaData(std::vector<LiveFileMetaData> *metadata) {
|
2013-08-22 23:32:53 +02:00
|
|
|
MutexLock l(&mutex_);
|
|
|
|
return versions_->GetLiveFilesMetaData(metadata);
|
|
|
|
}
|
|
|
|
|
2013-12-03 15:39:07 +01:00
|
|
|
Status DBImpl::GetDbIdentity(std::string& identity) {
|
|
|
|
std::string idfilename = IdentityFileName(dbname_);
|
|
|
|
unique_ptr<SequentialFile> idfile;
|
|
|
|
const EnvOptions soptions;
|
|
|
|
Status s = env_->NewSequentialFile(idfilename, &idfile, soptions);
|
|
|
|
if (!s.ok()) {
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
uint64_t file_size;
|
|
|
|
s = env_->GetFileSize(idfilename, &file_size);
|
|
|
|
if (!s.ok()) {
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
char buffer[file_size];
|
|
|
|
Slice id;
|
|
|
|
s = idfile->Read(file_size, &id, buffer);
|
|
|
|
if (!s.ok()) {
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
identity.assign(id.ToString());
|
|
|
|
// If last character is '\n' remove it from identity
|
|
|
|
if (identity.size() > 0 && identity.back() == '\n') {
|
|
|
|
identity.pop_back();
|
|
|
|
}
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
// Default implementations of convenience methods that subclasses of DB
|
|
|
|
// can call if they wish
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 20:14:09 +01:00
|
|
|
Status DB::Put(const WriteOptions& opt, const ColumnFamilyHandle& column_family,
|
|
|
|
const Slice& key, const Slice& value) {
|
2014-01-14 19:42:36 +01:00
|
|
|
// Pre-allocate size of write batch conservatively.
|
|
|
|
// 8 bytes are taken by header, 4 bytes for count, 1 byte for type,
|
|
|
|
// and we allocate 11 extra bytes for key length, as well as value length.
|
|
|
|
WriteBatch batch(key.size() + value.size() + 24);
|
2014-01-07 23:41:42 +01:00
|
|
|
batch.Put(column_family.id, key, value);
|
2011-03-18 23:37:00 +01:00
|
|
|
return Write(opt, &batch);
|
|
|
|
}
|
|
|
|
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 20:14:09 +01:00
|
|
|
Status DB::Delete(const WriteOptions& opt,
|
|
|
|
const ColumnFamilyHandle& column_family, const Slice& key) {
|
2011-03-18 23:37:00 +01:00
|
|
|
WriteBatch batch;
|
2014-01-07 23:41:42 +01:00
|
|
|
batch.Delete(column_family.id, key);
|
2011-03-18 23:37:00 +01:00
|
|
|
return Write(opt, &batch);
|
|
|
|
}
|
|
|
|
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 20:14:09 +01:00
|
|
|
Status DB::Merge(const WriteOptions& opt,
|
|
|
|
const ColumnFamilyHandle& column_family, const Slice& key,
|
2013-03-21 23:59:47 +01:00
|
|
|
const Slice& value) {
|
|
|
|
WriteBatch batch;
|
2014-01-07 23:41:42 +01:00
|
|
|
batch.Merge(column_family.id, key, value);
|
2013-03-21 23:59:47 +01:00
|
|
|
return Write(opt, &batch);
|
|
|
|
}
|
|
|
|
|
2014-01-02 18:08:12 +01:00
|
|
|
// Default implementation -- returns not supported status
|
|
|
|
Status DB::CreateColumnFamily(const ColumnFamilyOptions& options,
|
2014-01-06 22:31:06 +01:00
|
|
|
const std::string& column_family_name,
|
2014-01-02 18:08:12 +01:00
|
|
|
ColumnFamilyHandle* handle) {
|
|
|
|
return Status::NotSupported("");
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 20:14:09 +01:00
|
|
|
}
|
|
|
|
Status DB::DropColumnFamily(const ColumnFamilyHandle& column_family) {
|
2014-01-02 18:08:12 +01:00
|
|
|
return Status::NotSupported("");
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 20:14:09 +01:00
|
|
|
}
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
DB::~DB() { }
|
|
|
|
|
2013-07-23 23:42:27 +02:00
|
|
|
Status DB::Open(const Options& options, const std::string& dbname, DB** dbptr) {
|
2014-01-06 22:31:06 +01:00
|
|
|
DBOptions db_options(options);
|
|
|
|
ColumnFamilyOptions cf_options(options);
|
|
|
|
std::vector<ColumnFamilyDescriptor> column_families;
|
|
|
|
column_families.push_back(
|
|
|
|
ColumnFamilyDescriptor(default_column_family_name, cf_options));
|
|
|
|
std::vector<ColumnFamilyHandle> handles;
|
|
|
|
return DB::OpenWithColumnFamilies(db_options, dbname, column_families,
|
|
|
|
&handles, dbptr);
|
|
|
|
}
|
|
|
|
|
|
|
|
Status DB::OpenWithColumnFamilies(
|
|
|
|
const DBOptions& db_options, const std::string& dbname,
|
|
|
|
const std::vector<ColumnFamilyDescriptor>& column_families,
|
|
|
|
std::vector<ColumnFamilyHandle>* handles, DB** dbptr) {
|
2013-02-15 20:53:17 +01:00
|
|
|
*dbptr = nullptr;
|
2013-06-08 00:35:17 +02:00
|
|
|
EnvOptions soptions;
|
2014-01-06 22:31:06 +01:00
|
|
|
// TODO temporary until we change DBImpl to accept
|
|
|
|
// DBOptions instead of Options
|
|
|
|
ColumnFamilyOptions default_column_family_options;
|
|
|
|
for (auto cfd : column_families) {
|
|
|
|
if (cfd.name == default_column_family_name) {
|
|
|
|
default_column_family_options = cfd.options;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
// default options
|
|
|
|
Options options(db_options, default_column_family_options);
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2013-02-15 20:53:17 +01:00
|
|
|
if (options.block_cache != nullptr && options.no_block_cache) {
|
2012-11-01 01:02:24 +01:00
|
|
|
return Status::InvalidArgument(
|
2013-02-15 20:53:17 +01:00
|
|
|
"no_block_cache is true while block_cache is not nullptr");
|
2012-11-01 01:02:24 +01:00
|
|
|
}
|
2013-10-01 23:46:52 +02:00
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
DBImpl* impl = new DBImpl(options, dbname);
|
2013-10-01 23:46:52 +02:00
|
|
|
Status s = impl->env_->CreateDirIfMissing(impl->options_.wal_dir);
|
|
|
|
if (!s.ok()) {
|
|
|
|
delete impl;
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
|
|
|
s = impl->CreateArchivalDirectory();
|
2012-11-26 22:56:45 +01:00
|
|
|
if (!s.ok()) {
|
|
|
|
delete impl;
|
|
|
|
return s;
|
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
impl->mutex_.Lock();
|
2014-01-24 23:30:28 +01:00
|
|
|
// Handles create_if_missing, error_if_exists
|
|
|
|
s = impl->Recover(column_families);
|
2011-03-18 23:37:00 +01:00
|
|
|
if (s.ok()) {
|
2011-04-12 21:38:58 +02:00
|
|
|
uint64_t new_log_number = impl->versions_->NewFileNumber();
|
2013-01-20 11:07:13 +01:00
|
|
|
unique_ptr<WritableFile> lfile;
|
2013-06-08 00:35:17 +02:00
|
|
|
soptions.use_mmap_writes = false;
|
2013-11-16 02:17:13 +01:00
|
|
|
s = impl->options_.env->NewWritableFile(
|
2013-10-01 23:46:52 +02:00
|
|
|
LogFileName(impl->options_.wal_dir, new_log_number),
|
|
|
|
&lfile,
|
|
|
|
soptions
|
|
|
|
);
|
2011-03-18 23:37:00 +01:00
|
|
|
if (s.ok()) {
|
2013-11-16 02:17:13 +01:00
|
|
|
lfile->SetPreallocationBlockSize(1.1 * impl->options_.write_buffer_size);
|
Refactor Recover() code
Summary:
This diff does two things:
* Rethinks how we call Recover() with read_only option. Before, we call it with pointer to memtable where we'd like to apply those changes to. This memtable is set in db_impl_readonly.cc and it's actually DBImpl::mem_. Why don't we just apply updates to mem_ right away? It seems more intuitive.
* Changes when we apply updates to manifest. Before, the process is to recover all the logs, flush it to sst files and then do one giant commit that atomically adds all recovered sst files and sets the next log number. This works good enough, but causes some small troubles for my column family approach, since I can't have one VersionEdit apply to more than single column family[1]. The change here is to commit the files recovered from logs right away. Here is the state of the world before the change:
1. Recover log 5, add new sst files to edit
2. Recover log 7, add new sst files to edit
3. Recover log 8, add new sst files to edit
4. Commit all added sst files to manifest and mark log files 5, 7 and 8 as recoverd (via SetLogNumber(9) function)
After the change, we'll do:
1. Recover log 5, commit the new sst files and set log 5 as recovered
2. Recover log 7, commit the new sst files and set log 7 as recovered
3. Recover log 8, commit the new sst files and set log 8 as recovered
The added (small) benefit is that if we fail after (2), the new recovery will only have to recover log 8. In previous case, we'll have to restart the recovery from the beginning. The bigger benefit will be to enable easier integration of multiple column families in Recovery code path.
[1] I'm happy to dicuss this decison, but I believe this is the cleanest way to go. It also makes backward compatibility much easier. We don't have a requirement of adding multiple column families atomically.
Test Plan: make check
Reviewers: dhruba, haobo, kailiu, sdong
Reviewed By: kailiu
CC: leveldb
Differential Revision: https://reviews.facebook.net/D15237
2014-01-22 19:45:26 +01:00
|
|
|
VersionEdit edit;
|
2011-06-22 04:36:45 +02:00
|
|
|
impl->logfile_number_ = new_log_number;
|
2013-01-20 11:07:13 +01:00
|
|
|
impl->log_.reset(new log::Writer(std::move(lfile)));
|
2014-01-28 20:05:04 +01:00
|
|
|
// We use this LogAndApply just to store the next file number, the one
|
|
|
|
// that we used by calling impl->versions_->NewFileNumber()
|
|
|
|
// The used log number are already written to manifest in RecoverLogFile()
|
|
|
|
// method
|
2014-01-27 22:55:47 +01:00
|
|
|
s = impl->versions_->LogAndApply(impl->default_cfd_, &edit, &impl->mutex_,
|
2014-01-27 20:02:21 +01:00
|
|
|
impl->db_directory_.get());
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
if (s.ok()) {
|
2014-01-06 22:31:06 +01:00
|
|
|
// set column family handles
|
|
|
|
handles->clear();
|
|
|
|
for (auto cf : column_families) {
|
2014-01-22 20:44:53 +01:00
|
|
|
if (!impl->versions_->GetColumnFamilySet()->Exists(cf.name)) {
|
|
|
|
s = Status::InvalidArgument("Column family not found: ", cf.name);
|
|
|
|
handles->clear();
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
uint32_t id = impl->versions_->GetColumnFamilySet()->GetID(cf.name);
|
|
|
|
handles->push_back(ColumnFamilyHandle(id));
|
2014-01-06 22:31:06 +01:00
|
|
|
}
|
2014-01-22 20:44:53 +01:00
|
|
|
}
|
|
|
|
if (s.ok()) {
|
2014-01-24 23:30:28 +01:00
|
|
|
for (auto cfd : *impl->versions_->GetColumnFamilySet()) {
|
2014-01-29 22:28:50 +01:00
|
|
|
delete cfd->InstallSuperVersion(new SuperVersion());
|
|
|
|
cfd->mem()->SetLogNumber(impl->logfile_number_);
|
2014-01-24 23:30:28 +01:00
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
impl->DeleteObsoleteFiles();
|
2013-10-15 00:12:15 +02:00
|
|
|
impl->MaybeScheduleFlushOrCompaction();
|
2012-08-15 00:20:36 +02:00
|
|
|
impl->MaybeScheduleLogDBDeployStats();
|
2014-01-27 20:02:21 +01:00
|
|
|
s = impl->db_directory_->Fsync();
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
}
|
2013-09-04 22:13:08 +02:00
|
|
|
|
2014-01-16 01:15:43 +01:00
|
|
|
if (s.ok() && impl->options_.compaction_style == kCompactionStyleUniversal) {
|
2014-01-29 22:28:50 +01:00
|
|
|
Version* current = impl->default_cfd_->current();
|
2013-09-04 22:13:08 +02:00
|
|
|
for (int i = 1; i < impl->NumberLevels(); i++) {
|
2014-01-16 01:15:43 +01:00
|
|
|
int num_files = current->NumLevelFiles(i);
|
2013-09-04 22:13:08 +02:00
|
|
|
if (num_files > 0) {
|
|
|
|
s = Status::InvalidArgument("Not all files are at level 0. Cannot "
|
|
|
|
"open with universal compaction style.");
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-01-16 01:15:43 +01:00
|
|
|
impl->mutex_.Unlock();
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
if (s.ok()) {
|
|
|
|
*dbptr = impl;
|
|
|
|
} else {
|
2014-01-06 22:31:06 +01:00
|
|
|
handles->clear();
|
2011-03-18 23:37:00 +01:00
|
|
|
delete impl;
|
|
|
|
}
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
2014-01-02 18:08:12 +01:00
|
|
|
Status DB::ListColumnFamilies(const DBOptions& db_options,
|
|
|
|
const std::string& name,
|
|
|
|
std::vector<std::string>* column_families) {
|
2014-01-22 20:44:53 +01:00
|
|
|
return VersionSet::ListColumnFamilies(column_families, name, db_options.env);
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 20:14:09 +01:00
|
|
|
}
|
|
|
|
|
2011-05-21 04:17:43 +02:00
|
|
|
Snapshot::~Snapshot() {
|
|
|
|
}
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
Status DestroyDB(const std::string& dbname, const Options& options) {
|
2013-10-01 23:46:52 +02:00
|
|
|
const InternalKeyComparator comparator(options.comparator);
|
|
|
|
const InternalFilterPolicy filter_policy(options.filter_policy);
|
|
|
|
const Options& soptions(SanitizeOptions(
|
|
|
|
dbname, &comparator, &filter_policy, options));
|
|
|
|
Env* env = soptions.env;
|
2011-03-18 23:37:00 +01:00
|
|
|
std::vector<std::string> filenames;
|
2012-12-10 20:02:07 +01:00
|
|
|
std::vector<std::string> archiveFiles;
|
|
|
|
|
2013-10-01 23:46:52 +02:00
|
|
|
std::string archivedir = ArchivalDirectory(dbname);
|
2011-03-18 23:37:00 +01:00
|
|
|
// Ignore error in case directory does not exist
|
|
|
|
env->GetChildren(dbname, &filenames);
|
2013-10-01 23:46:52 +02:00
|
|
|
|
|
|
|
if (dbname != soptions.wal_dir) {
|
|
|
|
std::vector<std::string> logfilenames;
|
|
|
|
env->GetChildren(soptions.wal_dir, &logfilenames);
|
|
|
|
filenames.insert(filenames.end(), logfilenames.begin(), logfilenames.end());
|
|
|
|
archivedir = ArchivalDirectory(soptions.wal_dir);
|
|
|
|
}
|
2012-12-10 20:02:07 +01:00
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
if (filenames.empty()) {
|
|
|
|
return Status::OK();
|
|
|
|
}
|
|
|
|
|
|
|
|
FileLock* lock;
|
2011-07-15 02:20:57 +02:00
|
|
|
const std::string lockname = LockFileName(dbname);
|
|
|
|
Status result = env->LockFile(lockname, &lock);
|
2011-03-18 23:37:00 +01:00
|
|
|
if (result.ok()) {
|
|
|
|
uint64_t number;
|
|
|
|
FileType type;
|
2011-04-21 00:48:11 +02:00
|
|
|
for (size_t i = 0; i < filenames.size(); i++) {
|
2011-07-15 02:20:57 +02:00
|
|
|
if (ParseFileName(filenames[i], &number, &type) &&
|
2012-03-09 16:51:04 +01:00
|
|
|
type != kDBLockFile) { // Lock file will be deleted at end
|
2012-12-17 20:26:59 +01:00
|
|
|
Status del;
|
|
|
|
if (type == kMetaDatabase) {
|
|
|
|
del = DestroyDB(dbname + "/" + filenames[i], options);
|
2013-10-01 23:46:52 +02:00
|
|
|
} else if (type == kLogFile) {
|
|
|
|
del = env->DeleteFile(soptions.wal_dir + "/" + filenames[i]);
|
2012-12-17 20:26:59 +01:00
|
|
|
} else {
|
|
|
|
del = env->DeleteFile(dbname + "/" + filenames[i]);
|
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
if (result.ok() && !del.ok()) {
|
|
|
|
result = del;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2012-12-10 20:02:07 +01:00
|
|
|
|
2013-10-01 23:46:52 +02:00
|
|
|
env->GetChildren(archivedir, &archiveFiles);
|
2012-12-10 20:02:07 +01:00
|
|
|
// Delete archival files.
|
|
|
|
for (size_t i = 0; i < archiveFiles.size(); ++i) {
|
2013-10-11 03:02:10 +02:00
|
|
|
if (ParseFileName(archiveFiles[i], &number, &type) &&
|
|
|
|
type == kLogFile) {
|
2013-10-01 23:46:52 +02:00
|
|
|
Status del = env->DeleteFile(archivedir + "/" + archiveFiles[i]);
|
2012-12-10 20:02:07 +01:00
|
|
|
if (result.ok() && !del.ok()) {
|
|
|
|
result = del;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2012-12-14 22:21:53 +01:00
|
|
|
// ignore case where no archival directory is present.
|
2013-10-01 23:46:52 +02:00
|
|
|
env->DeleteDir(archivedir);
|
2012-12-10 20:02:07 +01:00
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
env->UnlockFile(lock); // Ignore error since state is already gone
|
2011-07-15 02:20:57 +02:00
|
|
|
env->DeleteFile(lockname);
|
2011-03-18 23:37:00 +01:00
|
|
|
env->DeleteDir(dbname); // Ignore error in case dir contains other files
|
2013-10-01 23:46:52 +02:00
|
|
|
env->DeleteDir(soptions.wal_dir);
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2012-08-21 09:33:21 +02:00
|
|
|
//
|
|
|
|
// A global method that can dump out the build version
|
2012-09-18 22:18:32 +02:00
|
|
|
void dumpLeveldbBuildVersion(Logger * log) {
|
2013-10-05 07:32:05 +02:00
|
|
|
Log(log, "Git sha %s", rocksdb_build_git_sha);
|
2013-01-24 20:45:11 +01:00
|
|
|
Log(log, "Compile time %s %s",
|
2013-10-05 07:32:05 +02:00
|
|
|
rocksdb_build_compile_time, rocksdb_build_compile_date);
|
2012-08-21 09:33:21 +02:00
|
|
|
}
|
|
|
|
|
2013-10-04 06:49:15 +02:00
|
|
|
} // namespace rocksdb
|