2013-10-16 23:59:46 +02:00
|
|
|
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
|
|
|
// This source code is licensed under the BSD-style license found in the
|
|
|
|
// LICENSE file in the root directory of this source tree. An additional grant
|
|
|
|
// of patent rights can be found in the PATENTS file in the same directory.
|
|
|
|
//
|
2012-10-19 23:00:53 +02:00
|
|
|
|
2013-10-05 07:32:05 +02:00
|
|
|
#pragma once
|
2012-10-19 23:00:53 +02:00
|
|
|
#include <string>
|
|
|
|
#include <list>
|
[RocksDB] [MergeOperator] The new Merge Interface! Uses merge sequences.
Summary:
Here are the major changes to the Merge Interface. It has been expanded
to handle cases where the MergeOperator is not associative. It does so by stacking
up merge operations while scanning through the key history (i.e.: during Get() or
Compaction), until a valid Put/Delete/end-of-history is encountered; it then
applies all of the merge operations in the correct sequence starting with the
base/sentinel value.
I have also introduced an "AssociativeMerge" function which allows the user to
take advantage of associative merge operations (such as in the case of counters).
The implementation will always attempt to merge the operations/operands themselves
together when they are encountered, and will resort to the "stacking" method if
and only if the "associative-merge" fails.
This implementation is conjectured to allow MergeOperator to handle the general
case, while still providing the user with the ability to take advantage of certain
efficiencies in their own merge-operator / data-structure.
NOTE: This is a preliminary diff. This must still go through a lot of review,
revision, and testing. Feedback welcome!
Test Plan:
-This is a preliminary diff. I have only just begun testing/debugging it.
-I will be testing this with the existing MergeOperator use-cases and unit-tests
(counters, string-append, and redis-lists)
-I will be "desk-checking" and walking through the code with the help gdb.
-I will find a way of stress-testing the new interface / implementation using
db_bench, db_test, merge_test, and/or db_stress.
-I will ensure that my tests cover all cases: Get-Memtable,
Get-Immutable-Memtable, Get-from-Disk, Iterator-Range-Scan, Flush-Memtable-to-L0,
Compaction-L0-L1, Compaction-Ln-L(n+1), Put/Delete found, Put/Delete not-found,
end-of-history, end-of-file, etc.
-A lot of feedback from the reviewers.
Reviewers: haobo, dhruba, zshao, emayanke
Reviewed By: haobo
CC: leveldb
Differential Revision: https://reviews.facebook.net/D11499
2013-08-06 05:14:32 +02:00
|
|
|
#include <deque>
|
2013-08-23 17:38:13 +02:00
|
|
|
#include "rocksdb/db.h"
|
2012-10-19 23:00:53 +02:00
|
|
|
#include "db/dbformat.h"
|
|
|
|
#include "db/skiplist.h"
|
|
|
|
#include "memtable.h"
|
|
|
|
|
2013-10-04 06:49:15 +02:00
|
|
|
namespace rocksdb {
|
2012-10-19 23:00:53 +02:00
|
|
|
|
|
|
|
class InternalKeyComparator;
|
|
|
|
class Mutex;
|
|
|
|
class MemTableListIterator;
|
|
|
|
|
|
|
|
//
|
2013-06-11 23:23:58 +02:00
|
|
|
// This class stores references to all the immutable memtables.
|
2012-11-29 01:42:36 +01:00
|
|
|
// The memtables are flushed to L0 as soon as possible and in
|
2012-10-19 23:00:53 +02:00
|
|
|
// any order. If there are more than one immutable memtable, their
|
2012-11-29 01:42:36 +01:00
|
|
|
// flushes can occur concurrently. However, they are 'committed'
|
2012-10-19 23:00:53 +02:00
|
|
|
// to the manifest in FIFO order to maintain correctness and
|
|
|
|
// recoverability from a crash.
|
|
|
|
//
|
|
|
|
class MemTableList {
|
|
|
|
public:
|
2012-11-29 01:42:36 +01:00
|
|
|
// A list of memtables.
|
2012-10-19 23:00:53 +02:00
|
|
|
MemTableList() : size_(0), num_flush_not_started_(0),
|
2013-09-05 02:24:35 +02:00
|
|
|
commit_in_progress_(false),
|
|
|
|
flush_requested_(false) {
|
2013-03-01 03:04:58 +01:00
|
|
|
imm_flush_needed.Release_Store(nullptr);
|
2012-10-19 23:00:53 +02:00
|
|
|
}
|
|
|
|
~MemTableList() {};
|
|
|
|
|
2013-03-01 03:04:58 +01:00
|
|
|
// so that backgrund threads can detect non-nullptr pointer to
|
2012-10-19 23:00:53 +02:00
|
|
|
// determine whether this is anything more to start flushing.
|
|
|
|
port::AtomicPointer imm_flush_needed;
|
|
|
|
|
|
|
|
// Increase reference count on all underling memtables
|
|
|
|
void RefAll();
|
|
|
|
|
2013-11-25 20:55:36 +01:00
|
|
|
// Drop reference count on all underling memtables. If the refcount
|
|
|
|
// on an underlying memtable drops to zero, then return it in
|
|
|
|
// to_delete vector.
|
|
|
|
void UnrefAll(std::vector<MemTable*>* to_delete);
|
2012-10-19 23:00:53 +02:00
|
|
|
|
|
|
|
// Returns the total number of memtables in the list
|
|
|
|
int size();
|
|
|
|
|
|
|
|
// Returns true if there is at least one memtable on which flush has
|
|
|
|
// not yet started.
|
2013-06-11 23:23:58 +02:00
|
|
|
bool IsFlushPending(int min_write_buffer_number_to_merge);
|
2012-10-19 23:00:53 +02:00
|
|
|
|
2013-10-25 00:58:00 +02:00
|
|
|
// Returns the earliest memtables that needs to be flushed. The returned
|
|
|
|
// memtables are guaranteed to be in the ascending order of created time.
|
2013-06-11 23:23:58 +02:00
|
|
|
void PickMemtablesToFlush(std::vector<MemTable*>* mems);
|
2012-10-19 23:00:53 +02:00
|
|
|
|
|
|
|
// Commit a successful flush in the manifest file
|
2013-06-11 23:23:58 +02:00
|
|
|
Status InstallMemtableFlushResults(const std::vector<MemTable*> &m,
|
2012-10-19 23:00:53 +02:00
|
|
|
VersionSet* vset, Status flushStatus,
|
|
|
|
port::Mutex* mu, Logger* info_log,
|
|
|
|
uint64_t file_number,
|
2013-11-27 23:56:20 +01:00
|
|
|
std::set<uint64_t>& pending_outputs,
|
|
|
|
std::vector<MemTable*>* to_delete);
|
2012-10-19 23:00:53 +02:00
|
|
|
|
2012-11-29 01:42:36 +01:00
|
|
|
// New memtables are inserted at the front of the list.
|
2013-01-04 02:13:56 +01:00
|
|
|
// Takes ownership of the referenced held on *m by the caller of Add().
|
2012-10-19 23:00:53 +02:00
|
|
|
void Add(MemTable* m);
|
|
|
|
|
|
|
|
// Returns an estimate of the number of bytes of data in use.
|
|
|
|
size_t ApproximateMemoryUsage();
|
|
|
|
|
|
|
|
// Search all the memtables starting from the most recent one.
|
|
|
|
// Return the most recent value found, if any.
|
2013-03-21 23:59:47 +01:00
|
|
|
bool Get(const LookupKey& key, std::string* value, Status* s,
|
[RocksDB] [MergeOperator] The new Merge Interface! Uses merge sequences.
Summary:
Here are the major changes to the Merge Interface. It has been expanded
to handle cases where the MergeOperator is not associative. It does so by stacking
up merge operations while scanning through the key history (i.e.: during Get() or
Compaction), until a valid Put/Delete/end-of-history is encountered; it then
applies all of the merge operations in the correct sequence starting with the
base/sentinel value.
I have also introduced an "AssociativeMerge" function which allows the user to
take advantage of associative merge operations (such as in the case of counters).
The implementation will always attempt to merge the operations/operands themselves
together when they are encountered, and will resort to the "stacking" method if
and only if the "associative-merge" fails.
This implementation is conjectured to allow MergeOperator to handle the general
case, while still providing the user with the ability to take advantage of certain
efficiencies in their own merge-operator / data-structure.
NOTE: This is a preliminary diff. This must still go through a lot of review,
revision, and testing. Feedback welcome!
Test Plan:
-This is a preliminary diff. I have only just begun testing/debugging it.
-I will be testing this with the existing MergeOperator use-cases and unit-tests
(counters, string-append, and redis-lists)
-I will be "desk-checking" and walking through the code with the help gdb.
-I will find a way of stress-testing the new interface / implementation using
db_bench, db_test, merge_test, and/or db_stress.
-I will ensure that my tests cover all cases: Get-Memtable,
Get-Immutable-Memtable, Get-from-Disk, Iterator-Range-Scan, Flush-Memtable-to-L0,
Compaction-L0-L1, Compaction-Ln-L(n+1), Put/Delete found, Put/Delete not-found,
end-of-history, end-of-file, etc.
-A lot of feedback from the reviewers.
Reviewers: haobo, dhruba, zshao, emayanke
Reviewed By: haobo
CC: leveldb
Differential Revision: https://reviews.facebook.net/D11499
2013-08-06 05:14:32 +02:00
|
|
|
std::deque<std::string>* operands, const Options& options);
|
2012-10-19 23:00:53 +02:00
|
|
|
|
2012-11-29 01:42:36 +01:00
|
|
|
// Returns the list of underlying memtables.
|
2012-10-19 23:00:53 +02:00
|
|
|
void GetMemTables(std::vector<MemTable*>* list);
|
|
|
|
|
2013-09-05 02:24:35 +02:00
|
|
|
// Request a flush of all existing memtables to storage
|
|
|
|
void FlushRequested() { flush_requested_ = true; }
|
|
|
|
|
2012-10-19 23:00:53 +02:00
|
|
|
// Copying allowed
|
|
|
|
// MemTableList(const MemTableList&);
|
|
|
|
// void operator=(const MemTableList&);
|
|
|
|
|
|
|
|
private:
|
|
|
|
std::list<MemTable*> memlist_;
|
|
|
|
int size_;
|
|
|
|
|
|
|
|
// the number of elements that still need flushing
|
|
|
|
int num_flush_not_started_;
|
|
|
|
|
|
|
|
// committing in progress
|
|
|
|
bool commit_in_progress_;
|
|
|
|
|
2013-09-05 02:24:35 +02:00
|
|
|
// Requested a flush of all memtables to storage
|
|
|
|
bool flush_requested_;
|
|
|
|
|
2012-10-19 23:00:53 +02:00
|
|
|
};
|
|
|
|
|
2013-10-04 06:49:15 +02:00
|
|
|
} // namespace rocksdb
|