47c4191fe8
Summary: There is a new option called hybrid_mode which, when switched on, causes HBase style compactions. Files from L0 are compacted back into L0. This meat of this compaction algorithm is in PickCompactionHybrid(). All files reside in L0. That means all files have overlapping keys. Each file has a time-bound, i.e. each file contains a range of keys that were inserted around the same time. The start-seqno and the end-seqno refers to the timeframe when these keys were inserted. Files that have contiguous seqno are compacted together into a larger file. All files are ordered from most recent to the oldest. The current compaction algorithm starts to look for candidate files starting from the most recent file. It continues to add more files to the same compaction run as long as the sum of the files chosen till now is smaller than the next candidate file size. This logic needs to be debated and validated. The above logic should reduce write amplification to a large extent... will publish numbers shortly. Test Plan: dbstress runs for 6 hours with no data corruption (tested so far). Differential Revision: https://reviews.facebook.net/D11289
128 lines
3.6 KiB
C++
128 lines
3.6 KiB
C++
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
|
// Use of this source code is governed by a BSD-style license that can be
|
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
|
|
|
#ifndef STORAGE_LEVELDB_DB_VERSION_EDIT_H_
|
|
#define STORAGE_LEVELDB_DB_VERSION_EDIT_H_
|
|
|
|
#include <set>
|
|
#include <utility>
|
|
#include <vector>
|
|
#include "db/dbformat.h"
|
|
|
|
namespace leveldb {
|
|
|
|
class VersionSet;
|
|
|
|
struct FileMetaData {
|
|
int refs;
|
|
int allowed_seeks; // Seeks allowed until compaction
|
|
uint64_t number;
|
|
uint64_t file_size; // File size in bytes
|
|
InternalKey smallest; // Smallest internal key served by table
|
|
InternalKey largest; // Largest internal key served by table
|
|
bool being_compacted; // Is this file undergoing compaction?
|
|
SequenceNumber smallest_seqno;// The smallest seqno in this file
|
|
SequenceNumber largest_seqno; // The largest seqno in this file
|
|
|
|
FileMetaData() : refs(0), allowed_seeks(1 << 30), file_size(0),
|
|
being_compacted(false) { }
|
|
};
|
|
|
|
class VersionEdit {
|
|
public:
|
|
/* implicit */ VersionEdit(int number_levels) :
|
|
number_levels_(number_levels) {
|
|
Clear();
|
|
}
|
|
~VersionEdit() { }
|
|
|
|
void Clear();
|
|
|
|
void SetComparatorName(const Slice& name) {
|
|
has_comparator_ = true;
|
|
comparator_ = name.ToString();
|
|
}
|
|
void SetLogNumber(uint64_t num) {
|
|
has_log_number_ = true;
|
|
log_number_ = num;
|
|
}
|
|
void SetPrevLogNumber(uint64_t num) {
|
|
has_prev_log_number_ = true;
|
|
prev_log_number_ = num;
|
|
}
|
|
void SetNextFile(uint64_t num) {
|
|
has_next_file_number_ = true;
|
|
next_file_number_ = num;
|
|
}
|
|
void SetLastSequence(SequenceNumber seq) {
|
|
has_last_sequence_ = true;
|
|
last_sequence_ = seq;
|
|
}
|
|
void SetCompactPointer(int level, const InternalKey& key) {
|
|
compact_pointers_.push_back(std::make_pair(level, key));
|
|
}
|
|
|
|
// Add the specified file at the specified number.
|
|
// REQUIRES: This version has not been saved (see VersionSet::SaveTo)
|
|
// REQUIRES: "smallest" and "largest" are smallest and largest keys in file
|
|
void AddFile(int level, uint64_t file,
|
|
uint64_t file_size,
|
|
const InternalKey& smallest,
|
|
const InternalKey& largest,
|
|
const SequenceNumber& smallest_seqno,
|
|
const SequenceNumber& largest_seqno) {
|
|
FileMetaData f;
|
|
f.number = file;
|
|
f.file_size = file_size;
|
|
f.smallest = smallest;
|
|
f.largest = largest;
|
|
f.smallest_seqno = smallest_seqno;
|
|
f.largest_seqno = largest_seqno;
|
|
assert(smallest_seqno <= largest_seqno);
|
|
new_files_.push_back(std::make_pair(level, f));
|
|
}
|
|
|
|
// Delete the specified "file" from the specified "level".
|
|
void DeleteFile(int level, uint64_t file) {
|
|
deleted_files_.insert(std::make_pair(level, file));
|
|
}
|
|
|
|
// Number of edits
|
|
int NumEntries() {
|
|
return new_files_.size() + deleted_files_.size();
|
|
}
|
|
|
|
void EncodeTo(std::string* dst) const;
|
|
Status DecodeFrom(const Slice& src);
|
|
|
|
std::string DebugString() const;
|
|
|
|
private:
|
|
friend class VersionSet;
|
|
|
|
typedef std::set< std::pair<int, uint64_t> > DeletedFileSet;
|
|
|
|
bool GetLevel(Slice* input, int* level, const char** msg);
|
|
|
|
int number_levels_;
|
|
std::string comparator_;
|
|
uint64_t log_number_;
|
|
uint64_t prev_log_number_;
|
|
uint64_t next_file_number_;
|
|
SequenceNumber last_sequence_;
|
|
bool has_comparator_;
|
|
bool has_log_number_;
|
|
bool has_prev_log_number_;
|
|
bool has_next_file_number_;
|
|
bool has_last_sequence_;
|
|
|
|
std::vector< std::pair<int, InternalKey> > compact_pointers_;
|
|
DeletedFileSet deleted_files_;
|
|
std::vector< std::pair<int, FileMetaData> > new_files_;
|
|
};
|
|
|
|
} // namespace leveldb
|
|
|
|
#endif // STORAGE_LEVELDB_DB_VERSION_EDIT_H_
|