rocksdb/util/autovector.h
Peter Dillinger a8a422e962 Add manifest fix-up utility for file temperatures (#9683)
Summary:
The goal of this change is to allow changes to the "current" (in
FileSystem) file temperatures to feed back into DB metadata, so that
they can inform decisions and stats reporting. In part because of
modular code factoring, it doesn't seem easy to do this automagically,
where opening an SST file and observing current Temperature different
from expected would trigger a change in metadata and DB manifest write
(essentially giving the deep read path access to the write path). It is also
difficult to do this while the DB is open because of the limitations of
LogAndApply.

This change allows updating file temperature metadata on a closed DB
using an experimental utility function UpdateManifestForFilesState()
or `ldb update_manifest --update_temperatures`. This should suffice for
"migration" scenarios where outside tooling has placed or re-arranged DB
files into a (different) tiered configuration without going through
RocksDB itself (currently, only compaction can change temperature
metadata).

Some details:
* Refactored and added unit test for `ldb unsafe_remove_sst_file` because
of shared functionality
* Pulled in autovector.h changes from https://github.com/facebook/rocksdb/issues/9546 to fix SuperVersionContext
move constructor (related to an older draft of this change)

Possible follow-up work:
* Support updating manifest with file checksums, such as when a
new checksum function is used and want existing DB metadata updated
for it.
* It's possible that for some repair scenarios, lighter weight than
full repair, we might want to support UpdateManifestForFilesState() to
modify critical file details like size or checksum using same
algorithm. But let's make sure these are differentiated from modifying
file details in ways that don't suspect corruption (or require extreme
trust).

Pull Request resolved: https://github.com/facebook/rocksdb/pull/9683

Test Plan: unit tests added

Reviewed By: jay-zhuang

Differential Revision: D34798828

Pulled By: pdillinger

fbshipit-source-id: cfd83e8fb10761d8c9e7f9c020d68c9106a95554
2022-03-18 16:35:51 -07:00

388 lines
10 KiB
C++

// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
#pragma once
#include <algorithm>
#include <cassert>
#include <initializer_list>
#include <iterator>
#include <stdexcept>
#include <vector>
#include "port/lang.h"
#include "rocksdb/rocksdb_namespace.h"
namespace ROCKSDB_NAMESPACE {
#ifdef ROCKSDB_LITE
template <class T, size_t kSize = 8>
class autovector : public std::vector<T> {
using std::vector<T>::vector;
public:
autovector() {
// Make sure the initial vector has space for kSize elements
std::vector<T>::reserve(kSize);
}
};
#else
// A vector that leverages pre-allocated stack-based array to achieve better
// performance for array with small amount of items.
//
// The interface resembles that of vector, but with less features since we aim
// to solve the problem that we have in hand, rather than implementing a
// full-fledged generic container.
//
// Currently we don't support:
// * reserve()/shrink_to_fit()
// If used correctly, in most cases, people should not touch the
// underlying vector at all.
// * random insert()/erase(), please only use push_back()/pop_back().
// * No move/swap operations. Each autovector instance has a
// stack-allocated array and if we want support move/swap operations, we
// need to copy the arrays other than just swapping the pointers. In this
// case we'll just explicitly forbid these operations since they may
// lead users to make false assumption by thinking they are inexpensive
// operations.
//
// Naming style of public methods almost follows that of the STL's.
template <class T, size_t kSize = 8>
class autovector {
public:
// General STL-style container member types.
using value_type = T;
using difference_type = typename std::vector<T>::difference_type;
using size_type = typename std::vector<T>::size_type;
using reference = value_type&;
using const_reference = const value_type&;
using pointer = value_type*;
using const_pointer = const value_type*;
// This class is the base for regular/const iterator
template <class TAutoVector, class TValueType>
class iterator_impl {
public:
// -- iterator traits
using self_type = iterator_impl<TAutoVector, TValueType>;
using value_type = TValueType;
using reference = TValueType&;
using pointer = TValueType*;
using difference_type = typename TAutoVector::difference_type;
using iterator_category = std::random_access_iterator_tag;
iterator_impl(TAutoVector* vect, size_t index)
: vect_(vect), index_(index) {};
iterator_impl(const iterator_impl&) = default;
~iterator_impl() {}
iterator_impl& operator=(const iterator_impl&) = default;
// -- Advancement
// ++iterator
self_type& operator++() {
++index_;
return *this;
}
// iterator++
self_type operator++(int) {
auto old = *this;
++index_;
return old;
}
// --iterator
self_type& operator--() {
--index_;
return *this;
}
// iterator--
self_type operator--(int) {
auto old = *this;
--index_;
return old;
}
self_type operator-(difference_type len) const {
return self_type(vect_, index_ - len);
}
difference_type operator-(const self_type& other) const {
assert(vect_ == other.vect_);
return index_ - other.index_;
}
self_type operator+(difference_type len) const {
return self_type(vect_, index_ + len);
}
self_type& operator+=(difference_type len) {
index_ += len;
return *this;
}
self_type& operator-=(difference_type len) {
index_ -= len;
return *this;
}
// -- Reference
reference operator*() const {
assert(vect_->size() >= index_);
return (*vect_)[index_];
}
pointer operator->() const {
assert(vect_->size() >= index_);
return &(*vect_)[index_];
}
reference operator[](difference_type len) const {
return *(*this + len);
}
// -- Logical Operators
bool operator==(const self_type& other) const {
assert(vect_ == other.vect_);
return index_ == other.index_;
}
bool operator!=(const self_type& other) const { return !(*this == other); }
bool operator>(const self_type& other) const {
assert(vect_ == other.vect_);
return index_ > other.index_;
}
bool operator<(const self_type& other) const {
assert(vect_ == other.vect_);
return index_ < other.index_;
}
bool operator>=(const self_type& other) const {
assert(vect_ == other.vect_);
return index_ >= other.index_;
}
bool operator<=(const self_type& other) const {
assert(vect_ == other.vect_);
return index_ <= other.index_;
}
private:
TAutoVector* vect_ = nullptr;
size_t index_ = 0;
};
using iterator = iterator_impl<autovector, value_type>;
using const_iterator = iterator_impl<const autovector, const value_type>;
using reverse_iterator = std::reverse_iterator<iterator>;
using const_reverse_iterator = std::reverse_iterator<const_iterator>;
autovector() : values_(reinterpret_cast<pointer>(buf_)) {}
autovector(std::initializer_list<T> init_list)
: values_(reinterpret_cast<pointer>(buf_)) {
for (const T& item : init_list) {
push_back(item);
}
}
~autovector() { clear(); }
// -- Immutable operations
// Indicate if all data resides in in-stack data structure.
bool only_in_stack() const {
// If no element was inserted at all, the vector's capacity will be `0`.
return vect_.capacity() == 0;
}
size_type size() const { return num_stack_items_ + vect_.size(); }
// resize does not guarantee anything about the contents of the newly
// available elements
void resize(size_type n) {
if (n > kSize) {
vect_.resize(n - kSize);
while (num_stack_items_ < kSize) {
new ((void*)(&values_[num_stack_items_++])) value_type();
}
num_stack_items_ = kSize;
} else {
vect_.clear();
while (num_stack_items_ < n) {
new ((void*)(&values_[num_stack_items_++])) value_type();
}
while (num_stack_items_ > n) {
values_[--num_stack_items_].~value_type();
}
}
}
bool empty() const { return size() == 0; }
const_reference operator[](size_type n) const {
assert(n < size());
if (n < kSize) {
return values_[n];
}
return vect_[n - kSize];
}
reference operator[](size_type n) {
assert(n < size());
if (n < kSize) {
return values_[n];
}
return vect_[n - kSize];
}
const_reference at(size_type n) const {
assert(n < size());
return (*this)[n];
}
reference at(size_type n) {
assert(n < size());
return (*this)[n];
}
reference front() {
assert(!empty());
return *begin();
}
const_reference front() const {
assert(!empty());
return *begin();
}
reference back() {
assert(!empty());
return *(end() - 1);
}
const_reference back() const {
assert(!empty());
return *(end() - 1);
}
// -- Mutable Operations
void push_back(T&& item) {
if (num_stack_items_ < kSize) {
new ((void*)(&values_[num_stack_items_])) value_type();
values_[num_stack_items_++] = std::move(item);
} else {
vect_.push_back(item);
}
}
void push_back(const T& item) {
if (num_stack_items_ < kSize) {
new ((void*)(&values_[num_stack_items_])) value_type();
values_[num_stack_items_++] = item;
} else {
vect_.push_back(item);
}
}
template <class... Args>
void emplace_back(Args&&... args) {
if (num_stack_items_ < kSize) {
new ((void*)(&values_[num_stack_items_++]))
value_type(std::forward<Args>(args)...);
} else {
vect_.emplace_back(std::forward<Args>(args)...);
}
}
void pop_back() {
assert(!empty());
if (!vect_.empty()) {
vect_.pop_back();
} else {
values_[--num_stack_items_].~value_type();
}
}
void clear() {
while (num_stack_items_ > 0) {
values_[--num_stack_items_].~value_type();
}
vect_.clear();
}
// -- Copy and Assignment
autovector& assign(const autovector& other);
autovector(const autovector& other) { assign(other); }
autovector& operator=(const autovector& other) { return assign(other); }
autovector(autovector&& other) noexcept { *this = std::move(other); }
autovector& operator=(autovector&& other);
// -- Iterator Operations
iterator begin() { return iterator(this, 0); }
const_iterator begin() const { return const_iterator(this, 0); }
iterator end() { return iterator(this, this->size()); }
const_iterator end() const { return const_iterator(this, this->size()); }
reverse_iterator rbegin() { return reverse_iterator(end()); }
const_reverse_iterator rbegin() const {
return const_reverse_iterator(end());
}
reverse_iterator rend() { return reverse_iterator(begin()); }
const_reverse_iterator rend() const {
return const_reverse_iterator(begin());
}
private:
size_type num_stack_items_ = 0; // current number of items
alignas(alignof(
value_type)) char buf_[kSize *
sizeof(value_type)]; // the first `kSize` items
pointer values_;
// used only if there are more than `kSize` items.
std::vector<T> vect_;
};
template <class T, size_t kSize>
autovector<T, kSize>& autovector<T, kSize>::assign(
const autovector<T, kSize>& other) {
values_ = reinterpret_cast<pointer>(buf_);
// copy the internal vector
vect_.assign(other.vect_.begin(), other.vect_.end());
// copy array
num_stack_items_ = other.num_stack_items_;
std::copy(other.values_, other.values_ + num_stack_items_, values_);
return *this;
}
template <class T, size_t kSize>
autovector<T, kSize>& autovector<T, kSize>::operator=(
autovector<T, kSize>&& other) {
values_ = reinterpret_cast<pointer>(buf_);
vect_ = std::move(other.vect_);
size_t n = other.num_stack_items_;
num_stack_items_ = n;
other.num_stack_items_ = 0;
for (size_t i = 0; i < n; ++i) {
values_[i] = std::move(other.values_[i]);
}
return *this;
}
#endif // ROCKSDB_LITE
} // namespace ROCKSDB_NAMESPACE